Blame view
mm/hugetlb_cgroup.c
21.4 KB
2bc64a204
|
1 2 3 4 5 |
/* * * Copyright IBM Corporation, 2012 * Author Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com> * |
faced7e08
|
6 7 8 9 |
* Cgroup v2 * Copyright (C) 2019 Red Hat, Inc. * Author: Giuseppe Scrivano <gscrivan@redhat.com> * |
2bc64a204
|
10 11 12 13 14 15 16 17 18 19 20 |
* This program is free software; you can redistribute it and/or modify it * under the terms of version 2.1 of the GNU Lesser General Public License * as published by the Free Software Foundation. * * This program is distributed in the hope that it would be useful, but * WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. * */ #include <linux/cgroup.h> |
71f87bee3
|
21 |
#include <linux/page_counter.h> |
2bc64a204
|
22 23 24 |
#include <linux/slab.h> #include <linux/hugetlb.h> #include <linux/hugetlb_cgroup.h> |
abb8206cb
|
25 26 27 |
#define MEMFILE_PRIVATE(x, val) (((x) << 16) | (val)) #define MEMFILE_IDX(val) (((val) >> 16) & 0xffff) #define MEMFILE_ATTR(val) ((val) & 0xffff) |
faced7e08
|
28 29 |
#define hugetlb_cgroup_from_counter(counter, idx) \ container_of(counter, struct hugetlb_cgroup, hugepage[idx]) |
2bc64a204
|
30 |
static struct hugetlb_cgroup *root_h_cgroup __read_mostly; |
cdc2fcfea
|
31 |
static inline struct page_counter * |
1adc4d419
|
32 33 |
__hugetlb_cgroup_counter_from_cgroup(struct hugetlb_cgroup *h_cg, int idx, bool rsvd) |
cdc2fcfea
|
34 35 36 37 38 |
{ if (rsvd) return &h_cg->rsvd_hugepage[idx]; return &h_cg->hugepage[idx]; } |
1adc4d419
|
39 40 41 42 43 44 45 46 47 48 49 |
static inline struct page_counter * hugetlb_cgroup_counter_from_cgroup(struct hugetlb_cgroup *h_cg, int idx) { return __hugetlb_cgroup_counter_from_cgroup(h_cg, idx, false); } static inline struct page_counter * hugetlb_cgroup_counter_from_cgroup_rsvd(struct hugetlb_cgroup *h_cg, int idx) { return __hugetlb_cgroup_counter_from_cgroup(h_cg, idx, true); } |
2bc64a204
|
50 51 52 |
static inline struct hugetlb_cgroup *hugetlb_cgroup_from_css(struct cgroup_subsys_state *s) { |
a7c6d554a
|
53 |
return s ? container_of(s, struct hugetlb_cgroup, css) : NULL; |
2bc64a204
|
54 55 56 |
} static inline |
2bc64a204
|
57 58 |
struct hugetlb_cgroup *hugetlb_cgroup_from_task(struct task_struct *task) { |
073219e99
|
59 |
return hugetlb_cgroup_from_css(task_css(task, hugetlb_cgrp_id)); |
2bc64a204
|
60 61 62 63 64 65 |
} static inline bool hugetlb_cgroup_is_root(struct hugetlb_cgroup *h_cg) { return (h_cg == root_h_cgroup); } |
3f7985183
|
66 67 |
static inline struct hugetlb_cgroup * parent_hugetlb_cgroup(struct hugetlb_cgroup *h_cg) |
2bc64a204
|
68 |
{ |
5c9d535b8
|
69 |
return hugetlb_cgroup_from_css(h_cg->css.parent); |
2bc64a204
|
70 |
} |
3f7985183
|
71 |
static inline bool hugetlb_cgroup_have_usage(struct hugetlb_cgroup *h_cg) |
2bc64a204
|
72 73 |
{ int idx; |
2bc64a204
|
74 75 |
for (idx = 0; idx < hugetlb_max_hstate; idx++) { |
1adc4d419
|
76 |
if (page_counter_read( |
7a5bde379
|
77 |
hugetlb_cgroup_counter_from_cgroup(h_cg, idx))) |
2bc64a204
|
78 79 80 81 |
return true; } return false; } |
297880f4a
|
82 83 84 85 86 87 |
static void hugetlb_cgroup_init(struct hugetlb_cgroup *h_cgroup, struct hugetlb_cgroup *parent_h_cgroup) { int idx; for (idx = 0; idx < HUGE_MAX_HSTATE; idx++) { |
1adc4d419
|
88 89 |
struct page_counter *fault_parent = NULL; struct page_counter *rsvd_parent = NULL; |
297880f4a
|
90 91 |
unsigned long limit; int ret; |
1adc4d419
|
92 93 94 95 96 97 98 99 100 101 102 103 |
if (parent_h_cgroup) { fault_parent = hugetlb_cgroup_counter_from_cgroup( parent_h_cgroup, idx); rsvd_parent = hugetlb_cgroup_counter_from_cgroup_rsvd( parent_h_cgroup, idx); } page_counter_init(hugetlb_cgroup_counter_from_cgroup(h_cgroup, idx), fault_parent); page_counter_init( hugetlb_cgroup_counter_from_cgroup_rsvd(h_cgroup, idx), rsvd_parent); |
297880f4a
|
104 105 106 |
limit = round_down(PAGE_COUNTER_MAX, 1 << huge_page_order(&hstates[idx])); |
1adc4d419
|
107 108 109 110 111 112 113 114 |
ret = page_counter_set_max( hugetlb_cgroup_counter_from_cgroup(h_cgroup, idx), limit); VM_BUG_ON(ret); ret = page_counter_set_max( hugetlb_cgroup_counter_from_cgroup_rsvd(h_cgroup, idx), limit); |
297880f4a
|
115 116 117 |
VM_BUG_ON(ret); } } |
eb95419b0
|
118 119 |
static struct cgroup_subsys_state * hugetlb_cgroup_css_alloc(struct cgroup_subsys_state *parent_css) |
2bc64a204
|
120 |
{ |
eb95419b0
|
121 122 |
struct hugetlb_cgroup *parent_h_cgroup = hugetlb_cgroup_from_css(parent_css); struct hugetlb_cgroup *h_cgroup; |
2bc64a204
|
123 124 125 126 |
h_cgroup = kzalloc(sizeof(*h_cgroup), GFP_KERNEL); if (!h_cgroup) return ERR_PTR(-ENOMEM); |
297880f4a
|
127 |
if (!parent_h_cgroup) |
2bc64a204
|
128 |
root_h_cgroup = h_cgroup; |
297880f4a
|
129 130 |
hugetlb_cgroup_init(h_cgroup, parent_h_cgroup); |
2bc64a204
|
131 132 |
return &h_cgroup->css; } |
eb95419b0
|
133 |
static void hugetlb_cgroup_css_free(struct cgroup_subsys_state *css) |
2bc64a204
|
134 135 |
{ struct hugetlb_cgroup *h_cgroup; |
eb95419b0
|
136 |
h_cgroup = hugetlb_cgroup_from_css(css); |
2bc64a204
|
137 138 |
kfree(h_cgroup); } |
da1def559
|
139 140 141 142 143 144 145 |
/* * Should be called with hugetlb_lock held. * Since we are holding hugetlb_lock, pages cannot get moved from * active list or uncharged from the cgroup, So no need to get * page reference and test for page active here. This function * cannot fail. */ |
3f7985183
|
146 |
static void hugetlb_cgroup_move_parent(int idx, struct hugetlb_cgroup *h_cg, |
da1def559
|
147 148 |
struct page *page) { |
71f87bee3
|
149 150 |
unsigned int nr_pages; struct page_counter *counter; |
da1def559
|
151 |
struct hugetlb_cgroup *page_hcg; |
3f7985183
|
152 |
struct hugetlb_cgroup *parent = parent_hugetlb_cgroup(h_cg); |
da1def559
|
153 154 155 156 157 158 159 160 161 |
page_hcg = hugetlb_cgroup_from_page(page); /* * We can have pages in active list without any cgroup * ie, hugepage with less than 3 pages. We can safely * ignore those pages. */ if (!page_hcg || page_hcg != h_cg) goto out; |
d8c6546b1
|
162 |
nr_pages = compound_nr(page); |
da1def559
|
163 164 165 |
if (!parent) { parent = root_h_cgroup; /* root has no limit */ |
71f87bee3
|
166 |
page_counter_charge(&parent->hugepage[idx], nr_pages); |
da1def559
|
167 168 |
} counter = &h_cg->hugepage[idx]; |
71f87bee3
|
169 170 |
/* Take the pages off the local counter */ page_counter_cancel(counter, nr_pages); |
da1def559
|
171 172 173 174 175 176 177 178 179 180 |
set_hugetlb_cgroup(page, parent); out: return; } /* * Force the hugetlb cgroup to empty the hugetlb resources by moving them to * the parent cgroup. */ |
eb95419b0
|
181 |
static void hugetlb_cgroup_css_offline(struct cgroup_subsys_state *css) |
2bc64a204
|
182 |
{ |
eb95419b0
|
183 |
struct hugetlb_cgroup *h_cg = hugetlb_cgroup_from_css(css); |
da1def559
|
184 185 |
struct hstate *h; struct page *page; |
7a5bde379
|
186 |
int idx; |
da1def559
|
187 188 |
do { |
7a5bde379
|
189 |
idx = 0; |
da1def559
|
190 191 192 |
for_each_hstate(h) { spin_lock(&hugetlb_lock); list_for_each_entry(page, &h->hugepage_activelist, lru) |
3f7985183
|
193 |
hugetlb_cgroup_move_parent(idx, h_cg, page); |
da1def559
|
194 195 196 197 198 |
spin_unlock(&hugetlb_lock); idx++; } cond_resched(); |
3f7985183
|
199 |
} while (hugetlb_cgroup_have_usage(h_cg)); |
2bc64a204
|
200 |
} |
faced7e08
|
201 202 203 204 205 206 207 208 209 210 211 212 |
static inline void hugetlb_event(struct hugetlb_cgroup *hugetlb, int idx, enum hugetlb_memory_event event) { atomic_long_inc(&hugetlb->events_local[idx][event]); cgroup_file_notify(&hugetlb->events_local_file[idx]); do { atomic_long_inc(&hugetlb->events[idx][event]); cgroup_file_notify(&hugetlb->events_file[idx]); } while ((hugetlb = parent_hugetlb_cgroup(hugetlb)) && !hugetlb_cgroup_is_root(hugetlb)); } |
1adc4d419
|
213 214 215 |
static int __hugetlb_cgroup_charge_cgroup(int idx, unsigned long nr_pages, struct hugetlb_cgroup **ptr, bool rsvd) |
6d76dcf40
|
216 217 |
{ int ret = 0; |
71f87bee3
|
218 |
struct page_counter *counter; |
6d76dcf40
|
219 |
struct hugetlb_cgroup *h_cg = NULL; |
6d76dcf40
|
220 221 222 223 224 225 226 227 228 229 230 231 |
if (hugetlb_cgroup_disabled()) goto done; /* * We don't charge any cgroup if the compound page have less * than 3 pages. */ if (huge_page_order(&hstates[idx]) < HUGETLB_CGROUP_MIN_ORDER) goto done; again: rcu_read_lock(); h_cg = hugetlb_cgroup_from_task(current); |
0362f326d
|
232 |
if (!css_tryget(&h_cg->css)) { |
6d76dcf40
|
233 234 235 236 |
rcu_read_unlock(); goto again; } rcu_read_unlock(); |
1adc4d419
|
237 238 239 |
if (!page_counter_try_charge( __hugetlb_cgroup_counter_from_cgroup(h_cg, idx, rsvd), nr_pages, &counter)) { |
6071ca520
|
240 |
ret = -ENOMEM; |
726b7bbea
|
241 |
hugetlb_event(h_cg, idx, HUGETLB_MAX); |
1adc4d419
|
242 243 |
css_put(&h_cg->css); goto done; |
faced7e08
|
244 |
} |
1adc4d419
|
245 246 247 248 249 |
/* Reservations take a reference to the css because they do not get * reparented. */ if (!rsvd) css_put(&h_cg->css); |
6d76dcf40
|
250 251 252 253 |
done: *ptr = h_cg; return ret; } |
1adc4d419
|
254 255 256 257 258 259 260 261 262 263 264 |
int hugetlb_cgroup_charge_cgroup(int idx, unsigned long nr_pages, struct hugetlb_cgroup **ptr) { return __hugetlb_cgroup_charge_cgroup(idx, nr_pages, ptr, false); } int hugetlb_cgroup_charge_cgroup_rsvd(int idx, unsigned long nr_pages, struct hugetlb_cgroup **ptr) { return __hugetlb_cgroup_charge_cgroup(idx, nr_pages, ptr, true); } |
94ae8ba71
|
265 |
/* Should be called with hugetlb_lock held */ |
1adc4d419
|
266 267 268 |
static void __hugetlb_cgroup_commit_charge(int idx, unsigned long nr_pages, struct hugetlb_cgroup *h_cg, struct page *page, bool rsvd) |
6d76dcf40
|
269 270 271 |
{ if (hugetlb_cgroup_disabled() || !h_cg) return; |
1adc4d419
|
272 |
__set_hugetlb_cgroup(page, h_cg, rsvd); |
6d76dcf40
|
273 274 |
return; } |
1adc4d419
|
275 276 277 278 279 280 281 282 283 284 285 286 287 |
void hugetlb_cgroup_commit_charge(int idx, unsigned long nr_pages, struct hugetlb_cgroup *h_cg, struct page *page) { __hugetlb_cgroup_commit_charge(idx, nr_pages, h_cg, page, false); } void hugetlb_cgroup_commit_charge_rsvd(int idx, unsigned long nr_pages, struct hugetlb_cgroup *h_cg, struct page *page) { __hugetlb_cgroup_commit_charge(idx, nr_pages, h_cg, page, true); } |
6d76dcf40
|
288 289 290 |
/* * Should be called with hugetlb_lock held */ |
1adc4d419
|
291 292 |
static void __hugetlb_cgroup_uncharge_page(int idx, unsigned long nr_pages, struct page *page, bool rsvd) |
6d76dcf40
|
293 294 |
{ struct hugetlb_cgroup *h_cg; |
6d76dcf40
|
295 296 297 |
if (hugetlb_cgroup_disabled()) return; |
7ea8574e5
|
298 |
lockdep_assert_held(&hugetlb_lock); |
1adc4d419
|
299 |
h_cg = __hugetlb_cgroup_from_page(page, rsvd); |
6d76dcf40
|
300 301 |
if (unlikely(!h_cg)) return; |
1adc4d419
|
302 303 304 305 306 307 308 309 |
__set_hugetlb_cgroup(page, NULL, rsvd); page_counter_uncharge(__hugetlb_cgroup_counter_from_cgroup(h_cg, idx, rsvd), nr_pages); if (rsvd) css_put(&h_cg->css); |
6d76dcf40
|
310 311 |
return; } |
1adc4d419
|
312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 |
void hugetlb_cgroup_uncharge_page(int idx, unsigned long nr_pages, struct page *page) { __hugetlb_cgroup_uncharge_page(idx, nr_pages, page, false); } void hugetlb_cgroup_uncharge_page_rsvd(int idx, unsigned long nr_pages, struct page *page) { __hugetlb_cgroup_uncharge_page(idx, nr_pages, page, true); } static void __hugetlb_cgroup_uncharge_cgroup(int idx, unsigned long nr_pages, struct hugetlb_cgroup *h_cg, bool rsvd) |
6d76dcf40
|
327 |
{ |
6d76dcf40
|
328 329 330 331 332 |
if (hugetlb_cgroup_disabled() || !h_cg) return; if (huge_page_order(&hstates[idx]) < HUGETLB_CGROUP_MIN_ORDER) return; |
1adc4d419
|
333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 |
page_counter_uncharge(__hugetlb_cgroup_counter_from_cgroup(h_cg, idx, rsvd), nr_pages); if (rsvd) css_put(&h_cg->css); } void hugetlb_cgroup_uncharge_cgroup(int idx, unsigned long nr_pages, struct hugetlb_cgroup *h_cg) { __hugetlb_cgroup_uncharge_cgroup(idx, nr_pages, h_cg, false); } void hugetlb_cgroup_uncharge_cgroup_rsvd(int idx, unsigned long nr_pages, struct hugetlb_cgroup *h_cg) { __hugetlb_cgroup_uncharge_cgroup(idx, nr_pages, h_cg, true); } |
e9fe92ae0
|
352 353 |
void hugetlb_cgroup_uncharge_counter(struct resv_map *resv, unsigned long start, unsigned long end) |
1adc4d419
|
354 |
{ |
e9fe92ae0
|
355 356 |
if (hugetlb_cgroup_disabled() || !resv || !resv->reservation_counter || !resv->css) |
1adc4d419
|
357 |
return; |
e9fe92ae0
|
358 359 360 |
page_counter_uncharge(resv->reservation_counter, (end - start) * resv->pages_per_hpage); css_put(resv->css); |
6d76dcf40
|
361 |
} |
075a61d07
|
362 363 364 365 366 367 368 369 370 371 372 373 374 375 |
void hugetlb_cgroup_uncharge_file_region(struct resv_map *resv, struct file_region *rg, unsigned long nr_pages) { if (hugetlb_cgroup_disabled() || !resv || !rg || !nr_pages) return; if (rg->reservation_counter && resv->pages_per_hpage && nr_pages > 0 && !resv->reservation_counter) { page_counter_uncharge(rg->reservation_counter, nr_pages * resv->pages_per_hpage); css_put(rg->css); } } |
71f87bee3
|
376 377 |
enum { RES_USAGE, |
cdc2fcfea
|
378 |
RES_RSVD_USAGE, |
71f87bee3
|
379 |
RES_LIMIT, |
cdc2fcfea
|
380 |
RES_RSVD_LIMIT, |
71f87bee3
|
381 |
RES_MAX_USAGE, |
cdc2fcfea
|
382 |
RES_RSVD_MAX_USAGE, |
71f87bee3
|
383 |
RES_FAILCNT, |
cdc2fcfea
|
384 |
RES_RSVD_FAILCNT, |
71f87bee3
|
385 |
}; |
716f479d2
|
386 387 |
static u64 hugetlb_cgroup_read_u64(struct cgroup_subsys_state *css, struct cftype *cft) |
abb8206cb
|
388 |
{ |
71f87bee3
|
389 |
struct page_counter *counter; |
cdc2fcfea
|
390 |
struct page_counter *rsvd_counter; |
182446d08
|
391 |
struct hugetlb_cgroup *h_cg = hugetlb_cgroup_from_css(css); |
abb8206cb
|
392 |
|
71f87bee3
|
393 |
counter = &h_cg->hugepage[MEMFILE_IDX(cft->private)]; |
cdc2fcfea
|
394 |
rsvd_counter = &h_cg->rsvd_hugepage[MEMFILE_IDX(cft->private)]; |
abb8206cb
|
395 |
|
71f87bee3
|
396 397 398 |
switch (MEMFILE_ATTR(cft->private)) { case RES_USAGE: return (u64)page_counter_read(counter) * PAGE_SIZE; |
cdc2fcfea
|
399 400 |
case RES_RSVD_USAGE: return (u64)page_counter_read(rsvd_counter) * PAGE_SIZE; |
71f87bee3
|
401 |
case RES_LIMIT: |
bbec2e151
|
402 |
return (u64)counter->max * PAGE_SIZE; |
cdc2fcfea
|
403 404 |
case RES_RSVD_LIMIT: return (u64)rsvd_counter->max * PAGE_SIZE; |
71f87bee3
|
405 406 |
case RES_MAX_USAGE: return (u64)counter->watermark * PAGE_SIZE; |
cdc2fcfea
|
407 408 |
case RES_RSVD_MAX_USAGE: return (u64)rsvd_counter->watermark * PAGE_SIZE; |
71f87bee3
|
409 410 |
case RES_FAILCNT: return counter->failcnt; |
cdc2fcfea
|
411 412 |
case RES_RSVD_FAILCNT: return rsvd_counter->failcnt; |
71f87bee3
|
413 414 415 |
default: BUG(); } |
abb8206cb
|
416 |
} |
faced7e08
|
417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 |
static int hugetlb_cgroup_read_u64_max(struct seq_file *seq, void *v) { int idx; u64 val; struct cftype *cft = seq_cft(seq); unsigned long limit; struct page_counter *counter; struct hugetlb_cgroup *h_cg = hugetlb_cgroup_from_css(seq_css(seq)); idx = MEMFILE_IDX(cft->private); counter = &h_cg->hugepage[idx]; limit = round_down(PAGE_COUNTER_MAX, 1 << huge_page_order(&hstates[idx])); switch (MEMFILE_ATTR(cft->private)) { |
cdc2fcfea
|
433 434 |
case RES_RSVD_USAGE: counter = &h_cg->rsvd_hugepage[idx]; |
e4a9bc589
|
435 |
fallthrough; |
faced7e08
|
436 437 438 439 440 |
case RES_USAGE: val = (u64)page_counter_read(counter); seq_printf(seq, "%llu ", val * PAGE_SIZE); break; |
cdc2fcfea
|
441 442 |
case RES_RSVD_LIMIT: counter = &h_cg->rsvd_hugepage[idx]; |
e4a9bc589
|
443 |
fallthrough; |
faced7e08
|
444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 |
case RES_LIMIT: val = (u64)counter->max; if (val == limit) seq_puts(seq, "max "); else seq_printf(seq, "%llu ", val * PAGE_SIZE); break; default: BUG(); } return 0; } |
71f87bee3
|
459 |
static DEFINE_MUTEX(hugetlb_limit_mutex); |
451af504d
|
460 |
static ssize_t hugetlb_cgroup_write(struct kernfs_open_file *of, |
faced7e08
|
461 462 |
char *buf, size_t nbytes, loff_t off, const char *max) |
abb8206cb
|
463 |
{ |
71f87bee3
|
464 465 |
int ret, idx; unsigned long nr_pages; |
451af504d
|
466 |
struct hugetlb_cgroup *h_cg = hugetlb_cgroup_from_css(of_css(of)); |
cdc2fcfea
|
467 |
bool rsvd = false; |
abb8206cb
|
468 |
|
71f87bee3
|
469 470 |
if (hugetlb_cgroup_is_root(h_cg)) /* Can't set limit on root */ return -EINVAL; |
451af504d
|
471 |
buf = strstrip(buf); |
faced7e08
|
472 |
ret = page_counter_memparse(buf, max, &nr_pages); |
71f87bee3
|
473 474 |
if (ret) return ret; |
451af504d
|
475 |
idx = MEMFILE_IDX(of_cft(of)->private); |
297880f4a
|
476 |
nr_pages = round_down(nr_pages, 1 << huge_page_order(&hstates[idx])); |
abb8206cb
|
477 |
|
71f87bee3
|
478 |
switch (MEMFILE_ATTR(of_cft(of)->private)) { |
cdc2fcfea
|
479 480 |
case RES_RSVD_LIMIT: rsvd = true; |
e4a9bc589
|
481 |
fallthrough; |
abb8206cb
|
482 |
case RES_LIMIT: |
71f87bee3
|
483 |
mutex_lock(&hugetlb_limit_mutex); |
cdc2fcfea
|
484 |
ret = page_counter_set_max( |
1adc4d419
|
485 |
__hugetlb_cgroup_counter_from_cgroup(h_cg, idx, rsvd), |
cdc2fcfea
|
486 |
nr_pages); |
71f87bee3
|
487 |
mutex_unlock(&hugetlb_limit_mutex); |
abb8206cb
|
488 489 490 491 492 |
break; default: ret = -EINVAL; break; } |
451af504d
|
493 |
return ret ?: nbytes; |
abb8206cb
|
494 |
} |
faced7e08
|
495 496 497 498 499 500 501 502 503 504 505 |
static ssize_t hugetlb_cgroup_write_legacy(struct kernfs_open_file *of, char *buf, size_t nbytes, loff_t off) { return hugetlb_cgroup_write(of, buf, nbytes, off, "-1"); } static ssize_t hugetlb_cgroup_write_dfl(struct kernfs_open_file *of, char *buf, size_t nbytes, loff_t off) { return hugetlb_cgroup_write(of, buf, nbytes, off, "max"); } |
6770c64e5
|
506 507 |
static ssize_t hugetlb_cgroup_reset(struct kernfs_open_file *of, char *buf, size_t nbytes, loff_t off) |
abb8206cb
|
508 |
{ |
71f87bee3
|
509 |
int ret = 0; |
cdc2fcfea
|
510 |
struct page_counter *counter, *rsvd_counter; |
6770c64e5
|
511 |
struct hugetlb_cgroup *h_cg = hugetlb_cgroup_from_css(of_css(of)); |
abb8206cb
|
512 |
|
71f87bee3
|
513 |
counter = &h_cg->hugepage[MEMFILE_IDX(of_cft(of)->private)]; |
cdc2fcfea
|
514 |
rsvd_counter = &h_cg->rsvd_hugepage[MEMFILE_IDX(of_cft(of)->private)]; |
abb8206cb
|
515 |
|
71f87bee3
|
516 |
switch (MEMFILE_ATTR(of_cft(of)->private)) { |
abb8206cb
|
517 |
case RES_MAX_USAGE: |
71f87bee3
|
518 |
page_counter_reset_watermark(counter); |
abb8206cb
|
519 |
break; |
cdc2fcfea
|
520 521 522 |
case RES_RSVD_MAX_USAGE: page_counter_reset_watermark(rsvd_counter); break; |
abb8206cb
|
523 |
case RES_FAILCNT: |
71f87bee3
|
524 |
counter->failcnt = 0; |
abb8206cb
|
525 |
break; |
cdc2fcfea
|
526 527 528 |
case RES_RSVD_FAILCNT: rsvd_counter->failcnt = 0; break; |
abb8206cb
|
529 530 531 532 |
default: ret = -EINVAL; break; } |
6770c64e5
|
533 |
return ret ?: nbytes; |
abb8206cb
|
534 535 536 537 538 539 540 541 542 543 544 545 |
} static char *mem_fmt(char *buf, int size, unsigned long hsize) { if (hsize >= (1UL << 30)) snprintf(buf, size, "%luGB", hsize >> 30); else if (hsize >= (1UL << 20)) snprintf(buf, size, "%luMB", hsize >> 20); else snprintf(buf, size, "%luKB", hsize >> 10); return buf; } |
faced7e08
|
546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 |
static int __hugetlb_events_show(struct seq_file *seq, bool local) { int idx; long max; struct cftype *cft = seq_cft(seq); struct hugetlb_cgroup *h_cg = hugetlb_cgroup_from_css(seq_css(seq)); idx = MEMFILE_IDX(cft->private); if (local) max = atomic_long_read(&h_cg->events_local[idx][HUGETLB_MAX]); else max = atomic_long_read(&h_cg->events[idx][HUGETLB_MAX]); seq_printf(seq, "max %lu ", max); return 0; } static int hugetlb_events_show(struct seq_file *seq, void *v) { return __hugetlb_events_show(seq, false); } static int hugetlb_events_local_show(struct seq_file *seq, void *v) { return __hugetlb_events_show(seq, true); } static void __init __hugetlb_cgroup_file_dfl_init(int idx) |
abb8206cb
|
577 578 579 580 581 582 |
{ char buf[32]; struct cftype *cft; struct hstate *h = &hstates[idx]; /* format the size */ |
cdc2fcfea
|
583 |
mem_fmt(buf, sizeof(buf), huge_page_size(h)); |
abb8206cb
|
584 585 |
/* Add the limit file */ |
faced7e08
|
586 587 588 589 590 591 |
cft = &h->cgroup_files_dfl[0]; snprintf(cft->name, MAX_CFTYPE_NAME, "%s.max", buf); cft->private = MEMFILE_PRIVATE(idx, RES_LIMIT); cft->seq_show = hugetlb_cgroup_read_u64_max; cft->write = hugetlb_cgroup_write_dfl; cft->flags = CFTYPE_NOT_ON_ROOT; |
cdc2fcfea
|
592 |
/* Add the reservation limit file */ |
faced7e08
|
593 |
cft = &h->cgroup_files_dfl[1]; |
cdc2fcfea
|
594 595 596 597 598 599 600 601 |
snprintf(cft->name, MAX_CFTYPE_NAME, "%s.rsvd.max", buf); cft->private = MEMFILE_PRIVATE(idx, RES_RSVD_LIMIT); cft->seq_show = hugetlb_cgroup_read_u64_max; cft->write = hugetlb_cgroup_write_dfl; cft->flags = CFTYPE_NOT_ON_ROOT; /* Add the current usage file */ cft = &h->cgroup_files_dfl[2]; |
faced7e08
|
602 603 604 605 |
snprintf(cft->name, MAX_CFTYPE_NAME, "%s.current", buf); cft->private = MEMFILE_PRIVATE(idx, RES_USAGE); cft->seq_show = hugetlb_cgroup_read_u64_max; cft->flags = CFTYPE_NOT_ON_ROOT; |
cdc2fcfea
|
606 607 608 609 610 611 |
/* Add the current reservation usage file */ cft = &h->cgroup_files_dfl[3]; snprintf(cft->name, MAX_CFTYPE_NAME, "%s.rsvd.current", buf); cft->private = MEMFILE_PRIVATE(idx, RES_RSVD_USAGE); cft->seq_show = hugetlb_cgroup_read_u64_max; cft->flags = CFTYPE_NOT_ON_ROOT; |
faced7e08
|
612 |
/* Add the events file */ |
cdc2fcfea
|
613 |
cft = &h->cgroup_files_dfl[4]; |
faced7e08
|
614 615 616 |
snprintf(cft->name, MAX_CFTYPE_NAME, "%s.events", buf); cft->private = MEMFILE_PRIVATE(idx, 0); cft->seq_show = hugetlb_events_show; |
d5a169597
|
617 |
cft->file_offset = offsetof(struct hugetlb_cgroup, events_file[idx]); |
faced7e08
|
618 619 620 |
cft->flags = CFTYPE_NOT_ON_ROOT; /* Add the events.local file */ |
cdc2fcfea
|
621 |
cft = &h->cgroup_files_dfl[5]; |
faced7e08
|
622 623 624 625 |
snprintf(cft->name, MAX_CFTYPE_NAME, "%s.events.local", buf); cft->private = MEMFILE_PRIVATE(idx, 0); cft->seq_show = hugetlb_events_local_show; cft->file_offset = offsetof(struct hugetlb_cgroup, |
d5a169597
|
626 |
events_local_file[idx]); |
faced7e08
|
627 628 629 |
cft->flags = CFTYPE_NOT_ON_ROOT; /* NULL terminate the last cft */ |
cdc2fcfea
|
630 |
cft = &h->cgroup_files_dfl[6]; |
faced7e08
|
631 632 633 634 635 636 637 638 639 640 641 642 643 |
memset(cft, 0, sizeof(*cft)); WARN_ON(cgroup_add_dfl_cftypes(&hugetlb_cgrp_subsys, h->cgroup_files_dfl)); } static void __init __hugetlb_cgroup_file_legacy_init(int idx) { char buf[32]; struct cftype *cft; struct hstate *h = &hstates[idx]; /* format the size */ |
cdc2fcfea
|
644 |
mem_fmt(buf, sizeof(buf), huge_page_size(h)); |
faced7e08
|
645 646 647 |
/* Add the limit file */ cft = &h->cgroup_files_legacy[0]; |
abb8206cb
|
648 649 |
snprintf(cft->name, MAX_CFTYPE_NAME, "%s.limit_in_bytes", buf); cft->private = MEMFILE_PRIVATE(idx, RES_LIMIT); |
716f479d2
|
650 |
cft->read_u64 = hugetlb_cgroup_read_u64; |
faced7e08
|
651 |
cft->write = hugetlb_cgroup_write_legacy; |
abb8206cb
|
652 |
|
cdc2fcfea
|
653 |
/* Add the reservation limit file */ |
faced7e08
|
654 |
cft = &h->cgroup_files_legacy[1]; |
cdc2fcfea
|
655 656 657 658 659 660 661 |
snprintf(cft->name, MAX_CFTYPE_NAME, "%s.rsvd.limit_in_bytes", buf); cft->private = MEMFILE_PRIVATE(idx, RES_RSVD_LIMIT); cft->read_u64 = hugetlb_cgroup_read_u64; cft->write = hugetlb_cgroup_write_legacy; /* Add the usage file */ cft = &h->cgroup_files_legacy[2]; |
abb8206cb
|
662 663 |
snprintf(cft->name, MAX_CFTYPE_NAME, "%s.usage_in_bytes", buf); cft->private = MEMFILE_PRIVATE(idx, RES_USAGE); |
716f479d2
|
664 |
cft->read_u64 = hugetlb_cgroup_read_u64; |
abb8206cb
|
665 |
|
cdc2fcfea
|
666 667 668 669 670 |
/* Add the reservation usage file */ cft = &h->cgroup_files_legacy[3]; snprintf(cft->name, MAX_CFTYPE_NAME, "%s.rsvd.usage_in_bytes", buf); cft->private = MEMFILE_PRIVATE(idx, RES_RSVD_USAGE); cft->read_u64 = hugetlb_cgroup_read_u64; |
abb8206cb
|
671 |
/* Add the MAX usage file */ |
cdc2fcfea
|
672 |
cft = &h->cgroup_files_legacy[4]; |
abb8206cb
|
673 674 |
snprintf(cft->name, MAX_CFTYPE_NAME, "%s.max_usage_in_bytes", buf); cft->private = MEMFILE_PRIVATE(idx, RES_MAX_USAGE); |
6770c64e5
|
675 |
cft->write = hugetlb_cgroup_reset; |
716f479d2
|
676 |
cft->read_u64 = hugetlb_cgroup_read_u64; |
abb8206cb
|
677 |
|
cdc2fcfea
|
678 679 680 681 682 683 |
/* Add the MAX reservation usage file */ cft = &h->cgroup_files_legacy[5]; snprintf(cft->name, MAX_CFTYPE_NAME, "%s.rsvd.max_usage_in_bytes", buf); cft->private = MEMFILE_PRIVATE(idx, RES_RSVD_MAX_USAGE); cft->write = hugetlb_cgroup_reset; cft->read_u64 = hugetlb_cgroup_read_u64; |
abb8206cb
|
684 |
/* Add the failcntfile */ |
cdc2fcfea
|
685 |
cft = &h->cgroup_files_legacy[6]; |
abb8206cb
|
686 |
snprintf(cft->name, MAX_CFTYPE_NAME, "%s.failcnt", buf); |
cdc2fcfea
|
687 688 689 690 691 692 693 694 |
cft->private = MEMFILE_PRIVATE(idx, RES_FAILCNT); cft->write = hugetlb_cgroup_reset; cft->read_u64 = hugetlb_cgroup_read_u64; /* Add the reservation failcntfile */ cft = &h->cgroup_files_legacy[7]; snprintf(cft->name, MAX_CFTYPE_NAME, "%s.rsvd.failcnt", buf); cft->private = MEMFILE_PRIVATE(idx, RES_RSVD_FAILCNT); |
6770c64e5
|
695 |
cft->write = hugetlb_cgroup_reset; |
716f479d2
|
696 |
cft->read_u64 = hugetlb_cgroup_read_u64; |
abb8206cb
|
697 698 |
/* NULL terminate the last cft */ |
cdc2fcfea
|
699 |
cft = &h->cgroup_files_legacy[8]; |
abb8206cb
|
700 |
memset(cft, 0, sizeof(*cft)); |
2cf669a58
|
701 |
WARN_ON(cgroup_add_legacy_cftypes(&hugetlb_cgrp_subsys, |
faced7e08
|
702 703 704 705 706 707 708 |
h->cgroup_files_legacy)); } static void __init __hugetlb_cgroup_file_init(int idx) { __hugetlb_cgroup_file_dfl_init(idx); __hugetlb_cgroup_file_legacy_init(idx); |
7179e7bf4
|
709 710 711 712 713 714 715 716 717 718 |
} void __init hugetlb_cgroup_file_init(void) { struct hstate *h; for_each_hstate(h) { /* * Add cgroup control files only if the huge page consists * of more than two normal pages. This is because we use |
1d798ca3f
|
719 |
* page[2].private for storing cgroup details. |
7179e7bf4
|
720 721 722 723 |
*/ if (huge_page_order(h) >= HUGETLB_CGROUP_MIN_ORDER) __hugetlb_cgroup_file_init(hstate_index(h)); } |
abb8206cb
|
724 |
} |
75754681f
|
725 726 727 728 |
/* * hugetlb_lock will make sure a parallel cgroup rmdir won't happen * when we migrate hugepages */ |
8e6ac7fab
|
729 730 731 |
void hugetlb_cgroup_migrate(struct page *oldhpage, struct page *newhpage) { struct hugetlb_cgroup *h_cg; |
1adc4d419
|
732 |
struct hugetlb_cgroup *h_cg_rsvd; |
94ae8ba71
|
733 |
struct hstate *h = page_hstate(oldhpage); |
8e6ac7fab
|
734 735 736 |
if (hugetlb_cgroup_disabled()) return; |
309381fea
|
737 |
VM_BUG_ON_PAGE(!PageHuge(oldhpage), oldhpage); |
8e6ac7fab
|
738 739 |
spin_lock(&hugetlb_lock); h_cg = hugetlb_cgroup_from_page(oldhpage); |
1adc4d419
|
740 |
h_cg_rsvd = hugetlb_cgroup_from_page_rsvd(oldhpage); |
8e6ac7fab
|
741 |
set_hugetlb_cgroup(oldhpage, NULL); |
9808895e1
|
742 |
set_hugetlb_cgroup_rsvd(oldhpage, NULL); |
8e6ac7fab
|
743 744 |
/* move the h_cg details to new cgroup */ |
9808895e1
|
745 |
set_hugetlb_cgroup(newhpage, h_cg); |
1adc4d419
|
746 |
set_hugetlb_cgroup_rsvd(newhpage, h_cg_rsvd); |
94ae8ba71
|
747 |
list_move(&newhpage->lru, &h->hugepage_activelist); |
8e6ac7fab
|
748 |
spin_unlock(&hugetlb_lock); |
8e6ac7fab
|
749 750 |
return; } |
faced7e08
|
751 752 753 |
static struct cftype hugetlb_files[] = { {} /* terminate */ }; |
073219e99
|
754 |
struct cgroup_subsys hugetlb_cgrp_subsys = { |
92fb97487
|
755 756 757 |
.css_alloc = hugetlb_cgroup_css_alloc, .css_offline = hugetlb_cgroup_css_offline, .css_free = hugetlb_cgroup_css_free, |
faced7e08
|
758 759 |
.dfl_cftypes = hugetlb_files, .legacy_cftypes = hugetlb_files, |
2bc64a204
|
760 |
}; |