Blame view
mm/page_owner.c
16.3 KB
b24413180 License cleanup: ... |
1 |
// SPDX-License-Identifier: GPL-2.0 |
48c96a368 mm/page_owner: ke... |
2 3 4 5 |
#include <linux/debugfs.h> #include <linux/mm.h> #include <linux/slab.h> #include <linux/uaccess.h> |
57c8a661d mm: remove includ... |
6 |
#include <linux/memblock.h> |
48c96a368 mm/page_owner: ke... |
7 8 |
#include <linux/stacktrace.h> #include <linux/page_owner.h> |
7dd80b8af mm, page_owner: c... |
9 |
#include <linux/jump_label.h> |
7cd12b4ab mm, page_owner: t... |
10 |
#include <linux/migrate.h> |
f2ca0b557 mm/page_owner: us... |
11 |
#include <linux/stackdepot.h> |
e2f612e67 mm/page_owner: mo... |
12 |
#include <linux/seq_file.h> |
fd0328e37 UPSTREAM: mm/page... |
13 |
#include <linux/sched/clock.h> |
f2ca0b557 mm/page_owner: us... |
14 |
|
48c96a368 mm/page_owner: ke... |
15 |
#include "internal.h" |
f2ca0b557 mm/page_owner: us... |
16 17 18 19 20 |
/* * TODO: teach PAGE_OWNER_STACK_DEPTH (__dump_page_owner and save_stack) * to use off stack temporal storage */ #define PAGE_OWNER_STACK_DEPTH (16) |
9300d8dfd mm/page_owner: do... |
21 |
struct page_owner { |
6b4c54e37 mm/page_owner.c: ... |
22 23 |
unsigned short order; short last_migrate_reason; |
9300d8dfd mm/page_owner: do... |
24 |
gfp_t gfp_mask; |
9300d8dfd mm/page_owner: do... |
25 |
depot_stack_handle_t handle; |
8974558f4 mm, page_owner, d... |
26 |
depot_stack_handle_t free_handle; |
fd0328e37 UPSTREAM: mm/page... |
27 28 |
u64 ts_nsec; pid_t pid; |
9300d8dfd mm/page_owner: do... |
29 |
}; |
0fe9a448a mm, page_owner: d... |
30 |
static bool page_owner_enabled = false; |
7dd80b8af mm, page_owner: c... |
31 |
DEFINE_STATIC_KEY_FALSE(page_owner_inited); |
48c96a368 mm/page_owner: ke... |
32 |
|
f2ca0b557 mm/page_owner: us... |
33 34 |
static depot_stack_handle_t dummy_handle; static depot_stack_handle_t failure_handle; |
dab4ead1a mm, page_owner: m... |
35 |
static depot_stack_handle_t early_handle; |
f2ca0b557 mm/page_owner: us... |
36 |
|
61cf5febd mm/page_owner: co... |
37 |
static void init_early_allocated_pages(void); |
1173194e1 mm/page_owner.c: ... |
38 |
static int __init early_page_owner_param(char *buf) |
48c96a368 mm/page_owner: ke... |
39 40 41 42 43 |
{ if (!buf) return -EINVAL; if (strcmp(buf, "on") == 0) |
0fe9a448a mm, page_owner: d... |
44 |
page_owner_enabled = true; |
48c96a368 mm/page_owner: ke... |
45 46 47 48 49 50 51 |
return 0; } early_param("page_owner", early_page_owner_param); static bool need_page_owner(void) { |
0fe9a448a mm, page_owner: d... |
52 |
return page_owner_enabled; |
48c96a368 mm/page_owner: ke... |
53 |
} |
dab4ead1a mm, page_owner: m... |
54 |
static __always_inline depot_stack_handle_t create_dummy_stack(void) |
f2ca0b557 mm/page_owner: us... |
55 56 |
{ unsigned long entries[4]; |
af52bf6b9 mm/page_owner: Si... |
57 |
unsigned int nr_entries; |
f2ca0b557 mm/page_owner: us... |
58 |
|
af52bf6b9 mm/page_owner: Si... |
59 60 |
nr_entries = stack_trace_save(entries, ARRAY_SIZE(entries), 0); return stack_depot_save(entries, nr_entries, GFP_KERNEL); |
f2ca0b557 mm/page_owner: us... |
61 |
} |
dab4ead1a mm, page_owner: m... |
62 |
static noinline void register_dummy_stack(void) |
f2ca0b557 mm/page_owner: us... |
63 |
{ |
dab4ead1a mm, page_owner: m... |
64 65 |
dummy_handle = create_dummy_stack(); } |
f2ca0b557 mm/page_owner: us... |
66 |
|
dab4ead1a mm, page_owner: m... |
67 68 69 70 |
static noinline void register_failure_stack(void) { failure_handle = create_dummy_stack(); } |
f2ca0b557 mm/page_owner: us... |
71 |
|
dab4ead1a mm, page_owner: m... |
72 73 74 |
static noinline void register_early_stack(void) { early_handle = create_dummy_stack(); |
f2ca0b557 mm/page_owner: us... |
75 |
} |
48c96a368 mm/page_owner: ke... |
76 77 |
static void init_page_owner(void) { |
0fe9a448a mm, page_owner: d... |
78 |
if (!page_owner_enabled) |
48c96a368 mm/page_owner: ke... |
79 |
return; |
f2ca0b557 mm/page_owner: us... |
80 81 |
register_dummy_stack(); register_failure_stack(); |
dab4ead1a mm, page_owner: m... |
82 |
register_early_stack(); |
7dd80b8af mm, page_owner: c... |
83 |
static_branch_enable(&page_owner_inited); |
61cf5febd mm/page_owner: co... |
84 |
init_early_allocated_pages(); |
48c96a368 mm/page_owner: ke... |
85 86 87 |
} struct page_ext_operations page_owner_ops = { |
9300d8dfd mm/page_owner: do... |
88 |
.size = sizeof(struct page_owner), |
48c96a368 mm/page_owner: ke... |
89 90 91 |
.need = need_page_owner, .init = init_page_owner, }; |
b76264c26 ANDROID: mm: Expo... |
92 |
struct page_owner *get_page_owner(struct page_ext *page_ext) |
9300d8dfd mm/page_owner: do... |
93 94 95 |
{ return (void *)page_ext + page_owner_ops.offset; } |
b76264c26 ANDROID: mm: Expo... |
96 |
EXPORT_SYMBOL_GPL(get_page_owner); |
9300d8dfd mm/page_owner: do... |
97 |
|
af52bf6b9 mm/page_owner: Si... |
98 99 100 |
static inline bool check_recursive_alloc(unsigned long *entries, unsigned int nr_entries, unsigned long ip) |
48c96a368 mm/page_owner: ke... |
101 |
{ |
af52bf6b9 mm/page_owner: Si... |
102 |
unsigned int i; |
f2ca0b557 mm/page_owner: us... |
103 |
|
af52bf6b9 mm/page_owner: Si... |
104 105 |
for (i = 0; i < nr_entries; i++) { if (entries[i] == ip) |
f2ca0b557 mm/page_owner: us... |
106 107 |
return true; } |
f2ca0b557 mm/page_owner: us... |
108 109 110 111 112 113 |
return false; } static noinline depot_stack_handle_t save_stack(gfp_t flags) { unsigned long entries[PAGE_OWNER_STACK_DEPTH]; |
f2ca0b557 mm/page_owner: us... |
114 |
depot_stack_handle_t handle; |
af52bf6b9 mm/page_owner: Si... |
115 |
unsigned int nr_entries; |
f2ca0b557 mm/page_owner: us... |
116 |
|
af52bf6b9 mm/page_owner: Si... |
117 |
nr_entries = stack_trace_save(entries, ARRAY_SIZE(entries), 2); |
f2ca0b557 mm/page_owner: us... |
118 119 |
/* |
af52bf6b9 mm/page_owner: Si... |
120 121 122 123 124 125 |
* We need to check recursion here because our request to * stackdepot could trigger memory allocation to save new * entry. New memory allocation would reach here and call * stack_depot_save_entries() again if we don't catch it. There is * still not enough memory in stackdepot so it would try to * allocate memory again and loop forever. |
f2ca0b557 mm/page_owner: us... |
126 |
*/ |
af52bf6b9 mm/page_owner: Si... |
127 |
if (check_recursive_alloc(entries, nr_entries, _RET_IP_)) |
f2ca0b557 mm/page_owner: us... |
128 |
return dummy_handle; |
af52bf6b9 mm/page_owner: Si... |
129 |
handle = stack_depot_save(entries, nr_entries, flags); |
f2ca0b557 mm/page_owner: us... |
130 131 132 133 134 |
if (!handle) handle = failure_handle; return handle; } |
8974558f4 mm, page_owner, d... |
135 136 137 138 |
void __reset_page_owner(struct page *page, unsigned int order) { int i; struct page_ext *page_ext; |
8974558f4 mm, page_owner, d... |
139 140 |
depot_stack_handle_t handle = 0; struct page_owner *page_owner; |
0fe9a448a mm, page_owner: d... |
141 |
handle = save_stack(GFP_NOWAIT | __GFP_NOWARN); |
8974558f4 mm, page_owner, d... |
142 |
|
5556cfe8d mm, page_owner: f... |
143 144 145 |
page_ext = lookup_page_ext(page); if (unlikely(!page_ext)) return; |
8974558f4 mm, page_owner, d... |
146 |
for (i = 0; i < (1 << order); i++) { |
fdf3bf809 mm, page_owner: r... |
147 |
__clear_bit(PAGE_EXT_OWNER_ALLOCATED, &page_ext->flags); |
0fe9a448a mm, page_owner: d... |
148 149 |
page_owner = get_page_owner(page_ext); page_owner->free_handle = handle; |
5556cfe8d mm, page_owner: f... |
150 |
page_ext = page_ext_next(page_ext); |
8974558f4 mm, page_owner, d... |
151 152 |
} } |
7e2f2a0cd mm, page_owner: r... |
153 154 155 |
static inline void __set_page_owner_handle(struct page *page, struct page_ext *page_ext, depot_stack_handle_t handle, unsigned int order, gfp_t gfp_mask) |
f2ca0b557 mm/page_owner: us... |
156 |
{ |
9300d8dfd mm/page_owner: do... |
157 |
struct page_owner *page_owner; |
7e2f2a0cd mm, page_owner: r... |
158 |
int i; |
48c96a368 mm/page_owner: ke... |
159 |
|
7e2f2a0cd mm, page_owner: r... |
160 161 162 163 164 165 |
for (i = 0; i < (1 << order); i++) { page_owner = get_page_owner(page_ext); page_owner->handle = handle; page_owner->order = order; page_owner->gfp_mask = gfp_mask; page_owner->last_migrate_reason = -1; |
fd0328e37 UPSTREAM: mm/page... |
166 167 |
page_owner->pid = current->pid; page_owner->ts_nsec = local_clock(); |
7e2f2a0cd mm, page_owner: r... |
168 |
__set_bit(PAGE_EXT_OWNER, &page_ext->flags); |
fdf3bf809 mm, page_owner: r... |
169 |
__set_bit(PAGE_EXT_OWNER_ALLOCATED, &page_ext->flags); |
48c96a368 mm/page_owner: ke... |
170 |
|
5556cfe8d mm, page_owner: f... |
171 |
page_ext = page_ext_next(page_ext); |
7e2f2a0cd mm, page_owner: r... |
172 |
} |
48c96a368 mm/page_owner: ke... |
173 |
} |
dab4ead1a mm, page_owner: m... |
174 175 176 177 178 179 180 181 182 183 |
noinline void __set_page_owner(struct page *page, unsigned int order, gfp_t gfp_mask) { struct page_ext *page_ext = lookup_page_ext(page); depot_stack_handle_t handle; if (unlikely(!page_ext)) return; handle = save_stack(gfp_mask); |
7e2f2a0cd mm, page_owner: r... |
184 |
__set_page_owner_handle(page, page_ext, handle, order, gfp_mask); |
dab4ead1a mm, page_owner: m... |
185 |
} |
7cd12b4ab mm, page_owner: t... |
186 187 188 |
void __set_page_owner_migrate_reason(struct page *page, int reason) { struct page_ext *page_ext = lookup_page_ext(page); |
9300d8dfd mm/page_owner: do... |
189 |
struct page_owner *page_owner; |
f86e42719 mm: check the ret... |
190 191 |
if (unlikely(!page_ext)) return; |
7cd12b4ab mm, page_owner: t... |
192 |
|
9300d8dfd mm/page_owner: do... |
193 194 |
page_owner = get_page_owner(page_ext); page_owner->last_migrate_reason = reason; |
7cd12b4ab mm, page_owner: t... |
195 |
} |
8fb156c9e mm/page_owner: ch... |
196 |
void __split_page_owner(struct page *page, unsigned int nr) |
e2cfc9112 mm/page_owner: se... |
197 |
{ |
a9627bc5e mm/page_owner: in... |
198 |
int i; |
e2cfc9112 mm/page_owner: se... |
199 |
struct page_ext *page_ext = lookup_page_ext(page); |
9300d8dfd mm/page_owner: do... |
200 |
struct page_owner *page_owner; |
a9627bc5e mm/page_owner: in... |
201 |
|
f86e42719 mm: check the ret... |
202 |
if (unlikely(!page_ext)) |
a9627bc5e mm/page_owner: in... |
203 |
return; |
e2cfc9112 mm/page_owner: se... |
204 |
|
8fb156c9e mm/page_owner: ch... |
205 |
for (i = 0; i < nr; i++) { |
7e2f2a0cd mm, page_owner: r... |
206 207 |
page_owner = get_page_owner(page_ext); page_owner->order = 0; |
5556cfe8d mm, page_owner: f... |
208 |
page_ext = page_ext_next(page_ext); |
7e2f2a0cd mm, page_owner: r... |
209 |
} |
e2cfc9112 mm/page_owner: se... |
210 |
} |
d435edca9 mm, page_owner: c... |
211 212 213 214 |
void __copy_page_owner(struct page *oldpage, struct page *newpage) { struct page_ext *old_ext = lookup_page_ext(oldpage); struct page_ext *new_ext = lookup_page_ext(newpage); |
9300d8dfd mm/page_owner: do... |
215 |
struct page_owner *old_page_owner, *new_page_owner; |
d435edca9 mm, page_owner: c... |
216 |
|
f86e42719 mm: check the ret... |
217 218 |
if (unlikely(!old_ext || !new_ext)) return; |
9300d8dfd mm/page_owner: do... |
219 220 221 222 223 224 225 |
old_page_owner = get_page_owner(old_ext); new_page_owner = get_page_owner(new_ext); new_page_owner->order = old_page_owner->order; new_page_owner->gfp_mask = old_page_owner->gfp_mask; new_page_owner->last_migrate_reason = old_page_owner->last_migrate_reason; new_page_owner->handle = old_page_owner->handle; |
fd0328e37 UPSTREAM: mm/page... |
226 227 |
new_page_owner->pid = old_page_owner->pid; new_page_owner->ts_nsec = old_page_owner->ts_nsec; |
d435edca9 mm, page_owner: c... |
228 229 230 231 232 233 234 235 236 237 238 |
/* * We don't clear the bit on the oldpage as it's going to be freed * after migration. Until then, the info can be useful in case of * a bug, and the overal stats will be off a bit only temporarily. * Also, migrate_misplaced_transhuge_page() can still fail the * migration and then we want the oldpage to retain the info. But * in that case we also don't need to explicitly clear the info from * the new page, which will be freed. */ __set_bit(PAGE_EXT_OWNER, &new_ext->flags); |
fdf3bf809 mm, page_owner: r... |
239 |
__set_bit(PAGE_EXT_OWNER_ALLOCATED, &new_ext->flags); |
d435edca9 mm, page_owner: c... |
240 |
} |
e2f612e67 mm/page_owner: mo... |
241 242 243 244 245 |
void pagetypeinfo_showmixedcount_print(struct seq_file *m, pg_data_t *pgdat, struct zone *zone) { struct page *page; struct page_ext *page_ext; |
9300d8dfd mm/page_owner: do... |
246 |
struct page_owner *page_owner; |
e2f612e67 mm/page_owner: mo... |
247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 |
unsigned long pfn = zone->zone_start_pfn, block_end_pfn; unsigned long end_pfn = pfn + zone->spanned_pages; unsigned long count[MIGRATE_TYPES] = { 0, }; int pageblock_mt, page_mt; int i; /* Scan block by block. First and last block may be incomplete */ pfn = zone->zone_start_pfn; /* * Walk the zone in pageblock_nr_pages steps. If a page block spans * a zone boundary, it will be double counted between zones. This does * not matter as the mixed block count will still be correct */ for (; pfn < end_pfn; ) { |
a26ee565b mm/page_owner: do... |
262 263 |
page = pfn_to_online_page(pfn); if (!page) { |
e2f612e67 mm/page_owner: mo... |
264 265 266 267 268 269 |
pfn = ALIGN(pfn + 1, MAX_ORDER_NR_PAGES); continue; } block_end_pfn = ALIGN(pfn + 1, pageblock_nr_pages); block_end_pfn = min(block_end_pfn, end_pfn); |
e2f612e67 mm/page_owner: mo... |
270 271 272 273 274 |
pageblock_mt = get_pageblock_migratetype(page); for (; pfn < block_end_pfn; pfn++) { if (!pfn_valid_within(pfn)) continue; |
a26ee565b mm/page_owner: do... |
275 |
/* The pageblock is online, no need to recheck. */ |
e2f612e67 mm/page_owner: mo... |
276 277 278 279 280 281 |
page = pfn_to_page(pfn); if (page_zone(page) != zone) continue; if (PageBuddy(page)) { |
727c080f0 mm: avoid taking ... |
282 |
unsigned long freepage_order; |
ab130f910 mm: rename page_o... |
283 |
freepage_order = buddy_order_unsafe(page); |
727c080f0 mm: avoid taking ... |
284 285 |
if (freepage_order < MAX_ORDER) pfn += (1UL << freepage_order) - 1; |
e2f612e67 mm/page_owner: mo... |
286 287 288 289 290 291 292 293 294 |
continue; } if (PageReserved(page)) continue; page_ext = lookup_page_ext(page); if (unlikely(!page_ext)) continue; |
fdf3bf809 mm, page_owner: r... |
295 |
if (!test_bit(PAGE_EXT_OWNER_ALLOCATED, &page_ext->flags)) |
e2f612e67 mm/page_owner: mo... |
296 |
continue; |
9300d8dfd mm/page_owner: do... |
297 |
page_owner = get_page_owner(page_ext); |
01c0bfe06 mm: rename gfpfla... |
298 |
page_mt = gfp_migratetype(page_owner->gfp_mask); |
e2f612e67 mm/page_owner: mo... |
299 300 301 302 303 304 305 306 307 |
if (pageblock_mt != page_mt) { if (is_migrate_cma(pageblock_mt)) count[MIGRATE_MOVABLE]++; else count[pageblock_mt]++; pfn = block_end_pfn; break; } |
9300d8dfd mm/page_owner: do... |
308 |
pfn += (1UL << page_owner->order) - 1; |
e2f612e67 mm/page_owner: mo... |
309 310 311 312 313 314 315 316 317 318 |
} } /* Print counts */ seq_printf(m, "Node %d, zone %8s ", pgdat->node_id, zone->name); for (i = 0; i < MIGRATE_TYPES; i++) seq_printf(m, "%12lu ", count[i]); seq_putc(m, ' '); } |
48c96a368 mm/page_owner: ke... |
319 320 |
static ssize_t print_page_owner(char __user *buf, size_t count, unsigned long pfn, |
9300d8dfd mm/page_owner: do... |
321 |
struct page *page, struct page_owner *page_owner, |
f2ca0b557 mm/page_owner: us... |
322 |
depot_stack_handle_t handle) |
48c96a368 mm/page_owner: ke... |
323 |
{ |
af52bf6b9 mm/page_owner: Si... |
324 325 326 |
int ret, pageblock_mt, page_mt; unsigned long *entries; unsigned int nr_entries; |
48c96a368 mm/page_owner: ke... |
327 |
char *kbuf; |
c8f61cfc8 mm/page_owner: cl... |
328 |
count = min_t(size_t, count, PAGE_SIZE); |
48c96a368 mm/page_owner: ke... |
329 330 331 332 333 |
kbuf = kmalloc(count, GFP_KERNEL); if (!kbuf) return -ENOMEM; ret = snprintf(kbuf, count, |
fd0328e37 UPSTREAM: mm/page... |
334 335 |
"Page allocated via order %u, mask %#x(%pGg), pid %d, ts %llu ns ", |
9300d8dfd mm/page_owner: do... |
336 |
page_owner->order, page_owner->gfp_mask, |
fd0328e37 UPSTREAM: mm/page... |
337 338 |
&page_owner->gfp_mask, page_owner->pid, page_owner->ts_nsec); |
48c96a368 mm/page_owner: ke... |
339 340 341 342 343 |
if (ret >= count) goto err; /* Print information relevant to grouping pages by mobility */ |
0b423ca22 mm, page_alloc: i... |
344 |
pageblock_mt = get_pageblock_migratetype(page); |
01c0bfe06 mm: rename gfpfla... |
345 |
page_mt = gfp_migratetype(page_owner->gfp_mask); |
48c96a368 mm/page_owner: ke... |
346 |
ret += snprintf(kbuf + ret, count - ret, |
60f30350f mm, page_owner: p... |
347 348 |
"PFN %lu type %s Block %lu type %s Flags %#lx(%pGp) ", |
48c96a368 mm/page_owner: ke... |
349 |
pfn, |
60f30350f mm, page_owner: p... |
350 |
migratetype_names[page_mt], |
48c96a368 mm/page_owner: ke... |
351 |
pfn >> pageblock_order, |
60f30350f mm, page_owner: p... |
352 353 |
migratetype_names[pageblock_mt], page->flags, &page->flags); |
48c96a368 mm/page_owner: ke... |
354 355 356 |
if (ret >= count) goto err; |
af52bf6b9 mm/page_owner: Si... |
357 358 |
nr_entries = stack_depot_fetch(handle, &entries); ret += stack_trace_snprint(kbuf + ret, count - ret, entries, nr_entries, 0); |
48c96a368 mm/page_owner: ke... |
359 360 |
if (ret >= count) goto err; |
9300d8dfd mm/page_owner: do... |
361 |
if (page_owner->last_migrate_reason != -1) { |
7cd12b4ab mm, page_owner: t... |
362 363 364 |
ret += snprintf(kbuf + ret, count - ret, "Page has been migrated, last migrate reason: %s ", |
9300d8dfd mm/page_owner: do... |
365 |
migrate_reason_names[page_owner->last_migrate_reason]); |
7cd12b4ab mm, page_owner: t... |
366 367 368 |
if (ret >= count) goto err; } |
48c96a368 mm/page_owner: ke... |
369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 |
ret += snprintf(kbuf + ret, count - ret, " "); if (ret >= count) goto err; if (copy_to_user(buf, kbuf, ret)) ret = -EFAULT; kfree(kbuf); return ret; err: kfree(kbuf); return -ENOMEM; } |
4e462112e mm, page_owner: d... |
384 385 386 |
void __dump_page_owner(struct page *page) { struct page_ext *page_ext = lookup_page_ext(page); |
9300d8dfd mm/page_owner: do... |
387 |
struct page_owner *page_owner; |
f2ca0b557 mm/page_owner: us... |
388 |
depot_stack_handle_t handle; |
af52bf6b9 mm/page_owner: Si... |
389 390 |
unsigned long *entries; unsigned int nr_entries; |
8285027fc mm/page_owner: av... |
391 392 |
gfp_t gfp_mask; int mt; |
4e462112e mm, page_owner: d... |
393 |
|
f86e42719 mm: check the ret... |
394 395 396 397 398 |
if (unlikely(!page_ext)) { pr_alert("There is not page extension available. "); return; } |
9300d8dfd mm/page_owner: do... |
399 400 401 |
page_owner = get_page_owner(page_ext); gfp_mask = page_owner->gfp_mask; |
01c0bfe06 mm: rename gfpfla... |
402 |
mt = gfp_migratetype(gfp_mask); |
f86e42719 mm: check the ret... |
403 |
|
4e462112e mm, page_owner: d... |
404 |
if (!test_bit(PAGE_EXT_OWNER, &page_ext->flags)) { |
37389167a mm, page_owner: k... |
405 406 |
pr_alert("page_owner info is not present (never set?) "); |
4e462112e mm, page_owner: d... |
407 408 |
return; } |
fdf3bf809 mm, page_owner: r... |
409 |
if (test_bit(PAGE_EXT_OWNER_ALLOCATED, &page_ext->flags)) |
37389167a mm, page_owner: k... |
410 411 412 413 414 |
pr_alert("page_owner tracks the page as allocated "); else pr_alert("page_owner tracks the page as freed "); |
fd0328e37 UPSTREAM: mm/page... |
415 416 417 418 |
pr_alert("page last allocated via order %u, migratetype %s, gfp_mask %#x(%pGg), pid %d, ts %llu ", page_owner->order, migratetype_names[mt], gfp_mask, &gfp_mask, page_owner->pid, page_owner->ts_nsec); |
37389167a mm, page_owner: k... |
419 |
|
9300d8dfd mm/page_owner: do... |
420 |
handle = READ_ONCE(page_owner->handle); |
f2ca0b557 mm/page_owner: us... |
421 |
if (!handle) { |
37389167a mm, page_owner: k... |
422 423 424 425 426 |
pr_alert("page_owner allocation stack trace missing "); } else { nr_entries = stack_depot_fetch(handle, &entries); stack_trace_print(entries, nr_entries, 0); |
f2ca0b557 mm/page_owner: us... |
427 |
} |
8974558f4 mm, page_owner, d... |
428 429 430 431 432 433 434 435 436 437 |
handle = READ_ONCE(page_owner->free_handle); if (!handle) { pr_alert("page_owner free stack trace missing "); } else { nr_entries = stack_depot_fetch(handle, &entries); pr_alert("page last free stack trace: "); stack_trace_print(entries, nr_entries, 0); } |
8974558f4 mm, page_owner, d... |
438 |
|
9300d8dfd mm/page_owner: do... |
439 |
if (page_owner->last_migrate_reason != -1) |
4e462112e mm, page_owner: d... |
440 441 |
pr_alert("page has been migrated, last migrate reason: %s ", |
9300d8dfd mm/page_owner: do... |
442 |
migrate_reason_names[page_owner->last_migrate_reason]); |
4e462112e mm, page_owner: d... |
443 |
} |
48c96a368 mm/page_owner: ke... |
444 445 446 447 448 449 |
static ssize_t read_page_owner(struct file *file, char __user *buf, size_t count, loff_t *ppos) { unsigned long pfn; struct page *page; struct page_ext *page_ext; |
9300d8dfd mm/page_owner: do... |
450 |
struct page_owner *page_owner; |
f2ca0b557 mm/page_owner: us... |
451 |
depot_stack_handle_t handle; |
48c96a368 mm/page_owner: ke... |
452 |
|
7dd80b8af mm, page_owner: c... |
453 |
if (!static_branch_unlikely(&page_owner_inited)) |
48c96a368 mm/page_owner: ke... |
454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 |
return -EINVAL; page = NULL; pfn = min_low_pfn + *ppos; /* Find a valid PFN or the start of a MAX_ORDER_NR_PAGES area */ while (!pfn_valid(pfn) && (pfn & (MAX_ORDER_NR_PAGES - 1)) != 0) pfn++; drain_all_pages(NULL); /* Find an allocated page */ for (; pfn < max_pfn; pfn++) { /* * If the new page is in a new MAX_ORDER_NR_PAGES area, * validate the area as existing, skip it if not */ if ((pfn & (MAX_ORDER_NR_PAGES - 1)) == 0 && !pfn_valid(pfn)) { pfn += MAX_ORDER_NR_PAGES - 1; continue; } /* Check for holes within a MAX_ORDER area */ if (!pfn_valid_within(pfn)) continue; page = pfn_to_page(pfn); if (PageBuddy(page)) { |
ab130f910 mm: rename page_o... |
482 |
unsigned long freepage_order = buddy_order_unsafe(page); |
48c96a368 mm/page_owner: ke... |
483 484 485 486 487 488 489 |
if (freepage_order < MAX_ORDER) pfn += (1UL << freepage_order) - 1; continue; } page_ext = lookup_page_ext(page); |
f86e42719 mm: check the ret... |
490 491 |
if (unlikely(!page_ext)) continue; |
48c96a368 mm/page_owner: ke... |
492 493 |
/* |
61cf5febd mm/page_owner: co... |
494 495 |
* Some pages could be missed by concurrent allocation or free, * because we don't hold the zone lock. |
48c96a368 mm/page_owner: ke... |
496 497 498 |
*/ if (!test_bit(PAGE_EXT_OWNER, &page_ext->flags)) continue; |
37389167a mm, page_owner: k... |
499 500 501 502 |
/* * Although we do have the info about past allocation of free * pages, it's not relevant for current memory usage. */ |
fdf3bf809 mm, page_owner: r... |
503 |
if (!test_bit(PAGE_EXT_OWNER_ALLOCATED, &page_ext->flags)) |
37389167a mm, page_owner: k... |
504 |
continue; |
9300d8dfd mm/page_owner: do... |
505 |
page_owner = get_page_owner(page_ext); |
f2ca0b557 mm/page_owner: us... |
506 |
/* |
7e2f2a0cd mm, page_owner: r... |
507 508 509 510 511 512 513 |
* Don't print "tail" pages of high-order allocations as that * would inflate the stats. */ if (!IS_ALIGNED(pfn, 1 << page_owner->order)) continue; /* |
f2ca0b557 mm/page_owner: us... |
514 515 516 |
* Access to page_ext->handle isn't synchronous so we should * be careful to access it. */ |
9300d8dfd mm/page_owner: do... |
517 |
handle = READ_ONCE(page_owner->handle); |
f2ca0b557 mm/page_owner: us... |
518 519 |
if (!handle) continue; |
48c96a368 mm/page_owner: ke... |
520 521 |
/* Record the next PFN to read in the file offset */ *ppos = (pfn - min_low_pfn) + 1; |
f2ca0b557 mm/page_owner: us... |
522 |
return print_page_owner(buf, count, pfn, page, |
9300d8dfd mm/page_owner: do... |
523 |
page_owner, handle); |
48c96a368 mm/page_owner: ke... |
524 525 526 527 |
} return 0; } |
61cf5febd mm/page_owner: co... |
528 529 |
static void init_pages_in_zone(pg_data_t *pgdat, struct zone *zone) { |
6787c1dab mm/page_owner.c: ... |
530 531 |
unsigned long pfn = zone->zone_start_pfn; unsigned long end_pfn = zone_end_pfn(zone); |
61cf5febd mm/page_owner: co... |
532 |
unsigned long count = 0; |
61cf5febd mm/page_owner: co... |
533 534 535 536 537 538 |
/* * Walk the zone in pageblock_nr_pages steps. If a page block spans * a zone boundary, it will be double counted between zones. This does * not matter as the mixed block count will still be correct */ for (; pfn < end_pfn; ) { |
6787c1dab mm/page_owner.c: ... |
539 |
unsigned long block_end_pfn; |
61cf5febd mm/page_owner: co... |
540 541 542 543 544 545 546 |
if (!pfn_valid(pfn)) { pfn = ALIGN(pfn + 1, MAX_ORDER_NR_PAGES); continue; } block_end_pfn = ALIGN(pfn + 1, pageblock_nr_pages); block_end_pfn = min(block_end_pfn, end_pfn); |
61cf5febd mm/page_owner: co... |
547 |
for (; pfn < block_end_pfn; pfn++) { |
6787c1dab mm/page_owner.c: ... |
548 549 |
struct page *page; struct page_ext *page_ext; |
61cf5febd mm/page_owner: co... |
550 551 552 553 |
if (!pfn_valid_within(pfn)) continue; page = pfn_to_page(pfn); |
9d43f5aec mm/page_owner: ad... |
554 555 |
if (page_zone(page) != zone) continue; |
61cf5febd mm/page_owner: co... |
556 |
/* |
109030279 mm, page_owner: d... |
557 558 559 560 561 |
* To avoid having to grab zone->lock, be a little * careful when reading buddy page order. The only * danger is that we skip too much and potentially miss * some early allocated pages, which is better than * heavy lock contention. |
61cf5febd mm/page_owner: co... |
562 563 |
*/ if (PageBuddy(page)) { |
ab130f910 mm: rename page_o... |
564 |
unsigned long order = buddy_order_unsafe(page); |
109030279 mm, page_owner: d... |
565 566 567 |
if (order > 0 && order < MAX_ORDER) pfn += (1UL << order) - 1; |
61cf5febd mm/page_owner: co... |
568 569 570 571 572 573 574 |
continue; } if (PageReserved(page)) continue; page_ext = lookup_page_ext(page); |
f86e42719 mm: check the ret... |
575 576 |
if (unlikely(!page_ext)) continue; |
61cf5febd mm/page_owner: co... |
577 |
|
dab4ead1a mm, page_owner: m... |
578 |
/* Maybe overlapping zone */ |
61cf5febd mm/page_owner: co... |
579 580 581 582 |
if (test_bit(PAGE_EXT_OWNER, &page_ext->flags)) continue; /* Found early allocated page */ |
7e2f2a0cd mm, page_owner: r... |
583 584 |
__set_page_owner_handle(page, page_ext, early_handle, 0, 0); |
61cf5febd mm/page_owner: co... |
585 586 |
count++; } |
109030279 mm, page_owner: d... |
587 |
cond_resched(); |
61cf5febd mm/page_owner: co... |
588 589 590 591 592 593 594 595 596 597 598 |
} pr_info("Node %d, zone %8s: page owner found early allocated %lu pages ", pgdat->node_id, zone->name, count); } static void init_zones_in_node(pg_data_t *pgdat) { struct zone *zone; struct zone *node_zones = pgdat->node_zones; |
61cf5febd mm/page_owner: co... |
599 600 601 602 |
for (zone = node_zones; zone - node_zones < MAX_NR_ZONES; ++zone) { if (!populated_zone(zone)) continue; |
61cf5febd mm/page_owner: co... |
603 |
init_pages_in_zone(pgdat, zone); |
61cf5febd mm/page_owner: co... |
604 605 606 607 608 609 |
} } static void init_early_allocated_pages(void) { pg_data_t *pgdat; |
61cf5febd mm/page_owner: co... |
610 611 612 |
for_each_online_pgdat(pgdat) init_zones_in_node(pgdat); } |
48c96a368 mm/page_owner: ke... |
613 614 615 616 617 618 |
static const struct file_operations proc_page_owner_operations = { .read = read_page_owner, }; static int __init pageowner_init(void) { |
7dd80b8af mm, page_owner: c... |
619 |
if (!static_branch_unlikely(&page_owner_inited)) { |
48c96a368 mm/page_owner: ke... |
620 621 622 623 |
pr_info("page_owner is disabled "); return 0; } |
d9f7979c9 mm: no need to ch... |
624 625 |
debugfs_create_file("page_owner", 0400, NULL, NULL, &proc_page_owner_operations); |
48c96a368 mm/page_owner: ke... |
626 |
|
d9f7979c9 mm: no need to ch... |
627 |
return 0; |
48c96a368 mm/page_owner: ke... |
628 |
} |
44c5af96d mm/page_owner.c: ... |
629 |
late_initcall(pageowner_init) |