Blame view
mm/compaction.c
34.8 KB
748446bb6
|
1 2 3 4 5 6 7 8 9 10 11 12 13 14 |
/* * linux/mm/compaction.c * * Memory compaction for the reduction of external fragmentation. Note that * this heavily depends upon page migration to do all the real heavy * lifting * * Copyright IBM Corp. 2007-2010 Mel Gorman <mel@csn.ul.ie> */ #include <linux/swap.h> #include <linux/migrate.h> #include <linux/compaction.h> #include <linux/mm_inline.h> #include <linux/backing-dev.h> |
76ab0f530
|
15 |
#include <linux/sysctl.h> |
ed4a6d7f0
|
16 |
#include <linux/sysfs.h> |
bf6bddf19
|
17 |
#include <linux/balloon_compaction.h> |
194159fbc
|
18 |
#include <linux/page-isolation.h> |
748446bb6
|
19 |
#include "internal.h" |
010fc29a4
|
20 21 22 23 24 25 26 27 28 29 30 31 32 33 |
#ifdef CONFIG_COMPACTION static inline void count_compact_event(enum vm_event_item item) { count_vm_event(item); } static inline void count_compact_events(enum vm_event_item item, long delta) { count_vm_events(item, delta); } #else #define count_compact_event(item) do { } while (0) #define count_compact_events(item, delta) do { } while (0) #endif |
ff9543fd3
|
34 |
#if defined CONFIG_COMPACTION || defined CONFIG_CMA |
b7aba6984
|
35 36 |
#define CREATE_TRACE_POINTS #include <trace/events/compaction.h> |
748446bb6
|
37 38 39 40 41 42 43 44 45 46 47 48 49 |
static unsigned long release_freepages(struct list_head *freelist) { struct page *page, *next; unsigned long count = 0; list_for_each_entry_safe(page, next, freelist, lru) { list_del(&page->lru); __free_page(page); count++; } return count; } |
ff9543fd3
|
50 51 52 53 54 55 56 57 58 |
static void map_pages(struct list_head *list) { struct page *page; list_for_each_entry(page, list, lru) { arch_alloc_page(page, 0); kernel_map_pages(page, 1, 1); } } |
47118af07
|
59 60 61 62 |
static inline bool migrate_async_suitable(int migratetype) { return is_migrate_cma(migratetype) || migratetype == MIGRATE_MOVABLE; } |
bb13ffeb9
|
63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 |
#ifdef CONFIG_COMPACTION /* Returns true if the pageblock should be scanned for pages to isolate. */ static inline bool isolation_suitable(struct compact_control *cc, struct page *page) { if (cc->ignore_skip_hint) return true; return !get_pageblock_skip(page); } /* * This function is called to clear all cached information on pageblocks that * should be skipped for page isolation when the migrate and free page scanner * meet. */ |
62997027c
|
79 |
static void __reset_isolation_suitable(struct zone *zone) |
bb13ffeb9
|
80 81 |
{ unsigned long start_pfn = zone->zone_start_pfn; |
108bcc96e
|
82 |
unsigned long end_pfn = zone_end_pfn(zone); |
bb13ffeb9
|
83 |
unsigned long pfn; |
35979ef33
|
84 85 |
zone->compact_cached_migrate_pfn[0] = start_pfn; zone->compact_cached_migrate_pfn[1] = start_pfn; |
c89511ab2
|
86 |
zone->compact_cached_free_pfn = end_pfn; |
62997027c
|
87 |
zone->compact_blockskip_flush = false; |
bb13ffeb9
|
88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 |
/* Walk the zone and mark every pageblock as suitable for isolation */ for (pfn = start_pfn; pfn < end_pfn; pfn += pageblock_nr_pages) { struct page *page; cond_resched(); if (!pfn_valid(pfn)) continue; page = pfn_to_page(pfn); if (zone != page_zone(page)) continue; clear_pageblock_skip(page); } } |
62997027c
|
105 106 107 108 109 110 111 112 113 114 115 116 117 118 |
void reset_isolation_suitable(pg_data_t *pgdat) { int zoneid; for (zoneid = 0; zoneid < MAX_NR_ZONES; zoneid++) { struct zone *zone = &pgdat->node_zones[zoneid]; if (!populated_zone(zone)) continue; /* Only flush if a full compaction finished recently */ if (zone->compact_blockskip_flush) __reset_isolation_suitable(zone); } } |
bb13ffeb9
|
119 120 |
/* * If no pages were isolated then mark this pageblock to be skipped in the |
62997027c
|
121 |
* future. The information is later cleared by __reset_isolation_suitable(). |
bb13ffeb9
|
122 |
*/ |
c89511ab2
|
123 124 |
static void update_pageblock_skip(struct compact_control *cc, struct page *page, unsigned long nr_isolated, |
35979ef33
|
125 |
bool set_unsuitable, bool migrate_scanner) |
bb13ffeb9
|
126 |
{ |
c89511ab2
|
127 |
struct zone *zone = cc->zone; |
35979ef33
|
128 |
unsigned long pfn; |
6815bf3f2
|
129 130 131 |
if (cc->ignore_skip_hint) return; |
bb13ffeb9
|
132 133 |
if (!page) return; |
35979ef33
|
134 135 136 137 138 139 140 141 |
if (nr_isolated) return; /* * Only skip pageblocks when all forms of compaction will be known to * fail in the near future. */ if (set_unsuitable) |
bb13ffeb9
|
142 |
set_pageblock_skip(page); |
c89511ab2
|
143 |
|
35979ef33
|
144 145 146 147 148 149 150 151 |
pfn = page_to_pfn(page); /* Update where async and sync compaction should restart */ if (migrate_scanner) { if (cc->finished_update_migrate) return; if (pfn > zone->compact_cached_migrate_pfn[0]) zone->compact_cached_migrate_pfn[0] = pfn; |
e0b9daeb4
|
152 153 |
if (cc->mode != MIGRATE_ASYNC && pfn > zone->compact_cached_migrate_pfn[1]) |
35979ef33
|
154 155 156 157 158 159 |
zone->compact_cached_migrate_pfn[1] = pfn; } else { if (cc->finished_update_free) return; if (pfn < zone->compact_cached_free_pfn) zone->compact_cached_free_pfn = pfn; |
c89511ab2
|
160 |
} |
bb13ffeb9
|
161 162 163 164 165 166 167 |
} #else static inline bool isolation_suitable(struct compact_control *cc, struct page *page) { return true; } |
c89511ab2
|
168 169 |
static void update_pageblock_skip(struct compact_control *cc, struct page *page, unsigned long nr_isolated, |
35979ef33
|
170 |
bool set_unsuitable, bool migrate_scanner) |
bb13ffeb9
|
171 172 173 |
{ } #endif /* CONFIG_COMPACTION */ |
2a1402aa0
|
174 175 176 177 |
static inline bool should_release_lock(spinlock_t *lock) { return need_resched() || spin_is_contended(lock); } |
85aa125f0
|
178 |
/* |
c67fe3752
|
179 180 181 182 183 184 185 186 187 188 189 |
* Compaction requires the taking of some coarse locks that are potentially * very heavily contended. Check if the process needs to be scheduled or * if the lock is contended. For async compaction, back out in the event * if contention is severe. For sync compaction, schedule. * * Returns true if the lock is held. * Returns false if the lock is released and compaction should abort */ static bool compact_checklock_irqsave(spinlock_t *lock, unsigned long *flags, bool locked, struct compact_control *cc) { |
2a1402aa0
|
190 |
if (should_release_lock(lock)) { |
c67fe3752
|
191 192 193 194 195 196 |
if (locked) { spin_unlock_irqrestore(lock, *flags); locked = false; } /* async aborts if taking too long or contended */ |
e0b9daeb4
|
197 |
if (cc->mode == MIGRATE_ASYNC) { |
e64c5237c
|
198 |
cc->contended = true; |
c67fe3752
|
199 200 201 202 |
return false; } cond_resched(); |
c67fe3752
|
203 204 205 206 207 208 |
} if (!locked) spin_lock_irqsave(lock, *flags); return true; } |
be9765722
|
209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 |
/* * Aside from avoiding lock contention, compaction also periodically checks * need_resched() and either schedules in sync compaction or aborts async * compaction. This is similar to what compact_checklock_irqsave() does, but * is used where no lock is concerned. * * Returns false when no scheduling was needed, or sync compaction scheduled. * Returns true when async compaction should abort. */ static inline bool compact_should_abort(struct compact_control *cc) { /* async compaction aborts if contended */ if (need_resched()) { if (cc->mode == MIGRATE_ASYNC) { cc->contended = true; return true; } cond_resched(); } return false; } |
f40d1e42b
|
232 233 234 |
/* Returns true if the page is within a block suitable for migration to */ static bool suitable_migration_target(struct page *page) { |
7d348b9ea
|
235 |
/* If the page is a large free page, then disallow migration */ |
f40d1e42b
|
236 |
if (PageBuddy(page) && page_order(page) >= pageblock_order) |
7d348b9ea
|
237 |
return false; |
f40d1e42b
|
238 239 |
/* If the block is MIGRATE_MOVABLE or MIGRATE_CMA, allow migration */ |
7d348b9ea
|
240 |
if (migrate_async_suitable(get_pageblock_migratetype(page))) |
f40d1e42b
|
241 242 243 244 245 |
return true; /* Otherwise skip the block */ return false; } |
c67fe3752
|
246 |
/* |
9e4be4708
|
247 248 249 |
* Isolate free pages onto a private freelist. If @strict is true, will abort * returning 0 on any invalid PFNs or non-free pages inside of the pageblock * (even though it may still end up isolating some pages). |
85aa125f0
|
250 |
*/ |
f40d1e42b
|
251 252 |
static unsigned long isolate_freepages_block(struct compact_control *cc, unsigned long blockpfn, |
85aa125f0
|
253 254 255 |
unsigned long end_pfn, struct list_head *freelist, bool strict) |
748446bb6
|
256 |
{ |
b7aba6984
|
257 |
int nr_scanned = 0, total_isolated = 0; |
bb13ffeb9
|
258 |
struct page *cursor, *valid_page = NULL; |
f40d1e42b
|
259 260 |
unsigned long flags; bool locked = false; |
01ead5340
|
261 |
bool checked_pageblock = false; |
748446bb6
|
262 |
|
748446bb6
|
263 |
cursor = pfn_to_page(blockpfn); |
f40d1e42b
|
264 |
/* Isolate free pages. */ |
748446bb6
|
265 266 267 |
for (; blockpfn < end_pfn; blockpfn++, cursor++) { int isolated, i; struct page *page = cursor; |
b7aba6984
|
268 |
nr_scanned++; |
f40d1e42b
|
269 |
if (!pfn_valid_within(blockpfn)) |
2af120bc0
|
270 |
goto isolate_fail; |
bb13ffeb9
|
271 272 |
if (!valid_page) valid_page = page; |
f40d1e42b
|
273 |
if (!PageBuddy(page)) |
2af120bc0
|
274 |
goto isolate_fail; |
f40d1e42b
|
275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 |
/* * The zone lock must be held to isolate freepages. * Unfortunately this is a very coarse lock and can be * heavily contended if there are parallel allocations * or parallel compactions. For async compaction do not * spin on the lock and we acquire the lock as late as * possible. */ locked = compact_checklock_irqsave(&cc->zone->lock, &flags, locked, cc); if (!locked) break; /* Recheck this is a suitable migration target under lock */ |
01ead5340
|
290 291 292 293 294 295 296 297 298 299 |
if (!strict && !checked_pageblock) { /* * We need to check suitability of pageblock only once * and this isolate_freepages_block() is called with * pageblock range, so just check once is sufficient. */ checked_pageblock = true; if (!suitable_migration_target(page)) break; } |
748446bb6
|
300 |
|
f40d1e42b
|
301 302 |
/* Recheck this is a buddy page under lock */ if (!PageBuddy(page)) |
2af120bc0
|
303 |
goto isolate_fail; |
748446bb6
|
304 305 306 307 308 309 310 311 312 313 314 315 316 |
/* Found a free page, break it into order-0 pages */ isolated = split_free_page(page); total_isolated += isolated; for (i = 0; i < isolated; i++) { list_add(&page->lru, freelist); page++; } /* If a page was split, advance to the end of it */ if (isolated) { blockpfn += isolated - 1; cursor += isolated - 1; |
2af120bc0
|
317 |
continue; |
748446bb6
|
318 |
} |
2af120bc0
|
319 320 321 322 323 324 |
isolate_fail: if (strict) break; else continue; |
748446bb6
|
325 |
} |
b7aba6984
|
326 |
trace_mm_compaction_isolate_freepages(nr_scanned, total_isolated); |
f40d1e42b
|
327 328 329 330 331 332 |
/* * If strict isolation is requested by CMA then check that all the * pages requested were isolated. If there were any failures, 0 is * returned and CMA will fail. */ |
2af120bc0
|
333 |
if (strict && blockpfn < end_pfn) |
f40d1e42b
|
334 335 336 337 |
total_isolated = 0; if (locked) spin_unlock_irqrestore(&cc->zone->lock, flags); |
bb13ffeb9
|
338 339 |
/* Update the pageblock-skip if the whole pageblock was scanned */ if (blockpfn == end_pfn) |
35979ef33
|
340 341 |
update_pageblock_skip(cc, valid_page, total_isolated, true, false); |
bb13ffeb9
|
342 |
|
010fc29a4
|
343 |
count_compact_events(COMPACTFREE_SCANNED, nr_scanned); |
397487db6
|
344 |
if (total_isolated) |
010fc29a4
|
345 |
count_compact_events(COMPACTISOLATED, total_isolated); |
748446bb6
|
346 347 |
return total_isolated; } |
85aa125f0
|
348 349 350 351 352 353 354 355 356 357 358 359 360 |
/** * isolate_freepages_range() - isolate free pages. * @start_pfn: The first PFN to start isolating. * @end_pfn: The one-past-last PFN. * * Non-free pages, invalid PFNs, or zone boundaries within the * [start_pfn, end_pfn) range are considered errors, cause function to * undo its actions and return zero. * * Otherwise, function returns one-past-the-last PFN of isolated page * (which may be greater then end_pfn if end fell in a middle of * a free page). */ |
ff9543fd3
|
361 |
unsigned long |
bb13ffeb9
|
362 363 |
isolate_freepages_range(struct compact_control *cc, unsigned long start_pfn, unsigned long end_pfn) |
85aa125f0
|
364 |
{ |
f40d1e42b
|
365 |
unsigned long isolated, pfn, block_end_pfn; |
85aa125f0
|
366 |
LIST_HEAD(freelist); |
85aa125f0
|
367 |
for (pfn = start_pfn; pfn < end_pfn; pfn += isolated) { |
bb13ffeb9
|
368 |
if (!pfn_valid(pfn) || cc->zone != page_zone(pfn_to_page(pfn))) |
85aa125f0
|
369 370 371 372 373 374 375 376 |
break; /* * On subsequent iterations ALIGN() is actually not needed, * but we keep it that we not to complicate the code. */ block_end_pfn = ALIGN(pfn + 1, pageblock_nr_pages); block_end_pfn = min(block_end_pfn, end_pfn); |
bb13ffeb9
|
377 |
isolated = isolate_freepages_block(cc, pfn, block_end_pfn, |
85aa125f0
|
378 |
&freelist, true); |
85aa125f0
|
379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 |
/* * In strict mode, isolate_freepages_block() returns 0 if * there are any holes in the block (ie. invalid PFNs or * non-free pages). */ if (!isolated) break; /* * If we managed to isolate pages, it is always (1 << n) * * pageblock_nr_pages for some non-negative n. (Max order * page may span two pageblocks). */ } /* split_free_page does not map the pages */ map_pages(&freelist); if (pfn < end_pfn) { /* Loop terminated early, cleanup. */ release_freepages(&freelist); return 0; } /* We don't use freelists for anything. */ return pfn; } |
748446bb6
|
407 |
/* Update the number of anon and file isolated pages in the zone */ |
c67fe3752
|
408 |
static void acct_isolated(struct zone *zone, bool locked, struct compact_control *cc) |
748446bb6
|
409 410 |
{ struct page *page; |
b9e84ac15
|
411 |
unsigned int count[2] = { 0, }; |
748446bb6
|
412 |
|
b9e84ac15
|
413 414 |
list_for_each_entry(page, &cc->migratepages, lru) count[!!page_is_file_cache(page)]++; |
748446bb6
|
415 |
|
c67fe3752
|
416 417 418 419 420 421 422 423 |
/* If locked we can use the interrupt unsafe versions */ if (locked) { __mod_zone_page_state(zone, NR_ISOLATED_ANON, count[0]); __mod_zone_page_state(zone, NR_ISOLATED_FILE, count[1]); } else { mod_zone_page_state(zone, NR_ISOLATED_ANON, count[0]); mod_zone_page_state(zone, NR_ISOLATED_FILE, count[1]); } |
748446bb6
|
424 425 426 427 428 |
} /* Similar to reclaim, but different enough that they don't share logic */ static bool too_many_isolated(struct zone *zone) { |
bc6930457
|
429 |
unsigned long active, inactive, isolated; |
748446bb6
|
430 431 432 |
inactive = zone_page_state(zone, NR_INACTIVE_FILE) + zone_page_state(zone, NR_INACTIVE_ANON); |
bc6930457
|
433 434 |
active = zone_page_state(zone, NR_ACTIVE_FILE) + zone_page_state(zone, NR_ACTIVE_ANON); |
748446bb6
|
435 436 |
isolated = zone_page_state(zone, NR_ISOLATED_FILE) + zone_page_state(zone, NR_ISOLATED_ANON); |
bc6930457
|
437 |
return isolated > (inactive + active) / 2; |
748446bb6
|
438 |
} |
2fe86e000
|
439 440 441 442 443 444 |
/** * isolate_migratepages_range() - isolate all migrate-able pages in range. * @zone: Zone pages are in. * @cc: Compaction control structure. * @low_pfn: The first PFN of the range. * @end_pfn: The one-past-the-last PFN of the range. |
e46a28790
|
445 |
* @unevictable: true if it allows to isolate unevictable pages |
2fe86e000
|
446 447 448 449 450 451 452 453 454 455 456 457 |
* * Isolate all pages that can be migrated from the range specified by * [low_pfn, end_pfn). Returns zero if there is a fatal signal * pending), otherwise PFN of the first page that was not scanned * (which may be both less, equal to or more then end_pfn). * * Assumes that cc->migratepages is empty and cc->nr_migratepages is * zero. * * Apart from cc->migratepages and cc->nr_migratetypes this function * does not modify any cc's fields, in particular it does not modify * (or read for that matter) cc->migrate_pfn. |
748446bb6
|
458 |
*/ |
ff9543fd3
|
459 |
unsigned long |
2fe86e000
|
460 |
isolate_migratepages_range(struct zone *zone, struct compact_control *cc, |
e46a28790
|
461 |
unsigned long low_pfn, unsigned long end_pfn, bool unevictable) |
748446bb6
|
462 |
{ |
9927af740
|
463 |
unsigned long last_pageblock_nr = 0, pageblock_nr; |
b7aba6984
|
464 |
unsigned long nr_scanned = 0, nr_isolated = 0; |
748446bb6
|
465 |
struct list_head *migratelist = &cc->migratepages; |
fa9add641
|
466 |
struct lruvec *lruvec; |
c67fe3752
|
467 |
unsigned long flags; |
2a1402aa0
|
468 |
bool locked = false; |
bb13ffeb9
|
469 |
struct page *page = NULL, *valid_page = NULL; |
35979ef33
|
470 |
bool set_unsuitable = true; |
e0b9daeb4
|
471 472 |
const isolate_mode_t mode = (cc->mode == MIGRATE_ASYNC ? ISOLATE_ASYNC_MIGRATE : 0) | |
da1c67a76
|
473 |
(unevictable ? ISOLATE_UNEVICTABLE : 0); |
748446bb6
|
474 |
|
748446bb6
|
475 476 477 478 479 480 |
/* * Ensure that there are not too many pages isolated from the LRU * list by either parallel reclaimers or compaction. If there are, * delay for some time until fewer pages are isolated */ while (unlikely(too_many_isolated(zone))) { |
f9e35b3b4
|
481 |
/* async migration should just abort */ |
e0b9daeb4
|
482 |
if (cc->mode == MIGRATE_ASYNC) |
2fe86e000
|
483 |
return 0; |
f9e35b3b4
|
484 |
|
748446bb6
|
485 486 487 |
congestion_wait(BLK_RW_ASYNC, HZ/10); if (fatal_signal_pending(current)) |
2fe86e000
|
488 |
return 0; |
748446bb6
|
489 |
} |
be9765722
|
490 491 |
if (compact_should_abort(cc)) return 0; |
aeef4b838
|
492 |
|
748446bb6
|
493 |
/* Time to isolate some pages for migration */ |
748446bb6
|
494 |
for (; low_pfn < end_pfn; low_pfn++) { |
b2eef8c0d
|
495 |
/* give a chance to irqs before checking need_resched() */ |
be1aa03b9
|
496 |
if (locked && !(low_pfn % SWAP_CLUSTER_MAX)) { |
2a1402aa0
|
497 498 499 500 |
if (should_release_lock(&zone->lru_lock)) { spin_unlock_irqrestore(&zone->lru_lock, flags); locked = false; } |
b2eef8c0d
|
501 |
} |
c67fe3752
|
502 |
|
0bf380bc7
|
503 504 505 506 507 508 509 510 511 512 513 514 |
/* * migrate_pfn does not necessarily start aligned to a * pageblock. Ensure that pfn_valid is called when moving * into a new MAX_ORDER_NR_PAGES range in case of large * memory holes within the zone */ if ((low_pfn & (MAX_ORDER_NR_PAGES - 1)) == 0) { if (!pfn_valid(low_pfn)) { low_pfn += MAX_ORDER_NR_PAGES - 1; continue; } } |
748446bb6
|
515 516 |
if (!pfn_valid_within(low_pfn)) continue; |
b7aba6984
|
517 |
nr_scanned++; |
748446bb6
|
518 |
|
dc9086004
|
519 520 521 522 523 524 |
/* * Get the page and ensure the page is within the same zone. * See the comment in isolate_freepages about overlapping * nodes. It is deliberate that the new zone lock is not taken * as memory compaction should not move pages between nodes. */ |
748446bb6
|
525 |
page = pfn_to_page(low_pfn); |
dc9086004
|
526 527 |
if (page_zone(page) != zone) continue; |
bb13ffeb9
|
528 529 530 531 532 |
if (!valid_page) valid_page = page; /* If isolation recently failed, do not retry */ pageblock_nr = low_pfn >> pageblock_order; |
c122b2087
|
533 534 535 536 537 538 539 540 541 542 543 544 545 |
if (last_pageblock_nr != pageblock_nr) { int mt; last_pageblock_nr = pageblock_nr; if (!isolation_suitable(cc, page)) goto next_pageblock; /* * For async migration, also only scan in MOVABLE * blocks. Async migration is optimistic to see if * the minimum amount of work satisfies the allocation */ mt = get_pageblock_migratetype(page); |
e0b9daeb4
|
546 547 |
if (cc->mode == MIGRATE_ASYNC && !migrate_async_suitable(mt)) { |
35979ef33
|
548 |
set_unsuitable = false; |
c122b2087
|
549 550 551 |
goto next_pageblock; } } |
bb13ffeb9
|
552 |
|
6c14466cc
|
553 554 555 556 |
/* * Skip if free. page_order cannot be used without zone->lock * as nothing prevents parallel allocations or buddy merging. */ |
748446bb6
|
557 558 |
if (PageBuddy(page)) continue; |
9927af740
|
559 |
/* |
bf6bddf19
|
560 561 562 563 564 565 566 567 |
* Check may be lockless but that's ok as we recheck later. * It's possible to migrate LRU pages and balloon pages * Skip any other type of page */ if (!PageLRU(page)) { if (unlikely(balloon_page_movable(page))) { if (locked && balloon_page_isolate(page)) { /* Successfully isolated */ |
b6c750163
|
568 |
goto isolate_success; |
bf6bddf19
|
569 570 |
} } |
bc835011a
|
571 |
continue; |
bf6bddf19
|
572 |
} |
bc835011a
|
573 574 |
/* |
2a1402aa0
|
575 576 577 578 579 580 581 582 |
* PageLRU is set. lru_lock normally excludes isolation * splitting and collapsing (collapsing has already happened * if PageLRU is set) but the lock is not necessarily taken * here and it is wasteful to take it just to check transhuge. * Check TransHuge without lock and skip the whole pageblock if * it's either a transhuge or hugetlbfs page, as calling * compound_order() without preventing THP from splitting the * page underneath us may return surprising results. |
bc835011a
|
583 584 |
*/ if (PageTransHuge(page)) { |
2a1402aa0
|
585 586 587 588 589 |
if (!locked) goto next_pageblock; low_pfn += (1 << compound_order(page)) - 1; continue; } |
119d6d59d
|
590 591 592 593 594 595 596 597 |
/* * Migration will fail if an anonymous page is pinned in memory, * so avoid taking lru_lock and isolating it unnecessarily in an * admittedly racy check. */ if (!page_mapping(page) && page_count(page) > page_mapcount(page)) continue; |
2a1402aa0
|
598 599 600 601 602 603 604 605 606 607 |
/* Check if it is ok to still hold the lock */ locked = compact_checklock_irqsave(&zone->lru_lock, &flags, locked, cc); if (!locked || fatal_signal_pending(current)) break; /* Recheck PageLRU and PageTransHuge under lock */ if (!PageLRU(page)) continue; if (PageTransHuge(page)) { |
bc835011a
|
608 609 610 |
low_pfn += (1 << compound_order(page)) - 1; continue; } |
fa9add641
|
611 |
lruvec = mem_cgroup_page_lruvec(page, zone); |
748446bb6
|
612 |
/* Try isolate the page */ |
f3fd4a619
|
613 |
if (__isolate_lru_page(page, mode) != 0) |
748446bb6
|
614 |
continue; |
309381fea
|
615 |
VM_BUG_ON_PAGE(PageTransCompound(page), page); |
bc835011a
|
616 |
|
748446bb6
|
617 |
/* Successfully isolated */ |
fa9add641
|
618 |
del_page_from_lru_list(page, lruvec, page_lru(page)); |
b6c750163
|
619 620 621 |
isolate_success: cc->finished_update_migrate = true; |
748446bb6
|
622 |
list_add(&page->lru, migratelist); |
748446bb6
|
623 |
cc->nr_migratepages++; |
b7aba6984
|
624 |
nr_isolated++; |
748446bb6
|
625 626 |
/* Avoid isolating too much */ |
31b8384a5
|
627 628 |
if (cc->nr_migratepages == COMPACT_CLUSTER_MAX) { ++low_pfn; |
748446bb6
|
629 |
break; |
31b8384a5
|
630 |
} |
2a1402aa0
|
631 632 633 634 |
continue; next_pageblock: |
a9aacbccf
|
635 |
low_pfn = ALIGN(low_pfn + 1, pageblock_nr_pages) - 1; |
748446bb6
|
636 |
} |
c67fe3752
|
637 |
acct_isolated(zone, locked, cc); |
748446bb6
|
638 |
|
c67fe3752
|
639 640 |
if (locked) spin_unlock_irqrestore(&zone->lru_lock, flags); |
748446bb6
|
641 |
|
50b5b094e
|
642 643 644 |
/* * Update the pageblock-skip information and cached scanner pfn, * if the whole pageblock was scanned without isolating any page. |
50b5b094e
|
645 |
*/ |
35979ef33
|
646 647 648 |
if (low_pfn == end_pfn) update_pageblock_skip(cc, valid_page, nr_isolated, set_unsuitable, true); |
bb13ffeb9
|
649 |
|
b7aba6984
|
650 |
trace_mm_compaction_isolate_migratepages(nr_scanned, nr_isolated); |
010fc29a4
|
651 |
count_compact_events(COMPACTMIGRATE_SCANNED, nr_scanned); |
397487db6
|
652 |
if (nr_isolated) |
010fc29a4
|
653 |
count_compact_events(COMPACTISOLATED, nr_isolated); |
397487db6
|
654 |
|
2fe86e000
|
655 656 |
return low_pfn; } |
ff9543fd3
|
657 658 |
#endif /* CONFIG_COMPACTION || CONFIG_CMA */ #ifdef CONFIG_COMPACTION |
2fe86e000
|
659 |
/* |
ff9543fd3
|
660 661 |
* Based on information in the current compact_control, find blocks * suitable for isolating free pages from and then isolate them. |
2fe86e000
|
662 |
*/ |
ff9543fd3
|
663 664 |
static void isolate_freepages(struct zone *zone, struct compact_control *cc) |
2fe86e000
|
665 |
{ |
ff9543fd3
|
666 |
struct page *page; |
c96b9e508
|
667 668 669 |
unsigned long block_start_pfn; /* start of current pageblock */ unsigned long block_end_pfn; /* end of current pageblock */ unsigned long low_pfn; /* lowest pfn scanner is able to scan */ |
ff9543fd3
|
670 671 |
int nr_freepages = cc->nr_freepages; struct list_head *freelist = &cc->freepages; |
2fe86e000
|
672 |
|
ff9543fd3
|
673 674 |
/* * Initialise the free scanner. The starting point is where we last |
49e068f0b
|
675 676 |
* successfully isolated from, zone-cached value, or the end of the * zone when isolating for the first time. We need this aligned to |
c96b9e508
|
677 678 679 680 |
* the pageblock boundary, because we do * block_start_pfn -= pageblock_nr_pages in the for loop. * For ending point, take care when isolating in last pageblock of a * a zone which ends in the middle of a pageblock. |
49e068f0b
|
681 682 |
* The low boundary is the end of the pageblock the migration scanner * is using. |
ff9543fd3
|
683 |
*/ |
c96b9e508
|
684 685 686 |
block_start_pfn = cc->free_pfn & ~(pageblock_nr_pages-1); block_end_pfn = min(block_start_pfn + pageblock_nr_pages, zone_end_pfn(zone)); |
7ed695e06
|
687 |
low_pfn = ALIGN(cc->migrate_pfn + 1, pageblock_nr_pages); |
2fe86e000
|
688 |
|
ff9543fd3
|
689 |
/* |
ff9543fd3
|
690 691 692 693 |
* Isolate free pages until enough are available to migrate the * pages on cc->migratepages. We stop searching if the migrate * and free page scanners meet or enough free pages are isolated. */ |
c96b9e508
|
694 695 696 |
for (; block_start_pfn >= low_pfn && cc->nr_migratepages > nr_freepages; block_end_pfn = block_start_pfn, block_start_pfn -= pageblock_nr_pages) { |
ff9543fd3
|
697 |
unsigned long isolated; |
2fe86e000
|
698 |
|
f6ea3adb7
|
699 700 701 |
/* * This can iterate a massively long zone without finding any * suitable migration targets, so periodically check if we need |
be9765722
|
702 |
* to schedule, or even abort async compaction. |
f6ea3adb7
|
703 |
*/ |
be9765722
|
704 705 706 |
if (!(block_start_pfn % (SWAP_CLUSTER_MAX * pageblock_nr_pages)) && compact_should_abort(cc)) break; |
f6ea3adb7
|
707 |
|
c96b9e508
|
708 |
if (!pfn_valid(block_start_pfn)) |
ff9543fd3
|
709 |
continue; |
2fe86e000
|
710 |
|
ff9543fd3
|
711 712 713 714 715 716 717 |
/* * Check for overlapping nodes/zones. It's possible on some * configurations to have a setup like * node0 node1 node0 * i.e. it's possible that all pages within a zones range of * pages do not belong to a single zone. */ |
c96b9e508
|
718 |
page = pfn_to_page(block_start_pfn); |
ff9543fd3
|
719 720 721 722 |
if (page_zone(page) != zone) continue; /* Check the block is suitable for migration */ |
68e3e9262
|
723 |
if (!suitable_migration_target(page)) |
ff9543fd3
|
724 |
continue; |
68e3e9262
|
725 |
|
bb13ffeb9
|
726 727 728 |
/* If isolation recently failed, do not retry */ if (!isolation_suitable(cc, page)) continue; |
f40d1e42b
|
729 |
/* Found a block suitable for isolating free pages from */ |
e9ade5699
|
730 |
cc->free_pfn = block_start_pfn; |
c96b9e508
|
731 732 |
isolated = isolate_freepages_block(cc, block_start_pfn, block_end_pfn, freelist, false); |
f40d1e42b
|
733 |
nr_freepages += isolated; |
ff9543fd3
|
734 735 |
/* |
e9ade5699
|
736 737 738 739 |
* Set a flag that we successfully isolated in this pageblock. * In the next loop iteration, zone->compact_cached_free_pfn * will not be updated and thus it will effectively contain the * highest pageblock we isolated pages from. |
ff9543fd3
|
740 |
*/ |
e9ade5699
|
741 |
if (isolated) |
c89511ab2
|
742 |
cc->finished_update_free = true; |
be9765722
|
743 744 745 746 747 748 749 |
/* * isolate_freepages_block() might have aborted due to async * compaction being contended */ if (cc->contended) break; |
ff9543fd3
|
750 751 752 753 |
} /* split_free_page does not map the pages */ map_pages(freelist); |
7ed695e06
|
754 755 756 757 |
/* * If we crossed the migrate scanner, we want to keep it that way * so that compact_finished() may detect this */ |
c96b9e508
|
758 |
if (block_start_pfn < low_pfn) |
e9ade5699
|
759 |
cc->free_pfn = cc->migrate_pfn; |
c96b9e508
|
760 |
|
ff9543fd3
|
761 |
cc->nr_freepages = nr_freepages; |
748446bb6
|
762 763 764 765 766 767 768 769 770 771 772 773 |
} /* * This is a migrate-callback that "allocates" freepages by taking pages * from the isolated freelists in the block we are migrating to. */ static struct page *compaction_alloc(struct page *migratepage, unsigned long data, int **result) { struct compact_control *cc = (struct compact_control *)data; struct page *freepage; |
be9765722
|
774 775 776 777 |
/* * Isolate free pages if necessary, and if we are not aborting due to * contention. */ |
748446bb6
|
778 |
if (list_empty(&cc->freepages)) { |
be9765722
|
779 780 |
if (!cc->contended) isolate_freepages(cc->zone, cc); |
748446bb6
|
781 782 783 784 785 786 787 788 789 790 791 792 793 |
if (list_empty(&cc->freepages)) return NULL; } freepage = list_entry(cc->freepages.next, struct page, lru); list_del(&freepage->lru); cc->nr_freepages--; return freepage; } /* |
d53aea3d4
|
794 795 796 797 798 799 800 801 802 803 804 |
* This is a migrate-callback that "frees" freepages back to the isolated * freelist. All pages on the freelist are from the same zone, so there is no * special handling needed for NUMA. */ static void compaction_free(struct page *page, unsigned long data) { struct compact_control *cc = (struct compact_control *)data; list_add(&page->lru, &cc->freepages); cc->nr_freepages++; } |
ff9543fd3
|
805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 |
/* possible outcome of isolate_migratepages */ typedef enum { ISOLATE_ABORT, /* Abort compaction now */ ISOLATE_NONE, /* No pages isolated, continue scanning */ ISOLATE_SUCCESS, /* Pages isolated, migrate */ } isolate_migrate_t; /* * Isolate all pages that can be migrated from the block pointed to by * the migrate scanner within compact_control. */ static isolate_migrate_t isolate_migratepages(struct zone *zone, struct compact_control *cc) { unsigned long low_pfn, end_pfn; /* Do not scan outside zone boundaries */ low_pfn = max(cc->migrate_pfn, zone->zone_start_pfn); /* Only scan within a pageblock boundary */ |
a9aacbccf
|
825 |
end_pfn = ALIGN(low_pfn + 1, pageblock_nr_pages); |
ff9543fd3
|
826 827 828 829 830 831 832 833 |
/* Do not cross the free scanner or scan within a memory hole */ if (end_pfn > cc->free_pfn || !pfn_valid(low_pfn)) { cc->migrate_pfn = end_pfn; return ISOLATE_NONE; } /* Perform the isolation */ |
e46a28790
|
834 |
low_pfn = isolate_migratepages_range(zone, cc, low_pfn, end_pfn, false); |
e64c5237c
|
835 |
if (!low_pfn || cc->contended) |
ff9543fd3
|
836 837 838 839 840 841 |
return ISOLATE_ABORT; cc->migrate_pfn = low_pfn; return ISOLATE_SUCCESS; } |
748446bb6
|
842 |
static int compact_finished(struct zone *zone, |
5a03b051e
|
843 |
struct compact_control *cc) |
748446bb6
|
844 |
{ |
8fb74b9fb
|
845 |
unsigned int order; |
5a03b051e
|
846 |
unsigned long watermark; |
56de7263f
|
847 |
|
be9765722
|
848 |
if (cc->contended || fatal_signal_pending(current)) |
748446bb6
|
849 |
return COMPACT_PARTIAL; |
753341a4b
|
850 |
/* Compaction run completes if the migrate and free scanner meet */ |
bb13ffeb9
|
851 |
if (cc->free_pfn <= cc->migrate_pfn) { |
55b7c4c99
|
852 |
/* Let the next compaction start anew. */ |
35979ef33
|
853 854 |
zone->compact_cached_migrate_pfn[0] = zone->zone_start_pfn; zone->compact_cached_migrate_pfn[1] = zone->zone_start_pfn; |
55b7c4c99
|
855 |
zone->compact_cached_free_pfn = zone_end_pfn(zone); |
62997027c
|
856 857 858 859 860 861 862 863 |
/* * Mark that the PG_migrate_skip information should be cleared * by kswapd when it goes to sleep. kswapd does not set the * flag itself as the decision to be clear should be directly * based on an allocation request. */ if (!current_is_kswapd()) zone->compact_blockskip_flush = true; |
748446bb6
|
864 |
return COMPACT_COMPLETE; |
bb13ffeb9
|
865 |
} |
748446bb6
|
866 |
|
82478fb7b
|
867 868 869 870 |
/* * order == -1 is expected when compacting via * /proc/sys/vm/compact_memory */ |
56de7263f
|
871 872 |
if (cc->order == -1) return COMPACT_CONTINUE; |
3957c7768
|
873 874 875 876 877 878 |
/* Compaction run is not finished if the watermark is not met */ watermark = low_wmark_pages(zone); watermark += (1 << cc->order); if (!zone_watermark_ok(zone, cc->order, watermark, 0, 0)) return COMPACT_CONTINUE; |
56de7263f
|
879 |
/* Direct compactor: Is a suitable page free? */ |
8fb74b9fb
|
880 881 882 883 884 885 886 887 888 |
for (order = cc->order; order < MAX_ORDER; order++) { struct free_area *area = &zone->free_area[order]; /* Job done if page is free of the right migratetype */ if (!list_empty(&area->free_list[cc->migratetype])) return COMPACT_PARTIAL; /* Job done if allocation would set block type */ if (cc->order >= pageblock_order && area->nr_free) |
56de7263f
|
889 890 |
return COMPACT_PARTIAL; } |
748446bb6
|
891 892 |
return COMPACT_CONTINUE; } |
3e7d34497
|
893 894 895 896 897 898 899 900 901 902 903 904 905 |
/* * compaction_suitable: Is this suitable to run compaction on this zone now? * Returns * COMPACT_SKIPPED - If there are too few free pages for compaction * COMPACT_PARTIAL - If the allocation would succeed without compaction * COMPACT_CONTINUE - If compaction should run now */ unsigned long compaction_suitable(struct zone *zone, int order) { int fragindex; unsigned long watermark; /* |
3957c7768
|
906 907 908 909 910 911 912 |
* order == -1 is expected when compacting via * /proc/sys/vm/compact_memory */ if (order == -1) return COMPACT_CONTINUE; /* |
3e7d34497
|
913 914 915 916 917 918 919 920 921 922 923 924 |
* Watermarks for order-0 must be met for compaction. Note the 2UL. * This is because during migration, copies of pages need to be * allocated and for a short time, the footprint is higher */ watermark = low_wmark_pages(zone) + (2UL << order); if (!zone_watermark_ok(zone, 0, watermark, 0, 0)) return COMPACT_SKIPPED; /* * fragmentation index determines if allocation failures are due to * low memory or external fragmentation * |
a582a738c
|
925 926 |
* index of -1000 implies allocations might succeed depending on * watermarks |
3e7d34497
|
927 928 929 930 931 932 933 934 |
* index towards 0 implies failure is due to lack of memory * index towards 1000 implies failure is due to fragmentation * * Only compact if a failure would be due to fragmentation. */ fragindex = fragmentation_index(zone, order); if (fragindex >= 0 && fragindex <= sysctl_extfrag_threshold) return COMPACT_SKIPPED; |
a582a738c
|
935 936 |
if (fragindex == -1000 && zone_watermark_ok(zone, order, watermark, 0, 0)) |
3e7d34497
|
937 938 939 940 |
return COMPACT_PARTIAL; return COMPACT_CONTINUE; } |
748446bb6
|
941 942 943 |
static int compact_zone(struct zone *zone, struct compact_control *cc) { int ret; |
c89511ab2
|
944 |
unsigned long start_pfn = zone->zone_start_pfn; |
108bcc96e
|
945 |
unsigned long end_pfn = zone_end_pfn(zone); |
e0b9daeb4
|
946 |
const bool sync = cc->mode != MIGRATE_ASYNC; |
748446bb6
|
947 |
|
3e7d34497
|
948 949 950 951 952 953 954 955 956 957 |
ret = compaction_suitable(zone, cc->order); switch (ret) { case COMPACT_PARTIAL: case COMPACT_SKIPPED: /* Compaction is likely to fail */ return ret; case COMPACT_CONTINUE: /* Fall through to compaction */ ; } |
c89511ab2
|
958 |
/* |
d3132e4b8
|
959 960 961 962 963 964 965 966 |
* Clear pageblock skip if there were failures recently and compaction * is about to be retried after being deferred. kswapd does not do * this reset as it'll reset the cached information when going to sleep. */ if (compaction_restarting(zone, cc->order) && !current_is_kswapd()) __reset_isolation_suitable(zone); /* |
c89511ab2
|
967 968 969 970 |
* Setup to move all movable pages to the end of the zone. Used cached * information on where the scanners should start but check that it * is initialised by ensuring the values are within zone boundaries. */ |
e0b9daeb4
|
971 |
cc->migrate_pfn = zone->compact_cached_migrate_pfn[sync]; |
c89511ab2
|
972 973 974 975 976 977 978 |
cc->free_pfn = zone->compact_cached_free_pfn; if (cc->free_pfn < start_pfn || cc->free_pfn > end_pfn) { cc->free_pfn = end_pfn & ~(pageblock_nr_pages-1); zone->compact_cached_free_pfn = cc->free_pfn; } if (cc->migrate_pfn < start_pfn || cc->migrate_pfn > end_pfn) { cc->migrate_pfn = start_pfn; |
35979ef33
|
979 980 |
zone->compact_cached_migrate_pfn[0] = cc->migrate_pfn; zone->compact_cached_migrate_pfn[1] = cc->migrate_pfn; |
c89511ab2
|
981 |
} |
748446bb6
|
982 |
|
0eb927c0a
|
983 |
trace_mm_compaction_begin(start_pfn, cc->migrate_pfn, cc->free_pfn, end_pfn); |
748446bb6
|
984 985 986 |
migrate_prep_local(); while ((ret = compact_finished(zone, cc)) == COMPACT_CONTINUE) { |
9d502c1c8
|
987 |
int err; |
748446bb6
|
988 |
|
f9e35b3b4
|
989 990 991 |
switch (isolate_migratepages(zone, cc)) { case ISOLATE_ABORT: ret = COMPACT_PARTIAL; |
5733c7d11
|
992 |
putback_movable_pages(&cc->migratepages); |
e64c5237c
|
993 |
cc->nr_migratepages = 0; |
f9e35b3b4
|
994 995 |
goto out; case ISOLATE_NONE: |
748446bb6
|
996 |
continue; |
f9e35b3b4
|
997 998 999 |
case ISOLATE_SUCCESS: ; } |
748446bb6
|
1000 |
|
f8c9301fa
|
1001 1002 |
if (!cc->nr_migratepages) continue; |
d53aea3d4
|
1003 |
err = migrate_pages(&cc->migratepages, compaction_alloc, |
e0b9daeb4
|
1004 |
compaction_free, (unsigned long)cc, cc->mode, |
7b2a2d4a1
|
1005 |
MR_COMPACTION); |
748446bb6
|
1006 |
|
f8c9301fa
|
1007 1008 |
trace_mm_compaction_migratepages(cc->nr_migratepages, err, &cc->migratepages); |
748446bb6
|
1009 |
|
f8c9301fa
|
1010 1011 |
/* All pages were either migrated or will be released */ cc->nr_migratepages = 0; |
9d502c1c8
|
1012 |
if (err) { |
5733c7d11
|
1013 |
putback_movable_pages(&cc->migratepages); |
7ed695e06
|
1014 1015 1016 1017 1018 |
/* * migrate_pages() may return -ENOMEM when scanners meet * and we want compact_finished() to detect it */ if (err == -ENOMEM && cc->free_pfn > cc->migrate_pfn) { |
4bf2bba37
|
1019 1020 1021 |
ret = COMPACT_PARTIAL; goto out; } |
748446bb6
|
1022 |
} |
748446bb6
|
1023 |
} |
f9e35b3b4
|
1024 |
out: |
748446bb6
|
1025 1026 1027 |
/* Release free pages and check accounting */ cc->nr_freepages -= release_freepages(&cc->freepages); VM_BUG_ON(cc->nr_freepages != 0); |
0eb927c0a
|
1028 |
trace_mm_compaction_end(ret); |
748446bb6
|
1029 1030 |
return ret; } |
76ab0f530
|
1031 |
|
e0b9daeb4
|
1032 1033 |
static unsigned long compact_zone_order(struct zone *zone, int order, gfp_t gfp_mask, enum migrate_mode mode, bool *contended) |
56de7263f
|
1034 |
{ |
e64c5237c
|
1035 |
unsigned long ret; |
56de7263f
|
1036 1037 1038 1039 1040 1041 |
struct compact_control cc = { .nr_freepages = 0, .nr_migratepages = 0, .order = order, .migratetype = allocflags_to_migratetype(gfp_mask), .zone = zone, |
e0b9daeb4
|
1042 |
.mode = mode, |
56de7263f
|
1043 1044 1045 |
}; INIT_LIST_HEAD(&cc.freepages); INIT_LIST_HEAD(&cc.migratepages); |
e64c5237c
|
1046 1047 1048 1049 1050 1051 1052 |
ret = compact_zone(zone, &cc); VM_BUG_ON(!list_empty(&cc.freepages)); VM_BUG_ON(!list_empty(&cc.migratepages)); *contended = cc.contended; return ret; |
56de7263f
|
1053 |
} |
5e7719058
|
1054 |
int sysctl_extfrag_threshold = 500; |
56de7263f
|
1055 1056 1057 1058 1059 1060 |
/** * try_to_compact_pages - Direct compact to satisfy a high-order allocation * @zonelist: The zonelist used for the current allocation * @order: The order of the current allocation * @gfp_mask: The GFP mask of the current allocation * @nodemask: The allowed nodes to allocate from |
e0b9daeb4
|
1061 |
* @mode: The migration mode for async, sync light, or sync migration |
661c4cb9b
|
1062 1063 |
* @contended: Return value that is true if compaction was aborted due to lock contention * @page: Optionally capture a free page of the requested order during compaction |
56de7263f
|
1064 1065 1066 1067 |
* * This is the main entry point for direct page compaction. */ unsigned long try_to_compact_pages(struct zonelist *zonelist, |
77f1fe6b0
|
1068 |
int order, gfp_t gfp_mask, nodemask_t *nodemask, |
e0b9daeb4
|
1069 |
enum migrate_mode mode, bool *contended) |
56de7263f
|
1070 1071 1072 1073 |
{ enum zone_type high_zoneidx = gfp_zone(gfp_mask); int may_enter_fs = gfp_mask & __GFP_FS; int may_perform_io = gfp_mask & __GFP_IO; |
56de7263f
|
1074 1075 1076 |
struct zoneref *z; struct zone *zone; int rc = COMPACT_SKIPPED; |
d95ea5d18
|
1077 |
int alloc_flags = 0; |
56de7263f
|
1078 |
|
4ffb6335d
|
1079 |
/* Check if the GFP flags allow compaction */ |
c5a73c3d5
|
1080 |
if (!order || !may_enter_fs || !may_perform_io) |
56de7263f
|
1081 |
return rc; |
010fc29a4
|
1082 |
count_compact_event(COMPACTSTALL); |
56de7263f
|
1083 |
|
d95ea5d18
|
1084 1085 1086 1087 |
#ifdef CONFIG_CMA if (allocflags_to_migratetype(gfp_mask) == MIGRATE_MOVABLE) alloc_flags |= ALLOC_CMA; #endif |
56de7263f
|
1088 1089 1090 |
/* Compact each zone in the list */ for_each_zone_zonelist_nodemask(zone, z, zonelist, high_zoneidx, nodemask) { |
56de7263f
|
1091 |
int status; |
e0b9daeb4
|
1092 |
status = compact_zone_order(zone, order, gfp_mask, mode, |
8fb74b9fb
|
1093 |
contended); |
56de7263f
|
1094 |
rc = max(status, rc); |
3e7d34497
|
1095 |
/* If a normal allocation would succeed, stop compacting */ |
d95ea5d18
|
1096 1097 |
if (zone_watermark_ok(zone, order, low_wmark_pages(zone), 0, alloc_flags)) |
56de7263f
|
1098 1099 1100 1101 1102 |
break; } return rc; } |
76ab0f530
|
1103 |
/* Compact all zones within a node */ |
7103f16db
|
1104 |
static void __compact_pgdat(pg_data_t *pgdat, struct compact_control *cc) |
76ab0f530
|
1105 1106 |
{ int zoneid; |
76ab0f530
|
1107 |
struct zone *zone; |
76ab0f530
|
1108 |
for (zoneid = 0; zoneid < MAX_NR_ZONES; zoneid++) { |
76ab0f530
|
1109 1110 1111 1112 |
zone = &pgdat->node_zones[zoneid]; if (!populated_zone(zone)) continue; |
7be62de99
|
1113 1114 1115 1116 1117 |
cc->nr_freepages = 0; cc->nr_migratepages = 0; cc->zone = zone; INIT_LIST_HEAD(&cc->freepages); INIT_LIST_HEAD(&cc->migratepages); |
76ab0f530
|
1118 |
|
aad6ec377
|
1119 |
if (cc->order == -1 || !compaction_deferred(zone, cc->order)) |
7be62de99
|
1120 |
compact_zone(zone, cc); |
76ab0f530
|
1121 |
|
aff622495
|
1122 |
if (cc->order > 0) { |
de6c60a6c
|
1123 1124 1125 |
if (zone_watermark_ok(zone, cc->order, low_wmark_pages(zone), 0, 0)) compaction_defer_reset(zone, cc->order, false); |
aff622495
|
1126 |
} |
7be62de99
|
1127 1128 |
VM_BUG_ON(!list_empty(&cc->freepages)); VM_BUG_ON(!list_empty(&cc->migratepages)); |
76ab0f530
|
1129 |
} |
76ab0f530
|
1130 |
} |
7103f16db
|
1131 |
void compact_pgdat(pg_data_t *pgdat, int order) |
7be62de99
|
1132 1133 1134 |
{ struct compact_control cc = { .order = order, |
e0b9daeb4
|
1135 |
.mode = MIGRATE_ASYNC, |
7be62de99
|
1136 |
}; |
3a7200af3
|
1137 1138 |
if (!order) return; |
7103f16db
|
1139 |
__compact_pgdat(pgdat, &cc); |
7be62de99
|
1140 |
} |
7103f16db
|
1141 |
static void compact_node(int nid) |
7be62de99
|
1142 |
{ |
7be62de99
|
1143 1144 |
struct compact_control cc = { .order = -1, |
e0b9daeb4
|
1145 |
.mode = MIGRATE_SYNC, |
91ca91864
|
1146 |
.ignore_skip_hint = true, |
7be62de99
|
1147 |
}; |
7103f16db
|
1148 |
__compact_pgdat(NODE_DATA(nid), &cc); |
7be62de99
|
1149 |
} |
76ab0f530
|
1150 |
/* Compact all nodes in the system */ |
7964c06d6
|
1151 |
static void compact_nodes(void) |
76ab0f530
|
1152 1153 |
{ int nid; |
8575ec29f
|
1154 1155 |
/* Flush pending updates to the LRU lists */ lru_add_drain_all(); |
76ab0f530
|
1156 1157 |
for_each_online_node(nid) compact_node(nid); |
76ab0f530
|
1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 |
} /* The written value is actually unused, all memory is compacted */ int sysctl_compact_memory; /* This is the entry point for compacting all nodes via /proc/sys/vm */ int sysctl_compaction_handler(struct ctl_table *table, int write, void __user *buffer, size_t *length, loff_t *ppos) { if (write) |
7964c06d6
|
1168 |
compact_nodes(); |
76ab0f530
|
1169 1170 1171 |
return 0; } |
ed4a6d7f0
|
1172 |
|
5e7719058
|
1173 1174 1175 1176 1177 1178 1179 |
int sysctl_extfrag_handler(struct ctl_table *table, int write, void __user *buffer, size_t *length, loff_t *ppos) { proc_dointvec_minmax(table, write, buffer, length, ppos); return 0; } |
ed4a6d7f0
|
1180 |
#if defined(CONFIG_SYSFS) && defined(CONFIG_NUMA) |
74e77fb9a
|
1181 |
static ssize_t sysfs_compact_node(struct device *dev, |
10fbcf4c6
|
1182 |
struct device_attribute *attr, |
ed4a6d7f0
|
1183 1184 |
const char *buf, size_t count) { |
8575ec29f
|
1185 1186 1187 1188 1189 1190 1191 1192 |
int nid = dev->id; if (nid >= 0 && nid < nr_node_ids && node_online(nid)) { /* Flush pending updates to the LRU lists */ lru_add_drain_all(); compact_node(nid); } |
ed4a6d7f0
|
1193 1194 1195 |
return count; } |
10fbcf4c6
|
1196 |
static DEVICE_ATTR(compact, S_IWUSR, NULL, sysfs_compact_node); |
ed4a6d7f0
|
1197 1198 1199 |
int compaction_register_node(struct node *node) { |
10fbcf4c6
|
1200 |
return device_create_file(&node->dev, &dev_attr_compact); |
ed4a6d7f0
|
1201 1202 1203 1204 |
} void compaction_unregister_node(struct node *node) { |
10fbcf4c6
|
1205 |
return device_remove_file(&node->dev, &dev_attr_compact); |
ed4a6d7f0
|
1206 1207 |
} #endif /* CONFIG_SYSFS && CONFIG_NUMA */ |
ff9543fd3
|
1208 1209 |
#endif /* CONFIG_COMPACTION */ |