Blame view
mm/compaction.c
27.1 KB
748446bb6
|
1 2 3 4 5 6 7 8 9 10 11 12 13 14 |
/* * linux/mm/compaction.c * * Memory compaction for the reduction of external fragmentation. Note that * this heavily depends upon page migration to do all the real heavy * lifting * * Copyright IBM Corp. 2007-2010 Mel Gorman <mel@csn.ul.ie> */ #include <linux/swap.h> #include <linux/migrate.h> #include <linux/compaction.h> #include <linux/mm_inline.h> #include <linux/backing-dev.h> |
76ab0f530
|
15 |
#include <linux/sysctl.h> |
ed4a6d7f0
|
16 |
#include <linux/sysfs.h> |
748446bb6
|
17 |
#include "internal.h" |
ff9543fd3
|
18 |
#if defined CONFIG_COMPACTION || defined CONFIG_CMA |
b7aba6984
|
19 20 |
#define CREATE_TRACE_POINTS #include <trace/events/compaction.h> |
748446bb6
|
21 22 23 24 25 26 27 28 29 30 31 32 33 |
static unsigned long release_freepages(struct list_head *freelist) { struct page *page, *next; unsigned long count = 0; list_for_each_entry_safe(page, next, freelist, lru) { list_del(&page->lru); __free_page(page); count++; } return count; } |
ff9543fd3
|
34 35 36 37 38 39 40 41 42 |
static void map_pages(struct list_head *list) { struct page *page; list_for_each_entry(page, list, lru) { arch_alloc_page(page, 0); kernel_map_pages(page, 1, 1); } } |
47118af07
|
43 44 45 46 |
static inline bool migrate_async_suitable(int migratetype) { return is_migrate_cma(migratetype) || migratetype == MIGRATE_MOVABLE; } |
85aa125f0
|
47 |
/* |
c67fe3752
|
48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 |
* Compaction requires the taking of some coarse locks that are potentially * very heavily contended. Check if the process needs to be scheduled or * if the lock is contended. For async compaction, back out in the event * if contention is severe. For sync compaction, schedule. * * Returns true if the lock is held. * Returns false if the lock is released and compaction should abort */ static bool compact_checklock_irqsave(spinlock_t *lock, unsigned long *flags, bool locked, struct compact_control *cc) { if (need_resched() || spin_is_contended(lock)) { if (locked) { spin_unlock_irqrestore(lock, *flags); locked = false; } /* async aborts if taking too long or contended */ if (!cc->sync) { if (cc->contended) *cc->contended = true; return false; } cond_resched(); if (fatal_signal_pending(current)) return false; } if (!locked) spin_lock_irqsave(lock, *flags); return true; } static inline bool compact_trylock_irqsave(spinlock_t *lock, unsigned long *flags, struct compact_control *cc) { return compact_checklock_irqsave(lock, flags, false, cc); } /* |
85aa125f0
|
89 90 91 92 93 94 95 96 97 |
* Isolate free pages onto a private freelist. Caller must hold zone->lock. * If @strict is true, will abort returning 0 on any invalid PFNs or non-free * pages inside of the pageblock (even though it may still end up isolating * some pages). */ static unsigned long isolate_freepages_block(unsigned long blockpfn, unsigned long end_pfn, struct list_head *freelist, bool strict) |
748446bb6
|
98 |
{ |
b7aba6984
|
99 |
int nr_scanned = 0, total_isolated = 0; |
748446bb6
|
100 |
struct page *cursor; |
748446bb6
|
101 102 103 104 105 106 |
cursor = pfn_to_page(blockpfn); /* Isolate free pages. This assumes the block is valid */ for (; blockpfn < end_pfn; blockpfn++, cursor++) { int isolated, i; struct page *page = cursor; |
85aa125f0
|
107 108 109 |
if (!pfn_valid_within(blockpfn)) { if (strict) return 0; |
748446bb6
|
110 |
continue; |
85aa125f0
|
111 |
} |
b7aba6984
|
112 |
nr_scanned++; |
748446bb6
|
113 |
|
85aa125f0
|
114 115 116 |
if (!PageBuddy(page)) { if (strict) return 0; |
748446bb6
|
117 |
continue; |
85aa125f0
|
118 |
} |
748446bb6
|
119 120 121 |
/* Found a free page, break it into order-0 pages */ isolated = split_free_page(page); |
85aa125f0
|
122 123 |
if (!isolated && strict) return 0; |
748446bb6
|
124 125 126 127 128 129 130 131 132 133 134 135 |
total_isolated += isolated; for (i = 0; i < isolated; i++) { list_add(&page->lru, freelist); page++; } /* If a page was split, advance to the end of it */ if (isolated) { blockpfn += isolated - 1; cursor += isolated - 1; } } |
b7aba6984
|
136 |
trace_mm_compaction_isolate_freepages(nr_scanned, total_isolated); |
748446bb6
|
137 138 |
return total_isolated; } |
85aa125f0
|
139 140 141 142 143 144 145 146 147 148 149 150 151 |
/** * isolate_freepages_range() - isolate free pages. * @start_pfn: The first PFN to start isolating. * @end_pfn: The one-past-last PFN. * * Non-free pages, invalid PFNs, or zone boundaries within the * [start_pfn, end_pfn) range are considered errors, cause function to * undo its actions and return zero. * * Otherwise, function returns one-past-the-last PFN of isolated page * (which may be greater then end_pfn if end fell in a middle of * a free page). */ |
ff9543fd3
|
152 |
unsigned long |
85aa125f0
|
153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 |
isolate_freepages_range(unsigned long start_pfn, unsigned long end_pfn) { unsigned long isolated, pfn, block_end_pfn, flags; struct zone *zone = NULL; LIST_HEAD(freelist); if (pfn_valid(start_pfn)) zone = page_zone(pfn_to_page(start_pfn)); for (pfn = start_pfn; pfn < end_pfn; pfn += isolated) { if (!pfn_valid(pfn) || zone != page_zone(pfn_to_page(pfn))) break; /* * On subsequent iterations ALIGN() is actually not needed, * but we keep it that we not to complicate the code. */ block_end_pfn = ALIGN(pfn + 1, pageblock_nr_pages); block_end_pfn = min(block_end_pfn, end_pfn); spin_lock_irqsave(&zone->lock, flags); isolated = isolate_freepages_block(pfn, block_end_pfn, &freelist, true); spin_unlock_irqrestore(&zone->lock, flags); /* * In strict mode, isolate_freepages_block() returns 0 if * there are any holes in the block (ie. invalid PFNs or * non-free pages). */ if (!isolated) break; /* * If we managed to isolate pages, it is always (1 << n) * * pageblock_nr_pages for some non-negative n. (Max order * page may span two pageblocks). */ } /* split_free_page does not map the pages */ map_pages(&freelist); if (pfn < end_pfn) { /* Loop terminated early, cleanup. */ release_freepages(&freelist); return 0; } /* We don't use freelists for anything. */ return pfn; } |
748446bb6
|
205 |
/* Update the number of anon and file isolated pages in the zone */ |
c67fe3752
|
206 |
static void acct_isolated(struct zone *zone, bool locked, struct compact_control *cc) |
748446bb6
|
207 208 |
{ struct page *page; |
b9e84ac15
|
209 |
unsigned int count[2] = { 0, }; |
748446bb6
|
210 |
|
b9e84ac15
|
211 212 |
list_for_each_entry(page, &cc->migratepages, lru) count[!!page_is_file_cache(page)]++; |
748446bb6
|
213 |
|
c67fe3752
|
214 215 216 217 218 219 220 221 |
/* If locked we can use the interrupt unsafe versions */ if (locked) { __mod_zone_page_state(zone, NR_ISOLATED_ANON, count[0]); __mod_zone_page_state(zone, NR_ISOLATED_FILE, count[1]); } else { mod_zone_page_state(zone, NR_ISOLATED_ANON, count[0]); mod_zone_page_state(zone, NR_ISOLATED_FILE, count[1]); } |
748446bb6
|
222 223 224 225 226 |
} /* Similar to reclaim, but different enough that they don't share logic */ static bool too_many_isolated(struct zone *zone) { |
bc6930457
|
227 |
unsigned long active, inactive, isolated; |
748446bb6
|
228 229 230 |
inactive = zone_page_state(zone, NR_INACTIVE_FILE) + zone_page_state(zone, NR_INACTIVE_ANON); |
bc6930457
|
231 232 |
active = zone_page_state(zone, NR_ACTIVE_FILE) + zone_page_state(zone, NR_ACTIVE_ANON); |
748446bb6
|
233 234 |
isolated = zone_page_state(zone, NR_ISOLATED_FILE) + zone_page_state(zone, NR_ISOLATED_ANON); |
bc6930457
|
235 |
return isolated > (inactive + active) / 2; |
748446bb6
|
236 |
} |
2fe86e000
|
237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 |
/** * isolate_migratepages_range() - isolate all migrate-able pages in range. * @zone: Zone pages are in. * @cc: Compaction control structure. * @low_pfn: The first PFN of the range. * @end_pfn: The one-past-the-last PFN of the range. * * Isolate all pages that can be migrated from the range specified by * [low_pfn, end_pfn). Returns zero if there is a fatal signal * pending), otherwise PFN of the first page that was not scanned * (which may be both less, equal to or more then end_pfn). * * Assumes that cc->migratepages is empty and cc->nr_migratepages is * zero. * * Apart from cc->migratepages and cc->nr_migratetypes this function * does not modify any cc's fields, in particular it does not modify * (or read for that matter) cc->migrate_pfn. |
748446bb6
|
255 |
*/ |
ff9543fd3
|
256 |
unsigned long |
2fe86e000
|
257 258 |
isolate_migratepages_range(struct zone *zone, struct compact_control *cc, unsigned long low_pfn, unsigned long end_pfn) |
748446bb6
|
259 |
{ |
9927af740
|
260 |
unsigned long last_pageblock_nr = 0, pageblock_nr; |
b7aba6984
|
261 |
unsigned long nr_scanned = 0, nr_isolated = 0; |
748446bb6
|
262 |
struct list_head *migratelist = &cc->migratepages; |
f3fd4a619
|
263 |
isolate_mode_t mode = 0; |
fa9add641
|
264 |
struct lruvec *lruvec; |
c67fe3752
|
265 266 |
unsigned long flags; bool locked; |
748446bb6
|
267 |
|
748446bb6
|
268 269 270 271 272 273 |
/* * Ensure that there are not too many pages isolated from the LRU * list by either parallel reclaimers or compaction. If there are, * delay for some time until fewer pages are isolated */ while (unlikely(too_many_isolated(zone))) { |
f9e35b3b4
|
274 |
/* async migration should just abort */ |
68e3e9262
|
275 |
if (!cc->sync) |
2fe86e000
|
276 |
return 0; |
f9e35b3b4
|
277 |
|
748446bb6
|
278 279 280 |
congestion_wait(BLK_RW_ASYNC, HZ/10); if (fatal_signal_pending(current)) |
2fe86e000
|
281 |
return 0; |
748446bb6
|
282 283 284 |
} /* Time to isolate some pages for migration */ |
b2eef8c0d
|
285 |
cond_resched(); |
c67fe3752
|
286 287 |
spin_lock_irqsave(&zone->lru_lock, flags); locked = true; |
748446bb6
|
288 289 |
for (; low_pfn < end_pfn; low_pfn++) { struct page *page; |
b2eef8c0d
|
290 291 292 |
/* give a chance to irqs before checking need_resched() */ if (!((low_pfn+1) % SWAP_CLUSTER_MAX)) { |
c67fe3752
|
293 |
spin_unlock_irqrestore(&zone->lru_lock, flags); |
b2eef8c0d
|
294 295 |
locked = false; } |
c67fe3752
|
296 297 298 299 300 301 |
/* Check if it is ok to still hold the lock */ locked = compact_checklock_irqsave(&zone->lru_lock, &flags, locked, cc); if (!locked) break; |
b2eef8c0d
|
302 |
|
0bf380bc7
|
303 304 305 306 307 308 309 310 311 312 313 314 |
/* * migrate_pfn does not necessarily start aligned to a * pageblock. Ensure that pfn_valid is called when moving * into a new MAX_ORDER_NR_PAGES range in case of large * memory holes within the zone */ if ((low_pfn & (MAX_ORDER_NR_PAGES - 1)) == 0) { if (!pfn_valid(low_pfn)) { low_pfn += MAX_ORDER_NR_PAGES - 1; continue; } } |
748446bb6
|
315 316 |
if (!pfn_valid_within(low_pfn)) continue; |
b7aba6984
|
317 |
nr_scanned++; |
748446bb6
|
318 |
|
dc9086004
|
319 320 321 322 323 324 |
/* * Get the page and ensure the page is within the same zone. * See the comment in isolate_freepages about overlapping * nodes. It is deliberate that the new zone lock is not taken * as memory compaction should not move pages between nodes. */ |
748446bb6
|
325 |
page = pfn_to_page(low_pfn); |
dc9086004
|
326 327 328 329 |
if (page_zone(page) != zone) continue; /* Skip if free */ |
748446bb6
|
330 331 |
if (PageBuddy(page)) continue; |
9927af740
|
332 333 334 335 336 337 |
/* * For async migration, also only scan in MOVABLE blocks. Async * migration is optimistic to see if the minimum amount of work * satisfies the allocation */ pageblock_nr = low_pfn >> pageblock_order; |
68e3e9262
|
338 |
if (!cc->sync && last_pageblock_nr != pageblock_nr && |
47118af07
|
339 |
!migrate_async_suitable(get_pageblock_migratetype(page))) { |
9927af740
|
340 341 342 343 344 |
low_pfn += pageblock_nr_pages; low_pfn = ALIGN(low_pfn, pageblock_nr_pages) - 1; last_pageblock_nr = pageblock_nr; continue; } |
bc835011a
|
345 346 347 348 349 350 351 352 353 354 355 356 |
if (!PageLRU(page)) continue; /* * PageLRU is set, and lru_lock excludes isolation, * splitting and collapsing (collapsing has already * happened if PageLRU is set). */ if (PageTransHuge(page)) { low_pfn += (1 << compound_order(page)) - 1; continue; } |
68e3e9262
|
357 |
if (!cc->sync) |
c82449352
|
358 |
mode |= ISOLATE_ASYNC_MIGRATE; |
fa9add641
|
359 |
lruvec = mem_cgroup_page_lruvec(page, zone); |
748446bb6
|
360 |
/* Try isolate the page */ |
f3fd4a619
|
361 |
if (__isolate_lru_page(page, mode) != 0) |
748446bb6
|
362 |
continue; |
bc835011a
|
363 |
VM_BUG_ON(PageTransCompound(page)); |
748446bb6
|
364 |
/* Successfully isolated */ |
fa9add641
|
365 |
del_page_from_lru_list(page, lruvec, page_lru(page)); |
748446bb6
|
366 |
list_add(&page->lru, migratelist); |
748446bb6
|
367 |
cc->nr_migratepages++; |
b7aba6984
|
368 |
nr_isolated++; |
748446bb6
|
369 370 |
/* Avoid isolating too much */ |
31b8384a5
|
371 372 |
if (cc->nr_migratepages == COMPACT_CLUSTER_MAX) { ++low_pfn; |
748446bb6
|
373 |
break; |
31b8384a5
|
374 |
} |
748446bb6
|
375 |
} |
c67fe3752
|
376 |
acct_isolated(zone, locked, cc); |
748446bb6
|
377 |
|
c67fe3752
|
378 379 |
if (locked) spin_unlock_irqrestore(&zone->lru_lock, flags); |
748446bb6
|
380 |
|
b7aba6984
|
381 |
trace_mm_compaction_isolate_migratepages(nr_scanned, nr_isolated); |
2fe86e000
|
382 383 |
return low_pfn; } |
ff9543fd3
|
384 385 |
#endif /* CONFIG_COMPACTION || CONFIG_CMA */ #ifdef CONFIG_COMPACTION |
68e3e9262
|
386 387 |
/* Returns true if the page is within a block suitable for migration to */ static bool suitable_migration_target(struct page *page) |
ff9543fd3
|
388 389 390 391 392 393 |
{ int migratetype = get_pageblock_migratetype(page); /* Don't interfere with memory hot-remove or the min_free_kbytes blocks */ if (migratetype == MIGRATE_ISOLATE || migratetype == MIGRATE_RESERVE) |
68e3e9262
|
394 |
return false; |
ff9543fd3
|
395 396 397 |
/* If the page is a large free page, then allow migration */ if (PageBuddy(page) && page_order(page) >= pageblock_order) |
68e3e9262
|
398 |
return true; |
ff9543fd3
|
399 |
|
47118af07
|
400 |
/* If the block is MIGRATE_MOVABLE or MIGRATE_CMA, allow migration */ |
68e3e9262
|
401 402 |
if (migrate_async_suitable(migratetype)) return true; |
ff9543fd3
|
403 404 |
/* Otherwise skip the block */ |
68e3e9262
|
405 |
return false; |
ff9543fd3
|
406 |
} |
2fe86e000
|
407 |
/* |
de74f1cc3
|
408 409 410 411 412 413 414 415 416 417 418 419 420 421 |
* Returns the start pfn of the last page block in a zone. This is the starting * point for full compaction of a zone. Compaction searches for free pages from * the end of each zone, while isolate_freepages_block scans forward inside each * page block. */ static unsigned long start_free_pfn(struct zone *zone) { unsigned long free_pfn; free_pfn = zone->zone_start_pfn + zone->spanned_pages; free_pfn &= ~(pageblock_nr_pages-1); return free_pfn; } /* |
ff9543fd3
|
422 423 |
* Based on information in the current compact_control, find blocks * suitable for isolating free pages from and then isolate them. |
2fe86e000
|
424 |
*/ |
ff9543fd3
|
425 426 |
static void isolate_freepages(struct zone *zone, struct compact_control *cc) |
2fe86e000
|
427 |
{ |
ff9543fd3
|
428 429 430 431 432 |
struct page *page; unsigned long high_pfn, low_pfn, pfn, zone_end_pfn, end_pfn; unsigned long flags; int nr_freepages = cc->nr_freepages; struct list_head *freelist = &cc->freepages; |
2fe86e000
|
433 |
|
ff9543fd3
|
434 435 436 437 438 439 440 |
/* * Initialise the free scanner. The starting point is where we last * scanned from (or the end of the zone if starting). The low point * is the end of the pageblock the migration scanner is using. */ pfn = cc->free_pfn; low_pfn = cc->migrate_pfn + pageblock_nr_pages; |
2fe86e000
|
441 |
|
ff9543fd3
|
442 443 444 445 446 447 |
/* * Take care that if the migration scanner is at the end of the zone * that the free scanner does not accidentally move to the next zone * in the next isolation cycle. */ high_pfn = min(low_pfn, pfn); |
2fe86e000
|
448 |
|
ff9543fd3
|
449 |
zone_end_pfn = zone->zone_start_pfn + zone->spanned_pages; |
2fe86e000
|
450 |
|
ff9543fd3
|
451 452 453 454 455 456 457 458 |
/* * Isolate free pages until enough are available to migrate the * pages on cc->migratepages. We stop searching if the migrate * and free page scanners meet or enough free pages are isolated. */ for (; pfn > low_pfn && cc->nr_migratepages > nr_freepages; pfn -= pageblock_nr_pages) { unsigned long isolated; |
2fe86e000
|
459 |
|
ff9543fd3
|
460 461 |
if (!pfn_valid(pfn)) continue; |
2fe86e000
|
462 |
|
ff9543fd3
|
463 464 465 466 467 468 469 470 471 472 473 474 |
/* * Check for overlapping nodes/zones. It's possible on some * configurations to have a setup like * node0 node1 node0 * i.e. it's possible that all pages within a zones range of * pages do not belong to a single zone. */ page = pfn_to_page(pfn); if (page_zone(page) != zone) continue; /* Check the block is suitable for migration */ |
68e3e9262
|
475 |
if (!suitable_migration_target(page)) |
ff9543fd3
|
476 |
continue; |
68e3e9262
|
477 |
|
ff9543fd3
|
478 479 480 481 482 483 484 |
/* * Found a block suitable for isolating free pages from. Now * we disabled interrupts, double check things are ok and * isolate the pages. This is to minimise the time IRQs * are disabled */ isolated = 0; |
c67fe3752
|
485 486 487 488 489 490 491 492 493 494 |
/* * The zone lock must be held to isolate freepages. This * unfortunately this is a very coarse lock and can be * heavily contended if there are parallel allocations * or parallel compactions. For async compaction do not * spin on the lock */ if (!compact_trylock_irqsave(&zone->lock, &flags, cc)) break; |
68e3e9262
|
495 |
if (suitable_migration_target(page)) { |
ff9543fd3
|
496 497 498 499 |
end_pfn = min(pfn + pageblock_nr_pages, zone_end_pfn); isolated = isolate_freepages_block(pfn, end_pfn, freelist, false); nr_freepages += isolated; |
68e3e9262
|
500 |
} |
ff9543fd3
|
501 502 503 504 505 506 507 |
spin_unlock_irqrestore(&zone->lock, flags); /* * Record the highest PFN we isolated pages from. When next * looking for free pages, the search will restart here as * page migration may have returned some pages to the allocator */ |
7db8889ab
|
508 |
if (isolated) { |
ff9543fd3
|
509 |
high_pfn = max(high_pfn, pfn); |
de74f1cc3
|
510 511 512 513 514 515 516 517 518 |
/* * If the free scanner has wrapped, update * compact_cached_free_pfn to point to the highest * pageblock with free pages. This reduces excessive * scanning of full pageblocks near the end of the * zone */ if (cc->order > 0 && cc->wrapped) |
7db8889ab
|
519 520 |
zone->compact_cached_free_pfn = high_pfn; } |
ff9543fd3
|
521 522 523 524 525 526 527 |
} /* split_free_page does not map the pages */ map_pages(freelist); cc->free_pfn = high_pfn; cc->nr_freepages = nr_freepages; |
de74f1cc3
|
528 529 530 531 532 |
/* If compact_cached_free_pfn is reset then set it now */ if (cc->order > 0 && !cc->wrapped && zone->compact_cached_free_pfn == start_free_pfn(zone)) zone->compact_cached_free_pfn = high_pfn; |
748446bb6
|
533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 |
} /* * This is a migrate-callback that "allocates" freepages by taking pages * from the isolated freelists in the block we are migrating to. */ static struct page *compaction_alloc(struct page *migratepage, unsigned long data, int **result) { struct compact_control *cc = (struct compact_control *)data; struct page *freepage; /* Isolate free pages if necessary */ if (list_empty(&cc->freepages)) { isolate_freepages(cc->zone, cc); if (list_empty(&cc->freepages)) return NULL; } freepage = list_entry(cc->freepages.next, struct page, lru); list_del(&freepage->lru); cc->nr_freepages--; return freepage; } /* * We cannot control nr_migratepages and nr_freepages fully when migration is * running as migrate_pages() has no knowledge of compact_control. When * migration is complete, we count the number of pages on the lists by hand. */ static void update_nr_listpages(struct compact_control *cc) { int nr_migratepages = 0; int nr_freepages = 0; struct page *page; list_for_each_entry(page, &cc->migratepages, lru) nr_migratepages++; list_for_each_entry(page, &cc->freepages, lru) nr_freepages++; cc->nr_migratepages = nr_migratepages; cc->nr_freepages = nr_freepages; } |
ff9543fd3
|
580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 |
/* possible outcome of isolate_migratepages */ typedef enum { ISOLATE_ABORT, /* Abort compaction now */ ISOLATE_NONE, /* No pages isolated, continue scanning */ ISOLATE_SUCCESS, /* Pages isolated, migrate */ } isolate_migrate_t; /* * Isolate all pages that can be migrated from the block pointed to by * the migrate scanner within compact_control. */ static isolate_migrate_t isolate_migratepages(struct zone *zone, struct compact_control *cc) { unsigned long low_pfn, end_pfn; /* Do not scan outside zone boundaries */ low_pfn = max(cc->migrate_pfn, zone->zone_start_pfn); /* Only scan within a pageblock boundary */ end_pfn = ALIGN(low_pfn + pageblock_nr_pages, pageblock_nr_pages); /* Do not cross the free scanner or scan within a memory hole */ if (end_pfn > cc->free_pfn || !pfn_valid(low_pfn)) { cc->migrate_pfn = end_pfn; return ISOLATE_NONE; } /* Perform the isolation */ low_pfn = isolate_migratepages_range(zone, cc, low_pfn, end_pfn); if (!low_pfn) return ISOLATE_ABORT; cc->migrate_pfn = low_pfn; return ISOLATE_SUCCESS; } |
748446bb6
|
617 |
static int compact_finished(struct zone *zone, |
5a03b051e
|
618 |
struct compact_control *cc) |
748446bb6
|
619 |
{ |
56de7263f
|
620 |
unsigned int order; |
5a03b051e
|
621 |
unsigned long watermark; |
56de7263f
|
622 |
|
748446bb6
|
623 624 |
if (fatal_signal_pending(current)) return COMPACT_PARTIAL; |
7db8889ab
|
625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 |
/* * A full (order == -1) compaction run starts at the beginning and * end of a zone; it completes when the migrate and free scanner meet. * A partial (order > 0) compaction can start with the free scanner * at a random point in the zone, and may have to restart. */ if (cc->free_pfn <= cc->migrate_pfn) { if (cc->order > 0 && !cc->wrapped) { /* We started partway through; restart at the end. */ unsigned long free_pfn = start_free_pfn(zone); zone->compact_cached_free_pfn = free_pfn; cc->free_pfn = free_pfn; cc->wrapped = 1; return COMPACT_CONTINUE; } return COMPACT_COMPLETE; } /* We wrapped around and ended up where we started. */ if (cc->wrapped && cc->free_pfn <= cc->start_free_pfn) |
748446bb6
|
645 |
return COMPACT_COMPLETE; |
82478fb7b
|
646 647 648 649 |
/* * order == -1 is expected when compacting via * /proc/sys/vm/compact_memory */ |
56de7263f
|
650 651 |
if (cc->order == -1) return COMPACT_CONTINUE; |
3957c7768
|
652 653 654 655 656 657 |
/* Compaction run is not finished if the watermark is not met */ watermark = low_wmark_pages(zone); watermark += (1 << cc->order); if (!zone_watermark_ok(zone, cc->order, watermark, 0, 0)) return COMPACT_CONTINUE; |
56de7263f
|
658 659 660 661 662 663 664 665 666 667 |
/* Direct compactor: Is a suitable page free? */ for (order = cc->order; order < MAX_ORDER; order++) { /* Job done if page is free of the right migratetype */ if (!list_empty(&zone->free_area[order].free_list[cc->migratetype])) return COMPACT_PARTIAL; /* Job done if allocation would set block type */ if (order >= pageblock_order && zone->free_area[order].nr_free) return COMPACT_PARTIAL; } |
748446bb6
|
668 669 |
return COMPACT_CONTINUE; } |
3e7d34497
|
670 671 672 673 674 675 676 677 678 679 680 681 682 |
/* * compaction_suitable: Is this suitable to run compaction on this zone now? * Returns * COMPACT_SKIPPED - If there are too few free pages for compaction * COMPACT_PARTIAL - If the allocation would succeed without compaction * COMPACT_CONTINUE - If compaction should run now */ unsigned long compaction_suitable(struct zone *zone, int order) { int fragindex; unsigned long watermark; /* |
3957c7768
|
683 684 685 686 687 688 689 |
* order == -1 is expected when compacting via * /proc/sys/vm/compact_memory */ if (order == -1) return COMPACT_CONTINUE; /* |
3e7d34497
|
690 691 692 693 694 695 696 697 698 699 700 701 |
* Watermarks for order-0 must be met for compaction. Note the 2UL. * This is because during migration, copies of pages need to be * allocated and for a short time, the footprint is higher */ watermark = low_wmark_pages(zone) + (2UL << order); if (!zone_watermark_ok(zone, 0, watermark, 0, 0)) return COMPACT_SKIPPED; /* * fragmentation index determines if allocation failures are due to * low memory or external fragmentation * |
a582a738c
|
702 703 |
* index of -1000 implies allocations might succeed depending on * watermarks |
3e7d34497
|
704 705 706 707 708 709 710 711 |
* index towards 0 implies failure is due to lack of memory * index towards 1000 implies failure is due to fragmentation * * Only compact if a failure would be due to fragmentation. */ fragindex = fragmentation_index(zone, order); if (fragindex >= 0 && fragindex <= sysctl_extfrag_threshold) return COMPACT_SKIPPED; |
a582a738c
|
712 713 |
if (fragindex == -1000 && zone_watermark_ok(zone, order, watermark, 0, 0)) |
3e7d34497
|
714 715 716 717 |
return COMPACT_PARTIAL; return COMPACT_CONTINUE; } |
748446bb6
|
718 719 720 |
static int compact_zone(struct zone *zone, struct compact_control *cc) { int ret; |
3e7d34497
|
721 722 723 724 725 726 727 728 729 730 |
ret = compaction_suitable(zone, cc->order); switch (ret) { case COMPACT_PARTIAL: case COMPACT_SKIPPED: /* Compaction is likely to fail */ return ret; case COMPACT_CONTINUE: /* Fall through to compaction */ ; } |
748446bb6
|
731 732 |
/* Setup to move all movable pages to the end of the zone */ cc->migrate_pfn = zone->zone_start_pfn; |
7db8889ab
|
733 734 735 736 737 738 739 740 741 |
if (cc->order > 0) { /* Incremental compaction. Start where the last one stopped. */ cc->free_pfn = zone->compact_cached_free_pfn; cc->start_free_pfn = cc->free_pfn; } else { /* Order == -1 starts at the end of the zone. */ cc->free_pfn = start_free_pfn(zone); } |
748446bb6
|
742 743 744 745 746 |
migrate_prep_local(); while ((ret = compact_finished(zone, cc)) == COMPACT_CONTINUE) { unsigned long nr_migrate, nr_remaining; |
9d502c1c8
|
747 |
int err; |
748446bb6
|
748 |
|
f9e35b3b4
|
749 750 751 752 753 |
switch (isolate_migratepages(zone, cc)) { case ISOLATE_ABORT: ret = COMPACT_PARTIAL; goto out; case ISOLATE_NONE: |
748446bb6
|
754 |
continue; |
f9e35b3b4
|
755 756 757 |
case ISOLATE_SUCCESS: ; } |
748446bb6
|
758 759 |
nr_migrate = cc->nr_migratepages; |
9d502c1c8
|
760 |
err = migrate_pages(&cc->migratepages, compaction_alloc, |
68e3e9262
|
761 762 |
(unsigned long)cc, false, cc->sync ? MIGRATE_SYNC_LIGHT : MIGRATE_ASYNC); |
748446bb6
|
763 764 765 766 767 768 769 |
update_nr_listpages(cc); nr_remaining = cc->nr_migratepages; count_vm_event(COMPACTBLOCKS); count_vm_events(COMPACTPAGES, nr_migrate - nr_remaining); if (nr_remaining) count_vm_events(COMPACTPAGEFAILED, nr_remaining); |
b7aba6984
|
770 771 |
trace_mm_compaction_migratepages(nr_migrate - nr_remaining, nr_remaining); |
748446bb6
|
772 773 |
/* Release LRU pages not migrated */ |
9d502c1c8
|
774 |
if (err) { |
748446bb6
|
775 776 |
putback_lru_pages(&cc->migratepages); cc->nr_migratepages = 0; |
4bf2bba37
|
777 778 779 780 |
if (err == -ENOMEM) { ret = COMPACT_PARTIAL; goto out; } |
748446bb6
|
781 |
} |
748446bb6
|
782 |
} |
f9e35b3b4
|
783 |
out: |
748446bb6
|
784 785 786 787 788 789 |
/* Release free pages and check accounting */ cc->nr_freepages -= release_freepages(&cc->freepages); VM_BUG_ON(cc->nr_freepages != 0); return ret; } |
76ab0f530
|
790 |
|
d43a87e68
|
791 |
static unsigned long compact_zone_order(struct zone *zone, |
5a03b051e
|
792 |
int order, gfp_t gfp_mask, |
c67fe3752
|
793 |
bool sync, bool *contended) |
56de7263f
|
794 795 796 797 798 799 800 |
{ struct compact_control cc = { .nr_freepages = 0, .nr_migratepages = 0, .order = order, .migratetype = allocflags_to_migratetype(gfp_mask), .zone = zone, |
68e3e9262
|
801 |
.sync = sync, |
c67fe3752
|
802 |
.contended = contended, |
56de7263f
|
803 804 805 |
}; INIT_LIST_HEAD(&cc.freepages); INIT_LIST_HEAD(&cc.migratepages); |
68e3e9262
|
806 |
return compact_zone(zone, &cc); |
56de7263f
|
807 |
} |
5e7719058
|
808 |
int sysctl_extfrag_threshold = 500; |
56de7263f
|
809 810 811 812 813 814 |
/** * try_to_compact_pages - Direct compact to satisfy a high-order allocation * @zonelist: The zonelist used for the current allocation * @order: The order of the current allocation * @gfp_mask: The GFP mask of the current allocation * @nodemask: The allowed nodes to allocate from |
77f1fe6b0
|
815 |
* @sync: Whether migration is synchronous or not |
56de7263f
|
816 817 818 819 |
* * This is the main entry point for direct page compaction. */ unsigned long try_to_compact_pages(struct zonelist *zonelist, |
77f1fe6b0
|
820 |
int order, gfp_t gfp_mask, nodemask_t *nodemask, |
c67fe3752
|
821 |
bool sync, bool *contended) |
56de7263f
|
822 823 824 825 |
{ enum zone_type high_zoneidx = gfp_zone(gfp_mask); int may_enter_fs = gfp_mask & __GFP_FS; int may_perform_io = gfp_mask & __GFP_IO; |
56de7263f
|
826 827 828 829 830 831 832 833 834 |
struct zoneref *z; struct zone *zone; int rc = COMPACT_SKIPPED; /* * Check whether it is worth even starting compaction. The order check is * made because an assumption is made that the page allocator can satisfy * the "cheaper" orders without taking special steps */ |
c5a73c3d5
|
835 |
if (!order || !may_enter_fs || !may_perform_io) |
56de7263f
|
836 837 838 839 840 841 842 |
return rc; count_vm_event(COMPACTSTALL); /* Compact each zone in the list */ for_each_zone_zonelist_nodemask(zone, z, zonelist, high_zoneidx, nodemask) { |
56de7263f
|
843 |
int status; |
c67fe3752
|
844 845 |
status = compact_zone_order(zone, order, gfp_mask, sync, contended); |
56de7263f
|
846 |
rc = max(status, rc); |
3e7d34497
|
847 848 |
/* If a normal allocation would succeed, stop compacting */ if (zone_watermark_ok(zone, order, low_wmark_pages(zone), 0, 0)) |
56de7263f
|
849 850 851 852 853 |
break; } return rc; } |
76ab0f530
|
854 |
/* Compact all zones within a node */ |
7be62de99
|
855 |
static int __compact_pgdat(pg_data_t *pgdat, struct compact_control *cc) |
76ab0f530
|
856 857 |
{ int zoneid; |
76ab0f530
|
858 |
struct zone *zone; |
76ab0f530
|
859 |
for (zoneid = 0; zoneid < MAX_NR_ZONES; zoneid++) { |
76ab0f530
|
860 861 862 863 |
zone = &pgdat->node_zones[zoneid]; if (!populated_zone(zone)) continue; |
7be62de99
|
864 865 866 867 868 |
cc->nr_freepages = 0; cc->nr_migratepages = 0; cc->zone = zone; INIT_LIST_HEAD(&cc->freepages); INIT_LIST_HEAD(&cc->migratepages); |
76ab0f530
|
869 |
|
aad6ec377
|
870 |
if (cc->order == -1 || !compaction_deferred(zone, cc->order)) |
7be62de99
|
871 |
compact_zone(zone, cc); |
76ab0f530
|
872 |
|
aff622495
|
873 874 875 |
if (cc->order > 0) { int ok = zone_watermark_ok(zone, cc->order, low_wmark_pages(zone), 0, 0); |
c81758fbe
|
876 |
if (ok && cc->order >= zone->compact_order_failed) |
aff622495
|
877 878 |
zone->compact_order_failed = cc->order + 1; /* Currently async compaction is never deferred. */ |
68e3e9262
|
879 |
else if (!ok && cc->sync) |
aff622495
|
880 881 |
defer_compaction(zone, cc->order); } |
7be62de99
|
882 883 |
VM_BUG_ON(!list_empty(&cc->freepages)); VM_BUG_ON(!list_empty(&cc->migratepages)); |
76ab0f530
|
884 885 886 887 |
} return 0; } |
7be62de99
|
888 889 890 891 |
int compact_pgdat(pg_data_t *pgdat, int order) { struct compact_control cc = { .order = order, |
68e3e9262
|
892 |
.sync = false, |
7be62de99
|
893 894 895 896 897 898 899 |
}; return __compact_pgdat(pgdat, &cc); } static int compact_node(int nid) { |
7be62de99
|
900 901 |
struct compact_control cc = { .order = -1, |
68e3e9262
|
902 |
.sync = true, |
7be62de99
|
903 |
}; |
8575ec29f
|
904 |
return __compact_pgdat(NODE_DATA(nid), &cc); |
7be62de99
|
905 |
} |
76ab0f530
|
906 907 908 909 |
/* Compact all nodes in the system */ static int compact_nodes(void) { int nid; |
8575ec29f
|
910 911 |
/* Flush pending updates to the LRU lists */ lru_add_drain_all(); |
76ab0f530
|
912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 |
for_each_online_node(nid) compact_node(nid); return COMPACT_COMPLETE; } /* The written value is actually unused, all memory is compacted */ int sysctl_compact_memory; /* This is the entry point for compacting all nodes via /proc/sys/vm */ int sysctl_compaction_handler(struct ctl_table *table, int write, void __user *buffer, size_t *length, loff_t *ppos) { if (write) return compact_nodes(); return 0; } |
ed4a6d7f0
|
930 |
|
5e7719058
|
931 932 933 934 935 936 937 |
int sysctl_extfrag_handler(struct ctl_table *table, int write, void __user *buffer, size_t *length, loff_t *ppos) { proc_dointvec_minmax(table, write, buffer, length, ppos); return 0; } |
ed4a6d7f0
|
938 |
#if defined(CONFIG_SYSFS) && defined(CONFIG_NUMA) |
10fbcf4c6
|
939 940 |
ssize_t sysfs_compact_node(struct device *dev, struct device_attribute *attr, |
ed4a6d7f0
|
941 942 |
const char *buf, size_t count) { |
8575ec29f
|
943 944 945 946 947 948 949 950 |
int nid = dev->id; if (nid >= 0 && nid < nr_node_ids && node_online(nid)) { /* Flush pending updates to the LRU lists */ lru_add_drain_all(); compact_node(nid); } |
ed4a6d7f0
|
951 952 953 |
return count; } |
10fbcf4c6
|
954 |
static DEVICE_ATTR(compact, S_IWUSR, NULL, sysfs_compact_node); |
ed4a6d7f0
|
955 956 957 |
int compaction_register_node(struct node *node) { |
10fbcf4c6
|
958 |
return device_create_file(&node->dev, &dev_attr_compact); |
ed4a6d7f0
|
959 960 961 962 |
} void compaction_unregister_node(struct node *node) { |
10fbcf4c6
|
963 |
return device_remove_file(&node->dev, &dev_attr_compact); |
ed4a6d7f0
|
964 965 |
} #endif /* CONFIG_SYSFS && CONFIG_NUMA */ |
ff9543fd3
|
966 967 |
#endif /* CONFIG_COMPACTION */ |