Blame view
mm/compaction.c
33.7 KB
748446bb6 mm: compaction: m... |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 |
/* * linux/mm/compaction.c * * Memory compaction for the reduction of external fragmentation. Note that * this heavily depends upon page migration to do all the real heavy * lifting * * Copyright IBM Corp. 2007-2010 Mel Gorman <mel@csn.ul.ie> */ #include <linux/swap.h> #include <linux/migrate.h> #include <linux/compaction.h> #include <linux/mm_inline.h> #include <linux/backing-dev.h> |
76ab0f530 mm: compaction: a... |
15 |
#include <linux/sysctl.h> |
ed4a6d7f0 mm: compaction: a... |
16 |
#include <linux/sysfs.h> |
bf6bddf19 mm: introduce com... |
17 |
#include <linux/balloon_compaction.h> |
194159fbc mm: remove MIGRAT... |
18 |
#include <linux/page-isolation.h> |
748446bb6 mm: compaction: m... |
19 |
#include "internal.h" |
010fc29a4 compaction: fix b... |
20 21 22 23 24 25 26 27 28 29 30 31 32 33 |
#ifdef CONFIG_COMPACTION static inline void count_compact_event(enum vm_event_item item) { count_vm_event(item); } static inline void count_compact_events(enum vm_event_item item, long delta) { count_vm_events(item, delta); } #else #define count_compact_event(item) do { } while (0) #define count_compact_events(item, delta) do { } while (0) #endif |
ff9543fd3 mm: compaction: e... |
34 |
#if defined CONFIG_COMPACTION || defined CONFIG_CMA |
b7aba6984 mm: compaction: a... |
35 36 |
#define CREATE_TRACE_POINTS #include <trace/events/compaction.h> |
748446bb6 mm: compaction: m... |
37 38 39 40 41 42 43 44 45 46 47 48 49 |
static unsigned long release_freepages(struct list_head *freelist) { struct page *page, *next; unsigned long count = 0; list_for_each_entry_safe(page, next, freelist, lru) { list_del(&page->lru); __free_page(page); count++; } return count; } |
ff9543fd3 mm: compaction: e... |
50 51 52 53 54 55 56 57 58 |
static void map_pages(struct list_head *list) { struct page *page; list_for_each_entry(page, list, lru) { arch_alloc_page(page, 0); kernel_map_pages(page, 1, 1); } } |
47118af07 mm: mmzone: MIGRA... |
59 60 61 62 |
static inline bool migrate_async_suitable(int migratetype) { return is_migrate_cma(migratetype) || migratetype == MIGRATE_MOVABLE; } |
bb13ffeb9 mm: compaction: c... |
63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 |
#ifdef CONFIG_COMPACTION /* Returns true if the pageblock should be scanned for pages to isolate. */ static inline bool isolation_suitable(struct compact_control *cc, struct page *page) { if (cc->ignore_skip_hint) return true; return !get_pageblock_skip(page); } /* * This function is called to clear all cached information on pageblocks that * should be skipped for page isolation when the migrate and free page scanner * meet. */ |
62997027c mm: compaction: c... |
79 |
static void __reset_isolation_suitable(struct zone *zone) |
bb13ffeb9 mm: compaction: c... |
80 81 |
{ unsigned long start_pfn = zone->zone_start_pfn; |
108bcc96e mm: add & use zon... |
82 |
unsigned long end_pfn = zone_end_pfn(zone); |
bb13ffeb9 mm: compaction: c... |
83 |
unsigned long pfn; |
c89511ab2 mm: compaction: R... |
84 85 |
zone->compact_cached_migrate_pfn = start_pfn; zone->compact_cached_free_pfn = end_pfn; |
62997027c mm: compaction: c... |
86 |
zone->compact_blockskip_flush = false; |
bb13ffeb9 mm: compaction: c... |
87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 |
/* Walk the zone and mark every pageblock as suitable for isolation */ for (pfn = start_pfn; pfn < end_pfn; pfn += pageblock_nr_pages) { struct page *page; cond_resched(); if (!pfn_valid(pfn)) continue; page = pfn_to_page(pfn); if (zone != page_zone(page)) continue; clear_pageblock_skip(page); } } |
62997027c mm: compaction: c... |
104 105 106 107 108 109 110 111 112 113 114 115 116 117 |
void reset_isolation_suitable(pg_data_t *pgdat) { int zoneid; for (zoneid = 0; zoneid < MAX_NR_ZONES; zoneid++) { struct zone *zone = &pgdat->node_zones[zoneid]; if (!populated_zone(zone)) continue; /* Only flush if a full compaction finished recently */ if (zone->compact_blockskip_flush) __reset_isolation_suitable(zone); } } |
bb13ffeb9 mm: compaction: c... |
118 119 |
/* * If no pages were isolated then mark this pageblock to be skipped in the |
62997027c mm: compaction: c... |
120 |
* future. The information is later cleared by __reset_isolation_suitable(). |
bb13ffeb9 mm: compaction: c... |
121 |
*/ |
c89511ab2 mm: compaction: R... |
122 123 124 |
static void update_pageblock_skip(struct compact_control *cc, struct page *page, unsigned long nr_isolated, bool migrate_scanner) |
bb13ffeb9 mm: compaction: c... |
125 |
{ |
c89511ab2 mm: compaction: R... |
126 |
struct zone *zone = cc->zone; |
6815bf3f2 mm/compaction: re... |
127 128 129 |
if (cc->ignore_skip_hint) return; |
bb13ffeb9 mm: compaction: c... |
130 131 |
if (!page) return; |
c89511ab2 mm: compaction: R... |
132 133 |
if (!nr_isolated) { unsigned long pfn = page_to_pfn(page); |
bb13ffeb9 mm: compaction: c... |
134 |
set_pageblock_skip(page); |
c89511ab2 mm: compaction: R... |
135 136 137 138 139 140 141 142 143 144 145 146 |
/* Update where compaction should restart */ if (migrate_scanner) { if (!cc->finished_update_migrate && pfn > zone->compact_cached_migrate_pfn) zone->compact_cached_migrate_pfn = pfn; } else { if (!cc->finished_update_free && pfn < zone->compact_cached_free_pfn) zone->compact_cached_free_pfn = pfn; } } |
bb13ffeb9 mm: compaction: c... |
147 148 149 150 151 152 153 |
} #else static inline bool isolation_suitable(struct compact_control *cc, struct page *page) { return true; } |
c89511ab2 mm: compaction: R... |
154 155 156 |
static void update_pageblock_skip(struct compact_control *cc, struct page *page, unsigned long nr_isolated, bool migrate_scanner) |
bb13ffeb9 mm: compaction: c... |
157 158 159 |
{ } #endif /* CONFIG_COMPACTION */ |
2a1402aa0 mm: compaction: a... |
160 161 162 163 |
static inline bool should_release_lock(spinlock_t *lock) { return need_resched() || spin_is_contended(lock); } |
85aa125f0 mm: compaction: i... |
164 |
/* |
c67fe3752 mm: compaction: A... |
165 166 167 168 169 170 171 172 173 174 175 |
* Compaction requires the taking of some coarse locks that are potentially * very heavily contended. Check if the process needs to be scheduled or * if the lock is contended. For async compaction, back out in the event * if contention is severe. For sync compaction, schedule. * * Returns true if the lock is held. * Returns false if the lock is released and compaction should abort */ static bool compact_checklock_irqsave(spinlock_t *lock, unsigned long *flags, bool locked, struct compact_control *cc) { |
2a1402aa0 mm: compaction: a... |
176 |
if (should_release_lock(lock)) { |
c67fe3752 mm: compaction: A... |
177 178 179 180 181 182 183 |
if (locked) { spin_unlock_irqrestore(lock, *flags); locked = false; } /* async aborts if taking too long or contended */ if (!cc->sync) { |
e64c5237c mm: compaction: a... |
184 |
cc->contended = true; |
c67fe3752 mm: compaction: A... |
185 186 187 188 |
return false; } cond_resched(); |
c67fe3752 mm: compaction: A... |
189 190 191 192 193 194 195 196 197 198 199 200 |
} if (!locked) spin_lock_irqsave(lock, *flags); return true; } static inline bool compact_trylock_irqsave(spinlock_t *lock, unsigned long *flags, struct compact_control *cc) { return compact_checklock_irqsave(lock, flags, false, cc); } |
f40d1e42b mm: compaction: a... |
201 202 203 |
/* Returns true if the page is within a block suitable for migration to */ static bool suitable_migration_target(struct page *page) { |
7d348b9ea mm/compaction: di... |
204 |
/* If the page is a large free page, then disallow migration */ |
f40d1e42b mm: compaction: a... |
205 |
if (PageBuddy(page) && page_order(page) >= pageblock_order) |
7d348b9ea mm/compaction: di... |
206 |
return false; |
f40d1e42b mm: compaction: a... |
207 208 |
/* If the block is MIGRATE_MOVABLE or MIGRATE_CMA, allow migration */ |
7d348b9ea mm/compaction: di... |
209 |
if (migrate_async_suitable(get_pageblock_migratetype(page))) |
f40d1e42b mm: compaction: a... |
210 211 212 213 214 |
return true; /* Otherwise skip the block */ return false; } |
c67fe3752 mm: compaction: A... |
215 |
/* |
9e4be4708 mm/compaction.c: ... |
216 217 218 |
* Isolate free pages onto a private freelist. If @strict is true, will abort * returning 0 on any invalid PFNs or non-free pages inside of the pageblock * (even though it may still end up isolating some pages). |
85aa125f0 mm: compaction: i... |
219 |
*/ |
f40d1e42b mm: compaction: a... |
220 221 |
static unsigned long isolate_freepages_block(struct compact_control *cc, unsigned long blockpfn, |
85aa125f0 mm: compaction: i... |
222 223 224 |
unsigned long end_pfn, struct list_head *freelist, bool strict) |
748446bb6 mm: compaction: m... |
225 |
{ |
b7aba6984 mm: compaction: a... |
226 |
int nr_scanned = 0, total_isolated = 0; |
bb13ffeb9 mm: compaction: c... |
227 |
struct page *cursor, *valid_page = NULL; |
f40d1e42b mm: compaction: a... |
228 229 |
unsigned long flags; bool locked = false; |
01ead5340 mm/compaction: do... |
230 |
bool checked_pageblock = false; |
748446bb6 mm: compaction: m... |
231 |
|
748446bb6 mm: compaction: m... |
232 |
cursor = pfn_to_page(blockpfn); |
f40d1e42b mm: compaction: a... |
233 |
/* Isolate free pages. */ |
748446bb6 mm: compaction: m... |
234 235 236 |
for (; blockpfn < end_pfn; blockpfn++, cursor++) { int isolated, i; struct page *page = cursor; |
b7aba6984 mm: compaction: a... |
237 |
nr_scanned++; |
f40d1e42b mm: compaction: a... |
238 |
if (!pfn_valid_within(blockpfn)) |
2af120bc0 mm/compaction: br... |
239 |
goto isolate_fail; |
bb13ffeb9 mm: compaction: c... |
240 241 |
if (!valid_page) valid_page = page; |
f40d1e42b mm: compaction: a... |
242 |
if (!PageBuddy(page)) |
2af120bc0 mm/compaction: br... |
243 |
goto isolate_fail; |
f40d1e42b mm: compaction: a... |
244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 |
/* * The zone lock must be held to isolate freepages. * Unfortunately this is a very coarse lock and can be * heavily contended if there are parallel allocations * or parallel compactions. For async compaction do not * spin on the lock and we acquire the lock as late as * possible. */ locked = compact_checklock_irqsave(&cc->zone->lock, &flags, locked, cc); if (!locked) break; /* Recheck this is a suitable migration target under lock */ |
01ead5340 mm/compaction: do... |
259 260 261 262 263 264 265 266 267 268 |
if (!strict && !checked_pageblock) { /* * We need to check suitability of pageblock only once * and this isolate_freepages_block() is called with * pageblock range, so just check once is sufficient. */ checked_pageblock = true; if (!suitable_migration_target(page)) break; } |
748446bb6 mm: compaction: m... |
269 |
|
f40d1e42b mm: compaction: a... |
270 271 |
/* Recheck this is a buddy page under lock */ if (!PageBuddy(page)) |
2af120bc0 mm/compaction: br... |
272 |
goto isolate_fail; |
748446bb6 mm: compaction: m... |
273 274 275 276 277 278 279 280 281 282 283 284 285 |
/* Found a free page, break it into order-0 pages */ isolated = split_free_page(page); total_isolated += isolated; for (i = 0; i < isolated; i++) { list_add(&page->lru, freelist); page++; } /* If a page was split, advance to the end of it */ if (isolated) { blockpfn += isolated - 1; cursor += isolated - 1; |
2af120bc0 mm/compaction: br... |
286 |
continue; |
748446bb6 mm: compaction: m... |
287 |
} |
2af120bc0 mm/compaction: br... |
288 289 290 291 292 293 |
isolate_fail: if (strict) break; else continue; |
748446bb6 mm: compaction: m... |
294 |
} |
b7aba6984 mm: compaction: a... |
295 |
trace_mm_compaction_isolate_freepages(nr_scanned, total_isolated); |
f40d1e42b mm: compaction: a... |
296 297 298 299 300 301 |
/* * If strict isolation is requested by CMA then check that all the * pages requested were isolated. If there were any failures, 0 is * returned and CMA will fail. */ |
2af120bc0 mm/compaction: br... |
302 |
if (strict && blockpfn < end_pfn) |
f40d1e42b mm: compaction: a... |
303 304 305 306 |
total_isolated = 0; if (locked) spin_unlock_irqrestore(&cc->zone->lock, flags); |
bb13ffeb9 mm: compaction: c... |
307 308 |
/* Update the pageblock-skip if the whole pageblock was scanned */ if (blockpfn == end_pfn) |
c89511ab2 mm: compaction: R... |
309 |
update_pageblock_skip(cc, valid_page, total_isolated, false); |
bb13ffeb9 mm: compaction: c... |
310 |
|
010fc29a4 compaction: fix b... |
311 |
count_compact_events(COMPACTFREE_SCANNED, nr_scanned); |
397487db6 mm: compaction: A... |
312 |
if (total_isolated) |
010fc29a4 compaction: fix b... |
313 |
count_compact_events(COMPACTISOLATED, total_isolated); |
748446bb6 mm: compaction: m... |
314 315 |
return total_isolated; } |
85aa125f0 mm: compaction: i... |
316 317 318 319 320 321 322 323 324 325 326 327 328 |
/** * isolate_freepages_range() - isolate free pages. * @start_pfn: The first PFN to start isolating. * @end_pfn: The one-past-last PFN. * * Non-free pages, invalid PFNs, or zone boundaries within the * [start_pfn, end_pfn) range are considered errors, cause function to * undo its actions and return zero. * * Otherwise, function returns one-past-the-last PFN of isolated page * (which may be greater then end_pfn if end fell in a middle of * a free page). */ |
ff9543fd3 mm: compaction: e... |
329 |
unsigned long |
bb13ffeb9 mm: compaction: c... |
330 331 |
isolate_freepages_range(struct compact_control *cc, unsigned long start_pfn, unsigned long end_pfn) |
85aa125f0 mm: compaction: i... |
332 |
{ |
f40d1e42b mm: compaction: a... |
333 |
unsigned long isolated, pfn, block_end_pfn; |
85aa125f0 mm: compaction: i... |
334 |
LIST_HEAD(freelist); |
85aa125f0 mm: compaction: i... |
335 |
for (pfn = start_pfn; pfn < end_pfn; pfn += isolated) { |
bb13ffeb9 mm: compaction: c... |
336 |
if (!pfn_valid(pfn) || cc->zone != page_zone(pfn_to_page(pfn))) |
85aa125f0 mm: compaction: i... |
337 338 339 340 341 342 343 344 |
break; /* * On subsequent iterations ALIGN() is actually not needed, * but we keep it that we not to complicate the code. */ block_end_pfn = ALIGN(pfn + 1, pageblock_nr_pages); block_end_pfn = min(block_end_pfn, end_pfn); |
bb13ffeb9 mm: compaction: c... |
345 |
isolated = isolate_freepages_block(cc, pfn, block_end_pfn, |
85aa125f0 mm: compaction: i... |
346 |
&freelist, true); |
85aa125f0 mm: compaction: i... |
347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 |
/* * In strict mode, isolate_freepages_block() returns 0 if * there are any holes in the block (ie. invalid PFNs or * non-free pages). */ if (!isolated) break; /* * If we managed to isolate pages, it is always (1 << n) * * pageblock_nr_pages for some non-negative n. (Max order * page may span two pageblocks). */ } /* split_free_page does not map the pages */ map_pages(&freelist); if (pfn < end_pfn) { /* Loop terminated early, cleanup. */ release_freepages(&freelist); return 0; } /* We don't use freelists for anything. */ return pfn; } |
748446bb6 mm: compaction: m... |
375 |
/* Update the number of anon and file isolated pages in the zone */ |
c67fe3752 mm: compaction: A... |
376 |
static void acct_isolated(struct zone *zone, bool locked, struct compact_control *cc) |
748446bb6 mm: compaction: m... |
377 378 |
{ struct page *page; |
b9e84ac15 mm: compaction: t... |
379 |
unsigned int count[2] = { 0, }; |
748446bb6 mm: compaction: m... |
380 |
|
b9e84ac15 mm: compaction: t... |
381 382 |
list_for_each_entry(page, &cc->migratepages, lru) count[!!page_is_file_cache(page)]++; |
748446bb6 mm: compaction: m... |
383 |
|
c67fe3752 mm: compaction: A... |
384 385 386 387 388 389 390 391 |
/* If locked we can use the interrupt unsafe versions */ if (locked) { __mod_zone_page_state(zone, NR_ISOLATED_ANON, count[0]); __mod_zone_page_state(zone, NR_ISOLATED_FILE, count[1]); } else { mod_zone_page_state(zone, NR_ISOLATED_ANON, count[0]); mod_zone_page_state(zone, NR_ISOLATED_FILE, count[1]); } |
748446bb6 mm: compaction: m... |
392 393 394 395 396 |
} /* Similar to reclaim, but different enough that they don't share logic */ static bool too_many_isolated(struct zone *zone) { |
bc6930457 mm: compaction: h... |
397 |
unsigned long active, inactive, isolated; |
748446bb6 mm: compaction: m... |
398 399 400 |
inactive = zone_page_state(zone, NR_INACTIVE_FILE) + zone_page_state(zone, NR_INACTIVE_ANON); |
bc6930457 mm: compaction: h... |
401 402 |
active = zone_page_state(zone, NR_ACTIVE_FILE) + zone_page_state(zone, NR_ACTIVE_ANON); |
748446bb6 mm: compaction: m... |
403 404 |
isolated = zone_page_state(zone, NR_ISOLATED_FILE) + zone_page_state(zone, NR_ISOLATED_ANON); |
bc6930457 mm: compaction: h... |
405 |
return isolated > (inactive + active) / 2; |
748446bb6 mm: compaction: m... |
406 |
} |
2fe86e000 mm: compaction: i... |
407 408 409 410 411 412 |
/** * isolate_migratepages_range() - isolate all migrate-able pages in range. * @zone: Zone pages are in. * @cc: Compaction control structure. * @low_pfn: The first PFN of the range. * @end_pfn: The one-past-the-last PFN of the range. |
e46a28790 CMA: migrate mloc... |
413 |
* @unevictable: true if it allows to isolate unevictable pages |
2fe86e000 mm: compaction: i... |
414 415 416 417 418 419 420 421 422 423 424 425 |
* * Isolate all pages that can be migrated from the range specified by * [low_pfn, end_pfn). Returns zero if there is a fatal signal * pending), otherwise PFN of the first page that was not scanned * (which may be both less, equal to or more then end_pfn). * * Assumes that cc->migratepages is empty and cc->nr_migratepages is * zero. * * Apart from cc->migratepages and cc->nr_migratetypes this function * does not modify any cc's fields, in particular it does not modify * (or read for that matter) cc->migrate_pfn. |
748446bb6 mm: compaction: m... |
426 |
*/ |
ff9543fd3 mm: compaction: e... |
427 |
unsigned long |
2fe86e000 mm: compaction: i... |
428 |
isolate_migratepages_range(struct zone *zone, struct compact_control *cc, |
e46a28790 CMA: migrate mloc... |
429 |
unsigned long low_pfn, unsigned long end_pfn, bool unevictable) |
748446bb6 mm: compaction: m... |
430 |
{ |
9927af740 mm: compaction: p... |
431 |
unsigned long last_pageblock_nr = 0, pageblock_nr; |
b7aba6984 mm: compaction: a... |
432 |
unsigned long nr_scanned = 0, nr_isolated = 0; |
748446bb6 mm: compaction: m... |
433 |
struct list_head *migratelist = &cc->migratepages; |
fa9add641 mm/memcg: apply a... |
434 |
struct lruvec *lruvec; |
c67fe3752 mm: compaction: A... |
435 |
unsigned long flags; |
2a1402aa0 mm: compaction: a... |
436 |
bool locked = false; |
bb13ffeb9 mm: compaction: c... |
437 |
struct page *page = NULL, *valid_page = NULL; |
50b5b094e mm: compaction: d... |
438 |
bool skipped_async_unsuitable = false; |
da1c67a76 mm, compaction: d... |
439 440 |
const isolate_mode_t mode = (!cc->sync ? ISOLATE_ASYNC_MIGRATE : 0) | (unevictable ? ISOLATE_UNEVICTABLE : 0); |
748446bb6 mm: compaction: m... |
441 |
|
748446bb6 mm: compaction: m... |
442 443 444 445 446 447 |
/* * Ensure that there are not too many pages isolated from the LRU * list by either parallel reclaimers or compaction. If there are, * delay for some time until fewer pages are isolated */ while (unlikely(too_many_isolated(zone))) { |
f9e35b3b4 mm: compaction: a... |
448 |
/* async migration should just abort */ |
68e3e9262 Revert "mm: compa... |
449 |
if (!cc->sync) |
2fe86e000 mm: compaction: i... |
450 |
return 0; |
f9e35b3b4 mm: compaction: a... |
451 |
|
748446bb6 mm: compaction: m... |
452 453 454 |
congestion_wait(BLK_RW_ASYNC, HZ/10); if (fatal_signal_pending(current)) |
2fe86e000 mm: compaction: i... |
455 |
return 0; |
748446bb6 mm: compaction: m... |
456 457 458 |
} /* Time to isolate some pages for migration */ |
b2eef8c0d mm: compaction: m... |
459 |
cond_resched(); |
748446bb6 mm: compaction: m... |
460 |
for (; low_pfn < end_pfn; low_pfn++) { |
b2eef8c0d mm: compaction: m... |
461 |
/* give a chance to irqs before checking need_resched() */ |
be1aa03b9 mm/compaction: ch... |
462 |
if (locked && !(low_pfn % SWAP_CLUSTER_MAX)) { |
2a1402aa0 mm: compaction: a... |
463 464 465 466 |
if (should_release_lock(&zone->lru_lock)) { spin_unlock_irqrestore(&zone->lru_lock, flags); locked = false; } |
b2eef8c0d mm: compaction: m... |
467 |
} |
c67fe3752 mm: compaction: A... |
468 |
|
0bf380bc7 mm: compaction: c... |
469 470 471 472 473 474 475 476 477 478 479 480 |
/* * migrate_pfn does not necessarily start aligned to a * pageblock. Ensure that pfn_valid is called when moving * into a new MAX_ORDER_NR_PAGES range in case of large * memory holes within the zone */ if ((low_pfn & (MAX_ORDER_NR_PAGES - 1)) == 0) { if (!pfn_valid(low_pfn)) { low_pfn += MAX_ORDER_NR_PAGES - 1; continue; } } |
748446bb6 mm: compaction: m... |
481 482 |
if (!pfn_valid_within(low_pfn)) continue; |
b7aba6984 mm: compaction: a... |
483 |
nr_scanned++; |
748446bb6 mm: compaction: m... |
484 |
|
dc9086004 mm: compaction: c... |
485 486 487 488 489 490 |
/* * Get the page and ensure the page is within the same zone. * See the comment in isolate_freepages about overlapping * nodes. It is deliberate that the new zone lock is not taken * as memory compaction should not move pages between nodes. */ |
748446bb6 mm: compaction: m... |
491 |
page = pfn_to_page(low_pfn); |
dc9086004 mm: compaction: c... |
492 493 |
if (page_zone(page) != zone) continue; |
bb13ffeb9 mm: compaction: c... |
494 495 496 497 498 |
if (!valid_page) valid_page = page; /* If isolation recently failed, do not retry */ pageblock_nr = low_pfn >> pageblock_order; |
c122b2087 mm/compaction: ch... |
499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 |
if (last_pageblock_nr != pageblock_nr) { int mt; last_pageblock_nr = pageblock_nr; if (!isolation_suitable(cc, page)) goto next_pageblock; /* * For async migration, also only scan in MOVABLE * blocks. Async migration is optimistic to see if * the minimum amount of work satisfies the allocation */ mt = get_pageblock_migratetype(page); if (!cc->sync && !migrate_async_suitable(mt)) { cc->finished_update_migrate = true; skipped_async_unsuitable = true; goto next_pageblock; } } |
bb13ffeb9 mm: compaction: c... |
518 |
|
6c14466cc mm: improve docum... |
519 520 521 522 |
/* * Skip if free. page_order cannot be used without zone->lock * as nothing prevents parallel allocations or buddy merging. */ |
748446bb6 mm: compaction: m... |
523 524 |
if (PageBuddy(page)) continue; |
9927af740 mm: compaction: p... |
525 |
/* |
bf6bddf19 mm: introduce com... |
526 527 528 529 530 531 532 533 |
* Check may be lockless but that's ok as we recheck later. * It's possible to migrate LRU pages and balloon pages * Skip any other type of page */ if (!PageLRU(page)) { if (unlikely(balloon_page_movable(page))) { if (locked && balloon_page_isolate(page)) { /* Successfully isolated */ |
b6c750163 mm/compaction: cl... |
534 |
goto isolate_success; |
bf6bddf19 mm: introduce com... |
535 536 |
} } |
bc835011a thp: transhuge is... |
537 |
continue; |
bf6bddf19 mm: introduce com... |
538 |
} |
bc835011a thp: transhuge is... |
539 540 |
/* |
2a1402aa0 mm: compaction: a... |
541 542 543 544 545 546 547 548 |
* PageLRU is set. lru_lock normally excludes isolation * splitting and collapsing (collapsing has already happened * if PageLRU is set) but the lock is not necessarily taken * here and it is wasteful to take it just to check transhuge. * Check TransHuge without lock and skip the whole pageblock if * it's either a transhuge or hugetlbfs page, as calling * compound_order() without preventing THP from splitting the * page underneath us may return surprising results. |
bc835011a thp: transhuge is... |
549 550 |
*/ if (PageTransHuge(page)) { |
2a1402aa0 mm: compaction: a... |
551 552 553 554 555 |
if (!locked) goto next_pageblock; low_pfn += (1 << compound_order(page)) - 1; continue; } |
119d6d59d mm, compaction: a... |
556 557 558 559 560 561 562 563 |
/* * Migration will fail if an anonymous page is pinned in memory, * so avoid taking lru_lock and isolating it unnecessarily in an * admittedly racy check. */ if (!page_mapping(page) && page_count(page) > page_mapcount(page)) continue; |
2a1402aa0 mm: compaction: a... |
564 565 566 567 568 569 570 571 572 573 |
/* Check if it is ok to still hold the lock */ locked = compact_checklock_irqsave(&zone->lru_lock, &flags, locked, cc); if (!locked || fatal_signal_pending(current)) break; /* Recheck PageLRU and PageTransHuge under lock */ if (!PageLRU(page)) continue; if (PageTransHuge(page)) { |
bc835011a thp: transhuge is... |
574 575 576 |
low_pfn += (1 << compound_order(page)) - 1; continue; } |
fa9add641 mm/memcg: apply a... |
577 |
lruvec = mem_cgroup_page_lruvec(page, zone); |
748446bb6 mm: compaction: m... |
578 |
/* Try isolate the page */ |
f3fd4a619 mm: remove lru ty... |
579 |
if (__isolate_lru_page(page, mode) != 0) |
748446bb6 mm: compaction: m... |
580 |
continue; |
309381fea mm: dump page whe... |
581 |
VM_BUG_ON_PAGE(PageTransCompound(page), page); |
bc835011a thp: transhuge is... |
582 |
|
748446bb6 mm: compaction: m... |
583 |
/* Successfully isolated */ |
fa9add641 mm/memcg: apply a... |
584 |
del_page_from_lru_list(page, lruvec, page_lru(page)); |
b6c750163 mm/compaction: cl... |
585 586 587 |
isolate_success: cc->finished_update_migrate = true; |
748446bb6 mm: compaction: m... |
588 |
list_add(&page->lru, migratelist); |
748446bb6 mm: compaction: m... |
589 |
cc->nr_migratepages++; |
b7aba6984 mm: compaction: a... |
590 |
nr_isolated++; |
748446bb6 mm: compaction: m... |
591 592 |
/* Avoid isolating too much */ |
31b8384a5 mm: compaction: p... |
593 594 |
if (cc->nr_migratepages == COMPACT_CLUSTER_MAX) { ++low_pfn; |
748446bb6 mm: compaction: m... |
595 |
break; |
31b8384a5 mm: compaction: p... |
596 |
} |
2a1402aa0 mm: compaction: a... |
597 598 599 600 |
continue; next_pageblock: |
a9aacbccf mm: compaction: d... |
601 |
low_pfn = ALIGN(low_pfn + 1, pageblock_nr_pages) - 1; |
748446bb6 mm: compaction: m... |
602 |
} |
c67fe3752 mm: compaction: A... |
603 |
acct_isolated(zone, locked, cc); |
748446bb6 mm: compaction: m... |
604 |
|
c67fe3752 mm: compaction: A... |
605 606 |
if (locked) spin_unlock_irqrestore(&zone->lru_lock, flags); |
748446bb6 mm: compaction: m... |
607 |
|
50b5b094e mm: compaction: d... |
608 609 610 611 612 613 614 |
/* * Update the pageblock-skip information and cached scanner pfn, * if the whole pageblock was scanned without isolating any page. * This is not done when pageblock was skipped due to being unsuitable * for async compaction, so that eventual sync compaction can try. */ if (low_pfn == end_pfn && !skipped_async_unsuitable) |
c89511ab2 mm: compaction: R... |
615 |
update_pageblock_skip(cc, valid_page, nr_isolated, true); |
bb13ffeb9 mm: compaction: c... |
616 |
|
b7aba6984 mm: compaction: a... |
617 |
trace_mm_compaction_isolate_migratepages(nr_scanned, nr_isolated); |
010fc29a4 compaction: fix b... |
618 |
count_compact_events(COMPACTMIGRATE_SCANNED, nr_scanned); |
397487db6 mm: compaction: A... |
619 |
if (nr_isolated) |
010fc29a4 compaction: fix b... |
620 |
count_compact_events(COMPACTISOLATED, nr_isolated); |
397487db6 mm: compaction: A... |
621 |
|
2fe86e000 mm: compaction: i... |
622 623 |
return low_pfn; } |
ff9543fd3 mm: compaction: e... |
624 625 |
#endif /* CONFIG_COMPACTION || CONFIG_CMA */ #ifdef CONFIG_COMPACTION |
2fe86e000 mm: compaction: i... |
626 |
/* |
ff9543fd3 mm: compaction: e... |
627 628 |
* Based on information in the current compact_control, find blocks * suitable for isolating free pages from and then isolate them. |
2fe86e000 mm: compaction: i... |
629 |
*/ |
ff9543fd3 mm: compaction: e... |
630 631 |
static void isolate_freepages(struct zone *zone, struct compact_control *cc) |
2fe86e000 mm: compaction: i... |
632 |
{ |
ff9543fd3 mm: compaction: e... |
633 |
struct page *page; |
108bcc96e mm: add & use zon... |
634 |
unsigned long high_pfn, low_pfn, pfn, z_end_pfn, end_pfn; |
ff9543fd3 mm: compaction: e... |
635 636 |
int nr_freepages = cc->nr_freepages; struct list_head *freelist = &cc->freepages; |
2fe86e000 mm: compaction: i... |
637 |
|
ff9543fd3 mm: compaction: e... |
638 639 640 641 642 643 |
/* * Initialise the free scanner. The starting point is where we last * scanned from (or the end of the zone if starting). The low point * is the end of the pageblock the migration scanner is using. */ pfn = cc->free_pfn; |
7ed695e06 mm: compaction: d... |
644 |
low_pfn = ALIGN(cc->migrate_pfn + 1, pageblock_nr_pages); |
2fe86e000 mm: compaction: i... |
645 |
|
ff9543fd3 mm: compaction: e... |
646 647 648 649 650 651 |
/* * Take care that if the migration scanner is at the end of the zone * that the free scanner does not accidentally move to the next zone * in the next isolation cycle. */ high_pfn = min(low_pfn, pfn); |
2fe86e000 mm: compaction: i... |
652 |
|
108bcc96e mm: add & use zon... |
653 |
z_end_pfn = zone_end_pfn(zone); |
2fe86e000 mm: compaction: i... |
654 |
|
ff9543fd3 mm: compaction: e... |
655 656 657 658 659 |
/* * Isolate free pages until enough are available to migrate the * pages on cc->migratepages. We stop searching if the migrate * and free page scanners meet or enough free pages are isolated. */ |
7ed695e06 mm: compaction: d... |
660 |
for (; pfn >= low_pfn && cc->nr_migratepages > nr_freepages; |
ff9543fd3 mm: compaction: e... |
661 662 |
pfn -= pageblock_nr_pages) { unsigned long isolated; |
2fe86e000 mm: compaction: i... |
663 |
|
f6ea3adb7 mm/compaction.c: ... |
664 665 666 667 668 669 |
/* * This can iterate a massively long zone without finding any * suitable migration targets, so periodically check if we need * to schedule. */ cond_resched(); |
ff9543fd3 mm: compaction: e... |
670 671 |
if (!pfn_valid(pfn)) continue; |
2fe86e000 mm: compaction: i... |
672 |
|
ff9543fd3 mm: compaction: e... |
673 674 675 676 677 678 679 680 681 682 683 684 |
/* * Check for overlapping nodes/zones. It's possible on some * configurations to have a setup like * node0 node1 node0 * i.e. it's possible that all pages within a zones range of * pages do not belong to a single zone. */ page = pfn_to_page(pfn); if (page_zone(page) != zone) continue; /* Check the block is suitable for migration */ |
68e3e9262 Revert "mm: compa... |
685 |
if (!suitable_migration_target(page)) |
ff9543fd3 mm: compaction: e... |
686 |
continue; |
68e3e9262 Revert "mm: compa... |
687 |
|
bb13ffeb9 mm: compaction: c... |
688 689 690 |
/* If isolation recently failed, do not retry */ if (!isolation_suitable(cc, page)) continue; |
f40d1e42b mm: compaction: a... |
691 |
/* Found a block suitable for isolating free pages from */ |
ff9543fd3 mm: compaction: e... |
692 |
isolated = 0; |
60177d31d mm: compaction: v... |
693 694 695 696 697 698 699 700 |
/* * As pfn may not start aligned, pfn+pageblock_nr_page * may cross a MAX_ORDER_NR_PAGES boundary and miss * a pfn_valid check. Ensure isolate_freepages_block() * only scans within a pageblock */ end_pfn = ALIGN(pfn + 1, pageblock_nr_pages); |
108bcc96e mm: add & use zon... |
701 |
end_pfn = min(end_pfn, z_end_pfn); |
f40d1e42b mm: compaction: a... |
702 703 704 |
isolated = isolate_freepages_block(cc, pfn, end_pfn, freelist, false); nr_freepages += isolated; |
ff9543fd3 mm: compaction: e... |
705 706 707 708 709 710 |
/* * Record the highest PFN we isolated pages from. When next * looking for free pages, the search will restart here as * page migration may have returned some pages to the allocator */ |
c89511ab2 mm: compaction: R... |
711 712 |
if (isolated) { cc->finished_update_free = true; |
ff9543fd3 mm: compaction: e... |
713 |
high_pfn = max(high_pfn, pfn); |
c89511ab2 mm: compaction: R... |
714 |
} |
ff9543fd3 mm: compaction: e... |
715 716 717 718 |
} /* split_free_page does not map the pages */ map_pages(freelist); |
7ed695e06 mm: compaction: d... |
719 720 721 722 723 724 725 726 |
/* * If we crossed the migrate scanner, we want to keep it that way * so that compact_finished() may detect this */ if (pfn < low_pfn) cc->free_pfn = max(pfn, zone->zone_start_pfn); else cc->free_pfn = high_pfn; |
ff9543fd3 mm: compaction: e... |
727 |
cc->nr_freepages = nr_freepages; |
748446bb6 mm: compaction: m... |
728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 |
} /* * This is a migrate-callback that "allocates" freepages by taking pages * from the isolated freelists in the block we are migrating to. */ static struct page *compaction_alloc(struct page *migratepage, unsigned long data, int **result) { struct compact_control *cc = (struct compact_control *)data; struct page *freepage; /* Isolate free pages if necessary */ if (list_empty(&cc->freepages)) { isolate_freepages(cc->zone, cc); if (list_empty(&cc->freepages)) return NULL; } freepage = list_entry(cc->freepages.next, struct page, lru); list_del(&freepage->lru); cc->nr_freepages--; return freepage; } /* * We cannot control nr_migratepages and nr_freepages fully when migration is * running as migrate_pages() has no knowledge of compact_control. When * migration is complete, we count the number of pages on the lists by hand. */ static void update_nr_listpages(struct compact_control *cc) { int nr_migratepages = 0; int nr_freepages = 0; struct page *page; list_for_each_entry(page, &cc->migratepages, lru) nr_migratepages++; list_for_each_entry(page, &cc->freepages, lru) nr_freepages++; cc->nr_migratepages = nr_migratepages; cc->nr_freepages = nr_freepages; } |
ff9543fd3 mm: compaction: e... |
775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 |
/* possible outcome of isolate_migratepages */ typedef enum { ISOLATE_ABORT, /* Abort compaction now */ ISOLATE_NONE, /* No pages isolated, continue scanning */ ISOLATE_SUCCESS, /* Pages isolated, migrate */ } isolate_migrate_t; /* * Isolate all pages that can be migrated from the block pointed to by * the migrate scanner within compact_control. */ static isolate_migrate_t isolate_migratepages(struct zone *zone, struct compact_control *cc) { unsigned long low_pfn, end_pfn; /* Do not scan outside zone boundaries */ low_pfn = max(cc->migrate_pfn, zone->zone_start_pfn); /* Only scan within a pageblock boundary */ |
a9aacbccf mm: compaction: d... |
795 |
end_pfn = ALIGN(low_pfn + 1, pageblock_nr_pages); |
ff9543fd3 mm: compaction: e... |
796 797 798 799 800 801 802 803 |
/* Do not cross the free scanner or scan within a memory hole */ if (end_pfn > cc->free_pfn || !pfn_valid(low_pfn)) { cc->migrate_pfn = end_pfn; return ISOLATE_NONE; } /* Perform the isolation */ |
e46a28790 CMA: migrate mloc... |
804 |
low_pfn = isolate_migratepages_range(zone, cc, low_pfn, end_pfn, false); |
e64c5237c mm: compaction: a... |
805 |
if (!low_pfn || cc->contended) |
ff9543fd3 mm: compaction: e... |
806 807 808 809 810 811 |
return ISOLATE_ABORT; cc->migrate_pfn = low_pfn; return ISOLATE_SUCCESS; } |
748446bb6 mm: compaction: m... |
812 |
static int compact_finished(struct zone *zone, |
5a03b051e thp: use compacti... |
813 |
struct compact_control *cc) |
748446bb6 mm: compaction: m... |
814 |
{ |
8fb74b9fb mm: compaction: p... |
815 |
unsigned int order; |
5a03b051e thp: use compacti... |
816 |
unsigned long watermark; |
56de7263f mm: compaction: d... |
817 |
|
748446bb6 mm: compaction: m... |
818 819 |
if (fatal_signal_pending(current)) return COMPACT_PARTIAL; |
753341a4b revert "mm: have ... |
820 |
/* Compaction run completes if the migrate and free scanner meet */ |
bb13ffeb9 mm: compaction: c... |
821 |
if (cc->free_pfn <= cc->migrate_pfn) { |
55b7c4c99 mm: compaction: r... |
822 823 824 |
/* Let the next compaction start anew. */ zone->compact_cached_migrate_pfn = zone->zone_start_pfn; zone->compact_cached_free_pfn = zone_end_pfn(zone); |
62997027c mm: compaction: c... |
825 826 827 828 829 830 831 832 |
/* * Mark that the PG_migrate_skip information should be cleared * by kswapd when it goes to sleep. kswapd does not set the * flag itself as the decision to be clear should be directly * based on an allocation request. */ if (!current_is_kswapd()) zone->compact_blockskip_flush = true; |
748446bb6 mm: compaction: m... |
833 |
return COMPACT_COMPLETE; |
bb13ffeb9 mm: compaction: c... |
834 |
} |
748446bb6 mm: compaction: m... |
835 |
|
82478fb7b mm: compaction: p... |
836 837 838 839 |
/* * order == -1 is expected when compacting via * /proc/sys/vm/compact_memory */ |
56de7263f mm: compaction: d... |
840 841 |
if (cc->order == -1) return COMPACT_CONTINUE; |
3957c7768 mm: compaction: f... |
842 843 844 845 846 847 |
/* Compaction run is not finished if the watermark is not met */ watermark = low_wmark_pages(zone); watermark += (1 << cc->order); if (!zone_watermark_ok(zone, cc->order, watermark, 0, 0)) return COMPACT_CONTINUE; |
56de7263f mm: compaction: d... |
848 |
/* Direct compactor: Is a suitable page free? */ |
8fb74b9fb mm: compaction: p... |
849 850 851 852 853 854 855 856 857 |
for (order = cc->order; order < MAX_ORDER; order++) { struct free_area *area = &zone->free_area[order]; /* Job done if page is free of the right migratetype */ if (!list_empty(&area->free_list[cc->migratetype])) return COMPACT_PARTIAL; /* Job done if allocation would set block type */ if (cc->order >= pageblock_order && area->nr_free) |
56de7263f mm: compaction: d... |
858 859 |
return COMPACT_PARTIAL; } |
748446bb6 mm: compaction: m... |
860 861 |
return COMPACT_CONTINUE; } |
3e7d34497 mm: vmscan: recla... |
862 863 864 865 866 867 868 869 870 871 872 873 874 |
/* * compaction_suitable: Is this suitable to run compaction on this zone now? * Returns * COMPACT_SKIPPED - If there are too few free pages for compaction * COMPACT_PARTIAL - If the allocation would succeed without compaction * COMPACT_CONTINUE - If compaction should run now */ unsigned long compaction_suitable(struct zone *zone, int order) { int fragindex; unsigned long watermark; /* |
3957c7768 mm: compaction: f... |
875 876 877 878 879 880 881 |
* order == -1 is expected when compacting via * /proc/sys/vm/compact_memory */ if (order == -1) return COMPACT_CONTINUE; /* |
3e7d34497 mm: vmscan: recla... |
882 883 884 885 886 887 888 889 890 891 892 893 |
* Watermarks for order-0 must be met for compaction. Note the 2UL. * This is because during migration, copies of pages need to be * allocated and for a short time, the footprint is higher */ watermark = low_wmark_pages(zone) + (2UL << order); if (!zone_watermark_ok(zone, 0, watermark, 0, 0)) return COMPACT_SKIPPED; /* * fragmentation index determines if allocation failures are due to * low memory or external fragmentation * |
a582a738c compaction: check... |
894 895 |
* index of -1000 implies allocations might succeed depending on * watermarks |
3e7d34497 mm: vmscan: recla... |
896 897 898 899 900 901 902 903 |
* index towards 0 implies failure is due to lack of memory * index towards 1000 implies failure is due to fragmentation * * Only compact if a failure would be due to fragmentation. */ fragindex = fragmentation_index(zone, order); if (fragindex >= 0 && fragindex <= sysctl_extfrag_threshold) return COMPACT_SKIPPED; |
a582a738c compaction: check... |
904 905 |
if (fragindex == -1000 && zone_watermark_ok(zone, order, watermark, 0, 0)) |
3e7d34497 mm: vmscan: recla... |
906 907 908 909 |
return COMPACT_PARTIAL; return COMPACT_CONTINUE; } |
748446bb6 mm: compaction: m... |
910 911 912 |
static int compact_zone(struct zone *zone, struct compact_control *cc) { int ret; |
c89511ab2 mm: compaction: R... |
913 |
unsigned long start_pfn = zone->zone_start_pfn; |
108bcc96e mm: add & use zon... |
914 |
unsigned long end_pfn = zone_end_pfn(zone); |
748446bb6 mm: compaction: m... |
915 |
|
3e7d34497 mm: vmscan: recla... |
916 917 918 919 920 921 922 923 924 925 |
ret = compaction_suitable(zone, cc->order); switch (ret) { case COMPACT_PARTIAL: case COMPACT_SKIPPED: /* Compaction is likely to fail */ return ret; case COMPACT_CONTINUE: /* Fall through to compaction */ ; } |
c89511ab2 mm: compaction: R... |
926 |
/* |
d3132e4b8 mm: compaction: r... |
927 928 929 930 931 932 933 934 |
* Clear pageblock skip if there were failures recently and compaction * is about to be retried after being deferred. kswapd does not do * this reset as it'll reset the cached information when going to sleep. */ if (compaction_restarting(zone, cc->order) && !current_is_kswapd()) __reset_isolation_suitable(zone); /* |
c89511ab2 mm: compaction: R... |
935 936 937 938 939 940 941 942 943 944 945 946 947 948 |
* Setup to move all movable pages to the end of the zone. Used cached * information on where the scanners should start but check that it * is initialised by ensuring the values are within zone boundaries. */ cc->migrate_pfn = zone->compact_cached_migrate_pfn; cc->free_pfn = zone->compact_cached_free_pfn; if (cc->free_pfn < start_pfn || cc->free_pfn > end_pfn) { cc->free_pfn = end_pfn & ~(pageblock_nr_pages-1); zone->compact_cached_free_pfn = cc->free_pfn; } if (cc->migrate_pfn < start_pfn || cc->migrate_pfn > end_pfn) { cc->migrate_pfn = start_pfn; zone->compact_cached_migrate_pfn = cc->migrate_pfn; } |
748446bb6 mm: compaction: m... |
949 |
|
0eb927c0a mm: compaction: t... |
950 |
trace_mm_compaction_begin(start_pfn, cc->migrate_pfn, cc->free_pfn, end_pfn); |
748446bb6 mm: compaction: m... |
951 952 953 954 |
migrate_prep_local(); while ((ret = compact_finished(zone, cc)) == COMPACT_CONTINUE) { unsigned long nr_migrate, nr_remaining; |
9d502c1c8 mm/compaction: ch... |
955 |
int err; |
748446bb6 mm: compaction: m... |
956 |
|
f9e35b3b4 mm: compaction: a... |
957 958 959 |
switch (isolate_migratepages(zone, cc)) { case ISOLATE_ABORT: ret = COMPACT_PARTIAL; |
5733c7d11 mm: introduce put... |
960 |
putback_movable_pages(&cc->migratepages); |
e64c5237c mm: compaction: a... |
961 |
cc->nr_migratepages = 0; |
f9e35b3b4 mm: compaction: a... |
962 963 |
goto out; case ISOLATE_NONE: |
748446bb6 mm: compaction: m... |
964 |
continue; |
f9e35b3b4 mm: compaction: a... |
965 966 967 |
case ISOLATE_SUCCESS: ; } |
748446bb6 mm: compaction: m... |
968 969 |
nr_migrate = cc->nr_migratepages; |
9d502c1c8 mm/compaction: ch... |
970 |
err = migrate_pages(&cc->migratepages, compaction_alloc, |
9c620e2bc mm: remove offlin... |
971 |
(unsigned long)cc, |
7b2a2d4a1 mm: migrate: Add ... |
972 973 |
cc->sync ? MIGRATE_SYNC_LIGHT : MIGRATE_ASYNC, MR_COMPACTION); |
748446bb6 mm: compaction: m... |
974 975 |
update_nr_listpages(cc); nr_remaining = cc->nr_migratepages; |
b7aba6984 mm: compaction: a... |
976 977 |
trace_mm_compaction_migratepages(nr_migrate - nr_remaining, nr_remaining); |
748446bb6 mm: compaction: m... |
978 |
|
5733c7d11 mm: introduce put... |
979 |
/* Release isolated pages not migrated */ |
9d502c1c8 mm/compaction: ch... |
980 |
if (err) { |
5733c7d11 mm: introduce put... |
981 |
putback_movable_pages(&cc->migratepages); |
748446bb6 mm: compaction: m... |
982 |
cc->nr_migratepages = 0; |
7ed695e06 mm: compaction: d... |
983 984 985 986 987 |
/* * migrate_pages() may return -ENOMEM when scanners meet * and we want compact_finished() to detect it */ if (err == -ENOMEM && cc->free_pfn > cc->migrate_pfn) { |
4bf2bba37 mm, thp: abort co... |
988 989 990 |
ret = COMPACT_PARTIAL; goto out; } |
748446bb6 mm: compaction: m... |
991 |
} |
748446bb6 mm: compaction: m... |
992 |
} |
f9e35b3b4 mm: compaction: a... |
993 |
out: |
748446bb6 mm: compaction: m... |
994 995 996 |
/* Release free pages and check accounting */ cc->nr_freepages -= release_freepages(&cc->freepages); VM_BUG_ON(cc->nr_freepages != 0); |
0eb927c0a mm: compaction: t... |
997 |
trace_mm_compaction_end(ret); |
748446bb6 mm: compaction: m... |
998 999 |
return ret; } |
76ab0f530 mm: compaction: a... |
1000 |
|
d43a87e68 mm: compaction: m... |
1001 |
static unsigned long compact_zone_order(struct zone *zone, |
5a03b051e thp: use compacti... |
1002 |
int order, gfp_t gfp_mask, |
8fb74b9fb mm: compaction: p... |
1003 |
bool sync, bool *contended) |
56de7263f mm: compaction: d... |
1004 |
{ |
e64c5237c mm: compaction: a... |
1005 |
unsigned long ret; |
56de7263f mm: compaction: d... |
1006 1007 1008 1009 1010 1011 |
struct compact_control cc = { .nr_freepages = 0, .nr_migratepages = 0, .order = order, .migratetype = allocflags_to_migratetype(gfp_mask), .zone = zone, |
68e3e9262 Revert "mm: compa... |
1012 |
.sync = sync, |
56de7263f mm: compaction: d... |
1013 1014 1015 |
}; INIT_LIST_HEAD(&cc.freepages); INIT_LIST_HEAD(&cc.migratepages); |
e64c5237c mm: compaction: a... |
1016 1017 1018 1019 1020 1021 1022 |
ret = compact_zone(zone, &cc); VM_BUG_ON(!list_empty(&cc.freepages)); VM_BUG_ON(!list_empty(&cc.migratepages)); *contended = cc.contended; return ret; |
56de7263f mm: compaction: d... |
1023 |
} |
5e7719058 mm: compaction: a... |
1024 |
int sysctl_extfrag_threshold = 500; |
56de7263f mm: compaction: d... |
1025 1026 1027 1028 1029 1030 |
/** * try_to_compact_pages - Direct compact to satisfy a high-order allocation * @zonelist: The zonelist used for the current allocation * @order: The order of the current allocation * @gfp_mask: The GFP mask of the current allocation * @nodemask: The allowed nodes to allocate from |
77f1fe6b0 mm: migration: al... |
1031 |
* @sync: Whether migration is synchronous or not |
661c4cb9b mm: compaction: U... |
1032 1033 |
* @contended: Return value that is true if compaction was aborted due to lock contention * @page: Optionally capture a free page of the requested order during compaction |
56de7263f mm: compaction: d... |
1034 1035 1036 1037 |
* * This is the main entry point for direct page compaction. */ unsigned long try_to_compact_pages(struct zonelist *zonelist, |
77f1fe6b0 mm: migration: al... |
1038 |
int order, gfp_t gfp_mask, nodemask_t *nodemask, |
8fb74b9fb mm: compaction: p... |
1039 |
bool sync, bool *contended) |
56de7263f mm: compaction: d... |
1040 1041 1042 1043 |
{ enum zone_type high_zoneidx = gfp_zone(gfp_mask); int may_enter_fs = gfp_mask & __GFP_FS; int may_perform_io = gfp_mask & __GFP_IO; |
56de7263f mm: compaction: d... |
1044 1045 1046 |
struct zoneref *z; struct zone *zone; int rc = COMPACT_SKIPPED; |
d95ea5d18 cma: fix watermar... |
1047 |
int alloc_flags = 0; |
56de7263f mm: compaction: d... |
1048 |
|
4ffb6335d mm: compaction: u... |
1049 |
/* Check if the GFP flags allow compaction */ |
c5a73c3d5 thp: use compacti... |
1050 |
if (!order || !may_enter_fs || !may_perform_io) |
56de7263f mm: compaction: d... |
1051 |
return rc; |
010fc29a4 compaction: fix b... |
1052 |
count_compact_event(COMPACTSTALL); |
56de7263f mm: compaction: d... |
1053 |
|
d95ea5d18 cma: fix watermar... |
1054 1055 1056 1057 |
#ifdef CONFIG_CMA if (allocflags_to_migratetype(gfp_mask) == MIGRATE_MOVABLE) alloc_flags |= ALLOC_CMA; #endif |
56de7263f mm: compaction: d... |
1058 1059 1060 |
/* Compact each zone in the list */ for_each_zone_zonelist_nodemask(zone, z, zonelist, high_zoneidx, nodemask) { |
56de7263f mm: compaction: d... |
1061 |
int status; |
c67fe3752 mm: compaction: A... |
1062 |
status = compact_zone_order(zone, order, gfp_mask, sync, |
8fb74b9fb mm: compaction: p... |
1063 |
contended); |
56de7263f mm: compaction: d... |
1064 |
rc = max(status, rc); |
3e7d34497 mm: vmscan: recla... |
1065 |
/* If a normal allocation would succeed, stop compacting */ |
d95ea5d18 cma: fix watermar... |
1066 1067 |
if (zone_watermark_ok(zone, order, low_wmark_pages(zone), 0, alloc_flags)) |
56de7263f mm: compaction: d... |
1068 1069 1070 1071 1072 |
break; } return rc; } |
76ab0f530 mm: compaction: a... |
1073 |
/* Compact all zones within a node */ |
7103f16db mm: compaction: m... |
1074 |
static void __compact_pgdat(pg_data_t *pgdat, struct compact_control *cc) |
76ab0f530 mm: compaction: a... |
1075 1076 |
{ int zoneid; |
76ab0f530 mm: compaction: a... |
1077 |
struct zone *zone; |
76ab0f530 mm: compaction: a... |
1078 |
for (zoneid = 0; zoneid < MAX_NR_ZONES; zoneid++) { |
76ab0f530 mm: compaction: a... |
1079 1080 1081 1082 |
zone = &pgdat->node_zones[zoneid]; if (!populated_zone(zone)) continue; |
7be62de99 vmscan: kswapd ca... |
1083 1084 1085 1086 1087 |
cc->nr_freepages = 0; cc->nr_migratepages = 0; cc->zone = zone; INIT_LIST_HEAD(&cc->freepages); INIT_LIST_HEAD(&cc->migratepages); |
76ab0f530 mm: compaction: a... |
1088 |
|
aad6ec377 mm: compaction: m... |
1089 |
if (cc->order == -1 || !compaction_deferred(zone, cc->order)) |
7be62de99 vmscan: kswapd ca... |
1090 |
compact_zone(zone, cc); |
76ab0f530 mm: compaction: a... |
1091 |
|
aff622495 vmscan: only defe... |
1092 |
if (cc->order > 0) { |
de6c60a6c mm: compaction: e... |
1093 1094 1095 |
if (zone_watermark_ok(zone, cc->order, low_wmark_pages(zone), 0, 0)) compaction_defer_reset(zone, cc->order, false); |
aff622495 vmscan: only defe... |
1096 |
/* Currently async compaction is never deferred. */ |
de6c60a6c mm: compaction: e... |
1097 |
else if (cc->sync) |
aff622495 vmscan: only defe... |
1098 1099 |
defer_compaction(zone, cc->order); } |
7be62de99 vmscan: kswapd ca... |
1100 1101 |
VM_BUG_ON(!list_empty(&cc->freepages)); VM_BUG_ON(!list_empty(&cc->migratepages)); |
76ab0f530 mm: compaction: a... |
1102 |
} |
76ab0f530 mm: compaction: a... |
1103 |
} |
7103f16db mm: compaction: m... |
1104 |
void compact_pgdat(pg_data_t *pgdat, int order) |
7be62de99 vmscan: kswapd ca... |
1105 1106 1107 |
{ struct compact_control cc = { .order = order, |
68e3e9262 Revert "mm: compa... |
1108 |
.sync = false, |
7be62de99 vmscan: kswapd ca... |
1109 |
}; |
3a7200af3 mm: compaction: d... |
1110 1111 |
if (!order) return; |
7103f16db mm: compaction: m... |
1112 |
__compact_pgdat(pgdat, &cc); |
7be62de99 vmscan: kswapd ca... |
1113 |
} |
7103f16db mm: compaction: m... |
1114 |
static void compact_node(int nid) |
7be62de99 vmscan: kswapd ca... |
1115 |
{ |
7be62de99 vmscan: kswapd ca... |
1116 1117 |
struct compact_control cc = { .order = -1, |
68e3e9262 Revert "mm: compa... |
1118 |
.sync = true, |
91ca91864 mm, compaction: i... |
1119 |
.ignore_skip_hint = true, |
7be62de99 vmscan: kswapd ca... |
1120 |
}; |
7103f16db mm: compaction: m... |
1121 |
__compact_pgdat(NODE_DATA(nid), &cc); |
7be62de99 vmscan: kswapd ca... |
1122 |
} |
76ab0f530 mm: compaction: a... |
1123 |
/* Compact all nodes in the system */ |
7964c06d6 mm: compaction: f... |
1124 |
static void compact_nodes(void) |
76ab0f530 mm: compaction: a... |
1125 1126 |
{ int nid; |
8575ec29f compact_pgdat: wo... |
1127 1128 |
/* Flush pending updates to the LRU lists */ lru_add_drain_all(); |
76ab0f530 mm: compaction: a... |
1129 1130 |
for_each_online_node(nid) compact_node(nid); |
76ab0f530 mm: compaction: a... |
1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 |
} /* The written value is actually unused, all memory is compacted */ int sysctl_compact_memory; /* This is the entry point for compacting all nodes via /proc/sys/vm */ int sysctl_compaction_handler(struct ctl_table *table, int write, void __user *buffer, size_t *length, loff_t *ppos) { if (write) |
7964c06d6 mm: compaction: f... |
1141 |
compact_nodes(); |
76ab0f530 mm: compaction: a... |
1142 1143 1144 |
return 0; } |
ed4a6d7f0 mm: compaction: a... |
1145 |
|
5e7719058 mm: compaction: a... |
1146 1147 1148 1149 1150 1151 1152 |
int sysctl_extfrag_handler(struct ctl_table *table, int write, void __user *buffer, size_t *length, loff_t *ppos) { proc_dointvec_minmax(table, write, buffer, length, ppos); return 0; } |
ed4a6d7f0 mm: compaction: a... |
1153 |
#if defined(CONFIG_SYSFS) && defined(CONFIG_NUMA) |
74e77fb9a mm/compaction.c: ... |
1154 |
static ssize_t sysfs_compact_node(struct device *dev, |
10fbcf4c6 convert 'memory' ... |
1155 |
struct device_attribute *attr, |
ed4a6d7f0 mm: compaction: a... |
1156 1157 |
const char *buf, size_t count) { |
8575ec29f compact_pgdat: wo... |
1158 1159 1160 1161 1162 1163 1164 1165 |
int nid = dev->id; if (nid >= 0 && nid < nr_node_ids && node_online(nid)) { /* Flush pending updates to the LRU lists */ lru_add_drain_all(); compact_node(nid); } |
ed4a6d7f0 mm: compaction: a... |
1166 1167 1168 |
return count; } |
10fbcf4c6 convert 'memory' ... |
1169 |
static DEVICE_ATTR(compact, S_IWUSR, NULL, sysfs_compact_node); |
ed4a6d7f0 mm: compaction: a... |
1170 1171 1172 |
int compaction_register_node(struct node *node) { |
10fbcf4c6 convert 'memory' ... |
1173 |
return device_create_file(&node->dev, &dev_attr_compact); |
ed4a6d7f0 mm: compaction: a... |
1174 1175 1176 1177 |
} void compaction_unregister_node(struct node *node) { |
10fbcf4c6 convert 'memory' ... |
1178 |
return device_remove_file(&node->dev, &dev_attr_compact); |
ed4a6d7f0 mm: compaction: a... |
1179 1180 |
} #endif /* CONFIG_SYSFS && CONFIG_NUMA */ |
ff9543fd3 mm: compaction: e... |
1181 1182 |
#endif /* CONFIG_COMPACTION */ |