Doug / smarc-fsl-linux-kernel

Download as
Browse Code ยป

Commit 645747462435d84c6c6a64269ed49cc3015f753d

Authored by Johannes Weiner
Committed by Linus Torvalds
1 parent 31c0569c3b
Exists in master and in 7 other branches imx_3.0.35_4.1.0, imx_3.10.17_1.0.1_ga, imx_3.10.53_1.1.0_ga, imx_3.14.28_1.0.0_ga, smarc-imx_3.10.53_1.1.0_ga, smarc-imx_3.14.28_1.0.0_ga, smarc-l5.0.0_1.0.0-ga

vmscan: detect mapped file pages used only once 

The VM currently assumes that an inactive, mapped and referenced file page
is in use and promotes it to the active list.

However, every mapped file page starts out like this and thus a problem
arises when workloads create a stream of such pages that are used only for
a short time.  By flooding the active list with those pages, the VM
quickly gets into trouble finding eligible reclaim canditates.  The result
is long allocation latencies and eviction of the wrong pages.

This patch reuses the PG_referenced page flag (used for unmapped file
pages) to implement a usage detection that scales with the speed of LRU
list cycling (i.e.  memory pressure).

If the scanner encounters those pages, the flag is set and the page cycled
again on the inactive list.  Only if it returns with another page table
reference it is activated.  Otherwise it is reclaimed as 'not recently
used cache'.

This effectively changes the minimum lifetime of a used-once mapped file
page from a full memory cycle to an inactive list cycle, which allows it
to occur in linear streams without affecting the stable working set of the
system.

Signed-off-by: Johannes Weiner <hannes@cmpxchg.org>
Reviewed-by: Rik van Riel <riel@redhat.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: OSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Cc: Lee Schermerhorn <lee.schermerhorn@hp.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

Showing 3 changed files with 36 additions and 14 deletions Side-by-side Diff

include/linux/rmap.h
Diff comments   View file @ 6457474
... ... @@ -209,7 +209,7 @@
209 209 unsigned long *vm_flags)
210 210 {
211 211 *vm_flags = 0;
212   - return TestClearPageReferenced(page);
  212 + return 0;
213 213 }
214 214  
215 215 #define try_to_unmap(page, refs) SWAP_FAIL
... ... @@ -601,9 +601,6 @@
601 601 int referenced = 0;
602 602 int we_locked = 0;
603 603  
604   - if (TestClearPageReferenced(page))
605   - referenced++;
606   -
607 604 *vm_flags = 0;
608 605 if (page_mapped(page) && page_rmapping(page)) {
609 606 if (!is_locked && (!PageAnon(page) || PageKsm(page))) {
... ... @@ -561,18 +561,18 @@
561 561 enum page_references {
562 562 PAGEREF_RECLAIM,
563 563 PAGEREF_RECLAIM_CLEAN,
  564 + PAGEREF_KEEP,
564 565 PAGEREF_ACTIVATE,
565 566 };
566 567  
567 568 static enum page_references page_check_references(struct page *page,
568 569 struct scan_control *sc)
569 570 {
  571 + int referenced_ptes, referenced_page;
570 572 unsigned long vm_flags;
571   - int referenced;
572 573  
573   - referenced = page_referenced(page, 1, sc->mem_cgroup, &vm_flags);
574   - if (!referenced)
575   - return PAGEREF_RECLAIM;
  574 + referenced_ptes = page_referenced(page, 1, sc->mem_cgroup, &vm_flags);
  575 + referenced_page = TestClearPageReferenced(page);
576 576  
577 577 /* Lumpy reclaim - ignore references */
578 578 if (sc->order > PAGE_ALLOC_COSTLY_ORDER)
579 579  
580 580  
... ... @@ -585,11 +585,36 @@
585 585 if (vm_flags & VM_LOCKED)
586 586 return PAGEREF_RECLAIM;
587 587  
588   - if (page_mapped(page))
589   - return PAGEREF_ACTIVATE;
  588 + if (referenced_ptes) {
  589 + if (PageAnon(page))
  590 + return PAGEREF_ACTIVATE;
  591 + /*
  592 + * All mapped pages start out with page table
  593 + * references from the instantiating fault, so we need
  594 + * to look twice if a mapped file page is used more
  595 + * than once.
  596 + *
  597 + * Mark it and spare it for another trip around the
  598 + * inactive list. Another page table reference will
  599 + * lead to its activation.
  600 + *
  601 + * Note: the mark is set for activated pages as well
  602 + * so that recently deactivated but used pages are
  603 + * quickly recovered.
  604 + */
  605 + SetPageReferenced(page);
590 606  
  607 + if (referenced_page)
  608 + return PAGEREF_ACTIVATE;
  609 +
  610 + return PAGEREF_KEEP;
  611 + }
  612 +
591 613 /* Reclaim if clean, defer dirty pages to writeback */
592   - return PAGEREF_RECLAIM_CLEAN;
  614 + if (referenced_page)
  615 + return PAGEREF_RECLAIM_CLEAN;
  616 +
  617 + return PAGEREF_RECLAIM;
593 618 }
594 619  
595 620 /*
... ... @@ -657,6 +682,8 @@
657 682 switch (references) {
658 683 case PAGEREF_ACTIVATE:
659 684 goto activate_locked;
  685 + case PAGEREF_KEEP:
  686 + goto keep_locked;
660 687 case PAGEREF_RECLAIM:
661 688 case PAGEREF_RECLAIM_CLEAN:
662 689 ; /* try to reclaim the page below */
... ... @@ -1359,9 +1386,7 @@
1359 1386 continue;
1360 1387 }
1361 1388  
1362   - /* page_referenced clears PageReferenced */
1363   - if (page_mapped(page) &&
1364   - page_referenced(page, 0, sc->mem_cgroup, &vm_flags)) {
  1389 + if (page_referenced(page, 0, sc->mem_cgroup, &vm_flags)) {
1365 1390 nr_rotated++;
1366 1391 /*
1367 1392 * Identify referenced, file-backed active pages and