Commit 2bcf887963812c075f80a14e1fad8ec7e1c67acf
Committed by
Linus Torvalds
1 parent
90b3feaec8
Exists in
master
and in
6 other branches
mm: take pagevecs off reclaim stack
Replace pagevecs in putback_lru_pages() and move_active_pages_to_lru() by lists of pages_to_free: then apply Konstantin Khlebnikov's free_hot_cold_page_list() to them instead of pagevec_release(). Which simplifies the flow (no need to drop and retake lock whenever pagevec fills up) and reduces stale addresses in stack backtraces (which often showed through the pagevecs); but more importantly, removes another 120 bytes from the deepest stacks in page reclaim. Although I've not recently seen an actual stack overflow here with a vanilla kernel, move_active_pages_to_lru() has often featured in deep backtraces. However, free_hot_cold_page_list() does not handle compound pages (nor need it: a Transparent HugePage would have been split by the time it reaches the call in shrink_page_list()), but it is possible for putback_lru_pages() or move_active_pages_to_lru() to be left holding the last reference on a THP, so must exclude the unlikely compound case before putting on pages_to_free. Remove pagevec_strip(), its work now done in move_active_pages_to_lru(). The pagevec in scan_mapping_unevictable_pages() remains in mm/vmscan.c, but that is never on the reclaim path, and cannot be replaced by a list. Signed-off-by: Hugh Dickins <hughd@google.com> Reviewed-by: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com> Reviewed-by: Konstantin Khlebnikov <khlebnikov@openvz.org> Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com> Cc: Mel Gorman <mel@csn.ul.ie> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
Showing 3 changed files with 40 additions and 39 deletions Side-by-side Diff
include/linux/pagevec.h
... | ... | @@ -22,7 +22,6 @@ |
22 | 22 | |
23 | 23 | void __pagevec_release(struct pagevec *pvec); |
24 | 24 | void ____pagevec_lru_add(struct pagevec *pvec, enum lru_list lru); |
25 | -void pagevec_strip(struct pagevec *pvec); | |
26 | 25 | unsigned pagevec_lookup(struct pagevec *pvec, struct address_space *mapping, |
27 | 26 | pgoff_t start, unsigned nr_pages); |
28 | 27 | unsigned pagevec_lookup_tag(struct pagevec *pvec, |
... | ... | @@ -58,7 +57,6 @@ |
58 | 57 | pvec->pages[pvec->nr++] = page; |
59 | 58 | return pagevec_space(pvec); |
60 | 59 | } |
61 | - | |
62 | 60 | |
63 | 61 | static inline void pagevec_release(struct pagevec *pvec) |
64 | 62 | { |
mm/swap.c
... | ... | @@ -23,7 +23,6 @@ |
23 | 23 | #include <linux/init.h> |
24 | 24 | #include <linux/export.h> |
25 | 25 | #include <linux/mm_inline.h> |
26 | -#include <linux/buffer_head.h> /* for try_to_release_page() */ | |
27 | 26 | #include <linux/percpu_counter.h> |
28 | 27 | #include <linux/percpu.h> |
29 | 28 | #include <linux/cpu.h> |
... | ... | @@ -729,24 +728,6 @@ |
729 | 728 | } |
730 | 729 | |
731 | 730 | EXPORT_SYMBOL(____pagevec_lru_add); |
732 | - | |
733 | -/* | |
734 | - * Try to drop buffers from the pages in a pagevec | |
735 | - */ | |
736 | -void pagevec_strip(struct pagevec *pvec) | |
737 | -{ | |
738 | - int i; | |
739 | - | |
740 | - for (i = 0; i < pagevec_count(pvec); i++) { | |
741 | - struct page *page = pvec->pages[i]; | |
742 | - | |
743 | - if (page_has_private(page) && trylock_page(page)) { | |
744 | - if (page_has_private(page)) | |
745 | - try_to_release_page(page, 0); | |
746 | - unlock_page(page); | |
747 | - } | |
748 | - } | |
749 | -} | |
750 | 731 | |
751 | 732 | /** |
752 | 733 | * pagevec_lookup - gang pagecache lookup |
mm/vmscan.c
... | ... | @@ -1398,12 +1398,10 @@ |
1398 | 1398 | struct list_head *page_list) |
1399 | 1399 | { |
1400 | 1400 | struct page *page; |
1401 | - struct pagevec pvec; | |
1401 | + LIST_HEAD(pages_to_free); | |
1402 | 1402 | struct zone *zone = mz->zone; |
1403 | 1403 | struct zone_reclaim_stat *reclaim_stat = get_reclaim_stat(mz); |
1404 | 1404 | |
1405 | - pagevec_init(&pvec, 1); | |
1406 | - | |
1407 | 1405 | /* |
1408 | 1406 | * Put back any unfreeable pages. |
1409 | 1407 | */ |
1410 | 1408 | |
... | ... | @@ -1427,17 +1425,24 @@ |
1427 | 1425 | int numpages = hpage_nr_pages(page); |
1428 | 1426 | reclaim_stat->recent_rotated[file] += numpages; |
1429 | 1427 | } |
1430 | - if (!pagevec_add(&pvec, page)) { | |
1431 | - spin_unlock_irq(&zone->lru_lock); | |
1432 | - __pagevec_release(&pvec); | |
1433 | - spin_lock_irq(&zone->lru_lock); | |
1428 | + if (put_page_testzero(page)) { | |
1429 | + __ClearPageLRU(page); | |
1430 | + __ClearPageActive(page); | |
1431 | + del_page_from_lru_list(zone, page, lru); | |
1432 | + | |
1433 | + if (unlikely(PageCompound(page))) { | |
1434 | + spin_unlock_irq(&zone->lru_lock); | |
1435 | + (*get_compound_page_dtor(page))(page); | |
1436 | + spin_lock_irq(&zone->lru_lock); | |
1437 | + } else | |
1438 | + list_add(&page->lru, &pages_to_free); | |
1434 | 1439 | } |
1435 | 1440 | } |
1436 | 1441 | __mod_zone_page_state(zone, NR_ISOLATED_ANON, -nr_anon); |
1437 | 1442 | __mod_zone_page_state(zone, NR_ISOLATED_FILE, -nr_file); |
1438 | 1443 | |
1439 | 1444 | spin_unlock_irq(&zone->lru_lock); |
1440 | - pagevec_release(&pvec); | |
1445 | + free_hot_cold_page_list(&pages_to_free, 1); | |
1441 | 1446 | } |
1442 | 1447 | |
1443 | 1448 | static noinline_for_stack void |
1444 | 1449 | |
1445 | 1450 | |
... | ... | @@ -1647,13 +1652,23 @@ |
1647 | 1652 | |
1648 | 1653 | static void move_active_pages_to_lru(struct zone *zone, |
1649 | 1654 | struct list_head *list, |
1655 | + struct list_head *pages_to_free, | |
1650 | 1656 | enum lru_list lru) |
1651 | 1657 | { |
1652 | 1658 | unsigned long pgmoved = 0; |
1653 | - struct pagevec pvec; | |
1654 | 1659 | struct page *page; |
1655 | 1660 | |
1656 | - pagevec_init(&pvec, 1); | |
1661 | + if (buffer_heads_over_limit) { | |
1662 | + spin_unlock_irq(&zone->lru_lock); | |
1663 | + list_for_each_entry(page, list, lru) { | |
1664 | + if (page_has_private(page) && trylock_page(page)) { | |
1665 | + if (page_has_private(page)) | |
1666 | + try_to_release_page(page, 0); | |
1667 | + unlock_page(page); | |
1668 | + } | |
1669 | + } | |
1670 | + spin_lock_irq(&zone->lru_lock); | |
1671 | + } | |
1657 | 1672 | |
1658 | 1673 | while (!list_empty(list)) { |
1659 | 1674 | struct lruvec *lruvec; |
... | ... | @@ -1667,12 +1682,17 @@ |
1667 | 1682 | list_move(&page->lru, &lruvec->lists[lru]); |
1668 | 1683 | pgmoved += hpage_nr_pages(page); |
1669 | 1684 | |
1670 | - if (!pagevec_add(&pvec, page) || list_empty(list)) { | |
1671 | - spin_unlock_irq(&zone->lru_lock); | |
1672 | - if (buffer_heads_over_limit) | |
1673 | - pagevec_strip(&pvec); | |
1674 | - __pagevec_release(&pvec); | |
1675 | - spin_lock_irq(&zone->lru_lock); | |
1685 | + if (put_page_testzero(page)) { | |
1686 | + __ClearPageLRU(page); | |
1687 | + __ClearPageActive(page); | |
1688 | + del_page_from_lru_list(zone, page, lru); | |
1689 | + | |
1690 | + if (unlikely(PageCompound(page))) { | |
1691 | + spin_unlock_irq(&zone->lru_lock); | |
1692 | + (*get_compound_page_dtor(page))(page); | |
1693 | + spin_lock_irq(&zone->lru_lock); | |
1694 | + } else | |
1695 | + list_add(&page->lru, pages_to_free); | |
1676 | 1696 | } |
1677 | 1697 | } |
1678 | 1698 | __mod_zone_page_state(zone, NR_LRU_BASE + lru, pgmoved); |
1679 | 1699 | |
1680 | 1700 | |
... | ... | @@ -1766,12 +1786,14 @@ |
1766 | 1786 | */ |
1767 | 1787 | reclaim_stat->recent_rotated[file] += nr_rotated; |
1768 | 1788 | |
1769 | - move_active_pages_to_lru(zone, &l_active, | |
1789 | + move_active_pages_to_lru(zone, &l_active, &l_hold, | |
1770 | 1790 | LRU_ACTIVE + file * LRU_FILE); |
1771 | - move_active_pages_to_lru(zone, &l_inactive, | |
1791 | + move_active_pages_to_lru(zone, &l_inactive, &l_hold, | |
1772 | 1792 | LRU_BASE + file * LRU_FILE); |
1773 | 1793 | __mod_zone_page_state(zone, NR_ISOLATED_ANON + file, -nr_taken); |
1774 | 1794 | spin_unlock_irq(&zone->lru_lock); |
1795 | + | |
1796 | + free_hot_cold_page_list(&l_hold, 1); | |
1775 | 1797 | } |
1776 | 1798 | |
1777 | 1799 | #ifdef CONFIG_SWAP |