mm, compaction: always update cached scanner positions

Compaction caches the migration and free scanner positions between compaction invocations, so that the whole zone gets eventually scanned and there is no bias towards the initial scanner positions at the beginning/end of the zone. The cached positions are continuously updated as scanners progress and the updating stops as soon as a page is successfully isolated. The reasoning behind this is that a pageblock where isolation succeeded is likely to succeed again in near future and it should be worth revisiting it. However, the downside is that potentially many pages are rescanned without successful isolation. At worst, there might be a page where isolation from LRU succeeds but migration fails (potentially always). So upon encountering this page, cached position would always stop being updated for no good reason. It might have been useful to let such page be rescanned with sync compaction after async one failed, but this is now handled by caching scanner position for async and sync mode separately since commit 35979ef33931 ("mm, compaction: add per-zone migration pfn cache for async compaction"). After this patch, cached positions are updated unconditionally. In stress-highalloc benchmark, this has decreased the numbers of scanned pages by few percent, without affecting allocation success rates. To prevent free scanner from leaving free pages behind after they are returned due to page migration failure, the cached scanner pfn is changed to point to the pageblock of the returned free page with the highest pfn, before leaving compact_zone(). [akpm@linux-foundation.org: coding-style fixes] Signed-off-by: Vlastimil Babka <vbabka@suse.cz> Cc: Minchan Kim <minchan@kernel.org> Cc: Mel Gorman <mgorman@suse.de> Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com> Cc: Michal Nazarewicz <mina86@mina86.com> Cc: Naoya Horiguchi <n-horiguchi@ah.jp.nec.com> Cc: Christoph Lameter <cl@linux.com> Acked-by: Rik van Riel <riel@redhat.com> Cc: David Rientjes <rientjes@google.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

mm, compaction: always update cached scanner positions
Compaction caches the migration and free scanner positions between compaction invocations, so that the whole zone gets eventually scanned and there is no bias towards the initial scanner positions at the beginning/end of the zone. The cached positions are continuously updated as scanners progress and the updating stops as soon as a page is successfully isolated. The reasoning behind this is that a pageblock where isolation succeeded is likely to succeed again in near future and it should be worth revisiting it. However, the downside is that potentially many pages are rescanned without successful isolation. At worst, there might be a page where isolation from LRU succeeds but migration fails (potentially always). So upon encountering this page, cached position would always stop being updated for no good reason. It might have been useful to let such page be rescanned with sync compaction after async one failed, but this is now handled by caching scanner position for async and sync mode separately since commit 35979ef33931 ("mm, compaction: add per-zone migration pfn cache for async compaction"). After this patch, cached positions are updated unconditionally. In stress-highalloc benchmark, this has decreased the numbers of scanned pages by few percent, without affecting allocation success rates. To prevent free scanner from leaving free pages behind after they are returned due to page migration failure, the cached scanner pfn is changed to point to the pageblock of the returned free page with the highest pfn, before leaving compact_zone(). [akpm@linux-foundation.org: coding-style fixes] Signed-off-by: Vlastimil Babka <vbabka@suse.cz> Cc: Minchan Kim <minchan@kernel.org> Cc: Mel Gorman <mgorman@suse.de> Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com> Cc: Michal Nazarewicz <mina86@mina86.com> Cc: Naoya Horiguchi <n-horiguchi@ah.jp.nec.com> Cc: Christoph Lameter <cl@linux.com> Acked-by: Rik van Riel <riel@redhat.com> Cc: David Rientjes <rientjes@google.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
Vlastimil Babka · Linus Torvalds
1 parent f866979539
Showing 2 changed files with 23 additions and 25 deletions Side-by-side Diff
mm/compaction.c
mm/internal.h
@@ -41,15 +41,17 @@
 static unsigned long release_freepages(struct list_head *freelist)
 {
 	struct page *page, *next;
-	unsigned long count = 0;
+	unsigned long high_pfn = 0;
  
 	list_for_each_entry_safe(page, next, freelist, lru) {
+		unsigned long pfn = page_to_pfn(page);
 		list_del(&page->lru);
 		__free_page(page);
-		count++;
+		if (pfn > high_pfn)
+			high_pfn = pfn;
 	}
  
-	return count;
+	return high_pfn;
 }
  
 static void map_pages(struct list_head *list)
  
@@ -195,16 +197,12 @@
  
 	/* Update where async and sync compaction should restart */
 	if (migrate_scanner) {
-		if (cc->finished_update_migrate)
-			return;
 		if (pfn > zone->compact_cached_migrate_pfn[0])
 			zone->compact_cached_migrate_pfn[0] = pfn;
 		if (cc->mode != MIGRATE_ASYNC &&
 		    pfn > zone->compact_cached_migrate_pfn[1])
 			zone->compact_cached_migrate_pfn[1] = pfn;
 	} else {
-		if (cc->finished_update_free)
-			return;
 		if (pfn < zone->compact_cached_free_pfn)
 			zone->compact_cached_free_pfn = pfn;
 	}
@@ -715,7 +713,6 @@
 		del_page_from_lru_list(page, lruvec, page_lru(page));
  
 isolate_success:
-		cc->finished_update_migrate = true;
 		list_add(&page->lru, migratelist);
 		cc->nr_migratepages++;
 		nr_isolated++;
@@ -889,15 +886,6 @@
 				block_start_pfn - pageblock_nr_pages;
  
 		/*
-		 * Set a flag that we successfully isolated in this pageblock.
-		 * In the next loop iteration, zone->compact_cached_free_pfn
-		 * will not be updated and thus it will effectively contain the
-		 * highest pageblock we isolated pages from.
-		 */
-		if (isolated)
-			cc->finished_update_free = true;
-
-		/*
 		 * isolate_freepages_block() might have aborted due to async
 		 * compaction being contended
 		 */
@@ -1251,9 +1239,24 @@
 	}
  
 out:
-	/* Release free pages and check accounting */
-	cc->nr_freepages -= release_freepages(&cc->freepages);
-	VM_BUG_ON(cc->nr_freepages != 0);
+	/*
+	 * Release free pages and update where the free scanner should restart,
+	 * so we don't leave any returned pages behind in the next attempt.
+	 */
+	if (cc->nr_freepages > 0) {
+		unsigned long free_pfn = release_freepages(&cc->freepages);
+
+		cc->nr_freepages = 0;
+		VM_BUG_ON(free_pfn == 0);
+		/* The cached pfn is always the first in a pageblock */
+		free_pfn &= ~(pageblock_nr_pages-1);
+		/*
+		 * Only go back, not forward. The cached pfn might have been
+		 * already reset to zone end in compact_finished()
+		 */
+		if (free_pfn > zone->compact_cached_free_pfn)
+			zone->compact_cached_free_pfn = free_pfn;
+	}
  
 	trace_mm_compaction_end(ret);
  
@@ -161,11 +161,6 @@
 	unsigned long migrate_pfn;	/* isolate_migratepages search base */
 	enum migrate_mode mode;		/* Async or sync migration mode */
 	bool ignore_skip_hint;		/* Scan blocks even if marked skip */
-	bool finished_update_free;	/* True when the zone cached pfns are
-					 * no longer being updated
-					 */
-	bool finished_update_migrate;
-
 	int order;			/* order a direct compactor needs */
 	const gfp_t gfp_mask;		/* gfp mask of a direct compactor */
 	const int alloc_flags;		/* alloc flags of a direct compactor */
...	...	@@ -41,15 +41,17 @@
41	41	static unsigned long release_freepages(struct list_head *freelist)
42	42	{
43	43	struct page page, next;
44		- unsigned long count = 0;
	44	+ unsigned long high_pfn = 0;
45	45
46	46	list_for_each_entry_safe(page, next, freelist, lru) {
	47	+ unsigned long pfn = page_to_pfn(page);
47	48	list_del(&page->lru);
48	49	__free_page(page);
49		- count++;
	50	+ if (pfn > high_pfn)
	51	+ high_pfn = pfn;
50	52	}
51	53
52		- return count;
	54	+ return high_pfn;
53	55	}
54	56
55	57	static void map_pages(struct list_head *list)
56	58
...	...	@@ -195,16 +197,12 @@
195	197
196	198	/* Update where async and sync compaction should restart */
197	199	if (migrate_scanner) {
198		- if (cc->finished_update_migrate)
199		- return;
200	200	if (pfn > zone->compact_cached_migrate_pfn[0])
201	201	zone->compact_cached_migrate_pfn[0] = pfn;
202	202	if (cc->mode != MIGRATE_ASYNC &&
203	203	pfn > zone->compact_cached_migrate_pfn[1])
204	204	zone->compact_cached_migrate_pfn[1] = pfn;
205	205	} else {
206		- if (cc->finished_update_free)
207		- return;
208	206	if (pfn < zone->compact_cached_free_pfn)
209	207	zone->compact_cached_free_pfn = pfn;
210	208	}
...	...	@@ -715,7 +713,6 @@
715	713	del_page_from_lru_list(page, lruvec, page_lru(page));
716	714
717	715	isolate_success:
718		- cc->finished_update_migrate = true;
719	716	list_add(&page->lru, migratelist);
720	717	cc->nr_migratepages++;
721	718	nr_isolated++;
...	...	@@ -889,15 +886,6 @@
889	886	block_start_pfn - pageblock_nr_pages;
890	887
891	888	/*
892		- * Set a flag that we successfully isolated in this pageblock.
893		- * In the next loop iteration, zone->compact_cached_free_pfn
894		- * will not be updated and thus it will effectively contain the
895		- * highest pageblock we isolated pages from.
896		- */
897		- if (isolated)
898		- cc->finished_update_free = true;
899		-
900		- /*
901	889	* isolate_freepages_block() might have aborted due to async
902	890	* compaction being contended
903	891	*/
...	...	@@ -1251,9 +1239,24 @@
1251	1239	}
1252	1240
1253	1241	out:
1254		- /* Release free pages and check accounting */
1255		- cc->nr_freepages -= release_freepages(&cc->freepages);
1256		- VM_BUG_ON(cc->nr_freepages != 0);
	1242	+ /*
	1243	+ * Release free pages and update where the free scanner should restart,
	1244	+ * so we don't leave any returned pages behind in the next attempt.
	1245	+ */
	1246	+ if (cc->nr_freepages > 0) {
	1247	+ unsigned long free_pfn = release_freepages(&cc->freepages);
	1248	+
	1249	+ cc->nr_freepages = 0;
	1250	+ VM_BUG_ON(free_pfn == 0);
	1251	+ /* The cached pfn is always the first in a pageblock */
	1252	+ free_pfn &= ~(pageblock_nr_pages-1);
	1253	+ /*
	1254	+ * Only go back, not forward. The cached pfn might have been
	1255	+ * already reset to zone end in compact_finished()
	1256	+ */
	1257	+ if (free_pfn > zone->compact_cached_free_pfn)
	1258	+ zone->compact_cached_free_pfn = free_pfn;
	1259	+ }
1257	1260
1258	1261	trace_mm_compaction_end(ret);
1259	1262
...	...	@@ -161,11 +161,6 @@
161	161	unsigned long migrate_pfn; /* isolate_migratepages search base */
162	162	enum migrate_mode mode; /* Async or sync migration mode */
163	163	bool ignore_skip_hint; /* Scan blocks even if marked skip */
164		- bool finished_update_free; /* True when the zone cached pfns are
165		- * no longer being updated
166		- */
167		- bool finished_update_migrate;
168		-
169	164	int order; /* order a direct compactor needs */
170	165	const gfp_t gfp_mask; /* gfp mask of a direct compactor */
171	166	const int alloc_flags; /* alloc flags of a direct compactor */