mm: page allocator: do not call direct reclaim for THP allocations while compaction is deferred

If compaction is deferred, direct reclaim is used to try to free enough pages for the allocation to succeed. For small high-orders, this has a reasonable chance of success. However, if the caller has specified __GFP_NO_KSWAPD to limit the disruption to the system, it makes more sense to fail the allocation rather than stall the caller in direct reclaim. This patch skips direct reclaim if compaction is deferred and the caller specifies __GFP_NO_KSWAPD. Async compaction only considers a subset of pages so it is possible for compaction to be deferred prematurely and not enter direct reclaim even in cases where it should. To compensate for this, this patch also defers compaction only if sync compaction failed. Signed-off-by: Mel Gorman <mgorman@suse.de> Acked-by: Minchan Kim <minchan.kim@gmail.com> Reviewed-by: Rik van Riel<riel@redhat.com> Cc: Andrea Arcangeli <aarcange@redhat.com> Cc: Dave Jones <davej@redhat.com> Cc: Jan Kara <jack@suse.cz> Cc: Andy Isaacson <adi@hexapodia.org> Cc: Nai Xia <nai.xia@gmail.com> Cc: Johannes Weiner <jweiner@redhat.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

mm: page allocator: do not call direct reclaim for THP allocations while compaction is deferred
If compaction is deferred, direct reclaim is used to try to free enough pages for the allocation to succeed. For small high-orders, this has a reasonable chance of success. However, if the caller has specified __GFP_NO_KSWAPD to limit the disruption to the system, it makes more sense to fail the allocation rather than stall the caller in direct reclaim. This patch skips direct reclaim if compaction is deferred and the caller specifies __GFP_NO_KSWAPD. Async compaction only considers a subset of pages so it is possible for compaction to be deferred prematurely and not enter direct reclaim even in cases where it should. To compensate for this, this patch also defers compaction only if sync compaction failed. Signed-off-by: Mel Gorman <mgorman@suse.de> Acked-by: Minchan Kim <minchan.kim@gmail.com> Reviewed-by: Rik van Riel<riel@redhat.com> Cc: Andrea Arcangeli <aarcange@redhat.com> Cc: Dave Jones <davej@redhat.com> Cc: Jan Kara <jack@suse.cz> Cc: Andy Isaacson <adi@hexapodia.org> Cc: Nai Xia <nai.xia@gmail.com> Cc: Johannes Weiner <jweiner@redhat.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
Mel Gorman · Linus Torvalds
1 parent c824493528
Showing 1 changed file with 35 additions and 10 deletions Side-by-side Diff
mm/page_alloc.c
@@ -1981,14 +1981,20 @@
 __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
 	struct zonelist *zonelist, enum zone_type high_zoneidx,
 	nodemask_t *nodemask, int alloc_flags, struct zone *preferred_zone,
-	int migratetype, unsigned long *did_some_progress,
-	bool sync_migration)
+	int migratetype, bool sync_migration,
+	bool *deferred_compaction,
+	unsigned long *did_some_progress)
 {
 	struct page *page;
  
-	if (!order || compaction_deferred(preferred_zone))
+	if (!order)
 		return NULL;
  
+	if (compaction_deferred(preferred_zone)) {
+		*deferred_compaction = true;
+		return NULL;
+	}
+
 	current->flags |= PF_MEMALLOC;
 	*did_some_progress = try_to_compact_pages(zonelist, order, gfp_mask,
 						nodemask, sync_migration);
  
@@ -2016,8 +2022,14 @@
 		 * but not enough to satisfy watermarks.
 		 */
 		count_vm_event(COMPACTFAIL);
-		defer_compaction(preferred_zone);
  
+		/*
+		 * As async compaction considers a subset of pageblocks, only
+		 * defer if the failure was a sync compaction failure.
+		 */
+		if (sync_migration)
+			defer_compaction(preferred_zone);
+
 		cond_resched();
 	}
  
@@ -2028,8 +2040,9 @@
 __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
 	struct zonelist *zonelist, enum zone_type high_zoneidx,
 	nodemask_t *nodemask, int alloc_flags, struct zone *preferred_zone,
-	int migratetype, unsigned long *did_some_progress,
-	bool sync_migration)
+	int migratetype, bool sync_migration,
+	bool *deferred_compaction,
+	unsigned long *did_some_progress)
 {
 	return NULL;
 }
@@ -2179,6 +2192,7 @@
 	unsigned long pages_reclaimed = 0;
 	unsigned long did_some_progress;
 	bool sync_migration = false;
+	bool deferred_compaction = false;
  
 	/*
 	 * In the slowpath, we sanity check order to avoid ever trying to
  
@@ -2259,12 +2273,22 @@
 					zonelist, high_zoneidx,
 					nodemask,
 					alloc_flags, preferred_zone,
-					migratetype, &did_some_progress,
-					sync_migration);
+					migratetype, sync_migration,
+					&deferred_compaction,
+					&did_some_progress);
 	if (page)
 		goto got_pg;
 	sync_migration = true;
  
+	/*
+	 * If compaction is deferred for high-order allocations, it is because
+	 * sync compaction recently failed. In this is the case and the caller
+	 * has requested the system not be heavily disrupted, fail the
+	 * allocation now instead of entering direct reclaim
+	 */
+	if (deferred_compaction && (gfp_mask & __GFP_NO_KSWAPD))
+		goto nopage;
+
 	/* Try direct reclaim and then allocating */
 	page = __alloc_pages_direct_reclaim(gfp_mask, order,
 					zonelist, high_zoneidx,
@@ -2328,8 +2352,9 @@
 					zonelist, high_zoneidx,
 					nodemask,
 					alloc_flags, preferred_zone,
-					migratetype, &did_some_progress,
-					sync_migration);
+					migratetype, sync_migration,
+					&deferred_compaction,
+					&did_some_progress);
 		if (page)
 			goto got_pg;
 	}
...	...	@@ -1981,14 +1981,20 @@
1981	1981	__alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
1982	1982	struct zonelist *zonelist, enum zone_type high_zoneidx,
1983	1983	nodemask_t nodemask, int alloc_flags, struct zone preferred_zone,
1984		- int migratetype, unsigned long *did_some_progress,
1985		- bool sync_migration)
	1984	+ int migratetype, bool sync_migration,
	1985	+ bool *deferred_compaction,
	1986	+ unsigned long *did_some_progress)
1986	1987	{
1987	1988	struct page *page;
1988	1989
1989		- if (!order \|\| compaction_deferred(preferred_zone))
	1990	+ if (!order)
1990	1991	return NULL;
1991	1992
	1993	+ if (compaction_deferred(preferred_zone)) {
	1994	+ *deferred_compaction = true;
	1995	+ return NULL;
	1996	+ }
	1997	+
1992	1998	current->flags \|= PF_MEMALLOC;
1993	1999	*did_some_progress = try_to_compact_pages(zonelist, order, gfp_mask,
1994	2000	nodemask, sync_migration);
1995	2001
...	...	@@ -2016,8 +2022,14 @@
2016	2022	* but not enough to satisfy watermarks.
2017	2023	*/
2018	2024	count_vm_event(COMPACTFAIL);
2019		- defer_compaction(preferred_zone);
2020	2025
	2026	+ /*
	2027	+ * As async compaction considers a subset of pageblocks, only
	2028	+ * defer if the failure was a sync compaction failure.
	2029	+ */
	2030	+ if (sync_migration)
	2031	+ defer_compaction(preferred_zone);
	2032	+
2021	2033	cond_resched();
2022	2034	}
2023	2035
...	...	@@ -2028,8 +2040,9 @@
2028	2040	__alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
2029	2041	struct zonelist *zonelist, enum zone_type high_zoneidx,
2030	2042	nodemask_t nodemask, int alloc_flags, struct zone preferred_zone,
2031		- int migratetype, unsigned long *did_some_progress,
2032		- bool sync_migration)
	2043	+ int migratetype, bool sync_migration,
	2044	+ bool *deferred_compaction,
	2045	+ unsigned long *did_some_progress)
2033	2046	{
2034	2047	return NULL;
2035	2048	}
...	...	@@ -2179,6 +2192,7 @@
2179	2192	unsigned long pages_reclaimed = 0;
2180	2193	unsigned long did_some_progress;
2181	2194	bool sync_migration = false;
	2195	+ bool deferred_compaction = false;
2182	2196
2183	2197	/*
2184	2198	* In the slowpath, we sanity check order to avoid ever trying to
2185	2199
...	...	@@ -2259,12 +2273,22 @@
2259	2273	zonelist, high_zoneidx,
2260	2274	nodemask,
2261	2275	alloc_flags, preferred_zone,
2262		- migratetype, &did_some_progress,
2263		- sync_migration);
	2276	+ migratetype, sync_migration,
	2277	+ &deferred_compaction,
	2278	+ &did_some_progress);
2264	2279	if (page)
2265	2280	goto got_pg;
2266	2281	sync_migration = true;
2267	2282
	2283	+ /*
	2284	+ * If compaction is deferred for high-order allocations, it is because
	2285	+ * sync compaction recently failed. In this is the case and the caller
	2286	+ * has requested the system not be heavily disrupted, fail the
	2287	+ * allocation now instead of entering direct reclaim
	2288	+ */
	2289	+ if (deferred_compaction && (gfp_mask & __GFP_NO_KSWAPD))
	2290	+ goto nopage;
	2291	+
2268	2292	/* Try direct reclaim and then allocating */
2269	2293	page = __alloc_pages_direct_reclaim(gfp_mask, order,
2270	2294	zonelist, high_zoneidx,
...	...	@@ -2328,8 +2352,9 @@
2328	2352	zonelist, high_zoneidx,
2329	2353	nodemask,
2330	2354	alloc_flags, preferred_zone,
2331		- migratetype, &did_some_progress,
2332		- sync_migration);
	2355	+ migratetype, sync_migration,
	2356	+ &deferred_compaction,
	2357	+ &did_some_progress);
2333	2358	if (page)
2334	2359	goto got_pg;
2335	2360	}