vmscan: kswapd carefully call compaction

With CONFIG_COMPACTION enabled, kswapd does not try to free contiguous free pages, even when it is woken for a higher order request. This could be bad for eg. jumbo frame network allocations, which are done from interrupt context and cannot compact memory themselves. Higher than before allocation failure rates in the network receive path have been observed in kernels with compaction enabled. Teach kswapd to defragment the memory zones in a node, but only if required and compaction is not deferred in a zone. [akpm@linux-foundation.org: reduce scope of zones_need_compaction] Signed-off-by: Rik van Riel <riel@redhat.com> Acked-by: Mel Gorman <mel@csn.ul.ie> Cc: Andrea Arcangeli <aarcange@redhat.com> Cc: Johannes Weiner <hannes@cmpxchg.org> Cc: Minchan Kim <minchan.kim@gmail.com> Cc: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com> Cc: Hillf Danton <dhillf@gmail.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

vmscan: kswapd carefully call compaction
With CONFIG_COMPACTION enabled, kswapd does not try to free contiguous free pages, even when it is woken for a higher order request. This could be bad for eg. jumbo frame network allocations, which are done from interrupt context and cannot compact memory themselves. Higher than before allocation failure rates in the network receive path have been observed in kernels with compaction enabled. Teach kswapd to defragment the memory zones in a node, but only if required and compaction is not deferred in a zone. [akpm@linux-foundation.org: reduce scope of zones_need_compaction] Signed-off-by: Rik van Riel <riel@redhat.com> Acked-by: Mel Gorman <mel@csn.ul.ie> Cc: Andrea Arcangeli <aarcange@redhat.com> Cc: Johannes Weiner <hannes@cmpxchg.org> Cc: Minchan Kim <minchan.kim@gmail.com> Cc: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com> Cc: Hillf Danton <dhillf@gmail.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
Rik van Riel · Linus Torvalds
1 parent fe2c2a1066
Showing 3 changed files with 51 additions and 18 deletions Side-by-side Diff
include/linux/compaction.h
mm/compaction.c
mm/vmscan.c
@@ -23,6 +23,7 @@
 extern unsigned long try_to_compact_pages(struct zonelist *zonelist,
 			int order, gfp_t gfp_mask, nodemask_t *mask,
 			bool sync);
+extern int compact_pgdat(pg_data_t *pgdat, int order);
 extern unsigned long compaction_suitable(struct zone *zone, int order);
  
 /* Do not skip compaction more than 64 times */
@@ -58,6 +59,11 @@
 static inline unsigned long try_to_compact_pages(struct zonelist *zonelist,
 			int order, gfp_t gfp_mask, nodemask_t *nodemask,
 			bool sync)
+{
+	return COMPACT_CONTINUE;
+}
+
+static inline int compact_pgdat(pg_data_t *pgdat, int order)
 {
 	return COMPACT_CONTINUE;
 }
@@ -675,42 +675,59 @@
  
  
 /* Compact all zones within a node */
-static int compact_node(int nid)
+static int __compact_pgdat(pg_data_t *pgdat, struct compact_control *cc)
 {
 	int zoneid;
-	pg_data_t *pgdat;
 	struct zone *zone;
  
-	if (nid < 0 || nid >= nr_node_ids || !node_online(nid))
-		return -EINVAL;
-	pgdat = NODE_DATA(nid);
-
 	/* Flush pending updates to the LRU lists */
 	lru_add_drain_all();
  
 	for (zoneid = 0; zoneid < MAX_NR_ZONES; zoneid++) {
-		struct compact_control cc = {
-			.nr_freepages = 0,
-			.nr_migratepages = 0,
-			.order = -1,
-			.sync = true,
-		};
  
 		zone = &pgdat->node_zones[zoneid];
 		if (!populated_zone(zone))
 			continue;
  
-		cc.zone = zone;
-		INIT_LIST_HEAD(&cc.freepages);
-		INIT_LIST_HEAD(&cc.migratepages);
+		cc->nr_freepages = 0;
+		cc->nr_migratepages = 0;
+		cc->zone = zone;
+		INIT_LIST_HEAD(&cc->freepages);
+		INIT_LIST_HEAD(&cc->migratepages);
  
-		compact_zone(zone, &cc);
+		if (cc->order < 0 || !compaction_deferred(zone))
+			compact_zone(zone, cc);
  
-		VM_BUG_ON(!list_empty(&cc.freepages));
-		VM_BUG_ON(!list_empty(&cc.migratepages));
+		VM_BUG_ON(!list_empty(&cc->freepages));
+		VM_BUG_ON(!list_empty(&cc->migratepages));
 	}
  
 	return 0;
+}
+
+int compact_pgdat(pg_data_t *pgdat, int order)
+{
+	struct compact_control cc = {
+		.order = order,
+		.sync = false,
+	};
+
+	return __compact_pgdat(pgdat, &cc);
+}
+
+static int compact_node(int nid)
+{
+	pg_data_t *pgdat;
+	struct compact_control cc = {
+		.order = -1,
+		.sync = true,
+	};
+
+	if (nid < 0 || nid >= nr_node_ids || !node_online(nid))
+		return -EINVAL;
+	pgdat = NODE_DATA(nid);
+
+	return __compact_pgdat(pgdat, &cc);
 }
  
 /* Compact all nodes in the system */
@@ -2919,6 +2919,8 @@
 	 * and it is potentially going to sleep here.
 	 */
 	if (order) {
+		int zones_need_compaction = 1;
+
 		for (i = 0; i <= end_zone; i++) {
 			struct zone *zone = pgdat->node_zones + i;
  
  
@@ -2939,9 +2941,17 @@
 				goto loop_again;
 			}
  
+			/* Check if the memory needs to be defragmented. */
+			if (zone_watermark_ok(zone, order,
+				    low_wmark_pages(zone), *classzone_idx, 0))
+				zones_need_compaction = 0;
+
 			/* If balanced, clear the congested flag */
 			zone_clear_flag(zone, ZONE_CONGESTED);
 		}
+
+		if (zones_need_compaction)
+			compact_pgdat(pgdat, order);
 	}
  
 	/*
...	...	@@ -23,6 +23,7 @@
23	23	extern unsigned long try_to_compact_pages(struct zonelist *zonelist,
24	24	int order, gfp_t gfp_mask, nodemask_t *mask,
25	25	bool sync);
	26	+extern int compact_pgdat(pg_data_t *pgdat, int order);
26	27	extern unsigned long compaction_suitable(struct zone *zone, int order);
27	28
28	29	/* Do not skip compaction more than 64 times */
...	...	@@ -58,6 +59,11 @@
58	59	static inline unsigned long try_to_compact_pages(struct zonelist *zonelist,
59	60	int order, gfp_t gfp_mask, nodemask_t *nodemask,
60	61	bool sync)
	62	+{
	63	+ return COMPACT_CONTINUE;
	64	+}
	65	+
	66	+static inline int compact_pgdat(pg_data_t *pgdat, int order)
61	67	{
62	68	return COMPACT_CONTINUE;
63	69	}
...	...	@@ -675,42 +675,59 @@
675	675
676	676
677	677	/* Compact all zones within a node */
678		-static int compact_node(int nid)
	678	+static int __compact_pgdat(pg_data_t pgdat, struct compact_control cc)
679	679	{
680	680	int zoneid;
681		- pg_data_t *pgdat;
682	681	struct zone *zone;
683	682
684		- if (nid < 0 \|\| nid >= nr_node_ids \|\| !node_online(nid))
685		- return -EINVAL;
686		- pgdat = NODE_DATA(nid);
687		-
688	683	/* Flush pending updates to the LRU lists */
689	684	lru_add_drain_all();
690	685
691	686	for (zoneid = 0; zoneid < MAX_NR_ZONES; zoneid++) {
692		- struct compact_control cc = {
693		- .nr_freepages = 0,
694		- .nr_migratepages = 0,
695		- .order = -1,
696		- .sync = true,
697		- };
698	687
699	688	zone = &pgdat->node_zones[zoneid];
700	689	if (!populated_zone(zone))
701	690	continue;
702	691
703		- cc.zone = zone;
704		- INIT_LIST_HEAD(&cc.freepages);
705		- INIT_LIST_HEAD(&cc.migratepages);
	692	+ cc->nr_freepages = 0;
	693	+ cc->nr_migratepages = 0;
	694	+ cc->zone = zone;
	695	+ INIT_LIST_HEAD(&cc->freepages);
	696	+ INIT_LIST_HEAD(&cc->migratepages);
706	697
707		- compact_zone(zone, &cc);
	698	+ if (cc->order < 0 \|\| !compaction_deferred(zone))
	699	+ compact_zone(zone, cc);
708	700
709		- VM_BUG_ON(!list_empty(&cc.freepages));
710		- VM_BUG_ON(!list_empty(&cc.migratepages));
	701	+ VM_BUG_ON(!list_empty(&cc->freepages));
	702	+ VM_BUG_ON(!list_empty(&cc->migratepages));
711	703	}
712	704
713	705	return 0;
	706	+}
	707	+
	708	+int compact_pgdat(pg_data_t *pgdat, int order)
	709	+{
	710	+ struct compact_control cc = {
	711	+ .order = order,
	712	+ .sync = false,
	713	+ };
	714	+
	715	+ return __compact_pgdat(pgdat, &cc);
	716	+}
	717	+
	718	+static int compact_node(int nid)
	719	+{
	720	+ pg_data_t *pgdat;
	721	+ struct compact_control cc = {
	722	+ .order = -1,
	723	+ .sync = true,
	724	+ };
	725	+
	726	+ if (nid < 0 \|\| nid >= nr_node_ids \|\| !node_online(nid))
	727	+ return -EINVAL;
	728	+ pgdat = NODE_DATA(nid);
	729	+
	730	+ return __compact_pgdat(pgdat, &cc);
714	731	}
715	732
716	733	/* Compact all nodes in the system */
...	...	@@ -2919,6 +2919,8 @@
2919	2919	* and it is potentially going to sleep here.
2920	2920	*/
2921	2921	if (order) {
	2922	+ int zones_need_compaction = 1;
	2923	+
2922	2924	for (i = 0; i <= end_zone; i++) {
2923	2925	struct zone *zone = pgdat->node_zones + i;
2924	2926
2925	2927
...	...	@@ -2939,9 +2941,17 @@
2939	2941	goto loop_again;
2940	2942	}
2941	2943
	2944	+ /* Check if the memory needs to be defragmented. */
	2945	+ if (zone_watermark_ok(zone, order,
	2946	+ low_wmark_pages(zone), *classzone_idx, 0))
	2947	+ zones_need_compaction = 0;
	2948	+
2942	2949	/* If balanced, clear the congested flag */
2943	2950	zone_clear_flag(zone, ZONE_CONGESTED);
2944	2951	}
	2952	+
	2953	+ if (zones_need_compaction)
	2954	+ compact_pgdat(pgdat, order);
2945	2955	}
2946	2956
2947	2957	/*