mm: migration: allow migration to operate asynchronously and avoid synchronous c…

…ompaction in the faster path Migration synchronously waits for writeback if the initial passes fails. Callers of memory compaction do not necessarily want this behaviour if the caller is latency sensitive or expects that synchronous migration is not going to have a significantly better success rate. This patch adds a sync parameter to migrate_pages() allowing the caller to indicate if wait_on_page_writeback() is allowed within migration or not. For reclaim/compaction, try_to_compact_pages() is first called asynchronously, direct reclaim runs and then try_to_compact_pages() is called synchronously as there is a greater expectation that it'll succeed. [akpm@linux-foundation.org: build/merge fix] Signed-off-by: Mel Gorman <mel@csn.ul.ie> Cc: Andrea Arcangeli <aarcange@redhat.com> Cc: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com> Cc: Rik van Riel <riel@redhat.com> Acked-by: Johannes Weiner <hannes@cmpxchg.org> Cc: Andy Whitcroft <apw@shadowen.org> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

mm: migration: allow migration to operate asynchronously and avoid synchronous c…
…ompaction in the faster path Migration synchronously waits for writeback if the initial passes fails. Callers of memory compaction do not necessarily want this behaviour if the caller is latency sensitive or expects that synchronous migration is not going to have a significantly better success rate. This patch adds a sync parameter to migrate_pages() allowing the caller to indicate if wait_on_page_writeback() is allowed within migration or not. For reclaim/compaction, try_to_compact_pages() is first called asynchronously, direct reclaim runs and then try_to_compact_pages() is called synchronously as there is a greater expectation that it'll succeed. [akpm@linux-foundation.org: build/merge fix] Signed-off-by: Mel Gorman <mel@csn.ul.ie> Cc: Andrea Arcangeli <aarcange@redhat.com> Cc: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com> Cc: Rik van Riel <riel@redhat.com> Acked-by: Johannes Weiner <hannes@cmpxchg.org> Cc: Andy Whitcroft <apw@shadowen.org> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
Mel Gorman · Linus Torvalds
1 parent 3e7d344970
Showing 9 changed files with 63 additions and 34 deletions Side-by-side Diff
include/linux/compaction.h
include/linux/migrate.h
mm/compaction.c
mm/memory-failure.c
mm/memory_hotplug.c
mm/mempolicy.c
mm/migrate.c
mm/page_alloc.c
mm/vmscan.c
@@ -21,10 +21,11 @@
  
 extern int fragmentation_index(struct zone *zone, unsigned int order);
 extern unsigned long try_to_compact_pages(struct zonelist *zonelist,
-			int order, gfp_t gfp_mask, nodemask_t *mask);
+			int order, gfp_t gfp_mask, nodemask_t *mask,
+			bool sync);
 extern unsigned long compaction_suitable(struct zone *zone, int order);
 extern unsigned long compact_zone_order(struct zone *zone, int order,
-						gfp_t gfp_mask);
+						gfp_t gfp_mask, bool sync);
  
 /* Do not skip compaction more than 64 times */
 #define COMPACT_MAX_DEFER_SHIFT 6
@@ -57,7 +58,8 @@
  
 #else
 static inline unsigned long try_to_compact_pages(struct zonelist *zonelist,
-			int order, gfp_t gfp_mask, nodemask_t *nodemask)
+			int order, gfp_t gfp_mask, nodemask_t *nodemask,
+			bool sync)
 {
 	return COMPACT_CONTINUE;
 }
@@ -68,7 +70,7 @@
 }
  
 static inline unsigned long compact_zone_order(struct zone *zone, int order,
-						gfp_t gfp_mask)
+						gfp_t gfp_mask, bool sync)
 {
 	return 0;
 }
@@ -13,9 +13,11 @@
 extern int migrate_page(struct address_space *,
 			struct page *, struct page *);
 extern int migrate_pages(struct list_head *l, new_page_t x,
-			unsigned long private, int offlining);
+			unsigned long private, int offlining,
+			bool sync);
 extern int migrate_huge_pages(struct list_head *l, new_page_t x,
-			unsigned long private, int offlining);
+			unsigned long private, int offlining,
+			bool sync);
  
 extern int fail_migrate_page(struct address_space *,
 			struct page *, struct page *);
  
@@ -33,9 +35,11 @@
  
 static inline void putback_lru_pages(struct list_head *l) {}
 static inline int migrate_pages(struct list_head *l, new_page_t x,
-		unsigned long private, int offlining) { return -ENOSYS; }
+		unsigned long private, int offlining,
+		bool sync) { return -ENOSYS; }
 static inline int migrate_huge_pages(struct list_head *l, new_page_t x,
-		unsigned long private, int offlining) { return -ENOSYS; }
+		unsigned long private, int offlining,
+		bool sync) { return -ENOSYS; }
  
 static inline int migrate_prep(void) { return -ENOSYS; }
 static inline int migrate_prep_local(void) { return -ENOSYS; }
@@ -33,6 +33,7 @@
 	unsigned long nr_migratepages;	/* Number of pages to migrate */
 	unsigned long free_pfn;		/* isolate_freepages search base */
 	unsigned long migrate_pfn;	/* isolate_migratepages search base */
+	bool sync;			/* Synchronous migration */
  
 	/* Account for isolated anon and file pages */
 	unsigned long nr_anon;
@@ -455,7 +456,8 @@
  
 		nr_migrate = cc->nr_migratepages;
 		migrate_pages(&cc->migratepages, compaction_alloc,
-						(unsigned long)cc, 0);
+				(unsigned long)cc, 0,
+				cc->sync);
 		update_nr_listpages(cc);
 		nr_remaining = cc->nr_migratepages;
  
@@ -482,7 +484,8 @@
 }
  
 unsigned long compact_zone_order(struct zone *zone,
-						int order, gfp_t gfp_mask)
+						int order, gfp_t gfp_mask,
+						bool sync)
 {
 	struct compact_control cc = {
 		.nr_freepages = 0,
@@ -490,6 +493,7 @@
 		.order = order,
 		.migratetype = allocflags_to_migratetype(gfp_mask),
 		.zone = zone,
+		.sync = sync,
 	};
 	INIT_LIST_HEAD(&cc.freepages);
 	INIT_LIST_HEAD(&cc.migratepages);
  
@@ -505,11 +509,13 @@
  * @order: The order of the current allocation
  * @gfp_mask: The GFP mask of the current allocation
  * @nodemask: The allowed nodes to allocate from
+ * @sync: Whether migration is synchronous or not
  *
  * This is the main entry point for direct page compaction.
  */
 unsigned long try_to_compact_pages(struct zonelist *zonelist,
-			int order, gfp_t gfp_mask, nodemask_t *nodemask)
+			int order, gfp_t gfp_mask, nodemask_t *nodemask,
+			bool sync)
 {
 	enum zone_type high_zoneidx = gfp_zone(gfp_mask);
 	int may_enter_fs = gfp_mask & __GFP_FS;
@@ -533,7 +539,7 @@
 								nodemask) {
 		int status;
  
-		status = compact_zone_order(zone, order, gfp_mask);
+		status = compact_zone_order(zone, order, gfp_mask, sync);
 		rc = max(status, rc);
  
 		/* If a normal allocation would succeed, stop compacting */
@@ -1290,9 +1290,10 @@
 	/* Keep page count to indicate a given hugepage is isolated. */
  
 	list_add(&hpage->lru, &pagelist);
-	ret = migrate_huge_pages(&pagelist, new_page, MPOL_MF_MOVE_ALL, 0);
+	ret = migrate_huge_pages(&pagelist, new_page, MPOL_MF_MOVE_ALL, 0,
+				true);
 	if (ret) {
-			putback_lru_pages(&pagelist);
+		putback_lru_pages(&pagelist);
 		pr_debug("soft offline: %#lx: migration failed %d, type %lx\n",
 			 pfn, ret, page->flags);
 		if (ret > 0)
@@ -1413,7 +1414,8 @@
 		LIST_HEAD(pagelist);
  
 		list_add(&page->lru, &pagelist);
-		ret = migrate_pages(&pagelist, new_page, MPOL_MF_MOVE_ALL, 0);
+		ret = migrate_pages(&pagelist, new_page, MPOL_MF_MOVE_ALL,
+								0, true);
 		if (ret) {
 			pr_info("soft offline: %#lx: migration failed %d, type %lx\n",
 				pfn, ret, page->flags);
@@ -733,7 +733,8 @@
 			goto out;
 		}
 		/* this function returns # of failed pages */
-		ret = migrate_pages(&source, hotremove_migrate_alloc, 0, 1);
+		ret = migrate_pages(&source, hotremove_migrate_alloc, 0,
+								1, true);
 		if (ret)
 			putback_lru_pages(&source);
 	}
@@ -935,7 +935,7 @@
 		return PTR_ERR(vma);
  
 	if (!list_empty(&pagelist)) {
-		err = migrate_pages(&pagelist, new_node_page, dest, 0);
+		err = migrate_pages(&pagelist, new_node_page, dest, 0, true);
 		if (err)
 			putback_lru_pages(&pagelist);
 	}
@@ -1155,7 +1155,7 @@
  
 		if (!list_empty(&pagelist)) {
 			nr_failed = migrate_pages(&pagelist, new_vma_page,
-						(unsigned long)vma, 0);
+						(unsigned long)vma, 0, true);
 			if (nr_failed)
 				putback_lru_pages(&pagelist);
 		}
@@ -614,7 +614,7 @@
  * to the newly allocated page in newpage.
  */
 static int unmap_and_move(new_page_t get_new_page, unsigned long private,
-			struct page *page, int force, int offlining)
+			struct page *page, int force, int offlining, bool sync)
 {
 	int rc = 0;
 	int *result = NULL;
@@ -682,7 +682,7 @@
 	BUG_ON(charge);
  
 	if (PageWriteback(page)) {
-		if (!force)
+		if (!force || !sync)
 			goto uncharge;
 		wait_on_page_writeback(page);
 	}
@@ -827,7 +827,7 @@
  */
 static int unmap_and_move_huge_page(new_page_t get_new_page,
 				unsigned long private, struct page *hpage,
-				int force, int offlining)
+				int force, int offlining, bool sync)
 {
 	int rc = 0;
 	int *result = NULL;
@@ -841,7 +841,7 @@
 	rc = -EAGAIN;
  
 	if (!trylock_page(hpage)) {
-		if (!force)
+		if (!force || !sync)
 			goto out;
 		lock_page(hpage);
 	}
@@ -909,7 +909,8 @@
  * Return: Number of pages not migrated or error code.
  */
 int migrate_pages(struct list_head *from,
-		new_page_t get_new_page, unsigned long private, int offlining)
+		new_page_t get_new_page, unsigned long private, int offlining,
+		bool sync)
 {
 	int retry = 1;
 	int nr_failed = 0;
@@ -929,7 +930,8 @@
 			cond_resched();
  
 			rc = unmap_and_move(get_new_page, private,
-						page, pass > 2, offlining);
+						page, pass > 2, offlining,
+						sync);
  
 			switch(rc) {
 			case -ENOMEM:
@@ -958,7 +960,8 @@
 }
  
 int migrate_huge_pages(struct list_head *from,
-		new_page_t get_new_page, unsigned long private, int offlining)
+		new_page_t get_new_page, unsigned long private, int offlining,
+		bool sync)
 {
 	int retry = 1;
 	int nr_failed = 0;
@@ -974,7 +977,8 @@
 			cond_resched();
  
 			rc = unmap_and_move_huge_page(get_new_page,
-					private, page, pass > 2, offlining);
+					private, page, pass > 2, offlining,
+					sync);
  
 			switch(rc) {
 			case -ENOMEM:
@@ -1107,7 +1111,7 @@
 	err = 0;
 	if (!list_empty(&pagelist)) {
 		err = migrate_pages(&pagelist, new_page_node,
-				(unsigned long)pm, 0);
+				(unsigned long)pm, 0, true);
 		if (err)
 			putback_lru_pages(&pagelist);
 	}
@@ -1812,7 +1812,8 @@
 __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
 	struct zonelist *zonelist, enum zone_type high_zoneidx,
 	nodemask_t *nodemask, int alloc_flags, struct zone *preferred_zone,
-	int migratetype, unsigned long *did_some_progress)
+	int migratetype, unsigned long *did_some_progress,
+	bool sync_migration)
 {
 	struct page *page;
 	struct task_struct *tsk = current;
@@ -1822,7 +1823,7 @@
  
 	tsk->flags |= PF_MEMALLOC;
 	*did_some_progress = try_to_compact_pages(zonelist, order, gfp_mask,
-								nodemask);
+						nodemask, sync_migration);
 	tsk->flags &= ~PF_MEMALLOC;
 	if (*did_some_progress != COMPACT_SKIPPED) {
  
@@ -1859,7 +1860,8 @@
 __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
 	struct zonelist *zonelist, enum zone_type high_zoneidx,
 	nodemask_t *nodemask, int alloc_flags, struct zone *preferred_zone,
-	int migratetype, unsigned long *did_some_progress)
+	int migratetype, unsigned long *did_some_progress,
+	bool sync_migration)
 {
 	return NULL;
 }
@@ -2001,6 +2003,7 @@
 	unsigned long pages_reclaimed = 0;
 	unsigned long did_some_progress;
 	struct task_struct *p = current;
+	bool sync_migration = false;
  
 	/*
 	 * In the slowpath, we sanity check order to avoid ever trying to
  
  
@@ -2063,14 +2066,19 @@
 	if (test_thread_flag(TIF_MEMDIE) && !(gfp_mask & __GFP_NOFAIL))
 		goto nopage;
  
-	/* Try direct compaction */
+	/*
+	 * Try direct compaction. The first pass is asynchronous. Subsequent
+	 * attempts after direct reclaim are synchronous
+	 */
 	page = __alloc_pages_direct_compact(gfp_mask, order,
 					zonelist, high_zoneidx,
 					nodemask,
 					alloc_flags, preferred_zone,
-					migratetype, &did_some_progress);
+					migratetype, &did_some_progress,
+					sync_migration);
 	if (page)
 		goto got_pg;
+	sync_migration = true;
  
 	/* Try direct reclaim and then allocating */
 	page = __alloc_pages_direct_reclaim(gfp_mask, order,
@@ -2134,7 +2142,8 @@
 					zonelist, high_zoneidx,
 					nodemask,
 					alloc_flags, preferred_zone,
-					migratetype, &did_some_progress);
+					migratetype, &did_some_progress,
+					sync_migration);
 		if (page)
 			goto got_pg;
 	}
@@ -2377,7 +2377,8 @@
 			 * would ordinarily call try_to_compact_pages()
 			 */
 			if (sc.order > PAGE_ALLOC_COSTLY_ORDER)
-				compact_zone_order(zone, sc.order, sc.gfp_mask);
+				compact_zone_order(zone, sc.order, sc.gfp_mask,
+							false);
  
 			if (!zone_watermark_ok_safe(zone, order,
 					high_wmark_pages(zone), end_zone, 0)) {
...	...	@@ -21,10 +21,11 @@
21	21
22	22	extern int fragmentation_index(struct zone *zone, unsigned int order);
23	23	extern unsigned long try_to_compact_pages(struct zonelist *zonelist,
24		- int order, gfp_t gfp_mask, nodemask_t *mask);
	24	+ int order, gfp_t gfp_mask, nodemask_t *mask,
	25	+ bool sync);
25	26	extern unsigned long compaction_suitable(struct zone *zone, int order);
26	27	extern unsigned long compact_zone_order(struct zone *zone, int order,
27		- gfp_t gfp_mask);
	28	+ gfp_t gfp_mask, bool sync);
28	29
29	30	/* Do not skip compaction more than 64 times */
30	31	#define COMPACT_MAX_DEFER_SHIFT 6
...	...	@@ -57,7 +58,8 @@
57	58
58	59	#else
59	60	static inline unsigned long try_to_compact_pages(struct zonelist *zonelist,
60		- int order, gfp_t gfp_mask, nodemask_t *nodemask)
	61	+ int order, gfp_t gfp_mask, nodemask_t *nodemask,
	62	+ bool sync)
61	63	{
62	64	return COMPACT_CONTINUE;
63	65	}
...	...	@@ -68,7 +70,7 @@
68	70	}
69	71
70	72	static inline unsigned long compact_zone_order(struct zone *zone, int order,
71		- gfp_t gfp_mask)
	73	+ gfp_t gfp_mask, bool sync)
72	74	{
73	75	return 0;
74	76	}
...	...	@@ -13,9 +13,11 @@
13	13	extern int migrate_page(struct address_space *,
14	14	struct page , struct page );
15	15	extern int migrate_pages(struct list_head *l, new_page_t x,
16		- unsigned long private, int offlining);
	16	+ unsigned long private, int offlining,
	17	+ bool sync);
17	18	extern int migrate_huge_pages(struct list_head *l, new_page_t x,
18		- unsigned long private, int offlining);
	19	+ unsigned long private, int offlining,
	20	+ bool sync);
19	21
20	22	extern int fail_migrate_page(struct address_space *,
21	23	struct page , struct page );
22	24
...	...	@@ -33,9 +35,11 @@
33	35
34	36	static inline void putback_lru_pages(struct list_head *l) {}
35	37	static inline int migrate_pages(struct list_head *l, new_page_t x,
36		- unsigned long private, int offlining) { return -ENOSYS; }
	38	+ unsigned long private, int offlining,
	39	+ bool sync) { return -ENOSYS; }
37	40	static inline int migrate_huge_pages(struct list_head *l, new_page_t x,
38		- unsigned long private, int offlining) { return -ENOSYS; }
	41	+ unsigned long private, int offlining,
	42	+ bool sync) { return -ENOSYS; }
39	43
40	44	static inline int migrate_prep(void) { return -ENOSYS; }
41	45	static inline int migrate_prep_local(void) { return -ENOSYS; }
...	...	@@ -33,6 +33,7 @@
33	33	unsigned long nr_migratepages; /* Number of pages to migrate */
34	34	unsigned long free_pfn; /* isolate_freepages search base */
35	35	unsigned long migrate_pfn; /* isolate_migratepages search base */
	36	+ bool sync; /* Synchronous migration */
36	37
37	38	/* Account for isolated anon and file pages */
38	39	unsigned long nr_anon;
...	...	@@ -455,7 +456,8 @@
455	456
456	457	nr_migrate = cc->nr_migratepages;
457	458	migrate_pages(&cc->migratepages, compaction_alloc,
458		- (unsigned long)cc, 0);
	459	+ (unsigned long)cc, 0,
	460	+ cc->sync);
459	461	update_nr_listpages(cc);
460	462	nr_remaining = cc->nr_migratepages;
461	463
...	...	@@ -482,7 +484,8 @@
482	484	}
483	485
484	486	unsigned long compact_zone_order(struct zone *zone,
485		- int order, gfp_t gfp_mask)
	487	+ int order, gfp_t gfp_mask,
	488	+ bool sync)
486	489	{
487	490	struct compact_control cc = {
488	491	.nr_freepages = 0,
...	...	@@ -490,6 +493,7 @@
490	493	.order = order,
491	494	.migratetype = allocflags_to_migratetype(gfp_mask),
492	495	.zone = zone,
	496	+ .sync = sync,
493	497	};
494	498	INIT_LIST_HEAD(&cc.freepages);
495	499	INIT_LIST_HEAD(&cc.migratepages);
496	500
...	...	@@ -505,11 +509,13 @@
505	509	* @order: The order of the current allocation
506	510	* @gfp_mask: The GFP mask of the current allocation
507	511	* @nodemask: The allowed nodes to allocate from
	512	+ * @sync: Whether migration is synchronous or not
508	513	*
509	514	* This is the main entry point for direct page compaction.
510	515	*/
511	516	unsigned long try_to_compact_pages(struct zonelist *zonelist,
512		- int order, gfp_t gfp_mask, nodemask_t *nodemask)
	517	+ int order, gfp_t gfp_mask, nodemask_t *nodemask,
	518	+ bool sync)
513	519	{
514	520	enum zone_type high_zoneidx = gfp_zone(gfp_mask);
515	521	int may_enter_fs = gfp_mask & __GFP_FS;
...	...	@@ -533,7 +539,7 @@
533	539	nodemask) {
534	540	int status;
535	541
536		- status = compact_zone_order(zone, order, gfp_mask);
	542	+ status = compact_zone_order(zone, order, gfp_mask, sync);
537	543	rc = max(status, rc);
538	544
539	545	/* If a normal allocation would succeed, stop compacting */
...	...	@@ -1290,9 +1290,10 @@
1290	1290	/* Keep page count to indicate a given hugepage is isolated. */
1291	1291
1292	1292	list_add(&hpage->lru, &pagelist);
1293		- ret = migrate_huge_pages(&pagelist, new_page, MPOL_MF_MOVE_ALL, 0);
	1293	+ ret = migrate_huge_pages(&pagelist, new_page, MPOL_MF_MOVE_ALL, 0,
	1294	+ true);
1294	1295	if (ret) {
1295		- putback_lru_pages(&pagelist);
	1296	+ putback_lru_pages(&pagelist);
1296	1297	pr_debug("soft offline: %#lx: migration failed %d, type %lx\n",
1297	1298	pfn, ret, page->flags);
1298	1299	if (ret > 0)
...	...	@@ -1413,7 +1414,8 @@
1413	1414	LIST_HEAD(pagelist);
1414	1415
1415	1416	list_add(&page->lru, &pagelist);
1416		- ret = migrate_pages(&pagelist, new_page, MPOL_MF_MOVE_ALL, 0);
	1417	+ ret = migrate_pages(&pagelist, new_page, MPOL_MF_MOVE_ALL,
	1418	+ 0, true);
1417	1419	if (ret) {
1418	1420	pr_info("soft offline: %#lx: migration failed %d, type %lx\n",
1419	1421	pfn, ret, page->flags);
...	...	@@ -733,7 +733,8 @@
733	733	goto out;
734	734	}
735	735	/* this function returns # of failed pages */
736		- ret = migrate_pages(&source, hotremove_migrate_alloc, 0, 1);
	736	+ ret = migrate_pages(&source, hotremove_migrate_alloc, 0,
	737	+ 1, true);
737	738	if (ret)
738	739	putback_lru_pages(&source);
739	740	}
...	...	@@ -935,7 +935,7 @@
935	935	return PTR_ERR(vma);
936	936
937	937	if (!list_empty(&pagelist)) {
938		- err = migrate_pages(&pagelist, new_node_page, dest, 0);
	938	+ err = migrate_pages(&pagelist, new_node_page, dest, 0, true);
939	939	if (err)
940	940	putback_lru_pages(&pagelist);
941	941	}
...	...	@@ -1155,7 +1155,7 @@
1155	1155
1156	1156	if (!list_empty(&pagelist)) {
1157	1157	nr_failed = migrate_pages(&pagelist, new_vma_page,
1158		- (unsigned long)vma, 0);
	1158	+ (unsigned long)vma, 0, true);
1159	1159	if (nr_failed)
1160	1160	putback_lru_pages(&pagelist);
1161	1161	}
...	...	@@ -614,7 +614,7 @@
614	614	* to the newly allocated page in newpage.
615	615	*/
616	616	static int unmap_and_move(new_page_t get_new_page, unsigned long private,
617		- struct page *page, int force, int offlining)
	617	+ struct page *page, int force, int offlining, bool sync)
618	618	{
619	619	int rc = 0;
620	620	int *result = NULL;
...	...	@@ -682,7 +682,7 @@
682	682	BUG_ON(charge);
683	683
684	684	if (PageWriteback(page)) {
685		- if (!force)
	685	+ if (!force \|\| !sync)
686	686	goto uncharge;
687	687	wait_on_page_writeback(page);
688	688	}
...	...	@@ -827,7 +827,7 @@
827	827	*/
828	828	static int unmap_and_move_huge_page(new_page_t get_new_page,
829	829	unsigned long private, struct page *hpage,
830		- int force, int offlining)
	830	+ int force, int offlining, bool sync)
831	831	{
832	832	int rc = 0;
833	833	int *result = NULL;
...	...	@@ -841,7 +841,7 @@
841	841	rc = -EAGAIN;
842	842
843	843	if (!trylock_page(hpage)) {
844		- if (!force)
	844	+ if (!force \|\| !sync)
845	845	goto out;
846	846	lock_page(hpage);
847	847	}
...	...	@@ -909,7 +909,8 @@
909	909	* Return: Number of pages not migrated or error code.
910	910	*/
911	911	int migrate_pages(struct list_head *from,
912		- new_page_t get_new_page, unsigned long private, int offlining)
	912	+ new_page_t get_new_page, unsigned long private, int offlining,
	913	+ bool sync)
913	914	{
914	915	int retry = 1;
915	916	int nr_failed = 0;
...	...	@@ -929,7 +930,8 @@
929	930	cond_resched();
930	931
931	932	rc = unmap_and_move(get_new_page, private,
932		- page, pass > 2, offlining);
	933	+ page, pass > 2, offlining,
	934	+ sync);
933	935
934	936	switch(rc) {
935	937	case -ENOMEM:
...	...	@@ -958,7 +960,8 @@
958	960	}
959	961
960	962	int migrate_huge_pages(struct list_head *from,
961		- new_page_t get_new_page, unsigned long private, int offlining)
	963	+ new_page_t get_new_page, unsigned long private, int offlining,
	964	+ bool sync)
962	965	{
963	966	int retry = 1;
964	967	int nr_failed = 0;
...	...	@@ -974,7 +977,8 @@
974	977	cond_resched();
975	978
976	979	rc = unmap_and_move_huge_page(get_new_page,
977		- private, page, pass > 2, offlining);
	980	+ private, page, pass > 2, offlining,
	981	+ sync);
978	982
979	983	switch(rc) {
980	984	case -ENOMEM:
...	...	@@ -1107,7 +1111,7 @@
1107	1111	err = 0;
1108	1112	if (!list_empty(&pagelist)) {
1109	1113	err = migrate_pages(&pagelist, new_page_node,
1110		- (unsigned long)pm, 0);
	1114	+ (unsigned long)pm, 0, true);
1111	1115	if (err)
1112	1116	putback_lru_pages(&pagelist);
1113	1117	}
...	...	@@ -1812,7 +1812,8 @@
1812	1812	__alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
1813	1813	struct zonelist *zonelist, enum zone_type high_zoneidx,
1814	1814	nodemask_t nodemask, int alloc_flags, struct zone preferred_zone,
1815		- int migratetype, unsigned long *did_some_progress)
	1815	+ int migratetype, unsigned long *did_some_progress,
	1816	+ bool sync_migration)
1816	1817	{
1817	1818	struct page *page;
1818	1819	struct task_struct *tsk = current;
...	...	@@ -1822,7 +1823,7 @@
1822	1823
1823	1824	tsk->flags \|= PF_MEMALLOC;
1824	1825	*did_some_progress = try_to_compact_pages(zonelist, order, gfp_mask,
1825		- nodemask);
	1826	+ nodemask, sync_migration);
1826	1827	tsk->flags &= ~PF_MEMALLOC;
1827	1828	if (*did_some_progress != COMPACT_SKIPPED) {
1828	1829
...	...	@@ -1859,7 +1860,8 @@
1859	1860	__alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
1860	1861	struct zonelist *zonelist, enum zone_type high_zoneidx,
1861	1862	nodemask_t nodemask, int alloc_flags, struct zone preferred_zone,
1862		- int migratetype, unsigned long *did_some_progress)
	1863	+ int migratetype, unsigned long *did_some_progress,
	1864	+ bool sync_migration)
1863	1865	{
1864	1866	return NULL;
1865	1867	}
...	...	@@ -2001,6 +2003,7 @@
2001	2003	unsigned long pages_reclaimed = 0;
2002	2004	unsigned long did_some_progress;
2003	2005	struct task_struct *p = current;
	2006	+ bool sync_migration = false;
2004	2007
2005	2008	/*
2006	2009	* In the slowpath, we sanity check order to avoid ever trying to
2007	2010
2008	2011
...	...	@@ -2063,14 +2066,19 @@
2063	2066	if (test_thread_flag(TIF_MEMDIE) && !(gfp_mask & __GFP_NOFAIL))
2064	2067	goto nopage;
2065	2068
2066		- /* Try direct compaction */
	2069	+ /*
	2070	+ * Try direct compaction. The first pass is asynchronous. Subsequent
	2071	+ * attempts after direct reclaim are synchronous
	2072	+ */
2067	2073	page = __alloc_pages_direct_compact(gfp_mask, order,
2068	2074	zonelist, high_zoneidx,
2069	2075	nodemask,
2070	2076	alloc_flags, preferred_zone,
2071		- migratetype, &did_some_progress);
	2077	+ migratetype, &did_some_progress,
	2078	+ sync_migration);
2072	2079	if (page)
2073	2080	goto got_pg;
	2081	+ sync_migration = true;
2074	2082
2075	2083	/* Try direct reclaim and then allocating */
2076	2084	page = __alloc_pages_direct_reclaim(gfp_mask, order,
...	...	@@ -2134,7 +2142,8 @@
2134	2142	zonelist, high_zoneidx,
2135	2143	nodemask,
2136	2144	alloc_flags, preferred_zone,
2137		- migratetype, &did_some_progress);
	2145	+ migratetype, &did_some_progress,
	2146	+ sync_migration);
2138	2147	if (page)
2139	2148	goto got_pg;
2140	2149	}
...	...	@@ -2377,7 +2377,8 @@
2377	2377	* would ordinarily call try_to_compact_pages()
2378	2378	*/
2379	2379	if (sc.order > PAGE_ALLOC_COSTLY_ORDER)
2380		- compact_zone_order(zone, sc.order, sc.gfp_mask);
	2380	+ compact_zone_order(zone, sc.order, sc.gfp_mask,
	2381	+ false);
2381	2382
2382	2383	if (!zone_watermark_ok_safe(zone, order,
2383	2384	high_wmark_pages(zone), end_zone, 0)) {