mm: numa: avoid unnecessary disruption of NUMA hinting during migration

do_huge_pmd_numa_page() handles the case where there is parallel THP migration. However, by the time it is checked the NUMA hinting information has already been disrupted. This patch adds an earlier check with some helpers. Signed-off-by: Mel Gorman <mgorman@suse.de> Reviewed-by: Rik van Riel <riel@redhat.com> Cc: Alex Thorlton <athorlton@sgi.com> Cc: <stable@vger.kernel.org> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

mm: numa: avoid unnecessary disruption of NUMA hinting during migration
do_huge_pmd_numa_page() handles the case where there is parallel THP migration. However, by the time it is checked the NUMA hinting information has already been disrupted. This patch adds an earlier check with some helpers. Signed-off-by: Mel Gorman <mgorman@suse.de> Reviewed-by: Rik van Riel <riel@redhat.com> Cc: Alex Thorlton <athorlton@sgi.com> Cc: <stable@vger.kernel.org> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
Mel Gorman · Linus Torvalds
1 parent 1667918b64
Showing 3 changed files with 37 additions and 6 deletions Side-by-side Diff
include/linux/migrate.h
mm/huge_memory.c
mm/migrate.c
@@ -90,10 +90,19 @@
 #endif /* CONFIG_MIGRATION */
  
 #ifdef CONFIG_NUMA_BALANCING
+extern bool pmd_trans_migrating(pmd_t pmd);
+extern void wait_migrate_huge_page(struct anon_vma *anon_vma, pmd_t *pmd);
 extern int migrate_misplaced_page(struct page *page,
 				  struct vm_area_struct *vma, int node);
 extern bool migrate_ratelimited(int node);
 #else
+static inline bool pmd_trans_migrating(pmd_t pmd)
+{
+	return false;
+}
+static inline void wait_migrate_huge_page(struct anon_vma *anon_vma, pmd_t *pmd)
+{
+}
 static inline int migrate_misplaced_page(struct page *page,
 					 struct vm_area_struct *vma, int node)
 {
@@ -882,6 +882,10 @@
 		ret = 0;
 		goto out_unlock;
 	}
+
+	/* mmap_sem prevents this happening but warn if that changes */
+	WARN_ON(pmd_trans_migrating(pmd));
+
 	if (unlikely(pmd_trans_splitting(pmd))) {
 		/* split huge page running from under us */
 		spin_unlock(src_ptl);
@@ -1299,6 +1303,17 @@
 	if (unlikely(!pmd_same(pmd, *pmdp)))
 		goto out_unlock;
  
+	/*
+	 * If there are potential migrations, wait for completion and retry
+	 * without disrupting NUMA hinting information. Do not relock and
+	 * check_same as the page may no longer be mapped.
+	 */
+	if (unlikely(pmd_trans_migrating(*pmdp))) {
+		spin_unlock(ptl);
+		wait_migrate_huge_page(vma->anon_vma, pmdp);
+		goto out;
+	}
+
 	page = pmd_page(pmd);
 	BUG_ON(is_huge_zero_page(page));
 	page_nid = page_to_nid(page);
@@ -1329,12 +1344,7 @@
 			goto clear_pmdnuma;
 	}
  
-	/*
-	 * If there are potential migrations, wait for completion and retry. We
-	 * do not relock and check_same as the page may no longer be mapped.
-	 * Furtermore, even if the page is currently misplaced, there is no
-	 * guarantee it is still misplaced after the migration completes.
-	 */
+	/* Migration could have started since the pmd_trans_migrating check */
 	if (!page_locked) {
 		spin_unlock(ptl);
 		wait_on_page_locked(page);
@@ -1655,6 +1655,18 @@
 	return 1;
 }
  
+bool pmd_trans_migrating(pmd_t pmd)
+{
+	struct page *page = pmd_page(pmd);
+	return PageLocked(page);
+}
+
+void wait_migrate_huge_page(struct anon_vma *anon_vma, pmd_t *pmd)
+{
+	struct page *page = pmd_page(*pmd);
+	wait_on_page_locked(page);
+}
+
 /*
  * Attempt to migrate a misplaced page to the specified destination
  * node. Caller is expected to have an elevated reference count on
...	...	@@ -90,10 +90,19 @@
90	90	#endif /* CONFIG_MIGRATION */
91	91
92	92	#ifdef CONFIG_NUMA_BALANCING
	93	+extern bool pmd_trans_migrating(pmd_t pmd);
	94	+extern void wait_migrate_huge_page(struct anon_vma anon_vma, pmd_t pmd);
93	95	extern int migrate_misplaced_page(struct page *page,
94	96	struct vm_area_struct *vma, int node);
95	97	extern bool migrate_ratelimited(int node);
96	98	#else
	99	+static inline bool pmd_trans_migrating(pmd_t pmd)
	100	+{
	101	+ return false;
	102	+}
	103	+static inline void wait_migrate_huge_page(struct anon_vma anon_vma, pmd_t pmd)
	104	+{
	105	+}
97	106	static inline int migrate_misplaced_page(struct page *page,
98	107	struct vm_area_struct *vma, int node)
99	108	{
...	...	@@ -882,6 +882,10 @@
882	882	ret = 0;
883	883	goto out_unlock;
884	884	}
	885	+
	886	+ /* mmap_sem prevents this happening but warn if that changes */
	887	+ WARN_ON(pmd_trans_migrating(pmd));
	888	+
885	889	if (unlikely(pmd_trans_splitting(pmd))) {
886	890	/* split huge page running from under us */
887	891	spin_unlock(src_ptl);
...	...	@@ -1299,6 +1303,17 @@
1299	1303	if (unlikely(!pmd_same(pmd, *pmdp)))
1300	1304	goto out_unlock;
1301	1305
	1306	+ /*
	1307	+ * If there are potential migrations, wait for completion and retry
	1308	+ * without disrupting NUMA hinting information. Do not relock and
	1309	+ * check_same as the page may no longer be mapped.
	1310	+ */
	1311	+ if (unlikely(pmd_trans_migrating(*pmdp))) {
	1312	+ spin_unlock(ptl);
	1313	+ wait_migrate_huge_page(vma->anon_vma, pmdp);
	1314	+ goto out;
	1315	+ }
	1316	+
1302	1317	page = pmd_page(pmd);
1303	1318	BUG_ON(is_huge_zero_page(page));
1304	1319	page_nid = page_to_nid(page);
...	...	@@ -1329,12 +1344,7 @@
1329	1344	goto clear_pmdnuma;
1330	1345	}
1331	1346
1332		- /*
1333		- * If there are potential migrations, wait for completion and retry. We
1334		- * do not relock and check_same as the page may no longer be mapped.
1335		- * Furtermore, even if the page is currently misplaced, there is no
1336		- * guarantee it is still misplaced after the migration completes.
1337		- */
	1347	+ /* Migration could have started since the pmd_trans_migrating check */
1338	1348	if (!page_locked) {
1339	1349	spin_unlock(ptl);
1340	1350	wait_on_page_locked(page);
...	...	@@ -1655,6 +1655,18 @@
1655	1655	return 1;
1656	1656	}
1657	1657
	1658	+bool pmd_trans_migrating(pmd_t pmd)
	1659	+{
	1660	+ struct page *page = pmd_page(pmd);
	1661	+ return PageLocked(page);
	1662	+}
	1663	+
	1664	+void wait_migrate_huge_page(struct anon_vma anon_vma, pmd_t pmd)
	1665	+{
	1666	+ struct page page = pmd_page(pmd);
	1667	+ wait_on_page_locked(page);
	1668	+}
	1669	+
1658	1670	/*
1659	1671	* Attempt to migrate a misplaced page to the specified destination
1660	1672	* node. Caller is expected to have an elevated reference count on