mm: thp: optimize memcg charge in khugepaged

We don't need to hold the mmmap_sem through mem_cgroup_newpage_charge(), the mmap_sem is only hold for keeping the vma stable and we don't need the vma stable anymore after we return from alloc_hugepage_vma(). Signed-off-by: Andrea Arcangeli <aarcange@redhat.com> Cc: Johannes Weiner <jweiner@redhat.com> Cc: Hugh Dickins <hughd@google.com> Cc: David Rientjes <rientjes@google.com> Cc: Minchan Kim <minchan.kim@gmail.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

mm: thp: optimize memcg charge in khugepaged
We don't need to hold the mmmap_sem through mem_cgroup_newpage_charge(), the mmap_sem is only hold for keeping the vma stable and we don't need the vma stable anymore after we return from alloc_hugepage_vma(). Signed-off-by: Andrea Arcangeli <aarcange@redhat.com> Cc: Johannes Weiner <jweiner@redhat.com> Cc: Hugh Dickins <hughd@google.com> Cc: David Rientjes <rientjes@google.com> Cc: Minchan Kim <minchan.kim@gmail.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
Andrea Arcangeli · Linus Torvalds
1 parent 9547d01bfb
Showing 1 changed file with 11 additions and 10 deletions Side-by-side Diff
mm/huge_memory.c
@@ -1771,12 +1771,9 @@
  
 	VM_BUG_ON(address & ~HPAGE_PMD_MASK);
 #ifndef CONFIG_NUMA
+	up_read(&mm->mmap_sem);
 	VM_BUG_ON(!*hpage);
 	new_page = *hpage;
-	if (unlikely(mem_cgroup_newpage_charge(new_page, mm, GFP_KERNEL))) {
-		up_read(&mm->mmap_sem);
-		return;
-	}
 #else
 	VM_BUG_ON(*hpage);
 	/*
  
  
  
  
  
@@ -1791,22 +1788,26 @@
 	 */
 	new_page = alloc_hugepage_vma(khugepaged_defrag(), vma, address,
 				      node, __GFP_OTHER_NODE);
+
+	/*
+	 * After allocating the hugepage, release the mmap_sem read lock in
+	 * preparation for taking it in write mode.
+	 */
+	up_read(&mm->mmap_sem);
 	if (unlikely(!new_page)) {
-		up_read(&mm->mmap_sem);
 		count_vm_event(THP_COLLAPSE_ALLOC_FAILED);
 		*hpage = ERR_PTR(-ENOMEM);
 		return;
 	}
+#endif
+
 	count_vm_event(THP_COLLAPSE_ALLOC);
 	if (unlikely(mem_cgroup_newpage_charge(new_page, mm, GFP_KERNEL))) {
-		up_read(&mm->mmap_sem);
+#ifdef CONFIG_NUMA
 		put_page(new_page);
+#endif
 		return;
 	}
-#endif
-
-	/* after allocating the hugepage upgrade to mmap_sem write mode */
-	up_read(&mm->mmap_sem);
  
 	/*
 	 * Prevent all access to pagetables with the exception of
...	...	@@ -1771,12 +1771,9 @@
1771	1771
1772	1772	VM_BUG_ON(address & ~HPAGE_PMD_MASK);
1773	1773	#ifndef CONFIG_NUMA
	1774	+ up_read(&mm->mmap_sem);
1774	1775	VM_BUG_ON(!*hpage);
1775	1776	new_page = *hpage;
1776		- if (unlikely(mem_cgroup_newpage_charge(new_page, mm, GFP_KERNEL))) {
1777		- up_read(&mm->mmap_sem);
1778		- return;
1779		- }
1780	1777	#else
1781	1778	VM_BUG_ON(*hpage);
1782	1779	/*
1783	1780
1784	1781
1785	1782
1786	1783
1787	1784
...	...	@@ -1791,22 +1788,26 @@
1791	1788	*/
1792	1789	new_page = alloc_hugepage_vma(khugepaged_defrag(), vma, address,
1793	1790	node, __GFP_OTHER_NODE);
	1791	+
	1792	+ /*
	1793	+ * After allocating the hugepage, release the mmap_sem read lock in
	1794	+ * preparation for taking it in write mode.
	1795	+ */
	1796	+ up_read(&mm->mmap_sem);
1794	1797	if (unlikely(!new_page)) {
1795		- up_read(&mm->mmap_sem);
1796	1798	count_vm_event(THP_COLLAPSE_ALLOC_FAILED);
1797	1799	*hpage = ERR_PTR(-ENOMEM);
1798	1800	return;
1799	1801	}
	1802	+#endif
	1803	+
1800	1804	count_vm_event(THP_COLLAPSE_ALLOC);
1801	1805	if (unlikely(mem_cgroup_newpage_charge(new_page, mm, GFP_KERNEL))) {
1802		- up_read(&mm->mmap_sem);
	1806	+#ifdef CONFIG_NUMA
1803	1807	put_page(new_page);
	1808	+#endif
1804	1809	return;
1805	1810	}
1806		-#endif
1807		-
1808		- /* after allocating the hugepage upgrade to mmap_sem write mode */
1809		- up_read(&mm->mmap_sem);
1810	1811
1811	1812	/*
1812	1813	* Prevent all access to pagetables with the exception of