Commit e009bb30c8df8a52a9622b616b67436b6a03a0cd

Authored by Kirill A. Shutemov
Committed by Linus Torvalds
1 parent c4088ebdca

mm: implement split page table lock for PMD level

The basic idea is the same as with PTE level: the lock is embedded into
struct page of table's page.

We can't use mm->pmd_huge_pte to store pgtables for THP, since we don't
take mm->page_table_lock anymore.  Let's reuse page->lru of table's page
for that.

pgtable_pmd_page_ctor() returns true, if initialization is successful
and false otherwise.  Current implementation never fails, but assumption
that constructor can fail will help to port it to -rt where spinlock_t
is rather huge and cannot be embedded into struct page -- dynamic
allocation is required.

Signed-off-by: Naoya Horiguchi <n-horiguchi@ah.jp.nec.com>
Signed-off-by: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Tested-by: Alex Thorlton <athorlton@sgi.com>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: "Eric W . Biederman" <ebiederm@xmission.com>
Cc: "Paul E . McKenney" <paulmck@linux.vnet.ibm.com>
Cc: Al Viro <viro@zeniv.linux.org.uk>
Cc: Andi Kleen <ak@linux.intel.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Dave Hansen <dave.hansen@intel.com>
Cc: Dave Jones <davej@redhat.com>
Cc: David Howells <dhowells@redhat.com>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Kees Cook <keescook@chromium.org>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Michael Kerrisk <mtk.manpages@gmail.com>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Rik van Riel <riel@redhat.com>
Cc: Robin Holt <robinmholt@gmail.com>
Cc: Sedat Dilek <sedat.dilek@gmail.com>
Cc: Srikar Dronamraju <srikar@linux.vnet.ibm.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Hugh Dickins <hughd@google.com>
Reviewed-by: Steven Rostedt <rostedt@goodmis.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

Showing 4 changed files with 43 additions and 3 deletions Side-by-side Diff

... ... @@ -1378,12 +1378,44 @@
1378 1378 ((unlikely(pmd_none(*(pmd))) && __pte_alloc_kernel(pmd, address))? \
1379 1379 NULL: pte_offset_kernel(pmd, address))
1380 1380  
  1381 +#if USE_SPLIT_PMD_PTLOCKS
  1382 +
1381 1383 static inline spinlock_t *pmd_lockptr(struct mm_struct *mm, pmd_t *pmd)
1382 1384 {
  1385 + return &virt_to_page(pmd)->ptl;
  1386 +}
  1387 +
  1388 +static inline bool pgtable_pmd_page_ctor(struct page *page)
  1389 +{
  1390 + spin_lock_init(&page->ptl);
  1391 +#ifdef CONFIG_TRANSPARENT_HUGEPAGE
  1392 + page->pmd_huge_pte = NULL;
  1393 +#endif
  1394 + return true;
  1395 +}
  1396 +
  1397 +static inline void pgtable_pmd_page_dtor(struct page *page)
  1398 +{
  1399 +#ifdef CONFIG_TRANSPARENT_HUGEPAGE
  1400 + VM_BUG_ON(page->pmd_huge_pte);
  1401 +#endif
  1402 +}
  1403 +
  1404 +#define pmd_huge_pte(mm, pmd) (virt_to_page(pmd)->pmd_huge_pte)
  1405 +
  1406 +#else
  1407 +
  1408 +static inline spinlock_t *pmd_lockptr(struct mm_struct *mm, pmd_t *pmd)
  1409 +{
1383 1410 return &mm->page_table_lock;
1384 1411 }
1385 1412  
  1413 +static inline bool pgtable_pmd_page_ctor(struct page *page) { return true; }
  1414 +static inline void pgtable_pmd_page_dtor(struct page *page) {}
  1415 +
1386 1416 #define pmd_huge_pte(mm, pmd) ((mm)->pmd_huge_pte)
  1417 +
  1418 +#endif
1387 1419  
1388 1420 static inline spinlock_t *pmd_lock(struct mm_struct *mm, pmd_t *pmd)
1389 1421 {
include/linux/mm_types.h
... ... @@ -24,6 +24,8 @@
24 24 struct address_space;
25 25  
26 26 #define USE_SPLIT_PTE_PTLOCKS (NR_CPUS >= CONFIG_SPLIT_PTLOCK_CPUS)
  27 +#define USE_SPLIT_PMD_PTLOCKS (USE_SPLIT_PTE_PTLOCKS && \
  28 + IS_ENABLED(CONFIG_ARCH_ENABLE_SPLIT_PMD_PTLOCK))
27 29  
28 30 /*
29 31 * Each physical page in the system has a struct page associated with
... ... @@ -63,6 +65,9 @@
63 65 * this page is only used to
64 66 * free other pages.
65 67 */
  68 +#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && USE_SPLIT_PMD_PTLOCKS
  69 + pgtable_t pmd_huge_pte; /* protected by page->ptl */
  70 +#endif
66 71 };
67 72  
68 73 union {
... ... @@ -406,7 +411,7 @@
406 411 #ifdef CONFIG_MMU_NOTIFIER
407 412 struct mmu_notifier_mm *mmu_notifier_mm;
408 413 #endif
409   -#ifdef CONFIG_TRANSPARENT_HUGEPAGE
  414 +#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && !USE_SPLIT_PMD_PTLOCKS
410 415 pgtable_t pmd_huge_pte; /* protected by page_table_lock */
411 416 #endif
412 417 #ifdef CONFIG_CPUMASK_OFFSTACK
... ... @@ -560,7 +560,7 @@
560 560 "mm:%p idx:%d val:%ld\n", mm, i, x);
561 561 }
562 562  
563   -#ifdef CONFIG_TRANSPARENT_HUGEPAGE
  563 +#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && !USE_SPLIT_PMD_PTLOCKS
564 564 VM_BUG_ON(mm->pmd_huge_pte);
565 565 #endif
566 566 }
... ... @@ -814,7 +814,7 @@
814 814 memcpy(mm, oldmm, sizeof(*mm));
815 815 mm_init_cpumask(mm);
816 816  
817   -#ifdef CONFIG_TRANSPARENT_HUGEPAGE
  817 +#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && !USE_SPLIT_PMD_PTLOCKS
818 818 mm->pmd_huge_pte = NULL;
819 819 #endif
820 820 if (!mm_init(mm, tsk))
... ... @@ -222,6 +222,9 @@
222 222 default "999999" if !64BIT && GENERIC_LOCKBREAK
223 223 default "4"
224 224  
  225 +config ARCH_ENABLE_SPLIT_PMD_PTLOCK
  226 + boolean
  227 +
225 228 #
226 229 # support for memory balloon compaction
227 230 config BALLOON_COMPACTION