Commit e009bb30c8df8a52a9622b616b67436b6a03a0cd
Committed by
Linus Torvalds
1 parent
c4088ebdca
Exists in
smarc-imx_3.14.28_1.0.0_ga
and in
1 other branch
mm: implement split page table lock for PMD level
The basic idea is the same as with PTE level: the lock is embedded into struct page of table's page. We can't use mm->pmd_huge_pte to store pgtables for THP, since we don't take mm->page_table_lock anymore. Let's reuse page->lru of table's page for that. pgtable_pmd_page_ctor() returns true, if initialization is successful and false otherwise. Current implementation never fails, but assumption that constructor can fail will help to port it to -rt where spinlock_t is rather huge and cannot be embedded into struct page -- dynamic allocation is required. Signed-off-by: Naoya Horiguchi <n-horiguchi@ah.jp.nec.com> Signed-off-by: Kirill A. Shutemov <kirill.shutemov@linux.intel.com> Tested-by: Alex Thorlton <athorlton@sgi.com> Cc: Ingo Molnar <mingo@redhat.com> Cc: "Eric W . Biederman" <ebiederm@xmission.com> Cc: "Paul E . McKenney" <paulmck@linux.vnet.ibm.com> Cc: Al Viro <viro@zeniv.linux.org.uk> Cc: Andi Kleen <ak@linux.intel.com> Cc: Andrea Arcangeli <aarcange@redhat.com> Cc: Dave Hansen <dave.hansen@intel.com> Cc: Dave Jones <davej@redhat.com> Cc: David Howells <dhowells@redhat.com> Cc: Frederic Weisbecker <fweisbec@gmail.com> Cc: Johannes Weiner <hannes@cmpxchg.org> Cc: Kees Cook <keescook@chromium.org> Cc: Mel Gorman <mgorman@suse.de> Cc: Michael Kerrisk <mtk.manpages@gmail.com> Cc: Oleg Nesterov <oleg@redhat.com> Cc: Peter Zijlstra <peterz@infradead.org> Cc: Rik van Riel <riel@redhat.com> Cc: Robin Holt <robinmholt@gmail.com> Cc: Sedat Dilek <sedat.dilek@gmail.com> Cc: Srikar Dronamraju <srikar@linux.vnet.ibm.com> Cc: Thomas Gleixner <tglx@linutronix.de> Cc: Hugh Dickins <hughd@google.com> Reviewed-by: Steven Rostedt <rostedt@goodmis.org> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
Showing 4 changed files with 43 additions and 3 deletions Side-by-side Diff
include/linux/mm.h
... | ... | @@ -1378,12 +1378,44 @@ |
1378 | 1378 | ((unlikely(pmd_none(*(pmd))) && __pte_alloc_kernel(pmd, address))? \ |
1379 | 1379 | NULL: pte_offset_kernel(pmd, address)) |
1380 | 1380 | |
1381 | +#if USE_SPLIT_PMD_PTLOCKS | |
1382 | + | |
1381 | 1383 | static inline spinlock_t *pmd_lockptr(struct mm_struct *mm, pmd_t *pmd) |
1382 | 1384 | { |
1385 | + return &virt_to_page(pmd)->ptl; | |
1386 | +} | |
1387 | + | |
1388 | +static inline bool pgtable_pmd_page_ctor(struct page *page) | |
1389 | +{ | |
1390 | + spin_lock_init(&page->ptl); | |
1391 | +#ifdef CONFIG_TRANSPARENT_HUGEPAGE | |
1392 | + page->pmd_huge_pte = NULL; | |
1393 | +#endif | |
1394 | + return true; | |
1395 | +} | |
1396 | + | |
1397 | +static inline void pgtable_pmd_page_dtor(struct page *page) | |
1398 | +{ | |
1399 | +#ifdef CONFIG_TRANSPARENT_HUGEPAGE | |
1400 | + VM_BUG_ON(page->pmd_huge_pte); | |
1401 | +#endif | |
1402 | +} | |
1403 | + | |
1404 | +#define pmd_huge_pte(mm, pmd) (virt_to_page(pmd)->pmd_huge_pte) | |
1405 | + | |
1406 | +#else | |
1407 | + | |
1408 | +static inline spinlock_t *pmd_lockptr(struct mm_struct *mm, pmd_t *pmd) | |
1409 | +{ | |
1383 | 1410 | return &mm->page_table_lock; |
1384 | 1411 | } |
1385 | 1412 | |
1413 | +static inline bool pgtable_pmd_page_ctor(struct page *page) { return true; } | |
1414 | +static inline void pgtable_pmd_page_dtor(struct page *page) {} | |
1415 | + | |
1386 | 1416 | #define pmd_huge_pte(mm, pmd) ((mm)->pmd_huge_pte) |
1417 | + | |
1418 | +#endif | |
1387 | 1419 | |
1388 | 1420 | static inline spinlock_t *pmd_lock(struct mm_struct *mm, pmd_t *pmd) |
1389 | 1421 | { |
include/linux/mm_types.h
... | ... | @@ -24,6 +24,8 @@ |
24 | 24 | struct address_space; |
25 | 25 | |
26 | 26 | #define USE_SPLIT_PTE_PTLOCKS (NR_CPUS >= CONFIG_SPLIT_PTLOCK_CPUS) |
27 | +#define USE_SPLIT_PMD_PTLOCKS (USE_SPLIT_PTE_PTLOCKS && \ | |
28 | + IS_ENABLED(CONFIG_ARCH_ENABLE_SPLIT_PMD_PTLOCK)) | |
27 | 29 | |
28 | 30 | /* |
29 | 31 | * Each physical page in the system has a struct page associated with |
... | ... | @@ -63,6 +65,9 @@ |
63 | 65 | * this page is only used to |
64 | 66 | * free other pages. |
65 | 67 | */ |
68 | +#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && USE_SPLIT_PMD_PTLOCKS | |
69 | + pgtable_t pmd_huge_pte; /* protected by page->ptl */ | |
70 | +#endif | |
66 | 71 | }; |
67 | 72 | |
68 | 73 | union { |
... | ... | @@ -406,7 +411,7 @@ |
406 | 411 | #ifdef CONFIG_MMU_NOTIFIER |
407 | 412 | struct mmu_notifier_mm *mmu_notifier_mm; |
408 | 413 | #endif |
409 | -#ifdef CONFIG_TRANSPARENT_HUGEPAGE | |
414 | +#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && !USE_SPLIT_PMD_PTLOCKS | |
410 | 415 | pgtable_t pmd_huge_pte; /* protected by page_table_lock */ |
411 | 416 | #endif |
412 | 417 | #ifdef CONFIG_CPUMASK_OFFSTACK |
kernel/fork.c
... | ... | @@ -560,7 +560,7 @@ |
560 | 560 | "mm:%p idx:%d val:%ld\n", mm, i, x); |
561 | 561 | } |
562 | 562 | |
563 | -#ifdef CONFIG_TRANSPARENT_HUGEPAGE | |
563 | +#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && !USE_SPLIT_PMD_PTLOCKS | |
564 | 564 | VM_BUG_ON(mm->pmd_huge_pte); |
565 | 565 | #endif |
566 | 566 | } |
... | ... | @@ -814,7 +814,7 @@ |
814 | 814 | memcpy(mm, oldmm, sizeof(*mm)); |
815 | 815 | mm_init_cpumask(mm); |
816 | 816 | |
817 | -#ifdef CONFIG_TRANSPARENT_HUGEPAGE | |
817 | +#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && !USE_SPLIT_PMD_PTLOCKS | |
818 | 818 | mm->pmd_huge_pte = NULL; |
819 | 819 | #endif |
820 | 820 | if (!mm_init(mm, tsk)) |
mm/Kconfig