mm: Count the number of pages affected in change_protection()

This will be used for three kinds of purposes: - to optimize mprotect() - to speed up working set scanning for working set areas that have not been touched - to more accurately scan per real working set No change in functionality from this patch. Suggested-by: Ingo Molnar <mingo@kernel.org> Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl> Cc: Linus Torvalds <torvalds@linux-foundation.org> Cc: Andrew Morton <akpm@linux-foundation.org> Cc: Andrea Arcangeli <aarcange@redhat.com> Cc: Rik van Riel <riel@redhat.com> Cc: Mel Gorman <mgorman@suse.de> Cc: Hugh Dickins <hughd@google.com> Cc: Thomas Gleixner <tglx@linutronix.de> Signed-off-by: Ingo Molnar <mingo@kernel.org>

mm: Count the number of pages affected in change_protection()
This will be used for three kinds of purposes: - to optimize mprotect() - to speed up working set scanning for working set areas that have not been touched - to more accurately scan per real working set No change in functionality from this patch. Suggested-by: Ingo Molnar <mingo@kernel.org> Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl> Cc: Linus Torvalds <torvalds@linux-foundation.org> Cc: Andrew Morton <akpm@linux-foundation.org> Cc: Andrea Arcangeli <aarcange@redhat.com> Cc: Rik van Riel <riel@redhat.com> Cc: Mel Gorman <mgorman@suse.de> Cc: Hugh Dickins <hughd@google.com> Cc: Thomas Gleixner <tglx@linutronix.de> Signed-off-by: Ingo Molnar <mingo@kernel.org>
Peter Zijlstra · Mel Gorman
1 parent 4fd017708c
Showing 4 changed files with 61 additions and 18 deletions Side-by-side Diff
include/linux/hugetlb.h
include/linux/mm.h
mm/hugetlb.c
mm/mprotect.c
@@ -87,7 +87,7 @@
 				pud_t *pud, int write);
 int pmd_huge(pmd_t pmd);
 int pud_huge(pud_t pmd);
-void hugetlb_change_protection(struct vm_area_struct *vma,
+unsigned long hugetlb_change_protection(struct vm_area_struct *vma,
 		unsigned long address, unsigned long end, pgprot_t newprot);
  
 #else /* !CONFIG_HUGETLB_PAGE */
@@ -132,7 +132,11 @@
 {
 }
  
-#define hugetlb_change_protection(vma, address, end, newprot)
+static inline unsigned long hugetlb_change_protection(struct vm_area_struct *vma,
+		unsigned long address, unsigned long end, pgprot_t newprot)
+{
+	return 0;
+}
  
 static inline void __unmap_hugepage_range_final(struct mmu_gather *tlb,
 			struct vm_area_struct *vma, unsigned long start,
@@ -1078,6 +1078,9 @@
 extern unsigned long do_mremap(unsigned long addr,
 			       unsigned long old_len, unsigned long new_len,
 			       unsigned long flags, unsigned long new_addr);
+extern unsigned long change_protection(struct vm_area_struct *vma, unsigned long start,
+			      unsigned long end, pgprot_t newprot,
+			      int dirty_accountable);
 extern int mprotect_fixup(struct vm_area_struct *vma,
 			  struct vm_area_struct **pprev, unsigned long start,
 			  unsigned long end, unsigned long newflags);
@@ -3014,7 +3014,7 @@
 	return i ? i : -EFAULT;
 }
  
-void hugetlb_change_protection(struct vm_area_struct *vma,
+unsigned long hugetlb_change_protection(struct vm_area_struct *vma,
 		unsigned long address, unsigned long end, pgprot_t newprot)
 {
 	struct mm_struct *mm = vma->vm_mm;
@@ -3022,6 +3022,7 @@
 	pte_t *ptep;
 	pte_t pte;
 	struct hstate *h = hstate_vma(vma);
+	unsigned long pages = 0;
  
 	BUG_ON(address >= end);
 	flush_cache_range(vma, address, end);
  
  
@@ -3032,12 +3033,15 @@
 		ptep = huge_pte_offset(mm, address);
 		if (!ptep)
 			continue;
-		if (huge_pmd_unshare(mm, &address, ptep))
+		if (huge_pmd_unshare(mm, &address, ptep)) {
+			pages++;
 			continue;
+		}
 		if (!huge_pte_none(huge_ptep_get(ptep))) {
 			pte = huge_ptep_get_and_clear(mm, address, ptep);
 			pte = pte_mkhuge(pte_modify(pte, newprot));
 			set_huge_pte_at(mm, address, ptep, pte);
+			pages++;
 		}
 	}
 	spin_unlock(&mm->page_table_lock);
@@ -3049,6 +3053,8 @@
 	 */
 	flush_tlb_range(vma, start, end);
 	mutex_unlock(&vma->vm_file->f_mapping->i_mmap_mutex);
+
+	return pages << h->order;
 }
  
 int hugetlb_reserve_pages(struct inode *inode,
@@ -35,12 +35,13 @@
 }
 #endif
  
-static void change_pte_range(struct mm_struct *mm, pmd_t *pmd,
+static unsigned long change_pte_range(struct mm_struct *mm, pmd_t *pmd,
 		unsigned long addr, unsigned long end, pgprot_t newprot,
 		int dirty_accountable)
 {
 	pte_t *pte, oldpte;
 	spinlock_t *ptl;
+	unsigned long pages = 0;
  
 	pte = pte_offset_map_lock(mm, pmd, addr, &ptl);
 	arch_enter_lazy_mmu_mode();
@@ -60,6 +61,7 @@
 				ptent = pte_mkwrite(ptent);
  
 			ptep_modify_prot_commit(mm, addr, pte, ptent);
+			pages++;
 		} else if (IS_ENABLED(CONFIG_MIGRATION) && !pte_file(oldpte)) {
 			swp_entry_t entry = pte_to_swp_entry(oldpte);
  
  
  
  
@@ -72,18 +74,22 @@
 				set_pte_at(mm, addr, pte,
 					swp_entry_to_pte(entry));
 			}
+			pages++;
 		}
 	} while (pte++, addr += PAGE_SIZE, addr != end);
 	arch_leave_lazy_mmu_mode();
 	pte_unmap_unlock(pte - 1, ptl);
+
+	return pages;
 }
  
-static inline void change_pmd_range(struct vm_area_struct *vma, pud_t *pud,
+static inline unsigned long change_pmd_range(struct vm_area_struct *vma, pud_t *pud,
 		unsigned long addr, unsigned long end, pgprot_t newprot,
 		int dirty_accountable)
 {
 	pmd_t *pmd;
 	unsigned long next;
+	unsigned long pages = 0;
  
 	pmd = pmd_offset(pud, addr);
 	do {
  
  
  
  
  
  
  
  
@@ -91,35 +97,42 @@
 		if (pmd_trans_huge(*pmd)) {
 			if (next - addr != HPAGE_PMD_SIZE)
 				split_huge_page_pmd(vma->vm_mm, pmd);
-			else if (change_huge_pmd(vma, pmd, addr, newprot))
+			else if (change_huge_pmd(vma, pmd, addr, newprot)) {
+				pages += HPAGE_PMD_NR;
 				continue;
+			}
 			/* fall through */
 		}
 		if (pmd_none_or_clear_bad(pmd))
 			continue;
-		change_pte_range(vma->vm_mm, pmd, addr, next, newprot,
+		pages += change_pte_range(vma->vm_mm, pmd, addr, next, newprot,
 				 dirty_accountable);
 	} while (pmd++, addr = next, addr != end);
+
+	return pages;
 }
  
-static inline void change_pud_range(struct vm_area_struct *vma, pgd_t *pgd,
+static inline unsigned long change_pud_range(struct vm_area_struct *vma, pgd_t *pgd,
 		unsigned long addr, unsigned long end, pgprot_t newprot,
 		int dirty_accountable)
 {
 	pud_t *pud;
 	unsigned long next;
+	unsigned long pages = 0;
  
 	pud = pud_offset(pgd, addr);
 	do {
 		next = pud_addr_end(addr, end);
 		if (pud_none_or_clear_bad(pud))
 			continue;
-		change_pmd_range(vma, pud, addr, next, newprot,
+		pages += change_pmd_range(vma, pud, addr, next, newprot,
 				 dirty_accountable);
 	} while (pud++, addr = next, addr != end);
+
+	return pages;
 }
  
-static void change_protection(struct vm_area_struct *vma,
+static unsigned long change_protection_range(struct vm_area_struct *vma,
 		unsigned long addr, unsigned long end, pgprot_t newprot,
 		int dirty_accountable)
 {
@@ -127,6 +140,7 @@
 	pgd_t *pgd;
 	unsigned long next;
 	unsigned long start = addr;
+	unsigned long pages = 0;
  
 	BUG_ON(addr >= end);
 	pgd = pgd_offset(mm, addr);
  
  
  
@@ -135,12 +149,32 @@
 		next = pgd_addr_end(addr, end);
 		if (pgd_none_or_clear_bad(pgd))
 			continue;
-		change_pud_range(vma, pgd, addr, next, newprot,
+		pages += change_pud_range(vma, pgd, addr, next, newprot,
 				 dirty_accountable);
 	} while (pgd++, addr = next, addr != end);
+
 	flush_tlb_range(vma, start, end);
+
+	return pages;
 }
  
+unsigned long change_protection(struct vm_area_struct *vma, unsigned long start,
+		       unsigned long end, pgprot_t newprot,
+		       int dirty_accountable)
+{
+	struct mm_struct *mm = vma->vm_mm;
+	unsigned long pages;
+
+	mmu_notifier_invalidate_range_start(mm, start, end);
+	if (is_vm_hugetlb_page(vma))
+		pages = hugetlb_change_protection(vma, start, end, newprot);
+	else
+		pages = change_protection_range(vma, start, end, newprot, dirty_accountable);
+	mmu_notifier_invalidate_range_end(mm, start, end);
+
+	return pages;
+}
+
 int
 mprotect_fixup(struct vm_area_struct *vma, struct vm_area_struct **pprev,
 	unsigned long start, unsigned long end, unsigned long newflags)
@@ -213,12 +247,8 @@
 		dirty_accountable = 1;
 	}
  
-	mmu_notifier_invalidate_range_start(mm, start, end);
-	if (is_vm_hugetlb_page(vma))
-		hugetlb_change_protection(vma, start, end, vma->vm_page_prot);
-	else
-		change_protection(vma, start, end, vma->vm_page_prot, dirty_accountable);
-	mmu_notifier_invalidate_range_end(mm, start, end);
+	change_protection(vma, start, end, vma->vm_page_prot, dirty_accountable);
+
 	vm_stat_account(mm, oldflags, vma->vm_file, -nrpages);
 	vm_stat_account(mm, newflags, vma->vm_file, nrpages);
 	perf_event_mmap(vma);
...	...	@@ -87,7 +87,7 @@
87	87	pud_t *pud, int write);
88	88	int pmd_huge(pmd_t pmd);
89	89	int pud_huge(pud_t pmd);
90		-void hugetlb_change_protection(struct vm_area_struct *vma,
	90	+unsigned long hugetlb_change_protection(struct vm_area_struct *vma,
91	91	unsigned long address, unsigned long end, pgprot_t newprot);
92	92
93	93	#else /* !CONFIG_HUGETLB_PAGE */
...	...	@@ -132,7 +132,11 @@
132	132	{
133	133	}
134	134
135		-#define hugetlb_change_protection(vma, address, end, newprot)
	135	+static inline unsigned long hugetlb_change_protection(struct vm_area_struct *vma,
	136	+ unsigned long address, unsigned long end, pgprot_t newprot)
	137	+{
	138	+ return 0;
	139	+}
136	140
137	141	static inline void __unmap_hugepage_range_final(struct mmu_gather *tlb,
138	142	struct vm_area_struct *vma, unsigned long start,
...	...	@@ -1078,6 +1078,9 @@
1078	1078	extern unsigned long do_mremap(unsigned long addr,
1079	1079	unsigned long old_len, unsigned long new_len,
1080	1080	unsigned long flags, unsigned long new_addr);
	1081	+extern unsigned long change_protection(struct vm_area_struct *vma, unsigned long start,
	1082	+ unsigned long end, pgprot_t newprot,
	1083	+ int dirty_accountable);
1081	1084	extern int mprotect_fixup(struct vm_area_struct *vma,
1082	1085	struct vm_area_struct **pprev, unsigned long start,
1083	1086	unsigned long end, unsigned long newflags);
...	...	@@ -3014,7 +3014,7 @@
3014	3014	return i ? i : -EFAULT;
3015	3015	}
3016	3016
3017		-void hugetlb_change_protection(struct vm_area_struct *vma,
	3017	+unsigned long hugetlb_change_protection(struct vm_area_struct *vma,
3018	3018	unsigned long address, unsigned long end, pgprot_t newprot)
3019	3019	{
3020	3020	struct mm_struct *mm = vma->vm_mm;
...	...	@@ -3022,6 +3022,7 @@
3022	3022	pte_t *ptep;
3023	3023	pte_t pte;
3024	3024	struct hstate *h = hstate_vma(vma);
	3025	+ unsigned long pages = 0;
3025	3026
3026	3027	BUG_ON(address >= end);
3027	3028	flush_cache_range(vma, address, end);
3028	3029
3029	3030
...	...	@@ -3032,12 +3033,15 @@
3032	3033	ptep = huge_pte_offset(mm, address);
3033	3034	if (!ptep)
3034	3035	continue;
3035		- if (huge_pmd_unshare(mm, &address, ptep))
	3036	+ if (huge_pmd_unshare(mm, &address, ptep)) {
	3037	+ pages++;
3036	3038	continue;
	3039	+ }
3037	3040	if (!huge_pte_none(huge_ptep_get(ptep))) {
3038	3041	pte = huge_ptep_get_and_clear(mm, address, ptep);
3039	3042	pte = pte_mkhuge(pte_modify(pte, newprot));
3040	3043	set_huge_pte_at(mm, address, ptep, pte);
	3044	+ pages++;
3041	3045	}
3042	3046	}
3043	3047	spin_unlock(&mm->page_table_lock);
...	...	@@ -3049,6 +3053,8 @@
3049	3053	*/
3050	3054	flush_tlb_range(vma, start, end);
3051	3055	mutex_unlock(&vma->vm_file->f_mapping->i_mmap_mutex);
	3056	+
	3057	+ return pages << h->order;
3052	3058	}
3053	3059
3054	3060	int hugetlb_reserve_pages(struct inode *inode,
...	...	@@ -35,12 +35,13 @@
35	35	}
36	36	#endif
37	37
38		-static void change_pte_range(struct mm_struct mm, pmd_t pmd,
	38	+static unsigned long change_pte_range(struct mm_struct mm, pmd_t pmd,
39	39	unsigned long addr, unsigned long end, pgprot_t newprot,
40	40	int dirty_accountable)
41	41	{
42	42	pte_t *pte, oldpte;
43	43	spinlock_t *ptl;
	44	+ unsigned long pages = 0;
44	45
45	46	pte = pte_offset_map_lock(mm, pmd, addr, &ptl);
46	47	arch_enter_lazy_mmu_mode();
...	...	@@ -60,6 +61,7 @@
60	61	ptent = pte_mkwrite(ptent);
61	62
62	63	ptep_modify_prot_commit(mm, addr, pte, ptent);
	64	+ pages++;
63	65	} else if (IS_ENABLED(CONFIG_MIGRATION) && !pte_file(oldpte)) {
64	66	swp_entry_t entry = pte_to_swp_entry(oldpte);
65	67
66	68
67	69
68	70
...	...	@@ -72,18 +74,22 @@
72	74	set_pte_at(mm, addr, pte,
73	75	swp_entry_to_pte(entry));
74	76	}
	77	+ pages++;
75	78	}
76	79	} while (pte++, addr += PAGE_SIZE, addr != end);
77	80	arch_leave_lazy_mmu_mode();
78	81	pte_unmap_unlock(pte - 1, ptl);
	82	+
	83	+ return pages;
79	84	}
80	85
81		-static inline void change_pmd_range(struct vm_area_struct vma, pud_t pud,
	86	+static inline unsigned long change_pmd_range(struct vm_area_struct vma, pud_t pud,
82	87	unsigned long addr, unsigned long end, pgprot_t newprot,
83	88	int dirty_accountable)
84	89	{
85	90	pmd_t *pmd;
86	91	unsigned long next;
	92	+ unsigned long pages = 0;
87	93
88	94	pmd = pmd_offset(pud, addr);
89	95	do {
90	96
91	97
92	98
93	99
94	100
95	101
96	102
97	103
...	...	@@ -91,35 +97,42 @@
91	97	if (pmd_trans_huge(*pmd)) {
92	98	if (next - addr != HPAGE_PMD_SIZE)
93	99	split_huge_page_pmd(vma->vm_mm, pmd);
94		- else if (change_huge_pmd(vma, pmd, addr, newprot))
	100	+ else if (change_huge_pmd(vma, pmd, addr, newprot)) {
	101	+ pages += HPAGE_PMD_NR;
95	102	continue;
	103	+ }
96	104	/* fall through */
97	105	}
98	106	if (pmd_none_or_clear_bad(pmd))
99	107	continue;
100		- change_pte_range(vma->vm_mm, pmd, addr, next, newprot,
	108	+ pages += change_pte_range(vma->vm_mm, pmd, addr, next, newprot,
101	109	dirty_accountable);
102	110	} while (pmd++, addr = next, addr != end);
	111	+
	112	+ return pages;
103	113	}
104	114
105		-static inline void change_pud_range(struct vm_area_struct vma, pgd_t pgd,
	115	+static inline unsigned long change_pud_range(struct vm_area_struct vma, pgd_t pgd,
106	116	unsigned long addr, unsigned long end, pgprot_t newprot,
107	117	int dirty_accountable)
108	118	{
109	119	pud_t *pud;
110	120	unsigned long next;
	121	+ unsigned long pages = 0;
111	122
112	123	pud = pud_offset(pgd, addr);
113	124	do {
114	125	next = pud_addr_end(addr, end);
115	126	if (pud_none_or_clear_bad(pud))
116	127	continue;
117		- change_pmd_range(vma, pud, addr, next, newprot,
	128	+ pages += change_pmd_range(vma, pud, addr, next, newprot,
118	129	dirty_accountable);
119	130	} while (pud++, addr = next, addr != end);
	131	+
	132	+ return pages;
120	133	}
121	134
122		-static void change_protection(struct vm_area_struct *vma,
	135	+static unsigned long change_protection_range(struct vm_area_struct *vma,
123	136	unsigned long addr, unsigned long end, pgprot_t newprot,
124	137	int dirty_accountable)
125	138	{
...	...	@@ -127,6 +140,7 @@
127	140	pgd_t *pgd;
128	141	unsigned long next;
129	142	unsigned long start = addr;
	143	+ unsigned long pages = 0;
130	144
131	145	BUG_ON(addr >= end);
132	146	pgd = pgd_offset(mm, addr);
133	147
134	148
135	149
...	...	@@ -135,12 +149,32 @@
135	149	next = pgd_addr_end(addr, end);
136	150	if (pgd_none_or_clear_bad(pgd))
137	151	continue;
138		- change_pud_range(vma, pgd, addr, next, newprot,
	152	+ pages += change_pud_range(vma, pgd, addr, next, newprot,
139	153	dirty_accountable);
140	154	} while (pgd++, addr = next, addr != end);
	155	+
141	156	flush_tlb_range(vma, start, end);
	157	+
	158	+ return pages;
142	159	}
143	160
	161	+unsigned long change_protection(struct vm_area_struct *vma, unsigned long start,
	162	+ unsigned long end, pgprot_t newprot,
	163	+ int dirty_accountable)
	164	+{
	165	+ struct mm_struct *mm = vma->vm_mm;
	166	+ unsigned long pages;
	167	+
	168	+ mmu_notifier_invalidate_range_start(mm, start, end);
	169	+ if (is_vm_hugetlb_page(vma))
	170	+ pages = hugetlb_change_protection(vma, start, end, newprot);
	171	+ else
	172	+ pages = change_protection_range(vma, start, end, newprot, dirty_accountable);
	173	+ mmu_notifier_invalidate_range_end(mm, start, end);
	174	+
	175	+ return pages;
	176	+}
	177	+
144	178	int
145	179	mprotect_fixup(struct vm_area_struct vma, struct vm_area_struct *pprev,
146	180	unsigned long start, unsigned long end, unsigned long newflags)
...	...	@@ -213,12 +247,8 @@
213	247	dirty_accountable = 1;
214	248	}
215	249
216		- mmu_notifier_invalidate_range_start(mm, start, end);
217		- if (is_vm_hugetlb_page(vma))
218		- hugetlb_change_protection(vma, start, end, vma->vm_page_prot);
219		- else
220		- change_protection(vma, start, end, vma->vm_page_prot, dirty_accountable);
221		- mmu_notifier_invalidate_range_end(mm, start, end);
	250	+ change_protection(vma, start, end, vma->vm_page_prot, dirty_accountable);
	251	+
222	252	vm_stat_account(mm, oldflags, vma->vm_file, -nrpages);
223	253	vm_stat_account(mm, newflags, vma->vm_file, nrpages);
224	254	perf_event_mmap(vma);