mm: split 'tlb_flush_mmu()' into tlb flushing and memory freeing parts

The mmu-gather operation 'tlb_flush_mmu()' has done two things: the actual tlb flush operation, and the batched freeing of the pages that the TLB entries pointed at. This splits the operation into separate phases, so that the forced batched flushing done by zap_pte_range() can now do the actual TLB flush while still holding the page table lock, but delay the batched freeing of all the pages to after the lock has been dropped. This in turn allows us to avoid a race condition between set_page_dirty() (as called by zap_pte_range() when it finds a dirty shared memory pte) and page_mkclean(): because we now flush all the dirty page data from the TLB's while holding the pte lock, page_mkclean() will be held up walking the (recently cleaned) page tables until after the TLB entries have been flushed from all CPU's. Reported-by: Benjamin Herrenschmidt <benh@kernel.crashing.org> Tested-by: Dave Hansen <dave.hansen@intel.com> Acked-by: Hugh Dickins <hughd@google.com> Cc: Peter Zijlstra <peterz@infradead.org> Cc: Russell King - ARM Linux <linux@arm.linux.org.uk> Cc: Tony Luck <tony.luck@intel.com> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

mm: split 'tlb_flush_mmu()' into tlb flushing and memory freeing parts
The mmu-gather operation 'tlb_flush_mmu()' has done two things: the actual tlb flush operation, and the batched freeing of the pages that the TLB entries pointed at. This splits the operation into separate phases, so that the forced batched flushing done by zap_pte_range() can now do the actual TLB flush while still holding the page table lock, but delay the batched freeing of all the pages to after the lock has been dropped. This in turn allows us to avoid a race condition between set_page_dirty() (as called by zap_pte_range() when it finds a dirty shared memory pte) and page_mkclean(): because we now flush all the dirty page data from the TLB's while holding the pte lock, page_mkclean() will be held up walking the (recently cleaned) page tables until after the TLB entries have been flushed from all CPU's. Reported-by: Benjamin Herrenschmidt <benh@kernel.crashing.org> Tested-by: Dave Hansen <dave.hansen@intel.com> Acked-by: Hugh Dickins <hughd@google.com> Cc: Peter Zijlstra <peterz@infradead.org> Cc: Russell King - ARM Linux <linux@arm.linux.org.uk> Cc: Tony Luck <tony.luck@intel.com> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
Linus Torvalds · Eric Lee · Eric Lee
1 parent 9a60ee117b
Showing 6 changed files with 111 additions and 33 deletions Side-by-side Diff
arch/arm/include/asm/tlb.h
arch/ia64/include/asm/tlb.h
arch/s390/include/asm/tlb.h
arch/sh/include/asm/tlb.h
arch/um/include/asm/tlb.h
mm/memory.c
@@ -98,13 +98,23 @@
 	}
 }
  
-static inline void tlb_flush_mmu(struct mmu_gather *tlb)
+static inline void tlb_flush_mmu_tlbonly(struct mmu_gather *tlb)
 {
 	tlb_flush(tlb);
+}
+
+static inline void tlb_flush_mmu_free(struct mmu_gather *tlb)
+{
 	free_pages_and_swap_cache(tlb->pages, tlb->nr);
 	tlb->nr = 0;
 	if (tlb->pages == tlb->local)
 		__tlb_alloc_page(tlb);
+}
+
+static inline void tlb_flush_mmu(struct mmu_gather *tlb)
+{
+	tlb_flush_mmu_tlbonly(tlb);
+	tlb_flush_mmu_free(tlb);
 }
  
 static inline void
@@ -91,18 +91,9 @@
 #define RR_RID_MASK	0x00000000ffffff00L
 #define RR_TO_RID(val) 	((val >> 8) & 0xffffff)
  
-/*
- * Flush the TLB for address range START to END and, if not in fast mode, release the
- * freed pages that where gathered up to this point.
- */
 static inline void
-ia64_tlb_flush_mmu (struct mmu_gather *tlb, unsigned long start, unsigned long end)
+ia64_tlb_flush_mmu_tlbonly(struct mmu_gather *tlb, unsigned long start, unsigned long end)
 {
-	unsigned long i;
-	unsigned int nr;
-
-	if (!tlb->need_flush)
-		return;
 	tlb->need_flush = 0;
  
 	if (tlb->fullmm) {
@@ -135,6 +126,14 @@
 		flush_tlb_range(&vma, ia64_thash(start), ia64_thash(end));
 	}
  
+}
+
+static inline void
+ia64_tlb_flush_mmu_free(struct mmu_gather *tlb)
+{
+	unsigned long i;
+	unsigned int nr;
+
 	/* lastly, release the freed pages */
 	nr = tlb->nr;
  
@@ -144,6 +143,19 @@
 		free_page_and_swap_cache(tlb->pages[i]);
 }
  
+/*
+ * Flush the TLB for address range START to END and, if not in fast mode, release the
+ * freed pages that where gathered up to this point.
+ */
+static inline void
+ia64_tlb_flush_mmu (struct mmu_gather *tlb, unsigned long start, unsigned long end)
+{
+	if (!tlb->need_flush)
+		return;
+	ia64_tlb_flush_mmu_tlbonly(tlb, start, end);
+	ia64_tlb_flush_mmu_free(tlb);
+}
+
 static inline void __tlb_alloc_page(struct mmu_gather *tlb)
 {
 	unsigned long addr = __get_free_pages(GFP_NOWAIT | __GFP_NOWARN, 0);
@@ -204,6 +216,16 @@
 	VM_BUG_ON(tlb->nr > tlb->max);
  
 	return tlb->max - tlb->nr;
+}
+
+static inline void tlb_flush_mmu_tlbonly(struct mmu_gather *tlb)
+{
+	ia64_tlb_flush_mmu_tlbonly(tlb, tlb->start_addr, tlb->end_addr);
+}
+
+static inline void tlb_flush_mmu_free(struct mmu_gather *tlb)
+{
+	ia64_tlb_flush_mmu_free(tlb);
 }
  
 static inline void tlb_flush_mmu(struct mmu_gather *tlb)
@@ -59,10 +59,21 @@
 	tlb->batch = NULL;
 }
  
-static inline void tlb_flush_mmu(struct mmu_gather *tlb)
+static inline void tlb_flush_mmu_tlbonly(struct mmu_gather *tlb)
 {
 	__tlb_flush_mm_lazy(tlb->mm);
+}
+
+static inline void tlb_flush_mmu_free(struct mmu_gather *tlb)
+{
 	tlb_table_flush(tlb);
+}
+
+
+static inline void tlb_flush_mmu(struct mmu_gather *tlb)
+{
+	tlb_flush_mmu_tlbonly(tlb);
+	tlb_flush_mmu_free(tlb);
 }
  
 static inline void tlb_finish_mmu(struct mmu_gather *tlb,
@@ -86,6 +86,14 @@
 	}
 }
  
+static inline void tlb_flush_mmu_tlbonly(struct mmu_gather *tlb)
+{
+}
+
+static inline void tlb_flush_mmu_free(struct mmu_gather *tlb)
+{
+}
+
 static inline void tlb_flush_mmu(struct mmu_gather *tlb)
 {
 }
@@ -59,13 +59,25 @@
 			       unsigned long end);
  
 static inline void
+tlb_flush_mmu_tlbonly(struct mmu_gather *tlb)
+{
+	flush_tlb_mm_range(tlb->mm, tlb->start, tlb->end);
+}
+
+static inline void
+tlb_flush_mmu_free(struct mmu_gather *tlb)
+{
+	init_tlb_gather(tlb);
+}
+
+static inline void
 tlb_flush_mmu(struct mmu_gather *tlb)
 {
 	if (!tlb->need_flush)
 		return;
  
-	flush_tlb_mm_range(tlb->mm, tlb->start, tlb->end);
-	init_tlb_gather(tlb);
+	tlb_flush_mmu_tlbonly(tlb);
+	tlb_flush_mmu_free(tlb);
 }
  
 /* tlb_finish_mmu
@@ -232,18 +232,19 @@
 #endif
 }
  
-void tlb_flush_mmu(struct mmu_gather *tlb)
+static void tlb_flush_mmu_tlbonly(struct mmu_gather *tlb)
 {
-	struct mmu_gather_batch *batch;
-
-	if (!tlb->need_flush)
-		return;
 	tlb->need_flush = 0;
 	tlb_flush(tlb);
 #ifdef CONFIG_HAVE_RCU_TABLE_FREE
 	tlb_table_flush(tlb);
 #endif
+}
  
+static void tlb_flush_mmu_free(struct mmu_gather *tlb)
+{
+	struct mmu_gather_batch *batch;
+
 	for (batch = &tlb->local; batch; batch = batch->next) {
 		free_pages_and_swap_cache(batch->pages, batch->nr);
 		batch->nr = 0;
@@ -251,6 +252,14 @@
 	tlb->active = &tlb->local;
 }
  
+void tlb_flush_mmu(struct mmu_gather *tlb)
+{
+	if (!tlb->need_flush)
+		return;
+	tlb_flush_mmu_tlbonly(tlb);
+	tlb_flush_mmu_free(tlb);
+}
+
 /* tlb_finish_mmu
  *	Called at the end of the shootdown operation to free up any resources
  *	that were required.
  
@@ -1127,8 +1136,10 @@
 			if (PageAnon(page))
 				rss[MM_ANONPAGES]--;
 			else {
-				if (pte_dirty(ptent))
+				if (pte_dirty(ptent)) {
+					force_flush = 1;
 					set_page_dirty(page);
+				}
 				if (pte_young(ptent) &&
 				    likely(!(vma->vm_flags & VM_SEQ_READ)))
 					mark_page_accessed(page);
  
@@ -1137,9 +1148,10 @@
 			page_remove_rmap(page);
 			if (unlikely(page_mapcount(page) < 0))
 				print_bad_pte(vma, addr, ptent, page);
-			force_flush = !__tlb_remove_page(tlb, page);
-			if (force_flush)
+			if (unlikely(!__tlb_remove_page(tlb, page))) {
+				force_flush = 1;
 				break;
+			}
 			continue;
 		}
 		/*
  
  
@@ -1174,18 +1186,11 @@
  
 	add_mm_rss_vec(mm, rss);
 	arch_leave_lazy_mmu_mode();
-	pte_unmap_unlock(start_pte, ptl);
  
-	/*
-	 * mmu_gather ran out of room to batch pages, we break out of
-	 * the PTE lock to avoid doing the potential expensive TLB invalidate
-	 * and page-free while holding it.
-	 */
+	/* Do the actual TLB flush before dropping ptl */
 	if (force_flush) {
 		unsigned long old_end;
  
-		force_flush = 0;
-
 		/*
 		 * Flush the TLB just for the previous segment,
 		 * then update the range to be the remaining
  
@@ -1193,11 +1198,21 @@
 		 */
 		old_end = tlb->end;
 		tlb->end = addr;
-
-		tlb_flush_mmu(tlb);
-
+		tlb_flush_mmu_tlbonly(tlb);
 		tlb->start = addr;
 		tlb->end = old_end;
+	}
+	pte_unmap_unlock(start_pte, ptl);
+
+	/*
+	 * If we forced a TLB flush (either due to running out of
+	 * batch buffers or because we needed to flush dirty TLB
+	 * entries before releasing the ptl), free the batched
+	 * memory too. Restart if we didn't do everything.
+	 */
+	if (force_flush) {
+		force_flush = 0;
+		tlb_flush_mmu_free(tlb);
  
 		if (addr != end)
 			goto again;
...	...	@@ -98,13 +98,23 @@
98	98	}
99	99	}
100	100
101		-static inline void tlb_flush_mmu(struct mmu_gather *tlb)
	101	+static inline void tlb_flush_mmu_tlbonly(struct mmu_gather *tlb)
102	102	{
103	103	tlb_flush(tlb);
	104	+}
	105	+
	106	+static inline void tlb_flush_mmu_free(struct mmu_gather *tlb)
	107	+{
104	108	free_pages_and_swap_cache(tlb->pages, tlb->nr);
105	109	tlb->nr = 0;
106	110	if (tlb->pages == tlb->local)
107	111	__tlb_alloc_page(tlb);
	112	+}
	113	+
	114	+static inline void tlb_flush_mmu(struct mmu_gather *tlb)
	115	+{
	116	+ tlb_flush_mmu_tlbonly(tlb);
	117	+ tlb_flush_mmu_free(tlb);
108	118	}
109	119
110	120	static inline void
...	...	@@ -91,18 +91,9 @@
91	91	#define RR_RID_MASK 0x00000000ffffff00L
92	92	#define RR_TO_RID(val) ((val >> 8) & 0xffffff)
93	93
94		-/*
95		- * Flush the TLB for address range START to END and, if not in fast mode, release the
96		- * freed pages that where gathered up to this point.
97		- */
98	94	static inline void
99		-ia64_tlb_flush_mmu (struct mmu_gather *tlb, unsigned long start, unsigned long end)
	95	+ia64_tlb_flush_mmu_tlbonly(struct mmu_gather *tlb, unsigned long start, unsigned long end)
100	96	{
101		- unsigned long i;
102		- unsigned int nr;
103		-
104		- if (!tlb->need_flush)
105		- return;
106	97	tlb->need_flush = 0;
107	98
108	99	if (tlb->fullmm) {
...	...	@@ -135,6 +126,14 @@
135	126	flush_tlb_range(&vma, ia64_thash(start), ia64_thash(end));
136	127	}
137	128
	129	+}
	130	+
	131	+static inline void
	132	+ia64_tlb_flush_mmu_free(struct mmu_gather *tlb)
	133	+{
	134	+ unsigned long i;
	135	+ unsigned int nr;
	136	+
138	137	/* lastly, release the freed pages */
139	138	nr = tlb->nr;
140	139
...	...	@@ -144,6 +143,19 @@
144	143	free_page_and_swap_cache(tlb->pages[i]);
145	144	}
146	145
	146	+/*
	147	+ * Flush the TLB for address range START to END and, if not in fast mode, release the
	148	+ * freed pages that where gathered up to this point.
	149	+ */
	150	+static inline void
	151	+ia64_tlb_flush_mmu (struct mmu_gather *tlb, unsigned long start, unsigned long end)
	152	+{
	153	+ if (!tlb->need_flush)
	154	+ return;
	155	+ ia64_tlb_flush_mmu_tlbonly(tlb, start, end);
	156	+ ia64_tlb_flush_mmu_free(tlb);
	157	+}
	158	+
147	159	static inline void __tlb_alloc_page(struct mmu_gather *tlb)
148	160	{
149	161	unsigned long addr = __get_free_pages(GFP_NOWAIT \| __GFP_NOWARN, 0);
...	...	@@ -204,6 +216,16 @@
204	216	VM_BUG_ON(tlb->nr > tlb->max);
205	217
206	218	return tlb->max - tlb->nr;
	219	+}
	220	+
	221	+static inline void tlb_flush_mmu_tlbonly(struct mmu_gather *tlb)
	222	+{
	223	+ ia64_tlb_flush_mmu_tlbonly(tlb, tlb->start_addr, tlb->end_addr);
	224	+}
	225	+
	226	+static inline void tlb_flush_mmu_free(struct mmu_gather *tlb)
	227	+{
	228	+ ia64_tlb_flush_mmu_free(tlb);
207	229	}
208	230
209	231	static inline void tlb_flush_mmu(struct mmu_gather *tlb)
...	...	@@ -59,10 +59,21 @@
59	59	tlb->batch = NULL;
60	60	}
61	61
62		-static inline void tlb_flush_mmu(struct mmu_gather *tlb)
	62	+static inline void tlb_flush_mmu_tlbonly(struct mmu_gather *tlb)
63	63	{
64	64	__tlb_flush_mm_lazy(tlb->mm);
	65	+}
	66	+
	67	+static inline void tlb_flush_mmu_free(struct mmu_gather *tlb)
	68	+{
65	69	tlb_table_flush(tlb);
	70	+}
	71	+
	72	+
	73	+static inline void tlb_flush_mmu(struct mmu_gather *tlb)
	74	+{
	75	+ tlb_flush_mmu_tlbonly(tlb);
	76	+ tlb_flush_mmu_free(tlb);
66	77	}
67	78
68	79	static inline void tlb_finish_mmu(struct mmu_gather *tlb,
...	...	@@ -86,6 +86,14 @@
86	86	}
87	87	}
88	88
	89	+static inline void tlb_flush_mmu_tlbonly(struct mmu_gather *tlb)
	90	+{
	91	+}
	92	+
	93	+static inline void tlb_flush_mmu_free(struct mmu_gather *tlb)
	94	+{
	95	+}
	96	+
89	97	static inline void tlb_flush_mmu(struct mmu_gather *tlb)
90	98	{
91	99	}
...	...	@@ -59,13 +59,25 @@
59	59	unsigned long end);
60	60
61	61	static inline void
	62	+tlb_flush_mmu_tlbonly(struct mmu_gather *tlb)
	63	+{
	64	+ flush_tlb_mm_range(tlb->mm, tlb->start, tlb->end);
	65	+}
	66	+
	67	+static inline void
	68	+tlb_flush_mmu_free(struct mmu_gather *tlb)
	69	+{
	70	+ init_tlb_gather(tlb);
	71	+}
	72	+
	73	+static inline void
62	74	tlb_flush_mmu(struct mmu_gather *tlb)
63	75	{
64	76	if (!tlb->need_flush)
65	77	return;
66	78
67		- flush_tlb_mm_range(tlb->mm, tlb->start, tlb->end);
68		- init_tlb_gather(tlb);
	79	+ tlb_flush_mmu_tlbonly(tlb);
	80	+ tlb_flush_mmu_free(tlb);
69	81	}
70	82
71	83	/* tlb_finish_mmu
...	...	@@ -232,18 +232,19 @@
232	232	#endif
233	233	}
234	234
235		-void tlb_flush_mmu(struct mmu_gather *tlb)
	235	+static void tlb_flush_mmu_tlbonly(struct mmu_gather *tlb)
236	236	{
237		- struct mmu_gather_batch *batch;
238		-
239		- if (!tlb->need_flush)
240		- return;
241	237	tlb->need_flush = 0;
242	238	tlb_flush(tlb);
243	239	#ifdef CONFIG_HAVE_RCU_TABLE_FREE
244	240	tlb_table_flush(tlb);
245	241	#endif
	242	+}
246	243
	244	+static void tlb_flush_mmu_free(struct mmu_gather *tlb)
	245	+{
	246	+ struct mmu_gather_batch *batch;
	247	+
247	248	for (batch = &tlb->local; batch; batch = batch->next) {
248	249	free_pages_and_swap_cache(batch->pages, batch->nr);
249	250	batch->nr = 0;
...	...	@@ -251,6 +252,14 @@
251	252	tlb->active = &tlb->local;
252	253	}
253	254
	255	+void tlb_flush_mmu(struct mmu_gather *tlb)
	256	+{
	257	+ if (!tlb->need_flush)
	258	+ return;
	259	+ tlb_flush_mmu_tlbonly(tlb);
	260	+ tlb_flush_mmu_free(tlb);
	261	+}
	262	+
254	263	/* tlb_finish_mmu
255	264	* Called at the end of the shootdown operation to free up any resources
256	265	* that were required.
257	266
...	...	@@ -1127,8 +1136,10 @@
1127	1136	if (PageAnon(page))
1128	1137	rss[MM_ANONPAGES]--;
1129	1138	else {
1130		- if (pte_dirty(ptent))
	1139	+ if (pte_dirty(ptent)) {
	1140	+ force_flush = 1;
1131	1141	set_page_dirty(page);
	1142	+ }
1132	1143	if (pte_young(ptent) &&
1133	1144	likely(!(vma->vm_flags & VM_SEQ_READ)))
1134	1145	mark_page_accessed(page);
1135	1146
...	...	@@ -1137,9 +1148,10 @@
1137	1148	page_remove_rmap(page);
1138	1149	if (unlikely(page_mapcount(page) < 0))
1139	1150	print_bad_pte(vma, addr, ptent, page);
1140		- force_flush = !__tlb_remove_page(tlb, page);
1141		- if (force_flush)
	1151	+ if (unlikely(!__tlb_remove_page(tlb, page))) {
	1152	+ force_flush = 1;
1142	1153	break;
	1154	+ }
1143	1155	continue;
1144	1156	}
1145	1157	/*
1146	1158
1147	1159
...	...	@@ -1174,18 +1186,11 @@
1174	1186
1175	1187	add_mm_rss_vec(mm, rss);
1176	1188	arch_leave_lazy_mmu_mode();
1177		- pte_unmap_unlock(start_pte, ptl);
1178	1189
1179		- /*
1180		- * mmu_gather ran out of room to batch pages, we break out of
1181		- * the PTE lock to avoid doing the potential expensive TLB invalidate
1182		- * and page-free while holding it.
1183		- */
	1190	+ /* Do the actual TLB flush before dropping ptl */
1184	1191	if (force_flush) {
1185	1192	unsigned long old_end;
1186	1193
1187		- force_flush = 0;
1188		-
1189	1194	/*
1190	1195	* Flush the TLB just for the previous segment,
1191	1196	* then update the range to be the remaining
1192	1197
...	...	@@ -1193,11 +1198,21 @@
1193	1198	*/
1194	1199	old_end = tlb->end;
1195	1200	tlb->end = addr;
1196		-
1197		- tlb_flush_mmu(tlb);
1198		-
	1201	+ tlb_flush_mmu_tlbonly(tlb);
1199	1202	tlb->start = addr;
1200	1203	tlb->end = old_end;
	1204	+ }
	1205	+ pte_unmap_unlock(start_pte, ptl);
	1206	+
	1207	+ /*
	1208	+ * If we forced a TLB flush (either due to running out of
	1209	+ * batch buffers or because we needed to flush dirty TLB
	1210	+ * entries before releasing the ptl), free the batched
	1211	+ * memory too. Restart if we didn't do everything.
	1212	+ */
	1213	+ if (force_flush) {
	1214	+ force_flush = 0;
	1215	+ tlb_flush_mmu_free(tlb);
1201	1216
1202	1217	if (addr != end)
1203	1218	goto again;