mm/swap.c: reorganize put_compound_page()

Tweak it so save a tab stop, make code layout slightly less nutty. Signed-off-by: Andrea Arcangeli <aarcange@redhat.com> Cc: Khalid Aziz <khalid.aziz@oracle.com> Cc: Pravin Shelar <pshelar@nicira.com> Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org> Cc: Ben Hutchings <bhutchings@solarflare.com> Cc: Christoph Lameter <cl@linux.com> Cc: Johannes Weiner <jweiner@redhat.com> Cc: Mel Gorman <mgorman@suse.de> Cc: Rik van Riel <riel@redhat.com> Cc: Andi Kleen <andi@firstfloor.org> Cc: Minchan Kim <minchan@kernel.org> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

mm/swap.c: reorganize put_compound_page()
Tweak it so save a tab stop, make code layout slightly less nutty. Signed-off-by: Andrea Arcangeli <aarcange@redhat.com> Cc: Khalid Aziz <khalid.aziz@oracle.com> Cc: Pravin Shelar <pshelar@nicira.com> Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org> Cc: Ben Hutchings <bhutchings@solarflare.com> Cc: Christoph Lameter <cl@linux.com> Cc: Johannes Weiner <jweiner@redhat.com> Cc: Mel Gorman <mgorman@suse.de> Cc: Rik van Riel <riel@redhat.com> Cc: Andi Kleen <andi@firstfloor.org> Cc: Minchan Kim <minchan@kernel.org> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
Andrew Morton · Linus Torvalds
1 parent 758f66a29c
Showing 1 changed file with 125 additions and 129 deletions Side-by-side Diff
mm/swap.c
@@ -81,154 +81,150 @@
  
 static void put_compound_page(struct page *page)
 {
-	if (unlikely(PageTail(page))) {
-		/* __split_huge_page_refcount can run under us */
-		struct page *page_head = compound_trans_head(page);
+	struct page *page_head;
  
-		/*
-		 * THP can not break up slab pages so avoid taking
-		 * compound_lock() and skip the tail page refcounting
-		 * (in _mapcount) too. Slab performs non-atomic bit
-		 * ops on page->flags for better performance. In
-		 * particular slab_unlock() in slub used to be a hot
-		 * path. It is still hot on arches that do not support
-		 * this_cpu_cmpxchg_double().
-		 *
-		 * If "page" is part of a slab or hugetlbfs page it
-		 * cannot be splitted and the head page cannot change
-		 * from under us. And if "page" is part of a THP page
-		 * under splitting, if the head page pointed by the
-		 * THP tail isn't a THP head anymore, we'll find
-		 * PageTail clear after smp_rmb() and we'll treat it
-		 * as a single page.
-		 */
-		if (!__compound_tail_refcounted(page_head)) {
+	if (likely(!PageTail(page))) {
+		if (put_page_testzero(page)) {
 			/*
-			 * If "page" is a THP tail, we must read the tail page
-			 * flags after the head page flags. The
-			 * split_huge_page side enforces write memory
-			 * barriers between clearing PageTail and before the
-			 * head page can be freed and reallocated.
+			 * By the time all refcounts have been released
+			 * split_huge_page cannot run anymore from under us.
 			 */
-			smp_rmb();
-			if (likely(PageTail(page))) {
-				/*
-				 * __split_huge_page_refcount
-				 * cannot race here.
-				 */
-				VM_BUG_ON(!PageHead(page_head));
-				VM_BUG_ON(page_mapcount(page) != 0);
-				if (put_page_testzero(page_head)) {
-					/*
-					 * If this is the tail of a
-					 * slab compound page, the
-					 * tail pin must not be the
-					 * last reference held on the
-					 * page, because the PG_slab
-					 * cannot be cleared before
-					 * all tail pins (which skips
-					 * the _mapcount tail
-					 * refcounting) have been
-					 * released. For hugetlbfs the
-					 * tail pin may be the last
-					 * reference on the page
-					 * instead, because
-					 * PageHeadHuge will not go
-					 * away until the compound
-					 * page enters the buddy
-					 * allocator.
-					 */
-					VM_BUG_ON(PageSlab(page_head));
-					__put_compound_page(page_head);
-				}
-				return;
-			} else
-				/*
-				 * __split_huge_page_refcount
-				 * run before us, "page" was a
-				 * THP tail. The split
-				 * page_head has been freed
-				 * and reallocated as slab or
-				 * hugetlbfs page of smaller
-				 * order (only possible if
-				 * reallocated as slab on
-				 * x86).
-				 */
-				goto out_put_single;
+			if (PageHead(page))
+				__put_compound_page(page);
+			else
+				__put_single_page(page);
 		}
+		return;
+	}
  
-		if (likely(page != page_head &&
-			   get_page_unless_zero(page_head))) {
-			unsigned long flags;
+	/* __split_huge_page_refcount can run under us */
+	page_head = compound_trans_head(page);
  
+	/*
+	 * THP can not break up slab pages so avoid taking
+	 * compound_lock() and skip the tail page refcounting (in
+	 * _mapcount) too. Slab performs non-atomic bit ops on
+	 * page->flags for better performance. In particular
+	 * slab_unlock() in slub used to be a hot path. It is still
+	 * hot on arches that do not support
+	 * this_cpu_cmpxchg_double().
+	 *
+	 * If "page" is part of a slab or hugetlbfs page it cannot be
+	 * splitted and the head page cannot change from under us. And
+	 * if "page" is part of a THP page under splitting, if the
+	 * head page pointed by the THP tail isn't a THP head anymore,
+	 * we'll find PageTail clear after smp_rmb() and we'll treat
+	 * it as a single page.
+	 */
+	if (!__compound_tail_refcounted(page_head)) {
+		/*
+		 * If "page" is a THP tail, we must read the tail page
+		 * flags after the head page flags. The
+		 * split_huge_page side enforces write memory barriers
+		 * between clearing PageTail and before the head page
+		 * can be freed and reallocated.
+		 */
+		smp_rmb();
+		if (likely(PageTail(page))) {
 			/*
-			 * page_head wasn't a dangling pointer but it
-			 * may not be a head page anymore by the time
-			 * we obtain the lock. That is ok as long as it
-			 * can't be freed from under us.
+			 * __split_huge_page_refcount cannot race
+			 * here.
 			 */
-			flags = compound_lock_irqsave(page_head);
-			if (unlikely(!PageTail(page))) {
-				/* __split_huge_page_refcount run before us */
-				compound_unlock_irqrestore(page_head, flags);
-				if (put_page_testzero(page_head)) {
-					/*
-					 * The head page may have been
-					 * freed and reallocated as a
-					 * compound page of smaller
-					 * order and then freed again.
-					 * All we know is that it
-					 * cannot have become: a THP
-					 * page, a compound page of
-					 * higher order, a tail page.
-					 * That is because we still
-					 * hold the refcount of the
-					 * split THP tail and
-					 * page_head was the THP head
-					 * before the split.
-					 */
-					if (PageHead(page_head))
-						__put_compound_page(page_head);
-					else
-						__put_single_page(page_head);
-				}
-out_put_single:
-				if (put_page_testzero(page))
-					__put_single_page(page);
-				return;
+			VM_BUG_ON(!PageHead(page_head));
+			VM_BUG_ON(page_mapcount(page) != 0);
+			if (put_page_testzero(page_head)) {
+				/*
+				 * If this is the tail of a slab
+				 * compound page, the tail pin must
+				 * not be the last reference held on
+				 * the page, because the PG_slab
+				 * cannot be cleared before all tail
+				 * pins (which skips the _mapcount
+				 * tail refcounting) have been
+				 * released. For hugetlbfs the tail
+				 * pin may be the last reference on
+				 * the page instead, because
+				 * PageHeadHuge will not go away until
+				 * the compound page enters the buddy
+				 * allocator.
+				 */
+				VM_BUG_ON(PageSlab(page_head));
+				__put_compound_page(page_head);
 			}
-			VM_BUG_ON(page_head != page->first_page);
+			return;
+		} else
 			/*
-			 * We can release the refcount taken by
-			 * get_page_unless_zero() now that
-			 * __split_huge_page_refcount() is blocked on
-			 * the compound_lock.
+			 * __split_huge_page_refcount run before us,
+			 * "page" was a THP tail. The split page_head
+			 * has been freed and reallocated as slab or
+			 * hugetlbfs page of smaller order (only
+			 * possible if reallocated as slab on x86).
 			 */
-			if (put_page_testzero(page_head))
-				VM_BUG_ON(1);
-			/* __split_huge_page_refcount will wait now */
-			VM_BUG_ON(page_mapcount(page) <= 0);
-			atomic_dec(&page->_mapcount);
-			VM_BUG_ON(atomic_read(&page_head->_count) <= 0);
-			VM_BUG_ON(atomic_read(&page->_count) != 0);
-			compound_unlock_irqrestore(page_head, flags);
+			goto out_put_single;
+	}
  
+	if (likely(page != page_head && get_page_unless_zero(page_head))) {
+		unsigned long flags;
+
+		/*
+		 * page_head wasn't a dangling pointer but it may not
+		 * be a head page anymore by the time we obtain the
+		 * lock. That is ok as long as it can't be freed from
+		 * under us.
+		 */
+		flags = compound_lock_irqsave(page_head);
+		if (unlikely(!PageTail(page))) {
+			/* __split_huge_page_refcount run before us */
+			compound_unlock_irqrestore(page_head, flags);
 			if (put_page_testzero(page_head)) {
+				/*
+				 * The head page may have been freed
+				 * and reallocated as a compound page
+				 * of smaller order and then freed
+				 * again.  All we know is that it
+				 * cannot have become: a THP page, a
+				 * compound page of higher order, a
+				 * tail page.  That is because we
+				 * still hold the refcount of the
+				 * split THP tail and page_head was
+				 * the THP head before the split.
+				 */
 				if (PageHead(page_head))
 					__put_compound_page(page_head);
 				else
 					__put_single_page(page_head);
 			}
-		} else {
-			/* page_head is a dangling pointer */
-			VM_BUG_ON(PageTail(page));
-			goto out_put_single;
+out_put_single:
+			if (put_page_testzero(page))
+				__put_single_page(page);
+			return;
 		}
-	} else if (put_page_testzero(page)) {
-		if (PageHead(page))
-			__put_compound_page(page);
-		else
-			__put_single_page(page);
+		VM_BUG_ON(page_head != page->first_page);
+		/*
+		 * We can release the refcount taken by
+		 * get_page_unless_zero() now that
+		 * __split_huge_page_refcount() is blocked on the
+		 * compound_lock.
+		 */
+		if (put_page_testzero(page_head))
+			VM_BUG_ON(1);
+		/* __split_huge_page_refcount will wait now */
+		VM_BUG_ON(page_mapcount(page) <= 0);
+		atomic_dec(&page->_mapcount);
+		VM_BUG_ON(atomic_read(&page_head->_count) <= 0);
+		VM_BUG_ON(atomic_read(&page->_count) != 0);
+		compound_unlock_irqrestore(page_head, flags);
+
+		if (put_page_testzero(page_head)) {
+			if (PageHead(page_head))
+				__put_compound_page(page_head);
+			else
+				__put_single_page(page_head);
+		}
+	} else {
+		/* page_head is a dangling pointer */
+		VM_BUG_ON(PageTail(page));
+		goto out_put_single;
 	}
 }
...	...	@@ -81,154 +81,150 @@
81	81
82	82	static void put_compound_page(struct page *page)
83	83	{
84		- if (unlikely(PageTail(page))) {
85		- /* __split_huge_page_refcount can run under us */
86		- struct page *page_head = compound_trans_head(page);
	84	+ struct page *page_head;
87	85
88		- /*
89		- * THP can not break up slab pages so avoid taking
90		- * compound_lock() and skip the tail page refcounting
91		- * (in _mapcount) too. Slab performs non-atomic bit
92		- * ops on page->flags for better performance. In
93		- * particular slab_unlock() in slub used to be a hot
94		- * path. It is still hot on arches that do not support
95		- * this_cpu_cmpxchg_double().
96		- *
97		- * If "page" is part of a slab or hugetlbfs page it
98		- * cannot be splitted and the head page cannot change
99		- * from under us. And if "page" is part of a THP page
100		- * under splitting, if the head page pointed by the
101		- * THP tail isn't a THP head anymore, we'll find
102		- * PageTail clear after smp_rmb() and we'll treat it
103		- * as a single page.
104		- */
105		- if (!__compound_tail_refcounted(page_head)) {
	86	+ if (likely(!PageTail(page))) {
	87	+ if (put_page_testzero(page)) {
106	88	/*
107		- * If "page" is a THP tail, we must read the tail page
108		- * flags after the head page flags. The
109		- * split_huge_page side enforces write memory
110		- * barriers between clearing PageTail and before the
111		- * head page can be freed and reallocated.
	89	+ * By the time all refcounts have been released
	90	+ * split_huge_page cannot run anymore from under us.
112	91	*/
113		- smp_rmb();
114		- if (likely(PageTail(page))) {
115		- /*
116		- * __split_huge_page_refcount
117		- * cannot race here.
118		- */
119		- VM_BUG_ON(!PageHead(page_head));
120		- VM_BUG_ON(page_mapcount(page) != 0);
121		- if (put_page_testzero(page_head)) {
122		- /*
123		- * If this is the tail of a
124		- * slab compound page, the
125		- * tail pin must not be the
126		- * last reference held on the
127		- * page, because the PG_slab
128		- * cannot be cleared before
129		- * all tail pins (which skips
130		- * the _mapcount tail
131		- * refcounting) have been
132		- * released. For hugetlbfs the
133		- * tail pin may be the last
134		- * reference on the page
135		- * instead, because
136		- * PageHeadHuge will not go
137		- * away until the compound
138		- * page enters the buddy
139		- * allocator.
140		- */
141		- VM_BUG_ON(PageSlab(page_head));
142		- __put_compound_page(page_head);
143		- }
144		- return;
145		- } else
146		- /*
147		- * __split_huge_page_refcount
148		- * run before us, "page" was a
149		- * THP tail. The split
150		- * page_head has been freed
151		- * and reallocated as slab or
152		- * hugetlbfs page of smaller
153		- * order (only possible if
154		- * reallocated as slab on
155		- * x86).
156		- */
157		- goto out_put_single;
	92	+ if (PageHead(page))
	93	+ __put_compound_page(page);
	94	+ else
	95	+ __put_single_page(page);
158	96	}
	97	+ return;
	98	+ }
159	99
160		- if (likely(page != page_head &&
161		- get_page_unless_zero(page_head))) {
162		- unsigned long flags;
	100	+ /* __split_huge_page_refcount can run under us */
	101	+ page_head = compound_trans_head(page);
163	102
	103	+ /*
	104	+ * THP can not break up slab pages so avoid taking
	105	+ * compound_lock() and skip the tail page refcounting (in
	106	+ * _mapcount) too. Slab performs non-atomic bit ops on
	107	+ * page->flags for better performance. In particular
	108	+ * slab_unlock() in slub used to be a hot path. It is still
	109	+ * hot on arches that do not support
	110	+ * this_cpu_cmpxchg_double().
	111	+ *
	112	+ * If "page" is part of a slab or hugetlbfs page it cannot be
	113	+ * splitted and the head page cannot change from under us. And
	114	+ * if "page" is part of a THP page under splitting, if the
	115	+ * head page pointed by the THP tail isn't a THP head anymore,
	116	+ * we'll find PageTail clear after smp_rmb() and we'll treat
	117	+ * it as a single page.
	118	+ */
	119	+ if (!__compound_tail_refcounted(page_head)) {
	120	+ /*
	121	+ * If "page" is a THP tail, we must read the tail page
	122	+ * flags after the head page flags. The
	123	+ * split_huge_page side enforces write memory barriers
	124	+ * between clearing PageTail and before the head page
	125	+ * can be freed and reallocated.
	126	+ */
	127	+ smp_rmb();
	128	+ if (likely(PageTail(page))) {
164	129	/*
165		- * page_head wasn't a dangling pointer but it
166		- * may not be a head page anymore by the time
167		- * we obtain the lock. That is ok as long as it
168		- * can't be freed from under us.
	130	+ * __split_huge_page_refcount cannot race
	131	+ * here.
169	132	*/
170		- flags = compound_lock_irqsave(page_head);
171		- if (unlikely(!PageTail(page))) {
172		- /* __split_huge_page_refcount run before us */
173		- compound_unlock_irqrestore(page_head, flags);
174		- if (put_page_testzero(page_head)) {
175		- /*
176		- * The head page may have been
177		- * freed and reallocated as a
178		- * compound page of smaller
179		- * order and then freed again.
180		- * All we know is that it
181		- * cannot have become: a THP
182		- * page, a compound page of
183		- * higher order, a tail page.
184		- * That is because we still
185		- * hold the refcount of the
186		- * split THP tail and
187		- * page_head was the THP head
188		- * before the split.
189		- */
190		- if (PageHead(page_head))
191		- __put_compound_page(page_head);
192		- else
193		- __put_single_page(page_head);
194		- }
195		-out_put_single:
196		- if (put_page_testzero(page))
197		- __put_single_page(page);
198		- return;
	133	+ VM_BUG_ON(!PageHead(page_head));
	134	+ VM_BUG_ON(page_mapcount(page) != 0);
	135	+ if (put_page_testzero(page_head)) {
	136	+ /*
	137	+ * If this is the tail of a slab
	138	+ * compound page, the tail pin must
	139	+ * not be the last reference held on
	140	+ * the page, because the PG_slab
	141	+ * cannot be cleared before all tail
	142	+ * pins (which skips the _mapcount
	143	+ * tail refcounting) have been
	144	+ * released. For hugetlbfs the tail
	145	+ * pin may be the last reference on
	146	+ * the page instead, because
	147	+ * PageHeadHuge will not go away until
	148	+ * the compound page enters the buddy
	149	+ * allocator.
	150	+ */
	151	+ VM_BUG_ON(PageSlab(page_head));
	152	+ __put_compound_page(page_head);
199	153	}
200		- VM_BUG_ON(page_head != page->first_page);
	154	+ return;
	155	+ } else
201	156	/*
202		- * We can release the refcount taken by
203		- * get_page_unless_zero() now that
204		- * __split_huge_page_refcount() is blocked on
205		- * the compound_lock.
	157	+ * __split_huge_page_refcount run before us,
	158	+ * "page" was a THP tail. The split page_head
	159	+ * has been freed and reallocated as slab or
	160	+ * hugetlbfs page of smaller order (only
	161	+ * possible if reallocated as slab on x86).
206	162	*/
207		- if (put_page_testzero(page_head))
208		- VM_BUG_ON(1);
209		- /* __split_huge_page_refcount will wait now */
210		- VM_BUG_ON(page_mapcount(page) <= 0);
211		- atomic_dec(&page->_mapcount);
212		- VM_BUG_ON(atomic_read(&page_head->_count) <= 0);
213		- VM_BUG_ON(atomic_read(&page->_count) != 0);
214		- compound_unlock_irqrestore(page_head, flags);
	163	+ goto out_put_single;
	164	+ }
215	165
	166	+ if (likely(page != page_head && get_page_unless_zero(page_head))) {
	167	+ unsigned long flags;
	168	+
	169	+ /*
	170	+ * page_head wasn't a dangling pointer but it may not
	171	+ * be a head page anymore by the time we obtain the
	172	+ * lock. That is ok as long as it can't be freed from
	173	+ * under us.
	174	+ */
	175	+ flags = compound_lock_irqsave(page_head);
	176	+ if (unlikely(!PageTail(page))) {
	177	+ /* __split_huge_page_refcount run before us */
	178	+ compound_unlock_irqrestore(page_head, flags);
216	179	if (put_page_testzero(page_head)) {
	180	+ /*
	181	+ * The head page may have been freed
	182	+ * and reallocated as a compound page
	183	+ * of smaller order and then freed
	184	+ * again. All we know is that it
	185	+ * cannot have become: a THP page, a
	186	+ * compound page of higher order, a
	187	+ * tail page. That is because we
	188	+ * still hold the refcount of the
	189	+ * split THP tail and page_head was
	190	+ * the THP head before the split.
	191	+ */
217	192	if (PageHead(page_head))
218	193	__put_compound_page(page_head);
219	194	else
220	195	__put_single_page(page_head);
221	196	}
222		- } else {
223		- /* page_head is a dangling pointer */
224		- VM_BUG_ON(PageTail(page));
225		- goto out_put_single;
	197	+out_put_single:
	198	+ if (put_page_testzero(page))
	199	+ __put_single_page(page);
	200	+ return;
226	201	}
227		- } else if (put_page_testzero(page)) {
228		- if (PageHead(page))
229		- __put_compound_page(page);
230		- else
231		- __put_single_page(page);
	202	+ VM_BUG_ON(page_head != page->first_page);
	203	+ /*
	204	+ * We can release the refcount taken by
	205	+ * get_page_unless_zero() now that
	206	+ * __split_huge_page_refcount() is blocked on the
	207	+ * compound_lock.
	208	+ */
	209	+ if (put_page_testzero(page_head))
	210	+ VM_BUG_ON(1);
	211	+ /* __split_huge_page_refcount will wait now */
	212	+ VM_BUG_ON(page_mapcount(page) <= 0);
	213	+ atomic_dec(&page->_mapcount);
	214	+ VM_BUG_ON(atomic_read(&page_head->_count) <= 0);
	215	+ VM_BUG_ON(atomic_read(&page->_count) != 0);
	216	+ compound_unlock_irqrestore(page_head, flags);
	217	+
	218	+ if (put_page_testzero(page_head)) {
	219	+ if (PageHead(page_head))
	220	+ __put_compound_page(page_head);
	221	+ else
	222	+ __put_single_page(page_head);
	223	+ }
	224	+ } else {
	225	+ /* page_head is a dangling pointer */
	226	+ VM_BUG_ON(PageTail(page));
	227	+ goto out_put_single;
232	228	}
233	229	}
234	230