mm: filemap: move radix tree hole searching here

The radix tree hole searching code is only used for page cache, for example the readahead code trying to get a a picture of the area surrounding a fault. It sufficed to rely on the radix tree definition of holes, which is "empty tree slot". But this is about to change, though, as shadow page descriptors will be stored in the page cache after the actual pages get evicted from memory. Move the functions over to mm/filemap.c and make them native page cache operations, where they can later be adapted to handle the new definition of "page cache hole". Signed-off-by: Johannes Weiner <hannes@cmpxchg.org> Reviewed-by: Rik van Riel <riel@redhat.com> Reviewed-by: Minchan Kim <minchan@kernel.org> Acked-by: Mel Gorman <mgorman@suse.de> Cc: Andrea Arcangeli <aarcange@redhat.com> Cc: Bob Liu <bob.liu@oracle.com> Cc: Christoph Hellwig <hch@infradead.org> Cc: Dave Chinner <david@fromorbit.com> Cc: Greg Thelen <gthelen@google.com> Cc: Hugh Dickins <hughd@google.com> Cc: Jan Kara <jack@suse.cz> Cc: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com> Cc: Luigi Semenzato <semenzato@google.com> Cc: Metin Doslu <metin@citusdata.com> Cc: Michel Lespinasse <walken@google.com> Cc: Ozgun Erdogan <ozgun@citusdata.com> Cc: Peter Zijlstra <peterz@infradead.org> Cc: Roman Gushchin <klamm@yandex-team.ru> Cc: Ryan Mallon <rmallon@gmail.com> Cc: Tejun Heo <tj@kernel.org> Cc: Vlastimil Babka <vbabka@suse.cz> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

mm: filemap: move radix tree hole searching here
The radix tree hole searching code is only used for page cache, for example the readahead code trying to get a a picture of the area surrounding a fault. It sufficed to rely on the radix tree definition of holes, which is "empty tree slot". But this is about to change, though, as shadow page descriptors will be stored in the page cache after the actual pages get evicted from memory. Move the functions over to mm/filemap.c and make them native page cache operations, where they can later be adapted to handle the new definition of "page cache hole". Signed-off-by: Johannes Weiner <hannes@cmpxchg.org> Reviewed-by: Rik van Riel <riel@redhat.com> Reviewed-by: Minchan Kim <minchan@kernel.org> Acked-by: Mel Gorman <mgorman@suse.de> Cc: Andrea Arcangeli <aarcange@redhat.com> Cc: Bob Liu <bob.liu@oracle.com> Cc: Christoph Hellwig <hch@infradead.org> Cc: Dave Chinner <david@fromorbit.com> Cc: Greg Thelen <gthelen@google.com> Cc: Hugh Dickins <hughd@google.com> Cc: Jan Kara <jack@suse.cz> Cc: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com> Cc: Luigi Semenzato <semenzato@google.com> Cc: Metin Doslu <metin@citusdata.com> Cc: Michel Lespinasse <walken@google.com> Cc: Ozgun Erdogan <ozgun@citusdata.com> Cc: Peter Zijlstra <peterz@infradead.org> Cc: Roman Gushchin <klamm@yandex-team.ru> Cc: Ryan Mallon <rmallon@gmail.com> Cc: Tejun Heo <tj@kernel.org> Cc: Vlastimil Babka <vbabka@suse.cz> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
Johannes Weiner · Linus Torvalds · Eric Lee · Eric Lee · Eric Lee · Eric Lee
1 parent 6dbaf22ce1
Showing 6 changed files with 84 additions and 82 deletions Side-by-side Diff
fs/nfs/blocklayout/blocklayout.c
include/linux/pagemap.h
include/linux/radix-tree.h
lib/radix-tree.c
mm/filemap.c
mm/readahead.c
@@ -1213,7 +1213,7 @@
 	end = DIV_ROUND_UP(i_size_read(inode), PAGE_CACHE_SIZE);
 	if (end != NFS_I(inode)->npages) {
 		rcu_read_lock();
-		end = radix_tree_next_hole(&mapping->page_tree, idx + 1, ULONG_MAX);
+		end = page_cache_next_hole(mapping, idx + 1, ULONG_MAX);
 		rcu_read_unlock();
 	}
  
@@ -243,6 +243,11 @@
  
 typedef int filler_t(void *, struct page *);
  
+pgoff_t page_cache_next_hole(struct address_space *mapping,
+			     pgoff_t index, unsigned long max_scan);
+pgoff_t page_cache_prev_hole(struct address_space *mapping,
+			     pgoff_t index, unsigned long max_scan);
+
 extern struct page * find_get_page(struct address_space *mapping,
 				pgoff_t index);
 extern struct page * find_lock_page(struct address_space *mapping,
@@ -227,10 +227,6 @@
 unsigned int radix_tree_gang_lookup_slot(struct radix_tree_root *root,
 			void ***results, unsigned long *indices,
 			unsigned long first_index, unsigned int max_items);
-unsigned long radix_tree_next_hole(struct radix_tree_root *root,
-				unsigned long index, unsigned long max_scan);
-unsigned long radix_tree_prev_hole(struct radix_tree_root *root,
-				unsigned long index, unsigned long max_scan);
 int radix_tree_preload(gfp_t gfp_mask);
 int radix_tree_maybe_preload(gfp_t gfp_mask);
 void radix_tree_init(void);
@@ -946,81 +946,6 @@
 }
 EXPORT_SYMBOL(radix_tree_range_tag_if_tagged);
  
-
-/**
- *	radix_tree_next_hole    -    find the next hole (not-present entry)
- *	@root:		tree root
- *	@index:		index key
- *	@max_scan:	maximum range to search
- *
- *	Search the set [index, min(index+max_scan-1, MAX_INDEX)] for the lowest
- *	indexed hole.
- *
- *	Returns: the index of the hole if found, otherwise returns an index
- *	outside of the set specified (in which case 'return - index >= max_scan'
- *	will be true). In rare cases of index wrap-around, 0 will be returned.
- *
- *	radix_tree_next_hole may be called under rcu_read_lock. However, like
- *	radix_tree_gang_lookup, this will not atomically search a snapshot of
- *	the tree at a single point in time. For example, if a hole is created
- *	at index 5, then subsequently a hole is created at index 10,
- *	radix_tree_next_hole covering both indexes may return 10 if called
- *	under rcu_read_lock.
- */
-unsigned long radix_tree_next_hole(struct radix_tree_root *root,
-				unsigned long index, unsigned long max_scan)
-{
-	unsigned long i;
-
-	for (i = 0; i < max_scan; i++) {
-		if (!radix_tree_lookup(root, index))
-			break;
-		index++;
-		if (index == 0)
-			break;
-	}
-
-	return index;
-}
-EXPORT_SYMBOL(radix_tree_next_hole);
-
-/**
- *	radix_tree_prev_hole    -    find the prev hole (not-present entry)
- *	@root:		tree root
- *	@index:		index key
- *	@max_scan:	maximum range to search
- *
- *	Search backwards in the range [max(index-max_scan+1, 0), index]
- *	for the first hole.
- *
- *	Returns: the index of the hole if found, otherwise returns an index
- *	outside of the set specified (in which case 'index - return >= max_scan'
- *	will be true). In rare cases of wrap-around, ULONG_MAX will be returned.
- *
- *	radix_tree_next_hole may be called under rcu_read_lock. However, like
- *	radix_tree_gang_lookup, this will not atomically search a snapshot of
- *	the tree at a single point in time. For example, if a hole is created
- *	at index 10, then subsequently a hole is created at index 5,
- *	radix_tree_prev_hole covering both indexes may return 5 if called under
- *	rcu_read_lock.
- */
-unsigned long radix_tree_prev_hole(struct radix_tree_root *root,
-				   unsigned long index, unsigned long max_scan)
-{
-	unsigned long i;
-
-	for (i = 0; i < max_scan; i++) {
-		if (!radix_tree_lookup(root, index))
-			break;
-		index--;
-		if (index == ULONG_MAX)
-			break;
-	}
-
-	return index;
-}
-EXPORT_SYMBOL(radix_tree_prev_hole);
-
 /**
  *	radix_tree_gang_lookup - perform multiple lookup on a radix tree
  *	@root:		radix tree root
@@ -686,6 +686,82 @@
 }
  
 /**
+ * page_cache_next_hole - find the next hole (not-present entry)
+ * @mapping: mapping
+ * @index: index
+ * @max_scan: maximum range to search
+ *
+ * Search the set [index, min(index+max_scan-1, MAX_INDEX)] for the
+ * lowest indexed hole.
+ *
+ * Returns: the index of the hole if found, otherwise returns an index
+ * outside of the set specified (in which case 'return - index >=
+ * max_scan' will be true). In rare cases of index wrap-around, 0 will
+ * be returned.
+ *
+ * page_cache_next_hole may be called under rcu_read_lock. However,
+ * like radix_tree_gang_lookup, this will not atomically search a
+ * snapshot of the tree at a single point in time. For example, if a
+ * hole is created at index 5, then subsequently a hole is created at
+ * index 10, page_cache_next_hole covering both indexes may return 10
+ * if called under rcu_read_lock.
+ */
+pgoff_t page_cache_next_hole(struct address_space *mapping,
+			     pgoff_t index, unsigned long max_scan)
+{
+	unsigned long i;
+
+	for (i = 0; i < max_scan; i++) {
+		if (!radix_tree_lookup(&mapping->page_tree, index))
+			break;
+		index++;
+		if (index == 0)
+			break;
+	}
+
+	return index;
+}
+EXPORT_SYMBOL(page_cache_next_hole);
+
+/**
+ * page_cache_prev_hole - find the prev hole (not-present entry)
+ * @mapping: mapping
+ * @index: index
+ * @max_scan: maximum range to search
+ *
+ * Search backwards in the range [max(index-max_scan+1, 0), index] for
+ * the first hole.
+ *
+ * Returns: the index of the hole if found, otherwise returns an index
+ * outside of the set specified (in which case 'index - return >=
+ * max_scan' will be true). In rare cases of wrap-around, ULONG_MAX
+ * will be returned.
+ *
+ * page_cache_prev_hole may be called under rcu_read_lock. However,
+ * like radix_tree_gang_lookup, this will not atomically search a
+ * snapshot of the tree at a single point in time. For example, if a
+ * hole is created at index 10, then subsequently a hole is created at
+ * index 5, page_cache_prev_hole covering both indexes may return 5 if
+ * called under rcu_read_lock.
+ */
+pgoff_t page_cache_prev_hole(struct address_space *mapping,
+			     pgoff_t index, unsigned long max_scan)
+{
+	unsigned long i;
+
+	for (i = 0; i < max_scan; i++) {
+		if (!radix_tree_lookup(&mapping->page_tree, index))
+			break;
+		index--;
+		if (index == ULONG_MAX)
+			break;
+	}
+
+	return index;
+}
+EXPORT_SYMBOL(page_cache_prev_hole);
+
+/**
  * find_get_page - find and get a page reference
  * @mapping: the address_space to search
  * @offset: the page index
@@ -347,7 +347,7 @@
 	pgoff_t head;
  
 	rcu_read_lock();
-	head = radix_tree_prev_hole(&mapping->page_tree, offset - 1, max);
+	head = page_cache_prev_hole(mapping, offset - 1, max);
 	rcu_read_unlock();
  
 	return offset - 1 - head;
@@ -427,7 +427,7 @@
 		pgoff_t start;
  
 		rcu_read_lock();
-		start = radix_tree_next_hole(&mapping->page_tree, offset+1,max);
+		start = page_cache_next_hole(mapping, offset + 1, max);
 		rcu_read_unlock();
  
 		if (!start || start - offset > max)
...	...	@@ -1213,7 +1213,7 @@
1213	1213	end = DIV_ROUND_UP(i_size_read(inode), PAGE_CACHE_SIZE);
1214	1214	if (end != NFS_I(inode)->npages) {
1215	1215	rcu_read_lock();
1216		- end = radix_tree_next_hole(&mapping->page_tree, idx + 1, ULONG_MAX);
	1216	+ end = page_cache_next_hole(mapping, idx + 1, ULONG_MAX);
1217	1217	rcu_read_unlock();
1218	1218	}
1219	1219
...	...	@@ -243,6 +243,11 @@
243	243
244	244	typedef int filler_t(void , struct page );
245	245
	246	+pgoff_t page_cache_next_hole(struct address_space *mapping,
	247	+ pgoff_t index, unsigned long max_scan);
	248	+pgoff_t page_cache_prev_hole(struct address_space *mapping,
	249	+ pgoff_t index, unsigned long max_scan);
	250	+
246	251	extern struct page * find_get_page(struct address_space *mapping,
247	252	pgoff_t index);
248	253	extern struct page * find_lock_page(struct address_space *mapping,
...	...	@@ -227,10 +227,6 @@
227	227	unsigned int radix_tree_gang_lookup_slot(struct radix_tree_root *root,
228	228	void **results, unsigned long indices,
229	229	unsigned long first_index, unsigned int max_items);
230		-unsigned long radix_tree_next_hole(struct radix_tree_root *root,
231		- unsigned long index, unsigned long max_scan);
232		-unsigned long radix_tree_prev_hole(struct radix_tree_root *root,
233		- unsigned long index, unsigned long max_scan);
234	230	int radix_tree_preload(gfp_t gfp_mask);
235	231	int radix_tree_maybe_preload(gfp_t gfp_mask);
236	232	void radix_tree_init(void);
...	...	@@ -946,81 +946,6 @@
946	946	}
947	947	EXPORT_SYMBOL(radix_tree_range_tag_if_tagged);
948	948
949		-
950		-/**
951		- * radix_tree_next_hole - find the next hole (not-present entry)
952		- * @root: tree root
953		- * @index: index key
954		- * @max_scan: maximum range to search
955		- *
956		- * Search the set [index, min(index+max_scan-1, MAX_INDEX)] for the lowest
957		- * indexed hole.
958		- *
959		- * Returns: the index of the hole if found, otherwise returns an index
960		- * outside of the set specified (in which case 'return - index >= max_scan'
961		- * will be true). In rare cases of index wrap-around, 0 will be returned.
962		- *
963		- * radix_tree_next_hole may be called under rcu_read_lock. However, like
964		- * radix_tree_gang_lookup, this will not atomically search a snapshot of
965		- * the tree at a single point in time. For example, if a hole is created
966		- * at index 5, then subsequently a hole is created at index 10,
967		- * radix_tree_next_hole covering both indexes may return 10 if called
968		- * under rcu_read_lock.
969		- */
970		-unsigned long radix_tree_next_hole(struct radix_tree_root *root,
971		- unsigned long index, unsigned long max_scan)
972		-{
973		- unsigned long i;
974		-
975		- for (i = 0; i < max_scan; i++) {
976		- if (!radix_tree_lookup(root, index))
977		- break;
978		- index++;
979		- if (index == 0)
980		- break;
981		- }
982		-
983		- return index;
984		-}
985		-EXPORT_SYMBOL(radix_tree_next_hole);
986		-
987		-/**
988		- * radix_tree_prev_hole - find the prev hole (not-present entry)
989		- * @root: tree root
990		- * @index: index key
991		- * @max_scan: maximum range to search
992		- *
993		- * Search backwards in the range [max(index-max_scan+1, 0), index]
994		- * for the first hole.
995		- *
996		- * Returns: the index of the hole if found, otherwise returns an index
997		- * outside of the set specified (in which case 'index - return >= max_scan'
998		- * will be true). In rare cases of wrap-around, ULONG_MAX will be returned.
999		- *
1000		- * radix_tree_next_hole may be called under rcu_read_lock. However, like
1001		- * radix_tree_gang_lookup, this will not atomically search a snapshot of
1002		- * the tree at a single point in time. For example, if a hole is created
1003		- * at index 10, then subsequently a hole is created at index 5,
1004		- * radix_tree_prev_hole covering both indexes may return 5 if called under
1005		- * rcu_read_lock.
1006		- */
1007		-unsigned long radix_tree_prev_hole(struct radix_tree_root *root,
1008		- unsigned long index, unsigned long max_scan)
1009		-{
1010		- unsigned long i;
1011		-
1012		- for (i = 0; i < max_scan; i++) {
1013		- if (!radix_tree_lookup(root, index))
1014		- break;
1015		- index--;
1016		- if (index == ULONG_MAX)
1017		- break;
1018		- }
1019		-
1020		- return index;
1021		-}
1022		-EXPORT_SYMBOL(radix_tree_prev_hole);
1023		-
1024	949	/**
1025	950	* radix_tree_gang_lookup - perform multiple lookup on a radix tree
1026	951	* @root: radix tree root
...	...	@@ -686,6 +686,82 @@
686	686	}
687	687
688	688	/**
	689	+ * page_cache_next_hole - find the next hole (not-present entry)
	690	+ * @mapping: mapping
	691	+ * @index: index
	692	+ * @max_scan: maximum range to search
	693	+ *
	694	+ * Search the set [index, min(index+max_scan-1, MAX_INDEX)] for the
	695	+ * lowest indexed hole.
	696	+ *
	697	+ * Returns: the index of the hole if found, otherwise returns an index
	698	+ * outside of the set specified (in which case 'return - index >=
	699	+ * max_scan' will be true). In rare cases of index wrap-around, 0 will
	700	+ * be returned.
	701	+ *
	702	+ * page_cache_next_hole may be called under rcu_read_lock. However,
	703	+ * like radix_tree_gang_lookup, this will not atomically search a
	704	+ * snapshot of the tree at a single point in time. For example, if a
	705	+ * hole is created at index 5, then subsequently a hole is created at
	706	+ * index 10, page_cache_next_hole covering both indexes may return 10
	707	+ * if called under rcu_read_lock.
	708	+ */
	709	+pgoff_t page_cache_next_hole(struct address_space *mapping,
	710	+ pgoff_t index, unsigned long max_scan)
	711	+{
	712	+ unsigned long i;
	713	+
	714	+ for (i = 0; i < max_scan; i++) {
	715	+ if (!radix_tree_lookup(&mapping->page_tree, index))
	716	+ break;
	717	+ index++;
	718	+ if (index == 0)
	719	+ break;
	720	+ }
	721	+
	722	+ return index;
	723	+}
	724	+EXPORT_SYMBOL(page_cache_next_hole);
	725	+
	726	+/**
	727	+ * page_cache_prev_hole - find the prev hole (not-present entry)
	728	+ * @mapping: mapping
	729	+ * @index: index
	730	+ * @max_scan: maximum range to search
	731	+ *
	732	+ * Search backwards in the range [max(index-max_scan+1, 0), index] for
	733	+ * the first hole.
	734	+ *
	735	+ * Returns: the index of the hole if found, otherwise returns an index
	736	+ * outside of the set specified (in which case 'index - return >=
	737	+ * max_scan' will be true). In rare cases of wrap-around, ULONG_MAX
	738	+ * will be returned.
	739	+ *
	740	+ * page_cache_prev_hole may be called under rcu_read_lock. However,
	741	+ * like radix_tree_gang_lookup, this will not atomically search a
	742	+ * snapshot of the tree at a single point in time. For example, if a
	743	+ * hole is created at index 10, then subsequently a hole is created at
	744	+ * index 5, page_cache_prev_hole covering both indexes may return 5 if
	745	+ * called under rcu_read_lock.
	746	+ */
	747	+pgoff_t page_cache_prev_hole(struct address_space *mapping,
	748	+ pgoff_t index, unsigned long max_scan)
	749	+{
	750	+ unsigned long i;
	751	+
	752	+ for (i = 0; i < max_scan; i++) {
	753	+ if (!radix_tree_lookup(&mapping->page_tree, index))
	754	+ break;
	755	+ index--;
	756	+ if (index == ULONG_MAX)
	757	+ break;
	758	+ }
	759	+
	760	+ return index;
	761	+}
	762	+EXPORT_SYMBOL(page_cache_prev_hole);
	763	+
	764	+/**
689	765	* find_get_page - find and get a page reference
690	766	* @mapping: the address_space to search
691	767	* @offset: the page index
...	...	@@ -347,7 +347,7 @@
347	347	pgoff_t head;
348	348
349	349	rcu_read_lock();
350		- head = radix_tree_prev_hole(&mapping->page_tree, offset - 1, max);
	350	+ head = page_cache_prev_hole(mapping, offset - 1, max);
351	351	rcu_read_unlock();
352	352
353	353	return offset - 1 - head;
...	...	@@ -427,7 +427,7 @@
427	427	pgoff_t start;
428	428
429	429	rcu_read_lock();
430		- start = radix_tree_next_hole(&mapping->page_tree, offset+1,max);
	430	+ start = page_cache_next_hole(mapping, offset + 1, max);
431	431	rcu_read_unlock();
432	432
433	433	if (!start \|\| start - offset > max)