vfs: make the dentry cache use the lockref infrastructure

This just replaces the dentry count/lock combination with the lockref structure that contains both a count and a spinlock, and does the mechanical conversion to use the lockref infrastructure. There are no semantic changes here, it's purely syntactic. The reference lockref implementation uses the spinlock exactly the same way that the old dcache code did, and the bulk of this patch is just expanding the internal "d_count" use in the dcache code to use "d_lockref.count" instead. This is purely preparation for the real change to make the reference count updates be lockless during the 3.12 merge window. [ As with the previous commit, this is a rewritten version of a concept originally from Waiman, so credit goes to him, blame for any errors goes to me. Waiman's patch had some semantic differences for taking advantage of the lockless update in dget_parent(), while this patch is intentionally a pure search-and-replace change with no semantic changes. - Linus ] Signed-off-by: Waiman Long <Waiman.Long@hp.com> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

vfs: make the dentry cache use the lockref infrastructure
This just replaces the dentry count/lock combination with the lockref structure that contains both a count and a spinlock, and does the mechanical conversion to use the lockref infrastructure. There are no semantic changes here, it's purely syntactic. The reference lockref implementation uses the spinlock exactly the same way that the old dcache code did, and the bulk of this patch is just expanding the internal "d_count" use in the dcache code to use "d_lockref.count" instead. This is purely preparation for the real change to make the reference count updates be lockless during the 3.12 merge window. [ As with the previous commit, this is a rewritten version of a concept originally from Waiman, so credit goes to him, blame for any errors goes to me. Waiman's patch had some semantic differences for taking advantage of the lockless update in dget_parent(), while this patch is intentionally a pure search-and-replace change with no semantic changes. - Linus ] Signed-off-by: Waiman Long <Waiman.Long@hp.com> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
Waiman Long · Linus Torvalds
1 parent 0f8f2aaaab
Showing 3 changed files with 35 additions and 47 deletions Side-by-side Diff
fs/dcache.c
fs/namei.c
include/linux/dcache.h
@@ -229,7 +229,7 @@
  */
 static void d_free(struct dentry *dentry)
 {
-	BUG_ON(dentry->d_count);
+	BUG_ON(dentry->d_lockref.count);
 	this_cpu_dec(nr_dentry);
 	if (dentry->d_op && dentry->d_op->d_release)
 		dentry->d_op->d_release(dentry);
@@ -467,7 +467,7 @@
 	}
  
 	if (ref)
-		dentry->d_count--;
+		dentry->d_lockref.count--;
 	/*
 	 * inform the fs via d_prune that this dentry is about to be
 	 * unhashed and destroyed.
  
  
@@ -513,15 +513,10 @@
 		return;
  
 repeat:
-	if (dentry->d_count == 1)
+	if (dentry->d_lockref.count == 1)
 		might_sleep();
-	spin_lock(&dentry->d_lock);
-	BUG_ON(!dentry->d_count);
-	if (dentry->d_count > 1) {
-		dentry->d_count--;
-		spin_unlock(&dentry->d_lock);
+	if (lockref_put_or_lock(&dentry->d_lockref))
 		return;
-	}
  
 	if (dentry->d_flags & DCACHE_OP_DELETE) {
 		if (dentry->d_op->d_delete(dentry))
@@ -535,7 +530,7 @@
 	dentry->d_flags |= DCACHE_REFERENCED;
 	dentry_lru_add(dentry);
  
-	dentry->d_count--;
+	dentry->d_lockref.count--;
 	spin_unlock(&dentry->d_lock);
 	return;
  
@@ -590,7 +585,7 @@
 	 * We also need to leave mountpoints alone,
 	 * directory or not.
 	 */
-	if (dentry->d_count > 1 && dentry->d_inode) {
+	if (dentry->d_lockref.count > 1 && dentry->d_inode) {
 		if (S_ISDIR(dentry->d_inode->i_mode) || d_mountpoint(dentry)) {
 			spin_unlock(&dentry->d_lock);
 			return -EBUSY;
  
@@ -606,14 +601,12 @@
 /* This must be called with d_lock held */
 static inline void __dget_dlock(struct dentry *dentry)
 {
-	dentry->d_count++;
+	dentry->d_lockref.count++;
 }
  
 static inline void __dget(struct dentry *dentry)
 {
-	spin_lock(&dentry->d_lock);
-	__dget_dlock(dentry);
-	spin_unlock(&dentry->d_lock);
+	lockref_get(&dentry->d_lockref);
 }
  
 struct dentry *dget_parent(struct dentry *dentry)
@@ -634,8 +627,8 @@
 		goto repeat;
 	}
 	rcu_read_unlock();
-	BUG_ON(!ret->d_count);
-	ret->d_count++;
+	BUG_ON(!ret->d_lockref.count);
+	ret->d_lockref.count++;
 	spin_unlock(&ret->d_lock);
 	return ret;
 }
@@ -718,7 +711,7 @@
 	spin_lock(&inode->i_lock);
 	hlist_for_each_entry(dentry, &inode->i_dentry, d_alias) {
 		spin_lock(&dentry->d_lock);
-		if (!dentry->d_count) {
+		if (!dentry->d_lockref.count) {
 			__dget_dlock(dentry);
 			__d_drop(dentry);
 			spin_unlock(&dentry->d_lock);
  
@@ -763,12 +756,8 @@
 	/* Prune ancestors. */
 	dentry = parent;
 	while (dentry) {
-		spin_lock(&dentry->d_lock);
-		if (dentry->d_count > 1) {
-			dentry->d_count--;
-			spin_unlock(&dentry->d_lock);
+		if (lockref_put_or_lock(&dentry->d_lockref))
 			return;
-		}
 		dentry = dentry_kill(dentry, 1);
 	}
 }
@@ -793,7 +782,7 @@
 		 * the LRU because of laziness during lookup.  Do not free
 		 * it - just keep it off the LRU list.
 		 */
-		if (dentry->d_count) {
+		if (dentry->d_lockref.count) {
 			dentry_lru_del(dentry);
 			spin_unlock(&dentry->d_lock);
 			continue;
@@ -913,7 +902,7 @@
 			dentry_lru_del(dentry);
 			__d_shrink(dentry);
  
-			if (dentry->d_count != 0) {
+			if (dentry->d_lockref.count != 0) {
 				printk(KERN_ERR
 				       "BUG: Dentry %p{i=%lx,n=%s}"
 				       " still in use (%d)"
@@ -922,7 +911,7 @@
 				       dentry->d_inode ?
 				       dentry->d_inode->i_ino : 0UL,
 				       dentry->d_name.name,
-				       dentry->d_count,
+				       dentry->d_lockref.count,
 				       dentry->d_sb->s_type->name,
 				       dentry->d_sb->s_id);
 				BUG();
@@ -933,7 +922,7 @@
 				list_del(&dentry->d_u.d_child);
 			} else {
 				parent = dentry->d_parent;
-				parent->d_count--;
+				parent->d_lockref.count--;
 				list_del(&dentry->d_u.d_child);
 			}
  
@@ -981,7 +970,7 @@
  
 	dentry = sb->s_root;
 	sb->s_root = NULL;
-	dentry->d_count--;
+	dentry->d_lockref.count--;
 	shrink_dcache_for_umount_subtree(dentry);
  
 	while (!hlist_bl_empty(&sb->s_anon)) {
@@ -1147,7 +1136,7 @@
 		 * loop in shrink_dcache_parent() might not make any progress
 		 * and loop forever.
 		 */
-		if (dentry->d_count) {
+		if (dentry->d_lockref.count) {
 			dentry_lru_del(dentry);
 		} else if (!(dentry->d_flags & DCACHE_SHRINK_LIST)) {
 			dentry_lru_move_list(dentry, dispose);
@@ -1269,7 +1258,7 @@
 	smp_wmb();
 	dentry->d_name.name = dname;
  
-	dentry->d_count = 1;
+	dentry->d_lockref.count = 1;
 	dentry->d_flags = 0;
 	spin_lock_init(&dentry->d_lock);
 	seqcount_init(&dentry->d_seq);
@@ -1970,7 +1959,7 @@
 				goto next;
 		}
  
-		dentry->d_count++;
+		dentry->d_lockref.count++;
 		found = dentry;
 		spin_unlock(&dentry->d_lock);
 		break;
@@ -2069,7 +2058,7 @@
 	spin_lock(&dentry->d_lock);
 	inode = dentry->d_inode;
 	isdir = S_ISDIR(inode->i_mode);
-	if (dentry->d_count == 1) {
+	if (dentry->d_lockref.count == 1) {
 		if (!spin_trylock(&inode->i_lock)) {
 			spin_unlock(&dentry->d_lock);
 			cpu_relax();
@@ -2948,7 +2937,7 @@
 		}
 		if (!(dentry->d_flags & DCACHE_GENOCIDE)) {
 			dentry->d_flags |= DCACHE_GENOCIDE;
-			dentry->d_count--;
+			dentry->d_lockref.count--;
 		}
 		spin_unlock(&dentry->d_lock);
 	}
@@ -2956,7 +2945,7 @@
 		struct dentry *child = this_parent;
 		if (!(this_parent->d_flags & DCACHE_GENOCIDE)) {
 			this_parent->d_flags |= DCACHE_GENOCIDE;
-			this_parent->d_count--;
+			this_parent->d_lockref.count--;
 		}
 		this_parent = try_to_ascend(this_parent, locked, seq);
 		if (!this_parent)
@@ -536,8 +536,8 @@
 		 * a reference at this point.
 		 */
 		BUG_ON(!IS_ROOT(dentry) && dentry->d_parent != parent);
-		BUG_ON(!parent->d_count);
-		parent->d_count++;
+		BUG_ON(!parent->d_lockref.count);
+		parent->d_lockref.count++;
 		spin_unlock(&dentry->d_lock);
 	}
 	spin_unlock(&parent->d_lock);
@@ -3327,7 +3327,7 @@
 {
 	shrink_dcache_parent(dentry);
 	spin_lock(&dentry->d_lock);
-	if (dentry->d_count == 1)
+	if (dentry->d_lockref.count == 1)
 		__d_drop(dentry);
 	spin_unlock(&dentry->d_lock);
 }
@@ -9,6 +9,7 @@
 #include <linux/seqlock.h>
 #include <linux/cache.h>
 #include <linux/rcupdate.h>
+#include <linux/lockref.h>
  
 struct nameidata;
 struct path;
@@ -100,6 +101,8 @@
 # endif
 #endif
  
+#define d_lock	d_lockref.lock
+
 struct dentry {
 	/* RCU lookup touched fields */
 	unsigned int d_flags;		/* protected by d_lock */
@@ -112,8 +115,7 @@
 	unsigned char d_iname[DNAME_INLINE_LEN];	/* small names */
  
 	/* Ref lookup also touches following */
-	unsigned int d_count;		/* protected by d_lock */
-	spinlock_t d_lock;		/* per dentry lock */
+	struct lockref d_lockref;	/* per-dentry lock and refcount */
 	const struct dentry_operations *d_op;
 	struct super_block *d_sb;	/* The root of the dentry tree */
 	unsigned long d_time;		/* used by d_revalidate */
@@ -318,7 +320,7 @@
 	assert_spin_locked(&dentry->d_lock);
 	if (!read_seqcount_retry(&dentry->d_seq, seq)) {
 		ret = 1;
-		dentry->d_count++;
+		dentry->d_lockref.count++;
 	}
  
 	return ret;
@@ -326,7 +328,7 @@
  
 static inline unsigned d_count(const struct dentry *dentry)
 {
-	return dentry->d_count;
+	return dentry->d_lockref.count;
 }
  
 /* validate "insecure" dentry pointer */
  
@@ -357,17 +359,14 @@
 static inline struct dentry *dget_dlock(struct dentry *dentry)
 {
 	if (dentry)
-		dentry->d_count++;
+		dentry->d_lockref.count++;
 	return dentry;
 }
  
 static inline struct dentry *dget(struct dentry *dentry)
 {
-	if (dentry) {
-		spin_lock(&dentry->d_lock);
-		dget_dlock(dentry);
-		spin_unlock(&dentry->d_lock);
-	}
+	if (dentry)
+		lockref_get(&dentry->d_lockref);
 	return dentry;
 }
...	...	@@ -229,7 +229,7 @@
229	229	*/
230	230	static void d_free(struct dentry *dentry)
231	231	{
232		- BUG_ON(dentry->d_count);
	232	+ BUG_ON(dentry->d_lockref.count);
233	233	this_cpu_dec(nr_dentry);
234	234	if (dentry->d_op && dentry->d_op->d_release)
235	235	dentry->d_op->d_release(dentry);
...	...	@@ -467,7 +467,7 @@
467	467	}
468	468
469	469	if (ref)
470		- dentry->d_count--;
	470	+ dentry->d_lockref.count--;
471	471	/*
472	472	* inform the fs via d_prune that this dentry is about to be
473	473	* unhashed and destroyed.
474	474
475	475
...	...	@@ -513,15 +513,10 @@
513	513	return;
514	514
515	515	repeat:
516		- if (dentry->d_count == 1)
	516	+ if (dentry->d_lockref.count == 1)
517	517	might_sleep();
518		- spin_lock(&dentry->d_lock);
519		- BUG_ON(!dentry->d_count);
520		- if (dentry->d_count > 1) {
521		- dentry->d_count--;
522		- spin_unlock(&dentry->d_lock);
	518	+ if (lockref_put_or_lock(&dentry->d_lockref))
523	519	return;
524		- }
525	520
526	521	if (dentry->d_flags & DCACHE_OP_DELETE) {
527	522	if (dentry->d_op->d_delete(dentry))
...	...	@@ -535,7 +530,7 @@
535	530	dentry->d_flags \|= DCACHE_REFERENCED;
536	531	dentry_lru_add(dentry);
537	532
538		- dentry->d_count--;
	533	+ dentry->d_lockref.count--;
539	534	spin_unlock(&dentry->d_lock);
540	535	return;
541	536
...	...	@@ -590,7 +585,7 @@
590	585	* We also need to leave mountpoints alone,
591	586	* directory or not.
592	587	*/
593		- if (dentry->d_count > 1 && dentry->d_inode) {
	588	+ if (dentry->d_lockref.count > 1 && dentry->d_inode) {
594	589	if (S_ISDIR(dentry->d_inode->i_mode) \|\| d_mountpoint(dentry)) {
595	590	spin_unlock(&dentry->d_lock);
596	591	return -EBUSY;
597	592
...	...	@@ -606,14 +601,12 @@
606	601	/* This must be called with d_lock held */
607	602	static inline void __dget_dlock(struct dentry *dentry)
608	603	{
609		- dentry->d_count++;
	604	+ dentry->d_lockref.count++;
610	605	}
611	606
612	607	static inline void __dget(struct dentry *dentry)
613	608	{
614		- spin_lock(&dentry->d_lock);
615		- __dget_dlock(dentry);
616		- spin_unlock(&dentry->d_lock);
	609	+ lockref_get(&dentry->d_lockref);
617	610	}
618	611
619	612	struct dentry dget_parent(struct dentry dentry)
...	...	@@ -634,8 +627,8 @@
634	627	goto repeat;
635	628	}
636	629	rcu_read_unlock();
637		- BUG_ON(!ret->d_count);
638		- ret->d_count++;
	630	+ BUG_ON(!ret->d_lockref.count);
	631	+ ret->d_lockref.count++;
639	632	spin_unlock(&ret->d_lock);
640	633	return ret;
641	634	}
...	...	@@ -718,7 +711,7 @@
718	711	spin_lock(&inode->i_lock);
719	712	hlist_for_each_entry(dentry, &inode->i_dentry, d_alias) {
720	713	spin_lock(&dentry->d_lock);
721		- if (!dentry->d_count) {
	714	+ if (!dentry->d_lockref.count) {
722	715	__dget_dlock(dentry);
723	716	__d_drop(dentry);
724	717	spin_unlock(&dentry->d_lock);
725	718
...	...	@@ -763,12 +756,8 @@
763	756	/* Prune ancestors. */
764	757	dentry = parent;
765	758	while (dentry) {
766		- spin_lock(&dentry->d_lock);
767		- if (dentry->d_count > 1) {
768		- dentry->d_count--;
769		- spin_unlock(&dentry->d_lock);
	759	+ if (lockref_put_or_lock(&dentry->d_lockref))
770	760	return;
771		- }
772	761	dentry = dentry_kill(dentry, 1);
773	762	}
774	763	}
...	...	@@ -793,7 +782,7 @@
793	782	* the LRU because of laziness during lookup. Do not free
794	783	* it - just keep it off the LRU list.
795	784	*/
796		- if (dentry->d_count) {
	785	+ if (dentry->d_lockref.count) {
797	786	dentry_lru_del(dentry);
798	787	spin_unlock(&dentry->d_lock);
799	788	continue;
...	...	@@ -913,7 +902,7 @@
913	902	dentry_lru_del(dentry);
914	903	__d_shrink(dentry);
915	904
916		- if (dentry->d_count != 0) {
	905	+ if (dentry->d_lockref.count != 0) {
917	906	printk(KERN_ERR
918	907	"BUG: Dentry %p{i=%lx,n=%s}"
919	908	" still in use (%d)"
...	...	@@ -922,7 +911,7 @@
922	911	dentry->d_inode ?
923	912	dentry->d_inode->i_ino : 0UL,
924	913	dentry->d_name.name,
925		- dentry->d_count,
	914	+ dentry->d_lockref.count,
926	915	dentry->d_sb->s_type->name,
927	916	dentry->d_sb->s_id);
928	917	BUG();
...	...	@@ -933,7 +922,7 @@
933	922	list_del(&dentry->d_u.d_child);
934	923	} else {
935	924	parent = dentry->d_parent;
936		- parent->d_count--;
	925	+ parent->d_lockref.count--;
937	926	list_del(&dentry->d_u.d_child);
938	927	}
939	928
...	...	@@ -981,7 +970,7 @@
981	970
982	971	dentry = sb->s_root;
983	972	sb->s_root = NULL;
984		- dentry->d_count--;
	973	+ dentry->d_lockref.count--;
985	974	shrink_dcache_for_umount_subtree(dentry);
986	975
987	976	while (!hlist_bl_empty(&sb->s_anon)) {
...	...	@@ -1147,7 +1136,7 @@
1147	1136	* loop in shrink_dcache_parent() might not make any progress
1148	1137	* and loop forever.
1149	1138	*/
1150		- if (dentry->d_count) {
	1139	+ if (dentry->d_lockref.count) {
1151	1140	dentry_lru_del(dentry);
1152	1141	} else if (!(dentry->d_flags & DCACHE_SHRINK_LIST)) {
1153	1142	dentry_lru_move_list(dentry, dispose);
...	...	@@ -1269,7 +1258,7 @@
1269	1258	smp_wmb();
1270	1259	dentry->d_name.name = dname;
1271	1260
1272		- dentry->d_count = 1;
	1261	+ dentry->d_lockref.count = 1;
1273	1262	dentry->d_flags = 0;
1274	1263	spin_lock_init(&dentry->d_lock);
1275	1264	seqcount_init(&dentry->d_seq);
...	...	@@ -1970,7 +1959,7 @@
1970	1959	goto next;
1971	1960	}
1972	1961
1973		- dentry->d_count++;
	1962	+ dentry->d_lockref.count++;
1974	1963	found = dentry;
1975	1964	spin_unlock(&dentry->d_lock);
1976	1965	break;
...	...	@@ -2069,7 +2058,7 @@
2069	2058	spin_lock(&dentry->d_lock);
2070	2059	inode = dentry->d_inode;
2071	2060	isdir = S_ISDIR(inode->i_mode);
2072		- if (dentry->d_count == 1) {
	2061	+ if (dentry->d_lockref.count == 1) {
2073	2062	if (!spin_trylock(&inode->i_lock)) {
2074	2063	spin_unlock(&dentry->d_lock);
2075	2064	cpu_relax();
...	...	@@ -2948,7 +2937,7 @@
2948	2937	}
2949	2938	if (!(dentry->d_flags & DCACHE_GENOCIDE)) {
2950	2939	dentry->d_flags \|= DCACHE_GENOCIDE;
2951		- dentry->d_count--;
	2940	+ dentry->d_lockref.count--;
2952	2941	}
2953	2942	spin_unlock(&dentry->d_lock);
2954	2943	}
...	...	@@ -2956,7 +2945,7 @@
2956	2945	struct dentry *child = this_parent;
2957	2946	if (!(this_parent->d_flags & DCACHE_GENOCIDE)) {
2958	2947	this_parent->d_flags \|= DCACHE_GENOCIDE;
2959		- this_parent->d_count--;
	2948	+ this_parent->d_lockref.count--;
2960	2949	}
2961	2950	this_parent = try_to_ascend(this_parent, locked, seq);
2962	2951	if (!this_parent)
...	...	@@ -536,8 +536,8 @@
536	536	* a reference at this point.
537	537	*/
538	538	BUG_ON(!IS_ROOT(dentry) && dentry->d_parent != parent);
539		- BUG_ON(!parent->d_count);
540		- parent->d_count++;
	539	+ BUG_ON(!parent->d_lockref.count);
	540	+ parent->d_lockref.count++;
541	541	spin_unlock(&dentry->d_lock);
542	542	}
543	543	spin_unlock(&parent->d_lock);
...	...	@@ -3327,7 +3327,7 @@
3327	3327	{
3328	3328	shrink_dcache_parent(dentry);
3329	3329	spin_lock(&dentry->d_lock);
3330		- if (dentry->d_count == 1)
	3330	+ if (dentry->d_lockref.count == 1)
3331	3331	__d_drop(dentry);
3332	3332	spin_unlock(&dentry->d_lock);
3333	3333	}
...	...	@@ -9,6 +9,7 @@
9	9	#include <linux/seqlock.h>
10	10	#include <linux/cache.h>
11	11	#include <linux/rcupdate.h>
	12	+#include <linux/lockref.h>
12	13
13	14	struct nameidata;
14	15	struct path;
...	...	@@ -100,6 +101,8 @@
100	101	# endif
101	102	#endif
102	103
	104	+#define d_lock d_lockref.lock
	105	+
103	106	struct dentry {
104	107	/* RCU lookup touched fields */
105	108	unsigned int d_flags; /* protected by d_lock */
...	...	@@ -112,8 +115,7 @@
112	115	unsigned char d_iname[DNAME_INLINE_LEN]; /* small names */
113	116
114	117	/* Ref lookup also touches following */
115		- unsigned int d_count; /* protected by d_lock */
116		- spinlock_t d_lock; /* per dentry lock */
	118	+ struct lockref d_lockref; /* per-dentry lock and refcount */
117	119	const struct dentry_operations *d_op;
118	120	struct super_block d_sb; / The root of the dentry tree */
119	121	unsigned long d_time; /* used by d_revalidate */
...	...	@@ -318,7 +320,7 @@
318	320	assert_spin_locked(&dentry->d_lock);
319	321	if (!read_seqcount_retry(&dentry->d_seq, seq)) {
320	322	ret = 1;
321		- dentry->d_count++;
	323	+ dentry->d_lockref.count++;
322	324	}
323	325
324	326	return ret;
...	...	@@ -326,7 +328,7 @@
326	328
327	329	static inline unsigned d_count(const struct dentry *dentry)
328	330	{
329		- return dentry->d_count;
	331	+ return dentry->d_lockref.count;
330	332	}
331	333
332	334	/* validate "insecure" dentry pointer */
333	335
...	...	@@ -357,17 +359,14 @@
357	359	static inline struct dentry dget_dlock(struct dentry dentry)
358	360	{
359	361	if (dentry)
360		- dentry->d_count++;
	362	+ dentry->d_lockref.count++;
361	363	return dentry;
362	364	}
363	365
364	366	static inline struct dentry dget(struct dentry dentry)
365	367	{
366		- if (dentry) {
367		- spin_lock(&dentry->d_lock);
368		- dget_dlock(dentry);
369		- spin_unlock(&dentry->d_lock);
370		- }
	368	+ if (dentry)
	369	+ lockref_get(&dentry->d_lockref);
371	370	return dentry;
372	371	}
373	372