SUNRPC: cleanup rpc credential cache garbage collection

Signed-off-by: Trond Myklebust <Trond.Myklebust@netapp.com>

SUNRPC: cleanup rpc credential cache garbage collection
Signed-off-by: Trond Myklebust <Trond.Myklebust@netapp.com>
Trond Myklebust
1 parent fc432dd907
Showing 3 changed files with 74 additions and 49 deletions Side-by-side Diff
include/linux/sunrpc/auth.h
net/sunrpc/auth.c
net/sunrpc/auth_null.c
@@ -34,6 +34,7 @@
 struct rpc_credops;
 struct rpc_cred {
 	struct hlist_node	cr_hash;	/* hash chain */
+	struct list_head	cr_lru;		/* lru garbage collection */
 	struct rpc_auth *	cr_auth;
 	const struct rpc_credops *cr_ops;
 #ifdef RPC_DEBUG
@@ -25,6 +25,8 @@
 	NULL,			/* others can be loadable modules */
 };
  
+static LIST_HEAD(cred_unused);
+
 static u32
 pseudoflavor_to_flavor(u32 flavor) {
 	if (flavor >= RPC_AUTH_MAXFLAVOR)
  
@@ -134,13 +136,13 @@
  * Destroy a list of credentials
  */
 static inline
-void rpcauth_destroy_credlist(struct hlist_head *head)
+void rpcauth_destroy_credlist(struct list_head *head)
 {
 	struct rpc_cred *cred;
  
-	while (!hlist_empty(head)) {
-		cred = hlist_entry(head->first, struct rpc_cred, cr_hash);
-		hlist_del_init(&cred->cr_hash);
+	while (!list_empty(head)) {
+		cred = list_entry(head->next, struct rpc_cred, cr_lru);
+		list_del_init(&cred->cr_lru);
 		put_rpccred(cred);
 	}
 }
  
@@ -152,17 +154,20 @@
 void
 rpcauth_clear_credcache(struct rpc_cred_cache *cache)
 {
-	HLIST_HEAD(free);
-	struct hlist_node *pos, *next;
+	LIST_HEAD(free);
+	struct hlist_head *head;
 	struct rpc_cred	*cred;
 	int		i;
  
 	spin_lock(&rpc_credcache_lock);
 	for (i = 0; i < RPC_CREDCACHE_NR; i++) {
-		hlist_for_each_safe(pos, next, &cache->hashtable[i]) {
-			cred = hlist_entry(pos, struct rpc_cred, cr_hash);
-			__hlist_del(&cred->cr_hash);
-			hlist_add_head(&cred->cr_hash, &free);
+		head = &cache->hashtable[i];
+		while (!hlist_empty(head)) {
+			cred = hlist_entry(head->first, struct rpc_cred, cr_hash);
+			get_rpccred(cred);
+			list_move_tail(&cred->cr_lru, &free);
+			smp_wmb();
+			hlist_del_init(&cred->cr_hash);
 		}
 	}
 	spin_unlock(&rpc_credcache_lock);
  
  
  
  
  
  
@@ -184,38 +189,39 @@
 	}
 }
  
+/*
+ * Remove stale credentials. Avoid sleeping inside the loop.
+ */
 static void
-rpcauth_prune_expired(struct rpc_auth *auth, struct rpc_cred *cred, struct hlist_head *free)
+rpcauth_prune_expired(struct list_head *free)
 {
-	if (atomic_read(&cred->cr_count) != 1)
-	       return;
-	if (time_after(jiffies, cred->cr_expire + auth->au_credcache->expire))
-		clear_bit(RPCAUTH_CRED_UPTODATE, &cred->cr_flags);
-	if (test_bit(RPCAUTH_CRED_UPTODATE, &cred->cr_flags) == 0) {
-		__hlist_del(&cred->cr_hash);
-		hlist_add_head(&cred->cr_hash, free);
+	struct rpc_cred *cred;
+
+	while (!list_empty(&cred_unused)) {
+		cred = list_entry(cred_unused.next, struct rpc_cred, cr_lru);
+		if (time_after(jiffies, cred->cr_expire +
+					cred->cr_auth->au_credcache->expire))
+			break;
+		list_del_init(&cred->cr_lru);
+		if (atomic_read(&cred->cr_count) != 0)
+			continue;
+		get_rpccred(cred);
+		list_add_tail(&cred->cr_lru, free);
+		smp_wmb();
+		hlist_del_init(&cred->cr_hash);
 	}
 }
  
 /*
- * Remove stale credentials. Avoid sleeping inside the loop.
+ * Run garbage collector.
  */
 static void
-rpcauth_gc_credcache(struct rpc_auth *auth, struct hlist_head *free)
+rpcauth_gc_credcache(struct rpc_cred_cache *cache, struct list_head *free)
 {
-	struct rpc_cred_cache *cache = auth->au_credcache;
-	struct hlist_node *pos, *next;
-	struct rpc_cred	*cred;
-	int		i;
-
-	dprintk("RPC:       gc'ing RPC credentials for auth %p\n", auth);
-	for (i = 0; i < RPC_CREDCACHE_NR; i++) {
-		hlist_for_each_safe(pos, next, &cache->hashtable[i]) {
-			cred = hlist_entry(pos, struct rpc_cred, cr_hash);
-			rpcauth_prune_expired(auth, cred, free);
-		}
-	}
+	if (time_before(jiffies, cache->nextgc))
+		return;
 	cache->nextgc = jiffies + cache->expire;
+	rpcauth_prune_expired(free);
 }
  
 /*
  
  
  
  
  
  
@@ -225,39 +231,35 @@
 rpcauth_lookup_credcache(struct rpc_auth *auth, struct auth_cred * acred,
 		int flags)
 {
+	LIST_HEAD(free);
 	struct rpc_cred_cache *cache = auth->au_credcache;
-	HLIST_HEAD(free);
-	struct hlist_node *pos, *next;
+	struct hlist_node *pos;
 	struct rpc_cred	*new = NULL,
-			*cred = NULL;
+			*cred = NULL,
+			*entry;
 	int		nr = 0;
  
 	if (!(flags & RPCAUTH_LOOKUP_ROOTCREDS))
 		nr = acred->uid & RPC_CREDCACHE_MASK;
 retry:
 	spin_lock(&rpc_credcache_lock);
-	if (time_before(cache->nextgc, jiffies))
-		rpcauth_gc_credcache(auth, &free);
-	hlist_for_each_safe(pos, next, &cache->hashtable[nr]) {
-		struct rpc_cred *entry;
-		entry = hlist_entry(pos, struct rpc_cred, cr_hash);
-		if (entry->cr_ops->crmatch(acred, entry, flags)) {
-			hlist_del(&entry->cr_hash);
-			cred = entry;
-			break;
-		}
-		rpcauth_prune_expired(auth, entry, &free);
+	hlist_for_each_entry(entry, pos, &cache->hashtable[nr], cr_hash) {
+		if (!entry->cr_ops->crmatch(acred, entry, flags))
+			continue;
+		cred = get_rpccred(entry);
+		hlist_del(&entry->cr_hash);
+		break;
 	}
 	if (new) {
 		if (cred)
-			hlist_add_head(&new->cr_hash, &free);
+			list_add_tail(&new->cr_lru, &free);
 		else
 			cred = new;
 	}
 	if (cred) {
 		hlist_add_head(&cred->cr_hash, &cache->hashtable[nr]);
-		get_rpccred(cred);
 	}
+	rpcauth_gc_credcache(cache, &free);
 	spin_unlock(&rpc_credcache_lock);
  
 	rpcauth_destroy_credlist(&free);
@@ -303,6 +305,7 @@
 		  struct rpc_auth *auth, const struct rpc_credops *ops)
 {
 	INIT_HLIST_NODE(&cred->cr_hash);
+	INIT_LIST_HEAD(&cred->cr_lru);
 	atomic_set(&cred->cr_count, 1);
 	cred->cr_auth = auth;
 	cred->cr_ops = ops;
  
@@ -353,9 +356,29 @@
 void
 put_rpccred(struct rpc_cred *cred)
 {
-	cred->cr_expire = jiffies;
+	/* Fast path for unhashed credentials */
+	if (!hlist_unhashed(&cred->cr_hash))
+		goto need_lock;
+
 	if (!atomic_dec_and_test(&cred->cr_count))
 		return;
+	goto out_destroy;
+
+need_lock:
+	if (!atomic_dec_and_lock(&cred->cr_count, &rpc_credcache_lock))
+		return;
+	if (!list_empty(&cred->cr_lru))
+		list_del_init(&cred->cr_lru);
+	if (test_bit(RPCAUTH_CRED_UPTODATE, &cred->cr_flags) == 0)
+		hlist_del(&cred->cr_hash);
+	else if (!hlist_unhashed(&cred->cr_hash)) {
+		cred->cr_expire = jiffies;
+		list_add_tail(&cred->cr_lru, &cred_unused);
+		spin_unlock(&rpc_credcache_lock);
+		return;
+	}
+	spin_unlock(&rpc_credcache_lock);
+out_destroy:
 	cred->cr_ops->crdestroy(cred);
 }
  
@@ -133,6 +133,7 @@
  
 static
 struct rpc_cred null_cred = {
+	.cr_lru		= LIST_HEAD_INIT(null_cred.cr_lru),
 	.cr_auth	= &null_auth,
 	.cr_ops		= &null_credops,
 	.cr_count	= ATOMIC_INIT(1),
...	...	@@ -34,6 +34,7 @@
34	34	struct rpc_credops;
35	35	struct rpc_cred {
36	36	struct hlist_node cr_hash; /* hash chain */
	37	+ struct list_head cr_lru; /* lru garbage collection */
37	38	struct rpc_auth * cr_auth;
38	39	const struct rpc_credops *cr_ops;
39	40	#ifdef RPC_DEBUG
...	...	@@ -25,6 +25,8 @@
25	25	NULL, /* others can be loadable modules */
26	26	};
27	27
	28	+static LIST_HEAD(cred_unused);
	29	+
28	30	static u32
29	31	pseudoflavor_to_flavor(u32 flavor) {
30	32	if (flavor >= RPC_AUTH_MAXFLAVOR)
31	33
...	...	@@ -134,13 +136,13 @@
134	136	* Destroy a list of credentials
135	137	*/
136	138	static inline
137		-void rpcauth_destroy_credlist(struct hlist_head *head)
	139	+void rpcauth_destroy_credlist(struct list_head *head)
138	140	{
139	141	struct rpc_cred *cred;
140	142
141		- while (!hlist_empty(head)) {
142		- cred = hlist_entry(head->first, struct rpc_cred, cr_hash);
143		- hlist_del_init(&cred->cr_hash);
	143	+ while (!list_empty(head)) {
	144	+ cred = list_entry(head->next, struct rpc_cred, cr_lru);
	145	+ list_del_init(&cred->cr_lru);
144	146	put_rpccred(cred);
145	147	}
146	148	}
147	149
...	...	@@ -152,17 +154,20 @@
152	154	void
153	155	rpcauth_clear_credcache(struct rpc_cred_cache *cache)
154	156	{
155		- HLIST_HEAD(free);
156		- struct hlist_node pos, next;
	157	+ LIST_HEAD(free);
	158	+ struct hlist_head *head;
157	159	struct rpc_cred *cred;
158	160	int i;
159	161
160	162	spin_lock(&rpc_credcache_lock);
161	163	for (i = 0; i < RPC_CREDCACHE_NR; i++) {
162		- hlist_for_each_safe(pos, next, &cache->hashtable[i]) {
163		- cred = hlist_entry(pos, struct rpc_cred, cr_hash);
164		- __hlist_del(&cred->cr_hash);
165		- hlist_add_head(&cred->cr_hash, &free);
	164	+ head = &cache->hashtable[i];
	165	+ while (!hlist_empty(head)) {
	166	+ cred = hlist_entry(head->first, struct rpc_cred, cr_hash);
	167	+ get_rpccred(cred);
	168	+ list_move_tail(&cred->cr_lru, &free);
	169	+ smp_wmb();
	170	+ hlist_del_init(&cred->cr_hash);
166	171	}
167	172	}
168	173	spin_unlock(&rpc_credcache_lock);
169	174
170	175
171	176
172	177
173	178
174	179
...	...	@@ -184,38 +189,39 @@
184	189	}
185	190	}
186	191
	192	+/*
	193	+ * Remove stale credentials. Avoid sleeping inside the loop.
	194	+ */
187	195	static void
188		-rpcauth_prune_expired(struct rpc_auth auth, struct rpc_cred cred, struct hlist_head *free)
	196	+rpcauth_prune_expired(struct list_head *free)
189	197	{
190		- if (atomic_read(&cred->cr_count) != 1)
191		- return;
192		- if (time_after(jiffies, cred->cr_expire + auth->au_credcache->expire))
193		- clear_bit(RPCAUTH_CRED_UPTODATE, &cred->cr_flags);
194		- if (test_bit(RPCAUTH_CRED_UPTODATE, &cred->cr_flags) == 0) {
195		- __hlist_del(&cred->cr_hash);
196		- hlist_add_head(&cred->cr_hash, free);
	198	+ struct rpc_cred *cred;
	199	+
	200	+ while (!list_empty(&cred_unused)) {
	201	+ cred = list_entry(cred_unused.next, struct rpc_cred, cr_lru);
	202	+ if (time_after(jiffies, cred->cr_expire +
	203	+ cred->cr_auth->au_credcache->expire))
	204	+ break;
	205	+ list_del_init(&cred->cr_lru);
	206	+ if (atomic_read(&cred->cr_count) != 0)
	207	+ continue;
	208	+ get_rpccred(cred);
	209	+ list_add_tail(&cred->cr_lru, free);
	210	+ smp_wmb();
	211	+ hlist_del_init(&cred->cr_hash);
197	212	}
198	213	}
199	214
200	215	/*
201		- * Remove stale credentials. Avoid sleeping inside the loop.
	216	+ * Run garbage collector.
202	217	*/
203	218	static void
204		-rpcauth_gc_credcache(struct rpc_auth auth, struct hlist_head free)
	219	+rpcauth_gc_credcache(struct rpc_cred_cache cache, struct list_head free)
205	220	{
206		- struct rpc_cred_cache *cache = auth->au_credcache;
207		- struct hlist_node pos, next;
208		- struct rpc_cred *cred;
209		- int i;
210		-
211		- dprintk("RPC: gc'ing RPC credentials for auth %p\n", auth);
212		- for (i = 0; i < RPC_CREDCACHE_NR; i++) {
213		- hlist_for_each_safe(pos, next, &cache->hashtable[i]) {
214		- cred = hlist_entry(pos, struct rpc_cred, cr_hash);
215		- rpcauth_prune_expired(auth, cred, free);
216		- }
217		- }
	221	+ if (time_before(jiffies, cache->nextgc))
	222	+ return;
218	223	cache->nextgc = jiffies + cache->expire;
	224	+ rpcauth_prune_expired(free);
219	225	}
220	226
221	227	/*
222	228
223	229
224	230
225	231
226	232
227	233
...	...	@@ -225,39 +231,35 @@
225	231	rpcauth_lookup_credcache(struct rpc_auth auth, struct auth_cred acred,
226	232	int flags)
227	233	{
	234	+ LIST_HEAD(free);
228	235	struct rpc_cred_cache *cache = auth->au_credcache;
229		- HLIST_HEAD(free);
230		- struct hlist_node pos, next;
	236	+ struct hlist_node *pos;
231	237	struct rpc_cred *new = NULL,
232		- *cred = NULL;
	238	+ *cred = NULL,
	239	+ *entry;
233	240	int nr = 0;
234	241
235	242	if (!(flags & RPCAUTH_LOOKUP_ROOTCREDS))
236	243	nr = acred->uid & RPC_CREDCACHE_MASK;
237	244	retry:
238	245	spin_lock(&rpc_credcache_lock);
239		- if (time_before(cache->nextgc, jiffies))
240		- rpcauth_gc_credcache(auth, &free);
241		- hlist_for_each_safe(pos, next, &cache->hashtable[nr]) {
242		- struct rpc_cred *entry;
243		- entry = hlist_entry(pos, struct rpc_cred, cr_hash);
244		- if (entry->cr_ops->crmatch(acred, entry, flags)) {
245		- hlist_del(&entry->cr_hash);
246		- cred = entry;
247		- break;
248		- }
249		- rpcauth_prune_expired(auth, entry, &free);
	246	+ hlist_for_each_entry(entry, pos, &cache->hashtable[nr], cr_hash) {
	247	+ if (!entry->cr_ops->crmatch(acred, entry, flags))
	248	+ continue;
	249	+ cred = get_rpccred(entry);
	250	+ hlist_del(&entry->cr_hash);
	251	+ break;
250	252	}
251	253	if (new) {
252	254	if (cred)
253		- hlist_add_head(&new->cr_hash, &free);
	255	+ list_add_tail(&new->cr_lru, &free);
254	256	else
255	257	cred = new;
256	258	}
257	259	if (cred) {
258	260	hlist_add_head(&cred->cr_hash, &cache->hashtable[nr]);
259		- get_rpccred(cred);
260	261	}
	262	+ rpcauth_gc_credcache(cache, &free);
261	263	spin_unlock(&rpc_credcache_lock);
262	264
263	265	rpcauth_destroy_credlist(&free);
...	...	@@ -303,6 +305,7 @@
303	305	struct rpc_auth auth, const struct rpc_credops ops)
304	306	{
305	307	INIT_HLIST_NODE(&cred->cr_hash);
	308	+ INIT_LIST_HEAD(&cred->cr_lru);
306	309	atomic_set(&cred->cr_count, 1);
307	310	cred->cr_auth = auth;
308	311	cred->cr_ops = ops;
309	312
...	...	@@ -353,9 +356,29 @@
353	356	void
354	357	put_rpccred(struct rpc_cred *cred)
355	358	{
356		- cred->cr_expire = jiffies;
	359	+ /* Fast path for unhashed credentials */
	360	+ if (!hlist_unhashed(&cred->cr_hash))
	361	+ goto need_lock;
	362	+
357	363	if (!atomic_dec_and_test(&cred->cr_count))
358	364	return;
	365	+ goto out_destroy;
	366	+
	367	+need_lock:
	368	+ if (!atomic_dec_and_lock(&cred->cr_count, &rpc_credcache_lock))
	369	+ return;
	370	+ if (!list_empty(&cred->cr_lru))
	371	+ list_del_init(&cred->cr_lru);
	372	+ if (test_bit(RPCAUTH_CRED_UPTODATE, &cred->cr_flags) == 0)
	373	+ hlist_del(&cred->cr_hash);
	374	+ else if (!hlist_unhashed(&cred->cr_hash)) {
	375	+ cred->cr_expire = jiffies;
	376	+ list_add_tail(&cred->cr_lru, &cred_unused);
	377	+ spin_unlock(&rpc_credcache_lock);
	378	+ return;
	379	+ }
	380	+ spin_unlock(&rpc_credcache_lock);
	381	+out_destroy:
359	382	cred->cr_ops->crdestroy(cred);
360	383	}
361	384
...	...	@@ -133,6 +133,7 @@
133	133
134	134	static
135	135	struct rpc_cred null_cred = {
	136	+ .cr_lru = LIST_HEAD_INIT(null_cred.cr_lru),
136	137	.cr_auth = &null_auth,
137	138	.cr_ops = &null_credops,
138	139	.cr_count = ATOMIC_INIT(1),