NFSv4: Remove nfs_client->cl_sem

Now that we're using the flags to indicate state that needs to be recovered, as well as having implemented proper refcounting and spinlocking on the state and open_owners, we can get rid of nfs_client->cl_sem. The only remaining case that was dubious was the file locking, and that case is now covered by the nfsi->rwsem. Signed-off-by: Trond Myklebust <Trond.Myklebust@netapp.com>

NFSv4: Remove nfs_client->cl_sem
Now that we're using the flags to indicate state that needs to be recovered, as well as having implemented proper refcounting and spinlocking on the state and open_owners, we can get rid of nfs_client->cl_sem. The only remaining case that was dubious was the file locking, and that case is now covered by the nfsi->rwsem. Signed-off-by: Trond Myklebust <Trond.Myklebust@netapp.com>
Trond Myklebust
1 parent 19e03c570e
Showing 6 changed files with 1 additions and 51 deletions Side-by-side Diff
fs/nfs/client.c
fs/nfs/delegation.c
fs/nfs/nfs4proc.c
fs/nfs/nfs4renewd.c
fs/nfs/nfs4state.c
include/linux/nfs_fs_sb.h
@@ -143,7 +143,6 @@
 	clp->cl_proto = cl_init->proto;
  
 #ifdef CONFIG_NFS_V4
-	init_rwsem(&clp->cl_sem);
 	INIT_LIST_HEAD(&clp->cl_delegations);
 	spin_lock_init(&clp->cl_lock);
 	INIT_DELAYED_WORK(&clp->cl_renewd, nfs4_renew_state);
@@ -243,16 +243,13 @@
  */
 static int __nfs_inode_return_delegation(struct inode *inode, struct nfs_delegation *delegation)
 {
-	struct nfs_client *clp = NFS_SERVER(inode)->nfs_client;
 	struct nfs_inode *nfsi = NFS_I(inode);
  
 	nfs_msync_inode(inode);
-	down_read(&clp->cl_sem);
 	/* Guard against new delegated open calls */
 	down_write(&nfsi->rwsem);
 	nfs_delegation_claim_opens(inode, &delegation->stateid);
 	up_write(&nfsi->rwsem);
-	up_read(&clp->cl_sem);
 	nfs_msync_inode(inode);
  
 	return nfs_do_return_delegation(inode, delegation, 1);
@@ -425,7 +422,6 @@
 	daemonize("nfsv4-delegreturn");
  
 	nfs_msync_inode(inode);
-	down_read(&clp->cl_sem);
 	down_write(&nfsi->rwsem);
 	spin_lock(&clp->cl_lock);
 	delegation = nfs_detach_delegation_locked(nfsi, args->stateid);
@@ -437,7 +433,6 @@
 	complete(&args->started);
 	nfs_delegation_claim_opens(inode, args->stateid);
 	up_write(&nfsi->rwsem);
-	up_read(&clp->cl_sem);
 	nfs_msync_inode(inode);
  
 	if (delegation != NULL)
@@ -207,12 +207,8 @@
  
 	might_sleep();
  
-	rwsem_acquire(&clp->cl_sem.dep_map, 0, 0, _RET_IP_);
-
 	res = wait_on_bit(&clp->cl_state, NFS4CLNT_STATE_RECOVER,
 			nfs4_wait_bit_killable, TASK_KILLABLE);
-
-	rwsem_release(&clp->cl_sem.dep_map, 1, _RET_IP_);
 	return res;
 }
  
@@ -1135,7 +1131,6 @@
 	struct nfs4_state_owner  *sp;
 	struct nfs4_state     *state = NULL;
 	struct nfs_server       *server = NFS_SERVER(dir);
-	struct nfs_client *clp = server->nfs_client;
 	struct nfs4_opendata *opendata;
 	int status;
  
  
@@ -1150,11 +1145,10 @@
 		goto err_put_state_owner;
 	if (path->dentry->d_inode != NULL)
 		nfs4_return_incompatible_delegation(path->dentry->d_inode, flags & (FMODE_READ|FMODE_WRITE));
-	down_read(&clp->cl_sem);
 	status = -ENOMEM;
 	opendata = nfs4_opendata_alloc(path, sp, flags, sattr);
 	if (opendata == NULL)
-		goto err_release_rwsem;
+		goto err_put_state_owner;
  
 	if (path->dentry->d_inode != NULL)
 		opendata->state = nfs4_get_open_state(path->dentry->d_inode, sp);
  
@@ -1172,13 +1166,10 @@
 		goto err_opendata_put;
 	nfs4_opendata_put(opendata);
 	nfs4_put_state_owner(sp);
-	up_read(&clp->cl_sem);
 	*res = state;
 	return 0;
 err_opendata_put:
 	nfs4_opendata_put(opendata);
-err_release_rwsem:
-	up_read(&clp->cl_sem);
 err_put_state_owner:
 	nfs4_put_state_owner(sp);
 out_err:
@@ -3099,7 +3090,6 @@
 	struct nfs4_lock_state *lsp;
 	int status;
  
-	down_read(&clp->cl_sem);
 	arg.lock_owner.clientid = clp->cl_clientid;
 	status = nfs4_set_lock_state(state, request);
 	if (status != 0)
@@ -3116,7 +3106,6 @@
 	}
 	request->fl_ops->fl_release_private(request);
 out:
-	up_read(&clp->cl_sem);
 	return status;
 }
  
@@ -3273,7 +3262,6 @@
  
 static int nfs4_proc_unlck(struct nfs4_state *state, int cmd, struct file_lock *request)
 {
-	struct nfs_client *clp = state->owner->so_client;
 	struct nfs_inode *nfsi = NFS_I(state->inode);
 	struct nfs_seqid *seqid;
 	struct nfs4_lock_state *lsp;
  
  
@@ -3284,15 +3272,12 @@
 	status = nfs4_set_lock_state(state, request);
 	/* Unlock _before_ we do the RPC call */
 	request->fl_flags |= FL_EXISTS;
-	down_read(&clp->cl_sem);
 	down_read(&nfsi->rwsem);
 	if (do_vfs_lock(request->fl_file, request) == -ENOENT) {
 		up_read(&nfsi->rwsem);
-		up_read(&clp->cl_sem);
 		goto out;
 	}
 	up_read(&nfsi->rwsem);
-	up_read(&clp->cl_sem);
 	if (status != 0)
 		goto out;
 	/* Is this a delegated lock? */
@@ -3518,7 +3503,6 @@
  
 static int _nfs4_proc_setlk(struct nfs4_state *state, int cmd, struct file_lock *request)
 {
-	struct nfs_client *clp = state->owner->so_client;
 	struct nfs_inode *nfsi = NFS_I(state->inode);
 	unsigned char fl_flags = request->fl_flags;
 	int status;
@@ -3531,7 +3515,6 @@
 	status = do_vfs_lock(request->fl_file, request);
 	if (status < 0)
 		goto out;
-	down_read(&clp->cl_sem);
 	down_read(&nfsi->rwsem);
 	if (test_bit(NFS_DELEGATED_STATE, &state->flags)) {
 		/* Yes: cache locks! */
@@ -3549,7 +3532,6 @@
 		printk(KERN_WARNING "%s: VFS is out of sync with lock manager!\n", __func__);
 out_unlock:
 	up_read(&nfsi->rwsem);
-	up_read(&clp->cl_sem);
 out:
 	request->fl_flags = fl_flags;
 	return status;
@@ -65,7 +65,6 @@
 	long lease, timeout;
 	unsigned long last, now;
  
-	down_read(&clp->cl_sem);
 	dprintk("%s: start\n", __func__);
 	/* Are there any active superblocks? */
 	if (list_empty(&clp->cl_superblocks))
  
@@ -101,11 +100,9 @@
 	schedule_delayed_work(&clp->cl_renewd, timeout);
 	spin_unlock(&clp->cl_lock);
 out:
-	up_read(&clp->cl_sem);
 	dprintk("%s: done\n", __func__);
 }
  
-/* Must be called with clp->cl_sem locked for writes */
 void
 nfs4_schedule_state_renewal(struct nfs_client *clp)
 {
@@ -305,10 +305,6 @@
 	}
 }
  
-/*
- * Note: must be called with clp->cl_sem held in order to prevent races
- *       with reboot recovery!
- */
 struct nfs4_state_owner *nfs4_get_state_owner(struct nfs_server *server, struct rpc_cred *cred)
 {
 	struct nfs_client *clp = server->nfs_client;
@@ -337,10 +333,6 @@
 	return sp;
 }
  
-/*
- * Must be called with clp->cl_sem held in order to avoid races
- * with state recovery...
- */
 void nfs4_put_state_owner(struct nfs4_state_owner *sp)
 {
 	struct nfs_client *clp = sp->so_client;
@@ -442,10 +434,6 @@
 	return state;
 }
  
-/*
- * Beware! Caller must be holding exactly one
- * reference to clp->cl_sem!
- */
 void nfs4_put_open_state(struct nfs4_state *state)
 {
 	struct inode *inode = state->inode;
@@ -578,7 +566,6 @@
  * Return a compatible lock_state. If no initialized lock_state structure
  * exists, return an uninitialized one.
  *
- * The caller must be holding clp->cl_sem
  */
 static struct nfs4_lock_state *nfs4_get_lock_state(struct nfs4_state *state, fl_owner_t owner)
 {
@@ -1127,7 +1114,6 @@
 	allow_signal(SIGKILL);
  
 	/* Ensure exclusive access to NFSv4 state */
-	down_write(&clp->cl_sem);
 	while (!list_empty(&clp->cl_superblocks)) {
 		if (test_and_clear_bit(NFS4CLNT_LEASE_EXPIRED, &clp->cl_state)) {
 			/* We're going to have to re-establish a clientid */
@@ -1149,7 +1135,6 @@
  
 		/* First recover reboot state... */
 		if (test_and_clear_bit(NFS4CLNT_RECLAIM_REBOOT, &clp->cl_state)) {
-			/* Note: list is protected by exclusive lock on cl->cl_sem */
 			status = nfs4_do_reclaim(clp, &nfs4_reboot_recovery_ops);
 			if (status == -NFS4ERR_STALE_CLIENTID)
 				continue;
@@ -1159,7 +1144,6 @@
  
 		/* Now recover expired state... */
 		if (test_and_clear_bit(NFS4CLNT_RECLAIM_NOGRACE, &clp->cl_state)) {
-			/* Note: list is protected by exclusive lock on cl->cl_sem */
 			status = nfs4_do_reclaim(clp, &nfs4_nograce_recovery_ops);
 			if (status < 0) {
 				set_bit(NFS4CLNT_RECLAIM_NOGRACE, &clp->cl_state);
@@ -1175,7 +1159,6 @@
 		break;
 	}
 out:
-	up_write(&clp->cl_sem);
 	if (test_and_clear_bit(NFS4CLNT_CB_PATH_DOWN, &clp->cl_state))
 		nfs_handle_cb_pathdown(clp);
 	nfs4_clear_recover_bit(clp);
@@ -42,12 +42,6 @@
 	struct rb_root		cl_openowner_id;
 	struct rb_root		cl_lockowner_id;
  
-	/*
-	 * The following rwsem ensures exclusive access to the server
-	 * while we recover the state following a lease expiration.
-	 */
-	struct rw_semaphore	cl_sem;
-
 	struct list_head	cl_delegations;
 	struct rb_root		cl_state_owners;
 	spinlock_t		cl_lock;
...	...	@@ -143,7 +143,6 @@
143	143	clp->cl_proto = cl_init->proto;
144	144
145	145	#ifdef CONFIG_NFS_V4
146		- init_rwsem(&clp->cl_sem);
147	146	INIT_LIST_HEAD(&clp->cl_delegations);
148	147	spin_lock_init(&clp->cl_lock);
149	148	INIT_DELAYED_WORK(&clp->cl_renewd, nfs4_renew_state);
...	...	@@ -243,16 +243,13 @@
243	243	*/
244	244	static int __nfs_inode_return_delegation(struct inode inode, struct nfs_delegation delegation)
245	245	{
246		- struct nfs_client *clp = NFS_SERVER(inode)->nfs_client;
247	246	struct nfs_inode *nfsi = NFS_I(inode);
248	247
249	248	nfs_msync_inode(inode);
250		- down_read(&clp->cl_sem);
251	249	/* Guard against new delegated open calls */
252	250	down_write(&nfsi->rwsem);
253	251	nfs_delegation_claim_opens(inode, &delegation->stateid);
254	252	up_write(&nfsi->rwsem);
255		- up_read(&clp->cl_sem);
256	253	nfs_msync_inode(inode);
257	254
258	255	return nfs_do_return_delegation(inode, delegation, 1);
...	...	@@ -425,7 +422,6 @@
425	422	daemonize("nfsv4-delegreturn");
426	423
427	424	nfs_msync_inode(inode);
428		- down_read(&clp->cl_sem);
429	425	down_write(&nfsi->rwsem);
430	426	spin_lock(&clp->cl_lock);
431	427	delegation = nfs_detach_delegation_locked(nfsi, args->stateid);
...	...	@@ -437,7 +433,6 @@
437	433	complete(&args->started);
438	434	nfs_delegation_claim_opens(inode, args->stateid);
439	435	up_write(&nfsi->rwsem);
440		- up_read(&clp->cl_sem);
441	436	nfs_msync_inode(inode);
442	437
443	438	if (delegation != NULL)
...	...	@@ -207,12 +207,8 @@
207	207
208	208	might_sleep();
209	209
210		- rwsem_acquire(&clp->cl_sem.dep_map, 0, 0, _RET_IP_);
211		-
212	210	res = wait_on_bit(&clp->cl_state, NFS4CLNT_STATE_RECOVER,
213	211	nfs4_wait_bit_killable, TASK_KILLABLE);
214		-
215		- rwsem_release(&clp->cl_sem.dep_map, 1, _RET_IP_);
216	212	return res;
217	213	}
218	214
...	...	@@ -1135,7 +1131,6 @@
1135	1131	struct nfs4_state_owner *sp;
1136	1132	struct nfs4_state *state = NULL;
1137	1133	struct nfs_server *server = NFS_SERVER(dir);
1138		- struct nfs_client *clp = server->nfs_client;
1139	1134	struct nfs4_opendata *opendata;
1140	1135	int status;
1141	1136
1142	1137
...	...	@@ -1150,11 +1145,10 @@
1150	1145	goto err_put_state_owner;
1151	1146	if (path->dentry->d_inode != NULL)
1152	1147	nfs4_return_incompatible_delegation(path->dentry->d_inode, flags & (FMODE_READ\|FMODE_WRITE));
1153		- down_read(&clp->cl_sem);
1154	1148	status = -ENOMEM;
1155	1149	opendata = nfs4_opendata_alloc(path, sp, flags, sattr);
1156	1150	if (opendata == NULL)
1157		- goto err_release_rwsem;
	1151	+ goto err_put_state_owner;
1158	1152
1159	1153	if (path->dentry->d_inode != NULL)
1160	1154	opendata->state = nfs4_get_open_state(path->dentry->d_inode, sp);
1161	1155
...	...	@@ -1172,13 +1166,10 @@
1172	1166	goto err_opendata_put;
1173	1167	nfs4_opendata_put(opendata);
1174	1168	nfs4_put_state_owner(sp);
1175		- up_read(&clp->cl_sem);
1176	1169	*res = state;
1177	1170	return 0;
1178	1171	err_opendata_put:
1179	1172	nfs4_opendata_put(opendata);
1180		-err_release_rwsem:
1181		- up_read(&clp->cl_sem);
1182	1173	err_put_state_owner:
1183	1174	nfs4_put_state_owner(sp);
1184	1175	out_err:
...	...	@@ -3099,7 +3090,6 @@
3099	3090	struct nfs4_lock_state *lsp;
3100	3091	int status;
3101	3092
3102		- down_read(&clp->cl_sem);
3103	3093	arg.lock_owner.clientid = clp->cl_clientid;
3104	3094	status = nfs4_set_lock_state(state, request);
3105	3095	if (status != 0)
...	...	@@ -3116,7 +3106,6 @@
3116	3106	}
3117	3107	request->fl_ops->fl_release_private(request);
3118	3108	out:
3119		- up_read(&clp->cl_sem);
3120	3109	return status;
3121	3110	}
3122	3111
...	...	@@ -3273,7 +3262,6 @@
3273	3262
3274	3263	static int nfs4_proc_unlck(struct nfs4_state state, int cmd, struct file_lock request)
3275	3264	{
3276		- struct nfs_client *clp = state->owner->so_client;
3277	3265	struct nfs_inode *nfsi = NFS_I(state->inode);
3278	3266	struct nfs_seqid *seqid;
3279	3267	struct nfs4_lock_state *lsp;
3280	3268
3281	3269
...	...	@@ -3284,15 +3272,12 @@
3284	3272	status = nfs4_set_lock_state(state, request);
3285	3273	/* Unlock _before_ we do the RPC call */
3286	3274	request->fl_flags \|= FL_EXISTS;
3287		- down_read(&clp->cl_sem);
3288	3275	down_read(&nfsi->rwsem);
3289	3276	if (do_vfs_lock(request->fl_file, request) == -ENOENT) {
3290	3277	up_read(&nfsi->rwsem);
3291		- up_read(&clp->cl_sem);
3292	3278	goto out;
3293	3279	}
3294	3280	up_read(&nfsi->rwsem);
3295		- up_read(&clp->cl_sem);
3296	3281	if (status != 0)
3297	3282	goto out;
3298	3283	/* Is this a delegated lock? */
...	...	@@ -3518,7 +3503,6 @@
3518	3503
3519	3504	static int _nfs4_proc_setlk(struct nfs4_state state, int cmd, struct file_lock request)
3520	3505	{
3521		- struct nfs_client *clp = state->owner->so_client;
3522	3506	struct nfs_inode *nfsi = NFS_I(state->inode);
3523	3507	unsigned char fl_flags = request->fl_flags;
3524	3508	int status;
...	...	@@ -3531,7 +3515,6 @@
3531	3515	status = do_vfs_lock(request->fl_file, request);
3532	3516	if (status < 0)
3533	3517	goto out;
3534		- down_read(&clp->cl_sem);
3535	3518	down_read(&nfsi->rwsem);
3536	3519	if (test_bit(NFS_DELEGATED_STATE, &state->flags)) {
3537	3520	/* Yes: cache locks! */
...	...	@@ -3549,7 +3532,6 @@
3549	3532	printk(KERN_WARNING "%s: VFS is out of sync with lock manager!\n", __func__);
3550	3533	out_unlock:
3551	3534	up_read(&nfsi->rwsem);
3552		- up_read(&clp->cl_sem);
3553	3535	out:
3554	3536	request->fl_flags = fl_flags;
3555	3537	return status;
...	...	@@ -65,7 +65,6 @@
65	65	long lease, timeout;
66	66	unsigned long last, now;
67	67
68		- down_read(&clp->cl_sem);
69	68	dprintk("%s: start\n", __func__);
70	69	/* Are there any active superblocks? */
71	70	if (list_empty(&clp->cl_superblocks))
72	71
...	...	@@ -101,11 +100,9 @@
101	100	schedule_delayed_work(&clp->cl_renewd, timeout);
102	101	spin_unlock(&clp->cl_lock);
103	102	out:
104		- up_read(&clp->cl_sem);
105	103	dprintk("%s: done\n", __func__);
106	104	}
107	105
108		-/* Must be called with clp->cl_sem locked for writes */
109	106	void
110	107	nfs4_schedule_state_renewal(struct nfs_client *clp)
111	108	{
...	...	@@ -305,10 +305,6 @@
305	305	}
306	306	}
307	307
308		-/*
309		- * Note: must be called with clp->cl_sem held in order to prevent races
310		- * with reboot recovery!
311		- */
312	308	struct nfs4_state_owner nfs4_get_state_owner(struct nfs_server server, struct rpc_cred *cred)
313	309	{
314	310	struct nfs_client *clp = server->nfs_client;
...	...	@@ -337,10 +333,6 @@
337	333	return sp;
338	334	}
339	335
340		-/*
341		- * Must be called with clp->cl_sem held in order to avoid races
342		- * with state recovery...
343		- */
344	336	void nfs4_put_state_owner(struct nfs4_state_owner *sp)
345	337	{
346	338	struct nfs_client *clp = sp->so_client;
...	...	@@ -442,10 +434,6 @@
442	434	return state;
443	435	}
444	436
445		-/*
446		- * Beware! Caller must be holding exactly one
447		- * reference to clp->cl_sem!
448		- */
449	437	void nfs4_put_open_state(struct nfs4_state *state)
450	438	{
451	439	struct inode *inode = state->inode;
...	...	@@ -578,7 +566,6 @@
578	566	* Return a compatible lock_state. If no initialized lock_state structure
579	567	* exists, return an uninitialized one.
580	568	*
581		- * The caller must be holding clp->cl_sem
582	569	*/
583	570	static struct nfs4_lock_state nfs4_get_lock_state(struct nfs4_state state, fl_owner_t owner)
584	571	{
...	...	@@ -1127,7 +1114,6 @@
1127	1114	allow_signal(SIGKILL);
1128	1115
1129	1116	/* Ensure exclusive access to NFSv4 state */
1130		- down_write(&clp->cl_sem);
1131	1117	while (!list_empty(&clp->cl_superblocks)) {
1132	1118	if (test_and_clear_bit(NFS4CLNT_LEASE_EXPIRED, &clp->cl_state)) {
1133	1119	/* We're going to have to re-establish a clientid */
...	...	@@ -1149,7 +1135,6 @@
1149	1135
1150	1136	/* First recover reboot state... */
1151	1137	if (test_and_clear_bit(NFS4CLNT_RECLAIM_REBOOT, &clp->cl_state)) {
1152		- /* Note: list is protected by exclusive lock on cl->cl_sem */
1153	1138	status = nfs4_do_reclaim(clp, &nfs4_reboot_recovery_ops);
1154	1139	if (status == -NFS4ERR_STALE_CLIENTID)
1155	1140	continue;
...	...	@@ -1159,7 +1144,6 @@
1159	1144
1160	1145	/* Now recover expired state... */
1161	1146	if (test_and_clear_bit(NFS4CLNT_RECLAIM_NOGRACE, &clp->cl_state)) {
1162		- /* Note: list is protected by exclusive lock on cl->cl_sem */
1163	1147	status = nfs4_do_reclaim(clp, &nfs4_nograce_recovery_ops);
1164	1148	if (status < 0) {
1165	1149	set_bit(NFS4CLNT_RECLAIM_NOGRACE, &clp->cl_state);
...	...	@@ -1175,7 +1159,6 @@
1175	1159	break;
1176	1160	}
1177	1161	out:
1178		- up_write(&clp->cl_sem);
1179	1162	if (test_and_clear_bit(NFS4CLNT_CB_PATH_DOWN, &clp->cl_state))
1180	1163	nfs_handle_cb_pathdown(clp);
1181	1164	nfs4_clear_recover_bit(clp);
...	...	@@ -42,12 +42,6 @@
42	42	struct rb_root cl_openowner_id;
43	43	struct rb_root cl_lockowner_id;
44	44
45		- /*
46		- * The following rwsem ensures exclusive access to the server
47		- * while we recover the state following a lease expiration.
48		- */
49		- struct rw_semaphore cl_sem;
50		-
51	45	struct list_head cl_delegations;
52	46	struct rb_root cl_state_owners;
53	47	spinlock_t cl_lock;