NFSv4.1 Use MDS auth flavor for data server connection

Commit 4edaa308 "NFS: Use "krb5i" to establish NFSv4 state whenever possible" uses the nfs_client cl_rpcclient for all state management operations, and will use krb5i or auth_sys with no regard to the mount command authflavor choice. The MDS, as any NFSv4.1 mount point, uses the nfs_server rpc client for all non-state management operations with a different nfs_server for each fsid encountered traversing the mount point, each with a potentially different auth flavor. pNFS data servers are not mounted in the normal sense as there is no associated nfs_server structure. Data servers can also export multiple fsids, each with a potentially different auth flavor. Data servers need to use the same authflavor as the MDS server rpc client for non-state management operations. Populate a list of rpc clients with the MDS server rpc client auth flavor for the DS to use. Signed-off-by: Andy Adamson <andros@netapp.com> Signed-off-by: Trond Myklebust <Trond.Myklebust@netapp.com>

NFSv4.1 Use MDS auth flavor for data server connection
Commit 4edaa308 "NFS: Use "krb5i" to establish NFSv4 state whenever possible" uses the nfs_client cl_rpcclient for all state management operations, and will use krb5i or auth_sys with no regard to the mount command authflavor choice. The MDS, as any NFSv4.1 mount point, uses the nfs_server rpc client for all non-state management operations with a different nfs_server for each fsid encountered traversing the mount point, each with a potentially different auth flavor. pNFS data servers are not mounted in the normal sense as there is no associated nfs_server structure. Data servers can also export multiple fsids, each with a potentially different auth flavor. Data servers need to use the same authflavor as the MDS server rpc client for non-state management operations. Populate a list of rpc clients with the MDS server rpc client auth flavor for the DS to use. Signed-off-by: Andy Adamson <andros@netapp.com> Signed-off-by: Trond Myklebust <Trond.Myklebust@netapp.com>
Andy Adamson · Trond Myklebust
1 parent 4109bb7496
Showing 4 changed files with 146 additions and 8 deletions Side-by-side Diff
fs/nfs/internal.h
fs/nfs/nfs4client.c
fs/nfs/nfs4filelayout.c
include/linux/nfs_fs_sb.h
@@ -186,6 +186,8 @@
 					     int ds_addrlen, int ds_proto,
 					     unsigned int ds_timeo,
 					     unsigned int ds_retrans);
+extern struct rpc_clnt *nfs4_find_or_create_ds_client(struct nfs_client *,
+						struct inode *);
 #ifdef CONFIG_PROC_FS
 extern int __init nfs_fs_proc_init(void);
 extern void nfs_fs_proc_exit(void);
@@ -41,9 +41,124 @@
 }
  
 #ifdef CONFIG_NFS_V4_1
+/**
+ * Per auth flavor data server rpc clients
+ */
+struct nfs4_ds_server {
+	struct list_head	list;   /* ds_clp->cl_ds_clients */
+	struct rpc_clnt		*rpc_clnt;
+};
+
+/**
+ * Common lookup case for DS I/O
+ */
+static struct nfs4_ds_server *
+nfs4_find_ds_client(struct nfs_client *ds_clp, rpc_authflavor_t flavor)
+{
+	struct nfs4_ds_server *dss;
+
+	rcu_read_lock();
+	list_for_each_entry_rcu(dss, &ds_clp->cl_ds_clients, list) {
+		if (dss->rpc_clnt->cl_auth->au_flavor != flavor)
+			continue;
+		goto out;
+	}
+	dss = NULL;
+out:
+	rcu_read_unlock();
+	return dss;
+}
+
+static struct nfs4_ds_server *
+nfs4_add_ds_client(struct nfs_client *ds_clp, rpc_authflavor_t flavor,
+			   struct nfs4_ds_server *new)
+{
+	struct nfs4_ds_server *dss;
+
+	spin_lock(&ds_clp->cl_lock);
+	list_for_each_entry(dss, &ds_clp->cl_ds_clients, list) {
+		if (dss->rpc_clnt->cl_auth->au_flavor != flavor)
+			continue;
+		goto out;
+	}
+	if (new)
+		list_add_rcu(&new->list, &ds_clp->cl_ds_clients);
+	dss = new;
+out:
+	spin_unlock(&ds_clp->cl_lock); /* need some lock to protect list */
+	return dss;
+}
+
+static struct nfs4_ds_server *
+nfs4_alloc_ds_server(struct nfs_client *ds_clp, rpc_authflavor_t flavor)
+{
+	struct nfs4_ds_server *dss;
+
+	dss = kmalloc(sizeof(*dss), GFP_NOFS);
+	if (dss == NULL)
+		return ERR_PTR(-ENOMEM);
+
+	dss->rpc_clnt = rpc_clone_client_set_auth(ds_clp->cl_rpcclient, flavor);
+	if (IS_ERR(dss->rpc_clnt)) {
+		int err = PTR_ERR(dss->rpc_clnt);
+		kfree (dss);
+		return ERR_PTR(err);
+	}
+	INIT_LIST_HEAD(&dss->list);
+
+	return dss;
+}
+
+static void
+nfs4_free_ds_server(struct nfs4_ds_server *dss)
+{
+	rpc_release_client(dss->rpc_clnt);
+	kfree(dss);
+}
+
+/**
+* Find or create a DS rpc client with th MDS server rpc client auth flavor
+* in the nfs_client cl_ds_clients list.
+*/
+struct rpc_clnt *
+nfs4_find_or_create_ds_client(struct nfs_client *ds_clp, struct inode *inode)
+{
+	struct nfs4_ds_server *dss, *new;
+	rpc_authflavor_t flavor = NFS_SERVER(inode)->client->cl_auth->au_flavor;
+
+	dss = nfs4_find_ds_client(ds_clp, flavor);
+	if (dss != NULL)
+		goto out;
+	new = nfs4_alloc_ds_server(ds_clp, flavor);
+	if (IS_ERR(new))
+		return ERR_CAST(new);
+	dss = nfs4_add_ds_client(ds_clp, flavor, new);
+	if (dss != new)
+		nfs4_free_ds_server(new);
+out:
+	return dss->rpc_clnt;
+}
+EXPORT_SYMBOL_GPL(nfs4_find_or_create_ds_client);
+
+static void
+nfs4_shutdown_ds_clients(struct nfs_client *clp)
+{
+	struct nfs4_ds_server *dss;
+	LIST_HEAD(shutdown_list);
+
+	while (!list_empty(&clp->cl_ds_clients)) {
+		dss = list_entry(clp->cl_ds_clients.next,
+					struct nfs4_ds_server, list);
+		list_del(&dss->list);
+		rpc_shutdown_client(dss->rpc_clnt);
+		kfree (dss);
+	}
+}
+
 void nfs41_shutdown_client(struct nfs_client *clp)
 {
 	if (nfs4_has_session(clp)) {
+		nfs4_shutdown_ds_clients(clp);
 		nfs4_destroy_session(clp->cl_session);
 		nfs4_destroy_clientid(clp);
 	}
@@ -77,6 +192,7 @@
  
 	spin_lock_init(&clp->cl_lock);
 	INIT_DELAYED_WORK(&clp->cl_renewd, nfs4_renew_state);
+	INIT_LIST_HEAD(&clp->cl_ds_clients);
 	rpc_init_wait_queue(&clp->cl_rpcwaitq, "NFS client");
 	clp->cl_state = 1 << NFS4CLNT_LEASE_EXPIRED;
 	clp->cl_minorversion = cl_init->minorversion;
@@ -528,6 +528,7 @@
 	struct nfs_pgio_header *hdr = data->header;
 	struct pnfs_layout_segment *lseg = hdr->lseg;
 	struct nfs4_pnfs_ds *ds;
+	struct rpc_clnt *ds_clnt;
 	loff_t offset = data->args.offset;
 	u32 j, idx;
 	struct nfs_fh *fh;
@@ -542,6 +543,11 @@
 	ds = nfs4_fl_prepare_ds(lseg, idx);
 	if (!ds)
 		return PNFS_NOT_ATTEMPTED;
+
+	ds_clnt = nfs4_find_or_create_ds_client(ds->ds_clp, hdr->inode);
+	if (IS_ERR(ds_clnt))
+		return PNFS_NOT_ATTEMPTED;
+
 	dprintk("%s USE DS: %s cl_count %d\n", __func__,
 		ds->ds_remotestr, atomic_read(&ds->ds_clp->cl_count));
  
@@ -556,7 +562,7 @@
 	data->mds_offset = offset;
  
 	/* Perform an asynchronous read to ds */
-	nfs_initiate_read(ds->ds_clp->cl_rpcclient, data,
+	nfs_initiate_read(ds_clnt, data,
 				  &filelayout_read_call_ops, RPC_TASK_SOFTCONN);
 	return PNFS_ATTEMPTED;
 }
@@ -568,6 +574,7 @@
 	struct nfs_pgio_header *hdr = data->header;
 	struct pnfs_layout_segment *lseg = hdr->lseg;
 	struct nfs4_pnfs_ds *ds;
+	struct rpc_clnt *ds_clnt;
 	loff_t offset = data->args.offset;
 	u32 j, idx;
 	struct nfs_fh *fh;
@@ -578,6 +585,11 @@
 	ds = nfs4_fl_prepare_ds(lseg, idx);
 	if (!ds)
 		return PNFS_NOT_ATTEMPTED;
+
+	ds_clnt = nfs4_find_or_create_ds_client(ds->ds_clp, hdr->inode);
+	if (IS_ERR(ds_clnt))
+		return PNFS_NOT_ATTEMPTED;
+
 	dprintk("%s ino %lu sync %d req %Zu@%llu DS: %s cl_count %d\n",
 		__func__, hdr->inode->i_ino, sync, (size_t) data->args.count,
 		offset, ds->ds_remotestr, atomic_read(&ds->ds_clp->cl_count));
@@ -595,7 +607,7 @@
 	data->args.offset = filelayout_get_dserver_offset(lseg, offset);
  
 	/* Perform an asynchronous write */
-	nfs_initiate_write(ds->ds_clp->cl_rpcclient, data,
+	nfs_initiate_write(ds_clnt, data,
 				    &filelayout_write_call_ops, sync,
 				    RPC_TASK_SOFTCONN);
 	return PNFS_ATTEMPTED;
  
@@ -1105,16 +1117,19 @@
 {
 	struct pnfs_layout_segment *lseg = data->lseg;
 	struct nfs4_pnfs_ds *ds;
+	struct rpc_clnt *ds_clnt;
 	u32 idx;
 	struct nfs_fh *fh;
  
 	idx = calc_ds_index_from_commit(lseg, data->ds_commit_index);
 	ds = nfs4_fl_prepare_ds(lseg, idx);
-	if (!ds) {
-		prepare_to_resend_writes(data);
-		filelayout_commit_release(data);
-		return -EAGAIN;
-	}
+	if (!ds)
+		goto out_err;
+
+	ds_clnt = nfs4_find_or_create_ds_client(ds->ds_clp, data->inode);
+	if (IS_ERR(ds_clnt))
+		goto out_err;
+
 	dprintk("%s ino %lu, how %d cl_count %d\n", __func__,
 		data->inode->i_ino, how, atomic_read(&ds->ds_clp->cl_count));
 	data->commit_done_cb = filelayout_commit_done_cb;
  
@@ -1123,9 +1138,13 @@
 	fh = select_ds_fh_from_commit(lseg, data->ds_commit_index);
 	if (fh)
 		data->args.fh = fh;
-	return nfs_initiate_commit(ds->ds_clp->cl_rpcclient, data,
+	return nfs_initiate_commit(ds_clnt, data,
 				   &filelayout_commit_call_ops, how,
 				   RPC_TASK_SOFTCONN);
+out_err:
+	prepare_to_resend_writes(data);
+	filelayout_commit_release(data);
+	return -EAGAIN;
 }
  
 static int
@@ -56,6 +56,7 @@
 	struct rpc_cred		*cl_machine_cred;
  
 #if IS_ENABLED(CONFIG_NFS_V4)
+	struct list_head	cl_ds_clients; /* auth flavor data servers */
 	u64			cl_clientid;	/* constant */
 	nfs4_verifier		cl_confirm;	/* Clientid verifier */
 	unsigned long		cl_state;
...	...	@@ -186,6 +186,8 @@
186	186	int ds_addrlen, int ds_proto,
187	187	unsigned int ds_timeo,
188	188	unsigned int ds_retrans);
	189	+extern struct rpc_clnt nfs4_find_or_create_ds_client(struct nfs_client ,
	190	+ struct inode *);
189	191	#ifdef CONFIG_PROC_FS
190	192	extern int __init nfs_fs_proc_init(void);
191	193	extern void nfs_fs_proc_exit(void);
...	...	@@ -41,9 +41,124 @@
41	41	}
42	42
43	43	#ifdef CONFIG_NFS_V4_1
	44	+/**
	45	+ * Per auth flavor data server rpc clients
	46	+ */
	47	+struct nfs4_ds_server {
	48	+ struct list_head list; /* ds_clp->cl_ds_clients */
	49	+ struct rpc_clnt *rpc_clnt;
	50	+};
	51	+
	52	+/**
	53	+ * Common lookup case for DS I/O
	54	+ */
	55	+static struct nfs4_ds_server *
	56	+nfs4_find_ds_client(struct nfs_client *ds_clp, rpc_authflavor_t flavor)
	57	+{
	58	+ struct nfs4_ds_server *dss;
	59	+
	60	+ rcu_read_lock();
	61	+ list_for_each_entry_rcu(dss, &ds_clp->cl_ds_clients, list) {
	62	+ if (dss->rpc_clnt->cl_auth->au_flavor != flavor)
	63	+ continue;
	64	+ goto out;
	65	+ }
	66	+ dss = NULL;
	67	+out:
	68	+ rcu_read_unlock();
	69	+ return dss;
	70	+}
	71	+
	72	+static struct nfs4_ds_server *
	73	+nfs4_add_ds_client(struct nfs_client *ds_clp, rpc_authflavor_t flavor,
	74	+ struct nfs4_ds_server *new)
	75	+{
	76	+ struct nfs4_ds_server *dss;
	77	+
	78	+ spin_lock(&ds_clp->cl_lock);
	79	+ list_for_each_entry(dss, &ds_clp->cl_ds_clients, list) {
	80	+ if (dss->rpc_clnt->cl_auth->au_flavor != flavor)
	81	+ continue;
	82	+ goto out;
	83	+ }
	84	+ if (new)
	85	+ list_add_rcu(&new->list, &ds_clp->cl_ds_clients);
	86	+ dss = new;
	87	+out:
	88	+ spin_unlock(&ds_clp->cl_lock); /* need some lock to protect list */
	89	+ return dss;
	90	+}
	91	+
	92	+static struct nfs4_ds_server *
	93	+nfs4_alloc_ds_server(struct nfs_client *ds_clp, rpc_authflavor_t flavor)
	94	+{
	95	+ struct nfs4_ds_server *dss;
	96	+
	97	+ dss = kmalloc(sizeof(*dss), GFP_NOFS);
	98	+ if (dss == NULL)
	99	+ return ERR_PTR(-ENOMEM);
	100	+
	101	+ dss->rpc_clnt = rpc_clone_client_set_auth(ds_clp->cl_rpcclient, flavor);
	102	+ if (IS_ERR(dss->rpc_clnt)) {
	103	+ int err = PTR_ERR(dss->rpc_clnt);
	104	+ kfree (dss);
	105	+ return ERR_PTR(err);
	106	+ }
	107	+ INIT_LIST_HEAD(&dss->list);
	108	+
	109	+ return dss;
	110	+}
	111	+
	112	+static void
	113	+nfs4_free_ds_server(struct nfs4_ds_server *dss)
	114	+{
	115	+ rpc_release_client(dss->rpc_clnt);
	116	+ kfree(dss);
	117	+}
	118	+
	119	+/**
	120	+* Find or create a DS rpc client with th MDS server rpc client auth flavor
	121	+* in the nfs_client cl_ds_clients list.
	122	+*/
	123	+struct rpc_clnt *
	124	+nfs4_find_or_create_ds_client(struct nfs_client ds_clp, struct inode inode)
	125	+{
	126	+ struct nfs4_ds_server dss, new;
	127	+ rpc_authflavor_t flavor = NFS_SERVER(inode)->client->cl_auth->au_flavor;
	128	+
	129	+ dss = nfs4_find_ds_client(ds_clp, flavor);
	130	+ if (dss != NULL)
	131	+ goto out;
	132	+ new = nfs4_alloc_ds_server(ds_clp, flavor);
	133	+ if (IS_ERR(new))
	134	+ return ERR_CAST(new);
	135	+ dss = nfs4_add_ds_client(ds_clp, flavor, new);
	136	+ if (dss != new)
	137	+ nfs4_free_ds_server(new);
	138	+out:
	139	+ return dss->rpc_clnt;
	140	+}
	141	+EXPORT_SYMBOL_GPL(nfs4_find_or_create_ds_client);
	142	+
	143	+static void
	144	+nfs4_shutdown_ds_clients(struct nfs_client *clp)
	145	+{
	146	+ struct nfs4_ds_server *dss;
	147	+ LIST_HEAD(shutdown_list);
	148	+
	149	+ while (!list_empty(&clp->cl_ds_clients)) {
	150	+ dss = list_entry(clp->cl_ds_clients.next,
	151	+ struct nfs4_ds_server, list);
	152	+ list_del(&dss->list);
	153	+ rpc_shutdown_client(dss->rpc_clnt);
	154	+ kfree (dss);
	155	+ }
	156	+}
	157	+
44	158	void nfs41_shutdown_client(struct nfs_client *clp)
45	159	{
46	160	if (nfs4_has_session(clp)) {
	161	+ nfs4_shutdown_ds_clients(clp);
47	162	nfs4_destroy_session(clp->cl_session);
48	163	nfs4_destroy_clientid(clp);
49	164	}
...	...	@@ -77,6 +192,7 @@
77	192
78	193	spin_lock_init(&clp->cl_lock);
79	194	INIT_DELAYED_WORK(&clp->cl_renewd, nfs4_renew_state);
	195	+ INIT_LIST_HEAD(&clp->cl_ds_clients);
80	196	rpc_init_wait_queue(&clp->cl_rpcwaitq, "NFS client");
81	197	clp->cl_state = 1 << NFS4CLNT_LEASE_EXPIRED;
82	198	clp->cl_minorversion = cl_init->minorversion;
...	...	@@ -528,6 +528,7 @@
528	528	struct nfs_pgio_header *hdr = data->header;
529	529	struct pnfs_layout_segment *lseg = hdr->lseg;
530	530	struct nfs4_pnfs_ds *ds;
	531	+ struct rpc_clnt *ds_clnt;
531	532	loff_t offset = data->args.offset;
532	533	u32 j, idx;
533	534	struct nfs_fh *fh;
...	...	@@ -542,6 +543,11 @@
542	543	ds = nfs4_fl_prepare_ds(lseg, idx);
543	544	if (!ds)
544	545	return PNFS_NOT_ATTEMPTED;
	546	+
	547	+ ds_clnt = nfs4_find_or_create_ds_client(ds->ds_clp, hdr->inode);
	548	+ if (IS_ERR(ds_clnt))
	549	+ return PNFS_NOT_ATTEMPTED;
	550	+
545	551	dprintk("%s USE DS: %s cl_count %d\n", __func__,
546	552	ds->ds_remotestr, atomic_read(&ds->ds_clp->cl_count));
547	553
...	...	@@ -556,7 +562,7 @@
556	562	data->mds_offset = offset;
557	563
558	564	/* Perform an asynchronous read to ds */
559		- nfs_initiate_read(ds->ds_clp->cl_rpcclient, data,
	565	+ nfs_initiate_read(ds_clnt, data,
560	566	&filelayout_read_call_ops, RPC_TASK_SOFTCONN);
561	567	return PNFS_ATTEMPTED;
562	568	}
...	...	@@ -568,6 +574,7 @@
568	574	struct nfs_pgio_header *hdr = data->header;
569	575	struct pnfs_layout_segment *lseg = hdr->lseg;
570	576	struct nfs4_pnfs_ds *ds;
	577	+ struct rpc_clnt *ds_clnt;
571	578	loff_t offset = data->args.offset;
572	579	u32 j, idx;
573	580	struct nfs_fh *fh;
...	...	@@ -578,6 +585,11 @@
578	585	ds = nfs4_fl_prepare_ds(lseg, idx);
579	586	if (!ds)
580	587	return PNFS_NOT_ATTEMPTED;
	588	+
	589	+ ds_clnt = nfs4_find_or_create_ds_client(ds->ds_clp, hdr->inode);
	590	+ if (IS_ERR(ds_clnt))
	591	+ return PNFS_NOT_ATTEMPTED;
	592	+
581	593	dprintk("%s ino %lu sync %d req %Zu@%llu DS: %s cl_count %d\n",
582	594	__func__, hdr->inode->i_ino, sync, (size_t) data->args.count,
583	595	offset, ds->ds_remotestr, atomic_read(&ds->ds_clp->cl_count));
...	...	@@ -595,7 +607,7 @@
595	607	data->args.offset = filelayout_get_dserver_offset(lseg, offset);
596	608
597	609	/* Perform an asynchronous write */
598		- nfs_initiate_write(ds->ds_clp->cl_rpcclient, data,
	610	+ nfs_initiate_write(ds_clnt, data,
599	611	&filelayout_write_call_ops, sync,
600	612	RPC_TASK_SOFTCONN);
601	613	return PNFS_ATTEMPTED;
602	614
...	...	@@ -1105,16 +1117,19 @@
1105	1117	{
1106	1118	struct pnfs_layout_segment *lseg = data->lseg;
1107	1119	struct nfs4_pnfs_ds *ds;
	1120	+ struct rpc_clnt *ds_clnt;
1108	1121	u32 idx;
1109	1122	struct nfs_fh *fh;
1110	1123
1111	1124	idx = calc_ds_index_from_commit(lseg, data->ds_commit_index);
1112	1125	ds = nfs4_fl_prepare_ds(lseg, idx);
1113		- if (!ds) {
1114		- prepare_to_resend_writes(data);
1115		- filelayout_commit_release(data);
1116		- return -EAGAIN;
1117		- }
	1126	+ if (!ds)
	1127	+ goto out_err;
	1128	+
	1129	+ ds_clnt = nfs4_find_or_create_ds_client(ds->ds_clp, data->inode);
	1130	+ if (IS_ERR(ds_clnt))
	1131	+ goto out_err;
	1132	+
1118	1133	dprintk("%s ino %lu, how %d cl_count %d\n", __func__,
1119	1134	data->inode->i_ino, how, atomic_read(&ds->ds_clp->cl_count));
1120	1135	data->commit_done_cb = filelayout_commit_done_cb;
1121	1136
...	...	@@ -1123,9 +1138,13 @@
1123	1138	fh = select_ds_fh_from_commit(lseg, data->ds_commit_index);
1124	1139	if (fh)
1125	1140	data->args.fh = fh;
1126		- return nfs_initiate_commit(ds->ds_clp->cl_rpcclient, data,
	1141	+ return nfs_initiate_commit(ds_clnt, data,
1127	1142	&filelayout_commit_call_ops, how,
1128	1143	RPC_TASK_SOFTCONN);
	1144	+out_err:
	1145	+ prepare_to_resend_writes(data);
	1146	+ filelayout_commit_release(data);
	1147	+ return -EAGAIN;
1129	1148	}
1130	1149
1131	1150	static int
...	...	@@ -56,6 +56,7 @@
56	56	struct rpc_cred *cl_machine_cred;
57	57
58	58	#if IS_ENABLED(CONFIG_NFS_V4)
	59	+ struct list_head cl_ds_clients; /* auth flavor data servers */
59	60	u64 cl_clientid; /* constant */
60	61	nfs4_verifier cl_confirm; /* Clientid verifier */
61	62	unsigned long cl_state;