NFSv4.1: do not use deviceids after MDS clientid invalidation

Mark all deviceids established under an expired MDS clientid as invalid. Stop all new i/o through DS and send through the MDS. Don't use any new LAYOUTGETs that use the invalid deviceid. Purge all layouts established under the expired MDS clientid. Remove the MDS clientid deviceid and data servers reference Signed-off-by: Andy Adamson <andros@netapp.com> Signed-off-by: Trond Myklebust <Trond.Myklebust@netapp.com>

NFSv4.1: do not use deviceids after MDS clientid invalidation
Mark all deviceids established under an expired MDS clientid as invalid. Stop all new i/o through DS and send through the MDS. Don't use any new LAYOUTGETs that use the invalid deviceid. Purge all layouts established under the expired MDS clientid. Remove the MDS clientid deviceid and data servers reference Signed-off-by: Andy Adamson <andros@netapp.com> Signed-off-by: Trond Myklebust <Trond.Myklebust@netapp.com>
Andy Adamson · Trond Myklebust
1 parent a56aaa02b1
Showing 5 changed files with 47 additions and 0 deletions Side-by-side Diff
fs/nfs/nfs4filelayout.c
fs/nfs/nfs4filelayout.h
fs/nfs/pnfs.c
fs/nfs/pnfs.h
fs/nfs/pnfs_dev.c
@@ -334,6 +334,9 @@
 		__func__, data->inode->i_ino,
 		data->args.pgbase, (size_t)data->args.count, offset);
  
+	if (test_bit(NFS_DEVICEID_INVALID, &FILELAYOUT_DEVID_NODE(lseg)->flags))
+		return PNFS_NOT_ATTEMPTED;
+
 	/* Retrieve the correct rpc_client for the byte range */
 	j = nfs4_fl_calc_j_index(lseg, offset);
 	idx = nfs4_fl_calc_ds_index(lseg, j);
@@ -373,6 +376,9 @@
 	struct nfs_fh *fh;
 	int status;
  
+	if (test_bit(NFS_DEVICEID_INVALID, &FILELAYOUT_DEVID_NODE(lseg)->flags))
+		return PNFS_NOT_ATTEMPTED;
+
 	/* Retrieve the correct rpc_client for the byte range */
 	j = nfs4_fl_calc_j_index(lseg, offset);
 	idx = nfs4_fl_calc_ds_index(lseg, j);
@@ -456,6 +462,10 @@
 			goto out;
 	} else
 		dsaddr = container_of(d, struct nfs4_file_layout_dsaddr, id_node);
+	/* Found deviceid is being reaped */
+	if (test_bit(NFS_DEVICEID_INVALID, &dsaddr->id_node.flags))
+			goto out_put;
+
 	fl->dsaddr = dsaddr;
  
 	if (fl->first_stripe_index < 0 ||
@@ -96,6 +96,12 @@
 			    generic_hdr);
 }
  
+static inline struct nfs4_deviceid_node *
+FILELAYOUT_DEVID_NODE(struct pnfs_layout_segment *lseg)
+{
+	return &FILELAYOUT_LSEG(lseg)->dsaddr->id_node;
+}
+
 extern struct nfs_fh *
 nfs4_fl_select_ds_fh(struct pnfs_layout_segment *lseg, u32 j);
  
@@ -452,6 +452,9 @@
 	struct pnfs_layout_hdr *lo;
 	LIST_HEAD(tmp_list);
  
+	nfs4_deviceid_mark_client_invalid(clp);
+	nfs4_deviceid_purge_client(clp);
+
 	spin_lock(&clp->cl_lock);
 	rcu_read_lock();
 	list_for_each_entry_rcu(server, &clp->cl_superblocks, client_link) {
@@ -192,12 +192,20 @@
 					       enum pnfs_iomode iomode,
 					       gfp_t gfp_flags);
  
+void nfs4_deviceid_mark_client_invalid(struct nfs_client *clp);
+
+/* nfs4_deviceid_flags */
+enum {
+	NFS_DEVICEID_INVALID = 0,       /* set when MDS clientid recalled */
+};
+
 /* pnfs_dev.c */
 struct nfs4_deviceid_node {
 	struct hlist_node		node;
 	struct hlist_node		tmpnode;
 	const struct pnfs_layoutdriver_type *ld;
 	const struct nfs_client		*nfs_client;
+	unsigned long 			flags;
 	struct nfs4_deviceid		deviceid;
 	atomic_t			ref;
 };
@@ -156,6 +156,7 @@
 	INIT_HLIST_NODE(&d->tmpnode);
 	d->ld = ld;
 	d->nfs_client = nfs_client;
+	d->flags = 0;
 	d->deviceid = *id;
 	atomic_set(&d->ref, 1);
 }
@@ -252,5 +253,24 @@
 		return;
 	for (h = 0; h < NFS4_DEVICE_ID_HASH_SIZE; h++)
 		_deviceid_purge_client(clp, h);
+}
+
+/*
+ * Stop use of all deviceids associated with an nfs_client
+ */
+void
+nfs4_deviceid_mark_client_invalid(struct nfs_client *clp)
+{
+	struct nfs4_deviceid_node *d;
+	struct hlist_node *n;
+	int i;
+
+	rcu_read_lock();
+	for (i = 0; i < NFS4_DEVICE_ID_HASH_SIZE; i ++){
+		hlist_for_each_entry_rcu(d, n, &nfs4_deviceid_cache[i], node)
+			if (d->nfs_client == clp)
+				set_bit(NFS_DEVICEID_INVALID, &d->flags);
+	}
+	rcu_read_unlock();
 }
...	...	@@ -334,6 +334,9 @@
334	334	__func__, data->inode->i_ino,
335	335	data->args.pgbase, (size_t)data->args.count, offset);
336	336
	337	+ if (test_bit(NFS_DEVICEID_INVALID, &FILELAYOUT_DEVID_NODE(lseg)->flags))
	338	+ return PNFS_NOT_ATTEMPTED;
	339	+
337	340	/* Retrieve the correct rpc_client for the byte range */
338	341	j = nfs4_fl_calc_j_index(lseg, offset);
339	342	idx = nfs4_fl_calc_ds_index(lseg, j);
...	...	@@ -373,6 +376,9 @@
373	376	struct nfs_fh *fh;
374	377	int status;
375	378
	379	+ if (test_bit(NFS_DEVICEID_INVALID, &FILELAYOUT_DEVID_NODE(lseg)->flags))
	380	+ return PNFS_NOT_ATTEMPTED;
	381	+
376	382	/* Retrieve the correct rpc_client for the byte range */
377	383	j = nfs4_fl_calc_j_index(lseg, offset);
378	384	idx = nfs4_fl_calc_ds_index(lseg, j);
...	...	@@ -456,6 +462,10 @@
456	462	goto out;
457	463	} else
458	464	dsaddr = container_of(d, struct nfs4_file_layout_dsaddr, id_node);
	465	+ /* Found deviceid is being reaped */
	466	+ if (test_bit(NFS_DEVICEID_INVALID, &dsaddr->id_node.flags))
	467	+ goto out_put;
	468	+
459	469	fl->dsaddr = dsaddr;
460	470
461	471	if (fl->first_stripe_index < 0 \|\|
...	...	@@ -96,6 +96,12 @@
96	96	generic_hdr);
97	97	}
98	98
	99	+static inline struct nfs4_deviceid_node *
	100	+FILELAYOUT_DEVID_NODE(struct pnfs_layout_segment *lseg)
	101	+{
	102	+ return &FILELAYOUT_LSEG(lseg)->dsaddr->id_node;
	103	+}
	104	+
99	105	extern struct nfs_fh *
100	106	nfs4_fl_select_ds_fh(struct pnfs_layout_segment *lseg, u32 j);
101	107
...	...	@@ -452,6 +452,9 @@
452	452	struct pnfs_layout_hdr *lo;
453	453	LIST_HEAD(tmp_list);
454	454
	455	+ nfs4_deviceid_mark_client_invalid(clp);
	456	+ nfs4_deviceid_purge_client(clp);
	457	+
455	458	spin_lock(&clp->cl_lock);
456	459	rcu_read_lock();
457	460	list_for_each_entry_rcu(server, &clp->cl_superblocks, client_link) {
...	...	@@ -192,12 +192,20 @@
192	192	enum pnfs_iomode iomode,
193	193	gfp_t gfp_flags);
194	194
	195	+void nfs4_deviceid_mark_client_invalid(struct nfs_client *clp);
	196	+
	197	+/* nfs4_deviceid_flags */
	198	+enum {
	199	+ NFS_DEVICEID_INVALID = 0, /* set when MDS clientid recalled */
	200	+};
	201	+
195	202	/* pnfs_dev.c */
196	203	struct nfs4_deviceid_node {
197	204	struct hlist_node node;
198	205	struct hlist_node tmpnode;
199	206	const struct pnfs_layoutdriver_type *ld;
200	207	const struct nfs_client *nfs_client;
	208	+ unsigned long flags;
201	209	struct nfs4_deviceid deviceid;
202	210	atomic_t ref;
203	211	};
...	...	@@ -156,6 +156,7 @@
156	156	INIT_HLIST_NODE(&d->tmpnode);
157	157	d->ld = ld;
158	158	d->nfs_client = nfs_client;
	159	+ d->flags = 0;
159	160	d->deviceid = *id;
160	161	atomic_set(&d->ref, 1);
161	162	}
...	...	@@ -252,5 +253,24 @@
252	253	return;
253	254	for (h = 0; h < NFS4_DEVICE_ID_HASH_SIZE; h++)
254	255	_deviceid_purge_client(clp, h);
	256	+}
	257	+
	258	+/*
	259	+ * Stop use of all deviceids associated with an nfs_client
	260	+ */
	261	+void
	262	+nfs4_deviceid_mark_client_invalid(struct nfs_client *clp)
	263	+{
	264	+ struct nfs4_deviceid_node *d;
	265	+ struct hlist_node *n;
	266	+ int i;
	267	+
	268	+ rcu_read_lock();
	269	+ for (i = 0; i < NFS4_DEVICE_ID_HASH_SIZE; i ++){
	270	+ hlist_for_each_entry_rcu(d, n, &nfs4_deviceid_cache[i], node)
	271	+ if (d->nfs_client == clp)
	272	+ set_bit(NFS_DEVICEID_INVALID, &d->flags);
	273	+ }
	274	+ rcu_read_unlock();
255	275	}