pnfsblock: call and parse getdevicelist

Call GETDEVICELIST during mount, then call and parse GETDEVICEINFO for each device returned. [pnfsblock: get rid of deprecated xdr macros] Signed-off-by: Jim Rees <rees@umich.edu> [pnfsblock: fix pnfs_deviceid references] Signed-off-by: Fred Isaman <iisaman@citi.umich.edu> [pnfsblock: fix print format warnings for sector_t and size_t] [pnfs-block: #include <linux/vmalloc.h>] [pnfsblock: no PNFS_NFS_SERVER] Signed-off-by: Benny Halevy <bhalevy@panasas.com> [pnfsblock: fix bug determining size of striped volume] [pnfsblock: fix oops when using multiple devices] Signed-off-by: Fred Isaman <iisaman@citi.umich.edu> Signed-off-by: Benny Halevy <bhalevy@panasas.com> Signed-off-by: Benny Halevy <bhalevy@tonian.com> [pnfsblock: get rid of vmap and deviceid->area structure] Signed-off-by: Peng Tao <peng_tao@emc.com> Signed-off-by: Jim Rees <rees@umich.edu> Signed-off-by: Trond Myklebust <Trond.Myklebust@netapp.com>

pnfsblock: call and parse getdevicelist
Call GETDEVICELIST during mount, then call and parse GETDEVICEINFO for each device returned. [pnfsblock: get rid of deprecated xdr macros] Signed-off-by: Jim Rees <rees@umich.edu> [pnfsblock: fix pnfs_deviceid references] Signed-off-by: Fred Isaman <iisaman@citi.umich.edu> [pnfsblock: fix print format warnings for sector_t and size_t] [pnfs-block: #include <linux/vmalloc.h>] [pnfsblock: no PNFS_NFS_SERVER] Signed-off-by: Benny Halevy <bhalevy@panasas.com> [pnfsblock: fix bug determining size of striped volume] [pnfsblock: fix oops when using multiple devices] Signed-off-by: Fred Isaman <iisaman@citi.umich.edu> Signed-off-by: Benny Halevy <bhalevy@panasas.com> Signed-off-by: Benny Halevy <bhalevy@tonian.com> [pnfsblock: get rid of vmap and deviceid->area structure] Signed-off-by: Peng Tao <peng_tao@emc.com> Signed-off-by: Jim Rees <rees@umich.edu> Signed-off-by: Trond Myklebust <Trond.Myklebust@netapp.com>
Fred Isaman · Trond Myklebust
1 parent 03341d2cc9
Showing 5 changed files with 158 additions and 8 deletions Side-by-side Diff
fs/nfs/blocklayout/blocklayout.c
fs/nfs/blocklayout/blocklayout.h
fs/nfs/blocklayout/blocklayoutdev.c
fs/nfs/pnfs.h
include/linux/nfs_fs_sb.h
@@ -157,17 +157,153 @@
 {
 }
  
+static void free_blk_mountid(struct block_mount_id *mid)
+{
+	if (mid) {
+		struct pnfs_block_dev *dev;
+		spin_lock(&mid->bm_lock);
+		while (!list_empty(&mid->bm_devlist)) {
+			dev = list_first_entry(&mid->bm_devlist,
+					       struct pnfs_block_dev,
+					       bm_node);
+			list_del(&dev->bm_node);
+			bl_free_block_dev(dev);
+		}
+		spin_unlock(&mid->bm_lock);
+		kfree(mid);
+	}
+}
+
+/* This is mostly copied from the filelayout's get_device_info function.
+ * It seems much of this should be at the generic pnfs level.
+ */
+static struct pnfs_block_dev *
+nfs4_blk_get_deviceinfo(struct nfs_server *server, const struct nfs_fh *fh,
+			struct nfs4_deviceid *d_id)
+{
+	struct pnfs_device *dev;
+	struct pnfs_block_dev *rv = NULL;
+	u32 max_resp_sz;
+	int max_pages;
+	struct page **pages = NULL;
+	int i, rc;
+
+	/*
+	 * Use the session max response size as the basis for setting
+	 * GETDEVICEINFO's maxcount
+	 */
+	max_resp_sz = server->nfs_client->cl_session->fc_attrs.max_resp_sz;
+	max_pages = max_resp_sz >> PAGE_SHIFT;
+	dprintk("%s max_resp_sz %u max_pages %d\n",
+		__func__, max_resp_sz, max_pages);
+
+	dev = kmalloc(sizeof(*dev), GFP_NOFS);
+	if (!dev) {
+		dprintk("%s kmalloc failed\n", __func__);
+		return NULL;
+	}
+
+	pages = kzalloc(max_pages * sizeof(struct page *), GFP_NOFS);
+	if (pages == NULL) {
+		kfree(dev);
+		return NULL;
+	}
+	for (i = 0; i < max_pages; i++) {
+		pages[i] = alloc_page(GFP_NOFS);
+		if (!pages[i])
+			goto out_free;
+	}
+
+	memcpy(&dev->dev_id, d_id, sizeof(*d_id));
+	dev->layout_type = LAYOUT_BLOCK_VOLUME;
+	dev->pages = pages;
+	dev->pgbase = 0;
+	dev->pglen = PAGE_SIZE * max_pages;
+	dev->mincount = 0;
+
+	dprintk("%s: dev_id: %s\n", __func__, dev->dev_id.data);
+	rc = nfs4_proc_getdeviceinfo(server, dev);
+	dprintk("%s getdevice info returns %d\n", __func__, rc);
+	if (rc)
+		goto out_free;
+
+	rv = nfs4_blk_decode_device(server, dev);
+ out_free:
+	for (i = 0; i < max_pages; i++)
+		__free_page(pages[i]);
+	kfree(pages);
+	kfree(dev);
+	return rv;
+}
+
 static int
 bl_set_layoutdriver(struct nfs_server *server, const struct nfs_fh *fh)
 {
+	struct block_mount_id *b_mt_id = NULL;
+	struct pnfs_devicelist *dlist = NULL;
+	struct pnfs_block_dev *bdev;
+	LIST_HEAD(block_disklist);
+	int status = 0, i;
+
 	dprintk("%s enter\n", __func__);
-	return 0;
+
+	if (server->pnfs_blksize == 0) {
+		dprintk("%s Server did not return blksize\n", __func__);
+		return -EINVAL;
+	}
+	b_mt_id = kzalloc(sizeof(struct block_mount_id), GFP_NOFS);
+	if (!b_mt_id) {
+		status = -ENOMEM;
+		goto out_error;
+	}
+	/* Initialize nfs4 block layout mount id */
+	spin_lock_init(&b_mt_id->bm_lock);
+	INIT_LIST_HEAD(&b_mt_id->bm_devlist);
+
+	dlist = kmalloc(sizeof(struct pnfs_devicelist), GFP_NOFS);
+	if (!dlist) {
+		status = -ENOMEM;
+		goto out_error;
+	}
+	dlist->eof = 0;
+	while (!dlist->eof) {
+		status = nfs4_proc_getdevicelist(server, fh, dlist);
+		if (status)
+			goto out_error;
+		dprintk("%s GETDEVICELIST numdevs=%i, eof=%i\n",
+			__func__, dlist->num_devs, dlist->eof);
+		for (i = 0; i < dlist->num_devs; i++) {
+			bdev = nfs4_blk_get_deviceinfo(server, fh,
+						       &dlist->dev_id[i]);
+			if (!bdev) {
+				status = -ENODEV;
+				goto out_error;
+			}
+			spin_lock(&b_mt_id->bm_lock);
+			list_add(&bdev->bm_node, &b_mt_id->bm_devlist);
+			spin_unlock(&b_mt_id->bm_lock);
+		}
+	}
+	dprintk("%s SUCCESS\n", __func__);
+	server->pnfs_ld_data = b_mt_id;
+
+ out_return:
+	kfree(dlist);
+	return status;
+
+ out_error:
+	free_blk_mountid(b_mt_id);
+	goto out_return;
 }
  
 static int
 bl_clear_layoutdriver(struct nfs_server *server)
 {
+	struct block_mount_id *b_mt_id = server->pnfs_ld_data;
+
 	dprintk("%s enter\n", __func__);
+	free_blk_mountid(b_mt_id);
+	dprintk("%s RETURNS\n", __func__);
 	return 0;
 }
  
@@ -38,6 +38,11 @@
  
 #include "../pnfs.h"
  
+struct block_mount_id {
+	spinlock_t			bm_lock;    /* protects list */
+	struct list_head		bm_devlist; /* holds pnfs_block_dev */
+};
+
 struct pnfs_block_dev {
 	struct list_head		bm_node;
 	struct nfs4_deviceid		bm_mdevid;    /* associated devid */
@@ -99,7 +104,10 @@
 	sector_t		bl_blocksize;  /* Server blocksize in sectors */
 };
  
-static inline struct pnfs_block_layout *BLK_LO2EXT(struct pnfs_layout_hdr *lo)
+#define BLK_ID(lo) ((struct block_mount_id *)(NFS_SERVER(lo->plh_inode)->pnfs_ld_data))
+
+static inline struct pnfs_block_layout *
+BLK_LO2EXT(struct pnfs_layout_hdr *lo)
 {
 	return container_of(lo, struct pnfs_block_layout, bl_layout);
 }
@@ -137,8 +145,7 @@
 struct block_device *nfs4_blkdev_get(dev_t dev);
 int nfs4_blkdev_put(struct block_device *bdev);
 struct pnfs_block_dev *nfs4_blk_decode_device(struct nfs_server *server,
-						struct pnfs_device *dev,
-						struct list_head *sdlist);
+						struct pnfs_device *dev);
 int nfs4_blk_process_layoutget(struct pnfs_layout_hdr *lo,
 				struct nfs4_layoutget_res *lgr, gfp_t gfp_flags);
  
@@ -116,8 +116,7 @@
  */
 struct pnfs_block_dev *
 nfs4_blk_decode_device(struct nfs_server *server,
-		       struct pnfs_device *dev,
-		       struct list_head *sdlist)
+		       struct pnfs_device *dev)
 {
 	struct pnfs_block_dev *rv = NULL;
 	struct block_device *bd = NULL;
@@ -129,6 +128,7 @@
 	uint8_t *dataptr;
 	DECLARE_WAITQUEUE(wq, current);
 	struct bl_dev_msg *reply = &bl_mount_reply;
+	int offset, len, i;
  
 	dprintk("%s CREATING PIPEFS MESSAGE\n", __func__);
 	dprintk("%s: deviceid: %s, mincount: %d\n", __func__, dev->dev_id.data,
@@ -143,7 +143,14 @@
  
 	memcpy(msg.data, &bl_msg, sizeof(bl_msg));
 	dataptr = (uint8_t *) msg.data;
-	memcpy(&dataptr[sizeof(bl_msg)], dev->area, dev->mincount);
+	len = dev->mincount;
+	offset = sizeof(bl_msg);
+	for (i = 0; len > 0; i++) {
+		memcpy(&dataptr[offset], page_address(dev->pages[i]),
+				len < PAGE_CACHE_SIZE ? len : PAGE_CACHE_SIZE);
+		len -= PAGE_CACHE_SIZE;
+		offset += PAGE_CACHE_SIZE;
+	}
 	msg.len = sizeof(bl_msg) + dev->mincount;
  
 	dprintk("%s CALLING USERSPACE DAEMON\n", __func__);
@@ -140,7 +140,6 @@
 	unsigned int  layout_type;
 	unsigned int  mincount;
 	struct page **pages;
-	void          *area;
 	unsigned int  pgbase;
 	unsigned int  pglen;
 };
@@ -146,6 +146,7 @@
 	struct pnfs_layoutdriver_type  *pnfs_curr_ld; /* Active layout driver */
 	struct rpc_wait_queue	roc_rpcwaitq;
 	u32			pnfs_blksize;	/* layout_blksize attr */
+	void			*pnfs_ld_data;	/* per mount point data */
  
 	/* the following fields are protected by nfs_client->cl_lock */
 	struct rb_root		state_owners;
...	...	@@ -157,17 +157,153 @@
157	157	{
158	158	}
159	159
	160	+static void free_blk_mountid(struct block_mount_id *mid)
	161	+{
	162	+ if (mid) {
	163	+ struct pnfs_block_dev *dev;
	164	+ spin_lock(&mid->bm_lock);
	165	+ while (!list_empty(&mid->bm_devlist)) {
	166	+ dev = list_first_entry(&mid->bm_devlist,
	167	+ struct pnfs_block_dev,
	168	+ bm_node);
	169	+ list_del(&dev->bm_node);
	170	+ bl_free_block_dev(dev);
	171	+ }
	172	+ spin_unlock(&mid->bm_lock);
	173	+ kfree(mid);
	174	+ }
	175	+}
	176	+
	177	+/* This is mostly copied from the filelayout's get_device_info function.
	178	+ * It seems much of this should be at the generic pnfs level.
	179	+ */
	180	+static struct pnfs_block_dev *
	181	+nfs4_blk_get_deviceinfo(struct nfs_server server, const struct nfs_fh fh,
	182	+ struct nfs4_deviceid *d_id)
	183	+{
	184	+ struct pnfs_device *dev;
	185	+ struct pnfs_block_dev *rv = NULL;
	186	+ u32 max_resp_sz;
	187	+ int max_pages;
	188	+ struct page **pages = NULL;
	189	+ int i, rc;
	190	+
	191	+ /*
	192	+ * Use the session max response size as the basis for setting
	193	+ * GETDEVICEINFO's maxcount
	194	+ */
	195	+ max_resp_sz = server->nfs_client->cl_session->fc_attrs.max_resp_sz;
	196	+ max_pages = max_resp_sz >> PAGE_SHIFT;
	197	+ dprintk("%s max_resp_sz %u max_pages %d\n",
	198	+ __func__, max_resp_sz, max_pages);
	199	+
	200	+ dev = kmalloc(sizeof(*dev), GFP_NOFS);
	201	+ if (!dev) {
	202	+ dprintk("%s kmalloc failed\n", __func__);
	203	+ return NULL;
	204	+ }
	205	+
	206	+ pages = kzalloc(max_pages * sizeof(struct page *), GFP_NOFS);
	207	+ if (pages == NULL) {
	208	+ kfree(dev);
	209	+ return NULL;
	210	+ }
	211	+ for (i = 0; i < max_pages; i++) {
	212	+ pages[i] = alloc_page(GFP_NOFS);
	213	+ if (!pages[i])
	214	+ goto out_free;
	215	+ }
	216	+
	217	+ memcpy(&dev->dev_id, d_id, sizeof(*d_id));
	218	+ dev->layout_type = LAYOUT_BLOCK_VOLUME;
	219	+ dev->pages = pages;
	220	+ dev->pgbase = 0;
	221	+ dev->pglen = PAGE_SIZE * max_pages;
	222	+ dev->mincount = 0;
	223	+
	224	+ dprintk("%s: dev_id: %s\n", __func__, dev->dev_id.data);
	225	+ rc = nfs4_proc_getdeviceinfo(server, dev);
	226	+ dprintk("%s getdevice info returns %d\n", __func__, rc);
	227	+ if (rc)
	228	+ goto out_free;
	229	+
	230	+ rv = nfs4_blk_decode_device(server, dev);
	231	+ out_free:
	232	+ for (i = 0; i < max_pages; i++)
	233	+ __free_page(pages[i]);
	234	+ kfree(pages);
	235	+ kfree(dev);
	236	+ return rv;
	237	+}
	238	+
160	239	static int
161	240	bl_set_layoutdriver(struct nfs_server server, const struct nfs_fh fh)
162	241	{
	242	+ struct block_mount_id *b_mt_id = NULL;
	243	+ struct pnfs_devicelist *dlist = NULL;
	244	+ struct pnfs_block_dev *bdev;
	245	+ LIST_HEAD(block_disklist);
	246	+ int status = 0, i;
	247	+
163	248	dprintk("%s enter\n", __func__);
164		- return 0;
	249	+
	250	+ if (server->pnfs_blksize == 0) {
	251	+ dprintk("%s Server did not return blksize\n", __func__);
	252	+ return -EINVAL;
	253	+ }
	254	+ b_mt_id = kzalloc(sizeof(struct block_mount_id), GFP_NOFS);
	255	+ if (!b_mt_id) {
	256	+ status = -ENOMEM;
	257	+ goto out_error;
	258	+ }
	259	+ /* Initialize nfs4 block layout mount id */
	260	+ spin_lock_init(&b_mt_id->bm_lock);
	261	+ INIT_LIST_HEAD(&b_mt_id->bm_devlist);
	262	+
	263	+ dlist = kmalloc(sizeof(struct pnfs_devicelist), GFP_NOFS);
	264	+ if (!dlist) {
	265	+ status = -ENOMEM;
	266	+ goto out_error;
	267	+ }
	268	+ dlist->eof = 0;
	269	+ while (!dlist->eof) {
	270	+ status = nfs4_proc_getdevicelist(server, fh, dlist);
	271	+ if (status)
	272	+ goto out_error;
	273	+ dprintk("%s GETDEVICELIST numdevs=%i, eof=%i\n",
	274	+ __func__, dlist->num_devs, dlist->eof);
	275	+ for (i = 0; i < dlist->num_devs; i++) {
	276	+ bdev = nfs4_blk_get_deviceinfo(server, fh,
	277	+ &dlist->dev_id[i]);
	278	+ if (!bdev) {
	279	+ status = -ENODEV;
	280	+ goto out_error;
	281	+ }
	282	+ spin_lock(&b_mt_id->bm_lock);
	283	+ list_add(&bdev->bm_node, &b_mt_id->bm_devlist);
	284	+ spin_unlock(&b_mt_id->bm_lock);
	285	+ }
	286	+ }
	287	+ dprintk("%s SUCCESS\n", __func__);
	288	+ server->pnfs_ld_data = b_mt_id;
	289	+
	290	+ out_return:
	291	+ kfree(dlist);
	292	+ return status;
	293	+
	294	+ out_error:
	295	+ free_blk_mountid(b_mt_id);
	296	+ goto out_return;
165	297	}
166	298
167	299	static int
168	300	bl_clear_layoutdriver(struct nfs_server *server)
169	301	{
	302	+ struct block_mount_id *b_mt_id = server->pnfs_ld_data;
	303	+
170	304	dprintk("%s enter\n", __func__);
	305	+ free_blk_mountid(b_mt_id);
	306	+ dprintk("%s RETURNS\n", __func__);
171	307	return 0;
172	308	}
173	309
...	...	@@ -38,6 +38,11 @@
38	38
39	39	#include "../pnfs.h"
40	40
	41	+struct block_mount_id {
	42	+ spinlock_t bm_lock; /* protects list */
	43	+ struct list_head bm_devlist; /* holds pnfs_block_dev */
	44	+};
	45	+
41	46	struct pnfs_block_dev {
42	47	struct list_head bm_node;
43	48	struct nfs4_deviceid bm_mdevid; /* associated devid */
...	...	@@ -99,7 +104,10 @@
99	104	sector_t bl_blocksize; /* Server blocksize in sectors */
100	105	};
101	106
102		-static inline struct pnfs_block_layout BLK_LO2EXT(struct pnfs_layout_hdr lo)
	107	+#define BLK_ID(lo) ((struct block_mount_id *)(NFS_SERVER(lo->plh_inode)->pnfs_ld_data))
	108	+
	109	+static inline struct pnfs_block_layout *
	110	+BLK_LO2EXT(struct pnfs_layout_hdr *lo)
103	111	{
104	112	return container_of(lo, struct pnfs_block_layout, bl_layout);
105	113	}
...	...	@@ -137,8 +145,7 @@
137	145	struct block_device *nfs4_blkdev_get(dev_t dev);
138	146	int nfs4_blkdev_put(struct block_device *bdev);
139	147	struct pnfs_block_dev nfs4_blk_decode_device(struct nfs_server server,
140		- struct pnfs_device *dev,
141		- struct list_head *sdlist);
	148	+ struct pnfs_device *dev);
142	149	int nfs4_blk_process_layoutget(struct pnfs_layout_hdr *lo,
143	150	struct nfs4_layoutget_res *lgr, gfp_t gfp_flags);
144	151
...	...	@@ -116,8 +116,7 @@
116	116	*/
117	117	struct pnfs_block_dev *
118	118	nfs4_blk_decode_device(struct nfs_server *server,
119		- struct pnfs_device *dev,
120		- struct list_head *sdlist)
	119	+ struct pnfs_device *dev)
121	120	{
122	121	struct pnfs_block_dev *rv = NULL;
123	122	struct block_device *bd = NULL;
...	...	@@ -129,6 +128,7 @@
129	128	uint8_t *dataptr;
130	129	DECLARE_WAITQUEUE(wq, current);
131	130	struct bl_dev_msg *reply = &bl_mount_reply;
	131	+ int offset, len, i;
132	132
133	133	dprintk("%s CREATING PIPEFS MESSAGE\n", __func__);
134	134	dprintk("%s: deviceid: %s, mincount: %d\n", __func__, dev->dev_id.data,
...	...	@@ -143,7 +143,14 @@
143	143
144	144	memcpy(msg.data, &bl_msg, sizeof(bl_msg));
145	145	dataptr = (uint8_t *) msg.data;
146		- memcpy(&dataptr[sizeof(bl_msg)], dev->area, dev->mincount);
	146	+ len = dev->mincount;
	147	+ offset = sizeof(bl_msg);
	148	+ for (i = 0; len > 0; i++) {
	149	+ memcpy(&dataptr[offset], page_address(dev->pages[i]),
	150	+ len < PAGE_CACHE_SIZE ? len : PAGE_CACHE_SIZE);
	151	+ len -= PAGE_CACHE_SIZE;
	152	+ offset += PAGE_CACHE_SIZE;
	153	+ }
147	154	msg.len = sizeof(bl_msg) + dev->mincount;
148	155
149	156	dprintk("%s CALLING USERSPACE DAEMON\n", __func__);
...	...	@@ -140,7 +140,6 @@
140	140	unsigned int layout_type;
141	141	unsigned int mincount;
142	142	struct page **pages;
143		- void *area;
144	143	unsigned int pgbase;
145	144	unsigned int pglen;
146	145	};
...	...	@@ -146,6 +146,7 @@
146	146	struct pnfs_layoutdriver_type pnfs_curr_ld; / Active layout driver */
147	147	struct rpc_wait_queue roc_rpcwaitq;
148	148	u32 pnfs_blksize; /* layout_blksize attr */
	149	+ void pnfs_ld_data; / per mount point data */
149	150
150	151	/* the following fields are protected by nfs_client->cl_lock */
151	152	struct rb_root state_owners;