tcm_vhost: Optimize gup in vhost_scsi_map_to_sgl

We can get all the pages in one time instead of calling gup N times. Signed-off-by: Asias He <asias@redhat.com> Signed-off-by: Nicholas Bellinger <nab@linux-iscsi.org>

tcm_vhost: Optimize gup in vhost_scsi_map_to_sgl
We can get all the pages in one time instead of calling gup N times. Signed-off-by: Asias He <asias@redhat.com> Signed-off-by: Nicholas Bellinger <nab@linux-iscsi.org>
Asias He · Nicholas Bellinger
1 parent f3158f362c
Showing 1 changed file with 30 additions and 24 deletions Side-by-side Diff
drivers/vhost/tcm_vhost.c
@@ -430,40 +430,47 @@
  * Returns the number of scatterlist entries used or -errno on error.
  */
 static int vhost_scsi_map_to_sgl(struct scatterlist *sgl,
-	unsigned int sgl_count, void __user *ptr, size_t len, int write)
+	unsigned int sgl_count, struct iovec *iov, int write)
 {
+	unsigned int npages = 0, pages_nr, offset, nbytes;
 	struct scatterlist *sg = sgl;
-	unsigned int npages = 0;
-	int ret;
+	void __user *ptr = iov->iov_base;
+	size_t len = iov->iov_len;
+	struct page **pages;
+	int ret, i;
  
-	while (len > 0) {
-		struct page *page;
-		unsigned int offset = (uintptr_t)ptr & ~PAGE_MASK;
-		unsigned int nbytes = min_t(unsigned int,
-				PAGE_SIZE - offset, len);
+	pages_nr = iov_num_pages(iov);
+	if (pages_nr > sgl_count)
+		return -ENOBUFS;
  
-		if (npages == sgl_count) {
-			ret = -ENOBUFS;
-			goto err;
-		}
+	pages = kmalloc(pages_nr * sizeof(struct page *), GFP_KERNEL);
+	if (!pages)
+		return -ENOMEM;
  
-		ret = get_user_pages_fast((unsigned long)ptr, 1, write, &page);
-		BUG_ON(ret == 0); /* we should either get our page or fail */
-		if (ret < 0)
-			goto err;
+	ret = get_user_pages_fast((unsigned long)ptr, pages_nr, write, pages);
+	/* No pages were pinned */
+	if (ret < 0)
+		goto out;
+	/* Less pages pinned than wanted */
+	if (ret != pages_nr) {
+		for (i = 0; i < ret; i++)
+			put_page(pages[i]);
+		ret = -EFAULT;
+		goto out;
+	}
  
-		sg_set_page(sg, page, nbytes, offset);
+	while (len > 0) {
+		offset = (uintptr_t)ptr & ~PAGE_MASK;
+		nbytes = min_t(unsigned int, PAGE_SIZE - offset, len);
+		sg_set_page(sg, pages[npages], nbytes, offset);
 		ptr += nbytes;
 		len -= nbytes;
 		sg++;
 		npages++;
 	}
-	return npages;
  
-err:
-	/* Put pages that we hold */
-	for (sg = sgl; sg != &sgl[npages]; sg++)
-		put_page(sg_page(sg));
+out:
+	kfree(pages);
 	return ret;
 }
  
@@ -496,8 +503,7 @@
  
 	pr_debug("Mapping %u iovecs for %u pages\n", niov, sgl_count);
 	for (i = 0; i < niov; i++) {
-		ret = vhost_scsi_map_to_sgl(sg, sgl_count, iov[i].iov_base,
-					iov[i].iov_len, write);
+		ret = vhost_scsi_map_to_sgl(sg, sgl_count, &iov[i], write);
 		if (ret < 0) {
 			for (i = 0; i < tv_cmd->tvc_sgl_count; i++)
 				put_page(sg_page(&tv_cmd->tvc_sgl[i]));
...	...	@@ -430,40 +430,47 @@
430	430	* Returns the number of scatterlist entries used or -errno on error.
431	431	*/
432	432	static int vhost_scsi_map_to_sgl(struct scatterlist *sgl,
433		- unsigned int sgl_count, void __user *ptr, size_t len, int write)
	433	+ unsigned int sgl_count, struct iovec *iov, int write)
434	434	{
	435	+ unsigned int npages = 0, pages_nr, offset, nbytes;
435	436	struct scatterlist *sg = sgl;
436		- unsigned int npages = 0;
437		- int ret;
	437	+ void __user *ptr = iov->iov_base;
	438	+ size_t len = iov->iov_len;
	439	+ struct page **pages;
	440	+ int ret, i;
438	441
439		- while (len > 0) {
440		- struct page *page;
441		- unsigned int offset = (uintptr_t)ptr & ~PAGE_MASK;
442		- unsigned int nbytes = min_t(unsigned int,
443		- PAGE_SIZE - offset, len);
	442	+ pages_nr = iov_num_pages(iov);
	443	+ if (pages_nr > sgl_count)
	444	+ return -ENOBUFS;
444	445
445		- if (npages == sgl_count) {
446		- ret = -ENOBUFS;
447		- goto err;
448		- }
	446	+ pages = kmalloc(pages_nr * sizeof(struct page *), GFP_KERNEL);
	447	+ if (!pages)
	448	+ return -ENOMEM;
449	449
450		- ret = get_user_pages_fast((unsigned long)ptr, 1, write, &page);
451		- BUG_ON(ret == 0); /* we should either get our page or fail */
452		- if (ret < 0)
453		- goto err;
	450	+ ret = get_user_pages_fast((unsigned long)ptr, pages_nr, write, pages);
	451	+ /* No pages were pinned */
	452	+ if (ret < 0)
	453	+ goto out;
	454	+ /* Less pages pinned than wanted */
	455	+ if (ret != pages_nr) {
	456	+ for (i = 0; i < ret; i++)
	457	+ put_page(pages[i]);
	458	+ ret = -EFAULT;
	459	+ goto out;
	460	+ }
454	461
455		- sg_set_page(sg, page, nbytes, offset);
	462	+ while (len > 0) {
	463	+ offset = (uintptr_t)ptr & ~PAGE_MASK;
	464	+ nbytes = min_t(unsigned int, PAGE_SIZE - offset, len);
	465	+ sg_set_page(sg, pages[npages], nbytes, offset);
456	466	ptr += nbytes;
457	467	len -= nbytes;
458	468	sg++;
459	469	npages++;
460	470	}
461		- return npages;
462	471
463		-err:
464		- /* Put pages that we hold */
465		- for (sg = sgl; sg != &sgl[npages]; sg++)
466		- put_page(sg_page(sg));
	472	+out:
	473	+ kfree(pages);
467	474	return ret;
468	475	}
469	476
...	...	@@ -496,8 +503,7 @@
496	503
497	504	pr_debug("Mapping %u iovecs for %u pages\n", niov, sgl_count);
498	505	for (i = 0; i < niov; i++) {
499		- ret = vhost_scsi_map_to_sgl(sg, sgl_count, iov[i].iov_base,
500		- iov[i].iov_len, write);
	506	+ ret = vhost_scsi_map_to_sgl(sg, sgl_count, &iov[i], write);
501	507	if (ret < 0) {
502	508	for (i = 0; i < tv_cmd->tvc_sgl_count; i++)
503	509	put_page(sg_page(&tv_cmd->tvc_sgl[i]));