splice: implement default splice_read method

If f_op->splice_read() is not implemented, fall back to a plain read. Use vfs_readv() to read into previously allocated pages. This will allow splice and functions using splice, such as the loop device, to work on all filesystems. This includes "direct_io" files in fuse which bypass the page cache. Signed-off-by: Miklos Szeredi <mszeredi@suse.cz> Signed-off-by: Jens Axboe <jens.axboe@oracle.com>

splice: implement default splice_read method
If f_op->splice_read() is not implemented, fall back to a plain read. Use vfs_readv() to read into previously allocated pages. This will allow splice and functions using splice, such as the loop device, to work on all filesystems. This includes "direct_io" files in fuse which bypass the page cache. Signed-off-by: Miklos Szeredi <mszeredi@suse.cz> Signed-off-by: Jens Axboe <jens.axboe@oracle.com>
Miklos Szeredi · Jens Axboe
1 parent 7c77f0b3f9
Showing 7 changed files with 140 additions and 24 deletions Side-by-side Diff
drivers/block/loop.c
fs/coda/file.c
fs/pipe.c
fs/read_write.c
fs/splice.c
include/linux/fs.h
include/linux/pipe_fs_i.h
@@ -709,10 +709,6 @@
 	if (!S_ISREG(inode->i_mode) && !S_ISBLK(inode->i_mode))
 		goto out_putf;
  
-	/* new backing store needs to support loop (eg splice_read) */
-	if (!inode->i_fop->splice_read)
-		goto out_putf;
-
 	/* size of the new backing store needs to be the same */
 	if (get_loop_size(lo, file) != get_loop_size(lo, old_file))
 		goto out_putf;
@@ -788,12 +784,7 @@
 	error = -EINVAL;
 	if (S_ISREG(inode->i_mode) || S_ISBLK(inode->i_mode)) {
 		const struct address_space_operations *aops = mapping->a_ops;
-		/*
-		 * If we can't read - sorry. If we only can't write - well,
-		 * it's going to be read-only.
-		 */
-		if (!file->f_op->splice_read)
-			goto out_putf;
+
 		if (aops->write_begin)
 			lo_flags |= LO_FLAGS_USE_AOPS;
 		if (!(lo_flags & LO_FLAGS_USE_AOPS) && !file->f_op->write)
@@ -47,6 +47,8 @@
 		      struct pipe_inode_info *pipe, size_t count,
 		      unsigned int flags)
 {
+	ssize_t (*splice_read)(struct file *, loff_t *,
+			       struct pipe_inode_info *, size_t, unsigned int);
 	struct coda_file_info *cfi;
 	struct file *host_file;
  
  
@@ -54,10 +56,11 @@
 	BUG_ON(!cfi || cfi->cfi_magic != CODA_MAGIC);
 	host_file = cfi->cfi_container;
  
-	if (!host_file->f_op || !host_file->f_op->splice_read)
-		return -EINVAL;
+	splice_read = host_file->f_op->splice_read;
+	if (!splice_read)
+		splice_read = default_file_splice_read;
  
-	return host_file->f_op->splice_read(host_file, ppos, pipe, count,flags);
+	return splice_read(host_file, ppos, pipe, count, flags);
 }
  
 static ssize_t
@@ -302,6 +302,20 @@
 	return 0;
 }
  
+/**
+ * generic_pipe_buf_release - put a reference to a &struct pipe_buffer
+ * @pipe:	the pipe that the buffer belongs to
+ * @buf:	the buffer to put a reference to
+ *
+ * Description:
+ *	This function releases a reference to @buf.
+ */
+void generic_pipe_buf_release(struct pipe_inode_info *pipe,
+			      struct pipe_buffer *buf)
+{
+	page_cache_release(buf->page);
+}
+
 static const struct pipe_buf_operations anon_pipe_buf_ops = {
 	.can_merge = 1,
 	.map = generic_pipe_buf_map,
@@ -805,12 +805,6 @@
 		goto out;
 	if (!(in_file->f_mode & FMODE_READ))
 		goto fput_in;
-	retval = -EINVAL;
-	in_inode = in_file->f_path.dentry->d_inode;
-	if (!in_inode)
-		goto fput_in;
-	if (!in_file->f_op || !in_file->f_op->splice_read)
-		goto fput_in;
 	retval = -ESPIPE;
 	if (!ppos)
 		ppos = &in_file->f_pos;
@@ -834,6 +828,7 @@
 	retval = -EINVAL;
 	if (!out_file->f_op || !out_file->f_op->sendpage)
 		goto fput_out;
+	in_inode = in_file->f_path.dentry->d_inode;
 	out_inode = out_file->f_path.dentry->d_inode;
 	retval = rw_verify_area(WRITE, out_file, &out_file->f_pos, count);
 	if (retval < 0)
@@ -507,9 +507,116 @@
  
 	return ret;
 }
-
 EXPORT_SYMBOL(generic_file_splice_read);
  
+static const struct pipe_buf_operations default_pipe_buf_ops = {
+	.can_merge = 0,
+	.map = generic_pipe_buf_map,
+	.unmap = generic_pipe_buf_unmap,
+	.confirm = generic_pipe_buf_confirm,
+	.release = generic_pipe_buf_release,
+	.steal = generic_pipe_buf_steal,
+	.get = generic_pipe_buf_get,
+};
+
+static ssize_t kernel_readv(struct file *file, const struct iovec *vec,
+			    unsigned long vlen, loff_t offset)
+{
+	mm_segment_t old_fs;
+	loff_t pos = offset;
+	ssize_t res;
+
+	old_fs = get_fs();
+	set_fs(get_ds());
+	/* The cast to a user pointer is valid due to the set_fs() */
+	res = vfs_readv(file, (const struct iovec __user *)vec, vlen, &pos);
+	set_fs(old_fs);
+
+	return res;
+}
+
+ssize_t default_file_splice_read(struct file *in, loff_t *ppos,
+				 struct pipe_inode_info *pipe, size_t len,
+				 unsigned int flags)
+{
+	unsigned int nr_pages;
+	unsigned int nr_freed;
+	size_t offset;
+	struct page *pages[PIPE_BUFFERS];
+	struct partial_page partial[PIPE_BUFFERS];
+	struct iovec vec[PIPE_BUFFERS];
+	pgoff_t index;
+	ssize_t res;
+	size_t this_len;
+	int error;
+	int i;
+	struct splice_pipe_desc spd = {
+		.pages = pages,
+		.partial = partial,
+		.flags = flags,
+		.ops = &default_pipe_buf_ops,
+		.spd_release = spd_release_page,
+	};
+
+	index = *ppos >> PAGE_CACHE_SHIFT;
+	offset = *ppos & ~PAGE_CACHE_MASK;
+	nr_pages = (len + offset + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
+
+	for (i = 0; i < nr_pages && i < PIPE_BUFFERS && len; i++) {
+		struct page *page;
+
+		page = alloc_page(GFP_HIGHUSER);
+		error = -ENOMEM;
+		if (!page)
+			goto err;
+
+		this_len = min_t(size_t, len, PAGE_CACHE_SIZE - offset);
+		vec[i].iov_base = (void __user *) kmap(page);
+		vec[i].iov_len = this_len;
+		pages[i] = page;
+		spd.nr_pages++;
+		len -= this_len;
+		offset = 0;
+	}
+
+	res = kernel_readv(in, vec, spd.nr_pages, *ppos);
+	if (res < 0)
+		goto err;
+
+	error = 0;
+	if (!res)
+		goto err;
+
+	nr_freed = 0;
+	for (i = 0; i < spd.nr_pages; i++) {
+		kunmap(pages[i]);
+		this_len = min_t(size_t, vec[i].iov_len, res);
+		partial[i].offset = 0;
+		partial[i].len = this_len;
+		if (!this_len) {
+			__free_page(pages[i]);
+			pages[i] = NULL;
+			nr_freed++;
+		}
+		res -= this_len;
+	}
+	spd.nr_pages -= nr_freed;
+
+	res = splice_to_pipe(pipe, &spd);
+	if (res > 0)
+		*ppos += res;
+
+	return res;
+
+err:
+	for (i = 0; i < spd.nr_pages; i++) {
+		kunmap(pages[i]);
+		__free_page(pages[i]);
+	}
+	return error;
+}
+EXPORT_SYMBOL(default_file_splice_read);
+
 /*
  * Send 'sd->len' bytes to socket from 'sd->file' at position 'sd->pos'
  * using sendpage(). Return the number of bytes sent.
  
@@ -933,11 +1040,10 @@
 			 struct pipe_inode_info *pipe, size_t len,
 			 unsigned int flags)
 {
+	ssize_t (*splice_read)(struct file *, loff_t *,
+			       struct pipe_inode_info *, size_t, unsigned int);
 	int ret;
  
-	if (unlikely(!in->f_op || !in->f_op->splice_read))
-		return -EINVAL;
-
 	if (unlikely(!(in->f_mode & FMODE_READ)))
 		return -EBADF;
  
@@ -945,7 +1051,11 @@
 	if (unlikely(ret < 0))
 		return ret;
  
-	return in->f_op->splice_read(in, ppos, pipe, len, flags);
+	splice_read = in->f_op->splice_read;
+	if (!splice_read)
+		splice_read = default_file_splice_read;
+
+	return splice_read(in, ppos, pipe, len, flags);
 }
  
 /**
@@ -2204,6 +2204,8 @@
 /* fs/splice.c */
 extern ssize_t generic_file_splice_read(struct file *, loff_t *,
 		struct pipe_inode_info *, size_t, unsigned int);
+extern ssize_t default_file_splice_read(struct file *, loff_t *,
+		struct pipe_inode_info *, size_t, unsigned int);
 extern ssize_t generic_file_splice_write(struct pipe_inode_info *,
 		struct file *, loff_t *, size_t, unsigned int);
 extern ssize_t generic_splice_sendpage(struct pipe_inode_info *pipe,
@@ -152,6 +152,7 @@
 void generic_pipe_buf_get(struct pipe_inode_info *, struct pipe_buffer *);
 int generic_pipe_buf_confirm(struct pipe_inode_info *, struct pipe_buffer *);
 int generic_pipe_buf_steal(struct pipe_inode_info *, struct pipe_buffer *);
+void generic_pipe_buf_release(struct pipe_inode_info *, struct pipe_buffer *);
  
 #endif
...	...	@@ -709,10 +709,6 @@
709	709	if (!S_ISREG(inode->i_mode) && !S_ISBLK(inode->i_mode))
710	710	goto out_putf;
711	711
712		- /* new backing store needs to support loop (eg splice_read) */
713		- if (!inode->i_fop->splice_read)
714		- goto out_putf;
715		-
716	712	/* size of the new backing store needs to be the same */
717	713	if (get_loop_size(lo, file) != get_loop_size(lo, old_file))
718	714	goto out_putf;
...	...	@@ -788,12 +784,7 @@
788	784	error = -EINVAL;
789	785	if (S_ISREG(inode->i_mode) \|\| S_ISBLK(inode->i_mode)) {
790	786	const struct address_space_operations *aops = mapping->a_ops;
791		- /*
792		- * If we can't read - sorry. If we only can't write - well,
793		- * it's going to be read-only.
794		- */
795		- if (!file->f_op->splice_read)
796		- goto out_putf;
	787	+
797	788	if (aops->write_begin)
798	789	lo_flags \|= LO_FLAGS_USE_AOPS;
799	790	if (!(lo_flags & LO_FLAGS_USE_AOPS) && !file->f_op->write)
...	...	@@ -47,6 +47,8 @@
47	47	struct pipe_inode_info *pipe, size_t count,
48	48	unsigned int flags)
49	49	{
	50	+ ssize_t (splice_read)(struct file , loff_t *,
	51	+ struct pipe_inode_info *, size_t, unsigned int);
50	52	struct coda_file_info *cfi;
51	53	struct file *host_file;
52	54
53	55
...	...	@@ -54,10 +56,11 @@
54	56	BUG_ON(!cfi \|\| cfi->cfi_magic != CODA_MAGIC);
55	57	host_file = cfi->cfi_container;
56	58
57		- if (!host_file->f_op \|\| !host_file->f_op->splice_read)
58		- return -EINVAL;
	59	+ splice_read = host_file->f_op->splice_read;
	60	+ if (!splice_read)
	61	+ splice_read = default_file_splice_read;
59	62
60		- return host_file->f_op->splice_read(host_file, ppos, pipe, count,flags);
	63	+ return splice_read(host_file, ppos, pipe, count, flags);
61	64	}
62	65
63	66	static ssize_t
...	...	@@ -302,6 +302,20 @@
302	302	return 0;
303	303	}
304	304
	305	+/**
	306	+ * generic_pipe_buf_release - put a reference to a &struct pipe_buffer
	307	+ * @pipe: the pipe that the buffer belongs to
	308	+ * @buf: the buffer to put a reference to
	309	+ *
	310	+ * Description:
	311	+ * This function releases a reference to @buf.
	312	+ */
	313	+void generic_pipe_buf_release(struct pipe_inode_info *pipe,
	314	+ struct pipe_buffer *buf)
	315	+{
	316	+ page_cache_release(buf->page);
	317	+}
	318	+
305	319	static const struct pipe_buf_operations anon_pipe_buf_ops = {
306	320	.can_merge = 1,
307	321	.map = generic_pipe_buf_map,
...	...	@@ -805,12 +805,6 @@
805	805	goto out;
806	806	if (!(in_file->f_mode & FMODE_READ))
807	807	goto fput_in;
808		- retval = -EINVAL;
809		- in_inode = in_file->f_path.dentry->d_inode;
810		- if (!in_inode)
811		- goto fput_in;
812		- if (!in_file->f_op \|\| !in_file->f_op->splice_read)
813		- goto fput_in;
814	808	retval = -ESPIPE;
815	809	if (!ppos)
816	810	ppos = &in_file->f_pos;
...	...	@@ -834,6 +828,7 @@
834	828	retval = -EINVAL;
835	829	if (!out_file->f_op \|\| !out_file->f_op->sendpage)
836	830	goto fput_out;
	831	+ in_inode = in_file->f_path.dentry->d_inode;
837	832	out_inode = out_file->f_path.dentry->d_inode;
838	833	retval = rw_verify_area(WRITE, out_file, &out_file->f_pos, count);
839	834	if (retval < 0)
...	...	@@ -507,9 +507,116 @@
507	507
508	508	return ret;
509	509	}
510		-
511	510	EXPORT_SYMBOL(generic_file_splice_read);
512	511
	512	+static const struct pipe_buf_operations default_pipe_buf_ops = {
	513	+ .can_merge = 0,
	514	+ .map = generic_pipe_buf_map,
	515	+ .unmap = generic_pipe_buf_unmap,
	516	+ .confirm = generic_pipe_buf_confirm,
	517	+ .release = generic_pipe_buf_release,
	518	+ .steal = generic_pipe_buf_steal,
	519	+ .get = generic_pipe_buf_get,
	520	+};
	521	+
	522	+static ssize_t kernel_readv(struct file file, const struct iovec vec,
	523	+ unsigned long vlen, loff_t offset)
	524	+{
	525	+ mm_segment_t old_fs;
	526	+ loff_t pos = offset;
	527	+ ssize_t res;
	528	+
	529	+ old_fs = get_fs();
	530	+ set_fs(get_ds());
	531	+ /* The cast to a user pointer is valid due to the set_fs() */
	532	+ res = vfs_readv(file, (const struct iovec __user *)vec, vlen, &pos);
	533	+ set_fs(old_fs);
	534	+
	535	+ return res;
	536	+}
	537	+
	538	+ssize_t default_file_splice_read(struct file in, loff_t ppos,
	539	+ struct pipe_inode_info *pipe, size_t len,
	540	+ unsigned int flags)
	541	+{
	542	+ unsigned int nr_pages;
	543	+ unsigned int nr_freed;
	544	+ size_t offset;
	545	+ struct page *pages[PIPE_BUFFERS];
	546	+ struct partial_page partial[PIPE_BUFFERS];
	547	+ struct iovec vec[PIPE_BUFFERS];
	548	+ pgoff_t index;
	549	+ ssize_t res;
	550	+ size_t this_len;
	551	+ int error;
	552	+ int i;
	553	+ struct splice_pipe_desc spd = {
	554	+ .pages = pages,
	555	+ .partial = partial,
	556	+ .flags = flags,
	557	+ .ops = &default_pipe_buf_ops,
	558	+ .spd_release = spd_release_page,
	559	+ };
	560	+
	561	+ index = *ppos >> PAGE_CACHE_SHIFT;
	562	+ offset = *ppos & ~PAGE_CACHE_MASK;
	563	+ nr_pages = (len + offset + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
	564	+
	565	+ for (i = 0; i < nr_pages && i < PIPE_BUFFERS && len; i++) {
	566	+ struct page *page;
	567	+
	568	+ page = alloc_page(GFP_HIGHUSER);
	569	+ error = -ENOMEM;
	570	+ if (!page)
	571	+ goto err;
	572	+
	573	+ this_len = min_t(size_t, len, PAGE_CACHE_SIZE - offset);
	574	+ vec[i].iov_base = (void __user *) kmap(page);
	575	+ vec[i].iov_len = this_len;
	576	+ pages[i] = page;
	577	+ spd.nr_pages++;
	578	+ len -= this_len;
	579	+ offset = 0;
	580	+ }
	581	+
	582	+ res = kernel_readv(in, vec, spd.nr_pages, *ppos);
	583	+ if (res < 0)
	584	+ goto err;
	585	+
	586	+ error = 0;
	587	+ if (!res)
	588	+ goto err;
	589	+
	590	+ nr_freed = 0;
	591	+ for (i = 0; i < spd.nr_pages; i++) {
	592	+ kunmap(pages[i]);
	593	+ this_len = min_t(size_t, vec[i].iov_len, res);
	594	+ partial[i].offset = 0;
	595	+ partial[i].len = this_len;
	596	+ if (!this_len) {
	597	+ __free_page(pages[i]);
	598	+ pages[i] = NULL;
	599	+ nr_freed++;
	600	+ }
	601	+ res -= this_len;
	602	+ }
	603	+ spd.nr_pages -= nr_freed;
	604	+
	605	+ res = splice_to_pipe(pipe, &spd);
	606	+ if (res > 0)
	607	+ *ppos += res;
	608	+
	609	+ return res;
	610	+
	611	+err:
	612	+ for (i = 0; i < spd.nr_pages; i++) {
	613	+ kunmap(pages[i]);
	614	+ __free_page(pages[i]);
	615	+ }
	616	+ return error;
	617	+}
	618	+EXPORT_SYMBOL(default_file_splice_read);
	619	+
513	620	/*
514	621	* Send 'sd->len' bytes to socket from 'sd->file' at position 'sd->pos'
515	622	* using sendpage(). Return the number of bytes sent.
516	623
...	...	@@ -933,11 +1040,10 @@
933	1040	struct pipe_inode_info *pipe, size_t len,
934	1041	unsigned int flags)
935	1042	{
	1043	+ ssize_t (splice_read)(struct file , loff_t *,
	1044	+ struct pipe_inode_info *, size_t, unsigned int);
936	1045	int ret;
937	1046
938		- if (unlikely(!in->f_op \|\| !in->f_op->splice_read))
939		- return -EINVAL;
940		-
941	1047	if (unlikely(!(in->f_mode & FMODE_READ)))
942	1048	return -EBADF;
943	1049
...	...	@@ -945,7 +1051,11 @@
945	1051	if (unlikely(ret < 0))
946	1052	return ret;
947	1053
948		- return in->f_op->splice_read(in, ppos, pipe, len, flags);
	1054	+ splice_read = in->f_op->splice_read;
	1055	+ if (!splice_read)
	1056	+ splice_read = default_file_splice_read;
	1057	+
	1058	+ return splice_read(in, ppos, pipe, len, flags);
949	1059	}
950	1060
951	1061	/**
...	...	@@ -2204,6 +2204,8 @@
2204	2204	/* fs/splice.c */
2205	2205	extern ssize_t generic_file_splice_read(struct file , loff_t ,
2206	2206	struct pipe_inode_info *, size_t, unsigned int);
	2207	+extern ssize_t default_file_splice_read(struct file , loff_t ,
	2208	+ struct pipe_inode_info *, size_t, unsigned int);
2207	2209	extern ssize_t generic_file_splice_write(struct pipe_inode_info *,
2208	2210	struct file , loff_t , size_t, unsigned int);
2209	2211	extern ssize_t generic_splice_sendpage(struct pipe_inode_info *pipe,
...	...	@@ -152,6 +152,7 @@
152	152	void generic_pipe_buf_get(struct pipe_inode_info , struct pipe_buffer );
153	153	int generic_pipe_buf_confirm(struct pipe_inode_info , struct pipe_buffer );
154	154	int generic_pipe_buf_steal(struct pipe_inode_info , struct pipe_buffer );
	155	+void generic_pipe_buf_release(struct pipe_inode_info , struct pipe_buffer );
155	156
156	157	#endif