[PATCH] introduce a "kernel-internal pipe object" abstraction

separate out the 'internal pipe object' abstraction, and make it usable to splice. This cleans up and fixes several aspects of the internal splice APIs and the pipe code: - pipes: the allocation and freeing of pipe_inode_info is now more symmetric and more streamlined with existing kernel practices. - splice: small micro-optimization: less pointer dereferencing in splice methods Signed-off-by: Ingo Molnar <mingo@elte.hu> Update XFS for the ->splice_read/->splice_write changes. Signed-off-by: Jens Axboe <axboe@suse.de>

[PATCH] introduce a "kernel-internal pipe object" abstraction
separate out the 'internal pipe object' abstraction, and make it usable to splice. This cleans up and fixes several aspects of the internal splice APIs and the pipe code: - pipes: the allocation and freeing of pipe_inode_info is now more symmetric and more streamlined with existing kernel practices. - splice: small micro-optimization: less pointer dereferencing in splice methods Signed-off-by: Ingo Molnar <mingo@elte.hu> Update XFS for the ->splice_read/->splice_write changes. Signed-off-by: Jens Axboe <axboe@suse.de>
Ingo Molnar · Jens Axboe
1 parent 0b749ce380
Showing 9 changed files with 114 additions and 106 deletions Side-by-side Diff
fs/fifo.c
fs/pipe.c
fs/splice.c
fs/xfs/linux-2.6/xfs_file.c
fs/xfs/linux-2.6/xfs_lrw.c
fs/xfs/linux-2.6/xfs_lrw.h
fs/xfs/linux-2.6/xfs_vnode.h
include/linux/fs.h
include/linux/pipe_fs_i.h
@@ -15,12 +15,13 @@
 #include <linux/fs.h>
 #include <linux/pipe_fs_i.h>
  
-static void wait_for_partner(struct inode* inode, unsigned int* cnt)
+static void wait_for_partner(struct inode* inode, unsigned int *cnt)
 {
 	int cur = *cnt;	
-	while(cur == *cnt) {
-		pipe_wait(inode);
-		if(signal_pending(current))
+
+	while (cur == *cnt) {
+		pipe_wait(inode->i_pipe);
+		if (signal_pending(current))
 			break;
 	}
 }
@@ -37,7 +38,8 @@
 	mutex_lock(PIPE_MUTEX(*inode));
 	if (!inode->i_pipe) {
 		ret = -ENOMEM;
-		if(!pipe_new(inode))
+		inode->i_pipe = alloc_pipe_info(inode);
+		if (!inode->i_pipe)
 			goto err_nocleanup;
 	}
 	filp->f_version = 0;
@@ -36,7 +36,7 @@
  */
  
 /* Drop the inode semaphore and wait for a pipe event, atomically */
-void pipe_wait(struct inode * inode)
+void pipe_wait(struct pipe_inode_info *pipe)
 {
 	DEFINE_WAIT(wait);
  
  
@@ -44,11 +44,13 @@
 	 * Pipes are system-local resources, so sleeping on them
 	 * is considered a noninteractive wait:
 	 */
-	prepare_to_wait(PIPE_WAIT(*inode), &wait, TASK_INTERRUPTIBLE|TASK_NONINTERACTIVE);
-	mutex_unlock(PIPE_MUTEX(*inode));
+	prepare_to_wait(&pipe->wait, &wait, TASK_INTERRUPTIBLE|TASK_NONINTERACTIVE);
+	if (pipe->inode)
+		mutex_unlock(&pipe->inode->i_mutex);
 	schedule();
-	finish_wait(PIPE_WAIT(*inode), &wait);
-	mutex_lock(PIPE_MUTEX(*inode));
+	finish_wait(&pipe->wait, &wait);
+	if (pipe->inode)
+		mutex_lock(&pipe->inode->i_mutex);
 }
  
 static int
@@ -223,7 +225,7 @@
 			wake_up_interruptible_sync(PIPE_WAIT(*inode));
  			kill_fasync(PIPE_FASYNC_WRITERS(*inode), SIGIO, POLL_OUT);
 		}
-		pipe_wait(inode);
+		pipe_wait(inode->i_pipe);
 	}
 	mutex_unlock(PIPE_MUTEX(*inode));
 	/* Signal writers asynchronously that there is more room.  */
@@ -370,7 +372,7 @@
 			do_wakeup = 0;
 		}
 		PIPE_WAITING_WRITERS(*inode)++;
-		pipe_wait(inode);
+		pipe_wait(inode->i_pipe);
 		PIPE_WAITING_WRITERS(*inode)--;
 	}
 out:
@@ -675,6 +677,20 @@
 	.fasync		= pipe_rdwr_fasync,
 };
  
+struct pipe_inode_info * alloc_pipe_info(struct inode *inode)
+{
+	struct pipe_inode_info *info;
+
+	info = kzalloc(sizeof(struct pipe_inode_info), GFP_KERNEL);
+	if (info) {
+		init_waitqueue_head(&info->wait);
+		info->r_counter = info->w_counter = 1;
+		info->inode = inode;
+	}
+
+	return info;
+}
+
 void free_pipe_info(struct inode *inode)
 {
 	int i;
@@ -691,23 +707,6 @@
 	kfree(info);
 }
  
-struct inode* pipe_new(struct inode* inode)
-{
-	struct pipe_inode_info *info;
-
-	info = kzalloc(sizeof(struct pipe_inode_info), GFP_KERNEL);
-	if (!info)
-		goto fail_page;
-	inode->i_pipe = info;
-
-	init_waitqueue_head(PIPE_WAIT(*inode));
-	PIPE_RCOUNTER(*inode) = PIPE_WCOUNTER(*inode) = 1;
-
-	return inode;
-fail_page:
-	return NULL;
-}
-
 static struct vfsmount *pipe_mnt __read_mostly;
 static int pipefs_delete_dentry(struct dentry *dentry)
 {
  
@@ -724,8 +723,10 @@
 	if (!inode)
 		goto fail_inode;
  
-	if(!pipe_new(inode))
+	inode->i_pipe = alloc_pipe_info(inode);
+	if (!inode->i_pipe)
 		goto fail_iput;
+
 	PIPE_READERS(*inode) = PIPE_WRITERS(*inode) = 1;
 	inode->i_fop = &rdwr_pipe_fops;
  
@@ -136,34 +136,33 @@
  * Pipe output worker. This sets up our pipe format with the page cache
  * pipe buffer operations. Otherwise very similar to the regular pipe_writev().
  */
-static ssize_t move_to_pipe(struct inode *inode, struct page **pages,
+static ssize_t move_to_pipe(struct pipe_inode_info *pipe, struct page **pages,
 			    int nr_pages, unsigned long offset,
 			    unsigned long len, unsigned int flags)
 {
-	struct pipe_inode_info *info;
 	int ret, do_wakeup, i;
  
 	ret = 0;
 	do_wakeup = 0;
 	i = 0;
  
-	mutex_lock(PIPE_MUTEX(*inode));
+	if (pipe->inode)
+		mutex_lock(&pipe->inode->i_mutex);
  
-	info = inode->i_pipe;
 	for (;;) {
 		int bufs;
  
-		if (!PIPE_READERS(*inode)) {
+		if (!pipe->readers) {
 			send_sig(SIGPIPE, current, 0);
 			if (!ret)
 				ret = -EPIPE;
 			break;
 		}
  
-		bufs = info->nrbufs;
+		bufs = pipe->nrbufs;
 		if (bufs < PIPE_BUFFERS) {
-			int newbuf = (info->curbuf + bufs) & (PIPE_BUFFERS - 1);
-			struct pipe_buffer *buf = info->bufs + newbuf;
+			int newbuf = (pipe->curbuf + bufs) & (PIPE_BUFFERS - 1);
+			struct pipe_buffer *buf = pipe->bufs + newbuf;
 			struct page *page = pages[i++];
 			unsigned long this_len;
  
@@ -175,7 +174,7 @@
 			buf->offset = offset;
 			buf->len = this_len;
 			buf->ops = &page_cache_pipe_buf_ops;
-			info->nrbufs = ++bufs;
+			pipe->nrbufs = ++bufs;
 			do_wakeup = 1;
  
 			ret += this_len;
  
  
  
@@ -205,25 +204,25 @@
  
 		if (do_wakeup) {
 			smp_mb();
-			if (waitqueue_active(PIPE_WAIT(*inode)))
-				wake_up_interruptible_sync(PIPE_WAIT(*inode));
-			kill_fasync(PIPE_FASYNC_READERS(*inode), SIGIO,
-				    POLL_IN);
+			if (waitqueue_active(&pipe->wait))
+				wake_up_interruptible_sync(&pipe->wait);
+			kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
 			do_wakeup = 0;
 		}
  
-		PIPE_WAITING_WRITERS(*inode)++;
-		pipe_wait(inode);
-		PIPE_WAITING_WRITERS(*inode)--;
+		pipe->waiting_writers++;
+		pipe_wait(pipe);
+		pipe->waiting_writers--;
 	}
  
-	mutex_unlock(PIPE_MUTEX(*inode));
+	if (pipe->inode)
+		mutex_unlock(&pipe->inode->i_mutex);
  
 	if (do_wakeup) {
 		smp_mb();
-		if (waitqueue_active(PIPE_WAIT(*inode)))
-			wake_up_interruptible(PIPE_WAIT(*inode));
-		kill_fasync(PIPE_FASYNC_READERS(*inode), SIGIO, POLL_IN);
+		if (waitqueue_active(&pipe->wait))
+			wake_up_interruptible(&pipe->wait);
+		kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
 	}
  
 	while (i < nr_pages)
@@ -232,8 +231,9 @@
 	return ret;
 }
  
-static int __generic_file_splice_read(struct file *in, struct inode *pipe,
-				      size_t len, unsigned int flags)
+static int
+__generic_file_splice_read(struct file *in, struct pipe_inode_info *pipe,
+			   size_t len, unsigned int flags)
 {
 	struct address_space *mapping = in->f_mapping;
 	unsigned int offset, nr_pages;
@@ -298,7 +298,7 @@
  * Will read pages from given file and fill them into a pipe.
  *
  */
-ssize_t generic_file_splice_read(struct file *in, struct inode *pipe,
+ssize_t generic_file_splice_read(struct file *in, struct pipe_inode_info *pipe,
 				 size_t len, unsigned int flags)
 {
 	ssize_t spliced;
@@ -306,6 +306,7 @@
  
 	ret = 0;
 	spliced = 0;
+
 	while (len) {
 		ret = __generic_file_splice_read(in, pipe, len, flags);
  
  
@@ -509,11 +510,10 @@
  * key here is the 'actor' worker passed in that actually moves the data
  * to the wanted destination. See pipe_to_file/pipe_to_sendpage above.
  */
-static ssize_t move_from_pipe(struct inode *inode, struct file *out,
+static ssize_t move_from_pipe(struct pipe_inode_info *pipe, struct file *out,
 			      size_t len, unsigned int flags,
 			      splice_actor *actor)
 {
-	struct pipe_inode_info *info;
 	int ret, do_wakeup, err;
 	struct splice_desc sd;
  
  
  
  
  
@@ -525,22 +525,22 @@
 	sd.file = out;
 	sd.pos = out->f_pos;
  
-	mutex_lock(PIPE_MUTEX(*inode));
+	if (pipe->inode)
+		mutex_lock(&pipe->inode->i_mutex);
  
-	info = inode->i_pipe;
 	for (;;) {
-		int bufs = info->nrbufs;
+		int bufs = pipe->nrbufs;
  
 		if (bufs) {
-			int curbuf = info->curbuf;
-			struct pipe_buffer *buf = info->bufs + curbuf;
+			int curbuf = pipe->curbuf;
+			struct pipe_buffer *buf = pipe->bufs + curbuf;
 			struct pipe_buf_operations *ops = buf->ops;
  
 			sd.len = buf->len;
 			if (sd.len > sd.total_len)
 				sd.len = sd.total_len;
  
-			err = actor(info, buf, &sd);
+			err = actor(pipe, buf, &sd);
 			if (err) {
 				if (!ret && err != -ENODATA)
 					ret = err;
  
@@ -553,10 +553,10 @@
 			buf->len -= sd.len;
 			if (!buf->len) {
 				buf->ops = NULL;
-				ops->release(info, buf);
+				ops->release(pipe, buf);
 				curbuf = (curbuf + 1) & (PIPE_BUFFERS - 1);
-				info->curbuf = curbuf;
-				info->nrbufs = --bufs;
+				pipe->curbuf = curbuf;
+				pipe->nrbufs = --bufs;
 				do_wakeup = 1;
 			}
  
  
@@ -568,9 +568,9 @@
  
 		if (bufs)
 			continue;
-		if (!PIPE_WRITERS(*inode))
+		if (!pipe->writers)
 			break;
-		if (!PIPE_WAITING_WRITERS(*inode)) {
+		if (!pipe->waiting_writers) {
 			if (ret)
 				break;
 		}
  
  
  
@@ -589,22 +589,23 @@
  
 		if (do_wakeup) {
 			smp_mb();
-			if (waitqueue_active(PIPE_WAIT(*inode)))
-				wake_up_interruptible_sync(PIPE_WAIT(*inode));
-			kill_fasync(PIPE_FASYNC_WRITERS(*inode),SIGIO,POLL_OUT);
+			if (waitqueue_active(&pipe->wait))
+				wake_up_interruptible_sync(&pipe->wait);
+			kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);
 			do_wakeup = 0;
 		}
  
-		pipe_wait(inode);
+		pipe_wait(pipe);
 	}
  
-	mutex_unlock(PIPE_MUTEX(*inode));
+	if (pipe->inode)
+		mutex_unlock(&pipe->inode->i_mutex);
  
 	if (do_wakeup) {
 		smp_mb();
-		if (waitqueue_active(PIPE_WAIT(*inode)))
-			wake_up_interruptible(PIPE_WAIT(*inode));
-		kill_fasync(PIPE_FASYNC_WRITERS(*inode), SIGIO, POLL_OUT);
+		if (waitqueue_active(&pipe->wait))
+			wake_up_interruptible(&pipe->wait);
+		kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);
 	}
  
 	mutex_lock(&out->f_mapping->host->i_mutex);
@@ -616,7 +617,7 @@
  
 /**
  * generic_file_splice_write - splice data from a pipe to a file
- * @inode:	pipe inode
+ * @pipe:	pipe info
  * @out:	file to write to
  * @len:	number of bytes to splice
  * @flags:	splice modifier flags
  
  
@@ -625,12 +626,15 @@
  * the given pipe inode to the given file.
  *
  */
-ssize_t generic_file_splice_write(struct inode *inode, struct file *out,
-				  size_t len, unsigned int flags)
+ssize_t
+generic_file_splice_write(struct pipe_inode_info *pipe, struct file *out,
+			  size_t len, unsigned int flags)
 {
 	struct address_space *mapping = out->f_mapping;
-	ssize_t ret = move_from_pipe(inode, out, len, flags, pipe_to_file);
+	ssize_t ret;
  
+	ret = move_from_pipe(pipe, out, len, flags, pipe_to_file);
+
 	/*
 	 * if file or inode is SYNC and we actually wrote some data, sync it
 	 */
  
@@ -664,10 +668,10 @@
  * is involved.
  *
  */
-ssize_t generic_splice_sendpage(struct inode *inode, struct file *out,
+ssize_t generic_splice_sendpage(struct pipe_inode_info *pipe, struct file *out,
 				size_t len, unsigned int flags)
 {
-	return move_from_pipe(inode, out, len, flags, pipe_to_sendpage);
+	return move_from_pipe(pipe, out, len, flags, pipe_to_sendpage);
 }
  
 EXPORT_SYMBOL(generic_splice_sendpage);
@@ -675,8 +679,8 @@
 /*
  * Attempt to initiate a splice from pipe to file.
  */
-static long do_splice_from(struct inode *pipe, struct file *out, size_t len,
-			   unsigned int flags)
+static long do_splice_from(struct pipe_inode_info *pipe, struct file *out,
+			   size_t len, unsigned int flags)
 {
 	loff_t pos;
 	int ret;
@@ -698,8 +702,8 @@
 /*
  * Attempt to initiate a splice from a file to a pipe.
  */
-static long do_splice_to(struct file *in, struct inode *pipe, size_t len,
-			 unsigned int flags)
+static long do_splice_to(struct file *in, struct pipe_inode_info *pipe,
+			 size_t len, unsigned int flags)
 {
 	loff_t pos, isize, left;
 	int ret;
  
  
@@ -732,14 +736,14 @@
 static long do_splice(struct file *in, struct file *out, size_t len,
 		      unsigned int flags)
 {
-	struct inode *pipe;
+	struct pipe_inode_info *pipe;
  
-	pipe = in->f_dentry->d_inode;
-	if (pipe->i_pipe)
+	pipe = in->f_dentry->d_inode->i_pipe;
+	if (pipe)
 		return do_splice_from(pipe, out, len, flags);
  
-	pipe = out->f_dentry->d_inode;
-	if (pipe->i_pipe)
+	pipe = out->f_dentry->d_inode->i_pipe;
+	if (pipe)
 		return do_splice_to(in, pipe, len, flags);
  
 	return -EINVAL;
@@ -252,7 +252,7 @@
 STATIC ssize_t
 xfs_file_splice_read(
 	struct file		*infilp,
-	struct inode		*pipe,
+	struct pipe_inode_info	*pipe,
 	size_t			len,
 	unsigned int		flags)
 {
@@ -266,7 +266,7 @@
 STATIC ssize_t
 xfs_file_splice_read_invis(
 	struct file		*infilp,
-	struct inode		*pipe,
+	struct pipe_inode_info	*pipe,
 	size_t			len,
 	unsigned int		flags)
 {
@@ -279,7 +279,7 @@
  
 STATIC ssize_t
 xfs_file_splice_write(
-	struct inode		*pipe,
+	struct pipe_inode_info	*pipe,
 	struct file		*outfilp,
 	size_t			len,
 	unsigned int		flags)
@@ -293,7 +293,7 @@
  
 STATIC ssize_t
 xfs_file_splice_write_invis(
-	struct inode		*pipe,
+	struct pipe_inode_info	*pipe,
 	struct file		*outfilp,
 	size_t			len,
 	unsigned int		flags)
@@ -338,7 +338,7 @@
 xfs_splice_read(
 	bhv_desc_t		*bdp,
 	struct file		*infilp,
-	struct inode		*pipe,
+	struct pipe_inode_info	*pipe,
 	size_t			count,
 	int			flags,
 	int			ioflags,
@@ -380,7 +380,7 @@
 ssize_t
 xfs_splice_write(
 	bhv_desc_t		*bdp,
-	struct inode		*pipe,
+	struct pipe_inode_info	*pipe,
 	struct file		*outfilp,
 	size_t			count,
 	int			flags,
@@ -94,9 +94,9 @@
 				loff_t *, int, size_t, read_actor_t,
 				void *, struct cred *);
 extern ssize_t xfs_splice_read(struct bhv_desc *, struct file *,
-				struct inode *, size_t, int, int,
+				struct pipe_inode_info *, size_t, int, int,
 				struct cred *);
-extern ssize_t xfs_splice_write(struct bhv_desc *, struct inode *,
+extern ssize_t xfs_splice_write(struct bhv_desc *, struct pipe_inode_info *,
 				struct file *, size_t, int, int,
 				struct cred *);
  
@@ -174,9 +174,9 @@
 				loff_t *, int, size_t, read_actor_t,
 				void *, struct cred *);
 typedef ssize_t (*vop_splice_read_t)(bhv_desc_t *, struct file *,
-				struct inode *, size_t, int, int,
+				struct pipe_inode_info *, size_t, int, int,
 				struct cred *);
-typedef ssize_t (*vop_splice_write_t)(bhv_desc_t *, struct inode *,
+typedef ssize_t (*vop_splice_write_t)(bhv_desc_t *, struct pipe_inode_info *,
 				struct file *, size_t, int, int,
 				struct cred *);
 typedef int	(*vop_ioctl_t)(bhv_desc_t *, struct inode *, struct file *,
@@ -1039,8 +1039,8 @@
 	int (*check_flags)(int);
 	int (*dir_notify)(struct file *filp, unsigned long arg);
 	int (*flock) (struct file *, int, struct file_lock *);
-	ssize_t (*splice_write)(struct inode *, struct file *, size_t, unsigned int);
-	ssize_t (*splice_read)(struct file *, struct inode *, size_t, unsigned int);
+	ssize_t (*splice_write)(struct pipe_inode_info *, struct file *, size_t, unsigned int);
+	ssize_t (*splice_read)(struct file *, struct pipe_inode_info *, size_t, unsigned int);
 };
  
 struct inode_operations {
@@ -1611,8 +1611,8 @@
 extern void do_generic_mapping_read(struct address_space *mapping,
 				    struct file_ra_state *, struct file *,
 				    loff_t *, read_descriptor_t *, read_actor_t);
-extern ssize_t generic_file_splice_read(struct file *, struct inode *, size_t, unsigned int);
-extern ssize_t generic_file_splice_write(struct inode *, struct file *, size_t, unsigned int);
+extern ssize_t generic_file_splice_read(struct file *, struct pipe_inode_info *, size_t, unsigned int);
+extern ssize_t generic_file_splice_write(struct pipe_inode_info *, struct file *, size_t, unsigned int);
 extern void
 file_ra_state_init(struct file_ra_state *ra, struct address_space *mapping);
 extern ssize_t generic_file_readv(struct file *filp, const struct iovec *iov, 
@@ -36,6 +36,7 @@
 	unsigned int w_counter;
 	struct fasync_struct *fasync_readers;
 	struct fasync_struct *fasync_writers;
+	struct inode *inode;
 };
  
 /* Differs from PIPE_BUF in that PIPE_SIZE is the length of the actual
  
@@ -53,10 +54,10 @@
 #define PIPE_FASYNC_WRITERS(inode)     (&((inode).i_pipe->fasync_writers))
  
 /* Drop the inode semaphore and wait for a pipe event, atomically */
-void pipe_wait(struct inode * inode);
+void pipe_wait(struct pipe_inode_info *pipe);
  
-struct inode* pipe_new(struct inode* inode);
-void free_pipe_info(struct inode* inode);
+struct pipe_inode_info * alloc_pipe_info(struct inode * inode);
+void free_pipe_info(struct inode * inode);
  
 /*
  * splice is tied to pipes as a transport (at least for now), so we'll just
...	...	@@ -15,12 +15,13 @@
15	15	#include <linux/fs.h>
16	16	#include <linux/pipe_fs_i.h>
17	17
18		-static void wait_for_partner(struct inode* inode, unsigned int* cnt)
	18	+static void wait_for_partner(struct inode* inode, unsigned int *cnt)
19	19	{
20	20	int cur = *cnt;
21		- while(cur == *cnt) {
22		- pipe_wait(inode);
23		- if(signal_pending(current))
	21	+
	22	+ while (cur == *cnt) {
	23	+ pipe_wait(inode->i_pipe);
	24	+ if (signal_pending(current))
24	25	break;
25	26	}
26	27	}
...	...	@@ -37,7 +38,8 @@
37	38	mutex_lock(PIPE_MUTEX(*inode));
38	39	if (!inode->i_pipe) {
39	40	ret = -ENOMEM;
40		- if(!pipe_new(inode))
	41	+ inode->i_pipe = alloc_pipe_info(inode);
	42	+ if (!inode->i_pipe)
41	43	goto err_nocleanup;
42	44	}
43	45	filp->f_version = 0;
...	...	@@ -36,7 +36,7 @@
36	36	*/
37	37
38	38	/* Drop the inode semaphore and wait for a pipe event, atomically */
39		-void pipe_wait(struct inode * inode)
	39	+void pipe_wait(struct pipe_inode_info *pipe)
40	40	{
41	41	DEFINE_WAIT(wait);
42	42
43	43
...	...	@@ -44,11 +44,13 @@
44	44	* Pipes are system-local resources, so sleeping on them
45	45	* is considered a noninteractive wait:
46	46	*/
47		- prepare_to_wait(PIPE_WAIT(*inode), &wait, TASK_INTERRUPTIBLE\|TASK_NONINTERACTIVE);
48		- mutex_unlock(PIPE_MUTEX(*inode));
	47	+ prepare_to_wait(&pipe->wait, &wait, TASK_INTERRUPTIBLE\|TASK_NONINTERACTIVE);
	48	+ if (pipe->inode)
	49	+ mutex_unlock(&pipe->inode->i_mutex);
49	50	schedule();
50		- finish_wait(PIPE_WAIT(*inode), &wait);
51		- mutex_lock(PIPE_MUTEX(*inode));
	51	+ finish_wait(&pipe->wait, &wait);
	52	+ if (pipe->inode)
	53	+ mutex_lock(&pipe->inode->i_mutex);
52	54	}
53	55
54	56	static int
...	...	@@ -223,7 +225,7 @@
223	225	wake_up_interruptible_sync(PIPE_WAIT(*inode));
224	226	kill_fasync(PIPE_FASYNC_WRITERS(*inode), SIGIO, POLL_OUT);
225	227	}
226		- pipe_wait(inode);
	228	+ pipe_wait(inode->i_pipe);
227	229	}
228	230	mutex_unlock(PIPE_MUTEX(*inode));
229	231	/* Signal writers asynchronously that there is more room. */
...	...	@@ -370,7 +372,7 @@
370	372	do_wakeup = 0;
371	373	}
372	374	PIPE_WAITING_WRITERS(*inode)++;
373		- pipe_wait(inode);
	375	+ pipe_wait(inode->i_pipe);
374	376	PIPE_WAITING_WRITERS(*inode)--;
375	377	}
376	378	out:
...	...	@@ -675,6 +677,20 @@
675	677	.fasync = pipe_rdwr_fasync,
676	678	};
677	679
	680	+struct pipe_inode_info * alloc_pipe_info(struct inode *inode)
	681	+{
	682	+ struct pipe_inode_info *info;
	683	+
	684	+ info = kzalloc(sizeof(struct pipe_inode_info), GFP_KERNEL);
	685	+ if (info) {
	686	+ init_waitqueue_head(&info->wait);
	687	+ info->r_counter = info->w_counter = 1;
	688	+ info->inode = inode;
	689	+ }
	690	+
	691	+ return info;
	692	+}
	693	+
678	694	void free_pipe_info(struct inode *inode)
679	695	{
680	696	int i;
...	...	@@ -691,23 +707,6 @@
691	707	kfree(info);
692	708	}
693	709
694		-struct inode* pipe_new(struct inode* inode)
695		-{
696		- struct pipe_inode_info *info;
697		-
698		- info = kzalloc(sizeof(struct pipe_inode_info), GFP_KERNEL);
699		- if (!info)
700		- goto fail_page;
701		- inode->i_pipe = info;
702		-
703		- init_waitqueue_head(PIPE_WAIT(*inode));
704		- PIPE_RCOUNTER(inode) = PIPE_WCOUNTER(inode) = 1;
705		-
706		- return inode;
707		-fail_page:
708		- return NULL;
709		-}
710		-
711	710	static struct vfsmount *pipe_mnt __read_mostly;
712	711	static int pipefs_delete_dentry(struct dentry *dentry)
713	712	{
714	713
...	...	@@ -724,8 +723,10 @@
724	723	if (!inode)
725	724	goto fail_inode;
726	725
727		- if(!pipe_new(inode))
	726	+ inode->i_pipe = alloc_pipe_info(inode);
	727	+ if (!inode->i_pipe)
728	728	goto fail_iput;
	729	+
729	730	PIPE_READERS(inode) = PIPE_WRITERS(inode) = 1;
730	731	inode->i_fop = &rdwr_pipe_fops;
731	732
...	...	@@ -136,34 +136,33 @@
136	136	* Pipe output worker. This sets up our pipe format with the page cache
137	137	* pipe buffer operations. Otherwise very similar to the regular pipe_writev().
138	138	*/
139		-static ssize_t move_to_pipe(struct inode inode, struct page *pages,
	139	+static ssize_t move_to_pipe(struct pipe_inode_info pipe, struct page *pages,
140	140	int nr_pages, unsigned long offset,
141	141	unsigned long len, unsigned int flags)
142	142	{
143		- struct pipe_inode_info *info;
144	143	int ret, do_wakeup, i;
145	144
146	145	ret = 0;
147	146	do_wakeup = 0;
148	147	i = 0;
149	148
150		- mutex_lock(PIPE_MUTEX(*inode));
	149	+ if (pipe->inode)
	150	+ mutex_lock(&pipe->inode->i_mutex);
151	151
152		- info = inode->i_pipe;
153	152	for (;;) {
154	153	int bufs;
155	154
156		- if (!PIPE_READERS(*inode)) {
	155	+ if (!pipe->readers) {
157	156	send_sig(SIGPIPE, current, 0);
158	157	if (!ret)
159	158	ret = -EPIPE;
160	159	break;
161	160	}
162	161
163		- bufs = info->nrbufs;
	162	+ bufs = pipe->nrbufs;
164	163	if (bufs < PIPE_BUFFERS) {
165		- int newbuf = (info->curbuf + bufs) & (PIPE_BUFFERS - 1);
166		- struct pipe_buffer *buf = info->bufs + newbuf;
	164	+ int newbuf = (pipe->curbuf + bufs) & (PIPE_BUFFERS - 1);
	165	+ struct pipe_buffer *buf = pipe->bufs + newbuf;
167	166	struct page *page = pages[i++];
168	167	unsigned long this_len;
169	168
...	...	@@ -175,7 +174,7 @@
175	174	buf->offset = offset;
176	175	buf->len = this_len;
177	176	buf->ops = &page_cache_pipe_buf_ops;
178		- info->nrbufs = ++bufs;
	177	+ pipe->nrbufs = ++bufs;
179	178	do_wakeup = 1;
180	179
181	180	ret += this_len;
182	181
183	182
184	183
...	...	@@ -205,25 +204,25 @@
205	204
206	205	if (do_wakeup) {
207	206	smp_mb();
208		- if (waitqueue_active(PIPE_WAIT(*inode)))
209		- wake_up_interruptible_sync(PIPE_WAIT(*inode));
210		- kill_fasync(PIPE_FASYNC_READERS(*inode), SIGIO,
211		- POLL_IN);
	207	+ if (waitqueue_active(&pipe->wait))
	208	+ wake_up_interruptible_sync(&pipe->wait);
	209	+ kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
212	210	do_wakeup = 0;
213	211	}
214	212
215		- PIPE_WAITING_WRITERS(*inode)++;
216		- pipe_wait(inode);
217		- PIPE_WAITING_WRITERS(*inode)--;
	213	+ pipe->waiting_writers++;
	214	+ pipe_wait(pipe);
	215	+ pipe->waiting_writers--;
218	216	}
219	217
220		- mutex_unlock(PIPE_MUTEX(*inode));
	218	+ if (pipe->inode)
	219	+ mutex_unlock(&pipe->inode->i_mutex);
221	220
222	221	if (do_wakeup) {
223	222	smp_mb();
224		- if (waitqueue_active(PIPE_WAIT(*inode)))
225		- wake_up_interruptible(PIPE_WAIT(*inode));
226		- kill_fasync(PIPE_FASYNC_READERS(*inode), SIGIO, POLL_IN);
	223	+ if (waitqueue_active(&pipe->wait))
	224	+ wake_up_interruptible(&pipe->wait);
	225	+ kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
227	226	}
228	227
229	228	while (i < nr_pages)
...	...	@@ -232,8 +231,9 @@
232	231	return ret;
233	232	}
234	233
235		-static int __generic_file_splice_read(struct file in, struct inode pipe,
236		- size_t len, unsigned int flags)
	234	+static int
	235	+__generic_file_splice_read(struct file in, struct pipe_inode_info pipe,
	236	+ size_t len, unsigned int flags)
237	237	{
238	238	struct address_space *mapping = in->f_mapping;
239	239	unsigned int offset, nr_pages;
...	...	@@ -298,7 +298,7 @@
298	298	* Will read pages from given file and fill them into a pipe.
299	299	*
300	300	*/
301		-ssize_t generic_file_splice_read(struct file in, struct inode pipe,
	301	+ssize_t generic_file_splice_read(struct file in, struct pipe_inode_info pipe,
302	302	size_t len, unsigned int flags)
303	303	{
304	304	ssize_t spliced;
...	...	@@ -306,6 +306,7 @@
306	306
307	307	ret = 0;
308	308	spliced = 0;
	309	+
309	310	while (len) {
310	311	ret = __generic_file_splice_read(in, pipe, len, flags);
311	312
312	313
...	...	@@ -509,11 +510,10 @@
509	510	* key here is the 'actor' worker passed in that actually moves the data
510	511	* to the wanted destination. See pipe_to_file/pipe_to_sendpage above.
511	512	*/
512		-static ssize_t move_from_pipe(struct inode inode, struct file out,
	513	+static ssize_t move_from_pipe(struct pipe_inode_info pipe, struct file out,
513	514	size_t len, unsigned int flags,
514	515	splice_actor *actor)
515	516	{
516		- struct pipe_inode_info *info;
517	517	int ret, do_wakeup, err;
518	518	struct splice_desc sd;
519	519
520	520
521	521
522	522
523	523
...	...	@@ -525,22 +525,22 @@
525	525	sd.file = out;
526	526	sd.pos = out->f_pos;
527	527
528		- mutex_lock(PIPE_MUTEX(*inode));
	528	+ if (pipe->inode)
	529	+ mutex_lock(&pipe->inode->i_mutex);
529	530
530		- info = inode->i_pipe;
531	531	for (;;) {
532		- int bufs = info->nrbufs;
	532	+ int bufs = pipe->nrbufs;
533	533
534	534	if (bufs) {
535		- int curbuf = info->curbuf;
536		- struct pipe_buffer *buf = info->bufs + curbuf;
	535	+ int curbuf = pipe->curbuf;
	536	+ struct pipe_buffer *buf = pipe->bufs + curbuf;
537	537	struct pipe_buf_operations *ops = buf->ops;
538	538
539	539	sd.len = buf->len;
540	540	if (sd.len > sd.total_len)
541	541	sd.len = sd.total_len;
542	542
543		- err = actor(info, buf, &sd);
	543	+ err = actor(pipe, buf, &sd);
544	544	if (err) {
545	545	if (!ret && err != -ENODATA)
546	546	ret = err;
547	547
...	...	@@ -553,10 +553,10 @@
553	553	buf->len -= sd.len;
554	554	if (!buf->len) {
555	555	buf->ops = NULL;
556		- ops->release(info, buf);
	556	+ ops->release(pipe, buf);
557	557	curbuf = (curbuf + 1) & (PIPE_BUFFERS - 1);
558		- info->curbuf = curbuf;
559		- info->nrbufs = --bufs;
	558	+ pipe->curbuf = curbuf;
	559	+ pipe->nrbufs = --bufs;
560	560	do_wakeup = 1;
561	561	}
562	562
563	563
...	...	@@ -568,9 +568,9 @@
568	568
569	569	if (bufs)
570	570	continue;
571		- if (!PIPE_WRITERS(*inode))
	571	+ if (!pipe->writers)
572	572	break;
573		- if (!PIPE_WAITING_WRITERS(*inode)) {
	573	+ if (!pipe->waiting_writers) {
574	574	if (ret)
575	575	break;
576	576	}
577	577
578	578
579	579
...	...	@@ -589,22 +589,23 @@
589	589
590	590	if (do_wakeup) {
591	591	smp_mb();
592		- if (waitqueue_active(PIPE_WAIT(*inode)))
593		- wake_up_interruptible_sync(PIPE_WAIT(*inode));
594		- kill_fasync(PIPE_FASYNC_WRITERS(*inode),SIGIO,POLL_OUT);
	592	+ if (waitqueue_active(&pipe->wait))
	593	+ wake_up_interruptible_sync(&pipe->wait);
	594	+ kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);
595	595	do_wakeup = 0;
596	596	}
597	597
598		- pipe_wait(inode);
	598	+ pipe_wait(pipe);
599	599	}
600	600
601		- mutex_unlock(PIPE_MUTEX(*inode));
	601	+ if (pipe->inode)
	602	+ mutex_unlock(&pipe->inode->i_mutex);
602	603
603	604	if (do_wakeup) {
604	605	smp_mb();
605		- if (waitqueue_active(PIPE_WAIT(*inode)))
606		- wake_up_interruptible(PIPE_WAIT(*inode));
607		- kill_fasync(PIPE_FASYNC_WRITERS(*inode), SIGIO, POLL_OUT);
	606	+ if (waitqueue_active(&pipe->wait))
	607	+ wake_up_interruptible(&pipe->wait);
	608	+ kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);
608	609	}
609	610
610	611	mutex_lock(&out->f_mapping->host->i_mutex);
...	...	@@ -616,7 +617,7 @@
616	617
617	618	/**
618	619	* generic_file_splice_write - splice data from a pipe to a file
619		- * @inode: pipe inode
	620	+ * @pipe: pipe info
620	621	* @out: file to write to
621	622	* @len: number of bytes to splice
622	623	* @flags: splice modifier flags
623	624
624	625
...	...	@@ -625,12 +626,15 @@
625	626	* the given pipe inode to the given file.
626	627	*
627	628	*/
628		-ssize_t generic_file_splice_write(struct inode inode, struct file out,
629		- size_t len, unsigned int flags)
	629	+ssize_t
	630	+generic_file_splice_write(struct pipe_inode_info pipe, struct file out,
	631	+ size_t len, unsigned int flags)
630	632	{
631	633	struct address_space *mapping = out->f_mapping;
632		- ssize_t ret = move_from_pipe(inode, out, len, flags, pipe_to_file);
	634	+ ssize_t ret;
633	635
	636	+ ret = move_from_pipe(pipe, out, len, flags, pipe_to_file);
	637	+
634	638	/*
635	639	* if file or inode is SYNC and we actually wrote some data, sync it
636	640	*/
637	641
...	...	@@ -664,10 +668,10 @@
664	668	* is involved.
665	669	*
666	670	*/
667		-ssize_t generic_splice_sendpage(struct inode inode, struct file out,
	671	+ssize_t generic_splice_sendpage(struct pipe_inode_info pipe, struct file out,
668	672	size_t len, unsigned int flags)
669	673	{
670		- return move_from_pipe(inode, out, len, flags, pipe_to_sendpage);
	674	+ return move_from_pipe(pipe, out, len, flags, pipe_to_sendpage);
671	675	}
672	676
673	677	EXPORT_SYMBOL(generic_splice_sendpage);
...	...	@@ -675,8 +679,8 @@
675	679	/*
676	680	* Attempt to initiate a splice from pipe to file.
677	681	*/
678		-static long do_splice_from(struct inode pipe, struct file out, size_t len,
679		- unsigned int flags)
	682	+static long do_splice_from(struct pipe_inode_info pipe, struct file out,
	683	+ size_t len, unsigned int flags)
680	684	{
681	685	loff_t pos;
682	686	int ret;
...	...	@@ -698,8 +702,8 @@
698	702	/*
699	703	* Attempt to initiate a splice from a file to a pipe.
700	704	*/
701		-static long do_splice_to(struct file in, struct inode pipe, size_t len,
702		- unsigned int flags)
	705	+static long do_splice_to(struct file in, struct pipe_inode_info pipe,
	706	+ size_t len, unsigned int flags)
703	707	{
704	708	loff_t pos, isize, left;
705	709	int ret;
706	710
707	711
...	...	@@ -732,14 +736,14 @@
732	736	static long do_splice(struct file in, struct file out, size_t len,
733	737	unsigned int flags)
734	738	{
735		- struct inode *pipe;
	739	+ struct pipe_inode_info *pipe;
736	740
737		- pipe = in->f_dentry->d_inode;
738		- if (pipe->i_pipe)
	741	+ pipe = in->f_dentry->d_inode->i_pipe;
	742	+ if (pipe)
739	743	return do_splice_from(pipe, out, len, flags);
740	744
741		- pipe = out->f_dentry->d_inode;
742		- if (pipe->i_pipe)
	745	+ pipe = out->f_dentry->d_inode->i_pipe;
	746	+ if (pipe)
743	747	return do_splice_to(in, pipe, len, flags);
744	748
745	749	return -EINVAL;
...	...	@@ -252,7 +252,7 @@
252	252	STATIC ssize_t
253	253	xfs_file_splice_read(
254	254	struct file *infilp,
255		- struct inode *pipe,
	255	+ struct pipe_inode_info *pipe,
256	256	size_t len,
257	257	unsigned int flags)
258	258	{
...	...	@@ -266,7 +266,7 @@
266	266	STATIC ssize_t
267	267	xfs_file_splice_read_invis(
268	268	struct file *infilp,
269		- struct inode *pipe,
	269	+ struct pipe_inode_info *pipe,
270	270	size_t len,
271	271	unsigned int flags)
272	272	{
...	...	@@ -279,7 +279,7 @@
279	279
280	280	STATIC ssize_t
281	281	xfs_file_splice_write(
282		- struct inode *pipe,
	282	+ struct pipe_inode_info *pipe,
283	283	struct file *outfilp,
284	284	size_t len,
285	285	unsigned int flags)
...	...	@@ -293,7 +293,7 @@
293	293
294	294	STATIC ssize_t
295	295	xfs_file_splice_write_invis(
296		- struct inode *pipe,
	296	+ struct pipe_inode_info *pipe,
297	297	struct file *outfilp,
298	298	size_t len,
299	299	unsigned int flags)
...	...	@@ -338,7 +338,7 @@
338	338	xfs_splice_read(
339	339	bhv_desc_t *bdp,
340	340	struct file *infilp,
341		- struct inode *pipe,
	341	+ struct pipe_inode_info *pipe,
342	342	size_t count,
343	343	int flags,
344	344	int ioflags,
...	...	@@ -380,7 +380,7 @@
380	380	ssize_t
381	381	xfs_splice_write(
382	382	bhv_desc_t *bdp,
383		- struct inode *pipe,
	383	+ struct pipe_inode_info *pipe,
384	384	struct file *outfilp,
385	385	size_t count,
386	386	int flags,
...	...	@@ -94,9 +94,9 @@
94	94	loff_t *, int, size_t, read_actor_t,
95	95	void , struct cred );
96	96	extern ssize_t xfs_splice_read(struct bhv_desc , struct file ,
97		- struct inode *, size_t, int, int,
	97	+ struct pipe_inode_info *, size_t, int, int,
98	98	struct cred *);
99		-extern ssize_t xfs_splice_write(struct bhv_desc , struct inode ,
	99	+extern ssize_t xfs_splice_write(struct bhv_desc , struct pipe_inode_info ,
100	100	struct file *, size_t, int, int,
101	101	struct cred *);
102	102
...	...	@@ -174,9 +174,9 @@
174	174	loff_t *, int, size_t, read_actor_t,
175	175	void , struct cred );
176	176	typedef ssize_t (vop_splice_read_t)(bhv_desc_t , struct file *,
177		- struct inode *, size_t, int, int,
	177	+ struct pipe_inode_info *, size_t, int, int,
178	178	struct cred *);
179		-typedef ssize_t (vop_splice_write_t)(bhv_desc_t , struct inode *,
	179	+typedef ssize_t (vop_splice_write_t)(bhv_desc_t , struct pipe_inode_info *,
180	180	struct file *, size_t, int, int,
181	181	struct cred *);
182	182	typedef int (vop_ioctl_t)(bhv_desc_t , struct inode , struct file ,
...	...	@@ -1039,8 +1039,8 @@
1039	1039	int (*check_flags)(int);
1040	1040	int (dir_notify)(struct file filp, unsigned long arg);
1041	1041	int (flock) (struct file , int, struct file_lock *);
1042		- ssize_t (splice_write)(struct inode , struct file *, size_t, unsigned int);
1043		- ssize_t (splice_read)(struct file , struct inode *, size_t, unsigned int);
	1042	+ ssize_t (splice_write)(struct pipe_inode_info , struct file *, size_t, unsigned int);
	1043	+ ssize_t (splice_read)(struct file , struct pipe_inode_info *, size_t, unsigned int);
1044	1044	};
1045	1045
1046	1046	struct inode_operations {
...	...	@@ -1611,8 +1611,8 @@
1611	1611	extern void do_generic_mapping_read(struct address_space *mapping,
1612	1612	struct file_ra_state , struct file ,
1613	1613	loff_t , read_descriptor_t , read_actor_t);
1614		-extern ssize_t generic_file_splice_read(struct file , struct inode , size_t, unsigned int);
1615		-extern ssize_t generic_file_splice_write(struct inode , struct file , size_t, unsigned int);
	1614	+extern ssize_t generic_file_splice_read(struct file , struct pipe_inode_info , size_t, unsigned int);
	1615	+extern ssize_t generic_file_splice_write(struct pipe_inode_info , struct file , size_t, unsigned int);
1616	1616	extern void
1617	1617	file_ra_state_init(struct file_ra_state ra, struct address_space mapping);
1618	1618	extern ssize_t generic_file_readv(struct file filp, const struct iovec iov,
...	...	@@ -36,6 +36,7 @@
36	36	unsigned int w_counter;
37	37	struct fasync_struct *fasync_readers;
38	38	struct fasync_struct *fasync_writers;
	39	+ struct inode *inode;
39	40	};
40	41
41	42	/* Differs from PIPE_BUF in that PIPE_SIZE is the length of the actual
42	43
...	...	@@ -53,10 +54,10 @@
53	54	#define PIPE_FASYNC_WRITERS(inode) (&((inode).i_pipe->fasync_writers))
54	55
55	56	/* Drop the inode semaphore and wait for a pipe event, atomically */
56		-void pipe_wait(struct inode * inode);
	57	+void pipe_wait(struct pipe_inode_info *pipe);
57	58
58		-struct inode* pipe_new(struct inode* inode);
59		-void free_pipe_info(struct inode* inode);
	59	+struct pipe_inode_info * alloc_pipe_info(struct inode * inode);
	60	+void free_pipe_info(struct inode * inode);
60	61
61	62	/*
62	63	* splice is tied to pipes as a transport (at least for now), so we'll just