Doug / smarc-fsl-linux-kernel

Download as
Browse Code ยป

Commit 7bee130e222dfb3a7a70c0404dc09f104cddd7d6

Authored by Al Viro
1 parent 6447a3cf19
Exists in smarc-l5.0.0_1.0.0-ga and in 5 other branches imx_3.10.17_1.0.1_ga, imx_3.10.53_1.1.0_ga, imx_3.14.28_1.0.0_ga, smarc-imx_3.10.53_1.1.0_ga, smarc-imx_3.14.28_1.0.0_ga

get rid of alloc_pipe_info() argument 

not used anymore

Signed-off-by: Al Viro <viro@zeniv.linux.org.uk>

Showing 3 changed files with 5 additions and 5 deletions Inline Diff

1 /* 1 /*
2 * linux/fs/pipe.c 2 * linux/fs/pipe.c
3 * 3 *
4 * Copyright (C) 1991, 1992, 1999 Linus Torvalds 4 * Copyright (C) 1991, 1992, 1999 Linus Torvalds
5 */ 5 */
6 6
7 #include <linux/mm.h> 7 #include <linux/mm.h>
8 #include <linux/file.h> 8 #include <linux/file.h>
9 #include <linux/poll.h> 9 #include <linux/poll.h>
10 #include <linux/slab.h> 10 #include <linux/slab.h>
11 #include <linux/module.h> 11 #include <linux/module.h>
12 #include <linux/init.h> 12 #include <linux/init.h>
13 #include <linux/fs.h> 13 #include <linux/fs.h>
14 #include <linux/log2.h> 14 #include <linux/log2.h>
15 #include <linux/mount.h> 15 #include <linux/mount.h>
16 #include <linux/magic.h> 16 #include <linux/magic.h>
17 #include <linux/pipe_fs_i.h> 17 #include <linux/pipe_fs_i.h>
18 #include <linux/uio.h> 18 #include <linux/uio.h>
19 #include <linux/highmem.h> 19 #include <linux/highmem.h>
20 #include <linux/pagemap.h> 20 #include <linux/pagemap.h>
21 #include <linux/audit.h> 21 #include <linux/audit.h>
22 #include <linux/syscalls.h> 22 #include <linux/syscalls.h>
23 #include <linux/fcntl.h> 23 #include <linux/fcntl.h>
24 24
25 #include <asm/uaccess.h> 25 #include <asm/uaccess.h>
26 #include <asm/ioctls.h> 26 #include <asm/ioctls.h>
27 27
28 #include "internal.h" 28 #include "internal.h"
29 29
30 /* 30 /*
31 * The max size that a non-root user is allowed to grow the pipe. Can 31 * The max size that a non-root user is allowed to grow the pipe. Can
32 * be set by root in /proc/sys/fs/pipe-max-size 32 * be set by root in /proc/sys/fs/pipe-max-size
33 */ 33 */
34 unsigned int pipe_max_size = 1048576; 34 unsigned int pipe_max_size = 1048576;
35 35
36 /* 36 /*
37 * Minimum pipe size, as required by POSIX 37 * Minimum pipe size, as required by POSIX
38 */ 38 */
39 unsigned int pipe_min_size = PAGE_SIZE; 39 unsigned int pipe_min_size = PAGE_SIZE;
40 40
41 /* 41 /*
42 * We use a start+len construction, which provides full use of the 42 * We use a start+len construction, which provides full use of the
43 * allocated memory. 43 * allocated memory.
44 * -- Florian Coosmann (FGC) 44 * -- Florian Coosmann (FGC)
45 * 45 *
46 * Reads with count = 0 should always return 0. 46 * Reads with count = 0 should always return 0.
47 * -- Julian Bradfield 1999-06-07. 47 * -- Julian Bradfield 1999-06-07.
48 * 48 *
49 * FIFOs and Pipes now generate SIGIO for both readers and writers. 49 * FIFOs and Pipes now generate SIGIO for both readers and writers.
50 * -- Jeremy Elson <jelson@circlemud.org> 2001-08-16 50 * -- Jeremy Elson <jelson@circlemud.org> 2001-08-16
51 * 51 *
52 * pipe_read & write cleanup 52 * pipe_read & write cleanup
53 * -- Manfred Spraul <manfred@colorfullife.com> 2002-05-09 53 * -- Manfred Spraul <manfred@colorfullife.com> 2002-05-09
54 */ 54 */
55 55
56 static void pipe_lock_nested(struct pipe_inode_info *pipe, int subclass) 56 static void pipe_lock_nested(struct pipe_inode_info *pipe, int subclass)
57 { 57 {
58 if (pipe->files) 58 if (pipe->files)
59 mutex_lock_nested(&pipe->mutex, subclass); 59 mutex_lock_nested(&pipe->mutex, subclass);
60 } 60 }
61 61
62 void pipe_lock(struct pipe_inode_info *pipe) 62 void pipe_lock(struct pipe_inode_info *pipe)
63 { 63 {
64 /* 64 /*
65 * pipe_lock() nests non-pipe inode locks (for writing to a file) 65 * pipe_lock() nests non-pipe inode locks (for writing to a file)
66 */ 66 */
67 pipe_lock_nested(pipe, I_MUTEX_PARENT); 67 pipe_lock_nested(pipe, I_MUTEX_PARENT);
68 } 68 }
69 EXPORT_SYMBOL(pipe_lock); 69 EXPORT_SYMBOL(pipe_lock);
70 70
71 void pipe_unlock(struct pipe_inode_info *pipe) 71 void pipe_unlock(struct pipe_inode_info *pipe)
72 { 72 {
73 if (pipe->files) 73 if (pipe->files)
74 mutex_unlock(&pipe->mutex); 74 mutex_unlock(&pipe->mutex);
75 } 75 }
76 EXPORT_SYMBOL(pipe_unlock); 76 EXPORT_SYMBOL(pipe_unlock);
77 77
78 static inline void __pipe_lock(struct pipe_inode_info *pipe) 78 static inline void __pipe_lock(struct pipe_inode_info *pipe)
79 { 79 {
80 mutex_lock_nested(&pipe->mutex, I_MUTEX_PARENT); 80 mutex_lock_nested(&pipe->mutex, I_MUTEX_PARENT);
81 } 81 }
82 82
83 static inline void __pipe_unlock(struct pipe_inode_info *pipe) 83 static inline void __pipe_unlock(struct pipe_inode_info *pipe)
84 { 84 {
85 mutex_unlock(&pipe->mutex); 85 mutex_unlock(&pipe->mutex);
86 } 86 }
87 87
88 void pipe_double_lock(struct pipe_inode_info *pipe1, 88 void pipe_double_lock(struct pipe_inode_info *pipe1,
89 struct pipe_inode_info *pipe2) 89 struct pipe_inode_info *pipe2)
90 { 90 {
91 BUG_ON(pipe1 == pipe2); 91 BUG_ON(pipe1 == pipe2);
92 92
93 if (pipe1 < pipe2) { 93 if (pipe1 < pipe2) {
94 pipe_lock_nested(pipe1, I_MUTEX_PARENT); 94 pipe_lock_nested(pipe1, I_MUTEX_PARENT);
95 pipe_lock_nested(pipe2, I_MUTEX_CHILD); 95 pipe_lock_nested(pipe2, I_MUTEX_CHILD);
96 } else { 96 } else {
97 pipe_lock_nested(pipe2, I_MUTEX_PARENT); 97 pipe_lock_nested(pipe2, I_MUTEX_PARENT);
98 pipe_lock_nested(pipe1, I_MUTEX_CHILD); 98 pipe_lock_nested(pipe1, I_MUTEX_CHILD);
99 } 99 }
100 } 100 }
101 101
102 /* Drop the inode semaphore and wait for a pipe event, atomically */ 102 /* Drop the inode semaphore and wait for a pipe event, atomically */
103 void pipe_wait(struct pipe_inode_info *pipe) 103 void pipe_wait(struct pipe_inode_info *pipe)
104 { 104 {
105 DEFINE_WAIT(wait); 105 DEFINE_WAIT(wait);
106 106
107 /* 107 /*
108 * Pipes are system-local resources, so sleeping on them 108 * Pipes are system-local resources, so sleeping on them
109 * is considered a noninteractive wait: 109 * is considered a noninteractive wait:
110 */ 110 */
111 prepare_to_wait(&pipe->wait, &wait, TASK_INTERRUPTIBLE); 111 prepare_to_wait(&pipe->wait, &wait, TASK_INTERRUPTIBLE);
112 pipe_unlock(pipe); 112 pipe_unlock(pipe);
113 schedule(); 113 schedule();
114 finish_wait(&pipe->wait, &wait); 114 finish_wait(&pipe->wait, &wait);
115 pipe_lock(pipe); 115 pipe_lock(pipe);
116 } 116 }
117 117
118 static int 118 static int
119 pipe_iov_copy_from_user(void *to, struct iovec *iov, unsigned long len, 119 pipe_iov_copy_from_user(void *to, struct iovec *iov, unsigned long len,
120 int atomic) 120 int atomic)
121 { 121 {
122 unsigned long copy; 122 unsigned long copy;
123 123
124 while (len > 0) { 124 while (len > 0) {
125 while (!iov->iov_len) 125 while (!iov->iov_len)
126 iov++; 126 iov++;
127 copy = min_t(unsigned long, len, iov->iov_len); 127 copy = min_t(unsigned long, len, iov->iov_len);
128 128
129 if (atomic) { 129 if (atomic) {
130 if (__copy_from_user_inatomic(to, iov->iov_base, copy)) 130 if (__copy_from_user_inatomic(to, iov->iov_base, copy))
131 return -EFAULT; 131 return -EFAULT;
132 } else { 132 } else {
133 if (copy_from_user(to, iov->iov_base, copy)) 133 if (copy_from_user(to, iov->iov_base, copy))
134 return -EFAULT; 134 return -EFAULT;
135 } 135 }
136 to += copy; 136 to += copy;
137 len -= copy; 137 len -= copy;
138 iov->iov_base += copy; 138 iov->iov_base += copy;
139 iov->iov_len -= copy; 139 iov->iov_len -= copy;
140 } 140 }
141 return 0; 141 return 0;
142 } 142 }
143 143
144 static int 144 static int
145 pipe_iov_copy_to_user(struct iovec *iov, const void *from, unsigned long len, 145 pipe_iov_copy_to_user(struct iovec *iov, const void *from, unsigned long len,
146 int atomic) 146 int atomic)
147 { 147 {
148 unsigned long copy; 148 unsigned long copy;
149 149
150 while (len > 0) { 150 while (len > 0) {
151 while (!iov->iov_len) 151 while (!iov->iov_len)
152 iov++; 152 iov++;
153 copy = min_t(unsigned long, len, iov->iov_len); 153 copy = min_t(unsigned long, len, iov->iov_len);
154 154
155 if (atomic) { 155 if (atomic) {
156 if (__copy_to_user_inatomic(iov->iov_base, from, copy)) 156 if (__copy_to_user_inatomic(iov->iov_base, from, copy))
157 return -EFAULT; 157 return -EFAULT;
158 } else { 158 } else {
159 if (copy_to_user(iov->iov_base, from, copy)) 159 if (copy_to_user(iov->iov_base, from, copy))
160 return -EFAULT; 160 return -EFAULT;
161 } 161 }
162 from += copy; 162 from += copy;
163 len -= copy; 163 len -= copy;
164 iov->iov_base += copy; 164 iov->iov_base += copy;
165 iov->iov_len -= copy; 165 iov->iov_len -= copy;
166 } 166 }
167 return 0; 167 return 0;
168 } 168 }
169 169
170 /* 170 /*
171 * Attempt to pre-fault in the user memory, so we can use atomic copies. 171 * Attempt to pre-fault in the user memory, so we can use atomic copies.
172 * Returns the number of bytes not faulted in. 172 * Returns the number of bytes not faulted in.
173 */ 173 */
174 static int iov_fault_in_pages_write(struct iovec *iov, unsigned long len) 174 static int iov_fault_in_pages_write(struct iovec *iov, unsigned long len)
175 { 175 {
176 while (!iov->iov_len) 176 while (!iov->iov_len)
177 iov++; 177 iov++;
178 178
179 while (len > 0) { 179 while (len > 0) {
180 unsigned long this_len; 180 unsigned long this_len;
181 181
182 this_len = min_t(unsigned long, len, iov->iov_len); 182 this_len = min_t(unsigned long, len, iov->iov_len);
183 if (fault_in_pages_writeable(iov->iov_base, this_len)) 183 if (fault_in_pages_writeable(iov->iov_base, this_len))
184 break; 184 break;
185 185
186 len -= this_len; 186 len -= this_len;
187 iov++; 187 iov++;
188 } 188 }
189 189
190 return len; 190 return len;
191 } 191 }
192 192
193 /* 193 /*
194 * Pre-fault in the user memory, so we can use atomic copies. 194 * Pre-fault in the user memory, so we can use atomic copies.
195 */ 195 */
196 static void iov_fault_in_pages_read(struct iovec *iov, unsigned long len) 196 static void iov_fault_in_pages_read(struct iovec *iov, unsigned long len)
197 { 197 {
198 while (!iov->iov_len) 198 while (!iov->iov_len)
199 iov++; 199 iov++;
200 200
201 while (len > 0) { 201 while (len > 0) {
202 unsigned long this_len; 202 unsigned long this_len;
203 203
204 this_len = min_t(unsigned long, len, iov->iov_len); 204 this_len = min_t(unsigned long, len, iov->iov_len);
205 fault_in_pages_readable(iov->iov_base, this_len); 205 fault_in_pages_readable(iov->iov_base, this_len);
206 len -= this_len; 206 len -= this_len;
207 iov++; 207 iov++;
208 } 208 }
209 } 209 }
210 210
211 static void anon_pipe_buf_release(struct pipe_inode_info *pipe, 211 static void anon_pipe_buf_release(struct pipe_inode_info *pipe,
212 struct pipe_buffer *buf) 212 struct pipe_buffer *buf)
213 { 213 {
214 struct page *page = buf->page; 214 struct page *page = buf->page;
215 215
216 /* 216 /*
217 * If nobody else uses this page, and we don't already have a 217 * If nobody else uses this page, and we don't already have a
218 * temporary page, let's keep track of it as a one-deep 218 * temporary page, let's keep track of it as a one-deep
219 * allocation cache. (Otherwise just release our reference to it) 219 * allocation cache. (Otherwise just release our reference to it)
220 */ 220 */
221 if (page_count(page) == 1 && !pipe->tmp_page) 221 if (page_count(page) == 1 && !pipe->tmp_page)
222 pipe->tmp_page = page; 222 pipe->tmp_page = page;
223 else 223 else
224 page_cache_release(page); 224 page_cache_release(page);
225 } 225 }
226 226
227 /** 227 /**
228 * generic_pipe_buf_map - virtually map a pipe buffer 228 * generic_pipe_buf_map - virtually map a pipe buffer
229 * @pipe: the pipe that the buffer belongs to 229 * @pipe: the pipe that the buffer belongs to
230 * @buf: the buffer that should be mapped 230 * @buf: the buffer that should be mapped
231 * @atomic: whether to use an atomic map 231 * @atomic: whether to use an atomic map
232 * 232 *
233 * Description: 233 * Description:
234 * This function returns a kernel virtual address mapping for the 234 * This function returns a kernel virtual address mapping for the
235 * pipe_buffer passed in @buf. If @atomic is set, an atomic map is provided 235 * pipe_buffer passed in @buf. If @atomic is set, an atomic map is provided
236 * and the caller has to be careful not to fault before calling 236 * and the caller has to be careful not to fault before calling
237 * the unmap function. 237 * the unmap function.
238 * 238 *
239 * Note that this function calls kmap_atomic() if @atomic != 0. 239 * Note that this function calls kmap_atomic() if @atomic != 0.
240 */ 240 */
241 void *generic_pipe_buf_map(struct pipe_inode_info *pipe, 241 void *generic_pipe_buf_map(struct pipe_inode_info *pipe,
242 struct pipe_buffer *buf, int atomic) 242 struct pipe_buffer *buf, int atomic)
243 { 243 {
244 if (atomic) { 244 if (atomic) {
245 buf->flags |= PIPE_BUF_FLAG_ATOMIC; 245 buf->flags |= PIPE_BUF_FLAG_ATOMIC;
246 return kmap_atomic(buf->page); 246 return kmap_atomic(buf->page);
247 } 247 }
248 248
249 return kmap(buf->page); 249 return kmap(buf->page);
250 } 250 }
251 EXPORT_SYMBOL(generic_pipe_buf_map); 251 EXPORT_SYMBOL(generic_pipe_buf_map);
252 252
253 /** 253 /**
254 * generic_pipe_buf_unmap - unmap a previously mapped pipe buffer 254 * generic_pipe_buf_unmap - unmap a previously mapped pipe buffer
255 * @pipe: the pipe that the buffer belongs to 255 * @pipe: the pipe that the buffer belongs to
256 * @buf: the buffer that should be unmapped 256 * @buf: the buffer that should be unmapped
257 * @map_data: the data that the mapping function returned 257 * @map_data: the data that the mapping function returned
258 * 258 *
259 * Description: 259 * Description:
260 * This function undoes the mapping that ->map() provided. 260 * This function undoes the mapping that ->map() provided.
261 */ 261 */
262 void generic_pipe_buf_unmap(struct pipe_inode_info *pipe, 262 void generic_pipe_buf_unmap(struct pipe_inode_info *pipe,
263 struct pipe_buffer *buf, void *map_data) 263 struct pipe_buffer *buf, void *map_data)
264 { 264 {
265 if (buf->flags & PIPE_BUF_FLAG_ATOMIC) { 265 if (buf->flags & PIPE_BUF_FLAG_ATOMIC) {
266 buf->flags &= ~PIPE_BUF_FLAG_ATOMIC; 266 buf->flags &= ~PIPE_BUF_FLAG_ATOMIC;
267 kunmap_atomic(map_data); 267 kunmap_atomic(map_data);
268 } else 268 } else
269 kunmap(buf->page); 269 kunmap(buf->page);
270 } 270 }
271 EXPORT_SYMBOL(generic_pipe_buf_unmap); 271 EXPORT_SYMBOL(generic_pipe_buf_unmap);
272 272
273 /** 273 /**
274 * generic_pipe_buf_steal - attempt to take ownership of a &pipe_buffer 274 * generic_pipe_buf_steal - attempt to take ownership of a &pipe_buffer
275 * @pipe: the pipe that the buffer belongs to 275 * @pipe: the pipe that the buffer belongs to
276 * @buf: the buffer to attempt to steal 276 * @buf: the buffer to attempt to steal
277 * 277 *
278 * Description: 278 * Description:
279 * This function attempts to steal the &struct page attached to 279 * This function attempts to steal the &struct page attached to
280 * @buf. If successful, this function returns 0 and returns with 280 * @buf. If successful, this function returns 0 and returns with
281 * the page locked. The caller may then reuse the page for whatever 281 * the page locked. The caller may then reuse the page for whatever
282 * he wishes; the typical use is insertion into a different file 282 * he wishes; the typical use is insertion into a different file
283 * page cache. 283 * page cache.
284 */ 284 */
285 int generic_pipe_buf_steal(struct pipe_inode_info *pipe, 285 int generic_pipe_buf_steal(struct pipe_inode_info *pipe,
286 struct pipe_buffer *buf) 286 struct pipe_buffer *buf)
287 { 287 {
288 struct page *page = buf->page; 288 struct page *page = buf->page;
289 289
290 /* 290 /*
291 * A reference of one is golden, that means that the owner of this 291 * A reference of one is golden, that means that the owner of this
292 * page is the only one holding a reference to it. lock the page 292 * page is the only one holding a reference to it. lock the page
293 * and return OK. 293 * and return OK.
294 */ 294 */
295 if (page_count(page) == 1) { 295 if (page_count(page) == 1) {
296 lock_page(page); 296 lock_page(page);
297 return 0; 297 return 0;
298 } 298 }
299 299
300 return 1; 300 return 1;
301 } 301 }
302 EXPORT_SYMBOL(generic_pipe_buf_steal); 302 EXPORT_SYMBOL(generic_pipe_buf_steal);
303 303
304 /** 304 /**
305 * generic_pipe_buf_get - get a reference to a &struct pipe_buffer 305 * generic_pipe_buf_get - get a reference to a &struct pipe_buffer
306 * @pipe: the pipe that the buffer belongs to 306 * @pipe: the pipe that the buffer belongs to
307 * @buf: the buffer to get a reference to 307 * @buf: the buffer to get a reference to
308 * 308 *
309 * Description: 309 * Description:
310 * This function grabs an extra reference to @buf. It's used in 310 * This function grabs an extra reference to @buf. It's used in
311 * in the tee() system call, when we duplicate the buffers in one 311 * in the tee() system call, when we duplicate the buffers in one
312 * pipe into another. 312 * pipe into another.
313 */ 313 */
314 void generic_pipe_buf_get(struct pipe_inode_info *pipe, struct pipe_buffer *buf) 314 void generic_pipe_buf_get(struct pipe_inode_info *pipe, struct pipe_buffer *buf)
315 { 315 {
316 page_cache_get(buf->page); 316 page_cache_get(buf->page);
317 } 317 }
318 EXPORT_SYMBOL(generic_pipe_buf_get); 318 EXPORT_SYMBOL(generic_pipe_buf_get);
319 319
320 /** 320 /**
321 * generic_pipe_buf_confirm - verify contents of the pipe buffer 321 * generic_pipe_buf_confirm - verify contents of the pipe buffer
322 * @info: the pipe that the buffer belongs to 322 * @info: the pipe that the buffer belongs to
323 * @buf: the buffer to confirm 323 * @buf: the buffer to confirm
324 * 324 *
325 * Description: 325 * Description:
326 * This function does nothing, because the generic pipe code uses 326 * This function does nothing, because the generic pipe code uses
327 * pages that are always good when inserted into the pipe. 327 * pages that are always good when inserted into the pipe.
328 */ 328 */
329 int generic_pipe_buf_confirm(struct pipe_inode_info *info, 329 int generic_pipe_buf_confirm(struct pipe_inode_info *info,
330 struct pipe_buffer *buf) 330 struct pipe_buffer *buf)
331 { 331 {
332 return 0; 332 return 0;
333 } 333 }
334 EXPORT_SYMBOL(generic_pipe_buf_confirm); 334 EXPORT_SYMBOL(generic_pipe_buf_confirm);
335 335
336 /** 336 /**
337 * generic_pipe_buf_release - put a reference to a &struct pipe_buffer 337 * generic_pipe_buf_release - put a reference to a &struct pipe_buffer
338 * @pipe: the pipe that the buffer belongs to 338 * @pipe: the pipe that the buffer belongs to
339 * @buf: the buffer to put a reference to 339 * @buf: the buffer to put a reference to
340 * 340 *
341 * Description: 341 * Description:
342 * This function releases a reference to @buf. 342 * This function releases a reference to @buf.
343 */ 343 */
344 void generic_pipe_buf_release(struct pipe_inode_info *pipe, 344 void generic_pipe_buf_release(struct pipe_inode_info *pipe,
345 struct pipe_buffer *buf) 345 struct pipe_buffer *buf)
346 { 346 {
347 page_cache_release(buf->page); 347 page_cache_release(buf->page);
348 } 348 }
349 EXPORT_SYMBOL(generic_pipe_buf_release); 349 EXPORT_SYMBOL(generic_pipe_buf_release);
350 350
351 static const struct pipe_buf_operations anon_pipe_buf_ops = { 351 static const struct pipe_buf_operations anon_pipe_buf_ops = {
352 .can_merge = 1, 352 .can_merge = 1,
353 .map = generic_pipe_buf_map, 353 .map = generic_pipe_buf_map,
354 .unmap = generic_pipe_buf_unmap, 354 .unmap = generic_pipe_buf_unmap,
355 .confirm = generic_pipe_buf_confirm, 355 .confirm = generic_pipe_buf_confirm,
356 .release = anon_pipe_buf_release, 356 .release = anon_pipe_buf_release,
357 .steal = generic_pipe_buf_steal, 357 .steal = generic_pipe_buf_steal,
358 .get = generic_pipe_buf_get, 358 .get = generic_pipe_buf_get,
359 }; 359 };
360 360
361 static const struct pipe_buf_operations packet_pipe_buf_ops = { 361 static const struct pipe_buf_operations packet_pipe_buf_ops = {
362 .can_merge = 0, 362 .can_merge = 0,
363 .map = generic_pipe_buf_map, 363 .map = generic_pipe_buf_map,
364 .unmap = generic_pipe_buf_unmap, 364 .unmap = generic_pipe_buf_unmap,
365 .confirm = generic_pipe_buf_confirm, 365 .confirm = generic_pipe_buf_confirm,
366 .release = anon_pipe_buf_release, 366 .release = anon_pipe_buf_release,
367 .steal = generic_pipe_buf_steal, 367 .steal = generic_pipe_buf_steal,
368 .get = generic_pipe_buf_get, 368 .get = generic_pipe_buf_get,
369 }; 369 };
370 370
371 static ssize_t 371 static ssize_t
372 pipe_read(struct kiocb *iocb, const struct iovec *_iov, 372 pipe_read(struct kiocb *iocb, const struct iovec *_iov,
373 unsigned long nr_segs, loff_t pos) 373 unsigned long nr_segs, loff_t pos)
374 { 374 {
375 struct file *filp = iocb->ki_filp; 375 struct file *filp = iocb->ki_filp;
376 struct pipe_inode_info *pipe = filp->private_data; 376 struct pipe_inode_info *pipe = filp->private_data;
377 int do_wakeup; 377 int do_wakeup;
378 ssize_t ret; 378 ssize_t ret;
379 struct iovec *iov = (struct iovec *)_iov; 379 struct iovec *iov = (struct iovec *)_iov;
380 size_t total_len; 380 size_t total_len;
381 381
382 total_len = iov_length(iov, nr_segs); 382 total_len = iov_length(iov, nr_segs);
383 /* Null read succeeds. */ 383 /* Null read succeeds. */
384 if (unlikely(total_len == 0)) 384 if (unlikely(total_len == 0))
385 return 0; 385 return 0;
386 386
387 do_wakeup = 0; 387 do_wakeup = 0;
388 ret = 0; 388 ret = 0;
389 __pipe_lock(pipe); 389 __pipe_lock(pipe);
390 for (;;) { 390 for (;;) {
391 int bufs = pipe->nrbufs; 391 int bufs = pipe->nrbufs;
392 if (bufs) { 392 if (bufs) {
393 int curbuf = pipe->curbuf; 393 int curbuf = pipe->curbuf;
394 struct pipe_buffer *buf = pipe->bufs + curbuf; 394 struct pipe_buffer *buf = pipe->bufs + curbuf;
395 const struct pipe_buf_operations *ops = buf->ops; 395 const struct pipe_buf_operations *ops = buf->ops;
396 void *addr; 396 void *addr;
397 size_t chars = buf->len; 397 size_t chars = buf->len;
398 int error, atomic; 398 int error, atomic;
399 399
400 if (chars > total_len) 400 if (chars > total_len)
401 chars = total_len; 401 chars = total_len;
402 402
403 error = ops->confirm(pipe, buf); 403 error = ops->confirm(pipe, buf);
404 if (error) { 404 if (error) {
405 if (!ret) 405 if (!ret)
406 ret = error; 406 ret = error;
407 break; 407 break;
408 } 408 }
409 409
410 atomic = !iov_fault_in_pages_write(iov, chars); 410 atomic = !iov_fault_in_pages_write(iov, chars);
411 redo: 411 redo:
412 addr = ops->map(pipe, buf, atomic); 412 addr = ops->map(pipe, buf, atomic);
413 error = pipe_iov_copy_to_user(iov, addr + buf->offset, chars, atomic); 413 error = pipe_iov_copy_to_user(iov, addr + buf->offset, chars, atomic);
414 ops->unmap(pipe, buf, addr); 414 ops->unmap(pipe, buf, addr);
415 if (unlikely(error)) { 415 if (unlikely(error)) {
416 /* 416 /*
417 * Just retry with the slow path if we failed. 417 * Just retry with the slow path if we failed.
418 */ 418 */
419 if (atomic) { 419 if (atomic) {
420 atomic = 0; 420 atomic = 0;
421 goto redo; 421 goto redo;
422 } 422 }
423 if (!ret) 423 if (!ret)
424 ret = error; 424 ret = error;
425 break; 425 break;
426 } 426 }
427 ret += chars; 427 ret += chars;
428 buf->offset += chars; 428 buf->offset += chars;
429 buf->len -= chars; 429 buf->len -= chars;
430 430
431 /* Was it a packet buffer? Clean up and exit */ 431 /* Was it a packet buffer? Clean up and exit */
432 if (buf->flags & PIPE_BUF_FLAG_PACKET) { 432 if (buf->flags & PIPE_BUF_FLAG_PACKET) {
433 total_len = chars; 433 total_len = chars;
434 buf->len = 0; 434 buf->len = 0;
435 } 435 }
436 436
437 if (!buf->len) { 437 if (!buf->len) {
438 buf->ops = NULL; 438 buf->ops = NULL;
439 ops->release(pipe, buf); 439 ops->release(pipe, buf);
440 curbuf = (curbuf + 1) & (pipe->buffers - 1); 440 curbuf = (curbuf + 1) & (pipe->buffers - 1);
441 pipe->curbuf = curbuf; 441 pipe->curbuf = curbuf;
442 pipe->nrbufs = --bufs; 442 pipe->nrbufs = --bufs;
443 do_wakeup = 1; 443 do_wakeup = 1;
444 } 444 }
445 total_len -= chars; 445 total_len -= chars;
446 if (!total_len) 446 if (!total_len)
447 break; /* common path: read succeeded */ 447 break; /* common path: read succeeded */
448 } 448 }
449 if (bufs) /* More to do? */ 449 if (bufs) /* More to do? */
450 continue; 450 continue;
451 if (!pipe->writers) 451 if (!pipe->writers)
452 break; 452 break;
453 if (!pipe->waiting_writers) { 453 if (!pipe->waiting_writers) {
454 /* syscall merging: Usually we must not sleep 454 /* syscall merging: Usually we must not sleep
455 * if O_NONBLOCK is set, or if we got some data. 455 * if O_NONBLOCK is set, or if we got some data.
456 * But if a writer sleeps in kernel space, then 456 * But if a writer sleeps in kernel space, then
457 * we can wait for that data without violating POSIX. 457 * we can wait for that data without violating POSIX.
458 */ 458 */
459 if (ret) 459 if (ret)
460 break; 460 break;
461 if (filp->f_flags & O_NONBLOCK) { 461 if (filp->f_flags & O_NONBLOCK) {
462 ret = -EAGAIN; 462 ret = -EAGAIN;
463 break; 463 break;
464 } 464 }
465 } 465 }
466 if (signal_pending(current)) { 466 if (signal_pending(current)) {
467 if (!ret) 467 if (!ret)
468 ret = -ERESTARTSYS; 468 ret = -ERESTARTSYS;
469 break; 469 break;
470 } 470 }
471 if (do_wakeup) { 471 if (do_wakeup) {
472 wake_up_interruptible_sync_poll(&pipe->wait, POLLOUT | POLLWRNORM); 472 wake_up_interruptible_sync_poll(&pipe->wait, POLLOUT | POLLWRNORM);
473 kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT); 473 kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);
474 } 474 }
475 pipe_wait(pipe); 475 pipe_wait(pipe);
476 } 476 }
477 __pipe_unlock(pipe); 477 __pipe_unlock(pipe);
478 478
479 /* Signal writers asynchronously that there is more room. */ 479 /* Signal writers asynchronously that there is more room. */
480 if (do_wakeup) { 480 if (do_wakeup) {
481 wake_up_interruptible_sync_poll(&pipe->wait, POLLOUT | POLLWRNORM); 481 wake_up_interruptible_sync_poll(&pipe->wait, POLLOUT | POLLWRNORM);
482 kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT); 482 kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);
483 } 483 }
484 if (ret > 0) 484 if (ret > 0)
485 file_accessed(filp); 485 file_accessed(filp);
486 return ret; 486 return ret;
487 } 487 }
488 488
489 static inline int is_packetized(struct file *file) 489 static inline int is_packetized(struct file *file)
490 { 490 {
491 return (file->f_flags & O_DIRECT) != 0; 491 return (file->f_flags & O_DIRECT) != 0;
492 } 492 }
493 493
494 static ssize_t 494 static ssize_t
495 pipe_write(struct kiocb *iocb, const struct iovec *_iov, 495 pipe_write(struct kiocb *iocb, const struct iovec *_iov,
496 unsigned long nr_segs, loff_t ppos) 496 unsigned long nr_segs, loff_t ppos)
497 { 497 {
498 struct file *filp = iocb->ki_filp; 498 struct file *filp = iocb->ki_filp;
499 struct pipe_inode_info *pipe = filp->private_data; 499 struct pipe_inode_info *pipe = filp->private_data;
500 ssize_t ret; 500 ssize_t ret;
501 int do_wakeup; 501 int do_wakeup;
502 struct iovec *iov = (struct iovec *)_iov; 502 struct iovec *iov = (struct iovec *)_iov;
503 size_t total_len; 503 size_t total_len;
504 ssize_t chars; 504 ssize_t chars;
505 505
506 total_len = iov_length(iov, nr_segs); 506 total_len = iov_length(iov, nr_segs);
507 /* Null write succeeds. */ 507 /* Null write succeeds. */
508 if (unlikely(total_len == 0)) 508 if (unlikely(total_len == 0))
509 return 0; 509 return 0;
510 510
511 do_wakeup = 0; 511 do_wakeup = 0;
512 ret = 0; 512 ret = 0;
513 __pipe_lock(pipe); 513 __pipe_lock(pipe);
514 514
515 if (!pipe->readers) { 515 if (!pipe->readers) {
516 send_sig(SIGPIPE, current, 0); 516 send_sig(SIGPIPE, current, 0);
517 ret = -EPIPE; 517 ret = -EPIPE;
518 goto out; 518 goto out;
519 } 519 }
520 520
521 /* We try to merge small writes */ 521 /* We try to merge small writes */
522 chars = total_len & (PAGE_SIZE-1); /* size of the last buffer */ 522 chars = total_len & (PAGE_SIZE-1); /* size of the last buffer */
523 if (pipe->nrbufs && chars != 0) { 523 if (pipe->nrbufs && chars != 0) {
524 int lastbuf = (pipe->curbuf + pipe->nrbufs - 1) & 524 int lastbuf = (pipe->curbuf + pipe->nrbufs - 1) &
525 (pipe->buffers - 1); 525 (pipe->buffers - 1);
526 struct pipe_buffer *buf = pipe->bufs + lastbuf; 526 struct pipe_buffer *buf = pipe->bufs + lastbuf;
527 const struct pipe_buf_operations *ops = buf->ops; 527 const struct pipe_buf_operations *ops = buf->ops;
528 int offset = buf->offset + buf->len; 528 int offset = buf->offset + buf->len;
529 529
530 if (ops->can_merge && offset + chars <= PAGE_SIZE) { 530 if (ops->can_merge && offset + chars <= PAGE_SIZE) {
531 int error, atomic = 1; 531 int error, atomic = 1;
532 void *addr; 532 void *addr;
533 533
534 error = ops->confirm(pipe, buf); 534 error = ops->confirm(pipe, buf);
535 if (error) 535 if (error)
536 goto out; 536 goto out;
537 537
538 iov_fault_in_pages_read(iov, chars); 538 iov_fault_in_pages_read(iov, chars);
539 redo1: 539 redo1:
540 addr = ops->map(pipe, buf, atomic); 540 addr = ops->map(pipe, buf, atomic);
541 error = pipe_iov_copy_from_user(offset + addr, iov, 541 error = pipe_iov_copy_from_user(offset + addr, iov,
542 chars, atomic); 542 chars, atomic);
543 ops->unmap(pipe, buf, addr); 543 ops->unmap(pipe, buf, addr);
544 ret = error; 544 ret = error;
545 do_wakeup = 1; 545 do_wakeup = 1;
546 if (error) { 546 if (error) {
547 if (atomic) { 547 if (atomic) {
548 atomic = 0; 548 atomic = 0;
549 goto redo1; 549 goto redo1;
550 } 550 }
551 goto out; 551 goto out;
552 } 552 }
553 buf->len += chars; 553 buf->len += chars;
554 total_len -= chars; 554 total_len -= chars;
555 ret = chars; 555 ret = chars;
556 if (!total_len) 556 if (!total_len)
557 goto out; 557 goto out;
558 } 558 }
559 } 559 }
560 560
561 for (;;) { 561 for (;;) {
562 int bufs; 562 int bufs;
563 563
564 if (!pipe->readers) { 564 if (!pipe->readers) {
565 send_sig(SIGPIPE, current, 0); 565 send_sig(SIGPIPE, current, 0);
566 if (!ret) 566 if (!ret)
567 ret = -EPIPE; 567 ret = -EPIPE;
568 break; 568 break;
569 } 569 }
570 bufs = pipe->nrbufs; 570 bufs = pipe->nrbufs;
571 if (bufs < pipe->buffers) { 571 if (bufs < pipe->buffers) {
572 int newbuf = (pipe->curbuf + bufs) & (pipe->buffers-1); 572 int newbuf = (pipe->curbuf + bufs) & (pipe->buffers-1);
573 struct pipe_buffer *buf = pipe->bufs + newbuf; 573 struct pipe_buffer *buf = pipe->bufs + newbuf;
574 struct page *page = pipe->tmp_page; 574 struct page *page = pipe->tmp_page;
575 char *src; 575 char *src;
576 int error, atomic = 1; 576 int error, atomic = 1;
577 577
578 if (!page) { 578 if (!page) {
579 page = alloc_page(GFP_HIGHUSER); 579 page = alloc_page(GFP_HIGHUSER);
580 if (unlikely(!page)) { 580 if (unlikely(!page)) {
581 ret = ret ? : -ENOMEM; 581 ret = ret ? : -ENOMEM;
582 break; 582 break;
583 } 583 }
584 pipe->tmp_page = page; 584 pipe->tmp_page = page;
585 } 585 }
586 /* Always wake up, even if the copy fails. Otherwise 586 /* Always wake up, even if the copy fails. Otherwise
587 * we lock up (O_NONBLOCK-)readers that sleep due to 587 * we lock up (O_NONBLOCK-)readers that sleep due to
588 * syscall merging. 588 * syscall merging.
589 * FIXME! Is this really true? 589 * FIXME! Is this really true?
590 */ 590 */
591 do_wakeup = 1; 591 do_wakeup = 1;
592 chars = PAGE_SIZE; 592 chars = PAGE_SIZE;
593 if (chars > total_len) 593 if (chars > total_len)
594 chars = total_len; 594 chars = total_len;
595 595
596 iov_fault_in_pages_read(iov, chars); 596 iov_fault_in_pages_read(iov, chars);
597 redo2: 597 redo2:
598 if (atomic) 598 if (atomic)
599 src = kmap_atomic(page); 599 src = kmap_atomic(page);
600 else 600 else
601 src = kmap(page); 601 src = kmap(page);
602 602
603 error = pipe_iov_copy_from_user(src, iov, chars, 603 error = pipe_iov_copy_from_user(src, iov, chars,
604 atomic); 604 atomic);
605 if (atomic) 605 if (atomic)
606 kunmap_atomic(src); 606 kunmap_atomic(src);
607 else 607 else
608 kunmap(page); 608 kunmap(page);
609 609
610 if (unlikely(error)) { 610 if (unlikely(error)) {
611 if (atomic) { 611 if (atomic) {
612 atomic = 0; 612 atomic = 0;
613 goto redo2; 613 goto redo2;
614 } 614 }
615 if (!ret) 615 if (!ret)
616 ret = error; 616 ret = error;
617 break; 617 break;
618 } 618 }
619 ret += chars; 619 ret += chars;
620 620
621 /* Insert it into the buffer array */ 621 /* Insert it into the buffer array */
622 buf->page = page; 622 buf->page = page;
623 buf->ops = &anon_pipe_buf_ops; 623 buf->ops = &anon_pipe_buf_ops;
624 buf->offset = 0; 624 buf->offset = 0;
625 buf->len = chars; 625 buf->len = chars;
626 buf->flags = 0; 626 buf->flags = 0;
627 if (is_packetized(filp)) { 627 if (is_packetized(filp)) {
628 buf->ops = &packet_pipe_buf_ops; 628 buf->ops = &packet_pipe_buf_ops;
629 buf->flags = PIPE_BUF_FLAG_PACKET; 629 buf->flags = PIPE_BUF_FLAG_PACKET;
630 } 630 }
631 pipe->nrbufs = ++bufs; 631 pipe->nrbufs = ++bufs;
632 pipe->tmp_page = NULL; 632 pipe->tmp_page = NULL;
633 633
634 total_len -= chars; 634 total_len -= chars;
635 if (!total_len) 635 if (!total_len)
636 break; 636 break;
637 } 637 }
638 if (bufs < pipe->buffers) 638 if (bufs < pipe->buffers)
639 continue; 639 continue;
640 if (filp->f_flags & O_NONBLOCK) { 640 if (filp->f_flags & O_NONBLOCK) {
641 if (!ret) 641 if (!ret)
642 ret = -EAGAIN; 642 ret = -EAGAIN;
643 break; 643 break;
644 } 644 }
645 if (signal_pending(current)) { 645 if (signal_pending(current)) {
646 if (!ret) 646 if (!ret)
647 ret = -ERESTARTSYS; 647 ret = -ERESTARTSYS;
648 break; 648 break;
649 } 649 }
650 if (do_wakeup) { 650 if (do_wakeup) {
651 wake_up_interruptible_sync_poll(&pipe->wait, POLLIN | POLLRDNORM); 651 wake_up_interruptible_sync_poll(&pipe->wait, POLLIN | POLLRDNORM);
652 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN); 652 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
653 do_wakeup = 0; 653 do_wakeup = 0;
654 } 654 }
655 pipe->waiting_writers++; 655 pipe->waiting_writers++;
656 pipe_wait(pipe); 656 pipe_wait(pipe);
657 pipe->waiting_writers--; 657 pipe->waiting_writers--;
658 } 658 }
659 out: 659 out:
660 __pipe_unlock(pipe); 660 __pipe_unlock(pipe);
661 if (do_wakeup) { 661 if (do_wakeup) {
662 wake_up_interruptible_sync_poll(&pipe->wait, POLLIN | POLLRDNORM); 662 wake_up_interruptible_sync_poll(&pipe->wait, POLLIN | POLLRDNORM);
663 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN); 663 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
664 } 664 }
665 if (ret > 0) { 665 if (ret > 0) {
666 int err = file_update_time(filp); 666 int err = file_update_time(filp);
667 if (err) 667 if (err)
668 ret = err; 668 ret = err;
669 } 669 }
670 return ret; 670 return ret;
671 } 671 }
672 672
673 static long pipe_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) 673 static long pipe_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
674 { 674 {
675 struct pipe_inode_info *pipe = filp->private_data; 675 struct pipe_inode_info *pipe = filp->private_data;
676 int count, buf, nrbufs; 676 int count, buf, nrbufs;
677 677
678 switch (cmd) { 678 switch (cmd) {
679 case FIONREAD: 679 case FIONREAD:
680 __pipe_lock(pipe); 680 __pipe_lock(pipe);
681 count = 0; 681 count = 0;
682 buf = pipe->curbuf; 682 buf = pipe->curbuf;
683 nrbufs = pipe->nrbufs; 683 nrbufs = pipe->nrbufs;
684 while (--nrbufs >= 0) { 684 while (--nrbufs >= 0) {
685 count += pipe->bufs[buf].len; 685 count += pipe->bufs[buf].len;
686 buf = (buf+1) & (pipe->buffers - 1); 686 buf = (buf+1) & (pipe->buffers - 1);
687 } 687 }
688 __pipe_unlock(pipe); 688 __pipe_unlock(pipe);
689 689
690 return put_user(count, (int __user *)arg); 690 return put_user(count, (int __user *)arg);
691 default: 691 default:
692 return -ENOIOCTLCMD; 692 return -ENOIOCTLCMD;
693 } 693 }
694 } 694 }
695 695
696 /* No kernel lock held - fine */ 696 /* No kernel lock held - fine */
697 static unsigned int 697 static unsigned int
698 pipe_poll(struct file *filp, poll_table *wait) 698 pipe_poll(struct file *filp, poll_table *wait)
699 { 699 {
700 unsigned int mask; 700 unsigned int mask;
701 struct pipe_inode_info *pipe = filp->private_data; 701 struct pipe_inode_info *pipe = filp->private_data;
702 int nrbufs; 702 int nrbufs;
703 703
704 poll_wait(filp, &pipe->wait, wait); 704 poll_wait(filp, &pipe->wait, wait);
705 705
706 /* Reading only -- no need for acquiring the semaphore. */ 706 /* Reading only -- no need for acquiring the semaphore. */
707 nrbufs = pipe->nrbufs; 707 nrbufs = pipe->nrbufs;
708 mask = 0; 708 mask = 0;
709 if (filp->f_mode & FMODE_READ) { 709 if (filp->f_mode & FMODE_READ) {
710 mask = (nrbufs > 0) ? POLLIN | POLLRDNORM : 0; 710 mask = (nrbufs > 0) ? POLLIN | POLLRDNORM : 0;
711 if (!pipe->writers && filp->f_version != pipe->w_counter) 711 if (!pipe->writers && filp->f_version != pipe->w_counter)
712 mask |= POLLHUP; 712 mask |= POLLHUP;
713 } 713 }
714 714
715 if (filp->f_mode & FMODE_WRITE) { 715 if (filp->f_mode & FMODE_WRITE) {
716 mask |= (nrbufs < pipe->buffers) ? POLLOUT | POLLWRNORM : 0; 716 mask |= (nrbufs < pipe->buffers) ? POLLOUT | POLLWRNORM : 0;
717 /* 717 /*
718 * Most Unices do not set POLLERR for FIFOs but on Linux they 718 * Most Unices do not set POLLERR for FIFOs but on Linux they
719 * behave exactly like pipes for poll(). 719 * behave exactly like pipes for poll().
720 */ 720 */
721 if (!pipe->readers) 721 if (!pipe->readers)
722 mask |= POLLERR; 722 mask |= POLLERR;
723 } 723 }
724 724
725 return mask; 725 return mask;
726 } 726 }
727 727
728 static int 728 static int
729 pipe_release(struct inode *inode, struct file *file) 729 pipe_release(struct inode *inode, struct file *file)
730 { 730 {
731 struct pipe_inode_info *pipe = inode->i_pipe; 731 struct pipe_inode_info *pipe = inode->i_pipe;
732 int kill = 0; 732 int kill = 0;
733 733
734 __pipe_lock(pipe); 734 __pipe_lock(pipe);
735 if (file->f_mode & FMODE_READ) 735 if (file->f_mode & FMODE_READ)
736 pipe->readers--; 736 pipe->readers--;
737 if (file->f_mode & FMODE_WRITE) 737 if (file->f_mode & FMODE_WRITE)
738 pipe->writers--; 738 pipe->writers--;
739 739
740 if (pipe->readers || pipe->writers) { 740 if (pipe->readers || pipe->writers) {
741 wake_up_interruptible_sync_poll(&pipe->wait, POLLIN | POLLOUT | POLLRDNORM | POLLWRNORM | POLLERR | POLLHUP); 741 wake_up_interruptible_sync_poll(&pipe->wait, POLLIN | POLLOUT | POLLRDNORM | POLLWRNORM | POLLERR | POLLHUP);
742 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN); 742 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
743 kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT); 743 kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);
744 } 744 }
745 spin_lock(&inode->i_lock); 745 spin_lock(&inode->i_lock);
746 if (!--pipe->files) { 746 if (!--pipe->files) {
747 inode->i_pipe = NULL; 747 inode->i_pipe = NULL;
748 kill = 1; 748 kill = 1;
749 } 749 }
750 spin_unlock(&inode->i_lock); 750 spin_unlock(&inode->i_lock);
751 __pipe_unlock(pipe); 751 __pipe_unlock(pipe);
752 752
753 if (kill) 753 if (kill)
754 __free_pipe_info(pipe); 754 __free_pipe_info(pipe);
755 755
756 return 0; 756 return 0;
757 } 757 }
758 758
759 static int 759 static int
760 pipe_fasync(int fd, struct file *filp, int on) 760 pipe_fasync(int fd, struct file *filp, int on)
761 { 761 {
762 struct pipe_inode_info *pipe = filp->private_data; 762 struct pipe_inode_info *pipe = filp->private_data;
763 int retval = 0; 763 int retval = 0;
764 764
765 __pipe_lock(pipe); 765 __pipe_lock(pipe);
766 if (filp->f_mode & FMODE_READ) 766 if (filp->f_mode & FMODE_READ)
767 retval = fasync_helper(fd, filp, on, &pipe->fasync_readers); 767 retval = fasync_helper(fd, filp, on, &pipe->fasync_readers);
768 if ((filp->f_mode & FMODE_WRITE) && retval >= 0) { 768 if ((filp->f_mode & FMODE_WRITE) && retval >= 0) {
769 retval = fasync_helper(fd, filp, on, &pipe->fasync_writers); 769 retval = fasync_helper(fd, filp, on, &pipe->fasync_writers);
770 if (retval < 0 && (filp->f_mode & FMODE_READ)) 770 if (retval < 0 && (filp->f_mode & FMODE_READ))
771 /* this can happen only if on == T */ 771 /* this can happen only if on == T */
772 fasync_helper(-1, filp, 0, &pipe->fasync_readers); 772 fasync_helper(-1, filp, 0, &pipe->fasync_readers);
773 } 773 }
774 __pipe_unlock(pipe); 774 __pipe_unlock(pipe);
775 return retval; 775 return retval;
776 } 776 }
777 777
778 struct pipe_inode_info * alloc_pipe_info(struct inode *inode) 778 struct pipe_inode_info *alloc_pipe_info(void)
779 { 779 {
780 struct pipe_inode_info *pipe; 780 struct pipe_inode_info *pipe;
781 781
782 pipe = kzalloc(sizeof(struct pipe_inode_info), GFP_KERNEL); 782 pipe = kzalloc(sizeof(struct pipe_inode_info), GFP_KERNEL);
783 if (pipe) { 783 if (pipe) {
784 pipe->bufs = kzalloc(sizeof(struct pipe_buffer) * PIPE_DEF_BUFFERS, GFP_KERNEL); 784 pipe->bufs = kzalloc(sizeof(struct pipe_buffer) * PIPE_DEF_BUFFERS, GFP_KERNEL);
785 if (pipe->bufs) { 785 if (pipe->bufs) {
786 init_waitqueue_head(&pipe->wait); 786 init_waitqueue_head(&pipe->wait);
787 pipe->r_counter = pipe->w_counter = 1; 787 pipe->r_counter = pipe->w_counter = 1;
788 pipe->buffers = PIPE_DEF_BUFFERS; 788 pipe->buffers = PIPE_DEF_BUFFERS;
789 mutex_init(&pipe->mutex); 789 mutex_init(&pipe->mutex);
790 return pipe; 790 return pipe;
791 } 791 }
792 kfree(pipe); 792 kfree(pipe);
793 } 793 }
794 794
795 return NULL; 795 return NULL;
796 } 796 }
797 797
798 void __free_pipe_info(struct pipe_inode_info *pipe) 798 void __free_pipe_info(struct pipe_inode_info *pipe)
799 { 799 {
800 int i; 800 int i;
801 801
802 for (i = 0; i < pipe->buffers; i++) { 802 for (i = 0; i < pipe->buffers; i++) {
803 struct pipe_buffer *buf = pipe->bufs + i; 803 struct pipe_buffer *buf = pipe->bufs + i;
804 if (buf->ops) 804 if (buf->ops)
805 buf->ops->release(pipe, buf); 805 buf->ops->release(pipe, buf);
806 } 806 }
807 if (pipe->tmp_page) 807 if (pipe->tmp_page)
808 __free_page(pipe->tmp_page); 808 __free_page(pipe->tmp_page);
809 kfree(pipe->bufs); 809 kfree(pipe->bufs);
810 kfree(pipe); 810 kfree(pipe);
811 } 811 }
812 812
813 void free_pipe_info(struct inode *inode) 813 void free_pipe_info(struct inode *inode)
814 { 814 {
815 __free_pipe_info(inode->i_pipe); 815 __free_pipe_info(inode->i_pipe);
816 inode->i_pipe = NULL; 816 inode->i_pipe = NULL;
817 } 817 }
818 818
819 static struct vfsmount *pipe_mnt __read_mostly; 819 static struct vfsmount *pipe_mnt __read_mostly;
820 820
821 /* 821 /*
822 * pipefs_dname() is called from d_path(). 822 * pipefs_dname() is called from d_path().
823 */ 823 */
824 static char *pipefs_dname(struct dentry *dentry, char *buffer, int buflen) 824 static char *pipefs_dname(struct dentry *dentry, char *buffer, int buflen)
825 { 825 {
826 return dynamic_dname(dentry, buffer, buflen, "pipe:[%lu]", 826 return dynamic_dname(dentry, buffer, buflen, "pipe:[%lu]",
827 dentry->d_inode->i_ino); 827 dentry->d_inode->i_ino);
828 } 828 }
829 829
830 static const struct dentry_operations pipefs_dentry_operations = { 830 static const struct dentry_operations pipefs_dentry_operations = {
831 .d_dname = pipefs_dname, 831 .d_dname = pipefs_dname,
832 }; 832 };
833 833
834 static struct inode * get_pipe_inode(void) 834 static struct inode * get_pipe_inode(void)
835 { 835 {
836 struct inode *inode = new_inode_pseudo(pipe_mnt->mnt_sb); 836 struct inode *inode = new_inode_pseudo(pipe_mnt->mnt_sb);
837 struct pipe_inode_info *pipe; 837 struct pipe_inode_info *pipe;
838 838
839 if (!inode) 839 if (!inode)
840 goto fail_inode; 840 goto fail_inode;
841 841
842 inode->i_ino = get_next_ino(); 842 inode->i_ino = get_next_ino();
843 843
844 pipe = alloc_pipe_info(inode); 844 pipe = alloc_pipe_info();
845 if (!pipe) 845 if (!pipe)
846 goto fail_iput; 846 goto fail_iput;
847 847
848 inode->i_pipe = pipe; 848 inode->i_pipe = pipe;
849 pipe->files = 2; 849 pipe->files = 2;
850 pipe->readers = pipe->writers = 1; 850 pipe->readers = pipe->writers = 1;
851 inode->i_fop = &pipefifo_fops; 851 inode->i_fop = &pipefifo_fops;
852 852
853 /* 853 /*
854 * Mark the inode dirty from the very beginning, 854 * Mark the inode dirty from the very beginning,
855 * that way it will never be moved to the dirty 855 * that way it will never be moved to the dirty
856 * list because "mark_inode_dirty()" will think 856 * list because "mark_inode_dirty()" will think
857 * that it already _is_ on the dirty list. 857 * that it already _is_ on the dirty list.
858 */ 858 */
859 inode->i_state = I_DIRTY; 859 inode->i_state = I_DIRTY;
860 inode->i_mode = S_IFIFO | S_IRUSR | S_IWUSR; 860 inode->i_mode = S_IFIFO | S_IRUSR | S_IWUSR;
861 inode->i_uid = current_fsuid(); 861 inode->i_uid = current_fsuid();
862 inode->i_gid = current_fsgid(); 862 inode->i_gid = current_fsgid();
863 inode->i_atime = inode->i_mtime = inode->i_ctime = CURRENT_TIME; 863 inode->i_atime = inode->i_mtime = inode->i_ctime = CURRENT_TIME;
864 864
865 return inode; 865 return inode;
866 866
867 fail_iput: 867 fail_iput:
868 iput(inode); 868 iput(inode);
869 869
870 fail_inode: 870 fail_inode:
871 return NULL; 871 return NULL;
872 } 872 }
873 873
874 int create_pipe_files(struct file **res, int flags) 874 int create_pipe_files(struct file **res, int flags)
875 { 875 {
876 int err; 876 int err;
877 struct inode *inode = get_pipe_inode(); 877 struct inode *inode = get_pipe_inode();
878 struct file *f; 878 struct file *f;
879 struct path path; 879 struct path path;
880 static struct qstr name = { .name = "" }; 880 static struct qstr name = { .name = "" };
881 881
882 if (!inode) 882 if (!inode)
883 return -ENFILE; 883 return -ENFILE;
884 884
885 err = -ENOMEM; 885 err = -ENOMEM;
886 path.dentry = d_alloc_pseudo(pipe_mnt->mnt_sb, &name); 886 path.dentry = d_alloc_pseudo(pipe_mnt->mnt_sb, &name);
887 if (!path.dentry) 887 if (!path.dentry)
888 goto err_inode; 888 goto err_inode;
889 path.mnt = mntget(pipe_mnt); 889 path.mnt = mntget(pipe_mnt);
890 890
891 d_instantiate(path.dentry, inode); 891 d_instantiate(path.dentry, inode);
892 892
893 err = -ENFILE; 893 err = -ENFILE;
894 f = alloc_file(&path, FMODE_WRITE, &pipefifo_fops); 894 f = alloc_file(&path, FMODE_WRITE, &pipefifo_fops);
895 if (IS_ERR(f)) 895 if (IS_ERR(f))
896 goto err_dentry; 896 goto err_dentry;
897 897
898 f->f_flags = O_WRONLY | (flags & (O_NONBLOCK | O_DIRECT)); 898 f->f_flags = O_WRONLY | (flags & (O_NONBLOCK | O_DIRECT));
899 f->private_data = inode->i_pipe; 899 f->private_data = inode->i_pipe;
900 900
901 res[0] = alloc_file(&path, FMODE_READ, &pipefifo_fops); 901 res[0] = alloc_file(&path, FMODE_READ, &pipefifo_fops);
902 if (IS_ERR(res[0])) 902 if (IS_ERR(res[0]))
903 goto err_file; 903 goto err_file;
904 904
905 path_get(&path); 905 path_get(&path);
906 res[0]->private_data = inode->i_pipe; 906 res[0]->private_data = inode->i_pipe;
907 res[0]->f_flags = O_RDONLY | (flags & O_NONBLOCK); 907 res[0]->f_flags = O_RDONLY | (flags & O_NONBLOCK);
908 res[1] = f; 908 res[1] = f;
909 return 0; 909 return 0;
910 910
911 err_file: 911 err_file:
912 put_filp(f); 912 put_filp(f);
913 err_dentry: 913 err_dentry:
914 free_pipe_info(inode); 914 free_pipe_info(inode);
915 path_put(&path); 915 path_put(&path);
916 return err; 916 return err;
917 917
918 err_inode: 918 err_inode:
919 free_pipe_info(inode); 919 free_pipe_info(inode);
920 iput(inode); 920 iput(inode);
921 return err; 921 return err;
922 } 922 }
923 923
924 static int __do_pipe_flags(int *fd, struct file **files, int flags) 924 static int __do_pipe_flags(int *fd, struct file **files, int flags)
925 { 925 {
926 int error; 926 int error;
927 int fdw, fdr; 927 int fdw, fdr;
928 928
929 if (flags & ~(O_CLOEXEC | O_NONBLOCK | O_DIRECT)) 929 if (flags & ~(O_CLOEXEC | O_NONBLOCK | O_DIRECT))
930 return -EINVAL; 930 return -EINVAL;
931 931
932 error = create_pipe_files(files, flags); 932 error = create_pipe_files(files, flags);
933 if (error) 933 if (error)
934 return error; 934 return error;
935 935
936 error = get_unused_fd_flags(flags); 936 error = get_unused_fd_flags(flags);
937 if (error < 0) 937 if (error < 0)
938 goto err_read_pipe; 938 goto err_read_pipe;
939 fdr = error; 939 fdr = error;
940 940
941 error = get_unused_fd_flags(flags); 941 error = get_unused_fd_flags(flags);
942 if (error < 0) 942 if (error < 0)
943 goto err_fdr; 943 goto err_fdr;
944 fdw = error; 944 fdw = error;
945 945
946 audit_fd_pair(fdr, fdw); 946 audit_fd_pair(fdr, fdw);
947 fd[0] = fdr; 947 fd[0] = fdr;
948 fd[1] = fdw; 948 fd[1] = fdw;
949 return 0; 949 return 0;
950 950
951 err_fdr: 951 err_fdr:
952 put_unused_fd(fdr); 952 put_unused_fd(fdr);
953 err_read_pipe: 953 err_read_pipe:
954 fput(files[0]); 954 fput(files[0]);
955 fput(files[1]); 955 fput(files[1]);
956 return error; 956 return error;
957 } 957 }
958 958
959 int do_pipe_flags(int *fd, int flags) 959 int do_pipe_flags(int *fd, int flags)
960 { 960 {
961 struct file *files[2]; 961 struct file *files[2];
962 int error = __do_pipe_flags(fd, files, flags); 962 int error = __do_pipe_flags(fd, files, flags);
963 if (!error) { 963 if (!error) {
964 fd_install(fd[0], files[0]); 964 fd_install(fd[0], files[0]);
965 fd_install(fd[1], files[1]); 965 fd_install(fd[1], files[1]);
966 } 966 }
967 return error; 967 return error;
968 } 968 }
969 969
970 /* 970 /*
971 * sys_pipe() is the normal C calling standard for creating 971 * sys_pipe() is the normal C calling standard for creating
972 * a pipe. It's not the way Unix traditionally does this, though. 972 * a pipe. It's not the way Unix traditionally does this, though.
973 */ 973 */
974 SYSCALL_DEFINE2(pipe2, int __user *, fildes, int, flags) 974 SYSCALL_DEFINE2(pipe2, int __user *, fildes, int, flags)
975 { 975 {
976 struct file *files[2]; 976 struct file *files[2];
977 int fd[2]; 977 int fd[2];
978 int error; 978 int error;
979 979
980 error = __do_pipe_flags(fd, files, flags); 980 error = __do_pipe_flags(fd, files, flags);
981 if (!error) { 981 if (!error) {
982 if (unlikely(copy_to_user(fildes, fd, sizeof(fd)))) { 982 if (unlikely(copy_to_user(fildes, fd, sizeof(fd)))) {
983 fput(files[0]); 983 fput(files[0]);
984 fput(files[1]); 984 fput(files[1]);
985 put_unused_fd(fd[0]); 985 put_unused_fd(fd[0]);
986 put_unused_fd(fd[1]); 986 put_unused_fd(fd[1]);
987 error = -EFAULT; 987 error = -EFAULT;
988 } else { 988 } else {
989 fd_install(fd[0], files[0]); 989 fd_install(fd[0], files[0]);
990 fd_install(fd[1], files[1]); 990 fd_install(fd[1], files[1]);
991 } 991 }
992 } 992 }
993 return error; 993 return error;
994 } 994 }
995 995
996 SYSCALL_DEFINE1(pipe, int __user *, fildes) 996 SYSCALL_DEFINE1(pipe, int __user *, fildes)
997 { 997 {
998 return sys_pipe2(fildes, 0); 998 return sys_pipe2(fildes, 0);
999 } 999 }
1000 1000
1001 static int wait_for_partner(struct pipe_inode_info *pipe, unsigned int *cnt) 1001 static int wait_for_partner(struct pipe_inode_info *pipe, unsigned int *cnt)
1002 { 1002 {
1003 int cur = *cnt; 1003 int cur = *cnt;
1004 1004
1005 while (cur == *cnt) { 1005 while (cur == *cnt) {
1006 pipe_wait(pipe); 1006 pipe_wait(pipe);
1007 if (signal_pending(current)) 1007 if (signal_pending(current))
1008 break; 1008 break;
1009 } 1009 }
1010 return cur == *cnt ? -ERESTARTSYS : 0; 1010 return cur == *cnt ? -ERESTARTSYS : 0;
1011 } 1011 }
1012 1012
1013 static void wake_up_partner(struct pipe_inode_info *pipe) 1013 static void wake_up_partner(struct pipe_inode_info *pipe)
1014 { 1014 {
1015 wake_up_interruptible(&pipe->wait); 1015 wake_up_interruptible(&pipe->wait);
1016 } 1016 }
1017 1017
1018 static int fifo_open(struct inode *inode, struct file *filp) 1018 static int fifo_open(struct inode *inode, struct file *filp)
1019 { 1019 {
1020 struct pipe_inode_info *pipe; 1020 struct pipe_inode_info *pipe;
1021 bool is_pipe = inode->i_sb->s_magic == PIPEFS_MAGIC; 1021 bool is_pipe = inode->i_sb->s_magic == PIPEFS_MAGIC;
1022 int kill = 0; 1022 int kill = 0;
1023 int ret; 1023 int ret;
1024 1024
1025 filp->f_version = 0; 1025 filp->f_version = 0;
1026 1026
1027 spin_lock(&inode->i_lock); 1027 spin_lock(&inode->i_lock);
1028 if (inode->i_pipe) { 1028 if (inode->i_pipe) {
1029 pipe = inode->i_pipe; 1029 pipe = inode->i_pipe;
1030 pipe->files++; 1030 pipe->files++;
1031 spin_unlock(&inode->i_lock); 1031 spin_unlock(&inode->i_lock);
1032 } else { 1032 } else {
1033 spin_unlock(&inode->i_lock); 1033 spin_unlock(&inode->i_lock);
1034 pipe = alloc_pipe_info(inode); 1034 pipe = alloc_pipe_info();
1035 if (!pipe) 1035 if (!pipe)
1036 return -ENOMEM; 1036 return -ENOMEM;
1037 pipe->files = 1; 1037 pipe->files = 1;
1038 spin_lock(&inode->i_lock); 1038 spin_lock(&inode->i_lock);
1039 if (unlikely(inode->i_pipe)) { 1039 if (unlikely(inode->i_pipe)) {
1040 inode->i_pipe->files++; 1040 inode->i_pipe->files++;
1041 spin_unlock(&inode->i_lock); 1041 spin_unlock(&inode->i_lock);
1042 __free_pipe_info(pipe); 1042 __free_pipe_info(pipe);
1043 pipe = inode->i_pipe; 1043 pipe = inode->i_pipe;
1044 } else { 1044 } else {
1045 inode->i_pipe = pipe; 1045 inode->i_pipe = pipe;
1046 spin_unlock(&inode->i_lock); 1046 spin_unlock(&inode->i_lock);
1047 } 1047 }
1048 } 1048 }
1049 filp->private_data = pipe; 1049 filp->private_data = pipe;
1050 /* OK, we have a pipe and it's pinned down */ 1050 /* OK, we have a pipe and it's pinned down */
1051 1051
1052 __pipe_lock(pipe); 1052 __pipe_lock(pipe);
1053 1053
1054 /* We can only do regular read/write on fifos */ 1054 /* We can only do regular read/write on fifos */
1055 filp->f_mode &= (FMODE_READ | FMODE_WRITE); 1055 filp->f_mode &= (FMODE_READ | FMODE_WRITE);
1056 1056
1057 switch (filp->f_mode) { 1057 switch (filp->f_mode) {
1058 case FMODE_READ: 1058 case FMODE_READ:
1059 /* 1059 /*
1060 * O_RDONLY 1060 * O_RDONLY
1061 * POSIX.1 says that O_NONBLOCK means return with the FIFO 1061 * POSIX.1 says that O_NONBLOCK means return with the FIFO
1062 * opened, even when there is no process writing the FIFO. 1062 * opened, even when there is no process writing the FIFO.
1063 */ 1063 */
1064 pipe->r_counter++; 1064 pipe->r_counter++;
1065 if (pipe->readers++ == 0) 1065 if (pipe->readers++ == 0)
1066 wake_up_partner(pipe); 1066 wake_up_partner(pipe);
1067 1067
1068 if (!is_pipe && !pipe->writers) { 1068 if (!is_pipe && !pipe->writers) {
1069 if ((filp->f_flags & O_NONBLOCK)) { 1069 if ((filp->f_flags & O_NONBLOCK)) {
1070 /* suppress POLLHUP until we have 1070 /* suppress POLLHUP until we have
1071 * seen a writer */ 1071 * seen a writer */
1072 filp->f_version = pipe->w_counter; 1072 filp->f_version = pipe->w_counter;
1073 } else { 1073 } else {
1074 if (wait_for_partner(pipe, &pipe->w_counter)) 1074 if (wait_for_partner(pipe, &pipe->w_counter))
1075 goto err_rd; 1075 goto err_rd;
1076 } 1076 }
1077 } 1077 }
1078 break; 1078 break;
1079 1079
1080 case FMODE_WRITE: 1080 case FMODE_WRITE:
1081 /* 1081 /*
1082 * O_WRONLY 1082 * O_WRONLY
1083 * POSIX.1 says that O_NONBLOCK means return -1 with 1083 * POSIX.1 says that O_NONBLOCK means return -1 with
1084 * errno=ENXIO when there is no process reading the FIFO. 1084 * errno=ENXIO when there is no process reading the FIFO.
1085 */ 1085 */
1086 ret = -ENXIO; 1086 ret = -ENXIO;
1087 if (!is_pipe && (filp->f_flags & O_NONBLOCK) && !pipe->readers) 1087 if (!is_pipe && (filp->f_flags & O_NONBLOCK) && !pipe->readers)
1088 goto err; 1088 goto err;
1089 1089
1090 pipe->w_counter++; 1090 pipe->w_counter++;
1091 if (!pipe->writers++) 1091 if (!pipe->writers++)
1092 wake_up_partner(pipe); 1092 wake_up_partner(pipe);
1093 1093
1094 if (!is_pipe && !pipe->readers) { 1094 if (!is_pipe && !pipe->readers) {
1095 if (wait_for_partner(pipe, &pipe->r_counter)) 1095 if (wait_for_partner(pipe, &pipe->r_counter))
1096 goto err_wr; 1096 goto err_wr;
1097 } 1097 }
1098 break; 1098 break;
1099 1099
1100 case FMODE_READ | FMODE_WRITE: 1100 case FMODE_READ | FMODE_WRITE:
1101 /* 1101 /*
1102 * O_RDWR 1102 * O_RDWR
1103 * POSIX.1 leaves this case "undefined" when O_NONBLOCK is set. 1103 * POSIX.1 leaves this case "undefined" when O_NONBLOCK is set.
1104 * This implementation will NEVER block on a O_RDWR open, since 1104 * This implementation will NEVER block on a O_RDWR open, since
1105 * the process can at least talk to itself. 1105 * the process can at least talk to itself.
1106 */ 1106 */
1107 1107
1108 pipe->readers++; 1108 pipe->readers++;
1109 pipe->writers++; 1109 pipe->writers++;
1110 pipe->r_counter++; 1110 pipe->r_counter++;
1111 pipe->w_counter++; 1111 pipe->w_counter++;
1112 if (pipe->readers == 1 || pipe->writers == 1) 1112 if (pipe->readers == 1 || pipe->writers == 1)
1113 wake_up_partner(pipe); 1113 wake_up_partner(pipe);
1114 break; 1114 break;
1115 1115
1116 default: 1116 default:
1117 ret = -EINVAL; 1117 ret = -EINVAL;
1118 goto err; 1118 goto err;
1119 } 1119 }
1120 1120
1121 /* Ok! */ 1121 /* Ok! */
1122 __pipe_unlock(pipe); 1122 __pipe_unlock(pipe);
1123 return 0; 1123 return 0;
1124 1124
1125 err_rd: 1125 err_rd:
1126 if (!--pipe->readers) 1126 if (!--pipe->readers)
1127 wake_up_interruptible(&pipe->wait); 1127 wake_up_interruptible(&pipe->wait);
1128 ret = -ERESTARTSYS; 1128 ret = -ERESTARTSYS;
1129 goto err; 1129 goto err;
1130 1130
1131 err_wr: 1131 err_wr:
1132 if (!--pipe->writers) 1132 if (!--pipe->writers)
1133 wake_up_interruptible(&pipe->wait); 1133 wake_up_interruptible(&pipe->wait);
1134 ret = -ERESTARTSYS; 1134 ret = -ERESTARTSYS;
1135 goto err; 1135 goto err;
1136 1136
1137 err: 1137 err:
1138 spin_lock(&inode->i_lock); 1138 spin_lock(&inode->i_lock);
1139 if (!--pipe->files) { 1139 if (!--pipe->files) {
1140 inode->i_pipe = NULL; 1140 inode->i_pipe = NULL;
1141 kill = 1; 1141 kill = 1;
1142 } 1142 }
1143 spin_unlock(&inode->i_lock); 1143 spin_unlock(&inode->i_lock);
1144 __pipe_unlock(pipe); 1144 __pipe_unlock(pipe);
1145 if (kill) 1145 if (kill)
1146 __free_pipe_info(pipe); 1146 __free_pipe_info(pipe);
1147 return ret; 1147 return ret;
1148 } 1148 }
1149 1149
1150 const struct file_operations pipefifo_fops = { 1150 const struct file_operations pipefifo_fops = {
1151 .open = fifo_open, 1151 .open = fifo_open,
1152 .llseek = no_llseek, 1152 .llseek = no_llseek,
1153 .read = do_sync_read, 1153 .read = do_sync_read,
1154 .aio_read = pipe_read, 1154 .aio_read = pipe_read,
1155 .write = do_sync_write, 1155 .write = do_sync_write,
1156 .aio_write = pipe_write, 1156 .aio_write = pipe_write,
1157 .poll = pipe_poll, 1157 .poll = pipe_poll,
1158 .unlocked_ioctl = pipe_ioctl, 1158 .unlocked_ioctl = pipe_ioctl,
1159 .release = pipe_release, 1159 .release = pipe_release,
1160 .fasync = pipe_fasync, 1160 .fasync = pipe_fasync,
1161 }; 1161 };
1162 1162
1163 /* 1163 /*
1164 * Allocate a new array of pipe buffers and copy the info over. Returns the 1164 * Allocate a new array of pipe buffers and copy the info over. Returns the
1165 * pipe size if successful, or return -ERROR on error. 1165 * pipe size if successful, or return -ERROR on error.
1166 */ 1166 */
1167 static long pipe_set_size(struct pipe_inode_info *pipe, unsigned long nr_pages) 1167 static long pipe_set_size(struct pipe_inode_info *pipe, unsigned long nr_pages)
1168 { 1168 {
1169 struct pipe_buffer *bufs; 1169 struct pipe_buffer *bufs;
1170 1170
1171 /* 1171 /*
1172 * We can shrink the pipe, if arg >= pipe->nrbufs. Since we don't 1172 * We can shrink the pipe, if arg >= pipe->nrbufs. Since we don't
1173 * expect a lot of shrink+grow operations, just free and allocate 1173 * expect a lot of shrink+grow operations, just free and allocate
1174 * again like we would do for growing. If the pipe currently 1174 * again like we would do for growing. If the pipe currently
1175 * contains more buffers than arg, then return busy. 1175 * contains more buffers than arg, then return busy.
1176 */ 1176 */
1177 if (nr_pages < pipe->nrbufs) 1177 if (nr_pages < pipe->nrbufs)
1178 return -EBUSY; 1178 return -EBUSY;
1179 1179
1180 bufs = kcalloc(nr_pages, sizeof(*bufs), GFP_KERNEL | __GFP_NOWARN); 1180 bufs = kcalloc(nr_pages, sizeof(*bufs), GFP_KERNEL | __GFP_NOWARN);
1181 if (unlikely(!bufs)) 1181 if (unlikely(!bufs))
1182 return -ENOMEM; 1182 return -ENOMEM;
1183 1183
1184 /* 1184 /*
1185 * The pipe array wraps around, so just start the new one at zero 1185 * The pipe array wraps around, so just start the new one at zero
1186 * and adjust the indexes. 1186 * and adjust the indexes.
1187 */ 1187 */
1188 if (pipe->nrbufs) { 1188 if (pipe->nrbufs) {
1189 unsigned int tail; 1189 unsigned int tail;
1190 unsigned int head; 1190 unsigned int head;
1191 1191
1192 tail = pipe->curbuf + pipe->nrbufs; 1192 tail = pipe->curbuf + pipe->nrbufs;
1193 if (tail < pipe->buffers) 1193 if (tail < pipe->buffers)
1194 tail = 0; 1194 tail = 0;
1195 else 1195 else
1196 tail &= (pipe->buffers - 1); 1196 tail &= (pipe->buffers - 1);
1197 1197
1198 head = pipe->nrbufs - tail; 1198 head = pipe->nrbufs - tail;
1199 if (head) 1199 if (head)
1200 memcpy(bufs, pipe->bufs + pipe->curbuf, head * sizeof(struct pipe_buffer)); 1200 memcpy(bufs, pipe->bufs + pipe->curbuf, head * sizeof(struct pipe_buffer));
1201 if (tail) 1201 if (tail)
1202 memcpy(bufs + head, pipe->bufs, tail * sizeof(struct pipe_buffer)); 1202 memcpy(bufs + head, pipe->bufs, tail * sizeof(struct pipe_buffer));
1203 } 1203 }
1204 1204
1205 pipe->curbuf = 0; 1205 pipe->curbuf = 0;
1206 kfree(pipe->bufs); 1206 kfree(pipe->bufs);
1207 pipe->bufs = bufs; 1207 pipe->bufs = bufs;
1208 pipe->buffers = nr_pages; 1208 pipe->buffers = nr_pages;
1209 return nr_pages * PAGE_SIZE; 1209 return nr_pages * PAGE_SIZE;
1210 } 1210 }
1211 1211
1212 /* 1212 /*
1213 * Currently we rely on the pipe array holding a power-of-2 number 1213 * Currently we rely on the pipe array holding a power-of-2 number
1214 * of pages. 1214 * of pages.
1215 */ 1215 */
1216 static inline unsigned int round_pipe_size(unsigned int size) 1216 static inline unsigned int round_pipe_size(unsigned int size)
1217 { 1217 {
1218 unsigned long nr_pages; 1218 unsigned long nr_pages;
1219 1219
1220 nr_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT; 1220 nr_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
1221 return roundup_pow_of_two(nr_pages) << PAGE_SHIFT; 1221 return roundup_pow_of_two(nr_pages) << PAGE_SHIFT;
1222 } 1222 }
1223 1223
1224 /* 1224 /*
1225 * This should work even if CONFIG_PROC_FS isn't set, as proc_dointvec_minmax 1225 * This should work even if CONFIG_PROC_FS isn't set, as proc_dointvec_minmax
1226 * will return an error. 1226 * will return an error.
1227 */ 1227 */
1228 int pipe_proc_fn(struct ctl_table *table, int write, void __user *buf, 1228 int pipe_proc_fn(struct ctl_table *table, int write, void __user *buf,
1229 size_t *lenp, loff_t *ppos) 1229 size_t *lenp, loff_t *ppos)
1230 { 1230 {
1231 int ret; 1231 int ret;
1232 1232
1233 ret = proc_dointvec_minmax(table, write, buf, lenp, ppos); 1233 ret = proc_dointvec_minmax(table, write, buf, lenp, ppos);
1234 if (ret < 0 || !write) 1234 if (ret < 0 || !write)
1235 return ret; 1235 return ret;
1236 1236
1237 pipe_max_size = round_pipe_size(pipe_max_size); 1237 pipe_max_size = round_pipe_size(pipe_max_size);
1238 return ret; 1238 return ret;
1239 } 1239 }
1240 1240
1241 /* 1241 /*
1242 * After the inode slimming patch, i_pipe/i_bdev/i_cdev share the same 1242 * After the inode slimming patch, i_pipe/i_bdev/i_cdev share the same
1243 * location, so checking ->i_pipe is not enough to verify that this is a 1243 * location, so checking ->i_pipe is not enough to verify that this is a
1244 * pipe. 1244 * pipe.
1245 */ 1245 */
1246 struct pipe_inode_info *get_pipe_info(struct file *file) 1246 struct pipe_inode_info *get_pipe_info(struct file *file)
1247 { 1247 {
1248 return file->f_op == &pipefifo_fops ? file->private_data : NULL; 1248 return file->f_op == &pipefifo_fops ? file->private_data : NULL;
1249 } 1249 }
1250 1250
1251 long pipe_fcntl(struct file *file, unsigned int cmd, unsigned long arg) 1251 long pipe_fcntl(struct file *file, unsigned int cmd, unsigned long arg)
1252 { 1252 {
1253 struct pipe_inode_info *pipe; 1253 struct pipe_inode_info *pipe;
1254 long ret; 1254 long ret;
1255 1255
1256 pipe = get_pipe_info(file); 1256 pipe = get_pipe_info(file);
1257 if (!pipe) 1257 if (!pipe)
1258 return -EBADF; 1258 return -EBADF;
1259 1259
1260 __pipe_lock(pipe); 1260 __pipe_lock(pipe);
1261 1261
1262 switch (cmd) { 1262 switch (cmd) {
1263 case F_SETPIPE_SZ: { 1263 case F_SETPIPE_SZ: {
1264 unsigned int size, nr_pages; 1264 unsigned int size, nr_pages;
1265 1265
1266 size = round_pipe_size(arg); 1266 size = round_pipe_size(arg);
1267 nr_pages = size >> PAGE_SHIFT; 1267 nr_pages = size >> PAGE_SHIFT;
1268 1268
1269 ret = -EINVAL; 1269 ret = -EINVAL;
1270 if (!nr_pages) 1270 if (!nr_pages)
1271 goto out; 1271 goto out;
1272 1272
1273 if (!capable(CAP_SYS_RESOURCE) && size > pipe_max_size) { 1273 if (!capable(CAP_SYS_RESOURCE) && size > pipe_max_size) {
1274 ret = -EPERM; 1274 ret = -EPERM;
1275 goto out; 1275 goto out;
1276 } 1276 }
1277 ret = pipe_set_size(pipe, nr_pages); 1277 ret = pipe_set_size(pipe, nr_pages);
1278 break; 1278 break;
1279 } 1279 }
1280 case F_GETPIPE_SZ: 1280 case F_GETPIPE_SZ:
1281 ret = pipe->buffers * PAGE_SIZE; 1281 ret = pipe->buffers * PAGE_SIZE;
1282 break; 1282 break;
1283 default: 1283 default:
1284 ret = -EINVAL; 1284 ret = -EINVAL;
1285 break; 1285 break;
1286 } 1286 }
1287 1287
1288 out: 1288 out:
1289 __pipe_unlock(pipe); 1289 __pipe_unlock(pipe);
1290 return ret; 1290 return ret;
1291 } 1291 }
1292 1292
1293 static const struct super_operations pipefs_ops = { 1293 static const struct super_operations pipefs_ops = {
1294 .destroy_inode = free_inode_nonrcu, 1294 .destroy_inode = free_inode_nonrcu,
1295 .statfs = simple_statfs, 1295 .statfs = simple_statfs,
1296 }; 1296 };
1297 1297
1298 /* 1298 /*
1299 * pipefs should _never_ be mounted by userland - too much of security hassle, 1299 * pipefs should _never_ be mounted by userland - too much of security hassle,
1300 * no real gain from having the whole whorehouse mounted. So we don't need 1300 * no real gain from having the whole whorehouse mounted. So we don't need
1301 * any operations on the root directory. However, we need a non-trivial 1301 * any operations on the root directory. However, we need a non-trivial
1302 * d_name - pipe: will go nicely and kill the special-casing in procfs. 1302 * d_name - pipe: will go nicely and kill the special-casing in procfs.
1303 */ 1303 */
1304 static struct dentry *pipefs_mount(struct file_system_type *fs_type, 1304 static struct dentry *pipefs_mount(struct file_system_type *fs_type,
1305 int flags, const char *dev_name, void *data) 1305 int flags, const char *dev_name, void *data)
1306 { 1306 {
1307 return mount_pseudo(fs_type, "pipe:", &pipefs_ops, 1307 return mount_pseudo(fs_type, "pipe:", &pipefs_ops,
1308 &pipefs_dentry_operations, PIPEFS_MAGIC); 1308 &pipefs_dentry_operations, PIPEFS_MAGIC);
1309 } 1309 }
1310 1310
1311 static struct file_system_type pipe_fs_type = { 1311 static struct file_system_type pipe_fs_type = {
1312 .name = "pipefs", 1312 .name = "pipefs",
1313 .mount = pipefs_mount, 1313 .mount = pipefs_mount,
1314 .kill_sb = kill_anon_super, 1314 .kill_sb = kill_anon_super,
1315 }; 1315 };
1316 1316
1317 static int __init init_pipe_fs(void) 1317 static int __init init_pipe_fs(void)
1318 { 1318 {
1319 int err = register_filesystem(&pipe_fs_type); 1319 int err = register_filesystem(&pipe_fs_type);
1320 1320
1321 if (!err) { 1321 if (!err) {
1322 pipe_mnt = kern_mount(&pipe_fs_type); 1322 pipe_mnt = kern_mount(&pipe_fs_type);
1323 if (IS_ERR(pipe_mnt)) { 1323 if (IS_ERR(pipe_mnt)) {
1324 err = PTR_ERR(pipe_mnt); 1324 err = PTR_ERR(pipe_mnt);
1325 unregister_filesystem(&pipe_fs_type); 1325 unregister_filesystem(&pipe_fs_type);
1326 } 1326 }
1327 } 1327 }
1328 return err; 1328 return err;
1329 } 1329 }
1330 1330
1331 fs_initcall(init_pipe_fs); 1331 fs_initcall(init_pipe_fs);
1332 1332
1 /* 1 /*
2 * "splice": joining two ropes together by interweaving their strands. 2 * "splice": joining two ropes together by interweaving their strands.
3 * 3 *
4 * This is the "extended pipe" functionality, where a pipe is used as 4 * This is the "extended pipe" functionality, where a pipe is used as
5 * an arbitrary in-memory buffer. Think of a pipe as a small kernel 5 * an arbitrary in-memory buffer. Think of a pipe as a small kernel
6 * buffer that you can use to transfer data from one end to the other. 6 * buffer that you can use to transfer data from one end to the other.
7 * 7 *
8 * The traditional unix read/write is extended with a "splice()" operation 8 * The traditional unix read/write is extended with a "splice()" operation
9 * that transfers data buffers to or from a pipe buffer. 9 * that transfers data buffers to or from a pipe buffer.
10 * 10 *
11 * Named by Larry McVoy, original implementation from Linus, extended by 11 * Named by Larry McVoy, original implementation from Linus, extended by
12 * Jens to support splicing to files, network, direct splicing, etc and 12 * Jens to support splicing to files, network, direct splicing, etc and
13 * fixing lots of bugs. 13 * fixing lots of bugs.
14 * 14 *
15 * Copyright (C) 2005-2006 Jens Axboe <axboe@kernel.dk> 15 * Copyright (C) 2005-2006 Jens Axboe <axboe@kernel.dk>
16 * Copyright (C) 2005-2006 Linus Torvalds <torvalds@osdl.org> 16 * Copyright (C) 2005-2006 Linus Torvalds <torvalds@osdl.org>
17 * Copyright (C) 2006 Ingo Molnar <mingo@elte.hu> 17 * Copyright (C) 2006 Ingo Molnar <mingo@elte.hu>
18 * 18 *
19 */ 19 */
20 #include <linux/fs.h> 20 #include <linux/fs.h>
21 #include <linux/file.h> 21 #include <linux/file.h>
22 #include <linux/pagemap.h> 22 #include <linux/pagemap.h>
23 #include <linux/splice.h> 23 #include <linux/splice.h>
24 #include <linux/memcontrol.h> 24 #include <linux/memcontrol.h>
25 #include <linux/mm_inline.h> 25 #include <linux/mm_inline.h>
26 #include <linux/swap.h> 26 #include <linux/swap.h>
27 #include <linux/writeback.h> 27 #include <linux/writeback.h>
28 #include <linux/export.h> 28 #include <linux/export.h>
29 #include <linux/syscalls.h> 29 #include <linux/syscalls.h>
30 #include <linux/uio.h> 30 #include <linux/uio.h>
31 #include <linux/security.h> 31 #include <linux/security.h>
32 #include <linux/gfp.h> 32 #include <linux/gfp.h>
33 #include <linux/socket.h> 33 #include <linux/socket.h>
34 #include "internal.h" 34 #include "internal.h"
35 35
36 /* 36 /*
37 * Attempt to steal a page from a pipe buffer. This should perhaps go into 37 * Attempt to steal a page from a pipe buffer. This should perhaps go into
38 * a vm helper function, it's already simplified quite a bit by the 38 * a vm helper function, it's already simplified quite a bit by the
39 * addition of remove_mapping(). If success is returned, the caller may 39 * addition of remove_mapping(). If success is returned, the caller may
40 * attempt to reuse this page for another destination. 40 * attempt to reuse this page for another destination.
41 */ 41 */
42 static int page_cache_pipe_buf_steal(struct pipe_inode_info *pipe, 42 static int page_cache_pipe_buf_steal(struct pipe_inode_info *pipe,
43 struct pipe_buffer *buf) 43 struct pipe_buffer *buf)
44 { 44 {
45 struct page *page = buf->page; 45 struct page *page = buf->page;
46 struct address_space *mapping; 46 struct address_space *mapping;
47 47
48 lock_page(page); 48 lock_page(page);
49 49
50 mapping = page_mapping(page); 50 mapping = page_mapping(page);
51 if (mapping) { 51 if (mapping) {
52 WARN_ON(!PageUptodate(page)); 52 WARN_ON(!PageUptodate(page));
53 53
54 /* 54 /*
55 * At least for ext2 with nobh option, we need to wait on 55 * At least for ext2 with nobh option, we need to wait on
56 * writeback completing on this page, since we'll remove it 56 * writeback completing on this page, since we'll remove it
57 * from the pagecache. Otherwise truncate wont wait on the 57 * from the pagecache. Otherwise truncate wont wait on the
58 * page, allowing the disk blocks to be reused by someone else 58 * page, allowing the disk blocks to be reused by someone else
59 * before we actually wrote our data to them. fs corruption 59 * before we actually wrote our data to them. fs corruption
60 * ensues. 60 * ensues.
61 */ 61 */
62 wait_on_page_writeback(page); 62 wait_on_page_writeback(page);
63 63
64 if (page_has_private(page) && 64 if (page_has_private(page) &&
65 !try_to_release_page(page, GFP_KERNEL)) 65 !try_to_release_page(page, GFP_KERNEL))
66 goto out_unlock; 66 goto out_unlock;
67 67
68 /* 68 /*
69 * If we succeeded in removing the mapping, set LRU flag 69 * If we succeeded in removing the mapping, set LRU flag
70 * and return good. 70 * and return good.
71 */ 71 */
72 if (remove_mapping(mapping, page)) { 72 if (remove_mapping(mapping, page)) {
73 buf->flags |= PIPE_BUF_FLAG_LRU; 73 buf->flags |= PIPE_BUF_FLAG_LRU;
74 return 0; 74 return 0;
75 } 75 }
76 } 76 }
77 77
78 /* 78 /*
79 * Raced with truncate or failed to remove page from current 79 * Raced with truncate or failed to remove page from current
80 * address space, unlock and return failure. 80 * address space, unlock and return failure.
81 */ 81 */
82 out_unlock: 82 out_unlock:
83 unlock_page(page); 83 unlock_page(page);
84 return 1; 84 return 1;
85 } 85 }
86 86
87 static void page_cache_pipe_buf_release(struct pipe_inode_info *pipe, 87 static void page_cache_pipe_buf_release(struct pipe_inode_info *pipe,
88 struct pipe_buffer *buf) 88 struct pipe_buffer *buf)
89 { 89 {
90 page_cache_release(buf->page); 90 page_cache_release(buf->page);
91 buf->flags &= ~PIPE_BUF_FLAG_LRU; 91 buf->flags &= ~PIPE_BUF_FLAG_LRU;
92 } 92 }
93 93
94 /* 94 /*
95 * Check whether the contents of buf is OK to access. Since the content 95 * Check whether the contents of buf is OK to access. Since the content
96 * is a page cache page, IO may be in flight. 96 * is a page cache page, IO may be in flight.
97 */ 97 */
98 static int page_cache_pipe_buf_confirm(struct pipe_inode_info *pipe, 98 static int page_cache_pipe_buf_confirm(struct pipe_inode_info *pipe,
99 struct pipe_buffer *buf) 99 struct pipe_buffer *buf)
100 { 100 {
101 struct page *page = buf->page; 101 struct page *page = buf->page;
102 int err; 102 int err;
103 103
104 if (!PageUptodate(page)) { 104 if (!PageUptodate(page)) {
105 lock_page(page); 105 lock_page(page);
106 106
107 /* 107 /*
108 * Page got truncated/unhashed. This will cause a 0-byte 108 * Page got truncated/unhashed. This will cause a 0-byte
109 * splice, if this is the first page. 109 * splice, if this is the first page.
110 */ 110 */
111 if (!page->mapping) { 111 if (!page->mapping) {
112 err = -ENODATA; 112 err = -ENODATA;
113 goto error; 113 goto error;
114 } 114 }
115 115
116 /* 116 /*
117 * Uh oh, read-error from disk. 117 * Uh oh, read-error from disk.
118 */ 118 */
119 if (!PageUptodate(page)) { 119 if (!PageUptodate(page)) {
120 err = -EIO; 120 err = -EIO;
121 goto error; 121 goto error;
122 } 122 }
123 123
124 /* 124 /*
125 * Page is ok afterall, we are done. 125 * Page is ok afterall, we are done.
126 */ 126 */
127 unlock_page(page); 127 unlock_page(page);
128 } 128 }
129 129
130 return 0; 130 return 0;
131 error: 131 error:
132 unlock_page(page); 132 unlock_page(page);
133 return err; 133 return err;
134 } 134 }
135 135
136 const struct pipe_buf_operations page_cache_pipe_buf_ops = { 136 const struct pipe_buf_operations page_cache_pipe_buf_ops = {
137 .can_merge = 0, 137 .can_merge = 0,
138 .map = generic_pipe_buf_map, 138 .map = generic_pipe_buf_map,
139 .unmap = generic_pipe_buf_unmap, 139 .unmap = generic_pipe_buf_unmap,
140 .confirm = page_cache_pipe_buf_confirm, 140 .confirm = page_cache_pipe_buf_confirm,
141 .release = page_cache_pipe_buf_release, 141 .release = page_cache_pipe_buf_release,
142 .steal = page_cache_pipe_buf_steal, 142 .steal = page_cache_pipe_buf_steal,
143 .get = generic_pipe_buf_get, 143 .get = generic_pipe_buf_get,
144 }; 144 };
145 145
146 static int user_page_pipe_buf_steal(struct pipe_inode_info *pipe, 146 static int user_page_pipe_buf_steal(struct pipe_inode_info *pipe,
147 struct pipe_buffer *buf) 147 struct pipe_buffer *buf)
148 { 148 {
149 if (!(buf->flags & PIPE_BUF_FLAG_GIFT)) 149 if (!(buf->flags & PIPE_BUF_FLAG_GIFT))
150 return 1; 150 return 1;
151 151
152 buf->flags |= PIPE_BUF_FLAG_LRU; 152 buf->flags |= PIPE_BUF_FLAG_LRU;
153 return generic_pipe_buf_steal(pipe, buf); 153 return generic_pipe_buf_steal(pipe, buf);
154 } 154 }
155 155
156 static const struct pipe_buf_operations user_page_pipe_buf_ops = { 156 static const struct pipe_buf_operations user_page_pipe_buf_ops = {
157 .can_merge = 0, 157 .can_merge = 0,
158 .map = generic_pipe_buf_map, 158 .map = generic_pipe_buf_map,
159 .unmap = generic_pipe_buf_unmap, 159 .unmap = generic_pipe_buf_unmap,
160 .confirm = generic_pipe_buf_confirm, 160 .confirm = generic_pipe_buf_confirm,
161 .release = page_cache_pipe_buf_release, 161 .release = page_cache_pipe_buf_release,
162 .steal = user_page_pipe_buf_steal, 162 .steal = user_page_pipe_buf_steal,
163 .get = generic_pipe_buf_get, 163 .get = generic_pipe_buf_get,
164 }; 164 };
165 165
166 static void wakeup_pipe_readers(struct pipe_inode_info *pipe) 166 static void wakeup_pipe_readers(struct pipe_inode_info *pipe)
167 { 167 {
168 smp_mb(); 168 smp_mb();
169 if (waitqueue_active(&pipe->wait)) 169 if (waitqueue_active(&pipe->wait))
170 wake_up_interruptible(&pipe->wait); 170 wake_up_interruptible(&pipe->wait);
171 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN); 171 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
172 } 172 }
173 173
174 /** 174 /**
175 * splice_to_pipe - fill passed data into a pipe 175 * splice_to_pipe - fill passed data into a pipe
176 * @pipe: pipe to fill 176 * @pipe: pipe to fill
177 * @spd: data to fill 177 * @spd: data to fill
178 * 178 *
179 * Description: 179 * Description:
180 * @spd contains a map of pages and len/offset tuples, along with 180 * @spd contains a map of pages and len/offset tuples, along with
181 * the struct pipe_buf_operations associated with these pages. This 181 * the struct pipe_buf_operations associated with these pages. This
182 * function will link that data to the pipe. 182 * function will link that data to the pipe.
183 * 183 *
184 */ 184 */
185 ssize_t splice_to_pipe(struct pipe_inode_info *pipe, 185 ssize_t splice_to_pipe(struct pipe_inode_info *pipe,
186 struct splice_pipe_desc *spd) 186 struct splice_pipe_desc *spd)
187 { 187 {
188 unsigned int spd_pages = spd->nr_pages; 188 unsigned int spd_pages = spd->nr_pages;
189 int ret, do_wakeup, page_nr; 189 int ret, do_wakeup, page_nr;
190 190
191 ret = 0; 191 ret = 0;
192 do_wakeup = 0; 192 do_wakeup = 0;
193 page_nr = 0; 193 page_nr = 0;
194 194
195 pipe_lock(pipe); 195 pipe_lock(pipe);
196 196
197 for (;;) { 197 for (;;) {
198 if (!pipe->readers) { 198 if (!pipe->readers) {
199 send_sig(SIGPIPE, current, 0); 199 send_sig(SIGPIPE, current, 0);
200 if (!ret) 200 if (!ret)
201 ret = -EPIPE; 201 ret = -EPIPE;
202 break; 202 break;
203 } 203 }
204 204
205 if (pipe->nrbufs < pipe->buffers) { 205 if (pipe->nrbufs < pipe->buffers) {
206 int newbuf = (pipe->curbuf + pipe->nrbufs) & (pipe->buffers - 1); 206 int newbuf = (pipe->curbuf + pipe->nrbufs) & (pipe->buffers - 1);
207 struct pipe_buffer *buf = pipe->bufs + newbuf; 207 struct pipe_buffer *buf = pipe->bufs + newbuf;
208 208
209 buf->page = spd->pages[page_nr]; 209 buf->page = spd->pages[page_nr];
210 buf->offset = spd->partial[page_nr].offset; 210 buf->offset = spd->partial[page_nr].offset;
211 buf->len = spd->partial[page_nr].len; 211 buf->len = spd->partial[page_nr].len;
212 buf->private = spd->partial[page_nr].private; 212 buf->private = spd->partial[page_nr].private;
213 buf->ops = spd->ops; 213 buf->ops = spd->ops;
214 if (spd->flags & SPLICE_F_GIFT) 214 if (spd->flags & SPLICE_F_GIFT)
215 buf->flags |= PIPE_BUF_FLAG_GIFT; 215 buf->flags |= PIPE_BUF_FLAG_GIFT;
216 216
217 pipe->nrbufs++; 217 pipe->nrbufs++;
218 page_nr++; 218 page_nr++;
219 ret += buf->len; 219 ret += buf->len;
220 220
221 if (pipe->files) 221 if (pipe->files)
222 do_wakeup = 1; 222 do_wakeup = 1;
223 223
224 if (!--spd->nr_pages) 224 if (!--spd->nr_pages)
225 break; 225 break;
226 if (pipe->nrbufs < pipe->buffers) 226 if (pipe->nrbufs < pipe->buffers)
227 continue; 227 continue;
228 228
229 break; 229 break;
230 } 230 }
231 231
232 if (spd->flags & SPLICE_F_NONBLOCK) { 232 if (spd->flags & SPLICE_F_NONBLOCK) {
233 if (!ret) 233 if (!ret)
234 ret = -EAGAIN; 234 ret = -EAGAIN;
235 break; 235 break;
236 } 236 }
237 237
238 if (signal_pending(current)) { 238 if (signal_pending(current)) {
239 if (!ret) 239 if (!ret)
240 ret = -ERESTARTSYS; 240 ret = -ERESTARTSYS;
241 break; 241 break;
242 } 242 }
243 243
244 if (do_wakeup) { 244 if (do_wakeup) {
245 smp_mb(); 245 smp_mb();
246 if (waitqueue_active(&pipe->wait)) 246 if (waitqueue_active(&pipe->wait))
247 wake_up_interruptible_sync(&pipe->wait); 247 wake_up_interruptible_sync(&pipe->wait);
248 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN); 248 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
249 do_wakeup = 0; 249 do_wakeup = 0;
250 } 250 }
251 251
252 pipe->waiting_writers++; 252 pipe->waiting_writers++;
253 pipe_wait(pipe); 253 pipe_wait(pipe);
254 pipe->waiting_writers--; 254 pipe->waiting_writers--;
255 } 255 }
256 256
257 pipe_unlock(pipe); 257 pipe_unlock(pipe);
258 258
259 if (do_wakeup) 259 if (do_wakeup)
260 wakeup_pipe_readers(pipe); 260 wakeup_pipe_readers(pipe);
261 261
262 while (page_nr < spd_pages) 262 while (page_nr < spd_pages)
263 spd->spd_release(spd, page_nr++); 263 spd->spd_release(spd, page_nr++);
264 264
265 return ret; 265 return ret;
266 } 266 }
267 267
268 void spd_release_page(struct splice_pipe_desc *spd, unsigned int i) 268 void spd_release_page(struct splice_pipe_desc *spd, unsigned int i)
269 { 269 {
270 page_cache_release(spd->pages[i]); 270 page_cache_release(spd->pages[i]);
271 } 271 }
272 272
273 /* 273 /*
274 * Check if we need to grow the arrays holding pages and partial page 274 * Check if we need to grow the arrays holding pages and partial page
275 * descriptions. 275 * descriptions.
276 */ 276 */
277 int splice_grow_spd(const struct pipe_inode_info *pipe, struct splice_pipe_desc *spd) 277 int splice_grow_spd(const struct pipe_inode_info *pipe, struct splice_pipe_desc *spd)
278 { 278 {
279 unsigned int buffers = ACCESS_ONCE(pipe->buffers); 279 unsigned int buffers = ACCESS_ONCE(pipe->buffers);
280 280
281 spd->nr_pages_max = buffers; 281 spd->nr_pages_max = buffers;
282 if (buffers <= PIPE_DEF_BUFFERS) 282 if (buffers <= PIPE_DEF_BUFFERS)
283 return 0; 283 return 0;
284 284
285 spd->pages = kmalloc(buffers * sizeof(struct page *), GFP_KERNEL); 285 spd->pages = kmalloc(buffers * sizeof(struct page *), GFP_KERNEL);
286 spd->partial = kmalloc(buffers * sizeof(struct partial_page), GFP_KERNEL); 286 spd->partial = kmalloc(buffers * sizeof(struct partial_page), GFP_KERNEL);
287 287
288 if (spd->pages && spd->partial) 288 if (spd->pages && spd->partial)
289 return 0; 289 return 0;
290 290
291 kfree(spd->pages); 291 kfree(spd->pages);
292 kfree(spd->partial); 292 kfree(spd->partial);
293 return -ENOMEM; 293 return -ENOMEM;
294 } 294 }
295 295
296 void splice_shrink_spd(struct splice_pipe_desc *spd) 296 void splice_shrink_spd(struct splice_pipe_desc *spd)
297 { 297 {
298 if (spd->nr_pages_max <= PIPE_DEF_BUFFERS) 298 if (spd->nr_pages_max <= PIPE_DEF_BUFFERS)
299 return; 299 return;
300 300
301 kfree(spd->pages); 301 kfree(spd->pages);
302 kfree(spd->partial); 302 kfree(spd->partial);
303 } 303 }
304 304
305 static int 305 static int
306 __generic_file_splice_read(struct file *in, loff_t *ppos, 306 __generic_file_splice_read(struct file *in, loff_t *ppos,
307 struct pipe_inode_info *pipe, size_t len, 307 struct pipe_inode_info *pipe, size_t len,
308 unsigned int flags) 308 unsigned int flags)
309 { 309 {
310 struct address_space *mapping = in->f_mapping; 310 struct address_space *mapping = in->f_mapping;
311 unsigned int loff, nr_pages, req_pages; 311 unsigned int loff, nr_pages, req_pages;
312 struct page *pages[PIPE_DEF_BUFFERS]; 312 struct page *pages[PIPE_DEF_BUFFERS];
313 struct partial_page partial[PIPE_DEF_BUFFERS]; 313 struct partial_page partial[PIPE_DEF_BUFFERS];
314 struct page *page; 314 struct page *page;
315 pgoff_t index, end_index; 315 pgoff_t index, end_index;
316 loff_t isize; 316 loff_t isize;
317 int error, page_nr; 317 int error, page_nr;
318 struct splice_pipe_desc spd = { 318 struct splice_pipe_desc spd = {
319 .pages = pages, 319 .pages = pages,
320 .partial = partial, 320 .partial = partial,
321 .nr_pages_max = PIPE_DEF_BUFFERS, 321 .nr_pages_max = PIPE_DEF_BUFFERS,
322 .flags = flags, 322 .flags = flags,
323 .ops = &page_cache_pipe_buf_ops, 323 .ops = &page_cache_pipe_buf_ops,
324 .spd_release = spd_release_page, 324 .spd_release = spd_release_page,
325 }; 325 };
326 326
327 if (splice_grow_spd(pipe, &spd)) 327 if (splice_grow_spd(pipe, &spd))
328 return -ENOMEM; 328 return -ENOMEM;
329 329
330 index = *ppos >> PAGE_CACHE_SHIFT; 330 index = *ppos >> PAGE_CACHE_SHIFT;
331 loff = *ppos & ~PAGE_CACHE_MASK; 331 loff = *ppos & ~PAGE_CACHE_MASK;
332 req_pages = (len + loff + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT; 332 req_pages = (len + loff + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
333 nr_pages = min(req_pages, spd.nr_pages_max); 333 nr_pages = min(req_pages, spd.nr_pages_max);
334 334
335 /* 335 /*
336 * Lookup the (hopefully) full range of pages we need. 336 * Lookup the (hopefully) full range of pages we need.
337 */ 337 */
338 spd.nr_pages = find_get_pages_contig(mapping, index, nr_pages, spd.pages); 338 spd.nr_pages = find_get_pages_contig(mapping, index, nr_pages, spd.pages);
339 index += spd.nr_pages; 339 index += spd.nr_pages;
340 340
341 /* 341 /*
342 * If find_get_pages_contig() returned fewer pages than we needed, 342 * If find_get_pages_contig() returned fewer pages than we needed,
343 * readahead/allocate the rest and fill in the holes. 343 * readahead/allocate the rest and fill in the holes.
344 */ 344 */
345 if (spd.nr_pages < nr_pages) 345 if (spd.nr_pages < nr_pages)
346 page_cache_sync_readahead(mapping, &in->f_ra, in, 346 page_cache_sync_readahead(mapping, &in->f_ra, in,
347 index, req_pages - spd.nr_pages); 347 index, req_pages - spd.nr_pages);
348 348
349 error = 0; 349 error = 0;
350 while (spd.nr_pages < nr_pages) { 350 while (spd.nr_pages < nr_pages) {
351 /* 351 /*
352 * Page could be there, find_get_pages_contig() breaks on 352 * Page could be there, find_get_pages_contig() breaks on
353 * the first hole. 353 * the first hole.
354 */ 354 */
355 page = find_get_page(mapping, index); 355 page = find_get_page(mapping, index);
356 if (!page) { 356 if (!page) {
357 /* 357 /*
358 * page didn't exist, allocate one. 358 * page didn't exist, allocate one.
359 */ 359 */
360 page = page_cache_alloc_cold(mapping); 360 page = page_cache_alloc_cold(mapping);
361 if (!page) 361 if (!page)
362 break; 362 break;
363 363
364 error = add_to_page_cache_lru(page, mapping, index, 364 error = add_to_page_cache_lru(page, mapping, index,
365 GFP_KERNEL); 365 GFP_KERNEL);
366 if (unlikely(error)) { 366 if (unlikely(error)) {
367 page_cache_release(page); 367 page_cache_release(page);
368 if (error == -EEXIST) 368 if (error == -EEXIST)
369 continue; 369 continue;
370 break; 370 break;
371 } 371 }
372 /* 372 /*
373 * add_to_page_cache() locks the page, unlock it 373 * add_to_page_cache() locks the page, unlock it
374 * to avoid convoluting the logic below even more. 374 * to avoid convoluting the logic below even more.
375 */ 375 */
376 unlock_page(page); 376 unlock_page(page);
377 } 377 }
378 378
379 spd.pages[spd.nr_pages++] = page; 379 spd.pages[spd.nr_pages++] = page;
380 index++; 380 index++;
381 } 381 }
382 382
383 /* 383 /*
384 * Now loop over the map and see if we need to start IO on any 384 * Now loop over the map and see if we need to start IO on any
385 * pages, fill in the partial map, etc. 385 * pages, fill in the partial map, etc.
386 */ 386 */
387 index = *ppos >> PAGE_CACHE_SHIFT; 387 index = *ppos >> PAGE_CACHE_SHIFT;
388 nr_pages = spd.nr_pages; 388 nr_pages = spd.nr_pages;
389 spd.nr_pages = 0; 389 spd.nr_pages = 0;
390 for (page_nr = 0; page_nr < nr_pages; page_nr++) { 390 for (page_nr = 0; page_nr < nr_pages; page_nr++) {
391 unsigned int this_len; 391 unsigned int this_len;
392 392
393 if (!len) 393 if (!len)
394 break; 394 break;
395 395
396 /* 396 /*
397 * this_len is the max we'll use from this page 397 * this_len is the max we'll use from this page
398 */ 398 */
399 this_len = min_t(unsigned long, len, PAGE_CACHE_SIZE - loff); 399 this_len = min_t(unsigned long, len, PAGE_CACHE_SIZE - loff);
400 page = spd.pages[page_nr]; 400 page = spd.pages[page_nr];
401 401
402 if (PageReadahead(page)) 402 if (PageReadahead(page))
403 page_cache_async_readahead(mapping, &in->f_ra, in, 403 page_cache_async_readahead(mapping, &in->f_ra, in,
404 page, index, req_pages - page_nr); 404 page, index, req_pages - page_nr);
405 405
406 /* 406 /*
407 * If the page isn't uptodate, we may need to start io on it 407 * If the page isn't uptodate, we may need to start io on it
408 */ 408 */
409 if (!PageUptodate(page)) { 409 if (!PageUptodate(page)) {
410 lock_page(page); 410 lock_page(page);
411 411
412 /* 412 /*
413 * Page was truncated, or invalidated by the 413 * Page was truncated, or invalidated by the
414 * filesystem. Redo the find/create, but this time the 414 * filesystem. Redo the find/create, but this time the
415 * page is kept locked, so there's no chance of another 415 * page is kept locked, so there's no chance of another
416 * race with truncate/invalidate. 416 * race with truncate/invalidate.
417 */ 417 */
418 if (!page->mapping) { 418 if (!page->mapping) {
419 unlock_page(page); 419 unlock_page(page);
420 page = find_or_create_page(mapping, index, 420 page = find_or_create_page(mapping, index,
421 mapping_gfp_mask(mapping)); 421 mapping_gfp_mask(mapping));
422 422
423 if (!page) { 423 if (!page) {
424 error = -ENOMEM; 424 error = -ENOMEM;
425 break; 425 break;
426 } 426 }
427 page_cache_release(spd.pages[page_nr]); 427 page_cache_release(spd.pages[page_nr]);
428 spd.pages[page_nr] = page; 428 spd.pages[page_nr] = page;
429 } 429 }
430 /* 430 /*
431 * page was already under io and is now done, great 431 * page was already under io and is now done, great
432 */ 432 */
433 if (PageUptodate(page)) { 433 if (PageUptodate(page)) {
434 unlock_page(page); 434 unlock_page(page);
435 goto fill_it; 435 goto fill_it;
436 } 436 }
437 437
438 /* 438 /*
439 * need to read in the page 439 * need to read in the page
440 */ 440 */
441 error = mapping->a_ops->readpage(in, page); 441 error = mapping->a_ops->readpage(in, page);
442 if (unlikely(error)) { 442 if (unlikely(error)) {
443 /* 443 /*
444 * We really should re-lookup the page here, 444 * We really should re-lookup the page here,
445 * but it complicates things a lot. Instead 445 * but it complicates things a lot. Instead
446 * lets just do what we already stored, and 446 * lets just do what we already stored, and
447 * we'll get it the next time we are called. 447 * we'll get it the next time we are called.
448 */ 448 */
449 if (error == AOP_TRUNCATED_PAGE) 449 if (error == AOP_TRUNCATED_PAGE)
450 error = 0; 450 error = 0;
451 451
452 break; 452 break;
453 } 453 }
454 } 454 }
455 fill_it: 455 fill_it:
456 /* 456 /*
457 * i_size must be checked after PageUptodate. 457 * i_size must be checked after PageUptodate.
458 */ 458 */
459 isize = i_size_read(mapping->host); 459 isize = i_size_read(mapping->host);
460 end_index = (isize - 1) >> PAGE_CACHE_SHIFT; 460 end_index = (isize - 1) >> PAGE_CACHE_SHIFT;
461 if (unlikely(!isize || index > end_index)) 461 if (unlikely(!isize || index > end_index))
462 break; 462 break;
463 463
464 /* 464 /*
465 * if this is the last page, see if we need to shrink 465 * if this is the last page, see if we need to shrink
466 * the length and stop 466 * the length and stop
467 */ 467 */
468 if (end_index == index) { 468 if (end_index == index) {
469 unsigned int plen; 469 unsigned int plen;
470 470
471 /* 471 /*
472 * max good bytes in this page 472 * max good bytes in this page
473 */ 473 */
474 plen = ((isize - 1) & ~PAGE_CACHE_MASK) + 1; 474 plen = ((isize - 1) & ~PAGE_CACHE_MASK) + 1;
475 if (plen <= loff) 475 if (plen <= loff)
476 break; 476 break;
477 477
478 /* 478 /*
479 * force quit after adding this page 479 * force quit after adding this page
480 */ 480 */
481 this_len = min(this_len, plen - loff); 481 this_len = min(this_len, plen - loff);
482 len = this_len; 482 len = this_len;
483 } 483 }
484 484
485 spd.partial[page_nr].offset = loff; 485 spd.partial[page_nr].offset = loff;
486 spd.partial[page_nr].len = this_len; 486 spd.partial[page_nr].len = this_len;
487 len -= this_len; 487 len -= this_len;
488 loff = 0; 488 loff = 0;
489 spd.nr_pages++; 489 spd.nr_pages++;
490 index++; 490 index++;
491 } 491 }
492 492
493 /* 493 /*
494 * Release any pages at the end, if we quit early. 'page_nr' is how far 494 * Release any pages at the end, if we quit early. 'page_nr' is how far
495 * we got, 'nr_pages' is how many pages are in the map. 495 * we got, 'nr_pages' is how many pages are in the map.
496 */ 496 */
497 while (page_nr < nr_pages) 497 while (page_nr < nr_pages)
498 page_cache_release(spd.pages[page_nr++]); 498 page_cache_release(spd.pages[page_nr++]);
499 in->f_ra.prev_pos = (loff_t)index << PAGE_CACHE_SHIFT; 499 in->f_ra.prev_pos = (loff_t)index << PAGE_CACHE_SHIFT;
500 500
501 if (spd.nr_pages) 501 if (spd.nr_pages)
502 error = splice_to_pipe(pipe, &spd); 502 error = splice_to_pipe(pipe, &spd);
503 503
504 splice_shrink_spd(&spd); 504 splice_shrink_spd(&spd);
505 return error; 505 return error;
506 } 506 }
507 507
508 /** 508 /**
509 * generic_file_splice_read - splice data from file to a pipe 509 * generic_file_splice_read - splice data from file to a pipe
510 * @in: file to splice from 510 * @in: file to splice from
511 * @ppos: position in @in 511 * @ppos: position in @in
512 * @pipe: pipe to splice to 512 * @pipe: pipe to splice to
513 * @len: number of bytes to splice 513 * @len: number of bytes to splice
514 * @flags: splice modifier flags 514 * @flags: splice modifier flags
515 * 515 *
516 * Description: 516 * Description:
517 * Will read pages from given file and fill them into a pipe. Can be 517 * Will read pages from given file and fill them into a pipe. Can be
518 * used as long as the address_space operations for the source implements 518 * used as long as the address_space operations for the source implements
519 * a readpage() hook. 519 * a readpage() hook.
520 * 520 *
521 */ 521 */
522 ssize_t generic_file_splice_read(struct file *in, loff_t *ppos, 522 ssize_t generic_file_splice_read(struct file *in, loff_t *ppos,
523 struct pipe_inode_info *pipe, size_t len, 523 struct pipe_inode_info *pipe, size_t len,
524 unsigned int flags) 524 unsigned int flags)
525 { 525 {
526 loff_t isize, left; 526 loff_t isize, left;
527 int ret; 527 int ret;
528 528
529 isize = i_size_read(in->f_mapping->host); 529 isize = i_size_read(in->f_mapping->host);
530 if (unlikely(*ppos >= isize)) 530 if (unlikely(*ppos >= isize))
531 return 0; 531 return 0;
532 532
533 left = isize - *ppos; 533 left = isize - *ppos;
534 if (unlikely(left < len)) 534 if (unlikely(left < len))
535 len = left; 535 len = left;
536 536
537 ret = __generic_file_splice_read(in, ppos, pipe, len, flags); 537 ret = __generic_file_splice_read(in, ppos, pipe, len, flags);
538 if (ret > 0) { 538 if (ret > 0) {
539 *ppos += ret; 539 *ppos += ret;
540 file_accessed(in); 540 file_accessed(in);
541 } 541 }
542 542
543 return ret; 543 return ret;
544 } 544 }
545 EXPORT_SYMBOL(generic_file_splice_read); 545 EXPORT_SYMBOL(generic_file_splice_read);
546 546
547 static const struct pipe_buf_operations default_pipe_buf_ops = { 547 static const struct pipe_buf_operations default_pipe_buf_ops = {
548 .can_merge = 0, 548 .can_merge = 0,
549 .map = generic_pipe_buf_map, 549 .map = generic_pipe_buf_map,
550 .unmap = generic_pipe_buf_unmap, 550 .unmap = generic_pipe_buf_unmap,
551 .confirm = generic_pipe_buf_confirm, 551 .confirm = generic_pipe_buf_confirm,
552 .release = generic_pipe_buf_release, 552 .release = generic_pipe_buf_release,
553 .steal = generic_pipe_buf_steal, 553 .steal = generic_pipe_buf_steal,
554 .get = generic_pipe_buf_get, 554 .get = generic_pipe_buf_get,
555 }; 555 };
556 556
557 static ssize_t kernel_readv(struct file *file, const struct iovec *vec, 557 static ssize_t kernel_readv(struct file *file, const struct iovec *vec,
558 unsigned long vlen, loff_t offset) 558 unsigned long vlen, loff_t offset)
559 { 559 {
560 mm_segment_t old_fs; 560 mm_segment_t old_fs;
561 loff_t pos = offset; 561 loff_t pos = offset;
562 ssize_t res; 562 ssize_t res;
563 563
564 old_fs = get_fs(); 564 old_fs = get_fs();
565 set_fs(get_ds()); 565 set_fs(get_ds());
566 /* The cast to a user pointer is valid due to the set_fs() */ 566 /* The cast to a user pointer is valid due to the set_fs() */
567 res = vfs_readv(file, (const struct iovec __user *)vec, vlen, &pos); 567 res = vfs_readv(file, (const struct iovec __user *)vec, vlen, &pos);
568 set_fs(old_fs); 568 set_fs(old_fs);
569 569
570 return res; 570 return res;
571 } 571 }
572 572
573 ssize_t kernel_write(struct file *file, const char *buf, size_t count, 573 ssize_t kernel_write(struct file *file, const char *buf, size_t count,
574 loff_t pos) 574 loff_t pos)
575 { 575 {
576 mm_segment_t old_fs; 576 mm_segment_t old_fs;
577 ssize_t res; 577 ssize_t res;
578 578
579 old_fs = get_fs(); 579 old_fs = get_fs();
580 set_fs(get_ds()); 580 set_fs(get_ds());
581 /* The cast to a user pointer is valid due to the set_fs() */ 581 /* The cast to a user pointer is valid due to the set_fs() */
582 res = vfs_write(file, (__force const char __user *)buf, count, &pos); 582 res = vfs_write(file, (__force const char __user *)buf, count, &pos);
583 set_fs(old_fs); 583 set_fs(old_fs);
584 584
585 return res; 585 return res;
586 } 586 }
587 EXPORT_SYMBOL(kernel_write); 587 EXPORT_SYMBOL(kernel_write);
588 588
589 ssize_t default_file_splice_read(struct file *in, loff_t *ppos, 589 ssize_t default_file_splice_read(struct file *in, loff_t *ppos,
590 struct pipe_inode_info *pipe, size_t len, 590 struct pipe_inode_info *pipe, size_t len,
591 unsigned int flags) 591 unsigned int flags)
592 { 592 {
593 unsigned int nr_pages; 593 unsigned int nr_pages;
594 unsigned int nr_freed; 594 unsigned int nr_freed;
595 size_t offset; 595 size_t offset;
596 struct page *pages[PIPE_DEF_BUFFERS]; 596 struct page *pages[PIPE_DEF_BUFFERS];
597 struct partial_page partial[PIPE_DEF_BUFFERS]; 597 struct partial_page partial[PIPE_DEF_BUFFERS];
598 struct iovec *vec, __vec[PIPE_DEF_BUFFERS]; 598 struct iovec *vec, __vec[PIPE_DEF_BUFFERS];
599 ssize_t res; 599 ssize_t res;
600 size_t this_len; 600 size_t this_len;
601 int error; 601 int error;
602 int i; 602 int i;
603 struct splice_pipe_desc spd = { 603 struct splice_pipe_desc spd = {
604 .pages = pages, 604 .pages = pages,
605 .partial = partial, 605 .partial = partial,
606 .nr_pages_max = PIPE_DEF_BUFFERS, 606 .nr_pages_max = PIPE_DEF_BUFFERS,
607 .flags = flags, 607 .flags = flags,
608 .ops = &default_pipe_buf_ops, 608 .ops = &default_pipe_buf_ops,
609 .spd_release = spd_release_page, 609 .spd_release = spd_release_page,
610 }; 610 };
611 611
612 if (splice_grow_spd(pipe, &spd)) 612 if (splice_grow_spd(pipe, &spd))
613 return -ENOMEM; 613 return -ENOMEM;
614 614
615 res = -ENOMEM; 615 res = -ENOMEM;
616 vec = __vec; 616 vec = __vec;
617 if (spd.nr_pages_max > PIPE_DEF_BUFFERS) { 617 if (spd.nr_pages_max > PIPE_DEF_BUFFERS) {
618 vec = kmalloc(spd.nr_pages_max * sizeof(struct iovec), GFP_KERNEL); 618 vec = kmalloc(spd.nr_pages_max * sizeof(struct iovec), GFP_KERNEL);
619 if (!vec) 619 if (!vec)
620 goto shrink_ret; 620 goto shrink_ret;
621 } 621 }
622 622
623 offset = *ppos & ~PAGE_CACHE_MASK; 623 offset = *ppos & ~PAGE_CACHE_MASK;
624 nr_pages = (len + offset + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT; 624 nr_pages = (len + offset + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
625 625
626 for (i = 0; i < nr_pages && i < spd.nr_pages_max && len; i++) { 626 for (i = 0; i < nr_pages && i < spd.nr_pages_max && len; i++) {
627 struct page *page; 627 struct page *page;
628 628
629 page = alloc_page(GFP_USER); 629 page = alloc_page(GFP_USER);
630 error = -ENOMEM; 630 error = -ENOMEM;
631 if (!page) 631 if (!page)
632 goto err; 632 goto err;
633 633
634 this_len = min_t(size_t, len, PAGE_CACHE_SIZE - offset); 634 this_len = min_t(size_t, len, PAGE_CACHE_SIZE - offset);
635 vec[i].iov_base = (void __user *) page_address(page); 635 vec[i].iov_base = (void __user *) page_address(page);
636 vec[i].iov_len = this_len; 636 vec[i].iov_len = this_len;
637 spd.pages[i] = page; 637 spd.pages[i] = page;
638 spd.nr_pages++; 638 spd.nr_pages++;
639 len -= this_len; 639 len -= this_len;
640 offset = 0; 640 offset = 0;
641 } 641 }
642 642
643 res = kernel_readv(in, vec, spd.nr_pages, *ppos); 643 res = kernel_readv(in, vec, spd.nr_pages, *ppos);
644 if (res < 0) { 644 if (res < 0) {
645 error = res; 645 error = res;
646 goto err; 646 goto err;
647 } 647 }
648 648
649 error = 0; 649 error = 0;
650 if (!res) 650 if (!res)
651 goto err; 651 goto err;
652 652
653 nr_freed = 0; 653 nr_freed = 0;
654 for (i = 0; i < spd.nr_pages; i++) { 654 for (i = 0; i < spd.nr_pages; i++) {
655 this_len = min_t(size_t, vec[i].iov_len, res); 655 this_len = min_t(size_t, vec[i].iov_len, res);
656 spd.partial[i].offset = 0; 656 spd.partial[i].offset = 0;
657 spd.partial[i].len = this_len; 657 spd.partial[i].len = this_len;
658 if (!this_len) { 658 if (!this_len) {
659 __free_page(spd.pages[i]); 659 __free_page(spd.pages[i]);
660 spd.pages[i] = NULL; 660 spd.pages[i] = NULL;
661 nr_freed++; 661 nr_freed++;
662 } 662 }
663 res -= this_len; 663 res -= this_len;
664 } 664 }
665 spd.nr_pages -= nr_freed; 665 spd.nr_pages -= nr_freed;
666 666
667 res = splice_to_pipe(pipe, &spd); 667 res = splice_to_pipe(pipe, &spd);
668 if (res > 0) 668 if (res > 0)
669 *ppos += res; 669 *ppos += res;
670 670
671 shrink_ret: 671 shrink_ret:
672 if (vec != __vec) 672 if (vec != __vec)
673 kfree(vec); 673 kfree(vec);
674 splice_shrink_spd(&spd); 674 splice_shrink_spd(&spd);
675 return res; 675 return res;
676 676
677 err: 677 err:
678 for (i = 0; i < spd.nr_pages; i++) 678 for (i = 0; i < spd.nr_pages; i++)
679 __free_page(spd.pages[i]); 679 __free_page(spd.pages[i]);
680 680
681 res = error; 681 res = error;
682 goto shrink_ret; 682 goto shrink_ret;
683 } 683 }
684 EXPORT_SYMBOL(default_file_splice_read); 684 EXPORT_SYMBOL(default_file_splice_read);
685 685
686 /* 686 /*
687 * Send 'sd->len' bytes to socket from 'sd->file' at position 'sd->pos' 687 * Send 'sd->len' bytes to socket from 'sd->file' at position 'sd->pos'
688 * using sendpage(). Return the number of bytes sent. 688 * using sendpage(). Return the number of bytes sent.
689 */ 689 */
690 static int pipe_to_sendpage(struct pipe_inode_info *pipe, 690 static int pipe_to_sendpage(struct pipe_inode_info *pipe,
691 struct pipe_buffer *buf, struct splice_desc *sd) 691 struct pipe_buffer *buf, struct splice_desc *sd)
692 { 692 {
693 struct file *file = sd->u.file; 693 struct file *file = sd->u.file;
694 loff_t pos = sd->pos; 694 loff_t pos = sd->pos;
695 int more; 695 int more;
696 696
697 if (!likely(file->f_op && file->f_op->sendpage)) 697 if (!likely(file->f_op && file->f_op->sendpage))
698 return -EINVAL; 698 return -EINVAL;
699 699
700 more = (sd->flags & SPLICE_F_MORE) ? MSG_MORE : 0; 700 more = (sd->flags & SPLICE_F_MORE) ? MSG_MORE : 0;
701 701
702 if (sd->len < sd->total_len && pipe->nrbufs > 1) 702 if (sd->len < sd->total_len && pipe->nrbufs > 1)
703 more |= MSG_SENDPAGE_NOTLAST; 703 more |= MSG_SENDPAGE_NOTLAST;
704 704
705 return file->f_op->sendpage(file, buf->page, buf->offset, 705 return file->f_op->sendpage(file, buf->page, buf->offset,
706 sd->len, &pos, more); 706 sd->len, &pos, more);
707 } 707 }
708 708
709 /* 709 /*
710 * This is a little more tricky than the file -> pipe splicing. There are 710 * This is a little more tricky than the file -> pipe splicing. There are
711 * basically three cases: 711 * basically three cases:
712 * 712 *
713 * - Destination page already exists in the address space and there 713 * - Destination page already exists in the address space and there
714 * are users of it. For that case we have no other option that 714 * are users of it. For that case we have no other option that
715 * copying the data. Tough luck. 715 * copying the data. Tough luck.
716 * - Destination page already exists in the address space, but there 716 * - Destination page already exists in the address space, but there
717 * are no users of it. Make sure it's uptodate, then drop it. Fall 717 * are no users of it. Make sure it's uptodate, then drop it. Fall
718 * through to last case. 718 * through to last case.
719 * - Destination page does not exist, we can add the pipe page to 719 * - Destination page does not exist, we can add the pipe page to
720 * the page cache and avoid the copy. 720 * the page cache and avoid the copy.
721 * 721 *
722 * If asked to move pages to the output file (SPLICE_F_MOVE is set in 722 * If asked to move pages to the output file (SPLICE_F_MOVE is set in
723 * sd->flags), we attempt to migrate pages from the pipe to the output 723 * sd->flags), we attempt to migrate pages from the pipe to the output
724 * file address space page cache. This is possible if no one else has 724 * file address space page cache. This is possible if no one else has
725 * the pipe page referenced outside of the pipe and page cache. If 725 * the pipe page referenced outside of the pipe and page cache. If
726 * SPLICE_F_MOVE isn't set, or we cannot move the page, we simply create 726 * SPLICE_F_MOVE isn't set, or we cannot move the page, we simply create
727 * a new page in the output file page cache and fill/dirty that. 727 * a new page in the output file page cache and fill/dirty that.
728 */ 728 */
729 int pipe_to_file(struct pipe_inode_info *pipe, struct pipe_buffer *buf, 729 int pipe_to_file(struct pipe_inode_info *pipe, struct pipe_buffer *buf,
730 struct splice_desc *sd) 730 struct splice_desc *sd)
731 { 731 {
732 struct file *file = sd->u.file; 732 struct file *file = sd->u.file;
733 struct address_space *mapping = file->f_mapping; 733 struct address_space *mapping = file->f_mapping;
734 unsigned int offset, this_len; 734 unsigned int offset, this_len;
735 struct page *page; 735 struct page *page;
736 void *fsdata; 736 void *fsdata;
737 int ret; 737 int ret;
738 738
739 offset = sd->pos & ~PAGE_CACHE_MASK; 739 offset = sd->pos & ~PAGE_CACHE_MASK;
740 740
741 this_len = sd->len; 741 this_len = sd->len;
742 if (this_len + offset > PAGE_CACHE_SIZE) 742 if (this_len + offset > PAGE_CACHE_SIZE)
743 this_len = PAGE_CACHE_SIZE - offset; 743 this_len = PAGE_CACHE_SIZE - offset;
744 744
745 ret = pagecache_write_begin(file, mapping, sd->pos, this_len, 745 ret = pagecache_write_begin(file, mapping, sd->pos, this_len,
746 AOP_FLAG_UNINTERRUPTIBLE, &page, &fsdata); 746 AOP_FLAG_UNINTERRUPTIBLE, &page, &fsdata);
747 if (unlikely(ret)) 747 if (unlikely(ret))
748 goto out; 748 goto out;
749 749
750 if (buf->page != page) { 750 if (buf->page != page) {
751 char *src = buf->ops->map(pipe, buf, 1); 751 char *src = buf->ops->map(pipe, buf, 1);
752 char *dst = kmap_atomic(page); 752 char *dst = kmap_atomic(page);
753 753
754 memcpy(dst + offset, src + buf->offset, this_len); 754 memcpy(dst + offset, src + buf->offset, this_len);
755 flush_dcache_page(page); 755 flush_dcache_page(page);
756 kunmap_atomic(dst); 756 kunmap_atomic(dst);
757 buf->ops->unmap(pipe, buf, src); 757 buf->ops->unmap(pipe, buf, src);
758 } 758 }
759 ret = pagecache_write_end(file, mapping, sd->pos, this_len, this_len, 759 ret = pagecache_write_end(file, mapping, sd->pos, this_len, this_len,
760 page, fsdata); 760 page, fsdata);
761 out: 761 out:
762 return ret; 762 return ret;
763 } 763 }
764 EXPORT_SYMBOL(pipe_to_file); 764 EXPORT_SYMBOL(pipe_to_file);
765 765
766 static void wakeup_pipe_writers(struct pipe_inode_info *pipe) 766 static void wakeup_pipe_writers(struct pipe_inode_info *pipe)
767 { 767 {
768 smp_mb(); 768 smp_mb();
769 if (waitqueue_active(&pipe->wait)) 769 if (waitqueue_active(&pipe->wait))
770 wake_up_interruptible(&pipe->wait); 770 wake_up_interruptible(&pipe->wait);
771 kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT); 771 kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);
772 } 772 }
773 773
774 /** 774 /**
775 * splice_from_pipe_feed - feed available data from a pipe to a file 775 * splice_from_pipe_feed - feed available data from a pipe to a file
776 * @pipe: pipe to splice from 776 * @pipe: pipe to splice from
777 * @sd: information to @actor 777 * @sd: information to @actor
778 * @actor: handler that splices the data 778 * @actor: handler that splices the data
779 * 779 *
780 * Description: 780 * Description:
781 * This function loops over the pipe and calls @actor to do the 781 * This function loops over the pipe and calls @actor to do the
782 * actual moving of a single struct pipe_buffer to the desired 782 * actual moving of a single struct pipe_buffer to the desired
783 * destination. It returns when there's no more buffers left in 783 * destination. It returns when there's no more buffers left in
784 * the pipe or if the requested number of bytes (@sd->total_len) 784 * the pipe or if the requested number of bytes (@sd->total_len)
785 * have been copied. It returns a positive number (one) if the 785 * have been copied. It returns a positive number (one) if the
786 * pipe needs to be filled with more data, zero if the required 786 * pipe needs to be filled with more data, zero if the required
787 * number of bytes have been copied and -errno on error. 787 * number of bytes have been copied and -errno on error.
788 * 788 *
789 * This, together with splice_from_pipe_{begin,end,next}, may be 789 * This, together with splice_from_pipe_{begin,end,next}, may be
790 * used to implement the functionality of __splice_from_pipe() when 790 * used to implement the functionality of __splice_from_pipe() when
791 * locking is required around copying the pipe buffers to the 791 * locking is required around copying the pipe buffers to the
792 * destination. 792 * destination.
793 */ 793 */
794 int splice_from_pipe_feed(struct pipe_inode_info *pipe, struct splice_desc *sd, 794 int splice_from_pipe_feed(struct pipe_inode_info *pipe, struct splice_desc *sd,
795 splice_actor *actor) 795 splice_actor *actor)
796 { 796 {
797 int ret; 797 int ret;
798 798
799 while (pipe->nrbufs) { 799 while (pipe->nrbufs) {
800 struct pipe_buffer *buf = pipe->bufs + pipe->curbuf; 800 struct pipe_buffer *buf = pipe->bufs + pipe->curbuf;
801 const struct pipe_buf_operations *ops = buf->ops; 801 const struct pipe_buf_operations *ops = buf->ops;
802 802
803 sd->len = buf->len; 803 sd->len = buf->len;
804 if (sd->len > sd->total_len) 804 if (sd->len > sd->total_len)
805 sd->len = sd->total_len; 805 sd->len = sd->total_len;
806 806
807 ret = buf->ops->confirm(pipe, buf); 807 ret = buf->ops->confirm(pipe, buf);
808 if (unlikely(ret)) { 808 if (unlikely(ret)) {
809 if (ret == -ENODATA) 809 if (ret == -ENODATA)
810 ret = 0; 810 ret = 0;
811 return ret; 811 return ret;
812 } 812 }
813 813
814 ret = actor(pipe, buf, sd); 814 ret = actor(pipe, buf, sd);
815 if (ret <= 0) 815 if (ret <= 0)
816 return ret; 816 return ret;
817 817
818 buf->offset += ret; 818 buf->offset += ret;
819 buf->len -= ret; 819 buf->len -= ret;
820 820
821 sd->num_spliced += ret; 821 sd->num_spliced += ret;
822 sd->len -= ret; 822 sd->len -= ret;
823 sd->pos += ret; 823 sd->pos += ret;
824 sd->total_len -= ret; 824 sd->total_len -= ret;
825 825
826 if (!buf->len) { 826 if (!buf->len) {
827 buf->ops = NULL; 827 buf->ops = NULL;
828 ops->release(pipe, buf); 828 ops->release(pipe, buf);
829 pipe->curbuf = (pipe->curbuf + 1) & (pipe->buffers - 1); 829 pipe->curbuf = (pipe->curbuf + 1) & (pipe->buffers - 1);
830 pipe->nrbufs--; 830 pipe->nrbufs--;
831 if (pipe->files) 831 if (pipe->files)
832 sd->need_wakeup = true; 832 sd->need_wakeup = true;
833 } 833 }
834 834
835 if (!sd->total_len) 835 if (!sd->total_len)
836 return 0; 836 return 0;
837 } 837 }
838 838
839 return 1; 839 return 1;
840 } 840 }
841 EXPORT_SYMBOL(splice_from_pipe_feed); 841 EXPORT_SYMBOL(splice_from_pipe_feed);
842 842
843 /** 843 /**
844 * splice_from_pipe_next - wait for some data to splice from 844 * splice_from_pipe_next - wait for some data to splice from
845 * @pipe: pipe to splice from 845 * @pipe: pipe to splice from
846 * @sd: information about the splice operation 846 * @sd: information about the splice operation
847 * 847 *
848 * Description: 848 * Description:
849 * This function will wait for some data and return a positive 849 * This function will wait for some data and return a positive
850 * value (one) if pipe buffers are available. It will return zero 850 * value (one) if pipe buffers are available. It will return zero
851 * or -errno if no more data needs to be spliced. 851 * or -errno if no more data needs to be spliced.
852 */ 852 */
853 int splice_from_pipe_next(struct pipe_inode_info *pipe, struct splice_desc *sd) 853 int splice_from_pipe_next(struct pipe_inode_info *pipe, struct splice_desc *sd)
854 { 854 {
855 while (!pipe->nrbufs) { 855 while (!pipe->nrbufs) {
856 if (!pipe->writers) 856 if (!pipe->writers)
857 return 0; 857 return 0;
858 858
859 if (!pipe->waiting_writers && sd->num_spliced) 859 if (!pipe->waiting_writers && sd->num_spliced)
860 return 0; 860 return 0;
861 861
862 if (sd->flags & SPLICE_F_NONBLOCK) 862 if (sd->flags & SPLICE_F_NONBLOCK)
863 return -EAGAIN; 863 return -EAGAIN;
864 864
865 if (signal_pending(current)) 865 if (signal_pending(current))
866 return -ERESTARTSYS; 866 return -ERESTARTSYS;
867 867
868 if (sd->need_wakeup) { 868 if (sd->need_wakeup) {
869 wakeup_pipe_writers(pipe); 869 wakeup_pipe_writers(pipe);
870 sd->need_wakeup = false; 870 sd->need_wakeup = false;
871 } 871 }
872 872
873 pipe_wait(pipe); 873 pipe_wait(pipe);
874 } 874 }
875 875
876 return 1; 876 return 1;
877 } 877 }
878 EXPORT_SYMBOL(splice_from_pipe_next); 878 EXPORT_SYMBOL(splice_from_pipe_next);
879 879
880 /** 880 /**
881 * splice_from_pipe_begin - start splicing from pipe 881 * splice_from_pipe_begin - start splicing from pipe
882 * @sd: information about the splice operation 882 * @sd: information about the splice operation
883 * 883 *
884 * Description: 884 * Description:
885 * This function should be called before a loop containing 885 * This function should be called before a loop containing
886 * splice_from_pipe_next() and splice_from_pipe_feed() to 886 * splice_from_pipe_next() and splice_from_pipe_feed() to
887 * initialize the necessary fields of @sd. 887 * initialize the necessary fields of @sd.
888 */ 888 */
889 void splice_from_pipe_begin(struct splice_desc *sd) 889 void splice_from_pipe_begin(struct splice_desc *sd)
890 { 890 {
891 sd->num_spliced = 0; 891 sd->num_spliced = 0;
892 sd->need_wakeup = false; 892 sd->need_wakeup = false;
893 } 893 }
894 EXPORT_SYMBOL(splice_from_pipe_begin); 894 EXPORT_SYMBOL(splice_from_pipe_begin);
895 895
896 /** 896 /**
897 * splice_from_pipe_end - finish splicing from pipe 897 * splice_from_pipe_end - finish splicing from pipe
898 * @pipe: pipe to splice from 898 * @pipe: pipe to splice from
899 * @sd: information about the splice operation 899 * @sd: information about the splice operation
900 * 900 *
901 * Description: 901 * Description:
902 * This function will wake up pipe writers if necessary. It should 902 * This function will wake up pipe writers if necessary. It should
903 * be called after a loop containing splice_from_pipe_next() and 903 * be called after a loop containing splice_from_pipe_next() and
904 * splice_from_pipe_feed(). 904 * splice_from_pipe_feed().
905 */ 905 */
906 void splice_from_pipe_end(struct pipe_inode_info *pipe, struct splice_desc *sd) 906 void splice_from_pipe_end(struct pipe_inode_info *pipe, struct splice_desc *sd)
907 { 907 {
908 if (sd->need_wakeup) 908 if (sd->need_wakeup)
909 wakeup_pipe_writers(pipe); 909 wakeup_pipe_writers(pipe);
910 } 910 }
911 EXPORT_SYMBOL(splice_from_pipe_end); 911 EXPORT_SYMBOL(splice_from_pipe_end);
912 912
913 /** 913 /**
914 * __splice_from_pipe - splice data from a pipe to given actor 914 * __splice_from_pipe - splice data from a pipe to given actor
915 * @pipe: pipe to splice from 915 * @pipe: pipe to splice from
916 * @sd: information to @actor 916 * @sd: information to @actor
917 * @actor: handler that splices the data 917 * @actor: handler that splices the data
918 * 918 *
919 * Description: 919 * Description:
920 * This function does little more than loop over the pipe and call 920 * This function does little more than loop over the pipe and call
921 * @actor to do the actual moving of a single struct pipe_buffer to 921 * @actor to do the actual moving of a single struct pipe_buffer to
922 * the desired destination. See pipe_to_file, pipe_to_sendpage, or 922 * the desired destination. See pipe_to_file, pipe_to_sendpage, or
923 * pipe_to_user. 923 * pipe_to_user.
924 * 924 *
925 */ 925 */
926 ssize_t __splice_from_pipe(struct pipe_inode_info *pipe, struct splice_desc *sd, 926 ssize_t __splice_from_pipe(struct pipe_inode_info *pipe, struct splice_desc *sd,
927 splice_actor *actor) 927 splice_actor *actor)
928 { 928 {
929 int ret; 929 int ret;
930 930
931 splice_from_pipe_begin(sd); 931 splice_from_pipe_begin(sd);
932 do { 932 do {
933 ret = splice_from_pipe_next(pipe, sd); 933 ret = splice_from_pipe_next(pipe, sd);
934 if (ret > 0) 934 if (ret > 0)
935 ret = splice_from_pipe_feed(pipe, sd, actor); 935 ret = splice_from_pipe_feed(pipe, sd, actor);
936 } while (ret > 0); 936 } while (ret > 0);
937 splice_from_pipe_end(pipe, sd); 937 splice_from_pipe_end(pipe, sd);
938 938
939 return sd->num_spliced ? sd->num_spliced : ret; 939 return sd->num_spliced ? sd->num_spliced : ret;
940 } 940 }
941 EXPORT_SYMBOL(__splice_from_pipe); 941 EXPORT_SYMBOL(__splice_from_pipe);
942 942
943 /** 943 /**
944 * splice_from_pipe - splice data from a pipe to a file 944 * splice_from_pipe - splice data from a pipe to a file
945 * @pipe: pipe to splice from 945 * @pipe: pipe to splice from
946 * @out: file to splice to 946 * @out: file to splice to
947 * @ppos: position in @out 947 * @ppos: position in @out
948 * @len: how many bytes to splice 948 * @len: how many bytes to splice
949 * @flags: splice modifier flags 949 * @flags: splice modifier flags
950 * @actor: handler that splices the data 950 * @actor: handler that splices the data
951 * 951 *
952 * Description: 952 * Description:
953 * See __splice_from_pipe. This function locks the pipe inode, 953 * See __splice_from_pipe. This function locks the pipe inode,
954 * otherwise it's identical to __splice_from_pipe(). 954 * otherwise it's identical to __splice_from_pipe().
955 * 955 *
956 */ 956 */
957 ssize_t splice_from_pipe(struct pipe_inode_info *pipe, struct file *out, 957 ssize_t splice_from_pipe(struct pipe_inode_info *pipe, struct file *out,
958 loff_t *ppos, size_t len, unsigned int flags, 958 loff_t *ppos, size_t len, unsigned int flags,
959 splice_actor *actor) 959 splice_actor *actor)
960 { 960 {
961 ssize_t ret; 961 ssize_t ret;
962 struct splice_desc sd = { 962 struct splice_desc sd = {
963 .total_len = len, 963 .total_len = len,
964 .flags = flags, 964 .flags = flags,
965 .pos = *ppos, 965 .pos = *ppos,
966 .u.file = out, 966 .u.file = out,
967 }; 967 };
968 968
969 pipe_lock(pipe); 969 pipe_lock(pipe);
970 ret = __splice_from_pipe(pipe, &sd, actor); 970 ret = __splice_from_pipe(pipe, &sd, actor);
971 pipe_unlock(pipe); 971 pipe_unlock(pipe);
972 972
973 return ret; 973 return ret;
974 } 974 }
975 975
976 /** 976 /**
977 * generic_file_splice_write - splice data from a pipe to a file 977 * generic_file_splice_write - splice data from a pipe to a file
978 * @pipe: pipe info 978 * @pipe: pipe info
979 * @out: file to write to 979 * @out: file to write to
980 * @ppos: position in @out 980 * @ppos: position in @out
981 * @len: number of bytes to splice 981 * @len: number of bytes to splice
982 * @flags: splice modifier flags 982 * @flags: splice modifier flags
983 * 983 *
984 * Description: 984 * Description:
985 * Will either move or copy pages (determined by @flags options) from 985 * Will either move or copy pages (determined by @flags options) from
986 * the given pipe inode to the given file. 986 * the given pipe inode to the given file.
987 * 987 *
988 */ 988 */
989 ssize_t 989 ssize_t
990 generic_file_splice_write(struct pipe_inode_info *pipe, struct file *out, 990 generic_file_splice_write(struct pipe_inode_info *pipe, struct file *out,
991 loff_t *ppos, size_t len, unsigned int flags) 991 loff_t *ppos, size_t len, unsigned int flags)
992 { 992 {
993 struct address_space *mapping = out->f_mapping; 993 struct address_space *mapping = out->f_mapping;
994 struct inode *inode = mapping->host; 994 struct inode *inode = mapping->host;
995 struct splice_desc sd = { 995 struct splice_desc sd = {
996 .total_len = len, 996 .total_len = len,
997 .flags = flags, 997 .flags = flags,
998 .pos = *ppos, 998 .pos = *ppos,
999 .u.file = out, 999 .u.file = out,
1000 }; 1000 };
1001 ssize_t ret; 1001 ssize_t ret;
1002 1002
1003 pipe_lock(pipe); 1003 pipe_lock(pipe);
1004 1004
1005 splice_from_pipe_begin(&sd); 1005 splice_from_pipe_begin(&sd);
1006 do { 1006 do {
1007 ret = splice_from_pipe_next(pipe, &sd); 1007 ret = splice_from_pipe_next(pipe, &sd);
1008 if (ret <= 0) 1008 if (ret <= 0)
1009 break; 1009 break;
1010 1010
1011 mutex_lock_nested(&inode->i_mutex, I_MUTEX_CHILD); 1011 mutex_lock_nested(&inode->i_mutex, I_MUTEX_CHILD);
1012 ret = file_remove_suid(out); 1012 ret = file_remove_suid(out);
1013 if (!ret) { 1013 if (!ret) {
1014 ret = file_update_time(out); 1014 ret = file_update_time(out);
1015 if (!ret) 1015 if (!ret)
1016 ret = splice_from_pipe_feed(pipe, &sd, 1016 ret = splice_from_pipe_feed(pipe, &sd,
1017 pipe_to_file); 1017 pipe_to_file);
1018 } 1018 }
1019 mutex_unlock(&inode->i_mutex); 1019 mutex_unlock(&inode->i_mutex);
1020 } while (ret > 0); 1020 } while (ret > 0);
1021 splice_from_pipe_end(pipe, &sd); 1021 splice_from_pipe_end(pipe, &sd);
1022 1022
1023 pipe_unlock(pipe); 1023 pipe_unlock(pipe);
1024 1024
1025 if (sd.num_spliced) 1025 if (sd.num_spliced)
1026 ret = sd.num_spliced; 1026 ret = sd.num_spliced;
1027 1027
1028 if (ret > 0) { 1028 if (ret > 0) {
1029 int err; 1029 int err;
1030 1030
1031 err = generic_write_sync(out, *ppos, ret); 1031 err = generic_write_sync(out, *ppos, ret);
1032 if (err) 1032 if (err)
1033 ret = err; 1033 ret = err;
1034 else 1034 else
1035 *ppos += ret; 1035 *ppos += ret;
1036 balance_dirty_pages_ratelimited(mapping); 1036 balance_dirty_pages_ratelimited(mapping);
1037 } 1037 }
1038 1038
1039 return ret; 1039 return ret;
1040 } 1040 }
1041 1041
1042 EXPORT_SYMBOL(generic_file_splice_write); 1042 EXPORT_SYMBOL(generic_file_splice_write);
1043 1043
1044 static int write_pipe_buf(struct pipe_inode_info *pipe, struct pipe_buffer *buf, 1044 static int write_pipe_buf(struct pipe_inode_info *pipe, struct pipe_buffer *buf,
1045 struct splice_desc *sd) 1045 struct splice_desc *sd)
1046 { 1046 {
1047 int ret; 1047 int ret;
1048 void *data; 1048 void *data;
1049 loff_t tmp = sd->pos; 1049 loff_t tmp = sd->pos;
1050 1050
1051 data = buf->ops->map(pipe, buf, 0); 1051 data = buf->ops->map(pipe, buf, 0);
1052 ret = __kernel_write(sd->u.file, data + buf->offset, sd->len, &tmp); 1052 ret = __kernel_write(sd->u.file, data + buf->offset, sd->len, &tmp);
1053 buf->ops->unmap(pipe, buf, data); 1053 buf->ops->unmap(pipe, buf, data);
1054 1054
1055 return ret; 1055 return ret;
1056 } 1056 }
1057 1057
1058 static ssize_t default_file_splice_write(struct pipe_inode_info *pipe, 1058 static ssize_t default_file_splice_write(struct pipe_inode_info *pipe,
1059 struct file *out, loff_t *ppos, 1059 struct file *out, loff_t *ppos,
1060 size_t len, unsigned int flags) 1060 size_t len, unsigned int flags)
1061 { 1061 {
1062 ssize_t ret; 1062 ssize_t ret;
1063 1063
1064 ret = splice_from_pipe(pipe, out, ppos, len, flags, write_pipe_buf); 1064 ret = splice_from_pipe(pipe, out, ppos, len, flags, write_pipe_buf);
1065 if (ret > 0) 1065 if (ret > 0)
1066 *ppos += ret; 1066 *ppos += ret;
1067 1067
1068 return ret; 1068 return ret;
1069 } 1069 }
1070 1070
1071 /** 1071 /**
1072 * generic_splice_sendpage - splice data from a pipe to a socket 1072 * generic_splice_sendpage - splice data from a pipe to a socket
1073 * @pipe: pipe to splice from 1073 * @pipe: pipe to splice from
1074 * @out: socket to write to 1074 * @out: socket to write to
1075 * @ppos: position in @out 1075 * @ppos: position in @out
1076 * @len: number of bytes to splice 1076 * @len: number of bytes to splice
1077 * @flags: splice modifier flags 1077 * @flags: splice modifier flags
1078 * 1078 *
1079 * Description: 1079 * Description:
1080 * Will send @len bytes from the pipe to a network socket. No data copying 1080 * Will send @len bytes from the pipe to a network socket. No data copying
1081 * is involved. 1081 * is involved.
1082 * 1082 *
1083 */ 1083 */
1084 ssize_t generic_splice_sendpage(struct pipe_inode_info *pipe, struct file *out, 1084 ssize_t generic_splice_sendpage(struct pipe_inode_info *pipe, struct file *out,
1085 loff_t *ppos, size_t len, unsigned int flags) 1085 loff_t *ppos, size_t len, unsigned int flags)
1086 { 1086 {
1087 return splice_from_pipe(pipe, out, ppos, len, flags, pipe_to_sendpage); 1087 return splice_from_pipe(pipe, out, ppos, len, flags, pipe_to_sendpage);
1088 } 1088 }
1089 1089
1090 EXPORT_SYMBOL(generic_splice_sendpage); 1090 EXPORT_SYMBOL(generic_splice_sendpage);
1091 1091
1092 /* 1092 /*
1093 * Attempt to initiate a splice from pipe to file. 1093 * Attempt to initiate a splice from pipe to file.
1094 */ 1094 */
1095 static long do_splice_from(struct pipe_inode_info *pipe, struct file *out, 1095 static long do_splice_from(struct pipe_inode_info *pipe, struct file *out,
1096 loff_t *ppos, size_t len, unsigned int flags) 1096 loff_t *ppos, size_t len, unsigned int flags)
1097 { 1097 {
1098 ssize_t (*splice_write)(struct pipe_inode_info *, struct file *, 1098 ssize_t (*splice_write)(struct pipe_inode_info *, struct file *,
1099 loff_t *, size_t, unsigned int); 1099 loff_t *, size_t, unsigned int);
1100 int ret; 1100 int ret;
1101 1101
1102 if (unlikely(!(out->f_mode & FMODE_WRITE))) 1102 if (unlikely(!(out->f_mode & FMODE_WRITE)))
1103 return -EBADF; 1103 return -EBADF;
1104 1104
1105 if (unlikely(out->f_flags & O_APPEND)) 1105 if (unlikely(out->f_flags & O_APPEND))
1106 return -EINVAL; 1106 return -EINVAL;
1107 1107
1108 ret = rw_verify_area(WRITE, out, ppos, len); 1108 ret = rw_verify_area(WRITE, out, ppos, len);
1109 if (unlikely(ret < 0)) 1109 if (unlikely(ret < 0))
1110 return ret; 1110 return ret;
1111 1111
1112 if (out->f_op && out->f_op->splice_write) 1112 if (out->f_op && out->f_op->splice_write)
1113 splice_write = out->f_op->splice_write; 1113 splice_write = out->f_op->splice_write;
1114 else 1114 else
1115 splice_write = default_file_splice_write; 1115 splice_write = default_file_splice_write;
1116 1116
1117 file_start_write(out); 1117 file_start_write(out);
1118 ret = splice_write(pipe, out, ppos, len, flags); 1118 ret = splice_write(pipe, out, ppos, len, flags);
1119 file_end_write(out); 1119 file_end_write(out);
1120 return ret; 1120 return ret;
1121 } 1121 }
1122 1122
1123 /* 1123 /*
1124 * Attempt to initiate a splice from a file to a pipe. 1124 * Attempt to initiate a splice from a file to a pipe.
1125 */ 1125 */
1126 static long do_splice_to(struct file *in, loff_t *ppos, 1126 static long do_splice_to(struct file *in, loff_t *ppos,
1127 struct pipe_inode_info *pipe, size_t len, 1127 struct pipe_inode_info *pipe, size_t len,
1128 unsigned int flags) 1128 unsigned int flags)
1129 { 1129 {
1130 ssize_t (*splice_read)(struct file *, loff_t *, 1130 ssize_t (*splice_read)(struct file *, loff_t *,
1131 struct pipe_inode_info *, size_t, unsigned int); 1131 struct pipe_inode_info *, size_t, unsigned int);
1132 int ret; 1132 int ret;
1133 1133
1134 if (unlikely(!(in->f_mode & FMODE_READ))) 1134 if (unlikely(!(in->f_mode & FMODE_READ)))
1135 return -EBADF; 1135 return -EBADF;
1136 1136
1137 ret = rw_verify_area(READ, in, ppos, len); 1137 ret = rw_verify_area(READ, in, ppos, len);
1138 if (unlikely(ret < 0)) 1138 if (unlikely(ret < 0))
1139 return ret; 1139 return ret;
1140 1140
1141 if (in->f_op && in->f_op->splice_read) 1141 if (in->f_op && in->f_op->splice_read)
1142 splice_read = in->f_op->splice_read; 1142 splice_read = in->f_op->splice_read;
1143 else 1143 else
1144 splice_read = default_file_splice_read; 1144 splice_read = default_file_splice_read;
1145 1145
1146 return splice_read(in, ppos, pipe, len, flags); 1146 return splice_read(in, ppos, pipe, len, flags);
1147 } 1147 }
1148 1148
1149 /** 1149 /**
1150 * splice_direct_to_actor - splices data directly between two non-pipes 1150 * splice_direct_to_actor - splices data directly between two non-pipes
1151 * @in: file to splice from 1151 * @in: file to splice from
1152 * @sd: actor information on where to splice to 1152 * @sd: actor information on where to splice to
1153 * @actor: handles the data splicing 1153 * @actor: handles the data splicing
1154 * 1154 *
1155 * Description: 1155 * Description:
1156 * This is a special case helper to splice directly between two 1156 * This is a special case helper to splice directly between two
1157 * points, without requiring an explicit pipe. Internally an allocated 1157 * points, without requiring an explicit pipe. Internally an allocated
1158 * pipe is cached in the process, and reused during the lifetime of 1158 * pipe is cached in the process, and reused during the lifetime of
1159 * that process. 1159 * that process.
1160 * 1160 *
1161 */ 1161 */
1162 ssize_t splice_direct_to_actor(struct file *in, struct splice_desc *sd, 1162 ssize_t splice_direct_to_actor(struct file *in, struct splice_desc *sd,
1163 splice_direct_actor *actor) 1163 splice_direct_actor *actor)
1164 { 1164 {
1165 struct pipe_inode_info *pipe; 1165 struct pipe_inode_info *pipe;
1166 long ret, bytes; 1166 long ret, bytes;
1167 umode_t i_mode; 1167 umode_t i_mode;
1168 size_t len; 1168 size_t len;
1169 int i, flags; 1169 int i, flags;
1170 1170
1171 /* 1171 /*
1172 * We require the input being a regular file, as we don't want to 1172 * We require the input being a regular file, as we don't want to
1173 * randomly drop data for eg socket -> socket splicing. Use the 1173 * randomly drop data for eg socket -> socket splicing. Use the
1174 * piped splicing for that! 1174 * piped splicing for that!
1175 */ 1175 */
1176 i_mode = file_inode(in)->i_mode; 1176 i_mode = file_inode(in)->i_mode;
1177 if (unlikely(!S_ISREG(i_mode) && !S_ISBLK(i_mode))) 1177 if (unlikely(!S_ISREG(i_mode) && !S_ISBLK(i_mode)))
1178 return -EINVAL; 1178 return -EINVAL;
1179 1179
1180 /* 1180 /*
1181 * neither in nor out is a pipe, setup an internal pipe attached to 1181 * neither in nor out is a pipe, setup an internal pipe attached to
1182 * 'out' and transfer the wanted data from 'in' to 'out' through that 1182 * 'out' and transfer the wanted data from 'in' to 'out' through that
1183 */ 1183 */
1184 pipe = current->splice_pipe; 1184 pipe = current->splice_pipe;
1185 if (unlikely(!pipe)) { 1185 if (unlikely(!pipe)) {
1186 pipe = alloc_pipe_info(NULL); 1186 pipe = alloc_pipe_info();
1187 if (!pipe) 1187 if (!pipe)
1188 return -ENOMEM; 1188 return -ENOMEM;
1189 1189
1190 /* 1190 /*
1191 * We don't have an immediate reader, but we'll read the stuff 1191 * We don't have an immediate reader, but we'll read the stuff
1192 * out of the pipe right after the splice_to_pipe(). So set 1192 * out of the pipe right after the splice_to_pipe(). So set
1193 * PIPE_READERS appropriately. 1193 * PIPE_READERS appropriately.
1194 */ 1194 */
1195 pipe->readers = 1; 1195 pipe->readers = 1;
1196 1196
1197 current->splice_pipe = pipe; 1197 current->splice_pipe = pipe;
1198 } 1198 }
1199 1199
1200 /* 1200 /*
1201 * Do the splice. 1201 * Do the splice.
1202 */ 1202 */
1203 ret = 0; 1203 ret = 0;
1204 bytes = 0; 1204 bytes = 0;
1205 len = sd->total_len; 1205 len = sd->total_len;
1206 flags = sd->flags; 1206 flags = sd->flags;
1207 1207
1208 /* 1208 /*
1209 * Don't block on output, we have to drain the direct pipe. 1209 * Don't block on output, we have to drain the direct pipe.
1210 */ 1210 */
1211 sd->flags &= ~SPLICE_F_NONBLOCK; 1211 sd->flags &= ~SPLICE_F_NONBLOCK;
1212 1212
1213 while (len) { 1213 while (len) {
1214 size_t read_len; 1214 size_t read_len;
1215 loff_t pos = sd->pos, prev_pos = pos; 1215 loff_t pos = sd->pos, prev_pos = pos;
1216 1216
1217 ret = do_splice_to(in, &pos, pipe, len, flags); 1217 ret = do_splice_to(in, &pos, pipe, len, flags);
1218 if (unlikely(ret <= 0)) 1218 if (unlikely(ret <= 0))
1219 goto out_release; 1219 goto out_release;
1220 1220
1221 read_len = ret; 1221 read_len = ret;
1222 sd->total_len = read_len; 1222 sd->total_len = read_len;
1223 1223
1224 /* 1224 /*
1225 * NOTE: nonblocking mode only applies to the input. We 1225 * NOTE: nonblocking mode only applies to the input. We
1226 * must not do the output in nonblocking mode as then we 1226 * must not do the output in nonblocking mode as then we
1227 * could get stuck data in the internal pipe: 1227 * could get stuck data in the internal pipe:
1228 */ 1228 */
1229 ret = actor(pipe, sd); 1229 ret = actor(pipe, sd);
1230 if (unlikely(ret <= 0)) { 1230 if (unlikely(ret <= 0)) {
1231 sd->pos = prev_pos; 1231 sd->pos = prev_pos;
1232 goto out_release; 1232 goto out_release;
1233 } 1233 }
1234 1234
1235 bytes += ret; 1235 bytes += ret;
1236 len -= ret; 1236 len -= ret;
1237 sd->pos = pos; 1237 sd->pos = pos;
1238 1238
1239 if (ret < read_len) { 1239 if (ret < read_len) {
1240 sd->pos = prev_pos + ret; 1240 sd->pos = prev_pos + ret;
1241 goto out_release; 1241 goto out_release;
1242 } 1242 }
1243 } 1243 }
1244 1244
1245 done: 1245 done:
1246 pipe->nrbufs = pipe->curbuf = 0; 1246 pipe->nrbufs = pipe->curbuf = 0;
1247 file_accessed(in); 1247 file_accessed(in);
1248 return bytes; 1248 return bytes;
1249 1249
1250 out_release: 1250 out_release:
1251 /* 1251 /*
1252 * If we did an incomplete transfer we must release 1252 * If we did an incomplete transfer we must release
1253 * the pipe buffers in question: 1253 * the pipe buffers in question:
1254 */ 1254 */
1255 for (i = 0; i < pipe->buffers; i++) { 1255 for (i = 0; i < pipe->buffers; i++) {
1256 struct pipe_buffer *buf = pipe->bufs + i; 1256 struct pipe_buffer *buf = pipe->bufs + i;
1257 1257
1258 if (buf->ops) { 1258 if (buf->ops) {
1259 buf->ops->release(pipe, buf); 1259 buf->ops->release(pipe, buf);
1260 buf->ops = NULL; 1260 buf->ops = NULL;
1261 } 1261 }
1262 } 1262 }
1263 1263
1264 if (!bytes) 1264 if (!bytes)
1265 bytes = ret; 1265 bytes = ret;
1266 1266
1267 goto done; 1267 goto done;
1268 } 1268 }
1269 EXPORT_SYMBOL(splice_direct_to_actor); 1269 EXPORT_SYMBOL(splice_direct_to_actor);
1270 1270
1271 static int direct_splice_actor(struct pipe_inode_info *pipe, 1271 static int direct_splice_actor(struct pipe_inode_info *pipe,
1272 struct splice_desc *sd) 1272 struct splice_desc *sd)
1273 { 1273 {
1274 struct file *file = sd->u.file; 1274 struct file *file = sd->u.file;
1275 1275
1276 return do_splice_from(pipe, file, &file->f_pos, sd->total_len, 1276 return do_splice_from(pipe, file, &file->f_pos, sd->total_len,
1277 sd->flags); 1277 sd->flags);
1278 } 1278 }
1279 1279
1280 /** 1280 /**
1281 * do_splice_direct - splices data directly between two files 1281 * do_splice_direct - splices data directly between two files
1282 * @in: file to splice from 1282 * @in: file to splice from
1283 * @ppos: input file offset 1283 * @ppos: input file offset
1284 * @out: file to splice to 1284 * @out: file to splice to
1285 * @len: number of bytes to splice 1285 * @len: number of bytes to splice
1286 * @flags: splice modifier flags 1286 * @flags: splice modifier flags
1287 * 1287 *
1288 * Description: 1288 * Description:
1289 * For use by do_sendfile(). splice can easily emulate sendfile, but 1289 * For use by do_sendfile(). splice can easily emulate sendfile, but
1290 * doing it in the application would incur an extra system call 1290 * doing it in the application would incur an extra system call
1291 * (splice in + splice out, as compared to just sendfile()). So this helper 1291 * (splice in + splice out, as compared to just sendfile()). So this helper
1292 * can splice directly through a process-private pipe. 1292 * can splice directly through a process-private pipe.
1293 * 1293 *
1294 */ 1294 */
1295 long do_splice_direct(struct file *in, loff_t *ppos, struct file *out, 1295 long do_splice_direct(struct file *in, loff_t *ppos, struct file *out,
1296 size_t len, unsigned int flags) 1296 size_t len, unsigned int flags)
1297 { 1297 {
1298 struct splice_desc sd = { 1298 struct splice_desc sd = {
1299 .len = len, 1299 .len = len,
1300 .total_len = len, 1300 .total_len = len,
1301 .flags = flags, 1301 .flags = flags,
1302 .pos = *ppos, 1302 .pos = *ppos,
1303 .u.file = out, 1303 .u.file = out,
1304 }; 1304 };
1305 long ret; 1305 long ret;
1306 1306
1307 ret = splice_direct_to_actor(in, &sd, direct_splice_actor); 1307 ret = splice_direct_to_actor(in, &sd, direct_splice_actor);
1308 if (ret > 0) 1308 if (ret > 0)
1309 *ppos = sd.pos; 1309 *ppos = sd.pos;
1310 1310
1311 return ret; 1311 return ret;
1312 } 1312 }
1313 1313
1314 static int splice_pipe_to_pipe(struct pipe_inode_info *ipipe, 1314 static int splice_pipe_to_pipe(struct pipe_inode_info *ipipe,
1315 struct pipe_inode_info *opipe, 1315 struct pipe_inode_info *opipe,
1316 size_t len, unsigned int flags); 1316 size_t len, unsigned int flags);
1317 1317
1318 /* 1318 /*
1319 * Determine where to splice to/from. 1319 * Determine where to splice to/from.
1320 */ 1320 */
1321 static long do_splice(struct file *in, loff_t __user *off_in, 1321 static long do_splice(struct file *in, loff_t __user *off_in,
1322 struct file *out, loff_t __user *off_out, 1322 struct file *out, loff_t __user *off_out,
1323 size_t len, unsigned int flags) 1323 size_t len, unsigned int flags)
1324 { 1324 {
1325 struct pipe_inode_info *ipipe; 1325 struct pipe_inode_info *ipipe;
1326 struct pipe_inode_info *opipe; 1326 struct pipe_inode_info *opipe;
1327 loff_t offset, *off; 1327 loff_t offset, *off;
1328 long ret; 1328 long ret;
1329 1329
1330 ipipe = get_pipe_info(in); 1330 ipipe = get_pipe_info(in);
1331 opipe = get_pipe_info(out); 1331 opipe = get_pipe_info(out);
1332 1332
1333 if (ipipe && opipe) { 1333 if (ipipe && opipe) {
1334 if (off_in || off_out) 1334 if (off_in || off_out)
1335 return -ESPIPE; 1335 return -ESPIPE;
1336 1336
1337 if (!(in->f_mode & FMODE_READ)) 1337 if (!(in->f_mode & FMODE_READ))
1338 return -EBADF; 1338 return -EBADF;
1339 1339
1340 if (!(out->f_mode & FMODE_WRITE)) 1340 if (!(out->f_mode & FMODE_WRITE))
1341 return -EBADF; 1341 return -EBADF;
1342 1342
1343 /* Splicing to self would be fun, but... */ 1343 /* Splicing to self would be fun, but... */
1344 if (ipipe == opipe) 1344 if (ipipe == opipe)
1345 return -EINVAL; 1345 return -EINVAL;
1346 1346
1347 return splice_pipe_to_pipe(ipipe, opipe, len, flags); 1347 return splice_pipe_to_pipe(ipipe, opipe, len, flags);
1348 } 1348 }
1349 1349
1350 if (ipipe) { 1350 if (ipipe) {
1351 if (off_in) 1351 if (off_in)
1352 return -ESPIPE; 1352 return -ESPIPE;
1353 if (off_out) { 1353 if (off_out) {
1354 if (!(out->f_mode & FMODE_PWRITE)) 1354 if (!(out->f_mode & FMODE_PWRITE))
1355 return -EINVAL; 1355 return -EINVAL;
1356 if (copy_from_user(&offset, off_out, sizeof(loff_t))) 1356 if (copy_from_user(&offset, off_out, sizeof(loff_t)))
1357 return -EFAULT; 1357 return -EFAULT;
1358 off = &offset; 1358 off = &offset;
1359 } else 1359 } else
1360 off = &out->f_pos; 1360 off = &out->f_pos;
1361 1361
1362 ret = do_splice_from(ipipe, out, off, len, flags); 1362 ret = do_splice_from(ipipe, out, off, len, flags);
1363 1363
1364 if (off_out && copy_to_user(off_out, off, sizeof(loff_t))) 1364 if (off_out && copy_to_user(off_out, off, sizeof(loff_t)))
1365 ret = -EFAULT; 1365 ret = -EFAULT;
1366 1366
1367 return ret; 1367 return ret;
1368 } 1368 }
1369 1369
1370 if (opipe) { 1370 if (opipe) {
1371 if (off_out) 1371 if (off_out)
1372 return -ESPIPE; 1372 return -ESPIPE;
1373 if (off_in) { 1373 if (off_in) {
1374 if (!(in->f_mode & FMODE_PREAD)) 1374 if (!(in->f_mode & FMODE_PREAD))
1375 return -EINVAL; 1375 return -EINVAL;
1376 if (copy_from_user(&offset, off_in, sizeof(loff_t))) 1376 if (copy_from_user(&offset, off_in, sizeof(loff_t)))
1377 return -EFAULT; 1377 return -EFAULT;
1378 off = &offset; 1378 off = &offset;
1379 } else 1379 } else
1380 off = &in->f_pos; 1380 off = &in->f_pos;
1381 1381
1382 ret = do_splice_to(in, off, opipe, len, flags); 1382 ret = do_splice_to(in, off, opipe, len, flags);
1383 1383
1384 if (off_in && copy_to_user(off_in, off, sizeof(loff_t))) 1384 if (off_in && copy_to_user(off_in, off, sizeof(loff_t)))
1385 ret = -EFAULT; 1385 ret = -EFAULT;
1386 1386
1387 return ret; 1387 return ret;
1388 } 1388 }
1389 1389
1390 return -EINVAL; 1390 return -EINVAL;
1391 } 1391 }
1392 1392
1393 /* 1393 /*
1394 * Map an iov into an array of pages and offset/length tupples. With the 1394 * Map an iov into an array of pages and offset/length tupples. With the
1395 * partial_page structure, we can map several non-contiguous ranges into 1395 * partial_page structure, we can map several non-contiguous ranges into
1396 * our ones pages[] map instead of splitting that operation into pieces. 1396 * our ones pages[] map instead of splitting that operation into pieces.
1397 * Could easily be exported as a generic helper for other users, in which 1397 * Could easily be exported as a generic helper for other users, in which
1398 * case one would probably want to add a 'max_nr_pages' parameter as well. 1398 * case one would probably want to add a 'max_nr_pages' parameter as well.
1399 */ 1399 */
1400 static int get_iovec_page_array(const struct iovec __user *iov, 1400 static int get_iovec_page_array(const struct iovec __user *iov,
1401 unsigned int nr_vecs, struct page **pages, 1401 unsigned int nr_vecs, struct page **pages,
1402 struct partial_page *partial, bool aligned, 1402 struct partial_page *partial, bool aligned,
1403 unsigned int pipe_buffers) 1403 unsigned int pipe_buffers)
1404 { 1404 {
1405 int buffers = 0, error = 0; 1405 int buffers = 0, error = 0;
1406 1406
1407 while (nr_vecs) { 1407 while (nr_vecs) {
1408 unsigned long off, npages; 1408 unsigned long off, npages;
1409 struct iovec entry; 1409 struct iovec entry;
1410 void __user *base; 1410 void __user *base;
1411 size_t len; 1411 size_t len;
1412 int i; 1412 int i;
1413 1413
1414 error = -EFAULT; 1414 error = -EFAULT;
1415 if (copy_from_user(&entry, iov, sizeof(entry))) 1415 if (copy_from_user(&entry, iov, sizeof(entry)))
1416 break; 1416 break;
1417 1417
1418 base = entry.iov_base; 1418 base = entry.iov_base;
1419 len = entry.iov_len; 1419 len = entry.iov_len;
1420 1420
1421 /* 1421 /*
1422 * Sanity check this iovec. 0 read succeeds. 1422 * Sanity check this iovec. 0 read succeeds.
1423 */ 1423 */
1424 error = 0; 1424 error = 0;
1425 if (unlikely(!len)) 1425 if (unlikely(!len))
1426 break; 1426 break;
1427 error = -EFAULT; 1427 error = -EFAULT;
1428 if (!access_ok(VERIFY_READ, base, len)) 1428 if (!access_ok(VERIFY_READ, base, len))
1429 break; 1429 break;
1430 1430
1431 /* 1431 /*
1432 * Get this base offset and number of pages, then map 1432 * Get this base offset and number of pages, then map
1433 * in the user pages. 1433 * in the user pages.
1434 */ 1434 */
1435 off = (unsigned long) base & ~PAGE_MASK; 1435 off = (unsigned long) base & ~PAGE_MASK;
1436 1436
1437 /* 1437 /*
1438 * If asked for alignment, the offset must be zero and the 1438 * If asked for alignment, the offset must be zero and the
1439 * length a multiple of the PAGE_SIZE. 1439 * length a multiple of the PAGE_SIZE.
1440 */ 1440 */
1441 error = -EINVAL; 1441 error = -EINVAL;
1442 if (aligned && (off || len & ~PAGE_MASK)) 1442 if (aligned && (off || len & ~PAGE_MASK))
1443 break; 1443 break;
1444 1444
1445 npages = (off + len + PAGE_SIZE - 1) >> PAGE_SHIFT; 1445 npages = (off + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
1446 if (npages > pipe_buffers - buffers) 1446 if (npages > pipe_buffers - buffers)
1447 npages = pipe_buffers - buffers; 1447 npages = pipe_buffers - buffers;
1448 1448
1449 error = get_user_pages_fast((unsigned long)base, npages, 1449 error = get_user_pages_fast((unsigned long)base, npages,
1450 0, &pages[buffers]); 1450 0, &pages[buffers]);
1451 1451
1452 if (unlikely(error <= 0)) 1452 if (unlikely(error <= 0))
1453 break; 1453 break;
1454 1454
1455 /* 1455 /*
1456 * Fill this contiguous range into the partial page map. 1456 * Fill this contiguous range into the partial page map.
1457 */ 1457 */
1458 for (i = 0; i < error; i++) { 1458 for (i = 0; i < error; i++) {
1459 const int plen = min_t(size_t, len, PAGE_SIZE - off); 1459 const int plen = min_t(size_t, len, PAGE_SIZE - off);
1460 1460
1461 partial[buffers].offset = off; 1461 partial[buffers].offset = off;
1462 partial[buffers].len = plen; 1462 partial[buffers].len = plen;
1463 1463
1464 off = 0; 1464 off = 0;
1465 len -= plen; 1465 len -= plen;
1466 buffers++; 1466 buffers++;
1467 } 1467 }
1468 1468
1469 /* 1469 /*
1470 * We didn't complete this iov, stop here since it probably 1470 * We didn't complete this iov, stop here since it probably
1471 * means we have to move some of this into a pipe to 1471 * means we have to move some of this into a pipe to
1472 * be able to continue. 1472 * be able to continue.
1473 */ 1473 */
1474 if (len) 1474 if (len)
1475 break; 1475 break;
1476 1476
1477 /* 1477 /*
1478 * Don't continue if we mapped fewer pages than we asked for, 1478 * Don't continue if we mapped fewer pages than we asked for,
1479 * or if we mapped the max number of pages that we have 1479 * or if we mapped the max number of pages that we have
1480 * room for. 1480 * room for.
1481 */ 1481 */
1482 if (error < npages || buffers == pipe_buffers) 1482 if (error < npages || buffers == pipe_buffers)
1483 break; 1483 break;
1484 1484
1485 nr_vecs--; 1485 nr_vecs--;
1486 iov++; 1486 iov++;
1487 } 1487 }
1488 1488
1489 if (buffers) 1489 if (buffers)
1490 return buffers; 1490 return buffers;
1491 1491
1492 return error; 1492 return error;
1493 } 1493 }
1494 1494
1495 static int pipe_to_user(struct pipe_inode_info *pipe, struct pipe_buffer *buf, 1495 static int pipe_to_user(struct pipe_inode_info *pipe, struct pipe_buffer *buf,
1496 struct splice_desc *sd) 1496 struct splice_desc *sd)
1497 { 1497 {
1498 char *src; 1498 char *src;
1499 int ret; 1499 int ret;
1500 1500
1501 /* 1501 /*
1502 * See if we can use the atomic maps, by prefaulting in the 1502 * See if we can use the atomic maps, by prefaulting in the
1503 * pages and doing an atomic copy 1503 * pages and doing an atomic copy
1504 */ 1504 */
1505 if (!fault_in_pages_writeable(sd->u.userptr, sd->len)) { 1505 if (!fault_in_pages_writeable(sd->u.userptr, sd->len)) {
1506 src = buf->ops->map(pipe, buf, 1); 1506 src = buf->ops->map(pipe, buf, 1);
1507 ret = __copy_to_user_inatomic(sd->u.userptr, src + buf->offset, 1507 ret = __copy_to_user_inatomic(sd->u.userptr, src + buf->offset,
1508 sd->len); 1508 sd->len);
1509 buf->ops->unmap(pipe, buf, src); 1509 buf->ops->unmap(pipe, buf, src);
1510 if (!ret) { 1510 if (!ret) {
1511 ret = sd->len; 1511 ret = sd->len;
1512 goto out; 1512 goto out;
1513 } 1513 }
1514 } 1514 }
1515 1515
1516 /* 1516 /*
1517 * No dice, use slow non-atomic map and copy 1517 * No dice, use slow non-atomic map and copy
1518 */ 1518 */
1519 src = buf->ops->map(pipe, buf, 0); 1519 src = buf->ops->map(pipe, buf, 0);
1520 1520
1521 ret = sd->len; 1521 ret = sd->len;
1522 if (copy_to_user(sd->u.userptr, src + buf->offset, sd->len)) 1522 if (copy_to_user(sd->u.userptr, src + buf->offset, sd->len))
1523 ret = -EFAULT; 1523 ret = -EFAULT;
1524 1524
1525 buf->ops->unmap(pipe, buf, src); 1525 buf->ops->unmap(pipe, buf, src);
1526 out: 1526 out:
1527 if (ret > 0) 1527 if (ret > 0)
1528 sd->u.userptr += ret; 1528 sd->u.userptr += ret;
1529 return ret; 1529 return ret;
1530 } 1530 }
1531 1531
1532 /* 1532 /*
1533 * For lack of a better implementation, implement vmsplice() to userspace 1533 * For lack of a better implementation, implement vmsplice() to userspace
1534 * as a simple copy of the pipes pages to the user iov. 1534 * as a simple copy of the pipes pages to the user iov.
1535 */ 1535 */
1536 static long vmsplice_to_user(struct file *file, const struct iovec __user *iov, 1536 static long vmsplice_to_user(struct file *file, const struct iovec __user *iov,
1537 unsigned long nr_segs, unsigned int flags) 1537 unsigned long nr_segs, unsigned int flags)
1538 { 1538 {
1539 struct pipe_inode_info *pipe; 1539 struct pipe_inode_info *pipe;
1540 struct splice_desc sd; 1540 struct splice_desc sd;
1541 ssize_t size; 1541 ssize_t size;
1542 int error; 1542 int error;
1543 long ret; 1543 long ret;
1544 1544
1545 pipe = get_pipe_info(file); 1545 pipe = get_pipe_info(file);
1546 if (!pipe) 1546 if (!pipe)
1547 return -EBADF; 1547 return -EBADF;
1548 1548
1549 pipe_lock(pipe); 1549 pipe_lock(pipe);
1550 1550
1551 error = ret = 0; 1551 error = ret = 0;
1552 while (nr_segs) { 1552 while (nr_segs) {
1553 void __user *base; 1553 void __user *base;
1554 size_t len; 1554 size_t len;
1555 1555
1556 /* 1556 /*
1557 * Get user address base and length for this iovec. 1557 * Get user address base and length for this iovec.
1558 */ 1558 */
1559 error = get_user(base, &iov->iov_base); 1559 error = get_user(base, &iov->iov_base);
1560 if (unlikely(error)) 1560 if (unlikely(error))
1561 break; 1561 break;
1562 error = get_user(len, &iov->iov_len); 1562 error = get_user(len, &iov->iov_len);
1563 if (unlikely(error)) 1563 if (unlikely(error))
1564 break; 1564 break;
1565 1565
1566 /* 1566 /*
1567 * Sanity check this iovec. 0 read succeeds. 1567 * Sanity check this iovec. 0 read succeeds.
1568 */ 1568 */
1569 if (unlikely(!len)) 1569 if (unlikely(!len))
1570 break; 1570 break;
1571 if (unlikely(!base)) { 1571 if (unlikely(!base)) {
1572 error = -EFAULT; 1572 error = -EFAULT;
1573 break; 1573 break;
1574 } 1574 }
1575 1575
1576 if (unlikely(!access_ok(VERIFY_WRITE, base, len))) { 1576 if (unlikely(!access_ok(VERIFY_WRITE, base, len))) {
1577 error = -EFAULT; 1577 error = -EFAULT;
1578 break; 1578 break;
1579 } 1579 }
1580 1580
1581 sd.len = 0; 1581 sd.len = 0;
1582 sd.total_len = len; 1582 sd.total_len = len;
1583 sd.flags = flags; 1583 sd.flags = flags;
1584 sd.u.userptr = base; 1584 sd.u.userptr = base;
1585 sd.pos = 0; 1585 sd.pos = 0;
1586 1586
1587 size = __splice_from_pipe(pipe, &sd, pipe_to_user); 1587 size = __splice_from_pipe(pipe, &sd, pipe_to_user);
1588 if (size < 0) { 1588 if (size < 0) {
1589 if (!ret) 1589 if (!ret)
1590 ret = size; 1590 ret = size;
1591 1591
1592 break; 1592 break;
1593 } 1593 }
1594 1594
1595 ret += size; 1595 ret += size;
1596 1596
1597 if (size < len) 1597 if (size < len)
1598 break; 1598 break;
1599 1599
1600 nr_segs--; 1600 nr_segs--;
1601 iov++; 1601 iov++;
1602 } 1602 }
1603 1603
1604 pipe_unlock(pipe); 1604 pipe_unlock(pipe);
1605 1605
1606 if (!ret) 1606 if (!ret)
1607 ret = error; 1607 ret = error;
1608 1608
1609 return ret; 1609 return ret;
1610 } 1610 }
1611 1611
1612 /* 1612 /*
1613 * vmsplice splices a user address range into a pipe. It can be thought of 1613 * vmsplice splices a user address range into a pipe. It can be thought of
1614 * as splice-from-memory, where the regular splice is splice-from-file (or 1614 * as splice-from-memory, where the regular splice is splice-from-file (or
1615 * to file). In both cases the output is a pipe, naturally. 1615 * to file). In both cases the output is a pipe, naturally.
1616 */ 1616 */
1617 static long vmsplice_to_pipe(struct file *file, const struct iovec __user *iov, 1617 static long vmsplice_to_pipe(struct file *file, const struct iovec __user *iov,
1618 unsigned long nr_segs, unsigned int flags) 1618 unsigned long nr_segs, unsigned int flags)
1619 { 1619 {
1620 struct pipe_inode_info *pipe; 1620 struct pipe_inode_info *pipe;
1621 struct page *pages[PIPE_DEF_BUFFERS]; 1621 struct page *pages[PIPE_DEF_BUFFERS];
1622 struct partial_page partial[PIPE_DEF_BUFFERS]; 1622 struct partial_page partial[PIPE_DEF_BUFFERS];
1623 struct splice_pipe_desc spd = { 1623 struct splice_pipe_desc spd = {
1624 .pages = pages, 1624 .pages = pages,
1625 .partial = partial, 1625 .partial = partial,
1626 .nr_pages_max = PIPE_DEF_BUFFERS, 1626 .nr_pages_max = PIPE_DEF_BUFFERS,
1627 .flags = flags, 1627 .flags = flags,
1628 .ops = &user_page_pipe_buf_ops, 1628 .ops = &user_page_pipe_buf_ops,
1629 .spd_release = spd_release_page, 1629 .spd_release = spd_release_page,
1630 }; 1630 };
1631 long ret; 1631 long ret;
1632 1632
1633 pipe = get_pipe_info(file); 1633 pipe = get_pipe_info(file);
1634 if (!pipe) 1634 if (!pipe)
1635 return -EBADF; 1635 return -EBADF;
1636 1636
1637 if (splice_grow_spd(pipe, &spd)) 1637 if (splice_grow_spd(pipe, &spd))
1638 return -ENOMEM; 1638 return -ENOMEM;
1639 1639
1640 spd.nr_pages = get_iovec_page_array(iov, nr_segs, spd.pages, 1640 spd.nr_pages = get_iovec_page_array(iov, nr_segs, spd.pages,
1641 spd.partial, false, 1641 spd.partial, false,
1642 spd.nr_pages_max); 1642 spd.nr_pages_max);
1643 if (spd.nr_pages <= 0) 1643 if (spd.nr_pages <= 0)
1644 ret = spd.nr_pages; 1644 ret = spd.nr_pages;
1645 else 1645 else
1646 ret = splice_to_pipe(pipe, &spd); 1646 ret = splice_to_pipe(pipe, &spd);
1647 1647
1648 splice_shrink_spd(&spd); 1648 splice_shrink_spd(&spd);
1649 return ret; 1649 return ret;
1650 } 1650 }
1651 1651
1652 /* 1652 /*
1653 * Note that vmsplice only really supports true splicing _from_ user memory 1653 * Note that vmsplice only really supports true splicing _from_ user memory
1654 * to a pipe, not the other way around. Splicing from user memory is a simple 1654 * to a pipe, not the other way around. Splicing from user memory is a simple
1655 * operation that can be supported without any funky alignment restrictions 1655 * operation that can be supported without any funky alignment restrictions
1656 * or nasty vm tricks. We simply map in the user memory and fill them into 1656 * or nasty vm tricks. We simply map in the user memory and fill them into
1657 * a pipe. The reverse isn't quite as easy, though. There are two possible 1657 * a pipe. The reverse isn't quite as easy, though. There are two possible
1658 * solutions for that: 1658 * solutions for that:
1659 * 1659 *
1660 * - memcpy() the data internally, at which point we might as well just 1660 * - memcpy() the data internally, at which point we might as well just
1661 * do a regular read() on the buffer anyway. 1661 * do a regular read() on the buffer anyway.
1662 * - Lots of nasty vm tricks, that are neither fast nor flexible (it 1662 * - Lots of nasty vm tricks, that are neither fast nor flexible (it
1663 * has restriction limitations on both ends of the pipe). 1663 * has restriction limitations on both ends of the pipe).
1664 * 1664 *
1665 * Currently we punt and implement it as a normal copy, see pipe_to_user(). 1665 * Currently we punt and implement it as a normal copy, see pipe_to_user().
1666 * 1666 *
1667 */ 1667 */
1668 SYSCALL_DEFINE4(vmsplice, int, fd, const struct iovec __user *, iov, 1668 SYSCALL_DEFINE4(vmsplice, int, fd, const struct iovec __user *, iov,
1669 unsigned long, nr_segs, unsigned int, flags) 1669 unsigned long, nr_segs, unsigned int, flags)
1670 { 1670 {
1671 struct fd f; 1671 struct fd f;
1672 long error; 1672 long error;
1673 1673
1674 if (unlikely(nr_segs > UIO_MAXIOV)) 1674 if (unlikely(nr_segs > UIO_MAXIOV))
1675 return -EINVAL; 1675 return -EINVAL;
1676 else if (unlikely(!nr_segs)) 1676 else if (unlikely(!nr_segs))
1677 return 0; 1677 return 0;
1678 1678
1679 error = -EBADF; 1679 error = -EBADF;
1680 f = fdget(fd); 1680 f = fdget(fd);
1681 if (f.file) { 1681 if (f.file) {
1682 if (f.file->f_mode & FMODE_WRITE) 1682 if (f.file->f_mode & FMODE_WRITE)
1683 error = vmsplice_to_pipe(f.file, iov, nr_segs, flags); 1683 error = vmsplice_to_pipe(f.file, iov, nr_segs, flags);
1684 else if (f.file->f_mode & FMODE_READ) 1684 else if (f.file->f_mode & FMODE_READ)
1685 error = vmsplice_to_user(f.file, iov, nr_segs, flags); 1685 error = vmsplice_to_user(f.file, iov, nr_segs, flags);
1686 1686
1687 fdput(f); 1687 fdput(f);
1688 } 1688 }
1689 1689
1690 return error; 1690 return error;
1691 } 1691 }
1692 1692
1693 SYSCALL_DEFINE6(splice, int, fd_in, loff_t __user *, off_in, 1693 SYSCALL_DEFINE6(splice, int, fd_in, loff_t __user *, off_in,
1694 int, fd_out, loff_t __user *, off_out, 1694 int, fd_out, loff_t __user *, off_out,
1695 size_t, len, unsigned int, flags) 1695 size_t, len, unsigned int, flags)
1696 { 1696 {
1697 struct fd in, out; 1697 struct fd in, out;
1698 long error; 1698 long error;
1699 1699
1700 if (unlikely(!len)) 1700 if (unlikely(!len))
1701 return 0; 1701 return 0;
1702 1702
1703 error = -EBADF; 1703 error = -EBADF;
1704 in = fdget(fd_in); 1704 in = fdget(fd_in);
1705 if (in.file) { 1705 if (in.file) {
1706 if (in.file->f_mode & FMODE_READ) { 1706 if (in.file->f_mode & FMODE_READ) {
1707 out = fdget(fd_out); 1707 out = fdget(fd_out);
1708 if (out.file) { 1708 if (out.file) {
1709 if (out.file->f_mode & FMODE_WRITE) 1709 if (out.file->f_mode & FMODE_WRITE)
1710 error = do_splice(in.file, off_in, 1710 error = do_splice(in.file, off_in,
1711 out.file, off_out, 1711 out.file, off_out,
1712 len, flags); 1712 len, flags);
1713 fdput(out); 1713 fdput(out);
1714 } 1714 }
1715 } 1715 }
1716 fdput(in); 1716 fdput(in);
1717 } 1717 }
1718 return error; 1718 return error;
1719 } 1719 }
1720 1720
1721 /* 1721 /*
1722 * Make sure there's data to read. Wait for input if we can, otherwise 1722 * Make sure there's data to read. Wait for input if we can, otherwise
1723 * return an appropriate error. 1723 * return an appropriate error.
1724 */ 1724 */
1725 static int ipipe_prep(struct pipe_inode_info *pipe, unsigned int flags) 1725 static int ipipe_prep(struct pipe_inode_info *pipe, unsigned int flags)
1726 { 1726 {
1727 int ret; 1727 int ret;
1728 1728
1729 /* 1729 /*
1730 * Check ->nrbufs without the inode lock first. This function 1730 * Check ->nrbufs without the inode lock first. This function
1731 * is speculative anyways, so missing one is ok. 1731 * is speculative anyways, so missing one is ok.
1732 */ 1732 */
1733 if (pipe->nrbufs) 1733 if (pipe->nrbufs)
1734 return 0; 1734 return 0;
1735 1735
1736 ret = 0; 1736 ret = 0;
1737 pipe_lock(pipe); 1737 pipe_lock(pipe);
1738 1738
1739 while (!pipe->nrbufs) { 1739 while (!pipe->nrbufs) {
1740 if (signal_pending(current)) { 1740 if (signal_pending(current)) {
1741 ret = -ERESTARTSYS; 1741 ret = -ERESTARTSYS;
1742 break; 1742 break;
1743 } 1743 }
1744 if (!pipe->writers) 1744 if (!pipe->writers)
1745 break; 1745 break;
1746 if (!pipe->waiting_writers) { 1746 if (!pipe->waiting_writers) {
1747 if (flags & SPLICE_F_NONBLOCK) { 1747 if (flags & SPLICE_F_NONBLOCK) {
1748 ret = -EAGAIN; 1748 ret = -EAGAIN;
1749 break; 1749 break;
1750 } 1750 }
1751 } 1751 }
1752 pipe_wait(pipe); 1752 pipe_wait(pipe);
1753 } 1753 }
1754 1754
1755 pipe_unlock(pipe); 1755 pipe_unlock(pipe);
1756 return ret; 1756 return ret;
1757 } 1757 }
1758 1758
1759 /* 1759 /*
1760 * Make sure there's writeable room. Wait for room if we can, otherwise 1760 * Make sure there's writeable room. Wait for room if we can, otherwise
1761 * return an appropriate error. 1761 * return an appropriate error.
1762 */ 1762 */
1763 static int opipe_prep(struct pipe_inode_info *pipe, unsigned int flags) 1763 static int opipe_prep(struct pipe_inode_info *pipe, unsigned int flags)
1764 { 1764 {
1765 int ret; 1765 int ret;
1766 1766
1767 /* 1767 /*
1768 * Check ->nrbufs without the inode lock first. This function 1768 * Check ->nrbufs without the inode lock first. This function
1769 * is speculative anyways, so missing one is ok. 1769 * is speculative anyways, so missing one is ok.
1770 */ 1770 */
1771 if (pipe->nrbufs < pipe->buffers) 1771 if (pipe->nrbufs < pipe->buffers)
1772 return 0; 1772 return 0;
1773 1773
1774 ret = 0; 1774 ret = 0;
1775 pipe_lock(pipe); 1775 pipe_lock(pipe);
1776 1776
1777 while (pipe->nrbufs >= pipe->buffers) { 1777 while (pipe->nrbufs >= pipe->buffers) {
1778 if (!pipe->readers) { 1778 if (!pipe->readers) {
1779 send_sig(SIGPIPE, current, 0); 1779 send_sig(SIGPIPE, current, 0);
1780 ret = -EPIPE; 1780 ret = -EPIPE;
1781 break; 1781 break;
1782 } 1782 }
1783 if (flags & SPLICE_F_NONBLOCK) { 1783 if (flags & SPLICE_F_NONBLOCK) {
1784 ret = -EAGAIN; 1784 ret = -EAGAIN;
1785 break; 1785 break;
1786 } 1786 }
1787 if (signal_pending(current)) { 1787 if (signal_pending(current)) {
1788 ret = -ERESTARTSYS; 1788 ret = -ERESTARTSYS;
1789 break; 1789 break;
1790 } 1790 }
1791 pipe->waiting_writers++; 1791 pipe->waiting_writers++;
1792 pipe_wait(pipe); 1792 pipe_wait(pipe);
1793 pipe->waiting_writers--; 1793 pipe->waiting_writers--;
1794 } 1794 }
1795 1795
1796 pipe_unlock(pipe); 1796 pipe_unlock(pipe);
1797 return ret; 1797 return ret;
1798 } 1798 }
1799 1799
1800 /* 1800 /*
1801 * Splice contents of ipipe to opipe. 1801 * Splice contents of ipipe to opipe.
1802 */ 1802 */
1803 static int splice_pipe_to_pipe(struct pipe_inode_info *ipipe, 1803 static int splice_pipe_to_pipe(struct pipe_inode_info *ipipe,
1804 struct pipe_inode_info *opipe, 1804 struct pipe_inode_info *opipe,
1805 size_t len, unsigned int flags) 1805 size_t len, unsigned int flags)
1806 { 1806 {
1807 struct pipe_buffer *ibuf, *obuf; 1807 struct pipe_buffer *ibuf, *obuf;
1808 int ret = 0, nbuf; 1808 int ret = 0, nbuf;
1809 bool input_wakeup = false; 1809 bool input_wakeup = false;
1810 1810
1811 1811
1812 retry: 1812 retry:
1813 ret = ipipe_prep(ipipe, flags); 1813 ret = ipipe_prep(ipipe, flags);
1814 if (ret) 1814 if (ret)
1815 return ret; 1815 return ret;
1816 1816
1817 ret = opipe_prep(opipe, flags); 1817 ret = opipe_prep(opipe, flags);
1818 if (ret) 1818 if (ret)
1819 return ret; 1819 return ret;
1820 1820
1821 /* 1821 /*
1822 * Potential ABBA deadlock, work around it by ordering lock 1822 * Potential ABBA deadlock, work around it by ordering lock
1823 * grabbing by pipe info address. Otherwise two different processes 1823 * grabbing by pipe info address. Otherwise two different processes
1824 * could deadlock (one doing tee from A -> B, the other from B -> A). 1824 * could deadlock (one doing tee from A -> B, the other from B -> A).
1825 */ 1825 */
1826 pipe_double_lock(ipipe, opipe); 1826 pipe_double_lock(ipipe, opipe);
1827 1827
1828 do { 1828 do {
1829 if (!opipe->readers) { 1829 if (!opipe->readers) {
1830 send_sig(SIGPIPE, current, 0); 1830 send_sig(SIGPIPE, current, 0);
1831 if (!ret) 1831 if (!ret)
1832 ret = -EPIPE; 1832 ret = -EPIPE;
1833 break; 1833 break;
1834 } 1834 }
1835 1835
1836 if (!ipipe->nrbufs && !ipipe->writers) 1836 if (!ipipe->nrbufs && !ipipe->writers)
1837 break; 1837 break;
1838 1838
1839 /* 1839 /*
1840 * Cannot make any progress, because either the input 1840 * Cannot make any progress, because either the input
1841 * pipe is empty or the output pipe is full. 1841 * pipe is empty or the output pipe is full.
1842 */ 1842 */
1843 if (!ipipe->nrbufs || opipe->nrbufs >= opipe->buffers) { 1843 if (!ipipe->nrbufs || opipe->nrbufs >= opipe->buffers) {
1844 /* Already processed some buffers, break */ 1844 /* Already processed some buffers, break */
1845 if (ret) 1845 if (ret)
1846 break; 1846 break;
1847 1847
1848 if (flags & SPLICE_F_NONBLOCK) { 1848 if (flags & SPLICE_F_NONBLOCK) {
1849 ret = -EAGAIN; 1849 ret = -EAGAIN;
1850 break; 1850 break;
1851 } 1851 }
1852 1852
1853 /* 1853 /*
1854 * We raced with another reader/writer and haven't 1854 * We raced with another reader/writer and haven't
1855 * managed to process any buffers. A zero return 1855 * managed to process any buffers. A zero return
1856 * value means EOF, so retry instead. 1856 * value means EOF, so retry instead.
1857 */ 1857 */
1858 pipe_unlock(ipipe); 1858 pipe_unlock(ipipe);
1859 pipe_unlock(opipe); 1859 pipe_unlock(opipe);
1860 goto retry; 1860 goto retry;
1861 } 1861 }
1862 1862
1863 ibuf = ipipe->bufs + ipipe->curbuf; 1863 ibuf = ipipe->bufs + ipipe->curbuf;
1864 nbuf = (opipe->curbuf + opipe->nrbufs) & (opipe->buffers - 1); 1864 nbuf = (opipe->curbuf + opipe->nrbufs) & (opipe->buffers - 1);
1865 obuf = opipe->bufs + nbuf; 1865 obuf = opipe->bufs + nbuf;
1866 1866
1867 if (len >= ibuf->len) { 1867 if (len >= ibuf->len) {
1868 /* 1868 /*
1869 * Simply move the whole buffer from ipipe to opipe 1869 * Simply move the whole buffer from ipipe to opipe
1870 */ 1870 */
1871 *obuf = *ibuf; 1871 *obuf = *ibuf;
1872 ibuf->ops = NULL; 1872 ibuf->ops = NULL;
1873 opipe->nrbufs++; 1873 opipe->nrbufs++;
1874 ipipe->curbuf = (ipipe->curbuf + 1) & (ipipe->buffers - 1); 1874 ipipe->curbuf = (ipipe->curbuf + 1) & (ipipe->buffers - 1);
1875 ipipe->nrbufs--; 1875 ipipe->nrbufs--;
1876 input_wakeup = true; 1876 input_wakeup = true;
1877 } else { 1877 } else {
1878 /* 1878 /*
1879 * Get a reference to this pipe buffer, 1879 * Get a reference to this pipe buffer,
1880 * so we can copy the contents over. 1880 * so we can copy the contents over.
1881 */ 1881 */
1882 ibuf->ops->get(ipipe, ibuf); 1882 ibuf->ops->get(ipipe, ibuf);
1883 *obuf = *ibuf; 1883 *obuf = *ibuf;
1884 1884
1885 /* 1885 /*
1886 * Don't inherit the gift flag, we need to 1886 * Don't inherit the gift flag, we need to
1887 * prevent multiple steals of this page. 1887 * prevent multiple steals of this page.
1888 */ 1888 */
1889 obuf->flags &= ~PIPE_BUF_FLAG_GIFT; 1889 obuf->flags &= ~PIPE_BUF_FLAG_GIFT;
1890 1890
1891 obuf->len = len; 1891 obuf->len = len;
1892 opipe->nrbufs++; 1892 opipe->nrbufs++;
1893 ibuf->offset += obuf->len; 1893 ibuf->offset += obuf->len;
1894 ibuf->len -= obuf->len; 1894 ibuf->len -= obuf->len;
1895 } 1895 }
1896 ret += obuf->len; 1896 ret += obuf->len;
1897 len -= obuf->len; 1897 len -= obuf->len;
1898 } while (len); 1898 } while (len);
1899 1899
1900 pipe_unlock(ipipe); 1900 pipe_unlock(ipipe);
1901 pipe_unlock(opipe); 1901 pipe_unlock(opipe);
1902 1902
1903 /* 1903 /*
1904 * If we put data in the output pipe, wakeup any potential readers. 1904 * If we put data in the output pipe, wakeup any potential readers.
1905 */ 1905 */
1906 if (ret > 0) 1906 if (ret > 0)
1907 wakeup_pipe_readers(opipe); 1907 wakeup_pipe_readers(opipe);
1908 1908
1909 if (input_wakeup) 1909 if (input_wakeup)
1910 wakeup_pipe_writers(ipipe); 1910 wakeup_pipe_writers(ipipe);
1911 1911
1912 return ret; 1912 return ret;
1913 } 1913 }
1914 1914
1915 /* 1915 /*
1916 * Link contents of ipipe to opipe. 1916 * Link contents of ipipe to opipe.
1917 */ 1917 */
1918 static int link_pipe(struct pipe_inode_info *ipipe, 1918 static int link_pipe(struct pipe_inode_info *ipipe,
1919 struct pipe_inode_info *opipe, 1919 struct pipe_inode_info *opipe,
1920 size_t len, unsigned int flags) 1920 size_t len, unsigned int flags)
1921 { 1921 {
1922 struct pipe_buffer *ibuf, *obuf; 1922 struct pipe_buffer *ibuf, *obuf;
1923 int ret = 0, i = 0, nbuf; 1923 int ret = 0, i = 0, nbuf;
1924 1924
1925 /* 1925 /*
1926 * Potential ABBA deadlock, work around it by ordering lock 1926 * Potential ABBA deadlock, work around it by ordering lock
1927 * grabbing by pipe info address. Otherwise two different processes 1927 * grabbing by pipe info address. Otherwise two different processes
1928 * could deadlock (one doing tee from A -> B, the other from B -> A). 1928 * could deadlock (one doing tee from A -> B, the other from B -> A).
1929 */ 1929 */
1930 pipe_double_lock(ipipe, opipe); 1930 pipe_double_lock(ipipe, opipe);
1931 1931
1932 do { 1932 do {
1933 if (!opipe->readers) { 1933 if (!opipe->readers) {
1934 send_sig(SIGPIPE, current, 0); 1934 send_sig(SIGPIPE, current, 0);
1935 if (!ret) 1935 if (!ret)
1936 ret = -EPIPE; 1936 ret = -EPIPE;
1937 break; 1937 break;
1938 } 1938 }
1939 1939
1940 /* 1940 /*
1941 * If we have iterated all input buffers or ran out of 1941 * If we have iterated all input buffers or ran out of
1942 * output room, break. 1942 * output room, break.
1943 */ 1943 */
1944 if (i >= ipipe->nrbufs || opipe->nrbufs >= opipe->buffers) 1944 if (i >= ipipe->nrbufs || opipe->nrbufs >= opipe->buffers)
1945 break; 1945 break;
1946 1946
1947 ibuf = ipipe->bufs + ((ipipe->curbuf + i) & (ipipe->buffers-1)); 1947 ibuf = ipipe->bufs + ((ipipe->curbuf + i) & (ipipe->buffers-1));
1948 nbuf = (opipe->curbuf + opipe->nrbufs) & (opipe->buffers - 1); 1948 nbuf = (opipe->curbuf + opipe->nrbufs) & (opipe->buffers - 1);
1949 1949
1950 /* 1950 /*
1951 * Get a reference to this pipe buffer, 1951 * Get a reference to this pipe buffer,
1952 * so we can copy the contents over. 1952 * so we can copy the contents over.
1953 */ 1953 */
1954 ibuf->ops->get(ipipe, ibuf); 1954 ibuf->ops->get(ipipe, ibuf);
1955 1955
1956 obuf = opipe->bufs + nbuf; 1956 obuf = opipe->bufs + nbuf;
1957 *obuf = *ibuf; 1957 *obuf = *ibuf;
1958 1958
1959 /* 1959 /*
1960 * Don't inherit the gift flag, we need to 1960 * Don't inherit the gift flag, we need to
1961 * prevent multiple steals of this page. 1961 * prevent multiple steals of this page.
1962 */ 1962 */
1963 obuf->flags &= ~PIPE_BUF_FLAG_GIFT; 1963 obuf->flags &= ~PIPE_BUF_FLAG_GIFT;
1964 1964
1965 if (obuf->len > len) 1965 if (obuf->len > len)
1966 obuf->len = len; 1966 obuf->len = len;
1967 1967
1968 opipe->nrbufs++; 1968 opipe->nrbufs++;
1969 ret += obuf->len; 1969 ret += obuf->len;
1970 len -= obuf->len; 1970 len -= obuf->len;
1971 i++; 1971 i++;
1972 } while (len); 1972 } while (len);
1973 1973
1974 /* 1974 /*
1975 * return EAGAIN if we have the potential of some data in the 1975 * return EAGAIN if we have the potential of some data in the
1976 * future, otherwise just return 0 1976 * future, otherwise just return 0
1977 */ 1977 */
1978 if (!ret && ipipe->waiting_writers && (flags & SPLICE_F_NONBLOCK)) 1978 if (!ret && ipipe->waiting_writers && (flags & SPLICE_F_NONBLOCK))
1979 ret = -EAGAIN; 1979 ret = -EAGAIN;
1980 1980
1981 pipe_unlock(ipipe); 1981 pipe_unlock(ipipe);
1982 pipe_unlock(opipe); 1982 pipe_unlock(opipe);
1983 1983
1984 /* 1984 /*
1985 * If we put data in the output pipe, wakeup any potential readers. 1985 * If we put data in the output pipe, wakeup any potential readers.
1986 */ 1986 */
1987 if (ret > 0) 1987 if (ret > 0)
1988 wakeup_pipe_readers(opipe); 1988 wakeup_pipe_readers(opipe);
1989 1989
1990 return ret; 1990 return ret;
1991 } 1991 }
1992 1992
1993 /* 1993 /*
1994 * This is a tee(1) implementation that works on pipes. It doesn't copy 1994 * This is a tee(1) implementation that works on pipes. It doesn't copy
1995 * any data, it simply references the 'in' pages on the 'out' pipe. 1995 * any data, it simply references the 'in' pages on the 'out' pipe.
1996 * The 'flags' used are the SPLICE_F_* variants, currently the only 1996 * The 'flags' used are the SPLICE_F_* variants, currently the only
1997 * applicable one is SPLICE_F_NONBLOCK. 1997 * applicable one is SPLICE_F_NONBLOCK.
1998 */ 1998 */
1999 static long do_tee(struct file *in, struct file *out, size_t len, 1999 static long do_tee(struct file *in, struct file *out, size_t len,
2000 unsigned int flags) 2000 unsigned int flags)
2001 { 2001 {
2002 struct pipe_inode_info *ipipe = get_pipe_info(in); 2002 struct pipe_inode_info *ipipe = get_pipe_info(in);
2003 struct pipe_inode_info *opipe = get_pipe_info(out); 2003 struct pipe_inode_info *opipe = get_pipe_info(out);
2004 int ret = -EINVAL; 2004 int ret = -EINVAL;
2005 2005
2006 /* 2006 /*
2007 * Duplicate the contents of ipipe to opipe without actually 2007 * Duplicate the contents of ipipe to opipe without actually
2008 * copying the data. 2008 * copying the data.
2009 */ 2009 */
2010 if (ipipe && opipe && ipipe != opipe) { 2010 if (ipipe && opipe && ipipe != opipe) {
2011 /* 2011 /*
2012 * Keep going, unless we encounter an error. The ipipe/opipe 2012 * Keep going, unless we encounter an error. The ipipe/opipe
2013 * ordering doesn't really matter. 2013 * ordering doesn't really matter.
2014 */ 2014 */
2015 ret = ipipe_prep(ipipe, flags); 2015 ret = ipipe_prep(ipipe, flags);
2016 if (!ret) { 2016 if (!ret) {
2017 ret = opipe_prep(opipe, flags); 2017 ret = opipe_prep(opipe, flags);
2018 if (!ret) 2018 if (!ret)
2019 ret = link_pipe(ipipe, opipe, len, flags); 2019 ret = link_pipe(ipipe, opipe, len, flags);
2020 } 2020 }
2021 } 2021 }
2022 2022
2023 return ret; 2023 return ret;
2024 } 2024 }
2025 2025
2026 SYSCALL_DEFINE4(tee, int, fdin, int, fdout, size_t, len, unsigned int, flags) 2026 SYSCALL_DEFINE4(tee, int, fdin, int, fdout, size_t, len, unsigned int, flags)
2027 { 2027 {
2028 struct fd in; 2028 struct fd in;
2029 int error; 2029 int error;
2030 2030
2031 if (unlikely(!len)) 2031 if (unlikely(!len))
2032 return 0; 2032 return 0;
2033 2033
2034 error = -EBADF; 2034 error = -EBADF;
2035 in = fdget(fdin); 2035 in = fdget(fdin);
2036 if (in.file) { 2036 if (in.file) {
2037 if (in.file->f_mode & FMODE_READ) { 2037 if (in.file->f_mode & FMODE_READ) {
2038 struct fd out = fdget(fdout); 2038 struct fd out = fdget(fdout);
2039 if (out.file) { 2039 if (out.file) {
2040 if (out.file->f_mode & FMODE_WRITE) 2040 if (out.file->f_mode & FMODE_WRITE)
2041 error = do_tee(in.file, out.file, 2041 error = do_tee(in.file, out.file,
2042 len, flags); 2042 len, flags);
2043 fdput(out); 2043 fdput(out);
2044 } 2044 }
2045 } 2045 }
2046 fdput(in); 2046 fdput(in);
2047 } 2047 }
2048 2048
2049 return error; 2049 return error;
2050 } 2050 }
2051 2051
include/linux/pipe_fs_i.h
Diff comments   View file @ 7bee130
1 #ifndef _LINUX_PIPE_FS_I_H 1 #ifndef _LINUX_PIPE_FS_I_H
2 #define _LINUX_PIPE_FS_I_H 2 #define _LINUX_PIPE_FS_I_H
3 3
4 #define PIPE_DEF_BUFFERS 16 4 #define PIPE_DEF_BUFFERS 16
5 5
6 #define PIPE_BUF_FLAG_LRU 0x01 /* page is on the LRU */ 6 #define PIPE_BUF_FLAG_LRU 0x01 /* page is on the LRU */
7 #define PIPE_BUF_FLAG_ATOMIC 0x02 /* was atomically mapped */ 7 #define PIPE_BUF_FLAG_ATOMIC 0x02 /* was atomically mapped */
8 #define PIPE_BUF_FLAG_GIFT 0x04 /* page is a gift */ 8 #define PIPE_BUF_FLAG_GIFT 0x04 /* page is a gift */
9 #define PIPE_BUF_FLAG_PACKET 0x08 /* read() as a packet */ 9 #define PIPE_BUF_FLAG_PACKET 0x08 /* read() as a packet */
10 10
11 /** 11 /**
12 * struct pipe_buffer - a linux kernel pipe buffer 12 * struct pipe_buffer - a linux kernel pipe buffer
13 * @page: the page containing the data for the pipe buffer 13 * @page: the page containing the data for the pipe buffer
14 * @offset: offset of data inside the @page 14 * @offset: offset of data inside the @page
15 * @len: length of data inside the @page 15 * @len: length of data inside the @page
16 * @ops: operations associated with this buffer. See @pipe_buf_operations. 16 * @ops: operations associated with this buffer. See @pipe_buf_operations.
17 * @flags: pipe buffer flags. See above. 17 * @flags: pipe buffer flags. See above.
18 * @private: private data owned by the ops. 18 * @private: private data owned by the ops.
19 **/ 19 **/
20 struct pipe_buffer { 20 struct pipe_buffer {
21 struct page *page; 21 struct page *page;
22 unsigned int offset, len; 22 unsigned int offset, len;
23 const struct pipe_buf_operations *ops; 23 const struct pipe_buf_operations *ops;
24 unsigned int flags; 24 unsigned int flags;
25 unsigned long private; 25 unsigned long private;
26 }; 26 };
27 27
28 /** 28 /**
29 * struct pipe_inode_info - a linux kernel pipe 29 * struct pipe_inode_info - a linux kernel pipe
30 * @mutex: mutex protecting the whole thing 30 * @mutex: mutex protecting the whole thing
31 * @wait: reader/writer wait point in case of empty/full pipe 31 * @wait: reader/writer wait point in case of empty/full pipe
32 * @nrbufs: the number of non-empty pipe buffers in this pipe 32 * @nrbufs: the number of non-empty pipe buffers in this pipe
33 * @buffers: total number of buffers (should be a power of 2) 33 * @buffers: total number of buffers (should be a power of 2)
34 * @curbuf: the current pipe buffer entry 34 * @curbuf: the current pipe buffer entry
35 * @tmp_page: cached released page 35 * @tmp_page: cached released page
36 * @readers: number of current readers of this pipe 36 * @readers: number of current readers of this pipe
37 * @writers: number of current writers of this pipe 37 * @writers: number of current writers of this pipe
38 * @files: number of struct file refering this pipe (protected by ->i_lock) 38 * @files: number of struct file refering this pipe (protected by ->i_lock)
39 * @waiting_writers: number of writers blocked waiting for room 39 * @waiting_writers: number of writers blocked waiting for room
40 * @r_counter: reader counter 40 * @r_counter: reader counter
41 * @w_counter: writer counter 41 * @w_counter: writer counter
42 * @fasync_readers: reader side fasync 42 * @fasync_readers: reader side fasync
43 * @fasync_writers: writer side fasync 43 * @fasync_writers: writer side fasync
44 * @bufs: the circular array of pipe buffers 44 * @bufs: the circular array of pipe buffers
45 **/ 45 **/
46 struct pipe_inode_info { 46 struct pipe_inode_info {
47 struct mutex mutex; 47 struct mutex mutex;
48 wait_queue_head_t wait; 48 wait_queue_head_t wait;
49 unsigned int nrbufs, curbuf, buffers; 49 unsigned int nrbufs, curbuf, buffers;
50 unsigned int readers; 50 unsigned int readers;
51 unsigned int writers; 51 unsigned int writers;
52 unsigned int files; 52 unsigned int files;
53 unsigned int waiting_writers; 53 unsigned int waiting_writers;
54 unsigned int r_counter; 54 unsigned int r_counter;
55 unsigned int w_counter; 55 unsigned int w_counter;
56 struct page *tmp_page; 56 struct page *tmp_page;
57 struct fasync_struct *fasync_readers; 57 struct fasync_struct *fasync_readers;
58 struct fasync_struct *fasync_writers; 58 struct fasync_struct *fasync_writers;
59 struct pipe_buffer *bufs; 59 struct pipe_buffer *bufs;
60 }; 60 };
61 61
62 /* 62 /*
63 * Note on the nesting of these functions: 63 * Note on the nesting of these functions:
64 * 64 *
65 * ->confirm() 65 * ->confirm()
66 * ->steal() 66 * ->steal()
67 * ... 67 * ...
68 * ->map() 68 * ->map()
69 * ... 69 * ...
70 * ->unmap() 70 * ->unmap()
71 * 71 *
72 * That is, ->map() must be called on a confirmed buffer, 72 * That is, ->map() must be called on a confirmed buffer,
73 * same goes for ->steal(). See below for the meaning of each 73 * same goes for ->steal(). See below for the meaning of each
74 * operation. Also see kerneldoc in fs/pipe.c for the pipe 74 * operation. Also see kerneldoc in fs/pipe.c for the pipe
75 * and generic variants of these hooks. 75 * and generic variants of these hooks.
76 */ 76 */
77 struct pipe_buf_operations { 77 struct pipe_buf_operations {
78 /* 78 /*
79 * This is set to 1, if the generic pipe read/write may coalesce 79 * This is set to 1, if the generic pipe read/write may coalesce
80 * data into an existing buffer. If this is set to 0, a new pipe 80 * data into an existing buffer. If this is set to 0, a new pipe
81 * page segment is always used for new data. 81 * page segment is always used for new data.
82 */ 82 */
83 int can_merge; 83 int can_merge;
84 84
85 /* 85 /*
86 * ->map() returns a virtual address mapping of the pipe buffer. 86 * ->map() returns a virtual address mapping of the pipe buffer.
87 * The last integer flag reflects whether this should be an atomic 87 * The last integer flag reflects whether this should be an atomic
88 * mapping or not. The atomic map is faster, however you can't take 88 * mapping or not. The atomic map is faster, however you can't take
89 * page faults before calling ->unmap() again. So if you need to eg 89 * page faults before calling ->unmap() again. So if you need to eg
90 * access user data through copy_to/from_user(), then you must get 90 * access user data through copy_to/from_user(), then you must get
91 * a non-atomic map. ->map() uses the kmap_atomic slot for 91 * a non-atomic map. ->map() uses the kmap_atomic slot for
92 * atomic maps, you have to be careful if mapping another page as 92 * atomic maps, you have to be careful if mapping another page as
93 * source or destination for a copy. 93 * source or destination for a copy.
94 */ 94 */
95 void * (*map)(struct pipe_inode_info *, struct pipe_buffer *, int); 95 void * (*map)(struct pipe_inode_info *, struct pipe_buffer *, int);
96 96
97 /* 97 /*
98 * Undoes ->map(), finishes the virtual mapping of the pipe buffer. 98 * Undoes ->map(), finishes the virtual mapping of the pipe buffer.
99 */ 99 */
100 void (*unmap)(struct pipe_inode_info *, struct pipe_buffer *, void *); 100 void (*unmap)(struct pipe_inode_info *, struct pipe_buffer *, void *);
101 101
102 /* 102 /*
103 * ->confirm() verifies that the data in the pipe buffer is there 103 * ->confirm() verifies that the data in the pipe buffer is there
104 * and that the contents are good. If the pages in the pipe belong 104 * and that the contents are good. If the pages in the pipe belong
105 * to a file system, we may need to wait for IO completion in this 105 * to a file system, we may need to wait for IO completion in this
106 * hook. Returns 0 for good, or a negative error value in case of 106 * hook. Returns 0 for good, or a negative error value in case of
107 * error. 107 * error.
108 */ 108 */
109 int (*confirm)(struct pipe_inode_info *, struct pipe_buffer *); 109 int (*confirm)(struct pipe_inode_info *, struct pipe_buffer *);
110 110
111 /* 111 /*
112 * When the contents of this pipe buffer has been completely 112 * When the contents of this pipe buffer has been completely
113 * consumed by a reader, ->release() is called. 113 * consumed by a reader, ->release() is called.
114 */ 114 */
115 void (*release)(struct pipe_inode_info *, struct pipe_buffer *); 115 void (*release)(struct pipe_inode_info *, struct pipe_buffer *);
116 116
117 /* 117 /*
118 * Attempt to take ownership of the pipe buffer and its contents. 118 * Attempt to take ownership of the pipe buffer and its contents.
119 * ->steal() returns 0 for success, in which case the contents 119 * ->steal() returns 0 for success, in which case the contents
120 * of the pipe (the buf->page) is locked and now completely owned 120 * of the pipe (the buf->page) is locked and now completely owned
121 * by the caller. The page may then be transferred to a different 121 * by the caller. The page may then be transferred to a different
122 * mapping, the most often used case is insertion into different 122 * mapping, the most often used case is insertion into different
123 * file address space cache. 123 * file address space cache.
124 */ 124 */
125 int (*steal)(struct pipe_inode_info *, struct pipe_buffer *); 125 int (*steal)(struct pipe_inode_info *, struct pipe_buffer *);
126 126
127 /* 127 /*
128 * Get a reference to the pipe buffer. 128 * Get a reference to the pipe buffer.
129 */ 129 */
130 void (*get)(struct pipe_inode_info *, struct pipe_buffer *); 130 void (*get)(struct pipe_inode_info *, struct pipe_buffer *);
131 }; 131 };
132 132
133 /* Differs from PIPE_BUF in that PIPE_SIZE is the length of the actual 133 /* Differs from PIPE_BUF in that PIPE_SIZE is the length of the actual
134 memory allocation, whereas PIPE_BUF makes atomicity guarantees. */ 134 memory allocation, whereas PIPE_BUF makes atomicity guarantees. */
135 #define PIPE_SIZE PAGE_SIZE 135 #define PIPE_SIZE PAGE_SIZE
136 136
137 /* Pipe lock and unlock operations */ 137 /* Pipe lock and unlock operations */
138 void pipe_lock(struct pipe_inode_info *); 138 void pipe_lock(struct pipe_inode_info *);
139 void pipe_unlock(struct pipe_inode_info *); 139 void pipe_unlock(struct pipe_inode_info *);
140 void pipe_double_lock(struct pipe_inode_info *, struct pipe_inode_info *); 140 void pipe_double_lock(struct pipe_inode_info *, struct pipe_inode_info *);
141 141
142 extern unsigned int pipe_max_size, pipe_min_size; 142 extern unsigned int pipe_max_size, pipe_min_size;
143 int pipe_proc_fn(struct ctl_table *, int, void __user *, size_t *, loff_t *); 143 int pipe_proc_fn(struct ctl_table *, int, void __user *, size_t *, loff_t *);
144 144
145 145
146 /* Drop the inode semaphore and wait for a pipe event, atomically */ 146 /* Drop the inode semaphore and wait for a pipe event, atomically */
147 void pipe_wait(struct pipe_inode_info *pipe); 147 void pipe_wait(struct pipe_inode_info *pipe);
148 148
149 struct pipe_inode_info * alloc_pipe_info(struct inode * inode); 149 struct pipe_inode_info *alloc_pipe_info(void);
150 void free_pipe_info(struct inode * inode); 150 void free_pipe_info(struct inode * inode);
151 void __free_pipe_info(struct pipe_inode_info *); 151 void __free_pipe_info(struct pipe_inode_info *);
152 152
153 /* Generic pipe buffer ops functions */ 153 /* Generic pipe buffer ops functions */
154 void *generic_pipe_buf_map(struct pipe_inode_info *, struct pipe_buffer *, int); 154 void *generic_pipe_buf_map(struct pipe_inode_info *, struct pipe_buffer *, int);
155 void generic_pipe_buf_unmap(struct pipe_inode_info *, struct pipe_buffer *, void *); 155 void generic_pipe_buf_unmap(struct pipe_inode_info *, struct pipe_buffer *, void *);
156 void generic_pipe_buf_get(struct pipe_inode_info *, struct pipe_buffer *); 156 void generic_pipe_buf_get(struct pipe_inode_info *, struct pipe_buffer *);
157 int generic_pipe_buf_confirm(struct pipe_inode_info *, struct pipe_buffer *); 157 int generic_pipe_buf_confirm(struct pipe_inode_info *, struct pipe_buffer *);
158 int generic_pipe_buf_steal(struct pipe_inode_info *, struct pipe_buffer *); 158 int generic_pipe_buf_steal(struct pipe_inode_info *, struct pipe_buffer *);
159 void generic_pipe_buf_release(struct pipe_inode_info *, struct pipe_buffer *); 159 void generic_pipe_buf_release(struct pipe_inode_info *, struct pipe_buffer *);
160 160
161 /* for F_SETPIPE_SZ and F_GETPIPE_SZ */ 161 /* for F_SETPIPE_SZ and F_GETPIPE_SZ */
162 long pipe_fcntl(struct file *, unsigned int, unsigned long arg); 162 long pipe_fcntl(struct file *, unsigned int, unsigned long arg);
163 struct pipe_inode_info *get_pipe_info(struct file *file); 163 struct pipe_inode_info *get_pipe_info(struct file *file);
164 164
165 int create_pipe_files(struct file **, int); 165 int create_pipe_files(struct file **, int);
166 166
167 #endif 167 #endif
168 168