Blame view
fs/splice.c
41.1 KB
457c89965
|
1 |
// SPDX-License-Identifier: GPL-2.0-only |
5274f052e
|
2 3 4 5 6 7 8 9 10 11 12 |
/* * "splice": joining two ropes together by interweaving their strands. * * This is the "extended pipe" functionality, where a pipe is used as * an arbitrary in-memory buffer. Think of a pipe as a small kernel * buffer that you can use to transfer data from one end to the other. * * The traditional unix read/write is extended with a "splice()" operation * that transfers data buffers to or from a pipe buffer. * * Named by Larry McVoy, original implementation from Linus, extended by |
c2058e061
|
13 14 |
* Jens to support splicing to files, network, direct splicing, etc and * fixing lots of bugs. |
5274f052e
|
15 |
* |
0fe234795
|
16 |
* Copyright (C) 2005-2006 Jens Axboe <axboe@kernel.dk> |
c2058e061
|
17 18 |
* Copyright (C) 2005-2006 Linus Torvalds <torvalds@osdl.org> * Copyright (C) 2006 Ingo Molnar <mingo@elte.hu> |
5274f052e
|
19 20 |
* */ |
be297968d
|
21 |
#include <linux/bvec.h> |
5274f052e
|
22 23 24 |
#include <linux/fs.h> #include <linux/file.h> #include <linux/pagemap.h> |
d6b29d7ce
|
25 |
#include <linux/splice.h> |
08e552c69
|
26 |
#include <linux/memcontrol.h> |
5274f052e
|
27 |
#include <linux/mm_inline.h> |
5abc97aa2
|
28 |
#include <linux/swap.h> |
4f6f0bd2f
|
29 |
#include <linux/writeback.h> |
630d9c472
|
30 |
#include <linux/export.h> |
4f6f0bd2f
|
31 |
#include <linux/syscalls.h> |
912d35f86
|
32 |
#include <linux/uio.h> |
29ce20586
|
33 |
#include <linux/security.h> |
5a0e3ad6a
|
34 |
#include <linux/gfp.h> |
35f9c09fe
|
35 |
#include <linux/socket.h> |
76b021d05
|
36 |
#include <linux/compat.h> |
174cd4b1e
|
37 |
#include <linux/sched/signal.h> |
06ae43f34
|
38 |
#include "internal.h" |
5274f052e
|
39 |
|
83f9135bd
|
40 41 42 43 44 45 |
/* * Attempt to steal a page from a pipe buffer. This should perhaps go into * a vm helper function, it's already simplified quite a bit by the * addition of remove_mapping(). If success is returned, the caller may * attempt to reuse this page for another destination. */ |
76ad4d111
|
46 |
static int page_cache_pipe_buf_steal(struct pipe_inode_info *pipe, |
5abc97aa2
|
47 48 49 |
struct pipe_buffer *buf) { struct page *page = buf->page; |
9e94cd4fd
|
50 |
struct address_space *mapping; |
5abc97aa2
|
51 |
|
9e0267c26
|
52 |
lock_page(page); |
9e94cd4fd
|
53 54 55 |
mapping = page_mapping(page); if (mapping) { WARN_ON(!PageUptodate(page)); |
5abc97aa2
|
56 |
|
9e94cd4fd
|
57 58 59 60 61 62 63 64 65 |
/* * At least for ext2 with nobh option, we need to wait on * writeback completing on this page, since we'll remove it * from the pagecache. Otherwise truncate wont wait on the * page, allowing the disk blocks to be reused by someone else * before we actually wrote our data to them. fs corruption * ensues. */ wait_on_page_writeback(page); |
ad8d6f0a7
|
66 |
|
266cf658e
|
67 68 |
if (page_has_private(page) && !try_to_release_page(page, GFP_KERNEL)) |
ca39d651d
|
69 |
goto out_unlock; |
4f6f0bd2f
|
70 |
|
9e94cd4fd
|
71 72 73 74 75 76 77 78 |
/* * If we succeeded in removing the mapping, set LRU flag * and return good. */ if (remove_mapping(mapping, page)) { buf->flags |= PIPE_BUF_FLAG_LRU; return 0; } |
9e0267c26
|
79 |
} |
5abc97aa2
|
80 |
|
9e94cd4fd
|
81 82 83 84 |
/* * Raced with truncate or failed to remove page from current * address space, unlock and return failure. */ |
ca39d651d
|
85 |
out_unlock: |
9e94cd4fd
|
86 87 |
unlock_page(page); return 1; |
5abc97aa2
|
88 |
} |
76ad4d111
|
89 |
static void page_cache_pipe_buf_release(struct pipe_inode_info *pipe, |
5274f052e
|
90 91 |
struct pipe_buffer *buf) { |
09cbfeaf1
|
92 |
put_page(buf->page); |
1432873af
|
93 |
buf->flags &= ~PIPE_BUF_FLAG_LRU; |
5274f052e
|
94 |
} |
0845718da
|
95 96 97 98 |
/* * Check whether the contents of buf is OK to access. Since the content * is a page cache page, IO may be in flight. */ |
cac36bb06
|
99 100 |
static int page_cache_pipe_buf_confirm(struct pipe_inode_info *pipe, struct pipe_buffer *buf) |
5274f052e
|
101 102 |
{ struct page *page = buf->page; |
49d0b21be
|
103 |
int err; |
5274f052e
|
104 105 |
if (!PageUptodate(page)) { |
49d0b21be
|
106 107 108 109 |
lock_page(page); /* * Page got truncated/unhashed. This will cause a 0-byte |
73d62d83e
|
110 |
* splice, if this is the first page. |
49d0b21be
|
111 112 113 114 115 |
*/ if (!page->mapping) { err = -ENODATA; goto error; } |
5274f052e
|
116 |
|
49d0b21be
|
117 |
/* |
73d62d83e
|
118 |
* Uh oh, read-error from disk. |
49d0b21be
|
119 120 121 122 123 124 125 |
*/ if (!PageUptodate(page)) { err = -EIO; goto error; } /* |
f84d75199
|
126 |
* Page is ok afterall, we are done. |
49d0b21be
|
127 |
*/ |
5274f052e
|
128 |
unlock_page(page); |
5274f052e
|
129 |
} |
f84d75199
|
130 |
return 0; |
49d0b21be
|
131 132 |
error: unlock_page(page); |
f84d75199
|
133 |
return err; |
70524490e
|
134 |
} |
708e3508c
|
135 |
const struct pipe_buf_operations page_cache_pipe_buf_ops = { |
cac36bb06
|
136 |
.confirm = page_cache_pipe_buf_confirm, |
5274f052e
|
137 |
.release = page_cache_pipe_buf_release, |
5abc97aa2
|
138 |
.steal = page_cache_pipe_buf_steal, |
f84d75199
|
139 |
.get = generic_pipe_buf_get, |
5274f052e
|
140 |
}; |
912d35f86
|
141 142 143 |
static int user_page_pipe_buf_steal(struct pipe_inode_info *pipe, struct pipe_buffer *buf) { |
7afa6fd03
|
144 145 |
if (!(buf->flags & PIPE_BUF_FLAG_GIFT)) return 1; |
1432873af
|
146 |
buf->flags |= PIPE_BUF_FLAG_LRU; |
330ab7161
|
147 |
return generic_pipe_buf_steal(pipe, buf); |
912d35f86
|
148 |
} |
d4c3cca94
|
149 |
static const struct pipe_buf_operations user_page_pipe_buf_ops = { |
cac36bb06
|
150 |
.confirm = generic_pipe_buf_confirm, |
912d35f86
|
151 152 |
.release = page_cache_pipe_buf_release, .steal = user_page_pipe_buf_steal, |
f84d75199
|
153 |
.get = generic_pipe_buf_get, |
912d35f86
|
154 |
}; |
825cdcb1a
|
155 156 157 |
static void wakeup_pipe_readers(struct pipe_inode_info *pipe) { smp_mb(); |
0ddad21d3
|
158 159 |
if (waitqueue_active(&pipe->rd_wait)) wake_up_interruptible(&pipe->rd_wait); |
825cdcb1a
|
160 161 |
kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN); } |
932cc6d4f
|
162 163 164 165 166 167 |
/** * splice_to_pipe - fill passed data into a pipe * @pipe: pipe to fill * @spd: data to fill * * Description: |
79685b8de
|
168 |
* @spd contains a map of pages and len/offset tuples, along with |
932cc6d4f
|
169 170 171 |
* the struct pipe_buf_operations associated with these pages. This * function will link that data to the pipe. * |
83f9135bd
|
172 |
*/ |
d6b29d7ce
|
173 174 |
ssize_t splice_to_pipe(struct pipe_inode_info *pipe, struct splice_pipe_desc *spd) |
5274f052e
|
175 |
{ |
00de00bda
|
176 |
unsigned int spd_pages = spd->nr_pages; |
8cefc107c
|
177 178 179 |
unsigned int tail = pipe->tail; unsigned int head = pipe->head; unsigned int mask = pipe->ring_size - 1; |
8924feff6
|
180 |
int ret = 0, page_nr = 0; |
5274f052e
|
181 |
|
d6785d915
|
182 183 |
if (!spd_pages) return 0; |
8924feff6
|
184 185 186 187 188 |
if (unlikely(!pipe->readers)) { send_sig(SIGPIPE, current, 0); ret = -EPIPE; goto out; } |
5274f052e
|
189 |
|
6718b6f85
|
190 |
while (!pipe_full(head, tail, pipe->max_usage)) { |
8cefc107c
|
191 |
struct pipe_buffer *buf = &pipe->bufs[head & mask]; |
5274f052e
|
192 |
|
8924feff6
|
193 194 195 196 197 |
buf->page = spd->pages[page_nr]; buf->offset = spd->partial[page_nr].offset; buf->len = spd->partial[page_nr].len; buf->private = spd->partial[page_nr].private; buf->ops = spd->ops; |
5a81e6a17
|
198 |
buf->flags = 0; |
5274f052e
|
199 |
|
8cefc107c
|
200 201 |
head++; pipe->head = head; |
8924feff6
|
202 203 |
page_nr++; ret += buf->len; |
29e350944
|
204 |
|
8924feff6
|
205 |
if (!--spd->nr_pages) |
5274f052e
|
206 |
break; |
5274f052e
|
207 |
} |
8924feff6
|
208 209 |
if (!ret) ret = -EAGAIN; |
5274f052e
|
210 |
|
8924feff6
|
211 |
out: |
00de00bda
|
212 |
while (page_nr < spd_pages) |
bbdfc2f70
|
213 |
spd->spd_release(spd, page_nr++); |
5274f052e
|
214 215 216 |
return ret; } |
2b514574f
|
217 |
EXPORT_SYMBOL_GPL(splice_to_pipe); |
5274f052e
|
218 |
|
79fddc4ef
|
219 220 |
ssize_t add_to_pipe(struct pipe_inode_info *pipe, struct pipe_buffer *buf) { |
8cefc107c
|
221 222 223 |
unsigned int head = pipe->head; unsigned int tail = pipe->tail; unsigned int mask = pipe->ring_size - 1; |
79fddc4ef
|
224 225 226 227 228 |
int ret; if (unlikely(!pipe->readers)) { send_sig(SIGPIPE, current, 0); ret = -EPIPE; |
6718b6f85
|
229 |
} else if (pipe_full(head, tail, pipe->max_usage)) { |
79fddc4ef
|
230 231 |
ret = -EAGAIN; } else { |
8cefc107c
|
232 233 |
pipe->bufs[head & mask] = *buf; pipe->head = head + 1; |
79fddc4ef
|
234 235 |
return buf->len; } |
a779638cf
|
236 |
pipe_buf_release(pipe, buf); |
79fddc4ef
|
237 238 239 |
return ret; } EXPORT_SYMBOL(add_to_pipe); |
35f3d14db
|
240 241 242 243 |
/* * Check if we need to grow the arrays holding pages and partial page * descriptions. */ |
047fe3605
|
244 |
int splice_grow_spd(const struct pipe_inode_info *pipe, struct splice_pipe_desc *spd) |
35f3d14db
|
245 |
{ |
6718b6f85
|
246 |
unsigned int max_usage = READ_ONCE(pipe->max_usage); |
047fe3605
|
247 |
|
8cefc107c
|
248 249 |
spd->nr_pages_max = max_usage; if (max_usage <= PIPE_DEF_BUFFERS) |
35f3d14db
|
250 |
return 0; |
8cefc107c
|
251 252 |
spd->pages = kmalloc_array(max_usage, sizeof(struct page *), GFP_KERNEL); spd->partial = kmalloc_array(max_usage, sizeof(struct partial_page), |
6da2ec560
|
253 |
GFP_KERNEL); |
35f3d14db
|
254 255 256 257 258 259 260 261 |
if (spd->pages && spd->partial) return 0; kfree(spd->pages); kfree(spd->partial); return -ENOMEM; } |
047fe3605
|
262 |
void splice_shrink_spd(struct splice_pipe_desc *spd) |
35f3d14db
|
263 |
{ |
047fe3605
|
264 |
if (spd->nr_pages_max <= PIPE_DEF_BUFFERS) |
35f3d14db
|
265 266 267 268 269 |
return; kfree(spd->pages); kfree(spd->partial); } |
83f9135bd
|
270 271 272 |
/** * generic_file_splice_read - splice data from file to a pipe * @in: file to splice from |
932cc6d4f
|
273 |
* @ppos: position in @in |
83f9135bd
|
274 275 276 277 |
* @pipe: pipe to splice to * @len: number of bytes to splice * @flags: splice modifier flags * |
932cc6d4f
|
278 279 |
* Description: * Will read pages from given file and fill them into a pipe. Can be |
82c156f85
|
280 |
* used as long as it has more or less sane ->read_iter(). |
932cc6d4f
|
281 |
* |
83f9135bd
|
282 |
*/ |
cbb7e577e
|
283 284 285 |
ssize_t generic_file_splice_read(struct file *in, loff_t *ppos, struct pipe_inode_info *pipe, size_t len, unsigned int flags) |
5274f052e
|
286 |
{ |
82c156f85
|
287 288 |
struct iov_iter to; struct kiocb kiocb; |
8cefc107c
|
289 290 |
unsigned int i_head; int ret; |
be64f884b
|
291 |
|
aa563d7bc
|
292 |
iov_iter_pipe(&to, READ, pipe, len); |
8cefc107c
|
293 |
i_head = to.head; |
82c156f85
|
294 295 |
init_sync_kiocb(&kiocb, in); kiocb.ki_pos = *ppos; |
bb7462b6f
|
296 |
ret = call_read_iter(in, &kiocb, &to); |
723590ed5
|
297 |
if (ret > 0) { |
82c156f85
|
298 |
*ppos = kiocb.ki_pos; |
723590ed5
|
299 |
file_accessed(in); |
82c156f85
|
300 |
} else if (ret < 0) { |
8cefc107c
|
301 |
to.head = i_head; |
c3a690240
|
302 303 |
to.iov_offset = 0; iov_iter_advance(&to, 0); /* to free what was emitted */ |
82c156f85
|
304 305 306 307 308 309 |
/* * callers of ->splice_read() expect -EAGAIN on * "can't put anything in there", rather than -EFAULT. */ if (ret == -EFAULT) ret = -EAGAIN; |
723590ed5
|
310 |
} |
5274f052e
|
311 312 313 |
return ret; } |
059a8f373
|
314 |
EXPORT_SYMBOL(generic_file_splice_read); |
241699cd7
|
315 |
const struct pipe_buf_operations default_pipe_buf_ops = { |
6818173bd
|
316 317 318 319 320 |
.confirm = generic_pipe_buf_confirm, .release = generic_pipe_buf_release, .steal = generic_pipe_buf_steal, .get = generic_pipe_buf_get, }; |
b98722265
|
321 322 |
int generic_pipe_buf_nosteal(struct pipe_inode_info *pipe, struct pipe_buffer *buf) |
28a625cbc
|
323 324 325 326 327 328 |
{ return 1; } /* Pipe buffer operations for a socket and similar. */ const struct pipe_buf_operations nosteal_pipe_buf_ops = { |
28a625cbc
|
329 330 331 332 333 334 |
.confirm = generic_pipe_buf_confirm, .release = generic_pipe_buf_release, .steal = generic_pipe_buf_nosteal, .get = generic_pipe_buf_get, }; EXPORT_SYMBOL(nosteal_pipe_buf_ops); |
523ac9afc
|
335 |
static ssize_t kernel_readv(struct file *file, const struct kvec *vec, |
6818173bd
|
336 337 338 339 340 341 342 |
unsigned long vlen, loff_t offset) { mm_segment_t old_fs; loff_t pos = offset; ssize_t res; old_fs = get_fs(); |
736706bee
|
343 |
set_fs(KERNEL_DS); |
6818173bd
|
344 |
/* The cast to a user pointer is valid due to the set_fs() */ |
793b80ef1
|
345 |
res = vfs_readv(file, (const struct iovec __user *)vec, vlen, &pos, 0); |
6818173bd
|
346 347 348 349 |
set_fs(old_fs); return res; } |
82c156f85
|
350 |
static ssize_t default_file_splice_read(struct file *in, loff_t *ppos, |
6818173bd
|
351 352 353 |
struct pipe_inode_info *pipe, size_t len, unsigned int flags) { |
523ac9afc
|
354 355 356 |
struct kvec *vec, __vec[PIPE_DEF_BUFFERS]; struct iov_iter to; struct page **pages; |
6818173bd
|
357 |
unsigned int nr_pages; |
8cefc107c
|
358 |
unsigned int mask; |
13c0f52be
|
359 |
size_t offset, base, copied = 0; |
6818173bd
|
360 |
ssize_t res; |
6818173bd
|
361 |
int i; |
35f3d14db
|
362 |
|
6718b6f85
|
363 |
if (pipe_full(pipe->head, pipe->tail, pipe->max_usage)) |
523ac9afc
|
364 |
return -EAGAIN; |
35f3d14db
|
365 |
|
523ac9afc
|
366 367 368 369 |
/* * Try to keep page boundaries matching to source pagecache ones - * it probably won't be much help, but... */ |
09cbfeaf1
|
370 |
offset = *ppos & ~PAGE_MASK; |
6818173bd
|
371 |
|
aa563d7bc
|
372 |
iov_iter_pipe(&to, READ, pipe, len + offset); |
6818173bd
|
373 |
|
13c0f52be
|
374 |
res = iov_iter_get_pages_alloc(&to, &pages, len + offset, &base); |
523ac9afc
|
375 376 |
if (res <= 0) return -ENOMEM; |
6818173bd
|
377 |
|
13c0f52be
|
378 |
nr_pages = DIV_ROUND_UP(res + base, PAGE_SIZE); |
6818173bd
|
379 |
|
523ac9afc
|
380 381 |
vec = __vec; if (nr_pages > PIPE_DEF_BUFFERS) { |
6da2ec560
|
382 |
vec = kmalloc_array(nr_pages, sizeof(struct kvec), GFP_KERNEL); |
523ac9afc
|
383 384 385 386 |
if (unlikely(!vec)) { res = -ENOMEM; goto out; } |
77f6bf57b
|
387 |
} |
6818173bd
|
388 |
|
8cefc107c
|
389 390 391 |
mask = pipe->ring_size - 1; pipe->bufs[to.head & mask].offset = offset; pipe->bufs[to.head & mask].len -= offset; |
523ac9afc
|
392 393 394 395 396 397 398 |
for (i = 0; i < nr_pages; i++) { size_t this_len = min_t(size_t, len, PAGE_SIZE - offset); vec[i].iov_base = page_address(pages[i]) + offset; vec[i].iov_len = this_len; len -= this_len; offset = 0; |
6818173bd
|
399 |
} |
6818173bd
|
400 |
|
523ac9afc
|
401 402 403 |
res = kernel_readv(in, vec, nr_pages, *ppos); if (res > 0) { copied = res; |
6818173bd
|
404 |
*ppos += res; |
523ac9afc
|
405 |
} |
6818173bd
|
406 |
|
35f3d14db
|
407 408 |
if (vec != __vec) kfree(vec); |
523ac9afc
|
409 410 411 412 413 |
out: for (i = 0; i < nr_pages; i++) put_page(pages[i]); kvfree(pages); iov_iter_advance(&to, copied); /* truncates and discards */ |
6818173bd
|
414 |
return res; |
6818173bd
|
415 |
} |
6818173bd
|
416 |
|
5274f052e
|
417 |
/* |
4f6f0bd2f
|
418 |
* Send 'sd->len' bytes to socket from 'sd->file' at position 'sd->pos' |
016b661e2
|
419 |
* using sendpage(). Return the number of bytes sent. |
5274f052e
|
420 |
*/ |
76ad4d111
|
421 |
static int pipe_to_sendpage(struct pipe_inode_info *pipe, |
5274f052e
|
422 423 |
struct pipe_buffer *buf, struct splice_desc *sd) { |
6a14b90bb
|
424 |
struct file *file = sd->u.file; |
5274f052e
|
425 |
loff_t pos = sd->pos; |
a8adbe378
|
426 |
int more; |
5274f052e
|
427 |
|
72c2d5319
|
428 |
if (!likely(file->f_op->sendpage)) |
a8adbe378
|
429 |
return -EINVAL; |
35f9c09fe
|
430 |
more = (sd->flags & SPLICE_F_MORE) ? MSG_MORE : 0; |
ae62ca7b0
|
431 |
|
8cefc107c
|
432 433 |
if (sd->len < sd->total_len && pipe_occupancy(pipe->head, pipe->tail) > 1) |
35f9c09fe
|
434 |
more |= MSG_SENDPAGE_NOTLAST; |
ae62ca7b0
|
435 |
|
a8adbe378
|
436 437 |
return file->f_op->sendpage(file, buf->page, buf->offset, sd->len, &pos, more); |
5274f052e
|
438 |
} |
b3c2d2ddd
|
439 440 441 |
static void wakeup_pipe_writers(struct pipe_inode_info *pipe) { smp_mb(); |
0ddad21d3
|
442 443 |
if (waitqueue_active(&pipe->wr_wait)) wake_up_interruptible(&pipe->wr_wait); |
b3c2d2ddd
|
444 445 |
kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT); } |
932cc6d4f
|
446 |
/** |
b3c2d2ddd
|
447 |
* splice_from_pipe_feed - feed available data from a pipe to a file |
932cc6d4f
|
448 449 450 451 452 |
* @pipe: pipe to splice from * @sd: information to @actor * @actor: handler that splices the data * * Description: |
b3c2d2ddd
|
453 454 455 456 457 458 459 |
* This function loops over the pipe and calls @actor to do the * actual moving of a single struct pipe_buffer to the desired * destination. It returns when there's no more buffers left in * the pipe or if the requested number of bytes (@sd->total_len) * have been copied. It returns a positive number (one) if the * pipe needs to be filled with more data, zero if the required * number of bytes have been copied and -errno on error. |
932cc6d4f
|
460 |
* |
b3c2d2ddd
|
461 462 463 464 |
* This, together with splice_from_pipe_{begin,end,next}, may be * used to implement the functionality of __splice_from_pipe() when * locking is required around copying the pipe buffers to the * destination. |
83f9135bd
|
465 |
*/ |
96f9bc8fb
|
466 |
static int splice_from_pipe_feed(struct pipe_inode_info *pipe, struct splice_desc *sd, |
b3c2d2ddd
|
467 |
splice_actor *actor) |
5274f052e
|
468 |
{ |
8cefc107c
|
469 470 471 |
unsigned int head = pipe->head; unsigned int tail = pipe->tail; unsigned int mask = pipe->ring_size - 1; |
b3c2d2ddd
|
472 |
int ret; |
5274f052e
|
473 |
|
ec057595c
|
474 |
while (!pipe_empty(head, tail)) { |
8cefc107c
|
475 |
struct pipe_buffer *buf = &pipe->bufs[tail & mask]; |
5274f052e
|
476 |
|
b3c2d2ddd
|
477 478 479 |
sd->len = buf->len; if (sd->len > sd->total_len) sd->len = sd->total_len; |
5274f052e
|
480 |
|
fba597db4
|
481 |
ret = pipe_buf_confirm(pipe, buf); |
a8adbe378
|
482 |
if (unlikely(ret)) { |
b3c2d2ddd
|
483 484 485 486 |
if (ret == -ENODATA) ret = 0; return ret; } |
a8adbe378
|
487 488 489 490 |
ret = actor(pipe, buf, sd); if (ret <= 0) return ret; |
b3c2d2ddd
|
491 492 493 494 495 496 497 498 499 |
buf->offset += ret; buf->len -= ret; sd->num_spliced += ret; sd->len -= ret; sd->pos += ret; sd->total_len -= ret; if (!buf->len) { |
a779638cf
|
500 |
pipe_buf_release(pipe, buf); |
8cefc107c
|
501 502 |
tail++; pipe->tail = tail; |
6447a3cf1
|
503 |
if (pipe->files) |
b3c2d2ddd
|
504 505 |
sd->need_wakeup = true; } |
5274f052e
|
506 |
|
b3c2d2ddd
|
507 508 509 |
if (!sd->total_len) return 0; } |
5274f052e
|
510 |
|
b3c2d2ddd
|
511 512 |
return 1; } |
5274f052e
|
513 |
|
b3c2d2ddd
|
514 515 516 517 518 519 520 521 522 523 |
/** * splice_from_pipe_next - wait for some data to splice from * @pipe: pipe to splice from * @sd: information about the splice operation * * Description: * This function will wait for some data and return a positive * value (one) if pipe buffers are available. It will return zero * or -errno if no more data needs to be spliced. */ |
96f9bc8fb
|
524 |
static int splice_from_pipe_next(struct pipe_inode_info *pipe, struct splice_desc *sd) |
b3c2d2ddd
|
525 |
{ |
c725bfce7
|
526 527 528 529 530 531 |
/* * Check for signal early to make process killable when there are * always buffers available */ if (signal_pending(current)) return -ERESTARTSYS; |
8cefc107c
|
532 |
while (pipe_empty(pipe->head, pipe->tail)) { |
b3c2d2ddd
|
533 534 |
if (!pipe->writers) return 0; |
016b661e2
|
535 |
|
a28c8b9db
|
536 |
if (sd->num_spliced) |
b3c2d2ddd
|
537 |
return 0; |
73d62d83e
|
538 |
|
b3c2d2ddd
|
539 540 |
if (sd->flags & SPLICE_F_NONBLOCK) return -EAGAIN; |
5274f052e
|
541 |
|
b3c2d2ddd
|
542 543 |
if (signal_pending(current)) return -ERESTARTSYS; |
5274f052e
|
544 |
|
b3c2d2ddd
|
545 546 547 |
if (sd->need_wakeup) { wakeup_pipe_writers(pipe); sd->need_wakeup = false; |
5274f052e
|
548 |
} |
b3c2d2ddd
|
549 550 |
pipe_wait(pipe); } |
29e350944
|
551 |
|
b3c2d2ddd
|
552 553 |
return 1; } |
5274f052e
|
554 |
|
b3c2d2ddd
|
555 556 |
/** * splice_from_pipe_begin - start splicing from pipe |
b80901bbf
|
557 |
* @sd: information about the splice operation |
b3c2d2ddd
|
558 559 560 561 562 563 |
* * Description: * This function should be called before a loop containing * splice_from_pipe_next() and splice_from_pipe_feed() to * initialize the necessary fields of @sd. */ |
96f9bc8fb
|
564 |
static void splice_from_pipe_begin(struct splice_desc *sd) |
b3c2d2ddd
|
565 566 567 568 |
{ sd->num_spliced = 0; sd->need_wakeup = false; } |
5274f052e
|
569 |
|
b3c2d2ddd
|
570 571 572 573 574 575 576 577 578 579 |
/** * splice_from_pipe_end - finish splicing from pipe * @pipe: pipe to splice from * @sd: information about the splice operation * * Description: * This function will wake up pipe writers if necessary. It should * be called after a loop containing splice_from_pipe_next() and * splice_from_pipe_feed(). */ |
96f9bc8fb
|
580 |
static void splice_from_pipe_end(struct pipe_inode_info *pipe, struct splice_desc *sd) |
b3c2d2ddd
|
581 582 583 584 |
{ if (sd->need_wakeup) wakeup_pipe_writers(pipe); } |
5274f052e
|
585 |
|
b3c2d2ddd
|
586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 |
/** * __splice_from_pipe - splice data from a pipe to given actor * @pipe: pipe to splice from * @sd: information to @actor * @actor: handler that splices the data * * Description: * This function does little more than loop over the pipe and call * @actor to do the actual moving of a single struct pipe_buffer to * the desired destination. See pipe_to_file, pipe_to_sendpage, or * pipe_to_user. * */ ssize_t __splice_from_pipe(struct pipe_inode_info *pipe, struct splice_desc *sd, splice_actor *actor) { int ret; |
5274f052e
|
603 |
|
b3c2d2ddd
|
604 605 |
splice_from_pipe_begin(sd); do { |
c2489e07c
|
606 |
cond_resched(); |
b3c2d2ddd
|
607 608 609 610 611 612 613 |
ret = splice_from_pipe_next(pipe, sd); if (ret > 0) ret = splice_from_pipe_feed(pipe, sd, actor); } while (ret > 0); splice_from_pipe_end(pipe, sd); return sd->num_spliced ? sd->num_spliced : ret; |
5274f052e
|
614 |
} |
40bee44ea
|
615 |
EXPORT_SYMBOL(__splice_from_pipe); |
5274f052e
|
616 |
|
932cc6d4f
|
617 618 619 620 621 622 623 624 625 626 |
/** * splice_from_pipe - splice data from a pipe to a file * @pipe: pipe to splice from * @out: file to splice to * @ppos: position in @out * @len: how many bytes to splice * @flags: splice modifier flags * @actor: handler that splices the data * * Description: |
2933970b9
|
627 |
* See __splice_from_pipe. This function locks the pipe inode, |
932cc6d4f
|
628 629 630 |
* otherwise it's identical to __splice_from_pipe(). * */ |
6da618098
|
631 632 633 634 635 |
ssize_t splice_from_pipe(struct pipe_inode_info *pipe, struct file *out, loff_t *ppos, size_t len, unsigned int flags, splice_actor *actor) { ssize_t ret; |
c66ab6fa7
|
636 637 638 639 |
struct splice_desc sd = { .total_len = len, .flags = flags, .pos = *ppos, |
6a14b90bb
|
640 |
.u.file = out, |
c66ab6fa7
|
641 |
}; |
6da618098
|
642 |
|
61e0d47c3
|
643 |
pipe_lock(pipe); |
c66ab6fa7
|
644 |
ret = __splice_from_pipe(pipe, &sd, actor); |
61e0d47c3
|
645 |
pipe_unlock(pipe); |
6da618098
|
646 647 648 649 650 |
return ret; } /** |
8d0207652
|
651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 |
* iter_file_splice_write - splice data from a pipe to a file * @pipe: pipe info * @out: file to write to * @ppos: position in @out * @len: number of bytes to splice * @flags: splice modifier flags * * Description: * Will either move or copy pages (determined by @flags options) from * the given pipe inode to the given file. * This one is ->write_iter-based. * */ ssize_t iter_file_splice_write(struct pipe_inode_info *pipe, struct file *out, loff_t *ppos, size_t len, unsigned int flags) { struct splice_desc sd = { .total_len = len, .flags = flags, .pos = *ppos, .u.file = out, }; |
6718b6f85
|
674 |
int nbufs = pipe->max_usage; |
8d0207652
|
675 676 677 678 679 680 681 682 683 684 685 686 |
struct bio_vec *array = kcalloc(nbufs, sizeof(struct bio_vec), GFP_KERNEL); ssize_t ret; if (unlikely(!array)) return -ENOMEM; pipe_lock(pipe); splice_from_pipe_begin(&sd); while (sd.total_len) { struct iov_iter from; |
ec057595c
|
687 |
unsigned int head, tail, mask; |
8d0207652
|
688 |
size_t left; |
8cefc107c
|
689 |
int n; |
8d0207652
|
690 691 692 693 |
ret = splice_from_pipe_next(pipe, &sd); if (ret <= 0) break; |
6718b6f85
|
694 |
if (unlikely(nbufs < pipe->max_usage)) { |
8d0207652
|
695 |
kfree(array); |
6718b6f85
|
696 |
nbufs = pipe->max_usage; |
8d0207652
|
697 698 699 700 701 702 703 |
array = kcalloc(nbufs, sizeof(struct bio_vec), GFP_KERNEL); if (!array) { ret = -ENOMEM; break; } } |
ec057595c
|
704 705 706 |
head = pipe->head; tail = pipe->tail; mask = pipe->ring_size - 1; |
8d0207652
|
707 708 |
/* build the vector */ left = sd.total_len; |
8cefc107c
|
709 710 |
for (n = 0; !pipe_empty(head, tail) && left && n < nbufs; tail++, n++) { struct pipe_buffer *buf = &pipe->bufs[tail & mask]; |
8d0207652
|
711 712 713 714 |
size_t this_len = buf->len; if (this_len > left) this_len = left; |
fba597db4
|
715 |
ret = pipe_buf_confirm(pipe, buf); |
8d0207652
|
716 717 718 719 720 721 722 723 724 725 726 |
if (unlikely(ret)) { if (ret == -ENODATA) ret = 0; goto done; } array[n].bv_page = buf->page; array[n].bv_len = this_len; array[n].bv_offset = buf->offset; left -= this_len; } |
aa563d7bc
|
727 |
iov_iter_bvec(&from, WRITE, array, n, sd.total_len - left); |
abbb65899
|
728 |
ret = vfs_iter_write(out, &from, &sd.pos, 0); |
8d0207652
|
729 730 731 732 733 |
if (ret <= 0) break; sd.num_spliced += ret; sd.total_len -= ret; |
dbe4e192a
|
734 |
*ppos = sd.pos; |
8d0207652
|
735 736 |
/* dismiss the fully eaten buffers, adjust the partial one */ |
8cefc107c
|
737 |
tail = pipe->tail; |
8d0207652
|
738 |
while (ret) { |
8cefc107c
|
739 |
struct pipe_buffer *buf = &pipe->bufs[tail & mask]; |
8d0207652
|
740 |
if (ret >= buf->len) { |
8d0207652
|
741 742 |
ret -= buf->len; buf->len = 0; |
a779638cf
|
743 |
pipe_buf_release(pipe, buf); |
8cefc107c
|
744 745 |
tail++; pipe->tail = tail; |
8d0207652
|
746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 |
if (pipe->files) sd.need_wakeup = true; } else { buf->offset += ret; buf->len -= ret; ret = 0; } } } done: kfree(array); splice_from_pipe_end(pipe, &sd); pipe_unlock(pipe); if (sd.num_spliced) ret = sd.num_spliced; return ret; } EXPORT_SYMBOL(iter_file_splice_write); |
b2858d7d1
|
768 769 |
static int write_pipe_buf(struct pipe_inode_info *pipe, struct pipe_buffer *buf, struct splice_desc *sd) |
0b0a47f5c
|
770 |
{ |
b2858d7d1
|
771 772 |
int ret; void *data; |
06ae43f34
|
773 |
loff_t tmp = sd->pos; |
b2858d7d1
|
774 |
|
fbb32750a
|
775 |
data = kmap(buf->page); |
06ae43f34
|
776 |
ret = __kernel_write(sd->u.file, data + buf->offset, sd->len, &tmp); |
fbb32750a
|
777 |
kunmap(buf->page); |
b2858d7d1
|
778 779 |
return ret; |
0b0a47f5c
|
780 781 782 783 784 785 |
} static ssize_t default_file_splice_write(struct pipe_inode_info *pipe, struct file *out, loff_t *ppos, size_t len, unsigned int flags) { |
b2858d7d1
|
786 |
ssize_t ret; |
0b0a47f5c
|
787 |
|
b2858d7d1
|
788 789 790 |
ret = splice_from_pipe(pipe, out, ppos, len, flags, write_pipe_buf); if (ret > 0) *ppos += ret; |
0b0a47f5c
|
791 |
|
b2858d7d1
|
792 |
return ret; |
0b0a47f5c
|
793 |
} |
83f9135bd
|
794 795 |
/** * generic_splice_sendpage - splice data from a pipe to a socket |
932cc6d4f
|
796 |
* @pipe: pipe to splice from |
83f9135bd
|
797 |
* @out: socket to write to |
932cc6d4f
|
798 |
* @ppos: position in @out |
83f9135bd
|
799 800 801 |
* @len: number of bytes to splice * @flags: splice modifier flags * |
932cc6d4f
|
802 803 804 |
* Description: * Will send @len bytes from the pipe to a network socket. No data copying * is involved. |
83f9135bd
|
805 806 |
* */ |
3a326a2ce
|
807 |
ssize_t generic_splice_sendpage(struct pipe_inode_info *pipe, struct file *out, |
cbb7e577e
|
808 |
loff_t *ppos, size_t len, unsigned int flags) |
5274f052e
|
809 |
{ |
00522fb41
|
810 |
return splice_from_pipe(pipe, out, ppos, len, flags, pipe_to_sendpage); |
5274f052e
|
811 |
} |
059a8f373
|
812 |
EXPORT_SYMBOL(generic_splice_sendpage); |
a0f067802
|
813 |
|
83f9135bd
|
814 815 816 |
/* * Attempt to initiate a splice from pipe to file. */ |
3a326a2ce
|
817 |
static long do_splice_from(struct pipe_inode_info *pipe, struct file *out, |
cbb7e577e
|
818 |
loff_t *ppos, size_t len, unsigned int flags) |
5274f052e
|
819 |
{ |
0b0a47f5c
|
820 821 |
ssize_t (*splice_write)(struct pipe_inode_info *, struct file *, loff_t *, size_t, unsigned int); |
5274f052e
|
822 |
|
72c2d5319
|
823 |
if (out->f_op->splice_write) |
cc56f7de7
|
824 825 |
splice_write = out->f_op->splice_write; else |
0b0a47f5c
|
826 |
splice_write = default_file_splice_write; |
500368f7f
|
827 |
return splice_write(pipe, out, ppos, len, flags); |
5274f052e
|
828 |
} |
83f9135bd
|
829 830 831 |
/* * Attempt to initiate a splice from a file to a pipe. */ |
cbb7e577e
|
832 833 834 |
static long do_splice_to(struct file *in, loff_t *ppos, struct pipe_inode_info *pipe, size_t len, unsigned int flags) |
5274f052e
|
835 |
{ |
6818173bd
|
836 837 |
ssize_t (*splice_read)(struct file *, loff_t *, struct pipe_inode_info *, size_t, unsigned int); |
5274f052e
|
838 |
int ret; |
49570e9b2
|
839 |
if (unlikely(!(in->f_mode & FMODE_READ))) |
5274f052e
|
840 |
return -EBADF; |
cbb7e577e
|
841 |
ret = rw_verify_area(READ, in, ppos, len); |
5274f052e
|
842 843 |
if (unlikely(ret < 0)) return ret; |
03cc0789a
|
844 845 |
if (unlikely(len > MAX_RW_COUNT)) len = MAX_RW_COUNT; |
72c2d5319
|
846 |
if (in->f_op->splice_read) |
cc56f7de7
|
847 848 |
splice_read = in->f_op->splice_read; else |
6818173bd
|
849 850 851 |
splice_read = default_file_splice_read; return splice_read(in, ppos, pipe, len, flags); |
5274f052e
|
852 |
} |
932cc6d4f
|
853 854 855 856 857 858 859 860 861 |
/** * splice_direct_to_actor - splices data directly between two non-pipes * @in: file to splice from * @sd: actor information on where to splice to * @actor: handles the data splicing * * Description: * This is a special case helper to splice directly between two * points, without requiring an explicit pipe. Internally an allocated |
79685b8de
|
862 |
* pipe is cached in the process, and reused during the lifetime of |
932cc6d4f
|
863 864 |
* that process. * |
c66ab6fa7
|
865 866 867 |
*/ ssize_t splice_direct_to_actor(struct file *in, struct splice_desc *sd, splice_direct_actor *actor) |
b92ce5589
|
868 869 870 871 |
{ struct pipe_inode_info *pipe; long ret, bytes; umode_t i_mode; |
c66ab6fa7
|
872 |
size_t len; |
0ff28d9f4
|
873 |
int i, flags, more; |
b92ce5589
|
874 875 876 877 878 879 |
/* * We require the input being a regular file, as we don't want to * randomly drop data for eg socket -> socket splicing. Use the * piped splicing for that! */ |
496ad9aa8
|
880 |
i_mode = file_inode(in)->i_mode; |
b92ce5589
|
881 882 883 884 885 886 887 888 |
if (unlikely(!S_ISREG(i_mode) && !S_ISBLK(i_mode))) return -EINVAL; /* * neither in nor out is a pipe, setup an internal pipe attached to * 'out' and transfer the wanted data from 'in' to 'out' through that */ pipe = current->splice_pipe; |
49570e9b2
|
889 |
if (unlikely(!pipe)) { |
7bee130e2
|
890 |
pipe = alloc_pipe_info(); |
b92ce5589
|
891 892 893 894 895 |
if (!pipe) return -ENOMEM; /* * We don't have an immediate reader, but we'll read the stuff |
00522fb41
|
896 |
* out of the pipe right after the splice_to_pipe(). So set |
b92ce5589
|
897 898 899 900 901 902 903 904 |
* PIPE_READERS appropriately. */ pipe->readers = 1; current->splice_pipe = pipe; } /* |
73d62d83e
|
905 |
* Do the splice. |
b92ce5589
|
906 907 908 |
*/ ret = 0; bytes = 0; |
c66ab6fa7
|
909 910 911 912 913 914 915 |
len = sd->total_len; flags = sd->flags; /* * Don't block on output, we have to drain the direct pipe. */ sd->flags &= ~SPLICE_F_NONBLOCK; |
0ff28d9f4
|
916 |
more = sd->flags & SPLICE_F_MORE; |
b92ce5589
|
917 |
|
8cefc107c
|
918 |
WARN_ON_ONCE(!pipe_empty(pipe->head, pipe->tail)); |
176144455
|
919 |
|
b92ce5589
|
920 |
while (len) { |
8cefc107c
|
921 |
unsigned int p_space; |
51a92c0f6
|
922 |
size_t read_len; |
a82c53a0e
|
923 |
loff_t pos = sd->pos, prev_pos = pos; |
b92ce5589
|
924 |
|
176144455
|
925 |
/* Don't try to read more the pipe has space for. */ |
6718b6f85
|
926 |
p_space = pipe->max_usage - |
8cefc107c
|
927 928 |
pipe_occupancy(pipe->head, pipe->tail); read_len = min_t(size_t, len, p_space << PAGE_SHIFT); |
176144455
|
929 |
ret = do_splice_to(in, &pos, pipe, read_len, flags); |
51a92c0f6
|
930 |
if (unlikely(ret <= 0)) |
b92ce5589
|
931 932 933 |
goto out_release; read_len = ret; |
c66ab6fa7
|
934 |
sd->total_len = read_len; |
b92ce5589
|
935 936 |
/* |
0ff28d9f4
|
937 938 939 940 941 942 943 944 945 |
* If more data is pending, set SPLICE_F_MORE * If this is the last data and SPLICE_F_MORE was not set * initially, clears it. */ if (read_len < len) sd->flags |= SPLICE_F_MORE; else if (!more) sd->flags &= ~SPLICE_F_MORE; /* |
b92ce5589
|
946 947 948 949 |
* NOTE: nonblocking mode only applies to the input. We * must not do the output in nonblocking mode as then we * could get stuck data in the internal pipe: */ |
c66ab6fa7
|
950 |
ret = actor(pipe, sd); |
a82c53a0e
|
951 952 |
if (unlikely(ret <= 0)) { sd->pos = prev_pos; |
b92ce5589
|
953 |
goto out_release; |
a82c53a0e
|
954 |
} |
b92ce5589
|
955 956 957 |
bytes += ret; len -= ret; |
bcd4f3acb
|
958 |
sd->pos = pos; |
b92ce5589
|
959 |
|
a82c53a0e
|
960 961 |
if (ret < read_len) { sd->pos = prev_pos + ret; |
51a92c0f6
|
962 |
goto out_release; |
a82c53a0e
|
963 |
} |
b92ce5589
|
964 |
} |
9e97198db
|
965 |
done: |
8cefc107c
|
966 |
pipe->tail = pipe->head = 0; |
808487085
|
967 |
file_accessed(in); |
b92ce5589
|
968 969 970 971 972 973 974 |
return bytes; out_release: /* * If we did an incomplete transfer we must release * the pipe buffers in question: */ |
8cefc107c
|
975 976 |
for (i = 0; i < pipe->ring_size; i++) { struct pipe_buffer *buf = &pipe->bufs[i]; |
b92ce5589
|
977 |
|
a779638cf
|
978 979 |
if (buf->ops) pipe_buf_release(pipe, buf); |
b92ce5589
|
980 |
} |
b92ce5589
|
981 |
|
9e97198db
|
982 983 |
if (!bytes) bytes = ret; |
c66ab6fa7
|
984 |
|
9e97198db
|
985 |
goto done; |
c66ab6fa7
|
986 987 988 989 990 991 |
} EXPORT_SYMBOL(splice_direct_to_actor); static int direct_splice_actor(struct pipe_inode_info *pipe, struct splice_desc *sd) { |
6a14b90bb
|
992 |
struct file *file = sd->u.file; |
c66ab6fa7
|
993 |
|
7995bd287
|
994 |
return do_splice_from(pipe, file, sd->opos, sd->total_len, |
2cb4b05e7
|
995 |
sd->flags); |
c66ab6fa7
|
996 |
} |
932cc6d4f
|
997 998 999 1000 1001 |
/** * do_splice_direct - splices data directly between two files * @in: file to splice from * @ppos: input file offset * @out: file to splice to |
acdb37c36
|
1002 |
* @opos: output file offset |
932cc6d4f
|
1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 |
* @len: number of bytes to splice * @flags: splice modifier flags * * Description: * For use by do_sendfile(). splice can easily emulate sendfile, but * doing it in the application would incur an extra system call * (splice in + splice out, as compared to just sendfile()). So this helper * can splice directly through a process-private pipe. * */ |
c66ab6fa7
|
1013 |
long do_splice_direct(struct file *in, loff_t *ppos, struct file *out, |
7995bd287
|
1014 |
loff_t *opos, size_t len, unsigned int flags) |
c66ab6fa7
|
1015 1016 1017 1018 1019 1020 |
{ struct splice_desc sd = { .len = len, .total_len = len, .flags = flags, .pos = *ppos, |
6a14b90bb
|
1021 |
.u.file = out, |
7995bd287
|
1022 |
.opos = opos, |
c66ab6fa7
|
1023 |
}; |
51a92c0f6
|
1024 |
long ret; |
c66ab6fa7
|
1025 |
|
18c67cb9f
|
1026 1027 1028 1029 1030 1031 1032 1033 1034 |
if (unlikely(!(out->f_mode & FMODE_WRITE))) return -EBADF; if (unlikely(out->f_flags & O_APPEND)) return -EINVAL; ret = rw_verify_area(WRITE, out, opos, len); if (unlikely(ret < 0)) return ret; |
c66ab6fa7
|
1035 |
ret = splice_direct_to_actor(in, &sd, direct_splice_actor); |
51a92c0f6
|
1036 |
if (ret > 0) |
a82c53a0e
|
1037 |
*ppos = sd.pos; |
51a92c0f6
|
1038 |
|
c66ab6fa7
|
1039 |
return ret; |
b92ce5589
|
1040 |
} |
1c118596a
|
1041 |
EXPORT_SYMBOL(do_splice_direct); |
b92ce5589
|
1042 |
|
8924feff6
|
1043 1044 |
static int wait_for_space(struct pipe_inode_info *pipe, unsigned flags) { |
52bce9116
|
1045 1046 1047 1048 1049 |
for (;;) { if (unlikely(!pipe->readers)) { send_sig(SIGPIPE, current, 0); return -EPIPE; } |
6718b6f85
|
1050 |
if (!pipe_full(pipe->head, pipe->tail, pipe->max_usage)) |
52bce9116
|
1051 |
return 0; |
8924feff6
|
1052 1053 1054 1055 |
if (flags & SPLICE_F_NONBLOCK) return -EAGAIN; if (signal_pending(current)) return -ERESTARTSYS; |
8924feff6
|
1056 |
pipe_wait(pipe); |
8924feff6
|
1057 |
} |
8924feff6
|
1058 |
} |
7c77f0b3f
|
1059 1060 1061 |
static int splice_pipe_to_pipe(struct pipe_inode_info *ipipe, struct pipe_inode_info *opipe, size_t len, unsigned int flags); |
ddac0d39c
|
1062 1063 |
/* |
83f9135bd
|
1064 1065 |
* Determine where to splice to/from. */ |
529565dcb
|
1066 1067 1068 |
static long do_splice(struct file *in, loff_t __user *off_in, struct file *out, loff_t __user *off_out, size_t len, unsigned int flags) |
5274f052e
|
1069 |
{ |
7c77f0b3f
|
1070 1071 |
struct pipe_inode_info *ipipe; struct pipe_inode_info *opipe; |
7995bd287
|
1072 |
loff_t offset; |
a4514ebd8
|
1073 |
long ret; |
5274f052e
|
1074 |
|
71993e62a
|
1075 1076 |
ipipe = get_pipe_info(in); opipe = get_pipe_info(out); |
7c77f0b3f
|
1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 |
if (ipipe && opipe) { if (off_in || off_out) return -ESPIPE; if (!(in->f_mode & FMODE_READ)) return -EBADF; if (!(out->f_mode & FMODE_WRITE)) return -EBADF; /* Splicing to self would be fun, but... */ if (ipipe == opipe) return -EINVAL; |
ee5e00119
|
1091 1092 |
if ((in->f_flags | out->f_flags) & O_NONBLOCK) flags |= SPLICE_F_NONBLOCK; |
7c77f0b3f
|
1093 1094 1095 1096 |
return splice_pipe_to_pipe(ipipe, opipe, len, flags); } if (ipipe) { |
529565dcb
|
1097 1098 |
if (off_in) return -ESPIPE; |
b92ce5589
|
1099 |
if (off_out) { |
19c9a49b4
|
1100 |
if (!(out->f_mode & FMODE_PWRITE)) |
b92ce5589
|
1101 |
return -EINVAL; |
cbb7e577e
|
1102 |
if (copy_from_user(&offset, off_out, sizeof(loff_t))) |
b92ce5589
|
1103 |
return -EFAULT; |
7995bd287
|
1104 1105 1106 |
} else { offset = out->f_pos; } |
529565dcb
|
1107 |
|
18c67cb9f
|
1108 1109 1110 1111 1112 1113 1114 1115 1116 |
if (unlikely(!(out->f_mode & FMODE_WRITE))) return -EBADF; if (unlikely(out->f_flags & O_APPEND)) return -EINVAL; ret = rw_verify_area(WRITE, out, &offset, len); if (unlikely(ret < 0)) return ret; |
ee5e00119
|
1117 1118 |
if (in->f_flags & O_NONBLOCK) flags |= SPLICE_F_NONBLOCK; |
500368f7f
|
1119 |
file_start_write(out); |
7995bd287
|
1120 |
ret = do_splice_from(ipipe, out, &offset, len, flags); |
500368f7f
|
1121 |
file_end_write(out); |
a4514ebd8
|
1122 |
|
7995bd287
|
1123 1124 1125 |
if (!off_out) out->f_pos = offset; else if (copy_to_user(off_out, &offset, sizeof(loff_t))) |
a4514ebd8
|
1126 1127 1128 |
ret = -EFAULT; return ret; |
529565dcb
|
1129 |
} |
5274f052e
|
1130 |
|
7c77f0b3f
|
1131 |
if (opipe) { |
529565dcb
|
1132 1133 |
if (off_out) return -ESPIPE; |
b92ce5589
|
1134 |
if (off_in) { |
19c9a49b4
|
1135 |
if (!(in->f_mode & FMODE_PREAD)) |
b92ce5589
|
1136 |
return -EINVAL; |
cbb7e577e
|
1137 |
if (copy_from_user(&offset, off_in, sizeof(loff_t))) |
b92ce5589
|
1138 |
return -EFAULT; |
7995bd287
|
1139 1140 1141 |
} else { offset = in->f_pos; } |
529565dcb
|
1142 |
|
ee5e00119
|
1143 1144 |
if (out->f_flags & O_NONBLOCK) flags |= SPLICE_F_NONBLOCK; |
8924feff6
|
1145 1146 |
pipe_lock(opipe); ret = wait_for_space(opipe, flags); |
3253d9d09
|
1147 |
if (!ret) { |
6a965666b
|
1148 |
unsigned int p_space; |
3253d9d09
|
1149 1150 |
/* Don't try to read more the pipe has space for. */ |
6a965666b
|
1151 1152 |
p_space = opipe->max_usage - pipe_occupancy(opipe->head, opipe->tail); len = min_t(size_t, len, p_space << PAGE_SHIFT); |
3253d9d09
|
1153 |
|
8924feff6
|
1154 |
ret = do_splice_to(in, &offset, opipe, len, flags); |
3253d9d09
|
1155 |
} |
8924feff6
|
1156 1157 1158 |
pipe_unlock(opipe); if (ret > 0) wakeup_pipe_readers(opipe); |
7995bd287
|
1159 1160 1161 |
if (!off_in) in->f_pos = offset; else if (copy_to_user(off_in, &offset, sizeof(loff_t))) |
a4514ebd8
|
1162 1163 1164 |
ret = -EFAULT; return ret; |
529565dcb
|
1165 |
} |
5274f052e
|
1166 1167 1168 |
return -EINVAL; } |
79fddc4ef
|
1169 1170 1171 |
static int iter_to_pipe(struct iov_iter *from, struct pipe_inode_info *pipe, unsigned flags) |
912d35f86
|
1172 |
{ |
79fddc4ef
|
1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 |
struct pipe_buffer buf = { .ops = &user_page_pipe_buf_ops, .flags = flags }; size_t total = 0; int ret = 0; bool failed = false; while (iov_iter_count(from) && !failed) { struct page *pages[16]; |
db85a9eb2
|
1183 1184 |
ssize_t copied; size_t start; |
79fddc4ef
|
1185 |
int n; |
db85a9eb2
|
1186 |
|
79fddc4ef
|
1187 1188 1189 1190 1191 |
copied = iov_iter_get_pages(from, pages, ~0UL, 16, &start); if (copied <= 0) { ret = copied; break; } |
db85a9eb2
|
1192 |
|
79fddc4ef
|
1193 |
for (n = 0; copied; n++, start = 0) { |
db85a9eb2
|
1194 |
int size = min_t(int, copied, PAGE_SIZE - start); |
79fddc4ef
|
1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 |
if (!failed) { buf.page = pages[n]; buf.offset = start; buf.len = size; ret = add_to_pipe(pipe, &buf); if (unlikely(ret < 0)) { failed = true; } else { iov_iter_advance(from, ret); total += ret; } } else { put_page(pages[n]); } |
db85a9eb2
|
1209 |
copied -= size; |
912d35f86
|
1210 |
} |
912d35f86
|
1211 |
} |
79fddc4ef
|
1212 |
return total ? total : ret; |
912d35f86
|
1213 |
} |
6a14b90bb
|
1214 1215 1216 |
static int pipe_to_user(struct pipe_inode_info *pipe, struct pipe_buffer *buf, struct splice_desc *sd) { |
6130f5315
|
1217 1218 |
int n = copy_page_to_iter(buf->page, buf->offset, sd->len, sd->u.data); return n == sd->len ? n : -EFAULT; |
6a14b90bb
|
1219 1220 1221 1222 1223 1224 |
} /* * For lack of a better implementation, implement vmsplice() to userspace * as a simple copy of the pipes pages to the user iov. */ |
87a3002af
|
1225 1226 |
static long vmsplice_to_user(struct file *file, struct iov_iter *iter, unsigned int flags) |
6a14b90bb
|
1227 |
{ |
87a3002af
|
1228 1229 1230 1231 1232 1233 1234 |
struct pipe_inode_info *pipe = get_pipe_info(file); struct splice_desc sd = { .total_len = iov_iter_count(iter), .flags = flags, .u.data = iter }; long ret = 0; |
6a14b90bb
|
1235 |
|
6a14b90bb
|
1236 1237 |
if (!pipe) return -EBADF; |
345995fa4
|
1238 1239 1240 1241 1242 |
if (sd.total_len) { pipe_lock(pipe); ret = __splice_from_pipe(pipe, &sd, pipe_to_user); pipe_unlock(pipe); } |
6a14b90bb
|
1243 1244 1245 |
return ret; } |
912d35f86
|
1246 1247 1248 1249 |
/* * vmsplice splices a user address range into a pipe. It can be thought of * as splice-from-memory, where the regular splice is splice-from-file (or * to file). In both cases the output is a pipe, naturally. |
912d35f86
|
1250 |
*/ |
87a3002af
|
1251 1252 |
static long vmsplice_to_pipe(struct file *file, struct iov_iter *iter, unsigned int flags) |
912d35f86
|
1253 |
{ |
ddac0d39c
|
1254 |
struct pipe_inode_info *pipe; |
87a3002af
|
1255 |
long ret = 0; |
79fddc4ef
|
1256 1257 1258 1259 |
unsigned buf_flag = 0; if (flags & SPLICE_F_GIFT) buf_flag = PIPE_BUF_FLAG_GIFT; |
912d35f86
|
1260 |
|
71993e62a
|
1261 |
pipe = get_pipe_info(file); |
ddac0d39c
|
1262 |
if (!pipe) |
912d35f86
|
1263 |
return -EBADF; |
912d35f86
|
1264 |
|
8924feff6
|
1265 1266 |
pipe_lock(pipe); ret = wait_for_space(pipe, flags); |
79fddc4ef
|
1267 |
if (!ret) |
87a3002af
|
1268 |
ret = iter_to_pipe(iter, pipe, buf_flag); |
8924feff6
|
1269 1270 1271 |
pipe_unlock(pipe); if (ret > 0) wakeup_pipe_readers(pipe); |
35f3d14db
|
1272 |
return ret; |
912d35f86
|
1273 |
} |
87a3002af
|
1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 |
static int vmsplice_type(struct fd f, int *type) { if (!f.file) return -EBADF; if (f.file->f_mode & FMODE_WRITE) { *type = WRITE; } else if (f.file->f_mode & FMODE_READ) { *type = READ; } else { fdput(f); return -EBADF; } return 0; } |
6a14b90bb
|
1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 |
/* * Note that vmsplice only really supports true splicing _from_ user memory * to a pipe, not the other way around. Splicing from user memory is a simple * operation that can be supported without any funky alignment restrictions * or nasty vm tricks. We simply map in the user memory and fill them into * a pipe. The reverse isn't quite as easy, though. There are two possible * solutions for that: * * - memcpy() the data internally, at which point we might as well just * do a regular read() on the buffer anyway. * - Lots of nasty vm tricks, that are neither fast nor flexible (it * has restriction limitations on both ends of the pipe). * * Currently we punt and implement it as a normal copy, see pipe_to_user(). * */ |
87a3002af
|
1304 |
static long do_vmsplice(struct file *f, struct iov_iter *iter, unsigned int flags) |
912d35f86
|
1305 |
{ |
3d6ea290f
|
1306 1307 |
if (unlikely(flags & ~SPLICE_F_ALL)) return -EINVAL; |
6a14b90bb
|
1308 |
|
87a3002af
|
1309 1310 |
if (!iov_iter_count(iter)) return 0; |
912d35f86
|
1311 |
|
87a3002af
|
1312 1313 1314 1315 |
if (iov_iter_rw(iter) == WRITE) return vmsplice_to_pipe(f, iter, flags); else return vmsplice_to_user(f, iter, flags); |
912d35f86
|
1316 |
} |
87a3002af
|
1317 |
SYSCALL_DEFINE4(vmsplice, int, fd, const struct iovec __user *, uiov, |
30cfe4ef8
|
1318 1319 |
unsigned long, nr_segs, unsigned int, flags) { |
87a3002af
|
1320 1321 1322 |
struct iovec iovstack[UIO_FASTIOV]; struct iovec *iov = iovstack; struct iov_iter iter; |
87e5e6dab
|
1323 |
ssize_t error; |
87a3002af
|
1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 |
struct fd f; int type; f = fdget(fd); error = vmsplice_type(f, &type); if (error) return error; error = import_iovec(type, uiov, nr_segs, ARRAY_SIZE(iovstack), &iov, &iter); |
87e5e6dab
|
1334 |
if (error >= 0) { |
87a3002af
|
1335 1336 1337 1338 1339 |
error = do_vmsplice(f.file, &iter, flags); kfree(iov); } fdput(f); return error; |
30cfe4ef8
|
1340 |
} |
76b021d05
|
1341 1342 1343 1344 |
#ifdef CONFIG_COMPAT COMPAT_SYSCALL_DEFINE4(vmsplice, int, fd, const struct compat_iovec __user *, iov32, unsigned int, nr_segs, unsigned int, flags) { |
87a3002af
|
1345 1346 1347 |
struct iovec iovstack[UIO_FASTIOV]; struct iovec *iov = iovstack; struct iov_iter iter; |
87e5e6dab
|
1348 |
ssize_t error; |
87a3002af
|
1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 |
struct fd f; int type; f = fdget(fd); error = vmsplice_type(f, &type); if (error) return error; error = compat_import_iovec(type, iov32, nr_segs, ARRAY_SIZE(iovstack), &iov, &iter); |
87e5e6dab
|
1359 |
if (error >= 0) { |
87a3002af
|
1360 1361 |
error = do_vmsplice(f.file, &iter, flags); kfree(iov); |
76b021d05
|
1362 |
} |
87a3002af
|
1363 1364 |
fdput(f); return error; |
76b021d05
|
1365 1366 |
} #endif |
836f92adf
|
1367 1368 1369 |
SYSCALL_DEFINE6(splice, int, fd_in, loff_t __user *, off_in, int, fd_out, loff_t __user *, off_out, size_t, len, unsigned int, flags) |
5274f052e
|
1370 |
{ |
2903ff019
|
1371 |
struct fd in, out; |
5274f052e
|
1372 |
long error; |
5274f052e
|
1373 1374 1375 |
if (unlikely(!len)) return 0; |
3d6ea290f
|
1376 1377 |
if (unlikely(flags & ~SPLICE_F_ALL)) return -EINVAL; |
5274f052e
|
1378 |
error = -EBADF; |
2903ff019
|
1379 1380 1381 1382 1383 1384 1385 1386 |
in = fdget(fd_in); if (in.file) { if (in.file->f_mode & FMODE_READ) { out = fdget(fd_out); if (out.file) { if (out.file->f_mode & FMODE_WRITE) error = do_splice(in.file, off_in, out.file, off_out, |
529565dcb
|
1387 |
len, flags); |
2903ff019
|
1388 |
fdput(out); |
5274f052e
|
1389 1390 |
} } |
2903ff019
|
1391 |
fdput(in); |
5274f052e
|
1392 |
} |
5274f052e
|
1393 1394 |
return error; } |
70524490e
|
1395 1396 |
/* |
aadd06e5c
|
1397 1398 1399 |
* Make sure there's data to read. Wait for input if we can, otherwise * return an appropriate error. */ |
7c77f0b3f
|
1400 |
static int ipipe_prep(struct pipe_inode_info *pipe, unsigned int flags) |
aadd06e5c
|
1401 1402 1403 1404 |
{ int ret; /* |
8cefc107c
|
1405 |
* Check the pipe occupancy without the inode lock first. This function |
aadd06e5c
|
1406 1407 |
* is speculative anyways, so missing one is ok. */ |
8cefc107c
|
1408 |
if (!pipe_empty(pipe->head, pipe->tail)) |
aadd06e5c
|
1409 1410 1411 |
return 0; ret = 0; |
61e0d47c3
|
1412 |
pipe_lock(pipe); |
aadd06e5c
|
1413 |
|
8cefc107c
|
1414 |
while (pipe_empty(pipe->head, pipe->tail)) { |
aadd06e5c
|
1415 1416 1417 1418 1419 1420 |
if (signal_pending(current)) { ret = -ERESTARTSYS; break; } if (!pipe->writers) break; |
a28c8b9db
|
1421 1422 1423 |
if (flags & SPLICE_F_NONBLOCK) { ret = -EAGAIN; break; |
aadd06e5c
|
1424 1425 1426 |
} pipe_wait(pipe); } |
61e0d47c3
|
1427 |
pipe_unlock(pipe); |
aadd06e5c
|
1428 1429 1430 1431 1432 1433 1434 |
return ret; } /* * Make sure there's writeable room. Wait for room if we can, otherwise * return an appropriate error. */ |
7c77f0b3f
|
1435 |
static int opipe_prep(struct pipe_inode_info *pipe, unsigned int flags) |
aadd06e5c
|
1436 1437 1438 1439 |
{ int ret; /* |
8cefc107c
|
1440 |
* Check pipe occupancy without the inode lock first. This function |
aadd06e5c
|
1441 1442 |
* is speculative anyways, so missing one is ok. */ |
6718b6f85
|
1443 |
if (pipe_full(pipe->head, pipe->tail, pipe->max_usage)) |
aadd06e5c
|
1444 1445 1446 |
return 0; ret = 0; |
61e0d47c3
|
1447 |
pipe_lock(pipe); |
aadd06e5c
|
1448 |
|
6718b6f85
|
1449 |
while (pipe_full(pipe->head, pipe->tail, pipe->max_usage)) { |
aadd06e5c
|
1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 |
if (!pipe->readers) { send_sig(SIGPIPE, current, 0); ret = -EPIPE; break; } if (flags & SPLICE_F_NONBLOCK) { ret = -EAGAIN; break; } if (signal_pending(current)) { ret = -ERESTARTSYS; break; } |
aadd06e5c
|
1463 |
pipe_wait(pipe); |
aadd06e5c
|
1464 |
} |
61e0d47c3
|
1465 |
pipe_unlock(pipe); |
aadd06e5c
|
1466 1467 1468 1469 |
return ret; } /* |
7c77f0b3f
|
1470 1471 1472 1473 1474 1475 1476 |
* Splice contents of ipipe to opipe. */ static int splice_pipe_to_pipe(struct pipe_inode_info *ipipe, struct pipe_inode_info *opipe, size_t len, unsigned int flags) { struct pipe_buffer *ibuf, *obuf; |
8cefc107c
|
1477 1478 1479 1480 |
unsigned int i_head, o_head; unsigned int i_tail, o_tail; unsigned int i_mask, o_mask; int ret = 0; |
7c77f0b3f
|
1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 |
bool input_wakeup = false; retry: ret = ipipe_prep(ipipe, flags); if (ret) return ret; ret = opipe_prep(opipe, flags); if (ret) return ret; /* * Potential ABBA deadlock, work around it by ordering lock * grabbing by pipe info address. Otherwise two different processes * could deadlock (one doing tee from A -> B, the other from B -> A). */ pipe_double_lock(ipipe, opipe); |
8cefc107c
|
1499 1500 1501 1502 |
i_tail = ipipe->tail; i_mask = ipipe->ring_size - 1; o_head = opipe->head; o_mask = opipe->ring_size - 1; |
7c77f0b3f
|
1503 |
do { |
8cefc107c
|
1504 |
size_t o_len; |
7c77f0b3f
|
1505 1506 1507 1508 1509 1510 |
if (!opipe->readers) { send_sig(SIGPIPE, current, 0); if (!ret) ret = -EPIPE; break; } |
8cefc107c
|
1511 1512 1513 1514 |
i_head = ipipe->head; o_tail = opipe->tail; if (pipe_empty(i_head, i_tail) && !ipipe->writers) |
7c77f0b3f
|
1515 1516 1517 1518 1519 1520 |
break; /* * Cannot make any progress, because either the input * pipe is empty or the output pipe is full. */ |
8cefc107c
|
1521 |
if (pipe_empty(i_head, i_tail) || |
6718b6f85
|
1522 |
pipe_full(o_head, o_tail, opipe->max_usage)) { |
7c77f0b3f
|
1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 |
/* Already processed some buffers, break */ if (ret) break; if (flags & SPLICE_F_NONBLOCK) { ret = -EAGAIN; break; } /* * We raced with another reader/writer and haven't * managed to process any buffers. A zero return * value means EOF, so retry instead. */ pipe_unlock(ipipe); pipe_unlock(opipe); goto retry; } |
8cefc107c
|
1541 1542 |
ibuf = &ipipe->bufs[i_tail & i_mask]; obuf = &opipe->bufs[o_head & o_mask]; |
7c77f0b3f
|
1543 1544 1545 1546 1547 1548 1549 |
if (len >= ibuf->len) { /* * Simply move the whole buffer from ipipe to opipe */ *obuf = *ibuf; ibuf->ops = NULL; |
8cefc107c
|
1550 1551 |
i_tail++; ipipe->tail = i_tail; |
7c77f0b3f
|
1552 |
input_wakeup = true; |
8cefc107c
|
1553 1554 1555 |
o_len = obuf->len; o_head++; opipe->head = o_head; |
7c77f0b3f
|
1556 1557 1558 1559 1560 |
} else { /* * Get a reference to this pipe buffer, * so we can copy the contents over. */ |
15fab63e1
|
1561 1562 1563 1564 1565 |
if (!pipe_buf_get(ipipe, ibuf)) { if (ret == 0) ret = -EFAULT; break; } |
7c77f0b3f
|
1566 1567 1568 1569 1570 1571 1572 |
*obuf = *ibuf; /* * Don't inherit the gift flag, we need to * prevent multiple steals of this page. */ obuf->flags &= ~PIPE_BUF_FLAG_GIFT; |
a0ce2f0aa
|
1573 |
pipe_buf_mark_unmergeable(obuf); |
7c77f0b3f
|
1574 |
obuf->len = len; |
8cefc107c
|
1575 1576 1577 1578 1579 |
ibuf->offset += len; ibuf->len -= len; o_len = len; o_head++; opipe->head = o_head; |
7c77f0b3f
|
1580 |
} |
8cefc107c
|
1581 1582 |
ret += o_len; len -= o_len; |
7c77f0b3f
|
1583 1584 1585 1586 1587 1588 1589 1590 |
} while (len); pipe_unlock(ipipe); pipe_unlock(opipe); /* * If we put data in the output pipe, wakeup any potential readers. */ |
825cdcb1a
|
1591 1592 |
if (ret > 0) wakeup_pipe_readers(opipe); |
7c77f0b3f
|
1593 1594 1595 1596 1597 1598 1599 |
if (input_wakeup) wakeup_pipe_writers(ipipe); return ret; } /* |
70524490e
|
1600 1601 1602 1603 1604 1605 1606 |
* Link contents of ipipe to opipe. */ static int link_pipe(struct pipe_inode_info *ipipe, struct pipe_inode_info *opipe, size_t len, unsigned int flags) { struct pipe_buffer *ibuf, *obuf; |
8cefc107c
|
1607 1608 1609 1610 |
unsigned int i_head, o_head; unsigned int i_tail, o_tail; unsigned int i_mask, o_mask; int ret = 0; |
70524490e
|
1611 1612 1613 |
/* * Potential ABBA deadlock, work around it by ordering lock |
61e0d47c3
|
1614 |
* grabbing by pipe info address. Otherwise two different processes |
70524490e
|
1615 1616 |
* could deadlock (one doing tee from A -> B, the other from B -> A). */ |
61e0d47c3
|
1617 |
pipe_double_lock(ipipe, opipe); |
70524490e
|
1618 |
|
8cefc107c
|
1619 1620 1621 1622 |
i_tail = ipipe->tail; i_mask = ipipe->ring_size - 1; o_head = opipe->head; o_mask = opipe->ring_size - 1; |
aadd06e5c
|
1623 |
do { |
70524490e
|
1624 1625 1626 1627 1628 1629 |
if (!opipe->readers) { send_sig(SIGPIPE, current, 0); if (!ret) ret = -EPIPE; break; } |
70524490e
|
1630 |
|
8cefc107c
|
1631 1632 |
i_head = ipipe->head; o_tail = opipe->tail; |
aadd06e5c
|
1633 |
/* |
8cefc107c
|
1634 |
* If we have iterated all input buffers or run out of |
aadd06e5c
|
1635 1636 |
* output room, break. */ |
8cefc107c
|
1637 |
if (pipe_empty(i_head, i_tail) || |
6718b6f85
|
1638 |
pipe_full(o_head, o_tail, opipe->max_usage)) |
aadd06e5c
|
1639 |
break; |
70524490e
|
1640 |
|
8cefc107c
|
1641 1642 |
ibuf = &ipipe->bufs[i_tail & i_mask]; obuf = &opipe->bufs[o_head & o_mask]; |
70524490e
|
1643 1644 |
/* |
aadd06e5c
|
1645 1646 |
* Get a reference to this pipe buffer, * so we can copy the contents over. |
70524490e
|
1647 |
*/ |
15fab63e1
|
1648 1649 1650 1651 1652 |
if (!pipe_buf_get(ipipe, ibuf)) { if (ret == 0) ret = -EFAULT; break; } |
aadd06e5c
|
1653 |
|
aadd06e5c
|
1654 |
*obuf = *ibuf; |
2a27250e6
|
1655 |
/* |
aadd06e5c
|
1656 1657 |
* Don't inherit the gift flag, we need to * prevent multiple steals of this page. |
2a27250e6
|
1658 |
*/ |
aadd06e5c
|
1659 |
obuf->flags &= ~PIPE_BUF_FLAG_GIFT; |
70524490e
|
1660 |
|
a0ce2f0aa
|
1661 |
pipe_buf_mark_unmergeable(obuf); |
aadd06e5c
|
1662 1663 |
if (obuf->len > len) obuf->len = len; |
aadd06e5c
|
1664 1665 |
ret += obuf->len; len -= obuf->len; |
8cefc107c
|
1666 1667 1668 1669 |
o_head++; opipe->head = o_head; i_tail++; |
aadd06e5c
|
1670 |
} while (len); |
70524490e
|
1671 |
|
61e0d47c3
|
1672 1673 |
pipe_unlock(ipipe); pipe_unlock(opipe); |
70524490e
|
1674 |
|
aadd06e5c
|
1675 1676 1677 |
/* * If we put data in the output pipe, wakeup any potential readers. */ |
825cdcb1a
|
1678 1679 |
if (ret > 0) wakeup_pipe_readers(opipe); |
70524490e
|
1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 |
return ret; } /* * This is a tee(1) implementation that works on pipes. It doesn't copy * any data, it simply references the 'in' pages on the 'out' pipe. * The 'flags' used are the SPLICE_F_* variants, currently the only * applicable one is SPLICE_F_NONBLOCK. */ static long do_tee(struct file *in, struct file *out, size_t len, unsigned int flags) { |
71993e62a
|
1693 1694 |
struct pipe_inode_info *ipipe = get_pipe_info(in); struct pipe_inode_info *opipe = get_pipe_info(out); |
aadd06e5c
|
1695 |
int ret = -EINVAL; |
70524490e
|
1696 1697 |
/* |
aadd06e5c
|
1698 1699 |
* Duplicate the contents of ipipe to opipe without actually * copying the data. |
70524490e
|
1700 |
*/ |
aadd06e5c
|
1701 |
if (ipipe && opipe && ipipe != opipe) { |
ee5e00119
|
1702 1703 |
if ((in->f_flags | out->f_flags) & O_NONBLOCK) flags |= SPLICE_F_NONBLOCK; |
aadd06e5c
|
1704 1705 1706 1707 |
/* * Keep going, unless we encounter an error. The ipipe/opipe * ordering doesn't really matter. */ |
7c77f0b3f
|
1708 |
ret = ipipe_prep(ipipe, flags); |
aadd06e5c
|
1709 |
if (!ret) { |
7c77f0b3f
|
1710 |
ret = opipe_prep(opipe, flags); |
02cf01aea
|
1711 |
if (!ret) |
aadd06e5c
|
1712 |
ret = link_pipe(ipipe, opipe, len, flags); |
aadd06e5c
|
1713 1714 |
} } |
70524490e
|
1715 |
|
aadd06e5c
|
1716 |
return ret; |
70524490e
|
1717 |
} |
836f92adf
|
1718 |
SYSCALL_DEFINE4(tee, int, fdin, int, fdout, size_t, len, unsigned int, flags) |
70524490e
|
1719 |
{ |
2903ff019
|
1720 1721 |
struct fd in; int error; |
70524490e
|
1722 |
|
3d6ea290f
|
1723 1724 |
if (unlikely(flags & ~SPLICE_F_ALL)) return -EINVAL; |
70524490e
|
1725 1726 1727 1728 |
if (unlikely(!len)) return 0; error = -EBADF; |
2903ff019
|
1729 1730 1731 1732 1733 1734 1735 1736 1737 |
in = fdget(fdin); if (in.file) { if (in.file->f_mode & FMODE_READ) { struct fd out = fdget(fdout); if (out.file) { if (out.file->f_mode & FMODE_WRITE) error = do_tee(in.file, out.file, len, flags); fdput(out); |
70524490e
|
1738 1739 |
} } |
2903ff019
|
1740 |
fdput(in); |
70524490e
|
1741 1742 1743 1744 |
} return error; } |