Commit 003386fff3e02e51cea882e60f7d28290113964c

Authored by Linus Torvalds

Merge branch 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/mszeredi/fuse

* 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/mszeredi/fuse:
  mm: export generic_pipe_buf_*() to modules
  fuse: support splice() reading from fuse device
  fuse: allow splice to move pages
  mm: export remove_from_page_cache() to modules
  mm: export lru_cache_add_*() to modules
  fuse: support splice() writing to fuse device
  fuse: get page reference for readpages
  fuse: use get_user_pages_fast()
  fuse: remove unneeded variable

Showing 7 changed files Side-by-side Diff

... ... @@ -16,6 +16,9 @@
16 16 #include <linux/pagemap.h>
17 17 #include <linux/file.h>
18 18 #include <linux/slab.h>
  19 +#include <linux/pipe_fs_i.h>
  20 +#include <linux/swap.h>
  21 +#include <linux/splice.h>
19 22  
20 23 MODULE_ALIAS_MISCDEV(FUSE_MINOR);
21 24 MODULE_ALIAS("devname:fuse");
... ... @@ -499,6 +502,9 @@
499 502 int write;
500 503 struct fuse_req *req;
501 504 const struct iovec *iov;
  505 + struct pipe_buffer *pipebufs;
  506 + struct pipe_buffer *currbuf;
  507 + struct pipe_inode_info *pipe;
502 508 unsigned long nr_segs;
503 509 unsigned long seglen;
504 510 unsigned long addr;
505 511  
506 512  
... ... @@ -506,16 +512,16 @@
506 512 void *mapaddr;
507 513 void *buf;
508 514 unsigned len;
  515 + unsigned move_pages:1;
509 516 };
510 517  
511 518 static void fuse_copy_init(struct fuse_copy_state *cs, struct fuse_conn *fc,
512   - int write, struct fuse_req *req,
  519 + int write,
513 520 const struct iovec *iov, unsigned long nr_segs)
514 521 {
515 522 memset(cs, 0, sizeof(*cs));
516 523 cs->fc = fc;
517 524 cs->write = write;
518   - cs->req = req;
519 525 cs->iov = iov;
520 526 cs->nr_segs = nr_segs;
521 527 }
... ... @@ -523,7 +529,18 @@
523 529 /* Unmap and put previous page of userspace buffer */
524 530 static void fuse_copy_finish(struct fuse_copy_state *cs)
525 531 {
526   - if (cs->mapaddr) {
  532 + if (cs->currbuf) {
  533 + struct pipe_buffer *buf = cs->currbuf;
  534 +
  535 + if (!cs->write) {
  536 + buf->ops->unmap(cs->pipe, buf, cs->mapaddr);
  537 + } else {
  538 + kunmap_atomic(cs->mapaddr, KM_USER0);
  539 + buf->len = PAGE_SIZE - cs->len;
  540 + }
  541 + cs->currbuf = NULL;
  542 + cs->mapaddr = NULL;
  543 + } else if (cs->mapaddr) {
527 544 kunmap_atomic(cs->mapaddr, KM_USER0);
528 545 if (cs->write) {
529 546 flush_dcache_page(cs->pg);
530 547  
... ... @@ -545,26 +562,61 @@
545 562  
546 563 unlock_request(cs->fc, cs->req);
547 564 fuse_copy_finish(cs);
548   - if (!cs->seglen) {
549   - BUG_ON(!cs->nr_segs);
550   - cs->seglen = cs->iov[0].iov_len;
551   - cs->addr = (unsigned long) cs->iov[0].iov_base;
552   - cs->iov++;
553   - cs->nr_segs--;
  565 + if (cs->pipebufs) {
  566 + struct pipe_buffer *buf = cs->pipebufs;
  567 +
  568 + if (!cs->write) {
  569 + err = buf->ops->confirm(cs->pipe, buf);
  570 + if (err)
  571 + return err;
  572 +
  573 + BUG_ON(!cs->nr_segs);
  574 + cs->currbuf = buf;
  575 + cs->mapaddr = buf->ops->map(cs->pipe, buf, 1);
  576 + cs->len = buf->len;
  577 + cs->buf = cs->mapaddr + buf->offset;
  578 + cs->pipebufs++;
  579 + cs->nr_segs--;
  580 + } else {
  581 + struct page *page;
  582 +
  583 + if (cs->nr_segs == cs->pipe->buffers)
  584 + return -EIO;
  585 +
  586 + page = alloc_page(GFP_HIGHUSER);
  587 + if (!page)
  588 + return -ENOMEM;
  589 +
  590 + buf->page = page;
  591 + buf->offset = 0;
  592 + buf->len = 0;
  593 +
  594 + cs->currbuf = buf;
  595 + cs->mapaddr = kmap_atomic(page, KM_USER0);
  596 + cs->buf = cs->mapaddr;
  597 + cs->len = PAGE_SIZE;
  598 + cs->pipebufs++;
  599 + cs->nr_segs++;
  600 + }
  601 + } else {
  602 + if (!cs->seglen) {
  603 + BUG_ON(!cs->nr_segs);
  604 + cs->seglen = cs->iov[0].iov_len;
  605 + cs->addr = (unsigned long) cs->iov[0].iov_base;
  606 + cs->iov++;
  607 + cs->nr_segs--;
  608 + }
  609 + err = get_user_pages_fast(cs->addr, 1, cs->write, &cs->pg);
  610 + if (err < 0)
  611 + return err;
  612 + BUG_ON(err != 1);
  613 + offset = cs->addr % PAGE_SIZE;
  614 + cs->mapaddr = kmap_atomic(cs->pg, KM_USER0);
  615 + cs->buf = cs->mapaddr + offset;
  616 + cs->len = min(PAGE_SIZE - offset, cs->seglen);
  617 + cs->seglen -= cs->len;
  618 + cs->addr += cs->len;
554 619 }
555   - down_read(&current->mm->mmap_sem);
556   - err = get_user_pages(current, current->mm, cs->addr, 1, cs->write, 0,
557   - &cs->pg, NULL);
558   - up_read(&current->mm->mmap_sem);
559   - if (err < 0)
560   - return err;
561   - BUG_ON(err != 1);
562   - offset = cs->addr % PAGE_SIZE;
563   - cs->mapaddr = kmap_atomic(cs->pg, KM_USER0);
564   - cs->buf = cs->mapaddr + offset;
565   - cs->len = min(PAGE_SIZE - offset, cs->seglen);
566   - cs->seglen -= cs->len;
567   - cs->addr += cs->len;
568 620  
569 621 return lock_request(cs->fc, cs->req);
570 622 }
571 623  
572 624  
573 625  
... ... @@ -586,23 +638,178 @@
586 638 return ncpy;
587 639 }
588 640  
  641 +static int fuse_check_page(struct page *page)
  642 +{
  643 + if (page_mapcount(page) ||
  644 + page->mapping != NULL ||
  645 + page_count(page) != 1 ||
  646 + (page->flags & PAGE_FLAGS_CHECK_AT_PREP &
  647 + ~(1 << PG_locked |
  648 + 1 << PG_referenced |
  649 + 1 << PG_uptodate |
  650 + 1 << PG_lru |
  651 + 1 << PG_active |
  652 + 1 << PG_reclaim))) {
  653 + printk(KERN_WARNING "fuse: trying to steal weird page\n");
  654 + printk(KERN_WARNING " page=%p index=%li flags=%08lx, count=%i, mapcount=%i, mapping=%p\n", page, page->index, page->flags, page_count(page), page_mapcount(page), page->mapping);
  655 + return 1;
  656 + }
  657 + return 0;
  658 +}
  659 +
  660 +static int fuse_try_move_page(struct fuse_copy_state *cs, struct page **pagep)
  661 +{
  662 + int err;
  663 + struct page *oldpage = *pagep;
  664 + struct page *newpage;
  665 + struct pipe_buffer *buf = cs->pipebufs;
  666 + struct address_space *mapping;
  667 + pgoff_t index;
  668 +
  669 + unlock_request(cs->fc, cs->req);
  670 + fuse_copy_finish(cs);
  671 +
  672 + err = buf->ops->confirm(cs->pipe, buf);
  673 + if (err)
  674 + return err;
  675 +
  676 + BUG_ON(!cs->nr_segs);
  677 + cs->currbuf = buf;
  678 + cs->len = buf->len;
  679 + cs->pipebufs++;
  680 + cs->nr_segs--;
  681 +
  682 + if (cs->len != PAGE_SIZE)
  683 + goto out_fallback;
  684 +
  685 + if (buf->ops->steal(cs->pipe, buf) != 0)
  686 + goto out_fallback;
  687 +
  688 + newpage = buf->page;
  689 +
  690 + if (WARN_ON(!PageUptodate(newpage)))
  691 + return -EIO;
  692 +
  693 + ClearPageMappedToDisk(newpage);
  694 +
  695 + if (fuse_check_page(newpage) != 0)
  696 + goto out_fallback_unlock;
  697 +
  698 + mapping = oldpage->mapping;
  699 + index = oldpage->index;
  700 +
  701 + /*
  702 + * This is a new and locked page, it shouldn't be mapped or
  703 + * have any special flags on it
  704 + */
  705 + if (WARN_ON(page_mapped(oldpage)))
  706 + goto out_fallback_unlock;
  707 + if (WARN_ON(page_has_private(oldpage)))
  708 + goto out_fallback_unlock;
  709 + if (WARN_ON(PageDirty(oldpage) || PageWriteback(oldpage)))
  710 + goto out_fallback_unlock;
  711 + if (WARN_ON(PageMlocked(oldpage)))
  712 + goto out_fallback_unlock;
  713 +
  714 + remove_from_page_cache(oldpage);
  715 + page_cache_release(oldpage);
  716 +
  717 + err = add_to_page_cache_locked(newpage, mapping, index, GFP_KERNEL);
  718 + if (err) {
  719 + printk(KERN_WARNING "fuse_try_move_page: failed to add page");
  720 + goto out_fallback_unlock;
  721 + }
  722 + page_cache_get(newpage);
  723 +
  724 + if (!(buf->flags & PIPE_BUF_FLAG_LRU))
  725 + lru_cache_add_file(newpage);
  726 +
  727 + err = 0;
  728 + spin_lock(&cs->fc->lock);
  729 + if (cs->req->aborted)
  730 + err = -ENOENT;
  731 + else
  732 + *pagep = newpage;
  733 + spin_unlock(&cs->fc->lock);
  734 +
  735 + if (err) {
  736 + unlock_page(newpage);
  737 + page_cache_release(newpage);
  738 + return err;
  739 + }
  740 +
  741 + unlock_page(oldpage);
  742 + page_cache_release(oldpage);
  743 + cs->len = 0;
  744 +
  745 + return 0;
  746 +
  747 +out_fallback_unlock:
  748 + unlock_page(newpage);
  749 +out_fallback:
  750 + cs->mapaddr = buf->ops->map(cs->pipe, buf, 1);
  751 + cs->buf = cs->mapaddr + buf->offset;
  752 +
  753 + err = lock_request(cs->fc, cs->req);
  754 + if (err)
  755 + return err;
  756 +
  757 + return 1;
  758 +}
  759 +
  760 +static int fuse_ref_page(struct fuse_copy_state *cs, struct page *page,
  761 + unsigned offset, unsigned count)
  762 +{
  763 + struct pipe_buffer *buf;
  764 +
  765 + if (cs->nr_segs == cs->pipe->buffers)
  766 + return -EIO;
  767 +
  768 + unlock_request(cs->fc, cs->req);
  769 + fuse_copy_finish(cs);
  770 +
  771 + buf = cs->pipebufs;
  772 + page_cache_get(page);
  773 + buf->page = page;
  774 + buf->offset = offset;
  775 + buf->len = count;
  776 +
  777 + cs->pipebufs++;
  778 + cs->nr_segs++;
  779 + cs->len = 0;
  780 +
  781 + return 0;
  782 +}
  783 +
589 784 /*
590 785 * Copy a page in the request to/from the userspace buffer. Must be
591 786 * done atomically
592 787 */
593   -static int fuse_copy_page(struct fuse_copy_state *cs, struct page *page,
  788 +static int fuse_copy_page(struct fuse_copy_state *cs, struct page **pagep,
594 789 unsigned offset, unsigned count, int zeroing)
595 790 {
  791 + int err;
  792 + struct page *page = *pagep;
  793 +
596 794 if (page && zeroing && count < PAGE_SIZE) {
597 795 void *mapaddr = kmap_atomic(page, KM_USER1);
598 796 memset(mapaddr, 0, PAGE_SIZE);
599 797 kunmap_atomic(mapaddr, KM_USER1);
600 798 }
601 799 while (count) {
602   - if (!cs->len) {
603   - int err = fuse_copy_fill(cs);
604   - if (err)
605   - return err;
  800 + if (cs->write && cs->pipebufs && page) {
  801 + return fuse_ref_page(cs, page, offset, count);
  802 + } else if (!cs->len) {
  803 + if (cs->move_pages && page &&
  804 + offset == 0 && count == PAGE_SIZE) {
  805 + err = fuse_try_move_page(cs, pagep);
  806 + if (err <= 0)
  807 + return err;
  808 + } else {
  809 + err = fuse_copy_fill(cs);
  810 + if (err)
  811 + return err;
  812 + }
606 813 }
607 814 if (page) {
608 815 void *mapaddr = kmap_atomic(page, KM_USER1);
... ... @@ -627,8 +834,10 @@
627 834 unsigned count = min(nbytes, (unsigned) PAGE_SIZE - offset);
628 835  
629 836 for (i = 0; i < req->num_pages && (nbytes || zeroing); i++) {
630   - struct page *page = req->pages[i];
631   - int err = fuse_copy_page(cs, page, offset, count, zeroing);
  837 + int err;
  838 +
  839 + err = fuse_copy_page(cs, &req->pages[i], offset, count,
  840 + zeroing);
632 841 if (err)
633 842 return err;
634 843  
635 844  
... ... @@ -705,11 +914,10 @@
705 914 *
706 915 * Called with fc->lock held, releases it
707 916 */
708   -static int fuse_read_interrupt(struct fuse_conn *fc, struct fuse_req *req,
709   - const struct iovec *iov, unsigned long nr_segs)
  917 +static int fuse_read_interrupt(struct fuse_conn *fc, struct fuse_copy_state *cs,
  918 + size_t nbytes, struct fuse_req *req)
710 919 __releases(&fc->lock)
711 920 {
712   - struct fuse_copy_state cs;
713 921 struct fuse_in_header ih;
714 922 struct fuse_interrupt_in arg;
715 923 unsigned reqsize = sizeof(ih) + sizeof(arg);
716 924  
717 925  
... ... @@ -725,14 +933,13 @@
725 933 arg.unique = req->in.h.unique;
726 934  
727 935 spin_unlock(&fc->lock);
728   - if (iov_length(iov, nr_segs) < reqsize)
  936 + if (nbytes < reqsize)
729 937 return -EINVAL;
730 938  
731   - fuse_copy_init(&cs, fc, 1, NULL, iov, nr_segs);
732   - err = fuse_copy_one(&cs, &ih, sizeof(ih));
  939 + err = fuse_copy_one(cs, &ih, sizeof(ih));
733 940 if (!err)
734   - err = fuse_copy_one(&cs, &arg, sizeof(arg));
735   - fuse_copy_finish(&cs);
  941 + err = fuse_copy_one(cs, &arg, sizeof(arg));
  942 + fuse_copy_finish(cs);
736 943  
737 944 return err ? err : reqsize;
738 945 }
739 946  
740 947  
... ... @@ -746,18 +953,13 @@
746 953 * request_end(). Otherwise add it to the processing list, and set
747 954 * the 'sent' flag.
748 955 */
749   -static ssize_t fuse_dev_read(struct kiocb *iocb, const struct iovec *iov,
750   - unsigned long nr_segs, loff_t pos)
  956 +static ssize_t fuse_dev_do_read(struct fuse_conn *fc, struct file *file,
  957 + struct fuse_copy_state *cs, size_t nbytes)
751 958 {
752 959 int err;
753 960 struct fuse_req *req;
754 961 struct fuse_in *in;
755   - struct fuse_copy_state cs;
756 962 unsigned reqsize;
757   - struct file *file = iocb->ki_filp;
758   - struct fuse_conn *fc = fuse_get_conn(file);
759   - if (!fc)
760   - return -EPERM;
761 963  
762 964 restart:
763 965 spin_lock(&fc->lock);
... ... @@ -777,7 +979,7 @@
777 979 if (!list_empty(&fc->interrupts)) {
778 980 req = list_entry(fc->interrupts.next, struct fuse_req,
779 981 intr_entry);
780   - return fuse_read_interrupt(fc, req, iov, nr_segs);
  982 + return fuse_read_interrupt(fc, cs, nbytes, req);
781 983 }
782 984  
783 985 req = list_entry(fc->pending.next, struct fuse_req, list);
... ... @@ -787,7 +989,7 @@
787 989 in = &req->in;
788 990 reqsize = in->h.len;
789 991 /* If request is too large, reply with an error and restart the read */
790   - if (iov_length(iov, nr_segs) < reqsize) {
  992 + if (nbytes < reqsize) {
791 993 req->out.h.error = -EIO;
792 994 /* SETXATTR is special, since it may contain too large data */
793 995 if (in->h.opcode == FUSE_SETXATTR)
794 996  
795 997  
... ... @@ -796,12 +998,12 @@
796 998 goto restart;
797 999 }
798 1000 spin_unlock(&fc->lock);
799   - fuse_copy_init(&cs, fc, 1, req, iov, nr_segs);
800   - err = fuse_copy_one(&cs, &in->h, sizeof(in->h));
  1001 + cs->req = req;
  1002 + err = fuse_copy_one(cs, &in->h, sizeof(in->h));
801 1003 if (!err)
802   - err = fuse_copy_args(&cs, in->numargs, in->argpages,
  1004 + err = fuse_copy_args(cs, in->numargs, in->argpages,
803 1005 (struct fuse_arg *) in->args, 0);
804   - fuse_copy_finish(&cs);
  1006 + fuse_copy_finish(cs);
805 1007 spin_lock(&fc->lock);
806 1008 req->locked = 0;
807 1009 if (req->aborted) {
... ... @@ -829,6 +1031,110 @@
829 1031 return err;
830 1032 }
831 1033  
  1034 +static ssize_t fuse_dev_read(struct kiocb *iocb, const struct iovec *iov,
  1035 + unsigned long nr_segs, loff_t pos)
  1036 +{
  1037 + struct fuse_copy_state cs;
  1038 + struct file *file = iocb->ki_filp;
  1039 + struct fuse_conn *fc = fuse_get_conn(file);
  1040 + if (!fc)
  1041 + return -EPERM;
  1042 +
  1043 + fuse_copy_init(&cs, fc, 1, iov, nr_segs);
  1044 +
  1045 + return fuse_dev_do_read(fc, file, &cs, iov_length(iov, nr_segs));
  1046 +}
  1047 +
  1048 +static int fuse_dev_pipe_buf_steal(struct pipe_inode_info *pipe,
  1049 + struct pipe_buffer *buf)
  1050 +{
  1051 + return 1;
  1052 +}
  1053 +
  1054 +static const struct pipe_buf_operations fuse_dev_pipe_buf_ops = {
  1055 + .can_merge = 0,
  1056 + .map = generic_pipe_buf_map,
  1057 + .unmap = generic_pipe_buf_unmap,
  1058 + .confirm = generic_pipe_buf_confirm,
  1059 + .release = generic_pipe_buf_release,
  1060 + .steal = fuse_dev_pipe_buf_steal,
  1061 + .get = generic_pipe_buf_get,
  1062 +};
  1063 +
  1064 +static ssize_t fuse_dev_splice_read(struct file *in, loff_t *ppos,
  1065 + struct pipe_inode_info *pipe,
  1066 + size_t len, unsigned int flags)
  1067 +{
  1068 + int ret;
  1069 + int page_nr = 0;
  1070 + int do_wakeup = 0;
  1071 + struct pipe_buffer *bufs;
  1072 + struct fuse_copy_state cs;
  1073 + struct fuse_conn *fc = fuse_get_conn(in);
  1074 + if (!fc)
  1075 + return -EPERM;
  1076 +
  1077 + bufs = kmalloc(pipe->buffers * sizeof (struct pipe_buffer), GFP_KERNEL);
  1078 + if (!bufs)
  1079 + return -ENOMEM;
  1080 +
  1081 + fuse_copy_init(&cs, fc, 1, NULL, 0);
  1082 + cs.pipebufs = bufs;
  1083 + cs.pipe = pipe;
  1084 + ret = fuse_dev_do_read(fc, in, &cs, len);
  1085 + if (ret < 0)
  1086 + goto out;
  1087 +
  1088 + ret = 0;
  1089 + pipe_lock(pipe);
  1090 +
  1091 + if (!pipe->readers) {
  1092 + send_sig(SIGPIPE, current, 0);
  1093 + if (!ret)
  1094 + ret = -EPIPE;
  1095 + goto out_unlock;
  1096 + }
  1097 +
  1098 + if (pipe->nrbufs + cs.nr_segs > pipe->buffers) {
  1099 + ret = -EIO;
  1100 + goto out_unlock;
  1101 + }
  1102 +
  1103 + while (page_nr < cs.nr_segs) {
  1104 + int newbuf = (pipe->curbuf + pipe->nrbufs) & (pipe->buffers - 1);
  1105 + struct pipe_buffer *buf = pipe->bufs + newbuf;
  1106 +
  1107 + buf->page = bufs[page_nr].page;
  1108 + buf->offset = bufs[page_nr].offset;
  1109 + buf->len = bufs[page_nr].len;
  1110 + buf->ops = &fuse_dev_pipe_buf_ops;
  1111 +
  1112 + pipe->nrbufs++;
  1113 + page_nr++;
  1114 + ret += buf->len;
  1115 +
  1116 + if (pipe->inode)
  1117 + do_wakeup = 1;
  1118 + }
  1119 +
  1120 +out_unlock:
  1121 + pipe_unlock(pipe);
  1122 +
  1123 + if (do_wakeup) {
  1124 + smp_mb();
  1125 + if (waitqueue_active(&pipe->wait))
  1126 + wake_up_interruptible(&pipe->wait);
  1127 + kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
  1128 + }
  1129 +
  1130 +out:
  1131 + for (; page_nr < cs.nr_segs; page_nr++)
  1132 + page_cache_release(bufs[page_nr].page);
  1133 +
  1134 + kfree(bufs);
  1135 + return ret;
  1136 +}
  1137 +
832 1138 static int fuse_notify_poll(struct fuse_conn *fc, unsigned int size,
833 1139 struct fuse_copy_state *cs)
834 1140 {
835 1141  
836 1142  
837 1143  
838 1144  
... ... @@ -988,23 +1294,17 @@
988 1294 * it from the list and copy the rest of the buffer to the request.
989 1295 * The request is finished by calling request_end()
990 1296 */
991   -static ssize_t fuse_dev_write(struct kiocb *iocb, const struct iovec *iov,
992   - unsigned long nr_segs, loff_t pos)
  1297 +static ssize_t fuse_dev_do_write(struct fuse_conn *fc,
  1298 + struct fuse_copy_state *cs, size_t nbytes)
993 1299 {
994 1300 int err;
995   - size_t nbytes = iov_length(iov, nr_segs);
996 1301 struct fuse_req *req;
997 1302 struct fuse_out_header oh;
998   - struct fuse_copy_state cs;
999   - struct fuse_conn *fc = fuse_get_conn(iocb->ki_filp);
1000   - if (!fc)
1001   - return -EPERM;
1002 1303  
1003   - fuse_copy_init(&cs, fc, 0, NULL, iov, nr_segs);
1004 1304 if (nbytes < sizeof(struct fuse_out_header))
1005 1305 return -EINVAL;
1006 1306  
1007   - err = fuse_copy_one(&cs, &oh, sizeof(oh));
  1307 + err = fuse_copy_one(cs, &oh, sizeof(oh));
1008 1308 if (err)
1009 1309 goto err_finish;
1010 1310  
... ... @@ -1017,7 +1317,7 @@
1017 1317 * and error contains notification code.
1018 1318 */
1019 1319 if (!oh.unique) {
1020   - err = fuse_notify(fc, oh.error, nbytes - sizeof(oh), &cs);
  1320 + err = fuse_notify(fc, oh.error, nbytes - sizeof(oh), cs);
1021 1321 return err ? err : nbytes;
1022 1322 }
1023 1323  
... ... @@ -1036,7 +1336,7 @@
1036 1336  
1037 1337 if (req->aborted) {
1038 1338 spin_unlock(&fc->lock);
1039   - fuse_copy_finish(&cs);
  1339 + fuse_copy_finish(cs);
1040 1340 spin_lock(&fc->lock);
1041 1341 request_end(fc, req);
1042 1342 return -ENOENT;
... ... @@ -1053,7 +1353,7 @@
1053 1353 queue_interrupt(fc, req);
1054 1354  
1055 1355 spin_unlock(&fc->lock);
1056   - fuse_copy_finish(&cs);
  1356 + fuse_copy_finish(cs);
1057 1357 return nbytes;
1058 1358 }
1059 1359  
1060 1360  
... ... @@ -1061,11 +1361,13 @@
1061 1361 list_move(&req->list, &fc->io);
1062 1362 req->out.h = oh;
1063 1363 req->locked = 1;
1064   - cs.req = req;
  1364 + cs->req = req;
  1365 + if (!req->out.page_replace)
  1366 + cs->move_pages = 0;
1065 1367 spin_unlock(&fc->lock);
1066 1368  
1067   - err = copy_out_args(&cs, &req->out, nbytes);
1068   - fuse_copy_finish(&cs);
  1369 + err = copy_out_args(cs, &req->out, nbytes);
  1370 + fuse_copy_finish(cs);
1069 1371  
1070 1372 spin_lock(&fc->lock);
1071 1373 req->locked = 0;
1072 1374  
... ... @@ -1081,10 +1383,101 @@
1081 1383 err_unlock:
1082 1384 spin_unlock(&fc->lock);
1083 1385 err_finish:
1084   - fuse_copy_finish(&cs);
  1386 + fuse_copy_finish(cs);
1085 1387 return err;
1086 1388 }
1087 1389  
  1390 +static ssize_t fuse_dev_write(struct kiocb *iocb, const struct iovec *iov,
  1391 + unsigned long nr_segs, loff_t pos)
  1392 +{
  1393 + struct fuse_copy_state cs;
  1394 + struct fuse_conn *fc = fuse_get_conn(iocb->ki_filp);
  1395 + if (!fc)
  1396 + return -EPERM;
  1397 +
  1398 + fuse_copy_init(&cs, fc, 0, iov, nr_segs);
  1399 +
  1400 + return fuse_dev_do_write(fc, &cs, iov_length(iov, nr_segs));
  1401 +}
  1402 +
  1403 +static ssize_t fuse_dev_splice_write(struct pipe_inode_info *pipe,
  1404 + struct file *out, loff_t *ppos,
  1405 + size_t len, unsigned int flags)
  1406 +{
  1407 + unsigned nbuf;
  1408 + unsigned idx;
  1409 + struct pipe_buffer *bufs;
  1410 + struct fuse_copy_state cs;
  1411 + struct fuse_conn *fc;
  1412 + size_t rem;
  1413 + ssize_t ret;
  1414 +
  1415 + fc = fuse_get_conn(out);
  1416 + if (!fc)
  1417 + return -EPERM;
  1418 +
  1419 + bufs = kmalloc(pipe->buffers * sizeof (struct pipe_buffer), GFP_KERNEL);
  1420 + if (!bufs)
  1421 + return -ENOMEM;
  1422 +
  1423 + pipe_lock(pipe);
  1424 + nbuf = 0;
  1425 + rem = 0;
  1426 + for (idx = 0; idx < pipe->nrbufs && rem < len; idx++)
  1427 + rem += pipe->bufs[(pipe->curbuf + idx) & (pipe->buffers - 1)].len;
  1428 +
  1429 + ret = -EINVAL;
  1430 + if (rem < len) {
  1431 + pipe_unlock(pipe);
  1432 + goto out;
  1433 + }
  1434 +
  1435 + rem = len;
  1436 + while (rem) {
  1437 + struct pipe_buffer *ibuf;
  1438 + struct pipe_buffer *obuf;
  1439 +
  1440 + BUG_ON(nbuf >= pipe->buffers);
  1441 + BUG_ON(!pipe->nrbufs);
  1442 + ibuf = &pipe->bufs[pipe->curbuf];
  1443 + obuf = &bufs[nbuf];
  1444 +
  1445 + if (rem >= ibuf->len) {
  1446 + *obuf = *ibuf;
  1447 + ibuf->ops = NULL;
  1448 + pipe->curbuf = (pipe->curbuf + 1) & (pipe->buffers - 1);
  1449 + pipe->nrbufs--;
  1450 + } else {
  1451 + ibuf->ops->get(pipe, ibuf);
  1452 + *obuf = *ibuf;
  1453 + obuf->flags &= ~PIPE_BUF_FLAG_GIFT;
  1454 + obuf->len = rem;
  1455 + ibuf->offset += obuf->len;
  1456 + ibuf->len -= obuf->len;
  1457 + }
  1458 + nbuf++;
  1459 + rem -= obuf->len;
  1460 + }
  1461 + pipe_unlock(pipe);
  1462 +
  1463 + fuse_copy_init(&cs, fc, 0, NULL, nbuf);
  1464 + cs.pipebufs = bufs;
  1465 + cs.pipe = pipe;
  1466 +
  1467 + if (flags & SPLICE_F_MOVE)
  1468 + cs.move_pages = 1;
  1469 +
  1470 + ret = fuse_dev_do_write(fc, &cs, len);
  1471 +
  1472 + for (idx = 0; idx < nbuf; idx++) {
  1473 + struct pipe_buffer *buf = &bufs[idx];
  1474 + buf->ops->release(pipe, buf);
  1475 + }
  1476 +out:
  1477 + kfree(bufs);
  1478 + return ret;
  1479 +}
  1480 +
1088 1481 static unsigned fuse_dev_poll(struct file *file, poll_table *wait)
1089 1482 {
1090 1483 unsigned mask = POLLOUT | POLLWRNORM;
1091 1484  
... ... @@ -1226,8 +1619,10 @@
1226 1619 .llseek = no_llseek,
1227 1620 .read = do_sync_read,
1228 1621 .aio_read = fuse_dev_read,
  1622 + .splice_read = fuse_dev_splice_read,
1229 1623 .write = do_sync_write,
1230 1624 .aio_write = fuse_dev_write,
  1625 + .splice_write = fuse_dev_splice_write,
1231 1626 .poll = fuse_dev_poll,
1232 1627 .release = fuse_dev_release,
1233 1628 .fasync = fuse_dev_fasync,
... ... @@ -516,18 +516,27 @@
516 516 int i;
517 517 size_t count = req->misc.read.in.size;
518 518 size_t num_read = req->out.args[0].size;
519   - struct inode *inode = req->pages[0]->mapping->host;
  519 + struct address_space *mapping = NULL;
520 520  
521   - /*
522   - * Short read means EOF. If file size is larger, truncate it
523   - */
524   - if (!req->out.h.error && num_read < count) {
525   - loff_t pos = page_offset(req->pages[0]) + num_read;
526   - fuse_read_update_size(inode, pos, req->misc.read.attr_ver);
527   - }
  521 + for (i = 0; mapping == NULL && i < req->num_pages; i++)
  522 + mapping = req->pages[i]->mapping;
528 523  
529   - fuse_invalidate_attr(inode); /* atime changed */
  524 + if (mapping) {
  525 + struct inode *inode = mapping->host;
530 526  
  527 + /*
  528 + * Short read means EOF. If file size is larger, truncate it
  529 + */
  530 + if (!req->out.h.error && num_read < count) {
  531 + loff_t pos;
  532 +
  533 + pos = page_offset(req->pages[0]) + num_read;
  534 + fuse_read_update_size(inode, pos,
  535 + req->misc.read.attr_ver);
  536 + }
  537 + fuse_invalidate_attr(inode); /* atime changed */
  538 + }
  539 +
531 540 for (i = 0; i < req->num_pages; i++) {
532 541 struct page *page = req->pages[i];
533 542 if (!req->out.h.error)
... ... @@ -535,6 +544,7 @@
535 544 else
536 545 SetPageError(page);
537 546 unlock_page(page);
  547 + page_cache_release(page);
538 548 }
539 549 if (req->ff)
540 550 fuse_file_put(req->ff);
... ... @@ -549,6 +559,7 @@
549 559  
550 560 req->out.argpages = 1;
551 561 req->out.page_zeroing = 1;
  562 + req->out.page_replace = 1;
552 563 fuse_read_fill(req, file, pos, count, FUSE_READ);
553 564 req->misc.read.attr_ver = fuse_get_attr_version(fc);
554 565 if (fc->async_read) {
... ... @@ -588,6 +599,7 @@
588 599 return PTR_ERR(req);
589 600 }
590 601 }
  602 + page_cache_get(page);
591 603 req->pages[req->num_pages] = page;
592 604 req->num_pages++;
593 605 return 0;
... ... @@ -993,10 +1005,7 @@
993 1005 nbytes = min_t(size_t, nbytes, FUSE_MAX_PAGES_PER_REQ << PAGE_SHIFT);
994 1006 npages = (nbytes + offset + PAGE_SIZE - 1) >> PAGE_SHIFT;
995 1007 npages = clamp(npages, 1, FUSE_MAX_PAGES_PER_REQ);
996   - down_read(&current->mm->mmap_sem);
997   - npages = get_user_pages(current, current->mm, user_addr, npages, !write,
998   - 0, req->pages, NULL);
999   - up_read(&current->mm->mmap_sem);
  1008 + npages = get_user_pages_fast(user_addr, npages, !write, req->pages);
1000 1009 if (npages < 0)
1001 1010 return npages;
1002 1011  
1003 1012  
... ... @@ -1579,9 +1588,9 @@
1579 1588 while (iov_iter_count(&ii)) {
1580 1589 struct page *page = pages[page_idx++];
1581 1590 size_t todo = min_t(size_t, PAGE_SIZE, iov_iter_count(&ii));
1582   - void *kaddr, *map;
  1591 + void *kaddr;
1583 1592  
1584   - kaddr = map = kmap(page);
  1593 + kaddr = kmap(page);
1585 1594  
1586 1595 while (todo) {
1587 1596 char __user *uaddr = ii.iov->iov_base + ii.iov_offset;
... ... @@ -177,6 +177,9 @@
177 177 /** Zero partially or not copied pages */
178 178 unsigned page_zeroing:1;
179 179  
  180 + /** Pages may be replaced with new ones */
  181 + unsigned page_replace:1;
  182 +
180 183 /** Number or arguments */
181 184 unsigned numargs;
182 185  
... ... @@ -230,6 +230,7 @@
230 230  
231 231 return kmap(buf->page);
232 232 }
  233 +EXPORT_SYMBOL(generic_pipe_buf_map);
233 234  
234 235 /**
235 236 * generic_pipe_buf_unmap - unmap a previously mapped pipe buffer
... ... @@ -249,6 +250,7 @@
249 250 } else
250 251 kunmap(buf->page);
251 252 }
  253 +EXPORT_SYMBOL(generic_pipe_buf_unmap);
252 254  
253 255 /**
254 256 * generic_pipe_buf_steal - attempt to take ownership of a &pipe_buffer
... ... @@ -279,6 +281,7 @@
279 281  
280 282 return 1;
281 283 }
  284 +EXPORT_SYMBOL(generic_pipe_buf_steal);
282 285  
283 286 /**
284 287 * generic_pipe_buf_get - get a reference to a &struct pipe_buffer
... ... @@ -294,6 +297,7 @@
294 297 {
295 298 page_cache_get(buf->page);
296 299 }
  300 +EXPORT_SYMBOL(generic_pipe_buf_get);
297 301  
298 302 /**
299 303 * generic_pipe_buf_confirm - verify contents of the pipe buffer
... ... @@ -309,6 +313,7 @@
309 313 {
310 314 return 0;
311 315 }
  316 +EXPORT_SYMBOL(generic_pipe_buf_confirm);
312 317  
313 318 /**
314 319 * generic_pipe_buf_release - put a reference to a &struct pipe_buffer
... ... @@ -323,6 +328,7 @@
323 328 {
324 329 page_cache_release(buf->page);
325 330 }
  331 +EXPORT_SYMBOL(generic_pipe_buf_release);
326 332  
327 333 static const struct pipe_buf_operations anon_pipe_buf_ops = {
328 334 .can_merge = 1,
include/linux/fuse.h
... ... @@ -34,6 +34,9 @@
34 34 * 7.13
35 35 * - make max number of background requests and congestion threshold
36 36 * tunables
  37 + *
  38 + * 7.14
  39 + * - add splice support to fuse device
37 40 */
38 41  
39 42 #ifndef _LINUX_FUSE_H
... ... @@ -65,7 +68,7 @@
65 68 #define FUSE_KERNEL_VERSION 7
66 69  
67 70 /** Minor version number of this interface */
68   -#define FUSE_KERNEL_MINOR_VERSION 13
  71 +#define FUSE_KERNEL_MINOR_VERSION 14
69 72  
70 73 /** The node ID of the root inode */
71 74 #define FUSE_ROOT_ID 1
... ... @@ -151,6 +151,7 @@
151 151 spin_unlock_irq(&mapping->tree_lock);
152 152 mem_cgroup_uncharge_cache_page(page);
153 153 }
  154 +EXPORT_SYMBOL(remove_from_page_cache);
154 155  
155 156 static int sync_page(void *word)
156 157 {
... ... @@ -224,6 +224,7 @@
224 224 ____pagevec_lru_add(pvec, lru);
225 225 put_cpu_var(lru_add_pvecs);
226 226 }
  227 +EXPORT_SYMBOL(__lru_cache_add);
227 228  
228 229 /**
229 230 * lru_cache_add_lru - add a page to a page list