Commit 003386fff3e02e51cea882e60f7d28290113964c
Exists in
master
and in
39 other branches
Merge branch 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/mszeredi/fuse
* 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/mszeredi/fuse: mm: export generic_pipe_buf_*() to modules fuse: support splice() reading from fuse device fuse: allow splice to move pages mm: export remove_from_page_cache() to modules mm: export lru_cache_add_*() to modules fuse: support splice() writing to fuse device fuse: get page reference for readpages fuse: use get_user_pages_fast() fuse: remove unneeded variable
Showing 7 changed files Side-by-side Diff
fs/fuse/dev.c
... | ... | @@ -16,6 +16,9 @@ |
16 | 16 | #include <linux/pagemap.h> |
17 | 17 | #include <linux/file.h> |
18 | 18 | #include <linux/slab.h> |
19 | +#include <linux/pipe_fs_i.h> | |
20 | +#include <linux/swap.h> | |
21 | +#include <linux/splice.h> | |
19 | 22 | |
20 | 23 | MODULE_ALIAS_MISCDEV(FUSE_MINOR); |
21 | 24 | MODULE_ALIAS("devname:fuse"); |
... | ... | @@ -499,6 +502,9 @@ |
499 | 502 | int write; |
500 | 503 | struct fuse_req *req; |
501 | 504 | const struct iovec *iov; |
505 | + struct pipe_buffer *pipebufs; | |
506 | + struct pipe_buffer *currbuf; | |
507 | + struct pipe_inode_info *pipe; | |
502 | 508 | unsigned long nr_segs; |
503 | 509 | unsigned long seglen; |
504 | 510 | unsigned long addr; |
505 | 511 | |
506 | 512 | |
... | ... | @@ -506,16 +512,16 @@ |
506 | 512 | void *mapaddr; |
507 | 513 | void *buf; |
508 | 514 | unsigned len; |
515 | + unsigned move_pages:1; | |
509 | 516 | }; |
510 | 517 | |
511 | 518 | static void fuse_copy_init(struct fuse_copy_state *cs, struct fuse_conn *fc, |
512 | - int write, struct fuse_req *req, | |
519 | + int write, | |
513 | 520 | const struct iovec *iov, unsigned long nr_segs) |
514 | 521 | { |
515 | 522 | memset(cs, 0, sizeof(*cs)); |
516 | 523 | cs->fc = fc; |
517 | 524 | cs->write = write; |
518 | - cs->req = req; | |
519 | 525 | cs->iov = iov; |
520 | 526 | cs->nr_segs = nr_segs; |
521 | 527 | } |
... | ... | @@ -523,7 +529,18 @@ |
523 | 529 | /* Unmap and put previous page of userspace buffer */ |
524 | 530 | static void fuse_copy_finish(struct fuse_copy_state *cs) |
525 | 531 | { |
526 | - if (cs->mapaddr) { | |
532 | + if (cs->currbuf) { | |
533 | + struct pipe_buffer *buf = cs->currbuf; | |
534 | + | |
535 | + if (!cs->write) { | |
536 | + buf->ops->unmap(cs->pipe, buf, cs->mapaddr); | |
537 | + } else { | |
538 | + kunmap_atomic(cs->mapaddr, KM_USER0); | |
539 | + buf->len = PAGE_SIZE - cs->len; | |
540 | + } | |
541 | + cs->currbuf = NULL; | |
542 | + cs->mapaddr = NULL; | |
543 | + } else if (cs->mapaddr) { | |
527 | 544 | kunmap_atomic(cs->mapaddr, KM_USER0); |
528 | 545 | if (cs->write) { |
529 | 546 | flush_dcache_page(cs->pg); |
530 | 547 | |
... | ... | @@ -545,26 +562,61 @@ |
545 | 562 | |
546 | 563 | unlock_request(cs->fc, cs->req); |
547 | 564 | fuse_copy_finish(cs); |
548 | - if (!cs->seglen) { | |
549 | - BUG_ON(!cs->nr_segs); | |
550 | - cs->seglen = cs->iov[0].iov_len; | |
551 | - cs->addr = (unsigned long) cs->iov[0].iov_base; | |
552 | - cs->iov++; | |
553 | - cs->nr_segs--; | |
565 | + if (cs->pipebufs) { | |
566 | + struct pipe_buffer *buf = cs->pipebufs; | |
567 | + | |
568 | + if (!cs->write) { | |
569 | + err = buf->ops->confirm(cs->pipe, buf); | |
570 | + if (err) | |
571 | + return err; | |
572 | + | |
573 | + BUG_ON(!cs->nr_segs); | |
574 | + cs->currbuf = buf; | |
575 | + cs->mapaddr = buf->ops->map(cs->pipe, buf, 1); | |
576 | + cs->len = buf->len; | |
577 | + cs->buf = cs->mapaddr + buf->offset; | |
578 | + cs->pipebufs++; | |
579 | + cs->nr_segs--; | |
580 | + } else { | |
581 | + struct page *page; | |
582 | + | |
583 | + if (cs->nr_segs == cs->pipe->buffers) | |
584 | + return -EIO; | |
585 | + | |
586 | + page = alloc_page(GFP_HIGHUSER); | |
587 | + if (!page) | |
588 | + return -ENOMEM; | |
589 | + | |
590 | + buf->page = page; | |
591 | + buf->offset = 0; | |
592 | + buf->len = 0; | |
593 | + | |
594 | + cs->currbuf = buf; | |
595 | + cs->mapaddr = kmap_atomic(page, KM_USER0); | |
596 | + cs->buf = cs->mapaddr; | |
597 | + cs->len = PAGE_SIZE; | |
598 | + cs->pipebufs++; | |
599 | + cs->nr_segs++; | |
600 | + } | |
601 | + } else { | |
602 | + if (!cs->seglen) { | |
603 | + BUG_ON(!cs->nr_segs); | |
604 | + cs->seglen = cs->iov[0].iov_len; | |
605 | + cs->addr = (unsigned long) cs->iov[0].iov_base; | |
606 | + cs->iov++; | |
607 | + cs->nr_segs--; | |
608 | + } | |
609 | + err = get_user_pages_fast(cs->addr, 1, cs->write, &cs->pg); | |
610 | + if (err < 0) | |
611 | + return err; | |
612 | + BUG_ON(err != 1); | |
613 | + offset = cs->addr % PAGE_SIZE; | |
614 | + cs->mapaddr = kmap_atomic(cs->pg, KM_USER0); | |
615 | + cs->buf = cs->mapaddr + offset; | |
616 | + cs->len = min(PAGE_SIZE - offset, cs->seglen); | |
617 | + cs->seglen -= cs->len; | |
618 | + cs->addr += cs->len; | |
554 | 619 | } |
555 | - down_read(¤t->mm->mmap_sem); | |
556 | - err = get_user_pages(current, current->mm, cs->addr, 1, cs->write, 0, | |
557 | - &cs->pg, NULL); | |
558 | - up_read(¤t->mm->mmap_sem); | |
559 | - if (err < 0) | |
560 | - return err; | |
561 | - BUG_ON(err != 1); | |
562 | - offset = cs->addr % PAGE_SIZE; | |
563 | - cs->mapaddr = kmap_atomic(cs->pg, KM_USER0); | |
564 | - cs->buf = cs->mapaddr + offset; | |
565 | - cs->len = min(PAGE_SIZE - offset, cs->seglen); | |
566 | - cs->seglen -= cs->len; | |
567 | - cs->addr += cs->len; | |
568 | 620 | |
569 | 621 | return lock_request(cs->fc, cs->req); |
570 | 622 | } |
571 | 623 | |
572 | 624 | |
573 | 625 | |
... | ... | @@ -586,23 +638,178 @@ |
586 | 638 | return ncpy; |
587 | 639 | } |
588 | 640 | |
641 | +static int fuse_check_page(struct page *page) | |
642 | +{ | |
643 | + if (page_mapcount(page) || | |
644 | + page->mapping != NULL || | |
645 | + page_count(page) != 1 || | |
646 | + (page->flags & PAGE_FLAGS_CHECK_AT_PREP & | |
647 | + ~(1 << PG_locked | | |
648 | + 1 << PG_referenced | | |
649 | + 1 << PG_uptodate | | |
650 | + 1 << PG_lru | | |
651 | + 1 << PG_active | | |
652 | + 1 << PG_reclaim))) { | |
653 | + printk(KERN_WARNING "fuse: trying to steal weird page\n"); | |
654 | + printk(KERN_WARNING " page=%p index=%li flags=%08lx, count=%i, mapcount=%i, mapping=%p\n", page, page->index, page->flags, page_count(page), page_mapcount(page), page->mapping); | |
655 | + return 1; | |
656 | + } | |
657 | + return 0; | |
658 | +} | |
659 | + | |
660 | +static int fuse_try_move_page(struct fuse_copy_state *cs, struct page **pagep) | |
661 | +{ | |
662 | + int err; | |
663 | + struct page *oldpage = *pagep; | |
664 | + struct page *newpage; | |
665 | + struct pipe_buffer *buf = cs->pipebufs; | |
666 | + struct address_space *mapping; | |
667 | + pgoff_t index; | |
668 | + | |
669 | + unlock_request(cs->fc, cs->req); | |
670 | + fuse_copy_finish(cs); | |
671 | + | |
672 | + err = buf->ops->confirm(cs->pipe, buf); | |
673 | + if (err) | |
674 | + return err; | |
675 | + | |
676 | + BUG_ON(!cs->nr_segs); | |
677 | + cs->currbuf = buf; | |
678 | + cs->len = buf->len; | |
679 | + cs->pipebufs++; | |
680 | + cs->nr_segs--; | |
681 | + | |
682 | + if (cs->len != PAGE_SIZE) | |
683 | + goto out_fallback; | |
684 | + | |
685 | + if (buf->ops->steal(cs->pipe, buf) != 0) | |
686 | + goto out_fallback; | |
687 | + | |
688 | + newpage = buf->page; | |
689 | + | |
690 | + if (WARN_ON(!PageUptodate(newpage))) | |
691 | + return -EIO; | |
692 | + | |
693 | + ClearPageMappedToDisk(newpage); | |
694 | + | |
695 | + if (fuse_check_page(newpage) != 0) | |
696 | + goto out_fallback_unlock; | |
697 | + | |
698 | + mapping = oldpage->mapping; | |
699 | + index = oldpage->index; | |
700 | + | |
701 | + /* | |
702 | + * This is a new and locked page, it shouldn't be mapped or | |
703 | + * have any special flags on it | |
704 | + */ | |
705 | + if (WARN_ON(page_mapped(oldpage))) | |
706 | + goto out_fallback_unlock; | |
707 | + if (WARN_ON(page_has_private(oldpage))) | |
708 | + goto out_fallback_unlock; | |
709 | + if (WARN_ON(PageDirty(oldpage) || PageWriteback(oldpage))) | |
710 | + goto out_fallback_unlock; | |
711 | + if (WARN_ON(PageMlocked(oldpage))) | |
712 | + goto out_fallback_unlock; | |
713 | + | |
714 | + remove_from_page_cache(oldpage); | |
715 | + page_cache_release(oldpage); | |
716 | + | |
717 | + err = add_to_page_cache_locked(newpage, mapping, index, GFP_KERNEL); | |
718 | + if (err) { | |
719 | + printk(KERN_WARNING "fuse_try_move_page: failed to add page"); | |
720 | + goto out_fallback_unlock; | |
721 | + } | |
722 | + page_cache_get(newpage); | |
723 | + | |
724 | + if (!(buf->flags & PIPE_BUF_FLAG_LRU)) | |
725 | + lru_cache_add_file(newpage); | |
726 | + | |
727 | + err = 0; | |
728 | + spin_lock(&cs->fc->lock); | |
729 | + if (cs->req->aborted) | |
730 | + err = -ENOENT; | |
731 | + else | |
732 | + *pagep = newpage; | |
733 | + spin_unlock(&cs->fc->lock); | |
734 | + | |
735 | + if (err) { | |
736 | + unlock_page(newpage); | |
737 | + page_cache_release(newpage); | |
738 | + return err; | |
739 | + } | |
740 | + | |
741 | + unlock_page(oldpage); | |
742 | + page_cache_release(oldpage); | |
743 | + cs->len = 0; | |
744 | + | |
745 | + return 0; | |
746 | + | |
747 | +out_fallback_unlock: | |
748 | + unlock_page(newpage); | |
749 | +out_fallback: | |
750 | + cs->mapaddr = buf->ops->map(cs->pipe, buf, 1); | |
751 | + cs->buf = cs->mapaddr + buf->offset; | |
752 | + | |
753 | + err = lock_request(cs->fc, cs->req); | |
754 | + if (err) | |
755 | + return err; | |
756 | + | |
757 | + return 1; | |
758 | +} | |
759 | + | |
760 | +static int fuse_ref_page(struct fuse_copy_state *cs, struct page *page, | |
761 | + unsigned offset, unsigned count) | |
762 | +{ | |
763 | + struct pipe_buffer *buf; | |
764 | + | |
765 | + if (cs->nr_segs == cs->pipe->buffers) | |
766 | + return -EIO; | |
767 | + | |
768 | + unlock_request(cs->fc, cs->req); | |
769 | + fuse_copy_finish(cs); | |
770 | + | |
771 | + buf = cs->pipebufs; | |
772 | + page_cache_get(page); | |
773 | + buf->page = page; | |
774 | + buf->offset = offset; | |
775 | + buf->len = count; | |
776 | + | |
777 | + cs->pipebufs++; | |
778 | + cs->nr_segs++; | |
779 | + cs->len = 0; | |
780 | + | |
781 | + return 0; | |
782 | +} | |
783 | + | |
589 | 784 | /* |
590 | 785 | * Copy a page in the request to/from the userspace buffer. Must be |
591 | 786 | * done atomically |
592 | 787 | */ |
593 | -static int fuse_copy_page(struct fuse_copy_state *cs, struct page *page, | |
788 | +static int fuse_copy_page(struct fuse_copy_state *cs, struct page **pagep, | |
594 | 789 | unsigned offset, unsigned count, int zeroing) |
595 | 790 | { |
791 | + int err; | |
792 | + struct page *page = *pagep; | |
793 | + | |
596 | 794 | if (page && zeroing && count < PAGE_SIZE) { |
597 | 795 | void *mapaddr = kmap_atomic(page, KM_USER1); |
598 | 796 | memset(mapaddr, 0, PAGE_SIZE); |
599 | 797 | kunmap_atomic(mapaddr, KM_USER1); |
600 | 798 | } |
601 | 799 | while (count) { |
602 | - if (!cs->len) { | |
603 | - int err = fuse_copy_fill(cs); | |
604 | - if (err) | |
605 | - return err; | |
800 | + if (cs->write && cs->pipebufs && page) { | |
801 | + return fuse_ref_page(cs, page, offset, count); | |
802 | + } else if (!cs->len) { | |
803 | + if (cs->move_pages && page && | |
804 | + offset == 0 && count == PAGE_SIZE) { | |
805 | + err = fuse_try_move_page(cs, pagep); | |
806 | + if (err <= 0) | |
807 | + return err; | |
808 | + } else { | |
809 | + err = fuse_copy_fill(cs); | |
810 | + if (err) | |
811 | + return err; | |
812 | + } | |
606 | 813 | } |
607 | 814 | if (page) { |
608 | 815 | void *mapaddr = kmap_atomic(page, KM_USER1); |
... | ... | @@ -627,8 +834,10 @@ |
627 | 834 | unsigned count = min(nbytes, (unsigned) PAGE_SIZE - offset); |
628 | 835 | |
629 | 836 | for (i = 0; i < req->num_pages && (nbytes || zeroing); i++) { |
630 | - struct page *page = req->pages[i]; | |
631 | - int err = fuse_copy_page(cs, page, offset, count, zeroing); | |
837 | + int err; | |
838 | + | |
839 | + err = fuse_copy_page(cs, &req->pages[i], offset, count, | |
840 | + zeroing); | |
632 | 841 | if (err) |
633 | 842 | return err; |
634 | 843 | |
635 | 844 | |
... | ... | @@ -705,11 +914,10 @@ |
705 | 914 | * |
706 | 915 | * Called with fc->lock held, releases it |
707 | 916 | */ |
708 | -static int fuse_read_interrupt(struct fuse_conn *fc, struct fuse_req *req, | |
709 | - const struct iovec *iov, unsigned long nr_segs) | |
917 | +static int fuse_read_interrupt(struct fuse_conn *fc, struct fuse_copy_state *cs, | |
918 | + size_t nbytes, struct fuse_req *req) | |
710 | 919 | __releases(&fc->lock) |
711 | 920 | { |
712 | - struct fuse_copy_state cs; | |
713 | 921 | struct fuse_in_header ih; |
714 | 922 | struct fuse_interrupt_in arg; |
715 | 923 | unsigned reqsize = sizeof(ih) + sizeof(arg); |
716 | 924 | |
717 | 925 | |
... | ... | @@ -725,14 +933,13 @@ |
725 | 933 | arg.unique = req->in.h.unique; |
726 | 934 | |
727 | 935 | spin_unlock(&fc->lock); |
728 | - if (iov_length(iov, nr_segs) < reqsize) | |
936 | + if (nbytes < reqsize) | |
729 | 937 | return -EINVAL; |
730 | 938 | |
731 | - fuse_copy_init(&cs, fc, 1, NULL, iov, nr_segs); | |
732 | - err = fuse_copy_one(&cs, &ih, sizeof(ih)); | |
939 | + err = fuse_copy_one(cs, &ih, sizeof(ih)); | |
733 | 940 | if (!err) |
734 | - err = fuse_copy_one(&cs, &arg, sizeof(arg)); | |
735 | - fuse_copy_finish(&cs); | |
941 | + err = fuse_copy_one(cs, &arg, sizeof(arg)); | |
942 | + fuse_copy_finish(cs); | |
736 | 943 | |
737 | 944 | return err ? err : reqsize; |
738 | 945 | } |
739 | 946 | |
740 | 947 | |
... | ... | @@ -746,18 +953,13 @@ |
746 | 953 | * request_end(). Otherwise add it to the processing list, and set |
747 | 954 | * the 'sent' flag. |
748 | 955 | */ |
749 | -static ssize_t fuse_dev_read(struct kiocb *iocb, const struct iovec *iov, | |
750 | - unsigned long nr_segs, loff_t pos) | |
956 | +static ssize_t fuse_dev_do_read(struct fuse_conn *fc, struct file *file, | |
957 | + struct fuse_copy_state *cs, size_t nbytes) | |
751 | 958 | { |
752 | 959 | int err; |
753 | 960 | struct fuse_req *req; |
754 | 961 | struct fuse_in *in; |
755 | - struct fuse_copy_state cs; | |
756 | 962 | unsigned reqsize; |
757 | - struct file *file = iocb->ki_filp; | |
758 | - struct fuse_conn *fc = fuse_get_conn(file); | |
759 | - if (!fc) | |
760 | - return -EPERM; | |
761 | 963 | |
762 | 964 | restart: |
763 | 965 | spin_lock(&fc->lock); |
... | ... | @@ -777,7 +979,7 @@ |
777 | 979 | if (!list_empty(&fc->interrupts)) { |
778 | 980 | req = list_entry(fc->interrupts.next, struct fuse_req, |
779 | 981 | intr_entry); |
780 | - return fuse_read_interrupt(fc, req, iov, nr_segs); | |
982 | + return fuse_read_interrupt(fc, cs, nbytes, req); | |
781 | 983 | } |
782 | 984 | |
783 | 985 | req = list_entry(fc->pending.next, struct fuse_req, list); |
... | ... | @@ -787,7 +989,7 @@ |
787 | 989 | in = &req->in; |
788 | 990 | reqsize = in->h.len; |
789 | 991 | /* If request is too large, reply with an error and restart the read */ |
790 | - if (iov_length(iov, nr_segs) < reqsize) { | |
992 | + if (nbytes < reqsize) { | |
791 | 993 | req->out.h.error = -EIO; |
792 | 994 | /* SETXATTR is special, since it may contain too large data */ |
793 | 995 | if (in->h.opcode == FUSE_SETXATTR) |
794 | 996 | |
795 | 997 | |
... | ... | @@ -796,12 +998,12 @@ |
796 | 998 | goto restart; |
797 | 999 | } |
798 | 1000 | spin_unlock(&fc->lock); |
799 | - fuse_copy_init(&cs, fc, 1, req, iov, nr_segs); | |
800 | - err = fuse_copy_one(&cs, &in->h, sizeof(in->h)); | |
1001 | + cs->req = req; | |
1002 | + err = fuse_copy_one(cs, &in->h, sizeof(in->h)); | |
801 | 1003 | if (!err) |
802 | - err = fuse_copy_args(&cs, in->numargs, in->argpages, | |
1004 | + err = fuse_copy_args(cs, in->numargs, in->argpages, | |
803 | 1005 | (struct fuse_arg *) in->args, 0); |
804 | - fuse_copy_finish(&cs); | |
1006 | + fuse_copy_finish(cs); | |
805 | 1007 | spin_lock(&fc->lock); |
806 | 1008 | req->locked = 0; |
807 | 1009 | if (req->aborted) { |
... | ... | @@ -829,6 +1031,110 @@ |
829 | 1031 | return err; |
830 | 1032 | } |
831 | 1033 | |
1034 | +static ssize_t fuse_dev_read(struct kiocb *iocb, const struct iovec *iov, | |
1035 | + unsigned long nr_segs, loff_t pos) | |
1036 | +{ | |
1037 | + struct fuse_copy_state cs; | |
1038 | + struct file *file = iocb->ki_filp; | |
1039 | + struct fuse_conn *fc = fuse_get_conn(file); | |
1040 | + if (!fc) | |
1041 | + return -EPERM; | |
1042 | + | |
1043 | + fuse_copy_init(&cs, fc, 1, iov, nr_segs); | |
1044 | + | |
1045 | + return fuse_dev_do_read(fc, file, &cs, iov_length(iov, nr_segs)); | |
1046 | +} | |
1047 | + | |
1048 | +static int fuse_dev_pipe_buf_steal(struct pipe_inode_info *pipe, | |
1049 | + struct pipe_buffer *buf) | |
1050 | +{ | |
1051 | + return 1; | |
1052 | +} | |
1053 | + | |
1054 | +static const struct pipe_buf_operations fuse_dev_pipe_buf_ops = { | |
1055 | + .can_merge = 0, | |
1056 | + .map = generic_pipe_buf_map, | |
1057 | + .unmap = generic_pipe_buf_unmap, | |
1058 | + .confirm = generic_pipe_buf_confirm, | |
1059 | + .release = generic_pipe_buf_release, | |
1060 | + .steal = fuse_dev_pipe_buf_steal, | |
1061 | + .get = generic_pipe_buf_get, | |
1062 | +}; | |
1063 | + | |
1064 | +static ssize_t fuse_dev_splice_read(struct file *in, loff_t *ppos, | |
1065 | + struct pipe_inode_info *pipe, | |
1066 | + size_t len, unsigned int flags) | |
1067 | +{ | |
1068 | + int ret; | |
1069 | + int page_nr = 0; | |
1070 | + int do_wakeup = 0; | |
1071 | + struct pipe_buffer *bufs; | |
1072 | + struct fuse_copy_state cs; | |
1073 | + struct fuse_conn *fc = fuse_get_conn(in); | |
1074 | + if (!fc) | |
1075 | + return -EPERM; | |
1076 | + | |
1077 | + bufs = kmalloc(pipe->buffers * sizeof (struct pipe_buffer), GFP_KERNEL); | |
1078 | + if (!bufs) | |
1079 | + return -ENOMEM; | |
1080 | + | |
1081 | + fuse_copy_init(&cs, fc, 1, NULL, 0); | |
1082 | + cs.pipebufs = bufs; | |
1083 | + cs.pipe = pipe; | |
1084 | + ret = fuse_dev_do_read(fc, in, &cs, len); | |
1085 | + if (ret < 0) | |
1086 | + goto out; | |
1087 | + | |
1088 | + ret = 0; | |
1089 | + pipe_lock(pipe); | |
1090 | + | |
1091 | + if (!pipe->readers) { | |
1092 | + send_sig(SIGPIPE, current, 0); | |
1093 | + if (!ret) | |
1094 | + ret = -EPIPE; | |
1095 | + goto out_unlock; | |
1096 | + } | |
1097 | + | |
1098 | + if (pipe->nrbufs + cs.nr_segs > pipe->buffers) { | |
1099 | + ret = -EIO; | |
1100 | + goto out_unlock; | |
1101 | + } | |
1102 | + | |
1103 | + while (page_nr < cs.nr_segs) { | |
1104 | + int newbuf = (pipe->curbuf + pipe->nrbufs) & (pipe->buffers - 1); | |
1105 | + struct pipe_buffer *buf = pipe->bufs + newbuf; | |
1106 | + | |
1107 | + buf->page = bufs[page_nr].page; | |
1108 | + buf->offset = bufs[page_nr].offset; | |
1109 | + buf->len = bufs[page_nr].len; | |
1110 | + buf->ops = &fuse_dev_pipe_buf_ops; | |
1111 | + | |
1112 | + pipe->nrbufs++; | |
1113 | + page_nr++; | |
1114 | + ret += buf->len; | |
1115 | + | |
1116 | + if (pipe->inode) | |
1117 | + do_wakeup = 1; | |
1118 | + } | |
1119 | + | |
1120 | +out_unlock: | |
1121 | + pipe_unlock(pipe); | |
1122 | + | |
1123 | + if (do_wakeup) { | |
1124 | + smp_mb(); | |
1125 | + if (waitqueue_active(&pipe->wait)) | |
1126 | + wake_up_interruptible(&pipe->wait); | |
1127 | + kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN); | |
1128 | + } | |
1129 | + | |
1130 | +out: | |
1131 | + for (; page_nr < cs.nr_segs; page_nr++) | |
1132 | + page_cache_release(bufs[page_nr].page); | |
1133 | + | |
1134 | + kfree(bufs); | |
1135 | + return ret; | |
1136 | +} | |
1137 | + | |
832 | 1138 | static int fuse_notify_poll(struct fuse_conn *fc, unsigned int size, |
833 | 1139 | struct fuse_copy_state *cs) |
834 | 1140 | { |
835 | 1141 | |
836 | 1142 | |
837 | 1143 | |
838 | 1144 | |
... | ... | @@ -988,23 +1294,17 @@ |
988 | 1294 | * it from the list and copy the rest of the buffer to the request. |
989 | 1295 | * The request is finished by calling request_end() |
990 | 1296 | */ |
991 | -static ssize_t fuse_dev_write(struct kiocb *iocb, const struct iovec *iov, | |
992 | - unsigned long nr_segs, loff_t pos) | |
1297 | +static ssize_t fuse_dev_do_write(struct fuse_conn *fc, | |
1298 | + struct fuse_copy_state *cs, size_t nbytes) | |
993 | 1299 | { |
994 | 1300 | int err; |
995 | - size_t nbytes = iov_length(iov, nr_segs); | |
996 | 1301 | struct fuse_req *req; |
997 | 1302 | struct fuse_out_header oh; |
998 | - struct fuse_copy_state cs; | |
999 | - struct fuse_conn *fc = fuse_get_conn(iocb->ki_filp); | |
1000 | - if (!fc) | |
1001 | - return -EPERM; | |
1002 | 1303 | |
1003 | - fuse_copy_init(&cs, fc, 0, NULL, iov, nr_segs); | |
1004 | 1304 | if (nbytes < sizeof(struct fuse_out_header)) |
1005 | 1305 | return -EINVAL; |
1006 | 1306 | |
1007 | - err = fuse_copy_one(&cs, &oh, sizeof(oh)); | |
1307 | + err = fuse_copy_one(cs, &oh, sizeof(oh)); | |
1008 | 1308 | if (err) |
1009 | 1309 | goto err_finish; |
1010 | 1310 | |
... | ... | @@ -1017,7 +1317,7 @@ |
1017 | 1317 | * and error contains notification code. |
1018 | 1318 | */ |
1019 | 1319 | if (!oh.unique) { |
1020 | - err = fuse_notify(fc, oh.error, nbytes - sizeof(oh), &cs); | |
1320 | + err = fuse_notify(fc, oh.error, nbytes - sizeof(oh), cs); | |
1021 | 1321 | return err ? err : nbytes; |
1022 | 1322 | } |
1023 | 1323 | |
... | ... | @@ -1036,7 +1336,7 @@ |
1036 | 1336 | |
1037 | 1337 | if (req->aborted) { |
1038 | 1338 | spin_unlock(&fc->lock); |
1039 | - fuse_copy_finish(&cs); | |
1339 | + fuse_copy_finish(cs); | |
1040 | 1340 | spin_lock(&fc->lock); |
1041 | 1341 | request_end(fc, req); |
1042 | 1342 | return -ENOENT; |
... | ... | @@ -1053,7 +1353,7 @@ |
1053 | 1353 | queue_interrupt(fc, req); |
1054 | 1354 | |
1055 | 1355 | spin_unlock(&fc->lock); |
1056 | - fuse_copy_finish(&cs); | |
1356 | + fuse_copy_finish(cs); | |
1057 | 1357 | return nbytes; |
1058 | 1358 | } |
1059 | 1359 | |
1060 | 1360 | |
... | ... | @@ -1061,11 +1361,13 @@ |
1061 | 1361 | list_move(&req->list, &fc->io); |
1062 | 1362 | req->out.h = oh; |
1063 | 1363 | req->locked = 1; |
1064 | - cs.req = req; | |
1364 | + cs->req = req; | |
1365 | + if (!req->out.page_replace) | |
1366 | + cs->move_pages = 0; | |
1065 | 1367 | spin_unlock(&fc->lock); |
1066 | 1368 | |
1067 | - err = copy_out_args(&cs, &req->out, nbytes); | |
1068 | - fuse_copy_finish(&cs); | |
1369 | + err = copy_out_args(cs, &req->out, nbytes); | |
1370 | + fuse_copy_finish(cs); | |
1069 | 1371 | |
1070 | 1372 | spin_lock(&fc->lock); |
1071 | 1373 | req->locked = 0; |
1072 | 1374 | |
... | ... | @@ -1081,10 +1383,101 @@ |
1081 | 1383 | err_unlock: |
1082 | 1384 | spin_unlock(&fc->lock); |
1083 | 1385 | err_finish: |
1084 | - fuse_copy_finish(&cs); | |
1386 | + fuse_copy_finish(cs); | |
1085 | 1387 | return err; |
1086 | 1388 | } |
1087 | 1389 | |
1390 | +static ssize_t fuse_dev_write(struct kiocb *iocb, const struct iovec *iov, | |
1391 | + unsigned long nr_segs, loff_t pos) | |
1392 | +{ | |
1393 | + struct fuse_copy_state cs; | |
1394 | + struct fuse_conn *fc = fuse_get_conn(iocb->ki_filp); | |
1395 | + if (!fc) | |
1396 | + return -EPERM; | |
1397 | + | |
1398 | + fuse_copy_init(&cs, fc, 0, iov, nr_segs); | |
1399 | + | |
1400 | + return fuse_dev_do_write(fc, &cs, iov_length(iov, nr_segs)); | |
1401 | +} | |
1402 | + | |
1403 | +static ssize_t fuse_dev_splice_write(struct pipe_inode_info *pipe, | |
1404 | + struct file *out, loff_t *ppos, | |
1405 | + size_t len, unsigned int flags) | |
1406 | +{ | |
1407 | + unsigned nbuf; | |
1408 | + unsigned idx; | |
1409 | + struct pipe_buffer *bufs; | |
1410 | + struct fuse_copy_state cs; | |
1411 | + struct fuse_conn *fc; | |
1412 | + size_t rem; | |
1413 | + ssize_t ret; | |
1414 | + | |
1415 | + fc = fuse_get_conn(out); | |
1416 | + if (!fc) | |
1417 | + return -EPERM; | |
1418 | + | |
1419 | + bufs = kmalloc(pipe->buffers * sizeof (struct pipe_buffer), GFP_KERNEL); | |
1420 | + if (!bufs) | |
1421 | + return -ENOMEM; | |
1422 | + | |
1423 | + pipe_lock(pipe); | |
1424 | + nbuf = 0; | |
1425 | + rem = 0; | |
1426 | + for (idx = 0; idx < pipe->nrbufs && rem < len; idx++) | |
1427 | + rem += pipe->bufs[(pipe->curbuf + idx) & (pipe->buffers - 1)].len; | |
1428 | + | |
1429 | + ret = -EINVAL; | |
1430 | + if (rem < len) { | |
1431 | + pipe_unlock(pipe); | |
1432 | + goto out; | |
1433 | + } | |
1434 | + | |
1435 | + rem = len; | |
1436 | + while (rem) { | |
1437 | + struct pipe_buffer *ibuf; | |
1438 | + struct pipe_buffer *obuf; | |
1439 | + | |
1440 | + BUG_ON(nbuf >= pipe->buffers); | |
1441 | + BUG_ON(!pipe->nrbufs); | |
1442 | + ibuf = &pipe->bufs[pipe->curbuf]; | |
1443 | + obuf = &bufs[nbuf]; | |
1444 | + | |
1445 | + if (rem >= ibuf->len) { | |
1446 | + *obuf = *ibuf; | |
1447 | + ibuf->ops = NULL; | |
1448 | + pipe->curbuf = (pipe->curbuf + 1) & (pipe->buffers - 1); | |
1449 | + pipe->nrbufs--; | |
1450 | + } else { | |
1451 | + ibuf->ops->get(pipe, ibuf); | |
1452 | + *obuf = *ibuf; | |
1453 | + obuf->flags &= ~PIPE_BUF_FLAG_GIFT; | |
1454 | + obuf->len = rem; | |
1455 | + ibuf->offset += obuf->len; | |
1456 | + ibuf->len -= obuf->len; | |
1457 | + } | |
1458 | + nbuf++; | |
1459 | + rem -= obuf->len; | |
1460 | + } | |
1461 | + pipe_unlock(pipe); | |
1462 | + | |
1463 | + fuse_copy_init(&cs, fc, 0, NULL, nbuf); | |
1464 | + cs.pipebufs = bufs; | |
1465 | + cs.pipe = pipe; | |
1466 | + | |
1467 | + if (flags & SPLICE_F_MOVE) | |
1468 | + cs.move_pages = 1; | |
1469 | + | |
1470 | + ret = fuse_dev_do_write(fc, &cs, len); | |
1471 | + | |
1472 | + for (idx = 0; idx < nbuf; idx++) { | |
1473 | + struct pipe_buffer *buf = &bufs[idx]; | |
1474 | + buf->ops->release(pipe, buf); | |
1475 | + } | |
1476 | +out: | |
1477 | + kfree(bufs); | |
1478 | + return ret; | |
1479 | +} | |
1480 | + | |
1088 | 1481 | static unsigned fuse_dev_poll(struct file *file, poll_table *wait) |
1089 | 1482 | { |
1090 | 1483 | unsigned mask = POLLOUT | POLLWRNORM; |
1091 | 1484 | |
... | ... | @@ -1226,8 +1619,10 @@ |
1226 | 1619 | .llseek = no_llseek, |
1227 | 1620 | .read = do_sync_read, |
1228 | 1621 | .aio_read = fuse_dev_read, |
1622 | + .splice_read = fuse_dev_splice_read, | |
1229 | 1623 | .write = do_sync_write, |
1230 | 1624 | .aio_write = fuse_dev_write, |
1625 | + .splice_write = fuse_dev_splice_write, | |
1231 | 1626 | .poll = fuse_dev_poll, |
1232 | 1627 | .release = fuse_dev_release, |
1233 | 1628 | .fasync = fuse_dev_fasync, |
fs/fuse/file.c
... | ... | @@ -516,18 +516,27 @@ |
516 | 516 | int i; |
517 | 517 | size_t count = req->misc.read.in.size; |
518 | 518 | size_t num_read = req->out.args[0].size; |
519 | - struct inode *inode = req->pages[0]->mapping->host; | |
519 | + struct address_space *mapping = NULL; | |
520 | 520 | |
521 | - /* | |
522 | - * Short read means EOF. If file size is larger, truncate it | |
523 | - */ | |
524 | - if (!req->out.h.error && num_read < count) { | |
525 | - loff_t pos = page_offset(req->pages[0]) + num_read; | |
526 | - fuse_read_update_size(inode, pos, req->misc.read.attr_ver); | |
527 | - } | |
521 | + for (i = 0; mapping == NULL && i < req->num_pages; i++) | |
522 | + mapping = req->pages[i]->mapping; | |
528 | 523 | |
529 | - fuse_invalidate_attr(inode); /* atime changed */ | |
524 | + if (mapping) { | |
525 | + struct inode *inode = mapping->host; | |
530 | 526 | |
527 | + /* | |
528 | + * Short read means EOF. If file size is larger, truncate it | |
529 | + */ | |
530 | + if (!req->out.h.error && num_read < count) { | |
531 | + loff_t pos; | |
532 | + | |
533 | + pos = page_offset(req->pages[0]) + num_read; | |
534 | + fuse_read_update_size(inode, pos, | |
535 | + req->misc.read.attr_ver); | |
536 | + } | |
537 | + fuse_invalidate_attr(inode); /* atime changed */ | |
538 | + } | |
539 | + | |
531 | 540 | for (i = 0; i < req->num_pages; i++) { |
532 | 541 | struct page *page = req->pages[i]; |
533 | 542 | if (!req->out.h.error) |
... | ... | @@ -535,6 +544,7 @@ |
535 | 544 | else |
536 | 545 | SetPageError(page); |
537 | 546 | unlock_page(page); |
547 | + page_cache_release(page); | |
538 | 548 | } |
539 | 549 | if (req->ff) |
540 | 550 | fuse_file_put(req->ff); |
... | ... | @@ -549,6 +559,7 @@ |
549 | 559 | |
550 | 560 | req->out.argpages = 1; |
551 | 561 | req->out.page_zeroing = 1; |
562 | + req->out.page_replace = 1; | |
552 | 563 | fuse_read_fill(req, file, pos, count, FUSE_READ); |
553 | 564 | req->misc.read.attr_ver = fuse_get_attr_version(fc); |
554 | 565 | if (fc->async_read) { |
... | ... | @@ -588,6 +599,7 @@ |
588 | 599 | return PTR_ERR(req); |
589 | 600 | } |
590 | 601 | } |
602 | + page_cache_get(page); | |
591 | 603 | req->pages[req->num_pages] = page; |
592 | 604 | req->num_pages++; |
593 | 605 | return 0; |
... | ... | @@ -993,10 +1005,7 @@ |
993 | 1005 | nbytes = min_t(size_t, nbytes, FUSE_MAX_PAGES_PER_REQ << PAGE_SHIFT); |
994 | 1006 | npages = (nbytes + offset + PAGE_SIZE - 1) >> PAGE_SHIFT; |
995 | 1007 | npages = clamp(npages, 1, FUSE_MAX_PAGES_PER_REQ); |
996 | - down_read(¤t->mm->mmap_sem); | |
997 | - npages = get_user_pages(current, current->mm, user_addr, npages, !write, | |
998 | - 0, req->pages, NULL); | |
999 | - up_read(¤t->mm->mmap_sem); | |
1008 | + npages = get_user_pages_fast(user_addr, npages, !write, req->pages); | |
1000 | 1009 | if (npages < 0) |
1001 | 1010 | return npages; |
1002 | 1011 | |
1003 | 1012 | |
... | ... | @@ -1579,9 +1588,9 @@ |
1579 | 1588 | while (iov_iter_count(&ii)) { |
1580 | 1589 | struct page *page = pages[page_idx++]; |
1581 | 1590 | size_t todo = min_t(size_t, PAGE_SIZE, iov_iter_count(&ii)); |
1582 | - void *kaddr, *map; | |
1591 | + void *kaddr; | |
1583 | 1592 | |
1584 | - kaddr = map = kmap(page); | |
1593 | + kaddr = kmap(page); | |
1585 | 1594 | |
1586 | 1595 | while (todo) { |
1587 | 1596 | char __user *uaddr = ii.iov->iov_base + ii.iov_offset; |
fs/fuse/fuse_i.h
fs/pipe.c
... | ... | @@ -230,6 +230,7 @@ |
230 | 230 | |
231 | 231 | return kmap(buf->page); |
232 | 232 | } |
233 | +EXPORT_SYMBOL(generic_pipe_buf_map); | |
233 | 234 | |
234 | 235 | /** |
235 | 236 | * generic_pipe_buf_unmap - unmap a previously mapped pipe buffer |
... | ... | @@ -249,6 +250,7 @@ |
249 | 250 | } else |
250 | 251 | kunmap(buf->page); |
251 | 252 | } |
253 | +EXPORT_SYMBOL(generic_pipe_buf_unmap); | |
252 | 254 | |
253 | 255 | /** |
254 | 256 | * generic_pipe_buf_steal - attempt to take ownership of a &pipe_buffer |
... | ... | @@ -279,6 +281,7 @@ |
279 | 281 | |
280 | 282 | return 1; |
281 | 283 | } |
284 | +EXPORT_SYMBOL(generic_pipe_buf_steal); | |
282 | 285 | |
283 | 286 | /** |
284 | 287 | * generic_pipe_buf_get - get a reference to a &struct pipe_buffer |
... | ... | @@ -294,6 +297,7 @@ |
294 | 297 | { |
295 | 298 | page_cache_get(buf->page); |
296 | 299 | } |
300 | +EXPORT_SYMBOL(generic_pipe_buf_get); | |
297 | 301 | |
298 | 302 | /** |
299 | 303 | * generic_pipe_buf_confirm - verify contents of the pipe buffer |
... | ... | @@ -309,6 +313,7 @@ |
309 | 313 | { |
310 | 314 | return 0; |
311 | 315 | } |
316 | +EXPORT_SYMBOL(generic_pipe_buf_confirm); | |
312 | 317 | |
313 | 318 | /** |
314 | 319 | * generic_pipe_buf_release - put a reference to a &struct pipe_buffer |
... | ... | @@ -323,6 +328,7 @@ |
323 | 328 | { |
324 | 329 | page_cache_release(buf->page); |
325 | 330 | } |
331 | +EXPORT_SYMBOL(generic_pipe_buf_release); | |
326 | 332 | |
327 | 333 | static const struct pipe_buf_operations anon_pipe_buf_ops = { |
328 | 334 | .can_merge = 1, |
include/linux/fuse.h
... | ... | @@ -34,6 +34,9 @@ |
34 | 34 | * 7.13 |
35 | 35 | * - make max number of background requests and congestion threshold |
36 | 36 | * tunables |
37 | + * | |
38 | + * 7.14 | |
39 | + * - add splice support to fuse device | |
37 | 40 | */ |
38 | 41 | |
39 | 42 | #ifndef _LINUX_FUSE_H |
... | ... | @@ -65,7 +68,7 @@ |
65 | 68 | #define FUSE_KERNEL_VERSION 7 |
66 | 69 | |
67 | 70 | /** Minor version number of this interface */ |
68 | -#define FUSE_KERNEL_MINOR_VERSION 13 | |
71 | +#define FUSE_KERNEL_MINOR_VERSION 14 | |
69 | 72 | |
70 | 73 | /** The node ID of the root inode */ |
71 | 74 | #define FUSE_ROOT_ID 1 |
mm/filemap.c