/*
    FUSE: Filesystem in Userspace

    Copyright (C) 2001-2008  Miklos Szeredi <miklos@szeredi.hu>

  
    This program can be distributed under the terms of the GNU GPL.
    See the file COPYING.
  */
  
  #include "fuse_i.h"
  
  #include <linux/pagemap.h>
  #include <linux/slab.h>
  #include <linux/kernel.h>

  #include <linux/sched.h>

  #include <linux/module.h>

  #include <linux/compat.h>

  #include <linux/swap.h>

  #include <linux/falloc.h>

  #include <linux/uio.h>

  static const struct file_operations fuse_direct_io_file_operations;

  static int fuse_send_open(struct fuse_conn *fc, u64 nodeid, struct file *file,
  			  int opcode, struct fuse_open_out *outargp)

  {

  	struct fuse_open_in inarg;

  	FUSE_ARGS(args);

  
  	memset(&inarg, 0, sizeof(inarg));

  	inarg.flags = file->f_flags & ~(O_CREAT | O_EXCL | O_NOCTTY);
  	if (!fc->atomic_o_trunc)
  		inarg.flags &= ~O_TRUNC;

  	args.in.h.opcode = opcode;
  	args.in.h.nodeid = nodeid;
  	args.in.numargs = 1;
  	args.in.args[0].size = sizeof(inarg);
  	args.in.args[0].value = &inarg;
  	args.out.numargs = 1;
  	args.out.args[0].size = sizeof(*outargp);
  	args.out.args[0].value = outargp;

  	return fuse_simple_request(fc, &args);

  }

  struct fuse_file *fuse_file_alloc(struct fuse_conn *fc)

  {
  	struct fuse_file *ff;

  	ff = kzalloc(sizeof(struct fuse_file), GFP_KERNEL);

  	if (unlikely(!ff))
  		return NULL;

  	ff->fc = fc;

  	ff->reserved_req = fuse_request_alloc(0);

  	if (unlikely(!ff->reserved_req)) {
  		kfree(ff);
  		return NULL;

  	}

  
  	INIT_LIST_HEAD(&ff->write_entry);

  	refcount_set(&ff->count, 1);

  	RB_CLEAR_NODE(&ff->polled_node);
  	init_waitqueue_head(&ff->poll_wait);
  
  	spin_lock(&fc->lock);
  	ff->kh = ++fc->khctr;
  	spin_unlock(&fc->lock);

  	return ff;
  }
  
  void fuse_file_free(struct fuse_file *ff)
  {

  	fuse_request_free(ff->reserved_req);

  	kfree(ff);
  }

  static struct fuse_file *fuse_file_get(struct fuse_file *ff)

  {

  	refcount_inc(&ff->count);

  	return ff;
  }

  static void fuse_release_end(struct fuse_conn *fc, struct fuse_req *req)
  {

  	iput(req->misc.release.inode);

  }

  static void fuse_file_put(struct fuse_file *ff, bool sync)

  {

  	if (refcount_dec_and_test(&ff->count)) {

  		struct fuse_req *req = ff->reserved_req;

  		if (ff->fc->no_open) {
  			/*
  			 * Drop the release request when client does not
  			 * implement 'open'
  			 */

  			__clear_bit(FR_BACKGROUND, &req->flags);

  			iput(req->misc.release.inode);

  			fuse_put_request(ff->fc, req);
  		} else if (sync) {

  			__set_bit(FR_FORCE, &req->flags);

  			__clear_bit(FR_BACKGROUND, &req->flags);

  			fuse_request_send(ff->fc, req);

  			iput(req->misc.release.inode);

  			fuse_put_request(ff->fc, req);
  		} else {
  			req->end = fuse_release_end;

  			__set_bit(FR_BACKGROUND, &req->flags);

  			fuse_request_send_background(ff->fc, req);
  		}

  		kfree(ff);
  	}
  }

  int fuse_do_open(struct fuse_conn *fc, u64 nodeid, struct file *file,
  		 bool isdir)

  {

  	struct fuse_file *ff;

  	int opcode = isdir ? FUSE_OPENDIR : FUSE_OPEN;
  
  	ff = fuse_file_alloc(fc);
  	if (!ff)
  		return -ENOMEM;

  	ff->fh = 0;
  	ff->open_flags = FOPEN_KEEP_CACHE; /* Default for no-open */
  	if (!fc->no_open || isdir) {
  		struct fuse_open_out outarg;
  		int err;
  
  		err = fuse_send_open(fc, nodeid, file, opcode, &outarg);
  		if (!err) {
  			ff->fh = outarg.fh;
  			ff->open_flags = outarg.open_flags;
  
  		} else if (err != -ENOSYS || isdir) {
  			fuse_file_free(ff);
  			return err;
  		} else {
  			fc->no_open = 1;
  		}

  	}
  
  	if (isdir)

  		ff->open_flags &= ~FOPEN_DIRECT_IO;

  	ff->nodeid = nodeid;

  	file->private_data = ff;

  
  	return 0;
  }

  EXPORT_SYMBOL_GPL(fuse_do_open);

  static void fuse_link_write_file(struct file *file)
  {
  	struct inode *inode = file_inode(file);
  	struct fuse_conn *fc = get_fuse_conn(inode);
  	struct fuse_inode *fi = get_fuse_inode(inode);
  	struct fuse_file *ff = file->private_data;
  	/*
  	 * file may be written through mmap, so chain it onto the
  	 * inodes's write_file list
  	 */
  	spin_lock(&fc->lock);
  	if (list_empty(&ff->write_entry))
  		list_add(&ff->write_entry, &fi->write_files);
  	spin_unlock(&fc->lock);
  }

  void fuse_finish_open(struct inode *inode, struct file *file)

  {

  	struct fuse_file *ff = file->private_data;

  	struct fuse_conn *fc = get_fuse_conn(inode);

  
  	if (ff->open_flags & FOPEN_DIRECT_IO)

  		file->f_op = &fuse_direct_io_file_operations;

  	if (!(ff->open_flags & FOPEN_KEEP_CACHE))

  		invalidate_inode_pages2(inode->i_mapping);

  	if (ff->open_flags & FOPEN_NONSEEKABLE)

  		nonseekable_open(inode, file);

  	if (fc->atomic_o_trunc && (file->f_flags & O_TRUNC)) {
  		struct fuse_inode *fi = get_fuse_inode(inode);
  
  		spin_lock(&fc->lock);
  		fi->attr_version = ++fc->attr_version;
  		i_size_write(inode, 0);
  		spin_unlock(&fc->lock);
  		fuse_invalidate_attr(inode);

  		if (fc->writeback_cache)
  			file_update_time(file);

  	}

  	if ((file->f_mode & FMODE_WRITE) && fc->writeback_cache)
  		fuse_link_write_file(file);

  }

  int fuse_open_common(struct inode *inode, struct file *file, bool isdir)

  {

  	struct fuse_conn *fc = get_fuse_conn(inode);

  	int err;

  	bool lock_inode = (file->f_flags & O_TRUNC) &&
  			  fc->atomic_o_trunc &&
  			  fc->writeback_cache;

  
  	err = generic_file_open(inode, file);
  	if (err)
  		return err;

  	if (lock_inode)

  		inode_lock(inode);

  	err = fuse_do_open(fc, get_node_id(inode), file, isdir);

  	if (!err)
  		fuse_finish_open(inode, file);

  	if (lock_inode)

  		inode_unlock(inode);

  
  	return err;

  }

  static void fuse_prepare_release(struct fuse_file *ff, int flags, int opcode)

  {

  	struct fuse_conn *fc = ff->fc;

  	struct fuse_req *req = ff->reserved_req;

  	struct fuse_release_in *inarg = &req->misc.release.in;

  	spin_lock(&fc->lock);
  	list_del(&ff->write_entry);
  	if (!RB_EMPTY_NODE(&ff->polled_node))
  		rb_erase(&ff->polled_node, &fc->polled_files);
  	spin_unlock(&fc->lock);

  	wake_up_interruptible_all(&ff->poll_wait);

  	inarg->fh = ff->fh;

  	inarg->flags = flags;

  	req->in.h.opcode = opcode;

  	req->in.h.nodeid = ff->nodeid;

  	req->in.numargs = 1;
  	req->in.args[0].size = sizeof(struct fuse_release_in);
  	req->in.args[0].value = inarg;

  }

  void fuse_release_common(struct file *file, int opcode)

  {

  	struct fuse_file *ff = file->private_data;
  	struct fuse_req *req = ff->reserved_req;

  	fuse_prepare_release(ff, file->f_flags, opcode);

  	if (ff->flock) {
  		struct fuse_release_in *inarg = &req->misc.release.in;
  		inarg->release_flags |= FUSE_RELEASE_FLOCK_UNLOCK;
  		inarg->lock_owner = fuse_lock_owner_id(ff->fc,
  						       (fl_owner_t) file);
  	}

  	/* Hold inode until release is finished */
  	req->misc.release.inode = igrab(file_inode(file));

  	/*
  	 * Normally this will send the RELEASE request, however if
  	 * some asynchronous READ or WRITE requests are outstanding,
  	 * the sending will be delayed.

  	 *
  	 * Make the release synchronous if this is a fuseblk mount,
  	 * synchronous RELEASE is allowed (and desirable) in this case
  	 * because the server can be trusted not to screw up.

  	 */

  	fuse_file_put(ff, ff->fc->destroy_req != NULL);

  }

  static int fuse_open(struct inode *inode, struct file *file)
  {

  	return fuse_open_common(inode, file, false);

  }
  
  static int fuse_release(struct inode *inode, struct file *file)
  {

  	struct fuse_conn *fc = get_fuse_conn(inode);
  
  	/* see fuse_vma_close() for !writeback_cache case */
  	if (fc->writeback_cache)

  		write_inode_now(inode, 1);

  	fuse_release_common(file, FUSE_RELEASE);
  
  	/* return value is ignored by VFS */
  	return 0;
  }
  
  void fuse_sync_release(struct fuse_file *ff, int flags)
  {

  	WARN_ON(refcount_read(&ff->count) > 1);

  	fuse_prepare_release(ff, flags, FUSE_RELEASE);

  	/*
  	 * iput(NULL) is a no-op and since the refcount is 1 and everything's
  	 * synchronous, we are fine with not doing igrab() here"
  	 */
  	fuse_file_put(ff, true);

  }

  EXPORT_SYMBOL_GPL(fuse_sync_release);

  /*

   * Scramble the ID space with XTEA, so that the value of the files_struct
   * pointer is not exposed to userspace.

   */

  u64 fuse_lock_owner_id(struct fuse_conn *fc, fl_owner_t id)

  {

  	u32 *k = fc->scramble_key;
  	u64 v = (unsigned long) id;
  	u32 v0 = v;
  	u32 v1 = v >> 32;
  	u32 sum = 0;
  	int i;
  
  	for (i = 0; i < 32; i++) {
  		v0 += ((v1 << 4 ^ v1 >> 5) + v1) ^ (sum + k[sum & 3]);
  		sum += 0x9E3779B9;
  		v1 += ((v0 << 4 ^ v0 >> 5) + v0) ^ (sum + k[sum>>11 & 3]);
  	}
  
  	return (u64) v0 + ((u64) v1 << 32);

  }

  /*

   * Check if any page in a range is under writeback

   *
   * This is currently done by walking the list of writepage requests
   * for the inode, which can be pretty inefficient.
   */

  static bool fuse_range_is_writeback(struct inode *inode, pgoff_t idx_from,
  				   pgoff_t idx_to)

  {
  	struct fuse_conn *fc = get_fuse_conn(inode);
  	struct fuse_inode *fi = get_fuse_inode(inode);
  	struct fuse_req *req;
  	bool found = false;
  
  	spin_lock(&fc->lock);
  	list_for_each_entry(req, &fi->writepages, writepages_entry) {
  		pgoff_t curr_index;
  
  		BUG_ON(req->inode != inode);

  		curr_index = req->misc.write.in.offset >> PAGE_SHIFT;

  		if (idx_from < curr_index + req->num_pages &&
  		    curr_index <= idx_to) {

  			found = true;
  			break;
  		}
  	}
  	spin_unlock(&fc->lock);
  
  	return found;
  }

  static inline bool fuse_page_is_writeback(struct inode *inode, pgoff_t index)
  {
  	return fuse_range_is_writeback(inode, index, index);
  }

  /*
   * Wait for page writeback to be completed.
   *
   * Since fuse doesn't rely on the VM writeback tracking, this has to
   * use some other means.
   */
  static int fuse_wait_on_page_writeback(struct inode *inode, pgoff_t index)
  {
  	struct fuse_inode *fi = get_fuse_inode(inode);
  
  	wait_event(fi->page_waitq, !fuse_page_is_writeback(inode, index));
  	return 0;
  }

  /*
   * Wait for all pending writepages on the inode to finish.
   *
   * This is currently done by blocking further writes with FUSE_NOWRITE
   * and waiting for all sent writes to complete.
   *
   * This must be called under i_mutex, otherwise the FUSE_NOWRITE usage
   * could conflict with truncation.
   */
  static void fuse_sync_writes(struct inode *inode)
  {
  	fuse_set_nowrite(inode);
  	fuse_release_nowrite(inode);
  }

  static int fuse_flush(struct file *file, fl_owner_t id)

  {

  	struct inode *inode = file_inode(file);

  	struct fuse_conn *fc = get_fuse_conn(inode);
  	struct fuse_file *ff = file->private_data;
  	struct fuse_req *req;
  	struct fuse_flush_in inarg;
  	int err;

  	if (is_bad_inode(inode))
  		return -EIO;

  	if (fc->no_flush)
  		return 0;

  	err = write_inode_now(inode, 1);

  	if (err)
  		return err;

  	inode_lock(inode);

  	fuse_sync_writes(inode);

  	inode_unlock(inode);

  	err = filemap_check_errors(file->f_mapping);

  	if (err)
  		return err;

  	req = fuse_get_req_nofail_nopages(fc, file);

  	memset(&inarg, 0, sizeof(inarg));
  	inarg.fh = ff->fh;

  	inarg.lock_owner = fuse_lock_owner_id(fc, id);

  	req->in.h.opcode = FUSE_FLUSH;
  	req->in.h.nodeid = get_node_id(inode);

  	req->in.numargs = 1;
  	req->in.args[0].size = sizeof(inarg);
  	req->in.args[0].value = &inarg;

  	__set_bit(FR_FORCE, &req->flags);

  	fuse_request_send(fc, req);

  	err = req->out.h.error;
  	fuse_put_request(fc, req);
  	if (err == -ENOSYS) {
  		fc->no_flush = 1;
  		err = 0;
  	}
  	return err;
  }

  int fuse_fsync_common(struct file *file, loff_t start, loff_t end,
  		      int datasync, int isdir)

  {

  	struct inode *inode = file->f_mapping->host;

  	struct fuse_conn *fc = get_fuse_conn(inode);
  	struct fuse_file *ff = file->private_data;

  	FUSE_ARGS(args);

  	struct fuse_fsync_in inarg;
  	int err;

  	if (is_bad_inode(inode))
  		return -EIO;

  	inode_lock(inode);

  	/*
  	 * Start writeback against all dirty pages of the inode, then
  	 * wait for all outstanding writes, before sending the FSYNC
  	 * request.
  	 */

  	err = file_write_and_wait_range(file, start, end);

  	if (err)

  		goto out;

  
  	fuse_sync_writes(inode);

  
  	/*
  	 * Due to implementation of fuse writeback

  	 * file_write_and_wait_range() does not catch errors.

  	 * We have to do this directly after fuse_sync_writes()
  	 */

  	err = file_check_and_advance_wb_err(file);

  	if (err)
  		goto out;

  	err = sync_inode_metadata(inode, 1);
  	if (err)
  		goto out;

  	if ((!isdir && fc->no_fsync) || (isdir && fc->no_fsyncdir))
  		goto out;

  	memset(&inarg, 0, sizeof(inarg));
  	inarg.fh = ff->fh;
  	inarg.fsync_flags = datasync ? 1 : 0;

  	args.in.h.opcode = isdir ? FUSE_FSYNCDIR : FUSE_FSYNC;
  	args.in.h.nodeid = get_node_id(inode);
  	args.in.numargs = 1;
  	args.in.args[0].size = sizeof(inarg);
  	args.in.args[0].value = &inarg;
  	err = fuse_simple_request(fc, &args);

  	if (err == -ENOSYS) {

  		if (isdir)
  			fc->no_fsyncdir = 1;
  		else
  			fc->no_fsync = 1;

  		err = 0;
  	}

  out:

  	inode_unlock(inode);

  	return err;
  }

  static int fuse_fsync(struct file *file, loff_t start, loff_t end,
  		      int datasync)

  {

  	return fuse_fsync_common(file, start, end, datasync, 0);

  }

  void fuse_read_fill(struct fuse_req *req, struct file *file, loff_t pos,
  		    size_t count, int opcode)

  {

  	struct fuse_read_in *inarg = &req->misc.read.in;

  	struct fuse_file *ff = file->private_data;

  	inarg->fh = ff->fh;
  	inarg->offset = pos;
  	inarg->size = count;

  	inarg->flags = file->f_flags;

  	req->in.h.opcode = opcode;

  	req->in.h.nodeid = ff->nodeid;

  	req->in.numargs = 1;
  	req->in.args[0].size = sizeof(struct fuse_read_in);

  	req->in.args[0].value = inarg;

  	req->out.argvar = 1;
  	req->out.numargs = 1;
  	req->out.args[0].size = count;

  }

  static void fuse_release_user_pages(struct fuse_req *req, bool should_dirty)

  {
  	unsigned i;
  
  	for (i = 0; i < req->num_pages; i++) {
  		struct page *page = req->pages[i];

  		if (should_dirty)

  			set_page_dirty_lock(page);
  		put_page(page);
  	}
  }

  static void fuse_io_release(struct kref *kref)
  {
  	kfree(container_of(kref, struct fuse_io_priv, refcnt));
  }

  static ssize_t fuse_get_res_by_io(struct fuse_io_priv *io)
  {
  	if (io->err)
  		return io->err;
  
  	if (io->bytes >= 0 && io->write)
  		return -EIO;
  
  	return io->bytes < 0 ? io->size : io->bytes;
  }

  /**
   * In case of short read, the caller sets 'pos' to the position of
   * actual end of fuse request in IO request. Otherwise, if bytes_requested
   * == bytes_transferred or rw == WRITE, the caller sets 'pos' to -1.
   *
   * An example:
   * User requested DIO read of 64K. It was splitted into two 32K fuse requests,
   * both submitted asynchronously. The first of them was ACKed by userspace as
   * fully completed (req->out.args[0].size == 32K) resulting in pos == -1. The
   * second request was ACKed as short, e.g. only 1K was read, resulting in
   * pos == 33K.
   *
   * Thus, when all fuse requests are completed, the minimal non-negative 'pos'
   * will be equal to the length of the longest contiguous fragment of
   * transferred data starting from the beginning of IO request.
   */
  static void fuse_aio_complete(struct fuse_io_priv *io, int err, ssize_t pos)
  {
  	int left;
  
  	spin_lock(&io->lock);
  	if (err)
  		io->err = io->err ? : err;
  	else if (pos >= 0 && (io->bytes < 0 || pos < io->bytes))
  		io->bytes = pos;
  
  	left = --io->reqs;

  	if (!left && io->blocking)

  		complete(io->done);

  	spin_unlock(&io->lock);

  	if (!left && !io->blocking) {

  		ssize_t res = fuse_get_res_by_io(io);

  		if (res >= 0) {
  			struct inode *inode = file_inode(io->iocb->ki_filp);
  			struct fuse_conn *fc = get_fuse_conn(inode);
  			struct fuse_inode *fi = get_fuse_inode(inode);

  			spin_lock(&fc->lock);
  			fi->attr_version = ++fc->attr_version;
  			spin_unlock(&fc->lock);

  		}

  		io->iocb->ki_complete(io->iocb, res, 0);

  	}

  
  	kref_put(&io->refcnt, fuse_io_release);

  }
  
  static void fuse_aio_complete_req(struct fuse_conn *fc, struct fuse_req *req)
  {
  	struct fuse_io_priv *io = req->io;
  	ssize_t pos = -1;

  	fuse_release_user_pages(req, io->should_dirty);

  
  	if (io->write) {
  		if (req->misc.write.in.size != req->misc.write.out.size)
  			pos = req->misc.write.in.offset - io->offset +
  				req->misc.write.out.size;
  	} else {
  		if (req->misc.read.in.size != req->out.args[0].size)
  			pos = req->misc.read.in.offset - io->offset +
  				req->out.args[0].size;
  	}
  
  	fuse_aio_complete(io, req->out.h.error, pos);
  }
  
  static size_t fuse_async_req_send(struct fuse_conn *fc, struct fuse_req *req,
  		size_t num_bytes, struct fuse_io_priv *io)
  {
  	spin_lock(&io->lock);

  	kref_get(&io->refcnt);

  	io->size += num_bytes;
  	io->reqs++;
  	spin_unlock(&io->lock);
  
  	req->io = io;
  	req->end = fuse_aio_complete_req;

  	__fuse_get_request(req);

  	fuse_request_send_background(fc, req);
  
  	return num_bytes;
  }

  static size_t fuse_send_read(struct fuse_req *req, struct fuse_io_priv *io,

  			     loff_t pos, size_t count, fl_owner_t owner)

  {

  	struct file *file = io->iocb->ki_filp;

  	struct fuse_file *ff = file->private_data;
  	struct fuse_conn *fc = ff->fc;

  	fuse_read_fill(req, file, pos, count, FUSE_READ);

  	if (owner != NULL) {

  		struct fuse_read_in *inarg = &req->misc.read.in;

  
  		inarg->read_flags |= FUSE_READ_LOCKOWNER;
  		inarg->lock_owner = fuse_lock_owner_id(fc, owner);
  	}

  
  	if (io->async)
  		return fuse_async_req_send(fc, req, count, io);

  	fuse_request_send(fc, req);

  	return req->out.args[0].size;

  }

  static void fuse_read_update_size(struct inode *inode, loff_t size,
  				  u64 attr_ver)
  {
  	struct fuse_conn *fc = get_fuse_conn(inode);
  	struct fuse_inode *fi = get_fuse_inode(inode);
  
  	spin_lock(&fc->lock);

  	if (attr_ver == fi->attr_version && size < inode->i_size &&
  	    !test_bit(FUSE_I_SIZE_UNSTABLE, &fi->state)) {

  		fi->attr_version = ++fc->attr_version;
  		i_size_write(inode, size);
  	}
  	spin_unlock(&fc->lock);
  }

  static void fuse_short_read(struct fuse_req *req, struct inode *inode,
  			    u64 attr_ver)
  {
  	size_t num_read = req->out.args[0].size;

  	struct fuse_conn *fc = get_fuse_conn(inode);
  
  	if (fc->writeback_cache) {
  		/*
  		 * A hole in a file. Some data after the hole are in page cache,
  		 * but have not reached the client fs yet. So, the hole is not
  		 * present there.
  		 */
  		int i;

  		int start_idx = num_read >> PAGE_SHIFT;
  		size_t off = num_read & (PAGE_SIZE - 1);

  		for (i = start_idx; i < req->num_pages; i++) {

  			zero_user_segment(req->pages[i], off, PAGE_SIZE);

  			off = 0;
  		}
  	} else {
  		loff_t pos = page_offset(req->pages[0]) + num_read;
  		fuse_read_update_size(inode, pos, attr_ver);
  	}

  }

  static int fuse_do_readpage(struct file *file, struct page *page)

  {

  	struct kiocb iocb;
  	struct fuse_io_priv io;

  	struct inode *inode = page->mapping->host;
  	struct fuse_conn *fc = get_fuse_conn(inode);

  	struct fuse_req *req;

  	size_t num_read;
  	loff_t pos = page_offset(page);

  	size_t count = PAGE_SIZE;

  	u64 attr_ver;

  	int err;

  	/*

  	 * Page writeback can extend beyond the lifetime of the

  	 * page-cache page, so make sure we read a properly synced
  	 * page.
  	 */
  	fuse_wait_on_page_writeback(inode, page->index);

  	req = fuse_get_req(fc, 1);

  	if (IS_ERR(req))

  		return PTR_ERR(req);

  	attr_ver = fuse_get_attr_version(fc);

  	req->out.page_zeroing = 1;

  	req->out.argpages = 1;

  	req->num_pages = 1;
  	req->pages[0] = page;

  	req->page_descs[0].length = count;

  	init_sync_kiocb(&iocb, file);
  	io = (struct fuse_io_priv) FUSE_IO_PRIV_SYNC(&iocb);

  	num_read = fuse_send_read(req, &io, pos, count, NULL);

  	err = req->out.h.error;

  
  	if (!err) {
  		/*
  		 * Short read means EOF.  If file size is larger, truncate it
  		 */
  		if (num_read < count)

  			fuse_short_read(req, inode, attr_ver);

  		SetPageUptodate(page);

  	}

  	fuse_put_request(fc, req);

  
  	return err;
  }
  
  static int fuse_readpage(struct file *file, struct page *page)
  {
  	struct inode *inode = page->mapping->host;
  	int err;
  
  	err = -EIO;
  	if (is_bad_inode(inode))
  		goto out;
  
  	err = fuse_do_readpage(file, page);

  	fuse_invalidate_atime(inode);

   out:
  	unlock_page(page);
  	return err;
  }

  static void fuse_readpages_end(struct fuse_conn *fc, struct fuse_req *req)

  {

  	int i;

  	size_t count = req->misc.read.in.size;
  	size_t num_read = req->out.args[0].size;

  	struct address_space *mapping = NULL;

  	for (i = 0; mapping == NULL && i < req->num_pages; i++)
  		mapping = req->pages[i]->mapping;

  	if (mapping) {
  		struct inode *inode = mapping->host;
  
  		/*
  		 * Short read means EOF. If file size is larger, truncate it
  		 */

  		if (!req->out.h.error && num_read < count)
  			fuse_short_read(req, inode, req->misc.read.attr_ver);

  		fuse_invalidate_atime(inode);

  	}

  	for (i = 0; i < req->num_pages; i++) {
  		struct page *page = req->pages[i];
  		if (!req->out.h.error)
  			SetPageUptodate(page);

  		else
  			SetPageError(page);

  		unlock_page(page);

  		put_page(page);

  	}

  	if (req->ff)

  		fuse_file_put(req->ff, false);

  }

  static void fuse_send_readpages(struct fuse_req *req, struct file *file)

  {

  	struct fuse_file *ff = file->private_data;
  	struct fuse_conn *fc = ff->fc;

  	loff_t pos = page_offset(req->pages[0]);

  	size_t count = req->num_pages << PAGE_SHIFT;

  
  	req->out.argpages = 1;

  	req->out.page_zeroing = 1;

  	req->out.page_replace = 1;

  	fuse_read_fill(req, file, pos, count, FUSE_READ);

  	req->misc.read.attr_ver = fuse_get_attr_version(fc);

  	if (fc->async_read) {

  		req->ff = fuse_file_get(ff);

  		req->end = fuse_readpages_end;

  		fuse_request_send_background(fc, req);

  	} else {

  		fuse_request_send(fc, req);

  		fuse_readpages_end(fc, req);

  		fuse_put_request(fc, req);

  	}

  }

  struct fuse_fill_data {

  	struct fuse_req *req;

  	struct file *file;

  	struct inode *inode;

  	unsigned nr_pages;

  };
  
  static int fuse_readpages_fill(void *_data, struct page *page)
  {

  	struct fuse_fill_data *data = _data;

  	struct fuse_req *req = data->req;
  	struct inode *inode = data->inode;
  	struct fuse_conn *fc = get_fuse_conn(inode);

  	fuse_wait_on_page_writeback(inode, page->index);

  	if (req->num_pages &&
  	    (req->num_pages == FUSE_MAX_PAGES_PER_REQ ||

  	     (req->num_pages + 1) * PAGE_SIZE > fc->max_read ||

  	     req->pages[req->num_pages - 1]->index + 1 != page->index)) {

  		int nr_alloc = min_t(unsigned, data->nr_pages,
  				     FUSE_MAX_PAGES_PER_REQ);

  		fuse_send_readpages(req, data->file);

  		if (fc->async_read)
  			req = fuse_get_req_for_background(fc, nr_alloc);
  		else
  			req = fuse_get_req(fc, nr_alloc);
  
  		data->req = req;

  		if (IS_ERR(req)) {

  			unlock_page(page);

  			return PTR_ERR(req);

  		}

  	}

  
  	if (WARN_ON(req->num_pages >= req->max_pages)) {
  		fuse_put_request(fc, req);
  		return -EIO;
  	}

  	get_page(page);

  	req->pages[req->num_pages] = page;

  	req->page_descs[req->num_pages].length = PAGE_SIZE;

  	req->num_pages++;

  	data->nr_pages--;

  	return 0;
  }
  
  static int fuse_readpages(struct file *file, struct address_space *mapping,
  			  struct list_head *pages, unsigned nr_pages)
  {
  	struct inode *inode = mapping->host;
  	struct fuse_conn *fc = get_fuse_conn(inode);

  	struct fuse_fill_data data;

  	int err;

  	int nr_alloc = min_t(unsigned, nr_pages, FUSE_MAX_PAGES_PER_REQ);

  	err = -EIO;

  	if (is_bad_inode(inode))

  		goto out;

  	data.file = file;

  	data.inode = inode;

  	if (fc->async_read)
  		data.req = fuse_get_req_for_background(fc, nr_alloc);
  	else
  		data.req = fuse_get_req(fc, nr_alloc);

  	data.nr_pages = nr_pages;

  	err = PTR_ERR(data.req);

  	if (IS_ERR(data.req))

  		goto out;

  
  	err = read_cache_pages(mapping, pages, fuse_readpages_fill, &data);

  	if (!err) {
  		if (data.req->num_pages)

  			fuse_send_readpages(data.req, file);

  		else
  			fuse_put_request(fc, data.req);
  	}

  out:

  	return err;

  }

  static ssize_t fuse_file_read_iter(struct kiocb *iocb, struct iov_iter *to)

  {
  	struct inode *inode = iocb->ki_filp->f_mapping->host;

  	struct fuse_conn *fc = get_fuse_conn(inode);

  	/*
  	 * In auto invalidate mode, always update attributes on read.
  	 * Otherwise, only update if we attempt to read past EOF (to ensure
  	 * i_size is up to date).
  	 */
  	if (fc->auto_inval_data ||

  	    (iocb->ki_pos + iov_iter_count(to) > i_size_read(inode))) {

  		int err;

  		err = fuse_update_attributes(inode, iocb->ki_filp);

  		if (err)
  			return err;
  	}

  	return generic_file_read_iter(iocb, to);

  }

  static void fuse_write_fill(struct fuse_req *req, struct fuse_file *ff,

  			    loff_t pos, size_t count)

  {

  	struct fuse_write_in *inarg = &req->misc.write.in;
  	struct fuse_write_out *outarg = &req->misc.write.out;

  	inarg->fh = ff->fh;
  	inarg->offset = pos;
  	inarg->size = count;

  	req->in.h.opcode = FUSE_WRITE;

  	req->in.h.nodeid = ff->nodeid;

  	req->in.numargs = 2;

  	if (ff->fc->minor < 9)

  		req->in.args[0].size = FUSE_COMPAT_WRITE_IN_SIZE;
  	else
  		req->in.args[0].size = sizeof(struct fuse_write_in);

  	req->in.args[0].value = inarg;

  	req->in.args[1].size = count;
  	req->out.numargs = 1;
  	req->out.args[0].size = sizeof(struct fuse_write_out);

  	req->out.args[0].value = outarg;
  }

  static size_t fuse_send_write(struct fuse_req *req, struct fuse_io_priv *io,

  			      loff_t pos, size_t count, fl_owner_t owner)

  {

  	struct kiocb *iocb = io->iocb;
  	struct file *file = iocb->ki_filp;

  	struct fuse_file *ff = file->private_data;
  	struct fuse_conn *fc = ff->fc;

  	struct fuse_write_in *inarg = &req->misc.write.in;

  	fuse_write_fill(req, ff, pos, count);

  	inarg->flags = file->f_flags;

  	if (iocb->ki_flags & IOCB_DSYNC)
  		inarg->flags |= O_DSYNC;
  	if (iocb->ki_flags & IOCB_SYNC)
  		inarg->flags |= O_SYNC;

  	if (owner != NULL) {

  		inarg->write_flags |= FUSE_WRITE_LOCKOWNER;
  		inarg->lock_owner = fuse_lock_owner_id(fc, owner);
  	}

  
  	if (io->async)
  		return fuse_async_req_send(fc, req, count, io);

  	fuse_request_send(fc, req);

  	return req->misc.write.out.size;

  }

  bool fuse_write_update_size(struct inode *inode, loff_t pos)

  {
  	struct fuse_conn *fc = get_fuse_conn(inode);
  	struct fuse_inode *fi = get_fuse_inode(inode);

  	bool ret = false;

  
  	spin_lock(&fc->lock);
  	fi->attr_version = ++fc->attr_version;

  	if (pos > inode->i_size) {

  		i_size_write(inode, pos);

  		ret = true;
  	}

  	spin_unlock(&fc->lock);

  
  	return ret;

  }

  static size_t fuse_send_write_pages(struct fuse_req *req, struct kiocb *iocb,

  				    struct inode *inode, loff_t pos,
  				    size_t count)
  {
  	size_t res;
  	unsigned offset;
  	unsigned i;

  	struct fuse_io_priv io = FUSE_IO_PRIV_SYNC(iocb);

  
  	for (i = 0; i < req->num_pages; i++)
  		fuse_wait_on_page_writeback(inode, req->pages[i]->index);

  	res = fuse_send_write(req, &io, pos, count, NULL);

  	offset = req->page_descs[0].offset;

  	count = res;
  	for (i = 0; i < req->num_pages; i++) {
  		struct page *page = req->pages[i];

  		if (!req->out.h.error && !offset && count >= PAGE_SIZE)

  			SetPageUptodate(page);

  		if (count > PAGE_SIZE - offset)
  			count -= PAGE_SIZE - offset;

  		else
  			count = 0;
  		offset = 0;
  
  		unlock_page(page);

  		put_page(page);

  	}
  
  	return res;
  }
  
  static ssize_t fuse_fill_write_pages(struct fuse_req *req,
  			       struct address_space *mapping,
  			       struct iov_iter *ii, loff_t pos)
  {
  	struct fuse_conn *fc = get_fuse_conn(mapping->host);

  	unsigned offset = pos & (PAGE_SIZE - 1);

  	size_t count = 0;
  	int err;

  	req->in.argpages = 1;

  	req->page_descs[0].offset = offset;

  
  	do {
  		size_t tmp;
  		struct page *page;

  		pgoff_t index = pos >> PAGE_SHIFT;
  		size_t bytes = min_t(size_t, PAGE_SIZE - offset,

  				     iov_iter_count(ii));
  
  		bytes = min_t(size_t, bytes, fc->max_write - count);
  
   again:
  		err = -EFAULT;
  		if (iov_iter_fault_in_readable(ii, bytes))
  			break;
  
  		err = -ENOMEM;

  		page = grab_cache_page_write_begin(mapping, index, 0);

  		if (!page)
  			break;

  		if (mapping_writably_mapped(mapping))
  			flush_dcache_page(page);

  		tmp = iov_iter_copy_from_user_atomic(page, ii, offset, bytes);

  		flush_dcache_page(page);

  		iov_iter_advance(ii, tmp);

  		if (!tmp) {
  			unlock_page(page);

  			put_page(page);

  			bytes = min(bytes, iov_iter_single_seg_count(ii));
  			goto again;
  		}
  
  		err = 0;
  		req->pages[req->num_pages] = page;

  		req->page_descs[req->num_pages].length = tmp;

  		req->num_pages++;

  		count += tmp;
  		pos += tmp;
  		offset += tmp;

  		if (offset == PAGE_SIZE)

  			offset = 0;

  		if (!fc->big_writes)
  			break;

  	} while (iov_iter_count(ii) && count < fc->max_write &&

  		 req->num_pages < req->max_pages && offset == 0);

  
  	return count > 0 ? count : err;
  }

  static inline unsigned fuse_wr_pages(loff_t pos, size_t len)
  {
  	return min_t(unsigned,

  		     ((pos + len - 1) >> PAGE_SHIFT) -
  		     (pos >> PAGE_SHIFT) + 1,

  		     FUSE_MAX_PAGES_PER_REQ);
  }

  static ssize_t fuse_perform_write(struct kiocb *iocb,

  				  struct address_space *mapping,
  				  struct iov_iter *ii, loff_t pos)
  {
  	struct inode *inode = mapping->host;
  	struct fuse_conn *fc = get_fuse_conn(inode);

  	struct fuse_inode *fi = get_fuse_inode(inode);

  	int err = 0;
  	ssize_t res = 0;
  
  	if (is_bad_inode(inode))
  		return -EIO;

  	if (inode->i_size < pos + iov_iter_count(ii))
  		set_bit(FUSE_I_SIZE_UNSTABLE, &fi->state);

  	do {
  		struct fuse_req *req;
  		ssize_t count;

  		unsigned nr_pages = fuse_wr_pages(pos, iov_iter_count(ii));

  		req = fuse_get_req(fc, nr_pages);

  		if (IS_ERR(req)) {
  			err = PTR_ERR(req);
  			break;
  		}
  
  		count = fuse_fill_write_pages(req, mapping, ii, pos);
  		if (count <= 0) {
  			err = count;
  		} else {
  			size_t num_written;

  			num_written = fuse_send_write_pages(req, iocb, inode,

  							    pos, count);
  			err = req->out.h.error;
  			if (!err) {
  				res += num_written;
  				pos += num_written;
  
  				/* break out of the loop on short write */
  				if (num_written != count)
  					err = -EIO;
  			}
  		}
  		fuse_put_request(fc, req);
  	} while (!err && iov_iter_count(ii));
  
  	if (res > 0)
  		fuse_write_update_size(inode, pos);

  	clear_bit(FUSE_I_SIZE_UNSTABLE, &fi->state);

  	fuse_invalidate_attr(inode);
  
  	return res > 0 ? res : err;
  }

  static ssize_t fuse_file_write_iter(struct kiocb *iocb, struct iov_iter *from)

  {
  	struct file *file = iocb->ki_filp;
  	struct address_space *mapping = file->f_mapping;

  	ssize_t written = 0;

  	ssize_t written_buffered = 0;

  	struct inode *inode = mapping->host;
  	ssize_t err;

  	loff_t endbyte = 0;

  	if (get_fuse_conn(inode)->writeback_cache) {
  		/* Update size (EOF optimization) and mode (SUID clearing) */

  		err = fuse_update_attributes(mapping->host, file);

  		if (err)
  			return err;

  		return generic_file_write_iter(iocb, from);

  	}

  	inode_lock(inode);

  
  	/* We can write back this queue in page reclaim */

  	current->backing_dev_info = inode_to_bdi(inode);

  	err = generic_write_checks(iocb, from);
  	if (err <= 0)

  		goto out;

  	err = file_remove_privs(file);

  	if (err)
  		goto out;

  	err = file_update_time(file);
  	if (err)
  		goto out;

  	if (iocb->ki_flags & IOCB_DIRECT) {

  		loff_t pos = iocb->ki_pos;

  		written = generic_file_direct_write(iocb, from);

  		if (written < 0 || !iov_iter_count(from))

  			goto out;
  
  		pos += written;

  		written_buffered = fuse_perform_write(iocb, mapping, from, pos);

  		if (written_buffered < 0) {
  			err = written_buffered;
  			goto out;
  		}
  		endbyte = pos + written_buffered - 1;
  
  		err = filemap_write_and_wait_range(file->f_mapping, pos,
  						   endbyte);
  		if (err)
  			goto out;
  
  		invalidate_mapping_pages(file->f_mapping,

  					 pos >> PAGE_SHIFT,
  					 endbyte >> PAGE_SHIFT);

  
  		written += written_buffered;
  		iocb->ki_pos = pos + written_buffered;
  	} else {

  		written = fuse_perform_write(iocb, mapping, from, iocb->ki_pos);

  		if (written >= 0)

  			iocb->ki_pos += written;

  	}

  out:
  	current->backing_dev_info = NULL;

  	inode_unlock(inode);

  	if (written > 0)
  		written = generic_write_sync(iocb, written);

  
  	return written ? written : err;
  }

  static inline void fuse_page_descs_length_init(struct fuse_req *req,
  		unsigned index, unsigned nr_pages)

  {
  	int i;

  	for (i = index; i < index + nr_pages; i++)

  		req->page_descs[i].length = PAGE_SIZE -
  			req->page_descs[i].offset;
  }

  static inline unsigned long fuse_get_user_addr(const struct iov_iter *ii)
  {
  	return (unsigned long)ii->iov->iov_base + ii->iov_offset;
  }
  
  static inline size_t fuse_get_frag_size(const struct iov_iter *ii,
  					size_t max_size)
  {
  	return min(iov_iter_single_seg_count(ii), max_size);
  }

  static int fuse_get_user_pages(struct fuse_req *req, struct iov_iter *ii,

  			       size_t *nbytesp, int write)

  {

  	size_t nbytes = 0;  /* # bytes already packed in req */

  	ssize_t ret = 0;

  	/* Special case for kernel I/O: can copy directly into the buffer */

  	if (ii->type & ITER_KVEC) {

  		unsigned long user_addr = fuse_get_user_addr(ii);
  		size_t frag_size = fuse_get_frag_size(ii, *nbytesp);

  		if (write)
  			req->in.args[1].value = (void *) user_addr;
  		else
  			req->out.args[0].value = (void *) user_addr;

  		iov_iter_advance(ii, frag_size);
  		*nbytesp = frag_size;

  		return 0;
  	}

  	while (nbytes < *nbytesp && req->num_pages < req->max_pages) {

  		unsigned npages;

  		size_t start;

  		ret = iov_iter_get_pages(ii, &req->pages[req->num_pages],

  					*nbytesp - nbytes,

  					req->max_pages - req->num_pages,
  					&start);

  		if (ret < 0)

  			break;

  		iov_iter_advance(ii, ret);
  		nbytes += ret;

  		ret += start;
  		npages = (ret + PAGE_SIZE - 1) / PAGE_SIZE;

  		req->page_descs[req->num_pages].offset = start;

  		fuse_page_descs_length_init(req, req->num_pages, npages);
  
  		req->num_pages += npages;
  		req->page_descs[req->num_pages - 1].length -=

  			(PAGE_SIZE - ret) & (PAGE_SIZE - 1);

  	}

  
  	if (write)
  		req->in.argpages = 1;
  	else
  		req->out.argpages = 1;

  	*nbytesp = nbytes;

  	return ret < 0 ? ret : 0;

  }

  static inline int fuse_iter_npages(const struct iov_iter *ii_p)
  {

  	return iov_iter_npages(ii_p, FUSE_MAX_PAGES_PER_REQ);

  }

  ssize_t fuse_direct_io(struct fuse_io_priv *io, struct iov_iter *iter,
  		       loff_t *ppos, int flags)

  {

  	int write = flags & FUSE_DIO_WRITE;
  	int cuse = flags & FUSE_DIO_CUSE;

  	struct file *file = io->iocb->ki_filp;

  	struct inode *inode = file->f_mapping->host;

  	struct fuse_file *ff = file->private_data;
  	struct fuse_conn *fc = ff->fc;

  	size_t nmax = write ? fc->max_write : fc->max_read;
  	loff_t pos = *ppos;

  	size_t count = iov_iter_count(iter);

  	pgoff_t idx_from = pos >> PAGE_SHIFT;
  	pgoff_t idx_to = (pos + count - 1) >> PAGE_SHIFT;

  	ssize_t res = 0;

  	struct fuse_req *req;

  	int err = 0;

  	if (io->async)

  		req = fuse_get_req_for_background(fc, fuse_iter_npages(iter));

  	else

  		req = fuse_get_req(fc, fuse_iter_npages(iter));

  	if (IS_ERR(req))
  		return PTR_ERR(req);

  	if (!cuse && fuse_range_is_writeback(inode, idx_from, idx_to)) {
  		if (!write)

  			inode_lock(inode);

  		fuse_sync_writes(inode);
  		if (!write)

  			inode_unlock(inode);

  	}

  	io->should_dirty = !write && iter_is_iovec(iter);

  	while (count) {

  		size_t nres;

  		fl_owner_t owner = current->files;

  		size_t nbytes = min(count, nmax);

  		err = fuse_get_user_pages(req, iter, &nbytes, write);
  		if (err && !nbytes)

  			break;

  		if (write)

  			nres = fuse_send_write(req, io, pos, nbytes, owner);

  		else

  			nres = fuse_send_read(req, io, pos, nbytes, owner);

  		if (!io->async)

  			fuse_release_user_pages(req, io->should_dirty);

  		if (req->out.h.error) {

  			err = req->out.h.error;

  			break;
  		} else if (nres > nbytes) {

  			res = 0;
  			err = -EIO;

  			break;
  		}
  		count -= nres;
  		res += nres;
  		pos += nres;

  		if (nres != nbytes)
  			break;

  		if (count) {
  			fuse_put_request(fc, req);

  			if (io->async)
  				req = fuse_get_req_for_background(fc,

  					fuse_iter_npages(iter));

  			else

  				req = fuse_get_req(fc, fuse_iter_npages(iter));

  			if (IS_ERR(req))
  				break;
  		}

  	}

  	if (!IS_ERR(req))
  		fuse_put_request(fc, req);

  	if (res > 0)

  		*ppos = pos;

  	return res > 0 ? res : err;

  }

  EXPORT_SYMBOL_GPL(fuse_direct_io);

  static ssize_t __fuse_direct_read(struct fuse_io_priv *io,