/*
   * Copyright (C) 2001, 2002 Sistina Software (UK) Limited.

   * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved.

   *
   * This file is released under the GPL.
   */

  #include "dm-core.h"
  #include "dm-rq.h"

  #include "dm-uevent.h"

  
  #include <linux/init.h>
  #include <linux/module.h>

  #include <linux/mutex.h>

  #include <linux/sched/mm.h>

  #include <linux/sched/signal.h>

  #include <linux/blkpg.h>
  #include <linux/bio.h>

  #include <linux/mempool.h>

  #include <linux/dax.h>

  #include <linux/slab.h>
  #include <linux/idr.h>

  #include <linux/uio.h>

  #include <linux/hdreg.h>

  #include <linux/delay.h>

  #include <linux/wait.h>

  #include <linux/pr.h>

  #include <linux/refcount.h>

  #include <linux/part_stat.h>

  #include <linux/blk-crypto.h>
  #include <linux/keyslot-manager.h>

  #define DM_MSG_PREFIX "core"

  /*
   * Cookies are numeric values sent with CHANGE and REMOVE
   * uevents while resuming, removing or renaming the device.
   */
  #define DM_COOKIE_ENV_VAR_NAME "DM_COOKIE"
  #define DM_COOKIE_LENGTH 24

  static const char *_name = DM_NAME;
  
  static unsigned int major = 0;
  static unsigned int _major = 0;

  static DEFINE_IDR(_minor_idr);

  static DEFINE_SPINLOCK(_minor_lock);

  
  static void do_deferred_remove(struct work_struct *w);
  
  static DECLARE_WORK(deferred_remove_work, do_deferred_remove);

  static struct workqueue_struct *deferred_remove_workqueue;

  atomic_t dm_global_event_nr = ATOMIC_INIT(0);
  DECLARE_WAIT_QUEUE_HEAD(dm_global_eventq);

  void dm_issue_global_event(void)
  {
  	atomic_inc(&dm_global_event_nr);
  	wake_up(&dm_global_eventq);
  }

  /*

   * One of these is allocated (on-stack) per original bio.

   */

  struct clone_info {

  	struct dm_table *map;
  	struct bio *bio;
  	struct dm_io *io;
  	sector_t sector;
  	unsigned sector_count;
  };
  
  /*
   * One of these is allocated per clone bio.
   */
  #define DM_TIO_MAGIC 7282014
  struct dm_target_io {
  	unsigned magic;
  	struct dm_io *io;
  	struct dm_target *ti;
  	unsigned target_bio_nr;
  	unsigned *len_ptr;
  	bool inside_dm_io;
  	struct bio clone;
  };
  
  /*

   * One of these is allocated per original bio.

   * It contains the first clone used for that original.

   */

  #define DM_IO_MAGIC 5191977

  struct dm_io {

  	unsigned magic;

  	struct mapped_device *md;

  	blk_status_t status;

  	atomic_t io_count;

  	struct bio *orig_bio;

  	unsigned long start_time;

  	spinlock_t endio_lock;

  	struct dm_stats_aux stats_aux;

  	/* last member of dm_target_io is 'struct bio' */
  	struct dm_target_io tio;

  };

  void *dm_per_bio_data(struct bio *bio, size_t data_size)
  {
  	struct dm_target_io *tio = container_of(bio, struct dm_target_io, clone);
  	if (!tio->inside_dm_io)
  		return (char *)bio - offsetof(struct dm_target_io, clone) - data_size;
  	return (char *)bio - offsetof(struct dm_target_io, clone) - offsetof(struct dm_io, tio) - data_size;
  }
  EXPORT_SYMBOL_GPL(dm_per_bio_data);
  
  struct bio *dm_bio_from_per_bio_data(void *data, size_t data_size)
  {
  	struct dm_io *io = (struct dm_io *)((char *)data + data_size);
  	if (io->magic == DM_IO_MAGIC)
  		return (struct bio *)((char *)io + offsetof(struct dm_io, tio) + offsetof(struct dm_target_io, clone));
  	BUG_ON(io->magic != DM_TIO_MAGIC);
  	return (struct bio *)((char *)io + offsetof(struct dm_target_io, clone));
  }
  EXPORT_SYMBOL_GPL(dm_bio_from_per_bio_data);
  
  unsigned dm_bio_get_target_bio_nr(const struct bio *bio)
  {
  	return container_of(bio, struct dm_target_io, clone)->target_bio_nr;
  }
  EXPORT_SYMBOL_GPL(dm_bio_get_target_bio_nr);

  #define MINOR_ALLOCED ((void *)-1)

  /*
   * Bits for the md->flags field.
   */

  #define DMF_BLOCK_IO_FOR_SUSPEND 0

  #define DMF_SUSPENDED 1

  #define DMF_FROZEN 2

  #define DMF_FREEING 3

  #define DMF_DELETING 4

  #define DMF_NOFLUSH_SUSPENDING 5

  #define DMF_DEFERRED_REMOVE 6
  #define DMF_SUSPENDED_INTERNALLY 7

  #define DMF_POST_SUSPENDING 8

  #define DM_NUMA_NODE NUMA_NO_NODE

  static int dm_numa_node = DM_NUMA_NODE;

  /*
   * For mempools pre-allocation at the table loading time.
   */
  struct dm_md_mempools {

  	struct bio_set bs;
  	struct bio_set io_bs;

  };

  struct table_device {
  	struct list_head list;

  	refcount_t count;

  	struct dm_dev dm_dev;
  };

  /*

   * Bio-based DM's mempools' reserved IOs set by the user.
   */

  #define RESERVED_BIO_BASED_IOS		16

  static unsigned reserved_bio_based_ios = RESERVED_BIO_BASED_IOS;

  static int __dm_get_module_param_int(int *module_param, int min, int max)
  {

  	int param = READ_ONCE(*module_param);

  	int modified_param = 0;
  	bool modified = true;
  
  	if (param < min)
  		modified_param = min;
  	else if (param > max)
  		modified_param = max;
  	else
  		modified = false;
  
  	if (modified) {
  		(void)cmpxchg(module_param, param, modified_param);
  		param = modified_param;
  	}
  
  	return param;
  }

  unsigned __dm_get_module_param(unsigned *module_param,
  			       unsigned def, unsigned max)

  {

  	unsigned param = READ_ONCE(*module_param);

  	unsigned modified_param = 0;

  	if (!param)
  		modified_param = def;
  	else if (param > max)
  		modified_param = max;

  	if (modified_param) {
  		(void)cmpxchg(module_param, param, modified_param);
  		param = modified_param;

  	}

  	return param;

  }

  unsigned dm_get_reserved_bio_based_ios(void)
  {

  	return __dm_get_module_param(&reserved_bio_based_ios,

  				     RESERVED_BIO_BASED_IOS, DM_RESERVED_MAX_IOS);

  }
  EXPORT_SYMBOL_GPL(dm_get_reserved_bio_based_ios);

  static unsigned dm_get_numa_node(void)
  {
  	return __dm_get_module_param_int(&dm_numa_node,
  					 DM_NUMA_NODE, num_online_nodes() - 1);
  }

  static int __init local_init(void)
  {

  	int r;

  	r = dm_uevent_init();

  	if (r)

  		return r;

  	deferred_remove_workqueue = alloc_workqueue("kdmremove", WQ_UNBOUND, 1);
  	if (!deferred_remove_workqueue) {
  		r = -ENOMEM;
  		goto out_uevent_exit;
  	}

  	_major = major;
  	r = register_blkdev(_major, _name);

  	if (r < 0)

  		goto out_free_workqueue;

  
  	if (!_major)
  		_major = r;
  
  	return 0;

  out_free_workqueue:
  	destroy_workqueue(deferred_remove_workqueue);

  out_uevent_exit:
  	dm_uevent_exit();

  
  	return r;

  }
  
  static void local_exit(void)
  {

  	flush_scheduled_work();

  	destroy_workqueue(deferred_remove_workqueue);

  	unregister_blkdev(_major, _name);

  	dm_uevent_exit();

  
  	_major = 0;
  
  	DMINFO("cleaned up");
  }

  static int (*_inits[])(void) __initdata = {

  	local_init,
  	dm_target_init,
  	dm_linear_init,
  	dm_stripe_init,

  	dm_io_init,

  	dm_kcopyd_init,

  	dm_interface_init,

  	dm_statistics_init,

  };

  static void (*_exits[])(void) = {

  	local_exit,
  	dm_target_exit,
  	dm_linear_exit,
  	dm_stripe_exit,

  	dm_io_exit,

  	dm_kcopyd_exit,

  	dm_interface_exit,

  	dm_statistics_exit,

  };
  
  static int __init dm_init(void)
  {
  	const int count = ARRAY_SIZE(_inits);
  
  	int r, i;
  
  	for (i = 0; i < count; i++) {
  		r = _inits[i]();
  		if (r)
  			goto bad;
  	}
  
  	return 0;
  
        bad:
  	while (i--)
  		_exits[i]();
  
  	return r;
  }
  
  static void __exit dm_exit(void)
  {
  	int i = ARRAY_SIZE(_exits);
  
  	while (i--)
  		_exits[i]();

  
  	/*
  	 * Should be empty by this point.
  	 */

  	idr_destroy(&_minor_idr);

  }
  
  /*
   * Block device functions
   */

  int dm_deleting_md(struct mapped_device *md)
  {
  	return test_bit(DMF_DELETING, &md->flags);
  }

  static int dm_blk_open(struct block_device *bdev, fmode_t mode)

  {
  	struct mapped_device *md;

  	spin_lock(&_minor_lock);

  	md = bdev->bd_disk->private_data;

  	if (!md)
  		goto out;

  	if (test_bit(DMF_FREEING, &md->flags) ||

  	    dm_deleting_md(md)) {

  		md = NULL;
  		goto out;
  	}

  	dm_get(md);

  	atomic_inc(&md->open_count);

  out:
  	spin_unlock(&_minor_lock);
  
  	return md ? 0 : -ENXIO;

  }

  static void dm_blk_close(struct gendisk *disk, fmode_t mode)

  {

  	struct mapped_device *md;

  	spin_lock(&_minor_lock);

  	md = disk->private_data;
  	if (WARN_ON(!md))
  		goto out;

  	if (atomic_dec_and_test(&md->open_count) &&
  	    (test_bit(DMF_DEFERRED_REMOVE, &md->flags)))

  		queue_work(deferred_remove_workqueue, &deferred_remove_work);

  	dm_put(md);

  out:

  	spin_unlock(&_minor_lock);

  }

  int dm_open_count(struct mapped_device *md)
  {
  	return atomic_read(&md->open_count);
  }
  
  /*
   * Guarantees nothing is using the device before it's deleted.
   */

  int dm_lock_for_deletion(struct mapped_device *md, bool mark_deferred, bool only_deferred)

  {
  	int r = 0;
  
  	spin_lock(&_minor_lock);

  	if (dm_open_count(md)) {

  		r = -EBUSY;

  		if (mark_deferred)
  			set_bit(DMF_DEFERRED_REMOVE, &md->flags);
  	} else if (only_deferred && !test_bit(DMF_DEFERRED_REMOVE, &md->flags))
  		r = -EEXIST;

  	else
  		set_bit(DMF_DELETING, &md->flags);
  
  	spin_unlock(&_minor_lock);
  
  	return r;
  }

  int dm_cancel_deferred_remove(struct mapped_device *md)
  {
  	int r = 0;
  
  	spin_lock(&_minor_lock);
  
  	if (test_bit(DMF_DELETING, &md->flags))
  		r = -EBUSY;
  	else
  		clear_bit(DMF_DEFERRED_REMOVE, &md->flags);
  
  	spin_unlock(&_minor_lock);
  
  	return r;
  }
  
  static void do_deferred_remove(struct work_struct *w)
  {
  	dm_deferred_remove();
  }

  static int dm_blk_getgeo(struct block_device *bdev, struct hd_geometry *geo)
  {
  	struct mapped_device *md = bdev->bd_disk->private_data;
  
  	return dm_get_geometry(md, geo);
  }

  #ifdef CONFIG_BLK_DEV_ZONED
  int dm_report_zones_cb(struct blk_zone *zone, unsigned int idx, void *data)
  {
  	struct dm_report_zones_args *args = data;
  	sector_t sector_diff = args->tgt->begin - args->start;
  
  	/*
  	 * Ignore zones beyond the target range.
  	 */
  	if (zone->start >= args->start + args->tgt->len)
  		return 0;
  
  	/*
  	 * Remap the start sector and write pointer position of the zone
  	 * to match its position in the target range.
  	 */
  	zone->start += sector_diff;
  	if (zone->type != BLK_ZONE_TYPE_CONVENTIONAL) {
  		if (zone->cond == BLK_ZONE_COND_FULL)
  			zone->wp = zone->start + zone->len;
  		else if (zone->cond == BLK_ZONE_COND_EMPTY)
  			zone->wp = zone->start;
  		else
  			zone->wp += sector_diff;
  	}
  
  	args->next_sector = zone->start + zone->len;
  	return args->orig_cb(zone, args->zone_idx++, args->orig_data);
  }
  EXPORT_SYMBOL_GPL(dm_report_zones_cb);

  static int dm_blk_report_zones(struct gendisk *disk, sector_t sector,

  		unsigned int nr_zones, report_zones_cb cb, void *data)

  {

  	struct mapped_device *md = disk->private_data;

  	struct dm_table *map;
  	int srcu_idx, ret;

  	struct dm_report_zones_args args = {
  		.next_sector = sector,
  		.orig_data = data,
  		.orig_cb = cb,
  	};

  
  	if (dm_suspended_md(md))
  		return -EAGAIN;
  
  	map = dm_get_live_table(md, &srcu_idx);

  	if (!map) {
  		ret = -EIO;
  		goto out;
  	}

  	do {
  		struct dm_target *tgt;

  		tgt = dm_table_find_target(map, args.next_sector);
  		if (WARN_ON_ONCE(!tgt->type->report_zones)) {
  			ret = -EIO;
  			goto out;
  		}

  		args.tgt = tgt;

  		ret = tgt->type->report_zones(tgt, &args,
  					      nr_zones - args.zone_idx);

  		if (ret < 0)
  			goto out;
  	} while (args.zone_idx < nr_zones &&
  		 args.next_sector < get_capacity(disk));

  	ret = args.zone_idx;

  out:
  	dm_put_live_table(md, srcu_idx);
  	return ret;

  }

  #else
  #define dm_blk_report_zones		NULL
  #endif /* CONFIG_BLK_DEV_ZONED */

  static int dm_prepare_ioctl(struct mapped_device *md, int *srcu_idx,

  			    struct block_device **bdev)

  {

  	struct dm_target *tgt;

  	struct dm_table *map;

  	int r;

  retry:

  	r = -ENOTTY;

  	map = dm_get_live_table(md, srcu_idx);

  	if (!map || !dm_table_get_size(map))

  		return r;

  
  	/* We only support devices that have a single target */
  	if (dm_table_get_num_targets(map) != 1)

  		return r;

  	tgt = dm_table_get_target(map, 0);
  	if (!tgt->type->prepare_ioctl)

  		return r;

  	if (dm_suspended_md(md))
  		return -EAGAIN;

  	r = tgt->type->prepare_ioctl(tgt, bdev);

  	if (r == -ENOTCONN && !fatal_signal_pending(current)) {

  		dm_put_live_table(md, *srcu_idx);

  		msleep(10);
  		goto retry;
  	}

  	return r;
  }

  static void dm_unprepare_ioctl(struct mapped_device *md, int srcu_idx)

  {
  	dm_put_live_table(md, srcu_idx);
  }

  static int dm_blk_ioctl(struct block_device *bdev, fmode_t mode,
  			unsigned int cmd, unsigned long arg)
  {
  	struct mapped_device *md = bdev->bd_disk->private_data;

  	int r, srcu_idx;

  	r = dm_prepare_ioctl(md, &srcu_idx, &bdev);

  	if (r < 0)

  		goto out;

  	if (r > 0) {
  		/*

  		 * Target determined this ioctl is being issued against a
  		 * subset of the parent bdev; require extra privileges.

  		 */

  		if (!capable(CAP_SYS_RAWIO)) {

  			DMDEBUG_LIMIT(

  	"%s: sending ioctl %x to DM device without required privilege.",
  				current->comm, cmd);
  			r = -ENOIOCTLCMD;

  			goto out;

  		}

  	}

  	r =  __blkdev_driver_ioctl(bdev, mode, cmd, arg);

  out:

  	dm_unprepare_ioctl(md, srcu_idx);

  	return r;
  }

  u64 dm_start_time_ns_from_clone(struct bio *bio)
  {
  	struct dm_target_io *tio = container_of(bio, struct dm_target_io, clone);
  	struct dm_io *io = tio->io;
  
  	return jiffies_to_nsecs(io->start_time);
  }
  EXPORT_SYMBOL_GPL(dm_start_time_ns_from_clone);
  
  static void start_io_acct(struct dm_io *io)
  {
  	struct mapped_device *md = io->md;
  	struct bio *bio = io->orig_bio;
  
  	io->start_time = bio_start_io_acct(bio);
  	if (unlikely(dm_stats_used(&md->stats)))
  		dm_stats_account_io(&md->stats, bio_data_dir(bio),
  				    bio->bi_iter.bi_sector, bio_sectors(bio),
  				    false, 0, &io->stats_aux);
  }
  
  static void end_io_acct(struct dm_io *io)
  {
  	struct mapped_device *md = io->md;
  	struct bio *bio = io->orig_bio;
  	unsigned long duration = jiffies - io->start_time;
  
  	bio_end_io_acct(bio, io->start_time);
  
  	if (unlikely(dm_stats_used(&md->stats)))
  		dm_stats_account_io(&md->stats, bio_data_dir(bio),
  				    bio->bi_iter.bi_sector, bio_sectors(bio),
  				    true, duration, &io->stats_aux);
  
  	/* nudge anyone waiting on suspend queue */
  	if (unlikely(wq_has_sleeper(&md->wait)))
  		wake_up(&md->wait);
  }

  
  static struct dm_io *alloc_io(struct mapped_device *md, struct bio *bio)

  {

  	struct dm_io *io;
  	struct dm_target_io *tio;
  	struct bio *clone;

  	clone = bio_alloc_bioset(GFP_NOIO, 0, &md->io_bs);

  	if (!clone)
  		return NULL;
  
  	tio = container_of(clone, struct dm_target_io, clone);
  	tio->inside_dm_io = true;
  	tio->io = NULL;
  
  	io = container_of(tio, struct dm_io, tio);
  	io->magic = DM_IO_MAGIC;

  	io->status = 0;
  	atomic_set(&io->io_count, 1);
  	io->orig_bio = bio;
  	io->md = md;
  	spin_lock_init(&io->endio_lock);
  
  	start_io_acct(io);

  
  	return io;

  }

  static void free_io(struct mapped_device *md, struct dm_io *io)

  {

  	bio_put(&io->tio.clone);
  }
  
  static struct dm_target_io *alloc_tio(struct clone_info *ci, struct dm_target *ti,
  				      unsigned target_bio_nr, gfp_t gfp_mask)
  {
  	struct dm_target_io *tio;
  
  	if (!ci->io->tio.io) {
  		/* the dm_target_io embedded in ci->io is available */
  		tio = &ci->io->tio;
  	} else {

  		struct bio *clone = bio_alloc_bioset(gfp_mask, 0, &ci->io->md->bs);

  		if (!clone)
  			return NULL;
  
  		tio = container_of(clone, struct dm_target_io, clone);
  		tio->inside_dm_io = false;
  	}
  
  	tio->magic = DM_TIO_MAGIC;
  	tio->io = ci->io;
  	tio->ti = ti;
  	tio->target_bio_nr = target_bio_nr;
  
  	return tio;

  }

  static void free_tio(struct dm_target_io *tio)

  {

  	if (tio->inside_dm_io)
  		return;

  	bio_put(&tio->clone);

  }
  
  /*
   * Add the bio to the list of deferred io.
   */

  static void queue_io(struct mapped_device *md, struct bio *bio)

  {

  	unsigned long flags;

  	spin_lock_irqsave(&md->deferred_lock, flags);

  	bio_list_add(&md->deferred, bio);

  	spin_unlock_irqrestore(&md->deferred_lock, flags);

  	queue_work(md->wq, &md->work);

  }
  
  /*
   * Everyone (including functions in this file), should use this
   * function to access the md->map field, and make sure they call

   * dm_put_live_table() when finished.

   */

  struct dm_table *dm_get_live_table(struct mapped_device *md, int *srcu_idx) __acquires(md->io_barrier)

  {

  	*srcu_idx = srcu_read_lock(&md->io_barrier);
  
  	return srcu_dereference(md->map, &md->io_barrier);
  }

  void dm_put_live_table(struct mapped_device *md, int srcu_idx) __releases(md->io_barrier)
  {
  	srcu_read_unlock(&md->io_barrier, srcu_idx);
  }
  
  void dm_sync_table(struct mapped_device *md)
  {
  	synchronize_srcu(&md->io_barrier);
  	synchronize_rcu_expedited();
  }
  
  /*
   * A fast alternative to dm_get_live_table/dm_put_live_table.
   * The caller must not block between these two functions.
   */
  static struct dm_table *dm_get_live_table_fast(struct mapped_device *md) __acquires(RCU)
  {
  	rcu_read_lock();
  	return rcu_dereference(md->map);
  }

  static void dm_put_live_table_fast(struct mapped_device *md) __releases(RCU)
  {
  	rcu_read_unlock();

  }

  static char *_dm_claim_ptr = "I belong to device-mapper";

  /*

   * Open a table device so we can use it as a map destination.
   */
  static int open_table_device(struct table_device *td, dev_t dev,
  			     struct mapped_device *md)
  {

  	struct block_device *bdev;
  
  	int r;
  
  	BUG_ON(td->dm_dev.bdev);

  	bdev = blkdev_get_by_dev(dev, td->dm_dev.mode | FMODE_EXCL, _dm_claim_ptr);

  	if (IS_ERR(bdev))
  		return PTR_ERR(bdev);
  
  	r = bd_link_disk_holder(bdev, dm_disk(md));
  	if (r) {
  		blkdev_put(bdev, td->dm_dev.mode | FMODE_EXCL);
  		return r;
  	}
  
  	td->dm_dev.bdev = bdev;

  	td->dm_dev.dax_dev = dax_get_by_host(bdev->bd_disk->disk_name);

  	return 0;
  }
  
  /*
   * Close a table device that we've been using.
   */
  static void close_table_device(struct table_device *td, struct mapped_device *md)
  {
  	if (!td->dm_dev.bdev)
  		return;
  
  	bd_unlink_disk_holder(td->dm_dev.bdev, dm_disk(md));
  	blkdev_put(td->dm_dev.bdev, td->dm_dev.mode | FMODE_EXCL);

  	put_dax(td->dm_dev.dax_dev);

  	td->dm_dev.bdev = NULL;

  	td->dm_dev.dax_dev = NULL;

  }
  
  static struct table_device *find_table_device(struct list_head *l, dev_t dev,

  					      fmode_t mode)
  {

  	struct table_device *td;
  
  	list_for_each_entry(td, l, list)
  		if (td->dm_dev.bdev->bd_dev == dev && td->dm_dev.mode == mode)
  			return td;
  
  	return NULL;
  }
  
  int dm_get_table_device(struct mapped_device *md, dev_t dev, fmode_t mode,

  			struct dm_dev **result)
  {

  	int r;
  	struct table_device *td;
  
  	mutex_lock(&md->table_devices_lock);
  	td = find_table_device(&md->table_devices, dev, mode);
  	if (!td) {

  		td = kmalloc_node(sizeof(*td), GFP_KERNEL, md->numa_node_id);

  		if (!td) {
  			mutex_unlock(&md->table_devices_lock);
  			return -ENOMEM;
  		}
  
  		td->dm_dev.mode = mode;
  		td->dm_dev.bdev = NULL;
  
  		if ((r = open_table_device(td, dev, md))) {
  			mutex_unlock(&md->table_devices_lock);
  			kfree(td);
  			return r;
  		}
  
  		format_dev_t(td->dm_dev.name, dev);

  		refcount_set(&td->count, 1);

  		list_add(&td->list, &md->table_devices);

  	} else {
  		refcount_inc(&td->count);

  	}

  	mutex_unlock(&md->table_devices_lock);
  
  	*result = &td->dm_dev;
  	return 0;
  }
  EXPORT_SYMBOL_GPL(dm_get_table_device);
  
  void dm_put_table_device(struct mapped_device *md, struct dm_dev *d)
  {
  	struct table_device *td = container_of(d, struct table_device, dm_dev);
  
  	mutex_lock(&md->table_devices_lock);

  	if (refcount_dec_and_test(&td->count)) {

  		close_table_device(td, md);
  		list_del(&td->list);
  		kfree(td);
  	}
  	mutex_unlock(&md->table_devices_lock);
  }
  EXPORT_SYMBOL(dm_put_table_device);
  
  static void free_table_devices(struct list_head *devices)
  {
  	struct list_head *tmp, *next;
  
  	list_for_each_safe(tmp, next, devices) {
  		struct table_device *td = list_entry(tmp, struct table_device, list);
  
  		DMWARN("dm_destroy: %s still exists with %d references",

  		       td->dm_dev.name, refcount_read(&td->count));

  		kfree(td);
  	}
  }
  
  /*

   * Get the geometry associated with a dm device
   */
  int dm_get_geometry(struct mapped_device *md, struct hd_geometry *geo)
  {
  	*geo = md->geometry;
  
  	return 0;
  }
  
  /*
   * Set the geometry of a device.
   */
  int dm_set_geometry(struct mapped_device *md, struct hd_geometry *geo)
  {
  	sector_t sz = (sector_t)geo->cylinders * geo->heads * geo->sectors;
  
  	if (geo->start > sz) {
  		DMWARN("Start sector is beyond the geometry limits.");
  		return -EINVAL;
  	}
  
  	md->geometry = *geo;
  
  	return 0;
  }

  static int __noflush_suspending(struct mapped_device *md)
  {
  	return test_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
  }

  /*
   * Decrements the number of outstanding ios that a bio has been
   * cloned into, completing the original io if necc.
   */

  static void dec_pending(struct dm_io *io, blk_status_t error)

  {

  	unsigned long flags;

  	blk_status_t io_error;

  	struct bio *bio;
  	struct mapped_device *md = io->md;

  
  	/* Push-back supersedes any I/O errors */

  	if (unlikely(error)) {
  		spin_lock_irqsave(&io->endio_lock, flags);

  		if (!(io->status == BLK_STS_DM_REQUEUE && __noflush_suspending(md)))

  			io->status = error;

  		spin_unlock_irqrestore(&io->endio_lock, flags);
  	}

  
  	if (atomic_dec_and_test(&io->io_count)) {

  		if (io->status == BLK_STS_DM_REQUEUE) {

  			/*
  			 * Target requested pushing back the I/O.

  			 */

  			spin_lock_irqsave(&md->deferred_lock, flags);

  			if (__noflush_suspending(md))

  				/* NOTE early return due to BLK_STS_DM_REQUEUE below */
  				bio_list_add_head(&md->deferred, io->orig_bio);

  			else

  				/* noflush suspend was interrupted. */

  				io->status = BLK_STS_IOERR;

  			spin_unlock_irqrestore(&md->deferred_lock, flags);

  		}

  		io_error = io->status;

  		bio = io->orig_bio;

  		end_io_acct(io);
  		free_io(md, io);

  		if (io_error == BLK_STS_DM_REQUEUE)

  			return;

  		if ((bio->bi_opf & REQ_PREFLUSH) && bio->bi_iter.bi_size) {

  			/*

  			 * Preflush done for flush with data, reissue

  			 * without REQ_PREFLUSH.

  			 */

  			bio->bi_opf &= ~REQ_PREFLUSH;

  			queue_io(md, bio);

  		} else {

  			/* done with normal IO or empty flush */

  			if (io_error)
  				bio->bi_status = io_error;

  			bio_endio(bio);

  		}

  	}
  }

  void disable_discard(struct mapped_device *md)
  {
  	struct queue_limits *limits = dm_get_queue_limits(md);
  
  	/* device doesn't really support DISCARD, disable it */
  	limits->max_discard_sectors = 0;
  	blk_queue_flag_clear(QUEUE_FLAG_DISCARD, md->queue);
  }

  void disable_write_same(struct mapped_device *md)

  {
  	struct queue_limits *limits = dm_get_queue_limits(md);
  
  	/* device doesn't really support WRITE SAME, disable it */
  	limits->max_write_same_sectors = 0;
  }

  void disable_write_zeroes(struct mapped_device *md)
  {
  	struct queue_limits *limits = dm_get_queue_limits(md);
  
  	/* device doesn't really support WRITE ZEROES, disable it */
  	limits->max_write_zeroes_sectors = 0;
  }

  static void clone_endio(struct bio *bio)

  {

  	blk_status_t error = bio->bi_status;

  	struct dm_target_io *tio = container_of(bio, struct dm_target_io, clone);

  	struct dm_io *io = tio->io;

  	struct mapped_device *md = tio->io->md;

  	dm_endio_fn endio = tio->ti->type->end_io;

  	struct bio *orig_bio = io->orig_bio;

  	if (unlikely(error == BLK_STS_TARGET)) {

  		if (bio_op(bio) == REQ_OP_DISCARD &&
  		    !bio->bi_disk->queue->limits.max_discard_sectors)
  			disable_discard(md);
  		else if (bio_op(bio) == REQ_OP_WRITE_SAME &&
  			 !bio->bi_disk->queue->limits.max_write_same_sectors)

  			disable_write_same(md);

  		else if (bio_op(bio) == REQ_OP_WRITE_ZEROES &&
  			 !bio->bi_disk->queue->limits.max_write_zeroes_sectors)

  			disable_write_zeroes(md);
  	}

  	/*
  	 * For zone-append bios get offset in zone of the written
  	 * sector and add that to the original bio sector pos.
  	 */
  	if (bio_op(orig_bio) == REQ_OP_ZONE_APPEND) {
  		sector_t written_sector = bio->bi_iter.bi_sector;
  		struct request_queue *q = orig_bio->bi_disk->queue;
  		u64 mask = (u64)blk_queue_zone_sectors(q) - 1;
  
  		orig_bio->bi_iter.bi_sector += written_sector & mask;
  	}

  	if (endio) {

  		int r = endio(tio->ti, bio, &error);

  		switch (r) {
  		case DM_ENDIO_REQUEUE:

  			error = BLK_STS_DM_REQUEUE;

  			fallthrough;

  		case DM_ENDIO_DONE:
  			break;
  		case DM_ENDIO_INCOMPLETE:
  			/* The target will handle the io */
  			return;
  		default:
  			DMWARN("unimplemented target endio return value: %d", r);
  			BUG();
  		}
  	}

  	free_tio(tio);

  	dec_pending(io, error);

  }

  /*

   * Return maximum size of I/O possible at the supplied sector up to the current
   * target boundary.
   */

  static inline sector_t max_io_len_target_boundary(struct dm_target *ti,
  						  sector_t target_offset)

  {

  	return ti->len - target_offset;
  }

  static sector_t max_io_len(struct dm_target *ti, sector_t sector)

  {

  	sector_t target_offset = dm_target_offset(ti, sector);
  	sector_t len = max_io_len_target_boundary(ti, target_offset);

  	sector_t max_len;

  
  	/*

  	 * Does the target need to split IO even further?
  	 * - varied (per target) IO splitting is a tenet of DM; this
  	 *   explains why stacked chunk_sectors based splitting via
  	 *   blk_max_size_offset() isn't possible here. So pass in
  	 *   ti->max_io_len to override stacked chunk_sectors.

  	 */

  	if (ti->max_io_len) {
  		max_len = blk_max_size_offset(ti->table->md->queue,
  					      target_offset, ti->max_io_len);
  		if (len > max_len)
  			len = max_len;
  	}

  
  	return len;
  }

  int dm_set_target_max_io_len(struct dm_target *ti, sector_t len)
  {
  	if (len > UINT_MAX) {
  		DMERR("Specified maximum size of target IO (%llu) exceeds limit (%u)",
  		      (unsigned long long)len, UINT_MAX);
  		ti->error = "Maximum size of target IO is too large";
  		return -EINVAL;
  	}

  	ti->max_io_len = (uint32_t) len;

  
  	return 0;
  }
  EXPORT_SYMBOL_GPL(dm_set_target_max_io_len);

  static struct dm_target *dm_dax_get_live_target(struct mapped_device *md,

  						sector_t sector, int *srcu_idx)
  	__acquires(md->io_barrier)

  {

  	struct dm_table *map;
  	struct dm_target *ti;

  	map = dm_get_live_table(md, srcu_idx);

  	if (!map)

  		return NULL;

  
  	ti = dm_table_find_target(map, sector);

  	if (!ti)

  		return NULL;

  	return ti;
  }

  static long dm_dax_direct_access(struct dax_device *dax_dev, pgoff_t pgoff,

  				 long nr_pages, void **kaddr, pfn_t *pfn)

  {
  	struct mapped_device *md = dax_get_private(dax_dev);
  	sector_t sector = pgoff * PAGE_SECTORS;
  	struct dm_target *ti;
  	long len, ret = -EIO;
  	int srcu_idx;

  	ti = dm_dax_get_live_target(md, sector, &srcu_idx);

  	if (!ti)
  		goto out;
  	if (!ti->type->direct_access)
  		goto out;

  	len = max_io_len(ti, sector) / PAGE_SECTORS;

  	if (len < 1)
  		goto out;
  	nr_pages = min(len, nr_pages);

  	ret = ti->type->direct_access(ti, pgoff, nr_pages, kaddr, pfn);

   out:

  	dm_put_live_table(md, srcu_idx);

  
  	return ret;

  }

  static bool dm_dax_supported(struct dax_device *dax_dev, struct block_device *bdev,
  		int blocksize, sector_t start, sector_t len)
  {
  	struct mapped_device *md = dax_get_private(dax_dev);
  	struct dm_table *map;

  	bool ret = false;

  	int srcu_idx;

  
  	map = dm_get_live_table(md, &srcu_idx);
  	if (!map)

  		goto out;

  	ret = dm_table_supports_dax(map, device_supports_dax, &blocksize);

  out:

  	dm_put_live_table(md, srcu_idx);
  
  	return ret;
  }

  static size_t dm_dax_copy_from_iter(struct dax_device *dax_dev, pgoff_t pgoff,

  				    void *addr, size_t bytes, struct iov_iter *i)

  {
  	struct mapped_device *md = dax_get_private(dax_dev);
  	sector_t sector = pgoff * PAGE_SECTORS;
  	struct dm_target *ti;
  	long ret = 0;
  	int srcu_idx;
  
  	ti = dm_dax_get_live_target(md, sector, &srcu_idx);
  
  	if (!ti)
  		goto out;
  	if (!ti->type->dax_copy_from_iter) {
  		ret = copy_from_iter(addr, bytes, i);
  		goto out;
  	}
  	ret = ti->type->dax_copy_from_iter(ti, pgoff, addr, bytes, i);
   out:
  	dm_put_live_table(md, srcu_idx);
  
  	return ret;
  }

  static size_t dm_dax_copy_to_iter(struct dax_device *dax_dev, pgoff_t pgoff,
  		void *addr, size_t bytes, struct iov_iter *i)
  {
  	struct mapped_device *md = dax_get_private(dax_dev);
  	sector_t sector = pgoff * PAGE_SECTORS;
  	struct dm_target *ti;
  	long ret = 0;
  	int srcu_idx;
  
  	ti = dm_dax_get_live_target(md, sector, &srcu_idx);
  
  	if (!ti)
  		goto out;
  	if (!ti->type->dax_copy_to_iter) {
  		ret = copy_to_iter(addr, bytes, i);
  		goto out;
  	}
  	ret = ti->type->dax_copy_to_iter(ti, pgoff, addr, bytes, i);
   out:
  	dm_put_live_table(md, srcu_idx);
  
  	return ret;
  }

  static int dm_dax_zero_page_range(struct dax_device *dax_dev, pgoff_t pgoff,
  				  size_t nr_pages)
  {
  	struct mapped_device *md = dax_get_private(dax_dev);
  	sector_t sector = pgoff * PAGE_SECTORS;
  	struct dm_target *ti;
  	int ret = -EIO;
  	int srcu_idx;
  
  	ti = dm_dax_get_live_target(md, sector, &srcu_idx);
  
  	if (!ti)
  		goto out;
  	if (WARN_ON(!ti->type->dax_zero_page_range)) {
  		/*
  		 * ->zero_page_range() is mandatory dax operation. If we are
  		 *  here, something is wrong.
  		 */

  		goto out;
  	}
  	ret = ti->type->dax_zero_page_range(ti, pgoff, nr_pages);

   out:
  	dm_put_live_table(md, srcu_idx);
  
  	return ret;
  }

  /*
   * A target may call dm_accept_partial_bio only from the map routine.  It is

   * allowed for all bio types except REQ_PREFLUSH, REQ_OP_ZONE_RESET,
   * REQ_OP_ZONE_OPEN, REQ_OP_ZONE_CLOSE and REQ_OP_ZONE_FINISH.

   *
   * dm_accept_partial_bio informs the dm that the target only wants to process
   * additional n_sectors sectors of the bio and the rest of the data should be
   * sent in a next bio.
   *
   * A diagram that explains the arithmetics:
   * +--------------------+---------------+-------+
   * |         1          |       2       |   3   |
   * +--------------------+---------------+-------+
   *
   * <-------------- *tio->len_ptr --------------->
   *                      <------- bi_size ------->
   *                      <-- n_sectors -->
   *
   * Region 1 was already iterated over with bio_advance or similar function.
   *	(it may be empty if the target doesn't use bio_advance)
   * Region 2 is the remaining bio size that the target wants to process.
   *	(it may be empty if region 1 is non-empty, although there is no reason
   *	 to make it empty)
   * The target requires that region 3 is to be sent in the next bio.
   *
   * If the target wants to receive multiple copies of the bio (via num_*bios, etc),
   * the partially processed part (the sum of regions 1+2) must be the same for all
   * copies of the bio.
   */
  void dm_accept_partial_bio(struct bio *bio, unsigned n_sectors)
  {
  	struct dm_target_io *tio = container_of(bio, struct dm_target_io, clone);
  	unsigned bi_size = bio->bi_iter.bi_size >> SECTOR_SHIFT;

  	BUG_ON(bio->bi_opf & REQ_PREFLUSH);

  	BUG_ON(bi_size > *tio->len_ptr);
  	BUG_ON(n_sectors > bi_size);
  	*tio->len_ptr -= bi_size - n_sectors;
  	bio->bi_iter.bi_size = n_sectors << SECTOR_SHIFT;
  }
  EXPORT_SYMBOL_GPL(dm_accept_partial_bio);

  static blk_qc_t __map_bio(struct dm_target_io *tio)

  {
  	int r;

  	sector_t sector;

  	struct bio *clone = &tio->clone;

  	struct dm_io *io = tio->io;

  	struct dm_target *ti = tio->ti;

  	blk_qc_t ret = BLK_QC_T_NONE;

  	clone->bi_end_io = clone_endio;

  
  	/*
  	 * Map the clone.  If r == 0 we don't need to do
  	 * anything, the target has assumed ownership of
  	 * this io.
  	 */

  	atomic_inc(&io->io_count);

  	sector = clone->bi_iter.bi_sector;

  	r = ti->type->map(ti, clone);

  	switch (r) {
  	case DM_MAPIO_SUBMITTED:
  		break;
  	case DM_MAPIO_REMAPPED:

  		/* the bio has been remapped so dispatch it */

  		trace_block_bio_remap(clone->bi_disk->queue, clone,

  				      bio_dev(io->orig_bio), sector);

  		ret = submit_bio_noacct(clone);

  		break;
  	case DM_MAPIO_KILL:

  		free_tio(tio);

  		dec_pending(io, BLK_STS_IOERR);

  		break;

  	case DM_MAPIO_REQUEUE:

  		free_tio(tio);

  		dec_pending(io, BLK_STS_DM_REQUEUE);

  		break;
  	default:

  		DMWARN("unimplemented target map return value: %d", r);
  		BUG();

  	}

  	return ret;

  }

  static void bio_setup_sector(struct bio *bio, sector_t sector, unsigned len)

  {

  	bio->bi_iter.bi_sector = sector;
  	bio->bi_iter.bi_size = to_bytes(len);

  }
  
  /*
   * Creates a bio that consists of range of complete bvecs.
   */

  static int clone_bio(struct dm_target_io *tio, struct bio *bio,
  		     sector_t sector, unsigned len)

  {

  	struct bio *clone = &tio->clone;

  	int r;

  	__bio_clone_fast(clone, bio);

  	r = bio_crypt_clone(clone, bio, GFP_NOIO);
  	if (r < 0)
  		return r;

  	if (bio_integrity(bio)) {

  		if (unlikely(!dm_target_has_integrity(tio->ti->type) &&
  			     !dm_target_passes_integrity(tio->ti->type))) {
  			DMWARN("%s: the target %s doesn't support integrity data.",
  				dm_device_name(tio->io->md),
  				tio->ti->type->name);
  			return -EIO;
  		}

  		r = bio_integrity_clone(clone, bio, GFP_NOIO);
  		if (r < 0)
  			return r;

  	}

  	bio_advance(clone, to_bytes(sector - clone->bi_iter.bi_sector));
  	clone->bi_iter.bi_size = to_bytes(len);
  
  	if (bio_integrity(bio))
  		bio_integrity_trim(clone);

  
  	return 0;

  }

  static void alloc_multiple_bios(struct bio_list *blist, struct clone_info *ci,
  				struct dm_target *ti, unsigned num_bios)

  {

  	struct dm_target_io *tio;

  	int try;

  	if (!num_bios)
  		return;

  	if (num_bios == 1) {
  		tio = alloc_tio(ci, ti, 0, GFP_NOIO);
  		bio_list_add(blist, &tio->clone);
  		return;
  	}

  	for (try = 0; try < 2; try++) {
  		int bio_nr;
  		struct bio *bio;
  
  		if (try)

  			mutex_lock(&ci->io->md->table_devices_lock);

  		for (bio_nr = 0; bio_nr < num_bios; bio_nr++) {
  			tio = alloc_tio(ci, ti, bio_nr, try ? GFP_NOIO : GFP_NOWAIT);
  			if (!tio)
  				break;
  
  			bio_list_add(blist, &tio->clone);
  		}
  		if (try)

  			mutex_unlock(&ci->io->md->table_devices_lock);

  		if (bio_nr == num_bios)
  			return;
  
  		while ((bio = bio_list_pop(blist))) {
  			tio = container_of(bio, struct dm_target_io, clone);
  			free_tio(tio);
  		}
  	}

  }

  static blk_qc_t __clone_and_map_simple_bio(struct clone_info *ci,
  					   struct dm_target_io *tio, unsigned *len)

  {

  	struct bio *clone = &tio->clone;

  	tio->len_ptr = len;

  	__bio_clone_fast(clone, ci->bio);

  	if (len)

  		bio_setup_sector(clone, ci->sector, *len);

  	return __map_bio(tio);

  }

  static void __send_duplicate_bios(struct clone_info *ci, struct dm_target *ti,

  				  unsigned num_bios, unsigned *len)

  {

  	struct bio_list blist = BIO_EMPTY_LIST;
  	struct bio *bio;
  	struct dm_target_io *tio;
  
  	alloc_multiple_bios(&blist, ci, ti, num_bios);

  	while ((bio = bio_list_pop(&blist))) {
  		tio = container_of(bio, struct dm_target_io, clone);

  		(void) __clone_and_map_simple_bio(ci, tio, len);

  	}

  }

  static int __send_empty_flush(struct clone_info *ci)

  {

  	unsigned target_nr = 0;

  	struct dm_target *ti;

  	struct bio flush_bio;
  
  	/*
  	 * Use an on-stack bio for this, it's safe since we don't
  	 * need to reference it after submit. It's just used as
  	 * the basis for the clone(s).
  	 */
  	bio_init(&flush_bio, NULL, 0);
  	flush_bio.bi_opf = REQ_OP_WRITE | REQ_PREFLUSH | REQ_SYNC;
  	ci->bio = &flush_bio;
  	ci->sector_count = 0;

  	/*

  	 * Empty flush uses a statically initialized bio, as the base for
  	 * cloning.  However, blkg association requires that a bdev is
  	 * associated with a gendisk, which doesn't happen until the bdev is
  	 * opened.  So, blkg association is done at issue time of the flush
  	 * rather than when the device is created in alloc_dev().

  	 */
  	bio_set_dev(ci->bio, ci->io->md->bdev);

  	BUG_ON(bio_has_data(ci->bio));

  	while ((ti = dm_table_get_target(ci->map, target_nr++)))

  		__send_duplicate_bios(ci, ti, ti->num_flush_bios, NULL);

  
  	bio_uninit(ci->bio);

  	return 0;
  }

  static int __clone_and_map_data_bio(struct clone_info *ci, struct dm_target *ti,

  				    sector_t sector, unsigned *len)

  {

  	struct bio *bio = ci->bio;

  	struct dm_target_io *tio;

  	int r;

  	tio = alloc_tio(ci, ti, 0, GFP_NOIO);

  	tio->len_ptr = len;
  	r = clone_bio(tio, bio, sector, *len);
  	if (r < 0) {
  		free_tio(tio);
  		return r;

  	}

  	(void) __map_bio(tio);

  	return 0;

  }

  static int __send_changing_extent_only(struct clone_info *ci, struct dm_target *ti,

  				       unsigned num_bios)

  {

  	unsigned len;

  	/*
  	 * Even though the device advertised support for this type of
  	 * request, that does not mean every target supports it, and
  	 * reconfiguration might also have changed that since the
  	 * check was performed.
  	 */

  	if (!num_bios)
  		return -EOPNOTSUPP;

  	len = min_t(sector_t, ci->sector_count,
  		    max_io_len_target_boundary(ti, dm_target_offset(ti, ci->sector)));

  	__send_duplicate_bios(ci, ti, num_bios, &len);

  	ci->sector += len;
  	ci->sector_count -= len;

  
  	return 0;

  }

  static bool is_abnormal_io(struct bio *bio)
  {
  	bool r = false;
  
  	switch (bio_op(bio)) {
  	case REQ_OP_DISCARD:
  	case REQ_OP_SECURE_ERASE:
  	case REQ_OP_WRITE_SAME:
  	case REQ_OP_WRITE_ZEROES:
  		r = true;
  		break;
  	}
  
  	return r;
  }

  static bool __process_abnormal_io(struct clone_info *ci, struct dm_target *ti,
  				  int *result)
  {
  	struct bio *bio = ci->bio;

  	unsigned num_bios = 0;

  	switch (bio_op(bio)) {
  	case REQ_OP_DISCARD:
  		num_bios = ti->num_discard_bios;
  		break;
  	case REQ_OP_SECURE_ERASE:
  		num_bios = ti->num_secure_erase_bios;
  		break;
  	case REQ_OP_WRITE_SAME:
  		num_bios = ti->num_write_same_bios;
  		break;
  	case REQ_OP_WRITE_ZEROES:
  		num_bios = ti->num_write_zeroes_bios;
  		break;
  	default:

  		return false;

  	}

  	*result = __send_changing_extent_only(ci, ti, num_bios);

  	return true;
  }

  /*

   * Select the correct strategy for processing a non-flush bio.
   */

  static int __split_and_process_non_flush(struct clone_info *ci)

  {

  	struct dm_target *ti;

  	unsigned len;

  	int r;

  	ti = dm_table_find_target(ci->map, ci->sector);

  	if (!ti)

  		return -EIO;

  	if (__process_abnormal_io(ci, ti, &r))

  		return r;

  	len = min_t(sector_t, max_io_len(ti, ci->sector), ci->sector_count);

  	r = __clone_and_map_data_bio(ci, ti, ci->sector, &len);
  	if (r < 0)
  		return r;

  	ci->sector += len;
  	ci->sector_count -= len;

  	return 0;

  }

  static void init_clone_info(struct clone_info *ci, struct mapped_device *md,
  			    struct dm_table *map, struct bio *bio)
  {
  	ci->map = map;
  	ci->io = alloc_io(md, bio);
  	ci->sector = bio->bi_iter.bi_sector;
  }

  #define __dm_part_stat_sub(part, field, subnd)	\
  	(part_stat_get(part, field) -= (subnd))

  /*

   * Entry point to split a bio into clones and submit them to the targets.

   */

  static blk_qc_t __split_and_process_bio(struct mapped_device *md,
  					struct dm_table *map, struct bio *bio)

  {

  	struct clone_info ci;

  	blk_qc_t ret = BLK_QC_T_NONE;

  	int error = 0;

  	init_clone_info(&ci, md, map, bio);

  	if (bio->bi_opf & REQ_PREFLUSH) {

  		error = __send_empty_flush(&ci);

  		/* dec_pending submits any data associated with flush */

  	} else if (op_is_zone_mgmt(bio_op(bio))) {

  		ci.bio = bio;
  		ci.sector_count = 0;
  		error = __split_and_process_non_flush(&ci);

  	} else {

  		ci.bio = bio;

  		ci.sector_count = bio_sectors(bio);

  		while (ci.sector_count && !error) {

  			error = __split_and_process_non_flush(&ci);

  			if (current->bio_list && ci.sector_count && !error) {
  				/*

  				 * Remainder must be passed to submit_bio_noacct()

  				 * so that it gets handled *after* bios already submitted
  				 * have been completely processed.
  				 * We take a clone of the original to store in

  				 * ci.io->orig_bio to be used by end_io_acct() and

  				 * for dec_pending to use for completion handling.

  				 */

  				struct bio *b = bio_split(bio, bio_sectors(bio) - ci.sector_count,
  							  GFP_NOIO, &md->queue->bio_split);

  				ci.io->orig_bio = b;

  
  				/*
  				 * Adjust IO stats for each split, otherwise upon queue
  				 * reentry there will be redundant IO accounting.
  				 * NOTE: this is a stop-gap fix, a proper fix involves
  				 * significant refactoring of DM core's bio splitting
  				 * (by eliminating DM's splitting and just using bio_split)
  				 */
  				part_stat_lock();
  				__dm_part_stat_sub(&dm_disk(md)->part0,
  						   sectors[op_stat_group(bio_op(bio))], ci.sector_count);
  				part_stat_unlock();

  				bio_chain(b, bio);

  				trace_block_split(md->queue, b, bio->bi_iter.bi_sector);

  				ret = submit_bio_noacct(bio);

  				break;
  			}
  		}

  	}

  	/* drop the extra reference count */

  	dec_pending(ci.io, errno_to_blk_status(error));

  	return ret;

  }

  static blk_qc_t dm_submit_bio(struct bio *bio)

  {

  	struct mapped_device *md = bio->bi_disk->private_data;

  	blk_qc_t ret = BLK_QC_T_NONE;

  	int srcu_idx;
  	struct dm_table *map;

  	map = dm_get_live_table(md, &srcu_idx);

  	if (unlikely(!map)) {
  		DMERR_LIMIT("%s: mapping table unavailable, erroring io",
  			    dm_device_name(md));
  		bio_io_error(bio);
  		goto out;
  	}