// SPDX-License-Identifier: GPL-2.0-only

  /*

   * Copyright (C) 1999 Eric Youngdale
   * Copyright (C) 2014 Christoph Hellwig

   *
   *  SCSI queueing library.
   *      Initial versions: Eric Youngdale (eric@andante.org).
   *                        Based upon conversations with large numbers
   *                        of people at Linux Expo.
   */
  
  #include <linux/bio.h>

  #include <linux/bitops.h>

  #include <linux/blkdev.h>
  #include <linux/completion.h>
  #include <linux/kernel.h>

  #include <linux/export.h>

  #include <linux/init.h>
  #include <linux/pci.h>
  #include <linux/delay.h>

  #include <linux/hardirq.h>

  #include <linux/scatterlist.h>

  #include <linux/blk-mq.h>

  #include <linux/ratelimit.h>

  #include <asm/unaligned.h>

  
  #include <scsi/scsi.h>

  #include <scsi/scsi_cmnd.h>

  #include <scsi/scsi_dbg.h>
  #include <scsi/scsi_device.h>
  #include <scsi/scsi_driver.h>
  #include <scsi/scsi_eh.h>
  #include <scsi/scsi_host.h>

  #include <scsi/scsi_transport.h> /* __scsi_init_queue() */

  #include <scsi/scsi_dh.h>

  #include <trace/events/scsi.h>

  #include "scsi_debugfs.h"

  #include "scsi_priv.h"
  #include "scsi_logging.h"

  /*
   * Size of integrity metadata is usually small, 1 inline sg should
   * cover normal cases.
   */

  #ifdef CONFIG_ARCH_NO_SG_CHAIN
  #define  SCSI_INLINE_PROT_SG_CNT  0
  #define  SCSI_INLINE_SG_CNT  0
  #else

  #define  SCSI_INLINE_PROT_SG_CNT  1

  #define  SCSI_INLINE_SG_CNT  2

  #endif

  static struct kmem_cache *scsi_sdb_cache;

  static struct kmem_cache *scsi_sense_cache;
  static struct kmem_cache *scsi_sense_isadma_cache;
  static DEFINE_MUTEX(scsi_sense_cache_mutex);

  static void scsi_mq_uninit_cmd(struct scsi_cmnd *cmd);

  static inline struct kmem_cache *

  scsi_select_sense_cache(bool unchecked_isa_dma)

  {

  	return unchecked_isa_dma ? scsi_sense_isadma_cache : scsi_sense_cache;

  }

  static void scsi_free_sense_buffer(bool unchecked_isa_dma,
  				   unsigned char *sense_buffer)

  {

  	kmem_cache_free(scsi_select_sense_cache(unchecked_isa_dma),
  			sense_buffer);

  }

  static unsigned char *scsi_alloc_sense_buffer(bool unchecked_isa_dma,

  	gfp_t gfp_mask, int numa_node)

  {

  	return kmem_cache_alloc_node(scsi_select_sense_cache(unchecked_isa_dma),
  				     gfp_mask, numa_node);

  }
  
  int scsi_init_sense_cache(struct Scsi_Host *shost)
  {
  	struct kmem_cache *cache;
  	int ret = 0;

  	mutex_lock(&scsi_sense_cache_mutex);

  	cache = scsi_select_sense_cache(shost->unchecked_isa_dma);

  	if (cache)

  		goto exit;

  	if (shost->unchecked_isa_dma) {
  		scsi_sense_isadma_cache =
  			kmem_cache_create("scsi_sense_cache(DMA)",

  				SCSI_SENSE_BUFFERSIZE, 0,
  				SLAB_HWCACHE_ALIGN | SLAB_CACHE_DMA, NULL);

  		if (!scsi_sense_isadma_cache)
  			ret = -ENOMEM;
  	} else {
  		scsi_sense_cache =

  			kmem_cache_create_usercopy("scsi_sense_cache",
  				SCSI_SENSE_BUFFERSIZE, 0, SLAB_HWCACHE_ALIGN,
  				0, SCSI_SENSE_BUFFERSIZE, NULL);

  		if (!scsi_sense_cache)
  			ret = -ENOMEM;
  	}

   exit:

  	mutex_unlock(&scsi_sense_cache_mutex);
  	return ret;
  }

  /*
   * When to reinvoke queueing after a resource shortage. It's 3 msecs to
   * not change behaviour from the previous unplug mechanism, experimentation
   * may prove this needs changing.
   */
  #define SCSI_QUEUE_DELAY	3

  static void
  scsi_set_blocked(struct scsi_cmnd *cmd, int reason)

  {
  	struct Scsi_Host *host = cmd->device->host;
  	struct scsi_device *device = cmd->device;

  	struct scsi_target *starget = scsi_target(device);

  
  	/*

  	 * Set the appropriate busy bit for the device/host.

  	 *
  	 * If the host/device isn't busy, assume that something actually
  	 * completed, and that we should be able to queue a command now.
  	 *
  	 * Note that the prior mid-layer assumption that any host could
  	 * always queue at least one command is now broken.  The mid-layer
  	 * will implement a user specifiable stall (see
  	 * scsi_host.max_host_blocked and scsi_device.max_device_blocked)
  	 * if a command is requeued with no other commands outstanding
  	 * either for the device or for the host.
  	 */

  	switch (reason) {
  	case SCSI_MLQUEUE_HOST_BUSY:

  		atomic_set(&host->host_blocked, host->max_host_blocked);

  		break;
  	case SCSI_MLQUEUE_DEVICE_BUSY:

  	case SCSI_MLQUEUE_EH_RETRY:

  		atomic_set(&device->device_blocked,
  			   device->max_device_blocked);

  		break;
  	case SCSI_MLQUEUE_TARGET_BUSY:

  		atomic_set(&starget->target_blocked,
  			   starget->max_target_blocked);

  		break;
  	}

  }

  static void scsi_mq_requeue_cmd(struct scsi_cmnd *cmd)
  {

  	if (cmd->request->rq_flags & RQF_DONTPREP) {
  		cmd->request->rq_flags &= ~RQF_DONTPREP;
  		scsi_mq_uninit_cmd(cmd);
  	} else {
  		WARN_ON_ONCE(true);
  	}

  	blk_mq_requeue_request(cmd->request, true);

  }

  /**
   * __scsi_queue_insert - private queue insertion
   * @cmd: The SCSI command being requeued
   * @reason:  The reason for the requeue
   * @unbusy: Whether the queue should be unbusied
   *
   * This is a private queue insertion.  The public interface
   * scsi_queue_insert() always assumes the queue should be unbusied
   * because it's always called before the completion.  This function is
   * for a requeue after completion, which should only occur in this
   * file.
   */

  static void __scsi_queue_insert(struct scsi_cmnd *cmd, int reason, bool unbusy)

  {
  	struct scsi_device *device = cmd->device;

  
  	SCSI_LOG_MLQUEUE(1, scmd_printk(KERN_INFO, cmd,
  		"Inserting command %p into mlqueue
  ", cmd));
  
  	scsi_set_blocked(cmd, reason);

  
  	/*

  	 * Decrement the counters, since these commands are no longer
  	 * active on the host/device.
  	 */

  	if (unbusy)

  		scsi_device_unbusy(device, cmd);

  
  	/*

  	 * Requeue this command.  It will go before all other commands

  	 * that are already in the queue. Schedule requeue work under
  	 * lock such that the kblockd_schedule_work() call happens
  	 * before blk_cleanup_queue() finishes.

  	 */

  	cmd->result = 0;

  	blk_mq_requeue_request(cmd->request, true);

  }

  /*
   * Function:    scsi_queue_insert()
   *
   * Purpose:     Insert a command in the midlevel queue.
   *
   * Arguments:   cmd    - command that we are adding to queue.
   *              reason - why we are inserting command to queue.
   *
   * Lock status: Assumed that lock is not held upon entry.
   *
   * Returns:     Nothing.
   *
   * Notes:       We do this for one of two cases.  Either the host is busy
   *              and it cannot accept any more commands for the time being,
   *              or the device returned QUEUE_FULL and can accept no more
   *              commands.
   * Notes:       This could be called either from an interrupt context or a
   *              normal process context.
   */

  void scsi_queue_insert(struct scsi_cmnd *cmd, int reason)

  {

  	__scsi_queue_insert(cmd, reason, true);

  }

  
  /**

   * __scsi_execute - insert request and wait for the result

   * @sdev:	scsi device
   * @cmd:	scsi command
   * @data_direction: data direction
   * @buffer:	data buffer
   * @bufflen:	len of buffer
   * @sense:	optional sense buffer
   * @sshdr:	optional decoded sense header
   * @timeout:	request timeout in seconds
   * @retries:	number of times to retry request
   * @flags:	flags for ->cmd_flags
   * @rq_flags:	flags for ->rq_flags
   * @resid:	optional residual length
   *

   * Returns the scsi_cmnd result field if a command was executed, or a negative
   * Linux error code if we didn't get that far.

   */

  int __scsi_execute(struct scsi_device *sdev, const unsigned char *cmd,

  		 int data_direction, void *buffer, unsigned bufflen,

  		 unsigned char *sense, struct scsi_sense_hdr *sshdr,
  		 int timeout, int retries, u64 flags, req_flags_t rq_flags,
  		 int *resid)

  {
  	struct request *req;

  	struct scsi_request *rq;

  	int ret = DRIVER_ERROR << 24;

  	req = blk_get_request(sdev->request_queue,

  			data_direction == DMA_TO_DEVICE ?

  			REQ_OP_SCSI_OUT : REQ_OP_SCSI_IN, BLK_MQ_REQ_PREEMPT);

  	if (IS_ERR(req))

  		return ret;

  	rq = scsi_req(req);

  
  	if (bufflen &&	blk_rq_map_kern(sdev->request_queue, req,

  					buffer, bufflen, GFP_NOIO))

  		goto out;

  	rq->cmd_len = COMMAND_SIZE(cmd[0]);
  	memcpy(rq->cmd, cmd, rq->cmd_len);

  	rq->retries = retries;

  	req->timeout = timeout;

  	req->cmd_flags |= flags;

  	req->rq_flags |= rq_flags | RQF_QUIET;

  
  	/*
  	 * head injection *required* here otherwise quiesce won't work
  	 */
  	blk_execute_rq(req->q, NULL, req, 1);

  	/*
  	 * Some devices (USB mass-storage in particular) may transfer
  	 * garbage data together with a residue indicating that the data
  	 * is invalid.  Prevent the garbage from being misinterpreted
  	 * and prevent security leaks by zeroing out the excess data.
  	 */

  	if (unlikely(rq->resid_len > 0 && rq->resid_len <= bufflen))
  		memset(buffer + (bufflen - rq->resid_len), 0, rq->resid_len);

  	if (resid)

  		*resid = rq->resid_len;
  	if (sense && rq->sense_len)
  		memcpy(sense, rq->sense, SCSI_SENSE_BUFFERSIZE);

  	if (sshdr)
  		scsi_normalize_sense(rq->sense, rq->sense_len, sshdr);

  	ret = rq->result;

   out:
  	blk_put_request(req);
  
  	return ret;
  }

  EXPORT_SYMBOL(__scsi_execute);

  /*
   * Function:    scsi_init_cmd_errh()
   *
   * Purpose:     Initialize cmd fields related to error handling.
   *
   * Arguments:   cmd	- command that is ready to be queued.
   *

   * Notes:       This function has the job of initializing a number of
   *              fields related to error handling.   Typically this will
   *              be called once for each command, as required.
   */

  static void scsi_init_cmd_errh(struct scsi_cmnd *cmd)

  {

  	scsi_set_resid(cmd, 0);

  	memset(cmd->sense_buffer, 0, SCSI_SENSE_BUFFERSIZE);

  	if (cmd->cmd_len == 0)

  		cmd->cmd_len = scsi_command_size(cmd->cmnd);

  }

  /*

   * Wake up the error handler if necessary. Avoid as follows that the error
   * handler is not woken up if host in-flight requests number ==
   * shost->host_failed: use call_rcu() in scsi_eh_scmd_add() in combination
   * with an RCU read lock in this function to ensure that this function in
   * its entirety either finishes before scsi_eh_scmd_add() increases the

   * host_failed counter or that it notices the shost state change made by
   * scsi_eh_scmd_add().
   */

  static void scsi_dec_host_busy(struct Scsi_Host *shost, struct scsi_cmnd *cmd)

  {

  	unsigned long flags;

  	rcu_read_lock();

  	__clear_bit(SCMD_STATE_INFLIGHT, &cmd->state);

  	if (unlikely(scsi_host_in_recovery(shost))) {

  		spin_lock_irqsave(shost->host_lock, flags);

  		if (shost->host_failed || shost->host_eh_scheduled)
  			scsi_eh_wakeup(shost);

  		spin_unlock_irqrestore(shost->host_lock, flags);
  	}

  	rcu_read_unlock();
  }

  void scsi_device_unbusy(struct scsi_device *sdev, struct scsi_cmnd *cmd)

  {
  	struct Scsi_Host *shost = sdev->host;
  	struct scsi_target *starget = scsi_target(sdev);

  	scsi_dec_host_busy(shost, cmd);

  
  	if (starget->can_queue > 0)
  		atomic_dec(&starget->target_busy);

  	atomic_dec(&sdev->device_busy);

  }

  static void scsi_kick_queue(struct request_queue *q)
  {

  	blk_mq_run_hw_queues(q, false);

  }

  /*
   * Called for single_lun devices on IO completion. Clear starget_sdev_user,
   * and call blk_run_queue for all the scsi_devices on the target -
   * including current_sdev first.
   *
   * Called with *no* scsi locks held.
   */
  static void scsi_single_lun_run(struct scsi_device *current_sdev)
  {
  	struct Scsi_Host *shost = current_sdev->host;
  	struct scsi_device *sdev, *tmp;
  	struct scsi_target *starget = scsi_target(current_sdev);
  	unsigned long flags;
  
  	spin_lock_irqsave(shost->host_lock, flags);
  	starget->starget_sdev_user = NULL;
  	spin_unlock_irqrestore(shost->host_lock, flags);
  
  	/*
  	 * Call blk_run_queue for all LUNs on the target, starting with
  	 * current_sdev. We race with others (to set starget_sdev_user),
  	 * but in most cases, we will be first. Ideally, each LU on the
  	 * target would get some limited time or requests on the target.
  	 */

  	scsi_kick_queue(current_sdev->request_queue);

  
  	spin_lock_irqsave(shost->host_lock, flags);
  	if (starget->starget_sdev_user)
  		goto out;
  	list_for_each_entry_safe(sdev, tmp, &starget->devices,
  			same_target_siblings) {
  		if (sdev == current_sdev)
  			continue;
  		if (scsi_device_get(sdev))
  			continue;
  
  		spin_unlock_irqrestore(shost->host_lock, flags);

  		scsi_kick_queue(sdev->request_queue);

  		spin_lock_irqsave(shost->host_lock, flags);
  	
  		scsi_device_put(sdev);
  	}
   out:
  	spin_unlock_irqrestore(shost->host_lock, flags);
  }

  static inline bool scsi_device_is_busy(struct scsi_device *sdev)

  {

  	if (atomic_read(&sdev->device_busy) >= sdev->queue_depth)
  		return true;
  	if (atomic_read(&sdev->device_blocked) > 0)
  		return true;
  	return false;

  }

  static inline bool scsi_target_is_busy(struct scsi_target *starget)

  {

  	if (starget->can_queue > 0) {
  		if (atomic_read(&starget->target_busy) >= starget->can_queue)
  			return true;
  		if (atomic_read(&starget->target_blocked) > 0)
  			return true;
  	}

  	return false;

  }

  static inline bool scsi_host_is_busy(struct Scsi_Host *shost)

  {

  	if (atomic_read(&shost->host_blocked) > 0)
  		return true;
  	if (shost->host_self_blocked)
  		return true;
  	return false;

  }

  static void scsi_starved_list_run(struct Scsi_Host *shost)

  {

  	LIST_HEAD(starved_list);

  	struct scsi_device *sdev;

  	unsigned long flags;

  	spin_lock_irqsave(shost->host_lock, flags);

  	list_splice_init(&shost->starved_list, &starved_list);
  
  	while (!list_empty(&starved_list)) {

  		struct request_queue *slq;

  		/*
  		 * As long as shost is accepting commands and we have
  		 * starved queues, call blk_run_queue. scsi_request_fn
  		 * drops the queue_lock and can add us back to the
  		 * starved_list.
  		 *
  		 * host_lock protects the starved_list and starved_entry.
  		 * scsi_request_fn must get the host_lock before checking
  		 * or modifying starved_list or starved_entry.
  		 */

  		if (scsi_host_is_busy(shost))

  			break;

  		sdev = list_entry(starved_list.next,
  				  struct scsi_device, starved_entry);
  		list_del_init(&sdev->starved_entry);

  		if (scsi_target_is_busy(scsi_target(sdev))) {
  			list_move_tail(&sdev->starved_entry,
  				       &shost->starved_list);
  			continue;
  		}

  		/*
  		 * Once we drop the host lock, a racing scsi_remove_device()
  		 * call may remove the sdev from the starved list and destroy
  		 * it and the queue.  Mitigate by taking a reference to the
  		 * queue and never touching the sdev again after we drop the
  		 * host lock.  Note: if __scsi_remove_device() invokes
  		 * blk_cleanup_queue() before the queue is run from this
  		 * function then blk_run_queue() will return immediately since
  		 * blk_cleanup_queue() marks the queue with QUEUE_FLAG_DYING.
  		 */
  		slq = sdev->request_queue;
  		if (!blk_get_queue(slq))
  			continue;
  		spin_unlock_irqrestore(shost->host_lock, flags);

  		scsi_kick_queue(slq);

  		blk_put_queue(slq);
  
  		spin_lock_irqsave(shost->host_lock, flags);

  	}

  	/* put any unprocessed entries back */
  	list_splice(&starved_list, &shost->starved_list);

  	spin_unlock_irqrestore(shost->host_lock, flags);

  }
  
  /*
   * Function:   scsi_run_queue()
   *
   * Purpose:    Select a proper request queue to serve next
   *
   * Arguments:  q       - last request's queue
   *
   * Returns:     Nothing
   *
   * Notes:      The previous command was completely finished, start
   *             a new one if possible.
   */
  static void scsi_run_queue(struct request_queue *q)
  {
  	struct scsi_device *sdev = q->queuedata;
  
  	if (scsi_target(sdev)->single_lun)
  		scsi_single_lun_run(sdev);
  	if (!list_empty(&sdev->host->starved_list))
  		scsi_starved_list_run(sdev->host);

  	blk_mq_run_hw_queues(q, false);

  }

  void scsi_requeue_run_queue(struct work_struct *work)
  {
  	struct scsi_device *sdev;
  	struct request_queue *q;
  
  	sdev = container_of(work, struct scsi_device, requeue_work);
  	q = sdev->request_queue;
  	scsi_run_queue(q);
  }

  void scsi_run_host_queues(struct Scsi_Host *shost)
  {
  	struct scsi_device *sdev;
  
  	shost_for_each_device(sdev, shost)
  		scsi_run_queue(sdev->request_queue);
  }

  static void scsi_uninit_cmd(struct scsi_cmnd *cmd)
  {

  	if (!blk_rq_is_passthrough(cmd->request)) {

  		struct scsi_driver *drv = scsi_cmd_to_driver(cmd);
  
  		if (drv->uninit_command)
  			drv->uninit_command(cmd);
  	}
  }
  
  static void scsi_mq_free_sgtables(struct scsi_cmnd *cmd)
  {
  	if (cmd->sdb.table.nents)

  		sg_free_table_chained(&cmd->sdb.table,
  				SCSI_INLINE_SG_CNT);

  	if (scsi_prot_sg_count(cmd))

  		sg_free_table_chained(&cmd->prot_sdb->table,
  				SCSI_INLINE_PROT_SG_CNT);

  }
  
  static void scsi_mq_uninit_cmd(struct scsi_cmnd *cmd)
  {

  	scsi_mq_free_sgtables(cmd);
  	scsi_uninit_cmd(cmd);

  }

  /* Returns false when no more bytes to process, true if there are more */

  static bool scsi_end_request(struct request *req, blk_status_t error,

  		unsigned int bytes)

  {

  	struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(req);

  	struct scsi_device *sdev = cmd->device;
  	struct request_queue *q = sdev->request_queue;

  
  	if (blk_update_request(req, error, bytes))
  		return true;

  	if (blk_queue_add_random(q))
  		add_disk_randomness(req->rq_disk);

  	if (!blk_rq_is_scsi(req)) {
  		WARN_ON_ONCE(!(cmd->flags & SCMD_INITIALIZED));
  		cmd->flags &= ~SCMD_INITIALIZED;
  	}

  	/*

  	 * Calling rcu_barrier() is not necessary here because the
  	 * SCSI error handler guarantees that the function called by
  	 * call_rcu() has been called before scsi_end_request() is
  	 * called.
  	 */
  	destroy_rcu_head(&cmd->rcu);
  
  	/*

  	 * In the MQ case the command gets freed by __blk_mq_end_request,
  	 * so we have to do all cleanup that depends on it earlier.
  	 *
  	 * We also can't kick the queues from irq context, so we
  	 * will have to defer it to a workqueue.
  	 */
  	scsi_mq_uninit_cmd(cmd);

  	/*
  	 * queue is still alive, so grab the ref for preventing it
  	 * from being cleaned up during running queue.
  	 */
  	percpu_ref_get(&q->q_usage_counter);

  	__blk_mq_end_request(req, error);

  	if (scsi_target(sdev)->single_lun ||
  	    !list_empty(&sdev->host->starved_list))
  		kblockd_schedule_work(&sdev->requeue_work);
  	else
  		blk_mq_run_hw_queues(q, true);

  	percpu_ref_put(&q->q_usage_counter);

  	return false;
  }

  /**

   * scsi_result_to_blk_status - translate a SCSI result code into blk_status_t
   * @cmd:	SCSI command

   * @result:	scsi error code
   *

   * Translate a SCSI result code into a blk_status_t value. May reset the host
   * byte of @cmd->result.

   */

  static blk_status_t scsi_result_to_blk_status(struct scsi_cmnd *cmd, int result)

  {

  	switch (host_byte(result)) {

  	case DID_OK:
  		/*
  		 * Also check the other bytes than the status byte in result
  		 * to handle the case when a SCSI LLD sets result to
  		 * DRIVER_SENSE << 24 without setting SAM_STAT_CHECK_CONDITION.
  		 */
  		if (scsi_status_is_good(result) && (result & ~0xff) == 0)
  			return BLK_STS_OK;
  		return BLK_STS_IOERR;

  	case DID_TRANSPORT_FAILFAST:

  		return BLK_STS_TRANSPORT;

  	case DID_TARGET_FAILURE:

  		set_host_byte(cmd, DID_OK);

  		return BLK_STS_TARGET;

  	case DID_NEXUS_FAILURE:

  		set_host_byte(cmd, DID_OK);

  		return BLK_STS_NEXUS;

  	case DID_ALLOC_FAILURE:
  		set_host_byte(cmd, DID_OK);

  		return BLK_STS_NOSPC;

  	case DID_MEDIUM_ERROR:
  		set_host_byte(cmd, DID_OK);

  		return BLK_STS_MEDIUM;

  	default:

  		return BLK_STS_IOERR;

  	}

  }

  /* Helper for scsi_io_completion() when "reprep" action required. */
  static void scsi_io_completion_reprep(struct scsi_cmnd *cmd,
  				      struct request_queue *q)
  {
  	/* A new command will be prepared and issued. */

  	scsi_mq_requeue_cmd(cmd);

  }

  /* Helper for scsi_io_completion() when special action required. */
  static void scsi_io_completion_action(struct scsi_cmnd *cmd, int result)

  {

  	struct request_queue *q = cmd->device->request_queue;

  	struct request *req = cmd->request;

  	int level = 0;

  	enum {ACTION_FAIL, ACTION_REPREP, ACTION_RETRY,
  	      ACTION_DELAYED_RETRY} action;

  	unsigned long wait_for = (cmd->allowed + 1) * req->timeout;

  	struct scsi_sense_hdr sshdr;
  	bool sense_valid;
  	bool sense_current = true;      /* false implies "deferred sense" */
  	blk_status_t blk_stat;

  	sense_valid = scsi_command_normalize_sense(cmd, &sshdr);
  	if (sense_valid)
  		sense_current = !scsi_sense_is_deferred(&sshdr);

  	blk_stat = scsi_result_to_blk_status(cmd, result);

  	if (host_byte(result) == DID_RESET) {
  		/* Third party bus reset or reset for error recovery
  		 * reasons.  Just retry the command and see what
  		 * happens.
  		 */
  		action = ACTION_RETRY;

  	} else if (sense_valid && sense_current) {

  		switch (sshdr.sense_key) {
  		case UNIT_ATTENTION:
  			if (cmd->device->removable) {

  				/* Detected disc change.  Set a bit

  				 * and quietly refuse further access.
  				 */
  				cmd->device->changed = 1;

  				action = ACTION_FAIL;

  			} else {

  				/* Must have been a power glitch, or a
  				 * bus reset.  Could not have been a
  				 * media change, so we just retry the

  				 * command and see what happens.

  				 */

  				action = ACTION_RETRY;

  			}
  			break;
  		case ILLEGAL_REQUEST:

  			/* If we had an ILLEGAL REQUEST returned, then
  			 * we may have performed an unsupported
  			 * command.  The only thing this should be
  			 * would be a ten byte read where only a six
  			 * byte read was supported.  Also, on a system
  			 * where READ CAPACITY failed, we may have
  			 * read past the end of the disk.
  			 */

  			if ((cmd->device->use_10_for_rw &&
  			    sshdr.asc == 0x20 && sshdr.ascq == 0x00) &&

  			    (cmd->cmnd[0] == READ_10 ||
  			     cmd->cmnd[0] == WRITE_10)) {

  				/* This will issue a new 6-byte command. */

  				cmd->device->use_10_for_rw = 0;

  				action = ACTION_REPREP;

  			} else if (sshdr.asc == 0x10) /* DIX */ {

  				action = ACTION_FAIL;

  				blk_stat = BLK_STS_PROTECTION;

  			/* INVALID COMMAND OPCODE or INVALID FIELD IN CDB */

  			} else if (sshdr.asc == 0x20 || sshdr.asc == 0x24) {

  				action = ACTION_FAIL;

  				blk_stat = BLK_STS_TARGET;

  			} else
  				action = ACTION_FAIL;
  			break;

  		case ABORTED_COMMAND:

  			action = ACTION_FAIL;

  			if (sshdr.asc == 0x10) /* DIF */

  				blk_stat = BLK_STS_PROTECTION;

  			break;
  		case NOT_READY:

  			/* If the device is in the process of becoming

  			 * ready, or has a temporary blockage, retry.

  			 */

  			if (sshdr.asc == 0x04) {
  				switch (sshdr.ascq) {
  				case 0x01: /* becoming ready */
  				case 0x04: /* format in progress */
  				case 0x05: /* rebuild in progress */
  				case 0x06: /* recalculation in progress */
  				case 0x07: /* operation in progress */
  				case 0x08: /* Long write in progress */
  				case 0x09: /* self test in progress */

  				case 0x14: /* space allocation in progress */

  				case 0x1a: /* start stop unit in progress */
  				case 0x1b: /* sanitize in progress */
  				case 0x1d: /* configuration in progress */
  				case 0x24: /* depopulation in progress */

  					action = ACTION_DELAYED_RETRY;

  					break;

  				default:

  					action = ACTION_FAIL;
  					break;

  				}

  			} else

  				action = ACTION_FAIL;

  			break;

  		case VOLUME_OVERFLOW:

  			/* See SSC3rXX or current. */

  			action = ACTION_FAIL;
  			break;

  		default:

  			action = ACTION_FAIL;

  			break;
  		}

  	} else

  		action = ACTION_FAIL;

  	if (action != ACTION_FAIL &&

  	    time_before(cmd->jiffies_at_alloc + wait_for, jiffies))

  		action = ACTION_FAIL;

  	switch (action) {
  	case ACTION_FAIL:
  		/* Give up and fail the remainder of the request */

  		if (!(req->rq_flags & RQF_QUIET)) {

  			static DEFINE_RATELIMIT_STATE(_rs,
  					DEFAULT_RATELIMIT_INTERVAL,
  					DEFAULT_RATELIMIT_BURST);
  
  			if (unlikely(scsi_logging_level))

  				level =
  				     SCSI_LOG_LEVEL(SCSI_LOG_MLCOMPLETE_SHIFT,
  						    SCSI_LOG_MLCOMPLETE_BITS);

  
  			/*
  			 * if logging is enabled the failure will be printed
  			 * in scsi_log_completion(), so avoid duplicate messages
  			 */
  			if (!level && __ratelimit(&_rs)) {
  				scsi_print_result(cmd, NULL, FAILED);

  				if (driver_byte(result) == DRIVER_SENSE)

  					scsi_print_sense(cmd);
  				scsi_print_command(cmd);
  			}

  		}

  		if (!scsi_end_request(req, blk_stat, blk_rq_err_bytes(req)))

  			return;

  		/*FALLTHRU*/

  	case ACTION_REPREP:

  		scsi_io_completion_reprep(cmd, q);

  		break;
  	case ACTION_RETRY:
  		/* Retry the same command immediately */

  		__scsi_queue_insert(cmd, SCSI_MLQUEUE_EH_RETRY, false);

  		break;
  	case ACTION_DELAYED_RETRY:
  		/* Retry the same command after a delay */

  		__scsi_queue_insert(cmd, SCSI_MLQUEUE_DEVICE_BUSY, false);

  		break;

  	}
  }

  /*

   * Helper for scsi_io_completion() when cmd->result is non-zero. Returns a
   * new result that may suppress further error checking. Also modifies
   * *blk_statp in some cases.
   */
  static int scsi_io_completion_nz_result(struct scsi_cmnd *cmd, int result,
  					blk_status_t *blk_statp)
  {
  	bool sense_valid;
  	bool sense_current = true;	/* false implies "deferred sense" */
  	struct request *req = cmd->request;
  	struct scsi_sense_hdr sshdr;
  
  	sense_valid = scsi_command_normalize_sense(cmd, &sshdr);
  	if (sense_valid)
  		sense_current = !scsi_sense_is_deferred(&sshdr);
  
  	if (blk_rq_is_passthrough(req)) {
  		if (sense_valid) {
  			/*
  			 * SG_IO wants current and deferred errors
  			 */
  			scsi_req(req)->sense_len =
  				min(8 + cmd->sense_buffer[7],
  				    SCSI_SENSE_BUFFERSIZE);
  		}
  		if (sense_current)
  			*blk_statp = scsi_result_to_blk_status(cmd, result);
  	} else if (blk_rq_bytes(req) == 0 && sense_current) {
  		/*
  		 * Flush commands do not transfers any data, and thus cannot use
  		 * good_bytes != blk_rq_bytes(req) as the signal for an error.
  		 * This sets *blk_statp explicitly for the problem case.
  		 */
  		*blk_statp = scsi_result_to_blk_status(cmd, result);
  	}
  	/*
  	 * Recovered errors need reporting, but they're always treated as
  	 * success, so fiddle the result code here.  For passthrough requests
  	 * we already took a copy of the original into sreq->result which
  	 * is what gets returned to the user
  	 */
  	if (sense_valid && (sshdr.sense_key == RECOVERED_ERROR)) {
  		bool do_print = true;
  		/*
  		 * if ATA PASS-THROUGH INFORMATION AVAILABLE [0x0, 0x1d]
  		 * skip print since caller wants ATA registers. Only occurs
  		 * on SCSI ATA PASS_THROUGH commands when CK_COND=1
  		 */
  		if ((sshdr.asc == 0x0) && (sshdr.ascq == 0x1d))
  			do_print = false;
  		else if (req->rq_flags & RQF_QUIET)
  			do_print = false;
  		if (do_print)
  			scsi_print_sense(cmd);
  		result = 0;
  		/* for passthrough, *blk_statp may be set */
  		*blk_statp = BLK_STS_OK;
  	}
  	/*
  	 * Another corner case: the SCSI status byte is non-zero but 'good'.
  	 * Example: PRE-FETCH command returns SAM_STAT_CONDITION_MET when
  	 * it is able to fit nominated LBs in its cache (and SAM_STAT_GOOD
  	 * if it can't fit). Treat SAM_STAT_CONDITION_MET and the related
  	 * intermediate statuses (both obsolete in SAM-4) as good.
  	 */
  	if (status_byte(result) && scsi_status_is_good(result)) {
  		result = 0;
  		*blk_statp = BLK_STS_OK;
  	}
  	return result;
  }
  
  /*

   * Function:    scsi_io_completion()
   *
   * Purpose:     Completion processing for block device I/O requests.
   *
   * Arguments:   cmd   - command that is finished.
   *
   * Lock status: Assumed that no lock is held upon entry.
   *
   * Returns:     Nothing
   *

   * Notes:       We will finish off the specified number of sectors.  If we
   *		are done, the command block will be released and the queue
   *		function will be goosed.  If we are not done then we have to

   *		figure out what to do next:

   *

   *		a) We can call scsi_requeue_command().  The request
   *		   will be unprepared and put back on the queue.  Then
   *		   a new command will be created for it.  This should
   *		   be used if we made forward progress, or if we want
   *		   to switch from READ(10) to READ(6) for example.

   *

   *		b) We can call __scsi_queue_insert().  The request will

   *		   be put back on the queue and retried using the same
   *		   command as before, possibly after a delay.
   *

   *		c) We can call scsi_end_request() with blk_stat other than
   *		   BLK_STS_OK, to fail the remainder of the request.

   */

  void scsi_io_completion(struct scsi_cmnd *cmd, unsigned int good_bytes)

  {
  	int result = cmd->result;

  	struct request_queue *q = cmd->device->request_queue;

  	struct request *req = cmd->request;

  	blk_status_t blk_stat = BLK_STS_OK;

  	if (unlikely(result))	/* a nz result may or may not be an error */

  		result = scsi_io_completion_nz_result(cmd, result, &blk_stat);

  	if (unlikely(blk_rq_is_passthrough(req))) {

  		/*

  		 * scsi_result_to_blk_status may have reset the host_byte

  		 */

  		scsi_req(req)->result = cmd->result;

  	}

  	/*

  	 * Next deal with any sectors which we were able to correctly
  	 * handle.
  	 */

  	SCSI_LOG_HLCOMPLETE(1, scmd_printk(KERN_INFO, cmd,
  		"%u sectors total, %d bytes done.
  ",
  		blk_rq_sectors(req), good_bytes));

  	/*

  	 * Next deal with any sectors which we were able to correctly
  	 * handle. Failed, zero length commands always need to drop down
  	 * to retry code. Fast path should return in this block.

  	 */

  	if (likely(blk_rq_bytes(req) > 0 || blk_stat == BLK_STS_OK)) {

  		if (likely(!scsi_end_request(req, blk_stat, good_bytes)))

  			return; /* no bytes remaining */
  	}

  	/* Kill remainder if no retries. */
  	if (unlikely(blk_stat && scsi_noretry_cmd(cmd))) {

  		if (scsi_end_request(req, blk_stat, blk_rq_bytes(req)))

  			WARN_ONCE(true,
  			    "Bytes remaining after failed, no-retry command");

  		return;

  	}
  
  	/*
  	 * If there had been no error, but we have leftover bytes in the
  	 * requeues just queue the command up again.

  	 */

  	if (likely(result == 0))

  		scsi_io_completion_reprep(cmd, q);
  	else

  		scsi_io_completion_action(cmd, result);

  }

  static blk_status_t scsi_init_sgtable(struct request *req,
  		struct scsi_data_buffer *sdb)

  {

  	int count;

  
  	/*

  	 * If sg table allocation fails, requeue request later.

  	 */

  	if (unlikely(sg_alloc_table_chained(&sdb->table,

  			blk_rq_nr_phys_segments(req), sdb->table.sgl,

  			SCSI_INLINE_SG_CNT)))

  		return BLK_STS_RESOURCE;

  	/* 
  	 * Next, walk the list, and fill in the addresses and sizes of
  	 * each segment.
  	 */

  	count = blk_rq_map_sg(req->q, req, sdb->table.sgl);
  	BUG_ON(count > sdb->table.nents);
  	sdb->table.nents = count;

  	sdb->length = blk_rq_payload_bytes(req);

  	return BLK_STS_OK;

  }

  
  /*
   * Function:    scsi_init_io()
   *
   * Purpose:     SCSI I/O initialize function.
   *
   * Arguments:   cmd   - Command descriptor we wish to initialize
   *

   * Returns:     BLK_STS_OK on success
   *		BLK_STS_RESOURCE if the failure is retryable
   *		BLK_STS_IOERR if the failure is fatal

   */

  blk_status_t scsi_init_io(struct scsi_cmnd *cmd)

  {

  	struct request *rq = cmd->request;

  	blk_status_t ret;

  	if (WARN_ON_ONCE(!blk_rq_nr_phys_segments(rq)))

  		return BLK_STS_IOERR;

  	ret = scsi_init_sgtable(rq, &cmd->sdb);
  	if (ret)
  		return ret;

  	if (blk_integrity_rq(rq)) {

  		struct scsi_data_buffer *prot_sdb = cmd->prot_sdb;
  		int ivecs, count;

  		if (WARN_ON_ONCE(!prot_sdb)) {

  			/*
  			 * This can happen if someone (e.g. multipath)
  			 * queues a command to a device on an adapter
  			 * that does not support DIX.
  			 */

  			ret = BLK_STS_IOERR;

  			goto out_free_sgtables;

  		}

  		ivecs = blk_rq_count_integrity_sg(rq->q, rq->bio);

  		if (sg_alloc_table_chained(&prot_sdb->table, ivecs,

  				prot_sdb->table.sgl,

  				SCSI_INLINE_PROT_SG_CNT)) {

  			ret = BLK_STS_RESOURCE;

  			goto out_free_sgtables;

  		}

  		count = blk_rq_map_integrity_sg(rq->q, rq->bio,

  						prot_sdb->table.sgl);

  		BUG_ON(count > ivecs);
  		BUG_ON(count > queue_max_integrity_segments(rq->q));

  
  		cmd->prot_sdb = prot_sdb;
  		cmd->prot_sdb->table.nents = count;
  	}

  	return BLK_STS_OK;

  out_free_sgtables:

  	scsi_mq_free_sgtables(cmd);

  	return ret;

  }

  EXPORT_SYMBOL(scsi_init_io);

  /**

   * scsi_initialize_rq - initialize struct scsi_cmnd partially

   * @rq: Request associated with the SCSI command to be initialized.

   *

   * This function initializes the members of struct scsi_cmnd that must be
   * initialized before request processing starts and that won't be
   * reinitialized if a SCSI command is requeued.
   *

   * Called from inside blk_get_request() for pass-through requests and from
   * inside scsi_init_command() for filesystem requests.

   */

  static void scsi_initialize_rq(struct request *rq)

  {

  	struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(rq);
  
  	scsi_req_init(&cmd->req);

  	init_rcu_head(&cmd->rcu);

  	cmd->jiffies_at_alloc = jiffies;
  	cmd->retries = 0;

  }

  /*
   * Only called when the request isn't completed by SCSI, and not freed by
   * SCSI
   */
  static void scsi_cleanup_rq(struct request *rq)
  {
  	if (rq->rq_flags & RQF_DONTPREP) {
  		scsi_mq_uninit_cmd(blk_mq_rq_to_pdu(rq));
  		rq->rq_flags &= ~RQF_DONTPREP;
  	}
  }

  /* Called before a request is prepared. See also scsi_mq_prep_fn(). */

  void scsi_init_command(struct scsi_device *dev, struct scsi_cmnd *cmd)

  {

  	void *buf = cmd->sense_buffer;
  	void *prot = cmd->prot_sdb;

  	struct request *rq = blk_mq_rq_from_pdu(cmd);
  	unsigned int flags = cmd->flags & SCMD_PRESERVED_FLAGS;

  	unsigned long jiffies_at_alloc;

  	int retries, to_clear;

  	bool in_flight;

  
  	if (!blk_rq_is_scsi(rq) && !(flags & SCMD_INITIALIZED)) {
  		flags |= SCMD_INITIALIZED;
  		scsi_initialize_rq(rq);
  	}

  	jiffies_at_alloc = cmd->jiffies_at_alloc;
  	retries = cmd->retries;

  	in_flight = test_bit(SCMD_STATE_INFLIGHT, &cmd->state);

  	/*
  	 * Zero out the cmd, except for the embedded scsi_request. Only clear
  	 * the driver-private command data if the LLD does not supply a
  	 * function to initialize that data.
  	 */
  	to_clear = sizeof(*cmd) - sizeof(cmd->req);
  	if (!dev->host->hostt->init_cmd_priv)
  		to_clear += dev->host->hostt->cmd_size;
  	memset((char *)cmd + sizeof(cmd->req), 0, to_clear);

  	cmd->device = dev;
  	cmd->sense_buffer = buf;
  	cmd->prot_sdb = prot;

  	cmd->flags = flags;

  	INIT_DELAYED_WORK(&cmd->abort_work, scmd_eh_abort_handler);

  	cmd->jiffies_at_alloc = jiffies_at_alloc;
  	cmd->retries = retries;

  	if (in_flight)
  		__set_bit(SCMD_STATE_INFLIGHT, &cmd->state);

  }

  static blk_status_t scsi_setup_scsi_cmnd(struct scsi_device *sdev,
  		struct request *req)

  {

  	struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(req);

  
  	/*

  	 * Passthrough requests may transfer data, in which case they must

  	 * a bio attached to them.  Or they might contain a SCSI command
  	 * that does not transfer data, in which case they may optionally
  	 * submit a request without an attached bio.
  	 */
  	if (req->bio) {

  		blk_status_t ret = scsi_init_io(cmd);
  		if (unlikely(ret != BLK_STS_OK))
  			return ret;

  	} else {

  		BUG_ON(blk_rq_bytes(req));

  		memset(&cmd->sdb, 0, sizeof(cmd->sdb));

  	}

  	cmd->cmd_len = scsi_req(req)->cmd_len;
  	cmd->cmnd = scsi_req(req)->cmd;

  	cmd->transfersize = blk_rq_bytes(req);

  	cmd->allowed = scsi_req(req)->retries;

  	return BLK_STS_OK;

  }

  /*

   * Setup a normal block command.  These are simple request from filesystems

   * that still need to be translated to SCSI CDBs from the ULD.

   */

  static blk_status_t scsi_setup_fs_cmnd(struct scsi_device *sdev,
  		struct request *req)

  {

  	struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(req);

  	if (unlikely(sdev->handler && sdev->handler->prep_fn)) {

  		blk_status_t ret = sdev->handler->prep_fn(sdev, req);
  		if (ret != BLK_STS_OK)
  			return ret;

  	}

  	cmd->cmnd = scsi_req(req)->cmd = scsi_req(req)->__cmd;

  	memset(cmd->cmnd, 0, BLK_MAX_CDB);

  	return scsi_cmd_to_driver(cmd)->init_command(cmd);

  }

  static blk_status_t scsi_setup_cmnd(struct scsi_device *sdev,
  		struct request *req)

  {

  	struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(req);

  
  	if (!blk_rq_bytes(req))
  		cmd->sc_data_direction = DMA_NONE;
  	else if (rq_data_dir(req) == WRITE)
  		cmd->sc_data_direction = DMA_TO_DEVICE;
  	else
  		cmd->sc_data_direction = DMA_FROM_DEVICE;

  	if (blk_rq_is_scsi(req))
  		return scsi_setup_scsi_cmnd(sdev, req);
  	else

  		return scsi_setup_fs_cmnd(sdev, req);

  }

  static blk_status_t

  scsi_prep_state_check(struct scsi_device *sdev, struct request *req)

  {

  	switch (sdev->sdev_state) {
  	case SDEV_OFFLINE:
  	case SDEV_TRANSPORT_OFFLINE:
  		/*
  		 * If the device is offline we refuse to process any
  		 * commands.  The device must be brought online
  		 * before trying any recovery commands.
  		 */

  		if (!sdev->offline_already) {
  			sdev->offline_already = true;
  			sdev_printk(KERN_ERR, sdev,
  				    "rejecting I/O to offline device
  ");
  		}

  		return BLK_STS_IOERR;
  	case SDEV_DEL:
  		/*
  		 * If the device is fully deleted, we refuse to
  		 * process any commands as well.
  		 */
  		sdev_printk(KERN_ERR, sdev,
  			    "rejecting I/O to dead device
  ");
  		return BLK_STS_IOERR;
  	case SDEV_BLOCK:
  	case SDEV_CREATED_BLOCK:
  		return BLK_STS_RESOURCE;
  	case SDEV_QUIESCE:
  		/*
  		 * If the devices is blocked we defer normal commands.
  		 */
  		if (req && !(req->rq_flags & RQF_PREEMPT))
  			return BLK_STS_RESOURCE;
  		return BLK_STS_OK;
  	default:
  		/*
  		 * For any other not fully online state we only allow
  		 * special commands.  In particular any user initiated
  		 * command is not allowed.
  		 */
  		if (req && !(req->rq_flags & RQF_PREEMPT))
  			return BLK_STS_IOERR;
  		return BLK_STS_OK;

  	}

  }

  /*
   * scsi_dev_queue_ready: if we can send requests to sdev, return 1 else
   * return 0.
   *
   * Called with the queue_lock held.
   */
  static inline int scsi_dev_queue_ready(struct request_queue *q,
  				  struct scsi_device *sdev)
  {

  	unsigned int busy;
  
  	busy = atomic_inc_return(&sdev->device_busy) - 1;

  	if (atomic_read(&sdev->device_blocked)) {

  		if (busy)
  			goto out_dec;

  		/*
  		 * unblock after device_blocked iterates to zero
  		 */

  		if (atomic_dec_return(&sdev->device_blocked) > 0)

  			goto out_dec;

  		SCSI_LOG_MLQUEUE(3, sdev_printk(KERN_INFO, sdev,
  				   "unblocking device at zero depth
  "));

  	}

  
  	if (busy >= sdev->queue_depth)
  		goto out_dec;

  
  	return 1;

  out_dec:
  	atomic_dec(&sdev->device_busy);
  	return 0;

  }

  /*
   * scsi_target_queue_ready: checks if there we can send commands to target
   * @sdev: scsi device on starget to check.

   */
  static inline int scsi_target_queue_ready(struct Scsi_Host *shost,
  					   struct scsi_device *sdev)
  {
  	struct scsi_target *starget = scsi_target(sdev);

  	unsigned int busy;

  
  	if (starget->single_lun) {

  		spin_lock_irq(shost->host_lock);

  		if (starget->starget_sdev_user &&

  		    starget->starget_sdev_user != sdev) {
  			spin_unlock_irq(shost->host_lock);
  			return 0;
  		}

  		starget->starget_sdev_user = sdev;

  		spin_unlock_irq(shost->host_lock);

  	}

  	if (starget->can_queue <= 0)
  		return 1;

  	busy = atomic_inc_return(&starget->target_busy) - 1;

  	if (atomic_read(&starget->target_blocked) > 0) {

  		if (busy)
  			goto starved;

  		/*
  		 * unblock after target_blocked iterates to zero
  		 */

  		if (atomic_dec_return(&starget->target_blocked) > 0)

  			goto out_dec;

  
  		SCSI_LOG_MLQUEUE(3, starget_printk(KERN_INFO, starget,
  				 "unblocking target at zero depth
  "));

  	}

  	if (busy >= starget->can_queue)

  		goto starved;

  	return 1;
  
  starved:
  	spin_lock_irq(shost->host_lock);
  	list_move_tail(&sdev->starved_entry, &shost->starved_list);

  	spin_unlock_irq(shost->host_lock);

  out_dec:

  	if (starget->can_queue > 0)
  		atomic_dec(&starget->target_busy);

  	return 0;

  }

  /*
   * scsi_host_queue_ready: if we can send requests to shost, return 1 else
   * return 0. We must end up running the queue again whenever 0 is
   * returned, else IO can hang.

   */
  static inline int scsi_host_queue_ready(struct request_queue *q,
  				   struct Scsi_Host *shost,

  				   struct scsi_device *sdev,
  				   struct scsi_cmnd *cmd)

  {

  	if (scsi_host_in_recovery(shost))

  		return 0;

  	if (atomic_read(&shost->host_blocked) > 0) {

  		if (scsi_host_busy(shost) > 0)

  			goto starved;

  		/*
  		 * unblock after host_blocked iterates to zero
  		 */

  		if (atomic_dec_return(&shost->host_blocked) > 0)

  			goto out_dec;

  
  		SCSI_LOG_MLQUEUE(3,
  			shost_printk(KERN_INFO, shost,
  				     "unblocking host at zero depth
  "));

  	}

  	if (shost->host_self_blocked)
  		goto starved;

  
  	/* We're OK to process the command, so we can't be starved */

  	if (!list_empty(&sdev->starved_entry)) {
  		spin_lock_irq(shost->host_lock);
  		if (!list_empty(&sdev->starved_entry))
  			list_del_init(&sdev->starved_entry);
  		spin_unlock_irq(shost->host_lock);
  	}

  	__set_bit(SCMD_STATE_INFLIGHT, &cmd->state);

  	return 1;
  
  starved:
  	spin_lock_irq(shost->host_lock);
  	if (list_empty(&sdev->starved_entry))
  		list_add_tail(&sdev->starved_entry, &shost->starved_list);

  	spin_unlock_irq(shost->host_lock);

  out_dec:

  	scsi_dec_host_busy(shost, cmd);

  	return 0;

  }
  
  /*

   * Busy state exporting function for request stacking drivers.
   *
   * For efficiency, no lock is taken to check the busy state of
   * shost/starget/sdev, since the returned value is not guaranteed and
   * may be changed after request stacking drivers call the function,
   * regardless of taking lock or not.
   *

   * When scsi can't dispatch I/Os anymore and needs to kill I/Os scsi
   * needs to return 'not busy'. Otherwise, request stacking drivers
   * may hold requests forever.

   */

  static bool scsi_mq_lld_busy(struct request_queue *q)

  {
  	struct scsi_device *sdev = q->queuedata;
  	struct Scsi_Host *shost;

  	if (blk_queue_dying(q))

  		return false;

  
  	shost = sdev->host;

  	/*
  	 * Ignore host/starget busy state.
  	 * Since block layer does not have a concept of fairness across
  	 * multiple queues, congestion of host/starget needs to be handled
  	 * in SCSI layer.
  	 */
  	if (scsi_host_in_recovery(shost) || scsi_device_is_busy(sdev))

  		return true;

  	return false;

  }

  static void scsi_softirq_done(struct request *rq)
  {

  	struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(rq);

  	unsigned long wait_for = (cmd->allowed + 1) * rq->timeout;

  	int disposition;
  
  	INIT_LIST_HEAD(&cmd->eh_entry);

  	atomic_inc(&cmd->device->iodone_cnt);
  	if (cmd->result)
  		atomic_inc(&cmd->device->ioerr_cnt);

  	disposition = scsi_decide_disposition(cmd);
  	if (disposition != SUCCESS &&
  	    time_before(cmd->jiffies_at_alloc + wait_for, jiffies)) {

  		scmd_printk(KERN_ERR, cmd,

  			    "timing out command, waited %lus
  ",
  			    wait_for/HZ);
  		disposition = SUCCESS;
  	}

  	scsi_log_completion(cmd, disposition);
  
  	switch (disposition) {
  		case SUCCESS:
  			scsi_finish_command(cmd);
  			break;
  		case NEEDS_RETRY:

  			scsi_queue_insert(cmd, SCSI_MLQUEUE_EH_RETRY);

  			break;
  		case ADD_TO_MLQUEUE:
  			scsi_queue_insert(cmd, SCSI_MLQUEUE_DEVICE_BUSY);
  			break;
  		default:

  			scsi_eh_scmd_add(cmd);

  			break;

  	}
  }

  /**

   * scsi_dispatch_command - Dispatch a command to the low-level driver.
   * @cmd: command block we are dispatching.
   *
   * Return: nonzero return request was rejected and device's queue needs to be
   * plugged.
   */
  static int scsi_dispatch_cmd(struct scsi_cmnd *cmd)
  {
  	struct Scsi_Host *host = cmd->device->host;
  	int rtn = 0;
  
  	atomic_inc(&cmd->device->iorequest_cnt);
  
  	/* check if the device is still usable */
  	if (unlikely(cmd->device->sdev_state == SDEV_DEL)) {
  		/* in SDEV_DEL we error all commands. DID_NO_CONNECT
  		 * returns an immediate error upwards, and signals
  		 * that the device is no longer present */
  		cmd->result = DID_NO_CONNECT << 16;
  		goto done;
  	}
  
  	/* Check to see if the scsi lld made this device blocked. */
  	if (unlikely(scsi_device_blocked(cmd->device))) {
  		/*
  		 * in blocked state, the command is just put back on
  		 * the device queue.  The suspend state has already
  		 * blocked the queue so future requests should not
  		 * occur until the device transitions out of the
  		 * suspend state.
  		 */
  		SCSI_LOG_MLQUEUE(3, scmd_printk(KERN_INFO, cmd,
  			"queuecommand : device blocked
  "));
  		return SCSI_MLQUEUE_DEVICE_BUSY;
  	}
  
  	/* Store the LUN value in cmnd, if needed. */
  	if (cmd->device->lun_in_cdb)
  		cmd->cmnd[1] = (cmd->cmnd[1] & 0x1f) |
  			       (cmd->device->lun << 5 & 0xe0);
  
  	scsi_log_send(cmd);
  
  	/*
  	 * Before we queue this command, check if the command
  	 * length exceeds what the host adapter can handle.
  	 */
  	if (cmd->cmd_len > cmd->device->host->max_cmd_len) {
  		SCSI_LOG_MLQUEUE(3, scmd_printk(KERN_INFO, cmd,
  			       "queuecommand : command too long. "
  			       "cdb_size=%d host->max_cmd_len=%d
  ",
  			       cmd->cmd_len, cmd->device->host->max_cmd_len));
  		cmd->result = (DID_ABORT << 16);
  		goto done;
  	}
  
  	if (unlikely(host->shost_state == SHOST_DEL)) {
  		cmd->result = (DID_NO_CONNECT << 16);
  		goto done;
  
  	}
  
  	trace_scsi_dispatch_cmd_start(cmd);
  	rtn = host->hostt->queuecommand(host, cmd);
  	if (rtn) {
  		trace_scsi_dispatch_cmd_error(cmd, rtn);
  		if (rtn != SCSI_MLQUEUE_DEVICE_BUSY &&
  		    rtn != SCSI_MLQUEUE_TARGET_BUSY)
  			rtn = SCSI_MLQUEUE_HOST_BUSY;
  
  		SCSI_LOG_MLQUEUE(3, scmd_printk(KERN_INFO, cmd,
  			"queuecommand : request rejected
  "));
  	}
  
  	return rtn;
   done:
  	cmd->scsi_done(cmd);
  	return 0;
  }

  /* Size in bytes of the sg-list stored in the scsi-mq command-private data. */

  static unsigned int scsi_mq_inline_sgl_size(struct Scsi_Host *shost)

  {

  	return min_t(unsigned int, shost->sg_tablesize, SCSI_INLINE_SG_CNT) *

  		sizeof(struct scatterlist);
  }

  static blk_status_t scsi_mq_prep_fn(struct request *req)

  {
  	struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(req);
  	struct scsi_device *sdev = req->q->queuedata;
  	struct Scsi_Host *shost = sdev->host;

  	struct scatterlist *sg;

  	scsi_init_command(sdev, cmd);

  	cmd->request = req;