// SPDX-License-Identifier: GPL-2.0-or-later

  // SPI init/core code
  //
  // Copyright (C) 2005 David Brownell
  // Copyright (C) 2008 Secret Lab Technologies Ltd.

  #include <linux/kernel.h>
  #include <linux/device.h>
  #include <linux/init.h>
  #include <linux/cache.h>

  #include <linux/dma-mapping.h>
  #include <linux/dmaengine.h>

  #include <linux/mutex.h>

  #include <linux/of_device.h>

  #include <linux/of_irq.h>

  #include <linux/clk/clk-conf.h>

  #include <linux/slab.h>

  #include <linux/mod_devicetable.h>

  #include <linux/spi/spi.h>

  #include <linux/spi/spi-mem.h>

  #include <linux/of_gpio.h>

  #include <linux/gpio/consumer.h>

  #include <linux/pm_runtime.h>

  #include <linux/pm_domain.h>

  #include <linux/property.h>

  #include <linux/export.h>

  #include <linux/sched/rt.h>

  #include <uapi/linux/sched/types.h>

  #include <linux/delay.h>
  #include <linux/kthread.h>

  #include <linux/ioport.h>
  #include <linux/acpi.h>

  #include <linux/highmem.h>

  #include <linux/idr.h>

  #include <linux/platform_data/x86/apple.h>

  #define CREATE_TRACE_POINTS
  #include <trace/events/spi.h>

  EXPORT_TRACEPOINT_SYMBOL(spi_transfer_start);
  EXPORT_TRACEPOINT_SYMBOL(spi_transfer_stop);

  #include "internals.h"

  static DEFINE_IDR(spi_master_idr);

  static void spidev_release(struct device *dev)
  {

  	struct spi_device	*spi = to_spi_device(dev);

  	/* spi controllers may cleanup for released devices */
  	if (spi->controller->cleanup)
  		spi->controller->cleanup(spi);

  	spi_controller_put(spi->controller);

  	kfree(spi->driver_override);

  	kfree(spi);

  }
  
  static ssize_t
  modalias_show(struct device *dev, struct device_attribute *a, char *buf)
  {
  	const struct spi_device	*spi = to_spi_device(dev);

  	int len;
  
  	len = acpi_device_modalias(dev, buf, PAGE_SIZE - 1);
  	if (len != -ENODEV)
  		return len;

  	return sprintf(buf, "%s%s
  ", SPI_MODULE_PREFIX, spi->modalias);

  }

  static DEVICE_ATTR_RO(modalias);

  static ssize_t driver_override_store(struct device *dev,
  				     struct device_attribute *a,
  				     const char *buf, size_t count)
  {
  	struct spi_device *spi = to_spi_device(dev);
  	const char *end = memchr(buf, '
  ', count);
  	const size_t len = end ? end - buf : count;
  	const char *driver_override, *old;
  
  	/* We need to keep extra room for a newline when displaying value */
  	if (len >= (PAGE_SIZE - 1))
  		return -EINVAL;
  
  	driver_override = kstrndup(buf, len, GFP_KERNEL);
  	if (!driver_override)
  		return -ENOMEM;
  
  	device_lock(dev);
  	old = spi->driver_override;
  	if (len) {
  		spi->driver_override = driver_override;
  	} else {

  		/* Empty string, disable driver override */

  		spi->driver_override = NULL;
  		kfree(driver_override);
  	}
  	device_unlock(dev);
  	kfree(old);
  
  	return count;
  }
  
  static ssize_t driver_override_show(struct device *dev,
  				    struct device_attribute *a, char *buf)
  {
  	const struct spi_device *spi = to_spi_device(dev);
  	ssize_t len;
  
  	device_lock(dev);
  	len = snprintf(buf, PAGE_SIZE, "%s
  ", spi->driver_override ? : "");
  	device_unlock(dev);
  	return len;
  }
  static DEVICE_ATTR_RW(driver_override);

  #define SPI_STATISTICS_ATTRS(field, file)				\

  static ssize_t spi_controller_##field##_show(struct device *dev,	\
  					     struct device_attribute *attr, \
  					     char *buf)			\

  {									\

  	struct spi_controller *ctlr = container_of(dev,			\
  					 struct spi_controller, dev);	\
  	return spi_statistics_##field##_show(&ctlr->statistics, buf);	\

  }									\

  static struct device_attribute dev_attr_spi_controller_##field = {	\

  	.attr = { .name = file, .mode = 0444 },				\

  	.show = spi_controller_##field##_show,				\

  };									\
  static ssize_t spi_device_##field##_show(struct device *dev,		\
  					 struct device_attribute *attr,	\
  					char *buf)			\
  {									\

  	struct spi_device *spi = to_spi_device(dev);			\

  	return spi_statistics_##field##_show(&spi->statistics, buf);	\
  }									\
  static struct device_attribute dev_attr_spi_device_##field = {		\

  	.attr = { .name = file, .mode = 0444 },				\

  	.show = spi_device_##field##_show,				\
  }
  
  #define SPI_STATISTICS_SHOW_NAME(name, file, field, format_string)	\
  static ssize_t spi_statistics_##name##_show(struct spi_statistics *stat, \
  					    char *buf)			\
  {									\
  	unsigned long flags;						\
  	ssize_t len;							\
  	spin_lock_irqsave(&stat->lock, flags);				\
  	len = sprintf(buf, format_string, stat->field);			\
  	spin_unlock_irqrestore(&stat->lock, flags);			\
  	return len;							\
  }									\
  SPI_STATISTICS_ATTRS(name, file)
  
  #define SPI_STATISTICS_SHOW(field, format_string)			\
  	SPI_STATISTICS_SHOW_NAME(field, __stringify(field),		\
  				 field, format_string)
  
  SPI_STATISTICS_SHOW(messages, "%lu");
  SPI_STATISTICS_SHOW(transfers, "%lu");
  SPI_STATISTICS_SHOW(errors, "%lu");
  SPI_STATISTICS_SHOW(timedout, "%lu");
  
  SPI_STATISTICS_SHOW(spi_sync, "%lu");
  SPI_STATISTICS_SHOW(spi_sync_immediate, "%lu");
  SPI_STATISTICS_SHOW(spi_async, "%lu");
  
  SPI_STATISTICS_SHOW(bytes, "%llu");
  SPI_STATISTICS_SHOW(bytes_rx, "%llu");
  SPI_STATISTICS_SHOW(bytes_tx, "%llu");

  #define SPI_STATISTICS_TRANSFER_BYTES_HISTO(index, number)		\
  	SPI_STATISTICS_SHOW_NAME(transfer_bytes_histo##index,		\
  				 "transfer_bytes_histo_" number,	\
  				 transfer_bytes_histo[index],  "%lu")
  SPI_STATISTICS_TRANSFER_BYTES_HISTO(0,  "0-1");
  SPI_STATISTICS_TRANSFER_BYTES_HISTO(1,  "2-3");
  SPI_STATISTICS_TRANSFER_BYTES_HISTO(2,  "4-7");
  SPI_STATISTICS_TRANSFER_BYTES_HISTO(3,  "8-15");
  SPI_STATISTICS_TRANSFER_BYTES_HISTO(4,  "16-31");
  SPI_STATISTICS_TRANSFER_BYTES_HISTO(5,  "32-63");
  SPI_STATISTICS_TRANSFER_BYTES_HISTO(6,  "64-127");
  SPI_STATISTICS_TRANSFER_BYTES_HISTO(7,  "128-255");
  SPI_STATISTICS_TRANSFER_BYTES_HISTO(8,  "256-511");
  SPI_STATISTICS_TRANSFER_BYTES_HISTO(9,  "512-1023");
  SPI_STATISTICS_TRANSFER_BYTES_HISTO(10, "1024-2047");
  SPI_STATISTICS_TRANSFER_BYTES_HISTO(11, "2048-4095");
  SPI_STATISTICS_TRANSFER_BYTES_HISTO(12, "4096-8191");
  SPI_STATISTICS_TRANSFER_BYTES_HISTO(13, "8192-16383");
  SPI_STATISTICS_TRANSFER_BYTES_HISTO(14, "16384-32767");
  SPI_STATISTICS_TRANSFER_BYTES_HISTO(15, "32768-65535");
  SPI_STATISTICS_TRANSFER_BYTES_HISTO(16, "65536+");

  SPI_STATISTICS_SHOW(transfers_split_maxsize, "%lu");

  static struct attribute *spi_dev_attrs[] = {
  	&dev_attr_modalias.attr,

  	&dev_attr_driver_override.attr,

  	NULL,

  };

  
  static const struct attribute_group spi_dev_group = {
  	.attrs  = spi_dev_attrs,
  };
  
  static struct attribute *spi_device_statistics_attrs[] = {
  	&dev_attr_spi_device_messages.attr,
  	&dev_attr_spi_device_transfers.attr,
  	&dev_attr_spi_device_errors.attr,
  	&dev_attr_spi_device_timedout.attr,
  	&dev_attr_spi_device_spi_sync.attr,
  	&dev_attr_spi_device_spi_sync_immediate.attr,
  	&dev_attr_spi_device_spi_async.attr,
  	&dev_attr_spi_device_bytes.attr,
  	&dev_attr_spi_device_bytes_rx.attr,
  	&dev_attr_spi_device_bytes_tx.attr,

  	&dev_attr_spi_device_transfer_bytes_histo0.attr,
  	&dev_attr_spi_device_transfer_bytes_histo1.attr,
  	&dev_attr_spi_device_transfer_bytes_histo2.attr,
  	&dev_attr_spi_device_transfer_bytes_histo3.attr,
  	&dev_attr_spi_device_transfer_bytes_histo4.attr,
  	&dev_attr_spi_device_transfer_bytes_histo5.attr,
  	&dev_attr_spi_device_transfer_bytes_histo6.attr,
  	&dev_attr_spi_device_transfer_bytes_histo7.attr,
  	&dev_attr_spi_device_transfer_bytes_histo8.attr,
  	&dev_attr_spi_device_transfer_bytes_histo9.attr,
  	&dev_attr_spi_device_transfer_bytes_histo10.attr,
  	&dev_attr_spi_device_transfer_bytes_histo11.attr,
  	&dev_attr_spi_device_transfer_bytes_histo12.attr,
  	&dev_attr_spi_device_transfer_bytes_histo13.attr,
  	&dev_attr_spi_device_transfer_bytes_histo14.attr,
  	&dev_attr_spi_device_transfer_bytes_histo15.attr,
  	&dev_attr_spi_device_transfer_bytes_histo16.attr,

  	&dev_attr_spi_device_transfers_split_maxsize.attr,

  	NULL,
  };
  
  static const struct attribute_group spi_device_statistics_group = {
  	.name  = "statistics",
  	.attrs  = spi_device_statistics_attrs,
  };
  
  static const struct attribute_group *spi_dev_groups[] = {
  	&spi_dev_group,
  	&spi_device_statistics_group,
  	NULL,
  };

  static struct attribute *spi_controller_statistics_attrs[] = {
  	&dev_attr_spi_controller_messages.attr,
  	&dev_attr_spi_controller_transfers.attr,
  	&dev_attr_spi_controller_errors.attr,
  	&dev_attr_spi_controller_timedout.attr,
  	&dev_attr_spi_controller_spi_sync.attr,
  	&dev_attr_spi_controller_spi_sync_immediate.attr,
  	&dev_attr_spi_controller_spi_async.attr,
  	&dev_attr_spi_controller_bytes.attr,
  	&dev_attr_spi_controller_bytes_rx.attr,
  	&dev_attr_spi_controller_bytes_tx.attr,
  	&dev_attr_spi_controller_transfer_bytes_histo0.attr,
  	&dev_attr_spi_controller_transfer_bytes_histo1.attr,
  	&dev_attr_spi_controller_transfer_bytes_histo2.attr,
  	&dev_attr_spi_controller_transfer_bytes_histo3.attr,
  	&dev_attr_spi_controller_transfer_bytes_histo4.attr,
  	&dev_attr_spi_controller_transfer_bytes_histo5.attr,
  	&dev_attr_spi_controller_transfer_bytes_histo6.attr,
  	&dev_attr_spi_controller_transfer_bytes_histo7.attr,
  	&dev_attr_spi_controller_transfer_bytes_histo8.attr,
  	&dev_attr_spi_controller_transfer_bytes_histo9.attr,
  	&dev_attr_spi_controller_transfer_bytes_histo10.attr,
  	&dev_attr_spi_controller_transfer_bytes_histo11.attr,
  	&dev_attr_spi_controller_transfer_bytes_histo12.attr,
  	&dev_attr_spi_controller_transfer_bytes_histo13.attr,
  	&dev_attr_spi_controller_transfer_bytes_histo14.attr,
  	&dev_attr_spi_controller_transfer_bytes_histo15.attr,
  	&dev_attr_spi_controller_transfer_bytes_histo16.attr,
  	&dev_attr_spi_controller_transfers_split_maxsize.attr,

  	NULL,
  };

  static const struct attribute_group spi_controller_statistics_group = {

  	.name  = "statistics",

  	.attrs  = spi_controller_statistics_attrs,

  };
  
  static const struct attribute_group *spi_master_groups[] = {

  	&spi_controller_statistics_group,

  	NULL,
  };
  
  void spi_statistics_add_transfer_stats(struct spi_statistics *stats,
  				       struct spi_transfer *xfer,

  				       struct spi_controller *ctlr)

  {
  	unsigned long flags;

  	int l2len = min(fls(xfer->len), SPI_STATISTICS_HISTO_SIZE) - 1;
  
  	if (l2len < 0)
  		l2len = 0;

  
  	spin_lock_irqsave(&stats->lock, flags);
  
  	stats->transfers++;

  	stats->transfer_bytes_histo[l2len]++;

  
  	stats->bytes += xfer->len;
  	if ((xfer->tx_buf) &&

  	    (xfer->tx_buf != ctlr->dummy_tx))

  		stats->bytes_tx += xfer->len;
  	if ((xfer->rx_buf) &&

  	    (xfer->rx_buf != ctlr->dummy_rx))

  		stats->bytes_rx += xfer->len;
  
  	spin_unlock_irqrestore(&stats->lock, flags);
  }
  EXPORT_SYMBOL_GPL(spi_statistics_add_transfer_stats);

  
  /* modalias support makes "modprobe $MODALIAS" new-style hotplug work,
   * and the sysfs version makes coldplug work too.
   */

  static const struct spi_device_id *spi_match_id(const struct spi_device_id *id,
  						const struct spi_device *sdev)
  {
  	while (id->name[0]) {
  		if (!strcmp(sdev->modalias, id->name))
  			return id;
  		id++;
  	}
  	return NULL;
  }
  
  const struct spi_device_id *spi_get_device_id(const struct spi_device *sdev)
  {
  	const struct spi_driver *sdrv = to_spi_driver(sdev->dev.driver);
  
  	return spi_match_id(sdrv->id_table, sdev);
  }
  EXPORT_SYMBOL_GPL(spi_get_device_id);

  static int spi_match_device(struct device *dev, struct device_driver *drv)
  {
  	const struct spi_device	*spi = to_spi_device(dev);

  	const struct spi_driver	*sdrv = to_spi_driver(drv);

  	/* Check override first, and if set, only use the named driver */
  	if (spi->driver_override)
  		return strcmp(spi->driver_override, drv->name) == 0;

  	/* Attempt an OF style match */
  	if (of_driver_match_device(dev, drv))
  		return 1;

  	/* Then try ACPI */
  	if (acpi_driver_match_device(dev, drv))
  		return 1;

  	if (sdrv->id_table)
  		return !!spi_match_id(sdrv->id_table, spi);

  	return strcmp(spi->modalias, drv->name) == 0;

  }

  static int spi_uevent(struct device *dev, struct kobj_uevent_env *env)

  {
  	const struct spi_device		*spi = to_spi_device(dev);

  	int rc;
  
  	rc = acpi_device_uevent_modalias(dev, env);
  	if (rc != -ENODEV)
  		return rc;

  	return add_uevent_var(env, "MODALIAS=%s%s", SPI_MODULE_PREFIX, spi->modalias);

  }

  struct bus_type spi_bus_type = {
  	.name		= "spi",

  	.dev_groups	= spi_dev_groups,

  	.match		= spi_match_device,
  	.uevent		= spi_uevent,

  };
  EXPORT_SYMBOL_GPL(spi_bus_type);

  
  static int spi_drv_probe(struct device *dev)
  {
  	const struct spi_driver		*sdrv = to_spi_driver(dev->driver);

  	struct spi_device		*spi = to_spi_device(dev);

  	int ret;

  	ret = of_clk_set_defaults(dev->of_node, false);
  	if (ret)
  		return ret;

  	if (dev->of_node) {
  		spi->irq = of_irq_get(dev->of_node, 0);
  		if (spi->irq == -EPROBE_DEFER)
  			return -EPROBE_DEFER;
  		if (spi->irq < 0)
  			spi->irq = 0;
  	}

  	ret = dev_pm_domain_attach(dev, true);

  	if (ret)
  		return ret;

  	if (sdrv->probe) {
  		ret = sdrv->probe(spi);
  		if (ret)
  			dev_pm_domain_detach(dev, true);
  	}

  	return ret;

  }
  
  static int spi_drv_remove(struct device *dev)
  {
  	const struct spi_driver		*sdrv = to_spi_driver(dev->driver);

  	int ret = 0;

  	if (sdrv->remove)
  		ret = sdrv->remove(to_spi_device(dev));

  	dev_pm_domain_detach(dev, true);

  	return ret;

  }
  
  static void spi_drv_shutdown(struct device *dev)
  {
  	const struct spi_driver		*sdrv = to_spi_driver(dev->driver);
  
  	sdrv->shutdown(to_spi_device(dev));
  }

  /**

   * __spi_register_driver - register a SPI driver

   * @owner: owner module of the driver to register

   * @sdrv: the driver to register
   * Context: can sleep

   *
   * Return: zero on success, else a negative error code.

   */

  int __spi_register_driver(struct module *owner, struct spi_driver *sdrv)

  {

  	sdrv->driver.owner = owner;

  	sdrv->driver.bus = &spi_bus_type;

  	sdrv->driver.probe = spi_drv_probe;
  	sdrv->driver.remove = spi_drv_remove;

  	if (sdrv->shutdown)
  		sdrv->driver.shutdown = spi_drv_shutdown;
  	return driver_register(&sdrv->driver);
  }

  EXPORT_SYMBOL_GPL(__spi_register_driver);

  /*-------------------------------------------------------------------------*/
  
  /* SPI devices should normally not be created by SPI device drivers; that

   * would make them board-specific.  Similarly with SPI controller drivers.

   * Device registration normally goes into like arch/.../mach.../board-YYY.c
   * with other readonly (flashable) information about mainboard devices.
   */
  
  struct boardinfo {
  	struct list_head	list;

  	struct spi_board_info	board_info;

  };
  
  static LIST_HEAD(board_list);

  static LIST_HEAD(spi_controller_list);

  
  /*

   * Used to protect add/del operation for board_info list and

   * spi_controller list, and their matching process

   * also used to protect object of type struct idr

   */

  static DEFINE_MUTEX(board_lock);

  /*
   * Prevents addition of devices with same chip select and
   * addition of devices below an unregistering controller.
   */
  static DEFINE_MUTEX(spi_add_lock);

  /**
   * spi_alloc_device - Allocate a new SPI device

   * @ctlr: Controller to which device is connected

   * Context: can sleep
   *
   * Allows a driver to allocate and initialize a spi_device without
   * registering it immediately.  This allows a driver to directly
   * fill the spi_device with device parameters before calling
   * spi_add_device() on it.
   *
   * Caller is responsible to call spi_add_device() on the returned

   * spi_device structure to add it to the SPI controller.  If the caller

   * needs to discard the spi_device without adding it, then it should
   * call spi_dev_put() on it.
   *

   * Return: a pointer to the new device, or NULL.

   */

  struct spi_device *spi_alloc_device(struct spi_controller *ctlr)

  {
  	struct spi_device	*spi;

  	if (!spi_controller_get(ctlr))

  		return NULL;

  	spi = kzalloc(sizeof(*spi), GFP_KERNEL);

  	if (!spi) {

  		spi_controller_put(ctlr);

  		return NULL;
  	}

  	spi->master = spi->controller = ctlr;
  	spi->dev.parent = &ctlr->dev;

  	spi->dev.bus = &spi_bus_type;
  	spi->dev.release = spidev_release;

  	spi->cs_gpio = -ENOENT;

  	spi->mode = ctlr->buswidth_override_bits;

  
  	spin_lock_init(&spi->statistics.lock);

  	device_initialize(&spi->dev);
  	return spi;
  }
  EXPORT_SYMBOL_GPL(spi_alloc_device);

  static void spi_dev_set_name(struct spi_device *spi)
  {
  	struct acpi_device *adev = ACPI_COMPANION(&spi->dev);
  
  	if (adev) {
  		dev_set_name(&spi->dev, "spi-%s", acpi_dev_name(adev));
  		return;
  	}

  	dev_set_name(&spi->dev, "%s.%u", dev_name(&spi->controller->dev),

  		     spi->chip_select);
  }

  static int spi_dev_check(struct device *dev, void *data)
  {
  	struct spi_device *spi = to_spi_device(dev);
  	struct spi_device *new_spi = data;

  	if (spi->controller == new_spi->controller &&

  	    spi->chip_select == new_spi->chip_select)
  		return -EBUSY;
  	return 0;
  }

  /**
   * spi_add_device - Add spi_device allocated with spi_alloc_device
   * @spi: spi_device to register
   *
   * Companion function to spi_alloc_device.  Devices allocated with
   * spi_alloc_device can be added onto the spi bus with this function.
   *

   * Return: 0 on success; negative errno on failure

   */
  int spi_add_device(struct spi_device *spi)
  {

  	struct spi_controller *ctlr = spi->controller;
  	struct device *dev = ctlr->dev.parent;

  	int status;
  
  	/* Chipselects are numbered 0..max; validate. */

  	if (spi->chip_select >= ctlr->num_chipselect) {
  		dev_err(dev, "cs%d >= max %d
  ", spi->chip_select,
  			ctlr->num_chipselect);

  		return -EINVAL;
  	}
  
  	/* Set the bus ID string */

  	spi_dev_set_name(spi);

  
  	/* We need to make sure there's no other device with this
  	 * chipselect **BEFORE** we call setup(), else we'll trash
  	 * its configuration.  Lock against concurrent add() calls.
  	 */
  	mutex_lock(&spi_add_lock);

  	status = bus_for_each_dev(&spi_bus_type, NULL, spi, spi_dev_check);
  	if (status) {

  		dev_err(dev, "chipselect %d already in use
  ",
  				spi->chip_select);

  		goto done;
  	}

  	/* Controller may unregister concurrently */
  	if (IS_ENABLED(CONFIG_SPI_DYNAMIC) &&
  	    !device_is_registered(&ctlr->dev)) {
  		status = -ENODEV;
  		goto done;
  	}

  	/* Descriptors take precedence */
  	if (ctlr->cs_gpiods)
  		spi->cs_gpiod = ctlr->cs_gpiods[spi->chip_select];
  	else if (ctlr->cs_gpios)

  		spi->cs_gpio = ctlr->cs_gpios[spi->chip_select];

  	/* Drivers may modify this initial i/o setup, but will
  	 * normally rely on the device being setup.  Devices
  	 * using SPI_CS_HIGH can't coexist well otherwise...
  	 */

  	status = spi_setup(spi);

  	if (status < 0) {

  		dev_err(dev, "can't setup %s, status %d
  ",
  				dev_name(&spi->dev), status);

  		goto done;

  	}

  	/* Device may be bound to an active driver when this returns */

  	status = device_add(&spi->dev);

  	if (status < 0)

  		dev_err(dev, "can't add %s, status %d
  ",
  				dev_name(&spi->dev), status);

  	else

  		dev_dbg(dev, "registered child %s
  ", dev_name(&spi->dev));

  done:
  	mutex_unlock(&spi_add_lock);
  	return status;

  }
  EXPORT_SYMBOL_GPL(spi_add_device);

  /**
   * spi_new_device - instantiate one new SPI device

   * @ctlr: Controller to which device is connected

   * @chip: Describes the SPI device
   * Context: can sleep
   *
   * On typical mainboards, this is purely internal; and it's not needed

   * after board init creates the hard-wired devices.  Some development
   * platforms may not be able to use spi_register_board_info though, and
   * this is exported so that for example a USB or parport based adapter
   * driver could add devices (which it would learn about out-of-band).

   *

   * Return: the new device, or NULL.

   */

  struct spi_device *spi_new_device(struct spi_controller *ctlr,

  				  struct spi_board_info *chip)

  {
  	struct spi_device	*proxy;

  	int			status;

  	/* NOTE:  caller did any chip->bus_num checks necessary.
  	 *
  	 * Also, unless we change the return value convention to use
  	 * error-or-pointer (not NULL-or-pointer), troubleshootability
  	 * suggests syslogged diagnostics are best here (ugh).
  	 */

  	proxy = spi_alloc_device(ctlr);

  	if (!proxy)

  		return NULL;

  	WARN_ON(strlen(chip->modalias) >= sizeof(proxy->modalias));

  	proxy->chip_select = chip->chip_select;
  	proxy->max_speed_hz = chip->max_speed_hz;

  	proxy->mode = chip->mode;

  	proxy->irq = chip->irq;

  	strlcpy(proxy->modalias, chip->modalias, sizeof(proxy->modalias));

  	proxy->dev.platform_data = (void *) chip->platform_data;
  	proxy->controller_data = chip->controller_data;
  	proxy->controller_state = NULL;

  	if (chip->properties) {
  		status = device_add_properties(&proxy->dev, chip->properties);
  		if (status) {

  			dev_err(&ctlr->dev,

  				"failed to add properties to '%s': %d
  ",
  				chip->modalias, status);
  			goto err_dev_put;
  		}

  	}

  	status = spi_add_device(proxy);
  	if (status < 0)
  		goto err_remove_props;

  	return proxy;

  
  err_remove_props:
  	if (chip->properties)
  		device_remove_properties(&proxy->dev);
  err_dev_put:
  	spi_dev_put(proxy);
  	return NULL;

  }
  EXPORT_SYMBOL_GPL(spi_new_device);

  /**
   * spi_unregister_device - unregister a single SPI device
   * @spi: spi_device to unregister
   *
   * Start making the passed SPI device vanish. Normally this would be handled

   * by spi_unregister_controller().

   */
  void spi_unregister_device(struct spi_device *spi)
  {

  	if (!spi)
  		return;

  	if (spi->dev.of_node) {

  		of_node_clear_flag(spi->dev.of_node, OF_POPULATED);

  		of_node_put(spi->dev.of_node);
  	}

  	if (ACPI_COMPANION(&spi->dev))
  		acpi_device_clear_enumerated(ACPI_COMPANION(&spi->dev));

  	device_unregister(&spi->dev);

  }
  EXPORT_SYMBOL_GPL(spi_unregister_device);

  static void spi_match_controller_to_boardinfo(struct spi_controller *ctlr,
  					      struct spi_board_info *bi)

  {
  	struct spi_device *dev;

  	if (ctlr->bus_num != bi->bus_num)

  		return;

  	dev = spi_new_device(ctlr, bi);

  	if (!dev)

  		dev_err(ctlr->dev.parent, "can't create new device for %s
  ",

  			bi->modalias);
  }

  /**
   * spi_register_board_info - register SPI devices for a given board
   * @info: array of chip descriptors
   * @n: how many descriptors are provided
   * Context: can sleep
   *

   * Board-specific early init code calls this (probably during arch_initcall)
   * with segments of the SPI device table.  Any device nodes are created later,
   * after the relevant parent SPI controller (bus_num) is defined.  We keep
   * this table of devices forever, so that reloading a controller driver will
   * not make Linux forget about these hard-wired devices.
   *
   * Other code can also call this, e.g. a particular add-on board might provide
   * SPI devices through its expansion connector, so code initializing that board
   * would naturally declare its SPI devices.
   *
   * The board info passed can safely be __initdata ... but be careful of
   * any embedded pointers (platform_data, etc), they're copied as-is.

   * Device properties are deep-copied though.

   *
   * Return: zero on success, else a negative error code.

   */

  int spi_register_board_info(struct spi_board_info const *info, unsigned n)

  {

  	struct boardinfo *bi;
  	int i;

  	if (!n)

  		return 0;

  	bi = kcalloc(n, sizeof(*bi), GFP_KERNEL);

  	if (!bi)
  		return -ENOMEM;

  	for (i = 0; i < n; i++, bi++, info++) {

  		struct spi_controller *ctlr;

  		memcpy(&bi->board_info, info, sizeof(*info));

  		if (info->properties) {
  			bi->board_info.properties =
  					property_entries_dup(info->properties);
  			if (IS_ERR(bi->board_info.properties))
  				return PTR_ERR(bi->board_info.properties);
  		}

  		mutex_lock(&board_lock);
  		list_add_tail(&bi->list, &board_list);

  		list_for_each_entry(ctlr, &spi_controller_list, list)
  			spi_match_controller_to_boardinfo(ctlr,
  							  &bi->board_info);

  		mutex_unlock(&board_lock);

  	}

  
  	return 0;

  }
  
  /*-------------------------------------------------------------------------*/

  static void spi_set_cs(struct spi_device *spi, bool enable)
  {

  	bool enable1 = enable;

  	/*
  	 * Avoid calling into the driver (or doing delays) if the chip select
  	 * isn't actually changing from the last time this was called.
  	 */
  	if ((spi->controller->last_cs_enable == enable) &&
  	    (spi->controller->last_cs_mode_high == (spi->mode & SPI_CS_HIGH)))
  		return;
  
  	spi->controller->last_cs_enable = enable;
  	spi->controller->last_cs_mode_high = spi->mode & SPI_CS_HIGH;

  	if (!spi->controller->set_cs_timing) {
  		if (enable1)
  			spi_delay_exec(&spi->controller->cs_setup, NULL);
  		else
  			spi_delay_exec(&spi->controller->cs_hold, NULL);
  	}

  	if (spi->mode & SPI_CS_HIGH)
  		enable = !enable;

  	if (spi->cs_gpiod || gpio_is_valid(spi->cs_gpio)) {

  		if (!(spi->mode & SPI_NO_CS)) {
  			if (spi->cs_gpiod)

  				/* polarity handled by gpiolib */

  				gpiod_set_value_cansleep(spi->cs_gpiod,

  							 enable1);

  			else

  				/*
  				 * invert the enable line, as active low is
  				 * default for SPI.
  				 */

  				gpio_set_value_cansleep(spi->cs_gpio, !enable);

  		}

  		/* Some SPI masters need both GPIO CS & slave_select */

  		if ((spi->controller->flags & SPI_MASTER_GPIO_SS) &&
  		    spi->controller->set_cs)
  			spi->controller->set_cs(spi, !enable);
  	} else if (spi->controller->set_cs) {
  		spi->controller->set_cs(spi, !enable);

  	}

  
  	if (!spi->controller->set_cs_timing) {
  		if (!enable1)
  			spi_delay_exec(&spi->controller->cs_inactive, NULL);
  	}

  }

  #ifdef CONFIG_HAS_DMA

  int spi_map_buf(struct spi_controller *ctlr, struct device *dev,
  		struct sg_table *sgt, void *buf, size_t len,
  		enum dma_data_direction dir)

  {
  	const bool vmalloced_buf = is_vmalloc_addr(buf);

  	unsigned int max_seg_size = dma_get_max_seg_size(dev);

  #ifdef CONFIG_HIGHMEM
  	const bool kmap_buf = ((unsigned long)buf >= PKMAP_BASE &&
  				(unsigned long)buf < (PKMAP_BASE +
  					(LAST_PKMAP * PAGE_SIZE)));
  #else
  	const bool kmap_buf = false;
  #endif

  	int desc_len;
  	int sgs;

  	struct page *vm_page;

  	struct scatterlist *sg;

  	void *sg_buf;
  	size_t min;
  	int i, ret;

  	if (vmalloced_buf || kmap_buf) {

  		desc_len = min_t(int, max_seg_size, PAGE_SIZE);

  		sgs = DIV_ROUND_UP(len + offset_in_page(buf), desc_len);

  	} else if (virt_addr_valid(buf)) {

  		desc_len = min_t(int, max_seg_size, ctlr->max_dma_len);

  		sgs = DIV_ROUND_UP(len, desc_len);

  	} else {
  		return -EINVAL;

  	}

  	ret = sg_alloc_table(sgt, sgs, GFP_KERNEL);
  	if (ret != 0)
  		return ret;

  	sg = &sgt->sgl[0];

  	for (i = 0; i < sgs; i++) {

  		if (vmalloced_buf || kmap_buf) {

  			/*
  			 * Next scatterlist entry size is the minimum between
  			 * the desc_len and the remaining buffer length that
  			 * fits in a page.
  			 */
  			min = min_t(size_t, desc_len,
  				    min_t(size_t, len,
  					  PAGE_SIZE - offset_in_page(buf)));

  			if (vmalloced_buf)
  				vm_page = vmalloc_to_page(buf);
  			else
  				vm_page = kmap_to_page(buf);

  			if (!vm_page) {
  				sg_free_table(sgt);
  				return -ENOMEM;
  			}

  			sg_set_page(sg, vm_page,

  				    min, offset_in_page(buf));

  		} else {

  			min = min_t(size_t, len, desc_len);

  			sg_buf = buf;

  			sg_set_buf(sg, sg_buf, min);

  		}

  		buf += min;
  		len -= min;

  		sg = sg_next(sg);

  	}
  
  	ret = dma_map_sg(dev, sgt->sgl, sgt->nents, dir);

  	if (!ret)
  		ret = -ENOMEM;

  	if (ret < 0) {
  		sg_free_table(sgt);
  		return ret;
  	}
  
  	sgt->nents = ret;
  
  	return 0;
  }

  void spi_unmap_buf(struct spi_controller *ctlr, struct device *dev,
  		   struct sg_table *sgt, enum dma_data_direction dir)

  {
  	if (sgt->orig_nents) {
  		dma_unmap_sg(dev, sgt->sgl, sgt->orig_nents, dir);
  		sg_free_table(sgt);
  	}
  }

  static int __spi_map_msg(struct spi_controller *ctlr, struct spi_message *msg)

  {

  	struct device *tx_dev, *rx_dev;
  	struct spi_transfer *xfer;

  	int ret;

  	if (!ctlr->can_dma)

  		return 0;

  	if (ctlr->dma_tx)
  		tx_dev = ctlr->dma_tx->device->dev;

  	else

  		tx_dev = ctlr->dev.parent;

  	if (ctlr->dma_rx)
  		rx_dev = ctlr->dma_rx->device->dev;

  	else

  		rx_dev = ctlr->dev.parent;

  
  	list_for_each_entry(xfer, &msg->transfers, transfer_list) {

  		if (!ctlr->can_dma(ctlr, msg->spi, xfer))

  			continue;
  
  		if (xfer->tx_buf != NULL) {

  			ret = spi_map_buf(ctlr, tx_dev, &xfer->tx_sg,

  					  (void *)xfer->tx_buf, xfer->len,
  					  DMA_TO_DEVICE);
  			if (ret != 0)
  				return ret;

  		}
  
  		if (xfer->rx_buf != NULL) {

  			ret = spi_map_buf(ctlr, rx_dev, &xfer->rx_sg,

  					  xfer->rx_buf, xfer->len,
  					  DMA_FROM_DEVICE);
  			if (ret != 0) {

  				spi_unmap_buf(ctlr, tx_dev, &xfer->tx_sg,

  					      DMA_TO_DEVICE);
  				return ret;

  			}
  		}
  	}

  	ctlr->cur_msg_mapped = true;

  
  	return 0;
  }

  static int __spi_unmap_msg(struct spi_controller *ctlr, struct spi_message *msg)

  {
  	struct spi_transfer *xfer;
  	struct device *tx_dev, *rx_dev;

  	if (!ctlr->cur_msg_mapped || !ctlr->can_dma)

  		return 0;

  	if (ctlr->dma_tx)
  		tx_dev = ctlr->dma_tx->device->dev;

  	else

  		tx_dev = ctlr->dev.parent;

  	if (ctlr->dma_rx)
  		rx_dev = ctlr->dma_rx->device->dev;

  	else

  		rx_dev = ctlr->dev.parent;

  
  	list_for_each_entry(xfer, &msg->transfers, transfer_list) {

  		if (!ctlr->can_dma(ctlr, msg->spi, xfer))

  			continue;

  		spi_unmap_buf(ctlr, rx_dev, &xfer->rx_sg, DMA_FROM_DEVICE);
  		spi_unmap_buf(ctlr, tx_dev, &xfer->tx_sg, DMA_TO_DEVICE);

  	}

  	ctlr->cur_msg_mapped = false;

  	return 0;
  }

  #else /* !CONFIG_HAS_DMA */

  static inline int __spi_map_msg(struct spi_controller *ctlr,

  				struct spi_message *msg)
  {
  	return 0;
  }

  static inline int __spi_unmap_msg(struct spi_controller *ctlr,

  				  struct spi_message *msg)

  {
  	return 0;
  }
  #endif /* !CONFIG_HAS_DMA */

  static inline int spi_unmap_msg(struct spi_controller *ctlr,

  				struct spi_message *msg)
  {
  	struct spi_transfer *xfer;
  
  	list_for_each_entry(xfer, &msg->transfers, transfer_list) {
  		/*
  		 * Restore the original value of tx_buf or rx_buf if they are
  		 * NULL.
  		 */

  		if (xfer->tx_buf == ctlr->dummy_tx)

  			xfer->tx_buf = NULL;

  		if (xfer->rx_buf == ctlr->dummy_rx)

  			xfer->rx_buf = NULL;
  	}

  	return __spi_unmap_msg(ctlr, msg);

  }

  static int spi_map_msg(struct spi_controller *ctlr, struct spi_message *msg)

  {
  	struct spi_transfer *xfer;
  	void *tmp;
  	unsigned int max_tx, max_rx;

  	if ((ctlr->flags & (SPI_CONTROLLER_MUST_RX | SPI_CONTROLLER_MUST_TX))
  		&& !(msg->spi->mode & SPI_3WIRE)) {

  		max_tx = 0;
  		max_rx = 0;
  
  		list_for_each_entry(xfer, &msg->transfers, transfer_list) {

  			if ((ctlr->flags & SPI_CONTROLLER_MUST_TX) &&

  			    !xfer->tx_buf)
  				max_tx = max(xfer->len, max_tx);

  			if ((ctlr->flags & SPI_CONTROLLER_MUST_RX) &&

  			    !xfer->rx_buf)
  				max_rx = max(xfer->len, max_rx);
  		}
  
  		if (max_tx) {

  			tmp = krealloc(ctlr->dummy_tx, max_tx,

  				       GFP_KERNEL | GFP_DMA);
  			if (!tmp)
  				return -ENOMEM;

  			ctlr->dummy_tx = tmp;

  			memset(tmp, 0, max_tx);
  		}
  
  		if (max_rx) {

  			tmp = krealloc(ctlr->dummy_rx, max_rx,

  				       GFP_KERNEL | GFP_DMA);
  			if (!tmp)
  				return -ENOMEM;

  			ctlr->dummy_rx = tmp;

  		}
  
  		if (max_tx || max_rx) {
  			list_for_each_entry(xfer, &msg->transfers,
  					    transfer_list) {

  				if (!xfer->len)
  					continue;

  				if (!xfer->tx_buf)

  					xfer->tx_buf = ctlr->dummy_tx;

  				if (!xfer->rx_buf)

  					xfer->rx_buf = ctlr->dummy_rx;

  			}
  		}
  	}

  	return __spi_map_msg(ctlr, msg);

  }

  static int spi_transfer_wait(struct spi_controller *ctlr,
  			     struct spi_message *msg,
  			     struct spi_transfer *xfer)
  {
  	struct spi_statistics *statm = &ctlr->statistics;
  	struct spi_statistics *stats = &msg->spi->statistics;

  	u32 speed_hz = xfer->speed_hz;

  	unsigned long long ms;

  
  	if (spi_controller_is_slave(ctlr)) {
  		if (wait_for_completion_interruptible(&ctlr->xfer_completion)) {
  			dev_dbg(&msg->spi->dev, "SPI transfer interrupted
  ");
  			return -EINTR;
  		}
  	} else {

  		if (!speed_hz)
  			speed_hz = 100000;

  		ms = 8LL * 1000LL * xfer->len;

  		do_div(ms, speed_hz);

  		ms += ms + 200; /* some tolerance */
  
  		if (ms > UINT_MAX)
  			ms = UINT_MAX;
  
  		ms = wait_for_completion_timeout(&ctlr->xfer_completion,
  						 msecs_to_jiffies(ms));
  
  		if (ms == 0) {
  			SPI_STATISTICS_INCREMENT_FIELD(statm, timedout);
  			SPI_STATISTICS_INCREMENT_FIELD(stats, timedout);
  			dev_err(&msg->spi->dev,
  				"SPI transfer timed out
  ");
  			return -ETIMEDOUT;
  		}
  	}
  
  	return 0;
  }

  static void _spi_transfer_delay_ns(u32 ns)
  {
  	if (!ns)
  		return;
  	if (ns <= 1000) {
  		ndelay(ns);
  	} else {
  		u32 us = DIV_ROUND_UP(ns, 1000);
  
  		if (us <= 10)
  			udelay(us);
  		else
  			usleep_range(us, us + DIV_ROUND_UP(us, 10));
  	}
  }

  int spi_delay_to_ns(struct spi_delay *_delay, struct spi_transfer *xfer)

  {

  	u32 delay = _delay->value;
  	u32 unit = _delay->unit;

  	u32 hz;

  	if (!delay)
  		return 0;

  
  	switch (unit) {
  	case SPI_DELAY_UNIT_USECS:

  		delay *= 1000;

  		break;
  	case SPI_DELAY_UNIT_NSECS: /* nothing to do here */
  		break;

  	case SPI_DELAY_UNIT_SCK:

  		/* clock cycles need to be obtained from spi_transfer */
  		if (!xfer)
  			return -EINVAL;

  		/* if there is no effective speed know, then approximate
  		 * by underestimating with half the requested hz
  		 */
  		hz = xfer->effective_speed_hz ?: xfer->speed_hz / 2;

  		if (!hz)
  			return -EINVAL;

  		delay *= DIV_ROUND_UP(1000000000, hz);
  		break;

  	default:

  		return -EINVAL;
  	}
  
  	return delay;
  }

  EXPORT_SYMBOL_GPL(spi_delay_to_ns);

  
  int spi_delay_exec(struct spi_delay *_delay, struct spi_transfer *xfer)
  {
  	int delay;

  	might_sleep();

  	if (!_delay)
  		return -EINVAL;

  	delay = spi_delay_to_ns(_delay, xfer);

  	if (delay < 0)
  		return delay;
  
  	_spi_transfer_delay_ns(delay);
  
  	return 0;
  }
  EXPORT_SYMBOL_GPL(spi_delay_exec);

  static void _spi_transfer_cs_change_delay(struct spi_message *msg,
  					  struct spi_transfer *xfer)
  {

  	u32 delay = xfer->cs_change_delay.value;
  	u32 unit = xfer->cs_change_delay.unit;
  	int ret;

  
  	/* return early on "fast" mode - for everything but USECS */

  	if (!delay) {
  		if (unit == SPI_DELAY_UNIT_USECS)
  			_spi_transfer_delay_ns(10000);

  		return;

  	}

  	ret = spi_delay_exec(&xfer->cs_change_delay, xfer);
  	if (ret) {

  		dev_err_once(&msg->spi->dev,
  			     "Use of unsupported delay unit %i, using default of 10us
  ",

  			     unit);
  		_spi_transfer_delay_ns(10000);

  	}

  }

  /*
   * spi_transfer_one_message - Default implementation of transfer_one_message()
   *
   * This is a standard implementation of transfer_one_message() for

   * drivers which implement a transfer_one() operation.  It provides

   * standard handling of delays and chip select management.
   */

  static int spi_transfer_one_message(struct spi_controller *ctlr,

  				    struct spi_message *msg)
  {
  	struct spi_transfer *xfer;

  	bool keep_cs = false;
  	int ret = 0;

  	struct spi_statistics *statm = &ctlr->statistics;

  	struct spi_statistics *stats = &msg->spi->statistics;

  
  	spi_set_cs(msg->spi, true);

  	SPI_STATISTICS_INCREMENT_FIELD(statm, messages);
  	SPI_STATISTICS_INCREMENT_FIELD(stats, messages);

  	list_for_each_entry(xfer, &msg->transfers, transfer_list) {
  		trace_spi_transfer_start(msg, xfer);

  		spi_statistics_add_transfer_stats(statm, xfer, ctlr);
  		spi_statistics_add_transfer_stats(stats, xfer, ctlr);

  		if (!ctlr->ptp_sts_supported) {
  			xfer->ptp_sts_word_pre = 0;
  			ptp_read_system_prets(xfer->ptp_sts);
  		}

  		if (xfer->tx_buf || xfer->rx_buf) {

  			reinit_completion(&ctlr->xfer_completion);

  fallback_pio:

  			ret = ctlr->transfer_one(ctlr, msg->spi, xfer);

  			if (ret < 0) {

  				if (ctlr->cur_msg_mapped &&
  				   (xfer->error & SPI_TRANS_FAIL_NO_START)) {
  					__spi_unmap_msg(ctlr, msg);
  					ctlr->fallback = true;
  					xfer->error &= ~SPI_TRANS_FAIL_NO_START;
  					goto fallback_pio;
  				}

  				SPI_STATISTICS_INCREMENT_FIELD(statm,
  							       errors);
  				SPI_STATISTICS_INCREMENT_FIELD(stats,
  							       errors);

  				dev_err(&msg->spi->dev,
  					"SPI transfer failed: %d
  ", ret);
  				goto out;
  			}

  			if (ret > 0) {
  				ret = spi_transfer_wait(ctlr, msg, xfer);
  				if (ret < 0)
  					msg->status = ret;
  			}

  		} else {
  			if (xfer->len)
  				dev_err(&msg->spi->dev,
  					"Bufferless transfer has length %u
  ",
  					xfer->len);

  		}

  		if (!ctlr->ptp_sts_supported) {
  			ptp_read_system_postts(xfer->ptp_sts);
  			xfer->ptp_sts_word_post = xfer->len;
  		}

  		trace_spi_transfer_stop(msg, xfer);
  
  		if (msg->status != -EINPROGRESS)
  			goto out;

  		spi_transfer_delay_exec(xfer);

  
  		if (xfer->cs_change) {
  			if (list_is_last(&xfer->transfer_list,
  					 &msg->transfers)) {
  				keep_cs = true;
  			} else {

  				spi_set_cs(msg->spi, false);

  				_spi_transfer_cs_change_delay(msg, xfer);

  				spi_set_cs(msg->spi, true);

  			}
  		}
  
  		msg->actual_length += xfer->len;
  	}
  
  out:
  	if (ret != 0 || !keep_cs)
  		spi_set_cs(msg->spi, false);
  
  	if (msg->status == -EINPROGRESS)
  		msg->status = ret;

  	if (msg->status && ctlr->handle_err)
  		ctlr->handle_err(ctlr, msg);

  	spi_finalize_current_message(ctlr);

  	return ret;
  }
  
  /**
   * spi_finalize_current_transfer - report completion of a transfer

   * @ctlr: the controller reporting completion

   *
   * Called by SPI drivers using the core transfer_one_message()
   * implementation to notify it that the current interrupt driven

   * transfer has finished and the next one may be scheduled.

   */

  void spi_finalize_current_transfer(struct spi_controller *ctlr)

  {

  	complete(&ctlr->xfer_completion);

  }
  EXPORT_SYMBOL_GPL(spi_finalize_current_transfer);

  static void spi_idle_runtime_pm(struct spi_controller *ctlr)
  {
  	if (ctlr->auto_runtime_pm) {
  		pm_runtime_mark_last_busy(ctlr->dev.parent);
  		pm_runtime_put_autosuspend(ctlr->dev.parent);
  	}
  }

  /**

   * __spi_pump_messages - function which processes spi message queue

   * @ctlr: controller to process queue for

   * @in_kthread: true if we are in the context of the message pump thread

   *
   * This function checks if there is any spi message in the queue that
   * needs processing and if so call out to the driver to initialize hardware
   * and transfer each message.
   *

   * Note that it is called both from the kthread itself and also from
   * inside spi_sync(); the queue extraction handling at the top of the
   * function should deal with this safely.

   */

  static void __spi_pump_messages(struct spi_controller *ctlr, bool in_kthread)

  {

  	struct spi_transfer *xfer;

  	struct spi_message *msg;

  	bool was_busy = false;

  	unsigned long flags;

  	int ret;

  	/* Lock queue */

  	spin_lock_irqsave(&ctlr->queue_lock, flags);

  
  	/* Make sure we are not already running a message */

  	if (ctlr->cur_msg) {
  		spin_unlock_irqrestore(&ctlr->queue_lock, flags);

  		return;
  	}

  	/* If another context is idling the device then defer */

  	if (ctlr->idling) {

  		kthread_queue_work(ctlr->kworker, &ctlr->pump_messages);

  		spin_unlock_irqrestore(&ctlr->queue_lock, flags);

  		return;
  	}

  	/* Check if the queue is idle */

  	if (list_empty(&ctlr->queue) || !ctlr->running) {
  		if (!ctlr->busy) {
  			spin_unlock_irqrestore(&ctlr->queue_lock, flags);

  			return;

  		}

  		/* Defer any non-atomic teardown to the thread */

  		if (!in_kthread) {

  			if (!ctlr->dummy_rx && !ctlr->dummy_tx &&
  			    !ctlr->unprepare_transfer_hardware) {
  				spi_idle_runtime_pm(ctlr);
  				ctlr->busy = false;
  				trace_spi_controller_idle(ctlr);
  			} else {
  				kthread_queue_work(ctlr->kworker,
  						   &ctlr->pump_messages);
  			}

  			spin_unlock_irqrestore(&ctlr->queue_lock, flags);
  			return;
  		}
  
  		ctlr->busy = false;
  		ctlr->idling = true;
  		spin_unlock_irqrestore(&ctlr->queue_lock, flags);
  
  		kfree(ctlr->dummy_rx);
  		ctlr->dummy_rx = NULL;
  		kfree(ctlr->dummy_tx);
  		ctlr->dummy_tx = NULL;
  		if (ctlr->unprepare_transfer_hardware &&
  		    ctlr->unprepare_transfer_hardware(ctlr))
  			dev_err(&ctlr->dev,
  				"failed to unprepare transfer hardware
  ");

  		spi_idle_runtime_pm(ctlr);

  		trace_spi_controller_idle(ctlr);
  
  		spin_lock_irqsave(&ctlr->queue_lock, flags);
  		ctlr->idling = false;

  		spin_unlock_irqrestore(&ctlr->queue_lock, flags);

  		return;
  	}

  	/* Extract head of queue */

  	msg = list_first_entry(&ctlr->queue, struct spi_message, queue);
  	ctlr->cur_msg = msg;

  	list_del_init(&msg->queue);

  	if (ctlr->busy)

  		was_busy = true;
  	else

  		ctlr->busy = true;
  	spin_unlock_irqrestore(&ctlr->queue_lock, flags);

  	mutex_lock(&ctlr->io_mutex);

  	if (!was_busy && ctlr->auto_runtime_pm) {
  		ret = pm_runtime_get_sync(ctlr->dev.parent);

  		if (ret < 0) {

  			pm_runtime_put_noidle(ctlr->dev.parent);

  			dev_err(&ctlr->dev, "Failed to power device: %d
  ",

  				ret);

  			mutex_unlock(&ctlr->io_mutex);

  			return;
  		}
  	}

  	if (!was_busy)

  		trace_spi_controller_busy(ctlr);

  	if (!was_busy && ctlr->prepare_transfer_hardware) {
  		ret = ctlr->prepare_transfer_hardware(ctlr);

  		if (ret) {

  			dev_err(&ctlr->dev,

  				"failed to prepare transfer hardware: %d
  ",
  				ret);

  			if (ctlr->auto_runtime_pm)
  				pm_runtime_put(ctlr->dev.parent);

  			msg->status = ret;

  			spi_finalize_current_message(ctlr);

  			mutex_unlock(&ctlr->io_mutex);

  			return;
  		}
  	}

  	trace_spi_message_start(msg);

  	if (ctlr->prepare_message) {

  		ret = ctlr->prepare_message(ctlr, msg);

  		if (ret) {

  			dev_err(&ctlr->dev, "failed to prepare message: %d
  ",
  				ret);

  			msg->status = ret;

  			spi_finalize_current_message(ctlr);

  			goto out;

  		}

  		ctlr->cur_msg_prepared = true;

  	}

  	ret = spi_map_msg(ctlr, msg);

  	if (ret) {

  		msg->status = ret;

  		spi_finalize_current_message(ctlr);

  		goto out;

  	}

  	if (!ctlr->ptp_sts_supported && !ctlr->transfer_one) {
  		list_for_each_entry(xfer, &msg->transfers, transfer_list) {
  			xfer->ptp_sts_word_pre = 0;
  			ptp_read_system_prets(xfer->ptp_sts);
  		}
  	}

  	ret = ctlr->transfer_one_message(ctlr, msg);

  	if (ret) {

  		dev_err(&ctlr->dev,

  			"failed to transfer one message from queue
  ");

  		goto out;

  	}

  
  out:

  	mutex_unlock(&ctlr->io_mutex);

  
  	/* Prod the scheduler in case transfer_one() was busy waiting */

  	if (!ret)
  		cond_resched();

  }

  /**
   * spi_pump_messages - kthread work function which processes spi message queue

   * @work: pointer to kthread work struct contained in the controller struct

   */
  static void spi_pump_messages(struct kthread_work *work)
  {

  	struct spi_controller *ctlr =
  		container_of(work, struct spi_controller, pump_messages);

  	__spi_pump_messages(ctlr, true);

  }

  /**

   * spi_take_timestamp_pre - helper for drivers to collect the beginning of the
   *			    TX timestamp for the requested byte from the SPI
   *			    transfer. The frequency with which this function
   *			    must be called (once per word, once for the whole
   *			    transfer, once per batch of words etc) is arbitrary
   *			    as long as the @tx buffer offset is greater than or
   *			    equal to the requested byte at the time of the
   *			    call. The timestamp is only taken once, at the
   *			    first such call. It is assumed that the driver
   *			    advances its @tx buffer pointer monotonically.
   * @ctlr: Pointer to the spi_controller structure of the driver
   * @xfer: Pointer to the transfer being timestamped

   * @progress: How many words (not bytes) have been transferred so far

   * @irqs_off: If true, will disable IRQs and preemption for the duration of the
   *	      transfer, for less jitter in time measurement. Only compatible
   *	      with PIO drivers. If true, must follow up with
   *	      spi_take_timestamp_post or otherwise system will crash.
   *	      WARNING: for fully predictable results, the CPU frequency must
   *	      also be under control (governor).
   */
  void spi_take_timestamp_pre(struct spi_controller *ctlr,
  			    struct spi_transfer *xfer,

  			    size_t progress, bool irqs_off)

  {

  	if (!xfer->ptp_sts)
  		return;

  	if (xfer->timestamped)

  		return;

  	if (progress > xfer->ptp_sts_word_pre)

  		return;
  
  	/* Capture the resolution of the timestamp */

  	xfer->ptp_sts_word_pre = progress;

  	if (irqs_off) {
  		local_irq_save(ctlr->irq_flags);
  		preempt_disable();
  	}
  
  	ptp_read_system_prets(xfer->ptp_sts);
  }
  EXPORT_SYMBOL_GPL(spi_take_timestamp_pre);
  
  /**
   * spi_take_timestamp_post - helper for drivers to collect the end of the
   *			     TX timestamp for the requested byte from the SPI
   *			     transfer. Can be called with an arbitrary
   *			     frequency: only the first call where @tx exceeds
   *			     or is equal to the requested word will be
   *			     timestamped.
   * @ctlr: Pointer to the spi_controller structure of the driver
   * @xfer: Pointer to the transfer being timestamped

   * @progress: How many words (not bytes) have been transferred so far

   * @irqs_off: If true, will re-enable IRQs and preemption for the local CPU.
   */
  void spi_take_timestamp_post(struct spi_controller *ctlr,
  			     struct spi_transfer *xfer,

  			     size_t progress, bool irqs_off)

  {

  	if (!xfer->ptp_sts)
  		return;

  	if (xfer->timestamped)

  		return;

  	if (progress < xfer->ptp_sts_word_post)

  		return;
  
  	ptp_read_system_postts(xfer->ptp_sts);
  
  	if (irqs_off) {
  		local_irq_restore(ctlr->irq_flags);
  		preempt_enable();
  	}
  
  	/* Capture the resolution of the timestamp */

  	xfer->ptp_sts_word_post = progress;

  	xfer->timestamped = true;

  }
  EXPORT_SYMBOL_GPL(spi_take_timestamp_post);
  
  /**

   * spi_set_thread_rt - set the controller to pump at realtime priority
   * @ctlr: controller to boost priority of
   *
   * This can be called because the controller requested realtime priority
   * (by setting the ->rt value before calling spi_register_controller()) or
   * because a device on the bus said that its transfers needed realtime
   * priority.
   *
   * NOTE: at the moment if any device on a bus says it needs realtime then
   * the thread will be at realtime priority for all transfers on that
   * controller.  If this eventually becomes a problem we may see if we can
   * find a way to boost the priority only temporarily during relevant
   * transfers.
   */
  static void spi_set_thread_rt(struct spi_controller *ctlr)

  {

  	dev_info(&ctlr->dev,
  		"will run message pump with realtime priority
  ");