// SPDX-License-Identifier: GPL-2.0+
  //
  // Actions Semi Owl SoCs DMA driver
  //
  // Copyright (c) 2014 Actions Semi Inc.
  // Author: David Liu <liuwei@actions-semi.com>
  //
  // Copyright (c) 2018 Linaro Ltd.
  // Author: Manivannan Sadhasivam <manivannan.sadhasivam@linaro.org>
  
  #include <linux/bitops.h>
  #include <linux/clk.h>
  #include <linux/delay.h>
  #include <linux/dmaengine.h>
  #include <linux/dma-mapping.h>
  #include <linux/dmapool.h>
  #include <linux/err.h>
  #include <linux/init.h>
  #include <linux/interrupt.h>
  #include <linux/io.h>
  #include <linux/mm.h>
  #include <linux/module.h>
  #include <linux/of_device.h>

  #include <linux/of_dma.h>

  #include <linux/slab.h>
  #include "virt-dma.h"
  
  #define OWL_DMA_FRAME_MAX_LENGTH		0xfffff
  
  /* Global DMA Controller Registers */
  #define OWL_DMA_IRQ_PD0				0x00
  #define OWL_DMA_IRQ_PD1				0x04
  #define OWL_DMA_IRQ_PD2				0x08
  #define OWL_DMA_IRQ_PD3				0x0C
  #define OWL_DMA_IRQ_EN0				0x10
  #define OWL_DMA_IRQ_EN1				0x14
  #define OWL_DMA_IRQ_EN2				0x18
  #define OWL_DMA_IRQ_EN3				0x1C
  #define OWL_DMA_SECURE_ACCESS_CTL		0x20
  #define OWL_DMA_NIC_QOS				0x24
  #define OWL_DMA_DBGSEL				0x28
  #define OWL_DMA_IDLE_STAT			0x2C
  
  /* Channel Registers */
  #define OWL_DMA_CHAN_BASE(i)			(0x100 + (i) * 0x100)
  #define OWL_DMAX_MODE				0x00
  #define OWL_DMAX_SOURCE				0x04
  #define OWL_DMAX_DESTINATION			0x08
  #define OWL_DMAX_FRAME_LEN			0x0C
  #define OWL_DMAX_FRAME_CNT			0x10
  #define OWL_DMAX_REMAIN_FRAME_CNT		0x14
  #define OWL_DMAX_REMAIN_CNT			0x18
  #define OWL_DMAX_SOURCE_STRIDE			0x1C
  #define OWL_DMAX_DESTINATION_STRIDE		0x20
  #define OWL_DMAX_START				0x24
  #define OWL_DMAX_PAUSE				0x28
  #define OWL_DMAX_CHAINED_CTL			0x2C
  #define OWL_DMAX_CONSTANT			0x30
  #define OWL_DMAX_LINKLIST_CTL			0x34
  #define OWL_DMAX_NEXT_DESCRIPTOR		0x38
  #define OWL_DMAX_CURRENT_DESCRIPTOR_NUM		0x3C
  #define OWL_DMAX_INT_CTL			0x40
  #define OWL_DMAX_INT_STATUS			0x44
  #define OWL_DMAX_CURRENT_SOURCE_POINTER		0x48
  #define OWL_DMAX_CURRENT_DESTINATION_POINTER	0x4C
  
  /* OWL_DMAX_MODE Bits */
  #define OWL_DMA_MODE_TS(x)			(((x) & GENMASK(5, 0)) << 0)
  #define OWL_DMA_MODE_ST(x)			(((x) & GENMASK(1, 0)) << 8)
  #define	OWL_DMA_MODE_ST_DEV			OWL_DMA_MODE_ST(0)
  #define	OWL_DMA_MODE_ST_DCU			OWL_DMA_MODE_ST(2)
  #define	OWL_DMA_MODE_ST_SRAM			OWL_DMA_MODE_ST(3)
  #define OWL_DMA_MODE_DT(x)			(((x) & GENMASK(1, 0)) << 10)
  #define	OWL_DMA_MODE_DT_DEV			OWL_DMA_MODE_DT(0)
  #define	OWL_DMA_MODE_DT_DCU			OWL_DMA_MODE_DT(2)
  #define	OWL_DMA_MODE_DT_SRAM			OWL_DMA_MODE_DT(3)
  #define OWL_DMA_MODE_SAM(x)			(((x) & GENMASK(1, 0)) << 16)
  #define	OWL_DMA_MODE_SAM_CONST			OWL_DMA_MODE_SAM(0)
  #define	OWL_DMA_MODE_SAM_INC			OWL_DMA_MODE_SAM(1)
  #define	OWL_DMA_MODE_SAM_STRIDE			OWL_DMA_MODE_SAM(2)
  #define OWL_DMA_MODE_DAM(x)			(((x) & GENMASK(1, 0)) << 18)
  #define	OWL_DMA_MODE_DAM_CONST			OWL_DMA_MODE_DAM(0)
  #define	OWL_DMA_MODE_DAM_INC			OWL_DMA_MODE_DAM(1)
  #define	OWL_DMA_MODE_DAM_STRIDE			OWL_DMA_MODE_DAM(2)
  #define OWL_DMA_MODE_PW(x)			(((x) & GENMASK(2, 0)) << 20)
  #define OWL_DMA_MODE_CB				BIT(23)
  #define OWL_DMA_MODE_NDDBW(x)			(((x) & 0x1) << 28)
  #define	OWL_DMA_MODE_NDDBW_32BIT		OWL_DMA_MODE_NDDBW(0)
  #define	OWL_DMA_MODE_NDDBW_8BIT			OWL_DMA_MODE_NDDBW(1)
  #define OWL_DMA_MODE_CFE			BIT(29)
  #define OWL_DMA_MODE_LME			BIT(30)
  #define OWL_DMA_MODE_CME			BIT(31)
  
  /* OWL_DMAX_LINKLIST_CTL Bits */
  #define OWL_DMA_LLC_SAV(x)			(((x) & GENMASK(1, 0)) << 8)
  #define	OWL_DMA_LLC_SAV_INC			OWL_DMA_LLC_SAV(0)
  #define	OWL_DMA_LLC_SAV_LOAD_NEXT		OWL_DMA_LLC_SAV(1)
  #define	OWL_DMA_LLC_SAV_LOAD_PREV		OWL_DMA_LLC_SAV(2)
  #define OWL_DMA_LLC_DAV(x)			(((x) & GENMASK(1, 0)) << 10)
  #define	OWL_DMA_LLC_DAV_INC			OWL_DMA_LLC_DAV(0)
  #define	OWL_DMA_LLC_DAV_LOAD_NEXT		OWL_DMA_LLC_DAV(1)
  #define	OWL_DMA_LLC_DAV_LOAD_PREV		OWL_DMA_LLC_DAV(2)
  #define OWL_DMA_LLC_SUSPEND			BIT(16)
  
  /* OWL_DMAX_INT_CTL Bits */
  #define OWL_DMA_INTCTL_BLOCK			BIT(0)
  #define OWL_DMA_INTCTL_SUPER_BLOCK		BIT(1)
  #define OWL_DMA_INTCTL_FRAME			BIT(2)
  #define OWL_DMA_INTCTL_HALF_FRAME		BIT(3)
  #define OWL_DMA_INTCTL_LAST_FRAME		BIT(4)
  
  /* OWL_DMAX_INT_STATUS Bits */
  #define OWL_DMA_INTSTAT_BLOCK			BIT(0)
  #define OWL_DMA_INTSTAT_SUPER_BLOCK		BIT(1)
  #define OWL_DMA_INTSTAT_FRAME			BIT(2)
  #define OWL_DMA_INTSTAT_HALF_FRAME		BIT(3)
  #define OWL_DMA_INTSTAT_LAST_FRAME		BIT(4)
  
  /* Pack shift and newshift in a single word */
  #define BIT_FIELD(val, width, shift, newshift)	\
  		((((val) >> (shift)) & ((BIT(width)) - 1)) << (newshift))

  /* Frame count value is fixed as 1 */
  #define FCNT_VAL				0x1

  /**

   * enum owl_dmadesc_offsets - Describe DMA descriptor, hardware link

   * list for dma transfer
   * @OWL_DMADESC_NEXT_LLI: physical address of the next link list
   * @OWL_DMADESC_SADDR: source physical address
   * @OWL_DMADESC_DADDR: destination physical address
   * @OWL_DMADESC_FLEN: frame length
   * @OWL_DMADESC_SRC_STRIDE: source stride
   * @OWL_DMADESC_DST_STRIDE: destination stride
   * @OWL_DMADESC_CTRLA: dma_mode and linklist ctrl config
   * @OWL_DMADESC_CTRLB: interrupt config
   * @OWL_DMADESC_CONST_NUM: data for constant fill

   * @OWL_DMADESC_SIZE: max size of this enum

   */

  enum owl_dmadesc_offsets {
  	OWL_DMADESC_NEXT_LLI = 0,
  	OWL_DMADESC_SADDR,
  	OWL_DMADESC_DADDR,
  	OWL_DMADESC_FLEN,
  	OWL_DMADESC_SRC_STRIDE,
  	OWL_DMADESC_DST_STRIDE,
  	OWL_DMADESC_CTRLA,
  	OWL_DMADESC_CTRLB,
  	OWL_DMADESC_CONST_NUM,
  	OWL_DMADESC_SIZE

  };

  enum owl_dma_id {
  	S900_DMA,
  	S700_DMA,
  };

  /**
   * struct owl_dma_lli - Link list for dma transfer
   * @hw: hardware link list
   * @phys: physical address of hardware link list
   * @node: node for txd's lli_list
   */
  struct owl_dma_lli {

  	u32			hw[OWL_DMADESC_SIZE];

  	dma_addr_t		phys;
  	struct list_head	node;
  };
  
  /**
   * struct owl_dma_txd - Wrapper for struct dma_async_tx_descriptor
   * @vd: virtual DMA descriptor
   * @lli_list: link list of lli nodes

   * @cyclic: flag to indicate cyclic transfers

   */
  struct owl_dma_txd {
  	struct virt_dma_desc	vd;
  	struct list_head	lli_list;

  	bool			cyclic;

  };
  
  /**
   * struct owl_dma_pchan - Holder for the physical channels
   * @id: physical index to this channel
   * @base: virtual memory base for the dma channel
   * @vchan: the virtual channel currently being served by this physical channel

   */
  struct owl_dma_pchan {
  	u32			id;
  	void __iomem		*base;
  	struct owl_dma_vchan	*vchan;

  };
  
  /**
   * struct owl_dma_pchan - Wrapper for DMA ENGINE channel
   * @vc: wrappped virtual channel
   * @pchan: the physical channel utilized by this channel
   * @txd: active transaction on this channel

   * @cfg: slave configuration for this channel
   * @drq: physical DMA request ID for this channel

   */
  struct owl_dma_vchan {
  	struct virt_dma_chan	vc;
  	struct owl_dma_pchan	*pchan;
  	struct owl_dma_txd	*txd;

  	struct dma_slave_config cfg;
  	u8			drq;

  };
  
  /**
   * struct owl_dma - Holder for the Owl DMA controller
   * @dma: dma engine for this instance
   * @base: virtual memory base for the DMA controller
   * @clk: clock for the DMA controller
   * @lock: a lock to use when change DMA controller global register
   * @lli_pool: a pool for the LLI descriptors

   * @irq: interrupt ID for the DMA controller

   * @nr_pchans: the number of physical channels
   * @pchans: array of data for the physical channels
   * @nr_vchans: the number of physical channels
   * @vchans: array of data for the physical channels

   * @devid: device id based on OWL SoC

   */
  struct owl_dma {
  	struct dma_device	dma;
  	void __iomem		*base;
  	struct clk		*clk;
  	spinlock_t		lock;
  	struct dma_pool		*lli_pool;
  	int			irq;
  
  	unsigned int		nr_pchans;
  	struct owl_dma_pchan	*pchans;
  
  	unsigned int		nr_vchans;
  	struct owl_dma_vchan	*vchans;

  	enum owl_dma_id		devid;

  };
  
  static void pchan_update(struct owl_dma_pchan *pchan, u32 reg,
  			 u32 val, bool state)
  {
  	u32 regval;
  
  	regval = readl(pchan->base + reg);
  
  	if (state)
  		regval |= val;
  	else
  		regval &= ~val;
  
  	writel(val, pchan->base + reg);
  }
  
  static void pchan_writel(struct owl_dma_pchan *pchan, u32 reg, u32 data)
  {
  	writel(data, pchan->base + reg);
  }
  
  static u32 pchan_readl(struct owl_dma_pchan *pchan, u32 reg)
  {
  	return readl(pchan->base + reg);
  }
  
  static void dma_update(struct owl_dma *od, u32 reg, u32 val, bool state)
  {
  	u32 regval;
  
  	regval = readl(od->base + reg);
  
  	if (state)
  		regval |= val;
  	else
  		regval &= ~val;
  
  	writel(val, od->base + reg);
  }
  
  static void dma_writel(struct owl_dma *od, u32 reg, u32 data)
  {
  	writel(data, od->base + reg);
  }
  
  static u32 dma_readl(struct owl_dma *od, u32 reg)
  {
  	return readl(od->base + reg);
  }
  
  static inline struct owl_dma *to_owl_dma(struct dma_device *dd)
  {
  	return container_of(dd, struct owl_dma, dma);
  }
  
  static struct device *chan2dev(struct dma_chan *chan)
  {
  	return &chan->dev->device;
  }
  
  static inline struct owl_dma_vchan *to_owl_vchan(struct dma_chan *chan)
  {
  	return container_of(chan, struct owl_dma_vchan, vc.chan);
  }
  
  static inline struct owl_dma_txd *to_owl_txd(struct dma_async_tx_descriptor *tx)
  {
  	return container_of(tx, struct owl_dma_txd, vd.tx);
  }
  
  static inline u32 llc_hw_ctrla(u32 mode, u32 llc_ctl)
  {
  	u32 ctl;
  
  	ctl = BIT_FIELD(mode, 4, 28, 28) |
  	      BIT_FIELD(mode, 8, 16, 20) |
  	      BIT_FIELD(mode, 4, 8, 16) |
  	      BIT_FIELD(mode, 6, 0, 10) |
  	      BIT_FIELD(llc_ctl, 2, 10, 8) |
  	      BIT_FIELD(llc_ctl, 2, 8, 6);
  
  	return ctl;
  }
  
  static inline u32 llc_hw_ctrlb(u32 int_ctl)
  {
  	u32 ctl;

  	/*
  	 * Irrespective of the SoC, ctrlb value starts filling from
  	 * bit 18.
  	 */

  	ctl = BIT_FIELD(int_ctl, 7, 0, 18);
  
  	return ctl;
  }

  static u32 llc_hw_flen(struct owl_dma_lli *lli)
  {
  	return lli->hw[OWL_DMADESC_FLEN] & GENMASK(19, 0);
  }

  static void owl_dma_free_lli(struct owl_dma *od,
  			     struct owl_dma_lli *lli)
  {
  	list_del(&lli->node);
  	dma_pool_free(od->lli_pool, lli, lli->phys);
  }
  
  static struct owl_dma_lli *owl_dma_alloc_lli(struct owl_dma *od)
  {
  	struct owl_dma_lli *lli;
  	dma_addr_t phys;
  
  	lli = dma_pool_alloc(od->lli_pool, GFP_NOWAIT, &phys);
  	if (!lli)
  		return NULL;
  
  	INIT_LIST_HEAD(&lli->node);
  	lli->phys = phys;
  
  	return lli;
  }
  
  static struct owl_dma_lli *owl_dma_add_lli(struct owl_dma_txd *txd,
  					   struct owl_dma_lli *prev,

  					   struct owl_dma_lli *next,
  					   bool is_cyclic)

  {

  	if (!is_cyclic)
  		list_add_tail(&next->node, &txd->lli_list);

  
  	if (prev) {

  		prev->hw[OWL_DMADESC_NEXT_LLI] = next->phys;
  		prev->hw[OWL_DMADESC_CTRLA] |=
  					llc_hw_ctrla(OWL_DMA_MODE_LME, 0);

  	}
  
  	return next;
  }
  
  static inline int owl_dma_cfg_lli(struct owl_dma_vchan *vchan,
  				  struct owl_dma_lli *lli,
  				  dma_addr_t src, dma_addr_t dst,

  				  u32 len, enum dma_transfer_direction dir,
  				  struct dma_slave_config *sconfig,
  				  bool is_cyclic)

  {

  	struct owl_dma *od = to_owl_dma(vchan->vc.chan.device);

  	u32 mode, ctrlb;

  
  	mode = OWL_DMA_MODE_PW(0);
  
  	switch (dir) {
  	case DMA_MEM_TO_MEM:
  		mode |= OWL_DMA_MODE_TS(0) | OWL_DMA_MODE_ST_DCU |
  			OWL_DMA_MODE_DT_DCU | OWL_DMA_MODE_SAM_INC |
  			OWL_DMA_MODE_DAM_INC;
  
  		break;

  	case DMA_MEM_TO_DEV:
  		mode |= OWL_DMA_MODE_TS(vchan->drq)
  			| OWL_DMA_MODE_ST_DCU | OWL_DMA_MODE_DT_DEV
  			| OWL_DMA_MODE_SAM_INC | OWL_DMA_MODE_DAM_CONST;
  
  		/*
  		 * Hardware only supports 32bit and 8bit buswidth. Since the
  		 * default is 32bit, select 8bit only when requested.
  		 */
  		if (sconfig->dst_addr_width == DMA_SLAVE_BUSWIDTH_1_BYTE)
  			mode |= OWL_DMA_MODE_NDDBW_8BIT;
  
  		break;
  	case DMA_DEV_TO_MEM:
  		 mode |= OWL_DMA_MODE_TS(vchan->drq)
  			| OWL_DMA_MODE_ST_DEV | OWL_DMA_MODE_DT_DCU
  			| OWL_DMA_MODE_SAM_CONST | OWL_DMA_MODE_DAM_INC;
  
  		/*
  		 * Hardware only supports 32bit and 8bit buswidth. Since the
  		 * default is 32bit, select 8bit only when requested.
  		 */
  		if (sconfig->src_addr_width == DMA_SLAVE_BUSWIDTH_1_BYTE)
  			mode |= OWL_DMA_MODE_NDDBW_8BIT;
  
  		break;

  	default:
  		return -EINVAL;
  	}

  	lli->hw[OWL_DMADESC_CTRLA] = llc_hw_ctrla(mode,
  						  OWL_DMA_LLC_SAV_LOAD_NEXT |
  						  OWL_DMA_LLC_DAV_LOAD_NEXT);

  	if (is_cyclic)

  		ctrlb = llc_hw_ctrlb(OWL_DMA_INTCTL_BLOCK);

  	else

  		ctrlb = llc_hw_ctrlb(OWL_DMA_INTCTL_SUPER_BLOCK);
  
  	lli->hw[OWL_DMADESC_NEXT_LLI] = 0; /* One link list by default */
  	lli->hw[OWL_DMADESC_SADDR] = src;
  	lli->hw[OWL_DMADESC_DADDR] = dst;
  	lli->hw[OWL_DMADESC_SRC_STRIDE] = 0;
  	lli->hw[OWL_DMADESC_DST_STRIDE] = 0;

  
  	if (od->devid == S700_DMA) {
  		/* Max frame length is 1MB */
  		lli->hw[OWL_DMADESC_FLEN] = len;
  		/*
  		 * On S700, word starts from offset 0x1C is shared between
  		 * frame count and ctrlb, where first 12 bits are for frame
  		 * count and rest of 20 bits are for ctrlb.
  		 */
  		lli->hw[OWL_DMADESC_CTRLB] = FCNT_VAL | ctrlb;
  	} else {
  		/*
  		 * On S900, word starts from offset 0xC is shared between
  		 * frame length (max frame length is 1MB) and frame count,
  		 * where first 20 bits are for frame length and rest of
  		 * 12 bits are for frame count.
  		 */
  		lli->hw[OWL_DMADESC_FLEN] = len | FCNT_VAL << 20;
  		lli->hw[OWL_DMADESC_CTRLB] = ctrlb;
  	}

  
  	return 0;
  }
  
  static struct owl_dma_pchan *owl_dma_get_pchan(struct owl_dma *od,
  					       struct owl_dma_vchan *vchan)
  {
  	struct owl_dma_pchan *pchan = NULL;
  	unsigned long flags;
  	int i;
  
  	for (i = 0; i < od->nr_pchans; i++) {
  		pchan = &od->pchans[i];

  		spin_lock_irqsave(&od->lock, flags);

  		if (!pchan->vchan) {
  			pchan->vchan = vchan;

  			spin_unlock_irqrestore(&od->lock, flags);

  			break;
  		}

  		spin_unlock_irqrestore(&od->lock, flags);

  	}
  
  	return pchan;
  }
  
  static int owl_dma_pchan_busy(struct owl_dma *od, struct owl_dma_pchan *pchan)
  {
  	unsigned int val;
  
  	val = dma_readl(od, OWL_DMA_IDLE_STAT);
  
  	return !(val & (1 << pchan->id));
  }
  
  static void owl_dma_terminate_pchan(struct owl_dma *od,
  				    struct owl_dma_pchan *pchan)
  {
  	unsigned long flags;
  	u32 irq_pd;
  
  	pchan_writel(pchan, OWL_DMAX_START, 0);
  	pchan_update(pchan, OWL_DMAX_INT_STATUS, 0xff, false);
  
  	spin_lock_irqsave(&od->lock, flags);
  	dma_update(od, OWL_DMA_IRQ_EN0, (1 << pchan->id), false);
  
  	irq_pd = dma_readl(od, OWL_DMA_IRQ_PD0);
  	if (irq_pd & (1 << pchan->id)) {
  		dev_warn(od->dma.dev,
  			 "terminating pchan %d that still has pending irq
  ",
  			 pchan->id);
  		dma_writel(od, OWL_DMA_IRQ_PD0, (1 << pchan->id));
  	}
  
  	pchan->vchan = NULL;
  
  	spin_unlock_irqrestore(&od->lock, flags);
  }

  static void owl_dma_pause_pchan(struct owl_dma_pchan *pchan)
  {
  	pchan_writel(pchan, 1, OWL_DMAX_PAUSE);
  }
  
  static void owl_dma_resume_pchan(struct owl_dma_pchan *pchan)
  {
  	pchan_writel(pchan, 0, OWL_DMAX_PAUSE);
  }

  static int owl_dma_start_next_txd(struct owl_dma_vchan *vchan)
  {
  	struct owl_dma *od = to_owl_dma(vchan->vc.chan.device);
  	struct virt_dma_desc *vd = vchan_next_desc(&vchan->vc);
  	struct owl_dma_pchan *pchan = vchan->pchan;
  	struct owl_dma_txd *txd = to_owl_txd(&vd->tx);
  	struct owl_dma_lli *lli;
  	unsigned long flags;
  	u32 int_ctl;
  
  	list_del(&vd->node);
  
  	vchan->txd = txd;
  
  	/* Wait for channel inactive */
  	while (owl_dma_pchan_busy(od, pchan))
  		cpu_relax();
  
  	lli = list_first_entry(&txd->lli_list,
  			       struct owl_dma_lli, node);

  	if (txd->cyclic)
  		int_ctl = OWL_DMA_INTCTL_BLOCK;
  	else
  		int_ctl = OWL_DMA_INTCTL_SUPER_BLOCK;

  
  	pchan_writel(pchan, OWL_DMAX_MODE, OWL_DMA_MODE_LME);
  	pchan_writel(pchan, OWL_DMAX_LINKLIST_CTL,
  		     OWL_DMA_LLC_SAV_LOAD_NEXT | OWL_DMA_LLC_DAV_LOAD_NEXT);
  	pchan_writel(pchan, OWL_DMAX_NEXT_DESCRIPTOR, lli->phys);
  	pchan_writel(pchan, OWL_DMAX_INT_CTL, int_ctl);
  
  	/* Clear IRQ status for this pchan */
  	pchan_update(pchan, OWL_DMAX_INT_STATUS, 0xff, false);
  
  	spin_lock_irqsave(&od->lock, flags);
  
  	dma_update(od, OWL_DMA_IRQ_EN0, (1 << pchan->id), true);
  
  	spin_unlock_irqrestore(&od->lock, flags);
  
  	dev_dbg(chan2dev(&vchan->vc.chan), "starting pchan %d
  ", pchan->id);
  
  	/* Start DMA transfer for this pchan */
  	pchan_writel(pchan, OWL_DMAX_START, 0x1);
  
  	return 0;
  }
  
  static void owl_dma_phy_free(struct owl_dma *od, struct owl_dma_vchan *vchan)
  {
  	/* Ensure that the physical channel is stopped */
  	owl_dma_terminate_pchan(od, vchan->pchan);
  
  	vchan->pchan = NULL;
  }
  
  static irqreturn_t owl_dma_interrupt(int irq, void *dev_id)
  {
  	struct owl_dma *od = dev_id;
  	struct owl_dma_vchan *vchan;
  	struct owl_dma_pchan *pchan;
  	unsigned long pending;
  	int i;
  	unsigned int global_irq_pending, chan_irq_pending;
  
  	spin_lock(&od->lock);
  
  	pending = dma_readl(od, OWL_DMA_IRQ_PD0);
  
  	/* Clear IRQ status for each pchan */
  	for_each_set_bit(i, &pending, od->nr_pchans) {
  		pchan = &od->pchans[i];
  		pchan_update(pchan, OWL_DMAX_INT_STATUS, 0xff, false);
  	}
  
  	/* Clear pending IRQ */
  	dma_writel(od, OWL_DMA_IRQ_PD0, pending);
  
  	/* Check missed pending IRQ */
  	for (i = 0; i < od->nr_pchans; i++) {
  		pchan = &od->pchans[i];
  		chan_irq_pending = pchan_readl(pchan, OWL_DMAX_INT_CTL) &
  				   pchan_readl(pchan, OWL_DMAX_INT_STATUS);
  
  		/* Dummy read to ensure OWL_DMA_IRQ_PD0 value is updated */
  		dma_readl(od, OWL_DMA_IRQ_PD0);
  
  		global_irq_pending = dma_readl(od, OWL_DMA_IRQ_PD0);

  		if (chan_irq_pending && !(global_irq_pending & BIT(i))) {

  			dev_dbg(od->dma.dev,
  				"global and channel IRQ pending match err
  ");
  
  			/* Clear IRQ status for this pchan */
  			pchan_update(pchan, OWL_DMAX_INT_STATUS,
  				     0xff, false);
  
  			/* Update global IRQ pending */
  			pending |= BIT(i);
  		}
  	}
  
  	spin_unlock(&od->lock);
  
  	for_each_set_bit(i, &pending, od->nr_pchans) {
  		struct owl_dma_txd *txd;
  
  		pchan = &od->pchans[i];
  
  		vchan = pchan->vchan;
  		if (!vchan) {
  			dev_warn(od->dma.dev, "no vchan attached on pchan %d
  ",
  				 pchan->id);
  			continue;
  		}
  
  		spin_lock(&vchan->vc.lock);
  
  		txd = vchan->txd;
  		if (txd) {
  			vchan->txd = NULL;
  
  			vchan_cookie_complete(&txd->vd);
  
  			/*
  			 * Start the next descriptor (if any),
  			 * otherwise free this channel.
  			 */
  			if (vchan_next_desc(&vchan->vc))
  				owl_dma_start_next_txd(vchan);
  			else
  				owl_dma_phy_free(od, vchan);
  		}
  
  		spin_unlock(&vchan->vc.lock);
  	}
  
  	return IRQ_HANDLED;
  }
  
  static void owl_dma_free_txd(struct owl_dma *od, struct owl_dma_txd *txd)
  {
  	struct owl_dma_lli *lli, *_lli;
  
  	if (unlikely(!txd))
  		return;
  
  	list_for_each_entry_safe(lli, _lli, &txd->lli_list, node)
  		owl_dma_free_lli(od, lli);
  
  	kfree(txd);
  }
  
  static void owl_dma_desc_free(struct virt_dma_desc *vd)
  {
  	struct owl_dma *od = to_owl_dma(vd->tx.chan->device);
  	struct owl_dma_txd *txd = to_owl_txd(&vd->tx);
  
  	owl_dma_free_txd(od, txd);
  }
  
  static int owl_dma_terminate_all(struct dma_chan *chan)
  {
  	struct owl_dma *od = to_owl_dma(chan->device);
  	struct owl_dma_vchan *vchan = to_owl_vchan(chan);
  	unsigned long flags;
  	LIST_HEAD(head);
  
  	spin_lock_irqsave(&vchan->vc.lock, flags);
  
  	if (vchan->pchan)
  		owl_dma_phy_free(od, vchan);
  
  	if (vchan->txd) {
  		owl_dma_desc_free(&vchan->txd->vd);
  		vchan->txd = NULL;
  	}
  
  	vchan_get_all_descriptors(&vchan->vc, &head);

  
  	spin_unlock_irqrestore(&vchan->vc.lock, flags);

  	vchan_dma_desc_free_list(&vchan->vc, &head);

  	return 0;
  }

  static int owl_dma_config(struct dma_chan *chan,
  			  struct dma_slave_config *config)
  {
  	struct owl_dma_vchan *vchan = to_owl_vchan(chan);
  
  	/* Reject definitely invalid configurations */
  	if (config->src_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES ||
  	    config->dst_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES)
  		return -EINVAL;
  
  	memcpy(&vchan->cfg, config, sizeof(struct dma_slave_config));
  
  	return 0;
  }
  
  static int owl_dma_pause(struct dma_chan *chan)
  {
  	struct owl_dma_vchan *vchan = to_owl_vchan(chan);
  	unsigned long flags;
  
  	spin_lock_irqsave(&vchan->vc.lock, flags);
  
  	owl_dma_pause_pchan(vchan->pchan);
  
  	spin_unlock_irqrestore(&vchan->vc.lock, flags);
  
  	return 0;
  }
  
  static int owl_dma_resume(struct dma_chan *chan)
  {
  	struct owl_dma_vchan *vchan = to_owl_vchan(chan);
  	unsigned long flags;
  
  	if (!vchan->pchan && !vchan->txd)
  		return 0;
  
  	dev_dbg(chan2dev(chan), "vchan %p: resume
  ", &vchan->vc);
  
  	spin_lock_irqsave(&vchan->vc.lock, flags);
  
  	owl_dma_resume_pchan(vchan->pchan);
  
  	spin_unlock_irqrestore(&vchan->vc.lock, flags);
  
  	return 0;
  }

  static u32 owl_dma_getbytes_chan(struct owl_dma_vchan *vchan)
  {
  	struct owl_dma_pchan *pchan;
  	struct owl_dma_txd *txd;
  	struct owl_dma_lli *lli;
  	unsigned int next_lli_phy;
  	size_t bytes;
  
  	pchan = vchan->pchan;
  	txd = vchan->txd;
  
  	if (!pchan || !txd)
  		return 0;
  
  	/* Get remain count of current node in link list */
  	bytes = pchan_readl(pchan, OWL_DMAX_REMAIN_CNT);
  
  	/* Loop through the preceding nodes to get total remaining bytes */
  	if (pchan_readl(pchan, OWL_DMAX_MODE) & OWL_DMA_MODE_LME) {
  		next_lli_phy = pchan_readl(pchan, OWL_DMAX_NEXT_DESCRIPTOR);
  		list_for_each_entry(lli, &txd->lli_list, node) {
  			/* Start from the next active node */
  			if (lli->phys == next_lli_phy) {
  				list_for_each_entry(lli, &txd->lli_list, node)

  					bytes += llc_hw_flen(lli);

  				break;
  			}
  		}
  	}
  
  	return bytes;
  }
  
  static enum dma_status owl_dma_tx_status(struct dma_chan *chan,
  					 dma_cookie_t cookie,
  					 struct dma_tx_state *state)
  {
  	struct owl_dma_vchan *vchan = to_owl_vchan(chan);
  	struct owl_dma_lli *lli;
  	struct virt_dma_desc *vd;
  	struct owl_dma_txd *txd;
  	enum dma_status ret;
  	unsigned long flags;
  	size_t bytes = 0;
  
  	ret = dma_cookie_status(chan, cookie, state);
  	if (ret == DMA_COMPLETE || !state)
  		return ret;
  
  	spin_lock_irqsave(&vchan->vc.lock, flags);
  
  	vd = vchan_find_desc(&vchan->vc, cookie);
  	if (vd) {
  		txd = to_owl_txd(&vd->tx);
  		list_for_each_entry(lli, &txd->lli_list, node)

  			bytes += llc_hw_flen(lli);

  	} else {
  		bytes = owl_dma_getbytes_chan(vchan);
  	}
  
  	spin_unlock_irqrestore(&vchan->vc.lock, flags);
  
  	dma_set_residue(state, bytes);
  
  	return ret;
  }
  
  static void owl_dma_phy_alloc_and_start(struct owl_dma_vchan *vchan)
  {
  	struct owl_dma *od = to_owl_dma(vchan->vc.chan.device);
  	struct owl_dma_pchan *pchan;
  
  	pchan = owl_dma_get_pchan(od, vchan);
  	if (!pchan)
  		return;
  
  	dev_dbg(od->dma.dev, "allocated pchan %d
  ", pchan->id);
  
  	vchan->pchan = pchan;
  	owl_dma_start_next_txd(vchan);
  }
  
  static void owl_dma_issue_pending(struct dma_chan *chan)
  {
  	struct owl_dma_vchan *vchan = to_owl_vchan(chan);
  	unsigned long flags;
  
  	spin_lock_irqsave(&vchan->vc.lock, flags);
  	if (vchan_issue_pending(&vchan->vc)) {
  		if (!vchan->pchan)
  			owl_dma_phy_alloc_and_start(vchan);
  	}
  	spin_unlock_irqrestore(&vchan->vc.lock, flags);
  }
  
  static struct dma_async_tx_descriptor
  		*owl_dma_prep_memcpy(struct dma_chan *chan,
  				     dma_addr_t dst, dma_addr_t src,
  				     size_t len, unsigned long flags)
  {
  	struct owl_dma *od = to_owl_dma(chan->device);
  	struct owl_dma_vchan *vchan = to_owl_vchan(chan);
  	struct owl_dma_txd *txd;
  	struct owl_dma_lli *lli, *prev = NULL;
  	size_t offset, bytes;
  	int ret;
  
  	if (!len)
  		return NULL;
  
  	txd = kzalloc(sizeof(*txd), GFP_NOWAIT);
  	if (!txd)
  		return NULL;
  
  	INIT_LIST_HEAD(&txd->lli_list);
  
  	/* Process the transfer as frame by frame */
  	for (offset = 0; offset < len; offset += bytes) {
  		lli = owl_dma_alloc_lli(od);
  		if (!lli) {
  			dev_warn(chan2dev(chan), "failed to allocate lli
  ");
  			goto err_txd_free;
  		}
  
  		bytes = min_t(size_t, (len - offset), OWL_DMA_FRAME_MAX_LENGTH);
  
  		ret = owl_dma_cfg_lli(vchan, lli, src + offset, dst + offset,

  				      bytes, DMA_MEM_TO_MEM,
  				      &vchan->cfg, txd->cyclic);

  		if (ret) {
  			dev_warn(chan2dev(chan), "failed to config lli
  ");
  			goto err_txd_free;
  		}

  		prev = owl_dma_add_lli(txd, prev, lli, false);

  	}
  
  	return vchan_tx_prep(&vchan->vc, &txd->vd, flags);
  
  err_txd_free:
  	owl_dma_free_txd(od, txd);
  	return NULL;
  }

  static struct dma_async_tx_descriptor
  		*owl_dma_prep_slave_sg(struct dma_chan *chan,
  				       struct scatterlist *sgl,
  				       unsigned int sg_len,
  				       enum dma_transfer_direction dir,
  				       unsigned long flags, void *context)
  {
  	struct owl_dma *od = to_owl_dma(chan->device);
  	struct owl_dma_vchan *vchan = to_owl_vchan(chan);
  	struct dma_slave_config *sconfig = &vchan->cfg;
  	struct owl_dma_txd *txd;
  	struct owl_dma_lli *lli, *prev = NULL;
  	struct scatterlist *sg;
  	dma_addr_t addr, src = 0, dst = 0;
  	size_t len;
  	int ret, i;
  
  	txd = kzalloc(sizeof(*txd), GFP_NOWAIT);
  	if (!txd)
  		return NULL;
  
  	INIT_LIST_HEAD(&txd->lli_list);
  
  	for_each_sg(sgl, sg, sg_len, i) {
  		addr = sg_dma_address(sg);
  		len = sg_dma_len(sg);
  
  		if (len > OWL_DMA_FRAME_MAX_LENGTH) {
  			dev_err(od->dma.dev,
  				"frame length exceeds max supported length");
  			goto err_txd_free;
  		}
  
  		lli = owl_dma_alloc_lli(od);
  		if (!lli) {
  			dev_err(chan2dev(chan), "failed to allocate lli");
  			goto err_txd_free;
  		}
  
  		if (dir == DMA_MEM_TO_DEV) {
  			src = addr;
  			dst = sconfig->dst_addr;
  		} else {
  			src = sconfig->src_addr;
  			dst = addr;
  		}
  
  		ret = owl_dma_cfg_lli(vchan, lli, src, dst, len, dir, sconfig,
  				      txd->cyclic);
  		if (ret) {
  			dev_warn(chan2dev(chan), "failed to config lli");
  			goto err_txd_free;
  		}
  
  		prev = owl_dma_add_lli(txd, prev, lli, false);
  	}
  
  	return vchan_tx_prep(&vchan->vc, &txd->vd, flags);
  
  err_txd_free:
  	owl_dma_free_txd(od, txd);
  
  	return NULL;
  }
  
  static struct dma_async_tx_descriptor
  		*owl_prep_dma_cyclic(struct dma_chan *chan,
  				     dma_addr_t buf_addr, size_t buf_len,
  				     size_t period_len,
  				     enum dma_transfer_direction dir,
  				     unsigned long flags)
  {
  	struct owl_dma *od = to_owl_dma(chan->device);
  	struct owl_dma_vchan *vchan = to_owl_vchan(chan);
  	struct dma_slave_config *sconfig = &vchan->cfg;
  	struct owl_dma_txd *txd;
  	struct owl_dma_lli *lli, *prev = NULL, *first = NULL;
  	dma_addr_t src = 0, dst = 0;
  	unsigned int periods = buf_len / period_len;
  	int ret, i;
  
  	txd = kzalloc(sizeof(*txd), GFP_NOWAIT);
  	if (!txd)
  		return NULL;
  
  	INIT_LIST_HEAD(&txd->lli_list);
  	txd->cyclic = true;
  
  	for (i = 0; i < periods; i++) {
  		lli = owl_dma_alloc_lli(od);
  		if (!lli) {
  			dev_warn(chan2dev(chan), "failed to allocate lli");
  			goto err_txd_free;
  		}
  
  		if (dir == DMA_MEM_TO_DEV) {
  			src = buf_addr + (period_len * i);
  			dst = sconfig->dst_addr;
  		} else if (dir == DMA_DEV_TO_MEM) {
  			src = sconfig->src_addr;
  			dst = buf_addr + (period_len * i);
  		}
  
  		ret = owl_dma_cfg_lli(vchan, lli, src, dst, period_len,
  				      dir, sconfig, txd->cyclic);
  		if (ret) {
  			dev_warn(chan2dev(chan), "failed to config lli");
  			goto err_txd_free;
  		}
  
  		if (!first)
  			first = lli;
  
  		prev = owl_dma_add_lli(txd, prev, lli, false);
  	}
  
  	/* close the cyclic list */
  	owl_dma_add_lli(txd, prev, first, true);
  
  	return vchan_tx_prep(&vchan->vc, &txd->vd, flags);
  
  err_txd_free:
  	owl_dma_free_txd(od, txd);
  
  	return NULL;
  }

  static void owl_dma_free_chan_resources(struct dma_chan *chan)
  {
  	struct owl_dma_vchan *vchan = to_owl_vchan(chan);
  
  	/* Ensure all queued descriptors are freed */
  	vchan_free_chan_resources(&vchan->vc);
  }
  
  static inline void owl_dma_free(struct owl_dma *od)
  {
  	struct owl_dma_vchan *vchan = NULL;
  	struct owl_dma_vchan *next;
  
  	list_for_each_entry_safe(vchan,
  				 next, &od->dma.channels, vc.chan.device_node) {
  		list_del(&vchan->vc.chan.device_node);
  		tasklet_kill(&vchan->vc.task);
  	}
  }

  static struct dma_chan *owl_dma_of_xlate(struct of_phandle_args *dma_spec,
  					 struct of_dma *ofdma)
  {
  	struct owl_dma *od = ofdma->of_dma_data;
  	struct owl_dma_vchan *vchan;
  	struct dma_chan *chan;
  	u8 drq = dma_spec->args[0];
  
  	if (drq > od->nr_vchans)
  		return NULL;
  
  	chan = dma_get_any_slave_channel(&od->dma);
  	if (!chan)
  		return NULL;
  
  	vchan = to_owl_vchan(chan);
  	vchan->drq = drq;
  
  	return chan;
  }

  static const struct of_device_id owl_dma_match[] = {
  	{ .compatible = "actions,s900-dma", .data = (void *)S900_DMA,},
  	{ .compatible = "actions,s700-dma", .data = (void *)S700_DMA,},
  	{ /* sentinel */ },
  };
  MODULE_DEVICE_TABLE(of, owl_dma_match);

  static int owl_dma_probe(struct platform_device *pdev)
  {
  	struct device_node *np = pdev->dev.of_node;
  	struct owl_dma *od;

  	int ret, i, nr_channels, nr_requests;
  
  	od = devm_kzalloc(&pdev->dev, sizeof(*od), GFP_KERNEL);
  	if (!od)
  		return -ENOMEM;

  	od->base = devm_platform_ioremap_resource(pdev, 0);

  	if (IS_ERR(od->base))
  		return PTR_ERR(od->base);
  
  	ret = of_property_read_u32(np, "dma-channels", &nr_channels);
  	if (ret) {
  		dev_err(&pdev->dev, "can't get dma-channels
  ");
  		return ret;
  	}
  
  	ret = of_property_read_u32(np, "dma-requests", &nr_requests);
  	if (ret) {
  		dev_err(&pdev->dev, "can't get dma-requests
  ");
  		return ret;
  	}
  
  	dev_info(&pdev->dev, "dma-channels %d, dma-requests %d
  ",
  		 nr_channels, nr_requests);

  	od->devid = (enum owl_dma_id)of_device_get_match_data(&pdev->dev);

  	od->nr_pchans = nr_channels;
  	od->nr_vchans = nr_requests;
  
  	pdev->dev.coherent_dma_mask = DMA_BIT_MASK(32);
  
  	platform_set_drvdata(pdev, od);
  	spin_lock_init(&od->lock);
  
  	dma_cap_set(DMA_MEMCPY, od->dma.cap_mask);

  	dma_cap_set(DMA_SLAVE, od->dma.cap_mask);
  	dma_cap_set(DMA_CYCLIC, od->dma.cap_mask);

  
  	od->dma.dev = &pdev->dev;
  	od->dma.device_free_chan_resources = owl_dma_free_chan_resources;
  	od->dma.device_tx_status = owl_dma_tx_status;
  	od->dma.device_issue_pending = owl_dma_issue_pending;
  	od->dma.device_prep_dma_memcpy = owl_dma_prep_memcpy;

  	od->dma.device_prep_slave_sg = owl_dma_prep_slave_sg;
  	od->dma.device_prep_dma_cyclic = owl_prep_dma_cyclic;
  	od->dma.device_config = owl_dma_config;
  	od->dma.device_pause = owl_dma_pause;
  	od->dma.device_resume = owl_dma_resume;

  	od->dma.device_terminate_all = owl_dma_terminate_all;
  	od->dma.src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
  	od->dma.dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
  	od->dma.directions = BIT(DMA_MEM_TO_MEM);
  	od->dma.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
  
  	INIT_LIST_HEAD(&od->dma.channels);
  
  	od->clk = devm_clk_get(&pdev->dev, NULL);
  	if (IS_ERR(od->clk)) {
  		dev_err(&pdev->dev, "unable to get clock
  ");
  		return PTR_ERR(od->clk);
  	}
  
  	/*
  	 * Eventhough the DMA controller is capable of generating 4
  	 * IRQ's for DMA priority feature, we only use 1 IRQ for
  	 * simplification.
  	 */
  	od->irq = platform_get_irq(pdev, 0);
  	ret = devm_request_irq(&pdev->dev, od->irq, owl_dma_interrupt, 0,
  			       dev_name(&pdev->dev), od);
  	if (ret) {
  		dev_err(&pdev->dev, "unable to request IRQ
  ");
  		return ret;
  	}
  
  	/* Init physical channel */
  	od->pchans = devm_kcalloc(&pdev->dev, od->nr_pchans,
  				  sizeof(struct owl_dma_pchan), GFP_KERNEL);
  	if (!od->pchans)
  		return -ENOMEM;
  
  	for (i = 0; i < od->nr_pchans; i++) {
  		struct owl_dma_pchan *pchan = &od->pchans[i];
  
  		pchan->id = i;
  		pchan->base = od->base + OWL_DMA_CHAN_BASE(i);
  	}
  
  	/* Init virtual channel */
  	od->vchans = devm_kcalloc(&pdev->dev, od->nr_vchans,
  				  sizeof(struct owl_dma_vchan), GFP_KERNEL);
  	if (!od->vchans)
  		return -ENOMEM;
  
  	for (i = 0; i < od->nr_vchans; i++) {
  		struct owl_dma_vchan *vchan = &od->vchans[i];
  
  		vchan->vc.desc_free = owl_dma_desc_free;
  		vchan_init(&vchan->vc, &od->dma);
  	}
  
  	/* Create a pool of consistent memory blocks for hardware descriptors */
  	od->lli_pool = dma_pool_create(dev_name(od->dma.dev), od->dma.dev,
  				       sizeof(struct owl_dma_lli),
  				       __alignof__(struct owl_dma_lli),
  				       0);
  	if (!od->lli_pool) {
  		dev_err(&pdev->dev, "unable to allocate DMA descriptor pool
  ");
  		return -ENOMEM;
  	}
  
  	clk_prepare_enable(od->clk);
  
  	ret = dma_async_device_register(&od->dma);
  	if (ret) {
  		dev_err(&pdev->dev, "failed to register DMA engine device
  ");
  		goto err_pool_free;
  	}

  	/* Device-tree DMA controller registration */
  	ret = of_dma_controller_register(pdev->dev.of_node,
  					 owl_dma_of_xlate, od);
  	if (ret) {
  		dev_err(&pdev->dev, "of_dma_controller_register failed
  ");
  		goto err_dma_unregister;
  	}

  	return 0;

  err_dma_unregister:
  	dma_async_device_unregister(&od->dma);

  err_pool_free:
  	clk_disable_unprepare(od->clk);
  	dma_pool_destroy(od->lli_pool);
  
  	return ret;
  }
  
  static int owl_dma_remove(struct platform_device *pdev)
  {
  	struct owl_dma *od = platform_get_drvdata(pdev);

  	of_dma_controller_free(pdev->dev.of_node);

  	dma_async_device_unregister(&od->dma);
  
  	/* Mask all interrupts for this execution environment */
  	dma_writel(od, OWL_DMA_IRQ_EN0, 0x0);
  
  	/* Make sure we won't have any further interrupts */
  	devm_free_irq(od->dma.dev, od->irq, od);
  
  	owl_dma_free(od);
  
  	clk_disable_unprepare(od->clk);
  
  	return 0;
  }

  static struct platform_driver owl_dma_driver = {
  	.probe	= owl_dma_probe,
  	.remove	= owl_dma_remove,
  	.driver = {
  		.name = "dma-owl",
  		.of_match_table = of_match_ptr(owl_dma_match),
  	},
  };
  
  static int owl_dma_init(void)
  {
  	return platform_driver_register(&owl_dma_driver);
  }
  subsys_initcall(owl_dma_init);
  
  static void __exit owl_dma_exit(void)
  {
  	platform_driver_unregister(&owl_dma_driver);
  }
  module_exit(owl_dma_exit);
  
  MODULE_AUTHOR("David Liu <liuwei@actions-semi.com>");
  MODULE_AUTHOR("Manivannan Sadhasivam <manivannan.sadhasivam@linaro.org>");
  MODULE_DESCRIPTION("Actions Semi Owl SoCs DMA driver");
  MODULE_LICENSE("GPL");