dmaengine: at_hdmac: add DMA slave transfers

This patch for at_hdmac adds the slave transfers capability to the Atmel DMA controller available on some AT91 SOCs. This allow peripheral to memory and memory to peripheral transfers with hardware handshaking. Slave structure for controller specific information is passed through channel private data. This at_dma_slave structure is defined in at_hdmac.h header file and relative hardware definition are moved to this file from at_hdmac_regs.h. Doing this we allow the channel configuration from platform definition code. This work is intensively based on dw_dmac and several slave implementations. Signed-off-by: Nicolas Ferre <nicolas.ferre@atmel.com> Signed-off-by: Dan Williams <dan.j.williams@intel.com>

dmaengine: at_hdmac: add DMA slave transfers
This patch for at_hdmac adds the slave transfers capability to the Atmel DMA controller available on some AT91 SOCs. This allow peripheral to memory and memory to peripheral transfers with hardware handshaking. Slave structure for controller specific information is passed through channel private data. This at_dma_slave structure is defined in at_hdmac.h header file and relative hardware definition are moved to this file from at_hdmac_regs.h. Doing this we allow the channel configuration from platform definition code. This work is intensively based on dw_dmac and several slave implementations. Signed-off-by: Nicolas Ferre <nicolas.ferre@atmel.com> Signed-off-by: Dan Williams <dan.j.williams@intel.com>
Nicolas Ferre · Dan Williams
1 parent dc78baa2b9
Showing 3 changed files with 290 additions and 43 deletions Side-by-side Diff
arch/arm/mach-at91/include/mach/at_hdmac.h
drivers/dma/at_hdmac.c
drivers/dma/at_hdmac_regs.h
@@ -23,5 +23,81 @@
 	dma_cap_mask_t  cap_mask;
 };
  
+/**
+ * enum at_dma_slave_width - DMA slave register access width.
+ * @AT_DMA_SLAVE_WIDTH_8BIT: Do 8-bit slave register accesses
+ * @AT_DMA_SLAVE_WIDTH_16BIT: Do 16-bit slave register accesses
+ * @AT_DMA_SLAVE_WIDTH_32BIT: Do 32-bit slave register accesses
+ */
+enum at_dma_slave_width {
+	AT_DMA_SLAVE_WIDTH_8BIT = 0,
+	AT_DMA_SLAVE_WIDTH_16BIT,
+	AT_DMA_SLAVE_WIDTH_32BIT,
+};
+
+/**
+ * struct at_dma_slave - Controller-specific information about a slave
+ * @dma_dev: required DMA master device
+ * @tx_reg: physical address of data register used for
+ *	memory-to-peripheral transfers
+ * @rx_reg: physical address of data register used for
+ *	peripheral-to-memory transfers
+ * @reg_width: peripheral register width
+ * @cfg: Platform-specific initializer for the CFG register
+ * @ctrla: Platform-specific initializer for the CTRLA register
+ */
+struct at_dma_slave {
+	struct device		*dma_dev;
+	dma_addr_t		tx_reg;
+	dma_addr_t		rx_reg;
+	enum at_dma_slave_width	reg_width;
+	u32			cfg;
+	u32			ctrla;
+};
+
+
+/* Platform-configurable bits in CFG */
+#define	ATC_SRC_PER(h)		(0xFU & (h))	/* Channel src rq associated with periph handshaking ifc h */
+#define	ATC_DST_PER(h)		((0xFU & (h)) <<  4)	/* Channel dst rq associated with periph handshaking ifc h */
+#define	ATC_SRC_REP		(0x1 <<  8)	/* Source Replay Mod */
+#define	ATC_SRC_H2SEL		(0x1 <<  9)	/* Source Handshaking Mod */
+#define		ATC_SRC_H2SEL_SW	(0x0 <<  9)
+#define		ATC_SRC_H2SEL_HW	(0x1 <<  9)
+#define	ATC_DST_REP		(0x1 << 12)	/* Destination Replay Mod */
+#define	ATC_DST_H2SEL		(0x1 << 13)	/* Destination Handshaking Mod */
+#define		ATC_DST_H2SEL_SW	(0x0 << 13)
+#define		ATC_DST_H2SEL_HW	(0x1 << 13)
+#define	ATC_SOD			(0x1 << 16)	/* Stop On Done */
+#define	ATC_LOCK_IF		(0x1 << 20)	/* Interface Lock */
+#define	ATC_LOCK_B		(0x1 << 21)	/* AHB Bus Lock */
+#define	ATC_LOCK_IF_L		(0x1 << 22)	/* Master Interface Arbiter Lock */
+#define		ATC_LOCK_IF_L_CHUNK	(0x0 << 22)
+#define		ATC_LOCK_IF_L_BUFFER	(0x1 << 22)
+#define	ATC_AHB_PROT_MASK	(0x7 << 24)	/* AHB Protection */
+#define	ATC_FIFOCFG_MASK	(0x3 << 28)	/* FIFO Request Configuration */
+#define		ATC_FIFOCFG_LARGESTBURST	(0x0 << 28)
+#define		ATC_FIFOCFG_HALFFIFO		(0x1 << 28)
+#define		ATC_FIFOCFG_ENOUGHSPACE		(0x2 << 28)
+
+/* Platform-configurable bits in CTRLA */
+#define	ATC_SCSIZE_MASK		(0x7 << 16)	/* Source Chunk Transfer Size */
+#define		ATC_SCSIZE_1		(0x0 << 16)
+#define		ATC_SCSIZE_4		(0x1 << 16)
+#define		ATC_SCSIZE_8		(0x2 << 16)
+#define		ATC_SCSIZE_16		(0x3 << 16)
+#define		ATC_SCSIZE_32		(0x4 << 16)
+#define		ATC_SCSIZE_64		(0x5 << 16)
+#define		ATC_SCSIZE_128		(0x6 << 16)
+#define		ATC_SCSIZE_256		(0x7 << 16)
+#define	ATC_DCSIZE_MASK		(0x7 << 20)	/* Destination Chunk Transfer Size */
+#define		ATC_DCSIZE_1		(0x0 << 20)
+#define		ATC_DCSIZE_4		(0x1 << 20)
+#define		ATC_DCSIZE_8		(0x2 << 20)
+#define		ATC_DCSIZE_16		(0x3 << 20)
+#define		ATC_DCSIZE_32		(0x4 << 20)
+#define		ATC_DCSIZE_64		(0x5 << 20)
+#define		ATC_DCSIZE_128		(0x6 << 20)
+#define		ATC_DCSIZE_256		(0x7 << 20)
+
 #endif /* AT_HDMAC_H */
@@ -608,7 +608,188 @@
 	return NULL;
 }
  
+
 /**
+ * atc_prep_slave_sg - prepare descriptors for a DMA_SLAVE transaction
+ * @chan: DMA channel
+ * @sgl: scatterlist to transfer to/from
+ * @sg_len: number of entries in @scatterlist
+ * @direction: DMA direction
+ * @flags: tx descriptor status flags
+ */
+static struct dma_async_tx_descriptor *
+atc_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
+		unsigned int sg_len, enum dma_data_direction direction,
+		unsigned long flags)
+{
+	struct at_dma_chan	*atchan = to_at_dma_chan(chan);
+	struct at_dma_slave	*atslave = chan->private;
+	struct at_desc		*first = NULL;
+	struct at_desc		*prev = NULL;
+	u32			ctrla;
+	u32			ctrlb;
+	dma_addr_t		reg;
+	unsigned int		reg_width;
+	unsigned int		mem_width;
+	unsigned int		i;
+	struct scatterlist	*sg;
+	size_t			total_len = 0;
+
+	dev_vdbg(chan2dev(chan), "prep_slave_sg: %s f0x%lx\n",
+			direction == DMA_TO_DEVICE ? "TO DEVICE" : "FROM DEVICE",
+			flags);
+
+	if (unlikely(!atslave || !sg_len)) {
+		dev_dbg(chan2dev(chan), "prep_dma_memcpy: length is zero!\n");
+		return NULL;
+	}
+
+	reg_width = atslave->reg_width;
+
+	sg_len = dma_map_sg(chan2parent(chan), sgl, sg_len, direction);
+
+	ctrla = ATC_DEFAULT_CTRLA | atslave->ctrla;
+	ctrlb = ATC_DEFAULT_CTRLB | ATC_IEN;
+
+	switch (direction) {
+	case DMA_TO_DEVICE:
+		ctrla |=  ATC_DST_WIDTH(reg_width);
+		ctrlb |=  ATC_DST_ADDR_MODE_FIXED
+			| ATC_SRC_ADDR_MODE_INCR
+			| ATC_FC_MEM2PER;
+		reg = atslave->tx_reg;
+		for_each_sg(sgl, sg, sg_len, i) {
+			struct at_desc	*desc;
+			u32		len;
+			u32		mem;
+
+			desc = atc_desc_get(atchan);
+			if (!desc)
+				goto err_desc_get;
+
+			mem = sg_phys(sg);
+			len = sg_dma_len(sg);
+			mem_width = 2;
+			if (unlikely(mem & 3 || len & 3))
+				mem_width = 0;
+
+			desc->lli.saddr = mem;
+			desc->lli.daddr = reg;
+			desc->lli.ctrla = ctrla
+					| ATC_SRC_WIDTH(mem_width)
+					| len >> mem_width;
+			desc->lli.ctrlb = ctrlb;
+
+			if (!first) {
+				first = desc;
+			} else {
+				/* inform the HW lli about chaining */
+				prev->lli.dscr = desc->txd.phys;
+				/* insert the link descriptor to the LD ring */
+				list_add_tail(&desc->desc_node,
+						&first->txd.tx_list);
+			}
+			prev = desc;
+			total_len += len;
+		}
+		break;
+	case DMA_FROM_DEVICE:
+		ctrla |=  ATC_SRC_WIDTH(reg_width);
+		ctrlb |=  ATC_DST_ADDR_MODE_INCR
+			| ATC_SRC_ADDR_MODE_FIXED
+			| ATC_FC_PER2MEM;
+
+		reg = atslave->rx_reg;
+		for_each_sg(sgl, sg, sg_len, i) {
+			struct at_desc	*desc;
+			u32		len;
+			u32		mem;
+
+			desc = atc_desc_get(atchan);
+			if (!desc)
+				goto err_desc_get;
+
+			mem = sg_phys(sg);
+			len = sg_dma_len(sg);
+			mem_width = 2;
+			if (unlikely(mem & 3 || len & 3))
+				mem_width = 0;
+
+			desc->lli.saddr = reg;
+			desc->lli.daddr = mem;
+			desc->lli.ctrla = ctrla
+					| ATC_DST_WIDTH(mem_width)
+					| len >> mem_width;
+			desc->lli.ctrlb = ctrlb;
+
+			if (!first) {
+				first = desc;
+			} else {
+				/* inform the HW lli about chaining */
+				prev->lli.dscr = desc->txd.phys;
+				/* insert the link descriptor to the LD ring */
+				list_add_tail(&desc->desc_node,
+						&first->txd.tx_list);
+			}
+			prev = desc;
+			total_len += len;
+		}
+		break;
+	default:
+		return NULL;
+	}
+
+	/* set end-of-link to the last link descriptor of list*/
+	set_desc_eol(prev);
+
+	/* First descriptor of the chain embedds additional information */
+	first->txd.cookie = -EBUSY;
+	first->len = total_len;
+
+	/* last link descriptor of list is responsible of flags */
+	prev->txd.flags = flags; /* client is in control of this ack */
+
+	return &first->txd;
+
+err_desc_get:
+	dev_err(chan2dev(chan), "not enough descriptors available\n");
+	atc_desc_put(atchan, first);
+	return NULL;
+}
+
+static void atc_terminate_all(struct dma_chan *chan)
+{
+	struct at_dma_chan	*atchan = to_at_dma_chan(chan);
+	struct at_dma		*atdma = to_at_dma(chan->device);
+	struct at_desc		*desc, *_desc;
+	LIST_HEAD(list);
+
+	/*
+	 * This is only called when something went wrong elsewhere, so
+	 * we don't really care about the data. Just disable the
+	 * channel. We still have to poll the channel enable bit due
+	 * to AHB/HSB limitations.
+	 */
+	spin_lock_bh(&atchan->lock);
+
+	dma_writel(atdma, CHDR, atchan->mask);
+
+	/* confirm that this channel is disabled */
+	while (dma_readl(atdma, CHSR) & atchan->mask)
+		cpu_relax();
+
+	/* active_list entries will end up before queued entries */
+	list_splice_init(&atchan->queue, &list);
+	list_splice_init(&atchan->active_list, &list);
+
+	spin_unlock_bh(&atchan->lock);
+
+	/* Flush all pending and queued descriptors */
+	list_for_each_entry_safe(desc, _desc, &list, desc_node)
+		atc_chain_complete(atchan, desc);
+}
+
+/**
  * atc_is_tx_complete - poll for transaction completion
  * @chan: DMA channel
  * @cookie: transaction identifier to check status of
  
@@ -686,7 +867,9 @@
 	struct at_dma_chan	*atchan = to_at_dma_chan(chan);
 	struct at_dma		*atdma = to_at_dma(chan->device);
 	struct at_desc		*desc;
+	struct at_dma_slave	*atslave;
 	int			i;
+	u32			cfg;
 	LIST_HEAD(tmp_list);
  
 	dev_vdbg(chan2dev(chan), "alloc_chan_resources\n");
@@ -697,7 +880,23 @@
 		return -EIO;
 	}
  
-	/* have we already been set up? */
+	cfg = ATC_DEFAULT_CFG;
+
+	atslave = chan->private;
+	if (atslave) {
+		/*
+		 * We need controller-specific data to set up slave
+		 * transfers.
+		 */
+		BUG_ON(!atslave->dma_dev || atslave->dma_dev != atdma->dma_common.dev);
+
+		/* if cfg configuration specified take it instad of default */
+		if (atslave->cfg)
+			cfg = atslave->cfg;
+	}
+
+	/* have we already been set up?
+	 * reconfigure channel but no need to reallocate descriptors */
 	if (!list_empty(&atchan->free_list))
 		return atchan->descs_allocated;
  
@@ -719,7 +918,7 @@
 	spin_unlock_bh(&atchan->lock);
  
 	/* channel parameters */
-	channel_writel(atchan, CFG, ATC_DEFAULT_CFG);
+	channel_writel(atchan, CFG, cfg);
  
 	dev_dbg(chan2dev(chan),
 		"alloc_chan_resources: allocated %d descriptors\n",
@@ -887,6 +1086,11 @@
 	/* set prep routines based on capability */
 	if (dma_has_cap(DMA_MEMCPY, atdma->dma_common.cap_mask))
 		atdma->dma_common.device_prep_dma_memcpy = atc_prep_dma_memcpy;
+
+	if (dma_has_cap(DMA_SLAVE, atdma->dma_common.cap_mask)) {
+		atdma->dma_common.device_prep_slave_sg = atc_prep_slave_sg;
+		atdma->dma_common.device_terminate_all = atc_terminate_all;
+	}
  
 	dma_writel(atdma, EN, AT_DMA_ENABLE);
  
@@ -87,29 +87,14 @@
 /* Bitfields in CTRLA */
 #define	ATC_BTSIZE_MAX		0xFFFFUL	/* Maximum Buffer Transfer Size */
 #define	ATC_BTSIZE(x)		(ATC_BTSIZE_MAX & (x)) /* Buffer Transfer Size */
-#define	ATC_SCSIZE_MASK		(0x7 << 16)	/* Source Chunk Transfer Size */
-#define		ATC_SCSIZE_1		(0x0 << 16)
-#define		ATC_SCSIZE_4		(0x1 << 16)
-#define		ATC_SCSIZE_8		(0x2 << 16)
-#define		ATC_SCSIZE_16		(0x3 << 16)
-#define		ATC_SCSIZE_32		(0x4 << 16)
-#define		ATC_SCSIZE_64		(0x5 << 16)
-#define		ATC_SCSIZE_128		(0x6 << 16)
-#define		ATC_SCSIZE_256		(0x7 << 16)
-#define	ATC_DCSIZE_MASK		(0x7 << 20)	/* Destination Chunk Transfer Size */
-#define		ATC_DCSIZE_1		(0x0 << 20)
-#define		ATC_DCSIZE_4		(0x1 << 20)
-#define		ATC_DCSIZE_8		(0x2 << 20)
-#define		ATC_DCSIZE_16		(0x3 << 20)
-#define		ATC_DCSIZE_32		(0x4 << 20)
-#define		ATC_DCSIZE_64		(0x5 << 20)
-#define		ATC_DCSIZE_128		(0x6 << 20)
-#define		ATC_DCSIZE_256		(0x7 << 20)
+/* Chunck Tranfer size definitions are in at_hdmac.h */
 #define	ATC_SRC_WIDTH_MASK	(0x3 << 24)	/* Source Single Transfer Size */
+#define		ATC_SRC_WIDTH(x)	((x) << 24)
 #define		ATC_SRC_WIDTH_BYTE	(0x0 << 24)
 #define		ATC_SRC_WIDTH_HALFWORD	(0x1 << 24)
 #define		ATC_SRC_WIDTH_WORD	(0x2 << 24)
 #define	ATC_DST_WIDTH_MASK	(0x3 << 28)	/* Destination Single Transfer Size */
+#define		ATC_DST_WIDTH(x)	((x) << 28)
 #define		ATC_DST_WIDTH_BYTE	(0x0 << 28)
 #define		ATC_DST_WIDTH_HALFWORD	(0x1 << 28)
 #define		ATC_DST_WIDTH_WORD	(0x2 << 28)
@@ -129,7 +114,8 @@
 #define		ATC_FC_PER2PER		(0x3 << 21)	/* Periph-to-Periph (DMA) */
 #define		ATC_FC_PER2MEM_PER	(0x4 << 21)	/* Periph-to-Mem (Peripheral) */
 #define		ATC_FC_MEM2PER_PER	(0x5 << 21)	/* Mem-to-Periph (Peripheral) */
-#define		ATC_FC_PER2PER_PER	(0x6 << 21)	/* Periph-to-Periph (Src Peripheral) */
+#define		ATC_FC_PER2PER_SRCPER	(0x6 << 21)	/* Periph-to-Periph (Src Peripheral) */
+#define		ATC_FC_PER2PER_DSTPER	(0x7 << 21)	/* Periph-to-Periph (Dst Peripheral) */
 #define	ATC_SRC_ADDR_MODE_MASK	(0x3 << 24)
 #define		ATC_SRC_ADDR_MODE_INCR	(0x0 << 24)	/* Incrementing Mode */
 #define		ATC_SRC_ADDR_MODE_DECR	(0x1 << 24)	/* Decrementing Mode */
@@ -142,27 +128,7 @@
 #define	ATC_AUTO		(0x1 << 31)	/* Auto multiple buffer tx enable */
  
 /* Bitfields in CFG */
-#define	ATC_SRC_PER(h)		(0xFU & (h))	/* Channel src rq associated with periph handshaking ifc h */
-#define	ATC_DST_PER(h)		((0xFU & (h)) <<  4)	/* Channel dst rq associated with periph handshaking ifc h */
-#define	ATC_SRC_REP		(0x1 <<  8)	/* Source Replay Mod */
-#define	ATC_SRC_H2SEL		(0x1 <<  9)	/* Source Handshaking Mod */
-#define		ATC_SRC_H2SEL_SW	(0x0 <<  9)
-#define		ATC_SRC_H2SEL_HW	(0x1 <<  9)
-#define	ATC_DST_REP		(0x1 << 12)	/* Destination Replay Mod */
-#define	ATC_DST_H2SEL		(0x1 << 13)	/* Destination Handshaking Mod */
-#define		ATC_DST_H2SEL_SW	(0x0 << 13)
-#define		ATC_DST_H2SEL_HW	(0x1 << 13)
-#define	ATC_SOD			(0x1 << 16)	/* Stop On Done */
-#define	ATC_LOCK_IF		(0x1 << 20)	/* Interface Lock */
-#define	ATC_LOCK_B		(0x1 << 21)	/* AHB Bus Lock */
-#define	ATC_LOCK_IF_L		(0x1 << 22)	/* Master Interface Arbiter Lock */
-#define		ATC_LOCK_IF_L_CHUNK	(0x0 << 22)
-#define		ATC_LOCK_IF_L_BUFFER	(0x1 << 22)
-#define	ATC_AHB_PROT_MASK	(0x7 << 24)	/* AHB Protection */
-#define	ATC_FIFOCFG_MASK	(0x3 << 28)	/* FIFO Request Configuration */
-#define		ATC_FIFOCFG_LARGESTBURST	(0x0 << 28)
-#define		ATC_FIFOCFG_HALFFIFO		(0x1 << 28)
-#define		ATC_FIFOCFG_ENOUGHSPACE		(0x2 << 28)
+/* are in at_hdmac.h */
  
 /* Bitfields in SPIP */
 #define	ATC_SPIP_HOLE(x)	(0xFFFFU & (x))
  
@@ -316,11 +282,12 @@
 		dma_readl(atdma, CHSR));
  
 	dev_err(chan2dev(&atchan->chan_common),
-		"  channel: s0x%x d0x%x ctrl0x%x:0x%x l0x%x\n",
+		"  channel: s0x%x d0x%x ctrl0x%x:0x%x cfg0x%x l0x%x\n",
 		channel_readl(atchan, SADDR),
 		channel_readl(atchan, DADDR),
 		channel_readl(atchan, CTRLA),
 		channel_readl(atchan, CTRLB),
+		channel_readl(atchan, CFG),
 		channel_readl(atchan, DSCR));
 }
 #else
...	...	@@ -23,5 +23,81 @@
23	23	dma_cap_mask_t cap_mask;
24	24	};
25	25
	26	+/**
	27	+ * enum at_dma_slave_width - DMA slave register access width.
	28	+ * @AT_DMA_SLAVE_WIDTH_8BIT: Do 8-bit slave register accesses
	29	+ * @AT_DMA_SLAVE_WIDTH_16BIT: Do 16-bit slave register accesses
	30	+ * @AT_DMA_SLAVE_WIDTH_32BIT: Do 32-bit slave register accesses
	31	+ */
	32	+enum at_dma_slave_width {
	33	+ AT_DMA_SLAVE_WIDTH_8BIT = 0,
	34	+ AT_DMA_SLAVE_WIDTH_16BIT,
	35	+ AT_DMA_SLAVE_WIDTH_32BIT,
	36	+};
	37	+
	38	+/**
	39	+ * struct at_dma_slave - Controller-specific information about a slave
	40	+ * @dma_dev: required DMA master device
	41	+ * @tx_reg: physical address of data register used for
	42	+ * memory-to-peripheral transfers
	43	+ * @rx_reg: physical address of data register used for
	44	+ * peripheral-to-memory transfers
	45	+ * @reg_width: peripheral register width
	46	+ * @cfg: Platform-specific initializer for the CFG register
	47	+ * @ctrla: Platform-specific initializer for the CTRLA register
	48	+ */
	49	+struct at_dma_slave {
	50	+ struct device *dma_dev;
	51	+ dma_addr_t tx_reg;
	52	+ dma_addr_t rx_reg;
	53	+ enum at_dma_slave_width reg_width;
	54	+ u32 cfg;
	55	+ u32 ctrla;
	56	+};
	57	+
	58	+
	59	+/* Platform-configurable bits in CFG */
	60	+#define ATC_SRC_PER(h) (0xFU & (h)) /* Channel src rq associated with periph handshaking ifc h */
	61	+#define ATC_DST_PER(h) ((0xFU & (h)) << 4) /* Channel dst rq associated with periph handshaking ifc h */
	62	+#define ATC_SRC_REP (0x1 << 8) /* Source Replay Mod */
	63	+#define ATC_SRC_H2SEL (0x1 << 9) /* Source Handshaking Mod */
	64	+#define ATC_SRC_H2SEL_SW (0x0 << 9)
	65	+#define ATC_SRC_H2SEL_HW (0x1 << 9)
	66	+#define ATC_DST_REP (0x1 << 12) /* Destination Replay Mod */
	67	+#define ATC_DST_H2SEL (0x1 << 13) /* Destination Handshaking Mod */
	68	+#define ATC_DST_H2SEL_SW (0x0 << 13)
	69	+#define ATC_DST_H2SEL_HW (0x1 << 13)
	70	+#define ATC_SOD (0x1 << 16) /* Stop On Done */
	71	+#define ATC_LOCK_IF (0x1 << 20) /* Interface Lock */
	72	+#define ATC_LOCK_B (0x1 << 21) /* AHB Bus Lock */
	73	+#define ATC_LOCK_IF_L (0x1 << 22) /* Master Interface Arbiter Lock */
	74	+#define ATC_LOCK_IF_L_CHUNK (0x0 << 22)
	75	+#define ATC_LOCK_IF_L_BUFFER (0x1 << 22)
	76	+#define ATC_AHB_PROT_MASK (0x7 << 24) /* AHB Protection */
	77	+#define ATC_FIFOCFG_MASK (0x3 << 28) /* FIFO Request Configuration */
	78	+#define ATC_FIFOCFG_LARGESTBURST (0x0 << 28)
	79	+#define ATC_FIFOCFG_HALFFIFO (0x1 << 28)
	80	+#define ATC_FIFOCFG_ENOUGHSPACE (0x2 << 28)
	81	+
	82	+/* Platform-configurable bits in CTRLA */
	83	+#define ATC_SCSIZE_MASK (0x7 << 16) /* Source Chunk Transfer Size */
	84	+#define ATC_SCSIZE_1 (0x0 << 16)
	85	+#define ATC_SCSIZE_4 (0x1 << 16)
	86	+#define ATC_SCSIZE_8 (0x2 << 16)
	87	+#define ATC_SCSIZE_16 (0x3 << 16)
	88	+#define ATC_SCSIZE_32 (0x4 << 16)
	89	+#define ATC_SCSIZE_64 (0x5 << 16)
	90	+#define ATC_SCSIZE_128 (0x6 << 16)
	91	+#define ATC_SCSIZE_256 (0x7 << 16)
	92	+#define ATC_DCSIZE_MASK (0x7 << 20) /* Destination Chunk Transfer Size */
	93	+#define ATC_DCSIZE_1 (0x0 << 20)
	94	+#define ATC_DCSIZE_4 (0x1 << 20)
	95	+#define ATC_DCSIZE_8 (0x2 << 20)
	96	+#define ATC_DCSIZE_16 (0x3 << 20)
	97	+#define ATC_DCSIZE_32 (0x4 << 20)
	98	+#define ATC_DCSIZE_64 (0x5 << 20)
	99	+#define ATC_DCSIZE_128 (0x6 << 20)
	100	+#define ATC_DCSIZE_256 (0x7 << 20)
	101	+
26	102	#endif /* AT_HDMAC_H */
...	...	@@ -608,7 +608,188 @@
608	608	return NULL;
609	609	}
610	610
	611	+
611	612	/**
	613	+ * atc_prep_slave_sg - prepare descriptors for a DMA_SLAVE transaction
	614	+ * @chan: DMA channel
	615	+ * @sgl: scatterlist to transfer to/from
	616	+ * @sg_len: number of entries in @scatterlist
	617	+ * @direction: DMA direction
	618	+ * @flags: tx descriptor status flags
	619	+ */
	620	+static struct dma_async_tx_descriptor *
	621	+atc_prep_slave_sg(struct dma_chan chan, struct scatterlist sgl,
	622	+ unsigned int sg_len, enum dma_data_direction direction,
	623	+ unsigned long flags)
	624	+{
	625	+ struct at_dma_chan *atchan = to_at_dma_chan(chan);
	626	+ struct at_dma_slave *atslave = chan->private;
	627	+ struct at_desc *first = NULL;
	628	+ struct at_desc *prev = NULL;
	629	+ u32 ctrla;
	630	+ u32 ctrlb;
	631	+ dma_addr_t reg;
	632	+ unsigned int reg_width;
	633	+ unsigned int mem_width;
	634	+ unsigned int i;
	635	+ struct scatterlist *sg;
	636	+ size_t total_len = 0;
	637	+
	638	+ dev_vdbg(chan2dev(chan), "prep_slave_sg: %s f0x%lx\n",
	639	+ direction == DMA_TO_DEVICE ? "TO DEVICE" : "FROM DEVICE",
	640	+ flags);
	641	+
	642	+ if (unlikely(!atslave \|\| !sg_len)) {
	643	+ dev_dbg(chan2dev(chan), "prep_dma_memcpy: length is zero!\n");
	644	+ return NULL;
	645	+ }
	646	+
	647	+ reg_width = atslave->reg_width;
	648	+
	649	+ sg_len = dma_map_sg(chan2parent(chan), sgl, sg_len, direction);
	650	+
	651	+ ctrla = ATC_DEFAULT_CTRLA \| atslave->ctrla;
	652	+ ctrlb = ATC_DEFAULT_CTRLB \| ATC_IEN;
	653	+
	654	+ switch (direction) {
	655	+ case DMA_TO_DEVICE:
	656	+ ctrla \|= ATC_DST_WIDTH(reg_width);
	657	+ ctrlb \|= ATC_DST_ADDR_MODE_FIXED
	658	+ \| ATC_SRC_ADDR_MODE_INCR
	659	+ \| ATC_FC_MEM2PER;
	660	+ reg = atslave->tx_reg;
	661	+ for_each_sg(sgl, sg, sg_len, i) {
	662	+ struct at_desc *desc;
	663	+ u32 len;
	664	+ u32 mem;
	665	+
	666	+ desc = atc_desc_get(atchan);
	667	+ if (!desc)
	668	+ goto err_desc_get;
	669	+
	670	+ mem = sg_phys(sg);
	671	+ len = sg_dma_len(sg);
	672	+ mem_width = 2;
	673	+ if (unlikely(mem & 3 \|\| len & 3))
	674	+ mem_width = 0;
	675	+
	676	+ desc->lli.saddr = mem;
	677	+ desc->lli.daddr = reg;
	678	+ desc->lli.ctrla = ctrla
	679	+ \| ATC_SRC_WIDTH(mem_width)
	680	+ \| len >> mem_width;
	681	+ desc->lli.ctrlb = ctrlb;
	682	+
	683	+ if (!first) {
	684	+ first = desc;
	685	+ } else {
	686	+ /* inform the HW lli about chaining */
	687	+ prev->lli.dscr = desc->txd.phys;
	688	+ /* insert the link descriptor to the LD ring */
	689	+ list_add_tail(&desc->desc_node,
	690	+ &first->txd.tx_list);
	691	+ }
	692	+ prev = desc;
	693	+ total_len += len;
	694	+ }
	695	+ break;
	696	+ case DMA_FROM_DEVICE:
	697	+ ctrla \|= ATC_SRC_WIDTH(reg_width);
	698	+ ctrlb \|= ATC_DST_ADDR_MODE_INCR
	699	+ \| ATC_SRC_ADDR_MODE_FIXED
	700	+ \| ATC_FC_PER2MEM;
	701	+
	702	+ reg = atslave->rx_reg;
	703	+ for_each_sg(sgl, sg, sg_len, i) {
	704	+ struct at_desc *desc;
	705	+ u32 len;
	706	+ u32 mem;
	707	+
	708	+ desc = atc_desc_get(atchan);
	709	+ if (!desc)
	710	+ goto err_desc_get;
	711	+
	712	+ mem = sg_phys(sg);
	713	+ len = sg_dma_len(sg);
	714	+ mem_width = 2;
	715	+ if (unlikely(mem & 3 \|\| len & 3))
	716	+ mem_width = 0;
	717	+
	718	+ desc->lli.saddr = reg;
	719	+ desc->lli.daddr = mem;
	720	+ desc->lli.ctrla = ctrla
	721	+ \| ATC_DST_WIDTH(mem_width)
	722	+ \| len >> mem_width;
	723	+ desc->lli.ctrlb = ctrlb;
	724	+
	725	+ if (!first) {
	726	+ first = desc;
	727	+ } else {
	728	+ /* inform the HW lli about chaining */
	729	+ prev->lli.dscr = desc->txd.phys;
	730	+ /* insert the link descriptor to the LD ring */
	731	+ list_add_tail(&desc->desc_node,
	732	+ &first->txd.tx_list);
	733	+ }
	734	+ prev = desc;
	735	+ total_len += len;
	736	+ }
	737	+ break;
	738	+ default:
	739	+ return NULL;
	740	+ }
	741	+
	742	+ /* set end-of-link to the last link descriptor of list*/
	743	+ set_desc_eol(prev);
	744	+
	745	+ /* First descriptor of the chain embedds additional information */
	746	+ first->txd.cookie = -EBUSY;
	747	+ first->len = total_len;
	748	+
	749	+ /* last link descriptor of list is responsible of flags */
	750	+ prev->txd.flags = flags; /* client is in control of this ack */
	751	+
	752	+ return &first->txd;
	753	+
	754	+err_desc_get:
	755	+ dev_err(chan2dev(chan), "not enough descriptors available\n");
	756	+ atc_desc_put(atchan, first);
	757	+ return NULL;
	758	+}
	759	+
	760	+static void atc_terminate_all(struct dma_chan *chan)
	761	+{
	762	+ struct at_dma_chan *atchan = to_at_dma_chan(chan);
	763	+ struct at_dma *atdma = to_at_dma(chan->device);
	764	+ struct at_desc desc, _desc;
	765	+ LIST_HEAD(list);
	766	+
	767	+ /*
	768	+ * This is only called when something went wrong elsewhere, so
	769	+ * we don't really care about the data. Just disable the
	770	+ * channel. We still have to poll the channel enable bit due
	771	+ * to AHB/HSB limitations.
	772	+ */
	773	+ spin_lock_bh(&atchan->lock);
	774	+
	775	+ dma_writel(atdma, CHDR, atchan->mask);
	776	+
	777	+ /* confirm that this channel is disabled */
	778	+ while (dma_readl(atdma, CHSR) & atchan->mask)
	779	+ cpu_relax();
	780	+
	781	+ /* active_list entries will end up before queued entries */
	782	+ list_splice_init(&atchan->queue, &list);
	783	+ list_splice_init(&atchan->active_list, &list);
	784	+
	785	+ spin_unlock_bh(&atchan->lock);
	786	+
	787	+ /* Flush all pending and queued descriptors */
	788	+ list_for_each_entry_safe(desc, _desc, &list, desc_node)
	789	+ atc_chain_complete(atchan, desc);
	790	+}
	791	+
	792	+/**
612	793	* atc_is_tx_complete - poll for transaction completion
613	794	* @chan: DMA channel
614	795	* @cookie: transaction identifier to check status of
615	796
...	...	@@ -686,7 +867,9 @@
686	867	struct at_dma_chan *atchan = to_at_dma_chan(chan);
687	868	struct at_dma *atdma = to_at_dma(chan->device);
688	869	struct at_desc *desc;
	870	+ struct at_dma_slave *atslave;
689	871	int i;
	872	+ u32 cfg;
690	873	LIST_HEAD(tmp_list);
691	874
692	875	dev_vdbg(chan2dev(chan), "alloc_chan_resources\n");
...	...	@@ -697,7 +880,23 @@
697	880	return -EIO;
698	881	}
699	882
700		- /* have we already been set up? */
	883	+ cfg = ATC_DEFAULT_CFG;
	884	+
	885	+ atslave = chan->private;
	886	+ if (atslave) {
	887	+ /*
	888	+ * We need controller-specific data to set up slave
	889	+ * transfers.
	890	+ */
	891	+ BUG_ON(!atslave->dma_dev \|\| atslave->dma_dev != atdma->dma_common.dev);
	892	+
	893	+ /* if cfg configuration specified take it instad of default */
	894	+ if (atslave->cfg)
	895	+ cfg = atslave->cfg;
	896	+ }
	897	+
	898	+ /* have we already been set up?
	899	+ * reconfigure channel but no need to reallocate descriptors */
701	900	if (!list_empty(&atchan->free_list))
702	901	return atchan->descs_allocated;
703	902
...	...	@@ -719,7 +918,7 @@
719	918	spin_unlock_bh(&atchan->lock);
720	919
721	920	/* channel parameters */
722		- channel_writel(atchan, CFG, ATC_DEFAULT_CFG);
	921	+ channel_writel(atchan, CFG, cfg);
723	922
724	923	dev_dbg(chan2dev(chan),
725	924	"alloc_chan_resources: allocated %d descriptors\n",
...	...	@@ -887,6 +1086,11 @@
887	1086	/* set prep routines based on capability */
888	1087	if (dma_has_cap(DMA_MEMCPY, atdma->dma_common.cap_mask))
889	1088	atdma->dma_common.device_prep_dma_memcpy = atc_prep_dma_memcpy;
	1089	+
	1090	+ if (dma_has_cap(DMA_SLAVE, atdma->dma_common.cap_mask)) {
	1091	+ atdma->dma_common.device_prep_slave_sg = atc_prep_slave_sg;
	1092	+ atdma->dma_common.device_terminate_all = atc_terminate_all;
	1093	+ }
890	1094
891	1095	dma_writel(atdma, EN, AT_DMA_ENABLE);
892	1096
...	...	@@ -87,29 +87,14 @@
87	87	/* Bitfields in CTRLA */
88	88	#define ATC_BTSIZE_MAX 0xFFFFUL /* Maximum Buffer Transfer Size */
89	89	#define ATC_BTSIZE(x) (ATC_BTSIZE_MAX & (x)) /* Buffer Transfer Size */
90		-#define ATC_SCSIZE_MASK (0x7 << 16) /* Source Chunk Transfer Size */
91		-#define ATC_SCSIZE_1 (0x0 << 16)
92		-#define ATC_SCSIZE_4 (0x1 << 16)
93		-#define ATC_SCSIZE_8 (0x2 << 16)
94		-#define ATC_SCSIZE_16 (0x3 << 16)
95		-#define ATC_SCSIZE_32 (0x4 << 16)
96		-#define ATC_SCSIZE_64 (0x5 << 16)
97		-#define ATC_SCSIZE_128 (0x6 << 16)
98		-#define ATC_SCSIZE_256 (0x7 << 16)
99		-#define ATC_DCSIZE_MASK (0x7 << 20) /* Destination Chunk Transfer Size */
100		-#define ATC_DCSIZE_1 (0x0 << 20)
101		-#define ATC_DCSIZE_4 (0x1 << 20)
102		-#define ATC_DCSIZE_8 (0x2 << 20)
103		-#define ATC_DCSIZE_16 (0x3 << 20)
104		-#define ATC_DCSIZE_32 (0x4 << 20)
105		-#define ATC_DCSIZE_64 (0x5 << 20)
106		-#define ATC_DCSIZE_128 (0x6 << 20)
107		-#define ATC_DCSIZE_256 (0x7 << 20)
	90	+/* Chunck Tranfer size definitions are in at_hdmac.h */
108	91	#define ATC_SRC_WIDTH_MASK (0x3 << 24) /* Source Single Transfer Size */
	92	+#define ATC_SRC_WIDTH(x) ((x) << 24)
109	93	#define ATC_SRC_WIDTH_BYTE (0x0 << 24)
110	94	#define ATC_SRC_WIDTH_HALFWORD (0x1 << 24)
111	95	#define ATC_SRC_WIDTH_WORD (0x2 << 24)
112	96	#define ATC_DST_WIDTH_MASK (0x3 << 28) /* Destination Single Transfer Size */
	97	+#define ATC_DST_WIDTH(x) ((x) << 28)
113	98	#define ATC_DST_WIDTH_BYTE (0x0 << 28)
114	99	#define ATC_DST_WIDTH_HALFWORD (0x1 << 28)
115	100	#define ATC_DST_WIDTH_WORD (0x2 << 28)
...	...	@@ -129,7 +114,8 @@
129	114	#define ATC_FC_PER2PER (0x3 << 21) /* Periph-to-Periph (DMA) */
130	115	#define ATC_FC_PER2MEM_PER (0x4 << 21) /* Periph-to-Mem (Peripheral) */
131	116	#define ATC_FC_MEM2PER_PER (0x5 << 21) /* Mem-to-Periph (Peripheral) */
132		-#define ATC_FC_PER2PER_PER (0x6 << 21) /* Periph-to-Periph (Src Peripheral) */
	117	+#define ATC_FC_PER2PER_SRCPER (0x6 << 21) /* Periph-to-Periph (Src Peripheral) */
	118	+#define ATC_FC_PER2PER_DSTPER (0x7 << 21) /* Periph-to-Periph (Dst Peripheral) */
133	119	#define ATC_SRC_ADDR_MODE_MASK (0x3 << 24)
134	120	#define ATC_SRC_ADDR_MODE_INCR (0x0 << 24) /* Incrementing Mode */
135	121	#define ATC_SRC_ADDR_MODE_DECR (0x1 << 24) /* Decrementing Mode */
...	...	@@ -142,27 +128,7 @@
142	128	#define ATC_AUTO (0x1 << 31) /* Auto multiple buffer tx enable */
143	129
144	130	/* Bitfields in CFG */
145		-#define ATC_SRC_PER(h) (0xFU & (h)) /* Channel src rq associated with periph handshaking ifc h */
146		-#define ATC_DST_PER(h) ((0xFU & (h)) << 4) /* Channel dst rq associated with periph handshaking ifc h */
147		-#define ATC_SRC_REP (0x1 << 8) /* Source Replay Mod */
148		-#define ATC_SRC_H2SEL (0x1 << 9) /* Source Handshaking Mod */
149		-#define ATC_SRC_H2SEL_SW (0x0 << 9)
150		-#define ATC_SRC_H2SEL_HW (0x1 << 9)
151		-#define ATC_DST_REP (0x1 << 12) /* Destination Replay Mod */
152		-#define ATC_DST_H2SEL (0x1 << 13) /* Destination Handshaking Mod */
153		-#define ATC_DST_H2SEL_SW (0x0 << 13)
154		-#define ATC_DST_H2SEL_HW (0x1 << 13)
155		-#define ATC_SOD (0x1 << 16) /* Stop On Done */
156		-#define ATC_LOCK_IF (0x1 << 20) /* Interface Lock */
157		-#define ATC_LOCK_B (0x1 << 21) /* AHB Bus Lock */
158		-#define ATC_LOCK_IF_L (0x1 << 22) /* Master Interface Arbiter Lock */
159		-#define ATC_LOCK_IF_L_CHUNK (0x0 << 22)
160		-#define ATC_LOCK_IF_L_BUFFER (0x1 << 22)
161		-#define ATC_AHB_PROT_MASK (0x7 << 24) /* AHB Protection */
162		-#define ATC_FIFOCFG_MASK (0x3 << 28) /* FIFO Request Configuration */
163		-#define ATC_FIFOCFG_LARGESTBURST (0x0 << 28)
164		-#define ATC_FIFOCFG_HALFFIFO (0x1 << 28)
165		-#define ATC_FIFOCFG_ENOUGHSPACE (0x2 << 28)
	131	+/* are in at_hdmac.h */
166	132
167	133	/* Bitfields in SPIP */
168	134	#define ATC_SPIP_HOLE(x) (0xFFFFU & (x))
169	135
...	...	@@ -316,11 +282,12 @@
316	282	dma_readl(atdma, CHSR));
317	283
318	284	dev_err(chan2dev(&atchan->chan_common),
319		- " channel: s0x%x d0x%x ctrl0x%x:0x%x l0x%x\n",
	285	+ " channel: s0x%x d0x%x ctrl0x%x:0x%x cfg0x%x l0x%x\n",
320	286	channel_readl(atchan, SADDR),
321	287	channel_readl(atchan, DADDR),
322	288	channel_readl(atchan, CTRLA),
323	289	channel_readl(atchan, CTRLB),
	290	+ channel_readl(atchan, CFG),
324	291	channel_readl(atchan, DSCR));
325	292	}
326	293	#else