fsldma: Add DMA_SLAVE support

Use the DMA_SLAVE capability of the DMAEngine API to copy/from a scatterlist into an arbitrary list of hardware address/length pairs. This allows a single DMA transaction to copy data from several different devices into a scatterlist at the same time. This also adds support to enable some controller-specific features such as external start and external pause for a DMA transaction. [dan.j.williams@intel.com: rebased on tx_list movement] Signed-off-by: Ira W. Snyder <iws@ovro.caltech.edu> Acked-by: Li Yang <leoli@freescale.com> Acked-by: Kumar Gala <galak@kernel.crashing.org> Signed-off-by: Dan Williams <dan.j.williams@intel.com>

fsldma: Add DMA_SLAVE support
Use the DMA_SLAVE capability of the DMAEngine API to copy/from a scatterlist into an arbitrary list of hardware address/length pairs. This allows a single DMA transaction to copy data from several different devices into a scatterlist at the same time. This also adds support to enable some controller-specific features such as external start and external pause for a DMA transaction. [dan.j.williams@intel.com: rebased on tx_list movement] Signed-off-by: Ira W. Snyder <iws@ovro.caltech.edu> Acked-by: Li Yang <leoli@freescale.com> Acked-by: Kumar Gala <galak@kernel.crashing.org> Signed-off-by: Dan Williams <dan.j.williams@intel.com>
Ira Snyder · Dan Williams
1 parent e6c7ecb64e
Showing 2 changed files with 363 additions and 0 deletions Side-by-side Diff
arch/powerpc/include/asm/fsldma.h
drivers/dma/fsldma.c
+/*
+ * Freescale MPC83XX / MPC85XX DMA Controller
+ *
+ * Copyright (c) 2009 Ira W. Snyder <iws@ovro.caltech.edu>
+ *
+ * This file is licensed under the terms of the GNU General Public License
+ * version 2. This program is licensed "as is" without any warranty of any
+ * kind, whether express or implied.
+ */
+
+#ifndef __ARCH_POWERPC_ASM_FSLDMA_H__
+#define __ARCH_POWERPC_ASM_FSLDMA_H__
+
+#include <linux/dmaengine.h>
+
+/*
+ * Definitions for the Freescale DMA controller's DMA_SLAVE implemention
+ *
+ * The Freescale DMA_SLAVE implementation was designed to handle many-to-many
+ * transfers. An example usage would be an accelerated copy between two
+ * scatterlists. Another example use would be an accelerated copy from
+ * multiple non-contiguous device buffers into a single scatterlist.
+ *
+ * A DMA_SLAVE transaction is defined by a struct fsl_dma_slave. This
+ * structure contains a list of hardware addresses that should be copied
+ * to/from the scatterlist passed into device_prep_slave_sg(). The structure
+ * also has some fields to enable hardware-specific features.
+ */
+
+/**
+ * struct fsl_dma_hw_addr
+ * @entry: linked list entry
+ * @address: the hardware address
+ * @length: length to transfer
+ *
+ * Holds a single physical hardware address / length pair for use
+ * with the DMAEngine DMA_SLAVE API.
+ */
+struct fsl_dma_hw_addr {
+	struct list_head entry;
+
+	dma_addr_t address;
+	size_t length;
+};
+
+/**
+ * struct fsl_dma_slave
+ * @addresses: a linked list of struct fsl_dma_hw_addr structures
+ * @request_count: value for DMA request count
+ * @src_loop_size: setup and enable constant source-address DMA transfers
+ * @dst_loop_size: setup and enable constant destination address DMA transfers
+ * @external_start: enable externally started DMA transfers
+ * @external_pause: enable externally paused DMA transfers
+ *
+ * Holds a list of address / length pairs for use with the DMAEngine
+ * DMA_SLAVE API implementation for the Freescale DMA controller.
+ */
+struct fsl_dma_slave {
+
+	/* List of hardware address/length pairs */
+	struct list_head addresses;
+
+	/* Support for extra controller features */
+	unsigned int request_count;
+	unsigned int src_loop_size;
+	unsigned int dst_loop_size;
+	bool external_start;
+	bool external_pause;
+};
+
+/**
+ * fsl_dma_slave_append - add an address/length pair to a struct fsl_dma_slave
+ * @slave: the &struct fsl_dma_slave to add to
+ * @address: the hardware address to add
+ * @length: the length of bytes to transfer from @address
+ *
+ * Add a hardware address/length pair to a struct fsl_dma_slave. Returns 0 on
+ * success, -ERRNO otherwise.
+ */
+static inline int fsl_dma_slave_append(struct fsl_dma_slave *slave,
+				       dma_addr_t address, size_t length)
+{
+	struct fsl_dma_hw_addr *addr;
+
+	addr = kzalloc(sizeof(*addr), GFP_ATOMIC);
+	if (!addr)
+		return -ENOMEM;
+
+	INIT_LIST_HEAD(&addr->entry);
+	addr->address = address;
+	addr->length = length;
+
+	list_add_tail(&addr->entry, &slave->addresses);
+	return 0;
+}
+
+/**
+ * fsl_dma_slave_free - free a struct fsl_dma_slave
+ * @slave: the struct fsl_dma_slave to free
+ *
+ * Free a struct fsl_dma_slave and all associated address/length pairs
+ */
+static inline void fsl_dma_slave_free(struct fsl_dma_slave *slave)
+{
+	struct fsl_dma_hw_addr *addr, *tmp;
+
+	if (slave) {
+		list_for_each_entry_safe(addr, tmp, &slave->addresses, entry) {
+			list_del(&addr->entry);
+			kfree(addr);
+		}
+
+		kfree(slave);
+	}
+}
+
+/**
+ * fsl_dma_slave_alloc - allocate a struct fsl_dma_slave
+ * @gfp: the flags to pass to kmalloc when allocating this structure
+ *
+ * Allocate a struct fsl_dma_slave for use by the DMA_SLAVE API. Returns a new
+ * struct fsl_dma_slave on success, or NULL on failure.
+ */
+static inline struct fsl_dma_slave *fsl_dma_slave_alloc(gfp_t gfp)
+{
+	struct fsl_dma_slave *slave;
+
+	slave = kzalloc(sizeof(*slave), gfp);
+	if (!slave)
+		return NULL;
+
+	INIT_LIST_HEAD(&slave->addresses);
+	return slave;
+}
+
+#endif /* __ARCH_POWERPC_ASM_FSLDMA_H__ */
@@ -34,6 +34,7 @@
 #include <linux/dmapool.h>
 #include <linux/of_platform.h>
  
+#include <asm/fsldma.h>
 #include "fsldma.h"
  
 static void dma_init(struct fsl_dma_chan *fsl_chan)
@@ -552,6 +553,229 @@
 }
  
 /**
+ * fsl_dma_prep_slave_sg - prepare descriptors for a DMA_SLAVE transaction
+ * @chan: DMA channel
+ * @sgl: scatterlist to transfer to/from
+ * @sg_len: number of entries in @scatterlist
+ * @direction: DMA direction
+ * @flags: DMAEngine flags
+ *
+ * Prepare a set of descriptors for a DMA_SLAVE transaction. Following the
+ * DMA_SLAVE API, this gets the device-specific information from the
+ * chan->private variable.
+ */
+static struct dma_async_tx_descriptor *fsl_dma_prep_slave_sg(
+	struct dma_chan *chan, struct scatterlist *sgl, unsigned int sg_len,
+	enum dma_data_direction direction, unsigned long flags)
+{
+	struct fsl_dma_chan *fsl_chan;
+	struct fsl_desc_sw *first = NULL, *prev = NULL, *new = NULL;
+	struct fsl_dma_slave *slave;
+	struct list_head *tx_list;
+	size_t copy;
+
+	int i;
+	struct scatterlist *sg;
+	size_t sg_used;
+	size_t hw_used;
+	struct fsl_dma_hw_addr *hw;
+	dma_addr_t dma_dst, dma_src;
+
+	if (!chan)
+		return NULL;
+
+	if (!chan->private)
+		return NULL;
+
+	fsl_chan = to_fsl_chan(chan);
+	slave = chan->private;
+
+	if (list_empty(&slave->addresses))
+		return NULL;
+
+	hw = list_first_entry(&slave->addresses, struct fsl_dma_hw_addr, entry);
+	hw_used = 0;
+
+	/*
+	 * Build the hardware transaction to copy from the scatterlist to
+	 * the hardware, or from the hardware to the scatterlist
+	 *
+	 * If you are copying from the hardware to the scatterlist and it
+	 * takes two hardware entries to fill an entire page, then both
+	 * hardware entries will be coalesced into the same page
+	 *
+	 * If you are copying from the scatterlist to the hardware and a
+	 * single page can fill two hardware entries, then the data will
+	 * be read out of the page into the first hardware entry, and so on
+	 */
+	for_each_sg(sgl, sg, sg_len, i) {
+		sg_used = 0;
+
+		/* Loop until the entire scatterlist entry is used */
+		while (sg_used < sg_dma_len(sg)) {
+
+			/*
+			 * If we've used up the current hardware address/length
+			 * pair, we need to load a new one
+			 *
+			 * This is done in a while loop so that descriptors with
+			 * length == 0 will be skipped
+			 */
+			while (hw_used >= hw->length) {
+
+				/*
+				 * If the current hardware entry is the last
+				 * entry in the list, we're finished
+				 */
+				if (list_is_last(&hw->entry, &slave->addresses))
+					goto finished;
+
+				/* Get the next hardware address/length pair */
+				hw = list_entry(hw->entry.next,
+						struct fsl_dma_hw_addr, entry);
+				hw_used = 0;
+			}
+
+			/* Allocate the link descriptor from DMA pool */
+			new = fsl_dma_alloc_descriptor(fsl_chan);
+			if (!new) {
+				dev_err(fsl_chan->dev, "No free memory for "
+						       "link descriptor\n");
+				goto fail;
+			}
+#ifdef FSL_DMA_LD_DEBUG
+			dev_dbg(fsl_chan->dev, "new link desc alloc %p\n", new);
+#endif
+
+			/*
+			 * Calculate the maximum number of bytes to transfer,
+			 * making sure it is less than the DMA controller limit
+			 */
+			copy = min_t(size_t, sg_dma_len(sg) - sg_used,
+					     hw->length - hw_used);
+			copy = min_t(size_t, copy, FSL_DMA_BCR_MAX_CNT);
+
+			/*
+			 * DMA_FROM_DEVICE
+			 * from the hardware to the scatterlist
+			 *
+			 * DMA_TO_DEVICE
+			 * from the scatterlist to the hardware
+			 */
+			if (direction == DMA_FROM_DEVICE) {
+				dma_src = hw->address + hw_used;
+				dma_dst = sg_dma_address(sg) + sg_used;
+			} else {
+				dma_src = sg_dma_address(sg) + sg_used;
+				dma_dst = hw->address + hw_used;
+			}
+
+			/* Fill in the descriptor */
+			set_desc_cnt(fsl_chan, &new->hw, copy);
+			set_desc_src(fsl_chan, &new->hw, dma_src);
+			set_desc_dest(fsl_chan, &new->hw, dma_dst);
+
+			/*
+			 * If this is not the first descriptor, chain the
+			 * current descriptor after the previous descriptor
+			 */
+			if (!first) {
+				first = new;
+			} else {
+				set_desc_next(fsl_chan, &prev->hw,
+					      new->async_tx.phys);
+			}
+
+			new->async_tx.cookie = 0;
+			async_tx_ack(&new->async_tx);
+
+			prev = new;
+			sg_used += copy;
+			hw_used += copy;
+
+			/* Insert the link descriptor into the LD ring */
+			list_add_tail(&new->node, &first->tx_list);
+		}
+	}
+
+finished:
+
+	/* All of the hardware address/length pairs had length == 0 */
+	if (!first || !new)
+		return NULL;
+
+	new->async_tx.flags = flags;
+	new->async_tx.cookie = -EBUSY;
+
+	/* Set End-of-link to the last link descriptor of new list */
+	set_ld_eol(fsl_chan, new);
+
+	/* Enable extra controller features */
+	if (fsl_chan->set_src_loop_size)
+		fsl_chan->set_src_loop_size(fsl_chan, slave->src_loop_size);
+
+	if (fsl_chan->set_dest_loop_size)
+		fsl_chan->set_dest_loop_size(fsl_chan, slave->dst_loop_size);
+
+	if (fsl_chan->toggle_ext_start)
+		fsl_chan->toggle_ext_start(fsl_chan, slave->external_start);
+
+	if (fsl_chan->toggle_ext_pause)
+		fsl_chan->toggle_ext_pause(fsl_chan, slave->external_pause);
+
+	if (fsl_chan->set_request_count)
+		fsl_chan->set_request_count(fsl_chan, slave->request_count);
+
+	return &first->async_tx;
+
+fail:
+	/* If first was not set, then we failed to allocate the very first
+	 * descriptor, and we're done */
+	if (!first)
+		return NULL;
+
+	/*
+	 * First is set, so all of the descriptors we allocated have been added
+	 * to first->tx_list, INCLUDING "first" itself. Therefore we
+	 * must traverse the list backwards freeing each descriptor in turn
+	 *
+	 * We're re-using variables for the loop, oh well
+	 */
+	tx_list = &first->tx_list;
+	list_for_each_entry_safe_reverse(new, prev, tx_list, node) {
+		list_del_init(&new->node);
+		dma_pool_free(fsl_chan->desc_pool, new, new->async_tx.phys);
+	}
+
+	return NULL;
+}
+
+static void fsl_dma_device_terminate_all(struct dma_chan *chan)
+{
+	struct fsl_dma_chan *fsl_chan;
+	struct fsl_desc_sw *desc, *tmp;
+	unsigned long flags;
+
+	if (!chan)
+		return;
+
+	fsl_chan = to_fsl_chan(chan);
+
+	/* Halt the DMA engine */
+	dma_halt(fsl_chan);
+
+	spin_lock_irqsave(&fsl_chan->desc_lock, flags);
+
+	/* Remove and free all of the descriptors in the LD queue */
+	list_for_each_entry_safe(desc, tmp, &fsl_chan->ld_queue, node) {
+		list_del(&desc->node);
+		dma_pool_free(fsl_chan->desc_pool, desc, desc->async_tx.phys);
+	}
+
+	spin_unlock_irqrestore(&fsl_chan->desc_lock, flags);
+}
+
+/**
  * fsl_dma_update_completed_cookie - Update the completed cookie.
  * @fsl_chan : Freescale DMA channel
  */
  
@@ -977,12 +1201,15 @@
  
 	dma_cap_set(DMA_MEMCPY, fdev->common.cap_mask);
 	dma_cap_set(DMA_INTERRUPT, fdev->common.cap_mask);
+	dma_cap_set(DMA_SLAVE, fdev->common.cap_mask);
 	fdev->common.device_alloc_chan_resources = fsl_dma_alloc_chan_resources;
 	fdev->common.device_free_chan_resources = fsl_dma_free_chan_resources;
 	fdev->common.device_prep_dma_interrupt = fsl_dma_prep_interrupt;
 	fdev->common.device_prep_dma_memcpy = fsl_dma_prep_memcpy;
 	fdev->common.device_is_tx_complete = fsl_dma_is_complete;
 	fdev->common.device_issue_pending = fsl_dma_memcpy_issue_pending;
+	fdev->common.device_prep_slave_sg = fsl_dma_prep_slave_sg;
+	fdev->common.device_terminate_all = fsl_dma_device_terminate_all;
 	fdev->common.dev = &dev->dev;
  
 	fdev->irq = irq_of_parse_and_map(dev->node, 0);
	1	+/*
	2	+ * Freescale MPC83XX / MPC85XX DMA Controller
	3	+ *
	4	+ * Copyright (c) 2009 Ira W. Snyder <iws@ovro.caltech.edu>
	5	+ *
	6	+ * This file is licensed under the terms of the GNU General Public License
	7	+ * version 2. This program is licensed "as is" without any warranty of any
	8	+ * kind, whether express or implied.
	9	+ */
	10	+
	11	+#ifndef __ARCH_POWERPC_ASM_FSLDMA_H__
	12	+#define __ARCH_POWERPC_ASM_FSLDMA_H__
	13	+
	14	+#include <linux/dmaengine.h>
	15	+
	16	+/*
	17	+ * Definitions for the Freescale DMA controller's DMA_SLAVE implemention
	18	+ *
	19	+ * The Freescale DMA_SLAVE implementation was designed to handle many-to-many
	20	+ * transfers. An example usage would be an accelerated copy between two
	21	+ * scatterlists. Another example use would be an accelerated copy from
	22	+ * multiple non-contiguous device buffers into a single scatterlist.
	23	+ *
	24	+ * A DMA_SLAVE transaction is defined by a struct fsl_dma_slave. This
	25	+ * structure contains a list of hardware addresses that should be copied
	26	+ * to/from the scatterlist passed into device_prep_slave_sg(). The structure
	27	+ * also has some fields to enable hardware-specific features.
	28	+ */
	29	+
	30	+/**
	31	+ * struct fsl_dma_hw_addr
	32	+ * @entry: linked list entry
	33	+ * @address: the hardware address
	34	+ * @length: length to transfer
	35	+ *
	36	+ * Holds a single physical hardware address / length pair for use
	37	+ * with the DMAEngine DMA_SLAVE API.
	38	+ */
	39	+struct fsl_dma_hw_addr {
	40	+ struct list_head entry;
	41	+
	42	+ dma_addr_t address;
	43	+ size_t length;
	44	+};
	45	+
	46	+/**
	47	+ * struct fsl_dma_slave
	48	+ * @addresses: a linked list of struct fsl_dma_hw_addr structures
	49	+ * @request_count: value for DMA request count
	50	+ * @src_loop_size: setup and enable constant source-address DMA transfers
	51	+ * @dst_loop_size: setup and enable constant destination address DMA transfers
	52	+ * @external_start: enable externally started DMA transfers
	53	+ * @external_pause: enable externally paused DMA transfers
	54	+ *
	55	+ * Holds a list of address / length pairs for use with the DMAEngine
	56	+ * DMA_SLAVE API implementation for the Freescale DMA controller.
	57	+ */
	58	+struct fsl_dma_slave {
	59	+
	60	+ /* List of hardware address/length pairs */
	61	+ struct list_head addresses;
	62	+
	63	+ /* Support for extra controller features */
	64	+ unsigned int request_count;
	65	+ unsigned int src_loop_size;
	66	+ unsigned int dst_loop_size;
	67	+ bool external_start;
	68	+ bool external_pause;
	69	+};
	70	+
	71	+/**
	72	+ * fsl_dma_slave_append - add an address/length pair to a struct fsl_dma_slave
	73	+ * @slave: the &struct fsl_dma_slave to add to
	74	+ * @address: the hardware address to add
	75	+ * @length: the length of bytes to transfer from @address
	76	+ *
	77	+ * Add a hardware address/length pair to a struct fsl_dma_slave. Returns 0 on
	78	+ * success, -ERRNO otherwise.
	79	+ */
	80	+static inline int fsl_dma_slave_append(struct fsl_dma_slave *slave,
	81	+ dma_addr_t address, size_t length)
	82	+{
	83	+ struct fsl_dma_hw_addr *addr;
	84	+
	85	+ addr = kzalloc(sizeof(*addr), GFP_ATOMIC);
	86	+ if (!addr)
	87	+ return -ENOMEM;
	88	+
	89	+ INIT_LIST_HEAD(&addr->entry);
	90	+ addr->address = address;
	91	+ addr->length = length;
	92	+
	93	+ list_add_tail(&addr->entry, &slave->addresses);
	94	+ return 0;
	95	+}
	96	+
	97	+/**
	98	+ * fsl_dma_slave_free - free a struct fsl_dma_slave
	99	+ * @slave: the struct fsl_dma_slave to free
	100	+ *
	101	+ * Free a struct fsl_dma_slave and all associated address/length pairs
	102	+ */
	103	+static inline void fsl_dma_slave_free(struct fsl_dma_slave *slave)
	104	+{
	105	+ struct fsl_dma_hw_addr addr, tmp;
	106	+
	107	+ if (slave) {
	108	+ list_for_each_entry_safe(addr, tmp, &slave->addresses, entry) {
	109	+ list_del(&addr->entry);
	110	+ kfree(addr);
	111	+ }
	112	+
	113	+ kfree(slave);
	114	+ }
	115	+}
	116	+
	117	+/**
	118	+ * fsl_dma_slave_alloc - allocate a struct fsl_dma_slave
	119	+ * @gfp: the flags to pass to kmalloc when allocating this structure
	120	+ *
	121	+ * Allocate a struct fsl_dma_slave for use by the DMA_SLAVE API. Returns a new
	122	+ * struct fsl_dma_slave on success, or NULL on failure.
	123	+ */
	124	+static inline struct fsl_dma_slave *fsl_dma_slave_alloc(gfp_t gfp)
	125	+{
	126	+ struct fsl_dma_slave *slave;
	127	+
	128	+ slave = kzalloc(sizeof(*slave), gfp);
	129	+ if (!slave)
	130	+ return NULL;
	131	+
	132	+ INIT_LIST_HEAD(&slave->addresses);
	133	+ return slave;
	134	+}
	135	+
	136	+#endif /* __ARCH_POWERPC_ASM_FSLDMA_H__ */
...	...	@@ -34,6 +34,7 @@
34	34	#include <linux/dmapool.h>
35	35	#include <linux/of_platform.h>
36	36
	37	+#include <asm/fsldma.h>
37	38	#include "fsldma.h"
38	39
39	40	static void dma_init(struct fsl_dma_chan *fsl_chan)
...	...	@@ -552,6 +553,229 @@
552	553	}
553	554
554	555	/**
	556	+ * fsl_dma_prep_slave_sg - prepare descriptors for a DMA_SLAVE transaction
	557	+ * @chan: DMA channel
	558	+ * @sgl: scatterlist to transfer to/from
	559	+ * @sg_len: number of entries in @scatterlist
	560	+ * @direction: DMA direction
	561	+ * @flags: DMAEngine flags
	562	+ *
	563	+ * Prepare a set of descriptors for a DMA_SLAVE transaction. Following the
	564	+ * DMA_SLAVE API, this gets the device-specific information from the
	565	+ * chan->private variable.
	566	+ */
	567	+static struct dma_async_tx_descriptor *fsl_dma_prep_slave_sg(
	568	+ struct dma_chan chan, struct scatterlist sgl, unsigned int sg_len,
	569	+ enum dma_data_direction direction, unsigned long flags)
	570	+{
	571	+ struct fsl_dma_chan *fsl_chan;
	572	+ struct fsl_desc_sw first = NULL, prev = NULL, *new = NULL;
	573	+ struct fsl_dma_slave *slave;
	574	+ struct list_head *tx_list;
	575	+ size_t copy;
	576	+
	577	+ int i;
	578	+ struct scatterlist *sg;
	579	+ size_t sg_used;
	580	+ size_t hw_used;
	581	+ struct fsl_dma_hw_addr *hw;
	582	+ dma_addr_t dma_dst, dma_src;
	583	+
	584	+ if (!chan)
	585	+ return NULL;
	586	+
	587	+ if (!chan->private)
	588	+ return NULL;
	589	+
	590	+ fsl_chan = to_fsl_chan(chan);
	591	+ slave = chan->private;
	592	+
	593	+ if (list_empty(&slave->addresses))
	594	+ return NULL;
	595	+
	596	+ hw = list_first_entry(&slave->addresses, struct fsl_dma_hw_addr, entry);
	597	+ hw_used = 0;
	598	+
	599	+ /*
	600	+ * Build the hardware transaction to copy from the scatterlist to
	601	+ * the hardware, or from the hardware to the scatterlist
	602	+ *
	603	+ * If you are copying from the hardware to the scatterlist and it
	604	+ * takes two hardware entries to fill an entire page, then both
	605	+ * hardware entries will be coalesced into the same page
	606	+ *
	607	+ * If you are copying from the scatterlist to the hardware and a
	608	+ * single page can fill two hardware entries, then the data will
	609	+ * be read out of the page into the first hardware entry, and so on
	610	+ */
	611	+ for_each_sg(sgl, sg, sg_len, i) {
	612	+ sg_used = 0;
	613	+
	614	+ /* Loop until the entire scatterlist entry is used */
	615	+ while (sg_used < sg_dma_len(sg)) {
	616	+
	617	+ /*
	618	+ * If we've used up the current hardware address/length
	619	+ * pair, we need to load a new one
	620	+ *
	621	+ * This is done in a while loop so that descriptors with
	622	+ * length == 0 will be skipped
	623	+ */
	624	+ while (hw_used >= hw->length) {
	625	+
	626	+ /*
	627	+ * If the current hardware entry is the last
	628	+ * entry in the list, we're finished
	629	+ */
	630	+ if (list_is_last(&hw->entry, &slave->addresses))
	631	+ goto finished;
	632	+
	633	+ /* Get the next hardware address/length pair */
	634	+ hw = list_entry(hw->entry.next,
	635	+ struct fsl_dma_hw_addr, entry);
	636	+ hw_used = 0;
	637	+ }
	638	+
	639	+ /* Allocate the link descriptor from DMA pool */
	640	+ new = fsl_dma_alloc_descriptor(fsl_chan);
	641	+ if (!new) {
	642	+ dev_err(fsl_chan->dev, "No free memory for "
	643	+ "link descriptor\n");
	644	+ goto fail;
	645	+ }
	646	+#ifdef FSL_DMA_LD_DEBUG
	647	+ dev_dbg(fsl_chan->dev, "new link desc alloc %p\n", new);
	648	+#endif
	649	+
	650	+ /*
	651	+ * Calculate the maximum number of bytes to transfer,
	652	+ * making sure it is less than the DMA controller limit
	653	+ */
	654	+ copy = min_t(size_t, sg_dma_len(sg) - sg_used,
	655	+ hw->length - hw_used);
	656	+ copy = min_t(size_t, copy, FSL_DMA_BCR_MAX_CNT);
	657	+
	658	+ /*
	659	+ * DMA_FROM_DEVICE
	660	+ * from the hardware to the scatterlist
	661	+ *
	662	+ * DMA_TO_DEVICE
	663	+ * from the scatterlist to the hardware
	664	+ */
	665	+ if (direction == DMA_FROM_DEVICE) {
	666	+ dma_src = hw->address + hw_used;
	667	+ dma_dst = sg_dma_address(sg) + sg_used;
	668	+ } else {
	669	+ dma_src = sg_dma_address(sg) + sg_used;
	670	+ dma_dst = hw->address + hw_used;
	671	+ }
	672	+
	673	+ /* Fill in the descriptor */
	674	+ set_desc_cnt(fsl_chan, &new->hw, copy);
	675	+ set_desc_src(fsl_chan, &new->hw, dma_src);
	676	+ set_desc_dest(fsl_chan, &new->hw, dma_dst);
	677	+
	678	+ /*
	679	+ * If this is not the first descriptor, chain the
	680	+ * current descriptor after the previous descriptor
	681	+ */
	682	+ if (!first) {
	683	+ first = new;
	684	+ } else {
	685	+ set_desc_next(fsl_chan, &prev->hw,
	686	+ new->async_tx.phys);
	687	+ }
	688	+
	689	+ new->async_tx.cookie = 0;
	690	+ async_tx_ack(&new->async_tx);
	691	+
	692	+ prev = new;
	693	+ sg_used += copy;
	694	+ hw_used += copy;
	695	+
	696	+ /* Insert the link descriptor into the LD ring */
	697	+ list_add_tail(&new->node, &first->tx_list);
	698	+ }
	699	+ }
	700	+
	701	+finished:
	702	+
	703	+ /* All of the hardware address/length pairs had length == 0 */
	704	+ if (!first \|\| !new)
	705	+ return NULL;
	706	+
	707	+ new->async_tx.flags = flags;
	708	+ new->async_tx.cookie = -EBUSY;
	709	+
	710	+ /* Set End-of-link to the last link descriptor of new list */
	711	+ set_ld_eol(fsl_chan, new);
	712	+
	713	+ /* Enable extra controller features */
	714	+ if (fsl_chan->set_src_loop_size)
	715	+ fsl_chan->set_src_loop_size(fsl_chan, slave->src_loop_size);
	716	+
	717	+ if (fsl_chan->set_dest_loop_size)
	718	+ fsl_chan->set_dest_loop_size(fsl_chan, slave->dst_loop_size);
	719	+
	720	+ if (fsl_chan->toggle_ext_start)
	721	+ fsl_chan->toggle_ext_start(fsl_chan, slave->external_start);
	722	+
	723	+ if (fsl_chan->toggle_ext_pause)
	724	+ fsl_chan->toggle_ext_pause(fsl_chan, slave->external_pause);
	725	+
	726	+ if (fsl_chan->set_request_count)
	727	+ fsl_chan->set_request_count(fsl_chan, slave->request_count);
	728	+
	729	+ return &first->async_tx;
	730	+
	731	+fail:
	732	+ /* If first was not set, then we failed to allocate the very first
	733	+ * descriptor, and we're done */
	734	+ if (!first)
	735	+ return NULL;
	736	+
	737	+ /*
	738	+ * First is set, so all of the descriptors we allocated have been added
	739	+ * to first->tx_list, INCLUDING "first" itself. Therefore we
	740	+ * must traverse the list backwards freeing each descriptor in turn
	741	+ *
	742	+ * We're re-using variables for the loop, oh well
	743	+ */
	744	+ tx_list = &first->tx_list;
	745	+ list_for_each_entry_safe_reverse(new, prev, tx_list, node) {
	746	+ list_del_init(&new->node);
	747	+ dma_pool_free(fsl_chan->desc_pool, new, new->async_tx.phys);
	748	+ }
	749	+
	750	+ return NULL;
	751	+}
	752	+
	753	+static void fsl_dma_device_terminate_all(struct dma_chan *chan)
	754	+{
	755	+ struct fsl_dma_chan *fsl_chan;
	756	+ struct fsl_desc_sw desc, tmp;
	757	+ unsigned long flags;
	758	+
	759	+ if (!chan)
	760	+ return;
	761	+
	762	+ fsl_chan = to_fsl_chan(chan);
	763	+
	764	+ /* Halt the DMA engine */
	765	+ dma_halt(fsl_chan);
	766	+
	767	+ spin_lock_irqsave(&fsl_chan->desc_lock, flags);
	768	+
	769	+ /* Remove and free all of the descriptors in the LD queue */
	770	+ list_for_each_entry_safe(desc, tmp, &fsl_chan->ld_queue, node) {
	771	+ list_del(&desc->node);
	772	+ dma_pool_free(fsl_chan->desc_pool, desc, desc->async_tx.phys);
	773	+ }
	774	+
	775	+ spin_unlock_irqrestore(&fsl_chan->desc_lock, flags);
	776	+}
	777	+
	778	+/**
555	779	* fsl_dma_update_completed_cookie - Update the completed cookie.
556	780	* @fsl_chan : Freescale DMA channel
557	781	*/
558	782
...	...	@@ -977,12 +1201,15 @@
977	1201
978	1202	dma_cap_set(DMA_MEMCPY, fdev->common.cap_mask);
979	1203	dma_cap_set(DMA_INTERRUPT, fdev->common.cap_mask);
	1204	+ dma_cap_set(DMA_SLAVE, fdev->common.cap_mask);
980	1205	fdev->common.device_alloc_chan_resources = fsl_dma_alloc_chan_resources;
981	1206	fdev->common.device_free_chan_resources = fsl_dma_free_chan_resources;
982	1207	fdev->common.device_prep_dma_interrupt = fsl_dma_prep_interrupt;
983	1208	fdev->common.device_prep_dma_memcpy = fsl_dma_prep_memcpy;
984	1209	fdev->common.device_is_tx_complete = fsl_dma_is_complete;
985	1210	fdev->common.device_issue_pending = fsl_dma_memcpy_issue_pending;
	1211	+ fdev->common.device_prep_slave_sg = fsl_dma_prep_slave_sg;
	1212	+ fdev->common.device_terminate_all = fsl_dma_device_terminate_all;
986	1213	fdev->common.dev = &dev->dev;
987	1214
988	1215	fdev->irq = irq_of_parse_and_map(dev->node, 0);