dmaengine: refactor dmaengine around dma_async_tx_descriptor

The current dmaengine interface defines mutliple routines per operation, i.e. dma_async_memcpy_buf_to_buf, dma_async_memcpy_buf_to_page etc. Adding more operation types (xor, crc, etc) to this model would result in an unmanageable number of method permutations. Are we really going to add a set of hooks for each DMA engine whizbang feature? - Jeff Garzik The descriptor creation process is refactored using the new common dma_async_tx_descriptor structure. Instead of per driver do_<operation>_<dest>_to_<src> methods, drivers integrate dma_async_tx_descriptor into their private software descriptor and then define a 'prep' routine per operation. The prep routine allocates a descriptor and ensures that the tx_set_src, tx_set_dest, tx_submit routines are valid. Descriptor creation and submission becomes: struct dma_device *dev; struct dma_chan *chan; struct dma_async_tx_descriptor *tx; tx = dev->device_prep_dma_<operation>(chan, len, int_flag) tx->tx_set_src(dma_addr_t, tx, index /* for multi-source ops */) tx->tx_set_dest(dma_addr_t, tx, index) tx->tx_submit(tx) In addition to the refactoring, dma_async_tx_descriptor also lays the groundwork for definining cross-channel-operation dependencies, and a callback facility for asynchronous notification of operation completion. Changelog: * drop dma mapping methods, suggested by Chris Leech * fix ioat_dma_dependency_added, also caught by Andrew Morton * fix dma_sync_wait, change from Andrew Morton * uninline large functions, change from Andrew Morton * add tx->callback = NULL to dmaengine calls to interoperate with async_tx calls * hookup ioat_tx_submit * convert channel capabilities to a 'cpumask_t like' bitmap * removed DMA_TX_ARRAY_INIT, no longer needed * checkpatch.pl fixes * make set_src, set_dest, and tx_submit descriptor specific methods * fixup git-ioat merge * move group_list and phys to dma_async_tx_descriptor Cc: Jeff Garzik <jeff@garzik.org> Cc: Chris Leech <christopher.leech@intel.com> Signed-off-by: Shannon Nelson <shannon.nelson@intel.com> Signed-off-by: Dan Williams <dan.j.williams@intel.com> Acked-by: David S. Miller <davem@davemloft.net>

dmaengine: refactor dmaengine around dma_async_tx_descriptor
The current dmaengine interface defines mutliple routines per operation, i.e. dma_async_memcpy_buf_to_buf, dma_async_memcpy_buf_to_page etc. Adding more operation types (xor, crc, etc) to this model would result in an unmanageable number of method permutations. Are we really going to add a set of hooks for each DMA engine whizbang feature? - Jeff Garzik The descriptor creation process is refactored using the new common dma_async_tx_descriptor structure. Instead of per driver do_<operation>_<dest>_to_<src> methods, drivers integrate dma_async_tx_descriptor into their private software descriptor and then define a 'prep' routine per operation. The prep routine allocates a descriptor and ensures that the tx_set_src, tx_set_dest, tx_submit routines are valid. Descriptor creation and submission becomes: struct dma_device *dev; struct dma_chan *chan; struct dma_async_tx_descriptor *tx; tx = dev->device_prep_dma_<operation>(chan, len, int_flag) tx->tx_set_src(dma_addr_t, tx, index /* for multi-source ops */) tx->tx_set_dest(dma_addr_t, tx, index) tx->tx_submit(tx) In addition to the refactoring, dma_async_tx_descriptor also lays the groundwork for definining cross-channel-operation dependencies, and a callback facility for asynchronous notification of operation completion. Changelog: * drop dma mapping methods, suggested by Chris Leech * fix ioat_dma_dependency_added, also caught by Andrew Morton * fix dma_sync_wait, change from Andrew Morton * uninline large functions, change from Andrew Morton * add tx->callback = NULL to dmaengine calls to interoperate with async_tx calls * hookup ioat_tx_submit * convert channel capabilities to a 'cpumask_t like' bitmap * removed DMA_TX_ARRAY_INIT, no longer needed * checkpatch.pl fixes * make set_src, set_dest, and tx_submit descriptor specific methods * fixup git-ioat merge * move group_list and phys to dma_async_tx_descriptor Cc: Jeff Garzik <jeff@garzik.org> Cc: Chris Leech <christopher.leech@intel.com> Signed-off-by: Shannon Nelson <shannon.nelson@intel.com> Signed-off-by: Dan Williams <dan.j.williams@intel.com> Acked-by: David S. Miller <davem@davemloft.net>
Dan Williams
1 parent 428ed6024f
Showing 4 changed files with 474 additions and 253 deletions Side-by-side Diff
drivers/dma/dmaengine.c
drivers/dma/ioatdma.c
drivers/dma/ioatdma.h
include/linux/dmaengine.h
@@ -59,6 +59,7 @@
  
 #include <linux/init.h>
 #include <linux/module.h>
+#include <linux/mm.h>
 #include <linux/device.h>
 #include <linux/dmaengine.h>
 #include <linux/hardirq.h>
@@ -66,6 +67,7 @@
 #include <linux/percpu.h>
 #include <linux/rcupdate.h>
 #include <linux/mutex.h>
+#include <linux/jiffies.h>
  
 static DEFINE_MUTEX(dma_list_mutex);
 static LIST_HEAD(dma_device_list);
@@ -165,6 +167,24 @@
 	return NULL;
 }
  
+enum dma_status dma_sync_wait(struct dma_chan *chan, dma_cookie_t cookie)
+{
+	enum dma_status status;
+	unsigned long dma_sync_wait_timeout = jiffies + msecs_to_jiffies(5000);
+
+	dma_async_issue_pending(chan);
+	do {
+		status = dma_async_is_tx_complete(chan, cookie, NULL, NULL);
+		if (time_after_eq(jiffies, dma_sync_wait_timeout)) {
+			printk(KERN_ERR "dma_sync_wait_timeout!\n");
+			return DMA_ERROR;
+		}
+	} while (status == DMA_IN_PROGRESS);
+
+	return status;
+}
+EXPORT_SYMBOL(dma_sync_wait);
+
 /**
  * dma_chan_cleanup - release a DMA channel's resources
  * @kref: kernel reference structure that contains the DMA channel device
@@ -322,6 +342,25 @@
 	if (!device)
 		return -ENODEV;
  
+	/* validate device routines */
+	BUG_ON(dma_has_cap(DMA_MEMCPY, device->cap_mask) &&
+		!device->device_prep_dma_memcpy);
+	BUG_ON(dma_has_cap(DMA_XOR, device->cap_mask) &&
+		!device->device_prep_dma_xor);
+	BUG_ON(dma_has_cap(DMA_ZERO_SUM, device->cap_mask) &&
+		!device->device_prep_dma_zero_sum);
+	BUG_ON(dma_has_cap(DMA_MEMSET, device->cap_mask) &&
+		!device->device_prep_dma_memset);
+	BUG_ON(dma_has_cap(DMA_ZERO_SUM, device->cap_mask) &&
+		!device->device_prep_dma_interrupt);
+
+	BUG_ON(!device->device_alloc_chan_resources);
+	BUG_ON(!device->device_free_chan_resources);
+	BUG_ON(!device->device_dependency_added);
+	BUG_ON(!device->device_is_tx_complete);
+	BUG_ON(!device->device_issue_pending);
+	BUG_ON(!device->dev);
+
 	init_completion(&device->done);
 	kref_init(&device->refcount);
 	device->dev_id = id++;
@@ -414,6 +453,149 @@
 	wait_for_completion(&device->done);
 }
 EXPORT_SYMBOL(dma_async_device_unregister);
+
+/**
+ * dma_async_memcpy_buf_to_buf - offloaded copy between virtual addresses
+ * @chan: DMA channel to offload copy to
+ * @dest: destination address (virtual)
+ * @src: source address (virtual)
+ * @len: length
+ *
+ * Both @dest and @src must be mappable to a bus address according to the
+ * DMA mapping API rules for streaming mappings.
+ * Both @dest and @src must stay memory resident (kernel memory or locked
+ * user space pages).
+ */
+dma_cookie_t
+dma_async_memcpy_buf_to_buf(struct dma_chan *chan, void *dest,
+			void *src, size_t len)
+{
+	struct dma_device *dev = chan->device;
+	struct dma_async_tx_descriptor *tx;
+	dma_addr_t addr;
+	dma_cookie_t cookie;
+	int cpu;
+
+	tx = dev->device_prep_dma_memcpy(chan, len, 0);
+	if (!tx)
+		return -ENOMEM;
+
+	tx->ack = 1;
+	tx->callback = NULL;
+	addr = dma_map_single(dev->dev, src, len, DMA_TO_DEVICE);
+	tx->tx_set_src(addr, tx, 0);
+	addr = dma_map_single(dev->dev, dest, len, DMA_FROM_DEVICE);
+	tx->tx_set_dest(addr, tx, 0);
+	cookie = tx->tx_submit(tx);
+
+	cpu = get_cpu();
+	per_cpu_ptr(chan->local, cpu)->bytes_transferred += len;
+	per_cpu_ptr(chan->local, cpu)->memcpy_count++;
+	put_cpu();
+
+	return cookie;
+}
+EXPORT_SYMBOL(dma_async_memcpy_buf_to_buf);
+
+/**
+ * dma_async_memcpy_buf_to_pg - offloaded copy from address to page
+ * @chan: DMA channel to offload copy to
+ * @page: destination page
+ * @offset: offset in page to copy to
+ * @kdata: source address (virtual)
+ * @len: length
+ *
+ * Both @page/@offset and @kdata must be mappable to a bus address according
+ * to the DMA mapping API rules for streaming mappings.
+ * Both @page/@offset and @kdata must stay memory resident (kernel memory or
+ * locked user space pages)
+ */
+dma_cookie_t
+dma_async_memcpy_buf_to_pg(struct dma_chan *chan, struct page *page,
+			unsigned int offset, void *kdata, size_t len)
+{
+	struct dma_device *dev = chan->device;
+	struct dma_async_tx_descriptor *tx;
+	dma_addr_t addr;
+	dma_cookie_t cookie;
+	int cpu;
+
+	tx = dev->device_prep_dma_memcpy(chan, len, 0);
+	if (!tx)
+		return -ENOMEM;
+
+	tx->ack = 1;
+	tx->callback = NULL;
+	addr = dma_map_single(dev->dev, kdata, len, DMA_TO_DEVICE);
+	tx->tx_set_src(addr, tx, 0);
+	addr = dma_map_page(dev->dev, page, offset, len, DMA_FROM_DEVICE);
+	tx->tx_set_dest(addr, tx, 0);
+	cookie = tx->tx_submit(tx);
+
+	cpu = get_cpu();
+	per_cpu_ptr(chan->local, cpu)->bytes_transferred += len;
+	per_cpu_ptr(chan->local, cpu)->memcpy_count++;
+	put_cpu();
+
+	return cookie;
+}
+EXPORT_SYMBOL(dma_async_memcpy_buf_to_pg);
+
+/**
+ * dma_async_memcpy_pg_to_pg - offloaded copy from page to page
+ * @chan: DMA channel to offload copy to
+ * @dest_pg: destination page
+ * @dest_off: offset in page to copy to
+ * @src_pg: source page
+ * @src_off: offset in page to copy from
+ * @len: length
+ *
+ * Both @dest_page/@dest_off and @src_page/@src_off must be mappable to a bus
+ * address according to the DMA mapping API rules for streaming mappings.
+ * Both @dest_page/@dest_off and @src_page/@src_off must stay memory resident
+ * (kernel memory or locked user space pages).
+ */
+dma_cookie_t
+dma_async_memcpy_pg_to_pg(struct dma_chan *chan, struct page *dest_pg,
+	unsigned int dest_off, struct page *src_pg, unsigned int src_off,
+	size_t len)
+{
+	struct dma_device *dev = chan->device;
+	struct dma_async_tx_descriptor *tx;
+	dma_addr_t addr;
+	dma_cookie_t cookie;
+	int cpu;
+
+	tx = dev->device_prep_dma_memcpy(chan, len, 0);
+	if (!tx)
+		return -ENOMEM;
+
+	tx->ack = 1;
+	tx->callback = NULL;
+	addr = dma_map_page(dev->dev, src_pg, src_off, len, DMA_TO_DEVICE);
+	tx->tx_set_src(addr, tx, 0);
+	addr = dma_map_page(dev->dev, dest_pg, dest_off, len, DMA_FROM_DEVICE);
+	tx->tx_set_dest(addr, tx, 0);
+	cookie = tx->tx_submit(tx);
+
+	cpu = get_cpu();
+	per_cpu_ptr(chan->local, cpu)->bytes_transferred += len;
+	per_cpu_ptr(chan->local, cpu)->memcpy_count++;
+	put_cpu();
+
+	return cookie;
+}
+EXPORT_SYMBOL(dma_async_memcpy_pg_to_pg);
+
+void dma_async_tx_descriptor_init(struct dma_async_tx_descriptor *tx,
+	struct dma_chan *chan)
+{
+	tx->chan = chan;
+	spin_lock_init(&tx->lock);
+	INIT_LIST_HEAD(&tx->depend_node);
+	INIT_LIST_HEAD(&tx->depend_list);
+}
+EXPORT_SYMBOL(dma_async_tx_descriptor_init);
  
 static int __init dma_bus_init(void)
 {
@@ -38,6 +38,7 @@
 #define to_ioat_chan(chan) container_of(chan, struct ioat_dma_chan, common)
 #define to_ioat_device(dev) container_of(dev, struct ioat_device, common)
 #define to_ioat_desc(lh) container_of(lh, struct ioat_desc_sw, node)
+#define tx_to_ioat_desc(tx) container_of(tx, struct ioat_desc_sw, async_tx)
  
 /* internal functions */
 static int __devinit ioat_probe(struct pci_dev *pdev, const struct pci_device_id *ent);
@@ -78,6 +79,73 @@
 	return device->common.chancnt;
 }
  
+static void
+ioat_set_src(dma_addr_t addr, struct dma_async_tx_descriptor *tx, int index)
+{
+	struct ioat_desc_sw *iter, *desc = tx_to_ioat_desc(tx);
+	struct ioat_dma_chan *ioat_chan = to_ioat_chan(tx->chan);
+
+	pci_unmap_addr_set(desc, src, addr);
+
+	list_for_each_entry(iter, &desc->async_tx.tx_list, node) {
+		iter->hw->src_addr = addr;
+		addr += ioat_chan->xfercap;
+	}
+
+}
+
+static void
+ioat_set_dest(dma_addr_t addr, struct dma_async_tx_descriptor *tx, int index)
+{
+	struct ioat_desc_sw *iter, *desc = tx_to_ioat_desc(tx);
+	struct ioat_dma_chan *ioat_chan = to_ioat_chan(tx->chan);
+
+	pci_unmap_addr_set(desc, dst, addr);
+
+	list_for_each_entry(iter, &desc->async_tx.tx_list, node) {
+		iter->hw->dst_addr = addr;
+		addr += ioat_chan->xfercap;
+	}
+}
+
+static dma_cookie_t
+ioat_tx_submit(struct dma_async_tx_descriptor *tx)
+{
+	struct ioat_dma_chan *ioat_chan = to_ioat_chan(tx->chan);
+	struct ioat_desc_sw *desc = tx_to_ioat_desc(tx);
+	int append = 0;
+	dma_cookie_t cookie;
+	struct ioat_desc_sw *group_start;
+
+	group_start = list_entry(desc->async_tx.tx_list.next,
+				 struct ioat_desc_sw, node);
+	spin_lock_bh(&ioat_chan->desc_lock);
+	/* cookie incr and addition to used_list must be atomic */
+	cookie = ioat_chan->common.cookie;
+	cookie++;
+	if (cookie < 0)
+		cookie = 1;
+	ioat_chan->common.cookie = desc->async_tx.cookie = cookie;
+
+	/* write address into NextDescriptor field of last desc in chain */
+	to_ioat_desc(ioat_chan->used_desc.prev)->hw->next =
+						group_start->async_tx.phys;
+	list_splice_init(&desc->async_tx.tx_list, ioat_chan->used_desc.prev);
+
+	ioat_chan->pending += desc->tx_cnt;
+	if (ioat_chan->pending >= 4) {
+		append = 1;
+		ioat_chan->pending = 0;
+	}
+	spin_unlock_bh(&ioat_chan->desc_lock);
+
+	if (append)
+		writeb(IOAT_CHANCMD_APPEND,
+			ioat_chan->reg_base + IOAT_CHANCMD_OFFSET);
+	
+	return cookie;
+}
+
 static struct ioat_desc_sw *ioat_dma_alloc_descriptor(
 	struct ioat_dma_chan *ioat_chan,
 	gfp_t flags)
  
@@ -99,8 +167,13 @@
 	}
  
 	memset(desc, 0, sizeof(*desc));
+	dma_async_tx_descriptor_init(&desc_sw->async_tx, &ioat_chan->common);
+	desc_sw->async_tx.tx_set_src = ioat_set_src;
+	desc_sw->async_tx.tx_set_dest = ioat_set_dest;
+	desc_sw->async_tx.tx_submit = ioat_tx_submit;
+	INIT_LIST_HEAD(&desc_sw->async_tx.tx_list);
 	desc_sw->hw = desc;
-	desc_sw->phys = phys;
+	desc_sw->async_tx.phys = phys;
  
 	return desc_sw;
 }
  
@@ -188,12 +261,14 @@
 	list_for_each_entry_safe(desc, _desc, &ioat_chan->used_desc, node) {
 		in_use_descs++;
 		list_del(&desc->node);
-		pci_pool_free(ioat_device->dma_pool, desc->hw, desc->phys);
+		pci_pool_free(ioat_device->dma_pool, desc->hw,
+			      desc->async_tx.phys);
 		kfree(desc);
 	}
 	list_for_each_entry_safe(desc, _desc, &ioat_chan->free_desc, node) {
 		list_del(&desc->node);
-		pci_pool_free(ioat_device->dma_pool, desc->hw, desc->phys);
+		pci_pool_free(ioat_device->dma_pool, desc->hw,
+			      desc->async_tx.phys);
 		kfree(desc);
 	}
 	spin_unlock_bh(&ioat_chan->desc_lock);
  
  
  
  
  
  
@@ -215,45 +290,25 @@
 	writew(chanctrl, ioat_chan->reg_base + IOAT_CHANCTRL_OFFSET);
 }
  
-/**
- * do_ioat_dma_memcpy - actual function that initiates a IOAT DMA transaction
- * @ioat_chan: IOAT DMA channel handle
- * @dest: DMA destination address
- * @src: DMA source address
- * @len: transaction length in bytes
- */
-
-static dma_cookie_t do_ioat_dma_memcpy(struct ioat_dma_chan *ioat_chan,
-                                       dma_addr_t dest,
-                                       dma_addr_t src,
-                                       size_t len)
+static struct dma_async_tx_descriptor *
+ioat_dma_prep_memcpy(struct dma_chan *chan, size_t len, int int_en)
 {
-	struct ioat_desc_sw *first;
-	struct ioat_desc_sw *prev;
-	struct ioat_desc_sw *new;
-	dma_cookie_t cookie;
+	struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
+	struct ioat_desc_sw *first, *prev, *new;
 	LIST_HEAD(new_chain);
 	u32 copy;
 	size_t orig_len;
-	dma_addr_t orig_src, orig_dst;
-	unsigned int desc_count = 0;
-	unsigned int append = 0;
+	int desc_count = 0;
  
-	if (!ioat_chan || !dest || !src)
-		return -EFAULT;
-
 	if (!len)
-		return ioat_chan->common.cookie;
+		return NULL;
  
 	orig_len = len;
-	orig_src = src;
-	orig_dst = dest;
  
 	first = NULL;
 	prev = NULL;
  
 	spin_lock_bh(&ioat_chan->desc_lock);
-
 	while (len) {
 		if (!list_empty(&ioat_chan->free_desc)) {
 			new = to_ioat_desc(ioat_chan->free_desc.next);
  
  
  
  
  
  
  
  
  
  
  
@@ -270,142 +325,38 @@
  
 		new->hw->size = copy;
 		new->hw->ctl = 0;
-		new->hw->src_addr = src;
-		new->hw->dst_addr = dest;
-		new->cookie = 0;
+		new->async_tx.cookie = 0;
+		new->async_tx.ack = 1;
  
 		/* chain together the physical address list for the HW */
 		if (!first)
 			first = new;
 		else
-			prev->hw->next = (u64) new->phys;
+			prev->hw->next = (u64) new->async_tx.phys;
  
 		prev = new;
-
 		len  -= copy;
-		dest += copy;
-		src  += copy;
-
 		list_add_tail(&new->node, &new_chain);
 		desc_count++;
 	}
+
+	list_splice(&new_chain, &new->async_tx.tx_list);
+
 	new->hw->ctl = IOAT_DMA_DESCRIPTOR_CTL_CP_STS;
 	new->hw->next = 0;
+	new->tx_cnt = desc_count;
+	new->async_tx.ack = 0; /* client is in control of this ack */
+	new->async_tx.cookie = -EBUSY;
  
-	/* cookie incr and addition to used_list must be atomic */
-
-	cookie = ioat_chan->common.cookie;
-	cookie++;
-	if (cookie < 0)
-		cookie = 1;
-	ioat_chan->common.cookie = new->cookie = cookie;
-
-	pci_unmap_addr_set(new, src, orig_src);
-	pci_unmap_addr_set(new, dst, orig_dst);
 	pci_unmap_len_set(new, src_len, orig_len);
 	pci_unmap_len_set(new, dst_len, orig_len);
-
-	/* write address into NextDescriptor field of last desc in chain */
-	to_ioat_desc(ioat_chan->used_desc.prev)->hw->next = first->phys;
-	list_splice_init(&new_chain, ioat_chan->used_desc.prev);
-
-	ioat_chan->pending += desc_count;
-	if (ioat_chan->pending >= 4) {
-		append = 1;
-		ioat_chan->pending = 0;
-	}
-
 	spin_unlock_bh(&ioat_chan->desc_lock);
  
-	if (append)
-		writeb(IOAT_CHANCMD_APPEND,
-		       ioat_chan->reg_base + IOAT_CHANCMD_OFFSET);
-	return cookie;
+	return new ? &new->async_tx : NULL;
 }
  
-/**
- * ioat_dma_memcpy_buf_to_buf - wrapper that takes src & dest bufs
- * @chan: IOAT DMA channel handle
- * @dest: DMA destination address
- * @src: DMA source address
- * @len: transaction length in bytes
- */
  
-static dma_cookie_t ioat_dma_memcpy_buf_to_buf(struct dma_chan *chan,
-                                               void *dest,
-                                               void *src,
-                                               size_t len)
-{
-	dma_addr_t dest_addr;
-	dma_addr_t src_addr;
-	struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
-
-	dest_addr = pci_map_single(ioat_chan->device->pdev,
-		dest, len, PCI_DMA_FROMDEVICE);
-	src_addr = pci_map_single(ioat_chan->device->pdev,
-		src, len, PCI_DMA_TODEVICE);
-
-	return do_ioat_dma_memcpy(ioat_chan, dest_addr, src_addr, len);
-}
-
 /**
- * ioat_dma_memcpy_buf_to_pg - wrapper, copying from a buf to a page
- * @chan: IOAT DMA channel handle
- * @page: pointer to the page to copy to
- * @offset: offset into that page
- * @src: DMA source address
- * @len: transaction length in bytes
- */
-
-static dma_cookie_t ioat_dma_memcpy_buf_to_pg(struct dma_chan *chan,
-                                              struct page *page,
-                                              unsigned int offset,
-                                              void *src,
-                                              size_t len)
-{
-	dma_addr_t dest_addr;
-	dma_addr_t src_addr;
-	struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
-
-	dest_addr = pci_map_page(ioat_chan->device->pdev,
-		page, offset, len, PCI_DMA_FROMDEVICE);
-	src_addr = pci_map_single(ioat_chan->device->pdev,
-		src, len, PCI_DMA_TODEVICE);
-
-	return do_ioat_dma_memcpy(ioat_chan, dest_addr, src_addr, len);
-}
-
-/**
- * ioat_dma_memcpy_pg_to_pg - wrapper, copying between two pages
- * @chan: IOAT DMA channel handle
- * @dest_pg: pointer to the page to copy to
- * @dest_off: offset into that page
- * @src_pg: pointer to the page to copy from
- * @src_off: offset into that page
- * @len: transaction length in bytes. This is guaranteed not to make a copy
- *	 across a page boundary.
- */
-
-static dma_cookie_t ioat_dma_memcpy_pg_to_pg(struct dma_chan *chan,
-                                             struct page *dest_pg,
-                                             unsigned int dest_off,
-                                             struct page *src_pg,
-                                             unsigned int src_off,
-                                             size_t len)
-{
-	dma_addr_t dest_addr;
-	dma_addr_t src_addr;
-	struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
-
-	dest_addr = pci_map_page(ioat_chan->device->pdev,
-		dest_pg, dest_off, len, PCI_DMA_FROMDEVICE);
-	src_addr = pci_map_page(ioat_chan->device->pdev,
-		src_pg, src_off, len, PCI_DMA_TODEVICE);
-
-	return do_ioat_dma_memcpy(ioat_chan, dest_addr, src_addr, len);
-}
-
-/**
  * ioat_dma_memcpy_issue_pending - push potentially unrecognized appended descriptors to hw
  * @chan: DMA channel handle
  */
@@ -465,8 +416,8 @@
 		 * exceeding xfercap, perhaps. If so, only the last one will
 		 * have a cookie, and require unmapping.
 		 */
-		if (desc->cookie) {
-			cookie = desc->cookie;
+		if (desc->async_tx.cookie) {
+			cookie = desc->async_tx.cookie;
  
 			/* yes we are unmapping both _page and _single alloc'd
 			   regions with unmap_page. Is this *really* that bad?
  
@@ -481,14 +432,19 @@
 					PCI_DMA_TODEVICE);
 		}
  
-		if (desc->phys != phys_complete) {
-			/* a completed entry, but not the last, so cleanup */
-			list_del(&desc->node);
-			list_add_tail(&desc->node, &chan->free_desc);
+		if (desc->async_tx.phys != phys_complete) {
+			/* a completed entry, but not the last, so cleanup
+			 * if the client is done with the descriptor
+			 */
+			if (desc->async_tx.ack) {
+				list_del(&desc->node);
+				list_add_tail(&desc->node, &chan->free_desc);
+			} else
+				desc->async_tx.cookie = 0;
 		} else {
 			/* last used desc. Do not remove, so we can append from
 			   it, but don't look at it next time, either */
-			desc->cookie = 0;
+			desc->async_tx.cookie = 0;
  
 			/* TODO check status bits? */
 			break;
@@ -504,6 +460,17 @@
 	spin_unlock(&chan->cleanup_lock);
 }
  
+static void ioat_dma_dependency_added(struct dma_chan *chan)
+{
+	struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
+	spin_lock_bh(&ioat_chan->desc_lock);
+	if (ioat_chan->pending == 0) {
+		spin_unlock_bh(&ioat_chan->desc_lock);
+		ioat_dma_memcpy_cleanup(ioat_chan);
+	} else
+		spin_unlock_bh(&ioat_chan->desc_lock);
+}
+
 /**
  * ioat_dma_is_complete - poll the status of a IOAT DMA transaction
  * @chan: IOAT DMA channel handle
  
  
@@ -606,13 +573,14 @@
  
 	desc->hw->ctl = IOAT_DMA_DESCRIPTOR_NUL;
 	desc->hw->next = 0;
+	desc->async_tx.ack = 1;
  
 	list_add_tail(&desc->node, &ioat_chan->used_desc);
 	spin_unlock_bh(&ioat_chan->desc_lock);
  
-	writel(((u64) desc->phys) & 0x00000000FFFFFFFF,
+	writel(((u64) desc->async_tx.phys) & 0x00000000FFFFFFFF,
 	       ioat_chan->reg_base + IOAT_CHAINADDR_OFFSET_LOW);
-	writel(((u64) desc->phys) >> 32,
+	writel(((u64) desc->async_tx.phys) >> 32,
 	       ioat_chan->reg_base + IOAT_CHAINADDR_OFFSET_HIGH);
  
 	writeb(IOAT_CHANCMD_START, ioat_chan->reg_base + IOAT_CHANCMD_OFFSET);
@@ -629,6 +597,8 @@
 	u8 *src;
 	u8 *dest;
 	struct dma_chan *dma_chan;
+	struct dma_async_tx_descriptor *tx;
+	dma_addr_t addr;
 	dma_cookie_t cookie;
 	int err = 0;
  
@@ -654,7 +624,15 @@
 		goto out;
 	}
  
-	cookie = ioat_dma_memcpy_buf_to_buf(dma_chan, dest, src, IOAT_TEST_SIZE);
+	tx = ioat_dma_prep_memcpy(dma_chan, IOAT_TEST_SIZE, 0);
+	async_tx_ack(tx);
+	addr = dma_map_single(dma_chan->device->dev, src, IOAT_TEST_SIZE,
+			DMA_TO_DEVICE);
+	ioat_set_src(addr, tx, 0);
+	addr = dma_map_single(dma_chan->device->dev, dest, IOAT_TEST_SIZE,
+			DMA_FROM_DEVICE);
+	ioat_set_dest(addr, tx, 0);
+	cookie = ioat_tx_submit(tx);
 	ioat_dma_memcpy_issue_pending(dma_chan);
 	msleep(1);
  
  
@@ -750,13 +728,14 @@
 	INIT_LIST_HEAD(&device->common.channels);
 	enumerate_dma_channels(device);
  
+	dma_cap_set(DMA_MEMCPY, device->common.cap_mask);
 	device->common.device_alloc_chan_resources = ioat_dma_alloc_chan_resources;
 	device->common.device_free_chan_resources = ioat_dma_free_chan_resources;
-	device->common.device_memcpy_buf_to_buf = ioat_dma_memcpy_buf_to_buf;
-	device->common.device_memcpy_buf_to_pg = ioat_dma_memcpy_buf_to_pg;
-	device->common.device_memcpy_pg_to_pg = ioat_dma_memcpy_pg_to_pg;
-	device->common.device_memcpy_complete = ioat_dma_is_complete;
-	device->common.device_memcpy_issue_pending = ioat_dma_memcpy_issue_pending;
+	device->common.device_prep_dma_memcpy = ioat_dma_prep_memcpy;
+	device->common.device_is_tx_complete = ioat_dma_is_complete;
+	device->common.device_issue_pending = ioat_dma_memcpy_issue_pending;
+	device->common.device_dependency_added = ioat_dma_dependency_added;
+	device->common.dev = &pdev->dev;
 	printk(KERN_INFO "Intel(R) I/OAT DMA Engine found, %d channels\n",
 		device->common.chancnt);
  
@@ -105,20 +105,20 @@
 /**
  * struct ioat_desc_sw - wrapper around hardware descriptor
  * @hw: hardware DMA descriptor
- * @node:
- * @cookie:
- * @phys:
+ * @node: this descriptor will either be on the free list,
+ *     or attached to a transaction list (async_tx.tx_list)
+ * @tx_cnt: number of descriptors required to complete the transaction
+ * @async_tx: the generic software descriptor for all engines
  */
-
 struct ioat_desc_sw {
 	struct ioat_dma_descriptor *hw;
 	struct list_head node;
-	dma_cookie_t cookie;
-	dma_addr_t phys;
+	int tx_cnt;
 	DECLARE_PCI_UNMAP_ADDR(src)
 	DECLARE_PCI_UNMAP_LEN(src_len)
 	DECLARE_PCI_UNMAP_ADDR(dst)
 	DECLARE_PCI_UNMAP_LEN(dst_len)
+	struct dma_async_tx_descriptor async_tx;
 };
  
 #endif /* IOATDMA_H */
@@ -21,13 +21,12 @@
 #ifndef DMAENGINE_H
 #define DMAENGINE_H
  
-#ifdef CONFIG_DMA_ENGINE
-
 #include <linux/device.h>
 #include <linux/uio.h>
 #include <linux/kref.h>
 #include <linux/completion.h>
 #include <linux/rcupdate.h>
+#include <linux/dma-mapping.h>
  
 /**
  * enum dma_event - resource PNP/power managment events
@@ -65,6 +64,31 @@
 };
  
 /**
+ * enum dma_transaction_type - DMA transaction types/indexes
+ */
+enum dma_transaction_type {
+	DMA_MEMCPY,
+	DMA_XOR,
+	DMA_PQ_XOR,
+	DMA_DUAL_XOR,
+	DMA_PQ_UPDATE,
+	DMA_ZERO_SUM,
+	DMA_PQ_ZERO_SUM,
+	DMA_MEMSET,
+	DMA_MEMCPY_CRC32C,
+	DMA_INTERRUPT,
+};
+
+/* last transaction type for creation of the capabilities mask */
+#define DMA_TX_TYPE_END (DMA_INTERRUPT + 1)
+
+/**
+ * dma_cap_mask_t - capabilities bitmap modeled after cpumask_t.
+ * See linux/cpumask.h
+ */
+typedef struct { DECLARE_BITMAP(bits, DMA_TX_TYPE_END); } dma_cap_mask_t;
+
+/**
  * struct dma_chan_percpu - the per-CPU part of struct dma_chan
  * @refcount: local_t used for open-coded "bigref" counting
  * @memcpy_count: transaction counter
  
  
  
  
  
  
  
  
@@ -157,48 +181,106 @@
 	struct list_head	global_node;
 };
  
+typedef void (*dma_async_tx_callback)(void *dma_async_param);
 /**
+ * struct dma_async_tx_descriptor - async transaction descriptor
+ * ---dma generic offload fields---
+ * @cookie: tracking cookie for this transaction, set to -EBUSY if
+ *	this tx is sitting on a dependency list
+ * @ack: the descriptor can not be reused until the client acknowledges
+ *	receipt, i.e. has has a chance to establish any dependency chains
+ * @phys: physical address of the descriptor
+ * @tx_list: driver common field for operations that require multiple
+ *	descriptors
+ * @chan: target channel for this operation
+ * @tx_submit: set the prepared descriptor(s) to be executed by the engine
+ * @tx_set_dest: set a destination address in a hardware descriptor
+ * @tx_set_src: set a source address in a hardware descriptor
+ * @callback: routine to call after this operation is complete
+ * @callback_param: general parameter to pass to the callback routine
+ * ---async_tx api specific fields---
+ * @depend_list: at completion this list of transactions are submitted
+ * @depend_node: allow this transaction to be executed after another
+ *	transaction has completed, possibly on another channel
+ * @parent: pointer to the next level up in the dependency chain
+ * @lock: protect the dependency list
+ */
+struct dma_async_tx_descriptor {
+	dma_cookie_t cookie;
+	int ack;
+	dma_addr_t phys;
+	struct list_head tx_list;
+	struct dma_chan *chan;
+	dma_cookie_t (*tx_submit)(struct dma_async_tx_descriptor *tx);
+	void (*tx_set_dest)(dma_addr_t addr,
+		struct dma_async_tx_descriptor *tx, int index);
+	void (*tx_set_src)(dma_addr_t addr,
+		struct dma_async_tx_descriptor *tx, int index);
+	dma_async_tx_callback callback;
+	void *callback_param;
+	struct list_head depend_list;
+	struct list_head depend_node;
+	struct dma_async_tx_descriptor *parent;
+	spinlock_t lock;
+};
+
+/**
  * struct dma_device - info on the entity supplying DMA services
  * @chancnt: how many DMA channels are supported
  * @channels: the list of struct dma_chan
  * @global_node: list_head for global dma_device_list
+ * @cap_mask: one or more dma_capability flags
+ * @max_xor: maximum number of xor sources, 0 if no capability
  * @refcount: reference count
  * @done: IO completion struct
  * @dev_id: unique device ID
+ * @dev: struct device reference for dma mapping api
  * @device_alloc_chan_resources: allocate resources and return the
  *	number of allocated descriptors
  * @device_free_chan_resources: release DMA channel's resources
- * @device_memcpy_buf_to_buf: memcpy buf pointer to buf pointer
- * @device_memcpy_buf_to_pg: memcpy buf pointer to struct page
- * @device_memcpy_pg_to_pg: memcpy struct page/offset to struct page/offset
- * @device_memcpy_complete: poll the status of an IOAT DMA transaction
- * @device_memcpy_issue_pending: push appended descriptors to hardware
+ * @device_prep_dma_memcpy: prepares a memcpy operation
+ * @device_prep_dma_xor: prepares a xor operation
+ * @device_prep_dma_zero_sum: prepares a zero_sum operation
+ * @device_prep_dma_memset: prepares a memset operation
+ * @device_prep_dma_interrupt: prepares an end of chain interrupt operation
+ * @device_dependency_added: async_tx notifies the channel about new deps
+ * @device_issue_pending: push pending transactions to hardware
  */
 struct dma_device {
  
 	unsigned int chancnt;
 	struct list_head channels;
 	struct list_head global_node;
+	dma_cap_mask_t  cap_mask;
+	int max_xor;
  
 	struct kref refcount;
 	struct completion done;
  
 	int dev_id;
+	struct device *dev;
  
 	int (*device_alloc_chan_resources)(struct dma_chan *chan);
 	void (*device_free_chan_resources)(struct dma_chan *chan);
-	dma_cookie_t (*device_memcpy_buf_to_buf)(struct dma_chan *chan,
-			void *dest, void *src, size_t len);
-	dma_cookie_t (*device_memcpy_buf_to_pg)(struct dma_chan *chan,
-			struct page *page, unsigned int offset, void *kdata,
-			size_t len);
-	dma_cookie_t (*device_memcpy_pg_to_pg)(struct dma_chan *chan,
-			struct page *dest_pg, unsigned int dest_off,
-			struct page *src_pg, unsigned int src_off, size_t len);
-	enum dma_status (*device_memcpy_complete)(struct dma_chan *chan,
+
+	struct dma_async_tx_descriptor *(*device_prep_dma_memcpy)(
+		struct dma_chan *chan, size_t len, int int_en);
+	struct dma_async_tx_descriptor *(*device_prep_dma_xor)(
+		struct dma_chan *chan, unsigned int src_cnt, size_t len,
+		int int_en);
+	struct dma_async_tx_descriptor *(*device_prep_dma_zero_sum)(
+		struct dma_chan *chan, unsigned int src_cnt, size_t len,
+		u32 *result, int int_en);
+	struct dma_async_tx_descriptor *(*device_prep_dma_memset)(
+		struct dma_chan *chan, int value, size_t len, int int_en);
+	struct dma_async_tx_descriptor *(*device_prep_dma_interrupt)(
+		struct dma_chan *chan);
+
+	void (*device_dependency_added)(struct dma_chan *chan);
+	enum dma_status (*device_is_tx_complete)(struct dma_chan *chan,
 			dma_cookie_t cookie, dma_cookie_t *last,
 			dma_cookie_t *used);
-	void (*device_memcpy_issue_pending)(struct dma_chan *chan);
+	void (*device_issue_pending)(struct dma_chan *chan);
 };
  
 /* --- public DMA engine API --- */
  
  
  
  
  
  
  
  
  
  
  
  
  
  
@@ -207,96 +289,72 @@
 void dma_async_client_unregister(struct dma_client *client);
 void dma_async_client_chan_request(struct dma_client *client,
 		unsigned int number);
+dma_cookie_t dma_async_memcpy_buf_to_buf(struct dma_chan *chan,
+	void *dest, void *src, size_t len);
+dma_cookie_t dma_async_memcpy_buf_to_pg(struct dma_chan *chan,
+	struct page *page, unsigned int offset, void *kdata, size_t len);
+dma_cookie_t dma_async_memcpy_pg_to_pg(struct dma_chan *chan,
+	struct page *dest_pg, unsigned int dest_off, struct page *src_pg,
+	unsigned int src_off, size_t len);
+void dma_async_tx_descriptor_init(struct dma_async_tx_descriptor *tx,
+	struct dma_chan *chan);
  
-/**
- * dma_async_memcpy_buf_to_buf - offloaded copy between virtual addresses
- * @chan: DMA channel to offload copy to
- * @dest: destination address (virtual)
- * @src: source address (virtual)
- * @len: length
- *
- * Both @dest and @src must be mappable to a bus address according to the
- * DMA mapping API rules for streaming mappings.
- * Both @dest and @src must stay memory resident (kernel memory or locked
- * user space pages).
- */
-static inline dma_cookie_t dma_async_memcpy_buf_to_buf(struct dma_chan *chan,
-	void *dest, void *src, size_t len)
-{
-	int cpu = get_cpu();
-	per_cpu_ptr(chan->local, cpu)->bytes_transferred += len;
-	per_cpu_ptr(chan->local, cpu)->memcpy_count++;
-	put_cpu();
  
-	return chan->device->device_memcpy_buf_to_buf(chan, dest, src, len);
+static inline void
+async_tx_ack(struct dma_async_tx_descriptor *tx)
+{
+	tx->ack = 1;
 }
  
-/**
- * dma_async_memcpy_buf_to_pg - offloaded copy from address to page
- * @chan: DMA channel to offload copy to
- * @page: destination page
- * @offset: offset in page to copy to
- * @kdata: source address (virtual)
- * @len: length
- *
- * Both @page/@offset and @kdata must be mappable to a bus address according
- * to the DMA mapping API rules for streaming mappings.
- * Both @page/@offset and @kdata must stay memory resident (kernel memory or
- * locked user space pages)
- */
-static inline dma_cookie_t dma_async_memcpy_buf_to_pg(struct dma_chan *chan,
-	struct page *page, unsigned int offset, void *kdata, size_t len)
+#define first_dma_cap(mask) __first_dma_cap(&(mask))
+static inline int __first_dma_cap(const dma_cap_mask_t *srcp)
 {
-	int cpu = get_cpu();
-	per_cpu_ptr(chan->local, cpu)->bytes_transferred += len;
-	per_cpu_ptr(chan->local, cpu)->memcpy_count++;
-	put_cpu();
+	return min_t(int, DMA_TX_TYPE_END,
+		find_first_bit(srcp->bits, DMA_TX_TYPE_END));
+}
  
-	return chan->device->device_memcpy_buf_to_pg(chan, page, offset,
-	                                             kdata, len);
+#define next_dma_cap(n, mask) __next_dma_cap((n), &(mask))
+static inline int __next_dma_cap(int n, const dma_cap_mask_t *srcp)
+{
+	return min_t(int, DMA_TX_TYPE_END,
+		find_next_bit(srcp->bits, DMA_TX_TYPE_END, n+1));
 }
  
-/**
- * dma_async_memcpy_pg_to_pg - offloaded copy from page to page
- * @chan: DMA channel to offload copy to
- * @dest_pg: destination page
- * @dest_off: offset in page to copy to
- * @src_pg: source page
- * @src_off: offset in page to copy from
- * @len: length
- *
- * Both @dest_page/@dest_off and @src_page/@src_off must be mappable to a bus
- * address according to the DMA mapping API rules for streaming mappings.
- * Both @dest_page/@dest_off and @src_page/@src_off must stay memory resident
- * (kernel memory or locked user space pages).
- */
-static inline dma_cookie_t dma_async_memcpy_pg_to_pg(struct dma_chan *chan,
-	struct page *dest_pg, unsigned int dest_off, struct page *src_pg,
-	unsigned int src_off, size_t len)
+#define dma_cap_set(tx, mask) __dma_cap_set((tx), &(mask))
+static inline void
+__dma_cap_set(enum dma_transaction_type tx_type, dma_cap_mask_t *dstp)
 {
-	int cpu = get_cpu();
-	per_cpu_ptr(chan->local, cpu)->bytes_transferred += len;
-	per_cpu_ptr(chan->local, cpu)->memcpy_count++;
-	put_cpu();
+	set_bit(tx_type, dstp->bits);
+}
  
-	return chan->device->device_memcpy_pg_to_pg(chan, dest_pg, dest_off,
-	                                            src_pg, src_off, len);
+#define dma_has_cap(tx, mask) __dma_has_cap((tx), &(mask))
+static inline int
+__dma_has_cap(enum dma_transaction_type tx_type, dma_cap_mask_t *srcp)
+{
+	return test_bit(tx_type, srcp->bits);
 }
  
+#define for_each_dma_cap_mask(cap, mask) \
+	for ((cap) = first_dma_cap(mask);	\
+		(cap) < DMA_TX_TYPE_END;	\
+		(cap) = next_dma_cap((cap), (mask)))
+
 /**
- * dma_async_memcpy_issue_pending - flush pending copies to HW
+ * dma_async_issue_pending - flush pending transactions to HW
  * @chan: target DMA channel
  *
  * This allows drivers to push copies to HW in batches,
  * reducing MMIO writes where possible.
  */
-static inline void dma_async_memcpy_issue_pending(struct dma_chan *chan)
+static inline void dma_async_issue_pending(struct dma_chan *chan)
 {
-	return chan->device->device_memcpy_issue_pending(chan);
+	return chan->device->device_issue_pending(chan);
 }
  
+#define dma_async_memcpy_issue_pending(chan) dma_async_issue_pending(chan)
+
 /**
- * dma_async_memcpy_complete - poll for transaction completion
+ * dma_async_is_tx_complete - poll for transaction completion
  * @chan: DMA channel
  * @cookie: transaction identifier to check status of
  * @last: returns last completed cookie, can be NULL
  
  
@@ -306,12 +364,15 @@
  * internal state and can be used with dma_async_is_complete() to check
  * the status of multiple cookies without re-checking hardware state.
  */
-static inline enum dma_status dma_async_memcpy_complete(struct dma_chan *chan,
+static inline enum dma_status dma_async_is_tx_complete(struct dma_chan *chan,
 	dma_cookie_t cookie, dma_cookie_t *last, dma_cookie_t *used)
 {
-	return chan->device->device_memcpy_complete(chan, cookie, last, used);
+	return chan->device->device_is_tx_complete(chan, cookie, last, used);
 }
  
+#define dma_async_memcpy_complete(chan, cookie, last, used)\
+	dma_async_is_tx_complete(chan, cookie, last, used)
+
 /**
  * dma_async_is_complete - test a cookie against chan state
  * @cookie: transaction identifier to test status of
@@ -334,6 +395,7 @@
 	return DMA_IN_PROGRESS;
 }
  
+enum dma_status dma_sync_wait(struct dma_chan *chan, dma_cookie_t cookie);
  
 /* --- DMA device --- */
  
@@ -362,6 +424,5 @@
 	struct dma_pinned_list *pinned_list, struct page *page,
 	unsigned int offset, size_t len);
  
-#endif /* CONFIG_DMA_ENGINE */
 #endif /* DMAENGINE_H */
...	...	@@ -59,6 +59,7 @@
59	59
60	60	#include <linux/init.h>
61	61	#include <linux/module.h>
	62	+#include <linux/mm.h>
62	63	#include <linux/device.h>
63	64	#include <linux/dmaengine.h>
64	65	#include <linux/hardirq.h>
...	...	@@ -66,6 +67,7 @@
66	67	#include <linux/percpu.h>
67	68	#include <linux/rcupdate.h>
68	69	#include <linux/mutex.h>
	70	+#include <linux/jiffies.h>
69	71
70	72	static DEFINE_MUTEX(dma_list_mutex);
71	73	static LIST_HEAD(dma_device_list);
...	...	@@ -165,6 +167,24 @@
165	167	return NULL;
166	168	}
167	169
	170	+enum dma_status dma_sync_wait(struct dma_chan *chan, dma_cookie_t cookie)
	171	+{
	172	+ enum dma_status status;
	173	+ unsigned long dma_sync_wait_timeout = jiffies + msecs_to_jiffies(5000);
	174	+
	175	+ dma_async_issue_pending(chan);
	176	+ do {
	177	+ status = dma_async_is_tx_complete(chan, cookie, NULL, NULL);
	178	+ if (time_after_eq(jiffies, dma_sync_wait_timeout)) {
	179	+ printk(KERN_ERR "dma_sync_wait_timeout!\n");
	180	+ return DMA_ERROR;
	181	+ }
	182	+ } while (status == DMA_IN_PROGRESS);
	183	+
	184	+ return status;
	185	+}
	186	+EXPORT_SYMBOL(dma_sync_wait);
	187	+
168	188	/**
169	189	* dma_chan_cleanup - release a DMA channel's resources
170	190	* @kref: kernel reference structure that contains the DMA channel device
...	...	@@ -322,6 +342,25 @@
322	342	if (!device)
323	343	return -ENODEV;
324	344
	345	+ /* validate device routines */
	346	+ BUG_ON(dma_has_cap(DMA_MEMCPY, device->cap_mask) &&
	347	+ !device->device_prep_dma_memcpy);
	348	+ BUG_ON(dma_has_cap(DMA_XOR, device->cap_mask) &&
	349	+ !device->device_prep_dma_xor);
	350	+ BUG_ON(dma_has_cap(DMA_ZERO_SUM, device->cap_mask) &&
	351	+ !device->device_prep_dma_zero_sum);
	352	+ BUG_ON(dma_has_cap(DMA_MEMSET, device->cap_mask) &&
	353	+ !device->device_prep_dma_memset);
	354	+ BUG_ON(dma_has_cap(DMA_ZERO_SUM, device->cap_mask) &&
	355	+ !device->device_prep_dma_interrupt);
	356	+
	357	+ BUG_ON(!device->device_alloc_chan_resources);
	358	+ BUG_ON(!device->device_free_chan_resources);
	359	+ BUG_ON(!device->device_dependency_added);
	360	+ BUG_ON(!device->device_is_tx_complete);
	361	+ BUG_ON(!device->device_issue_pending);
	362	+ BUG_ON(!device->dev);
	363	+
325	364	init_completion(&device->done);
326	365	kref_init(&device->refcount);
327	366	device->dev_id = id++;
...	...	@@ -414,6 +453,149 @@
414	453	wait_for_completion(&device->done);
415	454	}
416	455	EXPORT_SYMBOL(dma_async_device_unregister);
	456	+
	457	+/**
	458	+ * dma_async_memcpy_buf_to_buf - offloaded copy between virtual addresses
	459	+ * @chan: DMA channel to offload copy to
	460	+ * @dest: destination address (virtual)
	461	+ * @src: source address (virtual)
	462	+ * @len: length
	463	+ *
	464	+ * Both @dest and @src must be mappable to a bus address according to the
	465	+ * DMA mapping API rules for streaming mappings.
	466	+ * Both @dest and @src must stay memory resident (kernel memory or locked
	467	+ * user space pages).
	468	+ */
	469	+dma_cookie_t
	470	+dma_async_memcpy_buf_to_buf(struct dma_chan chan, void dest,
	471	+ void *src, size_t len)
	472	+{
	473	+ struct dma_device *dev = chan->device;
	474	+ struct dma_async_tx_descriptor *tx;
	475	+ dma_addr_t addr;
	476	+ dma_cookie_t cookie;
	477	+ int cpu;
	478	+
	479	+ tx = dev->device_prep_dma_memcpy(chan, len, 0);
	480	+ if (!tx)
	481	+ return -ENOMEM;
	482	+
	483	+ tx->ack = 1;
	484	+ tx->callback = NULL;
	485	+ addr = dma_map_single(dev->dev, src, len, DMA_TO_DEVICE);
	486	+ tx->tx_set_src(addr, tx, 0);
	487	+ addr = dma_map_single(dev->dev, dest, len, DMA_FROM_DEVICE);
	488	+ tx->tx_set_dest(addr, tx, 0);
	489	+ cookie = tx->tx_submit(tx);
	490	+
	491	+ cpu = get_cpu();
	492	+ per_cpu_ptr(chan->local, cpu)->bytes_transferred += len;
	493	+ per_cpu_ptr(chan->local, cpu)->memcpy_count++;
	494	+ put_cpu();
	495	+
	496	+ return cookie;
	497	+}
	498	+EXPORT_SYMBOL(dma_async_memcpy_buf_to_buf);
	499	+
	500	+/**
	501	+ * dma_async_memcpy_buf_to_pg - offloaded copy from address to page
	502	+ * @chan: DMA channel to offload copy to
	503	+ * @page: destination page
	504	+ * @offset: offset in page to copy to
	505	+ * @kdata: source address (virtual)
	506	+ * @len: length
	507	+ *
	508	+ * Both @page/@offset and @kdata must be mappable to a bus address according
	509	+ * to the DMA mapping API rules for streaming mappings.
	510	+ * Both @page/@offset and @kdata must stay memory resident (kernel memory or
	511	+ * locked user space pages)
	512	+ */
	513	+dma_cookie_t
	514	+dma_async_memcpy_buf_to_pg(struct dma_chan chan, struct page page,
	515	+ unsigned int offset, void *kdata, size_t len)
	516	+{
	517	+ struct dma_device *dev = chan->device;
	518	+ struct dma_async_tx_descriptor *tx;
	519	+ dma_addr_t addr;
	520	+ dma_cookie_t cookie;
	521	+ int cpu;
	522	+
	523	+ tx = dev->device_prep_dma_memcpy(chan, len, 0);
	524	+ if (!tx)
	525	+ return -ENOMEM;
	526	+
	527	+ tx->ack = 1;
	528	+ tx->callback = NULL;
	529	+ addr = dma_map_single(dev->dev, kdata, len, DMA_TO_DEVICE);
	530	+ tx->tx_set_src(addr, tx, 0);
	531	+ addr = dma_map_page(dev->dev, page, offset, len, DMA_FROM_DEVICE);
	532	+ tx->tx_set_dest(addr, tx, 0);
	533	+ cookie = tx->tx_submit(tx);
	534	+
	535	+ cpu = get_cpu();
	536	+ per_cpu_ptr(chan->local, cpu)->bytes_transferred += len;
	537	+ per_cpu_ptr(chan->local, cpu)->memcpy_count++;
	538	+ put_cpu();
	539	+
	540	+ return cookie;
	541	+}
	542	+EXPORT_SYMBOL(dma_async_memcpy_buf_to_pg);
	543	+
	544	+/**
	545	+ * dma_async_memcpy_pg_to_pg - offloaded copy from page to page
	546	+ * @chan: DMA channel to offload copy to
	547	+ * @dest_pg: destination page
	548	+ * @dest_off: offset in page to copy to
	549	+ * @src_pg: source page
	550	+ * @src_off: offset in page to copy from
	551	+ * @len: length
	552	+ *
	553	+ * Both @dest_page/@dest_off and @src_page/@src_off must be mappable to a bus
	554	+ * address according to the DMA mapping API rules for streaming mappings.
	555	+ * Both @dest_page/@dest_off and @src_page/@src_off must stay memory resident
	556	+ * (kernel memory or locked user space pages).
	557	+ */
	558	+dma_cookie_t
	559	+dma_async_memcpy_pg_to_pg(struct dma_chan chan, struct page dest_pg,
	560	+ unsigned int dest_off, struct page *src_pg, unsigned int src_off,
	561	+ size_t len)
	562	+{
	563	+ struct dma_device *dev = chan->device;
	564	+ struct dma_async_tx_descriptor *tx;
	565	+ dma_addr_t addr;
	566	+ dma_cookie_t cookie;
	567	+ int cpu;
	568	+
	569	+ tx = dev->device_prep_dma_memcpy(chan, len, 0);
	570	+ if (!tx)
	571	+ return -ENOMEM;
	572	+
	573	+ tx->ack = 1;
	574	+ tx->callback = NULL;
	575	+ addr = dma_map_page(dev->dev, src_pg, src_off, len, DMA_TO_DEVICE);
	576	+ tx->tx_set_src(addr, tx, 0);
	577	+ addr = dma_map_page(dev->dev, dest_pg, dest_off, len, DMA_FROM_DEVICE);
	578	+ tx->tx_set_dest(addr, tx, 0);
	579	+ cookie = tx->tx_submit(tx);
	580	+
	581	+ cpu = get_cpu();
	582	+ per_cpu_ptr(chan->local, cpu)->bytes_transferred += len;
	583	+ per_cpu_ptr(chan->local, cpu)->memcpy_count++;
	584	+ put_cpu();
	585	+
	586	+ return cookie;
	587	+}
	588	+EXPORT_SYMBOL(dma_async_memcpy_pg_to_pg);
	589	+
	590	+void dma_async_tx_descriptor_init(struct dma_async_tx_descriptor *tx,
	591	+ struct dma_chan *chan)
	592	+{
	593	+ tx->chan = chan;
	594	+ spin_lock_init(&tx->lock);
	595	+ INIT_LIST_HEAD(&tx->depend_node);
	596	+ INIT_LIST_HEAD(&tx->depend_list);
	597	+}
	598	+EXPORT_SYMBOL(dma_async_tx_descriptor_init);
417	599
418	600	static int __init dma_bus_init(void)
419	601	{
...	...	@@ -38,6 +38,7 @@
38	38	#define to_ioat_chan(chan) container_of(chan, struct ioat_dma_chan, common)
39	39	#define to_ioat_device(dev) container_of(dev, struct ioat_device, common)
40	40	#define to_ioat_desc(lh) container_of(lh, struct ioat_desc_sw, node)
	41	+#define tx_to_ioat_desc(tx) container_of(tx, struct ioat_desc_sw, async_tx)
41	42
42	43	/* internal functions */
43	44	static int __devinit ioat_probe(struct pci_dev pdev, const struct pci_device_id ent);
...	...	@@ -78,6 +79,73 @@
78	79	return device->common.chancnt;
79	80	}
80	81
	82	+static void
	83	+ioat_set_src(dma_addr_t addr, struct dma_async_tx_descriptor *tx, int index)
	84	+{
	85	+ struct ioat_desc_sw iter, desc = tx_to_ioat_desc(tx);
	86	+ struct ioat_dma_chan *ioat_chan = to_ioat_chan(tx->chan);
	87	+
	88	+ pci_unmap_addr_set(desc, src, addr);
	89	+
	90	+ list_for_each_entry(iter, &desc->async_tx.tx_list, node) {
	91	+ iter->hw->src_addr = addr;
	92	+ addr += ioat_chan->xfercap;
	93	+ }
	94	+
	95	+}
	96	+
	97	+static void
	98	+ioat_set_dest(dma_addr_t addr, struct dma_async_tx_descriptor *tx, int index)
	99	+{
	100	+ struct ioat_desc_sw iter, desc = tx_to_ioat_desc(tx);
	101	+ struct ioat_dma_chan *ioat_chan = to_ioat_chan(tx->chan);
	102	+
	103	+ pci_unmap_addr_set(desc, dst, addr);
	104	+
	105	+ list_for_each_entry(iter, &desc->async_tx.tx_list, node) {
	106	+ iter->hw->dst_addr = addr;
	107	+ addr += ioat_chan->xfercap;
	108	+ }
	109	+}
	110	+
	111	+static dma_cookie_t
	112	+ioat_tx_submit(struct dma_async_tx_descriptor *tx)
	113	+{
	114	+ struct ioat_dma_chan *ioat_chan = to_ioat_chan(tx->chan);
	115	+ struct ioat_desc_sw *desc = tx_to_ioat_desc(tx);
	116	+ int append = 0;
	117	+ dma_cookie_t cookie;
	118	+ struct ioat_desc_sw *group_start;
	119	+
	120	+ group_start = list_entry(desc->async_tx.tx_list.next,
	121	+ struct ioat_desc_sw, node);
	122	+ spin_lock_bh(&ioat_chan->desc_lock);
	123	+ /* cookie incr and addition to used_list must be atomic */
	124	+ cookie = ioat_chan->common.cookie;
	125	+ cookie++;
	126	+ if (cookie < 0)
	127	+ cookie = 1;
	128	+ ioat_chan->common.cookie = desc->async_tx.cookie = cookie;
	129	+
	130	+ /* write address into NextDescriptor field of last desc in chain */
	131	+ to_ioat_desc(ioat_chan->used_desc.prev)->hw->next =
	132	+ group_start->async_tx.phys;
	133	+ list_splice_init(&desc->async_tx.tx_list, ioat_chan->used_desc.prev);
	134	+
	135	+ ioat_chan->pending += desc->tx_cnt;
	136	+ if (ioat_chan->pending >= 4) {
	137	+ append = 1;
	138	+ ioat_chan->pending = 0;
	139	+ }
	140	+ spin_unlock_bh(&ioat_chan->desc_lock);
	141	+
	142	+ if (append)
	143	+ writeb(IOAT_CHANCMD_APPEND,
	144	+ ioat_chan->reg_base + IOAT_CHANCMD_OFFSET);
	145	+
	146	+ return cookie;
	147	+}
	148	+
81	149	static struct ioat_desc_sw *ioat_dma_alloc_descriptor(
82	150	struct ioat_dma_chan *ioat_chan,
83	151	gfp_t flags)
84	152
...	...	@@ -99,8 +167,13 @@
99	167	}
100	168
101	169	memset(desc, 0, sizeof(*desc));
	170	+ dma_async_tx_descriptor_init(&desc_sw->async_tx, &ioat_chan->common);
	171	+ desc_sw->async_tx.tx_set_src = ioat_set_src;
	172	+ desc_sw->async_tx.tx_set_dest = ioat_set_dest;
	173	+ desc_sw->async_tx.tx_submit = ioat_tx_submit;
	174	+ INIT_LIST_HEAD(&desc_sw->async_tx.tx_list);
102	175	desc_sw->hw = desc;
103		- desc_sw->phys = phys;
	176	+ desc_sw->async_tx.phys = phys;
104	177
105	178	return desc_sw;
106	179	}
107	180
...	...	@@ -188,12 +261,14 @@
188	261	list_for_each_entry_safe(desc, _desc, &ioat_chan->used_desc, node) {
189	262	in_use_descs++;
190	263	list_del(&desc->node);
191		- pci_pool_free(ioat_device->dma_pool, desc->hw, desc->phys);
	264	+ pci_pool_free(ioat_device->dma_pool, desc->hw,
	265	+ desc->async_tx.phys);
192	266	kfree(desc);
193	267	}
194	268	list_for_each_entry_safe(desc, _desc, &ioat_chan->free_desc, node) {
195	269	list_del(&desc->node);
196		- pci_pool_free(ioat_device->dma_pool, desc->hw, desc->phys);
	270	+ pci_pool_free(ioat_device->dma_pool, desc->hw,
	271	+ desc->async_tx.phys);
197	272	kfree(desc);
198	273	}
199	274	spin_unlock_bh(&ioat_chan->desc_lock);
200	275
201	276
202	277
203	278
204	279
205	280
...	...	@@ -215,45 +290,25 @@
215	290	writew(chanctrl, ioat_chan->reg_base + IOAT_CHANCTRL_OFFSET);
216	291	}
217	292
218		-/**
219		- * do_ioat_dma_memcpy - actual function that initiates a IOAT DMA transaction
220		- * @ioat_chan: IOAT DMA channel handle
221		- * @dest: DMA destination address
222		- * @src: DMA source address
223		- * @len: transaction length in bytes
224		- */
225		-
226		-static dma_cookie_t do_ioat_dma_memcpy(struct ioat_dma_chan *ioat_chan,
227		- dma_addr_t dest,
228		- dma_addr_t src,
229		- size_t len)
	293	+static struct dma_async_tx_descriptor *
	294	+ioat_dma_prep_memcpy(struct dma_chan *chan, size_t len, int int_en)
230	295	{
231		- struct ioat_desc_sw *first;
232		- struct ioat_desc_sw *prev;
233		- struct ioat_desc_sw *new;
234		- dma_cookie_t cookie;
	296	+ struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
	297	+ struct ioat_desc_sw first, prev, *new;
235	298	LIST_HEAD(new_chain);
236	299	u32 copy;
237	300	size_t orig_len;
238		- dma_addr_t orig_src, orig_dst;
239		- unsigned int desc_count = 0;
240		- unsigned int append = 0;
	301	+ int desc_count = 0;
241	302
242		- if (!ioat_chan \|\| !dest \|\| !src)
243		- return -EFAULT;
244		-
245	303	if (!len)
246		- return ioat_chan->common.cookie;
	304	+ return NULL;
247	305
248	306	orig_len = len;
249		- orig_src = src;
250		- orig_dst = dest;
251	307
252	308	first = NULL;
253	309	prev = NULL;
254	310
255	311	spin_lock_bh(&ioat_chan->desc_lock);
256		-
257	312	while (len) {
258	313	if (!list_empty(&ioat_chan->free_desc)) {
259	314	new = to_ioat_desc(ioat_chan->free_desc.next);
260	315
261	316
262	317
263	318
264	319
265	320
266	321
267	322
268	323
269	324
270	325
...	...	@@ -270,142 +325,38 @@
270	325
271	326	new->hw->size = copy;
272	327	new->hw->ctl = 0;
273		- new->hw->src_addr = src;
274		- new->hw->dst_addr = dest;
275		- new->cookie = 0;
	328	+ new->async_tx.cookie = 0;
	329	+ new->async_tx.ack = 1;
276	330
277	331	/* chain together the physical address list for the HW */
278	332	if (!first)
279	333	first = new;
280	334	else
281		- prev->hw->next = (u64) new->phys;
	335	+ prev->hw->next = (u64) new->async_tx.phys;
282	336
283	337	prev = new;
284		-
285	338	len -= copy;
286		- dest += copy;
287		- src += copy;
288		-
289	339	list_add_tail(&new->node, &new_chain);
290	340	desc_count++;
291	341	}
	342	+
	343	+ list_splice(&new_chain, &new->async_tx.tx_list);
	344	+
292	345	new->hw->ctl = IOAT_DMA_DESCRIPTOR_CTL_CP_STS;
293	346	new->hw->next = 0;
	347	+ new->tx_cnt = desc_count;
	348	+ new->async_tx.ack = 0; /* client is in control of this ack */
	349	+ new->async_tx.cookie = -EBUSY;
294	350
295		- /* cookie incr and addition to used_list must be atomic */
296		-
297		- cookie = ioat_chan->common.cookie;
298		- cookie++;
299		- if (cookie < 0)
300		- cookie = 1;
301		- ioat_chan->common.cookie = new->cookie = cookie;
302		-
303		- pci_unmap_addr_set(new, src, orig_src);
304		- pci_unmap_addr_set(new, dst, orig_dst);
305	351	pci_unmap_len_set(new, src_len, orig_len);
306	352	pci_unmap_len_set(new, dst_len, orig_len);
307		-
308		- /* write address into NextDescriptor field of last desc in chain */
309		- to_ioat_desc(ioat_chan->used_desc.prev)->hw->next = first->phys;
310		- list_splice_init(&new_chain, ioat_chan->used_desc.prev);
311		-
312		- ioat_chan->pending += desc_count;
313		- if (ioat_chan->pending >= 4) {
314		- append = 1;
315		- ioat_chan->pending = 0;
316		- }
317		-
318	353	spin_unlock_bh(&ioat_chan->desc_lock);
319	354
320		- if (append)
321		- writeb(IOAT_CHANCMD_APPEND,
322		- ioat_chan->reg_base + IOAT_CHANCMD_OFFSET);
323		- return cookie;
	355	+ return new ? &new->async_tx : NULL;
324	356	}
325	357
326		-/**
327		- * ioat_dma_memcpy_buf_to_buf - wrapper that takes src & dest bufs
328		- * @chan: IOAT DMA channel handle
329		- * @dest: DMA destination address
330		- * @src: DMA source address
331		- * @len: transaction length in bytes
332		- */
333	358
334		-static dma_cookie_t ioat_dma_memcpy_buf_to_buf(struct dma_chan *chan,
335		- void *dest,
336		- void *src,
337		- size_t len)
338		-{
339		- dma_addr_t dest_addr;
340		- dma_addr_t src_addr;
341		- struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
342		-
343		- dest_addr = pci_map_single(ioat_chan->device->pdev,
344		- dest, len, PCI_DMA_FROMDEVICE);
345		- src_addr = pci_map_single(ioat_chan->device->pdev,
346		- src, len, PCI_DMA_TODEVICE);
347		-
348		- return do_ioat_dma_memcpy(ioat_chan, dest_addr, src_addr, len);
349		-}
350		-
351	359	/**
352		- * ioat_dma_memcpy_buf_to_pg - wrapper, copying from a buf to a page
353		- * @chan: IOAT DMA channel handle
354		- * @page: pointer to the page to copy to
355		- * @offset: offset into that page
356		- * @src: DMA source address
357		- * @len: transaction length in bytes
358		- */
359		-
360		-static dma_cookie_t ioat_dma_memcpy_buf_to_pg(struct dma_chan *chan,
361		- struct page *page,
362		- unsigned int offset,
363		- void *src,
364		- size_t len)
365		-{
366		- dma_addr_t dest_addr;
367		- dma_addr_t src_addr;
368		- struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
369		-
370		- dest_addr = pci_map_page(ioat_chan->device->pdev,
371		- page, offset, len, PCI_DMA_FROMDEVICE);
372		- src_addr = pci_map_single(ioat_chan->device->pdev,
373		- src, len, PCI_DMA_TODEVICE);
374		-
375		- return do_ioat_dma_memcpy(ioat_chan, dest_addr, src_addr, len);
376		-}
377		-
378		-/**
379		- * ioat_dma_memcpy_pg_to_pg - wrapper, copying between two pages
380		- * @chan: IOAT DMA channel handle
381		- * @dest_pg: pointer to the page to copy to
382		- * @dest_off: offset into that page
383		- * @src_pg: pointer to the page to copy from
384		- * @src_off: offset into that page
385		- * @len: transaction length in bytes. This is guaranteed not to make a copy
386		- * across a page boundary.
387		- */
388		-
389		-static dma_cookie_t ioat_dma_memcpy_pg_to_pg(struct dma_chan *chan,
390		- struct page *dest_pg,
391		- unsigned int dest_off,
392		- struct page *src_pg,
393		- unsigned int src_off,
394		- size_t len)
395		-{
396		- dma_addr_t dest_addr;
397		- dma_addr_t src_addr;
398		- struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
399		-
400		- dest_addr = pci_map_page(ioat_chan->device->pdev,
401		- dest_pg, dest_off, len, PCI_DMA_FROMDEVICE);
402		- src_addr = pci_map_page(ioat_chan->device->pdev,
403		- src_pg, src_off, len, PCI_DMA_TODEVICE);
404		-
405		- return do_ioat_dma_memcpy(ioat_chan, dest_addr, src_addr, len);
406		-}
407		-
408		-/**
409	360	* ioat_dma_memcpy_issue_pending - push potentially unrecognized appended descriptors to hw
410	361	* @chan: DMA channel handle
411	362	*/
...	...	@@ -465,8 +416,8 @@
465	416	* exceeding xfercap, perhaps. If so, only the last one will
466	417	* have a cookie, and require unmapping.
467	418	*/
468		- if (desc->cookie) {
469		- cookie = desc->cookie;
	419	+ if (desc->async_tx.cookie) {
	420	+ cookie = desc->async_tx.cookie;
470	421
471	422	/* yes we are unmapping both _page and _single alloc'd
472	423	regions with unmap_page. Is this really that bad?
473	424
...	...	@@ -481,14 +432,19 @@
481	432	PCI_DMA_TODEVICE);
482	433	}
483	434
484		- if (desc->phys != phys_complete) {
485		- /* a completed entry, but not the last, so cleanup */
486		- list_del(&desc->node);
487		- list_add_tail(&desc->node, &chan->free_desc);
	435	+ if (desc->async_tx.phys != phys_complete) {
	436	+ /* a completed entry, but not the last, so cleanup
	437	+ * if the client is done with the descriptor
	438	+ */
	439	+ if (desc->async_tx.ack) {
	440	+ list_del(&desc->node);
	441	+ list_add_tail(&desc->node, &chan->free_desc);
	442	+ } else
	443	+ desc->async_tx.cookie = 0;
488	444	} else {
489	445	/* last used desc. Do not remove, so we can append from
490	446	it, but don't look at it next time, either */
491		- desc->cookie = 0;
	447	+ desc->async_tx.cookie = 0;
492	448
493	449	/* TODO check status bits? */
494	450	break;
...	...	@@ -504,6 +460,17 @@
504	460	spin_unlock(&chan->cleanup_lock);
505	461	}
506	462
	463	+static void ioat_dma_dependency_added(struct dma_chan *chan)
	464	+{
	465	+ struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
	466	+ spin_lock_bh(&ioat_chan->desc_lock);
	467	+ if (ioat_chan->pending == 0) {
	468	+ spin_unlock_bh(&ioat_chan->desc_lock);
	469	+ ioat_dma_memcpy_cleanup(ioat_chan);
	470	+ } else
	471	+ spin_unlock_bh(&ioat_chan->desc_lock);
	472	+}
	473	+
507	474	/**
508	475	* ioat_dma_is_complete - poll the status of a IOAT DMA transaction
509	476	* @chan: IOAT DMA channel handle
510	477
511	478
...	...	@@ -606,13 +573,14 @@
606	573
607	574	desc->hw->ctl = IOAT_DMA_DESCRIPTOR_NUL;
608	575	desc->hw->next = 0;
	576	+ desc->async_tx.ack = 1;
609	577
610	578	list_add_tail(&desc->node, &ioat_chan->used_desc);
611	579	spin_unlock_bh(&ioat_chan->desc_lock);
612	580
613		- writel(((u64) desc->phys) & 0x00000000FFFFFFFF,
	581	+ writel(((u64) desc->async_tx.phys) & 0x00000000FFFFFFFF,
614	582	ioat_chan->reg_base + IOAT_CHAINADDR_OFFSET_LOW);
615		- writel(((u64) desc->phys) >> 32,
	583	+ writel(((u64) desc->async_tx.phys) >> 32,
616	584	ioat_chan->reg_base + IOAT_CHAINADDR_OFFSET_HIGH);
617	585
618	586	writeb(IOAT_CHANCMD_START, ioat_chan->reg_base + IOAT_CHANCMD_OFFSET);
...	...	@@ -629,6 +597,8 @@
629	597	u8 *src;
630	598	u8 *dest;
631	599	struct dma_chan *dma_chan;
	600	+ struct dma_async_tx_descriptor *tx;
	601	+ dma_addr_t addr;
632	602	dma_cookie_t cookie;
633	603	int err = 0;
634	604
...	...	@@ -654,7 +624,15 @@
654	624	goto out;
655	625	}
656	626
657		- cookie = ioat_dma_memcpy_buf_to_buf(dma_chan, dest, src, IOAT_TEST_SIZE);
	627	+ tx = ioat_dma_prep_memcpy(dma_chan, IOAT_TEST_SIZE, 0);
	628	+ async_tx_ack(tx);
	629	+ addr = dma_map_single(dma_chan->device->dev, src, IOAT_TEST_SIZE,
	630	+ DMA_TO_DEVICE);
	631	+ ioat_set_src(addr, tx, 0);
	632	+ addr = dma_map_single(dma_chan->device->dev, dest, IOAT_TEST_SIZE,
	633	+ DMA_FROM_DEVICE);
	634	+ ioat_set_dest(addr, tx, 0);
	635	+ cookie = ioat_tx_submit(tx);
658	636	ioat_dma_memcpy_issue_pending(dma_chan);
659	637	msleep(1);
660	638
661	639
...	...	@@ -750,13 +728,14 @@
750	728	INIT_LIST_HEAD(&device->common.channels);
751	729	enumerate_dma_channels(device);
752	730
	731	+ dma_cap_set(DMA_MEMCPY, device->common.cap_mask);
753	732	device->common.device_alloc_chan_resources = ioat_dma_alloc_chan_resources;
754	733	device->common.device_free_chan_resources = ioat_dma_free_chan_resources;
755		- device->common.device_memcpy_buf_to_buf = ioat_dma_memcpy_buf_to_buf;
756		- device->common.device_memcpy_buf_to_pg = ioat_dma_memcpy_buf_to_pg;
757		- device->common.device_memcpy_pg_to_pg = ioat_dma_memcpy_pg_to_pg;
758		- device->common.device_memcpy_complete = ioat_dma_is_complete;
759		- device->common.device_memcpy_issue_pending = ioat_dma_memcpy_issue_pending;
	734	+ device->common.device_prep_dma_memcpy = ioat_dma_prep_memcpy;
	735	+ device->common.device_is_tx_complete = ioat_dma_is_complete;
	736	+ device->common.device_issue_pending = ioat_dma_memcpy_issue_pending;
	737	+ device->common.device_dependency_added = ioat_dma_dependency_added;
	738	+ device->common.dev = &pdev->dev;
760	739	printk(KERN_INFO "Intel(R) I/OAT DMA Engine found, %d channels\n",
761	740	device->common.chancnt);
762	741
...	...	@@ -105,20 +105,20 @@
105	105	/**
106	106	* struct ioat_desc_sw - wrapper around hardware descriptor
107	107	* @hw: hardware DMA descriptor
108		- * @node:
109		- * @cookie:
110		- * @phys:
	108	+ * @node: this descriptor will either be on the free list,
	109	+ * or attached to a transaction list (async_tx.tx_list)
	110	+ * @tx_cnt: number of descriptors required to complete the transaction
	111	+ * @async_tx: the generic software descriptor for all engines
111	112	*/
112		-
113	113	struct ioat_desc_sw {
114	114	struct ioat_dma_descriptor *hw;
115	115	struct list_head node;
116		- dma_cookie_t cookie;
117		- dma_addr_t phys;
	116	+ int tx_cnt;
118	117	DECLARE_PCI_UNMAP_ADDR(src)
119	118	DECLARE_PCI_UNMAP_LEN(src_len)
120	119	DECLARE_PCI_UNMAP_ADDR(dst)
121	120	DECLARE_PCI_UNMAP_LEN(dst_len)
	121	+ struct dma_async_tx_descriptor async_tx;
122	122	};
123	123
124	124	#endif /* IOATDMA_H */
...	...	@@ -21,13 +21,12 @@
21	21	#ifndef DMAENGINE_H
22	22	#define DMAENGINE_H
23	23
24		-#ifdef CONFIG_DMA_ENGINE
25		-
26	24	#include <linux/device.h>
27	25	#include <linux/uio.h>
28	26	#include <linux/kref.h>
29	27	#include <linux/completion.h>
30	28	#include <linux/rcupdate.h>
	29	+#include <linux/dma-mapping.h>
31	30
32	31	/**
33	32	* enum dma_event - resource PNP/power managment events
...	...	@@ -65,6 +64,31 @@
65	64	};
66	65
67	66	/**
	67	+ * enum dma_transaction_type - DMA transaction types/indexes
	68	+ */
	69	+enum dma_transaction_type {
	70	+ DMA_MEMCPY,
	71	+ DMA_XOR,
	72	+ DMA_PQ_XOR,
	73	+ DMA_DUAL_XOR,
	74	+ DMA_PQ_UPDATE,
	75	+ DMA_ZERO_SUM,
	76	+ DMA_PQ_ZERO_SUM,
	77	+ DMA_MEMSET,
	78	+ DMA_MEMCPY_CRC32C,
	79	+ DMA_INTERRUPT,
	80	+};
	81	+
	82	+/* last transaction type for creation of the capabilities mask */
	83	+#define DMA_TX_TYPE_END (DMA_INTERRUPT + 1)
	84	+
	85	+/**
	86	+ * dma_cap_mask_t - capabilities bitmap modeled after cpumask_t.
	87	+ * See linux/cpumask.h
	88	+ */
	89	+typedef struct { DECLARE_BITMAP(bits, DMA_TX_TYPE_END); } dma_cap_mask_t;
	90	+
	91	+/**
68	92	* struct dma_chan_percpu - the per-CPU part of struct dma_chan
69	93	* @refcount: local_t used for open-coded "bigref" counting
70	94	* @memcpy_count: transaction counter
71	95
72	96
73	97
74	98
75	99
76	100
77	101
78	102
...	...	@@ -157,48 +181,106 @@
157	181	struct list_head global_node;
158	182	};
159	183
	184	+typedef void (dma_async_tx_callback)(void dma_async_param);
160	185	/**
	186	+ * struct dma_async_tx_descriptor - async transaction descriptor
	187	+ * ---dma generic offload fields---
	188	+ * @cookie: tracking cookie for this transaction, set to -EBUSY if
	189	+ * this tx is sitting on a dependency list
	190	+ * @ack: the descriptor can not be reused until the client acknowledges
	191	+ * receipt, i.e. has has a chance to establish any dependency chains
	192	+ * @phys: physical address of the descriptor
	193	+ * @tx_list: driver common field for operations that require multiple
	194	+ * descriptors
	195	+ * @chan: target channel for this operation
	196	+ * @tx_submit: set the prepared descriptor(s) to be executed by the engine
	197	+ * @tx_set_dest: set a destination address in a hardware descriptor
	198	+ * @tx_set_src: set a source address in a hardware descriptor
	199	+ * @callback: routine to call after this operation is complete
	200	+ * @callback_param: general parameter to pass to the callback routine
	201	+ * ---async_tx api specific fields---
	202	+ * @depend_list: at completion this list of transactions are submitted
	203	+ * @depend_node: allow this transaction to be executed after another
	204	+ * transaction has completed, possibly on another channel
	205	+ * @parent: pointer to the next level up in the dependency chain
	206	+ * @lock: protect the dependency list
	207	+ */
	208	+struct dma_async_tx_descriptor {
	209	+ dma_cookie_t cookie;
	210	+ int ack;
	211	+ dma_addr_t phys;
	212	+ struct list_head tx_list;
	213	+ struct dma_chan *chan;
	214	+ dma_cookie_t (tx_submit)(struct dma_async_tx_descriptor tx);
	215	+ void (*tx_set_dest)(dma_addr_t addr,
	216	+ struct dma_async_tx_descriptor *tx, int index);
	217	+ void (*tx_set_src)(dma_addr_t addr,
	218	+ struct dma_async_tx_descriptor *tx, int index);
	219	+ dma_async_tx_callback callback;
	220	+ void *callback_param;
	221	+ struct list_head depend_list;
	222	+ struct list_head depend_node;
	223	+ struct dma_async_tx_descriptor *parent;
	224	+ spinlock_t lock;
	225	+};
	226	+
	227	+/**
161	228	* struct dma_device - info on the entity supplying DMA services
162	229	* @chancnt: how many DMA channels are supported
163	230	* @channels: the list of struct dma_chan
164	231	* @global_node: list_head for global dma_device_list
	232	+ * @cap_mask: one or more dma_capability flags
	233	+ * @max_xor: maximum number of xor sources, 0 if no capability
165	234	* @refcount: reference count
166	235	* @done: IO completion struct
167	236	* @dev_id: unique device ID
	237	+ * @dev: struct device reference for dma mapping api
168	238	* @device_alloc_chan_resources: allocate resources and return the
169	239	* number of allocated descriptors
170	240	* @device_free_chan_resources: release DMA channel's resources
171		- * @device_memcpy_buf_to_buf: memcpy buf pointer to buf pointer
172		- * @device_memcpy_buf_to_pg: memcpy buf pointer to struct page
173		- * @device_memcpy_pg_to_pg: memcpy struct page/offset to struct page/offset
174		- * @device_memcpy_complete: poll the status of an IOAT DMA transaction
175		- * @device_memcpy_issue_pending: push appended descriptors to hardware
	241	+ * @device_prep_dma_memcpy: prepares a memcpy operation
	242	+ * @device_prep_dma_xor: prepares a xor operation
	243	+ * @device_prep_dma_zero_sum: prepares a zero_sum operation
	244	+ * @device_prep_dma_memset: prepares a memset operation
	245	+ * @device_prep_dma_interrupt: prepares an end of chain interrupt operation
	246	+ * @device_dependency_added: async_tx notifies the channel about new deps
	247	+ * @device_issue_pending: push pending transactions to hardware
176	248	*/
177	249	struct dma_device {
178	250
179	251	unsigned int chancnt;
180	252	struct list_head channels;
181	253	struct list_head global_node;
	254	+ dma_cap_mask_t cap_mask;
	255	+ int max_xor;
182	256
183	257	struct kref refcount;
184	258	struct completion done;
185	259
186	260	int dev_id;
	261	+ struct device *dev;
187	262
188	263	int (device_alloc_chan_resources)(struct dma_chan chan);
189	264	void (device_free_chan_resources)(struct dma_chan chan);
190		- dma_cookie_t (device_memcpy_buf_to_buf)(struct dma_chan chan,
191		- void dest, void src, size_t len);
192		- dma_cookie_t (device_memcpy_buf_to_pg)(struct dma_chan chan,
193		- struct page page, unsigned int offset, void kdata,
194		- size_t len);
195		- dma_cookie_t (device_memcpy_pg_to_pg)(struct dma_chan chan,
196		- struct page *dest_pg, unsigned int dest_off,
197		- struct page *src_pg, unsigned int src_off, size_t len);
198		- enum dma_status (device_memcpy_complete)(struct dma_chan chan,
	265	+
	266	+ struct dma_async_tx_descriptor (device_prep_dma_memcpy)(
	267	+ struct dma_chan *chan, size_t len, int int_en);
	268	+ struct dma_async_tx_descriptor (device_prep_dma_xor)(
	269	+ struct dma_chan *chan, unsigned int src_cnt, size_t len,
	270	+ int int_en);
	271	+ struct dma_async_tx_descriptor (device_prep_dma_zero_sum)(
	272	+ struct dma_chan *chan, unsigned int src_cnt, size_t len,
	273	+ u32 *result, int int_en);
	274	+ struct dma_async_tx_descriptor (device_prep_dma_memset)(
	275	+ struct dma_chan *chan, int value, size_t len, int int_en);
	276	+ struct dma_async_tx_descriptor (device_prep_dma_interrupt)(
	277	+ struct dma_chan *chan);
	278	+
	279	+ void (device_dependency_added)(struct dma_chan chan);
	280	+ enum dma_status (device_is_tx_complete)(struct dma_chan chan,
199	281	dma_cookie_t cookie, dma_cookie_t *last,
200	282	dma_cookie_t *used);
201		- void (device_memcpy_issue_pending)(struct dma_chan chan);
	283	+ void (device_issue_pending)(struct dma_chan chan);
202	284	};
203	285
204	286	/* --- public DMA engine API --- */
205	287
206	288
207	289
208	290
209	291
210	292
211	293
212	294
213	295
214	296
215	297
216	298
217	299
218	300
...	...	@@ -207,96 +289,72 @@
207	289	void dma_async_client_unregister(struct dma_client *client);
208	290	void dma_async_client_chan_request(struct dma_client *client,
209	291	unsigned int number);
	292	+dma_cookie_t dma_async_memcpy_buf_to_buf(struct dma_chan *chan,
	293	+ void dest, void src, size_t len);
	294	+dma_cookie_t dma_async_memcpy_buf_to_pg(struct dma_chan *chan,
	295	+ struct page page, unsigned int offset, void kdata, size_t len);
	296	+dma_cookie_t dma_async_memcpy_pg_to_pg(struct dma_chan *chan,
	297	+ struct page dest_pg, unsigned int dest_off, struct page src_pg,
	298	+ unsigned int src_off, size_t len);
	299	+void dma_async_tx_descriptor_init(struct dma_async_tx_descriptor *tx,
	300	+ struct dma_chan *chan);
210	301
211		-/**
212		- * dma_async_memcpy_buf_to_buf - offloaded copy between virtual addresses
213		- * @chan: DMA channel to offload copy to
214		- * @dest: destination address (virtual)
215		- * @src: source address (virtual)
216		- * @len: length
217		- *
218		- * Both @dest and @src must be mappable to a bus address according to the
219		- * DMA mapping API rules for streaming mappings.
220		- * Both @dest and @src must stay memory resident (kernel memory or locked
221		- * user space pages).
222		- */
223		-static inline dma_cookie_t dma_async_memcpy_buf_to_buf(struct dma_chan *chan,
224		- void dest, void src, size_t len)
225		-{
226		- int cpu = get_cpu();
227		- per_cpu_ptr(chan->local, cpu)->bytes_transferred += len;
228		- per_cpu_ptr(chan->local, cpu)->memcpy_count++;
229		- put_cpu();
230	302
231		- return chan->device->device_memcpy_buf_to_buf(chan, dest, src, len);
	303	+static inline void
	304	+async_tx_ack(struct dma_async_tx_descriptor *tx)
	305	+{
	306	+ tx->ack = 1;
232	307	}
233	308
234		-/**
235		- * dma_async_memcpy_buf_to_pg - offloaded copy from address to page
236		- * @chan: DMA channel to offload copy to
237		- * @page: destination page
238		- * @offset: offset in page to copy to
239		- * @kdata: source address (virtual)
240		- * @len: length
241		- *
242		- * Both @page/@offset and @kdata must be mappable to a bus address according
243		- * to the DMA mapping API rules for streaming mappings.
244		- * Both @page/@offset and @kdata must stay memory resident (kernel memory or
245		- * locked user space pages)
246		- */
247		-static inline dma_cookie_t dma_async_memcpy_buf_to_pg(struct dma_chan *chan,
248		- struct page page, unsigned int offset, void kdata, size_t len)
	309	+#define first_dma_cap(mask) __first_dma_cap(&(mask))
	310	+static inline int __first_dma_cap(const dma_cap_mask_t *srcp)
249	311	{
250		- int cpu = get_cpu();
251		- per_cpu_ptr(chan->local, cpu)->bytes_transferred += len;
252		- per_cpu_ptr(chan->local, cpu)->memcpy_count++;
253		- put_cpu();
	312	+ return min_t(int, DMA_TX_TYPE_END,
	313	+ find_first_bit(srcp->bits, DMA_TX_TYPE_END));
	314	+}
254	315
255		- return chan->device->device_memcpy_buf_to_pg(chan, page, offset,
256		- kdata, len);
	316	+#define next_dma_cap(n, mask) __next_dma_cap((n), &(mask))
	317	+static inline int __next_dma_cap(int n, const dma_cap_mask_t *srcp)
	318	+{
	319	+ return min_t(int, DMA_TX_TYPE_END,
	320	+ find_next_bit(srcp->bits, DMA_TX_TYPE_END, n+1));
257	321	}
258	322
259		-/**
260		- * dma_async_memcpy_pg_to_pg - offloaded copy from page to page
261		- * @chan: DMA channel to offload copy to
262		- * @dest_pg: destination page
263		- * @dest_off: offset in page to copy to
264		- * @src_pg: source page
265		- * @src_off: offset in page to copy from
266		- * @len: length
267		- *
268		- * Both @dest_page/@dest_off and @src_page/@src_off must be mappable to a bus
269		- * address according to the DMA mapping API rules for streaming mappings.
270		- * Both @dest_page/@dest_off and @src_page/@src_off must stay memory resident
271		- * (kernel memory or locked user space pages).
272		- */
273		-static inline dma_cookie_t dma_async_memcpy_pg_to_pg(struct dma_chan *chan,
274		- struct page dest_pg, unsigned int dest_off, struct page src_pg,
275		- unsigned int src_off, size_t len)
	323	+#define dma_cap_set(tx, mask) __dma_cap_set((tx), &(mask))
	324	+static inline void
	325	+__dma_cap_set(enum dma_transaction_type tx_type, dma_cap_mask_t *dstp)
276	326	{
277		- int cpu = get_cpu();
278		- per_cpu_ptr(chan->local, cpu)->bytes_transferred += len;
279		- per_cpu_ptr(chan->local, cpu)->memcpy_count++;
280		- put_cpu();
	327	+ set_bit(tx_type, dstp->bits);
	328	+}
281	329
282		- return chan->device->device_memcpy_pg_to_pg(chan, dest_pg, dest_off,
283		- src_pg, src_off, len);
	330	+#define dma_has_cap(tx, mask) __dma_has_cap((tx), &(mask))
	331	+static inline int
	332	+__dma_has_cap(enum dma_transaction_type tx_type, dma_cap_mask_t *srcp)
	333	+{
	334	+ return test_bit(tx_type, srcp->bits);
284	335	}
285	336
	337	+#define for_each_dma_cap_mask(cap, mask) \
	338	+ for ((cap) = first_dma_cap(mask); \
	339	+ (cap) < DMA_TX_TYPE_END; \
	340	+ (cap) = next_dma_cap((cap), (mask)))
	341	+
286	342	/**
287		- * dma_async_memcpy_issue_pending - flush pending copies to HW
	343	+ * dma_async_issue_pending - flush pending transactions to HW
288	344	* @chan: target DMA channel
289	345	*
290	346	* This allows drivers to push copies to HW in batches,
291	347	* reducing MMIO writes where possible.
292	348	*/
293		-static inline void dma_async_memcpy_issue_pending(struct dma_chan *chan)
	349	+static inline void dma_async_issue_pending(struct dma_chan *chan)
294	350	{
295		- return chan->device->device_memcpy_issue_pending(chan);
	351	+ return chan->device->device_issue_pending(chan);
296	352	}
297	353
	354	+#define dma_async_memcpy_issue_pending(chan) dma_async_issue_pending(chan)
	355	+
298	356	/**
299		- * dma_async_memcpy_complete - poll for transaction completion
	357	+ * dma_async_is_tx_complete - poll for transaction completion
300	358	* @chan: DMA channel
301	359	* @cookie: transaction identifier to check status of
302	360	* @last: returns last completed cookie, can be NULL
303	361
304	362
...	...	@@ -306,12 +364,15 @@
306	364	* internal state and can be used with dma_async_is_complete() to check
307	365	* the status of multiple cookies without re-checking hardware state.
308	366	*/
309		-static inline enum dma_status dma_async_memcpy_complete(struct dma_chan *chan,
	367	+static inline enum dma_status dma_async_is_tx_complete(struct dma_chan *chan,
310	368	dma_cookie_t cookie, dma_cookie_t last, dma_cookie_t used)
311	369	{
312		- return chan->device->device_memcpy_complete(chan, cookie, last, used);
	370	+ return chan->device->device_is_tx_complete(chan, cookie, last, used);
313	371	}
314	372
	373	+#define dma_async_memcpy_complete(chan, cookie, last, used)\
	374	+ dma_async_is_tx_complete(chan, cookie, last, used)
	375	+
315	376	/**
316	377	* dma_async_is_complete - test a cookie against chan state
317	378	* @cookie: transaction identifier to test status of
...	...	@@ -334,6 +395,7 @@
334	395	return DMA_IN_PROGRESS;
335	396	}
336	397
	398	+enum dma_status dma_sync_wait(struct dma_chan *chan, dma_cookie_t cookie);
337	399
338	400	/* --- DMA device --- */
339	401
...	...	@@ -362,6 +424,5 @@
362	424	struct dma_pinned_list pinned_list, struct page page,
363	425	unsigned int offset, size_t len);
364	426
365		-#endif /* CONFIG_DMA_ENGINE */
366	427	#endif /* DMAENGINE_H */