Doug / smarc-fsl-linux-kernel | Embedian Git Server

Commit 729b5d1b8ec72c28e99840b3f300ba67726e3ab9

Authored by Dan Williams 2009-03-26 00:13:25 +0800

Exists in master and in 7 other branches

dmaengine: allow dma support for async_tx to be toggled

Provide a config option for blocking the allocation of dma channels to
the async_tx api.

Signed-off-by: Dan Williams <dan.j.williams@intel.com>

Showing 3 changed files with 32 additions and 3 deletions Inline Diff

crypto/async_tx/async_tx.c
drivers/dma/Kconfig
include/linux/dmaengine.h

crypto/async_tx/async_tx.c

Diff comments View file @ 729b5d1

1	/*	1	/*
2	* core routines for the asynchronous memory transfer/transform api	2	* core routines for the asynchronous memory transfer/transform api
3	*	3	*
4	* Copyright © 2006, Intel Corporation.	4	* Copyright © 2006, Intel Corporation.
5	*	5	*
6	* Dan Williams <dan.j.williams@intel.com>	6	* Dan Williams <dan.j.williams@intel.com>
7	*	7	*
8	* with architecture considerations by:	8	* with architecture considerations by:
9	* Neil Brown <neilb@suse.de>	9	* Neil Brown <neilb@suse.de>
10	* Jeff Garzik <jeff@garzik.org>	10	* Jeff Garzik <jeff@garzik.org>
11	*	11	*
12	* This program is free software; you can redistribute it and/or modify it	12	* This program is free software; you can redistribute it and/or modify it
13	* under the terms and conditions of the GNU General Public License,	13	* under the terms and conditions of the GNU General Public License,
14	* version 2, as published by the Free Software Foundation.	14	* version 2, as published by the Free Software Foundation.
15	*	15	*
16	* This program is distributed in the hope it will be useful, but WITHOUT	16	* This program is distributed in the hope it will be useful, but WITHOUT
17	* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or	17	* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
18	* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for	18	* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
19	* more details.	19	* more details.
20	*	20	*
21	* You should have received a copy of the GNU General Public License along with	21	* You should have received a copy of the GNU General Public License along with
22	* this program; if not, write to the Free Software Foundation, Inc.,	22	* this program; if not, write to the Free Software Foundation, Inc.,
23	* 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.	23	* 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
24	*	24	*
25	*/	25	*/
26	#include <linux/rculist.h>	26	#include <linux/rculist.h>
27	#include <linux/kernel.h>	27	#include <linux/kernel.h>
28	#include <linux/async_tx.h>	28	#include <linux/async_tx.h>
29		29
30	#ifdef CONFIG_DMA_ENGINE	30	#ifdef CONFIG_DMA_ENGINE
31	static int __init async_tx_init(void)	31	static int __init async_tx_init(void)
32	{	32	{
33	dmaengine_get();	33	async_dmaengine_get();
34		34
35	printk(KERN_INFO "async_tx: api initialized (async)\n");	35	printk(KERN_INFO "async_tx: api initialized (async)\n");
36		36
37	return 0;	37	return 0;
38	}	38	}
39		39
40	static void __exit async_tx_exit(void)	40	static void __exit async_tx_exit(void)
41	{	41	{
42	dmaengine_put();	42	async_dmaengine_put();
43	}	43	}
44		44
45	/**	45	/**
46	* __async_tx_find_channel - find a channel to carry out the operation or let	46	* __async_tx_find_channel - find a channel to carry out the operation or let
47	* the transaction execute synchronously	47	* the transaction execute synchronously
48	* @depend_tx: transaction dependency	48	* @depend_tx: transaction dependency
49	* @tx_type: transaction type	49	* @tx_type: transaction type
50	*/	50	*/
51	struct dma_chan *	51	struct dma_chan *
52	__async_tx_find_channel(struct dma_async_tx_descriptor *depend_tx,	52	__async_tx_find_channel(struct dma_async_tx_descriptor *depend_tx,
53	enum dma_transaction_type tx_type)	53	enum dma_transaction_type tx_type)
54	{	54	{
55	/* see if we can keep the chain on one channel */	55	/* see if we can keep the chain on one channel */
56	if (depend_tx &&	56	if (depend_tx &&
57	dma_has_cap(tx_type, depend_tx->chan->device->cap_mask))	57	dma_has_cap(tx_type, depend_tx->chan->device->cap_mask))
58	return depend_tx->chan;	58	return depend_tx->chan;
59	return dma_find_channel(tx_type);	59	return async_dma_find_channel(tx_type);
60	}	60	}
61	EXPORT_SYMBOL_GPL(__async_tx_find_channel);	61	EXPORT_SYMBOL_GPL(__async_tx_find_channel);
62	#else	62	#else
63	static int __init async_tx_init(void)	63	static int __init async_tx_init(void)
64	{	64	{
65	printk(KERN_INFO "async_tx: api initialized (sync-only)\n");	65	printk(KERN_INFO "async_tx: api initialized (sync-only)\n");
66	return 0;	66	return 0;
67	}	67	}
68		68
69	static void __exit async_tx_exit(void)	69	static void __exit async_tx_exit(void)
70	{	70	{
71	do { } while (0);	71	do { } while (0);
72	}	72	}
73	#endif	73	#endif
74		74
75		75
76	/**	76	/**
77	* async_tx_channel_switch - queue an interrupt descriptor with a dependency	77	* async_tx_channel_switch - queue an interrupt descriptor with a dependency
78	* pre-attached.	78	* pre-attached.
79	* @depend_tx: the operation that must finish before the new operation runs	79	* @depend_tx: the operation that must finish before the new operation runs
80	* @tx: the new operation	80	* @tx: the new operation
81	*/	81	*/
82	static void	82	static void
83	async_tx_channel_switch(struct dma_async_tx_descriptor *depend_tx,	83	async_tx_channel_switch(struct dma_async_tx_descriptor *depend_tx,
84	struct dma_async_tx_descriptor *tx)	84	struct dma_async_tx_descriptor *tx)
85	{	85	{
86	struct dma_chan *chan;	86	struct dma_chan *chan;
87	struct dma_device *device;	87	struct dma_device *device;
88	struct dma_async_tx_descriptor intr_tx = (void ) ~0;	88	struct dma_async_tx_descriptor intr_tx = (void ) ~0;
89		89
90	/* first check to see if we can still append to depend_tx */	90	/* first check to see if we can still append to depend_tx */
91	spin_lock_bh(&depend_tx->lock);	91	spin_lock_bh(&depend_tx->lock);
92	if (depend_tx->parent && depend_tx->chan == tx->chan) {	92	if (depend_tx->parent && depend_tx->chan == tx->chan) {
93	tx->parent = depend_tx;	93	tx->parent = depend_tx;
94	depend_tx->next = tx;	94	depend_tx->next = tx;
95	intr_tx = NULL;	95	intr_tx = NULL;
96	}	96	}
97	spin_unlock_bh(&depend_tx->lock);	97	spin_unlock_bh(&depend_tx->lock);
98		98
99	if (!intr_tx)	99	if (!intr_tx)
100	return;	100	return;
101		101
102	chan = depend_tx->chan;	102	chan = depend_tx->chan;
103	device = chan->device;	103	device = chan->device;
104		104
105	/* see if we can schedule an interrupt	105	/* see if we can schedule an interrupt
106	* otherwise poll for completion	106	* otherwise poll for completion
107	*/	107	*/
108	if (dma_has_cap(DMA_INTERRUPT, device->cap_mask))	108	if (dma_has_cap(DMA_INTERRUPT, device->cap_mask))
109	intr_tx = device->device_prep_dma_interrupt(chan, 0);	109	intr_tx = device->device_prep_dma_interrupt(chan, 0);
110	else	110	else
111	intr_tx = NULL;	111	intr_tx = NULL;
112		112
113	if (intr_tx) {	113	if (intr_tx) {
114	intr_tx->callback = NULL;	114	intr_tx->callback = NULL;
115	intr_tx->callback_param = NULL;	115	intr_tx->callback_param = NULL;
116	tx->parent = intr_tx;	116	tx->parent = intr_tx;
117	/* safe to set ->next outside the lock since we know we are	117	/* safe to set ->next outside the lock since we know we are
118	* not submitted yet	118	* not submitted yet
119	*/	119	*/
120	intr_tx->next = tx;	120	intr_tx->next = tx;
121		121
122	/* check if we need to append */	122	/* check if we need to append */
123	spin_lock_bh(&depend_tx->lock);	123	spin_lock_bh(&depend_tx->lock);
124	if (depend_tx->parent) {	124	if (depend_tx->parent) {
125	intr_tx->parent = depend_tx;	125	intr_tx->parent = depend_tx;
126	depend_tx->next = intr_tx;	126	depend_tx->next = intr_tx;
127	async_tx_ack(intr_tx);	127	async_tx_ack(intr_tx);
128	intr_tx = NULL;	128	intr_tx = NULL;
129	}	129	}
130	spin_unlock_bh(&depend_tx->lock);	130	spin_unlock_bh(&depend_tx->lock);
131		131
132	if (intr_tx) {	132	if (intr_tx) {
133	intr_tx->parent = NULL;	133	intr_tx->parent = NULL;
134	intr_tx->tx_submit(intr_tx);	134	intr_tx->tx_submit(intr_tx);
135	async_tx_ack(intr_tx);	135	async_tx_ack(intr_tx);
136	}	136	}
137	} else {	137	} else {
138	if (dma_wait_for_async_tx(depend_tx) == DMA_ERROR)	138	if (dma_wait_for_async_tx(depend_tx) == DMA_ERROR)
139	panic("%s: DMA_ERROR waiting for depend_tx\n",	139	panic("%s: DMA_ERROR waiting for depend_tx\n",
140	__func__);	140	__func__);
141	tx->tx_submit(tx);	141	tx->tx_submit(tx);
142	}	142	}
143	}	143	}
144		144
145		145
146	/**	146	/**
147	* submit_disposition - while holding depend_tx->lock we must avoid submitting	147	* submit_disposition - while holding depend_tx->lock we must avoid submitting
148	* new operations to prevent a circular locking dependency with	148	* new operations to prevent a circular locking dependency with
149	* drivers that already hold a channel lock when calling	149	* drivers that already hold a channel lock when calling
150	* async_tx_run_dependencies.	150	* async_tx_run_dependencies.
151	* @ASYNC_TX_SUBMITTED: we were able to append the new operation under the lock	151	* @ASYNC_TX_SUBMITTED: we were able to append the new operation under the lock
152	* @ASYNC_TX_CHANNEL_SWITCH: when the lock is dropped schedule a channel switch	152	* @ASYNC_TX_CHANNEL_SWITCH: when the lock is dropped schedule a channel switch
153	* @ASYNC_TX_DIRECT_SUBMIT: when the lock is dropped submit directly	153	* @ASYNC_TX_DIRECT_SUBMIT: when the lock is dropped submit directly
154	*/	154	*/
155	enum submit_disposition {	155	enum submit_disposition {
156	ASYNC_TX_SUBMITTED,	156	ASYNC_TX_SUBMITTED,
157	ASYNC_TX_CHANNEL_SWITCH,	157	ASYNC_TX_CHANNEL_SWITCH,
158	ASYNC_TX_DIRECT_SUBMIT,	158	ASYNC_TX_DIRECT_SUBMIT,
159	};	159	};
160		160
161	void	161	void
162	async_tx_submit(struct dma_chan chan, struct dma_async_tx_descriptor tx,	162	async_tx_submit(struct dma_chan chan, struct dma_async_tx_descriptor tx,
163	enum async_tx_flags flags, struct dma_async_tx_descriptor *depend_tx,	163	enum async_tx_flags flags, struct dma_async_tx_descriptor *depend_tx,
164	dma_async_tx_callback cb_fn, void *cb_param)	164	dma_async_tx_callback cb_fn, void *cb_param)
165	{	165	{
166	tx->callback = cb_fn;	166	tx->callback = cb_fn;
167	tx->callback_param = cb_param;	167	tx->callback_param = cb_param;
168		168
169	if (depend_tx) {	169	if (depend_tx) {
170	enum submit_disposition s;	170	enum submit_disposition s;
171		171
172	/* sanity check the dependency chain:	172	/* sanity check the dependency chain:
173	* 1/ if ack is already set then we cannot be sure	173	* 1/ if ack is already set then we cannot be sure
174	* we are referring to the correct operation	174	* we are referring to the correct operation
175	* 2/ dependencies are 1:1 i.e. two transactions can	175	* 2/ dependencies are 1:1 i.e. two transactions can
176	* not depend on the same parent	176	* not depend on the same parent
177	*/	177	*/
178	BUG_ON(async_tx_test_ack(depend_tx) \|\| depend_tx->next \|\|	178	BUG_ON(async_tx_test_ack(depend_tx) \|\| depend_tx->next \|\|
179	tx->parent);	179	tx->parent);
180		180
181	/* the lock prevents async_tx_run_dependencies from missing	181	/* the lock prevents async_tx_run_dependencies from missing
182	* the setting of ->next when ->parent != NULL	182	* the setting of ->next when ->parent != NULL
183	*/	183	*/
184	spin_lock_bh(&depend_tx->lock);	184	spin_lock_bh(&depend_tx->lock);
185	if (depend_tx->parent) {	185	if (depend_tx->parent) {
186	/* we have a parent so we can not submit directly	186	/* we have a parent so we can not submit directly
187	* if we are staying on the same channel: append	187	* if we are staying on the same channel: append
188	* else: channel switch	188	* else: channel switch
189	*/	189	*/
190	if (depend_tx->chan == chan) {	190	if (depend_tx->chan == chan) {
191	tx->parent = depend_tx;	191	tx->parent = depend_tx;
192	depend_tx->next = tx;	192	depend_tx->next = tx;
193	s = ASYNC_TX_SUBMITTED;	193	s = ASYNC_TX_SUBMITTED;
194	} else	194	} else
195	s = ASYNC_TX_CHANNEL_SWITCH;	195	s = ASYNC_TX_CHANNEL_SWITCH;
196	} else {	196	} else {
197	/* we do not have a parent so we may be able to submit	197	/* we do not have a parent so we may be able to submit
198	* directly if we are staying on the same channel	198	* directly if we are staying on the same channel
199	*/	199	*/
200	if (depend_tx->chan == chan)	200	if (depend_tx->chan == chan)
201	s = ASYNC_TX_DIRECT_SUBMIT;	201	s = ASYNC_TX_DIRECT_SUBMIT;
202	else	202	else
203	s = ASYNC_TX_CHANNEL_SWITCH;	203	s = ASYNC_TX_CHANNEL_SWITCH;
204	}	204	}
205	spin_unlock_bh(&depend_tx->lock);	205	spin_unlock_bh(&depend_tx->lock);
206		206
207	switch (s) {	207	switch (s) {
208	case ASYNC_TX_SUBMITTED:	208	case ASYNC_TX_SUBMITTED:
209	break;	209	break;
210	case ASYNC_TX_CHANNEL_SWITCH:	210	case ASYNC_TX_CHANNEL_SWITCH:
211	async_tx_channel_switch(depend_tx, tx);	211	async_tx_channel_switch(depend_tx, tx);
212	break;	212	break;
213	case ASYNC_TX_DIRECT_SUBMIT:	213	case ASYNC_TX_DIRECT_SUBMIT:
214	tx->parent = NULL;	214	tx->parent = NULL;
215	tx->tx_submit(tx);	215	tx->tx_submit(tx);
216	break;	216	break;
217	}	217	}
218	} else {	218	} else {
219	tx->parent = NULL;	219	tx->parent = NULL;
220	tx->tx_submit(tx);	220	tx->tx_submit(tx);
221	}	221	}
222		222
223	if (flags & ASYNC_TX_ACK)	223	if (flags & ASYNC_TX_ACK)
224	async_tx_ack(tx);	224	async_tx_ack(tx);
225		225
226	if (depend_tx && (flags & ASYNC_TX_DEP_ACK))	226	if (depend_tx && (flags & ASYNC_TX_DEP_ACK))
227	async_tx_ack(depend_tx);	227	async_tx_ack(depend_tx);
228	}	228	}
229	EXPORT_SYMBOL_GPL(async_tx_submit);	229	EXPORT_SYMBOL_GPL(async_tx_submit);
230		230
231	/**	231	/**
232	* async_trigger_callback - schedules the callback function to be run after	232	* async_trigger_callback - schedules the callback function to be run after
233	* any dependent operations have been completed.	233	* any dependent operations have been completed.
234	* @flags: ASYNC_TX_ACK, ASYNC_TX_DEP_ACK	234	* @flags: ASYNC_TX_ACK, ASYNC_TX_DEP_ACK
235	* @depend_tx: 'callback' requires the completion of this transaction	235	* @depend_tx: 'callback' requires the completion of this transaction
236	* @cb_fn: function to call after depend_tx completes	236	* @cb_fn: function to call after depend_tx completes
237	* @cb_param: parameter to pass to the callback routine	237	* @cb_param: parameter to pass to the callback routine
238	*/	238	*/
239	struct dma_async_tx_descriptor *	239	struct dma_async_tx_descriptor *
240	async_trigger_callback(enum async_tx_flags flags,	240	async_trigger_callback(enum async_tx_flags flags,
241	struct dma_async_tx_descriptor *depend_tx,	241	struct dma_async_tx_descriptor *depend_tx,
242	dma_async_tx_callback cb_fn, void *cb_param)	242	dma_async_tx_callback cb_fn, void *cb_param)
243	{	243	{
244	struct dma_chan *chan;	244	struct dma_chan *chan;
245	struct dma_device *device;	245	struct dma_device *device;
246	struct dma_async_tx_descriptor *tx;	246	struct dma_async_tx_descriptor *tx;
247		247
248	if (depend_tx) {	248	if (depend_tx) {
249	chan = depend_tx->chan;	249	chan = depend_tx->chan;
250	device = chan->device;	250	device = chan->device;
251		251
252	/* see if we can schedule an interrupt	252	/* see if we can schedule an interrupt
253	* otherwise poll for completion	253	* otherwise poll for completion
254	*/	254	*/
255	if (device && !dma_has_cap(DMA_INTERRUPT, device->cap_mask))	255	if (device && !dma_has_cap(DMA_INTERRUPT, device->cap_mask))
256	device = NULL;	256	device = NULL;
257		257
258	tx = device ? device->device_prep_dma_interrupt(chan, 0) : NULL;	258	tx = device ? device->device_prep_dma_interrupt(chan, 0) : NULL;
259	} else	259	} else
260	tx = NULL;	260	tx = NULL;
261		261
262	if (tx) {	262	if (tx) {
263	pr_debug("%s: (async)\n", __func__);	263	pr_debug("%s: (async)\n", __func__);
264		264
265	async_tx_submit(chan, tx, flags, depend_tx, cb_fn, cb_param);	265	async_tx_submit(chan, tx, flags, depend_tx, cb_fn, cb_param);
266	} else {	266	} else {
267	pr_debug("%s: (sync)\n", __func__);	267	pr_debug("%s: (sync)\n", __func__);
268		268
269	/* wait for any prerequisite operations */	269	/* wait for any prerequisite operations */
270	async_tx_quiesce(&depend_tx);	270	async_tx_quiesce(&depend_tx);
271		271
272	async_tx_sync_epilog(cb_fn, cb_param);	272	async_tx_sync_epilog(cb_fn, cb_param);
273	}	273	}
274		274
275	return tx;	275	return tx;
276	}	276	}
277	EXPORT_SYMBOL_GPL(async_trigger_callback);	277	EXPORT_SYMBOL_GPL(async_trigger_callback);
278		278
279	/**	279	/**
280	* async_tx_quiesce - ensure tx is complete and freeable upon return	280	* async_tx_quiesce - ensure tx is complete and freeable upon return
281	* @tx - transaction to quiesce	281	* @tx - transaction to quiesce
282	*/	282	*/
283	void async_tx_quiesce(struct dma_async_tx_descriptor **tx)	283	void async_tx_quiesce(struct dma_async_tx_descriptor **tx)
284	{	284	{
285	if (*tx) {	285	if (*tx) {
286	/* if ack is already set then we cannot be sure	286	/* if ack is already set then we cannot be sure
287	* we are referring to the correct operation	287	* we are referring to the correct operation
288	*/	288	*/
289	BUG_ON(async_tx_test_ack(*tx));	289	BUG_ON(async_tx_test_ack(*tx));
290	if (dma_wait_for_async_tx(*tx) == DMA_ERROR)	290	if (dma_wait_for_async_tx(*tx) == DMA_ERROR)
291	panic("DMA_ERROR waiting for transaction\n");	291	panic("DMA_ERROR waiting for transaction\n");
292	async_tx_ack(*tx);	292	async_tx_ack(*tx);
293	*tx = NULL;	293	*tx = NULL;
294	}	294	}
295	}	295	}
296	EXPORT_SYMBOL_GPL(async_tx_quiesce);	296	EXPORT_SYMBOL_GPL(async_tx_quiesce);
297		297
298	module_init(async_tx_init);	298	module_init(async_tx_init);
299	module_exit(async_tx_exit);	299	module_exit(async_tx_exit);
300		300
301	MODULE_AUTHOR("Intel Corporation");	301	MODULE_AUTHOR("Intel Corporation");
302	MODULE_DESCRIPTION("Asynchronous Bulk Memory Transactions API");	302	MODULE_DESCRIPTION("Asynchronous Bulk Memory Transactions API");
303	MODULE_LICENSE("GPL");	303	MODULE_LICENSE("GPL");
304		304

drivers/dma/Kconfig

Diff comments View file @ 729b5d1

 #
 # DMA engine configuration
 #
 menuconfig DMADEVICES
 	bool "DMA Engine support"
 	depends on !HIGHMEM64G && HAS_DMA
 	help
 	  DMA engines can do asynchronous data transfers without
 	  involving the host CPU.  Currently, this framework can be
 	  used to offload memory copies in the network stack and
 	  RAID operations in the MD driver.  This menu only presents
 	  DMA Device drivers supported by the configured arch, it may
 	  be empty in some cases.
 if DMADEVICES
 comment "DMA Devices"
 config INTEL_IOATDMA
 	tristate "Intel I/OAT DMA support"
 	depends on PCI && X86
 	select DMA_ENGINE
 	select DCA
 	help
 	  Enable support for the Intel(R) I/OAT DMA engine present
 	  in recent Intel Xeon chipsets.
 	  Say Y here if you have such a chipset.
 	  If unsure, say N.
 config INTEL_IOP_ADMA
 	tristate "Intel IOP ADMA support"
 	depends on ARCH_IOP32X || ARCH_IOP33X || ARCH_IOP13XX
 	select DMA_ENGINE
 	help
 	  Enable support for the Intel(R) IOP Series RAID engines.
 config DW_DMAC
 	tristate "Synopsys DesignWare AHB DMA support"
 	depends on AVR32
 	select DMA_ENGINE
 	default y if CPU_AT32AP7000
 	help
 	  Support the Synopsys DesignWare AHB DMA controller.  This
 	  can be integrated in chips such as the Atmel AT32ap7000.
 config FSL_DMA
 	tristate "Freescale Elo and Elo Plus DMA support"
 	depends on FSL_SOC
 	select DMA_ENGINE
 	---help---
 	  Enable support for the Freescale Elo and Elo Plus DMA controllers.
 	  The Elo is the DMA controller on some 82xx and 83xx parts, and the
 	  Elo Plus is the DMA controller on 85xx and 86xx parts.
 config MV_XOR
 	bool "Marvell XOR engine support"
 	depends on PLAT_ORION
 	select DMA_ENGINE
 	---help---
 	  Enable support for the Marvell XOR engine.
 config MX3_IPU
 	bool "MX3x Image Processing Unit support"
 	depends on ARCH_MX3
 	select DMA_ENGINE
 	default y
 	help
 	  If you plan to use the Image Processing unit in the i.MX3x, say
 	  Y here. If unsure, select Y.
 config MX3_IPU_IRQS
 	int "Number of dynamically mapped interrupts for IPU"
 	depends on MX3_IPU
 	range 2 137
 	default 4
 	help
 	  Out of 137 interrupt sources on i.MX31 IPU only very few are used.
 	  To avoid bloating the irq_desc[] array we allocate a sufficient
 	  number of IRQ slots and map them dynamically to specific sources.
 config DMA_ENGINE
 	bool
 comment "DMA Clients"
 	depends on DMA_ENGINE
 config NET_DMA
 	bool "Network: TCP receive copy offload"
 	depends on DMA_ENGINE && NET
 	default (INTEL_IOATDMA || FSL_DMA)
 	help
 	  This enables the use of DMA engines in the network stack to
 	  offload receive copy-to-user operations, freeing CPU cycles.
 	  Say Y here if you enabled INTEL_IOATDMA or FSL_DMA, otherwise
 	  say N.
+config ASYNC_TX_DMA
+	bool "Async_tx: Offload support for the async_tx api"
+	depends on DMA_ENGINE
+	help
+	  This allows the async_tx api to take advantage of offload engines for
+	  memcpy, memset, xor, and raid6 p+q operations.  If your platform has
+	  a dma engine that can perform raid operations and you have enabled
+	  MD_RAID456 say Y.
+	  If unsure, say N.
 config DMATEST
 	tristate "DMA Test client"
 	depends on DMA_ENGINE
 	help
 	  Simple DMA test client. Say N unless you're debugging a
 	  DMA Device driver.
 endif

include/linux/dmaengine.h

Diff comments View file @ 729b5d1

 /*
  * Copyright(c) 2004 - 2006 Intel Corporation. All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the Free
  * Software Foundation; either version 2 of the License, or (at your option)
  * any later version.
  *
  * This program is distributed in the hope that it will be useful, but WITHOUT
  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  * more details.
  *
  * You should have received a copy of the GNU General Public License along with
  * this program; if not, write to the Free Software Foundation, Inc., 59
  * Temple Place - Suite 330, Boston, MA  02111-1307, USA.
  *
  * The full GNU General Public License is included in this distribution in the
  * file called COPYING.
  */
 #ifndef DMAENGINE_H
 #define DMAENGINE_H
 #include <linux/device.h>
 #include <linux/uio.h>
 #include <linux/dma-mapping.h>
 /**
  * typedef dma_cookie_t - an opaque DMA cookie
  *
  * if dma_cookie_t is >0 it's a DMA request cookie, <0 it's an error code
  */
 typedef s32 dma_cookie_t;
 #define dma_submit_error(cookie) ((cookie) < 0 ? 1 : 0)
 /**
  * enum dma_status - DMA transaction status
  * @DMA_SUCCESS: transaction completed successfully
  * @DMA_IN_PROGRESS: transaction not yet processed
  * @DMA_ERROR: transaction failed
  */
 enum dma_status {
 	DMA_SUCCESS,
 	DMA_IN_PROGRESS,
 	DMA_ERROR,
 };
 /**
  * enum dma_transaction_type - DMA transaction types/indexes
  */
 enum dma_transaction_type {
 	DMA_MEMCPY,
 	DMA_XOR,
 	DMA_PQ_XOR,
 	DMA_DUAL_XOR,
 	DMA_PQ_UPDATE,
 	DMA_ZERO_SUM,
 	DMA_PQ_ZERO_SUM,
 	DMA_MEMSET,
 	DMA_MEMCPY_CRC32C,
 	DMA_INTERRUPT,
 	DMA_PRIVATE,
 	DMA_SLAVE,
 };
 /* last transaction type for creation of the capabilities mask */
 #define DMA_TX_TYPE_END (DMA_SLAVE + 1)
 /**
  * enum dma_ctrl_flags - DMA flags to augment operation preparation,
  * 	control completion, and communicate status.
  * @DMA_PREP_INTERRUPT - trigger an interrupt (callback) upon completion of
  * 	this transaction
  * @DMA_CTRL_ACK - the descriptor cannot be reused until the client
  * 	acknowledges receipt, i.e. has has a chance to establish any
  * 	dependency chains
  * @DMA_COMPL_SKIP_SRC_UNMAP - set to disable dma-unmapping the source buffer(s)
  * @DMA_COMPL_SKIP_DEST_UNMAP - set to disable dma-unmapping the destination(s)
  */
 enum dma_ctrl_flags {
 	DMA_PREP_INTERRUPT = (1 << 0),
 	DMA_CTRL_ACK = (1 << 1),
 	DMA_COMPL_SKIP_SRC_UNMAP = (1 << 2),
 	DMA_COMPL_SKIP_DEST_UNMAP = (1 << 3),
 };
 /**
  * dma_cap_mask_t - capabilities bitmap modeled after cpumask_t.
  * See linux/cpumask.h
  */
 typedef struct { DECLARE_BITMAP(bits, DMA_TX_TYPE_END); } dma_cap_mask_t;
 /**
  * struct dma_chan_percpu - the per-CPU part of struct dma_chan
  * @memcpy_count: transaction counter
  * @bytes_transferred: byte counter
  */
 struct dma_chan_percpu {
 	/* stats */
 	unsigned long memcpy_count;
 	unsigned long bytes_transferred;
 };
 /**
  * struct dma_chan - devices supply DMA channels, clients use them
  * @device: ptr to the dma device who supplies this channel, always !%NULL
  * @cookie: last cookie value returned to client
  * @chan_id: channel ID for sysfs
  * @dev: class device for sysfs
  * @device_node: used to add this to the device chan list
  * @local: per-cpu pointer to a struct dma_chan_percpu
  * @client-count: how many clients are using this channel
  * @table_count: number of appearances in the mem-to-mem allocation table
  * @private: private data for certain client-channel associations
  */
 struct dma_chan {
 	struct dma_device *device;
 	dma_cookie_t cookie;
 	/* sysfs */
 	int chan_id;
 	struct dma_chan_dev *dev;
 	struct list_head device_node;
 	struct dma_chan_percpu *local;
 	int client_count;
 	int table_count;
 	void *private;
 };
 /**
  * struct dma_chan_dev - relate sysfs device node to backing channel device
  * @chan - driver channel device
  * @device - sysfs device
  * @dev_id - parent dma_device dev_id
  * @idr_ref - reference count to gate release of dma_device dev_id
  */
 struct dma_chan_dev {
 	struct dma_chan *chan;
 	struct device device;
 	int dev_id;
 	atomic_t *idr_ref;
 };
 static inline const char *dma_chan_name(struct dma_chan *chan)
 {
 	return dev_name(&chan->dev->device);
 }
 void dma_chan_cleanup(struct kref *kref);
 /**
  * typedef dma_filter_fn - callback filter for dma_request_channel
  * @chan: channel to be reviewed
  * @filter_param: opaque parameter passed through dma_request_channel
  *
  * When this optional parameter is specified in a call to dma_request_channel a
  * suitable channel is passed to this routine for further dispositioning before
  * being returned.  Where 'suitable' indicates a non-busy channel that
  * satisfies the given capability mask.  It returns 'true' to indicate that the
  * channel is suitable.
  */
 typedef bool (*dma_filter_fn)(struct dma_chan *chan, void *filter_param);
 typedef void (*dma_async_tx_callback)(void *dma_async_param);
 /**
  * struct dma_async_tx_descriptor - async transaction descriptor
  * ---dma generic offload fields---
  * @cookie: tracking cookie for this transaction, set to -EBUSY if
  *	this tx is sitting on a dependency list
  * @flags: flags to augment operation preparation, control completion, and
  * 	communicate status
  * @phys: physical address of the descriptor
  * @tx_list: driver common field for operations that require multiple
  *	descriptors
  * @chan: target channel for this operation
  * @tx_submit: set the prepared descriptor(s) to be executed by the engine
  * @callback: routine to call after this operation is complete
  * @callback_param: general parameter to pass to the callback routine
  * ---async_tx api specific fields---
  * @next: at completion submit this descriptor
  * @parent: pointer to the next level up in the dependency chain
  * @lock: protect the parent and next pointers
  */
 struct dma_async_tx_descriptor {
 	dma_cookie_t cookie;
 	enum dma_ctrl_flags flags; /* not a 'long' to pack with cookie */
 	dma_addr_t phys;
 	struct list_head tx_list;
 	struct dma_chan *chan;
 	dma_cookie_t (*tx_submit)(struct dma_async_tx_descriptor *tx);
 	dma_async_tx_callback callback;
 	void *callback_param;
 	struct dma_async_tx_descriptor *next;
 	struct dma_async_tx_descriptor *parent;
 	spinlock_t lock;
 };
 /**
  * struct dma_device - info on the entity supplying DMA services
  * @chancnt: how many DMA channels are supported
  * @channels: the list of struct dma_chan
  * @global_node: list_head for global dma_device_list
  * @cap_mask: one or more dma_capability flags
  * @max_xor: maximum number of xor sources, 0 if no capability
  * @dev_id: unique device ID
  * @dev: struct device reference for dma mapping api
  * @device_alloc_chan_resources: allocate resources and return the
  *	number of allocated descriptors
  * @device_free_chan_resources: release DMA channel's resources
  * @device_prep_dma_memcpy: prepares a memcpy operation
  * @device_prep_dma_xor: prepares a xor operation
  * @device_prep_dma_zero_sum: prepares a zero_sum operation
  * @device_prep_dma_memset: prepares a memset operation
  * @device_prep_dma_interrupt: prepares an end of chain interrupt operation
  * @device_prep_slave_sg: prepares a slave dma operation
  * @device_terminate_all: terminate all pending operations
  * @device_is_tx_complete: poll for transaction completion
  * @device_issue_pending: push pending transactions to hardware
  */
 struct dma_device {
 	unsigned int chancnt;
 	struct list_head channels;
 	struct list_head global_node;
 	dma_cap_mask_t  cap_mask;
 	int max_xor;
 	int dev_id;
 	struct device *dev;
 	int (*device_alloc_chan_resources)(struct dma_chan *chan);
 	void (*device_free_chan_resources)(struct dma_chan *chan);
 	struct dma_async_tx_descriptor *(*device_prep_dma_memcpy)(
 		struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
 		size_t len, unsigned long flags);
 	struct dma_async_tx_descriptor *(*device_prep_dma_xor)(
 		struct dma_chan *chan, dma_addr_t dest, dma_addr_t *src,
 		unsigned int src_cnt, size_t len, unsigned long flags);
 	struct dma_async_tx_descriptor *(*device_prep_dma_zero_sum)(
 		struct dma_chan *chan, dma_addr_t *src,	unsigned int src_cnt,
 		size_t len, u32 *result, unsigned long flags);
 	struct dma_async_tx_descriptor *(*device_prep_dma_memset)(
 		struct dma_chan *chan, dma_addr_t dest, int value, size_t len,
 		unsigned long flags);
 	struct dma_async_tx_descriptor *(*device_prep_dma_interrupt)(
 		struct dma_chan *chan, unsigned long flags);
 	struct dma_async_tx_descriptor *(*device_prep_slave_sg)(
 		struct dma_chan *chan, struct scatterlist *sgl,
 		unsigned int sg_len, enum dma_data_direction direction,
 		unsigned long flags);
 	void (*device_terminate_all)(struct dma_chan *chan);
 	enum dma_status (*device_is_tx_complete)(struct dma_chan *chan,
 			dma_cookie_t cookie, dma_cookie_t *last,
 			dma_cookie_t *used);
 	void (*device_issue_pending)(struct dma_chan *chan);
 };
 /* --- public DMA engine API --- */
 #ifdef CONFIG_DMA_ENGINE
 void dmaengine_get(void);
 void dmaengine_put(void);
 #else
 static inline void dmaengine_get(void)
 {
 }
 static inline void dmaengine_put(void)
 {
 }
 #endif
 #ifdef CONFIG_NET_DMA
 #define net_dmaengine_get()	dmaengine_get()
 #define net_dmaengine_put()	dmaengine_put()
 #else
 static inline void net_dmaengine_get(void)
 {
 }
 static inline void net_dmaengine_put(void)
 {
 }
 #endif
+#ifdef CONFIG_ASYNC_TX_DMA
+#define async_dmaengine_get()	dmaengine_get()
+#define async_dmaengine_put()	dmaengine_put()
+#define async_dma_find_channel(type) dma_find_channel(type)
+#else
+static inline void async_dmaengine_get(void)
+{
+}
+static inline void async_dmaengine_put(void)
+{
+}
+static inline struct dma_chan *
+async_dma_find_channel(enum dma_transaction_type type)
+{
+	return NULL;
+}
+#endif
 dma_cookie_t dma_async_memcpy_buf_to_buf(struct dma_chan *chan,
 	void *dest, void *src, size_t len);
 dma_cookie_t dma_async_memcpy_buf_to_pg(struct dma_chan *chan,
 	struct page *page, unsigned int offset, void *kdata, size_t len);
 dma_cookie_t dma_async_memcpy_pg_to_pg(struct dma_chan *chan,
 	struct page *dest_pg, unsigned int dest_off, struct page *src_pg,
 	unsigned int src_off, size_t len);
 void dma_async_tx_descriptor_init(struct dma_async_tx_descriptor *tx,
 	struct dma_chan *chan);
 static inline void async_tx_ack(struct dma_async_tx_descriptor *tx)
 {
 	tx->flags |= DMA_CTRL_ACK;
 }
 static inline void async_tx_clear_ack(struct dma_async_tx_descriptor *tx)
 {
 	tx->flags &= ~DMA_CTRL_ACK;
 }
 static inline bool async_tx_test_ack(struct dma_async_tx_descriptor *tx)
 {
 	return (tx->flags & DMA_CTRL_ACK) == DMA_CTRL_ACK;
 }
 #define first_dma_cap(mask) __first_dma_cap(&(mask))
 static inline int __first_dma_cap(const dma_cap_mask_t *srcp)
 {
 	return min_t(int, DMA_TX_TYPE_END,
 		find_first_bit(srcp->bits, DMA_TX_TYPE_END));
 }
 #define next_dma_cap(n, mask) __next_dma_cap((n), &(mask))
 static inline int __next_dma_cap(int n, const dma_cap_mask_t *srcp)
 {
 	return min_t(int, DMA_TX_TYPE_END,
 		find_next_bit(srcp->bits, DMA_TX_TYPE_END, n+1));
 }
 #define dma_cap_set(tx, mask) __dma_cap_set((tx), &(mask))
 static inline void
 __dma_cap_set(enum dma_transaction_type tx_type, dma_cap_mask_t *dstp)
 {
 	set_bit(tx_type, dstp->bits);
 }
 #define dma_cap_zero(mask) __dma_cap_zero(&(mask))
 static inline void __dma_cap_zero(dma_cap_mask_t *dstp)
 {
 	bitmap_zero(dstp->bits, DMA_TX_TYPE_END);
 }
 #define dma_has_cap(tx, mask) __dma_has_cap((tx), &(mask))
 static inline int
 __dma_has_cap(enum dma_transaction_type tx_type, dma_cap_mask_t *srcp)
 {
 	return test_bit(tx_type, srcp->bits);
 }
 #define for_each_dma_cap_mask(cap, mask) \
 	for ((cap) = first_dma_cap(mask);	\
 		(cap) < DMA_TX_TYPE_END;	\
 		(cap) = next_dma_cap((cap), (mask)))
 /**
  * dma_async_issue_pending - flush pending transactions to HW
  * @chan: target DMA channel
  *
  * This allows drivers to push copies to HW in batches,
  * reducing MMIO writes where possible.
  */
 static inline void dma_async_issue_pending(struct dma_chan *chan)
 {
 	chan->device->device_issue_pending(chan);
 }
 #define dma_async_memcpy_issue_pending(chan) dma_async_issue_pending(chan)
 /**
  * dma_async_is_tx_complete - poll for transaction completion
  * @chan: DMA channel
  * @cookie: transaction identifier to check status of
  * @last: returns last completed cookie, can be NULL
  * @used: returns last issued cookie, can be NULL
  *
  * If @last and @used are passed in, upon return they reflect the driver
  * internal state and can be used with dma_async_is_complete() to check
  * the status of multiple cookies without re-checking hardware state.
  */
 static inline enum dma_status dma_async_is_tx_complete(struct dma_chan *chan,
 	dma_cookie_t cookie, dma_cookie_t *last, dma_cookie_t *used)
 {
 	return chan->device->device_is_tx_complete(chan, cookie, last, used);
 }
 #define dma_async_memcpy_complete(chan, cookie, last, used)\
 	dma_async_is_tx_complete(chan, cookie, last, used)
 /**
  * dma_async_is_complete - test a cookie against chan state
  * @cookie: transaction identifier to test status of
  * @last_complete: last know completed transaction
  * @last_used: last cookie value handed out
  *
  * dma_async_is_complete() is used in dma_async_memcpy_complete()
  * the test logic is separated for lightweight testing of multiple cookies
  */
 static inline enum dma_status dma_async_is_complete(dma_cookie_t cookie,
 			dma_cookie_t last_complete, dma_cookie_t last_used)
 {
 	if (last_complete <= last_used) {
 		if ((cookie <= last_complete) || (cookie > last_used))
 			return DMA_SUCCESS;
 	} else {
 		if ((cookie <= last_complete) && (cookie > last_used))
 			return DMA_SUCCESS;
 	}
 	return DMA_IN_PROGRESS;
 }
 enum dma_status dma_sync_wait(struct dma_chan *chan, dma_cookie_t cookie);
 #ifdef CONFIG_DMA_ENGINE
 enum dma_status dma_wait_for_async_tx(struct dma_async_tx_descriptor *tx);
 void dma_issue_pending_all(void);
 #else
 static inline enum dma_status dma_wait_for_async_tx(struct dma_async_tx_descriptor *tx)
 {
 	return DMA_SUCCESS;
 }
 static inline void dma_issue_pending_all(void)
 {
 	do { } while (0);
 }
 #endif
 /* --- DMA device --- */
 int dma_async_device_register(struct dma_device *device);
 void dma_async_device_unregister(struct dma_device *device);
 void dma_run_dependencies(struct dma_async_tx_descriptor *tx);
 struct dma_chan *dma_find_channel(enum dma_transaction_type tx_type);
 #define dma_request_channel(mask, x, y) __dma_request_channel(&(mask), x, y)
 struct dma_chan *__dma_request_channel(dma_cap_mask_t *mask, dma_filter_fn fn, void *fn_param);
 void dma_release_channel(struct dma_chan *chan);
 /* --- Helper iov-locking functions --- */
 struct dma_page_list {
 	char __user *base_address;
 	int nr_pages;
 	struct page **pages;
 };
 struct dma_pinned_list {
 	int nr_iovecs;
 	struct dma_page_list page_list[0];
 };
 struct dma_pinned_list *dma_pin_iovec_pages(struct iovec *iov, size_t len);
 void dma_unpin_iovec_pages(struct dma_pinned_list* pinned_list);
 dma_cookie_t dma_memcpy_to_iovec(struct dma_chan *chan, struct iovec *iov,
 	struct dma_pinned_list *pinned_list, unsigned char *kdata, size_t len);
 dma_cookie_t dma_memcpy_pg_to_iovec(struct dma_chan *chan, struct iovec *iov,
 	struct dma_pinned_list *pinned_list, struct page *page,
 	unsigned int offset, size_t len);
 #endif /* DMAENGINE_H */