Doug / smarc-fsl-linux-kernel

Download as
Browse Code ยป

Commit 855372c013bbad8369223f7c75242bd3c94f9345

Authored by Cong Ding
Committed by Vinod Koul
1 parent ed30933e6f
Exists in smarc-l5.0.0_1.0.0-ga and in 5 other branches imx_3.10.17_1.0.1_ga, imx_3.10.53_1.1.0_ga, imx_3.14.28_1.0.0_ga, smarc-imx_3.10.53_1.1.0_ga, smarc-imx_3.14.28_1.0.0_ga

dma: sh/shdma-base.c: remove unnecessary null pointer check 

the variable chan is dereferenced in line 635, so it is no reason to check
null again in line 641.

Signed-off-by: Cong Ding <dinggnu@gmail.com>
Signed-off-by: Vinod Koul <vinod.koul@intel.com>

Showing 1 changed file with 0 additions and 3 deletions Inline Diff

drivers/dma/sh/shdma-base.c
Diff comments   View file @ 855372c
1 /* 1 /*
2 * Dmaengine driver base library for DMA controllers, found on SH-based SoCs 2 * Dmaengine driver base library for DMA controllers, found on SH-based SoCs
3 * 3 *
4 * extracted from shdma.c 4 * extracted from shdma.c
5 * 5 *
6 * Copyright (C) 2011-2012 Guennadi Liakhovetski <g.liakhovetski@gmx.de> 6 * Copyright (C) 2011-2012 Guennadi Liakhovetski <g.liakhovetski@gmx.de>
7 * Copyright (C) 2009 Nobuhiro Iwamatsu <iwamatsu.nobuhiro@renesas.com> 7 * Copyright (C) 2009 Nobuhiro Iwamatsu <iwamatsu.nobuhiro@renesas.com>
8 * Copyright (C) 2009 Renesas Solutions, Inc. All rights reserved. 8 * Copyright (C) 2009 Renesas Solutions, Inc. All rights reserved.
9 * Copyright (C) 2007 Freescale Semiconductor, Inc. All rights reserved. 9 * Copyright (C) 2007 Freescale Semiconductor, Inc. All rights reserved.
10 * 10 *
11 * This is free software; you can redistribute it and/or modify 11 * This is free software; you can redistribute it and/or modify
12 * it under the terms of version 2 of the GNU General Public License as 12 * it under the terms of version 2 of the GNU General Public License as
13 * published by the Free Software Foundation. 13 * published by the Free Software Foundation.
14 */ 14 */
15 15
16 #include <linux/delay.h> 16 #include <linux/delay.h>
17 #include <linux/shdma-base.h> 17 #include <linux/shdma-base.h>
18 #include <linux/dmaengine.h> 18 #include <linux/dmaengine.h>
19 #include <linux/init.h> 19 #include <linux/init.h>
20 #include <linux/interrupt.h> 20 #include <linux/interrupt.h>
21 #include <linux/module.h> 21 #include <linux/module.h>
22 #include <linux/pm_runtime.h> 22 #include <linux/pm_runtime.h>
23 #include <linux/slab.h> 23 #include <linux/slab.h>
24 #include <linux/spinlock.h> 24 #include <linux/spinlock.h>
25 25
26 #include "../dmaengine.h" 26 #include "../dmaengine.h"
27 27
28 /* DMA descriptor control */ 28 /* DMA descriptor control */
29 enum shdma_desc_status { 29 enum shdma_desc_status {
30 DESC_IDLE, 30 DESC_IDLE,
31 DESC_PREPARED, 31 DESC_PREPARED,
32 DESC_SUBMITTED, 32 DESC_SUBMITTED,
33 DESC_COMPLETED, /* completed, have to call callback */ 33 DESC_COMPLETED, /* completed, have to call callback */
34 DESC_WAITING, /* callback called, waiting for ack / re-submit */ 34 DESC_WAITING, /* callback called, waiting for ack / re-submit */
35 }; 35 };
36 36
37 #define NR_DESCS_PER_CHANNEL 32 37 #define NR_DESCS_PER_CHANNEL 32
38 38
39 #define to_shdma_chan(c) container_of(c, struct shdma_chan, dma_chan) 39 #define to_shdma_chan(c) container_of(c, struct shdma_chan, dma_chan)
40 #define to_shdma_dev(d) container_of(d, struct shdma_dev, dma_dev) 40 #define to_shdma_dev(d) container_of(d, struct shdma_dev, dma_dev)
41 41
42 /* 42 /*
43 * For slave DMA we assume, that there is a finite number of DMA slaves in the 43 * For slave DMA we assume, that there is a finite number of DMA slaves in the
44 * system, and that each such slave can only use a finite number of channels. 44 * system, and that each such slave can only use a finite number of channels.
45 * We use slave channel IDs to make sure, that no such slave channel ID is 45 * We use slave channel IDs to make sure, that no such slave channel ID is
46 * allocated more than once. 46 * allocated more than once.
47 */ 47 */
48 static unsigned int slave_num = 256; 48 static unsigned int slave_num = 256;
49 module_param(slave_num, uint, 0444); 49 module_param(slave_num, uint, 0444);
50 50
51 /* A bitmask with slave_num bits */ 51 /* A bitmask with slave_num bits */
52 static unsigned long *shdma_slave_used; 52 static unsigned long *shdma_slave_used;
53 53
54 /* Called under spin_lock_irq(&schan->chan_lock") */ 54 /* Called under spin_lock_irq(&schan->chan_lock") */
55 static void shdma_chan_xfer_ld_queue(struct shdma_chan *schan) 55 static void shdma_chan_xfer_ld_queue(struct shdma_chan *schan)
56 { 56 {
57 struct shdma_dev *sdev = to_shdma_dev(schan->dma_chan.device); 57 struct shdma_dev *sdev = to_shdma_dev(schan->dma_chan.device);
58 const struct shdma_ops *ops = sdev->ops; 58 const struct shdma_ops *ops = sdev->ops;
59 struct shdma_desc *sdesc; 59 struct shdma_desc *sdesc;
60 60
61 /* DMA work check */ 61 /* DMA work check */
62 if (ops->channel_busy(schan)) 62 if (ops->channel_busy(schan))
63 return; 63 return;
64 64
65 /* Find the first not transferred descriptor */ 65 /* Find the first not transferred descriptor */
66 list_for_each_entry(sdesc, &schan->ld_queue, node) 66 list_for_each_entry(sdesc, &schan->ld_queue, node)
67 if (sdesc->mark == DESC_SUBMITTED) { 67 if (sdesc->mark == DESC_SUBMITTED) {
68 ops->start_xfer(schan, sdesc); 68 ops->start_xfer(schan, sdesc);
69 break; 69 break;
70 } 70 }
71 } 71 }
72 72
73 static dma_cookie_t shdma_tx_submit(struct dma_async_tx_descriptor *tx) 73 static dma_cookie_t shdma_tx_submit(struct dma_async_tx_descriptor *tx)
74 { 74 {
75 struct shdma_desc *chunk, *c, *desc = 75 struct shdma_desc *chunk, *c, *desc =
76 container_of(tx, struct shdma_desc, async_tx), 76 container_of(tx, struct shdma_desc, async_tx),
77 *last = desc; 77 *last = desc;
78 struct shdma_chan *schan = to_shdma_chan(tx->chan); 78 struct shdma_chan *schan = to_shdma_chan(tx->chan);
79 dma_async_tx_callback callback = tx->callback; 79 dma_async_tx_callback callback = tx->callback;
80 dma_cookie_t cookie; 80 dma_cookie_t cookie;
81 bool power_up; 81 bool power_up;
82 82
83 spin_lock_irq(&schan->chan_lock); 83 spin_lock_irq(&schan->chan_lock);
84 84
85 power_up = list_empty(&schan->ld_queue); 85 power_up = list_empty(&schan->ld_queue);
86 86
87 cookie = dma_cookie_assign(tx); 87 cookie = dma_cookie_assign(tx);
88 88
89 /* Mark all chunks of this descriptor as submitted, move to the queue */ 89 /* Mark all chunks of this descriptor as submitted, move to the queue */
90 list_for_each_entry_safe(chunk, c, desc->node.prev, node) { 90 list_for_each_entry_safe(chunk, c, desc->node.prev, node) {
91 /* 91 /*
92 * All chunks are on the global ld_free, so, we have to find 92 * All chunks are on the global ld_free, so, we have to find
93 * the end of the chain ourselves 93 * the end of the chain ourselves
94 */ 94 */
95 if (chunk != desc && (chunk->mark == DESC_IDLE || 95 if (chunk != desc && (chunk->mark == DESC_IDLE ||
96 chunk->async_tx.cookie > 0 || 96 chunk->async_tx.cookie > 0 ||
97 chunk->async_tx.cookie == -EBUSY || 97 chunk->async_tx.cookie == -EBUSY ||
98 &chunk->node == &schan->ld_free)) 98 &chunk->node == &schan->ld_free))
99 break; 99 break;
100 chunk->mark = DESC_SUBMITTED; 100 chunk->mark = DESC_SUBMITTED;
101 /* Callback goes to the last chunk */ 101 /* Callback goes to the last chunk */
102 chunk->async_tx.callback = NULL; 102 chunk->async_tx.callback = NULL;
103 chunk->cookie = cookie; 103 chunk->cookie = cookie;
104 list_move_tail(&chunk->node, &schan->ld_queue); 104 list_move_tail(&chunk->node, &schan->ld_queue);
105 last = chunk; 105 last = chunk;
106 106
107 dev_dbg(schan->dev, "submit #%d@%p on %d\n", 107 dev_dbg(schan->dev, "submit #%d@%p on %d\n",
108 tx->cookie, &last->async_tx, schan->id); 108 tx->cookie, &last->async_tx, schan->id);
109 } 109 }
110 110
111 last->async_tx.callback = callback; 111 last->async_tx.callback = callback;
112 last->async_tx.callback_param = tx->callback_param; 112 last->async_tx.callback_param = tx->callback_param;
113 113
114 if (power_up) { 114 if (power_up) {
115 int ret; 115 int ret;
116 schan->pm_state = SHDMA_PM_BUSY; 116 schan->pm_state = SHDMA_PM_BUSY;
117 117
118 ret = pm_runtime_get(schan->dev); 118 ret = pm_runtime_get(schan->dev);
119 119
120 spin_unlock_irq(&schan->chan_lock); 120 spin_unlock_irq(&schan->chan_lock);
121 if (ret < 0) 121 if (ret < 0)
122 dev_err(schan->dev, "%s(): GET = %d\n", __func__, ret); 122 dev_err(schan->dev, "%s(): GET = %d\n", __func__, ret);
123 123
124 pm_runtime_barrier(schan->dev); 124 pm_runtime_barrier(schan->dev);
125 125
126 spin_lock_irq(&schan->chan_lock); 126 spin_lock_irq(&schan->chan_lock);
127 127
128 /* Have we been reset, while waiting? */ 128 /* Have we been reset, while waiting? */
129 if (schan->pm_state != SHDMA_PM_ESTABLISHED) { 129 if (schan->pm_state != SHDMA_PM_ESTABLISHED) {
130 struct shdma_dev *sdev = 130 struct shdma_dev *sdev =
131 to_shdma_dev(schan->dma_chan.device); 131 to_shdma_dev(schan->dma_chan.device);
132 const struct shdma_ops *ops = sdev->ops; 132 const struct shdma_ops *ops = sdev->ops;
133 dev_dbg(schan->dev, "Bring up channel %d\n", 133 dev_dbg(schan->dev, "Bring up channel %d\n",
134 schan->id); 134 schan->id);
135 /* 135 /*
136 * TODO: .xfer_setup() might fail on some platforms. 136 * TODO: .xfer_setup() might fail on some platforms.
137 * Make it int then, on error remove chunks from the 137 * Make it int then, on error remove chunks from the
138 * queue again 138 * queue again
139 */ 139 */
140 ops->setup_xfer(schan, schan->slave_id); 140 ops->setup_xfer(schan, schan->slave_id);
141 141
142 if (schan->pm_state == SHDMA_PM_PENDING) 142 if (schan->pm_state == SHDMA_PM_PENDING)
143 shdma_chan_xfer_ld_queue(schan); 143 shdma_chan_xfer_ld_queue(schan);
144 schan->pm_state = SHDMA_PM_ESTABLISHED; 144 schan->pm_state = SHDMA_PM_ESTABLISHED;
145 } 145 }
146 } else { 146 } else {
147 /* 147 /*
148 * Tell .device_issue_pending() not to run the queue, interrupts 148 * Tell .device_issue_pending() not to run the queue, interrupts
149 * will do it anyway 149 * will do it anyway
150 */ 150 */
151 schan->pm_state = SHDMA_PM_PENDING; 151 schan->pm_state = SHDMA_PM_PENDING;
152 } 152 }
153 153
154 spin_unlock_irq(&schan->chan_lock); 154 spin_unlock_irq(&schan->chan_lock);
155 155
156 return cookie; 156 return cookie;
157 } 157 }
158 158
159 /* Called with desc_lock held */ 159 /* Called with desc_lock held */
160 static struct shdma_desc *shdma_get_desc(struct shdma_chan *schan) 160 static struct shdma_desc *shdma_get_desc(struct shdma_chan *schan)
161 { 161 {
162 struct shdma_desc *sdesc; 162 struct shdma_desc *sdesc;
163 163
164 list_for_each_entry(sdesc, &schan->ld_free, node) 164 list_for_each_entry(sdesc, &schan->ld_free, node)
165 if (sdesc->mark != DESC_PREPARED) { 165 if (sdesc->mark != DESC_PREPARED) {
166 BUG_ON(sdesc->mark != DESC_IDLE); 166 BUG_ON(sdesc->mark != DESC_IDLE);
167 list_del(&sdesc->node); 167 list_del(&sdesc->node);
168 return sdesc; 168 return sdesc;
169 } 169 }
170 170
171 return NULL; 171 return NULL;
172 } 172 }
173 173
174 static int shdma_setup_slave(struct shdma_chan *schan, int slave_id) 174 static int shdma_setup_slave(struct shdma_chan *schan, int slave_id)
175 { 175 {
176 struct shdma_dev *sdev = to_shdma_dev(schan->dma_chan.device); 176 struct shdma_dev *sdev = to_shdma_dev(schan->dma_chan.device);
177 const struct shdma_ops *ops = sdev->ops; 177 const struct shdma_ops *ops = sdev->ops;
178 int ret; 178 int ret;
179 179
180 if (slave_id < 0 || slave_id >= slave_num) 180 if (slave_id < 0 || slave_id >= slave_num)
181 return -EINVAL; 181 return -EINVAL;
182 182
183 if (test_and_set_bit(slave_id, shdma_slave_used)) 183 if (test_and_set_bit(slave_id, shdma_slave_used))
184 return -EBUSY; 184 return -EBUSY;
185 185
186 ret = ops->set_slave(schan, slave_id, false); 186 ret = ops->set_slave(schan, slave_id, false);
187 if (ret < 0) { 187 if (ret < 0) {
188 clear_bit(slave_id, shdma_slave_used); 188 clear_bit(slave_id, shdma_slave_used);
189 return ret; 189 return ret;
190 } 190 }
191 191
192 schan->slave_id = slave_id; 192 schan->slave_id = slave_id;
193 193
194 return 0; 194 return 0;
195 } 195 }
196 196
197 /* 197 /*
198 * This is the standard shdma filter function to be used as a replacement to the 198 * This is the standard shdma filter function to be used as a replacement to the
199 * "old" method, using the .private pointer. If for some reason you allocate a 199 * "old" method, using the .private pointer. If for some reason you allocate a
200 * channel without slave data, use something like ERR_PTR(-EINVAL) as a filter 200 * channel without slave data, use something like ERR_PTR(-EINVAL) as a filter
201 * parameter. If this filter is used, the slave driver, after calling 201 * parameter. If this filter is used, the slave driver, after calling
202 * dma_request_channel(), will also have to call dmaengine_slave_config() with 202 * dma_request_channel(), will also have to call dmaengine_slave_config() with
203 * .slave_id, .direction, and either .src_addr or .dst_addr set. 203 * .slave_id, .direction, and either .src_addr or .dst_addr set.
204 * NOTE: this filter doesn't support multiple DMAC drivers with the DMA_SLAVE 204 * NOTE: this filter doesn't support multiple DMAC drivers with the DMA_SLAVE
205 * capability! If this becomes a requirement, hardware glue drivers, using this 205 * capability! If this becomes a requirement, hardware glue drivers, using this
206 * services would have to provide their own filters, which first would check 206 * services would have to provide their own filters, which first would check
207 * the device driver, similar to how other DMAC drivers, e.g., sa11x0-dma.c, do 207 * the device driver, similar to how other DMAC drivers, e.g., sa11x0-dma.c, do
208 * this, and only then, in case of a match, call this common filter. 208 * this, and only then, in case of a match, call this common filter.
209 */ 209 */
210 bool shdma_chan_filter(struct dma_chan *chan, void *arg) 210 bool shdma_chan_filter(struct dma_chan *chan, void *arg)
211 { 211 {
212 struct shdma_chan *schan = to_shdma_chan(chan); 212 struct shdma_chan *schan = to_shdma_chan(chan);
213 struct shdma_dev *sdev = to_shdma_dev(schan->dma_chan.device); 213 struct shdma_dev *sdev = to_shdma_dev(schan->dma_chan.device);
214 const struct shdma_ops *ops = sdev->ops; 214 const struct shdma_ops *ops = sdev->ops;
215 int slave_id = (int)arg; 215 int slave_id = (int)arg;
216 int ret; 216 int ret;
217 217
218 if (slave_id < 0) 218 if (slave_id < 0)
219 /* No slave requested - arbitrary channel */ 219 /* No slave requested - arbitrary channel */
220 return true; 220 return true;
221 221
222 if (slave_id >= slave_num) 222 if (slave_id >= slave_num)
223 return false; 223 return false;
224 224
225 ret = ops->set_slave(schan, slave_id, true); 225 ret = ops->set_slave(schan, slave_id, true);
226 if (ret < 0) 226 if (ret < 0)
227 return false; 227 return false;
228 228
229 return true; 229 return true;
230 } 230 }
231 EXPORT_SYMBOL(shdma_chan_filter); 231 EXPORT_SYMBOL(shdma_chan_filter);
232 232
233 static int shdma_alloc_chan_resources(struct dma_chan *chan) 233 static int shdma_alloc_chan_resources(struct dma_chan *chan)
234 { 234 {
235 struct shdma_chan *schan = to_shdma_chan(chan); 235 struct shdma_chan *schan = to_shdma_chan(chan);
236 struct shdma_dev *sdev = to_shdma_dev(schan->dma_chan.device); 236 struct shdma_dev *sdev = to_shdma_dev(schan->dma_chan.device);
237 const struct shdma_ops *ops = sdev->ops; 237 const struct shdma_ops *ops = sdev->ops;
238 struct shdma_desc *desc; 238 struct shdma_desc *desc;
239 struct shdma_slave *slave = chan->private; 239 struct shdma_slave *slave = chan->private;
240 int ret, i; 240 int ret, i;
241 241
242 /* 242 /*
243 * This relies on the guarantee from dmaengine that alloc_chan_resources 243 * This relies on the guarantee from dmaengine that alloc_chan_resources
244 * never runs concurrently with itself or free_chan_resources. 244 * never runs concurrently with itself or free_chan_resources.
245 */ 245 */
246 if (slave) { 246 if (slave) {
247 /* Legacy mode: .private is set in filter */ 247 /* Legacy mode: .private is set in filter */
248 ret = shdma_setup_slave(schan, slave->slave_id); 248 ret = shdma_setup_slave(schan, slave->slave_id);
249 if (ret < 0) 249 if (ret < 0)
250 goto esetslave; 250 goto esetslave;
251 } else { 251 } else {
252 schan->slave_id = -EINVAL; 252 schan->slave_id = -EINVAL;
253 } 253 }
254 254
255 schan->desc = kcalloc(NR_DESCS_PER_CHANNEL, 255 schan->desc = kcalloc(NR_DESCS_PER_CHANNEL,
256 sdev->desc_size, GFP_KERNEL); 256 sdev->desc_size, GFP_KERNEL);
257 if (!schan->desc) { 257 if (!schan->desc) {
258 ret = -ENOMEM; 258 ret = -ENOMEM;
259 goto edescalloc; 259 goto edescalloc;
260 } 260 }
261 schan->desc_num = NR_DESCS_PER_CHANNEL; 261 schan->desc_num = NR_DESCS_PER_CHANNEL;
262 262
263 for (i = 0; i < NR_DESCS_PER_CHANNEL; i++) { 263 for (i = 0; i < NR_DESCS_PER_CHANNEL; i++) {
264 desc = ops->embedded_desc(schan->desc, i); 264 desc = ops->embedded_desc(schan->desc, i);
265 dma_async_tx_descriptor_init(&desc->async_tx, 265 dma_async_tx_descriptor_init(&desc->async_tx,
266 &schan->dma_chan); 266 &schan->dma_chan);
267 desc->async_tx.tx_submit = shdma_tx_submit; 267 desc->async_tx.tx_submit = shdma_tx_submit;
268 desc->mark = DESC_IDLE; 268 desc->mark = DESC_IDLE;
269 269
270 list_add(&desc->node, &schan->ld_free); 270 list_add(&desc->node, &schan->ld_free);
271 } 271 }
272 272
273 return NR_DESCS_PER_CHANNEL; 273 return NR_DESCS_PER_CHANNEL;
274 274
275 edescalloc: 275 edescalloc:
276 if (slave) 276 if (slave)
277 esetslave: 277 esetslave:
278 clear_bit(slave->slave_id, shdma_slave_used); 278 clear_bit(slave->slave_id, shdma_slave_used);
279 chan->private = NULL; 279 chan->private = NULL;
280 return ret; 280 return ret;
281 } 281 }
282 282
283 static dma_async_tx_callback __ld_cleanup(struct shdma_chan *schan, bool all) 283 static dma_async_tx_callback __ld_cleanup(struct shdma_chan *schan, bool all)
284 { 284 {
285 struct shdma_desc *desc, *_desc; 285 struct shdma_desc *desc, *_desc;
286 /* Is the "exposed" head of a chain acked? */ 286 /* Is the "exposed" head of a chain acked? */
287 bool head_acked = false; 287 bool head_acked = false;
288 dma_cookie_t cookie = 0; 288 dma_cookie_t cookie = 0;
289 dma_async_tx_callback callback = NULL; 289 dma_async_tx_callback callback = NULL;
290 void *param = NULL; 290 void *param = NULL;
291 unsigned long flags; 291 unsigned long flags;
292 292
293 spin_lock_irqsave(&schan->chan_lock, flags); 293 spin_lock_irqsave(&schan->chan_lock, flags);
294 list_for_each_entry_safe(desc, _desc, &schan->ld_queue, node) { 294 list_for_each_entry_safe(desc, _desc, &schan->ld_queue, node) {
295 struct dma_async_tx_descriptor *tx = &desc->async_tx; 295 struct dma_async_tx_descriptor *tx = &desc->async_tx;
296 296
297 BUG_ON(tx->cookie > 0 && tx->cookie != desc->cookie); 297 BUG_ON(tx->cookie > 0 && tx->cookie != desc->cookie);
298 BUG_ON(desc->mark != DESC_SUBMITTED && 298 BUG_ON(desc->mark != DESC_SUBMITTED &&
299 desc->mark != DESC_COMPLETED && 299 desc->mark != DESC_COMPLETED &&
300 desc->mark != DESC_WAITING); 300 desc->mark != DESC_WAITING);
301 301
302 /* 302 /*
303 * queue is ordered, and we use this loop to (1) clean up all 303 * queue is ordered, and we use this loop to (1) clean up all
304 * completed descriptors, and to (2) update descriptor flags of 304 * completed descriptors, and to (2) update descriptor flags of
305 * any chunks in a (partially) completed chain 305 * any chunks in a (partially) completed chain
306 */ 306 */
307 if (!all && desc->mark == DESC_SUBMITTED && 307 if (!all && desc->mark == DESC_SUBMITTED &&
308 desc->cookie != cookie) 308 desc->cookie != cookie)
309 break; 309 break;
310 310
311 if (tx->cookie > 0) 311 if (tx->cookie > 0)
312 cookie = tx->cookie; 312 cookie = tx->cookie;
313 313
314 if (desc->mark == DESC_COMPLETED && desc->chunks == 1) { 314 if (desc->mark == DESC_COMPLETED && desc->chunks == 1) {
315 if (schan->dma_chan.completed_cookie != desc->cookie - 1) 315 if (schan->dma_chan.completed_cookie != desc->cookie - 1)
316 dev_dbg(schan->dev, 316 dev_dbg(schan->dev,
317 "Completing cookie %d, expected %d\n", 317 "Completing cookie %d, expected %d\n",
318 desc->cookie, 318 desc->cookie,
319 schan->dma_chan.completed_cookie + 1); 319 schan->dma_chan.completed_cookie + 1);
320 schan->dma_chan.completed_cookie = desc->cookie; 320 schan->dma_chan.completed_cookie = desc->cookie;
321 } 321 }
322 322
323 /* Call callback on the last chunk */ 323 /* Call callback on the last chunk */
324 if (desc->mark == DESC_COMPLETED && tx->callback) { 324 if (desc->mark == DESC_COMPLETED && tx->callback) {
325 desc->mark = DESC_WAITING; 325 desc->mark = DESC_WAITING;
326 callback = tx->callback; 326 callback = tx->callback;
327 param = tx->callback_param; 327 param = tx->callback_param;
328 dev_dbg(schan->dev, "descriptor #%d@%p on %d callback\n", 328 dev_dbg(schan->dev, "descriptor #%d@%p on %d callback\n",
329 tx->cookie, tx, schan->id); 329 tx->cookie, tx, schan->id);
330 BUG_ON(desc->chunks != 1); 330 BUG_ON(desc->chunks != 1);
331 break; 331 break;
332 } 332 }
333 333
334 if (tx->cookie > 0 || tx->cookie == -EBUSY) { 334 if (tx->cookie > 0 || tx->cookie == -EBUSY) {
335 if (desc->mark == DESC_COMPLETED) { 335 if (desc->mark == DESC_COMPLETED) {
336 BUG_ON(tx->cookie < 0); 336 BUG_ON(tx->cookie < 0);
337 desc->mark = DESC_WAITING; 337 desc->mark = DESC_WAITING;
338 } 338 }
339 head_acked = async_tx_test_ack(tx); 339 head_acked = async_tx_test_ack(tx);
340 } else { 340 } else {
341 switch (desc->mark) { 341 switch (desc->mark) {
342 case DESC_COMPLETED: 342 case DESC_COMPLETED:
343 desc->mark = DESC_WAITING; 343 desc->mark = DESC_WAITING;
344 /* Fall through */ 344 /* Fall through */
345 case DESC_WAITING: 345 case DESC_WAITING:
346 if (head_acked) 346 if (head_acked)
347 async_tx_ack(&desc->async_tx); 347 async_tx_ack(&desc->async_tx);
348 } 348 }
349 } 349 }
350 350
351 dev_dbg(schan->dev, "descriptor %p #%d completed.\n", 351 dev_dbg(schan->dev, "descriptor %p #%d completed.\n",
352 tx, tx->cookie); 352 tx, tx->cookie);
353 353
354 if (((desc->mark == DESC_COMPLETED || 354 if (((desc->mark == DESC_COMPLETED ||
355 desc->mark == DESC_WAITING) && 355 desc->mark == DESC_WAITING) &&
356 async_tx_test_ack(&desc->async_tx)) || all) { 356 async_tx_test_ack(&desc->async_tx)) || all) {
357 /* Remove from ld_queue list */ 357 /* Remove from ld_queue list */
358 desc->mark = DESC_IDLE; 358 desc->mark = DESC_IDLE;
359 359
360 list_move(&desc->node, &schan->ld_free); 360 list_move(&desc->node, &schan->ld_free);
361 361
362 if (list_empty(&schan->ld_queue)) { 362 if (list_empty(&schan->ld_queue)) {
363 dev_dbg(schan->dev, "Bring down channel %d\n", schan->id); 363 dev_dbg(schan->dev, "Bring down channel %d\n", schan->id);
364 pm_runtime_put(schan->dev); 364 pm_runtime_put(schan->dev);
365 schan->pm_state = SHDMA_PM_ESTABLISHED; 365 schan->pm_state = SHDMA_PM_ESTABLISHED;
366 } 366 }
367 } 367 }
368 } 368 }
369 369
370 if (all && !callback) 370 if (all && !callback)
371 /* 371 /*
372 * Terminating and the loop completed normally: forgive 372 * Terminating and the loop completed normally: forgive
373 * uncompleted cookies 373 * uncompleted cookies
374 */ 374 */
375 schan->dma_chan.completed_cookie = schan->dma_chan.cookie; 375 schan->dma_chan.completed_cookie = schan->dma_chan.cookie;
376 376
377 spin_unlock_irqrestore(&schan->chan_lock, flags); 377 spin_unlock_irqrestore(&schan->chan_lock, flags);
378 378
379 if (callback) 379 if (callback)
380 callback(param); 380 callback(param);
381 381
382 return callback; 382 return callback;
383 } 383 }
384 384
385 /* 385 /*
386 * shdma_chan_ld_cleanup - Clean up link descriptors 386 * shdma_chan_ld_cleanup - Clean up link descriptors
387 * 387 *
388 * Clean up the ld_queue of DMA channel. 388 * Clean up the ld_queue of DMA channel.
389 */ 389 */
390 static void shdma_chan_ld_cleanup(struct shdma_chan *schan, bool all) 390 static void shdma_chan_ld_cleanup(struct shdma_chan *schan, bool all)
391 { 391 {
392 while (__ld_cleanup(schan, all)) 392 while (__ld_cleanup(schan, all))
393 ; 393 ;
394 } 394 }
395 395
396 /* 396 /*
397 * shdma_free_chan_resources - Free all resources of the channel. 397 * shdma_free_chan_resources - Free all resources of the channel.
398 */ 398 */
399 static void shdma_free_chan_resources(struct dma_chan *chan) 399 static void shdma_free_chan_resources(struct dma_chan *chan)
400 { 400 {
401 struct shdma_chan *schan = to_shdma_chan(chan); 401 struct shdma_chan *schan = to_shdma_chan(chan);
402 struct shdma_dev *sdev = to_shdma_dev(chan->device); 402 struct shdma_dev *sdev = to_shdma_dev(chan->device);
403 const struct shdma_ops *ops = sdev->ops; 403 const struct shdma_ops *ops = sdev->ops;
404 LIST_HEAD(list); 404 LIST_HEAD(list);
405 405
406 /* Protect against ISR */ 406 /* Protect against ISR */
407 spin_lock_irq(&schan->chan_lock); 407 spin_lock_irq(&schan->chan_lock);
408 ops->halt_channel(schan); 408 ops->halt_channel(schan);
409 spin_unlock_irq(&schan->chan_lock); 409 spin_unlock_irq(&schan->chan_lock);
410 410
411 /* Now no new interrupts will occur */ 411 /* Now no new interrupts will occur */
412 412
413 /* Prepared and not submitted descriptors can still be on the queue */ 413 /* Prepared and not submitted descriptors can still be on the queue */
414 if (!list_empty(&schan->ld_queue)) 414 if (!list_empty(&schan->ld_queue))
415 shdma_chan_ld_cleanup(schan, true); 415 shdma_chan_ld_cleanup(schan, true);
416 416
417 if (schan->slave_id >= 0) { 417 if (schan->slave_id >= 0) {
418 /* The caller is holding dma_list_mutex */ 418 /* The caller is holding dma_list_mutex */
419 clear_bit(schan->slave_id, shdma_slave_used); 419 clear_bit(schan->slave_id, shdma_slave_used);
420 chan->private = NULL; 420 chan->private = NULL;
421 } 421 }
422 422
423 spin_lock_irq(&schan->chan_lock); 423 spin_lock_irq(&schan->chan_lock);
424 424
425 list_splice_init(&schan->ld_free, &list); 425 list_splice_init(&schan->ld_free, &list);
426 schan->desc_num = 0; 426 schan->desc_num = 0;
427 427
428 spin_unlock_irq(&schan->chan_lock); 428 spin_unlock_irq(&schan->chan_lock);
429 429
430 kfree(schan->desc); 430 kfree(schan->desc);
431 } 431 }
432 432
433 /** 433 /**
434 * shdma_add_desc - get, set up and return one transfer descriptor 434 * shdma_add_desc - get, set up and return one transfer descriptor
435 * @schan: DMA channel 435 * @schan: DMA channel
436 * @flags: DMA transfer flags 436 * @flags: DMA transfer flags
437 * @dst: destination DMA address, incremented when direction equals 437 * @dst: destination DMA address, incremented when direction equals
438 * DMA_DEV_TO_MEM or DMA_MEM_TO_MEM 438 * DMA_DEV_TO_MEM or DMA_MEM_TO_MEM
439 * @src: source DMA address, incremented when direction equals 439 * @src: source DMA address, incremented when direction equals
440 * DMA_MEM_TO_DEV or DMA_MEM_TO_MEM 440 * DMA_MEM_TO_DEV or DMA_MEM_TO_MEM
441 * @len: DMA transfer length 441 * @len: DMA transfer length
442 * @first: if NULL, set to the current descriptor and cookie set to -EBUSY 442 * @first: if NULL, set to the current descriptor and cookie set to -EBUSY
443 * @direction: needed for slave DMA to decide which address to keep constant, 443 * @direction: needed for slave DMA to decide which address to keep constant,
444 * equals DMA_MEM_TO_MEM for MEMCPY 444 * equals DMA_MEM_TO_MEM for MEMCPY
445 * Returns 0 or an error 445 * Returns 0 or an error
446 * Locks: called with desc_lock held 446 * Locks: called with desc_lock held
447 */ 447 */
448 static struct shdma_desc *shdma_add_desc(struct shdma_chan *schan, 448 static struct shdma_desc *shdma_add_desc(struct shdma_chan *schan,
449 unsigned long flags, dma_addr_t *dst, dma_addr_t *src, size_t *len, 449 unsigned long flags, dma_addr_t *dst, dma_addr_t *src, size_t *len,
450 struct shdma_desc **first, enum dma_transfer_direction direction) 450 struct shdma_desc **first, enum dma_transfer_direction direction)
451 { 451 {
452 struct shdma_dev *sdev = to_shdma_dev(schan->dma_chan.device); 452 struct shdma_dev *sdev = to_shdma_dev(schan->dma_chan.device);
453 const struct shdma_ops *ops = sdev->ops; 453 const struct shdma_ops *ops = sdev->ops;
454 struct shdma_desc *new; 454 struct shdma_desc *new;
455 size_t copy_size = *len; 455 size_t copy_size = *len;
456 456
457 if (!copy_size) 457 if (!copy_size)
458 return NULL; 458 return NULL;
459 459
460 /* Allocate the link descriptor from the free list */ 460 /* Allocate the link descriptor from the free list */
461 new = shdma_get_desc(schan); 461 new = shdma_get_desc(schan);
462 if (!new) { 462 if (!new) {
463 dev_err(schan->dev, "No free link descriptor available\n"); 463 dev_err(schan->dev, "No free link descriptor available\n");
464 return NULL; 464 return NULL;
465 } 465 }
466 466
467 ops->desc_setup(schan, new, *src, *dst, &copy_size); 467 ops->desc_setup(schan, new, *src, *dst, &copy_size);
468 468
469 if (!*first) { 469 if (!*first) {
470 /* First desc */ 470 /* First desc */
471 new->async_tx.cookie = -EBUSY; 471 new->async_tx.cookie = -EBUSY;
472 *first = new; 472 *first = new;
473 } else { 473 } else {
474 /* Other desc - invisible to the user */ 474 /* Other desc - invisible to the user */
475 new->async_tx.cookie = -EINVAL; 475 new->async_tx.cookie = -EINVAL;
476 } 476 }
477 477
478 dev_dbg(schan->dev, 478 dev_dbg(schan->dev,
479 "chaining (%u/%u)@%x -> %x with %p, cookie %d\n", 479 "chaining (%u/%u)@%x -> %x with %p, cookie %d\n",
480 copy_size, *len, *src, *dst, &new->async_tx, 480 copy_size, *len, *src, *dst, &new->async_tx,
481 new->async_tx.cookie); 481 new->async_tx.cookie);
482 482
483 new->mark = DESC_PREPARED; 483 new->mark = DESC_PREPARED;
484 new->async_tx.flags = flags; 484 new->async_tx.flags = flags;
485 new->direction = direction; 485 new->direction = direction;
486 new->partial = 0; 486 new->partial = 0;
487 487
488 *len -= copy_size; 488 *len -= copy_size;
489 if (direction == DMA_MEM_TO_MEM || direction == DMA_MEM_TO_DEV) 489 if (direction == DMA_MEM_TO_MEM || direction == DMA_MEM_TO_DEV)
490 *src += copy_size; 490 *src += copy_size;
491 if (direction == DMA_MEM_TO_MEM || direction == DMA_DEV_TO_MEM) 491 if (direction == DMA_MEM_TO_MEM || direction == DMA_DEV_TO_MEM)
492 *dst += copy_size; 492 *dst += copy_size;
493 493
494 return new; 494 return new;
495 } 495 }
496 496
497 /* 497 /*
498 * shdma_prep_sg - prepare transfer descriptors from an SG list 498 * shdma_prep_sg - prepare transfer descriptors from an SG list
499 * 499 *
500 * Common routine for public (MEMCPY) and slave DMA. The MEMCPY case is also 500 * Common routine for public (MEMCPY) and slave DMA. The MEMCPY case is also
501 * converted to scatter-gather to guarantee consistent locking and a correct 501 * converted to scatter-gather to guarantee consistent locking and a correct
502 * list manipulation. For slave DMA direction carries the usual meaning, and, 502 * list manipulation. For slave DMA direction carries the usual meaning, and,
503 * logically, the SG list is RAM and the addr variable contains slave address, 503 * logically, the SG list is RAM and the addr variable contains slave address,
504 * e.g., the FIFO I/O register. For MEMCPY direction equals DMA_MEM_TO_MEM 504 * e.g., the FIFO I/O register. For MEMCPY direction equals DMA_MEM_TO_MEM
505 * and the SG list contains only one element and points at the source buffer. 505 * and the SG list contains only one element and points at the source buffer.
506 */ 506 */
507 static struct dma_async_tx_descriptor *shdma_prep_sg(struct shdma_chan *schan, 507 static struct dma_async_tx_descriptor *shdma_prep_sg(struct shdma_chan *schan,
508 struct scatterlist *sgl, unsigned int sg_len, dma_addr_t *addr, 508 struct scatterlist *sgl, unsigned int sg_len, dma_addr_t *addr,
509 enum dma_transfer_direction direction, unsigned long flags) 509 enum dma_transfer_direction direction, unsigned long flags)
510 { 510 {
511 struct scatterlist *sg; 511 struct scatterlist *sg;
512 struct shdma_desc *first = NULL, *new = NULL /* compiler... */; 512 struct shdma_desc *first = NULL, *new = NULL /* compiler... */;
513 LIST_HEAD(tx_list); 513 LIST_HEAD(tx_list);
514 int chunks = 0; 514 int chunks = 0;
515 unsigned long irq_flags; 515 unsigned long irq_flags;
516 int i; 516 int i;
517 517
518 for_each_sg(sgl, sg, sg_len, i) 518 for_each_sg(sgl, sg, sg_len, i)
519 chunks += DIV_ROUND_UP(sg_dma_len(sg), schan->max_xfer_len); 519 chunks += DIV_ROUND_UP(sg_dma_len(sg), schan->max_xfer_len);
520 520
521 /* Have to lock the whole loop to protect against concurrent release */ 521 /* Have to lock the whole loop to protect against concurrent release */
522 spin_lock_irqsave(&schan->chan_lock, irq_flags); 522 spin_lock_irqsave(&schan->chan_lock, irq_flags);
523 523
524 /* 524 /*
525 * Chaining: 525 * Chaining:
526 * first descriptor is what user is dealing with in all API calls, its 526 * first descriptor is what user is dealing with in all API calls, its
527 * cookie is at first set to -EBUSY, at tx-submit to a positive 527 * cookie is at first set to -EBUSY, at tx-submit to a positive
528 * number 528 * number
529 * if more than one chunk is needed further chunks have cookie = -EINVAL 529 * if more than one chunk is needed further chunks have cookie = -EINVAL
530 * the last chunk, if not equal to the first, has cookie = -ENOSPC 530 * the last chunk, if not equal to the first, has cookie = -ENOSPC
531 * all chunks are linked onto the tx_list head with their .node heads 531 * all chunks are linked onto the tx_list head with their .node heads
532 * only during this function, then they are immediately spliced 532 * only during this function, then they are immediately spliced
533 * back onto the free list in form of a chain 533 * back onto the free list in form of a chain
534 */ 534 */
535 for_each_sg(sgl, sg, sg_len, i) { 535 for_each_sg(sgl, sg, sg_len, i) {
536 dma_addr_t sg_addr = sg_dma_address(sg); 536 dma_addr_t sg_addr = sg_dma_address(sg);
537 size_t len = sg_dma_len(sg); 537 size_t len = sg_dma_len(sg);
538 538
539 if (!len) 539 if (!len)
540 goto err_get_desc; 540 goto err_get_desc;
541 541
542 do { 542 do {
543 dev_dbg(schan->dev, "Add SG #%d@%p[%d], dma %llx\n", 543 dev_dbg(schan->dev, "Add SG #%d@%p[%d], dma %llx\n",
544 i, sg, len, (unsigned long long)sg_addr); 544 i, sg, len, (unsigned long long)sg_addr);
545 545
546 if (direction == DMA_DEV_TO_MEM) 546 if (direction == DMA_DEV_TO_MEM)
547 new = shdma_add_desc(schan, flags, 547 new = shdma_add_desc(schan, flags,
548 &sg_addr, addr, &len, &first, 548 &sg_addr, addr, &len, &first,
549 direction); 549 direction);
550 else 550 else
551 new = shdma_add_desc(schan, flags, 551 new = shdma_add_desc(schan, flags,
552 addr, &sg_addr, &len, &first, 552 addr, &sg_addr, &len, &first,
553 direction); 553 direction);
554 if (!new) 554 if (!new)
555 goto err_get_desc; 555 goto err_get_desc;
556 556
557 new->chunks = chunks--; 557 new->chunks = chunks--;
558 list_add_tail(&new->node, &tx_list); 558 list_add_tail(&new->node, &tx_list);
559 } while (len); 559 } while (len);
560 } 560 }
561 561
562 if (new != first) 562 if (new != first)
563 new->async_tx.cookie = -ENOSPC; 563 new->async_tx.cookie = -ENOSPC;
564 564
565 /* Put them back on the free list, so, they don't get lost */ 565 /* Put them back on the free list, so, they don't get lost */
566 list_splice_tail(&tx_list, &schan->ld_free); 566 list_splice_tail(&tx_list, &schan->ld_free);
567 567
568 spin_unlock_irqrestore(&schan->chan_lock, irq_flags); 568 spin_unlock_irqrestore(&schan->chan_lock, irq_flags);
569 569
570 return &first->async_tx; 570 return &first->async_tx;
571 571
572 err_get_desc: 572 err_get_desc:
573 list_for_each_entry(new, &tx_list, node) 573 list_for_each_entry(new, &tx_list, node)
574 new->mark = DESC_IDLE; 574 new->mark = DESC_IDLE;
575 list_splice(&tx_list, &schan->ld_free); 575 list_splice(&tx_list, &schan->ld_free);
576 576
577 spin_unlock_irqrestore(&schan->chan_lock, irq_flags); 577 spin_unlock_irqrestore(&schan->chan_lock, irq_flags);
578 578
579 return NULL; 579 return NULL;
580 } 580 }
581 581
582 static struct dma_async_tx_descriptor *shdma_prep_memcpy( 582 static struct dma_async_tx_descriptor *shdma_prep_memcpy(
583 struct dma_chan *chan, dma_addr_t dma_dest, dma_addr_t dma_src, 583 struct dma_chan *chan, dma_addr_t dma_dest, dma_addr_t dma_src,
584 size_t len, unsigned long flags) 584 size_t len, unsigned long flags)
585 { 585 {
586 struct shdma_chan *schan = to_shdma_chan(chan); 586 struct shdma_chan *schan = to_shdma_chan(chan);
587 struct scatterlist sg; 587 struct scatterlist sg;
588 588
589 if (!chan || !len) 589 if (!chan || !len)
590 return NULL; 590 return NULL;
591 591
592 BUG_ON(!schan->desc_num); 592 BUG_ON(!schan->desc_num);
593 593
594 sg_init_table(&sg, 1); 594 sg_init_table(&sg, 1);
595 sg_set_page(&sg, pfn_to_page(PFN_DOWN(dma_src)), len, 595 sg_set_page(&sg, pfn_to_page(PFN_DOWN(dma_src)), len,
596 offset_in_page(dma_src)); 596 offset_in_page(dma_src));
597 sg_dma_address(&sg) = dma_src; 597 sg_dma_address(&sg) = dma_src;
598 sg_dma_len(&sg) = len; 598 sg_dma_len(&sg) = len;
599 599
600 return shdma_prep_sg(schan, &sg, 1, &dma_dest, DMA_MEM_TO_MEM, flags); 600 return shdma_prep_sg(schan, &sg, 1, &dma_dest, DMA_MEM_TO_MEM, flags);
601 } 601 }
602 602
603 static struct dma_async_tx_descriptor *shdma_prep_slave_sg( 603 static struct dma_async_tx_descriptor *shdma_prep_slave_sg(
604 struct dma_chan *chan, struct scatterlist *sgl, unsigned int sg_len, 604 struct dma_chan *chan, struct scatterlist *sgl, unsigned int sg_len,
605 enum dma_transfer_direction direction, unsigned long flags, void *context) 605 enum dma_transfer_direction direction, unsigned long flags, void *context)
606 { 606 {
607 struct shdma_chan *schan = to_shdma_chan(chan); 607 struct shdma_chan *schan = to_shdma_chan(chan);
608 struct shdma_dev *sdev = to_shdma_dev(schan->dma_chan.device); 608 struct shdma_dev *sdev = to_shdma_dev(schan->dma_chan.device);
609 const struct shdma_ops *ops = sdev->ops; 609 const struct shdma_ops *ops = sdev->ops;
610 int slave_id = schan->slave_id; 610 int slave_id = schan->slave_id;
611 dma_addr_t slave_addr; 611 dma_addr_t slave_addr;
612 612
613 if (!chan) 613 if (!chan)
614 return NULL; 614 return NULL;
615 615
616 BUG_ON(!schan->desc_num); 616 BUG_ON(!schan->desc_num);
617 617
618 /* Someone calling slave DMA on a generic channel? */ 618 /* Someone calling slave DMA on a generic channel? */
619 if (slave_id < 0 || !sg_len) { 619 if (slave_id < 0 || !sg_len) {
620 dev_warn(schan->dev, "%s: bad parameter: len=%d, id=%d\n", 620 dev_warn(schan->dev, "%s: bad parameter: len=%d, id=%d\n",
621 __func__, sg_len, slave_id); 621 __func__, sg_len, slave_id);
622 return NULL; 622 return NULL;
623 } 623 }
624 624
625 slave_addr = ops->slave_addr(schan); 625 slave_addr = ops->slave_addr(schan);
626 626
627 return shdma_prep_sg(schan, sgl, sg_len, &slave_addr, 627 return shdma_prep_sg(schan, sgl, sg_len, &slave_addr,
628 direction, flags); 628 direction, flags);
629 } 629 }
630 630
631 static int shdma_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd, 631 static int shdma_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd,
632 unsigned long arg) 632 unsigned long arg)
633 { 633 {
634 struct shdma_chan *schan = to_shdma_chan(chan); 634 struct shdma_chan *schan = to_shdma_chan(chan);
635 struct shdma_dev *sdev = to_shdma_dev(chan->device); 635 struct shdma_dev *sdev = to_shdma_dev(chan->device);
636 const struct shdma_ops *ops = sdev->ops; 636 const struct shdma_ops *ops = sdev->ops;
637 struct dma_slave_config *config; 637 struct dma_slave_config *config;
638 unsigned long flags; 638 unsigned long flags;
639 int ret; 639 int ret;
640 640
641 if (!chan)
642 return -EINVAL;
643
644 switch (cmd) { 641 switch (cmd) {
645 case DMA_TERMINATE_ALL: 642 case DMA_TERMINATE_ALL:
646 spin_lock_irqsave(&schan->chan_lock, flags); 643 spin_lock_irqsave(&schan->chan_lock, flags);
647 ops->halt_channel(schan); 644 ops->halt_channel(schan);
648 645
649 if (ops->get_partial && !list_empty(&schan->ld_queue)) { 646 if (ops->get_partial && !list_empty(&schan->ld_queue)) {
650 /* Record partial transfer */ 647 /* Record partial transfer */
651 struct shdma_desc *desc = list_first_entry(&schan->ld_queue, 648 struct shdma_desc *desc = list_first_entry(&schan->ld_queue,
652 struct shdma_desc, node); 649 struct shdma_desc, node);
653 desc->partial = ops->get_partial(schan, desc); 650 desc->partial = ops->get_partial(schan, desc);
654 } 651 }
655 652
656 spin_unlock_irqrestore(&schan->chan_lock, flags); 653 spin_unlock_irqrestore(&schan->chan_lock, flags);
657 654
658 shdma_chan_ld_cleanup(schan, true); 655 shdma_chan_ld_cleanup(schan, true);
659 break; 656 break;
660 case DMA_SLAVE_CONFIG: 657 case DMA_SLAVE_CONFIG:
661 /* 658 /*
662 * So far only .slave_id is used, but the slave drivers are 659 * So far only .slave_id is used, but the slave drivers are
663 * encouraged to also set a transfer direction and an address. 660 * encouraged to also set a transfer direction and an address.
664 */ 661 */
665 if (!arg) 662 if (!arg)
666 return -EINVAL; 663 return -EINVAL;
667 /* 664 /*
668 * We could lock this, but you shouldn't be configuring the 665 * We could lock this, but you shouldn't be configuring the
669 * channel, while using it... 666 * channel, while using it...
670 */ 667 */
671 config = (struct dma_slave_config *)arg; 668 config = (struct dma_slave_config *)arg;
672 ret = shdma_setup_slave(schan, config->slave_id); 669 ret = shdma_setup_slave(schan, config->slave_id);
673 if (ret < 0) 670 if (ret < 0)
674 return ret; 671 return ret;
675 break; 672 break;
676 default: 673 default:
677 return -ENXIO; 674 return -ENXIO;
678 } 675 }
679 676
680 return 0; 677 return 0;
681 } 678 }
682 679
683 static void shdma_issue_pending(struct dma_chan *chan) 680 static void shdma_issue_pending(struct dma_chan *chan)
684 { 681 {
685 struct shdma_chan *schan = to_shdma_chan(chan); 682 struct shdma_chan *schan = to_shdma_chan(chan);
686 683
687 spin_lock_irq(&schan->chan_lock); 684 spin_lock_irq(&schan->chan_lock);
688 if (schan->pm_state == SHDMA_PM_ESTABLISHED) 685 if (schan->pm_state == SHDMA_PM_ESTABLISHED)
689 shdma_chan_xfer_ld_queue(schan); 686 shdma_chan_xfer_ld_queue(schan);
690 else 687 else
691 schan->pm_state = SHDMA_PM_PENDING; 688 schan->pm_state = SHDMA_PM_PENDING;
692 spin_unlock_irq(&schan->chan_lock); 689 spin_unlock_irq(&schan->chan_lock);
693 } 690 }
694 691
695 static enum dma_status shdma_tx_status(struct dma_chan *chan, 692 static enum dma_status shdma_tx_status(struct dma_chan *chan,
696 dma_cookie_t cookie, 693 dma_cookie_t cookie,
697 struct dma_tx_state *txstate) 694 struct dma_tx_state *txstate)
698 { 695 {
699 struct shdma_chan *schan = to_shdma_chan(chan); 696 struct shdma_chan *schan = to_shdma_chan(chan);
700 enum dma_status status; 697 enum dma_status status;
701 unsigned long flags; 698 unsigned long flags;
702 699
703 shdma_chan_ld_cleanup(schan, false); 700 shdma_chan_ld_cleanup(schan, false);
704 701
705 spin_lock_irqsave(&schan->chan_lock, flags); 702 spin_lock_irqsave(&schan->chan_lock, flags);
706 703
707 status = dma_cookie_status(chan, cookie, txstate); 704 status = dma_cookie_status(chan, cookie, txstate);
708 705
709 /* 706 /*
710 * If we don't find cookie on the queue, it has been aborted and we have 707 * If we don't find cookie on the queue, it has been aborted and we have
711 * to report error 708 * to report error
712 */ 709 */
713 if (status != DMA_SUCCESS) { 710 if (status != DMA_SUCCESS) {
714 struct shdma_desc *sdesc; 711 struct shdma_desc *sdesc;
715 status = DMA_ERROR; 712 status = DMA_ERROR;
716 list_for_each_entry(sdesc, &schan->ld_queue, node) 713 list_for_each_entry(sdesc, &schan->ld_queue, node)
717 if (sdesc->cookie == cookie) { 714 if (sdesc->cookie == cookie) {
718 status = DMA_IN_PROGRESS; 715 status = DMA_IN_PROGRESS;
719 break; 716 break;
720 } 717 }
721 } 718 }
722 719
723 spin_unlock_irqrestore(&schan->chan_lock, flags); 720 spin_unlock_irqrestore(&schan->chan_lock, flags);
724 721
725 return status; 722 return status;
726 } 723 }
727 724
728 /* Called from error IRQ or NMI */ 725 /* Called from error IRQ or NMI */
729 bool shdma_reset(struct shdma_dev *sdev) 726 bool shdma_reset(struct shdma_dev *sdev)
730 { 727 {
731 const struct shdma_ops *ops = sdev->ops; 728 const struct shdma_ops *ops = sdev->ops;
732 struct shdma_chan *schan; 729 struct shdma_chan *schan;
733 unsigned int handled = 0; 730 unsigned int handled = 0;
734 int i; 731 int i;
735 732
736 /* Reset all channels */ 733 /* Reset all channels */
737 shdma_for_each_chan(schan, sdev, i) { 734 shdma_for_each_chan(schan, sdev, i) {
738 struct shdma_desc *sdesc; 735 struct shdma_desc *sdesc;
739 LIST_HEAD(dl); 736 LIST_HEAD(dl);
740 737
741 if (!schan) 738 if (!schan)
742 continue; 739 continue;
743 740
744 spin_lock(&schan->chan_lock); 741 spin_lock(&schan->chan_lock);
745 742
746 /* Stop the channel */ 743 /* Stop the channel */
747 ops->halt_channel(schan); 744 ops->halt_channel(schan);
748 745
749 list_splice_init(&schan->ld_queue, &dl); 746 list_splice_init(&schan->ld_queue, &dl);
750 747
751 if (!list_empty(&dl)) { 748 if (!list_empty(&dl)) {
752 dev_dbg(schan->dev, "Bring down channel %d\n", schan->id); 749 dev_dbg(schan->dev, "Bring down channel %d\n", schan->id);
753 pm_runtime_put(schan->dev); 750 pm_runtime_put(schan->dev);
754 } 751 }
755 schan->pm_state = SHDMA_PM_ESTABLISHED; 752 schan->pm_state = SHDMA_PM_ESTABLISHED;
756 753
757 spin_unlock(&schan->chan_lock); 754 spin_unlock(&schan->chan_lock);
758 755
759 /* Complete all */ 756 /* Complete all */
760 list_for_each_entry(sdesc, &dl, node) { 757 list_for_each_entry(sdesc, &dl, node) {
761 struct dma_async_tx_descriptor *tx = &sdesc->async_tx; 758 struct dma_async_tx_descriptor *tx = &sdesc->async_tx;
762 sdesc->mark = DESC_IDLE; 759 sdesc->mark = DESC_IDLE;
763 if (tx->callback) 760 if (tx->callback)
764 tx->callback(tx->callback_param); 761 tx->callback(tx->callback_param);
765 } 762 }
766 763
767 spin_lock(&schan->chan_lock); 764 spin_lock(&schan->chan_lock);
768 list_splice(&dl, &schan->ld_free); 765 list_splice(&dl, &schan->ld_free);
769 spin_unlock(&schan->chan_lock); 766 spin_unlock(&schan->chan_lock);
770 767
771 handled++; 768 handled++;
772 } 769 }
773 770
774 return !!handled; 771 return !!handled;
775 } 772 }
776 EXPORT_SYMBOL(shdma_reset); 773 EXPORT_SYMBOL(shdma_reset);
777 774
778 static irqreturn_t chan_irq(int irq, void *dev) 775 static irqreturn_t chan_irq(int irq, void *dev)
779 { 776 {
780 struct shdma_chan *schan = dev; 777 struct shdma_chan *schan = dev;
781 const struct shdma_ops *ops = 778 const struct shdma_ops *ops =
782 to_shdma_dev(schan->dma_chan.device)->ops; 779 to_shdma_dev(schan->dma_chan.device)->ops;
783 irqreturn_t ret; 780 irqreturn_t ret;
784 781
785 spin_lock(&schan->chan_lock); 782 spin_lock(&schan->chan_lock);
786 783
787 ret = ops->chan_irq(schan, irq) ? IRQ_WAKE_THREAD : IRQ_NONE; 784 ret = ops->chan_irq(schan, irq) ? IRQ_WAKE_THREAD : IRQ_NONE;
788 785
789 spin_unlock(&schan->chan_lock); 786 spin_unlock(&schan->chan_lock);
790 787
791 return ret; 788 return ret;
792 } 789 }
793 790
794 static irqreturn_t chan_irqt(int irq, void *dev) 791 static irqreturn_t chan_irqt(int irq, void *dev)
795 { 792 {
796 struct shdma_chan *schan = dev; 793 struct shdma_chan *schan = dev;
797 const struct shdma_ops *ops = 794 const struct shdma_ops *ops =
798 to_shdma_dev(schan->dma_chan.device)->ops; 795 to_shdma_dev(schan->dma_chan.device)->ops;
799 struct shdma_desc *sdesc; 796 struct shdma_desc *sdesc;
800 797
801 spin_lock_irq(&schan->chan_lock); 798 spin_lock_irq(&schan->chan_lock);
802 list_for_each_entry(sdesc, &schan->ld_queue, node) { 799 list_for_each_entry(sdesc, &schan->ld_queue, node) {
803 if (sdesc->mark == DESC_SUBMITTED && 800 if (sdesc->mark == DESC_SUBMITTED &&
804 ops->desc_completed(schan, sdesc)) { 801 ops->desc_completed(schan, sdesc)) {
805 dev_dbg(schan->dev, "done #%d@%p\n", 802 dev_dbg(schan->dev, "done #%d@%p\n",
806 sdesc->async_tx.cookie, &sdesc->async_tx); 803 sdesc->async_tx.cookie, &sdesc->async_tx);
807 sdesc->mark = DESC_COMPLETED; 804 sdesc->mark = DESC_COMPLETED;
808 break; 805 break;
809 } 806 }
810 } 807 }
811 /* Next desc */ 808 /* Next desc */
812 shdma_chan_xfer_ld_queue(schan); 809 shdma_chan_xfer_ld_queue(schan);
813 spin_unlock_irq(&schan->chan_lock); 810 spin_unlock_irq(&schan->chan_lock);
814 811
815 shdma_chan_ld_cleanup(schan, false); 812 shdma_chan_ld_cleanup(schan, false);
816 813
817 return IRQ_HANDLED; 814 return IRQ_HANDLED;
818 } 815 }
819 816
820 int shdma_request_irq(struct shdma_chan *schan, int irq, 817 int shdma_request_irq(struct shdma_chan *schan, int irq,
821 unsigned long flags, const char *name) 818 unsigned long flags, const char *name)
822 { 819 {
823 int ret = request_threaded_irq(irq, chan_irq, chan_irqt, 820 int ret = request_threaded_irq(irq, chan_irq, chan_irqt,
824 flags, name, schan); 821 flags, name, schan);
825 822
826 schan->irq = ret < 0 ? ret : irq; 823 schan->irq = ret < 0 ? ret : irq;
827 824
828 return ret; 825 return ret;
829 } 826 }
830 EXPORT_SYMBOL(shdma_request_irq); 827 EXPORT_SYMBOL(shdma_request_irq);
831 828
832 void shdma_free_irq(struct shdma_chan *schan) 829 void shdma_free_irq(struct shdma_chan *schan)
833 { 830 {
834 if (schan->irq >= 0) 831 if (schan->irq >= 0)
835 free_irq(schan->irq, schan); 832 free_irq(schan->irq, schan);
836 } 833 }
837 EXPORT_SYMBOL(shdma_free_irq); 834 EXPORT_SYMBOL(shdma_free_irq);
838 835
839 void shdma_chan_probe(struct shdma_dev *sdev, 836 void shdma_chan_probe(struct shdma_dev *sdev,
840 struct shdma_chan *schan, int id) 837 struct shdma_chan *schan, int id)
841 { 838 {
842 schan->pm_state = SHDMA_PM_ESTABLISHED; 839 schan->pm_state = SHDMA_PM_ESTABLISHED;
843 840
844 /* reference struct dma_device */ 841 /* reference struct dma_device */
845 schan->dma_chan.device = &sdev->dma_dev; 842 schan->dma_chan.device = &sdev->dma_dev;
846 dma_cookie_init(&schan->dma_chan); 843 dma_cookie_init(&schan->dma_chan);
847 844
848 schan->dev = sdev->dma_dev.dev; 845 schan->dev = sdev->dma_dev.dev;
849 schan->id = id; 846 schan->id = id;
850 847
851 if (!schan->max_xfer_len) 848 if (!schan->max_xfer_len)
852 schan->max_xfer_len = PAGE_SIZE; 849 schan->max_xfer_len = PAGE_SIZE;
853 850
854 spin_lock_init(&schan->chan_lock); 851 spin_lock_init(&schan->chan_lock);
855 852
856 /* Init descripter manage list */ 853 /* Init descripter manage list */
857 INIT_LIST_HEAD(&schan->ld_queue); 854 INIT_LIST_HEAD(&schan->ld_queue);
858 INIT_LIST_HEAD(&schan->ld_free); 855 INIT_LIST_HEAD(&schan->ld_free);
859 856
860 /* Add the channel to DMA device channel list */ 857 /* Add the channel to DMA device channel list */
861 list_add_tail(&schan->dma_chan.device_node, 858 list_add_tail(&schan->dma_chan.device_node,
862 &sdev->dma_dev.channels); 859 &sdev->dma_dev.channels);
863 sdev->schan[sdev->dma_dev.chancnt++] = schan; 860 sdev->schan[sdev->dma_dev.chancnt++] = schan;
864 } 861 }
865 EXPORT_SYMBOL(shdma_chan_probe); 862 EXPORT_SYMBOL(shdma_chan_probe);
866 863
867 void shdma_chan_remove(struct shdma_chan *schan) 864 void shdma_chan_remove(struct shdma_chan *schan)
868 { 865 {
869 list_del(&schan->dma_chan.device_node); 866 list_del(&schan->dma_chan.device_node);
870 } 867 }
871 EXPORT_SYMBOL(shdma_chan_remove); 868 EXPORT_SYMBOL(shdma_chan_remove);
872 869
873 int shdma_init(struct device *dev, struct shdma_dev *sdev, 870 int shdma_init(struct device *dev, struct shdma_dev *sdev,
874 int chan_num) 871 int chan_num)
875 { 872 {
876 struct dma_device *dma_dev = &sdev->dma_dev; 873 struct dma_device *dma_dev = &sdev->dma_dev;
877 874
878 /* 875 /*
879 * Require all call-backs for now, they can trivially be made optional 876 * Require all call-backs for now, they can trivially be made optional
880 * later as required 877 * later as required
881 */ 878 */
882 if (!sdev->ops || 879 if (!sdev->ops ||
883 !sdev->desc_size || 880 !sdev->desc_size ||
884 !sdev->ops->embedded_desc || 881 !sdev->ops->embedded_desc ||
885 !sdev->ops->start_xfer || 882 !sdev->ops->start_xfer ||
886 !sdev->ops->setup_xfer || 883 !sdev->ops->setup_xfer ||
887 !sdev->ops->set_slave || 884 !sdev->ops->set_slave ||
888 !sdev->ops->desc_setup || 885 !sdev->ops->desc_setup ||
889 !sdev->ops->slave_addr || 886 !sdev->ops->slave_addr ||
890 !sdev->ops->channel_busy || 887 !sdev->ops->channel_busy ||
891 !sdev->ops->halt_channel || 888 !sdev->ops->halt_channel ||
892 !sdev->ops->desc_completed) 889 !sdev->ops->desc_completed)
893 return -EINVAL; 890 return -EINVAL;
894 891
895 sdev->schan = kcalloc(chan_num, sizeof(*sdev->schan), GFP_KERNEL); 892 sdev->schan = kcalloc(chan_num, sizeof(*sdev->schan), GFP_KERNEL);
896 if (!sdev->schan) 893 if (!sdev->schan)
897 return -ENOMEM; 894 return -ENOMEM;
898 895
899 INIT_LIST_HEAD(&dma_dev->channels); 896 INIT_LIST_HEAD(&dma_dev->channels);
900 897
901 /* Common and MEMCPY operations */ 898 /* Common and MEMCPY operations */
902 dma_dev->device_alloc_chan_resources 899 dma_dev->device_alloc_chan_resources
903 = shdma_alloc_chan_resources; 900 = shdma_alloc_chan_resources;
904 dma_dev->device_free_chan_resources = shdma_free_chan_resources; 901 dma_dev->device_free_chan_resources = shdma_free_chan_resources;
905 dma_dev->device_prep_dma_memcpy = shdma_prep_memcpy; 902 dma_dev->device_prep_dma_memcpy = shdma_prep_memcpy;
906 dma_dev->device_tx_status = shdma_tx_status; 903 dma_dev->device_tx_status = shdma_tx_status;
907 dma_dev->device_issue_pending = shdma_issue_pending; 904 dma_dev->device_issue_pending = shdma_issue_pending;
908 905
909 /* Compulsory for DMA_SLAVE fields */ 906 /* Compulsory for DMA_SLAVE fields */
910 dma_dev->device_prep_slave_sg = shdma_prep_slave_sg; 907 dma_dev->device_prep_slave_sg = shdma_prep_slave_sg;
911 dma_dev->device_control = shdma_control; 908 dma_dev->device_control = shdma_control;
912 909
913 dma_dev->dev = dev; 910 dma_dev->dev = dev;
914 911
915 return 0; 912 return 0;
916 } 913 }
917 EXPORT_SYMBOL(shdma_init); 914 EXPORT_SYMBOL(shdma_init);
918 915
919 void shdma_cleanup(struct shdma_dev *sdev) 916 void shdma_cleanup(struct shdma_dev *sdev)
920 { 917 {
921 kfree(sdev->schan); 918 kfree(sdev->schan);
922 } 919 }
923 EXPORT_SYMBOL(shdma_cleanup); 920 EXPORT_SYMBOL(shdma_cleanup);
924 921
925 static int __init shdma_enter(void) 922 static int __init shdma_enter(void)
926 { 923 {
927 shdma_slave_used = kzalloc(DIV_ROUND_UP(slave_num, BITS_PER_LONG) * 924 shdma_slave_used = kzalloc(DIV_ROUND_UP(slave_num, BITS_PER_LONG) *
928 sizeof(long), GFP_KERNEL); 925 sizeof(long), GFP_KERNEL);
929 if (!shdma_slave_used) 926 if (!shdma_slave_used)
930 return -ENOMEM; 927 return -ENOMEM;
931 return 0; 928 return 0;
932 } 929 }
933 module_init(shdma_enter); 930 module_init(shdma_enter);
934 931
935 static void __exit shdma_exit(void) 932 static void __exit shdma_exit(void)
936 { 933 {
937 kfree(shdma_slave_used); 934 kfree(shdma_slave_used);
938 } 935 }
939 module_exit(shdma_exit); 936 module_exit(shdma_exit);
940 937
941 MODULE_LICENSE("GPL v2"); 938 MODULE_LICENSE("GPL v2");
942 MODULE_DESCRIPTION("SH-DMA driver base library"); 939 MODULE_DESCRIPTION("SH-DMA driver base library");
943 MODULE_AUTHOR("Guennadi Liakhovetski <g.liakhovetski@gmx.de>"); 940 MODULE_AUTHOR("Guennadi Liakhovetski <g.liakhovetski@gmx.de>");
944 941