Doug / smarc-fsl-linux-kernel

Download as
Browse Code ยป

Commit 05a625486efc3209ae4d98e253dafa6ce0124385

Authored by Linus Torvalds
2 parents 1f11abc966 f80ca163d6
Exists in master and in 7 other branches imx_3.0.35_4.1.0, imx_3.10.17_1.0.1_ga, imx_3.10.53_1.1.0_ga, imx_3.14.28_1.0.0_ga, smarc-imx_3.10.53_1.1.0_ga, smarc-imx_3.14.28_1.0.0_ga, smarc-l5.0.0_1.0.0-ga

Merge branch 'fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/djbw/async_tx 

* 'fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/djbw/async_tx:
  drivers/dma: Correct use after free
  drivers/dma: drop unnecesary memset
  ioat2,3: put channel hardware in known state at init
  async_tx: expand async raid6 test to cover ioatdma corner case
  ioat3: fix p-disabled q-continuation
  sh: fix DMA driver's descriptor chaining and cookie assignment
  dma: at_hdmac: correct incompatible type for argument 1 of 'spin_lock_bh'

Showing 12 changed files Side-by-side Diff

crypto/async_tx/raid6test.c
Diff comments   View file @ 05a6254
... ... @@ -214,6 +214,13 @@
214 214 err += test(4, &tests);
215 215 if (NDISKS > 5)
216 216 err += test(5, &tests);
  217 + /* the 11 and 12 disk cases are special for ioatdma (p-disabled
  218 + * q-continuation without extended descriptor)
  219 + */
  220 + if (NDISKS > 12) {
  221 + err += test(11, &tests);
  222 + err += test(12, &tests);
  223 + }
217 224 err += test(NDISKS, &tests);
218 225  
219 226 pr("\n");
drivers/dma/at_hdmac.c
Diff comments   View file @ 05a6254
... ... @@ -815,7 +815,7 @@
815 815 dev_vdbg(chan2dev(chan), "is_tx_complete: %d (d%d, u%d)\n",
816 816 cookie, done ? *done : 0, used ? *used : 0);
817 817  
818   - spin_lock_bh(atchan->lock);
  818 + spin_lock_bh(&atchan->lock);
819 819  
820 820 last_complete = atchan->completed_cookie;
821 821 last_used = chan->cookie;
... ... @@ -830,7 +830,7 @@
830 830 ret = dma_async_is_complete(cookie, last_complete, last_used);
831 831 }
832 832  
833   - spin_unlock_bh(atchan->lock);
  833 + spin_unlock_bh(&atchan->lock);
834 834  
835 835 if (done)
836 836 *done = last_complete;
drivers/dma/coh901318.c
Diff comments   View file @ 05a6254
... ... @@ -1294,8 +1294,8 @@
1294 1294 dma_async_device_unregister(&base->dma_slave);
1295 1295 coh901318_pool_destroy(&base->pool);
1296 1296 free_irq(platform_get_irq(pdev, 0), base);
1297   - kfree(base);
1298 1297 iounmap(base->virtbase);
  1298 + kfree(base);
1299 1299 release_mem_region(pdev->resource->start,
1300 1300 resource_size(pdev->resource));
1301 1301 return 0;
drivers/dma/dw_dmac.c
Diff comments   View file @ 05a6254
... ... @@ -1270,8 +1270,6 @@
1270 1270 goto err_kfree;
1271 1271 }
1272 1272  
1273   - memset(dw, 0, sizeof *dw);
1274   -
1275 1273 dw->regs = ioremap(io->start, DW_REGLEN);
1276 1274 if (!dw->regs) {
1277 1275 err = -ENOMEM;
drivers/dma/ioat/dma.c
Diff comments   View file @ 05a6254
... ... @@ -1032,7 +1032,7 @@
1032 1032 dma->dev = &pdev->dev;
1033 1033  
1034 1034 if (!dma->chancnt) {
1035   - dev_err(dev, "zero channels detected\n");
  1035 + dev_err(dev, "channel enumeration error\n");
1036 1036 goto err_setup_interrupts;
1037 1037 }
1038 1038  
drivers/dma/ioat/dma.h
Diff comments   View file @ 05a6254
... ... @@ -60,6 +60,7 @@
60 60 * @dca: direct cache access context
61 61 * @intr_quirk: interrupt setup quirk (for ioat_v1 devices)
62 62 * @enumerate_channels: hw version specific channel enumeration
  63 + * @reset_hw: hw version specific channel (re)initialization
63 64 * @cleanup_tasklet: select between the v2 and v3 cleanup routines
64 65 * @timer_fn: select between the v2 and v3 timer watchdog routines
65 66 * @self_test: hardware version specific self test for each supported op type
... ... @@ -78,6 +79,7 @@
78 79 struct dca_provider *dca;
79 80 void (*intr_quirk)(struct ioatdma_device *device);
80 81 int (*enumerate_channels)(struct ioatdma_device *device);
  82 + int (*reset_hw)(struct ioat_chan_common *chan);
81 83 void (*cleanup_tasklet)(unsigned long data);
82 84 void (*timer_fn)(unsigned long data);
83 85 int (*self_test)(struct ioatdma_device *device);
... ... @@ -262,6 +264,22 @@
262 264 u8 ver = chan->device->version;
263 265  
264 266 writeb(IOAT_CHANCMD_SUSPEND, chan->reg_base + IOAT_CHANCMD_OFFSET(ver));
  267 +}
  268 +
  269 +static inline void ioat_reset(struct ioat_chan_common *chan)
  270 +{
  271 + u8 ver = chan->device->version;
  272 +
  273 + writeb(IOAT_CHANCMD_RESET, chan->reg_base + IOAT_CHANCMD_OFFSET(ver));
  274 +}
  275 +
  276 +static inline bool ioat_reset_pending(struct ioat_chan_common *chan)
  277 +{
  278 + u8 ver = chan->device->version;
  279 + u8 cmd;
  280 +
  281 + cmd = readb(chan->reg_base + IOAT_CHANCMD_OFFSET(ver));
  282 + return (cmd & IOAT_CHANCMD_RESET) == IOAT_CHANCMD_RESET;
265 283 }
266 284  
267 285 static inline void ioat_set_chainaddr(struct ioat_dma_chan *ioat, u64 addr)
drivers/dma/ioat/dma_v2.c
Diff comments   View file @ 05a6254
... ... @@ -239,20 +239,50 @@
239 239 __ioat2_start_null_desc(ioat);
240 240 }
241 241  
242   -static void ioat2_restart_channel(struct ioat2_dma_chan *ioat)
  242 +int ioat2_quiesce(struct ioat_chan_common *chan, unsigned long tmo)
243 243 {
244   - struct ioat_chan_common *chan = &ioat->base;
245   - unsigned long phys_complete;
  244 + unsigned long end = jiffies + tmo;
  245 + int err = 0;
246 246 u32 status;
247 247  
248 248 status = ioat_chansts(chan);
249 249 if (is_ioat_active(status) || is_ioat_idle(status))
250 250 ioat_suspend(chan);
251 251 while (is_ioat_active(status) || is_ioat_idle(status)) {
  252 + if (end && time_after(jiffies, end)) {
  253 + err = -ETIMEDOUT;
  254 + break;
  255 + }
252 256 status = ioat_chansts(chan);
253 257 cpu_relax();
254 258 }
255 259  
  260 + return err;
  261 +}
  262 +
  263 +int ioat2_reset_sync(struct ioat_chan_common *chan, unsigned long tmo)
  264 +{
  265 + unsigned long end = jiffies + tmo;
  266 + int err = 0;
  267 +
  268 + ioat_reset(chan);
  269 + while (ioat_reset_pending(chan)) {
  270 + if (end && time_after(jiffies, end)) {
  271 + err = -ETIMEDOUT;
  272 + break;
  273 + }
  274 + cpu_relax();
  275 + }
  276 +
  277 + return err;
  278 +}
  279 +
  280 +static void ioat2_restart_channel(struct ioat2_dma_chan *ioat)
  281 +{
  282 + struct ioat_chan_common *chan = &ioat->base;
  283 + unsigned long phys_complete;
  284 +
  285 + ioat2_quiesce(chan, 0);
256 286 if (ioat_cleanup_preamble(chan, &phys_complete))
257 287 __cleanup(ioat, phys_complete);
258 288  
... ... @@ -318,6 +348,19 @@
318 348 spin_unlock_bh(&chan->cleanup_lock);
319 349 }
320 350  
  351 +static int ioat2_reset_hw(struct ioat_chan_common *chan)
  352 +{
  353 + /* throw away whatever the channel was doing and get it initialized */
  354 + u32 chanerr;
  355 +
  356 + ioat2_quiesce(chan, msecs_to_jiffies(100));
  357 +
  358 + chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET);
  359 + writel(chanerr, chan->reg_base + IOAT_CHANERR_OFFSET);
  360 +
  361 + return ioat2_reset_sync(chan, msecs_to_jiffies(200));
  362 +}
  363 +
321 364 /**
322 365 * ioat2_enumerate_channels - find and initialize the device's channels
323 366 * @device: the device to be enumerated
... ... @@ -360,6 +403,10 @@
360 403 (unsigned long) ioat);
361 404 ioat->xfercap_log = xfercap_log;
362 405 spin_lock_init(&ioat->ring_lock);
  406 + if (device->reset_hw(&ioat->base)) {
  407 + i = 0;
  408 + break;
  409 + }
363 410 }
364 411 dma->chancnt = i;
365 412 return i;
... ... @@ -467,7 +514,6 @@
467 514 struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
468 515 struct ioat_chan_common *chan = &ioat->base;
469 516 struct ioat_ring_ent **ring;
470   - u32 chanerr;
471 517 int order;
472 518  
473 519 /* have we already been set up? */
... ... @@ -477,12 +523,6 @@
477 523 /* Setup register to interrupt and write completion status on error */
478 524 writew(IOAT_CHANCTRL_RUN, chan->reg_base + IOAT_CHANCTRL_OFFSET);
479 525  
480   - chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET);
481   - if (chanerr) {
482   - dev_err(to_dev(chan), "CHANERR = %x, clearing\n", chanerr);
483   - writel(chanerr, chan->reg_base + IOAT_CHANERR_OFFSET);
484   - }
485   -
486 526 /* allocate a completion writeback area */
487 527 /* doing 2 32bit writes to mmio since 1 64b write doesn't work */
488 528 chan->completion = pci_pool_alloc(chan->device->completion_pool,
489 529  
... ... @@ -746,14 +786,8 @@
746 786 tasklet_disable(&chan->cleanup_task);
747 787 del_timer_sync(&chan->timer);
748 788 device->cleanup_tasklet((unsigned long) ioat);
  789 + device->reset_hw(chan);
749 790  
750   - /* Delay 100ms after reset to allow internal DMA logic to quiesce
751   - * before removing DMA descriptor resources.
752   - */
753   - writeb(IOAT_CHANCMD_RESET,
754   - chan->reg_base + IOAT_CHANCMD_OFFSET(chan->device->version));
755   - mdelay(100);
756   -
757 791 spin_lock_bh(&ioat->ring_lock);
758 792 descs = ioat2_ring_space(ioat);
759 793 dev_dbg(to_dev(chan), "freeing %d idle descriptors\n", descs);
... ... @@ -839,6 +873,7 @@
839 873 int err;
840 874  
841 875 device->enumerate_channels = ioat2_enumerate_channels;
  876 + device->reset_hw = ioat2_reset_hw;
842 877 device->cleanup_tasklet = ioat2_cleanup_tasklet;
843 878 device->timer_fn = ioat2_timer_event;
844 879 device->self_test = ioat_dma_self_test;
drivers/dma/ioat/dma_v2.h
Diff comments   View file @ 05a6254
... ... @@ -185,6 +185,8 @@
185 185 void __ioat2_issue_pending(struct ioat2_dma_chan *ioat);
186 186 void ioat2_cleanup_tasklet(unsigned long data);
187 187 void ioat2_timer_event(unsigned long data);
  188 +int ioat2_quiesce(struct ioat_chan_common *chan, unsigned long tmo);
  189 +int ioat2_reset_sync(struct ioat_chan_common *chan, unsigned long tmo);
188 190 extern struct kobj_type ioat2_ktype;
189 191 extern struct kmem_cache *ioat2_cache;
190 192 #endif /* IOATDMA_V2_H */
drivers/dma/ioat/dma_v3.c
Diff comments   View file @ 05a6254
... ... @@ -650,9 +650,11 @@
650 650  
651 651 num_descs = ioat2_xferlen_to_descs(ioat, len);
652 652 /* we need 2x the number of descriptors to cover greater than 3
653   - * sources
  653 + * sources (we need 1 extra source in the q-only continuation
  654 + * case and 3 extra sources in the p+q continuation case.
654 655 */
655   - if (src_cnt > 3 || flags & DMA_PREP_CONTINUE) {
  656 + if (src_cnt + dmaf_p_disabled_continue(flags) > 3 ||
  657 + (dmaf_continue(flags) && !dmaf_p_disabled_continue(flags))) {
656 658 with_ext = 1;
657 659 num_descs *= 2;
658 660 } else
... ... @@ -1128,6 +1130,45 @@
1128 1130 return 0;
1129 1131 }
1130 1132  
  1133 +static int ioat3_reset_hw(struct ioat_chan_common *chan)
  1134 +{
  1135 + /* throw away whatever the channel was doing and get it
  1136 + * initialized, with ioat3 specific workarounds
  1137 + */
  1138 + struct ioatdma_device *device = chan->device;
  1139 + struct pci_dev *pdev = device->pdev;
  1140 + u32 chanerr;
  1141 + u16 dev_id;
  1142 + int err;
  1143 +
  1144 + ioat2_quiesce(chan, msecs_to_jiffies(100));
  1145 +
  1146 + chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET);
  1147 + writel(chanerr, chan->reg_base + IOAT_CHANERR_OFFSET);
  1148 +
  1149 + /* -= IOAT ver.3 workarounds =- */
  1150 + /* Write CHANERRMSK_INT with 3E07h to mask out the errors
  1151 + * that can cause stability issues for IOAT ver.3, and clear any
  1152 + * pending errors
  1153 + */
  1154 + pci_write_config_dword(pdev, IOAT_PCI_CHANERRMASK_INT_OFFSET, 0x3e07);
  1155 + err = pci_read_config_dword(pdev, IOAT_PCI_CHANERR_INT_OFFSET, &chanerr);
  1156 + if (err) {
  1157 + dev_err(&pdev->dev, "channel error register unreachable\n");
  1158 + return err;
  1159 + }
  1160 + pci_write_config_dword(pdev, IOAT_PCI_CHANERR_INT_OFFSET, chanerr);
  1161 +
  1162 + /* Clear DMAUNCERRSTS Cfg-Reg Parity Error status bit
  1163 + * (workaround for spurious config parity error after restart)
  1164 + */
  1165 + pci_read_config_word(pdev, IOAT_PCI_DEVICE_ID_OFFSET, &dev_id);
  1166 + if (dev_id == PCI_DEVICE_ID_INTEL_IOAT_TBG0)
  1167 + pci_write_config_dword(pdev, IOAT_PCI_DMAUNCERRSTS_OFFSET, 0x10);
  1168 +
  1169 + return ioat2_reset_sync(chan, msecs_to_jiffies(200));
  1170 +}
  1171 +
1131 1172 int __devinit ioat3_dma_probe(struct ioatdma_device *device, int dca)
1132 1173 {
1133 1174 struct pci_dev *pdev = device->pdev;
1134 1175  
... ... @@ -1137,10 +1178,10 @@
1137 1178 struct ioat_chan_common *chan;
1138 1179 bool is_raid_device = false;
1139 1180 int err;
1140   - u16 dev_id;
1141 1181 u32 cap;
1142 1182  
1143 1183 device->enumerate_channels = ioat2_enumerate_channels;
  1184 + device->reset_hw = ioat3_reset_hw;
1144 1185 device->self_test = ioat3_dma_self_test;
1145 1186 dma = &device->common;
1146 1187 dma->device_prep_dma_memcpy = ioat2_dma_prep_memcpy_lock;
... ... @@ -1215,19 +1256,6 @@
1215 1256 dma_cap_clear(DMA_XOR_VAL, dma->cap_mask);
1216 1257 dma->device_prep_dma_xor_val = NULL;
1217 1258 #endif
1218   -
1219   - /* -= IOAT ver.3 workarounds =- */
1220   - /* Write CHANERRMSK_INT with 3E07h to mask out the errors
1221   - * that can cause stability issues for IOAT ver.3
1222   - */
1223   - pci_write_config_dword(pdev, IOAT_PCI_CHANERRMASK_INT_OFFSET, 0x3e07);
1224   -
1225   - /* Clear DMAUNCERRSTS Cfg-Reg Parity Error status bit
1226   - * (workaround for spurious config parity error after restart)
1227   - */
1228   - pci_read_config_word(pdev, IOAT_PCI_DEVICE_ID_OFFSET, &dev_id);
1229   - if (dev_id == PCI_DEVICE_ID_INTEL_IOAT_TBG0)
1230   - pci_write_config_dword(pdev, IOAT_PCI_DMAUNCERRSTS_OFFSET, 0x10);
1231 1259  
1232 1260 err = ioat_probe(device);
1233 1261 if (err)
drivers/dma/ioat/registers.h
Diff comments   View file @ 05a6254
... ... @@ -27,6 +27,7 @@
27 27  
28 28 #define IOAT_PCI_DEVICE_ID_OFFSET 0x02
29 29 #define IOAT_PCI_DMAUNCERRSTS_OFFSET 0x148
  30 +#define IOAT_PCI_CHANERR_INT_OFFSET 0x180
30 31 #define IOAT_PCI_CHANERRMASK_INT_OFFSET 0x184
31 32  
32 33 /* MMIO Device Registers */
... ... @@ -23,16 +23,19 @@
23 23 #include <linux/dmaengine.h>
24 24 #include <linux/delay.h>
25 25 #include <linux/dma-mapping.h>
26   -#include <linux/dmapool.h>
27 26 #include <linux/platform_device.h>
28 27 #include <cpu/dma.h>
29 28 #include <asm/dma-sh.h>
30 29 #include "shdma.h"
31 30  
32 31 /* DMA descriptor control */
33   -#define DESC_LAST (-1)
34   -#define DESC_COMP (1)
35   -#define DESC_NCOMP (0)
  32 +enum sh_dmae_desc_status {
  33 + DESC_IDLE,
  34 + DESC_PREPARED,
  35 + DESC_SUBMITTED,
  36 + DESC_COMPLETED, /* completed, have to call callback */
  37 + DESC_WAITING, /* callback called, waiting for ack / re-submit */
  38 +};
36 39  
37 40 #define NR_DESCS_PER_CHANNEL 32
38 41 /*
... ... @@ -45,6 +48,8 @@
45 48 */
46 49 #define RS_DEFAULT (RS_DUAL)
47 50  
  51 +static void sh_dmae_chan_ld_cleanup(struct sh_dmae_chan *sh_chan, bool all);
  52 +
48 53 #define SH_DMAC_CHAN_BASE(id) (dma_base_addr[id])
49 54 static void sh_dmae_writel(struct sh_dmae_chan *sh_dc, u32 data, u32 reg)
50 55 {
51 56  
... ... @@ -106,11 +111,11 @@
106 111 return ts_shift[(chcr & CHCR_TS_MASK) >> CHCR_TS_SHIFT];
107 112 }
108 113  
109   -static void dmae_set_reg(struct sh_dmae_chan *sh_chan, struct sh_dmae_regs hw)
  114 +static void dmae_set_reg(struct sh_dmae_chan *sh_chan, struct sh_dmae_regs *hw)
110 115 {
111   - sh_dmae_writel(sh_chan, hw.sar, SAR);
112   - sh_dmae_writel(sh_chan, hw.dar, DAR);
113   - sh_dmae_writel(sh_chan, hw.tcr >> calc_xmit_shift(sh_chan), TCR);
  116 + sh_dmae_writel(sh_chan, hw->sar, SAR);
  117 + sh_dmae_writel(sh_chan, hw->dar, DAR);
  118 + sh_dmae_writel(sh_chan, hw->tcr >> calc_xmit_shift(sh_chan), TCR);
114 119 }
115 120  
116 121 static void dmae_start(struct sh_dmae_chan *sh_chan)
117 122  
... ... @@ -184,8 +189,9 @@
184 189  
185 190 static dma_cookie_t sh_dmae_tx_submit(struct dma_async_tx_descriptor *tx)
186 191 {
187   - struct sh_desc *desc = tx_to_sh_desc(tx);
  192 + struct sh_desc *desc = tx_to_sh_desc(tx), *chunk, *last = desc, *c;
188 193 struct sh_dmae_chan *sh_chan = to_sh_chan(tx->chan);
  194 + dma_async_tx_callback callback = tx->callback;
189 195 dma_cookie_t cookie;
190 196  
191 197 spin_lock_bh(&sh_chan->desc_lock);
192 198  
193 199  
194 200  
195 201  
196 202  
197 203  
198 204  
199 205  
200 206  
... ... @@ -195,47 +201,55 @@
195 201 if (cookie < 0)
196 202 cookie = 1;
197 203  
198   - /* If desc only in the case of 1 */
199   - if (desc->async_tx.cookie != -EBUSY)
200   - desc->async_tx.cookie = cookie;
201   - sh_chan->common.cookie = desc->async_tx.cookie;
  204 + sh_chan->common.cookie = cookie;
  205 + tx->cookie = cookie;
202 206  
203   - list_splice_init(&desc->tx_list, sh_chan->ld_queue.prev);
  207 + /* Mark all chunks of this descriptor as submitted, move to the queue */
  208 + list_for_each_entry_safe(chunk, c, desc->node.prev, node) {
  209 + /*
  210 + * All chunks are on the global ld_free, so, we have to find
  211 + * the end of the chain ourselves
  212 + */
  213 + if (chunk != desc && (chunk->mark == DESC_IDLE ||
  214 + chunk->async_tx.cookie > 0 ||
  215 + chunk->async_tx.cookie == -EBUSY ||
  216 + &chunk->node == &sh_chan->ld_free))
  217 + break;
  218 + chunk->mark = DESC_SUBMITTED;
  219 + /* Callback goes to the last chunk */
  220 + chunk->async_tx.callback = NULL;
  221 + chunk->cookie = cookie;
  222 + list_move_tail(&chunk->node, &sh_chan->ld_queue);
  223 + last = chunk;
  224 + }
204 225  
  226 + last->async_tx.callback = callback;
  227 + last->async_tx.callback_param = tx->callback_param;
  228 +
  229 + dev_dbg(sh_chan->dev, "submit #%d@%p on %d: %x[%d] -> %x\n",
  230 + tx->cookie, &last->async_tx, sh_chan->id,
  231 + desc->hw.sar, desc->hw.tcr, desc->hw.dar);
  232 +
205 233 spin_unlock_bh(&sh_chan->desc_lock);
206 234  
207 235 return cookie;
208 236 }
209 237  
  238 +/* Called with desc_lock held */
210 239 static struct sh_desc *sh_dmae_get_desc(struct sh_dmae_chan *sh_chan)
211 240 {
212   - struct sh_desc *desc, *_desc, *ret = NULL;
  241 + struct sh_desc *desc;
213 242  
214   - spin_lock_bh(&sh_chan->desc_lock);
215   - list_for_each_entry_safe(desc, _desc, &sh_chan->ld_free, node) {
216   - if (async_tx_test_ack(&desc->async_tx)) {
  243 + list_for_each_entry(desc, &sh_chan->ld_free, node)
  244 + if (desc->mark != DESC_PREPARED) {
  245 + BUG_ON(desc->mark != DESC_IDLE);
217 246 list_del(&desc->node);
218   - ret = desc;
219   - break;
  247 + return desc;
220 248 }
221   - }
222   - spin_unlock_bh(&sh_chan->desc_lock);
223 249  
224   - return ret;
  250 + return NULL;
225 251 }
226 252  
227   -static void sh_dmae_put_desc(struct sh_dmae_chan *sh_chan, struct sh_desc *desc)
228   -{
229   - if (desc) {
230   - spin_lock_bh(&sh_chan->desc_lock);
231   -
232   - list_splice_init(&desc->tx_list, &sh_chan->ld_free);
233   - list_add(&desc->node, &sh_chan->ld_free);
234   -
235   - spin_unlock_bh(&sh_chan->desc_lock);
236   - }
237   -}
238   -
239 253 static int sh_dmae_alloc_chan_resources(struct dma_chan *chan)
240 254 {
241 255 struct sh_dmae_chan *sh_chan = to_sh_chan(chan);
242 256  
... ... @@ -252,11 +266,10 @@
252 266 dma_async_tx_descriptor_init(&desc->async_tx,
253 267 &sh_chan->common);
254 268 desc->async_tx.tx_submit = sh_dmae_tx_submit;
255   - desc->async_tx.flags = DMA_CTRL_ACK;
256   - INIT_LIST_HEAD(&desc->tx_list);
257   - sh_dmae_put_desc(sh_chan, desc);
  269 + desc->mark = DESC_IDLE;
258 270  
259 271 spin_lock_bh(&sh_chan->desc_lock);
  272 + list_add(&desc->node, &sh_chan->ld_free);
260 273 sh_chan->descs_allocated++;
261 274 }
262 275 spin_unlock_bh(&sh_chan->desc_lock);
... ... @@ -273,7 +286,10 @@
273 286 struct sh_desc *desc, *_desc;
274 287 LIST_HEAD(list);
275 288  
276   - BUG_ON(!list_empty(&sh_chan->ld_queue));
  289 + /* Prepared and not submitted descriptors can still be on the queue */
  290 + if (!list_empty(&sh_chan->ld_queue))
  291 + sh_dmae_chan_ld_cleanup(sh_chan, true);
  292 +
277 293 spin_lock_bh(&sh_chan->desc_lock);
278 294  
279 295 list_splice_init(&sh_chan->ld_free, &list);
... ... @@ -292,6 +308,8 @@
292 308 struct sh_dmae_chan *sh_chan;
293 309 struct sh_desc *first = NULL, *prev = NULL, *new;
294 310 size_t copy_size;
  311 + LIST_HEAD(tx_list);
  312 + int chunks = (len + SH_DMA_TCR_MAX) / (SH_DMA_TCR_MAX + 1);
295 313  
296 314 if (!chan)
297 315 return NULL;
298 316  
299 317  
300 318  
301 319  
302 320  
303 321  
304 322  
305 323  
306 324  
307 325  
308 326  
309 327  
310 328  
311 329  
312 330  
313 331  
314 332  
315 333  
316 334  
317 335  
318 336  
319 337  
320 338  
321 339  
322 340  
323 341  
324 342  
325 343  
326 344  
... ... @@ -301,108 +319,189 @@
301 319  
302 320 sh_chan = to_sh_chan(chan);
303 321  
  322 + /* Have to lock the whole loop to protect against concurrent release */
  323 + spin_lock_bh(&sh_chan->desc_lock);
  324 +
  325 + /*
  326 + * Chaining:
  327 + * first descriptor is what user is dealing with in all API calls, its
  328 + * cookie is at first set to -EBUSY, at tx-submit to a positive
  329 + * number
  330 + * if more than one chunk is needed further chunks have cookie = -EINVAL
  331 + * the last chunk, if not equal to the first, has cookie = -ENOSPC
  332 + * all chunks are linked onto the tx_list head with their .node heads
  333 + * only during this function, then they are immediately spliced
  334 + * back onto the free list in form of a chain
  335 + */
304 336 do {
305   - /* Allocate the link descriptor from DMA pool */
  337 + /* Allocate the link descriptor from the free list */
306 338 new = sh_dmae_get_desc(sh_chan);
307 339 if (!new) {
308 340 dev_err(sh_chan->dev,
309 341 "No free memory for link descriptor\n");
310   - goto err_get_desc;
  342 + list_for_each_entry(new, &tx_list, node)
  343 + new->mark = DESC_IDLE;
  344 + list_splice(&tx_list, &sh_chan->ld_free);
  345 + spin_unlock_bh(&sh_chan->desc_lock);
  346 + return NULL;
311 347 }
312 348  
313   - copy_size = min(len, (size_t)SH_DMA_TCR_MAX);
  349 + copy_size = min(len, (size_t)SH_DMA_TCR_MAX + 1);
314 350  
315 351 new->hw.sar = dma_src;
316 352 new->hw.dar = dma_dest;
317 353 new->hw.tcr = copy_size;
318   - if (!first)
  354 + if (!first) {
  355 + /* First desc */
  356 + new->async_tx.cookie = -EBUSY;
319 357 first = new;
  358 + } else {
  359 + /* Other desc - invisible to the user */
  360 + new->async_tx.cookie = -EINVAL;
  361 + }
320 362  
321   - new->mark = DESC_NCOMP;
322   - async_tx_ack(&new->async_tx);
  363 + dev_dbg(sh_chan->dev,
  364 + "chaining %u of %u with %p, dst %x, cookie %d\n",
  365 + copy_size, len, &new->async_tx, dma_dest,
  366 + new->async_tx.cookie);
323 367  
  368 + new->mark = DESC_PREPARED;
  369 + new->async_tx.flags = flags;
  370 + new->chunks = chunks--;
  371 +
324 372 prev = new;
325 373 len -= copy_size;
326 374 dma_src += copy_size;
327 375 dma_dest += copy_size;
328 376 /* Insert the link descriptor to the LD ring */
329   - list_add_tail(&new->node, &first->tx_list);
  377 + list_add_tail(&new->node, &tx_list);
330 378 } while (len);
331 379  
332   - new->async_tx.flags = flags; /* client is in control of this ack */
333   - new->async_tx.cookie = -EBUSY; /* Last desc */
  380 + if (new != first)
  381 + new->async_tx.cookie = -ENOSPC;
334 382  
335   - return &first->async_tx;
  383 + /* Put them back on the free list, so, they don't get lost */
  384 + list_splice_tail(&tx_list, &sh_chan->ld_free);
336 385  
337   -err_get_desc:
338   - sh_dmae_put_desc(sh_chan, first);
339   - return NULL;
  386 + spin_unlock_bh(&sh_chan->desc_lock);
340 387  
  388 + return &first->async_tx;
341 389 }
342 390  
343   -/*
344   - * sh_chan_ld_cleanup - Clean up link descriptors
345   - *
346   - * This function clean up the ld_queue of DMA channel.
347   - */
348   -static void sh_dmae_chan_ld_cleanup(struct sh_dmae_chan *sh_chan)
  391 +static dma_async_tx_callback __ld_cleanup(struct sh_dmae_chan *sh_chan, bool all)
349 392 {
350 393 struct sh_desc *desc, *_desc;
  394 + /* Is the "exposed" head of a chain acked? */
  395 + bool head_acked = false;
  396 + dma_cookie_t cookie = 0;
  397 + dma_async_tx_callback callback = NULL;
  398 + void *param = NULL;
351 399  
352 400 spin_lock_bh(&sh_chan->desc_lock);
353 401 list_for_each_entry_safe(desc, _desc, &sh_chan->ld_queue, node) {
354   - dma_async_tx_callback callback;
355   - void *callback_param;
  402 + struct dma_async_tx_descriptor *tx = &desc->async_tx;
356 403  
357   - /* non send data */
358   - if (desc->mark == DESC_NCOMP)
  404 + BUG_ON(tx->cookie > 0 && tx->cookie != desc->cookie);
  405 + BUG_ON(desc->mark != DESC_SUBMITTED &&
  406 + desc->mark != DESC_COMPLETED &&
  407 + desc->mark != DESC_WAITING);
  408 +
  409 + /*
  410 + * queue is ordered, and we use this loop to (1) clean up all
  411 + * completed descriptors, and to (2) update descriptor flags of
  412 + * any chunks in a (partially) completed chain
  413 + */
  414 + if (!all && desc->mark == DESC_SUBMITTED &&
  415 + desc->cookie != cookie)
359 416 break;
360 417  
361   - /* send data sesc */
362   - callback = desc->async_tx.callback;
363   - callback_param = desc->async_tx.callback_param;
  418 + if (tx->cookie > 0)
  419 + cookie = tx->cookie;
364 420  
365   - /* Remove from ld_queue list */
366   - list_splice_init(&desc->tx_list, &sh_chan->ld_free);
  421 + if (desc->mark == DESC_COMPLETED && desc->chunks == 1) {
  422 + BUG_ON(sh_chan->completed_cookie != desc->cookie - 1);
  423 + sh_chan->completed_cookie = desc->cookie;
  424 + }
367 425  
368   - dev_dbg(sh_chan->dev, "link descriptor %p will be recycle.\n",
369   - desc);
  426 + /* Call callback on the last chunk */
  427 + if (desc->mark == DESC_COMPLETED && tx->callback) {
  428 + desc->mark = DESC_WAITING;
  429 + callback = tx->callback;
  430 + param = tx->callback_param;
  431 + dev_dbg(sh_chan->dev, "descriptor #%d@%p on %d callback\n",
  432 + tx->cookie, tx, sh_chan->id);
  433 + BUG_ON(desc->chunks != 1);
  434 + break;
  435 + }
370 436  
371   - list_move(&desc->node, &sh_chan->ld_free);
372   - /* Run the link descriptor callback function */
373   - if (callback) {
374   - spin_unlock_bh(&sh_chan->desc_lock);
375   - dev_dbg(sh_chan->dev, "link descriptor %p callback\n",
376   - desc);
377   - callback(callback_param);
378   - spin_lock_bh(&sh_chan->desc_lock);
  437 + if (tx->cookie > 0 || tx->cookie == -EBUSY) {
  438 + if (desc->mark == DESC_COMPLETED) {
  439 + BUG_ON(tx->cookie < 0);
  440 + desc->mark = DESC_WAITING;
  441 + }
  442 + head_acked = async_tx_test_ack(tx);
  443 + } else {
  444 + switch (desc->mark) {
  445 + case DESC_COMPLETED:
  446 + desc->mark = DESC_WAITING;
  447 + /* Fall through */
  448 + case DESC_WAITING:
  449 + if (head_acked)
  450 + async_tx_ack(&desc->async_tx);
  451 + }
379 452 }
  453 +
  454 + dev_dbg(sh_chan->dev, "descriptor %p #%d completed.\n",
  455 + tx, tx->cookie);
  456 +
  457 + if (((desc->mark == DESC_COMPLETED ||
  458 + desc->mark == DESC_WAITING) &&
  459 + async_tx_test_ack(&desc->async_tx)) || all) {
  460 + /* Remove from ld_queue list */
  461 + desc->mark = DESC_IDLE;
  462 + list_move(&desc->node, &sh_chan->ld_free);
  463 + }
380 464 }
381 465 spin_unlock_bh(&sh_chan->desc_lock);
  466 +
  467 + if (callback)
  468 + callback(param);
  469 +
  470 + return callback;
382 471 }
383 472  
  473 +/*
  474 + * sh_chan_ld_cleanup - Clean up link descriptors
  475 + *
  476 + * This function cleans up the ld_queue of DMA channel.
  477 + */
  478 +static void sh_dmae_chan_ld_cleanup(struct sh_dmae_chan *sh_chan, bool all)
  479 +{
  480 + while (__ld_cleanup(sh_chan, all))
  481 + ;
  482 +}
  483 +
384 484 static void sh_chan_xfer_ld_queue(struct sh_dmae_chan *sh_chan)
385 485 {
386   - struct list_head *ld_node;
387   - struct sh_dmae_regs hw;
  486 + struct sh_desc *sd;
388 487  
  488 + spin_lock_bh(&sh_chan->desc_lock);
389 489 /* DMA work check */
390   - if (dmae_is_busy(sh_chan))
  490 + if (dmae_is_busy(sh_chan)) {
  491 + spin_unlock_bh(&sh_chan->desc_lock);
391 492 return;
  493 + }
392 494  
393 495 /* Find the first un-transfer desciptor */
394   - for (ld_node = sh_chan->ld_queue.next;
395   - (ld_node != &sh_chan->ld_queue)
396   - && (to_sh_desc(ld_node)->mark == DESC_COMP);
397   - ld_node = ld_node->next)
398   - cpu_relax();
  496 + list_for_each_entry(sd, &sh_chan->ld_queue, node)
  497 + if (sd->mark == DESC_SUBMITTED) {
  498 + /* Get the ld start address from ld_queue */
  499 + dmae_set_reg(sh_chan, &sd->hw);
  500 + dmae_start(sh_chan);
  501 + break;
  502 + }
399 503  
400   - if (ld_node != &sh_chan->ld_queue) {
401   - /* Get the ld start address from ld_queue */
402   - hw = to_sh_desc(ld_node)->hw;
403   - dmae_set_reg(sh_chan, hw);
404   - dmae_start(sh_chan);
405   - }
  504 + spin_unlock_bh(&sh_chan->desc_lock);
406 505 }
407 506  
408 507 static void sh_dmae_memcpy_issue_pending(struct dma_chan *chan)
409 508  
... ... @@ -420,12 +519,11 @@
420 519 dma_cookie_t last_used;
421 520 dma_cookie_t last_complete;
422 521  
423   - sh_dmae_chan_ld_cleanup(sh_chan);
  522 + sh_dmae_chan_ld_cleanup(sh_chan, false);
424 523  
425 524 last_used = chan->cookie;
426 525 last_complete = sh_chan->completed_cookie;
427   - if (last_complete == -EBUSY)
428   - last_complete = last_used;
  526 + BUG_ON(last_complete < 0);
429 527  
430 528 if (done)
431 529 *done = last_complete;
432 530  
... ... @@ -480,11 +578,13 @@
480 578 err = sh_dmae_rst(0);
481 579 if (err)
482 580 return err;
  581 +#ifdef SH_DMAC_BASE1
483 582 if (shdev->pdata.mode & SHDMA_DMAOR1) {
484 583 err = sh_dmae_rst(1);
485 584 if (err)
486 585 return err;
487 586 }
  587 +#endif
488 588 disable_irq(irq);
489 589 return IRQ_HANDLED;
490 590 }
491 591  
492 592  
493 593  
494 594  
... ... @@ -494,35 +594,25 @@
494 594 static void dmae_do_tasklet(unsigned long data)
495 595 {
496 596 struct sh_dmae_chan *sh_chan = (struct sh_dmae_chan *)data;
497   - struct sh_desc *desc, *_desc, *cur_desc = NULL;
  597 + struct sh_desc *desc;
498 598 u32 sar_buf = sh_dmae_readl(sh_chan, SAR);
499 599  
500   - list_for_each_entry_safe(desc, _desc,
501   - &sh_chan->ld_queue, node) {
502   - if ((desc->hw.sar + desc->hw.tcr) == sar_buf) {
503   - cur_desc = desc;
  600 + spin_lock(&sh_chan->desc_lock);
  601 + list_for_each_entry(desc, &sh_chan->ld_queue, node) {
  602 + if ((desc->hw.sar + desc->hw.tcr) == sar_buf &&
  603 + desc->mark == DESC_SUBMITTED) {
  604 + dev_dbg(sh_chan->dev, "done #%d@%p dst %u\n",
  605 + desc->async_tx.cookie, &desc->async_tx,
  606 + desc->hw.dar);
  607 + desc->mark = DESC_COMPLETED;
504 608 break;
505 609 }
506 610 }
  611 + spin_unlock(&sh_chan->desc_lock);
507 612  
508   - if (cur_desc) {
509   - switch (cur_desc->async_tx.cookie) {
510   - case 0: /* other desc data */
511   - break;
512   - case -EBUSY: /* last desc */
513   - sh_chan->completed_cookie =
514   - cur_desc->async_tx.cookie;
515   - break;
516   - default: /* first desc ( 0 < )*/
517   - sh_chan->completed_cookie =
518   - cur_desc->async_tx.cookie - 1;
519   - break;
520   - }
521   - cur_desc->mark = DESC_COMP;
522   - }
523 613 /* Next desc */
524 614 sh_chan_xfer_ld_queue(sh_chan);
525   - sh_dmae_chan_ld_cleanup(sh_chan);
  615 + sh_dmae_chan_ld_cleanup(sh_chan, false);
526 616 }
527 617  
528 618 static unsigned int get_dmae_irq(unsigned int id)
... ... @@ -13,9 +13,9 @@
13 13 #ifndef __DMA_SHDMA_H
14 14 #define __DMA_SHDMA_H
15 15  
16   -#include <linux/device.h>
17   -#include <linux/dmapool.h>
18 16 #include <linux/dmaengine.h>
  17 +#include <linux/interrupt.h>
  18 +#include <linux/list.h>
19 19  
20 20 #define SH_DMA_TCR_MAX 0x00FFFFFF /* 16MB */
21 21  
22 22  
23 23  
... ... @@ -26,12 +26,15 @@
26 26 };
27 27  
28 28 struct sh_desc {
29   - struct list_head tx_list;
30 29 struct sh_dmae_regs hw;
31 30 struct list_head node;
32 31 struct dma_async_tx_descriptor async_tx;
  32 + dma_cookie_t cookie;
  33 + int chunks;
33 34 int mark;
34 35 };
  36 +
  37 +struct device;
35 38  
36 39 struct sh_dmae_chan {
37 40 dma_cookie_t completed_cookie; /* The maximum cookie completed */