Eric Lee / smarc-fsl-linux-kernel

Download as
Browse Code ยป

Commit 636bdeaa1243327501edfd2a597ed7443eb4239a

Authored by Dan Williams
1 parent c4fe15541d

dmaengine: ack to flags: make use of the unused bits in the 'ack' field 

'ack' is currently a simple integer that flags whether or not a client is done
touching fields in the given descriptor.  It is effectively just a single bit
of information.  Converting this to a flags parameter allows the other bits to
be put to use to control completion actions, like dma-unmap, and capture
results, like xor-zero-sum == 0.

Changes are one of:
1/ convert all open-coded ->ack manipulations to use async_tx_ack
   and async_tx_test_ack.
2/ set the ack bit at prep time where possible
3/ make drivers store the flags at prep time
4/ add flags to the device_prep_dma_interrupt prototype

Acked-by: Maciej Sosnowski <maciej.sosnowski@intel.com>
Signed-off-by: Dan Williams <dan.j.williams@intel.com>

Showing 8 changed files with 69 additions and 54 deletions Side-by-side Diff

crypto/async_tx/async_memcpy.c
Diff comments   View file @ 636bdea
... ... @@ -77,7 +77,7 @@
77 77 /* if ack is already set then we cannot be sure
78 78 * we are referring to the correct operation
79 79 */
80   - BUG_ON(depend_tx->ack);
  80 + BUG_ON(async_tx_test_ack(depend_tx));
81 81 if (dma_wait_for_async_tx(depend_tx) == DMA_ERROR)
82 82 panic("%s: DMA_ERROR waiting for depend_tx\n",
83 83 __func__);
crypto/async_tx/async_tx.c
Diff comments   View file @ 636bdea
... ... @@ -446,7 +446,7 @@
446 446 * otherwise poll for completion
447 447 */
448 448 if (dma_has_cap(DMA_INTERRUPT, device->cap_mask))
449   - intr_tx = device->device_prep_dma_interrupt(chan);
  449 + intr_tx = device->device_prep_dma_interrupt(chan, 0);
450 450 else
451 451 intr_tx = NULL;
452 452  
... ... @@ -515,7 +515,8 @@
515 515 * 2/ dependencies are 1:1 i.e. two transactions can
516 516 * not depend on the same parent
517 517 */
518   - BUG_ON(depend_tx->ack || depend_tx->next || tx->parent);
  518 + BUG_ON(async_tx_test_ack(depend_tx) || depend_tx->next ||
  519 + tx->parent);
519 520  
520 521 /* the lock prevents async_tx_run_dependencies from missing
521 522 * the setting of ->next when ->parent != NULL
... ... @@ -594,7 +595,7 @@
594 595 if (device && !dma_has_cap(DMA_INTERRUPT, device->cap_mask))
595 596 device = NULL;
596 597  
597   - tx = device ? device->device_prep_dma_interrupt(chan) : NULL;
  598 + tx = device ? device->device_prep_dma_interrupt(chan, 0) : NULL;
598 599 } else
599 600 tx = NULL;
600 601  
... ... @@ -610,7 +611,7 @@
610 611 /* if ack is already set then we cannot be sure
611 612 * we are referring to the correct operation
612 613 */
613   - BUG_ON(depend_tx->ack);
  614 + BUG_ON(async_tx_test_ack(depend_tx));
614 615 if (dma_wait_for_async_tx(depend_tx) == DMA_ERROR)
615 616 panic("%s: DMA_ERROR waiting for depend_tx\n",
616 617 __func__);
crypto/async_tx/async_xor.c
Diff comments   View file @ 636bdea
... ... @@ -191,7 +191,7 @@
191 191 /* if ack is already set then we cannot be sure
192 192 * we are referring to the correct operation
193 193 */
194   - BUG_ON(depend_tx->ack);
  194 + BUG_ON(async_tx_test_ack(depend_tx));
195 195 if (dma_wait_for_async_tx(depend_tx) ==
196 196 DMA_ERROR)
197 197 panic("%s: DMA_ERROR waiting for "
drivers/dma/dmaengine.c
Diff comments   View file @ 636bdea
... ... @@ -478,7 +478,8 @@
478 478  
479 479 dma_src = dma_map_single(dev->dev, src, len, DMA_TO_DEVICE);
480 480 dma_dest = dma_map_single(dev->dev, dest, len, DMA_FROM_DEVICE);
481   - tx = dev->device_prep_dma_memcpy(chan, dma_dest, dma_src, len, 0);
  481 + tx = dev->device_prep_dma_memcpy(chan, dma_dest, dma_src, len,
  482 + DMA_CTRL_ACK);
482 483  
483 484 if (!tx) {
484 485 dma_unmap_single(dev->dev, dma_src, len, DMA_TO_DEVICE);
... ... @@ -486,7 +487,6 @@
486 487 return -ENOMEM;
487 488 }
488 489  
489   - tx->ack = 1;
490 490 tx->callback = NULL;
491 491 cookie = tx->tx_submit(tx);
492 492  
... ... @@ -524,7 +524,8 @@
524 524  
525 525 dma_src = dma_map_single(dev->dev, kdata, len, DMA_TO_DEVICE);
526 526 dma_dest = dma_map_page(dev->dev, page, offset, len, DMA_FROM_DEVICE);
527   - tx = dev->device_prep_dma_memcpy(chan, dma_dest, dma_src, len, 0);
  527 + tx = dev->device_prep_dma_memcpy(chan, dma_dest, dma_src, len,
  528 + DMA_CTRL_ACK);
528 529  
529 530 if (!tx) {
530 531 dma_unmap_single(dev->dev, dma_src, len, DMA_TO_DEVICE);
... ... @@ -532,7 +533,6 @@
532 533 return -ENOMEM;
533 534 }
534 535  
535   - tx->ack = 1;
536 536 tx->callback = NULL;
537 537 cookie = tx->tx_submit(tx);
538 538  
... ... @@ -573,7 +573,8 @@
573 573 dma_src = dma_map_page(dev->dev, src_pg, src_off, len, DMA_TO_DEVICE);
574 574 dma_dest = dma_map_page(dev->dev, dest_pg, dest_off, len,
575 575 DMA_FROM_DEVICE);
576   - tx = dev->device_prep_dma_memcpy(chan, dma_dest, dma_src, len, 0);
  576 + tx = dev->device_prep_dma_memcpy(chan, dma_dest, dma_src, len,
  577 + DMA_CTRL_ACK);
577 578  
578 579 if (!tx) {
579 580 dma_unmap_page(dev->dev, dma_src, len, DMA_TO_DEVICE);
... ... @@ -581,7 +582,6 @@
581 582 return -ENOMEM;
582 583 }
583 584  
584   - tx->ack = 1;
585 585 tx->callback = NULL;
586 586 cookie = tx->tx_submit(tx);
587 587  
drivers/dma/fsldma.c
Diff comments   View file @ 636bdea
... ... @@ -412,7 +412,7 @@
412 412 }
413 413  
414 414 static struct dma_async_tx_descriptor *
415   -fsl_dma_prep_interrupt(struct dma_chan *chan)
  415 +fsl_dma_prep_interrupt(struct dma_chan *chan, unsigned long flags)
416 416 {
417 417 struct fsl_dma_chan *fsl_chan;
418 418 struct fsl_desc_sw *new;
... ... @@ -429,7 +429,7 @@
429 429 }
430 430  
431 431 new->async_tx.cookie = -EBUSY;
432   - new->async_tx.ack = 0;
  432 + new->async_tx.flags = flags;
433 433  
434 434 /* Insert the link descriptor to the LD ring */
435 435 list_add_tail(&new->node, &new->async_tx.tx_list);
... ... @@ -482,7 +482,7 @@
482 482 set_desc_next(fsl_chan, &prev->hw, new->async_tx.phys);
483 483  
484 484 new->async_tx.cookie = 0;
485   - new->async_tx.ack = 1;
  485 + async_tx_ack(&new->async_tx);
486 486  
487 487 prev = new;
488 488 len -= copy;
... ... @@ -493,7 +493,7 @@
493 493 list_add_tail(&new->node, &first->async_tx.tx_list);
494 494 } while (len);
495 495  
496   - new->async_tx.ack = 0; /* client is in control of this ack */
  496 + new->async_tx.flags = flags; /* client is in control of this ack */
497 497 new->async_tx.cookie = -EBUSY;
498 498  
499 499 /* Set End-of-link to the last link descriptor of new list*/
... ... @@ -874,7 +874,7 @@
874 874 async_tx_ack(tx3);
875 875  
876 876 /* Interrupt tx test */
877   - tx1 = fsl_dma_prep_interrupt(chan);
  877 + tx1 = fsl_dma_prep_interrupt(chan, 0);
878 878 async_tx_ack(tx1);
879 879 cookie = fsl_dma_tx_submit(tx1);
880 880  
drivers/dma/ioat_dma.c
Diff comments   View file @ 636bdea
... ... @@ -212,14 +212,14 @@
212 212 u32 copy;
213 213 size_t len;
214 214 dma_addr_t src, dst;
215   - int orig_ack;
  215 + unsigned long orig_flags;
216 216 unsigned int desc_count = 0;
217 217  
218 218 /* src and dest and len are stored in the initial descriptor */
219 219 len = first->len;
220 220 src = first->src;
221 221 dst = first->dst;
222   - orig_ack = first->async_tx.ack;
  222 + orig_flags = first->async_tx.flags;
223 223 new = first;
224 224  
225 225 spin_lock_bh(&ioat_chan->desc_lock);
... ... @@ -228,7 +228,7 @@
228 228 do {
229 229 copy = min_t(size_t, len, ioat_chan->xfercap);
230 230  
231   - new->async_tx.ack = 1;
  231 + async_tx_ack(&new->async_tx);
232 232  
233 233 hw = new->hw;
234 234 hw->size = copy;
... ... @@ -264,7 +264,7 @@
264 264 }
265 265  
266 266 new->tx_cnt = desc_count;
267   - new->async_tx.ack = orig_ack; /* client is in control of this ack */
  267 + new->async_tx.flags = orig_flags; /* client is in control of this ack */
268 268  
269 269 /* store the original values for use in later cleanup */
270 270 if (new != first) {
271 271  
... ... @@ -304,14 +304,14 @@
304 304 u32 copy;
305 305 size_t len;
306 306 dma_addr_t src, dst;
307   - int orig_ack;
  307 + unsigned long orig_flags;
308 308 unsigned int desc_count = 0;
309 309  
310 310 /* src and dest and len are stored in the initial descriptor */
311 311 len = first->len;
312 312 src = first->src;
313 313 dst = first->dst;
314   - orig_ack = first->async_tx.ack;
  314 + orig_flags = first->async_tx.flags;
315 315 new = first;
316 316  
317 317 /*
... ... @@ -321,7 +321,7 @@
321 321 do {
322 322 copy = min_t(size_t, len, ioat_chan->xfercap);
323 323  
324   - new->async_tx.ack = 1;
  324 + async_tx_ack(&new->async_tx);
325 325  
326 326 hw = new->hw;
327 327 hw->size = copy;
... ... @@ -349,7 +349,7 @@
349 349 }
350 350  
351 351 new->tx_cnt = desc_count;
352   - new->async_tx.ack = orig_ack; /* client is in control of this ack */
  352 + new->async_tx.flags = orig_flags; /* client is in control of this ack */
353 353  
354 354 /* store the original values for use in later cleanup */
355 355 if (new != first) {
... ... @@ -714,7 +714,7 @@
714 714 new->len = len;
715 715 new->dst = dma_dest;
716 716 new->src = dma_src;
717   - new->async_tx.ack = 0;
  717 + new->async_tx.flags = flags;
718 718 return &new->async_tx;
719 719 } else
720 720 return NULL;
... ... @@ -742,7 +742,7 @@
742 742 new->len = len;
743 743 new->dst = dma_dest;
744 744 new->src = dma_src;
745   - new->async_tx.ack = 0;
  745 + new->async_tx.flags = flags;
746 746 return &new->async_tx;
747 747 } else
748 748 return NULL;
... ... @@ -842,7 +842,7 @@
842 842 * a completed entry, but not the last, so clean
843 843 * up if the client is done with the descriptor
844 844 */
845   - if (desc->async_tx.ack) {
  845 + if (async_tx_test_ack(&desc->async_tx)) {
846 846 list_del(&desc->node);
847 847 list_add_tail(&desc->node,
848 848 &ioat_chan->free_desc);
... ... @@ -979,7 +979,7 @@
979 979 desc->hw->size = 0;
980 980 desc->hw->src_addr = 0;
981 981 desc->hw->dst_addr = 0;
982   - desc->async_tx.ack = 1;
  982 + async_tx_ack(&desc->async_tx);
983 983 switch (ioat_chan->device->version) {
984 984 case IOAT_VER_1_2:
985 985 desc->hw->next = 0;
drivers/dma/iop-adma.c
Diff comments   View file @ 636bdea
... ... @@ -111,7 +111,7 @@
111 111 /* the client is allowed to attach dependent operations
112 112 * until 'ack' is set
113 113 */
114   - if (!desc->async_tx.ack)
  114 + if (!async_tx_test_ack(&desc->async_tx))
115 115 return 0;
116 116  
117 117 /* leave the last descriptor in the chain
... ... @@ -148,7 +148,7 @@
148 148 "this_desc: %#x next_desc: %#x ack: %d\n",
149 149 iter->async_tx.cookie, iter->idx, busy,
150 150 iter->async_tx.phys, iop_desc_get_next_desc(iter),
151   - iter->async_tx.ack);
  151 + async_tx_test_ack(&iter->async_tx));
152 152 prefetch(_iter);
153 153 prefetch(&_iter->async_tx);
154 154  
... ... @@ -338,9 +338,7 @@
338 338  
339 339 /* pre-ack all but the last descriptor */
340 340 if (num_slots != slots_per_op)
341   - iter->async_tx.ack = 1;
342   - else
343   - iter->async_tx.ack = 0;
  341 + async_tx_ack(&iter->async_tx);
344 342  
345 343 list_add_tail(&iter->chain_node, &chain);
346 344 alloc_tail = iter;
... ... @@ -513,7 +511,7 @@
513 511 }
514 512  
515 513 static struct dma_async_tx_descriptor *
516   -iop_adma_prep_dma_interrupt(struct dma_chan *chan)
  514 +iop_adma_prep_dma_interrupt(struct dma_chan *chan, unsigned long flags)
517 515 {
518 516 struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
519 517 struct iop_adma_desc_slot *sw_desc, *grp_start;
... ... @@ -528,6 +526,7 @@
528 526 grp_start = sw_desc->group_head;
529 527 iop_desc_init_interrupt(grp_start, iop_chan);
530 528 grp_start->unmap_len = 0;
  529 + sw_desc->async_tx.flags = flags;
531 530 }
532 531 spin_unlock_bh(&iop_chan->lock);
533 532  
... ... @@ -560,6 +559,7 @@
560 559 iop_desc_set_memcpy_src_addr(grp_start, dma_src);
561 560 sw_desc->unmap_src_cnt = 1;
562 561 sw_desc->unmap_len = len;
  562 + sw_desc->async_tx.flags = flags;
563 563 }
564 564 spin_unlock_bh(&iop_chan->lock);
565 565  
... ... @@ -592,6 +592,7 @@
592 592 iop_desc_set_dest_addr(grp_start, iop_chan, dma_dest);
593 593 sw_desc->unmap_src_cnt = 1;
594 594 sw_desc->unmap_len = len;
  595 + sw_desc->async_tx.flags = flags;
595 596 }
596 597 spin_unlock_bh(&iop_chan->lock);
597 598  
... ... @@ -625,6 +626,7 @@
625 626 iop_desc_set_dest_addr(grp_start, iop_chan, dma_dest);
626 627 sw_desc->unmap_src_cnt = src_cnt;
627 628 sw_desc->unmap_len = len;
  629 + sw_desc->async_tx.flags = flags;
628 630 while (src_cnt--)
629 631 iop_desc_set_xor_src_addr(grp_start, src_cnt,
630 632 dma_src[src_cnt]);
... ... @@ -661,6 +663,7 @@
661 663 __func__, grp_start->xor_check_result);
662 664 sw_desc->unmap_src_cnt = src_cnt;
663 665 sw_desc->unmap_len = len;
  666 + sw_desc->async_tx.flags = flags;
664 667 while (src_cnt--)
665 668 iop_desc_set_zero_sum_src_addr(grp_start, src_cnt,
666 669 dma_src[src_cnt]);
667 670  
... ... @@ -847,11 +850,11 @@
847 850 src_dma = dma_map_single(dma_chan->device->dev, src,
848 851 IOP_ADMA_TEST_SIZE, DMA_TO_DEVICE);
849 852 tx = iop_adma_prep_dma_memcpy(dma_chan, dest_dma, src_dma,
850   - IOP_ADMA_TEST_SIZE, 1);
  853 + IOP_ADMA_TEST_SIZE,
  854 + DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
851 855  
852 856 cookie = iop_adma_tx_submit(tx);
853 857 iop_adma_issue_pending(dma_chan);
854   - async_tx_ack(tx);
855 858 msleep(1);
856 859  
857 860 if (iop_adma_is_complete(dma_chan, cookie, NULL, NULL) !=
858 861  
... ... @@ -947,11 +950,11 @@
947 950 dma_srcs[i] = dma_map_page(dma_chan->device->dev, xor_srcs[i],
948 951 0, PAGE_SIZE, DMA_TO_DEVICE);
949 952 tx = iop_adma_prep_dma_xor(dma_chan, dest_dma, dma_srcs,
950   - IOP_ADMA_NUM_SRC_TEST, PAGE_SIZE, 1);
  953 + IOP_ADMA_NUM_SRC_TEST, PAGE_SIZE,
  954 + DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
951 955  
952 956 cookie = iop_adma_tx_submit(tx);
953 957 iop_adma_issue_pending(dma_chan);
954   - async_tx_ack(tx);
955 958 msleep(8);
956 959  
957 960 if (iop_adma_is_complete(dma_chan, cookie, NULL, NULL) !=
958 961  
... ... @@ -994,11 +997,11 @@
994 997 DMA_TO_DEVICE);
995 998 tx = iop_adma_prep_dma_zero_sum(dma_chan, dma_srcs,
996 999 IOP_ADMA_NUM_SRC_TEST + 1, PAGE_SIZE,
997   - &zero_sum_result, 1);
  1000 + &zero_sum_result,
  1001 + DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
998 1002  
999 1003 cookie = iop_adma_tx_submit(tx);
1000 1004 iop_adma_issue_pending(dma_chan);
1001   - async_tx_ack(tx);
1002 1005 msleep(8);
1003 1006  
1004 1007 if (iop_adma_is_complete(dma_chan, cookie, NULL, NULL) != DMA_SUCCESS) {
1005 1008  
... ... @@ -1018,11 +1021,11 @@
1018 1021 /* test memset */
1019 1022 dma_addr = dma_map_page(dma_chan->device->dev, dest, 0,
1020 1023 PAGE_SIZE, DMA_FROM_DEVICE);
1021   - tx = iop_adma_prep_dma_memset(dma_chan, dma_addr, 0, PAGE_SIZE, 1);
  1024 + tx = iop_adma_prep_dma_memset(dma_chan, dma_addr, 0, PAGE_SIZE,
  1025 + DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
1022 1026  
1023 1027 cookie = iop_adma_tx_submit(tx);
1024 1028 iop_adma_issue_pending(dma_chan);
1025   - async_tx_ack(tx);
1026 1029 msleep(8);
1027 1030  
1028 1031 if (iop_adma_is_complete(dma_chan, cookie, NULL, NULL) != DMA_SUCCESS) {
1029 1032  
... ... @@ -1050,11 +1053,11 @@
1050 1053 DMA_TO_DEVICE);
1051 1054 tx = iop_adma_prep_dma_zero_sum(dma_chan, dma_srcs,
1052 1055 IOP_ADMA_NUM_SRC_TEST + 1, PAGE_SIZE,
1053   - &zero_sum_result, 1);
  1056 + &zero_sum_result,
  1057 + DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
1054 1058  
1055 1059 cookie = iop_adma_tx_submit(tx);
1056 1060 iop_adma_issue_pending(dma_chan);
1057   - async_tx_ack(tx);
1058 1061 msleep(8);
1059 1062  
1060 1063 if (iop_adma_is_complete(dma_chan, cookie, NULL, NULL) != DMA_SUCCESS) {
... ... @@ -1287,7 +1290,7 @@
1287 1290 grp_start = sw_desc->group_head;
1288 1291  
1289 1292 list_splice_init(&sw_desc->async_tx.tx_list, &iop_chan->chain);
1290   - sw_desc->async_tx.ack = 1;
  1293 + async_tx_ack(&sw_desc->async_tx);
1291 1294 iop_desc_init_memcpy(grp_start, 0);
1292 1295 iop_desc_set_byte_count(grp_start, iop_chan, 0);
1293 1296 iop_desc_set_dest_addr(grp_start, iop_chan, 0);
... ... @@ -1343,7 +1346,7 @@
1343 1346 if (sw_desc) {
1344 1347 grp_start = sw_desc->group_head;
1345 1348 list_splice_init(&sw_desc->async_tx.tx_list, &iop_chan->chain);
1346   - sw_desc->async_tx.ack = 1;
  1349 + async_tx_ack(&sw_desc->async_tx);
1347 1350 iop_desc_init_null_xor(grp_start, 2, 0);
1348 1351 iop_desc_set_byte_count(grp_start, iop_chan, 0);
1349 1352 iop_desc_set_dest_addr(grp_start, iop_chan, 0);
include/linux/dmaengine.h
Diff comments   View file @ 636bdea
... ... @@ -95,12 +95,17 @@
95 95 #define DMA_TX_TYPE_END (DMA_INTERRUPT + 1)
96 96  
97 97 /**
98   - * enum dma_prep_flags - DMA flags to augment operation preparation
  98 + * enum dma_ctrl_flags - DMA flags to augment operation preparation,
  99 + * control completion, and communicate status.
99 100 * @DMA_PREP_INTERRUPT - trigger an interrupt (callback) upon completion of
100 101 * this transaction
  102 + * @DMA_CTRL_ACK - the descriptor cannot be reused until the client
  103 + * acknowledges receipt, i.e. has has a chance to establish any
  104 + * dependency chains
101 105 */
102   -enum dma_prep_flags {
  106 +enum dma_ctrl_flags {
103 107 DMA_PREP_INTERRUPT = (1 << 0),
  108 + DMA_CTRL_ACK = (1 << 1),
104 109 };
105 110  
106 111 /**
... ... @@ -211,8 +216,8 @@
211 216 * ---dma generic offload fields---
212 217 * @cookie: tracking cookie for this transaction, set to -EBUSY if
213 218 * this tx is sitting on a dependency list
214   - * @ack: the descriptor can not be reused until the client acknowledges
215   - * receipt, i.e. has has a chance to establish any dependency chains
  219 + * @flags: flags to augment operation preparation, control completion, and
  220 + * communicate status
216 221 * @phys: physical address of the descriptor
217 222 * @tx_list: driver common field for operations that require multiple
218 223 * descriptors
... ... @@ -227,7 +232,7 @@
227 232 */
228 233 struct dma_async_tx_descriptor {
229 234 dma_cookie_t cookie;
230   - int ack;
  235 + enum dma_ctrl_flags flags; /* not a 'long' to pack with cookie */
231 236 dma_addr_t phys;
232 237 struct list_head tx_list;
233 238 struct dma_chan *chan;
... ... @@ -290,7 +295,7 @@
290 295 struct dma_chan *chan, dma_addr_t dest, int value, size_t len,
291 296 unsigned long flags);
292 297 struct dma_async_tx_descriptor *(*device_prep_dma_interrupt)(
293   - struct dma_chan *chan);
  298 + struct dma_chan *chan, unsigned long flags);
294 299  
295 300 enum dma_status (*device_is_tx_complete)(struct dma_chan *chan,
296 301 dma_cookie_t cookie, dma_cookie_t *last,
... ... @@ -316,7 +321,13 @@
316 321 static inline void
317 322 async_tx_ack(struct dma_async_tx_descriptor *tx)
318 323 {
319   - tx->ack = 1;
  324 + tx->flags |= DMA_CTRL_ACK;
  325 +}
  326 +
  327 +static inline int
  328 +async_tx_test_ack(struct dma_async_tx_descriptor *tx)
  329 +{
  330 + return tx->flags & DMA_CTRL_ACK;
320 331 }
321 332  
322 333 #define first_dma_cap(mask) __first_dma_cap(&(mask))