Blame view
crypto/async_tx/async_xor.c
9.87 KB
9bc89cd82 async_tx: add the... |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 |
/* * xor offload engine api * * Copyright © 2006, Intel Corporation. * * Dan Williams <dan.j.williams@intel.com> * * with architecture considerations by: * Neil Brown <neilb@suse.de> * Jeff Garzik <jeff@garzik.org> * * This program is free software; you can redistribute it and/or modify it * under the terms and conditions of the GNU General Public License, * version 2, as published by the Free Software Foundation. * * This program is distributed in the hope it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for * more details. * * You should have received a copy of the GNU General Public License along with * this program; if not, write to the Free Software Foundation, Inc., * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. * */ #include <linux/kernel.h> #include <linux/interrupt.h> |
4bb33cc89 crypto: add modul... |
28 |
#include <linux/module.h> |
9bc89cd82 async_tx: add the... |
29 30 31 32 |
#include <linux/mm.h> #include <linux/dma-mapping.h> #include <linux/raid/xor.h> #include <linux/async_tx.h> |
06164f319 async_tx: provide... |
33 34 |
/* do_async_xor - dma map the pages and perform the xor with an engine */ static __async_inline struct dma_async_tx_descriptor * |
1e55db2d6 async_tx: ensure ... |
35 |
do_async_xor(struct dma_chan *chan, struct page *dest, struct page **src_list, |
04ce9ab38 async_xor: permit... |
36 |
unsigned int offset, int src_cnt, size_t len, dma_addr_t *dma_src, |
a08abd8ca async_tx: structi... |
37 |
struct async_submit_ctl *submit) |
9bc89cd82 async_tx: add the... |
38 |
{ |
1e55db2d6 async_tx: ensure ... |
39 |
struct dma_device *dma = chan->device; |
1e55db2d6 async_tx: ensure ... |
40 41 |
struct dma_async_tx_descriptor *tx = NULL; int src_off = 0; |
9bc89cd82 async_tx: add the... |
42 |
int i; |
a08abd8ca async_tx: structi... |
43 44 45 |
dma_async_tx_callback cb_fn_orig = submit->cb_fn; void *cb_param_orig = submit->cb_param; enum async_tx_flags flags_orig = submit->flags; |
1e55db2d6 async_tx: ensure ... |
46 |
enum dma_ctrl_flags dma_flags; |
b2141e695 raid6/async_tx: h... |
47 |
int xor_src_cnt = 0; |
1e55db2d6 async_tx: ensure ... |
48 |
dma_addr_t dma_dest; |
9bc89cd82 async_tx: add the... |
49 |
|
a06d568f7 async_xor: dma_ma... |
50 51 52 53 |
/* map the dest bidrectional in case it is re-used as a source */ dma_dest = dma_map_page(dma->dev, dest, offset, len, DMA_BIDIRECTIONAL); for (i = 0; i < src_cnt; i++) { /* only map the dest once */ |
b2141e695 raid6/async_tx: h... |
54 55 |
if (!src_list[i]) continue; |
a06d568f7 async_xor: dma_ma... |
56 |
if (unlikely(src_list[i] == dest)) { |
b2141e695 raid6/async_tx: h... |
57 |
dma_src[xor_src_cnt++] = dma_dest; |
a06d568f7 async_xor: dma_ma... |
58 59 |
continue; } |
b2141e695 raid6/async_tx: h... |
60 61 |
dma_src[xor_src_cnt++] = dma_map_page(dma->dev, src_list[i], offset, len, DMA_TO_DEVICE); |
a06d568f7 async_xor: dma_ma... |
62 |
} |
b2141e695 raid6/async_tx: h... |
63 |
src_cnt = xor_src_cnt; |
0036731c8 async_tx: kill tx... |
64 |
|
1e55db2d6 async_tx: ensure ... |
65 |
while (src_cnt) { |
a08abd8ca async_tx: structi... |
66 |
submit->flags = flags_orig; |
1e55db2d6 async_tx: ensure ... |
67 |
dma_flags = 0; |
b2f46fd8e async_tx: add sup... |
68 |
xor_src_cnt = min(src_cnt, (int)dma->max_xor); |
1e55db2d6 async_tx: ensure ... |
69 70 71 72 73 |
/* if we are submitting additional xors, leave the chain open, * clear the callback parameters, and leave the destination * buffer mapped */ if (src_cnt > xor_src_cnt) { |
a08abd8ca async_tx: structi... |
74 |
submit->flags &= ~ASYNC_TX_ACK; |
0403e3827 dmaengine: add fe... |
75 |
submit->flags |= ASYNC_TX_FENCE; |
1e55db2d6 async_tx: ensure ... |
76 |
dma_flags = DMA_COMPL_SKIP_DEST_UNMAP; |
a08abd8ca async_tx: structi... |
77 78 |
submit->cb_fn = NULL; submit->cb_param = NULL; |
1e55db2d6 async_tx: ensure ... |
79 |
} else { |
a08abd8ca async_tx: structi... |
80 81 |
submit->cb_fn = cb_fn_orig; submit->cb_param = cb_param_orig; |
1e55db2d6 async_tx: ensure ... |
82 |
} |
a08abd8ca async_tx: structi... |
83 |
if (submit->cb_fn) |
1e55db2d6 async_tx: ensure ... |
84 |
dma_flags |= DMA_PREP_INTERRUPT; |
0403e3827 dmaengine: add fe... |
85 86 |
if (submit->flags & ASYNC_TX_FENCE) dma_flags |= DMA_PREP_FENCE; |
1e55db2d6 async_tx: ensure ... |
87 88 89 90 91 92 |
/* Since we have clobbered the src_list we are committed * to doing this asynchronously. Drivers force forward progress * in case they can not provide a descriptor */ tx = dma->device_prep_dma_xor(chan, dma_dest, &dma_src[src_off], xor_src_cnt, len, dma_flags); |
669ab0b21 async_tx: fix han... |
93 |
if (unlikely(!tx)) |
a08abd8ca async_tx: structi... |
94 |
async_tx_quiesce(&submit->depend_tx); |
0036731c8 async_tx: kill tx... |
95 |
|
25985edce Fix common misspe... |
96 |
/* spin wait for the preceding transactions to complete */ |
669ab0b21 async_tx: fix han... |
97 98 |
while (unlikely(!tx)) { dma_async_issue_pending(chan); |
1e55db2d6 async_tx: ensure ... |
99 100 101 102 |
tx = dma->device_prep_dma_xor(chan, dma_dest, &dma_src[src_off], xor_src_cnt, len, dma_flags); |
669ab0b21 async_tx: fix han... |
103 |
} |
9bc89cd82 async_tx: add the... |
104 |
|
a08abd8ca async_tx: structi... |
105 106 |
async_tx_submit(chan, tx, submit); submit->depend_tx = tx; |
1e55db2d6 async_tx: ensure ... |
107 108 109 110 111 112 113 114 115 116 117 118 |
if (src_cnt > xor_src_cnt) { /* drop completed sources */ src_cnt -= xor_src_cnt; src_off += xor_src_cnt; /* use the intermediate result a source */ dma_src[--src_off] = dma_dest; src_cnt++; } else break; } |
0036731c8 async_tx: kill tx... |
119 120 |
return tx; |
9bc89cd82 async_tx: add the... |
121 122 123 124 |
} static void do_sync_xor(struct page *dest, struct page **src_list, unsigned int offset, |
a08abd8ca async_tx: structi... |
125 |
int src_cnt, size_t len, struct async_submit_ctl *submit) |
9bc89cd82 async_tx: add the... |
126 |
{ |
9bc89cd82 async_tx: add the... |
127 |
int i; |
b2141e695 raid6/async_tx: h... |
128 |
int xor_src_cnt = 0; |
1e55db2d6 async_tx: ensure ... |
129 130 |
int src_off = 0; void *dest_buf; |
04ce9ab38 async_xor: permit... |
131 |
void **srcs; |
9bc89cd82 async_tx: add the... |
132 |
|
04ce9ab38 async_xor: permit... |
133 134 135 136 137 138 |
if (submit->scribble) srcs = submit->scribble; else srcs = (void **) src_list; /* convert to buffer pointers */ |
9bc89cd82 async_tx: add the... |
139 |
for (i = 0; i < src_cnt; i++) |
b2141e695 raid6/async_tx: h... |
140 141 142 |
if (src_list[i]) srcs[xor_src_cnt++] = page_address(src_list[i]) + offset; src_cnt = xor_src_cnt; |
9bc89cd82 async_tx: add the... |
143 |
/* set destination address */ |
1e55db2d6 async_tx: ensure ... |
144 |
dest_buf = page_address(dest) + offset; |
9bc89cd82 async_tx: add the... |
145 |
|
a08abd8ca async_tx: structi... |
146 |
if (submit->flags & ASYNC_TX_XOR_ZERO_DST) |
1e55db2d6 async_tx: ensure ... |
147 |
memset(dest_buf, 0, len); |
9bc89cd82 async_tx: add the... |
148 |
|
1e55db2d6 async_tx: ensure ... |
149 150 151 152 153 154 155 156 157 |
while (src_cnt > 0) { /* process up to 'MAX_XOR_BLOCKS' sources */ xor_src_cnt = min(src_cnt, MAX_XOR_BLOCKS); xor_blocks(xor_src_cnt, len, dest_buf, &srcs[src_off]); /* drop completed sources */ src_cnt -= xor_src_cnt; src_off += xor_src_cnt; } |
9bc89cd82 async_tx: add the... |
158 |
|
a08abd8ca async_tx: structi... |
159 |
async_tx_sync_epilog(submit); |
9bc89cd82 async_tx: add the... |
160 161 162 163 |
} /** * async_xor - attempt to xor a set of blocks with a dma engine. |
9bc89cd82 async_tx: add the... |
164 |
* @dest: destination page |
a08abd8ca async_tx: structi... |
165 166 |
* @src_list: array of source pages * @offset: common src/dst offset to start transaction |
9bc89cd82 async_tx: add the... |
167 168 |
* @src_cnt: number of source pages * @len: length in bytes |
a08abd8ca async_tx: structi... |
169 170 171 172 173 174 175 176 177 178 179 180 181 |
* @submit: submission / completion modifiers * * honored flags: ASYNC_TX_ACK, ASYNC_TX_XOR_ZERO_DST, ASYNC_TX_XOR_DROP_DST * * xor_blocks always uses the dest as a source so the * ASYNC_TX_XOR_ZERO_DST flag must be set to not include dest data in * the calculation. The assumption with dma eninges is that they only * use the destination buffer as a source when it is explicity specified * in the source list. * * src_list note: if the dest is also a source it must be at index zero. * The contents of this array will be overwritten if a scribble region * is not specified. |
9bc89cd82 async_tx: add the... |
182 183 184 |
*/ struct dma_async_tx_descriptor * async_xor(struct page *dest, struct page **src_list, unsigned int offset, |
a08abd8ca async_tx: structi... |
185 |
int src_cnt, size_t len, struct async_submit_ctl *submit) |
9bc89cd82 async_tx: add the... |
186 |
{ |
a08abd8ca async_tx: structi... |
187 |
struct dma_chan *chan = async_tx_find_channel(submit, DMA_XOR, |
47437b2c9 async_tx: allow a... |
188 189 |
&dest, 1, src_list, src_cnt, len); |
04ce9ab38 async_xor: permit... |
190 |
dma_addr_t *dma_src = NULL; |
9bc89cd82 async_tx: add the... |
191 |
BUG_ON(src_cnt <= 1); |
04ce9ab38 async_xor: permit... |
192 193 194 195 |
if (submit->scribble) dma_src = submit->scribble; else if (sizeof(dma_addr_t) <= sizeof(struct page *)) dma_src = (dma_addr_t *) src_list; |
83544ae9f dmaengine, async_... |
196 |
if (dma_src && chan && is_dma_xor_aligned(chan->device, offset, 0, len)) { |
1e55db2d6 async_tx: ensure ... |
197 198 199 |
/* run the xor asynchronously */ pr_debug("%s (async): len: %zu ", __func__, len); |
9bc89cd82 async_tx: add the... |
200 |
|
1e55db2d6 async_tx: ensure ... |
201 |
return do_async_xor(chan, dest, src_list, offset, src_cnt, len, |
04ce9ab38 async_xor: permit... |
202 |
dma_src, submit); |
1e55db2d6 async_tx: ensure ... |
203 204 205 206 |
} else { /* run the xor synchronously */ pr_debug("%s (sync): len: %zu ", __func__, len); |
04ce9ab38 async_xor: permit... |
207 208 209 |
WARN_ONCE(chan, "%s: no space for dma address conversion ", __func__); |
9bc89cd82 async_tx: add the... |
210 |
|
1e55db2d6 async_tx: ensure ... |
211 212 |
/* in the sync case the dest is an implied source * (assumes the dest is the first source) |
9bc89cd82 async_tx: add the... |
213 |
*/ |
a08abd8ca async_tx: structi... |
214 |
if (submit->flags & ASYNC_TX_XOR_DROP_DST) { |
1e55db2d6 async_tx: ensure ... |
215 216 217 |
src_cnt--; src_list++; } |
9bc89cd82 async_tx: add the... |
218 |
|
1e55db2d6 async_tx: ensure ... |
219 |
/* wait for any prerequisite operations */ |
a08abd8ca async_tx: structi... |
220 |
async_tx_quiesce(&submit->depend_tx); |
9bc89cd82 async_tx: add the... |
221 |
|
a08abd8ca async_tx: structi... |
222 |
do_sync_xor(dest, src_list, offset, src_cnt, len, submit); |
9bc89cd82 async_tx: add the... |
223 |
|
1e55db2d6 async_tx: ensure ... |
224 |
return NULL; |
9bc89cd82 async_tx: add the... |
225 |
} |
9bc89cd82 async_tx: add the... |
226 227 228 229 230 231 232 233 234 |
} EXPORT_SYMBOL_GPL(async_xor); static int page_is_zero(struct page *p, unsigned int offset, size_t len) { char *a = page_address(p) + offset; return ((*(u32 *) a) == 0 && memcmp(a, a + 4, len - 4) == 0); } |
7b3cc2b1f async_tx: build-t... |
235 236 237 238 239 240 241 242 243 244 |
static inline struct dma_chan * xor_val_chan(struct async_submit_ctl *submit, struct page *dest, struct page **src_list, int src_cnt, size_t len) { #ifdef CONFIG_ASYNC_TX_DISABLE_XOR_VAL_DMA return NULL; #endif return async_tx_find_channel(submit, DMA_XOR_VAL, &dest, 1, src_list, src_cnt, len); } |
9bc89cd82 async_tx: add the... |
245 |
/** |
099f53cb5 async_tx: rename ... |
246 |
* async_xor_val - attempt a xor parity check with a dma engine. |
9bc89cd82 async_tx: add the... |
247 |
* @dest: destination page used if the xor is performed synchronously |
a08abd8ca async_tx: structi... |
248 |
* @src_list: array of source pages |
9bc89cd82 async_tx: add the... |
249 250 251 252 |
* @offset: offset in pages to start transaction * @src_cnt: number of source pages * @len: length in bytes * @result: 0 if sum == 0 else non-zero |
a08abd8ca async_tx: structi... |
253 254 255 256 257 258 259 |
* @submit: submission / completion modifiers * * honored flags: ASYNC_TX_ACK * * src_list note: if the dest is also a source it must be at index zero. * The contents of this array will be overwritten if a scribble region * is not specified. |
9bc89cd82 async_tx: add the... |
260 261 |
*/ struct dma_async_tx_descriptor * |
a08abd8ca async_tx: structi... |
262 |
async_xor_val(struct page *dest, struct page **src_list, unsigned int offset, |
ad283ea4a async_tx: add sum... |
263 |
int src_cnt, size_t len, enum sum_check_flags *result, |
a08abd8ca async_tx: structi... |
264 |
struct async_submit_ctl *submit) |
9bc89cd82 async_tx: add the... |
265 |
{ |
7b3cc2b1f async_tx: build-t... |
266 |
struct dma_chan *chan = xor_val_chan(submit, dest, src_list, src_cnt, len); |
9bc89cd82 async_tx: add the... |
267 |
struct dma_device *device = chan ? chan->device : NULL; |
0036731c8 async_tx: kill tx... |
268 |
struct dma_async_tx_descriptor *tx = NULL; |
04ce9ab38 async_xor: permit... |
269 |
dma_addr_t *dma_src = NULL; |
9bc89cd82 async_tx: add the... |
270 271 |
BUG_ON(src_cnt <= 1); |
04ce9ab38 async_xor: permit... |
272 273 274 275 |
if (submit->scribble) dma_src = submit->scribble; else if (sizeof(dma_addr_t) <= sizeof(struct page *)) dma_src = (dma_addr_t *) src_list; |
83544ae9f dmaengine, async_... |
276 277 |
if (dma_src && device && src_cnt <= device->max_xor && is_dma_xor_aligned(device, offset, 0, len)) { |
0403e3827 dmaengine: add fe... |
278 |
unsigned long dma_prep_flags = 0; |
0036731c8 async_tx: kill tx... |
279 |
int i; |
9bc89cd82 async_tx: add the... |
280 |
|
3280ab3e8 async_tx: checkpa... |
281 282 |
pr_debug("%s: (async) len: %zu ", __func__, len); |
9bc89cd82 async_tx: add the... |
283 |
|
0403e3827 dmaengine: add fe... |
284 285 286 287 |
if (submit->cb_fn) dma_prep_flags |= DMA_PREP_INTERRUPT; if (submit->flags & ASYNC_TX_FENCE) dma_prep_flags |= DMA_PREP_FENCE; |
0036731c8 async_tx: kill tx... |
288 289 290 |
for (i = 0; i < src_cnt; i++) dma_src[i] = dma_map_page(device->dev, src_list[i], offset, len, DMA_TO_DEVICE); |
099f53cb5 async_tx: rename ... |
291 292 293 |
tx = device->device_prep_dma_xor_val(chan, dma_src, src_cnt, len, result, dma_prep_flags); |
669ab0b21 async_tx: fix han... |
294 |
if (unlikely(!tx)) { |
a08abd8ca async_tx: structi... |
295 |
async_tx_quiesce(&submit->depend_tx); |
0036731c8 async_tx: kill tx... |
296 |
|
e34a8ae79 async_tx: fix mis... |
297 |
while (!tx) { |
669ab0b21 async_tx: fix han... |
298 |
dma_async_issue_pending(chan); |
099f53cb5 async_tx: rename ... |
299 |
tx = device->device_prep_dma_xor_val(chan, |
0036731c8 async_tx: kill tx... |
300 |
dma_src, src_cnt, len, result, |
d4c56f97f async_tx: replace... |
301 |
dma_prep_flags); |
e34a8ae79 async_tx: fix mis... |
302 |
} |
9bc89cd82 async_tx: add the... |
303 |
} |
a08abd8ca async_tx: structi... |
304 |
async_tx_submit(chan, tx, submit); |
9bc89cd82 async_tx: add the... |
305 |
} else { |
a08abd8ca async_tx: structi... |
306 |
enum async_tx_flags flags_orig = submit->flags; |
9bc89cd82 async_tx: add the... |
307 |
|
3280ab3e8 async_tx: checkpa... |
308 309 |
pr_debug("%s: (sync) len: %zu ", __func__, len); |
04ce9ab38 async_xor: permit... |
310 311 312 313 |
WARN_ONCE(device && src_cnt <= device->max_xor, "%s: no space for dma address conversion ", __func__); |
9bc89cd82 async_tx: add the... |
314 |
|
a08abd8ca async_tx: structi... |
315 316 |
submit->flags |= ASYNC_TX_XOR_DROP_DST; submit->flags &= ~ASYNC_TX_ACK; |
9bc89cd82 async_tx: add the... |
317 |
|
a08abd8ca async_tx: structi... |
318 |
tx = async_xor(dest, src_list, offset, src_cnt, len, submit); |
9bc89cd82 async_tx: add the... |
319 |
|
d2c52b798 async_tx: export ... |
320 |
async_tx_quiesce(&tx); |
9bc89cd82 async_tx: add the... |
321 |
|
ad283ea4a async_tx: add sum... |
322 |
*result = !page_is_zero(dest, offset, len) << SUM_CHECK_P; |
9bc89cd82 async_tx: add the... |
323 |
|
a08abd8ca async_tx: structi... |
324 325 |
async_tx_sync_epilog(submit); submit->flags = flags_orig; |
9bc89cd82 async_tx: add the... |
326 327 328 329 |
} return tx; } |
099f53cb5 async_tx: rename ... |
330 |
EXPORT_SYMBOL_GPL(async_xor_val); |
9bc89cd82 async_tx: add the... |
331 |
|
9bc89cd82 async_tx: add the... |
332 333 334 |
MODULE_AUTHOR("Intel Corporation"); MODULE_DESCRIPTION("asynchronous xor/xor-zero-sum api"); MODULE_LICENSE("GPL"); |