Blame view
crypto/async_tx/async_pq.c
12.4 KB
e62d94910 treewide: Replace... |
1 |
// SPDX-License-Identifier: GPL-2.0-or-later |
b2f46fd8e async_tx: add sup... |
2 3 4 |
/* * Copyright(c) 2007 Yuri Tikhonov <yur@emcraft.com> * Copyright(c) 2009 Intel Corporation |
b2f46fd8e async_tx: add sup... |
5 6 7 |
*/ #include <linux/kernel.h> #include <linux/interrupt.h> |
4bb33cc89 crypto: add modul... |
8 |
#include <linux/module.h> |
b2f46fd8e async_tx: add sup... |
9 10 11 |
#include <linux/dma-mapping.h> #include <linux/raid/pq.h> #include <linux/async_tx.h> |
5a0e3ad6a include cleanup: ... |
12 |
#include <linux/gfp.h> |
b2f46fd8e async_tx: add sup... |
13 14 |
/** |
030b07720 async_pq: rename ... |
15 16 |
* pq_scribble_page - space to hold throwaway P or Q buffer for * synchronous gen_syndrome |
b2f46fd8e async_tx: add sup... |
17 |
*/ |
030b07720 async_pq: rename ... |
18 |
static struct page *pq_scribble_page; |
b2f46fd8e async_tx: add sup... |
19 |
|
b2f46fd8e async_tx: add sup... |
20 21 22 23 24 25 26 27 |
/* the struct page *blocks[] parameter passed to async_gen_syndrome() * and async_syndrome_val() contains the 'P' destination address at * blocks[disks-2] and the 'Q' destination address at blocks[disks-1] * * note: these are macros as they are used as lvalues */ #define P(b, d) (b[d-2]) #define Q(b, d) (b[d-1]) |
89a7e2f75 async_pq: Remove ... |
28 |
#define MAX_DISKS 255 |
b2f46fd8e async_tx: add sup... |
29 30 31 32 |
/** * do_async_gen_syndrome - asynchronously calculate P and/or Q */ static __async_inline struct dma_async_tx_descriptor * |
7476bd79f async_pq: convert... |
33 34 35 36 |
do_async_gen_syndrome(struct dma_chan *chan, const unsigned char *scfs, int disks, struct dmaengine_unmap_data *unmap, enum dma_ctrl_flags dma_flags, |
b2f46fd8e async_tx: add sup... |
37 38 39 40 |
struct async_submit_ctl *submit) { struct dma_async_tx_descriptor *tx = NULL; struct dma_device *dma = chan->device; |
b2f46fd8e async_tx: add sup... |
41 42 43 44 |
enum async_tx_flags flags_orig = submit->flags; dma_async_tx_callback cb_fn_orig = submit->cb_fn; dma_async_tx_callback cb_param_orig = submit->cb_param; int src_cnt = disks - 2; |
b2f46fd8e async_tx: add sup... |
45 46 47 |
unsigned short pq_src_cnt; dma_addr_t dma_dest[2]; int src_off = 0; |
b2f46fd8e async_tx: add sup... |
48 |
|
b2f46fd8e async_tx: add sup... |
49 50 51 52 53 54 55 56 57 |
while (src_cnt > 0) { submit->flags = flags_orig; pq_src_cnt = min(src_cnt, dma_maxpq(dma, dma_flags)); /* if we are submitting additional pqs, leave the chain open, * clear the callback parameters, and leave the destination * buffers mapped */ if (src_cnt > pq_src_cnt) { submit->flags &= ~ASYNC_TX_ACK; |
0403e3827 dmaengine: add fe... |
58 |
submit->flags |= ASYNC_TX_FENCE; |
b2f46fd8e async_tx: add sup... |
59 60 61 |
submit->cb_fn = NULL; submit->cb_param = NULL; } else { |
b2f46fd8e async_tx: add sup... |
62 63 64 65 66 |
submit->cb_fn = cb_fn_orig; submit->cb_param = cb_param_orig; if (cb_fn_orig) dma_flags |= DMA_PREP_INTERRUPT; } |
baae03a0e async_tx: Fix DMA... |
67 68 |
if (submit->flags & ASYNC_TX_FENCE) dma_flags |= DMA_PREP_FENCE; |
b2f46fd8e async_tx: add sup... |
69 |
|
7476bd79f async_pq: convert... |
70 71 |
/* Drivers force forward progress in case they can not provide * a descriptor |
b2f46fd8e async_tx: add sup... |
72 73 |
*/ for (;;) { |
7476bd79f async_pq: convert... |
74 75 |
dma_dest[0] = unmap->addr[disks - 2]; dma_dest[1] = unmap->addr[disks - 1]; |
b2f46fd8e async_tx: add sup... |
76 |
tx = dma->device_prep_dma_pq(chan, dma_dest, |
7476bd79f async_pq: convert... |
77 |
&unmap->addr[src_off], |
b2f46fd8e async_tx: add sup... |
78 |
pq_src_cnt, |
7476bd79f async_pq: convert... |
79 |
&scfs[src_off], unmap->len, |
b2f46fd8e async_tx: add sup... |
80 81 82 83 84 85 |
dma_flags); if (likely(tx)) break; async_tx_quiesce(&submit->depend_tx); dma_async_issue_pending(chan); } |
7476bd79f async_pq: convert... |
86 |
dma_set_unmap(tx, unmap); |
b2f46fd8e async_tx: add sup... |
87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 |
async_tx_submit(chan, tx, submit); submit->depend_tx = tx; /* drop completed sources */ src_cnt -= pq_src_cnt; src_off += pq_src_cnt; dma_flags |= DMA_PREP_CONTINUE; } return tx; } /** * do_sync_gen_syndrome - synchronously calculate a raid6 syndrome */ static void do_sync_gen_syndrome(struct page **blocks, unsigned int offset, int disks, size_t len, struct async_submit_ctl *submit) { void **srcs; int i; |
584acdd49 md/raid5: activat... |
109 |
int start = -1, stop = disks - 3; |
b2f46fd8e async_tx: add sup... |
110 111 112 113 114 115 116 |
if (submit->scribble) srcs = submit->scribble; else srcs = (void **) blocks; for (i = 0; i < disks; i++) { |
5dd33c9a4 md/async: don't p... |
117 |
if (blocks[i] == NULL) { |
b2f46fd8e async_tx: add sup... |
118 |
BUG_ON(i > disks - 3); /* P or Q can't be zero */ |
5dd33c9a4 md/async: don't p... |
119 |
srcs[i] = (void*)raid6_empty_zero_page; |
584acdd49 md/raid5: activat... |
120 |
} else { |
b2f46fd8e async_tx: add sup... |
121 |
srcs[i] = page_address(blocks[i]) + offset; |
584acdd49 md/raid5: activat... |
122 123 124 125 126 127 |
if (i < disks - 2) { stop = i; if (start == -1) start = i; } } |
b2f46fd8e async_tx: add sup... |
128 |
} |
584acdd49 md/raid5: activat... |
129 130 131 132 133 134 |
if (submit->flags & ASYNC_TX_PQ_XOR_DST) { BUG_ON(!raid6_call.xor_syndrome); if (start >= 0) raid6_call.xor_syndrome(disks, start, stop, len, srcs); } else raid6_call.gen_syndrome(disks, len, srcs); |
b2f46fd8e async_tx: add sup... |
135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 |
async_tx_sync_epilog(submit); } /** * async_gen_syndrome - asynchronously calculate a raid6 syndrome * @blocks: source blocks from idx 0..disks-3, P @ disks-2 and Q @ disks-1 * @offset: common offset into each block (src and dest) to start transaction * @disks: number of blocks (including missing P or Q, see below) * @len: length of operation in bytes * @submit: submission/completion modifiers * * General note: This routine assumes a field of GF(2^8) with a * primitive polynomial of 0x11d and a generator of {02}. * * 'disks' note: callers can optionally omit either P or Q (but not * both) from the calculation by setting blocks[disks-2] or * blocks[disks-1] to NULL. When P or Q is omitted 'len' must be <= * PAGE_SIZE as a temporary buffer of this size is used in the * synchronous path. 'disks' always accounts for both destination |
5676470f0 async_pq: kill a ... |
154 155 156 157 |
* buffers. If any source buffers (blocks[i] where i < disks - 2) are * set to NULL those buffers will be replaced with the raid6_zero_page * in the synchronous path and omitted in the hardware-asynchronous * path. |
b2f46fd8e async_tx: add sup... |
158 159 160 161 162 163 164 165 166 167 |
*/ struct dma_async_tx_descriptor * async_gen_syndrome(struct page **blocks, unsigned int offset, int disks, size_t len, struct async_submit_ctl *submit) { int src_cnt = disks - 2; struct dma_chan *chan = async_tx_find_channel(submit, DMA_PQ, &P(blocks, disks), 2, blocks, src_cnt, len); struct dma_device *device = chan ? chan->device : NULL; |
7476bd79f async_pq: convert... |
168 |
struct dmaengine_unmap_data *unmap = NULL; |
b2f46fd8e async_tx: add sup... |
169 |
|
89a7e2f75 async_pq: Remove ... |
170 |
BUG_ON(disks > MAX_DISKS || !(P(blocks, disks) || Q(blocks, disks))); |
b2f46fd8e async_tx: add sup... |
171 |
|
7476bd79f async_pq: convert... |
172 |
if (device) |
b02bab6b0 async_tx: use GFP... |
173 |
unmap = dmaengine_get_unmap_data(device->dev, disks, GFP_NOWAIT); |
b2f46fd8e async_tx: add sup... |
174 |
|
584acdd49 md/raid5: activat... |
175 176 |
/* XORing P/Q is only implemented in software */ if (unmap && !(submit->flags & ASYNC_TX_PQ_XOR_DST) && |
b2f46fd8e async_tx: add sup... |
177 |
(src_cnt <= dma_maxpq(device, 0) || |
83544ae9f dmaengine, async_... |
178 179 |
dma_maxpq(device, DMA_PREP_CONTINUE) > 0) && is_dma_pq_aligned(device, offset, 0, len)) { |
7476bd79f async_pq: convert... |
180 181 |
struct dma_async_tx_descriptor *tx; enum dma_ctrl_flags dma_flags = 0; |
89a7e2f75 async_pq: Remove ... |
182 |
unsigned char coefs[MAX_DISKS]; |
7476bd79f async_pq: convert... |
183 |
int i, j; |
b2f46fd8e async_tx: add sup... |
184 185 186 187 |
/* run the p+q asynchronously */ pr_debug("%s: (async) disks: %d len: %zu ", __func__, disks, len); |
7476bd79f async_pq: convert... |
188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 |
/* convert source addresses being careful to collapse 'empty' * sources and update the coefficients accordingly */ unmap->len = len; for (i = 0, j = 0; i < src_cnt; i++) { if (blocks[i] == NULL) continue; unmap->addr[j] = dma_map_page(device->dev, blocks[i], offset, len, DMA_TO_DEVICE); coefs[j] = raid6_gfexp[i]; unmap->to_cnt++; j++; } /* * DMAs use destinations as sources, * so use BIDIRECTIONAL mapping */ unmap->bidi_cnt++; if (P(blocks, disks)) unmap->addr[j++] = dma_map_page(device->dev, P(blocks, disks), offset, len, DMA_BIDIRECTIONAL); else { unmap->addr[j++] = 0; dma_flags |= DMA_PREP_PQ_DISABLE_P; } unmap->bidi_cnt++; if (Q(blocks, disks)) unmap->addr[j++] = dma_map_page(device->dev, Q(blocks, disks), offset, len, DMA_BIDIRECTIONAL); else { unmap->addr[j++] = 0; dma_flags |= DMA_PREP_PQ_DISABLE_Q; } tx = do_async_gen_syndrome(chan, coefs, j, unmap, dma_flags, submit); dmaengine_unmap_put(unmap); return tx; |
b2f46fd8e async_tx: add sup... |
228 |
} |
7476bd79f async_pq: convert... |
229 |
dmaengine_unmap_put(unmap); |
b2f46fd8e async_tx: add sup... |
230 231 232 233 234 235 236 237 |
/* run the pq synchronously */ pr_debug("%s: (sync) disks: %d len: %zu ", __func__, disks, len); /* wait for any prerequisite operations */ async_tx_quiesce(&submit->depend_tx); if (!P(blocks, disks)) { |
030b07720 async_pq: rename ... |
238 |
P(blocks, disks) = pq_scribble_page; |
b2f46fd8e async_tx: add sup... |
239 240 241 |
BUG_ON(len + offset > PAGE_SIZE); } if (!Q(blocks, disks)) { |
030b07720 async_pq: rename ... |
242 |
Q(blocks, disks) = pq_scribble_page; |
b2f46fd8e async_tx: add sup... |
243 244 245 246 247 248 249 |
BUG_ON(len + offset > PAGE_SIZE); } do_sync_gen_syndrome(blocks, offset, disks, len, submit); return NULL; } EXPORT_SYMBOL_GPL(async_gen_syndrome); |
7b3cc2b1f async_tx: build-t... |
250 251 252 253 254 255 256 257 258 |
static inline struct dma_chan * pq_val_chan(struct async_submit_ctl *submit, struct page **blocks, int disks, size_t len) { #ifdef CONFIG_ASYNC_TX_DISABLE_PQ_VAL_DMA return NULL; #endif return async_tx_find_channel(submit, DMA_PQ_VAL, NULL, 0, blocks, disks, len); } |
b2f46fd8e async_tx: add sup... |
259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 |
/** * async_syndrome_val - asynchronously validate a raid6 syndrome * @blocks: source blocks from idx 0..disks-3, P @ disks-2 and Q @ disks-1 * @offset: common offset into each block (src and dest) to start transaction * @disks: number of blocks (including missing P or Q, see below) * @len: length of operation in bytes * @pqres: on val failure SUM_CHECK_P_RESULT and/or SUM_CHECK_Q_RESULT are set * @spare: temporary result buffer for the synchronous case * @submit: submission / completion modifiers * * The same notes from async_gen_syndrome apply to the 'blocks', * and 'disks' parameters of this routine. The synchronous path * requires a temporary result buffer and submit->scribble to be * specified. */ struct dma_async_tx_descriptor * async_syndrome_val(struct page **blocks, unsigned int offset, int disks, size_t len, enum sum_check_flags *pqres, struct page *spare, struct async_submit_ctl *submit) { |
7b3cc2b1f async_tx: build-t... |
279 |
struct dma_chan *chan = pq_val_chan(submit, blocks, disks, len); |
b2f46fd8e async_tx: add sup... |
280 281 |
struct dma_device *device = chan ? chan->device : NULL; struct dma_async_tx_descriptor *tx; |
89a7e2f75 async_pq: Remove ... |
282 |
unsigned char coefs[MAX_DISKS]; |
b2f46fd8e async_tx: add sup... |
283 |
enum dma_ctrl_flags dma_flags = submit->cb_fn ? DMA_PREP_INTERRUPT : 0; |
1786b943d async_pq_val: con... |
284 |
struct dmaengine_unmap_data *unmap = NULL; |
b2f46fd8e async_tx: add sup... |
285 |
|
89a7e2f75 async_pq: Remove ... |
286 |
BUG_ON(disks < 4 || disks > MAX_DISKS); |
b2f46fd8e async_tx: add sup... |
287 |
|
1786b943d async_pq_val: con... |
288 |
if (device) |
b02bab6b0 async_tx: use GFP... |
289 |
unmap = dmaengine_get_unmap_data(device->dev, disks, GFP_NOWAIT); |
b2f46fd8e async_tx: add sup... |
290 |
|
1786b943d async_pq_val: con... |
291 |
if (unmap && disks <= dma_maxpq(device, 0) && |
83544ae9f dmaengine, async_... |
292 |
is_dma_pq_aligned(device, offset, 0, len)) { |
b2f46fd8e async_tx: add sup... |
293 |
struct device *dev = device->dev; |
1786b943d async_pq_val: con... |
294 295 |
dma_addr_t pq[2]; int i, j = 0, src_cnt = 0; |
b2f46fd8e async_tx: add sup... |
296 297 298 299 |
pr_debug("%s: (async) disks: %d len: %zu ", __func__, disks, len); |
1786b943d async_pq_val: con... |
300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 |
unmap->len = len; for (i = 0; i < disks-2; i++) if (likely(blocks[i])) { unmap->addr[j] = dma_map_page(dev, blocks[i], offset, len, DMA_TO_DEVICE); coefs[j] = raid6_gfexp[i]; unmap->to_cnt++; src_cnt++; j++; } if (!P(blocks, disks)) { pq[0] = 0; |
b2f46fd8e async_tx: add sup... |
315 |
dma_flags |= DMA_PREP_PQ_DISABLE_P; |
1786b943d async_pq_val: con... |
316 |
} else { |
5676470f0 async_pq: kill a ... |
317 |
pq[0] = dma_map_page(dev, P(blocks, disks), |
b2141e695 raid6/async_tx: h... |
318 319 |
offset, len, DMA_TO_DEVICE); |
1786b943d async_pq_val: con... |
320 321 322 323 324 |
unmap->addr[j++] = pq[0]; unmap->to_cnt++; } if (!Q(blocks, disks)) { pq[1] = 0; |
b2f46fd8e async_tx: add sup... |
325 |
dma_flags |= DMA_PREP_PQ_DISABLE_Q; |
1786b943d async_pq_val: con... |
326 |
} else { |
5676470f0 async_pq: kill a ... |
327 |
pq[1] = dma_map_page(dev, Q(blocks, disks), |
b2141e695 raid6/async_tx: h... |
328 329 |
offset, len, DMA_TO_DEVICE); |
1786b943d async_pq_val: con... |
330 331 332 |
unmap->addr[j++] = pq[1]; unmap->to_cnt++; } |
b2141e695 raid6/async_tx: h... |
333 |
|
0403e3827 dmaengine: add fe... |
334 335 |
if (submit->flags & ASYNC_TX_FENCE) dma_flags |= DMA_PREP_FENCE; |
b2f46fd8e async_tx: add sup... |
336 |
for (;;) { |
1786b943d async_pq_val: con... |
337 338 |
tx = device->device_prep_dma_pq_val(chan, pq, unmap->addr, |
b2141e695 raid6/async_tx: h... |
339 340 |
src_cnt, coefs, |
b2f46fd8e async_tx: add sup... |
341 342 343 344 345 346 347 |
len, pqres, dma_flags); if (likely(tx)) break; async_tx_quiesce(&submit->depend_tx); dma_async_issue_pending(chan); } |
1786b943d async_pq_val: con... |
348 349 |
dma_set_unmap(tx, unmap); |
b2f46fd8e async_tx: add sup... |
350 |
async_tx_submit(chan, tx, submit); |
b2f46fd8e async_tx: add sup... |
351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 |
} else { struct page *p_src = P(blocks, disks); struct page *q_src = Q(blocks, disks); enum async_tx_flags flags_orig = submit->flags; dma_async_tx_callback cb_fn_orig = submit->cb_fn; void *scribble = submit->scribble; void *cb_param_orig = submit->cb_param; void *p, *q, *s; pr_debug("%s: (sync) disks: %d len: %zu ", __func__, disks, len); /* caller must provide a temporary result buffer and * allow the input parameters to be preserved */ BUG_ON(!spare || !scribble); /* wait for any prerequisite operations */ async_tx_quiesce(&submit->depend_tx); /* recompute p and/or q into the temporary buffer and then * check to see the result matches the current value */ tx = NULL; *pqres = 0; if (p_src) { init_async_submit(submit, ASYNC_TX_XOR_ZERO_DST, NULL, NULL, NULL, scribble); tx = async_xor(spare, blocks, offset, disks-2, len, submit); async_tx_quiesce(&tx); p = page_address(p_src) + offset; s = page_address(spare) + offset; *pqres |= !!memcmp(p, s, len) << SUM_CHECK_P; } if (q_src) { P(blocks, disks) = NULL; Q(blocks, disks) = spare; init_async_submit(submit, 0, NULL, NULL, NULL, scribble); tx = async_gen_syndrome(blocks, offset, disks, len, submit); async_tx_quiesce(&tx); q = page_address(q_src) + offset; s = page_address(spare) + offset; *pqres |= !!memcmp(q, s, len) << SUM_CHECK_Q; } /* restore P, Q and submit */ P(blocks, disks) = p_src; Q(blocks, disks) = q_src; submit->cb_fn = cb_fn_orig; submit->cb_param = cb_param_orig; submit->flags = flags_orig; async_tx_sync_epilog(submit); |
c84750906 async_pq_val: fix... |
406 |
tx = NULL; |
b2f46fd8e async_tx: add sup... |
407 |
} |
c84750906 async_pq_val: fix... |
408 409 410 |
dmaengine_unmap_put(unmap); return tx; |
b2f46fd8e async_tx: add sup... |
411 412 413 414 415 |
} EXPORT_SYMBOL_GPL(async_syndrome_val); static int __init async_pq_init(void) { |
030b07720 async_pq: rename ... |
416 |
pq_scribble_page = alloc_page(GFP_KERNEL); |
b2f46fd8e async_tx: add sup... |
417 |
|
030b07720 async_pq: rename ... |
418 |
if (pq_scribble_page) |
b2f46fd8e async_tx: add sup... |
419 420 421 422 423 424 425 426 427 428 |
return 0; pr_err("%s: failed to allocate required spare page ", __func__); return -ENOMEM; } static void __exit async_pq_exit(void) { |
95813b8fa mm/page_ref: add ... |
429 |
__free_page(pq_scribble_page); |
b2f46fd8e async_tx: add sup... |
430 431 432 433 434 435 436 |
} module_init(async_pq_init); module_exit(async_pq_exit); MODULE_DESCRIPTION("asynchronous raid6 syndrome generation/validation"); MODULE_LICENSE("GPL"); |