Commit 18d72668a3b49aed60ee23fdef64850b03385f5c
1 parent
bbfe0c547f
Exists in
v3.2_SMARCT335xPSP_04.06.00.11
and in
2 other branches
Add 0007-am33x-Create-driver-for-AES-crypto-module.patch that wasn't part of the…
… official PSP 04.06.00.11 release
Showing 1 changed file with 950 additions and 0 deletions Side-by-side Diff
drivers/crypto/omap4-aes.c
1 | +/* | |
2 | + * Cryptographic API. | |
3 | + * | |
4 | + * Support for OMAP AES HW acceleration. | |
5 | + * | |
6 | + * Copyright (c) 2010 Nokia Corporation | |
7 | + * Author: Dmitry Kasatkin <dmitry.kasatkin@nokia.com> | |
8 | + * | |
9 | + * This program is free software; you can redistribute it and/or modify | |
10 | + * it under the terms of the GNU General Public License version 2 as published | |
11 | + * by the Free Software Foundation. | |
12 | + * | |
13 | + */ | |
14 | +/* | |
15 | + * Copyright © 2011 Texas Instruments Incorporated | |
16 | + * Author: Herman Schuurman | |
17 | + * Change: July 2011 - Adapted the omap-aes.c driver to support Netra | |
18 | + * implementation of AES hardware accelerator. | |
19 | + */ | |
20 | +/* | |
21 | + * Copyright © 2011 Texas Instruments Incorporated | |
22 | + * Author: Greg Turner | |
23 | + * Change: November 2011 - Adapted for AM33x support HW accelerator. | |
24 | + */ | |
25 | + | |
26 | +//#define DEBUG | |
27 | + | |
28 | +#define pr_fmt(fmt) "%s: " fmt, __func__ | |
29 | + | |
30 | +#include <linux/err.h> | |
31 | +#include <linux/module.h> | |
32 | +#include <linux/init.h> | |
33 | +#include <linux/errno.h> | |
34 | +#include <linux/kernel.h> | |
35 | +#include <linux/clk.h> | |
36 | +#include <linux/platform_device.h> | |
37 | +#include <linux/scatterlist.h> | |
38 | +#include <linux/dma-mapping.h> | |
39 | +#include <linux/io.h> | |
40 | +#include <linux/crypto.h> | |
41 | +#include <linux/interrupt.h> | |
42 | +#include <crypto/scatterwalk.h> | |
43 | +#include <crypto/aes.h> | |
44 | + | |
45 | +#include <plat/cpu.h> | |
46 | +#include <plat/dma.h> | |
47 | +#include <mach/edma.h> | |
48 | +#include <mach/hardware.h> | |
49 | +#include "omap4.h" | |
50 | + | |
51 | +#define DEFAULT_TIMEOUT (5*HZ) | |
52 | + | |
53 | +#define FLAGS_MODE_MASK 0x000f | |
54 | +#define FLAGS_ENCRYPT BIT(0) | |
55 | +#define FLAGS_CBC BIT(1) | |
56 | +#define FLAGS_CTR BIT(2) | |
57 | +#define FLAGS_GIV BIT(3) | |
58 | + | |
59 | +#define FLAGS_INIT BIT(4) | |
60 | +#define FLAGS_FAST BIT(5) | |
61 | +#define FLAGS_BUSY BIT(6) | |
62 | + | |
63 | +struct omap4_aes_ctx { | |
64 | + struct omap4_aes_dev *dd; | |
65 | + | |
66 | + int keylen; | |
67 | + u32 key[AES_KEYSIZE_256 / sizeof(u32)]; | |
68 | + unsigned long flags; | |
69 | +}; | |
70 | + | |
71 | +struct omap4_aes_reqctx { | |
72 | + unsigned long mode; | |
73 | +}; | |
74 | + | |
75 | +#define AM33X_AES_QUEUE_LENGTH 1 | |
76 | +#define AM33X_AES_CACHE_SIZE 0 | |
77 | + | |
78 | +struct omap4_aes_dev { | |
79 | + struct list_head list; | |
80 | + unsigned long phys_base; | |
81 | + void __iomem *io_base; | |
82 | + struct clk *iclk; | |
83 | + struct omap4_aes_ctx *ctx; | |
84 | + struct device *dev; | |
85 | + unsigned long flags; | |
86 | + int err; | |
87 | + | |
88 | + spinlock_t lock; | |
89 | + struct crypto_queue queue; | |
90 | + | |
91 | + struct tasklet_struct done_task; | |
92 | + struct tasklet_struct queue_task; | |
93 | + | |
94 | + struct ablkcipher_request *req; | |
95 | + size_t total; | |
96 | + struct scatterlist *in_sg; | |
97 | + size_t in_offset; | |
98 | + struct scatterlist *out_sg; | |
99 | + size_t out_offset; | |
100 | + | |
101 | + size_t buflen; | |
102 | + void *buf_in; | |
103 | + size_t dma_size; | |
104 | + int dma_in; | |
105 | + int dma_lch_in; | |
106 | + dma_addr_t dma_addr_in; | |
107 | + void *buf_out; | |
108 | + int dma_out; | |
109 | + int dma_lch_out; | |
110 | + dma_addr_t dma_addr_out; | |
111 | +}; | |
112 | + | |
113 | +/* keep registered devices data here */ | |
114 | +static LIST_HEAD(dev_list); | |
115 | +static DEFINE_SPINLOCK(list_lock); | |
116 | + | |
117 | +static inline u32 omap4_aes_read(struct omap4_aes_dev *dd, u32 offset) | |
118 | +{ | |
119 | + return __raw_readl(dd->io_base + offset); | |
120 | +} | |
121 | + | |
122 | +static inline void omap4_aes_write(struct omap4_aes_dev *dd, u32 offset, | |
123 | + u32 value) | |
124 | +{ | |
125 | + __raw_writel(value, dd->io_base + offset); | |
126 | +} | |
127 | + | |
128 | +static inline void omap4_aes_write_mask(struct omap4_aes_dev *dd, u32 offset, | |
129 | + u32 value, u32 mask) | |
130 | +{ | |
131 | + u32 val; | |
132 | + | |
133 | + val = omap4_aes_read(dd, offset); | |
134 | + val &= ~mask; | |
135 | + val |= value; | |
136 | + omap4_aes_write(dd, offset, val); | |
137 | +} | |
138 | + | |
139 | +static void omap4_aes_write_n(struct omap4_aes_dev *dd, u32 offset, | |
140 | + u32 *value, int count) | |
141 | +{ | |
142 | + for (; count--; value++, offset += 4) | |
143 | + omap4_aes_write(dd, offset, *value); | |
144 | +} | |
145 | + | |
146 | +static int omap4_aes_hw_init(struct omap4_aes_dev *dd) | |
147 | +{ | |
148 | + /* | |
149 | + * clocks are enabled when request starts and disabled when finished. | |
150 | + * It may be long delays between requests. | |
151 | + * Device might go to off mode to save power. | |
152 | + */ | |
153 | + clk_enable(dd->iclk); | |
154 | + omap4_aes_write(dd, AES_REG_SYSCFG, 0); | |
155 | + | |
156 | + if (!(dd->flags & FLAGS_INIT)) { | |
157 | + dd->flags |= FLAGS_INIT; | |
158 | + dd->err = 0; | |
159 | + } | |
160 | + | |
161 | + return 0; | |
162 | +} | |
163 | + | |
164 | +static int omap4_aes_write_ctrl(struct omap4_aes_dev *dd) | |
165 | +{ | |
166 | + unsigned int key32; | |
167 | + int i, err; | |
168 | + u32 val, mask; | |
169 | + | |
170 | + err = omap4_aes_hw_init(dd); | |
171 | + if (err) | |
172 | + return err; | |
173 | + | |
174 | + pr_debug("Set key\n"); | |
175 | + key32 = dd->ctx->keylen / sizeof(u32); | |
176 | + | |
177 | + /* set a key */ | |
178 | + for (i = 0; i < key32; i++) { | |
179 | + omap4_aes_write(dd, AES_REG_KEY1(i), | |
180 | + __le32_to_cpu(dd->ctx->key[i])); | |
181 | + } | |
182 | + | |
183 | + if ((dd->flags & (FLAGS_CBC | FLAGS_CTR)) && dd->req->info) | |
184 | + omap4_aes_write_n(dd, AES_REG_IV(0), dd->req->info, 4); | |
185 | + | |
186 | + val = FLD_VAL(((dd->ctx->keylen >> 3) - 1), 4, 3); | |
187 | + if (dd->flags & FLAGS_CBC) | |
188 | + val |= AES_REG_CTRL_CBC; | |
189 | + else if (dd->flags & FLAGS_CTR) | |
190 | + val |= AES_REG_CTRL_CTR | AES_REG_CTRL_CTR_WIDTH_32; | |
191 | + if (dd->flags & FLAGS_ENCRYPT) | |
192 | + val |= AES_REG_CTRL_DIRECTION; | |
193 | + | |
194 | + mask = AES_REG_CTRL_CBC | AES_REG_CTRL_CTR | AES_REG_CTRL_DIRECTION | | |
195 | + AES_REG_CTRL_KEY_SIZE_MASK | AES_REG_CTRL_CTR_WIDTH_MASK; | |
196 | + | |
197 | + omap4_aes_write_mask(dd, AES_REG_CTRL, val, mask); | |
198 | + | |
199 | + return 0; | |
200 | +} | |
201 | + | |
202 | +static struct omap4_aes_dev *omap4_aes_find_dev(struct omap4_aes_ctx *ctx) | |
203 | +{ | |
204 | + struct omap4_aes_dev *dd = NULL, *tmp; | |
205 | + | |
206 | + spin_lock_bh(&list_lock); | |
207 | + if (!ctx->dd) { | |
208 | + list_for_each_entry(tmp, &dev_list, list) { | |
209 | + /* FIXME: take fist available aes core */ | |
210 | + dd = tmp; | |
211 | + break; | |
212 | + } | |
213 | + ctx->dd = dd; | |
214 | + } else { | |
215 | + /* already found before */ | |
216 | + dd = ctx->dd; | |
217 | + } | |
218 | + spin_unlock_bh(&list_lock); | |
219 | + | |
220 | + return dd; | |
221 | +} | |
222 | + | |
223 | +static void omap4_aes_dma_callback(unsigned int lch, u16 ch_status, void *data) | |
224 | +{ | |
225 | + struct omap4_aes_dev *dd = data; | |
226 | + | |
227 | + edma_stop(lch); | |
228 | + | |
229 | + if (ch_status != DMA_COMPLETE) { | |
230 | + pr_err("omap4-aes DMA error status: 0x%hx\n", ch_status); | |
231 | + dd->err = -EIO; | |
232 | + dd->flags &= ~FLAGS_INIT; /* request to re-initialize */ | |
233 | + } else if (lch == dd->dma_lch_in) { | |
234 | + return; | |
235 | + } | |
236 | + | |
237 | + /* dma_lch_out - completed */ | |
238 | + tasklet_schedule(&dd->done_task); | |
239 | +} | |
240 | + | |
241 | +static int omap4_aes_dma_init(struct omap4_aes_dev *dd) | |
242 | +{ | |
243 | + int err = -ENOMEM; | |
244 | + | |
245 | + dd->dma_lch_out = -1; | |
246 | + dd->dma_lch_in = -1; | |
247 | + | |
248 | + dd->buf_in = (void *)__get_free_pages(GFP_KERNEL, AM33X_AES_CACHE_SIZE); | |
249 | + dd->buf_out = (void *)__get_free_pages(GFP_KERNEL, AM33X_AES_CACHE_SIZE); | |
250 | + dd->buflen = PAGE_SIZE << AM33X_AES_CACHE_SIZE; | |
251 | + dd->buflen &= ~(AES_BLOCK_SIZE - 1); | |
252 | + | |
253 | + if (!dd->buf_in || !dd->buf_out) { | |
254 | + dev_err(dd->dev, "unable to alloc pages.\n"); | |
255 | + goto err_alloc; | |
256 | + } | |
257 | + | |
258 | + /* MAP here */ | |
259 | + dd->dma_addr_in = dma_map_single(dd->dev, dd->buf_in, dd->buflen, | |
260 | + DMA_TO_DEVICE); | |
261 | + if (dma_mapping_error(dd->dev, dd->dma_addr_in)) { | |
262 | + dev_err(dd->dev, "dma %d bytes error\n", dd->buflen); | |
263 | + err = -EINVAL; | |
264 | + goto err_map_in; | |
265 | + } | |
266 | + | |
267 | + dd->dma_addr_out = dma_map_single(dd->dev, dd->buf_out, dd->buflen, | |
268 | + DMA_FROM_DEVICE); | |
269 | + if (dma_mapping_error(dd->dev, dd->dma_addr_out)) { | |
270 | + dev_err(dd->dev, "dma %d bytes error\n", dd->buflen); | |
271 | + err = -EINVAL; | |
272 | + goto err_map_out; | |
273 | + } | |
274 | + | |
275 | + dd->dma_lch_in = edma_alloc_channel(dd->dma_in, omap4_aes_dma_callback, | |
276 | + dd, EVENTQ_DEFAULT); | |
277 | + | |
278 | + if (dd->dma_lch_in < 0) { | |
279 | + dev_err(dd->dev, "Unable to request DMA channel\n"); | |
280 | + goto err_dma_in; | |
281 | + } | |
282 | + | |
283 | + dd->dma_lch_out = edma_alloc_channel(dd->dma_out, omap4_aes_dma_callback, dd, EVENTQ_2); | |
284 | + | |
285 | + if (dd->dma_lch_out < 0) { | |
286 | + dev_err(dd->dev, "Unable to request DMA channel\n"); | |
287 | + goto err_dma_out; | |
288 | + } | |
289 | + | |
290 | + return 0; | |
291 | + | |
292 | +err_dma_out: | |
293 | + edma_free_channel(dd->dma_lch_in); | |
294 | +err_dma_in: | |
295 | + dma_unmap_single(dd->dev, dd->dma_addr_out, dd->buflen, | |
296 | + DMA_FROM_DEVICE); | |
297 | +err_map_out: | |
298 | + dma_unmap_single(dd->dev, dd->dma_addr_in, dd->buflen, DMA_TO_DEVICE); | |
299 | +err_map_in: | |
300 | + free_pages((unsigned long)dd->buf_out, AM33X_AES_CACHE_SIZE); | |
301 | + free_pages((unsigned long)dd->buf_in, AM33X_AES_CACHE_SIZE); | |
302 | +err_alloc: | |
303 | + if (err) | |
304 | + pr_err("error: %d\n", err); | |
305 | + return err; | |
306 | +} | |
307 | + | |
308 | +static void omap4_aes_dma_cleanup(struct omap4_aes_dev *dd) | |
309 | +{ | |
310 | + edma_free_channel(dd->dma_lch_out); | |
311 | + edma_free_channel(dd->dma_lch_in); | |
312 | + dma_unmap_single(dd->dev, dd->dma_addr_out, dd->buflen, | |
313 | + DMA_FROM_DEVICE); | |
314 | + dma_unmap_single(dd->dev, dd->dma_addr_in, dd->buflen, DMA_TO_DEVICE); | |
315 | + free_pages((unsigned long)dd->buf_out, AM33X_AES_CACHE_SIZE); | |
316 | + free_pages((unsigned long)dd->buf_in, AM33X_AES_CACHE_SIZE); | |
317 | +} | |
318 | + | |
319 | +static void sg_copy_buf(void *buf, struct scatterlist *sg, | |
320 | + unsigned int start, unsigned int nbytes, int out) | |
321 | +{ | |
322 | + struct scatter_walk walk; | |
323 | + | |
324 | + if (!nbytes) | |
325 | + return; | |
326 | + | |
327 | + scatterwalk_start(&walk, sg); | |
328 | + scatterwalk_advance(&walk, start); | |
329 | + scatterwalk_copychunks(buf, &walk, nbytes, out); | |
330 | + scatterwalk_done(&walk, out, 0); | |
331 | +} | |
332 | + | |
333 | +static int sg_copy(struct scatterlist **sg, size_t *offset, void *buf, | |
334 | + size_t buflen, size_t total, int out) | |
335 | +{ | |
336 | + unsigned int count, off = 0; | |
337 | + | |
338 | + while (buflen && total) { | |
339 | + count = min((*sg)->length - *offset, total); | |
340 | + count = min(count, buflen); | |
341 | + | |
342 | + if (!count) | |
343 | + return off; | |
344 | + | |
345 | + /* | |
346 | + * buflen and total are AES_BLOCK_SIZE size aligned, | |
347 | + * so count should be also aligned | |
348 | + */ | |
349 | + | |
350 | + sg_copy_buf(buf + off, *sg, *offset, count, out); | |
351 | + | |
352 | + off += count; | |
353 | + buflen -= count; | |
354 | + *offset += count; | |
355 | + total -= count; | |
356 | + | |
357 | + if (*offset == (*sg)->length) { | |
358 | + *sg = sg_next(*sg); | |
359 | + if (*sg) | |
360 | + *offset = 0; | |
361 | + else | |
362 | + total = 0; | |
363 | + } | |
364 | + } | |
365 | + | |
366 | + return off; | |
367 | +} | |
368 | + | |
369 | +static int omap4_aes_crypt_dma(struct crypto_tfm *tfm, dma_addr_t dma_addr_in, | |
370 | + dma_addr_t dma_addr_out, int length) | |
371 | +{ | |
372 | + struct omap4_aes_ctx *ctx = crypto_tfm_ctx(tfm); | |
373 | + struct omap4_aes_dev *dd = ctx->dd; | |
374 | + int nblocks; | |
375 | + struct edmacc_param p_ram; | |
376 | + | |
377 | + pr_debug("len: %d\n", length); | |
378 | + | |
379 | + dd->dma_size = length; | |
380 | + | |
381 | + if (!(dd->flags & FLAGS_FAST)) | |
382 | + dma_sync_single_for_device(dd->dev, dma_addr_in, length, | |
383 | + DMA_TO_DEVICE); | |
384 | + | |
385 | + nblocks = DIV_ROUND_UP(length, AES_BLOCK_SIZE); | |
386 | + | |
387 | + /* EDMA IN */ | |
388 | + p_ram.opt = TCINTEN | | |
389 | + EDMA_TCC(EDMA_CHAN_SLOT(dd->dma_lch_in)); | |
390 | + p_ram.src = dma_addr_in; | |
391 | + p_ram.a_b_cnt = AES_BLOCK_SIZE | nblocks << 16; | |
392 | + p_ram.dst = dd->phys_base + AES_REG_DATA; | |
393 | + p_ram.src_dst_bidx = AES_BLOCK_SIZE; | |
394 | + p_ram.link_bcntrld = 1 << 16 | 0xFFFF; | |
395 | + p_ram.src_dst_cidx = 0; | |
396 | + p_ram.ccnt = 1; | |
397 | + edma_write_slot(dd->dma_lch_in, &p_ram); | |
398 | + | |
399 | + /* EDMA OUT */ | |
400 | + p_ram.opt = TCINTEN | | |
401 | + EDMA_TCC(EDMA_CHAN_SLOT(dd->dma_lch_out)); | |
402 | + p_ram.src = dd->phys_base + AES_REG_DATA; | |
403 | + p_ram.dst = dma_addr_out; | |
404 | + p_ram.src_dst_bidx = AES_BLOCK_SIZE << 16; | |
405 | + edma_write_slot(dd->dma_lch_out, &p_ram); | |
406 | + | |
407 | + edma_start(dd->dma_lch_in); | |
408 | + edma_start(dd->dma_lch_out); | |
409 | + | |
410 | + /* write data length info out */ | |
411 | + omap4_aes_write(dd, AES_REG_LENGTH_N(0), length); | |
412 | + omap4_aes_write(dd, AES_REG_LENGTH_N(1), 0); | |
413 | + /* start DMA or disable idle mode */ | |
414 | + omap4_aes_write_mask(dd, AES_REG_SYSCFG, | |
415 | + AES_REG_SYSCFG_DREQ_DATA_OUT_EN | AES_REG_SYSCFG_DREQ_DATA_IN_EN, | |
416 | + AES_REG_SYSCFG_DREQ_MASK); | |
417 | + | |
418 | + return 0; | |
419 | +} | |
420 | + | |
421 | +static int omap4_aes_crypt_dma_start(struct omap4_aes_dev *dd) | |
422 | +{ | |
423 | + struct crypto_tfm *tfm = crypto_ablkcipher_tfm( | |
424 | + crypto_ablkcipher_reqtfm(dd->req)); | |
425 | + int err, fast = 0, in, out; | |
426 | + size_t count; | |
427 | + dma_addr_t addr_in, addr_out; | |
428 | + | |
429 | + pr_debug("total: %d\n", dd->total); | |
430 | + | |
431 | + if (sg_is_last(dd->in_sg) && sg_is_last(dd->out_sg)) { | |
432 | + /* check for alignment */ | |
433 | + in = IS_ALIGNED((u32)dd->in_sg->offset, sizeof(u32)); | |
434 | + out = IS_ALIGNED((u32)dd->out_sg->offset, sizeof(u32)); | |
435 | + | |
436 | + fast = in && out; | |
437 | + } | |
438 | + | |
439 | + if (fast) { | |
440 | + count = min(dd->total, sg_dma_len(dd->in_sg)); | |
441 | + count = min(count, sg_dma_len(dd->out_sg)); | |
442 | + | |
443 | + if (count != dd->total) { | |
444 | + pr_err("request length != buffer length\n"); | |
445 | + return -EINVAL; | |
446 | + } | |
447 | + | |
448 | + pr_debug("fast\n"); | |
449 | + | |
450 | + err = dma_map_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE); | |
451 | + if (!err) { | |
452 | + dev_err(dd->dev, "dma_map_sg() error\n"); | |
453 | + return -EINVAL; | |
454 | + } | |
455 | + | |
456 | + err = dma_map_sg(dd->dev, dd->out_sg, 1, DMA_FROM_DEVICE); | |
457 | + if (!err) { | |
458 | + dev_err(dd->dev, "dma_map_sg() error\n"); | |
459 | + dma_unmap_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE); | |
460 | + return -EINVAL; | |
461 | + } | |
462 | + | |
463 | + addr_in = sg_dma_address(dd->in_sg); | |
464 | + addr_out = sg_dma_address(dd->out_sg); | |
465 | + | |
466 | + dd->flags |= FLAGS_FAST; | |
467 | + | |
468 | + } else { | |
469 | + /* use cache buffers */ | |
470 | + count = sg_copy(&dd->in_sg, &dd->in_offset, dd->buf_in, | |
471 | + dd->buflen, dd->total, 0); | |
472 | + | |
473 | + addr_in = dd->dma_addr_in; | |
474 | + addr_out = dd->dma_addr_out; | |
475 | + | |
476 | + dd->flags &= ~FLAGS_FAST; | |
477 | + | |
478 | + } | |
479 | + | |
480 | + dd->total -= count; | |
481 | + | |
482 | + err = omap4_aes_crypt_dma(tfm, addr_in, addr_out, count); | |
483 | + if (err) { | |
484 | + dma_unmap_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE); | |
485 | + dma_unmap_sg(dd->dev, dd->out_sg, 1, DMA_TO_DEVICE); | |
486 | + } | |
487 | + | |
488 | + return err; | |
489 | +} | |
490 | + | |
491 | +static void omap4_aes_finish_req(struct omap4_aes_dev *dd, int err) | |
492 | +{ | |
493 | + struct ablkcipher_request *req = dd->req; | |
494 | + | |
495 | + pr_debug("err: %d\n", err); | |
496 | + | |
497 | + clk_disable(dd->iclk); | |
498 | + dd->flags &= ~FLAGS_BUSY; | |
499 | + | |
500 | + req->base.complete(&req->base, err); | |
501 | +} | |
502 | + | |
503 | +static int omap4_aes_crypt_dma_stop(struct omap4_aes_dev *dd) | |
504 | +{ | |
505 | + int err = 0; | |
506 | + size_t count; | |
507 | + | |
508 | + pr_debug("total: %d\n", dd->total); | |
509 | + | |
510 | + omap4_aes_write_mask(dd, AES_REG_SYSCFG, 0, AES_REG_SYSCFG_DREQ_MASK); | |
511 | + | |
512 | + edma_stop(dd->dma_lch_in); | |
513 | + edma_clean_channel(dd->dma_lch_in); | |
514 | + edma_stop(dd->dma_lch_out); | |
515 | + edma_clean_channel(dd->dma_lch_out); | |
516 | + | |
517 | + if (dd->flags & FLAGS_FAST) { | |
518 | + dma_unmap_sg(dd->dev, dd->out_sg, 1, DMA_FROM_DEVICE); | |
519 | + dma_unmap_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE); | |
520 | + } else { | |
521 | + dma_sync_single_for_device(dd->dev, dd->dma_addr_out, | |
522 | + dd->dma_size, DMA_FROM_DEVICE); | |
523 | + | |
524 | + /* copy data */ | |
525 | + count = sg_copy(&dd->out_sg, &dd->out_offset, dd->buf_out, | |
526 | + dd->buflen, dd->dma_size, 1); | |
527 | + if (count != dd->dma_size) { | |
528 | + err = -EINVAL; | |
529 | + pr_err("not all data converted: %u\n", count); | |
530 | + } | |
531 | + } | |
532 | + | |
533 | + return err; | |
534 | +} | |
535 | + | |
536 | +static int omap4_aes_handle_queue(struct omap4_aes_dev *dd, | |
537 | + struct ablkcipher_request *req) | |
538 | +{ | |
539 | + struct crypto_async_request *async_req, *backlog; | |
540 | + struct omap4_aes_ctx *ctx; | |
541 | + struct omap4_aes_reqctx *rctx; | |
542 | + unsigned long flags; | |
543 | + int err, ret = 0; | |
544 | + | |
545 | + spin_lock_irqsave(&dd->lock, flags); | |
546 | + if (req) | |
547 | + ret = ablkcipher_enqueue_request(&dd->queue, req); | |
548 | + | |
549 | + if (dd->flags & FLAGS_BUSY) { | |
550 | + spin_unlock_irqrestore(&dd->lock, flags); | |
551 | + return ret; | |
552 | + } | |
553 | + backlog = crypto_get_backlog(&dd->queue); | |
554 | + async_req = crypto_dequeue_request(&dd->queue); | |
555 | + if (async_req) | |
556 | + dd->flags |= FLAGS_BUSY; | |
557 | + spin_unlock_irqrestore(&dd->lock, flags); | |
558 | + | |
559 | + if (!async_req) | |
560 | + return ret; | |
561 | + | |
562 | + if (backlog) | |
563 | + backlog->complete(backlog, -EINPROGRESS); | |
564 | + | |
565 | + req = ablkcipher_request_cast(async_req); | |
566 | + | |
567 | + /* assign new request to device */ | |
568 | + dd->req = req; | |
569 | + dd->total = req->nbytes; | |
570 | + dd->in_offset = 0; | |
571 | + dd->in_sg = req->src; | |
572 | + dd->out_offset = 0; | |
573 | + dd->out_sg = req->dst; | |
574 | + | |
575 | + rctx = ablkcipher_request_ctx(req); | |
576 | + ctx = crypto_ablkcipher_ctx(crypto_ablkcipher_reqtfm(req)); | |
577 | + rctx->mode &= FLAGS_MODE_MASK; | |
578 | + dd->flags = (dd->flags & ~FLAGS_MODE_MASK) | rctx->mode; | |
579 | + | |
580 | + dd->ctx = ctx; | |
581 | + ctx->dd = dd; | |
582 | + | |
583 | + err = omap4_aes_write_ctrl(dd); | |
584 | + if (!err) | |
585 | + err = omap4_aes_crypt_dma_start(dd); | |
586 | + if (err) { | |
587 | + /* aes_task will not finish it, so do it here */ | |
588 | + omap4_aes_finish_req(dd, err); | |
589 | + tasklet_schedule(&dd->queue_task); | |
590 | + } | |
591 | + | |
592 | + return ret; /* return ret, which is enqueue return value */ | |
593 | +} | |
594 | + | |
595 | +static void omap4_aes_done_task(unsigned long data) | |
596 | +{ | |
597 | + struct omap4_aes_dev *dd = (struct omap4_aes_dev *)data; | |
598 | + int err; | |
599 | + | |
600 | + pr_debug("enter\n"); | |
601 | + | |
602 | + err = omap4_aes_crypt_dma_stop(dd); | |
603 | + | |
604 | + err = dd->err ? : err; | |
605 | + | |
606 | + if (dd->total && !err) { | |
607 | + err = omap4_aes_crypt_dma_start(dd); | |
608 | + if (!err) | |
609 | + return; /* DMA started. Not finishing. */ | |
610 | + } | |
611 | + | |
612 | + omap4_aes_finish_req(dd, err); | |
613 | + omap4_aes_handle_queue(dd, NULL); | |
614 | + | |
615 | + pr_debug("exit\n"); | |
616 | +} | |
617 | + | |
618 | +static void omap4_aes_queue_task(unsigned long data) | |
619 | +{ | |
620 | + struct omap4_aes_dev *dd = (struct omap4_aes_dev *)data; | |
621 | + | |
622 | + omap4_aes_handle_queue(dd, NULL); | |
623 | +} | |
624 | + | |
625 | +static int omap4_aes_crypt(struct ablkcipher_request *req, unsigned long mode) | |
626 | +{ | |
627 | + struct omap4_aes_ctx *ctx = crypto_ablkcipher_ctx( | |
628 | + crypto_ablkcipher_reqtfm(req)); | |
629 | + struct omap4_aes_reqctx *rctx = ablkcipher_request_ctx(req); | |
630 | + struct omap4_aes_dev *dd; | |
631 | + | |
632 | + pr_debug("nbytes: %d, enc: %d, cbc: %d, ctr: %d\n", req->nbytes, | |
633 | + !!(mode & FLAGS_ENCRYPT), | |
634 | + !!(mode & FLAGS_CBC), | |
635 | + !!(mode & FLAGS_CTR)); | |
636 | + | |
637 | + if (!IS_ALIGNED(req->nbytes, AES_BLOCK_SIZE)) { | |
638 | + pr_err("request size is not exact amount of AES blocks\n"); | |
639 | + return -EINVAL; | |
640 | + } | |
641 | + | |
642 | + dd = omap4_aes_find_dev(ctx); | |
643 | + if (!dd) | |
644 | + return -ENODEV; | |
645 | + | |
646 | + rctx->mode = mode; | |
647 | + | |
648 | + return omap4_aes_handle_queue(dd, req); | |
649 | +} | |
650 | + | |
651 | +/* ********************** ALG API ************************************ */ | |
652 | + | |
653 | +static int omap4_aes_setkey(struct crypto_ablkcipher *tfm, const u8 *key, | |
654 | + unsigned int keylen) | |
655 | +{ | |
656 | + struct omap4_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm); | |
657 | + | |
658 | + if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 && | |
659 | + keylen != AES_KEYSIZE_256) | |
660 | + return -EINVAL; | |
661 | + | |
662 | + pr_debug("enter, keylen: %d\n", keylen); | |
663 | + | |
664 | + memcpy(ctx->key, key, keylen); | |
665 | + ctx->keylen = keylen; | |
666 | + | |
667 | + return 0; | |
668 | +} | |
669 | + | |
670 | +static int omap4_aes_ecb_encrypt(struct ablkcipher_request *req) | |
671 | +{ | |
672 | + return omap4_aes_crypt(req, FLAGS_ENCRYPT); | |
673 | +} | |
674 | + | |
675 | +static int omap4_aes_ecb_decrypt(struct ablkcipher_request *req) | |
676 | +{ | |
677 | + return omap4_aes_crypt(req, 0); | |
678 | +} | |
679 | + | |
680 | +static int omap4_aes_cbc_encrypt(struct ablkcipher_request *req) | |
681 | +{ | |
682 | + return omap4_aes_crypt(req, FLAGS_ENCRYPT | FLAGS_CBC); | |
683 | +} | |
684 | + | |
685 | +static int omap4_aes_cbc_decrypt(struct ablkcipher_request *req) | |
686 | +{ | |
687 | + return omap4_aes_crypt(req, FLAGS_CBC); | |
688 | +} | |
689 | + | |
690 | +static int omap4_aes_ctr_encrypt(struct ablkcipher_request *req) | |
691 | +{ | |
692 | + return omap4_aes_crypt(req, FLAGS_ENCRYPT | FLAGS_CTR); | |
693 | +} | |
694 | + | |
695 | +static int omap4_aes_ctr_decrypt(struct ablkcipher_request *req) | |
696 | +{ | |
697 | + return omap4_aes_crypt(req, FLAGS_CTR); | |
698 | +} | |
699 | + | |
700 | +static int omap4_aes_cra_init(struct crypto_tfm *tfm) | |
701 | +{ | |
702 | + pr_debug("enter\n"); | |
703 | + | |
704 | + tfm->crt_ablkcipher.reqsize = sizeof(struct omap4_aes_reqctx); | |
705 | + | |
706 | + return 0; | |
707 | +} | |
708 | + | |
709 | +static void omap4_aes_cra_exit(struct crypto_tfm *tfm) | |
710 | +{ | |
711 | + pr_debug("enter\n"); | |
712 | +} | |
713 | + | |
714 | +/* ********************** ALGS ************************************ */ | |
715 | + | |
716 | +static struct crypto_alg algs[] = { | |
717 | + { | |
718 | + .cra_name = "ecb(aes)", | |
719 | + .cra_driver_name = "ecb-aes-omap4", | |
720 | + .cra_priority = 300, | |
721 | + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, | |
722 | + .cra_blocksize = AES_BLOCK_SIZE, | |
723 | + .cra_ctxsize = sizeof(struct omap4_aes_ctx), | |
724 | + .cra_alignmask = 0, | |
725 | + .cra_type = &crypto_ablkcipher_type, | |
726 | + .cra_module = THIS_MODULE, | |
727 | + .cra_init = omap4_aes_cra_init, | |
728 | + .cra_exit = omap4_aes_cra_exit, | |
729 | + .cra_u.ablkcipher = { | |
730 | + .min_keysize = AES_MIN_KEY_SIZE, | |
731 | + .max_keysize = AES_MAX_KEY_SIZE, | |
732 | + .setkey = omap4_aes_setkey, | |
733 | + .encrypt = omap4_aes_ecb_encrypt, | |
734 | + .decrypt = omap4_aes_ecb_decrypt, | |
735 | + } | |
736 | + }, | |
737 | + { | |
738 | + .cra_name = "cbc(aes)", | |
739 | + .cra_driver_name = "cbc-aes-omap4", | |
740 | + .cra_priority = 300, | |
741 | + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, | |
742 | + .cra_blocksize = AES_BLOCK_SIZE, | |
743 | + .cra_ctxsize = sizeof(struct omap4_aes_ctx), | |
744 | + .cra_alignmask = 0, | |
745 | + .cra_type = &crypto_ablkcipher_type, | |
746 | + .cra_module = THIS_MODULE, | |
747 | + .cra_init = omap4_aes_cra_init, | |
748 | + .cra_exit = omap4_aes_cra_exit, | |
749 | + .cra_u.ablkcipher = { | |
750 | + .min_keysize = AES_MIN_KEY_SIZE, | |
751 | + .max_keysize = AES_MAX_KEY_SIZE, | |
752 | + .geniv = "eseqiv", | |
753 | + .ivsize = AES_BLOCK_SIZE, | |
754 | + .setkey = omap4_aes_setkey, | |
755 | + .encrypt = omap4_aes_cbc_encrypt, | |
756 | + .decrypt = omap4_aes_cbc_decrypt, | |
757 | + | |
758 | + } | |
759 | + }, | |
760 | + { | |
761 | + .cra_name = "ctr(aes)", | |
762 | + .cra_driver_name = "ctr-aes-omap4", | |
763 | + .cra_priority = 300, | |
764 | + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, | |
765 | + .cra_blocksize = AES_BLOCK_SIZE, | |
766 | + .cra_ctxsize = sizeof(struct omap4_aes_ctx), | |
767 | + .cra_alignmask = 0, | |
768 | + .cra_type = &crypto_ablkcipher_type, | |
769 | + .cra_module = THIS_MODULE, | |
770 | + .cra_init = omap4_aes_cra_init, | |
771 | + .cra_exit = omap4_aes_cra_exit, | |
772 | + .cra_u.ablkcipher = { | |
773 | + .min_keysize = AES_MIN_KEY_SIZE, | |
774 | + .max_keysize = AES_MAX_KEY_SIZE, | |
775 | + .geniv = "eseqiv", | |
776 | + .ivsize = AES_BLOCK_SIZE, | |
777 | + .setkey = omap4_aes_setkey, | |
778 | + .encrypt = omap4_aes_ctr_encrypt, | |
779 | + .decrypt = omap4_aes_ctr_decrypt, | |
780 | + } | |
781 | + } | |
782 | +}; | |
783 | + | |
784 | +static int omap4_aes_probe(struct platform_device *pdev) | |
785 | +{ | |
786 | + struct device *dev = &pdev->dev; | |
787 | + struct omap4_aes_dev *dd; | |
788 | + struct resource *res; | |
789 | + int err = -ENOMEM, i, j; | |
790 | + u32 reg; | |
791 | + | |
792 | + dd = kzalloc(sizeof(struct omap4_aes_dev), GFP_KERNEL); | |
793 | + if (dd == NULL) { | |
794 | + dev_err(dev, "unable to alloc data struct.\n"); | |
795 | + goto err_data; | |
796 | + } | |
797 | + dd->dev = dev; | |
798 | + platform_set_drvdata(pdev, dd); | |
799 | + | |
800 | + spin_lock_init(&dd->lock); | |
801 | + crypto_init_queue(&dd->queue, AM33X_AES_QUEUE_LENGTH); | |
802 | + | |
803 | + /* Get the base address */ | |
804 | + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); | |
805 | + if (!res) { | |
806 | + dev_err(dev, "invalid resource type\n"); | |
807 | + err = -ENODEV; | |
808 | + goto err_res; | |
809 | + } | |
810 | + dd->phys_base = res->start; | |
811 | + | |
812 | + /* Get the DMA */ | |
813 | + res = platform_get_resource(pdev, IORESOURCE_DMA, 0); | |
814 | + if (!res) | |
815 | + dev_info(dev, "no DMA info\n"); | |
816 | + else | |
817 | + dd->dma_out = res->start; | |
818 | + | |
819 | + /* Get the DMA */ | |
820 | + res = platform_get_resource(pdev, IORESOURCE_DMA, 1); | |
821 | + if (!res) | |
822 | + dev_info(dev, "no DMA info\n"); | |
823 | + else | |
824 | + dd->dma_in = res->start; | |
825 | + | |
826 | + /* Initializing the clock */ | |
827 | + dd->iclk = clk_get(dev, "aes0_fck"); | |
828 | + if (IS_ERR(dd->iclk)) { | |
829 | + dev_err(dev, "clock initialization failed.\n"); | |
830 | + err = PTR_ERR(dd->iclk); | |
831 | + goto err_res; | |
832 | + } | |
833 | + | |
834 | + dd->io_base = ioremap(dd->phys_base, SZ_4K); | |
835 | + if (!dd->io_base) { | |
836 | + dev_err(dev, "can't ioremap\n"); | |
837 | + err = -ENOMEM; | |
838 | + goto err_io; | |
839 | + } | |
840 | + | |
841 | + omap4_aes_hw_init(dd); | |
842 | + reg = omap4_aes_read(dd, AES_REG_REV); | |
843 | + clk_disable(dd->iclk); | |
844 | + dev_info(dev, "AM33X AES hw accel rev: %u.%02u\n", | |
845 | + ((reg & AES_REG_REV_X_MAJOR_MASK) >> 8), | |
846 | + (reg & AES_REG_REV_Y_MINOR_MASK)); | |
847 | + | |
848 | + tasklet_init(&dd->done_task, omap4_aes_done_task, (unsigned long)dd); | |
849 | + tasklet_init(&dd->queue_task, omap4_aes_queue_task, (unsigned long)dd); | |
850 | + | |
851 | + err = omap4_aes_dma_init(dd); | |
852 | + if (err) | |
853 | + goto err_dma; | |
854 | + | |
855 | + INIT_LIST_HEAD(&dd->list); | |
856 | + spin_lock(&list_lock); | |
857 | + list_add_tail(&dd->list, &dev_list); | |
858 | + spin_unlock(&list_lock); | |
859 | + | |
860 | + for (i = 0; i < ARRAY_SIZE(algs); i++) { | |
861 | + pr_debug("reg alg: %s\n", algs[i].cra_name); | |
862 | + INIT_LIST_HEAD(&algs[i].cra_list); | |
863 | + err = crypto_register_alg(&algs[i]); | |
864 | + if (err) | |
865 | + goto err_algs; | |
866 | + } | |
867 | + | |
868 | + pr_info("probe() done\n"); | |
869 | + | |
870 | + return 0; | |
871 | + | |
872 | +err_algs: | |
873 | + for (j = 0; j < i; j++) | |
874 | + crypto_unregister_alg(&algs[j]); | |
875 | + omap4_aes_dma_cleanup(dd); | |
876 | +err_dma: | |
877 | + tasklet_kill(&dd->done_task); | |
878 | + tasklet_kill(&dd->queue_task); | |
879 | + iounmap(dd->io_base); | |
880 | + | |
881 | +err_io: | |
882 | + clk_put(dd->iclk); | |
883 | +err_res: | |
884 | + kfree(dd); | |
885 | + dd = NULL; | |
886 | +err_data: | |
887 | + dev_err(dev, "initialization failed.\n"); | |
888 | + return err; | |
889 | +} | |
890 | + | |
891 | +static int omap4_aes_remove(struct platform_device *pdev) | |
892 | +{ | |
893 | + struct omap4_aes_dev *dd = platform_get_drvdata(pdev); | |
894 | + int i; | |
895 | + | |
896 | + if (!dd) | |
897 | + return -ENODEV; | |
898 | + | |
899 | + spin_lock(&list_lock); | |
900 | + list_del(&dd->list); | |
901 | + spin_unlock(&list_lock); | |
902 | + | |
903 | + for (i = 0; i < ARRAY_SIZE(algs); i++) | |
904 | + crypto_unregister_alg(&algs[i]); | |
905 | + | |
906 | + tasklet_kill(&dd->done_task); | |
907 | + tasklet_kill(&dd->queue_task); | |
908 | + omap4_aes_dma_cleanup(dd); | |
909 | + iounmap(dd->io_base); | |
910 | + clk_put(dd->iclk); | |
911 | + kfree(dd); | |
912 | + dd = NULL; | |
913 | + | |
914 | + return 0; | |
915 | +} | |
916 | + | |
917 | +static struct platform_driver omap4_aes_driver = { | |
918 | + .probe = omap4_aes_probe, | |
919 | + .remove = omap4_aes_remove, | |
920 | + .driver = { | |
921 | + .name = "omap4-aes", | |
922 | + .owner = THIS_MODULE, | |
923 | + }, | |
924 | +}; | |
925 | + | |
926 | +static int __init omap4_aes_mod_init(void) | |
927 | +{ | |
928 | + pr_info("loading AM33X AES driver\n"); | |
929 | + | |
930 | + /* This only works on a GP device */ | |
931 | + if (!cpu_is_am33xx() || omap_type() != OMAP2_DEVICE_TYPE_GP) { | |
932 | + pr_err("Unsupported cpu\n"); | |
933 | + return -ENODEV; | |
934 | + } | |
935 | + return platform_driver_register(&omap4_aes_driver); | |
936 | +} | |
937 | + | |
938 | +static void __exit omap4_aes_mod_exit(void) | |
939 | +{ | |
940 | + pr_info("unloading AM33X AES driver\n"); | |
941 | + | |
942 | + platform_driver_unregister(&omap4_aes_driver); | |
943 | +} | |
944 | + | |
945 | +module_init(omap4_aes_mod_init); | |
946 | +module_exit(omap4_aes_mod_exit); | |
947 | + | |
948 | +MODULE_DESCRIPTION("AM33X AES acceleration support."); | |
949 | +MODULE_LICENSE("GPL v2"); | |
950 | +MODULE_AUTHOR("Herman Schuurman"); |