Eric Lee / linux-smarc-t335x-v3.2

Commit 159a82241b21691d8fe505d42f06e2a0dd3c03a4

Authored by Eric Lee 2014-02-14 16:39:38 +0800

Exists in v3.2_SMARCT335xPSP_04.06.00.11 and in 2 other branches

Add 0008-am33x-Create-driver-for-SHA-MD5-crypto-module.patch that wasn't part of…

… the official PSP 04.06.00.11 release

Showing 1 changed file with 1423 additions and 0 deletions Side-by-side Diff

drivers/crypto/omap4-sham.c

drivers/crypto/omap4-sham.c

Diff comments View file @ 159a822

Changes suppressed. Click to show

	1	+/*
	2	+ * Cryptographic API.
	3	+ *
	4	+ * Support for OMAP SHA1/MD5 HW acceleration.
	5	+ *
	6	+ * Copyright (c) 2010 Nokia Corporation
	7	+ * Author: Dmitry Kasatkin <dmitry.kasatkin@nokia.com>
	8	+ *
	9	+ * This program is free software; you can redistribute it and/or modify
	10	+ * it under the terms of the GNU General Public License version 2 as published
	11	+ * by the Free Software Foundation.
	12	+ *
	13	+ * Some ideas are from old omap-sha1-md5.c driver.
	14	+ */
	15	+/*
	16	+ * Copyright © 2011 Texas Instruments Incorporated
	17	+ * Author: Herman Schuurman
	18	+ * Change: July 2011 - Adapted the omap-sham.c driver to support Netra
	19	+ * implementation of SHA/MD5 hardware accelerator.
	20	+ * Dec 2011 - Updated with latest omap-sham.c driver changes.
	21	+ */
	22	+
	23	+//#define DEBUG
	24	+
	25	+#define pr_fmt(fmt) "%s: " fmt, __func__
	26	+
	27	+#include <linux/err.h>
	28	+#include <linux/device.h>
	29	+#include <linux/module.h>
	30	+#include <linux/init.h>
	31	+#include <linux/errno.h>
	32	+#include <linux/interrupt.h>
	33	+#include <linux/kernel.h>
	34	+#include <linux/clk.h>
	35	+#include <linux/irq.h>
	36	+#include <linux/io.h>
	37	+#include <linux/platform_device.h>
	38	+#include <linux/scatterlist.h>
	39	+#include <linux/dma-mapping.h>
	40	+#include <linux/delay.h>
	41	+#include <linux/crypto.h>
	42	+#include <linux/cryptohash.h>
	43	+#include <crypto/scatterwalk.h>
	44	+#include <crypto/algapi.h>
	45	+#include <crypto/sha.h>
	46	+#include <crypto/md5.h>
	47	+#include <crypto/hash.h>
	48	+#include <crypto/internal/hash.h>
	49	+
	50	+#include <mach/hardware.h>
	51	+#include <plat/cpu.h>
	52	+#include <plat/dma.h>
	53	+#include <mach/edma.h>
	54	+#include <mach/irqs.h>
	55	+#include "omap4.h"
	56	+
	57	+#define SHA2_MD5_BLOCK_SIZE SHA1_BLOCK_SIZE
	58	+
	59	+#define DEFAULT_TIMEOUT_INTERVAL HZ
	60	+
	61	+/* device flags */
	62	+#define FLAGS_BUSY 0
	63	+#define FLAGS_FINAL 1
	64	+#define FLAGS_DMA_ACTIVE 2
	65	+#define FLAGS_OUTPUT_READY 3 /* shared with context flags */
	66	+#define FLAGS_INIT 4
	67	+#define FLAGS_CPU 5 /* shared with context flags */
	68	+#define FLAGS_DMA_READY 6 /* shared with context flags */
	69	+
	70	+/* context flags */
	71	+#define FLAGS_FINUP 16
	72	+#define FLAGS_SG 17
	73	+#define FLAGS_MODE_SHIFT 18
	74	+#define FLAGS_MODE_MASK (SHA_REG_MODE_ALGO_MASK << (FLAGS_MODE_SHIFT - 1))
	75	+#define FLAGS_MD5 (SHA_REG_MODE_ALGO_MD5_128 << (FLAGS_MODE_SHIFT - 1))
	76	+#define FLAGS_SHA1 (SHA_REG_MODE_ALGO_SHA1_160 << (FLAGS_MODE_SHIFT - 1))
	77	+#define FLAGS_SHA224 (SHA_REG_MODE_ALGO_SHA2_224 << (FLAGS_MODE_SHIFT - 1))
	78	+#define FLAGS_SHA256 (SHA_REG_MODE_ALGO_SHA2_256 << (FLAGS_MODE_SHIFT - 1))
	79	+#define FLAGS_HMAC 20
	80	+#define FLAGS_ERROR 21
	81	+
	82	+#define OP_UPDATE 1
	83	+#define OP_FINAL 2
	84	+
	85	+#define AM33X_ALIGN_MASK (sizeof(u32)-1)
	86	+#define AM33X_ALIGNED __attribute__((aligned(sizeof(u32))))
	87	+
	88	+#define BUFLEN PAGE_SIZE
	89	+
	90	+struct omap4_sham_dev;
	91	+
	92	+struct omap4_sham_reqctx {
	93	+ struct omap4_sham_dev *dd;
	94	+ unsigned long rflags;
	95	+ unsigned long op;
	96	+
	97	+ u8 digest[SHA256_DIGEST_SIZE] AM33X_ALIGNED;
	98	+ size_t digcnt; /* total digest byte count */
	99	+ size_t bufcnt; /* bytes in buffer */
	100	+ size_t buflen; /* buffer length */
	101	+ dma_addr_t dma_addr;
	102	+
	103	+ /* walk state */
	104	+ struct scatterlist *sg;
	105	+ unsigned int offset; /* offset in current sg */
	106	+ unsigned int total; /* total request */
	107	+
	108	+ u8 buffer[0] AM33X_ALIGNED;
	109	+};
	110	+
	111	+/* This structure holds the initial HMAC key value, and subsequently
	112	+ * the outer digest in the first 32 bytes. The inner digest will be
	113	+ * kept within the request context to conform to hash only
	114	+ * computations.
	115	+ */
	116	+struct omap4_sham_hmac_ctx {
	117	+ struct crypto_shash *shash;
	118	+ u8 keypad[SHA2_MD5_BLOCK_SIZE] AM33X_ALIGNED;
	119	+ u32 odigest[SHA256_DIGEST_SIZE / sizeof(u32)];
	120	+};
	121	+
	122	+struct omap4_sham_ctx {
	123	+ struct omap4_sham_dev *dd;
	124	+
	125	+ unsigned long cflags;
	126	+
	127	+ /* fallback stuff */
	128	+ struct crypto_shash *fallback;
	129	+
	130	+ struct omap4_sham_hmac_ctx base[0];
	131	+};
	132	+
	133	+#define AM33X_SHAM_QUEUE_LENGTH 1
	134	+
	135	+struct omap4_sham_dev {
	136	+ struct list_head list;
	137	+ unsigned long phys_base;
	138	+ struct device *dev;
	139	+ void __iomem *io_base;
	140	+ int irq;
	141	+ struct clk *iclk;
	142	+ spinlock_t lock;
	143	+ int err;
	144	+ int dma;
	145	+ int dma_lch;
	146	+ struct tasklet_struct done_task;
	147	+
	148	+ unsigned long dflags;
	149	+ struct crypto_queue queue;
	150	+ struct ahash_request *req;
	151	+};
	152	+
	153	+struct omap4_sham_drv {
	154	+ struct list_head dev_list;
	155	+ spinlock_t lock;
	156	+ unsigned long flags; /* superfluous ???? */
	157	+};
	158	+
	159	+static struct omap4_sham_drv sham = {
	160	+ .dev_list = LIST_HEAD_INIT(sham.dev_list),
	161	+ .lock = __SPIN_LOCK_UNLOCKED(sham.lock),
	162	+};
	163	+
	164	+static inline u32 omap4_sham_read(struct omap4_sham_dev *dd, u32 offset)
	165	+{
	166	+ return __raw_readl(dd->io_base + offset);
	167	+}
	168	+
	169	+static inline void omap4_sham_write(struct omap4_sham_dev *dd,
	170	+ u32 offset, u32 value)
	171	+{
	172	+ __raw_writel(value, dd->io_base + offset);
	173	+}
	174	+
	175	+static inline void omap4_sham_write_mask(struct omap4_sham_dev *dd, u32 address,
	176	+ u32 value, u32 mask)
	177	+{
	178	+ u32 val;
	179	+
	180	+ val = omap4_sham_read(dd, address);
	181	+ val &= ~mask;
	182	+ val \|= value;
	183	+ omap4_sham_write(dd, address, val);
	184	+}
	185	+
	186	+static inline void omap4_sham_write_n(struct omap4_sham_dev *dd, u32 offset,
	187	+ u32 *value, int count)
	188	+{
	189	+ for (; count--; value++, offset += 4)
	190	+ omap4_sham_write(dd, offset, *value);
	191	+}
	192	+
	193	+static inline int omap4_sham_wait(struct omap4_sham_dev *dd, u32 offset, u32 bit)
	194	+{
	195	+ unsigned long timeout = jiffies + DEFAULT_TIMEOUT_INTERVAL;
	196	+
	197	+ while (!(omap4_sham_read(dd, offset) & bit)) {
	198	+ if (time_is_before_jiffies(timeout))
	199	+ return -ETIMEDOUT;
	200	+ }
	201	+
	202	+ return 0;
	203	+}
	204	+
	205	+static void omap4_sham_copy_hash(struct ahash_request *req, int out)
	206	+{
	207	+ struct omap4_sham_reqctx *ctx = ahash_request_ctx(req);
	208	+ u32 hash = (u32 )ctx->digest;
	209	+ int i;
	210	+
	211	+ if (ctx->rflags & BIT(FLAGS_HMAC)) {
	212	+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(ctx->dd->req);
	213	+ struct omap4_sham_ctx *tctx = crypto_ahash_ctx(tfm);
	214	+ struct omap4_sham_hmac_ctx *bctx = tctx->base;
	215	+
	216	+ for (i = 0; i < SHA256_DIGEST_SIZE / sizeof(u32); i++) {
	217	+ if (out)
	218	+ bctx->odigest[i] = omap4_sham_read(ctx->dd,
	219	+ SHA_REG_ODIGEST_N(i));
	220	+ else
	221	+ omap4_sham_write(ctx->dd,
	222	+ SHA_REG_ODIGEST_N(i), bctx->odigest[i]);
	223	+ }
	224	+ }
	225	+
	226	+ /* Copy sha256 size to reduce code */
	227	+ for (i = 0; i < SHA256_DIGEST_SIZE / sizeof(u32); i++) {
	228	+ if (out)
	229	+ hash[i] = omap4_sham_read(ctx->dd,
	230	+ SHA_REG_IDIGEST_N(i));
	231	+ else
	232	+ omap4_sham_write(ctx->dd,
	233	+ SHA_REG_IDIGEST_N(i), hash[i]);
	234	+ }
	235	+}
	236	+
	237	+static void omap4_sham_copy_ready_hash(struct ahash_request *req)
	238	+{
	239	+ struct omap4_sham_reqctx *ctx = ahash_request_ctx(req);
	240	+ u32 in = (u32 )ctx->digest;
	241	+ u32 hash = (u32 )req->result;
	242	+ int i, d;
	243	+
	244	+ if (!hash)
	245	+ return;
	246	+
	247	+ switch (ctx->rflags & FLAGS_MODE_MASK) {
	248	+ case FLAGS_MD5:
	249	+ d = MD5_DIGEST_SIZE / sizeof(u32);
	250	+ break;
	251	+ case FLAGS_SHA1:
	252	+ d = SHA1_DIGEST_SIZE / sizeof(u32);
	253	+ break;
	254	+ case FLAGS_SHA224:
	255	+ d = SHA224_DIGEST_SIZE / sizeof(u32);
	256	+ break;
	257	+ case FLAGS_SHA256:
	258	+ d = SHA256_DIGEST_SIZE / sizeof(u32);
	259	+ break;
	260	+ }
	261	+
	262	+ /* all results are in little endian */
	263	+ for (i = 0; i < d; i++)
	264	+ hash[i] = le32_to_cpu(in[i]);
	265	+}
	266	+
	267	+#if 0
	268	+static int omap4_sham_hw_init(struct omap4_sham_dev *dd)
	269	+{
	270	+ omap4_sham_write(dd, SHA_REG_SYSCFG, SHA_REG_SYSCFG_SOFTRESET);
	271	+ /*
	272	+ * prevent OCP bus error (SRESP) in case an access to the module
	273	+ * is performed while the module is coming out of soft reset
	274	+ */
	275	+ __asm__ __volatile__("nop");
	276	+ __asm__ __volatile__("nop");
	277	+
	278	+ if (omap4_sham_wait(dd, SHA_REG_SYSSTATUS, SHA_REG_SYSSTATUS_RESETDONE))
	279	+ return -ETIMEDOUT;
	280	+
	281	+ omap4_sham_write(dd, SHA_REG_SYSCFG,
	282	+ SHA_REG_SYSCFG_SIDLE_SMARTIDLE \| SHA_REG_SYSCFG_AUTOIDLE);
	283	+ set_bit(FLAGS_INIT, &dd->dflags);
	284	+ dd->err = 0;
	285	+
	286	+ return 0;
	287	+}
	288	+#endif
	289	+
	290	+static void omap4_sham_write_ctrl(struct omap4_sham_dev *dd, int final, int dma)
	291	+{
	292	+ struct omap4_sham_reqctx *ctx = ahash_request_ctx(dd->req);
	293	+ u32 val, mask;
	294	+
	295	+ /*
	296	+ * Setting ALGO_CONST only for the first iteration and
	297	+ * CLOSE_HASH only for the last one. Note that flags mode bits
	298	+ * correspond to algorithm encoding in mode register.
	299	+ */
	300	+ val = (ctx->rflags & FLAGS_MODE_MASK) >> (FLAGS_MODE_SHIFT - 1);
	301	+ if (!ctx->digcnt) {
	302	+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(dd->req);
	303	+ struct omap4_sham_ctx *tctx = crypto_ahash_ctx(tfm);
	304	+ struct omap4_sham_hmac_ctx *bctx = tctx->base;
	305	+
	306	+ val \|= SHA_REG_MODE_ALGO_CONSTANT;
	307	+ if (ctx->rflags & BIT(FLAGS_HMAC)) {
	308	+ val \|= SHA_REG_MODE_HMAC_KEY_PROC;
	309	+ omap4_sham_write_n(dd, SHA_REG_ODIGEST, (u32 *) bctx->keypad,
	310	+ SHA2_MD5_BLOCK_SIZE / sizeof(u32));
	311	+ ctx->digcnt += SHA2_MD5_BLOCK_SIZE;
	312	+ }
	313	+ }
	314	+ if (final) {
	315	+ val \|= SHA_REG_MODE_CLOSE_HASH;
	316	+
	317	+ if (ctx->rflags & BIT(FLAGS_HMAC)) {
	318	+ val \|= SHA_REG_MODE_HMAC_OUTER_HASH;
	319	+ }
	320	+ }
	321	+
	322	+ mask = SHA_REG_MODE_ALGO_CONSTANT \| SHA_REG_MODE_CLOSE_HASH \|
	323	+ SHA_REG_MODE_ALGO_MASK \| SHA_REG_MODE_HMAC_OUTER_HASH \|
	324	+ SHA_REG_MODE_HMAC_KEY_PROC;
	325	+
	326	+ dev_dbg(dd->dev, "ctrl: %08x, flags: %08lx\n", val, ctx->rflags);
	327	+ omap4_sham_write_mask(dd, SHA_REG_MODE, val, mask);
	328	+ omap4_sham_write(dd, SHA_REG_IRQENA, SHA_REG_IRQENA_OUTPUT_RDY);
	329	+ omap4_sham_write_mask(dd, SHA_REG_SYSCFG,
	330	+ SHA_REG_SYSCFG_SIT_EN \| (dma ? SHA_REG_SYSCFG_SDMA_EN : 0),
	331	+ SHA_REG_SYSCFG_SIT_EN \| SHA_REG_SYSCFG_SDMA_EN);
	332	+}
	333	+
	334	+static int omap4_sham_xmit_cpu(struct omap4_sham_dev dd, const u8 buf,
	335	+ size_t length, int final)
	336	+{
	337	+ struct omap4_sham_reqctx *ctx = ahash_request_ctx(dd->req);
	338	+ int count, len32;
	339	+ const u32 buffer = (const u32 )buf;
	340	+
	341	+ dev_dbg(dd->dev, "xmit_cpu: digcnt: %d, length: %d, final: %d\n",
	342	+ ctx->digcnt, length, final);
	343	+
	344	+ if (final)
	345	+ set_bit(FLAGS_FINAL, &dd->dflags); /* catch last interrupt */
	346	+
	347	+ set_bit(FLAGS_CPU, &dd->dflags);
	348	+
	349	+ omap4_sham_write_ctrl(dd, final, 0);
	350	+ /*
	351	+ * Setting the length field will also trigger start of
	352	+ * processing.
	353	+ */
	354	+ omap4_sham_write(dd, SHA_REG_LENGTH, length);
	355	+
	356	+ /* short-circuit zero length */
	357	+ if (likely(length)) {
	358	+ ctx->digcnt += length;
	359	+
	360	+ if (omap4_sham_wait(dd, SHA_REG_IRQSTATUS, SHA_REG_IRQSTATUS_INPUT_RDY))
	361	+ return -ETIMEDOUT;
	362	+
	363	+ len32 = DIV_ROUND_UP(length, sizeof(u32));
	364	+
	365	+ for (count = 0; count < len32; count++)
	366	+ omap4_sham_write(dd, SHA_REG_DATA_N(count), buffer[count]);
	367	+ }
	368	+
	369	+ return -EINPROGRESS;
	370	+}
	371	+
	372	+static int omap4_sham_xmit_dma(struct omap4_sham_dev *dd, dma_addr_t dma_addr,
	373	+ size_t length, int final)
	374	+{
	375	+ struct omap4_sham_reqctx *ctx = ahash_request_ctx(dd->req);
	376	+ int nblocks;
	377	+ struct edmacc_param p_ram;
	378	+
	379	+ dev_dbg(dd->dev, "xmit_dma: digcnt: %d, length: %d, final: %d\n",
	380	+ ctx->digcnt, length, final);
	381	+
	382	+ nblocks = DIV_ROUND_UP(length, SHA2_MD5_BLOCK_SIZE);
	383	+
	384	+ /* EDMA IN */
	385	+ p_ram.opt = TCINTEN \|
	386	+ EDMA_TCC(EDMA_CHAN_SLOT(dd->dma_lch));
	387	+ p_ram.src = dma_addr;
	388	+ p_ram.a_b_cnt = SHA2_MD5_BLOCK_SIZE \| nblocks << 16;
	389	+ p_ram.dst = dd->phys_base + SHA_REG_DATA;
	390	+ p_ram.src_dst_bidx = SHA2_MD5_BLOCK_SIZE;
	391	+ p_ram.link_bcntrld = 1 << 16 \| 0xFFFF;
	392	+ p_ram.src_dst_cidx = 0;
	393	+ p_ram.ccnt = 1;
	394	+ edma_write_slot(dd->dma_lch, &p_ram);
	395	+
	396	+ omap4_sham_write_ctrl(dd, final, 1);
	397	+
	398	+ ctx->digcnt += length;
	399	+
	400	+ if (final)
	401	+ set_bit(FLAGS_FINAL, &dd->dflags); /* catch last interrupt */
	402	+
	403	+ set_bit(FLAGS_DMA_ACTIVE, &dd->dflags);
	404	+
	405	+ edma_start(dd->dma_lch);
	406	+
	407	+ /*
	408	+ * Setting the length field will also trigger start of
	409	+ * processing.
	410	+ */
	411	+ omap4_sham_write(dd, SHA_REG_LENGTH, length);
	412	+
	413	+ return -EINPROGRESS;
	414	+}
	415	+
	416	+static size_t omap4_sham_append_buffer(struct omap4_sham_reqctx *ctx,
	417	+ const u8 *data, size_t length)
	418	+{
	419	+ size_t count = min(length, ctx->buflen - ctx->bufcnt);
	420	+
	421	+ count = min(count, ctx->total);
	422	+ if (count <= 0)
	423	+ return 0;
	424	+ memcpy(ctx->buffer + ctx->bufcnt, data, count);
	425	+ ctx->bufcnt += count;
	426	+
	427	+ return count;
	428	+}
	429	+
	430	+static size_t omap4_sham_append_sg(struct omap4_sham_reqctx *ctx)
	431	+{
	432	+ size_t count;
	433	+
	434	+ while (ctx->sg) {
	435	+ if (ctx->sg->length) {
	436	+ count = omap4_sham_append_buffer(ctx,
	437	+ sg_virt(ctx->sg) + ctx->offset,
	438	+ ctx->sg->length - ctx->offset);
	439	+ if (!count)
	440	+ break;
	441	+ ctx->offset += count;
	442	+ ctx->total -= count;
	443	+ }
	444	+ if (ctx->offset == ctx->sg->length) {
	445	+ ctx->sg = sg_next(ctx->sg);
	446	+ if (ctx->sg)
	447	+ ctx->offset = 0;
	448	+ else
	449	+ ctx->total = 0;
	450	+ }
	451	+ }
	452	+
	453	+ return 0;
	454	+}
	455	+
	456	+static int omap4_sham_xmit_dma_map(struct omap4_sham_dev *dd,
	457	+ struct omap4_sham_reqctx *ctx,
	458	+ size_t length, int final)
	459	+{
	460	+ ctx->dma_addr = dma_map_single(dd->dev, ctx->buffer, ctx->buflen,
	461	+ DMA_TO_DEVICE);
	462	+ if (dma_mapping_error(dd->dev, ctx->dma_addr)) {
	463	+ dev_err(dd->dev, "dma %u bytes error\n", ctx->buflen);
	464	+ return -EINVAL;
	465	+ }
	466	+
	467	+ ctx->rflags &= ~BIT(FLAGS_SG);
	468	+
	469	+ /* next call does not fail... so no unmap in the case of error */
	470	+ return omap4_sham_xmit_dma(dd, ctx->dma_addr, length, final);
	471	+}
	472	+
	473	+static int omap4_sham_update_dma_slow(struct omap4_sham_dev *dd)
	474	+{
	475	+ struct omap4_sham_reqctx *ctx = ahash_request_ctx(dd->req);
	476	+ unsigned int final;
	477	+ size_t count;
	478	+
	479	+ omap4_sham_append_sg(ctx);
	480	+
	481	+ final = (ctx->rflags & BIT(FLAGS_FINUP)) && !ctx->total;
	482	+
	483	+ dev_dbg(dd->dev, "slow: bufcnt: %u, digcnt: %d, final: %d\n",
	484	+ ctx->bufcnt, ctx->digcnt, final);
	485	+
	486	+ if (final \|\| (ctx->bufcnt == ctx->buflen && ctx->total)) {
	487	+ count = ctx->bufcnt;
	488	+ ctx->bufcnt = 0;
	489	+ return omap4_sham_xmit_dma_map(dd, ctx, count, final);
	490	+ }
	491	+
	492	+ return 0;
	493	+}
	494	+
	495	+/* Start address alignment */
	496	+#define SG_AA(sg) (IS_ALIGNED(sg->offset, sizeof(u32)))
	497	+/* SHA1 block size alignment */
	498	+#define SG_SA(sg) (IS_ALIGNED(sg->length, SHA2_MD5_BLOCK_SIZE))
	499	+
	500	+static int omap4_sham_update_dma_start(struct omap4_sham_dev *dd)
	501	+{
	502	+ struct omap4_sham_reqctx *ctx = ahash_request_ctx(dd->req);
	503	+ unsigned int length, final, tail;
	504	+ struct scatterlist *sg;
	505	+
	506	+ if (!ctx->total)
	507	+ return 0;
	508	+
	509	+ if (ctx->bufcnt \|\| ctx->offset)
	510	+ return omap4_sham_update_dma_slow(dd);
	511	+
	512	+ dev_dbg(dd->dev, "fast: digcnt: %d, bufcnt: %u, total: %u\n",
	513	+ ctx->digcnt, ctx->bufcnt, ctx->total);
	514	+
	515	+ sg = ctx->sg;
	516	+
	517	+ if (!SG_AA(sg))
	518	+ return omap4_sham_update_dma_slow(dd);
	519	+
	520	+ if (!sg_is_last(sg) && !SG_SA(sg))
	521	+ /* size is not SHA1_BLOCK_SIZE aligned */
	522	+ return omap4_sham_update_dma_slow(dd);
	523	+
	524	+ length = min(ctx->total, sg->length);
	525	+
	526	+ if (sg_is_last(sg)) {
	527	+ if (!(ctx->rflags & BIT(FLAGS_FINUP))) {
	528	+ /* not last sg must be SHA2_MD5_BLOCK_SIZE aligned */
	529	+ tail = length & (SHA2_MD5_BLOCK_SIZE - 1);
	530	+ /* without finup() we need one block to close hash */
	531	+ if (!tail)
	532	+ tail = SHA2_MD5_BLOCK_SIZE;
	533	+ length -= tail;
	534	+ }
	535	+ }
	536	+
	537	+ if (!dma_map_sg(dd->dev, ctx->sg, 1, DMA_TO_DEVICE)) {
	538	+ dev_err(dd->dev, "dma_map_sg error\n");
	539	+ return -EINVAL;
	540	+ }
	541	+
	542	+ ctx->rflags \|= BIT(FLAGS_SG);
	543	+
	544	+ ctx->total -= length;
	545	+ ctx->offset = length; /* offset where to start slow */
	546	+
	547	+ final = (ctx->rflags & BIT(FLAGS_FINUP)) && !ctx->total;
	548	+
	549	+ /* next call does not fail... so no unmap in the case of error */
	550	+ return omap4_sham_xmit_dma(dd, sg_dma_address(ctx->sg), length, final);
	551	+}
	552	+
	553	+static int omap4_sham_update_cpu(struct omap4_sham_dev *dd)
	554	+{
	555	+ struct omap4_sham_reqctx *ctx = ahash_request_ctx(dd->req);
	556	+ int bufcnt;
	557	+
	558	+ omap4_sham_append_sg(ctx);
	559	+ bufcnt = ctx->bufcnt;
	560	+ ctx->bufcnt = 0;
	561	+
	562	+ return omap4_sham_xmit_cpu(dd, ctx->buffer, bufcnt, 1);
	563	+}
	564	+
	565	+static int omap4_sham_update_dma_stop(struct omap4_sham_dev *dd)
	566	+{
	567	+ struct omap4_sham_reqctx *ctx = ahash_request_ctx(dd->req);
	568	+
	569	+ edma_stop(dd->dma_lch);
	570	+ if (ctx->rflags & BIT(FLAGS_SG)) {
	571	+ dma_unmap_sg(dd->dev, ctx->sg, 1, DMA_TO_DEVICE);
	572	+ if (ctx->sg->length == ctx->offset) {
	573	+ ctx->sg = sg_next(ctx->sg);
	574	+ if (ctx->sg)
	575	+ ctx->offset = 0;
	576	+ }
	577	+ } else {
	578	+ dma_unmap_single(dd->dev, ctx->dma_addr, ctx->buflen,
	579	+ DMA_TO_DEVICE);
	580	+ }
	581	+
	582	+ return 0;
	583	+}
	584	+
	585	+static int omap4_sham_init(struct ahash_request *req)
	586	+{
	587	+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
	588	+ struct omap4_sham_ctx *tctx = crypto_ahash_ctx(tfm);
	589	+ struct omap4_sham_reqctx *ctx = ahash_request_ctx(req);
	590	+ struct omap4_sham_dev dd = NULL, tmp;
	591	+
	592	+ spin_lock_bh(&sham.lock);
	593	+ if (!tctx->dd) {
	594	+ list_for_each_entry(tmp, &sham.dev_list, list) {
	595	+ dd = tmp;
	596	+ break;
	597	+ }
	598	+ tctx->dd = dd;
	599	+ } else {
	600	+ dd = tctx->dd;
	601	+ }
	602	+ spin_unlock_bh(&sham.lock);
	603	+
	604	+ ctx->dd = dd;
	605	+
	606	+ ctx->rflags = 0;
	607	+
	608	+ dev_dbg(dd->dev, "init: digest size: %d (@0x%08lx)\n",
	609	+ crypto_ahash_digestsize(tfm), dd->phys_base);
	610	+
	611	+ switch (crypto_ahash_digestsize(tfm)) {
	612	+ case MD5_DIGEST_SIZE:
	613	+ ctx->rflags \|= FLAGS_MD5;
	614	+ break;
	615	+ case SHA1_DIGEST_SIZE:
	616	+ ctx->rflags \|= FLAGS_SHA1;
	617	+ break;
	618	+ case SHA224_DIGEST_SIZE:
	619	+ ctx->rflags \|= FLAGS_SHA224;
	620	+ break;
	621	+ case SHA256_DIGEST_SIZE:
	622	+ ctx->rflags \|= FLAGS_SHA256;
	623	+ break;
	624	+ }
	625	+
	626	+ ctx->bufcnt = 0;
	627	+ ctx->digcnt = 0;
	628	+ ctx->buflen = BUFLEN;
	629	+
	630	+ if (tctx->cflags & BIT(FLAGS_HMAC)) {
	631	+ ctx->rflags \|= BIT(FLAGS_HMAC);
	632	+ }
	633	+
	634	+ return 0;
	635	+}
	636	+
	637	+static int omap4_sham_update_req(struct omap4_sham_dev *dd)
	638	+{
	639	+ struct ahash_request *req = dd->req;
	640	+ struct omap4_sham_reqctx *ctx = ahash_request_ctx(req);
	641	+ int err;
	642	+
	643	+ dev_dbg(dd->dev, "update_req: total: %u, digcnt: %d, finup: %d\n",
	644	+ ctx->total, ctx->digcnt, (ctx->rflags & BIT(FLAGS_FINUP)) != 0);
	645	+
	646	+ if (ctx->rflags & BIT(FLAGS_CPU))
	647	+ err = omap4_sham_update_cpu(dd);
	648	+ else
	649	+ err = omap4_sham_update_dma_start(dd);
	650	+
	651	+ /* wait for dma completion before can take more data */
	652	+ dev_dbg(dd->dev, "update: err: %d, digcnt: %d\n", err, ctx->digcnt);
	653	+
	654	+ return err;
	655	+}
	656	+
	657	+static int omap4_sham_final_req(struct omap4_sham_dev *dd)
	658	+{
	659	+ struct ahash_request *req = dd->req;
	660	+ struct omap4_sham_reqctx *ctx = ahash_request_ctx(req);
	661	+ int err = 0;
	662	+
	663	+ if (ctx->bufcnt <= SHA2_MD5_BLOCK_SIZE) /* faster to handle single block with CPU */
	664	+ err = omap4_sham_xmit_cpu(dd, ctx->buffer, ctx->bufcnt, 1);
	665	+ else
	666	+ err = omap4_sham_xmit_dma_map(dd, ctx, ctx->bufcnt, 1);
	667	+
	668	+ ctx->bufcnt = 0;
	669	+
	670	+ dev_dbg(dd->dev, "final_req: err: %d\n", err);
	671	+
	672	+ return err;
	673	+}
	674	+
	675	+static int omap4_sham_finish(struct ahash_request *req)
	676	+{
	677	+ struct omap4_sham_reqctx *ctx = ahash_request_ctx(req);
	678	+ struct omap4_sham_dev *dd = ctx->dd;
	679	+
	680	+ omap4_sham_copy_ready_hash(req);
	681	+ dev_dbg(dd->dev, "digcnt: %d, bufcnt: %d\n", ctx->digcnt, ctx->bufcnt);
	682	+
	683	+ return 0;
	684	+}
	685	+
	686	+static void omap4_sham_finish_req(struct ahash_request *req, int err)
	687	+{
	688	+ struct omap4_sham_reqctx *ctx = ahash_request_ctx(req);
	689	+ struct omap4_sham_dev *dd = ctx->dd;
	690	+
	691	+ if (!err) {
	692	+ omap4_sham_copy_hash(req, 1);
	693	+ if (test_bit(FLAGS_FINAL, &dd->dflags)) {
	694	+ err = omap4_sham_finish(req);
	695	+ }
	696	+ } else {
	697	+ ctx->rflags \|= BIT(FLAGS_ERROR);
	698	+ }
	699	+
	700	+ /* atomic operation is not needed here */
	701	+ dd->dflags &= ~(BIT(FLAGS_BUSY) \| BIT(FLAGS_FINAL) \| BIT(FLAGS_CPU) \|
	702	+ BIT(FLAGS_DMA_READY) \| BIT(FLAGS_OUTPUT_READY));
	703	+ clk_disable(dd->iclk);
	704	+
	705	+ if (req->base.complete)
	706	+ req->base.complete(&req->base, err);
	707	+
	708	+ /* handle new request */
	709	+ tasklet_schedule(&dd->done_task);
	710	+}
	711	+
	712	+static int omap4_sham_handle_queue(struct omap4_sham_dev *dd,
	713	+ struct ahash_request *req)
	714	+{
	715	+ struct crypto_async_request async_req, backlog;
	716	+ struct omap4_sham_reqctx *ctx;
	717	+ unsigned long flags;
	718	+ int err = 0, ret = 0;
	719	+
	720	+ spin_lock_irqsave(&dd->lock, flags);
	721	+ if (req)
	722	+ ret = ahash_enqueue_request(&dd->queue, req);
	723	+ if (test_bit(FLAGS_BUSY, &dd->dflags)) {
	724	+ spin_unlock_irqrestore(&dd->lock, flags);
	725	+ return ret;
	726	+ }
	727	+ backlog = crypto_get_backlog(&dd->queue);
	728	+ async_req = crypto_dequeue_request(&dd->queue);
	729	+ if (async_req)
	730	+ set_bit(FLAGS_BUSY, &dd->dflags);
	731	+ spin_unlock_irqrestore(&dd->lock, flags);
	732	+
	733	+ if (!async_req)
	734	+ return ret;
	735	+
	736	+ if (backlog)
	737	+ backlog->complete(backlog, -EINPROGRESS);
	738	+
	739	+ req = ahash_request_cast(async_req);
	740	+ dd->req = req;
	741	+ ctx = ahash_request_ctx(req);
	742	+
	743	+ dev_dbg(dd->dev, "handling new req, op: %lu, nbytes: %d\n",
	744	+ ctx->op, req->nbytes);
	745	+
	746	+ clk_enable(dd->iclk);
	747	+ if (!test_bit(FLAGS_INIT, &dd->dflags)) {
	748	+ set_bit(FLAGS_INIT, &dd->dflags);
	749	+ dd->err = 0;
	750	+ }
	751	+
	752	+ if (ctx->digcnt) /* not initial request - restore hash */
	753	+ omap4_sham_copy_hash(req, 0);
	754	+
	755	+ if (ctx->op == OP_UPDATE) {
	756	+ err = omap4_sham_update_req(dd);
	757	+ if (err != -EINPROGRESS && (ctx->rflags & BIT(FLAGS_FINUP)))
	758	+ /* no final() after finup() */
	759	+ err = omap4_sham_final_req(dd);
	760	+ } else if (ctx->op == OP_FINAL) {
	761	+ err = omap4_sham_final_req(dd);
	762	+ }
	763	+
	764	+ if (err != -EINPROGRESS)
	765	+ /* done_task will not finish it, so do it here */
	766	+ omap4_sham_finish_req(req, err);
	767	+
	768	+ dev_dbg(dd->dev, "exit, err: %d\n", err);
	769	+
	770	+ return ret;
	771	+}
	772	+
	773	+static int omap4_sham_enqueue(struct ahash_request *req, unsigned int op)
	774	+{
	775	+ struct omap4_sham_reqctx *ctx = ahash_request_ctx(req);
	776	+ struct omap4_sham_ctx *tctx = crypto_tfm_ctx(req->base.tfm);
	777	+ struct omap4_sham_dev *dd = tctx->dd;
	778	+
	779	+ ctx->op = op;
	780	+
	781	+ return omap4_sham_handle_queue(dd, req);
	782	+}
	783	+
	784	+static int omap4_sham_update(struct ahash_request *req)
	785	+{
	786	+ struct omap4_sham_reqctx *ctx = ahash_request_ctx(req);
	787	+
	788	+ if (!(ctx->rflags & BIT(FLAGS_FINUP)))
	789	+ if (!req->nbytes)
	790	+ return 0;
	791	+
	792	+ ctx->total = req->nbytes;
	793	+ ctx->sg = req->src;
	794	+ ctx->offset = 0;
	795	+
	796	+ if (ctx->rflags & BIT(FLAGS_FINUP)) {
	797	+ if (ctx->bufcnt + ctx->total <= SHA2_MD5_BLOCK_SIZE) {
	798	+ /*
	799	+ * faster to use CPU for short transfers
	800	+ */
	801	+ ctx->rflags \|= BIT(FLAGS_CPU);
	802	+ }
	803	+ } else if (ctx->bufcnt + ctx->total < ctx->buflen) {
	804	+ omap4_sham_append_sg(ctx);
	805	+ return 0;
	806	+ }
	807	+
	808	+ return omap4_sham_enqueue(req, OP_UPDATE);
	809	+}
	810	+
	811	+static int omap4_sham_shash_digest(struct crypto_shash *shash, u32 flags,
	812	+ const u8 data, unsigned int len, u8 out)
	813	+{
	814	+ struct {
	815	+ struct shash_desc shash;
	816	+ char ctx[crypto_shash_descsize(shash)];
	817	+ } desc;
	818	+
	819	+ desc.shash.tfm = shash;
	820	+ desc.shash.flags = flags & CRYPTO_TFM_REQ_MAY_SLEEP;
	821	+
	822	+ return crypto_shash_digest(&desc.shash, data, len, out);
	823	+}
	824	+
	825	+static int omap4_sham_final(struct ahash_request *req)
	826	+{
	827	+ struct omap4_sham_reqctx *ctx = ahash_request_ctx(req);
	828	+
	829	+ ctx->rflags \|= BIT(FLAGS_FINUP);
	830	+
	831	+ if (ctx->rflags & BIT(FLAGS_ERROR))
	832	+ return 0; /* uncompleted hash is not needed */
	833	+
	834	+ return omap4_sham_enqueue(req, OP_FINAL);
	835	+}
	836	+
	837	+static int omap4_sham_finup(struct ahash_request *req)
	838	+{
	839	+ struct omap4_sham_reqctx *ctx = ahash_request_ctx(req);
	840	+ int err1, err2;
	841	+
	842	+ ctx->rflags \|= BIT(FLAGS_FINUP);
	843	+
	844	+ err1 = omap4_sham_update(req);
	845	+ if (err1 == -EINPROGRESS \|\| err1 == -EBUSY)
	846	+ return err1;
	847	+ /*
	848	+ * final() has to be always called to cleanup resources
	849	+ * even if update() failed, except EINPROGRESS
	850	+ */
	851	+ err2 = omap4_sham_final(req);
	852	+
	853	+ return err1 ?: err2;
	854	+}
	855	+
	856	+static int omap4_sham_digest(struct ahash_request *req)
	857	+{
	858	+ return omap4_sham_init(req) ?: omap4_sham_finup(req);
	859	+}
	860	+
	861	+static int omap4_sham_setkey(struct crypto_ahash tfm, const u8 key,
	862	+ unsigned int keylen)
	863	+{
	864	+ struct omap4_sham_ctx *tctx = crypto_ahash_ctx(tfm);
	865	+ struct omap4_sham_hmac_ctx *bctx = tctx->base;
	866	+ int bs = crypto_shash_blocksize(bctx->shash);
	867	+ int ds = crypto_shash_digestsize(bctx->shash);
	868	+ int err;
	869	+
	870	+ /* If key is longer than block size, use hash of original key */
	871	+ if (keylen > bs) {
	872	+ err = crypto_shash_setkey(tctx->fallback, key, keylen) ?:
	873	+ omap4_sham_shash_digest(bctx->shash,
	874	+ crypto_shash_get_flags(bctx->shash),
	875	+ key, keylen, bctx->keypad);
	876	+ if (err)
	877	+ return err;
	878	+ keylen = ds;
	879	+ } else {
	880	+ memcpy(bctx->keypad, key, keylen);
	881	+ }
	882	+
	883	+ /* zero-pad the key (or its digest) */
	884	+ if (keylen < bs)
	885	+ memset(bctx->keypad + keylen, 0, bs - keylen);
	886	+
	887	+ return 0;
	888	+}
	889	+
	890	+static int omap4_sham_cra_init_alg(struct crypto_tfm tfm, const char alg_base)
	891	+{
	892	+ struct omap4_sham_ctx *tctx = crypto_tfm_ctx(tfm);
	893	+ const char *alg_name = crypto_tfm_alg_name(tfm);
	894	+
	895	+ /* Allocate a fallback and abort if it failed. */
	896	+ tctx->fallback = crypto_alloc_shash(alg_name, 0,
	897	+ CRYPTO_ALG_NEED_FALLBACK);
	898	+ if (IS_ERR(tctx->fallback)) {
	899	+ pr_err("omap4-sham: fallback driver '%s' "
	900	+ "could not be loaded.\n", alg_name);
	901	+ return PTR_ERR(tctx->fallback);
	902	+ }
	903	+
	904	+ crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
	905	+ sizeof(struct omap4_sham_reqctx) + BUFLEN);
	906	+
	907	+ if (alg_base) {
	908	+ struct omap4_sham_hmac_ctx *bctx = tctx->base;
	909	+ tctx->cflags \|= BIT(FLAGS_HMAC);
	910	+ bctx->shash = crypto_alloc_shash(alg_base, 0,
	911	+ CRYPTO_ALG_NEED_FALLBACK);
	912	+ if (IS_ERR(bctx->shash)) {
	913	+ pr_err("omap4-sham: base driver '%s' "
	914	+ "could not be loaded.\n", alg_base);
	915	+ crypto_free_shash(tctx->fallback);
	916	+ return PTR_ERR(bctx->shash);
	917	+ }
	918	+
	919	+ }
	920	+
	921	+ return 0;
	922	+}
	923	+
	924	+static int omap4_sham_cra_init(struct crypto_tfm *tfm)
	925	+{
	926	+ return omap4_sham_cra_init_alg(tfm, NULL);
	927	+}
	928	+
	929	+static int omap4_sham_cra_sha1_init(struct crypto_tfm *tfm)
	930	+{
	931	+ return omap4_sham_cra_init_alg(tfm, "sha1");
	932	+}
	933	+
	934	+static int omap4_sham_cra_sha224_init(struct crypto_tfm *tfm)
	935	+{
	936	+ return omap4_sham_cra_init_alg(tfm, "sha224");
	937	+}
	938	+
	939	+static int omap4_sham_cra_sha256_init(struct crypto_tfm *tfm)
	940	+{
	941	+ return omap4_sham_cra_init_alg(tfm, "sha256");
	942	+}
	943	+
	944	+static int omap4_sham_cra_md5_init(struct crypto_tfm *tfm)
	945	+{
	946	+ return omap4_sham_cra_init_alg(tfm, "md5");
	947	+}
	948	+
	949	+static void omap4_sham_cra_exit(struct crypto_tfm *tfm)
	950	+{
	951	+ struct omap4_sham_ctx *tctx = crypto_tfm_ctx(tfm);
	952	+
	953	+ crypto_free_shash(tctx->fallback);
	954	+ tctx->fallback = NULL;
	955	+
	956	+ if (tctx->cflags & BIT(FLAGS_HMAC)) {
	957	+ struct omap4_sham_hmac_ctx *bctx = tctx->base;
	958	+ crypto_free_shash(bctx->shash);
	959	+ }
	960	+}
	961	+
	962	+static struct ahash_alg algs[] = {
	963	+{
	964	+ .init = omap4_sham_init,
	965	+ .update = omap4_sham_update,
	966	+ .final = omap4_sham_final,
	967	+ .finup = omap4_sham_finup,
	968	+ .digest = omap4_sham_digest,
	969	+ .halg.digestsize = SHA1_DIGEST_SIZE,
	970	+ .halg.base = {
	971	+ .cra_name = "sha1",
	972	+ .cra_driver_name = "omap4-sha1",
	973	+ .cra_priority = 300,
	974	+ .cra_flags = CRYPTO_ALG_TYPE_AHASH \|
	975	+ CRYPTO_ALG_ASYNC \|
	976	+ CRYPTO_ALG_NEED_FALLBACK,
	977	+ .cra_blocksize = SHA1_BLOCK_SIZE,
	978	+ .cra_ctxsize = sizeof(struct omap4_sham_ctx),
	979	+ .cra_alignmask = 0,
	980	+ .cra_module = THIS_MODULE,
	981	+ .cra_init = omap4_sham_cra_init,
	982	+ .cra_exit = omap4_sham_cra_exit,
	983	+ }
	984	+},
	985	+{
	986	+ .init = omap4_sham_init,
	987	+ .update = omap4_sham_update,
	988	+ .final = omap4_sham_final,
	989	+ .finup = omap4_sham_finup,
	990	+ .digest = omap4_sham_digest,
	991	+ .halg.digestsize = SHA224_DIGEST_SIZE,
	992	+ .halg.base = {
	993	+ .cra_name = "sha224",
	994	+ .cra_driver_name = "omap4-sha224",
	995	+ .cra_priority = 300,
	996	+ .cra_flags = CRYPTO_ALG_TYPE_AHASH \|
	997	+ CRYPTO_ALG_ASYNC \|
	998	+ CRYPTO_ALG_NEED_FALLBACK,
	999	+ .cra_blocksize = SHA224_BLOCK_SIZE,
	1000	+ .cra_ctxsize = sizeof(struct omap4_sham_ctx),
	1001	+ .cra_alignmask = 0,
	1002	+ .cra_module = THIS_MODULE,
	1003	+ .cra_init = omap4_sham_cra_init,
	1004	+ .cra_exit = omap4_sham_cra_exit,
	1005	+ }
	1006	+},
	1007	+{
	1008	+ .init = omap4_sham_init,
	1009	+ .update = omap4_sham_update,
	1010	+ .final = omap4_sham_final,
	1011	+ .finup = omap4_sham_finup,
	1012	+ .digest = omap4_sham_digest,
	1013	+ .halg.digestsize = SHA256_DIGEST_SIZE,
	1014	+ .halg.base = {
	1015	+ .cra_name = "sha256",
	1016	+ .cra_driver_name = "omap4-sha256",
	1017	+ .cra_priority = 300,
	1018	+ .cra_flags = CRYPTO_ALG_TYPE_AHASH \|
	1019	+ CRYPTO_ALG_ASYNC \|
	1020	+ CRYPTO_ALG_NEED_FALLBACK,
	1021	+ .cra_blocksize = SHA256_BLOCK_SIZE,
	1022	+ .cra_ctxsize = sizeof(struct omap4_sham_ctx),
	1023	+ .cra_alignmask = 0,
	1024	+ .cra_module = THIS_MODULE,
	1025	+ .cra_init = omap4_sham_cra_init,
	1026	+ .cra_exit = omap4_sham_cra_exit,
	1027	+ }
	1028	+},
	1029	+{
	1030	+ .init = omap4_sham_init,
	1031	+ .update = omap4_sham_update,
	1032	+ .final = omap4_sham_final,
	1033	+ .finup = omap4_sham_finup,
	1034	+ .digest = omap4_sham_digest,
	1035	+ .halg.digestsize = MD5_DIGEST_SIZE,
	1036	+ .halg.base = {
	1037	+ .cra_name = "md5",
	1038	+ .cra_driver_name = "omap4-md5",
	1039	+ .cra_priority = 300,
	1040	+ .cra_flags = CRYPTO_ALG_TYPE_AHASH \|
	1041	+ CRYPTO_ALG_ASYNC \|
	1042	+ CRYPTO_ALG_NEED_FALLBACK,
	1043	+ .cra_blocksize = SHA1_BLOCK_SIZE,
	1044	+ .cra_ctxsize = sizeof(struct omap4_sham_ctx),
	1045	+ .cra_alignmask = AM33X_ALIGN_MASK,
	1046	+ .cra_module = THIS_MODULE,
	1047	+ .cra_init = omap4_sham_cra_init,
	1048	+ .cra_exit = omap4_sham_cra_exit,
	1049	+ }
	1050	+},
	1051	+{
	1052	+ .init = omap4_sham_init,
	1053	+ .update = omap4_sham_update,
	1054	+ .final = omap4_sham_final,
	1055	+ .finup = omap4_sham_finup,
	1056	+ .digest = omap4_sham_digest,
	1057	+ .setkey = omap4_sham_setkey,
	1058	+ .halg.digestsize = SHA1_DIGEST_SIZE,
	1059	+ .halg.base = {
	1060	+ .cra_name = "hmac(sha1)",
	1061	+ .cra_driver_name = "omap4-hmac-sha1",
	1062	+ .cra_priority = 300,
	1063	+ .cra_flags = CRYPTO_ALG_TYPE_AHASH \|
	1064	+ CRYPTO_ALG_ASYNC \|
	1065	+ CRYPTO_ALG_NEED_FALLBACK,
	1066	+ .cra_blocksize = SHA1_BLOCK_SIZE,
	1067	+ .cra_ctxsize = sizeof(struct omap4_sham_ctx) +
	1068	+ sizeof(struct omap4_sham_hmac_ctx),
	1069	+ .cra_alignmask = AM33X_ALIGN_MASK,
	1070	+ .cra_module = THIS_MODULE,
	1071	+ .cra_init = omap4_sham_cra_sha1_init,
	1072	+ .cra_exit = omap4_sham_cra_exit,
	1073	+ }
	1074	+},
	1075	+{
	1076	+ .init = omap4_sham_init,
	1077	+ .update = omap4_sham_update,
	1078	+ .final = omap4_sham_final,
	1079	+ .finup = omap4_sham_finup,
	1080	+ .digest = omap4_sham_digest,
	1081	+ .setkey = omap4_sham_setkey,
	1082	+ .halg.digestsize = SHA224_DIGEST_SIZE,
	1083	+ .halg.base = {
	1084	+ .cra_name = "hmac(sha224)",
	1085	+ .cra_driver_name = "omap4-hmac-sha224",
	1086	+ .cra_priority = 300,
	1087	+ .cra_flags = CRYPTO_ALG_TYPE_AHASH \|
	1088	+ CRYPTO_ALG_ASYNC \|
	1089	+ CRYPTO_ALG_NEED_FALLBACK,
	1090	+ .cra_blocksize = SHA224_BLOCK_SIZE,
	1091	+ .cra_ctxsize = sizeof(struct omap4_sham_ctx) +
	1092	+ sizeof(struct omap4_sham_hmac_ctx),
	1093	+ .cra_alignmask = AM33X_ALIGN_MASK,
	1094	+ .cra_module = THIS_MODULE,
	1095	+ .cra_init = omap4_sham_cra_sha224_init,
	1096	+ .cra_exit = omap4_sham_cra_exit,
	1097	+ }
	1098	+},
	1099	+{
	1100	+ .init = omap4_sham_init,
	1101	+ .update = omap4_sham_update,
	1102	+ .final = omap4_sham_final,
	1103	+ .finup = omap4_sham_finup,
	1104	+ .digest = omap4_sham_digest,
	1105	+ .setkey = omap4_sham_setkey,
	1106	+ .halg.digestsize = SHA256_DIGEST_SIZE,
	1107	+ .halg.base = {
	1108	+ .cra_name = "hmac(sha256)",
	1109	+ .cra_driver_name = "omap4-hmac-sha256",
	1110	+ .cra_priority = 300,
	1111	+ .cra_flags = CRYPTO_ALG_TYPE_AHASH \|
	1112	+ CRYPTO_ALG_ASYNC \|
	1113	+ CRYPTO_ALG_NEED_FALLBACK,
	1114	+ .cra_blocksize = SHA256_BLOCK_SIZE,
	1115	+ .cra_ctxsize = sizeof(struct omap4_sham_ctx) +
	1116	+ sizeof(struct omap4_sham_hmac_ctx),
	1117	+ .cra_alignmask = AM33X_ALIGN_MASK,
	1118	+ .cra_module = THIS_MODULE,
	1119	+ .cra_init = omap4_sham_cra_sha256_init,
	1120	+ .cra_exit = omap4_sham_cra_exit,
	1121	+ }
	1122	+},
	1123	+{
	1124	+ .init = omap4_sham_init,
	1125	+ .update = omap4_sham_update,
	1126	+ .final = omap4_sham_final,
	1127	+ .finup = omap4_sham_finup,
	1128	+ .digest = omap4_sham_digest,
	1129	+ .setkey = omap4_sham_setkey,
	1130	+ .halg.digestsize = MD5_DIGEST_SIZE,
	1131	+ .halg.base = {
	1132	+ .cra_name = "hmac(md5)",
	1133	+ .cra_driver_name = "omap4-hmac-md5",
	1134	+ .cra_priority = 300,
	1135	+ .cra_flags = CRYPTO_ALG_TYPE_AHASH \|
	1136	+ CRYPTO_ALG_ASYNC \|
	1137	+ CRYPTO_ALG_NEED_FALLBACK,
	1138	+ .cra_blocksize = SHA1_BLOCK_SIZE,
	1139	+ .cra_ctxsize = sizeof(struct omap4_sham_ctx) +
	1140	+ sizeof(struct omap4_sham_hmac_ctx),
	1141	+ .cra_alignmask = AM33X_ALIGN_MASK,
	1142	+ .cra_module = THIS_MODULE,
	1143	+ .cra_init = omap4_sham_cra_md5_init,
	1144	+ .cra_exit = omap4_sham_cra_exit,
	1145	+ }
	1146	+}
	1147	+};
	1148	+
	1149	+static void omap4_sham_done_task(unsigned long data)
	1150	+{
	1151	+ struct omap4_sham_dev dd = (struct omap4_sham_dev )data;
	1152	+ int err = 0;
	1153	+
	1154	+ if (!test_bit(FLAGS_BUSY, &dd->dflags)) {
	1155	+ omap4_sham_handle_queue(dd, NULL);
	1156	+ return;
	1157	+ }
	1158	+
	1159	+ if (test_bit(FLAGS_CPU, &dd->dflags)) {
	1160	+ if (test_and_clear_bit(FLAGS_OUTPUT_READY, &dd->dflags))
	1161	+ goto finish;
	1162	+ } else if (test_bit(FLAGS_OUTPUT_READY, &dd->dflags)) {
	1163	+ if (test_and_clear_bit(FLAGS_DMA_ACTIVE, &dd->dflags)) {
	1164	+ omap4_sham_update_dma_stop(dd);
	1165	+ if (dd->err) {
	1166	+ err = dd->err;
	1167	+ goto finish;
	1168	+ }
	1169	+ }
	1170	+ if (test_and_clear_bit(FLAGS_OUTPUT_READY, &dd->dflags)) {
	1171	+ /* hash or semi-hash ready */
	1172	+ clear_bit(FLAGS_DMA_READY, &dd->dflags);
	1173	+ err = omap4_sham_update_dma_start(dd);
	1174	+ if (err != -EINPROGRESS)
	1175	+ goto finish;
	1176	+ }
	1177	+ }
	1178	+
	1179	+ return;
	1180	+
	1181	+finish:
	1182	+ dev_dbg(dd->dev, "update done: err: %d\n", err);
	1183	+ /* finish current request */
	1184	+ omap4_sham_finish_req(dd->req, err);
	1185	+}
	1186	+
	1187	+static irqreturn_t omap4_sham_irq(int irq, void *dev_id)
	1188	+{
	1189	+ struct omap4_sham_dev *dd = dev_id;
	1190	+
	1191	+#if 0
	1192	+ if (unlikely(test_bit(FLAGS_FINAL, &dd->flags)))
	1193	+ /* final -> allow device to go to power-saving mode */
	1194	+ omap4_sham_write_mask(dd, SHA_REG_CTRL, 0, SHA_REG_CTRL_LENGTH);
	1195	+#endif
	1196	+
	1197	+ /* TODO check whether the result needs to be read out here,
	1198	+ or if we just disable the interrupt */
	1199	+ omap4_sham_write_mask(dd, SHA_REG_SYSCFG, 0, SHA_REG_SYSCFG_SIT_EN);
	1200	+
	1201	+ if (!test_bit(FLAGS_BUSY, &dd->dflags)) {
	1202	+ dev_warn(dd->dev, "Interrupt when no active requests.\n");
	1203	+ } else {
	1204	+ set_bit(FLAGS_OUTPUT_READY, &dd->dflags);
	1205	+ tasklet_schedule(&dd->done_task);
	1206	+ }
	1207	+
	1208	+ return IRQ_HANDLED;
	1209	+}
	1210	+
	1211	+static void omap4_sham_dma_callback(unsigned int lch, u16 ch_status, void *data)
	1212	+{
	1213	+ struct omap4_sham_dev *dd = data;
	1214	+
	1215	+ edma_stop(lch);
	1216	+
	1217	+ if (ch_status != DMA_COMPLETE) {
	1218	+ pr_err("omap4-sham DMA error status: 0x%hx\n", ch_status);
	1219	+ dd->err = -EIO;
	1220	+ clear_bit(FLAGS_INIT, &dd->dflags); /* request to re-initialize */
	1221	+ }
	1222	+
	1223	+ set_bit(FLAGS_DMA_READY, &dd->dflags);
	1224	+ tasklet_schedule(&dd->done_task);
	1225	+}
	1226	+
	1227	+static int omap4_sham_dma_init(struct omap4_sham_dev *dd)
	1228	+{
	1229	+ int err;
	1230	+
	1231	+ dd->dma_lch = -1;
	1232	+
	1233	+ dd->dma_lch = edma_alloc_channel(dd->dma, omap4_sham_dma_callback, dd, EVENTQ_2);
	1234	+ if (dd->dma_lch < 0) {
	1235	+ dev_err(dd->dev, "Unable to request EDMA channel\n");
	1236	+ return -1;
	1237	+ }
	1238	+
	1239	+ return 0;
	1240	+}
	1241	+
	1242	+static void omap4_sham_dma_cleanup(struct omap4_sham_dev *dd)
	1243	+{
	1244	+ if (dd->dma_lch >= 0) {
	1245	+ edma_free_channel(dd->dma_lch);
	1246	+ dd->dma_lch = -1;
	1247	+ }
	1248	+}
	1249	+
	1250	+static int __devinit omap4_sham_probe(struct platform_device *pdev)
	1251	+{
	1252	+ struct omap4_sham_dev *dd;
	1253	+ struct device *dev = &pdev->dev;
	1254	+ struct resource *res;
	1255	+ int err, i, j;
	1256	+ u32 reg;
	1257	+
	1258	+ dd = kzalloc(sizeof(struct omap4_sham_dev), GFP_KERNEL);
	1259	+ if (dd == NULL) {
	1260	+ dev_err(dev, "unable to alloc data struct.\n");
	1261	+ err = -ENOMEM;
	1262	+ goto data_err;
	1263	+ }
	1264	+ dd->dev = dev;
	1265	+ platform_set_drvdata(pdev, dd);
	1266	+
	1267	+ INIT_LIST_HEAD(&dd->list);
	1268	+ spin_lock_init(&dd->lock);
	1269	+ tasklet_init(&dd->done_task, omap4_sham_done_task, (unsigned long)dd);
	1270	+ crypto_init_queue(&dd->queue, AM33X_SHAM_QUEUE_LENGTH);
	1271	+
	1272	+ dd->irq = -1;
	1273	+
	1274	+ /* Get the base address */
	1275	+ res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
	1276	+ if (!res) {
	1277	+ dev_err(dev, "no MEM resource info\n");
	1278	+ err = -ENODEV;
	1279	+ goto res_err;
	1280	+ }
	1281	+ dd->phys_base = res->start;
	1282	+
	1283	+ /* Get the DMA */
	1284	+ res = platform_get_resource(pdev, IORESOURCE_DMA, 0);
	1285	+ if (!res) {
	1286	+ dev_err(dev, "no DMA resource info\n");
	1287	+ err = -ENODEV;
	1288	+ goto res_err;
	1289	+ }
	1290	+ dd->dma = res->start;
	1291	+
	1292	+ /* Get the IRQ */
	1293	+ dd->irq = platform_get_irq(pdev, 0);
	1294	+ if (dd->irq < 0) {
	1295	+ dev_err(dev, "no IRQ resource info\n");
	1296	+ err = dd->irq;
	1297	+ goto res_err;
	1298	+ }
	1299	+
	1300	+ err = request_irq(dd->irq, omap4_sham_irq,
	1301	+ IRQF_TRIGGER_LOW, dev_name(dev), dd);
	1302	+ if (err) {
	1303	+ dev_err(dev, "unable to request irq.\n");
	1304	+ goto res_err;
	1305	+ }
	1306	+
	1307	+ err = omap4_sham_dma_init(dd);
	1308	+ if (err)
	1309	+ goto dma_err;
	1310	+
	1311	+ /* Initializing the clock */
	1312	+ dd->iclk = clk_get(dev, "sha0_fck");
	1313	+ if (IS_ERR(dd->iclk)) {
	1314	+ dev_err(dev, "clock initialization failed.\n");
	1315	+ err = PTR_ERR(dd->iclk);
	1316	+ goto clk_err;
	1317	+ }
	1318	+
	1319	+ dd->io_base = ioremap(dd->phys_base, SZ_4K);
	1320	+ if (!dd->io_base) {
	1321	+ dev_err(dev, "can't ioremap\n");
	1322	+ err = -ENOMEM;
	1323	+ goto io_err;
	1324	+ }
	1325	+
	1326	+ clk_enable(dd->iclk);
	1327	+ reg = omap4_sham_read(dd, SHA_REG_REV);
	1328	+ clk_disable(dd->iclk);
	1329	+
	1330	+ dev_info(dev, "AM33X SHA/MD5 hw accel rev: %u.%02u\n",
	1331	+ (reg & SHA_REG_REV_X_MAJOR_MASK) >> 8, reg & SHA_REG_REV_Y_MINOR_MASK);
	1332	+
	1333	+ spin_lock(&sham.lock);
	1334	+ list_add_tail(&dd->list, &sham.dev_list);
	1335	+ spin_unlock(&sham.lock);
	1336	+
	1337	+ for (i = 0; i < ARRAY_SIZE(algs); i++) {
	1338	+ err = crypto_register_ahash(&algs[i]);
	1339	+ if (err)
	1340	+ goto err_algs;
	1341	+ }
	1342	+
	1343	+ pr_info("probe() done\n");
	1344	+
	1345	+ return 0;
	1346	+
	1347	+err_algs:
	1348	+ for (j = 0; j < i; j++)
	1349	+ crypto_unregister_ahash(&algs[j]);
	1350	+ iounmap(dd->io_base);
	1351	+io_err:
	1352	+ clk_put(dd->iclk);
	1353	+clk_err:
	1354	+ omap4_sham_dma_cleanup(dd);
	1355	+dma_err:
	1356	+ if (dd->irq >= 0)
	1357	+ free_irq(dd->irq, dd);
	1358	+res_err:
	1359	+ kfree(dd);
	1360	+ dd = NULL;
	1361	+data_err:
	1362	+ dev_err(dev, "initialization failed.\n");
	1363	+
	1364	+ return err;
	1365	+}
	1366	+
	1367	+static int __devexit omap4_sham_remove(struct platform_device *pdev)
	1368	+{
	1369	+ static struct omap4_sham_dev *dd;
	1370	+ int i;
	1371	+
	1372	+ dd = platform_get_drvdata(pdev);
	1373	+ if (!dd)
	1374	+ return -ENODEV;
	1375	+ spin_lock(&sham.lock);
	1376	+ list_del(&dd->list);
	1377	+ spin_unlock(&sham.lock);
	1378	+ for (i = 0; i < ARRAY_SIZE(algs); i++)
	1379	+ crypto_unregister_ahash(&algs[i]);
	1380	+ tasklet_kill(&dd->done_task);
	1381	+ iounmap(dd->io_base);
	1382	+ clk_put(dd->iclk);
	1383	+ omap4_sham_dma_cleanup(dd);
	1384	+ if (dd->irq >= 0)
	1385	+ free_irq(dd->irq, dd);
	1386	+ kfree(dd);
	1387	+ dd = NULL;
	1388	+
	1389	+ return 0;
	1390	+}
	1391	+
	1392	+static struct platform_driver omap4_sham_driver = {
	1393	+ .probe = omap4_sham_probe,
	1394	+ .remove = omap4_sham_remove,
	1395	+ .driver = {
	1396	+ .name = "omap4-sham",
	1397	+ .owner = THIS_MODULE,
	1398	+ },
	1399	+};
	1400	+
	1401	+static int __init omap4_sham_mod_init(void)
	1402	+{
	1403	+ pr_info("loading AM33X SHA/MD5 driver\n");
	1404	+
	1405	+ if (!cpu_is_am33xx() \|\| omap_type() != OMAP2_DEVICE_TYPE_GP) {
	1406	+ pr_err("Unsupported cpu\n");
	1407	+ return -ENODEV;
	1408	+ }
	1409	+
	1410	+ return platform_driver_register(&omap4_sham_driver);
	1411	+}
	1412	+
	1413	+static void __exit omap4_sham_mod_exit(void)
	1414	+{
	1415	+ platform_driver_unregister(&omap4_sham_driver);
	1416	+}
	1417	+
	1418	+module_init(omap4_sham_mod_init);
	1419	+module_exit(omap4_sham_mod_exit);
	1420	+
	1421	+MODULE_DESCRIPTION("AM33x SHA/MD5 hw acceleration support.");
	1422	+MODULE_LICENSE("GPL v2");
	1423	+MODULE_AUTHOR("Herman Schuurman");