crypto: keystone: keystone sideband crypto driver

This commits adds the driver support for keystone SA in sideband mode. The driver registers algorithm implementations in the Kernel crypto framework. Since the primary use case in kernel is to enable HW offload of IPSec ESP crypto operations, the driver is currently supporting only authenticated encryption (AEAD) mode. Following algorithms are currently supported: - authenc(hmac(sha1),cbc(aes)) - authenc(hmac(sha1),cbc(des3_ede)) - authenc(hmac(sha1),ecb(cipher_null)) Signed-off-by: Sandeep Nair <sandeep_n@ti.com> Signed-off-by: Tinku Mannan <tmannan@ti.com> Signed-off-by: Hao Zhang <hzhang@ti.com> Signed-off-by: Vitaly Andrianov <vitalya@ti.com>

crypto: keystone: keystone sideband crypto driver
This commits adds the driver support for keystone SA in sideband mode. The driver registers algorithm implementations in the Kernel crypto framework. Since the primary use case in kernel is to enable HW offload of IPSec ESP crypto operations, the driver is currently supporting only authenticated encryption (AEAD) mode. Following algorithms are currently supported: - authenc(hmac(sha1),cbc(aes)) - authenc(hmac(sha1),cbc(des3_ede)) - authenc(hmac(sha1),ecb(cipher_null)) Signed-off-by: Sandeep Nair <sandeep_n@ti.com> Signed-off-by: Tinku Mannan <tmannan@ti.com> Signed-off-by: Hao Zhang <hzhang@ti.com> Signed-off-by: Vitaly Andrianov <vitalya@ti.com>
Vitaly Andrianov · Hongmei Gou
1 parent ad6e0b7583
Showing 8 changed files with 4418 additions and 0 deletions Side-by-side Diff
drivers/crypto/Kconfig
drivers/crypto/Makefile
drivers/crypto/keystone-sa-hlp.h
drivers/crypto/keystone-sa-lld.c
drivers/crypto/keystone-sa-tbls.c
drivers/crypto/keystone-sa-utils.c
drivers/crypto/keystone-sa.c
drivers/crypto/keystone-sa.h
@@ -273,6 +273,22 @@
 	  the ECB and CBC modes of operation supported by the driver. Also
 	  accesses made on unaligned boundaries are also supported.
  
+config CRYPTO_DEV_KEYSTONE
+	tristate "Support for TI Keystone security accelerator"
+	depends on ARCH_KEYSTONE
+	select CRYPTO_AES
+	select CRYPTO_AES_ARM
+	select CRYPTO_SHA1
+	select CRYPTO_MD5
+	select CRYPTO_ALGAPI
+	select CRYPTO_AUTHENC
+	select HW_RANDOM
+	default y if ARCH_KEYSTONE
+	help
+	  Keystone devices include a security accelerator engine that may be
+	  used for crypto offload.  Select this if you want to use hardware
+	  acceleration for cryptographic algorithms on these devices.
+
 config CRYPTO_DEV_PICOXCELL
 	tristate "Support for picoXcell IPSEC and Layer2 crypto engines"
 	depends on ARCH_PICOXCELL && HAVE_CLK
@@ -8,6 +8,9 @@
 obj-$(CONFIG_CRYPTO_DEV_HIFN_795X) += hifn_795x.o
 obj-$(CONFIG_CRYPTO_DEV_IMGTEC_HASH) += img-hash.o
 obj-$(CONFIG_CRYPTO_DEV_IXP4XX) += ixp4xx_crypto.o
+obj-$(CONFIG_CRYPTO_DEV_KEYSTONE) += keystone-sa-driver.o
+keystone-sa-driver-objs := keystone-sa.o keystone-sa-utils.o \
+			   keystone-sa-lld.o keystone-sa-tbls.o
 obj-$(CONFIG_CRYPTO_DEV_MV_CESA) += mv_cesa.o
 obj-$(CONFIG_CRYPTO_DEV_MXS_DCP) += mxs-dcp.o
 obj-$(CONFIG_CRYPTO_DEV_NIAGARA2) += n2_crypto.o
+/*
+ * Keystone crypto accelerator driver
+ *
+ * Copyright (C) 2015 Texas Instruments Incorporated - http://www.ti.com
+ *
+ * Authors:	Sandeep Nair
+ *		Vitaly Andrianov
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ */
+
+#ifndef _KEYSTONE_SA_HLP_
+#define _KEYSTONE_SA_HLP_
+
+#include <linux/soc/ti/knav_dma.h>
+#include <linux/soc/ti/knav_qmss.h>
+#include <linux/interrupt.h>
+#include <linux/hw_random.h>
+#include <linux/skbuff.h>
+
+/* Enable the below macro for testing with run-time
+ * self tests in the cryptographic algorithm manager
+ * framework */
+/* #define TEST */
+
+/* For enabling debug prints */
+/* #define DEBUG */
+
+/* Algorithm constants */
+#define MD5_BLOCK_SIZE    64
+#define AES_XCBC_DIGEST_SIZE	16
+
+/* Values for NULL algorithms */
+#define NULL_KEY_SIZE		0
+#define NULL_BLOCK_SIZE	1
+#define NULL_DIGEST_SIZE	0
+#define NULL_IV_SIZE		0
+
+/* Number of 32 bit words in EPIB  */
+#define SA_DMA_NUM_EPIB_WORDS	4
+
+/* Number of 32 bit words in PS data  */
+#define SA_DMA_NUM_PS_WORDS	16
+
+/* Number of meta data elements passed in descriptor to SA */
+#define SA_NUM_DMA_META_ELEMS	2
+
+/* Maximum number of simultaeneous security contexts
+ * supported by the driver */
+#define SA_MAX_NUM_CTX	512
+
+/* Encoding used to identify the typo of crypto operation
+ * performed on the packet when the packet is returned
+ * by SA
+ */
+#define SA_REQ_SUBTYPE_ENC	0x0001
+#define SA_REQ_SUBTYPE_DEC	0x0002
+#define SA_REQ_SUBTYPE_SHIFT	16
+#define SA_REQ_SUBTYPE_MASK	0xffff
+
+/* Maximum size of authentication tag
+ * NOTE: update this macro as we start supporting
+ * algorithms with bigger digest size
+ */
+#define SA_MAX_AUTH_TAG_SZ SHA1_DIGEST_SIZE
+
+/* Memory map of the SA register set */
+struct sa_mmr_regs {
+	u32 PID;
+	u32 EFUSE_EN;
+	u32 CMD_STATUS;
+	u32 BLKMGR_PA_BLKS;
+	u32 PA_FLOWID;
+	u32 CDMA_FLOWID;
+	u32 PA_ENG_ID;
+	u32 CDMA_ENG_ID;
+	u8  RSVD0[224];
+	u32 CTXCACH_CTRL;
+	u32 CTXCACH_SC_PTR;
+	u32 CTXCACH_SC_ID;
+	u32 CTXCACH_MISSCNT;
+};
+
+/*
+ * Register Overlay Structure for TRNG module
+ */
+struct sa_trng_regs {
+	u32 TRNG_OUTPUT_L;
+	u32 TRNG_OUTPUT_H;
+	u32 TRNG_STATUS;
+	u32 TRNG_INTMASK;
+	u32 TRNG_INTACK;
+	u32 TRNG_CONTROL;
+	u32 TRNG_CONFIG;
+	u32 TRNG_ALARMCNT;
+	u32 TRNG_FROENABLE;
+	u32 TRNG_FRODETUNE;
+	u32 TRNG_ALARMMASK;
+	u32 TRNG_ALARMSTOP;
+	u32 TRNG_LFSR_L;
+	u32 TRNG_LFSR_M;
+	u32 TRNG_LFSR_H;
+	u32 TRNG_COUNT;
+	u32 TRNG_TEST;
+};
+
+struct sa_regs {
+	struct sa_mmr_regs mmr;
+};
+
+/* Driver statistics */
+struct sa_drv_stats {
+	/* Number of data pkts dropped while submitting to CP_ACE */
+	atomic_t tx_dropped;
+	/* Number of tear-down pkts dropped while submitting to CP_ACE */
+	atomic_t sc_tear_dropped;
+	/* Number of crypto requests sent to CP_ACE */
+	atomic_t tx_pkts;
+	/* Number of crypto request completions received from CP_ACE */
+	atomic_t rx_pkts;
+};
+
+/* Crypto driver instance data */
+struct keystone_crypto_data {
+	struct platform_device	*pdev;
+	struct clk		*clk;
+	struct tasklet_struct	rx_task;
+	struct tasklet_struct	tx_task;
+	struct dma_pool		*sc_pool;
+	struct kmem_cache	*dma_req_ctx_cache;
+	struct sa_regs		*regs;
+	struct sa_trng_regs	*trng_regs;
+
+	void		*rx_chan;
+	void		*rx_fdq[KNAV_DMA_FDQ_PER_CHAN];
+	void		*rx_compl_q;
+	void		*tx_chan;
+	void		*tx_submit_q;
+	void		*tx_compl_q;
+	u32		tx_submit_qid;
+	u32		tx_compl_qid;
+	u32		rx_compl_qid;
+	const char	*rx_chan_name;
+	const char	*tx_chan_name;
+	u32		tx_queue_depth;
+	u32		rx_queue_depths[KNAV_DMA_FDQ_PER_CHAN];
+	u32		rx_buffer_sizes[KNAV_DMA_FDQ_PER_CHAN];
+	u32		rx_pool_size;
+	u32		rx_pool_region_id;
+	void		*rx_pool;
+	u32		tx_pool_size;
+	u32		tx_pool_region_id;
+	void		*tx_pool;
+
+	struct hwrng	rng;
+
+	spinlock_t	scid_lock; /* lock for SC-ID allocation */
+	spinlock_t	trng_lock; /* reading random data from TRNG */
+
+	struct kobject	stats_kobj;
+
+	/* Security context data */
+	u16		sc_id_start;
+	u16		sc_id_end;
+	u16		sc_id;
+
+	/* Bitmap to keep track of Security context ID's */
+	unsigned long	ctx_bm[DIV_ROUND_UP(SA_MAX_NUM_CTX,
+				BITS_PER_LONG)];
+	/* Driver stats */
+	struct sa_drv_stats	stats;
+	atomic_t	rx_dma_page_cnt; /* N buf from 2nd pool available */
+	atomic_t	tx_dma_desc_cnt; /* Tx DMA desc-s available */
+};
+
+
+
+
+/* Packet structure used in Rx */
+#define SA_SGLIST_SIZE	(MAX_SKB_FRAGS + SA_NUM_DMA_META_ELEMS)
+struct sa_packet {
+	struct scatterlist		 sg[SA_SGLIST_SIZE];
+	int				 sg_ents;
+	struct keystone_crypto_data	*priv;
+	struct dma_chan			*chan;
+	struct dma_async_tx_descriptor	*desc;
+	dma_cookie_t			 cookie;
+	u32				 epib[SA_DMA_NUM_EPIB_WORDS];
+	u32				 psdata[SA_DMA_NUM_PS_WORDS];
+	struct completion		 complete;
+	void				*data;
+};
+
+/* Command label updation info */
+struct sa_cmdl_param_info {
+	u16	index;
+	u16	offset;
+	u16	size;
+};
+
+/* Maximum length of Auxiliary data in 32bit words */
+#define SA_MAX_AUX_DATA_WORDS	8
+
+struct sa_cmdl_upd_info {
+	u16	flags;
+	u16	submode;
+	struct sa_cmdl_param_info	enc_size;
+	struct sa_cmdl_param_info	enc_size2;
+	struct sa_cmdl_param_info	enc_offset;
+	struct sa_cmdl_param_info	enc_iv;
+	struct sa_cmdl_param_info	enc_iv2;
+	struct sa_cmdl_param_info	aad;
+	struct sa_cmdl_param_info	payload;
+	struct sa_cmdl_param_info	auth_size;
+	struct sa_cmdl_param_info	auth_size2;
+	struct sa_cmdl_param_info	auth_offset;
+	struct sa_cmdl_param_info	auth_iv;
+	struct sa_cmdl_param_info	aux_key_info;
+	u32				aux_key[SA_MAX_AUX_DATA_WORDS];
+};
+
+enum sa_submode {
+	SA_MODE_GEN = 0,
+	SA_MODE_CCM,
+	SA_MODE_GCM,
+	SA_MODE_GMAC
+};
+
+/* TFM Context info */
+
+/* Number of 32bit words appended after the command label
+ * in PSDATA to identify the crypto request context.
+ * word-0: Request type
+ * word-1: pointer to request
+ */
+#define SA_NUM_PSDATA_CTX_WORDS 4
+
+/* Maximum size of Command label in 32 words */
+#define SA_MAX_CMDL_WORDS (SA_DMA_NUM_PS_WORDS - SA_NUM_PSDATA_CTX_WORDS)
+
+struct sa_ctx_info {
+	u8		*sc;
+	dma_addr_t	sc_phys;
+	u16		sc_id;
+	u16		cmdl_size;
+	u32		cmdl[SA_MAX_CMDL_WORDS];
+	struct sa_cmdl_upd_info cmdl_upd_info;
+	/* Store Auxiliary data such as K2/K3 subkeys in AES-XCBC */
+	u32		epib[SA_DMA_NUM_EPIB_WORDS];
+	u32		rx_flow;
+	u32		rx_compl_qid;
+};
+
+struct sa_tfm_ctx {
+	struct keystone_crypto_data *dev_data;
+	struct sa_ctx_info enc;
+	struct sa_ctx_info dec;
+	struct sa_ctx_info auth;
+};
+
+/* Tx DMA callback param */
+struct sa_dma_req_ctx {
+	struct keystone_crypto_data *dev_data;
+	u32		cmdl[SA_MAX_CMDL_WORDS + SA_NUM_PSDATA_CTX_WORDS];
+	unsigned	map_idx;
+	struct sg_table sg_tbl;
+	dma_cookie_t	cookie;
+	struct dma_chan *tx_chan;
+	bool		pkt;
+};
+
+/* Encryption algorithms */
+enum sa_ealg_id {
+	SA_EALG_ID_NONE = 0,        /* No encryption */
+	SA_EALG_ID_NULL,            /* NULL encryption */
+	SA_EALG_ID_AES_CTR,         /* AES Counter mode */
+	SA_EALG_ID_AES_F8,          /* AES F8 mode */
+	SA_EALG_ID_AES_CBC,         /* AES CBC mode */
+	SA_EALG_ID_DES_CBC,         /* DES CBC mode */
+	SA_EALG_ID_3DES_CBC,        /* 3DES CBC mode */
+	SA_EALG_ID_CCM,             /* Counter with CBC-MAC mode */
+	SA_EALG_ID_GCM,             /* Galois Counter mode */
+	SA_EALG_ID_LAST
+};
+
+/* Authentication algorithms */
+enum sa_aalg_id {
+	SA_AALG_ID_NONE = 0,               /* No Authentication  */
+	SA_AALG_ID_NULL = SA_EALG_ID_LAST, /* NULL Authentication  */
+	SA_AALG_ID_MD5,                    /* MD5 mode */
+	SA_AALG_ID_SHA1,                   /* SHA1 mode */
+	SA_AALG_ID_SHA2_224,               /* 224-bit SHA2 mode */
+	SA_AALG_ID_SHA2_256,               /* 256-bit SHA2 mode */
+	SA_AALG_ID_HMAC_MD5,               /* HMAC with MD5 mode */
+	SA_AALG_ID_HMAC_SHA1,              /* HMAC with SHA1 mode */
+	SA_AALG_ID_HMAC_SHA2_224,          /* HMAC with 224-bit SHA2 mode */
+	SA_AALG_ID_HMAC_SHA2_256,          /* HMAC with 256-bit SHA2 mode */
+	SA_AALG_ID_GMAC,                   /* Galois Message
+					      Authentication Code mode */
+	SA_AALG_ID_CMAC,                   /* Cipher-based Message
+					      Authentication Code mode */
+	SA_AALG_ID_CBC_MAC,                /* Cipher Block Chaining */
+	SA_AALG_ID_AES_XCBC                /* AES Extended
+					      Cipher Block Chaining */
+};
+
+/* Mode control engine algorithms used to index the
+ * mode control instruction tables
+ */
+enum sa_eng_algo_id {
+	SA_ENG_ALGO_ECB = 0,
+	SA_ENG_ALGO_CBC,
+	SA_ENG_ALGO_CFB,
+	SA_ENG_ALGO_OFB,
+	SA_ENG_ALGO_CTR,
+	SA_ENG_ALGO_F8,
+	SA_ENG_ALGO_GCM,
+	SA_ENG_ALGO_GMAC,
+	SA_ENG_ALGO_CCM,
+	SA_ENG_ALGO_CMAC,
+	SA_ENG_ALGO_CBCMAC,
+	SA_NUM_ENG_ALGOS
+};
+
+struct sa_eng_info {
+	u8	eng_id;
+	u16	sc_size;
+};
+
+#define DMA_HAS_PSINFO		BIT(31)
+#define DMA_HAS_EPIB		BIT(30)
+
+void sa_register_algos(const struct device *dev);
+void sa_unregister_algos(const struct device *dev);
+void sa_tx_completion_process(struct keystone_crypto_data *dev_data);
+void sa_rx_completion_process(struct keystone_crypto_data *dev_data);
+
+void sa_set_sc_auth(u16 alg_id, const u8 *key, u16 key_sz, u8 *sc_buf);
+int sa_set_sc_enc(u16 alg_id, const u8 *key, u16 key_sz,
+				u16 aad_len, u8 enc, u8 *sc_buf);
+
+void sa_swiz_128(u8 *in, u8 *out, u16 len);
+void sa_conv_calg_to_salg(const char *cra_name, int *ealg_id, int *aalg_id);
+void sa_get_engine_info(int alg_id, struct sa_eng_info *info);
+int sa_get_hash_size(u16 aalg_id);
+
+#define AES_MAXNR 14
+struct asm_aes_key {
+	unsigned int rd_key[4 * (AES_MAXNR + 1)];
+	int rounds;
+};
+
+/* AES encryption functions defined in aes-armv4.S */
+asmlinkage void AES_encrypt(const u8 *in, u8 *out, struct asm_aes_key *key);
+asmlinkage int private_AES_set_encrypt_key(const unsigned char *user_key,
+				const int bits, struct asm_aes_key *key);
+/*
+ * Derive sub-key k1, k2 and k3 used in the AES XCBC MAC mode
+ * detailed in RFC 3566
+ */
+static inline int sa_aes_xcbc_subkey(u8 *sub_key1, u8 *sub_key2,
+					u8 *sub_key3, const u8 *key,
+					u16 key_sz)
+{
+	struct asm_aes_key enc_key;
+
+	if (private_AES_set_encrypt_key(key, (key_sz * 8), &enc_key) == -1) {
+		pr_err("%s: failed to set enc key\n", __func__);
+		return -1;
+	}
+
+	if (sub_key1) {
+		memset(sub_key1, 0x01, AES_BLOCK_SIZE);
+		AES_encrypt(sub_key1, sub_key1, &enc_key);
+	}
+
+	if (sub_key2) {
+		memset(sub_key2, 0x02, AES_BLOCK_SIZE);
+		AES_encrypt(sub_key2, sub_key2, &enc_key);
+	}
+
+	if (sub_key3) {
+		memset(sub_key3, 0x03, AES_BLOCK_SIZE);
+		AES_encrypt(sub_key3, sub_key3, &enc_key);
+	}
+
+	return 0;
+}
+
+
+extern const uint8_t sa_eng_aes_enc_mci_tbl[11][3][27];
+extern const uint8_t sa_eng_aes_dec_mci_tbl[11][3][27];
+extern const uint8_t sa_eng_3des_enc_mci_tbl[4][27];
+extern const uint8_t sa_eng_3des_dec_mci_tbl[4][27];
+extern struct device *sa_ks2_dev;
+
+#endif /* _KEYSTONE_SA_HLP_ */
+/*
+ * Keystone crypto accelerator driver
+ *
+ * Copyright (C) 2015 Texas Instruments Incorporated - http://www.ti.com
+ *
+ * Authors:	Sandeep Nair
+ *		Vitaly Andrianov
+ *
+ * Contributors:Tinku Mannan
+ *		Hao Zhang
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ */
+
+#include <linux/types.h>
+#include <linux/crypto.h>
+#include <linux/cryptohash.h>
+
+#include <crypto/algapi.h>
+#include <crypto/aead.h>
+#include <crypto/authenc.h>
+#include <crypto/hash.h>
+#include <crypto/internal/hash.h>
+#include <crypto/aes.h>
+#include <crypto/des.h>
+#include <crypto/sha.h>
+#include <crypto/md5.h>
+
+#include "keystone-sa.h"
+#include "keystone-sa-hlp.h"
+
+/* Perform 16 byte swizzling */
+void sa_swiz_128(u8 *in, u8 *out, u16 len)
+{
+	u8 data[16];
+	int i, j;
+
+	for (i = 0; i < len; i += 16) {
+		memcpy(data, &in[i], 16);
+		for (j = 0; j < 16; j++)
+			out[i + j] = data[15 - j];
+	}
+}
+
+/* Convert CRA name to internal algorithm ID */
+void sa_conv_calg_to_salg(const char *cra_name, int *ealg_id, int *aalg_id)
+{
+	*ealg_id = SA_EALG_ID_NONE;
+	*aalg_id = SA_AALG_ID_NONE;
+
+	if (!strcmp(cra_name, "authenc(hmac(sha1),cbc(aes))")) {
+		*ealg_id = SA_EALG_ID_AES_CBC;
+		*aalg_id = SA_AALG_ID_HMAC_SHA1;
+	} else if (!strcmp(cra_name, "authenc(hmac(sha1),ecb(cipher_null))")) {
+		*ealg_id = SA_EALG_ID_NULL;
+		*aalg_id = SA_AALG_ID_HMAC_SHA1;
+	} else if (!strcmp(cra_name, "authenc(hmac(sha1),cbc(des3_ede))")) {
+		*ealg_id = SA_EALG_ID_3DES_CBC;
+		*aalg_id = SA_AALG_ID_HMAC_SHA1;
+	} else if (!strcmp(cra_name, "authenc(xcbc(aes),cbc(aes))")) {
+		*ealg_id = SA_EALG_ID_AES_CBC;
+		*aalg_id = SA_AALG_ID_AES_XCBC;
+	} else if (!strcmp(cra_name, "authenc(xcbc(aes),cbc(des3_ede))")) {
+		*ealg_id = SA_EALG_ID_3DES_CBC;
+		*aalg_id = SA_AALG_ID_AES_XCBC;
+	} else if (!strcmp(cra_name, "cbc(aes)")) {
+		*ealg_id = SA_EALG_ID_AES_CBC;
+	} else if (!strcmp(cra_name, "cbc(des3_ede)")) {
+		*ealg_id = SA_EALG_ID_3DES_CBC;
+	} else if (!strcmp(cra_name, "hmac(sha1)")) {
+		*aalg_id = SA_AALG_ID_HMAC_SHA1;
+	} else if (!strcmp(cra_name, "xcbc(aes)")) {
+		*aalg_id = SA_AALG_ID_AES_XCBC;
+	}
+}
+
+/* Given an algorithm ID get the engine details */
+void sa_get_engine_info(int alg_id, struct sa_eng_info *info)
+{
+	switch (alg_id) {
+	case SA_EALG_ID_AES_CBC:
+	case SA_EALG_ID_3DES_CBC:
+	case SA_EALG_ID_DES_CBC:
+		info->eng_id = SA_ENG_ID_EM1;
+		info->sc_size = SA_CTX_ENC_TYPE1_SZ;
+		break;
+
+	case SA_EALG_ID_NULL:
+		info->eng_id = SA_ENG_ID_NONE;
+		info->sc_size = 0;
+		break;
+
+	case SA_AALG_ID_HMAC_SHA1:
+	case SA_AALG_ID_HMAC_MD5:
+		info->eng_id = SA_ENG_ID_AM1;
+		info->sc_size = SA_CTX_AUTH_TYPE2_SZ;
+		break;
+
+	case SA_AALG_ID_AES_XCBC:
+	case SA_AALG_ID_CMAC:
+		info->eng_id = SA_ENG_ID_EM1;
+		info->sc_size = SA_CTX_AUTH_TYPE1_SZ;
+		break;
+
+	default:
+		pr_err("%s: unsupported algo\n", __func__);
+		info->eng_id = SA_ENG_ID_NONE;
+		info->sc_size = 0;
+		break;
+	}
+}
+
+/* Given an algorithm get the hash size */
+int sa_get_hash_size(u16 aalg_id)
+{
+	int hash_size = 0;
+
+	switch (aalg_id) {
+	case SA_AALG_ID_MD5:
+	case SA_AALG_ID_HMAC_MD5:
+		hash_size = MD5_DIGEST_SIZE;
+		break;
+
+	case SA_AALG_ID_SHA1:
+	case SA_AALG_ID_HMAC_SHA1:
+		hash_size = SHA1_DIGEST_SIZE;
+		break;
+
+	case SA_AALG_ID_SHA2_224:
+	case SA_AALG_ID_HMAC_SHA2_224:
+		hash_size = SHA224_DIGEST_SIZE;
+		break;
+
+	case SA_AALG_ID_SHA2_256:
+	case SA_AALG_ID_HMAC_SHA2_256:
+		hash_size = SHA256_DIGEST_SIZE;
+		break;
+
+	case SA_AALG_ID_AES_XCBC:
+	case SA_AALG_ID_CMAC:
+		hash_size = AES_BLOCK_SIZE;
+		break;
+
+	default:
+		pr_err("%s: unsupported hash\n", __func__);
+		break;
+	}
+
+	return hash_size;
+}
+
+/* Initialize MD5 digest */
+static inline void md5_init(u32 *hash)
+{
+	/* Load magic initialization constants */
+	hash[0] = 0x67452301;
+	hash[1] = 0xefcdab89;
+	hash[2] = 0x98badcfe;
+	hash[3] = 0x10325476;
+}
+
+/* Generate HMAC-MD5 intermediate Hash */
+void sa_hmac_md5_get_pad(const u8 *key, u16 key_sz, u32 *ipad, u32 *opad)
+{
+	u8 k_ipad[MD5_MESSAGE_BYTES];
+	u8 k_opad[MD5_MESSAGE_BYTES];
+	int i;
+
+	for (i = 0; i < key_sz; i++) {
+		k_ipad[i] = key[i] ^ 0x36;
+		k_opad[i] = key[i] ^ 0x5c;
+	}
+	/* Instead of XOR with 0 */
+	for (; i < SHA_MESSAGE_BYTES; i++) {
+		k_ipad[i] = 0x36;
+		k_opad[i] = 0x5c;
+	}
+
+	/* SHA-1 on k_ipad */
+	md5_init(ipad);
+	md5_transform(ipad, (u32 *)k_ipad);
+
+	/* SHA-1 on k_opad */
+	md5_init(opad);
+	md5_transform(ipad, (u32 *)k_opad);
+}
+
+/* Generate HMAC-SHA1 intermediate Hash */
+void sa_hmac_sha1_get_pad(const u8 *key, u16 key_sz, u32 *ipad, u32 *opad)
+{
+	u32 ws[SHA_WORKSPACE_WORDS];
+	u8 k_ipad[SHA_MESSAGE_BYTES];
+	u8 k_opad[SHA_MESSAGE_BYTES];
+	int i;
+
+	for (i = 0; i < key_sz; i++) {
+		k_ipad[i] = key[i] ^ 0x36;
+		k_opad[i] = key[i] ^ 0x5c;
+	}
+	/* Instead of XOR with 0 */
+	for (; i < SHA_MESSAGE_BYTES; i++) {
+		k_ipad[i] = 0x36;
+		k_opad[i] = 0x5c;
+	}
+
+	/* SHA-1 on k_ipad */
+	sha_init(ipad);
+	sha_transform(ipad, k_ipad, ws);
+
+	for (i = 0; i < SHA_DIGEST_WORDS; i++)
+		ipad[i] = cpu_to_be32(ipad[i]);
+
+	/* SHA-1 on k_opad */
+	sha_init(opad);
+	sha_transform(opad, k_opad, ws);
+
+	for (i = 0; i < SHA_DIGEST_WORDS; i++)
+		opad[i] = cpu_to_be32(opad[i]);
+}
+
+/* Derive GHASH to be used in the GCM algorithm */
+void sa_calc_ghash(const u8 *key, u16 key_sz, u8 *ghash)
+{
+}
+
+/* Generate HMAC-SHA224 intermediate Hash */
+void sa_hmac_sha224_get_pad(const u8 *key, u16 key_sz, u32 *ipad, u32 *opad)
+{
+}
+
+/* Generate HMAC-SHA256 intermediate Hash */
+void sa_hmac_sha256_get_pad(const u8 *key, u16 key_sz, u32 *ipad, u32 *opad)
+{
+}
+
+
+/* Derive the inverse key used in AES-CBC decryption operation */
+static inline int sa_aes_inv_key(u8 *inv_key, const u8 *key, u16 key_sz)
+{
+	struct crypto_aes_ctx ctx;
+	int key_pos;
+
+	if (crypto_aes_expand_key(&ctx, key, key_sz)) {
+		pr_err("%s: bad key len(%d)\n", __func__, key_sz);
+		return -1;
+	}
+
+	/* Refer the implementation of crypto_aes_expand_key()
+	 * to understand the below logic
+	 */
+	switch (key_sz) {
+	case AES_KEYSIZE_128:
+	case AES_KEYSIZE_192:
+		key_pos = key_sz + 24;
+		break;
+
+	case AES_KEYSIZE_256:
+		key_pos = key_sz + 24 - 4;
+		break;
+
+	default:
+		pr_err("%s: bad key len(%d)\n", __func__, key_sz);
+		return -1;
+	}
+
+	memcpy(inv_key, &ctx.key_enc[key_pos], key_sz);
+	return 0;
+}
+
+
+/* Set Security context for the encryption engine */
+int sa_set_sc_enc(u16 alg_id, const u8 *key, u16 key_sz,
+				u16 aad_len, u8 enc, u8 *sc_buf)
+{
+/* Byte offset for key in encryption security context */
+#define SC_ENC_KEY_OFFSET (1 + 27 + 4)
+/* Byte offset for Aux-1 in encryption security context */
+#define SC_ENC_AUX1_OFFSET (1 + 27 + 4 + 32)
+
+	u8 ghash[16]; /* AES block size */
+	const u8 *mci = NULL;
+	/* Convert the key size (16/24/32) to the key size index (0/1/2) */
+	int key_idx = (key_sz >> 3) - 2;
+
+	/* Set Encryption mode selector to crypto processing */
+	sc_buf[0] = 0;
+
+	/* Select the mode control instruction */
+	switch (alg_id) {
+	case SA_EALG_ID_AES_CBC:
+		mci = (enc) ? sa_eng_aes_enc_mci_tbl[SA_ENG_ALGO_CBC][key_idx] :
+			sa_eng_aes_dec_mci_tbl[SA_ENG_ALGO_CBC][key_idx];
+		break;
+
+	case SA_EALG_ID_CCM:
+		mci = (enc) ? sa_eng_aes_enc_mci_tbl[SA_ENG_ALGO_CCM][key_idx] :
+			sa_eng_aes_dec_mci_tbl[SA_ENG_ALGO_CCM][key_idx];
+		break;
+
+	case SA_EALG_ID_AES_F8:
+		mci = sa_eng_aes_enc_mci_tbl[SA_ENG_ALGO_F8][key_idx];
+		break;
+
+	case SA_EALG_ID_AES_CTR:
+		mci = sa_eng_aes_enc_mci_tbl[SA_ENG_ALGO_CTR][key_idx];
+		break;
+
+	case SA_EALG_ID_GCM:
+		mci = (enc) ? sa_eng_aes_enc_mci_tbl[SA_ENG_ALGO_GCM][key_idx] :
+			sa_eng_aes_dec_mci_tbl[SA_ENG_ALGO_GCM][key_idx];
+		/* Set AAD length at byte offset 23 in Aux-1 */
+		sc_buf[SC_ENC_AUX1_OFFSET + 23] = (aad_len << 3);
+		/* fall through to GMAC */
+
+	case SA_AALG_ID_GMAC:
+		sa_calc_ghash(key, (key_sz << 3), ghash);
+		/* copy GCM Hash in Aux-1 */
+		memcpy(&sc_buf[SC_ENC_AUX1_OFFSET], ghash, 16);
+		break;
+
+	case SA_AALG_ID_AES_XCBC:
+	case SA_AALG_ID_CMAC:
+		mci = sa_eng_aes_enc_mci_tbl[SA_ENG_ALGO_CMAC][key_idx];
+		break;
+
+	case SA_AALG_ID_CBC_MAC:
+		mci = sa_eng_aes_enc_mci_tbl[SA_ENG_ALGO_CBCMAC][key_idx];
+		break;
+
+	case SA_EALG_ID_3DES_CBC:
+		mci = (enc) ? sa_eng_3des_enc_mci_tbl[SA_ENG_ALGO_CBC] :
+			sa_eng_3des_dec_mci_tbl[SA_ENG_ALGO_CBC];
+		break;
+	}
+
+	/* Set the mode control instructions in security context */
+	if (mci)
+		memcpy(&sc_buf[1], mci, 27);
+
+	/* For AES-CBC decryption get the inverse key */
+	if ((alg_id == SA_EALG_ID_AES_CBC) && !enc) {
+		if (sa_aes_inv_key(&sc_buf[SC_ENC_KEY_OFFSET], key, key_sz))
+			return -1;
+	}
+	/* For AES-XCBC-MAC get the subkey */
+	else if (alg_id == SA_AALG_ID_AES_XCBC) {
+		if (sa_aes_xcbc_subkey(&sc_buf[SC_ENC_KEY_OFFSET], NULL,
+					NULL, key, key_sz))
+			return -1;
+	}
+	/* For all other cases: key is used */
+	else
+		memcpy(&sc_buf[SC_ENC_KEY_OFFSET], key, key_sz);
+
+	return 0;
+}
+
+/* Set Security context for the authentication engine */
+void sa_set_sc_auth(u16 alg_id, const u8 *key, u16 key_sz, u8 *sc_buf)
+{
+	u32 ipad[8], opad[8];
+	u8 mac_sz, keyed_mac = 0;
+
+	/* Set Authentication mode selector to hash processing */
+	sc_buf[0] = 0;
+
+	/* Auth SW ctrl word: bit[6]=1 (upload computed hash to TLR section) */
+	sc_buf[1] = 0x40;
+
+	switch (alg_id) {
+	case SA_AALG_ID_MD5:
+		/* Auth SW ctrl word: bit[4]=1 (basic hash)
+		 * bit[3:0]=1 (MD5 operation)*/
+		sc_buf[1] |= (0x10 | 0x1);
+		break;
+
+	case SA_AALG_ID_SHA1:
+		/* Auth SW ctrl word: bit[4]=1 (basic hash)
+		 * bit[3:0]=2 (SHA1 operation)*/
+		sc_buf[1] |= (0x10 | 0x2);
+		break;
+
+	case SA_AALG_ID_SHA2_224:
+		/* Auth SW ctrl word: bit[4]=1 (basic hash)
+		 * bit[3:0]=3 (SHA2-224 operation)*/
+		sc_buf[1] |= (0x10 | 0x3);
+		break;
+
+	case SA_AALG_ID_SHA2_256:
+		/* Auth SW ctrl word: bit[4]=1 (basic hash)
+		 * bit[3:0]=4 (SHA2-256 operation)*/
+		sc_buf[1] |= (0x10 | 0x4);
+		break;
+
+	case SA_AALG_ID_HMAC_MD5:
+		/* Auth SW ctrl word: bit[4]=0 (HMAC)
+		 * bit[3:0]=1 (MD5 operation)*/
+		sc_buf[1] |= 0x1;
+		keyed_mac = 1;
+		mac_sz = MD5_DIGEST_SIZE;
+		sa_hmac_md5_get_pad(key, key_sz, ipad, opad);
+		break;
+
+	case SA_AALG_ID_HMAC_SHA1:
+		/* Auth SW ctrl word: bit[4]=0 (HMAC)
+		 * bit[3:0]=2 (SHA1 operation)*/
+		sc_buf[1] |= 0x2;
+		keyed_mac = 1;
+		mac_sz = SHA1_DIGEST_SIZE;
+		sa_hmac_sha1_get_pad(key, key_sz, ipad, opad);
+		break;
+
+	case SA_AALG_ID_HMAC_SHA2_224:
+		/* Auth SW ctrl word: bit[4]=0 (HMAC)
+		 * bit[3:0]=3 (SHA2-224 operation)*/
+		sc_buf[1] |= 0x3;
+		keyed_mac = 1;
+		mac_sz = SHA224_DIGEST_SIZE;
+		sa_hmac_sha224_get_pad(key, key_sz, ipad, opad);
+		break;
+
+	case SA_AALG_ID_HMAC_SHA2_256:
+		/* Auth SW ctrl word: bit[4]=0 (HMAC)
+		 * bit[3:0]=4 (SHA2-256 operation)*/
+		sc_buf[1] |= 0x4;
+		keyed_mac = 1;
+		mac_sz = SHA256_DIGEST_SIZE;
+		sa_hmac_sha256_get_pad(key, key_sz, ipad, opad);
+		break;
+	}
+
+	/* Copy the keys or ipad/opad */
+	if (keyed_mac) {
+		/* Copy ipad to AuthKey */
+		memcpy(&sc_buf[32], ipad, mac_sz);
+		/* Copy opad to Aux-1 */
+		memcpy(&sc_buf[64], opad, mac_sz);
+	}
+}
+/*
+ * Keystone crypto accelerator driver
+ *
+ * Copyright (C) 2015 Texas Instruments Incorporated - http://www.ti.com
+ *
+ * Authors:	Sandeep Nair
+ *		Vitaly Andrianov
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ */
+
+#include <linux/types.h>
+
+/************************************************************
+ * Note: The below tables are generated.
+ * Do not update it manually.
+ *
+ * Note: This is a special version of MCI file with
+ * 3GPP standard modes disabled.
+************************************************************/
+const uint8_t sa_eng_aes_enc_mci_tbl[11][3][27] = {
+	{
+		{
+			0x21, 0x00, 0x00, 0x80, 0x8a, 0x04, 0xb7, 0x90, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+		},
+		{
+			0x21, 0x00, 0x00, 0x84, 0x8a, 0x04, 0xb7, 0x90, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+		},
+		{
+			0x21, 0x00, 0x00, 0x88, 0x8a, 0x04, 0xb7, 0x90, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+		}
+	},
+	{
+		{
+			0x21, 0x00, 0x00, 0x18, 0x88, 0x0a, 0xaa, 0x4b, 0x7e,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+		},
+		{
+			0x21, 0x00, 0x00, 0x18, 0x88, 0x4a, 0xaa, 0x4b, 0x7e,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+		},
+		{
+			0x21, 0x00, 0x00, 0x18, 0x88, 0x8a, 0xaa, 0x4b, 0x7e,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+		}
+	},
+	{
+		{
+			0x21, 0x00, 0x00, 0x80, 0x9a, 0x09, 0x94, 0x7c, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+		},
+		{
+			0x21, 0x00, 0x00, 0x84, 0x9a, 0x09, 0x94, 0x7c, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+		},
+		{
+			0x21, 0x00, 0x00, 0x88, 0x9a, 0x09, 0x94, 0x7c, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+		}
+	},
+	{
+		{
+			0x21, 0x00, 0x00, 0x80, 0x9a, 0xa5, 0xb4, 0x60, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+		},
+		{
+			0x21, 0x00, 0x00, 0x84, 0x9a, 0xa5, 0xb4, 0x60, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+		},
+		{
+			0x21, 0x00, 0x00, 0x88, 0x9a, 0xa5, 0xb4, 0x60, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+		}
+	},
+	{
+		{
+			0x21, 0x00, 0x00, 0x80, 0x9a, 0x8f, 0x54, 0x1b, 0x82,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+		},
+		{
+			0x21, 0x00, 0x00, 0x84, 0x9a, 0x8f, 0x54, 0x1b, 0x82,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+		},
+		{
+			0x21, 0x00, 0x00, 0x88, 0x9a, 0x8f, 0x54, 0x1b, 0x82,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+		}
+	},
+	{
+		{
+			0x21, 0x00, 0x22, 0x3b, 0xa3, 0xfb, 0x19, 0x31, 0x91,
+			0x80, 0xa5, 0xc3, 0xa8, 0x89, 0x9e, 0x10, 0x2c, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+		},
+		{
+			0x21, 0x00, 0x22, 0x3b, 0xa3, 0xfb, 0x19, 0x31, 0x91,
+			0x84, 0xa5, 0xc3, 0xa8, 0x89, 0x9e, 0x10, 0x2c, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+		},
+		{
+			0x21, 0x00, 0x22, 0x3b, 0xa3, 0xfb, 0x19, 0x31, 0x91,
+			0x88, 0xa5, 0xc3, 0xa8, 0x89, 0x9e, 0x10, 0x2c, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+		}
+	},
+	{
+		{
+			0x61, 0x00, 0x44, 0x80, 0xa9, 0xfe, 0x83, 0x99, 0x7e,
+			0x58, 0x2e, 0x0a, 0x90, 0x71, 0x41, 0x83, 0x9d, 0x63,
+			0xaa, 0x0b, 0x7e, 0x9a, 0x78, 0x3a, 0xa3, 0x8b, 0x1e
+		},
+		{
+			0x61, 0x00, 0x44, 0x84, 0xa9, 0xfe, 0x83, 0x99, 0x7e,
+			0x58, 0x2e, 0x4a, 0x90, 0x71, 0x41, 0x83, 0x9d, 0x63,
+			0xaa, 0x0b, 0x7e, 0x9a, 0x78, 0x3a, 0xa3, 0x8b, 0x1e
+		},
+		{
+			0x61, 0x00, 0x44, 0x88, 0xa9, 0xfe, 0x83, 0x99, 0x7e,
+			0x58, 0x2e, 0x8a, 0x90, 0x71, 0x41, 0x83, 0x9d, 0x63,
+			0xaa, 0x0b, 0x7e, 0x9a, 0x78, 0x3a, 0xa3, 0x8b, 0x1e
+		}
+	},
+	{
+		{
+			0x41, 0x00, 0x44, 0x80, 0xa9, 0xfe, 0x83, 0x99, 0x7e,
+			0x14, 0x18, 0x39, 0xd4, 0xba, 0xa0, 0xb7, 0xe9, 0xa7,
+			0x83, 0xaa, 0x38, 0xb5, 0xe0, 0x00, 0x00, 0x00, 0x00
+		},
+		{
+			0x41, 0x00, 0x44, 0x84, 0xa9, 0xfe, 0x83, 0x99, 0x7e,
+			0x14, 0x18, 0x39, 0xd4, 0xba, 0xa0, 0xb7, 0xe9, 0xa7,
+			0x83, 0xaa, 0x38, 0xb5, 0xe0, 0x00, 0x00, 0x00, 0x00
+		},
+		{
+			0x41, 0x00, 0x44, 0x88, 0xa9, 0xfe, 0x83, 0x99, 0x7e,
+			0x14, 0x18, 0x39, 0xd4, 0xba, 0xa0, 0xb7, 0xe9, 0xa7,
+			0x83, 0xaa, 0x38, 0xb5, 0xe0, 0x00, 0x00, 0x00, 0x00
+		}
+	},
+	{
+		{
+			0x61, 0x00, 0x66, 0x80, 0xa9, 0x8f, 0x80, 0xa9, 0xbe,
+			0x80, 0xb9, 0x7e, 0x18, 0x28, 0x0a, 0x9b, 0xe5, 0xc3,
+			0x80, 0xbd, 0x6c, 0x15, 0x1a, 0x8e, 0xb0, 0x00, 0x00
+		},
+		{
+			0x61, 0x00, 0x66, 0x84, 0xa9, 0x8f, 0x84, 0xa9, 0xbe,
+			0x84, 0xb9, 0x7e, 0x18, 0x28, 0x4a, 0x9b, 0xe5, 0xc3,
+			0x84, 0xbd, 0x6c, 0x15, 0x1a, 0x8e, 0xb0, 0x00, 0x00
+		},
+		{
+			0x61, 0x00, 0x66, 0x88, 0xa9, 0x8f, 0x88, 0xa9, 0xbe,
+			0x88, 0xb9, 0x7e, 0x18, 0x28, 0x8a, 0x9b, 0xe5, 0xc3,
+			0x88, 0xbd, 0x6c, 0x15, 0x1a, 0x8e, 0xb0, 0x00, 0x00
+		}
+	},
+	{
+		{
+			0x41, 0x00, 0x00, 0xf1, 0x0d, 0x19, 0x10, 0x8d, 0x2c,
+			0x12, 0x88, 0x08, 0xa6, 0x4b, 0x7e, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+		},
+		{
+			0x41, 0x00, 0x00, 0xf1, 0x0d, 0x19, 0x10, 0x8d, 0x2c,
+			0x12, 0x88, 0x48, 0xa6, 0x4b, 0x7e, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+		},
+		{
+			0x41, 0x00, 0x00, 0xf1, 0x0d, 0x19, 0x10, 0x8d, 0x2c,
+			0x12, 0x88, 0x88, 0xa6, 0x4b, 0x7e, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+		}
+	},
+	{
+		{
+			0x01, 0x00, 0x11, 0x37, 0x91, 0x41, 0x80, 0x9a, 0x4c,
+			0x97, 0xec, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+		},
+		{
+			0x01, 0x00, 0x11, 0x37, 0x91, 0x41, 0x84, 0x9a, 0x4c,
+			0x97, 0xec, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+		},
+		{
+			0x01, 0x00, 0x11, 0x37, 0x91, 0x41, 0x88, 0x9a, 0x4c,
+			0x97, 0xec, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+		}
+	}
+};
+
+const uint8_t sa_eng_aes_dec_mci_tbl[11][3][27] = {
+	{
+		{
+			0x31, 0x00, 0x00, 0x80, 0x8a, 0x04, 0xb7, 0x90, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+		},
+		{
+			0x31, 0x00, 0x00, 0x84, 0x8a, 0x04, 0xb7, 0x90, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+		},
+		{
+			0x31, 0x00, 0x00, 0x88, 0x8a, 0x04, 0xb7, 0x90, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+		}
+	},
+	{
+		{
+			0x31, 0x00, 0x00, 0x80, 0x8a, 0xca, 0x98, 0xf4, 0x40,
+			0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+		},
+		{
+			0x31, 0x00, 0x00, 0x84, 0x8a, 0xca, 0x98, 0xf4, 0x40,
+			0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+		},
+		{
+			0x31, 0x00, 0x00, 0x88, 0x8a, 0xca, 0x98, 0xf4, 0x40,
+			0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+		}
+	},
+	{
+		{
+			0x21, 0x00, 0x00, 0x80, 0x9a, 0xc7, 0x44, 0x0b, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+		},
+		{
+			0x21, 0x00, 0x00, 0x84, 0x9a, 0xc7, 0x44, 0x0b, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+		},
+		{
+			0x21, 0x00, 0x00, 0x88, 0x9a, 0xc7, 0x44, 0x0b, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+		}
+	},
+	{
+		{
+			0x21, 0x00, 0x00, 0x80, 0x9a, 0xa5, 0xb4, 0x60, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+		},
+		{
+			0x21, 0x00, 0x00, 0x84, 0x9a, 0xa5, 0xb4, 0x60, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+		},
+		{
+			0x21, 0x00, 0x00, 0x88, 0x9a, 0xa5, 0xb4, 0x60, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+		}
+	},
+	{
+		{
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+		},
+		{
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+		},
+		{
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+		}
+	},
+	{
+		{
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+		},
+		{
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+		},
+		{
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+		}
+	},
+	{
+		{
+			0x61, 0x00, 0x44, 0x80, 0xa9, 0xfe, 0x83, 0x99, 0x7e,
+			0x58, 0x2e, 0x0a, 0x14, 0x19, 0x07, 0x83, 0x9d, 0x63,
+			0xaa, 0x0b, 0x7e, 0x9a, 0x78, 0x3a, 0xa3, 0x8b, 0x1e
+		},
+		{
+			0x61, 0x00, 0x44, 0x84, 0xa9, 0xfe, 0x83, 0x99, 0x7e,
+			0x58, 0x2e, 0x4a, 0x14, 0x19, 0x07, 0x83, 0x9d, 0x63,
+			0xaa, 0x0b, 0x7e, 0x9a, 0x78, 0x3a, 0xa3, 0x8b, 0x1e
+		},
+		{
+			0x61, 0x00, 0x44, 0x88, 0xa9, 0xfe, 0x83, 0x99, 0x7e,
+			0x58, 0x2e, 0x8a, 0x14, 0x19, 0x07, 0x83, 0x9d, 0x63,
+			0xaa, 0x0b, 0x7e, 0x9a, 0x78, 0x3a, 0xa3, 0x8b, 0x1e
+		}
+	},
+	{
+		{
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+		},
+		{
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+		},
+		{
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+		}
+	},
+	{
+		{
+			0x61, 0x00, 0x66, 0x80, 0xa9, 0x8f, 0x80, 0xa9, 0xbe,
+			0x80, 0xb9, 0x7e, 0x5c, 0x3e, 0x0b, 0x90, 0x71, 0x82,
+			0x80, 0xaa, 0x88, 0x9b, 0xed, 0x7c, 0x14, 0xac, 0x00
+		},
+		{
+			0x61, 0x00, 0x66, 0x84, 0xa9, 0x8f, 0x84, 0xa9, 0xbe,
+			0x84, 0xb9, 0x7e, 0x5c, 0x3e, 0x4b, 0x90, 0x71, 0x82,
+			0x84, 0xaa, 0x88, 0x9b, 0xed, 0x7c, 0x14, 0xac, 0x00
+		},
+		{
+			0x61, 0x00, 0x66, 0x88, 0xa9, 0x8f, 0x88, 0xa9, 0xbe,
+			0x88, 0xb9, 0x7e, 0x5c, 0x3e, 0x8b, 0x90, 0x71, 0x82,
+			0x88, 0xaa, 0x88, 0x9b, 0xed, 0x7c, 0x14, 0xac, 0x00
+		}
+	},
+	{
+		{
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+		},
+		{
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+		},
+		{
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+		}
+	},
+	{
+		{
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+		},
+		{
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+		},
+		{
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+		}
+	}
+};
+
+const uint8_t sa_eng_3des_enc_mci_tbl[4][27] = {
+	{
+		0x20, 0x00, 0x00, 0x85, 0x0a, 0x04, 0xb7, 0x90, 0x00, 0x00,
+		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+	},
+	{
+		0x20, 0x00, 0x00, 0x18, 0x88, 0x52, 0xaa, 0x4b, 0x7e, 0x00,
+		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+	},
+	{
+		0x20, 0x00, 0x00, 0x85, 0x1a, 0x09, 0x94, 0x7c, 0x00, 0x00,
+		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+	},
+	{
+		0x20, 0x00, 0x00, 0x85, 0x1a, 0xa5, 0xb4, 0x60, 0x00, 0x00,
+		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+	}
+};
+
+const uint8_t sa_eng_3des_dec_mci_tbl[4][27] = {
+	{
+		0x30, 0x00, 0x00, 0x85, 0x0a, 0x04, 0xb7, 0x90, 0x00, 0x00,
+		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+	},
+	{
+		0x30, 0x00, 0x00, 0x85, 0x0a, 0xca, 0x98, 0xf4, 0x40, 0xc0,
+		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+	},
+	{
+		0x20, 0x00, 0x00, 0x85, 0x1a, 0xc7, 0x44, 0x0b, 0x00, 0x00,
+		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+	},
+	{
+		0x20, 0x00, 0x00, 0x85, 0x1a, 0xa5, 0xb4, 0x60, 0x00, 0x00,
+		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+	}
+};
+/*
+ * Keystone crypto accelerator driver
+ *
+ * Copyright (C) 2015 Texas Instruments Incorporated - http://www.ti.com
+ *
+ * Authors:	Sandeep Nair
+ *		Vitaly Andrianov
+ *
+ * Contributors:Tinku Mannan
+ *		Hao Zhang
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ */
+
+#include <linux/clk.h>
+#include <linux/err.h>
+#include <linux/init.h>
+#include <linux/slab.h>
+#include <linux/module.h>
+#include <linux/interrupt.h>
+#include <linux/dmapool.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
+#include <linux/rtnetlink.h>
+#include <linux/dma-mapping.h>
+#include <linux/platform_device.h>
+#include <linux/soc/ti/knav_dma.h>
+#include <linux/soc/ti/knav_qmss.h>
+
+#include <linux/crypto.h>
+#include <linux/hw_random.h>
+#include <linux/cryptohash.h>
+#include <crypto/algapi.h>
+#include <crypto/aead.h>
+#include <crypto/authenc.h>
+#include <crypto/hash.h>
+#include <crypto/internal/hash.h>
+#include <crypto/aes.h>
+#include <crypto/des.h>
+#include <crypto/sha.h>
+#include <crypto/md5.h>
+#include <crypto/scatterwalk.h>
+
+#include "keystone-sa.h"
+#include "keystone-sa-hlp.h"
+
+struct device *sa_ks2_dev;
+
+/* Number of elements in scatterlist */
+static int sg_count(struct scatterlist *sg, int len)
+{
+	int sg_nents = 0;
+
+	while (sg && (len > 0)) {
+		sg_nents++;
+		len -= sg->length;
+		sg = sg_next(sg);
+	}
+	return sg_nents;
+}
+
+/* buffer capacity of scatterlist */
+static int sg_len(struct scatterlist *sg)
+{
+	int len = 0;
+
+	while (sg) {
+		len += sg->length;
+		sg = sg_next(sg);
+	}
+	return len;
+}
+
+/* Clone SG list without copying the buffer */
+static inline void sa_clone_sg(struct scatterlist *src,
+		struct scatterlist *dst, unsigned int nbytes)
+{
+	while ((nbytes > 0) && src && dst) {
+		struct page *pg = sg_page(src);
+		unsigned int len = min(nbytes, src->length);
+
+		sg_set_page(dst, pg, len, src->offset);
+		src = sg_next(src);
+		dst = sg_next(dst);
+		nbytes -= len;
+	}
+}
+
+static inline unsigned int sa_scatterwalk_sglen(struct scatter_walk *walk)
+{
+	return walk->sg->offset + walk->sg->length - walk->offset;
+}
+
+static inline void *sa_scatterwalk_vaddr(struct scatter_walk *walk)
+{
+	return sg_virt(walk->sg) + (walk->offset - walk->sg->offset);
+}
+
+static inline void sa_scatterwalk_sgdone(struct scatter_walk *walk, size_t len)
+{
+	if (walk->offset >= walk->sg->offset + walk->sg->length)
+		scatterwalk_start(walk, sg_next(walk->sg));
+}
+
+/* scatterwalk_copychunks() for mapped SG list */
+static inline void
+sa_scatterwalk_copychunks(void *buf,
+			  struct scatter_walk *walk, unsigned int nbytes,
+			  int out)
+{
+	unsigned int len_this_sg;
+
+	for (;;) {
+		len_this_sg = sa_scatterwalk_sglen(walk);
+
+		if (len_this_sg > nbytes)
+			len_this_sg = nbytes;
+
+		if (out)
+			memcpy(sa_scatterwalk_vaddr(walk), buf,
+					len_this_sg);
+		else
+			memcpy(buf, sa_scatterwalk_vaddr(walk),
+					len_this_sg);
+
+		scatterwalk_advance(walk, len_this_sg);
+
+		if (nbytes == len_this_sg)
+			break;
+
+		buf += len_this_sg;
+		nbytes -= len_this_sg;
+
+		sa_scatterwalk_sgdone(walk, len_this_sg);
+	}
+}
+
+/* Copy buffer content from list of hwdesc-s to DST SG list */
+static int sa_hwdesc2sg_copy(struct knav_dma_desc **hwdesc,
+			     struct scatterlist *dst,
+			     unsigned int src_offset, unsigned int dst_offset,
+			     size_t len, int num)
+{
+	struct scatter_walk walk;
+	int sglen, cplen;
+	int j = 0;
+
+	sglen = hwdesc[0]->desc_info & KNAV_DMA_DESC_PKT_LEN_MASK;
+	if (unlikely(len + src_offset > sglen)) {
+		pr_err("[%s] src len(%d) less than (%d)\n", __func__,
+		       sglen, len + src_offset);
+		return -1;
+	}
+
+	sglen = sg_len(dst);
+	if (unlikely(len + dst_offset > sglen)) {
+		pr_err("[%s] dst len(%d) less than (%d)\n", __func__,
+		       sglen, len + dst_offset);
+		return -1;
+	}
+
+	scatterwalk_start(&walk, dst);
+	scatterwalk_advance(&walk, dst_offset);
+	while ((j < num) && (len > 0)) {
+		cplen = min((int)len, (int)(hwdesc[j]->buff_len - src_offset));
+		if (likely(cplen)) {
+			sa_scatterwalk_copychunks(((char *)hwdesc[j]->pad[0] +
+						   src_offset),
+						  &walk, cplen, 1);
+		}
+		len -= cplen;
+		j++;
+		src_offset = 0;
+	}
+	return 0;
+}
+
+void sa_scatterwalk_copy(void *buf, struct scatterlist *sg,
+			 unsigned int start, unsigned int nbytes, int out)
+{
+	struct scatter_walk walk;
+	unsigned int offset = 0;
+
+	if (!nbytes)
+		return;
+
+	for (;;) {
+		scatterwalk_start(&walk, sg);
+
+		if (start < offset + sg->length)
+			break;
+
+		offset += sg->length;
+		sg = sg_next(sg);
+	}
+
+	scatterwalk_advance(&walk, start - offset);
+	sa_scatterwalk_copychunks(buf, &walk, nbytes, out);
+}
+
+/******************************************************************************
+ * Command Label Definitions and utility functions
+ ******************************************************************************/
+struct sa_cmdl_cfg {
+	int	enc1st;
+	int	aalg;
+	u8	enc_eng_id;
+	u8	auth_eng_id;
+	u8	iv_size;
+	const u8 *akey;
+	u16	akey_len;
+};
+
+#define SA_CMDL_UPD_ENC		0x0001
+#define SA_CMDL_UPD_AUTH	0x0002
+#define SA_CMDL_UPD_ENC_IV	0x0004
+#define SA_CMDL_UPD_AUTH_IV	0x0008
+#define SA_CMDL_UPD_AUX_KEY	0x0010
+
+/* Format general command label */
+static int sa_format_cmdl_gen(struct sa_cmdl_cfg *cfg, u8 *cmdl,
+				struct sa_cmdl_upd_info *upd_info)
+{
+	u8 offset = 0;
+	u32 *word_ptr = (u32 *)cmdl;
+	int i;
+
+	/* Clear the command label */
+	memset(cmdl, 0, (SA_MAX_CMDL_WORDS * sizeof(u32)));
+
+	/* Iniialize the command update structure */
+	memset(upd_info, 0, sizeof(*upd_info));
+	upd_info->enc_size.offset = 2;
+	upd_info->enc_size.size = 2;
+	upd_info->enc_offset.size = 1;
+	upd_info->enc_size2.size = 4;
+	upd_info->auth_size.offset = 2;
+	upd_info->auth_size.size = 2;
+	upd_info->auth_offset.size = 1;
+
+	if (cfg->aalg == SA_AALG_ID_AES_XCBC) {
+
+		/* Derive K2/K3 subkeys */
+		if (sa_aes_xcbc_subkey(NULL, (u8 *)&upd_info->aux_key[0],
+			(u8 *)&upd_info->aux_key[AES_BLOCK_SIZE/sizeof(u32)],
+			cfg->akey, cfg->akey_len))
+			return -1;
+
+		/*
+		 * Format the key into 32bit CPU words
+		 * from a big-endian stream
+		 */
+		for (i = 0; i < SA_MAX_AUX_DATA_WORDS; i++)
+			upd_info->aux_key[i] =
+				be32_to_cpu(upd_info->aux_key[i]);
+	}
+
+	if (cfg->enc1st) {
+		if (cfg->enc_eng_id != SA_ENG_ID_NONE) {
+			upd_info->flags |= SA_CMDL_UPD_ENC;
+			upd_info->enc_size.index = 0;
+			upd_info->enc_offset.index = 1;
+
+			if ((cfg->enc_eng_id == SA_ENG_ID_EM1) &&
+			    (cfg->auth_eng_id == SA_ENG_ID_EM1))
+				cfg->auth_eng_id = SA_ENG_ID_EM2;
+
+			/* Encryption command label */
+			if (cfg->auth_eng_id != SA_ENG_ID_NONE)
+				cmdl[SA_CMDL_OFFSET_NESC] = cfg->auth_eng_id;
+			else
+				cmdl[SA_CMDL_OFFSET_NESC] = SA_ENG_ID_OUTPORT2;
+
+			/* Encryption modes requiring IV */
+			if (cfg->iv_size) {
+				upd_info->flags |= SA_CMDL_UPD_ENC_IV;
+				upd_info->enc_iv.index =
+					SA_CMDL_HEADER_SIZE_BYTES >> 2;
+				upd_info->enc_iv.size = cfg->iv_size;
+
+				cmdl[SA_CMDL_OFFSET_LABEL_LEN] =
+					SA_CMDL_HEADER_SIZE_BYTES +
+					cfg->iv_size;
+
+				cmdl[SA_CMDL_OFFSET_OPTION_CTRL1] =
+					(SA_CTX_ENC_AUX2_OFFSET |
+					 (cfg->iv_size >> 3));
+
+				offset = SA_CMDL_HEADER_SIZE_BYTES +
+						cfg->iv_size;
+			} else {
+				cmdl[SA_CMDL_OFFSET_LABEL_LEN] =
+					SA_CMDL_HEADER_SIZE_BYTES;
+				offset = SA_CMDL_HEADER_SIZE_BYTES;
+			}
+		}
+
+		if (cfg->auth_eng_id != SA_ENG_ID_NONE) {
+			upd_info->flags |= SA_CMDL_UPD_AUTH;
+			upd_info->auth_size.index = offset >> 2;
+			upd_info->auth_offset.index =
+				upd_info->auth_size.index + 1;
+
+			cmdl[offset + SA_CMDL_OFFSET_NESC] = SA_ENG_ID_OUTPORT2;
+
+			/* Algorithm with subkeys */
+			if ((cfg->aalg == SA_AALG_ID_AES_XCBC) ||
+				(cfg->aalg == SA_AALG_ID_CMAC)) {
+				upd_info->flags |= SA_CMDL_UPD_AUX_KEY;
+				upd_info->aux_key_info.index =
+				(offset + SA_CMDL_HEADER_SIZE_BYTES) >> 2;
+
+				cmdl[offset + SA_CMDL_OFFSET_LABEL_LEN] =
+					SA_CMDL_HEADER_SIZE_BYTES + 16;
+				cmdl[offset + SA_CMDL_OFFSET_OPTION_CTRL1] =
+					(SA_CTX_ENC_AUX1_OFFSET | (16 >> 3));
+
+				offset += SA_CMDL_HEADER_SIZE_BYTES + 16;
+			} else {
+				cmdl[offset + SA_CMDL_OFFSET_LABEL_LEN] =
+					SA_CMDL_HEADER_SIZE_BYTES;
+				offset += SA_CMDL_HEADER_SIZE_BYTES;
+			}
+		}
+	} else {
+		/* Auth first */
+		if (cfg->auth_eng_id != SA_ENG_ID_NONE) {
+			upd_info->flags |= SA_CMDL_UPD_AUTH;
+			upd_info->auth_size.index = 0;
+			upd_info->auth_offset.index = 1;
+
+			if ((cfg->auth_eng_id == SA_ENG_ID_EM1) &&
+				(cfg->enc_eng_id == SA_ENG_ID_EM1))
+				cfg->enc_eng_id = SA_ENG_ID_EM2;
+
+			/* Authentication command label */
+			if (cfg->enc_eng_id != SA_ENG_ID_NONE)
+				cmdl[SA_CMDL_OFFSET_NESC] = cfg->enc_eng_id;
+			else
+				cmdl[SA_CMDL_OFFSET_NESC] = SA_ENG_ID_OUTPORT2;
+
+			/* Algorithm with subkeys */
+			if ((cfg->aalg == SA_AALG_ID_AES_XCBC) ||
+				(cfg->aalg == SA_AALG_ID_CMAC)) {
+				upd_info->flags |= SA_CMDL_UPD_AUX_KEY;
+				upd_info->aux_key_info.index =
+					(SA_CMDL_HEADER_SIZE_BYTES) >> 2;
+
+				cmdl[SA_CMDL_OFFSET_LABEL_LEN] =
+					SA_CMDL_HEADER_SIZE_BYTES + 16;
+				cmdl[offset + SA_CMDL_OFFSET_OPTION_CTRL1] =
+					(SA_CTX_ENC_AUX1_OFFSET | (16 >> 3));
+
+				offset = SA_CMDL_HEADER_SIZE_BYTES + 16;
+			} else {
+				cmdl[SA_CMDL_OFFSET_LABEL_LEN] =
+					SA_CMDL_HEADER_SIZE_BYTES;
+				offset = SA_CMDL_HEADER_SIZE_BYTES;
+			}
+		}
+
+		if (cfg->enc_eng_id != SA_ENG_ID_NONE) {
+			upd_info->flags |= SA_CMDL_UPD_ENC;
+			upd_info->enc_size.index = offset >> 2;
+			upd_info->enc_offset.index =
+				upd_info->enc_size.index + 1;
+
+			cmdl[offset + SA_CMDL_OFFSET_NESC] = SA_ENG_ID_OUTPORT2;
+
+			/* Encryption modes requiring IV */
+			if (cfg->iv_size) {
+				upd_info->flags |= SA_CMDL_UPD_ENC_IV;
+				upd_info->enc_iv.index =
+				(offset + SA_CMDL_HEADER_SIZE_BYTES) >> 2;
+				upd_info->enc_iv.size = cfg->iv_size;
+
+				cmdl[offset + SA_CMDL_OFFSET_LABEL_LEN] =
+				SA_CMDL_HEADER_SIZE_BYTES + cfg->iv_size;
+
+				cmdl[offset + SA_CMDL_OFFSET_OPTION_CTRL1] =
+				(SA_CTX_ENC_AUX2_OFFSET | (cfg->iv_size >> 3));
+
+				offset += SA_CMDL_HEADER_SIZE_BYTES +
+						cfg->iv_size;
+			} else {
+				cmdl[offset + SA_CMDL_OFFSET_LABEL_LEN] =
+					SA_CMDL_HEADER_SIZE_BYTES;
+				offset += SA_CMDL_HEADER_SIZE_BYTES;
+			}
+		}
+	}
+
+	offset = roundup(offset, 8);
+
+	for (i = 0; i < offset/4; i++)
+		word_ptr[i] = be32_to_cpu(word_ptr[i]);
+
+	return offset;
+}
+
+/* Make 32-bit word from 4 bytes */
+#define SA_MK_U32(b0, b1, b2, b3) (((b0) << 24) | ((b1) << 16) | \
+					((b2) << 8) | (b3))
+
+/* Update Command label */
+static inline void
+sa_update_cmdl(struct device *dev, u8 enc_offset, u16 enc_size,	u8 *enc_iv,
+	       u8 auth_offset, u16 auth_size, u8 *auth_iv, u8 aad_size,
+	       u8 *aad,	struct sa_cmdl_upd_info	*upd_info, u32 *cmdl)
+{
+	switch (upd_info->submode) {
+	case SA_MODE_GEN:
+		if (likely(upd_info->flags & SA_CMDL_UPD_ENC)) {
+			cmdl[upd_info->enc_size.index] &= 0xffff0000;
+			cmdl[upd_info->enc_size.index] |= enc_size;
+			cmdl[upd_info->enc_offset.index] &= 0x00ffffff;
+			cmdl[upd_info->enc_offset.index] |=
+						((u32)enc_offset << 24);
+
+			if (likely(upd_info->flags & SA_CMDL_UPD_ENC_IV)) {
+				u32 *data = &cmdl[upd_info->enc_iv.index];
+
+				data[0] = SA_MK_U32(enc_iv[0], enc_iv[1],
+						    enc_iv[2], enc_iv[3]);
+				data[1] = SA_MK_U32(enc_iv[4], enc_iv[5],
+						    enc_iv[6], enc_iv[7]);
+
+				if (upd_info->enc_iv.size > 8) {
+					data[2] = SA_MK_U32(enc_iv[8],
+							    enc_iv[9],
+							    enc_iv[10],
+							    enc_iv[11]);
+					data[3] = SA_MK_U32(enc_iv[12],
+							    enc_iv[13],
+							    enc_iv[14],
+							    enc_iv[15]);
+				}
+			}
+		}
+
+		if (likely(upd_info->flags & SA_CMDL_UPD_AUTH)) {
+			cmdl[upd_info->auth_size.index] &= 0xffff0000;
+			cmdl[upd_info->auth_size.index] |= auth_size;
+			cmdl[upd_info->auth_offset.index] &= 0x00ffffff;
+			cmdl[upd_info->auth_offset.index] |=
+					((u32)auth_offset << 24);
+
+			if (upd_info->flags & SA_CMDL_UPD_AUTH_IV) {
+				u32 *data = &cmdl[upd_info->auth_iv.index];
+
+				data[0] = SA_MK_U32(auth_iv[0], auth_iv[1],
+							auth_iv[2], auth_iv[3]);
+				data[1] = SA_MK_U32(auth_iv[4], auth_iv[5],
+							auth_iv[6], auth_iv[7]);
+
+				if (upd_info->auth_iv.size > 8) {
+					data[2] = SA_MK_U32(auth_iv[8],
+					auth_iv[9], auth_iv[10], auth_iv[11]);
+					data[3] = SA_MK_U32(auth_iv[12],
+					auth_iv[13], auth_iv[14], auth_iv[15]);
+				}
+			}
+
+			if (upd_info->flags & SA_CMDL_UPD_AUX_KEY) {
+				int offset = (auth_size & 0xF) ? 4 : 0;
+
+				memcpy(&cmdl[upd_info->aux_key_info.index],
+						&upd_info->aux_key[offset], 16);
+			}
+		}
+		break;
+
+	case SA_MODE_CCM:
+	case SA_MODE_GCM:
+	case SA_MODE_GMAC:
+	default:
+		dev_err(dev, "unsupported mode(%d)\n", upd_info->submode);
+		break;
+
+	}
+}
+
+/* Format SWINFO words to be sent to SA */
+static void sa_set_swinfo(u8 eng_id, u16 sc_id, dma_addr_t sc_phys,
+		u8 cmdl_present, u8 cmdl_offset, u8 flags, u16 queue_id,
+		u8 flow_id, u8 hash_size, u32 *swinfo)
+{
+	swinfo[0] = sc_id;
+	swinfo[0] |= (flags << 16);
+	if (likely(cmdl_present))
+		swinfo[0] |= ((cmdl_offset | 0x10)  << 20);
+	swinfo[0] |= (eng_id << 25);
+	swinfo[0] |= 0x40000000;
+	swinfo[1] = sc_phys;
+	swinfo[2] = (queue_id | (flow_id << 16) | (hash_size << 24));
+}
+
+/******************************************************************************
+ * Security context creation functions
+ ******************************************************************************/
+
+/* Dump the security context */
+static void sa_dump_sc(u8 *buf, u32 dma_addr)
+{
+#ifdef DEBUG
+	dev_info(sa_ks2_dev, "Security context dump for %p:\n",
+			(void *)dma_addr);
+	print_hex_dump(KERN_CONT, "", DUMP_PREFIX_OFFSET,
+			16, 1, buf, SA_CTX_MAX_SZ, false);
+#endif
+}
+
+/* size of SCCTL structure in bytes */
+#define SA_SCCTL_SZ 8
+
+/* Initialize Security context */
+static int sa_init_sc(struct sa_ctx_info *ctx, const u8 *enc_key,
+			u16 enc_key_sz, const u8 *auth_key, u16 auth_key_sz,
+			const char *cra_name, u8 enc,
+			u32 *swinfo)
+{
+	struct sa_eng_info enc_eng, auth_eng;
+	int ealg_id, aalg_id, use_enc = 0;
+	int enc_sc_offset, auth_sc_offset;
+	u8 php_f, php_e, eng0_f, eng1_f;
+	u8 *sc_buf = ctx->sc;
+	u16 sc_id = ctx->sc_id;
+	u16 aad_len = 0; /* Currently not supporting AEAD algo */
+	u8 first_engine;
+	u8 hash_size;
+
+	memset(sc_buf, 0, SA_CTX_MAX_SZ);
+	sa_conv_calg_to_salg(cra_name, &ealg_id, &aalg_id);
+	sa_get_engine_info(ealg_id, &enc_eng);
+	sa_get_engine_info(aalg_id, &auth_eng);
+
+	if (!enc_eng.sc_size && !auth_eng.sc_size)
+		return -1;
+
+	if (auth_eng.eng_id <= SA_ENG_ID_EM2)
+		use_enc = 1;
+
+	/* Determine the order of encryption & Authentication contexts */
+	if (enc || !use_enc) {
+		eng0_f = SA_CTX_SIZE_TO_DMA_SIZE(enc_eng.sc_size);
+		eng1_f = SA_CTX_SIZE_TO_DMA_SIZE(auth_eng.sc_size);
+		enc_sc_offset = SA_CTX_PHP_PE_CTX_SZ;
+		auth_sc_offset = enc_sc_offset + enc_eng.sc_size;
+	} else {
+		eng0_f = SA_CTX_SIZE_TO_DMA_SIZE(auth_eng.sc_size);
+		eng1_f = SA_CTX_SIZE_TO_DMA_SIZE(enc_eng.sc_size);
+		auth_sc_offset = SA_CTX_PHP_PE_CTX_SZ;
+		enc_sc_offset = auth_sc_offset + auth_eng.sc_size;
+	}
+
+	php_f = php_e = SA_CTX_DMA_SIZE_64;
+
+	/* SCCTL Owner info: 0=host, 1=CP_ACE */
+	sc_buf[SA_CTX_SCCTL_OWNER_OFFSET] = 0;
+	/* SCCTL F/E control */
+	sc_buf[1] = SA_CTX_SCCTL_MK_DMA_INFO(php_f, eng0_f, eng1_f, php_e);
+	memcpy(&sc_buf[2], &sc_id, 2); /*(optional)
+					 Filled here for reference only */
+	memcpy(&sc_buf[4], &ctx->sc_phys, 4); /*(optional)
+					Filled here for reference only */
+
+	/* Initialize the rest of PHP context */
+	memset(sc_buf + SA_SCCTL_SZ, 0, SA_CTX_PHP_PE_CTX_SZ - SA_SCCTL_SZ);
+
+	/* Prepare context for encryption engine */
+	if (enc_eng.sc_size) {
+		if (sa_set_sc_enc(ealg_id, enc_key, enc_key_sz, aad_len,
+				enc, &sc_buf[enc_sc_offset]))
+			return -1;
+	}
+
+	/* Prepare context for authentication engine */
+	if (auth_eng.sc_size) {
+		if (use_enc) {
+			if (sa_set_sc_enc(aalg_id, auth_key, auth_key_sz,
+					aad_len, 0, &sc_buf[auth_sc_offset]))
+				return -1;
+		} else
+			sa_set_sc_auth(aalg_id, auth_key, auth_key_sz,
+					&sc_buf[auth_sc_offset]);
+	}
+
+	/* Set the ownership of context to CP_ACE */
+	sc_buf[SA_CTX_SCCTL_OWNER_OFFSET] = 0x80;
+
+	/* swizzle the security context */
+	sa_swiz_128(sc_buf, sc_buf, SA_CTX_MAX_SZ);
+
+	/* Setup SWINFO */
+	if (ealg_id == SA_EALG_ID_NULL)
+		first_engine = auth_eng.eng_id;
+	else
+		first_engine = enc ? enc_eng.eng_id : auth_eng.eng_id;
+
+	/* TODO: take care of AEAD algorithms */
+	hash_size = sa_get_hash_size(aalg_id);
+	if (!hash_size)
+		return -1;
+	/* Round up the tag size to multiple of 8 */
+	hash_size = roundup(hash_size, 8);
+
+#ifndef TEST
+	sa_set_swinfo(first_engine, ctx->sc_id, ctx->sc_phys, 1, 0,
+			0, ctx->rx_compl_qid, ctx->rx_flow, hash_size, swinfo);
+#else
+	/* For run-time self tests in the cryptographic
+	 * algorithm manager framework */
+	sa_set_swinfo(first_engine, ctx->sc_id, ctx->sc_phys, 1, 0,
+			SA_SW_INFO_FLAG_EVICT, ctx->rx_compl_qid, ctx->rx_flow,
+			hash_size, swinfo);
+#endif
+	sa_dump_sc(sc_buf, ctx->sc_phys);
+
+	return 0;
+}
+
+/* Tear down the Security Context */
+#define SA_SC_TEAR_RETRIES	5
+#define SA_SC_TEAR_DELAY	20 /* msecs */
+static int sa_tear_sc(struct sa_ctx_info *ctx,
+			struct keystone_crypto_data *pdata)
+{
+	struct device *dev = &pdata->pdev->dev;
+	int own_off, cnt = SA_SC_TEAR_RETRIES;
+	struct knav_dma_desc *hwdesc;
+	struct sa_dma_req_ctx *dma_ctx;
+	int ret = 0;
+	u32 packet_info;
+	int j;
+	dma_addr_t dma_addr;
+	u32 dma_sz;
+
+	dma_ctx = kmem_cache_alloc(pdata->dma_req_ctx_cache, GFP_KERNEL);
+	if (!dma_ctx) {
+		ret = -ENOMEM;
+		goto err;
+	}
+
+	dma_ctx->dev_data = pdata;
+	dma_ctx->pkt = false;
+
+	sa_set_swinfo(SA_ENG_ID_OUTPORT2, ctx->sc_id, ctx->sc_phys, 0, 0,
+		(SA_SW_INFO_FLAG_TEAR | SA_SW_INFO_FLAG_EVICT |
+		SA_SW_INFO_FLAG_NOPD), ctx->rx_compl_qid, ctx->rx_flow, 0,
+		&ctx->epib[1]);
+
+	ctx->epib[0] = 0;
+
+	/* map the packet */
+	packet_info = KNAV_DMA_DESC_HAS_EPIB |
+		(pdata->tx_compl_qid << KNAV_DMA_DESC_RETQ_SHIFT);
+
+	hwdesc = knav_pool_desc_get(pdata->tx_pool);
+	if (IS_ERR_OR_NULL(hwdesc)) {
+		dev_dbg(dev, "out of tx pool desc\n");
+		ret = -ENOBUFS;
+		goto err;
+	}
+
+	memset(hwdesc, 0, sizeof(struct knav_dma_desc));
+	for (j = 0; j < 4; j++)
+		hwdesc->epib[j] = ctx->epib[j];
+
+	hwdesc->packet_info  = packet_info;
+
+	knav_pool_desc_map(pdata->tx_pool, hwdesc, sizeof(hwdesc),
+				   &dma_addr, &dma_sz);
+
+	hwdesc->pad[0] = (u32)dma_addr;
+	hwdesc->pad[1] = dma_sz;
+	hwdesc->pad[2] = (u32)dma_ctx;
+
+	knav_queue_push(pdata->tx_submit_q, dma_addr,
+			sizeof(struct knav_dma_desc), 0);
+
+	/*
+	 * Check that CP_ACE has released the context
+	 * by making sure that the owner bit is 0
+	 */
+	/*
+	 * Security context had been swizzled by 128 bits
+	 * before handing to CP_ACE
+	 */
+	own_off = ((SA_CTX_SCCTL_OWNER_OFFSET/16) * 16)
+		+ (15 - (SA_CTX_SCCTL_OWNER_OFFSET % 16));
+	while (__raw_readb(&ctx->sc[own_off])) {
+		if (!--cnt)
+			return -EAGAIN;
+		msleep_interruptible(SA_SC_TEAR_DELAY);
+	}
+	return 0;
+
+err:
+	atomic_inc(&pdata->stats.sc_tear_dropped);
+	if (dma_ctx)
+		kmem_cache_free(pdata->dma_req_ctx_cache, dma_ctx);
+	return ret;
+}
+
+/************************************************************/
+/*	Algorithm interface functions & templates	*/
+/************************************************************/
+struct sa_alg_tmpl {
+	u32 type; /* CRYPTO_ALG_TYPE from <linux/crypto.h> */
+	union {
+		struct crypto_alg crypto;
+		struct ahash_alg hash;
+	} alg;
+	int registered;
+};
+
+/* Free the per direction context memory */
+static void sa_free_ctx_info(struct sa_ctx_info *ctx,
+			     struct keystone_crypto_data *data)
+{
+	unsigned long bn;
+
+	if (sa_tear_sc(ctx, data)) {
+		dev_err(sa_ks2_dev,
+			"Failed to tear down context id(%x)\n", ctx->sc_id);
+		return;
+	}
+
+	bn = ctx->sc_id - data->sc_id_start;
+	spin_lock(&data->scid_lock);
+	__clear_bit(bn, data->ctx_bm);
+	data->sc_id--;
+	spin_unlock(&data->scid_lock);
+
+	if (ctx->sc) {
+		dma_pool_free(data->sc_pool, ctx->sc, ctx->sc_phys);
+		ctx->sc = NULL;
+	}
+}
+
+/* Initialize the per direction context memory */
+static int sa_init_ctx_info(struct sa_ctx_info *ctx,
+			    struct keystone_crypto_data *data)
+{
+	unsigned long bn;
+	int err;
+
+	spin_lock(&data->scid_lock);
+	if (data->sc_id > data->sc_id_end) {
+		spin_unlock(&data->scid_lock);
+		dev_err(&data->pdev->dev, "Out of SC IDs\n");
+		return -1;
+	}
+	bn = find_first_zero_bit(data->ctx_bm, SA_MAX_NUM_CTX);
+	__set_bit(bn, data->ctx_bm);
+	data->sc_id++;
+	spin_unlock(&data->scid_lock);
+
+	ctx->sc_id = (u16)(data->sc_id_start + bn);
+
+	ctx->rx_flow = knav_dma_get_flow(data->rx_chan);
+	ctx->rx_compl_qid = data->rx_compl_qid;
+
+	ctx->sc = dma_pool_alloc(data->sc_pool, GFP_KERNEL, &ctx->sc_phys);
+	if (!ctx->sc) {
+		dev_err(&data->pdev->dev, "Failed to allocate SC memory\n");
+		err = -ENOMEM;
+		goto scid_rollback;
+	}
+
+	return 0;
+
+scid_rollback:
+	spin_lock(&data->scid_lock);
+	__clear_bit(bn, data->ctx_bm);
+	data->sc_id--;
+	spin_unlock(&data->scid_lock);
+
+	return err;
+}
+
+/* Initialize TFM context */
+static int sa_init_tfm(struct crypto_tfm *tfm)
+{
+	struct crypto_alg *alg = tfm->__crt_alg;
+	struct sa_alg_tmpl *sa_alg;
+	struct sa_tfm_ctx *ctx = crypto_tfm_ctx(tfm);
+	struct keystone_crypto_data *data = dev_get_drvdata(sa_ks2_dev);
+	int ret;
+
+	if ((alg->cra_flags & CRYPTO_ALG_TYPE_MASK) == CRYPTO_ALG_TYPE_AHASH)
+		sa_alg = container_of(__crypto_ahash_alg(alg),
+					struct sa_alg_tmpl, alg.hash);
+	else
+		sa_alg = container_of(alg, struct sa_alg_tmpl, alg.crypto);
+
+	memset(ctx, 0, sizeof(*ctx));
+	ctx->dev_data = data;
+
+	if (sa_alg->type == CRYPTO_ALG_TYPE_AHASH) {
+		ret = sa_init_ctx_info(&ctx->auth, data);
+		if (ret)
+			return ret;
+	} else if (sa_alg->type == CRYPTO_ALG_TYPE_AEAD) {
+		ret = sa_init_ctx_info(&ctx->enc, data);
+		if (ret)
+			return ret;
+		ret = sa_init_ctx_info(&ctx->dec, data);
+		if (ret) {
+			sa_free_ctx_info(&ctx->enc, data);
+			return ret;
+		}
+	} else if (sa_alg->type == CRYPTO_ALG_TYPE_ABLKCIPHER) {
+		ret = sa_init_ctx_info(&ctx->enc, data);
+		if (ret)
+			return ret;
+		ret = sa_init_ctx_info(&ctx->dec, data);
+		if (ret) {
+			sa_free_ctx_info(&ctx->enc, data);
+			return ret;
+		}
+	}
+
+	dev_dbg(sa_ks2_dev, "%s(0x%p) sc-ids(0x%x(0x%x), 0x%x(0x%x))\n",
+			__func__, tfm, ctx->enc.sc_id, ctx->enc.sc_phys,
+			ctx->dec.sc_id, ctx->dec.sc_phys);
+	return 0;
+}
+
+/* Algorithm init */
+static int sa_cra_init_aead(struct crypto_tfm *tfm)
+{
+	return sa_init_tfm(tfm);
+}
+
+/* Algorithm init */
+static int sa_cra_init_ablkcipher(struct crypto_tfm *tfm)
+{
+	return sa_init_tfm(tfm);
+}
+
+/* Algorithm init */
+static int sa_cra_init_ahash(struct crypto_tfm *tfm)
+{
+	return sa_init_tfm(tfm);
+}
+
+/* Algorithm context teardown */
+static void sa_exit_tfm(struct crypto_tfm *tfm)
+{
+	struct crypto_alg *alg = tfm->__crt_alg;
+	struct sa_tfm_ctx *ctx = crypto_tfm_ctx(tfm);
+	struct keystone_crypto_data *data = dev_get_drvdata(sa_ks2_dev);
+
+	dev_dbg(sa_ks2_dev, "%s(0x%p) sc-ids(0x%x(0x%x), 0x%x(0x%x))\n",
+			__func__, tfm, ctx->enc.sc_id, ctx->enc.sc_phys,
+			ctx->dec.sc_id, ctx->dec.sc_phys);
+
+	if ((alg->cra_flags & CRYPTO_ALG_TYPE_MASK)
+			== CRYPTO_ALG_TYPE_AEAD) {
+		sa_free_ctx_info(&ctx->enc, data);
+		sa_free_ctx_info(&ctx->dec, data);
+	} else if ((alg->cra_flags & CRYPTO_ALG_TYPE_MASK)
+			== CRYPTO_ALG_TYPE_AHASH) {
+		sa_free_ctx_info(&ctx->auth, data);
+	} else if ((alg->cra_flags & CRYPTO_ALG_TYPE_MASK)
+			== CRYPTO_ALG_TYPE_ABLKCIPHER) {
+		sa_free_ctx_info(&ctx->enc, data);
+		sa_free_ctx_info(&ctx->dec, data);
+	}
+}
+
+/* AEAD algorithm configuration interface function */
+static int sa_aead_setkey(struct crypto_aead *authenc,
+		const u8 *key, unsigned int keylen)
+{
+	struct sa_tfm_ctx *ctx = crypto_aead_ctx(authenc);
+	unsigned int enckey_len, authkey_len, auth_size;
+	struct rtattr *rta = (struct rtattr *)key;
+	struct crypto_authenc_key_param *param;
+	struct sa_eng_info enc_eng, auth_eng;
+	int ealg_id, aalg_id, cmdl_len;
+	struct sa_cmdl_cfg cfg;
+	u8 const *enc_key;
+	u8 const *auth_key;
+	const char *cra_name;
+
+	if (!RTA_OK(rta, keylen))
+		goto badkey;
+	if (rta->rta_type != CRYPTO_AUTHENC_KEYA_PARAM)
+		goto badkey;
+	if (RTA_PAYLOAD(rta) < sizeof(*param))
+		goto badkey;
+
+	param = RTA_DATA(rta);
+	enckey_len = be32_to_cpu(param->enckeylen);
+
+	key += RTA_ALIGN(rta->rta_len);
+	keylen -= RTA_ALIGN(rta->rta_len);
+
+	if (keylen < enckey_len)
+		goto badkey;
+
+	authkey_len = keylen - enckey_len;
+	auth_size = crypto_aead_authsize(authenc);
+
+	enc_key = key + authkey_len;
+	auth_key = key;
+
+	cra_name = crypto_tfm_alg_name(crypto_aead_tfm(authenc));
+
+	sa_conv_calg_to_salg(cra_name, &ealg_id, &aalg_id);
+	sa_get_engine_info(ealg_id, &enc_eng);
+	sa_get_engine_info(aalg_id, &auth_eng);
+
+	memset(&cfg, 0, sizeof(cfg));
+	cfg.enc1st = 1;
+	cfg.aalg = aalg_id;
+	cfg.enc_eng_id = enc_eng.eng_id;
+	cfg.auth_eng_id = auth_eng.eng_id;
+	cfg.iv_size = crypto_aead_ivsize(authenc);
+	cfg.akey = auth_key;
+	cfg.akey_len = authkey_len;
+
+	/* Setup Encryption Security Context & Command label template */
+	if (sa_init_sc(&ctx->enc, enc_key, enckey_len, auth_key,
+				authkey_len, cra_name, 1, &ctx->enc.epib[1]))
+		goto badkey;
+
+	cmdl_len = sa_format_cmdl_gen(&cfg,
+				(u8 *)ctx->enc.cmdl, &ctx->enc.cmdl_upd_info);
+	if ((cmdl_len <= 0) || (cmdl_len > SA_MAX_CMDL_WORDS * sizeof(u32)))
+		goto badkey;
+
+	ctx->enc.cmdl_size = cmdl_len;
+
+	/* Setup Decryption Security Context & Command label template */
+	if (sa_init_sc(&ctx->dec, enc_key, enckey_len, auth_key,
+				authkey_len, cra_name, 0, &ctx->dec.epib[1]))
+		goto badkey;
+
+	cfg.enc1st = 0;
+	cfg.enc_eng_id = enc_eng.eng_id;
+	cfg.auth_eng_id = auth_eng.eng_id;
+	cmdl_len = sa_format_cmdl_gen(&cfg,
+				      (u8 *)ctx->dec.cmdl,
+				      &ctx->dec.cmdl_upd_info);
+
+	if ((cmdl_len <= 0) || (cmdl_len > SA_MAX_CMDL_WORDS * sizeof(u32)))
+		goto badkey;
+
+	ctx->dec.cmdl_size = cmdl_len;
+	return 0;
+
+badkey:
+	dev_err(sa_ks2_dev, "%s: badkey\n", __func__);
+	crypto_aead_set_flags(authenc, CRYPTO_TFM_RES_BAD_KEY_LEN);
+	return -EINVAL;
+}
+
+/* AEAD algorithm configuration interface function */
+static int sa_aead_setauthsize(struct crypto_aead *tfm,
+				unsigned int auth_size)
+{
+	if (auth_size > crypto_aead_alg(tfm)->maxauthsize)
+		return -EINVAL;
+	return 0;
+}
+
+dma_addr_t
+sa_prepare_tx_desc(struct keystone_crypto_data *pdata, struct scatterlist *_sg,
+		   int num_sg, u32 pslen, u32 *psdata,
+		   u32 epiblen, u32 *epib, struct sa_dma_req_ctx *ctx)
+{
+	struct device *dev = &pdata->pdev->dev;
+	struct knav_dma_desc *hwdesc = NULL;
+	struct scatterlist *sg = _sg;
+	u32 packet_len = 0;
+	u32 nsg;
+	u32 next_desc = 0;
+	u32 packet_info;
+
+	packet_info = KNAV_DMA_DESC_HAS_EPIB |
+		((pslen / sizeof(u32)) << KNAV_DMA_DESC_PSLEN_SHIFT) |
+		(pdata->tx_compl_qid << KNAV_DMA_DESC_RETQ_SHIFT);
+
+	for (sg += num_sg - 1, nsg = num_sg; nsg > 0; sg--, nsg--) {
+		u32 buflen, orig_len;
+		int i;
+		dma_addr_t dma_addr;
+		u32 dma_sz;
+		u32 *out, *in;
+
+		hwdesc = knav_pool_desc_get(pdata->tx_pool);
+		if (IS_ERR_OR_NULL(hwdesc)) {
+			dev_dbg(dev, "out of tx pool desc\n");
+			/* TODO: we need to return all pooped descriptors */
+			return 0;
+		}
+
+		buflen = sg_dma_len(sg) & MASK(22);
+		orig_len = (pdata->tx_submit_qid << 28) | buflen;
+		packet_len += buflen;
+
+		if (nsg == 1) { /* extra fileds for packed descriptor */
+			for (out = hwdesc->epib, in = epib, i = 0;
+			     i < epiblen / sizeof(u32); i++)
+				*out++ = *in++;
+			for (out = hwdesc->psdata, in = psdata, i = 0;
+			     i < pslen / sizeof(u32); i++)
+				*out++ = *in++;
+		}
+
+		hwdesc->desc_info    = packet_len;
+		hwdesc->tag_info     = 0;
+		hwdesc->packet_info  = packet_info;
+		hwdesc->buff_len     = buflen;
+		hwdesc->buff         = sg_dma_address(sg);
+		hwdesc->next_desc    = next_desc;
+		hwdesc->orig_len     = orig_len;
+		hwdesc->orig_buff    = sg_dma_address(sg);
+
+		knav_pool_desc_map(pdata->tx_pool, hwdesc, sizeof(hwdesc),
+				   &dma_addr, &dma_sz);
+
+		hwdesc->pad[0] = (u32)dma_addr;
+		hwdesc->pad[1] = dma_sz;
+		hwdesc->pad[2] = (u32)ctx;
+
+		next_desc = (u32)dma_addr;
+
+	}
+
+	return (unlikely(hwdesc == NULL)) ? 0 : hwdesc->pad[0];
+}
+
+void sa_tx_completion_process(struct keystone_crypto_data *dev_data)
+{
+	struct knav_dma_desc *hwdesc = NULL;
+	dma_addr_t dma;
+	struct sa_dma_req_ctx *ctx = NULL;
+	u32	pkt_len;
+	u32	calc_pkt_len;
+
+	for (;;) {
+		dma = knav_queue_pop(dev_data->tx_compl_q, NULL);
+		if (!dma) {
+			dev_dbg(sa_ks2_dev, "no desc in the queue %d\n",
+				dev_data->tx_compl_qid);
+			break;
+		}
+
+		ctx = NULL;
+		pkt_len = 0;
+		calc_pkt_len = 0;
+
+		do {
+			hwdesc = knav_pool_desc_unmap(dev_data->tx_pool, dma,
+						      sizeof(hwdesc));
+			if (!hwdesc) {
+				pr_err("failed to unmap descriptor 0x%08x\n",
+				       dma);
+				break;
+			}
+			/* take the req_ctx from the first descriptor */
+			if (!ctx) {
+				ctx = (struct sa_dma_req_ctx
+					   *)hwdesc->pad[2];
+				pkt_len = hwdesc->desc_info &
+					KNAV_DMA_DESC_PKT_LEN_MASK;
+			}
+			calc_pkt_len += hwdesc->buff_len;
+			/* do we need to unmap buffer here, or will do it
+			 * later
+			 */
+
+			dma = hwdesc->next_desc;
+
+			knav_pool_desc_put(dev_data->tx_pool, hwdesc);
+		} while (dma);
+
+		if (pkt_len != calc_pkt_len)
+			pr_err("[%s] calculated packet length doesn't match %d/%d\n",
+			       __func__, calc_pkt_len, pkt_len);
+
+		if ((pkt_len > 0) && ctx) {
+			dma_unmap_sg(&ctx->dev_data->pdev->dev, ctx->sg_tbl.sgl,
+				     ctx->sg_tbl.nents, DMA_TO_DEVICE);
+
+			if (likely(ctx->pkt)) {
+				atomic_add(ctx->sg_tbl.nents,
+					   &ctx->dev_data->tx_dma_desc_cnt);
+				atomic_inc(&ctx->dev_data->stats.tx_pkts);
+			}
+
+			if (likely(ctx->sg_tbl.sgl))
+				sg_free_table(&ctx->sg_tbl);
+		}
+
+		if (ctx)
+			kmem_cache_free(ctx->dev_data->dma_req_ctx_cache, ctx);
+	}
+}
+
+static void sa_rx_desc_process(struct keystone_crypto_data *dev_data,
+			struct knav_dma_desc **hwdesc, int num)
+{
+	int			j;
+	unsigned int		alg_type;
+	u32			req_sub_type;
+
+	alg_type = hwdesc[0]->psdata[0] & CRYPTO_ALG_TYPE_MASK;
+	req_sub_type = hwdesc[0]->psdata[0] >> SA_REQ_SUBTYPE_SHIFT;
+
+	if (likely(alg_type == CRYPTO_ALG_TYPE_AEAD)) {
+		int auth_words, auth_size, iv_size, enc_len, enc_offset, i;
+		struct aead_request *req;
+		struct crypto_aead *tfm;
+		int enc, err = 0;
+
+		req = (struct aead_request *)hwdesc[0]->psdata[1];
+		tfm = crypto_aead_reqtfm(req);
+		auth_size = crypto_aead_authsize(tfm);
+		iv_size = crypto_aead_ivsize(tfm);
+		enc_offset = req->assoclen + iv_size;
+
+		if (req_sub_type == SA_REQ_SUBTYPE_ENC) {
+			enc_len = req->cryptlen;
+			enc = 1;
+		} else if (req_sub_type == SA_REQ_SUBTYPE_DEC) {
+			enc_len = req->cryptlen - auth_size;
+			enc = 0;
+		} else {
+			err = -EBADMSG;
+			goto aead_err;
+		}
+
+		/* NOTE: We receive the tag as host endian 32bit words */
+		auth_words = auth_size/sizeof(u32);
+
+		for (i = 2; i < (auth_words + SA_NUM_PSDATA_CTX_WORDS); i++)
+			hwdesc[0]->psdata[i] = htonl(hwdesc[0]->psdata[i]);
+
+		/* if encryption, copy the authentication tag */
+		if (enc) {
+			sa_scatterwalk_copy(
+				&(hwdesc[0]->psdata[SA_NUM_PSDATA_CTX_WORDS]),
+				req->dst, enc_len, auth_size, 1);
+#ifdef DEBUG
+			dev_info(sa_ks2_dev, "computed tag:\n");
+			print_hex_dump(KERN_CONT, "", DUMP_PREFIX_OFFSET,
+			16, 1, &(hwdesc[0]->psdata[SA_NUM_PSDATA_CTX_WORDS]),
+			auth_size, false);
+#endif
+		} else  {
+			/* Verify the authentication tag */
+			u8 auth_tag[SA_MAX_AUTH_TAG_SZ];
+
+			sa_scatterwalk_copy(auth_tag, req->src, enc_len,
+					auth_size, 0);
+
+#ifdef DEBUG
+			dev_info(sa_ks2_dev, "expected tag:\n");
+			print_hex_dump(KERN_CONT, "", DUMP_PREFIX_OFFSET,
+			16, 1, auth_tag, auth_size, false);
+			dev_info(sa_ks2_dev, "computed tag:\n");
+			print_hex_dump(KERN_CONT, "", DUMP_PREFIX_OFFSET,
+			16, 1, &(hwdesc[0]->psdata[SA_NUM_PSDATA_CTX_WORDS]),
+			auth_size, false);
+#endif
+
+			err = memcmp(&(hwdesc[0]->psdata[SA_NUM_PSDATA_CTX_WORDS]),
+					auth_tag, auth_size) ? -EBADMSG : 0;
+			if (unlikely(err))
+				goto aead_err;
+		}
+
+		/* Copy the encrypted/decrypted data */
+		if (unlikely(sa_hwdesc2sg_copy(hwdesc, req->dst,
+					       enc_offset, 0, enc_len, num)))
+			err = -EBADMSG;
+
+aead_err:
+		aead_request_complete(req, err);
+	}
+
+	/* free buffers here */
+	for (j = 0; j < num; j++) {
+		if (hwdesc[j]->orig_len == PAGE_SIZE) {
+			__free_page((struct page *)hwdesc[j]->pad[1]);
+			atomic_dec(&dev_data->rx_dma_page_cnt);
+		} else
+			kfree((void *)hwdesc[j]->pad[0]);
+	}
+
+	atomic_inc(&dev_data->stats.rx_pkts);
+}
+
+void sa_rx_completion_process(struct keystone_crypto_data *dev_data)
+{
+	struct knav_dma_desc	*hwdesc[MAX_SKB_FRAGS];
+	int			j, desc_num;
+	dma_addr_t		dma;
+	u32			pkt_len;
+	u32			calc_pkt_len;
+	int			wait4pkt = 1;
+
+	for (;;) {
+		dma = knav_queue_pop(dev_data->rx_compl_q, NULL);
+		if (!dma) {
+			dev_dbg(sa_ks2_dev, "no desc in the queue %d\n",
+				dev_data->rx_compl_qid);
+			break;
+		}
+
+		pkt_len = 0;
+		calc_pkt_len = 0;
+		wait4pkt = 1;
+		desc_num = 0;
+
+		do {
+			hwdesc[desc_num] = knav_pool_desc_unmap(dev_data->rx_pool, dma,
+						      sizeof(hwdesc));
+			if (!hwdesc[desc_num]) {
+				pr_err("failed to unmap descriptor 0x%08x\n",
+				       dma);
+				break;
+			}
+
+			if (hwdesc[desc_num]->orig_len == PAGE_SIZE) {
+				dma_unmap_page(sa_ks2_dev,
+					       hwdesc[desc_num]->orig_buff,
+					       PAGE_SIZE,
+					       DMA_FROM_DEVICE);
+			} else {
+				dma_unmap_single(sa_ks2_dev,
+						 hwdesc[desc_num]->orig_buff,
+						 dev_data->rx_buffer_sizes[0],
+						 DMA_FROM_DEVICE);
+			}
+
+			/* take the req_ctx from the first descriptor */
+			if (wait4pkt) {
+				pkt_len = hwdesc[desc_num]->desc_info &
+					KNAV_DMA_DESC_PKT_LEN_MASK;
+				wait4pkt = 0;
+			}
+			calc_pkt_len += hwdesc[desc_num]->buff_len;
+
+			dma = hwdesc[desc_num]->next_desc;
+			desc_num++;
+		} while (dma);
+
+		if (pkt_len != calc_pkt_len)
+			pr_err("[%s] calculated packet length doesn't match %d/%d\n",
+			       __func__, calc_pkt_len, pkt_len);
+
+		/* retrieve data and copy it to the destination sg list */
+		sa_rx_desc_process(dev_data, hwdesc, desc_num);
+
+		/* return descriptor to the pool */
+		for (j = 0; j < desc_num; j++)
+			knav_pool_desc_put(dev_data->tx_pool, hwdesc[j]);
+
+		/* increment rx packet counter */
+	}
+}
+
+static int sa_aead_perform(struct aead_request *req, u8 *iv, int enc)
+{
+	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
+	struct sa_tfm_ctx *ctx = crypto_aead_ctx(tfm);
+	struct sa_ctx_info *sa_ctx = enc ? &ctx->enc : &ctx->dec;
+	dma_addr_t desc_dma_addr;
+	struct keystone_crypto_data *pdata = dev_get_drvdata(sa_ks2_dev);
+	unsigned ivsize = crypto_aead_ivsize(tfm);
+	u8 enc_offset = req->assoclen + ivsize;
+	struct sa_dma_req_ctx *req_ctx = NULL;
+	int sg_nents;
+	int assoc_sgents, src_sgents;
+	int psdata_offset, ret = 0;
+	u8 auth_offset = 0;
+	u8 *auth_iv = NULL;
+	u8 *aad = NULL;
+	u8 aad_len = 0;
+	int sg_idx = 0;
+	u16 enc_len;
+	u16 auth_len;
+	u32 req_type;
+	int n_bufs;
+
+	gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ?
+			GFP_KERNEL : GFP_ATOMIC;
+
+	enc_len = req->cryptlen;
+
+	/* req->cryptlen includes authsize when decrypting */
+	if (!enc)
+		enc_len -= crypto_aead_authsize(tfm);
+
+	auth_len = req->assoclen + ivsize + enc_len;
+
+	/* Allocate descriptor & submit packet */
+	assoc_sgents = sg_count(req->assoc, req->assoclen);
+	sg_nents = assoc_sgents;
+	src_sgents = sg_count(req->src, enc_len);
+	sg_nents += src_sgents;
+
+	if (likely(ivsize))
+		sg_nents += 1;
+
+	if (unlikely(atomic_sub_return(sg_nents, &pdata->tx_dma_desc_cnt)
+		     < 0)) {
+		ret = -EBUSY;
+		goto err_0;
+	}
+
+	n_bufs = auth_len - pdata->rx_buffer_sizes[0];
+
+	n_bufs = (n_bufs <= 0) ? 0 :
+		DIV_ROUND_UP(n_bufs, pdata->rx_buffer_sizes[1]);
+
+	if (unlikely(atomic_read(&pdata->rx_dma_page_cnt) < n_bufs)) {
+		ret = -EBUSY;
+		goto err_0;
+	}
+
+	req_ctx = kmem_cache_alloc(pdata->dma_req_ctx_cache, flags);
+
+	if (unlikely(req_ctx == NULL)) {
+		ret = -ENOMEM;
+		goto err_0;
+	}
+
+	if (unlikely(sg_alloc_table(&req_ctx->sg_tbl, sg_nents, flags))) {
+		ret = -ENOMEM;
+		goto err_1;
+	}
+
+	memcpy(req_ctx->cmdl, sa_ctx->cmdl, sa_ctx->cmdl_size);
+	/* Update Command Label */
+	sa_update_cmdl(sa_ks2_dev, enc_offset, enc_len,
+			iv, auth_offset, auth_len,
+			auth_iv, aad_len, aad,
+			&sa_ctx->cmdl_upd_info, req_ctx->cmdl);
+
+	/* Last 2 words in PSDATA will have the crypto alg type &
+	 * crypto request pointer
+	 */
+	req_type = CRYPTO_ALG_TYPE_AEAD;
+	if (enc)
+		req_type |= (SA_REQ_SUBTYPE_ENC << SA_REQ_SUBTYPE_SHIFT);
+	else
+		req_type |= (SA_REQ_SUBTYPE_DEC << SA_REQ_SUBTYPE_SHIFT);
+	psdata_offset = sa_ctx->cmdl_size/sizeof(u32);
+	/* Append the type of request */
+	req_ctx->cmdl[psdata_offset++] = req_type;
+	/* Append the pointer to request */
+	req_ctx->cmdl[psdata_offset] = (u32)req;
+
+#ifdef DEBUG
+	dev_info(sa_ks2_dev, "cmdl:\n");
+	print_hex_dump(KERN_CONT, "", DUMP_PREFIX_OFFSET,
+			16, 4, req_ctx->cmdl, sa_ctx->cmdl_size + 8, false);
+#endif
+	/* clone the assoc sg list */
+	if (likely(req->assoclen)) {
+		sa_clone_sg(req->assoc, &req_ctx->sg_tbl.sgl[sg_idx],
+				req->assoclen);
+		sg_idx += assoc_sgents;
+	}
+
+	if (likely(ivsize))
+		sg_set_buf(&req_ctx->sg_tbl.sgl[sg_idx++], iv, ivsize);
+
+	/* clone the src sg list */
+	if (likely(enc_len)) {
+		sa_clone_sg(req->src, &req_ctx->sg_tbl.sgl[sg_idx], enc_len);
+		sg_idx += src_sgents;
+	}
+
+	/* map the packet */
+	req_ctx->sg_tbl.nents = dma_map_sg(sa_ks2_dev, req_ctx->sg_tbl.sgl,
+					sg_nents, DMA_TO_DEVICE);
+
+	if (unlikely(req_ctx->sg_tbl.nents != sg_nents)) {
+		dev_warn_ratelimited(sa_ks2_dev, "failed to map tx pkt\n");
+		ret = -EIO;
+		goto err;
+	}
+
+	req_ctx->dev_data = pdata;
+	req_ctx->pkt = true;
+
+/*
+ * here we have the req_ctx->sg_tbl with a chain of packets ready to go.
+ * Let's start filling HW descriptors and submit packet to the queue
+ */
+	desc_dma_addr = sa_prepare_tx_desc(pdata, req_ctx->sg_tbl.sgl,
+					   sg_nents,
+					   (sa_ctx->cmdl_size +
+					    (SA_NUM_PSDATA_CTX_WORDS *
+					     sizeof(u32))),
+					   req_ctx->cmdl,
+					   sizeof(sa_ctx->epib),
+					   sa_ctx->epib,
+					   req_ctx);
+
+	if (desc_dma_addr == 0) {
+		ret = -EIO;
+		goto err;
+	}
+
+	knav_queue_push(pdata->tx_submit_q, desc_dma_addr,
+			sizeof(struct knav_dma_desc), 0);
+
+	return -EINPROGRESS;
+
+err:
+	if (req_ctx && req_ctx->sg_tbl.sgl)
+		sg_free_table(&req_ctx->sg_tbl);
+err_1:
+	if (req_ctx)
+		kmem_cache_free(pdata->dma_req_ctx_cache, req_ctx);
+err_0:
+	atomic_add((sg_nents - SA_NUM_DMA_META_ELEMS), &pdata->tx_dma_desc_cnt);
+	atomic_inc(&pdata->stats.tx_dropped);
+	return ret;
+}
+
+/* AEAD algorithm encrypt interface function */
+static int sa_aead_encrypt(struct aead_request *req)
+{
+	return sa_aead_perform(req, req->iv, 1);
+}
+
+/* AEAD algorithm decrypt interface function */
+static int sa_aead_decrypt(struct aead_request *req)
+{
+	return sa_aead_perform(req, req->iv, 0);
+}
+
+/* AEAD algorithm givencrypt interface function */
+static int sa_aead_givencrypt(struct aead_givcrypt_request *req)
+{
+	struct crypto_aead *tfm = aead_givcrypt_reqtfm(req);
+
+	get_random_bytes(req->giv, crypto_aead_ivsize(tfm));
+	return sa_aead_perform(&req->areq, req->giv, 1);
+}
+
+static int sa_ablkcipher_setkey(struct crypto_ablkcipher *cipher,
+			     const u8 *key, unsigned int keylen)
+{
+	return 0;
+}
+
+static int sa_ablkcipher_encrypt(struct ablkcipher_request *areq)
+{
+	return 0;
+}
+
+static int sa_ablkcipher_decrypt(struct ablkcipher_request *areq)
+{
+	return 0;
+}
+
+static int sa_ahash_init(struct ahash_request *areq)
+{
+	return 0;
+}
+
+static int sa_ahash_update(struct ahash_request *areq)
+{
+	return 0;
+}
+
+static int sa_ahash_final(struct ahash_request *areq)
+{
+	return 0;
+}
+
+static int sa_ahash_finup(struct ahash_request *areq)
+{
+	return 0;
+}
+
+static int sa_ahash_digest(struct ahash_request *areq)
+{
+	return 0;
+}
+
+static int sa_ahash_setkey(struct crypto_ahash *tfm, const u8 *key,
+			unsigned int keylen)
+{
+	return 0;
+}
+
+static struct sa_alg_tmpl sa_algs[] = {
+	/* AEAD algorithms */
+	{	.type = CRYPTO_ALG_TYPE_AEAD,
+		.alg.crypto = {
+			.cra_name = "authenc(hmac(sha1),cbc(aes))",
+			.cra_driver_name =
+				"authenc-hmac-sha1-cbc-aes-keystone-sa",
+			.cra_blocksize = AES_BLOCK_SIZE,
+			.cra_aead = {
+				.geniv = "custom",
+				.ivsize = AES_BLOCK_SIZE,
+				.maxauthsize = SHA1_DIGEST_SIZE,
+			}
+		}
+	},
+	{	.type = CRYPTO_ALG_TYPE_AEAD,
+		.alg.crypto = {
+			.cra_name = "authenc(hmac(sha1),cbc(des3_ede))",
+			.cra_driver_name =
+				"authenc-hmac-sha1-cbc-3des-keystone-sa",
+			.cra_blocksize = DES3_EDE_BLOCK_SIZE,
+			.cra_aead = {
+				.geniv = "custom",
+				.ivsize = DES3_EDE_BLOCK_SIZE,
+				.maxauthsize = SHA1_DIGEST_SIZE,
+			}
+		}
+	},
+	{       .type = CRYPTO_ALG_TYPE_AEAD,
+		.alg.crypto = {
+			.cra_name = "authenc(xcbc(aes),cbc(aes))",
+			.cra_driver_name =
+				"authenc-aes-xcbc-mac-cbc-aes-keystone-sa",
+			.cra_blocksize = AES_BLOCK_SIZE,
+			.cra_aead = {
+				.geniv = "custom",
+				.ivsize = AES_BLOCK_SIZE,
+				.maxauthsize = AES_XCBC_DIGEST_SIZE,
+			}
+		}
+	},
+	{	.type = CRYPTO_ALG_TYPE_AEAD,
+		.alg.crypto = {
+			.cra_name = "authenc(xcbc(aes),cbc(des3_ede))",
+			.cra_driver_name =
+				"authenc-aes-xcbc-mac-cbc-3des-keystone-sa",
+			.cra_blocksize = DES3_EDE_BLOCK_SIZE,
+			.cra_aead = {
+				.geniv = "custom",
+				.ivsize = DES3_EDE_BLOCK_SIZE,
+				.maxauthsize = AES_XCBC_DIGEST_SIZE,
+			}
+		}
+	},
+	{	.type = CRYPTO_ALG_TYPE_AEAD,
+		.alg.crypto = {
+			.cra_name = "authenc(hmac(sha1),ecb(cipher_null))",
+			.cra_driver_name =
+				"authenc-hmac-sha1-cipher_null-keystone-sa",
+			.cra_blocksize = NULL_BLOCK_SIZE,
+			.cra_aead = {
+				.geniv = "custom",
+				.ivsize = NULL_IV_SIZE,
+				.maxauthsize = SHA1_DIGEST_SIZE,
+			}
+		}
+	},
+
+#ifdef TODO
+	{	.type = CRYPTO_ALG_TYPE_AEAD,
+		.alg.crypto = {
+			.cra_name = "authenc(hmac(md5),cbc(aes))",
+			.cra_driver_name =
+				"authenc-hmac-md5-cbc-aes-keystone-sa",
+			.cra_blocksize = AES_BLOCK_SIZE,
+			.cra_aead = {
+				.geniv = "custom",
+				.ivsize = AES_BLOCK_SIZE,
+				.maxauthsize = MD5_DIGEST_SIZE,
+			}
+		}
+	},
+	{	.type = CRYPTO_ALG_TYPE_AEAD,
+		.alg.crypto = {
+			.cra_name = "authenc(hmac(md5),cbc(des3_ede))",
+			.cra_driver_name =
+				"authenc-hmac-md5-cbc-3des-keystone-sa",
+			.cra_blocksize = DES3_EDE_BLOCK_SIZE,
+			.cra_aead = {
+				.geniv = "custom",
+				.ivsize = DES3_EDE_BLOCK_SIZE,
+				.maxauthsize = MD5_DIGEST_SIZE,
+			}
+		}
+	},
+	/* ABLKCIPHER algorithms. */
+	{	.type = CRYPTO_ALG_TYPE_ABLKCIPHER,
+		.alg.crypto = {
+			.cra_name = "cbc(aes)",
+			.cra_driver_name = "cbc-aes-keystone-sa",
+			.cra_blocksize = AES_BLOCK_SIZE,
+			.cra_ablkcipher = {
+				.geniv = "custom",
+				.min_keysize = AES_MIN_KEY_SIZE,
+				.max_keysize = AES_MAX_KEY_SIZE,
+				.ivsize = AES_BLOCK_SIZE,
+			}
+		}
+	},
+	{	.type = CRYPTO_ALG_TYPE_ABLKCIPHER,
+		.alg.crypto = {
+			.cra_name = "cbc(des3_ede)",
+			.cra_driver_name = "cbc-3des-keystone-sa",
+			.cra_blocksize = DES3_EDE_BLOCK_SIZE,
+			.cra_ablkcipher = {
+				.geniv = "custom",
+				.min_keysize = DES3_EDE_KEY_SIZE,
+				.max_keysize = DES3_EDE_KEY_SIZE,
+				.ivsize = DES3_EDE_BLOCK_SIZE,
+			}
+		}
+	},
+	/* AHASH algorithms. */
+	{	.type = CRYPTO_ALG_TYPE_AHASH,
+		.alg.hash = {
+			.halg.digestsize = AES_XCBC_DIGEST_SIZE,
+			.halg.base = {
+				.cra_name = "xcbc(aes)",
+				.cra_driver_name =
+					"aes-xcbc-mac-keystone-sa",
+				.cra_blocksize = SHA224_BLOCK_SIZE,
+			}
+		}
+	},
+	{	.type = CRYPTO_ALG_TYPE_AHASH,
+		.alg.hash = {
+			.halg.digestsize = MD5_DIGEST_SIZE,
+			.halg.base = {
+				.cra_name = "hmac(md5)",
+				.cra_driver_name =
+					"hmac-md5-keystone-sa",
+				.cra_blocksize = MD5_BLOCK_SIZE,
+			}
+		}
+	},
+	{	.type = CRYPTO_ALG_TYPE_AHASH,
+		.alg.hash = {
+			.halg.digestsize = SHA1_DIGEST_SIZE,
+			.halg.base = {
+				.cra_name = "hmac(sha1)",
+				.cra_driver_name =
+					"hmac-sha1-keystone-sa",
+				.cra_blocksize = SHA1_BLOCK_SIZE,
+			}
+		}
+	}
+#endif
+};
+
+/* Register the algorithms in crypto framework */
+void sa_register_algos(const struct device *dev)
+{
+	struct crypto_alg *cra;
+	struct ahash_alg *hash = NULL;
+	char *alg_name;
+	u32 type;
+	int i, err, num_algs = ARRAY_SIZE(sa_algs);
+
+	for (i = 0; i < num_algs; i++) {
+		type = sa_algs[i].type;
+		if (type == CRYPTO_ALG_TYPE_AEAD) {
+			cra = &sa_algs[i].alg.crypto;
+			alg_name = cra->cra_name;
+			if (snprintf(cra->cra_driver_name, CRYPTO_MAX_ALG_NAME,
+			"%s-keystone-sa", alg_name) >= CRYPTO_MAX_ALG_NAME) {
+				continue;
+			}
+			cra->cra_type = &crypto_aead_type;
+			cra->cra_flags = CRYPTO_ALG_TYPE_AEAD |
+					CRYPTO_ALG_KERN_DRIVER_ONLY |
+					CRYPTO_ALG_ASYNC;
+			cra->cra_aead.setkey = sa_aead_setkey;
+			cra->cra_aead.setauthsize = sa_aead_setauthsize;
+			cra->cra_aead.encrypt = sa_aead_encrypt;
+			cra->cra_aead.decrypt = sa_aead_decrypt;
+			cra->cra_aead.givencrypt = sa_aead_givencrypt;
+			cra->cra_init = sa_cra_init_aead;
+		} else if (type == CRYPTO_ALG_TYPE_ABLKCIPHER) {
+			cra = &sa_algs[i].alg.crypto;
+			alg_name = cra->cra_name;
+			if (snprintf(cra->cra_driver_name, CRYPTO_MAX_ALG_NAME,
+			"%s-keystone-sa", alg_name) >= CRYPTO_MAX_ALG_NAME) {
+				continue;
+			}
+			cra->cra_type = &crypto_ablkcipher_type;
+			cra->cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
+					CRYPTO_ALG_KERN_DRIVER_ONLY |
+					CRYPTO_ALG_ASYNC;
+			cra->cra_ablkcipher.setkey = sa_ablkcipher_setkey;
+			cra->cra_ablkcipher.encrypt = sa_ablkcipher_encrypt;
+			cra->cra_ablkcipher.decrypt = sa_ablkcipher_decrypt;
+			cra->cra_init = sa_cra_init_ablkcipher;
+		} else if (type == CRYPTO_ALG_TYPE_AHASH) {
+			hash = &sa_algs[i].alg.hash;
+			alg_name = hash->halg.base.cra_name;
+			if (snprintf(hash->halg.base.cra_driver_name,
+				CRYPTO_MAX_ALG_NAME, "%s-keystone-sa",
+				alg_name) >= CRYPTO_MAX_ALG_NAME) {
+				continue;
+			}
+			hash->init = sa_ahash_init;
+			hash->update = sa_ahash_update;
+			hash->final = sa_ahash_final;
+			hash->finup = sa_ahash_finup;
+			hash->digest = sa_ahash_digest;
+			hash->setkey = sa_ahash_setkey;
+			cra = &hash->halg.base;
+			cra->cra_flags = CRYPTO_ALG_TYPE_AHASH |
+						CRYPTO_ALG_KERN_DRIVER_ONLY |
+						CRYPTO_ALG_ASYNC;
+			cra->cra_type = &crypto_ahash_type;
+			cra->cra_init = sa_cra_init_ahash;
+		} else {
+			dev_err(dev,
+				"un-supported crypto algorithm (%d)", type);
+			continue;
+		}
+
+		cra->cra_ctxsize = sizeof(struct sa_tfm_ctx);
+		cra->cra_module = THIS_MODULE;
+		cra->cra_alignmask = 0;
+		cra->cra_priority = 3000;
+		cra->cra_exit = sa_exit_tfm;
+
+		if (type == CRYPTO_ALG_TYPE_AHASH)
+			err = crypto_register_ahash(hash);
+		else
+			err = crypto_register_alg(cra);
+
+		if (err)
+			dev_err(dev, "Failed to register '%s'\n", alg_name);
+		else
+			sa_algs[i].registered = 1;
+	}
+}
+
+/* un-register the algorithms from crypto framework */
+void sa_unregister_algos(const struct device *dev)
+{
+	u32 type;
+	char *alg_name;
+	int err = 0, i, num_algs = ARRAY_SIZE(sa_algs);
+
+	for (i = 0; i < num_algs; i++) {
+		type = sa_algs[i].type;
+		if ((type == CRYPTO_ALG_TYPE_AHASH) &&
+				(sa_algs[i].registered)) {
+			alg_name = sa_algs[i].alg.hash.halg.base.cra_name;
+			err = crypto_unregister_ahash(&sa_algs[i].alg.hash);
+		} else if (sa_algs[i].registered) {
+			alg_name = sa_algs[i].alg.crypto.cra_name;
+			err = crypto_unregister_alg(&sa_algs[i].alg.crypto);
+		}
+
+		if (err)
+			dev_err(dev, "Failed to unregister '%s'", alg_name);
+	}
+}
+/*
+ * Keystone crypto accelerator driver
+ *
+ * Copyright (C) 2015 Texas Instruments Incorporated - http://www.ti.com
+ *
+ * Authors:	Sandeep Nair
+ *		Vitaly Andrianov
+ *
+ * Contributors:Tinku Mannan
+ *		Hao Zhang
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ */
+
+#include <linux/clk.h>
+#include <linux/err.h>
+#include <linux/init.h>
+#include <linux/slab.h>
+#include <linux/module.h>
+#include <linux/interrupt.h>
+#include <linux/dmapool.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
+#include <linux/rtnetlink.h>
+#include <linux/dma-mapping.h>
+#include <linux/platform_device.h>
+#include <linux/soc/ti/knav_dma.h>
+#include <linux/soc/ti/knav_qmss.h>
+
+#include <linux/crypto.h>
+#include <linux/hw_random.h>
+#include <linux/cryptohash.h>
+#include <crypto/algapi.h>
+#include <crypto/aead.h>
+#include <crypto/authenc.h>
+#include <crypto/hash.h>
+#include <crypto/internal/hash.h>
+#include <crypto/aes.h>
+#include <crypto/des.h>
+#include <crypto/sha.h>
+#include <crypto/md5.h>
+#include <crypto/scatterwalk.h>
+
+#include "keystone-sa.h"
+#include "keystone-sa-hlp.h"
+
+#define knav_queue_get_id(q)	knav_queue_device_control(q, \
+				KNAV_QUEUE_GET_ID, (unsigned long)NULL)
+
+#define knav_queue_enable_notify(q) knav_queue_device_control(q,	\
+					KNAV_QUEUE_ENABLE_NOTIFY,	\
+					(unsigned long)NULL)
+
+#define knav_queue_disable_notify(q) knav_queue_device_control(q,	\
+					KNAV_QUEUE_DISABLE_NOTIFY,	\
+					(unsigned long)NULL)
+
+#define knav_queue_get_count(q)	knav_queue_device_control(q, \
+				KNAV_QUEUE_GET_COUNT, (unsigned long)NULL)
+
+
+/**********************************************************************/
+/* Allocate ONE receive buffer for Rx descriptors */
+static void sa_allocate_rx_buf(struct keystone_crypto_data *dev_data,
+			       int fdq)
+{
+	struct device *dev = &dev_data->pdev->dev;
+	struct knav_dma_desc *hwdesc;
+	unsigned int buf_len, dma_sz;
+	u32 desc_info, pkt_info;
+	void *bufptr;
+	struct page *page;
+	dma_addr_t dma;
+	u32 pad[2];
+
+	/* Allocate descriptor */
+	hwdesc = knav_pool_desc_get(dev_data->rx_pool);
+	if (IS_ERR_OR_NULL(hwdesc)) {
+		dev_dbg(dev, "out of rx pool desc\n");
+		return;
+	}
+
+	if (fdq == 0) {
+		buf_len = dev_data->rx_buffer_sizes[0]; /* TODO is that size
+							   enough */
+		bufptr = kmalloc(buf_len, GFP_ATOMIC | GFP_DMA | __GFP_COLD);
+		if (unlikely(!bufptr)) {
+			dev_warn_ratelimited(dev, "Primary RX buffer alloc failed\n");
+			goto fail;
+		}
+		dma = dma_map_single(dev, bufptr, buf_len, DMA_TO_DEVICE);
+		pad[0] = (u32)bufptr;
+		pad[1] = 0;
+	} else {
+		/* Allocate a secondary receive queue entry */
+		page = alloc_page(GFP_ATOMIC | GFP_DMA | __GFP_COLD);
+		if (unlikely(!page)) {
+			dev_warn_ratelimited(dev, "Secondary page alloc failed\n");
+			goto fail;
+		}
+		buf_len = PAGE_SIZE;
+		dma = dma_map_page(dev, page, 0, buf_len, DMA_TO_DEVICE);
+		pad[0] = (u32)page_address(page);
+		pad[1] = (u32)page;
+
+		atomic_inc(&dev_data->rx_dma_page_cnt);
+	}
+
+	desc_info =  KNAV_DMA_DESC_PS_INFO_IN_DESC;
+	desc_info |= buf_len & KNAV_DMA_DESC_PKT_LEN_MASK;
+	pkt_info =  KNAV_DMA_DESC_HAS_EPIB;
+	pkt_info |= KNAV_DMA_NUM_PS_WORDS << KNAV_DMA_DESC_PSLEN_SHIFT;
+	pkt_info |= (dev_data->rx_compl_qid & KNAV_DMA_DESC_RETQ_MASK) <<
+		    KNAV_DMA_DESC_RETQ_SHIFT;
+	hwdesc->orig_buff = dma;
+	hwdesc->orig_len = buf_len;
+	hwdesc->pad[0] = pad[0];
+	hwdesc->pad[1] = pad[1];
+	hwdesc->desc_info = desc_info;
+	hwdesc->packet_info = pkt_info;
+
+	/* Push to FDQs */
+	knav_pool_desc_map(dev_data->rx_pool, hwdesc, sizeof(*hwdesc), &dma,
+			   &dma_sz);
+	knav_queue_push(dev_data->rx_fdq[fdq], dma, sizeof(*hwdesc), 0);
+
+	return;
+fail:
+	knav_pool_desc_put(dev_data->rx_pool, hwdesc);
+}
+
+/* Refill Rx FDQ with descriptors & attached buffers */
+static void sa_rxpool_refill(struct keystone_crypto_data *dev_data)
+{
+	u32 fdq_deficit;
+	int i;
+
+	/* Calculate the FDQ deficit and refill */
+	for (i = 0; i < KNAV_DMA_FDQ_PER_CHAN && dev_data->rx_fdq[i]; i++) {
+		fdq_deficit = dev_data->rx_queue_depths[i] -
+				 knav_queue_get_count(dev_data->rx_fdq[i]);
+		while (fdq_deficit--)
+			sa_allocate_rx_buf(dev_data, i);
+	} /* end for fdqs */
+}
+
+/* Release ALL descriptors and attached buffers from Rx FDQ */
+static void sa_free_rx_buf(struct keystone_crypto_data *dev_data,
+			       int fdq)
+{
+	struct device *dev = &dev_data->pdev->dev;
+
+	struct knav_dma_desc *desc;
+	unsigned int buf_len, dma_sz;
+	dma_addr_t dma;
+	void *buf_ptr;
+
+	/* Allocate descriptor */
+	while ((dma = knav_queue_pop(dev_data->rx_fdq[fdq], &dma_sz))) {
+		desc = knav_pool_desc_unmap(dev_data->rx_pool, dma, dma_sz);
+		if (unlikely(!desc)) {
+			dev_err(dev, "failed to unmap Rx desc\n");
+			continue;
+		}
+		dma = desc->orig_buff;
+		buf_len = desc->orig_len;
+		buf_ptr = (void *)desc->pad[0];
+
+		if (unlikely(!dma)) {
+			dev_err(dev, "NULL orig_buff in desc\n");
+			knav_pool_desc_put(dev_data->rx_pool, desc);
+			continue;
+		}
+
+		if (unlikely(!buf_ptr)) {
+			dev_err(dev, "NULL bufptr in desc\n");
+			knav_pool_desc_put(dev_data->rx_pool, desc);
+			continue;
+		}
+
+		if (fdq == 0) {
+			dma_unmap_single(dev, dma, buf_len, DMA_FROM_DEVICE);
+			kfree(buf_ptr);
+		} else {
+			dma_unmap_page(dev, dma, buf_len, DMA_FROM_DEVICE);
+			__free_page(buf_ptr);
+		}
+
+		knav_pool_desc_put(dev_data->rx_pool, desc);
+	}
+}
+
+static void sa_rxpool_free(struct keystone_crypto_data *dev_data)
+{
+	struct device *dev = &dev_data->pdev->dev;
+	int i;
+
+	for (i = 0; i < KNAV_DMA_FDQ_PER_CHAN &&
+	     !IS_ERR_OR_NULL(dev_data->rx_fdq[i]); i++)
+		sa_free_rx_buf(dev_data, i);
+
+	if (knav_pool_count(dev_data->rx_pool) != dev_data->rx_pool_size)
+		dev_err(dev, "Lost Rx (%d) descriptors\n",
+			dev_data->rx_pool_size -
+			knav_pool_count(dev_data->rx_pool));
+
+	knav_pool_destroy(dev_data->rx_pool);
+	dev_data->rx_pool = NULL;
+}
+
+/* DMA channel rx notify callback */
+static void sa_dma_notify_rx_compl(void *arg)
+{
+	struct keystone_crypto_data *dev_data = arg;
+
+	knav_queue_disable_notify(dev_data->rx_compl_q);
+	tasklet_schedule(&dev_data->rx_task);
+}
+
+/* Rx tast tasklet code */
+static void sa_rx_task(unsigned long data)
+{
+	struct keystone_crypto_data *dev_data =
+		(struct keystone_crypto_data *)data;
+
+	sa_rx_completion_process(dev_data);
+
+	sa_rxpool_refill(dev_data);
+	knav_queue_enable_notify(dev_data->rx_compl_q);
+}
+
+/* DMA channel tx notify callback */
+static void sa_dma_notify_tx_compl(void *arg)
+{
+	struct keystone_crypto_data *dev_data = arg;
+
+	knav_queue_disable_notify(dev_data->tx_compl_q);
+	tasklet_schedule(&dev_data->tx_task);
+}
+
+/* Tx task tasklet code */
+static void sa_tx_task(unsigned long data)
+{
+	struct keystone_crypto_data *dev_data =
+		(struct keystone_crypto_data *)data;
+
+	sa_tx_completion_process(dev_data);
+	knav_queue_enable_notify(dev_data->tx_compl_q);
+}
+
+static void sa_free_resources(struct keystone_crypto_data *dev_data)
+{
+	int	i;
+
+	if (!IS_ERR_OR_NULL(dev_data->rx_pool))
+		sa_rxpool_free(dev_data);
+
+	if (!IS_ERR_OR_NULL(dev_data->tx_pool)) {
+		knav_pool_destroy(dev_data->tx_pool);
+		dev_data->tx_pool = NULL;
+	}
+
+	if (!IS_ERR_OR_NULL(dev_data->tx_chan)) {
+		knav_dma_close_channel(dev_data->tx_chan);
+		dev_data->tx_chan = NULL;
+	}
+
+	if (!IS_ERR_OR_NULL(dev_data->rx_chan)) {
+		knav_dma_close_channel(dev_data->rx_chan);
+		dev_data->rx_chan = NULL;
+	}
+
+	if (!IS_ERR_OR_NULL(dev_data->tx_submit_q)) {
+		knav_queue_close(dev_data->tx_submit_q);
+		dev_data->tx_submit_q = NULL;
+	}
+
+	if (!IS_ERR_OR_NULL(dev_data->tx_compl_q)) {
+		knav_queue_close(dev_data->tx_compl_q);
+		dev_data->tx_compl_q = NULL;
+	}
+
+	if (!IS_ERR_OR_NULL(dev_data->rx_compl_q)) {
+		knav_queue_close(dev_data->rx_compl_q);
+		dev_data->rx_compl_q = NULL;
+	}
+
+	for (i = 0; i < KNAV_DMA_FDQ_PER_CHAN &&
+	     !IS_ERR_OR_NULL(dev_data->rx_fdq[i]) ; ++i) {
+		knav_queue_close(dev_data->rx_fdq[i]);
+		dev_data->rx_fdq[i] = NULL;
+	}
+}
+
+static int sa_setup_resources(struct keystone_crypto_data *dev_data)
+{
+	struct device *dev = &dev_data->pdev->dev;
+	u8	name[20];
+	int	ret = 0;
+	int	i;
+
+	snprintf(name, sizeof(name), "rx-pool-%s", dev_name(dev));
+	dev_data->rx_pool = knav_pool_create(name, dev_data->rx_pool_size,
+					     dev_data->rx_pool_region_id);
+	if (IS_ERR_OR_NULL(dev_data->rx_pool)) {
+		dev_err(dev, "Couldn't create rx pool\n");
+		ret = PTR_ERR(dev_data->rx_pool);
+		goto fail;
+	}
+
+	snprintf(name, sizeof(name), "tx-pool-%s", dev_name(dev));
+	dev_data->tx_pool = knav_pool_create(name, dev_data->tx_pool_size,
+					     dev_data->tx_pool_region_id);
+	if (IS_ERR_OR_NULL(dev_data->tx_pool)) {
+		dev_err(dev, "Couldn't create tx pool\n");
+		ret = PTR_ERR(dev_data->tx_pool);
+		goto fail;
+	}
+
+	snprintf(name, sizeof(name), "tx-subm_q-%s", dev_name(dev));
+	dev_data->tx_submit_q = knav_queue_open(name,
+						dev_data->tx_submit_qid, 0);
+	if (IS_ERR(dev_data->tx_submit_q)) {
+		ret = PTR_ERR(dev_data->tx_submit_q);
+		dev_err(dev, "Could not open \"%s\": %d\n", name, ret);
+		goto fail;
+	}
+
+	snprintf(name, sizeof(name), "tx-compl-q-%s", dev_name(dev));
+	dev_data->tx_compl_q = knav_queue_open(name, dev_data->tx_compl_qid, 0);
+	if (IS_ERR(dev_data->tx_compl_q)) {
+		ret = PTR_ERR(dev_data->tx_compl_q);
+		dev_err(dev, "Could not open \"%s\": %d\n", name, ret);
+		goto fail;
+	}
+
+	snprintf(name, sizeof(name), "rx-compl-q-%s", dev_name(dev));
+	dev_data->rx_compl_q = knav_queue_open(name, dev_data->rx_compl_qid, 0);
+	if (IS_ERR(dev_data->rx_compl_q)) {
+		ret = PTR_ERR(dev_data->rx_compl_q);
+		dev_err(dev, "Could not open \"%s\": %d\n", name, ret);
+		goto fail;
+	}
+
+	for (i = 0; i < KNAV_DMA_FDQ_PER_CHAN &&
+	     dev_data->rx_queue_depths[i] && dev_data->rx_buffer_sizes[i];
+	     i++) {
+		snprintf(name, sizeof(name), "rx-fdq%d-%s", i, dev_name(dev));
+		dev_data->rx_fdq[i] = knav_queue_open(name, KNAV_QUEUE_GP, 0);
+		if (IS_ERR_OR_NULL(dev_data->rx_fdq[i])) {
+			ret = PTR_ERR(dev_data->rx_fdq[i]);
+			goto fail;
+		}
+	}
+	sa_rxpool_refill(dev_data);
+
+	return 0;
+
+fail:
+	sa_free_resources(dev_data);
+	return ret;
+}
+
+static int sa_setup_dma(struct keystone_crypto_data *dev_data)
+{
+	struct device *dev = &dev_data->pdev->dev;
+	struct knav_queue_notify_config notify_cfg;
+	struct knav_dma_cfg config;
+	int error = 0;
+	int i;
+	u32 last_fdq = 0;
+	u8 name[16];
+
+	error = sa_setup_resources(dev_data);
+	if (error)
+		goto fail;
+
+	/* Setup Tx DMA channel */
+	memset(&config, 0, sizeof(config));
+	config.direction = DMA_MEM_TO_DEV;
+	config.u.tx.filt_einfo = false;
+	config.u.tx.filt_pswords = false;
+	config.u.tx.priority = DMA_PRIO_MED_L;
+
+	dev_data->tx_chan = knav_dma_open_channel(dev, dev_data->tx_chan_name,
+						  &config);
+	if (IS_ERR_OR_NULL(dev_data->tx_chan)) {
+		dev_err(dev, "(%s) failed to open dmachan\n",
+			dev_data->tx_chan_name);
+		error = -ENODEV;
+		goto fail;
+	}
+
+	notify_cfg.fn = sa_dma_notify_tx_compl;
+	notify_cfg.fn_arg = dev_data;
+	error = knav_queue_device_control(dev_data->tx_compl_q,
+					  KNAV_QUEUE_SET_NOTIFIER,
+					  (unsigned long)&notify_cfg);
+	if (error)
+		goto fail;
+
+	knav_queue_enable_notify(dev_data->tx_compl_q);
+
+	dev_dbg(dev, "opened tx channel %s\n", name);
+
+	/* Set notification for Rx completion */
+	notify_cfg.fn = sa_dma_notify_rx_compl;
+	notify_cfg.fn_arg = dev_data;
+	error = knav_queue_device_control(dev_data->rx_compl_q,
+					KNAV_QUEUE_SET_NOTIFIER,
+					(unsigned long)&notify_cfg);
+	if (error)
+		goto fail;
+
+	knav_queue_disable_notify(dev_data->rx_compl_q);
+
+	/* Setup Rx DMA channel */
+	memset(&config, 0, sizeof(config));
+	config.direction		= DMA_DEV_TO_MEM;
+	config.u.rx.einfo_present	= true;
+	config.u.rx.psinfo_present	= true;
+	config.u.rx.err_mode		= DMA_DROP;
+	config.u.rx.desc_type		= DMA_DESC_HOST;
+	config.u.rx.psinfo_at_sop	= false;
+	config.u.rx.sop_offset		= 0; /* NETCP_SOP_OFFSET */
+	config.u.rx.dst_q		= dev_data->rx_compl_qid;
+	config.u.rx.thresh		= DMA_THRESH_NONE;
+
+	for (i = 0; i < KNAV_DMA_FDQ_PER_CHAN; ++i) {
+		if (dev_data->rx_fdq[i])
+			last_fdq = knav_queue_get_id(dev_data->rx_fdq[i]);
+		config.u.rx.fdq[i] = last_fdq;
+	}
+
+	dev_data->rx_chan = knav_dma_open_channel(dev, dev_data->rx_chan_name,
+						  &config);
+	if (IS_ERR_OR_NULL(dev_data->rx_chan)) {
+		dev_err(dev, "(%s) failed to open dmachan\n",
+			dev_data->rx_chan_name);
+		error = -ENODEV;
+		goto fail;
+	}
+
+	knav_queue_enable_notify(dev_data->rx_compl_q);
+
+	return 0;
+
+fail:
+	sa_free_resources(dev_data);
+
+	return error;
+}
+
+/* Teardown DMA channels */
+static void sa_teardown_dma(struct keystone_crypto_data *dev_data)
+{
+	if (dev_data->tx_chan) {
+		knav_dma_close_channel(dev_data->tx_chan);
+		dev_data->tx_chan = NULL;
+	}
+
+	if (dev_data->rx_chan) {
+		knav_dma_close_channel(dev_data->rx_chan);
+		dev_data->rx_chan = NULL;
+	}
+}
+/******************************************************************************/
+/************************************************************/
+/*	SYSFS interface functions			    */
+/************************************************************/
+struct sa_kobj_attribute {
+	struct attribute attr;
+	ssize_t (*show)(struct keystone_crypto_data *crypto,
+		struct sa_kobj_attribute *attr, char *buf);
+	ssize_t	(*store)(struct keystone_crypto_data *crypto,
+		struct sa_kobj_attribute *attr, const char *, size_t);
+};
+
+#define SA_ATTR(_name, _mode, _show, _store) \
+	struct sa_kobj_attribute sa_attr_##_name = \
+__ATTR(_name, _mode, _show, _store)
+
+static ssize_t sa_stats_show_tx_pkts(struct keystone_crypto_data *crypto,
+		struct sa_kobj_attribute *attr, char *buf)
+{
+	return scnprintf(buf, PAGE_SIZE, "%d\n",
+			atomic_read(&crypto->stats.tx_pkts));
+}
+
+static ssize_t sa_stats_reset_tx_pkts(struct keystone_crypto_data *crypto,
+		struct sa_kobj_attribute *attr, const char *buf, size_t len)
+{
+	atomic_set(&crypto->stats.tx_pkts, 0);
+	return len;
+}
+
+static ssize_t sa_stats_show_rx_pkts(struct keystone_crypto_data *crypto,
+		struct sa_kobj_attribute *attr, char *buf)
+{
+	return scnprintf(buf, PAGE_SIZE, "%d\n",
+			atomic_read(&crypto->stats.rx_pkts));
+}
+
+static ssize_t sa_stats_reset_rx_pkts(struct keystone_crypto_data *crypto,
+		struct sa_kobj_attribute *attr, const char *buf, size_t len)
+{
+	atomic_set(&crypto->stats.rx_pkts, 0);
+	return len;
+}
+
+static ssize_t sa_stats_show_tx_drop_pkts(struct keystone_crypto_data *crypto,
+		struct sa_kobj_attribute *attr, char *buf)
+{
+	return scnprintf(buf, PAGE_SIZE, "%d\n",
+			atomic_read(&crypto->stats.tx_dropped));
+}
+
+static ssize_t sa_stats_reset_tx_drop_pkts(struct keystone_crypto_data *crypto,
+		struct sa_kobj_attribute *attr, const char *buf, size_t len)
+{
+	atomic_set(&crypto->stats.tx_dropped, 0);
+	return len;
+}
+
+static ssize_t
+sa_stats_show_sc_tear_drop_pkts(struct keystone_crypto_data *crypto,
+		struct sa_kobj_attribute *attr, char *buf)
+{
+	return scnprintf(buf, PAGE_SIZE, "%d\n",
+			atomic_read(&crypto->stats.sc_tear_dropped));
+}
+
+static SA_ATTR(tx_pkts, S_IRUGO | S_IWUSR,
+		sa_stats_show_tx_pkts, sa_stats_reset_tx_pkts);
+static SA_ATTR(rx_pkts, S_IRUGO | S_IWUSR,
+		sa_stats_show_rx_pkts, sa_stats_reset_rx_pkts);
+static SA_ATTR(tx_drop_pkts, S_IRUGO | S_IWUSR,
+		sa_stats_show_tx_drop_pkts, sa_stats_reset_tx_drop_pkts);
+static SA_ATTR(sc_tear_drop_pkts, S_IRUGO,
+		sa_stats_show_sc_tear_drop_pkts, NULL);
+
+static struct attribute *sa_stats_attrs[] = {
+	&sa_attr_tx_pkts.attr,
+	&sa_attr_rx_pkts.attr,
+	&sa_attr_tx_drop_pkts.attr,
+	&sa_attr_sc_tear_drop_pkts.attr,
+	NULL
+};
+
+#define to_sa_kobj_attr(_attr) \
+	container_of(_attr, struct sa_kobj_attribute, attr)
+#define to_crypto_data_from_stats_obj(obj) \
+	container_of(obj, struct keystone_crypto_data, stats_kobj)
+
+static ssize_t sa_kobj_attr_show(struct kobject *kobj, struct attribute *attr,
+			     char *buf)
+{
+	struct sa_kobj_attribute *sa_attr = to_sa_kobj_attr(attr);
+	struct keystone_crypto_data *crypto =
+		to_crypto_data_from_stats_obj(kobj);
+	ssize_t ret = -EIO;
+
+	if (sa_attr->show)
+		ret = sa_attr->show(crypto, sa_attr, buf);
+	return ret;
+}
+
+static ssize_t sa_kobj_attr_store(struct kobject *kobj, struct attribute *attr,
+			     const char *buf, size_t len)
+{
+	struct sa_kobj_attribute *sa_attr = to_sa_kobj_attr(attr);
+	struct keystone_crypto_data *crypto =
+		to_crypto_data_from_stats_obj(kobj);
+	ssize_t ret = -EIO;
+
+	if (sa_attr->store)
+		ret = sa_attr->store(crypto, sa_attr, buf, len);
+	return ret;
+}
+
+static const struct sysfs_ops sa_stats_sysfs_ops = {
+	.show = sa_kobj_attr_show,
+	.store = sa_kobj_attr_store,
+};
+
+static struct kobj_type sa_stats_ktype = {
+	.sysfs_ops = &sa_stats_sysfs_ops,
+	.default_attrs = sa_stats_attrs,
+};
+
+static int sa_create_sysfs_entries(struct keystone_crypto_data *crypto)
+{
+	struct device *dev = &crypto->pdev->dev;
+	int ret;
+
+	ret = kobject_init_and_add(&crypto->stats_kobj, &sa_stats_ktype,
+		kobject_get(&dev->kobj), "stats");
+
+	if (ret) {
+		dev_err(dev, "failed to create sysfs entry\n");
+		kobject_put(&crypto->stats_kobj);
+		kobject_put(&dev->kobj);
+	}
+	return ret;
+}
+
+static void sa_delete_sysfs_entries(struct keystone_crypto_data *crypto)
+{
+	kobject_del(&crypto->stats_kobj);
+}
+
+/*
+ * HW RNG functions
+ */
+
+static int sa_rng_init(struct hwrng *rng)
+{
+	u32 value;
+	struct device *dev = (struct device *)rng->priv;
+	struct keystone_crypto_data *crypto = dev_get_drvdata(dev);
+	u32 startup_cycles, min_refill_cycles, max_refill_cycles, clk_div;
+
+	crypto->trng_regs = (struct sa_trng_regs *)((void *)crypto->regs +
+				SA_REG_MAP_TRNG_OFFSET);
+
+	startup_cycles = SA_TRNG_DEF_STARTUP_CYCLES;
+	min_refill_cycles = SA_TRNG_DEF_MIN_REFILL_CYCLES;
+	max_refill_cycles = SA_TRNG_DEF_MAX_REFILL_CYCLES;
+	clk_div = SA_TRNG_DEF_CLK_DIV_CYCLES;
+
+	/* Enable RNG module */
+	value = __raw_readl(&crypto->regs->mmr.CMD_STATUS);
+	value |= SA_CMD_STATUS_REG_TRNG_ENABLE;
+	__raw_writel(value, &crypto->regs->mmr.CMD_STATUS);
+
+	/* Configure RNG module */
+	__raw_writel(0, &crypto->trng_regs->TRNG_CONTROL); /* Disable RNG */
+	value = startup_cycles << SA_TRNG_CONTROL_REG_STARTUP_CYCLES_SHIFT;
+	__raw_writel(value, &crypto->trng_regs->TRNG_CONTROL);
+	value =
+	(min_refill_cycles << SA_TRNG_CONFIG_REG_MIN_REFILL_CYCLES_SHIFT) |
+	(max_refill_cycles << SA_TRNG_CONFIG_REG_MAX_REFILL_CYCLES_SHIFT) |
+	(clk_div << SA_TRNG_CONFIG_REG_SAMPLE_DIV_SHIFT);
+	__raw_writel(value, &crypto->trng_regs->TRNG_CONFIG);
+	/* Disable all interrupts from TRNG */
+	__raw_writel(0, &crypto->trng_regs->TRNG_INTMASK);
+	/* Enable RNG */
+	value = __raw_readl(&crypto->trng_regs->TRNG_CONTROL);
+	value |= SA_TRNG_CONTROL_REG_TRNG_ENABLE;
+	__raw_writel(value, &crypto->trng_regs->TRNG_CONTROL);
+
+	/* Initialize the TRNG access lock */
+	spin_lock_init(&crypto->trng_lock);
+
+	return 0;
+}
+
+void sa_rng_cleanup(struct hwrng *rng)
+{
+	u32 value;
+	struct device *dev = (struct device *)rng->priv;
+	struct keystone_crypto_data *crypto = dev_get_drvdata(dev);
+
+	/* Disable RNG */
+	__raw_writel(0, &crypto->trng_regs->TRNG_CONTROL);
+	value = __raw_readl(&crypto->regs->mmr.CMD_STATUS);
+	value &= ~SA_CMD_STATUS_REG_TRNG_ENABLE;
+	__raw_writel(value, &crypto->regs->mmr.CMD_STATUS);
+}
+
+/* Maximum size of RNG data available in one read */
+#define SA_MAX_RNG_DATA	8
+/* Maximum retries to get rng data */
+#define SA_MAX_RNG_DATA_RETRIES	5
+/* Delay between retries (in usecs) */
+#define SA_RNG_DATA_RETRY_DELAY	5
+
+static int sa_rng_read(struct hwrng *rng, void *data, size_t max, bool wait)
+{
+	u32 value;
+	u32 st_ready;
+	u32 rng_lo, rng_hi;
+	int retries = SA_MAX_RNG_DATA_RETRIES;
+	int data_sz = min_t(u32, max, SA_MAX_RNG_DATA);
+	struct device *dev = (struct device *)rng->priv;
+	struct keystone_crypto_data *crypto = dev_get_drvdata(dev);
+
+	do {
+		spin_lock(&crypto->trng_lock);
+		value = __raw_readl(&crypto->trng_regs->TRNG_STATUS);
+		st_ready = value & SA_TRNG_STATUS_REG_READY;
+		if (st_ready) {
+			/* Read random data */
+			rng_hi = __raw_readl(&crypto->trng_regs->TRNG_OUTPUT_H);
+			rng_lo = __raw_readl(&crypto->trng_regs->TRNG_OUTPUT_L);
+			/* Clear ready status */
+			__raw_writel(SA_TRNG_INTACK_REG_READY,
+					&crypto->trng_regs->TRNG_INTACK);
+		}
+		spin_unlock(&crypto->trng_lock);
+		udelay(SA_RNG_DATA_RETRY_DELAY);
+	} while (wait && !st_ready && retries--);
+
+	if (!st_ready)
+		return -EAGAIN;
+
+	if (likely(data_sz > sizeof(rng_lo))) {
+		memcpy(data, &rng_lo, sizeof(rng_lo));
+		memcpy((data + sizeof(rng_lo)), &rng_hi,
+				(data_sz - sizeof(rng_lo)));
+	} else {
+		memcpy(data, &rng_lo, data_sz);
+	}
+
+	return data_sz;
+}
+
+static int sa_register_rng(struct device *dev)
+{
+	struct keystone_crypto_data *crypto = dev_get_drvdata(dev);
+
+	crypto->rng.name = dev_driver_string(dev);
+	crypto->rng.init = sa_rng_init;
+	crypto->rng.cleanup = sa_rng_cleanup;
+	crypto->rng.read = sa_rng_read;
+	crypto->rng.priv = (unsigned long)dev;
+
+	return hwrng_register(&crypto->rng);
+}
+
+static void sa_unregister_rng(struct device *dev)
+{
+	struct keystone_crypto_data *crypto = dev_get_drvdata(dev);
+
+	hwrng_unregister(&crypto->rng);
+}
+
+/************************************************************/
+/*	Driver registration functions			*/
+/************************************************************/
+#define OF_PROP_READ(type, node, prop, var) \
+	do { \
+		ret = of_property_read_##type(node, prop, &var); \
+		if (ret < 0) { \
+			dev_err(dev, "missing \""prop"\" parameter\n"); \
+			return -1; \
+		} \
+	} while (0)
+
+#define OF_PROP_READ_U32_ARRAY(node, prop, array, size) \
+	do { \
+		ret = of_property_read_u32_array(node, prop, array, size); \
+		if (ret < 0) { \
+			dev_err(dev, "missing \""prop"\" parameter\n"); \
+			return -1; \
+		} \
+	} while (0)
+
+static int sa_read_dtb(struct device_node *node,
+			struct keystone_crypto_data *dev_data)
+{
+	int i, ret = 0;
+	struct device *dev = &dev_data->pdev->dev;
+	u32 temp[2];
+
+	OF_PROP_READ(string, node, "tx-channel", dev_data->tx_chan_name);
+	OF_PROP_READ(u32, node, "tx-queue-depth", dev_data->tx_queue_depth);
+	atomic_set(&dev_data->tx_dma_desc_cnt, dev_data->tx_queue_depth);
+	OF_PROP_READ(u32, node, "tx-submit-queue", dev_data->tx_submit_qid);
+	OF_PROP_READ(u32, node, "tx-compl-queue", dev_data->tx_compl_qid);
+	OF_PROP_READ(string, node, "rx-channel", dev_data->rx_chan_name);
+
+	OF_PROP_READ_U32_ARRAY(node, "rx-queue-depth",
+			       dev_data->rx_queue_depths,
+			       KNAV_DMA_FDQ_PER_CHAN);
+
+	for (i = 0; i < KNAV_DMA_FDQ_PER_CHAN; i++)
+		dev_dbg(dev, "rx-queue-depth[%d]= %u\n", i,
+				dev_data->rx_queue_depths[i]);
+
+	OF_PROP_READ_U32_ARRAY(node, "rx-buffer-size",
+			       dev_data->rx_buffer_sizes,
+			       KNAV_DMA_FDQ_PER_CHAN);
+
+	for (i = 0; i < KNAV_DMA_FDQ_PER_CHAN; i++)
+		dev_dbg(dev, "rx-buffer-size[%d]= %u\n", i,
+				dev_data->rx_buffer_sizes[i]);
+
+	atomic_set(&dev_data->rx_dma_page_cnt, 0);
+
+	OF_PROP_READ(u32, node, "rx-compl-queue", dev_data->rx_compl_qid);
+
+	OF_PROP_READ_U32_ARRAY(node, "tx-pool", temp, 2);
+	dev_data->tx_pool_size = temp[0];
+	dev_data->tx_pool_region_id = temp[1];
+
+	OF_PROP_READ_U32_ARRAY(node, "rx-pool", temp, 2);
+	dev_data->rx_pool_size = temp[0];
+	dev_data->rx_pool_region_id = temp[1];
+
+	OF_PROP_READ_U32_ARRAY(node, "sc-id", temp, 2);
+	dev_data->sc_id_start = temp[0];
+	dev_data->sc_id_end = temp[1];
+	dev_data->sc_id = dev_data->sc_id_start;
+
+	dev_data->regs = of_iomap(node, 0);
+	if (!dev_data->regs) {
+		dev_err(dev, "failed to of_iomap\n");
+		return -ENOMEM;
+	}
+
+	return 0;
+}
+
+static int sa_init_mem(struct keystone_crypto_data *dev_data)
+{
+	struct device *dev = &dev_data->pdev->dev;
+	/* Setup dma pool for security context buffers */
+	dev_data->sc_pool = dma_pool_create("keystone-sc", dev,
+				SA_CTX_MAX_SZ, 64, 0);
+	if (!dev_data->sc_pool) {
+		dev_err(dev, "Failed to create dma pool");
+		return -1;
+	}
+
+	/* Create a cache for Tx DMA request context */
+	dev_data->dma_req_ctx_cache = KMEM_CACHE(sa_dma_req_ctx, 0);
+	if (!dev_data->dma_req_ctx_cache) {
+		dev_err(dev, "Failed to create dma req cache");
+		return -1;
+	}
+	return 0;
+}
+
+static void sa_free_mem(struct keystone_crypto_data *dev_data)
+{
+	if (dev_data->sc_pool)
+		dma_pool_destroy(dev_data->sc_pool);
+	if (dev_data->dma_req_ctx_cache)
+		kmem_cache_destroy(dev_data->dma_req_ctx_cache);
+}
+static int keystone_crypto_remove(struct platform_device *pdev)
+{
+	struct keystone_crypto_data *dev_data = platform_get_drvdata(pdev);
+
+	/* un-register crypto algorithms */
+	sa_unregister_algos(&pdev->dev);
+	/* un-register HW RNG */
+	sa_unregister_rng(&pdev->dev);
+	/* Delete SYSFS entries */
+	sa_delete_sysfs_entries(dev_data);
+	/* Release DMA channels */
+	sa_teardown_dma(dev_data);
+	/* Kill tasklets */
+	tasklet_kill(&dev_data->rx_task);
+	/* Free memory pools used by the driver */
+	sa_free_mem(dev_data);
+
+	clk_disable_unprepare(dev_data->clk);
+	clk_put(dev_data->clk);
+	kfree(dev_data);
+	platform_set_drvdata(pdev, NULL);
+	return 0;
+}
+
+static int keystone_crypto_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct device_node *node = pdev->dev.of_node;
+	struct keystone_crypto_data *dev_data;
+	u32 value;
+	int ret;
+
+	sa_ks2_dev = dev;
+	dev_data = devm_kzalloc(dev, sizeof(*dev_data), GFP_KERNEL);
+	if (!dev_data)
+		return -ENOMEM;
+
+	dev_data->clk = clk_get(dev, NULL);
+	if (IS_ERR_OR_NULL(dev_data->clk)) {
+		dev_err(dev, "Couldn't get clock\n");
+		ret = -ENODEV;
+		goto err;
+	}
+
+	ret = clk_prepare_enable(dev_data->clk);
+	if (ret < 0) {
+		dev_err(dev, "Couldn't enable clock\n");
+		clk_put(dev_data->clk);
+		ret = -ENODEV;
+		goto err;
+	}
+
+	dev_data->pdev = pdev;
+	platform_set_drvdata(pdev, dev_data);
+
+	/* Read configuration from device tree */
+	ret = sa_read_dtb(node, dev_data);
+	if (ret) {
+		dev_err(dev, "Failed to get all relevant configurations from DTB...\n");
+		goto err;
+	}
+
+	/* Enable the required sub-modules in SA */
+	value = __raw_readl(&dev_data->regs->mmr.CMD_STATUS);
+
+	value |= (0x00000001u  /* Enc SS */
+		| 0x00000002u /* Auth SS */
+		| 0x00000080u /* Context Cache */
+		| 0x00000100u /* PA in port */
+		| 0x00000200u /* CDMA in port */
+		| 0x00000400u /* PA out port */
+		| 0x00000800u /* CDMA out port */
+		| 0x00001000u /* Enc SS1 */
+		| 0x00002000u); /* Auth SS1 */
+
+	__raw_writel(value, &dev_data->regs->mmr.CMD_STATUS);
+
+	tasklet_init(&dev_data->rx_task, sa_rx_task,
+		     (unsigned long) dev_data);
+
+	tasklet_init(&dev_data->tx_task, sa_tx_task, (unsigned long) dev_data);
+
+	/* Initialize statistic counters */
+	atomic_set(&dev_data->stats.tx_dropped, 0);
+	atomic_set(&dev_data->stats.sc_tear_dropped, 0);
+	atomic_set(&dev_data->stats.tx_pkts, 0);
+	atomic_set(&dev_data->stats.rx_pkts, 0);
+
+	/* Initialize memory pools used by the driver */
+	if (sa_init_mem(dev_data)) {
+		dev_err(dev, "Failed to create dma pool");
+		ret = -ENOMEM;
+		goto err;
+	}
+
+	/* Setup DMA channels */
+	if (sa_setup_dma(dev_data)) {
+		dev_err(dev, "Failed to set DMA channels");
+		ret = -ENODEV;
+		goto err;
+	}
+
+	/* Initialize the SC-ID allocation lock */
+	spin_lock_init(&dev_data->scid_lock);
+
+	/* Create sysfs entries */
+	ret = sa_create_sysfs_entries(dev_data);
+	if (ret)
+		goto err;
+
+	/* Register HW RNG support */
+	ret = sa_register_rng(dev);
+	if (ret) {
+		dev_err(dev, "Failed to register HW RNG");
+		goto err;
+	}
+
+	/* Register crypto algorithms */
+	sa_register_algos(dev);
+	dev_info(dev, "crypto accelerator enabled\n");
+	return 0;
+
+err:
+	keystone_crypto_remove(pdev);
+	return ret;
+}
+
+static const struct of_device_id of_match[] = {
+	{ .compatible = "ti,keystone-crypto", },
+	{},
+};
+MODULE_DEVICE_TABLE(of, of_match);
+
+static struct platform_driver keystone_crypto_driver = {
+	.probe	= keystone_crypto_probe,
+	.remove	= keystone_crypto_remove,
+	.driver	= {
+		.name		= "keystone-crypto",
+		.owner		= THIS_MODULE,
+		.of_match_table	= of_match,
+	},
+};
+
+static int __init keystone_crypto_mod_init(void)
+{
+	return  platform_driver_register(&keystone_crypto_driver);
+}
+
+static void __exit keystone_crypto_mod_exit(void)
+{
+	platform_driver_unregister(&keystone_crypto_driver);
+}
+
+module_init(keystone_crypto_mod_init);
+module_exit(keystone_crypto_mod_exit);
+
+MODULE_DESCRIPTION("Keystone crypto acceleration support.");
+MODULE_LICENSE("GPL v2");
+MODULE_AUTHOR("Sandeep Nair");
+MODULE_AUTHOR("Vitaly Andrianov");
+/*
+ * Keystone crypto accelerator driver
+ *
+ * Copyright (C) 2013 Texas Instruments Incorporated - http://www.ti.com
+ * Contact: Sandeep Nair <sandeep_n@ti.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ */
+
+/*
+ *
+ *     0                   1                   2                   3
+ *     0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+ *   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -----
+ *   |                                                               |
+ *   |   Software only section (not fetched by CP_ACE)               |
+ *   |               (optional)                                      |
+ *   |                                                               |
+ *   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+<-Must be
+ *   |               SCCTL                                           |  64 byte
+ *   |               (8 bytes)                                       |  aligned
+ *   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ ------
+ *   |                                                               |
+ *   |   PHP module specific section (fetched by CP_ACE)             |
+ *   |               (56 bytes)                                      |
+ *   |                                                               |
+ *   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -----
+ *   |                                                               |
+ *   |   Encryption module specific section (fetched by CP_ACE)      |
+ *   |               (variable size)                                 |
+ *   |                                                               |
+ *   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+<-64 byte
+ *   |                                                               |  aligned
+ *   |   Authentication module specific section (fetched by CP_ACE)  |
+ *   |               (variable size)                                 |
+ *   |                                                               |
+ *   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -----
+ *
+ *              Figure: Security Context memory layout
+ *
+ *
+ *     0                   1                   2                   3
+ *     0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+ *   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -----
+ *   |O|Evict done   |  F/E control  |           SCID                |
+ *   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ *   |                 SCPTR (Security Context Pointer)              |
+ *   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -----
+ *              O : Owner
+ *              SCID & SCPTR are filled by hardware
+ *              Figure: Security Context control word (SCCTL)
+ *
+ *
+ *     0                   1                   2                   3
+ *     0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+ *   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ ------
+ *   |D|  Pkt Type   |  Flow Index   |   Dest Queue ID               |
+ *   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ ------
+ *   |               SWINFO-0 (4 bytes)                              |
+ *   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ ------
+ *   |               SWINFO-1 (4 bytes)                              |
+ *   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ ------
+ *   | PktID (16 bits)               |                               |
+ *   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ ------
+ *   |                                                               |
+ *   |                Protocol Specific Parameters                   |
+ *   |                (Variable Size up to 116 bytes                 |
+ *   ...                                                           ...
+ *   |                                                               |
+ *   |                                                               |
+ *   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ ------
+ *              D : Direction
+ *              Figure: PHP engine Security Context Format
+ *
+ *
+ *     0                   1                   2                   3
+ *     0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+ *   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ ------
+ *   |M| R |nEngineID|                                               |
+ *   +-+-+-+-+-+-+-+-+                                               +
+ *   |                  encryption mode ctrl word                    |
+ *   ...                                                           ...
+ *   |                       (27 bytes)                              |
+ *   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ ------
+ *   |          Reserved (4 bytes) (must initialize to 0)            |
+ *   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ ------
+ *   |          Encryption Key value (32 bytes)                      |
+ *   ...                                                           ...
+ *   |                                                               |
+ *   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ ------
+ *   |          Encryption Aux-1 (32 bytes) (optional)               |
+ *   ...                                                           ...
+ *   |                                                               |
+ *   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ ------
+ *   |          Encryption Aux-2 (16 bytes) (optional)               |
+ *   ...                                                           ...
+ *   |                                                               |
+ *   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ ------
+ *   |          Encryption Aux-3 (16 bytes) (optional)               |
+ *   ...                                                           ...
+ *   |                                                               |
+ *   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ ------
+ *   |          Encryption Aux-4 (16 bytes)                          |
+ *   ...                                                           ...
+ *   |                                                               |
+ *   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ ------
+ *   |          Pre-crypto data store (15 bytes)                     |
+ *   |                                                               |
+ *   |                                               +-+-+-+-+-+-+-+-|
+ *   |                                               |               |
+ *   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ ------
+ *              M : Encryption Mode selector (0=crypto processing, 1=NULL)
+ *              R : Reserved
+ *              Figure: Encryption engine Security Context Format
+ *
+ *
+ *     0                   1                   2                   3
+ *     0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+ *   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ ------
+ *   |M| R |nEngineID| Auth SW Ctrl  |                               |
+ *   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+                               +
+ *   |                  Reserved (6 bytes)                           |
+ *   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ ------
+ *   |          Authentication length (8 bytes)                      |
+ *   |          ( 0 = let h/w calculate the length )                 |
+ *   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ ------
+ *   |          Reserved (12 bytes)                                  |
+ *   ...                                                           ...
+ *   |                                                               |
+ *   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ ------
+ *   |          HW crtl word (4 bytes) must be set to 0 by SW        |
+ *   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ ------
+ *   |          Authentication Key value (32 bytes)                  |
+ *   ...        Master Key or pre-computed inner digest for HMAC   ...
+ *   |                                                               |
+ *   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ ------
+ *   |          Authentication Aux-1 (32 bytes) (optional)           |
+ *   ...        Pre-computed outer opad for HMAC                   ...
+ *   |                                                               |
+ *   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ ------
+ *   |          Authentication Aux-2 (32 bytes) (optional)           |
+ *   ...                                                           ...
+ *   |                                                               |
+ *   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ ------
+ *   |          Pre-crypto data store (32 bytes)                     |
+ *   ...        ( HW access only)                                   ...
+ *   |                                                               |
+ *   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ ------
+ *   |          Pre-crypto data store (32 bytes)                     |
+ *   ...        ( HW access only)                                   ...
+ *   |                                                               |
+ *   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ ------
+ *              M : Authentication Mode selector (0=hash processing, 1=NULL)
+ *              R : Reserved
+ *              Figure: Authentication engine Security Context Format
+ *
+ */
+
+/******************************************************************************
+ * This type represents the various packet types to be processed
+ * by the PHP engine in SA.
+ * It is used to identify the corresponding PHP processing function.
+ ******************************************************************************/
+typedef u8 SA_CTX_PE_PKT_TYPE_T;
+#define SA_CTX_PE_PKT_TYPE_3GPP_AIR    0    /* 3GPP Air Cipher */
+#define SA_CTX_PE_PKT_TYPE_SRTP        1    /* SRTP */
+#define SA_CTX_PE_PKT_TYPE_IPSEC_AH    2    /* IPSec Authentication Header */
+#define SA_CTX_PE_PKT_TYPE_IPSEC_ESP   3    /* IPSec Encapsulating
+					       Security Payload */
+#define SA_CTX_PE_PKT_TYPE_NONE        4    /* Indicates that it is in
+					       data mode, It may not be
+					       used by PHP */
+
+
+#define SA_CTX_ENC_TYPE1_SZ	64	/* Encryption SC with Key only */
+#define SA_CTX_ENC_TYPE2_SZ	96	/* Encryption SC with Key and Aux1 */
+
+#define SA_CTX_AUTH_TYPE1_SZ	64	/* Auth SC with Key only */
+#define SA_CTX_AUTH_TYPE2_SZ	96	/* Auth SC with Key and Aux1 */
+
+#define SA_CTX_PHP_PE_CTX_SZ	64	/* Size of security ctx for
+					   PHP engine */
+
+#define SA_CTX_MAX_SZ (64 + SA_CTX_ENC_TYPE2_SZ + SA_CTX_AUTH_TYPE2_SZ)
+
+/*
+ * Encoding of F/E control in SCCTL
+ *  Bit 0-1: Fetch PHP Bytes
+ *  Bit 2-3: Fetch Encryption/Air Ciphering Bytes
+ *  Bit 4-5: Fetch Authentication Bytes or Encr pass 2
+ *  Bit 6-7: Evict PHP Bytes
+ *
+ *  where   00 = 0 bytes
+ *          01 = 64 bytes
+ *          10 = 96 bytes
+ *          11 = 128 bytes
+ */
+#define SA_CTX_DMA_SIZE_0       0
+#define SA_CTX_DMA_SIZE_64      1
+#define SA_CTX_DMA_SIZE_96      2
+#define SA_CTX_DMA_SIZE_128     3
+
+#define SA_CTX_SCCTL_MK_DMA_INFO(php_f, eng0_f, eng1_f, php_e) \
+	((php_f) | \
+	 ((eng0_f) << 2) | \
+	 ((eng1_f) << 4) | \
+	 ((php_e) << 6))
+
+/*
+ * Byte offset of the owner word in SCCTL
+ * in the security context
+ */
+#define SA_CTX_SCCTL_OWNER_OFFSET 0
+
+/*
+ * Assumption: CTX size is multiple of 32
+ */
+#define SA_CTX_SIZE_TO_DMA_SIZE(ctx_sz)	\
+	((ctx_sz) ? ((ctx_sz)/32 - 1) : 0)
+
+#define SA_CTX_ENC_KEY_OFFSET	32
+#define SA_CTX_ENC_AUX1_OFFSET	64
+#define SA_CTX_ENC_AUX2_OFFSET	96
+#define SA_CTX_ENC_AUX3_OFFSET	112
+#define SA_CTX_ENC_AUX4_OFFSET	128
+
+/* Next Engine Select code in CP_ACE */
+#define SA_ENG_ID_EM1	2	/*  Encryption/Decryption engine
+				    with AES/DES core */
+#define SA_ENG_ID_EM2	3	/*  Encryption/Decryption enginefor pass 2 */
+#define SA_ENG_ID_AM1	4	/*  Authentication engine with
+				    SHA1/MD5/SHA2 core */
+#define SA_ENG_ID_AM2	5	/*  Authentication engine for pass 2 */
+#define SA_ENG_ID_OUTPORT2 20	/*  Egress module 2  */
+#define SA_ENG_ID_NONE  0xff
+
+/*
+ * Command Label Definitions
+ */
+#define SA_CMDL_OFFSET_NESC           0      /* Next Engine Select Code */
+#define SA_CMDL_OFFSET_LABEL_LEN      1      /* Engine Command Label Length */
+#define SA_CMDL_OFFSET_DATA_LEN       2      /* 16-bit Length of Data to be
+						processed */
+#define SA_CMDL_OFFSET_DATA_OFFSET    4      /* Stat Data Offset */
+#define SA_CMDL_OFFSET_OPTION_CTRL1   5      /* Option Control Byte 1 */
+#define SA_CMDL_OFFSET_OPTION_CTRL2   6      /* Option Control Byte 2 */
+#define SA_CMDL_OFFSET_OPTION_CTRL3   7      /* Option Control Byte 3 */
+#define SA_CMDL_OFFSET_OPTION_BYTE    8
+
+#define SA_CMDL_HEADER_SIZE_BYTES          8
+
+#define SA_CMDL_OPTION_BYTES_MAX_SIZE     72
+#define SA_CMDL_MAX_SIZE_BYTES (SA_CMDL_HEADER_SIZE_BYTES + \
+				SA_CMDL_OPTION_BYTES_MAX_SIZE)
+
+/* SWINFO word-0 flags */
+#define SA_SW_INFO_FLAG_EVICT	0x0001
+#define SA_SW_INFO_FLAG_TEAR	0x0002
+#define SA_SW_INFO_FLAG_NOPD	0x0004
+
+/*
+ * TRNG module definitions
+ */
+
+/* Offset to TRNG module in CP_ACE memory map */
+#define SA_REG_MAP_TRNG_OFFSET	0x24000
+
+/* TRNG enable control in CP_ACE */
+#define SA_CMD_STATUS_REG_TRNG_ENABLE	BIT(3)
+
+/* TRNG start control in TRNG module */
+#define SA_TRNG_CONTROL_REG_TRNG_ENABLE	BIT(10)
+
+/* Data ready indicator in STATUS register */
+#define SA_TRNG_STATUS_REG_READY BIT(0)
+
+/* Data ready clear control in INTACK register */
+#define SA_TRNG_INTACK_REG_READY BIT(0)
+
+/* Number of samples taken to gather entropy during startup.
+ * If value is 0, the number of samples is 2^24 else
+ * equals value times 2^8.
+ */
+#define SA_TRNG_DEF_STARTUP_CYCLES	0
+#define SA_TRNG_CONTROL_REG_STARTUP_CYCLES_SHIFT 16
+
+/* Minimum number of samples taken to regenerate entropy
+ * If value is 0, the number of samples is 2^24 else
+ * equals value times 2^6.
+ */
+#define SA_TRNG_DEF_MIN_REFILL_CYCLES	1
+#define SA_TRNG_CONFIG_REG_MIN_REFILL_CYCLES_SHIFT 0
+
+/* Maximum number of samples taken to regenerate entropy
+ * If value is 0, the number of samples is 2^24 else
+ * equals value times 2^8.
+ */
+#define SA_TRNG_DEF_MAX_REFILL_CYCLES	0
+#define SA_TRNG_CONFIG_REG_MAX_REFILL_CYCLES_SHIFT 16
+
+/* Number of CLK input cycles between samples */
+#define SA_TRNG_DEF_CLK_DIV_CYCLES	0
+#define SA_TRNG_CONFIG_REG_SAMPLE_DIV_SHIFT 8
...	...	@@ -273,6 +273,22 @@
273	273	the ECB and CBC modes of operation supported by the driver. Also
274	274	accesses made on unaligned boundaries are also supported.
275	275
	276	+config CRYPTO_DEV_KEYSTONE
	277	+ tristate "Support for TI Keystone security accelerator"
	278	+ depends on ARCH_KEYSTONE
	279	+ select CRYPTO_AES
	280	+ select CRYPTO_AES_ARM
	281	+ select CRYPTO_SHA1
	282	+ select CRYPTO_MD5
	283	+ select CRYPTO_ALGAPI
	284	+ select CRYPTO_AUTHENC
	285	+ select HW_RANDOM
	286	+ default y if ARCH_KEYSTONE
	287	+ help
	288	+ Keystone devices include a security accelerator engine that may be
	289	+ used for crypto offload. Select this if you want to use hardware
	290	+ acceleration for cryptographic algorithms on these devices.
	291	+
276	292	config CRYPTO_DEV_PICOXCELL
277	293	tristate "Support for picoXcell IPSEC and Layer2 crypto engines"
278	294	depends on ARCH_PICOXCELL && HAVE_CLK
...	...	@@ -8,6 +8,9 @@
8	8	obj-$(CONFIG_CRYPTO_DEV_HIFN_795X) += hifn_795x.o
9	9	obj-$(CONFIG_CRYPTO_DEV_IMGTEC_HASH) += img-hash.o
10	10	obj-$(CONFIG_CRYPTO_DEV_IXP4XX) += ixp4xx_crypto.o
	11	+obj-$(CONFIG_CRYPTO_DEV_KEYSTONE) += keystone-sa-driver.o
	12	+keystone-sa-driver-objs := keystone-sa.o keystone-sa-utils.o \
	13	+ keystone-sa-lld.o keystone-sa-tbls.o
11	14	obj-$(CONFIG_CRYPTO_DEV_MV_CESA) += mv_cesa.o
12	15	obj-$(CONFIG_CRYPTO_DEV_MXS_DCP) += mxs-dcp.o
13	16	obj-$(CONFIG_CRYPTO_DEV_NIAGARA2) += n2_crypto.o
	1	+/*
	2	+ * Keystone crypto accelerator driver
	3	+ *
	4	+ * Copyright (C) 2015 Texas Instruments Incorporated - http://www.ti.com
	5	+ *
	6	+ * Authors: Sandeep Nair
	7	+ * Vitaly Andrianov
	8	+ *
	9	+ * This program is free software; you can redistribute it and/or
	10	+ * modify it under the terms of the GNU General Public License
	11	+ * version 2 as published by the Free Software Foundation.
	12	+ *
	13	+ * This program is distributed in the hope that it will be useful, but
	14	+ * WITHOUT ANY WARRANTY; without even the implied warranty of
	15	+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	16	+ * General Public License for more details.
	17	+ */
	18	+
	19	+#ifndef _KEYSTONE_SA_HLP_
	20	+#define _KEYSTONE_SA_HLP_
	21	+
	22	+#include <linux/soc/ti/knav_dma.h>
	23	+#include <linux/soc/ti/knav_qmss.h>
	24	+#include <linux/interrupt.h>
	25	+#include <linux/hw_random.h>
	26	+#include <linux/skbuff.h>
	27	+
	28	+/* Enable the below macro for testing with run-time
	29	+ * self tests in the cryptographic algorithm manager
	30	+ * framework */
	31	+/* #define TEST */
	32	+
	33	+/* For enabling debug prints */
	34	+/* #define DEBUG */
	35	+
	36	+/* Algorithm constants */
	37	+#define MD5_BLOCK_SIZE 64
	38	+#define AES_XCBC_DIGEST_SIZE 16
	39	+
	40	+/* Values for NULL algorithms */
	41	+#define NULL_KEY_SIZE 0
	42	+#define NULL_BLOCK_SIZE 1
	43	+#define NULL_DIGEST_SIZE 0
	44	+#define NULL_IV_SIZE 0
	45	+
	46	+/* Number of 32 bit words in EPIB */
	47	+#define SA_DMA_NUM_EPIB_WORDS 4
	48	+
	49	+/* Number of 32 bit words in PS data */
	50	+#define SA_DMA_NUM_PS_WORDS 16
	51	+
	52	+/* Number of meta data elements passed in descriptor to SA */
	53	+#define SA_NUM_DMA_META_ELEMS 2
	54	+
	55	+/* Maximum number of simultaeneous security contexts
	56	+ * supported by the driver */
	57	+#define SA_MAX_NUM_CTX 512
	58	+
	59	+/* Encoding used to identify the typo of crypto operation
	60	+ * performed on the packet when the packet is returned
	61	+ * by SA
	62	+ */
	63	+#define SA_REQ_SUBTYPE_ENC 0x0001
	64	+#define SA_REQ_SUBTYPE_DEC 0x0002
	65	+#define SA_REQ_SUBTYPE_SHIFT 16
	66	+#define SA_REQ_SUBTYPE_MASK 0xffff
	67	+
	68	+/* Maximum size of authentication tag
	69	+ * NOTE: update this macro as we start supporting
	70	+ * algorithms with bigger digest size
	71	+ */
	72	+#define SA_MAX_AUTH_TAG_SZ SHA1_DIGEST_SIZE
	73	+
	74	+/* Memory map of the SA register set */
	75	+struct sa_mmr_regs {
	76	+ u32 PID;
	77	+ u32 EFUSE_EN;
	78	+ u32 CMD_STATUS;
	79	+ u32 BLKMGR_PA_BLKS;
	80	+ u32 PA_FLOWID;
	81	+ u32 CDMA_FLOWID;
	82	+ u32 PA_ENG_ID;
	83	+ u32 CDMA_ENG_ID;
	84	+ u8 RSVD0[224];
	85	+ u32 CTXCACH_CTRL;
	86	+ u32 CTXCACH_SC_PTR;
	87	+ u32 CTXCACH_SC_ID;
	88	+ u32 CTXCACH_MISSCNT;
	89	+};
	90	+
	91	+/*
	92	+ * Register Overlay Structure for TRNG module
	93	+ */
	94	+struct sa_trng_regs {
	95	+ u32 TRNG_OUTPUT_L;
	96	+ u32 TRNG_OUTPUT_H;
	97	+ u32 TRNG_STATUS;
	98	+ u32 TRNG_INTMASK;
	99	+ u32 TRNG_INTACK;
	100	+ u32 TRNG_CONTROL;
	101	+ u32 TRNG_CONFIG;
	102	+ u32 TRNG_ALARMCNT;
	103	+ u32 TRNG_FROENABLE;
	104	+ u32 TRNG_FRODETUNE;
	105	+ u32 TRNG_ALARMMASK;
	106	+ u32 TRNG_ALARMSTOP;
	107	+ u32 TRNG_LFSR_L;
	108	+ u32 TRNG_LFSR_M;
	109	+ u32 TRNG_LFSR_H;
	110	+ u32 TRNG_COUNT;
	111	+ u32 TRNG_TEST;
	112	+};
	113	+
	114	+struct sa_regs {
	115	+ struct sa_mmr_regs mmr;
	116	+};
	117	+
	118	+/* Driver statistics */
	119	+struct sa_drv_stats {
	120	+ /* Number of data pkts dropped while submitting to CP_ACE */
	121	+ atomic_t tx_dropped;
	122	+ /* Number of tear-down pkts dropped while submitting to CP_ACE */
	123	+ atomic_t sc_tear_dropped;
	124	+ /* Number of crypto requests sent to CP_ACE */
	125	+ atomic_t tx_pkts;
	126	+ /* Number of crypto request completions received from CP_ACE */
	127	+ atomic_t rx_pkts;
	128	+};
	129	+
	130	+/* Crypto driver instance data */
	131	+struct keystone_crypto_data {
	132	+ struct platform_device *pdev;
	133	+ struct clk *clk;
	134	+ struct tasklet_struct rx_task;
	135	+ struct tasklet_struct tx_task;
	136	+ struct dma_pool *sc_pool;
	137	+ struct kmem_cache *dma_req_ctx_cache;
	138	+ struct sa_regs *regs;
	139	+ struct sa_trng_regs *trng_regs;
	140	+
	141	+ void *rx_chan;
	142	+ void *rx_fdq[KNAV_DMA_FDQ_PER_CHAN];
	143	+ void *rx_compl_q;
	144	+ void *tx_chan;
	145	+ void *tx_submit_q;
	146	+ void *tx_compl_q;
	147	+ u32 tx_submit_qid;
	148	+ u32 tx_compl_qid;
	149	+ u32 rx_compl_qid;
	150	+ const char *rx_chan_name;
	151	+ const char *tx_chan_name;
	152	+ u32 tx_queue_depth;
	153	+ u32 rx_queue_depths[KNAV_DMA_FDQ_PER_CHAN];
	154	+ u32 rx_buffer_sizes[KNAV_DMA_FDQ_PER_CHAN];
	155	+ u32 rx_pool_size;
	156	+ u32 rx_pool_region_id;
	157	+ void *rx_pool;
	158	+ u32 tx_pool_size;
	159	+ u32 tx_pool_region_id;
	160	+ void *tx_pool;
	161	+
	162	+ struct hwrng rng;
	163	+
	164	+ spinlock_t scid_lock; /* lock for SC-ID allocation */
	165	+ spinlock_t trng_lock; /* reading random data from TRNG */
	166	+
	167	+ struct kobject stats_kobj;
	168	+
	169	+ /* Security context data */
	170	+ u16 sc_id_start;
	171	+ u16 sc_id_end;
	172	+ u16 sc_id;
	173	+
	174	+ /* Bitmap to keep track of Security context ID's */
	175	+ unsigned long ctx_bm[DIV_ROUND_UP(SA_MAX_NUM_CTX,
	176	+ BITS_PER_LONG)];
	177	+ /* Driver stats */
	178	+ struct sa_drv_stats stats;
	179	+ atomic_t rx_dma_page_cnt; /* N buf from 2nd pool available */
	180	+ atomic_t tx_dma_desc_cnt; /* Tx DMA desc-s available */
	181	+};
	182	+
	183	+
	184	+
	185	+
	186	+/* Packet structure used in Rx */
	187	+#define SA_SGLIST_SIZE (MAX_SKB_FRAGS + SA_NUM_DMA_META_ELEMS)
	188	+struct sa_packet {
	189	+ struct scatterlist sg[SA_SGLIST_SIZE];
	190	+ int sg_ents;
	191	+ struct keystone_crypto_data *priv;
	192	+ struct dma_chan *chan;
	193	+ struct dma_async_tx_descriptor *desc;
	194	+ dma_cookie_t cookie;
	195	+ u32 epib[SA_DMA_NUM_EPIB_WORDS];
	196	+ u32 psdata[SA_DMA_NUM_PS_WORDS];
	197	+ struct completion complete;
	198	+ void *data;
	199	+};
	200	+
	201	+/* Command label updation info */
	202	+struct sa_cmdl_param_info {
	203	+ u16 index;
	204	+ u16 offset;
	205	+ u16 size;
	206	+};
	207	+
	208	+/* Maximum length of Auxiliary data in 32bit words */
	209	+#define SA_MAX_AUX_DATA_WORDS 8
	210	+
	211	+struct sa_cmdl_upd_info {
	212	+ u16 flags;
	213	+ u16 submode;
	214	+ struct sa_cmdl_param_info enc_size;
	215	+ struct sa_cmdl_param_info enc_size2;
	216	+ struct sa_cmdl_param_info enc_offset;
	217	+ struct sa_cmdl_param_info enc_iv;
	218	+ struct sa_cmdl_param_info enc_iv2;
	219	+ struct sa_cmdl_param_info aad;
	220	+ struct sa_cmdl_param_info payload;
	221	+ struct sa_cmdl_param_info auth_size;
	222	+ struct sa_cmdl_param_info auth_size2;
	223	+ struct sa_cmdl_param_info auth_offset;
	224	+ struct sa_cmdl_param_info auth_iv;
	225	+ struct sa_cmdl_param_info aux_key_info;
	226	+ u32 aux_key[SA_MAX_AUX_DATA_WORDS];
	227	+};
	228	+
	229	+enum sa_submode {
	230	+ SA_MODE_GEN = 0,
	231	+ SA_MODE_CCM,
	232	+ SA_MODE_GCM,
	233	+ SA_MODE_GMAC
	234	+};
	235	+
	236	+/* TFM Context info */
	237	+
	238	+/* Number of 32bit words appended after the command label
	239	+ * in PSDATA to identify the crypto request context.
	240	+ * word-0: Request type
	241	+ * word-1: pointer to request
	242	+ */
	243	+#define SA_NUM_PSDATA_CTX_WORDS 4
	244	+
	245	+/* Maximum size of Command label in 32 words */
	246	+#define SA_MAX_CMDL_WORDS (SA_DMA_NUM_PS_WORDS - SA_NUM_PSDATA_CTX_WORDS)
	247	+
	248	+struct sa_ctx_info {
	249	+ u8 *sc;
	250	+ dma_addr_t sc_phys;
	251	+ u16 sc_id;
	252	+ u16 cmdl_size;
	253	+ u32 cmdl[SA_MAX_CMDL_WORDS];
	254	+ struct sa_cmdl_upd_info cmdl_upd_info;
	255	+ /* Store Auxiliary data such as K2/K3 subkeys in AES-XCBC */
	256	+ u32 epib[SA_DMA_NUM_EPIB_WORDS];
	257	+ u32 rx_flow;
	258	+ u32 rx_compl_qid;
	259	+};
	260	+
	261	+struct sa_tfm_ctx {
	262	+ struct keystone_crypto_data *dev_data;
	263	+ struct sa_ctx_info enc;
	264	+ struct sa_ctx_info dec;
	265	+ struct sa_ctx_info auth;
	266	+};
	267	+
	268	+/* Tx DMA callback param */
	269	+struct sa_dma_req_ctx {
	270	+ struct keystone_crypto_data *dev_data;
	271	+ u32 cmdl[SA_MAX_CMDL_WORDS + SA_NUM_PSDATA_CTX_WORDS];
	272	+ unsigned map_idx;
	273	+ struct sg_table sg_tbl;
	274	+ dma_cookie_t cookie;
	275	+ struct dma_chan *tx_chan;
	276	+ bool pkt;
	277	+};
	278	+
	279	+/* Encryption algorithms */
	280	+enum sa_ealg_id {
	281	+ SA_EALG_ID_NONE = 0, /* No encryption */
	282	+ SA_EALG_ID_NULL, /* NULL encryption */
	283	+ SA_EALG_ID_AES_CTR, /* AES Counter mode */
	284	+ SA_EALG_ID_AES_F8, /* AES F8 mode */
	285	+ SA_EALG_ID_AES_CBC, /* AES CBC mode */
	286	+ SA_EALG_ID_DES_CBC, /* DES CBC mode */
	287	+ SA_EALG_ID_3DES_CBC, /* 3DES CBC mode */
	288	+ SA_EALG_ID_CCM, /* Counter with CBC-MAC mode */
	289	+ SA_EALG_ID_GCM, /* Galois Counter mode */
	290	+ SA_EALG_ID_LAST
	291	+};
	292	+
	293	+/* Authentication algorithms */
	294	+enum sa_aalg_id {
	295	+ SA_AALG_ID_NONE = 0, /* No Authentication */
	296	+ SA_AALG_ID_NULL = SA_EALG_ID_LAST, /* NULL Authentication */
	297	+ SA_AALG_ID_MD5, /* MD5 mode */
	298	+ SA_AALG_ID_SHA1, /* SHA1 mode */
	299	+ SA_AALG_ID_SHA2_224, /* 224-bit SHA2 mode */
	300	+ SA_AALG_ID_SHA2_256, /* 256-bit SHA2 mode */
	301	+ SA_AALG_ID_HMAC_MD5, /* HMAC with MD5 mode */
	302	+ SA_AALG_ID_HMAC_SHA1, /* HMAC with SHA1 mode */
	303	+ SA_AALG_ID_HMAC_SHA2_224, /* HMAC with 224-bit SHA2 mode */
	304	+ SA_AALG_ID_HMAC_SHA2_256, /* HMAC with 256-bit SHA2 mode */
	305	+ SA_AALG_ID_GMAC, /* Galois Message
	306	+ Authentication Code mode */
	307	+ SA_AALG_ID_CMAC, /* Cipher-based Message
	308	+ Authentication Code mode */
	309	+ SA_AALG_ID_CBC_MAC, /* Cipher Block Chaining */
	310	+ SA_AALG_ID_AES_XCBC /* AES Extended
	311	+ Cipher Block Chaining */
	312	+};
	313	+
	314	+/* Mode control engine algorithms used to index the
	315	+ * mode control instruction tables
	316	+ */
	317	+enum sa_eng_algo_id {
	318	+ SA_ENG_ALGO_ECB = 0,
	319	+ SA_ENG_ALGO_CBC,
	320	+ SA_ENG_ALGO_CFB,
	321	+ SA_ENG_ALGO_OFB,
	322	+ SA_ENG_ALGO_CTR,
	323	+ SA_ENG_ALGO_F8,
	324	+ SA_ENG_ALGO_GCM,
	325	+ SA_ENG_ALGO_GMAC,
	326	+ SA_ENG_ALGO_CCM,
	327	+ SA_ENG_ALGO_CMAC,
	328	+ SA_ENG_ALGO_CBCMAC,
	329	+ SA_NUM_ENG_ALGOS
	330	+};
	331	+
	332	+struct sa_eng_info {
	333	+ u8 eng_id;
	334	+ u16 sc_size;
	335	+};
	336	+
	337	+#define DMA_HAS_PSINFO BIT(31)
	338	+#define DMA_HAS_EPIB BIT(30)
	339	+
	340	+void sa_register_algos(const struct device *dev);
	341	+void sa_unregister_algos(const struct device *dev);
	342	+void sa_tx_completion_process(struct keystone_crypto_data *dev_data);
	343	+void sa_rx_completion_process(struct keystone_crypto_data *dev_data);
	344	+
	345	+void sa_set_sc_auth(u16 alg_id, const u8 key, u16 key_sz, u8 sc_buf);
	346	+int sa_set_sc_enc(u16 alg_id, const u8 *key, u16 key_sz,
	347	+ u16 aad_len, u8 enc, u8 *sc_buf);
	348	+
	349	+void sa_swiz_128(u8 in, u8 out, u16 len);
	350	+void sa_conv_calg_to_salg(const char cra_name, int ealg_id, int *aalg_id);
	351	+void sa_get_engine_info(int alg_id, struct sa_eng_info *info);
	352	+int sa_get_hash_size(u16 aalg_id);
	353	+
	354	+#define AES_MAXNR 14
	355	+struct asm_aes_key {
	356	+ unsigned int rd_key[4 * (AES_MAXNR + 1)];
	357	+ int rounds;
	358	+};
	359	+
	360	+/* AES encryption functions defined in aes-armv4.S */
	361	+asmlinkage void AES_encrypt(const u8 in, u8 out, struct asm_aes_key *key);
	362	+asmlinkage int private_AES_set_encrypt_key(const unsigned char *user_key,
	363	+ const int bits, struct asm_aes_key *key);
	364	+/*
	365	+ * Derive sub-key k1, k2 and k3 used in the AES XCBC MAC mode
	366	+ * detailed in RFC 3566
	367	+ */
	368	+static inline int sa_aes_xcbc_subkey(u8 sub_key1, u8 sub_key2,
	369	+ u8 sub_key3, const u8 key,
	370	+ u16 key_sz)
	371	+{
	372	+ struct asm_aes_key enc_key;
	373	+
	374	+ if (private_AES_set_encrypt_key(key, (key_sz * 8), &enc_key) == -1) {
	375	+ pr_err("%s: failed to set enc key\n", __func__);
	376	+ return -1;
	377	+ }
	378	+
	379	+ if (sub_key1) {
	380	+ memset(sub_key1, 0x01, AES_BLOCK_SIZE);
	381	+ AES_encrypt(sub_key1, sub_key1, &enc_key);
	382	+ }
	383	+
	384	+ if (sub_key2) {
	385	+ memset(sub_key2, 0x02, AES_BLOCK_SIZE);
	386	+ AES_encrypt(sub_key2, sub_key2, &enc_key);
	387	+ }
	388	+
	389	+ if (sub_key3) {
	390	+ memset(sub_key3, 0x03, AES_BLOCK_SIZE);
	391	+ AES_encrypt(sub_key3, sub_key3, &enc_key);
	392	+ }
	393	+
	394	+ return 0;
	395	+}
	396	+
	397	+
	398	+extern const uint8_t sa_eng_aes_enc_mci_tbl[11][3][27];
	399	+extern const uint8_t sa_eng_aes_dec_mci_tbl[11][3][27];
	400	+extern const uint8_t sa_eng_3des_enc_mci_tbl[4][27];
	401	+extern const uint8_t sa_eng_3des_dec_mci_tbl[4][27];
	402	+extern struct device *sa_ks2_dev;
	403	+
	404	+#endif /* _KEYSTONE_SA_HLP_ */
	1	+/*
	2	+ * Keystone crypto accelerator driver
	3	+ *
	4	+ * Copyright (C) 2013 Texas Instruments Incorporated - http://www.ti.com
	5	+ * Contact: Sandeep Nair <sandeep_n@ti.com>
	6	+ *
	7	+ * This program is free software; you can redistribute it and/or
	8	+ * modify it under the terms of the GNU General Public License
	9	+ * version 2 as published by the Free Software Foundation.
	10	+ *
	11	+ * This program is distributed in the hope that it will be useful, but
	12	+ * WITHOUT ANY WARRANTY; without even the implied warranty of
	13	+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	14	+ * General Public License for more details.
	15	+ */
	16	+
	17	+/*
	18	+ *
	19	+ * 0 1 2 3
	20	+ * 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
	21	+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -----
	22	+ * \| \|
	23	+ * \| Software only section (not fetched by CP_ACE) \|
	24	+ * \| (optional) \|
	25	+ * \| \|
	26	+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+<-Must be
	27	+ * \| SCCTL \| 64 byte
	28	+ * \| (8 bytes) \| aligned
	29	+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ ------
	30	+ * \| \|
	31	+ * \| PHP module specific section (fetched by CP_ACE) \|
	32	+ * \| (56 bytes) \|
	33	+ * \| \|
	34	+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -----
	35	+ * \| \|
	36	+ * \| Encryption module specific section (fetched by CP_ACE) \|
	37	+ * \| (variable size) \|
	38	+ * \| \|
	39	+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+<-64 byte
	40	+ * \| \| aligned
	41	+ * \| Authentication module specific section (fetched by CP_ACE) \|
	42	+ * \| (variable size) \|
	43	+ * \| \|
	44	+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -----
	45	+ *
	46	+ * Figure: Security Context memory layout
	47	+ *
	48	+ *
	49	+ * 0 1 2 3
	50	+ * 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
	51	+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -----
	52	+ * \|O\|Evict done \| F/E control \| SCID \|
	53	+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
	54	+ * \| SCPTR (Security Context Pointer) \|
	55	+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -----
	56	+ * O : Owner
	57	+ * SCID & SCPTR are filled by hardware
	58	+ * Figure: Security Context control word (SCCTL)
	59	+ *
	60	+ *
	61	+ * 0 1 2 3
	62	+ * 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
	63	+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ ------
	64	+ * \|D\| Pkt Type \| Flow Index \| Dest Queue ID \|
	65	+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ ------
	66	+ * \| SWINFO-0 (4 bytes) \|
	67	+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ ------
	68	+ * \| SWINFO-1 (4 bytes) \|
	69	+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ ------
	70	+ * \| PktID (16 bits) \| \|
	71	+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ ------
	72	+ * \| \|
	73	+ * \| Protocol Specific Parameters \|
	74	+ * \| (Variable Size up to 116 bytes \|
	75	+ * ... ...
	76	+ * \| \|
	77	+ * \| \|
	78	+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ ------
	79	+ * D : Direction
	80	+ * Figure: PHP engine Security Context Format
	81	+ *
	82	+ *
	83	+ * 0 1 2 3
	84	+ * 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
	85	+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ ------
	86	+ * \|M\| R \|nEngineID\| \|
	87	+ * +-+-+-+-+-+-+-+-+ +
	88	+ * \| encryption mode ctrl word \|
	89	+ * ... ...
	90	+ * \| (27 bytes) \|
	91	+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ ------
	92	+ * \| Reserved (4 bytes) (must initialize to 0) \|
	93	+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ ------
	94	+ * \| Encryption Key value (32 bytes) \|
	95	+ * ... ...
	96	+ * \| \|
	97	+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ ------
	98	+ * \| Encryption Aux-1 (32 bytes) (optional) \|
	99	+ * ... ...
	100	+ * \| \|
	101	+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ ------
	102	+ * \| Encryption Aux-2 (16 bytes) (optional) \|
	103	+ * ... ...
	104	+ * \| \|
	105	+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ ------
	106	+ * \| Encryption Aux-3 (16 bytes) (optional) \|
	107	+ * ... ...
	108	+ * \| \|
	109	+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ ------
	110	+ * \| Encryption Aux-4 (16 bytes) \|
	111	+ * ... ...
	112	+ * \| \|
	113	+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ ------
	114	+ * \| Pre-crypto data store (15 bytes) \|
	115	+ * \| \|
	116	+ * \| +-+-+-+-+-+-+-+-\|
	117	+ * \| \| \|
	118	+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ ------
	119	+ * M : Encryption Mode selector (0=crypto processing, 1=NULL)
	120	+ * R : Reserved
	121	+ * Figure: Encryption engine Security Context Format
	122	+ *
	123	+ *
	124	+ * 0 1 2 3
	125	+ * 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
	126	+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ ------
	127	+ * \|M\| R \|nEngineID\| Auth SW Ctrl \| \|
	128	+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +
	129	+ * \| Reserved (6 bytes) \|
	130	+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ ------
	131	+ * \| Authentication length (8 bytes) \|
	132	+ * \| ( 0 = let h/w calculate the length ) \|
	133	+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ ------
	134	+ * \| Reserved (12 bytes) \|
	135	+ * ... ...
	136	+ * \| \|
	137	+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ ------
	138	+ * \| HW crtl word (4 bytes) must be set to 0 by SW \|
	139	+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ ------
	140	+ * \| Authentication Key value (32 bytes) \|
	141	+ * ... Master Key or pre-computed inner digest for HMAC ...
	142	+ * \| \|
	143	+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ ------
	144	+ * \| Authentication Aux-1 (32 bytes) (optional) \|
	145	+ * ... Pre-computed outer opad for HMAC ...
	146	+ * \| \|
	147	+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ ------
	148	+ * \| Authentication Aux-2 (32 bytes) (optional) \|
	149	+ * ... ...
	150	+ * \| \|
	151	+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ ------
	152	+ * \| Pre-crypto data store (32 bytes) \|
	153	+ * ... ( HW access only) ...
	154	+ * \| \|
	155	+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ ------
	156	+ * \| Pre-crypto data store (32 bytes) \|
	157	+ * ... ( HW access only) ...
	158	+ * \| \|
	159	+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ ------
	160	+ * M : Authentication Mode selector (0=hash processing, 1=NULL)
	161	+ * R : Reserved
	162	+ * Figure: Authentication engine Security Context Format
	163	+ *
	164	+ */
	165	+
	166	+/******************************************************************************
	167	+ * This type represents the various packet types to be processed
	168	+ * by the PHP engine in SA.
	169	+ * It is used to identify the corresponding PHP processing function.
	170	+ ******************************************************************************/
	171	+typedef u8 SA_CTX_PE_PKT_TYPE_T;
	172	+#define SA_CTX_PE_PKT_TYPE_3GPP_AIR 0 /* 3GPP Air Cipher */
	173	+#define SA_CTX_PE_PKT_TYPE_SRTP 1 /* SRTP */
	174	+#define SA_CTX_PE_PKT_TYPE_IPSEC_AH 2 /* IPSec Authentication Header */
	175	+#define SA_CTX_PE_PKT_TYPE_IPSEC_ESP 3 /* IPSec Encapsulating
	176	+ Security Payload */
	177	+#define SA_CTX_PE_PKT_TYPE_NONE 4 /* Indicates that it is in
	178	+ data mode, It may not be
	179	+ used by PHP */
	180	+
	181	+
	182	+#define SA_CTX_ENC_TYPE1_SZ 64 /* Encryption SC with Key only */
	183	+#define SA_CTX_ENC_TYPE2_SZ 96 /* Encryption SC with Key and Aux1 */
	184	+
	185	+#define SA_CTX_AUTH_TYPE1_SZ 64 /* Auth SC with Key only */
	186	+#define SA_CTX_AUTH_TYPE2_SZ 96 /* Auth SC with Key and Aux1 */
	187	+
	188	+#define SA_CTX_PHP_PE_CTX_SZ 64 /* Size of security ctx for
	189	+ PHP engine */
	190	+
	191	+#define SA_CTX_MAX_SZ (64 + SA_CTX_ENC_TYPE2_SZ + SA_CTX_AUTH_TYPE2_SZ)
	192	+
	193	+/*
	194	+ * Encoding of F/E control in SCCTL
	195	+ * Bit 0-1: Fetch PHP Bytes
	196	+ * Bit 2-3: Fetch Encryption/Air Ciphering Bytes
	197	+ * Bit 4-5: Fetch Authentication Bytes or Encr pass 2
	198	+ * Bit 6-7: Evict PHP Bytes
	199	+ *
	200	+ * where 00 = 0 bytes
	201	+ * 01 = 64 bytes
	202	+ * 10 = 96 bytes
	203	+ * 11 = 128 bytes
	204	+ */
	205	+#define SA_CTX_DMA_SIZE_0 0
	206	+#define SA_CTX_DMA_SIZE_64 1
	207	+#define SA_CTX_DMA_SIZE_96 2
	208	+#define SA_CTX_DMA_SIZE_128 3
	209	+
	210	+#define SA_CTX_SCCTL_MK_DMA_INFO(php_f, eng0_f, eng1_f, php_e) \
	211	+ ((php_f) \| \
	212	+ ((eng0_f) << 2) \| \
	213	+ ((eng1_f) << 4) \| \
	214	+ ((php_e) << 6))
	215	+
	216	+/*
	217	+ * Byte offset of the owner word in SCCTL
	218	+ * in the security context
	219	+ */
	220	+#define SA_CTX_SCCTL_OWNER_OFFSET 0
	221	+
	222	+/*
	223	+ * Assumption: CTX size is multiple of 32
	224	+ */
	225	+#define SA_CTX_SIZE_TO_DMA_SIZE(ctx_sz) \
	226	+ ((ctx_sz) ? ((ctx_sz)/32 - 1) : 0)
	227	+
	228	+#define SA_CTX_ENC_KEY_OFFSET 32
	229	+#define SA_CTX_ENC_AUX1_OFFSET 64
	230	+#define SA_CTX_ENC_AUX2_OFFSET 96
	231	+#define SA_CTX_ENC_AUX3_OFFSET 112
	232	+#define SA_CTX_ENC_AUX4_OFFSET 128
	233	+
	234	+/* Next Engine Select code in CP_ACE */
	235	+#define SA_ENG_ID_EM1 2 /* Encryption/Decryption engine
	236	+ with AES/DES core */
	237	+#define SA_ENG_ID_EM2 3 /* Encryption/Decryption enginefor pass 2 */
	238	+#define SA_ENG_ID_AM1 4 /* Authentication engine with
	239	+ SHA1/MD5/SHA2 core */
	240	+#define SA_ENG_ID_AM2 5 /* Authentication engine for pass 2 */
	241	+#define SA_ENG_ID_OUTPORT2 20 /* Egress module 2 */
	242	+#define SA_ENG_ID_NONE 0xff
	243	+
	244	+/*
	245	+ * Command Label Definitions
	246	+ */
	247	+#define SA_CMDL_OFFSET_NESC 0 /* Next Engine Select Code */
	248	+#define SA_CMDL_OFFSET_LABEL_LEN 1 /* Engine Command Label Length */
	249	+#define SA_CMDL_OFFSET_DATA_LEN 2 /* 16-bit Length of Data to be
	250	+ processed */
	251	+#define SA_CMDL_OFFSET_DATA_OFFSET 4 /* Stat Data Offset */
	252	+#define SA_CMDL_OFFSET_OPTION_CTRL1 5 /* Option Control Byte 1 */
	253	+#define SA_CMDL_OFFSET_OPTION_CTRL2 6 /* Option Control Byte 2 */
	254	+#define SA_CMDL_OFFSET_OPTION_CTRL3 7 /* Option Control Byte 3 */
	255	+#define SA_CMDL_OFFSET_OPTION_BYTE 8
	256	+
	257	+#define SA_CMDL_HEADER_SIZE_BYTES 8
	258	+
	259	+#define SA_CMDL_OPTION_BYTES_MAX_SIZE 72
	260	+#define SA_CMDL_MAX_SIZE_BYTES (SA_CMDL_HEADER_SIZE_BYTES + \
	261	+ SA_CMDL_OPTION_BYTES_MAX_SIZE)
	262	+
	263	+/* SWINFO word-0 flags */
	264	+#define SA_SW_INFO_FLAG_EVICT 0x0001
	265	+#define SA_SW_INFO_FLAG_TEAR 0x0002
	266	+#define SA_SW_INFO_FLAG_NOPD 0x0004
	267	+
	268	+/*
	269	+ * TRNG module definitions
	270	+ */
	271	+
	272	+/* Offset to TRNG module in CP_ACE memory map */
	273	+#define SA_REG_MAP_TRNG_OFFSET 0x24000
	274	+
	275	+/* TRNG enable control in CP_ACE */
	276	+#define SA_CMD_STATUS_REG_TRNG_ENABLE BIT(3)
	277	+
	278	+/* TRNG start control in TRNG module */
	279	+#define SA_TRNG_CONTROL_REG_TRNG_ENABLE BIT(10)
	280	+
	281	+/* Data ready indicator in STATUS register */
	282	+#define SA_TRNG_STATUS_REG_READY BIT(0)
	283	+
	284	+/* Data ready clear control in INTACK register */
	285	+#define SA_TRNG_INTACK_REG_READY BIT(0)
	286	+
	287	+/* Number of samples taken to gather entropy during startup.
	288	+ * If value is 0, the number of samples is 2^24 else
	289	+ * equals value times 2^8.
	290	+ */
	291	+#define SA_TRNG_DEF_STARTUP_CYCLES 0
	292	+#define SA_TRNG_CONTROL_REG_STARTUP_CYCLES_SHIFT 16
	293	+
	294	+/* Minimum number of samples taken to regenerate entropy
	295	+ * If value is 0, the number of samples is 2^24 else
	296	+ * equals value times 2^6.
	297	+ */
	298	+#define SA_TRNG_DEF_MIN_REFILL_CYCLES 1
	299	+#define SA_TRNG_CONFIG_REG_MIN_REFILL_CYCLES_SHIFT 0
	300	+
	301	+/* Maximum number of samples taken to regenerate entropy
	302	+ * If value is 0, the number of samples is 2^24 else
	303	+ * equals value times 2^8.
	304	+ */
	305	+#define SA_TRNG_DEF_MAX_REFILL_CYCLES 0
	306	+#define SA_TRNG_CONFIG_REG_MAX_REFILL_CYCLES_SHIFT 16
	307	+
	308	+/* Number of CLK input cycles between samples */
	309	+#define SA_TRNG_DEF_CLK_DIV_CYCLES 0
	310	+#define SA_TRNG_CONFIG_REG_SAMPLE_DIV_SHIFT 8