crypto: caam - add support for rfc3686 with authenc md5, sha1 and sha2

Add support for AES Counter Mode (CTR) compliant with RFC3686 to be used along with authenc algorithms (md5, sha1, sha224, sha256, sha384, sha512) as one-shot aead algorithms. Signed-off-by: Catalin Vasile <catalin.vasile@freescale.com> Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>

crypto: caam - add support for rfc3686 with authenc md5, sha1 and sha2
Add support for AES Counter Mode (CTR) compliant with RFC3686 to be used along with authenc algorithms (md5, sha1, sha224, sha256, sha384, sha512) as one-shot aead algorithms. Signed-off-by: Catalin Vasile <catalin.vasile@freescale.com> Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
Catalin Vasile · Herbert Xu
1 parent a5f57cffce
Showing 1 changed file with 236 additions and 26 deletions Side-by-side Diff
drivers/crypto/caam/caamalg.c
@@ -60,6 +60,7 @@
 #define CAAM_CRA_PRIORITY		3000
 /* max key is sum of AES_MAX_KEY_SIZE, max split key size */
 #define CAAM_MAX_KEY_SIZE		(AES_MAX_KEY_SIZE + \
+					 CTR_RFC3686_NONCE_SIZE + \
 					 SHA512_DIGEST_SIZE * 2)
 /* max IV is max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */
 #define CAAM_MAX_IV_LENGTH		16
@@ -70,6 +71,9 @@
 #define DESC_AEAD_DEC_LEN		(DESC_AEAD_BASE + 18 * CAAM_CMD_SZ)
 #define DESC_AEAD_GIVENC_LEN		(DESC_AEAD_ENC_LEN + 7 * CAAM_CMD_SZ)
  
+/* Note: Nonce is counted in enckeylen */
+#define DESC_AEAD_CTR_RFC3686_LEN	(6 * CAAM_CMD_SZ)
+
 #define DESC_AEAD_NULL_BASE		(3 * CAAM_CMD_SZ)
 #define DESC_AEAD_NULL_ENC_LEN		(DESC_AEAD_NULL_BASE + 14 * CAAM_CMD_SZ)
 #define DESC_AEAD_NULL_DEC_LEN		(DESC_AEAD_NULL_BASE + 17 * CAAM_CMD_SZ)
  
@@ -142,11 +146,13 @@
 /*
  * For aead encrypt and decrypt, read iv for both classes
  */
-static inline void aead_append_ld_iv(u32 *desc, int ivsize)
+static inline void aead_append_ld_iv(u32 *desc, int ivsize, int ivoffset)
 {
-	append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_BYTE_CONTEXT |
-		   LDST_CLASS_1_CCB | ivsize);
-	append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_CLASS2INFIFO | ivsize);
+	append_seq_load(desc, ivsize, LDST_CLASS_1_CCB |
+			LDST_SRCDST_BYTE_CONTEXT |
+			(ivoffset << LDST_OFFSET_SHIFT));
+	append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_CLASS2INFIFO |
+		    (ivoffset << MOVE_OFFSET_SHIFT) | ivsize);
 }
  
 /*
  
  
  
  
  
  
  
@@ -192,35 +198,60 @@
 };
  
 static void append_key_aead(u32 *desc, struct caam_ctx *ctx,
-			    int keys_fit_inline)
+			    int keys_fit_inline, bool is_rfc3686)
 {
+	u32 *nonce;
+	unsigned int enckeylen = ctx->enckeylen;
+
+	/*
+	 * RFC3686 specific:
+	 *	| ctx->key = {AUTH_KEY, ENC_KEY, NONCE}
+	 *	| enckeylen = encryption key size + nonce size
+	 */
+	if (is_rfc3686)
+		enckeylen -= CTR_RFC3686_NONCE_SIZE;
+
 	if (keys_fit_inline) {
 		append_key_as_imm(desc, ctx->key, ctx->split_key_pad_len,
 				  ctx->split_key_len, CLASS_2 |
 				  KEY_DEST_MDHA_SPLIT | KEY_ENC);
 		append_key_as_imm(desc, (void *)ctx->key +
-				  ctx->split_key_pad_len, ctx->enckeylen,
-				  ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
+				  ctx->split_key_pad_len, enckeylen,
+				  enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
 	} else {
 		append_key(desc, ctx->key_dma, ctx->split_key_len, CLASS_2 |
 			   KEY_DEST_MDHA_SPLIT | KEY_ENC);
 		append_key(desc, ctx->key_dma + ctx->split_key_pad_len,
-			   ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
+			   enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
 	}
+
+	/* Load Counter into CONTEXT1 reg */
+	if (is_rfc3686) {
+		nonce = (u32 *)((void *)ctx->key + ctx->split_key_pad_len +
+			       enckeylen);
+		append_load_imm_u32(desc, *nonce, LDST_CLASS_IND_CCB |
+				    LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
+		append_move(desc,
+			    MOVE_SRC_OUTFIFO |
+			    MOVE_DEST_CLASS1CTX |
+			    (16 << MOVE_OFFSET_SHIFT) |
+			    (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT));
+	}
 }
  
 static void init_sh_desc_key_aead(u32 *desc, struct caam_ctx *ctx,
-				  int keys_fit_inline)
+				  int keys_fit_inline, bool is_rfc3686)
 {
 	u32 *key_jump_cmd;
  
-	init_sh_desc(desc, HDR_SHARE_SERIAL);
+	/* Note: Context registers are saved. */
+	init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
  
 	/* Skip if already shared */
 	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
 				   JUMP_COND_SHRD);
  
-	append_key_aead(desc, ctx, keys_fit_inline);
+	append_key_aead(desc, ctx, keys_fit_inline, is_rfc3686);
  
 	set_jump_tgt_here(desc, key_jump_cmd);
 }
  
  
  
@@ -420,10 +451,17 @@
 {
 	struct aead_tfm *tfm = &aead->base.crt_aead;
 	struct caam_ctx *ctx = crypto_aead_ctx(aead);
+	struct crypto_tfm *ctfm = crypto_aead_tfm(aead);
+	const char *alg_name = crypto_tfm_alg_name(ctfm);
 	struct device *jrdev = ctx->jrdev;
-	bool keys_fit_inline = false;
+	bool keys_fit_inline;
 	u32 geniv, moveiv;
+	u32 ctx1_iv_off = 0;
 	u32 *desc;
+	const bool ctr_mode = ((ctx->class1_alg_type & OP_ALG_AAI_MASK) ==
+			       OP_ALG_AAI_CTR_MOD128);
+	const bool is_rfc3686 = (ctr_mode &&
+				 (strstr(alg_name, "rfc3686") != NULL));
  
 	if (!ctx->authsize)
 		return 0;
  
  
  
@@ -433,18 +471,36 @@
 		return aead_null_set_sh_desc(aead);
  
 	/*
+	 * AES-CTR needs to load IV in CONTEXT1 reg
+	 * at an offset of 128bits (16bytes)
+	 * CONTEXT1[255:128] = IV
+	 */
+	if (ctr_mode)
+		ctx1_iv_off = 16;
+
+	/*
+	 * RFC3686 specific:
+	 *	CONTEXT1[255:128] = {NONCE, IV, COUNTER}
+	 */
+	if (is_rfc3686)
+		ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE;
+
+	/*
 	 * Job Descriptor and Shared Descriptors
 	 * must all fit into the 64-word Descriptor h/w Buffer
 	 */
+	keys_fit_inline = false;
 	if (DESC_AEAD_ENC_LEN + DESC_JOB_IO_LEN +
-	    ctx->split_key_pad_len + ctx->enckeylen <=
+	    ctx->split_key_pad_len + ctx->enckeylen +
+	    (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0) <=
 	    CAAM_DESC_BYTES_MAX)
 		keys_fit_inline = true;
  
 	/* aead_encrypt shared descriptor */
 	desc = ctx->sh_desc_enc;
  
-	init_sh_desc_key_aead(desc, ctx, keys_fit_inline);
+	/* Note: Context registers are saved. */
+	init_sh_desc_key_aead(desc, ctx, keys_fit_inline, is_rfc3686);
  
 	/* Class 2 operation */
 	append_operation(desc, ctx->class2_alg_type |
  
@@ -462,8 +518,16 @@
 	/* read assoc before reading payload */
 	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
 			     KEY_VLF);
-	aead_append_ld_iv(desc, tfm->ivsize);
+	aead_append_ld_iv(desc, tfm->ivsize, ctx1_iv_off);
  
+	/* Load Counter into CONTEXT1 reg */
+	if (is_rfc3686)
+		append_load_imm_u32(desc, be32_to_cpu(1), LDST_IMM |
+				    LDST_CLASS_1_CCB |
+				    LDST_SRCDST_BYTE_CONTEXT |
+				    ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
+				     LDST_OFFSET_SHIFT));
+
 	/* Class 1 operation */
 	append_operation(desc, ctx->class1_alg_type |
 			 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
  
@@ -496,14 +560,16 @@
 	 */
 	keys_fit_inline = false;
 	if (DESC_AEAD_DEC_LEN + DESC_JOB_IO_LEN +
-	    ctx->split_key_pad_len + ctx->enckeylen <=
+	    ctx->split_key_pad_len + ctx->enckeylen +
+	    (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0) <=
 	    CAAM_DESC_BYTES_MAX)
 		keys_fit_inline = true;
  
 	/* aead_decrypt shared descriptor */
 	desc = ctx->sh_desc_dec;
  
-	init_sh_desc_key_aead(desc, ctx, keys_fit_inline);
+	/* Note: Context registers are saved. */
+	init_sh_desc_key_aead(desc, ctx, keys_fit_inline, is_rfc3686);
  
 	/* Class 2 operation */
 	append_operation(desc, ctx->class2_alg_type |
  
  
@@ -520,10 +586,23 @@
 	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
 			     KEY_VLF);
  
-	aead_append_ld_iv(desc, tfm->ivsize);
+	aead_append_ld_iv(desc, tfm->ivsize, ctx1_iv_off);
  
-	append_dec_op1(desc, ctx->class1_alg_type);
+	/* Load Counter into CONTEXT1 reg */
+	if (is_rfc3686)
+		append_load_imm_u32(desc, be32_to_cpu(1), LDST_IMM |
+				    LDST_CLASS_1_CCB |
+				    LDST_SRCDST_BYTE_CONTEXT |
+				    ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
+				     LDST_OFFSET_SHIFT));
  
+	/* Choose operation */
+	if (ctr_mode)
+		append_operation(desc, ctx->class1_alg_type |
+				 OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT);
+	else
+		append_dec_op1(desc, ctx->class1_alg_type);
+
 	/* Read and write cryptlen bytes */
 	append_math_add(desc, VARSEQINLEN, ZERO, REG2, CAAM_CMD_SZ);
 	append_math_add(desc, VARSEQOUTLEN, ZERO, REG2, CAAM_CMD_SZ);
  
@@ -552,14 +631,16 @@
 	 */
 	keys_fit_inline = false;
 	if (DESC_AEAD_GIVENC_LEN + DESC_JOB_IO_LEN +
-	    ctx->split_key_pad_len + ctx->enckeylen <=
+	    ctx->split_key_pad_len + ctx->enckeylen +
+	    (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0) <=
 	    CAAM_DESC_BYTES_MAX)
 		keys_fit_inline = true;
  
 	/* aead_givencrypt shared descriptor */
 	desc = ctx->sh_desc_givenc;
  
-	init_sh_desc_key_aead(desc, ctx, keys_fit_inline);
+	/* Note: Context registers are saved. */
+	init_sh_desc_key_aead(desc, ctx, keys_fit_inline, is_rfc3686);
  
 	/* Generate IV */
 	geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO |
  
@@ -568,13 +649,16 @@
 	append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB |
 			    LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM);
 	append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
-	append_move(desc, MOVE_SRC_INFIFO |
-		    MOVE_DEST_CLASS1CTX | (tfm->ivsize << MOVE_LEN_SHIFT));
+	append_move(desc, MOVE_WAITCOMP |
+		    MOVE_SRC_INFIFO | MOVE_DEST_CLASS1CTX |
+		    (ctx1_iv_off << MOVE_OFFSET_SHIFT) |
+		    (tfm->ivsize << MOVE_LEN_SHIFT));
 	append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
  
 	/* Copy IV to class 1 context */
-	append_move(desc, MOVE_SRC_CLASS1CTX |
-		    MOVE_DEST_OUTFIFO | (tfm->ivsize << MOVE_LEN_SHIFT));
+	append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_OUTFIFO |
+		    (ctx1_iv_off << MOVE_OFFSET_SHIFT) |
+		    (tfm->ivsize << MOVE_LEN_SHIFT));
  
 	/* Return to encryption */
 	append_operation(desc, ctx->class2_alg_type |
@@ -590,7 +674,7 @@
 	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
 			     KEY_VLF);
  
-	/* Copy iv from class 1 ctx to class 2 fifo*/
+	/* Copy iv from outfifo to class 2 fifo */
 	moveiv = NFIFOENTRY_STYPE_OFIFO | NFIFOENTRY_DEST_CLASS2 |
 		 NFIFOENTRY_DTYPE_MSG | (tfm->ivsize << NFIFOENTRY_DLEN_SHIFT);
 	append_load_imm_u32(desc, moveiv, LDST_CLASS_IND_CCB |
@@ -598,6 +682,14 @@
 	append_load_imm_u32(desc, tfm->ivsize, LDST_CLASS_2_CCB |
 			    LDST_SRCDST_WORD_DATASZ_REG | LDST_IMM);
  
+	/* Load Counter into CONTEXT1 reg */
+	if (is_rfc3686)
+		append_load_imm_u32(desc, be32_to_cpu(1), LDST_IMM |
+				    LDST_CLASS_1_CCB |
+				    LDST_SRCDST_BYTE_CONTEXT |
+				    ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
+				     LDST_OFFSET_SHIFT));
+
 	/* Class 1 operation */
 	append_operation(desc, ctx->class1_alg_type |
 			 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
@@ -3498,6 +3590,124 @@
 			.maxauthsize = SHA512_DIGEST_SIZE,
 			},
 		.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
+		.class2_alg_type = OP_ALG_ALGSEL_SHA512 |
+				   OP_ALG_AAI_HMAC_PRECOMP,
+		.alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC,
+	},
+	{
+		.name = "authenc(hmac(md5),rfc3686(ctr(aes)))",
+		.driver_name = "authenc-hmac-md5-rfc3686-ctr-aes-caam",
+		.blocksize = 1,
+		.type = CRYPTO_ALG_TYPE_AEAD,
+		.template_aead = {
+			.setkey = aead_setkey,
+			.setauthsize = aead_setauthsize,
+			.encrypt = aead_encrypt,
+			.decrypt = aead_decrypt,
+			.givencrypt = aead_givencrypt,
+			.geniv = "<built-in>",
+			.ivsize = CTR_RFC3686_IV_SIZE,
+			.maxauthsize = MD5_DIGEST_SIZE,
+			},
+		.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128,
+		.class2_alg_type = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC_PRECOMP,
+		.alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC,
+	},
+	{
+		.name = "authenc(hmac(sha1),rfc3686(ctr(aes)))",
+		.driver_name = "authenc-hmac-sha1-rfc3686-ctr-aes-caam",
+		.blocksize = 1,
+		.type = CRYPTO_ALG_TYPE_AEAD,
+		.template_aead = {
+			.setkey = aead_setkey,
+			.setauthsize = aead_setauthsize,
+			.encrypt = aead_encrypt,
+			.decrypt = aead_decrypt,
+			.givencrypt = aead_givencrypt,
+			.geniv = "<built-in>",
+			.ivsize = CTR_RFC3686_IV_SIZE,
+			.maxauthsize = SHA1_DIGEST_SIZE,
+			},
+		.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128,
+		.class2_alg_type = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC_PRECOMP,
+		.alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC,
+	},
+	{
+		.name = "authenc(hmac(sha224),rfc3686(ctr(aes)))",
+		.driver_name = "authenc-hmac-sha224-rfc3686-ctr-aes-caam",
+		.blocksize = 1,
+		.type = CRYPTO_ALG_TYPE_AEAD,
+		.template_aead = {
+			.setkey = aead_setkey,
+			.setauthsize = aead_setauthsize,
+			.encrypt = aead_encrypt,
+			.decrypt = aead_decrypt,
+			.givencrypt = aead_givencrypt,
+			.geniv = "<built-in>",
+			.ivsize = CTR_RFC3686_IV_SIZE,
+			.maxauthsize = SHA224_DIGEST_SIZE,
+			},
+		.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128,
+		.class2_alg_type = OP_ALG_ALGSEL_SHA224 |
+				   OP_ALG_AAI_HMAC_PRECOMP,
+		.alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC,
+	},
+	{
+		.name = "authenc(hmac(sha256),rfc3686(ctr(aes)))",
+		.driver_name = "authenc-hmac-sha256-rfc3686-ctr-aes-caam",
+		.blocksize = 1,
+		.type = CRYPTO_ALG_TYPE_AEAD,
+		.template_aead = {
+			.setkey = aead_setkey,
+			.setauthsize = aead_setauthsize,
+			.encrypt = aead_encrypt,
+			.decrypt = aead_decrypt,
+			.givencrypt = aead_givencrypt,
+			.geniv = "<built-in>",
+			.ivsize = CTR_RFC3686_IV_SIZE,
+			.maxauthsize = SHA256_DIGEST_SIZE,
+			},
+		.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128,
+		.class2_alg_type = OP_ALG_ALGSEL_SHA256 |
+				   OP_ALG_AAI_HMAC_PRECOMP,
+		.alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC,
+	},
+	{
+		.name = "authenc(hmac(sha384),rfc3686(ctr(aes)))",
+		.driver_name = "authenc-hmac-sha384-rfc3686-ctr-aes-caam",
+		.blocksize = 1,
+		.type = CRYPTO_ALG_TYPE_AEAD,
+		.template_aead = {
+			.setkey = aead_setkey,
+			.setauthsize = aead_setauthsize,
+			.encrypt = aead_encrypt,
+			.decrypt = aead_decrypt,
+			.givencrypt = aead_givencrypt,
+			.geniv = "<built-in>",
+			.ivsize = CTR_RFC3686_IV_SIZE,
+			.maxauthsize = SHA384_DIGEST_SIZE,
+			},
+		.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128,
+		.class2_alg_type = OP_ALG_ALGSEL_SHA384 |
+				   OP_ALG_AAI_HMAC_PRECOMP,
+		.alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC,
+	},
+	{
+		.name = "authenc(hmac(sha512),rfc3686(ctr(aes)))",
+		.driver_name = "authenc-hmac-sha512-rfc3686-ctr-aes-caam",
+		.blocksize = 1,
+		.type = CRYPTO_ALG_TYPE_AEAD,
+		.template_aead = {
+			.setkey = aead_setkey,
+			.setauthsize = aead_setauthsize,
+			.encrypt = aead_encrypt,
+			.decrypt = aead_decrypt,
+			.givencrypt = aead_givencrypt,
+			.geniv = "<built-in>",
+			.ivsize = CTR_RFC3686_IV_SIZE,
+			.maxauthsize = SHA512_DIGEST_SIZE,
+			},
+		.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128,
 		.class2_alg_type = OP_ALG_ALGSEL_SHA512 |
 				   OP_ALG_AAI_HMAC_PRECOMP,
 		.alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC,
...	...	@@ -60,6 +60,7 @@
60	60	#define CAAM_CRA_PRIORITY 3000
61	61	/* max key is sum of AES_MAX_KEY_SIZE, max split key size */
62	62	#define CAAM_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + \
	63	+ CTR_RFC3686_NONCE_SIZE + \
63	64	SHA512_DIGEST_SIZE * 2)
64	65	/* max IV is max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */
65	66	#define CAAM_MAX_IV_LENGTH 16
...	...	@@ -70,6 +71,9 @@
70	71	#define DESC_AEAD_DEC_LEN (DESC_AEAD_BASE + 18 * CAAM_CMD_SZ)
71	72	#define DESC_AEAD_GIVENC_LEN (DESC_AEAD_ENC_LEN + 7 * CAAM_CMD_SZ)
72	73
	74	+/* Note: Nonce is counted in enckeylen */
	75	+#define DESC_AEAD_CTR_RFC3686_LEN (6 * CAAM_CMD_SZ)
	76	+
73	77	#define DESC_AEAD_NULL_BASE (3 * CAAM_CMD_SZ)
74	78	#define DESC_AEAD_NULL_ENC_LEN (DESC_AEAD_NULL_BASE + 14 * CAAM_CMD_SZ)
75	79	#define DESC_AEAD_NULL_DEC_LEN (DESC_AEAD_NULL_BASE + 17 * CAAM_CMD_SZ)
76	80
...	...	@@ -142,11 +146,13 @@
142	146	/*
143	147	* For aead encrypt and decrypt, read iv for both classes
144	148	*/
145		-static inline void aead_append_ld_iv(u32 *desc, int ivsize)
	149	+static inline void aead_append_ld_iv(u32 *desc, int ivsize, int ivoffset)
146	150	{
147		- append_cmd(desc, CMD_SEQ_LOAD \| LDST_SRCDST_BYTE_CONTEXT \|
148		- LDST_CLASS_1_CCB \| ivsize);
149		- append_move(desc, MOVE_SRC_CLASS1CTX \| MOVE_DEST_CLASS2INFIFO \| ivsize);
	151	+ append_seq_load(desc, ivsize, LDST_CLASS_1_CCB \|
	152	+ LDST_SRCDST_BYTE_CONTEXT \|
	153	+ (ivoffset << LDST_OFFSET_SHIFT));
	154	+ append_move(desc, MOVE_SRC_CLASS1CTX \| MOVE_DEST_CLASS2INFIFO \|
	155	+ (ivoffset << MOVE_OFFSET_SHIFT) \| ivsize);
150	156	}
151	157
152	158	/*
153	159
154	160
155	161
156	162
157	163
158	164
159	165
...	...	@@ -192,35 +198,60 @@
192	198	};
193	199
194	200	static void append_key_aead(u32 desc, struct caam_ctx ctx,
195		- int keys_fit_inline)
	201	+ int keys_fit_inline, bool is_rfc3686)
196	202	{
	203	+ u32 *nonce;
	204	+ unsigned int enckeylen = ctx->enckeylen;
	205	+
	206	+ /*
	207	+ * RFC3686 specific:
	208	+ * \| ctx->key = {AUTH_KEY, ENC_KEY, NONCE}
	209	+ * \| enckeylen = encryption key size + nonce size
	210	+ */
	211	+ if (is_rfc3686)
	212	+ enckeylen -= CTR_RFC3686_NONCE_SIZE;
	213	+
197	214	if (keys_fit_inline) {
198	215	append_key_as_imm(desc, ctx->key, ctx->split_key_pad_len,
199	216	ctx->split_key_len, CLASS_2 \|
200	217	KEY_DEST_MDHA_SPLIT \| KEY_ENC);
201	218	append_key_as_imm(desc, (void *)ctx->key +
202		- ctx->split_key_pad_len, ctx->enckeylen,
203		- ctx->enckeylen, CLASS_1 \| KEY_DEST_CLASS_REG);
	219	+ ctx->split_key_pad_len, enckeylen,
	220	+ enckeylen, CLASS_1 \| KEY_DEST_CLASS_REG);
204	221	} else {
205	222	append_key(desc, ctx->key_dma, ctx->split_key_len, CLASS_2 \|
206	223	KEY_DEST_MDHA_SPLIT \| KEY_ENC);
207	224	append_key(desc, ctx->key_dma + ctx->split_key_pad_len,
208		- ctx->enckeylen, CLASS_1 \| KEY_DEST_CLASS_REG);
	225	+ enckeylen, CLASS_1 \| KEY_DEST_CLASS_REG);
209	226	}
	227	+
	228	+ /* Load Counter into CONTEXT1 reg */
	229	+ if (is_rfc3686) {
	230	+ nonce = (u32 )((void )ctx->key + ctx->split_key_pad_len +
	231	+ enckeylen);
	232	+ append_load_imm_u32(desc, *nonce, LDST_CLASS_IND_CCB \|
	233	+ LDST_SRCDST_BYTE_OUTFIFO \| LDST_IMM);
	234	+ append_move(desc,
	235	+ MOVE_SRC_OUTFIFO \|
	236	+ MOVE_DEST_CLASS1CTX \|
	237	+ (16 << MOVE_OFFSET_SHIFT) \|
	238	+ (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT));
	239	+ }
210	240	}
211	241
212	242	static void init_sh_desc_key_aead(u32 desc, struct caam_ctx ctx,
213		- int keys_fit_inline)
	243	+ int keys_fit_inline, bool is_rfc3686)
214	244	{
215	245	u32 *key_jump_cmd;
216	246
217		- init_sh_desc(desc, HDR_SHARE_SERIAL);
	247	+ /* Note: Context registers are saved. */
	248	+ init_sh_desc(desc, HDR_SHARE_SERIAL \| HDR_SAVECTX);
218	249
219	250	/* Skip if already shared */
220	251	key_jump_cmd = append_jump(desc, JUMP_JSL \| JUMP_TEST_ALL \|
221	252	JUMP_COND_SHRD);
222	253
223		- append_key_aead(desc, ctx, keys_fit_inline);
	254	+ append_key_aead(desc, ctx, keys_fit_inline, is_rfc3686);
224	255
225	256	set_jump_tgt_here(desc, key_jump_cmd);
226	257	}
227	258
228	259
229	260
...	...	@@ -420,10 +451,17 @@
420	451	{
421	452	struct aead_tfm *tfm = &aead->base.crt_aead;
422	453	struct caam_ctx *ctx = crypto_aead_ctx(aead);
	454	+ struct crypto_tfm *ctfm = crypto_aead_tfm(aead);
	455	+ const char *alg_name = crypto_tfm_alg_name(ctfm);
423	456	struct device *jrdev = ctx->jrdev;
424		- bool keys_fit_inline = false;
	457	+ bool keys_fit_inline;
425	458	u32 geniv, moveiv;
	459	+ u32 ctx1_iv_off = 0;
426	460	u32 *desc;
	461	+ const bool ctr_mode = ((ctx->class1_alg_type & OP_ALG_AAI_MASK) ==
	462	+ OP_ALG_AAI_CTR_MOD128);
	463	+ const bool is_rfc3686 = (ctr_mode &&
	464	+ (strstr(alg_name, "rfc3686") != NULL));
427	465
428	466	if (!ctx->authsize)
429	467	return 0;
430	468
431	469
432	470
...	...	@@ -433,18 +471,36 @@
433	471	return aead_null_set_sh_desc(aead);
434	472
435	473	/*
	474	+ * AES-CTR needs to load IV in CONTEXT1 reg
	475	+ * at an offset of 128bits (16bytes)
	476	+ * CONTEXT1[255:128] = IV
	477	+ */
	478	+ if (ctr_mode)
	479	+ ctx1_iv_off = 16;
	480	+
	481	+ /*
	482	+ * RFC3686 specific:
	483	+ * CONTEXT1[255:128] = {NONCE, IV, COUNTER}
	484	+ */
	485	+ if (is_rfc3686)
	486	+ ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE;
	487	+
	488	+ /*
436	489	* Job Descriptor and Shared Descriptors
437	490	* must all fit into the 64-word Descriptor h/w Buffer
438	491	*/
	492	+ keys_fit_inline = false;
439	493	if (DESC_AEAD_ENC_LEN + DESC_JOB_IO_LEN +
440		- ctx->split_key_pad_len + ctx->enckeylen <=
	494	+ ctx->split_key_pad_len + ctx->enckeylen +
	495	+ (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0) <=
441	496	CAAM_DESC_BYTES_MAX)
442	497	keys_fit_inline = true;
443	498
444	499	/* aead_encrypt shared descriptor */
445	500	desc = ctx->sh_desc_enc;
446	501
447		- init_sh_desc_key_aead(desc, ctx, keys_fit_inline);
	502	+ /* Note: Context registers are saved. */
	503	+ init_sh_desc_key_aead(desc, ctx, keys_fit_inline, is_rfc3686);
448	504
449	505	/* Class 2 operation */
450	506	append_operation(desc, ctx->class2_alg_type \|
451	507
...	...	@@ -462,8 +518,16 @@
462	518	/* read assoc before reading payload */
463	519	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 \| FIFOLD_TYPE_MSG \|
464	520	KEY_VLF);
465		- aead_append_ld_iv(desc, tfm->ivsize);
	521	+ aead_append_ld_iv(desc, tfm->ivsize, ctx1_iv_off);
466	522
	523	+ /* Load Counter into CONTEXT1 reg */
	524	+ if (is_rfc3686)
	525	+ append_load_imm_u32(desc, be32_to_cpu(1), LDST_IMM \|
	526	+ LDST_CLASS_1_CCB \|
	527	+ LDST_SRCDST_BYTE_CONTEXT \|
	528	+ ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
	529	+ LDST_OFFSET_SHIFT));
	530	+
467	531	/* Class 1 operation */
468	532	append_operation(desc, ctx->class1_alg_type \|
469	533	OP_ALG_AS_INITFINAL \| OP_ALG_ENCRYPT);
470	534
...	...	@@ -496,14 +560,16 @@
496	560	*/
497	561	keys_fit_inline = false;
498	562	if (DESC_AEAD_DEC_LEN + DESC_JOB_IO_LEN +
499		- ctx->split_key_pad_len + ctx->enckeylen <=
	563	+ ctx->split_key_pad_len + ctx->enckeylen +
	564	+ (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0) <=
500	565	CAAM_DESC_BYTES_MAX)
501	566	keys_fit_inline = true;
502	567
503	568	/* aead_decrypt shared descriptor */
504	569	desc = ctx->sh_desc_dec;
505	570
506		- init_sh_desc_key_aead(desc, ctx, keys_fit_inline);
	571	+ /* Note: Context registers are saved. */
	572	+ init_sh_desc_key_aead(desc, ctx, keys_fit_inline, is_rfc3686);
507	573
508	574	/* Class 2 operation */
509	575	append_operation(desc, ctx->class2_alg_type \|
510	576
511	577
...	...	@@ -520,10 +586,23 @@
520	586	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 \| FIFOLD_TYPE_MSG \|
521	587	KEY_VLF);
522	588
523		- aead_append_ld_iv(desc, tfm->ivsize);
	589	+ aead_append_ld_iv(desc, tfm->ivsize, ctx1_iv_off);
524	590
525		- append_dec_op1(desc, ctx->class1_alg_type);
	591	+ /* Load Counter into CONTEXT1 reg */
	592	+ if (is_rfc3686)
	593	+ append_load_imm_u32(desc, be32_to_cpu(1), LDST_IMM \|
	594	+ LDST_CLASS_1_CCB \|
	595	+ LDST_SRCDST_BYTE_CONTEXT \|
	596	+ ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
	597	+ LDST_OFFSET_SHIFT));
526	598
	599	+ /* Choose operation */
	600	+ if (ctr_mode)
	601	+ append_operation(desc, ctx->class1_alg_type \|
	602	+ OP_ALG_AS_INITFINAL \| OP_ALG_DECRYPT);
	603	+ else
	604	+ append_dec_op1(desc, ctx->class1_alg_type);
	605	+
527	606	/* Read and write cryptlen bytes */
528	607	append_math_add(desc, VARSEQINLEN, ZERO, REG2, CAAM_CMD_SZ);
529	608	append_math_add(desc, VARSEQOUTLEN, ZERO, REG2, CAAM_CMD_SZ);
530	609
...	...	@@ -552,14 +631,16 @@
552	631	*/
553	632	keys_fit_inline = false;
554	633	if (DESC_AEAD_GIVENC_LEN + DESC_JOB_IO_LEN +
555		- ctx->split_key_pad_len + ctx->enckeylen <=
	634	+ ctx->split_key_pad_len + ctx->enckeylen +
	635	+ (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0) <=
556	636	CAAM_DESC_BYTES_MAX)
557	637	keys_fit_inline = true;
558	638
559	639	/* aead_givencrypt shared descriptor */
560	640	desc = ctx->sh_desc_givenc;
561	641
562		- init_sh_desc_key_aead(desc, ctx, keys_fit_inline);
	642	+ /* Note: Context registers are saved. */
	643	+ init_sh_desc_key_aead(desc, ctx, keys_fit_inline, is_rfc3686);
563	644
564	645	/* Generate IV */
565	646	geniv = NFIFOENTRY_STYPE_PAD \| NFIFOENTRY_DEST_DECO \|
566	647
...	...	@@ -568,13 +649,16 @@
568	649	append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB \|
569	650	LDST_SRCDST_WORD_INFO_FIFO \| LDST_IMM);
570	651	append_cmd(desc, CMD_LOAD \| DISABLE_AUTO_INFO_FIFO);
571		- append_move(desc, MOVE_SRC_INFIFO \|
572		- MOVE_DEST_CLASS1CTX \| (tfm->ivsize << MOVE_LEN_SHIFT));
	652	+ append_move(desc, MOVE_WAITCOMP \|
	653	+ MOVE_SRC_INFIFO \| MOVE_DEST_CLASS1CTX \|
	654	+ (ctx1_iv_off << MOVE_OFFSET_SHIFT) \|
	655	+ (tfm->ivsize << MOVE_LEN_SHIFT));
573	656	append_cmd(desc, CMD_LOAD \| ENABLE_AUTO_INFO_FIFO);
574	657
575	658	/* Copy IV to class 1 context */
576		- append_move(desc, MOVE_SRC_CLASS1CTX \|
577		- MOVE_DEST_OUTFIFO \| (tfm->ivsize << MOVE_LEN_SHIFT));
	659	+ append_move(desc, MOVE_SRC_CLASS1CTX \| MOVE_DEST_OUTFIFO \|
	660	+ (ctx1_iv_off << MOVE_OFFSET_SHIFT) \|
	661	+ (tfm->ivsize << MOVE_LEN_SHIFT));
578	662
579	663	/* Return to encryption */
580	664	append_operation(desc, ctx->class2_alg_type \|
...	...	@@ -590,7 +674,7 @@
590	674	append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 \| FIFOLD_TYPE_MSG \|
591	675	KEY_VLF);
592	676
593		- /* Copy iv from class 1 ctx to class 2 fifo*/
	677	+ /* Copy iv from outfifo to class 2 fifo */
594	678	moveiv = NFIFOENTRY_STYPE_OFIFO \| NFIFOENTRY_DEST_CLASS2 \|
595	679	NFIFOENTRY_DTYPE_MSG \| (tfm->ivsize << NFIFOENTRY_DLEN_SHIFT);
596	680	append_load_imm_u32(desc, moveiv, LDST_CLASS_IND_CCB \|
...	...	@@ -598,6 +682,14 @@
598	682	append_load_imm_u32(desc, tfm->ivsize, LDST_CLASS_2_CCB \|
599	683	LDST_SRCDST_WORD_DATASZ_REG \| LDST_IMM);
600	684
	685	+ /* Load Counter into CONTEXT1 reg */
	686	+ if (is_rfc3686)
	687	+ append_load_imm_u32(desc, be32_to_cpu(1), LDST_IMM \|
	688	+ LDST_CLASS_1_CCB \|
	689	+ LDST_SRCDST_BYTE_CONTEXT \|
	690	+ ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
	691	+ LDST_OFFSET_SHIFT));
	692	+
601	693	/* Class 1 operation */
602	694	append_operation(desc, ctx->class1_alg_type \|
603	695	OP_ALG_AS_INITFINAL \| OP_ALG_ENCRYPT);
...	...	@@ -3498,6 +3590,124 @@
3498	3590	.maxauthsize = SHA512_DIGEST_SIZE,
3499	3591	},
3500	3592	.class1_alg_type = OP_ALG_ALGSEL_DES \| OP_ALG_AAI_CBC,
	3593	+ .class2_alg_type = OP_ALG_ALGSEL_SHA512 \|
	3594	+ OP_ALG_AAI_HMAC_PRECOMP,
	3595	+ .alg_op = OP_ALG_ALGSEL_SHA512 \| OP_ALG_AAI_HMAC,
	3596	+ },
	3597	+ {
	3598	+ .name = "authenc(hmac(md5),rfc3686(ctr(aes)))",
	3599	+ .driver_name = "authenc-hmac-md5-rfc3686-ctr-aes-caam",
	3600	+ .blocksize = 1,
	3601	+ .type = CRYPTO_ALG_TYPE_AEAD,
	3602	+ .template_aead = {
	3603	+ .setkey = aead_setkey,
	3604	+ .setauthsize = aead_setauthsize,
	3605	+ .encrypt = aead_encrypt,
	3606	+ .decrypt = aead_decrypt,
	3607	+ .givencrypt = aead_givencrypt,
	3608	+ .geniv = "<built-in>",
	3609	+ .ivsize = CTR_RFC3686_IV_SIZE,
	3610	+ .maxauthsize = MD5_DIGEST_SIZE,
	3611	+ },
	3612	+ .class1_alg_type = OP_ALG_ALGSEL_AES \| OP_ALG_AAI_CTR_MOD128,
	3613	+ .class2_alg_type = OP_ALG_ALGSEL_MD5 \| OP_ALG_AAI_HMAC_PRECOMP,
	3614	+ .alg_op = OP_ALG_ALGSEL_MD5 \| OP_ALG_AAI_HMAC,
	3615	+ },
	3616	+ {
	3617	+ .name = "authenc(hmac(sha1),rfc3686(ctr(aes)))",
	3618	+ .driver_name = "authenc-hmac-sha1-rfc3686-ctr-aes-caam",
	3619	+ .blocksize = 1,
	3620	+ .type = CRYPTO_ALG_TYPE_AEAD,
	3621	+ .template_aead = {
	3622	+ .setkey = aead_setkey,
	3623	+ .setauthsize = aead_setauthsize,
	3624	+ .encrypt = aead_encrypt,
	3625	+ .decrypt = aead_decrypt,
	3626	+ .givencrypt = aead_givencrypt,
	3627	+ .geniv = "<built-in>",
	3628	+ .ivsize = CTR_RFC3686_IV_SIZE,
	3629	+ .maxauthsize = SHA1_DIGEST_SIZE,
	3630	+ },
	3631	+ .class1_alg_type = OP_ALG_ALGSEL_AES \| OP_ALG_AAI_CTR_MOD128,
	3632	+ .class2_alg_type = OP_ALG_ALGSEL_SHA1 \| OP_ALG_AAI_HMAC_PRECOMP,
	3633	+ .alg_op = OP_ALG_ALGSEL_SHA1 \| OP_ALG_AAI_HMAC,
	3634	+ },
	3635	+ {
	3636	+ .name = "authenc(hmac(sha224),rfc3686(ctr(aes)))",
	3637	+ .driver_name = "authenc-hmac-sha224-rfc3686-ctr-aes-caam",
	3638	+ .blocksize = 1,
	3639	+ .type = CRYPTO_ALG_TYPE_AEAD,
	3640	+ .template_aead = {
	3641	+ .setkey = aead_setkey,
	3642	+ .setauthsize = aead_setauthsize,
	3643	+ .encrypt = aead_encrypt,
	3644	+ .decrypt = aead_decrypt,
	3645	+ .givencrypt = aead_givencrypt,
	3646	+ .geniv = "<built-in>",
	3647	+ .ivsize = CTR_RFC3686_IV_SIZE,
	3648	+ .maxauthsize = SHA224_DIGEST_SIZE,
	3649	+ },
	3650	+ .class1_alg_type = OP_ALG_ALGSEL_AES \| OP_ALG_AAI_CTR_MOD128,
	3651	+ .class2_alg_type = OP_ALG_ALGSEL_SHA224 \|
	3652	+ OP_ALG_AAI_HMAC_PRECOMP,
	3653	+ .alg_op = OP_ALG_ALGSEL_SHA224 \| OP_ALG_AAI_HMAC,
	3654	+ },
	3655	+ {
	3656	+ .name = "authenc(hmac(sha256),rfc3686(ctr(aes)))",
	3657	+ .driver_name = "authenc-hmac-sha256-rfc3686-ctr-aes-caam",
	3658	+ .blocksize = 1,
	3659	+ .type = CRYPTO_ALG_TYPE_AEAD,
	3660	+ .template_aead = {
	3661	+ .setkey = aead_setkey,
	3662	+ .setauthsize = aead_setauthsize,
	3663	+ .encrypt = aead_encrypt,
	3664	+ .decrypt = aead_decrypt,
	3665	+ .givencrypt = aead_givencrypt,
	3666	+ .geniv = "<built-in>",
	3667	+ .ivsize = CTR_RFC3686_IV_SIZE,
	3668	+ .maxauthsize = SHA256_DIGEST_SIZE,
	3669	+ },
	3670	+ .class1_alg_type = OP_ALG_ALGSEL_AES \| OP_ALG_AAI_CTR_MOD128,
	3671	+ .class2_alg_type = OP_ALG_ALGSEL_SHA256 \|
	3672	+ OP_ALG_AAI_HMAC_PRECOMP,
	3673	+ .alg_op = OP_ALG_ALGSEL_SHA256 \| OP_ALG_AAI_HMAC,
	3674	+ },
	3675	+ {
	3676	+ .name = "authenc(hmac(sha384),rfc3686(ctr(aes)))",
	3677	+ .driver_name = "authenc-hmac-sha384-rfc3686-ctr-aes-caam",
	3678	+ .blocksize = 1,
	3679	+ .type = CRYPTO_ALG_TYPE_AEAD,
	3680	+ .template_aead = {
	3681	+ .setkey = aead_setkey,
	3682	+ .setauthsize = aead_setauthsize,
	3683	+ .encrypt = aead_encrypt,
	3684	+ .decrypt = aead_decrypt,
	3685	+ .givencrypt = aead_givencrypt,
	3686	+ .geniv = "<built-in>",
	3687	+ .ivsize = CTR_RFC3686_IV_SIZE,
	3688	+ .maxauthsize = SHA384_DIGEST_SIZE,
	3689	+ },
	3690	+ .class1_alg_type = OP_ALG_ALGSEL_AES \| OP_ALG_AAI_CTR_MOD128,
	3691	+ .class2_alg_type = OP_ALG_ALGSEL_SHA384 \|
	3692	+ OP_ALG_AAI_HMAC_PRECOMP,
	3693	+ .alg_op = OP_ALG_ALGSEL_SHA384 \| OP_ALG_AAI_HMAC,
	3694	+ },
	3695	+ {
	3696	+ .name = "authenc(hmac(sha512),rfc3686(ctr(aes)))",
	3697	+ .driver_name = "authenc-hmac-sha512-rfc3686-ctr-aes-caam",
	3698	+ .blocksize = 1,
	3699	+ .type = CRYPTO_ALG_TYPE_AEAD,
	3700	+ .template_aead = {
	3701	+ .setkey = aead_setkey,
	3702	+ .setauthsize = aead_setauthsize,
	3703	+ .encrypt = aead_encrypt,
	3704	+ .decrypt = aead_decrypt,
	3705	+ .givencrypt = aead_givencrypt,
	3706	+ .geniv = "<built-in>",
	3707	+ .ivsize = CTR_RFC3686_IV_SIZE,
	3708	+ .maxauthsize = SHA512_DIGEST_SIZE,
	3709	+ },
	3710	+ .class1_alg_type = OP_ALG_ALGSEL_AES \| OP_ALG_AAI_CTR_MOD128,
3501	3711	.class2_alg_type = OP_ALG_ALGSEL_SHA512 \|
3502	3712	OP_ALG_AAI_HMAC_PRECOMP,
3503	3713	.alg_op = OP_ALG_ALGSEL_SHA512 \| OP_ALG_AAI_HMAC,