Eric Lee / smarc-ti-linux-kernel | Embedian Git Server

Commit acc5ccb9fe1c1d3840d49e181ae30b924cfc28b5

Authored by Kees Cook 2014-11-25 08:32:38 +0800

Committed by Greg Kroah-Hartman 2015-01-30 09:40:46 +0800

crypto: include crypto- module prefix in template

commit 4943ba16bbc2db05115707b3ff7b4874e9e3c560 upstream.

This adds the module loading prefix "crypto-" to the template lookup
as well.

For example, attempting to load 'vfat(blowfish)' via AF_ALG now correctly
includes the "crypto-" prefix at every level, correctly rejecting "vfat":

	net-pf-38
	algif-hash
	crypto-vfat(blowfish)
	crypto-vfat(blowfish)-all
	crypto-vfat

Reported-by: Mathias Krause <minipli@googlemail.com>
Signed-off-by: Kees Cook <keescook@chromium.org>
Acked-by: Mathias Krause <minipli@googlemail.com>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>

Showing 23 changed files with 26 additions and 2 deletions Inline Diff

arch/x86/crypto/fpu.c

Diff comments View file @ acc5ccb

 /*
  * FPU: Wrapper for blkcipher touching fpu
  *
  * Copyright (c) Intel Corp.
  *   Author: Huang Ying <ying.huang@intel.com>
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the Free
  * Software Foundation; either version 2 of the License, or (at your option)
  * any later version.
  *
  */
 #include <crypto/algapi.h>
 #include <linux/err.h>
 #include <linux/init.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/slab.h>
+#include <linux/crypto.h>
 #include <asm/i387.h>
 struct crypto_fpu_ctx {
 	struct crypto_blkcipher *child;
 };
 static int crypto_fpu_setkey(struct crypto_tfm *parent, const u8 *key,
 			     unsigned int keylen)
 {
 	struct crypto_fpu_ctx *ctx = crypto_tfm_ctx(parent);
 	struct crypto_blkcipher *child = ctx->child;
 	int err;
 	crypto_blkcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK);
 	crypto_blkcipher_set_flags(child, crypto_tfm_get_flags(parent) &
 				   CRYPTO_TFM_REQ_MASK);
 	err = crypto_blkcipher_setkey(child, key, keylen);
 	crypto_tfm_set_flags(parent, crypto_blkcipher_get_flags(child) &
 				     CRYPTO_TFM_RES_MASK);
 	return err;
 }
 static int crypto_fpu_encrypt(struct blkcipher_desc *desc_in,
 			      struct scatterlist *dst, struct scatterlist *src,
 			      unsigned int nbytes)
 {
 	int err;
 	struct crypto_fpu_ctx *ctx = crypto_blkcipher_ctx(desc_in->tfm);
 	struct crypto_blkcipher *child = ctx->child;
 	struct blkcipher_desc desc = {
 		.tfm = child,
 		.info = desc_in->info,
 		.flags = desc_in->flags & ~CRYPTO_TFM_REQ_MAY_SLEEP,
 	};
 	kernel_fpu_begin();
 	err = crypto_blkcipher_crt(desc.tfm)->encrypt(&desc, dst, src, nbytes);
 	kernel_fpu_end();
 	return err;
 }
 static int crypto_fpu_decrypt(struct blkcipher_desc *desc_in,
 			      struct scatterlist *dst, struct scatterlist *src,
 			      unsigned int nbytes)
 {
 	int err;
 	struct crypto_fpu_ctx *ctx = crypto_blkcipher_ctx(desc_in->tfm);
 	struct crypto_blkcipher *child = ctx->child;
 	struct blkcipher_desc desc = {
 		.tfm = child,
 		.info = desc_in->info,
 		.flags = desc_in->flags & ~CRYPTO_TFM_REQ_MAY_SLEEP,
 	};
 	kernel_fpu_begin();
 	err = crypto_blkcipher_crt(desc.tfm)->decrypt(&desc, dst, src, nbytes);
 	kernel_fpu_end();
 	return err;
 }
 static int crypto_fpu_init_tfm(struct crypto_tfm *tfm)
 {
 	struct crypto_instance *inst = crypto_tfm_alg_instance(tfm);
 	struct crypto_spawn *spawn = crypto_instance_ctx(inst);
 	struct crypto_fpu_ctx *ctx = crypto_tfm_ctx(tfm);
 	struct crypto_blkcipher *cipher;
 	cipher = crypto_spawn_blkcipher(spawn);
 	if (IS_ERR(cipher))
 		return PTR_ERR(cipher);
 	ctx->child = cipher;
 	return 0;
 }
 static void crypto_fpu_exit_tfm(struct crypto_tfm *tfm)
 {
 	struct crypto_fpu_ctx *ctx = crypto_tfm_ctx(tfm);
 	crypto_free_blkcipher(ctx->child);
 }
 static struct crypto_instance *crypto_fpu_alloc(struct rtattr **tb)
 {
 	struct crypto_instance *inst;
 	struct crypto_alg *alg;
 	int err;
 	err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_BLKCIPHER);
 	if (err)
 		return ERR_PTR(err);
 	alg = crypto_get_attr_alg(tb, CRYPTO_ALG_TYPE_BLKCIPHER,
 				  CRYPTO_ALG_TYPE_MASK);
 	if (IS_ERR(alg))
 		return ERR_CAST(alg);
 	inst = crypto_alloc_instance("fpu", alg);
 	if (IS_ERR(inst))
 		goto out_put_alg;
 	inst->alg.cra_flags = alg->cra_flags;
 	inst->alg.cra_priority = alg->cra_priority;
 	inst->alg.cra_blocksize = alg->cra_blocksize;
 	inst->alg.cra_alignmask = alg->cra_alignmask;
 	inst->alg.cra_type = alg->cra_type;
 	inst->alg.cra_blkcipher.ivsize = alg->cra_blkcipher.ivsize;
 	inst->alg.cra_blkcipher.min_keysize = alg->cra_blkcipher.min_keysize;
 	inst->alg.cra_blkcipher.max_keysize = alg->cra_blkcipher.max_keysize;
 	inst->alg.cra_ctxsize = sizeof(struct crypto_fpu_ctx);
 	inst->alg.cra_init = crypto_fpu_init_tfm;
 	inst->alg.cra_exit = crypto_fpu_exit_tfm;
 	inst->alg.cra_blkcipher.setkey = crypto_fpu_setkey;
 	inst->alg.cra_blkcipher.encrypt = crypto_fpu_encrypt;
 	inst->alg.cra_blkcipher.decrypt = crypto_fpu_decrypt;
 out_put_alg:
 	crypto_mod_put(alg);
 	return inst;
 }
 static void crypto_fpu_free(struct crypto_instance *inst)
 {
 	crypto_drop_spawn(crypto_instance_ctx(inst));
 	kfree(inst);
 }
 static struct crypto_template crypto_fpu_tmpl = {
 	.name = "fpu",
 	.alloc = crypto_fpu_alloc,
 	.free = crypto_fpu_free,
 	.module = THIS_MODULE,
 };
 int __init crypto_fpu_init(void)
 {
 	return crypto_register_template(&crypto_fpu_tmpl);
 }
 void __exit crypto_fpu_exit(void)
 {
 	crypto_unregister_template(&crypto_fpu_tmpl);
 }
+MODULE_ALIAS_CRYPTO("fpu");

crypto/algapi.c

Diff comments View file @ acc5ccb

1	/*	1	/*
2	* Cryptographic API for algorithms (i.e., low-level API).	2	* Cryptographic API for algorithms (i.e., low-level API).
3	*	3	*
4	* Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au>	4	* Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au>
5	*	5	*
6	* This program is free software; you can redistribute it and/or modify it	6	* This program is free software; you can redistribute it and/or modify it
7	* under the terms of the GNU General Public License as published by the Free	7	* under the terms of the GNU General Public License as published by the Free
8	* Software Foundation; either version 2 of the License, or (at your option)	8	* Software Foundation; either version 2 of the License, or (at your option)
9	* any later version.	9	* any later version.
10	*	10	*
11	*/	11	*/
12		12
13	#include <linux/err.h>	13	#include <linux/err.h>
14	#include <linux/errno.h>	14	#include <linux/errno.h>
15	#include <linux/init.h>	15	#include <linux/init.h>
16	#include <linux/kernel.h>	16	#include <linux/kernel.h>
17	#include <linux/list.h>	17	#include <linux/list.h>
18	#include <linux/module.h>	18	#include <linux/module.h>
19	#include <linux/rtnetlink.h>	19	#include <linux/rtnetlink.h>
20	#include <linux/slab.h>	20	#include <linux/slab.h>
21	#include <linux/string.h>	21	#include <linux/string.h>
22		22
23	#include "internal.h"	23	#include "internal.h"
24		24
25	static LIST_HEAD(crypto_template_list);	25	static LIST_HEAD(crypto_template_list);
26		26
27	static inline int crypto_set_driver_name(struct crypto_alg *alg)	27	static inline int crypto_set_driver_name(struct crypto_alg *alg)
28	{	28	{
29	static const char suffix[] = "-generic";	29	static const char suffix[] = "-generic";
30	char *driver_name = alg->cra_driver_name;	30	char *driver_name = alg->cra_driver_name;
31	int len;	31	int len;
32		32
33	if (*driver_name)	33	if (*driver_name)
34	return 0;	34	return 0;
35		35
36	len = strlcpy(driver_name, alg->cra_name, CRYPTO_MAX_ALG_NAME);	36	len = strlcpy(driver_name, alg->cra_name, CRYPTO_MAX_ALG_NAME);
37	if (len + sizeof(suffix) > CRYPTO_MAX_ALG_NAME)	37	if (len + sizeof(suffix) > CRYPTO_MAX_ALG_NAME)
38	return -ENAMETOOLONG;	38	return -ENAMETOOLONG;
39		39
40	memcpy(driver_name + len, suffix, sizeof(suffix));	40	memcpy(driver_name + len, suffix, sizeof(suffix));
41	return 0;	41	return 0;
42	}	42	}
43		43
44	static inline void crypto_check_module_sig(struct module *mod)	44	static inline void crypto_check_module_sig(struct module *mod)
45	{	45	{
46	#ifdef CONFIG_CRYPTO_FIPS	46	#ifdef CONFIG_CRYPTO_FIPS
47	if (fips_enabled && mod && !mod->sig_ok)	47	if (fips_enabled && mod && !mod->sig_ok)
48	panic("Module %s signature verification failed in FIPS mode\n",	48	panic("Module %s signature verification failed in FIPS mode\n",
49	mod->name);	49	mod->name);
50	#endif	50	#endif
51	return;	51	return;
52	}	52	}
53		53
54	static int crypto_check_alg(struct crypto_alg *alg)	54	static int crypto_check_alg(struct crypto_alg *alg)
55	{	55	{
56	crypto_check_module_sig(alg->cra_module);	56	crypto_check_module_sig(alg->cra_module);
57		57
58	if (alg->cra_alignmask & (alg->cra_alignmask + 1))	58	if (alg->cra_alignmask & (alg->cra_alignmask + 1))
59	return -EINVAL;	59	return -EINVAL;
60		60
61	if (alg->cra_blocksize > PAGE_SIZE / 8)	61	if (alg->cra_blocksize > PAGE_SIZE / 8)
62	return -EINVAL;	62	return -EINVAL;
63		63
64	if (alg->cra_priority < 0)	64	if (alg->cra_priority < 0)
65	return -EINVAL;	65	return -EINVAL;
66		66
67	return crypto_set_driver_name(alg);	67	return crypto_set_driver_name(alg);
68	}	68	}
69		69
70	static void crypto_destroy_instance(struct crypto_alg *alg)	70	static void crypto_destroy_instance(struct crypto_alg *alg)
71	{	71	{
72	struct crypto_instance inst = (void )alg;	72	struct crypto_instance inst = (void )alg;
73	struct crypto_template *tmpl = inst->tmpl;	73	struct crypto_template *tmpl = inst->tmpl;
74		74
75	tmpl->free(inst);	75	tmpl->free(inst);
76	crypto_tmpl_put(tmpl);	76	crypto_tmpl_put(tmpl);
77	}	77	}
78		78
79	static struct list_head crypto_more_spawns(struct crypto_alg alg,	79	static struct list_head crypto_more_spawns(struct crypto_alg alg,
80	struct list_head *stack,	80	struct list_head *stack,
81	struct list_head *top,	81	struct list_head *top,
82	struct list_head *secondary_spawns)	82	struct list_head *secondary_spawns)
83	{	83	{
84	struct crypto_spawn spawn, n;	84	struct crypto_spawn spawn, n;
85		85
86	if (list_empty(stack))	86	if (list_empty(stack))
87	return NULL;	87	return NULL;
88		88
89	spawn = list_first_entry(stack, struct crypto_spawn, list);	89	spawn = list_first_entry(stack, struct crypto_spawn, list);
90	n = list_entry(spawn->list.next, struct crypto_spawn, list);	90	n = list_entry(spawn->list.next, struct crypto_spawn, list);
91		91
92	if (spawn->alg && &n->list != stack && !n->alg)	92	if (spawn->alg && &n->list != stack && !n->alg)
93	n->alg = (n->list.next == stack) ? alg :	93	n->alg = (n->list.next == stack) ? alg :
94	&list_entry(n->list.next, struct crypto_spawn,	94	&list_entry(n->list.next, struct crypto_spawn,
95	list)->inst->alg;	95	list)->inst->alg;
96		96
97	list_move(&spawn->list, secondary_spawns);	97	list_move(&spawn->list, secondary_spawns);
98		98
99	return &n->list == stack ? top : &n->inst->alg.cra_users;	99	return &n->list == stack ? top : &n->inst->alg.cra_users;
100	}	100	}
101		101
102	static void crypto_remove_spawn(struct crypto_spawn *spawn,	102	static void crypto_remove_spawn(struct crypto_spawn *spawn,
103	struct list_head *list)	103	struct list_head *list)
104	{	104	{
105	struct crypto_instance *inst = spawn->inst;	105	struct crypto_instance *inst = spawn->inst;
106	struct crypto_template *tmpl = inst->tmpl;	106	struct crypto_template *tmpl = inst->tmpl;
107		107
108	if (crypto_is_dead(&inst->alg))	108	if (crypto_is_dead(&inst->alg))
109	return;	109	return;
110		110
111	inst->alg.cra_flags \|= CRYPTO_ALG_DEAD;	111	inst->alg.cra_flags \|= CRYPTO_ALG_DEAD;
112	if (hlist_unhashed(&inst->list))	112	if (hlist_unhashed(&inst->list))
113	return;	113	return;
114		114
115	if (!tmpl \|\| !crypto_tmpl_get(tmpl))	115	if (!tmpl \|\| !crypto_tmpl_get(tmpl))
116	return;	116	return;
117		117
118	crypto_notify(CRYPTO_MSG_ALG_UNREGISTER, &inst->alg);	118	crypto_notify(CRYPTO_MSG_ALG_UNREGISTER, &inst->alg);
119	list_move(&inst->alg.cra_list, list);	119	list_move(&inst->alg.cra_list, list);
120	hlist_del(&inst->list);	120	hlist_del(&inst->list);
121	inst->alg.cra_destroy = crypto_destroy_instance;	121	inst->alg.cra_destroy = crypto_destroy_instance;
122		122
123	BUG_ON(!list_empty(&inst->alg.cra_users));	123	BUG_ON(!list_empty(&inst->alg.cra_users));
124	}	124	}
125		125
126	void crypto_remove_spawns(struct crypto_alg alg, struct list_head list,	126	void crypto_remove_spawns(struct crypto_alg alg, struct list_head list,
127	struct crypto_alg *nalg)	127	struct crypto_alg *nalg)
128	{	128	{
129	u32 new_type = (nalg ?: alg)->cra_flags;	129	u32 new_type = (nalg ?: alg)->cra_flags;
130	struct crypto_spawn spawn, n;	130	struct crypto_spawn spawn, n;
131	LIST_HEAD(secondary_spawns);	131	LIST_HEAD(secondary_spawns);
132	struct list_head *spawns;	132	struct list_head *spawns;
133	LIST_HEAD(stack);	133	LIST_HEAD(stack);
134	LIST_HEAD(top);	134	LIST_HEAD(top);
135		135
136	spawns = &alg->cra_users;	136	spawns = &alg->cra_users;
137	list_for_each_entry_safe(spawn, n, spawns, list) {	137	list_for_each_entry_safe(spawn, n, spawns, list) {
138	if ((spawn->alg->cra_flags ^ new_type) & spawn->mask)	138	if ((spawn->alg->cra_flags ^ new_type) & spawn->mask)
139	continue;	139	continue;
140		140
141	list_move(&spawn->list, &top);	141	list_move(&spawn->list, &top);
142	}	142	}
143		143
144	spawns = &top;	144	spawns = &top;
145	do {	145	do {
146	while (!list_empty(spawns)) {	146	while (!list_empty(spawns)) {
147	struct crypto_instance *inst;	147	struct crypto_instance *inst;
148		148
149	spawn = list_first_entry(spawns, struct crypto_spawn,	149	spawn = list_first_entry(spawns, struct crypto_spawn,
150	list);	150	list);
151	inst = spawn->inst;	151	inst = spawn->inst;
152		152
153	BUG_ON(&inst->alg == alg);	153	BUG_ON(&inst->alg == alg);
154		154
155	list_move(&spawn->list, &stack);	155	list_move(&spawn->list, &stack);
156		156
157	if (&inst->alg == nalg)	157	if (&inst->alg == nalg)
158	break;	158	break;
159		159
160	spawn->alg = NULL;	160	spawn->alg = NULL;
161	spawns = &inst->alg.cra_users;	161	spawns = &inst->alg.cra_users;
162	}	162	}
163	} while ((spawns = crypto_more_spawns(alg, &stack, &top,	163	} while ((spawns = crypto_more_spawns(alg, &stack, &top,
164	&secondary_spawns)));	164	&secondary_spawns)));
165		165
166	list_for_each_entry_safe(spawn, n, &secondary_spawns, list) {	166	list_for_each_entry_safe(spawn, n, &secondary_spawns, list) {
167	if (spawn->alg)	167	if (spawn->alg)
168	list_move(&spawn->list, &spawn->alg->cra_users);	168	list_move(&spawn->list, &spawn->alg->cra_users);
169	else	169	else
170	crypto_remove_spawn(spawn, list);	170	crypto_remove_spawn(spawn, list);
171	}	171	}
172	}	172	}
173	EXPORT_SYMBOL_GPL(crypto_remove_spawns);	173	EXPORT_SYMBOL_GPL(crypto_remove_spawns);
174		174
175	static struct crypto_larval __crypto_register_alg(struct crypto_alg alg)	175	static struct crypto_larval __crypto_register_alg(struct crypto_alg alg)
176	{	176	{
177	struct crypto_alg *q;	177	struct crypto_alg *q;
178	struct crypto_larval *larval;	178	struct crypto_larval *larval;
179	int ret = -EAGAIN;	179	int ret = -EAGAIN;
180		180
181	if (crypto_is_dead(alg))	181	if (crypto_is_dead(alg))
182	goto err;	182	goto err;
183		183
184	INIT_LIST_HEAD(&alg->cra_users);	184	INIT_LIST_HEAD(&alg->cra_users);
185		185
186	/* No cheating! */	186	/* No cheating! */
187	alg->cra_flags &= ~CRYPTO_ALG_TESTED;	187	alg->cra_flags &= ~CRYPTO_ALG_TESTED;
188		188
189	ret = -EEXIST;	189	ret = -EEXIST;
190		190
191	atomic_set(&alg->cra_refcnt, 1);	191	atomic_set(&alg->cra_refcnt, 1);
192	list_for_each_entry(q, &crypto_alg_list, cra_list) {	192	list_for_each_entry(q, &crypto_alg_list, cra_list) {
193	if (q == alg)	193	if (q == alg)
194	goto err;	194	goto err;
195		195
196	if (crypto_is_moribund(q))	196	if (crypto_is_moribund(q))
197	continue;	197	continue;
198		198
199	if (crypto_is_larval(q)) {	199	if (crypto_is_larval(q)) {
200	if (!strcmp(alg->cra_driver_name, q->cra_driver_name))	200	if (!strcmp(alg->cra_driver_name, q->cra_driver_name))
201	goto err;	201	goto err;
202	continue;	202	continue;
203	}	203	}
204		204
205	if (!strcmp(q->cra_driver_name, alg->cra_name) \|\|	205	if (!strcmp(q->cra_driver_name, alg->cra_name) \|\|
206	!strcmp(q->cra_name, alg->cra_driver_name))	206	!strcmp(q->cra_name, alg->cra_driver_name))
207	goto err;	207	goto err;
208	}	208	}
209		209
210	larval = crypto_larval_alloc(alg->cra_name,	210	larval = crypto_larval_alloc(alg->cra_name,
211	alg->cra_flags \| CRYPTO_ALG_TESTED, 0);	211	alg->cra_flags \| CRYPTO_ALG_TESTED, 0);
212	if (IS_ERR(larval))	212	if (IS_ERR(larval))
213	goto out;	213	goto out;
214		214
215	ret = -ENOENT;	215	ret = -ENOENT;
216	larval->adult = crypto_mod_get(alg);	216	larval->adult = crypto_mod_get(alg);
217	if (!larval->adult)	217	if (!larval->adult)
218	goto free_larval;	218	goto free_larval;
219		219
220	atomic_set(&larval->alg.cra_refcnt, 1);	220	atomic_set(&larval->alg.cra_refcnt, 1);
221	memcpy(larval->alg.cra_driver_name, alg->cra_driver_name,	221	memcpy(larval->alg.cra_driver_name, alg->cra_driver_name,
222	CRYPTO_MAX_ALG_NAME);	222	CRYPTO_MAX_ALG_NAME);
223	larval->alg.cra_priority = alg->cra_priority;	223	larval->alg.cra_priority = alg->cra_priority;
224		224
225	list_add(&alg->cra_list, &crypto_alg_list);	225	list_add(&alg->cra_list, &crypto_alg_list);
226	list_add(&larval->alg.cra_list, &crypto_alg_list);	226	list_add(&larval->alg.cra_list, &crypto_alg_list);
227		227
228	out:	228	out:
229	return larval;	229	return larval;
230		230
231	free_larval:	231	free_larval:
232	kfree(larval);	232	kfree(larval);
233	err:	233	err:
234	larval = ERR_PTR(ret);	234	larval = ERR_PTR(ret);
235	goto out;	235	goto out;
236	}	236	}
237		237
238	void crypto_alg_tested(const char *name, int err)	238	void crypto_alg_tested(const char *name, int err)
239	{	239	{
240	struct crypto_larval *test;	240	struct crypto_larval *test;
241	struct crypto_alg *alg;	241	struct crypto_alg *alg;
242	struct crypto_alg *q;	242	struct crypto_alg *q;
243	LIST_HEAD(list);	243	LIST_HEAD(list);
244		244
245	down_write(&crypto_alg_sem);	245	down_write(&crypto_alg_sem);
246	list_for_each_entry(q, &crypto_alg_list, cra_list) {	246	list_for_each_entry(q, &crypto_alg_list, cra_list) {
247	if (crypto_is_moribund(q) \|\| !crypto_is_larval(q))	247	if (crypto_is_moribund(q) \|\| !crypto_is_larval(q))
248	continue;	248	continue;
249		249
250	test = (struct crypto_larval *)q;	250	test = (struct crypto_larval *)q;
251		251
252	if (!strcmp(q->cra_driver_name, name))	252	if (!strcmp(q->cra_driver_name, name))
253	goto found;	253	goto found;
254	}	254	}
255		255
256	printk(KERN_ERR "alg: Unexpected test result for %s: %d\n", name, err);	256	printk(KERN_ERR "alg: Unexpected test result for %s: %d\n", name, err);
257	goto unlock;	257	goto unlock;
258		258
259	found:	259	found:
260	q->cra_flags \|= CRYPTO_ALG_DEAD;	260	q->cra_flags \|= CRYPTO_ALG_DEAD;
261	alg = test->adult;	261	alg = test->adult;
262	if (err \|\| list_empty(&alg->cra_list))	262	if (err \|\| list_empty(&alg->cra_list))
263	goto complete;	263	goto complete;
264		264
265	alg->cra_flags \|= CRYPTO_ALG_TESTED;	265	alg->cra_flags \|= CRYPTO_ALG_TESTED;
266		266
267	list_for_each_entry(q, &crypto_alg_list, cra_list) {	267	list_for_each_entry(q, &crypto_alg_list, cra_list) {
268	if (q == alg)	268	if (q == alg)
269	continue;	269	continue;
270		270
271	if (crypto_is_moribund(q))	271	if (crypto_is_moribund(q))
272	continue;	272	continue;
273		273
274	if (crypto_is_larval(q)) {	274	if (crypto_is_larval(q)) {
275	struct crypto_larval larval = (void )q;	275	struct crypto_larval larval = (void )q;
276		276
277	/*	277	/*
278	* Check to see if either our generic name or	278	* Check to see if either our generic name or
279	* specific name can satisfy the name requested	279	* specific name can satisfy the name requested
280	* by the larval entry q.	280	* by the larval entry q.
281	*/	281	*/
282	if (strcmp(alg->cra_name, q->cra_name) &&	282	if (strcmp(alg->cra_name, q->cra_name) &&
283	strcmp(alg->cra_driver_name, q->cra_name))	283	strcmp(alg->cra_driver_name, q->cra_name))
284	continue;	284	continue;
285		285
286	if (larval->adult)	286	if (larval->adult)
287	continue;	287	continue;
288	if ((q->cra_flags ^ alg->cra_flags) & larval->mask)	288	if ((q->cra_flags ^ alg->cra_flags) & larval->mask)
289	continue;	289	continue;
290	if (!crypto_mod_get(alg))	290	if (!crypto_mod_get(alg))
291	continue;	291	continue;
292		292
293	larval->adult = alg;	293	larval->adult = alg;
294	continue;	294	continue;
295	}	295	}
296		296
297	if (strcmp(alg->cra_name, q->cra_name))	297	if (strcmp(alg->cra_name, q->cra_name))
298	continue;	298	continue;
299		299
300	if (strcmp(alg->cra_driver_name, q->cra_driver_name) &&	300	if (strcmp(alg->cra_driver_name, q->cra_driver_name) &&
301	q->cra_priority > alg->cra_priority)	301	q->cra_priority > alg->cra_priority)
302	continue;	302	continue;
303		303
304	crypto_remove_spawns(q, &list, alg);	304	crypto_remove_spawns(q, &list, alg);
305	}	305	}
306		306
307	complete:	307	complete:
308	complete_all(&test->completion);	308	complete_all(&test->completion);
309		309
310	unlock:	310	unlock:
311	up_write(&crypto_alg_sem);	311	up_write(&crypto_alg_sem);
312		312
313	crypto_remove_final(&list);	313	crypto_remove_final(&list);
314	}	314	}
315	EXPORT_SYMBOL_GPL(crypto_alg_tested);	315	EXPORT_SYMBOL_GPL(crypto_alg_tested);
316		316
317	void crypto_remove_final(struct list_head *list)	317	void crypto_remove_final(struct list_head *list)
318	{	318	{
319	struct crypto_alg *alg;	319	struct crypto_alg *alg;
320	struct crypto_alg *n;	320	struct crypto_alg *n;
321		321
322	list_for_each_entry_safe(alg, n, list, cra_list) {	322	list_for_each_entry_safe(alg, n, list, cra_list) {
323	list_del_init(&alg->cra_list);	323	list_del_init(&alg->cra_list);
324	crypto_alg_put(alg);	324	crypto_alg_put(alg);
325	}	325	}
326	}	326	}
327	EXPORT_SYMBOL_GPL(crypto_remove_final);	327	EXPORT_SYMBOL_GPL(crypto_remove_final);
328		328
329	static void crypto_wait_for_test(struct crypto_larval *larval)	329	static void crypto_wait_for_test(struct crypto_larval *larval)
330	{	330	{
331	int err;	331	int err;
332		332
333	err = crypto_probing_notify(CRYPTO_MSG_ALG_REGISTER, larval->adult);	333	err = crypto_probing_notify(CRYPTO_MSG_ALG_REGISTER, larval->adult);
334	if (err != NOTIFY_STOP) {	334	if (err != NOTIFY_STOP) {
335	if (WARN_ON(err != NOTIFY_DONE))	335	if (WARN_ON(err != NOTIFY_DONE))
336	goto out;	336	goto out;
337	crypto_alg_tested(larval->alg.cra_driver_name, 0);	337	crypto_alg_tested(larval->alg.cra_driver_name, 0);
338	}	338	}
339		339
340	err = wait_for_completion_interruptible(&larval->completion);	340	err = wait_for_completion_interruptible(&larval->completion);
341	WARN_ON(err);	341	WARN_ON(err);
342		342
343	out:	343	out:
344	crypto_larval_kill(&larval->alg);	344	crypto_larval_kill(&larval->alg);
345	}	345	}
346		346
347	int crypto_register_alg(struct crypto_alg *alg)	347	int crypto_register_alg(struct crypto_alg *alg)
348	{	348	{
349	struct crypto_larval *larval;	349	struct crypto_larval *larval;
350	int err;	350	int err;
351		351
352	err = crypto_check_alg(alg);	352	err = crypto_check_alg(alg);
353	if (err)	353	if (err)
354	return err;	354	return err;
355		355
356	down_write(&crypto_alg_sem);	356	down_write(&crypto_alg_sem);
357	larval = __crypto_register_alg(alg);	357	larval = __crypto_register_alg(alg);
358	up_write(&crypto_alg_sem);	358	up_write(&crypto_alg_sem);
359		359
360	if (IS_ERR(larval))	360	if (IS_ERR(larval))
361	return PTR_ERR(larval);	361	return PTR_ERR(larval);
362		362
363	crypto_wait_for_test(larval);	363	crypto_wait_for_test(larval);
364	return 0;	364	return 0;
365	}	365	}
366	EXPORT_SYMBOL_GPL(crypto_register_alg);	366	EXPORT_SYMBOL_GPL(crypto_register_alg);
367		367
368	static int crypto_remove_alg(struct crypto_alg alg, struct list_head list)	368	static int crypto_remove_alg(struct crypto_alg alg, struct list_head list)
369	{	369	{
370	if (unlikely(list_empty(&alg->cra_list)))	370	if (unlikely(list_empty(&alg->cra_list)))
371	return -ENOENT;	371	return -ENOENT;
372		372
373	alg->cra_flags \|= CRYPTO_ALG_DEAD;	373	alg->cra_flags \|= CRYPTO_ALG_DEAD;
374		374
375	crypto_notify(CRYPTO_MSG_ALG_UNREGISTER, alg);	375	crypto_notify(CRYPTO_MSG_ALG_UNREGISTER, alg);
376	list_del_init(&alg->cra_list);	376	list_del_init(&alg->cra_list);
377	crypto_remove_spawns(alg, list, NULL);	377	crypto_remove_spawns(alg, list, NULL);
378		378
379	return 0;	379	return 0;
380	}	380	}
381		381
382	int crypto_unregister_alg(struct crypto_alg *alg)	382	int crypto_unregister_alg(struct crypto_alg *alg)
383	{	383	{
384	int ret;	384	int ret;
385	LIST_HEAD(list);	385	LIST_HEAD(list);
386		386
387	down_write(&crypto_alg_sem);	387	down_write(&crypto_alg_sem);
388	ret = crypto_remove_alg(alg, &list);	388	ret = crypto_remove_alg(alg, &list);
389	up_write(&crypto_alg_sem);	389	up_write(&crypto_alg_sem);
390		390
391	if (ret)	391	if (ret)
392	return ret;	392	return ret;
393		393
394	BUG_ON(atomic_read(&alg->cra_refcnt) != 1);	394	BUG_ON(atomic_read(&alg->cra_refcnt) != 1);
395	if (alg->cra_destroy)	395	if (alg->cra_destroy)
396	alg->cra_destroy(alg);	396	alg->cra_destroy(alg);
397		397
398	crypto_remove_final(&list);	398	crypto_remove_final(&list);
399	return 0;	399	return 0;
400	}	400	}
401	EXPORT_SYMBOL_GPL(crypto_unregister_alg);	401	EXPORT_SYMBOL_GPL(crypto_unregister_alg);
402		402
403	int crypto_register_algs(struct crypto_alg *algs, int count)	403	int crypto_register_algs(struct crypto_alg *algs, int count)
404	{	404	{
405	int i, ret;	405	int i, ret;
406		406
407	for (i = 0; i < count; i++) {	407	for (i = 0; i < count; i++) {
408	ret = crypto_register_alg(&algs[i]);	408	ret = crypto_register_alg(&algs[i]);
409	if (ret)	409	if (ret)
410	goto err;	410	goto err;
411	}	411	}
412		412
413	return 0;	413	return 0;
414		414
415	err:	415	err:
416	for (--i; i >= 0; --i)	416	for (--i; i >= 0; --i)
417	crypto_unregister_alg(&algs[i]);	417	crypto_unregister_alg(&algs[i]);
418		418
419	return ret;	419	return ret;
420	}	420	}
421	EXPORT_SYMBOL_GPL(crypto_register_algs);	421	EXPORT_SYMBOL_GPL(crypto_register_algs);
422		422
423	int crypto_unregister_algs(struct crypto_alg *algs, int count)	423	int crypto_unregister_algs(struct crypto_alg *algs, int count)
424	{	424	{
425	int i, ret;	425	int i, ret;
426		426
427	for (i = 0; i < count; i++) {	427	for (i = 0; i < count; i++) {
428	ret = crypto_unregister_alg(&algs[i]);	428	ret = crypto_unregister_alg(&algs[i]);
429	if (ret)	429	if (ret)
430	pr_err("Failed to unregister %s %s: %d\n",	430	pr_err("Failed to unregister %s %s: %d\n",
431	algs[i].cra_driver_name, algs[i].cra_name, ret);	431	algs[i].cra_driver_name, algs[i].cra_name, ret);
432	}	432	}
433		433
434	return 0;	434	return 0;
435	}	435	}
436	EXPORT_SYMBOL_GPL(crypto_unregister_algs);	436	EXPORT_SYMBOL_GPL(crypto_unregister_algs);
437		437
438	int crypto_register_template(struct crypto_template *tmpl)	438	int crypto_register_template(struct crypto_template *tmpl)
439	{	439	{
440	struct crypto_template *q;	440	struct crypto_template *q;
441	int err = -EEXIST;	441	int err = -EEXIST;
442		442
443	down_write(&crypto_alg_sem);	443	down_write(&crypto_alg_sem);
444		444
445	crypto_check_module_sig(tmpl->module);	445	crypto_check_module_sig(tmpl->module);
446		446
447	list_for_each_entry(q, &crypto_template_list, list) {	447	list_for_each_entry(q, &crypto_template_list, list) {
448	if (q == tmpl)	448	if (q == tmpl)
449	goto out;	449	goto out;
450	}	450	}
451		451
452	list_add(&tmpl->list, &crypto_template_list);	452	list_add(&tmpl->list, &crypto_template_list);
453	crypto_notify(CRYPTO_MSG_TMPL_REGISTER, tmpl);	453	crypto_notify(CRYPTO_MSG_TMPL_REGISTER, tmpl);
454	err = 0;	454	err = 0;
455	out:	455	out:
456	up_write(&crypto_alg_sem);	456	up_write(&crypto_alg_sem);
457	return err;	457	return err;
458	}	458	}
459	EXPORT_SYMBOL_GPL(crypto_register_template);	459	EXPORT_SYMBOL_GPL(crypto_register_template);
460		460
461	void crypto_unregister_template(struct crypto_template *tmpl)	461	void crypto_unregister_template(struct crypto_template *tmpl)
462	{	462	{
463	struct crypto_instance *inst;	463	struct crypto_instance *inst;
464	struct hlist_node *n;	464	struct hlist_node *n;
465	struct hlist_head *list;	465	struct hlist_head *list;
466	LIST_HEAD(users);	466	LIST_HEAD(users);
467		467
468	down_write(&crypto_alg_sem);	468	down_write(&crypto_alg_sem);
469		469
470	BUG_ON(list_empty(&tmpl->list));	470	BUG_ON(list_empty(&tmpl->list));
471	list_del_init(&tmpl->list);	471	list_del_init(&tmpl->list);
472		472
473	list = &tmpl->instances;	473	list = &tmpl->instances;
474	hlist_for_each_entry(inst, list, list) {	474	hlist_for_each_entry(inst, list, list) {
475	int err = crypto_remove_alg(&inst->alg, &users);	475	int err = crypto_remove_alg(&inst->alg, &users);
476	BUG_ON(err);	476	BUG_ON(err);
477	}	477	}
478		478
479	crypto_notify(CRYPTO_MSG_TMPL_UNREGISTER, tmpl);	479	crypto_notify(CRYPTO_MSG_TMPL_UNREGISTER, tmpl);
480		480
481	up_write(&crypto_alg_sem);	481	up_write(&crypto_alg_sem);
482		482
483	hlist_for_each_entry_safe(inst, n, list, list) {	483	hlist_for_each_entry_safe(inst, n, list, list) {
484	BUG_ON(atomic_read(&inst->alg.cra_refcnt) != 1);	484	BUG_ON(atomic_read(&inst->alg.cra_refcnt) != 1);
485	tmpl->free(inst);	485	tmpl->free(inst);
486	}	486	}
487	crypto_remove_final(&users);	487	crypto_remove_final(&users);
488	}	488	}
489	EXPORT_SYMBOL_GPL(crypto_unregister_template);	489	EXPORT_SYMBOL_GPL(crypto_unregister_template);
490		490
491	static struct crypto_template __crypto_lookup_template(const char name)	491	static struct crypto_template __crypto_lookup_template(const char name)
492	{	492	{
493	struct crypto_template q, tmpl = NULL;	493	struct crypto_template q, tmpl = NULL;
494		494
495	down_read(&crypto_alg_sem);	495	down_read(&crypto_alg_sem);
496	list_for_each_entry(q, &crypto_template_list, list) {	496	list_for_each_entry(q, &crypto_template_list, list) {
497	if (strcmp(q->name, name))	497	if (strcmp(q->name, name))
498	continue;	498	continue;
499	if (unlikely(!crypto_tmpl_get(q)))	499	if (unlikely(!crypto_tmpl_get(q)))
500	continue;	500	continue;
501		501
502	tmpl = q;	502	tmpl = q;
503	break;	503	break;
504	}	504	}
505	up_read(&crypto_alg_sem);	505	up_read(&crypto_alg_sem);
506		506
507	return tmpl;	507	return tmpl;
508	}	508	}
509		509
510	struct crypto_template crypto_lookup_template(const char name)	510	struct crypto_template crypto_lookup_template(const char name)
511	{	511	{
512	return try_then_request_module(__crypto_lookup_template(name), "%s",	512	return try_then_request_module(__crypto_lookup_template(name),
513	name);	513	"crypto-%s", name);
514	}	514	}
515	EXPORT_SYMBOL_GPL(crypto_lookup_template);	515	EXPORT_SYMBOL_GPL(crypto_lookup_template);
516		516
517	int crypto_register_instance(struct crypto_template *tmpl,	517	int crypto_register_instance(struct crypto_template *tmpl,
518	struct crypto_instance *inst)	518	struct crypto_instance *inst)
519	{	519	{
520	struct crypto_larval *larval;	520	struct crypto_larval *larval;
521	int err;	521	int err;
522		522
523	err = crypto_check_alg(&inst->alg);	523	err = crypto_check_alg(&inst->alg);
524	if (err)	524	if (err)
525	goto err;	525	goto err;
526		526
527	inst->alg.cra_module = tmpl->module;	527	inst->alg.cra_module = tmpl->module;
528	inst->alg.cra_flags \|= CRYPTO_ALG_INSTANCE;	528	inst->alg.cra_flags \|= CRYPTO_ALG_INSTANCE;
529		529
530	down_write(&crypto_alg_sem);	530	down_write(&crypto_alg_sem);
531		531
532	larval = __crypto_register_alg(&inst->alg);	532	larval = __crypto_register_alg(&inst->alg);
533	if (IS_ERR(larval))	533	if (IS_ERR(larval))
534	goto unlock;	534	goto unlock;
535		535
536	hlist_add_head(&inst->list, &tmpl->instances);	536	hlist_add_head(&inst->list, &tmpl->instances);
537	inst->tmpl = tmpl;	537	inst->tmpl = tmpl;
538		538
539	unlock:	539	unlock:
540	up_write(&crypto_alg_sem);	540	up_write(&crypto_alg_sem);
541		541
542	err = PTR_ERR(larval);	542	err = PTR_ERR(larval);
543	if (IS_ERR(larval))	543	if (IS_ERR(larval))
544	goto err;	544	goto err;
545		545
546	crypto_wait_for_test(larval);	546	crypto_wait_for_test(larval);
547	err = 0;	547	err = 0;
548		548
549	err:	549	err:
550	return err;	550	return err;
551	}	551	}
552	EXPORT_SYMBOL_GPL(crypto_register_instance);	552	EXPORT_SYMBOL_GPL(crypto_register_instance);
553		553
554	int crypto_unregister_instance(struct crypto_alg *alg)	554	int crypto_unregister_instance(struct crypto_alg *alg)
555	{	555	{
556	int err;	556	int err;
557	struct crypto_instance inst = (void )alg;	557	struct crypto_instance inst = (void )alg;
558	struct crypto_template *tmpl = inst->tmpl;	558	struct crypto_template *tmpl = inst->tmpl;
559	LIST_HEAD(users);	559	LIST_HEAD(users);
560		560
561	if (!(alg->cra_flags & CRYPTO_ALG_INSTANCE))	561	if (!(alg->cra_flags & CRYPTO_ALG_INSTANCE))
562	return -EINVAL;	562	return -EINVAL;
563		563
564	BUG_ON(atomic_read(&alg->cra_refcnt) != 1);	564	BUG_ON(atomic_read(&alg->cra_refcnt) != 1);
565		565
566	down_write(&crypto_alg_sem);	566	down_write(&crypto_alg_sem);
567		567
568	hlist_del_init(&inst->list);	568	hlist_del_init(&inst->list);
569	err = crypto_remove_alg(alg, &users);	569	err = crypto_remove_alg(alg, &users);
570		570
571	up_write(&crypto_alg_sem);	571	up_write(&crypto_alg_sem);
572		572
573	if (err)	573	if (err)
574	return err;	574	return err;
575		575
576	tmpl->free(inst);	576	tmpl->free(inst);
577	crypto_remove_final(&users);	577	crypto_remove_final(&users);
578		578
579	return 0;	579	return 0;
580	}	580	}
581	EXPORT_SYMBOL_GPL(crypto_unregister_instance);	581	EXPORT_SYMBOL_GPL(crypto_unregister_instance);
582		582
583	int crypto_init_spawn(struct crypto_spawn spawn, struct crypto_alg alg,	583	int crypto_init_spawn(struct crypto_spawn spawn, struct crypto_alg alg,
584	struct crypto_instance *inst, u32 mask)	584	struct crypto_instance *inst, u32 mask)
585	{	585	{
586	int err = -EAGAIN;	586	int err = -EAGAIN;
587		587
588	spawn->inst = inst;	588	spawn->inst = inst;
589	spawn->mask = mask;	589	spawn->mask = mask;
590		590
591	down_write(&crypto_alg_sem);	591	down_write(&crypto_alg_sem);
592	if (!crypto_is_moribund(alg)) {	592	if (!crypto_is_moribund(alg)) {
593	list_add(&spawn->list, &alg->cra_users);	593	list_add(&spawn->list, &alg->cra_users);
594	spawn->alg = alg;	594	spawn->alg = alg;
595	err = 0;	595	err = 0;
596	}	596	}
597	up_write(&crypto_alg_sem);	597	up_write(&crypto_alg_sem);
598		598
599	return err;	599	return err;
600	}	600	}
601	EXPORT_SYMBOL_GPL(crypto_init_spawn);	601	EXPORT_SYMBOL_GPL(crypto_init_spawn);
602		602
603	int crypto_init_spawn2(struct crypto_spawn spawn, struct crypto_alg alg,	603	int crypto_init_spawn2(struct crypto_spawn spawn, struct crypto_alg alg,
604	struct crypto_instance *inst,	604	struct crypto_instance *inst,
605	const struct crypto_type *frontend)	605	const struct crypto_type *frontend)
606	{	606	{
607	int err = -EINVAL;	607	int err = -EINVAL;
608		608
609	if ((alg->cra_flags ^ frontend->type) & frontend->maskset)	609	if ((alg->cra_flags ^ frontend->type) & frontend->maskset)
610	goto out;	610	goto out;
611		611
612	spawn->frontend = frontend;	612	spawn->frontend = frontend;
613	err = crypto_init_spawn(spawn, alg, inst, frontend->maskset);	613	err = crypto_init_spawn(spawn, alg, inst, frontend->maskset);
614		614
615	out:	615	out:
616	return err;	616	return err;
617	}	617	}
618	EXPORT_SYMBOL_GPL(crypto_init_spawn2);	618	EXPORT_SYMBOL_GPL(crypto_init_spawn2);
619		619
620	void crypto_drop_spawn(struct crypto_spawn *spawn)	620	void crypto_drop_spawn(struct crypto_spawn *spawn)
621	{	621	{
622	if (!spawn->alg)	622	if (!spawn->alg)
623	return;	623	return;
624		624
625	down_write(&crypto_alg_sem);	625	down_write(&crypto_alg_sem);
626	list_del(&spawn->list);	626	list_del(&spawn->list);
627	up_write(&crypto_alg_sem);	627	up_write(&crypto_alg_sem);
628	}	628	}
629	EXPORT_SYMBOL_GPL(crypto_drop_spawn);	629	EXPORT_SYMBOL_GPL(crypto_drop_spawn);
630		630
631	static struct crypto_alg crypto_spawn_alg(struct crypto_spawn spawn)	631	static struct crypto_alg crypto_spawn_alg(struct crypto_spawn spawn)
632	{	632	{
633	struct crypto_alg *alg;	633	struct crypto_alg *alg;
634	struct crypto_alg *alg2;	634	struct crypto_alg *alg2;
635		635
636	down_read(&crypto_alg_sem);	636	down_read(&crypto_alg_sem);
637	alg = spawn->alg;	637	alg = spawn->alg;
638	alg2 = alg;	638	alg2 = alg;
639	if (alg2)	639	if (alg2)
640	alg2 = crypto_mod_get(alg2);	640	alg2 = crypto_mod_get(alg2);
641	up_read(&crypto_alg_sem);	641	up_read(&crypto_alg_sem);
642		642
643	if (!alg2) {	643	if (!alg2) {
644	if (alg)	644	if (alg)
645	crypto_shoot_alg(alg);	645	crypto_shoot_alg(alg);
646	return ERR_PTR(-EAGAIN);	646	return ERR_PTR(-EAGAIN);
647	}	647	}
648		648
649	return alg;	649	return alg;
650	}	650	}
651		651
652	struct crypto_tfm crypto_spawn_tfm(struct crypto_spawn spawn, u32 type,	652	struct crypto_tfm crypto_spawn_tfm(struct crypto_spawn spawn, u32 type,
653	u32 mask)	653	u32 mask)
654	{	654	{
655	struct crypto_alg *alg;	655	struct crypto_alg *alg;
656	struct crypto_tfm *tfm;	656	struct crypto_tfm *tfm;
657		657
658	alg = crypto_spawn_alg(spawn);	658	alg = crypto_spawn_alg(spawn);
659	if (IS_ERR(alg))	659	if (IS_ERR(alg))
660	return ERR_CAST(alg);	660	return ERR_CAST(alg);
661		661
662	tfm = ERR_PTR(-EINVAL);	662	tfm = ERR_PTR(-EINVAL);
663	if (unlikely((alg->cra_flags ^ type) & mask))	663	if (unlikely((alg->cra_flags ^ type) & mask))
664	goto out_put_alg;	664	goto out_put_alg;
665		665
666	tfm = __crypto_alloc_tfm(alg, type, mask);	666	tfm = __crypto_alloc_tfm(alg, type, mask);
667	if (IS_ERR(tfm))	667	if (IS_ERR(tfm))
668	goto out_put_alg;	668	goto out_put_alg;
669		669
670	return tfm;	670	return tfm;
671		671
672	out_put_alg:	672	out_put_alg:
673	crypto_mod_put(alg);	673	crypto_mod_put(alg);
674	return tfm;	674	return tfm;
675	}	675	}
676	EXPORT_SYMBOL_GPL(crypto_spawn_tfm);	676	EXPORT_SYMBOL_GPL(crypto_spawn_tfm);
677		677
678	void crypto_spawn_tfm2(struct crypto_spawn spawn)	678	void crypto_spawn_tfm2(struct crypto_spawn spawn)
679	{	679	{
680	struct crypto_alg *alg;	680	struct crypto_alg *alg;
681	struct crypto_tfm *tfm;	681	struct crypto_tfm *tfm;
682		682
683	alg = crypto_spawn_alg(spawn);	683	alg = crypto_spawn_alg(spawn);
684	if (IS_ERR(alg))	684	if (IS_ERR(alg))
685	return ERR_CAST(alg);	685	return ERR_CAST(alg);
686		686
687	tfm = crypto_create_tfm(alg, spawn->frontend);	687	tfm = crypto_create_tfm(alg, spawn->frontend);
688	if (IS_ERR(tfm))	688	if (IS_ERR(tfm))
689	goto out_put_alg;	689	goto out_put_alg;
690		690
691	return tfm;	691	return tfm;
692		692
693	out_put_alg:	693	out_put_alg:
694	crypto_mod_put(alg);	694	crypto_mod_put(alg);
695	return tfm;	695	return tfm;
696	}	696	}
697	EXPORT_SYMBOL_GPL(crypto_spawn_tfm2);	697	EXPORT_SYMBOL_GPL(crypto_spawn_tfm2);
698		698
699	int crypto_register_notifier(struct notifier_block *nb)	699	int crypto_register_notifier(struct notifier_block *nb)
700	{	700	{
701	return blocking_notifier_chain_register(&crypto_chain, nb);	701	return blocking_notifier_chain_register(&crypto_chain, nb);
702	}	702	}
703	EXPORT_SYMBOL_GPL(crypto_register_notifier);	703	EXPORT_SYMBOL_GPL(crypto_register_notifier);
704		704
705	int crypto_unregister_notifier(struct notifier_block *nb)	705	int crypto_unregister_notifier(struct notifier_block *nb)
706	{	706	{
707	return blocking_notifier_chain_unregister(&crypto_chain, nb);	707	return blocking_notifier_chain_unregister(&crypto_chain, nb);
708	}	708	}
709	EXPORT_SYMBOL_GPL(crypto_unregister_notifier);	709	EXPORT_SYMBOL_GPL(crypto_unregister_notifier);
710		710
711	struct crypto_attr_type crypto_get_attr_type(struct rtattr *tb)	711	struct crypto_attr_type crypto_get_attr_type(struct rtattr *tb)
712	{	712	{
713	struct rtattr *rta = tb[0];	713	struct rtattr *rta = tb[0];
714	struct crypto_attr_type *algt;	714	struct crypto_attr_type *algt;
715		715
716	if (!rta)	716	if (!rta)
717	return ERR_PTR(-ENOENT);	717	return ERR_PTR(-ENOENT);
718	if (RTA_PAYLOAD(rta) < sizeof(*algt))	718	if (RTA_PAYLOAD(rta) < sizeof(*algt))
719	return ERR_PTR(-EINVAL);	719	return ERR_PTR(-EINVAL);
720	if (rta->rta_type != CRYPTOA_TYPE)	720	if (rta->rta_type != CRYPTOA_TYPE)
721	return ERR_PTR(-EINVAL);	721	return ERR_PTR(-EINVAL);
722		722
723	algt = RTA_DATA(rta);	723	algt = RTA_DATA(rta);
724		724
725	return algt;	725	return algt;
726	}	726	}
727	EXPORT_SYMBOL_GPL(crypto_get_attr_type);	727	EXPORT_SYMBOL_GPL(crypto_get_attr_type);
728		728
729	int crypto_check_attr_type(struct rtattr **tb, u32 type)	729	int crypto_check_attr_type(struct rtattr **tb, u32 type)
730	{	730	{
731	struct crypto_attr_type *algt;	731	struct crypto_attr_type *algt;
732		732
733	algt = crypto_get_attr_type(tb);	733	algt = crypto_get_attr_type(tb);
734	if (IS_ERR(algt))	734	if (IS_ERR(algt))
735	return PTR_ERR(algt);	735	return PTR_ERR(algt);
736		736
737	if ((algt->type ^ type) & algt->mask)	737	if ((algt->type ^ type) & algt->mask)
738	return -EINVAL;	738	return -EINVAL;
739		739
740	return 0;	740	return 0;
741	}	741	}
742	EXPORT_SYMBOL_GPL(crypto_check_attr_type);	742	EXPORT_SYMBOL_GPL(crypto_check_attr_type);
743		743
744	const char crypto_attr_alg_name(struct rtattr rta)	744	const char crypto_attr_alg_name(struct rtattr rta)
745	{	745	{
746	struct crypto_attr_alg *alga;	746	struct crypto_attr_alg *alga;
747		747
748	if (!rta)	748	if (!rta)
749	return ERR_PTR(-ENOENT);	749	return ERR_PTR(-ENOENT);
750	if (RTA_PAYLOAD(rta) < sizeof(*alga))	750	if (RTA_PAYLOAD(rta) < sizeof(*alga))
751	return ERR_PTR(-EINVAL);	751	return ERR_PTR(-EINVAL);
752	if (rta->rta_type != CRYPTOA_ALG)	752	if (rta->rta_type != CRYPTOA_ALG)
753	return ERR_PTR(-EINVAL);	753	return ERR_PTR(-EINVAL);
754		754
755	alga = RTA_DATA(rta);	755	alga = RTA_DATA(rta);
756	alga->name[CRYPTO_MAX_ALG_NAME - 1] = 0;	756	alga->name[CRYPTO_MAX_ALG_NAME - 1] = 0;
757		757
758	return alga->name;	758	return alga->name;
759	}	759	}
760	EXPORT_SYMBOL_GPL(crypto_attr_alg_name);	760	EXPORT_SYMBOL_GPL(crypto_attr_alg_name);
761		761
762	struct crypto_alg crypto_attr_alg2(struct rtattr rta,	762	struct crypto_alg crypto_attr_alg2(struct rtattr rta,
763	const struct crypto_type *frontend,	763	const struct crypto_type *frontend,
764	u32 type, u32 mask)	764	u32 type, u32 mask)
765	{	765	{
766	const char *name;	766	const char *name;
767		767
768	name = crypto_attr_alg_name(rta);	768	name = crypto_attr_alg_name(rta);
769	if (IS_ERR(name))	769	if (IS_ERR(name))
770	return ERR_CAST(name);	770	return ERR_CAST(name);
771		771
772	return crypto_find_alg(name, frontend, type, mask);	772	return crypto_find_alg(name, frontend, type, mask);
773	}	773	}
774	EXPORT_SYMBOL_GPL(crypto_attr_alg2);	774	EXPORT_SYMBOL_GPL(crypto_attr_alg2);
775		775
776	int crypto_attr_u32(struct rtattr rta, u32 num)	776	int crypto_attr_u32(struct rtattr rta, u32 num)
777	{	777	{
778	struct crypto_attr_u32 *nu32;	778	struct crypto_attr_u32 *nu32;
779		779
780	if (!rta)	780	if (!rta)
781	return -ENOENT;	781	return -ENOENT;
782	if (RTA_PAYLOAD(rta) < sizeof(*nu32))	782	if (RTA_PAYLOAD(rta) < sizeof(*nu32))
783	return -EINVAL;	783	return -EINVAL;
784	if (rta->rta_type != CRYPTOA_U32)	784	if (rta->rta_type != CRYPTOA_U32)
785	return -EINVAL;	785	return -EINVAL;
786		786
787	nu32 = RTA_DATA(rta);	787	nu32 = RTA_DATA(rta);
788	*num = nu32->num;	788	*num = nu32->num;
789		789
790	return 0;	790	return 0;
791	}	791	}
792	EXPORT_SYMBOL_GPL(crypto_attr_u32);	792	EXPORT_SYMBOL_GPL(crypto_attr_u32);
793		793
794	void crypto_alloc_instance2(const char name, struct crypto_alg *alg,	794	void crypto_alloc_instance2(const char name, struct crypto_alg *alg,
795	unsigned int head)	795	unsigned int head)
796	{	796	{
797	struct crypto_instance *inst;	797	struct crypto_instance *inst;
798	char *p;	798	char *p;
799	int err;	799	int err;
800		800
801	p = kzalloc(head + sizeof(*inst) + sizeof(struct crypto_spawn),	801	p = kzalloc(head + sizeof(*inst) + sizeof(struct crypto_spawn),
802	GFP_KERNEL);	802	GFP_KERNEL);
803	if (!p)	803	if (!p)
804	return ERR_PTR(-ENOMEM);	804	return ERR_PTR(-ENOMEM);
805		805
806	inst = (void *)(p + head);	806	inst = (void *)(p + head);
807		807
808	err = -ENAMETOOLONG;	808	err = -ENAMETOOLONG;
809	if (snprintf(inst->alg.cra_name, CRYPTO_MAX_ALG_NAME, "%s(%s)", name,	809	if (snprintf(inst->alg.cra_name, CRYPTO_MAX_ALG_NAME, "%s(%s)", name,
810	alg->cra_name) >= CRYPTO_MAX_ALG_NAME)	810	alg->cra_name) >= CRYPTO_MAX_ALG_NAME)
811	goto err_free_inst;	811	goto err_free_inst;
812		812
813	if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s(%s)",	813	if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s(%s)",
814	name, alg->cra_driver_name) >= CRYPTO_MAX_ALG_NAME)	814	name, alg->cra_driver_name) >= CRYPTO_MAX_ALG_NAME)
815	goto err_free_inst;	815	goto err_free_inst;
816		816
817	return p;	817	return p;
818		818
819	err_free_inst:	819	err_free_inst:
820	kfree(p);	820	kfree(p);
821	return ERR_PTR(err);	821	return ERR_PTR(err);
822	}	822	}
823	EXPORT_SYMBOL_GPL(crypto_alloc_instance2);	823	EXPORT_SYMBOL_GPL(crypto_alloc_instance2);
824		824
825	struct crypto_instance crypto_alloc_instance(const char name,	825	struct crypto_instance crypto_alloc_instance(const char name,
826	struct crypto_alg *alg)	826	struct crypto_alg *alg)
827	{	827	{
828	struct crypto_instance *inst;	828	struct crypto_instance *inst;
829	struct crypto_spawn *spawn;	829	struct crypto_spawn *spawn;
830	int err;	830	int err;
831		831
832	inst = crypto_alloc_instance2(name, alg, 0);	832	inst = crypto_alloc_instance2(name, alg, 0);
833	if (IS_ERR(inst))	833	if (IS_ERR(inst))
834	goto out;	834	goto out;
835		835
836	spawn = crypto_instance_ctx(inst);	836	spawn = crypto_instance_ctx(inst);
837	err = crypto_init_spawn(spawn, alg, inst,	837	err = crypto_init_spawn(spawn, alg, inst,
838	CRYPTO_ALG_TYPE_MASK \| CRYPTO_ALG_ASYNC);	838	CRYPTO_ALG_TYPE_MASK \| CRYPTO_ALG_ASYNC);
839		839
840	if (err)	840	if (err)
841	goto err_free_inst;	841	goto err_free_inst;
842		842
843	return inst;	843	return inst;
844		844
845	err_free_inst:	845	err_free_inst:
846	kfree(inst);	846	kfree(inst);
847	inst = ERR_PTR(err);	847	inst = ERR_PTR(err);
848		848
849	out:	849	out:
850	return inst;	850	return inst;
851	}	851	}
852	EXPORT_SYMBOL_GPL(crypto_alloc_instance);	852	EXPORT_SYMBOL_GPL(crypto_alloc_instance);
853		853
854	void crypto_init_queue(struct crypto_queue *queue, unsigned int max_qlen)	854	void crypto_init_queue(struct crypto_queue *queue, unsigned int max_qlen)
855	{	855	{
856	INIT_LIST_HEAD(&queue->list);	856	INIT_LIST_HEAD(&queue->list);
857	queue->backlog = &queue->list;	857	queue->backlog = &queue->list;
858	queue->qlen = 0;	858	queue->qlen = 0;
859	queue->max_qlen = max_qlen;	859	queue->max_qlen = max_qlen;
860	}	860	}
861	EXPORT_SYMBOL_GPL(crypto_init_queue);	861	EXPORT_SYMBOL_GPL(crypto_init_queue);
862		862
863	int crypto_enqueue_request(struct crypto_queue *queue,	863	int crypto_enqueue_request(struct crypto_queue *queue,
864	struct crypto_async_request *request)	864	struct crypto_async_request *request)
865	{	865	{
866	int err = -EINPROGRESS;	866	int err = -EINPROGRESS;
867		867
868	if (unlikely(queue->qlen >= queue->max_qlen)) {	868	if (unlikely(queue->qlen >= queue->max_qlen)) {
869	err = -EBUSY;	869	err = -EBUSY;
870	if (!(request->flags & CRYPTO_TFM_REQ_MAY_BACKLOG))	870	if (!(request->flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
871	goto out;	871	goto out;
872	if (queue->backlog == &queue->list)	872	if (queue->backlog == &queue->list)
873	queue->backlog = &request->list;	873	queue->backlog = &request->list;
874	}	874	}
875		875
876	queue->qlen++;	876	queue->qlen++;
877	list_add_tail(&request->list, &queue->list);	877	list_add_tail(&request->list, &queue->list);
878		878
879	out:	879	out:
880	return err;	880	return err;
881	}	881	}
882	EXPORT_SYMBOL_GPL(crypto_enqueue_request);	882	EXPORT_SYMBOL_GPL(crypto_enqueue_request);
883		883
884	void __crypto_dequeue_request(struct crypto_queue queue, unsigned int offset)	884	void __crypto_dequeue_request(struct crypto_queue queue, unsigned int offset)
885	{	885	{
886	struct list_head *request;	886	struct list_head *request;
887		887
888	if (unlikely(!queue->qlen))	888	if (unlikely(!queue->qlen))
889	return NULL;	889	return NULL;
890		890
891	queue->qlen--;	891	queue->qlen--;
892		892
893	if (queue->backlog != &queue->list)	893	if (queue->backlog != &queue->list)
894	queue->backlog = queue->backlog->next;	894	queue->backlog = queue->backlog->next;
895		895
896	request = queue->list.next;	896	request = queue->list.next;
897	list_del(request);	897	list_del(request);
898		898
899	return (char *)list_entry(request, struct crypto_async_request, list) -	899	return (char *)list_entry(request, struct crypto_async_request, list) -
900	offset;	900	offset;
901	}	901	}
902	EXPORT_SYMBOL_GPL(__crypto_dequeue_request);	902	EXPORT_SYMBOL_GPL(__crypto_dequeue_request);
903		903
904	struct crypto_async_request crypto_dequeue_request(struct crypto_queue queue)	904	struct crypto_async_request crypto_dequeue_request(struct crypto_queue queue)
905	{	905	{
906	return __crypto_dequeue_request(queue, 0);	906	return __crypto_dequeue_request(queue, 0);
907	}	907	}
908	EXPORT_SYMBOL_GPL(crypto_dequeue_request);	908	EXPORT_SYMBOL_GPL(crypto_dequeue_request);
909		909
910	int crypto_tfm_in_queue(struct crypto_queue queue, struct crypto_tfm tfm)	910	int crypto_tfm_in_queue(struct crypto_queue queue, struct crypto_tfm tfm)
911	{	911	{
912	struct crypto_async_request *req;	912	struct crypto_async_request *req;
913		913
914	list_for_each_entry(req, &queue->list, list) {	914	list_for_each_entry(req, &queue->list, list) {
915	if (req->tfm == tfm)	915	if (req->tfm == tfm)
916	return 1;	916	return 1;
917	}	917	}
918		918
919	return 0;	919	return 0;
920	}	920	}
921	EXPORT_SYMBOL_GPL(crypto_tfm_in_queue);	921	EXPORT_SYMBOL_GPL(crypto_tfm_in_queue);
922		922
923	static inline void crypto_inc_byte(u8 *a, unsigned int size)	923	static inline void crypto_inc_byte(u8 *a, unsigned int size)
924	{	924	{
925	u8 *b = (a + size);	925	u8 *b = (a + size);
926	u8 c;	926	u8 c;
927		927
928	for (; size; size--) {	928	for (; size; size--) {
929	c = *--b + 1;	929	c = *--b + 1;
930	*b = c;	930	*b = c;
931	if (c)	931	if (c)
932	break;	932	break;
933	}	933	}
934	}	934	}
935		935
936	void crypto_inc(u8 *a, unsigned int size)	936	void crypto_inc(u8 *a, unsigned int size)
937	{	937	{
938	__be32 b = (__be32 )(a + size);	938	__be32 b = (__be32 )(a + size);
939	u32 c;	939	u32 c;
940		940
941	for (; size >= 4; size -= 4) {	941	for (; size >= 4; size -= 4) {
942	c = be32_to_cpu(*--b) + 1;	942	c = be32_to_cpu(*--b) + 1;
943	*b = cpu_to_be32(c);	943	*b = cpu_to_be32(c);
944	if (c)	944	if (c)
945	return;	945	return;
946	}	946	}
947		947
948	crypto_inc_byte(a, size);	948	crypto_inc_byte(a, size);
949	}	949	}
950	EXPORT_SYMBOL_GPL(crypto_inc);	950	EXPORT_SYMBOL_GPL(crypto_inc);
951		951
952	static inline void crypto_xor_byte(u8 a, const u8 b, unsigned int size)	952	static inline void crypto_xor_byte(u8 a, const u8 b, unsigned int size)
953	{	953	{
954	for (; size; size--)	954	for (; size; size--)
955	a++ ^= b++;	955	a++ ^= b++;
956	}	956	}
957		957
958	void crypto_xor(u8 dst, const u8 src, unsigned int size)	958	void crypto_xor(u8 dst, const u8 src, unsigned int size)
959	{	959	{
960	u32 a = (u32 )dst;	960	u32 a = (u32 )dst;
961	u32 b = (u32 )src;	961	u32 b = (u32 )src;
962		962
963	for (; size >= 4; size -= 4)	963	for (; size >= 4; size -= 4)
964	a++ ^= b++;	964	a++ ^= b++;
965		965
966	crypto_xor_byte((u8 )a, (u8 )b, size);	966	crypto_xor_byte((u8 )a, (u8 )b, size);
967	}	967	}
968	EXPORT_SYMBOL_GPL(crypto_xor);	968	EXPORT_SYMBOL_GPL(crypto_xor);
969		969
970	static int __init crypto_algapi_init(void)	970	static int __init crypto_algapi_init(void)
971	{	971	{
972	crypto_init_proc();	972	crypto_init_proc();
973	return 0;	973	return 0;
974	}	974	}
975		975
976	static void __exit crypto_algapi_exit(void)	976	static void __exit crypto_algapi_exit(void)
977	{	977	{
978	crypto_exit_proc();	978	crypto_exit_proc();
979	}	979	}
980		980
981	module_init(crypto_algapi_init);	981	module_init(crypto_algapi_init);
982	module_exit(crypto_algapi_exit);	982	module_exit(crypto_algapi_exit);
983		983
984	MODULE_LICENSE("GPL");	984	MODULE_LICENSE("GPL");
985	MODULE_DESCRIPTION("Cryptographic algorithms API");	985	MODULE_DESCRIPTION("Cryptographic algorithms API");
986		986

crypto/authenc.c

Diff comments View file @ acc5ccb

 /*
  * Authenc: Simple AEAD wrapper for IPsec
  *
  * Copyright (c) 2007 Herbert Xu <herbert@gondor.apana.org.au>
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the Free
  * Software Foundation; either version 2 of the License, or (at your option)
  * any later version.
  *
  */
 #include <crypto/aead.h>
 #include <crypto/internal/hash.h>
 #include <crypto/internal/skcipher.h>
 #include <crypto/authenc.h>
 #include <crypto/scatterwalk.h>
 #include <linux/err.h>
 #include <linux/init.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/rtnetlink.h>
 #include <linux/slab.h>
 #include <linux/spinlock.h>
 typedef u8 *(*authenc_ahash_t)(struct aead_request *req, unsigned int flags);
 struct authenc_instance_ctx {
 	struct crypto_ahash_spawn auth;
 	struct crypto_skcipher_spawn enc;
 };
 struct crypto_authenc_ctx {
 	unsigned int reqoff;
 	struct crypto_ahash *auth;
 	struct crypto_ablkcipher *enc;
 };
 struct authenc_request_ctx {
 	unsigned int cryptlen;
 	struct scatterlist *sg;
 	struct scatterlist asg[2];
 	struct scatterlist cipher[2];
 	crypto_completion_t complete;
 	crypto_completion_t update_complete;
 	char tail[];
 };
 static void authenc_request_complete(struct aead_request *req, int err)
 {
 	if (err != -EINPROGRESS)
 		aead_request_complete(req, err);
 }
 int crypto_authenc_extractkeys(struct crypto_authenc_keys *keys, const u8 *key,
 			       unsigned int keylen)
 {
 	struct rtattr *rta = (struct rtattr *)key;
 	struct crypto_authenc_key_param *param;
 	if (!RTA_OK(rta, keylen))
 		return -EINVAL;
 	if (rta->rta_type != CRYPTO_AUTHENC_KEYA_PARAM)
 		return -EINVAL;
 	if (RTA_PAYLOAD(rta) < sizeof(*param))
 		return -EINVAL;
 	param = RTA_DATA(rta);
 	keys->enckeylen = be32_to_cpu(param->enckeylen);
 	key += RTA_ALIGN(rta->rta_len);
 	keylen -= RTA_ALIGN(rta->rta_len);
 	if (keylen < keys->enckeylen)
 		return -EINVAL;
 	keys->authkeylen = keylen - keys->enckeylen;
 	keys->authkey = key;
 	keys->enckey = key + keys->authkeylen;
 	return 0;
 }
 EXPORT_SYMBOL_GPL(crypto_authenc_extractkeys);
 static int crypto_authenc_setkey(struct crypto_aead *authenc, const u8 *key,
 				 unsigned int keylen)
 {
 	struct crypto_authenc_ctx *ctx = crypto_aead_ctx(authenc);
 	struct crypto_ahash *auth = ctx->auth;
 	struct crypto_ablkcipher *enc = ctx->enc;
 	struct crypto_authenc_keys keys;
 	int err = -EINVAL;
 	if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
 		goto badkey;
 	crypto_ahash_clear_flags(auth, CRYPTO_TFM_REQ_MASK);
 	crypto_ahash_set_flags(auth, crypto_aead_get_flags(authenc) &
 				    CRYPTO_TFM_REQ_MASK);
 	err = crypto_ahash_setkey(auth, keys.authkey, keys.authkeylen);
 	crypto_aead_set_flags(authenc, crypto_ahash_get_flags(auth) &
 				       CRYPTO_TFM_RES_MASK);
 	if (err)
 		goto out;
 	crypto_ablkcipher_clear_flags(enc, CRYPTO_TFM_REQ_MASK);
 	crypto_ablkcipher_set_flags(enc, crypto_aead_get_flags(authenc) &
 					 CRYPTO_TFM_REQ_MASK);
 	err = crypto_ablkcipher_setkey(enc, keys.enckey, keys.enckeylen);
 	crypto_aead_set_flags(authenc, crypto_ablkcipher_get_flags(enc) &
 				       CRYPTO_TFM_RES_MASK);
 out:
 	return err;
 badkey:
 	crypto_aead_set_flags(authenc, CRYPTO_TFM_RES_BAD_KEY_LEN);
 	goto out;
 }
 static void authenc_geniv_ahash_update_done(struct crypto_async_request *areq,
 					    int err)
 {
 	struct aead_request *req = areq->data;
 	struct crypto_aead *authenc = crypto_aead_reqtfm(req);
 	struct crypto_authenc_ctx *ctx = crypto_aead_ctx(authenc);
 	struct authenc_request_ctx *areq_ctx = aead_request_ctx(req);
 	struct ahash_request *ahreq = (void *)(areq_ctx->tail + ctx->reqoff);
 	if (err)
 		goto out;
 	ahash_request_set_crypt(ahreq, areq_ctx->sg, ahreq->result,
 				areq_ctx->cryptlen);
 	ahash_request_set_callback(ahreq, aead_request_flags(req) &
 					  CRYPTO_TFM_REQ_MAY_SLEEP,
 				   areq_ctx->complete, req);
 	err = crypto_ahash_finup(ahreq);
 	if (err)
 		goto out;
 	scatterwalk_map_and_copy(ahreq->result, areq_ctx->sg,
 				 areq_ctx->cryptlen,
 				 crypto_aead_authsize(authenc), 1);
 out:
 	authenc_request_complete(req, err);
 }
 static void authenc_geniv_ahash_done(struct crypto_async_request *areq, int err)
 {
 	struct aead_request *req = areq->data;
 	struct crypto_aead *authenc = crypto_aead_reqtfm(req);
 	struct crypto_authenc_ctx *ctx = crypto_aead_ctx(authenc);
 	struct authenc_request_ctx *areq_ctx = aead_request_ctx(req);
 	struct ahash_request *ahreq = (void *)(areq_ctx->tail + ctx->reqoff);
 	if (err)
 		goto out;
 	scatterwalk_map_and_copy(ahreq->result, areq_ctx->sg,
 				 areq_ctx->cryptlen,
 				 crypto_aead_authsize(authenc), 1);
 out:
 	aead_request_complete(req, err);
 }
 static void authenc_verify_ahash_update_done(struct crypto_async_request *areq,
 					     int err)
 {
 	u8 *ihash;
 	unsigned int authsize;
 	struct ablkcipher_request *abreq;
 	struct aead_request *req = areq->data;
 	struct crypto_aead *authenc = crypto_aead_reqtfm(req);
 	struct crypto_authenc_ctx *ctx = crypto_aead_ctx(authenc);
 	struct authenc_request_ctx *areq_ctx = aead_request_ctx(req);
 	struct ahash_request *ahreq = (void *)(areq_ctx->tail + ctx->reqoff);
 	unsigned int cryptlen = req->cryptlen;
 	if (err)
 		goto out;
 	ahash_request_set_crypt(ahreq, areq_ctx->sg, ahreq->result,
 				areq_ctx->cryptlen);
 	ahash_request_set_callback(ahreq, aead_request_flags(req) &
 					  CRYPTO_TFM_REQ_MAY_SLEEP,
 				   areq_ctx->complete, req);
 	err = crypto_ahash_finup(ahreq);
 	if (err)
 		goto out;
 	authsize = crypto_aead_authsize(authenc);
 	cryptlen -= authsize;
 	ihash = ahreq->result + authsize;
 	scatterwalk_map_and_copy(ihash, areq_ctx->sg, areq_ctx->cryptlen,
 				 authsize, 0);
 	err = crypto_memneq(ihash, ahreq->result, authsize) ? -EBADMSG : 0;
 	if (err)
 		goto out;
 	abreq = aead_request_ctx(req);
 	ablkcipher_request_set_tfm(abreq, ctx->enc);
 	ablkcipher_request_set_callback(abreq, aead_request_flags(req),
 					req->base.complete, req->base.data);
 	ablkcipher_request_set_crypt(abreq, req->src, req->dst,
 				     cryptlen, req->iv);
 	err = crypto_ablkcipher_decrypt(abreq);
 out:
 	authenc_request_complete(req, err);
 }
 static void authenc_verify_ahash_done(struct crypto_async_request *areq,
 				      int err)
 {
 	u8 *ihash;
 	unsigned int authsize;
 	struct ablkcipher_request *abreq;
 	struct aead_request *req = areq->data;
 	struct crypto_aead *authenc = crypto_aead_reqtfm(req);
 	struct crypto_authenc_ctx *ctx = crypto_aead_ctx(authenc);
 	struct authenc_request_ctx *areq_ctx = aead_request_ctx(req);
 	struct ahash_request *ahreq = (void *)(areq_ctx->tail + ctx->reqoff);
 	unsigned int cryptlen = req->cryptlen;
 	if (err)
 		goto out;
 	authsize = crypto_aead_authsize(authenc);
 	cryptlen -= authsize;
 	ihash = ahreq->result + authsize;
 	scatterwalk_map_and_copy(ihash, areq_ctx->sg, areq_ctx->cryptlen,
 				 authsize, 0);
 	err = crypto_memneq(ihash, ahreq->result, authsize) ? -EBADMSG : 0;
 	if (err)
 		goto out;
 	abreq = aead_request_ctx(req);
 	ablkcipher_request_set_tfm(abreq, ctx->enc);
 	ablkcipher_request_set_callback(abreq, aead_request_flags(req),
 					req->base.complete, req->base.data);
 	ablkcipher_request_set_crypt(abreq, req->src, req->dst,
 				     cryptlen, req->iv);
 	err = crypto_ablkcipher_decrypt(abreq);
 out:
 	authenc_request_complete(req, err);
 }
 static u8 *crypto_authenc_ahash_fb(struct aead_request *req, unsigned int flags)
 {
 	struct crypto_aead *authenc = crypto_aead_reqtfm(req);
 	struct crypto_authenc_ctx *ctx = crypto_aead_ctx(authenc);
 	struct crypto_ahash *auth = ctx->auth;
 	struct authenc_request_ctx *areq_ctx = aead_request_ctx(req);
 	struct ahash_request *ahreq = (void *)(areq_ctx->tail + ctx->reqoff);
 	u8 *hash = areq_ctx->tail;
 	int err;
 	hash = (u8 *)ALIGN((unsigned long)hash + crypto_ahash_alignmask(auth),
 			    crypto_ahash_alignmask(auth) + 1);
 	ahash_request_set_tfm(ahreq, auth);
 	err = crypto_ahash_init(ahreq);
 	if (err)
 		return ERR_PTR(err);
 	ahash_request_set_crypt(ahreq, req->assoc, hash, req->assoclen);
 	ahash_request_set_callback(ahreq, aead_request_flags(req) & flags,
 				   areq_ctx->update_complete, req);
 	err = crypto_ahash_update(ahreq);
 	if (err)
 		return ERR_PTR(err);
 	ahash_request_set_crypt(ahreq, areq_ctx->sg, hash,
 				areq_ctx->cryptlen);
 	ahash_request_set_callback(ahreq, aead_request_flags(req) & flags,
 				   areq_ctx->complete, req);
 	err = crypto_ahash_finup(ahreq);
 	if (err)
 		return ERR_PTR(err);
 	return hash;
 }
 static u8 *crypto_authenc_ahash(struct aead_request *req, unsigned int flags)
 {
 	struct crypto_aead *authenc = crypto_aead_reqtfm(req);
 	struct crypto_authenc_ctx *ctx = crypto_aead_ctx(authenc);
 	struct crypto_ahash *auth = ctx->auth;
 	struct authenc_request_ctx *areq_ctx = aead_request_ctx(req);
 	struct ahash_request *ahreq = (void *)(areq_ctx->tail + ctx->reqoff);
 	u8 *hash = areq_ctx->tail;
 	int err;
 	hash = (u8 *)ALIGN((unsigned long)hash + crypto_ahash_alignmask(auth),
 			   crypto_ahash_alignmask(auth) + 1);
 	ahash_request_set_tfm(ahreq, auth);
 	ahash_request_set_crypt(ahreq, areq_ctx->sg, hash,
 				areq_ctx->cryptlen);
 	ahash_request_set_callback(ahreq, aead_request_flags(req) & flags,
 				   areq_ctx->complete, req);
 	err = crypto_ahash_digest(ahreq);
 	if (err)
 		return ERR_PTR(err);
 	return hash;
 }
 static int crypto_authenc_genicv(struct aead_request *req, u8 *iv,
 				 unsigned int flags)
 {
 	struct crypto_aead *authenc = crypto_aead_reqtfm(req);
 	struct authenc_request_ctx *areq_ctx = aead_request_ctx(req);
 	struct scatterlist *dst = req->dst;
 	struct scatterlist *assoc = req->assoc;
 	struct scatterlist *cipher = areq_ctx->cipher;
 	struct scatterlist *asg = areq_ctx->asg;
 	unsigned int ivsize = crypto_aead_ivsize(authenc);
 	unsigned int cryptlen = req->cryptlen;
 	authenc_ahash_t authenc_ahash_fn = crypto_authenc_ahash_fb;
 	struct page *dstp;
 	u8 *vdst;
 	u8 *hash;
 	dstp = sg_page(dst);
 	vdst = PageHighMem(dstp) ? NULL : page_address(dstp) + dst->offset;
 	if (ivsize) {
 		sg_init_table(cipher, 2);
 		sg_set_buf(cipher, iv, ivsize);
 		scatterwalk_crypto_chain(cipher, dst, vdst == iv + ivsize, 2);
 		dst = cipher;
 		cryptlen += ivsize;
 	}
 	if (req->assoclen && sg_is_last(assoc)) {
 		authenc_ahash_fn = crypto_authenc_ahash;
 		sg_init_table(asg, 2);
 		sg_set_page(asg, sg_page(assoc), assoc->length, assoc->offset);
 		scatterwalk_crypto_chain(asg, dst, 0, 2);
 		dst = asg;
 		cryptlen += req->assoclen;
 	}
 	areq_ctx->cryptlen = cryptlen;
 	areq_ctx->sg = dst;
 	areq_ctx->complete = authenc_geniv_ahash_done;
 	areq_ctx->update_complete = authenc_geniv_ahash_update_done;
 	hash = authenc_ahash_fn(req, flags);
 	if (IS_ERR(hash))
 		return PTR_ERR(hash);
 	scatterwalk_map_and_copy(hash, dst, cryptlen,
 				 crypto_aead_authsize(authenc), 1);
 	return 0;
 }
 static void crypto_authenc_encrypt_done(struct crypto_async_request *req,
 					int err)
 {
 	struct aead_request *areq = req->data;
 	if (!err) {
 		struct crypto_aead *authenc = crypto_aead_reqtfm(areq);
 		struct crypto_authenc_ctx *ctx = crypto_aead_ctx(authenc);
 		struct authenc_request_ctx *areq_ctx = aead_request_ctx(areq);
 		struct ablkcipher_request *abreq = (void *)(areq_ctx->tail
 							    + ctx->reqoff);
 		u8 *iv = (u8 *)abreq - crypto_ablkcipher_ivsize(ctx->enc);
 		err = crypto_authenc_genicv(areq, iv, 0);
 	}
 	authenc_request_complete(areq, err);
 }
 static int crypto_authenc_encrypt(struct aead_request *req)
 {
 	struct crypto_aead *authenc = crypto_aead_reqtfm(req);
 	struct crypto_authenc_ctx *ctx = crypto_aead_ctx(authenc);
 	struct authenc_request_ctx *areq_ctx = aead_request_ctx(req);
 	struct crypto_ablkcipher *enc = ctx->enc;
 	struct scatterlist *dst = req->dst;
 	unsigned int cryptlen = req->cryptlen;
 	struct ablkcipher_request *abreq = (void *)(areq_ctx->tail
 						    + ctx->reqoff);
 	u8 *iv = (u8 *)abreq - crypto_ablkcipher_ivsize(enc);
 	int err;
 	ablkcipher_request_set_tfm(abreq, enc);
 	ablkcipher_request_set_callback(abreq, aead_request_flags(req),
 					crypto_authenc_encrypt_done, req);
 	ablkcipher_request_set_crypt(abreq, req->src, dst, cryptlen, req->iv);
 	memcpy(iv, req->iv, crypto_aead_ivsize(authenc));
 	err = crypto_ablkcipher_encrypt(abreq);
 	if (err)
 		return err;
 	return crypto_authenc_genicv(req, iv, CRYPTO_TFM_REQ_MAY_SLEEP);
 }
 static void crypto_authenc_givencrypt_done(struct crypto_async_request *req,
 					   int err)
 {
 	struct aead_request *areq = req->data;
 	if (!err) {
 		struct skcipher_givcrypt_request *greq = aead_request_ctx(areq);
 		err = crypto_authenc_genicv(areq, greq->giv, 0);
 	}
 	authenc_request_complete(areq, err);
 }
 static int crypto_authenc_givencrypt(struct aead_givcrypt_request *req)
 {
 	struct crypto_aead *authenc = aead_givcrypt_reqtfm(req);
 	struct crypto_authenc_ctx *ctx = crypto_aead_ctx(authenc);
 	struct aead_request *areq = &req->areq;
 	struct skcipher_givcrypt_request *greq = aead_request_ctx(areq);
 	u8 *iv = req->giv;
 	int err;
 	skcipher_givcrypt_set_tfm(greq, ctx->enc);
 	skcipher_givcrypt_set_callback(greq, aead_request_flags(areq),
 				       crypto_authenc_givencrypt_done, areq);
 	skcipher_givcrypt_set_crypt(greq, areq->src, areq->dst, areq->cryptlen,
 				    areq->iv);
 	skcipher_givcrypt_set_giv(greq, iv, req->seq);
 	err = crypto_skcipher_givencrypt(greq);
 	if (err)
 		return err;
 	return crypto_authenc_genicv(areq, iv, CRYPTO_TFM_REQ_MAY_SLEEP);
 }
 static int crypto_authenc_verify(struct aead_request *req,
 				 authenc_ahash_t authenc_ahash_fn)
 {
 	struct crypto_aead *authenc = crypto_aead_reqtfm(req);
 	struct authenc_request_ctx *areq_ctx = aead_request_ctx(req);
 	u8 *ohash;
 	u8 *ihash;
 	unsigned int authsize;
 	areq_ctx->complete = authenc_verify_ahash_done;
 	areq_ctx->update_complete = authenc_verify_ahash_update_done;
 	ohash = authenc_ahash_fn(req, CRYPTO_TFM_REQ_MAY_SLEEP);
 	if (IS_ERR(ohash))
 		return PTR_ERR(ohash);
 	authsize = crypto_aead_authsize(authenc);
 	ihash = ohash + authsize;
 	scatterwalk_map_and_copy(ihash, areq_ctx->sg, areq_ctx->cryptlen,
 				 authsize, 0);
 	return crypto_memneq(ihash, ohash, authsize) ? -EBADMSG : 0;
 }
 static int crypto_authenc_iverify(struct aead_request *req, u8 *iv,
 				  unsigned int cryptlen)
 {
 	struct crypto_aead *authenc = crypto_aead_reqtfm(req);
 	struct authenc_request_ctx *areq_ctx = aead_request_ctx(req);
 	struct scatterlist *src = req->src;
 	struct scatterlist *assoc = req->assoc;
 	struct scatterlist *cipher = areq_ctx->cipher;
 	struct scatterlist *asg = areq_ctx->asg;
 	unsigned int ivsize = crypto_aead_ivsize(authenc);
 	authenc_ahash_t authenc_ahash_fn = crypto_authenc_ahash_fb;
 	struct page *srcp;
 	u8 *vsrc;
 	srcp = sg_page(src);
 	vsrc = PageHighMem(srcp) ? NULL : page_address(srcp) + src->offset;
 	if (ivsize) {
 		sg_init_table(cipher, 2);
 		sg_set_buf(cipher, iv, ivsize);
 		scatterwalk_crypto_chain(cipher, src, vsrc == iv + ivsize, 2);
 		src = cipher;
 		cryptlen += ivsize;
 	}
 	if (req->assoclen && sg_is_last(assoc)) {
 		authenc_ahash_fn = crypto_authenc_ahash;
 		sg_init_table(asg, 2);
 		sg_set_page(asg, sg_page(assoc), assoc->length, assoc->offset);
 		scatterwalk_crypto_chain(asg, src, 0, 2);
 		src = asg;
 		cryptlen += req->assoclen;
 	}
 	areq_ctx->cryptlen = cryptlen;
 	areq_ctx->sg = src;
 	return crypto_authenc_verify(req, authenc_ahash_fn);
 }
 static int crypto_authenc_decrypt(struct aead_request *req)
 {
 	struct crypto_aead *authenc = crypto_aead_reqtfm(req);
 	struct crypto_authenc_ctx *ctx = crypto_aead_ctx(authenc);
 	struct ablkcipher_request *abreq = aead_request_ctx(req);
 	unsigned int cryptlen = req->cryptlen;
 	unsigned int authsize = crypto_aead_authsize(authenc);
 	u8 *iv = req->iv;
 	int err;
 	if (cryptlen < authsize)
 		return -EINVAL;
 	cryptlen -= authsize;
 	err = crypto_authenc_iverify(req, iv, cryptlen);
 	if (err)
 		return err;
 	ablkcipher_request_set_tfm(abreq, ctx->enc);
 	ablkcipher_request_set_callback(abreq, aead_request_flags(req),
 					req->base.complete, req->base.data);
 	ablkcipher_request_set_crypt(abreq, req->src, req->dst, cryptlen, iv);
 	return crypto_ablkcipher_decrypt(abreq);
 }
 static int crypto_authenc_init_tfm(struct crypto_tfm *tfm)
 {
 	struct crypto_instance *inst = crypto_tfm_alg_instance(tfm);
 	struct authenc_instance_ctx *ictx = crypto_instance_ctx(inst);
 	struct crypto_authenc_ctx *ctx = crypto_tfm_ctx(tfm);
 	struct crypto_ahash *auth;
 	struct crypto_ablkcipher *enc;
 	int err;
 	auth = crypto_spawn_ahash(&ictx->auth);
 	if (IS_ERR(auth))
 		return PTR_ERR(auth);
 	enc = crypto_spawn_skcipher(&ictx->enc);
 	err = PTR_ERR(enc);
 	if (IS_ERR(enc))
 		goto err_free_ahash;
 	ctx->auth = auth;
 	ctx->enc = enc;
 	ctx->reqoff = ALIGN(2 * crypto_ahash_digestsize(auth) +
 			    crypto_ahash_alignmask(auth),
 			    crypto_ahash_alignmask(auth) + 1) +
 		      crypto_ablkcipher_ivsize(enc);
 	tfm->crt_aead.reqsize = sizeof(struct authenc_request_ctx) +
 				ctx->reqoff +
 				max_t(unsigned int,
 				crypto_ahash_reqsize(auth) +
 				sizeof(struct ahash_request),
 				sizeof(struct skcipher_givcrypt_request) +
 				crypto_ablkcipher_reqsize(enc));
 	return 0;
 err_free_ahash:
 	crypto_free_ahash(auth);
 	return err;
 }
 static void crypto_authenc_exit_tfm(struct crypto_tfm *tfm)
 {
 	struct crypto_authenc_ctx *ctx = crypto_tfm_ctx(tfm);
 	crypto_free_ahash(ctx->auth);
 	crypto_free_ablkcipher(ctx->enc);
 }
 static struct crypto_instance *crypto_authenc_alloc(struct rtattr **tb)
 {
 	struct crypto_attr_type *algt;
 	struct crypto_instance *inst;
 	struct hash_alg_common *auth;
 	struct crypto_alg *auth_base;
 	struct crypto_alg *enc;
 	struct authenc_instance_ctx *ctx;
 	const char *enc_name;
 	int err;
 	algt = crypto_get_attr_type(tb);
 	if (IS_ERR(algt))
 		return ERR_CAST(algt);
 	if ((algt->type ^ CRYPTO_ALG_TYPE_AEAD) & algt->mask)
 		return ERR_PTR(-EINVAL);
 	auth = ahash_attr_alg(tb[1], CRYPTO_ALG_TYPE_HASH,
 			       CRYPTO_ALG_TYPE_AHASH_MASK);
 	if (IS_ERR(auth))
 		return ERR_CAST(auth);
 	auth_base = &auth->base;
 	enc_name = crypto_attr_alg_name(tb[2]);
 	err = PTR_ERR(enc_name);
 	if (IS_ERR(enc_name))
 		goto out_put_auth;
 	inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL);
 	err = -ENOMEM;
 	if (!inst)
 		goto out_put_auth;
 	ctx = crypto_instance_ctx(inst);
 	err = crypto_init_ahash_spawn(&ctx->auth, auth, inst);
 	if (err)
 		goto err_free_inst;
 	crypto_set_skcipher_spawn(&ctx->enc, inst);
 	err = crypto_grab_skcipher(&ctx->enc, enc_name, 0,
 				   crypto_requires_sync(algt->type,
 							algt->mask));
 	if (err)
 		goto err_drop_auth;
 	enc = crypto_skcipher_spawn_alg(&ctx->enc);
 	err = -ENAMETOOLONG;
 	if (snprintf(inst->alg.cra_name, CRYPTO_MAX_ALG_NAME,
 		     "authenc(%s,%s)", auth_base->cra_name, enc->cra_name) >=
 	    CRYPTO_MAX_ALG_NAME)
 		goto err_drop_enc;
 	if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME,
 		     "authenc(%s,%s)", auth_base->cra_driver_name,
 		     enc->cra_driver_name) >= CRYPTO_MAX_ALG_NAME)
 		goto err_drop_enc;
 	inst->alg.cra_flags = CRYPTO_ALG_TYPE_AEAD;
 	inst->alg.cra_flags |= enc->cra_flags & CRYPTO_ALG_ASYNC;
 	inst->alg.cra_priority = enc->cra_priority *
 				 10 + auth_base->cra_priority;
 	inst->alg.cra_blocksize = enc->cra_blocksize;
 	inst->alg.cra_alignmask = auth_base->cra_alignmask | enc->cra_alignmask;
 	inst->alg.cra_type = &crypto_aead_type;
 	inst->alg.cra_aead.ivsize = enc->cra_ablkcipher.ivsize;
 	inst->alg.cra_aead.maxauthsize = auth->digestsize;
 	inst->alg.cra_ctxsize = sizeof(struct crypto_authenc_ctx);
 	inst->alg.cra_init = crypto_authenc_init_tfm;
 	inst->alg.cra_exit = crypto_authenc_exit_tfm;
 	inst->alg.cra_aead.setkey = crypto_authenc_setkey;
 	inst->alg.cra_aead.encrypt = crypto_authenc_encrypt;
 	inst->alg.cra_aead.decrypt = crypto_authenc_decrypt;
 	inst->alg.cra_aead.givencrypt = crypto_authenc_givencrypt;
 out:
 	crypto_mod_put(auth_base);
 	return inst;
 err_drop_enc:
 	crypto_drop_skcipher(&ctx->enc);
 err_drop_auth:
 	crypto_drop_ahash(&ctx->auth);
 err_free_inst:
 	kfree(inst);
 out_put_auth:
 	inst = ERR_PTR(err);
 	goto out;
 }
 static void crypto_authenc_free(struct crypto_instance *inst)
 {
 	struct authenc_instance_ctx *ctx = crypto_instance_ctx(inst);
 	crypto_drop_skcipher(&ctx->enc);
 	crypto_drop_ahash(&ctx->auth);
 	kfree(inst);
 }
 static struct crypto_template crypto_authenc_tmpl = {
 	.name = "authenc",
 	.alloc = crypto_authenc_alloc,
 	.free = crypto_authenc_free,
 	.module = THIS_MODULE,
 };
 static int __init crypto_authenc_module_init(void)
 {
 	return crypto_register_template(&crypto_authenc_tmpl);
 }
 static void __exit crypto_authenc_module_exit(void)
 {
 	crypto_unregister_template(&crypto_authenc_tmpl);
 }
 module_init(crypto_authenc_module_init);
 module_exit(crypto_authenc_module_exit);
 MODULE_LICENSE("GPL");
 MODULE_DESCRIPTION("Simple AEAD wrapper for IPsec");
+MODULE_ALIAS_CRYPTO("authenc");

crypto/authencesn.c

Diff comments View file @ acc5ccb

 /*
  * authencesn.c - AEAD wrapper for IPsec with extended sequence numbers,
  *                 derived from authenc.c
  *
  * Copyright (C) 2010 secunet Security Networks AG
  * Copyright (C) 2010 Steffen Klassert <steffen.klassert@secunet.com>
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the Free
  * Software Foundation; either version 2 of the License, or (at your option)
  * any later version.
  *
  */
 #include <crypto/aead.h>
 #include <crypto/internal/hash.h>
 #include <crypto/internal/skcipher.h>
 #include <crypto/authenc.h>
 #include <crypto/scatterwalk.h>
 #include <linux/err.h>
 #include <linux/init.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/rtnetlink.h>
 #include <linux/slab.h>
 #include <linux/spinlock.h>
 struct authenc_esn_instance_ctx {
 	struct crypto_ahash_spawn auth;
 	struct crypto_skcipher_spawn enc;
 };
 struct crypto_authenc_esn_ctx {
 	unsigned int reqoff;
 	struct crypto_ahash *auth;
 	struct crypto_ablkcipher *enc;
 };
 struct authenc_esn_request_ctx {
 	unsigned int cryptlen;
 	unsigned int headlen;
 	unsigned int trailen;
 	struct scatterlist *sg;
 	struct scatterlist hsg[2];
 	struct scatterlist tsg[1];
 	struct scatterlist cipher[2];
 	crypto_completion_t complete;
 	crypto_completion_t update_complete;
 	crypto_completion_t update_complete2;
 	char tail[];
 };
 static void authenc_esn_request_complete(struct aead_request *req, int err)
 {
 	if (err != -EINPROGRESS)
 		aead_request_complete(req, err);
 }
 static int crypto_authenc_esn_setkey(struct crypto_aead *authenc_esn, const u8 *key,
 				     unsigned int keylen)
 {
 	struct crypto_authenc_esn_ctx *ctx = crypto_aead_ctx(authenc_esn);
 	struct crypto_ahash *auth = ctx->auth;
 	struct crypto_ablkcipher *enc = ctx->enc;
 	struct crypto_authenc_keys keys;
 	int err = -EINVAL;
 	if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
 		goto badkey;
 	crypto_ahash_clear_flags(auth, CRYPTO_TFM_REQ_MASK);
 	crypto_ahash_set_flags(auth, crypto_aead_get_flags(authenc_esn) &
 				     CRYPTO_TFM_REQ_MASK);
 	err = crypto_ahash_setkey(auth, keys.authkey, keys.authkeylen);
 	crypto_aead_set_flags(authenc_esn, crypto_ahash_get_flags(auth) &
 					   CRYPTO_TFM_RES_MASK);
 	if (err)
 		goto out;
 	crypto_ablkcipher_clear_flags(enc, CRYPTO_TFM_REQ_MASK);
 	crypto_ablkcipher_set_flags(enc, crypto_aead_get_flags(authenc_esn) &
 					 CRYPTO_TFM_REQ_MASK);
 	err = crypto_ablkcipher_setkey(enc, keys.enckey, keys.enckeylen);
 	crypto_aead_set_flags(authenc_esn, crypto_ablkcipher_get_flags(enc) &
 					   CRYPTO_TFM_RES_MASK);
 out:
 	return err;
 badkey:
 	crypto_aead_set_flags(authenc_esn, CRYPTO_TFM_RES_BAD_KEY_LEN);
 	goto out;
 }
 static void authenc_esn_geniv_ahash_update_done(struct crypto_async_request *areq,
 						int err)
 {
 	struct aead_request *req = areq->data;
 	struct crypto_aead *authenc_esn = crypto_aead_reqtfm(req);
 	struct crypto_authenc_esn_ctx *ctx = crypto_aead_ctx(authenc_esn);
 	struct authenc_esn_request_ctx *areq_ctx = aead_request_ctx(req);
 	struct ahash_request *ahreq = (void *)(areq_ctx->tail + ctx->reqoff);
 	if (err)
 		goto out;
 	ahash_request_set_crypt(ahreq, areq_ctx->sg, ahreq->result,
 				areq_ctx->cryptlen);
 	ahash_request_set_callback(ahreq, aead_request_flags(req) &
 					  CRYPTO_TFM_REQ_MAY_SLEEP,
 				   areq_ctx->update_complete2, req);
 	err = crypto_ahash_update(ahreq);
 	if (err)
 		goto out;
 	ahash_request_set_crypt(ahreq, areq_ctx->tsg, ahreq->result,
 				areq_ctx->trailen);
 	ahash_request_set_callback(ahreq, aead_request_flags(req) &
 					  CRYPTO_TFM_REQ_MAY_SLEEP,
 				   areq_ctx->complete, req);
 	err = crypto_ahash_finup(ahreq);
 	if (err)
 		goto out;
 	scatterwalk_map_and_copy(ahreq->result, areq_ctx->sg,
 				 areq_ctx->cryptlen,
 				 crypto_aead_authsize(authenc_esn), 1);
 out:
 	authenc_esn_request_complete(req, err);
 }
 static void authenc_esn_geniv_ahash_update_done2(struct crypto_async_request *areq,
 						 int err)
 {
 	struct aead_request *req = areq->data;
 	struct crypto_aead *authenc_esn = crypto_aead_reqtfm(req);
 	struct crypto_authenc_esn_ctx *ctx = crypto_aead_ctx(authenc_esn);
 	struct authenc_esn_request_ctx *areq_ctx = aead_request_ctx(req);
 	struct ahash_request *ahreq = (void *)(areq_ctx->tail + ctx->reqoff);
 	if (err)
 		goto out;
 	ahash_request_set_crypt(ahreq, areq_ctx->tsg, ahreq->result,
 				areq_ctx->trailen);
 	ahash_request_set_callback(ahreq, aead_request_flags(req) &
 					  CRYPTO_TFM_REQ_MAY_SLEEP,
 				   areq_ctx->complete, req);
 	err = crypto_ahash_finup(ahreq);
 	if (err)
 		goto out;
 	scatterwalk_map_and_copy(ahreq->result, areq_ctx->sg,
 				 areq_ctx->cryptlen,
 				 crypto_aead_authsize(authenc_esn), 1);
 out:
 	authenc_esn_request_complete(req, err);
 }
 static void authenc_esn_geniv_ahash_done(struct crypto_async_request *areq,
 					 int err)
 {
 	struct aead_request *req = areq->data;
 	struct crypto_aead *authenc_esn = crypto_aead_reqtfm(req);
 	struct crypto_authenc_esn_ctx *ctx = crypto_aead_ctx(authenc_esn);
 	struct authenc_esn_request_ctx *areq_ctx = aead_request_ctx(req);
 	struct ahash_request *ahreq = (void *)(areq_ctx->tail + ctx->reqoff);
 	if (err)
 		goto out;
 	scatterwalk_map_and_copy(ahreq->result, areq_ctx->sg,
 				 areq_ctx->cryptlen,
 				 crypto_aead_authsize(authenc_esn), 1);
 out:
 	aead_request_complete(req, err);
 }
 static void authenc_esn_verify_ahash_update_done(struct crypto_async_request *areq,
 						 int err)
 {
 	u8 *ihash;
 	unsigned int authsize;
 	struct ablkcipher_request *abreq;
 	struct aead_request *req = areq->data;
 	struct crypto_aead *authenc_esn = crypto_aead_reqtfm(req);
 	struct crypto_authenc_esn_ctx *ctx = crypto_aead_ctx(authenc_esn);
 	struct authenc_esn_request_ctx *areq_ctx = aead_request_ctx(req);
 	struct ahash_request *ahreq = (void *)(areq_ctx->tail + ctx->reqoff);
 	unsigned int cryptlen = req->cryptlen;
 	if (err)
 		goto out;
 	ahash_request_set_crypt(ahreq, areq_ctx->sg, ahreq->result,
 				areq_ctx->cryptlen);
 	ahash_request_set_callback(ahreq,
 				   aead_request_flags(req) &
 				   CRYPTO_TFM_REQ_MAY_SLEEP,
 				   areq_ctx->update_complete2, req);
 	err = crypto_ahash_update(ahreq);
 	if (err)
 		goto out;
 	ahash_request_set_crypt(ahreq, areq_ctx->tsg, ahreq->result,
 				areq_ctx->trailen);
 	ahash_request_set_callback(ahreq, aead_request_flags(req) &
 					  CRYPTO_TFM_REQ_MAY_SLEEP,
 				   areq_ctx->complete, req);
 	err = crypto_ahash_finup(ahreq);
 	if (err)
 		goto out;
 	authsize = crypto_aead_authsize(authenc_esn);
 	cryptlen -= authsize;
 	ihash = ahreq->result + authsize;
 	scatterwalk_map_and_copy(ihash, areq_ctx->sg, areq_ctx->cryptlen,
 				 authsize, 0);
 	err = crypto_memneq(ihash, ahreq->result, authsize) ? -EBADMSG : 0;
 	if (err)
 		goto out;
 	abreq = aead_request_ctx(req);
 	ablkcipher_request_set_tfm(abreq, ctx->enc);
 	ablkcipher_request_set_callback(abreq, aead_request_flags(req),
 					req->base.complete, req->base.data);
 	ablkcipher_request_set_crypt(abreq, req->src, req->dst,
 				     cryptlen, req->iv);
 	err = crypto_ablkcipher_decrypt(abreq);
 out:
 	authenc_esn_request_complete(req, err);
 }
 static void authenc_esn_verify_ahash_update_done2(struct crypto_async_request *areq,
 						  int err)
 {
 	u8 *ihash;
 	unsigned int authsize;
 	struct ablkcipher_request *abreq;
 	struct aead_request *req = areq->data;
 	struct crypto_aead *authenc_esn = crypto_aead_reqtfm(req);
 	struct crypto_authenc_esn_ctx *ctx = crypto_aead_ctx(authenc_esn);
 	struct authenc_esn_request_ctx *areq_ctx = aead_request_ctx(req);
 	struct ahash_request *ahreq = (void *)(areq_ctx->tail + ctx->reqoff);
 	unsigned int cryptlen = req->cryptlen;
 	if (err)
 		goto out;
 	ahash_request_set_crypt(ahreq, areq_ctx->tsg, ahreq->result,
 				areq_ctx->trailen);
 	ahash_request_set_callback(ahreq, aead_request_flags(req) &
 					  CRYPTO_TFM_REQ_MAY_SLEEP,
 				   areq_ctx->complete, req);
 	err = crypto_ahash_finup(ahreq);
 	if (err)
 		goto out;
 	authsize = crypto_aead_authsize(authenc_esn);
 	cryptlen -= authsize;
 	ihash = ahreq->result + authsize;
 	scatterwalk_map_and_copy(ihash, areq_ctx->sg, areq_ctx->cryptlen,
 				 authsize, 0);
 	err = crypto_memneq(ihash, ahreq->result, authsize) ? -EBADMSG : 0;
 	if (err)
 		goto out;
 	abreq = aead_request_ctx(req);
 	ablkcipher_request_set_tfm(abreq, ctx->enc);
 	ablkcipher_request_set_callback(abreq, aead_request_flags(req),
 					req->base.complete, req->base.data);
 	ablkcipher_request_set_crypt(abreq, req->src, req->dst,
 				     cryptlen, req->iv);
 	err = crypto_ablkcipher_decrypt(abreq);
 out:
 	authenc_esn_request_complete(req, err);
 }
 static void authenc_esn_verify_ahash_done(struct crypto_async_request *areq,
 					  int err)
 {
 	u8 *ihash;
 	unsigned int authsize;
 	struct ablkcipher_request *abreq;
 	struct aead_request *req = areq->data;
 	struct crypto_aead *authenc_esn = crypto_aead_reqtfm(req);
 	struct crypto_authenc_esn_ctx *ctx = crypto_aead_ctx(authenc_esn);
 	struct authenc_esn_request_ctx *areq_ctx = aead_request_ctx(req);
 	struct ahash_request *ahreq = (void *)(areq_ctx->tail + ctx->reqoff);
 	unsigned int cryptlen = req->cryptlen;
 	if (err)
 		goto out;
 	authsize = crypto_aead_authsize(authenc_esn);
 	cryptlen -= authsize;
 	ihash = ahreq->result + authsize;
 	scatterwalk_map_and_copy(ihash, areq_ctx->sg, areq_ctx->cryptlen,
 				 authsize, 0);
 	err = crypto_memneq(ihash, ahreq->result, authsize) ? -EBADMSG : 0;
 	if (err)
 		goto out;
 	abreq = aead_request_ctx(req);
 	ablkcipher_request_set_tfm(abreq, ctx->enc);
 	ablkcipher_request_set_callback(abreq, aead_request_flags(req),
 					req->base.complete, req->base.data);
 	ablkcipher_request_set_crypt(abreq, req->src, req->dst,
 				     cryptlen, req->iv);
 	err = crypto_ablkcipher_decrypt(abreq);
 out:
 	authenc_esn_request_complete(req, err);
 }
 static u8 *crypto_authenc_esn_ahash(struct aead_request *req,
 				    unsigned int flags)
 {
 	struct crypto_aead *authenc_esn = crypto_aead_reqtfm(req);
 	struct crypto_authenc_esn_ctx *ctx = crypto_aead_ctx(authenc_esn);
 	struct crypto_ahash *auth = ctx->auth;
 	struct authenc_esn_request_ctx *areq_ctx = aead_request_ctx(req);
 	struct ahash_request *ahreq = (void *)(areq_ctx->tail + ctx->reqoff);
 	u8 *hash = areq_ctx->tail;
 	int err;
 	hash = (u8 *)ALIGN((unsigned long)hash + crypto_ahash_alignmask(auth),
 			    crypto_ahash_alignmask(auth) + 1);
 	ahash_request_set_tfm(ahreq, auth);
 	err = crypto_ahash_init(ahreq);
 	if (err)
 		return ERR_PTR(err);
 	ahash_request_set_crypt(ahreq, areq_ctx->hsg, hash, areq_ctx->headlen);
 	ahash_request_set_callback(ahreq, aead_request_flags(req) & flags,
 				   areq_ctx->update_complete, req);
 	err = crypto_ahash_update(ahreq);
 	if (err)
 		return ERR_PTR(err);
 	ahash_request_set_crypt(ahreq, areq_ctx->sg, hash, areq_ctx->cryptlen);
 	ahash_request_set_callback(ahreq, aead_request_flags(req) & flags,
 				   areq_ctx->update_complete2, req);
 	err = crypto_ahash_update(ahreq);
 	if (err)
 		return ERR_PTR(err);
 	ahash_request_set_crypt(ahreq, areq_ctx->tsg, hash,
 				areq_ctx->trailen);
 	ahash_request_set_callback(ahreq, aead_request_flags(req) & flags,
 				   areq_ctx->complete, req);
 	err = crypto_ahash_finup(ahreq);
 	if (err)
 		return ERR_PTR(err);
 	return hash;
 }
 static int crypto_authenc_esn_genicv(struct aead_request *req, u8 *iv,
 				     unsigned int flags)
 {
 	struct crypto_aead *authenc_esn = crypto_aead_reqtfm(req);
 	struct authenc_esn_request_ctx *areq_ctx = aead_request_ctx(req);
 	struct scatterlist *dst = req->dst;
 	struct scatterlist *assoc = req->assoc;
 	struct scatterlist *cipher = areq_ctx->cipher;
 	struct scatterlist *hsg = areq_ctx->hsg;
 	struct scatterlist *tsg = areq_ctx->tsg;
 	struct scatterlist *assoc1;
 	struct scatterlist *assoc2;
 	unsigned int ivsize = crypto_aead_ivsize(authenc_esn);
 	unsigned int cryptlen = req->cryptlen;
 	struct page *dstp;
 	u8 *vdst;
 	u8 *hash;
 	dstp = sg_page(dst);
 	vdst = PageHighMem(dstp) ? NULL : page_address(dstp) + dst->offset;
 	if (ivsize) {
 		sg_init_table(cipher, 2);
 		sg_set_buf(cipher, iv, ivsize);
 		scatterwalk_crypto_chain(cipher, dst, vdst == iv + ivsize, 2);
 		dst = cipher;
 		cryptlen += ivsize;
 	}
 	if (sg_is_last(assoc))
 		return -EINVAL;
 	assoc1 = assoc + 1;
 	if (sg_is_last(assoc1))
 		return -EINVAL;
 	assoc2 = assoc + 2;
 	if (!sg_is_last(assoc2))
 		return -EINVAL;
 	sg_init_table(hsg, 2);
 	sg_set_page(hsg, sg_page(assoc), assoc->length, assoc->offset);
 	sg_set_page(hsg + 1, sg_page(assoc2), assoc2->length, assoc2->offset);
 	sg_init_table(tsg, 1);
 	sg_set_page(tsg, sg_page(assoc1), assoc1->length, assoc1->offset);
 	areq_ctx->cryptlen = cryptlen;
 	areq_ctx->headlen = assoc->length + assoc2->length;
 	areq_ctx->trailen = assoc1->length;
 	areq_ctx->sg = dst;
 	areq_ctx->complete = authenc_esn_geniv_ahash_done;
 	areq_ctx->update_complete = authenc_esn_geniv_ahash_update_done;
 	areq_ctx->update_complete2 = authenc_esn_geniv_ahash_update_done2;
 	hash = crypto_authenc_esn_ahash(req, flags);
 	if (IS_ERR(hash))
 		return PTR_ERR(hash);
 	scatterwalk_map_and_copy(hash, dst, cryptlen,
 				 crypto_aead_authsize(authenc_esn), 1);
 	return 0;
 }
 static void crypto_authenc_esn_encrypt_done(struct crypto_async_request *req,
 					    int err)
 {
 	struct aead_request *areq = req->data;
 	if (!err) {
 		struct crypto_aead *authenc_esn = crypto_aead_reqtfm(areq);
 		struct crypto_authenc_esn_ctx *ctx = crypto_aead_ctx(authenc_esn);
 		struct ablkcipher_request *abreq = aead_request_ctx(areq);
 		u8 *iv = (u8 *)(abreq + 1) +
 			 crypto_ablkcipher_reqsize(ctx->enc);
 		err = crypto_authenc_esn_genicv(areq, iv, 0);
 	}
 	authenc_esn_request_complete(areq, err);
 }
 static int crypto_authenc_esn_encrypt(struct aead_request *req)
 {
 	struct crypto_aead *authenc_esn = crypto_aead_reqtfm(req);
 	struct crypto_authenc_esn_ctx *ctx = crypto_aead_ctx(authenc_esn);
 	struct authenc_esn_request_ctx *areq_ctx = aead_request_ctx(req);
 	struct crypto_ablkcipher *enc = ctx->enc;
 	struct scatterlist *dst = req->dst;
 	unsigned int cryptlen = req->cryptlen;
 	struct ablkcipher_request *abreq = (void *)(areq_ctx->tail
 						    + ctx->reqoff);
 	u8 *iv = (u8 *)abreq - crypto_ablkcipher_ivsize(enc);
 	int err;
 	ablkcipher_request_set_tfm(abreq, enc);
 	ablkcipher_request_set_callback(abreq, aead_request_flags(req),
 					crypto_authenc_esn_encrypt_done, req);
 	ablkcipher_request_set_crypt(abreq, req->src, dst, cryptlen, req->iv);
 	memcpy(iv, req->iv, crypto_aead_ivsize(authenc_esn));
 	err = crypto_ablkcipher_encrypt(abreq);
 	if (err)
 		return err;
 	return crypto_authenc_esn_genicv(req, iv, CRYPTO_TFM_REQ_MAY_SLEEP);
 }
 static void crypto_authenc_esn_givencrypt_done(struct crypto_async_request *req,
 					       int err)
 {
 	struct aead_request *areq = req->data;
 	if (!err) {
 		struct skcipher_givcrypt_request *greq = aead_request_ctx(areq);
 		err = crypto_authenc_esn_genicv(areq, greq->giv, 0);
 	}
 	authenc_esn_request_complete(areq, err);
 }
 static int crypto_authenc_esn_givencrypt(struct aead_givcrypt_request *req)
 {
 	struct crypto_aead *authenc_esn = aead_givcrypt_reqtfm(req);
 	struct crypto_authenc_esn_ctx *ctx = crypto_aead_ctx(authenc_esn);
 	struct aead_request *areq = &req->areq;
 	struct skcipher_givcrypt_request *greq = aead_request_ctx(areq);
 	u8 *iv = req->giv;
 	int err;
 	skcipher_givcrypt_set_tfm(greq, ctx->enc);
 	skcipher_givcrypt_set_callback(greq, aead_request_flags(areq),
 				       crypto_authenc_esn_givencrypt_done, areq);
 	skcipher_givcrypt_set_crypt(greq, areq->src, areq->dst, areq->cryptlen,
 				    areq->iv);
 	skcipher_givcrypt_set_giv(greq, iv, req->seq);
 	err = crypto_skcipher_givencrypt(greq);
 	if (err)
 		return err;
 	return crypto_authenc_esn_genicv(areq, iv, CRYPTO_TFM_REQ_MAY_SLEEP);
 }
 static int crypto_authenc_esn_verify(struct aead_request *req)
 {
 	struct crypto_aead *authenc_esn = crypto_aead_reqtfm(req);
 	struct authenc_esn_request_ctx *areq_ctx = aead_request_ctx(req);
 	u8 *ohash;
 	u8 *ihash;
 	unsigned int authsize;
 	areq_ctx->complete = authenc_esn_verify_ahash_done;
 	areq_ctx->update_complete = authenc_esn_verify_ahash_update_done;
 	ohash = crypto_authenc_esn_ahash(req, CRYPTO_TFM_REQ_MAY_SLEEP);
 	if (IS_ERR(ohash))
 		return PTR_ERR(ohash);
 	authsize = crypto_aead_authsize(authenc_esn);
 	ihash = ohash + authsize;
 	scatterwalk_map_and_copy(ihash, areq_ctx->sg, areq_ctx->cryptlen,
 				 authsize, 0);
 	return crypto_memneq(ihash, ohash, authsize) ? -EBADMSG : 0;
 }
 static int crypto_authenc_esn_iverify(struct aead_request *req, u8 *iv,
 				      unsigned int cryptlen)
 {
 	struct crypto_aead *authenc_esn = crypto_aead_reqtfm(req);
 	struct authenc_esn_request_ctx *areq_ctx = aead_request_ctx(req);
 	struct scatterlist *src = req->src;
 	struct scatterlist *assoc = req->assoc;
 	struct scatterlist *cipher = areq_ctx->cipher;
 	struct scatterlist *hsg = areq_ctx->hsg;
 	struct scatterlist *tsg = areq_ctx->tsg;
 	struct scatterlist *assoc1;
 	struct scatterlist *assoc2;
 	unsigned int ivsize = crypto_aead_ivsize(authenc_esn);
 	struct page *srcp;
 	u8 *vsrc;
 	srcp = sg_page(src);
 	vsrc = PageHighMem(srcp) ? NULL : page_address(srcp) + src->offset;
 	if (ivsize) {
 		sg_init_table(cipher, 2);
 		sg_set_buf(cipher, iv, ivsize);
 		scatterwalk_crypto_chain(cipher, src, vsrc == iv + ivsize, 2);
 		src = cipher;
 		cryptlen += ivsize;
 	}
 	if (sg_is_last(assoc))
 		return -EINVAL;
 	assoc1 = assoc + 1;
 	if (sg_is_last(assoc1))
 		return -EINVAL;
 	assoc2 = assoc + 2;
 	if (!sg_is_last(assoc2))
 		return -EINVAL;
 	sg_init_table(hsg, 2);
 	sg_set_page(hsg, sg_page(assoc), assoc->length, assoc->offset);
 	sg_set_page(hsg + 1, sg_page(assoc2), assoc2->length, assoc2->offset);
 	sg_init_table(tsg, 1);
 	sg_set_page(tsg, sg_page(assoc1), assoc1->length, assoc1->offset);
 	areq_ctx->cryptlen = cryptlen;
 	areq_ctx->headlen = assoc->length + assoc2->length;
 	areq_ctx->trailen = assoc1->length;
 	areq_ctx->sg = src;
 	areq_ctx->complete = authenc_esn_verify_ahash_done;
 	areq_ctx->update_complete = authenc_esn_verify_ahash_update_done;
 	areq_ctx->update_complete2 = authenc_esn_verify_ahash_update_done2;
 	return crypto_authenc_esn_verify(req);
 }
 static int crypto_authenc_esn_decrypt(struct aead_request *req)
 {
 	struct crypto_aead *authenc_esn = crypto_aead_reqtfm(req);
 	struct crypto_authenc_esn_ctx *ctx = crypto_aead_ctx(authenc_esn);
 	struct ablkcipher_request *abreq = aead_request_ctx(req);
 	unsigned int cryptlen = req->cryptlen;
 	unsigned int authsize = crypto_aead_authsize(authenc_esn);
 	u8 *iv = req->iv;
 	int err;
 	if (cryptlen < authsize)
 		return -EINVAL;
 	cryptlen -= authsize;
 	err = crypto_authenc_esn_iverify(req, iv, cryptlen);
 	if (err)
 		return err;
 	ablkcipher_request_set_tfm(abreq, ctx->enc);
 	ablkcipher_request_set_callback(abreq, aead_request_flags(req),
 					req->base.complete, req->base.data);
 	ablkcipher_request_set_crypt(abreq, req->src, req->dst, cryptlen, iv);
 	return crypto_ablkcipher_decrypt(abreq);
 }
 static int crypto_authenc_esn_init_tfm(struct crypto_tfm *tfm)
 {
 	struct crypto_instance *inst = crypto_tfm_alg_instance(tfm);
 	struct authenc_esn_instance_ctx *ictx = crypto_instance_ctx(inst);
 	struct crypto_authenc_esn_ctx *ctx = crypto_tfm_ctx(tfm);
 	struct crypto_ahash *auth;
 	struct crypto_ablkcipher *enc;
 	int err;
 	auth = crypto_spawn_ahash(&ictx->auth);
 	if (IS_ERR(auth))
 		return PTR_ERR(auth);
 	enc = crypto_spawn_skcipher(&ictx->enc);
 	err = PTR_ERR(enc);
 	if (IS_ERR(enc))
 		goto err_free_ahash;
 	ctx->auth = auth;
 	ctx->enc = enc;
 	ctx->reqoff = ALIGN(2 * crypto_ahash_digestsize(auth) +
 			    crypto_ahash_alignmask(auth),
 			    crypto_ahash_alignmask(auth) + 1) +
 		      crypto_ablkcipher_ivsize(enc);
 	tfm->crt_aead.reqsize = sizeof(struct authenc_esn_request_ctx) +
 				ctx->reqoff +
 				max_t(unsigned int,
 				crypto_ahash_reqsize(auth) +
 				sizeof(struct ahash_request),
 				sizeof(struct skcipher_givcrypt_request) +
 				crypto_ablkcipher_reqsize(enc));
 	return 0;
 err_free_ahash:
 	crypto_free_ahash(auth);
 	return err;
 }
 static void crypto_authenc_esn_exit_tfm(struct crypto_tfm *tfm)
 {
 	struct crypto_authenc_esn_ctx *ctx = crypto_tfm_ctx(tfm);
 	crypto_free_ahash(ctx->auth);
 	crypto_free_ablkcipher(ctx->enc);
 }
 static struct crypto_instance *crypto_authenc_esn_alloc(struct rtattr **tb)
 {
 	struct crypto_attr_type *algt;
 	struct crypto_instance *inst;
 	struct hash_alg_common *auth;
 	struct crypto_alg *auth_base;
 	struct crypto_alg *enc;
 	struct authenc_esn_instance_ctx *ctx;
 	const char *enc_name;
 	int err;
 	algt = crypto_get_attr_type(tb);
 	if (IS_ERR(algt))
 		return ERR_CAST(algt);
 	if ((algt->type ^ CRYPTO_ALG_TYPE_AEAD) & algt->mask)
 		return ERR_PTR(-EINVAL);
 	auth = ahash_attr_alg(tb[1], CRYPTO_ALG_TYPE_HASH,
 			       CRYPTO_ALG_TYPE_AHASH_MASK);
 	if (IS_ERR(auth))
 		return ERR_CAST(auth);
 	auth_base = &auth->base;
 	enc_name = crypto_attr_alg_name(tb[2]);
 	err = PTR_ERR(enc_name);
 	if (IS_ERR(enc_name))
 		goto out_put_auth;
 	inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL);
 	err = -ENOMEM;
 	if (!inst)
 		goto out_put_auth;
 	ctx = crypto_instance_ctx(inst);
 	err = crypto_init_ahash_spawn(&ctx->auth, auth, inst);
 	if (err)
 		goto err_free_inst;
 	crypto_set_skcipher_spawn(&ctx->enc, inst);
 	err = crypto_grab_skcipher(&ctx->enc, enc_name, 0,
 				   crypto_requires_sync(algt->type,
 							algt->mask));
 	if (err)
 		goto err_drop_auth;
 	enc = crypto_skcipher_spawn_alg(&ctx->enc);
 	err = -ENAMETOOLONG;
 	if (snprintf(inst->alg.cra_name, CRYPTO_MAX_ALG_NAME,
 		     "authencesn(%s,%s)", auth_base->cra_name, enc->cra_name) >=
 	    CRYPTO_MAX_ALG_NAME)
 		goto err_drop_enc;
 	if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME,
 		     "authencesn(%s,%s)", auth_base->cra_driver_name,
 		     enc->cra_driver_name) >= CRYPTO_MAX_ALG_NAME)
 		goto err_drop_enc;
 	inst->alg.cra_flags = CRYPTO_ALG_TYPE_AEAD;
 	inst->alg.cra_flags |= enc->cra_flags & CRYPTO_ALG_ASYNC;
 	inst->alg.cra_priority = enc->cra_priority *
 				 10 + auth_base->cra_priority;
 	inst->alg.cra_blocksize = enc->cra_blocksize;
 	inst->alg.cra_alignmask = auth_base->cra_alignmask | enc->cra_alignmask;
 	inst->alg.cra_type = &crypto_aead_type;
 	inst->alg.cra_aead.ivsize = enc->cra_ablkcipher.ivsize;
 	inst->alg.cra_aead.maxauthsize = auth->digestsize;
 	inst->alg.cra_ctxsize = sizeof(struct crypto_authenc_esn_ctx);
 	inst->alg.cra_init = crypto_authenc_esn_init_tfm;
 	inst->alg.cra_exit = crypto_authenc_esn_exit_tfm;
 	inst->alg.cra_aead.setkey = crypto_authenc_esn_setkey;
 	inst->alg.cra_aead.encrypt = crypto_authenc_esn_encrypt;
 	inst->alg.cra_aead.decrypt = crypto_authenc_esn_decrypt;
 	inst->alg.cra_aead.givencrypt = crypto_authenc_esn_givencrypt;
 out:
 	crypto_mod_put(auth_base);
 	return inst;
 err_drop_enc:
 	crypto_drop_skcipher(&ctx->enc);
 err_drop_auth:
 	crypto_drop_ahash(&ctx->auth);
 err_free_inst:
 	kfree(inst);
 out_put_auth:
 	inst = ERR_PTR(err);
 	goto out;
 }
 static void crypto_authenc_esn_free(struct crypto_instance *inst)
 {
 	struct authenc_esn_instance_ctx *ctx = crypto_instance_ctx(inst);
 	crypto_drop_skcipher(&ctx->enc);
 	crypto_drop_ahash(&ctx->auth);
 	kfree(inst);
 }
 static struct crypto_template crypto_authenc_esn_tmpl = {
 	.name = "authencesn",
 	.alloc = crypto_authenc_esn_alloc,
 	.free = crypto_authenc_esn_free,
 	.module = THIS_MODULE,
 };
 static int __init crypto_authenc_esn_module_init(void)
 {
 	return crypto_register_template(&crypto_authenc_esn_tmpl);
 }
 static void __exit crypto_authenc_esn_module_exit(void)
 {
 	crypto_unregister_template(&crypto_authenc_esn_tmpl);
 }
 module_init(crypto_authenc_esn_module_init);
 module_exit(crypto_authenc_esn_module_exit);
 MODULE_LICENSE("GPL");
 MODULE_AUTHOR("Steffen Klassert <steffen.klassert@secunet.com>");
 MODULE_DESCRIPTION("AEAD wrapper for IPsec with extended sequence numbers");
+MODULE_ALIAS_CRYPTO("authencesn");

crypto/cbc.c

Diff comments View file @ acc5ccb

 /*
  * CBC: Cipher Block Chaining mode
  *
  * Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au>
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the Free
  * Software Foundation; either version 2 of the License, or (at your option)
  * any later version.
  *
  */
 #include <crypto/algapi.h>
 #include <linux/err.h>
 #include <linux/init.h>
 #include <linux/kernel.h>
 #include <linux/log2.h>
 #include <linux/module.h>
 #include <linux/scatterlist.h>
 #include <linux/slab.h>
 struct crypto_cbc_ctx {
 	struct crypto_cipher *child;
 };
 static int crypto_cbc_setkey(struct crypto_tfm *parent, const u8 *key,
 			     unsigned int keylen)
 {
 	struct crypto_cbc_ctx *ctx = crypto_tfm_ctx(parent);
 	struct crypto_cipher *child = ctx->child;
 	int err;
 	crypto_cipher_clear_flags(child, CRYPTO_TFM_REQ_MASK);
 	crypto_cipher_set_flags(child, crypto_tfm_get_flags(parent) &
 				       CRYPTO_TFM_REQ_MASK);
 	err = crypto_cipher_setkey(child, key, keylen);
 	crypto_tfm_set_flags(parent, crypto_cipher_get_flags(child) &
 				     CRYPTO_TFM_RES_MASK);
 	return err;
 }
 static int crypto_cbc_encrypt_segment(struct blkcipher_desc *desc,
 				      struct blkcipher_walk *walk,
 				      struct crypto_cipher *tfm)
 {
 	void (*fn)(struct crypto_tfm *, u8 *, const u8 *) =
 		crypto_cipher_alg(tfm)->cia_encrypt;
 	int bsize = crypto_cipher_blocksize(tfm);
 	unsigned int nbytes = walk->nbytes;
 	u8 *src = walk->src.virt.addr;
 	u8 *dst = walk->dst.virt.addr;
 	u8 *iv = walk->iv;
 	do {
 		crypto_xor(iv, src, bsize);
 		fn(crypto_cipher_tfm(tfm), dst, iv);
 		memcpy(iv, dst, bsize);
 		src += bsize;
 		dst += bsize;
 	} while ((nbytes -= bsize) >= bsize);
 	return nbytes;
 }
 static int crypto_cbc_encrypt_inplace(struct blkcipher_desc *desc,
 				      struct blkcipher_walk *walk,
 				      struct crypto_cipher *tfm)
 {
 	void (*fn)(struct crypto_tfm *, u8 *, const u8 *) =
 		crypto_cipher_alg(tfm)->cia_encrypt;
 	int bsize = crypto_cipher_blocksize(tfm);
 	unsigned int nbytes = walk->nbytes;
 	u8 *src = walk->src.virt.addr;
 	u8 *iv = walk->iv;
 	do {
 		crypto_xor(src, iv, bsize);
 		fn(crypto_cipher_tfm(tfm), src, src);
 		iv = src;
 		src += bsize;
 	} while ((nbytes -= bsize) >= bsize);
 	memcpy(walk->iv, iv, bsize);
 	return nbytes;
 }
 static int crypto_cbc_encrypt(struct blkcipher_desc *desc,
 			      struct scatterlist *dst, struct scatterlist *src,
 			      unsigned int nbytes)
 {
 	struct blkcipher_walk walk;
 	struct crypto_blkcipher *tfm = desc->tfm;
 	struct crypto_cbc_ctx *ctx = crypto_blkcipher_ctx(tfm);
 	struct crypto_cipher *child = ctx->child;
 	int err;
 	blkcipher_walk_init(&walk, dst, src, nbytes);
 	err = blkcipher_walk_virt(desc, &walk);
 	while ((nbytes = walk.nbytes)) {
 		if (walk.src.virt.addr == walk.dst.virt.addr)
 			nbytes = crypto_cbc_encrypt_inplace(desc, &walk, child);
 		else
 			nbytes = crypto_cbc_encrypt_segment(desc, &walk, child);
 		err = blkcipher_walk_done(desc, &walk, nbytes);
 	}
 	return err;
 }
 static int crypto_cbc_decrypt_segment(struct blkcipher_desc *desc,
 				      struct blkcipher_walk *walk,
 				      struct crypto_cipher *tfm)
 {
 	void (*fn)(struct crypto_tfm *, u8 *, const u8 *) =
 		crypto_cipher_alg(tfm)->cia_decrypt;
 	int bsize = crypto_cipher_blocksize(tfm);
 	unsigned int nbytes = walk->nbytes;
 	u8 *src = walk->src.virt.addr;
 	u8 *dst = walk->dst.virt.addr;
 	u8 *iv = walk->iv;
 	do {
 		fn(crypto_cipher_tfm(tfm), dst, src);
 		crypto_xor(dst, iv, bsize);
 		iv = src;
 		src += bsize;
 		dst += bsize;
 	} while ((nbytes -= bsize) >= bsize);
 	memcpy(walk->iv, iv, bsize);
 	return nbytes;
 }
 static int crypto_cbc_decrypt_inplace(struct blkcipher_desc *desc,
 				      struct blkcipher_walk *walk,
 				      struct crypto_cipher *tfm)
 {
 	void (*fn)(struct crypto_tfm *, u8 *, const u8 *) =
 		crypto_cipher_alg(tfm)->cia_decrypt;
 	int bsize = crypto_cipher_blocksize(tfm);
 	unsigned int nbytes = walk->nbytes;
 	u8 *src = walk->src.virt.addr;
 	u8 last_iv[bsize];
 	/* Start of the last block. */
 	src += nbytes - (nbytes & (bsize - 1)) - bsize;
 	memcpy(last_iv, src, bsize);
 	for (;;) {
 		fn(crypto_cipher_tfm(tfm), src, src);
 		if ((nbytes -= bsize) < bsize)
 			break;
 		crypto_xor(src, src - bsize, bsize);
 		src -= bsize;
 	}
 	crypto_xor(src, walk->iv, bsize);
 	memcpy(walk->iv, last_iv, bsize);
 	return nbytes;
 }
 static int crypto_cbc_decrypt(struct blkcipher_desc *desc,
 			      struct scatterlist *dst, struct scatterlist *src,
 			      unsigned int nbytes)
 {
 	struct blkcipher_walk walk;
 	struct crypto_blkcipher *tfm = desc->tfm;
 	struct crypto_cbc_ctx *ctx = crypto_blkcipher_ctx(tfm);
 	struct crypto_cipher *child = ctx->child;
 	int err;
 	blkcipher_walk_init(&walk, dst, src, nbytes);
 	err = blkcipher_walk_virt(desc, &walk);
 	while ((nbytes = walk.nbytes)) {
 		if (walk.src.virt.addr == walk.dst.virt.addr)
 			nbytes = crypto_cbc_decrypt_inplace(desc, &walk, child);
 		else
 			nbytes = crypto_cbc_decrypt_segment(desc, &walk, child);
 		err = blkcipher_walk_done(desc, &walk, nbytes);
 	}
 	return err;
 }
 static int crypto_cbc_init_tfm(struct crypto_tfm *tfm)
 {
 	struct crypto_instance *inst = (void *)tfm->__crt_alg;
 	struct crypto_spawn *spawn = crypto_instance_ctx(inst);
 	struct crypto_cbc_ctx *ctx = crypto_tfm_ctx(tfm);
 	struct crypto_cipher *cipher;
 	cipher = crypto_spawn_cipher(spawn);
 	if (IS_ERR(cipher))
 		return PTR_ERR(cipher);
 	ctx->child = cipher;
 	return 0;
 }
 static void crypto_cbc_exit_tfm(struct crypto_tfm *tfm)
 {
 	struct crypto_cbc_ctx *ctx = crypto_tfm_ctx(tfm);
 	crypto_free_cipher(ctx->child);
 }
 static struct crypto_instance *crypto_cbc_alloc(struct rtattr **tb)
 {
 	struct crypto_instance *inst;
 	struct crypto_alg *alg;
 	int err;
 	err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_BLKCIPHER);
 	if (err)
 		return ERR_PTR(err);
 	alg = crypto_get_attr_alg(tb, CRYPTO_ALG_TYPE_CIPHER,
 				  CRYPTO_ALG_TYPE_MASK);
 	if (IS_ERR(alg))
 		return ERR_CAST(alg);
 	inst = ERR_PTR(-EINVAL);
 	if (!is_power_of_2(alg->cra_blocksize))
 		goto out_put_alg;
 	inst = crypto_alloc_instance("cbc", alg);
 	if (IS_ERR(inst))
 		goto out_put_alg;
 	inst->alg.cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER;
 	inst->alg.cra_priority = alg->cra_priority;
 	inst->alg.cra_blocksize = alg->cra_blocksize;
 	inst->alg.cra_alignmask = alg->cra_alignmask;
 	inst->alg.cra_type = &crypto_blkcipher_type;
 	/* We access the data as u32s when xoring. */
 	inst->alg.cra_alignmask |= __alignof__(u32) - 1;
 	inst->alg.cra_blkcipher.ivsize = alg->cra_blocksize;
 	inst->alg.cra_blkcipher.min_keysize = alg->cra_cipher.cia_min_keysize;
 	inst->alg.cra_blkcipher.max_keysize = alg->cra_cipher.cia_max_keysize;
 	inst->alg.cra_ctxsize = sizeof(struct crypto_cbc_ctx);
 	inst->alg.cra_init = crypto_cbc_init_tfm;
 	inst->alg.cra_exit = crypto_cbc_exit_tfm;
 	inst->alg.cra_blkcipher.setkey = crypto_cbc_setkey;
 	inst->alg.cra_blkcipher.encrypt = crypto_cbc_encrypt;
 	inst->alg.cra_blkcipher.decrypt = crypto_cbc_decrypt;
 out_put_alg:
 	crypto_mod_put(alg);
 	return inst;
 }
 static void crypto_cbc_free(struct crypto_instance *inst)
 {
 	crypto_drop_spawn(crypto_instance_ctx(inst));
 	kfree(inst);
 }
 static struct crypto_template crypto_cbc_tmpl = {
 	.name = "cbc",
 	.alloc = crypto_cbc_alloc,
 	.free = crypto_cbc_free,
 	.module = THIS_MODULE,
 };
 static int __init crypto_cbc_module_init(void)
 {
 	return crypto_register_template(&crypto_cbc_tmpl);
 }
 static void __exit crypto_cbc_module_exit(void)
 {
 	crypto_unregister_template(&crypto_cbc_tmpl);
 }
 module_init(crypto_cbc_module_init);
 module_exit(crypto_cbc_module_exit);
 MODULE_LICENSE("GPL");
 MODULE_DESCRIPTION("CBC block cipher algorithm");
+MODULE_ALIAS_CRYPTO("cbc");

crypto/ccm.c

Diff comments View file @ acc5ccb

 /*
  * CCM: Counter with CBC-MAC
  *
  * (C) Copyright IBM Corp. 2007 - Joy Latten <latten@us.ibm.com>
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the Free
  * Software Foundation; either version 2 of the License, or (at your option)
  * any later version.
  *
  */
 #include <crypto/internal/aead.h>
 #include <crypto/internal/skcipher.h>
 #include <crypto/scatterwalk.h>
 #include <linux/err.h>
 #include <linux/init.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/slab.h>
 #include "internal.h"
 struct ccm_instance_ctx {
 	struct crypto_skcipher_spawn ctr;
 	struct crypto_spawn cipher;
 };
 struct crypto_ccm_ctx {
 	struct crypto_cipher *cipher;
 	struct crypto_ablkcipher *ctr;
 };
 struct crypto_rfc4309_ctx {
 	struct crypto_aead *child;
 	u8 nonce[3];
 };
 struct crypto_ccm_req_priv_ctx {
 	u8 odata[16];
 	u8 idata[16];
 	u8 auth_tag[16];
 	u32 ilen;
 	u32 flags;
 	struct scatterlist src[2];
 	struct scatterlist dst[2];
 	struct ablkcipher_request abreq;
 };
 static inline struct crypto_ccm_req_priv_ctx *crypto_ccm_reqctx(
 	struct aead_request *req)
 {
 	unsigned long align = crypto_aead_alignmask(crypto_aead_reqtfm(req));
 	return (void *)PTR_ALIGN((u8 *)aead_request_ctx(req), align + 1);
 }
 static int set_msg_len(u8 *block, unsigned int msglen, int csize)
 {
 	__be32 data;
 	memset(block, 0, csize);
 	block += csize;
 	if (csize >= 4)
 		csize = 4;
 	else if (msglen > (1 << (8 * csize)))
 		return -EOVERFLOW;
 	data = cpu_to_be32(msglen);
 	memcpy(block - csize, (u8 *)&data + 4 - csize, csize);
 	return 0;
 }
 static int crypto_ccm_setkey(struct crypto_aead *aead, const u8 *key,
 			     unsigned int keylen)
 {
 	struct crypto_ccm_ctx *ctx = crypto_aead_ctx(aead);
 	struct crypto_ablkcipher *ctr = ctx->ctr;
 	struct crypto_cipher *tfm = ctx->cipher;
 	int err = 0;
 	crypto_ablkcipher_clear_flags(ctr, CRYPTO_TFM_REQ_MASK);
 	crypto_ablkcipher_set_flags(ctr, crypto_aead_get_flags(aead) &
 				    CRYPTO_TFM_REQ_MASK);
 	err = crypto_ablkcipher_setkey(ctr, key, keylen);
 	crypto_aead_set_flags(aead, crypto_ablkcipher_get_flags(ctr) &
 			      CRYPTO_TFM_RES_MASK);
 	if (err)
 		goto out;
 	crypto_cipher_clear_flags(tfm, CRYPTO_TFM_REQ_MASK);
 	crypto_cipher_set_flags(tfm, crypto_aead_get_flags(aead) &
 				    CRYPTO_TFM_REQ_MASK);
 	err = crypto_cipher_setkey(tfm, key, keylen);
 	crypto_aead_set_flags(aead, crypto_cipher_get_flags(tfm) &
 			      CRYPTO_TFM_RES_MASK);
 out:
 	return err;
 }
 static int crypto_ccm_setauthsize(struct crypto_aead *tfm,
 				  unsigned int authsize)
 {
 	switch (authsize) {
 	case 4:
 	case 6:
 	case 8:
 	case 10:
 	case 12:
 	case 14:
 	case 16:
 		break;
 	default:
 		return -EINVAL;
 	}
 	return 0;
 }
 static int format_input(u8 *info, struct aead_request *req,
 			unsigned int cryptlen)
 {
 	struct crypto_aead *aead = crypto_aead_reqtfm(req);
 	unsigned int lp = req->iv[0];
 	unsigned int l = lp + 1;
 	unsigned int m;
 	m = crypto_aead_authsize(aead);
 	memcpy(info, req->iv, 16);
 	/* format control info per RFC 3610 and
 	 * NIST Special Publication 800-38C
 	 */
 	*info |= (8 * ((m - 2) / 2));
 	if (req->assoclen)
 		*info |= 64;
 	return set_msg_len(info + 16 - l, cryptlen, l);
 }
 static int format_adata(u8 *adata, unsigned int a)
 {
 	int len = 0;
 	/* add control info for associated data
 	 * RFC 3610 and NIST Special Publication 800-38C
 	 */
 	if (a < 65280) {
 		*(__be16 *)adata = cpu_to_be16(a);
 		len = 2;
 	} else  {
 		*(__be16 *)adata = cpu_to_be16(0xfffe);
 		*(__be32 *)&adata[2] = cpu_to_be32(a);
 		len = 6;
 	}
 	return len;
 }
 static void compute_mac(struct crypto_cipher *tfm, u8 *data, int n,
 		       struct crypto_ccm_req_priv_ctx *pctx)
 {
 	unsigned int bs = 16;
 	u8 *odata = pctx->odata;
 	u8 *idata = pctx->idata;
 	int datalen, getlen;
 	datalen = n;
 	/* first time in here, block may be partially filled. */
 	getlen = bs - pctx->ilen;
 	if (datalen >= getlen) {
 		memcpy(idata + pctx->ilen, data, getlen);
 		crypto_xor(odata, idata, bs);
 		crypto_cipher_encrypt_one(tfm, odata, odata);
 		datalen -= getlen;
 		data += getlen;
 		pctx->ilen = 0;
 	}
 	/* now encrypt rest of data */
 	while (datalen >= bs) {
 		crypto_xor(odata, data, bs);
 		crypto_cipher_encrypt_one(tfm, odata, odata);
 		datalen -= bs;
 		data += bs;
 	}
 	/* check and see if there's leftover data that wasn't
 	 * enough to fill a block.
 	 */
 	if (datalen) {
 		memcpy(idata + pctx->ilen, data, datalen);
 		pctx->ilen += datalen;
 	}
 }
 static void get_data_to_compute(struct crypto_cipher *tfm,
 			       struct crypto_ccm_req_priv_ctx *pctx,
 			       struct scatterlist *sg, unsigned int len)
 {
 	struct scatter_walk walk;
 	u8 *data_src;
 	int n;
 	scatterwalk_start(&walk, sg);
 	while (len) {
 		n = scatterwalk_clamp(&walk, len);
 		if (!n) {
 			scatterwalk_start(&walk, sg_next(walk.sg));
 			n = scatterwalk_clamp(&walk, len);
 		}
 		data_src = scatterwalk_map(&walk);
 		compute_mac(tfm, data_src, n, pctx);
 		len -= n;
 		scatterwalk_unmap(data_src);
 		scatterwalk_advance(&walk, n);
 		scatterwalk_done(&walk, 0, len);
 		if (len)
 			crypto_yield(pctx->flags);
 	}
 	/* any leftover needs padding and then encrypted */
 	if (pctx->ilen) {
 		int padlen;
 		u8 *odata = pctx->odata;
 		u8 *idata = pctx->idata;
 		padlen = 16 - pctx->ilen;
 		memset(idata + pctx->ilen, 0, padlen);
 		crypto_xor(odata, idata, 16);
 		crypto_cipher_encrypt_one(tfm, odata, odata);
 		pctx->ilen = 0;
 	}
 }
 static int crypto_ccm_auth(struct aead_request *req, struct scatterlist *plain,
 			   unsigned int cryptlen)
 {
 	struct crypto_aead *aead = crypto_aead_reqtfm(req);
 	struct crypto_ccm_ctx *ctx = crypto_aead_ctx(aead);
 	struct crypto_ccm_req_priv_ctx *pctx = crypto_ccm_reqctx(req);
 	struct crypto_cipher *cipher = ctx->cipher;
 	unsigned int assoclen = req->assoclen;
 	u8 *odata = pctx->odata;
 	u8 *idata = pctx->idata;
 	int err;
 	/* format control data for input */
 	err = format_input(odata, req, cryptlen);
 	if (err)
 		goto out;
 	/* encrypt first block to use as start in computing mac  */
 	crypto_cipher_encrypt_one(cipher, odata, odata);
 	/* format associated data and compute into mac */
 	if (assoclen) {
 		pctx->ilen = format_adata(idata, assoclen);
 		get_data_to_compute(cipher, pctx, req->assoc, req->assoclen);
 	} else {
 		pctx->ilen = 0;
 	}
 	/* compute plaintext into mac */
 	if (cryptlen)
 		get_data_to_compute(cipher, pctx, plain, cryptlen);
 out:
 	return err;
 }
 static void crypto_ccm_encrypt_done(struct crypto_async_request *areq, int err)
 {
 	struct aead_request *req = areq->data;
 	struct crypto_aead *aead = crypto_aead_reqtfm(req);
 	struct crypto_ccm_req_priv_ctx *pctx = crypto_ccm_reqctx(req);
 	u8 *odata = pctx->odata;
 	if (!err)
 		scatterwalk_map_and_copy(odata, req->dst, req->cryptlen,
 					 crypto_aead_authsize(aead), 1);
 	aead_request_complete(req, err);
 }
 static inline int crypto_ccm_check_iv(const u8 *iv)
 {
 	/* 2 <= L <= 8, so 1 <= L' <= 7. */
 	if (1 > iv[0] || iv[0] > 7)
 		return -EINVAL;
 	return 0;
 }
 static int crypto_ccm_encrypt(struct aead_request *req)
 {
 	struct crypto_aead *aead = crypto_aead_reqtfm(req);
 	struct crypto_ccm_ctx *ctx = crypto_aead_ctx(aead);
 	struct crypto_ccm_req_priv_ctx *pctx = crypto_ccm_reqctx(req);
 	struct ablkcipher_request *abreq = &pctx->abreq;
 	struct scatterlist *dst;
 	unsigned int cryptlen = req->cryptlen;
 	u8 *odata = pctx->odata;
 	u8 *iv = req->iv;
 	int err;
 	err = crypto_ccm_check_iv(iv);
 	if (err)
 		return err;
 	pctx->flags = aead_request_flags(req);
 	err = crypto_ccm_auth(req, req->src, cryptlen);
 	if (err)
 		return err;
 	 /* Note: rfc 3610 and NIST 800-38C require counter of
 	 * zero to encrypt auth tag.
 	 */
 	memset(iv + 15 - iv[0], 0, iv[0] + 1);
 	sg_init_table(pctx->src, 2);
 	sg_set_buf(pctx->src, odata, 16);
 	scatterwalk_sg_chain(pctx->src, 2, req->src);
 	dst = pctx->src;
 	if (req->src != req->dst) {
 		sg_init_table(pctx->dst, 2);
 		sg_set_buf(pctx->dst, odata, 16);
 		scatterwalk_sg_chain(pctx->dst, 2, req->dst);
 		dst = pctx->dst;
 	}
 	ablkcipher_request_set_tfm(abreq, ctx->ctr);
 	ablkcipher_request_set_callback(abreq, pctx->flags,
 					crypto_ccm_encrypt_done, req);
 	ablkcipher_request_set_crypt(abreq, pctx->src, dst, cryptlen + 16, iv);
 	err = crypto_ablkcipher_encrypt(abreq);
 	if (err)
 		return err;
 	/* copy authtag to end of dst */
 	scatterwalk_map_and_copy(odata, req->dst, cryptlen,
 				 crypto_aead_authsize(aead), 1);
 	return err;
 }
 static void crypto_ccm_decrypt_done(struct crypto_async_request *areq,
 				   int err)
 {
 	struct aead_request *req = areq->data;
 	struct crypto_ccm_req_priv_ctx *pctx = crypto_ccm_reqctx(req);
 	struct crypto_aead *aead = crypto_aead_reqtfm(req);
 	unsigned int authsize = crypto_aead_authsize(aead);
 	unsigned int cryptlen = req->cryptlen - authsize;
 	if (!err) {
 		err = crypto_ccm_auth(req, req->dst, cryptlen);
 		if (!err && crypto_memneq(pctx->auth_tag, pctx->odata, authsize))
 			err = -EBADMSG;
 	}
 	aead_request_complete(req, err);
 }
 static int crypto_ccm_decrypt(struct aead_request *req)
 {
 	struct crypto_aead *aead = crypto_aead_reqtfm(req);
 	struct crypto_ccm_ctx *ctx = crypto_aead_ctx(aead);
 	struct crypto_ccm_req_priv_ctx *pctx = crypto_ccm_reqctx(req);
 	struct ablkcipher_request *abreq = &pctx->abreq;
 	struct scatterlist *dst;
 	unsigned int authsize = crypto_aead_authsize(aead);
 	unsigned int cryptlen = req->cryptlen;
 	u8 *authtag = pctx->auth_tag;
 	u8 *odata = pctx->odata;
 	u8 *iv = req->iv;
 	int err;
 	if (cryptlen < authsize)
 		return -EINVAL;
 	cryptlen -= authsize;
 	err = crypto_ccm_check_iv(iv);
 	if (err)
 		return err;
 	pctx->flags = aead_request_flags(req);
 	scatterwalk_map_and_copy(authtag, req->src, cryptlen, authsize, 0);
 	memset(iv + 15 - iv[0], 0, iv[0] + 1);
 	sg_init_table(pctx->src, 2);
 	sg_set_buf(pctx->src, authtag, 16);
 	scatterwalk_sg_chain(pctx->src, 2, req->src);
 	dst = pctx->src;
 	if (req->src != req->dst) {
 		sg_init_table(pctx->dst, 2);
 		sg_set_buf(pctx->dst, authtag, 16);
 		scatterwalk_sg_chain(pctx->dst, 2, req->dst);
 		dst = pctx->dst;
 	}
 	ablkcipher_request_set_tfm(abreq, ctx->ctr);
 	ablkcipher_request_set_callback(abreq, pctx->flags,
 					crypto_ccm_decrypt_done, req);
 	ablkcipher_request_set_crypt(abreq, pctx->src, dst, cryptlen + 16, iv);
 	err = crypto_ablkcipher_decrypt(abreq);
 	if (err)
 		return err;
 	err = crypto_ccm_auth(req, req->dst, cryptlen);
 	if (err)
 		return err;
 	/* verify */
 	if (crypto_memneq(authtag, odata, authsize))
 		return -EBADMSG;
 	return err;
 }
 static int crypto_ccm_init_tfm(struct crypto_tfm *tfm)
 {
 	struct crypto_instance *inst = (void *)tfm->__crt_alg;
 	struct ccm_instance_ctx *ictx = crypto_instance_ctx(inst);
 	struct crypto_ccm_ctx *ctx = crypto_tfm_ctx(tfm);
 	struct crypto_cipher *cipher;
 	struct crypto_ablkcipher *ctr;
 	unsigned long align;
 	int err;
 	cipher = crypto_spawn_cipher(&ictx->cipher);
 	if (IS_ERR(cipher))
 		return PTR_ERR(cipher);
 	ctr = crypto_spawn_skcipher(&ictx->ctr);
 	err = PTR_ERR(ctr);
 	if (IS_ERR(ctr))
 		goto err_free_cipher;
 	ctx->cipher = cipher;
 	ctx->ctr = ctr;
 	align = crypto_tfm_alg_alignmask(tfm);
 	align &= ~(crypto_tfm_ctx_alignment() - 1);
 	tfm->crt_aead.reqsize = align +
 				sizeof(struct crypto_ccm_req_priv_ctx) +
 				crypto_ablkcipher_reqsize(ctr);
 	return 0;
 err_free_cipher:
 	crypto_free_cipher(cipher);
 	return err;
 }
 static void crypto_ccm_exit_tfm(struct crypto_tfm *tfm)
 {
 	struct crypto_ccm_ctx *ctx = crypto_tfm_ctx(tfm);
 	crypto_free_cipher(ctx->cipher);
 	crypto_free_ablkcipher(ctx->ctr);
 }
 static struct crypto_instance *crypto_ccm_alloc_common(struct rtattr **tb,
 						       const char *full_name,
 						       const char *ctr_name,
 						       const char *cipher_name)
 {
 	struct crypto_attr_type *algt;
 	struct crypto_instance *inst;
 	struct crypto_alg *ctr;
 	struct crypto_alg *cipher;
 	struct ccm_instance_ctx *ictx;
 	int err;
 	algt = crypto_get_attr_type(tb);
 	if (IS_ERR(algt))
 		return ERR_CAST(algt);
 	if ((algt->type ^ CRYPTO_ALG_TYPE_AEAD) & algt->mask)
 		return ERR_PTR(-EINVAL);
 	cipher = crypto_alg_mod_lookup(cipher_name,  CRYPTO_ALG_TYPE_CIPHER,
 				       CRYPTO_ALG_TYPE_MASK);
 	if (IS_ERR(cipher))
 		return ERR_CAST(cipher);
 	err = -EINVAL;
 	if (cipher->cra_blocksize != 16)
 		goto out_put_cipher;
 	inst = kzalloc(sizeof(*inst) + sizeof(*ictx), GFP_KERNEL);
 	err = -ENOMEM;
 	if (!inst)
 		goto out_put_cipher;
 	ictx = crypto_instance_ctx(inst);
 	err = crypto_init_spawn(&ictx->cipher, cipher, inst,
 				CRYPTO_ALG_TYPE_MASK);
 	if (err)
 		goto err_free_inst;
 	crypto_set_skcipher_spawn(&ictx->ctr, inst);
 	err = crypto_grab_skcipher(&ictx->ctr, ctr_name, 0,
 				   crypto_requires_sync(algt->type,
 							algt->mask));
 	if (err)
 		goto err_drop_cipher;
 	ctr = crypto_skcipher_spawn_alg(&ictx->ctr);
 	/* Not a stream cipher? */
 	err = -EINVAL;
 	if (ctr->cra_blocksize != 1)
 		goto err_drop_ctr;
 	/* We want the real thing! */
 	if (ctr->cra_ablkcipher.ivsize != 16)
 		goto err_drop_ctr;
 	err = -ENAMETOOLONG;
 	if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME,
 		     "ccm_base(%s,%s)", ctr->cra_driver_name,
 		     cipher->cra_driver_name) >= CRYPTO_MAX_ALG_NAME)
 		goto err_drop_ctr;
 	memcpy(inst->alg.cra_name, full_name, CRYPTO_MAX_ALG_NAME);
 	inst->alg.cra_flags = CRYPTO_ALG_TYPE_AEAD;
 	inst->alg.cra_flags |= ctr->cra_flags & CRYPTO_ALG_ASYNC;
 	inst->alg.cra_priority = cipher->cra_priority + ctr->cra_priority;
 	inst->alg.cra_blocksize = 1;
 	inst->alg.cra_alignmask = cipher->cra_alignmask | ctr->cra_alignmask |
 				  (__alignof__(u32) - 1);
 	inst->alg.cra_type = &crypto_aead_type;
 	inst->alg.cra_aead.ivsize = 16;
 	inst->alg.cra_aead.maxauthsize = 16;
 	inst->alg.cra_ctxsize = sizeof(struct crypto_ccm_ctx);
 	inst->alg.cra_init = crypto_ccm_init_tfm;
 	inst->alg.cra_exit = crypto_ccm_exit_tfm;
 	inst->alg.cra_aead.setkey = crypto_ccm_setkey;
 	inst->alg.cra_aead.setauthsize = crypto_ccm_setauthsize;
 	inst->alg.cra_aead.encrypt = crypto_ccm_encrypt;
 	inst->alg.cra_aead.decrypt = crypto_ccm_decrypt;
 out:
 	crypto_mod_put(cipher);
 	return inst;
 err_drop_ctr:
 	crypto_drop_skcipher(&ictx->ctr);
 err_drop_cipher:
 	crypto_drop_spawn(&ictx->cipher);
 err_free_inst:
 	kfree(inst);
 out_put_cipher:
 	inst = ERR_PTR(err);
 	goto out;
 }
 static struct crypto_instance *crypto_ccm_alloc(struct rtattr **tb)
 {
 	const char *cipher_name;
 	char ctr_name[CRYPTO_MAX_ALG_NAME];
 	char full_name[CRYPTO_MAX_ALG_NAME];
 	cipher_name = crypto_attr_alg_name(tb[1]);
 	if (IS_ERR(cipher_name))
 		return ERR_CAST(cipher_name);
 	if (snprintf(ctr_name, CRYPTO_MAX_ALG_NAME, "ctr(%s)",
 		     cipher_name) >= CRYPTO_MAX_ALG_NAME)
 		return ERR_PTR(-ENAMETOOLONG);
 	if (snprintf(full_name, CRYPTO_MAX_ALG_NAME, "ccm(%s)", cipher_name) >=
 	    CRYPTO_MAX_ALG_NAME)
 		return ERR_PTR(-ENAMETOOLONG);
 	return crypto_ccm_alloc_common(tb, full_name, ctr_name, cipher_name);
 }
 static void crypto_ccm_free(struct crypto_instance *inst)
 {
 	struct ccm_instance_ctx *ctx = crypto_instance_ctx(inst);
 	crypto_drop_spawn(&ctx->cipher);
 	crypto_drop_skcipher(&ctx->ctr);
 	kfree(inst);
 }
 static struct crypto_template crypto_ccm_tmpl = {
 	.name = "ccm",
 	.alloc = crypto_ccm_alloc,
 	.free = crypto_ccm_free,
 	.module = THIS_MODULE,
 };
 static struct crypto_instance *crypto_ccm_base_alloc(struct rtattr **tb)
 {
 	const char *ctr_name;
 	const char *cipher_name;
 	char full_name[CRYPTO_MAX_ALG_NAME];
 	ctr_name = crypto_attr_alg_name(tb[1]);
 	if (IS_ERR(ctr_name))
 		return ERR_CAST(ctr_name);
 	cipher_name = crypto_attr_alg_name(tb[2]);
 	if (IS_ERR(cipher_name))
 		return ERR_CAST(cipher_name);
 	if (snprintf(full_name, CRYPTO_MAX_ALG_NAME, "ccm_base(%s,%s)",
 		     ctr_name, cipher_name) >= CRYPTO_MAX_ALG_NAME)
 		return ERR_PTR(-ENAMETOOLONG);
 	return crypto_ccm_alloc_common(tb, full_name, ctr_name, cipher_name);
 }
 static struct crypto_template crypto_ccm_base_tmpl = {
 	.name = "ccm_base",
 	.alloc = crypto_ccm_base_alloc,
 	.free = crypto_ccm_free,
 	.module = THIS_MODULE,
 };
 static int crypto_rfc4309_setkey(struct crypto_aead *parent, const u8 *key,
 				 unsigned int keylen)
 {
 	struct crypto_rfc4309_ctx *ctx = crypto_aead_ctx(parent);
 	struct crypto_aead *child = ctx->child;
 	int err;
 	if (keylen < 3)
 		return -EINVAL;
 	keylen -= 3;
 	memcpy(ctx->nonce, key + keylen, 3);
 	crypto_aead_clear_flags(child, CRYPTO_TFM_REQ_MASK);
 	crypto_aead_set_flags(child, crypto_aead_get_flags(parent) &
 				     CRYPTO_TFM_REQ_MASK);
 	err = crypto_aead_setkey(child, key, keylen);
 	crypto_aead_set_flags(parent, crypto_aead_get_flags(child) &
 				      CRYPTO_TFM_RES_MASK);
 	return err;
 }
 static int crypto_rfc4309_setauthsize(struct crypto_aead *parent,
 				      unsigned int authsize)
 {
 	struct crypto_rfc4309_ctx *ctx = crypto_aead_ctx(parent);
 	switch (authsize) {
 	case 8:
 	case 12:
 	case 16:
 		break;
 	default:
 		return -EINVAL;
 	}
 	return crypto_aead_setauthsize(ctx->child, authsize);
 }
 static struct aead_request *crypto_rfc4309_crypt(struct aead_request *req)
 {
 	struct aead_request *subreq = aead_request_ctx(req);
 	struct crypto_aead *aead = crypto_aead_reqtfm(req);
 	struct crypto_rfc4309_ctx *ctx = crypto_aead_ctx(aead);
 	struct crypto_aead *child = ctx->child;
 	u8 *iv = PTR_ALIGN((u8 *)(subreq + 1) + crypto_aead_reqsize(child),
 			   crypto_aead_alignmask(child) + 1);
 	/* L' */
 	iv[0] = 3;
 	memcpy(iv + 1, ctx->nonce, 3);
 	memcpy(iv + 4, req->iv, 8);
 	aead_request_set_tfm(subreq, child);
 	aead_request_set_callback(subreq, req->base.flags, req->base.complete,
 				  req->base.data);
 	aead_request_set_crypt(subreq, req->src, req->dst, req->cryptlen, iv);
 	aead_request_set_assoc(subreq, req->assoc, req->assoclen);
 	return subreq;
 }
 static int crypto_rfc4309_encrypt(struct aead_request *req)
 {
 	req = crypto_rfc4309_crypt(req);
 	return crypto_aead_encrypt(req);
 }
 static int crypto_rfc4309_decrypt(struct aead_request *req)
 {
 	req = crypto_rfc4309_crypt(req);
 	return crypto_aead_decrypt(req);
 }
 static int crypto_rfc4309_init_tfm(struct crypto_tfm *tfm)
 {
 	struct crypto_instance *inst = (void *)tfm->__crt_alg;
 	struct crypto_aead_spawn *spawn = crypto_instance_ctx(inst);
 	struct crypto_rfc4309_ctx *ctx = crypto_tfm_ctx(tfm);
 	struct crypto_aead *aead;
 	unsigned long align;
 	aead = crypto_spawn_aead(spawn);
 	if (IS_ERR(aead))
 		return PTR_ERR(aead);
 	ctx->child = aead;
 	align = crypto_aead_alignmask(aead);
 	align &= ~(crypto_tfm_ctx_alignment() - 1);
 	tfm->crt_aead.reqsize = sizeof(struct aead_request) +
 				ALIGN(crypto_aead_reqsize(aead),
 				      crypto_tfm_ctx_alignment()) +
 				align + 16;
 	return 0;
 }
 static void crypto_rfc4309_exit_tfm(struct crypto_tfm *tfm)
 {
 	struct crypto_rfc4309_ctx *ctx = crypto_tfm_ctx(tfm);
 	crypto_free_aead(ctx->child);
 }
 static struct crypto_instance *crypto_rfc4309_alloc(struct rtattr **tb)
 {
 	struct crypto_attr_type *algt;
 	struct crypto_instance *inst;
 	struct crypto_aead_spawn *spawn;
 	struct crypto_alg *alg;
 	const char *ccm_name;
 	int err;
 	algt = crypto_get_attr_type(tb);
 	if (IS_ERR(algt))
 		return ERR_CAST(algt);
 	if ((algt->type ^ CRYPTO_ALG_TYPE_AEAD) & algt->mask)
 		return ERR_PTR(-EINVAL);
 	ccm_name = crypto_attr_alg_name(tb[1]);
 	if (IS_ERR(ccm_name))
 		return ERR_CAST(ccm_name);
 	inst = kzalloc(sizeof(*inst) + sizeof(*spawn), GFP_KERNEL);
 	if (!inst)
 		return ERR_PTR(-ENOMEM);
 	spawn = crypto_instance_ctx(inst);
 	crypto_set_aead_spawn(spawn, inst);
 	err = crypto_grab_aead(spawn, ccm_name, 0,
 			       crypto_requires_sync(algt->type, algt->mask));
 	if (err)
 		goto out_free_inst;
 	alg = crypto_aead_spawn_alg(spawn);
 	err = -EINVAL;
 	/* We only support 16-byte blocks. */
 	if (alg->cra_aead.ivsize != 16)
 		goto out_drop_alg;
 	/* Not a stream cipher? */
 	if (alg->cra_blocksize != 1)
 		goto out_drop_alg;
 	err = -ENAMETOOLONG;
 	if (snprintf(inst->alg.cra_name, CRYPTO_MAX_ALG_NAME,
 		     "rfc4309(%s)", alg->cra_name) >= CRYPTO_MAX_ALG_NAME ||
 	    snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME,
 		     "rfc4309(%s)", alg->cra_driver_name) >=
 	    CRYPTO_MAX_ALG_NAME)
 		goto out_drop_alg;
 	inst->alg.cra_flags = CRYPTO_ALG_TYPE_AEAD;
 	inst->alg.cra_flags |= alg->cra_flags & CRYPTO_ALG_ASYNC;
 	inst->alg.cra_priority = alg->cra_priority;
 	inst->alg.cra_blocksize = 1;
 	inst->alg.cra_alignmask = alg->cra_alignmask;
 	inst->alg.cra_type = &crypto_nivaead_type;
 	inst->alg.cra_aead.ivsize = 8;
 	inst->alg.cra_aead.maxauthsize = 16;
 	inst->alg.cra_ctxsize = sizeof(struct crypto_rfc4309_ctx);
 	inst->alg.cra_init = crypto_rfc4309_init_tfm;
 	inst->alg.cra_exit = crypto_rfc4309_exit_tfm;
 	inst->alg.cra_aead.setkey = crypto_rfc4309_setkey;
 	inst->alg.cra_aead.setauthsize = crypto_rfc4309_setauthsize;
 	inst->alg.cra_aead.encrypt = crypto_rfc4309_encrypt;
 	inst->alg.cra_aead.decrypt = crypto_rfc4309_decrypt;
 	inst->alg.cra_aead.geniv = "seqiv";
 out:
 	return inst;
 out_drop_alg:
 	crypto_drop_aead(spawn);
 out_free_inst:
 	kfree(inst);
 	inst = ERR_PTR(err);
 	goto out;
 }
 static void crypto_rfc4309_free(struct crypto_instance *inst)
 {
 	crypto_drop_spawn(crypto_instance_ctx(inst));
 	kfree(inst);
 }
 static struct crypto_template crypto_rfc4309_tmpl = {
 	.name = "rfc4309",
 	.alloc = crypto_rfc4309_alloc,
 	.free = crypto_rfc4309_free,
 	.module = THIS_MODULE,
 };
 static int __init crypto_ccm_module_init(void)
 {
 	int err;
 	err = crypto_register_template(&crypto_ccm_base_tmpl);
 	if (err)
 		goto out;
 	err = crypto_register_template(&crypto_ccm_tmpl);
 	if (err)
 		goto out_undo_base;
 	err = crypto_register_template(&crypto_rfc4309_tmpl);
 	if (err)
 		goto out_undo_ccm;
 out:
 	return err;
 out_undo_ccm:
 	crypto_unregister_template(&crypto_ccm_tmpl);
 out_undo_base:
 	crypto_unregister_template(&crypto_ccm_base_tmpl);
 	goto out;
 }
 static void __exit crypto_ccm_module_exit(void)
 {
 	crypto_unregister_template(&crypto_rfc4309_tmpl);
 	crypto_unregister_template(&crypto_ccm_tmpl);
 	crypto_unregister_template(&crypto_ccm_base_tmpl);
 }
 module_init(crypto_ccm_module_init);
 module_exit(crypto_ccm_module_exit);
 MODULE_LICENSE("GPL");
 MODULE_DESCRIPTION("Counter with CBC MAC");
 MODULE_ALIAS_CRYPTO("ccm_base");
 MODULE_ALIAS_CRYPTO("rfc4309");
+MODULE_ALIAS_CRYPTO("ccm");

crypto/chainiv.c

Diff comments View file @ acc5ccb

 /*
  * chainiv: Chain IV Generator
  *
  * Generate IVs simply be using the last block of the previous encryption.
  * This is mainly useful for CBC with a synchronous algorithm.
  *
  * Copyright (c) 2007 Herbert Xu <herbert@gondor.apana.org.au>
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the Free
  * Software Foundation; either version 2 of the License, or (at your option)
  * any later version.
  *
  */
 #include <crypto/internal/skcipher.h>
 #include <crypto/rng.h>
 #include <crypto/crypto_wq.h>
 #include <linux/err.h>
 #include <linux/init.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/spinlock.h>
 #include <linux/string.h>
 #include <linux/workqueue.h>
 enum {
 	CHAINIV_STATE_INUSE = 0,
 };
 struct chainiv_ctx {
 	spinlock_t lock;
 	char iv[];
 };
 struct async_chainiv_ctx {
 	unsigned long state;
 	spinlock_t lock;
 	int err;
 	struct crypto_queue queue;
 	struct work_struct postponed;
 	char iv[];
 };
 static int chainiv_givencrypt(struct skcipher_givcrypt_request *req)
 {
 	struct crypto_ablkcipher *geniv = skcipher_givcrypt_reqtfm(req);
 	struct chainiv_ctx *ctx = crypto_ablkcipher_ctx(geniv);
 	struct ablkcipher_request *subreq = skcipher_givcrypt_reqctx(req);
 	unsigned int ivsize;
 	int err;
 	ablkcipher_request_set_tfm(subreq, skcipher_geniv_cipher(geniv));
 	ablkcipher_request_set_callback(subreq, req->creq.base.flags &
 						~CRYPTO_TFM_REQ_MAY_SLEEP,
 					req->creq.base.complete,
 					req->creq.base.data);
 	ablkcipher_request_set_crypt(subreq, req->creq.src, req->creq.dst,
 				     req->creq.nbytes, req->creq.info);
 	spin_lock_bh(&ctx->lock);
 	ivsize = crypto_ablkcipher_ivsize(geniv);
 	memcpy(req->giv, ctx->iv, ivsize);
 	memcpy(subreq->info, ctx->iv, ivsize);
 	err = crypto_ablkcipher_encrypt(subreq);
 	if (err)
 		goto unlock;
 	memcpy(ctx->iv, subreq->info, ivsize);
 unlock:
 	spin_unlock_bh(&ctx->lock);
 	return err;
 }
 static int chainiv_givencrypt_first(struct skcipher_givcrypt_request *req)
 {
 	struct crypto_ablkcipher *geniv = skcipher_givcrypt_reqtfm(req);
 	struct chainiv_ctx *ctx = crypto_ablkcipher_ctx(geniv);
 	int err = 0;
 	spin_lock_bh(&ctx->lock);
 	if (crypto_ablkcipher_crt(geniv)->givencrypt !=
 	    chainiv_givencrypt_first)
 		goto unlock;
 	crypto_ablkcipher_crt(geniv)->givencrypt = chainiv_givencrypt;
 	err = crypto_rng_get_bytes(crypto_default_rng, ctx->iv,
 				   crypto_ablkcipher_ivsize(geniv));
 unlock:
 	spin_unlock_bh(&ctx->lock);
 	if (err)
 		return err;
 	return chainiv_givencrypt(req);
 }
 static int chainiv_init_common(struct crypto_tfm *tfm)
 {
 	tfm->crt_ablkcipher.reqsize = sizeof(struct ablkcipher_request);
 	return skcipher_geniv_init(tfm);
 }
 static int chainiv_init(struct crypto_tfm *tfm)
 {
 	struct chainiv_ctx *ctx = crypto_tfm_ctx(tfm);
 	spin_lock_init(&ctx->lock);
 	return chainiv_init_common(tfm);
 }
 static int async_chainiv_schedule_work(struct async_chainiv_ctx *ctx)
 {
 	int queued;
 	int err = ctx->err;
 	if (!ctx->queue.qlen) {
 		smp_mb__before_atomic();
 		clear_bit(CHAINIV_STATE_INUSE, &ctx->state);
 		if (!ctx->queue.qlen ||
 		    test_and_set_bit(CHAINIV_STATE_INUSE, &ctx->state))
 			goto out;
 	}
 	queued = queue_work(kcrypto_wq, &ctx->postponed);
 	BUG_ON(!queued);
 out:
 	return err;
 }
 static int async_chainiv_postpone_request(struct skcipher_givcrypt_request *req)
 {
 	struct crypto_ablkcipher *geniv = skcipher_givcrypt_reqtfm(req);
 	struct async_chainiv_ctx *ctx = crypto_ablkcipher_ctx(geniv);
 	int err;
 	spin_lock_bh(&ctx->lock);
 	err = skcipher_enqueue_givcrypt(&ctx->queue, req);
 	spin_unlock_bh(&ctx->lock);
 	if (test_and_set_bit(CHAINIV_STATE_INUSE, &ctx->state))
 		return err;
 	ctx->err = err;
 	return async_chainiv_schedule_work(ctx);
 }
 static int async_chainiv_givencrypt_tail(struct skcipher_givcrypt_request *req)
 {
 	struct crypto_ablkcipher *geniv = skcipher_givcrypt_reqtfm(req);
 	struct async_chainiv_ctx *ctx = crypto_ablkcipher_ctx(geniv);
 	struct ablkcipher_request *subreq = skcipher_givcrypt_reqctx(req);
 	unsigned int ivsize = crypto_ablkcipher_ivsize(geniv);
 	memcpy(req->giv, ctx->iv, ivsize);
 	memcpy(subreq->info, ctx->iv, ivsize);
 	ctx->err = crypto_ablkcipher_encrypt(subreq);
 	if (ctx->err)
 		goto out;
 	memcpy(ctx->iv, subreq->info, ivsize);
 out:
 	return async_chainiv_schedule_work(ctx);
 }
 static int async_chainiv_givencrypt(struct skcipher_givcrypt_request *req)
 {
 	struct crypto_ablkcipher *geniv = skcipher_givcrypt_reqtfm(req);
 	struct async_chainiv_ctx *ctx = crypto_ablkcipher_ctx(geniv);
 	struct ablkcipher_request *subreq = skcipher_givcrypt_reqctx(req);
 	ablkcipher_request_set_tfm(subreq, skcipher_geniv_cipher(geniv));
 	ablkcipher_request_set_callback(subreq, req->creq.base.flags,
 					req->creq.base.complete,
 					req->creq.base.data);
 	ablkcipher_request_set_crypt(subreq, req->creq.src, req->creq.dst,
 				     req->creq.nbytes, req->creq.info);
 	if (test_and_set_bit(CHAINIV_STATE_INUSE, &ctx->state))
 		goto postpone;
 	if (ctx->queue.qlen) {
 		clear_bit(CHAINIV_STATE_INUSE, &ctx->state);
 		goto postpone;
 	}
 	return async_chainiv_givencrypt_tail(req);
 postpone:
 	return async_chainiv_postpone_request(req);
 }
 static int async_chainiv_givencrypt_first(struct skcipher_givcrypt_request *req)
 {
 	struct crypto_ablkcipher *geniv = skcipher_givcrypt_reqtfm(req);
 	struct async_chainiv_ctx *ctx = crypto_ablkcipher_ctx(geniv);
 	int err = 0;
 	if (test_and_set_bit(CHAINIV_STATE_INUSE, &ctx->state))
 		goto out;
 	if (crypto_ablkcipher_crt(geniv)->givencrypt !=
 	    async_chainiv_givencrypt_first)
 		goto unlock;
 	crypto_ablkcipher_crt(geniv)->givencrypt = async_chainiv_givencrypt;
 	err = crypto_rng_get_bytes(crypto_default_rng, ctx->iv,
 				   crypto_ablkcipher_ivsize(geniv));
 unlock:
 	clear_bit(CHAINIV_STATE_INUSE, &ctx->state);
 	if (err)
 		return err;
 out:
 	return async_chainiv_givencrypt(req);
 }
 static void async_chainiv_do_postponed(struct work_struct *work)
 {
 	struct async_chainiv_ctx *ctx = container_of(work,
 						     struct async_chainiv_ctx,
 						     postponed);
 	struct skcipher_givcrypt_request *req;
 	struct ablkcipher_request *subreq;
 	int err;
 	/* Only handle one request at a time to avoid hogging keventd. */
 	spin_lock_bh(&ctx->lock);
 	req = skcipher_dequeue_givcrypt(&ctx->queue);
 	spin_unlock_bh(&ctx->lock);
 	if (!req) {
 		async_chainiv_schedule_work(ctx);
 		return;
 	}
 	subreq = skcipher_givcrypt_reqctx(req);
 	subreq->base.flags |= CRYPTO_TFM_REQ_MAY_SLEEP;
 	err = async_chainiv_givencrypt_tail(req);
 	local_bh_disable();
 	skcipher_givcrypt_complete(req, err);
 	local_bh_enable();
 }
 static int async_chainiv_init(struct crypto_tfm *tfm)
 {
 	struct async_chainiv_ctx *ctx = crypto_tfm_ctx(tfm);
 	spin_lock_init(&ctx->lock);
 	crypto_init_queue(&ctx->queue, 100);
 	INIT_WORK(&ctx->postponed, async_chainiv_do_postponed);
 	return chainiv_init_common(tfm);
 }
 static void async_chainiv_exit(struct crypto_tfm *tfm)
 {
 	struct async_chainiv_ctx *ctx = crypto_tfm_ctx(tfm);
 	BUG_ON(test_bit(CHAINIV_STATE_INUSE, &ctx->state) || ctx->queue.qlen);
 	skcipher_geniv_exit(tfm);
 }
 static struct crypto_template chainiv_tmpl;
 static struct crypto_instance *chainiv_alloc(struct rtattr **tb)
 {
 	struct crypto_attr_type *algt;
 	struct crypto_instance *inst;
 	int err;
 	algt = crypto_get_attr_type(tb);
 	if (IS_ERR(algt))
 		return ERR_CAST(algt);
 	err = crypto_get_default_rng();
 	if (err)
 		return ERR_PTR(err);
 	inst = skcipher_geniv_alloc(&chainiv_tmpl, tb, 0, 0);
 	if (IS_ERR(inst))
 		goto put_rng;
 	inst->alg.cra_ablkcipher.givencrypt = chainiv_givencrypt_first;
 	inst->alg.cra_init = chainiv_init;
 	inst->alg.cra_exit = skcipher_geniv_exit;
 	inst->alg.cra_ctxsize = sizeof(struct chainiv_ctx);
 	if (!crypto_requires_sync(algt->type, algt->mask)) {
 		inst->alg.cra_flags |= CRYPTO_ALG_ASYNC;
 		inst->alg.cra_ablkcipher.givencrypt =
 			async_chainiv_givencrypt_first;
 		inst->alg.cra_init = async_chainiv_init;
 		inst->alg.cra_exit = async_chainiv_exit;
 		inst->alg.cra_ctxsize = sizeof(struct async_chainiv_ctx);
 	}
 	inst->alg.cra_ctxsize += inst->alg.cra_ablkcipher.ivsize;
 out:
 	return inst;
 put_rng:
 	crypto_put_default_rng();
 	goto out;
 }
 static void chainiv_free(struct crypto_instance *inst)
 {
 	skcipher_geniv_free(inst);
 	crypto_put_default_rng();
 }
 static struct crypto_template chainiv_tmpl = {
 	.name = "chainiv",
 	.alloc = chainiv_alloc,
 	.free = chainiv_free,
 	.module = THIS_MODULE,
 };
 static int __init chainiv_module_init(void)
 {
 	return crypto_register_template(&chainiv_tmpl);
 }
 static void chainiv_module_exit(void)
 {
 	crypto_unregister_template(&chainiv_tmpl);
 }
 module_init(chainiv_module_init);
 module_exit(chainiv_module_exit);
 MODULE_LICENSE("GPL");
 MODULE_DESCRIPTION("Chain IV Generator");
+MODULE_ALIAS_CRYPTO("chainiv");

crypto/cmac.c

Diff comments View file @ acc5ccb

 /*
  * CMAC: Cipher Block Mode for Authentication
  *
  * Copyright © 2013 Jussi Kivilinna <jussi.kivilinna@iki.fi>
  *
  * Based on work by:
  *  Copyright © 2013 Tom St Denis <tstdenis@elliptictech.com>
  * Based on crypto/xcbc.c:
  *  Copyright © 2006 USAGI/WIDE Project,
  *   Author: Kazunori Miyazawa <miyazawa@linux-ipv6.org>
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
  * the Free Software Foundation; either version 2 of the License, or
  * (at your option) any later version.
  *
  */
 #include <crypto/internal/hash.h>
 #include <linux/err.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
 /*
  * +------------------------
  * | <parent tfm>
  * +------------------------
  * | cmac_tfm_ctx
  * +------------------------
  * | consts (block size * 2)
  * +------------------------
  */
 struct cmac_tfm_ctx {
 	struct crypto_cipher *child;
 	u8 ctx[];
 };
 /*
  * +------------------------
  * | <shash desc>
  * +------------------------
  * | cmac_desc_ctx
  * +------------------------
  * | odds (block size)
  * +------------------------
  * | prev (block size)
  * +------------------------
  */
 struct cmac_desc_ctx {
 	unsigned int len;
 	u8 ctx[];
 };
 static int crypto_cmac_digest_setkey(struct crypto_shash *parent,
 				     const u8 *inkey, unsigned int keylen)
 {
 	unsigned long alignmask = crypto_shash_alignmask(parent);
 	struct cmac_tfm_ctx *ctx = crypto_shash_ctx(parent);
 	unsigned int bs = crypto_shash_blocksize(parent);
 	__be64 *consts = PTR_ALIGN((void *)ctx->ctx, alignmask + 1);
 	u64 _const[2];
 	int i, err = 0;
 	u8 msb_mask, gfmask;
 	err = crypto_cipher_setkey(ctx->child, inkey, keylen);
 	if (err)
 		return err;
 	/* encrypt the zero block */
 	memset(consts, 0, bs);
 	crypto_cipher_encrypt_one(ctx->child, (u8 *)consts, (u8 *)consts);
 	switch (bs) {
 	case 16:
 		gfmask = 0x87;
 		_const[0] = be64_to_cpu(consts[1]);
 		_const[1] = be64_to_cpu(consts[0]);
 		/* gf(2^128) multiply zero-ciphertext with u and u^2 */
 		for (i = 0; i < 4; i += 2) {
 			msb_mask = ((s64)_const[1] >> 63) & gfmask;
 			_const[1] = (_const[1] << 1) | (_const[0] >> 63);
 			_const[0] = (_const[0] << 1) ^ msb_mask;
 			consts[i + 0] = cpu_to_be64(_const[1]);
 			consts[i + 1] = cpu_to_be64(_const[0]);
 		}
 		break;
 	case 8:
 		gfmask = 0x1B;
 		_const[0] = be64_to_cpu(consts[0]);
 		/* gf(2^64) multiply zero-ciphertext with u and u^2 */
 		for (i = 0; i < 2; i++) {
 			msb_mask = ((s64)_const[0] >> 63) & gfmask;
 			_const[0] = (_const[0] << 1) ^ msb_mask;
 			consts[i] = cpu_to_be64(_const[0]);
 		}
 		break;
 	}
 	return 0;
 }
 static int crypto_cmac_digest_init(struct shash_desc *pdesc)
 {
 	unsigned long alignmask = crypto_shash_alignmask(pdesc->tfm);
 	struct cmac_desc_ctx *ctx = shash_desc_ctx(pdesc);
 	int bs = crypto_shash_blocksize(pdesc->tfm);
 	u8 *prev = PTR_ALIGN((void *)ctx->ctx, alignmask + 1) + bs;
 	ctx->len = 0;
 	memset(prev, 0, bs);
 	return 0;
 }
 static int crypto_cmac_digest_update(struct shash_desc *pdesc, const u8 *p,
 				     unsigned int len)
 {
 	struct crypto_shash *parent = pdesc->tfm;
 	unsigned long alignmask = crypto_shash_alignmask(parent);
 	struct cmac_tfm_ctx *tctx = crypto_shash_ctx(parent);
 	struct cmac_desc_ctx *ctx = shash_desc_ctx(pdesc);
 	struct crypto_cipher *tfm = tctx->child;
 	int bs = crypto_shash_blocksize(parent);
 	u8 *odds = PTR_ALIGN((void *)ctx->ctx, alignmask + 1);
 	u8 *prev = odds + bs;
 	/* checking the data can fill the block */
 	if ((ctx->len + len) <= bs) {
 		memcpy(odds + ctx->len, p, len);
 		ctx->len += len;
 		return 0;
 	}
 	/* filling odds with new data and encrypting it */
 	memcpy(odds + ctx->len, p, bs - ctx->len);
 	len -= bs - ctx->len;
 	p += bs - ctx->len;
 	crypto_xor(prev, odds, bs);
 	crypto_cipher_encrypt_one(tfm, prev, prev);
 	/* clearing the length */
 	ctx->len = 0;
 	/* encrypting the rest of data */
 	while (len > bs) {
 		crypto_xor(prev, p, bs);
 		crypto_cipher_encrypt_one(tfm, prev, prev);
 		p += bs;
 		len -= bs;
 	}
 	/* keeping the surplus of blocksize */
 	if (len) {
 		memcpy(odds, p, len);
 		ctx->len = len;
 	}
 	return 0;
 }
 static int crypto_cmac_digest_final(struct shash_desc *pdesc, u8 *out)
 {
 	struct crypto_shash *parent = pdesc->tfm;
 	unsigned long alignmask = crypto_shash_alignmask(parent);
 	struct cmac_tfm_ctx *tctx = crypto_shash_ctx(parent);
 	struct cmac_desc_ctx *ctx = shash_desc_ctx(pdesc);
 	struct crypto_cipher *tfm = tctx->child;
 	int bs = crypto_shash_blocksize(parent);
 	u8 *consts = PTR_ALIGN((void *)tctx->ctx, alignmask + 1);
 	u8 *odds = PTR_ALIGN((void *)ctx->ctx, alignmask + 1);
 	u8 *prev = odds + bs;
 	unsigned int offset = 0;
 	if (ctx->len != bs) {
 		unsigned int rlen;
 		u8 *p = odds + ctx->len;
 		*p = 0x80;
 		p++;
 		rlen = bs - ctx->len - 1;
 		if (rlen)
 			memset(p, 0, rlen);
 		offset += bs;
 	}
 	crypto_xor(prev, odds, bs);
 	crypto_xor(prev, consts + offset, bs);
 	crypto_cipher_encrypt_one(tfm, out, prev);
 	return 0;
 }
 static int cmac_init_tfm(struct crypto_tfm *tfm)
 {
 	struct crypto_cipher *cipher;
 	struct crypto_instance *inst = (void *)tfm->__crt_alg;
 	struct crypto_spawn *spawn = crypto_instance_ctx(inst);
 	struct cmac_tfm_ctx *ctx = crypto_tfm_ctx(tfm);
 	cipher = crypto_spawn_cipher(spawn);
 	if (IS_ERR(cipher))
 		return PTR_ERR(cipher);
 	ctx->child = cipher;
 	return 0;
 };
 static void cmac_exit_tfm(struct crypto_tfm *tfm)
 {
 	struct cmac_tfm_ctx *ctx = crypto_tfm_ctx(tfm);
 	crypto_free_cipher(ctx->child);
 }
 static int cmac_create(struct crypto_template *tmpl, struct rtattr **tb)
 {
 	struct shash_instance *inst;
 	struct crypto_alg *alg;
 	unsigned long alignmask;
 	int err;
 	err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_SHASH);
 	if (err)
 		return err;
 	alg = crypto_get_attr_alg(tb, CRYPTO_ALG_TYPE_CIPHER,
 				  CRYPTO_ALG_TYPE_MASK);
 	if (IS_ERR(alg))
 		return PTR_ERR(alg);
 	switch (alg->cra_blocksize) {
 	case 16:
 	case 8:
 		break;
 	default:
 		goto out_put_alg;
 	}
 	inst = shash_alloc_instance("cmac", alg);
 	err = PTR_ERR(inst);
 	if (IS_ERR(inst))
 		goto out_put_alg;
 	err = crypto_init_spawn(shash_instance_ctx(inst), alg,
 				shash_crypto_instance(inst),
 				CRYPTO_ALG_TYPE_MASK);
 	if (err)
 		goto out_free_inst;
 	alignmask = alg->cra_alignmask | (sizeof(long) - 1);
 	inst->alg.base.cra_alignmask = alignmask;
 	inst->alg.base.cra_priority = alg->cra_priority;
 	inst->alg.base.cra_blocksize = alg->cra_blocksize;
 	inst->alg.digestsize = alg->cra_blocksize;
 	inst->alg.descsize =
 		ALIGN(sizeof(struct cmac_desc_ctx), crypto_tfm_ctx_alignment())
 		+ (alignmask & ~(crypto_tfm_ctx_alignment() - 1))
 		+ alg->cra_blocksize * 2;
 	inst->alg.base.cra_ctxsize =
 		ALIGN(sizeof(struct cmac_tfm_ctx), alignmask + 1)
 		+ alg->cra_blocksize * 2;
 	inst->alg.base.cra_init = cmac_init_tfm;
 	inst->alg.base.cra_exit = cmac_exit_tfm;
 	inst->alg.init = crypto_cmac_digest_init;
 	inst->alg.update = crypto_cmac_digest_update;
 	inst->alg.final = crypto_cmac_digest_final;
 	inst->alg.setkey = crypto_cmac_digest_setkey;
 	err = shash_register_instance(tmpl, inst);
 	if (err) {
 out_free_inst:
 		shash_free_instance(shash_crypto_instance(inst));
 	}
 out_put_alg:
 	crypto_mod_put(alg);
 	return err;
 }
 static struct crypto_template crypto_cmac_tmpl = {
 	.name = "cmac",
 	.create = cmac_create,
 	.free = shash_free_instance,
 	.module = THIS_MODULE,
 };
 static int __init crypto_cmac_module_init(void)
 {
 	return crypto_register_template(&crypto_cmac_tmpl);
 }
 static void __exit crypto_cmac_module_exit(void)
 {
 	crypto_unregister_template(&crypto_cmac_tmpl);
 }
 module_init(crypto_cmac_module_init);
 module_exit(crypto_cmac_module_exit);
 MODULE_LICENSE("GPL");
 MODULE_DESCRIPTION("CMAC keyed hash algorithm");
+MODULE_ALIAS_CRYPTO("cmac");

crypto/cryptd.c

Diff comments View file @ acc5ccb

 /*
  * Software async crypto daemon.
  *
  * Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au>
  *
  * Added AEAD support to cryptd.
  *    Authors: Tadeusz Struk (tadeusz.struk@intel.com)
  *             Adrian Hoban <adrian.hoban@intel.com>
  *             Gabriele Paoloni <gabriele.paoloni@intel.com>
  *             Aidan O'Mahony (aidan.o.mahony@intel.com)
  *    Copyright (c) 2010, Intel Corporation.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the Free
  * Software Foundation; either version 2 of the License, or (at your option)
  * any later version.
  *
  */
 #include <crypto/algapi.h>
 #include <crypto/internal/hash.h>
 #include <crypto/internal/aead.h>
 #include <crypto/cryptd.h>
 #include <crypto/crypto_wq.h>
 #include <linux/err.h>
 #include <linux/init.h>
 #include <linux/kernel.h>
 #include <linux/list.h>
 #include <linux/module.h>
 #include <linux/scatterlist.h>
 #include <linux/sched.h>
 #include <linux/slab.h>
 #define CRYPTD_MAX_CPU_QLEN 100
 struct cryptd_cpu_queue {
 	struct crypto_queue queue;
 	struct work_struct work;
 };
 struct cryptd_queue {
 	struct cryptd_cpu_queue __percpu *cpu_queue;
 };
 struct cryptd_instance_ctx {
 	struct crypto_spawn spawn;
 	struct cryptd_queue *queue;
 };
 struct hashd_instance_ctx {
 	struct crypto_shash_spawn spawn;
 	struct cryptd_queue *queue;
 };
 struct aead_instance_ctx {
 	struct crypto_aead_spawn aead_spawn;
 	struct cryptd_queue *queue;
 };
 struct cryptd_blkcipher_ctx {
 	struct crypto_blkcipher *child;
 };
 struct cryptd_blkcipher_request_ctx {
 	crypto_completion_t complete;
 };
 struct cryptd_hash_ctx {
 	struct crypto_shash *child;
 };
 struct cryptd_hash_request_ctx {
 	crypto_completion_t complete;
 	struct shash_desc desc;
 };
 struct cryptd_aead_ctx {
 	struct crypto_aead *child;
 };
 struct cryptd_aead_request_ctx {
 	crypto_completion_t complete;
 };
 static void cryptd_queue_worker(struct work_struct *work);
 static int cryptd_init_queue(struct cryptd_queue *queue,
 			     unsigned int max_cpu_qlen)
 {
 	int cpu;
 	struct cryptd_cpu_queue *cpu_queue;
 	queue->cpu_queue = alloc_percpu(struct cryptd_cpu_queue);
 	if (!queue->cpu_queue)
 		return -ENOMEM;
 	for_each_possible_cpu(cpu) {
 		cpu_queue = per_cpu_ptr(queue->cpu_queue, cpu);
 		crypto_init_queue(&cpu_queue->queue, max_cpu_qlen);
 		INIT_WORK(&cpu_queue->work, cryptd_queue_worker);
 	}
 	return 0;
 }
 static void cryptd_fini_queue(struct cryptd_queue *queue)
 {
 	int cpu;
 	struct cryptd_cpu_queue *cpu_queue;
 	for_each_possible_cpu(cpu) {
 		cpu_queue = per_cpu_ptr(queue->cpu_queue, cpu);
 		BUG_ON(cpu_queue->queue.qlen);
 	}
 	free_percpu(queue->cpu_queue);
 }
 static int cryptd_enqueue_request(struct cryptd_queue *queue,
 				  struct crypto_async_request *request)
 {
 	int cpu, err;
 	struct cryptd_cpu_queue *cpu_queue;
 	cpu = get_cpu();
 	cpu_queue = this_cpu_ptr(queue->cpu_queue);
 	err = crypto_enqueue_request(&cpu_queue->queue, request);
 	queue_work_on(cpu, kcrypto_wq, &cpu_queue->work);
 	put_cpu();
 	return err;
 }
 /* Called in workqueue context, do one real cryption work (via
  * req->complete) and reschedule itself if there are more work to
  * do. */
 static void cryptd_queue_worker(struct work_struct *work)
 {
 	struct cryptd_cpu_queue *cpu_queue;
 	struct crypto_async_request *req, *backlog;
 	cpu_queue = container_of(work, struct cryptd_cpu_queue, work);
 	/*
 	 * Only handle one request at a time to avoid hogging crypto workqueue.
 	 * preempt_disable/enable is used to prevent being preempted by
 	 * cryptd_enqueue_request(). local_bh_disable/enable is used to prevent
 	 * cryptd_enqueue_request() being accessed from software interrupts.
 	 */
 	local_bh_disable();
 	preempt_disable();
 	backlog = crypto_get_backlog(&cpu_queue->queue);
 	req = crypto_dequeue_request(&cpu_queue->queue);
 	preempt_enable();
 	local_bh_enable();
 	if (!req)
 		return;
 	if (backlog)
 		backlog->complete(backlog, -EINPROGRESS);
 	req->complete(req, 0);
 	if (cpu_queue->queue.qlen)
 		queue_work(kcrypto_wq, &cpu_queue->work);
 }
 static inline struct cryptd_queue *cryptd_get_queue(struct crypto_tfm *tfm)
 {
 	struct crypto_instance *inst = crypto_tfm_alg_instance(tfm);
 	struct cryptd_instance_ctx *ictx = crypto_instance_ctx(inst);
 	return ictx->queue;
 }
 static int cryptd_blkcipher_setkey(struct crypto_ablkcipher *parent,
 				   const u8 *key, unsigned int keylen)
 {
 	struct cryptd_blkcipher_ctx *ctx = crypto_ablkcipher_ctx(parent);
 	struct crypto_blkcipher *child = ctx->child;
 	int err;
 	crypto_blkcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK);
 	crypto_blkcipher_set_flags(child, crypto_ablkcipher_get_flags(parent) &
 					  CRYPTO_TFM_REQ_MASK);
 	err = crypto_blkcipher_setkey(child, key, keylen);
 	crypto_ablkcipher_set_flags(parent, crypto_blkcipher_get_flags(child) &
 					    CRYPTO_TFM_RES_MASK);
 	return err;
 }
 static void cryptd_blkcipher_crypt(struct ablkcipher_request *req,
 				   struct crypto_blkcipher *child,
 				   int err,
 				   int (*crypt)(struct blkcipher_desc *desc,
 						struct scatterlist *dst,
 						struct scatterlist *src,
 						unsigned int len))
 {
 	struct cryptd_blkcipher_request_ctx *rctx;
 	struct blkcipher_desc desc;
 	rctx = ablkcipher_request_ctx(req);
 	if (unlikely(err == -EINPROGRESS))
 		goto out;
 	desc.tfm = child;
 	desc.info = req->info;
 	desc.flags = CRYPTO_TFM_REQ_MAY_SLEEP;
 	err = crypt(&desc, req->dst, req->src, req->nbytes);
 	req->base.complete = rctx->complete;
 out:
 	local_bh_disable();
 	rctx->complete(&req->base, err);
 	local_bh_enable();
 }
 static void cryptd_blkcipher_encrypt(struct crypto_async_request *req, int err)
 {
 	struct cryptd_blkcipher_ctx *ctx = crypto_tfm_ctx(req->tfm);
 	struct crypto_blkcipher *child = ctx->child;
 	cryptd_blkcipher_crypt(ablkcipher_request_cast(req), child, err,
 			       crypto_blkcipher_crt(child)->encrypt);
 }
 static void cryptd_blkcipher_decrypt(struct crypto_async_request *req, int err)
 {
 	struct cryptd_blkcipher_ctx *ctx = crypto_tfm_ctx(req->tfm);
 	struct crypto_blkcipher *child = ctx->child;
 	cryptd_blkcipher_crypt(ablkcipher_request_cast(req), child, err,
 			       crypto_blkcipher_crt(child)->decrypt);
 }
 static int cryptd_blkcipher_enqueue(struct ablkcipher_request *req,
 				    crypto_completion_t compl)
 {
 	struct cryptd_blkcipher_request_ctx *rctx = ablkcipher_request_ctx(req);
 	struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
 	struct cryptd_queue *queue;
 	queue = cryptd_get_queue(crypto_ablkcipher_tfm(tfm));
 	rctx->complete = req->base.complete;
 	req->base.complete = compl;
 	return cryptd_enqueue_request(queue, &req->base);
 }
 static int cryptd_blkcipher_encrypt_enqueue(struct ablkcipher_request *req)
 {
 	return cryptd_blkcipher_enqueue(req, cryptd_blkcipher_encrypt);
 }
 static int cryptd_blkcipher_decrypt_enqueue(struct ablkcipher_request *req)
 {
 	return cryptd_blkcipher_enqueue(req, cryptd_blkcipher_decrypt);
 }
 static int cryptd_blkcipher_init_tfm(struct crypto_tfm *tfm)
 {
 	struct crypto_instance *inst = crypto_tfm_alg_instance(tfm);
 	struct cryptd_instance_ctx *ictx = crypto_instance_ctx(inst);
 	struct crypto_spawn *spawn = &ictx->spawn;
 	struct cryptd_blkcipher_ctx *ctx = crypto_tfm_ctx(tfm);
 	struct crypto_blkcipher *cipher;
 	cipher = crypto_spawn_blkcipher(spawn);
 	if (IS_ERR(cipher))
 		return PTR_ERR(cipher);
 	ctx->child = cipher;
 	tfm->crt_ablkcipher.reqsize =
 		sizeof(struct cryptd_blkcipher_request_ctx);
 	return 0;
 }
 static void cryptd_blkcipher_exit_tfm(struct crypto_tfm *tfm)
 {
 	struct cryptd_blkcipher_ctx *ctx = crypto_tfm_ctx(tfm);
 	crypto_free_blkcipher(ctx->child);
 }
 static void *cryptd_alloc_instance(struct crypto_alg *alg, unsigned int head,
 				   unsigned int tail)
 {
 	char *p;
 	struct crypto_instance *inst;
 	int err;
 	p = kzalloc(head + sizeof(*inst) + tail, GFP_KERNEL);
 	if (!p)
 		return ERR_PTR(-ENOMEM);
 	inst = (void *)(p + head);
 	err = -ENAMETOOLONG;
 	if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME,
 		     "cryptd(%s)", alg->cra_driver_name) >= CRYPTO_MAX_ALG_NAME)
 		goto out_free_inst;
 	memcpy(inst->alg.cra_name, alg->cra_name, CRYPTO_MAX_ALG_NAME);
 	inst->alg.cra_priority = alg->cra_priority + 50;
 	inst->alg.cra_blocksize = alg->cra_blocksize;
 	inst->alg.cra_alignmask = alg->cra_alignmask;
 out:
 	return p;
 out_free_inst:
 	kfree(p);
 	p = ERR_PTR(err);
 	goto out;
 }
 static int cryptd_create_blkcipher(struct crypto_template *tmpl,
 				   struct rtattr **tb,
 				   struct cryptd_queue *queue)
 {
 	struct cryptd_instance_ctx *ctx;
 	struct crypto_instance *inst;
 	struct crypto_alg *alg;
 	int err;
 	alg = crypto_get_attr_alg(tb, CRYPTO_ALG_TYPE_BLKCIPHER,
 				  CRYPTO_ALG_TYPE_MASK);
 	if (IS_ERR(alg))
 		return PTR_ERR(alg);
 	inst = cryptd_alloc_instance(alg, 0, sizeof(*ctx));
 	err = PTR_ERR(inst);
 	if (IS_ERR(inst))
 		goto out_put_alg;
 	ctx = crypto_instance_ctx(inst);
 	ctx->queue = queue;
 	err = crypto_init_spawn(&ctx->spawn, alg, inst,
 				CRYPTO_ALG_TYPE_MASK | CRYPTO_ALG_ASYNC);
 	if (err)
 		goto out_free_inst;
 	inst->alg.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC;
 	inst->alg.cra_type = &crypto_ablkcipher_type;
 	inst->alg.cra_ablkcipher.ivsize = alg->cra_blkcipher.ivsize;
 	inst->alg.cra_ablkcipher.min_keysize = alg->cra_blkcipher.min_keysize;
 	inst->alg.cra_ablkcipher.max_keysize = alg->cra_blkcipher.max_keysize;
 	inst->alg.cra_ablkcipher.geniv = alg->cra_blkcipher.geniv;
 	inst->alg.cra_ctxsize = sizeof(struct cryptd_blkcipher_ctx);
 	inst->alg.cra_init = cryptd_blkcipher_init_tfm;
 	inst->alg.cra_exit = cryptd_blkcipher_exit_tfm;
 	inst->alg.cra_ablkcipher.setkey = cryptd_blkcipher_setkey;
 	inst->alg.cra_ablkcipher.encrypt = cryptd_blkcipher_encrypt_enqueue;
 	inst->alg.cra_ablkcipher.decrypt = cryptd_blkcipher_decrypt_enqueue;
 	err = crypto_register_instance(tmpl, inst);
 	if (err) {
 		crypto_drop_spawn(&ctx->spawn);
 out_free_inst:
 		kfree(inst);
 	}
 out_put_alg:
 	crypto_mod_put(alg);
 	return err;
 }
 static int cryptd_hash_init_tfm(struct crypto_tfm *tfm)
 {
 	struct crypto_instance *inst = crypto_tfm_alg_instance(tfm);
 	struct hashd_instance_ctx *ictx = crypto_instance_ctx(inst);
 	struct crypto_shash_spawn *spawn = &ictx->spawn;
 	struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(tfm);
 	struct crypto_shash *hash;
 	hash = crypto_spawn_shash(spawn);
 	if (IS_ERR(hash))
 		return PTR_ERR(hash);
 	ctx->child = hash;
 	crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
 				 sizeof(struct cryptd_hash_request_ctx) +
 				 crypto_shash_descsize(hash));
 	return 0;
 }
 static void cryptd_hash_exit_tfm(struct crypto_tfm *tfm)
 {
 	struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(tfm);
 	crypto_free_shash(ctx->child);
 }
 static int cryptd_hash_setkey(struct crypto_ahash *parent,
 				   const u8 *key, unsigned int keylen)
 {
 	struct cryptd_hash_ctx *ctx   = crypto_ahash_ctx(parent);
 	struct crypto_shash *child = ctx->child;
 	int err;
 	crypto_shash_clear_flags(child, CRYPTO_TFM_REQ_MASK);
 	crypto_shash_set_flags(child, crypto_ahash_get_flags(parent) &
 				      CRYPTO_TFM_REQ_MASK);
 	err = crypto_shash_setkey(child, key, keylen);
 	crypto_ahash_set_flags(parent, crypto_shash_get_flags(child) &
 				       CRYPTO_TFM_RES_MASK);
 	return err;
 }
 static int cryptd_hash_enqueue(struct ahash_request *req,
 				crypto_completion_t compl)
 {
 	struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 	struct cryptd_queue *queue =
 		cryptd_get_queue(crypto_ahash_tfm(tfm));
 	rctx->complete = req->base.complete;
 	req->base.complete = compl;
 	return cryptd_enqueue_request(queue, &req->base);
 }
 static void cryptd_hash_init(struct crypto_async_request *req_async, int err)
 {
 	struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(req_async->tfm);
 	struct crypto_shash *child = ctx->child;
 	struct ahash_request *req = ahash_request_cast(req_async);
 	struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
 	struct shash_desc *desc = &rctx->desc;
 	if (unlikely(err == -EINPROGRESS))
 		goto out;
 	desc->tfm = child;
 	desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP;
 	err = crypto_shash_init(desc);
 	req->base.complete = rctx->complete;
 out:
 	local_bh_disable();
 	rctx->complete(&req->base, err);
 	local_bh_enable();
 }
 static int cryptd_hash_init_enqueue(struct ahash_request *req)
 {
 	return cryptd_hash_enqueue(req, cryptd_hash_init);
 }
 static void cryptd_hash_update(struct crypto_async_request *req_async, int err)
 {
 	struct ahash_request *req = ahash_request_cast(req_async);
 	struct cryptd_hash_request_ctx *rctx;
 	rctx = ahash_request_ctx(req);
 	if (unlikely(err == -EINPROGRESS))
 		goto out;
 	err = shash_ahash_update(req, &rctx->desc);
 	req->base.complete = rctx->complete;
 out:
 	local_bh_disable();
 	rctx->complete(&req->base, err);
 	local_bh_enable();
 }
 static int cryptd_hash_update_enqueue(struct ahash_request *req)
 {
 	return cryptd_hash_enqueue(req, cryptd_hash_update);
 }
 static void cryptd_hash_final(struct crypto_async_request *req_async, int err)
 {
 	struct ahash_request *req = ahash_request_cast(req_async);
 	struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
 	if (unlikely(err == -EINPROGRESS))
 		goto out;
 	err = crypto_shash_final(&rctx->desc, req->result);
 	req->base.complete = rctx->complete;
 out:
 	local_bh_disable();
 	rctx->complete(&req->base, err);
 	local_bh_enable();
 }
 static int cryptd_hash_final_enqueue(struct ahash_request *req)
 {
 	return cryptd_hash_enqueue(req, cryptd_hash_final);
 }
 static void cryptd_hash_finup(struct crypto_async_request *req_async, int err)
 {
 	struct ahash_request *req = ahash_request_cast(req_async);
 	struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
 	if (unlikely(err == -EINPROGRESS))
 		goto out;
 	err = shash_ahash_finup(req, &rctx->desc);
 	req->base.complete = rctx->complete;
 out:
 	local_bh_disable();
 	rctx->complete(&req->base, err);
 	local_bh_enable();
 }
 static int cryptd_hash_finup_enqueue(struct ahash_request *req)
 {
 	return cryptd_hash_enqueue(req, cryptd_hash_finup);
 }
 static void cryptd_hash_digest(struct crypto_async_request *req_async, int err)
 {
 	struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(req_async->tfm);
 	struct crypto_shash *child = ctx->child;
 	struct ahash_request *req = ahash_request_cast(req_async);
 	struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
 	struct shash_desc *desc = &rctx->desc;
 	if (unlikely(err == -EINPROGRESS))
 		goto out;
 	desc->tfm = child;
 	desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP;
 	err = shash_ahash_digest(req, desc);
 	req->base.complete = rctx->complete;
 out:
 	local_bh_disable();
 	rctx->complete(&req->base, err);
 	local_bh_enable();
 }
 static int cryptd_hash_digest_enqueue(struct ahash_request *req)
 {
 	return cryptd_hash_enqueue(req, cryptd_hash_digest);
 }
 static int cryptd_hash_export(struct ahash_request *req, void *out)
 {
 	struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
 	return crypto_shash_export(&rctx->desc, out);
 }
 static int cryptd_hash_import(struct ahash_request *req, const void *in)
 {
 	struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
 	return crypto_shash_import(&rctx->desc, in);
 }
 static int cryptd_create_hash(struct crypto_template *tmpl, struct rtattr **tb,
 			      struct cryptd_queue *queue)
 {
 	struct hashd_instance_ctx *ctx;
 	struct ahash_instance *inst;
 	struct shash_alg *salg;
 	struct crypto_alg *alg;
 	int err;
 	salg = shash_attr_alg(tb[1], 0, 0);
 	if (IS_ERR(salg))
 		return PTR_ERR(salg);
 	alg = &salg->base;
 	inst = cryptd_alloc_instance(alg, ahash_instance_headroom(),
 				     sizeof(*ctx));
 	err = PTR_ERR(inst);
 	if (IS_ERR(inst))
 		goto out_put_alg;
 	ctx = ahash_instance_ctx(inst);
 	ctx->queue = queue;
 	err = crypto_init_shash_spawn(&ctx->spawn, salg,
 				      ahash_crypto_instance(inst));
 	if (err)
 		goto out_free_inst;
 	inst->alg.halg.base.cra_flags = CRYPTO_ALG_ASYNC;
 	inst->alg.halg.digestsize = salg->digestsize;
 	inst->alg.halg.base.cra_ctxsize = sizeof(struct cryptd_hash_ctx);
 	inst->alg.halg.base.cra_init = cryptd_hash_init_tfm;
 	inst->alg.halg.base.cra_exit = cryptd_hash_exit_tfm;
 	inst->alg.init   = cryptd_hash_init_enqueue;
 	inst->alg.update = cryptd_hash_update_enqueue;
 	inst->alg.final  = cryptd_hash_final_enqueue;
 	inst->alg.finup  = cryptd_hash_finup_enqueue;
 	inst->alg.export = cryptd_hash_export;
 	inst->alg.import = cryptd_hash_import;
 	inst->alg.setkey = cryptd_hash_setkey;
 	inst->alg.digest = cryptd_hash_digest_enqueue;
 	err = ahash_register_instance(tmpl, inst);
 	if (err) {
 		crypto_drop_shash(&ctx->spawn);
 out_free_inst:
 		kfree(inst);
 	}
 out_put_alg:
 	crypto_mod_put(alg);
 	return err;
 }
 static void cryptd_aead_crypt(struct aead_request *req,
 			struct crypto_aead *child,
 			int err,
 			int (*crypt)(struct aead_request *req))
 {
 	struct cryptd_aead_request_ctx *rctx;
 	rctx = aead_request_ctx(req);
 	if (unlikely(err == -EINPROGRESS))
 		goto out;
 	aead_request_set_tfm(req, child);
 	err = crypt( req );
 	req->base.complete = rctx->complete;
 out:
 	local_bh_disable();
 	rctx->complete(&req->base, err);
 	local_bh_enable();
 }
 static void cryptd_aead_encrypt(struct crypto_async_request *areq, int err)
 {
 	struct cryptd_aead_ctx *ctx = crypto_tfm_ctx(areq->tfm);
 	struct crypto_aead *child = ctx->child;
 	struct aead_request *req;
 	req = container_of(areq, struct aead_request, base);
 	cryptd_aead_crypt(req, child, err, crypto_aead_crt(child)->encrypt);
 }
 static void cryptd_aead_decrypt(struct crypto_async_request *areq, int err)
 {
 	struct cryptd_aead_ctx *ctx = crypto_tfm_ctx(areq->tfm);
 	struct crypto_aead *child = ctx->child;
 	struct aead_request *req;
 	req = container_of(areq, struct aead_request, base);
 	cryptd_aead_crypt(req, child, err, crypto_aead_crt(child)->decrypt);
 }
 static int cryptd_aead_enqueue(struct aead_request *req,
 				    crypto_completion_t compl)
 {
 	struct cryptd_aead_request_ctx *rctx = aead_request_ctx(req);
 	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
 	struct cryptd_queue *queue = cryptd_get_queue(crypto_aead_tfm(tfm));
 	rctx->complete = req->base.complete;
 	req->base.complete = compl;
 	return cryptd_enqueue_request(queue, &req->base);
 }
 static int cryptd_aead_encrypt_enqueue(struct aead_request *req)
 {
 	return cryptd_aead_enqueue(req, cryptd_aead_encrypt );
 }
 static int cryptd_aead_decrypt_enqueue(struct aead_request *req)
 {
 	return cryptd_aead_enqueue(req, cryptd_aead_decrypt );
 }
 static int cryptd_aead_init_tfm(struct crypto_tfm *tfm)
 {
 	struct crypto_instance *inst = crypto_tfm_alg_instance(tfm);
 	struct aead_instance_ctx *ictx = crypto_instance_ctx(inst);
 	struct crypto_aead_spawn *spawn = &ictx->aead_spawn;
 	struct cryptd_aead_ctx *ctx = crypto_tfm_ctx(tfm);
 	struct crypto_aead *cipher;
 	cipher = crypto_spawn_aead(spawn);
 	if (IS_ERR(cipher))
 		return PTR_ERR(cipher);
 	crypto_aead_set_flags(cipher, CRYPTO_TFM_REQ_MAY_SLEEP);
 	ctx->child = cipher;
 	tfm->crt_aead.reqsize = sizeof(struct cryptd_aead_request_ctx);
 	return 0;
 }
 static void cryptd_aead_exit_tfm(struct crypto_tfm *tfm)
 {
 	struct cryptd_aead_ctx *ctx = crypto_tfm_ctx(tfm);
 	crypto_free_aead(ctx->child);
 }
 static int cryptd_create_aead(struct crypto_template *tmpl,
 		              struct rtattr **tb,
 			      struct cryptd_queue *queue)
 {
 	struct aead_instance_ctx *ctx;
 	struct crypto_instance *inst;
 	struct crypto_alg *alg;
 	int err;
 	alg = crypto_get_attr_alg(tb, CRYPTO_ALG_TYPE_AEAD,
 				CRYPTO_ALG_TYPE_MASK);
         if (IS_ERR(alg))
 		return PTR_ERR(alg);
 	inst = cryptd_alloc_instance(alg, 0, sizeof(*ctx));
 	err = PTR_ERR(inst);
 	if (IS_ERR(inst))
 		goto out_put_alg;
 	ctx = crypto_instance_ctx(inst);
 	ctx->queue = queue;
 	err = crypto_init_spawn(&ctx->aead_spawn.base, alg, inst,
 			CRYPTO_ALG_TYPE_MASK | CRYPTO_ALG_ASYNC);
 	if (err)
 		goto out_free_inst;
 	inst->alg.cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC;
 	inst->alg.cra_type = alg->cra_type;
 	inst->alg.cra_ctxsize = sizeof(struct cryptd_aead_ctx);
 	inst->alg.cra_init = cryptd_aead_init_tfm;
 	inst->alg.cra_exit = cryptd_aead_exit_tfm;
 	inst->alg.cra_aead.setkey      = alg->cra_aead.setkey;
 	inst->alg.cra_aead.setauthsize = alg->cra_aead.setauthsize;
 	inst->alg.cra_aead.geniv       = alg->cra_aead.geniv;
 	inst->alg.cra_aead.ivsize      = alg->cra_aead.ivsize;
 	inst->alg.cra_aead.maxauthsize = alg->cra_aead.maxauthsize;
 	inst->alg.cra_aead.encrypt     = cryptd_aead_encrypt_enqueue;
 	inst->alg.cra_aead.decrypt     = cryptd_aead_decrypt_enqueue;
 	inst->alg.cra_aead.givencrypt  = alg->cra_aead.givencrypt;
 	inst->alg.cra_aead.givdecrypt  = alg->cra_aead.givdecrypt;
 	err = crypto_register_instance(tmpl, inst);
 	if (err) {
 		crypto_drop_spawn(&ctx->aead_spawn.base);
 out_free_inst:
 		kfree(inst);
 	}
 out_put_alg:
 	crypto_mod_put(alg);
 	return err;
 }
 static struct cryptd_queue queue;
 static int cryptd_create(struct crypto_template *tmpl, struct rtattr **tb)
 {
 	struct crypto_attr_type *algt;
 	algt = crypto_get_attr_type(tb);
 	if (IS_ERR(algt))
 		return PTR_ERR(algt);
 	switch (algt->type & algt->mask & CRYPTO_ALG_TYPE_MASK) {
 	case CRYPTO_ALG_TYPE_BLKCIPHER:
 		return cryptd_create_blkcipher(tmpl, tb, &queue);
 	case CRYPTO_ALG_TYPE_DIGEST:
 		return cryptd_create_hash(tmpl, tb, &queue);
 	case CRYPTO_ALG_TYPE_AEAD:
 		return cryptd_create_aead(tmpl, tb, &queue);
 	}
 	return -EINVAL;
 }
 static void cryptd_free(struct crypto_instance *inst)
 {
 	struct cryptd_instance_ctx *ctx = crypto_instance_ctx(inst);
 	struct hashd_instance_ctx *hctx = crypto_instance_ctx(inst);
 	struct aead_instance_ctx *aead_ctx = crypto_instance_ctx(inst);
 	switch (inst->alg.cra_flags & CRYPTO_ALG_TYPE_MASK) {
 	case CRYPTO_ALG_TYPE_AHASH:
 		crypto_drop_shash(&hctx->spawn);
 		kfree(ahash_instance(inst));
 		return;
 	case CRYPTO_ALG_TYPE_AEAD:
 		crypto_drop_spawn(&aead_ctx->aead_spawn.base);
 		kfree(inst);
 		return;
 	default:
 		crypto_drop_spawn(&ctx->spawn);
 		kfree(inst);
 	}
 }
 static struct crypto_template cryptd_tmpl = {
 	.name = "cryptd",
 	.create = cryptd_create,
 	.free = cryptd_free,
 	.module = THIS_MODULE,
 };
 struct cryptd_ablkcipher *cryptd_alloc_ablkcipher(const char *alg_name,
 						  u32 type, u32 mask)
 {
 	char cryptd_alg_name[CRYPTO_MAX_ALG_NAME];
 	struct crypto_tfm *tfm;
 	if (snprintf(cryptd_alg_name, CRYPTO_MAX_ALG_NAME,
 		     "cryptd(%s)", alg_name) >= CRYPTO_MAX_ALG_NAME)
 		return ERR_PTR(-EINVAL);
 	type &= ~(CRYPTO_ALG_TYPE_MASK | CRYPTO_ALG_GENIV);
 	type |= CRYPTO_ALG_TYPE_BLKCIPHER;
 	mask &= ~CRYPTO_ALG_TYPE_MASK;
 	mask |= (CRYPTO_ALG_GENIV | CRYPTO_ALG_TYPE_BLKCIPHER_MASK);
 	tfm = crypto_alloc_base(cryptd_alg_name, type, mask);
 	if (IS_ERR(tfm))
 		return ERR_CAST(tfm);
 	if (tfm->__crt_alg->cra_module != THIS_MODULE) {
 		crypto_free_tfm(tfm);
 		return ERR_PTR(-EINVAL);
 	}
 	return __cryptd_ablkcipher_cast(__crypto_ablkcipher_cast(tfm));
 }
 EXPORT_SYMBOL_GPL(cryptd_alloc_ablkcipher);
 struct crypto_blkcipher *cryptd_ablkcipher_child(struct cryptd_ablkcipher *tfm)
 {
 	struct cryptd_blkcipher_ctx *ctx = crypto_ablkcipher_ctx(&tfm->base);
 	return ctx->child;
 }
 EXPORT_SYMBOL_GPL(cryptd_ablkcipher_child);
 void cryptd_free_ablkcipher(struct cryptd_ablkcipher *tfm)
 {
 	crypto_free_ablkcipher(&tfm->base);
 }
 EXPORT_SYMBOL_GPL(cryptd_free_ablkcipher);
 struct cryptd_ahash *cryptd_alloc_ahash(const char *alg_name,
 					u32 type, u32 mask)
 {
 	char cryptd_alg_name[CRYPTO_MAX_ALG_NAME];
 	struct crypto_ahash *tfm;
 	if (snprintf(cryptd_alg_name, CRYPTO_MAX_ALG_NAME,
 		     "cryptd(%s)", alg_name) >= CRYPTO_MAX_ALG_NAME)
 		return ERR_PTR(-EINVAL);
 	tfm = crypto_alloc_ahash(cryptd_alg_name, type, mask);
 	if (IS_ERR(tfm))
 		return ERR_CAST(tfm);
 	if (tfm->base.__crt_alg->cra_module != THIS_MODULE) {
 		crypto_free_ahash(tfm);
 		return ERR_PTR(-EINVAL);
 	}
 	return __cryptd_ahash_cast(tfm);
 }
 EXPORT_SYMBOL_GPL(cryptd_alloc_ahash);
 struct crypto_shash *cryptd_ahash_child(struct cryptd_ahash *tfm)
 {
 	struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(&tfm->base);
 	return ctx->child;
 }
 EXPORT_SYMBOL_GPL(cryptd_ahash_child);
 struct shash_desc *cryptd_shash_desc(struct ahash_request *req)
 {
 	struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
 	return &rctx->desc;
 }
 EXPORT_SYMBOL_GPL(cryptd_shash_desc);
 void cryptd_free_ahash(struct cryptd_ahash *tfm)
 {
 	crypto_free_ahash(&tfm->base);
 }
 EXPORT_SYMBOL_GPL(cryptd_free_ahash);
 struct cryptd_aead *cryptd_alloc_aead(const char *alg_name,
 						  u32 type, u32 mask)
 {
 	char cryptd_alg_name[CRYPTO_MAX_ALG_NAME];
 	struct crypto_aead *tfm;
 	if (snprintf(cryptd_alg_name, CRYPTO_MAX_ALG_NAME,
 		     "cryptd(%s)", alg_name) >= CRYPTO_MAX_ALG_NAME)
 		return ERR_PTR(-EINVAL);
 	tfm = crypto_alloc_aead(cryptd_alg_name, type, mask);
 	if (IS_ERR(tfm))
 		return ERR_CAST(tfm);
 	if (tfm->base.__crt_alg->cra_module != THIS_MODULE) {
 		crypto_free_aead(tfm);
 		return ERR_PTR(-EINVAL);
 	}
 	return __cryptd_aead_cast(tfm);
 }
 EXPORT_SYMBOL_GPL(cryptd_alloc_aead);
 struct crypto_aead *cryptd_aead_child(struct cryptd_aead *tfm)
 {
 	struct cryptd_aead_ctx *ctx;
 	ctx = crypto_aead_ctx(&tfm->base);
 	return ctx->child;
 }
 EXPORT_SYMBOL_GPL(cryptd_aead_child);
 void cryptd_free_aead(struct cryptd_aead *tfm)
 {
 	crypto_free_aead(&tfm->base);
 }
 EXPORT_SYMBOL_GPL(cryptd_free_aead);
 static int __init cryptd_init(void)
 {
 	int err;
 	err = cryptd_init_queue(&queue, CRYPTD_MAX_CPU_QLEN);
 	if (err)
 		return err;
 	err = crypto_register_template(&cryptd_tmpl);
 	if (err)
 		cryptd_fini_queue(&queue);
 	return err;
 }
 static void __exit cryptd_exit(void)
 {
 	cryptd_fini_queue(&queue);
 	crypto_unregister_template(&cryptd_tmpl);
 }
 subsys_initcall(cryptd_init);
 module_exit(cryptd_exit);
 MODULE_LICENSE("GPL");
 MODULE_DESCRIPTION("Software async crypto daemon");
+MODULE_ALIAS_CRYPTO("cryptd");

crypto/ctr.c

Diff comments View file @ acc5ccb

 /*
  * CTR: Counter mode
  *
  * (C) Copyright IBM Corp. 2007 - Joy Latten <latten@us.ibm.com>
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the Free
  * Software Foundation; either version 2 of the License, or (at your option)
  * any later version.
  *
  */
 #include <crypto/algapi.h>
 #include <crypto/ctr.h>
 #include <crypto/internal/skcipher.h>
 #include <linux/err.h>
 #include <linux/init.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/random.h>
 #include <linux/scatterlist.h>
 #include <linux/slab.h>
 struct crypto_ctr_ctx {
 	struct crypto_cipher *child;
 };
 struct crypto_rfc3686_ctx {
 	struct crypto_ablkcipher *child;
 	u8 nonce[CTR_RFC3686_NONCE_SIZE];
 };
 struct crypto_rfc3686_req_ctx {
 	u8 iv[CTR_RFC3686_BLOCK_SIZE];
 	struct ablkcipher_request subreq CRYPTO_MINALIGN_ATTR;
 };
 static int crypto_ctr_setkey(struct crypto_tfm *parent, const u8 *key,
 			     unsigned int keylen)
 {
 	struct crypto_ctr_ctx *ctx = crypto_tfm_ctx(parent);
 	struct crypto_cipher *child = ctx->child;
 	int err;
 	crypto_cipher_clear_flags(child, CRYPTO_TFM_REQ_MASK);
 	crypto_cipher_set_flags(child, crypto_tfm_get_flags(parent) &
 				CRYPTO_TFM_REQ_MASK);
 	err = crypto_cipher_setkey(child, key, keylen);
 	crypto_tfm_set_flags(parent, crypto_cipher_get_flags(child) &
 			     CRYPTO_TFM_RES_MASK);
 	return err;
 }
 static void crypto_ctr_crypt_final(struct blkcipher_walk *walk,
 				   struct crypto_cipher *tfm)
 {
 	unsigned int bsize = crypto_cipher_blocksize(tfm);
 	unsigned long alignmask = crypto_cipher_alignmask(tfm);
 	u8 *ctrblk = walk->iv;
 	u8 tmp[bsize + alignmask];
 	u8 *keystream = PTR_ALIGN(tmp + 0, alignmask + 1);
 	u8 *src = walk->src.virt.addr;
 	u8 *dst = walk->dst.virt.addr;
 	unsigned int nbytes = walk->nbytes;
 	crypto_cipher_encrypt_one(tfm, keystream, ctrblk);
 	crypto_xor(keystream, src, nbytes);
 	memcpy(dst, keystream, nbytes);
 	crypto_inc(ctrblk, bsize);
 }
 static int crypto_ctr_crypt_segment(struct blkcipher_walk *walk,
 				    struct crypto_cipher *tfm)
 {
 	void (*fn)(struct crypto_tfm *, u8 *, const u8 *) =
 		   crypto_cipher_alg(tfm)->cia_encrypt;
 	unsigned int bsize = crypto_cipher_blocksize(tfm);
 	u8 *ctrblk = walk->iv;
 	u8 *src = walk->src.virt.addr;
 	u8 *dst = walk->dst.virt.addr;
 	unsigned int nbytes = walk->nbytes;
 	do {
 		/* create keystream */
 		fn(crypto_cipher_tfm(tfm), dst, ctrblk);
 		crypto_xor(dst, src, bsize);
 		/* increment counter in counterblock */
 		crypto_inc(ctrblk, bsize);
 		src += bsize;
 		dst += bsize;
 	} while ((nbytes -= bsize) >= bsize);
 	return nbytes;
 }
 static int crypto_ctr_crypt_inplace(struct blkcipher_walk *walk,
 				    struct crypto_cipher *tfm)
 {
 	void (*fn)(struct crypto_tfm *, u8 *, const u8 *) =
 		   crypto_cipher_alg(tfm)->cia_encrypt;
 	unsigned int bsize = crypto_cipher_blocksize(tfm);
 	unsigned long alignmask = crypto_cipher_alignmask(tfm);
 	unsigned int nbytes = walk->nbytes;
 	u8 *ctrblk = walk->iv;
 	u8 *src = walk->src.virt.addr;
 	u8 tmp[bsize + alignmask];
 	u8 *keystream = PTR_ALIGN(tmp + 0, alignmask + 1);
 	do {
 		/* create keystream */
 		fn(crypto_cipher_tfm(tfm), keystream, ctrblk);
 		crypto_xor(src, keystream, bsize);
 		/* increment counter in counterblock */
 		crypto_inc(ctrblk, bsize);
 		src += bsize;
 	} while ((nbytes -= bsize) >= bsize);
 	return nbytes;
 }
 static int crypto_ctr_crypt(struct blkcipher_desc *desc,
 			      struct scatterlist *dst, struct scatterlist *src,
 			      unsigned int nbytes)
 {
 	struct blkcipher_walk walk;
 	struct crypto_blkcipher *tfm = desc->tfm;
 	struct crypto_ctr_ctx *ctx = crypto_blkcipher_ctx(tfm);
 	struct crypto_cipher *child = ctx->child;
 	unsigned int bsize = crypto_cipher_blocksize(child);
 	int err;
 	blkcipher_walk_init(&walk, dst, src, nbytes);
 	err = blkcipher_walk_virt_block(desc, &walk, bsize);
 	while (walk.nbytes >= bsize) {
 		if (walk.src.virt.addr == walk.dst.virt.addr)
 			nbytes = crypto_ctr_crypt_inplace(&walk, child);
 		else
 			nbytes = crypto_ctr_crypt_segment(&walk, child);
 		err = blkcipher_walk_done(desc, &walk, nbytes);
 	}
 	if (walk.nbytes) {
 		crypto_ctr_crypt_final(&walk, child);
 		err = blkcipher_walk_done(desc, &walk, 0);
 	}
 	return err;
 }
 static int crypto_ctr_init_tfm(struct crypto_tfm *tfm)
 {
 	struct crypto_instance *inst = (void *)tfm->__crt_alg;
 	struct crypto_spawn *spawn = crypto_instance_ctx(inst);
 	struct crypto_ctr_ctx *ctx = crypto_tfm_ctx(tfm);
 	struct crypto_cipher *cipher;
 	cipher = crypto_spawn_cipher(spawn);
 	if (IS_ERR(cipher))
 		return PTR_ERR(cipher);
 	ctx->child = cipher;
 	return 0;
 }
 static void crypto_ctr_exit_tfm(struct crypto_tfm *tfm)
 {
 	struct crypto_ctr_ctx *ctx = crypto_tfm_ctx(tfm);
 	crypto_free_cipher(ctx->child);
 }
 static struct crypto_instance *crypto_ctr_alloc(struct rtattr **tb)
 {
 	struct crypto_instance *inst;
 	struct crypto_alg *alg;
 	int err;
 	err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_BLKCIPHER);
 	if (err)
 		return ERR_PTR(err);
 	alg = crypto_attr_alg(tb[1], CRYPTO_ALG_TYPE_CIPHER,
 				  CRYPTO_ALG_TYPE_MASK);
 	if (IS_ERR(alg))
 		return ERR_CAST(alg);
 	/* Block size must be >= 4 bytes. */
 	err = -EINVAL;
 	if (alg->cra_blocksize < 4)
 		goto out_put_alg;
 	/* If this is false we'd fail the alignment of crypto_inc. */
 	if (alg->cra_blocksize % 4)
 		goto out_put_alg;
 	inst = crypto_alloc_instance("ctr", alg);
 	if (IS_ERR(inst))
 		goto out;
 	inst->alg.cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER;
 	inst->alg.cra_priority = alg->cra_priority;
 	inst->alg.cra_blocksize = 1;
 	inst->alg.cra_alignmask = alg->cra_alignmask | (__alignof__(u32) - 1);
 	inst->alg.cra_type = &crypto_blkcipher_type;
 	inst->alg.cra_blkcipher.ivsize = alg->cra_blocksize;
 	inst->alg.cra_blkcipher.min_keysize = alg->cra_cipher.cia_min_keysize;
 	inst->alg.cra_blkcipher.max_keysize = alg->cra_cipher.cia_max_keysize;
 	inst->alg.cra_ctxsize = sizeof(struct crypto_ctr_ctx);
 	inst->alg.cra_init = crypto_ctr_init_tfm;
 	inst->alg.cra_exit = crypto_ctr_exit_tfm;
 	inst->alg.cra_blkcipher.setkey = crypto_ctr_setkey;
 	inst->alg.cra_blkcipher.encrypt = crypto_ctr_crypt;
 	inst->alg.cra_blkcipher.decrypt = crypto_ctr_crypt;
 	inst->alg.cra_blkcipher.geniv = "chainiv";
 out:
 	crypto_mod_put(alg);
 	return inst;
 out_put_alg:
 	inst = ERR_PTR(err);
 	goto out;
 }
 static void crypto_ctr_free(struct crypto_instance *inst)
 {
 	crypto_drop_spawn(crypto_instance_ctx(inst));
 	kfree(inst);
 }
 static struct crypto_template crypto_ctr_tmpl = {
 	.name = "ctr",
 	.alloc = crypto_ctr_alloc,
 	.free = crypto_ctr_free,
 	.module = THIS_MODULE,
 };
 static int crypto_rfc3686_setkey(struct crypto_ablkcipher *parent,
 				 const u8 *key, unsigned int keylen)
 {
 	struct crypto_rfc3686_ctx *ctx = crypto_ablkcipher_ctx(parent);
 	struct crypto_ablkcipher *child = ctx->child;
 	int err;
 	/* the nonce is stored in bytes at end of key */
 	if (keylen < CTR_RFC3686_NONCE_SIZE)
 		return -EINVAL;
 	memcpy(ctx->nonce, key + (keylen - CTR_RFC3686_NONCE_SIZE),
 	       CTR_RFC3686_NONCE_SIZE);
 	keylen -= CTR_RFC3686_NONCE_SIZE;
 	crypto_ablkcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK);
 	crypto_ablkcipher_set_flags(child, crypto_ablkcipher_get_flags(parent) &
 				    CRYPTO_TFM_REQ_MASK);
 	err = crypto_ablkcipher_setkey(child, key, keylen);
 	crypto_ablkcipher_set_flags(parent, crypto_ablkcipher_get_flags(child) &
 				    CRYPTO_TFM_RES_MASK);
 	return err;
 }
 static int crypto_rfc3686_crypt(struct ablkcipher_request *req)
 {
 	struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
 	struct crypto_rfc3686_ctx *ctx = crypto_ablkcipher_ctx(tfm);
 	struct crypto_ablkcipher *child = ctx->child;
 	unsigned long align = crypto_ablkcipher_alignmask(tfm);
 	struct crypto_rfc3686_req_ctx *rctx =
 		(void *)PTR_ALIGN((u8 *)ablkcipher_request_ctx(req), align + 1);
 	struct ablkcipher_request *subreq = &rctx->subreq;
 	u8 *iv = rctx->iv;
 	/* set up counter block */
 	memcpy(iv, ctx->nonce, CTR_RFC3686_NONCE_SIZE);
 	memcpy(iv + CTR_RFC3686_NONCE_SIZE, req->info, CTR_RFC3686_IV_SIZE);
 	/* initialize counter portion of counter block */
 	*(__be32 *)(iv + CTR_RFC3686_NONCE_SIZE + CTR_RFC3686_IV_SIZE) =
 		cpu_to_be32(1);
 	ablkcipher_request_set_tfm(subreq, child);
 	ablkcipher_request_set_callback(subreq, req->base.flags,
 					req->base.complete, req->base.data);
 	ablkcipher_request_set_crypt(subreq, req->src, req->dst, req->nbytes,
 				     iv);
 	return crypto_ablkcipher_encrypt(subreq);
 }
 static int crypto_rfc3686_init_tfm(struct crypto_tfm *tfm)
 {
 	struct crypto_instance *inst = (void *)tfm->__crt_alg;
 	struct crypto_skcipher_spawn *spawn = crypto_instance_ctx(inst);
 	struct crypto_rfc3686_ctx *ctx = crypto_tfm_ctx(tfm);
 	struct crypto_ablkcipher *cipher;
 	unsigned long align;
 	cipher = crypto_spawn_skcipher(spawn);
 	if (IS_ERR(cipher))
 		return PTR_ERR(cipher);
 	ctx->child = cipher;
 	align = crypto_tfm_alg_alignmask(tfm);
 	align &= ~(crypto_tfm_ctx_alignment() - 1);
 	tfm->crt_ablkcipher.reqsize = align +
 		sizeof(struct crypto_rfc3686_req_ctx) +
 		crypto_ablkcipher_reqsize(cipher);
 	return 0;
 }
 static void crypto_rfc3686_exit_tfm(struct crypto_tfm *tfm)
 {
 	struct crypto_rfc3686_ctx *ctx = crypto_tfm_ctx(tfm);
 	crypto_free_ablkcipher(ctx->child);
 }
 static struct crypto_instance *crypto_rfc3686_alloc(struct rtattr **tb)
 {
 	struct crypto_attr_type *algt;
 	struct crypto_instance *inst;
 	struct crypto_alg *alg;
 	struct crypto_skcipher_spawn *spawn;
 	const char *cipher_name;
 	int err;
 	algt = crypto_get_attr_type(tb);
 	if (IS_ERR(algt))
 		return ERR_CAST(algt);
 	if ((algt->type ^ CRYPTO_ALG_TYPE_BLKCIPHER) & algt->mask)
 		return ERR_PTR(-EINVAL);
 	cipher_name = crypto_attr_alg_name(tb[1]);
 	if (IS_ERR(cipher_name))
 		return ERR_CAST(cipher_name);
 	inst = kzalloc(sizeof(*inst) + sizeof(*spawn), GFP_KERNEL);
 	if (!inst)
 		return ERR_PTR(-ENOMEM);
 	spawn = crypto_instance_ctx(inst);
 	crypto_set_skcipher_spawn(spawn, inst);
 	err = crypto_grab_skcipher(spawn, cipher_name, 0,
 				   crypto_requires_sync(algt->type,
 							algt->mask));
 	if (err)
 		goto err_free_inst;
 	alg = crypto_skcipher_spawn_alg(spawn);
 	/* We only support 16-byte blocks. */
 	err = -EINVAL;
 	if (alg->cra_ablkcipher.ivsize != CTR_RFC3686_BLOCK_SIZE)
 		goto err_drop_spawn;
 	/* Not a stream cipher? */
 	if (alg->cra_blocksize != 1)
 		goto err_drop_spawn;
 	err = -ENAMETOOLONG;
 	if (snprintf(inst->alg.cra_name, CRYPTO_MAX_ALG_NAME, "rfc3686(%s)",
 		     alg->cra_name) >= CRYPTO_MAX_ALG_NAME)
 		goto err_drop_spawn;
 	if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME,
 		     "rfc3686(%s)", alg->cra_driver_name) >=
 			CRYPTO_MAX_ALG_NAME)
 		goto err_drop_spawn;
 	inst->alg.cra_priority = alg->cra_priority;
 	inst->alg.cra_blocksize = 1;
 	inst->alg.cra_alignmask = alg->cra_alignmask;
 	inst->alg.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
 			      (alg->cra_flags & CRYPTO_ALG_ASYNC);
 	inst->alg.cra_type = &crypto_ablkcipher_type;
 	inst->alg.cra_ablkcipher.ivsize = CTR_RFC3686_IV_SIZE;
 	inst->alg.cra_ablkcipher.min_keysize =
 		alg->cra_ablkcipher.min_keysize + CTR_RFC3686_NONCE_SIZE;
 	inst->alg.cra_ablkcipher.max_keysize =
 		alg->cra_ablkcipher.max_keysize + CTR_RFC3686_NONCE_SIZE;
 	inst->alg.cra_ablkcipher.geniv = "seqiv";
 	inst->alg.cra_ablkcipher.setkey = crypto_rfc3686_setkey;
 	inst->alg.cra_ablkcipher.encrypt = crypto_rfc3686_crypt;
 	inst->alg.cra_ablkcipher.decrypt = crypto_rfc3686_crypt;
 	inst->alg.cra_ctxsize = sizeof(struct crypto_rfc3686_ctx);
 	inst->alg.cra_init = crypto_rfc3686_init_tfm;
 	inst->alg.cra_exit = crypto_rfc3686_exit_tfm;
 	return inst;
 err_drop_spawn:
 	crypto_drop_skcipher(spawn);
 err_free_inst:
 	kfree(inst);
 	return ERR_PTR(err);
 }
 static void crypto_rfc3686_free(struct crypto_instance *inst)
 {
 	struct crypto_skcipher_spawn *spawn = crypto_instance_ctx(inst);
 	crypto_drop_skcipher(spawn);
 	kfree(inst);
 }
 static struct crypto_template crypto_rfc3686_tmpl = {
 	.name = "rfc3686",
 	.alloc = crypto_rfc3686_alloc,
 	.free = crypto_rfc3686_free,
 	.module = THIS_MODULE,
 };
 static int __init crypto_ctr_module_init(void)
 {
 	int err;
 	err = crypto_register_template(&crypto_ctr_tmpl);
 	if (err)
 		goto out;
 	err = crypto_register_template(&crypto_rfc3686_tmpl);
 	if (err)
 		goto out_drop_ctr;
 out:
 	return err;
 out_drop_ctr:
 	crypto_unregister_template(&crypto_ctr_tmpl);
 	goto out;
 }
 static void __exit crypto_ctr_module_exit(void)
 {
 	crypto_unregister_template(&crypto_rfc3686_tmpl);
 	crypto_unregister_template(&crypto_ctr_tmpl);
 }
 module_init(crypto_ctr_module_init);
 module_exit(crypto_ctr_module_exit);
 MODULE_LICENSE("GPL");
 MODULE_DESCRIPTION("CTR Counter block mode");
 MODULE_ALIAS_CRYPTO("rfc3686");
+MODULE_ALIAS_CRYPTO("ctr");

crypto/cts.c

Diff comments View file @ acc5ccb

 /*
  * CTS: Cipher Text Stealing mode
  *
  * COPYRIGHT (c) 2008
  * The Regents of the University of Michigan
  * ALL RIGHTS RESERVED
  *
  * Permission is granted to use, copy, create derivative works
  * and redistribute this software and such derivative works
  * for any purpose, so long as the name of The University of
  * Michigan is not used in any advertising or publicity
  * pertaining to the use of distribution of this software
  * without specific, written prior authorization.  If the
  * above copyright notice or any other identification of the
  * University of Michigan is included in any copy of any
  * portion of this software, then the disclaimer below must
  * also be included.
  *
  * THIS SOFTWARE IS PROVIDED AS IS, WITHOUT REPRESENTATION
  * FROM THE UNIVERSITY OF MICHIGAN AS TO ITS FITNESS FOR ANY
  * PURPOSE, AND WITHOUT WARRANTY BY THE UNIVERSITY OF
  * MICHIGAN OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING
  * WITHOUT LIMITATION THE IMPLIED WARRANTIES OF
  * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE
  * REGENTS OF THE UNIVERSITY OF MICHIGAN SHALL NOT BE LIABLE
  * FOR ANY DAMAGES, INCLUDING SPECIAL, INDIRECT, INCIDENTAL, OR
  * CONSEQUENTIAL DAMAGES, WITH RESPECT TO ANY CLAIM ARISING
  * OUT OF OR IN CONNECTION WITH THE USE OF THE SOFTWARE, EVEN
  * IF IT HAS BEEN OR IS HEREAFTER ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGES.
  */
 /* Derived from various:
  *	Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au>
  */
 /*
  * This is the Cipher Text Stealing mode as described by
  * Section 8 of rfc2040 and referenced by rfc3962.
  * rfc3962 includes errata information in its Appendix A.
  */
 #include <crypto/algapi.h>
 #include <linux/err.h>
 #include <linux/init.h>
 #include <linux/kernel.h>
 #include <linux/log2.h>
 #include <linux/module.h>
 #include <linux/scatterlist.h>
 #include <crypto/scatterwalk.h>
 #include <linux/slab.h>
 struct crypto_cts_ctx {
 	struct crypto_blkcipher *child;
 };
 static int crypto_cts_setkey(struct crypto_tfm *parent, const u8 *key,
 			     unsigned int keylen)
 {
 	struct crypto_cts_ctx *ctx = crypto_tfm_ctx(parent);
 	struct crypto_blkcipher *child = ctx->child;
 	int err;
 	crypto_blkcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK);
 	crypto_blkcipher_set_flags(child, crypto_tfm_get_flags(parent) &
 				       CRYPTO_TFM_REQ_MASK);
 	err = crypto_blkcipher_setkey(child, key, keylen);
 	crypto_tfm_set_flags(parent, crypto_blkcipher_get_flags(child) &
 				     CRYPTO_TFM_RES_MASK);
 	return err;
 }
 static int cts_cbc_encrypt(struct crypto_cts_ctx *ctx,
 			   struct blkcipher_desc *desc,
 			   struct scatterlist *dst,
 			   struct scatterlist *src,
 			   unsigned int offset,
 			   unsigned int nbytes)
 {
 	int bsize = crypto_blkcipher_blocksize(desc->tfm);
 	u8 tmp[bsize], tmp2[bsize];
 	struct blkcipher_desc lcldesc;
 	struct scatterlist sgsrc[1], sgdst[1];
 	int lastn = nbytes - bsize;
 	u8 iv[bsize];
 	u8 s[bsize * 2], d[bsize * 2];
 	int err;
 	if (lastn < 0)
 		return -EINVAL;
 	sg_init_table(sgsrc, 1);
 	sg_init_table(sgdst, 1);
 	memset(s, 0, sizeof(s));
 	scatterwalk_map_and_copy(s, src, offset, nbytes, 0);
 	memcpy(iv, desc->info, bsize);
 	lcldesc.tfm = ctx->child;
 	lcldesc.info = iv;
 	lcldesc.flags = desc->flags;
 	sg_set_buf(&sgsrc[0], s, bsize);
 	sg_set_buf(&sgdst[0], tmp, bsize);
 	err = crypto_blkcipher_encrypt_iv(&lcldesc, sgdst, sgsrc, bsize);
 	memcpy(d + bsize, tmp, lastn);
 	lcldesc.info = tmp;
 	sg_set_buf(&sgsrc[0], s + bsize, bsize);
 	sg_set_buf(&sgdst[0], tmp2, bsize);
 	err = crypto_blkcipher_encrypt_iv(&lcldesc, sgdst, sgsrc, bsize);
 	memcpy(d, tmp2, bsize);
 	scatterwalk_map_and_copy(d, dst, offset, nbytes, 1);
 	memcpy(desc->info, tmp2, bsize);
 	return err;
 }
 static int crypto_cts_encrypt(struct blkcipher_desc *desc,
 			      struct scatterlist *dst, struct scatterlist *src,
 			      unsigned int nbytes)
 {
 	struct crypto_cts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 	int bsize = crypto_blkcipher_blocksize(desc->tfm);
 	int tot_blocks = (nbytes + bsize - 1) / bsize;
 	int cbc_blocks = tot_blocks > 2 ? tot_blocks - 2 : 0;
 	struct blkcipher_desc lcldesc;
 	int err;
 	lcldesc.tfm = ctx->child;
 	lcldesc.info = desc->info;
 	lcldesc.flags = desc->flags;
 	if (tot_blocks == 1) {
 		err = crypto_blkcipher_encrypt_iv(&lcldesc, dst, src, bsize);
 	} else if (nbytes <= bsize * 2) {
 		err = cts_cbc_encrypt(ctx, desc, dst, src, 0, nbytes);
 	} else {
 		/* do normal function for tot_blocks - 2 */
 		err = crypto_blkcipher_encrypt_iv(&lcldesc, dst, src,
 							cbc_blocks * bsize);
 		if (err == 0) {
 			/* do cts for final two blocks */
 			err = cts_cbc_encrypt(ctx, desc, dst, src,
 						cbc_blocks * bsize,
 						nbytes - (cbc_blocks * bsize));
 		}
 	}
 	return err;
 }
 static int cts_cbc_decrypt(struct crypto_cts_ctx *ctx,
 			   struct blkcipher_desc *desc,
 			   struct scatterlist *dst,
 			   struct scatterlist *src,
 			   unsigned int offset,
 			   unsigned int nbytes)
 {
 	int bsize = crypto_blkcipher_blocksize(desc->tfm);
 	u8 tmp[bsize];
 	struct blkcipher_desc lcldesc;
 	struct scatterlist sgsrc[1], sgdst[1];
 	int lastn = nbytes - bsize;
 	u8 iv[bsize];
 	u8 s[bsize * 2], d[bsize * 2];
 	int err;
 	if (lastn < 0)
 		return -EINVAL;
 	sg_init_table(sgsrc, 1);
 	sg_init_table(sgdst, 1);
 	scatterwalk_map_and_copy(s, src, offset, nbytes, 0);
 	lcldesc.tfm = ctx->child;
 	lcldesc.info = iv;
 	lcldesc.flags = desc->flags;
 	/* 1. Decrypt Cn-1 (s) to create Dn (tmp)*/
 	memset(iv, 0, sizeof(iv));
 	sg_set_buf(&sgsrc[0], s, bsize);
 	sg_set_buf(&sgdst[0], tmp, bsize);
 	err = crypto_blkcipher_decrypt_iv(&lcldesc, sgdst, sgsrc, bsize);
 	if (err)
 		return err;
 	/* 2. Pad Cn with zeros at the end to create C of length BB */
 	memset(iv, 0, sizeof(iv));
 	memcpy(iv, s + bsize, lastn);
 	/* 3. Exclusive-or Dn (tmp) with C (iv) to create Xn (tmp) */
 	crypto_xor(tmp, iv, bsize);
 	/* 4. Select the first Ln bytes of Xn (tmp) to create Pn */
 	memcpy(d + bsize, tmp, lastn);
 	/* 5. Append the tail (BB - Ln) bytes of Xn (tmp) to Cn to create En */
 	memcpy(s + bsize + lastn, tmp + lastn, bsize - lastn);
 	/* 6. Decrypt En to create Pn-1 */
 	memzero_explicit(iv, sizeof(iv));
 	sg_set_buf(&sgsrc[0], s + bsize, bsize);
 	sg_set_buf(&sgdst[0], d, bsize);
 	err = crypto_blkcipher_decrypt_iv(&lcldesc, sgdst, sgsrc, bsize);
 	/* XOR with previous block */
 	crypto_xor(d, desc->info, bsize);
 	scatterwalk_map_and_copy(d, dst, offset, nbytes, 1);
 	memcpy(desc->info, s, bsize);
 	return err;
 }
 static int crypto_cts_decrypt(struct blkcipher_desc *desc,
 			      struct scatterlist *dst, struct scatterlist *src,
 			      unsigned int nbytes)
 {
 	struct crypto_cts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 	int bsize = crypto_blkcipher_blocksize(desc->tfm);
 	int tot_blocks = (nbytes + bsize - 1) / bsize;
 	int cbc_blocks = tot_blocks > 2 ? tot_blocks - 2 : 0;
 	struct blkcipher_desc lcldesc;
 	int err;
 	lcldesc.tfm = ctx->child;
 	lcldesc.info = desc->info;
 	lcldesc.flags = desc->flags;
 	if (tot_blocks == 1) {
 		err = crypto_blkcipher_decrypt_iv(&lcldesc, dst, src, bsize);
 	} else if (nbytes <= bsize * 2) {
 		err = cts_cbc_decrypt(ctx, desc, dst, src, 0, nbytes);
 	} else {
 		/* do normal function for tot_blocks - 2 */
 		err = crypto_blkcipher_decrypt_iv(&lcldesc, dst, src,
 							cbc_blocks * bsize);
 		if (err == 0) {
 			/* do cts for final two blocks */
 			err = cts_cbc_decrypt(ctx, desc, dst, src,
 						cbc_blocks * bsize,
 						nbytes - (cbc_blocks * bsize));
 		}
 	}
 	return err;
 }
 static int crypto_cts_init_tfm(struct crypto_tfm *tfm)
 {
 	struct crypto_instance *inst = (void *)tfm->__crt_alg;
 	struct crypto_spawn *spawn = crypto_instance_ctx(inst);
 	struct crypto_cts_ctx *ctx = crypto_tfm_ctx(tfm);
 	struct crypto_blkcipher *cipher;
 	cipher = crypto_spawn_blkcipher(spawn);
 	if (IS_ERR(cipher))
 		return PTR_ERR(cipher);
 	ctx->child = cipher;
 	return 0;
 }
 static void crypto_cts_exit_tfm(struct crypto_tfm *tfm)
 {
 	struct crypto_cts_ctx *ctx = crypto_tfm_ctx(tfm);
 	crypto_free_blkcipher(ctx->child);
 }
 static struct crypto_instance *crypto_cts_alloc(struct rtattr **tb)
 {
 	struct crypto_instance *inst;
 	struct crypto_alg *alg;
 	int err;
 	err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_BLKCIPHER);
 	if (err)
 		return ERR_PTR(err);
 	alg = crypto_attr_alg(tb[1], CRYPTO_ALG_TYPE_BLKCIPHER,
 				  CRYPTO_ALG_TYPE_MASK);
 	if (IS_ERR(alg))
 		return ERR_CAST(alg);
 	inst = ERR_PTR(-EINVAL);
 	if (!is_power_of_2(alg->cra_blocksize))
 		goto out_put_alg;
 	inst = crypto_alloc_instance("cts", alg);
 	if (IS_ERR(inst))
 		goto out_put_alg;
 	inst->alg.cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER;
 	inst->alg.cra_priority = alg->cra_priority;
 	inst->alg.cra_blocksize = alg->cra_blocksize;
 	inst->alg.cra_alignmask = alg->cra_alignmask;
 	inst->alg.cra_type = &crypto_blkcipher_type;
 	/* We access the data as u32s when xoring. */
 	inst->alg.cra_alignmask |= __alignof__(u32) - 1;
 	inst->alg.cra_blkcipher.ivsize = alg->cra_blocksize;
 	inst->alg.cra_blkcipher.min_keysize = alg->cra_blkcipher.min_keysize;
 	inst->alg.cra_blkcipher.max_keysize = alg->cra_blkcipher.max_keysize;
 	inst->alg.cra_blkcipher.geniv = "seqiv";
 	inst->alg.cra_ctxsize = sizeof(struct crypto_cts_ctx);
 	inst->alg.cra_init = crypto_cts_init_tfm;
 	inst->alg.cra_exit = crypto_cts_exit_tfm;
 	inst->alg.cra_blkcipher.setkey = crypto_cts_setkey;
 	inst->alg.cra_blkcipher.encrypt = crypto_cts_encrypt;
 	inst->alg.cra_blkcipher.decrypt = crypto_cts_decrypt;
 out_put_alg:
 	crypto_mod_put(alg);
 	return inst;
 }
 static void crypto_cts_free(struct crypto_instance *inst)
 {
 	crypto_drop_spawn(crypto_instance_ctx(inst));
 	kfree(inst);
 }
 static struct crypto_template crypto_cts_tmpl = {
 	.name = "cts",
 	.alloc = crypto_cts_alloc,
 	.free = crypto_cts_free,
 	.module = THIS_MODULE,
 };
 static int __init crypto_cts_module_init(void)
 {
 	return crypto_register_template(&crypto_cts_tmpl);
 }
 static void __exit crypto_cts_module_exit(void)
 {
 	crypto_unregister_template(&crypto_cts_tmpl);
 }
 module_init(crypto_cts_module_init);
 module_exit(crypto_cts_module_exit);
 MODULE_LICENSE("Dual BSD/GPL");
 MODULE_DESCRIPTION("CTS-CBC CipherText Stealing for CBC");
+MODULE_ALIAS_CRYPTO("cts");

crypto/ecb.c

Diff comments View file @ acc5ccb

 /*
  * ECB: Electronic CodeBook mode
  *
  * Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au>
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the Free
  * Software Foundation; either version 2 of the License, or (at your option)
  * any later version.
  *
  */
 #include <crypto/algapi.h>
 #include <linux/err.h>
 #include <linux/init.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/scatterlist.h>
 #include <linux/slab.h>
 struct crypto_ecb_ctx {
 	struct crypto_cipher *child;
 };
 static int crypto_ecb_setkey(struct crypto_tfm *parent, const u8 *key,
 			     unsigned int keylen)
 {
 	struct crypto_ecb_ctx *ctx = crypto_tfm_ctx(parent);
 	struct crypto_cipher *child = ctx->child;
 	int err;
 	crypto_cipher_clear_flags(child, CRYPTO_TFM_REQ_MASK);
 	crypto_cipher_set_flags(child, crypto_tfm_get_flags(parent) &
 				       CRYPTO_TFM_REQ_MASK);
 	err = crypto_cipher_setkey(child, key, keylen);
 	crypto_tfm_set_flags(parent, crypto_cipher_get_flags(child) &
 				     CRYPTO_TFM_RES_MASK);
 	return err;
 }
 static int crypto_ecb_crypt(struct blkcipher_desc *desc,
 			    struct blkcipher_walk *walk,
 			    struct crypto_cipher *tfm,
 			    void (*fn)(struct crypto_tfm *, u8 *, const u8 *))
 {
 	int bsize = crypto_cipher_blocksize(tfm);
 	unsigned int nbytes;
 	int err;
 	err = blkcipher_walk_virt(desc, walk);
 	while ((nbytes = walk->nbytes)) {
 		u8 *wsrc = walk->src.virt.addr;
 		u8 *wdst = walk->dst.virt.addr;
 		do {
 			fn(crypto_cipher_tfm(tfm), wdst, wsrc);
 			wsrc += bsize;
 			wdst += bsize;
 		} while ((nbytes -= bsize) >= bsize);
 		err = blkcipher_walk_done(desc, walk, nbytes);
 	}
 	return err;
 }
 static int crypto_ecb_encrypt(struct blkcipher_desc *desc,
 			      struct scatterlist *dst, struct scatterlist *src,
 			      unsigned int nbytes)
 {
 	struct blkcipher_walk walk;
 	struct crypto_blkcipher *tfm = desc->tfm;
 	struct crypto_ecb_ctx *ctx = crypto_blkcipher_ctx(tfm);
 	struct crypto_cipher *child = ctx->child;
 	blkcipher_walk_init(&walk, dst, src, nbytes);
 	return crypto_ecb_crypt(desc, &walk, child,
 				crypto_cipher_alg(child)->cia_encrypt);
 }
 static int crypto_ecb_decrypt(struct blkcipher_desc *desc,
 			      struct scatterlist *dst, struct scatterlist *src,
 			      unsigned int nbytes)
 {
 	struct blkcipher_walk walk;
 	struct crypto_blkcipher *tfm = desc->tfm;
 	struct crypto_ecb_ctx *ctx = crypto_blkcipher_ctx(tfm);
 	struct crypto_cipher *child = ctx->child;
 	blkcipher_walk_init(&walk, dst, src, nbytes);
 	return crypto_ecb_crypt(desc, &walk, child,
 				crypto_cipher_alg(child)->cia_decrypt);
 }
 static int crypto_ecb_init_tfm(struct crypto_tfm *tfm)
 {
 	struct crypto_instance *inst = (void *)tfm->__crt_alg;
 	struct crypto_spawn *spawn = crypto_instance_ctx(inst);
 	struct crypto_ecb_ctx *ctx = crypto_tfm_ctx(tfm);
 	struct crypto_cipher *cipher;
 	cipher = crypto_spawn_cipher(spawn);
 	if (IS_ERR(cipher))
 		return PTR_ERR(cipher);
 	ctx->child = cipher;
 	return 0;
 }
 static void crypto_ecb_exit_tfm(struct crypto_tfm *tfm)
 {
 	struct crypto_ecb_ctx *ctx = crypto_tfm_ctx(tfm);
 	crypto_free_cipher(ctx->child);
 }
 static struct crypto_instance *crypto_ecb_alloc(struct rtattr **tb)
 {
 	struct crypto_instance *inst;
 	struct crypto_alg *alg;
 	int err;
 	err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_BLKCIPHER);
 	if (err)
 		return ERR_PTR(err);
 	alg = crypto_get_attr_alg(tb, CRYPTO_ALG_TYPE_CIPHER,
 				  CRYPTO_ALG_TYPE_MASK);
 	if (IS_ERR(alg))
 		return ERR_CAST(alg);
 	inst = crypto_alloc_instance("ecb", alg);
 	if (IS_ERR(inst))
 		goto out_put_alg;
 	inst->alg.cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER;
 	inst->alg.cra_priority = alg->cra_priority;
 	inst->alg.cra_blocksize = alg->cra_blocksize;
 	inst->alg.cra_alignmask = alg->cra_alignmask;
 	inst->alg.cra_type = &crypto_blkcipher_type;
 	inst->alg.cra_blkcipher.min_keysize = alg->cra_cipher.cia_min_keysize;
 	inst->alg.cra_blkcipher.max_keysize = alg->cra_cipher.cia_max_keysize;
 	inst->alg.cra_ctxsize = sizeof(struct crypto_ecb_ctx);
 	inst->alg.cra_init = crypto_ecb_init_tfm;
 	inst->alg.cra_exit = crypto_ecb_exit_tfm;
 	inst->alg.cra_blkcipher.setkey = crypto_ecb_setkey;
 	inst->alg.cra_blkcipher.encrypt = crypto_ecb_encrypt;
 	inst->alg.cra_blkcipher.decrypt = crypto_ecb_decrypt;
 out_put_alg:
 	crypto_mod_put(alg);
 	return inst;
 }
 static void crypto_ecb_free(struct crypto_instance *inst)
 {
 	crypto_drop_spawn(crypto_instance_ctx(inst));
 	kfree(inst);
 }
 static struct crypto_template crypto_ecb_tmpl = {
 	.name = "ecb",
 	.alloc = crypto_ecb_alloc,
 	.free = crypto_ecb_free,
 	.module = THIS_MODULE,
 };
 static int __init crypto_ecb_module_init(void)
 {
 	return crypto_register_template(&crypto_ecb_tmpl);
 }
 static void __exit crypto_ecb_module_exit(void)
 {
 	crypto_unregister_template(&crypto_ecb_tmpl);
 }
 module_init(crypto_ecb_module_init);
 module_exit(crypto_ecb_module_exit);
 MODULE_LICENSE("GPL");
 MODULE_DESCRIPTION("ECB block cipher algorithm");
+MODULE_ALIAS_CRYPTO("ecb");

crypto/eseqiv.c

Diff comments View file @ acc5ccb

 /*
  * eseqiv: Encrypted Sequence Number IV Generator
  *
  * This generator generates an IV based on a sequence number by xoring it
  * with a salt and then encrypting it with the same key as used to encrypt
  * the plain text.  This algorithm requires that the block size be equal
  * to the IV size.  It is mainly useful for CBC.
  *
  * Copyright (c) 2007 Herbert Xu <herbert@gondor.apana.org.au>
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the Free
  * Software Foundation; either version 2 of the License, or (at your option)
  * any later version.
  *
  */
 #include <crypto/internal/skcipher.h>
 #include <crypto/rng.h>
 #include <crypto/scatterwalk.h>
 #include <linux/err.h>
 #include <linux/init.h>
 #include <linux/kernel.h>
 #include <linux/mm.h>
 #include <linux/module.h>
 #include <linux/scatterlist.h>
 #include <linux/spinlock.h>
 #include <linux/string.h>
 struct eseqiv_request_ctx {
 	struct scatterlist src[2];
 	struct scatterlist dst[2];
 	char tail[];
 };
 struct eseqiv_ctx {
 	spinlock_t lock;
 	unsigned int reqoff;
 	char salt[];
 };
 static void eseqiv_complete2(struct skcipher_givcrypt_request *req)
 {
 	struct crypto_ablkcipher *geniv = skcipher_givcrypt_reqtfm(req);
 	struct eseqiv_request_ctx *reqctx = skcipher_givcrypt_reqctx(req);
 	memcpy(req->giv, PTR_ALIGN((u8 *)reqctx->tail,
 			 crypto_ablkcipher_alignmask(geniv) + 1),
 	       crypto_ablkcipher_ivsize(geniv));
 }
 static void eseqiv_complete(struct crypto_async_request *base, int err)
 {
 	struct skcipher_givcrypt_request *req = base->data;
 	if (err)
 		goto out;
 	eseqiv_complete2(req);
 out:
 	skcipher_givcrypt_complete(req, err);
 }
 static int eseqiv_givencrypt(struct skcipher_givcrypt_request *req)
 {
 	struct crypto_ablkcipher *geniv = skcipher_givcrypt_reqtfm(req);
 	struct eseqiv_ctx *ctx = crypto_ablkcipher_ctx(geniv);
 	struct eseqiv_request_ctx *reqctx = skcipher_givcrypt_reqctx(req);
 	struct ablkcipher_request *subreq;
 	crypto_completion_t compl;
 	void *data;
 	struct scatterlist *osrc, *odst;
 	struct scatterlist *dst;
 	struct page *srcp;
 	struct page *dstp;
 	u8 *giv;
 	u8 *vsrc;
 	u8 *vdst;
 	__be64 seq;
 	unsigned int ivsize;
 	unsigned int len;
 	int err;
 	subreq = (void *)(reqctx->tail + ctx->reqoff);
 	ablkcipher_request_set_tfm(subreq, skcipher_geniv_cipher(geniv));
 	giv = req->giv;
 	compl = req->creq.base.complete;
 	data = req->creq.base.data;
 	osrc = req->creq.src;
 	odst = req->creq.dst;
 	srcp = sg_page(osrc);
 	dstp = sg_page(odst);
 	vsrc = PageHighMem(srcp) ? NULL : page_address(srcp) + osrc->offset;
 	vdst = PageHighMem(dstp) ? NULL : page_address(dstp) + odst->offset;
 	ivsize = crypto_ablkcipher_ivsize(geniv);
 	if (vsrc != giv + ivsize && vdst != giv + ivsize) {
 		giv = PTR_ALIGN((u8 *)reqctx->tail,
 				crypto_ablkcipher_alignmask(geniv) + 1);
 		compl = eseqiv_complete;
 		data = req;
 	}
 	ablkcipher_request_set_callback(subreq, req->creq.base.flags, compl,
 					data);
 	sg_init_table(reqctx->src, 2);
 	sg_set_buf(reqctx->src, giv, ivsize);
 	scatterwalk_crypto_chain(reqctx->src, osrc, vsrc == giv + ivsize, 2);
 	dst = reqctx->src;
 	if (osrc != odst) {
 		sg_init_table(reqctx->dst, 2);
 		sg_set_buf(reqctx->dst, giv, ivsize);
 		scatterwalk_crypto_chain(reqctx->dst, odst, vdst == giv + ivsize, 2);
 		dst = reqctx->dst;
 	}
 	ablkcipher_request_set_crypt(subreq, reqctx->src, dst,
 				     req->creq.nbytes + ivsize,
 				     req->creq.info);
 	memcpy(req->creq.info, ctx->salt, ivsize);
 	len = ivsize;
 	if (ivsize > sizeof(u64)) {
 		memset(req->giv, 0, ivsize - sizeof(u64));
 		len = sizeof(u64);
 	}
 	seq = cpu_to_be64(req->seq);
 	memcpy(req->giv + ivsize - len, &seq, len);
 	err = crypto_ablkcipher_encrypt(subreq);
 	if (err)
 		goto out;
 	if (giv != req->giv)
 		eseqiv_complete2(req);
 out:
 	return err;
 }
 static int eseqiv_givencrypt_first(struct skcipher_givcrypt_request *req)
 {
 	struct crypto_ablkcipher *geniv = skcipher_givcrypt_reqtfm(req);
 	struct eseqiv_ctx *ctx = crypto_ablkcipher_ctx(geniv);
 	int err = 0;
 	spin_lock_bh(&ctx->lock);
 	if (crypto_ablkcipher_crt(geniv)->givencrypt != eseqiv_givencrypt_first)
 		goto unlock;
 	crypto_ablkcipher_crt(geniv)->givencrypt = eseqiv_givencrypt;
 	err = crypto_rng_get_bytes(crypto_default_rng, ctx->salt,
 				   crypto_ablkcipher_ivsize(geniv));
 unlock:
 	spin_unlock_bh(&ctx->lock);
 	if (err)
 		return err;
 	return eseqiv_givencrypt(req);
 }
 static int eseqiv_init(struct crypto_tfm *tfm)
 {
 	struct crypto_ablkcipher *geniv = __crypto_ablkcipher_cast(tfm);
 	struct eseqiv_ctx *ctx = crypto_ablkcipher_ctx(geniv);
 	unsigned long alignmask;
 	unsigned int reqsize;
 	spin_lock_init(&ctx->lock);
 	alignmask = crypto_tfm_ctx_alignment() - 1;
 	reqsize = sizeof(struct eseqiv_request_ctx);
 	if (alignmask & reqsize) {
 		alignmask &= reqsize;
 		alignmask--;
 	}
 	alignmask = ~alignmask;
 	alignmask &= crypto_ablkcipher_alignmask(geniv);
 	reqsize += alignmask;
 	reqsize += crypto_ablkcipher_ivsize(geniv);
 	reqsize = ALIGN(reqsize, crypto_tfm_ctx_alignment());
 	ctx->reqoff = reqsize - sizeof(struct eseqiv_request_ctx);
 	tfm->crt_ablkcipher.reqsize = reqsize +
 				      sizeof(struct ablkcipher_request);
 	return skcipher_geniv_init(tfm);
 }
 static struct crypto_template eseqiv_tmpl;
 static struct crypto_instance *eseqiv_alloc(struct rtattr **tb)
 {
 	struct crypto_instance *inst;
 	int err;
 	err = crypto_get_default_rng();
 	if (err)
 		return ERR_PTR(err);
 	inst = skcipher_geniv_alloc(&eseqiv_tmpl, tb, 0, 0);
 	if (IS_ERR(inst))
 		goto put_rng;
 	err = -EINVAL;
 	if (inst->alg.cra_ablkcipher.ivsize != inst->alg.cra_blocksize)
 		goto free_inst;
 	inst->alg.cra_ablkcipher.givencrypt = eseqiv_givencrypt_first;
 	inst->alg.cra_init = eseqiv_init;
 	inst->alg.cra_exit = skcipher_geniv_exit;
 	inst->alg.cra_ctxsize = sizeof(struct eseqiv_ctx);
 	inst->alg.cra_ctxsize += inst->alg.cra_ablkcipher.ivsize;
 out:
 	return inst;
 free_inst:
 	skcipher_geniv_free(inst);
 	inst = ERR_PTR(err);
 put_rng:
 	crypto_put_default_rng();
 	goto out;
 }
 static void eseqiv_free(struct crypto_instance *inst)
 {
 	skcipher_geniv_free(inst);
 	crypto_put_default_rng();
 }
 static struct crypto_template eseqiv_tmpl = {
 	.name = "eseqiv",
 	.alloc = eseqiv_alloc,
 	.free = eseqiv_free,
 	.module = THIS_MODULE,
 };
 static int __init eseqiv_module_init(void)
 {
 	return crypto_register_template(&eseqiv_tmpl);
 }
 static void __exit eseqiv_module_exit(void)
 {
 	crypto_unregister_template(&eseqiv_tmpl);
 }
 module_init(eseqiv_module_init);
 module_exit(eseqiv_module_exit);
 MODULE_LICENSE("GPL");
 MODULE_DESCRIPTION("Encrypted Sequence Number IV Generator");
+MODULE_ALIAS_CRYPTO("eseqiv");

crypto/gcm.c

Diff comments View file @ acc5ccb

 /*
  * GCM: Galois/Counter Mode.
  *
  * Copyright (c) 2007 Nokia Siemens Networks - Mikko Herranen <mh1@iki.fi>
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License version 2 as published
  * by the Free Software Foundation.
  */
 #include <crypto/gf128mul.h>
 #include <crypto/internal/aead.h>
 #include <crypto/internal/skcipher.h>
 #include <crypto/internal/hash.h>
 #include <crypto/scatterwalk.h>
 #include <crypto/hash.h>
 #include "internal.h"
 #include <linux/completion.h>
 #include <linux/err.h>
 #include <linux/init.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/slab.h>
 struct gcm_instance_ctx {
 	struct crypto_skcipher_spawn ctr;
 	struct crypto_ahash_spawn ghash;
 };
 struct crypto_gcm_ctx {
 	struct crypto_ablkcipher *ctr;
 	struct crypto_ahash *ghash;
 };
 struct crypto_rfc4106_ctx {
 	struct crypto_aead *child;
 	u8 nonce[4];
 };
 struct crypto_rfc4543_instance_ctx {
 	struct crypto_aead_spawn aead;
 	struct crypto_skcipher_spawn null;
 };
 struct crypto_rfc4543_ctx {
 	struct crypto_aead *child;
 	struct crypto_blkcipher *null;
 	u8 nonce[4];
 };
 struct crypto_rfc4543_req_ctx {
 	u8 auth_tag[16];
 	u8 assocbuf[32];
 	struct scatterlist cipher[1];
 	struct scatterlist payload[2];
 	struct scatterlist assoc[2];
 	struct aead_request subreq;
 };
 struct crypto_gcm_ghash_ctx {
 	unsigned int cryptlen;
 	struct scatterlist *src;
 	void (*complete)(struct aead_request *req, int err);
 };
 struct crypto_gcm_req_priv_ctx {
 	u8 auth_tag[16];
 	u8 iauth_tag[16];
 	struct scatterlist src[2];
 	struct scatterlist dst[2];
 	struct crypto_gcm_ghash_ctx ghash_ctx;
 	union {
 		struct ahash_request ahreq;
 		struct ablkcipher_request abreq;
 	} u;
 };
 struct crypto_gcm_setkey_result {
 	int err;
 	struct completion completion;
 };
 static void *gcm_zeroes;
 static inline struct crypto_gcm_req_priv_ctx *crypto_gcm_reqctx(
 	struct aead_request *req)
 {
 	unsigned long align = crypto_aead_alignmask(crypto_aead_reqtfm(req));
 	return (void *)PTR_ALIGN((u8 *)aead_request_ctx(req), align + 1);
 }
 static void crypto_gcm_setkey_done(struct crypto_async_request *req, int err)
 {
 	struct crypto_gcm_setkey_result *result = req->data;
 	if (err == -EINPROGRESS)
 		return;
 	result->err = err;
 	complete(&result->completion);
 }
 static int crypto_gcm_setkey(struct crypto_aead *aead, const u8 *key,
 			     unsigned int keylen)
 {
 	struct crypto_gcm_ctx *ctx = crypto_aead_ctx(aead);
 	struct crypto_ahash *ghash = ctx->ghash;
 	struct crypto_ablkcipher *ctr = ctx->ctr;
 	struct {
 		be128 hash;
 		u8 iv[8];
 		struct crypto_gcm_setkey_result result;
 		struct scatterlist sg[1];
 		struct ablkcipher_request req;
 	} *data;
 	int err;
 	crypto_ablkcipher_clear_flags(ctr, CRYPTO_TFM_REQ_MASK);
 	crypto_ablkcipher_set_flags(ctr, crypto_aead_get_flags(aead) &
 				   CRYPTO_TFM_REQ_MASK);
 	err = crypto_ablkcipher_setkey(ctr, key, keylen);
 	if (err)
 		return err;
 	crypto_aead_set_flags(aead, crypto_ablkcipher_get_flags(ctr) &
 				       CRYPTO_TFM_RES_MASK);
 	data = kzalloc(sizeof(*data) + crypto_ablkcipher_reqsize(ctr),
 		       GFP_KERNEL);
 	if (!data)
 		return -ENOMEM;
 	init_completion(&data->result.completion);
 	sg_init_one(data->sg, &data->hash, sizeof(data->hash));
 	ablkcipher_request_set_tfm(&data->req, ctr);
 	ablkcipher_request_set_callback(&data->req, CRYPTO_TFM_REQ_MAY_SLEEP |
 						    CRYPTO_TFM_REQ_MAY_BACKLOG,
 					crypto_gcm_setkey_done,
 					&data->result);
 	ablkcipher_request_set_crypt(&data->req, data->sg, data->sg,
 				     sizeof(data->hash), data->iv);
 	err = crypto_ablkcipher_encrypt(&data->req);
 	if (err == -EINPROGRESS || err == -EBUSY) {
 		err = wait_for_completion_interruptible(
 			&data->result.completion);
 		if (!err)
 			err = data->result.err;
 	}
 	if (err)
 		goto out;
 	crypto_ahash_clear_flags(ghash, CRYPTO_TFM_REQ_MASK);
 	crypto_ahash_set_flags(ghash, crypto_aead_get_flags(aead) &
 			       CRYPTO_TFM_REQ_MASK);
 	err = crypto_ahash_setkey(ghash, (u8 *)&data->hash, sizeof(be128));
 	crypto_aead_set_flags(aead, crypto_ahash_get_flags(ghash) &
 			      CRYPTO_TFM_RES_MASK);
 out:
 	kfree(data);
 	return err;
 }
 static int crypto_gcm_setauthsize(struct crypto_aead *tfm,
 				  unsigned int authsize)
 {
 	switch (authsize) {
 	case 4:
 	case 8:
 	case 12:
 	case 13:
 	case 14:
 	case 15:
 	case 16:
 		break;
 	default:
 		return -EINVAL;
 	}
 	return 0;
 }
 static void crypto_gcm_init_crypt(struct ablkcipher_request *ablk_req,
 				  struct aead_request *req,
 				  unsigned int cryptlen)
 {
 	struct crypto_aead *aead = crypto_aead_reqtfm(req);
 	struct crypto_gcm_ctx *ctx = crypto_aead_ctx(aead);
 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
 	struct scatterlist *dst;
 	__be32 counter = cpu_to_be32(1);
 	memset(pctx->auth_tag, 0, sizeof(pctx->auth_tag));
 	memcpy(req->iv + 12, &counter, 4);
 	sg_init_table(pctx->src, 2);
 	sg_set_buf(pctx->src, pctx->auth_tag, sizeof(pctx->auth_tag));
 	scatterwalk_sg_chain(pctx->src, 2, req->src);
 	dst = pctx->src;
 	if (req->src != req->dst) {
 		sg_init_table(pctx->dst, 2);
 		sg_set_buf(pctx->dst, pctx->auth_tag, sizeof(pctx->auth_tag));
 		scatterwalk_sg_chain(pctx->dst, 2, req->dst);
 		dst = pctx->dst;
 	}
 	ablkcipher_request_set_tfm(ablk_req, ctx->ctr);
 	ablkcipher_request_set_crypt(ablk_req, pctx->src, dst,
 				     cryptlen + sizeof(pctx->auth_tag),
 				     req->iv);
 }
 static inline unsigned int gcm_remain(unsigned int len)
 {
 	len &= 0xfU;
 	return len ? 16 - len : 0;
 }
 static void gcm_hash_len_done(struct crypto_async_request *areq, int err);
 static void gcm_hash_final_done(struct crypto_async_request *areq, int err);
 static int gcm_hash_update(struct aead_request *req,
 			   struct crypto_gcm_req_priv_ctx *pctx,
 			   crypto_completion_t compl,
 			   struct scatterlist *src,
 			   unsigned int len)
 {
 	struct ahash_request *ahreq = &pctx->u.ahreq;
 	ahash_request_set_callback(ahreq, aead_request_flags(req),
 				   compl, req);
 	ahash_request_set_crypt(ahreq, src, NULL, len);
 	return crypto_ahash_update(ahreq);
 }
 static int gcm_hash_remain(struct aead_request *req,
 			   struct crypto_gcm_req_priv_ctx *pctx,
 			   unsigned int remain,
 			   crypto_completion_t compl)
 {
 	struct ahash_request *ahreq = &pctx->u.ahreq;
 	ahash_request_set_callback(ahreq, aead_request_flags(req),
 				   compl, req);
 	sg_init_one(pctx->src, gcm_zeroes, remain);
 	ahash_request_set_crypt(ahreq, pctx->src, NULL, remain);
 	return crypto_ahash_update(ahreq);
 }
 static int gcm_hash_len(struct aead_request *req,
 			struct crypto_gcm_req_priv_ctx *pctx)
 {
 	struct ahash_request *ahreq = &pctx->u.ahreq;
 	struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;
 	u128 lengths;
 	lengths.a = cpu_to_be64(req->assoclen * 8);
 	lengths.b = cpu_to_be64(gctx->cryptlen * 8);
 	memcpy(pctx->iauth_tag, &lengths, 16);
 	sg_init_one(pctx->src, pctx->iauth_tag, 16);
 	ahash_request_set_callback(ahreq, aead_request_flags(req),
 				   gcm_hash_len_done, req);
 	ahash_request_set_crypt(ahreq, pctx->src,
 				NULL, sizeof(lengths));
 	return crypto_ahash_update(ahreq);
 }
 static int gcm_hash_final(struct aead_request *req,
 			  struct crypto_gcm_req_priv_ctx *pctx)
 {
 	struct ahash_request *ahreq = &pctx->u.ahreq;
 	ahash_request_set_callback(ahreq, aead_request_flags(req),
 				   gcm_hash_final_done, req);
 	ahash_request_set_crypt(ahreq, NULL, pctx->iauth_tag, 0);
 	return crypto_ahash_final(ahreq);
 }
 static void __gcm_hash_final_done(struct aead_request *req, int err)
 {
 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
 	struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;
 	if (!err)
 		crypto_xor(pctx->auth_tag, pctx->iauth_tag, 16);
 	gctx->complete(req, err);
 }
 static void gcm_hash_final_done(struct crypto_async_request *areq, int err)
 {
 	struct aead_request *req = areq->data;
 	__gcm_hash_final_done(req, err);
 }
 static void __gcm_hash_len_done(struct aead_request *req, int err)
 {
 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
 	if (!err) {
 		err = gcm_hash_final(req, pctx);
 		if (err == -EINPROGRESS || err == -EBUSY)
 			return;
 	}
 	__gcm_hash_final_done(req, err);
 }
 static void gcm_hash_len_done(struct crypto_async_request *areq, int err)
 {
 	struct aead_request *req = areq->data;
 	__gcm_hash_len_done(req, err);
 }
 static void __gcm_hash_crypt_remain_done(struct aead_request *req, int err)
 {
 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
 	if (!err) {
 		err = gcm_hash_len(req, pctx);
 		if (err == -EINPROGRESS || err == -EBUSY)
 			return;
 	}
 	__gcm_hash_len_done(req, err);
 }
 static void gcm_hash_crypt_remain_done(struct crypto_async_request *areq,
 				       int err)
 {
 	struct aead_request *req = areq->data;
 	__gcm_hash_crypt_remain_done(req, err);
 }
 static void __gcm_hash_crypt_done(struct aead_request *req, int err)
 {
 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
 	struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;
 	unsigned int remain;
 	if (!err) {
 		remain = gcm_remain(gctx->cryptlen);
 		BUG_ON(!remain);
 		err = gcm_hash_remain(req, pctx, remain,
 				      gcm_hash_crypt_remain_done);
 		if (err == -EINPROGRESS || err == -EBUSY)
 			return;
 	}
 	__gcm_hash_crypt_remain_done(req, err);
 }
 static void gcm_hash_crypt_done(struct crypto_async_request *areq, int err)
 {
 	struct aead_request *req = areq->data;
 	__gcm_hash_crypt_done(req, err);
 }
 static void __gcm_hash_assoc_remain_done(struct aead_request *req, int err)
 {
 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
 	struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;
 	crypto_completion_t compl;
 	unsigned int remain = 0;
 	if (!err && gctx->cryptlen) {
 		remain = gcm_remain(gctx->cryptlen);
 		compl = remain ? gcm_hash_crypt_done :
 			gcm_hash_crypt_remain_done;
 		err = gcm_hash_update(req, pctx, compl,
 				      gctx->src, gctx->cryptlen);
 		if (err == -EINPROGRESS || err == -EBUSY)
 			return;
 	}
 	if (remain)
 		__gcm_hash_crypt_done(req, err);
 	else
 		__gcm_hash_crypt_remain_done(req, err);
 }
 static void gcm_hash_assoc_remain_done(struct crypto_async_request *areq,
 				       int err)
 {
 	struct aead_request *req = areq->data;
 	__gcm_hash_assoc_remain_done(req, err);
 }
 static void __gcm_hash_assoc_done(struct aead_request *req, int err)
 {
 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
 	unsigned int remain;
 	if (!err) {
 		remain = gcm_remain(req->assoclen);
 		BUG_ON(!remain);
 		err = gcm_hash_remain(req, pctx, remain,
 				      gcm_hash_assoc_remain_done);
 		if (err == -EINPROGRESS || err == -EBUSY)
 			return;
 	}
 	__gcm_hash_assoc_remain_done(req, err);
 }
 static void gcm_hash_assoc_done(struct crypto_async_request *areq, int err)
 {
 	struct aead_request *req = areq->data;
 	__gcm_hash_assoc_done(req, err);
 }
 static void __gcm_hash_init_done(struct aead_request *req, int err)
 {
 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
 	crypto_completion_t compl;
 	unsigned int remain = 0;
 	if (!err && req->assoclen) {
 		remain = gcm_remain(req->assoclen);
 		compl = remain ? gcm_hash_assoc_done :
 			gcm_hash_assoc_remain_done;
 		err = gcm_hash_update(req, pctx, compl,
 				      req->assoc, req->assoclen);
 		if (err == -EINPROGRESS || err == -EBUSY)
 			return;
 	}
 	if (remain)
 		__gcm_hash_assoc_done(req, err);
 	else
 		__gcm_hash_assoc_remain_done(req, err);
 }
 static void gcm_hash_init_done(struct crypto_async_request *areq, int err)
 {
 	struct aead_request *req = areq->data;
 	__gcm_hash_init_done(req, err);
 }
 static int gcm_hash(struct aead_request *req,
 		    struct crypto_gcm_req_priv_ctx *pctx)
 {
 	struct ahash_request *ahreq = &pctx->u.ahreq;
 	struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;
 	struct crypto_gcm_ctx *ctx = crypto_tfm_ctx(req->base.tfm);
 	unsigned int remain;
 	crypto_completion_t compl;
 	int err;
 	ahash_request_set_tfm(ahreq, ctx->ghash);
 	ahash_request_set_callback(ahreq, aead_request_flags(req),
 				   gcm_hash_init_done, req);
 	err = crypto_ahash_init(ahreq);
 	if (err)
 		return err;
 	remain = gcm_remain(req->assoclen);
 	compl = remain ? gcm_hash_assoc_done : gcm_hash_assoc_remain_done;
 	err = gcm_hash_update(req, pctx, compl, req->assoc, req->assoclen);
 	if (err)
 		return err;
 	if (remain) {
 		err = gcm_hash_remain(req, pctx, remain,
 				      gcm_hash_assoc_remain_done);
 		if (err)
 			return err;
 	}
 	remain = gcm_remain(gctx->cryptlen);
 	compl = remain ? gcm_hash_crypt_done : gcm_hash_crypt_remain_done;
 	err = gcm_hash_update(req, pctx, compl, gctx->src, gctx->cryptlen);
 	if (err)
 		return err;
 	if (remain) {
 		err = gcm_hash_remain(req, pctx, remain,
 				      gcm_hash_crypt_remain_done);
 		if (err)
 			return err;
 	}
 	err = gcm_hash_len(req, pctx);
 	if (err)
 		return err;
 	err = gcm_hash_final(req, pctx);
 	if (err)
 		return err;
 	return 0;
 }
 static void gcm_enc_copy_hash(struct aead_request *req,
 			      struct crypto_gcm_req_priv_ctx *pctx)
 {
 	struct crypto_aead *aead = crypto_aead_reqtfm(req);
 	u8 *auth_tag = pctx->auth_tag;
 	scatterwalk_map_and_copy(auth_tag, req->dst, req->cryptlen,
 				 crypto_aead_authsize(aead), 1);
 }
 static void gcm_enc_hash_done(struct aead_request *req, int err)
 {
 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
 	if (!err)
 		gcm_enc_copy_hash(req, pctx);
 	aead_request_complete(req, err);
 }
 static void gcm_encrypt_done(struct crypto_async_request *areq, int err)
 {
 	struct aead_request *req = areq->data;
 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
 	if (!err) {
 		err = gcm_hash(req, pctx);
 		if (err == -EINPROGRESS || err == -EBUSY)
 			return;
 		else if (!err) {
 			crypto_xor(pctx->auth_tag, pctx->iauth_tag, 16);
 			gcm_enc_copy_hash(req, pctx);
 		}
 	}
 	aead_request_complete(req, err);
 }
 static int crypto_gcm_encrypt(struct aead_request *req)
 {
 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
 	struct ablkcipher_request *abreq = &pctx->u.abreq;
 	struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;
 	int err;
 	crypto_gcm_init_crypt(abreq, req, req->cryptlen);
 	ablkcipher_request_set_callback(abreq, aead_request_flags(req),
 					gcm_encrypt_done, req);
 	gctx->src = req->dst;
 	gctx->cryptlen = req->cryptlen;
 	gctx->complete = gcm_enc_hash_done;
 	err = crypto_ablkcipher_encrypt(abreq);
 	if (err)
 		return err;
 	err = gcm_hash(req, pctx);
 	if (err)
 		return err;
 	crypto_xor(pctx->auth_tag, pctx->iauth_tag, 16);
 	gcm_enc_copy_hash(req, pctx);
 	return 0;
 }
 static int crypto_gcm_verify(struct aead_request *req,
 			     struct crypto_gcm_req_priv_ctx *pctx)
 {
 	struct crypto_aead *aead = crypto_aead_reqtfm(req);
 	u8 *auth_tag = pctx->auth_tag;
 	u8 *iauth_tag = pctx->iauth_tag;
 	unsigned int authsize = crypto_aead_authsize(aead);
 	unsigned int cryptlen = req->cryptlen - authsize;
 	crypto_xor(auth_tag, iauth_tag, 16);
 	scatterwalk_map_and_copy(iauth_tag, req->src, cryptlen, authsize, 0);
 	return crypto_memneq(iauth_tag, auth_tag, authsize) ? -EBADMSG : 0;
 }
 static void gcm_decrypt_done(struct crypto_async_request *areq, int err)
 {
 	struct aead_request *req = areq->data;
 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
 	if (!err)
 		err = crypto_gcm_verify(req, pctx);
 	aead_request_complete(req, err);
 }
 static void gcm_dec_hash_done(struct aead_request *req, int err)
 {
 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
 	struct ablkcipher_request *abreq = &pctx->u.abreq;
 	struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;
 	if (!err) {
 		ablkcipher_request_set_callback(abreq, aead_request_flags(req),
 						gcm_decrypt_done, req);
 		crypto_gcm_init_crypt(abreq, req, gctx->cryptlen);
 		err = crypto_ablkcipher_decrypt(abreq);
 		if (err == -EINPROGRESS || err == -EBUSY)
 			return;
 		else if (!err)
 			err = crypto_gcm_verify(req, pctx);
 	}
 	aead_request_complete(req, err);
 }
 static int crypto_gcm_decrypt(struct aead_request *req)
 {
 	struct crypto_aead *aead = crypto_aead_reqtfm(req);
 	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
 	struct ablkcipher_request *abreq = &pctx->u.abreq;
 	struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;
 	unsigned int authsize = crypto_aead_authsize(aead);
 	unsigned int cryptlen = req->cryptlen;
 	int err;
 	if (cryptlen < authsize)
 		return -EINVAL;
 	cryptlen -= authsize;
 	gctx->src = req->src;
 	gctx->cryptlen = cryptlen;
 	gctx->complete = gcm_dec_hash_done;
 	err = gcm_hash(req, pctx);
 	if (err)
 		return err;
 	ablkcipher_request_set_callback(abreq, aead_request_flags(req),
 					gcm_decrypt_done, req);
 	crypto_gcm_init_crypt(abreq, req, cryptlen);
 	err = crypto_ablkcipher_decrypt(abreq);
 	if (err)
 		return err;
 	return crypto_gcm_verify(req, pctx);
 }
 static int crypto_gcm_init_tfm(struct crypto_tfm *tfm)
 {
 	struct crypto_instance *inst = (void *)tfm->__crt_alg;
 	struct gcm_instance_ctx *ictx = crypto_instance_ctx(inst);
 	struct crypto_gcm_ctx *ctx = crypto_tfm_ctx(tfm);
 	struct crypto_ablkcipher *ctr;
 	struct crypto_ahash *ghash;
 	unsigned long align;
 	int err;
 	ghash = crypto_spawn_ahash(&ictx->ghash);
 	if (IS_ERR(ghash))
 		return PTR_ERR(ghash);
 	ctr = crypto_spawn_skcipher(&ictx->ctr);
 	err = PTR_ERR(ctr);
 	if (IS_ERR(ctr))
 		goto err_free_hash;
 	ctx->ctr = ctr;
 	ctx->ghash = ghash;
 	align = crypto_tfm_alg_alignmask(tfm);
 	align &= ~(crypto_tfm_ctx_alignment() - 1);
 	tfm->crt_aead.reqsize = align +
 		offsetof(struct crypto_gcm_req_priv_ctx, u) +
 		max(sizeof(struct ablkcipher_request) +
 		    crypto_ablkcipher_reqsize(ctr),
 		    sizeof(struct ahash_request) +
 		    crypto_ahash_reqsize(ghash));
 	return 0;
 err_free_hash:
 	crypto_free_ahash(ghash);
 	return err;
 }
 static void crypto_gcm_exit_tfm(struct crypto_tfm *tfm)
 {
 	struct crypto_gcm_ctx *ctx = crypto_tfm_ctx(tfm);
 	crypto_free_ahash(ctx->ghash);
 	crypto_free_ablkcipher(ctx->ctr);
 }
 static struct crypto_instance *crypto_gcm_alloc_common(struct rtattr **tb,
 						       const char *full_name,
 						       const char *ctr_name,
 						       const char *ghash_name)
 {
 	struct crypto_attr_type *algt;
 	struct crypto_instance *inst;
 	struct crypto_alg *ctr;
 	struct crypto_alg *ghash_alg;
 	struct ahash_alg *ghash_ahash_alg;
 	struct gcm_instance_ctx *ctx;
 	int err;
 	algt = crypto_get_attr_type(tb);
 	if (IS_ERR(algt))
 		return ERR_CAST(algt);
 	if ((algt->type ^ CRYPTO_ALG_TYPE_AEAD) & algt->mask)
 		return ERR_PTR(-EINVAL);
 	ghash_alg = crypto_find_alg(ghash_name, &crypto_ahash_type,
 				    CRYPTO_ALG_TYPE_HASH,
 				    CRYPTO_ALG_TYPE_AHASH_MASK);
 	if (IS_ERR(ghash_alg))
 		return ERR_CAST(ghash_alg);
 	err = -ENOMEM;
 	inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL);
 	if (!inst)
 		goto out_put_ghash;
 	ctx = crypto_instance_ctx(inst);
 	ghash_ahash_alg = container_of(ghash_alg, struct ahash_alg, halg.base);
 	err = crypto_init_ahash_spawn(&ctx->ghash, &ghash_ahash_alg->halg,
 				      inst);
 	if (err)
 		goto err_free_inst;
 	crypto_set_skcipher_spawn(&ctx->ctr, inst);
 	err = crypto_grab_skcipher(&ctx->ctr, ctr_name, 0,
 				   crypto_requires_sync(algt->type,
 							algt->mask));
 	if (err)
 		goto err_drop_ghash;
 	ctr = crypto_skcipher_spawn_alg(&ctx->ctr);
 	/* We only support 16-byte blocks. */
 	if (ctr->cra_ablkcipher.ivsize != 16)
 		goto out_put_ctr;
 	/* Not a stream cipher? */
 	err = -EINVAL;
 	if (ctr->cra_blocksize != 1)
 		goto out_put_ctr;
 	err = -ENAMETOOLONG;
 	if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME,
 		     "gcm_base(%s,%s)", ctr->cra_driver_name,
 		     ghash_alg->cra_driver_name) >=
 	    CRYPTO_MAX_ALG_NAME)
 		goto out_put_ctr;
 	memcpy(inst->alg.cra_name, full_name, CRYPTO_MAX_ALG_NAME);
 	inst->alg.cra_flags = CRYPTO_ALG_TYPE_AEAD;
 	inst->alg.cra_flags |= ctr->cra_flags & CRYPTO_ALG_ASYNC;
 	inst->alg.cra_priority = ctr->cra_priority;
 	inst->alg.cra_blocksize = 1;
 	inst->alg.cra_alignmask = ctr->cra_alignmask | (__alignof__(u64) - 1);
 	inst->alg.cra_type = &crypto_aead_type;
 	inst->alg.cra_aead.ivsize = 16;
 	inst->alg.cra_aead.maxauthsize = 16;
 	inst->alg.cra_ctxsize = sizeof(struct crypto_gcm_ctx);
 	inst->alg.cra_init = crypto_gcm_init_tfm;
 	inst->alg.cra_exit = crypto_gcm_exit_tfm;
 	inst->alg.cra_aead.setkey = crypto_gcm_setkey;
 	inst->alg.cra_aead.setauthsize = crypto_gcm_setauthsize;
 	inst->alg.cra_aead.encrypt = crypto_gcm_encrypt;
 	inst->alg.cra_aead.decrypt = crypto_gcm_decrypt;
 out:
 	crypto_mod_put(ghash_alg);
 	return inst;
 out_put_ctr:
 	crypto_drop_skcipher(&ctx->ctr);
 err_drop_ghash:
 	crypto_drop_ahash(&ctx->ghash);
 err_free_inst:
 	kfree(inst);
 out_put_ghash:
 	inst = ERR_PTR(err);
 	goto out;
 }
 static struct crypto_instance *crypto_gcm_alloc(struct rtattr **tb)
 {
 	const char *cipher_name;
 	char ctr_name[CRYPTO_MAX_ALG_NAME];
 	char full_name[CRYPTO_MAX_ALG_NAME];
 	cipher_name = crypto_attr_alg_name(tb[1]);
 	if (IS_ERR(cipher_name))
 		return ERR_CAST(cipher_name);
 	if (snprintf(ctr_name, CRYPTO_MAX_ALG_NAME, "ctr(%s)", cipher_name) >=
 	    CRYPTO_MAX_ALG_NAME)
 		return ERR_PTR(-ENAMETOOLONG);
 	if (snprintf(full_name, CRYPTO_MAX_ALG_NAME, "gcm(%s)", cipher_name) >=
 	    CRYPTO_MAX_ALG_NAME)
 		return ERR_PTR(-ENAMETOOLONG);
 	return crypto_gcm_alloc_common(tb, full_name, ctr_name, "ghash");
 }
 static void crypto_gcm_free(struct crypto_instance *inst)
 {
 	struct gcm_instance_ctx *ctx = crypto_instance_ctx(inst);
 	crypto_drop_skcipher(&ctx->ctr);
 	crypto_drop_ahash(&ctx->ghash);
 	kfree(inst);
 }
 static struct crypto_template crypto_gcm_tmpl = {
 	.name = "gcm",
 	.alloc = crypto_gcm_alloc,
 	.free = crypto_gcm_free,
 	.module = THIS_MODULE,
 };
 static struct crypto_instance *crypto_gcm_base_alloc(struct rtattr **tb)
 {
 	const char *ctr_name;
 	const char *ghash_name;
 	char full_name[CRYPTO_MAX_ALG_NAME];
 	ctr_name = crypto_attr_alg_name(tb[1]);
 	if (IS_ERR(ctr_name))
 		return ERR_CAST(ctr_name);
 	ghash_name = crypto_attr_alg_name(tb[2]);
 	if (IS_ERR(ghash_name))
 		return ERR_CAST(ghash_name);
 	if (snprintf(full_name, CRYPTO_MAX_ALG_NAME, "gcm_base(%s,%s)",
 		     ctr_name, ghash_name) >= CRYPTO_MAX_ALG_NAME)
 		return ERR_PTR(-ENAMETOOLONG);
 	return crypto_gcm_alloc_common(tb, full_name, ctr_name, ghash_name);
 }
 static struct crypto_template crypto_gcm_base_tmpl = {
 	.name = "gcm_base",
 	.alloc = crypto_gcm_base_alloc,
 	.free = crypto_gcm_free,
 	.module = THIS_MODULE,
 };
 static int crypto_rfc4106_setkey(struct crypto_aead *parent, const u8 *key,
 				 unsigned int keylen)
 {
 	struct crypto_rfc4106_ctx *ctx = crypto_aead_ctx(parent);
 	struct crypto_aead *child = ctx->child;
 	int err;
 	if (keylen < 4)
 		return -EINVAL;
 	keylen -= 4;
 	memcpy(ctx->nonce, key + keylen, 4);
 	crypto_aead_clear_flags(child, CRYPTO_TFM_REQ_MASK);
 	crypto_aead_set_flags(child, crypto_aead_get_flags(parent) &
 				     CRYPTO_TFM_REQ_MASK);
 	err = crypto_aead_setkey(child, key, keylen);
 	crypto_aead_set_flags(parent, crypto_aead_get_flags(child) &
 				      CRYPTO_TFM_RES_MASK);
 	return err;
 }
 static int crypto_rfc4106_setauthsize(struct crypto_aead *parent,
 				      unsigned int authsize)
 {
 	struct crypto_rfc4106_ctx *ctx = crypto_aead_ctx(parent);
 	switch (authsize) {
 	case 8:
 	case 12:
 	case 16:
 		break;
 	default:
 		return -EINVAL;
 	}
 	return crypto_aead_setauthsize(ctx->child, authsize);
 }
 static struct aead_request *crypto_rfc4106_crypt(struct aead_request *req)
 {
 	struct aead_request *subreq = aead_request_ctx(req);
 	struct crypto_aead *aead = crypto_aead_reqtfm(req);
 	struct crypto_rfc4106_ctx *ctx = crypto_aead_ctx(aead);
 	struct crypto_aead *child = ctx->child;
 	u8 *iv = PTR_ALIGN((u8 *)(subreq + 1) + crypto_aead_reqsize(child),
 			   crypto_aead_alignmask(child) + 1);
 	memcpy(iv, ctx->nonce, 4);
 	memcpy(iv + 4, req->iv, 8);
 	aead_request_set_tfm(subreq, child);
 	aead_request_set_callback(subreq, req->base.flags, req->base.complete,
 				  req->base.data);
 	aead_request_set_crypt(subreq, req->src, req->dst, req->cryptlen, iv);
 	aead_request_set_assoc(subreq, req->assoc, req->assoclen);
 	return subreq;
 }
 static int crypto_rfc4106_encrypt(struct aead_request *req)
 {
 	req = crypto_rfc4106_crypt(req);
 	return crypto_aead_encrypt(req);
 }
 static int crypto_rfc4106_decrypt(struct aead_request *req)
 {
 	req = crypto_rfc4106_crypt(req);
 	return crypto_aead_decrypt(req);
 }
 static int crypto_rfc4106_init_tfm(struct crypto_tfm *tfm)
 {
 	struct crypto_instance *inst = (void *)tfm->__crt_alg;
 	struct crypto_aead_spawn *spawn = crypto_instance_ctx(inst);
 	struct crypto_rfc4106_ctx *ctx = crypto_tfm_ctx(tfm);
 	struct crypto_aead *aead;
 	unsigned long align;
 	aead = crypto_spawn_aead(spawn);
 	if (IS_ERR(aead))
 		return PTR_ERR(aead);
 	ctx->child = aead;
 	align = crypto_aead_alignmask(aead);
 	align &= ~(crypto_tfm_ctx_alignment() - 1);
 	tfm->crt_aead.reqsize = sizeof(struct aead_request) +
 				ALIGN(crypto_aead_reqsize(aead),
 				      crypto_tfm_ctx_alignment()) +
 				align + 16;
 	return 0;
 }
 static void crypto_rfc4106_exit_tfm(struct crypto_tfm *tfm)
 {
 	struct crypto_rfc4106_ctx *ctx = crypto_tfm_ctx(tfm);
 	crypto_free_aead(ctx->child);
 }
 static struct crypto_instance *crypto_rfc4106_alloc(struct rtattr **tb)
 {
 	struct crypto_attr_type *algt;
 	struct crypto_instance *inst;
 	struct crypto_aead_spawn *spawn;
 	struct crypto_alg *alg;
 	const char *ccm_name;
 	int err;
 	algt = crypto_get_attr_type(tb);
 	if (IS_ERR(algt))
 		return ERR_CAST(algt);
 	if ((algt->type ^ CRYPTO_ALG_TYPE_AEAD) & algt->mask)
 		return ERR_PTR(-EINVAL);
 	ccm_name = crypto_attr_alg_name(tb[1]);
 	if (IS_ERR(ccm_name))
 		return ERR_CAST(ccm_name);
 	inst = kzalloc(sizeof(*inst) + sizeof(*spawn), GFP_KERNEL);
 	if (!inst)
 		return ERR_PTR(-ENOMEM);
 	spawn = crypto_instance_ctx(inst);
 	crypto_set_aead_spawn(spawn, inst);
 	err = crypto_grab_aead(spawn, ccm_name, 0,
 			       crypto_requires_sync(algt->type, algt->mask));
 	if (err)
 		goto out_free_inst;
 	alg = crypto_aead_spawn_alg(spawn);
 	err = -EINVAL;
 	/* We only support 16-byte blocks. */
 	if (alg->cra_aead.ivsize != 16)
 		goto out_drop_alg;
 	/* Not a stream cipher? */
 	if (alg->cra_blocksize != 1)
 		goto out_drop_alg;
 	err = -ENAMETOOLONG;
 	if (snprintf(inst->alg.cra_name, CRYPTO_MAX_ALG_NAME,
 		     "rfc4106(%s)", alg->cra_name) >= CRYPTO_MAX_ALG_NAME ||
 	    snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME,
 		     "rfc4106(%s)", alg->cra_driver_name) >=
 	    CRYPTO_MAX_ALG_NAME)
 		goto out_drop_alg;
 	inst->alg.cra_flags = CRYPTO_ALG_TYPE_AEAD;
 	inst->alg.cra_flags |= alg->cra_flags & CRYPTO_ALG_ASYNC;
 	inst->alg.cra_priority = alg->cra_priority;
 	inst->alg.cra_blocksize = 1;
 	inst->alg.cra_alignmask = alg->cra_alignmask;
 	inst->alg.cra_type = &crypto_nivaead_type;
 	inst->alg.cra_aead.ivsize = 8;
 	inst->alg.cra_aead.maxauthsize = 16;
 	inst->alg.cra_ctxsize = sizeof(struct crypto_rfc4106_ctx);
 	inst->alg.cra_init = crypto_rfc4106_init_tfm;
 	inst->alg.cra_exit = crypto_rfc4106_exit_tfm;
 	inst->alg.cra_aead.setkey = crypto_rfc4106_setkey;
 	inst->alg.cra_aead.setauthsize = crypto_rfc4106_setauthsize;
 	inst->alg.cra_aead.encrypt = crypto_rfc4106_encrypt;
 	inst->alg.cra_aead.decrypt = crypto_rfc4106_decrypt;
 	inst->alg.cra_aead.geniv = "seqiv";
 out:
 	return inst;
 out_drop_alg:
 	crypto_drop_aead(spawn);
 out_free_inst:
 	kfree(inst);
 	inst = ERR_PTR(err);
 	goto out;
 }
 static void crypto_rfc4106_free(struct crypto_instance *inst)
 {
 	crypto_drop_spawn(crypto_instance_ctx(inst));
 	kfree(inst);
 }
 static struct crypto_template crypto_rfc4106_tmpl = {
 	.name = "rfc4106",
 	.alloc = crypto_rfc4106_alloc,
 	.free = crypto_rfc4106_free,
 	.module = THIS_MODULE,
 };
 static inline struct crypto_rfc4543_req_ctx *crypto_rfc4543_reqctx(
 	struct aead_request *req)
 {
 	unsigned long align = crypto_aead_alignmask(crypto_aead_reqtfm(req));
 	return (void *)PTR_ALIGN((u8 *)aead_request_ctx(req), align + 1);
 }
 static int crypto_rfc4543_setkey(struct crypto_aead *parent, const u8 *key,
 				 unsigned int keylen)
 {
 	struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(parent);
 	struct crypto_aead *child = ctx->child;
 	int err;
 	if (keylen < 4)
 		return -EINVAL;
 	keylen -= 4;
 	memcpy(ctx->nonce, key + keylen, 4);
 	crypto_aead_clear_flags(child, CRYPTO_TFM_REQ_MASK);
 	crypto_aead_set_flags(child, crypto_aead_get_flags(parent) &
 				     CRYPTO_TFM_REQ_MASK);
 	err = crypto_aead_setkey(child, key, keylen);
 	crypto_aead_set_flags(parent, crypto_aead_get_flags(child) &
 				      CRYPTO_TFM_RES_MASK);
 	return err;
 }
 static int crypto_rfc4543_setauthsize(struct crypto_aead *parent,
 				      unsigned int authsize)
 {
 	struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(parent);
 	if (authsize != 16)
 		return -EINVAL;
 	return crypto_aead_setauthsize(ctx->child, authsize);
 }
 static void crypto_rfc4543_done(struct crypto_async_request *areq, int err)
 {
 	struct aead_request *req = areq->data;
 	struct crypto_aead *aead = crypto_aead_reqtfm(req);
 	struct crypto_rfc4543_req_ctx *rctx = crypto_rfc4543_reqctx(req);
 	if (!err) {
 		scatterwalk_map_and_copy(rctx->auth_tag, req->dst,
 					 req->cryptlen,
 					 crypto_aead_authsize(aead), 1);
 	}
 	aead_request_complete(req, err);
 }
 static struct aead_request *crypto_rfc4543_crypt(struct aead_request *req,
 						 bool enc)
 {
 	struct crypto_aead *aead = crypto_aead_reqtfm(req);
 	struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(aead);
 	struct crypto_rfc4543_req_ctx *rctx = crypto_rfc4543_reqctx(req);
 	struct aead_request *subreq = &rctx->subreq;
 	struct scatterlist *src = req->src;
 	struct scatterlist *cipher = rctx->cipher;
 	struct scatterlist *payload = rctx->payload;
 	struct scatterlist *assoc = rctx->assoc;
 	unsigned int authsize = crypto_aead_authsize(aead);
 	unsigned int assoclen = req->assoclen;
 	struct page *srcp;
 	u8 *vsrc;
 	u8 *iv = PTR_ALIGN((u8 *)(rctx + 1) + crypto_aead_reqsize(ctx->child),
 			   crypto_aead_alignmask(ctx->child) + 1);
 	memcpy(iv, ctx->nonce, 4);
 	memcpy(iv + 4, req->iv, 8);
 	/* construct cipher/plaintext */
 	if (enc)
 		memset(rctx->auth_tag, 0, authsize);
 	else
 		scatterwalk_map_and_copy(rctx->auth_tag, src,
 					 req->cryptlen - authsize,
 					 authsize, 0);
 	sg_init_one(cipher, rctx->auth_tag, authsize);
 	/* construct the aad */
 	srcp = sg_page(src);
 	vsrc = PageHighMem(srcp) ? NULL : page_address(srcp) + src->offset;
 	sg_init_table(payload, 2);
 	sg_set_buf(payload, req->iv, 8);
 	scatterwalk_crypto_chain(payload, src, vsrc == req->iv + 8, 2);
 	assoclen += 8 + req->cryptlen - (enc ? 0 : authsize);
 	if (req->assoc->length == req->assoclen) {
 		sg_init_table(assoc, 2);
 		sg_set_page(assoc, sg_page(req->assoc), req->assoc->length,
 			    req->assoc->offset);
 	} else {
 		BUG_ON(req->assoclen > sizeof(rctx->assocbuf));
 		scatterwalk_map_and_copy(rctx->assocbuf, req->assoc, 0,
 					 req->assoclen, 0);
 		sg_init_table(assoc, 2);
 		sg_set_buf(assoc, rctx->assocbuf, req->assoclen);
 	}
 	scatterwalk_crypto_chain(assoc, payload, 0, 2);
 	aead_request_set_tfm(subreq, ctx->child);
 	aead_request_set_callback(subreq, req->base.flags, crypto_rfc4543_done,
 				  req);
 	aead_request_set_crypt(subreq, cipher, cipher, enc ? 0 : authsize, iv);
 	aead_request_set_assoc(subreq, assoc, assoclen);
 	return subreq;
 }
 static int crypto_rfc4543_copy_src_to_dst(struct aead_request *req, bool enc)
 {
 	struct crypto_aead *aead = crypto_aead_reqtfm(req);
 	struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(aead);
 	unsigned int authsize = crypto_aead_authsize(aead);
 	unsigned int nbytes = req->cryptlen - (enc ? 0 : authsize);
 	struct blkcipher_desc desc = {
 		.tfm = ctx->null,
 	};
 	return crypto_blkcipher_encrypt(&desc, req->dst, req->src, nbytes);
 }
 static int crypto_rfc4543_encrypt(struct aead_request *req)
 {
 	struct crypto_aead *aead = crypto_aead_reqtfm(req);
 	struct crypto_rfc4543_req_ctx *rctx = crypto_rfc4543_reqctx(req);
 	struct aead_request *subreq;
 	int err;
 	if (req->src != req->dst) {
 		err = crypto_rfc4543_copy_src_to_dst(req, true);
 		if (err)
 			return err;
 	}
 	subreq = crypto_rfc4543_crypt(req, true);
 	err = crypto_aead_encrypt(subreq);
 	if (err)
 		return err;
 	scatterwalk_map_and_copy(rctx->auth_tag, req->dst, req->cryptlen,
 				 crypto_aead_authsize(aead), 1);
 	return 0;
 }
 static int crypto_rfc4543_decrypt(struct aead_request *req)
 {
 	int err;
 	if (req->src != req->dst) {
 		err = crypto_rfc4543_copy_src_to_dst(req, false);
 		if (err)
 			return err;
 	}
 	req = crypto_rfc4543_crypt(req, false);
 	return crypto_aead_decrypt(req);
 }
 static int crypto_rfc4543_init_tfm(struct crypto_tfm *tfm)
 {
 	struct crypto_instance *inst = (void *)tfm->__crt_alg;
 	struct crypto_rfc4543_instance_ctx *ictx = crypto_instance_ctx(inst);
 	struct crypto_aead_spawn *spawn = &ictx->aead;
 	struct crypto_rfc4543_ctx *ctx = crypto_tfm_ctx(tfm);
 	struct crypto_aead *aead;
 	struct crypto_blkcipher *null;
 	unsigned long align;
 	int err = 0;
 	aead = crypto_spawn_aead(spawn);
 	if (IS_ERR(aead))
 		return PTR_ERR(aead);
 	null = crypto_spawn_blkcipher(&ictx->null.base);
 	err = PTR_ERR(null);
 	if (IS_ERR(null))
 		goto err_free_aead;
 	ctx->child = aead;
 	ctx->null = null;
 	align = crypto_aead_alignmask(aead);
 	align &= ~(crypto_tfm_ctx_alignment() - 1);
 	tfm->crt_aead.reqsize = sizeof(struct crypto_rfc4543_req_ctx) +
 				ALIGN(crypto_aead_reqsize(aead),
 				      crypto_tfm_ctx_alignment()) +
 				align + 16;
 	return 0;
 err_free_aead:
 	crypto_free_aead(aead);
 	return err;
 }
 static void crypto_rfc4543_exit_tfm(struct crypto_tfm *tfm)
 {
 	struct crypto_rfc4543_ctx *ctx = crypto_tfm_ctx(tfm);
 	crypto_free_aead(ctx->child);
 	crypto_free_blkcipher(ctx->null);
 }
 static struct crypto_instance *crypto_rfc4543_alloc(struct rtattr **tb)
 {
 	struct crypto_attr_type *algt;
 	struct crypto_instance *inst;
 	struct crypto_aead_spawn *spawn;
 	struct crypto_alg *alg;
 	struct crypto_rfc4543_instance_ctx *ctx;
 	const char *ccm_name;
 	int err;
 	algt = crypto_get_attr_type(tb);
 	if (IS_ERR(algt))
 		return ERR_CAST(algt);
 	if ((algt->type ^ CRYPTO_ALG_TYPE_AEAD) & algt->mask)
 		return ERR_PTR(-EINVAL);
 	ccm_name = crypto_attr_alg_name(tb[1]);
 	if (IS_ERR(ccm_name))
 		return ERR_CAST(ccm_name);
 	inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL);
 	if (!inst)
 		return ERR_PTR(-ENOMEM);
 	ctx = crypto_instance_ctx(inst);
 	spawn = &ctx->aead;
 	crypto_set_aead_spawn(spawn, inst);
 	err = crypto_grab_aead(spawn, ccm_name, 0,
 			       crypto_requires_sync(algt->type, algt->mask));
 	if (err)
 		goto out_free_inst;
 	alg = crypto_aead_spawn_alg(spawn);
 	crypto_set_skcipher_spawn(&ctx->null, inst);
 	err = crypto_grab_skcipher(&ctx->null, "ecb(cipher_null)", 0,
 				   CRYPTO_ALG_ASYNC);
 	if (err)
 		goto out_drop_alg;
 	crypto_skcipher_spawn_alg(&ctx->null);
 	err = -EINVAL;
 	/* We only support 16-byte blocks. */
 	if (alg->cra_aead.ivsize != 16)
 		goto out_drop_ecbnull;
 	/* Not a stream cipher? */
 	if (alg->cra_blocksize != 1)
 		goto out_drop_ecbnull;
 	err = -ENAMETOOLONG;
 	if (snprintf(inst->alg.cra_name, CRYPTO_MAX_ALG_NAME,
 		     "rfc4543(%s)", alg->cra_name) >= CRYPTO_MAX_ALG_NAME ||
 	    snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME,
 		     "rfc4543(%s)", alg->cra_driver_name) >=
 	    CRYPTO_MAX_ALG_NAME)
 		goto out_drop_ecbnull;
 	inst->alg.cra_flags = CRYPTO_ALG_TYPE_AEAD;
 	inst->alg.cra_flags |= alg->cra_flags & CRYPTO_ALG_ASYNC;
 	inst->alg.cra_priority = alg->cra_priority;
 	inst->alg.cra_blocksize = 1;
 	inst->alg.cra_alignmask = alg->cra_alignmask;
 	inst->alg.cra_type = &crypto_nivaead_type;
 	inst->alg.cra_aead.ivsize = 8;
 	inst->alg.cra_aead.maxauthsize = 16;
 	inst->alg.cra_ctxsize = sizeof(struct crypto_rfc4543_ctx);
 	inst->alg.cra_init = crypto_rfc4543_init_tfm;
 	inst->alg.cra_exit = crypto_rfc4543_exit_tfm;
 	inst->alg.cra_aead.setkey = crypto_rfc4543_setkey;
 	inst->alg.cra_aead.setauthsize = crypto_rfc4543_setauthsize;
 	inst->alg.cra_aead.encrypt = crypto_rfc4543_encrypt;
 	inst->alg.cra_aead.decrypt = crypto_rfc4543_decrypt;
 	inst->alg.cra_aead.geniv = "seqiv";
 out:
 	return inst;
 out_drop_ecbnull:
 	crypto_drop_skcipher(&ctx->null);
 out_drop_alg:
 	crypto_drop_aead(spawn);
 out_free_inst:
 	kfree(inst);
 	inst = ERR_PTR(err);
 	goto out;
 }
 static void crypto_rfc4543_free(struct crypto_instance *inst)
 {
 	struct crypto_rfc4543_instance_ctx *ctx = crypto_instance_ctx(inst);
 	crypto_drop_aead(&ctx->aead);
 	crypto_drop_skcipher(&ctx->null);
 	kfree(inst);
 }
 static struct crypto_template crypto_rfc4543_tmpl = {
 	.name = "rfc4543",
 	.alloc = crypto_rfc4543_alloc,
 	.free = crypto_rfc4543_free,
 	.module = THIS_MODULE,
 };
 static int __init crypto_gcm_module_init(void)
 {
 	int err;
 	gcm_zeroes = kzalloc(16, GFP_KERNEL);
 	if (!gcm_zeroes)
 		return -ENOMEM;
 	err = crypto_register_template(&crypto_gcm_base_tmpl);
 	if (err)
 		goto out;
 	err = crypto_register_template(&crypto_gcm_tmpl);
 	if (err)
 		goto out_undo_base;
 	err = crypto_register_template(&crypto_rfc4106_tmpl);
 	if (err)
 		goto out_undo_gcm;
 	err = crypto_register_template(&crypto_rfc4543_tmpl);
 	if (err)
 		goto out_undo_rfc4106;
 	return 0;
 out_undo_rfc4106:
 	crypto_unregister_template(&crypto_rfc4106_tmpl);
 out_undo_gcm:
 	crypto_unregister_template(&crypto_gcm_tmpl);
 out_undo_base:
 	crypto_unregister_template(&crypto_gcm_base_tmpl);
 out:
 	kfree(gcm_zeroes);
 	return err;
 }
 static void __exit crypto_gcm_module_exit(void)
 {
 	kfree(gcm_zeroes);
 	crypto_unregister_template(&crypto_rfc4543_tmpl);
 	crypto_unregister_template(&crypto_rfc4106_tmpl);
 	crypto_unregister_template(&crypto_gcm_tmpl);
 	crypto_unregister_template(&crypto_gcm_base_tmpl);
 }
 module_init(crypto_gcm_module_init);
 module_exit(crypto_gcm_module_exit);
 MODULE_LICENSE("GPL");
 MODULE_DESCRIPTION("Galois/Counter Mode");
 MODULE_AUTHOR("Mikko Herranen <mh1@iki.fi>");
 MODULE_ALIAS_CRYPTO("gcm_base");
 MODULE_ALIAS_CRYPTO("rfc4106");
 MODULE_ALIAS_CRYPTO("rfc4543");
+MODULE_ALIAS_CRYPTO("gcm");

crypto/hmac.c

Diff comments View file @ acc5ccb

 /*
  * Cryptographic API.
  *
  * HMAC: Keyed-Hashing for Message Authentication (RFC2104).
  *
  * Copyright (c) 2002 James Morris <jmorris@intercode.com.au>
  * Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au>
  *
  * The HMAC implementation is derived from USAGI.
  * Copyright (c) 2002 Kazunori Miyazawa <miyazawa@linux-ipv6.org> / USAGI
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the Free
  * Software Foundation; either version 2 of the License, or (at your option)
  * any later version.
  *
  */
 #include <crypto/internal/hash.h>
 #include <crypto/scatterwalk.h>
 #include <linux/err.h>
 #include <linux/init.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/scatterlist.h>
 #include <linux/string.h>
 struct hmac_ctx {
 	struct crypto_shash *hash;
 };
 static inline void *align_ptr(void *p, unsigned int align)
 {
 	return (void *)ALIGN((unsigned long)p, align);
 }
 static inline struct hmac_ctx *hmac_ctx(struct crypto_shash *tfm)
 {
 	return align_ptr(crypto_shash_ctx_aligned(tfm) +
 			 crypto_shash_statesize(tfm) * 2,
 			 crypto_tfm_ctx_alignment());
 }
 static int hmac_setkey(struct crypto_shash *parent,
 		       const u8 *inkey, unsigned int keylen)
 {
 	int bs = crypto_shash_blocksize(parent);
 	int ds = crypto_shash_digestsize(parent);
 	int ss = crypto_shash_statesize(parent);
 	char *ipad = crypto_shash_ctx_aligned(parent);
 	char *opad = ipad + ss;
 	struct hmac_ctx *ctx = align_ptr(opad + ss,
 					 crypto_tfm_ctx_alignment());
 	struct crypto_shash *hash = ctx->hash;
 	SHASH_DESC_ON_STACK(shash, hash);
 	unsigned int i;
 	shash->tfm = hash;
 	shash->flags = crypto_shash_get_flags(parent)
 		& CRYPTO_TFM_REQ_MAY_SLEEP;
 	if (keylen > bs) {
 		int err;
 		err = crypto_shash_digest(shash, inkey, keylen, ipad);
 		if (err)
 			return err;
 		keylen = ds;
 	} else
 		memcpy(ipad, inkey, keylen);
 	memset(ipad + keylen, 0, bs - keylen);
 	memcpy(opad, ipad, bs);
 	for (i = 0; i < bs; i++) {
 		ipad[i] ^= 0x36;
 		opad[i] ^= 0x5c;
 	}
 	return crypto_shash_init(shash) ?:
 	       crypto_shash_update(shash, ipad, bs) ?:
 	       crypto_shash_export(shash, ipad) ?:
 	       crypto_shash_init(shash) ?:
 	       crypto_shash_update(shash, opad, bs) ?:
 	       crypto_shash_export(shash, opad);
 }
 static int hmac_export(struct shash_desc *pdesc, void *out)
 {
 	struct shash_desc *desc = shash_desc_ctx(pdesc);
 	desc->flags = pdesc->flags & CRYPTO_TFM_REQ_MAY_SLEEP;
 	return crypto_shash_export(desc, out);
 }
 static int hmac_import(struct shash_desc *pdesc, const void *in)
 {
 	struct shash_desc *desc = shash_desc_ctx(pdesc);
 	struct hmac_ctx *ctx = hmac_ctx(pdesc->tfm);
 	desc->tfm = ctx->hash;
 	desc->flags = pdesc->flags & CRYPTO_TFM_REQ_MAY_SLEEP;
 	return crypto_shash_import(desc, in);
 }
 static int hmac_init(struct shash_desc *pdesc)
 {
 	return hmac_import(pdesc, crypto_shash_ctx_aligned(pdesc->tfm));
 }
 static int hmac_update(struct shash_desc *pdesc,
 		       const u8 *data, unsigned int nbytes)
 {
 	struct shash_desc *desc = shash_desc_ctx(pdesc);
 	desc->flags = pdesc->flags & CRYPTO_TFM_REQ_MAY_SLEEP;
 	return crypto_shash_update(desc, data, nbytes);
 }
 static int hmac_final(struct shash_desc *pdesc, u8 *out)
 {
 	struct crypto_shash *parent = pdesc->tfm;
 	int ds = crypto_shash_digestsize(parent);
 	int ss = crypto_shash_statesize(parent);
 	char *opad = crypto_shash_ctx_aligned(parent) + ss;
 	struct shash_desc *desc = shash_desc_ctx(pdesc);
 	desc->flags = pdesc->flags & CRYPTO_TFM_REQ_MAY_SLEEP;
 	return crypto_shash_final(desc, out) ?:
 	       crypto_shash_import(desc, opad) ?:
 	       crypto_shash_finup(desc, out, ds, out);
 }
 static int hmac_finup(struct shash_desc *pdesc, const u8 *data,
 		      unsigned int nbytes, u8 *out)
 {
 	struct crypto_shash *parent = pdesc->tfm;
 	int ds = crypto_shash_digestsize(parent);
 	int ss = crypto_shash_statesize(parent);
 	char *opad = crypto_shash_ctx_aligned(parent) + ss;
 	struct shash_desc *desc = shash_desc_ctx(pdesc);
 	desc->flags = pdesc->flags & CRYPTO_TFM_REQ_MAY_SLEEP;
 	return crypto_shash_finup(desc, data, nbytes, out) ?:
 	       crypto_shash_import(desc, opad) ?:
 	       crypto_shash_finup(desc, out, ds, out);
 }
 static int hmac_init_tfm(struct crypto_tfm *tfm)
 {
 	struct crypto_shash *parent = __crypto_shash_cast(tfm);
 	struct crypto_shash *hash;
 	struct crypto_instance *inst = (void *)tfm->__crt_alg;
 	struct crypto_shash_spawn *spawn = crypto_instance_ctx(inst);
 	struct hmac_ctx *ctx = hmac_ctx(parent);
 	hash = crypto_spawn_shash(spawn);
 	if (IS_ERR(hash))
 		return PTR_ERR(hash);
 	parent->descsize = sizeof(struct shash_desc) +
 			   crypto_shash_descsize(hash);
 	ctx->hash = hash;
 	return 0;
 }
 static void hmac_exit_tfm(struct crypto_tfm *tfm)
 {
 	struct hmac_ctx *ctx = hmac_ctx(__crypto_shash_cast(tfm));
 	crypto_free_shash(ctx->hash);
 }
 static int hmac_create(struct crypto_template *tmpl, struct rtattr **tb)
 {
 	struct shash_instance *inst;
 	struct crypto_alg *alg;
 	struct shash_alg *salg;
 	int err;
 	int ds;
 	int ss;
 	err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_SHASH);
 	if (err)
 		return err;
 	salg = shash_attr_alg(tb[1], 0, 0);
 	if (IS_ERR(salg))
 		return PTR_ERR(salg);
 	err = -EINVAL;
 	ds = salg->digestsize;
 	ss = salg->statesize;
 	alg = &salg->base;
 	if (ds > alg->cra_blocksize ||
 	    ss < alg->cra_blocksize)
 		goto out_put_alg;
 	inst = shash_alloc_instance("hmac", alg);
 	err = PTR_ERR(inst);
 	if (IS_ERR(inst))
 		goto out_put_alg;
 	err = crypto_init_shash_spawn(shash_instance_ctx(inst), salg,
 				      shash_crypto_instance(inst));
 	if (err)
 		goto out_free_inst;
 	inst->alg.base.cra_priority = alg->cra_priority;
 	inst->alg.base.cra_blocksize = alg->cra_blocksize;
 	inst->alg.base.cra_alignmask = alg->cra_alignmask;
 	ss = ALIGN(ss, alg->cra_alignmask + 1);
 	inst->alg.digestsize = ds;
 	inst->alg.statesize = ss;
 	inst->alg.base.cra_ctxsize = sizeof(struct hmac_ctx) +
 				     ALIGN(ss * 2, crypto_tfm_ctx_alignment());
 	inst->alg.base.cra_init = hmac_init_tfm;
 	inst->alg.base.cra_exit = hmac_exit_tfm;
 	inst->alg.init = hmac_init;
 	inst->alg.update = hmac_update;
 	inst->alg.final = hmac_final;
 	inst->alg.finup = hmac_finup;
 	inst->alg.export = hmac_export;
 	inst->alg.import = hmac_import;
 	inst->alg.setkey = hmac_setkey;
 	err = shash_register_instance(tmpl, inst);
 	if (err) {
 out_free_inst:
 		shash_free_instance(shash_crypto_instance(inst));
 	}
 out_put_alg:
 	crypto_mod_put(alg);
 	return err;
 }
 static struct crypto_template hmac_tmpl = {
 	.name = "hmac",
 	.create = hmac_create,
 	.free = shash_free_instance,
 	.module = THIS_MODULE,
 };
 static int __init hmac_module_init(void)
 {
 	return crypto_register_template(&hmac_tmpl);
 }
 static void __exit hmac_module_exit(void)
 {
 	crypto_unregister_template(&hmac_tmpl);
 }
 module_init(hmac_module_init);
 module_exit(hmac_module_exit);
 MODULE_LICENSE("GPL");
 MODULE_DESCRIPTION("HMAC hash algorithm");
+MODULE_ALIAS_CRYPTO("hmac");

crypto/lrw.c

Diff comments View file @ acc5ccb

 /* LRW: as defined by Cyril Guyot in
  *	http://grouper.ieee.org/groups/1619/email/pdf00017.pdf
  *
  * Copyright (c) 2006 Rik Snel <rsnel@cube.dyndns.org>
  *
  * Based on ecb.c
  * Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au>
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the Free
  * Software Foundation; either version 2 of the License, or (at your option)
  * any later version.
  */
 /* This implementation is checked against the test vectors in the above
  * document and by a test vector provided by Ken Buchanan at
  * http://www.mail-archive.com/stds-p1619@listserv.ieee.org/msg00173.html
  *
  * The test vectors are included in the testing module tcrypt.[ch] */
 #include <crypto/algapi.h>
 #include <linux/err.h>
 #include <linux/init.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/scatterlist.h>
 #include <linux/slab.h>
 #include <crypto/b128ops.h>
 #include <crypto/gf128mul.h>
 #include <crypto/lrw.h>
 struct priv {
 	struct crypto_cipher *child;
 	struct lrw_table_ctx table;
 };
 static inline void setbit128_bbe(void *b, int bit)
 {
 	__set_bit(bit ^ (0x80 -
 #ifdef __BIG_ENDIAN
 			 BITS_PER_LONG
 #else
 			 BITS_PER_BYTE
 #endif
 			), b);
 }
 int lrw_init_table(struct lrw_table_ctx *ctx, const u8 *tweak)
 {
 	be128 tmp = { 0 };
 	int i;
 	if (ctx->table)
 		gf128mul_free_64k(ctx->table);
 	/* initialize multiplication table for Key2 */
 	ctx->table = gf128mul_init_64k_bbe((be128 *)tweak);
 	if (!ctx->table)
 		return -ENOMEM;
 	/* initialize optimization table */
 	for (i = 0; i < 128; i++) {
 		setbit128_bbe(&tmp, i);
 		ctx->mulinc[i] = tmp;
 		gf128mul_64k_bbe(&ctx->mulinc[i], ctx->table);
 	}
 	return 0;
 }
 EXPORT_SYMBOL_GPL(lrw_init_table);
 void lrw_free_table(struct lrw_table_ctx *ctx)
 {
 	if (ctx->table)
 		gf128mul_free_64k(ctx->table);
 }
 EXPORT_SYMBOL_GPL(lrw_free_table);
 static int setkey(struct crypto_tfm *parent, const u8 *key,
 		  unsigned int keylen)
 {
 	struct priv *ctx = crypto_tfm_ctx(parent);
 	struct crypto_cipher *child = ctx->child;
 	int err, bsize = LRW_BLOCK_SIZE;
 	const u8 *tweak = key + keylen - bsize;
 	crypto_cipher_clear_flags(child, CRYPTO_TFM_REQ_MASK);
 	crypto_cipher_set_flags(child, crypto_tfm_get_flags(parent) &
 				       CRYPTO_TFM_REQ_MASK);
 	err = crypto_cipher_setkey(child, key, keylen - bsize);
 	if (err)
 		return err;
 	crypto_tfm_set_flags(parent, crypto_cipher_get_flags(child) &
 				     CRYPTO_TFM_RES_MASK);
 	return lrw_init_table(&ctx->table, tweak);
 }
 struct sinfo {
 	be128 t;
 	struct crypto_tfm *tfm;
 	void (*fn)(struct crypto_tfm *, u8 *, const u8 *);
 };
 static inline void inc(be128 *iv)
 {
 	be64_add_cpu(&iv->b, 1);
 	if (!iv->b)
 		be64_add_cpu(&iv->a, 1);
 }
 static inline void lrw_round(struct sinfo *s, void *dst, const void *src)
 {
 	be128_xor(dst, &s->t, src);		/* PP <- T xor P */
 	s->fn(s->tfm, dst, dst);		/* CC <- E(Key2,PP) */
 	be128_xor(dst, dst, &s->t);		/* C <- T xor CC */
 }
 /* this returns the number of consequative 1 bits starting
  * from the right, get_index128(00 00 00 00 00 00 ... 00 00 10 FB) = 2 */
 static inline int get_index128(be128 *block)
 {
 	int x;
 	__be32 *p = (__be32 *) block;
 	for (p += 3, x = 0; x < 128; p--, x += 32) {
 		u32 val = be32_to_cpup(p);
 		if (!~val)
 			continue;
 		return x + ffz(val);
 	}
 	return x;
 }
 static int crypt(struct blkcipher_desc *d,
 		 struct blkcipher_walk *w, struct priv *ctx,
 		 void (*fn)(struct crypto_tfm *, u8 *, const u8 *))
 {
 	int err;
 	unsigned int avail;
 	const int bs = LRW_BLOCK_SIZE;
 	struct sinfo s = {
 		.tfm = crypto_cipher_tfm(ctx->child),
 		.fn = fn
 	};
 	be128 *iv;
 	u8 *wsrc;
 	u8 *wdst;
 	err = blkcipher_walk_virt(d, w);
 	if (!(avail = w->nbytes))
 		return err;
 	wsrc = w->src.virt.addr;
 	wdst = w->dst.virt.addr;
 	/* calculate first value of T */
 	iv = (be128 *)w->iv;
 	s.t = *iv;
 	/* T <- I*Key2 */
 	gf128mul_64k_bbe(&s.t, ctx->table.table);
 	goto first;
 	for (;;) {
 		do {
 			/* T <- I*Key2, using the optimization
 			 * discussed in the specification */
 			be128_xor(&s.t, &s.t,
 				  &ctx->table.mulinc[get_index128(iv)]);
 			inc(iv);
 first:
 			lrw_round(&s, wdst, wsrc);
 			wsrc += bs;
 			wdst += bs;
 		} while ((avail -= bs) >= bs);
 		err = blkcipher_walk_done(d, w, avail);
 		if (!(avail = w->nbytes))
 			break;
 		wsrc = w->src.virt.addr;
 		wdst = w->dst.virt.addr;
 	}
 	return err;
 }
 static int encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 		   struct scatterlist *src, unsigned int nbytes)
 {
 	struct priv *ctx = crypto_blkcipher_ctx(desc->tfm);
 	struct blkcipher_walk w;
 	blkcipher_walk_init(&w, dst, src, nbytes);
 	return crypt(desc, &w, ctx,
 		     crypto_cipher_alg(ctx->child)->cia_encrypt);
 }
 static int decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 		   struct scatterlist *src, unsigned int nbytes)
 {
 	struct priv *ctx = crypto_blkcipher_ctx(desc->tfm);
 	struct blkcipher_walk w;
 	blkcipher_walk_init(&w, dst, src, nbytes);
 	return crypt(desc, &w, ctx,
 		     crypto_cipher_alg(ctx->child)->cia_decrypt);
 }
 int lrw_crypt(struct blkcipher_desc *desc, struct scatterlist *sdst,
 	      struct scatterlist *ssrc, unsigned int nbytes,
 	      struct lrw_crypt_req *req)
 {
 	const unsigned int bsize = LRW_BLOCK_SIZE;
 	const unsigned int max_blks = req->tbuflen / bsize;
 	struct lrw_table_ctx *ctx = req->table_ctx;
 	struct blkcipher_walk walk;
 	unsigned int nblocks;
 	be128 *iv, *src, *dst, *t;
 	be128 *t_buf = req->tbuf;
 	int err, i;
 	BUG_ON(max_blks < 1);
 	blkcipher_walk_init(&walk, sdst, ssrc, nbytes);
 	err = blkcipher_walk_virt(desc, &walk);
 	nbytes = walk.nbytes;
 	if (!nbytes)
 		return err;
 	nblocks = min(walk.nbytes / bsize, max_blks);
 	src = (be128 *)walk.src.virt.addr;
 	dst = (be128 *)walk.dst.virt.addr;
 	/* calculate first value of T */
 	iv = (be128 *)walk.iv;
 	t_buf[0] = *iv;
 	/* T <- I*Key2 */
 	gf128mul_64k_bbe(&t_buf[0], ctx->table);
 	i = 0;
 	goto first;
 	for (;;) {
 		do {
 			for (i = 0; i < nblocks; i++) {
 				/* T <- I*Key2, using the optimization
 				 * discussed in the specification */
 				be128_xor(&t_buf[i], t,
 						&ctx->mulinc[get_index128(iv)]);
 				inc(iv);
 first:
 				t = &t_buf[i];
 				/* PP <- T xor P */
 				be128_xor(dst + i, t, src + i);
 			}
 			/* CC <- E(Key2,PP) */
 			req->crypt_fn(req->crypt_ctx, (u8 *)dst,
 				      nblocks * bsize);
 			/* C <- T xor CC */
 			for (i = 0; i < nblocks; i++)
 				be128_xor(dst + i, dst + i, &t_buf[i]);
 			src += nblocks;
 			dst += nblocks;
 			nbytes -= nblocks * bsize;
 			nblocks = min(nbytes / bsize, max_blks);
 		} while (nblocks > 0);
 		err = blkcipher_walk_done(desc, &walk, nbytes);
 		nbytes = walk.nbytes;
 		if (!nbytes)
 			break;
 		nblocks = min(nbytes / bsize, max_blks);
 		src = (be128 *)walk.src.virt.addr;
 		dst = (be128 *)walk.dst.virt.addr;
 	}
 	return err;
 }
 EXPORT_SYMBOL_GPL(lrw_crypt);
 static int init_tfm(struct crypto_tfm *tfm)
 {
 	struct crypto_cipher *cipher;
 	struct crypto_instance *inst = (void *)tfm->__crt_alg;
 	struct crypto_spawn *spawn = crypto_instance_ctx(inst);
 	struct priv *ctx = crypto_tfm_ctx(tfm);
 	u32 *flags = &tfm->crt_flags;
 	cipher = crypto_spawn_cipher(spawn);
 	if (IS_ERR(cipher))
 		return PTR_ERR(cipher);
 	if (crypto_cipher_blocksize(cipher) != LRW_BLOCK_SIZE) {
 		*flags |= CRYPTO_TFM_RES_BAD_BLOCK_LEN;
 		crypto_free_cipher(cipher);
 		return -EINVAL;
 	}
 	ctx->child = cipher;
 	return 0;
 }
 static void exit_tfm(struct crypto_tfm *tfm)
 {
 	struct priv *ctx = crypto_tfm_ctx(tfm);
 	lrw_free_table(&ctx->table);
 	crypto_free_cipher(ctx->child);
 }
 static struct crypto_instance *alloc(struct rtattr **tb)
 {
 	struct crypto_instance *inst;
 	struct crypto_alg *alg;
 	int err;
 	err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_BLKCIPHER);
 	if (err)
 		return ERR_PTR(err);
 	alg = crypto_get_attr_alg(tb, CRYPTO_ALG_TYPE_CIPHER,
 				  CRYPTO_ALG_TYPE_MASK);
 	if (IS_ERR(alg))
 		return ERR_CAST(alg);
 	inst = crypto_alloc_instance("lrw", alg);
 	if (IS_ERR(inst))
 		goto out_put_alg;
 	inst->alg.cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER;
 	inst->alg.cra_priority = alg->cra_priority;
 	inst->alg.cra_blocksize = alg->cra_blocksize;
 	if (alg->cra_alignmask < 7) inst->alg.cra_alignmask = 7;
 	else inst->alg.cra_alignmask = alg->cra_alignmask;
 	inst->alg.cra_type = &crypto_blkcipher_type;
 	if (!(alg->cra_blocksize % 4))
 		inst->alg.cra_alignmask |= 3;
 	inst->alg.cra_blkcipher.ivsize = alg->cra_blocksize;
 	inst->alg.cra_blkcipher.min_keysize =
 		alg->cra_cipher.cia_min_keysize + alg->cra_blocksize;
 	inst->alg.cra_blkcipher.max_keysize =
 		alg->cra_cipher.cia_max_keysize + alg->cra_blocksize;
 	inst->alg.cra_ctxsize = sizeof(struct priv);
 	inst->alg.cra_init = init_tfm;
 	inst->alg.cra_exit = exit_tfm;
 	inst->alg.cra_blkcipher.setkey = setkey;
 	inst->alg.cra_blkcipher.encrypt = encrypt;
 	inst->alg.cra_blkcipher.decrypt = decrypt;
 out_put_alg:
 	crypto_mod_put(alg);
 	return inst;
 }
 static void free(struct crypto_instance *inst)
 {
 	crypto_drop_spawn(crypto_instance_ctx(inst));
 	kfree(inst);
 }
 static struct crypto_template crypto_tmpl = {
 	.name = "lrw",
 	.alloc = alloc,
 	.free = free,
 	.module = THIS_MODULE,
 };
 static int __init crypto_module_init(void)
 {
 	return crypto_register_template(&crypto_tmpl);
 }
 static void __exit crypto_module_exit(void)
 {
 	crypto_unregister_template(&crypto_tmpl);
 }
 module_init(crypto_module_init);
 module_exit(crypto_module_exit);
 MODULE_LICENSE("GPL");
 MODULE_DESCRIPTION("LRW block cipher mode");
+MODULE_ALIAS_CRYPTO("lrw");

crypto/mcryptd.c

Diff comments View file @ acc5ccb

 /*
  * Software multibuffer async crypto daemon.
  *
  * Copyright (c) 2014 Tim Chen <tim.c.chen@linux.intel.com>
  *
  * Adapted from crypto daemon.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the Free
  * Software Foundation; either version 2 of the License, or (at your option)
  * any later version.
  *
  */
 #include <crypto/algapi.h>
 #include <crypto/internal/hash.h>
 #include <crypto/internal/aead.h>
 #include <crypto/mcryptd.h>
 #include <crypto/crypto_wq.h>
 #include <linux/err.h>
 #include <linux/init.h>
 #include <linux/kernel.h>
 #include <linux/list.h>
 #include <linux/module.h>
 #include <linux/scatterlist.h>
 #include <linux/sched.h>
 #include <linux/slab.h>
 #include <linux/hardirq.h>
 #define MCRYPTD_MAX_CPU_QLEN 100
 #define MCRYPTD_BATCH 9
 static void *mcryptd_alloc_instance(struct crypto_alg *alg, unsigned int head,
 				   unsigned int tail);
 struct mcryptd_flush_list {
 	struct list_head list;
 	struct mutex lock;
 };
 static struct mcryptd_flush_list __percpu *mcryptd_flist;
 struct hashd_instance_ctx {
 	struct crypto_shash_spawn spawn;
 	struct mcryptd_queue *queue;
 };
 static void mcryptd_queue_worker(struct work_struct *work);
 void mcryptd_arm_flusher(struct mcryptd_alg_cstate *cstate, unsigned long delay)
 {
 	struct mcryptd_flush_list *flist;
 	if (!cstate->flusher_engaged) {
 		/* put the flusher on the flush list */
 		flist = per_cpu_ptr(mcryptd_flist, smp_processor_id());
 		mutex_lock(&flist->lock);
 		list_add_tail(&cstate->flush_list, &flist->list);
 		cstate->flusher_engaged = true;
 		cstate->next_flush = jiffies + delay;
 		queue_delayed_work_on(smp_processor_id(), kcrypto_wq,
 			&cstate->flush, delay);
 		mutex_unlock(&flist->lock);
 	}
 }
 EXPORT_SYMBOL(mcryptd_arm_flusher);
 static int mcryptd_init_queue(struct mcryptd_queue *queue,
 			     unsigned int max_cpu_qlen)
 {
 	int cpu;
 	struct mcryptd_cpu_queue *cpu_queue;
 	queue->cpu_queue = alloc_percpu(struct mcryptd_cpu_queue);
 	pr_debug("mqueue:%p mcryptd_cpu_queue %p\n", queue, queue->cpu_queue);
 	if (!queue->cpu_queue)
 		return -ENOMEM;
 	for_each_possible_cpu(cpu) {
 		cpu_queue = per_cpu_ptr(queue->cpu_queue, cpu);
 		pr_debug("cpu_queue #%d %p\n", cpu, queue->cpu_queue);
 		crypto_init_queue(&cpu_queue->queue, max_cpu_qlen);
 		INIT_WORK(&cpu_queue->work, mcryptd_queue_worker);
 	}
 	return 0;
 }
 static void mcryptd_fini_queue(struct mcryptd_queue *queue)
 {
 	int cpu;
 	struct mcryptd_cpu_queue *cpu_queue;
 	for_each_possible_cpu(cpu) {
 		cpu_queue = per_cpu_ptr(queue->cpu_queue, cpu);
 		BUG_ON(cpu_queue->queue.qlen);
 	}
 	free_percpu(queue->cpu_queue);
 }
 static int mcryptd_enqueue_request(struct mcryptd_queue *queue,
 				  struct crypto_async_request *request,
 				  struct mcryptd_hash_request_ctx *rctx)
 {
 	int cpu, err;
 	struct mcryptd_cpu_queue *cpu_queue;
 	cpu = get_cpu();
 	cpu_queue = this_cpu_ptr(queue->cpu_queue);
 	rctx->tag.cpu = cpu;
 	err = crypto_enqueue_request(&cpu_queue->queue, request);
 	pr_debug("enqueue request: cpu %d cpu_queue %p request %p\n",
 		 cpu, cpu_queue, request);
 	queue_work_on(cpu, kcrypto_wq, &cpu_queue->work);
 	put_cpu();
 	return err;
 }
 /*
  * Try to opportunisticlly flush the partially completed jobs if
  * crypto daemon is the only task running.
  */
 static void mcryptd_opportunistic_flush(void)
 {
 	struct mcryptd_flush_list *flist;
 	struct mcryptd_alg_cstate *cstate;
 	flist = per_cpu_ptr(mcryptd_flist, smp_processor_id());
 	while (single_task_running()) {
 		mutex_lock(&flist->lock);
 		if (list_empty(&flist->list)) {
 			mutex_unlock(&flist->lock);
 			return;
 		}
 		cstate = list_entry(flist->list.next,
 				struct mcryptd_alg_cstate, flush_list);
 		if (!cstate->flusher_engaged) {
 			mutex_unlock(&flist->lock);
 			return;
 		}
 		list_del(&cstate->flush_list);
 		cstate->flusher_engaged = false;
 		mutex_unlock(&flist->lock);
 		cstate->alg_state->flusher(cstate);
 	}
 }
 /*
  * Called in workqueue context, do one real cryption work (via
  * req->complete) and reschedule itself if there are more work to
  * do.
  */
 static void mcryptd_queue_worker(struct work_struct *work)
 {
 	struct mcryptd_cpu_queue *cpu_queue;
 	struct crypto_async_request *req, *backlog;
 	int i;
 	/*
 	 * Need to loop through more than once for multi-buffer to
 	 * be effective.
 	 */
 	cpu_queue = container_of(work, struct mcryptd_cpu_queue, work);
 	i = 0;
 	while (i < MCRYPTD_BATCH || single_task_running()) {
 		/*
 		 * preempt_disable/enable is used to prevent
 		 * being preempted by mcryptd_enqueue_request()
 		 */
 		local_bh_disable();
 		preempt_disable();
 		backlog = crypto_get_backlog(&cpu_queue->queue);
 		req = crypto_dequeue_request(&cpu_queue->queue);
 		preempt_enable();
 		local_bh_enable();
 		if (!req) {
 			mcryptd_opportunistic_flush();
 			return;
 		}
 		if (backlog)
 			backlog->complete(backlog, -EINPROGRESS);
 		req->complete(req, 0);
 		if (!cpu_queue->queue.qlen)
 			return;
 		++i;
 	}
 	if (cpu_queue->queue.qlen)
 		queue_work(kcrypto_wq, &cpu_queue->work);
 }
 void mcryptd_flusher(struct work_struct *__work)
 {
 	struct	mcryptd_alg_cstate	*alg_cpu_state;
 	struct	mcryptd_alg_state	*alg_state;
 	struct	mcryptd_flush_list	*flist;
 	int	cpu;
 	cpu = smp_processor_id();
 	alg_cpu_state = container_of(to_delayed_work(__work),
 				     struct mcryptd_alg_cstate, flush);
 	alg_state = alg_cpu_state->alg_state;
 	if (alg_cpu_state->cpu != cpu)
 		pr_debug("mcryptd error: work on cpu %d, should be cpu %d\n",
 				cpu, alg_cpu_state->cpu);
 	if (alg_cpu_state->flusher_engaged) {
 		flist = per_cpu_ptr(mcryptd_flist, cpu);
 		mutex_lock(&flist->lock);
 		list_del(&alg_cpu_state->flush_list);
 		alg_cpu_state->flusher_engaged = false;
 		mutex_unlock(&flist->lock);
 		alg_state->flusher(alg_cpu_state);
 	}
 }
 EXPORT_SYMBOL_GPL(mcryptd_flusher);
 static inline struct mcryptd_queue *mcryptd_get_queue(struct crypto_tfm *tfm)
 {
 	struct crypto_instance *inst = crypto_tfm_alg_instance(tfm);
 	struct mcryptd_instance_ctx *ictx = crypto_instance_ctx(inst);
 	return ictx->queue;
 }
 static void *mcryptd_alloc_instance(struct crypto_alg *alg, unsigned int head,
 				   unsigned int tail)
 {
 	char *p;
 	struct crypto_instance *inst;
 	int err;
 	p = kzalloc(head + sizeof(*inst) + tail, GFP_KERNEL);
 	if (!p)
 		return ERR_PTR(-ENOMEM);
 	inst = (void *)(p + head);
 	err = -ENAMETOOLONG;
 	if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME,
 		    "mcryptd(%s)", alg->cra_driver_name) >= CRYPTO_MAX_ALG_NAME)
 		goto out_free_inst;
 	memcpy(inst->alg.cra_name, alg->cra_name, CRYPTO_MAX_ALG_NAME);
 	inst->alg.cra_priority = alg->cra_priority + 50;
 	inst->alg.cra_blocksize = alg->cra_blocksize;
 	inst->alg.cra_alignmask = alg->cra_alignmask;
 out:
 	return p;
 out_free_inst:
 	kfree(p);
 	p = ERR_PTR(err);
 	goto out;
 }
 static int mcryptd_hash_init_tfm(struct crypto_tfm *tfm)
 {
 	struct crypto_instance *inst = crypto_tfm_alg_instance(tfm);
 	struct hashd_instance_ctx *ictx = crypto_instance_ctx(inst);
 	struct crypto_shash_spawn *spawn = &ictx->spawn;
 	struct mcryptd_hash_ctx *ctx = crypto_tfm_ctx(tfm);
 	struct crypto_shash *hash;
 	hash = crypto_spawn_shash(spawn);
 	if (IS_ERR(hash))
 		return PTR_ERR(hash);
 	ctx->child = hash;
 	crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
 				 sizeof(struct mcryptd_hash_request_ctx) +
 				 crypto_shash_descsize(hash));
 	return 0;
 }
 static void mcryptd_hash_exit_tfm(struct crypto_tfm *tfm)
 {
 	struct mcryptd_hash_ctx *ctx = crypto_tfm_ctx(tfm);
 	crypto_free_shash(ctx->child);
 }
 static int mcryptd_hash_setkey(struct crypto_ahash *parent,
 				   const u8 *key, unsigned int keylen)
 {
 	struct mcryptd_hash_ctx *ctx   = crypto_ahash_ctx(parent);
 	struct crypto_shash *child = ctx->child;
 	int err;
 	crypto_shash_clear_flags(child, CRYPTO_TFM_REQ_MASK);
 	crypto_shash_set_flags(child, crypto_ahash_get_flags(parent) &
 				      CRYPTO_TFM_REQ_MASK);
 	err = crypto_shash_setkey(child, key, keylen);
 	crypto_ahash_set_flags(parent, crypto_shash_get_flags(child) &
 				       CRYPTO_TFM_RES_MASK);
 	return err;
 }
 static int mcryptd_hash_enqueue(struct ahash_request *req,
 				crypto_completion_t complete)
 {
 	int ret;
 	struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 	struct mcryptd_queue *queue =
 		mcryptd_get_queue(crypto_ahash_tfm(tfm));
 	rctx->complete = req->base.complete;
 	req->base.complete = complete;
 	ret = mcryptd_enqueue_request(queue, &req->base, rctx);
 	return ret;
 }
 static void mcryptd_hash_init(struct crypto_async_request *req_async, int err)
 {
 	struct mcryptd_hash_ctx *ctx = crypto_tfm_ctx(req_async->tfm);
 	struct crypto_shash *child = ctx->child;
 	struct ahash_request *req = ahash_request_cast(req_async);
 	struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
 	struct shash_desc *desc = &rctx->desc;
 	if (unlikely(err == -EINPROGRESS))
 		goto out;
 	desc->tfm = child;
 	desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP;
 	err = crypto_shash_init(desc);
 	req->base.complete = rctx->complete;
 out:
 	local_bh_disable();
 	rctx->complete(&req->base, err);
 	local_bh_enable();
 }
 static int mcryptd_hash_init_enqueue(struct ahash_request *req)
 {
 	return mcryptd_hash_enqueue(req, mcryptd_hash_init);
 }
 static void mcryptd_hash_update(struct crypto_async_request *req_async, int err)
 {
 	struct ahash_request *req = ahash_request_cast(req_async);
 	struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
 	if (unlikely(err == -EINPROGRESS))
 		goto out;
 	err = shash_ahash_mcryptd_update(req, &rctx->desc);
 	if (err) {
 		req->base.complete = rctx->complete;
 		goto out;
 	}
 	return;
 out:
 	local_bh_disable();
 	rctx->complete(&req->base, err);
 	local_bh_enable();
 }
 static int mcryptd_hash_update_enqueue(struct ahash_request *req)
 {
 	return mcryptd_hash_enqueue(req, mcryptd_hash_update);
 }
 static void mcryptd_hash_final(struct crypto_async_request *req_async, int err)
 {
 	struct ahash_request *req = ahash_request_cast(req_async);
 	struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
 	if (unlikely(err == -EINPROGRESS))
 		goto out;
 	err = shash_ahash_mcryptd_final(req, &rctx->desc);
 	if (err) {
 		req->base.complete = rctx->complete;
 		goto out;
 	}
 	return;
 out:
 	local_bh_disable();
 	rctx->complete(&req->base, err);
 	local_bh_enable();
 }
 static int mcryptd_hash_final_enqueue(struct ahash_request *req)
 {
 	return mcryptd_hash_enqueue(req, mcryptd_hash_final);
 }
 static void mcryptd_hash_finup(struct crypto_async_request *req_async, int err)
 {
 	struct ahash_request *req = ahash_request_cast(req_async);
 	struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
 	if (unlikely(err == -EINPROGRESS))
 		goto out;
 	err = shash_ahash_mcryptd_finup(req, &rctx->desc);
 	if (err) {
 		req->base.complete = rctx->complete;
 		goto out;
 	}
 	return;
 out:
 	local_bh_disable();
 	rctx->complete(&req->base, err);
 	local_bh_enable();
 }
 static int mcryptd_hash_finup_enqueue(struct ahash_request *req)
 {
 	return mcryptd_hash_enqueue(req, mcryptd_hash_finup);
 }
 static void mcryptd_hash_digest(struct crypto_async_request *req_async, int err)
 {
 	struct mcryptd_hash_ctx *ctx = crypto_tfm_ctx(req_async->tfm);
 	struct crypto_shash *child = ctx->child;
 	struct ahash_request *req = ahash_request_cast(req_async);
 	struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
 	struct shash_desc *desc = &rctx->desc;
 	if (unlikely(err == -EINPROGRESS))
 		goto out;
 	desc->tfm = child;
 	desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP;  /* check this again */
 	err = shash_ahash_mcryptd_digest(req, desc);
 	if (err) {
 		req->base.complete = rctx->complete;
 		goto out;
 	}
 	return;
 out:
 	local_bh_disable();
 	rctx->complete(&req->base, err);
 	local_bh_enable();
 }
 static int mcryptd_hash_digest_enqueue(struct ahash_request *req)
 {
 	return mcryptd_hash_enqueue(req, mcryptd_hash_digest);
 }
 static int mcryptd_hash_export(struct ahash_request *req, void *out)
 {
 	struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
 	return crypto_shash_export(&rctx->desc, out);
 }
 static int mcryptd_hash_import(struct ahash_request *req, const void *in)
 {
 	struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
 	return crypto_shash_import(&rctx->desc, in);
 }
 static int mcryptd_create_hash(struct crypto_template *tmpl, struct rtattr **tb,
 			      struct mcryptd_queue *queue)
 {
 	struct hashd_instance_ctx *ctx;
 	struct ahash_instance *inst;
 	struct shash_alg *salg;
 	struct crypto_alg *alg;
 	int err;
 	salg = shash_attr_alg(tb[1], 0, 0);
 	if (IS_ERR(salg))
 		return PTR_ERR(salg);
 	alg = &salg->base;
 	pr_debug("crypto: mcryptd hash alg: %s\n", alg->cra_name);
 	inst = mcryptd_alloc_instance(alg, ahash_instance_headroom(),
 					sizeof(*ctx));
 	err = PTR_ERR(inst);
 	if (IS_ERR(inst))
 		goto out_put_alg;
 	ctx = ahash_instance_ctx(inst);
 	ctx->queue = queue;
 	err = crypto_init_shash_spawn(&ctx->spawn, salg,
 				      ahash_crypto_instance(inst));
 	if (err)
 		goto out_free_inst;
 	inst->alg.halg.base.cra_flags = CRYPTO_ALG_ASYNC;
 	inst->alg.halg.digestsize = salg->digestsize;
 	inst->alg.halg.base.cra_ctxsize = sizeof(struct mcryptd_hash_ctx);
 	inst->alg.halg.base.cra_init = mcryptd_hash_init_tfm;
 	inst->alg.halg.base.cra_exit = mcryptd_hash_exit_tfm;
 	inst->alg.init   = mcryptd_hash_init_enqueue;
 	inst->alg.update = mcryptd_hash_update_enqueue;
 	inst->alg.final  = mcryptd_hash_final_enqueue;
 	inst->alg.finup  = mcryptd_hash_finup_enqueue;
 	inst->alg.export = mcryptd_hash_export;
 	inst->alg.import = mcryptd_hash_import;
 	inst->alg.setkey = mcryptd_hash_setkey;
 	inst->alg.digest = mcryptd_hash_digest_enqueue;
 	err = ahash_register_instance(tmpl, inst);
 	if (err) {
 		crypto_drop_shash(&ctx->spawn);
 out_free_inst:
 		kfree(inst);
 	}
 out_put_alg:
 	crypto_mod_put(alg);
 	return err;
 }
 static struct mcryptd_queue mqueue;
 static int mcryptd_create(struct crypto_template *tmpl, struct rtattr **tb)
 {
 	struct crypto_attr_type *algt;
 	algt = crypto_get_attr_type(tb);
 	if (IS_ERR(algt))
 		return PTR_ERR(algt);
 	switch (algt->type & algt->mask & CRYPTO_ALG_TYPE_MASK) {
 	case CRYPTO_ALG_TYPE_DIGEST:
 		return mcryptd_create_hash(tmpl, tb, &mqueue);
 	break;
 	}
 	return -EINVAL;
 }
 static void mcryptd_free(struct crypto_instance *inst)
 {
 	struct mcryptd_instance_ctx *ctx = crypto_instance_ctx(inst);
 	struct hashd_instance_ctx *hctx = crypto_instance_ctx(inst);
 	switch (inst->alg.cra_flags & CRYPTO_ALG_TYPE_MASK) {
 	case CRYPTO_ALG_TYPE_AHASH:
 		crypto_drop_shash(&hctx->spawn);
 		kfree(ahash_instance(inst));
 		return;
 	default:
 		crypto_drop_spawn(&ctx->spawn);
 		kfree(inst);
 	}
 }
 static struct crypto_template mcryptd_tmpl = {
 	.name = "mcryptd",
 	.create = mcryptd_create,
 	.free = mcryptd_free,
 	.module = THIS_MODULE,
 };
 struct mcryptd_ahash *mcryptd_alloc_ahash(const char *alg_name,
 					u32 type, u32 mask)
 {
 	char mcryptd_alg_name[CRYPTO_MAX_ALG_NAME];
 	struct crypto_ahash *tfm;
 	if (snprintf(mcryptd_alg_name, CRYPTO_MAX_ALG_NAME,
 		     "mcryptd(%s)", alg_name) >= CRYPTO_MAX_ALG_NAME)
 		return ERR_PTR(-EINVAL);
 	tfm = crypto_alloc_ahash(mcryptd_alg_name, type, mask);
 	if (IS_ERR(tfm))
 		return ERR_CAST(tfm);
 	if (tfm->base.__crt_alg->cra_module != THIS_MODULE) {
 		crypto_free_ahash(tfm);
 		return ERR_PTR(-EINVAL);
 	}
 	return __mcryptd_ahash_cast(tfm);
 }
 EXPORT_SYMBOL_GPL(mcryptd_alloc_ahash);
 int shash_ahash_mcryptd_digest(struct ahash_request *req,
 			       struct shash_desc *desc)
 {
 	int err;
 	err = crypto_shash_init(desc) ?:
 	      shash_ahash_mcryptd_finup(req, desc);
 	return err;
 }
 EXPORT_SYMBOL_GPL(shash_ahash_mcryptd_digest);
 int shash_ahash_mcryptd_update(struct ahash_request *req,
 			       struct shash_desc *desc)
 {
 	struct crypto_shash *tfm = desc->tfm;
 	struct shash_alg *shash = crypto_shash_alg(tfm);
 	/* alignment is to be done by multi-buffer crypto algorithm if needed */
 	return shash->update(desc, NULL, 0);
 }
 EXPORT_SYMBOL_GPL(shash_ahash_mcryptd_update);
 int shash_ahash_mcryptd_finup(struct ahash_request *req,
 			      struct shash_desc *desc)
 {
 	struct crypto_shash *tfm = desc->tfm;
 	struct shash_alg *shash = crypto_shash_alg(tfm);
 	/* alignment is to be done by multi-buffer crypto algorithm if needed */
 	return shash->finup(desc, NULL, 0, req->result);
 }
 EXPORT_SYMBOL_GPL(shash_ahash_mcryptd_finup);
 int shash_ahash_mcryptd_final(struct ahash_request *req,
 			      struct shash_desc *desc)
 {
 	struct crypto_shash *tfm = desc->tfm;
 	struct shash_alg *shash = crypto_shash_alg(tfm);
 	/* alignment is to be done by multi-buffer crypto algorithm if needed */
 	return shash->final(desc, req->result);
 }
 EXPORT_SYMBOL_GPL(shash_ahash_mcryptd_final);
 struct crypto_shash *mcryptd_ahash_child(struct mcryptd_ahash *tfm)
 {
 	struct mcryptd_hash_ctx *ctx = crypto_ahash_ctx(&tfm->base);
 	return ctx->child;
 }
 EXPORT_SYMBOL_GPL(mcryptd_ahash_child);
 struct shash_desc *mcryptd_shash_desc(struct ahash_request *req)
 {
 	struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
 	return &rctx->desc;
 }
 EXPORT_SYMBOL_GPL(mcryptd_shash_desc);
 void mcryptd_free_ahash(struct mcryptd_ahash *tfm)
 {
 	crypto_free_ahash(&tfm->base);
 }
 EXPORT_SYMBOL_GPL(mcryptd_free_ahash);
 static int __init mcryptd_init(void)
 {
 	int err, cpu;
 	struct mcryptd_flush_list *flist;
 	mcryptd_flist = alloc_percpu(struct mcryptd_flush_list);
 	for_each_possible_cpu(cpu) {
 		flist = per_cpu_ptr(mcryptd_flist, cpu);
 		INIT_LIST_HEAD(&flist->list);
 		mutex_init(&flist->lock);
 	}
 	err = mcryptd_init_queue(&mqueue, MCRYPTD_MAX_CPU_QLEN);
 	if (err) {
 		free_percpu(mcryptd_flist);
 		return err;
 	}
 	err = crypto_register_template(&mcryptd_tmpl);
 	if (err) {
 		mcryptd_fini_queue(&mqueue);
 		free_percpu(mcryptd_flist);
 	}
 	return err;
 }
 static void __exit mcryptd_exit(void)
 {
 	mcryptd_fini_queue(&mqueue);
 	crypto_unregister_template(&mcryptd_tmpl);
 	free_percpu(mcryptd_flist);
 }
 subsys_initcall(mcryptd_init);
 module_exit(mcryptd_exit);
 MODULE_LICENSE("GPL");
 MODULE_DESCRIPTION("Software async multibuffer crypto daemon");
+MODULE_ALIAS_CRYPTO("mcryptd");

crypto/pcbc.c

Diff comments View file @ acc5ccb

 /*
  * PCBC: Propagating Cipher Block Chaining mode
  *
  * Copyright (C) 2006 Red Hat, Inc. All Rights Reserved.
  * Written by David Howells (dhowells@redhat.com)
  *
  * Derived from cbc.c
  * - Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au>
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the Free
  * Software Foundation; either version 2 of the License, or (at your option)
  * any later version.
  *
  */
 #include <crypto/algapi.h>
 #include <linux/err.h>
 #include <linux/init.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/scatterlist.h>
 #include <linux/slab.h>
 struct crypto_pcbc_ctx {
 	struct crypto_cipher *child;
 };
 static int crypto_pcbc_setkey(struct crypto_tfm *parent, const u8 *key,
 			      unsigned int keylen)
 {
 	struct crypto_pcbc_ctx *ctx = crypto_tfm_ctx(parent);
 	struct crypto_cipher *child = ctx->child;
 	int err;
 	crypto_cipher_clear_flags(child, CRYPTO_TFM_REQ_MASK);
 	crypto_cipher_set_flags(child, crypto_tfm_get_flags(parent) &
 				CRYPTO_TFM_REQ_MASK);
 	err = crypto_cipher_setkey(child, key, keylen);
 	crypto_tfm_set_flags(parent, crypto_cipher_get_flags(child) &
 			     CRYPTO_TFM_RES_MASK);
 	return err;
 }
 static int crypto_pcbc_encrypt_segment(struct blkcipher_desc *desc,
 				       struct blkcipher_walk *walk,
 				       struct crypto_cipher *tfm)
 {
 	void (*fn)(struct crypto_tfm *, u8 *, const u8 *) =
 		crypto_cipher_alg(tfm)->cia_encrypt;
 	int bsize = crypto_cipher_blocksize(tfm);
 	unsigned int nbytes = walk->nbytes;
 	u8 *src = walk->src.virt.addr;
 	u8 *dst = walk->dst.virt.addr;
 	u8 *iv = walk->iv;
 	do {
 		crypto_xor(iv, src, bsize);
 		fn(crypto_cipher_tfm(tfm), dst, iv);
 		memcpy(iv, dst, bsize);
 		crypto_xor(iv, src, bsize);
 		src += bsize;
 		dst += bsize;
 	} while ((nbytes -= bsize) >= bsize);
 	return nbytes;
 }
 static int crypto_pcbc_encrypt_inplace(struct blkcipher_desc *desc,
 				       struct blkcipher_walk *walk,
 				       struct crypto_cipher *tfm)
 {
 	void (*fn)(struct crypto_tfm *, u8 *, const u8 *) =
 		crypto_cipher_alg(tfm)->cia_encrypt;
 	int bsize = crypto_cipher_blocksize(tfm);
 	unsigned int nbytes = walk->nbytes;
 	u8 *src = walk->src.virt.addr;
 	u8 *iv = walk->iv;
 	u8 tmpbuf[bsize];
 	do {
 		memcpy(tmpbuf, src, bsize);
 		crypto_xor(iv, src, bsize);
 		fn(crypto_cipher_tfm(tfm), src, iv);
 		memcpy(iv, tmpbuf, bsize);
 		crypto_xor(iv, src, bsize);
 		src += bsize;
 	} while ((nbytes -= bsize) >= bsize);
 	memcpy(walk->iv, iv, bsize);
 	return nbytes;
 }
 static int crypto_pcbc_encrypt(struct blkcipher_desc *desc,
 			       struct scatterlist *dst, struct scatterlist *src,
 			       unsigned int nbytes)
 {
 	struct blkcipher_walk walk;
 	struct crypto_blkcipher *tfm = desc->tfm;
 	struct crypto_pcbc_ctx *ctx = crypto_blkcipher_ctx(tfm);
 	struct crypto_cipher *child = ctx->child;
 	int err;
 	blkcipher_walk_init(&walk, dst, src, nbytes);
 	err = blkcipher_walk_virt(desc, &walk);
 	while ((nbytes = walk.nbytes)) {
 		if (walk.src.virt.addr == walk.dst.virt.addr)
 			nbytes = crypto_pcbc_encrypt_inplace(desc, &walk,
 							     child);
 		else
 			nbytes = crypto_pcbc_encrypt_segment(desc, &walk,
 							     child);
 		err = blkcipher_walk_done(desc, &walk, nbytes);
 	}
 	return err;
 }
 static int crypto_pcbc_decrypt_segment(struct blkcipher_desc *desc,
 				       struct blkcipher_walk *walk,
 				       struct crypto_cipher *tfm)
 {
 	void (*fn)(struct crypto_tfm *, u8 *, const u8 *) =
 		crypto_cipher_alg(tfm)->cia_decrypt;
 	int bsize = crypto_cipher_blocksize(tfm);
 	unsigned int nbytes = walk->nbytes;
 	u8 *src = walk->src.virt.addr;
 	u8 *dst = walk->dst.virt.addr;
 	u8 *iv = walk->iv;
 	do {
 		fn(crypto_cipher_tfm(tfm), dst, src);
 		crypto_xor(dst, iv, bsize);
 		memcpy(iv, src, bsize);
 		crypto_xor(iv, dst, bsize);
 		src += bsize;
 		dst += bsize;
 	} while ((nbytes -= bsize) >= bsize);
 	memcpy(walk->iv, iv, bsize);
 	return nbytes;
 }
 static int crypto_pcbc_decrypt_inplace(struct blkcipher_desc *desc,
 				       struct blkcipher_walk *walk,
 				       struct crypto_cipher *tfm)
 {
 	void (*fn)(struct crypto_tfm *, u8 *, const u8 *) =
 		crypto_cipher_alg(tfm)->cia_decrypt;
 	int bsize = crypto_cipher_blocksize(tfm);
 	unsigned int nbytes = walk->nbytes;
 	u8 *src = walk->src.virt.addr;
 	u8 *iv = walk->iv;
 	u8 tmpbuf[bsize];
 	do {
 		memcpy(tmpbuf, src, bsize);
 		fn(crypto_cipher_tfm(tfm), src, src);
 		crypto_xor(src, iv, bsize);
 		memcpy(iv, tmpbuf, bsize);
 		crypto_xor(iv, src, bsize);
 		src += bsize;
 	} while ((nbytes -= bsize) >= bsize);
 	memcpy(walk->iv, iv, bsize);
 	return nbytes;
 }
 static int crypto_pcbc_decrypt(struct blkcipher_desc *desc,
 			       struct scatterlist *dst, struct scatterlist *src,
 			       unsigned int nbytes)
 {
 	struct blkcipher_walk walk;
 	struct crypto_blkcipher *tfm = desc->tfm;
 	struct crypto_pcbc_ctx *ctx = crypto_blkcipher_ctx(tfm);
 	struct crypto_cipher *child = ctx->child;
 	int err;
 	blkcipher_walk_init(&walk, dst, src, nbytes);
 	err = blkcipher_walk_virt(desc, &walk);
 	while ((nbytes = walk.nbytes)) {
 		if (walk.src.virt.addr == walk.dst.virt.addr)
 			nbytes = crypto_pcbc_decrypt_inplace(desc, &walk,
 							     child);
 		else
 			nbytes = crypto_pcbc_decrypt_segment(desc, &walk,
 							     child);
 		err = blkcipher_walk_done(desc, &walk, nbytes);
 	}
 	return err;
 }
 static int crypto_pcbc_init_tfm(struct crypto_tfm *tfm)
 {
 	struct crypto_instance *inst = (void *)tfm->__crt_alg;
 	struct crypto_spawn *spawn = crypto_instance_ctx(inst);
 	struct crypto_pcbc_ctx *ctx = crypto_tfm_ctx(tfm);
 	struct crypto_cipher *cipher;
 	cipher = crypto_spawn_cipher(spawn);
 	if (IS_ERR(cipher))
 		return PTR_ERR(cipher);
 	ctx->child = cipher;
 	return 0;
 }
 static void crypto_pcbc_exit_tfm(struct crypto_tfm *tfm)
 {
 	struct crypto_pcbc_ctx *ctx = crypto_tfm_ctx(tfm);
 	crypto_free_cipher(ctx->child);
 }
 static struct crypto_instance *crypto_pcbc_alloc(struct rtattr **tb)
 {
 	struct crypto_instance *inst;
 	struct crypto_alg *alg;
 	int err;
 	err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_BLKCIPHER);
 	if (err)
 		return ERR_PTR(err);
 	alg = crypto_get_attr_alg(tb, CRYPTO_ALG_TYPE_CIPHER,
 				  CRYPTO_ALG_TYPE_MASK);
 	if (IS_ERR(alg))
 		return ERR_CAST(alg);
 	inst = crypto_alloc_instance("pcbc", alg);
 	if (IS_ERR(inst))
 		goto out_put_alg;
 	inst->alg.cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER;
 	inst->alg.cra_priority = alg->cra_priority;
 	inst->alg.cra_blocksize = alg->cra_blocksize;
 	inst->alg.cra_alignmask = alg->cra_alignmask;
 	inst->alg.cra_type = &crypto_blkcipher_type;
 	/* We access the data as u32s when xoring. */
 	inst->alg.cra_alignmask |= __alignof__(u32) - 1;
 	inst->alg.cra_blkcipher.ivsize = alg->cra_blocksize;
 	inst->alg.cra_blkcipher.min_keysize = alg->cra_cipher.cia_min_keysize;
 	inst->alg.cra_blkcipher.max_keysize = alg->cra_cipher.cia_max_keysize;
 	inst->alg.cra_ctxsize = sizeof(struct crypto_pcbc_ctx);
 	inst->alg.cra_init = crypto_pcbc_init_tfm;
 	inst->alg.cra_exit = crypto_pcbc_exit_tfm;
 	inst->alg.cra_blkcipher.setkey = crypto_pcbc_setkey;
 	inst->alg.cra_blkcipher.encrypt = crypto_pcbc_encrypt;
 	inst->alg.cra_blkcipher.decrypt = crypto_pcbc_decrypt;
 out_put_alg:
 	crypto_mod_put(alg);
 	return inst;
 }
 static void crypto_pcbc_free(struct crypto_instance *inst)
 {
 	crypto_drop_spawn(crypto_instance_ctx(inst));
 	kfree(inst);
 }
 static struct crypto_template crypto_pcbc_tmpl = {
 	.name = "pcbc",
 	.alloc = crypto_pcbc_alloc,
 	.free = crypto_pcbc_free,
 	.module = THIS_MODULE,
 };
 static int __init crypto_pcbc_module_init(void)
 {
 	return crypto_register_template(&crypto_pcbc_tmpl);
 }
 static void __exit crypto_pcbc_module_exit(void)
 {
 	crypto_unregister_template(&crypto_pcbc_tmpl);
 }
 module_init(crypto_pcbc_module_init);
 module_exit(crypto_pcbc_module_exit);
 MODULE_LICENSE("GPL");
 MODULE_DESCRIPTION("PCBC block cipher algorithm");
+MODULE_ALIAS_CRYPTO("pcbc");

crypto/pcrypt.c

Diff comments View file @ acc5ccb

 /*
  * pcrypt - Parallel crypto wrapper.
  *
  * Copyright (C) 2009 secunet Security Networks AG
  * Copyright (C) 2009 Steffen Klassert <steffen.klassert@secunet.com>
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms and conditions of the GNU General Public License,
  * version 2, as published by the Free Software Foundation.
  *
  * This program is distributed in the hope it will be useful, but WITHOUT
  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  * more details.
  *
  * You should have received a copy of the GNU General Public License along with
  * this program; if not, write to the Free Software Foundation, Inc.,
  * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
  */
 #include <crypto/algapi.h>
 #include <crypto/internal/aead.h>
 #include <linux/err.h>
 #include <linux/init.h>
 #include <linux/module.h>
 #include <linux/slab.h>
 #include <linux/notifier.h>
 #include <linux/kobject.h>
 #include <linux/cpu.h>
 #include <crypto/pcrypt.h>
 struct padata_pcrypt {
 	struct padata_instance *pinst;
 	struct workqueue_struct *wq;
 	/*
 	 * Cpumask for callback CPUs. It should be
 	 * equal to serial cpumask of corresponding padata instance,
 	 * so it is updated when padata notifies us about serial
 	 * cpumask change.
 	 *
 	 * cb_cpumask is protected by RCU. This fact prevents us from
 	 * using cpumask_var_t directly because the actual type of
 	 * cpumsak_var_t depends on kernel configuration(particularly on
 	 * CONFIG_CPUMASK_OFFSTACK macro). Depending on the configuration
 	 * cpumask_var_t may be either a pointer to the struct cpumask
 	 * or a variable allocated on the stack. Thus we can not safely use
 	 * cpumask_var_t with RCU operations such as rcu_assign_pointer or
 	 * rcu_dereference. So cpumask_var_t is wrapped with struct
 	 * pcrypt_cpumask which makes possible to use it with RCU.
 	 */
 	struct pcrypt_cpumask {
 		cpumask_var_t mask;
 	} *cb_cpumask;
 	struct notifier_block nblock;
 };
 static struct padata_pcrypt pencrypt;
 static struct padata_pcrypt pdecrypt;
 static struct kset           *pcrypt_kset;
 struct pcrypt_instance_ctx {
 	struct crypto_spawn spawn;
 	unsigned int tfm_count;
 };
 struct pcrypt_aead_ctx {
 	struct crypto_aead *child;
 	unsigned int cb_cpu;
 };
 static int pcrypt_do_parallel(struct padata_priv *padata, unsigned int *cb_cpu,
 			      struct padata_pcrypt *pcrypt)
 {
 	unsigned int cpu_index, cpu, i;
 	struct pcrypt_cpumask *cpumask;
 	cpu = *cb_cpu;
 	rcu_read_lock_bh();
 	cpumask = rcu_dereference_bh(pcrypt->cb_cpumask);
 	if (cpumask_test_cpu(cpu, cpumask->mask))
 			goto out;
 	if (!cpumask_weight(cpumask->mask))
 			goto out;
 	cpu_index = cpu % cpumask_weight(cpumask->mask);
 	cpu = cpumask_first(cpumask->mask);
 	for (i = 0; i < cpu_index; i++)
 		cpu = cpumask_next(cpu, cpumask->mask);
 	*cb_cpu = cpu;
 out:
 	rcu_read_unlock_bh();
 	return padata_do_parallel(pcrypt->pinst, padata, cpu);
 }
 static int pcrypt_aead_setkey(struct crypto_aead *parent,
 			      const u8 *key, unsigned int keylen)
 {
 	struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(parent);
 	return crypto_aead_setkey(ctx->child, key, keylen);
 }
 static int pcrypt_aead_setauthsize(struct crypto_aead *parent,
 				   unsigned int authsize)
 {
 	struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(parent);
 	return crypto_aead_setauthsize(ctx->child, authsize);
 }
 static void pcrypt_aead_serial(struct padata_priv *padata)
 {
 	struct pcrypt_request *preq = pcrypt_padata_request(padata);
 	struct aead_request *req = pcrypt_request_ctx(preq);
 	aead_request_complete(req->base.data, padata->info);
 }
 static void pcrypt_aead_giv_serial(struct padata_priv *padata)
 {
 	struct pcrypt_request *preq = pcrypt_padata_request(padata);
 	struct aead_givcrypt_request *req = pcrypt_request_ctx(preq);
 	aead_request_complete(req->areq.base.data, padata->info);
 }
 static void pcrypt_aead_done(struct crypto_async_request *areq, int err)
 {
 	struct aead_request *req = areq->data;
 	struct pcrypt_request *preq = aead_request_ctx(req);
 	struct padata_priv *padata = pcrypt_request_padata(preq);
 	padata->info = err;
 	req->base.flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 	padata_do_serial(padata);
 }
 static void pcrypt_aead_enc(struct padata_priv *padata)
 {
 	struct pcrypt_request *preq = pcrypt_padata_request(padata);
 	struct aead_request *req = pcrypt_request_ctx(preq);
 	padata->info = crypto_aead_encrypt(req);
 	if (padata->info == -EINPROGRESS)
 		return;
 	padata_do_serial(padata);
 }
 static int pcrypt_aead_encrypt(struct aead_request *req)
 {
 	int err;
 	struct pcrypt_request *preq = aead_request_ctx(req);
 	struct aead_request *creq = pcrypt_request_ctx(preq);
 	struct padata_priv *padata = pcrypt_request_padata(preq);
 	struct crypto_aead *aead = crypto_aead_reqtfm(req);
 	struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(aead);
 	u32 flags = aead_request_flags(req);
 	memset(padata, 0, sizeof(struct padata_priv));
 	padata->parallel = pcrypt_aead_enc;
 	padata->serial = pcrypt_aead_serial;
 	aead_request_set_tfm(creq, ctx->child);
 	aead_request_set_callback(creq, flags & ~CRYPTO_TFM_REQ_MAY_SLEEP,
 				  pcrypt_aead_done, req);
 	aead_request_set_crypt(creq, req->src, req->dst,
 			       req->cryptlen, req->iv);
 	aead_request_set_assoc(creq, req->assoc, req->assoclen);
 	err = pcrypt_do_parallel(padata, &ctx->cb_cpu, &pencrypt);
 	if (!err)
 		return -EINPROGRESS;
 	return err;
 }
 static void pcrypt_aead_dec(struct padata_priv *padata)
 {
 	struct pcrypt_request *preq = pcrypt_padata_request(padata);
 	struct aead_request *req = pcrypt_request_ctx(preq);
 	padata->info = crypto_aead_decrypt(req);
 	if (padata->info == -EINPROGRESS)
 		return;
 	padata_do_serial(padata);
 }
 static int pcrypt_aead_decrypt(struct aead_request *req)
 {
 	int err;
 	struct pcrypt_request *preq = aead_request_ctx(req);
 	struct aead_request *creq = pcrypt_request_ctx(preq);
 	struct padata_priv *padata = pcrypt_request_padata(preq);
 	struct crypto_aead *aead = crypto_aead_reqtfm(req);
 	struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(aead);
 	u32 flags = aead_request_flags(req);
 	memset(padata, 0, sizeof(struct padata_priv));
 	padata->parallel = pcrypt_aead_dec;
 	padata->serial = pcrypt_aead_serial;
 	aead_request_set_tfm(creq, ctx->child);
 	aead_request_set_callback(creq, flags & ~CRYPTO_TFM_REQ_MAY_SLEEP,
 				  pcrypt_aead_done, req);
 	aead_request_set_crypt(creq, req->src, req->dst,
 			       req->cryptlen, req->iv);
 	aead_request_set_assoc(creq, req->assoc, req->assoclen);
 	err = pcrypt_do_parallel(padata, &ctx->cb_cpu, &pdecrypt);
 	if (!err)
 		return -EINPROGRESS;
 	return err;
 }
 static void pcrypt_aead_givenc(struct padata_priv *padata)
 {
 	struct pcrypt_request *preq = pcrypt_padata_request(padata);
 	struct aead_givcrypt_request *req = pcrypt_request_ctx(preq);
 	padata->info = crypto_aead_givencrypt(req);
 	if (padata->info == -EINPROGRESS)
 		return;
 	padata_do_serial(padata);
 }
 static int pcrypt_aead_givencrypt(struct aead_givcrypt_request *req)
 {
 	int err;
 	struct aead_request *areq = &req->areq;
 	struct pcrypt_request *preq = aead_request_ctx(areq);
 	struct aead_givcrypt_request *creq = pcrypt_request_ctx(preq);
 	struct padata_priv *padata = pcrypt_request_padata(preq);
 	struct crypto_aead *aead = aead_givcrypt_reqtfm(req);
 	struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(aead);
 	u32 flags = aead_request_flags(areq);
 	memset(padata, 0, sizeof(struct padata_priv));
 	padata->parallel = pcrypt_aead_givenc;
 	padata->serial = pcrypt_aead_giv_serial;
 	aead_givcrypt_set_tfm(creq, ctx->child);
 	aead_givcrypt_set_callback(creq, flags & ~CRYPTO_TFM_REQ_MAY_SLEEP,
 				   pcrypt_aead_done, areq);
 	aead_givcrypt_set_crypt(creq, areq->src, areq->dst,
 				areq->cryptlen, areq->iv);
 	aead_givcrypt_set_assoc(creq, areq->assoc, areq->assoclen);
 	aead_givcrypt_set_giv(creq, req->giv, req->seq);
 	err = pcrypt_do_parallel(padata, &ctx->cb_cpu, &pencrypt);
 	if (!err)
 		return -EINPROGRESS;
 	return err;
 }
 static int pcrypt_aead_init_tfm(struct crypto_tfm *tfm)
 {
 	int cpu, cpu_index;
 	struct crypto_instance *inst = crypto_tfm_alg_instance(tfm);
 	struct pcrypt_instance_ctx *ictx = crypto_instance_ctx(inst);
 	struct pcrypt_aead_ctx *ctx = crypto_tfm_ctx(tfm);
 	struct crypto_aead *cipher;
 	ictx->tfm_count++;
 	cpu_index = ictx->tfm_count % cpumask_weight(cpu_online_mask);
 	ctx->cb_cpu = cpumask_first(cpu_online_mask);
 	for (cpu = 0; cpu < cpu_index; cpu++)
 		ctx->cb_cpu = cpumask_next(ctx->cb_cpu, cpu_online_mask);
 	cipher = crypto_spawn_aead(crypto_instance_ctx(inst));
 	if (IS_ERR(cipher))
 		return PTR_ERR(cipher);
 	ctx->child = cipher;
 	tfm->crt_aead.reqsize = sizeof(struct pcrypt_request)
 		+ sizeof(struct aead_givcrypt_request)
 		+ crypto_aead_reqsize(cipher);
 	return 0;
 }
 static void pcrypt_aead_exit_tfm(struct crypto_tfm *tfm)
 {
 	struct pcrypt_aead_ctx *ctx = crypto_tfm_ctx(tfm);
 	crypto_free_aead(ctx->child);
 }
 static struct crypto_instance *pcrypt_alloc_instance(struct crypto_alg *alg)
 {
 	struct crypto_instance *inst;
 	struct pcrypt_instance_ctx *ctx;
 	int err;
 	inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL);
 	if (!inst) {
 		inst = ERR_PTR(-ENOMEM);
 		goto out;
 	}
 	err = -ENAMETOOLONG;
 	if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME,
 		     "pcrypt(%s)", alg->cra_driver_name) >= CRYPTO_MAX_ALG_NAME)
 		goto out_free_inst;
 	memcpy(inst->alg.cra_name, alg->cra_name, CRYPTO_MAX_ALG_NAME);
 	ctx = crypto_instance_ctx(inst);
 	err = crypto_init_spawn(&ctx->spawn, alg, inst,
 				CRYPTO_ALG_TYPE_MASK);
 	if (err)
 		goto out_free_inst;
 	inst->alg.cra_priority = alg->cra_priority + 100;
 	inst->alg.cra_blocksize = alg->cra_blocksize;
 	inst->alg.cra_alignmask = alg->cra_alignmask;
 out:
 	return inst;
 out_free_inst:
 	kfree(inst);
 	inst = ERR_PTR(err);
 	goto out;
 }
 static struct crypto_instance *pcrypt_alloc_aead(struct rtattr **tb,
 						 u32 type, u32 mask)
 {
 	struct crypto_instance *inst;
 	struct crypto_alg *alg;
 	alg = crypto_get_attr_alg(tb, type, (mask & CRYPTO_ALG_TYPE_MASK));
 	if (IS_ERR(alg))
 		return ERR_CAST(alg);
 	inst = pcrypt_alloc_instance(alg);
 	if (IS_ERR(inst))
 		goto out_put_alg;
 	inst->alg.cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC;
 	inst->alg.cra_type = &crypto_aead_type;
 	inst->alg.cra_aead.ivsize = alg->cra_aead.ivsize;
 	inst->alg.cra_aead.geniv = alg->cra_aead.geniv;
 	inst->alg.cra_aead.maxauthsize = alg->cra_aead.maxauthsize;
 	inst->alg.cra_ctxsize = sizeof(struct pcrypt_aead_ctx);
 	inst->alg.cra_init = pcrypt_aead_init_tfm;
 	inst->alg.cra_exit = pcrypt_aead_exit_tfm;
 	inst->alg.cra_aead.setkey = pcrypt_aead_setkey;
 	inst->alg.cra_aead.setauthsize = pcrypt_aead_setauthsize;
 	inst->alg.cra_aead.encrypt = pcrypt_aead_encrypt;
 	inst->alg.cra_aead.decrypt = pcrypt_aead_decrypt;
 	inst->alg.cra_aead.givencrypt = pcrypt_aead_givencrypt;
 out_put_alg:
 	crypto_mod_put(alg);
 	return inst;
 }
 static struct crypto_instance *pcrypt_alloc(struct rtattr **tb)
 {
 	struct crypto_attr_type *algt;
 	algt = crypto_get_attr_type(tb);
 	if (IS_ERR(algt))
 		return ERR_CAST(algt);
 	switch (algt->type & algt->mask & CRYPTO_ALG_TYPE_MASK) {
 	case CRYPTO_ALG_TYPE_AEAD:
 		return pcrypt_alloc_aead(tb, algt->type, algt->mask);
 	}
 	return ERR_PTR(-EINVAL);
 }
 static void pcrypt_free(struct crypto_instance *inst)
 {
 	struct pcrypt_instance_ctx *ctx = crypto_instance_ctx(inst);
 	crypto_drop_spawn(&ctx->spawn);
 	kfree(inst);
 }
 static int pcrypt_cpumask_change_notify(struct notifier_block *self,
 					unsigned long val, void *data)
 {
 	struct padata_pcrypt *pcrypt;
 	struct pcrypt_cpumask *new_mask, *old_mask;
 	struct padata_cpumask *cpumask = (struct padata_cpumask *)data;
 	if (!(val & PADATA_CPU_SERIAL))
 		return 0;
 	pcrypt = container_of(self, struct padata_pcrypt, nblock);
 	new_mask = kmalloc(sizeof(*new_mask), GFP_KERNEL);
 	if (!new_mask)
 		return -ENOMEM;
 	if (!alloc_cpumask_var(&new_mask->mask, GFP_KERNEL)) {
 		kfree(new_mask);
 		return -ENOMEM;
 	}
 	old_mask = pcrypt->cb_cpumask;
 	cpumask_copy(new_mask->mask, cpumask->cbcpu);
 	rcu_assign_pointer(pcrypt->cb_cpumask, new_mask);
 	synchronize_rcu_bh();
 	free_cpumask_var(old_mask->mask);
 	kfree(old_mask);
 	return 0;
 }
 static int pcrypt_sysfs_add(struct padata_instance *pinst, const char *name)
 {
 	int ret;
 	pinst->kobj.kset = pcrypt_kset;
 	ret = kobject_add(&pinst->kobj, NULL, name);
 	if (!ret)
 		kobject_uevent(&pinst->kobj, KOBJ_ADD);
 	return ret;
 }
 static int pcrypt_init_padata(struct padata_pcrypt *pcrypt,
 			      const char *name)
 {
 	int ret = -ENOMEM;
 	struct pcrypt_cpumask *mask;
 	get_online_cpus();
 	pcrypt->wq = alloc_workqueue("%s", WQ_MEM_RECLAIM | WQ_CPU_INTENSIVE,
 				     1, name);
 	if (!pcrypt->wq)
 		goto err;
 	pcrypt->pinst = padata_alloc_possible(pcrypt->wq);
 	if (!pcrypt->pinst)
 		goto err_destroy_workqueue;
 	mask = kmalloc(sizeof(*mask), GFP_KERNEL);
 	if (!mask)
 		goto err_free_padata;
 	if (!alloc_cpumask_var(&mask->mask, GFP_KERNEL)) {
 		kfree(mask);
 		goto err_free_padata;
 	}
 	cpumask_and(mask->mask, cpu_possible_mask, cpu_online_mask);
 	rcu_assign_pointer(pcrypt->cb_cpumask, mask);
 	pcrypt->nblock.notifier_call = pcrypt_cpumask_change_notify;
 	ret = padata_register_cpumask_notifier(pcrypt->pinst, &pcrypt->nblock);
 	if (ret)
 		goto err_free_cpumask;
 	ret = pcrypt_sysfs_add(pcrypt->pinst, name);
 	if (ret)
 		goto err_unregister_notifier;
 	put_online_cpus();
 	return ret;
 err_unregister_notifier:
 	padata_unregister_cpumask_notifier(pcrypt->pinst, &pcrypt->nblock);
 err_free_cpumask:
 	free_cpumask_var(mask->mask);
 	kfree(mask);
 err_free_padata:
 	padata_free(pcrypt->pinst);
 err_destroy_workqueue:
 	destroy_workqueue(pcrypt->wq);
 err:
 	put_online_cpus();
 	return ret;
 }
 static void pcrypt_fini_padata(struct padata_pcrypt *pcrypt)
 {
 	free_cpumask_var(pcrypt->cb_cpumask->mask);
 	kfree(pcrypt->cb_cpumask);
 	padata_stop(pcrypt->pinst);
 	padata_unregister_cpumask_notifier(pcrypt->pinst, &pcrypt->nblock);
 	destroy_workqueue(pcrypt->wq);
 	padata_free(pcrypt->pinst);
 }
 static struct crypto_template pcrypt_tmpl = {
 	.name = "pcrypt",
 	.alloc = pcrypt_alloc,
 	.free = pcrypt_free,
 	.module = THIS_MODULE,
 };
 static int __init pcrypt_init(void)
 {
 	int err = -ENOMEM;
 	pcrypt_kset = kset_create_and_add("pcrypt", NULL, kernel_kobj);
 	if (!pcrypt_kset)
 		goto err;
 	err = pcrypt_init_padata(&pencrypt, "pencrypt");
 	if (err)
 		goto err_unreg_kset;
 	err = pcrypt_init_padata(&pdecrypt, "pdecrypt");
 	if (err)
 		goto err_deinit_pencrypt;
 	padata_start(pencrypt.pinst);
 	padata_start(pdecrypt.pinst);
 	return crypto_register_template(&pcrypt_tmpl);
 err_deinit_pencrypt:
 	pcrypt_fini_padata(&pencrypt);
 err_unreg_kset:
 	kset_unregister(pcrypt_kset);
 err:
 	return err;
 }
 static void __exit pcrypt_exit(void)
 {
 	pcrypt_fini_padata(&pencrypt);
 	pcrypt_fini_padata(&pdecrypt);
 	kset_unregister(pcrypt_kset);
 	crypto_unregister_template(&pcrypt_tmpl);
 }
 module_init(pcrypt_init);
 module_exit(pcrypt_exit);
 MODULE_LICENSE("GPL");
 MODULE_AUTHOR("Steffen Klassert <steffen.klassert@secunet.com>");
 MODULE_DESCRIPTION("Parallel crypto wrapper");
+MODULE_ALIAS_CRYPTO("pcrypt");

crypto/seqiv.c

Diff comments View file @ acc5ccb

 /*
  * seqiv: Sequence Number IV Generator
  *
  * This generator generates an IV based on a sequence number by xoring it
  * with a salt.  This algorithm is mainly useful for CTR and similar modes.
  *
  * Copyright (c) 2007 Herbert Xu <herbert@gondor.apana.org.au>
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the Free
  * Software Foundation; either version 2 of the License, or (at your option)
  * any later version.
  *
  */
 #include <crypto/internal/aead.h>
 #include <crypto/internal/skcipher.h>
 #include <crypto/rng.h>
 #include <linux/err.h>
 #include <linux/init.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/slab.h>
 #include <linux/spinlock.h>
 #include <linux/string.h>
 struct seqiv_ctx {
 	spinlock_t lock;
 	u8 salt[] __attribute__ ((aligned(__alignof__(u32))));
 };
 static void seqiv_complete2(struct skcipher_givcrypt_request *req, int err)
 {
 	struct ablkcipher_request *subreq = skcipher_givcrypt_reqctx(req);
 	struct crypto_ablkcipher *geniv;
 	if (err == -EINPROGRESS)
 		return;
 	if (err)
 		goto out;
 	geniv = skcipher_givcrypt_reqtfm(req);
 	memcpy(req->creq.info, subreq->info, crypto_ablkcipher_ivsize(geniv));
 out:
 	kfree(subreq->info);
 }
 static void seqiv_complete(struct crypto_async_request *base, int err)
 {
 	struct skcipher_givcrypt_request *req = base->data;
 	seqiv_complete2(req, err);
 	skcipher_givcrypt_complete(req, err);
 }
 static void seqiv_aead_complete2(struct aead_givcrypt_request *req, int err)
 {
 	struct aead_request *subreq = aead_givcrypt_reqctx(req);
 	struct crypto_aead *geniv;
 	if (err == -EINPROGRESS)
 		return;
 	if (err)
 		goto out;
 	geniv = aead_givcrypt_reqtfm(req);
 	memcpy(req->areq.iv, subreq->iv, crypto_aead_ivsize(geniv));
 out:
 	kfree(subreq->iv);
 }
 static void seqiv_aead_complete(struct crypto_async_request *base, int err)
 {
 	struct aead_givcrypt_request *req = base->data;
 	seqiv_aead_complete2(req, err);
 	aead_givcrypt_complete(req, err);
 }
 static void seqiv_geniv(struct seqiv_ctx *ctx, u8 *info, u64 seq,
 			unsigned int ivsize)
 {
 	unsigned int len = ivsize;
 	if (ivsize > sizeof(u64)) {
 		memset(info, 0, ivsize - sizeof(u64));
 		len = sizeof(u64);
 	}
 	seq = cpu_to_be64(seq);
 	memcpy(info + ivsize - len, &seq, len);
 	crypto_xor(info, ctx->salt, ivsize);
 }
 static int seqiv_givencrypt(struct skcipher_givcrypt_request *req)
 {
 	struct crypto_ablkcipher *geniv = skcipher_givcrypt_reqtfm(req);
 	struct seqiv_ctx *ctx = crypto_ablkcipher_ctx(geniv);
 	struct ablkcipher_request *subreq = skcipher_givcrypt_reqctx(req);
 	crypto_completion_t compl;
 	void *data;
 	u8 *info;
 	unsigned int ivsize;
 	int err;
 	ablkcipher_request_set_tfm(subreq, skcipher_geniv_cipher(geniv));
 	compl = req->creq.base.complete;
 	data = req->creq.base.data;
 	info = req->creq.info;
 	ivsize = crypto_ablkcipher_ivsize(geniv);
 	if (unlikely(!IS_ALIGNED((unsigned long)info,
 				 crypto_ablkcipher_alignmask(geniv) + 1))) {
 		info = kmalloc(ivsize, req->creq.base.flags &
 				       CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL:
 								  GFP_ATOMIC);
 		if (!info)
 			return -ENOMEM;
 		compl = seqiv_complete;
 		data = req;
 	}
 	ablkcipher_request_set_callback(subreq, req->creq.base.flags, compl,
 					data);
 	ablkcipher_request_set_crypt(subreq, req->creq.src, req->creq.dst,
 				     req->creq.nbytes, info);
 	seqiv_geniv(ctx, info, req->seq, ivsize);
 	memcpy(req->giv, info, ivsize);
 	err = crypto_ablkcipher_encrypt(subreq);
 	if (unlikely(info != req->creq.info))
 		seqiv_complete2(req, err);
 	return err;
 }
 static int seqiv_aead_givencrypt(struct aead_givcrypt_request *req)
 {
 	struct crypto_aead *geniv = aead_givcrypt_reqtfm(req);
 	struct seqiv_ctx *ctx = crypto_aead_ctx(geniv);
 	struct aead_request *areq = &req->areq;
 	struct aead_request *subreq = aead_givcrypt_reqctx(req);
 	crypto_completion_t compl;
 	void *data;
 	u8 *info;
 	unsigned int ivsize;
 	int err;
 	aead_request_set_tfm(subreq, aead_geniv_base(geniv));
 	compl = areq->base.complete;
 	data = areq->base.data;
 	info = areq->iv;
 	ivsize = crypto_aead_ivsize(geniv);
 	if (unlikely(!IS_ALIGNED((unsigned long)info,
 				 crypto_aead_alignmask(geniv) + 1))) {
 		info = kmalloc(ivsize, areq->base.flags &
 				       CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL:
 								  GFP_ATOMIC);
 		if (!info)
 			return -ENOMEM;
 		compl = seqiv_aead_complete;
 		data = req;
 	}
 	aead_request_set_callback(subreq, areq->base.flags, compl, data);
 	aead_request_set_crypt(subreq, areq->src, areq->dst, areq->cryptlen,
 			       info);
 	aead_request_set_assoc(subreq, areq->assoc, areq->assoclen);
 	seqiv_geniv(ctx, info, req->seq, ivsize);
 	memcpy(req->giv, info, ivsize);
 	err = crypto_aead_encrypt(subreq);
 	if (unlikely(info != areq->iv))
 		seqiv_aead_complete2(req, err);
 	return err;
 }
 static int seqiv_givencrypt_first(struct skcipher_givcrypt_request *req)
 {
 	struct crypto_ablkcipher *geniv = skcipher_givcrypt_reqtfm(req);
 	struct seqiv_ctx *ctx = crypto_ablkcipher_ctx(geniv);
 	int err = 0;
 	spin_lock_bh(&ctx->lock);
 	if (crypto_ablkcipher_crt(geniv)->givencrypt != seqiv_givencrypt_first)
 		goto unlock;
 	crypto_ablkcipher_crt(geniv)->givencrypt = seqiv_givencrypt;
 	err = crypto_rng_get_bytes(crypto_default_rng, ctx->salt,
 				   crypto_ablkcipher_ivsize(geniv));
 unlock:
 	spin_unlock_bh(&ctx->lock);
 	if (err)
 		return err;
 	return seqiv_givencrypt(req);
 }
 static int seqiv_aead_givencrypt_first(struct aead_givcrypt_request *req)
 {
 	struct crypto_aead *geniv = aead_givcrypt_reqtfm(req);
 	struct seqiv_ctx *ctx = crypto_aead_ctx(geniv);
 	int err = 0;
 	spin_lock_bh(&ctx->lock);
 	if (crypto_aead_crt(geniv)->givencrypt != seqiv_aead_givencrypt_first)
 		goto unlock;
 	crypto_aead_crt(geniv)->givencrypt = seqiv_aead_givencrypt;
 	err = crypto_rng_get_bytes(crypto_default_rng, ctx->salt,
 				   crypto_aead_ivsize(geniv));
 unlock:
 	spin_unlock_bh(&ctx->lock);
 	if (err)
 		return err;
 	return seqiv_aead_givencrypt(req);
 }
 static int seqiv_init(struct crypto_tfm *tfm)
 {
 	struct crypto_ablkcipher *geniv = __crypto_ablkcipher_cast(tfm);
 	struct seqiv_ctx *ctx = crypto_ablkcipher_ctx(geniv);
 	spin_lock_init(&ctx->lock);
 	tfm->crt_ablkcipher.reqsize = sizeof(struct ablkcipher_request);
 	return skcipher_geniv_init(tfm);
 }
 static int seqiv_aead_init(struct crypto_tfm *tfm)
 {
 	struct crypto_aead *geniv = __crypto_aead_cast(tfm);
 	struct seqiv_ctx *ctx = crypto_aead_ctx(geniv);
 	spin_lock_init(&ctx->lock);
 	tfm->crt_aead.reqsize = sizeof(struct aead_request);
 	return aead_geniv_init(tfm);
 }
 static struct crypto_template seqiv_tmpl;
 static struct crypto_instance *seqiv_ablkcipher_alloc(struct rtattr **tb)
 {
 	struct crypto_instance *inst;
 	inst = skcipher_geniv_alloc(&seqiv_tmpl, tb, 0, 0);
 	if (IS_ERR(inst))
 		goto out;
 	inst->alg.cra_ablkcipher.givencrypt = seqiv_givencrypt_first;
 	inst->alg.cra_init = seqiv_init;
 	inst->alg.cra_exit = skcipher_geniv_exit;
 	inst->alg.cra_ctxsize += inst->alg.cra_ablkcipher.ivsize;
 out:
 	return inst;
 }
 static struct crypto_instance *seqiv_aead_alloc(struct rtattr **tb)
 {
 	struct crypto_instance *inst;
 	inst = aead_geniv_alloc(&seqiv_tmpl, tb, 0, 0);
 	if (IS_ERR(inst))
 		goto out;
 	inst->alg.cra_aead.givencrypt = seqiv_aead_givencrypt_first;
 	inst->alg.cra_init = seqiv_aead_init;
 	inst->alg.cra_exit = aead_geniv_exit;
 	inst->alg.cra_ctxsize = inst->alg.cra_aead.ivsize;
 out:
 	return inst;
 }
 static struct crypto_instance *seqiv_alloc(struct rtattr **tb)
 {
 	struct crypto_attr_type *algt;
 	struct crypto_instance *inst;
 	int err;
 	algt = crypto_get_attr_type(tb);
 	if (IS_ERR(algt))
 		return ERR_CAST(algt);
 	err = crypto_get_default_rng();
 	if (err)
 		return ERR_PTR(err);
 	if ((algt->type ^ CRYPTO_ALG_TYPE_AEAD) & CRYPTO_ALG_TYPE_MASK)
 		inst = seqiv_ablkcipher_alloc(tb);
 	else
 		inst = seqiv_aead_alloc(tb);
 	if (IS_ERR(inst))
 		goto put_rng;
 	inst->alg.cra_alignmask |= __alignof__(u32) - 1;
 	inst->alg.cra_ctxsize += sizeof(struct seqiv_ctx);
 out:
 	return inst;
 put_rng:
 	crypto_put_default_rng();
 	goto out;
 }
 static void seqiv_free(struct crypto_instance *inst)
 {
 	if ((inst->alg.cra_flags ^ CRYPTO_ALG_TYPE_AEAD) & CRYPTO_ALG_TYPE_MASK)
 		skcipher_geniv_free(inst);
 	else
 		aead_geniv_free(inst);
 	crypto_put_default_rng();
 }
 static struct crypto_template seqiv_tmpl = {
 	.name = "seqiv",
 	.alloc = seqiv_alloc,
 	.free = seqiv_free,
 	.module = THIS_MODULE,
 };
 static int __init seqiv_module_init(void)
 {
 	return crypto_register_template(&seqiv_tmpl);
 }
 static void __exit seqiv_module_exit(void)
 {
 	crypto_unregister_template(&seqiv_tmpl);
 }
 module_init(seqiv_module_init);
 module_exit(seqiv_module_exit);
 MODULE_LICENSE("GPL");
 MODULE_DESCRIPTION("Sequence Number IV Generator");
+MODULE_ALIAS_CRYPTO("seqiv");

crypto/vmac.c

Diff comments View file @ acc5ccb

 /*
  * Modified to interface to the Linux kernel
  * Copyright (c) 2009, Intel Corporation.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms and conditions of the GNU General Public License,
  * version 2, as published by the Free Software Foundation.
  *
  * This program is distributed in the hope it will be useful, but WITHOUT
  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  * more details.
  *
  * You should have received a copy of the GNU General Public License along with
  * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
  * Place - Suite 330, Boston, MA 02111-1307 USA.
  */
 /* --------------------------------------------------------------------------
  * VMAC and VHASH Implementation by Ted Krovetz (tdk@acm.org) and Wei Dai.
  * This implementation is herby placed in the public domain.
  * The authors offers no warranty. Use at your own risk.
  * Please send bug reports to the authors.
  * Last modified: 17 APR 08, 1700 PDT
  * ----------------------------------------------------------------------- */
 #include <linux/init.h>
 #include <linux/types.h>
 #include <linux/crypto.h>
 #include <linux/module.h>
 #include <linux/scatterlist.h>
 #include <asm/byteorder.h>
 #include <crypto/scatterwalk.h>
 #include <crypto/vmac.h>
 #include <crypto/internal/hash.h>
 /*
  * Constants and masks
  */
 #define UINT64_C(x) x##ULL
 static const u64 p64   = UINT64_C(0xfffffffffffffeff);	/* 2^64 - 257 prime  */
 static const u64 m62   = UINT64_C(0x3fffffffffffffff);	/* 62-bit mask       */
 static const u64 m63   = UINT64_C(0x7fffffffffffffff);	/* 63-bit mask       */
 static const u64 m64   = UINT64_C(0xffffffffffffffff);	/* 64-bit mask       */
 static const u64 mpoly = UINT64_C(0x1fffffff1fffffff);	/* Poly key mask     */
 #define pe64_to_cpup le64_to_cpup		/* Prefer little endian */
 #ifdef __LITTLE_ENDIAN
 #define INDEX_HIGH 1
 #define INDEX_LOW 0
 #else
 #define INDEX_HIGH 0
 #define INDEX_LOW 1
 #endif
 /*
  * The following routines are used in this implementation. They are
  * written via macros to simulate zero-overhead call-by-reference.
  *
  * MUL64: 64x64->128-bit multiplication
  * PMUL64: assumes top bits cleared on inputs
  * ADD128: 128x128->128-bit addition
  */
 #define ADD128(rh, rl, ih, il)						\
 	do {								\
 		u64 _il = (il);						\
 		(rl) += (_il);						\
 		if ((rl) < (_il))					\
 			(rh)++;						\
 		(rh) += (ih);						\
 	} while (0)
 #define MUL32(i1, i2)	((u64)(u32)(i1)*(u32)(i2))
 #define PMUL64(rh, rl, i1, i2)	/* Assumes m doesn't overflow */	\
 	do {								\
 		u64 _i1 = (i1), _i2 = (i2);				\
 		u64 m = MUL32(_i1, _i2>>32) + MUL32(_i1>>32, _i2);	\
 		rh = MUL32(_i1>>32, _i2>>32);				\
 		rl = MUL32(_i1, _i2);					\
 		ADD128(rh, rl, (m >> 32), (m << 32));			\
 	} while (0)
 #define MUL64(rh, rl, i1, i2)						\
 	do {								\
 		u64 _i1 = (i1), _i2 = (i2);				\
 		u64 m1 = MUL32(_i1, _i2>>32);				\
 		u64 m2 = MUL32(_i1>>32, _i2);				\
 		rh = MUL32(_i1>>32, _i2>>32);				\
 		rl = MUL32(_i1, _i2);					\
 		ADD128(rh, rl, (m1 >> 32), (m1 << 32));			\
 		ADD128(rh, rl, (m2 >> 32), (m2 << 32));			\
 	} while (0)
 /*
  * For highest performance the L1 NH and L2 polynomial hashes should be
  * carefully implemented to take advantage of one's target architecture.
  * Here these two hash functions are defined multiple time; once for
  * 64-bit architectures, once for 32-bit SSE2 architectures, and once
  * for the rest (32-bit) architectures.
  * For each, nh_16 *must* be defined (works on multiples of 16 bytes).
  * Optionally, nh_vmac_nhbytes can be defined (for multiples of
  * VMAC_NHBYTES), and nh_16_2 and nh_vmac_nhbytes_2 (versions that do two
  * NH computations at once).
  */
 #ifdef CONFIG_64BIT
 #define nh_16(mp, kp, nw, rh, rl)					\
 	do {								\
 		int i; u64 th, tl;					\
 		rh = rl = 0;						\
 		for (i = 0; i < nw; i += 2) {				\
 			MUL64(th, tl, pe64_to_cpup((mp)+i)+(kp)[i],	\
 				pe64_to_cpup((mp)+i+1)+(kp)[i+1]);	\
 			ADD128(rh, rl, th, tl);				\
 		}							\
 	} while (0)
 #define nh_16_2(mp, kp, nw, rh, rl, rh1, rl1)				\
 	do {								\
 		int i; u64 th, tl;					\
 		rh1 = rl1 = rh = rl = 0;				\
 		for (i = 0; i < nw; i += 2) {				\
 			MUL64(th, tl, pe64_to_cpup((mp)+i)+(kp)[i],	\
 				pe64_to_cpup((mp)+i+1)+(kp)[i+1]);	\
 			ADD128(rh, rl, th, tl);				\
 			MUL64(th, tl, pe64_to_cpup((mp)+i)+(kp)[i+2],	\
 				pe64_to_cpup((mp)+i+1)+(kp)[i+3]);	\
 			ADD128(rh1, rl1, th, tl);			\
 		}							\
 	} while (0)
 #if (VMAC_NHBYTES >= 64) /* These versions do 64-bytes of message at a time */
 #define nh_vmac_nhbytes(mp, kp, nw, rh, rl)				\
 	do {								\
 		int i; u64 th, tl;					\
 		rh = rl = 0;						\
 		for (i = 0; i < nw; i += 8) {				\
 			MUL64(th, tl, pe64_to_cpup((mp)+i)+(kp)[i],	\
 				pe64_to_cpup((mp)+i+1)+(kp)[i+1]);	\
 			ADD128(rh, rl, th, tl);				\
 			MUL64(th, tl, pe64_to_cpup((mp)+i+2)+(kp)[i+2],	\
 				pe64_to_cpup((mp)+i+3)+(kp)[i+3]);	\
 			ADD128(rh, rl, th, tl);				\
 			MUL64(th, tl, pe64_to_cpup((mp)+i+4)+(kp)[i+4],	\
 				pe64_to_cpup((mp)+i+5)+(kp)[i+5]);	\
 			ADD128(rh, rl, th, tl);				\
 			MUL64(th, tl, pe64_to_cpup((mp)+i+6)+(kp)[i+6],	\
 				pe64_to_cpup((mp)+i+7)+(kp)[i+7]);	\
 			ADD128(rh, rl, th, tl);				\
 		}							\
 	} while (0)
 #define nh_vmac_nhbytes_2(mp, kp, nw, rh, rl, rh1, rl1)			\
 	do {								\
 		int i; u64 th, tl;					\
 		rh1 = rl1 = rh = rl = 0;				\
 		for (i = 0; i < nw; i += 8) {				\
 			MUL64(th, tl, pe64_to_cpup((mp)+i)+(kp)[i],	\
 				pe64_to_cpup((mp)+i+1)+(kp)[i+1]);	\
 			ADD128(rh, rl, th, tl);				\
 			MUL64(th, tl, pe64_to_cpup((mp)+i)+(kp)[i+2],	\
 				pe64_to_cpup((mp)+i+1)+(kp)[i+3]);	\
 			ADD128(rh1, rl1, th, tl);			\
 			MUL64(th, tl, pe64_to_cpup((mp)+i+2)+(kp)[i+2],	\
 				pe64_to_cpup((mp)+i+3)+(kp)[i+3]);	\
 			ADD128(rh, rl, th, tl);				\
 			MUL64(th, tl, pe64_to_cpup((mp)+i+2)+(kp)[i+4],	\
 				pe64_to_cpup((mp)+i+3)+(kp)[i+5]);	\
 			ADD128(rh1, rl1, th, tl);			\
 			MUL64(th, tl, pe64_to_cpup((mp)+i+4)+(kp)[i+4],	\
 				pe64_to_cpup((mp)+i+5)+(kp)[i+5]);	\
 			ADD128(rh, rl, th, tl);				\
 			MUL64(th, tl, pe64_to_cpup((mp)+i+4)+(kp)[i+6],	\
 				pe64_to_cpup((mp)+i+5)+(kp)[i+7]);	\
 			ADD128(rh1, rl1, th, tl);			\
 			MUL64(th, tl, pe64_to_cpup((mp)+i+6)+(kp)[i+6],	\
 				pe64_to_cpup((mp)+i+7)+(kp)[i+7]);	\
 			ADD128(rh, rl, th, tl);				\
 			MUL64(th, tl, pe64_to_cpup((mp)+i+6)+(kp)[i+8],	\
 				pe64_to_cpup((mp)+i+7)+(kp)[i+9]);	\
 			ADD128(rh1, rl1, th, tl);			\
 		}							\
 	} while (0)
 #endif
 #define poly_step(ah, al, kh, kl, mh, ml)				\
 	do {								\
 		u64 t1h, t1l, t2h, t2l, t3h, t3l, z = 0;		\
 		/* compute ab*cd, put bd into result registers */	\
 		PMUL64(t3h, t3l, al, kh);				\
 		PMUL64(t2h, t2l, ah, kl);				\
 		PMUL64(t1h, t1l, ah, 2*kh);				\
 		PMUL64(ah, al, al, kl);					\
 		/* add 2 * ac to result */				\
 		ADD128(ah, al, t1h, t1l);				\
 		/* add together ad + bc */				\
 		ADD128(t2h, t2l, t3h, t3l);				\
 		/* now (ah,al), (t2l,2*t2h) need summing */		\
 		/* first add the high registers, carrying into t2h */	\
 		ADD128(t2h, ah, z, t2l);				\
 		/* double t2h and add top bit of ah */			\
 		t2h = 2 * t2h + (ah >> 63);				\
 		ah &= m63;						\
 		/* now add the low registers */				\
 		ADD128(ah, al, mh, ml);					\
 		ADD128(ah, al, z, t2h);					\
 	} while (0)
 #else /* ! CONFIG_64BIT */
 #ifndef nh_16
 #define nh_16(mp, kp, nw, rh, rl)					\
 	do {								\
 		u64 t1, t2, m1, m2, t;					\
 		int i;							\
 		rh = rl = t = 0;					\
 		for (i = 0; i < nw; i += 2)  {				\
 			t1 = pe64_to_cpup(mp+i) + kp[i];		\
 			t2 = pe64_to_cpup(mp+i+1) + kp[i+1];		\
 			m2 = MUL32(t1 >> 32, t2);			\
 			m1 = MUL32(t1, t2 >> 32);			\
 			ADD128(rh, rl, MUL32(t1 >> 32, t2 >> 32),	\
 				MUL32(t1, t2));				\
 			rh += (u64)(u32)(m1 >> 32)			\
 				+ (u32)(m2 >> 32);			\
 			t += (u64)(u32)m1 + (u32)m2;			\
 		}							\
 		ADD128(rh, rl, (t >> 32), (t << 32));			\
 	} while (0)
 #endif
 static void poly_step_func(u64 *ahi, u64 *alo,
 			const u64 *kh, const u64 *kl,
 			const u64 *mh, const u64 *ml)
 {
 #define a0 (*(((u32 *)alo)+INDEX_LOW))
 #define a1 (*(((u32 *)alo)+INDEX_HIGH))
 #define a2 (*(((u32 *)ahi)+INDEX_LOW))
 #define a3 (*(((u32 *)ahi)+INDEX_HIGH))
 #define k0 (*(((u32 *)kl)+INDEX_LOW))
 #define k1 (*(((u32 *)kl)+INDEX_HIGH))
 #define k2 (*(((u32 *)kh)+INDEX_LOW))
 #define k3 (*(((u32 *)kh)+INDEX_HIGH))
 	u64 p, q, t;
 	u32 t2;
 	p = MUL32(a3, k3);
 	p += p;
 	p += *(u64 *)mh;
 	p += MUL32(a0, k2);
 	p += MUL32(a1, k1);
 	p += MUL32(a2, k0);
 	t = (u32)(p);
 	p >>= 32;
 	p += MUL32(a0, k3);
 	p += MUL32(a1, k2);
 	p += MUL32(a2, k1);
 	p += MUL32(a3, k0);
 	t |= ((u64)((u32)p & 0x7fffffff)) << 32;
 	p >>= 31;
 	p += (u64)(((u32 *)ml)[INDEX_LOW]);
 	p += MUL32(a0, k0);
 	q =  MUL32(a1, k3);
 	q += MUL32(a2, k2);
 	q += MUL32(a3, k1);
 	q += q;
 	p += q;
 	t2 = (u32)(p);
 	p >>= 32;
 	p += (u64)(((u32 *)ml)[INDEX_HIGH]);
 	p += MUL32(a0, k1);
 	p += MUL32(a1, k0);
 	q =  MUL32(a2, k3);
 	q += MUL32(a3, k2);
 	q += q;
 	p += q;
 	*(u64 *)(alo) = (p << 32) | t2;
 	p >>= 32;
 	*(u64 *)(ahi) = p + t;
 #undef a0
 #undef a1
 #undef a2
 #undef a3
 #undef k0
 #undef k1
 #undef k2
 #undef k3
 }
 #define poly_step(ah, al, kh, kl, mh, ml)				\
 	poly_step_func(&(ah), &(al), &(kh), &(kl), &(mh), &(ml))
 #endif  /* end of specialized NH and poly definitions */
 /* At least nh_16 is defined. Defined others as needed here */
 #ifndef nh_16_2
 #define nh_16_2(mp, kp, nw, rh, rl, rh2, rl2)				\
 	do { 								\
 		nh_16(mp, kp, nw, rh, rl);				\
 		nh_16(mp, ((kp)+2), nw, rh2, rl2);			\
 	} while (0)
 #endif
 #ifndef nh_vmac_nhbytes
 #define nh_vmac_nhbytes(mp, kp, nw, rh, rl)				\
 	nh_16(mp, kp, nw, rh, rl)
 #endif
 #ifndef nh_vmac_nhbytes_2
 #define nh_vmac_nhbytes_2(mp, kp, nw, rh, rl, rh2, rl2)			\
 	do {								\
 		nh_vmac_nhbytes(mp, kp, nw, rh, rl);			\
 		nh_vmac_nhbytes(mp, ((kp)+2), nw, rh2, rl2);		\
 	} while (0)
 #endif
 static void vhash_abort(struct vmac_ctx *ctx)
 {
 	ctx->polytmp[0] = ctx->polykey[0] ;
 	ctx->polytmp[1] = ctx->polykey[1] ;
 	ctx->first_block_processed = 0;
 }
 static u64 l3hash(u64 p1, u64 p2, u64 k1, u64 k2, u64 len)
 {
 	u64 rh, rl, t, z = 0;
 	/* fully reduce (p1,p2)+(len,0) mod p127 */
 	t = p1 >> 63;
 	p1 &= m63;
 	ADD128(p1, p2, len, t);
 	/* At this point, (p1,p2) is at most 2^127+(len<<64) */
 	t = (p1 > m63) + ((p1 == m63) && (p2 == m64));
 	ADD128(p1, p2, z, t);
 	p1 &= m63;
 	/* compute (p1,p2)/(2^64-2^32) and (p1,p2)%(2^64-2^32) */
 	t = p1 + (p2 >> 32);
 	t += (t >> 32);
 	t += (u32)t > 0xfffffffeu;
 	p1 += (t >> 32);
 	p2 += (p1 << 32);
 	/* compute (p1+k1)%p64 and (p2+k2)%p64 */
 	p1 += k1;
 	p1 += (0 - (p1 < k1)) & 257;
 	p2 += k2;
 	p2 += (0 - (p2 < k2)) & 257;
 	/* compute (p1+k1)*(p2+k2)%p64 */
 	MUL64(rh, rl, p1, p2);
 	t = rh >> 56;
 	ADD128(t, rl, z, rh);
 	rh <<= 8;
 	ADD128(t, rl, z, rh);
 	t += t << 8;
 	rl += t;
 	rl += (0 - (rl < t)) & 257;
 	rl += (0 - (rl > p64-1)) & 257;
 	return rl;
 }
 static void vhash_update(const unsigned char *m,
 			unsigned int mbytes, /* Pos multiple of VMAC_NHBYTES */
 			struct vmac_ctx *ctx)
 {
 	u64 rh, rl, *mptr;
 	const u64 *kptr = (u64 *)ctx->nhkey;
 	int i;
 	u64 ch, cl;
 	u64 pkh = ctx->polykey[0];
 	u64 pkl = ctx->polykey[1];
 	if (!mbytes)
 		return;
 	BUG_ON(mbytes % VMAC_NHBYTES);
 	mptr = (u64 *)m;
 	i = mbytes / VMAC_NHBYTES;  /* Must be non-zero */
 	ch = ctx->polytmp[0];
 	cl = ctx->polytmp[1];
 	if (!ctx->first_block_processed) {
 		ctx->first_block_processed = 1;
 		nh_vmac_nhbytes(mptr, kptr, VMAC_NHBYTES/8, rh, rl);
 		rh &= m62;
 		ADD128(ch, cl, rh, rl);
 		mptr += (VMAC_NHBYTES/sizeof(u64));
 		i--;
 	}
 	while (i--) {
 		nh_vmac_nhbytes(mptr, kptr, VMAC_NHBYTES/8, rh, rl);
 		rh &= m62;
 		poly_step(ch, cl, pkh, pkl, rh, rl);
 		mptr += (VMAC_NHBYTES/sizeof(u64));
 	}
 	ctx->polytmp[0] = ch;
 	ctx->polytmp[1] = cl;
 }
 static u64 vhash(unsigned char m[], unsigned int mbytes,
 			u64 *tagl, struct vmac_ctx *ctx)
 {
 	u64 rh, rl, *mptr;
 	const u64 *kptr = (u64 *)ctx->nhkey;
 	int i, remaining;
 	u64 ch, cl;
 	u64 pkh = ctx->polykey[0];
 	u64 pkl = ctx->polykey[1];
 	mptr = (u64 *)m;
 	i = mbytes / VMAC_NHBYTES;
 	remaining = mbytes % VMAC_NHBYTES;
 	if (ctx->first_block_processed) {
 		ch = ctx->polytmp[0];
 		cl = ctx->polytmp[1];
 	} else if (i) {
 		nh_vmac_nhbytes(mptr, kptr, VMAC_NHBYTES/8, ch, cl);
 		ch &= m62;
 		ADD128(ch, cl, pkh, pkl);
 		mptr += (VMAC_NHBYTES/sizeof(u64));
 		i--;
 	} else if (remaining) {
 		nh_16(mptr, kptr, 2*((remaining+15)/16), ch, cl);
 		ch &= m62;
 		ADD128(ch, cl, pkh, pkl);
 		mptr += (VMAC_NHBYTES/sizeof(u64));
 		goto do_l3;
 	} else {/* Empty String */
 		ch = pkh; cl = pkl;
 		goto do_l3;
 	}
 	while (i--) {
 		nh_vmac_nhbytes(mptr, kptr, VMAC_NHBYTES/8, rh, rl);
 		rh &= m62;
 		poly_step(ch, cl, pkh, pkl, rh, rl);
 		mptr += (VMAC_NHBYTES/sizeof(u64));
 	}
 	if (remaining) {
 		nh_16(mptr, kptr, 2*((remaining+15)/16), rh, rl);
 		rh &= m62;
 		poly_step(ch, cl, pkh, pkl, rh, rl);
 	}
 do_l3:
 	vhash_abort(ctx);
 	remaining *= 8;
 	return l3hash(ch, cl, ctx->l3key[0], ctx->l3key[1], remaining);
 }
 static u64 vmac(unsigned char m[], unsigned int mbytes,
 			const unsigned char n[16], u64 *tagl,
 			struct vmac_ctx_t *ctx)
 {
 	u64 *in_n, *out_p;
 	u64 p, h;
 	int i;
 	in_n = ctx->__vmac_ctx.cached_nonce;
 	out_p = ctx->__vmac_ctx.cached_aes;
 	i = n[15] & 1;
 	if ((*(u64 *)(n+8) != in_n[1]) || (*(u64 *)(n) != in_n[0])) {
 		in_n[0] = *(u64 *)(n);
 		in_n[1] = *(u64 *)(n+8);
 		((unsigned char *)in_n)[15] &= 0xFE;
 		crypto_cipher_encrypt_one(ctx->child,
 			(unsigned char *)out_p, (unsigned char *)in_n);
 		((unsigned char *)in_n)[15] |= (unsigned char)(1-i);
 	}
 	p = be64_to_cpup(out_p + i);
 	h = vhash(m, mbytes, (u64 *)0, &ctx->__vmac_ctx);
 	return le64_to_cpu(p + h);
 }
 static int vmac_set_key(unsigned char user_key[], struct vmac_ctx_t *ctx)
 {
 	u64 in[2] = {0}, out[2];
 	unsigned i;
 	int err = 0;
 	err = crypto_cipher_setkey(ctx->child, user_key, VMAC_KEY_LEN);
 	if (err)
 		return err;
 	/* Fill nh key */
 	((unsigned char *)in)[0] = 0x80;
 	for (i = 0; i < sizeof(ctx->__vmac_ctx.nhkey)/8; i += 2) {
 		crypto_cipher_encrypt_one(ctx->child,
 			(unsigned char *)out, (unsigned char *)in);
 		ctx->__vmac_ctx.nhkey[i] = be64_to_cpup(out);
 		ctx->__vmac_ctx.nhkey[i+1] = be64_to_cpup(out+1);
 		((unsigned char *)in)[15] += 1;
 	}
 	/* Fill poly key */
 	((unsigned char *)in)[0] = 0xC0;
 	in[1] = 0;
 	for (i = 0; i < sizeof(ctx->__vmac_ctx.polykey)/8; i += 2) {
 		crypto_cipher_encrypt_one(ctx->child,
 			(unsigned char *)out, (unsigned char *)in);
 		ctx->__vmac_ctx.polytmp[i] =
 			ctx->__vmac_ctx.polykey[i] =
 				be64_to_cpup(out) & mpoly;
 		ctx->__vmac_ctx.polytmp[i+1] =
 			ctx->__vmac_ctx.polykey[i+1] =
 				be64_to_cpup(out+1) & mpoly;
 		((unsigned char *)in)[15] += 1;
 	}
 	/* Fill ip key */
 	((unsigned char *)in)[0] = 0xE0;
 	in[1] = 0;
 	for (i = 0; i < sizeof(ctx->__vmac_ctx.l3key)/8; i += 2) {
 		do {
 			crypto_cipher_encrypt_one(ctx->child,
 				(unsigned char *)out, (unsigned char *)in);
 			ctx->__vmac_ctx.l3key[i] = be64_to_cpup(out);
 			ctx->__vmac_ctx.l3key[i+1] = be64_to_cpup(out+1);
 			((unsigned char *)in)[15] += 1;
 		} while (ctx->__vmac_ctx.l3key[i] >= p64
 			|| ctx->__vmac_ctx.l3key[i+1] >= p64);
 	}
 	/* Invalidate nonce/aes cache and reset other elements */
 	ctx->__vmac_ctx.cached_nonce[0] = (u64)-1; /* Ensure illegal nonce */
 	ctx->__vmac_ctx.cached_nonce[1] = (u64)0;  /* Ensure illegal nonce */
 	ctx->__vmac_ctx.first_block_processed = 0;
 	return err;
 }
 static int vmac_setkey(struct crypto_shash *parent,
 		const u8 *key, unsigned int keylen)
 {
 	struct vmac_ctx_t *ctx = crypto_shash_ctx(parent);
 	if (keylen != VMAC_KEY_LEN) {
 		crypto_shash_set_flags(parent, CRYPTO_TFM_RES_BAD_KEY_LEN);
 		return -EINVAL;
 	}
 	return vmac_set_key((u8 *)key, ctx);
 }
 static int vmac_init(struct shash_desc *pdesc)
 {
 	return 0;
 }
 static int vmac_update(struct shash_desc *pdesc, const u8 *p,
 		unsigned int len)
 {
 	struct crypto_shash *parent = pdesc->tfm;
 	struct vmac_ctx_t *ctx = crypto_shash_ctx(parent);
 	int expand;
 	int min;
 	expand = VMAC_NHBYTES - ctx->partial_size > 0 ?
 			VMAC_NHBYTES - ctx->partial_size : 0;
 	min = len < expand ? len : expand;
 	memcpy(ctx->partial + ctx->partial_size, p, min);
 	ctx->partial_size += min;
 	if (len < expand)
 		return 0;
 	vhash_update(ctx->partial, VMAC_NHBYTES, &ctx->__vmac_ctx);
 	ctx->partial_size = 0;
 	len -= expand;
 	p += expand;
 	if (len % VMAC_NHBYTES) {
 		memcpy(ctx->partial, p + len - (len % VMAC_NHBYTES),
 			len % VMAC_NHBYTES);
 		ctx->partial_size = len % VMAC_NHBYTES;
 	}
 	vhash_update(p, len - len % VMAC_NHBYTES, &ctx->__vmac_ctx);
 	return 0;
 }
 static int vmac_final(struct shash_desc *pdesc, u8 *out)
 {
 	struct crypto_shash *parent = pdesc->tfm;
 	struct vmac_ctx_t *ctx = crypto_shash_ctx(parent);
 	vmac_t mac;
 	u8 nonce[16] = {};
 	/* vmac() ends up accessing outside the array bounds that
 	 * we specify.  In appears to access up to the next 2-word
 	 * boundary.  We'll just be uber cautious and zero the
 	 * unwritten bytes in the buffer.
 	 */
 	if (ctx->partial_size) {
 		memset(ctx->partial + ctx->partial_size, 0,
 			VMAC_NHBYTES - ctx->partial_size);
 	}
 	mac = vmac(ctx->partial, ctx->partial_size, nonce, NULL, ctx);
 	memcpy(out, &mac, sizeof(vmac_t));
 	memzero_explicit(&mac, sizeof(vmac_t));
 	memset(&ctx->__vmac_ctx, 0, sizeof(struct vmac_ctx));
 	ctx->partial_size = 0;
 	return 0;
 }
 static int vmac_init_tfm(struct crypto_tfm *tfm)
 {
 	struct crypto_cipher *cipher;
 	struct crypto_instance *inst = (void *)tfm->__crt_alg;
 	struct crypto_spawn *spawn = crypto_instance_ctx(inst);
 	struct vmac_ctx_t *ctx = crypto_tfm_ctx(tfm);
 	cipher = crypto_spawn_cipher(spawn);
 	if (IS_ERR(cipher))
 		return PTR_ERR(cipher);
 	ctx->child = cipher;
 	return 0;
 }
 static void vmac_exit_tfm(struct crypto_tfm *tfm)
 {
 	struct vmac_ctx_t *ctx = crypto_tfm_ctx(tfm);
 	crypto_free_cipher(ctx->child);
 }
 static int vmac_create(struct crypto_template *tmpl, struct rtattr **tb)
 {
 	struct shash_instance *inst;
 	struct crypto_alg *alg;
 	int err;
 	err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_SHASH);
 	if (err)
 		return err;
 	alg = crypto_get_attr_alg(tb, CRYPTO_ALG_TYPE_CIPHER,
 			CRYPTO_ALG_TYPE_MASK);
 	if (IS_ERR(alg))
 		return PTR_ERR(alg);
 	inst = shash_alloc_instance("vmac", alg);
 	err = PTR_ERR(inst);
 	if (IS_ERR(inst))
 		goto out_put_alg;
 	err = crypto_init_spawn(shash_instance_ctx(inst), alg,
 			shash_crypto_instance(inst),
 			CRYPTO_ALG_TYPE_MASK);
 	if (err)
 		goto out_free_inst;
 	inst->alg.base.cra_priority = alg->cra_priority;
 	inst->alg.base.cra_blocksize = alg->cra_blocksize;
 	inst->alg.base.cra_alignmask = alg->cra_alignmask;
 	inst->alg.digestsize = sizeof(vmac_t);
 	inst->alg.base.cra_ctxsize = sizeof(struct vmac_ctx_t);
 	inst->alg.base.cra_init = vmac_init_tfm;
 	inst->alg.base.cra_exit = vmac_exit_tfm;
 	inst->alg.init = vmac_init;
 	inst->alg.update = vmac_update;
 	inst->alg.final = vmac_final;
 	inst->alg.setkey = vmac_setkey;
 	err = shash_register_instance(tmpl, inst);
 	if (err) {
 out_free_inst:
 		shash_free_instance(shash_crypto_instance(inst));
 	}
 out_put_alg:
 	crypto_mod_put(alg);
 	return err;
 }
 static struct crypto_template vmac_tmpl = {
 	.name = "vmac",
 	.create = vmac_create,
 	.free = shash_free_instance,
 	.module = THIS_MODULE,
 };
 static int __init vmac_module_init(void)
 {
 	return crypto_register_template(&vmac_tmpl);
 }
 static void __exit vmac_module_exit(void)
 {
 	crypto_unregister_template(&vmac_tmpl);
 }
 module_init(vmac_module_init);
 module_exit(vmac_module_exit);
 MODULE_LICENSE("GPL");
 MODULE_DESCRIPTION("VMAC hash algorithm");
+MODULE_ALIAS_CRYPTO("vmac");

crypto/xcbc.c

Diff comments View file @ acc5ccb

 /*
  * Copyright (C)2006 USAGI/WIDE Project
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
  * the Free Software Foundation; either version 2 of the License, or
  * (at your option) any later version.
  *
  * This program is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  *
  * Author:
  * 	Kazunori Miyazawa <miyazawa@linux-ipv6.org>
  */
 #include <crypto/internal/hash.h>
 #include <linux/err.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
 static u_int32_t ks[12] = {0x01010101, 0x01010101, 0x01010101, 0x01010101,
 			   0x02020202, 0x02020202, 0x02020202, 0x02020202,
 			   0x03030303, 0x03030303, 0x03030303, 0x03030303};
 /*
  * +------------------------
  * | <parent tfm>
  * +------------------------
  * | xcbc_tfm_ctx
  * +------------------------
  * | consts (block size * 2)
  * +------------------------
  */
 struct xcbc_tfm_ctx {
 	struct crypto_cipher *child;
 	u8 ctx[];
 };
 /*
  * +------------------------
  * | <shash desc>
  * +------------------------
  * | xcbc_desc_ctx
  * +------------------------
  * | odds (block size)
  * +------------------------
  * | prev (block size)
  * +------------------------
  */
 struct xcbc_desc_ctx {
 	unsigned int len;
 	u8 ctx[];
 };
 static int crypto_xcbc_digest_setkey(struct crypto_shash *parent,
 				     const u8 *inkey, unsigned int keylen)
 {
 	unsigned long alignmask = crypto_shash_alignmask(parent);
 	struct xcbc_tfm_ctx *ctx = crypto_shash_ctx(parent);
 	int bs = crypto_shash_blocksize(parent);
 	u8 *consts = PTR_ALIGN(&ctx->ctx[0], alignmask + 1);
 	int err = 0;
 	u8 key1[bs];
 	if ((err = crypto_cipher_setkey(ctx->child, inkey, keylen)))
 		return err;
 	crypto_cipher_encrypt_one(ctx->child, consts, (u8 *)ks + bs);
 	crypto_cipher_encrypt_one(ctx->child, consts + bs, (u8 *)ks + bs * 2);
 	crypto_cipher_encrypt_one(ctx->child, key1, (u8 *)ks);
 	return crypto_cipher_setkey(ctx->child, key1, bs);
 }
 static int crypto_xcbc_digest_init(struct shash_desc *pdesc)
 {
 	unsigned long alignmask = crypto_shash_alignmask(pdesc->tfm);
 	struct xcbc_desc_ctx *ctx = shash_desc_ctx(pdesc);
 	int bs = crypto_shash_blocksize(pdesc->tfm);
 	u8 *prev = PTR_ALIGN(&ctx->ctx[0], alignmask + 1) + bs;
 	ctx->len = 0;
 	memset(prev, 0, bs);
 	return 0;
 }
 static int crypto_xcbc_digest_update(struct shash_desc *pdesc, const u8 *p,
 				     unsigned int len)
 {
 	struct crypto_shash *parent = pdesc->tfm;
 	unsigned long alignmask = crypto_shash_alignmask(parent);
 	struct xcbc_tfm_ctx *tctx = crypto_shash_ctx(parent);
 	struct xcbc_desc_ctx *ctx = shash_desc_ctx(pdesc);
 	struct crypto_cipher *tfm = tctx->child;
 	int bs = crypto_shash_blocksize(parent);
 	u8 *odds = PTR_ALIGN(&ctx->ctx[0], alignmask + 1);
 	u8 *prev = odds + bs;
 	/* checking the data can fill the block */
 	if ((ctx->len + len) <= bs) {
 		memcpy(odds + ctx->len, p, len);
 		ctx->len += len;
 		return 0;
 	}
 	/* filling odds with new data and encrypting it */
 	memcpy(odds + ctx->len, p, bs - ctx->len);
 	len -= bs - ctx->len;
 	p += bs - ctx->len;
 	crypto_xor(prev, odds, bs);
 	crypto_cipher_encrypt_one(tfm, prev, prev);
 	/* clearing the length */
 	ctx->len = 0;
 	/* encrypting the rest of data */
 	while (len > bs) {
 		crypto_xor(prev, p, bs);
 		crypto_cipher_encrypt_one(tfm, prev, prev);
 		p += bs;
 		len -= bs;
 	}
 	/* keeping the surplus of blocksize */
 	if (len) {
 		memcpy(odds, p, len);
 		ctx->len = len;
 	}
 	return 0;
 }
 static int crypto_xcbc_digest_final(struct shash_desc *pdesc, u8 *out)
 {
 	struct crypto_shash *parent = pdesc->tfm;
 	unsigned long alignmask = crypto_shash_alignmask(parent);
 	struct xcbc_tfm_ctx *tctx = crypto_shash_ctx(parent);
 	struct xcbc_desc_ctx *ctx = shash_desc_ctx(pdesc);
 	struct crypto_cipher *tfm = tctx->child;
 	int bs = crypto_shash_blocksize(parent);
 	u8 *consts = PTR_ALIGN(&tctx->ctx[0], alignmask + 1);
 	u8 *odds = PTR_ALIGN(&ctx->ctx[0], alignmask + 1);
 	u8 *prev = odds + bs;
 	unsigned int offset = 0;
 	if (ctx->len != bs) {
 		unsigned int rlen;
 		u8 *p = odds + ctx->len;
 		*p = 0x80;
 		p++;
 		rlen = bs - ctx->len -1;
 		if (rlen)
 			memset(p, 0, rlen);
 		offset += bs;
 	}
 	crypto_xor(prev, odds, bs);
 	crypto_xor(prev, consts + offset, bs);
 	crypto_cipher_encrypt_one(tfm, out, prev);
 	return 0;
 }
 static int xcbc_init_tfm(struct crypto_tfm *tfm)
 {
 	struct crypto_cipher *cipher;
 	struct crypto_instance *inst = (void *)tfm->__crt_alg;
 	struct crypto_spawn *spawn = crypto_instance_ctx(inst);
 	struct xcbc_tfm_ctx *ctx = crypto_tfm_ctx(tfm);
 	cipher = crypto_spawn_cipher(spawn);
 	if (IS_ERR(cipher))
 		return PTR_ERR(cipher);
 	ctx->child = cipher;
 	return 0;
 };
 static void xcbc_exit_tfm(struct crypto_tfm *tfm)
 {
 	struct xcbc_tfm_ctx *ctx = crypto_tfm_ctx(tfm);
 	crypto_free_cipher(ctx->child);
 }
 static int xcbc_create(struct crypto_template *tmpl, struct rtattr **tb)
 {
 	struct shash_instance *inst;
 	struct crypto_alg *alg;
 	unsigned long alignmask;
 	int err;
 	err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_SHASH);
 	if (err)
 		return err;
 	alg = crypto_get_attr_alg(tb, CRYPTO_ALG_TYPE_CIPHER,
 				  CRYPTO_ALG_TYPE_MASK);
 	if (IS_ERR(alg))
 		return PTR_ERR(alg);
 	switch(alg->cra_blocksize) {
 	case 16:
 		break;
 	default:
 		goto out_put_alg;
 	}
 	inst = shash_alloc_instance("xcbc", alg);
 	err = PTR_ERR(inst);
 	if (IS_ERR(inst))
 		goto out_put_alg;
 	err = crypto_init_spawn(shash_instance_ctx(inst), alg,
 				shash_crypto_instance(inst),
 				CRYPTO_ALG_TYPE_MASK);
 	if (err)
 		goto out_free_inst;
 	alignmask = alg->cra_alignmask | 3;
 	inst->alg.base.cra_alignmask = alignmask;
 	inst->alg.base.cra_priority = alg->cra_priority;
 	inst->alg.base.cra_blocksize = alg->cra_blocksize;
 	inst->alg.digestsize = alg->cra_blocksize;
 	inst->alg.descsize = ALIGN(sizeof(struct xcbc_desc_ctx),
 				   crypto_tfm_ctx_alignment()) +
 			     (alignmask &
 			      ~(crypto_tfm_ctx_alignment() - 1)) +
 			     alg->cra_blocksize * 2;
 	inst->alg.base.cra_ctxsize = ALIGN(sizeof(struct xcbc_tfm_ctx),
 					   alignmask + 1) +
 				     alg->cra_blocksize * 2;
 	inst->alg.base.cra_init = xcbc_init_tfm;
 	inst->alg.base.cra_exit = xcbc_exit_tfm;
 	inst->alg.init = crypto_xcbc_digest_init;
 	inst->alg.update = crypto_xcbc_digest_update;
 	inst->alg.final = crypto_xcbc_digest_final;
 	inst->alg.setkey = crypto_xcbc_digest_setkey;
 	err = shash_register_instance(tmpl, inst);
 	if (err) {
 out_free_inst:
 		shash_free_instance(shash_crypto_instance(inst));
 	}
 out_put_alg:
 	crypto_mod_put(alg);
 	return err;
 }
 static struct crypto_template crypto_xcbc_tmpl = {
 	.name = "xcbc",
 	.create = xcbc_create,
 	.free = shash_free_instance,
 	.module = THIS_MODULE,
 };
 static int __init crypto_xcbc_module_init(void)
 {
 	return crypto_register_template(&crypto_xcbc_tmpl);
 }
 static void __exit crypto_xcbc_module_exit(void)
 {
 	crypto_unregister_template(&crypto_xcbc_tmpl);
 }
 module_init(crypto_xcbc_module_init);
 module_exit(crypto_xcbc_module_exit);
 MODULE_LICENSE("GPL");
 MODULE_DESCRIPTION("XCBC keyed hash algorithm");
+MODULE_ALIAS_CRYPTO("xcbc");

crypto/xts.c

Diff comments View file @ acc5ccb

 /* XTS: as defined in IEEE1619/D16
  *	http://grouper.ieee.org/groups/1619/email/pdf00086.pdf
  *	(sector sizes which are not a multiple of 16 bytes are,
  *	however currently unsupported)
  *
  * Copyright (c) 2007 Rik Snel <rsnel@cube.dyndns.org>
  *
  * Based om ecb.c
  * Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au>
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the Free
  * Software Foundation; either version 2 of the License, or (at your option)
  * any later version.
  */
 #include <crypto/algapi.h>
 #include <linux/err.h>
 #include <linux/init.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/scatterlist.h>
 #include <linux/slab.h>
 #include <crypto/xts.h>
 #include <crypto/b128ops.h>
 #include <crypto/gf128mul.h>
 struct priv {
 	struct crypto_cipher *child;
 	struct crypto_cipher *tweak;
 };
 static int setkey(struct crypto_tfm *parent, const u8 *key,
 		  unsigned int keylen)
 {
 	struct priv *ctx = crypto_tfm_ctx(parent);
 	struct crypto_cipher *child = ctx->tweak;
 	u32 *flags = &parent->crt_flags;
 	int err;
 	/* key consists of keys of equal size concatenated, therefore
 	 * the length must be even */
 	if (keylen % 2) {
 		/* tell the user why there was an error */
 		*flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
 		return -EINVAL;
 	}
 	/* we need two cipher instances: one to compute the initial 'tweak'
 	 * by encrypting the IV (usually the 'plain' iv) and the other
 	 * one to encrypt and decrypt the data */
 	/* tweak cipher, uses Key2 i.e. the second half of *key */
 	crypto_cipher_clear_flags(child, CRYPTO_TFM_REQ_MASK);
 	crypto_cipher_set_flags(child, crypto_tfm_get_flags(parent) &
 				       CRYPTO_TFM_REQ_MASK);
 	err = crypto_cipher_setkey(child, key + keylen/2, keylen/2);
 	if (err)
 		return err;
 	crypto_tfm_set_flags(parent, crypto_cipher_get_flags(child) &
 				     CRYPTO_TFM_RES_MASK);
 	child = ctx->child;
 	/* data cipher, uses Key1 i.e. the first half of *key */
 	crypto_cipher_clear_flags(child, CRYPTO_TFM_REQ_MASK);
 	crypto_cipher_set_flags(child, crypto_tfm_get_flags(parent) &
 				       CRYPTO_TFM_REQ_MASK);
 	err = crypto_cipher_setkey(child, key, keylen/2);
 	if (err)
 		return err;
 	crypto_tfm_set_flags(parent, crypto_cipher_get_flags(child) &
 				     CRYPTO_TFM_RES_MASK);
 	return 0;
 }
 struct sinfo {
 	be128 *t;
 	struct crypto_tfm *tfm;
 	void (*fn)(struct crypto_tfm *, u8 *, const u8 *);
 };
 static inline void xts_round(struct sinfo *s, void *dst, const void *src)
 {
 	be128_xor(dst, s->t, src);		/* PP <- T xor P */
 	s->fn(s->tfm, dst, dst);		/* CC <- E(Key1,PP) */
 	be128_xor(dst, dst, s->t);		/* C <- T xor CC */
 }
 static int crypt(struct blkcipher_desc *d,
 		 struct blkcipher_walk *w, struct priv *ctx,
 		 void (*tw)(struct crypto_tfm *, u8 *, const u8 *),
 		 void (*fn)(struct crypto_tfm *, u8 *, const u8 *))
 {
 	int err;
 	unsigned int avail;
 	const int bs = XTS_BLOCK_SIZE;
 	struct sinfo s = {
 		.tfm = crypto_cipher_tfm(ctx->child),
 		.fn = fn
 	};
 	u8 *wsrc;
 	u8 *wdst;
 	err = blkcipher_walk_virt(d, w);
 	if (!w->nbytes)
 		return err;
 	s.t = (be128 *)w->iv;
 	avail = w->nbytes;
 	wsrc = w->src.virt.addr;
 	wdst = w->dst.virt.addr;
 	/* calculate first value of T */
 	tw(crypto_cipher_tfm(ctx->tweak), w->iv, w->iv);
 	goto first;
 	for (;;) {
 		do {
 			gf128mul_x_ble(s.t, s.t);
 first:
 			xts_round(&s, wdst, wsrc);
 			wsrc += bs;
 			wdst += bs;
 		} while ((avail -= bs) >= bs);
 		err = blkcipher_walk_done(d, w, avail);
 		if (!w->nbytes)
 			break;
 		avail = w->nbytes;
 		wsrc = w->src.virt.addr;
 		wdst = w->dst.virt.addr;
 	}
 	return err;
 }
 static int encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 		   struct scatterlist *src, unsigned int nbytes)
 {
 	struct priv *ctx = crypto_blkcipher_ctx(desc->tfm);
 	struct blkcipher_walk w;
 	blkcipher_walk_init(&w, dst, src, nbytes);
 	return crypt(desc, &w, ctx, crypto_cipher_alg(ctx->tweak)->cia_encrypt,
 		     crypto_cipher_alg(ctx->child)->cia_encrypt);
 }
 static int decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 		   struct scatterlist *src, unsigned int nbytes)
 {
 	struct priv *ctx = crypto_blkcipher_ctx(desc->tfm);
 	struct blkcipher_walk w;
 	blkcipher_walk_init(&w, dst, src, nbytes);
 	return crypt(desc, &w, ctx, crypto_cipher_alg(ctx->tweak)->cia_encrypt,
 		     crypto_cipher_alg(ctx->child)->cia_decrypt);
 }
 int xts_crypt(struct blkcipher_desc *desc, struct scatterlist *sdst,
 	      struct scatterlist *ssrc, unsigned int nbytes,
 	      struct xts_crypt_req *req)
 {
 	const unsigned int bsize = XTS_BLOCK_SIZE;
 	const unsigned int max_blks = req->tbuflen / bsize;
 	struct blkcipher_walk walk;
 	unsigned int nblocks;
 	be128 *src, *dst, *t;
 	be128 *t_buf = req->tbuf;
 	int err, i;
 	BUG_ON(max_blks < 1);
 	blkcipher_walk_init(&walk, sdst, ssrc, nbytes);
 	err = blkcipher_walk_virt(desc, &walk);
 	nbytes = walk.nbytes;
 	if (!nbytes)
 		return err;
 	nblocks = min(nbytes / bsize, max_blks);
 	src = (be128 *)walk.src.virt.addr;
 	dst = (be128 *)walk.dst.virt.addr;
 	/* calculate first value of T */
 	req->tweak_fn(req->tweak_ctx, (u8 *)&t_buf[0], walk.iv);
 	i = 0;
 	goto first;
 	for (;;) {
 		do {
 			for (i = 0; i < nblocks; i++) {
 				gf128mul_x_ble(&t_buf[i], t);
 first:
 				t = &t_buf[i];
 				/* PP <- T xor P */
 				be128_xor(dst + i, t, src + i);
 			}
 			/* CC <- E(Key2,PP) */
 			req->crypt_fn(req->crypt_ctx, (u8 *)dst,
 				      nblocks * bsize);
 			/* C <- T xor CC */
 			for (i = 0; i < nblocks; i++)
 				be128_xor(dst + i, dst + i, &t_buf[i]);
 			src += nblocks;
 			dst += nblocks;
 			nbytes -= nblocks * bsize;
 			nblocks = min(nbytes / bsize, max_blks);
 		} while (nblocks > 0);
 		*(be128 *)walk.iv = *t;
 		err = blkcipher_walk_done(desc, &walk, nbytes);
 		nbytes = walk.nbytes;
 		if (!nbytes)
 			break;
 		nblocks = min(nbytes / bsize, max_blks);
 		src = (be128 *)walk.src.virt.addr;
 		dst = (be128 *)walk.dst.virt.addr;
 	}
 	return err;
 }
 EXPORT_SYMBOL_GPL(xts_crypt);
 static int init_tfm(struct crypto_tfm *tfm)
 {
 	struct crypto_cipher *cipher;
 	struct crypto_instance *inst = (void *)tfm->__crt_alg;
 	struct crypto_spawn *spawn = crypto_instance_ctx(inst);
 	struct priv *ctx = crypto_tfm_ctx(tfm);
 	u32 *flags = &tfm->crt_flags;
 	cipher = crypto_spawn_cipher(spawn);
 	if (IS_ERR(cipher))
 		return PTR_ERR(cipher);
 	if (crypto_cipher_blocksize(cipher) != XTS_BLOCK_SIZE) {
 		*flags |= CRYPTO_TFM_RES_BAD_BLOCK_LEN;
 		crypto_free_cipher(cipher);
 		return -EINVAL;
 	}
 	ctx->child = cipher;
 	cipher = crypto_spawn_cipher(spawn);
 	if (IS_ERR(cipher)) {
 		crypto_free_cipher(ctx->child);
 		return PTR_ERR(cipher);
 	}
 	/* this check isn't really needed, leave it here just in case */
 	if (crypto_cipher_blocksize(cipher) != XTS_BLOCK_SIZE) {
 		crypto_free_cipher(cipher);
 		crypto_free_cipher(ctx->child);
 		*flags |= CRYPTO_TFM_RES_BAD_BLOCK_LEN;
 		return -EINVAL;
 	}
 	ctx->tweak = cipher;
 	return 0;
 }
 static void exit_tfm(struct crypto_tfm *tfm)
 {
 	struct priv *ctx = crypto_tfm_ctx(tfm);
 	crypto_free_cipher(ctx->child);
 	crypto_free_cipher(ctx->tweak);
 }
 static struct crypto_instance *alloc(struct rtattr **tb)
 {
 	struct crypto_instance *inst;
 	struct crypto_alg *alg;
 	int err;
 	err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_BLKCIPHER);
 	if (err)
 		return ERR_PTR(err);
 	alg = crypto_get_attr_alg(tb, CRYPTO_ALG_TYPE_CIPHER,
 				  CRYPTO_ALG_TYPE_MASK);
 	if (IS_ERR(alg))
 		return ERR_CAST(alg);
 	inst = crypto_alloc_instance("xts", alg);
 	if (IS_ERR(inst))
 		goto out_put_alg;
 	inst->alg.cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER;
 	inst->alg.cra_priority = alg->cra_priority;
 	inst->alg.cra_blocksize = alg->cra_blocksize;
 	if (alg->cra_alignmask < 7)
 		inst->alg.cra_alignmask = 7;
 	else
 		inst->alg.cra_alignmask = alg->cra_alignmask;
 	inst->alg.cra_type = &crypto_blkcipher_type;
 	inst->alg.cra_blkcipher.ivsize = alg->cra_blocksize;
 	inst->alg.cra_blkcipher.min_keysize =
 		2 * alg->cra_cipher.cia_min_keysize;
 	inst->alg.cra_blkcipher.max_keysize =
 		2 * alg->cra_cipher.cia_max_keysize;
 	inst->alg.cra_ctxsize = sizeof(struct priv);
 	inst->alg.cra_init = init_tfm;
 	inst->alg.cra_exit = exit_tfm;
 	inst->alg.cra_blkcipher.setkey = setkey;
 	inst->alg.cra_blkcipher.encrypt = encrypt;
 	inst->alg.cra_blkcipher.decrypt = decrypt;
 out_put_alg:
 	crypto_mod_put(alg);
 	return inst;
 }
 static void free(struct crypto_instance *inst)
 {
 	crypto_drop_spawn(crypto_instance_ctx(inst));
 	kfree(inst);
 }
 static struct crypto_template crypto_tmpl = {
 	.name = "xts",
 	.alloc = alloc,
 	.free = free,
 	.module = THIS_MODULE,
 };
 static int __init crypto_module_init(void)
 {
 	return crypto_register_template(&crypto_tmpl);
 }
 static void __exit crypto_module_exit(void)
 {
 	crypto_unregister_template(&crypto_tmpl);
 }
 module_init(crypto_module_init);
 module_exit(crypto_module_exit);
 MODULE_LICENSE("GPL");
 MODULE_DESCRIPTION("XTS block cipher mode");
+MODULE_ALIAS_CRYPTO("xts");