Blame view
drivers/crypto/padlock-aes.c
15 KB
1da177e4c Linux-2.6.12-rc2 |
1 2 3 4 5 6 7 |
/* * Cryptographic API. * * Support for VIA PadLock hardware crypto engine. * * Copyright (c) 2004 Michal Ludvig <michal@logix.cz> * |
1da177e4c Linux-2.6.12-rc2 |
8 |
*/ |
28ce728a9 [CRYPTO] padlock:... |
9 |
#include <crypto/algapi.h> |
89e126543 [CRYPTO] aes: Mov... |
10 |
#include <crypto/aes.h> |
214930887 crypto: padlock -... |
11 |
#include <crypto/padlock.h> |
1da177e4c Linux-2.6.12-rc2 |
12 13 14 15 |
#include <linux/module.h> #include <linux/init.h> #include <linux/types.h> #include <linux/errno.h> |
1da177e4c Linux-2.6.12-rc2 |
16 |
#include <linux/interrupt.h> |
6789b2dc4 [PADLOCK] Move fa... |
17 |
#include <linux/kernel.h> |
420a4b20c crypto: padlock -... |
18 19 |
#include <linux/percpu.h> #include <linux/smp.h> |
5a0e3ad6a include cleanup: ... |
20 |
#include <linux/slab.h> |
1da177e4c Linux-2.6.12-rc2 |
21 |
#include <asm/byteorder.h> |
a76c1c23d crypto: padlock-a... |
22 |
#include <asm/processor.h> |
e49140120 crypto: padlock -... |
23 |
#include <asm/i387.h> |
1da177e4c Linux-2.6.12-rc2 |
24 |
|
8d8409f77 crypto: padlock-a... |
25 26 27 28 |
/* * Number of data blocks actually fetched for each xcrypt insn. * Processors with prefetch errata will fetch extra blocks. */ |
a76c1c23d crypto: padlock-a... |
29 |
static unsigned int ecb_fetch_blocks = 2; |
8d8409f77 crypto: padlock-a... |
30 |
#define MAX_ECB_FETCH_BLOCKS (8) |
a76c1c23d crypto: padlock-a... |
31 |
#define ecb_fetch_bytes (ecb_fetch_blocks * AES_BLOCK_SIZE) |
8d8409f77 crypto: padlock-a... |
32 33 34 |
static unsigned int cbc_fetch_blocks = 1; #define MAX_CBC_FETCH_BLOCKS (4) |
a76c1c23d crypto: padlock-a... |
35 |
#define cbc_fetch_bytes (cbc_fetch_blocks * AES_BLOCK_SIZE) |
ccc17c34d [CRYPTO] padlock:... |
36 37 38 39 40 41 42 43 44 45 |
/* Control word. */ struct cword { unsigned int __attribute__ ((__packed__)) rounds:4, algo:3, keygen:1, interm:1, encdec:1, ksize:2; } __attribute__ ((__aligned__(PADLOCK_ALIGNMENT))); |
cc08632f8 [CRYPTO] padlock:... |
46 47 |
/* Whenever making any changes to the following * structure *make sure* you keep E, d_data |
7dc748e4e [CRYPTO] padlock-... |
48 49 50 51 52 |
* and cword aligned on 16 Bytes boundaries and * the Hardware can access 16 * 16 bytes of E and d_data * (only the first 15 * 16 bytes matter but the HW reads * more). */ |
1da177e4c Linux-2.6.12-rc2 |
53 |
struct aes_ctx { |
7dc748e4e [CRYPTO] padlock-... |
54 55 56 57 |
u32 E[AES_MAX_KEYLENGTH_U32] __attribute__ ((__aligned__(PADLOCK_ALIGNMENT))); u32 d_data[AES_MAX_KEYLENGTH_U32] __attribute__ ((__aligned__(PADLOCK_ALIGNMENT))); |
6789b2dc4 [PADLOCK] Move fa... |
58 59 60 61 |
struct { struct cword encrypt; struct cword decrypt; } cword; |
82062c72c [CRYPTO] padlock:... |
62 |
u32 *D; |
1da177e4c Linux-2.6.12-rc2 |
63 |
}; |
390dfd95c percpu: make misc... |
64 |
static DEFINE_PER_CPU(struct cword *, paes_last_cword); |
420a4b20c crypto: padlock -... |
65 |
|
1da177e4c Linux-2.6.12-rc2 |
66 67 68 69 70 71 72 73 74 75 76 77 |
/* Tells whether the ACE is capable to generate the extended key for a given key_len. */ static inline int aes_hw_extkey_available(uint8_t key_len) { /* TODO: We should check the actual CPU model/stepping as it's possible that the capability will be added in the next CPU revisions. */ if (key_len == 16) return 1; return 0; } |
28ce728a9 [CRYPTO] padlock:... |
78 |
static inline struct aes_ctx *aes_ctx_common(void *ctx) |
6789b2dc4 [PADLOCK] Move fa... |
79 |
{ |
28ce728a9 [CRYPTO] padlock:... |
80 |
unsigned long addr = (unsigned long)ctx; |
f10b7897e [CRYPTO] api: Ali... |
81 82 83 84 |
unsigned long align = PADLOCK_ALIGNMENT; if (align <= crypto_tfm_ctx_alignment()) align = 1; |
6c2bb98bc [CRYPTO] all: Pas... |
85 |
return (struct aes_ctx *)ALIGN(addr, align); |
6789b2dc4 [PADLOCK] Move fa... |
86 |
} |
28ce728a9 [CRYPTO] padlock:... |
87 88 89 90 91 92 93 94 95 |
static inline struct aes_ctx *aes_ctx(struct crypto_tfm *tfm) { return aes_ctx_common(crypto_tfm_ctx(tfm)); } static inline struct aes_ctx *blk_aes_ctx(struct crypto_blkcipher *tfm) { return aes_ctx_common(crypto_blkcipher_ctx(tfm)); } |
6c2bb98bc [CRYPTO] all: Pas... |
96 |
static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key, |
560c06ae1 [CRYPTO] api: Get... |
97 |
unsigned int key_len) |
1da177e4c Linux-2.6.12-rc2 |
98 |
{ |
6c2bb98bc [CRYPTO] all: Pas... |
99 |
struct aes_ctx *ctx = aes_ctx(tfm); |
06ace7a9b [CRYPTO] Use stan... |
100 |
const __le32 *key = (const __le32 *)in_key; |
560c06ae1 [CRYPTO] api: Get... |
101 |
u32 *flags = &tfm->crt_flags; |
7dc748e4e [CRYPTO] padlock-... |
102 |
struct crypto_aes_ctx gen_aes; |
420a4b20c crypto: padlock -... |
103 |
int cpu; |
1da177e4c Linux-2.6.12-rc2 |
104 |
|
560c06ae1 [CRYPTO] api: Get... |
105 |
if (key_len % 8) { |
1da177e4c Linux-2.6.12-rc2 |
106 107 108 |
*flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; return -EINVAL; } |
6789b2dc4 [PADLOCK] Move fa... |
109 110 111 112 113 |
/* * If the hardware is capable of generating the extended key * itself we must supply the plain key for both encryption * and decryption. */ |
82062c72c [CRYPTO] padlock:... |
114 |
ctx->D = ctx->E; |
1da177e4c Linux-2.6.12-rc2 |
115 |
|
7dc748e4e [CRYPTO] padlock-... |
116 117 118 119 |
ctx->E[0] = le32_to_cpu(key[0]); ctx->E[1] = le32_to_cpu(key[1]); ctx->E[2] = le32_to_cpu(key[2]); ctx->E[3] = le32_to_cpu(key[3]); |
1da177e4c Linux-2.6.12-rc2 |
120 |
|
6789b2dc4 [PADLOCK] Move fa... |
121 122 123 124 125 126 127 128 |
/* Prepare control words. */ memset(&ctx->cword, 0, sizeof(ctx->cword)); ctx->cword.decrypt.encdec = 1; ctx->cword.encrypt.rounds = 10 + (key_len - 16) / 4; ctx->cword.decrypt.rounds = ctx->cword.encrypt.rounds; ctx->cword.encrypt.ksize = (key_len - 16) / 8; ctx->cword.decrypt.ksize = ctx->cword.encrypt.ksize; |
1da177e4c Linux-2.6.12-rc2 |
129 130 |
/* Don't generate extended keys if the hardware can do it. */ if (aes_hw_extkey_available(key_len)) |
420a4b20c crypto: padlock -... |
131 |
goto ok; |
1da177e4c Linux-2.6.12-rc2 |
132 |
|
6789b2dc4 [PADLOCK] Move fa... |
133 134 135 |
ctx->D = ctx->d_data; ctx->cword.encrypt.keygen = 1; ctx->cword.decrypt.keygen = 1; |
7dc748e4e [CRYPTO] padlock-... |
136 137 138 |
if (crypto_aes_expand_key(&gen_aes, in_key, key_len)) { *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; return -EINVAL; |
1da177e4c Linux-2.6.12-rc2 |
139 |
} |
7dc748e4e [CRYPTO] padlock-... |
140 141 |
memcpy(ctx->E, gen_aes.key_enc, AES_MAX_KEYLENGTH); memcpy(ctx->D, gen_aes.key_dec, AES_MAX_KEYLENGTH); |
420a4b20c crypto: padlock -... |
142 143 144 |
ok: for_each_online_cpu(cpu) |
390dfd95c percpu: make misc... |
145 146 147 |
if (&ctx->cword.encrypt == per_cpu(paes_last_cword, cpu) || &ctx->cword.decrypt == per_cpu(paes_last_cword, cpu)) per_cpu(paes_last_cword, cpu) = NULL; |
420a4b20c crypto: padlock -... |
148 |
|
1da177e4c Linux-2.6.12-rc2 |
149 150 151 152 |
return 0; } /* ====== Encryption/decryption routines ====== */ |
28e8c3ad9 [PADLOCK] Impleme... |
153 |
/* These are the real call to PadLock. */ |
420a4b20c crypto: padlock -... |
154 155 156 |
static inline void padlock_reset_key(struct cword *cword) { int cpu = raw_smp_processor_id(); |
390dfd95c percpu: make misc... |
157 |
if (cword != per_cpu(paes_last_cword, cpu)) |
d1c8b0a76 crypto: padlock -... |
158 |
#ifndef CONFIG_X86_64 |
420a4b20c crypto: padlock -... |
159 |
asm volatile ("pushfl; popfl"); |
d1c8b0a76 crypto: padlock -... |
160 161 162 |
#else asm volatile ("pushfq; popfq"); #endif |
420a4b20c crypto: padlock -... |
163 164 165 |
} static inline void padlock_store_cword(struct cword *cword) |
866cd902e [CRYPTO] padlock:... |
166 |
{ |
390dfd95c percpu: make misc... |
167 |
per_cpu(paes_last_cword, raw_smp_processor_id()) = cword; |
866cd902e [CRYPTO] padlock:... |
168 |
} |
e49140120 crypto: padlock -... |
169 170 171 172 173 |
/* * While the padlock instructions don't use FP/SSE registers, they * generate a spurious DNA fault when cr0.ts is '1'. These instructions * should be used only inside the irq_ts_save/restore() context */ |
8d8409f77 crypto: padlock-a... |
174 |
static inline void rep_xcrypt_ecb(const u8 *input, u8 *output, void *key, |
a76c1c23d crypto: padlock-a... |
175 |
struct cword *control_word, int count) |
d4a7dd8e6 [CRYPTO] padlock:... |
176 177 178 |
{ asm volatile (".byte 0xf3,0x0f,0xa7,0xc8" /* rep xcryptecb */ : "+S"(input), "+D"(output) |
a76c1c23d crypto: padlock-a... |
179 |
: "d"(control_word), "b"(key), "c"(count)); |
d4a7dd8e6 [CRYPTO] padlock:... |
180 |
} |
8d8409f77 crypto: padlock-a... |
181 182 183 184 185 186 187 188 189 190 |
static inline u8 *rep_xcrypt_cbc(const u8 *input, u8 *output, void *key, u8 *iv, struct cword *control_word, int count) { asm volatile (".byte 0xf3,0x0f,0xa7,0xd0" /* rep xcryptcbc */ : "+S" (input), "+D" (output), "+a" (iv) : "d" (control_word), "b" (key), "c" (count)); return iv; } static void ecb_crypt_copy(const u8 *in, u8 *out, u32 *key, |
a76c1c23d crypto: padlock-a... |
191 |
struct cword *cword, int count) |
d4a7dd8e6 [CRYPTO] padlock:... |
192 |
{ |
a76c1c23d crypto: padlock-a... |
193 194 195 196 |
/* * Padlock prefetches extra data so we must provide mapped input buffers. * Assume there are at least 16 bytes of stack already in use. */ |
8d8409f77 crypto: padlock-a... |
197 |
u8 buf[AES_BLOCK_SIZE * (MAX_ECB_FETCH_BLOCKS - 1) + PADLOCK_ALIGNMENT - 1]; |
490fe3f05 [CRYPTO] padlock:... |
198 |
u8 *tmp = PTR_ALIGN(&buf[0], PADLOCK_ALIGNMENT); |
d4a7dd8e6 [CRYPTO] padlock:... |
199 |
|
a76c1c23d crypto: padlock-a... |
200 |
memcpy(tmp, in, count * AES_BLOCK_SIZE); |
8d8409f77 crypto: padlock-a... |
201 |
rep_xcrypt_ecb(tmp, out, key, cword, count); |
d4a7dd8e6 [CRYPTO] padlock:... |
202 |
} |
8d8409f77 crypto: padlock-a... |
203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 |
static u8 *cbc_crypt_copy(const u8 *in, u8 *out, u32 *key, u8 *iv, struct cword *cword, int count) { /* * Padlock prefetches extra data so we must provide mapped input buffers. * Assume there are at least 16 bytes of stack already in use. */ u8 buf[AES_BLOCK_SIZE * (MAX_CBC_FETCH_BLOCKS - 1) + PADLOCK_ALIGNMENT - 1]; u8 *tmp = PTR_ALIGN(&buf[0], PADLOCK_ALIGNMENT); memcpy(tmp, in, count * AES_BLOCK_SIZE); return rep_xcrypt_cbc(tmp, out, key, iv, cword, count); } static inline void ecb_crypt(const u8 *in, u8 *out, u32 *key, |
a76c1c23d crypto: padlock-a... |
218 |
struct cword *cword, int count) |
d4a7dd8e6 [CRYPTO] padlock:... |
219 |
{ |
a76c1c23d crypto: padlock-a... |
220 221 222 |
/* Padlock in ECB mode fetches at least ecb_fetch_bytes of data. * We could avoid some copying here but it's probably not worth it. */ |
e8edb3cbd crypto: padlock-a... |
223 |
if (unlikely(((unsigned long)in & ~PAGE_MASK) + ecb_fetch_bytes > PAGE_SIZE)) { |
8d8409f77 crypto: padlock-a... |
224 |
ecb_crypt_copy(in, out, key, cword, count); |
d4a7dd8e6 [CRYPTO] padlock:... |
225 226 |
return; } |
8d8409f77 crypto: padlock-a... |
227 228 229 230 231 232 233 |
rep_xcrypt_ecb(in, out, key, cword, count); } static inline u8 *cbc_crypt(const u8 *in, u8 *out, u32 *key, u8 *iv, struct cword *cword, int count) { /* Padlock in CBC mode fetches at least cbc_fetch_bytes of data. */ |
e8edb3cbd crypto: padlock-a... |
234 |
if (unlikely(((unsigned long)in & ~PAGE_MASK) + cbc_fetch_bytes > PAGE_SIZE)) |
8d8409f77 crypto: padlock-a... |
235 236 237 |
return cbc_crypt_copy(in, out, key, iv, cword, count); return rep_xcrypt_cbc(in, out, key, iv, cword, count); |
d4a7dd8e6 [CRYPTO] padlock:... |
238 |
} |
6789b2dc4 [PADLOCK] Move fa... |
239 240 |
static inline void padlock_xcrypt_ecb(const u8 *input, u8 *output, void *key, void *control_word, u32 count) |
1da177e4c Linux-2.6.12-rc2 |
241 |
{ |
a76c1c23d crypto: padlock-a... |
242 243 244 |
u32 initial = count & (ecb_fetch_blocks - 1); if (count < ecb_fetch_blocks) { |
8d8409f77 crypto: padlock-a... |
245 |
ecb_crypt(input, output, key, control_word, count); |
d4a7dd8e6 [CRYPTO] padlock:... |
246 247 |
return; } |
a76c1c23d crypto: padlock-a... |
248 249 250 251 252 253 |
if (initial) asm volatile (".byte 0xf3,0x0f,0xa7,0xc8" /* rep xcryptecb */ : "+S"(input), "+D"(output) : "d"(control_word), "b"(key), "c"(initial)); asm volatile (".byte 0xf3,0x0f,0xa7,0xc8" /* rep xcryptecb */ |
1da177e4c Linux-2.6.12-rc2 |
254 |
: "+S"(input), "+D"(output) |
a76c1c23d crypto: padlock-a... |
255 |
: "d"(control_word), "b"(key), "c"(count - initial)); |
1da177e4c Linux-2.6.12-rc2 |
256 |
} |
476df259c [CRYPTO] Update I... |
257 258 |
static inline u8 *padlock_xcrypt_cbc(const u8 *input, u8 *output, void *key, u8 *iv, void *control_word, u32 count) |
28e8c3ad9 [PADLOCK] Impleme... |
259 |
{ |
8d8409f77 crypto: padlock-a... |
260 261 262 263 264 265 266 267 |
u32 initial = count & (cbc_fetch_blocks - 1); if (count < cbc_fetch_blocks) return cbc_crypt(input, output, key, iv, control_word, count); if (initial) asm volatile (".byte 0xf3,0x0f,0xa7,0xd0" /* rep xcryptcbc */ : "+S" (input), "+D" (output), "+a" (iv) |
c054a076a crypto: padlock -... |
268 |
: "d" (control_word), "b" (key), "c" (initial)); |
8d8409f77 crypto: padlock-a... |
269 270 |
asm volatile (".byte 0xf3,0x0f,0xa7,0xd0" /* rep xcryptcbc */ |
28e8c3ad9 [PADLOCK] Impleme... |
271 |
: "+S" (input), "+D" (output), "+a" (iv) |
8d8409f77 crypto: padlock-a... |
272 |
: "d" (control_word), "b" (key), "c" (count-initial)); |
476df259c [CRYPTO] Update I... |
273 |
return iv; |
28e8c3ad9 [PADLOCK] Impleme... |
274 |
} |
6c2bb98bc [CRYPTO] all: Pas... |
275 |
static void aes_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in) |
1da177e4c Linux-2.6.12-rc2 |
276 |
{ |
6c2bb98bc [CRYPTO] all: Pas... |
277 |
struct aes_ctx *ctx = aes_ctx(tfm); |
e49140120 crypto: padlock -... |
278 |
int ts_state; |
e49140120 crypto: padlock -... |
279 |
|
420a4b20c crypto: padlock -... |
280 |
padlock_reset_key(&ctx->cword.encrypt); |
e49140120 crypto: padlock -... |
281 |
ts_state = irq_ts_save(); |
8d8409f77 crypto: padlock-a... |
282 |
ecb_crypt(in, out, ctx->E, &ctx->cword.encrypt, 1); |
e49140120 crypto: padlock -... |
283 |
irq_ts_restore(ts_state); |
420a4b20c crypto: padlock -... |
284 |
padlock_store_cword(&ctx->cword.encrypt); |
1da177e4c Linux-2.6.12-rc2 |
285 |
} |
6c2bb98bc [CRYPTO] all: Pas... |
286 |
static void aes_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in) |
1da177e4c Linux-2.6.12-rc2 |
287 |
{ |
6c2bb98bc [CRYPTO] all: Pas... |
288 |
struct aes_ctx *ctx = aes_ctx(tfm); |
e49140120 crypto: padlock -... |
289 |
int ts_state; |
e49140120 crypto: padlock -... |
290 |
|
420a4b20c crypto: padlock -... |
291 |
padlock_reset_key(&ctx->cword.encrypt); |
e49140120 crypto: padlock -... |
292 |
ts_state = irq_ts_save(); |
8d8409f77 crypto: padlock-a... |
293 |
ecb_crypt(in, out, ctx->D, &ctx->cword.decrypt, 1); |
e49140120 crypto: padlock -... |
294 |
irq_ts_restore(ts_state); |
420a4b20c crypto: padlock -... |
295 |
padlock_store_cword(&ctx->cword.encrypt); |
1da177e4c Linux-2.6.12-rc2 |
296 297 298 299 |
} static struct crypto_alg aes_alg = { .cra_name = "aes", |
c8a19c91b [CRYPTO] Allow AE... |
300 |
.cra_driver_name = "aes-padlock", |
ccc17c34d [CRYPTO] padlock:... |
301 |
.cra_priority = PADLOCK_CRA_PRIORITY, |
1da177e4c Linux-2.6.12-rc2 |
302 303 |
.cra_flags = CRYPTO_ALG_TYPE_CIPHER, .cra_blocksize = AES_BLOCK_SIZE, |
fbdae9f3e [CRYPTO] Ensure c... |
304 |
.cra_ctxsize = sizeof(struct aes_ctx), |
6789b2dc4 [PADLOCK] Move fa... |
305 |
.cra_alignmask = PADLOCK_ALIGNMENT - 1, |
1da177e4c Linux-2.6.12-rc2 |
306 307 308 309 310 311 312 313 |
.cra_module = THIS_MODULE, .cra_list = LIST_HEAD_INIT(aes_alg.cra_list), .cra_u = { .cipher = { .cia_min_keysize = AES_MIN_KEY_SIZE, .cia_max_keysize = AES_MAX_KEY_SIZE, .cia_setkey = aes_set_key, .cia_encrypt = aes_encrypt, |
28e8c3ad9 [PADLOCK] Impleme... |
314 |
.cia_decrypt = aes_decrypt, |
1da177e4c Linux-2.6.12-rc2 |
315 316 317 |
} } }; |
28ce728a9 [CRYPTO] padlock:... |
318 319 320 321 322 323 324 |
static int ecb_aes_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { struct aes_ctx *ctx = blk_aes_ctx(desc->tfm); struct blkcipher_walk walk; int err; |
e49140120 crypto: padlock -... |
325 |
int ts_state; |
28ce728a9 [CRYPTO] padlock:... |
326 |
|
420a4b20c crypto: padlock -... |
327 |
padlock_reset_key(&ctx->cword.encrypt); |
866cd902e [CRYPTO] padlock:... |
328 |
|
28ce728a9 [CRYPTO] padlock:... |
329 330 |
blkcipher_walk_init(&walk, dst, src, nbytes); err = blkcipher_walk_virt(desc, &walk); |
e49140120 crypto: padlock -... |
331 |
ts_state = irq_ts_save(); |
28ce728a9 [CRYPTO] padlock:... |
332 333 334 335 336 337 338 |
while ((nbytes = walk.nbytes)) { padlock_xcrypt_ecb(walk.src.virt.addr, walk.dst.virt.addr, ctx->E, &ctx->cword.encrypt, nbytes / AES_BLOCK_SIZE); nbytes &= AES_BLOCK_SIZE - 1; err = blkcipher_walk_done(desc, &walk, nbytes); } |
e49140120 crypto: padlock -... |
339 |
irq_ts_restore(ts_state); |
28ce728a9 [CRYPTO] padlock:... |
340 |
|
420a4b20c crypto: padlock -... |
341 |
padlock_store_cword(&ctx->cword.encrypt); |
28ce728a9 [CRYPTO] padlock:... |
342 343 344 345 346 347 348 349 350 351 |
return err; } static int ecb_aes_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { struct aes_ctx *ctx = blk_aes_ctx(desc->tfm); struct blkcipher_walk walk; int err; |
e49140120 crypto: padlock -... |
352 |
int ts_state; |
28ce728a9 [CRYPTO] padlock:... |
353 |
|
420a4b20c crypto: padlock -... |
354 |
padlock_reset_key(&ctx->cword.decrypt); |
866cd902e [CRYPTO] padlock:... |
355 |
|
28ce728a9 [CRYPTO] padlock:... |
356 357 |
blkcipher_walk_init(&walk, dst, src, nbytes); err = blkcipher_walk_virt(desc, &walk); |
e49140120 crypto: padlock -... |
358 |
ts_state = irq_ts_save(); |
28ce728a9 [CRYPTO] padlock:... |
359 360 361 362 363 364 365 |
while ((nbytes = walk.nbytes)) { padlock_xcrypt_ecb(walk.src.virt.addr, walk.dst.virt.addr, ctx->D, &ctx->cword.decrypt, nbytes / AES_BLOCK_SIZE); nbytes &= AES_BLOCK_SIZE - 1; err = blkcipher_walk_done(desc, &walk, nbytes); } |
e49140120 crypto: padlock -... |
366 |
irq_ts_restore(ts_state); |
420a4b20c crypto: padlock -... |
367 368 |
padlock_store_cword(&ctx->cword.encrypt); |
28ce728a9 [CRYPTO] padlock:... |
369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 |
return err; } static struct crypto_alg ecb_aes_alg = { .cra_name = "ecb(aes)", .cra_driver_name = "ecb-aes-padlock", .cra_priority = PADLOCK_COMPOSITE_PRIORITY, .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, .cra_blocksize = AES_BLOCK_SIZE, .cra_ctxsize = sizeof(struct aes_ctx), .cra_alignmask = PADLOCK_ALIGNMENT - 1, .cra_type = &crypto_blkcipher_type, .cra_module = THIS_MODULE, .cra_list = LIST_HEAD_INIT(ecb_aes_alg.cra_list), .cra_u = { .blkcipher = { .min_keysize = AES_MIN_KEY_SIZE, .max_keysize = AES_MAX_KEY_SIZE, .setkey = aes_set_key, .encrypt = ecb_aes_encrypt, .decrypt = ecb_aes_decrypt, } } }; static int cbc_aes_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { struct aes_ctx *ctx = blk_aes_ctx(desc->tfm); struct blkcipher_walk walk; int err; |
e49140120 crypto: padlock -... |
401 |
int ts_state; |
28ce728a9 [CRYPTO] padlock:... |
402 |
|
420a4b20c crypto: padlock -... |
403 |
padlock_reset_key(&ctx->cword.encrypt); |
866cd902e [CRYPTO] padlock:... |
404 |
|
28ce728a9 [CRYPTO] padlock:... |
405 406 |
blkcipher_walk_init(&walk, dst, src, nbytes); err = blkcipher_walk_virt(desc, &walk); |
e49140120 crypto: padlock -... |
407 |
ts_state = irq_ts_save(); |
28ce728a9 [CRYPTO] padlock:... |
408 409 410 411 412 413 414 415 416 |
while ((nbytes = walk.nbytes)) { u8 *iv = padlock_xcrypt_cbc(walk.src.virt.addr, walk.dst.virt.addr, ctx->E, walk.iv, &ctx->cword.encrypt, nbytes / AES_BLOCK_SIZE); memcpy(walk.iv, iv, AES_BLOCK_SIZE); nbytes &= AES_BLOCK_SIZE - 1; err = blkcipher_walk_done(desc, &walk, nbytes); } |
e49140120 crypto: padlock -... |
417 |
irq_ts_restore(ts_state); |
28ce728a9 [CRYPTO] padlock:... |
418 |
|
420a4b20c crypto: padlock -... |
419 |
padlock_store_cword(&ctx->cword.decrypt); |
28ce728a9 [CRYPTO] padlock:... |
420 421 422 423 424 425 426 427 428 429 |
return err; } static int cbc_aes_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { struct aes_ctx *ctx = blk_aes_ctx(desc->tfm); struct blkcipher_walk walk; int err; |
e49140120 crypto: padlock -... |
430 |
int ts_state; |
28ce728a9 [CRYPTO] padlock:... |
431 |
|
420a4b20c crypto: padlock -... |
432 |
padlock_reset_key(&ctx->cword.encrypt); |
866cd902e [CRYPTO] padlock:... |
433 |
|
28ce728a9 [CRYPTO] padlock:... |
434 435 |
blkcipher_walk_init(&walk, dst, src, nbytes); err = blkcipher_walk_virt(desc, &walk); |
e49140120 crypto: padlock -... |
436 |
ts_state = irq_ts_save(); |
28ce728a9 [CRYPTO] padlock:... |
437 438 439 440 441 442 443 |
while ((nbytes = walk.nbytes)) { padlock_xcrypt_cbc(walk.src.virt.addr, walk.dst.virt.addr, ctx->D, walk.iv, &ctx->cword.decrypt, nbytes / AES_BLOCK_SIZE); nbytes &= AES_BLOCK_SIZE - 1; err = blkcipher_walk_done(desc, &walk, nbytes); } |
e49140120 crypto: padlock -... |
444 |
irq_ts_restore(ts_state); |
420a4b20c crypto: padlock -... |
445 446 |
padlock_store_cword(&ctx->cword.encrypt); |
28ce728a9 [CRYPTO] padlock:... |
447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 |
return err; } static struct crypto_alg cbc_aes_alg = { .cra_name = "cbc(aes)", .cra_driver_name = "cbc-aes-padlock", .cra_priority = PADLOCK_COMPOSITE_PRIORITY, .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, .cra_blocksize = AES_BLOCK_SIZE, .cra_ctxsize = sizeof(struct aes_ctx), .cra_alignmask = PADLOCK_ALIGNMENT - 1, .cra_type = &crypto_blkcipher_type, .cra_module = THIS_MODULE, .cra_list = LIST_HEAD_INIT(cbc_aes_alg.cra_list), .cra_u = { .blkcipher = { .min_keysize = AES_MIN_KEY_SIZE, .max_keysize = AES_MAX_KEY_SIZE, .ivsize = AES_BLOCK_SIZE, .setkey = aes_set_key, .encrypt = cbc_aes_encrypt, .decrypt = cbc_aes_decrypt, } } }; |
1191f0a49 [CRYPTO] padlock:... |
472 |
static int __init padlock_init(void) |
1da177e4c Linux-2.6.12-rc2 |
473 |
{ |
1191f0a49 [CRYPTO] padlock:... |
474 |
int ret; |
a76c1c23d crypto: padlock-a... |
475 |
struct cpuinfo_x86 *c = &cpu_data(0); |
1191f0a49 [CRYPTO] padlock:... |
476 |
|
c39cc377f crypto: padlock-a... |
477 |
if (!cpu_has_xcrypt) |
1191f0a49 [CRYPTO] padlock:... |
478 |
return -ENODEV; |
1191f0a49 [CRYPTO] padlock:... |
479 480 |
if (!cpu_has_xcrypt_enabled) { |
b43e726b3 crypto: padlock -... |
481 482 |
printk(KERN_NOTICE PFX "VIA PadLock detected, but not enabled. Hmm, strange... "); |
1191f0a49 [CRYPTO] padlock:... |
483 484 |
return -ENODEV; } |
1da177e4c Linux-2.6.12-rc2 |
485 |
|
28ce728a9 [CRYPTO] padlock:... |
486 487 488 489 490 491 492 493 |
if ((ret = crypto_register_alg(&aes_alg))) goto aes_err; if ((ret = crypto_register_alg(&ecb_aes_alg))) goto ecb_aes_err; if ((ret = crypto_register_alg(&cbc_aes_alg))) goto cbc_aes_err; |
1191f0a49 [CRYPTO] padlock:... |
494 495 496 |
printk(KERN_NOTICE PFX "Using VIA PadLock ACE for AES algorithm. "); |
a76c1c23d crypto: padlock-a... |
497 |
if (c->x86 == 6 && c->x86_model == 15 && c->x86_mask == 2) { |
8d8409f77 crypto: padlock-a... |
498 499 |
ecb_fetch_blocks = MAX_ECB_FETCH_BLOCKS; cbc_fetch_blocks = MAX_CBC_FETCH_BLOCKS; |
a76c1c23d crypto: padlock-a... |
500 501 502 |
printk(KERN_NOTICE PFX "VIA Nano stepping 2 detected: enabling workaround. "); } |
28ce728a9 [CRYPTO] padlock:... |
503 |
out: |
1191f0a49 [CRYPTO] padlock:... |
504 |
return ret; |
28ce728a9 [CRYPTO] padlock:... |
505 506 507 508 509 510 511 512 513 |
cbc_aes_err: crypto_unregister_alg(&ecb_aes_alg); ecb_aes_err: crypto_unregister_alg(&aes_alg); aes_err: printk(KERN_ERR PFX "VIA PadLock AES initialization failed. "); goto out; |
1da177e4c Linux-2.6.12-rc2 |
514 |
} |
1191f0a49 [CRYPTO] padlock:... |
515 |
static void __exit padlock_fini(void) |
1da177e4c Linux-2.6.12-rc2 |
516 |
{ |
28ce728a9 [CRYPTO] padlock:... |
517 518 |
crypto_unregister_alg(&cbc_aes_alg); crypto_unregister_alg(&ecb_aes_alg); |
1da177e4c Linux-2.6.12-rc2 |
519 520 |
crypto_unregister_alg(&aes_alg); } |
1191f0a49 [CRYPTO] padlock:... |
521 522 523 524 525 526 527 |
module_init(padlock_init); module_exit(padlock_fini); MODULE_DESCRIPTION("VIA PadLock AES algorithm support"); MODULE_LICENSE("GPL"); MODULE_AUTHOR("Michal Ludvig"); |
acd246b74 crypto: padlock -... |
528 |
MODULE_ALIAS("aes"); |