Blame view
arch/x86/crypto/sha256_ssse3_glue.c
10.5 KB
8275d1aa6
|
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 |
/* * Cryptographic API. * * Glue code for the SHA256 Secure Hash Algorithm assembler * implementation using supplemental SSE3 / AVX / AVX2 instructions. * * This file is based on sha256_generic.c * * Copyright (C) 2013 Intel Corporation. * * Author: * Tim Chen <tim.c.chen@linux.intel.com> * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the Free * Software Foundation; either version 2 of the License, or (at your option) * any later version. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt #include <crypto/internal/hash.h> |
f2abe0d72
|
33 |
#include <crypto/internal/simd.h> |
8275d1aa6
|
34 35 36 |
#include <linux/init.h> #include <linux/module.h> #include <linux/mm.h> |
8275d1aa6
|
37 38 |
#include <linux/types.h> #include <crypto/sha.h> |
1631030ae
|
39 |
#include <crypto/sha256_base.h> |
8275d1aa6
|
40 |
#include <linux/string.h> |
f2abe0d72
|
41 |
#include <asm/simd.h> |
8275d1aa6
|
42 |
|
41419a289
|
43 44 |
asmlinkage void sha256_transform_ssse3(struct sha256_state *state, const u8 *data, int blocks); |
8275d1aa6
|
45 |
|
eb7d6ba88
|
46 |
static int _sha256_update(struct shash_desc *desc, const u8 *data, |
41419a289
|
47 |
unsigned int len, sha256_block_fn *sha256_xform) |
8275d1aa6
|
48 49 |
{ struct sha256_state *sctx = shash_desc_ctx(desc); |
8275d1aa6
|
50 |
|
f2abe0d72
|
51 |
if (!crypto_simd_usable() || |
1631030ae
|
52 53 |
(sctx->count % SHA256_BLOCK_SIZE) + len < SHA256_BLOCK_SIZE) return crypto_sha256_update(desc, data, len); |
8275d1aa6
|
54 |
|
41419a289
|
55 56 57 58 |
/* * Make sure struct sha256_state begins directly with the SHA256 * 256-bit internal state, as this is what the asm functions expect. */ |
1631030ae
|
59 |
BUILD_BUG_ON(offsetof(struct sha256_state, state) != 0); |
8275d1aa6
|
60 |
|
1631030ae
|
61 |
kernel_fpu_begin(); |
41419a289
|
62 |
sha256_base_do_update(desc, data, len, sha256_xform); |
1631030ae
|
63 |
kernel_fpu_end(); |
8275d1aa6
|
64 65 66 |
return 0; } |
5dda42fc8
|
67 |
static int sha256_finup(struct shash_desc *desc, const u8 *data, |
41419a289
|
68 |
unsigned int len, u8 *out, sha256_block_fn *sha256_xform) |
8275d1aa6
|
69 |
{ |
f2abe0d72
|
70 |
if (!crypto_simd_usable()) |
1631030ae
|
71 |
return crypto_sha256_finup(desc, data, len, out); |
8275d1aa6
|
72 |
|
1631030ae
|
73 74 |
kernel_fpu_begin(); if (len) |
41419a289
|
75 76 |
sha256_base_do_update(desc, data, len, sha256_xform); sha256_base_do_finalize(desc, sha256_xform); |
1631030ae
|
77 |
kernel_fpu_end(); |
8275d1aa6
|
78 |
|
1631030ae
|
79 |
return sha256_base_finish(desc, out); |
8275d1aa6
|
80 |
} |
5dda42fc8
|
81 82 83 |
static int sha256_ssse3_update(struct shash_desc *desc, const u8 *data, unsigned int len) { |
eb7d6ba88
|
84 |
return _sha256_update(desc, data, len, sha256_transform_ssse3); |
5dda42fc8
|
85 86 87 88 89 90 91 |
} static int sha256_ssse3_finup(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out) { return sha256_finup(desc, data, len, out, sha256_transform_ssse3); } |
1631030ae
|
92 93 |
/* Add padding and return the message digest. */ static int sha256_ssse3_final(struct shash_desc *desc, u8 *out) |
a710f761f
|
94 |
{ |
1631030ae
|
95 |
return sha256_ssse3_finup(desc, NULL, 0, out); |
a710f761f
|
96 |
} |
5dda42fc8
|
97 |
static struct shash_alg sha256_ssse3_algs[] = { { |
8275d1aa6
|
98 |
.digestsize = SHA256_DIGEST_SIZE, |
1631030ae
|
99 |
.init = sha256_base_init, |
8275d1aa6
|
100 101 |
.update = sha256_ssse3_update, .final = sha256_ssse3_final, |
1631030ae
|
102 |
.finup = sha256_ssse3_finup, |
8275d1aa6
|
103 |
.descsize = sizeof(struct sha256_state), |
8275d1aa6
|
104 105 106 107 |
.base = { .cra_name = "sha256", .cra_driver_name = "sha256-ssse3", .cra_priority = 150, |
8275d1aa6
|
108 109 110 |
.cra_blocksize = SHA256_BLOCK_SIZE, .cra_module = THIS_MODULE, } |
a710f761f
|
111 112 |
}, { .digestsize = SHA224_DIGEST_SIZE, |
1631030ae
|
113 |
.init = sha224_base_init, |
a710f761f
|
114 |
.update = sha256_ssse3_update, |
1631030ae
|
115 116 |
.final = sha256_ssse3_final, .finup = sha256_ssse3_finup, |
a710f761f
|
117 |
.descsize = sizeof(struct sha256_state), |
a710f761f
|
118 119 120 121 |
.base = { .cra_name = "sha224", .cra_driver_name = "sha224-ssse3", .cra_priority = 150, |
a710f761f
|
122 123 124 125 |
.cra_blocksize = SHA224_BLOCK_SIZE, .cra_module = THIS_MODULE, } } }; |
8275d1aa6
|
126 |
|
5dda42fc8
|
127 128 129 130 131 132 133 134 135 136 137 138 139 140 |
static int register_sha256_ssse3(void) { if (boot_cpu_has(X86_FEATURE_SSSE3)) return crypto_register_shashes(sha256_ssse3_algs, ARRAY_SIZE(sha256_ssse3_algs)); return 0; } static void unregister_sha256_ssse3(void) { if (boot_cpu_has(X86_FEATURE_SSSE3)) crypto_unregister_shashes(sha256_ssse3_algs, ARRAY_SIZE(sha256_ssse3_algs)); } |
41419a289
|
141 142 |
asmlinkage void sha256_transform_avx(struct sha256_state *state, const u8 *data, int blocks); |
5dda42fc8
|
143 144 145 146 |
static int sha256_avx_update(struct shash_desc *desc, const u8 *data, unsigned int len) { |
eb7d6ba88
|
147 |
return _sha256_update(desc, data, len, sha256_transform_avx); |
5dda42fc8
|
148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 |
} static int sha256_avx_finup(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out) { return sha256_finup(desc, data, len, out, sha256_transform_avx); } static int sha256_avx_final(struct shash_desc *desc, u8 *out) { return sha256_avx_finup(desc, NULL, 0, out); } static struct shash_alg sha256_avx_algs[] = { { .digestsize = SHA256_DIGEST_SIZE, .init = sha256_base_init, .update = sha256_avx_update, .final = sha256_avx_final, .finup = sha256_avx_finup, .descsize = sizeof(struct sha256_state), .base = { .cra_name = "sha256", .cra_driver_name = "sha256-avx", .cra_priority = 160, |
5dda42fc8
|
172 173 174 175 176 177 178 179 180 181 182 183 184 185 |
.cra_blocksize = SHA256_BLOCK_SIZE, .cra_module = THIS_MODULE, } }, { .digestsize = SHA224_DIGEST_SIZE, .init = sha224_base_init, .update = sha256_avx_update, .final = sha256_avx_final, .finup = sha256_avx_finup, .descsize = sizeof(struct sha256_state), .base = { .cra_name = "sha224", .cra_driver_name = "sha224-avx", .cra_priority = 160, |
5dda42fc8
|
186 187 188 189 190 191 |
.cra_blocksize = SHA224_BLOCK_SIZE, .cra_module = THIS_MODULE, } } }; static bool avx_usable(void) |
8275d1aa6
|
192 |
{ |
d91cab781
|
193 |
if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL)) { |
da154e82a
|
194 |
if (boot_cpu_has(X86_FEATURE_AVX)) |
70d51eb65
|
195 196 |
pr_info("AVX detected but unusable. "); |
8275d1aa6
|
197 198 199 200 201 |
return false; } return true; } |
8275d1aa6
|
202 |
|
5dda42fc8
|
203 |
static int register_sha256_avx(void) |
8275d1aa6
|
204 |
{ |
5dda42fc8
|
205 206 207 208 209 |
if (avx_usable()) return crypto_register_shashes(sha256_avx_algs, ARRAY_SIZE(sha256_avx_algs)); return 0; } |
8275d1aa6
|
210 |
|
5dda42fc8
|
211 212 213 214 215 216 |
static void unregister_sha256_avx(void) { if (avx_usable()) crypto_unregister_shashes(sha256_avx_algs, ARRAY_SIZE(sha256_avx_algs)); } |
8275d1aa6
|
217 |
|
41419a289
|
218 219 |
asmlinkage void sha256_transform_rorx(struct sha256_state *state, const u8 *data, int blocks); |
5dda42fc8
|
220 221 222 223 |
static int sha256_avx2_update(struct shash_desc *desc, const u8 *data, unsigned int len) { |
eb7d6ba88
|
224 |
return _sha256_update(desc, data, len, sha256_transform_rorx); |
5dda42fc8
|
225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 |
} static int sha256_avx2_finup(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out) { return sha256_finup(desc, data, len, out, sha256_transform_rorx); } static int sha256_avx2_final(struct shash_desc *desc, u8 *out) { return sha256_avx2_finup(desc, NULL, 0, out); } static struct shash_alg sha256_avx2_algs[] = { { .digestsize = SHA256_DIGEST_SIZE, .init = sha256_base_init, .update = sha256_avx2_update, .final = sha256_avx2_final, .finup = sha256_avx2_finup, .descsize = sizeof(struct sha256_state), .base = { .cra_name = "sha256", .cra_driver_name = "sha256-avx2", .cra_priority = 170, |
5dda42fc8
|
249 250 |
.cra_blocksize = SHA256_BLOCK_SIZE, .cra_module = THIS_MODULE, |
8275d1aa6
|
251 |
} |
5dda42fc8
|
252 253 254 255 256 257 258 259 260 261 262 |
}, { .digestsize = SHA224_DIGEST_SIZE, .init = sha224_base_init, .update = sha256_avx2_update, .final = sha256_avx2_final, .finup = sha256_avx2_finup, .descsize = sizeof(struct sha256_state), .base = { .cra_name = "sha224", .cra_driver_name = "sha224-avx2", .cra_priority = 170, |
5dda42fc8
|
263 264 265 266 |
.cra_blocksize = SHA224_BLOCK_SIZE, .cra_module = THIS_MODULE, } } }; |
8275d1aa6
|
267 |
|
5dda42fc8
|
268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 |
static bool avx2_usable(void) { if (avx_usable() && boot_cpu_has(X86_FEATURE_AVX2) && boot_cpu_has(X86_FEATURE_BMI2)) return true; return false; } static int register_sha256_avx2(void) { if (avx2_usable()) return crypto_register_shashes(sha256_avx2_algs, ARRAY_SIZE(sha256_avx2_algs)); return 0; } static void unregister_sha256_avx2(void) { if (avx2_usable()) crypto_unregister_shashes(sha256_avx2_algs, ARRAY_SIZE(sha256_avx2_algs)); } |
95fca7df0
|
291 |
#ifdef CONFIG_AS_SHA256_NI |
41419a289
|
292 293 |
asmlinkage void sha256_ni_transform(struct sha256_state *digest, const u8 *data, int rounds); |
5dda42fc8
|
294 295 296 297 |
static int sha256_ni_update(struct shash_desc *desc, const u8 *data, unsigned int len) { |
eb7d6ba88
|
298 |
return _sha256_update(desc, data, len, sha256_ni_transform); |
5dda42fc8
|
299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 |
} static int sha256_ni_finup(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out) { return sha256_finup(desc, data, len, out, sha256_ni_transform); } static int sha256_ni_final(struct shash_desc *desc, u8 *out) { return sha256_ni_finup(desc, NULL, 0, out); } static struct shash_alg sha256_ni_algs[] = { { .digestsize = SHA256_DIGEST_SIZE, .init = sha256_base_init, .update = sha256_ni_update, .final = sha256_ni_final, .finup = sha256_ni_finup, .descsize = sizeof(struct sha256_state), .base = { .cra_name = "sha256", .cra_driver_name = "sha256-ni", .cra_priority = 250, |
5dda42fc8
|
323 324 325 326 327 328 329 330 331 332 333 334 335 336 |
.cra_blocksize = SHA256_BLOCK_SIZE, .cra_module = THIS_MODULE, } }, { .digestsize = SHA224_DIGEST_SIZE, .init = sha224_base_init, .update = sha256_ni_update, .final = sha256_ni_final, .finup = sha256_ni_finup, .descsize = sizeof(struct sha256_state), .base = { .cra_name = "sha224", .cra_driver_name = "sha224-ni", .cra_priority = 250, |
5dda42fc8
|
337 338 |
.cra_blocksize = SHA224_BLOCK_SIZE, .cra_module = THIS_MODULE, |
8275d1aa6
|
339 |
} |
5dda42fc8
|
340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 |
} }; static int register_sha256_ni(void) { if (boot_cpu_has(X86_FEATURE_SHA_NI)) return crypto_register_shashes(sha256_ni_algs, ARRAY_SIZE(sha256_ni_algs)); return 0; } static void unregister_sha256_ni(void) { if (boot_cpu_has(X86_FEATURE_SHA_NI)) crypto_unregister_shashes(sha256_ni_algs, ARRAY_SIZE(sha256_ni_algs)); } #else static inline int register_sha256_ni(void) { return 0; } static inline void unregister_sha256_ni(void) { } |
8275d1aa6
|
360 |
#endif |
8275d1aa6
|
361 |
|
5dda42fc8
|
362 363 364 365 366 367 368 369 |
static int __init sha256_ssse3_mod_init(void) { if (register_sha256_ssse3()) goto fail; if (register_sha256_avx()) { unregister_sha256_ssse3(); goto fail; |
8275d1aa6
|
370 |
} |
8275d1aa6
|
371 |
|
5dda42fc8
|
372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 |
if (register_sha256_avx2()) { unregister_sha256_avx(); unregister_sha256_ssse3(); goto fail; } if (register_sha256_ni()) { unregister_sha256_avx2(); unregister_sha256_avx(); unregister_sha256_ssse3(); goto fail; } return 0; fail: |
8275d1aa6
|
387 388 389 390 391 |
return -ENODEV; } static void __exit sha256_ssse3_mod_fini(void) { |
5dda42fc8
|
392 393 394 395 |
unregister_sha256_ni(); unregister_sha256_avx2(); unregister_sha256_avx(); unregister_sha256_ssse3(); |
8275d1aa6
|
396 397 398 399 400 401 402 |
} module_init(sha256_ssse3_mod_init); module_exit(sha256_ssse3_mod_fini); MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("SHA256 Secure Hash Algorithm, Supplemental SSE3 accelerated"); |
5d26a105b
|
403 |
MODULE_ALIAS_CRYPTO("sha256"); |
1a445e8ef
|
404 405 406 |
MODULE_ALIAS_CRYPTO("sha256-ssse3"); MODULE_ALIAS_CRYPTO("sha256-avx"); MODULE_ALIAS_CRYPTO("sha256-avx2"); |
5d26a105b
|
407 |
MODULE_ALIAS_CRYPTO("sha224"); |
1a445e8ef
|
408 409 410 411 412 413 414 |
MODULE_ALIAS_CRYPTO("sha224-ssse3"); MODULE_ALIAS_CRYPTO("sha224-avx"); MODULE_ALIAS_CRYPTO("sha224-avx2"); #ifdef CONFIG_AS_SHA256_NI MODULE_ALIAS_CRYPTO("sha256-ni"); MODULE_ALIAS_CRYPTO("sha224-ni"); #endif |