Blame view
arch/x86/crypto/sha512_ssse3_glue.c
8.62 KB
87de4579f
|
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 |
/* * Cryptographic API. * * Glue code for the SHA512 Secure Hash Algorithm assembler * implementation using supplemental SSE3 / AVX / AVX2 instructions. * * This file is based on sha512_generic.c * * Copyright (C) 2013 Intel Corporation * Author: Tim Chen <tim.c.chen@linux.intel.com> * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the Free * Software Foundation; either version 2 of the License, or (at your option) * any later version. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. * */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt #include <crypto/internal/hash.h> |
f2abe0d72
|
31 |
#include <crypto/internal/simd.h> |
87de4579f
|
32 33 34 |
#include <linux/init.h> #include <linux/module.h> #include <linux/mm.h> |
f2abe0d72
|
35 |
#include <linux/string.h> |
87de4579f
|
36 37 |
#include <linux/types.h> #include <crypto/sha.h> |
e68410ebf
|
38 |
#include <crypto/sha512_base.h> |
f2abe0d72
|
39 |
#include <asm/simd.h> |
87de4579f
|
40 |
|
41419a289
|
41 42 |
asmlinkage void sha512_transform_ssse3(struct sha512_state *state, const u8 *data, int blocks); |
87de4579f
|
43 |
|
be6ec98dd
|
44 |
static int sha512_update(struct shash_desc *desc, const u8 *data, |
41419a289
|
45 |
unsigned int len, sha512_block_fn *sha512_xform) |
87de4579f
|
46 47 |
{ struct sha512_state *sctx = shash_desc_ctx(desc); |
87de4579f
|
48 |
|
f2abe0d72
|
49 |
if (!crypto_simd_usable() || |
e68410ebf
|
50 51 |
(sctx->count[0] % SHA512_BLOCK_SIZE) + len < SHA512_BLOCK_SIZE) return crypto_sha512_update(desc, data, len); |
87de4579f
|
52 |
|
41419a289
|
53 54 55 56 |
/* * Make sure struct sha512_state begins directly with the SHA512 * 512-bit internal state, as this is what the asm functions expect. */ |
e68410ebf
|
57 |
BUILD_BUG_ON(offsetof(struct sha512_state, state) != 0); |
87de4579f
|
58 |
|
e68410ebf
|
59 |
kernel_fpu_begin(); |
41419a289
|
60 |
sha512_base_do_update(desc, data, len, sha512_xform); |
e68410ebf
|
61 |
kernel_fpu_end(); |
87de4579f
|
62 63 64 |
return 0; } |
be6ec98dd
|
65 |
static int sha512_finup(struct shash_desc *desc, const u8 *data, |
41419a289
|
66 |
unsigned int len, u8 *out, sha512_block_fn *sha512_xform) |
87de4579f
|
67 |
{ |
f2abe0d72
|
68 |
if (!crypto_simd_usable()) |
e68410ebf
|
69 |
return crypto_sha512_finup(desc, data, len, out); |
87de4579f
|
70 |
|
e68410ebf
|
71 72 |
kernel_fpu_begin(); if (len) |
41419a289
|
73 74 |
sha512_base_do_update(desc, data, len, sha512_xform); sha512_base_do_finalize(desc, sha512_xform); |
e68410ebf
|
75 |
kernel_fpu_end(); |
87de4579f
|
76 |
|
e68410ebf
|
77 |
return sha512_base_finish(desc, out); |
87de4579f
|
78 |
} |
be6ec98dd
|
79 80 81 82 83 84 85 86 87 88 89 |
static int sha512_ssse3_update(struct shash_desc *desc, const u8 *data, unsigned int len) { return sha512_update(desc, data, len, sha512_transform_ssse3); } static int sha512_ssse3_finup(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out) { return sha512_finup(desc, data, len, out, sha512_transform_ssse3); } |
87de4579f
|
90 91 92 |
/* Add padding and return the message digest. */ static int sha512_ssse3_final(struct shash_desc *desc, u8 *out) { |
e68410ebf
|
93 |
return sha512_ssse3_finup(desc, NULL, 0, out); |
340991e30
|
94 |
} |
be6ec98dd
|
95 |
static struct shash_alg sha512_ssse3_algs[] = { { |
87de4579f
|
96 |
.digestsize = SHA512_DIGEST_SIZE, |
e68410ebf
|
97 |
.init = sha512_base_init, |
87de4579f
|
98 99 |
.update = sha512_ssse3_update, .final = sha512_ssse3_final, |
e68410ebf
|
100 |
.finup = sha512_ssse3_finup, |
87de4579f
|
101 |
.descsize = sizeof(struct sha512_state), |
87de4579f
|
102 103 104 105 |
.base = { .cra_name = "sha512", .cra_driver_name = "sha512-ssse3", .cra_priority = 150, |
87de4579f
|
106 107 108 |
.cra_blocksize = SHA512_BLOCK_SIZE, .cra_module = THIS_MODULE, } |
340991e30
|
109 110 |
}, { .digestsize = SHA384_DIGEST_SIZE, |
e68410ebf
|
111 |
.init = sha384_base_init, |
340991e30
|
112 |
.update = sha512_ssse3_update, |
e68410ebf
|
113 114 |
.final = sha512_ssse3_final, .finup = sha512_ssse3_finup, |
340991e30
|
115 |
.descsize = sizeof(struct sha512_state), |
340991e30
|
116 117 118 119 |
.base = { .cra_name = "sha384", .cra_driver_name = "sha384-ssse3", .cra_priority = 150, |
340991e30
|
120 121 122 123 |
.cra_blocksize = SHA384_BLOCK_SIZE, .cra_module = THIS_MODULE, } } }; |
87de4579f
|
124 |
|
be6ec98dd
|
125 126 127 128 129 130 131 132 133 134 135 136 137 138 |
static int register_sha512_ssse3(void) { if (boot_cpu_has(X86_FEATURE_SSSE3)) return crypto_register_shashes(sha512_ssse3_algs, ARRAY_SIZE(sha512_ssse3_algs)); return 0; } static void unregister_sha512_ssse3(void) { if (boot_cpu_has(X86_FEATURE_SSSE3)) crypto_unregister_shashes(sha512_ssse3_algs, ARRAY_SIZE(sha512_ssse3_algs)); } |
41419a289
|
139 140 |
asmlinkage void sha512_transform_avx(struct sha512_state *state, const u8 *data, int blocks); |
be6ec98dd
|
141 |
static bool avx_usable(void) |
87de4579f
|
142 |
{ |
d91cab781
|
143 |
if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL)) { |
da154e82a
|
144 |
if (boot_cpu_has(X86_FEATURE_AVX)) |
c93b8a396
|
145 146 |
pr_info("AVX detected but unusable. "); |
87de4579f
|
147 148 149 150 151 |
return false; } return true; } |
87de4579f
|
152 |
|
be6ec98dd
|
153 154 |
static int sha512_avx_update(struct shash_desc *desc, const u8 *data, unsigned int len) |
87de4579f
|
155 |
{ |
be6ec98dd
|
156 157 |
return sha512_update(desc, data, len, sha512_transform_avx); } |
87de4579f
|
158 |
|
be6ec98dd
|
159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 |
static int sha512_avx_finup(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out) { return sha512_finup(desc, data, len, out, sha512_transform_avx); } /* Add padding and return the message digest. */ static int sha512_avx_final(struct shash_desc *desc, u8 *out) { return sha512_avx_finup(desc, NULL, 0, out); } static struct shash_alg sha512_avx_algs[] = { { .digestsize = SHA512_DIGEST_SIZE, .init = sha512_base_init, .update = sha512_avx_update, .final = sha512_avx_final, .finup = sha512_avx_finup, .descsize = sizeof(struct sha512_state), .base = { .cra_name = "sha512", .cra_driver_name = "sha512-avx", .cra_priority = 160, |
be6ec98dd
|
182 183 |
.cra_blocksize = SHA512_BLOCK_SIZE, .cra_module = THIS_MODULE, |
87de4579f
|
184 |
} |
be6ec98dd
|
185 186 187 188 189 190 191 192 193 194 195 |
}, { .digestsize = SHA384_DIGEST_SIZE, .init = sha384_base_init, .update = sha512_avx_update, .final = sha512_avx_final, .finup = sha512_avx_finup, .descsize = sizeof(struct sha512_state), .base = { .cra_name = "sha384", .cra_driver_name = "sha384-avx", .cra_priority = 160, |
be6ec98dd
|
196 197 198 199 |
.cra_blocksize = SHA384_BLOCK_SIZE, .cra_module = THIS_MODULE, } } }; |
87de4579f
|
200 |
|
be6ec98dd
|
201 202 203 204 205 206 207 208 209 210 211 212 213 214 |
static int register_sha512_avx(void) { if (avx_usable()) return crypto_register_shashes(sha512_avx_algs, ARRAY_SIZE(sha512_avx_algs)); return 0; } static void unregister_sha512_avx(void) { if (avx_usable()) crypto_unregister_shashes(sha512_avx_algs, ARRAY_SIZE(sha512_avx_algs)); } |
be6ec98dd
|
215 |
|
41419a289
|
216 217 |
asmlinkage void sha512_transform_rorx(struct sha512_state *state, const u8 *data, int blocks); |
be6ec98dd
|
218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 |
static int sha512_avx2_update(struct shash_desc *desc, const u8 *data, unsigned int len) { return sha512_update(desc, data, len, sha512_transform_rorx); } static int sha512_avx2_finup(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out) { return sha512_finup(desc, data, len, out, sha512_transform_rorx); } /* Add padding and return the message digest. */ static int sha512_avx2_final(struct shash_desc *desc, u8 *out) { return sha512_avx2_finup(desc, NULL, 0, out); } static struct shash_alg sha512_avx2_algs[] = { { .digestsize = SHA512_DIGEST_SIZE, .init = sha512_base_init, .update = sha512_avx2_update, .final = sha512_avx2_final, .finup = sha512_avx2_finup, .descsize = sizeof(struct sha512_state), .base = { .cra_name = "sha512", .cra_driver_name = "sha512-avx2", .cra_priority = 170, |
be6ec98dd
|
248 249 250 251 252 253 254 255 256 257 258 259 260 261 |
.cra_blocksize = SHA512_BLOCK_SIZE, .cra_module = THIS_MODULE, } }, { .digestsize = SHA384_DIGEST_SIZE, .init = sha384_base_init, .update = sha512_avx2_update, .final = sha512_avx2_final, .finup = sha512_avx2_finup, .descsize = sizeof(struct sha512_state), .base = { .cra_name = "sha384", .cra_driver_name = "sha384-avx2", .cra_priority = 170, |
be6ec98dd
|
262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 |
.cra_blocksize = SHA384_BLOCK_SIZE, .cra_module = THIS_MODULE, } } }; static bool avx2_usable(void) { if (avx_usable() && boot_cpu_has(X86_FEATURE_AVX2) && boot_cpu_has(X86_FEATURE_BMI2)) return true; return false; } static int register_sha512_avx2(void) { if (avx2_usable()) return crypto_register_shashes(sha512_avx2_algs, ARRAY_SIZE(sha512_avx2_algs)); return 0; } static void unregister_sha512_avx2(void) { if (avx2_usable()) crypto_unregister_shashes(sha512_avx2_algs, ARRAY_SIZE(sha512_avx2_algs)); } |
be6ec98dd
|
290 291 292 293 294 295 296 297 298 299 |
static int __init sha512_ssse3_mod_init(void) { if (register_sha512_ssse3()) goto fail; if (register_sha512_avx()) { unregister_sha512_ssse3(); goto fail; |
87de4579f
|
300 |
} |
87de4579f
|
301 |
|
be6ec98dd
|
302 303 304 305 306 307 308 309 |
if (register_sha512_avx2()) { unregister_sha512_avx(); unregister_sha512_ssse3(); goto fail; } return 0; fail: |
87de4579f
|
310 311 312 313 314 |
return -ENODEV; } static void __exit sha512_ssse3_mod_fini(void) { |
be6ec98dd
|
315 316 317 |
unregister_sha512_avx2(); unregister_sha512_avx(); unregister_sha512_ssse3(); |
87de4579f
|
318 319 320 321 322 323 324 |
} module_init(sha512_ssse3_mod_init); module_exit(sha512_ssse3_mod_fini); MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("SHA512 Secure Hash Algorithm, Supplemental SSE3 accelerated"); |
5d26a105b
|
325 |
MODULE_ALIAS_CRYPTO("sha512"); |
1a445e8ef
|
326 327 328 |
MODULE_ALIAS_CRYPTO("sha512-ssse3"); MODULE_ALIAS_CRYPTO("sha512-avx"); MODULE_ALIAS_CRYPTO("sha512-avx2"); |
5d26a105b
|
329 |
MODULE_ALIAS_CRYPTO("sha384"); |
1a445e8ef
|
330 331 332 |
MODULE_ALIAS_CRYPTO("sha384-ssse3"); MODULE_ALIAS_CRYPTO("sha384-avx"); MODULE_ALIAS_CRYPTO("sha384-avx2"); |