crypto: crc32c - Use Intel CRC32 instruction

From NHM processor onward, Intel processors can support hardware accelerated CRC32c algorithm with the new CRC32 instruction in SSE 4.2 instruction set. The patch detects the availability of the feature, and chooses the most proper way to calculate CRC32c checksum. Byte code instructions are used for compiler compatibility. No MMX / XMM registers is involved in the implementation. Signed-off-by: Austin Zhang <austin.zhang@intel.com> Signed-off-by: Kent Liu <kent.liu@intel.com> Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>

crypto: crc32c - Use Intel CRC32 instruction
From NHM processor onward, Intel processors can support hardware accelerated CRC32c algorithm with the new CRC32 instruction in SSE 4.2 instruction set. The patch detects the availability of the feature, and chooses the most proper way to calculate CRC32c checksum. Byte code instructions are used for compiler compatibility. No MMX / XMM registers is involved in the implementation. Signed-off-by: Austin Zhang <austin.zhang@intel.com> Signed-off-by: Kent Liu <kent.liu@intel.com> Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
Austin Zhang · Herbert Xu
1 parent f139cfa7cd
Showing 3 changed files with 211 additions and 0 deletions Side-by-side Diff
arch/x86/crypto/Makefile
arch/x86/crypto/crc32c-intel.c
crypto/Kconfig
@@ -10,6 +10,8 @@
 obj-$(CONFIG_CRYPTO_TWOFISH_X86_64) += twofish-x86_64.o
 obj-$(CONFIG_CRYPTO_SALSA20_X86_64) += salsa20-x86_64.o
  
+obj-$(CONFIG_CRYPTO_CRC32C_INTEL) += crc32c-intel.o
+
 aes-i586-y := aes-i586-asm_32.o aes_glue.o
 twofish-i586-y := twofish-i586-asm_32.o twofish_glue.o
 salsa20-i586-y := salsa20-i586-asm_32.o salsa20_glue.o
+/*
+ * Using hardware provided CRC32 instruction to accelerate the CRC32 disposal.
+ * CRC32C polynomial:0x1EDC6F41(BE)/0x82F63B78(LE)
+ * CRC32 is a new instruction in Intel SSE4.2, the reference can be found at:
+ * http://www.intel.com/products/processor/manuals/
+ * Intel(R) 64 and IA-32 Architectures Software Developer's Manual
+ * Volume 2A: Instruction Set Reference, A-M
+ *
+ * Copyright (c) 2008 Austin Zhang <austin_zhang@linux.intel.com>
+ * Copyright (c) 2008 Kent Liu <kent.liu@intel.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; either version 2 of the License, or (at your option)
+ * any later version.
+ *
+ */
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/string.h>
+#include <linux/kernel.h>
+#include <crypto/internal/hash.h>
+
+#include <asm/cpufeature.h>
+
+#define CHKSUM_BLOCK_SIZE	1
+#define CHKSUM_DIGEST_SIZE	4
+
+#define SCALE_F	sizeof(unsigned long)
+
+#ifdef CONFIG_X86_64
+#define REX_PRE "0x48, "
+#else
+#define REX_PRE
+#endif
+
+static u32 crc32c_intel_le_hw_byte(u32 crc, unsigned char const *data, size_t length)
+{
+	while (length--) {
+		__asm__ __volatile__(
+			".byte 0xf2, 0xf, 0x38, 0xf0, 0xf1"
+			:"=S"(crc)
+			:"0"(crc), "c"(*data)
+		);
+		data++;
+	}
+
+	return crc;
+}
+
+static u32 __pure crc32c_intel_le_hw(u32 crc, unsigned char const *p, size_t len)
+{
+	unsigned int iquotient = len / SCALE_F;
+	unsigned int iremainder = len % SCALE_F;
+	unsigned long *ptmp = (unsigned long *)p;
+
+	while (iquotient--) {
+		__asm__ __volatile__(
+			".byte 0xf2, " REX_PRE "0xf, 0x38, 0xf1, 0xf1;"
+			:"=S"(crc)
+			:"0"(crc), "c"(*ptmp)
+		);
+		ptmp++;
+	}
+
+	if (iremainder)
+		crc = crc32c_intel_le_hw_byte(crc, (unsigned char *)ptmp,
+				 iremainder);
+
+	return crc;
+}
+
+/*
+ * Setting the seed allows arbitrary accumulators and flexible XOR policy
+ * If your algorithm starts with ~0, then XOR with ~0 before you set
+ * the seed.
+ */
+static int crc32c_intel_setkey(struct crypto_ahash *hash, const u8 *key,
+			unsigned int keylen)
+{
+	u32 *mctx = crypto_ahash_ctx(hash);
+
+	if (keylen != sizeof(u32)) {
+		crypto_ahash_set_flags(hash, CRYPTO_TFM_RES_BAD_KEY_LEN);
+		return -EINVAL;
+	}
+	*mctx = le32_to_cpup((__le32 *)key);
+	return 0;
+}
+
+static int crc32c_intel_init(struct ahash_request *req)
+{
+	u32 *mctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req));
+	u32 *crcp = ahash_request_ctx(req);
+
+	*crcp = *mctx;
+
+	return 0;
+}
+
+static int crc32c_intel_update(struct ahash_request *req)
+{
+	struct crypto_hash_walk walk;
+	u32 *crcp = ahash_request_ctx(req);
+	u32 crc = *crcp;
+	int nbytes;
+
+	for (nbytes = crypto_hash_walk_first(req, &walk); nbytes;
+	   nbytes = crypto_hash_walk_done(&walk, 0))
+	crc = crc32c_intel_le_hw(crc, walk.data, nbytes);
+
+	*crcp = crc;
+	return 0;
+}
+
+static int crc32c_intel_final(struct ahash_request *req)
+{
+	u32 *crcp = ahash_request_ctx(req);
+
+	*(__le32 *)req->result = ~cpu_to_le32p(crcp);
+	return 0;
+}
+
+static int crc32c_intel_digest(struct ahash_request *req)
+{
+	struct crypto_hash_walk walk;
+	u32 *mctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req));
+	u32 crc = *mctx;
+	int nbytes;
+
+	for (nbytes = crypto_hash_walk_first(req, &walk); nbytes;
+	   nbytes = crypto_hash_walk_done(&walk, 0))
+		crc = crc32c_intel_le_hw(crc, walk.data, nbytes);
+
+	*(__le32 *)req->result = ~cpu_to_le32(crc);
+	return 0;
+}
+
+static int crc32c_intel_cra_init(struct crypto_tfm *tfm)
+{
+	u32 *key = crypto_tfm_ctx(tfm);
+
+	*key = ~0;
+
+	tfm->crt_ahash.reqsize = sizeof(u32);
+
+	return 0;
+}
+
+static struct crypto_alg alg = {
+	.cra_name               =       "crc32c",
+	.cra_driver_name        =       "crc32c-intel",
+	.cra_priority           =       200,
+	.cra_flags              =       CRYPTO_ALG_TYPE_AHASH,
+	.cra_blocksize          =       CHKSUM_BLOCK_SIZE,
+	.cra_alignmask          =       3,
+	.cra_ctxsize            =       sizeof(u32),
+	.cra_module             =       THIS_MODULE,
+	.cra_list               =       LIST_HEAD_INIT(alg.cra_list),
+	.cra_init               =       crc32c_intel_cra_init,
+	.cra_type               =       &crypto_ahash_type,
+	.cra_u                  =       {
+		.ahash = {
+			.digestsize    =       CHKSUM_DIGEST_SIZE,
+			.setkey        =       crc32c_intel_setkey,
+			.init          =       crc32c_intel_init,
+			.update        =       crc32c_intel_update,
+			.final         =       crc32c_intel_final,
+			.digest        =       crc32c_intel_digest,
+		}
+	}
+};
+
+
+static int __init crc32c_intel_mod_init(void)
+{
+	if (cpu_has_xmm4_2)
+		return crypto_register_alg(&alg);
+	else
+		return -ENODEV;
+}
+
+static void __exit crc32c_intel_mod_fini(void)
+{
+	crypto_unregister_alg(&alg);
+}
+
+module_init(crc32c_intel_mod_init);
+module_exit(crc32c_intel_mod_fini);
+
+MODULE_AUTHOR("Austin Zhang <austin.zhang@intel.com>, Kent Liu <kent.liu@intel.com>");
+MODULE_DESCRIPTION("CRC32c (Castagnoli) optimization using Intel Hardware.");
+MODULE_LICENSE("GPL");
+
+MODULE_ALIAS("crc32c");
+MODULE_ALIAS("crc32c-intel");
@@ -221,6 +221,18 @@
 	  See Castagnoli93.  This implementation uses lib/libcrc32c.
 	  Module will be crc32c.
  
+config CRYPTO_CRC32C_INTEL
+	tristate "CRC32c INTEL hardware acceleration"
+	depends on X86
+	select CRYPTO_HASH
+	help
+	  In Intel processor with SSE4.2 supported, the processor will
+	  support CRC32C implementation using hardware accelerated CRC32
+	  instruction. This option will create 'crc32c-intel' module,
+	  which will enable any routine to use the CRC32 instruction to
+	  gain performance compared with software implementation.
+	  Module will be crc32c-intel.
+
 config CRYPTO_MD4
 	tristate "MD4 digest algorithm"
 	select CRYPTO_ALGAPI
...	...	@@ -10,6 +10,8 @@
10	10	obj-$(CONFIG_CRYPTO_TWOFISH_X86_64) += twofish-x86_64.o
11	11	obj-$(CONFIG_CRYPTO_SALSA20_X86_64) += salsa20-x86_64.o
12	12
	13	+obj-$(CONFIG_CRYPTO_CRC32C_INTEL) += crc32c-intel.o
	14	+
13	15	aes-i586-y := aes-i586-asm_32.o aes_glue.o
14	16	twofish-i586-y := twofish-i586-asm_32.o twofish_glue.o
15	17	salsa20-i586-y := salsa20-i586-asm_32.o salsa20_glue.o
	1	+/*
	2	+ * Using hardware provided CRC32 instruction to accelerate the CRC32 disposal.
	3	+ * CRC32C polynomial:0x1EDC6F41(BE)/0x82F63B78(LE)
	4	+ * CRC32 is a new instruction in Intel SSE4.2, the reference can be found at:
	5	+ * http://www.intel.com/products/processor/manuals/
	6	+ * Intel(R) 64 and IA-32 Architectures Software Developer's Manual
	7	+ * Volume 2A: Instruction Set Reference, A-M
	8	+ *
	9	+ * Copyright (c) 2008 Austin Zhang <austin_zhang@linux.intel.com>
	10	+ * Copyright (c) 2008 Kent Liu <kent.liu@intel.com>
	11	+ *
	12	+ * This program is free software; you can redistribute it and/or modify it
	13	+ * under the terms of the GNU General Public License as published by the Free
	14	+ * Software Foundation; either version 2 of the License, or (at your option)
	15	+ * any later version.
	16	+ *
	17	+ */
	18	+#include <linux/init.h>
	19	+#include <linux/module.h>
	20	+#include <linux/string.h>
	21	+#include <linux/kernel.h>
	22	+#include <crypto/internal/hash.h>
	23	+
	24	+#include <asm/cpufeature.h>
	25	+
	26	+#define CHKSUM_BLOCK_SIZE 1
	27	+#define CHKSUM_DIGEST_SIZE 4
	28	+
	29	+#define SCALE_F sizeof(unsigned long)
	30	+
	31	+#ifdef CONFIG_X86_64
	32	+#define REX_PRE "0x48, "
	33	+#else
	34	+#define REX_PRE
	35	+#endif
	36	+
	37	+static u32 crc32c_intel_le_hw_byte(u32 crc, unsigned char const *data, size_t length)
	38	+{
	39	+ while (length--) {
	40	+ __asm__ __volatile__(
	41	+ ".byte 0xf2, 0xf, 0x38, 0xf0, 0xf1"
	42	+ :"=S"(crc)
	43	+ :"0"(crc), "c"(*data)
	44	+ );
	45	+ data++;
	46	+ }
	47	+
	48	+ return crc;
	49	+}
	50	+
	51	+static u32 __pure crc32c_intel_le_hw(u32 crc, unsigned char const *p, size_t len)
	52	+{
	53	+ unsigned int iquotient = len / SCALE_F;
	54	+ unsigned int iremainder = len % SCALE_F;
	55	+ unsigned long ptmp = (unsigned long )p;
	56	+
	57	+ while (iquotient--) {
	58	+ __asm__ __volatile__(
	59	+ ".byte 0xf2, " REX_PRE "0xf, 0x38, 0xf1, 0xf1;"
	60	+ :"=S"(crc)
	61	+ :"0"(crc), "c"(*ptmp)
	62	+ );
	63	+ ptmp++;
	64	+ }
	65	+
	66	+ if (iremainder)
	67	+ crc = crc32c_intel_le_hw_byte(crc, (unsigned char *)ptmp,
	68	+ iremainder);
	69	+
	70	+ return crc;
	71	+}
	72	+
	73	+/*
	74	+ * Setting the seed allows arbitrary accumulators and flexible XOR policy
	75	+ * If your algorithm starts with ~0, then XOR with ~0 before you set
	76	+ * the seed.
	77	+ */
	78	+static int crc32c_intel_setkey(struct crypto_ahash hash, const u8 key,
	79	+ unsigned int keylen)
	80	+{
	81	+ u32 *mctx = crypto_ahash_ctx(hash);
	82	+
	83	+ if (keylen != sizeof(u32)) {
	84	+ crypto_ahash_set_flags(hash, CRYPTO_TFM_RES_BAD_KEY_LEN);
	85	+ return -EINVAL;
	86	+ }
	87	+ mctx = le32_to_cpup((__le32 )key);
	88	+ return 0;
	89	+}
	90	+
	91	+static int crc32c_intel_init(struct ahash_request *req)
	92	+{
	93	+ u32 *mctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req));
	94	+ u32 *crcp = ahash_request_ctx(req);
	95	+
	96	+ crcp = mctx;
	97	+
	98	+ return 0;
	99	+}
	100	+
	101	+static int crc32c_intel_update(struct ahash_request *req)
	102	+{
	103	+ struct crypto_hash_walk walk;
	104	+ u32 *crcp = ahash_request_ctx(req);
	105	+ u32 crc = *crcp;
	106	+ int nbytes;
	107	+
	108	+ for (nbytes = crypto_hash_walk_first(req, &walk); nbytes;
	109	+ nbytes = crypto_hash_walk_done(&walk, 0))
	110	+ crc = crc32c_intel_le_hw(crc, walk.data, nbytes);
	111	+
	112	+ *crcp = crc;
	113	+ return 0;
	114	+}
	115	+
	116	+static int crc32c_intel_final(struct ahash_request *req)
	117	+{
	118	+ u32 *crcp = ahash_request_ctx(req);
	119	+
	120	+ (__le32 )req->result = ~cpu_to_le32p(crcp);
	121	+ return 0;
	122	+}
	123	+
	124	+static int crc32c_intel_digest(struct ahash_request *req)
	125	+{
	126	+ struct crypto_hash_walk walk;
	127	+ u32 *mctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req));
	128	+ u32 crc = *mctx;
	129	+ int nbytes;
	130	+
	131	+ for (nbytes = crypto_hash_walk_first(req, &walk); nbytes;
	132	+ nbytes = crypto_hash_walk_done(&walk, 0))
	133	+ crc = crc32c_intel_le_hw(crc, walk.data, nbytes);
	134	+
	135	+ (__le32 )req->result = ~cpu_to_le32(crc);
	136	+ return 0;
	137	+}
	138	+
	139	+static int crc32c_intel_cra_init(struct crypto_tfm *tfm)
	140	+{
	141	+ u32 *key = crypto_tfm_ctx(tfm);
	142	+
	143	+ *key = ~0;
	144	+
	145	+ tfm->crt_ahash.reqsize = sizeof(u32);
	146	+
	147	+ return 0;
	148	+}
	149	+
	150	+static struct crypto_alg alg = {
	151	+ .cra_name = "crc32c",
	152	+ .cra_driver_name = "crc32c-intel",
	153	+ .cra_priority = 200,
	154	+ .cra_flags = CRYPTO_ALG_TYPE_AHASH,
	155	+ .cra_blocksize = CHKSUM_BLOCK_SIZE,
	156	+ .cra_alignmask = 3,
	157	+ .cra_ctxsize = sizeof(u32),
	158	+ .cra_module = THIS_MODULE,
	159	+ .cra_list = LIST_HEAD_INIT(alg.cra_list),
	160	+ .cra_init = crc32c_intel_cra_init,
	161	+ .cra_type = &crypto_ahash_type,
	162	+ .cra_u = {
	163	+ .ahash = {
	164	+ .digestsize = CHKSUM_DIGEST_SIZE,
	165	+ .setkey = crc32c_intel_setkey,
	166	+ .init = crc32c_intel_init,
	167	+ .update = crc32c_intel_update,
	168	+ .final = crc32c_intel_final,
	169	+ .digest = crc32c_intel_digest,
	170	+ }
	171	+ }
	172	+};
	173	+
	174	+
	175	+static int __init crc32c_intel_mod_init(void)
	176	+{
	177	+ if (cpu_has_xmm4_2)
	178	+ return crypto_register_alg(&alg);
	179	+ else
	180	+ return -ENODEV;
	181	+}
	182	+
	183	+static void __exit crc32c_intel_mod_fini(void)
	184	+{
	185	+ crypto_unregister_alg(&alg);
	186	+}
	187	+
	188	+module_init(crc32c_intel_mod_init);
	189	+module_exit(crc32c_intel_mod_fini);
	190	+
	191	+MODULE_AUTHOR("Austin Zhang <austin.zhang@intel.com>, Kent Liu <kent.liu@intel.com>");
	192	+MODULE_DESCRIPTION("CRC32c (Castagnoli) optimization using Intel Hardware.");
	193	+MODULE_LICENSE("GPL");
	194	+
	195	+MODULE_ALIAS("crc32c");
	196	+MODULE_ALIAS("crc32c-intel");
...	...	@@ -221,6 +221,18 @@
221	221	See Castagnoli93. This implementation uses lib/libcrc32c.
222	222	Module will be crc32c.
223	223
	224	+config CRYPTO_CRC32C_INTEL
	225	+ tristate "CRC32c INTEL hardware acceleration"
	226	+ depends on X86
	227	+ select CRYPTO_HASH
	228	+ help
	229	+ In Intel processor with SSE4.2 supported, the processor will
	230	+ support CRC32C implementation using hardware accelerated CRC32
	231	+ instruction. This option will create 'crc32c-intel' module,
	232	+ which will enable any routine to use the CRC32 instruction to
	233	+ gain performance compared with software implementation.
	234	+ Module will be crc32c-intel.
	235	+
224	236	config CRYPTO_MD4
225	237	tristate "MD4 digest algorithm"
226	238	select CRYPTO_ALGAPI