lastdragon/linux-kernel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Resguardo inicial

Browse Source
This commit is contained in:
lastdragon
2026-09-06 12:53:21 -06:00
parent 662efe3c2a
commit ad2088d178
94853 changed files with 43200404 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
linux-7.2.3/arch/arm64/crypto/.gitignore
+4
View File
@@ -0,0 +1,4 @@
# SPDX-License-Identifier: GPL-2.0-only
sha256-core.S
sha512-core.S
poly1305-core.S
linux-7.2.3/arch/arm64/crypto/Kconfig
+167
View File
@@ -0,0 +1,167 @@
# SPDX-License-Identifier: GPL-2.0
menu "Accelerated Cryptographic Algorithms for CPU (arm64)"
config CRYPTO_GHASH_ARM64_CE
tristate "AEAD cipher: AES in GCM mode (ARMv8 Crypto Extensions)"
depends on KERNEL_MODE_NEON
select CRYPTO_LIB_AES
select CRYPTO_LIB_GF128MUL
select CRYPTO_AEAD
help
AEAD cipher: AES-GCM
Architecture: arm64 using:
- ARMv8 Crypto Extensions
config CRYPTO_AES_ARM64_CE_BLK
tristate "Ciphers: AES, modes: ECB/CBC/CTR/XTS (ARMv8 Crypto Extensions)"
depends on KERNEL_MODE_NEON
select CRYPTO_SKCIPHER
select CRYPTO_LIB_AES
select CRYPTO_LIB_SHA256
help
Length-preserving ciphers: AES cipher algorithms (FIPS-197)
with block cipher modes:
- ECB (Electronic Codebook) mode (NIST SP800-38A)
- CBC (Cipher Block Chaining) mode (NIST SP800-38A)
- CTR (Counter) mode (NIST SP800-38A)
- XTS (XOR Encrypt XOR with ciphertext stealing) mode (NIST SP800-38E
and IEEE 1619)
Architecture: arm64 using:
- ARMv8 Crypto Extensions
config CRYPTO_AES_ARM64_NEON_BLK
tristate "Ciphers: AES, modes: ECB/CBC/CTR/XTS (NEON)"
depends on KERNEL_MODE_NEON
select CRYPTO_SKCIPHER
select CRYPTO_LIB_AES
select CRYPTO_LIB_SHA256
help
Length-preserving ciphers: AES cipher algorithms (FIPS-197)
with block cipher modes:
- ECB (Electronic Codebook) mode (NIST SP800-38A)
- CBC (Cipher Block Chaining) mode (NIST SP800-38A)
- CTR (Counter) mode (NIST SP800-38A)
- XTS (XOR Encrypt XOR with ciphertext stealing) mode (NIST SP800-38E
and IEEE 1619)
Architecture: arm64 using:
- NEON (Advanced SIMD) extensions
config CRYPTO_AES_ARM64_BS
tristate "Ciphers: AES, modes: ECB/CBC/CTR/XCTR/XTS modes (bit-sliced NEON)"
depends on KERNEL_MODE_NEON
select CRYPTO_SKCIPHER
select CRYPTO_AES_ARM64_NEON_BLK
select CRYPTO_LIB_AES
help
Length-preserving ciphers: AES cipher algorithms (FIPS-197)
with block cipher modes:
- ECB (Electronic Codebook) mode (NIST SP800-38A)
- CBC (Cipher Block Chaining) mode (NIST SP800-38A)
- CTR (Counter) mode (NIST SP800-38A)
- XCTR mode for HCTR2
- XTS (XOR Encrypt XOR with ciphertext stealing) mode (NIST SP800-38E
and IEEE 1619)
Architecture: arm64 using:
- bit-sliced algorithm
- NEON (Advanced SIMD) extensions
config CRYPTO_SM4_ARM64_CE
tristate "Ciphers: SM4 (ARMv8.2 Crypto Extensions)"
depends on KERNEL_MODE_NEON
select CRYPTO_ALGAPI
select CRYPTO_SM4
help
Block ciphers: SM4 cipher algorithms (OSCCA GB/T 32907-2016)
Architecture: arm64 using:
- ARMv8.2 Crypto Extensions
- NEON (Advanced SIMD) extensions
config CRYPTO_SM4_ARM64_CE_BLK
tristate "Ciphers: SM4, modes: ECB/CBC/CTR/XTS (ARMv8 Crypto Extensions)"
depends on KERNEL_MODE_NEON
select CRYPTO_SKCIPHER
select CRYPTO_SM4
help
Length-preserving ciphers: SM4 cipher algorithms (OSCCA GB/T 32907-2016)
with block cipher modes:
- ECB (Electronic Codebook) mode (NIST SP800-38A)
- CBC (Cipher Block Chaining) mode (NIST SP800-38A)
- CTR (Counter) mode (NIST SP800-38A)
- XTS (XOR Encrypt XOR with ciphertext stealing) mode (NIST SP800-38E
and IEEE 1619)
Architecture: arm64 using:
- ARMv8 Crypto Extensions
- NEON (Advanced SIMD) extensions
config CRYPTO_SM4_ARM64_NEON_BLK
tristate "Ciphers: SM4, modes: ECB/CBC/CTR (NEON)"
depends on KERNEL_MODE_NEON
select CRYPTO_SKCIPHER
select CRYPTO_SM4
help
Length-preserving ciphers: SM4 cipher algorithms (OSCCA GB/T 32907-2016)
with block cipher modes:
- ECB (Electronic Codebook) mode (NIST SP800-38A)
- CBC (Cipher Block Chaining) mode (NIST SP800-38A)
- CTR (Counter) mode (NIST SP800-38A)
Architecture: arm64 using:
- NEON (Advanced SIMD) extensions
config CRYPTO_AES_ARM64_CE_CCM
tristate "AEAD cipher: AES in CCM mode (ARMv8 Crypto Extensions)"
depends on KERNEL_MODE_NEON
select CRYPTO_ALGAPI
select CRYPTO_AES_ARM64_CE_BLK
select CRYPTO_AEAD
select CRYPTO_LIB_AES_CBC_MACS
help
AEAD cipher: AES cipher algorithms (FIPS-197) with
CCM (Counter with Cipher Block Chaining-Message Authentication Code)
authenticated encryption mode (NIST SP800-38C)
Architecture: arm64 using:
- ARMv8 Crypto Extensions
- NEON (Advanced SIMD) extensions
config CRYPTO_SM4_ARM64_CE_CCM
tristate "AEAD cipher: SM4 in CCM mode (ARMv8 Crypto Extensions)"
depends on KERNEL_MODE_NEON
select CRYPTO_ALGAPI
select CRYPTO_AEAD
select CRYPTO_SM4
select CRYPTO_SM4_ARM64_CE_BLK
help
AEAD cipher: SM4 cipher algorithms (OSCCA GB/T 32907-2016) with
CCM (Counter with Cipher Block Chaining-Message Authentication Code)
authenticated encryption mode (NIST SP800-38C)
Architecture: arm64 using:
- ARMv8 Crypto Extensions
- NEON (Advanced SIMD) extensions
config CRYPTO_SM4_ARM64_CE_GCM
tristate "AEAD cipher: SM4 in GCM mode (ARMv8 Crypto Extensions)"
depends on KERNEL_MODE_NEON
select CRYPTO_ALGAPI
select CRYPTO_AEAD
select CRYPTO_SM4
select CRYPTO_SM4_ARM64_CE_BLK
help
AEAD cipher: SM4 cipher algorithms (OSCCA GB/T 32907-2016) with
GCM (Galois/Counter Mode) authenticated encryption mode (NIST SP800-38D)
Architecture: arm64 using:
- ARMv8 Crypto Extensions
- PMULL (Polynomial Multiply Long) instructions
- NEON (Advanced SIMD) extensions
endmenu
linux-7.2.3/arch/arm64/crypto/Makefile
+36
View File
@@ -0,0 +1,36 @@
# SPDX-License-Identifier: GPL-2.0-only
#
# linux/arch/arm64/crypto/Makefile
#
# Copyright (C) 2014 Linaro Ltd <ard.biesheuvel@linaro.org>
#
obj-$(CONFIG_CRYPTO_SM4_ARM64_CE) += sm4-ce-cipher.o
sm4-ce-cipher-y := sm4-ce-cipher-glue.o sm4-ce-cipher-core.o
obj-$(CONFIG_CRYPTO_SM4_ARM64_CE_BLK) += sm4-ce.o
sm4-ce-y := sm4-ce-glue.o sm4-ce-core.o
obj-$(CONFIG_CRYPTO_SM4_ARM64_CE_CCM) += sm4-ce-ccm.o
sm4-ce-ccm-y := sm4-ce-ccm-glue.o sm4-ce-ccm-core.o
obj-$(CONFIG_CRYPTO_SM4_ARM64_CE_GCM) += sm4-ce-gcm.o
sm4-ce-gcm-y := sm4-ce-gcm-glue.o sm4-ce-gcm-core.o
obj-$(CONFIG_CRYPTO_SM4_ARM64_NEON_BLK) += sm4-neon.o
sm4-neon-y := sm4-neon-glue.o sm4-neon-core.o
obj-$(CONFIG_CRYPTO_GHASH_ARM64_CE) += ghash-ce.o
ghash-ce-y := ghash-ce-glue.o ghash-ce-core.o
obj-$(CONFIG_CRYPTO_AES_ARM64_CE_CCM) += aes-ce-ccm.o
aes-ce-ccm-y := aes-ce-ccm-glue.o aes-ce-ccm-core.o
obj-$(CONFIG_CRYPTO_AES_ARM64_CE_BLK) += aes-ce-blk.o
aes-ce-blk-y := aes-glue-ce.o
obj-$(CONFIG_CRYPTO_AES_ARM64_NEON_BLK) += aes-neon-blk.o
aes-neon-blk-y := aes-glue-neon.o
obj-$(CONFIG_CRYPTO_AES_ARM64_BS) += aes-neon-bs.o
aes-neon-bs-y := aes-neonbs-core.o aes-neonbs-glue.o
linux-7.2.3/arch/arm64/crypto/aes-ce-ccm-core.S
+142
View File
@@ -0,0 +1,142 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* aes-ce-ccm-core.S - AES-CCM transform for ARMv8 with Crypto Extensions
*
* Copyright (C) 2013 - 2017 Linaro Ltd.
* Copyright (C) 2024 Google LLC
*
* Author: Ard Biesheuvel <ardb@kernel.org>
*/
#include <linux/linkage.h>
#include <asm/assembler.h>
.text
.arch armv8-a+crypto
.macro load_round_keys, rk, nr, tmp
sub w\tmp, \nr, #10
add \tmp, \rk, w\tmp, sxtw #4
ld1 {v10.4s-v13.4s}, [\rk]
ld1 {v14.4s-v17.4s}, [\tmp], #64
ld1 {v18.4s-v21.4s}, [\tmp], #64
ld1 {v3.4s-v5.4s}, [\tmp]
.endm
.macro dround, va, vb, vk
aese \va\().16b, \vk\().16b
aesmc \va\().16b, \va\().16b
aese \vb\().16b, \vk\().16b
aesmc \vb\().16b, \vb\().16b
.endm
.macro aes_encrypt, va, vb, nr
tbz \nr, #2, .L\@
dround \va, \vb, v10
dround \va, \vb, v11
tbz \nr, #1, .L\@
dround \va, \vb, v12
dround \va, \vb, v13
.L\@: .irp v, v14, v15, v16, v17, v18, v19, v20, v21, v3
dround \va, \vb, \v
.endr
aese \va\().16b, v4.16b
aese \vb\().16b, v4.16b
.endm
.macro aes_ccm_do_crypt,enc
load_round_keys x3, w4, x10
ld1 {v0.16b}, [x5] /* load mac */
cbz x2, ce_aes_ccm_final
ldr x8, [x6, #8] /* load lower ctr */
CPU_LE( rev x8, x8 ) /* keep swabbed ctr in reg */
0: /* outer loop */
ld1 {v1.8b}, [x6] /* load upper ctr */
prfm pldl1strm, [x1]
add x8, x8, #1
rev x9, x8
ins v1.d[1], x9 /* no carry in lower ctr */
aes_encrypt v0, v1, w4
subs w2, w2, #16
bmi ce_aes_ccm_crypt_tail
ld1 {v2.16b}, [x1], #16 /* load next input block */
.if \enc == 1
eor v2.16b, v2.16b, v5.16b /* final round enc+mac */
eor v6.16b, v1.16b, v2.16b /* xor with crypted ctr */
.else
eor v2.16b, v2.16b, v1.16b /* xor with crypted ctr */
eor v6.16b, v2.16b, v5.16b /* final round enc */
.endif
eor v0.16b, v0.16b, v2.16b /* xor mac with pt ^ rk[last] */
st1 {v6.16b}, [x0], #16 /* write output block */
bne 0b
CPU_LE( rev x8, x8 )
str x8, [x6, #8] /* store lsb end of ctr (BE) */
cbnz x7, ce_aes_ccm_final
st1 {v0.16b}, [x5] /* store mac */
ret
.endm
SYM_FUNC_START_LOCAL(ce_aes_ccm_crypt_tail)
eor v0.16b, v0.16b, v5.16b /* final round mac */
eor v1.16b, v1.16b, v5.16b /* final round enc */
add x1, x1, w2, sxtw /* rewind the input pointer (w2 < 0) */
add x0, x0, w2, sxtw /* rewind the output pointer */
adr_l x8, .Lpermute /* load permute vectors */
add x9, x8, w2, sxtw
sub x8, x8, w2, sxtw
ld1 {v7.16b-v8.16b}, [x9]
ld1 {v9.16b}, [x8]
ld1 {v2.16b}, [x1] /* load a full block of input */
tbl v1.16b, {v1.16b}, v7.16b /* move keystream to end of register */
eor v7.16b, v2.16b, v1.16b /* encrypt partial input block */
bif v2.16b, v7.16b, v22.16b /* select plaintext */
tbx v7.16b, {v6.16b}, v8.16b /* insert output from previous iteration */
tbl v2.16b, {v2.16b}, v9.16b /* copy plaintext to start of v2 */
eor v0.16b, v0.16b, v2.16b /* fold plaintext into mac */
st1 {v7.16b}, [x0] /* store output block */
cbz x7, 0f
SYM_INNER_LABEL(ce_aes_ccm_final, SYM_L_LOCAL)
ld1 {v1.16b}, [x7] /* load 1st ctriv */
aes_encrypt v0, v1, w4
/* final round key cancels out */
eor v0.16b, v0.16b, v1.16b /* en-/decrypt the mac */
0: st1 {v0.16b}, [x5] /* store result */
ret
SYM_FUNC_END(ce_aes_ccm_crypt_tail)
/*
* void ce_aes_ccm_encrypt(u8 out[], u8 const in[], u32 cbytes,
* u8 const rk[], u32 rounds, u8 mac[],
* u8 ctr[], u8 const final_iv[]);
* void ce_aes_ccm_decrypt(u8 out[], u8 const in[], u32 cbytes,
* u8 const rk[], u32 rounds, u8 mac[],
* u8 ctr[], u8 const final_iv[]);
*/
SYM_FUNC_START(ce_aes_ccm_encrypt)
movi v22.16b, #255
aes_ccm_do_crypt 1
SYM_FUNC_END(ce_aes_ccm_encrypt)
SYM_FUNC_START(ce_aes_ccm_decrypt)
movi v22.16b, #0
aes_ccm_do_crypt 0
SYM_FUNC_END(ce_aes_ccm_decrypt)
.section ".rodata", "a"
.align 6
.fill 15, 1, 0xff
.Lpermute:
.byte 0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7
.byte 0x8, 0x9, 0xa, 0xb, 0xc, 0xd, 0xe, 0xf
.fill 15, 1, 0xff
linux-7.2.3/arch/arm64/crypto/aes-ce-ccm-glue.c
+324
View File
@@ -0,0 +1,324 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* aes-ce-ccm-glue.c - AES-CCM transform for ARMv8 with Crypto Extensions
*
* Copyright (C) 2013 - 2017 Linaro Ltd.
* Copyright (C) 2024 Google LLC
*
* Author: Ard Biesheuvel <ardb@kernel.org>
*/
#include <linux/unaligned.h>
#include <crypto/aes.h>
#include <crypto/scatterwalk.h>
#include <crypto/internal/aead.h>
#include <crypto/internal/skcipher.h>
#include <linux/module.h>
#include <asm/simd.h>
MODULE_IMPORT_NS("CRYPTO_INTERNAL");
static int num_rounds(struct crypto_aes_ctx *ctx)
{
/*
* # of rounds specified by AES:
* 128 bit key 10 rounds
* 192 bit key 12 rounds
* 256 bit key 14 rounds
* => n byte key => 6 + (n/4) rounds
*/
return 6 + ctx->key_length / 4;
}
asmlinkage void ce_aes_ccm_encrypt(u8 out[], u8 const in[], u32 cbytes,
u32 const rk[], u32 rounds, u8 mac[],
u8 ctr[], u8 const final_iv[]);
asmlinkage void ce_aes_ccm_decrypt(u8 out[], u8 const in[], u32 cbytes,
u32 const rk[], u32 rounds, u8 mac[],
u8 ctr[], u8 const final_iv[]);
static int ccm_setkey(struct crypto_aead *tfm, const u8 *in_key,
unsigned int key_len)
{
struct crypto_aes_ctx *ctx = crypto_aead_ctx(tfm);
return ce_aes_expandkey(ctx, in_key, key_len);
}
static int ccm_setauthsize(struct crypto_aead *tfm, unsigned int authsize)
{
if ((authsize & 1) || authsize < 4)
return -EINVAL;
return 0;
}
static int ccm_init_mac(struct aead_request *req, u8 maciv[], u32 msglen)
{
struct crypto_aead *aead = crypto_aead_reqtfm(req);
__be32 *n = (__be32 *)&maciv[AES_BLOCK_SIZE - 8];
u32 l = req->iv[0] + 1;
/* verify that CCM dimension 'L' is set correctly in the IV */
if (l < 2 || l > 8)
return -EINVAL;
/* verify that msglen can in fact be represented in L bytes */
if (l < 4 && msglen >> (8 * l))
return -EOVERFLOW;
/*
* Even if the CCM spec allows L values of up to 8, the Linux cryptoapi
* uses a u32 type to represent msglen so the top 4 bytes are always 0.
*/
n[0] = 0;
n[1] = cpu_to_be32(msglen);
memcpy(maciv, req->iv, AES_BLOCK_SIZE - l);
/*
* Meaning of byte 0 according to CCM spec (RFC 3610/NIST 800-38C)
* - bits 0..2 : max # of bytes required to represent msglen, minus 1
* (already set by caller)
* - bits 3..5 : size of auth tag (1 => 4 bytes, 2 => 6 bytes, etc)
* - bit 6 : indicates presence of authenticate-only data
*/
maciv[0] |= (crypto_aead_authsize(aead) - 2) << 2;
if (req->assoclen)
maciv[0] |= 0x40;
memset(&req->iv[AES_BLOCK_SIZE - l], 0, l);
return 0;
}
static u32 ce_aes_ccm_auth_data(u8 mac[], u8 const in[], u32 abytes,
u32 macp, u32 const rk[], u32 rounds)
{
int enc_after = (macp + abytes) % AES_BLOCK_SIZE;
do {
u32 blocks = abytes / AES_BLOCK_SIZE;
if (macp == AES_BLOCK_SIZE || (!macp && blocks > 0)) {
ce_aes_mac_update(in, rk, rounds, blocks, mac, macp,
enc_after);
macp = enc_after ? 0 : AES_BLOCK_SIZE;
in += blocks * AES_BLOCK_SIZE;
abytes -= blocks * AES_BLOCK_SIZE;
} else {
u32 l = min(AES_BLOCK_SIZE - macp, abytes);
crypto_xor(&mac[macp], in, l);
in += l;
macp += l;
abytes -= l;
}
} while (abytes > 0);
return macp;
}
static void ccm_calculate_auth_mac(struct aead_request *req, u8 mac[])
{
struct crypto_aead *aead = crypto_aead_reqtfm(req);
struct crypto_aes_ctx *ctx = crypto_aead_ctx(aead);
struct __packed { __be16 l; __be32 h; u16 len; } ltag;
struct scatter_walk walk;
u32 len = req->assoclen;
u32 macp = AES_BLOCK_SIZE;
/* prepend the AAD with a length tag */
if (len < 0xff00) {
ltag.l = cpu_to_be16(len);
ltag.len = 2;
} else {
ltag.l = cpu_to_be16(0xfffe);
put_unaligned_be32(len, &ltag.h);
ltag.len = 6;
}
macp = ce_aes_ccm_auth_data(mac, (u8 *)&ltag, ltag.len, macp,
ctx->key_enc, num_rounds(ctx));
scatterwalk_start(&walk, req->src);
do {
unsigned int n;
n = scatterwalk_next(&walk, len);
macp = ce_aes_ccm_auth_data(mac, walk.addr, n, macp,
ctx->key_enc, num_rounds(ctx));
scatterwalk_done_src(&walk, n);
len -= n;
} while (len);
}
static int ccm_encrypt(struct aead_request *req)
{
struct crypto_aead *aead = crypto_aead_reqtfm(req);
struct crypto_aes_ctx *ctx = crypto_aead_ctx(aead);
struct skcipher_walk walk;
u8 __aligned(8) mac[AES_BLOCK_SIZE];
u8 orig_iv[AES_BLOCK_SIZE];
u32 len = req->cryptlen;
int err;
err = ccm_init_mac(req, mac, len);
if (err)
return err;
/* preserve the original iv for the final round */
memcpy(orig_iv, req->iv, AES_BLOCK_SIZE);
err = skcipher_walk_aead_encrypt(&walk, req, false);
if (unlikely(err))
return err;
scoped_ksimd() {
if (req->assoclen)
ccm_calculate_auth_mac(req, mac);
do {
u32 tail = walk.nbytes % AES_BLOCK_SIZE;
const u8 *src = walk.src.virt.addr;
u8 *dst = walk.dst.virt.addr;
u8 buf[AES_BLOCK_SIZE];
u8 *final_iv = NULL;
if (walk.nbytes == walk.total) {
tail = 0;
final_iv = orig_iv;
}
if (unlikely(walk.nbytes < AES_BLOCK_SIZE))
src = dst = memcpy(&buf[sizeof(buf) - walk.nbytes],
src, walk.nbytes);
ce_aes_ccm_encrypt(dst, src, walk.nbytes - tail,
ctx->key_enc, num_rounds(ctx),
mac, walk.iv, final_iv);
if (unlikely(walk.nbytes < AES_BLOCK_SIZE))
memcpy(walk.dst.virt.addr, dst, walk.nbytes);
if (walk.nbytes) {
err = skcipher_walk_done(&walk, tail);
}
} while (walk.nbytes);
}
if (unlikely(err))
return err;
/* copy authtag to end of dst */
scatterwalk_map_and_copy(mac, req->dst, req->assoclen + req->cryptlen,
crypto_aead_authsize(aead), 1);
return 0;
}
static int ccm_decrypt(struct aead_request *req)
{
struct crypto_aead *aead = crypto_aead_reqtfm(req);
struct crypto_aes_ctx *ctx = crypto_aead_ctx(aead);
unsigned int authsize = crypto_aead_authsize(aead);
struct skcipher_walk walk;
u8 __aligned(8) mac[AES_BLOCK_SIZE];
u8 orig_iv[AES_BLOCK_SIZE];
u32 len = req->cryptlen - authsize;
int err;
err = ccm_init_mac(req, mac, len);
if (err)
return err;
/* preserve the original iv for the final round */
memcpy(orig_iv, req->iv, AES_BLOCK_SIZE);
err = skcipher_walk_aead_decrypt(&walk, req, false);
if (unlikely(err))
return err;
scoped_ksimd() {
if (req->assoclen)
ccm_calculate_auth_mac(req, mac);
do {
u32 tail = walk.nbytes % AES_BLOCK_SIZE;
const u8 *src = walk.src.virt.addr;
u8 *dst = walk.dst.virt.addr;
u8 buf[AES_BLOCK_SIZE];
u8 *final_iv = NULL;
if (walk.nbytes == walk.total) {
tail = 0;
final_iv = orig_iv;
}
if (unlikely(walk.nbytes < AES_BLOCK_SIZE))
src = dst = memcpy(&buf[sizeof(buf) - walk.nbytes],
src, walk.nbytes);
ce_aes_ccm_decrypt(dst, src, walk.nbytes - tail,
ctx->key_enc, num_rounds(ctx),
mac, walk.iv, final_iv);
if (unlikely(walk.nbytes < AES_BLOCK_SIZE))
memcpy(walk.dst.virt.addr, dst, walk.nbytes);
if (walk.nbytes) {
err = skcipher_walk_done(&walk, tail);
}
} while (walk.nbytes);
}
if (unlikely(err))
return err;
/* compare calculated auth tag with the stored one */
scatterwalk_map_and_copy(orig_iv, req->src,
req->assoclen + req->cryptlen - authsize,
authsize, 0);
if (crypto_memneq(mac, orig_iv, authsize))
return -EBADMSG;
return 0;
}
static struct aead_alg ccm_aes_alg = {
.base = {
.cra_name = "ccm(aes)",
.cra_driver_name = "ccm-aes-ce",
.cra_priority = 300,
.cra_blocksize = 1,
.cra_ctxsize = sizeof(struct crypto_aes_ctx),
.cra_module = THIS_MODULE,
},
.ivsize = AES_BLOCK_SIZE,
.chunksize = AES_BLOCK_SIZE,
.maxauthsize = AES_BLOCK_SIZE,
.setkey = ccm_setkey,
.setauthsize = ccm_setauthsize,
.encrypt = ccm_encrypt,
.decrypt = ccm_decrypt,
};
static int __init aes_mod_init(void)
{
if (!cpu_have_named_feature(AES))
return -ENODEV;
return crypto_register_aead(&ccm_aes_alg);
}
static void __exit aes_mod_exit(void)
{
crypto_unregister_aead(&ccm_aes_alg);
}
module_init(aes_mod_init);
module_exit(aes_mod_exit);
MODULE_DESCRIPTION("Synchronous AES in CCM mode using ARMv8 Crypto Extensions");
MODULE_AUTHOR("Ard Biesheuvel <ardb@kernel.org>");
MODULE_LICENSE("GPL v2");
MODULE_ALIAS_CRYPTO("ccm(aes)");
linux-7.2.3/arch/arm64/crypto/aes-glue-ce.c
+2
View File
@@ -0,0 +1,2 @@
#define USE_V8_CRYPTO_EXTENSIONS
#include "aes-glue.c"
linux-7.2.3/arch/arm64/crypto/aes-glue-neon.c
+1
View File
@@ -0,0 +1 @@
#include "aes-glue.c"
linux-7.2.3/arch/arm64/crypto/aes-glue.c
+726
View File
@@ -0,0 +1,726 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* linux/arch/arm64/crypto/aes-glue.c - wrapper code for ARMv8 AES
*
* Copyright (C) 2013 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
*/
#include <crypto/aes.h>
#include <crypto/ctr.h>
#include <crypto/internal/skcipher.h>
#include <crypto/scatterwalk.h>
#include <crypto/sha2.h>
#include <crypto/utils.h>
#include <crypto/xts.h>
#include <linux/cpufeature.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/string.h>
#include <asm/hwcap.h>
#include <asm/simd.h>
#ifdef USE_V8_CRYPTO_EXTENSIONS
#define MODE "ce"
#define PRIO 300
#define aes_expandkey ce_aes_expandkey
#define aes_ecb_encrypt ce_aes_ecb_encrypt
#define aes_ecb_decrypt ce_aes_ecb_decrypt
#define aes_cbc_encrypt ce_aes_cbc_encrypt
#define aes_cbc_decrypt ce_aes_cbc_decrypt
#define aes_cbc_cts_encrypt ce_aes_cbc_cts_encrypt
#define aes_cbc_cts_decrypt ce_aes_cbc_cts_decrypt
#define aes_essiv_cbc_encrypt ce_aes_essiv_cbc_encrypt
#define aes_essiv_cbc_decrypt ce_aes_essiv_cbc_decrypt
#define aes_ctr_encrypt ce_aes_ctr_encrypt
#define aes_xctr_encrypt ce_aes_xctr_encrypt
#define aes_xts_encrypt ce_aes_xts_encrypt
#define aes_xts_decrypt ce_aes_xts_decrypt
MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS/XCTR using ARMv8 Crypto Extensions");
#else
#define MODE "neon"
#define PRIO 200
#define aes_ecb_encrypt neon_aes_ecb_encrypt
#define aes_ecb_decrypt neon_aes_ecb_decrypt
#define aes_cbc_encrypt neon_aes_cbc_encrypt
#define aes_cbc_decrypt neon_aes_cbc_decrypt
#define aes_cbc_cts_encrypt neon_aes_cbc_cts_encrypt
#define aes_cbc_cts_decrypt neon_aes_cbc_cts_decrypt
#define aes_essiv_cbc_encrypt neon_aes_essiv_cbc_encrypt
#define aes_essiv_cbc_decrypt neon_aes_essiv_cbc_decrypt
#define aes_ctr_encrypt neon_aes_ctr_encrypt
#define aes_xctr_encrypt neon_aes_xctr_encrypt
#define aes_xts_encrypt neon_aes_xts_encrypt
#define aes_xts_decrypt neon_aes_xts_decrypt
MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS/XCTR using ARMv8 NEON");
#endif
#if defined(USE_V8_CRYPTO_EXTENSIONS) || !IS_ENABLED(CONFIG_CRYPTO_AES_ARM64_BS)
MODULE_ALIAS_CRYPTO("ecb(aes)");
MODULE_ALIAS_CRYPTO("cbc(aes)");
MODULE_ALIAS_CRYPTO("ctr(aes)");
MODULE_ALIAS_CRYPTO("xts(aes)");
MODULE_ALIAS_CRYPTO("xctr(aes)");
#endif
MODULE_ALIAS_CRYPTO("cts(cbc(aes))");
MODULE_ALIAS_CRYPTO("essiv(cbc(aes),sha256)");
MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
MODULE_IMPORT_NS("CRYPTO_INTERNAL");
MODULE_LICENSE("GPL v2");
struct crypto_aes_xts_ctx {
struct crypto_aes_ctx key1;
struct crypto_aes_ctx __aligned(8) key2;
};
struct crypto_aes_essiv_cbc_ctx {
struct crypto_aes_ctx key1;
struct crypto_aes_ctx __aligned(8) key2;
};
static int skcipher_aes_setkey(struct crypto_skcipher *tfm, const u8 *in_key,
unsigned int key_len)
{
struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
return aes_expandkey(ctx, in_key, key_len);
}
static int __maybe_unused xts_set_key(struct crypto_skcipher *tfm,
const u8 *in_key, unsigned int key_len)
{
struct crypto_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
int ret;
ret = xts_verify_key(tfm, in_key, key_len);
if (ret)
return ret;
ret = aes_expandkey(&ctx->key1, in_key, key_len / 2);
if (!ret)
ret = aes_expandkey(&ctx->key2, &in_key[key_len / 2],
key_len / 2);
return ret;
}
static int __maybe_unused essiv_cbc_set_key(struct crypto_skcipher *tfm,
const u8 *in_key,
unsigned int key_len)
{
struct crypto_aes_essiv_cbc_ctx *ctx = crypto_skcipher_ctx(tfm);
u8 digest[SHA256_DIGEST_SIZE];
int ret;
ret = aes_expandkey(&ctx->key1, in_key, key_len);
if (ret)
return ret;
sha256(in_key, key_len, digest);
return aes_expandkey(&ctx->key2, digest, sizeof(digest));
}
static int __maybe_unused ecb_encrypt(struct skcipher_request *req)
{
struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
int err, rounds = 6 + ctx->key_length / 4;
struct skcipher_walk walk;
unsigned int blocks;
err = skcipher_walk_virt(&walk, req, false);
while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) {
scoped_ksimd()
aes_ecb_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
ctx->key_enc, rounds, blocks);
err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
}
return err;
}
static int __maybe_unused ecb_decrypt(struct skcipher_request *req)
{
struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
int err, rounds = 6 + ctx->key_length / 4;
struct skcipher_walk walk;
unsigned int blocks;
err = skcipher_walk_virt(&walk, req, false);
while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) {
scoped_ksimd()
aes_ecb_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
ctx->key_dec, rounds, blocks);
err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
}
return err;
}
static int cbc_encrypt_walk(struct skcipher_request *req,
struct skcipher_walk *walk)
{
struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
int err = 0, rounds = 6 + ctx->key_length / 4;
unsigned int blocks;
while ((blocks = (walk->nbytes / AES_BLOCK_SIZE))) {
scoped_ksimd()
aes_cbc_encrypt(walk->dst.virt.addr, walk->src.virt.addr,
ctx->key_enc, rounds, blocks, walk->iv);
err = skcipher_walk_done(walk, walk->nbytes % AES_BLOCK_SIZE);
}
return err;
}
static int __maybe_unused cbc_encrypt(struct skcipher_request *req)
{
struct skcipher_walk walk;
int err;
err = skcipher_walk_virt(&walk, req, false);
if (err)
return err;
return cbc_encrypt_walk(req, &walk);
}
static int cbc_decrypt_walk(struct skcipher_request *req,
struct skcipher_walk *walk)
{
struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
int err = 0, rounds = 6 + ctx->key_length / 4;
unsigned int blocks;
while ((blocks = (walk->nbytes / AES_BLOCK_SIZE))) {
scoped_ksimd()
aes_cbc_decrypt(walk->dst.virt.addr, walk->src.virt.addr,
ctx->key_dec, rounds, blocks, walk->iv);
err = skcipher_walk_done(walk, walk->nbytes % AES_BLOCK_SIZE);
}
return err;
}
static int __maybe_unused cbc_decrypt(struct skcipher_request *req)
{
struct skcipher_walk walk;
int err;
err = skcipher_walk_virt(&walk, req, false);
if (err)
return err;
return cbc_decrypt_walk(req, &walk);
}
static int cts_cbc_encrypt(struct skcipher_request *req)
{
struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
int err, rounds = 6 + ctx->key_length / 4;
int cbc_blocks = DIV_ROUND_UP(req->cryptlen, AES_BLOCK_SIZE) - 2;
struct scatterlist *src = req->src, *dst = req->dst;
struct scatterlist sg_src[2], sg_dst[2];
struct skcipher_request subreq;
struct skcipher_walk walk;
skcipher_request_set_tfm(&subreq, tfm);
skcipher_request_set_callback(&subreq, skcipher_request_flags(req),
NULL, NULL);
if (req->cryptlen <= AES_BLOCK_SIZE) {
if (req->cryptlen < AES_BLOCK_SIZE)
return -EINVAL;
cbc_blocks = 1;
}
if (cbc_blocks > 0) {
skcipher_request_set_crypt(&subreq, req->src, req->dst,
cbc_blocks * AES_BLOCK_SIZE,
req->iv);
err = skcipher_walk_virt(&walk, &subreq, false) ?:
cbc_encrypt_walk(&subreq, &walk);
if (err)
return err;
if (req->cryptlen == AES_BLOCK_SIZE)
return 0;
dst = src = scatterwalk_ffwd(sg_src, req->src, subreq.cryptlen);
if (req->dst != req->src)
dst = scatterwalk_ffwd(sg_dst, req->dst,
subreq.cryptlen);
}
/* handle ciphertext stealing */
skcipher_request_set_crypt(&subreq, src, dst,
req->cryptlen - cbc_blocks * AES_BLOCK_SIZE,
req->iv);
err = skcipher_walk_virt(&walk, &subreq, false);
if (err)
return err;
scoped_ksimd()
aes_cbc_cts_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
ctx->key_enc, rounds, walk.nbytes, walk.iv);
return skcipher_walk_done(&walk, 0);
}
static int cts_cbc_decrypt(struct skcipher_request *req)
{
struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
int err, rounds = 6 + ctx->key_length / 4;
int cbc_blocks = DIV_ROUND_UP(req->cryptlen, AES_BLOCK_SIZE) - 2;
struct scatterlist *src = req->src, *dst = req->dst;
struct scatterlist sg_src[2], sg_dst[2];
struct skcipher_request subreq;
struct skcipher_walk walk;
skcipher_request_set_tfm(&subreq, tfm);
skcipher_request_set_callback(&subreq, skcipher_request_flags(req),
NULL, NULL);
if (req->cryptlen <= AES_BLOCK_SIZE) {
if (req->cryptlen < AES_BLOCK_SIZE)
return -EINVAL;
cbc_blocks = 1;
}
if (cbc_blocks > 0) {
skcipher_request_set_crypt(&subreq, req->src, req->dst,
cbc_blocks * AES_BLOCK_SIZE,
req->iv);
err = skcipher_walk_virt(&walk, &subreq, false) ?:
cbc_decrypt_walk(&subreq, &walk);
if (err)
return err;
if (req->cryptlen == AES_BLOCK_SIZE)
return 0;
dst = src = scatterwalk_ffwd(sg_src, req->src, subreq.cryptlen);
if (req->dst != req->src)
dst = scatterwalk_ffwd(sg_dst, req->dst,
subreq.cryptlen);
}
/* handle ciphertext stealing */
skcipher_request_set_crypt(&subreq, src, dst,
req->cryptlen - cbc_blocks * AES_BLOCK_SIZE,
req->iv);
err = skcipher_walk_virt(&walk, &subreq, false);
if (err)
return err;
scoped_ksimd()
aes_cbc_cts_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
ctx->key_dec, rounds, walk.nbytes, walk.iv);
return skcipher_walk_done(&walk, 0);
}
static int __maybe_unused essiv_cbc_encrypt(struct skcipher_request *req)
{
struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
struct crypto_aes_essiv_cbc_ctx *ctx = crypto_skcipher_ctx(tfm);
int err, rounds = 6 + ctx->key1.key_length / 4;
struct skcipher_walk walk;
unsigned int blocks;
err = skcipher_walk_virt(&walk, req, false);
blocks = walk.nbytes / AES_BLOCK_SIZE;
if (blocks) {
scoped_ksimd()
aes_essiv_cbc_encrypt(walk.dst.virt.addr,
walk.src.virt.addr,
ctx->key1.key_enc, rounds, blocks,
req->iv, ctx->key2.key_enc);
err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
}
return err ?: cbc_encrypt_walk(req, &walk);
}
static int __maybe_unused essiv_cbc_decrypt(struct skcipher_request *req)
{
struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
struct crypto_aes_essiv_cbc_ctx *ctx = crypto_skcipher_ctx(tfm);
int err, rounds = 6 + ctx->key1.key_length / 4;
struct skcipher_walk walk;
unsigned int blocks;
err = skcipher_walk_virt(&walk, req, false);
blocks = walk.nbytes / AES_BLOCK_SIZE;
if (blocks) {
scoped_ksimd()
aes_essiv_cbc_decrypt(walk.dst.virt.addr,
walk.src.virt.addr,
ctx->key1.key_dec, rounds, blocks,
req->iv, ctx->key2.key_enc);
err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
}
return err ?: cbc_decrypt_walk(req, &walk);
}
static int __maybe_unused xctr_encrypt(struct skcipher_request *req)
{
struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
int err, rounds = 6 + ctx->key_length / 4;
struct skcipher_walk walk;
unsigned int byte_ctr = 0;
err = skcipher_walk_virt(&walk, req, false);
while (walk.nbytes > 0) {
const u8 *src = walk.src.virt.addr;
unsigned int nbytes = walk.nbytes;
u8 *dst = walk.dst.virt.addr;
u8 buf[AES_BLOCK_SIZE];
/*
* If given less than 16 bytes, we must copy the partial block
* into a temporary buffer of 16 bytes to avoid out of bounds
* reads and writes. Furthermore, this code is somewhat unusual
* in that it expects the end of the data to be at the end of
* the temporary buffer, rather than the start of the data at
* the start of the temporary buffer.
*/
if (unlikely(nbytes < AES_BLOCK_SIZE))
src = dst = memcpy(buf + sizeof(buf) - nbytes,
src, nbytes);
else if (nbytes < walk.total)
nbytes &= ~(AES_BLOCK_SIZE - 1);
scoped_ksimd()
aes_xctr_encrypt(dst, src, ctx->key_enc, rounds, nbytes,
walk.iv, byte_ctr);
if (unlikely(nbytes < AES_BLOCK_SIZE))
memcpy(walk.dst.virt.addr,
buf + sizeof(buf) - nbytes, nbytes);
byte_ctr += nbytes;
err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
}
return err;
}
static int __maybe_unused ctr_encrypt(struct skcipher_request *req)
{
struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
int err, rounds = 6 + ctx->key_length / 4;
struct skcipher_walk walk;
err = skcipher_walk_virt(&walk, req, false);
while (walk.nbytes > 0) {
const u8 *src = walk.src.virt.addr;
unsigned int nbytes = walk.nbytes;
u8 *dst = walk.dst.virt.addr;
u8 buf[AES_BLOCK_SIZE];
/*
* If given less than 16 bytes, we must copy the partial block
* into a temporary buffer of 16 bytes to avoid out of bounds
* reads and writes. Furthermore, this code is somewhat unusual
* in that it expects the end of the data to be at the end of
* the temporary buffer, rather than the start of the data at
* the start of the temporary buffer.
*/
if (unlikely(nbytes < AES_BLOCK_SIZE))
src = dst = memcpy(buf + sizeof(buf) - nbytes,
src, nbytes);
else if (nbytes < walk.total)
nbytes &= ~(AES_BLOCK_SIZE - 1);
scoped_ksimd()
aes_ctr_encrypt(dst, src, ctx->key_enc, rounds, nbytes,
walk.iv);
if (unlikely(nbytes < AES_BLOCK_SIZE))
memcpy(walk.dst.virt.addr,
buf + sizeof(buf) - nbytes, nbytes);
err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
}
return err;
}
static int __maybe_unused xts_encrypt(struct skcipher_request *req)
{
struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
struct crypto_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
int err, first, rounds = 6 + ctx->key1.key_length / 4;
int tail = req->cryptlen % AES_BLOCK_SIZE;
struct scatterlist sg_src[2], sg_dst[2];
struct skcipher_request subreq;
struct scatterlist *src, *dst;
struct skcipher_walk walk;
if (req->cryptlen < AES_BLOCK_SIZE)
return -EINVAL;
err = skcipher_walk_virt(&walk, req, false);
if (unlikely(tail > 0 && walk.nbytes < walk.total)) {
int xts_blocks = DIV_ROUND_UP(req->cryptlen,
AES_BLOCK_SIZE) - 2;
skcipher_walk_abort(&walk);
skcipher_request_set_tfm(&subreq, tfm);
skcipher_request_set_callback(&subreq,
skcipher_request_flags(req),
NULL, NULL);
skcipher_request_set_crypt(&subreq, req->src, req->dst,
xts_blocks * AES_BLOCK_SIZE,
req->iv);
req = &subreq;
err = skcipher_walk_virt(&walk, req, false);
} else {
tail = 0;
}
scoped_ksimd() {
for (first = 1; walk.nbytes >= AES_BLOCK_SIZE; first = 0) {
int nbytes = walk.nbytes;
if (walk.nbytes < walk.total)
nbytes &= ~(AES_BLOCK_SIZE - 1);
aes_xts_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
ctx->key1.key_enc, rounds, nbytes,
ctx->key2.key_enc, walk.iv, first);
err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
}
if (err || likely(!tail))
return err;
dst = src = scatterwalk_ffwd(sg_src, req->src, req->cryptlen);
if (req->dst != req->src)
dst = scatterwalk_ffwd(sg_dst, req->dst, req->cryptlen);
skcipher_request_set_crypt(req, src, dst, AES_BLOCK_SIZE + tail,
req->iv);
err = skcipher_walk_virt(&walk, &subreq, false);
if (err)
return err;
aes_xts_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
ctx->key1.key_enc, rounds, walk.nbytes,
ctx->key2.key_enc, walk.iv, first);
}
return skcipher_walk_done(&walk, 0);
}
static int __maybe_unused xts_decrypt(struct skcipher_request *req)
{
struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
struct crypto_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
int err, first, rounds = 6 + ctx->key1.key_length / 4;
int tail = req->cryptlen % AES_BLOCK_SIZE;
struct scatterlist sg_src[2], sg_dst[2];
struct skcipher_request subreq;
struct scatterlist *src, *dst;
struct skcipher_walk walk;
if (req->cryptlen < AES_BLOCK_SIZE)
return -EINVAL;
err = skcipher_walk_virt(&walk, req, false);
if (unlikely(tail > 0 && walk.nbytes < walk.total)) {
int xts_blocks = DIV_ROUND_UP(req->cryptlen,
AES_BLOCK_SIZE) - 2;
skcipher_walk_abort(&walk);
skcipher_request_set_tfm(&subreq, tfm);
skcipher_request_set_callback(&subreq,
skcipher_request_flags(req),
NULL, NULL);
skcipher_request_set_crypt(&subreq, req->src, req->dst,
xts_blocks * AES_BLOCK_SIZE,
req->iv);
req = &subreq;
err = skcipher_walk_virt(&walk, req, false);
} else {
tail = 0;
}
scoped_ksimd() {
for (first = 1; walk.nbytes >= AES_BLOCK_SIZE; first = 0) {
int nbytes = walk.nbytes;
if (walk.nbytes < walk.total)
nbytes &= ~(AES_BLOCK_SIZE - 1);
aes_xts_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
ctx->key1.key_dec, rounds, nbytes,
ctx->key2.key_enc, walk.iv, first);
err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
}
if (err || likely(!tail))
return err;
dst = src = scatterwalk_ffwd(sg_src, req->src, req->cryptlen);
if (req->dst != req->src)
dst = scatterwalk_ffwd(sg_dst, req->dst, req->cryptlen);
skcipher_request_set_crypt(req, src, dst, AES_BLOCK_SIZE + tail,
req->iv);
err = skcipher_walk_virt(&walk, &subreq, false);
if (err)
return err;
aes_xts_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
ctx->key1.key_dec, rounds, walk.nbytes,
ctx->key2.key_enc, walk.iv, first);
}
return skcipher_walk_done(&walk, 0);
}
static struct skcipher_alg aes_algs[] = { {
#if defined(USE_V8_CRYPTO_EXTENSIONS) || !IS_ENABLED(CONFIG_CRYPTO_AES_ARM64_BS)
.base = {
.cra_name = "ecb(aes)",
.cra_driver_name = "ecb-aes-" MODE,
.cra_priority = PRIO,
.cra_blocksize = AES_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct crypto_aes_ctx),
.cra_module = THIS_MODULE,
},
.min_keysize = AES_MIN_KEY_SIZE,
.max_keysize = AES_MAX_KEY_SIZE,
.setkey = skcipher_aes_setkey,
.encrypt = ecb_encrypt,
.decrypt = ecb_decrypt,
}, {
.base = {
.cra_name = "cbc(aes)",
.cra_driver_name = "cbc-aes-" MODE,
.cra_priority = PRIO,
.cra_blocksize = AES_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct crypto_aes_ctx),
.cra_module = THIS_MODULE,
},
.min_keysize = AES_MIN_KEY_SIZE,
.max_keysize = AES_MAX_KEY_SIZE,
.ivsize = AES_BLOCK_SIZE,
.setkey = skcipher_aes_setkey,
.encrypt = cbc_encrypt,
.decrypt = cbc_decrypt,
}, {
.base = {
.cra_name = "ctr(aes)",
.cra_driver_name = "ctr-aes-" MODE,
.cra_priority = PRIO,
.cra_blocksize = 1,
.cra_ctxsize = sizeof(struct crypto_aes_ctx),
.cra_module = THIS_MODULE,
},
.min_keysize = AES_MIN_KEY_SIZE,
.max_keysize = AES_MAX_KEY_SIZE,
.ivsize = AES_BLOCK_SIZE,
.chunksize = AES_BLOCK_SIZE,
.setkey = skcipher_aes_setkey,
.encrypt = ctr_encrypt,
.decrypt = ctr_encrypt,
}, {
.base = {
.cra_name = "xctr(aes)",
.cra_driver_name = "xctr-aes-" MODE,
.cra_priority = PRIO,
.cra_blocksize = 1,
.cra_ctxsize = sizeof(struct crypto_aes_ctx),
.cra_module = THIS_MODULE,
},
.min_keysize = AES_MIN_KEY_SIZE,
.max_keysize = AES_MAX_KEY_SIZE,
.ivsize = AES_BLOCK_SIZE,
.chunksize = AES_BLOCK_SIZE,
.setkey = skcipher_aes_setkey,
.encrypt = xctr_encrypt,
.decrypt = xctr_encrypt,
}, {
.base = {
.cra_name = "xts(aes)",
.cra_driver_name = "xts-aes-" MODE,
.cra_priority = PRIO,
.cra_blocksize = AES_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct crypto_aes_xts_ctx),
.cra_module = THIS_MODULE,
},
.min_keysize = 2 * AES_MIN_KEY_SIZE,
.max_keysize = 2 * AES_MAX_KEY_SIZE,
.ivsize = AES_BLOCK_SIZE,
.walksize = 2 * AES_BLOCK_SIZE,
.setkey = xts_set_key,
.encrypt = xts_encrypt,
.decrypt = xts_decrypt,
}, {
#endif
.base = {
.cra_name = "cts(cbc(aes))",
.cra_driver_name = "cts-cbc-aes-" MODE,
.cra_priority = PRIO,
.cra_blocksize = AES_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct crypto_aes_ctx),
.cra_module = THIS_MODULE,
},
.min_keysize = AES_MIN_KEY_SIZE,
.max_keysize = AES_MAX_KEY_SIZE,
.ivsize = AES_BLOCK_SIZE,
.walksize = 2 * AES_BLOCK_SIZE,
.setkey = skcipher_aes_setkey,
.encrypt = cts_cbc_encrypt,
.decrypt = cts_cbc_decrypt,
}, {
.base = {
.cra_name = "essiv(cbc(aes),sha256)",
.cra_driver_name = "essiv-cbc-aes-sha256-" MODE,
.cra_priority = PRIO + 1,
.cra_blocksize = AES_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct crypto_aes_essiv_cbc_ctx),
.cra_module = THIS_MODULE,
},
.min_keysize = AES_MIN_KEY_SIZE,
.max_keysize = AES_MAX_KEY_SIZE,
.ivsize = AES_BLOCK_SIZE,
.setkey = essiv_cbc_set_key,
.encrypt = essiv_cbc_encrypt,
.decrypt = essiv_cbc_decrypt,
} };
static void aes_exit(void)
{
crypto_unregister_skciphers(aes_algs, ARRAY_SIZE(aes_algs));
}
static int __init aes_init(void)
{
return crypto_register_skciphers(aes_algs, ARRAY_SIZE(aes_algs));
}
#ifdef USE_V8_CRYPTO_EXTENSIONS
module_cpu_feature_match(AES, aes_init);
#else
module_init(aes_init);
#endif
module_exit(aes_exit);
linux-7.2.3/arch/arm64/crypto/aes-neonbs-core.S
+866
View File
@@ -0,0 +1,866 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Bit sliced AES using NEON instructions
*
* Copyright (C) 2016 Linaro Ltd <ard.biesheuvel@linaro.org>
*/
/*
* The algorithm implemented here is described in detail by the paper
* 'Faster and Timing-Attack Resistant AES-GCM' by Emilia Kaesper and
* Peter Schwabe (https://eprint.iacr.org/2009/129.pdf)
*
* This implementation is based primarily on the OpenSSL implementation
* for 32-bit ARM written by Andy Polyakov <appro@openssl.org>
*/
#include <linux/linkage.h>
#include <linux/cfi_types.h>
#include <asm/assembler.h>
.text
rounds .req x11
bskey .req x12
.macro in_bs_ch, b0, b1, b2, b3, b4, b5, b6, b7
eor \b2, \b2, \b1
eor \b5, \b5, \b6
eor \b3, \b3, \b0
eor \b6, \b6, \b2
eor \b5, \b5, \b0
eor \b6, \b6, \b3
eor \b3, \b3, \b7
eor \b7, \b7, \b5
eor \b3, \b3, \b4
eor \b4, \b4, \b5
eor \b2, \b2, \b7
eor \b3, \b3, \b1
eor \b1, \b1, \b5
.endm
.macro out_bs_ch, b0, b1, b2, b3, b4, b5, b6, b7
eor \b0, \b0, \b6
eor \b1, \b1, \b4
eor \b4, \b4, \b6
eor \b2, \b2, \b0
eor \b6, \b6, \b1
eor \b1, \b1, \b5
eor \b5, \b5, \b3
eor \b3, \b3, \b7
eor \b7, \b7, \b5
eor \b2, \b2, \b5
eor \b4, \b4, \b7
.endm
.macro inv_in_bs_ch, b6, b1, b2, b4, b7, b0, b3, b5
eor \b1, \b1, \b7
eor \b4, \b4, \b7
eor \b7, \b7, \b5
eor \b1, \b1, \b3
eor \b2, \b2, \b5
eor \b3, \b3, \b7
eor \b6, \b6, \b1
eor \b2, \b2, \b0
eor \b5, \b5, \b3
eor \b4, \b4, \b6
eor \b0, \b0, \b6
eor \b1, \b1, \b4
.endm
.macro inv_out_bs_ch, b6, b5, b0, b3, b7, b1, b4, b2
eor \b1, \b1, \b5
eor \b2, \b2, \b7
eor \b3, \b3, \b1
eor \b4, \b4, \b5
eor \b7, \b7, \b5
eor \b3, \b3, \b4
eor \b5, \b5, \b0
eor \b3, \b3, \b7
eor \b6, \b6, \b2
eor \b2, \b2, \b1
eor \b6, \b6, \b3
eor \b3, \b3, \b0
eor \b5, \b5, \b6
.endm
.macro mul_gf4, x0, x1, y0, y1, t0, t1
eor \t0, \y0, \y1
and \t0, \t0, \x0
eor \x0, \x0, \x1
and \t1, \x1, \y0
and \x0, \x0, \y1
eor \x1, \t1, \t0
eor \x0, \x0, \t1
.endm
.macro mul_gf4_n_gf4, x0, x1, y0, y1, t0, x2, x3, y2, y3, t1
eor \t0, \y0, \y1
eor \t1, \y2, \y3
and \t0, \t0, \x0
and \t1, \t1, \x2
eor \x0, \x0, \x1
eor \x2, \x2, \x3
and \x1, \x1, \y0
and \x3, \x3, \y2
and \x0, \x0, \y1
and \x2, \x2, \y3
eor \x1, \x1, \x0
eor \x2, \x2, \x3
eor \x0, \x0, \t0
eor \x3, \x3, \t1
.endm
.macro mul_gf16_2, x0, x1, x2, x3, x4, x5, x6, x7, \
y0, y1, y2, y3, t0, t1, t2, t3
eor \t0, \x0, \x2
eor \t1, \x1, \x3
mul_gf4 \x0, \x1, \y0, \y1, \t2, \t3
eor \y0, \y0, \y2
eor \y1, \y1, \y3
mul_gf4_n_gf4 \t0, \t1, \y0, \y1, \t3, \x2, \x3, \y2, \y3, \t2
eor \x0, \x0, \t0
eor \x2, \x2, \t0
eor \x1, \x1, \t1
eor \x3, \x3, \t1
eor \t0, \x4, \x6
eor \t1, \x5, \x7
mul_gf4_n_gf4 \t0, \t1, \y0, \y1, \t3, \x6, \x7, \y2, \y3, \t2
eor \y0, \y0, \y2
eor \y1, \y1, \y3
mul_gf4 \x4, \x5, \y0, \y1, \t2, \t3
eor \x4, \x4, \t0
eor \x6, \x6, \t0
eor \x5, \x5, \t1
eor \x7, \x7, \t1
.endm
.macro inv_gf256, x0, x1, x2, x3, x4, x5, x6, x7, \
t0, t1, t2, t3, s0, s1, s2, s3
eor \t3, \x4, \x6
eor \t0, \x5, \x7
eor \t1, \x1, \x3
eor \s1, \x7, \x6
eor \s0, \x0, \x2
eor \s3, \t3, \t0
orr \t2, \t0, \t1
and \s2, \t3, \s0
orr \t3, \t3, \s0
eor \s0, \s0, \t1
and \t0, \t0, \t1
eor \t1, \x3, \x2
and \s3, \s3, \s0
and \s1, \s1, \t1
eor \t1, \x4, \x5
eor \s0, \x1, \x0
eor \t3, \t3, \s1
eor \t2, \t2, \s1
and \s1, \t1, \s0
orr \t1, \t1, \s0
eor \t3, \t3, \s3
eor \t0, \t0, \s1
eor \t2, \t2, \s2
eor \t1, \t1, \s3
eor \t0, \t0, \s2
and \s0, \x7, \x3
eor \t1, \t1, \s2
and \s1, \x6, \x2
and \s2, \x5, \x1
orr \s3, \x4, \x0
eor \t3, \t3, \s0
eor \t1, \t1, \s2
eor \s0, \t0, \s3
eor \t2, \t2, \s1
and \s2, \t3, \t1
eor \s1, \t2, \s2
eor \s3, \s0, \s2
bsl \s1, \t1, \s0
not \t0, \s0
bsl \s0, \s1, \s3
bsl \t0, \s1, \s3
bsl \s3, \t3, \t2
eor \t3, \t3, \t2
and \s2, \s0, \s3
eor \t1, \t1, \t0
eor \s2, \s2, \t3
mul_gf16_2 \x0, \x1, \x2, \x3, \x4, \x5, \x6, \x7, \
\s3, \s2, \s1, \t1, \s0, \t0, \t2, \t3
.endm
.macro sbox, b0, b1, b2, b3, b4, b5, b6, b7, \
t0, t1, t2, t3, s0, s1, s2, s3
in_bs_ch \b0\().16b, \b1\().16b, \b2\().16b, \b3\().16b, \
\b4\().16b, \b5\().16b, \b6\().16b, \b7\().16b
inv_gf256 \b6\().16b, \b5\().16b, \b0\().16b, \b3\().16b, \
\b7\().16b, \b1\().16b, \b4\().16b, \b2\().16b, \
\t0\().16b, \t1\().16b, \t2\().16b, \t3\().16b, \
\s0\().16b, \s1\().16b, \s2\().16b, \s3\().16b
out_bs_ch \b7\().16b, \b1\().16b, \b4\().16b, \b2\().16b, \
\b6\().16b, \b5\().16b, \b0\().16b, \b3\().16b
.endm
.macro inv_sbox, b0, b1, b2, b3, b4, b5, b6, b7, \
t0, t1, t2, t3, s0, s1, s2, s3
inv_in_bs_ch \b0\().16b, \b1\().16b, \b2\().16b, \b3\().16b, \
\b4\().16b, \b5\().16b, \b6\().16b, \b7\().16b
inv_gf256 \b5\().16b, \b1\().16b, \b2\().16b, \b6\().16b, \
\b3\().16b, \b7\().16b, \b0\().16b, \b4\().16b, \
\t0\().16b, \t1\().16b, \t2\().16b, \t3\().16b, \
\s0\().16b, \s1\().16b, \s2\().16b, \s3\().16b
inv_out_bs_ch \b3\().16b, \b7\().16b, \b0\().16b, \b4\().16b, \
\b5\().16b, \b1\().16b, \b2\().16b, \b6\().16b
.endm
.macro enc_next_rk
ldp q16, q17, [bskey], #128
ldp q18, q19, [bskey, #-96]
ldp q20, q21, [bskey, #-64]
ldp q22, q23, [bskey, #-32]
.endm
.macro dec_next_rk
ldp q16, q17, [bskey, #-128]!
ldp q18, q19, [bskey, #32]
ldp q20, q21, [bskey, #64]
ldp q22, q23, [bskey, #96]
.endm
.macro add_round_key, x0, x1, x2, x3, x4, x5, x6, x7
eor \x0\().16b, \x0\().16b, v16.16b
eor \x1\().16b, \x1\().16b, v17.16b
eor \x2\().16b, \x2\().16b, v18.16b
eor \x3\().16b, \x3\().16b, v19.16b
eor \x4\().16b, \x4\().16b, v20.16b
eor \x5\().16b, \x5\().16b, v21.16b
eor \x6\().16b, \x6\().16b, v22.16b
eor \x7\().16b, \x7\().16b, v23.16b
.endm
.macro shift_rows, x0, x1, x2, x3, x4, x5, x6, x7, mask
tbl \x0\().16b, {\x0\().16b}, \mask\().16b
tbl \x1\().16b, {\x1\().16b}, \mask\().16b
tbl \x2\().16b, {\x2\().16b}, \mask\().16b
tbl \x3\().16b, {\x3\().16b}, \mask\().16b
tbl \x4\().16b, {\x4\().16b}, \mask\().16b
tbl \x5\().16b, {\x5\().16b}, \mask\().16b
tbl \x6\().16b, {\x6\().16b}, \mask\().16b
tbl \x7\().16b, {\x7\().16b}, \mask\().16b
.endm
.macro mix_cols, x0, x1, x2, x3, x4, x5, x6, x7, \
t0, t1, t2, t3, t4, t5, t6, t7, inv
ext \t0\().16b, \x0\().16b, \x0\().16b, #12
ext \t1\().16b, \x1\().16b, \x1\().16b, #12
eor \x0\().16b, \x0\().16b, \t0\().16b
ext \t2\().16b, \x2\().16b, \x2\().16b, #12
eor \x1\().16b, \x1\().16b, \t1\().16b
ext \t3\().16b, \x3\().16b, \x3\().16b, #12
eor \x2\().16b, \x2\().16b, \t2\().16b
ext \t4\().16b, \x4\().16b, \x4\().16b, #12
eor \x3\().16b, \x3\().16b, \t3\().16b
ext \t5\().16b, \x5\().16b, \x5\().16b, #12
eor \x4\().16b, \x4\().16b, \t4\().16b
ext \t6\().16b, \x6\().16b, \x6\().16b, #12
eor \x5\().16b, \x5\().16b, \t5\().16b
ext \t7\().16b, \x7\().16b, \x7\().16b, #12
eor \x6\().16b, \x6\().16b, \t6\().16b
eor \t1\().16b, \t1\().16b, \x0\().16b
eor \x7\().16b, \x7\().16b, \t7\().16b
ext \x0\().16b, \x0\().16b, \x0\().16b, #8
eor \t2\().16b, \t2\().16b, \x1\().16b
eor \t0\().16b, \t0\().16b, \x7\().16b
eor \t1\().16b, \t1\().16b, \x7\().16b
ext \x1\().16b, \x1\().16b, \x1\().16b, #8
eor \t5\().16b, \t5\().16b, \x4\().16b
eor \x0\().16b, \x0\().16b, \t0\().16b
eor \t6\().16b, \t6\().16b, \x5\().16b
eor \x1\().16b, \x1\().16b, \t1\().16b
ext \t0\().16b, \x4\().16b, \x4\().16b, #8
eor \t4\().16b, \t4\().16b, \x3\().16b
ext \t1\().16b, \x5\().16b, \x5\().16b, #8
eor \t7\().16b, \t7\().16b, \x6\().16b
ext \x4\().16b, \x3\().16b, \x3\().16b, #8
eor \t3\().16b, \t3\().16b, \x2\().16b
ext \x5\().16b, \x7\().16b, \x7\().16b, #8
eor \t4\().16b, \t4\().16b, \x7\().16b
ext \x3\().16b, \x6\().16b, \x6\().16b, #8
eor \t3\().16b, \t3\().16b, \x7\().16b
ext \x6\().16b, \x2\().16b, \x2\().16b, #8
eor \x7\().16b, \t1\().16b, \t5\().16b
.ifb \inv
eor \x2\().16b, \t0\().16b, \t4\().16b
eor \x4\().16b, \x4\().16b, \t3\().16b
eor \x5\().16b, \x5\().16b, \t7\().16b
eor \x3\().16b, \x3\().16b, \t6\().16b
eor \x6\().16b, \x6\().16b, \t2\().16b
.else
eor \t3\().16b, \t3\().16b, \x4\().16b
eor \x5\().16b, \x5\().16b, \t7\().16b
eor \x2\().16b, \x3\().16b, \t6\().16b
eor \x3\().16b, \t0\().16b, \t4\().16b
eor \x4\().16b, \x6\().16b, \t2\().16b
mov \x6\().16b, \t3\().16b
.endif
.endm
.macro inv_mix_cols, x0, x1, x2, x3, x4, x5, x6, x7, \
t0, t1, t2, t3, t4, t5, t6, t7
ext \t0\().16b, \x0\().16b, \x0\().16b, #8
ext \t6\().16b, \x6\().16b, \x6\().16b, #8
ext \t7\().16b, \x7\().16b, \x7\().16b, #8
eor \t0\().16b, \t0\().16b, \x0\().16b
ext \t1\().16b, \x1\().16b, \x1\().16b, #8
eor \t6\().16b, \t6\().16b, \x6\().16b
ext \t2\().16b, \x2\().16b, \x2\().16b, #8
eor \t7\().16b, \t7\().16b, \x7\().16b
ext \t3\().16b, \x3\().16b, \x3\().16b, #8
eor \t1\().16b, \t1\().16b, \x1\().16b
ext \t4\().16b, \x4\().16b, \x4\().16b, #8
eor \t2\().16b, \t2\().16b, \x2\().16b
ext \t5\().16b, \x5\().16b, \x5\().16b, #8
eor \t3\().16b, \t3\().16b, \x3\().16b
eor \t4\().16b, \t4\().16b, \x4\().16b
eor \t5\().16b, \t5\().16b, \x5\().16b
eor \x0\().16b, \x0\().16b, \t6\().16b
eor \x1\().16b, \x1\().16b, \t6\().16b
eor \x2\().16b, \x2\().16b, \t0\().16b
eor \x4\().16b, \x4\().16b, \t2\().16b
eor \x3\().16b, \x3\().16b, \t1\().16b
eor \x1\().16b, \x1\().16b, \t7\().16b
eor \x2\().16b, \x2\().16b, \t7\().16b
eor \x4\().16b, \x4\().16b, \t6\().16b
eor \x5\().16b, \x5\().16b, \t3\().16b
eor \x3\().16b, \x3\().16b, \t6\().16b
eor \x6\().16b, \x6\().16b, \t4\().16b
eor \x4\().16b, \x4\().16b, \t7\().16b
eor \x5\().16b, \x5\().16b, \t7\().16b
eor \x7\().16b, \x7\().16b, \t5\().16b
mix_cols \x0, \x1, \x2, \x3, \x4, \x5, \x6, \x7, \
\t0, \t1, \t2, \t3, \t4, \t5, \t6, \t7, 1
.endm
.macro swapmove_2x, a0, b0, a1, b1, n, mask, t0, t1
ushr \t0\().2d, \b0\().2d, #\n
ushr \t1\().2d, \b1\().2d, #\n
eor \t0\().16b, \t0\().16b, \a0\().16b
eor \t1\().16b, \t1\().16b, \a1\().16b
and \t0\().16b, \t0\().16b, \mask\().16b
and \t1\().16b, \t1\().16b, \mask\().16b
eor \a0\().16b, \a0\().16b, \t0\().16b
shl \t0\().2d, \t0\().2d, #\n
eor \a1\().16b, \a1\().16b, \t1\().16b
shl \t1\().2d, \t1\().2d, #\n
eor \b0\().16b, \b0\().16b, \t0\().16b
eor \b1\().16b, \b1\().16b, \t1\().16b
.endm
.macro bitslice, x7, x6, x5, x4, x3, x2, x1, x0, t0, t1, t2, t3
movi \t0\().16b, #0x55
movi \t1\().16b, #0x33
swapmove_2x \x0, \x1, \x2, \x3, 1, \t0, \t2, \t3
swapmove_2x \x4, \x5, \x6, \x7, 1, \t0, \t2, \t3
movi \t0\().16b, #0x0f
swapmove_2x \x0, \x2, \x1, \x3, 2, \t1, \t2, \t3
swapmove_2x \x4, \x6, \x5, \x7, 2, \t1, \t2, \t3
swapmove_2x \x0, \x4, \x1, \x5, 4, \t0, \t2, \t3
swapmove_2x \x2, \x6, \x3, \x7, 4, \t0, \t2, \t3
.endm
.align 6
M0: .octa 0x0004080c0105090d02060a0e03070b0f
M0SR: .octa 0x0004080c05090d010a0e02060f03070b
SR: .octa 0x0f0e0d0c0a09080b0504070600030201
SRM0: .octa 0x01060b0c0207080d0304090e00050a0f
M0ISR: .octa 0x0004080c0d0105090a0e0206070b0f03
ISR: .octa 0x0f0e0d0c080b0a090504070602010003
ISRM0: .octa 0x0306090c00070a0d01040b0e0205080f
/*
* void aesbs_convert_key(u8 out[], u32 const rk[], int rounds)
*/
SYM_FUNC_START(aesbs_convert_key)
ld1 {v7.4s}, [x1], #16 // load round 0 key
ld1 {v17.4s}, [x1], #16 // load round 1 key
movi v8.16b, #0x01 // bit masks
movi v9.16b, #0x02
movi v10.16b, #0x04
movi v11.16b, #0x08
movi v12.16b, #0x10
movi v13.16b, #0x20
movi v14.16b, #0x40
movi v15.16b, #0x80
ldr q16, M0
sub x2, x2, #1
str q7, [x0], #16 // save round 0 key
.Lkey_loop:
tbl v7.16b ,{v17.16b}, v16.16b
ld1 {v17.4s}, [x1], #16 // load next round key
cmtst v0.16b, v7.16b, v8.16b
cmtst v1.16b, v7.16b, v9.16b
cmtst v2.16b, v7.16b, v10.16b
cmtst v3.16b, v7.16b, v11.16b
cmtst v4.16b, v7.16b, v12.16b
cmtst v5.16b, v7.16b, v13.16b
cmtst v6.16b, v7.16b, v14.16b
cmtst v7.16b, v7.16b, v15.16b
not v0.16b, v0.16b
not v1.16b, v1.16b
not v5.16b, v5.16b
not v6.16b, v6.16b
subs x2, x2, #1
stp q0, q1, [x0], #128
stp q2, q3, [x0, #-96]
stp q4, q5, [x0, #-64]
stp q6, q7, [x0, #-32]
b.ne .Lkey_loop
movi v7.16b, #0x63 // compose .L63
eor v17.16b, v17.16b, v7.16b
str q17, [x0]
ret
SYM_FUNC_END(aesbs_convert_key)
.align 4
SYM_FUNC_START_LOCAL(aesbs_encrypt8)
ldr q9, [bskey], #16 // round 0 key
ldr q8, M0SR
ldr q24, SR
eor v10.16b, v0.16b, v9.16b // xor with round0 key
eor v11.16b, v1.16b, v9.16b
tbl v0.16b, {v10.16b}, v8.16b
eor v12.16b, v2.16b, v9.16b
tbl v1.16b, {v11.16b}, v8.16b
eor v13.16b, v3.16b, v9.16b
tbl v2.16b, {v12.16b}, v8.16b
eor v14.16b, v4.16b, v9.16b
tbl v3.16b, {v13.16b}, v8.16b
eor v15.16b, v5.16b, v9.16b
tbl v4.16b, {v14.16b}, v8.16b
eor v10.16b, v6.16b, v9.16b
tbl v5.16b, {v15.16b}, v8.16b
eor v11.16b, v7.16b, v9.16b
tbl v6.16b, {v10.16b}, v8.16b
tbl v7.16b, {v11.16b}, v8.16b
bitslice v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11
sub rounds, rounds, #1
b .Lenc_sbox
.Lenc_loop:
shift_rows v0, v1, v2, v3, v4, v5, v6, v7, v24
.Lenc_sbox:
sbox v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, \
v13, v14, v15
subs rounds, rounds, #1
b.cc .Lenc_done
enc_next_rk
mix_cols v0, v1, v4, v6, v3, v7, v2, v5, v8, v9, v10, v11, v12, \
v13, v14, v15
add_round_key v0, v1, v2, v3, v4, v5, v6, v7
b.ne .Lenc_loop
ldr q24, SRM0
b .Lenc_loop
.Lenc_done:
ldr q12, [bskey] // last round key
bitslice v0, v1, v4, v6, v3, v7, v2, v5, v8, v9, v10, v11
eor v0.16b, v0.16b, v12.16b
eor v1.16b, v1.16b, v12.16b
eor v4.16b, v4.16b, v12.16b
eor v6.16b, v6.16b, v12.16b
eor v3.16b, v3.16b, v12.16b
eor v7.16b, v7.16b, v12.16b
eor v2.16b, v2.16b, v12.16b
eor v5.16b, v5.16b, v12.16b
ret
SYM_FUNC_END(aesbs_encrypt8)
.align 4
SYM_FUNC_START_LOCAL(aesbs_decrypt8)
lsl x9, rounds, #7
add bskey, bskey, x9
ldr q9, [bskey, #-112]! // round 0 key
ldr q8, M0ISR
ldr q24, ISR
eor v10.16b, v0.16b, v9.16b // xor with round0 key
eor v11.16b, v1.16b, v9.16b
tbl v0.16b, {v10.16b}, v8.16b
eor v12.16b, v2.16b, v9.16b
tbl v1.16b, {v11.16b}, v8.16b
eor v13.16b, v3.16b, v9.16b
tbl v2.16b, {v12.16b}, v8.16b
eor v14.16b, v4.16b, v9.16b
tbl v3.16b, {v13.16b}, v8.16b
eor v15.16b, v5.16b, v9.16b
tbl v4.16b, {v14.16b}, v8.16b
eor v10.16b, v6.16b, v9.16b
tbl v5.16b, {v15.16b}, v8.16b
eor v11.16b, v7.16b, v9.16b
tbl v6.16b, {v10.16b}, v8.16b
tbl v7.16b, {v11.16b}, v8.16b
bitslice v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11
sub rounds, rounds, #1
b .Ldec_sbox
.Ldec_loop:
shift_rows v0, v1, v2, v3, v4, v5, v6, v7, v24
.Ldec_sbox:
inv_sbox v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, \
v13, v14, v15
subs rounds, rounds, #1
b.cc .Ldec_done
dec_next_rk
add_round_key v0, v1, v6, v4, v2, v7, v3, v5
inv_mix_cols v0, v1, v6, v4, v2, v7, v3, v5, v8, v9, v10, v11, v12, \
v13, v14, v15
b.ne .Ldec_loop
ldr q24, ISRM0
b .Ldec_loop
.Ldec_done:
ldr q12, [bskey, #-16] // last round key
bitslice v0, v1, v6, v4, v2, v7, v3, v5, v8, v9, v10, v11
eor v0.16b, v0.16b, v12.16b
eor v1.16b, v1.16b, v12.16b
eor v6.16b, v6.16b, v12.16b
eor v4.16b, v4.16b, v12.16b
eor v2.16b, v2.16b, v12.16b
eor v7.16b, v7.16b, v12.16b
eor v3.16b, v3.16b, v12.16b
eor v5.16b, v5.16b, v12.16b
ret
SYM_FUNC_END(aesbs_decrypt8)
/*
* aesbs_ecb_encrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
* int blocks)
* aesbs_ecb_decrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
* int blocks)
*/
.macro __ecb_crypt, do8, o0, o1, o2, o3, o4, o5, o6, o7
frame_push 5
mov x19, x0
mov x20, x1
mov x21, x2
mov x22, x3
mov x23, x4
99: mov x5, #1
lsl x5, x5, x23
subs w23, w23, #8
csel x23, x23, xzr, pl
csel x5, x5, xzr, mi
ld1 {v0.16b}, [x20], #16
tbnz x5, #1, 0f
ld1 {v1.16b}, [x20], #16
tbnz x5, #2, 0f
ld1 {v2.16b}, [x20], #16
tbnz x5, #3, 0f
ld1 {v3.16b}, [x20], #16
tbnz x5, #4, 0f
ld1 {v4.16b}, [x20], #16
tbnz x5, #5, 0f
ld1 {v5.16b}, [x20], #16
tbnz x5, #6, 0f
ld1 {v6.16b}, [x20], #16
tbnz x5, #7, 0f
ld1 {v7.16b}, [x20], #16
0: mov bskey, x21
mov rounds, x22
bl \do8
st1 {\o0\().16b}, [x19], #16
tbnz x5, #1, 1f
st1 {\o1\().16b}, [x19], #16
tbnz x5, #2, 1f
st1 {\o2\().16b}, [x19], #16
tbnz x5, #3, 1f
st1 {\o3\().16b}, [x19], #16
tbnz x5, #4, 1f
st1 {\o4\().16b}, [x19], #16
tbnz x5, #5, 1f
st1 {\o5\().16b}, [x19], #16
tbnz x5, #6, 1f
st1 {\o6\().16b}, [x19], #16
tbnz x5, #7, 1f
st1 {\o7\().16b}, [x19], #16
cbz x23, 1f
b 99b
1: frame_pop
ret
.endm
.align 4
SYM_TYPED_FUNC_START(aesbs_ecb_encrypt)
__ecb_crypt aesbs_encrypt8, v0, v1, v4, v6, v3, v7, v2, v5
SYM_FUNC_END(aesbs_ecb_encrypt)
.align 4
SYM_TYPED_FUNC_START(aesbs_ecb_decrypt)
__ecb_crypt aesbs_decrypt8, v0, v1, v6, v4, v2, v7, v3, v5
SYM_FUNC_END(aesbs_ecb_decrypt)
/*
* aesbs_cbc_decrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
* int blocks, u8 iv[])
*/
.align 4
SYM_FUNC_START(aesbs_cbc_decrypt)
frame_push 6
mov x19, x0
mov x20, x1
mov x21, x2
mov x22, x3
mov x23, x4
mov x24, x5
99: mov x6, #1
lsl x6, x6, x23
subs w23, w23, #8
csel x23, x23, xzr, pl
csel x6, x6, xzr, mi
ld1 {v0.16b}, [x20], #16
mov v25.16b, v0.16b
tbnz x6, #1, 0f
ld1 {v1.16b}, [x20], #16
mov v26.16b, v1.16b
tbnz x6, #2, 0f
ld1 {v2.16b}, [x20], #16
mov v27.16b, v2.16b
tbnz x6, #3, 0f
ld1 {v3.16b}, [x20], #16
mov v28.16b, v3.16b
tbnz x6, #4, 0f
ld1 {v4.16b}, [x20], #16
mov v29.16b, v4.16b
tbnz x6, #5, 0f
ld1 {v5.16b}, [x20], #16
mov v30.16b, v5.16b
tbnz x6, #6, 0f
ld1 {v6.16b}, [x20], #16
mov v31.16b, v6.16b
tbnz x6, #7, 0f
ld1 {v7.16b}, [x20]
0: mov bskey, x21
mov rounds, x22
bl aesbs_decrypt8
ld1 {v24.16b}, [x24] // load IV
eor v1.16b, v1.16b, v25.16b
eor v6.16b, v6.16b, v26.16b
eor v4.16b, v4.16b, v27.16b
eor v2.16b, v2.16b, v28.16b
eor v7.16b, v7.16b, v29.16b
eor v0.16b, v0.16b, v24.16b
eor v3.16b, v3.16b, v30.16b
eor v5.16b, v5.16b, v31.16b
st1 {v0.16b}, [x19], #16
mov v24.16b, v25.16b
tbnz x6, #1, 1f
st1 {v1.16b}, [x19], #16
mov v24.16b, v26.16b
tbnz x6, #2, 1f
st1 {v6.16b}, [x19], #16
mov v24.16b, v27.16b
tbnz x6, #3, 1f
st1 {v4.16b}, [x19], #16
mov v24.16b, v28.16b
tbnz x6, #4, 1f
st1 {v2.16b}, [x19], #16
mov v24.16b, v29.16b
tbnz x6, #5, 1f
st1 {v7.16b}, [x19], #16
mov v24.16b, v30.16b
tbnz x6, #6, 1f
st1 {v3.16b}, [x19], #16
mov v24.16b, v31.16b
tbnz x6, #7, 1f
ld1 {v24.16b}, [x20], #16
st1 {v5.16b}, [x19], #16
1: st1 {v24.16b}, [x24] // store IV
cbz x23, 2f
b 99b
2: frame_pop
ret
SYM_FUNC_END(aesbs_cbc_decrypt)
.macro next_tweak, out, in, const, tmp
sshr \tmp\().2d, \in\().2d, #63
and \tmp\().16b, \tmp\().16b, \const\().16b
add \out\().2d, \in\().2d, \in\().2d
ext \tmp\().16b, \tmp\().16b, \tmp\().16b, #8
eor \out\().16b, \out\().16b, \tmp\().16b
.endm
/*
* aesbs_xts_encrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
* int blocks, u8 iv[])
* aesbs_xts_decrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
* int blocks, u8 iv[])
*/
SYM_FUNC_START_LOCAL(__xts_crypt8)
movi v18.2s, #0x1
movi v19.2s, #0x87
uzp1 v18.4s, v18.4s, v19.4s
ld1 {v0.16b-v3.16b}, [x1], #64
ld1 {v4.16b-v7.16b}, [x1], #64
next_tweak v26, v25, v18, v19
next_tweak v27, v26, v18, v19
next_tweak v28, v27, v18, v19
next_tweak v29, v28, v18, v19
next_tweak v30, v29, v18, v19
next_tweak v31, v30, v18, v19
next_tweak v16, v31, v18, v19
next_tweak v17, v16, v18, v19
eor v0.16b, v0.16b, v25.16b
eor v1.16b, v1.16b, v26.16b
eor v2.16b, v2.16b, v27.16b
eor v3.16b, v3.16b, v28.16b
eor v4.16b, v4.16b, v29.16b
eor v5.16b, v5.16b, v30.16b
eor v6.16b, v6.16b, v31.16b
eor v7.16b, v7.16b, v16.16b
stp q16, q17, [x6]
mov bskey, x2
mov rounds, x3
br x16
SYM_FUNC_END(__xts_crypt8)
.macro __xts_crypt, do8, o0, o1, o2, o3, o4, o5, o6, o7
frame_push 0, 32
add x6, sp, #.Lframe_local_offset
ld1 {v25.16b}, [x5]
0: adr x16, \do8
bl __xts_crypt8
eor v16.16b, \o0\().16b, v25.16b
eor v17.16b, \o1\().16b, v26.16b
eor v18.16b, \o2\().16b, v27.16b
eor v19.16b, \o3\().16b, v28.16b
ldp q24, q25, [x6]
eor v20.16b, \o4\().16b, v29.16b
eor v21.16b, \o5\().16b, v30.16b
eor v22.16b, \o6\().16b, v31.16b
eor v23.16b, \o7\().16b, v24.16b
st1 {v16.16b-v19.16b}, [x0], #64
st1 {v20.16b-v23.16b}, [x0], #64
subs x4, x4, #8
b.gt 0b
st1 {v25.16b}, [x5]
frame_pop
ret
.endm
SYM_TYPED_FUNC_START(aesbs_xts_encrypt)
__xts_crypt aesbs_encrypt8, v0, v1, v4, v6, v3, v7, v2, v5
SYM_FUNC_END(aesbs_xts_encrypt)
SYM_TYPED_FUNC_START(aesbs_xts_decrypt)
__xts_crypt aesbs_decrypt8, v0, v1, v6, v4, v2, v7, v3, v5
SYM_FUNC_END(aesbs_xts_decrypt)
.macro next_ctr, v
mov \v\().d[1], x8
adds x8, x8, #1
mov \v\().d[0], x7
adc x7, x7, xzr
rev64 \v\().16b, \v\().16b
.endm
/*
* aesbs_ctr_encrypt(u8 out[], u8 const in[], u8 const rk[],
* int rounds, int blocks, u8 iv[])
*/
SYM_FUNC_START(aesbs_ctr_encrypt)
frame_push 0
ldp x7, x8, [x5]
ld1 {v0.16b}, [x5]
CPU_LE( rev x7, x7 )
CPU_LE( rev x8, x8 )
adds x8, x8, #1
adc x7, x7, xzr
0: next_ctr v1
next_ctr v2
next_ctr v3
next_ctr v4
next_ctr v5
next_ctr v6
next_ctr v7
mov bskey, x2
mov rounds, x3
bl aesbs_encrypt8
ld1 { v8.16b-v11.16b}, [x1], #64
ld1 {v12.16b-v15.16b}, [x1], #64
eor v8.16b, v0.16b, v8.16b
eor v9.16b, v1.16b, v9.16b
eor v10.16b, v4.16b, v10.16b
eor v11.16b, v6.16b, v11.16b
eor v12.16b, v3.16b, v12.16b
eor v13.16b, v7.16b, v13.16b
eor v14.16b, v2.16b, v14.16b
eor v15.16b, v5.16b, v15.16b
st1 { v8.16b-v11.16b}, [x0], #64
st1 {v12.16b-v15.16b}, [x0], #64
next_ctr v0
subs x4, x4, #8
b.gt 0b
st1 {v0.16b}, [x5]
frame_pop
ret
SYM_FUNC_END(aesbs_ctr_encrypt)
linux-7.2.3/arch/arm64/crypto/aes-neonbs-glue.c
+464
View File
@@ -0,0 +1,464 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Bit sliced AES using NEON instructions
*
* Copyright (C) 2016 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
*/
#include <asm/neon.h>
#include <asm/simd.h>
#include <crypto/aes.h>
#include <crypto/ctr.h>
#include <crypto/internal/simd.h>
#include <crypto/internal/skcipher.h>
#include <crypto/scatterwalk.h>
#include <crypto/xts.h>
#include <linux/module.h>
MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
MODULE_DESCRIPTION("Bit sliced AES using NEON instructions");
MODULE_IMPORT_NS("CRYPTO_INTERNAL");
MODULE_LICENSE("GPL v2");
MODULE_ALIAS_CRYPTO("ecb(aes)");
MODULE_ALIAS_CRYPTO("cbc(aes)");
MODULE_ALIAS_CRYPTO("ctr(aes)");
MODULE_ALIAS_CRYPTO("xts(aes)");
asmlinkage void aesbs_convert_key(u8 out[], u32 const rk[], int rounds);
asmlinkage void aesbs_ecb_encrypt(u8 out[], u8 const in[], u8 const rk[],
int rounds, int blocks);
asmlinkage void aesbs_ecb_decrypt(u8 out[], u8 const in[], u8 const rk[],
int rounds, int blocks);
asmlinkage void aesbs_cbc_decrypt(u8 out[], u8 const in[], u8 const rk[],
int rounds, int blocks, u8 iv[]);
asmlinkage void aesbs_ctr_encrypt(u8 out[], u8 const in[], u8 const rk[],
int rounds, int blocks, u8 iv[]);
asmlinkage void aesbs_xts_encrypt(u8 out[], u8 const in[], u8 const rk[],
int rounds, int blocks, u8 iv[]);
asmlinkage void aesbs_xts_decrypt(u8 out[], u8 const in[], u8 const rk[],
int rounds, int blocks, u8 iv[]);
struct aesbs_ctx {
u8 rk[13 * (8 * AES_BLOCK_SIZE) + 32];
int rounds;
} __aligned(AES_BLOCK_SIZE);
struct aesbs_cbc_ctr_ctx {
struct aesbs_ctx key;
u32 enc[AES_MAX_KEYLENGTH_U32];
};
struct aesbs_xts_ctx {
struct aesbs_ctx key;
u32 twkey[AES_MAX_KEYLENGTH_U32];
struct crypto_aes_ctx cts;
};
static int aesbs_setkey(struct crypto_skcipher *tfm, const u8 *in_key,
unsigned int key_len)
{
struct aesbs_ctx *ctx = crypto_skcipher_ctx(tfm);
struct crypto_aes_ctx *rk;
int err;
rk = kmalloc(sizeof(*rk), GFP_KERNEL);
if (!rk)
return -ENOMEM;
err = aes_expandkey(rk, in_key, key_len);
if (err)
goto out;
ctx->rounds = 6 + key_len / 4;
scoped_ksimd()
aesbs_convert_key(ctx->rk, rk->key_enc, ctx->rounds);
out:
kfree_sensitive(rk);
return err;
}
static int __ecb_crypt(struct skcipher_request *req,
void (*fn)(u8 out[], u8 const in[], u8 const rk[],
int rounds, int blocks))
{
struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
struct aesbs_ctx *ctx = crypto_skcipher_ctx(tfm);
struct skcipher_walk walk;
int err;
err = skcipher_walk_virt(&walk, req, false);
while (walk.nbytes >= AES_BLOCK_SIZE) {
unsigned int blocks = walk.nbytes / AES_BLOCK_SIZE;
if (walk.nbytes < walk.total)
blocks = round_down(blocks,
walk.stride / AES_BLOCK_SIZE);
scoped_ksimd()
fn(walk.dst.virt.addr, walk.src.virt.addr, ctx->rk,
ctx->rounds, blocks);
err = skcipher_walk_done(&walk,
walk.nbytes - blocks * AES_BLOCK_SIZE);
}
return err;
}
static int ecb_encrypt(struct skcipher_request *req)
{
return __ecb_crypt(req, aesbs_ecb_encrypt);
}
static int ecb_decrypt(struct skcipher_request *req)
{
return __ecb_crypt(req, aesbs_ecb_decrypt);
}
static int aesbs_cbc_ctr_setkey(struct crypto_skcipher *tfm, const u8 *in_key,
unsigned int key_len)
{
struct aesbs_cbc_ctr_ctx *ctx = crypto_skcipher_ctx(tfm);
struct crypto_aes_ctx *rk;
int err;
rk = kmalloc(sizeof(*rk), GFP_KERNEL);
if (!rk)
return -ENOMEM;
err = aes_expandkey(rk, in_key, key_len);
if (err)
goto out;
ctx->key.rounds = 6 + key_len / 4;
memcpy(ctx->enc, rk->key_enc, sizeof(ctx->enc));
scoped_ksimd()
aesbs_convert_key(ctx->key.rk, rk->key_enc, ctx->key.rounds);
out:
kfree_sensitive(rk);
return err;
}
static int cbc_encrypt(struct skcipher_request *req)
{
struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
struct aesbs_cbc_ctr_ctx *ctx = crypto_skcipher_ctx(tfm);
struct skcipher_walk walk;
int err;
err = skcipher_walk_virt(&walk, req, false);
while (walk.nbytes >= AES_BLOCK_SIZE) {
unsigned int blocks = walk.nbytes / AES_BLOCK_SIZE;
/* fall back to the non-bitsliced NEON implementation */
scoped_ksimd()
neon_aes_cbc_encrypt(walk.dst.virt.addr,
walk.src.virt.addr,
ctx->enc, ctx->key.rounds, blocks,
walk.iv);
err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
}
return err;
}
static int cbc_decrypt(struct skcipher_request *req)
{
struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
struct aesbs_cbc_ctr_ctx *ctx = crypto_skcipher_ctx(tfm);
struct skcipher_walk walk;
int err;
err = skcipher_walk_virt(&walk, req, false);
while (walk.nbytes >= AES_BLOCK_SIZE) {
unsigned int blocks = walk.nbytes / AES_BLOCK_SIZE;
if (walk.nbytes < walk.total)
blocks = round_down(blocks,
walk.stride / AES_BLOCK_SIZE);
scoped_ksimd()
aesbs_cbc_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
ctx->key.rk, ctx->key.rounds, blocks,
walk.iv);
err = skcipher_walk_done(&walk,
walk.nbytes - blocks * AES_BLOCK_SIZE);
}
return err;
}
static int ctr_encrypt(struct skcipher_request *req)
{
struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
struct aesbs_cbc_ctr_ctx *ctx = crypto_skcipher_ctx(tfm);
struct skcipher_walk walk;
int err;
err = skcipher_walk_virt(&walk, req, false);
while (walk.nbytes > 0) {
int blocks = (walk.nbytes / AES_BLOCK_SIZE) & ~7;
int nbytes = walk.nbytes % (8 * AES_BLOCK_SIZE);
const u8 *src = walk.src.virt.addr;
u8 *dst = walk.dst.virt.addr;
scoped_ksimd() {
if (blocks >= 8) {
aesbs_ctr_encrypt(dst, src, ctx->key.rk,
ctx->key.rounds, blocks,
walk.iv);
dst += blocks * AES_BLOCK_SIZE;
src += blocks * AES_BLOCK_SIZE;
}
if (nbytes && walk.nbytes == walk.total) {
u8 buf[AES_BLOCK_SIZE];
u8 *d = dst;
if (unlikely(nbytes < AES_BLOCK_SIZE))
src = dst = memcpy(buf + sizeof(buf) -
nbytes, src, nbytes);
neon_aes_ctr_encrypt(dst, src, ctx->enc,
ctx->key.rounds, nbytes,
walk.iv);
if (unlikely(nbytes < AES_BLOCK_SIZE))
memcpy(d, dst, nbytes);
nbytes = 0;
}
}
err = skcipher_walk_done(&walk, nbytes);
}
return err;
}
static int aesbs_xts_setkey(struct crypto_skcipher *tfm, const u8 *in_key,
unsigned int key_len)
{
struct aesbs_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
struct crypto_aes_ctx rk;
int err;
err = xts_verify_key(tfm, in_key, key_len);
if (err)
return err;
key_len /= 2;
err = aes_expandkey(&ctx->cts, in_key, key_len);
if (err)
return err;
err = aes_expandkey(&rk, in_key + key_len, key_len);
if (err)
return err;
memcpy(ctx->twkey, rk.key_enc, sizeof(ctx->twkey));
return aesbs_setkey(tfm, in_key, key_len);
}
static int __xts_crypt(struct skcipher_request *req, bool encrypt,
void (*fn)(u8 out[], u8 const in[], u8 const rk[],
int rounds, int blocks, u8 iv[]))
{
struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
struct aesbs_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
int tail = req->cryptlen % (8 * AES_BLOCK_SIZE);
struct scatterlist sg_src[2], sg_dst[2];
struct skcipher_request subreq;
struct scatterlist *src, *dst;
struct skcipher_walk walk;
int nbytes, err;
int first = 1;
const u8 *in;
u8 *out;
if (req->cryptlen < AES_BLOCK_SIZE)
return -EINVAL;
/* ensure that the cts tail is covered by a single step */
if (unlikely(tail > 0 && tail < AES_BLOCK_SIZE)) {
int xts_blocks = DIV_ROUND_UP(req->cryptlen,
AES_BLOCK_SIZE) - 2;
skcipher_request_set_tfm(&subreq, tfm);
skcipher_request_set_callback(&subreq,
skcipher_request_flags(req),
NULL, NULL);
skcipher_request_set_crypt(&subreq, req->src, req->dst,
xts_blocks * AES_BLOCK_SIZE,
req->iv);
req = &subreq;
} else {
tail = 0;
}
err = skcipher_walk_virt(&walk, req, false);
if (err)
return err;
scoped_ksimd() {
while (walk.nbytes >= AES_BLOCK_SIZE) {
int blocks = (walk.nbytes / AES_BLOCK_SIZE) & ~7;
out = walk.dst.virt.addr;
in = walk.src.virt.addr;
nbytes = walk.nbytes;
if (blocks >= 8) {
if (first == 1)
neon_aes_ecb_encrypt(walk.iv, walk.iv,
ctx->twkey,
ctx->key.rounds, 1);
first = 2;
fn(out, in, ctx->key.rk, ctx->key.rounds, blocks,
walk.iv);
out += blocks * AES_BLOCK_SIZE;
in += blocks * AES_BLOCK_SIZE;
nbytes -= blocks * AES_BLOCK_SIZE;
}
if (walk.nbytes == walk.total && nbytes > 0) {
if (encrypt)
neon_aes_xts_encrypt(out, in, ctx->cts.key_enc,
ctx->key.rounds, nbytes,
ctx->twkey, walk.iv, first);
else
neon_aes_xts_decrypt(out, in, ctx->cts.key_dec,
ctx->key.rounds, nbytes,
ctx->twkey, walk.iv, first);
nbytes = first = 0;
}
err = skcipher_walk_done(&walk, nbytes);
}
if (err || likely(!tail))
return err;
/* handle ciphertext stealing */
dst = src = scatterwalk_ffwd(sg_src, req->src, req->cryptlen);
if (req->dst != req->src)
dst = scatterwalk_ffwd(sg_dst, req->dst, req->cryptlen);
skcipher_request_set_crypt(req, src, dst, AES_BLOCK_SIZE + tail,
req->iv);
err = skcipher_walk_virt(&walk, req, false);
if (err)
return err;
out = walk.dst.virt.addr;
in = walk.src.virt.addr;
nbytes = walk.nbytes;
if (encrypt)
neon_aes_xts_encrypt(out, in, ctx->cts.key_enc,
ctx->key.rounds, nbytes, ctx->twkey,
walk.iv, first);
else
neon_aes_xts_decrypt(out, in, ctx->cts.key_dec,
ctx->key.rounds, nbytes, ctx->twkey,
walk.iv, first);
}
return skcipher_walk_done(&walk, 0);
}
static int xts_encrypt(struct skcipher_request *req)
{
return __xts_crypt(req, true, aesbs_xts_encrypt);
}
static int xts_decrypt(struct skcipher_request *req)
{
return __xts_crypt(req, false, aesbs_xts_decrypt);
}
static struct skcipher_alg aes_algs[] = { {
.base.cra_name = "ecb(aes)",
.base.cra_driver_name = "ecb-aes-neonbs",
.base.cra_priority = 250,
.base.cra_blocksize = AES_BLOCK_SIZE,
.base.cra_ctxsize = sizeof(struct aesbs_ctx),
.base.cra_module = THIS_MODULE,
.min_keysize = AES_MIN_KEY_SIZE,
.max_keysize = AES_MAX_KEY_SIZE,
.walksize = 8 * AES_BLOCK_SIZE,
.setkey = aesbs_setkey,
.encrypt = ecb_encrypt,
.decrypt = ecb_decrypt,
}, {
.base.cra_name = "cbc(aes)",
.base.cra_driver_name = "cbc-aes-neonbs",
.base.cra_priority = 250,
.base.cra_blocksize = AES_BLOCK_SIZE,
.base.cra_ctxsize = sizeof(struct aesbs_cbc_ctr_ctx),
.base.cra_module = THIS_MODULE,
.min_keysize = AES_MIN_KEY_SIZE,
.max_keysize = AES_MAX_KEY_SIZE,
.walksize = 8 * AES_BLOCK_SIZE,
.ivsize = AES_BLOCK_SIZE,
.setkey = aesbs_cbc_ctr_setkey,
.encrypt = cbc_encrypt,
.decrypt = cbc_decrypt,
}, {
.base.cra_name = "ctr(aes)",
.base.cra_driver_name = "ctr-aes-neonbs",
.base.cra_priority = 250,
.base.cra_blocksize = 1,
.base.cra_ctxsize = sizeof(struct aesbs_cbc_ctr_ctx),
.base.cra_module = THIS_MODULE,
.min_keysize = AES_MIN_KEY_SIZE,
.max_keysize = AES_MAX_KEY_SIZE,
.chunksize = AES_BLOCK_SIZE,
.walksize = 8 * AES_BLOCK_SIZE,
.ivsize = AES_BLOCK_SIZE,
.setkey = aesbs_cbc_ctr_setkey,
.encrypt = ctr_encrypt,
.decrypt = ctr_encrypt,
}, {
.base.cra_name = "xts(aes)",
.base.cra_driver_name = "xts-aes-neonbs",
.base.cra_priority = 250,
.base.cra_blocksize = AES_BLOCK_SIZE,
.base.cra_ctxsize = sizeof(struct aesbs_xts_ctx),
.base.cra_module = THIS_MODULE,
.min_keysize = 2 * AES_MIN_KEY_SIZE,
.max_keysize = 2 * AES_MAX_KEY_SIZE,
.walksize = 8 * AES_BLOCK_SIZE,
.ivsize = AES_BLOCK_SIZE,
.setkey = aesbs_xts_setkey,
.encrypt = xts_encrypt,
.decrypt = xts_decrypt,
} };
static void aes_exit(void)
{
crypto_unregister_skciphers(aes_algs, ARRAY_SIZE(aes_algs));
}
static int __init aes_init(void)
{
if (!cpu_have_named_feature(ASIMD))
return -ENODEV;
return crypto_register_skciphers(aes_algs, ARRAY_SIZE(aes_algs));
}
module_init(aes_init);
module_exit(aes_exit);
linux-7.2.3/arch/arm64/crypto/ghash-ce-core.S
+593
View File
@@ -0,0 +1,593 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Accelerated AES-GCM implementation with ARMv8 Crypto Extensions.
*
* Copyright (C) 2014 - 2018 Linaro Ltd. <ard.biesheuvel@linaro.org>
*/
#include <linux/linkage.h>
#include <asm/assembler.h>
SHASH .req v0
SHASH2 .req v1
T1 .req v2
T2 .req v3
MASK .req v4
XM .req v5
XL .req v6
XH .req v7
IN1 .req v7
XL2 .req v8
XM2 .req v9
XH2 .req v10
XL3 .req v11
XM3 .req v12
XH3 .req v13
TT3 .req v14
TT4 .req v15
HH .req v16
HH3 .req v17
HH4 .req v18
HH34 .req v19
.text
.arch armv8-a+crypto
.macro __pmull_pre_p64
add x8, x3, #16
ld1 {HH.2d-HH4.2d}, [x8]
trn1 SHASH2.2d, SHASH.2d, HH.2d
trn2 T1.2d, SHASH.2d, HH.2d
eor SHASH2.16b, SHASH2.16b, T1.16b
trn1 HH34.2d, HH3.2d, HH4.2d
trn2 T1.2d, HH3.2d, HH4.2d
eor HH34.16b, HH34.16b, T1.16b
movi MASK.16b, #0xe1
shl MASK.2d, MASK.2d, #57
.endm
.macro __pmull_reduce_p64
pmull T2.1q, XL.1d, MASK.1d
eor XM.16b, XM.16b, T1.16b
mov XH.d[0], XM.d[1]
mov XM.d[1], XL.d[0]
eor XL.16b, XM.16b, T2.16b
ext T2.16b, XL.16b, XL.16b, #8
pmull XL.1q, XL.1d, MASK.1d
.endm
/*
* void pmull_ghash_update_p64(int blocks, u64 dg[], const char *src,
* u64 const h[4][2], const char *head)
*/
SYM_FUNC_START(pmull_ghash_update_p64)
ld1 {SHASH.2d}, [x3]
ld1 {XL.2d}, [x1]
__pmull_pre_p64
/* do the head block first, if supplied */
cbz x4, 0f
ld1 {T1.2d}, [x4]
mov x4, xzr
b 3f
0:
tbnz w0, #0, 2f // skip until #blocks is a
tbnz w0, #1, 2f // round multiple of 4
1: ld1 {XM3.16b-TT4.16b}, [x2], #64
sub w0, w0, #4
rev64 T1.16b, XM3.16b
rev64 T2.16b, XH3.16b
rev64 TT4.16b, TT4.16b
rev64 TT3.16b, TT3.16b
ext IN1.16b, TT4.16b, TT4.16b, #8
ext XL3.16b, TT3.16b, TT3.16b, #8
eor TT4.16b, TT4.16b, IN1.16b
pmull2 XH2.1q, SHASH.2d, IN1.2d // a1 * b1
pmull XL2.1q, SHASH.1d, IN1.1d // a0 * b0
pmull XM2.1q, SHASH2.1d, TT4.1d // (a1 + a0)(b1 + b0)
eor TT3.16b, TT3.16b, XL3.16b
pmull2 XH3.1q, HH.2d, XL3.2d // a1 * b1
pmull XL3.1q, HH.1d, XL3.1d // a0 * b0
pmull2 XM3.1q, SHASH2.2d, TT3.2d // (a1 + a0)(b1 + b0)
ext IN1.16b, T2.16b, T2.16b, #8
eor XL2.16b, XL2.16b, XL3.16b
eor XH2.16b, XH2.16b, XH3.16b
eor XM2.16b, XM2.16b, XM3.16b
eor T2.16b, T2.16b, IN1.16b
pmull2 XH3.1q, HH3.2d, IN1.2d // a1 * b1
pmull XL3.1q, HH3.1d, IN1.1d // a0 * b0
pmull XM3.1q, HH34.1d, T2.1d // (a1 + a0)(b1 + b0)
eor XL2.16b, XL2.16b, XL3.16b
eor XH2.16b, XH2.16b, XH3.16b
eor XM2.16b, XM2.16b, XM3.16b
ext IN1.16b, T1.16b, T1.16b, #8
ext TT3.16b, XL.16b, XL.16b, #8
eor XL.16b, XL.16b, IN1.16b
eor T1.16b, T1.16b, TT3.16b
pmull2 XH.1q, HH4.2d, XL.2d // a1 * b1
eor T1.16b, T1.16b, XL.16b
pmull XL.1q, HH4.1d, XL.1d // a0 * b0
pmull2 XM.1q, HH34.2d, T1.2d // (a1 + a0)(b1 + b0)
eor XL.16b, XL.16b, XL2.16b
eor XH.16b, XH.16b, XH2.16b
eor XM.16b, XM.16b, XM2.16b
eor T2.16b, XL.16b, XH.16b
ext T1.16b, XL.16b, XH.16b, #8
eor XM.16b, XM.16b, T2.16b
__pmull_reduce_p64
eor T2.16b, T2.16b, XH.16b
eor XL.16b, XL.16b, T2.16b
cbz w0, 5f
b 1b
2: ld1 {T1.2d}, [x2], #16
sub w0, w0, #1
3: /* multiply XL by SHASH in GF(2^128) */
CPU_LE( rev64 T1.16b, T1.16b )
ext T2.16b, XL.16b, XL.16b, #8
ext IN1.16b, T1.16b, T1.16b, #8
eor T1.16b, T1.16b, T2.16b
eor XL.16b, XL.16b, IN1.16b
pmull2 XH.1q, XL.2d, SHASH.2d // a1 * b1
eor T1.16b, T1.16b, XL.16b
pmull XL.1q, XL.1d, SHASH.1d // a0 * b0
pmull XM.1q, T1.1d, SHASH2.1d // (a1 + a0)(b1 + b0)
4: eor T2.16b, XL.16b, XH.16b
ext T1.16b, XL.16b, XH.16b, #8
eor XM.16b, XM.16b, T2.16b
__pmull_reduce_p64
eor T2.16b, T2.16b, XH.16b
eor XL.16b, XL.16b, T2.16b
cbnz w0, 0b
5: st1 {XL.2d}, [x1]
ret
SYM_FUNC_END(pmull_ghash_update_p64)
KS0 .req v8
KS1 .req v9
KS2 .req v10
KS3 .req v11
INP0 .req v21
INP1 .req v22
INP2 .req v23
INP3 .req v24
K0 .req v25
K1 .req v26
K2 .req v27
K3 .req v28
K4 .req v12
K5 .req v13
K6 .req v4
K7 .req v5
K8 .req v14
K9 .req v15
KK .req v29
KL .req v30
KM .req v31
.macro load_round_keys, rounds, rk, tmp
add \tmp, \rk, #64
ld1 {K0.4s-K3.4s}, [\rk]
ld1 {K4.4s-K5.4s}, [\tmp]
add \tmp, \rk, \rounds, lsl #4
sub \tmp, \tmp, #32
ld1 {KK.4s-KM.4s}, [\tmp]
.endm
.macro enc_round, state, key
aese \state\().16b, \key\().16b
aesmc \state\().16b, \state\().16b
.endm
.macro enc_qround, s0, s1, s2, s3, key
enc_round \s0, \key
enc_round \s1, \key
enc_round \s2, \key
enc_round \s3, \key
.endm
.macro enc_block, state, rounds, rk, tmp
add \tmp, \rk, #96
ld1 {K6.4s-K7.4s}, [\tmp], #32
.irp key, K0, K1, K2, K3, K4 K5
enc_round \state, \key
.endr
tbnz \rounds, #2, .Lnot128_\@
.Lout256_\@:
enc_round \state, K6
enc_round \state, K7
.Lout192_\@:
enc_round \state, KK
aese \state\().16b, KL.16b
eor \state\().16b, \state\().16b, KM.16b
.subsection 1
.Lnot128_\@:
ld1 {K8.4s-K9.4s}, [\tmp], #32
enc_round \state, K6
enc_round \state, K7
ld1 {K6.4s-K7.4s}, [\tmp]
enc_round \state, K8
enc_round \state, K9
tbz \rounds, #1, .Lout192_\@
b .Lout256_\@
.previous
.endm
.align 6
.macro pmull_gcm_do_crypt, enc
frame_push 1
load_round_keys x7, x6, x8
ld1 {SHASH.2d}, [x3], #16
ld1 {HH.2d-HH4.2d}, [x3]
trn1 SHASH2.2d, SHASH.2d, HH.2d
trn2 T1.2d, SHASH.2d, HH.2d
eor SHASH2.16b, SHASH2.16b, T1.16b
trn1 HH34.2d, HH3.2d, HH4.2d
trn2 T1.2d, HH3.2d, HH4.2d
eor HH34.16b, HH34.16b, T1.16b
ld1 {XL.2d}, [x4]
cbz x0, 3f // tag only?
ldr w8, [x5, #12] // load lower counter
CPU_LE( rev w8, w8 )
0: mov w9, #4 // max blocks per round
add x10, x0, #0xf
lsr x10, x10, #4 // remaining blocks
subs x0, x0, #64
csel w9, w10, w9, mi
add w8, w8, w9
bmi 1f
ld1 {INP0.16b-INP3.16b}, [x2], #64
.subsection 1
/*
* Populate the four input registers right to left with up to 63 bytes
* of data, using overlapping loads to avoid branches.
*
* INP0 INP1 INP2 INP3
* 1 byte | | | |x |
* 16 bytes | | | |xxxxxxxx|
* 17 bytes | | |xxxxxxxx|x |
* 47 bytes | |xxxxxxxx|xxxxxxxx|xxxxxxx |
* etc etc
*
* Note that this code may read up to 15 bytes before the start of
* the input. It is up to the calling code to ensure this is safe if
* this happens in the first iteration of the loop (i.e., when the
* input size is < 16 bytes)
*/
1: mov x15, #16
ands x19, x0, #0xf
csel x19, x19, x15, ne
adr_l x17, .Lpermute_table + 16
sub x11, x15, x19
add x12, x17, x11
sub x17, x17, x11
ld1 {T1.16b}, [x12]
sub x10, x1, x11
sub x11, x2, x11
cmp x0, #-16
csel x14, x15, xzr, gt
cmp x0, #-32
csel x15, x15, xzr, gt
cmp x0, #-48
csel x16, x19, xzr, gt
csel x1, x1, x10, gt
csel x2, x2, x11, gt
ld1 {INP0.16b}, [x2], x14
ld1 {INP1.16b}, [x2], x15
ld1 {INP2.16b}, [x2], x16
ld1 {INP3.16b}, [x2]
tbl INP3.16b, {INP3.16b}, T1.16b
b 2f
.previous
2: .if \enc == 0
bl pmull_gcm_ghash_4x
.endif
bl pmull_gcm_enc_4x
tbnz x0, #63, 6f
st1 {INP0.16b-INP3.16b}, [x1], #64
.if \enc == 1
bl pmull_gcm_ghash_4x
.endif
bne 0b
3: ldr x10, [sp, #.Lframe_local_offset]
cbz x10, 5f // output tag?
ld1 {INP3.16b}, [x10] // load lengths[]
mov w9, #1
bl pmull_gcm_ghash_4x
mov w11, #(0x1 << 24) // BE '1U'
ld1 {KS0.16b}, [x5]
mov KS0.s[3], w11
enc_block KS0, x7, x6, x12
ext XL.16b, XL.16b, XL.16b, #8
rev64 XL.16b, XL.16b
eor XL.16b, XL.16b, KS0.16b
.if \enc == 1
st1 {XL.16b}, [x10] // store tag
.else
ldp x11, x12, [sp, #40] // load tag pointer and authsize
adr_l x17, .Lpermute_table
ld1 {KS0.16b}, [x11] // load supplied tag
add x17, x17, x12
ld1 {KS1.16b}, [x17] // load permute vector
cmeq XL.16b, XL.16b, KS0.16b // compare tags
mvn XL.16b, XL.16b // -1 for fail, 0 for pass
tbl XL.16b, {XL.16b}, KS1.16b // keep authsize bytes only
sminv b0, XL.16b // signed minimum across XL
smov w0, v0.b[0] // return b0
.endif
4: frame_pop
ret
5:
CPU_LE( rev w8, w8 )
str w8, [x5, #12] // store lower counter
st1 {XL.2d}, [x4]
b 4b
6: ld1 {T1.16b-T2.16b}, [x17], #32 // permute vectors
sub x17, x17, x19, lsl #1
cmp w9, #1
beq 7f
.subsection 1
7: ld1 {INP2.16b}, [x1]
tbx INP2.16b, {INP3.16b}, T1.16b
mov INP3.16b, INP2.16b
b 8f
.previous
st1 {INP0.16b}, [x1], x14
st1 {INP1.16b}, [x1], x15
st1 {INP2.16b}, [x1], x16
tbl INP3.16b, {INP3.16b}, T1.16b
tbx INP3.16b, {INP2.16b}, T2.16b
8: st1 {INP3.16b}, [x1]
.if \enc == 1
ld1 {T1.16b}, [x17]
tbl INP3.16b, {INP3.16b}, T1.16b // clear non-data bits
bl pmull_gcm_ghash_4x
.endif
b 3b
.endm
/*
* void pmull_gcm_encrypt(int bytes, u8 dst[], const u8 src[],
* u64 const h[4][2], u64 dg[], u8 ctr[],
* u32 const rk[], int rounds, u8 tag[])
*/
SYM_FUNC_START(pmull_gcm_encrypt)
pmull_gcm_do_crypt 1
SYM_FUNC_END(pmull_gcm_encrypt)
/*
* int pmull_gcm_decrypt(int bytes, u8 dst[], const u8 src[],
* u64 const h[4][2], u64 dg[], u8 ctr[],
* u32 const rk[], int rounds, const u8 l[],
* const u8 tag[], u64 authsize)
*/
SYM_FUNC_START(pmull_gcm_decrypt)
pmull_gcm_do_crypt 0
SYM_FUNC_END(pmull_gcm_decrypt)
SYM_FUNC_START_LOCAL(pmull_gcm_ghash_4x)
movi MASK.16b, #0xe1
shl MASK.2d, MASK.2d, #57
rev64 T1.16b, INP0.16b
rev64 T2.16b, INP1.16b
rev64 TT3.16b, INP2.16b
rev64 TT4.16b, INP3.16b
ext XL.16b, XL.16b, XL.16b, #8
tbz w9, #2, 0f // <4 blocks?
.subsection 1
0: movi XH2.16b, #0
movi XM2.16b, #0
movi XL2.16b, #0
tbz w9, #0, 1f // 2 blocks?
tbz w9, #1, 2f // 1 block?
eor T2.16b, T2.16b, XL.16b
ext T1.16b, T2.16b, T2.16b, #8
b .Lgh3
1: eor TT3.16b, TT3.16b, XL.16b
ext T2.16b, TT3.16b, TT3.16b, #8
b .Lgh2
2: eor TT4.16b, TT4.16b, XL.16b
ext IN1.16b, TT4.16b, TT4.16b, #8
b .Lgh1
.previous
eor T1.16b, T1.16b, XL.16b
ext IN1.16b, T1.16b, T1.16b, #8
pmull2 XH2.1q, HH4.2d, IN1.2d // a1 * b1
eor T1.16b, T1.16b, IN1.16b
pmull XL2.1q, HH4.1d, IN1.1d // a0 * b0
pmull2 XM2.1q, HH34.2d, T1.2d // (a1 + a0)(b1 + b0)
ext T1.16b, T2.16b, T2.16b, #8
.Lgh3: eor T2.16b, T2.16b, T1.16b
pmull2 XH.1q, HH3.2d, T1.2d // a1 * b1
pmull XL.1q, HH3.1d, T1.1d // a0 * b0
pmull XM.1q, HH34.1d, T2.1d // (a1 + a0)(b1 + b0)
eor XH2.16b, XH2.16b, XH.16b
eor XL2.16b, XL2.16b, XL.16b
eor XM2.16b, XM2.16b, XM.16b
ext T2.16b, TT3.16b, TT3.16b, #8
.Lgh2: eor TT3.16b, TT3.16b, T2.16b
pmull2 XH.1q, HH.2d, T2.2d // a1 * b1
pmull XL.1q, HH.1d, T2.1d // a0 * b0
pmull2 XM.1q, SHASH2.2d, TT3.2d // (a1 + a0)(b1 + b0)
eor XH2.16b, XH2.16b, XH.16b
eor XL2.16b, XL2.16b, XL.16b
eor XM2.16b, XM2.16b, XM.16b
ext IN1.16b, TT4.16b, TT4.16b, #8
.Lgh1: eor TT4.16b, TT4.16b, IN1.16b
pmull XL.1q, SHASH.1d, IN1.1d // a0 * b0
pmull2 XH.1q, SHASH.2d, IN1.2d // a1 * b1
pmull XM.1q, SHASH2.1d, TT4.1d // (a1 + a0)(b1 + b0)
eor XH.16b, XH.16b, XH2.16b
eor XL.16b, XL.16b, XL2.16b
eor XM.16b, XM.16b, XM2.16b
eor T2.16b, XL.16b, XH.16b
ext T1.16b, XL.16b, XH.16b, #8
eor XM.16b, XM.16b, T2.16b
__pmull_reduce_p64
eor T2.16b, T2.16b, XH.16b
eor XL.16b, XL.16b, T2.16b
ret
SYM_FUNC_END(pmull_gcm_ghash_4x)
SYM_FUNC_START_LOCAL(pmull_gcm_enc_4x)
ld1 {KS0.16b}, [x5] // load upper counter
sub w10, w8, #4
sub w11, w8, #3
sub w12, w8, #2
sub w13, w8, #1
rev w10, w10
rev w11, w11
rev w12, w12
rev w13, w13
mov KS1.16b, KS0.16b
mov KS2.16b, KS0.16b
mov KS3.16b, KS0.16b
ins KS0.s[3], w10 // set lower counter
ins KS1.s[3], w11
ins KS2.s[3], w12
ins KS3.s[3], w13
add x10, x6, #96 // round key pointer
ld1 {K6.4s-K7.4s}, [x10], #32
.irp key, K0, K1, K2, K3, K4, K5
enc_qround KS0, KS1, KS2, KS3, \key
.endr
tbnz x7, #2, .Lnot128
.subsection 1
.Lnot128:
ld1 {K8.4s-K9.4s}, [x10], #32
.irp key, K6, K7
enc_qround KS0, KS1, KS2, KS3, \key
.endr
ld1 {K6.4s-K7.4s}, [x10]
.irp key, K8, K9
enc_qround KS0, KS1, KS2, KS3, \key
.endr
tbz x7, #1, .Lout192
b .Lout256
.previous
.Lout256:
.irp key, K6, K7
enc_qround KS0, KS1, KS2, KS3, \key
.endr
.Lout192:
enc_qround KS0, KS1, KS2, KS3, KK
aese KS0.16b, KL.16b
aese KS1.16b, KL.16b
aese KS2.16b, KL.16b
aese KS3.16b, KL.16b
eor KS0.16b, KS0.16b, KM.16b
eor KS1.16b, KS1.16b, KM.16b
eor KS2.16b, KS2.16b, KM.16b
eor KS3.16b, KS3.16b, KM.16b
eor INP0.16b, INP0.16b, KS0.16b
eor INP1.16b, INP1.16b, KS1.16b
eor INP2.16b, INP2.16b, KS2.16b
eor INP3.16b, INP3.16b, KS3.16b
ret
SYM_FUNC_END(pmull_gcm_enc_4x)
.section ".rodata", "a"
.align 6
.Lpermute_table:
.byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
.byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
.byte 0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7
.byte 0x8, 0x9, 0xa, 0xb, 0xc, 0xd, 0xe, 0xf
.byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
.byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
.byte 0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7
.byte 0x8, 0x9, 0xa, 0xb, 0xc, 0xd, 0xe, 0xf
.previous
linux-7.2.3/arch/arm64/crypto/ghash-ce-glue.c
+404
View File
@@ -0,0 +1,404 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* AES-GCM using ARMv8 Crypto Extensions
*
* Copyright (C) 2014 - 2018 Linaro Ltd. <ard.biesheuvel@linaro.org>
*/
#include <crypto/aes.h>
#include <crypto/b128ops.h>
#include <crypto/gcm.h>
#include <crypto/ghash.h>
#include <crypto/gf128mul.h>
#include <crypto/internal/aead.h>
#include <crypto/internal/skcipher.h>
#include <crypto/scatterwalk.h>
#include <linux/cpufeature.h>
#include <linux/errno.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/string.h>
#include <linux/unaligned.h>
#include <asm/simd.h>
MODULE_DESCRIPTION("AES-GCM using ARMv8 Crypto Extensions");
MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
MODULE_LICENSE("GPL v2");
MODULE_ALIAS_CRYPTO("gcm(aes)");
MODULE_ALIAS_CRYPTO("rfc4106(gcm(aes))");
#define RFC4106_NONCE_SIZE 4
struct arm_ghash_key {
be128 k;
u64 h[4][2];
};
struct gcm_aes_ctx {
struct aes_enckey aes_key;
u8 nonce[RFC4106_NONCE_SIZE];
struct arm_ghash_key ghash_key;
};
asmlinkage void pmull_ghash_update_p64(int blocks, u64 dg[], const char *src,
u64 const h[4][2], const char *head);
asmlinkage void pmull_gcm_encrypt(int bytes, u8 dst[], const u8 src[],
u64 const h[4][2], u64 dg[], u8 ctr[],
u32 const rk[], int rounds, u8 tag[]);
asmlinkage int pmull_gcm_decrypt(int bytes, u8 dst[], const u8 src[],
u64 const h[4][2], u64 dg[], u8 ctr[],
u32 const rk[], int rounds, const u8 l[],
const u8 tag[], u64 authsize);
static void ghash_do_simd_update(int blocks, u64 dg[], const char *src,
struct arm_ghash_key *key, const char *head)
{
scoped_ksimd()
pmull_ghash_update_p64(blocks, dg, src, key->h, head);
}
static void ghash_reflect(u64 h[], const be128 *k)
{
u64 carry = be64_to_cpu(k->a) & BIT(63) ? 1 : 0;
h[0] = (be64_to_cpu(k->b) << 1) | carry;
h[1] = (be64_to_cpu(k->a) << 1) | (be64_to_cpu(k->b) >> 63);
if (carry)
h[1] ^= 0xc200000000000000UL;
}
static int gcm_aes_setkey(struct crypto_aead *tfm, const u8 *inkey,
unsigned int keylen)
{
struct gcm_aes_ctx *ctx = crypto_aead_ctx(tfm);
u8 key[GHASH_BLOCK_SIZE];
be128 h;
int ret;
ret = aes_prepareenckey(&ctx->aes_key, inkey, keylen);
if (ret)
return -EINVAL;
aes_encrypt(&ctx->aes_key, key, (u8[AES_BLOCK_SIZE]){});
/* needed for the fallback */
memcpy(&ctx->ghash_key.k, key, GHASH_BLOCK_SIZE);
ghash_reflect(ctx->ghash_key.h[0], &ctx->ghash_key.k);
h = ctx->ghash_key.k;
gf128mul_lle(&h, &ctx->ghash_key.k);
ghash_reflect(ctx->ghash_key.h[1], &h);
gf128mul_lle(&h, &ctx->ghash_key.k);
ghash_reflect(ctx->ghash_key.h[2], &h);
gf128mul_lle(&h, &ctx->ghash_key.k);
ghash_reflect(ctx->ghash_key.h[3], &h);
return 0;
}
static int gcm_aes_setauthsize(struct crypto_aead *tfm, unsigned int authsize)
{
return crypto_gcm_check_authsize(authsize);
}
static void gcm_update_mac(u64 dg[], const u8 *src, int count, u8 buf[],
int *buf_count, struct gcm_aes_ctx *ctx)
{
if (*buf_count > 0) {
int buf_added = min(count, GHASH_BLOCK_SIZE - *buf_count);
memcpy(&buf[*buf_count], src, buf_added);
*buf_count += buf_added;
src += buf_added;
count -= buf_added;
}
if (count >= GHASH_BLOCK_SIZE || *buf_count == GHASH_BLOCK_SIZE) {
int blocks = count / GHASH_BLOCK_SIZE;
ghash_do_simd_update(blocks, dg, src, &ctx->ghash_key,
*buf_count ? buf : NULL);
src += blocks * GHASH_BLOCK_SIZE;
count %= GHASH_BLOCK_SIZE;
*buf_count = 0;
}
if (count > 0) {
memcpy(buf, src, count);
*buf_count = count;
}
}
static void gcm_calculate_auth_mac(struct aead_request *req, u64 dg[], u32 len)
{
struct crypto_aead *aead = crypto_aead_reqtfm(req);
struct gcm_aes_ctx *ctx = crypto_aead_ctx(aead);
u8 buf[GHASH_BLOCK_SIZE];
struct scatter_walk walk;
int buf_count = 0;
scatterwalk_start(&walk, req->src);
do {
unsigned int n;
n = scatterwalk_next(&walk, len);
gcm_update_mac(dg, walk.addr, n, buf, &buf_count, ctx);
scatterwalk_done_src(&walk, n);
len -= n;
} while (len);
if (buf_count) {
memset(&buf[buf_count], 0, GHASH_BLOCK_SIZE - buf_count);
ghash_do_simd_update(1, dg, buf, &ctx->ghash_key, NULL);
}
}
static int gcm_encrypt(struct aead_request *req, char *iv, int assoclen)
{
struct crypto_aead *aead = crypto_aead_reqtfm(req);
struct gcm_aes_ctx *ctx = crypto_aead_ctx(aead);
struct skcipher_walk walk;
u8 buf[AES_BLOCK_SIZE];
u64 dg[2] = {};
be128 lengths;
u8 *tag;
int err;
lengths.a = cpu_to_be64(assoclen * 8);
lengths.b = cpu_to_be64(req->cryptlen * 8);
if (assoclen)
gcm_calculate_auth_mac(req, dg, assoclen);
put_unaligned_be32(2, iv + GCM_AES_IV_SIZE);
err = skcipher_walk_aead_encrypt(&walk, req, false);
do {
const u8 *src = walk.src.virt.addr;
u8 *dst = walk.dst.virt.addr;
int nbytes = walk.nbytes;
tag = (u8 *)&lengths;
if (unlikely(nbytes > 0 && nbytes < AES_BLOCK_SIZE)) {
src = dst = memcpy(buf + sizeof(buf) - nbytes,
src, nbytes);
} else if (nbytes < walk.total) {
nbytes &= ~(AES_BLOCK_SIZE - 1);
tag = NULL;
}
scoped_ksimd()
pmull_gcm_encrypt(nbytes, dst, src, ctx->ghash_key.h,
dg, iv, ctx->aes_key.k.rndkeys,
ctx->aes_key.nrounds, tag);
if (unlikely(!nbytes))
break;
if (unlikely(nbytes > 0 && nbytes < AES_BLOCK_SIZE))
memcpy(walk.dst.virt.addr,
buf + sizeof(buf) - nbytes, nbytes);
err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
} while (walk.nbytes);
if (err)
return err;
/* copy authtag to end of dst */
scatterwalk_map_and_copy(tag, req->dst, req->assoclen + req->cryptlen,
crypto_aead_authsize(aead), 1);
return 0;
}
static int gcm_decrypt(struct aead_request *req, char *iv, int assoclen)
{
struct crypto_aead *aead = crypto_aead_reqtfm(req);
struct gcm_aes_ctx *ctx = crypto_aead_ctx(aead);
unsigned int authsize = crypto_aead_authsize(aead);
struct skcipher_walk walk;
u8 otag[AES_BLOCK_SIZE];
u8 buf[AES_BLOCK_SIZE];
u64 dg[2] = {};
be128 lengths;
u8 *tag;
int ret;
int err;
lengths.a = cpu_to_be64(assoclen * 8);
lengths.b = cpu_to_be64((req->cryptlen - authsize) * 8);
if (assoclen)
gcm_calculate_auth_mac(req, dg, assoclen);
put_unaligned_be32(2, iv + GCM_AES_IV_SIZE);
scatterwalk_map_and_copy(otag, req->src,
req->assoclen + req->cryptlen - authsize,
authsize, 0);
err = skcipher_walk_aead_decrypt(&walk, req, false);
do {
const u8 *src = walk.src.virt.addr;
u8 *dst = walk.dst.virt.addr;
int nbytes = walk.nbytes;
tag = (u8 *)&lengths;
if (unlikely(nbytes > 0 && nbytes < AES_BLOCK_SIZE)) {
src = dst = memcpy(buf + sizeof(buf) - nbytes,
src, nbytes);
} else if (nbytes < walk.total) {
nbytes &= ~(AES_BLOCK_SIZE - 1);
tag = NULL;
}
scoped_ksimd()
ret = pmull_gcm_decrypt(nbytes, dst, src,
ctx->ghash_key.h,
dg, iv, ctx->aes_key.k.rndkeys,
ctx->aes_key.nrounds, tag, otag,
authsize);
if (unlikely(!nbytes))
break;
if (unlikely(nbytes > 0 && nbytes < AES_BLOCK_SIZE))
memcpy(walk.dst.virt.addr,
buf + sizeof(buf) - nbytes, nbytes);
err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
} while (walk.nbytes);
if (err)
return err;
return ret ? -EBADMSG : 0;
}
static int gcm_aes_encrypt(struct aead_request *req)
{
u8 iv[AES_BLOCK_SIZE];
memcpy(iv, req->iv, GCM_AES_IV_SIZE);
return gcm_encrypt(req, iv, req->assoclen);
}
static int gcm_aes_decrypt(struct aead_request *req)
{
u8 iv[AES_BLOCK_SIZE];
memcpy(iv, req->iv, GCM_AES_IV_SIZE);
return gcm_decrypt(req, iv, req->assoclen);
}
static int rfc4106_setkey(struct crypto_aead *tfm, const u8 *inkey,
unsigned int keylen)
{
struct gcm_aes_ctx *ctx = crypto_aead_ctx(tfm);
int err;
keylen -= RFC4106_NONCE_SIZE;
err = gcm_aes_setkey(tfm, inkey, keylen);
if (err)
return err;
memcpy(ctx->nonce, inkey + keylen, RFC4106_NONCE_SIZE);
return 0;
}
static int rfc4106_setauthsize(struct crypto_aead *tfm, unsigned int authsize)
{
return crypto_rfc4106_check_authsize(authsize);
}
static int rfc4106_encrypt(struct aead_request *req)
{
struct crypto_aead *aead = crypto_aead_reqtfm(req);
struct gcm_aes_ctx *ctx = crypto_aead_ctx(aead);
u8 iv[AES_BLOCK_SIZE];
memcpy(iv, ctx->nonce, RFC4106_NONCE_SIZE);
memcpy(iv + RFC4106_NONCE_SIZE, req->iv, GCM_RFC4106_IV_SIZE);
return crypto_ipsec_check_assoclen(req->assoclen) ?:
gcm_encrypt(req, iv, req->assoclen - GCM_RFC4106_IV_SIZE);
}
static int rfc4106_decrypt(struct aead_request *req)
{
struct crypto_aead *aead = crypto_aead_reqtfm(req);
struct gcm_aes_ctx *ctx = crypto_aead_ctx(aead);
u8 iv[AES_BLOCK_SIZE];
memcpy(iv, ctx->nonce, RFC4106_NONCE_SIZE);
memcpy(iv + RFC4106_NONCE_SIZE, req->iv, GCM_RFC4106_IV_SIZE);
return crypto_ipsec_check_assoclen(req->assoclen) ?:
gcm_decrypt(req, iv, req->assoclen - GCM_RFC4106_IV_SIZE);
}
static struct aead_alg gcm_aes_algs[] = {{
.ivsize = GCM_AES_IV_SIZE,
.chunksize = AES_BLOCK_SIZE,
.maxauthsize = AES_BLOCK_SIZE,
.setkey = gcm_aes_setkey,
.setauthsize = gcm_aes_setauthsize,
.encrypt = gcm_aes_encrypt,
.decrypt = gcm_aes_decrypt,
.base.cra_name = "gcm(aes)",
.base.cra_driver_name = "gcm-aes-ce",
.base.cra_priority = 300,
.base.cra_blocksize = 1,
.base.cra_ctxsize = sizeof(struct gcm_aes_ctx),
.base.cra_module = THIS_MODULE,
}, {
.ivsize = GCM_RFC4106_IV_SIZE,
.chunksize = AES_BLOCK_SIZE,
.maxauthsize = AES_BLOCK_SIZE,
.setkey = rfc4106_setkey,
.setauthsize = rfc4106_setauthsize,
.encrypt = rfc4106_encrypt,
.decrypt = rfc4106_decrypt,
.base.cra_name = "rfc4106(gcm(aes))",
.base.cra_driver_name = "rfc4106-gcm-aes-ce",
.base.cra_priority = 300,
.base.cra_blocksize = 1,
.base.cra_ctxsize = sizeof(struct gcm_aes_ctx),
.base.cra_module = THIS_MODULE,
}};
static int __init ghash_ce_mod_init(void)
{
if (!cpu_have_named_feature(ASIMD) || !cpu_have_named_feature(PMULL))
return -ENODEV;
return crypto_register_aeads(gcm_aes_algs, ARRAY_SIZE(gcm_aes_algs));
}
static void __exit ghash_ce_mod_exit(void)
{
crypto_unregister_aeads(gcm_aes_algs, ARRAY_SIZE(gcm_aes_algs));
}
static const struct cpu_feature __maybe_unused ghash_cpu_feature[] = {
{ cpu_feature(PMULL) }, { }
};
MODULE_DEVICE_TABLE(cpu, ghash_cpu_feature);
module_init(ghash_ce_mod_init);
module_exit(ghash_ce_mod_exit);
linux-7.2.3/arch/arm64/crypto/sm4-ce-asm.h
+209
View File
@@ -0,0 +1,209 @@
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
* SM4 helper macros for Crypto Extensions
* Copyright (C) 2022 Tianjia Zhang <tianjia.zhang@linux.alibaba.com>
*/
#define SM4_PREPARE(ptr) \
ld1 {v24.16b-v27.16b}, [ptr], #64; \
ld1 {v28.16b-v31.16b}, [ptr];
#define SM4_CRYPT_BLK_BE(b0) \
sm4e b0.4s, v24.4s; \
sm4e b0.4s, v25.4s; \
sm4e b0.4s, v26.4s; \
sm4e b0.4s, v27.4s; \
sm4e b0.4s, v28.4s; \
sm4e b0.4s, v29.4s; \
sm4e b0.4s, v30.4s; \
sm4e b0.4s, v31.4s; \
rev64 b0.4s, b0.4s; \
ext b0.16b, b0.16b, b0.16b, #8; \
rev32 b0.16b, b0.16b;
#define SM4_CRYPT_BLK(b0) \
rev32 b0.16b, b0.16b; \
SM4_CRYPT_BLK_BE(b0);
#define SM4_CRYPT_BLK2_BE(b0, b1) \
sm4e b0.4s, v24.4s; \
sm4e b1.4s, v24.4s; \
sm4e b0.4s, v25.4s; \
sm4e b1.4s, v25.4s; \
sm4e b0.4s, v26.4s; \
sm4e b1.4s, v26.4s; \
sm4e b0.4s, v27.4s; \
sm4e b1.4s, v27.4s; \
sm4e b0.4s, v28.4s; \
sm4e b1.4s, v28.4s; \
sm4e b0.4s, v29.4s; \
sm4e b1.4s, v29.4s; \
sm4e b0.4s, v30.4s; \
sm4e b1.4s, v30.4s; \
sm4e b0.4s, v31.4s; \
sm4e b1.4s, v31.4s; \
rev64 b0.4s, b0.4s; \
rev64 b1.4s, b1.4s; \
ext b0.16b, b0.16b, b0.16b, #8; \
ext b1.16b, b1.16b, b1.16b, #8; \
rev32 b0.16b, b0.16b; \
rev32 b1.16b, b1.16b; \
#define SM4_CRYPT_BLK2(b0, b1) \
rev32 b0.16b, b0.16b; \
rev32 b1.16b, b1.16b; \
SM4_CRYPT_BLK2_BE(b0, b1);
#define SM4_CRYPT_BLK4_BE(b0, b1, b2, b3) \
sm4e b0.4s, v24.4s; \
sm4e b1.4s, v24.4s; \
sm4e b2.4s, v24.4s; \
sm4e b3.4s, v24.4s; \
sm4e b0.4s, v25.4s; \
sm4e b1.4s, v25.4s; \
sm4e b2.4s, v25.4s; \
sm4e b3.4s, v25.4s; \
sm4e b0.4s, v26.4s; \
sm4e b1.4s, v26.4s; \
sm4e b2.4s, v26.4s; \
sm4e b3.4s, v26.4s; \
sm4e b0.4s, v27.4s; \
sm4e b1.4s, v27.4s; \
sm4e b2.4s, v27.4s; \
sm4e b3.4s, v27.4s; \
sm4e b0.4s, v28.4s; \
sm4e b1.4s, v28.4s; \
sm4e b2.4s, v28.4s; \
sm4e b3.4s, v28.4s; \
sm4e b0.4s, v29.4s; \
sm4e b1.4s, v29.4s; \
sm4e b2.4s, v29.4s; \
sm4e b3.4s, v29.4s; \
sm4e b0.4s, v30.4s; \
sm4e b1.4s, v30.4s; \
sm4e b2.4s, v30.4s; \
sm4e b3.4s, v30.4s; \
sm4e b0.4s, v31.4s; \
sm4e b1.4s, v31.4s; \
sm4e b2.4s, v31.4s; \
sm4e b3.4s, v31.4s; \
rev64 b0.4s, b0.4s; \
rev64 b1.4s, b1.4s; \
rev64 b2.4s, b2.4s; \
rev64 b3.4s, b3.4s; \
ext b0.16b, b0.16b, b0.16b, #8; \
ext b1.16b, b1.16b, b1.16b, #8; \
ext b2.16b, b2.16b, b2.16b, #8; \
ext b3.16b, b3.16b, b3.16b, #8; \
rev32 b0.16b, b0.16b; \
rev32 b1.16b, b1.16b; \
rev32 b2.16b, b2.16b; \
rev32 b3.16b, b3.16b;
#define SM4_CRYPT_BLK4(b0, b1, b2, b3) \
rev32 b0.16b, b0.16b; \
rev32 b1.16b, b1.16b; \
rev32 b2.16b, b2.16b; \
rev32 b3.16b, b3.16b; \
SM4_CRYPT_BLK4_BE(b0, b1, b2, b3);
#define SM4_CRYPT_BLK8_BE(b0, b1, b2, b3, b4, b5, b6, b7) \
sm4e b0.4s, v24.4s; \
sm4e b1.4s, v24.4s; \
sm4e b2.4s, v24.4s; \
sm4e b3.4s, v24.4s; \
sm4e b4.4s, v24.4s; \
sm4e b5.4s, v24.4s; \
sm4e b6.4s, v24.4s; \
sm4e b7.4s, v24.4s; \
sm4e b0.4s, v25.4s; \
sm4e b1.4s, v25.4s; \
sm4e b2.4s, v25.4s; \
sm4e b3.4s, v25.4s; \
sm4e b4.4s, v25.4s; \
sm4e b5.4s, v25.4s; \
sm4e b6.4s, v25.4s; \
sm4e b7.4s, v25.4s; \
sm4e b0.4s, v26.4s; \
sm4e b1.4s, v26.4s; \
sm4e b2.4s, v26.4s; \
sm4e b3.4s, v26.4s; \
sm4e b4.4s, v26.4s; \
sm4e b5.4s, v26.4s; \
sm4e b6.4s, v26.4s; \
sm4e b7.4s, v26.4s; \
sm4e b0.4s, v27.4s; \
sm4e b1.4s, v27.4s; \
sm4e b2.4s, v27.4s; \
sm4e b3.4s, v27.4s; \
sm4e b4.4s, v27.4s; \
sm4e b5.4s, v27.4s; \
sm4e b6.4s, v27.4s; \
sm4e b7.4s, v27.4s; \
sm4e b0.4s, v28.4s; \
sm4e b1.4s, v28.4s; \
sm4e b2.4s, v28.4s; \
sm4e b3.4s, v28.4s; \
sm4e b4.4s, v28.4s; \
sm4e b5.4s, v28.4s; \
sm4e b6.4s, v28.4s; \
sm4e b7.4s, v28.4s; \
sm4e b0.4s, v29.4s; \
sm4e b1.4s, v29.4s; \
sm4e b2.4s, v29.4s; \
sm4e b3.4s, v29.4s; \
sm4e b4.4s, v29.4s; \
sm4e b5.4s, v29.4s; \
sm4e b6.4s, v29.4s; \
sm4e b7.4s, v29.4s; \
sm4e b0.4s, v30.4s; \
sm4e b1.4s, v30.4s; \
sm4e b2.4s, v30.4s; \
sm4e b3.4s, v30.4s; \
sm4e b4.4s, v30.4s; \
sm4e b5.4s, v30.4s; \
sm4e b6.4s, v30.4s; \
sm4e b7.4s, v30.4s; \
sm4e b0.4s, v31.4s; \
sm4e b1.4s, v31.4s; \
sm4e b2.4s, v31.4s; \
sm4e b3.4s, v31.4s; \
sm4e b4.4s, v31.4s; \
sm4e b5.4s, v31.4s; \
sm4e b6.4s, v31.4s; \
sm4e b7.4s, v31.4s; \
rev64 b0.4s, b0.4s; \
rev64 b1.4s, b1.4s; \
rev64 b2.4s, b2.4s; \
rev64 b3.4s, b3.4s; \
rev64 b4.4s, b4.4s; \
rev64 b5.4s, b5.4s; \
rev64 b6.4s, b6.4s; \
rev64 b7.4s, b7.4s; \
ext b0.16b, b0.16b, b0.16b, #8; \
ext b1.16b, b1.16b, b1.16b, #8; \
ext b2.16b, b2.16b, b2.16b, #8; \
ext b3.16b, b3.16b, b3.16b, #8; \
ext b4.16b, b4.16b, b4.16b, #8; \
ext b5.16b, b5.16b, b5.16b, #8; \
ext b6.16b, b6.16b, b6.16b, #8; \
ext b7.16b, b7.16b, b7.16b, #8; \
rev32 b0.16b, b0.16b; \
rev32 b1.16b, b1.16b; \
rev32 b2.16b, b2.16b; \
rev32 b3.16b, b3.16b; \
rev32 b4.16b, b4.16b; \
rev32 b5.16b, b5.16b; \
rev32 b6.16b, b6.16b; \
rev32 b7.16b, b7.16b;
#define SM4_CRYPT_BLK8(b0, b1, b2, b3, b4, b5, b6, b7) \
rev32 b0.16b, b0.16b; \
rev32 b1.16b, b1.16b; \
rev32 b2.16b, b2.16b; \
rev32 b3.16b, b3.16b; \
rev32 b4.16b, b4.16b; \
rev32 b5.16b, b5.16b; \
rev32 b6.16b, b6.16b; \
rev32 b7.16b, b7.16b; \
SM4_CRYPT_BLK8_BE(b0, b1, b2, b3, b4, b5, b6, b7);
linux-7.2.3/arch/arm64/crypto/sm4-ce-ccm-core.S
+329
View File
@@ -0,0 +1,329 @@
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
* SM4-CCM AEAD Algorithm using ARMv8 Crypto Extensions
* as specified in rfc8998
* https://datatracker.ietf.org/doc/html/rfc8998
*
* Copyright (C) 2022 Tianjia Zhang <tianjia.zhang@linux.alibaba.com>
*/
#include <linux/linkage.h>
#include <linux/cfi_types.h>
#include <asm/assembler.h>
#include "sm4-ce-asm.h"
.arch armv8-a+crypto
.irp b, 0, 1, 8, 9, 10, 11, 12, 13, 14, 15, 16, 24, 25, 26, 27, 28, 29, 30, 31
.set .Lv\b\().4s, \b
.endr
.macro sm4e, vd, vn
.inst 0xcec08400 | (.L\vn << 5) | .L\vd
.endm
/* Register macros */
#define RMAC v16
/* Helper macros. */
#define inc_le128(vctr) \
mov vctr.d[1], x8; \
mov vctr.d[0], x7; \
adds x8, x8, #1; \
rev64 vctr.16b, vctr.16b; \
adc x7, x7, xzr;
.align 3
SYM_FUNC_START(sm4_ce_cbcmac_update)
/* input:
* x0: round key array, CTX
* x1: mac
* x2: src
* w3: nblocks
*/
SM4_PREPARE(x0)
ld1 {RMAC.16b}, [x1]
.Lcbcmac_loop_4x:
cmp w3, #4
blt .Lcbcmac_loop_1x
sub w3, w3, #4
ld1 {v0.16b-v3.16b}, [x2], #64
SM4_CRYPT_BLK(RMAC)
eor RMAC.16b, RMAC.16b, v0.16b
SM4_CRYPT_BLK(RMAC)
eor RMAC.16b, RMAC.16b, v1.16b
SM4_CRYPT_BLK(RMAC)
eor RMAC.16b, RMAC.16b, v2.16b
SM4_CRYPT_BLK(RMAC)
eor RMAC.16b, RMAC.16b, v3.16b
cbz w3, .Lcbcmac_end
b .Lcbcmac_loop_4x
.Lcbcmac_loop_1x:
sub w3, w3, #1
ld1 {v0.16b}, [x2], #16
SM4_CRYPT_BLK(RMAC)
eor RMAC.16b, RMAC.16b, v0.16b
cbnz w3, .Lcbcmac_loop_1x
.Lcbcmac_end:
st1 {RMAC.16b}, [x1]
ret
SYM_FUNC_END(sm4_ce_cbcmac_update)
.align 3
SYM_FUNC_START(sm4_ce_ccm_final)
/* input:
* x0: round key array, CTX
* x1: ctr0 (big endian, 128 bit)
* x2: mac
*/
SM4_PREPARE(x0)
ld1 {RMAC.16b}, [x2]
ld1 {v0.16b}, [x1]
SM4_CRYPT_BLK2(RMAC, v0)
/* en-/decrypt the mac with ctr0 */
eor RMAC.16b, RMAC.16b, v0.16b
st1 {RMAC.16b}, [x2]
ret
SYM_FUNC_END(sm4_ce_ccm_final)
.align 3
SYM_TYPED_FUNC_START(sm4_ce_ccm_enc)
/* input:
* x0: round key array, CTX
* x1: dst
* x2: src
* x3: ctr (big endian, 128 bit)
* w4: nbytes
* x5: mac
*/
SM4_PREPARE(x0)
ldp x7, x8, [x3]
rev x7, x7
rev x8, x8
ld1 {RMAC.16b}, [x5]
.Lccm_enc_loop_4x:
cmp w4, #(4 * 16)
blt .Lccm_enc_loop_1x
sub w4, w4, #(4 * 16)
/* construct CTRs */
inc_le128(v8) /* +0 */
inc_le128(v9) /* +1 */
inc_le128(v10) /* +2 */
inc_le128(v11) /* +3 */
ld1 {v0.16b-v3.16b}, [x2], #64
SM4_CRYPT_BLK2(v8, RMAC)
eor v8.16b, v8.16b, v0.16b
eor RMAC.16b, RMAC.16b, v0.16b
SM4_CRYPT_BLK2(v9, RMAC)
eor v9.16b, v9.16b, v1.16b
eor RMAC.16b, RMAC.16b, v1.16b
SM4_CRYPT_BLK2(v10, RMAC)
eor v10.16b, v10.16b, v2.16b
eor RMAC.16b, RMAC.16b, v2.16b
SM4_CRYPT_BLK2(v11, RMAC)
eor v11.16b, v11.16b, v3.16b
eor RMAC.16b, RMAC.16b, v3.16b
st1 {v8.16b-v11.16b}, [x1], #64
cbz w4, .Lccm_enc_end
b .Lccm_enc_loop_4x
.Lccm_enc_loop_1x:
cmp w4, #16
blt .Lccm_enc_tail
sub w4, w4, #16
/* construct CTRs */
inc_le128(v8)
ld1 {v0.16b}, [x2], #16
SM4_CRYPT_BLK2(v8, RMAC)
eor v8.16b, v8.16b, v0.16b
eor RMAC.16b, RMAC.16b, v0.16b
st1 {v8.16b}, [x1], #16
cbz w4, .Lccm_enc_end
b .Lccm_enc_loop_1x
.Lccm_enc_tail:
/* construct CTRs */
inc_le128(v8)
SM4_CRYPT_BLK2(RMAC, v8)
/* store new MAC */
st1 {RMAC.16b}, [x5]
.Lccm_enc_tail_loop:
ldrb w0, [x2], #1 /* get 1 byte from input */
umov w9, v8.b[0] /* get top crypted CTR byte */
umov w6, RMAC.b[0] /* get top MAC byte */
eor w9, w9, w0 /* w9 = CTR ^ input */
eor w6, w6, w0 /* w6 = MAC ^ input */
strb w9, [x1], #1 /* store out byte */
strb w6, [x5], #1 /* store MAC byte */
subs w4, w4, #1
beq .Lccm_enc_ret
/* shift out one byte */
ext RMAC.16b, RMAC.16b, RMAC.16b, #1
ext v8.16b, v8.16b, v8.16b, #1
b .Lccm_enc_tail_loop
.Lccm_enc_end:
/* store new MAC */
st1 {RMAC.16b}, [x5]
/* store new CTR */
rev x7, x7
rev x8, x8
stp x7, x8, [x3]
.Lccm_enc_ret:
ret
SYM_FUNC_END(sm4_ce_ccm_enc)
.align 3
SYM_TYPED_FUNC_START(sm4_ce_ccm_dec)
/* input:
* x0: round key array, CTX
* x1: dst
* x2: src
* x3: ctr (big endian, 128 bit)
* w4: nbytes
* x5: mac
*/
SM4_PREPARE(x0)
ldp x7, x8, [x3]
rev x7, x7
rev x8, x8
ld1 {RMAC.16b}, [x5]
.Lccm_dec_loop_4x:
cmp w4, #(4 * 16)
blt .Lccm_dec_loop_1x
sub w4, w4, #(4 * 16)
/* construct CTRs */
inc_le128(v8) /* +0 */
inc_le128(v9) /* +1 */
inc_le128(v10) /* +2 */
inc_le128(v11) /* +3 */
ld1 {v0.16b-v3.16b}, [x2], #64
SM4_CRYPT_BLK2(v8, RMAC)
eor v8.16b, v8.16b, v0.16b
eor RMAC.16b, RMAC.16b, v8.16b
SM4_CRYPT_BLK2(v9, RMAC)
eor v9.16b, v9.16b, v1.16b
eor RMAC.16b, RMAC.16b, v9.16b
SM4_CRYPT_BLK2(v10, RMAC)
eor v10.16b, v10.16b, v2.16b
eor RMAC.16b, RMAC.16b, v10.16b
SM4_CRYPT_BLK2(v11, RMAC)
eor v11.16b, v11.16b, v3.16b
eor RMAC.16b, RMAC.16b, v11.16b
st1 {v8.16b-v11.16b}, [x1], #64
cbz w4, .Lccm_dec_end
b .Lccm_dec_loop_4x
.Lccm_dec_loop_1x:
cmp w4, #16
blt .Lccm_dec_tail
sub w4, w4, #16
/* construct CTRs */
inc_le128(v8)
ld1 {v0.16b}, [x2], #16
SM4_CRYPT_BLK2(v8, RMAC)
eor v8.16b, v8.16b, v0.16b
eor RMAC.16b, RMAC.16b, v8.16b
st1 {v8.16b}, [x1], #16
cbz w4, .Lccm_dec_end
b .Lccm_dec_loop_1x
.Lccm_dec_tail:
/* construct CTRs */
inc_le128(v8)
SM4_CRYPT_BLK2(RMAC, v8)
/* store new MAC */
st1 {RMAC.16b}, [x5]
.Lccm_dec_tail_loop:
ldrb w0, [x2], #1 /* get 1 byte from input */
umov w9, v8.b[0] /* get top crypted CTR byte */
umov w6, RMAC.b[0] /* get top MAC byte */
eor w9, w9, w0 /* w9 = CTR ^ input */
eor w6, w6, w9 /* w6 = MAC ^ output */
strb w9, [x1], #1 /* store out byte */
strb w6, [x5], #1 /* store MAC byte */
subs w4, w4, #1
beq .Lccm_dec_ret
/* shift out one byte */
ext RMAC.16b, RMAC.16b, RMAC.16b, #1
ext v8.16b, v8.16b, v8.16b, #1
b .Lccm_dec_tail_loop
.Lccm_dec_end:
/* store new MAC */
st1 {RMAC.16b}, [x5]
/* store new CTR */
rev x7, x7
rev x8, x8
stp x7, x8, [x3]
.Lccm_dec_ret:
ret
SYM_FUNC_END(sm4_ce_ccm_dec)
linux-7.2.3/arch/arm64/crypto/sm4-ce-ccm-glue.c
+285
View File
@@ -0,0 +1,285 @@
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
* SM4-CCM AEAD Algorithm using ARMv8 Crypto Extensions
* as specified in rfc8998
* https://datatracker.ietf.org/doc/html/rfc8998
*
* Copyright (C) 2022 Tianjia Zhang <tianjia.zhang@linux.alibaba.com>
*/
#include <linux/module.h>
#include <linux/crypto.h>
#include <linux/kernel.h>
#include <linux/cpufeature.h>
#include <asm/simd.h>
#include <crypto/scatterwalk.h>
#include <crypto/internal/aead.h>
#include <crypto/internal/skcipher.h>
#include <crypto/sm4.h>
#include "sm4-ce.h"
asmlinkage void sm4_ce_cbcmac_update(const u32 *rkey_enc, u8 *mac,
const u8 *src, unsigned int nblocks);
asmlinkage void sm4_ce_ccm_enc(const u32 *rkey_enc, u8 *dst, const u8 *src,
u8 *iv, unsigned int nbytes, u8 *mac);
asmlinkage void sm4_ce_ccm_dec(const u32 *rkey_enc, u8 *dst, const u8 *src,
u8 *iv, unsigned int nbytes, u8 *mac);
asmlinkage void sm4_ce_ccm_final(const u32 *rkey_enc, u8 *iv, u8 *mac);
static int ccm_setkey(struct crypto_aead *tfm, const u8 *key,
unsigned int key_len)
{
struct sm4_ctx *ctx = crypto_aead_ctx(tfm);
if (key_len != SM4_KEY_SIZE)
return -EINVAL;
scoped_ksimd()
sm4_ce_expand_key(key, ctx->rkey_enc, ctx->rkey_dec,
crypto_sm4_fk, crypto_sm4_ck);
return 0;
}
static int ccm_setauthsize(struct crypto_aead *tfm, unsigned int authsize)
{
if ((authsize & 1) || authsize < 4)
return -EINVAL;
return 0;
}
static int ccm_format_input(u8 info[], struct aead_request *req,
unsigned int msglen)
{
struct crypto_aead *aead = crypto_aead_reqtfm(req);
unsigned int l = req->iv[0] + 1;
unsigned int m;
__be32 len;
/* verify that CCM dimension 'L': 2 <= L <= 8 */
if (l < 2 || l > 8)
return -EINVAL;
if (l < 4 && msglen >> (8 * l))
return -EOVERFLOW;
memset(&req->iv[SM4_BLOCK_SIZE - l], 0, l);
memcpy(info, req->iv, SM4_BLOCK_SIZE);
m = crypto_aead_authsize(aead);
/* format flags field per RFC 3610/NIST 800-38C */
*info |= ((m - 2) / 2) << 3;
if (req->assoclen)
*info |= (1 << 6);
/*
* format message length field,
* Linux uses a u32 type to represent msglen
*/
if (l >= 4)
l = 4;
len = cpu_to_be32(msglen);
memcpy(&info[SM4_BLOCK_SIZE - l], (u8 *)&len + 4 - l, l);
return 0;
}
static void ccm_calculate_auth_mac(struct aead_request *req, u8 mac[])
{
struct crypto_aead *aead = crypto_aead_reqtfm(req);
struct sm4_ctx *ctx = crypto_aead_ctx(aead);
struct __packed { __be16 l; __be32 h; } aadlen;
u32 assoclen = req->assoclen;
struct scatter_walk walk;
unsigned int len;
if (assoclen < 0xff00) {
aadlen.l = cpu_to_be16(assoclen);
len = 2;
} else {
aadlen.l = cpu_to_be16(0xfffe);
put_unaligned_be32(assoclen, &aadlen.h);
len = 6;
}
sm4_ce_crypt_block(ctx->rkey_enc, mac, mac);
crypto_xor(mac, (const u8 *)&aadlen, len);
scatterwalk_start(&walk, req->src);
do {
unsigned int n, orig_n;
const u8 *p;
orig_n = scatterwalk_next(&walk, assoclen);
p = walk.addr;
n = orig_n;
while (n > 0) {
unsigned int l, nblocks;
if (len == SM4_BLOCK_SIZE) {
if (n < SM4_BLOCK_SIZE) {
sm4_ce_crypt_block(ctx->rkey_enc,
mac, mac);
len = 0;
} else {
nblocks = n / SM4_BLOCK_SIZE;
sm4_ce_cbcmac_update(ctx->rkey_enc,
mac, p, nblocks);
p += nblocks * SM4_BLOCK_SIZE;
n %= SM4_BLOCK_SIZE;
continue;
}
}
l = min(n, SM4_BLOCK_SIZE - len);
if (l) {
crypto_xor(mac + len, p, l);
len += l;
p += l;
n -= l;
}
}
scatterwalk_done_src(&walk, orig_n);
assoclen -= orig_n;
} while (assoclen);
}
static int ccm_crypt(struct aead_request *req, struct skcipher_walk *walk,
u32 *rkey_enc, u8 mac[],
void (*sm4_ce_ccm_crypt)(const u32 *rkey_enc, u8 *dst,
const u8 *src, u8 *iv,
unsigned int nbytes, u8 *mac))
{
u8 __aligned(8) ctr0[SM4_BLOCK_SIZE];
int err = 0;
/* preserve the initial ctr0 for the TAG */
memcpy(ctr0, walk->iv, SM4_BLOCK_SIZE);
crypto_inc(walk->iv, SM4_BLOCK_SIZE);
scoped_ksimd() {
if (req->assoclen)
ccm_calculate_auth_mac(req, mac);
while (walk->nbytes) {
unsigned int tail = walk->nbytes % SM4_BLOCK_SIZE;
if (walk->nbytes == walk->total)
tail = 0;
sm4_ce_ccm_crypt(rkey_enc, walk->dst.virt.addr,
walk->src.virt.addr, walk->iv,
walk->nbytes - tail, mac);
err = skcipher_walk_done(walk, tail);
}
sm4_ce_ccm_final(rkey_enc, ctr0, mac);
}
return err;
}
static int ccm_encrypt(struct aead_request *req)
{
struct crypto_aead *aead = crypto_aead_reqtfm(req);
struct sm4_ctx *ctx = crypto_aead_ctx(aead);
u8 __aligned(8) mac[SM4_BLOCK_SIZE];
struct skcipher_walk walk;
int err;
err = ccm_format_input(mac, req, req->cryptlen);
if (err)
return err;
err = skcipher_walk_aead_encrypt(&walk, req, false);
if (err)
return err;
err = ccm_crypt(req, &walk, ctx->rkey_enc, mac, sm4_ce_ccm_enc);
if (err)
return err;
/* copy authtag to end of dst */
scatterwalk_map_and_copy(mac, req->dst, req->assoclen + req->cryptlen,
crypto_aead_authsize(aead), 1);
return 0;
}
static int ccm_decrypt(struct aead_request *req)
{
struct crypto_aead *aead = crypto_aead_reqtfm(req);
unsigned int authsize = crypto_aead_authsize(aead);
struct sm4_ctx *ctx = crypto_aead_ctx(aead);
u8 __aligned(8) mac[SM4_BLOCK_SIZE];
u8 authtag[SM4_BLOCK_SIZE];
struct skcipher_walk walk;
int err;
err = ccm_format_input(mac, req, req->cryptlen - authsize);
if (err)
return err;
err = skcipher_walk_aead_decrypt(&walk, req, false);
if (err)
return err;
err = ccm_crypt(req, &walk, ctx->rkey_enc, mac, sm4_ce_ccm_dec);
if (err)
return err;
/* compare calculated auth tag with the stored one */
scatterwalk_map_and_copy(authtag, req->src,
req->assoclen + req->cryptlen - authsize,
authsize, 0);
if (crypto_memneq(authtag, mac, authsize))
return -EBADMSG;
return 0;
}
static struct aead_alg sm4_ccm_alg = {
.base = {
.cra_name = "ccm(sm4)",
.cra_driver_name = "ccm-sm4-ce",
.cra_priority = 400,
.cra_blocksize = 1,
.cra_ctxsize = sizeof(struct sm4_ctx),
.cra_module = THIS_MODULE,
},
.ivsize = SM4_BLOCK_SIZE,
.chunksize = SM4_BLOCK_SIZE,
.maxauthsize = SM4_BLOCK_SIZE,
.setkey = ccm_setkey,
.setauthsize = ccm_setauthsize,
.encrypt = ccm_encrypt,
.decrypt = ccm_decrypt,
};
static int __init sm4_ce_ccm_init(void)
{
return crypto_register_aead(&sm4_ccm_alg);
}
static void __exit sm4_ce_ccm_exit(void)
{
crypto_unregister_aead(&sm4_ccm_alg);
}
module_cpu_feature_match(SM4, sm4_ce_ccm_init);
module_exit(sm4_ce_ccm_exit);
MODULE_DESCRIPTION("Synchronous SM4 in CCM mode using ARMv8 Crypto Extensions");
MODULE_ALIAS_CRYPTO("ccm(sm4)");
MODULE_AUTHOR("Tianjia Zhang <tianjia.zhang@linux.alibaba.com>");
MODULE_LICENSE("GPL v2");
linux-7.2.3/arch/arm64/crypto/sm4-ce-cipher-core.S
+36
View File
@@ -0,0 +1,36 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/linkage.h>
#include <asm/assembler.h>
.irp b, 0, 1, 2, 3, 4, 5, 6, 7, 8
.set .Lv\b\().4s, \b
.endr
.macro sm4e, rd, rn
.inst 0xcec08400 | .L\rd | (.L\rn << 5)
.endm
/*
* void sm4_ce_do_crypt(const u32 *rk, u32 *out, const u32 *in);
*/
.text
SYM_FUNC_START(sm4_ce_do_crypt)
ld1 {v8.4s}, [x2]
ld1 {v0.4s-v3.4s}, [x0], #64
CPU_LE( rev32 v8.16b, v8.16b )
ld1 {v4.4s-v7.4s}, [x0]
sm4e v8.4s, v0.4s
sm4e v8.4s, v1.4s
sm4e v8.4s, v2.4s
sm4e v8.4s, v3.4s
sm4e v8.4s, v4.4s
sm4e v8.4s, v5.4s
sm4e v8.4s, v6.4s
sm4e v8.4s, v7.4s
rev64 v8.4s, v8.4s
ext v8.16b, v8.16b, v8.16b, #8
CPU_LE( rev32 v8.16b, v8.16b )
st1 {v8.4s}, [x1]
ret
SYM_FUNC_END(sm4_ce_do_crypt)
linux-7.2.3/arch/arm64/crypto/sm4-ce-cipher-glue.c
+80
View File
@@ -0,0 +1,80 @@
// SPDX-License-Identifier: GPL-2.0
#include <asm/neon.h>
#include <asm/simd.h>
#include <crypto/algapi.h>
#include <crypto/sm4.h>
#include <crypto/internal/simd.h>
#include <linux/module.h>
#include <linux/cpufeature.h>
#include <linux/types.h>
MODULE_ALIAS_CRYPTO("sm4");
MODULE_ALIAS_CRYPTO("sm4-ce");
MODULE_DESCRIPTION("SM4 symmetric cipher using ARMv8 Crypto Extensions");
MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
MODULE_LICENSE("GPL v2");
asmlinkage void sm4_ce_do_crypt(const u32 *rk, void *out, const void *in);
static int sm4_ce_setkey(struct crypto_tfm *tfm, const u8 *key,
unsigned int key_len)
{
struct sm4_ctx *ctx = crypto_tfm_ctx(tfm);
return sm4_expandkey(ctx, key, key_len);
}
static void sm4_ce_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
{
const struct sm4_ctx *ctx = crypto_tfm_ctx(tfm);
if (!crypto_simd_usable()) {
sm4_crypt_block(ctx->rkey_enc, out, in);
} else {
scoped_ksimd()
sm4_ce_do_crypt(ctx->rkey_enc, out, in);
}
}
static void sm4_ce_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
{
const struct sm4_ctx *ctx = crypto_tfm_ctx(tfm);
if (!crypto_simd_usable()) {
sm4_crypt_block(ctx->rkey_dec, out, in);
} else {
scoped_ksimd()
sm4_ce_do_crypt(ctx->rkey_dec, out, in);
}
}
static struct crypto_alg sm4_ce_alg = {
.cra_name = "sm4",
.cra_driver_name = "sm4-ce",
.cra_priority = 300,
.cra_flags = CRYPTO_ALG_TYPE_CIPHER,
.cra_blocksize = SM4_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct sm4_ctx),
.cra_module = THIS_MODULE,
.cra_u.cipher = {
.cia_min_keysize = SM4_KEY_SIZE,
.cia_max_keysize = SM4_KEY_SIZE,
.cia_setkey = sm4_ce_setkey,
.cia_encrypt = sm4_ce_encrypt,
.cia_decrypt = sm4_ce_decrypt
}
};
static int __init sm4_ce_mod_init(void)
{
return crypto_register_alg(&sm4_ce_alg);
}
static void __exit sm4_ce_mod_fini(void)
{
crypto_unregister_alg(&sm4_ce_alg);
}
module_cpu_feature_match(SM4, sm4_ce_mod_init);
module_exit(sm4_ce_mod_fini);
linux-7.2.3/arch/arm64/crypto/sm4-ce-core.S
+935
View File
@@ -0,0 +1,935 @@
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
* SM4 Cipher Algorithm for ARMv8 with Crypto Extensions
* as specified in
* https://tools.ietf.org/id/draft-ribose-cfrg-sm4-10.html
*
* Copyright (C) 2022, Alibaba Group.
* Copyright (C) 2022 Tianjia Zhang <tianjia.zhang@linux.alibaba.com>
*/
#include <linux/linkage.h>
#include <asm/assembler.h>
#include "sm4-ce-asm.h"
.arch armv8-a+crypto
.irp b, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, \
20, 24, 25, 26, 27, 28, 29, 30, 31
.set .Lv\b\().4s, \b
.endr
.macro sm4e, vd, vn
.inst 0xcec08400 | (.L\vn << 5) | .L\vd
.endm
.macro sm4ekey, vd, vn, vm
.inst 0xce60c800 | (.L\vm << 16) | (.L\vn << 5) | .L\vd
.endm
/* Register macros */
#define RTMP0 v16
#define RTMP1 v17
#define RTMP2 v18
#define RTMP3 v19
#define RIV v20
#define RMAC v20
#define RMASK v21
.align 3
SYM_FUNC_START(sm4_ce_expand_key)
/* input:
* x0: 128-bit key
* x1: rkey_enc
* x2: rkey_dec
* x3: fk array
* x4: ck array
*/
ld1 {v0.16b}, [x0];
rev32 v0.16b, v0.16b;
ld1 {v1.16b}, [x3];
/* load ck */
ld1 {v24.16b-v27.16b}, [x4], #64;
ld1 {v28.16b-v31.16b}, [x4];
/* input ^ fk */
eor v0.16b, v0.16b, v1.16b;
sm4ekey v0.4s, v0.4s, v24.4s;
sm4ekey v1.4s, v0.4s, v25.4s;
sm4ekey v2.4s, v1.4s, v26.4s;
sm4ekey v3.4s, v2.4s, v27.4s;
sm4ekey v4.4s, v3.4s, v28.4s;
sm4ekey v5.4s, v4.4s, v29.4s;
sm4ekey v6.4s, v5.4s, v30.4s;
sm4ekey v7.4s, v6.4s, v31.4s;
adr_l x5, .Lbswap128_mask
ld1 {v24.16b}, [x5]
st1 {v0.16b-v3.16b}, [x1], #64;
st1 {v4.16b-v7.16b}, [x1];
tbl v16.16b, {v7.16b}, v24.16b
tbl v17.16b, {v6.16b}, v24.16b
tbl v18.16b, {v5.16b}, v24.16b
tbl v19.16b, {v4.16b}, v24.16b
tbl v20.16b, {v3.16b}, v24.16b
tbl v21.16b, {v2.16b}, v24.16b
tbl v22.16b, {v1.16b}, v24.16b
tbl v23.16b, {v0.16b}, v24.16b
st1 {v16.16b-v19.16b}, [x2], #64
st1 {v20.16b-v23.16b}, [x2]
ret;
SYM_FUNC_END(sm4_ce_expand_key)
.align 3
SYM_FUNC_START(sm4_ce_crypt_block)
/* input:
* x0: round key array, CTX
* x1: dst
* x2: src
*/
SM4_PREPARE(x0)
ld1 {v0.16b}, [x2];
SM4_CRYPT_BLK(v0);
st1 {v0.16b}, [x1];
ret;
SYM_FUNC_END(sm4_ce_crypt_block)
.align 3
SYM_FUNC_START(sm4_ce_crypt)
/* input:
* x0: round key array, CTX
* x1: dst
* x2: src
* w3: nblocks
*/
SM4_PREPARE(x0)
.Lcrypt_loop_blk:
sub w3, w3, #8;
tbnz w3, #31, .Lcrypt_tail8;
ld1 {v0.16b-v3.16b}, [x2], #64;
ld1 {v4.16b-v7.16b}, [x2], #64;
SM4_CRYPT_BLK8(v0, v1, v2, v3, v4, v5, v6, v7);
st1 {v0.16b-v3.16b}, [x1], #64;
st1 {v4.16b-v7.16b}, [x1], #64;
cbz w3, .Lcrypt_end;
b .Lcrypt_loop_blk;
.Lcrypt_tail8:
add w3, w3, #8;
cmp w3, #4;
blt .Lcrypt_tail4;
sub w3, w3, #4;
ld1 {v0.16b-v3.16b}, [x2], #64;
SM4_CRYPT_BLK4(v0, v1, v2, v3);
st1 {v0.16b-v3.16b}, [x1], #64;
cbz w3, .Lcrypt_end;
.Lcrypt_tail4:
sub w3, w3, #1;
ld1 {v0.16b}, [x2], #16;
SM4_CRYPT_BLK(v0);
st1 {v0.16b}, [x1], #16;
cbnz w3, .Lcrypt_tail4;
.Lcrypt_end:
ret;
SYM_FUNC_END(sm4_ce_crypt)
.align 3
SYM_FUNC_START(sm4_ce_cbc_enc)
/* input:
* x0: round key array, CTX
* x1: dst
* x2: src
* x3: iv (big endian, 128 bit)
* w4: nblocks
*/
SM4_PREPARE(x0)
ld1 {RIV.16b}, [x3]
.Lcbc_enc_loop_4x:
cmp w4, #4
blt .Lcbc_enc_loop_1x
sub w4, w4, #4
ld1 {v0.16b-v3.16b}, [x2], #64
eor v0.16b, v0.16b, RIV.16b
SM4_CRYPT_BLK(v0)
eor v1.16b, v1.16b, v0.16b
SM4_CRYPT_BLK(v1)
eor v2.16b, v2.16b, v1.16b
SM4_CRYPT_BLK(v2)
eor v3.16b, v3.16b, v2.16b
SM4_CRYPT_BLK(v3)
st1 {v0.16b-v3.16b}, [x1], #64
mov RIV.16b, v3.16b
cbz w4, .Lcbc_enc_end
b .Lcbc_enc_loop_4x
.Lcbc_enc_loop_1x:
sub w4, w4, #1
ld1 {v0.16b}, [x2], #16
eor RIV.16b, RIV.16b, v0.16b
SM4_CRYPT_BLK(RIV)
st1 {RIV.16b}, [x1], #16
cbnz w4, .Lcbc_enc_loop_1x
.Lcbc_enc_end:
/* store new IV */
st1 {RIV.16b}, [x3]
ret
SYM_FUNC_END(sm4_ce_cbc_enc)
.align 3
SYM_FUNC_START(sm4_ce_cbc_dec)
/* input:
* x0: round key array, CTX
* x1: dst
* x2: src
* x3: iv (big endian, 128 bit)
* w4: nblocks
*/
SM4_PREPARE(x0)
ld1 {RIV.16b}, [x3]
.Lcbc_dec_loop_8x:
sub w4, w4, #8
tbnz w4, #31, .Lcbc_dec_4x
ld1 {v0.16b-v3.16b}, [x2], #64
ld1 {v4.16b-v7.16b}, [x2], #64
rev32 v8.16b, v0.16b
rev32 v9.16b, v1.16b
rev32 v10.16b, v2.16b
rev32 v11.16b, v3.16b
rev32 v12.16b, v4.16b
rev32 v13.16b, v5.16b
rev32 v14.16b, v6.16b
rev32 v15.16b, v7.16b
SM4_CRYPT_BLK8_BE(v8, v9, v10, v11, v12, v13, v14, v15)
eor v8.16b, v8.16b, RIV.16b
eor v9.16b, v9.16b, v0.16b
eor v10.16b, v10.16b, v1.16b
eor v11.16b, v11.16b, v2.16b
eor v12.16b, v12.16b, v3.16b
eor v13.16b, v13.16b, v4.16b
eor v14.16b, v14.16b, v5.16b
eor v15.16b, v15.16b, v6.16b
st1 {v8.16b-v11.16b}, [x1], #64
st1 {v12.16b-v15.16b}, [x1], #64
mov RIV.16b, v7.16b
cbz w4, .Lcbc_dec_end
b .Lcbc_dec_loop_8x
.Lcbc_dec_4x:
add w4, w4, #8
cmp w4, #4
blt .Lcbc_dec_loop_1x
sub w4, w4, #4
ld1 {v0.16b-v3.16b}, [x2], #64
rev32 v8.16b, v0.16b
rev32 v9.16b, v1.16b
rev32 v10.16b, v2.16b
rev32 v11.16b, v3.16b
SM4_CRYPT_BLK4_BE(v8, v9, v10, v11)
eor v8.16b, v8.16b, RIV.16b
eor v9.16b, v9.16b, v0.16b
eor v10.16b, v10.16b, v1.16b
eor v11.16b, v11.16b, v2.16b
st1 {v8.16b-v11.16b}, [x1], #64
mov RIV.16b, v3.16b
cbz w4, .Lcbc_dec_end
.Lcbc_dec_loop_1x:
sub w4, w4, #1
ld1 {v0.16b}, [x2], #16
rev32 v8.16b, v0.16b
SM4_CRYPT_BLK_BE(v8)
eor v8.16b, v8.16b, RIV.16b
st1 {v8.16b}, [x1], #16
mov RIV.16b, v0.16b
cbnz w4, .Lcbc_dec_loop_1x
.Lcbc_dec_end:
/* store new IV */
st1 {RIV.16b}, [x3]
ret
SYM_FUNC_END(sm4_ce_cbc_dec)
.align 3
SYM_FUNC_START(sm4_ce_cbc_cts_enc)
/* input:
* x0: round key array, CTX
* x1: dst
* x2: src
* x3: iv (big endian, 128 bit)
* w4: nbytes
*/
SM4_PREPARE(x0)
sub w5, w4, #16
uxtw x5, w5
ld1 {RIV.16b}, [x3]
ld1 {v0.16b}, [x2]
eor RIV.16b, RIV.16b, v0.16b
SM4_CRYPT_BLK(RIV)
/* load permute table */
adr_l x6, .Lcts_permute_table
add x7, x6, #32
add x6, x6, x5
sub x7, x7, x5
ld1 {v3.16b}, [x6]
ld1 {v4.16b}, [x7]
/* overlapping loads */
add x2, x2, x5
ld1 {v1.16b}, [x2]
/* create Cn from En-1 */
tbl v0.16b, {RIV.16b}, v3.16b
/* padding Pn with zeros */
tbl v1.16b, {v1.16b}, v4.16b
eor v1.16b, v1.16b, RIV.16b
SM4_CRYPT_BLK(v1)
/* overlapping stores */
add x5, x1, x5
st1 {v0.16b}, [x5]
st1 {v1.16b}, [x1]
ret
SYM_FUNC_END(sm4_ce_cbc_cts_enc)
.align 3
SYM_FUNC_START(sm4_ce_cbc_cts_dec)
/* input:
* x0: round key array, CTX
* x1: dst
* x2: src
* x3: iv (big endian, 128 bit)
* w4: nbytes
*/
SM4_PREPARE(x0)
sub w5, w4, #16
uxtw x5, w5
ld1 {RIV.16b}, [x3]
/* load permute table */
adr_l x6, .Lcts_permute_table
add x7, x6, #32
add x6, x6, x5
sub x7, x7, x5
ld1 {v3.16b}, [x6]
ld1 {v4.16b}, [x7]
/* overlapping loads */
ld1 {v0.16b}, [x2], x5
ld1 {v1.16b}, [x2]
SM4_CRYPT_BLK(v0)
/* select the first Ln bytes of Xn to create Pn */
tbl v2.16b, {v0.16b}, v3.16b
eor v2.16b, v2.16b, v1.16b
/* overwrite the first Ln bytes with Cn to create En-1 */
tbx v0.16b, {v1.16b}, v4.16b
SM4_CRYPT_BLK(v0)
eor v0.16b, v0.16b, RIV.16b
/* overlapping stores */
add x5, x1, x5
st1 {v2.16b}, [x5]
st1 {v0.16b}, [x1]
ret
SYM_FUNC_END(sm4_ce_cbc_cts_dec)
.align 3
SYM_FUNC_START(sm4_ce_ctr_enc)
/* input:
* x0: round key array, CTX
* x1: dst
* x2: src
* x3: ctr (big endian, 128 bit)
* w4: nblocks
*/
SM4_PREPARE(x0)
ldp x7, x8, [x3]
rev x7, x7
rev x8, x8
.Lctr_loop_8x:
sub w4, w4, #8
tbnz w4, #31, .Lctr_4x
#define inc_le128(vctr) \
mov vctr.d[1], x8; \
mov vctr.d[0], x7; \
adds x8, x8, #1; \
rev64 vctr.16b, vctr.16b; \
adc x7, x7, xzr;
/* construct CTRs */
inc_le128(v0) /* +0 */
inc_le128(v1) /* +1 */
inc_le128(v2) /* +2 */
inc_le128(v3) /* +3 */
inc_le128(v4) /* +4 */
inc_le128(v5) /* +5 */
inc_le128(v6) /* +6 */
inc_le128(v7) /* +7 */
ld1 {v8.16b-v11.16b}, [x2], #64
ld1 {v12.16b-v15.16b}, [x2], #64
SM4_CRYPT_BLK8(v0, v1, v2, v3, v4, v5, v6, v7)
eor v0.16b, v0.16b, v8.16b
eor v1.16b, v1.16b, v9.16b
eor v2.16b, v2.16b, v10.16b
eor v3.16b, v3.16b, v11.16b
eor v4.16b, v4.16b, v12.16b
eor v5.16b, v5.16b, v13.16b
eor v6.16b, v6.16b, v14.16b
eor v7.16b, v7.16b, v15.16b
st1 {v0.16b-v3.16b}, [x1], #64
st1 {v4.16b-v7.16b}, [x1], #64
cbz w4, .Lctr_end
b .Lctr_loop_8x
.Lctr_4x:
add w4, w4, #8
cmp w4, #4
blt .Lctr_loop_1x
sub w4, w4, #4
/* construct CTRs */
inc_le128(v0) /* +0 */
inc_le128(v1) /* +1 */
inc_le128(v2) /* +2 */
inc_le128(v3) /* +3 */
ld1 {v8.16b-v11.16b}, [x2], #64
SM4_CRYPT_BLK4(v0, v1, v2, v3)
eor v0.16b, v0.16b, v8.16b
eor v1.16b, v1.16b, v9.16b
eor v2.16b, v2.16b, v10.16b
eor v3.16b, v3.16b, v11.16b
st1 {v0.16b-v3.16b}, [x1], #64
cbz w4, .Lctr_end
.Lctr_loop_1x:
sub w4, w4, #1
/* construct CTRs */
inc_le128(v0)
ld1 {v8.16b}, [x2], #16
SM4_CRYPT_BLK(v0)
eor v0.16b, v0.16b, v8.16b
st1 {v0.16b}, [x1], #16
cbnz w4, .Lctr_loop_1x
.Lctr_end:
/* store new CTR */
rev x7, x7
rev x8, x8
stp x7, x8, [x3]
ret
SYM_FUNC_END(sm4_ce_ctr_enc)
#define tweak_next(vt, vin, RTMP) \
sshr RTMP.2d, vin.2d, #63; \
and RTMP.16b, RTMP.16b, RMASK.16b; \
add vt.2d, vin.2d, vin.2d; \
ext RTMP.16b, RTMP.16b, RTMP.16b, #8; \
eor vt.16b, vt.16b, RTMP.16b;
.align 3
SYM_FUNC_START(sm4_ce_xts_enc)
/* input:
* x0: round key array, CTX
* x1: dst
* x2: src
* x3: tweak (big endian, 128 bit)
* w4: nbytes
* x5: round key array for IV
*/
ld1 {v8.16b}, [x3]
cbz x5, .Lxts_enc_nofirst
SM4_PREPARE(x5)
/* Generate first tweak */
SM4_CRYPT_BLK(v8)
.Lxts_enc_nofirst:
SM4_PREPARE(x0)
ands w5, w4, #15
lsr w4, w4, #4
sub w6, w4, #1
csel w4, w4, w6, eq
uxtw x5, w5
movi RMASK.2s, #0x1
movi RTMP0.2s, #0x87
uzp1 RMASK.4s, RMASK.4s, RTMP0.4s
cbz w4, .Lxts_enc_cts
.Lxts_enc_loop_8x:
sub w4, w4, #8
tbnz w4, #31, .Lxts_enc_4x
tweak_next( v9, v8, RTMP0)
tweak_next(v10, v9, RTMP1)
tweak_next(v11, v10, RTMP2)
tweak_next(v12, v11, RTMP3)
tweak_next(v13, v12, RTMP0)
tweak_next(v14, v13, RTMP1)
tweak_next(v15, v14, RTMP2)
ld1 {v0.16b-v3.16b}, [x2], #64
ld1 {v4.16b-v7.16b}, [x2], #64
eor v0.16b, v0.16b, v8.16b
eor v1.16b, v1.16b, v9.16b
eor v2.16b, v2.16b, v10.16b
eor v3.16b, v3.16b, v11.16b
eor v4.16b, v4.16b, v12.16b
eor v5.16b, v5.16b, v13.16b
eor v6.16b, v6.16b, v14.16b
eor v7.16b, v7.16b, v15.16b
SM4_CRYPT_BLK8(v0, v1, v2, v3, v4, v5, v6, v7)
eor v0.16b, v0.16b, v8.16b
eor v1.16b, v1.16b, v9.16b
eor v2.16b, v2.16b, v10.16b
eor v3.16b, v3.16b, v11.16b
eor v4.16b, v4.16b, v12.16b
eor v5.16b, v5.16b, v13.16b
eor v6.16b, v6.16b, v14.16b
eor v7.16b, v7.16b, v15.16b
st1 {v0.16b-v3.16b}, [x1], #64
st1 {v4.16b-v7.16b}, [x1], #64
tweak_next(v8, v15, RTMP3)
cbz w4, .Lxts_enc_cts
b .Lxts_enc_loop_8x
.Lxts_enc_4x:
add w4, w4, #8
cmp w4, #4
blt .Lxts_enc_loop_1x
sub w4, w4, #4
tweak_next( v9, v8, RTMP0)
tweak_next(v10, v9, RTMP1)
tweak_next(v11, v10, RTMP2)
ld1 {v0.16b-v3.16b}, [x2], #64
eor v0.16b, v0.16b, v8.16b
eor v1.16b, v1.16b, v9.16b
eor v2.16b, v2.16b, v10.16b
eor v3.16b, v3.16b, v11.16b
SM4_CRYPT_BLK4(v0, v1, v2, v3)
eor v0.16b, v0.16b, v8.16b
eor v1.16b, v1.16b, v9.16b
eor v2.16b, v2.16b, v10.16b
eor v3.16b, v3.16b, v11.16b
st1 {v0.16b-v3.16b}, [x1], #64
tweak_next(v8, v11, RTMP3)
cbz w4, .Lxts_enc_cts
.Lxts_enc_loop_1x:
sub w4, w4, #1
ld1 {v0.16b}, [x2], #16
eor v0.16b, v0.16b, v8.16b
SM4_CRYPT_BLK(v0)
eor v0.16b, v0.16b, v8.16b
st1 {v0.16b}, [x1], #16
tweak_next(v8, v8, RTMP0)
cbnz w4, .Lxts_enc_loop_1x
.Lxts_enc_cts:
cbz x5, .Lxts_enc_end
/* cipher text stealing */
tweak_next(v9, v8, RTMP0)
ld1 {v0.16b}, [x2]
eor v0.16b, v0.16b, v8.16b
SM4_CRYPT_BLK(v0)
eor v0.16b, v0.16b, v8.16b
/* load permute table */
adr_l x6, .Lcts_permute_table
add x7, x6, #32
add x6, x6, x5
sub x7, x7, x5
ld1 {v3.16b}, [x6]
ld1 {v4.16b}, [x7]
/* overlapping loads */
add x2, x2, x5
ld1 {v1.16b}, [x2]
/* create Cn from En-1 */
tbl v2.16b, {v0.16b}, v3.16b
/* padding Pn with En-1 at the end */
tbx v0.16b, {v1.16b}, v4.16b
eor v0.16b, v0.16b, v9.16b
SM4_CRYPT_BLK(v0)
eor v0.16b, v0.16b, v9.16b
/* overlapping stores */
add x5, x1, x5
st1 {v2.16b}, [x5]
st1 {v0.16b}, [x1]
b .Lxts_enc_ret
.Lxts_enc_end:
/* store new tweak */
st1 {v8.16b}, [x3]
.Lxts_enc_ret:
ret
SYM_FUNC_END(sm4_ce_xts_enc)
.align 3
SYM_FUNC_START(sm4_ce_xts_dec)
/* input:
* x0: round key array, CTX
* x1: dst
* x2: src
* x3: tweak (big endian, 128 bit)
* w4: nbytes
* x5: round key array for IV
*/
ld1 {v8.16b}, [x3]
cbz x5, .Lxts_dec_nofirst
SM4_PREPARE(x5)
/* Generate first tweak */
SM4_CRYPT_BLK(v8)
.Lxts_dec_nofirst:
SM4_PREPARE(x0)
ands w5, w4, #15
lsr w4, w4, #4
sub w6, w4, #1
csel w4, w4, w6, eq
uxtw x5, w5
movi RMASK.2s, #0x1
movi RTMP0.2s, #0x87
uzp1 RMASK.4s, RMASK.4s, RTMP0.4s
cbz w4, .Lxts_dec_cts
.Lxts_dec_loop_8x:
sub w4, w4, #8
tbnz w4, #31, .Lxts_dec_4x
tweak_next( v9, v8, RTMP0)
tweak_next(v10, v9, RTMP1)
tweak_next(v11, v10, RTMP2)
tweak_next(v12, v11, RTMP3)
tweak_next(v13, v12, RTMP0)
tweak_next(v14, v13, RTMP1)
tweak_next(v15, v14, RTMP2)
ld1 {v0.16b-v3.16b}, [x2], #64
ld1 {v4.16b-v7.16b}, [x2], #64
eor v0.16b, v0.16b, v8.16b
eor v1.16b, v1.16b, v9.16b
eor v2.16b, v2.16b, v10.16b
eor v3.16b, v3.16b, v11.16b
eor v4.16b, v4.16b, v12.16b
eor v5.16b, v5.16b, v13.16b
eor v6.16b, v6.16b, v14.16b
eor v7.16b, v7.16b, v15.16b
SM4_CRYPT_BLK8(v0, v1, v2, v3, v4, v5, v6, v7)
eor v0.16b, v0.16b, v8.16b
eor v1.16b, v1.16b, v9.16b
eor v2.16b, v2.16b, v10.16b
eor v3.16b, v3.16b, v11.16b
eor v4.16b, v4.16b, v12.16b
eor v5.16b, v5.16b, v13.16b
eor v6.16b, v6.16b, v14.16b
eor v7.16b, v7.16b, v15.16b
st1 {v0.16b-v3.16b}, [x1], #64
st1 {v4.16b-v7.16b}, [x1], #64
tweak_next(v8, v15, RTMP3)
cbz w4, .Lxts_dec_cts
b .Lxts_dec_loop_8x
.Lxts_dec_4x:
add w4, w4, #8
cmp w4, #4
blt .Lxts_dec_loop_1x
sub w4, w4, #4
tweak_next( v9, v8, RTMP0)
tweak_next(v10, v9, RTMP1)
tweak_next(v11, v10, RTMP2)
ld1 {v0.16b-v3.16b}, [x2], #64
eor v0.16b, v0.16b, v8.16b
eor v1.16b, v1.16b, v9.16b
eor v2.16b, v2.16b, v10.16b
eor v3.16b, v3.16b, v11.16b
SM4_CRYPT_BLK4(v0, v1, v2, v3)
eor v0.16b, v0.16b, v8.16b
eor v1.16b, v1.16b, v9.16b
eor v2.16b, v2.16b, v10.16b
eor v3.16b, v3.16b, v11.16b
st1 {v0.16b-v3.16b}, [x1], #64
tweak_next(v8, v11, RTMP3)
cbz w4, .Lxts_dec_cts
.Lxts_dec_loop_1x:
sub w4, w4, #1
ld1 {v0.16b}, [x2], #16
eor v0.16b, v0.16b, v8.16b
SM4_CRYPT_BLK(v0)
eor v0.16b, v0.16b, v8.16b
st1 {v0.16b}, [x1], #16
tweak_next(v8, v8, RTMP0)
cbnz w4, .Lxts_dec_loop_1x
.Lxts_dec_cts:
cbz x5, .Lxts_dec_end
/* cipher text stealing */
tweak_next(v9, v8, RTMP0)
ld1 {v0.16b}, [x2]
eor v0.16b, v0.16b, v9.16b
SM4_CRYPT_BLK(v0)
eor v0.16b, v0.16b, v9.16b
/* load permute table */
adr_l x6, .Lcts_permute_table
add x7, x6, #32
add x6, x6, x5
sub x7, x7, x5
ld1 {v3.16b}, [x6]
ld1 {v4.16b}, [x7]
/* overlapping loads */
add x2, x2, x5
ld1 {v1.16b}, [x2]
/* create Cn from En-1 */
tbl v2.16b, {v0.16b}, v3.16b
/* padding Pn with En-1 at the end */
tbx v0.16b, {v1.16b}, v4.16b
eor v0.16b, v0.16b, v8.16b
SM4_CRYPT_BLK(v0)
eor v0.16b, v0.16b, v8.16b
/* overlapping stores */
add x5, x1, x5
st1 {v2.16b}, [x5]
st1 {v0.16b}, [x1]
b .Lxts_dec_ret
.Lxts_dec_end:
/* store new tweak */
st1 {v8.16b}, [x3]
.Lxts_dec_ret:
ret
SYM_FUNC_END(sm4_ce_xts_dec)
.align 3
SYM_FUNC_START(sm4_ce_mac_update)
/* input:
* x0: round key array, CTX
* x1: digest
* x2: src
* w3: nblocks
* w4: enc_before
* w5: enc_after
*/
SM4_PREPARE(x0)
ld1 {RMAC.16b}, [x1]
cbz w4, .Lmac_update
SM4_CRYPT_BLK(RMAC)
.Lmac_update:
cbz w3, .Lmac_ret
sub w6, w3, #1
cmp w5, wzr
csel w3, w3, w6, ne
cbz w3, .Lmac_end
.Lmac_loop_4x:
cmp w3, #4
blt .Lmac_loop_1x
sub w3, w3, #4
ld1 {v0.16b-v3.16b}, [x2], #64
eor RMAC.16b, RMAC.16b, v0.16b
SM4_CRYPT_BLK(RMAC)
eor RMAC.16b, RMAC.16b, v1.16b
SM4_CRYPT_BLK(RMAC)
eor RMAC.16b, RMAC.16b, v2.16b
SM4_CRYPT_BLK(RMAC)
eor RMAC.16b, RMAC.16b, v3.16b
SM4_CRYPT_BLK(RMAC)
cbz w3, .Lmac_end
b .Lmac_loop_4x
.Lmac_loop_1x:
sub w3, w3, #1
ld1 {v0.16b}, [x2], #16
eor RMAC.16b, RMAC.16b, v0.16b
SM4_CRYPT_BLK(RMAC)
cbnz w3, .Lmac_loop_1x
.Lmac_end:
cbnz w5, .Lmac_ret
ld1 {v0.16b}, [x2], #16
eor RMAC.16b, RMAC.16b, v0.16b
.Lmac_ret:
st1 {RMAC.16b}, [x1]
ret
SYM_FUNC_END(sm4_ce_mac_update)
.section ".rodata", "a"
.align 4
.Lbswap128_mask:
.byte 0x0c, 0x0d, 0x0e, 0x0f, 0x08, 0x09, 0x0a, 0x0b
.byte 0x04, 0x05, 0x06, 0x07, 0x00, 0x01, 0x02, 0x03
.Lcts_permute_table:
.byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
.byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
.byte 0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7
.byte 0x8, 0x9, 0xa, 0xb, 0xc, 0xd, 0xe, 0xf
.byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
.byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
linux-7.2.3/arch/arm64/crypto/sm4-ce-gcm-core.S
+742
View File
@@ -0,0 +1,742 @@
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
* SM4-GCM AEAD Algorithm using ARMv8 Crypto Extensions
* as specified in rfc8998
* https://datatracker.ietf.org/doc/html/rfc8998
*
* Copyright (C) 2016 Jussi Kivilinna <jussi.kivilinna@iki.fi>
* Copyright (C) 2022 Tianjia Zhang <tianjia.zhang@linux.alibaba.com>
*/
#include <linux/linkage.h>
#include <linux/cfi_types.h>
#include <asm/assembler.h>
#include "sm4-ce-asm.h"
.arch armv8-a+crypto
.irp b, 0, 1, 2, 3, 24, 25, 26, 27, 28, 29, 30, 31
.set .Lv\b\().4s, \b
.endr
.macro sm4e, vd, vn
.inst 0xcec08400 | (.L\vn << 5) | .L\vd
.endm
/* Register macros */
/* Used for both encryption and decryption */
#define RHASH v21
#define RRCONST v22
#define RZERO v23
/* Helper macros. */
/*
* input: m0, m1
* output: r0:r1 (low 128-bits in r0, high in r1)
*/
#define PMUL_128x128(r0, r1, m0, m1, T0, T1) \
ext T0.16b, m1.16b, m1.16b, #8; \
pmull r0.1q, m0.1d, m1.1d; \
pmull T1.1q, m0.1d, T0.1d; \
pmull2 T0.1q, m0.2d, T0.2d; \
pmull2 r1.1q, m0.2d, m1.2d; \
eor T0.16b, T0.16b, T1.16b; \
ext T1.16b, RZERO.16b, T0.16b, #8; \
ext T0.16b, T0.16b, RZERO.16b, #8; \
eor r0.16b, r0.16b, T1.16b; \
eor r1.16b, r1.16b, T0.16b;
#define PMUL_128x128_4x(r0, r1, m0, m1, T0, T1, \
r2, r3, m2, m3, T2, T3, \
r4, r5, m4, m5, T4, T5, \
r6, r7, m6, m7, T6, T7) \
ext T0.16b, m1.16b, m1.16b, #8; \
ext T2.16b, m3.16b, m3.16b, #8; \
ext T4.16b, m5.16b, m5.16b, #8; \
ext T6.16b, m7.16b, m7.16b, #8; \
pmull r0.1q, m0.1d, m1.1d; \
pmull r2.1q, m2.1d, m3.1d; \
pmull r4.1q, m4.1d, m5.1d; \
pmull r6.1q, m6.1d, m7.1d; \
pmull T1.1q, m0.1d, T0.1d; \
pmull T3.1q, m2.1d, T2.1d; \
pmull T5.1q, m4.1d, T4.1d; \
pmull T7.1q, m6.1d, T6.1d; \
pmull2 T0.1q, m0.2d, T0.2d; \
pmull2 T2.1q, m2.2d, T2.2d; \
pmull2 T4.1q, m4.2d, T4.2d; \
pmull2 T6.1q, m6.2d, T6.2d; \
pmull2 r1.1q, m0.2d, m1.2d; \
pmull2 r3.1q, m2.2d, m3.2d; \
pmull2 r5.1q, m4.2d, m5.2d; \
pmull2 r7.1q, m6.2d, m7.2d; \
eor T0.16b, T0.16b, T1.16b; \
eor T2.16b, T2.16b, T3.16b; \
eor T4.16b, T4.16b, T5.16b; \
eor T6.16b, T6.16b, T7.16b; \
ext T1.16b, RZERO.16b, T0.16b, #8; \
ext T3.16b, RZERO.16b, T2.16b, #8; \
ext T5.16b, RZERO.16b, T4.16b, #8; \
ext T7.16b, RZERO.16b, T6.16b, #8; \
ext T0.16b, T0.16b, RZERO.16b, #8; \
ext T2.16b, T2.16b, RZERO.16b, #8; \
ext T4.16b, T4.16b, RZERO.16b, #8; \
ext T6.16b, T6.16b, RZERO.16b, #8; \
eor r0.16b, r0.16b, T1.16b; \
eor r2.16b, r2.16b, T3.16b; \
eor r4.16b, r4.16b, T5.16b; \
eor r6.16b, r6.16b, T7.16b; \
eor r1.16b, r1.16b, T0.16b; \
eor r3.16b, r3.16b, T2.16b; \
eor r5.16b, r5.16b, T4.16b; \
eor r7.16b, r7.16b, T6.16b;
/*
* input: r0:r1 (low 128-bits in r0, high in r1)
* output: a
*/
#define REDUCTION(a, r0, r1, rconst, T0, T1) \
pmull2 T0.1q, r1.2d, rconst.2d; \
ext T1.16b, T0.16b, RZERO.16b, #8; \
ext T0.16b, RZERO.16b, T0.16b, #8; \
eor r1.16b, r1.16b, T1.16b; \
eor r0.16b, r0.16b, T0.16b; \
pmull T0.1q, r1.1d, rconst.1d; \
eor a.16b, r0.16b, T0.16b;
#define SM4_CRYPT_PMUL_128x128_BLK(b0, r0, r1, m0, m1, T0, T1) \
rev32 b0.16b, b0.16b; \
ext T0.16b, m1.16b, m1.16b, #8; \
sm4e b0.4s, v24.4s; \
pmull r0.1q, m0.1d, m1.1d; \
sm4e b0.4s, v25.4s; \
pmull T1.1q, m0.1d, T0.1d; \
sm4e b0.4s, v26.4s; \
pmull2 T0.1q, m0.2d, T0.2d; \
sm4e b0.4s, v27.4s; \
pmull2 r1.1q, m0.2d, m1.2d; \
sm4e b0.4s, v28.4s; \
eor T0.16b, T0.16b, T1.16b; \
sm4e b0.4s, v29.4s; \
ext T1.16b, RZERO.16b, T0.16b, #8; \
sm4e b0.4s, v30.4s; \
ext T0.16b, T0.16b, RZERO.16b, #8; \
sm4e b0.4s, v31.4s; \
eor r0.16b, r0.16b, T1.16b; \
rev64 b0.4s, b0.4s; \
eor r1.16b, r1.16b, T0.16b; \
ext b0.16b, b0.16b, b0.16b, #8; \
rev32 b0.16b, b0.16b;
#define SM4_CRYPT_PMUL_128x128_BLK3(b0, b1, b2, \
r0, r1, m0, m1, T0, T1, \
r2, r3, m2, m3, T2, T3, \
r4, r5, m4, m5, T4, T5) \
rev32 b0.16b, b0.16b; \
rev32 b1.16b, b1.16b; \
rev32 b2.16b, b2.16b; \
ext T0.16b, m1.16b, m1.16b, #8; \
ext T2.16b, m3.16b, m3.16b, #8; \
ext T4.16b, m5.16b, m5.16b, #8; \
sm4e b0.4s, v24.4s; \
sm4e b1.4s, v24.4s; \
sm4e b2.4s, v24.4s; \
pmull r0.1q, m0.1d, m1.1d; \
pmull r2.1q, m2.1d, m3.1d; \
pmull r4.1q, m4.1d, m5.1d; \
sm4e b0.4s, v25.4s; \
sm4e b1.4s, v25.4s; \
sm4e b2.4s, v25.4s; \
pmull T1.1q, m0.1d, T0.1d; \
pmull T3.1q, m2.1d, T2.1d; \
pmull T5.1q, m4.1d, T4.1d; \
sm4e b0.4s, v26.4s; \
sm4e b1.4s, v26.4s; \
sm4e b2.4s, v26.4s; \
pmull2 T0.1q, m0.2d, T0.2d; \
pmull2 T2.1q, m2.2d, T2.2d; \
pmull2 T4.1q, m4.2d, T4.2d; \
sm4e b0.4s, v27.4s; \
sm4e b1.4s, v27.4s; \
sm4e b2.4s, v27.4s; \
pmull2 r1.1q, m0.2d, m1.2d; \
pmull2 r3.1q, m2.2d, m3.2d; \
pmull2 r5.1q, m4.2d, m5.2d; \
sm4e b0.4s, v28.4s; \
sm4e b1.4s, v28.4s; \
sm4e b2.4s, v28.4s; \
eor T0.16b, T0.16b, T1.16b; \
eor T2.16b, T2.16b, T3.16b; \
eor T4.16b, T4.16b, T5.16b; \
sm4e b0.4s, v29.4s; \
sm4e b1.4s, v29.4s; \
sm4e b2.4s, v29.4s; \
ext T1.16b, RZERO.16b, T0.16b, #8; \
ext T3.16b, RZERO.16b, T2.16b, #8; \
ext T5.16b, RZERO.16b, T4.16b, #8; \
sm4e b0.4s, v30.4s; \
sm4e b1.4s, v30.4s; \
sm4e b2.4s, v30.4s; \
ext T0.16b, T0.16b, RZERO.16b, #8; \
ext T2.16b, T2.16b, RZERO.16b, #8; \
ext T4.16b, T4.16b, RZERO.16b, #8; \
sm4e b0.4s, v31.4s; \
sm4e b1.4s, v31.4s; \
sm4e b2.4s, v31.4s; \
eor r0.16b, r0.16b, T1.16b; \
eor r2.16b, r2.16b, T3.16b; \
eor r4.16b, r4.16b, T5.16b; \
rev64 b0.4s, b0.4s; \
rev64 b1.4s, b1.4s; \
rev64 b2.4s, b2.4s; \
eor r1.16b, r1.16b, T0.16b; \
eor r3.16b, r3.16b, T2.16b; \
eor r5.16b, r5.16b, T4.16b; \
ext b0.16b, b0.16b, b0.16b, #8; \
ext b1.16b, b1.16b, b1.16b, #8; \
ext b2.16b, b2.16b, b2.16b, #8; \
eor r0.16b, r0.16b, r2.16b; \
eor r1.16b, r1.16b, r3.16b; \
rev32 b0.16b, b0.16b; \
rev32 b1.16b, b1.16b; \
rev32 b2.16b, b2.16b; \
eor r0.16b, r0.16b, r4.16b; \
eor r1.16b, r1.16b, r5.16b;
#define inc32_le128(vctr) \
mov vctr.d[1], x9; \
add w6, w9, #1; \
mov vctr.d[0], x8; \
bfi x9, x6, #0, #32; \
rev64 vctr.16b, vctr.16b;
#define GTAG_HASH_LENGTHS(vctr0, vlen) \
ld1 {vlen.16b}, [x7]; \
/* construct CTR0 */ \
/* the lower 32-bits of initial IV is always be32(1) */ \
mov x6, #0x1; \
bfi x9, x6, #0, #32; \
mov vctr0.d[0], x8; \
mov vctr0.d[1], x9; \
rbit vlen.16b, vlen.16b; \
rev64 vctr0.16b, vctr0.16b; \
/* authtag = GCTR(CTR0, GHASH) */ \
eor RHASH.16b, RHASH.16b, vlen.16b; \
SM4_CRYPT_PMUL_128x128_BLK(vctr0, RR0, RR1, RHASH, RH1, \
RTMP0, RTMP1); \
REDUCTION(RHASH, RR0, RR1, RRCONST, RTMP2, RTMP3); \
rbit RHASH.16b, RHASH.16b; \
eor RHASH.16b, RHASH.16b, vctr0.16b;
/* Register macros for encrypt and ghash */
/* can be the same as input v0-v3 */
#define RR1 v0
#define RR3 v1
#define RR5 v2
#define RR7 v3
#define RR0 v4
#define RR2 v5
#define RR4 v6
#define RR6 v7
#define RTMP0 v8
#define RTMP1 v9
#define RTMP2 v10
#define RTMP3 v11
#define RTMP4 v12
#define RTMP5 v13
#define RTMP6 v14
#define RTMP7 v15
#define RH1 v16
#define RH2 v17
#define RH3 v18
#define RH4 v19
.align 3
SYM_FUNC_START(sm4_ce_pmull_ghash_setup)
/* input:
* x0: round key array, CTX
* x1: ghash table
*/
SM4_PREPARE(x0)
adr_l x2, .Lghash_rconst
ld1r {RRCONST.2d}, [x2]
eor RZERO.16b, RZERO.16b, RZERO.16b
/* H = E(K, 0^128) */
rev32 v0.16b, RZERO.16b
SM4_CRYPT_BLK_BE(v0)
/* H ^ 1 */
rbit RH1.16b, v0.16b
/* H ^ 2 */
PMUL_128x128(RR0, RR1, RH1, RH1, RTMP0, RTMP1)
REDUCTION(RH2, RR0, RR1, RRCONST, RTMP2, RTMP3)
/* H ^ 3 */
PMUL_128x128(RR0, RR1, RH2, RH1, RTMP0, RTMP1)
REDUCTION(RH3, RR0, RR1, RRCONST, RTMP2, RTMP3)
/* H ^ 4 */
PMUL_128x128(RR0, RR1, RH2, RH2, RTMP0, RTMP1)
REDUCTION(RH4, RR0, RR1, RRCONST, RTMP2, RTMP3)
st1 {RH1.16b-RH4.16b}, [x1]
ret
SYM_FUNC_END(sm4_ce_pmull_ghash_setup)
.align 3
SYM_FUNC_START(pmull_ghash_update)
/* input:
* x0: ghash table
* x1: ghash result
* x2: src
* w3: nblocks
*/
ld1 {RH1.16b-RH4.16b}, [x0]
ld1 {RHASH.16b}, [x1]
rbit RHASH.16b, RHASH.16b
adr_l x4, .Lghash_rconst
ld1r {RRCONST.2d}, [x4]
eor RZERO.16b, RZERO.16b, RZERO.16b
.Lghash_loop_4x:
cmp w3, #4
blt .Lghash_loop_1x
sub w3, w3, #4
ld1 {v0.16b-v3.16b}, [x2], #64
rbit v0.16b, v0.16b
rbit v1.16b, v1.16b
rbit v2.16b, v2.16b
rbit v3.16b, v3.16b
/*
* (in0 ^ HASH) * H^4 => rr0:rr1
* (in1) * H^3 => rr2:rr3
* (in2) * H^2 => rr4:rr5
* (in3) * H^1 => rr6:rr7
*/
eor RHASH.16b, RHASH.16b, v0.16b
PMUL_128x128_4x(RR0, RR1, RHASH, RH4, RTMP0, RTMP1,
RR2, RR3, v1, RH3, RTMP2, RTMP3,
RR4, RR5, v2, RH2, RTMP4, RTMP5,
RR6, RR7, v3, RH1, RTMP6, RTMP7)
eor RR0.16b, RR0.16b, RR2.16b
eor RR1.16b, RR1.16b, RR3.16b
eor RR0.16b, RR0.16b, RR4.16b
eor RR1.16b, RR1.16b, RR5.16b
eor RR0.16b, RR0.16b, RR6.16b
eor RR1.16b, RR1.16b, RR7.16b
REDUCTION(RHASH, RR0, RR1, RRCONST, RTMP0, RTMP1)
cbz w3, .Lghash_end
b .Lghash_loop_4x
.Lghash_loop_1x:
sub w3, w3, #1
ld1 {v0.16b}, [x2], #16
rbit v0.16b, v0.16b
eor RHASH.16b, RHASH.16b, v0.16b
PMUL_128x128(RR0, RR1, RHASH, RH1, RTMP0, RTMP1)
REDUCTION(RHASH, RR0, RR1, RRCONST, RTMP2, RTMP3)
cbnz w3, .Lghash_loop_1x
.Lghash_end:
rbit RHASH.16b, RHASH.16b
st1 {RHASH.2d}, [x1]
ret
SYM_FUNC_END(pmull_ghash_update)
.align 3
SYM_TYPED_FUNC_START(sm4_ce_pmull_gcm_enc)
/* input:
* x0: round key array, CTX
* x1: dst
* x2: src
* x3: ctr (big endian, 128 bit)
* w4: nbytes
* x5: ghash result
* x6: ghash table
* x7: lengths (only for last block)
*/
SM4_PREPARE(x0)
ldp x8, x9, [x3]
rev x8, x8
rev x9, x9
ld1 {RH1.16b-RH4.16b}, [x6]
ld1 {RHASH.16b}, [x5]
rbit RHASH.16b, RHASH.16b
adr_l x6, .Lghash_rconst
ld1r {RRCONST.2d}, [x6]
eor RZERO.16b, RZERO.16b, RZERO.16b
cbz w4, .Lgcm_enc_hash_len
.Lgcm_enc_loop_4x:
cmp w4, #(4 * 16)
blt .Lgcm_enc_loop_1x
sub w4, w4, #(4 * 16)
/* construct CTRs */
inc32_le128(v0) /* +0 */
inc32_le128(v1) /* +1 */
inc32_le128(v2) /* +2 */
inc32_le128(v3) /* +3 */
ld1 {RTMP0.16b-RTMP3.16b}, [x2], #64
SM4_CRYPT_BLK4(v0, v1, v2, v3)
eor v0.16b, v0.16b, RTMP0.16b
eor v1.16b, v1.16b, RTMP1.16b
eor v2.16b, v2.16b, RTMP2.16b
eor v3.16b, v3.16b, RTMP3.16b
st1 {v0.16b-v3.16b}, [x1], #64
/* ghash update */
rbit v0.16b, v0.16b
rbit v1.16b, v1.16b
rbit v2.16b, v2.16b
rbit v3.16b, v3.16b
/*
* (in0 ^ HASH) * H^4 => rr0:rr1
* (in1) * H^3 => rr2:rr3
* (in2) * H^2 => rr4:rr5
* (in3) * H^1 => rr6:rr7
*/
eor RHASH.16b, RHASH.16b, v0.16b
PMUL_128x128_4x(RR0, RR1, RHASH, RH4, RTMP0, RTMP1,
RR2, RR3, v1, RH3, RTMP2, RTMP3,
RR4, RR5, v2, RH2, RTMP4, RTMP5,
RR6, RR7, v3, RH1, RTMP6, RTMP7)
eor RR0.16b, RR0.16b, RR2.16b
eor RR1.16b, RR1.16b, RR3.16b
eor RR0.16b, RR0.16b, RR4.16b
eor RR1.16b, RR1.16b, RR5.16b
eor RR0.16b, RR0.16b, RR6.16b
eor RR1.16b, RR1.16b, RR7.16b
REDUCTION(RHASH, RR0, RR1, RRCONST, RTMP0, RTMP1)
cbz w4, .Lgcm_enc_hash_len
b .Lgcm_enc_loop_4x
.Lgcm_enc_loop_1x:
cmp w4, #16
blt .Lgcm_enc_tail
sub w4, w4, #16
/* construct CTRs */
inc32_le128(v0)
ld1 {RTMP0.16b}, [x2], #16
SM4_CRYPT_BLK(v0)
eor v0.16b, v0.16b, RTMP0.16b
st1 {v0.16b}, [x1], #16
/* ghash update */
rbit v0.16b, v0.16b
eor RHASH.16b, RHASH.16b, v0.16b
PMUL_128x128(RR0, RR1, RHASH, RH1, RTMP0, RTMP1)
REDUCTION(RHASH, RR0, RR1, RRCONST, RTMP2, RTMP3)
cbz w4, .Lgcm_enc_hash_len
b .Lgcm_enc_loop_1x
.Lgcm_enc_tail:
/* construct CTRs */
inc32_le128(v0)
SM4_CRYPT_BLK(v0)
/* load permute table */
adr_l x0, .Lcts_permute_table
add x0, x0, #32
sub x0, x0, w4, uxtw
ld1 {v3.16b}, [x0]
.Lgcm_enc_tail_loop:
/* do encrypt */
ldrb w0, [x2], #1 /* get 1 byte from input */
umov w6, v0.b[0] /* get top crypted byte */
eor w6, w6, w0 /* w6 = CTR ^ input */
strb w6, [x1], #1 /* store out byte */
/* shift right out one byte */
ext v0.16b, v0.16b, v0.16b, #1
/* the last ciphertext is placed in high bytes */
ins v0.b[15], w6
subs w4, w4, #1
bne .Lgcm_enc_tail_loop
/* padding last block with zeros */
tbl v0.16b, {v0.16b}, v3.16b
/* ghash update */
rbit v0.16b, v0.16b
eor RHASH.16b, RHASH.16b, v0.16b
PMUL_128x128(RR0, RR1, RHASH, RH1, RTMP0, RTMP1)
REDUCTION(RHASH, RR0, RR1, RRCONST, RTMP2, RTMP3)
.Lgcm_enc_hash_len:
cbz x7, .Lgcm_enc_end
GTAG_HASH_LENGTHS(v1, v3)
b .Lgcm_enc_ret
.Lgcm_enc_end:
/* store new CTR */
rev x8, x8
rev x9, x9
stp x8, x9, [x3]
rbit RHASH.16b, RHASH.16b
.Lgcm_enc_ret:
/* store new MAC */
st1 {RHASH.2d}, [x5]
ret
SYM_FUNC_END(sm4_ce_pmull_gcm_enc)
#undef RR1
#undef RR3
#undef RR5
#undef RR7
#undef RR0
#undef RR2
#undef RR4
#undef RR6
#undef RTMP0
#undef RTMP1
#undef RTMP2
#undef RTMP3
#undef RTMP4
#undef RTMP5
#undef RTMP6
#undef RTMP7
#undef RH1
#undef RH2
#undef RH3
#undef RH4
/* Register macros for decrypt */
/* v0-v2 for building CTRs, v3-v5 for saving inputs */
#define RR1 v6
#define RR3 v7
#define RR5 v8
#define RR0 v9
#define RR2 v10
#define RR4 v11
#define RTMP0 v12
#define RTMP1 v13
#define RTMP2 v14
#define RTMP3 v15
#define RTMP4 v16
#define RTMP5 v17
#define RH1 v18
#define RH2 v19
#define RH3 v20
.align 3
SYM_TYPED_FUNC_START(sm4_ce_pmull_gcm_dec)
/* input:
* x0: round key array, CTX
* x1: dst
* x2: src
* x3: ctr (big endian, 128 bit)
* w4: nbytes
* x5: ghash result
* x6: ghash table
* x7: lengths (only for last block)
*/
SM4_PREPARE(x0)
ldp x8, x9, [x3]
rev x8, x8
rev x9, x9
ld1 {RH1.16b-RH3.16b}, [x6]
ld1 {RHASH.16b}, [x5]
rbit RHASH.16b, RHASH.16b
adr_l x6, .Lghash_rconst
ld1r {RRCONST.2d}, [x6]
eor RZERO.16b, RZERO.16b, RZERO.16b
cbz w4, .Lgcm_dec_hash_len
.Lgcm_dec_loop_3x:
cmp w4, #(3 * 16)
blt .Lgcm_dec_loop_1x
sub w4, w4, #(3 * 16)
ld1 {v3.16b-v5.16b}, [x2], #(3 * 16)
/* construct CTRs */
inc32_le128(v0) /* +0 */
rbit v6.16b, v3.16b
inc32_le128(v1) /* +1 */
rbit v7.16b, v4.16b
inc32_le128(v2) /* +2 */
rbit v8.16b, v5.16b
eor RHASH.16b, RHASH.16b, v6.16b
/* decrypt & ghash update */
SM4_CRYPT_PMUL_128x128_BLK3(v0, v1, v2,
RR0, RR1, RHASH, RH3, RTMP0, RTMP1,
RR2, RR3, v7, RH2, RTMP2, RTMP3,
RR4, RR5, v8, RH1, RTMP4, RTMP5)
eor v0.16b, v0.16b, v3.16b
eor v1.16b, v1.16b, v4.16b
eor v2.16b, v2.16b, v5.16b
REDUCTION(RHASH, RR0, RR1, RRCONST, RTMP0, RTMP1)
st1 {v0.16b-v2.16b}, [x1], #(3 * 16)
cbz w4, .Lgcm_dec_hash_len
b .Lgcm_dec_loop_3x
.Lgcm_dec_loop_1x:
cmp w4, #16
blt .Lgcm_dec_tail
sub w4, w4, #16
ld1 {v3.16b}, [x2], #16
/* construct CTRs */
inc32_le128(v0)
rbit v6.16b, v3.16b
eor RHASH.16b, RHASH.16b, v6.16b
SM4_CRYPT_PMUL_128x128_BLK(v0, RR0, RR1, RHASH, RH1, RTMP0, RTMP1)
eor v0.16b, v0.16b, v3.16b
REDUCTION(RHASH, RR0, RR1, RRCONST, RTMP2, RTMP3)
st1 {v0.16b}, [x1], #16
cbz w4, .Lgcm_dec_hash_len
b .Lgcm_dec_loop_1x
.Lgcm_dec_tail:
/* construct CTRs */
inc32_le128(v0)
SM4_CRYPT_BLK(v0)
/* load permute table */
adr_l x0, .Lcts_permute_table
add x0, x0, #32
sub x0, x0, w4, uxtw
ld1 {v3.16b}, [x0]
.Lgcm_dec_tail_loop:
/* do decrypt */
ldrb w0, [x2], #1 /* get 1 byte from input */
umov w6, v0.b[0] /* get top crypted byte */
eor w6, w6, w0 /* w6 = CTR ^ input */
strb w6, [x1], #1 /* store out byte */
/* shift right out one byte */
ext v0.16b, v0.16b, v0.16b, #1
/* the last ciphertext is placed in high bytes */
ins v0.b[15], w0
subs w4, w4, #1
bne .Lgcm_dec_tail_loop
/* padding last block with zeros */
tbl v0.16b, {v0.16b}, v3.16b
/* ghash update */
rbit v0.16b, v0.16b
eor RHASH.16b, RHASH.16b, v0.16b
PMUL_128x128(RR0, RR1, RHASH, RH1, RTMP0, RTMP1)
REDUCTION(RHASH, RR0, RR1, RRCONST, RTMP2, RTMP3)
.Lgcm_dec_hash_len:
cbz x7, .Lgcm_dec_end
GTAG_HASH_LENGTHS(v1, v3)
b .Lgcm_dec_ret
.Lgcm_dec_end:
/* store new CTR */
rev x8, x8
rev x9, x9
stp x8, x9, [x3]
rbit RHASH.16b, RHASH.16b
.Lgcm_dec_ret:
/* store new MAC */
st1 {RHASH.2d}, [x5]
ret
SYM_FUNC_END(sm4_ce_pmull_gcm_dec)
.section ".rodata", "a"
.align 4
.Lcts_permute_table:
.byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
.byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
.byte 0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7
.byte 0x8, 0x9, 0xa, 0xb, 0xc, 0xd, 0xe, 0xf
.byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
.byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
.Lghash_rconst:
.quad 0x87
linux-7.2.3/arch/arm64/crypto/sm4-ce-gcm-glue.c
+262
View File
@@ -0,0 +1,262 @@
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
* SM4-GCM AEAD Algorithm using ARMv8 Crypto Extensions
* as specified in rfc8998
* https://datatracker.ietf.org/doc/html/rfc8998
*
* Copyright (C) 2022 Tianjia Zhang <tianjia.zhang@linux.alibaba.com>
*/
#include <linux/module.h>
#include <linux/crypto.h>
#include <linux/kernel.h>
#include <linux/cpufeature.h>
#include <asm/simd.h>
#include <crypto/b128ops.h>
#include <crypto/scatterwalk.h>
#include <crypto/internal/aead.h>
#include <crypto/internal/skcipher.h>
#include <crypto/sm4.h>
#include "sm4-ce.h"
asmlinkage void sm4_ce_pmull_ghash_setup(const u32 *rkey_enc, u8 *ghash_table);
asmlinkage void pmull_ghash_update(const u8 *ghash_table, u8 *ghash,
const u8 *src, unsigned int nblocks);
asmlinkage void sm4_ce_pmull_gcm_enc(const u32 *rkey_enc, u8 *dst,
const u8 *src, u8 *iv,
unsigned int nbytes, u8 *ghash,
const u8 *ghash_table, const u8 *lengths);
asmlinkage void sm4_ce_pmull_gcm_dec(const u32 *rkey_enc, u8 *dst,
const u8 *src, u8 *iv,
unsigned int nbytes, u8 *ghash,
const u8 *ghash_table, const u8 *lengths);
#define GHASH_BLOCK_SIZE 16
#define GCM_IV_SIZE 12
struct sm4_gcm_ctx {
struct sm4_ctx key;
u8 ghash_table[16 * 4];
};
static int gcm_setkey(struct crypto_aead *tfm, const u8 *key,
unsigned int key_len)
{
struct sm4_gcm_ctx *ctx = crypto_aead_ctx(tfm);
if (key_len != SM4_KEY_SIZE)
return -EINVAL;
scoped_ksimd() {
sm4_ce_expand_key(key, ctx->key.rkey_enc, ctx->key.rkey_dec,
crypto_sm4_fk, crypto_sm4_ck);
sm4_ce_pmull_ghash_setup(ctx->key.rkey_enc, ctx->ghash_table);
}
return 0;
}
static int gcm_setauthsize(struct crypto_aead *tfm, unsigned int authsize)
{
switch (authsize) {
case 4:
case 8:
case 12 ... 16:
return 0;
default:
return -EINVAL;
}
}
static void gcm_calculate_auth_mac(struct aead_request *req, u8 ghash[])
{
struct crypto_aead *aead = crypto_aead_reqtfm(req);
struct sm4_gcm_ctx *ctx = crypto_aead_ctx(aead);
u8 __aligned(8) buffer[GHASH_BLOCK_SIZE];
u32 assoclen = req->assoclen;
struct scatter_walk walk;
unsigned int buflen = 0;
scatterwalk_start(&walk, req->src);
do {
unsigned int n, orig_n;
const u8 *p;
orig_n = scatterwalk_next(&walk, assoclen);
p = walk.addr;
n = orig_n;
if (n + buflen < GHASH_BLOCK_SIZE) {
memcpy(&buffer[buflen], p, n);
buflen += n;
} else {
unsigned int nblocks;
if (buflen) {
unsigned int l = GHASH_BLOCK_SIZE - buflen;
memcpy(&buffer[buflen], p, l);
p += l;
n -= l;
pmull_ghash_update(ctx->ghash_table, ghash,
buffer, 1);
}
nblocks = n / GHASH_BLOCK_SIZE;
if (nblocks) {
pmull_ghash_update(ctx->ghash_table, ghash,
p, nblocks);
p += nblocks * GHASH_BLOCK_SIZE;
}
buflen = n % GHASH_BLOCK_SIZE;
if (buflen)
memcpy(&buffer[0], p, buflen);
}
scatterwalk_done_src(&walk, orig_n);
assoclen -= orig_n;
} while (assoclen);
/* padding with '0' */
if (buflen) {
memset(&buffer[buflen], 0, GHASH_BLOCK_SIZE - buflen);
pmull_ghash_update(ctx->ghash_table, ghash, buffer, 1);
}
}
static int gcm_crypt(struct aead_request *req, struct skcipher_walk *walk,
u8 ghash[], int err,
void (*sm4_ce_pmull_gcm_crypt)(const u32 *rkey_enc,
u8 *dst, const u8 *src, u8 *iv,
unsigned int nbytes, u8 *ghash,
const u8 *ghash_table, const u8 *lengths))
{
struct crypto_aead *aead = crypto_aead_reqtfm(req);
struct sm4_gcm_ctx *ctx = crypto_aead_ctx(aead);
u8 __aligned(8) iv[SM4_BLOCK_SIZE];
be128 __aligned(8) lengths;
memset(ghash, 0, SM4_BLOCK_SIZE);
lengths.a = cpu_to_be64(req->assoclen * 8);
lengths.b = cpu_to_be64(walk->total * 8);
memcpy(iv, req->iv, GCM_IV_SIZE);
put_unaligned_be32(2, iv + GCM_IV_SIZE);
scoped_ksimd() {
if (req->assoclen)
gcm_calculate_auth_mac(req, ghash);
do {
unsigned int tail = walk->nbytes % SM4_BLOCK_SIZE;
const u8 *src = walk->src.virt.addr;
u8 *dst = walk->dst.virt.addr;
const u8 *l = NULL;
if (walk->nbytes == walk->total) {
l = (const u8 *)&lengths;
tail = 0;
}
sm4_ce_pmull_gcm_crypt(ctx->key.rkey_enc, dst, src, iv,
walk->nbytes - tail, ghash,
ctx->ghash_table, l);
err = skcipher_walk_done(walk, tail);
} while (walk->nbytes);
}
return err;
}
static int gcm_encrypt(struct aead_request *req)
{
struct crypto_aead *aead = crypto_aead_reqtfm(req);
u8 __aligned(8) ghash[SM4_BLOCK_SIZE];
struct skcipher_walk walk;
int err;
err = skcipher_walk_aead_encrypt(&walk, req, false);
err = gcm_crypt(req, &walk, ghash, err, sm4_ce_pmull_gcm_enc);
if (err)
return err;
/* copy authtag to end of dst */
scatterwalk_map_and_copy(ghash, req->dst, req->assoclen + req->cryptlen,
crypto_aead_authsize(aead), 1);
return 0;
}
static int gcm_decrypt(struct aead_request *req)
{
struct crypto_aead *aead = crypto_aead_reqtfm(req);
unsigned int authsize = crypto_aead_authsize(aead);
u8 __aligned(8) ghash[SM4_BLOCK_SIZE];
u8 authtag[SM4_BLOCK_SIZE];
struct skcipher_walk walk;
int err;
err = skcipher_walk_aead_decrypt(&walk, req, false);
err = gcm_crypt(req, &walk, ghash, err, sm4_ce_pmull_gcm_dec);
if (err)
return err;
/* compare calculated auth tag with the stored one */
scatterwalk_map_and_copy(authtag, req->src,
req->assoclen + req->cryptlen - authsize,
authsize, 0);
if (crypto_memneq(authtag, ghash, authsize))
return -EBADMSG;
return 0;
}
static struct aead_alg sm4_gcm_alg = {
.base = {
.cra_name = "gcm(sm4)",
.cra_driver_name = "gcm-sm4-ce",
.cra_priority = 400,
.cra_blocksize = 1,
.cra_ctxsize = sizeof(struct sm4_gcm_ctx),
.cra_module = THIS_MODULE,
},
.ivsize = GCM_IV_SIZE,
.chunksize = SM4_BLOCK_SIZE,
.maxauthsize = SM4_BLOCK_SIZE,
.setkey = gcm_setkey,
.setauthsize = gcm_setauthsize,
.encrypt = gcm_encrypt,
.decrypt = gcm_decrypt,
};
static int __init sm4_ce_gcm_init(void)
{
if (!cpu_have_named_feature(PMULL))
return -ENODEV;
return crypto_register_aead(&sm4_gcm_alg);
}
static void __exit sm4_ce_gcm_exit(void)
{
crypto_unregister_aead(&sm4_gcm_alg);
}
static const struct cpu_feature __maybe_unused sm4_ce_gcm_cpu_feature[] = {
{ cpu_feature(PMULL) },
{}
};
MODULE_DEVICE_TABLE(cpu, sm4_ce_gcm_cpu_feature);
module_cpu_feature_match(SM4, sm4_ce_gcm_init);
module_exit(sm4_ce_gcm_exit);
MODULE_DESCRIPTION("Synchronous SM4 in GCM mode using ARMv8 Crypto Extensions");
MODULE_ALIAS_CRYPTO("gcm(sm4)");
MODULE_AUTHOR("Tianjia Zhang <tianjia.zhang@linux.alibaba.com>");
MODULE_LICENSE("GPL v2");
linux-7.2.3/arch/arm64/crypto/sm4-ce-glue.c
+719
View File
@@ -0,0 +1,719 @@
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
* SM4 Cipher Algorithm, using ARMv8 Crypto Extensions
* as specified in
* https://tools.ietf.org/id/draft-ribose-cfrg-sm4-10.html
*
* Copyright (C) 2022, Alibaba Group.
* Copyright (C) 2022 Tianjia Zhang <tianjia.zhang@linux.alibaba.com>
*/
#include <asm/simd.h>
#include <crypto/b128ops.h>
#include <crypto/internal/hash.h>
#include <crypto/internal/skcipher.h>
#include <crypto/scatterwalk.h>
#include <crypto/sm4.h>
#include <crypto/utils.h>
#include <crypto/xts.h>
#include <linux/cpufeature.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/string.h>
#define BYTES2BLKS(nbytes) ((nbytes) >> 4)
asmlinkage void sm4_ce_expand_key(const u8 *key, u32 *rkey_enc, u32 *rkey_dec,
const u32 *fk, const u32 *ck);
asmlinkage void sm4_ce_crypt_block(const u32 *rkey, u8 *dst, const u8 *src);
asmlinkage void sm4_ce_crypt(const u32 *rkey, u8 *dst, const u8 *src,
unsigned int nblks);
asmlinkage void sm4_ce_cbc_enc(const u32 *rkey, u8 *dst, const u8 *src,
u8 *iv, unsigned int nblocks);
asmlinkage void sm4_ce_cbc_dec(const u32 *rkey, u8 *dst, const u8 *src,
u8 *iv, unsigned int nblocks);
asmlinkage void sm4_ce_cbc_cts_enc(const u32 *rkey, u8 *dst, const u8 *src,
u8 *iv, unsigned int nbytes);
asmlinkage void sm4_ce_cbc_cts_dec(const u32 *rkey, u8 *dst, const u8 *src,
u8 *iv, unsigned int nbytes);
asmlinkage void sm4_ce_ctr_enc(const u32 *rkey, u8 *dst, const u8 *src,
u8 *iv, unsigned int nblks);
asmlinkage void sm4_ce_xts_enc(const u32 *rkey1, u8 *dst, const u8 *src,
u8 *tweak, unsigned int nbytes,
const u32 *rkey2_enc);
asmlinkage void sm4_ce_xts_dec(const u32 *rkey1, u8 *dst, const u8 *src,
u8 *tweak, unsigned int nbytes,
const u32 *rkey2_enc);
asmlinkage void sm4_ce_mac_update(const u32 *rkey_enc, u8 *digest,
const u8 *src, unsigned int nblocks,
bool enc_before, bool enc_after);
EXPORT_SYMBOL(sm4_ce_expand_key);
EXPORT_SYMBOL(sm4_ce_crypt_block);
EXPORT_SYMBOL(sm4_ce_cbc_enc);
struct sm4_xts_ctx {
struct sm4_ctx key1;
struct sm4_ctx key2;
};
struct sm4_mac_tfm_ctx {
struct sm4_ctx key;
u8 __aligned(8) consts[];
};
struct sm4_mac_desc_ctx {
u8 digest[SM4_BLOCK_SIZE];
};
static int sm4_setkey(struct crypto_skcipher *tfm, const u8 *key,
unsigned int key_len)
{
struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm);
if (key_len != SM4_KEY_SIZE)
return -EINVAL;
scoped_ksimd()
sm4_ce_expand_key(key, ctx->rkey_enc, ctx->rkey_dec,
crypto_sm4_fk, crypto_sm4_ck);
return 0;
}
static int sm4_xts_setkey(struct crypto_skcipher *tfm, const u8 *key,
unsigned int key_len)
{
struct sm4_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
int ret;
if (key_len != SM4_KEY_SIZE * 2)
return -EINVAL;
ret = xts_verify_key(tfm, key, key_len);
if (ret)
return ret;
scoped_ksimd() {
sm4_ce_expand_key(key, ctx->key1.rkey_enc,
ctx->key1.rkey_dec, crypto_sm4_fk, crypto_sm4_ck);
sm4_ce_expand_key(&key[SM4_KEY_SIZE], ctx->key2.rkey_enc,
ctx->key2.rkey_dec, crypto_sm4_fk, crypto_sm4_ck);
}
return 0;
}
static int sm4_ecb_do_crypt(struct skcipher_request *req, const u32 *rkey)
{
struct skcipher_walk walk;
unsigned int nbytes;
int err;
err = skcipher_walk_virt(&walk, req, false);
while ((nbytes = walk.nbytes) > 0) {
const u8 *src = walk.src.virt.addr;
u8 *dst = walk.dst.virt.addr;
unsigned int nblks;
scoped_ksimd() {
nblks = BYTES2BLKS(nbytes);
if (nblks) {
sm4_ce_crypt(rkey, dst, src, nblks);
nbytes -= nblks * SM4_BLOCK_SIZE;
}
}
err = skcipher_walk_done(&walk, nbytes);
}
return err;
}
static int sm4_ecb_encrypt(struct skcipher_request *req)
{
struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm);
return sm4_ecb_do_crypt(req, ctx->rkey_enc);
}
static int sm4_ecb_decrypt(struct skcipher_request *req)
{
struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm);
return sm4_ecb_do_crypt(req, ctx->rkey_dec);
}
static int sm4_cbc_crypt(struct skcipher_request *req,
struct sm4_ctx *ctx, bool encrypt)
{
struct skcipher_walk walk;
unsigned int nbytes;
int err;
err = skcipher_walk_virt(&walk, req, false);
if (err)
return err;
while ((nbytes = walk.nbytes) > 0) {
const u8 *src = walk.src.virt.addr;
u8 *dst = walk.dst.virt.addr;
unsigned int nblocks;
nblocks = nbytes / SM4_BLOCK_SIZE;
if (nblocks) {
scoped_ksimd() {
if (encrypt)
sm4_ce_cbc_enc(ctx->rkey_enc, dst, src,
walk.iv, nblocks);
else
sm4_ce_cbc_dec(ctx->rkey_dec, dst, src,
walk.iv, nblocks);
}
}
err = skcipher_walk_done(&walk, nbytes % SM4_BLOCK_SIZE);
}
return err;
}
static int sm4_cbc_encrypt(struct skcipher_request *req)
{
struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm);
return sm4_cbc_crypt(req, ctx, true);
}
static int sm4_cbc_decrypt(struct skcipher_request *req)
{
struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm);
return sm4_cbc_crypt(req, ctx, false);
}
static int sm4_cbc_cts_crypt(struct skcipher_request *req, bool encrypt)
{
struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm);
struct scatterlist *src = req->src;
struct scatterlist *dst = req->dst;
struct scatterlist sg_src[2], sg_dst[2];
struct skcipher_request subreq;
struct skcipher_walk walk;
int cbc_blocks;
int err;
if (req->cryptlen < SM4_BLOCK_SIZE)
return -EINVAL;
if (req->cryptlen == SM4_BLOCK_SIZE)
return sm4_cbc_crypt(req, ctx, encrypt);
skcipher_request_set_tfm(&subreq, tfm);
skcipher_request_set_callback(&subreq, skcipher_request_flags(req),
NULL, NULL);
/* handle the CBC cryption part */
cbc_blocks = DIV_ROUND_UP(req->cryptlen, SM4_BLOCK_SIZE) - 2;
if (cbc_blocks) {
skcipher_request_set_crypt(&subreq, src, dst,
cbc_blocks * SM4_BLOCK_SIZE,
req->iv);
err = sm4_cbc_crypt(&subreq, ctx, encrypt);
if (err)
return err;
dst = src = scatterwalk_ffwd(sg_src, src, subreq.cryptlen);
if (req->dst != req->src)
dst = scatterwalk_ffwd(sg_dst, req->dst,
subreq.cryptlen);
}
/* handle ciphertext stealing */
skcipher_request_set_crypt(&subreq, src, dst,
req->cryptlen - cbc_blocks * SM4_BLOCK_SIZE,
req->iv);
err = skcipher_walk_virt(&walk, &subreq, false);
if (err)
return err;
scoped_ksimd() {
if (encrypt)
sm4_ce_cbc_cts_enc(ctx->rkey_enc, walk.dst.virt.addr,
walk.src.virt.addr, walk.iv, walk.nbytes);
else
sm4_ce_cbc_cts_dec(ctx->rkey_dec, walk.dst.virt.addr,
walk.src.virt.addr, walk.iv, walk.nbytes);
}
return skcipher_walk_done(&walk, 0);
}
static int sm4_cbc_cts_encrypt(struct skcipher_request *req)
{
return sm4_cbc_cts_crypt(req, true);
}
static int sm4_cbc_cts_decrypt(struct skcipher_request *req)
{
return sm4_cbc_cts_crypt(req, false);
}
static int sm4_ctr_crypt(struct skcipher_request *req)
{
struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm);
struct skcipher_walk walk;
unsigned int nbytes;
int err;
err = skcipher_walk_virt(&walk, req, false);
while ((nbytes = walk.nbytes) > 0) {
const u8 *src = walk.src.virt.addr;
u8 *dst = walk.dst.virt.addr;
unsigned int nblks;
scoped_ksimd() {
nblks = BYTES2BLKS(nbytes);
if (nblks) {
sm4_ce_ctr_enc(ctx->rkey_enc, dst, src, walk.iv, nblks);
dst += nblks * SM4_BLOCK_SIZE;
src += nblks * SM4_BLOCK_SIZE;
nbytes -= nblks * SM4_BLOCK_SIZE;
}
/* tail */
if (walk.nbytes == walk.total && nbytes > 0) {
u8 keystream[SM4_BLOCK_SIZE];
sm4_ce_crypt_block(ctx->rkey_enc, keystream, walk.iv);
crypto_inc(walk.iv, SM4_BLOCK_SIZE);
crypto_xor_cpy(dst, src, keystream, nbytes);
nbytes = 0;
}
}
err = skcipher_walk_done(&walk, nbytes);
}
return err;
}
static int sm4_xts_crypt(struct skcipher_request *req, bool encrypt)
{
struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
struct sm4_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
int tail = req->cryptlen % SM4_BLOCK_SIZE;
const u32 *rkey2_enc = ctx->key2.rkey_enc;
struct scatterlist sg_src[2], sg_dst[2];
struct skcipher_request subreq;
struct scatterlist *src, *dst;
struct skcipher_walk walk;
unsigned int nbytes;
int err;
if (req->cryptlen < SM4_BLOCK_SIZE)
return -EINVAL;
err = skcipher_walk_virt(&walk, req, false);
if (err)
return err;
if (unlikely(tail > 0 && walk.nbytes < walk.total)) {
int nblocks = DIV_ROUND_UP(req->cryptlen, SM4_BLOCK_SIZE) - 2;
skcipher_walk_abort(&walk);
skcipher_request_set_tfm(&subreq, tfm);
skcipher_request_set_callback(&subreq,
skcipher_request_flags(req),
NULL, NULL);
skcipher_request_set_crypt(&subreq, req->src, req->dst,
nblocks * SM4_BLOCK_SIZE, req->iv);
err = skcipher_walk_virt(&walk, &subreq, false);
if (err)
return err;
} else {
tail = 0;
}
scoped_ksimd() {
while ((nbytes = walk.nbytes) >= SM4_BLOCK_SIZE) {
if (nbytes < walk.total)
nbytes &= ~(SM4_BLOCK_SIZE - 1);
if (encrypt)
sm4_ce_xts_enc(ctx->key1.rkey_enc, walk.dst.virt.addr,
walk.src.virt.addr, walk.iv, nbytes,
rkey2_enc);
else
sm4_ce_xts_dec(ctx->key1.rkey_dec, walk.dst.virt.addr,
walk.src.virt.addr, walk.iv, nbytes,
rkey2_enc);
rkey2_enc = NULL;
err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
if (err)
return err;
}
if (likely(tail == 0))
return 0;
/* handle ciphertext stealing */
dst = src = scatterwalk_ffwd(sg_src, req->src, subreq.cryptlen);
if (req->dst != req->src)
dst = scatterwalk_ffwd(sg_dst, req->dst, subreq.cryptlen);
skcipher_request_set_crypt(&subreq, src, dst,
SM4_BLOCK_SIZE + tail, req->iv);
err = skcipher_walk_virt(&walk, &subreq, false);
if (err)
return err;
if (encrypt)
sm4_ce_xts_enc(ctx->key1.rkey_enc, walk.dst.virt.addr,
walk.src.virt.addr, walk.iv, walk.nbytes,
rkey2_enc);
else
sm4_ce_xts_dec(ctx->key1.rkey_dec, walk.dst.virt.addr,
walk.src.virt.addr, walk.iv, walk.nbytes,
rkey2_enc);
}
return skcipher_walk_done(&walk, 0);
}
static int sm4_xts_encrypt(struct skcipher_request *req)
{
return sm4_xts_crypt(req, true);
}
static int sm4_xts_decrypt(struct skcipher_request *req)
{
return sm4_xts_crypt(req, false);
}
static struct skcipher_alg sm4_algs[] = {
{
.base = {
.cra_name = "ecb(sm4)",
.cra_driver_name = "ecb-sm4-ce",
.cra_priority = 400,
.cra_blocksize = SM4_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct sm4_ctx),
.cra_module = THIS_MODULE,
},
.min_keysize = SM4_KEY_SIZE,
.max_keysize = SM4_KEY_SIZE,
.setkey = sm4_setkey,
.encrypt = sm4_ecb_encrypt,
.decrypt = sm4_ecb_decrypt,
}, {
.base = {
.cra_name = "cbc(sm4)",
.cra_driver_name = "cbc-sm4-ce",
.cra_priority = 400,
.cra_blocksize = SM4_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct sm4_ctx),
.cra_module = THIS_MODULE,
},
.min_keysize = SM4_KEY_SIZE,
.max_keysize = SM4_KEY_SIZE,
.ivsize = SM4_BLOCK_SIZE,
.setkey = sm4_setkey,
.encrypt = sm4_cbc_encrypt,
.decrypt = sm4_cbc_decrypt,
}, {
.base = {
.cra_name = "ctr(sm4)",
.cra_driver_name = "ctr-sm4-ce",
.cra_priority = 400,
.cra_blocksize = 1,
.cra_ctxsize = sizeof(struct sm4_ctx),
.cra_module = THIS_MODULE,
},
.min_keysize = SM4_KEY_SIZE,
.max_keysize = SM4_KEY_SIZE,
.ivsize = SM4_BLOCK_SIZE,
.chunksize = SM4_BLOCK_SIZE,
.setkey = sm4_setkey,
.encrypt = sm4_ctr_crypt,
.decrypt = sm4_ctr_crypt,
}, {
.base = {
.cra_name = "cts(cbc(sm4))",
.cra_driver_name = "cts-cbc-sm4-ce",
.cra_priority = 400,
.cra_blocksize = SM4_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct sm4_ctx),
.cra_module = THIS_MODULE,
},
.min_keysize = SM4_KEY_SIZE,
.max_keysize = SM4_KEY_SIZE,
.ivsize = SM4_BLOCK_SIZE,
.walksize = SM4_BLOCK_SIZE * 2,
.setkey = sm4_setkey,
.encrypt = sm4_cbc_cts_encrypt,
.decrypt = sm4_cbc_cts_decrypt,
}, {
.base = {
.cra_name = "xts(sm4)",
.cra_driver_name = "xts-sm4-ce",
.cra_priority = 400,
.cra_blocksize = SM4_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct sm4_xts_ctx),
.cra_module = THIS_MODULE,
},
.min_keysize = SM4_KEY_SIZE * 2,
.max_keysize = SM4_KEY_SIZE * 2,
.ivsize = SM4_BLOCK_SIZE,
.walksize = SM4_BLOCK_SIZE * 2,
.setkey = sm4_xts_setkey,
.encrypt = sm4_xts_encrypt,
.decrypt = sm4_xts_decrypt,
}
};
static int sm4_cbcmac_setkey(struct crypto_shash *tfm, const u8 *key,
unsigned int key_len)
{
struct sm4_mac_tfm_ctx *ctx = crypto_shash_ctx(tfm);
if (key_len != SM4_KEY_SIZE)
return -EINVAL;
scoped_ksimd()
sm4_ce_expand_key(key, ctx->key.rkey_enc, ctx->key.rkey_dec,
crypto_sm4_fk, crypto_sm4_ck);
return 0;
}
static int sm4_cmac_setkey(struct crypto_shash *tfm, const u8 *key,
unsigned int key_len)
{
struct sm4_mac_tfm_ctx *ctx = crypto_shash_ctx(tfm);
be128 *consts = (be128 *)ctx->consts;
u64 a, b;
if (key_len != SM4_KEY_SIZE)
return -EINVAL;
memset(consts, 0, SM4_BLOCK_SIZE);
scoped_ksimd() {
sm4_ce_expand_key(key, ctx->key.rkey_enc, ctx->key.rkey_dec,
crypto_sm4_fk, crypto_sm4_ck);
/* encrypt the zero block */
sm4_ce_crypt_block(ctx->key.rkey_enc, (u8 *)consts, (const u8 *)consts);
}
/* gf(2^128) multiply zero-ciphertext with u and u^2 */
a = be64_to_cpu(consts[0].a);
b = be64_to_cpu(consts[0].b);
consts[0].a = cpu_to_be64((a << 1) | (b >> 63));
consts[0].b = cpu_to_be64((b << 1) ^ ((a >> 63) ? 0x87 : 0));
a = be64_to_cpu(consts[0].a);
b = be64_to_cpu(consts[0].b);
consts[1].a = cpu_to_be64((a << 1) | (b >> 63));
consts[1].b = cpu_to_be64((b << 1) ^ ((a >> 63) ? 0x87 : 0));
return 0;
}
static int sm4_xcbc_setkey(struct crypto_shash *tfm, const u8 *key,
unsigned int key_len)
{
struct sm4_mac_tfm_ctx *ctx = crypto_shash_ctx(tfm);
u8 __aligned(8) key2[SM4_BLOCK_SIZE];
static u8 const ks[3][SM4_BLOCK_SIZE] = {
{ [0 ... SM4_BLOCK_SIZE - 1] = 0x1},
{ [0 ... SM4_BLOCK_SIZE - 1] = 0x2},
{ [0 ... SM4_BLOCK_SIZE - 1] = 0x3},
};
if (key_len != SM4_KEY_SIZE)
return -EINVAL;
scoped_ksimd() {
sm4_ce_expand_key(key, ctx->key.rkey_enc, ctx->key.rkey_dec,
crypto_sm4_fk, crypto_sm4_ck);
sm4_ce_crypt_block(ctx->key.rkey_enc, key2, ks[0]);
sm4_ce_crypt(ctx->key.rkey_enc, ctx->consts, ks[1], 2);
sm4_ce_expand_key(key2, ctx->key.rkey_enc, ctx->key.rkey_dec,
crypto_sm4_fk, crypto_sm4_ck);
}
return 0;
}
static int sm4_mac_init(struct shash_desc *desc)
{
struct sm4_mac_desc_ctx *ctx = shash_desc_ctx(desc);
memset(ctx->digest, 0, SM4_BLOCK_SIZE);
return 0;
}
static int sm4_mac_update(struct shash_desc *desc, const u8 *p,
unsigned int len)
{
struct sm4_mac_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm);
struct sm4_mac_desc_ctx *ctx = shash_desc_ctx(desc);
unsigned int nblocks = len / SM4_BLOCK_SIZE;
len %= SM4_BLOCK_SIZE;
scoped_ksimd()
sm4_ce_mac_update(tctx->key.rkey_enc, ctx->digest, p,
nblocks, false, true);
return len;
}
static int sm4_cmac_finup(struct shash_desc *desc, const u8 *src,
unsigned int len, u8 *out)
{
struct sm4_mac_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm);
struct sm4_mac_desc_ctx *ctx = shash_desc_ctx(desc);
const u8 *consts = tctx->consts;
crypto_xor(ctx->digest, src, len);
if (len != SM4_BLOCK_SIZE) {
ctx->digest[len] ^= 0x80;
consts += SM4_BLOCK_SIZE;
}
scoped_ksimd()
sm4_ce_mac_update(tctx->key.rkey_enc, ctx->digest, consts, 1,
false, true);
memcpy(out, ctx->digest, SM4_BLOCK_SIZE);
return 0;
}
static int sm4_cbcmac_finup(struct shash_desc *desc, const u8 *src,
unsigned int len, u8 *out)
{
struct sm4_mac_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm);
struct sm4_mac_desc_ctx *ctx = shash_desc_ctx(desc);
if (len) {
crypto_xor(ctx->digest, src, len);
scoped_ksimd()
sm4_ce_crypt_block(tctx->key.rkey_enc, ctx->digest,
ctx->digest);
}
memcpy(out, ctx->digest, SM4_BLOCK_SIZE);
return 0;
}
static struct shash_alg sm4_mac_algs[] = {
{
.base = {
.cra_name = "cmac(sm4)",
.cra_driver_name = "cmac-sm4-ce",
.cra_priority = 400,
.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY |
CRYPTO_AHASH_ALG_FINAL_NONZERO,
.cra_blocksize = SM4_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct sm4_mac_tfm_ctx)
+ SM4_BLOCK_SIZE * 2,
.cra_module = THIS_MODULE,
},
.digestsize = SM4_BLOCK_SIZE,
.init = sm4_mac_init,
.update = sm4_mac_update,
.finup = sm4_cmac_finup,
.setkey = sm4_cmac_setkey,
.descsize = sizeof(struct sm4_mac_desc_ctx),
}, {
.base = {
.cra_name = "xcbc(sm4)",
.cra_driver_name = "xcbc-sm4-ce",
.cra_priority = 400,
.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY |
CRYPTO_AHASH_ALG_FINAL_NONZERO,
.cra_blocksize = SM4_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct sm4_mac_tfm_ctx)
+ SM4_BLOCK_SIZE * 2,
.cra_module = THIS_MODULE,
},
.digestsize = SM4_BLOCK_SIZE,
.init = sm4_mac_init,
.update = sm4_mac_update,
.finup = sm4_cmac_finup,
.setkey = sm4_xcbc_setkey,
.descsize = sizeof(struct sm4_mac_desc_ctx),
}, {
.base = {
.cra_name = "cbcmac(sm4)",
.cra_driver_name = "cbcmac-sm4-ce",
.cra_priority = 400,
.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY,
.cra_blocksize = SM4_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct sm4_mac_tfm_ctx),
.cra_module = THIS_MODULE,
},
.digestsize = SM4_BLOCK_SIZE,
.init = sm4_mac_init,
.update = sm4_mac_update,
.finup = sm4_cbcmac_finup,
.setkey = sm4_cbcmac_setkey,
.descsize = sizeof(struct sm4_mac_desc_ctx),
}
};
static int __init sm4_init(void)
{
int err;
err = crypto_register_skciphers(sm4_algs, ARRAY_SIZE(sm4_algs));
if (err)
return err;
err = crypto_register_shashes(sm4_mac_algs, ARRAY_SIZE(sm4_mac_algs));
if (err)
goto out_err;
return 0;
out_err:
crypto_unregister_skciphers(sm4_algs, ARRAY_SIZE(sm4_algs));
return err;
}
static void __exit sm4_exit(void)
{
crypto_unregister_shashes(sm4_mac_algs, ARRAY_SIZE(sm4_mac_algs));
crypto_unregister_skciphers(sm4_algs, ARRAY_SIZE(sm4_algs));
}
module_cpu_feature_match(SM4, sm4_init);
module_exit(sm4_exit);
MODULE_DESCRIPTION("SM4 ECB/CBC/CTR/XTS using ARMv8 Crypto Extensions");
MODULE_ALIAS_CRYPTO("sm4-ce");
MODULE_ALIAS_CRYPTO("sm4");
MODULE_ALIAS_CRYPTO("ecb(sm4)");
MODULE_ALIAS_CRYPTO("cbc(sm4)");
MODULE_ALIAS_CRYPTO("ctr(sm4)");
MODULE_ALIAS_CRYPTO("cts(cbc(sm4))");
MODULE_ALIAS_CRYPTO("xts(sm4)");
MODULE_ALIAS_CRYPTO("cmac(sm4)");
MODULE_ALIAS_CRYPTO("xcbc(sm4)");
MODULE_ALIAS_CRYPTO("cbcmac(sm4)");
MODULE_AUTHOR("Tianjia Zhang <tianjia.zhang@linux.alibaba.com>");
MODULE_LICENSE("GPL v2");
linux-7.2.3/arch/arm64/crypto/sm4-ce.h
+13
View File
@@ -0,0 +1,13 @@
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
* SM4 common functions for Crypto Extensions
* Copyright (C) 2022 Tianjia Zhang <tianjia.zhang@linux.alibaba.com>
*/
void sm4_ce_expand_key(const u8 *key, u32 *rkey_enc, u32 *rkey_dec,
const u32 *fk, const u32 *ck);
void sm4_ce_crypt_block(const u32 *rkey, u8 *dst, const u8 *src);
void sm4_ce_cbc_enc(const u32 *rkey_enc, u8 *dst, const u8 *src,
u8 *iv, unsigned int nblocks);
linux-7.2.3/arch/arm64/crypto/sm4-neon-core.S
+566
View File
@@ -0,0 +1,566 @@
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
* SM4 Cipher Algorithm for ARMv8 NEON
* as specified in
* https://tools.ietf.org/id/draft-ribose-cfrg-sm4-10.html
*
* Copyright (C) 2022, Alibaba Group.
* Copyright (C) 2022 Tianjia Zhang <tianjia.zhang@linux.alibaba.com>
*/
#include <linux/linkage.h>
#include <asm/assembler.h>
/* Register macros */
#define RTMP0 v8
#define RTMP1 v9
#define RTMP2 v10
#define RTMP3 v11
#define RTMP4 v12
#define RTMP5 v13
#define RTMP6 v14
#define RTMP7 v15
#define RX0 v12
#define RX1 v13
#define RKEY v14
#define RIV v15
/* Helper macros. */
#define SM4_PREPARE() \
adr_l x5, crypto_sm4_sbox; \
ld1 {v16.16b-v19.16b}, [x5], #64; \
ld1 {v20.16b-v23.16b}, [x5], #64; \
ld1 {v24.16b-v27.16b}, [x5], #64; \
ld1 {v28.16b-v31.16b}, [x5];
#define transpose_4x4(s0, s1, s2, s3) \
zip1 RTMP0.4s, s0.4s, s1.4s; \
zip1 RTMP1.4s, s2.4s, s3.4s; \
zip2 RTMP2.4s, s0.4s, s1.4s; \
zip2 RTMP3.4s, s2.4s, s3.4s; \
zip1 s0.2d, RTMP0.2d, RTMP1.2d; \
zip2 s1.2d, RTMP0.2d, RTMP1.2d; \
zip1 s2.2d, RTMP2.2d, RTMP3.2d; \
zip2 s3.2d, RTMP2.2d, RTMP3.2d;
#define transpose_4x4_2x(s0, s1, s2, s3, s4, s5, s6, s7) \
zip1 RTMP0.4s, s0.4s, s1.4s; \
zip1 RTMP1.4s, s2.4s, s3.4s; \
zip2 RTMP2.4s, s0.4s, s1.4s; \
zip2 RTMP3.4s, s2.4s, s3.4s; \
zip1 RTMP4.4s, s4.4s, s5.4s; \
zip1 RTMP5.4s, s6.4s, s7.4s; \
zip2 RTMP6.4s, s4.4s, s5.4s; \
zip2 RTMP7.4s, s6.4s, s7.4s; \
zip1 s0.2d, RTMP0.2d, RTMP1.2d; \
zip2 s1.2d, RTMP0.2d, RTMP1.2d; \
zip1 s2.2d, RTMP2.2d, RTMP3.2d; \
zip2 s3.2d, RTMP2.2d, RTMP3.2d; \
zip1 s4.2d, RTMP4.2d, RTMP5.2d; \
zip2 s5.2d, RTMP4.2d, RTMP5.2d; \
zip1 s6.2d, RTMP6.2d, RTMP7.2d; \
zip2 s7.2d, RTMP6.2d, RTMP7.2d;
#define rotate_clockwise_4x4(s0, s1, s2, s3) \
zip1 RTMP0.4s, s1.4s, s0.4s; \
zip2 RTMP1.4s, s1.4s, s0.4s; \
zip1 RTMP2.4s, s3.4s, s2.4s; \
zip2 RTMP3.4s, s3.4s, s2.4s; \
zip1 s0.2d, RTMP2.2d, RTMP0.2d; \
zip2 s1.2d, RTMP2.2d, RTMP0.2d; \
zip1 s2.2d, RTMP3.2d, RTMP1.2d; \
zip2 s3.2d, RTMP3.2d, RTMP1.2d;
#define rotate_clockwise_4x4_2x(s0, s1, s2, s3, s4, s5, s6, s7) \
zip1 RTMP0.4s, s1.4s, s0.4s; \
zip1 RTMP2.4s, s3.4s, s2.4s; \
zip2 RTMP1.4s, s1.4s, s0.4s; \
zip2 RTMP3.4s, s3.4s, s2.4s; \
zip1 RTMP4.4s, s5.4s, s4.4s; \
zip1 RTMP6.4s, s7.4s, s6.4s; \
zip2 RTMP5.4s, s5.4s, s4.4s; \
zip2 RTMP7.4s, s7.4s, s6.4s; \
zip1 s0.2d, RTMP2.2d, RTMP0.2d; \
zip2 s1.2d, RTMP2.2d, RTMP0.2d; \
zip1 s2.2d, RTMP3.2d, RTMP1.2d; \
zip2 s3.2d, RTMP3.2d, RTMP1.2d; \
zip1 s4.2d, RTMP6.2d, RTMP4.2d; \
zip2 s5.2d, RTMP6.2d, RTMP4.2d; \
zip1 s6.2d, RTMP7.2d, RTMP5.2d; \
zip2 s7.2d, RTMP7.2d, RTMP5.2d;
#define ROUND4(round, s0, s1, s2, s3) \
dup RX0.4s, RKEY.s[round]; \
/* rk ^ s1 ^ s2 ^ s3 */ \
eor RTMP1.16b, s2.16b, s3.16b; \
eor RX0.16b, RX0.16b, s1.16b; \
eor RX0.16b, RX0.16b, RTMP1.16b; \
\
/* sbox, non-linear part */ \
movi RTMP3.16b, #64; /* sizeof(sbox) / 4 */ \
tbl RTMP0.16b, {v16.16b-v19.16b}, RX0.16b; \
sub RX0.16b, RX0.16b, RTMP3.16b; \
tbx RTMP0.16b, {v20.16b-v23.16b}, RX0.16b; \
sub RX0.16b, RX0.16b, RTMP3.16b; \
tbx RTMP0.16b, {v24.16b-v27.16b}, RX0.16b; \
sub RX0.16b, RX0.16b, RTMP3.16b; \
tbx RTMP0.16b, {v28.16b-v31.16b}, RX0.16b; \
\
/* linear part */ \
shl RTMP1.4s, RTMP0.4s, #8; \
shl RTMP2.4s, RTMP0.4s, #16; \
shl RTMP3.4s, RTMP0.4s, #24; \
sri RTMP1.4s, RTMP0.4s, #(32-8); \
sri RTMP2.4s, RTMP0.4s, #(32-16); \
sri RTMP3.4s, RTMP0.4s, #(32-24); \
/* RTMP1 = x ^ rol32(x, 8) ^ rol32(x, 16) */ \
eor RTMP1.16b, RTMP1.16b, RTMP0.16b; \
eor RTMP1.16b, RTMP1.16b, RTMP2.16b; \
/* RTMP3 = x ^ rol32(x, 24) ^ rol32(RTMP1, 2) */ \
eor RTMP3.16b, RTMP3.16b, RTMP0.16b; \
shl RTMP2.4s, RTMP1.4s, 2; \
sri RTMP2.4s, RTMP1.4s, #(32-2); \
eor RTMP3.16b, RTMP3.16b, RTMP2.16b; \
/* s0 ^= RTMP3 */ \
eor s0.16b, s0.16b, RTMP3.16b;
#define SM4_CRYPT_BLK4_BE(b0, b1, b2, b3) \
mov x6, 8; \
4: \
ld1 {RKEY.4s}, [x0], #16; \
subs x6, x6, #1; \
\
ROUND4(0, b0, b1, b2, b3); \
ROUND4(1, b1, b2, b3, b0); \
ROUND4(2, b2, b3, b0, b1); \
ROUND4(3, b3, b0, b1, b2); \
\
bne 4b; \
\
rev32 b0.16b, b0.16b; \
rev32 b1.16b, b1.16b; \
rev32 b2.16b, b2.16b; \
rev32 b3.16b, b3.16b; \
\
rotate_clockwise_4x4(b0, b1, b2, b3); \
\
/* repoint to rkey */ \
sub x0, x0, #128;
#define SM4_CRYPT_BLK4(b0, b1, b2, b3) \
rev32 b0.16b, b0.16b; \
rev32 b1.16b, b1.16b; \
rev32 b2.16b, b2.16b; \
rev32 b3.16b, b3.16b; \
SM4_CRYPT_BLK4_BE(b0, b1, b2, b3);
#define ROUND8(round, s0, s1, s2, s3, t0, t1, t2, t3) \
/* rk ^ s1 ^ s2 ^ s3 */ \
dup RX0.4s, RKEY.s[round]; \
eor RTMP0.16b, s2.16b, s3.16b; \
mov RX1.16b, RX0.16b; \
eor RTMP1.16b, t2.16b, t3.16b; \
eor RX0.16b, RX0.16b, s1.16b; \
eor RX1.16b, RX1.16b, t1.16b; \
eor RX0.16b, RX0.16b, RTMP0.16b; \
eor RX1.16b, RX1.16b, RTMP1.16b; \
\
/* sbox, non-linear part */ \
movi RTMP3.16b, #64; /* sizeof(sbox) / 4 */ \
tbl RTMP0.16b, {v16.16b-v19.16b}, RX0.16b; \
tbl RTMP1.16b, {v16.16b-v19.16b}, RX1.16b; \
sub RX0.16b, RX0.16b, RTMP3.16b; \
sub RX1.16b, RX1.16b, RTMP3.16b; \
tbx RTMP0.16b, {v20.16b-v23.16b}, RX0.16b; \
tbx RTMP1.16b, {v20.16b-v23.16b}, RX1.16b; \
sub RX0.16b, RX0.16b, RTMP3.16b; \
sub RX1.16b, RX1.16b, RTMP3.16b; \
tbx RTMP0.16b, {v24.16b-v27.16b}, RX0.16b; \
tbx RTMP1.16b, {v24.16b-v27.16b}, RX1.16b; \
sub RX0.16b, RX0.16b, RTMP3.16b; \
sub RX1.16b, RX1.16b, RTMP3.16b; \
tbx RTMP0.16b, {v28.16b-v31.16b}, RX0.16b; \
tbx RTMP1.16b, {v28.16b-v31.16b}, RX1.16b; \
\
/* linear part */ \
shl RX0.4s, RTMP0.4s, #8; \
shl RX1.4s, RTMP1.4s, #8; \
shl RTMP2.4s, RTMP0.4s, #16; \
shl RTMP3.4s, RTMP1.4s, #16; \
sri RX0.4s, RTMP0.4s, #(32 - 8); \
sri RX1.4s, RTMP1.4s, #(32 - 8); \
sri RTMP2.4s, RTMP0.4s, #(32 - 16); \
sri RTMP3.4s, RTMP1.4s, #(32 - 16); \
/* RX = x ^ rol32(x, 8) ^ rol32(x, 16) */ \
eor RX0.16b, RX0.16b, RTMP0.16b; \
eor RX1.16b, RX1.16b, RTMP1.16b; \
eor RX0.16b, RX0.16b, RTMP2.16b; \
eor RX1.16b, RX1.16b, RTMP3.16b; \
/* RTMP0/1 ^= x ^ rol32(x, 24) ^ rol32(RX, 2) */ \
shl RTMP2.4s, RTMP0.4s, #24; \
shl RTMP3.4s, RTMP1.4s, #24; \
sri RTMP2.4s, RTMP0.4s, #(32 - 24); \
sri RTMP3.4s, RTMP1.4s, #(32 - 24); \
eor RTMP0.16b, RTMP0.16b, RTMP2.16b; \
eor RTMP1.16b, RTMP1.16b, RTMP3.16b; \
shl RTMP2.4s, RX0.4s, #2; \
shl RTMP3.4s, RX1.4s, #2; \
sri RTMP2.4s, RX0.4s, #(32 - 2); \
sri RTMP3.4s, RX1.4s, #(32 - 2); \
eor RTMP0.16b, RTMP0.16b, RTMP2.16b; \
eor RTMP1.16b, RTMP1.16b, RTMP3.16b; \
/* s0/t0 ^= RTMP0/1 */ \
eor s0.16b, s0.16b, RTMP0.16b; \
eor t0.16b, t0.16b, RTMP1.16b;
#define SM4_CRYPT_BLK8_norotate(b0, b1, b2, b3, b4, b5, b6, b7) \
rev32 b0.16b, b0.16b; \
rev32 b1.16b, b1.16b; \
rev32 b2.16b, b2.16b; \
rev32 b3.16b, b3.16b; \
rev32 b4.16b, b4.16b; \
rev32 b5.16b, b5.16b; \
rev32 b6.16b, b6.16b; \
rev32 b7.16b, b7.16b; \
\
mov x6, 8; \
8: \
ld1 {RKEY.4s}, [x0], #16; \
subs x6, x6, #1; \
\
ROUND8(0, b0, b1, b2, b3, b4, b5, b6, b7); \
ROUND8(1, b1, b2, b3, b0, b5, b6, b7, b4); \
ROUND8(2, b2, b3, b0, b1, b6, b7, b4, b5); \
ROUND8(3, b3, b0, b1, b2, b7, b4, b5, b6); \
\
bne 8b; \
\
rev32 b0.16b, b0.16b; \
rev32 b1.16b, b1.16b; \
rev32 b2.16b, b2.16b; \
rev32 b3.16b, b3.16b; \
rev32 b4.16b, b4.16b; \
rev32 b5.16b, b5.16b; \
rev32 b6.16b, b6.16b; \
rev32 b7.16b, b7.16b; \
\
/* repoint to rkey */ \
sub x0, x0, #128;
#define SM4_CRYPT_BLK8(b0, b1, b2, b3, b4, b5, b6, b7) \
SM4_CRYPT_BLK8_norotate(b0, b1, b2, b3, b4, b5, b6, b7); \
rotate_clockwise_4x4_2x(b0, b1, b2, b3, b4, b5, b6, b7); \
.align 3
SYM_FUNC_START(sm4_neon_crypt)
/* input:
* x0: round key array, CTX
* x1: dst
* x2: src
* w3: nblocks
*/
SM4_PREPARE()
.Lcrypt_loop_8x:
sub w3, w3, #8
tbnz w3, #31, .Lcrypt_4x
ld4 {v0.4s-v3.4s}, [x2], #64
ld4 {v4.4s-v7.4s}, [x2], #64
SM4_CRYPT_BLK8(v0, v1, v2, v3, v4, v5, v6, v7)
st1 {v0.16b-v3.16b}, [x1], #64
st1 {v4.16b-v7.16b}, [x1], #64
cbz w3, .Lcrypt_end
b .Lcrypt_loop_8x
.Lcrypt_4x:
add w3, w3, #8
cmp w3, #4
blt .Lcrypt_tail
sub w3, w3, #4
ld4 {v0.4s-v3.4s}, [x2], #64
SM4_CRYPT_BLK4(v0, v1, v2, v3)
st1 {v0.16b-v3.16b}, [x1], #64
cbz w3, .Lcrypt_end
.Lcrypt_tail:
cmp w3, #2
ld1 {v0.16b}, [x2], #16
blt .Lcrypt_tail_load_done
ld1 {v1.16b}, [x2], #16
beq .Lcrypt_tail_load_done
ld1 {v2.16b}, [x2], #16
.Lcrypt_tail_load_done:
transpose_4x4(v0, v1, v2, v3)
SM4_CRYPT_BLK4(v0, v1, v2, v3)
cmp w3, #2
st1 {v0.16b}, [x1], #16
blt .Lcrypt_end
st1 {v1.16b}, [x1], #16
beq .Lcrypt_end
st1 {v2.16b}, [x1], #16
.Lcrypt_end:
ret
SYM_FUNC_END(sm4_neon_crypt)
.align 3
SYM_FUNC_START(sm4_neon_cbc_dec)
/* input:
* x0: round key array, CTX
* x1: dst
* x2: src
* x3: iv (big endian, 128 bit)
* w4: nblocks
*/
SM4_PREPARE()
ld1 {RIV.16b}, [x3]
.Lcbc_dec_loop_8x:
sub w4, w4, #8
tbnz w4, #31, .Lcbc_dec_4x
ld4 {v0.4s-v3.4s}, [x2], #64
ld4 {v4.4s-v7.4s}, [x2]
SM4_CRYPT_BLK8_norotate(v0, v1, v2, v3, v4, v5, v6, v7)
/* Avoid overwriting the RIV register */
rotate_clockwise_4x4(v0, v1, v2, v3)
rotate_clockwise_4x4(v4, v5, v6, v7)
sub x2, x2, #64
eor v0.16b, v0.16b, RIV.16b
ld1 {RTMP0.16b-RTMP3.16b}, [x2], #64
ld1 {RTMP4.16b-RTMP7.16b}, [x2], #64
eor v1.16b, v1.16b, RTMP0.16b
eor v2.16b, v2.16b, RTMP1.16b
eor v3.16b, v3.16b, RTMP2.16b
eor v4.16b, v4.16b, RTMP3.16b
eor v5.16b, v5.16b, RTMP4.16b
eor v6.16b, v6.16b, RTMP5.16b
eor v7.16b, v7.16b, RTMP6.16b
mov RIV.16b, RTMP7.16b
st1 {v0.16b-v3.16b}, [x1], #64
st1 {v4.16b-v7.16b}, [x1], #64
cbz w4, .Lcbc_dec_end
b .Lcbc_dec_loop_8x
.Lcbc_dec_4x:
add w4, w4, #8
cmp w4, #4
blt .Lcbc_dec_tail
sub w4, w4, #4
ld1 {v0.16b-v3.16b}, [x2], #64
rev32 v4.16b, v0.16b
rev32 v5.16b, v1.16b
rev32 v6.16b, v2.16b
rev32 v7.16b, v3.16b
transpose_4x4(v4, v5, v6, v7)
SM4_CRYPT_BLK4_BE(v4, v5, v6, v7)
eor v4.16b, v4.16b, RIV.16b
eor v5.16b, v5.16b, v0.16b
eor v6.16b, v6.16b, v1.16b
eor v7.16b, v7.16b, v2.16b
mov RIV.16b, v3.16b
st1 {v4.16b-v7.16b}, [x1], #64
cbz w4, .Lcbc_dec_end
.Lcbc_dec_tail:
cmp w4, #2
ld1 {v0.16b}, [x2], #16
blt .Lcbc_dec_tail_load_done
ld1 {v1.16b}, [x2], #16
beq .Lcbc_dec_tail_load_done
ld1 {v2.16b}, [x2], #16
.Lcbc_dec_tail_load_done:
rev32 v4.16b, v0.16b
rev32 v5.16b, v1.16b
rev32 v6.16b, v2.16b
transpose_4x4(v4, v5, v6, v7)
SM4_CRYPT_BLK4_BE(v4, v5, v6, v7)
cmp w4, #2
eor v4.16b, v4.16b, RIV.16b
mov RIV.16b, v0.16b
st1 {v4.16b}, [x1], #16
blt .Lcbc_dec_end
eor v5.16b, v5.16b, v0.16b
mov RIV.16b, v1.16b
st1 {v5.16b}, [x1], #16
beq .Lcbc_dec_end
eor v6.16b, v6.16b, v1.16b
mov RIV.16b, v2.16b
st1 {v6.16b}, [x1], #16
.Lcbc_dec_end:
/* store new IV */
st1 {RIV.16b}, [x3]
ret
SYM_FUNC_END(sm4_neon_cbc_dec)
.align 3
SYM_FUNC_START(sm4_neon_ctr_crypt)
/* input:
* x0: round key array, CTX
* x1: dst
* x2: src
* x3: ctr (big endian, 128 bit)
* w4: nblocks
*/
SM4_PREPARE()
ldp x7, x8, [x3]
rev x7, x7
rev x8, x8
.Lctr_crypt_loop_8x:
sub w4, w4, #8
tbnz w4, #31, .Lctr_crypt_4x
#define inc_le128(vctr) \
mov vctr.d[1], x8; \
mov vctr.d[0], x7; \
adds x8, x8, #1; \
rev64 vctr.16b, vctr.16b; \
adc x7, x7, xzr;
/* construct CTRs */
inc_le128(v0) /* +0 */
inc_le128(v1) /* +1 */
inc_le128(v2) /* +2 */
inc_le128(v3) /* +3 */
inc_le128(v4) /* +4 */
inc_le128(v5) /* +5 */
inc_le128(v6) /* +6 */
inc_le128(v7) /* +7 */
transpose_4x4_2x(v0, v1, v2, v3, v4, v5, v6, v7)
SM4_CRYPT_BLK8(v0, v1, v2, v3, v4, v5, v6, v7)
ld1 {RTMP0.16b-RTMP3.16b}, [x2], #64
ld1 {RTMP4.16b-RTMP7.16b}, [x2], #64
eor v0.16b, v0.16b, RTMP0.16b
eor v1.16b, v1.16b, RTMP1.16b
eor v2.16b, v2.16b, RTMP2.16b
eor v3.16b, v3.16b, RTMP3.16b
eor v4.16b, v4.16b, RTMP4.16b
eor v5.16b, v5.16b, RTMP5.16b
eor v6.16b, v6.16b, RTMP6.16b
eor v7.16b, v7.16b, RTMP7.16b
st1 {v0.16b-v3.16b}, [x1], #64
st1 {v4.16b-v7.16b}, [x1], #64
cbz w4, .Lctr_crypt_end
b .Lctr_crypt_loop_8x
.Lctr_crypt_4x:
add w4, w4, #8
cmp w4, #4
blt .Lctr_crypt_tail
sub w4, w4, #4
/* construct CTRs */
inc_le128(v0) /* +0 */
inc_le128(v1) /* +1 */
inc_le128(v2) /* +2 */
inc_le128(v3) /* +3 */
ld1 {v4.16b-v7.16b}, [x2], #64
transpose_4x4(v0, v1, v2, v3)
SM4_CRYPT_BLK4(v0, v1, v2, v3)
eor v0.16b, v0.16b, v4.16b
eor v1.16b, v1.16b, v5.16b
eor v2.16b, v2.16b, v6.16b
eor v3.16b, v3.16b, v7.16b
st1 {v0.16b-v3.16b}, [x1], #64
cbz w4, .Lctr_crypt_end
.Lctr_crypt_tail:
/* inc_le128 will change the sign bit */
ld1 {v4.16b}, [x2], #16
inc_le128(v0)
cmp w4, #2
blt .Lctr_crypt_tail_load_done
ld1 {v5.16b}, [x2], #16
inc_le128(v1)
cmp w4, #2
beq .Lctr_crypt_tail_load_done
ld1 {v6.16b}, [x2], #16
inc_le128(v2)
.Lctr_crypt_tail_load_done:
transpose_4x4(v0, v1, v2, v3)
SM4_CRYPT_BLK4(v0, v1, v2, v3)
cmp w4, #2
eor v0.16b, v0.16b, v4.16b
st1 {v0.16b}, [x1], #16
blt .Lctr_crypt_end
eor v1.16b, v1.16b, v5.16b
st1 {v1.16b}, [x1], #16
beq .Lctr_crypt_end
eor v2.16b, v2.16b, v6.16b
st1 {v2.16b}, [x1], #16
.Lctr_crypt_end:
/* store new CTR */
rev x7, x7
rev x8, x8
stp x7, x8, [x3]
ret
SYM_FUNC_END(sm4_neon_ctr_crypt)
linux-7.2.3/arch/arm64/crypto/sm4-neon-glue.c
+248
View File
@@ -0,0 +1,248 @@
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
* SM4 Cipher Algorithm, using ARMv8 NEON
* as specified in
* https://tools.ietf.org/id/draft-ribose-cfrg-sm4-10.html
*
* Copyright (C) 2022, Alibaba Group.
* Copyright (C) 2022 Tianjia Zhang <tianjia.zhang@linux.alibaba.com>
*/
#include <linux/module.h>
#include <linux/crypto.h>
#include <linux/kernel.h>
#include <linux/cpufeature.h>
#include <asm/neon.h>
#include <asm/simd.h>
#include <crypto/internal/simd.h>
#include <crypto/internal/skcipher.h>
#include <crypto/sm4.h>
asmlinkage void sm4_neon_crypt(const u32 *rkey, u8 *dst, const u8 *src,
unsigned int nblocks);
asmlinkage void sm4_neon_cbc_dec(const u32 *rkey_dec, u8 *dst, const u8 *src,
u8 *iv, unsigned int nblocks);
asmlinkage void sm4_neon_ctr_crypt(const u32 *rkey_enc, u8 *dst, const u8 *src,
u8 *iv, unsigned int nblocks);
static int sm4_setkey(struct crypto_skcipher *tfm, const u8 *key,
unsigned int key_len)
{
struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm);
return sm4_expandkey(ctx, key, key_len);
}
static int sm4_ecb_do_crypt(struct skcipher_request *req, const u32 *rkey)
{
struct skcipher_walk walk;
unsigned int nbytes;
int err;
err = skcipher_walk_virt(&walk, req, false);
while ((nbytes = walk.nbytes) > 0) {
const u8 *src = walk.src.virt.addr;
u8 *dst = walk.dst.virt.addr;
unsigned int nblocks;
nblocks = nbytes / SM4_BLOCK_SIZE;
if (nblocks) {
scoped_ksimd()
sm4_neon_crypt(rkey, dst, src, nblocks);
}
err = skcipher_walk_done(&walk, nbytes % SM4_BLOCK_SIZE);
}
return err;
}
static int sm4_ecb_encrypt(struct skcipher_request *req)
{
struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm);
return sm4_ecb_do_crypt(req, ctx->rkey_enc);
}
static int sm4_ecb_decrypt(struct skcipher_request *req)
{
struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm);
return sm4_ecb_do_crypt(req, ctx->rkey_dec);
}
static int sm4_cbc_encrypt(struct skcipher_request *req)
{
struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm);
struct skcipher_walk walk;
unsigned int nbytes;
int err;
err = skcipher_walk_virt(&walk, req, false);
while ((nbytes = walk.nbytes) > 0) {
const u8 *iv = walk.iv;
const u8 *src = walk.src.virt.addr;
u8 *dst = walk.dst.virt.addr;
while (nbytes >= SM4_BLOCK_SIZE) {
crypto_xor_cpy(dst, src, iv, SM4_BLOCK_SIZE);
sm4_crypt_block(ctx->rkey_enc, dst, dst);
iv = dst;
src += SM4_BLOCK_SIZE;
dst += SM4_BLOCK_SIZE;
nbytes -= SM4_BLOCK_SIZE;
}
if (iv != walk.iv)
memcpy(walk.iv, iv, SM4_BLOCK_SIZE);
err = skcipher_walk_done(&walk, nbytes);
}
return err;
}
static int sm4_cbc_decrypt(struct skcipher_request *req)
{
struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm);
struct skcipher_walk walk;
unsigned int nbytes;
int err;
err = skcipher_walk_virt(&walk, req, false);
while ((nbytes = walk.nbytes) > 0) {
const u8 *src = walk.src.virt.addr;
u8 *dst = walk.dst.virt.addr;
unsigned int nblocks;
nblocks = nbytes / SM4_BLOCK_SIZE;
if (nblocks) {
scoped_ksimd()
sm4_neon_cbc_dec(ctx->rkey_dec, dst, src,
walk.iv, nblocks);
}
err = skcipher_walk_done(&walk, nbytes % SM4_BLOCK_SIZE);
}
return err;
}
static int sm4_ctr_crypt(struct skcipher_request *req)
{
struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm);
struct skcipher_walk walk;
unsigned int nbytes;
int err;
err = skcipher_walk_virt(&walk, req, false);
while ((nbytes = walk.nbytes) > 0) {
const u8 *src = walk.src.virt.addr;
u8 *dst = walk.dst.virt.addr;
unsigned int nblocks;
nblocks = nbytes / SM4_BLOCK_SIZE;
if (nblocks) {
scoped_ksimd()
sm4_neon_ctr_crypt(ctx->rkey_enc, dst, src,
walk.iv, nblocks);
dst += nblocks * SM4_BLOCK_SIZE;
src += nblocks * SM4_BLOCK_SIZE;
nbytes -= nblocks * SM4_BLOCK_SIZE;
}
/* tail */
if (walk.nbytes == walk.total && nbytes > 0) {
u8 keystream[SM4_BLOCK_SIZE];
sm4_crypt_block(ctx->rkey_enc, keystream, walk.iv);
crypto_inc(walk.iv, SM4_BLOCK_SIZE);
crypto_xor_cpy(dst, src, keystream, nbytes);
nbytes = 0;
}
err = skcipher_walk_done(&walk, nbytes);
}
return err;
}
static struct skcipher_alg sm4_algs[] = {
{
.base = {
.cra_name = "ecb(sm4)",
.cra_driver_name = "ecb-sm4-neon",
.cra_priority = 200,
.cra_blocksize = SM4_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct sm4_ctx),
.cra_module = THIS_MODULE,
},
.min_keysize = SM4_KEY_SIZE,
.max_keysize = SM4_KEY_SIZE,
.setkey = sm4_setkey,
.encrypt = sm4_ecb_encrypt,
.decrypt = sm4_ecb_decrypt,
}, {
.base = {
.cra_name = "cbc(sm4)",
.cra_driver_name = "cbc-sm4-neon",
.cra_priority = 200,
.cra_blocksize = SM4_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct sm4_ctx),
.cra_module = THIS_MODULE,
},
.min_keysize = SM4_KEY_SIZE,
.max_keysize = SM4_KEY_SIZE,
.ivsize = SM4_BLOCK_SIZE,
.setkey = sm4_setkey,
.encrypt = sm4_cbc_encrypt,
.decrypt = sm4_cbc_decrypt,
}, {
.base = {
.cra_name = "ctr(sm4)",
.cra_driver_name = "ctr-sm4-neon",
.cra_priority = 200,
.cra_blocksize = 1,
.cra_ctxsize = sizeof(struct sm4_ctx),
.cra_module = THIS_MODULE,
},
.min_keysize = SM4_KEY_SIZE,
.max_keysize = SM4_KEY_SIZE,
.ivsize = SM4_BLOCK_SIZE,
.chunksize = SM4_BLOCK_SIZE,
.setkey = sm4_setkey,
.encrypt = sm4_ctr_crypt,
.decrypt = sm4_ctr_crypt,
}
};
static int __init sm4_init(void)
{
return crypto_register_skciphers(sm4_algs, ARRAY_SIZE(sm4_algs));
}
static void __exit sm4_exit(void)
{
crypto_unregister_skciphers(sm4_algs, ARRAY_SIZE(sm4_algs));
}
module_init(sm4_init);
module_exit(sm4_exit);
MODULE_DESCRIPTION("SM4 ECB/CBC/CTR using ARMv8 NEON");
MODULE_ALIAS_CRYPTO("sm4-neon");
MODULE_ALIAS_CRYPTO("sm4");
MODULE_ALIAS_CRYPTO("ecb(sm4)");
MODULE_ALIAS_CRYPTO("cbc(sm4)");
MODULE_ALIAS_CRYPTO("ctr(sm4)");
MODULE_AUTHOR("Tianjia Zhang <tianjia.zhang@linux.alibaba.com>");
MODULE_LICENSE("GPL v2");