lastdragon/linux-kernel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Resguardo inicial

Browse Source
This commit is contained in:
lastdragon
2026-09-06 12:53:21 -06:00
parent 662efe3c2a
commit ad2088d178
94853 changed files with 43200404 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
linux-7.2.3/lib/crypto/arm64/.gitignore
+4
View File
@@ -0,0 +1,4 @@
# SPDX-License-Identifier: GPL-2.0-only
poly1305-core.S
sha256-core.S
sha512-core.S
linux-7.2.3/lib/crypto/arm64/aes-ce-core.S
+84
View File
@@ -0,0 +1,84 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (C) 2013 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
*/
#include <linux/linkage.h>
#include <asm/assembler.h>
.arch armv8-a+crypto
SYM_FUNC_START(__aes_ce_encrypt)
sub w3, w3, #2
ld1 {v0.16b}, [x2]
ld1 {v1.4s}, [x0], #16
cmp w3, #10
bmi 0f
bne 3f
mov v3.16b, v1.16b
b 2f
0: mov v2.16b, v1.16b
ld1 {v3.4s}, [x0], #16
1: aese v0.16b, v2.16b
aesmc v0.16b, v0.16b
2: ld1 {v1.4s}, [x0], #16
aese v0.16b, v3.16b
aesmc v0.16b, v0.16b
3: ld1 {v2.4s}, [x0], #16
subs w3, w3, #3
aese v0.16b, v1.16b
aesmc v0.16b, v0.16b
ld1 {v3.4s}, [x0], #16
bpl 1b
aese v0.16b, v2.16b
eor v0.16b, v0.16b, v3.16b
st1 {v0.16b}, [x1]
ret
SYM_FUNC_END(__aes_ce_encrypt)
SYM_FUNC_START(__aes_ce_decrypt)
sub w3, w3, #2
ld1 {v0.16b}, [x2]
ld1 {v1.4s}, [x0], #16
cmp w3, #10
bmi 0f
bne 3f
mov v3.16b, v1.16b
b 2f
0: mov v2.16b, v1.16b
ld1 {v3.4s}, [x0], #16
1: aesd v0.16b, v2.16b
aesimc v0.16b, v0.16b
2: ld1 {v1.4s}, [x0], #16
aesd v0.16b, v3.16b
aesimc v0.16b, v0.16b
3: ld1 {v2.4s}, [x0], #16
subs w3, w3, #3
aesd v0.16b, v1.16b
aesimc v0.16b, v0.16b
ld1 {v3.4s}, [x0], #16
bpl 1b
aesd v0.16b, v2.16b
eor v0.16b, v0.16b, v3.16b
st1 {v0.16b}, [x1]
ret
SYM_FUNC_END(__aes_ce_decrypt)
/*
* __aes_ce_sub() - use the aese instruction to perform the AES sbox
* substitution on each byte in 'input'
*/
SYM_FUNC_START(__aes_ce_sub)
dup v1.4s, w0
movi v0.16b, #0
aese v0.16b, v1.16b
umov w0, v0.s[0]
ret
SYM_FUNC_END(__aes_ce_sub)
SYM_FUNC_START(__aes_ce_invert)
ld1 {v0.4s}, [x1]
aesimc v1.16b, v0.16b
st1 {v1.4s}, [x0]
ret
SYM_FUNC_END(__aes_ce_invert)
linux-7.2.3/lib/crypto/arm64/aes-ce.S
+145
View File
@@ -0,0 +1,145 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* AES cipher for ARMv8 with Crypto Extensions
*
* Copyright (C) 2013 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
*/
#include <linux/linkage.h>
#include <asm/assembler.h>
#define AES_FUNC_START(func) SYM_FUNC_START(ce_ ## func)
#define AES_FUNC_END(func) SYM_FUNC_END(ce_ ## func)
.arch armv8-a+crypto
xtsmask .req v16
cbciv .req v16
vctr .req v16
.macro xts_reload_mask, tmp
.endm
.macro xts_cts_skip_tw, reg, lbl
.endm
/* preload all round keys */
.macro load_round_keys, rk, nr, tmp
add \tmp, \rk, \nr, sxtw #4
sub \tmp, \tmp, #160
ld1 {v17.4s-v20.4s}, [\rk]
ld1 {v21.4s-v24.4s}, [\tmp], #64
ld1 {v25.4s-v28.4s}, [\tmp], #64
ld1 {v29.4s-v31.4s}, [\tmp]
.endm
/* prepare for encryption with key in rk[] */
.macro enc_prepare, rounds, rk, temp
load_round_keys \rk, \rounds, \temp
.endm
/* prepare for encryption (again) but with new key in rk[] */
.macro enc_switch_key, rounds, rk, temp
load_round_keys \rk, \rounds, \temp
.endm
/* prepare for decryption with key in rk[] */
.macro dec_prepare, rounds, rk, temp
load_round_keys \rk, \rounds, \temp
.endm
.macro do_enc_Nx, de, mc, k, i0, i1, i2, i3, i4
aes\de \i0\().16b, \k\().16b
aes\mc \i0\().16b, \i0\().16b
.ifnb \i1
aes\de \i1\().16b, \k\().16b
aes\mc \i1\().16b, \i1\().16b
.ifnb \i3
aes\de \i2\().16b, \k\().16b
aes\mc \i2\().16b, \i2\().16b
aes\de \i3\().16b, \k\().16b
aes\mc \i3\().16b, \i3\().16b
.ifnb \i4
aes\de \i4\().16b, \k\().16b
aes\mc \i4\().16b, \i4\().16b
.endif
.endif
.endif
.endm
/* up to 5 interleaved encryption rounds with the same round key */
.macro round_Nx, enc, k, i0, i1, i2, i3, i4
.ifc \enc, e
do_enc_Nx e, mc, \k, \i0, \i1, \i2, \i3, \i4
.else
do_enc_Nx d, imc, \k, \i0, \i1, \i2, \i3, \i4
.endif
.endm
/* up to 5 interleaved final rounds */
.macro fin_round_Nx, de, k, k2, i0, i1, i2, i3, i4
aes\de \i0\().16b, \k\().16b
.ifnb \i1
aes\de \i1\().16b, \k\().16b
.ifnb \i3
aes\de \i2\().16b, \k\().16b
aes\de \i3\().16b, \k\().16b
.ifnb \i4
aes\de \i4\().16b, \k\().16b
.endif
.endif
.endif
eor \i0\().16b, \i0\().16b, \k2\().16b
.ifnb \i1
eor \i1\().16b, \i1\().16b, \k2\().16b
.ifnb \i3
eor \i2\().16b, \i2\().16b, \k2\().16b
eor \i3\().16b, \i3\().16b, \k2\().16b
.ifnb \i4
eor \i4\().16b, \i4\().16b, \k2\().16b
.endif
.endif
.endif
.endm
/* up to 5 interleaved blocks */
.macro do_block_Nx, enc, rounds, i0, i1, i2, i3, i4
tbz \rounds, #2, .L\@ /* 128 bits */
round_Nx \enc, v17, \i0, \i1, \i2, \i3, \i4
round_Nx \enc, v18, \i0, \i1, \i2, \i3, \i4
tbz \rounds, #1, .L\@ /* 192 bits */
round_Nx \enc, v19, \i0, \i1, \i2, \i3, \i4
round_Nx \enc, v20, \i0, \i1, \i2, \i3, \i4
.L\@: .irp key, v21, v22, v23, v24, v25, v26, v27, v28, v29
round_Nx \enc, \key, \i0, \i1, \i2, \i3, \i4
.endr
fin_round_Nx \enc, v30, v31, \i0, \i1, \i2, \i3, \i4
.endm
.macro encrypt_block, in, rounds, t0, t1, t2
do_block_Nx e, \rounds, \in
.endm
.macro encrypt_block4x, i0, i1, i2, i3, rounds, t0, t1, t2
do_block_Nx e, \rounds, \i0, \i1, \i2, \i3
.endm
.macro encrypt_block5x, i0, i1, i2, i3, i4, rounds, t0, t1, t2
do_block_Nx e, \rounds, \i0, \i1, \i2, \i3, \i4
.endm
.macro decrypt_block, in, rounds, t0, t1, t2
do_block_Nx d, \rounds, \in
.endm
.macro decrypt_block4x, i0, i1, i2, i3, rounds, t0, t1, t2
do_block_Nx d, \rounds, \i0, \i1, \i2, \i3
.endm
.macro decrypt_block5x, i0, i1, i2, i3, i4, rounds, t0, t1, t2
do_block_Nx d, \rounds, \i0, \i1, \i2, \i3, \i4
.endm
#define MAX_STRIDE 5
#include "aes-modes.S"
linux-7.2.3/lib/crypto/arm64/aes-cipher-core.S
+122
View File
@@ -0,0 +1,122 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Scalar AES core transform
*
* Copyright (C) 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
*/
#include <linux/linkage.h>
#include <asm/assembler.h>
#include <asm/cache.h>
.text
rk .req x0
out .req x1
in .req x2
rounds .req x3
tt .req x2
.macro __pair1, sz, op, reg0, reg1, in0, in1e, in1d, shift
.ifc \op\shift, b0
ubfiz \reg0, \in0, #2, #8
ubfiz \reg1, \in1e, #2, #8
.else
ubfx \reg0, \in0, #\shift, #8
ubfx \reg1, \in1e, #\shift, #8
.endif
/*
* AArch64 cannot do byte size indexed loads from a table containing
* 32-bit quantities, i.e., 'ldrb w12, [tt, w12, uxtw #2]' is not a
* valid instruction. So perform the shift explicitly first for the
* high bytes (the low byte is shifted implicitly by using ubfiz rather
* than ubfx above)
*/
.ifnc \op, b
ldr \reg0, [tt, \reg0, uxtw #2]
ldr \reg1, [tt, \reg1, uxtw #2]
.else
.if \shift > 0
lsl \reg0, \reg0, #2
lsl \reg1, \reg1, #2
.endif
ldrb \reg0, [tt, \reg0, uxtw]
ldrb \reg1, [tt, \reg1, uxtw]
.endif
.endm
.macro __pair0, sz, op, reg0, reg1, in0, in1e, in1d, shift
ubfx \reg0, \in0, #\shift, #8
ubfx \reg1, \in1d, #\shift, #8
ldr\op \reg0, [tt, \reg0, uxtw #\sz]
ldr\op \reg1, [tt, \reg1, uxtw #\sz]
.endm
.macro __hround, out0, out1, in0, in1, in2, in3, t0, t1, enc, sz, op
ldp \out0, \out1, [rk], #8
__pair\enc \sz, \op, w12, w13, \in0, \in1, \in3, 0
__pair\enc \sz, \op, w14, w15, \in1, \in2, \in0, 8
__pair\enc \sz, \op, w16, w17, \in2, \in3, \in1, 16
__pair\enc \sz, \op, \t0, \t1, \in3, \in0, \in2, 24
eor \out0, \out0, w12
eor \out1, \out1, w13
eor \out0, \out0, w14, ror #24
eor \out1, \out1, w15, ror #24
eor \out0, \out0, w16, ror #16
eor \out1, \out1, w17, ror #16
eor \out0, \out0, \t0, ror #8
eor \out1, \out1, \t1, ror #8
.endm
.macro fround, out0, out1, out2, out3, in0, in1, in2, in3, sz=2, op
__hround \out0, \out1, \in0, \in1, \in2, \in3, \out2, \out3, 1, \sz, \op
__hround \out2, \out3, \in2, \in3, \in0, \in1, \in1, \in2, 1, \sz, \op
.endm
.macro iround, out0, out1, out2, out3, in0, in1, in2, in3, sz=2, op
__hround \out0, \out1, \in0, \in3, \in2, \in1, \out2, \out3, 0, \sz, \op
__hround \out2, \out3, \in2, \in1, \in0, \in3, \in1, \in0, 0, \sz, \op
.endm
.macro do_crypt, round, ttab, ltab, bsz
ldp w4, w5, [in]
ldp w6, w7, [in, #8]
ldp w8, w9, [rk], #16
ldp w10, w11, [rk, #-8]
eor w4, w4, w8
eor w5, w5, w9
eor w6, w6, w10
eor w7, w7, w11
adr_l tt, \ttab
tbnz rounds, #1, 1f
0: \round w8, w9, w10, w11, w4, w5, w6, w7
\round w4, w5, w6, w7, w8, w9, w10, w11
1: subs rounds, rounds, #4
\round w8, w9, w10, w11, w4, w5, w6, w7
b.ls 3f
2: \round w4, w5, w6, w7, w8, w9, w10, w11
b 0b
3: adr_l tt, \ltab
\round w4, w5, w6, w7, w8, w9, w10, w11, \bsz, b
stp w4, w5, [out]
stp w6, w7, [out, #8]
ret
.endm
SYM_FUNC_START(__aes_arm64_encrypt)
do_crypt fround, aes_enc_tab, aes_enc_tab + 1, 2
SYM_FUNC_END(__aes_arm64_encrypt)
.align 5
SYM_FUNC_START(__aes_arm64_decrypt)
do_crypt iround, aes_dec_tab, crypto_aes_inv_sbox, 0
SYM_FUNC_END(__aes_arm64_decrypt)
linux-7.2.3/lib/crypto/arm64/aes-modes.S
+867
View File
@@ -0,0 +1,867 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Chaining mode wrappers for AES
*
* Copyright (C) 2013 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
*/
/* included by aes-ce.S and aes-neon.S */
.text
.align 4
#ifndef MAX_STRIDE
#define MAX_STRIDE 4
#endif
#if MAX_STRIDE == 4
#define ST4(x...) x
#define ST5(x...)
#else
#define ST4(x...)
#define ST5(x...) x
#endif
SYM_FUNC_START_LOCAL(aes_encrypt_block4x)
encrypt_block4x v0, v1, v2, v3, w3, x2, x8, w7
ret
SYM_FUNC_END(aes_encrypt_block4x)
SYM_FUNC_START_LOCAL(aes_decrypt_block4x)
decrypt_block4x v0, v1, v2, v3, w3, x2, x8, w7
ret
SYM_FUNC_END(aes_decrypt_block4x)
#if MAX_STRIDE == 5
SYM_FUNC_START_LOCAL(aes_encrypt_block5x)
encrypt_block5x v0, v1, v2, v3, v4, w3, x2, x8, w7
ret
SYM_FUNC_END(aes_encrypt_block5x)
SYM_FUNC_START_LOCAL(aes_decrypt_block5x)
decrypt_block5x v0, v1, v2, v3, v4, w3, x2, x8, w7
ret
SYM_FUNC_END(aes_decrypt_block5x)
#endif
/*
* aes_ecb_encrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
* int blocks)
* aes_ecb_decrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
* int blocks)
*/
AES_FUNC_START(aes_ecb_encrypt)
frame_push 0
enc_prepare w3, x2, x5
.LecbencloopNx:
subs w4, w4, #MAX_STRIDE
bmi .Lecbenc1x
ld1 {v0.16b-v3.16b}, [x1], #64 /* get 4 pt blocks */
ST4( bl aes_encrypt_block4x )
ST5( ld1 {v4.16b}, [x1], #16 )
ST5( bl aes_encrypt_block5x )
st1 {v0.16b-v3.16b}, [x0], #64
ST5( st1 {v4.16b}, [x0], #16 )
b .LecbencloopNx
.Lecbenc1x:
adds w4, w4, #MAX_STRIDE
beq .Lecbencout
.Lecbencloop:
ld1 {v0.16b}, [x1], #16 /* get next pt block */
encrypt_block v0, w3, x2, x5, w6
st1 {v0.16b}, [x0], #16
subs w4, w4, #1
bne .Lecbencloop
.Lecbencout:
frame_pop
ret
AES_FUNC_END(aes_ecb_encrypt)
AES_FUNC_START(aes_ecb_decrypt)
frame_push 0
dec_prepare w3, x2, x5
.LecbdecloopNx:
subs w4, w4, #MAX_STRIDE
bmi .Lecbdec1x
ld1 {v0.16b-v3.16b}, [x1], #64 /* get 4 ct blocks */
ST4( bl aes_decrypt_block4x )
ST5( ld1 {v4.16b}, [x1], #16 )
ST5( bl aes_decrypt_block5x )
st1 {v0.16b-v3.16b}, [x0], #64
ST5( st1 {v4.16b}, [x0], #16 )
b .LecbdecloopNx
.Lecbdec1x:
adds w4, w4, #MAX_STRIDE
beq .Lecbdecout
.Lecbdecloop:
ld1 {v0.16b}, [x1], #16 /* get next ct block */
decrypt_block v0, w3, x2, x5, w6
st1 {v0.16b}, [x0], #16
subs w4, w4, #1
bne .Lecbdecloop
.Lecbdecout:
frame_pop
ret
AES_FUNC_END(aes_ecb_decrypt)
/*
* aes_cbc_encrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
* int blocks, u8 iv[])
* aes_cbc_decrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
* int blocks, u8 iv[])
* aes_essiv_cbc_encrypt(u8 out[], u8 const in[], u32 const rk1[],
* int rounds, int blocks, u8 iv[],
* u32 const rk2[]);
* aes_essiv_cbc_decrypt(u8 out[], u8 const in[], u32 const rk1[],
* int rounds, int blocks, u8 iv[],
* u32 const rk2[]);
*/
AES_FUNC_START(aes_essiv_cbc_encrypt)
ld1 {v4.16b}, [x5] /* get iv */
mov w8, #14 /* AES-256: 14 rounds */
enc_prepare w8, x6, x7
encrypt_block v4, w8, x6, x7, w9
enc_switch_key w3, x2, x6
b .Lcbcencloop4x
AES_FUNC_START(aes_cbc_encrypt)
ld1 {v4.16b}, [x5] /* get iv */
enc_prepare w3, x2, x6
.Lcbcencloop4x:
subs w4, w4, #4
bmi .Lcbcenc1x
ld1 {v0.16b-v3.16b}, [x1], #64 /* get 4 pt blocks */
eor v0.16b, v0.16b, v4.16b /* ..and xor with iv */
encrypt_block v0, w3, x2, x6, w7
eor v1.16b, v1.16b, v0.16b
encrypt_block v1, w3, x2, x6, w7
eor v2.16b, v2.16b, v1.16b
encrypt_block v2, w3, x2, x6, w7
eor v3.16b, v3.16b, v2.16b
encrypt_block v3, w3, x2, x6, w7
st1 {v0.16b-v3.16b}, [x0], #64
mov v4.16b, v3.16b
b .Lcbcencloop4x
.Lcbcenc1x:
adds w4, w4, #4
beq .Lcbcencout
.Lcbcencloop:
ld1 {v0.16b}, [x1], #16 /* get next pt block */
eor v4.16b, v4.16b, v0.16b /* ..and xor with iv */
encrypt_block v4, w3, x2, x6, w7
st1 {v4.16b}, [x0], #16
subs w4, w4, #1
bne .Lcbcencloop
.Lcbcencout:
st1 {v4.16b}, [x5] /* return iv */
ret
AES_FUNC_END(aes_cbc_encrypt)
AES_FUNC_END(aes_essiv_cbc_encrypt)
AES_FUNC_START(aes_essiv_cbc_decrypt)
ld1 {cbciv.16b}, [x5] /* get iv */
mov w8, #14 /* AES-256: 14 rounds */
enc_prepare w8, x6, x7
encrypt_block cbciv, w8, x6, x7, w9
b .Lessivcbcdecstart
AES_FUNC_START(aes_cbc_decrypt)
ld1 {cbciv.16b}, [x5] /* get iv */
.Lessivcbcdecstart:
frame_push 0
dec_prepare w3, x2, x6
.LcbcdecloopNx:
subs w4, w4, #MAX_STRIDE
bmi .Lcbcdec1x
ld1 {v0.16b-v3.16b}, [x1], #64 /* get 4 ct blocks */
#if MAX_STRIDE == 5
ld1 {v4.16b}, [x1], #16 /* get 1 ct block */
mov v5.16b, v0.16b
mov v6.16b, v1.16b
mov v7.16b, v2.16b
bl aes_decrypt_block5x
sub x1, x1, #32
eor v0.16b, v0.16b, cbciv.16b
eor v1.16b, v1.16b, v5.16b
ld1 {v5.16b}, [x1], #16 /* reload 1 ct block */
ld1 {cbciv.16b}, [x1], #16 /* reload 1 ct block */
eor v2.16b, v2.16b, v6.16b
eor v3.16b, v3.16b, v7.16b
eor v4.16b, v4.16b, v5.16b
#else
mov v4.16b, v0.16b
mov v5.16b, v1.16b
mov v6.16b, v2.16b
bl aes_decrypt_block4x
sub x1, x1, #16
eor v0.16b, v0.16b, cbciv.16b
eor v1.16b, v1.16b, v4.16b
ld1 {cbciv.16b}, [x1], #16 /* reload 1 ct block */
eor v2.16b, v2.16b, v5.16b
eor v3.16b, v3.16b, v6.16b
#endif
st1 {v0.16b-v3.16b}, [x0], #64
ST5( st1 {v4.16b}, [x0], #16 )
b .LcbcdecloopNx
.Lcbcdec1x:
adds w4, w4, #MAX_STRIDE
beq .Lcbcdecout
.Lcbcdecloop:
ld1 {v1.16b}, [x1], #16 /* get next ct block */
mov v0.16b, v1.16b /* ...and copy to v0 */
decrypt_block v0, w3, x2, x6, w7
eor v0.16b, v0.16b, cbciv.16b /* xor with iv => pt */
mov cbciv.16b, v1.16b /* ct is next iv */
st1 {v0.16b}, [x0], #16
subs w4, w4, #1
bne .Lcbcdecloop
.Lcbcdecout:
st1 {cbciv.16b}, [x5] /* return iv */
frame_pop
ret
AES_FUNC_END(aes_cbc_decrypt)
AES_FUNC_END(aes_essiv_cbc_decrypt)
/*
* aes_cbc_cts_encrypt(u8 out[], u8 const in[], u32 const rk[],
* int rounds, int bytes, u8 const iv[])
* aes_cbc_cts_decrypt(u8 out[], u8 const in[], u32 const rk[],
* int rounds, int bytes, u8 const iv[])
*/
AES_FUNC_START(aes_cbc_cts_encrypt)
adr_l x8, .Lcts_permute_table
sub x4, x4, #16
add x9, x8, #32
add x8, x8, x4
sub x9, x9, x4
ld1 {v3.16b}, [x8]
ld1 {v4.16b}, [x9]
ld1 {v0.16b}, [x1], x4 /* overlapping loads */
ld1 {v1.16b}, [x1]
ld1 {v5.16b}, [x5] /* get iv */
enc_prepare w3, x2, x6
eor v0.16b, v0.16b, v5.16b /* xor with iv */
tbl v1.16b, {v1.16b}, v4.16b
encrypt_block v0, w3, x2, x6, w7
eor v1.16b, v1.16b, v0.16b
tbl v0.16b, {v0.16b}, v3.16b
encrypt_block v1, w3, x2, x6, w7
add x4, x0, x4
st1 {v0.16b}, [x4] /* overlapping stores */
st1 {v1.16b}, [x0]
ret
AES_FUNC_END(aes_cbc_cts_encrypt)
AES_FUNC_START(aes_cbc_cts_decrypt)
adr_l x8, .Lcts_permute_table
sub x4, x4, #16
add x9, x8, #32
add x8, x8, x4
sub x9, x9, x4
ld1 {v3.16b}, [x8]
ld1 {v4.16b}, [x9]
ld1 {v0.16b}, [x1], x4 /* overlapping loads */
ld1 {v1.16b}, [x1]
ld1 {v5.16b}, [x5] /* get iv */
dec_prepare w3, x2, x6
decrypt_block v0, w3, x2, x6, w7
tbl v2.16b, {v0.16b}, v3.16b
eor v2.16b, v2.16b, v1.16b
tbx v0.16b, {v1.16b}, v4.16b
decrypt_block v0, w3, x2, x6, w7
eor v0.16b, v0.16b, v5.16b /* xor with iv */
add x4, x0, x4
st1 {v2.16b}, [x4] /* overlapping stores */
st1 {v0.16b}, [x0]
ret
AES_FUNC_END(aes_cbc_cts_decrypt)
.section ".rodata", "a"
.align 6
.Lcts_permute_table:
.byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
.byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
.byte 0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7
.byte 0x8, 0x9, 0xa, 0xb, 0xc, 0xd, 0xe, 0xf
.byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
.byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
.previous
/*
* This macro generates the code for CTR and XCTR mode.
*/
.macro ctr_encrypt xctr
// Arguments
OUT .req x0
IN .req x1
KEY .req x2
ROUNDS_W .req w3
BYTES_W .req w4
IV .req x5
BYTE_CTR_W .req w6 // XCTR only
// Intermediate values
CTR_W .req w11 // XCTR only
CTR .req x11 // XCTR only
IV_PART .req x12
BLOCKS .req x13
BLOCKS_W .req w13
frame_push 0
enc_prepare ROUNDS_W, KEY, IV_PART
ld1 {vctr.16b}, [IV]
/*
* Keep 64 bits of the IV in a register. For CTR mode this lets us
* easily increment the IV. For XCTR mode this lets us efficiently XOR
* the 64-bit counter with the IV.
*/
.if \xctr
umov IV_PART, vctr.d[0]
lsr CTR_W, BYTE_CTR_W, #4
.else
umov IV_PART, vctr.d[1]
rev IV_PART, IV_PART
.endif
.LctrloopNx\xctr:
add BLOCKS_W, BYTES_W, #15
sub BYTES_W, BYTES_W, #MAX_STRIDE << 4
lsr BLOCKS_W, BLOCKS_W, #4
mov w8, #MAX_STRIDE
cmp BLOCKS_W, w8
csel BLOCKS_W, BLOCKS_W, w8, lt
/*
* Set up the counter values in v0-v{MAX_STRIDE-1}.
*
* If we are encrypting less than MAX_STRIDE blocks, the tail block
* handling code expects the last keystream block to be in
* v{MAX_STRIDE-1}. For example: if encrypting two blocks with
* MAX_STRIDE=5, then v3 and v4 should have the next two counter blocks.
*/
.if \xctr
add CTR, CTR, BLOCKS
.else
adds IV_PART, IV_PART, BLOCKS
.endif
mov v0.16b, vctr.16b
mov v1.16b, vctr.16b
mov v2.16b, vctr.16b
mov v3.16b, vctr.16b
ST5( mov v4.16b, vctr.16b )
.if \xctr
sub x6, CTR, #MAX_STRIDE - 1
sub x7, CTR, #MAX_STRIDE - 2
sub x8, CTR, #MAX_STRIDE - 3
sub x9, CTR, #MAX_STRIDE - 4
ST5( sub x10, CTR, #MAX_STRIDE - 5 )
eor x6, x6, IV_PART
eor x7, x7, IV_PART
eor x8, x8, IV_PART
eor x9, x9, IV_PART
ST5( eor x10, x10, IV_PART )
mov v0.d[0], x6
mov v1.d[0], x7
mov v2.d[0], x8
mov v3.d[0], x9
ST5( mov v4.d[0], x10 )
.else
bcs 0f
.subsection 1
/*
* This subsection handles carries.
*
* Conditional branching here is allowed with respect to time
* invariance since the branches are dependent on the IV instead
* of the plaintext or key. This code is rarely executed in
* practice anyway.
*/
/* Apply carry to outgoing counter. */
0: umov x8, vctr.d[0]
rev x8, x8
add x8, x8, #1
rev x8, x8
ins vctr.d[0], x8
/*
* Apply carry to counter blocks if needed.
*
* Since the carry flag was set, we know 0 <= IV_PART <
* MAX_STRIDE. Using the value of IV_PART we can determine how
* many counter blocks need to be updated.
*/
cbz IV_PART, 2f
adr x16, 1f
sub x16, x16, IV_PART, lsl #3
br x16
bti c
mov v0.d[0], vctr.d[0]
bti c
mov v1.d[0], vctr.d[0]
bti c
mov v2.d[0], vctr.d[0]
bti c
mov v3.d[0], vctr.d[0]
ST5( bti c )
ST5( mov v4.d[0], vctr.d[0] )
1: b 2f
.previous
2: rev x7, IV_PART
ins vctr.d[1], x7
sub x7, IV_PART, #MAX_STRIDE - 1
sub x8, IV_PART, #MAX_STRIDE - 2
sub x9, IV_PART, #MAX_STRIDE - 3
rev x7, x7
rev x8, x8
mov v1.d[1], x7
rev x9, x9
ST5( sub x10, IV_PART, #MAX_STRIDE - 4 )
mov v2.d[1], x8
ST5( rev x10, x10 )
mov v3.d[1], x9
ST5( mov v4.d[1], x10 )
.endif
/*
* If there are at least MAX_STRIDE blocks left, XOR the data with
* keystream and store. Otherwise jump to tail handling.
*/
tbnz BYTES_W, #31, .Lctrtail\xctr
ld1 {v5.16b-v7.16b}, [IN], #48
ST4( bl aes_encrypt_block4x )
ST5( bl aes_encrypt_block5x )
eor v0.16b, v5.16b, v0.16b
ST4( ld1 {v5.16b}, [IN], #16 )
eor v1.16b, v6.16b, v1.16b
ST5( ld1 {v5.16b-v6.16b}, [IN], #32 )
eor v2.16b, v7.16b, v2.16b
eor v3.16b, v5.16b, v3.16b
ST5( eor v4.16b, v6.16b, v4.16b )
st1 {v0.16b-v3.16b}, [OUT], #64
ST5( st1 {v4.16b}, [OUT], #16 )
cbz BYTES_W, .Lctrout\xctr
b .LctrloopNx\xctr
.Lctrout\xctr:
.if !\xctr
st1 {vctr.16b}, [IV] /* return next CTR value */
.endif
frame_pop
ret
.Lctrtail\xctr:
/*
* Handle up to MAX_STRIDE * 16 - 1 bytes of plaintext
*
* This code expects the last keystream block to be in v{MAX_STRIDE-1}.
* For example: if encrypting two blocks with MAX_STRIDE=5, then v3 and
* v4 should have the next two counter blocks.
*
* This allows us to store the ciphertext by writing to overlapping
* regions of memory. Any invalid ciphertext blocks get overwritten by
* correctly computed blocks. This approach greatly simplifies the
* logic for storing the ciphertext.
*/
mov x16, #16
ands w7, BYTES_W, #0xf
csel x13, x7, x16, ne
ST5( cmp BYTES_W, #64 - (MAX_STRIDE << 4))
ST5( csel x14, x16, xzr, gt )
cmp BYTES_W, #48 - (MAX_STRIDE << 4)
csel x15, x16, xzr, gt
cmp BYTES_W, #32 - (MAX_STRIDE << 4)
csel x16, x16, xzr, gt
cmp BYTES_W, #16 - (MAX_STRIDE << 4)
adr_l x9, .Lcts_permute_table
add x9, x9, x13
ble .Lctrtail1x\xctr
ST5( ld1 {v5.16b}, [IN], x14 )
ld1 {v6.16b}, [IN], x15
ld1 {v7.16b}, [IN], x16
ST4( bl aes_encrypt_block4x )
ST5( bl aes_encrypt_block5x )
ld1 {v8.16b}, [IN], x13
ld1 {v9.16b}, [IN]
ld1 {v10.16b}, [x9]
ST4( eor v6.16b, v6.16b, v0.16b )
ST4( eor v7.16b, v7.16b, v1.16b )
ST4( tbl v3.16b, {v3.16b}, v10.16b )
ST4( eor v8.16b, v8.16b, v2.16b )
ST4( eor v9.16b, v9.16b, v3.16b )
ST5( eor v5.16b, v5.16b, v0.16b )
ST5( eor v6.16b, v6.16b, v1.16b )
ST5( tbl v4.16b, {v4.16b}, v10.16b )
ST5( eor v7.16b, v7.16b, v2.16b )
ST5( eor v8.16b, v8.16b, v3.16b )
ST5( eor v9.16b, v9.16b, v4.16b )
ST5( st1 {v5.16b}, [OUT], x14 )
st1 {v6.16b}, [OUT], x15
st1 {v7.16b}, [OUT], x16
add x13, x13, OUT
st1 {v9.16b}, [x13] // overlapping stores
st1 {v8.16b}, [OUT]
b .Lctrout\xctr
.Lctrtail1x\xctr:
/*
* Handle <= 16 bytes of plaintext
*
* This code always reads and writes 16 bytes. To avoid out of bounds
* accesses, XCTR and CTR modes must use a temporary buffer when
* encrypting/decrypting less than 16 bytes.
*
* This code is unusual in that it loads the input and stores the output
* relative to the end of the buffers rather than relative to the start.
* This causes unusual behaviour when encrypting/decrypting less than 16
* bytes; the end of the data is expected to be at the end of the
* temporary buffer rather than the start of the data being at the start
* of the temporary buffer.
*/
sub x8, x7, #16
csel x7, x7, x8, eq
add IN, IN, x7
add OUT, OUT, x7
ld1 {v5.16b}, [IN]
ld1 {v6.16b}, [OUT]
ST5( mov v3.16b, v4.16b )
encrypt_block v3, ROUNDS_W, KEY, x8, w7
ld1 {v10.16b-v11.16b}, [x9]
tbl v3.16b, {v3.16b}, v10.16b
sshr v11.16b, v11.16b, #7
eor v5.16b, v5.16b, v3.16b
bif v5.16b, v6.16b, v11.16b
st1 {v5.16b}, [OUT]
b .Lctrout\xctr
// Arguments
.unreq OUT
.unreq IN
.unreq KEY
.unreq ROUNDS_W
.unreq BYTES_W
.unreq IV
.unreq BYTE_CTR_W // XCTR only
// Intermediate values
.unreq CTR_W // XCTR only
.unreq CTR // XCTR only
.unreq IV_PART
.unreq BLOCKS
.unreq BLOCKS_W
.endm
/*
* aes_ctr_encrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
* int bytes, u8 ctr[])
*
* The input and output buffers must always be at least 16 bytes even if
* encrypting/decrypting less than 16 bytes. Otherwise out of bounds
* accesses will occur. The data to be encrypted/decrypted is expected
* to be at the end of this 16-byte temporary buffer rather than the
* start.
*/
AES_FUNC_START(aes_ctr_encrypt)
ctr_encrypt 0
AES_FUNC_END(aes_ctr_encrypt)
/*
* aes_xctr_encrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
* int bytes, u8 const iv[], int byte_ctr)
*
* The input and output buffers must always be at least 16 bytes even if
* encrypting/decrypting less than 16 bytes. Otherwise out of bounds
* accesses will occur. The data to be encrypted/decrypted is expected
* to be at the end of this 16-byte temporary buffer rather than the
* start.
*/
AES_FUNC_START(aes_xctr_encrypt)
ctr_encrypt 1
AES_FUNC_END(aes_xctr_encrypt)
/*
* aes_xts_encrypt(u8 out[], u8 const in[], u8 const rk1[], int rounds,
* int bytes, u8 const rk2[], u8 iv[], int first)
* aes_xts_decrypt(u8 out[], u8 const in[], u8 const rk1[], int rounds,
* int bytes, u8 const rk2[], u8 iv[], int first)
*/
.macro next_tweak, out, in, tmp
sshr \tmp\().2d, \in\().2d, #63
and \tmp\().16b, \tmp\().16b, xtsmask.16b
add \out\().2d, \in\().2d, \in\().2d
ext \tmp\().16b, \tmp\().16b, \tmp\().16b, #8
eor \out\().16b, \out\().16b, \tmp\().16b
.endm
.macro xts_load_mask, tmp
movi xtsmask.2s, #0x1
movi \tmp\().2s, #0x87
uzp1 xtsmask.4s, xtsmask.4s, \tmp\().4s
.endm
AES_FUNC_START(aes_xts_encrypt)
frame_push 0
ld1 {v4.16b}, [x6]
xts_load_mask v8
cbz w7, .Lxtsencnotfirst
enc_prepare w3, x5, x8
xts_cts_skip_tw w7, .LxtsencNx
encrypt_block v4, w3, x5, x8, w7 /* first tweak */
enc_switch_key w3, x2, x8
b .LxtsencNx
.Lxtsencnotfirst:
enc_prepare w3, x2, x8
.LxtsencloopNx:
next_tweak v4, v4, v8
.LxtsencNx:
subs w4, w4, #64
bmi .Lxtsenc1x
ld1 {v0.16b-v3.16b}, [x1], #64 /* get 4 pt blocks */
next_tweak v5, v4, v8
eor v0.16b, v0.16b, v4.16b
next_tweak v6, v5, v8
eor v1.16b, v1.16b, v5.16b
eor v2.16b, v2.16b, v6.16b
next_tweak v7, v6, v8
eor v3.16b, v3.16b, v7.16b
bl aes_encrypt_block4x
eor v3.16b, v3.16b, v7.16b
eor v0.16b, v0.16b, v4.16b
eor v1.16b, v1.16b, v5.16b
eor v2.16b, v2.16b, v6.16b
st1 {v0.16b-v3.16b}, [x0], #64
mov v4.16b, v7.16b
cbz w4, .Lxtsencret
xts_reload_mask v8
b .LxtsencloopNx
.Lxtsenc1x:
adds w4, w4, #64
beq .Lxtsencout
subs w4, w4, #16
bmi .LxtsencctsNx
.Lxtsencloop:
ld1 {v0.16b}, [x1], #16
.Lxtsencctsout:
eor v0.16b, v0.16b, v4.16b
encrypt_block v0, w3, x2, x8, w7
eor v0.16b, v0.16b, v4.16b
cbz w4, .Lxtsencout
subs w4, w4, #16
next_tweak v4, v4, v8
bmi .Lxtsenccts
st1 {v0.16b}, [x0], #16
b .Lxtsencloop
.Lxtsencout:
st1 {v0.16b}, [x0]
.Lxtsencret:
st1 {v4.16b}, [x6]
frame_pop
ret
.LxtsencctsNx:
mov v0.16b, v3.16b
sub x0, x0, #16
.Lxtsenccts:
adr_l x8, .Lcts_permute_table
add x1, x1, w4, sxtw /* rewind input pointer */
add w4, w4, #16 /* # bytes in final block */
add x9, x8, #32
add x8, x8, x4
sub x9, x9, x4
add x4, x0, x4 /* output address of final block */
ld1 {v1.16b}, [x1] /* load final block */
ld1 {v2.16b}, [x8]
ld1 {v3.16b}, [x9]
tbl v2.16b, {v0.16b}, v2.16b
tbx v0.16b, {v1.16b}, v3.16b
st1 {v2.16b}, [x4] /* overlapping stores */
mov w4, wzr
b .Lxtsencctsout
AES_FUNC_END(aes_xts_encrypt)
AES_FUNC_START(aes_xts_decrypt)
frame_push 0
/* subtract 16 bytes if we are doing CTS */
sub w8, w4, #0x10
tst w4, #0xf
csel w4, w4, w8, eq
ld1 {v4.16b}, [x6]
xts_load_mask v8
xts_cts_skip_tw w7, .Lxtsdecskiptw
cbz w7, .Lxtsdecnotfirst
enc_prepare w3, x5, x8
encrypt_block v4, w3, x5, x8, w7 /* first tweak */
.Lxtsdecskiptw:
dec_prepare w3, x2, x8
b .LxtsdecNx
.Lxtsdecnotfirst:
dec_prepare w3, x2, x8
.LxtsdecloopNx:
next_tweak v4, v4, v8
.LxtsdecNx:
subs w4, w4, #64
bmi .Lxtsdec1x
ld1 {v0.16b-v3.16b}, [x1], #64 /* get 4 ct blocks */
next_tweak v5, v4, v8
eor v0.16b, v0.16b, v4.16b
next_tweak v6, v5, v8
eor v1.16b, v1.16b, v5.16b
eor v2.16b, v2.16b, v6.16b
next_tweak v7, v6, v8
eor v3.16b, v3.16b, v7.16b
bl aes_decrypt_block4x
eor v3.16b, v3.16b, v7.16b
eor v0.16b, v0.16b, v4.16b
eor v1.16b, v1.16b, v5.16b
eor v2.16b, v2.16b, v6.16b
st1 {v0.16b-v3.16b}, [x0], #64
mov v4.16b, v7.16b
cbz w4, .Lxtsdecout
xts_reload_mask v8
b .LxtsdecloopNx
.Lxtsdec1x:
adds w4, w4, #64
beq .Lxtsdecout
subs w4, w4, #16
.Lxtsdecloop:
ld1 {v0.16b}, [x1], #16
bmi .Lxtsdeccts
.Lxtsdecctsout:
eor v0.16b, v0.16b, v4.16b
decrypt_block v0, w3, x2, x8, w7
eor v0.16b, v0.16b, v4.16b
st1 {v0.16b}, [x0], #16
cbz w4, .Lxtsdecout
subs w4, w4, #16
next_tweak v4, v4, v8
b .Lxtsdecloop
.Lxtsdecout:
st1 {v4.16b}, [x6]
frame_pop
ret
.Lxtsdeccts:
adr_l x8, .Lcts_permute_table
add x1, x1, w4, sxtw /* rewind input pointer */
add w4, w4, #16 /* # bytes in final block */
add x9, x8, #32
add x8, x8, x4
sub x9, x9, x4
add x4, x0, x4 /* output address of final block */
next_tweak v5, v4, v8
ld1 {v1.16b}, [x1] /* load final block */
ld1 {v2.16b}, [x8]
ld1 {v3.16b}, [x9]
eor v0.16b, v0.16b, v5.16b
decrypt_block v0, w3, x2, x8, w7
eor v0.16b, v0.16b, v5.16b
tbl v2.16b, {v0.16b}, v2.16b
tbx v0.16b, {v1.16b}, v3.16b
st1 {v2.16b}, [x4] /* overlapping stores */
mov w4, wzr
b .Lxtsdecctsout
AES_FUNC_END(aes_xts_decrypt)
#if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CBC_MACS)
/*
* void aes_mac_update(u8 const in[], u32 const rk[], int rounds,
* size_t blocks, u8 dg[], int enc_before,
* int enc_after);
*/
AES_FUNC_START(aes_mac_update)
ld1 {v0.16b}, [x4] /* get dg */
enc_prepare w2, x1, x7
cbz w5, .Lmacloop4x
encrypt_block v0, w2, x1, x7, w8
.Lmacloop4x:
subs x3, x3, #4
bmi .Lmac1x
ld1 {v1.16b-v4.16b}, [x0], #64 /* get next pt block */
eor v0.16b, v0.16b, v1.16b /* ..and xor with dg */
encrypt_block v0, w2, x1, x7, w8
eor v0.16b, v0.16b, v2.16b
encrypt_block v0, w2, x1, x7, w8
eor v0.16b, v0.16b, v3.16b
encrypt_block v0, w2, x1, x7, w8
eor v0.16b, v0.16b, v4.16b
cmp x3, xzr
csinv w5, w6, wzr, eq
cbz w5, .Lmacout
encrypt_block v0, w2, x1, x7, w8
st1 {v0.16b}, [x4] /* return dg */
b .Lmacloop4x
.Lmac1x:
add x3, x3, #4
.Lmacloop:
cbz x3, .Lmacout
ld1 {v1.16b}, [x0], #16 /* get next pt block */
eor v0.16b, v0.16b, v1.16b /* ..and xor with dg */
subs x3, x3, #1
csinv w5, w6, wzr, eq
cbz w5, .Lmacout
.Lmacenc:
encrypt_block v0, w2, x1, x7, w8
b .Lmacloop
.Lmacout:
st1 {v0.16b}, [x4] /* return dg */
ret
AES_FUNC_END(aes_mac_update)
#endif /* CONFIG_CRYPTO_LIB_AES_CBC_MACS */
linux-7.2.3/lib/crypto/arm64/aes-neon.S
+250
View File
@@ -0,0 +1,250 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* AES cipher for ARMv8 NEON
*
* Copyright (C) 2013 - 2017 Linaro Ltd. <ard.biesheuvel@linaro.org>
*/
#include <linux/linkage.h>
#include <asm/assembler.h>
#define AES_FUNC_START(func) SYM_FUNC_START(neon_ ## func)
#define AES_FUNC_END(func) SYM_FUNC_END(neon_ ## func)
xtsmask .req v7
cbciv .req v7
vctr .req v4
.macro xts_reload_mask, tmp
xts_load_mask \tmp
.endm
/* special case for the neon-bs driver calling into this one for CTS */
.macro xts_cts_skip_tw, reg, lbl
tbnz \reg, #1, \lbl
.endm
/* multiply by polynomial 'x' in GF(2^8) */
.macro mul_by_x, out, in, temp, const
sshr \temp, \in, #7
shl \out, \in, #1
and \temp, \temp, \const
eor \out, \out, \temp
.endm
/* multiply by polynomial 'x^2' in GF(2^8) */
.macro mul_by_x2, out, in, temp, const
ushr \temp, \in, #6
shl \out, \in, #2
pmul \temp, \temp, \const
eor \out, \out, \temp
.endm
/* preload the entire Sbox */
.macro prepare, sbox, shiftrows, temp
movi v12.16b, #0x1b
ldr_l q13, \shiftrows, \temp
ldr_l q14, .Lror32by8, \temp
adr_l \temp, \sbox
ld1 {v16.16b-v19.16b}, [\temp], #64
ld1 {v20.16b-v23.16b}, [\temp], #64
ld1 {v24.16b-v27.16b}, [\temp], #64
ld1 {v28.16b-v31.16b}, [\temp]
.endm
/* do preload for encryption */
.macro enc_prepare, ignore0, ignore1, temp
prepare crypto_aes_sbox, .LForward_ShiftRows, \temp
.endm
.macro enc_switch_key, ignore0, ignore1, temp
/* do nothing */
.endm
/* do preload for decryption */
.macro dec_prepare, ignore0, ignore1, temp
prepare crypto_aes_inv_sbox, .LReverse_ShiftRows, \temp
.endm
/* apply SubBytes transformation using the preloaded Sbox */
.macro sub_bytes, in
sub v9.16b, \in\().16b, v15.16b
tbl \in\().16b, {v16.16b-v19.16b}, \in\().16b
sub v10.16b, v9.16b, v15.16b
tbx \in\().16b, {v20.16b-v23.16b}, v9.16b
sub v11.16b, v10.16b, v15.16b
tbx \in\().16b, {v24.16b-v27.16b}, v10.16b
tbx \in\().16b, {v28.16b-v31.16b}, v11.16b
.endm
/* apply MixColumns transformation */
.macro mix_columns, in, enc
.if \enc == 0
/* Inverse MixColumns: pre-multiply by { 5, 0, 4, 0 } */
mul_by_x2 v8.16b, \in\().16b, v9.16b, v12.16b
eor \in\().16b, \in\().16b, v8.16b
rev32 v8.8h, v8.8h
eor \in\().16b, \in\().16b, v8.16b
.endif
mul_by_x v9.16b, \in\().16b, v8.16b, v12.16b
rev32 v8.8h, \in\().8h
eor v8.16b, v8.16b, v9.16b
eor \in\().16b, \in\().16b, v8.16b
tbl \in\().16b, {\in\().16b}, v14.16b
eor \in\().16b, \in\().16b, v8.16b
.endm
.macro do_block, enc, in, rounds, rk, rkp, i
ld1 {v15.4s}, [\rk]
add \rkp, \rk, #16
mov \i, \rounds
.La\@: eor \in\().16b, \in\().16b, v15.16b /* ^round key */
movi v15.16b, #0x40
tbl \in\().16b, {\in\().16b}, v13.16b /* ShiftRows */
sub_bytes \in
sub \i, \i, #1
ld1 {v15.4s}, [\rkp], #16
cbz \i, .Lb\@
mix_columns \in, \enc
b .La\@
.Lb\@: eor \in\().16b, \in\().16b, v15.16b /* ^round key */
.endm
.macro encrypt_block, in, rounds, rk, rkp, i
do_block 1, \in, \rounds, \rk, \rkp, \i
.endm
.macro decrypt_block, in, rounds, rk, rkp, i
do_block 0, \in, \rounds, \rk, \rkp, \i
.endm
/*
* Interleaved versions: functionally equivalent to the
* ones above, but applied to AES states in parallel.
*/
.macro sub_bytes_4x, in0, in1, in2, in3
sub v8.16b, \in0\().16b, v15.16b
tbl \in0\().16b, {v16.16b-v19.16b}, \in0\().16b
sub v9.16b, \in1\().16b, v15.16b
tbl \in1\().16b, {v16.16b-v19.16b}, \in1\().16b
sub v10.16b, \in2\().16b, v15.16b
tbl \in2\().16b, {v16.16b-v19.16b}, \in2\().16b
sub v11.16b, \in3\().16b, v15.16b
tbl \in3\().16b, {v16.16b-v19.16b}, \in3\().16b
tbx \in0\().16b, {v20.16b-v23.16b}, v8.16b
tbx \in1\().16b, {v20.16b-v23.16b}, v9.16b
sub v8.16b, v8.16b, v15.16b
tbx \in2\().16b, {v20.16b-v23.16b}, v10.16b
sub v9.16b, v9.16b, v15.16b
tbx \in3\().16b, {v20.16b-v23.16b}, v11.16b
sub v10.16b, v10.16b, v15.16b
tbx \in0\().16b, {v24.16b-v27.16b}, v8.16b
sub v11.16b, v11.16b, v15.16b
tbx \in1\().16b, {v24.16b-v27.16b}, v9.16b
sub v8.16b, v8.16b, v15.16b
tbx \in2\().16b, {v24.16b-v27.16b}, v10.16b
sub v9.16b, v9.16b, v15.16b
tbx \in3\().16b, {v24.16b-v27.16b}, v11.16b
sub v10.16b, v10.16b, v15.16b
tbx \in0\().16b, {v28.16b-v31.16b}, v8.16b
sub v11.16b, v11.16b, v15.16b
tbx \in1\().16b, {v28.16b-v31.16b}, v9.16b
tbx \in2\().16b, {v28.16b-v31.16b}, v10.16b
tbx \in3\().16b, {v28.16b-v31.16b}, v11.16b
.endm
.macro mul_by_x_2x, out0, out1, in0, in1, tmp0, tmp1, const
sshr \tmp0\().16b, \in0\().16b, #7
shl \out0\().16b, \in0\().16b, #1
sshr \tmp1\().16b, \in1\().16b, #7
and \tmp0\().16b, \tmp0\().16b, \const\().16b
shl \out1\().16b, \in1\().16b, #1
and \tmp1\().16b, \tmp1\().16b, \const\().16b
eor \out0\().16b, \out0\().16b, \tmp0\().16b
eor \out1\().16b, \out1\().16b, \tmp1\().16b
.endm
.macro mul_by_x2_2x, out0, out1, in0, in1, tmp0, tmp1, const
ushr \tmp0\().16b, \in0\().16b, #6
shl \out0\().16b, \in0\().16b, #2
ushr \tmp1\().16b, \in1\().16b, #6
pmul \tmp0\().16b, \tmp0\().16b, \const\().16b
shl \out1\().16b, \in1\().16b, #2
pmul \tmp1\().16b, \tmp1\().16b, \const\().16b
eor \out0\().16b, \out0\().16b, \tmp0\().16b
eor \out1\().16b, \out1\().16b, \tmp1\().16b
.endm
.macro mix_columns_2x, in0, in1, enc
.if \enc == 0
/* Inverse MixColumns: pre-multiply by { 5, 0, 4, 0 } */
mul_by_x2_2x v8, v9, \in0, \in1, v10, v11, v12
eor \in0\().16b, \in0\().16b, v8.16b
rev32 v8.8h, v8.8h
eor \in1\().16b, \in1\().16b, v9.16b
rev32 v9.8h, v9.8h
eor \in0\().16b, \in0\().16b, v8.16b
eor \in1\().16b, \in1\().16b, v9.16b
.endif
mul_by_x_2x v8, v9, \in0, \in1, v10, v11, v12
rev32 v10.8h, \in0\().8h
rev32 v11.8h, \in1\().8h
eor v10.16b, v10.16b, v8.16b
eor v11.16b, v11.16b, v9.16b
eor \in0\().16b, \in0\().16b, v10.16b
eor \in1\().16b, \in1\().16b, v11.16b
tbl \in0\().16b, {\in0\().16b}, v14.16b
tbl \in1\().16b, {\in1\().16b}, v14.16b
eor \in0\().16b, \in0\().16b, v10.16b
eor \in1\().16b, \in1\().16b, v11.16b
.endm
.macro do_block_4x, enc, in0, in1, in2, in3, rounds, rk, rkp, i
ld1 {v15.4s}, [\rk]
add \rkp, \rk, #16
mov \i, \rounds
.La\@: eor \in0\().16b, \in0\().16b, v15.16b /* ^round key */
eor \in1\().16b, \in1\().16b, v15.16b /* ^round key */
eor \in2\().16b, \in2\().16b, v15.16b /* ^round key */
eor \in3\().16b, \in3\().16b, v15.16b /* ^round key */
movi v15.16b, #0x40
tbl \in0\().16b, {\in0\().16b}, v13.16b /* ShiftRows */
tbl \in1\().16b, {\in1\().16b}, v13.16b /* ShiftRows */
tbl \in2\().16b, {\in2\().16b}, v13.16b /* ShiftRows */
tbl \in3\().16b, {\in3\().16b}, v13.16b /* ShiftRows */
sub_bytes_4x \in0, \in1, \in2, \in3
sub \i, \i, #1
ld1 {v15.4s}, [\rkp], #16
cbz \i, .Lb\@
mix_columns_2x \in0, \in1, \enc
mix_columns_2x \in2, \in3, \enc
b .La\@
.Lb\@: eor \in0\().16b, \in0\().16b, v15.16b /* ^round key */
eor \in1\().16b, \in1\().16b, v15.16b /* ^round key */
eor \in2\().16b, \in2\().16b, v15.16b /* ^round key */
eor \in3\().16b, \in3\().16b, v15.16b /* ^round key */
.endm
.macro encrypt_block4x, in0, in1, in2, in3, rounds, rk, rkp, i
do_block_4x 1, \in0, \in1, \in2, \in3, \rounds, \rk, \rkp, \i
.endm
.macro decrypt_block4x, in0, in1, in2, in3, rounds, rk, rkp, i
do_block_4x 0, \in0, \in1, \in2, \in3, \rounds, \rk, \rkp, \i
.endm
#include "aes-modes.S"
.section ".rodata", "a"
.align 4
.LForward_ShiftRows:
.octa 0x0b06010c07020d08030e09040f0a0500
.LReverse_ShiftRows:
.octa 0x0306090c0f0205080b0e0104070a0d00
.Lror32by8:
.octa 0x0c0f0e0d080b0a090407060500030201
linux-7.2.3/lib/crypto/arm64/aes.h
+219
View File
@@ -0,0 +1,219 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* AES block cipher, optimized for ARM64
*
* Copyright (C) 2013 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
* Copyright 2026 Google LLC
*/
#include <asm/neon.h>
#include <asm/simd.h>
#include <linux/unaligned.h>
#include <linux/cpufeature.h>
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon);
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_aes);
struct aes_block {
u8 b[AES_BLOCK_SIZE];
};
asmlinkage void __aes_arm64_encrypt(const u32 rk[], u8 out[AES_BLOCK_SIZE],
const u8 in[AES_BLOCK_SIZE], int rounds);
asmlinkage void __aes_arm64_decrypt(const u32 inv_rk[], u8 out[AES_BLOCK_SIZE],
const u8 in[AES_BLOCK_SIZE], int rounds);
asmlinkage void __aes_ce_encrypt(const u32 rk[], u8 out[AES_BLOCK_SIZE],
const u8 in[AES_BLOCK_SIZE], int rounds);
asmlinkage void __aes_ce_decrypt(const u32 inv_rk[], u8 out[AES_BLOCK_SIZE],
const u8 in[AES_BLOCK_SIZE], int rounds);
asmlinkage u32 __aes_ce_sub(u32 l);
asmlinkage void __aes_ce_invert(struct aes_block *out,
const struct aes_block *in);
asmlinkage void neon_aes_mac_update(u8 const in[], u32 const rk[], int rounds,
size_t blocks, u8 dg[], int enc_before,
int enc_after);
/*
* Expand an AES key using the crypto extensions if supported and usable or
* generic code otherwise. The expanded key format is compatible between the
* two cases. The outputs are @rndkeys (required) and @inv_rndkeys (optional).
*/
static void aes_expandkey_arm64(u32 rndkeys[], u32 *inv_rndkeys,
const u8 *in_key, int key_len, int nrounds)
{
/*
* The AES key schedule round constants
*/
static u8 const rcon[] = {
0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1b, 0x36,
};
u32 kwords = key_len / sizeof(u32);
struct aes_block *key_enc, *key_dec;
int i, j;
if (!static_branch_likely(&have_aes) || unlikely(!may_use_simd())) {
aes_expandkey_generic(rndkeys, inv_rndkeys, in_key, key_len);
return;
}
for (i = 0; i < kwords; i++)
rndkeys[i] = get_unaligned_le32(&in_key[i * sizeof(u32)]);
scoped_ksimd() {
for (i = 0; i < sizeof(rcon); i++) {
u32 *rki = &rndkeys[i * kwords];
u32 *rko = rki + kwords;
rko[0] = ror32(__aes_ce_sub(rki[kwords - 1]), 8) ^
rcon[i] ^ rki[0];
rko[1] = rko[0] ^ rki[1];
rko[2] = rko[1] ^ rki[2];
rko[3] = rko[2] ^ rki[3];
if (key_len == AES_KEYSIZE_192) {
if (i >= 7)
break;
rko[4] = rko[3] ^ rki[4];
rko[5] = rko[4] ^ rki[5];
} else if (key_len == AES_KEYSIZE_256) {
if (i >= 6)
break;
rko[4] = __aes_ce_sub(rko[3]) ^ rki[4];
rko[5] = rko[4] ^ rki[5];
rko[6] = rko[5] ^ rki[6];
rko[7] = rko[6] ^ rki[7];
}
}
/*
* Generate the decryption keys for the Equivalent Inverse
* Cipher. This involves reversing the order of the round
* keys, and applying the Inverse Mix Columns transformation on
* all but the first and the last one.
*/
if (inv_rndkeys) {
key_enc = (struct aes_block *)rndkeys;
key_dec = (struct aes_block *)inv_rndkeys;
j = nrounds;
key_dec[0] = key_enc[j];
for (i = 1, j--; j > 0; i++, j--)
__aes_ce_invert(key_dec + i, key_enc + j);
key_dec[i] = key_enc[0];
}
}
}
static void aes_preparekey_arch(union aes_enckey_arch *k,
union aes_invkey_arch *inv_k,
const u8 *in_key, int key_len, int nrounds)
{
aes_expandkey_arm64(k->rndkeys, inv_k ? inv_k->inv_rndkeys : NULL,
in_key, key_len, nrounds);
}
/*
* This is here temporarily until the remaining AES mode implementations are
* migrated from arch/arm64/crypto/ to lib/crypto/arm64/.
*/
int ce_aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key,
unsigned int key_len)
{
if (aes_check_keylen(key_len) != 0)
return -EINVAL;
ctx->key_length = key_len;
aes_expandkey_arm64(ctx->key_enc, ctx->key_dec, in_key, key_len,
6 + key_len / 4);
return 0;
}
EXPORT_SYMBOL(ce_aes_expandkey);
EXPORT_SYMBOL_NS_GPL(neon_aes_ecb_encrypt, "CRYPTO_INTERNAL");
EXPORT_SYMBOL_NS_GPL(neon_aes_ecb_decrypt, "CRYPTO_INTERNAL");
EXPORT_SYMBOL_NS_GPL(neon_aes_cbc_encrypt, "CRYPTO_INTERNAL");
EXPORT_SYMBOL_NS_GPL(neon_aes_cbc_decrypt, "CRYPTO_INTERNAL");
EXPORT_SYMBOL_NS_GPL(neon_aes_cbc_cts_encrypt, "CRYPTO_INTERNAL");
EXPORT_SYMBOL_NS_GPL(neon_aes_cbc_cts_decrypt, "CRYPTO_INTERNAL");
EXPORT_SYMBOL_NS_GPL(neon_aes_ctr_encrypt, "CRYPTO_INTERNAL");
EXPORT_SYMBOL_NS_GPL(neon_aes_xctr_encrypt, "CRYPTO_INTERNAL");
EXPORT_SYMBOL_NS_GPL(neon_aes_xts_encrypt, "CRYPTO_INTERNAL");
EXPORT_SYMBOL_NS_GPL(neon_aes_xts_decrypt, "CRYPTO_INTERNAL");
EXPORT_SYMBOL_NS_GPL(neon_aes_essiv_cbc_encrypt, "CRYPTO_INTERNAL");
EXPORT_SYMBOL_NS_GPL(neon_aes_essiv_cbc_decrypt, "CRYPTO_INTERNAL");
EXPORT_SYMBOL_NS_GPL(ce_aes_ecb_encrypt, "CRYPTO_INTERNAL");
EXPORT_SYMBOL_NS_GPL(ce_aes_ecb_decrypt, "CRYPTO_INTERNAL");
EXPORT_SYMBOL_NS_GPL(ce_aes_cbc_encrypt, "CRYPTO_INTERNAL");
EXPORT_SYMBOL_NS_GPL(ce_aes_cbc_decrypt, "CRYPTO_INTERNAL");
EXPORT_SYMBOL_NS_GPL(ce_aes_cbc_cts_encrypt, "CRYPTO_INTERNAL");
EXPORT_SYMBOL_NS_GPL(ce_aes_cbc_cts_decrypt, "CRYPTO_INTERNAL");
EXPORT_SYMBOL_NS_GPL(ce_aes_ctr_encrypt, "CRYPTO_INTERNAL");
EXPORT_SYMBOL_NS_GPL(ce_aes_xctr_encrypt, "CRYPTO_INTERNAL");
EXPORT_SYMBOL_NS_GPL(ce_aes_xts_encrypt, "CRYPTO_INTERNAL");
EXPORT_SYMBOL_NS_GPL(ce_aes_xts_decrypt, "CRYPTO_INTERNAL");
EXPORT_SYMBOL_NS_GPL(ce_aes_essiv_cbc_encrypt, "CRYPTO_INTERNAL");
EXPORT_SYMBOL_NS_GPL(ce_aes_essiv_cbc_decrypt, "CRYPTO_INTERNAL");
#if IS_MODULE(CONFIG_CRYPTO_AES_ARM64_CE_CCM)
EXPORT_SYMBOL_NS_GPL(ce_aes_mac_update, "CRYPTO_INTERNAL");
#endif
static void aes_encrypt_arch(const struct aes_enckey *key,
u8 out[AES_BLOCK_SIZE],
const u8 in[AES_BLOCK_SIZE])
{
if (static_branch_likely(&have_aes) && likely(may_use_simd())) {
scoped_ksimd()
__aes_ce_encrypt(key->k.rndkeys, out, in, key->nrounds);
} else {
__aes_arm64_encrypt(key->k.rndkeys, out, in, key->nrounds);
}
}
static void aes_decrypt_arch(const struct aes_key *key,
u8 out[AES_BLOCK_SIZE],
const u8 in[AES_BLOCK_SIZE])
{
if (static_branch_likely(&have_aes) && likely(may_use_simd())) {
scoped_ksimd()
__aes_ce_decrypt(key->inv_k.inv_rndkeys, out, in,
key->nrounds);
} else {
__aes_arm64_decrypt(key->inv_k.inv_rndkeys, out, in,
key->nrounds);
}
}
#if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CBC_MACS)
#define aes_cbcmac_blocks_arch aes_cbcmac_blocks_arch
static bool aes_cbcmac_blocks_arch(u8 h[AES_BLOCK_SIZE],
const struct aes_enckey *key, const u8 *data,
size_t nblocks, bool enc_before,
bool enc_after)
{
if (static_branch_likely(&have_neon) && likely(may_use_simd())) {
scoped_ksimd() {
if (static_branch_likely(&have_aes))
ce_aes_mac_update(data, key->k.rndkeys,
key->nrounds, nblocks, h,
enc_before, enc_after);
else
neon_aes_mac_update(data, key->k.rndkeys,
key->nrounds, nblocks, h,
enc_before, enc_after);
}
return true;
}
return false;
}
#endif /* CONFIG_CRYPTO_LIB_AES_CBC_MACS */
#define aes_mod_init_arch aes_mod_init_arch
static void aes_mod_init_arch(void)
{
if (cpu_have_named_feature(ASIMD)) {
static_branch_enable(&have_neon);
if (cpu_have_named_feature(AES))
static_branch_enable(&have_aes);
}
}
linux-7.2.3/lib/crypto/arm64/chacha-neon-core.S
+789
View File
@@ -0,0 +1,789 @@
/*
* ChaCha/HChaCha NEON helper functions
*
* Copyright (C) 2016-2018 Linaro, Ltd. <ard.biesheuvel@linaro.org>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation.
*
* Originally based on:
* ChaCha20 256-bit cipher algorithm, RFC7539, x64 SSSE3 functions
*
* Copyright (C) 2015 Martin Willi
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*/
#include <linux/linkage.h>
#include <asm/assembler.h>
#include <asm/cache.h>
.text
.align 6
/*
* chacha_permute - permute one block
*
* Permute one 64-byte block where the state matrix is stored in the four NEON
* registers v0-v3. It performs matrix operations on four words in parallel,
* but requires shuffling to rearrange the words after each round.
*
* The round count is given in w3.
*
* Clobbers: w3, x10, v4, v12
*/
SYM_FUNC_START_LOCAL(chacha_permute)
adr_l x10, ROT8
ld1 {v12.4s}, [x10]
.Ldoubleround:
// x0 += x1, x3 = rotl32(x3 ^ x0, 16)
add v0.4s, v0.4s, v1.4s
eor v3.16b, v3.16b, v0.16b
rev32 v3.8h, v3.8h
// x2 += x3, x1 = rotl32(x1 ^ x2, 12)
add v2.4s, v2.4s, v3.4s
eor v4.16b, v1.16b, v2.16b
shl v1.4s, v4.4s, #12
sri v1.4s, v4.4s, #20
// x0 += x1, x3 = rotl32(x3 ^ x0, 8)
add v0.4s, v0.4s, v1.4s
eor v3.16b, v3.16b, v0.16b
tbl v3.16b, {v3.16b}, v12.16b
// x2 += x3, x1 = rotl32(x1 ^ x2, 7)
add v2.4s, v2.4s, v3.4s
eor v4.16b, v1.16b, v2.16b
shl v1.4s, v4.4s, #7
sri v1.4s, v4.4s, #25
// x1 = shuffle32(x1, MASK(0, 3, 2, 1))
ext v1.16b, v1.16b, v1.16b, #4
// x2 = shuffle32(x2, MASK(1, 0, 3, 2))
ext v2.16b, v2.16b, v2.16b, #8
// x3 = shuffle32(x3, MASK(2, 1, 0, 3))
ext v3.16b, v3.16b, v3.16b, #12
// x0 += x1, x3 = rotl32(x3 ^ x0, 16)
add v0.4s, v0.4s, v1.4s
eor v3.16b, v3.16b, v0.16b
rev32 v3.8h, v3.8h
// x2 += x3, x1 = rotl32(x1 ^ x2, 12)
add v2.4s, v2.4s, v3.4s
eor v4.16b, v1.16b, v2.16b
shl v1.4s, v4.4s, #12
sri v1.4s, v4.4s, #20
// x0 += x1, x3 = rotl32(x3 ^ x0, 8)
add v0.4s, v0.4s, v1.4s
eor v3.16b, v3.16b, v0.16b
tbl v3.16b, {v3.16b}, v12.16b
// x2 += x3, x1 = rotl32(x1 ^ x2, 7)
add v2.4s, v2.4s, v3.4s
eor v4.16b, v1.16b, v2.16b
shl v1.4s, v4.4s, #7
sri v1.4s, v4.4s, #25
// x1 = shuffle32(x1, MASK(2, 1, 0, 3))
ext v1.16b, v1.16b, v1.16b, #12
// x2 = shuffle32(x2, MASK(1, 0, 3, 2))
ext v2.16b, v2.16b, v2.16b, #8
// x3 = shuffle32(x3, MASK(0, 3, 2, 1))
ext v3.16b, v3.16b, v3.16b, #4
subs w3, w3, #2
b.ne .Ldoubleround
ret
SYM_FUNC_END(chacha_permute)
SYM_FUNC_START(chacha_block_xor_neon)
// x0: Input state matrix, s
// x1: 1 data block output, o
// x2: 1 data block input, i
// w3: nrounds
stp x29, x30, [sp, #-16]!
mov x29, sp
// x0..3 = s0..3
ld1 {v0.4s-v3.4s}, [x0]
ld1 {v8.4s-v11.4s}, [x0]
bl chacha_permute
ld1 {v4.16b-v7.16b}, [x2]
// o0 = i0 ^ (x0 + s0)
add v0.4s, v0.4s, v8.4s
eor v0.16b, v0.16b, v4.16b
// o1 = i1 ^ (x1 + s1)
add v1.4s, v1.4s, v9.4s
eor v1.16b, v1.16b, v5.16b
// o2 = i2 ^ (x2 + s2)
add v2.4s, v2.4s, v10.4s
eor v2.16b, v2.16b, v6.16b
// o3 = i3 ^ (x3 + s3)
add v3.4s, v3.4s, v11.4s
eor v3.16b, v3.16b, v7.16b
st1 {v0.16b-v3.16b}, [x1]
ldp x29, x30, [sp], #16
ret
SYM_FUNC_END(chacha_block_xor_neon)
SYM_FUNC_START(hchacha_block_neon)
// x0: Input state matrix, s
// x1: output (8 32-bit words)
// w2: nrounds
stp x29, x30, [sp, #-16]!
mov x29, sp
ld1 {v0.4s-v3.4s}, [x0]
mov w3, w2
bl chacha_permute
st1 {v0.4s}, [x1], #16
st1 {v3.4s}, [x1]
ldp x29, x30, [sp], #16
ret
SYM_FUNC_END(hchacha_block_neon)
a0 .req w12
a1 .req w13
a2 .req w14
a3 .req w15
a4 .req w16
a5 .req w17
a6 .req w19
a7 .req w20
a8 .req w21
a9 .req w22
a10 .req w23
a11 .req w24
a12 .req w25
a13 .req w26
a14 .req w27
a15 .req w28
.align 6
SYM_FUNC_START(chacha_4block_xor_neon)
frame_push 10
// x0: Input state matrix, s
// x1: 4 data blocks output, o
// x2: 4 data blocks input, i
// w3: nrounds
// x4: byte count
adr_l x10, .Lpermute
and x5, x4, #63
add x10, x10, x5
//
// This function encrypts four consecutive ChaCha blocks by loading
// the state matrix in NEON registers four times. The algorithm performs
// each operation on the corresponding word of each state matrix, hence
// requires no word shuffling. For final XORing step we transpose the
// matrix by interleaving 32- and then 64-bit words, which allows us to
// do XOR in NEON registers.
//
// At the same time, a fifth block is encrypted in parallel using
// scalar registers
//
adr_l x9, CTRINC // ... and ROT8
ld1 {v30.4s-v31.4s}, [x9]
// x0..15[0-3] = s0..3[0..3]
add x8, x0, #16
ld4r { v0.4s- v3.4s}, [x0]
ld4r { v4.4s- v7.4s}, [x8], #16
ld4r { v8.4s-v11.4s}, [x8], #16
ld4r {v12.4s-v15.4s}, [x8]
mov a0, v0.s[0]
mov a1, v1.s[0]
mov a2, v2.s[0]
mov a3, v3.s[0]
mov a4, v4.s[0]
mov a5, v5.s[0]
mov a6, v6.s[0]
mov a7, v7.s[0]
mov a8, v8.s[0]
mov a9, v9.s[0]
mov a10, v10.s[0]
mov a11, v11.s[0]
mov a12, v12.s[0]
mov a13, v13.s[0]
mov a14, v14.s[0]
mov a15, v15.s[0]
// x12 += counter values 1-4
add v12.4s, v12.4s, v30.4s
.Ldoubleround4:
// x0 += x4, x12 = rotl32(x12 ^ x0, 16)
// x1 += x5, x13 = rotl32(x13 ^ x1, 16)
// x2 += x6, x14 = rotl32(x14 ^ x2, 16)
// x3 += x7, x15 = rotl32(x15 ^ x3, 16)
add v0.4s, v0.4s, v4.4s
add a0, a0, a4
add v1.4s, v1.4s, v5.4s
add a1, a1, a5
add v2.4s, v2.4s, v6.4s
add a2, a2, a6
add v3.4s, v3.4s, v7.4s
add a3, a3, a7
eor v12.16b, v12.16b, v0.16b
eor a12, a12, a0
eor v13.16b, v13.16b, v1.16b
eor a13, a13, a1
eor v14.16b, v14.16b, v2.16b
eor a14, a14, a2
eor v15.16b, v15.16b, v3.16b
eor a15, a15, a3
rev32 v12.8h, v12.8h
ror a12, a12, #16
rev32 v13.8h, v13.8h
ror a13, a13, #16
rev32 v14.8h, v14.8h
ror a14, a14, #16
rev32 v15.8h, v15.8h
ror a15, a15, #16
// x8 += x12, x4 = rotl32(x4 ^ x8, 12)
// x9 += x13, x5 = rotl32(x5 ^ x9, 12)
// x10 += x14, x6 = rotl32(x6 ^ x10, 12)
// x11 += x15, x7 = rotl32(x7 ^ x11, 12)
add v8.4s, v8.4s, v12.4s
add a8, a8, a12
add v9.4s, v9.4s, v13.4s
add a9, a9, a13
add v10.4s, v10.4s, v14.4s
add a10, a10, a14
add v11.4s, v11.4s, v15.4s
add a11, a11, a15
eor v16.16b, v4.16b, v8.16b
eor a4, a4, a8
eor v17.16b, v5.16b, v9.16b
eor a5, a5, a9
eor v18.16b, v6.16b, v10.16b
eor a6, a6, a10
eor v19.16b, v7.16b, v11.16b
eor a7, a7, a11
shl v4.4s, v16.4s, #12
shl v5.4s, v17.4s, #12
shl v6.4s, v18.4s, #12
shl v7.4s, v19.4s, #12
sri v4.4s, v16.4s, #20
ror a4, a4, #20
sri v5.4s, v17.4s, #20
ror a5, a5, #20
sri v6.4s, v18.4s, #20
ror a6, a6, #20
sri v7.4s, v19.4s, #20
ror a7, a7, #20
// x0 += x4, x12 = rotl32(x12 ^ x0, 8)
// x1 += x5, x13 = rotl32(x13 ^ x1, 8)
// x2 += x6, x14 = rotl32(x14 ^ x2, 8)
// x3 += x7, x15 = rotl32(x15 ^ x3, 8)
add v0.4s, v0.4s, v4.4s
add a0, a0, a4
add v1.4s, v1.4s, v5.4s
add a1, a1, a5
add v2.4s, v2.4s, v6.4s
add a2, a2, a6
add v3.4s, v3.4s, v7.4s
add a3, a3, a7
eor v12.16b, v12.16b, v0.16b
eor a12, a12, a0
eor v13.16b, v13.16b, v1.16b
eor a13, a13, a1
eor v14.16b, v14.16b, v2.16b
eor a14, a14, a2
eor v15.16b, v15.16b, v3.16b
eor a15, a15, a3
tbl v12.16b, {v12.16b}, v31.16b
ror a12, a12, #24
tbl v13.16b, {v13.16b}, v31.16b
ror a13, a13, #24
tbl v14.16b, {v14.16b}, v31.16b
ror a14, a14, #24
tbl v15.16b, {v15.16b}, v31.16b
ror a15, a15, #24
// x8 += x12, x4 = rotl32(x4 ^ x8, 7)
// x9 += x13, x5 = rotl32(x5 ^ x9, 7)
// x10 += x14, x6 = rotl32(x6 ^ x10, 7)
// x11 += x15, x7 = rotl32(x7 ^ x11, 7)
add v8.4s, v8.4s, v12.4s
add a8, a8, a12
add v9.4s, v9.4s, v13.4s
add a9, a9, a13
add v10.4s, v10.4s, v14.4s
add a10, a10, a14
add v11.4s, v11.4s, v15.4s
add a11, a11, a15
eor v16.16b, v4.16b, v8.16b
eor a4, a4, a8
eor v17.16b, v5.16b, v9.16b
eor a5, a5, a9
eor v18.16b, v6.16b, v10.16b
eor a6, a6, a10
eor v19.16b, v7.16b, v11.16b
eor a7, a7, a11
shl v4.4s, v16.4s, #7
shl v5.4s, v17.4s, #7
shl v6.4s, v18.4s, #7
shl v7.4s, v19.4s, #7
sri v4.4s, v16.4s, #25
ror a4, a4, #25
sri v5.4s, v17.4s, #25
ror a5, a5, #25
sri v6.4s, v18.4s, #25
ror a6, a6, #25
sri v7.4s, v19.4s, #25
ror a7, a7, #25
// x0 += x5, x15 = rotl32(x15 ^ x0, 16)
// x1 += x6, x12 = rotl32(x12 ^ x1, 16)
// x2 += x7, x13 = rotl32(x13 ^ x2, 16)
// x3 += x4, x14 = rotl32(x14 ^ x3, 16)
add v0.4s, v0.4s, v5.4s
add a0, a0, a5
add v1.4s, v1.4s, v6.4s
add a1, a1, a6
add v2.4s, v2.4s, v7.4s
add a2, a2, a7
add v3.4s, v3.4s, v4.4s
add a3, a3, a4
eor v15.16b, v15.16b, v0.16b
eor a15, a15, a0
eor v12.16b, v12.16b, v1.16b
eor a12, a12, a1
eor v13.16b, v13.16b, v2.16b
eor a13, a13, a2
eor v14.16b, v14.16b, v3.16b
eor a14, a14, a3
rev32 v15.8h, v15.8h
ror a15, a15, #16
rev32 v12.8h, v12.8h
ror a12, a12, #16
rev32 v13.8h, v13.8h
ror a13, a13, #16
rev32 v14.8h, v14.8h
ror a14, a14, #16
// x10 += x15, x5 = rotl32(x5 ^ x10, 12)
// x11 += x12, x6 = rotl32(x6 ^ x11, 12)
// x8 += x13, x7 = rotl32(x7 ^ x8, 12)
// x9 += x14, x4 = rotl32(x4 ^ x9, 12)
add v10.4s, v10.4s, v15.4s
add a10, a10, a15
add v11.4s, v11.4s, v12.4s
add a11, a11, a12
add v8.4s, v8.4s, v13.4s
add a8, a8, a13
add v9.4s, v9.4s, v14.4s
add a9, a9, a14
eor v16.16b, v5.16b, v10.16b
eor a5, a5, a10
eor v17.16b, v6.16b, v11.16b
eor a6, a6, a11
eor v18.16b, v7.16b, v8.16b
eor a7, a7, a8
eor v19.16b, v4.16b, v9.16b
eor a4, a4, a9
shl v5.4s, v16.4s, #12
shl v6.4s, v17.4s, #12
shl v7.4s, v18.4s, #12
shl v4.4s, v19.4s, #12
sri v5.4s, v16.4s, #20
ror a5, a5, #20
sri v6.4s, v17.4s, #20
ror a6, a6, #20
sri v7.4s, v18.4s, #20
ror a7, a7, #20
sri v4.4s, v19.4s, #20
ror a4, a4, #20
// x0 += x5, x15 = rotl32(x15 ^ x0, 8)
// x1 += x6, x12 = rotl32(x12 ^ x1, 8)
// x2 += x7, x13 = rotl32(x13 ^ x2, 8)
// x3 += x4, x14 = rotl32(x14 ^ x3, 8)
add v0.4s, v0.4s, v5.4s
add a0, a0, a5
add v1.4s, v1.4s, v6.4s
add a1, a1, a6
add v2.4s, v2.4s, v7.4s
add a2, a2, a7
add v3.4s, v3.4s, v4.4s
add a3, a3, a4
eor v15.16b, v15.16b, v0.16b
eor a15, a15, a0
eor v12.16b, v12.16b, v1.16b
eor a12, a12, a1
eor v13.16b, v13.16b, v2.16b
eor a13, a13, a2
eor v14.16b, v14.16b, v3.16b
eor a14, a14, a3
tbl v15.16b, {v15.16b}, v31.16b
ror a15, a15, #24
tbl v12.16b, {v12.16b}, v31.16b
ror a12, a12, #24
tbl v13.16b, {v13.16b}, v31.16b
ror a13, a13, #24
tbl v14.16b, {v14.16b}, v31.16b
ror a14, a14, #24
// x10 += x15, x5 = rotl32(x5 ^ x10, 7)
// x11 += x12, x6 = rotl32(x6 ^ x11, 7)
// x8 += x13, x7 = rotl32(x7 ^ x8, 7)
// x9 += x14, x4 = rotl32(x4 ^ x9, 7)
add v10.4s, v10.4s, v15.4s
add a10, a10, a15
add v11.4s, v11.4s, v12.4s
add a11, a11, a12
add v8.4s, v8.4s, v13.4s
add a8, a8, a13
add v9.4s, v9.4s, v14.4s
add a9, a9, a14
eor v16.16b, v5.16b, v10.16b
eor a5, a5, a10
eor v17.16b, v6.16b, v11.16b
eor a6, a6, a11
eor v18.16b, v7.16b, v8.16b
eor a7, a7, a8
eor v19.16b, v4.16b, v9.16b
eor a4, a4, a9
shl v5.4s, v16.4s, #7
shl v6.4s, v17.4s, #7
shl v7.4s, v18.4s, #7
shl v4.4s, v19.4s, #7
sri v5.4s, v16.4s, #25
ror a5, a5, #25
sri v6.4s, v17.4s, #25
ror a6, a6, #25
sri v7.4s, v18.4s, #25
ror a7, a7, #25
sri v4.4s, v19.4s, #25
ror a4, a4, #25
subs w3, w3, #2
b.ne .Ldoubleround4
ld4r {v16.4s-v19.4s}, [x0], #16
ld4r {v20.4s-v23.4s}, [x0], #16
// x12 += counter values 0-3
add v12.4s, v12.4s, v30.4s
// x0[0-3] += s0[0]
// x1[0-3] += s0[1]
// x2[0-3] += s0[2]
// x3[0-3] += s0[3]
add v0.4s, v0.4s, v16.4s
mov w6, v16.s[0]
mov w7, v17.s[0]
add v1.4s, v1.4s, v17.4s
mov w8, v18.s[0]
mov w9, v19.s[0]
add v2.4s, v2.4s, v18.4s
add a0, a0, w6
add a1, a1, w7
add v3.4s, v3.4s, v19.4s
add a2, a2, w8
add a3, a3, w9
ld4r {v24.4s-v27.4s}, [x0], #16
ld4r {v28.4s-v31.4s}, [x0]
// x4[0-3] += s1[0]
// x5[0-3] += s1[1]
// x6[0-3] += s1[2]
// x7[0-3] += s1[3]
add v4.4s, v4.4s, v20.4s
mov w6, v20.s[0]
mov w7, v21.s[0]
add v5.4s, v5.4s, v21.4s
mov w8, v22.s[0]
mov w9, v23.s[0]
add v6.4s, v6.4s, v22.4s
add a4, a4, w6
add a5, a5, w7
add v7.4s, v7.4s, v23.4s
add a6, a6, w8
add a7, a7, w9
// x8[0-3] += s2[0]
// x9[0-3] += s2[1]
// x10[0-3] += s2[2]
// x11[0-3] += s2[3]
add v8.4s, v8.4s, v24.4s
mov w6, v24.s[0]
mov w7, v25.s[0]
add v9.4s, v9.4s, v25.4s
mov w8, v26.s[0]
mov w9, v27.s[0]
add v10.4s, v10.4s, v26.4s
add a8, a8, w6
add a9, a9, w7
add v11.4s, v11.4s, v27.4s
add a10, a10, w8
add a11, a11, w9
// x12[0-3] += s3[0]
// x13[0-3] += s3[1]
// x14[0-3] += s3[2]
// x15[0-3] += s3[3]
add v12.4s, v12.4s, v28.4s
mov w6, v28.s[0]
mov w7, v29.s[0]
add v13.4s, v13.4s, v29.4s
mov w8, v30.s[0]
mov w9, v31.s[0]
add v14.4s, v14.4s, v30.4s
add a12, a12, w6
add a13, a13, w7
add v15.4s, v15.4s, v31.4s
add a14, a14, w8
add a15, a15, w9
// interleave 32-bit words in state n, n+1
ldp w6, w7, [x2], #64
zip1 v16.4s, v0.4s, v1.4s
ldp w8, w9, [x2, #-56]
eor a0, a0, w6
zip2 v17.4s, v0.4s, v1.4s
eor a1, a1, w7
zip1 v18.4s, v2.4s, v3.4s
eor a2, a2, w8
zip2 v19.4s, v2.4s, v3.4s
eor a3, a3, w9
ldp w6, w7, [x2, #-48]
zip1 v20.4s, v4.4s, v5.4s
ldp w8, w9, [x2, #-40]
eor a4, a4, w6
zip2 v21.4s, v4.4s, v5.4s
eor a5, a5, w7
zip1 v22.4s, v6.4s, v7.4s
eor a6, a6, w8
zip2 v23.4s, v6.4s, v7.4s
eor a7, a7, w9
ldp w6, w7, [x2, #-32]
zip1 v24.4s, v8.4s, v9.4s
ldp w8, w9, [x2, #-24]
eor a8, a8, w6
zip2 v25.4s, v8.4s, v9.4s
eor a9, a9, w7
zip1 v26.4s, v10.4s, v11.4s
eor a10, a10, w8
zip2 v27.4s, v10.4s, v11.4s
eor a11, a11, w9
ldp w6, w7, [x2, #-16]
zip1 v28.4s, v12.4s, v13.4s
ldp w8, w9, [x2, #-8]
eor a12, a12, w6
zip2 v29.4s, v12.4s, v13.4s
eor a13, a13, w7
zip1 v30.4s, v14.4s, v15.4s
eor a14, a14, w8
zip2 v31.4s, v14.4s, v15.4s
eor a15, a15, w9
add x3, x2, x4
sub x3, x3, #128 // start of last block
subs x5, x4, #128
csel x2, x2, x3, ge
// interleave 64-bit words in state n, n+2
zip1 v0.2d, v16.2d, v18.2d
zip2 v4.2d, v16.2d, v18.2d
stp a0, a1, [x1], #64
zip1 v8.2d, v17.2d, v19.2d
zip2 v12.2d, v17.2d, v19.2d
stp a2, a3, [x1, #-56]
subs x6, x4, #192
ld1 {v16.16b-v19.16b}, [x2], #64
csel x2, x2, x3, ge
zip1 v1.2d, v20.2d, v22.2d
zip2 v5.2d, v20.2d, v22.2d
stp a4, a5, [x1, #-48]
zip1 v9.2d, v21.2d, v23.2d
zip2 v13.2d, v21.2d, v23.2d
stp a6, a7, [x1, #-40]
subs x7, x4, #256
ld1 {v20.16b-v23.16b}, [x2], #64
csel x2, x2, x3, ge
zip1 v2.2d, v24.2d, v26.2d
zip2 v6.2d, v24.2d, v26.2d
stp a8, a9, [x1, #-32]
zip1 v10.2d, v25.2d, v27.2d
zip2 v14.2d, v25.2d, v27.2d
stp a10, a11, [x1, #-24]
subs x8, x4, #320
ld1 {v24.16b-v27.16b}, [x2], #64
csel x2, x2, x3, ge
zip1 v3.2d, v28.2d, v30.2d
zip2 v7.2d, v28.2d, v30.2d
stp a12, a13, [x1, #-16]
zip1 v11.2d, v29.2d, v31.2d
zip2 v15.2d, v29.2d, v31.2d
stp a14, a15, [x1, #-8]
tbnz x5, #63, .Lt128
ld1 {v28.16b-v31.16b}, [x2]
// xor with corresponding input, write to output
eor v16.16b, v16.16b, v0.16b
eor v17.16b, v17.16b, v1.16b
eor v18.16b, v18.16b, v2.16b
eor v19.16b, v19.16b, v3.16b
tbnz x6, #63, .Lt192
eor v20.16b, v20.16b, v4.16b
eor v21.16b, v21.16b, v5.16b
eor v22.16b, v22.16b, v6.16b
eor v23.16b, v23.16b, v7.16b
st1 {v16.16b-v19.16b}, [x1], #64
tbnz x7, #63, .Lt256
eor v24.16b, v24.16b, v8.16b
eor v25.16b, v25.16b, v9.16b
eor v26.16b, v26.16b, v10.16b
eor v27.16b, v27.16b, v11.16b
st1 {v20.16b-v23.16b}, [x1], #64
tbnz x8, #63, .Lt320
eor v28.16b, v28.16b, v12.16b
eor v29.16b, v29.16b, v13.16b
eor v30.16b, v30.16b, v14.16b
eor v31.16b, v31.16b, v15.16b
st1 {v24.16b-v27.16b}, [x1], #64
st1 {v28.16b-v31.16b}, [x1]
.Lout: frame_pop
ret
// fewer than 192 bytes of in/output
.Lt192: cbz x5, 1f // exactly 128 bytes?
ld1 {v28.16b-v31.16b}, [x10]
add x5, x5, x1
tbl v28.16b, {v4.16b-v7.16b}, v28.16b
tbl v29.16b, {v4.16b-v7.16b}, v29.16b
tbl v30.16b, {v4.16b-v7.16b}, v30.16b
tbl v31.16b, {v4.16b-v7.16b}, v31.16b
0: eor v20.16b, v20.16b, v28.16b
eor v21.16b, v21.16b, v29.16b
eor v22.16b, v22.16b, v30.16b
eor v23.16b, v23.16b, v31.16b
st1 {v20.16b-v23.16b}, [x5] // overlapping stores
1: st1 {v16.16b-v19.16b}, [x1]
b .Lout
// fewer than 128 bytes of in/output
.Lt128: ld1 {v28.16b-v31.16b}, [x10]
add x5, x5, x1
sub x1, x1, #64
tbl v28.16b, {v0.16b-v3.16b}, v28.16b
tbl v29.16b, {v0.16b-v3.16b}, v29.16b
tbl v30.16b, {v0.16b-v3.16b}, v30.16b
tbl v31.16b, {v0.16b-v3.16b}, v31.16b
ld1 {v16.16b-v19.16b}, [x1] // reload first output block
b 0b
// fewer than 256 bytes of in/output
.Lt256: cbz x6, 2f // exactly 192 bytes?
ld1 {v4.16b-v7.16b}, [x10]
add x6, x6, x1
tbl v0.16b, {v8.16b-v11.16b}, v4.16b
tbl v1.16b, {v8.16b-v11.16b}, v5.16b
tbl v2.16b, {v8.16b-v11.16b}, v6.16b
tbl v3.16b, {v8.16b-v11.16b}, v7.16b
eor v28.16b, v28.16b, v0.16b
eor v29.16b, v29.16b, v1.16b
eor v30.16b, v30.16b, v2.16b
eor v31.16b, v31.16b, v3.16b
st1 {v28.16b-v31.16b}, [x6] // overlapping stores
2: st1 {v20.16b-v23.16b}, [x1]
b .Lout
// fewer than 320 bytes of in/output
.Lt320: cbz x7, 3f // exactly 256 bytes?
ld1 {v4.16b-v7.16b}, [x10]
add x7, x7, x1
tbl v0.16b, {v12.16b-v15.16b}, v4.16b
tbl v1.16b, {v12.16b-v15.16b}, v5.16b
tbl v2.16b, {v12.16b-v15.16b}, v6.16b
tbl v3.16b, {v12.16b-v15.16b}, v7.16b
eor v28.16b, v28.16b, v0.16b
eor v29.16b, v29.16b, v1.16b
eor v30.16b, v30.16b, v2.16b
eor v31.16b, v31.16b, v3.16b
st1 {v28.16b-v31.16b}, [x7] // overlapping stores
3: st1 {v24.16b-v27.16b}, [x1]
b .Lout
SYM_FUNC_END(chacha_4block_xor_neon)
.section ".rodata", "a", %progbits
.align L1_CACHE_SHIFT
.Lpermute:
.set .Li, 0
.rept 128
.byte (.Li - 64)
.set .Li, .Li + 1
.endr
CTRINC: .word 1, 2, 3, 4
ROT8: .word 0x02010003, 0x06050407, 0x0a09080b, 0x0e0d0c0f
linux-7.2.3/lib/crypto/arm64/chacha.h
+88
View File
@@ -0,0 +1,88 @@
/*
* ChaCha and HChaCha functions (ARM64 optimized)
*
* Copyright (C) 2016 - 2017 Linaro, Ltd. <ard.biesheuvel@linaro.org>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation.
*
* Based on:
* ChaCha20 256-bit cipher algorithm, RFC7539, SIMD glue code
*
* Copyright (C) 2015 Martin Willi
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*/
#include <crypto/internal/simd.h>
#include <linux/jump_label.h>
#include <linux/kernel.h>
#include <asm/hwcap.h>
#include <asm/simd.h>
asmlinkage void chacha_block_xor_neon(const struct chacha_state *state,
u8 *dst, const u8 *src, int nrounds);
asmlinkage void chacha_4block_xor_neon(const struct chacha_state *state,
u8 *dst, const u8 *src,
int nrounds, int bytes);
asmlinkage void hchacha_block_neon(const struct chacha_state *state,
u32 out[HCHACHA_OUT_WORDS], int nrounds);
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon);
static void chacha_doneon(struct chacha_state *state, u8 *dst, const u8 *src,
unsigned int bytes, int nrounds)
{
while (bytes) {
int l = min(bytes, CHACHA_BLOCK_SIZE * 5);
if (l <= CHACHA_BLOCK_SIZE) {
u8 buf[CHACHA_BLOCK_SIZE];
memcpy(buf, src, l);
chacha_block_xor_neon(state, buf, buf, nrounds);
memcpy(dst, buf, l);
state->x[12] += 1;
break;
}
chacha_4block_xor_neon(state, dst, src, nrounds, l);
bytes -= l;
src += l;
dst += l;
state->x[12] += DIV_ROUND_UP(l, CHACHA_BLOCK_SIZE);
}
}
static void hchacha_block_arch(const struct chacha_state *state,
u32 out[HCHACHA_OUT_WORDS], int nrounds)
{
if (!static_branch_likely(&have_neon) || !crypto_simd_usable()) {
hchacha_block_generic(state, out, nrounds);
} else {
scoped_ksimd()
hchacha_block_neon(state, out, nrounds);
}
}
static void chacha_crypt_arch(struct chacha_state *state, u8 *dst,
const u8 *src, unsigned int bytes, int nrounds)
{
if (!static_branch_likely(&have_neon) || bytes <= CHACHA_BLOCK_SIZE ||
!crypto_simd_usable())
return chacha_crypt_generic(state, dst, src, bytes, nrounds);
scoped_ksimd()
chacha_doneon(state, dst, src, bytes, nrounds);
}
#define chacha_mod_init_arch chacha_mod_init_arch
static void chacha_mod_init_arch(void)
{
if (cpu_have_named_feature(ASIMD))
static_branch_enable(&have_neon);
}
linux-7.2.3/lib/crypto/arm64/gf128hash.h
+121
View File
@@ -0,0 +1,121 @@
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
* GHASH and POLYVAL, arm64 optimized
*
* Copyright 2025 Google LLC
*/
#include <asm/simd.h>
#include <linux/cpufeature.h>
#define NUM_H_POWERS 8
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_asimd);
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_pmull);
asmlinkage void pmull_ghash_update_p8(size_t blocks, struct polyval_elem *dg,
const u8 *src,
const struct polyval_elem *h);
asmlinkage void polyval_mul_pmull(struct polyval_elem *a,
const struct polyval_elem *b);
asmlinkage void polyval_blocks_pmull(struct polyval_elem *acc,
const struct polyval_key *key,
const u8 *data, size_t nblocks);
#define polyval_preparekey_arch polyval_preparekey_arch
static void polyval_preparekey_arch(struct polyval_key *key,
const u8 raw_key[POLYVAL_BLOCK_SIZE])
{
static_assert(ARRAY_SIZE(key->h_powers) == NUM_H_POWERS);
memcpy(&key->h_powers[NUM_H_POWERS - 1], raw_key, POLYVAL_BLOCK_SIZE);
if (static_branch_likely(&have_pmull) && may_use_simd()) {
scoped_ksimd() {
for (int i = NUM_H_POWERS - 2; i >= 0; i--) {
key->h_powers[i] = key->h_powers[i + 1];
polyval_mul_pmull(
&key->h_powers[i],
&key->h_powers[NUM_H_POWERS - 1]);
}
}
} else {
for (int i = NUM_H_POWERS - 2; i >= 0; i--) {
key->h_powers[i] = key->h_powers[i + 1];
polyval_mul_generic(&key->h_powers[i],
&key->h_powers[NUM_H_POWERS - 1]);
}
}
}
static void polyval_mul_arm64(struct polyval_elem *a,
const struct polyval_elem *b)
{
if (static_branch_likely(&have_asimd) && may_use_simd()) {
static const u8 zeroes[GHASH_BLOCK_SIZE];
scoped_ksimd() {
if (static_branch_likely(&have_pmull)) {
polyval_mul_pmull(a, b);
} else {
/*
* Note that this is indeed equivalent to a
* POLYVAL multiplication, since it takes the
* accumulator and key in POLYVAL format, and
* byte-swapping a block of zeroes is a no-op.
*/
pmull_ghash_update_p8(1, a, zeroes, b);
}
}
} else {
polyval_mul_generic(a, b);
}
}
#define ghash_mul_arch ghash_mul_arch
static void ghash_mul_arch(struct polyval_elem *acc,
const struct ghash_key *key)
{
polyval_mul_arm64(acc, &key->h);
}
#define polyval_mul_arch polyval_mul_arch
static void polyval_mul_arch(struct polyval_elem *acc,
const struct polyval_key *key)
{
polyval_mul_arm64(acc, &key->h_powers[NUM_H_POWERS - 1]);
}
#define ghash_blocks_arch ghash_blocks_arch
static void ghash_blocks_arch(struct polyval_elem *acc,
const struct ghash_key *key,
const u8 *data, size_t nblocks)
{
if (static_branch_likely(&have_asimd) && may_use_simd()) {
scoped_ksimd()
pmull_ghash_update_p8(nblocks, acc, data, &key->h);
} else {
ghash_blocks_generic(acc, &key->h, data, nblocks);
}
}
#define polyval_blocks_arch polyval_blocks_arch
static void polyval_blocks_arch(struct polyval_elem *acc,
const struct polyval_key *key,
const u8 *data, size_t nblocks)
{
if (static_branch_likely(&have_pmull) && may_use_simd()) {
scoped_ksimd()
polyval_blocks_pmull(acc, key, data, nblocks);
} else {
polyval_blocks_generic(acc, &key->h_powers[NUM_H_POWERS - 1],
data, nblocks);
}
}
#define gf128hash_mod_init_arch gf128hash_mod_init_arch
static void gf128hash_mod_init_arch(void)
{
if (cpu_have_named_feature(ASIMD)) {
static_branch_enable(&have_asimd);
if (cpu_have_named_feature(PMULL))
static_branch_enable(&have_pmull);
}
}
linux-7.2.3/lib/crypto/arm64/ghash-neon-core.S
+220
View File
@@ -0,0 +1,220 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Accelerated GHASH implementation with ARMv8 ASIMD instructions.
*
* Copyright (C) 2014 - 2018 Linaro Ltd. <ard.biesheuvel@linaro.org>
*/
#include <linux/linkage.h>
#include <asm/assembler.h>
SHASH .req v0
SHASH2 .req v1
T1 .req v2
T2 .req v3
XM .req v5
XL .req v6
XH .req v7
IN1 .req v7
k00_16 .req v8
k32_48 .req v9
t3 .req v10
t4 .req v11
t5 .req v12
t6 .req v13
t7 .req v14
t8 .req v15
t9 .req v16
perm1 .req v17
perm2 .req v18
perm3 .req v19
sh1 .req v20
sh2 .req v21
sh3 .req v22
sh4 .req v23
ss1 .req v24
ss2 .req v25
ss3 .req v26
ss4 .req v27
.text
.macro __pmull_p8, rq, ad, bd
ext t3.8b, \ad\().8b, \ad\().8b, #1 // A1
ext t5.8b, \ad\().8b, \ad\().8b, #2 // A2
ext t7.8b, \ad\().8b, \ad\().8b, #3 // A3
__pmull_p8_\bd \rq, \ad
.endm
.macro __pmull2_p8, rq, ad, bd
tbl t3.16b, {\ad\().16b}, perm1.16b // A1
tbl t5.16b, {\ad\().16b}, perm2.16b // A2
tbl t7.16b, {\ad\().16b}, perm3.16b // A3
__pmull2_p8_\bd \rq, \ad
.endm
.macro __pmull_p8_SHASH, rq, ad
__pmull_p8_tail \rq, \ad\().8b, SHASH.8b, 8b,, sh1, sh2, sh3, sh4
.endm
.macro __pmull_p8_SHASH2, rq, ad
__pmull_p8_tail \rq, \ad\().8b, SHASH2.8b, 8b,, ss1, ss2, ss3, ss4
.endm
.macro __pmull2_p8_SHASH, rq, ad
__pmull_p8_tail \rq, \ad\().16b, SHASH.16b, 16b, 2, sh1, sh2, sh3, sh4
.endm
.macro __pmull_p8_tail, rq, ad, bd, nb, t, b1, b2, b3, b4
pmull\t t3.8h, t3.\nb, \bd // F = A1*B
pmull\t t4.8h, \ad, \b1\().\nb // E = A*B1
pmull\t t5.8h, t5.\nb, \bd // H = A2*B
pmull\t t6.8h, \ad, \b2\().\nb // G = A*B2
pmull\t t7.8h, t7.\nb, \bd // J = A3*B
pmull\t t8.8h, \ad, \b3\().\nb // I = A*B3
pmull\t t9.8h, \ad, \b4\().\nb // K = A*B4
pmull\t \rq\().8h, \ad, \bd // D = A*B
eor t3.16b, t3.16b, t4.16b // L = E + F
eor t5.16b, t5.16b, t6.16b // M = G + H
eor t7.16b, t7.16b, t8.16b // N = I + J
uzp1 t4.2d, t3.2d, t5.2d
uzp2 t3.2d, t3.2d, t5.2d
uzp1 t6.2d, t7.2d, t9.2d
uzp2 t7.2d, t7.2d, t9.2d
// t3 = (L) (P0 + P1) << 8
// t5 = (M) (P2 + P3) << 16
eor t4.16b, t4.16b, t3.16b
and t3.16b, t3.16b, k32_48.16b
// t7 = (N) (P4 + P5) << 24
// t9 = (K) (P6 + P7) << 32
eor t6.16b, t6.16b, t7.16b
and t7.16b, t7.16b, k00_16.16b
eor t4.16b, t4.16b, t3.16b
eor t6.16b, t6.16b, t7.16b
zip2 t5.2d, t4.2d, t3.2d
zip1 t3.2d, t4.2d, t3.2d
zip2 t9.2d, t6.2d, t7.2d
zip1 t7.2d, t6.2d, t7.2d
ext t3.16b, t3.16b, t3.16b, #15
ext t5.16b, t5.16b, t5.16b, #14
ext t7.16b, t7.16b, t7.16b, #13
ext t9.16b, t9.16b, t9.16b, #12
eor t3.16b, t3.16b, t5.16b
eor t7.16b, t7.16b, t9.16b
eor \rq\().16b, \rq\().16b, t3.16b
eor \rq\().16b, \rq\().16b, t7.16b
.endm
.macro __pmull_pre_p8
ext SHASH2.16b, SHASH.16b, SHASH.16b, #8
eor SHASH2.16b, SHASH2.16b, SHASH.16b
// k00_16 := 0x0000000000000000_000000000000ffff
// k32_48 := 0x00000000ffffffff_0000ffffffffffff
movi k32_48.2d, #0xffffffff
mov k32_48.h[2], k32_48.h[0]
ushr k00_16.2d, k32_48.2d, #32
// prepare the permutation vectors
mov_q x5, 0x080f0e0d0c0b0a09
movi T1.8b, #8
dup perm1.2d, x5
eor perm1.16b, perm1.16b, T1.16b
ushr perm2.2d, perm1.2d, #8
ushr perm3.2d, perm1.2d, #16
ushr T1.2d, perm1.2d, #24
sli perm2.2d, perm1.2d, #56
sli perm3.2d, perm1.2d, #48
sli T1.2d, perm1.2d, #40
// precompute loop invariants
tbl sh1.16b, {SHASH.16b}, perm1.16b
tbl sh2.16b, {SHASH.16b}, perm2.16b
tbl sh3.16b, {SHASH.16b}, perm3.16b
tbl sh4.16b, {SHASH.16b}, T1.16b
ext ss1.8b, SHASH2.8b, SHASH2.8b, #1
ext ss2.8b, SHASH2.8b, SHASH2.8b, #2
ext ss3.8b, SHASH2.8b, SHASH2.8b, #3
ext ss4.8b, SHASH2.8b, SHASH2.8b, #4
.endm
.macro __pmull_reduce_p8
eor XM.16b, XM.16b, T1.16b
mov XL.d[1], XM.d[0]
mov XH.d[0], XM.d[1]
shl T1.2d, XL.2d, #57
shl T2.2d, XL.2d, #62
eor T2.16b, T2.16b, T1.16b
shl T1.2d, XL.2d, #63
eor T2.16b, T2.16b, T1.16b
ext T1.16b, XL.16b, XH.16b, #8
eor T2.16b, T2.16b, T1.16b
mov XL.d[1], T2.d[0]
mov XH.d[0], T2.d[1]
ushr T2.2d, XL.2d, #1
eor XH.16b, XH.16b, XL.16b
eor XL.16b, XL.16b, T2.16b
ushr T2.2d, T2.2d, #6
ushr XL.2d, XL.2d, #1
.endm
/*
* void pmull_ghash_update_p8(size_t blocks, struct polyval_elem *dg,
* const u8 *src,
* const struct polyval_elem *h)
*/
SYM_FUNC_START(pmull_ghash_update_p8)
ld1 {SHASH.2d}, [x3]
ld1 {XL.2d}, [x1]
__pmull_pre_p8
0: ld1 {T1.2d}, [x2], #16
sub x0, x0, #1
/* multiply XL by SHASH in GF(2^128) */
rev64 T1.16b, T1.16b
ext T2.16b, XL.16b, XL.16b, #8
ext IN1.16b, T1.16b, T1.16b, #8
eor T1.16b, T1.16b, T2.16b
eor XL.16b, XL.16b, IN1.16b
__pmull2_p8 XH, XL, SHASH // a1 * b1
eor T1.16b, T1.16b, XL.16b
__pmull_p8 XL, XL, SHASH // a0 * b0
__pmull_p8 XM, T1, SHASH2 // (a1 + a0)(b1 + b0)
eor T2.16b, XL.16b, XH.16b
ext T1.16b, XL.16b, XH.16b, #8
eor XM.16b, XM.16b, T2.16b
__pmull_reduce_p8
eor T2.16b, T2.16b, XH.16b
eor XL.16b, XL.16b, T2.16b
cbnz x0, 0b
st1 {XL.2d}, [x1]
ret
SYM_FUNC_END(pmull_ghash_update_p8)
linux-7.2.3/lib/crypto/arm64/nh-neon-core.S
+103
View File
@@ -0,0 +1,103 @@
/* SPDX-License-Identifier: GPL-2.0 */
/*
* NH - ε-almost-universal hash function, ARM64 NEON accelerated version
*
* Copyright 2018 Google LLC
*
* Author: Eric Biggers <ebiggers@google.com>
*/
#include <linux/linkage.h>
KEY .req x0
MESSAGE .req x1
MESSAGE_LEN .req x2
HASH .req x3
PASS0_SUMS .req v0
PASS1_SUMS .req v1
PASS2_SUMS .req v2
PASS3_SUMS .req v3
K0 .req v4
K1 .req v5
K2 .req v6
K3 .req v7
T0 .req v8
T1 .req v9
T2 .req v10
T3 .req v11
T4 .req v12
T5 .req v13
T6 .req v14
T7 .req v15
.macro _nh_stride k0, k1, k2, k3
// Load next message stride
ld1 {T3.16b}, [MESSAGE], #16
// Load next key stride
ld1 {\k3\().4s}, [KEY], #16
// Add message words to key words
add T0.4s, T3.4s, \k0\().4s
add T1.4s, T3.4s, \k1\().4s
add T2.4s, T3.4s, \k2\().4s
add T3.4s, T3.4s, \k3\().4s
// Multiply 32x32 => 64 and accumulate
mov T4.d[0], T0.d[1]
mov T5.d[0], T1.d[1]
mov T6.d[0], T2.d[1]
mov T7.d[0], T3.d[1]
umlal PASS0_SUMS.2d, T0.2s, T4.2s
umlal PASS1_SUMS.2d, T1.2s, T5.2s
umlal PASS2_SUMS.2d, T2.2s, T6.2s
umlal PASS3_SUMS.2d, T3.2s, T7.2s
.endm
/*
* void nh_neon(const u32 *key, const u8 *message, size_t message_len,
* __le64 hash[NH_NUM_PASSES])
*
* It's guaranteed that message_len % 16 == 0.
*/
SYM_FUNC_START(nh_neon)
ld1 {K0.4s,K1.4s}, [KEY], #32
movi PASS0_SUMS.2d, #0
movi PASS1_SUMS.2d, #0
ld1 {K2.4s}, [KEY], #16
movi PASS2_SUMS.2d, #0
movi PASS3_SUMS.2d, #0
subs MESSAGE_LEN, MESSAGE_LEN, #64
blt .Lloop4_done
.Lloop4:
_nh_stride K0, K1, K2, K3
_nh_stride K1, K2, K3, K0
_nh_stride K2, K3, K0, K1
_nh_stride K3, K0, K1, K2
subs MESSAGE_LEN, MESSAGE_LEN, #64
bge .Lloop4
.Lloop4_done:
ands MESSAGE_LEN, MESSAGE_LEN, #63
beq .Ldone
_nh_stride K0, K1, K2, K3
subs MESSAGE_LEN, MESSAGE_LEN, #16
beq .Ldone
_nh_stride K1, K2, K3, K0
subs MESSAGE_LEN, MESSAGE_LEN, #16
beq .Ldone
_nh_stride K2, K3, K0, K1
.Ldone:
// Sum the accumulators for each pass, then store the sums to 'hash'
addp T0.2d, PASS0_SUMS.2d, PASS1_SUMS.2d
addp T1.2d, PASS2_SUMS.2d, PASS3_SUMS.2d
st1 {T0.16b,T1.16b}, [HASH]
ret
SYM_FUNC_END(nh_neon)
linux-7.2.3/lib/crypto/arm64/nh.h
+34
View File
@@ -0,0 +1,34 @@
/* SPDX-License-Identifier: GPL-2.0 */
/*
* ARM64 accelerated implementation of NH
*
* Copyright 2018 Google LLC
*/
#include <asm/hwcap.h>
#include <asm/simd.h>
#include <linux/cpufeature.h>
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon);
asmlinkage void nh_neon(const u32 *key, const u8 *message, size_t message_len,
__le64 hash[NH_NUM_PASSES]);
static bool nh_arch(const u32 *key, const u8 *message, size_t message_len,
__le64 hash[NH_NUM_PASSES])
{
if (static_branch_likely(&have_neon) && message_len >= 64 &&
may_use_simd()) {
scoped_ksimd()
nh_neon(key, message, message_len, hash);
return true;
}
return false;
}
#define nh_mod_init_arch nh_mod_init_arch
static void nh_mod_init_arch(void)
{
if (cpu_have_named_feature(ASIMD))
static_branch_enable(&have_neon);
}
linux-7.2.3/lib/crypto/arm64/poly1305-armv8.pl
+920
View File
@@ -0,0 +1,920 @@
#!/usr/bin/env perl
# SPDX-License-Identifier: GPL-1.0+ OR BSD-3-Clause
#
# ====================================================================
# Written by Andy Polyakov, @dot-asm, initially for the OpenSSL
# project.
# ====================================================================
#
# This module implements Poly1305 hash for ARMv8.
#
# June 2015
#
# Numbers are cycles per processed byte with poly1305_blocks alone.
#
# IALU/gcc-4.9 NEON
#
# Apple A7 1.86/+5% 0.72
# Cortex-A53 2.69/+58% 1.47
# Cortex-A57 2.70/+7% 1.14
# Denver 1.64/+50% 1.18(*)
# X-Gene 2.13/+68% 2.27
# Mongoose 1.77/+75% 1.12
# Kryo 2.70/+55% 1.13
# ThunderX2 1.17/+95% 1.36
#
# (*) estimate based on resources availability is less than 1.0,
# i.e. measured result is worse than expected, presumably binary
# translator is not almighty;
$flavour=shift;
$output=shift;
if ($flavour && $flavour ne "void") {
$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or
( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate) or
die "can't locate arm-xlate.pl";
open STDOUT,"| \"$^X\" $xlate $flavour $output";
} else {
open STDOUT,">$output";
}
my ($ctx,$inp,$len,$padbit) = map("x$_",(0..3));
my ($mac,$nonce)=($inp,$len);
my ($h0,$h1,$h2,$r0,$r1,$s1,$t0,$t1,$d0,$d1,$d2) = map("x$_",(4..14));
$code.=<<___;
#ifndef __KERNEL__
# include "arm_arch.h"
.extern OPENSSL_armcap_P
#else
# define poly1305_init poly1305_block_init
# define poly1305_blocks poly1305_blocks_arm64
#endif
.text
// forward "declarations" are required for Apple
.globl poly1305_blocks
.globl poly1305_emit
.globl poly1305_init
.type poly1305_init,%function
.align 5
poly1305_init:
cmp $inp,xzr
stp xzr,xzr,[$ctx] // zero hash value
stp xzr,xzr,[$ctx,#16] // [along with is_base2_26]
csel x0,xzr,x0,eq
b.eq .Lno_key
#ifndef __KERNEL__
adrp x17,OPENSSL_armcap_P
ldr w17,[x17,#:lo12:OPENSSL_armcap_P]
#endif
ldp $r0,$r1,[$inp] // load key
mov $s1,#0xfffffffc0fffffff
movk $s1,#0x0fff,lsl#48
#ifdef __AARCH64EB__
rev $r0,$r0 // flip bytes
rev $r1,$r1
#endif
and $r0,$r0,$s1 // &=0ffffffc0fffffff
and $s1,$s1,#-4
and $r1,$r1,$s1 // &=0ffffffc0ffffffc
mov w#$s1,#-1
stp $r0,$r1,[$ctx,#32] // save key value
str w#$s1,[$ctx,#48] // impossible key power value
#ifndef __KERNEL__
tst w17,#ARMV7_NEON
adr $d0,.Lpoly1305_blocks
adr $r0,.Lpoly1305_blocks_neon
adr $d1,.Lpoly1305_emit
csel $d0,$d0,$r0,eq
# ifdef __ILP32__
stp w#$d0,w#$d1,[$len]
# else
stp $d0,$d1,[$len]
# endif
#endif
mov x0,#1
.Lno_key:
ret
.size poly1305_init,.-poly1305_init
.type poly1305_blocks,%function
.align 5
poly1305_blocks:
.Lpoly1305_blocks:
ands $len,$len,#-16
b.eq .Lno_data
ldp $h0,$h1,[$ctx] // load hash value
ldp $h2,x17,[$ctx,#16] // [along with is_base2_26]
ldp $r0,$r1,[$ctx,#32] // load key value
#ifdef __AARCH64EB__
lsr $d0,$h0,#32
mov w#$d1,w#$h0
lsr $d2,$h1,#32
mov w15,w#$h1
lsr x16,$h2,#32
#else
mov w#$d0,w#$h0
lsr $d1,$h0,#32
mov w#$d2,w#$h1
lsr x15,$h1,#32
mov w16,w#$h2
#endif
add $d0,$d0,$d1,lsl#26 // base 2^26 -> base 2^64
lsr $d1,$d2,#12
adds $d0,$d0,$d2,lsl#52
add $d1,$d1,x15,lsl#14
adc $d1,$d1,xzr
lsr $d2,x16,#24
adds $d1,$d1,x16,lsl#40
adc $d2,$d2,xzr
cmp x17,#0 // is_base2_26?
add $s1,$r1,$r1,lsr#2 // s1 = r1 + (r1 >> 2)
csel $h0,$h0,$d0,eq // choose between radixes
csel $h1,$h1,$d1,eq
csel $h2,$h2,$d2,eq
.Loop:
ldp $t0,$t1,[$inp],#16 // load input
sub $len,$len,#16
#ifdef __AARCH64EB__
rev $t0,$t0
rev $t1,$t1
#endif
adds $h0,$h0,$t0 // accumulate input
adcs $h1,$h1,$t1
mul $d0,$h0,$r0 // h0*r0
adc $h2,$h2,$padbit
umulh $d1,$h0,$r0
mul $t0,$h1,$s1 // h1*5*r1
umulh $t1,$h1,$s1
adds $d0,$d0,$t0
mul $t0,$h0,$r1 // h0*r1
adc $d1,$d1,$t1
umulh $d2,$h0,$r1
adds $d1,$d1,$t0
mul $t0,$h1,$r0 // h1*r0
adc $d2,$d2,xzr
umulh $t1,$h1,$r0
adds $d1,$d1,$t0
mul $t0,$h2,$s1 // h2*5*r1
adc $d2,$d2,$t1
mul $t1,$h2,$r0 // h2*r0
adds $d1,$d1,$t0
adc $d2,$d2,$t1
and $t0,$d2,#-4 // final reduction
and $h2,$d2,#3
add $t0,$t0,$d2,lsr#2
adds $h0,$d0,$t0
adcs $h1,$d1,xzr
adc $h2,$h2,xzr
cbnz $len,.Loop
stp $h0,$h1,[$ctx] // store hash value
stp $h2,xzr,[$ctx,#16] // [and clear is_base2_26]
.Lno_data:
ret
.size poly1305_blocks,.-poly1305_blocks
.type poly1305_emit,%function
.align 5
poly1305_emit:
.Lpoly1305_emit:
ldp $h0,$h1,[$ctx] // load hash base 2^64
ldp $h2,$r0,[$ctx,#16] // [along with is_base2_26]
ldp $t0,$t1,[$nonce] // load nonce
#ifdef __AARCH64EB__
lsr $d0,$h0,#32
mov w#$d1,w#$h0
lsr $d2,$h1,#32
mov w15,w#$h1
lsr x16,$h2,#32
#else
mov w#$d0,w#$h0
lsr $d1,$h0,#32
mov w#$d2,w#$h1
lsr x15,$h1,#32
mov w16,w#$h2
#endif
add $d0,$d0,$d1,lsl#26 // base 2^26 -> base 2^64
lsr $d1,$d2,#12
adds $d0,$d0,$d2,lsl#52
add $d1,$d1,x15,lsl#14
adc $d1,$d1,xzr
lsr $d2,x16,#24
adds $d1,$d1,x16,lsl#40
adc $d2,$d2,xzr
cmp $r0,#0 // is_base2_26?
csel $h0,$h0,$d0,eq // choose between radixes
csel $h1,$h1,$d1,eq
csel $h2,$h2,$d2,eq
adds $d0,$h0,#5 // compare to modulus
adcs $d1,$h1,xzr
adc $d2,$h2,xzr
tst $d2,#-4 // see if it's carried/borrowed
csel $h0,$h0,$d0,eq
csel $h1,$h1,$d1,eq
#ifdef __AARCH64EB__
ror $t0,$t0,#32 // flip nonce words
ror $t1,$t1,#32
#endif
adds $h0,$h0,$t0 // accumulate nonce
adc $h1,$h1,$t1
#ifdef __AARCH64EB__
rev $h0,$h0 // flip output bytes
rev $h1,$h1
#endif
stp $h0,$h1,[$mac] // write result
ret
.size poly1305_emit,.-poly1305_emit
___
my ($R0,$R1,$S1,$R2,$S2,$R3,$S3,$R4,$S4) = map("v$_.4s",(0..8));
my ($IN01_0,$IN01_1,$IN01_2,$IN01_3,$IN01_4) = map("v$_.2s",(9..13));
my ($IN23_0,$IN23_1,$IN23_2,$IN23_3,$IN23_4) = map("v$_.2s",(14..18));
my ($ACC0,$ACC1,$ACC2,$ACC3,$ACC4) = map("v$_.2d",(19..23));
my ($H0,$H1,$H2,$H3,$H4) = map("v$_.2s",(24..28));
my ($T0,$T1,$MASK) = map("v$_",(29..31));
my ($in2,$zeros)=("x16","x17");
my $is_base2_26 = $zeros; # borrow
$code.=<<___;
.type poly1305_mult,%function
.align 5
poly1305_mult:
mul $d0,$h0,$r0 // h0*r0
umulh $d1,$h0,$r0
mul $t0,$h1,$s1 // h1*5*r1
umulh $t1,$h1,$s1
adds $d0,$d0,$t0
mul $t0,$h0,$r1 // h0*r1
adc $d1,$d1,$t1
umulh $d2,$h0,$r1
adds $d1,$d1,$t0
mul $t0,$h1,$r0 // h1*r0
adc $d2,$d2,xzr
umulh $t1,$h1,$r0
adds $d1,$d1,$t0
mul $t0,$h2,$s1 // h2*5*r1
adc $d2,$d2,$t1
mul $t1,$h2,$r0 // h2*r0
adds $d1,$d1,$t0
adc $d2,$d2,$t1
and $t0,$d2,#-4 // final reduction
and $h2,$d2,#3
add $t0,$t0,$d2,lsr#2
adds $h0,$d0,$t0
adcs $h1,$d1,xzr
adc $h2,$h2,xzr
ret
.size poly1305_mult,.-poly1305_mult
.type poly1305_splat,%function
.align 4
poly1305_splat:
and x12,$h0,#0x03ffffff // base 2^64 -> base 2^26
ubfx x13,$h0,#26,#26
extr x14,$h1,$h0,#52
and x14,x14,#0x03ffffff
ubfx x15,$h1,#14,#26
extr x16,$h2,$h1,#40
str w12,[$ctx,#16*0] // r0
add w12,w13,w13,lsl#2 // r1*5
str w13,[$ctx,#16*1] // r1
add w13,w14,w14,lsl#2 // r2*5
str w12,[$ctx,#16*2] // s1
str w14,[$ctx,#16*3] // r2
add w14,w15,w15,lsl#2 // r3*5
str w13,[$ctx,#16*4] // s2
str w15,[$ctx,#16*5] // r3
add w15,w16,w16,lsl#2 // r4*5
str w14,[$ctx,#16*6] // s3
str w16,[$ctx,#16*7] // r4
str w15,[$ctx,#16*8] // s4
ret
.size poly1305_splat,.-poly1305_splat
#ifdef __KERNEL__
.globl poly1305_blocks_neon
#endif
.type poly1305_blocks_neon,%function
.align 5
poly1305_blocks_neon:
.Lpoly1305_blocks_neon:
ldr $is_base2_26,[$ctx,#24]
cmp $len,#128
b.lo .Lpoly1305_blocks
.inst 0xd503233f // paciasp
stp x29,x30,[sp,#-80]!
add x29,sp,#0
stp d8,d9,[sp,#16] // meet ABI requirements
stp d10,d11,[sp,#32]
stp d12,d13,[sp,#48]
stp d14,d15,[sp,#64]
cbz $is_base2_26,.Lbase2_64_neon
ldp w10,w11,[$ctx] // load hash value base 2^26
ldp w12,w13,[$ctx,#8]
ldr w14,[$ctx,#16]
tst $len,#31
b.eq .Leven_neon
ldp $r0,$r1,[$ctx,#32] // load key value
add $h0,x10,x11,lsl#26 // base 2^26 -> base 2^64
lsr $h1,x12,#12
adds $h0,$h0,x12,lsl#52
add $h1,$h1,x13,lsl#14
adc $h1,$h1,xzr
lsr $h2,x14,#24
adds $h1,$h1,x14,lsl#40
adc $d2,$h2,xzr // can be partially reduced...
ldp $d0,$d1,[$inp],#16 // load input
sub $len,$len,#16
add $s1,$r1,$r1,lsr#2 // s1 = r1 + (r1 >> 2)
#ifdef __AARCH64EB__
rev $d0,$d0
rev $d1,$d1
#endif
adds $h0,$h0,$d0 // accumulate input
adcs $h1,$h1,$d1
adc $h2,$h2,$padbit
bl poly1305_mult
and x10,$h0,#0x03ffffff // base 2^64 -> base 2^26
ubfx x11,$h0,#26,#26
extr x12,$h1,$h0,#52
and x12,x12,#0x03ffffff
ubfx x13,$h1,#14,#26
extr x14,$h2,$h1,#40
b .Leven_neon
.align 4
.Lbase2_64_neon:
ldp $r0,$r1,[$ctx,#32] // load key value
ldp $h0,$h1,[$ctx] // load hash value base 2^64
ldr $h2,[$ctx,#16]
tst $len,#31
b.eq .Linit_neon
ldp $d0,$d1,[$inp],#16 // load input
sub $len,$len,#16
add $s1,$r1,$r1,lsr#2 // s1 = r1 + (r1 >> 2)
#ifdef __AARCH64EB__
rev $d0,$d0
rev $d1,$d1
#endif
adds $h0,$h0,$d0 // accumulate input
adcs $h1,$h1,$d1
adc $h2,$h2,$padbit
bl poly1305_mult
.Linit_neon:
ldr w17,[$ctx,#48] // first table element
and x10,$h0,#0x03ffffff // base 2^64 -> base 2^26
ubfx x11,$h0,#26,#26
extr x12,$h1,$h0,#52
and x12,x12,#0x03ffffff
ubfx x13,$h1,#14,#26
extr x14,$h2,$h1,#40
cmp w17,#-1 // is value impossible?
b.ne .Leven_neon
fmov ${H0},x10
fmov ${H1},x11
fmov ${H2},x12
fmov ${H3},x13
fmov ${H4},x14
////////////////////////////////// initialize r^n table
mov $h0,$r0 // r^1
add $s1,$r1,$r1,lsr#2 // s1 = r1 + (r1 >> 2)
mov $h1,$r1
mov $h2,xzr
add $ctx,$ctx,#48+12
bl poly1305_splat
bl poly1305_mult // r^2
sub $ctx,$ctx,#4
bl poly1305_splat
bl poly1305_mult // r^3
sub $ctx,$ctx,#4
bl poly1305_splat
bl poly1305_mult // r^4
sub $ctx,$ctx,#4
bl poly1305_splat
sub $ctx,$ctx,#48 // restore original $ctx
b .Ldo_neon
.align 4
.Leven_neon:
fmov ${H0},x10
fmov ${H1},x11
fmov ${H2},x12
fmov ${H3},x13
fmov ${H4},x14
.Ldo_neon:
ldp x8,x12,[$inp,#32] // inp[2:3]
subs $len,$len,#64
ldp x9,x13,[$inp,#48]
add $in2,$inp,#96
adrp $zeros,.Lzeros
add $zeros,$zeros,#:lo12:.Lzeros
lsl $padbit,$padbit,#24
add x15,$ctx,#48
#ifdef __AARCH64EB__
rev x8,x8
rev x12,x12
rev x9,x9
rev x13,x13
#endif
and x4,x8,#0x03ffffff // base 2^64 -> base 2^26
and x5,x9,#0x03ffffff
ubfx x6,x8,#26,#26
ubfx x7,x9,#26,#26
add x4,x4,x5,lsl#32 // bfi x4,x5,#32,#32
extr x8,x12,x8,#52
extr x9,x13,x9,#52
add x6,x6,x7,lsl#32 // bfi x6,x7,#32,#32
fmov $IN23_0,x4
and x8,x8,#0x03ffffff
and x9,x9,#0x03ffffff
ubfx x10,x12,#14,#26
ubfx x11,x13,#14,#26
add x12,$padbit,x12,lsr#40
add x13,$padbit,x13,lsr#40
add x8,x8,x9,lsl#32 // bfi x8,x9,#32,#32
fmov $IN23_1,x6
add x10,x10,x11,lsl#32 // bfi x10,x11,#32,#32
add x12,x12,x13,lsl#32 // bfi x12,x13,#32,#32
fmov $IN23_2,x8
fmov $IN23_3,x10
fmov $IN23_4,x12
ldp x8,x12,[$inp],#16 // inp[0:1]
ldp x9,x13,[$inp],#48
ld1 {$R0,$R1,$S1,$R2},[x15],#64
ld1 {$S2,$R3,$S3,$R4},[x15],#64
ld1 {$S4},[x15]
#ifdef __AARCH64EB__
rev x8,x8
rev x12,x12
rev x9,x9
rev x13,x13
#endif
and x4,x8,#0x03ffffff // base 2^64 -> base 2^26
and x5,x9,#0x03ffffff
ubfx x6,x8,#26,#26
ubfx x7,x9,#26,#26
add x4,x4,x5,lsl#32 // bfi x4,x5,#32,#32
extr x8,x12,x8,#52
extr x9,x13,x9,#52
add x6,x6,x7,lsl#32 // bfi x6,x7,#32,#32
fmov $IN01_0,x4
and x8,x8,#0x03ffffff
and x9,x9,#0x03ffffff
ubfx x10,x12,#14,#26
ubfx x11,x13,#14,#26
add x12,$padbit,x12,lsr#40
add x13,$padbit,x13,lsr#40
add x8,x8,x9,lsl#32 // bfi x8,x9,#32,#32
fmov $IN01_1,x6
add x10,x10,x11,lsl#32 // bfi x10,x11,#32,#32
add x12,x12,x13,lsl#32 // bfi x12,x13,#32,#32
movi $MASK.2d,#-1
fmov $IN01_2,x8
fmov $IN01_3,x10
fmov $IN01_4,x12
ushr $MASK.2d,$MASK.2d,#38
b.ls .Lskip_loop
.align 4
.Loop_neon:
////////////////////////////////////////////////////////////////
// ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2
// ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^3+inp[7]*r
// \___________________/
// ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2+inp[8])*r^2
// ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^4+inp[7]*r^2+inp[9])*r
// \___________________/ \____________________/
//
// Note that we start with inp[2:3]*r^2. This is because it
// doesn't depend on reduction in previous iteration.
////////////////////////////////////////////////////////////////
// d4 = h0*r4 + h1*r3 + h2*r2 + h3*r1 + h4*r0
// d3 = h0*r3 + h1*r2 + h2*r1 + h3*r0 + h4*5*r4
// d2 = h0*r2 + h1*r1 + h2*r0 + h3*5*r4 + h4*5*r3
// d1 = h0*r1 + h1*r0 + h2*5*r4 + h3*5*r3 + h4*5*r2
// d0 = h0*r0 + h1*5*r4 + h2*5*r3 + h3*5*r2 + h4*5*r1
subs $len,$len,#64
umull $ACC4,$IN23_0,${R4}[2]
csel $in2,$zeros,$in2,lo
umull $ACC3,$IN23_0,${R3}[2]
umull $ACC2,$IN23_0,${R2}[2]
ldp x8,x12,[$in2],#16 // inp[2:3] (or zero)
umull $ACC1,$IN23_0,${R1}[2]
ldp x9,x13,[$in2],#48
umull $ACC0,$IN23_0,${R0}[2]
#ifdef __AARCH64EB__
rev x8,x8
rev x12,x12
rev x9,x9
rev x13,x13
#endif
umlal $ACC4,$IN23_1,${R3}[2]
and x4,x8,#0x03ffffff // base 2^64 -> base 2^26
umlal $ACC3,$IN23_1,${R2}[2]
and x5,x9,#0x03ffffff
umlal $ACC2,$IN23_1,${R1}[2]
ubfx x6,x8,#26,#26
umlal $ACC1,$IN23_1,${R0}[2]
ubfx x7,x9,#26,#26
umlal $ACC0,$IN23_1,${S4}[2]
add x4,x4,x5,lsl#32 // bfi x4,x5,#32,#32
umlal $ACC4,$IN23_2,${R2}[2]
extr x8,x12,x8,#52
umlal $ACC3,$IN23_2,${R1}[2]
extr x9,x13,x9,#52
umlal $ACC2,$IN23_2,${R0}[2]
add x6,x6,x7,lsl#32 // bfi x6,x7,#32,#32
umlal $ACC1,$IN23_2,${S4}[2]
fmov $IN23_0,x4
umlal $ACC0,$IN23_2,${S3}[2]
and x8,x8,#0x03ffffff
umlal $ACC4,$IN23_3,${R1}[2]
and x9,x9,#0x03ffffff
umlal $ACC3,$IN23_3,${R0}[2]
ubfx x10,x12,#14,#26
umlal $ACC2,$IN23_3,${S4}[2]
ubfx x11,x13,#14,#26
umlal $ACC1,$IN23_3,${S3}[2]
add x8,x8,x9,lsl#32 // bfi x8,x9,#32,#32
umlal $ACC0,$IN23_3,${S2}[2]
fmov $IN23_1,x6
add $IN01_2,$IN01_2,$H2
add x12,$padbit,x12,lsr#40
umlal $ACC4,$IN23_4,${R0}[2]
add x13,$padbit,x13,lsr#40
umlal $ACC3,$IN23_4,${S4}[2]
add x10,x10,x11,lsl#32 // bfi x10,x11,#32,#32
umlal $ACC2,$IN23_4,${S3}[2]
add x12,x12,x13,lsl#32 // bfi x12,x13,#32,#32
umlal $ACC1,$IN23_4,${S2}[2]
fmov $IN23_2,x8
umlal $ACC0,$IN23_4,${S1}[2]
fmov $IN23_3,x10
////////////////////////////////////////////////////////////////
// (hash+inp[0:1])*r^4 and accumulate
add $IN01_0,$IN01_0,$H0
fmov $IN23_4,x12
umlal $ACC3,$IN01_2,${R1}[0]
ldp x8,x12,[$inp],#16 // inp[0:1]
umlal $ACC0,$IN01_2,${S3}[0]
ldp x9,x13,[$inp],#48
umlal $ACC4,$IN01_2,${R2}[0]
umlal $ACC1,$IN01_2,${S4}[0]
umlal $ACC2,$IN01_2,${R0}[0]
#ifdef __AARCH64EB__
rev x8,x8
rev x12,x12
rev x9,x9
rev x13,x13
#endif
add $IN01_1,$IN01_1,$H1
umlal $ACC3,$IN01_0,${R3}[0]
umlal $ACC4,$IN01_0,${R4}[0]
and x4,x8,#0x03ffffff // base 2^64 -> base 2^26
umlal $ACC2,$IN01_0,${R2}[0]
and x5,x9,#0x03ffffff
umlal $ACC0,$IN01_0,${R0}[0]
ubfx x6,x8,#26,#26
umlal $ACC1,$IN01_0,${R1}[0]
ubfx x7,x9,#26,#26
add $IN01_3,$IN01_3,$H3
add x4,x4,x5,lsl#32 // bfi x4,x5,#32,#32
umlal $ACC3,$IN01_1,${R2}[0]
extr x8,x12,x8,#52
umlal $ACC4,$IN01_1,${R3}[0]
extr x9,x13,x9,#52
umlal $ACC0,$IN01_1,${S4}[0]
add x6,x6,x7,lsl#32 // bfi x6,x7,#32,#32
umlal $ACC2,$IN01_1,${R1}[0]
fmov $IN01_0,x4
umlal $ACC1,$IN01_1,${R0}[0]
and x8,x8,#0x03ffffff
add $IN01_4,$IN01_4,$H4
and x9,x9,#0x03ffffff
umlal $ACC3,$IN01_3,${R0}[0]
ubfx x10,x12,#14,#26
umlal $ACC0,$IN01_3,${S2}[0]
ubfx x11,x13,#14,#26
umlal $ACC4,$IN01_3,${R1}[0]
add x8,x8,x9,lsl#32 // bfi x8,x9,#32,#32
umlal $ACC1,$IN01_3,${S3}[0]
fmov $IN01_1,x6
umlal $ACC2,$IN01_3,${S4}[0]
add x12,$padbit,x12,lsr#40
umlal $ACC3,$IN01_4,${S4}[0]
add x13,$padbit,x13,lsr#40
umlal $ACC0,$IN01_4,${S1}[0]
add x10,x10,x11,lsl#32 // bfi x10,x11,#32,#32
umlal $ACC4,$IN01_4,${R0}[0]
add x12,x12,x13,lsl#32 // bfi x12,x13,#32,#32
umlal $ACC1,$IN01_4,${S2}[0]
fmov $IN01_2,x8
umlal $ACC2,$IN01_4,${S3}[0]
fmov $IN01_3,x10
fmov $IN01_4,x12
/////////////////////////////////////////////////////////////////
// lazy reduction as discussed in "NEON crypto" by D.J. Bernstein
// and P. Schwabe
//
// [see discussion in poly1305-armv4 module]
ushr $T0.2d,$ACC3,#26
xtn $H3,$ACC3
ushr $T1.2d,$ACC0,#26
and $ACC0,$ACC0,$MASK.2d
add $ACC4,$ACC4,$T0.2d // h3 -> h4
bic $H3,#0xfc,lsl#24 // &=0x03ffffff
add $ACC1,$ACC1,$T1.2d // h0 -> h1
ushr $T0.2d,$ACC4,#26
xtn $H4,$ACC4
ushr $T1.2d,$ACC1,#26
xtn $H1,$ACC1
bic $H4,#0xfc,lsl#24
add $ACC2,$ACC2,$T1.2d // h1 -> h2
add $ACC0,$ACC0,$T0.2d
shl $T0.2d,$T0.2d,#2
shrn $T1.2s,$ACC2,#26
xtn $H2,$ACC2
add $ACC0,$ACC0,$T0.2d // h4 -> h0
bic $H1,#0xfc,lsl#24
add $H3,$H3,$T1.2s // h2 -> h3
bic $H2,#0xfc,lsl#24
shrn $T0.2s,$ACC0,#26
xtn $H0,$ACC0
ushr $T1.2s,$H3,#26
bic $H3,#0xfc,lsl#24
bic $H0,#0xfc,lsl#24
add $H1,$H1,$T0.2s // h0 -> h1
add $H4,$H4,$T1.2s // h3 -> h4
b.hi .Loop_neon
.Lskip_loop:
dup $IN23_2,${IN23_2}[0]
add $IN01_2,$IN01_2,$H2
////////////////////////////////////////////////////////////////
// multiply (inp[0:1]+hash) or inp[2:3] by r^2:r^1
adds $len,$len,#32
b.ne .Long_tail
dup $IN23_2,${IN01_2}[0]
add $IN23_0,$IN01_0,$H0
add $IN23_3,$IN01_3,$H3
add $IN23_1,$IN01_1,$H1
add $IN23_4,$IN01_4,$H4
.Long_tail:
dup $IN23_0,${IN23_0}[0]
umull2 $ACC0,$IN23_2,${S3}
umull2 $ACC3,$IN23_2,${R1}
umull2 $ACC4,$IN23_2,${R2}
umull2 $ACC2,$IN23_2,${R0}
umull2 $ACC1,$IN23_2,${S4}
dup $IN23_1,${IN23_1}[0]
umlal2 $ACC0,$IN23_0,${R0}
umlal2 $ACC2,$IN23_0,${R2}
umlal2 $ACC3,$IN23_0,${R3}
umlal2 $ACC4,$IN23_0,${R4}
umlal2 $ACC1,$IN23_0,${R1}
dup $IN23_3,${IN23_3}[0]
umlal2 $ACC0,$IN23_1,${S4}
umlal2 $ACC3,$IN23_1,${R2}
umlal2 $ACC2,$IN23_1,${R1}
umlal2 $ACC4,$IN23_1,${R3}
umlal2 $ACC1,$IN23_1,${R0}
dup $IN23_4,${IN23_4}[0]
umlal2 $ACC3,$IN23_3,${R0}
umlal2 $ACC4,$IN23_3,${R1}
umlal2 $ACC0,$IN23_3,${S2}
umlal2 $ACC1,$IN23_3,${S3}
umlal2 $ACC2,$IN23_3,${S4}
umlal2 $ACC3,$IN23_4,${S4}
umlal2 $ACC0,$IN23_4,${S1}
umlal2 $ACC4,$IN23_4,${R0}
umlal2 $ACC1,$IN23_4,${S2}
umlal2 $ACC2,$IN23_4,${S3}
b.eq .Lshort_tail
////////////////////////////////////////////////////////////////
// (hash+inp[0:1])*r^4:r^3 and accumulate
add $IN01_0,$IN01_0,$H0
umlal $ACC3,$IN01_2,${R1}
umlal $ACC0,$IN01_2,${S3}
umlal $ACC4,$IN01_2,${R2}
umlal $ACC1,$IN01_2,${S4}
umlal $ACC2,$IN01_2,${R0}
add $IN01_1,$IN01_1,$H1
umlal $ACC3,$IN01_0,${R3}
umlal $ACC0,$IN01_0,${R0}
umlal $ACC4,$IN01_0,${R4}
umlal $ACC1,$IN01_0,${R1}
umlal $ACC2,$IN01_0,${R2}
add $IN01_3,$IN01_3,$H3
umlal $ACC3,$IN01_1,${R2}
umlal $ACC0,$IN01_1,${S4}
umlal $ACC4,$IN01_1,${R3}
umlal $ACC1,$IN01_1,${R0}
umlal $ACC2,$IN01_1,${R1}
add $IN01_4,$IN01_4,$H4
umlal $ACC3,$IN01_3,${R0}
umlal $ACC0,$IN01_3,${S2}
umlal $ACC4,$IN01_3,${R1}
umlal $ACC1,$IN01_3,${S3}
umlal $ACC2,$IN01_3,${S4}
umlal $ACC3,$IN01_4,${S4}
umlal $ACC0,$IN01_4,${S1}
umlal $ACC4,$IN01_4,${R0}
umlal $ACC1,$IN01_4,${S2}
umlal $ACC2,$IN01_4,${S3}
.Lshort_tail:
////////////////////////////////////////////////////////////////
// horizontal add
addp $ACC3,$ACC3,$ACC3
ldp d8,d9,[sp,#16] // meet ABI requirements
addp $ACC0,$ACC0,$ACC0
ldp d10,d11,[sp,#32]
addp $ACC4,$ACC4,$ACC4
ldp d12,d13,[sp,#48]
addp $ACC1,$ACC1,$ACC1
ldp d14,d15,[sp,#64]
addp $ACC2,$ACC2,$ACC2
ldr x30,[sp,#8]
////////////////////////////////////////////////////////////////
// lazy reduction, but without narrowing
ushr $T0.2d,$ACC3,#26
and $ACC3,$ACC3,$MASK.2d
ushr $T1.2d,$ACC0,#26
and $ACC0,$ACC0,$MASK.2d
add $ACC4,$ACC4,$T0.2d // h3 -> h4
add $ACC1,$ACC1,$T1.2d // h0 -> h1
ushr $T0.2d,$ACC4,#26
and $ACC4,$ACC4,$MASK.2d
ushr $T1.2d,$ACC1,#26
and $ACC1,$ACC1,$MASK.2d
add $ACC2,$ACC2,$T1.2d // h1 -> h2
add $ACC0,$ACC0,$T0.2d
shl $T0.2d,$T0.2d,#2
ushr $T1.2d,$ACC2,#26
and $ACC2,$ACC2,$MASK.2d
add $ACC0,$ACC0,$T0.2d // h4 -> h0
add $ACC3,$ACC3,$T1.2d // h2 -> h3
ushr $T0.2d,$ACC0,#26
and $ACC0,$ACC0,$MASK.2d
ushr $T1.2d,$ACC3,#26
and $ACC3,$ACC3,$MASK.2d
add $ACC1,$ACC1,$T0.2d // h0 -> h1
add $ACC4,$ACC4,$T1.2d // h3 -> h4
////////////////////////////////////////////////////////////////
// write the result, can be partially reduced
st4 {$ACC0,$ACC1,$ACC2,$ACC3}[0],[$ctx],#16
mov x4,#1
st1 {$ACC4}[0],[$ctx]
str x4,[$ctx,#8] // set is_base2_26
ldr x29,[sp],#80
.inst 0xd50323bf // autiasp
ret
.size poly1305_blocks_neon,.-poly1305_blocks_neon
.pushsection .rodata
.align 5
.Lzeros:
.long 0,0,0,0,0,0,0,0
.asciz "Poly1305 for ARMv8, CRYPTOGAMS by \@dot-asm"
.popsection
.align 2
#if !defined(__KERNEL__) && !defined(_WIN64)
.comm OPENSSL_armcap_P,4,4
.hidden OPENSSL_armcap_P
#endif
___
foreach (split("\n",$code)) {
s/\b(shrn\s+v[0-9]+)\.[24]d/$1.2s/ or
s/\b(fmov\s+)v([0-9]+)[^,]*,\s*x([0-9]+)/$1d$2,x$3/ or
(m/\bdup\b/ and (s/\.[24]s/.2d/g or 1)) or
(m/\b(eor|and)/ and (s/\.[248][sdh]/.16b/g or 1)) or
(m/\bum(ul|la)l\b/ and (s/\.4s/.2s/g or 1)) or
(m/\bum(ul|la)l2\b/ and (s/\.2s/.4s/g or 1)) or
(m/\bst[1-4]\s+{[^}]+}\[/ and (s/\.[24]d/.s/g or 1));
s/\.[124]([sd])\[/.$1\[/;
s/w#x([0-9]+)/w$1/g;
print $_,"\n";
}
close STDOUT;
linux-7.2.3/lib/crypto/arm64/poly1305.h
+42
View File
@@ -0,0 +1,42 @@
/* SPDX-License-Identifier: GPL-2.0 */
/*
* OpenSSL/Cryptogams accelerated Poly1305 transform for arm64
*
* Copyright (C) 2019 Linaro Ltd. <ard.biesheuvel@linaro.org>
*/
#include <asm/hwcap.h>
#include <asm/simd.h>
#include <linux/cpufeature.h>
#include <linux/jump_label.h>
#include <linux/kernel.h>
asmlinkage void poly1305_block_init(struct poly1305_block_state *state,
const u8 raw_key[POLY1305_BLOCK_SIZE]);
asmlinkage void poly1305_blocks_arm64(struct poly1305_block_state *state,
const u8 *src, u32 len, u32 hibit);
asmlinkage void poly1305_blocks_neon(struct poly1305_block_state *state,
const u8 *src, u32 len, u32 hibit);
asmlinkage void poly1305_emit(const struct poly1305_state *state,
u8 digest[POLY1305_DIGEST_SIZE],
const u32 nonce[4]);
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon);
static void poly1305_blocks(struct poly1305_block_state *state, const u8 *src,
unsigned int len, u32 padbit)
{
if (static_branch_likely(&have_neon) && likely(may_use_simd())) {
scoped_ksimd()
poly1305_blocks_neon(state, src, len, padbit);
} else {
poly1305_blocks_arm64(state, src, len, padbit);
}
}
#define poly1305_mod_init_arch poly1305_mod_init_arch
static void poly1305_mod_init_arch(void)
{
if (cpu_have_named_feature(ASIMD))
static_branch_enable(&have_neon);
}
linux-7.2.3/lib/crypto/arm64/polyval-ce-core.S
+359
View File
@@ -0,0 +1,359 @@
/* SPDX-License-Identifier: GPL-2.0 */
/*
* Implementation of POLYVAL using ARMv8 Crypto Extensions.
*
* Copyright 2021 Google LLC
*/
/*
* This is an efficient implementation of POLYVAL using ARMv8 Crypto Extensions
* It works on 8 blocks at a time, by precomputing the first 8 keys powers h^8,
* ..., h^1 in the POLYVAL finite field. This precomputation allows us to split
* finite field multiplication into two steps.
*
* In the first step, we consider h^i, m_i as normal polynomials of degree less
* than 128. We then compute p(x) = h^8m_0 + ... + h^1m_7 where multiplication
* is simply polynomial multiplication.
*
* In the second step, we compute the reduction of p(x) modulo the finite field
* modulus g(x) = x^128 + x^127 + x^126 + x^121 + 1.
*
* This two step process is equivalent to computing h^8m_0 + ... + h^1m_7 where
* multiplication is finite field multiplication. The advantage is that the
* two-step process only requires 1 finite field reduction for every 8
* polynomial multiplications. Further parallelism is gained by interleaving the
* multiplications and polynomial reductions.
*/
#include <linux/linkage.h>
#define STRIDE_BLOCKS 8
ACCUMULATOR .req x0
KEY_POWERS .req x1
MSG .req x2
BLOCKS_LEFT .req x3
KEY_START .req x10
EXTRA_BYTES .req x11
TMP .req x13
M0 .req v0
M1 .req v1
M2 .req v2
M3 .req v3
M4 .req v4
M5 .req v5
M6 .req v6
M7 .req v7
KEY8 .req v8
KEY7 .req v9
KEY6 .req v10
KEY5 .req v11
KEY4 .req v12
KEY3 .req v13
KEY2 .req v14
KEY1 .req v15
PL .req v16
PH .req v17
TMP_V .req v18
LO .req v20
MI .req v21
HI .req v22
SUM .req v23
GSTAR .req v24
.text
.arch armv8-a+crypto
.align 4
.Lgstar:
.quad 0xc200000000000000, 0xc200000000000000
/*
* Computes the product of two 128-bit polynomials in X and Y and XORs the
* components of the 256-bit product into LO, MI, HI.
*
* Given:
* X = [X_1 : X_0]
* Y = [Y_1 : Y_0]
*
* We compute:
* LO += X_0 * Y_0
* MI += (X_0 + X_1) * (Y_0 + Y_1)
* HI += X_1 * Y_1
*
* Later, the 256-bit result can be extracted as:
* [HI_1 : HI_0 + HI_1 + MI_1 + LO_1 : LO_1 + HI_0 + MI_0 + LO_0 : LO_0]
* This step is done when computing the polynomial reduction for efficiency
* reasons.
*
* Karatsuba multiplication is used instead of Schoolbook multiplication because
* it was found to be slightly faster on ARM64 CPUs.
*
*/
.macro karatsuba1 X Y
X .req \X
Y .req \Y
ext v25.16b, X.16b, X.16b, #8
ext v26.16b, Y.16b, Y.16b, #8
eor v25.16b, v25.16b, X.16b
eor v26.16b, v26.16b, Y.16b
pmull2 v28.1q, X.2d, Y.2d
pmull v29.1q, X.1d, Y.1d
pmull v27.1q, v25.1d, v26.1d
eor HI.16b, HI.16b, v28.16b
eor LO.16b, LO.16b, v29.16b
eor MI.16b, MI.16b, v27.16b
.unreq X
.unreq Y
.endm
/*
* Same as karatsuba1, except overwrites HI, LO, MI rather than XORing into
* them.
*/
.macro karatsuba1_store X Y
X .req \X
Y .req \Y
ext v25.16b, X.16b, X.16b, #8
ext v26.16b, Y.16b, Y.16b, #8
eor v25.16b, v25.16b, X.16b
eor v26.16b, v26.16b, Y.16b
pmull2 HI.1q, X.2d, Y.2d
pmull LO.1q, X.1d, Y.1d
pmull MI.1q, v25.1d, v26.1d
.unreq X
.unreq Y
.endm
/*
* Computes the 256-bit polynomial represented by LO, HI, MI. Stores
* the result in PL, PH.
* [PH : PL] =
* [HI_1 : HI_1 + HI_0 + MI_1 + LO_1 : HI_0 + MI_0 + LO_1 + LO_0 : LO_0]
*/
.macro karatsuba2
// v4 = [HI_1 + MI_1 : HI_0 + MI_0]
eor v4.16b, HI.16b, MI.16b
// v4 = [HI_1 + MI_1 + LO_1 : HI_0 + MI_0 + LO_0]
eor v4.16b, v4.16b, LO.16b
// v5 = [HI_0 : LO_1]
ext v5.16b, LO.16b, HI.16b, #8
// v4 = [HI_1 + HI_0 + MI_1 + LO_1 : HI_0 + MI_0 + LO_1 + LO_0]
eor v4.16b, v4.16b, v5.16b
// HI = [HI_0 : HI_1]
ext HI.16b, HI.16b, HI.16b, #8
// LO = [LO_0 : LO_1]
ext LO.16b, LO.16b, LO.16b, #8
// PH = [HI_1 : HI_1 + HI_0 + MI_1 + LO_1]
ext PH.16b, v4.16b, HI.16b, #8
// PL = [HI_0 + MI_0 + LO_1 + LO_0 : LO_0]
ext PL.16b, LO.16b, v4.16b, #8
.endm
/*
* Computes the 128-bit reduction of PH : PL. Stores the result in dest.
*
* This macro computes p(x) mod g(x) where p(x) is in montgomery form and g(x) =
* x^128 + x^127 + x^126 + x^121 + 1.
*
* We have a 256-bit polynomial PH : PL = P_3 : P_2 : P_1 : P_0 that is the
* product of two 128-bit polynomials in Montgomery form. We need to reduce it
* mod g(x). Also, since polynomials in Montgomery form have an "extra" factor
* of x^128, this product has two extra factors of x^128. To get it back into
* Montgomery form, we need to remove one of these factors by dividing by x^128.
*
* To accomplish both of these goals, we add multiples of g(x) that cancel out
* the low 128 bits P_1 : P_0, leaving just the high 128 bits. Since the low
* bits are zero, the polynomial division by x^128 can be done by right
* shifting.
*
* Since the only nonzero term in the low 64 bits of g(x) is the constant term,
* the multiple of g(x) needed to cancel out P_0 is P_0 * g(x). The CPU can
* only do 64x64 bit multiplications, so split P_0 * g(x) into x^128 * P_0 +
* x^64 * g*(x) * P_0 + P_0, where g*(x) is bits 64-127 of g(x). Adding this to
* the original polynomial gives P_3 : P_2 + P_0 + T_1 : P_1 + T_0 : 0, where T
* = T_1 : T_0 = g*(x) * P_0. Thus, bits 0-63 got "folded" into bits 64-191.
*
* Repeating this same process on the next 64 bits "folds" bits 64-127 into bits
* 128-255, giving the answer in bits 128-255. This time, we need to cancel P_1
* + T_0 in bits 64-127. The multiple of g(x) required is (P_1 + T_0) * g(x) *
* x^64. Adding this to our previous computation gives P_3 + P_1 + T_0 + V_1 :
* P_2 + P_0 + T_1 + V_0 : 0 : 0, where V = V_1 : V_0 = g*(x) * (P_1 + T_0).
*
* So our final computation is:
* T = T_1 : T_0 = g*(x) * P_0
* V = V_1 : V_0 = g*(x) * (P_1 + T_0)
* p(x) / x^{128} mod g(x) = P_3 + P_1 + T_0 + V_1 : P_2 + P_0 + T_1 + V_0
*
* The implementation below saves a XOR instruction by computing P_1 + T_0 : P_0
* + T_1 and XORing into dest, rather than separately XORing P_1 : P_0 and T_0 :
* T_1 into dest. This allows us to reuse P_1 + T_0 when computing V.
*/
.macro montgomery_reduction dest
DEST .req \dest
// TMP_V = T_1 : T_0 = P_0 * g*(x)
pmull TMP_V.1q, PL.1d, GSTAR.1d
// TMP_V = T_0 : T_1
ext TMP_V.16b, TMP_V.16b, TMP_V.16b, #8
// TMP_V = P_1 + T_0 : P_0 + T_1
eor TMP_V.16b, PL.16b, TMP_V.16b
// PH = P_3 + P_1 + T_0 : P_2 + P_0 + T_1
eor PH.16b, PH.16b, TMP_V.16b
// TMP_V = V_1 : V_0 = (P_1 + T_0) * g*(x)
pmull2 TMP_V.1q, TMP_V.2d, GSTAR.2d
eor DEST.16b, PH.16b, TMP_V.16b
.unreq DEST
.endm
/*
* Compute Polyval on 8 blocks.
*
* If reduce is set, also computes the montgomery reduction of the
* previous full_stride call and XORs with the first message block.
* (m_0 + REDUCE(PL, PH))h^8 + ... + m_7h^1.
* I.e., the first multiplication uses m_0 + REDUCE(PL, PH) instead of m_0.
*
* Sets PL, PH.
*/
.macro full_stride reduce
eor LO.16b, LO.16b, LO.16b
eor MI.16b, MI.16b, MI.16b
eor HI.16b, HI.16b, HI.16b
ld1 {M0.16b, M1.16b, M2.16b, M3.16b}, [MSG], #64
ld1 {M4.16b, M5.16b, M6.16b, M7.16b}, [MSG], #64
karatsuba1 M7 KEY1
.if \reduce
pmull TMP_V.1q, PL.1d, GSTAR.1d
.endif
karatsuba1 M6 KEY2
.if \reduce
ext TMP_V.16b, TMP_V.16b, TMP_V.16b, #8
.endif
karatsuba1 M5 KEY3
.if \reduce
eor TMP_V.16b, PL.16b, TMP_V.16b
.endif
karatsuba1 M4 KEY4
.if \reduce
eor PH.16b, PH.16b, TMP_V.16b
.endif
karatsuba1 M3 KEY5
.if \reduce
pmull2 TMP_V.1q, TMP_V.2d, GSTAR.2d
.endif
karatsuba1 M2 KEY6
.if \reduce
eor SUM.16b, PH.16b, TMP_V.16b
.endif
karatsuba1 M1 KEY7
eor M0.16b, M0.16b, SUM.16b
karatsuba1 M0 KEY8
karatsuba2
.endm
/*
* Handle any extra blocks after full_stride loop.
*/
.macro partial_stride
add KEY_POWERS, KEY_START, #(STRIDE_BLOCKS << 4)
sub KEY_POWERS, KEY_POWERS, BLOCKS_LEFT, lsl #4
ld1 {KEY1.16b}, [KEY_POWERS], #16
ld1 {TMP_V.16b}, [MSG], #16
eor SUM.16b, SUM.16b, TMP_V.16b
karatsuba1_store KEY1 SUM
sub BLOCKS_LEFT, BLOCKS_LEFT, #1
tst BLOCKS_LEFT, #4
beq .Lpartial4BlocksDone
ld1 {M0.16b, M1.16b, M2.16b, M3.16b}, [MSG], #64
ld1 {KEY8.16b, KEY7.16b, KEY6.16b, KEY5.16b}, [KEY_POWERS], #64
karatsuba1 M0 KEY8
karatsuba1 M1 KEY7
karatsuba1 M2 KEY6
karatsuba1 M3 KEY5
.Lpartial4BlocksDone:
tst BLOCKS_LEFT, #2
beq .Lpartial2BlocksDone
ld1 {M0.16b, M1.16b}, [MSG], #32
ld1 {KEY8.16b, KEY7.16b}, [KEY_POWERS], #32
karatsuba1 M0 KEY8
karatsuba1 M1 KEY7
.Lpartial2BlocksDone:
tst BLOCKS_LEFT, #1
beq .LpartialDone
ld1 {M0.16b}, [MSG], #16
ld1 {KEY8.16b}, [KEY_POWERS], #16
karatsuba1 M0 KEY8
.LpartialDone:
karatsuba2
montgomery_reduction SUM
.endm
/*
* Computes a = a * b * x^{-128} mod x^128 + x^127 + x^126 + x^121 + 1.
*
* void polyval_mul_pmull(struct polyval_elem *a,
* const struct polyval_elem *b);
*/
SYM_FUNC_START(polyval_mul_pmull)
adr TMP, .Lgstar
ld1 {GSTAR.2d}, [TMP]
ld1 {v0.16b}, [x0]
ld1 {v1.16b}, [x1]
karatsuba1_store v0 v1
karatsuba2
montgomery_reduction SUM
st1 {SUM.16b}, [x0]
ret
SYM_FUNC_END(polyval_mul_pmull)
/*
* Perform polynomial evaluation as specified by POLYVAL. This computes:
* h^n * accumulator + h^n * m_0 + ... + h^1 * m_{n-1}
* where n=nblocks, h is the hash key, and m_i are the message blocks.
*
* x0 - pointer to accumulator
* x1 - pointer to precomputed key powers h^8 ... h^1
* x2 - pointer to message blocks
* x3 - number of blocks to hash
*
* void polyval_blocks_pmull(struct polyval_elem *acc,
* const struct polyval_key *key,
* const u8 *data, size_t nblocks);
*/
SYM_FUNC_START(polyval_blocks_pmull)
adr TMP, .Lgstar
mov KEY_START, KEY_POWERS
ld1 {GSTAR.2d}, [TMP]
ld1 {SUM.16b}, [ACCUMULATOR]
subs BLOCKS_LEFT, BLOCKS_LEFT, #STRIDE_BLOCKS
blt .LstrideLoopExit
ld1 {KEY8.16b, KEY7.16b, KEY6.16b, KEY5.16b}, [KEY_POWERS], #64
ld1 {KEY4.16b, KEY3.16b, KEY2.16b, KEY1.16b}, [KEY_POWERS], #64
full_stride 0
subs BLOCKS_LEFT, BLOCKS_LEFT, #STRIDE_BLOCKS
blt .LstrideLoopExitReduce
.LstrideLoop:
full_stride 1
subs BLOCKS_LEFT, BLOCKS_LEFT, #STRIDE_BLOCKS
bge .LstrideLoop
.LstrideLoopExitReduce:
montgomery_reduction SUM
.LstrideLoopExit:
adds BLOCKS_LEFT, BLOCKS_LEFT, #STRIDE_BLOCKS
beq .LskipPartial
partial_stride
.LskipPartial:
st1 {SUM.16b}, [ACCUMULATOR]
ret
SYM_FUNC_END(polyval_blocks_pmull)
linux-7.2.3/lib/crypto/arm64/sha1-ce-core.S
+126
View File
@@ -0,0 +1,126 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* SHA-1 secure hash using ARMv8 Crypto Extensions
*
* Copyright (C) 2014 Linaro Ltd <ard.biesheuvel@linaro.org>
*/
#include <linux/linkage.h>
#include <asm/assembler.h>
.text
.arch armv8-a+crypto
k0 .req v0
k1 .req v1
k2 .req v2
k3 .req v3
t0 .req v4
t1 .req v5
dga .req q6
dgav .req v6
dgb .req s7
dgbv .req v7
dg0q .req q12
dg0s .req s12
dg0v .req v12
dg1s .req s13
dg1v .req v13
dg2s .req s14
.macro add_only, op, ev, rc, s0, dg1
.ifc \ev, ev
add t1.4s, v\s0\().4s, \rc\().4s
sha1h dg2s, dg0s
.ifnb \dg1
sha1\op dg0q, \dg1, t0.4s
.else
sha1\op dg0q, dg1s, t0.4s
.endif
.else
.ifnb \s0
add t0.4s, v\s0\().4s, \rc\().4s
.endif
sha1h dg1s, dg0s
sha1\op dg0q, dg2s, t1.4s
.endif
.endm
.macro add_update, op, ev, rc, s0, s1, s2, s3, dg1
sha1su0 v\s0\().4s, v\s1\().4s, v\s2\().4s
add_only \op, \ev, \rc, \s1, \dg1
sha1su1 v\s0\().4s, v\s3\().4s
.endm
.macro loadrc, k, val, tmp
movz \tmp, :abs_g0_nc:\val
movk \tmp, :abs_g1:\val
dup \k, \tmp
.endm
/*
* void sha1_ce_transform(struct sha1_block_state *state,
* const u8 *data, size_t nblocks);
*/
SYM_FUNC_START(sha1_ce_transform)
/* load round constants */
loadrc k0.4s, 0x5a827999, w6
loadrc k1.4s, 0x6ed9eba1, w6
loadrc k2.4s, 0x8f1bbcdc, w6
loadrc k3.4s, 0xca62c1d6, w6
/* load state */
ld1 {dgav.4s}, [x0]
ldr dgb, [x0, #16]
/* load input */
0: ld1 {v8.4s-v11.4s}, [x1], #64
sub x2, x2, #1
rev32 v8.16b, v8.16b
rev32 v9.16b, v9.16b
rev32 v10.16b, v10.16b
rev32 v11.16b, v11.16b
add t0.4s, v8.4s, k0.4s
mov dg0v.16b, dgav.16b
add_update c, ev, k0, 8, 9, 10, 11, dgb
add_update c, od, k0, 9, 10, 11, 8
add_update c, ev, k0, 10, 11, 8, 9
add_update c, od, k0, 11, 8, 9, 10
add_update c, ev, k1, 8, 9, 10, 11
add_update p, od, k1, 9, 10, 11, 8
add_update p, ev, k1, 10, 11, 8, 9
add_update p, od, k1, 11, 8, 9, 10
add_update p, ev, k1, 8, 9, 10, 11
add_update p, od, k2, 9, 10, 11, 8
add_update m, ev, k2, 10, 11, 8, 9
add_update m, od, k2, 11, 8, 9, 10
add_update m, ev, k2, 8, 9, 10, 11
add_update m, od, k2, 9, 10, 11, 8
add_update m, ev, k3, 10, 11, 8, 9
add_update p, od, k3, 11, 8, 9, 10
add_only p, ev, k3, 9
add_only p, od, k3, 10
add_only p, ev, k3, 11
add_only p, od
/* update state */
add dgbv.2s, dgbv.2s, dg1v.2s
add dgav.4s, dgav.4s, dg0v.4s
/* handled all input blocks? */
cbnz x2, 0b
/* store new state */
st1 {dgav.4s}, [x0]
str dgb, [x0, #16]
ret
SYM_FUNC_END(sha1_ce_transform)
linux-7.2.3/lib/crypto/arm64/sha1.h
+31
View File
@@ -0,0 +1,31 @@
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
* SHA-1 optimized for ARM64
*
* Copyright 2025 Google LLC
*/
#include <asm/simd.h>
#include <linux/cpufeature.h>
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_ce);
asmlinkage void sha1_ce_transform(struct sha1_block_state *state,
const u8 *data, size_t nblocks);
static void sha1_blocks(struct sha1_block_state *state,
const u8 *data, size_t nblocks)
{
if (static_branch_likely(&have_ce) && likely(may_use_simd())) {
scoped_ksimd()
sha1_ce_transform(state, data, nblocks);
} else {
sha1_blocks_generic(state, data, nblocks);
}
}
#define sha1_mod_init_arch sha1_mod_init_arch
static void sha1_mod_init_arch(void)
{
if (cpu_have_named_feature(SHA1))
static_branch_enable(&have_ce);
}
linux-7.2.3/lib/crypto/arm64/sha2-armv8.pl
+786
View File
@@ -0,0 +1,786 @@
#! /usr/bin/env perl
# SPDX-License-Identifier: GPL-2.0
# This code is taken from the OpenSSL project but the author (Andy Polyakov)
# has relicensed it under the GPLv2. Therefore this program is free software;
# you can redistribute it and/or modify it under the terms of the GNU General
# Public License version 2 as published by the Free Software Foundation.
#
# The original headers, including the original license headers, are
# included below for completeness.
# Copyright 2014-2016 The OpenSSL Project Authors. All Rights Reserved.
#
# Licensed under the OpenSSL license (the "License"). You may not use
# this file except in compliance with the License. You can obtain a copy
# in the file LICENSE in the source distribution or at
# https://www.openssl.org/source/license.html
# ====================================================================
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see http://www.openssl.org/~appro/cryptogams/.
# ====================================================================
#
# SHA256/512 for ARMv8.
#
# Performance in cycles per processed byte and improvement coefficient
# over code generated with "default" compiler:
#
# SHA256-hw SHA256(*) SHA512
# Apple A7 1.97 10.5 (+33%) 6.73 (-1%(**))
# Cortex-A53 2.38 15.5 (+115%) 10.0 (+150%(***))
# Cortex-A57 2.31 11.6 (+86%) 7.51 (+260%(***))
# Denver 2.01 10.5 (+26%) 6.70 (+8%)
# X-Gene 20.0 (+100%) 12.8 (+300%(***))
# Mongoose 2.36 13.0 (+50%) 8.36 (+33%)
#
# (*) Software SHA256 results are of lesser relevance, presented
# mostly for informational purposes.
# (**) The result is a trade-off: it's possible to improve it by
# 10% (or by 1 cycle per round), but at the cost of 20% loss
# on Cortex-A53 (or by 4 cycles per round).
# (***) Super-impressive coefficients over gcc-generated code are
# indication of some compiler "pathology", most notably code
# generated with -mgeneral-regs-only is significantly faster
# and the gap is only 40-90%.
#
# October 2016.
#
# Originally it was reckoned that it makes no sense to implement NEON
# version of SHA256 for 64-bit processors. This is because performance
# improvement on most wide-spread Cortex-A5x processors was observed
# to be marginal, same on Cortex-A53 and ~10% on A57. But then it was
# observed that 32-bit NEON SHA256 performs significantly better than
# 64-bit scalar version on *some* of the more recent processors. As
# result 64-bit NEON version of SHA256 was added to provide best
# all-round performance. For example it executes ~30% faster on X-Gene
# and Mongoose. [For reference, NEON version of SHA512 is bound to
# deliver much less improvement, likely *negative* on Cortex-A5x.
# Which is why NEON support is limited to SHA256.]
$output=pop;
$flavour=pop;
if ($flavour && $flavour ne "void") {
$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or
( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate) or
die "can't locate arm-xlate.pl";
open OUT,"| \"$^X\" $xlate $flavour $output";
*STDOUT=*OUT;
} else {
open STDOUT,">$output";
}
if ($output =~ /512/) {
$BITS=512;
$SZ=8;
@Sigma0=(28,34,39);
@Sigma1=(14,18,41);
@sigma0=(1, 8, 7);
@sigma1=(19,61, 6);
$rounds=80;
$reg_t="x";
} else {
$BITS=256;
$SZ=4;
@Sigma0=( 2,13,22);
@Sigma1=( 6,11,25);
@sigma0=( 7,18, 3);
@sigma1=(17,19,10);
$rounds=64;
$reg_t="w";
}
$func="sha${BITS}_block_data_order";
($ctx,$inp,$num,$Ktbl)=map("x$_",(0..2,30));
@X=map("$reg_t$_",(3..15,0..2));
@V=($A,$B,$C,$D,$E,$F,$G,$H)=map("$reg_t$_",(20..27));
($t0,$t1,$t2,$t3)=map("$reg_t$_",(16,17,19,28));
sub BODY_00_xx {
my ($i,$a,$b,$c,$d,$e,$f,$g,$h)=@_;
my $j=($i+1)&15;
my ($T0,$T1,$T2)=(@X[($i-8)&15],@X[($i-9)&15],@X[($i-10)&15]);
$T0=@X[$i+3] if ($i<11);
$code.=<<___ if ($i<16);
#ifndef __AARCH64EB__
rev @X[$i],@X[$i] // $i
#endif
___
$code.=<<___ if ($i<13 && ($i&1));
ldp @X[$i+1],@X[$i+2],[$inp],#2*$SZ
___
$code.=<<___ if ($i==13);
ldp @X[14],@X[15],[$inp]
___
$code.=<<___ if ($i>=14);
ldr @X[($i-11)&15],[sp,#`$SZ*(($i-11)%4)`]
___
$code.=<<___ if ($i>0 && $i<16);
add $a,$a,$t1 // h+=Sigma0(a)
___
$code.=<<___ if ($i>=11);
str @X[($i-8)&15],[sp,#`$SZ*(($i-8)%4)`]
___
# While ARMv8 specifies merged rotate-n-logical operation such as
# 'eor x,y,z,ror#n', it was found to negatively affect performance
# on Apple A7. The reason seems to be that it requires even 'y' to
# be available earlier. This means that such merged instruction is
# not necessarily best choice on critical path... On the other hand
# Cortex-A5x handles merged instructions much better than disjoint
# rotate and logical... See (**) footnote above.
$code.=<<___ if ($i<15);
ror $t0,$e,#$Sigma1[0]
add $h,$h,$t2 // h+=K[i]
eor $T0,$e,$e,ror#`$Sigma1[2]-$Sigma1[1]`
and $t1,$f,$e
bic $t2,$g,$e
add $h,$h,@X[$i&15] // h+=X[i]
orr $t1,$t1,$t2 // Ch(e,f,g)
eor $t2,$a,$b // a^b, b^c in next round
eor $t0,$t0,$T0,ror#$Sigma1[1] // Sigma1(e)
ror $T0,$a,#$Sigma0[0]
add $h,$h,$t1 // h+=Ch(e,f,g)
eor $t1,$a,$a,ror#`$Sigma0[2]-$Sigma0[1]`
add $h,$h,$t0 // h+=Sigma1(e)
and $t3,$t3,$t2 // (b^c)&=(a^b)
add $d,$d,$h // d+=h
eor $t3,$t3,$b // Maj(a,b,c)
eor $t1,$T0,$t1,ror#$Sigma0[1] // Sigma0(a)
add $h,$h,$t3 // h+=Maj(a,b,c)
ldr $t3,[$Ktbl],#$SZ // *K++, $t2 in next round
//add $h,$h,$t1 // h+=Sigma0(a)
___
$code.=<<___ if ($i>=15);
ror $t0,$e,#$Sigma1[0]
add $h,$h,$t2 // h+=K[i]
ror $T1,@X[($j+1)&15],#$sigma0[0]
and $t1,$f,$e
ror $T2,@X[($j+14)&15],#$sigma1[0]
bic $t2,$g,$e
ror $T0,$a,#$Sigma0[0]
add $h,$h,@X[$i&15] // h+=X[i]
eor $t0,$t0,$e,ror#$Sigma1[1]
eor $T1,$T1,@X[($j+1)&15],ror#$sigma0[1]
orr $t1,$t1,$t2 // Ch(e,f,g)
eor $t2,$a,$b // a^b, b^c in next round
eor $t0,$t0,$e,ror#$Sigma1[2] // Sigma1(e)
eor $T0,$T0,$a,ror#$Sigma0[1]
add $h,$h,$t1 // h+=Ch(e,f,g)
and $t3,$t3,$t2 // (b^c)&=(a^b)
eor $T2,$T2,@X[($j+14)&15],ror#$sigma1[1]
eor $T1,$T1,@X[($j+1)&15],lsr#$sigma0[2] // sigma0(X[i+1])
add $h,$h,$t0 // h+=Sigma1(e)
eor $t3,$t3,$b // Maj(a,b,c)
eor $t1,$T0,$a,ror#$Sigma0[2] // Sigma0(a)
eor $T2,$T2,@X[($j+14)&15],lsr#$sigma1[2] // sigma1(X[i+14])
add @X[$j],@X[$j],@X[($j+9)&15]
add $d,$d,$h // d+=h
add $h,$h,$t3 // h+=Maj(a,b,c)
ldr $t3,[$Ktbl],#$SZ // *K++, $t2 in next round
add @X[$j],@X[$j],$T1
add $h,$h,$t1 // h+=Sigma0(a)
add @X[$j],@X[$j],$T2
___
($t2,$t3)=($t3,$t2);
}
$code.=<<___;
#ifndef __KERNEL__
# include "arm_arch.h"
#endif
.text
.extern OPENSSL_armcap_P
.globl $func
.type $func,%function
.align 6
$func:
___
$code.=<<___ if ($SZ==4);
#ifndef __KERNEL__
# ifdef __ILP32__
ldrsw x16,.LOPENSSL_armcap_P
# else
ldr x16,.LOPENSSL_armcap_P
# endif
adr x17,.LOPENSSL_armcap_P
add x16,x16,x17
ldr w16,[x16]
tst w16,#ARMV8_SHA256
b.ne .Lv8_entry
tst w16,#ARMV7_NEON
b.ne .Lneon_entry
#endif
___
$code.=<<___;
stp x29,x30,[sp,#-128]!
add x29,sp,#0
stp x19,x20,[sp,#16]
stp x21,x22,[sp,#32]
stp x23,x24,[sp,#48]
stp x25,x26,[sp,#64]
stp x27,x28,[sp,#80]
sub sp,sp,#4*$SZ
ldp $A,$B,[$ctx] // load context
ldp $C,$D,[$ctx,#2*$SZ]
ldp $E,$F,[$ctx,#4*$SZ]
add $num,$inp,$num,lsl#`log(16*$SZ)/log(2)` // end of input
ldp $G,$H,[$ctx,#6*$SZ]
adr $Ktbl,.LK$BITS
stp $ctx,$num,[x29,#96]
.Loop:
ldp @X[0],@X[1],[$inp],#2*$SZ
ldr $t2,[$Ktbl],#$SZ // *K++
eor $t3,$B,$C // magic seed
str $inp,[x29,#112]
___
for ($i=0;$i<16;$i++) { &BODY_00_xx($i,@V); unshift(@V,pop(@V)); }
$code.=".Loop_16_xx:\n";
for (;$i<32;$i++) { &BODY_00_xx($i,@V); unshift(@V,pop(@V)); }
$code.=<<___;
cbnz $t2,.Loop_16_xx
ldp $ctx,$num,[x29,#96]
ldr $inp,[x29,#112]
sub $Ktbl,$Ktbl,#`$SZ*($rounds+1)` // rewind
ldp @X[0],@X[1],[$ctx]
ldp @X[2],@X[3],[$ctx,#2*$SZ]
add $inp,$inp,#14*$SZ // advance input pointer
ldp @X[4],@X[5],[$ctx,#4*$SZ]
add $A,$A,@X[0]
ldp @X[6],@X[7],[$ctx,#6*$SZ]
add $B,$B,@X[1]
add $C,$C,@X[2]
add $D,$D,@X[3]
stp $A,$B,[$ctx]
add $E,$E,@X[4]
add $F,$F,@X[5]
stp $C,$D,[$ctx,#2*$SZ]
add $G,$G,@X[6]
add $H,$H,@X[7]
cmp $inp,$num
stp $E,$F,[$ctx,#4*$SZ]
stp $G,$H,[$ctx,#6*$SZ]
b.ne .Loop
ldp x19,x20,[x29,#16]
add sp,sp,#4*$SZ
ldp x21,x22,[x29,#32]
ldp x23,x24,[x29,#48]
ldp x25,x26,[x29,#64]
ldp x27,x28,[x29,#80]
ldp x29,x30,[sp],#128
ret
.size $func,.-$func
.align 6
.type .LK$BITS,%object
.LK$BITS:
___
$code.=<<___ if ($SZ==8);
.quad 0x428a2f98d728ae22,0x7137449123ef65cd
.quad 0xb5c0fbcfec4d3b2f,0xe9b5dba58189dbbc
.quad 0x3956c25bf348b538,0x59f111f1b605d019
.quad 0x923f82a4af194f9b,0xab1c5ed5da6d8118
.quad 0xd807aa98a3030242,0x12835b0145706fbe
.quad 0x243185be4ee4b28c,0x550c7dc3d5ffb4e2
.quad 0x72be5d74f27b896f,0x80deb1fe3b1696b1
.quad 0x9bdc06a725c71235,0xc19bf174cf692694
.quad 0xe49b69c19ef14ad2,0xefbe4786384f25e3
.quad 0x0fc19dc68b8cd5b5,0x240ca1cc77ac9c65
.quad 0x2de92c6f592b0275,0x4a7484aa6ea6e483
.quad 0x5cb0a9dcbd41fbd4,0x76f988da831153b5
.quad 0x983e5152ee66dfab,0xa831c66d2db43210
.quad 0xb00327c898fb213f,0xbf597fc7beef0ee4
.quad 0xc6e00bf33da88fc2,0xd5a79147930aa725
.quad 0x06ca6351e003826f,0x142929670a0e6e70
.quad 0x27b70a8546d22ffc,0x2e1b21385c26c926
.quad 0x4d2c6dfc5ac42aed,0x53380d139d95b3df
.quad 0x650a73548baf63de,0x766a0abb3c77b2a8
.quad 0x81c2c92e47edaee6,0x92722c851482353b
.quad 0xa2bfe8a14cf10364,0xa81a664bbc423001
.quad 0xc24b8b70d0f89791,0xc76c51a30654be30
.quad 0xd192e819d6ef5218,0xd69906245565a910
.quad 0xf40e35855771202a,0x106aa07032bbd1b8
.quad 0x19a4c116b8d2d0c8,0x1e376c085141ab53
.quad 0x2748774cdf8eeb99,0x34b0bcb5e19b48a8
.quad 0x391c0cb3c5c95a63,0x4ed8aa4ae3418acb
.quad 0x5b9cca4f7763e373,0x682e6ff3d6b2b8a3
.quad 0x748f82ee5defb2fc,0x78a5636f43172f60
.quad 0x84c87814a1f0ab72,0x8cc702081a6439ec
.quad 0x90befffa23631e28,0xa4506cebde82bde9
.quad 0xbef9a3f7b2c67915,0xc67178f2e372532b
.quad 0xca273eceea26619c,0xd186b8c721c0c207
.quad 0xeada7dd6cde0eb1e,0xf57d4f7fee6ed178
.quad 0x06f067aa72176fba,0x0a637dc5a2c898a6
.quad 0x113f9804bef90dae,0x1b710b35131c471b
.quad 0x28db77f523047d84,0x32caab7b40c72493
.quad 0x3c9ebe0a15c9bebc,0x431d67c49c100d4c
.quad 0x4cc5d4becb3e42b6,0x597f299cfc657e2a
.quad 0x5fcb6fab3ad6faec,0x6c44198c4a475817
.quad 0 // terminator
___
$code.=<<___ if ($SZ==4);
.long 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5
.long 0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5
.long 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3
.long 0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174
.long 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc
.long 0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da
.long 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7
.long 0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967
.long 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13
.long 0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85
.long 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3
.long 0xd192e819,0xd6990624,0xf40e3585,0x106aa070
.long 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5
.long 0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3
.long 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208
.long 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2
.long 0 //terminator
___
$code.=<<___;
.size .LK$BITS,.-.LK$BITS
#ifndef __KERNEL__
.align 3
.LOPENSSL_armcap_P:
# ifdef __ILP32__
.long OPENSSL_armcap_P-.
# else
.quad OPENSSL_armcap_P-.
# endif
#endif
.asciz "SHA$BITS block transform for ARMv8, CRYPTOGAMS by <appro\@openssl.org>"
.align 2
___
if ($SZ==4) {
my $Ktbl="x3";
my ($ABCD,$EFGH,$abcd)=map("v$_.16b",(0..2));
my @MSG=map("v$_.16b",(4..7));
my ($W0,$W1)=("v16.4s","v17.4s");
my ($ABCD_SAVE,$EFGH_SAVE)=("v18.16b","v19.16b");
$code.=<<___;
#ifndef __KERNEL__
.type sha256_block_armv8,%function
.align 6
sha256_block_armv8:
.Lv8_entry:
stp x29,x30,[sp,#-16]!
add x29,sp,#0
ld1.32 {$ABCD,$EFGH},[$ctx]
adr $Ktbl,.LK256
.Loop_hw:
ld1 {@MSG[0]-@MSG[3]},[$inp],#64
sub $num,$num,#1
ld1.32 {$W0},[$Ktbl],#16
rev32 @MSG[0],@MSG[0]
rev32 @MSG[1],@MSG[1]
rev32 @MSG[2],@MSG[2]
rev32 @MSG[3],@MSG[3]
orr $ABCD_SAVE,$ABCD,$ABCD // offload
orr $EFGH_SAVE,$EFGH,$EFGH
___
for($i=0;$i<12;$i++) {
$code.=<<___;
ld1.32 {$W1},[$Ktbl],#16
add.i32 $W0,$W0,@MSG[0]
sha256su0 @MSG[0],@MSG[1]
orr $abcd,$ABCD,$ABCD
sha256h $ABCD,$EFGH,$W0
sha256h2 $EFGH,$abcd,$W0
sha256su1 @MSG[0],@MSG[2],@MSG[3]
___
($W0,$W1)=($W1,$W0); push(@MSG,shift(@MSG));
}
$code.=<<___;
ld1.32 {$W1},[$Ktbl],#16
add.i32 $W0,$W0,@MSG[0]
orr $abcd,$ABCD,$ABCD
sha256h $ABCD,$EFGH,$W0
sha256h2 $EFGH,$abcd,$W0
ld1.32 {$W0},[$Ktbl],#16
add.i32 $W1,$W1,@MSG[1]
orr $abcd,$ABCD,$ABCD
sha256h $ABCD,$EFGH,$W1
sha256h2 $EFGH,$abcd,$W1
ld1.32 {$W1},[$Ktbl]
add.i32 $W0,$W0,@MSG[2]
sub $Ktbl,$Ktbl,#$rounds*$SZ-16 // rewind
orr $abcd,$ABCD,$ABCD
sha256h $ABCD,$EFGH,$W0
sha256h2 $EFGH,$abcd,$W0
add.i32 $W1,$W1,@MSG[3]
orr $abcd,$ABCD,$ABCD
sha256h $ABCD,$EFGH,$W1
sha256h2 $EFGH,$abcd,$W1
add.i32 $ABCD,$ABCD,$ABCD_SAVE
add.i32 $EFGH,$EFGH,$EFGH_SAVE
cbnz $num,.Loop_hw
st1.32 {$ABCD,$EFGH},[$ctx]
ldr x29,[sp],#16
ret
.size sha256_block_armv8,.-sha256_block_armv8
#endif
___
}
if ($SZ==4) { ######################################### NEON stuff #
# You'll surely note a lot of similarities with sha256-armv4 module,
# and of course it's not a coincidence. sha256-armv4 was used as
# initial template, but was adapted for ARMv8 instruction set and
# extensively re-tuned for all-round performance.
my @V = ($A,$B,$C,$D,$E,$F,$G,$H) = map("w$_",(3..10));
my ($t0,$t1,$t2,$t3,$t4) = map("w$_",(11..15));
my $Ktbl="x16";
my $Xfer="x17";
my @X = map("q$_",(0..3));
my ($T0,$T1,$T2,$T3,$T4,$T5,$T6,$T7) = map("q$_",(4..7,16..19));
my $j=0;
sub AUTOLOAD() # thunk [simplified] x86-style perlasm
{ my $opcode = $AUTOLOAD; $opcode =~ s/.*:://; $opcode =~ s/_/\./;
my $arg = pop;
$arg = "#$arg" if ($arg*1 eq $arg);
$code .= "\t$opcode\t".join(',',@_,$arg)."\n";
}
sub Dscalar { shift =~ m|[qv]([0-9]+)|?"d$1":""; }
sub Dlo { shift =~ m|[qv]([0-9]+)|?"v$1.d[0]":""; }
sub Dhi { shift =~ m|[qv]([0-9]+)|?"v$1.d[1]":""; }
sub Xupdate()
{ use integer;
my $body = shift;
my @insns = (&$body,&$body,&$body,&$body);
my ($a,$b,$c,$d,$e,$f,$g,$h);
&ext_8 ($T0,@X[0],@X[1],4); # X[1..4]
eval(shift(@insns));
eval(shift(@insns));
eval(shift(@insns));
&ext_8 ($T3,@X[2],@X[3],4); # X[9..12]
eval(shift(@insns));
eval(shift(@insns));
&mov (&Dscalar($T7),&Dhi(@X[3])); # X[14..15]
eval(shift(@insns));
eval(shift(@insns));
&ushr_32 ($T2,$T0,$sigma0[0]);
eval(shift(@insns));
&ushr_32 ($T1,$T0,$sigma0[2]);
eval(shift(@insns));
&add_32 (@X[0],@X[0],$T3); # X[0..3] += X[9..12]
eval(shift(@insns));
&sli_32 ($T2,$T0,32-$sigma0[0]);
eval(shift(@insns));
eval(shift(@insns));
&ushr_32 ($T3,$T0,$sigma0[1]);
eval(shift(@insns));
eval(shift(@insns));
&eor_8 ($T1,$T1,$T2);
eval(shift(@insns));
eval(shift(@insns));
&sli_32 ($T3,$T0,32-$sigma0[1]);
eval(shift(@insns));
eval(shift(@insns));
&ushr_32 ($T4,$T7,$sigma1[0]);
eval(shift(@insns));
eval(shift(@insns));
&eor_8 ($T1,$T1,$T3); # sigma0(X[1..4])
eval(shift(@insns));
eval(shift(@insns));
&sli_32 ($T4,$T7,32-$sigma1[0]);
eval(shift(@insns));
eval(shift(@insns));
&ushr_32 ($T5,$T7,$sigma1[2]);
eval(shift(@insns));
eval(shift(@insns));
&ushr_32 ($T3,$T7,$sigma1[1]);
eval(shift(@insns));
eval(shift(@insns));
&add_32 (@X[0],@X[0],$T1); # X[0..3] += sigma0(X[1..4])
eval(shift(@insns));
eval(shift(@insns));
&sli_u32 ($T3,$T7,32-$sigma1[1]);
eval(shift(@insns));
eval(shift(@insns));
&eor_8 ($T5,$T5,$T4);
eval(shift(@insns));
eval(shift(@insns));
eval(shift(@insns));
&eor_8 ($T5,$T5,$T3); # sigma1(X[14..15])
eval(shift(@insns));
eval(shift(@insns));
eval(shift(@insns));
&add_32 (@X[0],@X[0],$T5); # X[0..1] += sigma1(X[14..15])
eval(shift(@insns));
eval(shift(@insns));
eval(shift(@insns));
&ushr_32 ($T6,@X[0],$sigma1[0]);
eval(shift(@insns));
&ushr_32 ($T7,@X[0],$sigma1[2]);
eval(shift(@insns));
eval(shift(@insns));
&sli_32 ($T6,@X[0],32-$sigma1[0]);
eval(shift(@insns));
&ushr_32 ($T5,@X[0],$sigma1[1]);
eval(shift(@insns));
eval(shift(@insns));
&eor_8 ($T7,$T7,$T6);
eval(shift(@insns));
eval(shift(@insns));
&sli_32 ($T5,@X[0],32-$sigma1[1]);
eval(shift(@insns));
eval(shift(@insns));
&ld1_32 ("{$T0}","[$Ktbl], #16");
eval(shift(@insns));
&eor_8 ($T7,$T7,$T5); # sigma1(X[16..17])
eval(shift(@insns));
eval(shift(@insns));
&eor_8 ($T5,$T5,$T5);
eval(shift(@insns));
eval(shift(@insns));
&mov (&Dhi($T5), &Dlo($T7));
eval(shift(@insns));
eval(shift(@insns));
eval(shift(@insns));
&add_32 (@X[0],@X[0],$T5); # X[2..3] += sigma1(X[16..17])
eval(shift(@insns));
eval(shift(@insns));
eval(shift(@insns));
&add_32 ($T0,$T0,@X[0]);
while($#insns>=1) { eval(shift(@insns)); }
&st1_32 ("{$T0}","[$Xfer], #16");
eval(shift(@insns));
push(@X,shift(@X)); # "rotate" X[]
}
sub Xpreload()
{ use integer;
my $body = shift;
my @insns = (&$body,&$body,&$body,&$body);
my ($a,$b,$c,$d,$e,$f,$g,$h);
eval(shift(@insns));
eval(shift(@insns));
&ld1_8 ("{@X[0]}","[$inp],#16");
eval(shift(@insns));
eval(shift(@insns));
&ld1_32 ("{$T0}","[$Ktbl],#16");
eval(shift(@insns));
eval(shift(@insns));
eval(shift(@insns));
eval(shift(@insns));
&rev32 (@X[0],@X[0]);
eval(shift(@insns));
eval(shift(@insns));
eval(shift(@insns));
eval(shift(@insns));
&add_32 ($T0,$T0,@X[0]);
foreach (@insns) { eval; } # remaining instructions
&st1_32 ("{$T0}","[$Xfer], #16");
push(@X,shift(@X)); # "rotate" X[]
}
sub body_00_15 () {
(
'($a,$b,$c,$d,$e,$f,$g,$h)=@V;'.
'&add ($h,$h,$t1)', # h+=X[i]+K[i]
'&add ($a,$a,$t4);'. # h+=Sigma0(a) from the past
'&and ($t1,$f,$e)',
'&bic ($t4,$g,$e)',
'&eor ($t0,$e,$e,"ror#".($Sigma1[1]-$Sigma1[0]))',
'&add ($a,$a,$t2)', # h+=Maj(a,b,c) from the past
'&orr ($t1,$t1,$t4)', # Ch(e,f,g)
'&eor ($t0,$t0,$e,"ror#".($Sigma1[2]-$Sigma1[0]))', # Sigma1(e)
'&eor ($t4,$a,$a,"ror#".($Sigma0[1]-$Sigma0[0]))',
'&add ($h,$h,$t1)', # h+=Ch(e,f,g)
'&ror ($t0,$t0,"#$Sigma1[0]")',
'&eor ($t2,$a,$b)', # a^b, b^c in next round
'&eor ($t4,$t4,$a,"ror#".($Sigma0[2]-$Sigma0[0]))', # Sigma0(a)
'&add ($h,$h,$t0)', # h+=Sigma1(e)
'&ldr ($t1,sprintf "[sp,#%d]",4*(($j+1)&15)) if (($j&15)!=15);'.
'&ldr ($t1,"[$Ktbl]") if ($j==15);'.
'&and ($t3,$t3,$t2)', # (b^c)&=(a^b)
'&ror ($t4,$t4,"#$Sigma0[0]")',
'&add ($d,$d,$h)', # d+=h
'&eor ($t3,$t3,$b)', # Maj(a,b,c)
'$j++; unshift(@V,pop(@V)); ($t2,$t3)=($t3,$t2);'
)
}
$code.=<<___;
#ifdef __KERNEL__
.globl sha256_block_neon
#endif
.type sha256_block_neon,%function
.align 4
sha256_block_neon:
.Lneon_entry:
stp x29, x30, [sp, #-16]!
mov x29, sp
sub sp,sp,#16*4
adr $Ktbl,.LK256
add $num,$inp,$num,lsl#6 // len to point at the end of inp
ld1.8 {@X[0]},[$inp], #16
ld1.8 {@X[1]},[$inp], #16
ld1.8 {@X[2]},[$inp], #16
ld1.8 {@X[3]},[$inp], #16
ld1.32 {$T0},[$Ktbl], #16
ld1.32 {$T1},[$Ktbl], #16
ld1.32 {$T2},[$Ktbl], #16
ld1.32 {$T3},[$Ktbl], #16
rev32 @X[0],@X[0] // yes, even on
rev32 @X[1],@X[1] // big-endian
rev32 @X[2],@X[2]
rev32 @X[3],@X[3]
mov $Xfer,sp
add.32 $T0,$T0,@X[0]
add.32 $T1,$T1,@X[1]
add.32 $T2,$T2,@X[2]
st1.32 {$T0-$T1},[$Xfer], #32
add.32 $T3,$T3,@X[3]
st1.32 {$T2-$T3},[$Xfer]
sub $Xfer,$Xfer,#32
ldp $A,$B,[$ctx]
ldp $C,$D,[$ctx,#8]
ldp $E,$F,[$ctx,#16]
ldp $G,$H,[$ctx,#24]
ldr $t1,[sp,#0]
mov $t2,wzr
eor $t3,$B,$C
mov $t4,wzr
b .L_00_48
.align 4
.L_00_48:
___
&Xupdate(\&body_00_15);
&Xupdate(\&body_00_15);
&Xupdate(\&body_00_15);
&Xupdate(\&body_00_15);
$code.=<<___;
cmp $t1,#0 // check for K256 terminator
ldr $t1,[sp,#0]
sub $Xfer,$Xfer,#64
bne .L_00_48
sub $Ktbl,$Ktbl,#256 // rewind $Ktbl
cmp $inp,$num
mov $Xfer, #64
csel $Xfer, $Xfer, xzr, eq
sub $inp,$inp,$Xfer // avoid SEGV
mov $Xfer,sp
___
&Xpreload(\&body_00_15);
&Xpreload(\&body_00_15);
&Xpreload(\&body_00_15);
&Xpreload(\&body_00_15);
$code.=<<___;
add $A,$A,$t4 // h+=Sigma0(a) from the past
ldp $t0,$t1,[$ctx,#0]
add $A,$A,$t2 // h+=Maj(a,b,c) from the past
ldp $t2,$t3,[$ctx,#8]
add $A,$A,$t0 // accumulate
add $B,$B,$t1
ldp $t0,$t1,[$ctx,#16]
add $C,$C,$t2
add $D,$D,$t3
ldp $t2,$t3,[$ctx,#24]
add $E,$E,$t0
add $F,$F,$t1
ldr $t1,[sp,#0]
stp $A,$B,[$ctx,#0]
add $G,$G,$t2
mov $t2,wzr
stp $C,$D,[$ctx,#8]
add $H,$H,$t3
stp $E,$F,[$ctx,#16]
eor $t3,$B,$C
stp $G,$H,[$ctx,#24]
mov $t4,wzr
mov $Xfer,sp
b.ne .L_00_48
ldr x29,[x29]
add sp,sp,#16*4+16
ret
.size sha256_block_neon,.-sha256_block_neon
___
}
$code.=<<___;
#ifndef __KERNEL__
.comm OPENSSL_armcap_P,4,4
#endif
___
{ my %opcode = (
"sha256h" => 0x5e004000, "sha256h2" => 0x5e005000,
"sha256su0" => 0x5e282800, "sha256su1" => 0x5e006000 );
sub unsha256 {
my ($mnemonic,$arg)=@_;
$arg =~ m/[qv]([0-9]+)[^,]*,\s*[qv]([0-9]+)[^,]*(?:,\s*[qv]([0-9]+))?/o
&&
sprintf ".inst\t0x%08x\t//%s %s",
$opcode{$mnemonic}|$1|($2<<5)|($3<<16),
$mnemonic,$arg;
}
}
open SELF,$0;
while(<SELF>) {
next if (/^#!/);
last if (!s/^#/\/\// and !/^$/);
print;
}
close SELF;
foreach(split("\n",$code)) {
s/\`([^\`]*)\`/eval($1)/ge;
s/\b(sha256\w+)\s+([qv].*)/unsha256($1,$2)/ge;
s/\bq([0-9]+)\b/v$1.16b/g; # old->new registers
s/\.[ui]?8(\s)/$1/;
s/\.\w?32\b// and s/\.16b/\.4s/g;
m/(ld|st)1[^\[]+\[0\]/ and s/\.4s/\.s/g;
print $_,"\n";
}
close STDOUT;
linux-7.2.3/lib/crypto/arm64/sha256-ce.S
+401
View File
@@ -0,0 +1,401 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Core SHA-224/SHA-256 transform using v8 Crypto Extensions
*
* Copyright (C) 2014 Linaro Ltd <ard.biesheuvel@linaro.org>
*/
#include <linux/linkage.h>
#include <asm/assembler.h>
.text
.arch armv8-a+crypto
dga .req q20
dgav .req v20
dgb .req q21
dgbv .req v21
t0 .req v22
t1 .req v23
dg0q .req q24
dg0v .req v24
dg1q .req q25
dg1v .req v25
dg2q .req q26
dg2v .req v26
.macro add_only, ev, rc, s0
mov dg2v.16b, dg0v.16b
.ifeq \ev
add t1.4s, v\s0\().4s, \rc\().4s
sha256h dg0q, dg1q, t0.4s
sha256h2 dg1q, dg2q, t0.4s
.else
.ifnb \s0
add t0.4s, v\s0\().4s, \rc\().4s
.endif
sha256h dg0q, dg1q, t1.4s
sha256h2 dg1q, dg2q, t1.4s
.endif
.endm
.macro add_update, ev, rc, s0, s1, s2, s3
sha256su0 v\s0\().4s, v\s1\().4s
add_only \ev, \rc, \s1
sha256su1 v\s0\().4s, v\s2\().4s, v\s3\().4s
.endm
/*
* The SHA-256 round constants
*/
.section ".rodata", "a"
.align 4
.Lsha2_rcon:
.word 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5
.word 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5
.word 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3
.word 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174
.word 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc
.word 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da
.word 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7
.word 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967
.word 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13
.word 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85
.word 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3
.word 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070
.word 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5
.word 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3
.word 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208
.word 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2
.macro load_round_constants tmp
adr_l \tmp, .Lsha2_rcon
ld1 { v0.4s- v3.4s}, [\tmp], #64
ld1 { v4.4s- v7.4s}, [\tmp], #64
ld1 { v8.4s-v11.4s}, [\tmp], #64
ld1 {v12.4s-v15.4s}, [\tmp]
.endm
/*
* void sha256_ce_transform(struct sha256_block_state *state,
* const u8 *data, size_t nblocks);
*/
.text
SYM_FUNC_START(sha256_ce_transform)
load_round_constants x8
/* load state */
ld1 {dgav.4s, dgbv.4s}, [x0]
/* load input */
0: ld1 {v16.4s-v19.4s}, [x1], #64
sub x2, x2, #1
rev32 v16.16b, v16.16b
rev32 v17.16b, v17.16b
rev32 v18.16b, v18.16b
rev32 v19.16b, v19.16b
add t0.4s, v16.4s, v0.4s
mov dg0v.16b, dgav.16b
mov dg1v.16b, dgbv.16b
add_update 0, v1, 16, 17, 18, 19
add_update 1, v2, 17, 18, 19, 16
add_update 0, v3, 18, 19, 16, 17
add_update 1, v4, 19, 16, 17, 18
add_update 0, v5, 16, 17, 18, 19
add_update 1, v6, 17, 18, 19, 16
add_update 0, v7, 18, 19, 16, 17
add_update 1, v8, 19, 16, 17, 18
add_update 0, v9, 16, 17, 18, 19
add_update 1, v10, 17, 18, 19, 16
add_update 0, v11, 18, 19, 16, 17
add_update 1, v12, 19, 16, 17, 18
add_only 0, v13, 17
add_only 1, v14, 18
add_only 0, v15, 19
add_only 1
/* update state */
add dgav.4s, dgav.4s, dg0v.4s
add dgbv.4s, dgbv.4s, dg1v.4s
/* handled all input blocks? */
cbnz x2, 0b
/* store new state */
st1 {dgav.4s, dgbv.4s}, [x0]
ret
SYM_FUNC_END(sha256_ce_transform)
.unreq dga
.unreq dgav
.unreq dgb
.unreq dgbv
.unreq t0
.unreq t1
.unreq dg0q
.unreq dg0v
.unreq dg1q
.unreq dg1v
.unreq dg2q
.unreq dg2v
// parameters for sha256_ce_finup2x()
ctx .req x0
data1 .req x1
data2 .req x2
len .req w3
out1 .req x4
out2 .req x5
// other scalar variables
count .req x6
final_step .req w7
// x8-x9 are used as temporaries.
// v0-v15 are used to cache the SHA-256 round constants.
// v16-v19 are used for the message schedule for the first message.
// v20-v23 are used for the message schedule for the second message.
// v24-v31 are used for the state and temporaries as given below.
// *_a are for the first message and *_b for the second.
state0_a_q .req q24
state0_a .req v24
state1_a_q .req q25
state1_a .req v25
state0_b_q .req q26
state0_b .req v26
state1_b_q .req q27
state1_b .req v27
t0_a .req v28
t0_b .req v29
t1_a_q .req q30
t1_a .req v30
t1_b_q .req q31
t1_b .req v31
#define OFFSETOF_BYTECOUNT 32 // offsetof(struct __sha256_ctx, bytecount)
#define OFFSETOF_BUF 40 // offsetof(struct __sha256_ctx, buf)
// offsetof(struct __sha256_ctx, state) is assumed to be 0.
// Do 4 rounds of SHA-256 for each of two messages (interleaved). m0_a
// and m0_b contain the current 4 message schedule words for the first
// and second message respectively.
//
// If not all the message schedule words have been computed yet, then
// this also computes 4 more message schedule words for each message.
// m1_a-m3_a contain the next 3 groups of 4 message schedule words for
// the first message, and likewise m1_b-m3_b for the second. After
// consuming the current value of m0_a, this macro computes the group
// after m3_a and writes it to m0_a, and likewise for *_b. This means
// that the next (m0_a, m1_a, m2_a, m3_a) is the current (m1_a, m2_a,
// m3_a, m0_a), and likewise for *_b, so the caller must cycle through
// the registers accordingly.
.macro do_4rounds_2x i, k, m0_a, m1_a, m2_a, m3_a, \
m0_b, m1_b, m2_b, m3_b
add t0_a\().4s, \m0_a\().4s, \k\().4s
add t0_b\().4s, \m0_b\().4s, \k\().4s
.if \i < 48
sha256su0 \m0_a\().4s, \m1_a\().4s
sha256su0 \m0_b\().4s, \m1_b\().4s
sha256su1 \m0_a\().4s, \m2_a\().4s, \m3_a\().4s
sha256su1 \m0_b\().4s, \m2_b\().4s, \m3_b\().4s
.endif
mov t1_a.16b, state0_a.16b
mov t1_b.16b, state0_b.16b
sha256h state0_a_q, state1_a_q, t0_a\().4s
sha256h state0_b_q, state1_b_q, t0_b\().4s
sha256h2 state1_a_q, t1_a_q, t0_a\().4s
sha256h2 state1_b_q, t1_b_q, t0_b\().4s
.endm
.macro do_16rounds_2x i, k0, k1, k2, k3
do_4rounds_2x \i + 0, \k0, v16, v17, v18, v19, v20, v21, v22, v23
do_4rounds_2x \i + 4, \k1, v17, v18, v19, v16, v21, v22, v23, v20
do_4rounds_2x \i + 8, \k2, v18, v19, v16, v17, v22, v23, v20, v21
do_4rounds_2x \i + 12, \k3, v19, v16, v17, v18, v23, v20, v21, v22
.endm
//
// void sha256_ce_finup2x(const struct __sha256_ctx *ctx,
// const u8 *data1, const u8 *data2, int len,
// u8 out1[SHA256_DIGEST_SIZE],
// u8 out2[SHA256_DIGEST_SIZE]);
//
// This function computes the SHA-256 digests of two messages |data1| and
// |data2| that are both |len| bytes long, starting from the initial context
// |ctx|. |len| must be at least SHA256_BLOCK_SIZE.
//
// The instructions for the two SHA-256 operations are interleaved. On many
// CPUs, this is almost twice as fast as hashing each message individually due
// to taking better advantage of the CPU's SHA-256 and SIMD throughput.
//
SYM_FUNC_START(sha256_ce_finup2x)
sub sp, sp, #128
mov final_step, #0
load_round_constants x8
// Load the initial state from ctx->state.
ld1 {state0_a.4s-state1_a.4s}, [ctx]
// Load ctx->bytecount. Take the mod 64 of it to get the number of
// bytes that are buffered in ctx->buf. Also save it in a register with
// len added to it.
ldr x8, [ctx, #OFFSETOF_BYTECOUNT]
add count, x8, len, sxtw
and x8, x8, #63
cbz x8, .Lfinup2x_enter_loop // No bytes buffered?
// x8 bytes (1 to 63) are currently buffered in ctx->buf. Load them
// followed by the first 64 - x8 bytes of data. Since len >= 64, we
// just load 64 bytes from each of ctx->buf, data1, and data2
// unconditionally and rearrange the data as needed.
add x9, ctx, #OFFSETOF_BUF
ld1 {v16.16b-v19.16b}, [x9]
st1 {v16.16b-v19.16b}, [sp]
ld1 {v16.16b-v19.16b}, [data1], #64
add x9, sp, x8
st1 {v16.16b-v19.16b}, [x9]
ld1 {v16.4s-v19.4s}, [sp]
ld1 {v20.16b-v23.16b}, [data2], #64
st1 {v20.16b-v23.16b}, [x9]
ld1 {v20.4s-v23.4s}, [sp]
sub len, len, #64
sub data1, data1, x8
sub data2, data2, x8
add len, len, w8
mov state0_b.16b, state0_a.16b
mov state1_b.16b, state1_a.16b
b .Lfinup2x_loop_have_data
.Lfinup2x_enter_loop:
sub len, len, #64
mov state0_b.16b, state0_a.16b
mov state1_b.16b, state1_a.16b
.Lfinup2x_loop:
// Load the next two data blocks.
ld1 {v16.4s-v19.4s}, [data1], #64
ld1 {v20.4s-v23.4s}, [data2], #64
.Lfinup2x_loop_have_data:
// Convert the words of the data blocks from big endian.
rev32 v16.16b, v16.16b
rev32 v17.16b, v17.16b
rev32 v18.16b, v18.16b
rev32 v19.16b, v19.16b
rev32 v20.16b, v20.16b
rev32 v21.16b, v21.16b
rev32 v22.16b, v22.16b
rev32 v23.16b, v23.16b
.Lfinup2x_loop_have_bswapped_data:
// Save the original state for each block.
st1 {state0_a.4s-state1_b.4s}, [sp]
// Do the SHA-256 rounds on each block.
do_16rounds_2x 0, v0, v1, v2, v3
do_16rounds_2x 16, v4, v5, v6, v7
do_16rounds_2x 32, v8, v9, v10, v11
do_16rounds_2x 48, v12, v13, v14, v15
// Add the original state for each block.
ld1 {v16.4s-v19.4s}, [sp]
add state0_a.4s, state0_a.4s, v16.4s
add state1_a.4s, state1_a.4s, v17.4s
add state0_b.4s, state0_b.4s, v18.4s
add state1_b.4s, state1_b.4s, v19.4s
// Update len and loop back if more blocks remain.
sub len, len, #64
tbz len, #31, .Lfinup2x_loop // len >= 0?
// Check if any final blocks need to be handled.
// final_step = 2: all done
// final_step = 1: need to do count-only padding block
// final_step = 0: need to do the block with 0x80 padding byte
tbnz final_step, #1, .Lfinup2x_done
tbnz final_step, #0, .Lfinup2x_finalize_countonly
add len, len, #64
cbz len, .Lfinup2x_finalize_blockaligned
// Not block-aligned; 1 <= len <= 63 data bytes remain. Pad the block.
// To do this, write the padding starting with the 0x80 byte to
// &sp[64]. Then for each message, copy the last 64 data bytes to sp
// and load from &sp[64 - len] to get the needed padding block. This
// code relies on the data buffers being >= 64 bytes in length.
sub w8, len, #64 // w8 = len - 64
add data1, data1, w8, sxtw // data1 += len - 64
add data2, data2, w8, sxtw // data2 += len - 64
mov x9, #0x80
fmov d16, x9
movi v17.16b, #0
stp q16, q17, [sp, #64]
stp q17, q17, [sp, #96]
sub x9, sp, w8, sxtw // x9 = &sp[64 - len]
cmp len, #56
b.ge 1f // will count spill into its own block?
lsl count, count, #3
rev count, count
str count, [x9, #56]
mov final_step, #2 // won't need count-only block
b 2f
1:
mov final_step, #1 // will need count-only block
2:
ld1 {v16.16b-v19.16b}, [data1]
st1 {v16.16b-v19.16b}, [sp]
ld1 {v16.4s-v19.4s}, [x9]
ld1 {v20.16b-v23.16b}, [data2]
st1 {v20.16b-v23.16b}, [sp]
ld1 {v20.4s-v23.4s}, [x9]
b .Lfinup2x_loop_have_data
// Prepare a padding block, either:
//
// {0x80, 0, 0, 0, ..., count (as __be64)}
// This is for a block aligned message.
//
// { 0, 0, 0, 0, ..., count (as __be64)}
// This is for a message whose length mod 64 is >= 56.
//
// Pre-swap the endianness of the words.
.Lfinup2x_finalize_countonly:
movi v16.2d, #0
b 1f
.Lfinup2x_finalize_blockaligned:
mov x8, #0x80000000
fmov d16, x8
1:
movi v17.2d, #0
movi v18.2d, #0
ror count, count, #29 // ror(lsl(count, 3), 32)
mov v19.d[0], xzr
mov v19.d[1], count
mov v20.16b, v16.16b
movi v21.2d, #0
movi v22.2d, #0
mov v23.16b, v19.16b
mov final_step, #2
b .Lfinup2x_loop_have_bswapped_data
.Lfinup2x_done:
// Write the two digests with all bytes in the correct order.
rev32 state0_a.16b, state0_a.16b
rev32 state1_a.16b, state1_a.16b
rev32 state0_b.16b, state0_b.16b
rev32 state1_b.16b, state1_b.16b
st1 {state0_a.4s-state1_a.4s}, [out1]
st1 {state0_b.4s-state1_b.4s}, [out2]
add sp, sp, #128
ret
SYM_FUNC_END(sha256_ce_finup2x)
linux-7.2.3/lib/crypto/arm64/sha256.h
+74
View File
@@ -0,0 +1,74 @@
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
* SHA-256 optimized for ARM64
*
* Copyright 2025 Google LLC
*/
#include <asm/simd.h>
#include <linux/cpufeature.h>
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon);
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_ce);
asmlinkage void sha256_block_data_order(struct sha256_block_state *state,
const u8 *data, size_t nblocks);
asmlinkage void sha256_block_neon(struct sha256_block_state *state,
const u8 *data, size_t nblocks);
asmlinkage void sha256_ce_transform(struct sha256_block_state *state,
const u8 *data, size_t nblocks);
static void sha256_blocks(struct sha256_block_state *state,
const u8 *data, size_t nblocks)
{
if (static_branch_likely(&have_neon) && likely(may_use_simd())) {
scoped_ksimd() {
if (static_branch_likely(&have_ce))
sha256_ce_transform(state, data, nblocks);
else
sha256_block_neon(state, data, nblocks);
}
} else {
sha256_block_data_order(state, data, nblocks);
}
}
static_assert(offsetof(struct __sha256_ctx, state) == 0);
static_assert(offsetof(struct __sha256_ctx, bytecount) == 32);
static_assert(offsetof(struct __sha256_ctx, buf) == 40);
asmlinkage void sha256_ce_finup2x(const struct __sha256_ctx *ctx,
const u8 *data1, const u8 *data2, int len,
u8 out1[SHA256_DIGEST_SIZE],
u8 out2[SHA256_DIGEST_SIZE]);
#define sha256_finup_2x_arch sha256_finup_2x_arch
static bool sha256_finup_2x_arch(const struct __sha256_ctx *ctx,
const u8 *data1, const u8 *data2, size_t len,
u8 out1[SHA256_DIGEST_SIZE],
u8 out2[SHA256_DIGEST_SIZE])
{
/* The assembly requires len >= SHA256_BLOCK_SIZE && len <= INT_MAX. */
if (static_branch_likely(&have_ce) && len >= SHA256_BLOCK_SIZE &&
len <= INT_MAX && likely(may_use_simd())) {
scoped_ksimd()
sha256_ce_finup2x(ctx, data1, data2, len, out1, out2);
kmsan_unpoison_memory(out1, SHA256_DIGEST_SIZE);
kmsan_unpoison_memory(out2, SHA256_DIGEST_SIZE);
return true;
}
return false;
}
static bool sha256_finup_2x_is_optimized_arch(void)
{
return static_key_enabled(&have_ce);
}
#define sha256_mod_init_arch sha256_mod_init_arch
static void sha256_mod_init_arch(void)
{
if (cpu_have_named_feature(ASIMD)) {
static_branch_enable(&have_neon);
if (cpu_have_named_feature(SHA2))
static_branch_enable(&have_ce);
}
}
linux-7.2.3/lib/crypto/arm64/sha3-ce-core.S
+211
View File
@@ -0,0 +1,211 @@
/* SPDX-License-Identifier: GPL-2.0 */
/*
* Core SHA-3 transform using v8.2 Crypto Extensions
*
* Copyright (C) 2018 Linaro Ltd <ard.biesheuvel@linaro.org>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation.
*/
#include <linux/linkage.h>
#include <asm/assembler.h>
.irp b,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31
.set .Lv\b\().2d, \b
.set .Lv\b\().16b, \b
.endr
/*
* ARMv8.2 Crypto Extensions instructions
*/
.macro eor3, rd, rn, rm, ra
.inst 0xce000000 | .L\rd | (.L\rn << 5) | (.L\ra << 10) | (.L\rm << 16)
.endm
.macro rax1, rd, rn, rm
.inst 0xce608c00 | .L\rd | (.L\rn << 5) | (.L\rm << 16)
.endm
.macro bcax, rd, rn, rm, ra
.inst 0xce200000 | .L\rd | (.L\rn << 5) | (.L\ra << 10) | (.L\rm << 16)
.endm
.macro xar, rd, rn, rm, imm6
.inst 0xce800000 | .L\rd | (.L\rn << 5) | ((\imm6) << 10) | (.L\rm << 16)
.endm
/*
* void sha3_ce_transform(struct sha3_state *state, const u8 *data,
* size_t nblocks, size_t block_size)
*
* block_size is assumed to be one of 72 (SHA3-512), 104 (SHA3-384), 136
* (SHA3-256 and SHAKE256), 144 (SHA3-224), or 168 (SHAKE128).
*/
.text
SYM_FUNC_START(sha3_ce_transform)
/* load state */
add x8, x0, #32
ld1 { v0.1d- v3.1d}, [x0]
ld1 { v4.1d- v7.1d}, [x8], #32
ld1 { v8.1d-v11.1d}, [x8], #32
ld1 {v12.1d-v15.1d}, [x8], #32
ld1 {v16.1d-v19.1d}, [x8], #32
ld1 {v20.1d-v23.1d}, [x8], #32
ld1 {v24.1d}, [x8]
0: sub x2, x2, #1
mov w8, #24
adr_l x9, .Lsha3_rcon
/* load input */
ld1 {v25.8b-v28.8b}, [x1], #32
ld1 {v29.8b}, [x1], #8
eor v0.8b, v0.8b, v25.8b
eor v1.8b, v1.8b, v26.8b
eor v2.8b, v2.8b, v27.8b
eor v3.8b, v3.8b, v28.8b
eor v4.8b, v4.8b, v29.8b
ld1 {v25.8b-v28.8b}, [x1], #32
eor v5.8b, v5.8b, v25.8b
eor v6.8b, v6.8b, v26.8b
eor v7.8b, v7.8b, v27.8b
eor v8.8b, v8.8b, v28.8b
cmp x3, #72
b.eq 3f /* SHA3-512 (block_size=72)? */
ld1 {v25.8b-v28.8b}, [x1], #32
eor v9.8b, v9.8b, v25.8b
eor v10.8b, v10.8b, v26.8b
eor v11.8b, v11.8b, v27.8b
eor v12.8b, v12.8b, v28.8b
cmp x3, #104
b.eq 3f /* SHA3-384 (block_size=104)? */
ld1 {v25.8b-v28.8b}, [x1], #32
eor v13.8b, v13.8b, v25.8b
eor v14.8b, v14.8b, v26.8b
eor v15.8b, v15.8b, v27.8b
eor v16.8b, v16.8b, v28.8b
cmp x3, #144
b.lt 3f /* SHA3-256 or SHAKE256 (block_size=136)? */
b.eq 2f /* SHA3-224 (block_size=144)? */
/* SHAKE128 (block_size=168) */
ld1 {v25.8b-v28.8b}, [x1], #32
eor v17.8b, v17.8b, v25.8b
eor v18.8b, v18.8b, v26.8b
eor v19.8b, v19.8b, v27.8b
eor v20.8b, v20.8b, v28.8b
b 3f
2:
/* SHA3-224 (block_size=144) */
ld1 {v25.8b}, [x1], #8
eor v17.8b, v17.8b, v25.8b
3: sub w8, w8, #1
eor3 v29.16b, v4.16b, v9.16b, v14.16b
eor3 v26.16b, v1.16b, v6.16b, v11.16b
eor3 v28.16b, v3.16b, v8.16b, v13.16b
eor3 v25.16b, v0.16b, v5.16b, v10.16b
eor3 v27.16b, v2.16b, v7.16b, v12.16b
eor3 v29.16b, v29.16b, v19.16b, v24.16b
eor3 v26.16b, v26.16b, v16.16b, v21.16b
eor3 v28.16b, v28.16b, v18.16b, v23.16b
eor3 v25.16b, v25.16b, v15.16b, v20.16b
eor3 v27.16b, v27.16b, v17.16b, v22.16b
rax1 v30.2d, v29.2d, v26.2d // bc[0]
rax1 v26.2d, v26.2d, v28.2d // bc[2]
rax1 v28.2d, v28.2d, v25.2d // bc[4]
rax1 v25.2d, v25.2d, v27.2d // bc[1]
rax1 v27.2d, v27.2d, v29.2d // bc[3]
eor v0.16b, v0.16b, v30.16b
xar v29.2d, v1.2d, v25.2d, (64 - 1)
xar v1.2d, v6.2d, v25.2d, (64 - 44)
xar v6.2d, v9.2d, v28.2d, (64 - 20)
xar v9.2d, v22.2d, v26.2d, (64 - 61)
xar v22.2d, v14.2d, v28.2d, (64 - 39)
xar v14.2d, v20.2d, v30.2d, (64 - 18)
xar v31.2d, v2.2d, v26.2d, (64 - 62)
xar v2.2d, v12.2d, v26.2d, (64 - 43)
xar v12.2d, v13.2d, v27.2d, (64 - 25)
xar v13.2d, v19.2d, v28.2d, (64 - 8)
xar v19.2d, v23.2d, v27.2d, (64 - 56)
xar v23.2d, v15.2d, v30.2d, (64 - 41)
xar v15.2d, v4.2d, v28.2d, (64 - 27)
xar v28.2d, v24.2d, v28.2d, (64 - 14)
xar v24.2d, v21.2d, v25.2d, (64 - 2)
xar v8.2d, v8.2d, v27.2d, (64 - 55)
xar v4.2d, v16.2d, v25.2d, (64 - 45)
xar v16.2d, v5.2d, v30.2d, (64 - 36)
xar v5.2d, v3.2d, v27.2d, (64 - 28)
xar v27.2d, v18.2d, v27.2d, (64 - 21)
xar v3.2d, v17.2d, v26.2d, (64 - 15)
xar v25.2d, v11.2d, v25.2d, (64 - 10)
xar v26.2d, v7.2d, v26.2d, (64 - 6)
xar v30.2d, v10.2d, v30.2d, (64 - 3)
bcax v20.16b, v31.16b, v22.16b, v8.16b
bcax v21.16b, v8.16b, v23.16b, v22.16b
bcax v22.16b, v22.16b, v24.16b, v23.16b
bcax v23.16b, v23.16b, v31.16b, v24.16b
bcax v24.16b, v24.16b, v8.16b, v31.16b
ld1r {v31.2d}, [x9], #8
bcax v17.16b, v25.16b, v19.16b, v3.16b
bcax v18.16b, v3.16b, v15.16b, v19.16b
bcax v19.16b, v19.16b, v16.16b, v15.16b
bcax v15.16b, v15.16b, v25.16b, v16.16b
bcax v16.16b, v16.16b, v3.16b, v25.16b
bcax v10.16b, v29.16b, v12.16b, v26.16b
bcax v11.16b, v26.16b, v13.16b, v12.16b
bcax v12.16b, v12.16b, v14.16b, v13.16b
bcax v13.16b, v13.16b, v29.16b, v14.16b
bcax v14.16b, v14.16b, v26.16b, v29.16b
bcax v7.16b, v30.16b, v9.16b, v4.16b
bcax v8.16b, v4.16b, v5.16b, v9.16b
bcax v9.16b, v9.16b, v6.16b, v5.16b
bcax v5.16b, v5.16b, v30.16b, v6.16b
bcax v6.16b, v6.16b, v4.16b, v30.16b
bcax v3.16b, v27.16b, v0.16b, v28.16b
bcax v4.16b, v28.16b, v1.16b, v0.16b
bcax v0.16b, v0.16b, v2.16b, v1.16b
bcax v1.16b, v1.16b, v27.16b, v2.16b
bcax v2.16b, v2.16b, v28.16b, v27.16b
eor v0.16b, v0.16b, v31.16b
cbnz w8, 3b
cbnz x2, 0b
/* save state */
st1 { v0.1d- v3.1d}, [x0], #32
st1 { v4.1d- v7.1d}, [x0], #32
st1 { v8.1d-v11.1d}, [x0], #32
st1 {v12.1d-v15.1d}, [x0], #32
st1 {v16.1d-v19.1d}, [x0], #32
st1 {v20.1d-v23.1d}, [x0], #32
st1 {v24.1d}, [x0]
ret
SYM_FUNC_END(sha3_ce_transform)
.section ".rodata", "a"
.align 8
.Lsha3_rcon:
.quad 0x0000000000000001, 0x0000000000008082, 0x800000000000808a
.quad 0x8000000080008000, 0x000000000000808b, 0x0000000080000001
.quad 0x8000000080008081, 0x8000000000008009, 0x000000000000008a
.quad 0x0000000000000088, 0x0000000080008009, 0x000000008000000a
.quad 0x000000008000808b, 0x800000000000008b, 0x8000000000008089
.quad 0x8000000000008003, 0x8000000000008002, 0x8000000000000080
.quad 0x000000000000800a, 0x800000008000000a, 0x8000000080008081
.quad 0x8000000000008080, 0x0000000080000001, 0x8000000080008008
linux-7.2.3/lib/crypto/arm64/sha3.h
+52
View File
@@ -0,0 +1,52 @@
/* SPDX-License-Identifier: GPL-2.0 */
/*
* Copyright (C) 2018 Linaro Ltd <ard.biesheuvel@linaro.org>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation.
*/
#include <asm/simd.h>
#include <linux/cpufeature.h>
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_sha3);
asmlinkage void sha3_ce_transform(struct sha3_state *state, const u8 *data,
size_t nblocks, size_t block_size);
static void sha3_absorb_blocks(struct sha3_state *state, const u8 *data,
size_t nblocks, size_t block_size)
{
if (static_branch_likely(&have_sha3) && likely(may_use_simd())) {
scoped_ksimd()
sha3_ce_transform(state, data, nblocks, block_size);
} else {
sha3_absorb_blocks_generic(state, data, nblocks, block_size);
}
}
static void sha3_keccakf(struct sha3_state *state)
{
if (static_branch_likely(&have_sha3) && likely(may_use_simd())) {
/*
* Passing zeroes into sha3_ce_transform() gives the plain
* Keccak-f permutation, which is what we want here. Any
* supported block size may be used. Use SHA3_512_BLOCK_SIZE
* since it's the shortest.
*/
static const u8 zeroes[SHA3_512_BLOCK_SIZE];
scoped_ksimd()
sha3_ce_transform(state, zeroes, 1, sizeof(zeroes));
} else {
sha3_keccakf_generic(state);
}
}
#define sha3_mod_init_arch sha3_mod_init_arch
static void sha3_mod_init_arch(void)
{
if (cpu_have_named_feature(SHA3))
static_branch_enable(&have_sha3);
}
linux-7.2.3/lib/crypto/arm64/sha512-ce-core.S
+195
View File
@@ -0,0 +1,195 @@
/* SPDX-License-Identifier: GPL-2.0 */
/*
* Core SHA-384/SHA-512 transform using v8 Crypto Extensions
*
* Copyright (C) 2018 Linaro Ltd <ard.biesheuvel@linaro.org>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation.
*/
#include <linux/linkage.h>
#include <asm/assembler.h>
/*
* We have to specify the "sha3" feature here, since the GNU and clang
* assemblers both consider the SHA-512 instructions to be part of the
* "sha3" feature. (Except binutils 2.30 through 2.42, which used
* "sha2". But "sha3" implies "sha2", so "sha3" still works in those
* versions.) "sha3" doesn't make a lot of sense, since SHA-512 is part
* of the SHA-2 family of algorithms, and also the Arm Architecture
* Reference Manual defines FEAT_SHA512 and FEAT_SHA3 separately.
* Regardless, we must use "sha3" to be compatible with the assemblers.
*/
.arch armv8-a+sha3
/*
* The SHA-512 round constants
*/
.section ".rodata", "a"
.align 4
.Lsha512_rcon:
.quad 0x428a2f98d728ae22, 0x7137449123ef65cd
.quad 0xb5c0fbcfec4d3b2f, 0xe9b5dba58189dbbc
.quad 0x3956c25bf348b538, 0x59f111f1b605d019
.quad 0x923f82a4af194f9b, 0xab1c5ed5da6d8118
.quad 0xd807aa98a3030242, 0x12835b0145706fbe
.quad 0x243185be4ee4b28c, 0x550c7dc3d5ffb4e2
.quad 0x72be5d74f27b896f, 0x80deb1fe3b1696b1
.quad 0x9bdc06a725c71235, 0xc19bf174cf692694
.quad 0xe49b69c19ef14ad2, 0xefbe4786384f25e3
.quad 0x0fc19dc68b8cd5b5, 0x240ca1cc77ac9c65
.quad 0x2de92c6f592b0275, 0x4a7484aa6ea6e483
.quad 0x5cb0a9dcbd41fbd4, 0x76f988da831153b5
.quad 0x983e5152ee66dfab, 0xa831c66d2db43210
.quad 0xb00327c898fb213f, 0xbf597fc7beef0ee4
.quad 0xc6e00bf33da88fc2, 0xd5a79147930aa725
.quad 0x06ca6351e003826f, 0x142929670a0e6e70
.quad 0x27b70a8546d22ffc, 0x2e1b21385c26c926
.quad 0x4d2c6dfc5ac42aed, 0x53380d139d95b3df
.quad 0x650a73548baf63de, 0x766a0abb3c77b2a8
.quad 0x81c2c92e47edaee6, 0x92722c851482353b
.quad 0xa2bfe8a14cf10364, 0xa81a664bbc423001
.quad 0xc24b8b70d0f89791, 0xc76c51a30654be30
.quad 0xd192e819d6ef5218, 0xd69906245565a910
.quad 0xf40e35855771202a, 0x106aa07032bbd1b8
.quad 0x19a4c116b8d2d0c8, 0x1e376c085141ab53
.quad 0x2748774cdf8eeb99, 0x34b0bcb5e19b48a8
.quad 0x391c0cb3c5c95a63, 0x4ed8aa4ae3418acb
.quad 0x5b9cca4f7763e373, 0x682e6ff3d6b2b8a3
.quad 0x748f82ee5defb2fc, 0x78a5636f43172f60
.quad 0x84c87814a1f0ab72, 0x8cc702081a6439ec
.quad 0x90befffa23631e28, 0xa4506cebde82bde9
.quad 0xbef9a3f7b2c67915, 0xc67178f2e372532b
.quad 0xca273eceea26619c, 0xd186b8c721c0c207
.quad 0xeada7dd6cde0eb1e, 0xf57d4f7fee6ed178
.quad 0x06f067aa72176fba, 0x0a637dc5a2c898a6
.quad 0x113f9804bef90dae, 0x1b710b35131c471b
.quad 0x28db77f523047d84, 0x32caab7b40c72493
.quad 0x3c9ebe0a15c9bebc, 0x431d67c49c100d4c
.quad 0x4cc5d4becb3e42b6, 0x597f299cfc657e2a
.quad 0x5fcb6fab3ad6faec, 0x6c44198c4a475817
.macro dround, i0, i1, i2, i3, i4, rc0, rc1, in0, in1, in2, in3, in4
.ifnb \rc1
ld1 {v\rc1\().2d}, [x4], #16
.endif
add v5.2d, v\rc0\().2d, v\in0\().2d
ext v6.16b, v\i2\().16b, v\i3\().16b, #8
ext v5.16b, v5.16b, v5.16b, #8
ext v7.16b, v\i1\().16b, v\i2\().16b, #8
add v\i3\().2d, v\i3\().2d, v5.2d
.ifnb \in1
ext v5.16b, v\in3\().16b, v\in4\().16b, #8
sha512su0 v\in0\().2d, v\in1\().2d
.endif
sha512h q\i3, q6, v7.2d
.ifnb \in1
sha512su1 v\in0\().2d, v\in2\().2d, v5.2d
.endif
add v\i4\().2d, v\i1\().2d, v\i3\().2d
sha512h2 q\i3, q\i1, v\i0\().2d
.endm
/*
* void sha512_ce_transform(struct sha512_block_state *state,
* const u8 *data, size_t nblocks);
*/
.text
SYM_FUNC_START(sha512_ce_transform)
/* load state */
ld1 {v8.2d-v11.2d}, [x0]
/* load first 4 round constants */
adr_l x3, .Lsha512_rcon
ld1 {v20.2d-v23.2d}, [x3], #64
/* load input */
0: ld1 {v12.2d-v15.2d}, [x1], #64
ld1 {v16.2d-v19.2d}, [x1], #64
sub x2, x2, #1
rev64 v12.16b, v12.16b
rev64 v13.16b, v13.16b
rev64 v14.16b, v14.16b
rev64 v15.16b, v15.16b
rev64 v16.16b, v16.16b
rev64 v17.16b, v17.16b
rev64 v18.16b, v18.16b
rev64 v19.16b, v19.16b
mov x4, x3 // rc pointer
mov v0.16b, v8.16b
mov v1.16b, v9.16b
mov v2.16b, v10.16b
mov v3.16b, v11.16b
// v0 ab cd -- ef gh ab
// v1 cd -- ef gh ab cd
// v2 ef gh ab cd -- ef
// v3 gh ab cd -- ef gh
// v4 -- ef gh ab cd --
dround 0, 1, 2, 3, 4, 20, 24, 12, 13, 19, 16, 17
dround 3, 0, 4, 2, 1, 21, 25, 13, 14, 12, 17, 18
dround 2, 3, 1, 4, 0, 22, 26, 14, 15, 13, 18, 19
dround 4, 2, 0, 1, 3, 23, 27, 15, 16, 14, 19, 12
dround 1, 4, 3, 0, 2, 24, 28, 16, 17, 15, 12, 13
dround 0, 1, 2, 3, 4, 25, 29, 17, 18, 16, 13, 14
dround 3, 0, 4, 2, 1, 26, 30, 18, 19, 17, 14, 15
dround 2, 3, 1, 4, 0, 27, 31, 19, 12, 18, 15, 16
dround 4, 2, 0, 1, 3, 28, 24, 12, 13, 19, 16, 17
dround 1, 4, 3, 0, 2, 29, 25, 13, 14, 12, 17, 18
dround 0, 1, 2, 3, 4, 30, 26, 14, 15, 13, 18, 19
dround 3, 0, 4, 2, 1, 31, 27, 15, 16, 14, 19, 12
dround 2, 3, 1, 4, 0, 24, 28, 16, 17, 15, 12, 13
dround 4, 2, 0, 1, 3, 25, 29, 17, 18, 16, 13, 14
dround 1, 4, 3, 0, 2, 26, 30, 18, 19, 17, 14, 15
dround 0, 1, 2, 3, 4, 27, 31, 19, 12, 18, 15, 16
dround 3, 0, 4, 2, 1, 28, 24, 12, 13, 19, 16, 17
dround 2, 3, 1, 4, 0, 29, 25, 13, 14, 12, 17, 18
dround 4, 2, 0, 1, 3, 30, 26, 14, 15, 13, 18, 19
dround 1, 4, 3, 0, 2, 31, 27, 15, 16, 14, 19, 12
dround 0, 1, 2, 3, 4, 24, 28, 16, 17, 15, 12, 13
dround 3, 0, 4, 2, 1, 25, 29, 17, 18, 16, 13, 14
dround 2, 3, 1, 4, 0, 26, 30, 18, 19, 17, 14, 15
dround 4, 2, 0, 1, 3, 27, 31, 19, 12, 18, 15, 16
dround 1, 4, 3, 0, 2, 28, 24, 12, 13, 19, 16, 17
dround 0, 1, 2, 3, 4, 29, 25, 13, 14, 12, 17, 18
dround 3, 0, 4, 2, 1, 30, 26, 14, 15, 13, 18, 19
dround 2, 3, 1, 4, 0, 31, 27, 15, 16, 14, 19, 12
dround 4, 2, 0, 1, 3, 24, 28, 16, 17, 15, 12, 13
dround 1, 4, 3, 0, 2, 25, 29, 17, 18, 16, 13, 14
dround 0, 1, 2, 3, 4, 26, 30, 18, 19, 17, 14, 15
dround 3, 0, 4, 2, 1, 27, 31, 19, 12, 18, 15, 16
dround 2, 3, 1, 4, 0, 28, 24, 12
dround 4, 2, 0, 1, 3, 29, 25, 13
dround 1, 4, 3, 0, 2, 30, 26, 14
dround 0, 1, 2, 3, 4, 31, 27, 15
dround 3, 0, 4, 2, 1, 24, , 16
dround 2, 3, 1, 4, 0, 25, , 17
dround 4, 2, 0, 1, 3, 26, , 18
dround 1, 4, 3, 0, 2, 27, , 19
/* update state */
add v8.2d, v8.2d, v0.2d
add v9.2d, v9.2d, v1.2d
add v10.2d, v10.2d, v2.2d
add v11.2d, v11.2d, v3.2d
/* handled all input blocks? */
cbnz x2, 0b
/* store new state */
st1 {v8.2d-v11.2d}, [x0]
ret
SYM_FUNC_END(sha512_ce_transform)
linux-7.2.3/lib/crypto/arm64/sha512.h
+35
View File
@@ -0,0 +1,35 @@
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
* arm64-optimized SHA-512 block function
*
* Copyright 2025 Google LLC
*/
#include <asm/simd.h>
#include <linux/cpufeature.h>
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_sha512_insns);
asmlinkage void sha512_block_data_order(struct sha512_block_state *state,
const u8 *data, size_t nblocks);
asmlinkage void sha512_ce_transform(struct sha512_block_state *state,
const u8 *data, size_t nblocks);
static void sha512_blocks(struct sha512_block_state *state,
const u8 *data, size_t nblocks)
{
if (static_branch_likely(&have_sha512_insns) &&
likely(may_use_simd())) {
scoped_ksimd()
sha512_ce_transform(state, data, nblocks);
} else {
sha512_block_data_order(state, data, nblocks);
}
}
#define sha512_mod_init_arch sha512_mod_init_arch
static void sha512_mod_init_arch(void)
{
if (cpu_have_named_feature(SHA512))
static_branch_enable(&have_sha512_insns);
}
linux-7.2.3/lib/crypto/arm64/sm3-ce-core.S
+138
View File
@@ -0,0 +1,138 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* sm3-ce-core.S - SM3 secure hash using ARMv8.2 Crypto Extensions
*
* Copyright (C) 2018 Linaro Ltd <ard.biesheuvel@linaro.org>
*/
#include <linux/linkage.h>
#include <asm/assembler.h>
.irp b, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12
.set .Lv\b\().4s, \b
.endr
.macro sm3partw1, rd, rn, rm
.inst 0xce60c000 | .L\rd | (.L\rn << 5) | (.L\rm << 16)
.endm
.macro sm3partw2, rd, rn, rm
.inst 0xce60c400 | .L\rd | (.L\rn << 5) | (.L\rm << 16)
.endm
.macro sm3ss1, rd, rn, rm, ra
.inst 0xce400000 | .L\rd | (.L\rn << 5) | (.L\ra << 10) | (.L\rm << 16)
.endm
.macro sm3tt1a, rd, rn, rm, imm2
.inst 0xce408000 | .L\rd | (.L\rn << 5) | ((\imm2) << 12) | (.L\rm << 16)
.endm
.macro sm3tt1b, rd, rn, rm, imm2
.inst 0xce408400 | .L\rd | (.L\rn << 5) | ((\imm2) << 12) | (.L\rm << 16)
.endm
.macro sm3tt2a, rd, rn, rm, imm2
.inst 0xce408800 | .L\rd | (.L\rn << 5) | ((\imm2) << 12) | (.L\rm << 16)
.endm
.macro sm3tt2b, rd, rn, rm, imm2
.inst 0xce408c00 | .L\rd | (.L\rn << 5) | ((\imm2) << 12) | (.L\rm << 16)
.endm
.macro round, ab, s0, t0, t1, i
sm3ss1 v5.4s, v8.4s, \t0\().4s, v9.4s
shl \t1\().4s, \t0\().4s, #1
sri \t1\().4s, \t0\().4s, #31
sm3tt1\ab v8.4s, v5.4s, v10.4s, \i
sm3tt2\ab v9.4s, v5.4s, \s0\().4s, \i
.endm
.macro qround, ab, s0, s1, s2, s3, s4
.ifnb \s4
ext \s4\().16b, \s1\().16b, \s2\().16b, #12
ext v6.16b, \s0\().16b, \s1\().16b, #12
ext v7.16b, \s2\().16b, \s3\().16b, #8
sm3partw1 \s4\().4s, \s0\().4s, \s3\().4s
.endif
eor v10.16b, \s0\().16b, \s1\().16b
round \ab, \s0, v11, v12, 0
round \ab, \s0, v12, v11, 1
round \ab, \s0, v11, v12, 2
round \ab, \s0, v12, v11, 3
.ifnb \s4
sm3partw2 \s4\().4s, v7.4s, v6.4s
.endif
.endm
/*
* void sm3_ce_transform(struct sm3_block_state *state,
* const u8 *data, size_t nblocks)
*/
.text
SYM_FUNC_START(sm3_ce_transform)
/* load state */
ld1 {v8.4s-v9.4s}, [x0]
rev64 v8.4s, v8.4s
rev64 v9.4s, v9.4s
ext v8.16b, v8.16b, v8.16b, #8
ext v9.16b, v9.16b, v9.16b, #8
adr_l x8, .Lt
ldp s13, s14, [x8]
/* load input */
0: ld1 {v0.16b-v3.16b}, [x1], #64
sub x2, x2, #1
mov v15.16b, v8.16b
mov v16.16b, v9.16b
rev32 v0.16b, v0.16b
rev32 v1.16b, v1.16b
rev32 v2.16b, v2.16b
rev32 v3.16b, v3.16b
ext v11.16b, v13.16b, v13.16b, #4
qround a, v0, v1, v2, v3, v4
qround a, v1, v2, v3, v4, v0
qround a, v2, v3, v4, v0, v1
qround a, v3, v4, v0, v1, v2
ext v11.16b, v14.16b, v14.16b, #4
qround b, v4, v0, v1, v2, v3
qround b, v0, v1, v2, v3, v4
qround b, v1, v2, v3, v4, v0
qround b, v2, v3, v4, v0, v1
qround b, v3, v4, v0, v1, v2
qround b, v4, v0, v1, v2, v3
qround b, v0, v1, v2, v3, v4
qround b, v1, v2, v3, v4, v0
qround b, v2, v3, v4, v0, v1
qround b, v3, v4
qround b, v4, v0
qround b, v0, v1
eor v8.16b, v8.16b, v15.16b
eor v9.16b, v9.16b, v16.16b
/* handled all input blocks? */
cbnz x2, 0b
/* save state */
rev64 v8.4s, v8.4s
rev64 v9.4s, v9.4s
ext v8.16b, v8.16b, v8.16b, #8
ext v9.16b, v9.16b, v9.16b, #8
st1 {v8.4s-v9.4s}, [x0]
ret
SYM_FUNC_END(sm3_ce_transform)
.section ".rodata", "a"
.align 3
.Lt: .word 0x79cc4519, 0x9d8a7a87
linux-7.2.3/lib/crypto/arm64/sm3-neon-core.S
+600
View File
@@ -0,0 +1,600 @@
// SPDX-License-Identifier: GPL-2.0-or-later
/*
* sm3-neon-core.S - SM3 secure hash using NEON instructions
*
* Linux/arm64 port of the libgcrypt SM3 implementation for AArch64
*
* Copyright (C) 2021 Jussi Kivilinna <jussi.kivilinna@iki.fi>
* Copyright (c) 2022 Tianjia Zhang <tianjia.zhang@linux.alibaba.com>
*/
#include <linux/linkage.h>
#include <asm/assembler.h>
/* Context structure */
#define state_h0 0
#define state_h1 4
#define state_h2 8
#define state_h3 12
#define state_h4 16
#define state_h5 20
#define state_h6 24
#define state_h7 28
/* Stack structure */
#define STACK_W_SIZE (32 * 2 * 3)
#define STACK_W (0)
#define STACK_SIZE (STACK_W + STACK_W_SIZE)
/* Register macros */
#define RSTATE x0
#define RDATA x1
#define RNBLKS x2
#define RKPTR x28
#define RFRAME x29
#define ra w3
#define rb w4
#define rc w5
#define rd w6
#define re w7
#define rf w8
#define rg w9
#define rh w10
#define t0 w11
#define t1 w12
#define t2 w13
#define t3 w14
#define t4 w15
#define t5 w16
#define t6 w17
#define k_even w19
#define k_odd w20
#define addr0 x21
#define addr1 x22
#define s0 w23
#define s1 w24
#define s2 w25
#define s3 w26
#define W0 v0
#define W1 v1
#define W2 v2
#define W3 v3
#define W4 v4
#define W5 v5
#define XTMP0 v6
#define XTMP1 v7
#define XTMP2 v16
#define XTMP3 v17
#define XTMP4 v18
#define XTMP5 v19
#define XTMP6 v20
/* Helper macros. */
#define _(...) /*_*/
#define clear_vec(x) \
movi x.8h, #0;
#define rolw(o, a, n) \
ror o, a, #(32 - n);
/* Round function macros. */
#define GG1_1(x, y, z, o, t) \
eor o, x, y;
#define GG1_2(x, y, z, o, t) \
eor o, o, z;
#define GG1_3(x, y, z, o, t)
#define FF1_1(x, y, z, o, t) GG1_1(x, y, z, o, t)
#define FF1_2(x, y, z, o, t)
#define FF1_3(x, y, z, o, t) GG1_2(x, y, z, o, t)
#define GG2_1(x, y, z, o, t) \
bic o, z, x;
#define GG2_2(x, y, z, o, t) \
and t, y, x;
#define GG2_3(x, y, z, o, t) \
eor o, o, t;
#define FF2_1(x, y, z, o, t) \
eor o, x, y;
#define FF2_2(x, y, z, o, t) \
and t, x, y; \
and o, o, z;
#define FF2_3(x, y, z, o, t) \
eor o, o, t;
#define R(i, a, b, c, d, e, f, g, h, k, K_LOAD, round, widx, wtype, IOP, iop_param) \
K_LOAD(round); \
ldr t5, [sp, #(wtype##_W1_ADDR(round, widx))]; \
rolw(t0, a, 12); /* rol(a, 12) => t0 */ \
IOP(1, iop_param); \
FF##i##_1(a, b, c, t1, t2); \
ldr t6, [sp, #(wtype##_W1W2_ADDR(round, widx))]; \
add k, k, e; \
IOP(2, iop_param); \
GG##i##_1(e, f, g, t3, t4); \
FF##i##_2(a, b, c, t1, t2); \
IOP(3, iop_param); \
add k, k, t0; \
add h, h, t5; \
add d, d, t6; /* w1w2 + d => d */ \
IOP(4, iop_param); \
rolw(k, k, 7); /* rol (t0 + e + t), 7) => k */ \
GG##i##_2(e, f, g, t3, t4); \
add h, h, k; /* h + w1 + k => h */ \
IOP(5, iop_param); \
FF##i##_3(a, b, c, t1, t2); \
eor t0, t0, k; /* k ^ t0 => t0 */ \
GG##i##_3(e, f, g, t3, t4); \
add d, d, t1; /* FF(a,b,c) + d => d */ \
IOP(6, iop_param); \
add t3, t3, h; /* GG(e,f,g) + h => t3 */ \
rolw(b, b, 9); /* rol(b, 9) => b */ \
eor h, t3, t3, ror #(32-9); \
IOP(7, iop_param); \
add d, d, t0; /* t0 + d => d */ \
rolw(f, f, 19); /* rol(f, 19) => f */ \
IOP(8, iop_param); \
eor h, h, t3, ror #(32-17); /* P0(t3) => h */
#define R1(a, b, c, d, e, f, g, h, k, K_LOAD, round, widx, wtype, IOP, iop_param) \
R(1, ##a, ##b, ##c, ##d, ##e, ##f, ##g, ##h, ##k, K_LOAD, round, widx, wtype, IOP, iop_param)
#define R2(a, b, c, d, e, f, g, h, k, K_LOAD, round, widx, wtype, IOP, iop_param) \
R(2, ##a, ##b, ##c, ##d, ##e, ##f, ##g, ##h, ##k, K_LOAD, round, widx, wtype, IOP, iop_param)
#define KL(round) \
ldp k_even, k_odd, [RKPTR, #(4*(round))];
/* Input expansion macros. */
/* Byte-swapped input address. */
#define IW_W_ADDR(round, widx, offs) \
(STACK_W + ((round) / 4) * 64 + (offs) + ((widx) * 4))
/* Expanded input address. */
#define XW_W_ADDR(round, widx, offs) \
(STACK_W + ((((round) / 3) - 4) % 2) * 64 + (offs) + ((widx) * 4))
/* Rounds 1-12, byte-swapped input block addresses. */
#define IW_W1_ADDR(round, widx) IW_W_ADDR(round, widx, 32)
#define IW_W1W2_ADDR(round, widx) IW_W_ADDR(round, widx, 48)
/* Rounds 1-12, expanded input block addresses. */
#define XW_W1_ADDR(round, widx) XW_W_ADDR(round, widx, 0)
#define XW_W1W2_ADDR(round, widx) XW_W_ADDR(round, widx, 16)
/* Input block loading.
* Interleaving within round function needed for in-order CPUs. */
#define LOAD_W_VEC_1_1() \
add addr0, sp, #IW_W1_ADDR(0, 0);
#define LOAD_W_VEC_1_2() \
add addr1, sp, #IW_W1_ADDR(4, 0);
#define LOAD_W_VEC_1_3() \
ld1 {W0.16b}, [RDATA], #16;
#define LOAD_W_VEC_1_4() \
ld1 {W1.16b}, [RDATA], #16;
#define LOAD_W_VEC_1_5() \
ld1 {W2.16b}, [RDATA], #16;
#define LOAD_W_VEC_1_6() \
ld1 {W3.16b}, [RDATA], #16;
#define LOAD_W_VEC_1_7() \
rev32 XTMP0.16b, W0.16b;
#define LOAD_W_VEC_1_8() \
rev32 XTMP1.16b, W1.16b;
#define LOAD_W_VEC_2_1() \
rev32 XTMP2.16b, W2.16b;
#define LOAD_W_VEC_2_2() \
rev32 XTMP3.16b, W3.16b;
#define LOAD_W_VEC_2_3() \
eor XTMP4.16b, XTMP1.16b, XTMP0.16b;
#define LOAD_W_VEC_2_4() \
eor XTMP5.16b, XTMP2.16b, XTMP1.16b;
#define LOAD_W_VEC_2_5() \
st1 {XTMP0.16b}, [addr0], #16;
#define LOAD_W_VEC_2_6() \
st1 {XTMP4.16b}, [addr0]; \
add addr0, sp, #IW_W1_ADDR(8, 0);
#define LOAD_W_VEC_2_7() \
eor XTMP6.16b, XTMP3.16b, XTMP2.16b;
#define LOAD_W_VEC_2_8() \
ext W0.16b, XTMP0.16b, XTMP0.16b, #8; /* W0: xx, w0, xx, xx */
#define LOAD_W_VEC_3_1() \
mov W2.16b, XTMP1.16b; /* W2: xx, w6, w5, w4 */
#define LOAD_W_VEC_3_2() \
st1 {XTMP1.16b}, [addr1], #16;
#define LOAD_W_VEC_3_3() \
st1 {XTMP5.16b}, [addr1]; \
ext W1.16b, XTMP0.16b, XTMP0.16b, #4; /* W1: xx, w3, w2, w1 */
#define LOAD_W_VEC_3_4() \
ext W3.16b, XTMP1.16b, XTMP2.16b, #12; /* W3: xx, w9, w8, w7 */
#define LOAD_W_VEC_3_5() \
ext W4.16b, XTMP2.16b, XTMP3.16b, #8; /* W4: xx, w12, w11, w10 */
#define LOAD_W_VEC_3_6() \
st1 {XTMP2.16b}, [addr0], #16;
#define LOAD_W_VEC_3_7() \
st1 {XTMP6.16b}, [addr0];
#define LOAD_W_VEC_3_8() \
ext W5.16b, XTMP3.16b, XTMP3.16b, #4; /* W5: xx, w15, w14, w13 */
#define LOAD_W_VEC_1(iop_num, ...) \
LOAD_W_VEC_1_##iop_num()
#define LOAD_W_VEC_2(iop_num, ...) \
LOAD_W_VEC_2_##iop_num()
#define LOAD_W_VEC_3(iop_num, ...) \
LOAD_W_VEC_3_##iop_num()
/* Message scheduling. Note: 3 words per vector register.
* Interleaving within round function needed for in-order CPUs. */
#define SCHED_W_1_1(round, w0, w1, w2, w3, w4, w5) \
/* Load (w[i - 16]) => XTMP0 */ \
/* Load (w[i - 13]) => XTMP5 */ \
ext XTMP0.16b, w0.16b, w0.16b, #12; /* XTMP0: w0, xx, xx, xx */
#define SCHED_W_1_2(round, w0, w1, w2, w3, w4, w5) \
ext XTMP5.16b, w1.16b, w1.16b, #12;
#define SCHED_W_1_3(round, w0, w1, w2, w3, w4, w5) \
ext XTMP0.16b, XTMP0.16b, w1.16b, #12; /* XTMP0: xx, w2, w1, w0 */
#define SCHED_W_1_4(round, w0, w1, w2, w3, w4, w5) \
ext XTMP5.16b, XTMP5.16b, w2.16b, #12;
#define SCHED_W_1_5(round, w0, w1, w2, w3, w4, w5) \
/* w[i - 9] == w3 */ \
/* W3 ^ XTMP0 => XTMP0 */ \
eor XTMP0.16b, XTMP0.16b, w3.16b;
#define SCHED_W_1_6(round, w0, w1, w2, w3, w4, w5) \
/* w[i - 3] == w5 */ \
/* rol(XMM5, 15) ^ XTMP0 => XTMP0 */ \
/* rol(XTMP5, 7) => XTMP1 */ \
add addr0, sp, #XW_W1_ADDR((round), 0); \
shl XTMP2.4s, w5.4s, #15;
#define SCHED_W_1_7(round, w0, w1, w2, w3, w4, w5) \
shl XTMP1.4s, XTMP5.4s, #7;
#define SCHED_W_1_8(round, w0, w1, w2, w3, w4, w5) \
sri XTMP2.4s, w5.4s, #(32-15);
#define SCHED_W_2_1(round, w0, w1, w2, w3, w4, w5) \
sri XTMP1.4s, XTMP5.4s, #(32-7);
#define SCHED_W_2_2(round, w0, w1, w2, w3, w4, w5) \
eor XTMP0.16b, XTMP0.16b, XTMP2.16b;
#define SCHED_W_2_3(round, w0, w1, w2, w3, w4, w5) \
/* w[i - 6] == W4 */ \
/* W4 ^ XTMP1 => XTMP1 */ \
eor XTMP1.16b, XTMP1.16b, w4.16b;
#define SCHED_W_2_4(round, w0, w1, w2, w3, w4, w5) \
/* P1(XTMP0) ^ XTMP1 => W0 */ \
shl XTMP3.4s, XTMP0.4s, #15;
#define SCHED_W_2_5(round, w0, w1, w2, w3, w4, w5) \
shl XTMP4.4s, XTMP0.4s, #23;
#define SCHED_W_2_6(round, w0, w1, w2, w3, w4, w5) \
eor w0.16b, XTMP1.16b, XTMP0.16b;
#define SCHED_W_2_7(round, w0, w1, w2, w3, w4, w5) \
sri XTMP3.4s, XTMP0.4s, #(32-15);
#define SCHED_W_2_8(round, w0, w1, w2, w3, w4, w5) \
sri XTMP4.4s, XTMP0.4s, #(32-23);
#define SCHED_W_3_1(round, w0, w1, w2, w3, w4, w5) \
eor w0.16b, w0.16b, XTMP3.16b;
#define SCHED_W_3_2(round, w0, w1, w2, w3, w4, w5) \
/* Load (w[i - 3]) => XTMP2 */ \
ext XTMP2.16b, w4.16b, w4.16b, #12;
#define SCHED_W_3_3(round, w0, w1, w2, w3, w4, w5) \
eor w0.16b, w0.16b, XTMP4.16b;
#define SCHED_W_3_4(round, w0, w1, w2, w3, w4, w5) \
ext XTMP2.16b, XTMP2.16b, w5.16b, #12;
#define SCHED_W_3_5(round, w0, w1, w2, w3, w4, w5) \
/* W1 ^ W2 => XTMP3 */ \
eor XTMP3.16b, XTMP2.16b, w0.16b;
#define SCHED_W_3_6(round, w0, w1, w2, w3, w4, w5)
#define SCHED_W_3_7(round, w0, w1, w2, w3, w4, w5) \
st1 {XTMP2.16b-XTMP3.16b}, [addr0];
#define SCHED_W_3_8(round, w0, w1, w2, w3, w4, w5)
#define SCHED_W_W0W1W2W3W4W5_1(iop_num, round) \
SCHED_W_1_##iop_num(round, W0, W1, W2, W3, W4, W5)
#define SCHED_W_W0W1W2W3W4W5_2(iop_num, round) \
SCHED_W_2_##iop_num(round, W0, W1, W2, W3, W4, W5)
#define SCHED_W_W0W1W2W3W4W5_3(iop_num, round) \
SCHED_W_3_##iop_num(round, W0, W1, W2, W3, W4, W5)
#define SCHED_W_W1W2W3W4W5W0_1(iop_num, round) \
SCHED_W_1_##iop_num(round, W1, W2, W3, W4, W5, W0)
#define SCHED_W_W1W2W3W4W5W0_2(iop_num, round) \
SCHED_W_2_##iop_num(round, W1, W2, W3, W4, W5, W0)
#define SCHED_W_W1W2W3W4W5W0_3(iop_num, round) \
SCHED_W_3_##iop_num(round, W1, W2, W3, W4, W5, W0)
#define SCHED_W_W2W3W4W5W0W1_1(iop_num, round) \
SCHED_W_1_##iop_num(round, W2, W3, W4, W5, W0, W1)
#define SCHED_W_W2W3W4W5W0W1_2(iop_num, round) \
SCHED_W_2_##iop_num(round, W2, W3, W4, W5, W0, W1)
#define SCHED_W_W2W3W4W5W0W1_3(iop_num, round) \
SCHED_W_3_##iop_num(round, W2, W3, W4, W5, W0, W1)
#define SCHED_W_W3W4W5W0W1W2_1(iop_num, round) \
SCHED_W_1_##iop_num(round, W3, W4, W5, W0, W1, W2)
#define SCHED_W_W3W4W5W0W1W2_2(iop_num, round) \
SCHED_W_2_##iop_num(round, W3, W4, W5, W0, W1, W2)
#define SCHED_W_W3W4W5W0W1W2_3(iop_num, round) \
SCHED_W_3_##iop_num(round, W3, W4, W5, W0, W1, W2)
#define SCHED_W_W4W5W0W1W2W3_1(iop_num, round) \
SCHED_W_1_##iop_num(round, W4, W5, W0, W1, W2, W3)
#define SCHED_W_W4W5W0W1W2W3_2(iop_num, round) \
SCHED_W_2_##iop_num(round, W4, W5, W0, W1, W2, W3)
#define SCHED_W_W4W5W0W1W2W3_3(iop_num, round) \
SCHED_W_3_##iop_num(round, W4, W5, W0, W1, W2, W3)
#define SCHED_W_W5W0W1W2W3W4_1(iop_num, round) \
SCHED_W_1_##iop_num(round, W5, W0, W1, W2, W3, W4)
#define SCHED_W_W5W0W1W2W3W4_2(iop_num, round) \
SCHED_W_2_##iop_num(round, W5, W0, W1, W2, W3, W4)
#define SCHED_W_W5W0W1W2W3W4_3(iop_num, round) \
SCHED_W_3_##iop_num(round, W5, W0, W1, W2, W3, W4)
/*
* Transform nblocks*64 bytes (nblocks*16 32-bit words) at 'data'.
*
* void sm3_neon_transform(struct sm3_block_state *state,
* const u8 *data, size_t nblocks)
*/
.text
.align 3
SYM_FUNC_START(sm3_neon_transform)
ldp ra, rb, [RSTATE, #0]
ldp rc, rd, [RSTATE, #8]
ldp re, rf, [RSTATE, #16]
ldp rg, rh, [RSTATE, #24]
stp x28, x29, [sp, #-16]!
stp x19, x20, [sp, #-16]!
stp x21, x22, [sp, #-16]!
stp x23, x24, [sp, #-16]!
stp x25, x26, [sp, #-16]!
mov RFRAME, sp
sub addr0, sp, #STACK_SIZE
adr_l RKPTR, .LKtable
and sp, addr0, #(~63)
/* Preload first block. */
LOAD_W_VEC_1(1, 0)
LOAD_W_VEC_1(2, 0)
LOAD_W_VEC_1(3, 0)
LOAD_W_VEC_1(4, 0)
LOAD_W_VEC_1(5, 0)
LOAD_W_VEC_1(6, 0)
LOAD_W_VEC_1(7, 0)
LOAD_W_VEC_1(8, 0)
LOAD_W_VEC_2(1, 0)
LOAD_W_VEC_2(2, 0)
LOAD_W_VEC_2(3, 0)
LOAD_W_VEC_2(4, 0)
LOAD_W_VEC_2(5, 0)
LOAD_W_VEC_2(6, 0)
LOAD_W_VEC_2(7, 0)
LOAD_W_VEC_2(8, 0)
LOAD_W_VEC_3(1, 0)
LOAD_W_VEC_3(2, 0)
LOAD_W_VEC_3(3, 0)
LOAD_W_VEC_3(4, 0)
LOAD_W_VEC_3(5, 0)
LOAD_W_VEC_3(6, 0)
LOAD_W_VEC_3(7, 0)
LOAD_W_VEC_3(8, 0)
.balign 16
.Loop:
/* Transform 0-3 */
R1(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 0, 0, IW, _, 0)
R1(rd, ra, rb, rc, rh, re, rf, rg, k_odd, _, 1, 1, IW, _, 0)
R1(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 2, 2, IW, _, 0)
R1(rb, rc, rd, ra, rf, rg, rh, re, k_odd, _, 3, 3, IW, _, 0)
/* Transform 4-7 + Precalc 12-14 */
R1(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 4, 0, IW, _, 0)
R1(rd, ra, rb, rc, rh, re, rf, rg, k_odd, _, 5, 1, IW, _, 0)
R1(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 6, 2, IW, SCHED_W_W0W1W2W3W4W5_1, 12)
R1(rb, rc, rd, ra, rf, rg, rh, re, k_odd, _, 7, 3, IW, SCHED_W_W0W1W2W3W4W5_2, 12)
/* Transform 8-11 + Precalc 12-17 */
R1(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 8, 0, IW, SCHED_W_W0W1W2W3W4W5_3, 12)
R1(rd, ra, rb, rc, rh, re, rf, rg, k_odd, _, 9, 1, IW, SCHED_W_W1W2W3W4W5W0_1, 15)
R1(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 10, 2, IW, SCHED_W_W1W2W3W4W5W0_2, 15)
R1(rb, rc, rd, ra, rf, rg, rh, re, k_odd, _, 11, 3, IW, SCHED_W_W1W2W3W4W5W0_3, 15)
/* Transform 12-14 + Precalc 18-20 */
R1(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 12, 0, XW, SCHED_W_W2W3W4W5W0W1_1, 18)
R1(rd, ra, rb, rc, rh, re, rf, rg, k_odd, _, 13, 1, XW, SCHED_W_W2W3W4W5W0W1_2, 18)
R1(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 14, 2, XW, SCHED_W_W2W3W4W5W0W1_3, 18)
/* Transform 15-17 + Precalc 21-23 */
R1(rb, rc, rd, ra, rf, rg, rh, re, k_odd, _, 15, 0, XW, SCHED_W_W3W4W5W0W1W2_1, 21)
R2(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 16, 1, XW, SCHED_W_W3W4W5W0W1W2_2, 21)
R2(rd, ra, rb, rc, rh, re, rf, rg, k_odd, _, 17, 2, XW, SCHED_W_W3W4W5W0W1W2_3, 21)
/* Transform 18-20 + Precalc 24-26 */
R2(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 18, 0, XW, SCHED_W_W4W5W0W1W2W3_1, 24)
R2(rb, rc, rd, ra, rf, rg, rh, re, k_odd, _, 19, 1, XW, SCHED_W_W4W5W0W1W2W3_2, 24)
R2(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 20, 2, XW, SCHED_W_W4W5W0W1W2W3_3, 24)
/* Transform 21-23 + Precalc 27-29 */
R2(rd, ra, rb, rc, rh, re, rf, rg, k_odd, _, 21, 0, XW, SCHED_W_W5W0W1W2W3W4_1, 27)
R2(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 22, 1, XW, SCHED_W_W5W0W1W2W3W4_2, 27)
R2(rb, rc, rd, ra, rf, rg, rh, re, k_odd, _, 23, 2, XW, SCHED_W_W5W0W1W2W3W4_3, 27)
/* Transform 24-26 + Precalc 30-32 */
R2(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 24, 0, XW, SCHED_W_W0W1W2W3W4W5_1, 30)
R2(rd, ra, rb, rc, rh, re, rf, rg, k_odd, _, 25, 1, XW, SCHED_W_W0W1W2W3W4W5_2, 30)
R2(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 26, 2, XW, SCHED_W_W0W1W2W3W4W5_3, 30)
/* Transform 27-29 + Precalc 33-35 */
R2(rb, rc, rd, ra, rf, rg, rh, re, k_odd, _, 27, 0, XW, SCHED_W_W1W2W3W4W5W0_1, 33)
R2(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 28, 1, XW, SCHED_W_W1W2W3W4W5W0_2, 33)
R2(rd, ra, rb, rc, rh, re, rf, rg, k_odd, _, 29, 2, XW, SCHED_W_W1W2W3W4W5W0_3, 33)
/* Transform 30-32 + Precalc 36-38 */
R2(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 30, 0, XW, SCHED_W_W2W3W4W5W0W1_1, 36)
R2(rb, rc, rd, ra, rf, rg, rh, re, k_odd, _, 31, 1, XW, SCHED_W_W2W3W4W5W0W1_2, 36)
R2(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 32, 2, XW, SCHED_W_W2W3W4W5W0W1_3, 36)
/* Transform 33-35 + Precalc 39-41 */
R2(rd, ra, rb, rc, rh, re, rf, rg, k_odd, _, 33, 0, XW, SCHED_W_W3W4W5W0W1W2_1, 39)
R2(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 34, 1, XW, SCHED_W_W3W4W5W0W1W2_2, 39)
R2(rb, rc, rd, ra, rf, rg, rh, re, k_odd, _, 35, 2, XW, SCHED_W_W3W4W5W0W1W2_3, 39)
/* Transform 36-38 + Precalc 42-44 */
R2(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 36, 0, XW, SCHED_W_W4W5W0W1W2W3_1, 42)
R2(rd, ra, rb, rc, rh, re, rf, rg, k_odd, _, 37, 1, XW, SCHED_W_W4W5W0W1W2W3_2, 42)
R2(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 38, 2, XW, SCHED_W_W4W5W0W1W2W3_3, 42)
/* Transform 39-41 + Precalc 45-47 */
R2(rb, rc, rd, ra, rf, rg, rh, re, k_odd, _, 39, 0, XW, SCHED_W_W5W0W1W2W3W4_1, 45)
R2(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 40, 1, XW, SCHED_W_W5W0W1W2W3W4_2, 45)
R2(rd, ra, rb, rc, rh, re, rf, rg, k_odd, _, 41, 2, XW, SCHED_W_W5W0W1W2W3W4_3, 45)
/* Transform 42-44 + Precalc 48-50 */
R2(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 42, 0, XW, SCHED_W_W0W1W2W3W4W5_1, 48)
R2(rb, rc, rd, ra, rf, rg, rh, re, k_odd, _, 43, 1, XW, SCHED_W_W0W1W2W3W4W5_2, 48)
R2(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 44, 2, XW, SCHED_W_W0W1W2W3W4W5_3, 48)
/* Transform 45-47 + Precalc 51-53 */
R2(rd, ra, rb, rc, rh, re, rf, rg, k_odd, _, 45, 0, XW, SCHED_W_W1W2W3W4W5W0_1, 51)
R2(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 46, 1, XW, SCHED_W_W1W2W3W4W5W0_2, 51)
R2(rb, rc, rd, ra, rf, rg, rh, re, k_odd, _, 47, 2, XW, SCHED_W_W1W2W3W4W5W0_3, 51)
/* Transform 48-50 + Precalc 54-56 */
R2(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 48, 0, XW, SCHED_W_W2W3W4W5W0W1_1, 54)
R2(rd, ra, rb, rc, rh, re, rf, rg, k_odd, _, 49, 1, XW, SCHED_W_W2W3W4W5W0W1_2, 54)
R2(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 50, 2, XW, SCHED_W_W2W3W4W5W0W1_3, 54)
/* Transform 51-53 + Precalc 57-59 */
R2(rb, rc, rd, ra, rf, rg, rh, re, k_odd, _, 51, 0, XW, SCHED_W_W3W4W5W0W1W2_1, 57)
R2(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 52, 1, XW, SCHED_W_W3W4W5W0W1W2_2, 57)
R2(rd, ra, rb, rc, rh, re, rf, rg, k_odd, _, 53, 2, XW, SCHED_W_W3W4W5W0W1W2_3, 57)
/* Transform 54-56 + Precalc 60-62 */
R2(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 54, 0, XW, SCHED_W_W4W5W0W1W2W3_1, 60)
R2(rb, rc, rd, ra, rf, rg, rh, re, k_odd, _, 55, 1, XW, SCHED_W_W4W5W0W1W2W3_2, 60)
R2(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 56, 2, XW, SCHED_W_W4W5W0W1W2W3_3, 60)
/* Transform 57-59 + Precalc 63 */
R2(rd, ra, rb, rc, rh, re, rf, rg, k_odd, _, 57, 0, XW, SCHED_W_W5W0W1W2W3W4_1, 63)
R2(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 58, 1, XW, SCHED_W_W5W0W1W2W3W4_2, 63)
R2(rb, rc, rd, ra, rf, rg, rh, re, k_odd, _, 59, 2, XW, SCHED_W_W5W0W1W2W3W4_3, 63)
/* Transform 60 */
R2(ra, rb, rc, rd, re, rf, rg, rh, k_even, KL, 60, 0, XW, _, _)
subs RNBLKS, RNBLKS, #1
b.eq .Lend
/* Transform 61-63 + Preload next block */
R2(rd, ra, rb, rc, rh, re, rf, rg, k_odd, _, 61, 1, XW, LOAD_W_VEC_1, _)
ldp s0, s1, [RSTATE, #0]
R2(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 62, 2, XW, LOAD_W_VEC_2, _)
ldp s2, s3, [RSTATE, #8]
R2(rb, rc, rd, ra, rf, rg, rh, re, k_odd, _, 63, 0, XW, LOAD_W_VEC_3, _)
/* Update the chaining variables. */
eor ra, ra, s0
eor rb, rb, s1
ldp s0, s1, [RSTATE, #16]
eor rc, rc, s2
ldp k_even, k_odd, [RSTATE, #24]
eor rd, rd, s3
eor re, re, s0
stp ra, rb, [RSTATE, #0]
eor rf, rf, s1
stp rc, rd, [RSTATE, #8]
eor rg, rg, k_even
stp re, rf, [RSTATE, #16]
eor rh, rh, k_odd
stp rg, rh, [RSTATE, #24]
b .Loop
.Lend:
/* Transform 61-63 */
R2(rd, ra, rb, rc, rh, re, rf, rg, k_odd, _, 61, 1, XW, _, _)
ldp s0, s1, [RSTATE, #0]
R2(rc, rd, ra, rb, rg, rh, re, rf, k_even, KL, 62, 2, XW, _, _)
ldp s2, s3, [RSTATE, #8]
R2(rb, rc, rd, ra, rf, rg, rh, re, k_odd, _, 63, 0, XW, _, _)
/* Update the chaining variables. */
eor ra, ra, s0
clear_vec(W0)
eor rb, rb, s1
clear_vec(W1)
ldp s0, s1, [RSTATE, #16]
clear_vec(W2)
eor rc, rc, s2
clear_vec(W3)
ldp k_even, k_odd, [RSTATE, #24]
clear_vec(W4)
eor rd, rd, s3
clear_vec(W5)
eor re, re, s0
clear_vec(XTMP0)
stp ra, rb, [RSTATE, #0]
clear_vec(XTMP1)
eor rf, rf, s1
clear_vec(XTMP2)
stp rc, rd, [RSTATE, #8]
clear_vec(XTMP3)
eor rg, rg, k_even
clear_vec(XTMP4)
stp re, rf, [RSTATE, #16]
clear_vec(XTMP5)
eor rh, rh, k_odd
clear_vec(XTMP6)
stp rg, rh, [RSTATE, #24]
/* Clear message expansion area */
add addr0, sp, #STACK_W
st1 {W0.16b-W3.16b}, [addr0], #64
st1 {W0.16b-W3.16b}, [addr0], #64
st1 {W0.16b-W3.16b}, [addr0]
mov sp, RFRAME
ldp x25, x26, [sp], #16
ldp x23, x24, [sp], #16
ldp x21, x22, [sp], #16
ldp x19, x20, [sp], #16
ldp x28, x29, [sp], #16
ret
SYM_FUNC_END(sm3_neon_transform)
.section ".rodata", "a"
.align 4
.LKtable:
.long 0x79cc4519, 0xf3988a32, 0xe7311465, 0xce6228cb
.long 0x9cc45197, 0x3988a32f, 0x7311465e, 0xe6228cbc
.long 0xcc451979, 0x988a32f3, 0x311465e7, 0x6228cbce
.long 0xc451979c, 0x88a32f39, 0x11465e73, 0x228cbce6
.long 0x9d8a7a87, 0x3b14f50f, 0x7629ea1e, 0xec53d43c
.long 0xd8a7a879, 0xb14f50f3, 0x629ea1e7, 0xc53d43ce
.long 0x8a7a879d, 0x14f50f3b, 0x29ea1e76, 0x53d43cec
.long 0xa7a879d8, 0x4f50f3b1, 0x9ea1e762, 0x3d43cec5
.long 0x7a879d8a, 0xf50f3b14, 0xea1e7629, 0xd43cec53
.long 0xa879d8a7, 0x50f3b14f, 0xa1e7629e, 0x43cec53d
.long 0x879d8a7a, 0x0f3b14f5, 0x1e7629ea, 0x3cec53d4
.long 0x79d8a7a8, 0xf3b14f50, 0xe7629ea1, 0xcec53d43
.long 0x9d8a7a87, 0x3b14f50f, 0x7629ea1e, 0xec53d43c
.long 0xd8a7a879, 0xb14f50f3, 0x629ea1e7, 0xc53d43ce
.long 0x8a7a879d, 0x14f50f3b, 0x29ea1e76, 0x53d43cec
.long 0xa7a879d8, 0x4f50f3b1, 0x9ea1e762, 0x3d43cec5
linux-7.2.3/lib/crypto/arm64/sm3.h
+41
View File
@@ -0,0 +1,41 @@
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
* SM3 optimized for ARM64
*
* Copyright 2026 Google LLC
*/
#include <asm/simd.h>
#include <linux/cpufeature.h>
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon);
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_ce);
asmlinkage void sm3_neon_transform(struct sm3_block_state *state,
const u8 *data, size_t nblocks);
asmlinkage void sm3_ce_transform(struct sm3_block_state *state,
const u8 *data, size_t nblocks);
static void sm3_blocks(struct sm3_block_state *state,
const u8 *data, size_t nblocks)
{
if (static_branch_likely(&have_neon) && likely(may_use_simd())) {
scoped_ksimd() {
if (static_branch_likely(&have_ce))
sm3_ce_transform(state, data, nblocks);
else
sm3_neon_transform(state, data, nblocks);
}
} else {
sm3_blocks_generic(state, data, nblocks);
}
}
#define sm3_mod_init_arch sm3_mod_init_arch
static void sm3_mod_init_arch(void)
{
if (cpu_have_named_feature(ASIMD)) {
static_branch_enable(&have_neon);
if (cpu_have_named_feature(SM3))
static_branch_enable(&have_ce);
}
}