lastdragon/linux-kernel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Resguardo inicial

Browse Source
This commit is contained in:
lastdragon
2026-09-06 12:53:21 -06:00
parent 662efe3c2a
commit ad2088d178
94853 changed files with 43200404 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
linux-7.2.3/lib/crypto/powerpc/.gitignore
+3
View File
@@ -0,0 +1,3 @@
# SPDX-License-Identifier: GPL-2.0-only
aesp8-ppc.S
ghashp8-ppc.S
linux-7.2.3/lib/crypto/powerpc/aes-spe-core.S
+346
View File
@@ -0,0 +1,346 @@
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
* Fast AES implementation for SPE instruction set (PPC)
*
* This code makes use of the SPE SIMD instruction set as defined in
* http://cache.freescale.com/files/32bit/doc/ref_manual/SPEPIM.pdf
* Implementation is based on optimization guide notes from
* http://cache.freescale.com/files/32bit/doc/app_note/AN2665.pdf
*
* Copyright (c) 2015 Markus Stockhausen <stockhausen@collogia.de>
*/
#include <asm/ppc_asm.h>
#include "aes-spe-regs.h"
#define EAD(in, bpos) \
rlwimi rT0,in,28-((bpos+3)%4)*8,20,27;
#define DAD(in, bpos) \
rlwimi rT1,in,24-((bpos+3)%4)*8,24,31;
#define LWH(out, off) \
evlwwsplat out,off(rT0); /* load word high */
#define LWL(out, off) \
lwz out,off(rT0); /* load word low */
#define LBZ(out, tab, off) \
lbz out,off(tab); /* load byte */
#define LAH(out, in, bpos, off) \
EAD(in, bpos) /* calc addr + load word high */ \
LWH(out, off)
#define LAL(out, in, bpos, off) \
EAD(in, bpos) /* calc addr + load word low */ \
LWL(out, off)
#define LAE(out, in, bpos) \
EAD(in, bpos) /* calc addr + load enc byte */ \
LBZ(out, rT0, 8)
#define LBE(out) \
LBZ(out, rT0, 8) /* load enc byte */
#define LAD(out, in, bpos) \
DAD(in, bpos) /* calc addr + load dec byte */ \
LBZ(out, rT1, 0)
#define LBD(out) \
LBZ(out, rT1, 0)
/*
* ppc_encrypt_block: The central encryption function for a single 16 bytes
* block. It does no stack handling or register saving to support fast calls
* via bl/blr. It expects that caller has pre-xored input data with first
* 4 words of encryption key into rD0-rD3. Pointer/counter registers must
* have also been set up before (rT0, rKP, CTR). Output is stored in rD0-rD3
* and rW0-rW3 and caller must execute a final xor on the output registers.
* All working registers rD0-rD3 & rW0-rW7 are overwritten during processing.
*
*/
_GLOBAL(ppc_encrypt_block)
LAH(rW4, rD1, 2, 4)
LAH(rW6, rD0, 3, 0)
LAH(rW3, rD0, 1, 8)
ppc_encrypt_block_loop:
LAH(rW0, rD3, 0, 12)
LAL(rW0, rD0, 0, 12)
LAH(rW1, rD1, 0, 12)
LAH(rW2, rD2, 1, 8)
LAL(rW2, rD3, 1, 8)
LAL(rW3, rD1, 1, 8)
LAL(rW4, rD2, 2, 4)
LAL(rW6, rD1, 3, 0)
LAH(rW5, rD3, 2, 4)
LAL(rW5, rD0, 2, 4)
LAH(rW7, rD2, 3, 0)
evldw rD1,16(rKP)
EAD(rD3, 3)
evxor rW2,rW2,rW4
LWL(rW7, 0)
evxor rW2,rW2,rW6
EAD(rD2, 0)
evxor rD1,rD1,rW2
LWL(rW1, 12)
evxor rD1,rD1,rW0
evldw rD3,24(rKP)
evmergehi rD0,rD0,rD1
EAD(rD1, 2)
evxor rW3,rW3,rW5
LWH(rW4, 4)
evxor rW3,rW3,rW7
EAD(rD0, 3)
evxor rD3,rD3,rW3
LWH(rW6, 0)
evxor rD3,rD3,rW1
EAD(rD0, 1)
evmergehi rD2,rD2,rD3
LWH(rW3, 8)
LAH(rW0, rD3, 0, 12)
LAL(rW0, rD0, 0, 12)
LAH(rW1, rD1, 0, 12)
LAH(rW2, rD2, 1, 8)
LAL(rW2, rD3, 1, 8)
LAL(rW3, rD1, 1, 8)
LAL(rW4, rD2, 2, 4)
LAL(rW6, rD1, 3, 0)
LAH(rW5, rD3, 2, 4)
LAL(rW5, rD0, 2, 4)
LAH(rW7, rD2, 3, 0)
evldw rD1,32(rKP)
EAD(rD3, 3)
evxor rW2,rW2,rW4
LWL(rW7, 0)
evxor rW2,rW2,rW6
EAD(rD2, 0)
evxor rD1,rD1,rW2
LWL(rW1, 12)
evxor rD1,rD1,rW0
evldw rD3,40(rKP)
evmergehi rD0,rD0,rD1
EAD(rD1, 2)
evxor rW3,rW3,rW5
LWH(rW4, 4)
evxor rW3,rW3,rW7
EAD(rD0, 3)
evxor rD3,rD3,rW3
LWH(rW6, 0)
evxor rD3,rD3,rW1
EAD(rD0, 1)
evmergehi rD2,rD2,rD3
LWH(rW3, 8)
addi rKP,rKP,32
bdnz ppc_encrypt_block_loop
LAH(rW0, rD3, 0, 12)
LAL(rW0, rD0, 0, 12)
LAH(rW1, rD1, 0, 12)
LAH(rW2, rD2, 1, 8)
LAL(rW2, rD3, 1, 8)
LAL(rW3, rD1, 1, 8)
LAL(rW4, rD2, 2, 4)
LAH(rW5, rD3, 2, 4)
LAL(rW6, rD1, 3, 0)
LAL(rW5, rD0, 2, 4)
LAH(rW7, rD2, 3, 0)
evldw rD1,16(rKP)
EAD(rD3, 3)
evxor rW2,rW2,rW4
LWL(rW7, 0)
evxor rW2,rW2,rW6
EAD(rD2, 0)
evxor rD1,rD1,rW2
LWL(rW1, 12)
evxor rD1,rD1,rW0
evldw rD3,24(rKP)
evmergehi rD0,rD0,rD1
EAD(rD1, 0)
evxor rW3,rW3,rW5
LBE(rW2)
evxor rW3,rW3,rW7
EAD(rD0, 1)
evxor rD3,rD3,rW3
LBE(rW6)
evxor rD3,rD3,rW1
EAD(rD0, 0)
evmergehi rD2,rD2,rD3
LBE(rW1)
LAE(rW0, rD3, 0)
LAE(rW1, rD0, 0)
LAE(rW4, rD2, 1)
LAE(rW5, rD3, 1)
LAE(rW3, rD2, 0)
LAE(rW7, rD1, 1)
rlwimi rW0,rW4,8,16,23
rlwimi rW1,rW5,8,16,23
LAE(rW4, rD1, 2)
LAE(rW5, rD2, 2)
rlwimi rW2,rW6,8,16,23
rlwimi rW3,rW7,8,16,23
LAE(rW6, rD3, 2)
LAE(rW7, rD0, 2)
rlwimi rW0,rW4,16,8,15
rlwimi rW1,rW5,16,8,15
LAE(rW4, rD0, 3)
LAE(rW5, rD1, 3)
rlwimi rW2,rW6,16,8,15
lwz rD0,32(rKP)
rlwimi rW3,rW7,16,8,15
lwz rD1,36(rKP)
LAE(rW6, rD2, 3)
LAE(rW7, rD3, 3)
rlwimi rW0,rW4,24,0,7
lwz rD2,40(rKP)
rlwimi rW1,rW5,24,0,7
lwz rD3,44(rKP)
rlwimi rW2,rW6,24,0,7
rlwimi rW3,rW7,24,0,7
blr
/*
* ppc_decrypt_block: The central decryption function for a single 16 bytes
* block. It does no stack handling or register saving to support fast calls
* via bl/blr. It expects that caller has pre-xored input data with first
* 4 words of encryption key into rD0-rD3. Pointer/counter registers must
* have also been set up before (rT0, rKP, CTR). Output is stored in rD0-rD3
* and rW0-rW3 and caller must execute a final xor on the output registers.
* All working registers rD0-rD3 & rW0-rW7 are overwritten during processing.
*
*/
_GLOBAL(ppc_decrypt_block)
LAH(rW0, rD1, 0, 12)
LAH(rW6, rD0, 3, 0)
LAH(rW3, rD0, 1, 8)
ppc_decrypt_block_loop:
LAH(rW1, rD3, 0, 12)
LAL(rW0, rD2, 0, 12)
LAH(rW2, rD2, 1, 8)
LAL(rW2, rD3, 1, 8)
LAH(rW4, rD3, 2, 4)
LAL(rW4, rD0, 2, 4)
LAL(rW6, rD1, 3, 0)
LAH(rW5, rD1, 2, 4)
LAH(rW7, rD2, 3, 0)
LAL(rW7, rD3, 3, 0)
LAL(rW3, rD1, 1, 8)
evldw rD1,16(rKP)
EAD(rD0, 0)
evxor rW4,rW4,rW6
LWL(rW1, 12)
evxor rW0,rW0,rW4
EAD(rD2, 2)
evxor rW0,rW0,rW2
LWL(rW5, 4)
evxor rD1,rD1,rW0
evldw rD3,24(rKP)
evmergehi rD0,rD0,rD1
EAD(rD1, 0)
evxor rW3,rW3,rW7
LWH(rW0, 12)
evxor rW3,rW3,rW1
EAD(rD0, 3)
evxor rD3,rD3,rW3
LWH(rW6, 0)
evxor rD3,rD3,rW5
EAD(rD0, 1)
evmergehi rD2,rD2,rD3
LWH(rW3, 8)
LAH(rW1, rD3, 0, 12)
LAL(rW0, rD2, 0, 12)
LAH(rW2, rD2, 1, 8)
LAL(rW2, rD3, 1, 8)
LAH(rW4, rD3, 2, 4)
LAL(rW4, rD0, 2, 4)
LAL(rW6, rD1, 3, 0)
LAH(rW5, rD1, 2, 4)
LAH(rW7, rD2, 3, 0)
LAL(rW7, rD3, 3, 0)
LAL(rW3, rD1, 1, 8)
evldw rD1,32(rKP)
EAD(rD0, 0)
evxor rW4,rW4,rW6
LWL(rW1, 12)
evxor rW0,rW0,rW4
EAD(rD2, 2)
evxor rW0,rW0,rW2
LWL(rW5, 4)
evxor rD1,rD1,rW0
evldw rD3,40(rKP)
evmergehi rD0,rD0,rD1
EAD(rD1, 0)
evxor rW3,rW3,rW7
LWH(rW0, 12)
evxor rW3,rW3,rW1
EAD(rD0, 3)
evxor rD3,rD3,rW3
LWH(rW6, 0)
evxor rD3,rD3,rW5
EAD(rD0, 1)
evmergehi rD2,rD2,rD3
LWH(rW3, 8)
addi rKP,rKP,32
bdnz ppc_decrypt_block_loop
LAH(rW1, rD3, 0, 12)
LAL(rW0, rD2, 0, 12)
LAH(rW2, rD2, 1, 8)
LAL(rW2, rD3, 1, 8)
LAH(rW4, rD3, 2, 4)
LAL(rW4, rD0, 2, 4)
LAL(rW6, rD1, 3, 0)
LAH(rW5, rD1, 2, 4)
LAH(rW7, rD2, 3, 0)
LAL(rW7, rD3, 3, 0)
LAL(rW3, rD1, 1, 8)
evldw rD1,16(rKP)
EAD(rD0, 0)
evxor rW4,rW4,rW6
LWL(rW1, 12)
evxor rW0,rW0,rW4
EAD(rD2, 2)
evxor rW0,rW0,rW2
LWL(rW5, 4)
evxor rD1,rD1,rW0
evldw rD3,24(rKP)
evmergehi rD0,rD0,rD1
DAD(rD1, 0)
evxor rW3,rW3,rW7
LBD(rW0)
evxor rW3,rW3,rW1
DAD(rD0, 1)
evxor rD3,rD3,rW3
LBD(rW6)
evxor rD3,rD3,rW5
DAD(rD0, 0)
evmergehi rD2,rD2,rD3
LBD(rW3)
LAD(rW2, rD3, 0)
LAD(rW1, rD2, 0)
LAD(rW4, rD2, 1)
LAD(rW5, rD3, 1)
LAD(rW7, rD1, 1)
rlwimi rW0,rW4,8,16,23
rlwimi rW1,rW5,8,16,23
LAD(rW4, rD3, 2)
LAD(rW5, rD0, 2)
rlwimi rW2,rW6,8,16,23
rlwimi rW3,rW7,8,16,23
LAD(rW6, rD1, 2)
LAD(rW7, rD2, 2)
rlwimi rW0,rW4,16,8,15
rlwimi rW1,rW5,16,8,15
LAD(rW4, rD0, 3)
LAD(rW5, rD1, 3)
rlwimi rW2,rW6,16,8,15
lwz rD0,32(rKP)
rlwimi rW3,rW7,16,8,15
lwz rD1,36(rKP)
LAD(rW6, rD2, 3)
LAD(rW7, rD3, 3)
rlwimi rW0,rW4,24,0,7
lwz rD2,40(rKP)
rlwimi rW1,rW5,24,0,7
lwz rD3,44(rKP)
rlwimi rW2,rW6,24,0,7
rlwimi rW3,rW7,24,0,7
blr
linux-7.2.3/lib/crypto/powerpc/aes-spe-keys.S
+278
View File
@@ -0,0 +1,278 @@
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
* Key handling functions for PPC AES implementation
*
* Copyright (c) 2015 Markus Stockhausen <stockhausen@collogia.de>
*/
#include <asm/ppc_asm.h>
#ifdef __BIG_ENDIAN__
#define LOAD_KEY(d, s, off) \
lwz d,off(s);
#else
#define LOAD_KEY(d, s, off) \
li r0,off; \
lwbrx d,s,r0;
#endif
#define INITIALIZE_KEY \
stwu r1,-32(r1); /* create stack frame */ \
stw r14,8(r1); /* save registers */ \
stw r15,12(r1); \
stw r16,16(r1);
#define FINALIZE_KEY \
lwz r14,8(r1); /* restore registers */ \
lwz r15,12(r1); \
lwz r16,16(r1); \
xor r5,r5,r5; /* clear sensitive data */ \
xor r6,r6,r6; \
xor r7,r7,r7; \
xor r8,r8,r8; \
xor r9,r9,r9; \
xor r10,r10,r10; \
xor r11,r11,r11; \
xor r12,r12,r12; \
addi r1,r1,32; /* cleanup stack */
#define LS_BOX(r, t1, t2) \
lis t2,PPC_AES_4K_ENCTAB@h; \
ori t2,t2,PPC_AES_4K_ENCTAB@l; \
rlwimi t2,r,4,20,27; \
lbz t1,8(t2); \
rlwimi r,t1,0,24,31; \
rlwimi t2,r,28,20,27; \
lbz t1,8(t2); \
rlwimi r,t1,8,16,23; \
rlwimi t2,r,20,20,27; \
lbz t1,8(t2); \
rlwimi r,t1,16,8,15; \
rlwimi t2,r,12,20,27; \
lbz t1,8(t2); \
rlwimi r,t1,24,0,7;
#define GF8_MUL(out, in, t1, t2) \
lis t1,0x8080; /* multiplication in GF8 */ \
ori t1,t1,0x8080; \
and t1,t1,in; \
srwi t1,t1,7; \
mulli t1,t1,0x1b; \
lis t2,0x7f7f; \
ori t2,t2,0x7f7f; \
and t2,t2,in; \
slwi t2,t2,1; \
xor out,t1,t2;
/*
* ppc_expand_key_128(u32 *key_enc, const u8 *key)
*
* Expand 128 bit key into 176 bytes encryption key. It consists of
* key itself plus 10 rounds with 16 bytes each
*
*/
_GLOBAL(ppc_expand_key_128)
INITIALIZE_KEY
LOAD_KEY(r5,r4,0)
LOAD_KEY(r6,r4,4)
LOAD_KEY(r7,r4,8)
LOAD_KEY(r8,r4,12)
stw r5,0(r3) /* key[0..3] = input data */
stw r6,4(r3)
stw r7,8(r3)
stw r8,12(r3)
li r16,10 /* 10 expansion rounds */
lis r0,0x0100 /* RCO(1) */
ppc_expand_128_loop:
addi r3,r3,16
mr r14,r8 /* apply LS_BOX to 4th temp */
rotlwi r14,r14,8
LS_BOX(r14, r15, r4)
xor r14,r14,r0
xor r5,r5,r14 /* xor next 4 keys */
xor r6,r6,r5
xor r7,r7,r6
xor r8,r8,r7
stw r5,0(r3) /* store next 4 keys */
stw r6,4(r3)
stw r7,8(r3)
stw r8,12(r3)
GF8_MUL(r0, r0, r4, r14) /* multiply RCO by 2 in GF */
subi r16,r16,1
cmpwi r16,0
bt eq,ppc_expand_128_end
b ppc_expand_128_loop
ppc_expand_128_end:
FINALIZE_KEY
blr
/*
* ppc_expand_key_192(u32 *key_enc, const u8 *key)
*
* Expand 192 bit key into 208 bytes encryption key. It consists of key
* itself plus 12 rounds with 16 bytes each
*
*/
_GLOBAL(ppc_expand_key_192)
INITIALIZE_KEY
LOAD_KEY(r5,r4,0)
LOAD_KEY(r6,r4,4)
LOAD_KEY(r7,r4,8)
LOAD_KEY(r8,r4,12)
LOAD_KEY(r9,r4,16)
LOAD_KEY(r10,r4,20)
stw r5,0(r3)
stw r6,4(r3)
stw r7,8(r3)
stw r8,12(r3)
stw r9,16(r3)
stw r10,20(r3)
li r16,8 /* 8 expansion rounds */
lis r0,0x0100 /* RCO(1) */
ppc_expand_192_loop:
addi r3,r3,24
mr r14,r10 /* apply LS_BOX to 6th temp */
rotlwi r14,r14,8
LS_BOX(r14, r15, r4)
xor r14,r14,r0
xor r5,r5,r14 /* xor next 6 keys */
xor r6,r6,r5
xor r7,r7,r6
xor r8,r8,r7
xor r9,r9,r8
xor r10,r10,r9
stw r5,0(r3)
stw r6,4(r3)
stw r7,8(r3)
stw r8,12(r3)
subi r16,r16,1
cmpwi r16,0 /* last round early kick out */
bt eq,ppc_expand_192_end
stw r9,16(r3)
stw r10,20(r3)
GF8_MUL(r0, r0, r4, r14) /* multiply RCO GF8 */
b ppc_expand_192_loop
ppc_expand_192_end:
FINALIZE_KEY
blr
/*
* ppc_expand_key_256(u32 *key_enc, const u8 *key)
*
* Expand 256 bit key into 240 bytes encryption key. It consists of key
* itself plus 14 rounds with 16 bytes each
*
*/
_GLOBAL(ppc_expand_key_256)
INITIALIZE_KEY
LOAD_KEY(r5,r4,0)
LOAD_KEY(r6,r4,4)
LOAD_KEY(r7,r4,8)
LOAD_KEY(r8,r4,12)
LOAD_KEY(r9,r4,16)
LOAD_KEY(r10,r4,20)
LOAD_KEY(r11,r4,24)
LOAD_KEY(r12,r4,28)
stw r5,0(r3)
stw r6,4(r3)
stw r7,8(r3)
stw r8,12(r3)
stw r9,16(r3)
stw r10,20(r3)
stw r11,24(r3)
stw r12,28(r3)
li r16,7 /* 7 expansion rounds */
lis r0,0x0100 /* RCO(1) */
ppc_expand_256_loop:
addi r3,r3,32
mr r14,r12 /* apply LS_BOX to 8th temp */
rotlwi r14,r14,8
LS_BOX(r14, r15, r4)
xor r14,r14,r0
xor r5,r5,r14 /* xor 4 keys */
xor r6,r6,r5
xor r7,r7,r6
xor r8,r8,r7
mr r14,r8
LS_BOX(r14, r15, r4) /* apply LS_BOX to 4th temp */
xor r9,r9,r14 /* xor 4 keys */
xor r10,r10,r9
xor r11,r11,r10
xor r12,r12,r11
stw r5,0(r3)
stw r6,4(r3)
stw r7,8(r3)
stw r8,12(r3)
subi r16,r16,1
cmpwi r16,0 /* last round early kick out */
bt eq,ppc_expand_256_end
stw r9,16(r3)
stw r10,20(r3)
stw r11,24(r3)
stw r12,28(r3)
GF8_MUL(r0, r0, r4, r14)
b ppc_expand_256_loop
ppc_expand_256_end:
FINALIZE_KEY
blr
/*
* ppc_generate_decrypt_key: derive decryption key from encryption key
* number of bytes to handle are calculated from length of key (16/24/32)
*
*/
_GLOBAL(ppc_generate_decrypt_key)
addi r6,r5,24
slwi r6,r6,2
lwzx r7,r4,r6 /* first/last 4 words are same */
stw r7,0(r3)
lwz r7,0(r4)
stwx r7,r3,r6
addi r6,r6,4
lwzx r7,r4,r6
stw r7,4(r3)
lwz r7,4(r4)
stwx r7,r3,r6
addi r6,r6,4
lwzx r7,r4,r6
stw r7,8(r3)
lwz r7,8(r4)
stwx r7,r3,r6
addi r6,r6,4
lwzx r7,r4,r6
stw r7,12(r3)
lwz r7,12(r4)
stwx r7,r3,r6
addi r3,r3,16
add r4,r4,r6
subi r4,r4,28
addi r5,r5,20
srwi r5,r5,2
ppc_generate_decrypt_block:
li r6,4
mtctr r6
ppc_generate_decrypt_word:
lwz r6,0(r4)
GF8_MUL(r7, r6, r0, r7)
GF8_MUL(r8, r7, r0, r8)
GF8_MUL(r9, r8, r0, r9)
xor r10,r9,r6
xor r11,r7,r8
xor r11,r11,r9
xor r12,r7,r10
rotrwi r12,r12,24
xor r11,r11,r12
xor r12,r8,r10
rotrwi r12,r12,16
xor r11,r11,r12
rotrwi r12,r10,8
xor r11,r11,r12
stw r11,0(r3)
addi r3,r3,4
addi r4,r4,4
bdnz ppc_generate_decrypt_word
subi r4,r4,32
subi r5,r5,1
cmpwi r5,0
bt gt,ppc_generate_decrypt_block
blr
linux-7.2.3/lib/crypto/powerpc/aes-spe-modes.S
+625
View File
@@ -0,0 +1,625 @@
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
* AES modes (ECB/CBC/CTR/XTS) for PPC AES implementation
*
* Copyright (c) 2015 Markus Stockhausen <stockhausen@collogia.de>
*/
#include <asm/ppc_asm.h>
#include "aes-spe-regs.h"
#ifdef __BIG_ENDIAN__ /* Macros for big endian builds */
#define LOAD_DATA(reg, off) \
lwz reg,off(rSP); /* load with offset */
#define SAVE_DATA(reg, off) \
stw reg,off(rDP); /* save with offset */
#define NEXT_BLOCK \
addi rSP,rSP,16; /* increment pointers per bloc */ \
addi rDP,rDP,16;
#define LOAD_IV(reg, off) \
lwz reg,off(rIP); /* IV loading with offset */
#define SAVE_IV(reg, off) \
stw reg,off(rIP); /* IV saving with offset */
#define START_IV /* nothing to reset */
#define CBC_DEC 16 /* CBC decrement per block */
#define CTR_DEC 1 /* CTR decrement one byte */
#else /* Macros for little endian */
#define LOAD_DATA(reg, off) \
lwbrx reg,0,rSP; /* load reversed */ \
addi rSP,rSP,4; /* and increment pointer */
#define SAVE_DATA(reg, off) \
stwbrx reg,0,rDP; /* save reversed */ \
addi rDP,rDP,4; /* and increment pointer */
#define NEXT_BLOCK /* nothing todo */
#define LOAD_IV(reg, off) \
lwbrx reg,0,rIP; /* load reversed */ \
addi rIP,rIP,4; /* and increment pointer */
#define SAVE_IV(reg, off) \
stwbrx reg,0,rIP; /* load reversed */ \
addi rIP,rIP,4; /* and increment pointer */
#define START_IV \
subi rIP,rIP,16; /* must reset pointer */
#define CBC_DEC 32 /* 2 blocks because of incs */
#define CTR_DEC 17 /* 1 block because of incs */
#endif
#define SAVE_0_REGS
#define LOAD_0_REGS
#define SAVE_4_REGS \
stw rI0,96(r1); /* save 32 bit registers */ \
stw rI1,100(r1); \
stw rI2,104(r1); \
stw rI3,108(r1);
#define LOAD_4_REGS \
lwz rI0,96(r1); /* restore 32 bit registers */ \
lwz rI1,100(r1); \
lwz rI2,104(r1); \
lwz rI3,108(r1);
#define SAVE_8_REGS \
SAVE_4_REGS \
stw rG0,112(r1); /* save 32 bit registers */ \
stw rG1,116(r1); \
stw rG2,120(r1); \
stw rG3,124(r1);
#define LOAD_8_REGS \
LOAD_4_REGS \
lwz rG0,112(r1); /* restore 32 bit registers */ \
lwz rG1,116(r1); \
lwz rG2,120(r1); \
lwz rG3,124(r1);
#define INITIALIZE_CRYPT(tab,nr32bitregs) \
mflr r0; \
stwu r1,-160(r1); /* create stack frame */ \
lis rT0,tab@h; /* en-/decryption table pointer */ \
stw r0,8(r1); /* save link register */ \
ori rT0,rT0,tab@l; \
evstdw r14,16(r1); \
mr rKS,rKP; \
evstdw r15,24(r1); /* We must save non volatile */ \
evstdw r16,32(r1); /* registers. Take the chance */ \
evstdw r17,40(r1); /* and save the SPE part too */ \
evstdw r18,48(r1); \
evstdw r19,56(r1); \
evstdw r20,64(r1); \
evstdw r21,72(r1); \
evstdw r22,80(r1); \
evstdw r23,88(r1); \
SAVE_##nr32bitregs##_REGS
#define FINALIZE_CRYPT(nr32bitregs) \
lwz r0,8(r1); \
evldw r14,16(r1); /* restore SPE registers */ \
evldw r15,24(r1); \
evldw r16,32(r1); \
evldw r17,40(r1); \
evldw r18,48(r1); \
evldw r19,56(r1); \
evldw r20,64(r1); \
evldw r21,72(r1); \
evldw r22,80(r1); \
evldw r23,88(r1); \
LOAD_##nr32bitregs##_REGS \
mtlr r0; /* restore link register */ \
xor r0,r0,r0; \
stw r0,16(r1); /* delete sensitive data */ \
stw r0,24(r1); /* that we might have pushed */ \
stw r0,32(r1); /* from other context that runs */ \
stw r0,40(r1); /* the same code */ \
stw r0,48(r1); \
stw r0,56(r1); \
stw r0,64(r1); \
stw r0,72(r1); \
stw r0,80(r1); \
stw r0,88(r1); \
addi r1,r1,160; /* cleanup stack frame */
#define ENDIAN_SWAP(t0, t1, s0, s1) \
rotrwi t0,s0,8; /* swap endianness for 2 GPRs */ \
rotrwi t1,s1,8; \
rlwimi t0,s0,8,8,15; \
rlwimi t1,s1,8,8,15; \
rlwimi t0,s0,8,24,31; \
rlwimi t1,s1,8,24,31;
#define GF128_MUL(d0, d1, d2, d3, t0) \
li t0,0x87; /* multiplication in GF128 */ \
cmpwi d3,-1; \
iselgt t0,0,t0; \
rlwimi d3,d2,0,0,0; /* propagate "carry" bits */ \
rotlwi d3,d3,1; \
rlwimi d2,d1,0,0,0; \
rotlwi d2,d2,1; \
rlwimi d1,d0,0,0,0; \
slwi d0,d0,1; /* shift left 128 bit */ \
rotlwi d1,d1,1; \
xor d0,d0,t0;
#define START_KEY(d0, d1, d2, d3) \
lwz rW0,0(rKP); \
mtctr rRR; \
lwz rW1,4(rKP); \
lwz rW2,8(rKP); \
lwz rW3,12(rKP); \
xor rD0,d0,rW0; \
xor rD1,d1,rW1; \
xor rD2,d2,rW2; \
xor rD3,d3,rW3;
/*
* ppc_encrypt_aes(u8 *out, const u8 *in, u32 *key_enc,
* u32 rounds)
*
* called from glue layer to encrypt a single 16 byte block
* round values are AES128 = 4, AES192 = 5, AES256 = 6
*
*/
_GLOBAL(ppc_encrypt_aes)
INITIALIZE_CRYPT(PPC_AES_4K_ENCTAB, 0)
LOAD_DATA(rD0, 0)
LOAD_DATA(rD1, 4)
LOAD_DATA(rD2, 8)
LOAD_DATA(rD3, 12)
START_KEY(rD0, rD1, rD2, rD3)
bl ppc_encrypt_block
xor rD0,rD0,rW0
SAVE_DATA(rD0, 0)
xor rD1,rD1,rW1
SAVE_DATA(rD1, 4)
xor rD2,rD2,rW2
SAVE_DATA(rD2, 8)
xor rD3,rD3,rW3
SAVE_DATA(rD3, 12)
FINALIZE_CRYPT(0)
blr
/*
* ppc_decrypt_aes(u8 *out, const u8 *in, u32 *key_dec,
* u32 rounds)
*
* called from glue layer to decrypt a single 16 byte block
* round values are AES128 = 4, AES192 = 5, AES256 = 6
*
*/
_GLOBAL(ppc_decrypt_aes)
INITIALIZE_CRYPT(PPC_AES_4K_DECTAB,0)
LOAD_DATA(rD0, 0)
addi rT1,rT0,4096
LOAD_DATA(rD1, 4)
LOAD_DATA(rD2, 8)
LOAD_DATA(rD3, 12)
START_KEY(rD0, rD1, rD2, rD3)
bl ppc_decrypt_block
xor rD0,rD0,rW0
SAVE_DATA(rD0, 0)
xor rD1,rD1,rW1
SAVE_DATA(rD1, 4)
xor rD2,rD2,rW2
SAVE_DATA(rD2, 8)
xor rD3,rD3,rW3
SAVE_DATA(rD3, 12)
FINALIZE_CRYPT(0)
blr
/*
* ppc_encrypt_ecb(u8 *out, const u8 *in, u32 *key_enc,
* u32 rounds, u32 bytes);
*
* called from glue layer to encrypt multiple blocks via ECB
* Bytes must be larger or equal 16 and only whole blocks are
* processed. round values are AES128 = 4, AES192 = 5 and
* AES256 = 6
*
*/
_GLOBAL(ppc_encrypt_ecb)
INITIALIZE_CRYPT(PPC_AES_4K_ENCTAB, 0)
ppc_encrypt_ecb_loop:
LOAD_DATA(rD0, 0)
mr rKP,rKS
LOAD_DATA(rD1, 4)
subi rLN,rLN,16
LOAD_DATA(rD2, 8)
cmpwi rLN,15
LOAD_DATA(rD3, 12)
START_KEY(rD0, rD1, rD2, rD3)
bl ppc_encrypt_block
xor rD0,rD0,rW0
SAVE_DATA(rD0, 0)
xor rD1,rD1,rW1
SAVE_DATA(rD1, 4)
xor rD2,rD2,rW2
SAVE_DATA(rD2, 8)
xor rD3,rD3,rW3
SAVE_DATA(rD3, 12)
NEXT_BLOCK
bt gt,ppc_encrypt_ecb_loop
FINALIZE_CRYPT(0)
blr
/*
* ppc_decrypt_ecb(u8 *out, const u8 *in, u32 *key_dec,
* u32 rounds, u32 bytes);
*
* called from glue layer to decrypt multiple blocks via ECB
* Bytes must be larger or equal 16 and only whole blocks are
* processed. round values are AES128 = 4, AES192 = 5 and
* AES256 = 6
*
*/
_GLOBAL(ppc_decrypt_ecb)
INITIALIZE_CRYPT(PPC_AES_4K_DECTAB, 0)
addi rT1,rT0,4096
ppc_decrypt_ecb_loop:
LOAD_DATA(rD0, 0)
mr rKP,rKS
LOAD_DATA(rD1, 4)
subi rLN,rLN,16
LOAD_DATA(rD2, 8)
cmpwi rLN,15
LOAD_DATA(rD3, 12)
START_KEY(rD0, rD1, rD2, rD3)
bl ppc_decrypt_block
xor rD0,rD0,rW0
SAVE_DATA(rD0, 0)
xor rD1,rD1,rW1
SAVE_DATA(rD1, 4)
xor rD2,rD2,rW2
SAVE_DATA(rD2, 8)
xor rD3,rD3,rW3
SAVE_DATA(rD3, 12)
NEXT_BLOCK
bt gt,ppc_decrypt_ecb_loop
FINALIZE_CRYPT(0)
blr
/*
* ppc_encrypt_cbc(u8 *out, const u8 *in, u32 *key_enc,
* 32 rounds, u32 bytes, u8 *iv);
*
* called from glue layer to encrypt multiple blocks via CBC
* Bytes must be larger or equal 16 and only whole blocks are
* processed. round values are AES128 = 4, AES192 = 5 and
* AES256 = 6
*
*/
_GLOBAL(ppc_encrypt_cbc)
INITIALIZE_CRYPT(PPC_AES_4K_ENCTAB, 4)
LOAD_IV(rI0, 0)
LOAD_IV(rI1, 4)
LOAD_IV(rI2, 8)
LOAD_IV(rI3, 12)
ppc_encrypt_cbc_loop:
LOAD_DATA(rD0, 0)
mr rKP,rKS
LOAD_DATA(rD1, 4)
subi rLN,rLN,16
LOAD_DATA(rD2, 8)
cmpwi rLN,15
LOAD_DATA(rD3, 12)
xor rD0,rD0,rI0
xor rD1,rD1,rI1
xor rD2,rD2,rI2
xor rD3,rD3,rI3
START_KEY(rD0, rD1, rD2, rD3)
bl ppc_encrypt_block
xor rI0,rD0,rW0
SAVE_DATA(rI0, 0)
xor rI1,rD1,rW1
SAVE_DATA(rI1, 4)
xor rI2,rD2,rW2
SAVE_DATA(rI2, 8)
xor rI3,rD3,rW3
SAVE_DATA(rI3, 12)
NEXT_BLOCK
bt gt,ppc_encrypt_cbc_loop
START_IV
SAVE_IV(rI0, 0)
SAVE_IV(rI1, 4)
SAVE_IV(rI2, 8)
SAVE_IV(rI3, 12)
FINALIZE_CRYPT(4)
blr
/*
* ppc_decrypt_cbc(u8 *out, const u8 *in, u32 *key_dec,
* u32 rounds, u32 bytes, u8 *iv);
*
* called from glue layer to decrypt multiple blocks via CBC
* round values are AES128 = 4, AES192 = 5, AES256 = 6
*
*/
_GLOBAL(ppc_decrypt_cbc)
INITIALIZE_CRYPT(PPC_AES_4K_DECTAB, 4)
li rT1,15
LOAD_IV(rI0, 0)
andc rLN,rLN,rT1
LOAD_IV(rI1, 4)
subi rLN,rLN,16
LOAD_IV(rI2, 8)
add rSP,rSP,rLN /* reverse processing */
LOAD_IV(rI3, 12)
add rDP,rDP,rLN
LOAD_DATA(rD0, 0)
addi rT1,rT0,4096
LOAD_DATA(rD1, 4)
LOAD_DATA(rD2, 8)
LOAD_DATA(rD3, 12)
START_IV
SAVE_IV(rD0, 0)
SAVE_IV(rD1, 4)
SAVE_IV(rD2, 8)
cmpwi rLN,16
SAVE_IV(rD3, 12)
bt lt,ppc_decrypt_cbc_end
ppc_decrypt_cbc_loop:
mr rKP,rKS
START_KEY(rD0, rD1, rD2, rD3)
bl ppc_decrypt_block
subi rLN,rLN,16
subi rSP,rSP,CBC_DEC
xor rW0,rD0,rW0
LOAD_DATA(rD0, 0)
xor rW1,rD1,rW1
LOAD_DATA(rD1, 4)
xor rW2,rD2,rW2
LOAD_DATA(rD2, 8)
xor rW3,rD3,rW3
LOAD_DATA(rD3, 12)
xor rW0,rW0,rD0
SAVE_DATA(rW0, 0)
xor rW1,rW1,rD1
SAVE_DATA(rW1, 4)
xor rW2,rW2,rD2
SAVE_DATA(rW2, 8)
xor rW3,rW3,rD3
SAVE_DATA(rW3, 12)
cmpwi rLN,15
subi rDP,rDP,CBC_DEC
bt gt,ppc_decrypt_cbc_loop
ppc_decrypt_cbc_end:
mr rKP,rKS
START_KEY(rD0, rD1, rD2, rD3)
bl ppc_decrypt_block
xor rW0,rW0,rD0
xor rW1,rW1,rD1
xor rW2,rW2,rD2
xor rW3,rW3,rD3
xor rW0,rW0,rI0 /* decrypt with initial IV */
SAVE_DATA(rW0, 0)
xor rW1,rW1,rI1
SAVE_DATA(rW1, 4)
xor rW2,rW2,rI2
SAVE_DATA(rW2, 8)
xor rW3,rW3,rI3
SAVE_DATA(rW3, 12)
FINALIZE_CRYPT(4)
blr
/*
* ppc_crypt_ctr(u8 *out, const u8 *in, u32 *key_enc,
* u32 rounds, u32 bytes, u8 *iv);
*
* called from glue layer to encrypt/decrypt multiple blocks
* via CTR. Number of bytes does not need to be a multiple of
* 16. Round values are AES128 = 4, AES192 = 5, AES256 = 6
*
*/
_GLOBAL(ppc_crypt_ctr)
INITIALIZE_CRYPT(PPC_AES_4K_ENCTAB, 4)
LOAD_IV(rI0, 0)
LOAD_IV(rI1, 4)
LOAD_IV(rI2, 8)
cmpwi rLN,16
LOAD_IV(rI3, 12)
START_IV
bt lt,ppc_crypt_ctr_partial
ppc_crypt_ctr_loop:
mr rKP,rKS
START_KEY(rI0, rI1, rI2, rI3)
bl ppc_encrypt_block
xor rW0,rD0,rW0
xor rW1,rD1,rW1
xor rW2,rD2,rW2
xor rW3,rD3,rW3
LOAD_DATA(rD0, 0)
subi rLN,rLN,16
LOAD_DATA(rD1, 4)
LOAD_DATA(rD2, 8)
LOAD_DATA(rD3, 12)
xor rD0,rD0,rW0
SAVE_DATA(rD0, 0)
xor rD1,rD1,rW1
SAVE_DATA(rD1, 4)
xor rD2,rD2,rW2
SAVE_DATA(rD2, 8)
xor rD3,rD3,rW3
SAVE_DATA(rD3, 12)
addic rI3,rI3,1 /* increase counter */
addze rI2,rI2
addze rI1,rI1
addze rI0,rI0
NEXT_BLOCK
cmpwi rLN,15
bt gt,ppc_crypt_ctr_loop
ppc_crypt_ctr_partial:
cmpwi rLN,0
bt eq,ppc_crypt_ctr_end
mr rKP,rKS
START_KEY(rI0, rI1, rI2, rI3)
bl ppc_encrypt_block
xor rW0,rD0,rW0
SAVE_IV(rW0, 0)
xor rW1,rD1,rW1
SAVE_IV(rW1, 4)
xor rW2,rD2,rW2
SAVE_IV(rW2, 8)
xor rW3,rD3,rW3
SAVE_IV(rW3, 12)
mtctr rLN
subi rIP,rIP,CTR_DEC
subi rSP,rSP,1
subi rDP,rDP,1
ppc_crypt_ctr_xorbyte:
lbzu rW4,1(rIP) /* bytewise xor for partial block */
lbzu rW5,1(rSP)
xor rW4,rW4,rW5
stbu rW4,1(rDP)
bdnz ppc_crypt_ctr_xorbyte
subf rIP,rLN,rIP
addi rIP,rIP,1
addic rI3,rI3,1
addze rI2,rI2
addze rI1,rI1
addze rI0,rI0
ppc_crypt_ctr_end:
SAVE_IV(rI0, 0)
SAVE_IV(rI1, 4)
SAVE_IV(rI2, 8)
SAVE_IV(rI3, 12)
FINALIZE_CRYPT(4)
blr
/*
* ppc_encrypt_xts(u8 *out, const u8 *in, u32 *key_enc,
* u32 rounds, u32 bytes, u8 *iv, u32 *key_twk);
*
* called from glue layer to encrypt multiple blocks via XTS
* If key_twk is given, the initial IV encryption will be
* processed too. Round values are AES128 = 4, AES192 = 5,
* AES256 = 6
*
*/
_GLOBAL(ppc_encrypt_xts)
INITIALIZE_CRYPT(PPC_AES_4K_ENCTAB, 8)
LOAD_IV(rI0, 0)
LOAD_IV(rI1, 4)
LOAD_IV(rI2, 8)
cmpwi rKT,0
LOAD_IV(rI3, 12)
bt eq,ppc_encrypt_xts_notweak
mr rKP,rKT
START_KEY(rI0, rI1, rI2, rI3)
bl ppc_encrypt_block
xor rI0,rD0,rW0
xor rI1,rD1,rW1
xor rI2,rD2,rW2
xor rI3,rD3,rW3
ppc_encrypt_xts_notweak:
ENDIAN_SWAP(rG0, rG1, rI0, rI1)
ENDIAN_SWAP(rG2, rG3, rI2, rI3)
ppc_encrypt_xts_loop:
LOAD_DATA(rD0, 0)
mr rKP,rKS
LOAD_DATA(rD1, 4)
subi rLN,rLN,16
LOAD_DATA(rD2, 8)
LOAD_DATA(rD3, 12)
xor rD0,rD0,rI0
xor rD1,rD1,rI1
xor rD2,rD2,rI2
xor rD3,rD3,rI3
START_KEY(rD0, rD1, rD2, rD3)
bl ppc_encrypt_block
xor rD0,rD0,rW0
xor rD1,rD1,rW1
xor rD2,rD2,rW2
xor rD3,rD3,rW3
xor rD0,rD0,rI0
SAVE_DATA(rD0, 0)
xor rD1,rD1,rI1
SAVE_DATA(rD1, 4)
xor rD2,rD2,rI2
SAVE_DATA(rD2, 8)
xor rD3,rD3,rI3
SAVE_DATA(rD3, 12)
GF128_MUL(rG0, rG1, rG2, rG3, rW0)
ENDIAN_SWAP(rI0, rI1, rG0, rG1)
ENDIAN_SWAP(rI2, rI3, rG2, rG3)
cmpwi rLN,0
NEXT_BLOCK
bt gt,ppc_encrypt_xts_loop
START_IV
SAVE_IV(rI0, 0)
SAVE_IV(rI1, 4)
SAVE_IV(rI2, 8)
SAVE_IV(rI3, 12)
FINALIZE_CRYPT(8)
blr
/*
* ppc_decrypt_xts(u8 *out, const u8 *in, u32 *key_dec,
* u32 rounds, u32 blocks, u8 *iv, u32 *key_twk);
*
* called from glue layer to decrypt multiple blocks via XTS
* If key_twk is given, the initial IV encryption will be
* processed too. Round values are AES128 = 4, AES192 = 5,
* AES256 = 6
*
*/
_GLOBAL(ppc_decrypt_xts)
INITIALIZE_CRYPT(PPC_AES_4K_DECTAB, 8)
LOAD_IV(rI0, 0)
addi rT1,rT0,4096
LOAD_IV(rI1, 4)
LOAD_IV(rI2, 8)
cmpwi rKT,0
LOAD_IV(rI3, 12)
bt eq,ppc_decrypt_xts_notweak
subi rT0,rT0,4096
mr rKP,rKT
START_KEY(rI0, rI1, rI2, rI3)
bl ppc_encrypt_block
xor rI0,rD0,rW0
xor rI1,rD1,rW1
xor rI2,rD2,rW2
xor rI3,rD3,rW3
addi rT0,rT0,4096
ppc_decrypt_xts_notweak:
ENDIAN_SWAP(rG0, rG1, rI0, rI1)
ENDIAN_SWAP(rG2, rG3, rI2, rI3)
ppc_decrypt_xts_loop:
LOAD_DATA(rD0, 0)
mr rKP,rKS
LOAD_DATA(rD1, 4)
subi rLN,rLN,16
LOAD_DATA(rD2, 8)
LOAD_DATA(rD3, 12)
xor rD0,rD0,rI0
xor rD1,rD1,rI1
xor rD2,rD2,rI2
xor rD3,rD3,rI3
START_KEY(rD0, rD1, rD2, rD3)
bl ppc_decrypt_block
xor rD0,rD0,rW0
xor rD1,rD1,rW1
xor rD2,rD2,rW2
xor rD3,rD3,rW3
xor rD0,rD0,rI0
SAVE_DATA(rD0, 0)
xor rD1,rD1,rI1
SAVE_DATA(rD1, 4)
xor rD2,rD2,rI2
SAVE_DATA(rD2, 8)
xor rD3,rD3,rI3
SAVE_DATA(rD3, 12)
GF128_MUL(rG0, rG1, rG2, rG3, rW0)
ENDIAN_SWAP(rI0, rI1, rG0, rG1)
ENDIAN_SWAP(rI2, rI3, rG2, rG3)
cmpwi rLN,0
NEXT_BLOCK
bt gt,ppc_decrypt_xts_loop
START_IV
SAVE_IV(rI0, 0)
SAVE_IV(rI1, 4)
SAVE_IV(rI2, 8)
SAVE_IV(rI3, 12)
FINALIZE_CRYPT(8)
blr
linux-7.2.3/lib/crypto/powerpc/aes-spe-regs.h
+37
View File
@@ -0,0 +1,37 @@
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
* Common registers for PPC AES implementation
*
* Copyright (c) 2015 Markus Stockhausen <stockhausen@collogia.de>
*/
#define rKS r0 /* copy of en-/decryption key pointer */
#define rDP r3 /* destination pointer */
#define rSP r4 /* source pointer */
#define rKP r5 /* pointer to en-/decryption key pointer */
#define rRR r6 /* en-/decryption rounds */
#define rLN r7 /* length of data to be processed */
#define rIP r8 /* potiner to IV (CBC/CTR/XTS modes) */
#define rKT r9 /* pointer to tweak key (XTS mode) */
#define rT0 r11 /* pointers to en-/decryption tables */
#define rT1 r10
#define rD0 r9 /* data */
#define rD1 r14
#define rD2 r12
#define rD3 r15
#define rW0 r16 /* working registers */
#define rW1 r17
#define rW2 r18
#define rW3 r19
#define rW4 r20
#define rW5 r21
#define rW6 r22
#define rW7 r23
#define rI0 r24 /* IV */
#define rI1 r25
#define rI2 r26
#define rI3 r27
#define rG0 r28 /* endian reversed tweak (XTS mode) */
#define rG1 r29
#define rG2 r30
#define rG3 r31
linux-7.2.3/lib/crypto/powerpc/aes-tab-4k.S
+326
View File
@@ -0,0 +1,326 @@
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
* 4K AES tables for PPC AES implementation
*
* Copyright (c) 2015 Markus Stockhausen <stockhausen@collogia.de>
*/
/*
* These big endian AES encryption/decryption tables have been taken from
* crypto/aes_generic.c and are designed to be simply accessed by a combination
* of rlwimi/lwz instructions with a minimum of table registers (usually only
* one required). Thus they are aligned to 4K. The locality of rotated values
* is derived from the reduced offsets that are available in the SPE load
* instructions. E.g. evldw, evlwwsplat, ...
*
* For the safety-conscious it has to be noted that they might be vulnerable
* to cache timing attacks because of their size. Nevertheless in contrast to
* the generic tables they have been reduced from 16KB to 8KB + 256 bytes.
* This is a quite good tradeoff for low power devices (e.g. routers) without
* dedicated encryption hardware where we usually have no multiuser
* environment.
*
*/
#define R(a, b, c, d) \
0x##a##b##c##d, 0x##d##a##b##c, 0x##c##d##a##b, 0x##b##c##d##a
.data
.align 12
.globl PPC_AES_4K_ENCTAB
PPC_AES_4K_ENCTAB:
/* encryption table, same as crypto_ft_tab in crypto/aes-generic.c */
.long R(c6, 63, 63, a5), R(f8, 7c, 7c, 84)
.long R(ee, 77, 77, 99), R(f6, 7b, 7b, 8d)
.long R(ff, f2, f2, 0d), R(d6, 6b, 6b, bd)
.long R(de, 6f, 6f, b1), R(91, c5, c5, 54)
.long R(60, 30, 30, 50), R(02, 01, 01, 03)
.long R(ce, 67, 67, a9), R(56, 2b, 2b, 7d)
.long R(e7, fe, fe, 19), R(b5, d7, d7, 62)
.long R(4d, ab, ab, e6), R(ec, 76, 76, 9a)
.long R(8f, ca, ca, 45), R(1f, 82, 82, 9d)
.long R(89, c9, c9, 40), R(fa, 7d, 7d, 87)
.long R(ef, fa, fa, 15), R(b2, 59, 59, eb)
.long R(8e, 47, 47, c9), R(fb, f0, f0, 0b)
.long R(41, ad, ad, ec), R(b3, d4, d4, 67)
.long R(5f, a2, a2, fd), R(45, af, af, ea)
.long R(23, 9c, 9c, bf), R(53, a4, a4, f7)
.long R(e4, 72, 72, 96), R(9b, c0, c0, 5b)
.long R(75, b7, b7, c2), R(e1, fd, fd, 1c)
.long R(3d, 93, 93, ae), R(4c, 26, 26, 6a)
.long R(6c, 36, 36, 5a), R(7e, 3f, 3f, 41)
.long R(f5, f7, f7, 02), R(83, cc, cc, 4f)
.long R(68, 34, 34, 5c), R(51, a5, a5, f4)
.long R(d1, e5, e5, 34), R(f9, f1, f1, 08)
.long R(e2, 71, 71, 93), R(ab, d8, d8, 73)
.long R(62, 31, 31, 53), R(2a, 15, 15, 3f)
.long R(08, 04, 04, 0c), R(95, c7, c7, 52)
.long R(46, 23, 23, 65), R(9d, c3, c3, 5e)
.long R(30, 18, 18, 28), R(37, 96, 96, a1)
.long R(0a, 05, 05, 0f), R(2f, 9a, 9a, b5)
.long R(0e, 07, 07, 09), R(24, 12, 12, 36)
.long R(1b, 80, 80, 9b), R(df, e2, e2, 3d)
.long R(cd, eb, eb, 26), R(4e, 27, 27, 69)
.long R(7f, b2, b2, cd), R(ea, 75, 75, 9f)
.long R(12, 09, 09, 1b), R(1d, 83, 83, 9e)
.long R(58, 2c, 2c, 74), R(34, 1a, 1a, 2e)
.long R(36, 1b, 1b, 2d), R(dc, 6e, 6e, b2)
.long R(b4, 5a, 5a, ee), R(5b, a0, a0, fb)
.long R(a4, 52, 52, f6), R(76, 3b, 3b, 4d)
.long R(b7, d6, d6, 61), R(7d, b3, b3, ce)
.long R(52, 29, 29, 7b), R(dd, e3, e3, 3e)
.long R(5e, 2f, 2f, 71), R(13, 84, 84, 97)
.long R(a6, 53, 53, f5), R(b9, d1, d1, 68)
.long R(00, 00, 00, 00), R(c1, ed, ed, 2c)
.long R(40, 20, 20, 60), R(e3, fc, fc, 1f)
.long R(79, b1, b1, c8), R(b6, 5b, 5b, ed)
.long R(d4, 6a, 6a, be), R(8d, cb, cb, 46)
.long R(67, be, be, d9), R(72, 39, 39, 4b)
.long R(94, 4a, 4a, de), R(98, 4c, 4c, d4)
.long R(b0, 58, 58, e8), R(85, cf, cf, 4a)
.long R(bb, d0, d0, 6b), R(c5, ef, ef, 2a)
.long R(4f, aa, aa, e5), R(ed, fb, fb, 16)
.long R(86, 43, 43, c5), R(9a, 4d, 4d, d7)
.long R(66, 33, 33, 55), R(11, 85, 85, 94)
.long R(8a, 45, 45, cf), R(e9, f9, f9, 10)
.long R(04, 02, 02, 06), R(fe, 7f, 7f, 81)
.long R(a0, 50, 50, f0), R(78, 3c, 3c, 44)
.long R(25, 9f, 9f, ba), R(4b, a8, a8, e3)
.long R(a2, 51, 51, f3), R(5d, a3, a3, fe)
.long R(80, 40, 40, c0), R(05, 8f, 8f, 8a)
.long R(3f, 92, 92, ad), R(21, 9d, 9d, bc)
.long R(70, 38, 38, 48), R(f1, f5, f5, 04)
.long R(63, bc, bc, df), R(77, b6, b6, c1)
.long R(af, da, da, 75), R(42, 21, 21, 63)
.long R(20, 10, 10, 30), R(e5, ff, ff, 1a)
.long R(fd, f3, f3, 0e), R(bf, d2, d2, 6d)
.long R(81, cd, cd, 4c), R(18, 0c, 0c, 14)
.long R(26, 13, 13, 35), R(c3, ec, ec, 2f)
.long R(be, 5f, 5f, e1), R(35, 97, 97, a2)
.long R(88, 44, 44, cc), R(2e, 17, 17, 39)
.long R(93, c4, c4, 57), R(55, a7, a7, f2)
.long R(fc, 7e, 7e, 82), R(7a, 3d, 3d, 47)
.long R(c8, 64, 64, ac), R(ba, 5d, 5d, e7)
.long R(32, 19, 19, 2b), R(e6, 73, 73, 95)
.long R(c0, 60, 60, a0), R(19, 81, 81, 98)
.long R(9e, 4f, 4f, d1), R(a3, dc, dc, 7f)
.long R(44, 22, 22, 66), R(54, 2a, 2a, 7e)
.long R(3b, 90, 90, ab), R(0b, 88, 88, 83)
.long R(8c, 46, 46, ca), R(c7, ee, ee, 29)
.long R(6b, b8, b8, d3), R(28, 14, 14, 3c)
.long R(a7, de, de, 79), R(bc, 5e, 5e, e2)
.long R(16, 0b, 0b, 1d), R(ad, db, db, 76)
.long R(db, e0, e0, 3b), R(64, 32, 32, 56)
.long R(74, 3a, 3a, 4e), R(14, 0a, 0a, 1e)
.long R(92, 49, 49, db), R(0c, 06, 06, 0a)
.long R(48, 24, 24, 6c), R(b8, 5c, 5c, e4)
.long R(9f, c2, c2, 5d), R(bd, d3, d3, 6e)
.long R(43, ac, ac, ef), R(c4, 62, 62, a6)
.long R(39, 91, 91, a8), R(31, 95, 95, a4)
.long R(d3, e4, e4, 37), R(f2, 79, 79, 8b)
.long R(d5, e7, e7, 32), R(8b, c8, c8, 43)
.long R(6e, 37, 37, 59), R(da, 6d, 6d, b7)
.long R(01, 8d, 8d, 8c), R(b1, d5, d5, 64)
.long R(9c, 4e, 4e, d2), R(49, a9, a9, e0)
.long R(d8, 6c, 6c, b4), R(ac, 56, 56, fa)
.long R(f3, f4, f4, 07), R(cf, ea, ea, 25)
.long R(ca, 65, 65, af), R(f4, 7a, 7a, 8e)
.long R(47, ae, ae, e9), R(10, 08, 08, 18)
.long R(6f, ba, ba, d5), R(f0, 78, 78, 88)
.long R(4a, 25, 25, 6f), R(5c, 2e, 2e, 72)
.long R(38, 1c, 1c, 24), R(57, a6, a6, f1)
.long R(73, b4, b4, c7), R(97, c6, c6, 51)
.long R(cb, e8, e8, 23), R(a1, dd, dd, 7c)
.long R(e8, 74, 74, 9c), R(3e, 1f, 1f, 21)
.long R(96, 4b, 4b, dd), R(61, bd, bd, dc)
.long R(0d, 8b, 8b, 86), R(0f, 8a, 8a, 85)
.long R(e0, 70, 70, 90), R(7c, 3e, 3e, 42)
.long R(71, b5, b5, c4), R(cc, 66, 66, aa)
.long R(90, 48, 48, d8), R(06, 03, 03, 05)
.long R(f7, f6, f6, 01), R(1c, 0e, 0e, 12)
.long R(c2, 61, 61, a3), R(6a, 35, 35, 5f)
.long R(ae, 57, 57, f9), R(69, b9, b9, d0)
.long R(17, 86, 86, 91), R(99, c1, c1, 58)
.long R(3a, 1d, 1d, 27), R(27, 9e, 9e, b9)
.long R(d9, e1, e1, 38), R(eb, f8, f8, 13)
.long R(2b, 98, 98, b3), R(22, 11, 11, 33)
.long R(d2, 69, 69, bb), R(a9, d9, d9, 70)
.long R(07, 8e, 8e, 89), R(33, 94, 94, a7)
.long R(2d, 9b, 9b, b6), R(3c, 1e, 1e, 22)
.long R(15, 87, 87, 92), R(c9, e9, e9, 20)
.long R(87, ce, ce, 49), R(aa, 55, 55, ff)
.long R(50, 28, 28, 78), R(a5, df, df, 7a)
.long R(03, 8c, 8c, 8f), R(59, a1, a1, f8)
.long R(09, 89, 89, 80), R(1a, 0d, 0d, 17)
.long R(65, bf, bf, da), R(d7, e6, e6, 31)
.long R(84, 42, 42, c6), R(d0, 68, 68, b8)
.long R(82, 41, 41, c3), R(29, 99, 99, b0)
.long R(5a, 2d, 2d, 77), R(1e, 0f, 0f, 11)
.long R(7b, b0, b0, cb), R(a8, 54, 54, fc)
.long R(6d, bb, bb, d6), R(2c, 16, 16, 3a)
.globl PPC_AES_4K_DECTAB
PPC_AES_4K_DECTAB:
/* decryption table, same as crypto_it_tab in crypto/aes-generic.c */
.long R(51, f4, a7, 50), R(7e, 41, 65, 53)
.long R(1a, 17, a4, c3), R(3a, 27, 5e, 96)
.long R(3b, ab, 6b, cb), R(1f, 9d, 45, f1)
.long R(ac, fa, 58, ab), R(4b, e3, 03, 93)
.long R(20, 30, fa, 55), R(ad, 76, 6d, f6)
.long R(88, cc, 76, 91), R(f5, 02, 4c, 25)
.long R(4f, e5, d7, fc), R(c5, 2a, cb, d7)
.long R(26, 35, 44, 80), R(b5, 62, a3, 8f)
.long R(de, b1, 5a, 49), R(25, ba, 1b, 67)
.long R(45, ea, 0e, 98), R(5d, fe, c0, e1)
.long R(c3, 2f, 75, 02), R(81, 4c, f0, 12)
.long R(8d, 46, 97, a3), R(6b, d3, f9, c6)
.long R(03, 8f, 5f, e7), R(15, 92, 9c, 95)
.long R(bf, 6d, 7a, eb), R(95, 52, 59, da)
.long R(d4, be, 83, 2d), R(58, 74, 21, d3)
.long R(49, e0, 69, 29), R(8e, c9, c8, 44)
.long R(75, c2, 89, 6a), R(f4, 8e, 79, 78)
.long R(99, 58, 3e, 6b), R(27, b9, 71, dd)
.long R(be, e1, 4f, b6), R(f0, 88, ad, 17)
.long R(c9, 20, ac, 66), R(7d, ce, 3a, b4)
.long R(63, df, 4a, 18), R(e5, 1a, 31, 82)
.long R(97, 51, 33, 60), R(62, 53, 7f, 45)
.long R(b1, 64, 77, e0), R(bb, 6b, ae, 84)
.long R(fe, 81, a0, 1c), R(f9, 08, 2b, 94)
.long R(70, 48, 68, 58), R(8f, 45, fd, 19)
.long R(94, de, 6c, 87), R(52, 7b, f8, b7)
.long R(ab, 73, d3, 23), R(72, 4b, 02, e2)
.long R(e3, 1f, 8f, 57), R(66, 55, ab, 2a)
.long R(b2, eb, 28, 07), R(2f, b5, c2, 03)
.long R(86, c5, 7b, 9a), R(d3, 37, 08, a5)
.long R(30, 28, 87, f2), R(23, bf, a5, b2)
.long R(02, 03, 6a, ba), R(ed, 16, 82, 5c)
.long R(8a, cf, 1c, 2b), R(a7, 79, b4, 92)
.long R(f3, 07, f2, f0), R(4e, 69, e2, a1)
.long R(65, da, f4, cd), R(06, 05, be, d5)
.long R(d1, 34, 62, 1f), R(c4, a6, fe, 8a)
.long R(34, 2e, 53, 9d), R(a2, f3, 55, a0)
.long R(05, 8a, e1, 32), R(a4, f6, eb, 75)
.long R(0b, 83, ec, 39), R(40, 60, ef, aa)
.long R(5e, 71, 9f, 06), R(bd, 6e, 10, 51)
.long R(3e, 21, 8a, f9), R(96, dd, 06, 3d)
.long R(dd, 3e, 05, ae), R(4d, e6, bd, 46)
.long R(91, 54, 8d, b5), R(71, c4, 5d, 05)
.long R(04, 06, d4, 6f), R(60, 50, 15, ff)
.long R(19, 98, fb, 24), R(d6, bd, e9, 97)
.long R(89, 40, 43, cc), R(67, d9, 9e, 77)
.long R(b0, e8, 42, bd), R(07, 89, 8b, 88)
.long R(e7, 19, 5b, 38), R(79, c8, ee, db)
.long R(a1, 7c, 0a, 47), R(7c, 42, 0f, e9)
.long R(f8, 84, 1e, c9), R(00, 00, 00, 00)
.long R(09, 80, 86, 83), R(32, 2b, ed, 48)
.long R(1e, 11, 70, ac), R(6c, 5a, 72, 4e)
.long R(fd, 0e, ff, fb), R(0f, 85, 38, 56)
.long R(3d, ae, d5, 1e), R(36, 2d, 39, 27)
.long R(0a, 0f, d9, 64), R(68, 5c, a6, 21)
.long R(9b, 5b, 54, d1), R(24, 36, 2e, 3a)
.long R(0c, 0a, 67, b1), R(93, 57, e7, 0f)
.long R(b4, ee, 96, d2), R(1b, 9b, 91, 9e)
.long R(80, c0, c5, 4f), R(61, dc, 20, a2)
.long R(5a, 77, 4b, 69), R(1c, 12, 1a, 16)
.long R(e2, 93, ba, 0a), R(c0, a0, 2a, e5)
.long R(3c, 22, e0, 43), R(12, 1b, 17, 1d)
.long R(0e, 09, 0d, 0b), R(f2, 8b, c7, ad)
.long R(2d, b6, a8, b9), R(14, 1e, a9, c8)
.long R(57, f1, 19, 85), R(af, 75, 07, 4c)
.long R(ee, 99, dd, bb), R(a3, 7f, 60, fd)
.long R(f7, 01, 26, 9f), R(5c, 72, f5, bc)
.long R(44, 66, 3b, c5), R(5b, fb, 7e, 34)
.long R(8b, 43, 29, 76), R(cb, 23, c6, dc)
.long R(b6, ed, fc, 68), R(b8, e4, f1, 63)
.long R(d7, 31, dc, ca), R(42, 63, 85, 10)
.long R(13, 97, 22, 40), R(84, c6, 11, 20)
.long R(85, 4a, 24, 7d), R(d2, bb, 3d, f8)
.long R(ae, f9, 32, 11), R(c7, 29, a1, 6d)
.long R(1d, 9e, 2f, 4b), R(dc, b2, 30, f3)
.long R(0d, 86, 52, ec), R(77, c1, e3, d0)
.long R(2b, b3, 16, 6c), R(a9, 70, b9, 99)
.long R(11, 94, 48, fa), R(47, e9, 64, 22)
.long R(a8, fc, 8c, c4), R(a0, f0, 3f, 1a)
.long R(56, 7d, 2c, d8), R(22, 33, 90, ef)
.long R(87, 49, 4e, c7), R(d9, 38, d1, c1)
.long R(8c, ca, a2, fe), R(98, d4, 0b, 36)
.long R(a6, f5, 81, cf), R(a5, 7a, de, 28)
.long R(da, b7, 8e, 26), R(3f, ad, bf, a4)
.long R(2c, 3a, 9d, e4), R(50, 78, 92, 0d)
.long R(6a, 5f, cc, 9b), R(54, 7e, 46, 62)
.long R(f6, 8d, 13, c2), R(90, d8, b8, e8)
.long R(2e, 39, f7, 5e), R(82, c3, af, f5)
.long R(9f, 5d, 80, be), R(69, d0, 93, 7c)
.long R(6f, d5, 2d, a9), R(cf, 25, 12, b3)
.long R(c8, ac, 99, 3b), R(10, 18, 7d, a7)
.long R(e8, 9c, 63, 6e), R(db, 3b, bb, 7b)
.long R(cd, 26, 78, 09), R(6e, 59, 18, f4)
.long R(ec, 9a, b7, 01), R(83, 4f, 9a, a8)
.long R(e6, 95, 6e, 65), R(aa, ff, e6, 7e)
.long R(21, bc, cf, 08), R(ef, 15, e8, e6)
.long R(ba, e7, 9b, d9), R(4a, 6f, 36, ce)
.long R(ea, 9f, 09, d4), R(29, b0, 7c, d6)
.long R(31, a4, b2, af), R(2a, 3f, 23, 31)
.long R(c6, a5, 94, 30), R(35, a2, 66, c0)
.long R(74, 4e, bc, 37), R(fc, 82, ca, a6)
.long R(e0, 90, d0, b0), R(33, a7, d8, 15)
.long R(f1, 04, 98, 4a), R(41, ec, da, f7)
.long R(7f, cd, 50, 0e), R(17, 91, f6, 2f)
.long R(76, 4d, d6, 8d), R(43, ef, b0, 4d)
.long R(cc, aa, 4d, 54), R(e4, 96, 04, df)
.long R(9e, d1, b5, e3), R(4c, 6a, 88, 1b)
.long R(c1, 2c, 1f, b8), R(46, 65, 51, 7f)
.long R(9d, 5e, ea, 04), R(01, 8c, 35, 5d)
.long R(fa, 87, 74, 73), R(fb, 0b, 41, 2e)
.long R(b3, 67, 1d, 5a), R(92, db, d2, 52)
.long R(e9, 10, 56, 33), R(6d, d6, 47, 13)
.long R(9a, d7, 61, 8c), R(37, a1, 0c, 7a)
.long R(59, f8, 14, 8e), R(eb, 13, 3c, 89)
.long R(ce, a9, 27, ee), R(b7, 61, c9, 35)
.long R(e1, 1c, e5, ed), R(7a, 47, b1, 3c)
.long R(9c, d2, df, 59), R(55, f2, 73, 3f)
.long R(18, 14, ce, 79), R(73, c7, 37, bf)
.long R(53, f7, cd, ea), R(5f, fd, aa, 5b)
.long R(df, 3d, 6f, 14), R(78, 44, db, 86)
.long R(ca, af, f3, 81), R(b9, 68, c4, 3e)
.long R(38, 24, 34, 2c), R(c2, a3, 40, 5f)
.long R(16, 1d, c3, 72), R(bc, e2, 25, 0c)
.long R(28, 3c, 49, 8b), R(ff, 0d, 95, 41)
.long R(39, a8, 01, 71), R(08, 0c, b3, de)
.long R(d8, b4, e4, 9c), R(64, 56, c1, 90)
.long R(7b, cb, 84, 61), R(d5, 32, b6, 70)
.long R(48, 6c, 5c, 74), R(d0, b8, 57, 42)
.globl PPC_AES_4K_DECTAB2
PPC_AES_4K_DECTAB2:
/* decryption table, same as crypto_il_tab in crypto/aes-generic.c */
.byte 0x52, 0x09, 0x6a, 0xd5, 0x30, 0x36, 0xa5, 0x38
.byte 0xbf, 0x40, 0xa3, 0x9e, 0x81, 0xf3, 0xd7, 0xfb
.byte 0x7c, 0xe3, 0x39, 0x82, 0x9b, 0x2f, 0xff, 0x87
.byte 0x34, 0x8e, 0x43, 0x44, 0xc4, 0xde, 0xe9, 0xcb
.byte 0x54, 0x7b, 0x94, 0x32, 0xa6, 0xc2, 0x23, 0x3d
.byte 0xee, 0x4c, 0x95, 0x0b, 0x42, 0xfa, 0xc3, 0x4e
.byte 0x08, 0x2e, 0xa1, 0x66, 0x28, 0xd9, 0x24, 0xb2
.byte 0x76, 0x5b, 0xa2, 0x49, 0x6d, 0x8b, 0xd1, 0x25
.byte 0x72, 0xf8, 0xf6, 0x64, 0x86, 0x68, 0x98, 0x16
.byte 0xd4, 0xa4, 0x5c, 0xcc, 0x5d, 0x65, 0xb6, 0x92
.byte 0x6c, 0x70, 0x48, 0x50, 0xfd, 0xed, 0xb9, 0xda
.byte 0x5e, 0x15, 0x46, 0x57, 0xa7, 0x8d, 0x9d, 0x84
.byte 0x90, 0xd8, 0xab, 0x00, 0x8c, 0xbc, 0xd3, 0x0a
.byte 0xf7, 0xe4, 0x58, 0x05, 0xb8, 0xb3, 0x45, 0x06
.byte 0xd0, 0x2c, 0x1e, 0x8f, 0xca, 0x3f, 0x0f, 0x02
.byte 0xc1, 0xaf, 0xbd, 0x03, 0x01, 0x13, 0x8a, 0x6b
.byte 0x3a, 0x91, 0x11, 0x41, 0x4f, 0x67, 0xdc, 0xea
.byte 0x97, 0xf2, 0xcf, 0xce, 0xf0, 0xb4, 0xe6, 0x73
.byte 0x96, 0xac, 0x74, 0x22, 0xe7, 0xad, 0x35, 0x85
.byte 0xe2, 0xf9, 0x37, 0xe8, 0x1c, 0x75, 0xdf, 0x6e
.byte 0x47, 0xf1, 0x1a, 0x71, 0x1d, 0x29, 0xc5, 0x89
.byte 0x6f, 0xb7, 0x62, 0x0e, 0xaa, 0x18, 0xbe, 0x1b
.byte 0xfc, 0x56, 0x3e, 0x4b, 0xc6, 0xd2, 0x79, 0x20
.byte 0x9a, 0xdb, 0xc0, 0xfe, 0x78, 0xcd, 0x5a, 0xf4
.byte 0x1f, 0xdd, 0xa8, 0x33, 0x88, 0x07, 0xc7, 0x31
.byte 0xb1, 0x12, 0x10, 0x59, 0x27, 0x80, 0xec, 0x5f
.byte 0x60, 0x51, 0x7f, 0xa9, 0x19, 0xb5, 0x4a, 0x0d
.byte 0x2d, 0xe5, 0x7a, 0x9f, 0x93, 0xc9, 0x9c, 0xef
.byte 0xa0, 0xe0, 0x3b, 0x4d, 0xae, 0x2a, 0xf5, 0xb0
.byte 0xc8, 0xeb, 0xbb, 0x3c, 0x83, 0x53, 0x99, 0x61
.byte 0x17, 0x2b, 0x04, 0x7e, 0xba, 0x77, 0xd6, 0x26
.byte 0xe1, 0x69, 0x14, 0x63, 0x55, 0x21, 0x0c, 0x7d
linux-7.2.3/lib/crypto/powerpc/aes.h
+240
View File
@@ -0,0 +1,240 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (c) 2015 Markus Stockhausen <stockhausen@collogia.de>
* Copyright (C) 2015 International Business Machines Inc.
* Copyright 2026 Google LLC
*/
#include <asm/simd.h>
#include <asm/switch_to.h>
#include <linux/cpufeature.h>
#include <linux/jump_label.h>
#include <linux/preempt.h>
#include <linux/uaccess.h>
#ifdef CONFIG_SPE
EXPORT_SYMBOL_GPL(ppc_expand_key_128);
EXPORT_SYMBOL_GPL(ppc_expand_key_192);
EXPORT_SYMBOL_GPL(ppc_expand_key_256);
EXPORT_SYMBOL_GPL(ppc_generate_decrypt_key);
EXPORT_SYMBOL_GPL(ppc_encrypt_ecb);
EXPORT_SYMBOL_GPL(ppc_decrypt_ecb);
EXPORT_SYMBOL_GPL(ppc_encrypt_cbc);
EXPORT_SYMBOL_GPL(ppc_decrypt_cbc);
EXPORT_SYMBOL_GPL(ppc_crypt_ctr);
EXPORT_SYMBOL_GPL(ppc_encrypt_xts);
EXPORT_SYMBOL_GPL(ppc_decrypt_xts);
void ppc_encrypt_aes(u8 *out, const u8 *in, const u32 *key_enc, u32 rounds);
void ppc_decrypt_aes(u8 *out, const u8 *in, const u32 *key_dec, u32 rounds);
static void spe_begin(void)
{
/* disable preemption and save users SPE registers if required */
preempt_disable();
enable_kernel_spe();
}
static void spe_end(void)
{
disable_kernel_spe();
/* reenable preemption */
preempt_enable();
}
static void aes_preparekey_arch(union aes_enckey_arch *k,
union aes_invkey_arch *inv_k,
const u8 *in_key, int key_len, int nrounds)
{
if (key_len == AES_KEYSIZE_128)
ppc_expand_key_128(k->spe_enc_key, in_key);
else if (key_len == AES_KEYSIZE_192)
ppc_expand_key_192(k->spe_enc_key, in_key);
else
ppc_expand_key_256(k->spe_enc_key, in_key);
if (inv_k)
ppc_generate_decrypt_key(inv_k->spe_dec_key, k->spe_enc_key,
key_len);
}
static void aes_encrypt_arch(const struct aes_enckey *key,
u8 out[AES_BLOCK_SIZE],
const u8 in[AES_BLOCK_SIZE])
{
spe_begin();
ppc_encrypt_aes(out, in, key->k.spe_enc_key, key->nrounds / 2 - 1);
spe_end();
}
static void aes_decrypt_arch(const struct aes_key *key,
u8 out[AES_BLOCK_SIZE],
const u8 in[AES_BLOCK_SIZE])
{
spe_begin();
ppc_decrypt_aes(out, in, key->inv_k.spe_dec_key, key->nrounds / 2 - 1);
spe_end();
}
#else /* CONFIG_SPE */
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_vec_crypto);
EXPORT_SYMBOL_GPL(aes_p8_set_encrypt_key);
EXPORT_SYMBOL_GPL(aes_p8_set_decrypt_key);
EXPORT_SYMBOL_GPL(aes_p8_encrypt);
EXPORT_SYMBOL_GPL(aes_p8_decrypt);
EXPORT_SYMBOL_GPL(aes_p8_cbc_encrypt);
EXPORT_SYMBOL_GPL(aes_p8_ctr32_encrypt_blocks);
EXPORT_SYMBOL_GPL(aes_p8_xts_encrypt);
EXPORT_SYMBOL_GPL(aes_p8_xts_decrypt);
static inline bool is_vsx_format(const struct p8_aes_key *key)
{
return key->nrounds != 0;
}
/*
* Convert a round key from VSX to generic format by reflecting all 16 bytes (if
* little endian) or reflecting the bytes in each 4-byte word (if big endian),
* and (if apply_inv_mix=true) applying InvMixColumn to each column.
*
* It would be nice if the VSX and generic key formats would be compatible. But
* that's very difficult to do, with the assembly code having been borrowed from
* OpenSSL and also targeted to POWER8 rather than POWER9.
*
* Fortunately, this conversion should only be needed in extremely rare cases,
* possibly not at all in practice. It's just included for full correctness.
*/
static void rndkey_from_vsx(u32 out[4], const u32 in[4], bool apply_inv_mix)
{
const bool be = IS_ENABLED(CONFIG_CPU_BIG_ENDIAN);
u32 k0 = swab32(in[0]);
u32 k1 = swab32(in[1]);
u32 k2 = swab32(in[2]);
u32 k3 = swab32(in[3]);
if (apply_inv_mix) {
k0 = inv_mix_columns(k0);
k1 = inv_mix_columns(k1);
k2 = inv_mix_columns(k2);
k3 = inv_mix_columns(k3);
}
out[0] = be ? k0 : k3;
out[1] = be ? k1 : k2;
out[2] = be ? k2 : k1;
out[3] = be ? k3 : k0;
}
static void aes_preparekey_arch(union aes_enckey_arch *k,
union aes_invkey_arch *inv_k,
const u8 *in_key, int key_len, int nrounds)
{
const int keybits = 8 * key_len;
int ret;
if (static_branch_likely(&have_vec_crypto) && likely(may_use_simd())) {
preempt_disable();
pagefault_disable();
enable_kernel_vsx();
ret = aes_p8_set_encrypt_key(in_key, keybits, &k->p8);
/*
* aes_p8_set_encrypt_key() should never fail here, since the
* key length was already validated.
*/
WARN_ON_ONCE(ret);
if (inv_k) {
ret = aes_p8_set_decrypt_key(in_key, keybits,
&inv_k->p8);
/* ... and likewise for aes_p8_set_decrypt_key(). */
WARN_ON_ONCE(ret);
}
disable_kernel_vsx();
pagefault_enable();
preempt_enable();
} else {
aes_expandkey_generic(k->rndkeys,
inv_k ? inv_k->inv_rndkeys : NULL,
in_key, key_len);
/* Mark the key as using the generic format. */
k->p8.nrounds = 0;
if (inv_k)
inv_k->p8.nrounds = 0;
}
}
static void aes_encrypt_arch(const struct aes_enckey *key,
u8 out[AES_BLOCK_SIZE],
const u8 in[AES_BLOCK_SIZE])
{
if (static_branch_likely(&have_vec_crypto) &&
likely(is_vsx_format(&key->k.p8) && may_use_simd())) {
preempt_disable();
pagefault_disable();
enable_kernel_vsx();
aes_p8_encrypt(in, out, &key->k.p8);
disable_kernel_vsx();
pagefault_enable();
preempt_enable();
} else if (unlikely(is_vsx_format(&key->k.p8))) {
/*
* This handles (the hopefully extremely rare) case where a key
* was prepared using the VSX optimized format, then encryption
* is done in a context that cannot use VSX instructions.
*/
u32 rndkeys[AES_MAX_KEYLENGTH_U32];
for (int i = 0; i < 4 * (key->nrounds + 1); i += 4)
rndkey_from_vsx(&rndkeys[i],
&key->k.p8.rndkeys[i], false);
aes_encrypt_generic(rndkeys, key->nrounds, out, in);
} else {
aes_encrypt_generic(key->k.rndkeys, key->nrounds, out, in);
}
}
static void aes_decrypt_arch(const struct aes_key *key, u8 out[AES_BLOCK_SIZE],
const u8 in[AES_BLOCK_SIZE])
{
if (static_branch_likely(&have_vec_crypto) &&
likely(is_vsx_format(&key->inv_k.p8) && may_use_simd())) {
preempt_disable();
pagefault_disable();
enable_kernel_vsx();
aes_p8_decrypt(in, out, &key->inv_k.p8);
disable_kernel_vsx();
pagefault_enable();
preempt_enable();
} else if (unlikely(is_vsx_format(&key->inv_k.p8))) {
/*
* This handles (the hopefully extremely rare) case where a key
* was prepared using the VSX optimized format, then decryption
* is done in a context that cannot use VSX instructions.
*/
u32 inv_rndkeys[AES_MAX_KEYLENGTH_U32];
int i;
rndkey_from_vsx(&inv_rndkeys[0],
&key->inv_k.p8.rndkeys[0], false);
for (i = 4; i < 4 * key->nrounds; i += 4) {
rndkey_from_vsx(&inv_rndkeys[i],
&key->inv_k.p8.rndkeys[i], true);
}
rndkey_from_vsx(&inv_rndkeys[i],
&key->inv_k.p8.rndkeys[i], false);
aes_decrypt_generic(inv_rndkeys, key->nrounds, out, in);
} else {
aes_decrypt_generic(key->inv_k.inv_rndkeys, key->nrounds,
out, in);
}
}
#define aes_mod_init_arch aes_mod_init_arch
static void aes_mod_init_arch(void)
{
if (cpu_has_feature(CPU_FTR_ARCH_207S) &&
(cur_cpu_spec->cpu_user_features2 & PPC_FEATURE2_VEC_CRYPTO))
static_branch_enable(&have_vec_crypto);
}
#endif /* !CONFIG_SPE */
linux-7.2.3/lib/crypto/powerpc/aesp8-ppc.pl
+3890
View File
File diff suppressed because it is too large Load Diff
linux-7.2.3/lib/crypto/powerpc/chacha-p10le-8x.S
+840
View File
@@ -0,0 +1,840 @@
/* SPDX-License-Identifier: GPL-2.0-or-later */
#
# Accelerated chacha20 implementation for ppc64le.
#
# Copyright 2023- IBM Corp. All rights reserved
#
#===================================================================================
# Written by Danny Tsen <dtsen@us.ibm.com>
#
# do rounds, 8 quarter rounds
# 1. a += b; d ^= a; d <<<= 16;
# 2. c += d; b ^= c; b <<<= 12;
# 3. a += b; d ^= a; d <<<= 8;
# 4. c += d; b ^= c; b <<<= 7
#
# row1 = (row1 + row2), row4 = row1 xor row4, row4 rotate each word by 16
# row3 = (row3 + row4), row2 = row3 xor row2, row2 rotate each word by 12
# row1 = (row1 + row2), row4 = row1 xor row4, row4 rotate each word by 8
# row3 = (row3 + row4), row2 = row3 xor row2, row2 rotate each word by 7
#
# 4 blocks (a b c d)
#
# a0 b0 c0 d0
# a1 b1 c1 d1
# ...
# a4 b4 c4 d4
# ...
# a8 b8 c8 d8
# ...
# a12 b12 c12 d12
# a13 ...
# a14 ...
# a15 b15 c15 d15
#
# Column round (v0, v4, v8, v12, v1, v5, v9, v13, v2, v6, v10, v14, v3, v7, v11, v15)
# Diagnal round (v0, v5, v10, v15, v1, v6, v11, v12, v2, v7, v8, v13, v3, v4, v9, v14)
#
#include <asm/ppc_asm.h>
#include <asm/asm-offsets.h>
#include <asm/asm-compat.h>
#include <linux/linkage.h>
.machine "any"
.text
.macro SAVE_GPR GPR OFFSET FRAME
std \GPR,\OFFSET(\FRAME)
.endm
.macro SAVE_VRS VRS OFFSET FRAME
li 16, \OFFSET
stvx \VRS, 16, \FRAME
.endm
.macro SAVE_VSX VSX OFFSET FRAME
li 16, \OFFSET
stxvx \VSX, 16, \FRAME
.endm
.macro RESTORE_GPR GPR OFFSET FRAME
ld \GPR,\OFFSET(\FRAME)
.endm
.macro RESTORE_VRS VRS OFFSET FRAME
li 16, \OFFSET
lvx \VRS, 16, \FRAME
.endm
.macro RESTORE_VSX VSX OFFSET FRAME
li 16, \OFFSET
lxvx \VSX, 16, \FRAME
.endm
.macro SAVE_REGS
mflr 0
std 0, 16(1)
stdu 1,-752(1)
SAVE_GPR 14, 112, 1
SAVE_GPR 15, 120, 1
SAVE_GPR 16, 128, 1
SAVE_GPR 17, 136, 1
SAVE_GPR 18, 144, 1
SAVE_GPR 19, 152, 1
SAVE_GPR 20, 160, 1
SAVE_GPR 21, 168, 1
SAVE_GPR 22, 176, 1
SAVE_GPR 23, 184, 1
SAVE_GPR 24, 192, 1
SAVE_GPR 25, 200, 1
SAVE_GPR 26, 208, 1
SAVE_GPR 27, 216, 1
SAVE_GPR 28, 224, 1
SAVE_GPR 29, 232, 1
SAVE_GPR 30, 240, 1
SAVE_GPR 31, 248, 1
addi 9, 1, 256
SAVE_VRS 20, 0, 9
SAVE_VRS 21, 16, 9
SAVE_VRS 22, 32, 9
SAVE_VRS 23, 48, 9
SAVE_VRS 24, 64, 9
SAVE_VRS 25, 80, 9
SAVE_VRS 26, 96, 9
SAVE_VRS 27, 112, 9
SAVE_VRS 28, 128, 9
SAVE_VRS 29, 144, 9
SAVE_VRS 30, 160, 9
SAVE_VRS 31, 176, 9
SAVE_VSX 14, 192, 9
SAVE_VSX 15, 208, 9
SAVE_VSX 16, 224, 9
SAVE_VSX 17, 240, 9
SAVE_VSX 18, 256, 9
SAVE_VSX 19, 272, 9
SAVE_VSX 20, 288, 9
SAVE_VSX 21, 304, 9
SAVE_VSX 22, 320, 9
SAVE_VSX 23, 336, 9
SAVE_VSX 24, 352, 9
SAVE_VSX 25, 368, 9
SAVE_VSX 26, 384, 9
SAVE_VSX 27, 400, 9
SAVE_VSX 28, 416, 9
SAVE_VSX 29, 432, 9
SAVE_VSX 30, 448, 9
SAVE_VSX 31, 464, 9
.endm # SAVE_REGS
.macro RESTORE_REGS
addi 9, 1, 256
RESTORE_VRS 20, 0, 9
RESTORE_VRS 21, 16, 9
RESTORE_VRS 22, 32, 9
RESTORE_VRS 23, 48, 9
RESTORE_VRS 24, 64, 9
RESTORE_VRS 25, 80, 9
RESTORE_VRS 26, 96, 9
RESTORE_VRS 27, 112, 9
RESTORE_VRS 28, 128, 9
RESTORE_VRS 29, 144, 9
RESTORE_VRS 30, 160, 9
RESTORE_VRS 31, 176, 9
RESTORE_VSX 14, 192, 9
RESTORE_VSX 15, 208, 9
RESTORE_VSX 16, 224, 9
RESTORE_VSX 17, 240, 9
RESTORE_VSX 18, 256, 9
RESTORE_VSX 19, 272, 9
RESTORE_VSX 20, 288, 9
RESTORE_VSX 21, 304, 9
RESTORE_VSX 22, 320, 9
RESTORE_VSX 23, 336, 9
RESTORE_VSX 24, 352, 9
RESTORE_VSX 25, 368, 9
RESTORE_VSX 26, 384, 9
RESTORE_VSX 27, 400, 9
RESTORE_VSX 28, 416, 9
RESTORE_VSX 29, 432, 9
RESTORE_VSX 30, 448, 9
RESTORE_VSX 31, 464, 9
RESTORE_GPR 14, 112, 1
RESTORE_GPR 15, 120, 1
RESTORE_GPR 16, 128, 1
RESTORE_GPR 17, 136, 1
RESTORE_GPR 18, 144, 1
RESTORE_GPR 19, 152, 1
RESTORE_GPR 20, 160, 1
RESTORE_GPR 21, 168, 1
RESTORE_GPR 22, 176, 1
RESTORE_GPR 23, 184, 1
RESTORE_GPR 24, 192, 1
RESTORE_GPR 25, 200, 1
RESTORE_GPR 26, 208, 1
RESTORE_GPR 27, 216, 1
RESTORE_GPR 28, 224, 1
RESTORE_GPR 29, 232, 1
RESTORE_GPR 30, 240, 1
RESTORE_GPR 31, 248, 1
addi 1, 1, 752
ld 0, 16(1)
mtlr 0
.endm # RESTORE_REGS
.macro QT_loop_8x
# QR(v0, v4, v8, v12, v1, v5, v9, v13, v2, v6, v10, v14, v3, v7, v11, v15)
xxlor 0, 32+25, 32+25
xxlor 32+25, 20, 20
vadduwm 0, 0, 4
vadduwm 1, 1, 5
vadduwm 2, 2, 6
vadduwm 3, 3, 7
vadduwm 16, 16, 20
vadduwm 17, 17, 21
vadduwm 18, 18, 22
vadduwm 19, 19, 23
vpermxor 12, 12, 0, 25
vpermxor 13, 13, 1, 25
vpermxor 14, 14, 2, 25
vpermxor 15, 15, 3, 25
vpermxor 28, 28, 16, 25
vpermxor 29, 29, 17, 25
vpermxor 30, 30, 18, 25
vpermxor 31, 31, 19, 25
xxlor 32+25, 0, 0
vadduwm 8, 8, 12
vadduwm 9, 9, 13
vadduwm 10, 10, 14
vadduwm 11, 11, 15
vadduwm 24, 24, 28
vadduwm 25, 25, 29
vadduwm 26, 26, 30
vadduwm 27, 27, 31
vxor 4, 4, 8
vxor 5, 5, 9
vxor 6, 6, 10
vxor 7, 7, 11
vxor 20, 20, 24
vxor 21, 21, 25
vxor 22, 22, 26
vxor 23, 23, 27
xxlor 0, 32+25, 32+25
xxlor 32+25, 21, 21
vrlw 4, 4, 25 #
vrlw 5, 5, 25
vrlw 6, 6, 25
vrlw 7, 7, 25
vrlw 20, 20, 25 #
vrlw 21, 21, 25
vrlw 22, 22, 25
vrlw 23, 23, 25
xxlor 32+25, 0, 0
vadduwm 0, 0, 4
vadduwm 1, 1, 5
vadduwm 2, 2, 6
vadduwm 3, 3, 7
vadduwm 16, 16, 20
vadduwm 17, 17, 21
vadduwm 18, 18, 22
vadduwm 19, 19, 23
xxlor 0, 32+25, 32+25
xxlor 32+25, 22, 22
vpermxor 12, 12, 0, 25
vpermxor 13, 13, 1, 25
vpermxor 14, 14, 2, 25
vpermxor 15, 15, 3, 25
vpermxor 28, 28, 16, 25
vpermxor 29, 29, 17, 25
vpermxor 30, 30, 18, 25
vpermxor 31, 31, 19, 25
xxlor 32+25, 0, 0
vadduwm 8, 8, 12
vadduwm 9, 9, 13
vadduwm 10, 10, 14
vadduwm 11, 11, 15
vadduwm 24, 24, 28
vadduwm 25, 25, 29
vadduwm 26, 26, 30
vadduwm 27, 27, 31
xxlor 0, 32+28, 32+28
xxlor 32+28, 23, 23
vxor 4, 4, 8
vxor 5, 5, 9
vxor 6, 6, 10
vxor 7, 7, 11
vxor 20, 20, 24
vxor 21, 21, 25
vxor 22, 22, 26
vxor 23, 23, 27
vrlw 4, 4, 28 #
vrlw 5, 5, 28
vrlw 6, 6, 28
vrlw 7, 7, 28
vrlw 20, 20, 28 #
vrlw 21, 21, 28
vrlw 22, 22, 28
vrlw 23, 23, 28
xxlor 32+28, 0, 0
# QR(v0, v5, v10, v15, v1, v6, v11, v12, v2, v7, v8, v13, v3, v4, v9, v14)
xxlor 0, 32+25, 32+25
xxlor 32+25, 20, 20
vadduwm 0, 0, 5
vadduwm 1, 1, 6
vadduwm 2, 2, 7
vadduwm 3, 3, 4
vadduwm 16, 16, 21
vadduwm 17, 17, 22
vadduwm 18, 18, 23
vadduwm 19, 19, 20
vpermxor 15, 15, 0, 25
vpermxor 12, 12, 1, 25
vpermxor 13, 13, 2, 25
vpermxor 14, 14, 3, 25
vpermxor 31, 31, 16, 25
vpermxor 28, 28, 17, 25
vpermxor 29, 29, 18, 25
vpermxor 30, 30, 19, 25
xxlor 32+25, 0, 0
vadduwm 10, 10, 15
vadduwm 11, 11, 12
vadduwm 8, 8, 13
vadduwm 9, 9, 14
vadduwm 26, 26, 31
vadduwm 27, 27, 28
vadduwm 24, 24, 29
vadduwm 25, 25, 30
vxor 5, 5, 10
vxor 6, 6, 11
vxor 7, 7, 8
vxor 4, 4, 9
vxor 21, 21, 26
vxor 22, 22, 27
vxor 23, 23, 24
vxor 20, 20, 25
xxlor 0, 32+25, 32+25
xxlor 32+25, 21, 21
vrlw 5, 5, 25
vrlw 6, 6, 25
vrlw 7, 7, 25
vrlw 4, 4, 25
vrlw 21, 21, 25
vrlw 22, 22, 25
vrlw 23, 23, 25
vrlw 20, 20, 25
xxlor 32+25, 0, 0
vadduwm 0, 0, 5
vadduwm 1, 1, 6
vadduwm 2, 2, 7
vadduwm 3, 3, 4
vadduwm 16, 16, 21
vadduwm 17, 17, 22
vadduwm 18, 18, 23
vadduwm 19, 19, 20
xxlor 0, 32+25, 32+25
xxlor 32+25, 22, 22
vpermxor 15, 15, 0, 25
vpermxor 12, 12, 1, 25
vpermxor 13, 13, 2, 25
vpermxor 14, 14, 3, 25
vpermxor 31, 31, 16, 25
vpermxor 28, 28, 17, 25
vpermxor 29, 29, 18, 25
vpermxor 30, 30, 19, 25
xxlor 32+25, 0, 0
vadduwm 10, 10, 15
vadduwm 11, 11, 12
vadduwm 8, 8, 13
vadduwm 9, 9, 14
vadduwm 26, 26, 31
vadduwm 27, 27, 28
vadduwm 24, 24, 29
vadduwm 25, 25, 30
xxlor 0, 32+28, 32+28
xxlor 32+28, 23, 23
vxor 5, 5, 10
vxor 6, 6, 11
vxor 7, 7, 8
vxor 4, 4, 9
vxor 21, 21, 26
vxor 22, 22, 27
vxor 23, 23, 24
vxor 20, 20, 25
vrlw 5, 5, 28
vrlw 6, 6, 28
vrlw 7, 7, 28
vrlw 4, 4, 28
vrlw 21, 21, 28
vrlw 22, 22, 28
vrlw 23, 23, 28
vrlw 20, 20, 28
xxlor 32+28, 0, 0
.endm
.macro QT_loop_4x
# QR(v0, v4, v8, v12, v1, v5, v9, v13, v2, v6, v10, v14, v3, v7, v11, v15)
vadduwm 0, 0, 4
vadduwm 1, 1, 5
vadduwm 2, 2, 6
vadduwm 3, 3, 7
vpermxor 12, 12, 0, 20
vpermxor 13, 13, 1, 20
vpermxor 14, 14, 2, 20
vpermxor 15, 15, 3, 20
vadduwm 8, 8, 12
vadduwm 9, 9, 13
vadduwm 10, 10, 14
vadduwm 11, 11, 15
vxor 4, 4, 8
vxor 5, 5, 9
vxor 6, 6, 10
vxor 7, 7, 11
vrlw 4, 4, 21
vrlw 5, 5, 21
vrlw 6, 6, 21
vrlw 7, 7, 21
vadduwm 0, 0, 4
vadduwm 1, 1, 5
vadduwm 2, 2, 6
vadduwm 3, 3, 7
vpermxor 12, 12, 0, 22
vpermxor 13, 13, 1, 22
vpermxor 14, 14, 2, 22
vpermxor 15, 15, 3, 22
vadduwm 8, 8, 12
vadduwm 9, 9, 13
vadduwm 10, 10, 14
vadduwm 11, 11, 15
vxor 4, 4, 8
vxor 5, 5, 9
vxor 6, 6, 10
vxor 7, 7, 11
vrlw 4, 4, 23
vrlw 5, 5, 23
vrlw 6, 6, 23
vrlw 7, 7, 23
# QR(v0, v5, v10, v15, v1, v6, v11, v12, v2, v7, v8, v13, v3, v4, v9, v14)
vadduwm 0, 0, 5
vadduwm 1, 1, 6
vadduwm 2, 2, 7
vadduwm 3, 3, 4
vpermxor 15, 15, 0, 20
vpermxor 12, 12, 1, 20
vpermxor 13, 13, 2, 20
vpermxor 14, 14, 3, 20
vadduwm 10, 10, 15
vadduwm 11, 11, 12
vadduwm 8, 8, 13
vadduwm 9, 9, 14
vxor 5, 5, 10
vxor 6, 6, 11
vxor 7, 7, 8
vxor 4, 4, 9
vrlw 5, 5, 21
vrlw 6, 6, 21
vrlw 7, 7, 21
vrlw 4, 4, 21
vadduwm 0, 0, 5
vadduwm 1, 1, 6
vadduwm 2, 2, 7
vadduwm 3, 3, 4
vpermxor 15, 15, 0, 22
vpermxor 12, 12, 1, 22
vpermxor 13, 13, 2, 22
vpermxor 14, 14, 3, 22
vadduwm 10, 10, 15
vadduwm 11, 11, 12
vadduwm 8, 8, 13
vadduwm 9, 9, 14
vxor 5, 5, 10
vxor 6, 6, 11
vxor 7, 7, 8
vxor 4, 4, 9
vrlw 5, 5, 23
vrlw 6, 6, 23
vrlw 7, 7, 23
vrlw 4, 4, 23
.endm
# Transpose
.macro TP_4x a0 a1 a2 a3
xxmrghw 10, 32+\a0, 32+\a1 # a0, a1, b0, b1
xxmrghw 11, 32+\a2, 32+\a3 # a2, a3, b2, b3
xxmrglw 12, 32+\a0, 32+\a1 # c0, c1, d0, d1
xxmrglw 13, 32+\a2, 32+\a3 # c2, c3, d2, d3
xxpermdi 32+\a0, 10, 11, 0 # a0, a1, a2, a3
xxpermdi 32+\a1, 10, 11, 3 # b0, b1, b2, b3
xxpermdi 32+\a2, 12, 13, 0 # c0, c1, c2, c3
xxpermdi 32+\a3, 12, 13, 3 # d0, d1, d2, d3
.endm
# key stream = working state + state
.macro Add_state S
vadduwm \S+0, \S+0, 16-\S
vadduwm \S+4, \S+4, 17-\S
vadduwm \S+8, \S+8, 18-\S
vadduwm \S+12, \S+12, 19-\S
vadduwm \S+1, \S+1, 16-\S
vadduwm \S+5, \S+5, 17-\S
vadduwm \S+9, \S+9, 18-\S
vadduwm \S+13, \S+13, 19-\S
vadduwm \S+2, \S+2, 16-\S
vadduwm \S+6, \S+6, 17-\S
vadduwm \S+10, \S+10, 18-\S
vadduwm \S+14, \S+14, 19-\S
vadduwm \S+3, \S+3, 16-\S
vadduwm \S+7, \S+7, 17-\S
vadduwm \S+11, \S+11, 18-\S
vadduwm \S+15, \S+15, 19-\S
.endm
#
# write 256 bytes
#
.macro Write_256 S
add 9, 14, 5
add 16, 14, 4
lxvw4x 0, 0, 9
lxvw4x 1, 17, 9
lxvw4x 2, 18, 9
lxvw4x 3, 19, 9
lxvw4x 4, 20, 9
lxvw4x 5, 21, 9
lxvw4x 6, 22, 9
lxvw4x 7, 23, 9
lxvw4x 8, 24, 9
lxvw4x 9, 25, 9
lxvw4x 10, 26, 9
lxvw4x 11, 27, 9
lxvw4x 12, 28, 9
lxvw4x 13, 29, 9
lxvw4x 14, 30, 9
lxvw4x 15, 31, 9
xxlxor \S+32, \S+32, 0
xxlxor \S+36, \S+36, 1
xxlxor \S+40, \S+40, 2
xxlxor \S+44, \S+44, 3
xxlxor \S+33, \S+33, 4
xxlxor \S+37, \S+37, 5
xxlxor \S+41, \S+41, 6
xxlxor \S+45, \S+45, 7
xxlxor \S+34, \S+34, 8
xxlxor \S+38, \S+38, 9
xxlxor \S+42, \S+42, 10
xxlxor \S+46, \S+46, 11
xxlxor \S+35, \S+35, 12
xxlxor \S+39, \S+39, 13
xxlxor \S+43, \S+43, 14
xxlxor \S+47, \S+47, 15
stxvw4x \S+32, 0, 16
stxvw4x \S+36, 17, 16
stxvw4x \S+40, 18, 16
stxvw4x \S+44, 19, 16
stxvw4x \S+33, 20, 16
stxvw4x \S+37, 21, 16
stxvw4x \S+41, 22, 16
stxvw4x \S+45, 23, 16
stxvw4x \S+34, 24, 16
stxvw4x \S+38, 25, 16
stxvw4x \S+42, 26, 16
stxvw4x \S+46, 27, 16
stxvw4x \S+35, 28, 16
stxvw4x \S+39, 29, 16
stxvw4x \S+43, 30, 16
stxvw4x \S+47, 31, 16
.endm
#
# void chacha_p10le_8x(const struct chacha_state *state, u8 *dst, const u8 *src,
# unsigned int len, int nrounds);
#
SYM_FUNC_START(chacha_p10le_8x)
.align 5
cmpdi 6, 0
ble Out_no_chacha
SAVE_REGS
# r17 - r31 mainly for Write_256 macro.
li 17, 16
li 18, 32
li 19, 48
li 20, 64
li 21, 80
li 22, 96
li 23, 112
li 24, 128
li 25, 144
li 26, 160
li 27, 176
li 28, 192
li 29, 208
li 30, 224
li 31, 240
mr 15, 6 # len
li 14, 0 # offset to inp and outp
lxvw4x 48, 0, 3 # vr16, constants
lxvw4x 49, 17, 3 # vr17, key 1
lxvw4x 50, 18, 3 # vr18, key 2
lxvw4x 51, 19, 3 # vr19, counter, nonce
# create (0, 1, 2, 3) counters
vspltisw 0, 0
vspltisw 1, 1
vspltisw 2, 2
vspltisw 3, 3
vmrghw 4, 0, 1
vmrglw 5, 2, 3
vsldoi 30, 4, 5, 8 # vr30 counter, 4 (0, 1, 2, 3)
vspltisw 21, 12
vspltisw 23, 7
addis 11, 2, permx@toc@ha
addi 11, 11, permx@toc@l
lxvw4x 32+20, 0, 11
lxvw4x 32+22, 17, 11
sradi 8, 7, 1
mtctr 8
# save constants to vsx
xxlor 16, 48, 48
xxlor 17, 49, 49
xxlor 18, 50, 50
xxlor 19, 51, 51
vspltisw 25, 4
vspltisw 26, 8
xxlor 25, 32+26, 32+26
xxlor 24, 32+25, 32+25
vadduwm 31, 30, 25 # counter = (0, 1, 2, 3) + (4, 4, 4, 4)
xxlor 30, 32+30, 32+30
xxlor 31, 32+31, 32+31
xxlor 20, 32+20, 32+20
xxlor 21, 32+21, 32+21
xxlor 22, 32+22, 32+22
xxlor 23, 32+23, 32+23
cmpdi 6, 512
blt Loop_last
Loop_8x:
xxspltw 32+0, 16, 0
xxspltw 32+1, 16, 1
xxspltw 32+2, 16, 2
xxspltw 32+3, 16, 3
xxspltw 32+4, 17, 0
xxspltw 32+5, 17, 1
xxspltw 32+6, 17, 2
xxspltw 32+7, 17, 3
xxspltw 32+8, 18, 0
xxspltw 32+9, 18, 1
xxspltw 32+10, 18, 2
xxspltw 32+11, 18, 3
xxspltw 32+12, 19, 0
xxspltw 32+13, 19, 1
xxspltw 32+14, 19, 2
xxspltw 32+15, 19, 3
vadduwm 12, 12, 30 # increase counter
xxspltw 32+16, 16, 0
xxspltw 32+17, 16, 1
xxspltw 32+18, 16, 2
xxspltw 32+19, 16, 3
xxspltw 32+20, 17, 0
xxspltw 32+21, 17, 1
xxspltw 32+22, 17, 2
xxspltw 32+23, 17, 3
xxspltw 32+24, 18, 0
xxspltw 32+25, 18, 1
xxspltw 32+26, 18, 2
xxspltw 32+27, 18, 3
xxspltw 32+28, 19, 0
xxspltw 32+29, 19, 1
vadduwm 28, 28, 31 # increase counter
xxspltw 32+30, 19, 2
xxspltw 32+31, 19, 3
.align 5
quarter_loop_8x:
QT_loop_8x
bdnz quarter_loop_8x
xxlor 0, 32+30, 32+30
xxlor 32+30, 30, 30
vadduwm 12, 12, 30
xxlor 32+30, 0, 0
TP_4x 0, 1, 2, 3
TP_4x 4, 5, 6, 7
TP_4x 8, 9, 10, 11
TP_4x 12, 13, 14, 15
xxlor 0, 48, 48
xxlor 1, 49, 49
xxlor 2, 50, 50
xxlor 3, 51, 51
xxlor 48, 16, 16
xxlor 49, 17, 17
xxlor 50, 18, 18
xxlor 51, 19, 19
Add_state 0
xxlor 48, 0, 0
xxlor 49, 1, 1
xxlor 50, 2, 2
xxlor 51, 3, 3
Write_256 0
addi 14, 14, 256 # offset +=256
addi 15, 15, -256 # len -=256
xxlor 5, 32+31, 32+31
xxlor 32+31, 31, 31
vadduwm 28, 28, 31
xxlor 32+31, 5, 5
TP_4x 16+0, 16+1, 16+2, 16+3
TP_4x 16+4, 16+5, 16+6, 16+7
TP_4x 16+8, 16+9, 16+10, 16+11
TP_4x 16+12, 16+13, 16+14, 16+15
xxlor 32, 16, 16
xxlor 33, 17, 17
xxlor 34, 18, 18
xxlor 35, 19, 19
Add_state 16
Write_256 16
addi 14, 14, 256 # offset +=256
addi 15, 15, -256 # len +=256
xxlor 32+24, 24, 24
xxlor 32+25, 25, 25
xxlor 32+30, 30, 30
vadduwm 30, 30, 25
vadduwm 31, 30, 24
xxlor 30, 32+30, 32+30
xxlor 31, 32+31, 32+31
cmpdi 15, 0
beq Out_loop
cmpdi 15, 512
blt Loop_last
mtctr 8
b Loop_8x
Loop_last:
lxvw4x 48, 0, 3 # vr16, constants
lxvw4x 49, 17, 3 # vr17, key 1
lxvw4x 50, 18, 3 # vr18, key 2
lxvw4x 51, 19, 3 # vr19, counter, nonce
vspltisw 21, 12
vspltisw 23, 7
addis 11, 2, permx@toc@ha
addi 11, 11, permx@toc@l
lxvw4x 32+20, 0, 11
lxvw4x 32+22, 17, 11
sradi 8, 7, 1
mtctr 8
Loop_4x:
vspltw 0, 16, 0
vspltw 1, 16, 1
vspltw 2, 16, 2
vspltw 3, 16, 3
vspltw 4, 17, 0
vspltw 5, 17, 1
vspltw 6, 17, 2
vspltw 7, 17, 3
vspltw 8, 18, 0
vspltw 9, 18, 1
vspltw 10, 18, 2
vspltw 11, 18, 3
vspltw 12, 19, 0
vadduwm 12, 12, 30 # increase counter
vspltw 13, 19, 1
vspltw 14, 19, 2
vspltw 15, 19, 3
.align 5
quarter_loop:
QT_loop_4x
bdnz quarter_loop
vadduwm 12, 12, 30
TP_4x 0, 1, 2, 3
TP_4x 4, 5, 6, 7
TP_4x 8, 9, 10, 11
TP_4x 12, 13, 14, 15
Add_state 0
Write_256 0
addi 14, 14, 256 # offset += 256
addi 15, 15, -256 # len += 256
# Update state counter
vspltisw 25, 4
vadduwm 30, 30, 25
cmpdi 15, 0
beq Out_loop
cmpdi 15, 256
blt Out_loop
mtctr 8
b Loop_4x
Out_loop:
RESTORE_REGS
blr
Out_no_chacha:
li 3, 0
blr
SYM_FUNC_END(chacha_p10le_8x)
SYM_DATA_START_LOCAL(PERMX)
.align 5
permx:
.long 0x22330011, 0x66774455, 0xaabb8899, 0xeeffccdd
.long 0x11223300, 0x55667744, 0x99aabb88, 0xddeeffcc
SYM_DATA_END(PERMX)
linux-7.2.3/lib/crypto/powerpc/chacha.h
+76
View File
@@ -0,0 +1,76 @@
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
* ChaCha stream cipher (P10 accelerated)
*
* Copyright 2023- IBM Corp. All rights reserved.
*/
#include <crypto/internal/simd.h>
#include <linux/kernel.h>
#include <linux/cpufeature.h>
#include <linux/sizes.h>
#include <asm/simd.h>
#include <asm/switch_to.h>
asmlinkage void chacha_p10le_8x(const struct chacha_state *state, u8 *dst,
const u8 *src, unsigned int len, int nrounds);
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_p10);
static void vsx_begin(void)
{
preempt_disable();
enable_kernel_vsx();
}
static void vsx_end(void)
{
disable_kernel_vsx();
preempt_enable();
}
static void chacha_p10_do_8x(struct chacha_state *state, u8 *dst, const u8 *src,
unsigned int bytes, int nrounds)
{
unsigned int l = bytes & ~0x0FF;
if (l > 0) {
chacha_p10le_8x(state, dst, src, l, nrounds);
bytes -= l;
src += l;
dst += l;
state->x[12] += l / CHACHA_BLOCK_SIZE;
}
if (bytes > 0)
chacha_crypt_generic(state, dst, src, bytes, nrounds);
}
#define hchacha_block_arch hchacha_block_generic /* not implemented yet */
static void chacha_crypt_arch(struct chacha_state *state, u8 *dst,
const u8 *src, unsigned int bytes, int nrounds)
{
if (!static_branch_likely(&have_p10) || bytes <= CHACHA_BLOCK_SIZE ||
!crypto_simd_usable())
return chacha_crypt_generic(state, dst, src, bytes, nrounds);
do {
unsigned int todo = min_t(unsigned int, bytes, SZ_4K);
vsx_begin();
chacha_p10_do_8x(state, dst, src, todo, nrounds);
vsx_end();
bytes -= todo;
src += todo;
dst += todo;
} while (bytes);
}
#define chacha_mod_init_arch chacha_mod_init_arch
static void chacha_mod_init_arch(void)
{
if (cpu_has_feature(CPU_FTR_ARCH_31))
static_branch_enable(&have_p10);
}
linux-7.2.3/lib/crypto/powerpc/curve25519-ppc64le_asm.S
+671
View File
@@ -0,0 +1,671 @@
/* SPDX-License-Identifier: GPL-2.0-or-later */
#
# This code is taken from CRYPTOGAMs[1] and is included here using the option
# in the license to distribute the code under the GPL. Therefore this program
# is free software; you can redistribute it and/or modify it under the terms of
# the GNU General Public License version 2 as published by the Free Software
# Foundation.
#
# [1] https://github.com/dot-asm/cryptogams/
# Copyright (c) 2006-2017, CRYPTOGAMS by <appro@openssl.org>
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# * Redistributions of source code must retain copyright notices,
# this list of conditions and the following disclaimer.
#
# * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following
# disclaimer in the documentation and/or other materials
# provided with the distribution.
#
# * Neither the name of the CRYPTOGAMS nor the names of its
# copyright holder and contributors may be used to endorse or
# promote products derived from this software without specific
# prior written permission.
#
# ALTERNATIVELY, provided that this notice is retained in full, this
# product may be distributed under the terms of the GNU General Public
# License (GPL), in which case the provisions of the GPL apply INSTEAD OF
# those given above.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
# ====================================================================
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see https://www.openssl.org/~appro/cryptogams/.
# ====================================================================
#
# ====================================================================
# Written and Modified by Danny Tsen <dtsen@us.ibm.com>
# - Added x25519_fe51_sqr_times, x25519_fe51_frombytes, x25519_fe51_tobytes
# and x25519_cswap
#
# Copyright 2024- IBM Corp.
#
# X25519 lower-level primitives for PPC64.
#
#include <linux/linkage.h>
.text
.align 5
SYM_FUNC_START(x25519_fe51_mul)
stdu 1,-144(1)
std 21,56(1)
std 22,64(1)
std 23,72(1)
std 24,80(1)
std 25,88(1)
std 26,96(1)
std 27,104(1)
std 28,112(1)
std 29,120(1)
std 30,128(1)
std 31,136(1)
ld 6,0(5)
ld 7,0(4)
ld 8,8(4)
ld 9,16(4)
ld 10,24(4)
ld 11,32(4)
mulld 22,7,6
mulhdu 23,7,6
mulld 24,8,6
mulhdu 25,8,6
mulld 30,11,6
mulhdu 31,11,6
ld 4,8(5)
mulli 11,11,19
mulld 26,9,6
mulhdu 27,9,6
mulld 28,10,6
mulhdu 29,10,6
mulld 12,11,4
mulhdu 21,11,4
addc 22,22,12
adde 23,23,21
mulld 12,7,4
mulhdu 21,7,4
addc 24,24,12
adde 25,25,21
mulld 12,10,4
mulhdu 21,10,4
ld 6,16(5)
mulli 10,10,19
addc 30,30,12
adde 31,31,21
mulld 12,8,4
mulhdu 21,8,4
addc 26,26,12
adde 27,27,21
mulld 12,9,4
mulhdu 21,9,4
addc 28,28,12
adde 29,29,21
mulld 12,10,6
mulhdu 21,10,6
addc 22,22,12
adde 23,23,21
mulld 12,11,6
mulhdu 21,11,6
addc 24,24,12
adde 25,25,21
mulld 12,9,6
mulhdu 21,9,6
ld 4,24(5)
mulli 9,9,19
addc 30,30,12
adde 31,31,21
mulld 12,7,6
mulhdu 21,7,6
addc 26,26,12
adde 27,27,21
mulld 12,8,6
mulhdu 21,8,6
addc 28,28,12
adde 29,29,21
mulld 12,9,4
mulhdu 21,9,4
addc 22,22,12
adde 23,23,21
mulld 12,10,4
mulhdu 21,10,4
addc 24,24,12
adde 25,25,21
mulld 12,8,4
mulhdu 21,8,4
ld 6,32(5)
mulli 8,8,19
addc 30,30,12
adde 31,31,21
mulld 12,11,4
mulhdu 21,11,4
addc 26,26,12
adde 27,27,21
mulld 12,7,4
mulhdu 21,7,4
addc 28,28,12
adde 29,29,21
mulld 12,8,6
mulhdu 21,8,6
addc 22,22,12
adde 23,23,21
mulld 12,9,6
mulhdu 21,9,6
addc 24,24,12
adde 25,25,21
mulld 12,10,6
mulhdu 21,10,6
addc 26,26,12
adde 27,27,21
mulld 12,11,6
mulhdu 21,11,6
addc 28,28,12
adde 29,29,21
mulld 12,7,6
mulhdu 21,7,6
addc 30,30,12
adde 31,31,21
.Lfe51_reduce:
li 0,-1
srdi 0,0,13
srdi 12,26,51
and 9,26,0
insrdi 12,27,51,0
srdi 21,22,51
and 7,22,0
insrdi 21,23,51,0
addc 28,28,12
addze 29,29
addc 24,24,21
addze 25,25
srdi 12,28,51
and 10,28,0
insrdi 12,29,51,0
srdi 21,24,51
and 8,24,0
insrdi 21,25,51,0
addc 30,30,12
addze 31,31
add 9,9,21
srdi 12,30,51
and 11,30,0
insrdi 12,31,51,0
mulli 12,12,19
add 7,7,12
srdi 21,9,51
and 9,9,0
add 10,10,21
srdi 12,7,51
and 7,7,0
add 8,8,12
std 9,16(3)
std 10,24(3)
std 11,32(3)
std 7,0(3)
std 8,8(3)
ld 21,56(1)
ld 22,64(1)
ld 23,72(1)
ld 24,80(1)
ld 25,88(1)
ld 26,96(1)
ld 27,104(1)
ld 28,112(1)
ld 29,120(1)
ld 30,128(1)
ld 31,136(1)
addi 1,1,144
blr
SYM_FUNC_END(x25519_fe51_mul)
.align 5
SYM_FUNC_START(x25519_fe51_sqr)
stdu 1,-144(1)
std 21,56(1)
std 22,64(1)
std 23,72(1)
std 24,80(1)
std 25,88(1)
std 26,96(1)
std 27,104(1)
std 28,112(1)
std 29,120(1)
std 30,128(1)
std 31,136(1)
ld 7,0(4)
ld 8,8(4)
ld 9,16(4)
ld 10,24(4)
ld 11,32(4)
add 6,7,7
mulli 21,11,19
mulld 22,7,7
mulhdu 23,7,7
mulld 24,8,6
mulhdu 25,8,6
mulld 26,9,6
mulhdu 27,9,6
mulld 28,10,6
mulhdu 29,10,6
mulld 30,11,6
mulhdu 31,11,6
add 6,8,8
mulld 12,11,21
mulhdu 11,11,21
addc 28,28,12
adde 29,29,11
mulli 5,10,19
mulld 12,8,8
mulhdu 11,8,8
addc 26,26,12
adde 27,27,11
mulld 12,9,6
mulhdu 11,9,6
addc 28,28,12
adde 29,29,11
mulld 12,10,6
mulhdu 11,10,6
addc 30,30,12
adde 31,31,11
mulld 12,21,6
mulhdu 11,21,6
add 6,10,10
addc 22,22,12
adde 23,23,11
mulld 12,10,5
mulhdu 10,10,5
addc 24,24,12
adde 25,25,10
mulld 12,6,21
mulhdu 10,6,21
add 6,9,9
addc 26,26,12
adde 27,27,10
mulld 12,9,9
mulhdu 10,9,9
addc 30,30,12
adde 31,31,10
mulld 12,5,6
mulhdu 10,5,6
addc 22,22,12
adde 23,23,10
mulld 12,21,6
mulhdu 10,21,6
addc 24,24,12
adde 25,25,10
b .Lfe51_reduce
SYM_FUNC_END(x25519_fe51_sqr)
.align 5
SYM_FUNC_START(x25519_fe51_mul121666)
stdu 1,-144(1)
std 21,56(1)
std 22,64(1)
std 23,72(1)
std 24,80(1)
std 25,88(1)
std 26,96(1)
std 27,104(1)
std 28,112(1)
std 29,120(1)
std 30,128(1)
std 31,136(1)
lis 6,1
ori 6,6,56130
ld 7,0(4)
ld 8,8(4)
ld 9,16(4)
ld 10,24(4)
ld 11,32(4)
mulld 22,7,6
mulhdu 23,7,6
mulld 24,8,6
mulhdu 25,8,6
mulld 26,9,6
mulhdu 27,9,6
mulld 28,10,6
mulhdu 29,10,6
mulld 30,11,6
mulhdu 31,11,6
b .Lfe51_reduce
SYM_FUNC_END(x25519_fe51_mul121666)
.align 5
SYM_FUNC_START(x25519_fe51_sqr_times)
stdu 1,-144(1)
std 21,56(1)
std 22,64(1)
std 23,72(1)
std 24,80(1)
std 25,88(1)
std 26,96(1)
std 27,104(1)
std 28,112(1)
std 29,120(1)
std 30,128(1)
std 31,136(1)
ld 7,0(4)
ld 8,8(4)
ld 9,16(4)
ld 10,24(4)
ld 11,32(4)
mtctr 5
.Lsqr_times_loop:
add 6,7,7
mulli 21,11,19
mulld 22,7,7
mulhdu 23,7,7
mulld 24,8,6
mulhdu 25,8,6
mulld 26,9,6
mulhdu 27,9,6
mulld 28,10,6
mulhdu 29,10,6
mulld 30,11,6
mulhdu 31,11,6
add 6,8,8
mulld 12,11,21
mulhdu 11,11,21
addc 28,28,12
adde 29,29,11
mulli 5,10,19
mulld 12,8,8
mulhdu 11,8,8
addc 26,26,12
adde 27,27,11
mulld 12,9,6
mulhdu 11,9,6
addc 28,28,12
adde 29,29,11
mulld 12,10,6
mulhdu 11,10,6
addc 30,30,12
adde 31,31,11
mulld 12,21,6
mulhdu 11,21,6
add 6,10,10
addc 22,22,12
adde 23,23,11
mulld 12,10,5
mulhdu 10,10,5
addc 24,24,12
adde 25,25,10
mulld 12,6,21
mulhdu 10,6,21
add 6,9,9
addc 26,26,12
adde 27,27,10
mulld 12,9,9
mulhdu 10,9,9
addc 30,30,12
adde 31,31,10
mulld 12,5,6
mulhdu 10,5,6
addc 22,22,12
adde 23,23,10
mulld 12,21,6
mulhdu 10,21,6
addc 24,24,12
adde 25,25,10
# fe51_reduce
li 0,-1
srdi 0,0,13
srdi 12,26,51
and 9,26,0
insrdi 12,27,51,0
srdi 21,22,51
and 7,22,0
insrdi 21,23,51,0
addc 28,28,12
addze 29,29
addc 24,24,21
addze 25,25
srdi 12,28,51
and 10,28,0
insrdi 12,29,51,0
srdi 21,24,51
and 8,24,0
insrdi 21,25,51,0
addc 30,30,12
addze 31,31
add 9,9,21
srdi 12,30,51
and 11,30,0
insrdi 12,31,51,0
mulli 12,12,19
add 7,7,12
srdi 21,9,51
and 9,9,0
add 10,10,21
srdi 12,7,51
and 7,7,0
add 8,8,12
bdnz .Lsqr_times_loop
std 9,16(3)
std 10,24(3)
std 11,32(3)
std 7,0(3)
std 8,8(3)
ld 21,56(1)
ld 22,64(1)
ld 23,72(1)
ld 24,80(1)
ld 25,88(1)
ld 26,96(1)
ld 27,104(1)
ld 28,112(1)
ld 29,120(1)
ld 30,128(1)
ld 31,136(1)
addi 1,1,144
blr
SYM_FUNC_END(x25519_fe51_sqr_times)
.align 5
SYM_FUNC_START(x25519_fe51_frombytes)
li 12, -1
srdi 12, 12, 13 # 0x7ffffffffffff
ld 5, 0(4)
ld 6, 8(4)
ld 7, 16(4)
ld 8, 24(4)
srdi 10, 5, 51
and 5, 5, 12 # h0
sldi 11, 6, 13
or 11, 10, 11 # h1t
srdi 10, 6, 38
and 6, 11, 12 # h1
sldi 11, 7, 26
or 10, 10, 11 # h2t
srdi 11, 7, 25
and 7, 10, 12 # h2
sldi 10, 8, 39
or 11, 11, 10 # h3t
srdi 9, 8, 12
and 8, 11, 12 # h3
and 9, 9, 12 # h4
std 5, 0(3)
std 6, 8(3)
std 7, 16(3)
std 8, 24(3)
std 9, 32(3)
blr
SYM_FUNC_END(x25519_fe51_frombytes)
.align 5
SYM_FUNC_START(x25519_fe51_tobytes)
ld 5, 0(4)
ld 6, 8(4)
ld 7, 16(4)
ld 8, 24(4)
ld 9, 32(4)
li 12, -1
srdi 12, 12, 13 # 0x7ffffffffffff
# Full reducuction
addi 10, 5, 19
srdi 10, 10, 51
add 10, 10, 6
srdi 10, 10, 51
add 10, 10, 7
srdi 10, 10, 51
add 10, 10, 8
srdi 10, 10, 51
add 10, 10, 9
srdi 10, 10, 51
mulli 10, 10, 19
add 5, 5, 10
srdi 11, 5, 51
add 6, 6, 11
srdi 11, 6, 51
add 7, 7, 11
srdi 11, 7, 51
add 8, 8, 11
srdi 11, 8, 51
add 9, 9, 11
and 5, 5, 12
and 6, 6, 12
and 7, 7, 12
and 8, 8, 12
and 9, 9, 12
sldi 10, 6, 51
or 5, 5, 10 # s0
srdi 11, 6, 13
sldi 10, 7, 38
or 6, 11, 10 # s1
srdi 11, 7, 26
sldi 10, 8, 25
or 7, 11, 10 # s2
srdi 11, 8, 39
sldi 10, 9, 12
or 8, 11, 10 # s4
std 5, 0(3)
std 6, 8(3)
std 7, 16(3)
std 8, 24(3)
blr
SYM_FUNC_END(x25519_fe51_tobytes)
.align 5
SYM_FUNC_START(x25519_cswap)
li 7, 5
neg 6, 5
mtctr 7
.Lswap_loop:
ld 8, 0(3)
ld 9, 0(4)
xor 10, 8, 9
and 10, 10, 6
xor 11, 8, 10
xor 12, 9, 10
std 11, 0(3)
addi 3, 3, 8
std 12, 0(4)
addi 4, 4, 8
bdnz .Lswap_loop
blr
SYM_FUNC_END(x25519_cswap)
linux-7.2.3/lib/crypto/powerpc/curve25519.h
+186
View File
@@ -0,0 +1,186 @@
// SPDX-License-Identifier: GPL-2.0-or-later
/*
* Copyright 2024- IBM Corp.
*
* X25519 scalar multiplication with 51 bits limbs for PPC64le.
* Based on RFC7748 and AArch64 optimized implementation for X25519
* - Algorithm 1 Scalar multiplication of a variable point
*/
#include <linux/types.h>
#include <linux/jump_label.h>
#include <linux/kernel.h>
#include <linux/cpufeature.h>
#include <linux/processor.h>
typedef uint64_t fe51[5];
asmlinkage void x25519_fe51_mul(fe51 h, const fe51 f, const fe51 g);
asmlinkage void x25519_fe51_sqr(fe51 h, const fe51 f);
asmlinkage void x25519_fe51_mul121666(fe51 h, fe51 f);
asmlinkage void x25519_fe51_sqr_times(fe51 h, const fe51 f, int n);
asmlinkage void x25519_fe51_frombytes(fe51 h, const uint8_t *s);
asmlinkage void x25519_fe51_tobytes(uint8_t *s, const fe51 h);
asmlinkage void x25519_cswap(fe51 p, fe51 q, unsigned int bit);
#define fmul x25519_fe51_mul
#define fsqr x25519_fe51_sqr
#define fmul121666 x25519_fe51_mul121666
#define fe51_tobytes x25519_fe51_tobytes
static void fadd(fe51 h, const fe51 f, const fe51 g)
{
h[0] = f[0] + g[0];
h[1] = f[1] + g[1];
h[2] = f[2] + g[2];
h[3] = f[3] + g[3];
h[4] = f[4] + g[4];
}
/*
* Prime = 2 ** 255 - 19, 255 bits
* (0x7fffffff ffffffff ffffffff ffffffff ffffffff ffffffff ffffffff ffffffed)
*
* Prime in 5 51-bit limbs
*/
static fe51 prime51 = { 0x7ffffffffffed, 0x7ffffffffffff, 0x7ffffffffffff, 0x7ffffffffffff, 0x7ffffffffffff};
static void fsub(fe51 h, const fe51 f, const fe51 g)
{
h[0] = (f[0] + ((prime51[0] * 2))) - g[0];
h[1] = (f[1] + ((prime51[1] * 2))) - g[1];
h[2] = (f[2] + ((prime51[2] * 2))) - g[2];
h[3] = (f[3] + ((prime51[3] * 2))) - g[3];
h[4] = (f[4] + ((prime51[4] * 2))) - g[4];
}
static void fe51_frombytes(fe51 h, const uint8_t *s)
{
/*
* Make sure 64-bit aligned.
*/
unsigned char sbuf[32+8];
unsigned char *sb = PTR_ALIGN((void *)sbuf, 8);
memcpy(sb, s, 32);
x25519_fe51_frombytes(h, sb);
}
static void finv(fe51 o, const fe51 i)
{
fe51 a0, b, c, t00;
fsqr(a0, i);
x25519_fe51_sqr_times(t00, a0, 2);
fmul(b, t00, i);
fmul(a0, b, a0);
fsqr(t00, a0);
fmul(b, t00, b);
x25519_fe51_sqr_times(t00, b, 5);
fmul(b, t00, b);
x25519_fe51_sqr_times(t00, b, 10);
fmul(c, t00, b);
x25519_fe51_sqr_times(t00, c, 20);
fmul(t00, t00, c);
x25519_fe51_sqr_times(t00, t00, 10);
fmul(b, t00, b);
x25519_fe51_sqr_times(t00, b, 50);
fmul(c, t00, b);
x25519_fe51_sqr_times(t00, c, 100);
fmul(t00, t00, c);
x25519_fe51_sqr_times(t00, t00, 50);
fmul(t00, t00, b);
x25519_fe51_sqr_times(t00, t00, 5);
fmul(o, t00, a0);
}
static void curve25519_fe51(uint8_t out[32], const uint8_t scalar[32],
const uint8_t point[32])
{
fe51 x1, x2, z2, x3, z3;
uint8_t s[32];
unsigned int swap = 0;
int i;
memcpy(s, scalar, 32);
s[0] &= 0xf8;
s[31] &= 0x7f;
s[31] |= 0x40;
fe51_frombytes(x1, point);
z2[0] = z2[1] = z2[2] = z2[3] = z2[4] = 0;
x3[0] = x1[0];
x3[1] = x1[1];
x3[2] = x1[2];
x3[3] = x1[3];
x3[4] = x1[4];
x2[0] = z3[0] = 1;
x2[1] = z3[1] = 0;
x2[2] = z3[2] = 0;
x2[3] = z3[3] = 0;
x2[4] = z3[4] = 0;
for (i = 254; i >= 0; --i) {
unsigned int k_t = 1 & (s[i / 8] >> (i & 7));
fe51 a, b, c, d, e;
fe51 da, cb, aa, bb;
fe51 dacb_p, dacb_m;
swap ^= k_t;
x25519_cswap(x2, x3, swap);
x25519_cswap(z2, z3, swap);
swap = k_t;
fsub(b, x2, z2); // B = x_2 - z_2
fadd(a, x2, z2); // A = x_2 + z_2
fsub(d, x3, z3); // D = x_3 - z_3
fadd(c, x3, z3); // C = x_3 + z_3
fsqr(bb, b); // BB = B^2
fsqr(aa, a); // AA = A^2
fmul(da, d, a); // DA = D * A
fmul(cb, c, b); // CB = C * B
fsub(e, aa, bb); // E = AA - BB
fmul(x2, aa, bb); // x2 = AA * BB
fadd(dacb_p, da, cb); // DA + CB
fsub(dacb_m, da, cb); // DA - CB
fmul121666(z3, e); // 121666 * E
fsqr(z2, dacb_m); // (DA - CB)^2
fsqr(x3, dacb_p); // x3 = (DA + CB)^2
fadd(b, bb, z3); // BB + 121666 * E
fmul(z3, x1, z2); // z3 = x1 * (DA - CB)^2
fmul(z2, e, b); // z2 = e * (BB + (DA + CB)^2)
}
finv(z2, z2);
fmul(x2, x2, z2);
fe51_tobytes(out, x2);
}
static void curve25519_arch(u8 mypublic[CURVE25519_KEY_SIZE],
const u8 secret[CURVE25519_KEY_SIZE],
const u8 basepoint[CURVE25519_KEY_SIZE])
{
curve25519_fe51(mypublic, secret, basepoint);
}
static void curve25519_base_arch(u8 pub[CURVE25519_KEY_SIZE],
const u8 secret[CURVE25519_KEY_SIZE])
{
curve25519_fe51(pub, secret, curve25519_base_point);
}
linux-7.2.3/lib/crypto/powerpc/gf128hash.h
+109
View File
@@ -0,0 +1,109 @@
/* SPDX-License-Identifier: GPL-2.0 */
/*
* GHASH routines supporting VMX instructions on the Power 8
*
* Copyright (C) 2015, 2019 International Business Machines Inc.
* Copyright (C) 2014 - 2018 Linaro Ltd.
* Copyright 2026 Google LLC
*/
#include <asm/simd.h>
#include <asm/switch_to.h>
#include <linux/cpufeature.h>
#include <linux/jump_label.h>
#include <linux/preempt.h>
#include <linux/uaccess.h>
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_vec_crypto);
void gcm_init_p8(u64 htable[4][2], const u8 h[16]);
void gcm_gmult_p8(u8 Xi[16], const u64 htable[4][2]);
void gcm_ghash_p8(u8 Xi[16], const u64 htable[4][2], const u8 *in, size_t len);
#define ghash_preparekey_arch ghash_preparekey_arch
static void ghash_preparekey_arch(struct ghash_key *key,
const u8 raw_key[GHASH_BLOCK_SIZE])
{
ghash_key_to_polyval(raw_key, &key->h);
if (static_branch_likely(&have_vec_crypto) && likely(may_use_simd())) {
preempt_disable();
pagefault_disable();
enable_kernel_vsx();
gcm_init_p8(key->htable, raw_key);
disable_kernel_vsx();
pagefault_enable();
preempt_enable();
} else {
/* This reproduces gcm_init_p8() on both LE and BE systems. */
key->htable[0][0] = 0;
key->htable[0][1] = 0xc200000000000000;
key->htable[1][0] = 0;
key->htable[1][1] = le64_to_cpu(key->h.lo);
key->htable[2][0] = le64_to_cpu(key->h.lo);
key->htable[2][1] = le64_to_cpu(key->h.hi);
key->htable[3][0] = le64_to_cpu(key->h.hi);
key->htable[3][1] = 0;
}
}
#define ghash_mul_arch ghash_mul_arch
static void ghash_mul_arch(struct polyval_elem *acc,
const struct ghash_key *key)
{
if (static_branch_likely(&have_vec_crypto) && likely(may_use_simd())) {
u8 ghash_acc[GHASH_BLOCK_SIZE];
polyval_acc_to_ghash(acc, ghash_acc);
preempt_disable();
pagefault_disable();
enable_kernel_vsx();
gcm_gmult_p8(ghash_acc, key->htable);
disable_kernel_vsx();
pagefault_enable();
preempt_enable();
ghash_acc_to_polyval(ghash_acc, acc);
memzero_explicit(ghash_acc, sizeof(ghash_acc));
} else {
polyval_mul_generic(acc, &key->h);
}
}
#define ghash_blocks_arch ghash_blocks_arch
static void ghash_blocks_arch(struct polyval_elem *acc,
const struct ghash_key *key,
const u8 *data, size_t nblocks)
{
if (static_branch_likely(&have_vec_crypto) && likely(may_use_simd())) {
u8 ghash_acc[GHASH_BLOCK_SIZE];
polyval_acc_to_ghash(acc, ghash_acc);
preempt_disable();
pagefault_disable();
enable_kernel_vsx();
gcm_ghash_p8(ghash_acc, key->htable, data,
nblocks * GHASH_BLOCK_SIZE);
disable_kernel_vsx();
pagefault_enable();
preempt_enable();
ghash_acc_to_polyval(ghash_acc, acc);
memzero_explicit(ghash_acc, sizeof(ghash_acc));
} else {
ghash_blocks_generic(acc, &key->h, data, nblocks);
}
}
#define gf128hash_mod_init_arch gf128hash_mod_init_arch
static void gf128hash_mod_init_arch(void)
{
if (cpu_has_feature(CPU_FTR_ARCH_207S) &&
(cur_cpu_spec->cpu_user_features2 & PPC_FEATURE2_VEC_CRYPTO))
static_branch_enable(&have_vec_crypto);
}
linux-7.2.3/lib/crypto/powerpc/ghashp8-ppc.pl
+244
View File
@@ -0,0 +1,244 @@
#!/usr/bin/env perl
# SPDX-License-Identifier: GPL-2.0
# This code is taken from the OpenSSL project but the author (Andy Polyakov)
# has relicensed it under the GPLv2. Therefore this program is free software;
# you can redistribute it and/or modify it under the terms of the GNU General
# Public License version 2 as published by the Free Software Foundation.
#
# The original headers, including the original license headers, are
# included below for completeness.
# ====================================================================
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see https://www.openssl.org/~appro/cryptogams/.
# ====================================================================
#
# GHASH for PowerISA v2.07.
#
# July 2014
#
# Accurate performance measurements are problematic, because it's
# always virtualized setup with possibly throttled processor.
# Relative comparison is therefore more informative. This initial
# version is ~2.1x slower than hardware-assisted AES-128-CTR, ~12x
# faster than "4-bit" integer-only compiler-generated 64-bit code.
# "Initial version" means that there is room for futher improvement.
$flavour=shift;
$output =shift;
if ($flavour =~ /64/) {
$SIZE_T=8;
$LRSAVE=2*$SIZE_T;
$STU="stdu";
$POP="ld";
$PUSH="std";
} elsif ($flavour =~ /32/) {
$SIZE_T=4;
$LRSAVE=$SIZE_T;
$STU="stwu";
$POP="lwz";
$PUSH="stw";
} else { die "nonsense $flavour"; }
$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
( $xlate="${dir}ppc-xlate.pl" and -f $xlate ) or
( $xlate="${dir}../../perlasm/ppc-xlate.pl" and -f $xlate) or
( $xlate="${dir}../../../arch/powerpc/crypto/ppc-xlate.pl" and -f $xlate) or
die "can't locate ppc-xlate.pl";
open STDOUT,"| $^X $xlate $flavour $output" || die "can't call $xlate: $!";
my ($Xip,$Htbl,$inp,$len)=map("r$_",(3..6)); # argument block
my ($Xl,$Xm,$Xh,$IN)=map("v$_",(0..3));
my ($zero,$t0,$t1,$t2,$xC2,$H,$Hh,$Hl,$lemask)=map("v$_",(4..12));
my $vrsave="r12";
$code=<<___;
.machine "any"
.text
.globl .gcm_init_p8
lis r0,0xfff0
li r8,0x10
mfspr $vrsave,256
li r9,0x20
mtspr 256,r0
li r10,0x30
lvx_u $H,0,r4 # load H
le?xor r7,r7,r7
le?addi r7,r7,0x8 # need a vperm start with 08
le?lvsr 5,0,r7
le?vspltisb 6,0x0f
le?vxor 5,5,6 # set a b-endian mask
le?vperm $H,$H,$H,5
vspltisb $xC2,-16 # 0xf0
vspltisb $t0,1 # one
vaddubm $xC2,$xC2,$xC2 # 0xe0
vxor $zero,$zero,$zero
vor $xC2,$xC2,$t0 # 0xe1
vsldoi $xC2,$xC2,$zero,15 # 0xe1...
vsldoi $t1,$zero,$t0,1 # ...1
vaddubm $xC2,$xC2,$xC2 # 0xc2...
vspltisb $t2,7
vor $xC2,$xC2,$t1 # 0xc2....01
vspltb $t1,$H,0 # most significant byte
vsl $H,$H,$t0 # H<<=1
vsrab $t1,$t1,$t2 # broadcast carry bit
vand $t1,$t1,$xC2
vxor $H,$H,$t1 # twisted H
vsldoi $H,$H,$H,8 # twist even more ...
vsldoi $xC2,$zero,$xC2,8 # 0xc2.0
vsldoi $Hl,$zero,$H,8 # ... and split
vsldoi $Hh,$H,$zero,8
stvx_u $xC2,0,r3 # save pre-computed table
stvx_u $Hl,r8,r3
stvx_u $H, r9,r3
stvx_u $Hh,r10,r3
mtspr 256,$vrsave
blr
.long 0
.byte 0,12,0x14,0,0,0,2,0
.long 0
.size .gcm_init_p8,.-.gcm_init_p8
.globl .gcm_gmult_p8
lis r0,0xfff8
li r8,0x10
mfspr $vrsave,256
li r9,0x20
mtspr 256,r0
li r10,0x30
lvx_u $IN,0,$Xip # load Xi
lvx_u $Hl,r8,$Htbl # load pre-computed table
le?lvsl $lemask,r0,r0
lvx_u $H, r9,$Htbl
le?vspltisb $t0,0x07
lvx_u $Hh,r10,$Htbl
le?vxor $lemask,$lemask,$t0
lvx_u $xC2,0,$Htbl
le?vperm $IN,$IN,$IN,$lemask
vxor $zero,$zero,$zero
vpmsumd $Xl,$IN,$Hl # H.lo·Xi.lo
vpmsumd $Xm,$IN,$H # H.hi·Xi.lo+H.lo·Xi.hi
vpmsumd $Xh,$IN,$Hh # H.hi·Xi.hi
vpmsumd $t2,$Xl,$xC2 # 1st phase
vsldoi $t0,$Xm,$zero,8
vsldoi $t1,$zero,$Xm,8
vxor $Xl,$Xl,$t0
vxor $Xh,$Xh,$t1
vsldoi $Xl,$Xl,$Xl,8
vxor $Xl,$Xl,$t2
vsldoi $t1,$Xl,$Xl,8 # 2nd phase
vpmsumd $Xl,$Xl,$xC2
vxor $t1,$t1,$Xh
vxor $Xl,$Xl,$t1
le?vperm $Xl,$Xl,$Xl,$lemask
stvx_u $Xl,0,$Xip # write out Xi
mtspr 256,$vrsave
blr
.long 0
.byte 0,12,0x14,0,0,0,2,0
.long 0
.size .gcm_gmult_p8,.-.gcm_gmult_p8
.globl .gcm_ghash_p8
lis r0,0xfff8
li r8,0x10
mfspr $vrsave,256
li r9,0x20
mtspr 256,r0
li r10,0x30
lvx_u $Xl,0,$Xip # load Xi
lvx_u $Hl,r8,$Htbl # load pre-computed table
le?lvsl $lemask,r0,r0
lvx_u $H, r9,$Htbl
le?vspltisb $t0,0x07
lvx_u $Hh,r10,$Htbl
le?vxor $lemask,$lemask,$t0
lvx_u $xC2,0,$Htbl
le?vperm $Xl,$Xl,$Xl,$lemask
vxor $zero,$zero,$zero
lvx_u $IN,0,$inp
addi $inp,$inp,16
subi $len,$len,16
le?vperm $IN,$IN,$IN,$lemask
vxor $IN,$IN,$Xl
b Loop
.align 5
Loop:
subic $len,$len,16
vpmsumd $Xl,$IN,$Hl # H.lo·Xi.lo
subfe. r0,r0,r0 # borrow?-1:0
vpmsumd $Xm,$IN,$H # H.hi·Xi.lo+H.lo·Xi.hi
and r0,r0,$len
vpmsumd $Xh,$IN,$Hh # H.hi·Xi.hi
add $inp,$inp,r0
vpmsumd $t2,$Xl,$xC2 # 1st phase
vsldoi $t0,$Xm,$zero,8
vsldoi $t1,$zero,$Xm,8
vxor $Xl,$Xl,$t0
vxor $Xh,$Xh,$t1
vsldoi $Xl,$Xl,$Xl,8
vxor $Xl,$Xl,$t2
lvx_u $IN,0,$inp
addi $inp,$inp,16
vsldoi $t1,$Xl,$Xl,8 # 2nd phase
vpmsumd $Xl,$Xl,$xC2
le?vperm $IN,$IN,$IN,$lemask
vxor $t1,$t1,$Xh
vxor $IN,$IN,$t1
vxor $IN,$IN,$Xl
beq Loop # did $len-=16 borrow?
vxor $Xl,$Xl,$t1
le?vperm $Xl,$Xl,$Xl,$lemask
stvx_u $Xl,0,$Xip # write out Xi
mtspr 256,$vrsave
blr
.long 0
.byte 0,12,0x14,0,0,0,4,0
.long 0
.size .gcm_ghash_p8,.-.gcm_ghash_p8
.asciz "GHASH for PowerISA 2.07, CRYPTOGAMS by <appro\@openssl.org>"
.align 2
___
foreach (split("\n",$code)) {
if ($flavour =~ /le$/o) { # little-endian
s/le\?//o or
s/be\?/#be#/o;
} else {
s/le\?/#le#/o or
s/be\?//o;
}
print $_,"\n";
}
close STDOUT; # enforce flush
linux-7.2.3/lib/crypto/powerpc/poly1305-p10le_64.S
+1075
View File
File diff suppressed because it is too large Load Diff
linux-7.2.3/lib/crypto/powerpc/poly1305.h
+74
View File
@@ -0,0 +1,74 @@
/* SPDX-License-Identifier: GPL-2.0 */
/*
* Poly1305 authenticator algorithm, RFC7539.
*
* Copyright 2023- IBM Corp. All rights reserved.
*/
#include <asm/switch_to.h>
#include <linux/cpufeature.h>
#include <linux/jump_label.h>
#include <linux/kernel.h>
#include <linux/unaligned.h>
asmlinkage void poly1305_p10le_4blocks(struct poly1305_block_state *state, const u8 *m, u32 mlen);
asmlinkage void poly1305_64s(struct poly1305_block_state *state, const u8 *m, u32 mlen, int highbit);
asmlinkage void poly1305_emit_64(const struct poly1305_state *state, const u32 nonce[4], u8 digest[POLY1305_DIGEST_SIZE]);
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_p10);
static void vsx_begin(void)
{
preempt_disable();
enable_kernel_vsx();
}
static void vsx_end(void)
{
disable_kernel_vsx();
preempt_enable();
}
static void poly1305_block_init(struct poly1305_block_state *dctx,
const u8 raw_key[POLY1305_BLOCK_SIZE])
{
if (!static_key_enabled(&have_p10))
return poly1305_block_init_generic(dctx, raw_key);
dctx->h = (struct poly1305_state){};
dctx->core_r.key.r64[0] = get_unaligned_le64(raw_key + 0);
dctx->core_r.key.r64[1] = get_unaligned_le64(raw_key + 8);
}
static void poly1305_blocks(struct poly1305_block_state *state, const u8 *src,
unsigned int len, u32 padbit)
{
if (!static_key_enabled(&have_p10))
return poly1305_blocks_generic(state, src, len, padbit);
vsx_begin();
if (len >= POLY1305_BLOCK_SIZE * 4) {
poly1305_p10le_4blocks(state, src, len);
src += len - (len % (POLY1305_BLOCK_SIZE * 4));
len %= POLY1305_BLOCK_SIZE * 4;
}
while (len >= POLY1305_BLOCK_SIZE) {
poly1305_64s(state, src, POLY1305_BLOCK_SIZE, padbit);
len -= POLY1305_BLOCK_SIZE;
src += POLY1305_BLOCK_SIZE;
}
vsx_end();
}
static void poly1305_emit(const struct poly1305_state *state,
u8 digest[POLY1305_DIGEST_SIZE], const u32 nonce[4])
{
if (!static_key_enabled(&have_p10))
return poly1305_emit_generic(state, digest, nonce);
poly1305_emit_64(state, nonce, digest);
}
#define poly1305_mod_init_arch poly1305_mod_init_arch
static void poly1305_mod_init_arch(void)
{
if (cpu_has_feature(CPU_FTR_ARCH_31))
static_branch_enable(&have_p10);
}
linux-7.2.3/lib/crypto/powerpc/sha1-powerpc-asm.S
+188
View File
@@ -0,0 +1,188 @@
/* SPDX-License-Identifier: GPL-2.0 */
/*
* SHA-1 implementation for PowerPC.
*
* Copyright (C) 2005 Paul Mackerras <paulus@samba.org>
*/
#include <asm/ppc_asm.h>
#include <asm/asm-offsets.h>
#include <asm/asm-compat.h>
#ifdef __BIG_ENDIAN__
#define LWZ(rt, d, ra) \
lwz rt,d(ra)
#else
#define LWZ(rt, d, ra) \
li rt,d; \
lwbrx rt,rt,ra
#endif
/*
* We roll the registers for T, A, B, C, D, E around on each
* iteration; T on iteration t is A on iteration t+1, and so on.
* We use registers 7 - 12 for this.
*/
#define RT(t) ((((t)+5)%6)+7)
#define RA(t) ((((t)+4)%6)+7)
#define RB(t) ((((t)+3)%6)+7)
#define RC(t) ((((t)+2)%6)+7)
#define RD(t) ((((t)+1)%6)+7)
#define RE(t) ((((t)+0)%6)+7)
/* We use registers 16 - 31 for the W values */
#define W(t) (((t)%16)+16)
#define LOADW(t) \
LWZ(W(t),(t)*4,r4)
#define STEPD0_LOAD(t) \
andc r0,RD(t),RB(t); \
and r6,RB(t),RC(t); \
rotlwi RT(t),RA(t),5; \
or r6,r6,r0; \
add r0,RE(t),r15; \
add RT(t),RT(t),r6; \
add r14,r0,W(t); \
LWZ(W((t)+4),((t)+4)*4,r4); \
rotlwi RB(t),RB(t),30; \
add RT(t),RT(t),r14
#define STEPD0_UPDATE(t) \
and r6,RB(t),RC(t); \
andc r0,RD(t),RB(t); \
rotlwi RT(t),RA(t),5; \
rotlwi RB(t),RB(t),30; \
or r6,r6,r0; \
add r0,RE(t),r15; \
xor r5,W((t)+4-3),W((t)+4-8); \
add RT(t),RT(t),r6; \
xor W((t)+4),W((t)+4-16),W((t)+4-14); \
add r0,r0,W(t); \
xor W((t)+4),W((t)+4),r5; \
add RT(t),RT(t),r0; \
rotlwi W((t)+4),W((t)+4),1
#define STEPD1(t) \
xor r6,RB(t),RC(t); \
rotlwi RT(t),RA(t),5; \
rotlwi RB(t),RB(t),30; \
xor r6,r6,RD(t); \
add r0,RE(t),r15; \
add RT(t),RT(t),r6; \
add r0,r0,W(t); \
add RT(t),RT(t),r0
#define STEPD1_UPDATE(t) \
xor r6,RB(t),RC(t); \
rotlwi RT(t),RA(t),5; \
rotlwi RB(t),RB(t),30; \
xor r6,r6,RD(t); \
add r0,RE(t),r15; \
xor r5,W((t)+4-3),W((t)+4-8); \
add RT(t),RT(t),r6; \
xor W((t)+4),W((t)+4-16),W((t)+4-14); \
add r0,r0,W(t); \
xor W((t)+4),W((t)+4),r5; \
add RT(t),RT(t),r0; \
rotlwi W((t)+4),W((t)+4),1
#define STEPD2_UPDATE(t) \
and r6,RB(t),RC(t); \
and r0,RB(t),RD(t); \
rotlwi RT(t),RA(t),5; \
or r6,r6,r0; \
rotlwi RB(t),RB(t),30; \
and r0,RC(t),RD(t); \
xor r5,W((t)+4-3),W((t)+4-8); \
or r6,r6,r0; \
xor W((t)+4),W((t)+4-16),W((t)+4-14); \
add r0,RE(t),r15; \
add RT(t),RT(t),r6; \
add r0,r0,W(t); \
xor W((t)+4),W((t)+4),r5; \
add RT(t),RT(t),r0; \
rotlwi W((t)+4),W((t)+4),1
#define STEP0LD4(t) \
STEPD0_LOAD(t); \
STEPD0_LOAD((t)+1); \
STEPD0_LOAD((t)+2); \
STEPD0_LOAD((t)+3)
#define STEPUP4(t, fn) \
STEP##fn##_UPDATE(t); \
STEP##fn##_UPDATE((t)+1); \
STEP##fn##_UPDATE((t)+2); \
STEP##fn##_UPDATE((t)+3)
#define STEPUP20(t, fn) \
STEPUP4(t, fn); \
STEPUP4((t)+4, fn); \
STEPUP4((t)+8, fn); \
STEPUP4((t)+12, fn); \
STEPUP4((t)+16, fn)
_GLOBAL(powerpc_sha_transform)
PPC_STLU r1,-INT_FRAME_SIZE(r1)
SAVE_GPRS(14, 31, r1)
/* Load up A - E */
lwz RA(0),0(r3) /* A */
lwz RB(0),4(r3) /* B */
lwz RC(0),8(r3) /* C */
lwz RD(0),12(r3) /* D */
lwz RE(0),16(r3) /* E */
LOADW(0)
LOADW(1)
LOADW(2)
LOADW(3)
lis r15,0x5a82 /* K0-19 */
ori r15,r15,0x7999
STEP0LD4(0)
STEP0LD4(4)
STEP0LD4(8)
STEPUP4(12, D0)
STEPUP4(16, D0)
lis r15,0x6ed9 /* K20-39 */
ori r15,r15,0xeba1
STEPUP20(20, D1)
lis r15,0x8f1b /* K40-59 */
ori r15,r15,0xbcdc
STEPUP20(40, D2)
lis r15,0xca62 /* K60-79 */
ori r15,r15,0xc1d6
STEPUP4(60, D1)
STEPUP4(64, D1)
STEPUP4(68, D1)
STEPUP4(72, D1)
lwz r20,16(r3)
STEPD1(76)
lwz r19,12(r3)
STEPD1(77)
lwz r18,8(r3)
STEPD1(78)
lwz r17,4(r3)
STEPD1(79)
lwz r16,0(r3)
add r20,RE(80),r20
add RD(0),RD(80),r19
add RC(0),RC(80),r18
add RB(0),RB(80),r17
add RA(0),RA(80),r16
mr RE(0),r20
stw RA(0),0(r3)
stw RB(0),4(r3)
stw RC(0),8(r3)
stw RD(0),12(r3)
stw RE(0),16(r3)
REST_GPRS(14, 31, r1)
addi r1,r1,INT_FRAME_SIZE
blr
linux-7.2.3/lib/crypto/powerpc/sha1-spe-asm.S
+294
View File
@@ -0,0 +1,294 @@
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
* Fast SHA-1 implementation for SPE instruction set (PPC)
*
* This code makes use of the SPE SIMD instruction set as defined in
* http://cache.freescale.com/files/32bit/doc/ref_manual/SPEPIM.pdf
* Implementation is based on optimization guide notes from
* http://cache.freescale.com/files/32bit/doc/app_note/AN2665.pdf
*
* Copyright (c) 2015 Markus Stockhausen <stockhausen@collogia.de>
*/
#include <asm/ppc_asm.h>
#include <asm/asm-offsets.h>
#define rHP r3 /* pointer to hash value */
#define rWP r4 /* pointer to input */
#define rKP r5 /* pointer to constants */
#define rW0 r14 /* 64 bit round words */
#define rW1 r15
#define rW2 r16
#define rW3 r17
#define rW4 r18
#define rW5 r19
#define rW6 r20
#define rW7 r21
#define rH0 r6 /* 32 bit hash values */
#define rH1 r7
#define rH2 r8
#define rH3 r9
#define rH4 r10
#define rT0 r22 /* 64 bit temporary */
#define rT1 r0 /* 32 bit temporaries */
#define rT2 r11
#define rT3 r12
#define rK r23 /* 64 bit constant in volatile register */
#define LOAD_K01
#define LOAD_K11 \
evlwwsplat rK,0(rKP);
#define LOAD_K21 \
evlwwsplat rK,4(rKP);
#define LOAD_K31 \
evlwwsplat rK,8(rKP);
#define LOAD_K41 \
evlwwsplat rK,12(rKP);
#define INITIALIZE \
stwu r1,-128(r1); /* create stack frame */ \
evstdw r14,8(r1); /* We must save non volatile */ \
evstdw r15,16(r1); /* registers. Take the chance */ \
evstdw r16,24(r1); /* and save the SPE part too */ \
evstdw r17,32(r1); \
evstdw r18,40(r1); \
evstdw r19,48(r1); \
evstdw r20,56(r1); \
evstdw r21,64(r1); \
evstdw r22,72(r1); \
evstdw r23,80(r1);
#define FINALIZE \
evldw r14,8(r1); /* restore SPE registers */ \
evldw r15,16(r1); \
evldw r16,24(r1); \
evldw r17,32(r1); \
evldw r18,40(r1); \
evldw r19,48(r1); \
evldw r20,56(r1); \
evldw r21,64(r1); \
evldw r22,72(r1); \
evldw r23,80(r1); \
xor r0,r0,r0; \
stw r0,8(r1); /* Delete sensitive data */ \
stw r0,16(r1); /* that we might have pushed */ \
stw r0,24(r1); /* from other context that runs */ \
stw r0,32(r1); /* the same code. Assume that */ \
stw r0,40(r1); /* the lower part of the GPRs */ \
stw r0,48(r1); /* were already overwritten on */ \
stw r0,56(r1); /* the way down to here */ \
stw r0,64(r1); \
stw r0,72(r1); \
stw r0,80(r1); \
addi r1,r1,128; /* cleanup stack frame */
#ifdef __BIG_ENDIAN__
#define LOAD_DATA(reg, off) \
lwz reg,off(rWP); /* load data */
#define NEXT_BLOCK \
addi rWP,rWP,64; /* increment per block */
#else
#define LOAD_DATA(reg, off) \
lwbrx reg,0,rWP; /* load data */ \
addi rWP,rWP,4; /* increment per word */
#define NEXT_BLOCK /* nothing to do */
#endif
#define R_00_15(a, b, c, d, e, w0, w1, k, off) \
LOAD_DATA(w0, off) /* 1: W */ \
and rT2,b,c; /* 1: F' = B and C */ \
LOAD_K##k##1 \
andc rT1,d,b; /* 1: F" = ~B and D */ \
rotrwi rT0,a,27; /* 1: A' = A rotl 5 */ \
or rT2,rT2,rT1; /* 1: F = F' or F" */ \
add e,e,rT0; /* 1: E = E + A' */ \
rotrwi b,b,2; /* 1: B = B rotl 30 */ \
add e,e,w0; /* 1: E = E + W */ \
LOAD_DATA(w1, off+4) /* 2: W */ \
add e,e,rT2; /* 1: E = E + F */ \
and rT1,a,b; /* 2: F' = B and C */ \
add e,e,rK; /* 1: E = E + K */ \
andc rT2,c,a; /* 2: F" = ~B and D */ \
add d,d,rK; /* 2: E = E + K */ \
or rT2,rT2,rT1; /* 2: F = F' or F" */ \
rotrwi rT0,e,27; /* 2: A' = A rotl 5 */ \
add d,d,w1; /* 2: E = E + W */ \
rotrwi a,a,2; /* 2: B = B rotl 30 */ \
add d,d,rT0; /* 2: E = E + A' */ \
evmergelo w1,w1,w0; /* mix W[0]/W[1] */ \
add d,d,rT2 /* 2: E = E + F */
#define R_16_19(a, b, c, d, e, w0, w1, w4, w6, w7, k) \
and rT2,b,c; /* 1: F' = B and C */ \
evmergelohi rT0,w7,w6; /* W[-3] */ \
andc rT1,d,b; /* 1: F" = ~B and D */ \
evxor w0,w0,rT0; /* W = W[-16] xor W[-3] */ \
or rT1,rT1,rT2; /* 1: F = F' or F" */ \
evxor w0,w0,w4; /* W = W xor W[-8] */ \
add e,e,rT1; /* 1: E = E + F */ \
evxor w0,w0,w1; /* W = W xor W[-14] */ \
rotrwi rT2,a,27; /* 1: A' = A rotl 5 */ \
evrlwi w0,w0,1; /* W = W rotl 1 */ \
add e,e,rT2; /* 1: E = E + A' */ \
evaddw rT0,w0,rK; /* WK = W + K */ \
rotrwi b,b,2; /* 1: B = B rotl 30 */ \
LOAD_K##k##1 \
evmergehi rT1,rT1,rT0; /* WK1/WK2 */ \
add e,e,rT0; /* 1: E = E + WK */ \
add d,d,rT1; /* 2: E = E + WK */ \
and rT2,a,b; /* 2: F' = B and C */ \
andc rT1,c,a; /* 2: F" = ~B and D */ \
rotrwi rT0,e,27; /* 2: A' = A rotl 5 */ \
or rT1,rT1,rT2; /* 2: F = F' or F" */ \
add d,d,rT0; /* 2: E = E + A' */ \
rotrwi a,a,2; /* 2: B = B rotl 30 */ \
add d,d,rT1 /* 2: E = E + F */
#define R_20_39(a, b, c, d, e, w0, w1, w4, w6, w7, k) \
evmergelohi rT0,w7,w6; /* W[-3] */ \
xor rT2,b,c; /* 1: F' = B xor C */ \
evxor w0,w0,rT0; /* W = W[-16] xor W[-3] */ \
xor rT2,rT2,d; /* 1: F = F' xor D */ \
evxor w0,w0,w4; /* W = W xor W[-8] */ \
add e,e,rT2; /* 1: E = E + F */ \
evxor w0,w0,w1; /* W = W xor W[-14] */ \
rotrwi rT2,a,27; /* 1: A' = A rotl 5 */ \
evrlwi w0,w0,1; /* W = W rotl 1 */ \
add e,e,rT2; /* 1: E = E + A' */ \
evaddw rT0,w0,rK; /* WK = W + K */ \
rotrwi b,b,2; /* 1: B = B rotl 30 */ \
LOAD_K##k##1 \
evmergehi rT1,rT1,rT0; /* WK1/WK2 */ \
add e,e,rT0; /* 1: E = E + WK */ \
xor rT2,a,b; /* 2: F' = B xor C */ \
add d,d,rT1; /* 2: E = E + WK */ \
xor rT2,rT2,c; /* 2: F = F' xor D */ \
rotrwi rT0,e,27; /* 2: A' = A rotl 5 */ \
add d,d,rT2; /* 2: E = E + F */ \
rotrwi a,a,2; /* 2: B = B rotl 30 */ \
add d,d,rT0 /* 2: E = E + A' */
#define R_40_59(a, b, c, d, e, w0, w1, w4, w6, w7, k) \
and rT2,b,c; /* 1: F' = B and C */ \
evmergelohi rT0,w7,w6; /* W[-3] */ \
or rT1,b,c; /* 1: F" = B or C */ \
evxor w0,w0,rT0; /* W = W[-16] xor W[-3] */ \
and rT1,d,rT1; /* 1: F" = F" and D */ \
evxor w0,w0,w4; /* W = W xor W[-8] */ \
or rT2,rT2,rT1; /* 1: F = F' or F" */ \
evxor w0,w0,w1; /* W = W xor W[-14] */ \
add e,e,rT2; /* 1: E = E + F */ \
evrlwi w0,w0,1; /* W = W rotl 1 */ \
rotrwi rT2,a,27; /* 1: A' = A rotl 5 */ \
evaddw rT0,w0,rK; /* WK = W + K */ \
add e,e,rT2; /* 1: E = E + A' */ \
LOAD_K##k##1 \
evmergehi rT1,rT1,rT0; /* WK1/WK2 */ \
rotrwi b,b,2; /* 1: B = B rotl 30 */ \
add e,e,rT0; /* 1: E = E + WK */ \
and rT2,a,b; /* 2: F' = B and C */ \
or rT0,a,b; /* 2: F" = B or C */ \
add d,d,rT1; /* 2: E = E + WK */ \
and rT0,c,rT0; /* 2: F" = F" and D */ \
rotrwi a,a,2; /* 2: B = B rotl 30 */ \
or rT2,rT2,rT0; /* 2: F = F' or F" */ \
rotrwi rT0,e,27; /* 2: A' = A rotl 5 */ \
add d,d,rT2; /* 2: E = E + F */ \
add d,d,rT0 /* 2: E = E + A' */
#define R_60_79(a, b, c, d, e, w0, w1, w4, w6, w7, k) \
R_20_39(a, b, c, d, e, w0, w1, w4, w6, w7, k)
_GLOBAL(ppc_spe_sha1_transform)
INITIALIZE
lwz rH0,0(rHP)
lwz rH1,4(rHP)
mtctr r5
lwz rH2,8(rHP)
lis rKP,PPC_SPE_SHA1_K@h
lwz rH3,12(rHP)
ori rKP,rKP,PPC_SPE_SHA1_K@l
lwz rH4,16(rHP)
ppc_spe_sha1_main:
R_00_15(rH0, rH1, rH2, rH3, rH4, rW1, rW0, 1, 0)
R_00_15(rH3, rH4, rH0, rH1, rH2, rW2, rW1, 0, 8)
R_00_15(rH1, rH2, rH3, rH4, rH0, rW3, rW2, 0, 16)
R_00_15(rH4, rH0, rH1, rH2, rH3, rW4, rW3, 0, 24)
R_00_15(rH2, rH3, rH4, rH0, rH1, rW5, rW4, 0, 32)
R_00_15(rH0, rH1, rH2, rH3, rH4, rW6, rW5, 0, 40)
R_00_15(rH3, rH4, rH0, rH1, rH2, rT3, rW6, 0, 48)
R_00_15(rH1, rH2, rH3, rH4, rH0, rT3, rW7, 0, 56)
R_16_19(rH4, rH0, rH1, rH2, rH3, rW0, rW1, rW4, rW6, rW7, 0)
R_16_19(rH2, rH3, rH4, rH0, rH1, rW1, rW2, rW5, rW7, rW0, 2)
R_20_39(rH0, rH1, rH2, rH3, rH4, rW2, rW3, rW6, rW0, rW1, 0)
R_20_39(rH3, rH4, rH0, rH1, rH2, rW3, rW4, rW7, rW1, rW2, 0)
R_20_39(rH1, rH2, rH3, rH4, rH0, rW4, rW5, rW0, rW2, rW3, 0)
R_20_39(rH4, rH0, rH1, rH2, rH3, rW5, rW6, rW1, rW3, rW4, 0)
R_20_39(rH2, rH3, rH4, rH0, rH1, rW6, rW7, rW2, rW4, rW5, 0)
R_20_39(rH0, rH1, rH2, rH3, rH4, rW7, rW0, rW3, rW5, rW6, 0)
R_20_39(rH3, rH4, rH0, rH1, rH2, rW0, rW1, rW4, rW6, rW7, 0)
R_20_39(rH1, rH2, rH3, rH4, rH0, rW1, rW2, rW5, rW7, rW0, 0)
R_20_39(rH4, rH0, rH1, rH2, rH3, rW2, rW3, rW6, rW0, rW1, 0)
R_20_39(rH2, rH3, rH4, rH0, rH1, rW3, rW4, rW7, rW1, rW2, 3)
R_40_59(rH0, rH1, rH2, rH3, rH4, rW4, rW5, rW0, rW2, rW3, 0)
R_40_59(rH3, rH4, rH0, rH1, rH2, rW5, rW6, rW1, rW3, rW4, 0)
R_40_59(rH1, rH2, rH3, rH4, rH0, rW6, rW7, rW2, rW4, rW5, 0)
R_40_59(rH4, rH0, rH1, rH2, rH3, rW7, rW0, rW3, rW5, rW6, 0)
R_40_59(rH2, rH3, rH4, rH0, rH1, rW0, rW1, rW4, rW6, rW7, 0)
R_40_59(rH0, rH1, rH2, rH3, rH4, rW1, rW2, rW5, rW7, rW0, 0)
R_40_59(rH3, rH4, rH0, rH1, rH2, rW2, rW3, rW6, rW0, rW1, 0)
R_40_59(rH1, rH2, rH3, rH4, rH0, rW3, rW4, rW7, rW1, rW2, 0)
R_40_59(rH4, rH0, rH1, rH2, rH3, rW4, rW5, rW0, rW2, rW3, 0)
R_40_59(rH2, rH3, rH4, rH0, rH1, rW5, rW6, rW1, rW3, rW4, 4)
R_60_79(rH0, rH1, rH2, rH3, rH4, rW6, rW7, rW2, rW4, rW5, 0)
R_60_79(rH3, rH4, rH0, rH1, rH2, rW7, rW0, rW3, rW5, rW6, 0)
R_60_79(rH1, rH2, rH3, rH4, rH0, rW0, rW1, rW4, rW6, rW7, 0)
R_60_79(rH4, rH0, rH1, rH2, rH3, rW1, rW2, rW5, rW7, rW0, 0)
R_60_79(rH2, rH3, rH4, rH0, rH1, rW2, rW3, rW6, rW0, rW1, 0)
R_60_79(rH0, rH1, rH2, rH3, rH4, rW3, rW4, rW7, rW1, rW2, 0)
R_60_79(rH3, rH4, rH0, rH1, rH2, rW4, rW5, rW0, rW2, rW3, 0)
lwz rT3,0(rHP)
R_60_79(rH1, rH2, rH3, rH4, rH0, rW5, rW6, rW1, rW3, rW4, 0)
lwz rW1,4(rHP)
R_60_79(rH4, rH0, rH1, rH2, rH3, rW6, rW7, rW2, rW4, rW5, 0)
lwz rW2,8(rHP)
R_60_79(rH2, rH3, rH4, rH0, rH1, rW7, rW0, rW3, rW5, rW6, 0)
lwz rW3,12(rHP)
NEXT_BLOCK
lwz rW4,16(rHP)
add rH0,rH0,rT3
stw rH0,0(rHP)
add rH1,rH1,rW1
stw rH1,4(rHP)
add rH2,rH2,rW2
stw rH2,8(rHP)
add rH3,rH3,rW3
stw rH3,12(rHP)
add rH4,rH4,rW4
stw rH4,16(rHP)
bdnz ppc_spe_sha1_main
FINALIZE
blr
.data
.align 4
PPC_SPE_SHA1_K:
.long 0x5A827999,0x6ED9EBA1,0x8F1BBCDC,0xCA62C1D6
linux-7.2.3/lib/crypto/powerpc/sha1.h
+67
View File
@@ -0,0 +1,67 @@
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
* SHA-1 optimized for PowerPC
*
* Copyright (c) 2015 Markus Stockhausen <stockhausen@collogia.de>
*/
#include <asm/switch_to.h>
#include <linux/preempt.h>
#ifdef CONFIG_SPE
/*
* MAX_BYTES defines the number of bytes that are allowed to be processed
* between preempt_disable() and preempt_enable(). SHA1 takes ~1000
* operations per 64 bytes. e500 cores can issue two arithmetic instructions
* per clock cycle using one 32/64 bit unit (SU1) and one 32 bit unit (SU2).
* Thus 2KB of input data will need an estimated maximum of 18,000 cycles.
* Headroom for cache misses included. Even with the low end model clocked
* at 667 MHz this equals to a critical time window of less than 27us.
*
*/
#define MAX_BYTES 2048
asmlinkage void ppc_spe_sha1_transform(struct sha1_block_state *state,
const u8 *data, u32 nblocks);
static void spe_begin(void)
{
/* We just start SPE operations and will save SPE registers later. */
preempt_disable();
enable_kernel_spe();
}
static void spe_end(void)
{
disable_kernel_spe();
/* reenable preemption */
preempt_enable();
}
static void sha1_blocks(struct sha1_block_state *state,
const u8 *data, size_t nblocks)
{
do {
u32 unit = min_t(size_t, nblocks, MAX_BYTES / SHA1_BLOCK_SIZE);
spe_begin();
ppc_spe_sha1_transform(state, data, unit);
spe_end();
data += unit * SHA1_BLOCK_SIZE;
nblocks -= unit;
} while (nblocks);
}
#else /* CONFIG_SPE */
asmlinkage void powerpc_sha_transform(struct sha1_block_state *state,
const u8 data[SHA1_BLOCK_SIZE]);
static void sha1_blocks(struct sha1_block_state *state,
const u8 *data, size_t nblocks)
{
do {
powerpc_sha_transform(state, data);
data += SHA1_BLOCK_SIZE;
} while (--nblocks);
}
#endif /* !CONFIG_SPE */
linux-7.2.3/lib/crypto/powerpc/sha256-spe-asm.S
+318
View File
@@ -0,0 +1,318 @@
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
* Fast SHA-256 implementation for SPE instruction set (PPC)
*
* This code makes use of the SPE SIMD instruction set as defined in
* http://cache.freescale.com/files/32bit/doc/ref_manual/SPEPIM.pdf
* Implementation is based on optimization guide notes from
* http://cache.freescale.com/files/32bit/doc/app_note/AN2665.pdf
*
* Copyright (c) 2015 Markus Stockhausen <stockhausen@collogia.de>
*/
#include <asm/ppc_asm.h>
#include <asm/asm-offsets.h>
#define rHP r3 /* pointer to hash values in memory */
#define rKP r24 /* pointer to round constants */
#define rWP r4 /* pointer to input data */
#define rH0 r5 /* 8 32 bit hash values in 8 registers */
#define rH1 r6
#define rH2 r7
#define rH3 r8
#define rH4 r9
#define rH5 r10
#define rH6 r11
#define rH7 r12
#define rW0 r14 /* 64 bit registers. 16 words in 8 registers */
#define rW1 r15
#define rW2 r16
#define rW3 r17
#define rW4 r18
#define rW5 r19
#define rW6 r20
#define rW7 r21
#define rT0 r22 /* 64 bit temporaries */
#define rT1 r23
#define rT2 r0 /* 32 bit temporaries */
#define rT3 r25
#define CMP_KN_LOOP
#define CMP_KC_LOOP \
cmpwi rT1,0;
#define INITIALIZE \
stwu r1,-128(r1); /* create stack frame */ \
evstdw r14,8(r1); /* We must save non volatile */ \
evstdw r15,16(r1); /* registers. Take the chance */ \
evstdw r16,24(r1); /* and save the SPE part too */ \
evstdw r17,32(r1); \
evstdw r18,40(r1); \
evstdw r19,48(r1); \
evstdw r20,56(r1); \
evstdw r21,64(r1); \
evstdw r22,72(r1); \
evstdw r23,80(r1); \
stw r24,88(r1); /* save normal registers */ \
stw r25,92(r1);
#define FINALIZE \
evldw r14,8(r1); /* restore SPE registers */ \
evldw r15,16(r1); \
evldw r16,24(r1); \
evldw r17,32(r1); \
evldw r18,40(r1); \
evldw r19,48(r1); \
evldw r20,56(r1); \
evldw r21,64(r1); \
evldw r22,72(r1); \
evldw r23,80(r1); \
lwz r24,88(r1); /* restore normal registers */ \
lwz r25,92(r1); \
xor r0,r0,r0; \
stw r0,8(r1); /* Delete sensitive data */ \
stw r0,16(r1); /* that we might have pushed */ \
stw r0,24(r1); /* from other context that runs */ \
stw r0,32(r1); /* the same code. Assume that */ \
stw r0,40(r1); /* the lower part of the GPRs */ \
stw r0,48(r1); /* was already overwritten on */ \
stw r0,56(r1); /* the way down to here */ \
stw r0,64(r1); \
stw r0,72(r1); \
stw r0,80(r1); \
addi r1,r1,128; /* cleanup stack frame */
#ifdef __BIG_ENDIAN__
#define LOAD_DATA(reg, off) \
lwz reg,off(rWP); /* load data */
#define NEXT_BLOCK \
addi rWP,rWP,64; /* increment per block */
#else
#define LOAD_DATA(reg, off) \
lwbrx reg,0,rWP; /* load data */ \
addi rWP,rWP,4; /* increment per word */
#define NEXT_BLOCK /* nothing to do */
#endif
#define R_LOAD_W(a, b, c, d, e, f, g, h, w, off) \
LOAD_DATA(w, off) /* 1: W */ \
rotrwi rT0,e,6; /* 1: S1 = e rotr 6 */ \
rotrwi rT1,e,11; /* 1: S1' = e rotr 11 */ \
rotrwi rT2,e,25; /* 1: S1" = e rotr 25 */ \
xor rT0,rT0,rT1; /* 1: S1 = S1 xor S1' */ \
and rT3,e,f; /* 1: ch = e and f */ \
xor rT0,rT0,rT2; /* 1: S1 = S1 xor S1" */ \
andc rT1,g,e; /* 1: ch' = ~e and g */ \
lwz rT2,off(rKP); /* 1: K */ \
xor rT3,rT3,rT1; /* 1: ch = ch xor ch' */ \
add h,h,rT0; /* 1: temp1 = h + S1 */ \
add rT3,rT3,w; /* 1: temp1' = ch + w */ \
rotrwi rT0,a,2; /* 1: S0 = a rotr 2 */ \
add h,h,rT3; /* 1: temp1 = temp1 + temp1' */ \
rotrwi rT1,a,13; /* 1: S0' = a rotr 13 */ \
add h,h,rT2; /* 1: temp1 = temp1 + K */ \
rotrwi rT3,a,22; /* 1: S0" = a rotr 22 */ \
xor rT0,rT0,rT1; /* 1: S0 = S0 xor S0' */ \
add d,d,h; /* 1: d = d + temp1 */ \
xor rT3,rT0,rT3; /* 1: S0 = S0 xor S0" */ \
evmergelo w,w,w; /* shift W */ \
or rT2,a,b; /* 1: maj = a or b */ \
and rT1,a,b; /* 1: maj' = a and b */ \
and rT2,rT2,c; /* 1: maj = maj and c */ \
LOAD_DATA(w, off+4) /* 2: W */ \
or rT2,rT1,rT2; /* 1: maj = maj or maj' */ \
rotrwi rT0,d,6; /* 2: S1 = e rotr 6 */ \
add rT3,rT3,rT2; /* 1: temp2 = S0 + maj */ \
rotrwi rT1,d,11; /* 2: S1' = e rotr 11 */ \
add h,h,rT3; /* 1: h = temp1 + temp2 */ \
rotrwi rT2,d,25; /* 2: S1" = e rotr 25 */ \
xor rT0,rT0,rT1; /* 2: S1 = S1 xor S1' */ \
and rT3,d,e; /* 2: ch = e and f */ \
xor rT0,rT0,rT2; /* 2: S1 = S1 xor S1" */ \
andc rT1,f,d; /* 2: ch' = ~e and g */ \
lwz rT2,off+4(rKP); /* 2: K */ \
xor rT3,rT3,rT1; /* 2: ch = ch xor ch' */ \
add g,g,rT0; /* 2: temp1 = h + S1 */ \
add rT3,rT3,w; /* 2: temp1' = ch + w */ \
rotrwi rT0,h,2; /* 2: S0 = a rotr 2 */ \
add g,g,rT3; /* 2: temp1 = temp1 + temp1' */ \
rotrwi rT1,h,13; /* 2: S0' = a rotr 13 */ \
add g,g,rT2; /* 2: temp1 = temp1 + K */ \
rotrwi rT3,h,22; /* 2: S0" = a rotr 22 */ \
xor rT0,rT0,rT1; /* 2: S0 = S0 xor S0' */ \
or rT2,h,a; /* 2: maj = a or b */ \
xor rT3,rT0,rT3; /* 2: S0 = S0 xor S0" */ \
and rT1,h,a; /* 2: maj' = a and b */ \
and rT2,rT2,b; /* 2: maj = maj and c */ \
add c,c,g; /* 2: d = d + temp1 */ \
or rT2,rT1,rT2; /* 2: maj = maj or maj' */ \
add rT3,rT3,rT2; /* 2: temp2 = S0 + maj */ \
add g,g,rT3 /* 2: h = temp1 + temp2 */
#define R_CALC_W(a, b, c, d, e, f, g, h, w0, w1, w4, w5, w7, k, off) \
rotrwi rT2,e,6; /* 1: S1 = e rotr 6 */ \
evmergelohi rT0,w0,w1; /* w[-15] */ \
rotrwi rT3,e,11; /* 1: S1' = e rotr 11 */ \
evsrwiu rT1,rT0,3; /* s0 = w[-15] >> 3 */ \
xor rT2,rT2,rT3; /* 1: S1 = S1 xor S1' */ \
evrlwi rT0,rT0,25; /* s0' = w[-15] rotr 7 */ \
rotrwi rT3,e,25; /* 1: S1' = e rotr 25 */ \
evxor rT1,rT1,rT0; /* s0 = s0 xor s0' */ \
xor rT2,rT2,rT3; /* 1: S1 = S1 xor S1' */ \
evrlwi rT0,rT0,21; /* s0' = w[-15] rotr 18 */ \
add h,h,rT2; /* 1: temp1 = h + S1 */ \
evxor rT0,rT0,rT1; /* s0 = s0 xor s0' */ \
and rT2,e,f; /* 1: ch = e and f */ \
evaddw w0,w0,rT0; /* w = w[-16] + s0 */ \
andc rT3,g,e; /* 1: ch' = ~e and g */ \
evsrwiu rT0,w7,10; /* s1 = w[-2] >> 10 */ \
xor rT2,rT2,rT3; /* 1: ch = ch xor ch' */ \
evrlwi rT1,w7,15; /* s1' = w[-2] rotr 17 */ \
add h,h,rT2; /* 1: temp1 = temp1 + ch */ \
evxor rT0,rT0,rT1; /* s1 = s1 xor s1' */ \
rotrwi rT2,a,2; /* 1: S0 = a rotr 2 */ \
evrlwi rT1,w7,13; /* s1' = w[-2] rotr 19 */ \
rotrwi rT3,a,13; /* 1: S0' = a rotr 13 */ \
evxor rT0,rT0,rT1; /* s1 = s1 xor s1' */ \
xor rT2,rT2,rT3; /* 1: S0 = S0 xor S0' */ \
evldw rT1,off(rKP); /* k */ \
rotrwi rT3,a,22; /* 1: S0' = a rotr 22 */ \
evaddw w0,w0,rT0; /* w = w + s1 */ \
xor rT2,rT2,rT3; /* 1: S0 = S0 xor S0' */ \
evmergelohi rT0,w4,w5; /* w[-7] */ \
and rT3,a,b; /* 1: maj = a and b */ \
evaddw w0,w0,rT0; /* w = w + w[-7] */ \
CMP_K##k##_LOOP \
add rT2,rT2,rT3; /* 1: temp2 = S0 + maj */ \
evaddw rT1,rT1,w0; /* wk = w + k */ \
xor rT3,a,b; /* 1: maj = a xor b */ \
evmergehi rT0,rT1,rT1; /* wk1/wk2 */ \
and rT3,rT3,c; /* 1: maj = maj and c */ \
add h,h,rT0; /* 1: temp1 = temp1 + wk */ \
add rT2,rT2,rT3; /* 1: temp2 = temp2 + maj */ \
add g,g,rT1; /* 2: temp1 = temp1 + wk */ \
add d,d,h; /* 1: d = d + temp1 */ \
rotrwi rT0,d,6; /* 2: S1 = e rotr 6 */ \
add h,h,rT2; /* 1: h = temp1 + temp2 */ \
rotrwi rT1,d,11; /* 2: S1' = e rotr 11 */ \
rotrwi rT2,d,25; /* 2: S" = e rotr 25 */ \
xor rT0,rT0,rT1; /* 2: S1 = S1 xor S1' */ \
and rT3,d,e; /* 2: ch = e and f */ \
xor rT0,rT0,rT2; /* 2: S1 = S1 xor S1" */ \
andc rT1,f,d; /* 2: ch' = ~e and g */ \
add g,g,rT0; /* 2: temp1 = h + S1 */ \
xor rT3,rT3,rT1; /* 2: ch = ch xor ch' */ \
rotrwi rT0,h,2; /* 2: S0 = a rotr 2 */ \
add g,g,rT3; /* 2: temp1 = temp1 + ch */ \
rotrwi rT1,h,13; /* 2: S0' = a rotr 13 */ \
rotrwi rT3,h,22; /* 2: S0" = a rotr 22 */ \
xor rT0,rT0,rT1; /* 2: S0 = S0 xor S0' */ \
or rT2,h,a; /* 2: maj = a or b */ \
and rT1,h,a; /* 2: maj' = a and b */ \
and rT2,rT2,b; /* 2: maj = maj and c */ \
xor rT3,rT0,rT3; /* 2: S0 = S0 xor S0" */ \
or rT2,rT1,rT2; /* 2: maj = maj or maj' */ \
add c,c,g; /* 2: d = d + temp1 */ \
add rT3,rT3,rT2; /* 2: temp2 = S0 + maj */ \
add g,g,rT3 /* 2: h = temp1 + temp2 */
_GLOBAL(ppc_spe_sha256_transform)
INITIALIZE
mtctr r5
lwz rH0,0(rHP)
lwz rH1,4(rHP)
lwz rH2,8(rHP)
lwz rH3,12(rHP)
lwz rH4,16(rHP)
lwz rH5,20(rHP)
lwz rH6,24(rHP)
lwz rH7,28(rHP)
ppc_spe_sha256_main:
lis rKP,PPC_SPE_SHA256_K@ha
addi rKP,rKP,PPC_SPE_SHA256_K@l
R_LOAD_W(rH0, rH1, rH2, rH3, rH4, rH5, rH6, rH7, rW0, 0)
R_LOAD_W(rH6, rH7, rH0, rH1, rH2, rH3, rH4, rH5, rW1, 8)
R_LOAD_W(rH4, rH5, rH6, rH7, rH0, rH1, rH2, rH3, rW2, 16)
R_LOAD_W(rH2, rH3, rH4, rH5, rH6, rH7, rH0, rH1, rW3, 24)
R_LOAD_W(rH0, rH1, rH2, rH3, rH4, rH5, rH6, rH7, rW4, 32)
R_LOAD_W(rH6, rH7, rH0, rH1, rH2, rH3, rH4, rH5, rW5, 40)
R_LOAD_W(rH4, rH5, rH6, rH7, rH0, rH1, rH2, rH3, rW6, 48)
R_LOAD_W(rH2, rH3, rH4, rH5, rH6, rH7, rH0, rH1, rW7, 56)
ppc_spe_sha256_16_rounds:
addi rKP,rKP,64
R_CALC_W(rH0, rH1, rH2, rH3, rH4, rH5, rH6, rH7,
rW0, rW1, rW4, rW5, rW7, N, 0)
R_CALC_W(rH6, rH7, rH0, rH1, rH2, rH3, rH4, rH5,
rW1, rW2, rW5, rW6, rW0, N, 8)
R_CALC_W(rH4, rH5, rH6, rH7, rH0, rH1, rH2, rH3,
rW2, rW3, rW6, rW7, rW1, N, 16)
R_CALC_W(rH2, rH3, rH4, rH5, rH6, rH7, rH0, rH1,
rW3, rW4, rW7, rW0, rW2, N, 24)
R_CALC_W(rH0, rH1, rH2, rH3, rH4, rH5, rH6, rH7,
rW4, rW5, rW0, rW1, rW3, N, 32)
R_CALC_W(rH6, rH7, rH0, rH1, rH2, rH3, rH4, rH5,
rW5, rW6, rW1, rW2, rW4, N, 40)
R_CALC_W(rH4, rH5, rH6, rH7, rH0, rH1, rH2, rH3,
rW6, rW7, rW2, rW3, rW5, N, 48)
R_CALC_W(rH2, rH3, rH4, rH5, rH6, rH7, rH0, rH1,
rW7, rW0, rW3, rW4, rW6, C, 56)
bt gt,ppc_spe_sha256_16_rounds
lwz rW0,0(rHP)
NEXT_BLOCK
lwz rW1,4(rHP)
lwz rW2,8(rHP)
lwz rW3,12(rHP)
lwz rW4,16(rHP)
lwz rW5,20(rHP)
lwz rW6,24(rHP)
lwz rW7,28(rHP)
add rH0,rH0,rW0
stw rH0,0(rHP)
add rH1,rH1,rW1
stw rH1,4(rHP)
add rH2,rH2,rW2
stw rH2,8(rHP)
add rH3,rH3,rW3
stw rH3,12(rHP)
add rH4,rH4,rW4
stw rH4,16(rHP)
add rH5,rH5,rW5
stw rH5,20(rHP)
add rH6,rH6,rW6
stw rH6,24(rHP)
add rH7,rH7,rW7
stw rH7,28(rHP)
bdnz ppc_spe_sha256_main
FINALIZE
blr
.data
.align 5
PPC_SPE_SHA256_K:
.long 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5
.long 0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5
.long 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3
.long 0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174
.long 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc
.long 0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da
.long 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7
.long 0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967
.long 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13
.long 0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85
.long 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3
.long 0xd192e819,0xd6990624,0xf40e3585,0x106aa070
.long 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5
.long 0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3
.long 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208
.long 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2
linux-7.2.3/lib/crypto/powerpc/sha256.h
+58
View File
@@ -0,0 +1,58 @@
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
* SHA-256 Secure Hash Algorithm, SPE optimized
*
* Based on generic implementation. The assembler module takes care
* about the SPE registers so it can run from interrupt context.
*
* Copyright (c) 2015 Markus Stockhausen <stockhausen@collogia.de>
*/
#include <asm/switch_to.h>
#include <linux/preempt.h>
/*
* MAX_BYTES defines the number of bytes that are allowed to be processed
* between preempt_disable() and preempt_enable(). SHA256 takes ~2,000
* operations per 64 bytes. e500 cores can issue two arithmetic instructions
* per clock cycle using one 32/64 bit unit (SU1) and one 32 bit unit (SU2).
* Thus 1KB of input data will need an estimated maximum of 18,000 cycles.
* Headroom for cache misses included. Even with the low end model clocked
* at 667 MHz this equals to a critical time window of less than 27us.
*
*/
#define MAX_BYTES 1024
extern void ppc_spe_sha256_transform(struct sha256_block_state *state,
const u8 *src, u32 blocks);
static void spe_begin(void)
{
/* We just start SPE operations and will save SPE registers later. */
preempt_disable();
enable_kernel_spe();
}
static void spe_end(void)
{
disable_kernel_spe();
/* reenable preemption */
preempt_enable();
}
static void sha256_blocks(struct sha256_block_state *state,
const u8 *data, size_t nblocks)
{
do {
/* cut input data into smaller blocks */
u32 unit = min_t(size_t, nblocks,
MAX_BYTES / SHA256_BLOCK_SIZE);
spe_begin();
ppc_spe_sha256_transform(state, data, unit);
spe_end();
data += unit * SHA256_BLOCK_SIZE;
nblocks -= unit;
} while (nblocks);
}