lastdragon/linux-kernel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Resguardo inicial

Browse Source
This commit is contained in:
lastdragon
2026-09-06 12:53:21 -06:00
parent 662efe3c2a
commit ad2088d178
94853 changed files with 43200404 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
linux-7.2.3/lib/raid/xor/Makefile
+43
View File
@@ -0,0 +1,43 @@
# SPDX-License-Identifier: GPL-2.0
ccflags-y += -I $(src)
obj-$(CONFIG_XOR_BLOCKS) += xor.o
xor-y += xor-core.o
xor-y += xor-8regs.o
xor-y += xor-32regs.o
xor-y += xor-8regs-prefetch.o
xor-y += xor-32regs-prefetch.o
ifeq ($(CONFIG_XOR_BLOCKS_ARCH),y)
CFLAGS_xor-core.o += -I$(src)/$(SRCARCH)
endif
xor-$(CONFIG_ALPHA) += alpha/xor.o
xor-$(CONFIG_ARM) += arm/xor.o
ifeq ($(CONFIG_ARM),y)
xor-$(CONFIG_KERNEL_MODE_NEON) += xor-neon.o arm/xor-neon-glue.o
endif
xor-$(CONFIG_ARM64) += xor-neon.o arm64/xor-eor3.o \
arm64/xor-neon-glue.o
xor-$(CONFIG_CPU_HAS_LSX) += loongarch/xor_simd.o
xor-$(CONFIG_CPU_HAS_LSX) += loongarch/xor_simd_glue.o
xor-$(CONFIG_ALTIVEC) += powerpc/xor_vmx.o powerpc/xor_vmx_glue.o
xor-$(CONFIG_RISCV_ISA_V) += riscv/xor.o riscv/xor-glue.o
xor-$(CONFIG_SPARC32) += sparc/xor-sparc32.o
xor-$(CONFIG_SPARC64) += sparc/xor-sparc64.o sparc/xor-sparc64-glue.o
xor-$(CONFIG_S390) += s390/xor.o
xor-$(CONFIG_X86_32) += x86/xor-avx.o x86/xor-sse.o x86/xor-mmx.o
xor-$(CONFIG_X86_64) += x86/xor-avx.o x86/xor-sse.o
obj-y += tests/
CFLAGS_xor-neon.o += $(CC_FLAGS_FPU) -I$(src)/$(SRCARCH)
CFLAGS_REMOVE_xor-neon.o += $(CC_FLAGS_NO_FPU)
CFLAGS_arm64/xor-eor3.o += $(CC_FLAGS_FPU)
CFLAGS_REMOVE_arm64/xor-eor3.o += $(CC_FLAGS_NO_FPU)
CFLAGS_powerpc/xor_vmx.o += -mhard-float -maltivec \
$(call cc-option,-mabi=altivec) \
-isystem $(shell $(CC) -print-file-name=include)
linux-7.2.3/lib/raid/xor/alpha/xor.c
+848
View File
@@ -0,0 +1,848 @@
// SPDX-License-Identifier: GPL-2.0-or-later
/*
* Optimized XOR parity functions for alpha EV5 and EV6
*/
#include "xor_impl.h"
#include "xor_arch.h"
extern void
xor_alpha_2(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2);
extern void
xor_alpha_3(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2,
const unsigned long * __restrict p3);
extern void
xor_alpha_4(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2,
const unsigned long * __restrict p3,
const unsigned long * __restrict p4);
extern void
xor_alpha_5(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2,
const unsigned long * __restrict p3,
const unsigned long * __restrict p4,
const unsigned long * __restrict p5);
extern void
xor_alpha_prefetch_2(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2);
extern void
xor_alpha_prefetch_3(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2,
const unsigned long * __restrict p3);
extern void
xor_alpha_prefetch_4(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2,
const unsigned long * __restrict p3,
const unsigned long * __restrict p4);
extern void
xor_alpha_prefetch_5(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2,
const unsigned long * __restrict p3,
const unsigned long * __restrict p4,
const unsigned long * __restrict p5);
asm(" \n\
.text \n\
.align 3 \n\
.ent xor_alpha_2 \n\
xor_alpha_2: \n\
.prologue 0 \n\
srl $16, 6, $16 \n\
.align 4 \n\
2: \n\
ldq $0,0($17) \n\
ldq $1,0($18) \n\
ldq $2,8($17) \n\
ldq $3,8($18) \n\
\n\
ldq $4,16($17) \n\
ldq $5,16($18) \n\
ldq $6,24($17) \n\
ldq $7,24($18) \n\
\n\
ldq $19,32($17) \n\
ldq $20,32($18) \n\
ldq $21,40($17) \n\
ldq $22,40($18) \n\
\n\
ldq $23,48($17) \n\
ldq $24,48($18) \n\
ldq $25,56($17) \n\
xor $0,$1,$0 # 7 cycles from $1 load \n\
\n\
ldq $27,56($18) \n\
xor $2,$3,$2 \n\
stq $0,0($17) \n\
xor $4,$5,$4 \n\
\n\
stq $2,8($17) \n\
xor $6,$7,$6 \n\
stq $4,16($17) \n\
xor $19,$20,$19 \n\
\n\
stq $6,24($17) \n\
xor $21,$22,$21 \n\
stq $19,32($17) \n\
xor $23,$24,$23 \n\
\n\
stq $21,40($17) \n\
xor $25,$27,$25 \n\
stq $23,48($17) \n\
subq $16,1,$16 \n\
\n\
stq $25,56($17) \n\
addq $17,64,$17 \n\
addq $18,64,$18 \n\
bgt $16,2b \n\
\n\
ret \n\
.end xor_alpha_2 \n\
\n\
.align 3 \n\
.ent xor_alpha_3 \n\
xor_alpha_3: \n\
.prologue 0 \n\
srl $16, 6, $16 \n\
.align 4 \n\
3: \n\
ldq $0,0($17) \n\
ldq $1,0($18) \n\
ldq $2,0($19) \n\
ldq $3,8($17) \n\
\n\
ldq $4,8($18) \n\
ldq $6,16($17) \n\
ldq $7,16($18) \n\
ldq $21,24($17) \n\
\n\
ldq $22,24($18) \n\
ldq $24,32($17) \n\
ldq $25,32($18) \n\
ldq $5,8($19) \n\
\n\
ldq $20,16($19) \n\
ldq $23,24($19) \n\
ldq $27,32($19) \n\
nop \n\
\n\
xor $0,$1,$1 # 8 cycles from $0 load \n\
xor $3,$4,$4 # 6 cycles from $4 load \n\
xor $6,$7,$7 # 6 cycles from $7 load \n\
xor $21,$22,$22 # 5 cycles from $22 load \n\
\n\
xor $1,$2,$2 # 9 cycles from $2 load \n\
xor $24,$25,$25 # 5 cycles from $25 load \n\
stq $2,0($17) \n\
xor $4,$5,$5 # 6 cycles from $5 load \n\
\n\
stq $5,8($17) \n\
xor $7,$20,$20 # 7 cycles from $20 load \n\
stq $20,16($17) \n\
xor $22,$23,$23 # 7 cycles from $23 load \n\
\n\
stq $23,24($17) \n\
xor $25,$27,$27 # 7 cycles from $27 load \n\
stq $27,32($17) \n\
nop \n\
\n\
ldq $0,40($17) \n\
ldq $1,40($18) \n\
ldq $3,48($17) \n\
ldq $4,48($18) \n\
\n\
ldq $6,56($17) \n\
ldq $7,56($18) \n\
ldq $2,40($19) \n\
ldq $5,48($19) \n\
\n\
ldq $20,56($19) \n\
xor $0,$1,$1 # 4 cycles from $1 load \n\
xor $3,$4,$4 # 5 cycles from $4 load \n\
xor $6,$7,$7 # 5 cycles from $7 load \n\
\n\
xor $1,$2,$2 # 4 cycles from $2 load \n\
xor $4,$5,$5 # 5 cycles from $5 load \n\
stq $2,40($17) \n\
xor $7,$20,$20 # 4 cycles from $20 load \n\
\n\
stq $5,48($17) \n\
subq $16,1,$16 \n\
stq $20,56($17) \n\
addq $19,64,$19 \n\
\n\
addq $18,64,$18 \n\
addq $17,64,$17 \n\
bgt $16,3b \n\
ret \n\
.end xor_alpha_3 \n\
\n\
.align 3 \n\
.ent xor_alpha_4 \n\
xor_alpha_4: \n\
.prologue 0 \n\
srl $16, 6, $16 \n\
.align 4 \n\
4: \n\
ldq $0,0($17) \n\
ldq $1,0($18) \n\
ldq $2,0($19) \n\
ldq $3,0($20) \n\
\n\
ldq $4,8($17) \n\
ldq $5,8($18) \n\
ldq $6,8($19) \n\
ldq $7,8($20) \n\
\n\
ldq $21,16($17) \n\
ldq $22,16($18) \n\
ldq $23,16($19) \n\
ldq $24,16($20) \n\
\n\
ldq $25,24($17) \n\
xor $0,$1,$1 # 6 cycles from $1 load \n\
ldq $27,24($18) \n\
xor $2,$3,$3 # 6 cycles from $3 load \n\
\n\
ldq $0,24($19) \n\
xor $1,$3,$3 \n\
ldq $1,24($20) \n\
xor $4,$5,$5 # 7 cycles from $5 load \n\
\n\
stq $3,0($17) \n\
xor $6,$7,$7 \n\
xor $21,$22,$22 # 7 cycles from $22 load \n\
xor $5,$7,$7 \n\
\n\
stq $7,8($17) \n\
xor $23,$24,$24 # 7 cycles from $24 load \n\
ldq $2,32($17) \n\
xor $22,$24,$24 \n\
\n\
ldq $3,32($18) \n\
ldq $4,32($19) \n\
ldq $5,32($20) \n\
xor $25,$27,$27 # 8 cycles from $27 load \n\
\n\
ldq $6,40($17) \n\
ldq $7,40($18) \n\
ldq $21,40($19) \n\
ldq $22,40($20) \n\
\n\
stq $24,16($17) \n\
xor $0,$1,$1 # 9 cycles from $1 load \n\
xor $2,$3,$3 # 5 cycles from $3 load \n\
xor $27,$1,$1 \n\
\n\
stq $1,24($17) \n\
xor $4,$5,$5 # 5 cycles from $5 load \n\
ldq $23,48($17) \n\
ldq $24,48($18) \n\
\n\
ldq $25,48($19) \n\
xor $3,$5,$5 \n\
ldq $27,48($20) \n\
ldq $0,56($17) \n\
\n\
ldq $1,56($18) \n\
ldq $2,56($19) \n\
xor $6,$7,$7 # 8 cycles from $6 load \n\
ldq $3,56($20) \n\
\n\
stq $5,32($17) \n\
xor $21,$22,$22 # 8 cycles from $22 load \n\
xor $7,$22,$22 \n\
xor $23,$24,$24 # 5 cycles from $24 load \n\
\n\
stq $22,40($17) \n\
xor $25,$27,$27 # 5 cycles from $27 load \n\
xor $24,$27,$27 \n\
xor $0,$1,$1 # 5 cycles from $1 load \n\
\n\
stq $27,48($17) \n\
xor $2,$3,$3 # 4 cycles from $3 load \n\
xor $1,$3,$3 \n\
subq $16,1,$16 \n\
\n\
stq $3,56($17) \n\
addq $20,64,$20 \n\
addq $19,64,$19 \n\
addq $18,64,$18 \n\
\n\
addq $17,64,$17 \n\
bgt $16,4b \n\
ret \n\
.end xor_alpha_4 \n\
\n\
.align 3 \n\
.ent xor_alpha_5 \n\
xor_alpha_5: \n\
.prologue 0 \n\
srl $16, 6, $16 \n\
.align 4 \n\
5: \n\
ldq $0,0($17) \n\
ldq $1,0($18) \n\
ldq $2,0($19) \n\
ldq $3,0($20) \n\
\n\
ldq $4,0($21) \n\
ldq $5,8($17) \n\
ldq $6,8($18) \n\
ldq $7,8($19) \n\
\n\
ldq $22,8($20) \n\
ldq $23,8($21) \n\
ldq $24,16($17) \n\
ldq $25,16($18) \n\
\n\
ldq $27,16($19) \n\
xor $0,$1,$1 # 6 cycles from $1 load \n\
ldq $28,16($20) \n\
xor $2,$3,$3 # 6 cycles from $3 load \n\
\n\
ldq $0,16($21) \n\
xor $1,$3,$3 \n\
ldq $1,24($17) \n\
xor $3,$4,$4 # 7 cycles from $4 load \n\
\n\
stq $4,0($17) \n\
xor $5,$6,$6 # 7 cycles from $6 load \n\
xor $7,$22,$22 # 7 cycles from $22 load \n\
xor $6,$23,$23 # 7 cycles from $23 load \n\
\n\
ldq $2,24($18) \n\
xor $22,$23,$23 \n\
ldq $3,24($19) \n\
xor $24,$25,$25 # 8 cycles from $25 load \n\
\n\
stq $23,8($17) \n\
xor $25,$27,$27 # 8 cycles from $27 load \n\
ldq $4,24($20) \n\
xor $28,$0,$0 # 7 cycles from $0 load \n\
\n\
ldq $5,24($21) \n\
xor $27,$0,$0 \n\
ldq $6,32($17) \n\
ldq $7,32($18) \n\
\n\
stq $0,16($17) \n\
xor $1,$2,$2 # 6 cycles from $2 load \n\
ldq $22,32($19) \n\
xor $3,$4,$4 # 4 cycles from $4 load \n\
\n\
ldq $23,32($20) \n\
xor $2,$4,$4 \n\
ldq $24,32($21) \n\
ldq $25,40($17) \n\
\n\
ldq $27,40($18) \n\
ldq $28,40($19) \n\
ldq $0,40($20) \n\
xor $4,$5,$5 # 7 cycles from $5 load \n\
\n\
stq $5,24($17) \n\
xor $6,$7,$7 # 7 cycles from $7 load \n\
ldq $1,40($21) \n\
ldq $2,48($17) \n\
\n\
ldq $3,48($18) \n\
xor $7,$22,$22 # 7 cycles from $22 load \n\
ldq $4,48($19) \n\
xor $23,$24,$24 # 6 cycles from $24 load \n\
\n\
ldq $5,48($20) \n\
xor $22,$24,$24 \n\
ldq $6,48($21) \n\
xor $25,$27,$27 # 7 cycles from $27 load \n\
\n\
stq $24,32($17) \n\
xor $27,$28,$28 # 8 cycles from $28 load \n\
ldq $7,56($17) \n\
xor $0,$1,$1 # 6 cycles from $1 load \n\
\n\
ldq $22,56($18) \n\
ldq $23,56($19) \n\
ldq $24,56($20) \n\
ldq $25,56($21) \n\
\n\
xor $28,$1,$1 \n\
xor $2,$3,$3 # 9 cycles from $3 load \n\
xor $3,$4,$4 # 9 cycles from $4 load \n\
xor $5,$6,$6 # 8 cycles from $6 load \n\
\n\
stq $1,40($17) \n\
xor $4,$6,$6 \n\
xor $7,$22,$22 # 7 cycles from $22 load \n\
xor $23,$24,$24 # 6 cycles from $24 load \n\
\n\
stq $6,48($17) \n\
xor $22,$24,$24 \n\
subq $16,1,$16 \n\
xor $24,$25,$25 # 8 cycles from $25 load \n\
\n\
stq $25,56($17) \n\
addq $21,64,$21 \n\
addq $20,64,$20 \n\
addq $19,64,$19 \n\
\n\
addq $18,64,$18 \n\
addq $17,64,$17 \n\
bgt $16,5b \n\
ret \n\
.end xor_alpha_5 \n\
\n\
.align 3 \n\
.ent xor_alpha_prefetch_2 \n\
xor_alpha_prefetch_2: \n\
.prologue 0 \n\
srl $16, 6, $16 \n\
\n\
ldq $31, 0($17) \n\
ldq $31, 0($18) \n\
\n\
ldq $31, 64($17) \n\
ldq $31, 64($18) \n\
\n\
ldq $31, 128($17) \n\
ldq $31, 128($18) \n\
\n\
ldq $31, 192($17) \n\
ldq $31, 192($18) \n\
.align 4 \n\
2: \n\
ldq $0,0($17) \n\
ldq $1,0($18) \n\
ldq $2,8($17) \n\
ldq $3,8($18) \n\
\n\
ldq $4,16($17) \n\
ldq $5,16($18) \n\
ldq $6,24($17) \n\
ldq $7,24($18) \n\
\n\
ldq $19,32($17) \n\
ldq $20,32($18) \n\
ldq $21,40($17) \n\
ldq $22,40($18) \n\
\n\
ldq $23,48($17) \n\
ldq $24,48($18) \n\
ldq $25,56($17) \n\
ldq $27,56($18) \n\
\n\
ldq $31,256($17) \n\
xor $0,$1,$0 # 8 cycles from $1 load \n\
ldq $31,256($18) \n\
xor $2,$3,$2 \n\
\n\
stq $0,0($17) \n\
xor $4,$5,$4 \n\
stq $2,8($17) \n\
xor $6,$7,$6 \n\
\n\
stq $4,16($17) \n\
xor $19,$20,$19 \n\
stq $6,24($17) \n\
xor $21,$22,$21 \n\
\n\
stq $19,32($17) \n\
xor $23,$24,$23 \n\
stq $21,40($17) \n\
xor $25,$27,$25 \n\
\n\
stq $23,48($17) \n\
subq $16,1,$16 \n\
stq $25,56($17) \n\
addq $17,64,$17 \n\
\n\
addq $18,64,$18 \n\
bgt $16,2b \n\
ret \n\
.end xor_alpha_prefetch_2 \n\
\n\
.align 3 \n\
.ent xor_alpha_prefetch_3 \n\
xor_alpha_prefetch_3: \n\
.prologue 0 \n\
srl $16, 6, $16 \n\
\n\
ldq $31, 0($17) \n\
ldq $31, 0($18) \n\
ldq $31, 0($19) \n\
\n\
ldq $31, 64($17) \n\
ldq $31, 64($18) \n\
ldq $31, 64($19) \n\
\n\
ldq $31, 128($17) \n\
ldq $31, 128($18) \n\
ldq $31, 128($19) \n\
\n\
ldq $31, 192($17) \n\
ldq $31, 192($18) \n\
ldq $31, 192($19) \n\
.align 4 \n\
3: \n\
ldq $0,0($17) \n\
ldq $1,0($18) \n\
ldq $2,0($19) \n\
ldq $3,8($17) \n\
\n\
ldq $4,8($18) \n\
ldq $6,16($17) \n\
ldq $7,16($18) \n\
ldq $21,24($17) \n\
\n\
ldq $22,24($18) \n\
ldq $24,32($17) \n\
ldq $25,32($18) \n\
ldq $5,8($19) \n\
\n\
ldq $20,16($19) \n\
ldq $23,24($19) \n\
ldq $27,32($19) \n\
nop \n\
\n\
xor $0,$1,$1 # 8 cycles from $0 load \n\
xor $3,$4,$4 # 7 cycles from $4 load \n\
xor $6,$7,$7 # 6 cycles from $7 load \n\
xor $21,$22,$22 # 5 cycles from $22 load \n\
\n\
xor $1,$2,$2 # 9 cycles from $2 load \n\
xor $24,$25,$25 # 5 cycles from $25 load \n\
stq $2,0($17) \n\
xor $4,$5,$5 # 6 cycles from $5 load \n\
\n\
stq $5,8($17) \n\
xor $7,$20,$20 # 7 cycles from $20 load \n\
stq $20,16($17) \n\
xor $22,$23,$23 # 7 cycles from $23 load \n\
\n\
stq $23,24($17) \n\
xor $25,$27,$27 # 7 cycles from $27 load \n\
stq $27,32($17) \n\
nop \n\
\n\
ldq $0,40($17) \n\
ldq $1,40($18) \n\
ldq $3,48($17) \n\
ldq $4,48($18) \n\
\n\
ldq $6,56($17) \n\
ldq $7,56($18) \n\
ldq $2,40($19) \n\
ldq $5,48($19) \n\
\n\
ldq $20,56($19) \n\
ldq $31,256($17) \n\
ldq $31,256($18) \n\
ldq $31,256($19) \n\
\n\
xor $0,$1,$1 # 6 cycles from $1 load \n\
xor $3,$4,$4 # 5 cycles from $4 load \n\
xor $6,$7,$7 # 5 cycles from $7 load \n\
xor $1,$2,$2 # 4 cycles from $2 load \n\
\n\
xor $4,$5,$5 # 5 cycles from $5 load \n\
xor $7,$20,$20 # 4 cycles from $20 load \n\
stq $2,40($17) \n\
subq $16,1,$16 \n\
\n\
stq $5,48($17) \n\
addq $19,64,$19 \n\
stq $20,56($17) \n\
addq $18,64,$18 \n\
\n\
addq $17,64,$17 \n\
bgt $16,3b \n\
ret \n\
.end xor_alpha_prefetch_3 \n\
\n\
.align 3 \n\
.ent xor_alpha_prefetch_4 \n\
xor_alpha_prefetch_4: \n\
.prologue 0 \n\
srl $16, 6, $16 \n\
\n\
ldq $31, 0($17) \n\
ldq $31, 0($18) \n\
ldq $31, 0($19) \n\
ldq $31, 0($20) \n\
\n\
ldq $31, 64($17) \n\
ldq $31, 64($18) \n\
ldq $31, 64($19) \n\
ldq $31, 64($20) \n\
\n\
ldq $31, 128($17) \n\
ldq $31, 128($18) \n\
ldq $31, 128($19) \n\
ldq $31, 128($20) \n\
\n\
ldq $31, 192($17) \n\
ldq $31, 192($18) \n\
ldq $31, 192($19) \n\
ldq $31, 192($20) \n\
.align 4 \n\
4: \n\
ldq $0,0($17) \n\
ldq $1,0($18) \n\
ldq $2,0($19) \n\
ldq $3,0($20) \n\
\n\
ldq $4,8($17) \n\
ldq $5,8($18) \n\
ldq $6,8($19) \n\
ldq $7,8($20) \n\
\n\
ldq $21,16($17) \n\
ldq $22,16($18) \n\
ldq $23,16($19) \n\
ldq $24,16($20) \n\
\n\
ldq $25,24($17) \n\
xor $0,$1,$1 # 6 cycles from $1 load \n\
ldq $27,24($18) \n\
xor $2,$3,$3 # 6 cycles from $3 load \n\
\n\
ldq $0,24($19) \n\
xor $1,$3,$3 \n\
ldq $1,24($20) \n\
xor $4,$5,$5 # 7 cycles from $5 load \n\
\n\
stq $3,0($17) \n\
xor $6,$7,$7 \n\
xor $21,$22,$22 # 7 cycles from $22 load \n\
xor $5,$7,$7 \n\
\n\
stq $7,8($17) \n\
xor $23,$24,$24 # 7 cycles from $24 load \n\
ldq $2,32($17) \n\
xor $22,$24,$24 \n\
\n\
ldq $3,32($18) \n\
ldq $4,32($19) \n\
ldq $5,32($20) \n\
xor $25,$27,$27 # 8 cycles from $27 load \n\
\n\
ldq $6,40($17) \n\
ldq $7,40($18) \n\
ldq $21,40($19) \n\
ldq $22,40($20) \n\
\n\
stq $24,16($17) \n\
xor $0,$1,$1 # 9 cycles from $1 load \n\
xor $2,$3,$3 # 5 cycles from $3 load \n\
xor $27,$1,$1 \n\
\n\
stq $1,24($17) \n\
xor $4,$5,$5 # 5 cycles from $5 load \n\
ldq $23,48($17) \n\
xor $3,$5,$5 \n\
\n\
ldq $24,48($18) \n\
ldq $25,48($19) \n\
ldq $27,48($20) \n\
ldq $0,56($17) \n\
\n\
ldq $1,56($18) \n\
ldq $2,56($19) \n\
ldq $3,56($20) \n\
xor $6,$7,$7 # 8 cycles from $6 load \n\
\n\
ldq $31,256($17) \n\
xor $21,$22,$22 # 8 cycles from $22 load \n\
ldq $31,256($18) \n\
xor $7,$22,$22 \n\
\n\
ldq $31,256($19) \n\
xor $23,$24,$24 # 6 cycles from $24 load \n\
ldq $31,256($20) \n\
xor $25,$27,$27 # 6 cycles from $27 load \n\
\n\
stq $5,32($17) \n\
xor $24,$27,$27 \n\
xor $0,$1,$1 # 7 cycles from $1 load \n\
xor $2,$3,$3 # 6 cycles from $3 load \n\
\n\
stq $22,40($17) \n\
xor $1,$3,$3 \n\
stq $27,48($17) \n\
subq $16,1,$16 \n\
\n\
stq $3,56($17) \n\
addq $20,64,$20 \n\
addq $19,64,$19 \n\
addq $18,64,$18 \n\
\n\
addq $17,64,$17 \n\
bgt $16,4b \n\
ret \n\
.end xor_alpha_prefetch_4 \n\
\n\
.align 3 \n\
.ent xor_alpha_prefetch_5 \n\
xor_alpha_prefetch_5: \n\
.prologue 0 \n\
srl $16, 6, $16 \n\
\n\
ldq $31, 0($17) \n\
ldq $31, 0($18) \n\
ldq $31, 0($19) \n\
ldq $31, 0($20) \n\
ldq $31, 0($21) \n\
\n\
ldq $31, 64($17) \n\
ldq $31, 64($18) \n\
ldq $31, 64($19) \n\
ldq $31, 64($20) \n\
ldq $31, 64($21) \n\
\n\
ldq $31, 128($17) \n\
ldq $31, 128($18) \n\
ldq $31, 128($19) \n\
ldq $31, 128($20) \n\
ldq $31, 128($21) \n\
\n\
ldq $31, 192($17) \n\
ldq $31, 192($18) \n\
ldq $31, 192($19) \n\
ldq $31, 192($20) \n\
ldq $31, 192($21) \n\
.align 4 \n\
5: \n\
ldq $0,0($17) \n\
ldq $1,0($18) \n\
ldq $2,0($19) \n\
ldq $3,0($20) \n\
\n\
ldq $4,0($21) \n\
ldq $5,8($17) \n\
ldq $6,8($18) \n\
ldq $7,8($19) \n\
\n\
ldq $22,8($20) \n\
ldq $23,8($21) \n\
ldq $24,16($17) \n\
ldq $25,16($18) \n\
\n\
ldq $27,16($19) \n\
xor $0,$1,$1 # 6 cycles from $1 load \n\
ldq $28,16($20) \n\
xor $2,$3,$3 # 6 cycles from $3 load \n\
\n\
ldq $0,16($21) \n\
xor $1,$3,$3 \n\
ldq $1,24($17) \n\
xor $3,$4,$4 # 7 cycles from $4 load \n\
\n\
stq $4,0($17) \n\
xor $5,$6,$6 # 7 cycles from $6 load \n\
xor $7,$22,$22 # 7 cycles from $22 load \n\
xor $6,$23,$23 # 7 cycles from $23 load \n\
\n\
ldq $2,24($18) \n\
xor $22,$23,$23 \n\
ldq $3,24($19) \n\
xor $24,$25,$25 # 8 cycles from $25 load \n\
\n\
stq $23,8($17) \n\
xor $25,$27,$27 # 8 cycles from $27 load \n\
ldq $4,24($20) \n\
xor $28,$0,$0 # 7 cycles from $0 load \n\
\n\
ldq $5,24($21) \n\
xor $27,$0,$0 \n\
ldq $6,32($17) \n\
ldq $7,32($18) \n\
\n\
stq $0,16($17) \n\
xor $1,$2,$2 # 6 cycles from $2 load \n\
ldq $22,32($19) \n\
xor $3,$4,$4 # 4 cycles from $4 load \n\
\n\
ldq $23,32($20) \n\
xor $2,$4,$4 \n\
ldq $24,32($21) \n\
ldq $25,40($17) \n\
\n\
ldq $27,40($18) \n\
ldq $28,40($19) \n\
ldq $0,40($20) \n\
xor $4,$5,$5 # 7 cycles from $5 load \n\
\n\
stq $5,24($17) \n\
xor $6,$7,$7 # 7 cycles from $7 load \n\
ldq $1,40($21) \n\
ldq $2,48($17) \n\
\n\
ldq $3,48($18) \n\
xor $7,$22,$22 # 7 cycles from $22 load \n\
ldq $4,48($19) \n\
xor $23,$24,$24 # 6 cycles from $24 load \n\
\n\
ldq $5,48($20) \n\
xor $22,$24,$24 \n\
ldq $6,48($21) \n\
xor $25,$27,$27 # 7 cycles from $27 load \n\
\n\
stq $24,32($17) \n\
xor $27,$28,$28 # 8 cycles from $28 load \n\
ldq $7,56($17) \n\
xor $0,$1,$1 # 6 cycles from $1 load \n\
\n\
ldq $22,56($18) \n\
ldq $23,56($19) \n\
ldq $24,56($20) \n\
ldq $25,56($21) \n\
\n\
ldq $31,256($17) \n\
xor $28,$1,$1 \n\
ldq $31,256($18) \n\
xor $2,$3,$3 # 9 cycles from $3 load \n\
\n\
ldq $31,256($19) \n\
xor $3,$4,$4 # 9 cycles from $4 load \n\
ldq $31,256($20) \n\
xor $5,$6,$6 # 8 cycles from $6 load \n\
\n\
stq $1,40($17) \n\
xor $4,$6,$6 \n\
xor $7,$22,$22 # 7 cycles from $22 load \n\
xor $23,$24,$24 # 6 cycles from $24 load \n\
\n\
stq $6,48($17) \n\
xor $22,$24,$24 \n\
ldq $31,256($21) \n\
xor $24,$25,$25 # 8 cycles from $25 load \n\
\n\
stq $25,56($17) \n\
subq $16,1,$16 \n\
addq $21,64,$21 \n\
addq $20,64,$20 \n\
\n\
addq $19,64,$19 \n\
addq $18,64,$18 \n\
addq $17,64,$17 \n\
bgt $16,5b \n\
\n\
ret \n\
.end xor_alpha_prefetch_5 \n\
");
DO_XOR_BLOCKS(alpha, xor_alpha_2, xor_alpha_3, xor_alpha_4, xor_alpha_5);
struct xor_block_template xor_block_alpha = {
.name = "alpha",
.xor_gen = xor_gen_alpha,
};
DO_XOR_BLOCKS(alpha_prefetch, xor_alpha_prefetch_2, xor_alpha_prefetch_3,
xor_alpha_prefetch_4, xor_alpha_prefetch_5);
struct xor_block_template xor_block_alpha_prefetch = {
.name = "alpha prefetch",
.xor_gen = xor_gen_alpha_prefetch,
};
linux-7.2.3/lib/raid/xor/alpha/xor_arch.h
+22
View File
@@ -0,0 +1,22 @@
/* SPDX-License-Identifier: GPL-2.0-or-later */
#include <asm/special_insns.h>
extern struct xor_block_template xor_block_alpha;
extern struct xor_block_template xor_block_alpha_prefetch;
/*
* Force the use of alpha_prefetch if EV6, as it is significantly faster in the
* cold cache case.
*/
static __always_inline void __init arch_xor_init(void)
{
if (implver() == IMPLVER_EV6) {
xor_force(&xor_block_alpha_prefetch);
} else {
xor_register(&xor_block_8regs);
xor_register(&xor_block_32regs);
xor_register(&xor_block_alpha);
xor_register(&xor_block_alpha_prefetch);
}
}
linux-7.2.3/lib/raid/xor/arm/xor-neon-glue.c
+19
View File
@@ -0,0 +1,19 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (C) 2001 Russell King
*/
#include "xor_impl.h"
#include "xor_arch.h"
static void xor_gen_neon(void *dest, void **srcs, unsigned int src_cnt,
unsigned int bytes)
{
kernel_neon_begin();
xor_gen_neon_inner(dest, srcs, src_cnt, bytes);
kernel_neon_end();
}
struct xor_block_template xor_block_neon = {
.name = "neon",
.xor_gen = xor_gen_neon,
};
linux-7.2.3/lib/raid/xor/arm/xor-neon.h
+7
View File
@@ -0,0 +1,7 @@
/* SPDX-License-Identifier: GPL-2.0-only */
extern struct xor_block_template xor_block_arm4regs;
extern struct xor_block_template xor_block_neon;
void xor_gen_neon_inner(void *dest, void **srcs, unsigned int src_cnt,
unsigned int bytes);
linux-7.2.3/lib/raid/xor/arm/xor.c
+136
View File
@@ -0,0 +1,136 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (C) 2001 Russell King
*/
#include "xor_impl.h"
#include "xor_arch.h"
#define __XOR(a1, a2) a1 ^= a2
#define GET_BLOCK_2(dst) \
__asm__("ldmia %0, {%1, %2}" \
: "=r" (dst), "=r" (a1), "=r" (a2) \
: "0" (dst))
#define GET_BLOCK_4(dst) \
__asm__("ldmia %0, {%1, %2, %3, %4}" \
: "=r" (dst), "=r" (a1), "=r" (a2), "=r" (a3), "=r" (a4) \
: "0" (dst))
#define XOR_BLOCK_2(src) \
__asm__("ldmia %0!, {%1, %2}" \
: "=r" (src), "=r" (b1), "=r" (b2) \
: "0" (src)); \
__XOR(a1, b1); __XOR(a2, b2);
#define XOR_BLOCK_4(src) \
__asm__("ldmia %0!, {%1, %2, %3, %4}" \
: "=r" (src), "=r" (b1), "=r" (b2), "=r" (b3), "=r" (b4) \
: "0" (src)); \
__XOR(a1, b1); __XOR(a2, b2); __XOR(a3, b3); __XOR(a4, b4)
#define PUT_BLOCK_2(dst) \
__asm__ __volatile__("stmia %0!, {%2, %3}" \
: "=r" (dst) \
: "0" (dst), "r" (a1), "r" (a2))
#define PUT_BLOCK_4(dst) \
__asm__ __volatile__("stmia %0!, {%2, %3, %4, %5}" \
: "=r" (dst) \
: "0" (dst), "r" (a1), "r" (a2), "r" (a3), "r" (a4))
static void
xor_arm4regs_2(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2)
{
unsigned int lines = bytes / sizeof(unsigned long) / 4;
register unsigned int a1 __asm__("r4");
register unsigned int a2 __asm__("r5");
register unsigned int a3 __asm__("r6");
register unsigned int a4 __asm__("r10");
register unsigned int b1 __asm__("r8");
register unsigned int b2 __asm__("r9");
register unsigned int b3 __asm__("ip");
register unsigned int b4 __asm__("lr");
do {
GET_BLOCK_4(p1);
XOR_BLOCK_4(p2);
PUT_BLOCK_4(p1);
} while (--lines);
}
static void
xor_arm4regs_3(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2,
const unsigned long * __restrict p3)
{
unsigned int lines = bytes / sizeof(unsigned long) / 4;
register unsigned int a1 __asm__("r4");
register unsigned int a2 __asm__("r5");
register unsigned int a3 __asm__("r6");
register unsigned int a4 __asm__("r10");
register unsigned int b1 __asm__("r8");
register unsigned int b2 __asm__("r9");
register unsigned int b3 __asm__("ip");
register unsigned int b4 __asm__("lr");
do {
GET_BLOCK_4(p1);
XOR_BLOCK_4(p2);
XOR_BLOCK_4(p3);
PUT_BLOCK_4(p1);
} while (--lines);
}
static void
xor_arm4regs_4(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2,
const unsigned long * __restrict p3,
const unsigned long * __restrict p4)
{
unsigned int lines = bytes / sizeof(unsigned long) / 2;
register unsigned int a1 __asm__("r8");
register unsigned int a2 __asm__("r9");
register unsigned int b1 __asm__("ip");
register unsigned int b2 __asm__("lr");
do {
GET_BLOCK_2(p1);
XOR_BLOCK_2(p2);
XOR_BLOCK_2(p3);
XOR_BLOCK_2(p4);
PUT_BLOCK_2(p1);
} while (--lines);
}
static void
xor_arm4regs_5(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2,
const unsigned long * __restrict p3,
const unsigned long * __restrict p4,
const unsigned long * __restrict p5)
{
unsigned int lines = bytes / sizeof(unsigned long) / 2;
register unsigned int a1 __asm__("r8");
register unsigned int a2 __asm__("r9");
register unsigned int b1 __asm__("ip");
register unsigned int b2 __asm__("lr");
do {
GET_BLOCK_2(p1);
XOR_BLOCK_2(p2);
XOR_BLOCK_2(p3);
XOR_BLOCK_2(p4);
XOR_BLOCK_2(p5);
PUT_BLOCK_2(p1);
} while (--lines);
}
DO_XOR_BLOCKS(arm4regs, xor_arm4regs_2, xor_arm4regs_3, xor_arm4regs_4,
xor_arm4regs_5);
struct xor_block_template xor_block_arm4regs = {
.name = "arm4regs",
.xor_gen = xor_gen_arm4regs,
};
linux-7.2.3/lib/raid/xor/arm/xor_arch.h
+17
View File
@@ -0,0 +1,17 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (C) 2001 Russell King
*/
#include <asm/neon.h>
#include "xor-neon.h"
static __always_inline void __init arch_xor_init(void)
{
xor_register(&xor_block_arm4regs);
xor_register(&xor_block_8regs);
xor_register(&xor_block_32regs);
#ifdef CONFIG_KERNEL_MODE_NEON
if (cpu_has_neon())
xor_register(&xor_block_neon);
#endif
}
linux-7.2.3/lib/raid/xor/arm64/xor-eor3.c
+146
View File
@@ -0,0 +1,146 @@
// SPDX-License-Identifier: GPL-2.0-only
#include <linux/cache.h>
#include <asm/neon-intrinsics.h>
#include "xor_impl.h"
#include "xor_arch.h"
#include "xor-neon.h"
extern void __xor_eor3_2(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2);
static inline uint64x2_t eor3(uint64x2_t p, uint64x2_t q, uint64x2_t r)
{
uint64x2_t res;
asm(ARM64_ASM_PREAMBLE ".arch_extension sha3\n"
"eor3 %0.16b, %1.16b, %2.16b, %3.16b"
: "=w"(res) : "w"(p), "w"(q), "w"(r));
return res;
}
static void __xor_eor3_3(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2,
const unsigned long * __restrict p3)
{
uint64_t *dp1 = (uint64_t *)p1;
uint64_t *dp2 = (uint64_t *)p2;
uint64_t *dp3 = (uint64_t *)p3;
register uint64x2_t v0, v1, v2, v3;
long lines = bytes / (sizeof(uint64x2_t) * 4);
do {
/* p1 ^= p2 ^ p3 */
v0 = eor3(vld1q_u64(dp1 + 0), vld1q_u64(dp2 + 0),
vld1q_u64(dp3 + 0));
v1 = eor3(vld1q_u64(dp1 + 2), vld1q_u64(dp2 + 2),
vld1q_u64(dp3 + 2));
v2 = eor3(vld1q_u64(dp1 + 4), vld1q_u64(dp2 + 4),
vld1q_u64(dp3 + 4));
v3 = eor3(vld1q_u64(dp1 + 6), vld1q_u64(dp2 + 6),
vld1q_u64(dp3 + 6));
/* store */
vst1q_u64(dp1 + 0, v0);
vst1q_u64(dp1 + 2, v1);
vst1q_u64(dp1 + 4, v2);
vst1q_u64(dp1 + 6, v3);
dp1 += 8;
dp2 += 8;
dp3 += 8;
} while (--lines > 0);
}
static void __xor_eor3_4(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2,
const unsigned long * __restrict p3,
const unsigned long * __restrict p4)
{
uint64_t *dp1 = (uint64_t *)p1;
uint64_t *dp2 = (uint64_t *)p2;
uint64_t *dp3 = (uint64_t *)p3;
uint64_t *dp4 = (uint64_t *)p4;
register uint64x2_t v0, v1, v2, v3;
long lines = bytes / (sizeof(uint64x2_t) * 4);
do {
/* p1 ^= p2 ^ p3 */
v0 = eor3(vld1q_u64(dp1 + 0), vld1q_u64(dp2 + 0),
vld1q_u64(dp3 + 0));
v1 = eor3(vld1q_u64(dp1 + 2), vld1q_u64(dp2 + 2),
vld1q_u64(dp3 + 2));
v2 = eor3(vld1q_u64(dp1 + 4), vld1q_u64(dp2 + 4),
vld1q_u64(dp3 + 4));
v3 = eor3(vld1q_u64(dp1 + 6), vld1q_u64(dp2 + 6),
vld1q_u64(dp3 + 6));
/* p1 ^= p4 */
v0 = veorq_u64(v0, vld1q_u64(dp4 + 0));
v1 = veorq_u64(v1, vld1q_u64(dp4 + 2));
v2 = veorq_u64(v2, vld1q_u64(dp4 + 4));
v3 = veorq_u64(v3, vld1q_u64(dp4 + 6));
/* store */
vst1q_u64(dp1 + 0, v0);
vst1q_u64(dp1 + 2, v1);
vst1q_u64(dp1 + 4, v2);
vst1q_u64(dp1 + 6, v3);
dp1 += 8;
dp2 += 8;
dp3 += 8;
dp4 += 8;
} while (--lines > 0);
}
static void __xor_eor3_5(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2,
const unsigned long * __restrict p3,
const unsigned long * __restrict p4,
const unsigned long * __restrict p5)
{
uint64_t *dp1 = (uint64_t *)p1;
uint64_t *dp2 = (uint64_t *)p2;
uint64_t *dp3 = (uint64_t *)p3;
uint64_t *dp4 = (uint64_t *)p4;
uint64_t *dp5 = (uint64_t *)p5;
register uint64x2_t v0, v1, v2, v3;
long lines = bytes / (sizeof(uint64x2_t) * 4);
do {
/* p1 ^= p2 ^ p3 */
v0 = eor3(vld1q_u64(dp1 + 0), vld1q_u64(dp2 + 0),
vld1q_u64(dp3 + 0));
v1 = eor3(vld1q_u64(dp1 + 2), vld1q_u64(dp2 + 2),
vld1q_u64(dp3 + 2));
v2 = eor3(vld1q_u64(dp1 + 4), vld1q_u64(dp2 + 4),
vld1q_u64(dp3 + 4));
v3 = eor3(vld1q_u64(dp1 + 6), vld1q_u64(dp2 + 6),
vld1q_u64(dp3 + 6));
/* p1 ^= p4 ^ p5 */
v0 = eor3(v0, vld1q_u64(dp4 + 0), vld1q_u64(dp5 + 0));
v1 = eor3(v1, vld1q_u64(dp4 + 2), vld1q_u64(dp5 + 2));
v2 = eor3(v2, vld1q_u64(dp4 + 4), vld1q_u64(dp5 + 4));
v3 = eor3(v3, vld1q_u64(dp4 + 6), vld1q_u64(dp5 + 6));
/* store */
vst1q_u64(dp1 + 0, v0);
vst1q_u64(dp1 + 2, v1);
vst1q_u64(dp1 + 4, v2);
vst1q_u64(dp1 + 6, v3);
dp1 += 8;
dp2 += 8;
dp3 += 8;
dp4 += 8;
dp5 += 8;
} while (--lines > 0);
}
__DO_XOR_BLOCKS(eor3_inner, __xor_eor3_2, __xor_eor3_3, __xor_eor3_4,
__xor_eor3_5);
linux-7.2.3/lib/raid/xor/arm64/xor-neon-glue.c
+26
View File
@@ -0,0 +1,26 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Authors: Jackie Liu <liuyun01@kylinos.cn>
* Copyright (C) 2018,Tianjin KYLIN Information Technology Co., Ltd.
*/
#include <asm/simd.h>
#include "xor_impl.h"
#include "xor_arch.h"
#include "xor-neon.h"
#define XOR_TEMPLATE(_name) \
static void xor_gen_##_name(void *dest, void **srcs, unsigned int src_cnt, \
unsigned int bytes) \
{ \
scoped_ksimd() \
xor_gen_##_name##_inner(dest, srcs, src_cnt, bytes); \
} \
\
struct xor_block_template xor_block_##_name = { \
.name = __stringify(_name), \
.xor_gen = xor_gen_##_name, \
};
XOR_TEMPLATE(neon);
XOR_TEMPLATE(eor3);
linux-7.2.3/lib/raid/xor/arm64/xor-neon.h
+6
View File
@@ -0,0 +1,6 @@
/* SPDX-License-Identifier: GPL-2.0-only */
void xor_gen_neon_inner(void *dest, void **srcs, unsigned int src_cnt,
unsigned int bytes);
void xor_gen_eor3_inner(void *dest, void **srcs, unsigned int src_cnt,
unsigned int bytes);
linux-7.2.3/lib/raid/xor/arm64/xor_arch.h
+21
View File
@@ -0,0 +1,21 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Authors: Jackie Liu <liuyun01@kylinos.cn>
* Copyright (C) 2018,Tianjin KYLIN Information Technology Co., Ltd.
*/
#include <asm/simd.h>
extern struct xor_block_template xor_block_neon;
extern struct xor_block_template xor_block_eor3;
static __always_inline void __init arch_xor_init(void)
{
xor_register(&xor_block_8regs);
xor_register(&xor_block_32regs);
if (cpu_has_neon()) {
if (cpu_have_named_feature(SHA3))
xor_register(&xor_block_eor3);
else
xor_register(&xor_block_neon);
}
}
linux-7.2.3/lib/raid/xor/loongarch/xor_arch.h
+33
View File
@@ -0,0 +1,33 @@
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
* Copyright (C) 2023 WANG Xuerui <git@xen0n.name>
*/
#include <asm/cpu-features.h>
/*
* For grins, also test the generic routines.
*
* More importantly: it cannot be ruled out at this point of time, that some
* future (maybe reduced) models could run the vector algorithms slower than
* the scalar ones, maybe for errata or micro-op reasons. It may be
* appropriate to revisit this after one or two more uarch generations.
*/
extern struct xor_block_template xor_block_lsx;
extern struct xor_block_template xor_block_lasx;
static __always_inline void __init arch_xor_init(void)
{
xor_register(&xor_block_8regs);
xor_register(&xor_block_8regs_p);
xor_register(&xor_block_32regs);
xor_register(&xor_block_32regs_p);
#ifdef CONFIG_CPU_HAS_LSX
if (cpu_has_lsx)
xor_register(&xor_block_lsx);
#endif
#ifdef CONFIG_CPU_HAS_LASX
if (cpu_has_lasx)
xor_register(&xor_block_lasx);
#endif
}
linux-7.2.3/lib/raid/xor/loongarch/xor_simd.c
+93
View File
@@ -0,0 +1,93 @@
// SPDX-License-Identifier: GPL-2.0-or-later
/*
* LoongArch SIMD XOR operations
*
* Copyright (C) 2023 WANG Xuerui <git@xen0n.name>
*/
#include "xor_simd.h"
/*
* Process one cache line (64 bytes) per loop. This is assuming all future
* popular LoongArch cores are similar performance-characteristics-wise to the
* current models.
*/
#define LINE_WIDTH 64
#ifdef CONFIG_CPU_HAS_LSX
#define LD(reg, base, offset) \
"vld $vr" #reg ", %[" #base "], " #offset "\n\t"
#define ST(reg, base, offset) \
"vst $vr" #reg ", %[" #base "], " #offset "\n\t"
#define XOR(dj, k) "vxor.v $vr" #dj ", $vr" #dj ", $vr" #k "\n\t"
#define LD_INOUT_LINE(base) \
LD(0, base, 0) \
LD(1, base, 16) \
LD(2, base, 32) \
LD(3, base, 48)
#define LD_AND_XOR_LINE(base) \
LD(4, base, 0) \
LD(5, base, 16) \
LD(6, base, 32) \
LD(7, base, 48) \
XOR(0, 4) \
XOR(1, 5) \
XOR(2, 6) \
XOR(3, 7)
#define ST_LINE(base) \
ST(0, base, 0) \
ST(1, base, 16) \
ST(2, base, 32) \
ST(3, base, 48)
#define XOR_FUNC_NAME(nr) __xor_lsx_##nr
#include "xor_template.c"
#undef LD
#undef ST
#undef XOR
#undef LD_INOUT_LINE
#undef LD_AND_XOR_LINE
#undef ST_LINE
#undef XOR_FUNC_NAME
#endif /* CONFIG_CPU_HAS_LSX */
#ifdef CONFIG_CPU_HAS_LASX
#define LD(reg, base, offset) \
"xvld $xr" #reg ", %[" #base "], " #offset "\n\t"
#define ST(reg, base, offset) \
"xvst $xr" #reg ", %[" #base "], " #offset "\n\t"
#define XOR(dj, k) "xvxor.v $xr" #dj ", $xr" #dj ", $xr" #k "\n\t"
#define LD_INOUT_LINE(base) \
LD(0, base, 0) \
LD(1, base, 32)
#define LD_AND_XOR_LINE(base) \
LD(2, base, 0) \
LD(3, base, 32) \
XOR(0, 2) \
XOR(1, 3)
#define ST_LINE(base) \
ST(0, base, 0) \
ST(1, base, 32)
#define XOR_FUNC_NAME(nr) __xor_lasx_##nr
#include "xor_template.c"
#undef LD
#undef ST
#undef XOR
#undef LD_INOUT_LINE
#undef LD_AND_XOR_LINE
#undef ST_LINE
#undef XOR_FUNC_NAME
#endif /* CONFIG_CPU_HAS_LASX */
linux-7.2.3/lib/raid/xor/loongarch/xor_simd.h
+38
View File
@@ -0,0 +1,38 @@
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
* Simple interface to link xor_simd.c and xor_simd_glue.c
*
* Separating these files ensures that no SIMD instructions are run outside of
* the kfpu critical section.
*/
#ifndef __LOONGARCH_LIB_XOR_SIMD_H
#define __LOONGARCH_LIB_XOR_SIMD_H
#ifdef CONFIG_CPU_HAS_LSX
void __xor_lsx_2(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2);
void __xor_lsx_3(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2, const unsigned long * __restrict p3);
void __xor_lsx_4(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2, const unsigned long * __restrict p3,
const unsigned long * __restrict p4);
void __xor_lsx_5(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2, const unsigned long * __restrict p3,
const unsigned long * __restrict p4, const unsigned long * __restrict p5);
#endif /* CONFIG_CPU_HAS_LSX */
#ifdef CONFIG_CPU_HAS_LASX
void __xor_lasx_2(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2);
void __xor_lasx_3(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2, const unsigned long * __restrict p3);
void __xor_lasx_4(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2, const unsigned long * __restrict p3,
const unsigned long * __restrict p4);
void __xor_lasx_5(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2, const unsigned long * __restrict p3,
const unsigned long * __restrict p4, const unsigned long * __restrict p5);
#endif /* CONFIG_CPU_HAS_LASX */
#endif /* __LOONGARCH_LIB_XOR_SIMD_H */
linux-7.2.3/lib/raid/xor/loongarch/xor_simd_glue.c
+37
View File
@@ -0,0 +1,37 @@
// SPDX-License-Identifier: GPL-2.0-or-later
/*
* LoongArch SIMD XOR operations
*
* Copyright (C) 2023 WANG Xuerui <git@xen0n.name>
*/
#include <linux/sched.h>
#include <asm/fpu.h>
#include "xor_impl.h"
#include "xor_arch.h"
#include "xor_simd.h"
#define MAKE_XOR_GLUES(flavor) \
DO_XOR_BLOCKS(flavor##_inner, __xor_##flavor##_2, __xor_##flavor##_3, \
__xor_##flavor##_4, __xor_##flavor##_5); \
\
static void xor_gen_##flavor(void *dest, void **srcs, unsigned int src_cnt, \
unsigned int bytes) \
{ \
kernel_fpu_begin(); \
xor_gen_##flavor##_inner(dest, srcs, src_cnt, bytes); \
kernel_fpu_end(); \
} \
\
struct xor_block_template xor_block_##flavor = { \
.name = __stringify(flavor), \
.xor_gen = xor_gen_##flavor \
}
#ifdef CONFIG_CPU_HAS_LSX
MAKE_XOR_GLUES(lsx);
#endif /* CONFIG_CPU_HAS_LSX */
#ifdef CONFIG_CPU_HAS_LASX
MAKE_XOR_GLUES(lasx);
#endif /* CONFIG_CPU_HAS_LASX */
linux-7.2.3/lib/raid/xor/loongarch/xor_template.c
+110
View File
@@ -0,0 +1,110 @@
// SPDX-License-Identifier: GPL-2.0-or-later
/*
* Copyright (C) 2023 WANG Xuerui <git@xen0n.name>
*
* Template for XOR operations, instantiated in xor_simd.c.
*
* Expected preprocessor definitions:
*
* - LINE_WIDTH
* - XOR_FUNC_NAME(nr)
* - LD_INOUT_LINE(buf)
* - LD_AND_XOR_LINE(buf)
* - ST_LINE(buf)
*/
void XOR_FUNC_NAME(2)(unsigned long bytes,
unsigned long * __restrict v1,
const unsigned long * __restrict v2)
{
unsigned long lines = bytes / LINE_WIDTH;
do {
__asm__ __volatile__ (
LD_INOUT_LINE(v1)
LD_AND_XOR_LINE(v2)
ST_LINE(v1)
: : [v1] "r"(v1), [v2] "r"(v2) : "memory"
);
v1 += LINE_WIDTH / sizeof(unsigned long);
v2 += LINE_WIDTH / sizeof(unsigned long);
} while (--lines > 0);
}
void XOR_FUNC_NAME(3)(unsigned long bytes,
unsigned long * __restrict v1,
const unsigned long * __restrict v2,
const unsigned long * __restrict v3)
{
unsigned long lines = bytes / LINE_WIDTH;
do {
__asm__ __volatile__ (
LD_INOUT_LINE(v1)
LD_AND_XOR_LINE(v2)
LD_AND_XOR_LINE(v3)
ST_LINE(v1)
: : [v1] "r"(v1), [v2] "r"(v2), [v3] "r"(v3) : "memory"
);
v1 += LINE_WIDTH / sizeof(unsigned long);
v2 += LINE_WIDTH / sizeof(unsigned long);
v3 += LINE_WIDTH / sizeof(unsigned long);
} while (--lines > 0);
}
void XOR_FUNC_NAME(4)(unsigned long bytes,
unsigned long * __restrict v1,
const unsigned long * __restrict v2,
const unsigned long * __restrict v3,
const unsigned long * __restrict v4)
{
unsigned long lines = bytes / LINE_WIDTH;
do {
__asm__ __volatile__ (
LD_INOUT_LINE(v1)
LD_AND_XOR_LINE(v2)
LD_AND_XOR_LINE(v3)
LD_AND_XOR_LINE(v4)
ST_LINE(v1)
: : [v1] "r"(v1), [v2] "r"(v2), [v3] "r"(v3), [v4] "r"(v4)
: "memory"
);
v1 += LINE_WIDTH / sizeof(unsigned long);
v2 += LINE_WIDTH / sizeof(unsigned long);
v3 += LINE_WIDTH / sizeof(unsigned long);
v4 += LINE_WIDTH / sizeof(unsigned long);
} while (--lines > 0);
}
void XOR_FUNC_NAME(5)(unsigned long bytes,
unsigned long * __restrict v1,
const unsigned long * __restrict v2,
const unsigned long * __restrict v3,
const unsigned long * __restrict v4,
const unsigned long * __restrict v5)
{
unsigned long lines = bytes / LINE_WIDTH;
do {
__asm__ __volatile__ (
LD_INOUT_LINE(v1)
LD_AND_XOR_LINE(v2)
LD_AND_XOR_LINE(v3)
LD_AND_XOR_LINE(v4)
LD_AND_XOR_LINE(v5)
ST_LINE(v1)
: : [v1] "r"(v1), [v2] "r"(v2), [v3] "r"(v3), [v4] "r"(v4),
[v5] "r"(v5) : "memory"
);
v1 += LINE_WIDTH / sizeof(unsigned long);
v2 += LINE_WIDTH / sizeof(unsigned long);
v3 += LINE_WIDTH / sizeof(unsigned long);
v4 += LINE_WIDTH / sizeof(unsigned long);
v5 += LINE_WIDTH / sizeof(unsigned long);
} while (--lines > 0);
}
linux-7.2.3/lib/raid/xor/powerpc/xor_arch.h
+22
View File
@@ -0,0 +1,22 @@
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
*
* Copyright (C) IBM Corporation, 2012
*
* Author: Anton Blanchard <anton@au.ibm.com>
*/
#include <asm/cpu_has_feature.h>
extern struct xor_block_template xor_block_altivec;
static __always_inline void __init arch_xor_init(void)
{
xor_register(&xor_block_8regs);
xor_register(&xor_block_8regs_p);
xor_register(&xor_block_32regs);
xor_register(&xor_block_32regs_p);
#ifdef CONFIG_ALTIVEC
if (cpu_has_feature(CPU_FTR_ALTIVEC))
xor_register(&xor_block_altivec);
#endif
}
linux-7.2.3/lib/raid/xor/powerpc/xor_vmx.c
+160
View File
@@ -0,0 +1,160 @@
// SPDX-License-Identifier: GPL-2.0-or-later
/*
*
* Copyright (C) IBM Corporation, 2012
*
* Author: Anton Blanchard <anton@au.ibm.com>
*/
/*
* Sparse (as at v0.5.0) gets very, very confused by this file.
* Make it a bit simpler for it.
*/
#include "xor_impl.h"
#if !defined(__CHECKER__)
#include <altivec.h>
#else
#define vec_xor(a, b) a ^ b
#define vector __attribute__((vector_size(16)))
#endif
#include "xor_vmx.h"
typedef vector signed char unative_t;
#define DEFINE(V) \
unative_t *V = (unative_t *)V##_in; \
unative_t V##_0, V##_1, V##_2, V##_3
#define LOAD(V) \
do { \
V##_0 = V[0]; \
V##_1 = V[1]; \
V##_2 = V[2]; \
V##_3 = V[3]; \
} while (0)
#define STORE(V) \
do { \
V[0] = V##_0; \
V[1] = V##_1; \
V[2] = V##_2; \
V[3] = V##_3; \
} while (0)
#define XOR(V1, V2) \
do { \
V1##_0 = vec_xor(V1##_0, V2##_0); \
V1##_1 = vec_xor(V1##_1, V2##_1); \
V1##_2 = vec_xor(V1##_2, V2##_2); \
V1##_3 = vec_xor(V1##_3, V2##_3); \
} while (0)
static void __xor_altivec_2(unsigned long bytes,
unsigned long * __restrict v1_in,
const unsigned long * __restrict v2_in)
{
DEFINE(v1);
DEFINE(v2);
unsigned long lines = bytes / (sizeof(unative_t)) / 4;
do {
LOAD(v1);
LOAD(v2);
XOR(v1, v2);
STORE(v1);
v1 += 4;
v2 += 4;
} while (--lines > 0);
}
static void __xor_altivec_3(unsigned long bytes,
unsigned long * __restrict v1_in,
const unsigned long * __restrict v2_in,
const unsigned long * __restrict v3_in)
{
DEFINE(v1);
DEFINE(v2);
DEFINE(v3);
unsigned long lines = bytes / (sizeof(unative_t)) / 4;
do {
LOAD(v1);
LOAD(v2);
LOAD(v3);
XOR(v1, v2);
XOR(v1, v3);
STORE(v1);
v1 += 4;
v2 += 4;
v3 += 4;
} while (--lines > 0);
}
static void __xor_altivec_4(unsigned long bytes,
unsigned long * __restrict v1_in,
const unsigned long * __restrict v2_in,
const unsigned long * __restrict v3_in,
const unsigned long * __restrict v4_in)
{
DEFINE(v1);
DEFINE(v2);
DEFINE(v3);
DEFINE(v4);
unsigned long lines = bytes / (sizeof(unative_t)) / 4;
do {
LOAD(v1);
LOAD(v2);
LOAD(v3);
LOAD(v4);
XOR(v1, v2);
XOR(v3, v4);
XOR(v1, v3);
STORE(v1);
v1 += 4;
v2 += 4;
v3 += 4;
v4 += 4;
} while (--lines > 0);
}
static void __xor_altivec_5(unsigned long bytes,
unsigned long * __restrict v1_in,
const unsigned long * __restrict v2_in,
const unsigned long * __restrict v3_in,
const unsigned long * __restrict v4_in,
const unsigned long * __restrict v5_in)
{
DEFINE(v1);
DEFINE(v2);
DEFINE(v3);
DEFINE(v4);
DEFINE(v5);
unsigned long lines = bytes / (sizeof(unative_t)) / 4;
do {
LOAD(v1);
LOAD(v2);
LOAD(v3);
LOAD(v4);
LOAD(v5);
XOR(v1, v2);
XOR(v3, v4);
XOR(v1, v5);
XOR(v1, v3);
STORE(v1);
v1 += 4;
v2 += 4;
v3 += 4;
v4 += 4;
v5 += 4;
} while (--lines > 0);
}
__DO_XOR_BLOCKS(altivec_inner, __xor_altivec_2, __xor_altivec_3,
__xor_altivec_4, __xor_altivec_5);
linux-7.2.3/lib/raid/xor/powerpc/xor_vmx.h
+10
View File
@@ -0,0 +1,10 @@
/* SPDX-License-Identifier: GPL-2.0 */
/*
* Simple interface to link xor_vmx.c and xor_vmx_glue.c
*
* Separating these file ensures that no altivec instructions are run
* outside of the enable/disable altivec block.
*/
void xor_gen_altivec_inner(void *dest, void **srcs, unsigned int src_cnt,
unsigned int bytes);
linux-7.2.3/lib/raid/xor/powerpc/xor_vmx_glue.c
+28
View File
@@ -0,0 +1,28 @@
// SPDX-License-Identifier: GPL-2.0-or-later
/*
* Altivec XOR operations
*
* Copyright 2017 IBM Corp.
*/
#include <linux/preempt.h>
#include <linux/sched.h>
#include <asm/switch_to.h>
#include "xor_impl.h"
#include "xor_arch.h"
#include "xor_vmx.h"
static void xor_gen_altivec(void *dest, void **srcs, unsigned int src_cnt,
unsigned int bytes)
{
preempt_disable();
enable_kernel_altivec();
xor_gen_altivec_inner(dest, srcs, src_cnt, bytes);
disable_kernel_altivec();
preempt_enable();
}
struct xor_block_template xor_block_altivec = {
.name = "altivec",
.xor_gen = xor_gen_altivec,
};
linux-7.2.3/lib/raid/xor/riscv/xor-glue.c
+25
View File
@@ -0,0 +1,25 @@
// SPDX-License-Identifier: GPL-2.0-or-later
/*
* Copyright (C) 2021 SiFive
*/
#include <asm/vector.h>
#include <asm/switch_to.h>
#include <asm/asm-prototypes.h>
#include "xor_impl.h"
#include "xor_arch.h"
DO_XOR_BLOCKS(vector_inner, xor_regs_2_, xor_regs_3_, xor_regs_4_, xor_regs_5_);
static void xor_gen_vector(void *dest, void **srcs, unsigned int src_cnt,
unsigned int bytes)
{
kernel_vector_begin();
xor_gen_vector_inner(dest, srcs, src_cnt, bytes);
kernel_vector_end();
}
struct xor_block_template xor_block_rvv = {
.name = "rvv",
.xor_gen = xor_gen_vector,
};
linux-7.2.3/lib/raid/xor/riscv/xor.S
+77
View File
@@ -0,0 +1,77 @@
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
* Copyright (C) 2021 SiFive
*/
#include <linux/linkage.h>
#include <linux/export.h>
#include <asm/asm.h>
SYM_FUNC_START(xor_regs_2_)
vsetvli a3, a0, e8, m8, ta, ma
vle8.v v0, (a1)
vle8.v v8, (a2)
sub a0, a0, a3
vxor.vv v16, v0, v8
add a2, a2, a3
vse8.v v16, (a1)
add a1, a1, a3
bnez a0, xor_regs_2_
ret
SYM_FUNC_END(xor_regs_2_)
SYM_FUNC_START(xor_regs_3_)
vsetvli a4, a0, e8, m8, ta, ma
vle8.v v0, (a1)
vle8.v v8, (a2)
sub a0, a0, a4
vxor.vv v0, v0, v8
vle8.v v16, (a3)
add a2, a2, a4
vxor.vv v16, v0, v16
add a3, a3, a4
vse8.v v16, (a1)
add a1, a1, a4
bnez a0, xor_regs_3_
ret
SYM_FUNC_END(xor_regs_3_)
SYM_FUNC_START(xor_regs_4_)
vsetvli a5, a0, e8, m8, ta, ma
vle8.v v0, (a1)
vle8.v v8, (a2)
sub a0, a0, a5
vxor.vv v0, v0, v8
vle8.v v16, (a3)
add a2, a2, a5
vxor.vv v0, v0, v16
vle8.v v24, (a4)
add a3, a3, a5
vxor.vv v16, v0, v24
add a4, a4, a5
vse8.v v16, (a1)
add a1, a1, a5
bnez a0, xor_regs_4_
ret
SYM_FUNC_END(xor_regs_4_)
SYM_FUNC_START(xor_regs_5_)
vsetvli a6, a0, e8, m8, ta, ma
vle8.v v0, (a1)
vle8.v v8, (a2)
sub a0, a0, a6
vxor.vv v0, v0, v8
vle8.v v16, (a3)
add a2, a2, a6
vxor.vv v0, v0, v16
vle8.v v24, (a4)
add a3, a3, a6
vxor.vv v0, v0, v24
vle8.v v8, (a5)
add a4, a4, a6
vxor.vv v16, v0, v8
add a5, a5, a6
vse8.v v16, (a1)
add a1, a1, a6
bnez a0, xor_regs_5_
ret
SYM_FUNC_END(xor_regs_5_)
linux-7.2.3/lib/raid/xor/riscv/xor_arch.h
+17
View File
@@ -0,0 +1,17 @@
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
* Copyright (C) 2021 SiFive
*/
#include <asm/vector.h>
extern struct xor_block_template xor_block_rvv;
static __always_inline void __init arch_xor_init(void)
{
xor_register(&xor_block_8regs);
xor_register(&xor_block_32regs);
#ifdef CONFIG_RISCV_ISA_V
if (has_vector())
xor_register(&xor_block_rvv);
#endif
}
linux-7.2.3/lib/raid/xor/s390/xor.c
+133
View File
@@ -0,0 +1,133 @@
// SPDX-License-Identifier: GPL-2.0
/*
* Optimized xor_block operation for RAID4/5
*
* Copyright IBM Corp. 2016
* Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>
*/
#include <linux/types.h>
#include "xor_impl.h"
#include "xor_arch.h"
static void xor_xc_2(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2)
{
asm volatile(
" aghi %0,-1\n"
" jm 3f\n"
" srlg 0,%0,8\n"
" ltgr 0,0\n"
" jz 1f\n"
"0: xc 0(256,%1),0(%2)\n"
" la %1,256(%1)\n"
" la %2,256(%2)\n"
" brctg 0,0b\n"
"1: exrl %0,2f\n"
" j 3f\n"
"2: xc 0(1,%1),0(%2)\n"
"3:"
: "+a" (bytes), "+a" (p1), "+a" (p2)
: : "0", "cc", "memory");
}
static void xor_xc_3(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2,
const unsigned long * __restrict p3)
{
asm volatile(
" aghi %0,-1\n"
" jm 4f\n"
" srlg 0,%0,8\n"
" ltgr 0,0\n"
" jz 1f\n"
"0: xc 0(256,%1),0(%2)\n"
" xc 0(256,%1),0(%3)\n"
" la %1,256(%1)\n"
" la %2,256(%2)\n"
" la %3,256(%3)\n"
" brctg 0,0b\n"
"1: exrl %0,2f\n"
" exrl %0,3f\n"
" j 4f\n"
"2: xc 0(1,%1),0(%2)\n"
"3: xc 0(1,%1),0(%3)\n"
"4:"
: "+a" (bytes), "+a" (p1), "+a" (p2), "+a" (p3)
: : "0", "cc", "memory");
}
static void xor_xc_4(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2,
const unsigned long * __restrict p3,
const unsigned long * __restrict p4)
{
asm volatile(
" aghi %0,-1\n"
" jm 5f\n"
" srlg 0,%0,8\n"
" ltgr 0,0\n"
" jz 1f\n"
"0: xc 0(256,%1),0(%2)\n"
" xc 0(256,%1),0(%3)\n"
" xc 0(256,%1),0(%4)\n"
" la %1,256(%1)\n"
" la %2,256(%2)\n"
" la %3,256(%3)\n"
" la %4,256(%4)\n"
" brctg 0,0b\n"
"1: exrl %0,2f\n"
" exrl %0,3f\n"
" exrl %0,4f\n"
" j 5f\n"
"2: xc 0(1,%1),0(%2)\n"
"3: xc 0(1,%1),0(%3)\n"
"4: xc 0(1,%1),0(%4)\n"
"5:"
: "+a" (bytes), "+a" (p1), "+a" (p2), "+a" (p3), "+a" (p4)
: : "0", "cc", "memory");
}
static void xor_xc_5(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2,
const unsigned long * __restrict p3,
const unsigned long * __restrict p4,
const unsigned long * __restrict p5)
{
asm volatile(
" aghi %0,-1\n"
" jm 6f\n"
" srlg 0,%0,8\n"
" ltgr 0,0\n"
" jz 1f\n"
"0: xc 0(256,%1),0(%2)\n"
" xc 0(256,%1),0(%3)\n"
" xc 0(256,%1),0(%4)\n"
" xc 0(256,%1),0(%5)\n"
" la %1,256(%1)\n"
" la %2,256(%2)\n"
" la %3,256(%3)\n"
" la %4,256(%4)\n"
" la %5,256(%5)\n"
" brctg 0,0b\n"
"1: exrl %0,2f\n"
" exrl %0,3f\n"
" exrl %0,4f\n"
" exrl %0,5f\n"
" j 6f\n"
"2: xc 0(1,%1),0(%2)\n"
"3: xc 0(1,%1),0(%3)\n"
"4: xc 0(1,%1),0(%4)\n"
"5: xc 0(1,%1),0(%5)\n"
"6:"
: "+a" (bytes), "+a" (p1), "+a" (p2), "+a" (p3), "+a" (p4),
"+a" (p5)
: : "0", "cc", "memory");
}
DO_XOR_BLOCKS(xc, xor_xc_2, xor_xc_3, xor_xc_4, xor_xc_5);
struct xor_block_template xor_block_xc = {
.name = "xc",
.xor_gen = xor_gen_xc,
};
linux-7.2.3/lib/raid/xor/s390/xor_arch.h
+13
View File
@@ -0,0 +1,13 @@
/* SPDX-License-Identifier: GPL-2.0 */
/*
* Optimited xor routines
*
* Copyright IBM Corp. 2016
* Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>
*/
extern struct xor_block_template xor_block_xc;
static __always_inline void __init arch_xor_init(void)
{
xor_force(&xor_block_xc);
}
linux-7.2.3/lib/raid/xor/sparc/xor-sparc32.c
+252
View File
@@ -0,0 +1,252 @@
// SPDX-License-Identifier: GPL-2.0-or-later
/*
* High speed xor_block operation for RAID4/5 utilizing the
* ldd/std SPARC instructions.
*
* Copyright (C) 1999 Jakub Jelinek (jj@ultra.linux.cz)
*/
#include "xor_impl.h"
#include "xor_arch.h"
static void
sparc_2(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2)
{
int lines = bytes / (sizeof (long)) / 8;
do {
__asm__ __volatile__(
"ldd [%0 + 0x00], %%g2\n\t"
"ldd [%0 + 0x08], %%g4\n\t"
"ldd [%0 + 0x10], %%o0\n\t"
"ldd [%0 + 0x18], %%o2\n\t"
"ldd [%1 + 0x00], %%o4\n\t"
"ldd [%1 + 0x08], %%l0\n\t"
"ldd [%1 + 0x10], %%l2\n\t"
"ldd [%1 + 0x18], %%l4\n\t"
"xor %%g2, %%o4, %%g2\n\t"
"xor %%g3, %%o5, %%g3\n\t"
"xor %%g4, %%l0, %%g4\n\t"
"xor %%g5, %%l1, %%g5\n\t"
"xor %%o0, %%l2, %%o0\n\t"
"xor %%o1, %%l3, %%o1\n\t"
"xor %%o2, %%l4, %%o2\n\t"
"xor %%o3, %%l5, %%o3\n\t"
"std %%g2, [%0 + 0x00]\n\t"
"std %%g4, [%0 + 0x08]\n\t"
"std %%o0, [%0 + 0x10]\n\t"
"std %%o2, [%0 + 0x18]\n"
:
: "r" (p1), "r" (p2)
: "g2", "g3", "g4", "g5",
"o0", "o1", "o2", "o3", "o4", "o5",
"l0", "l1", "l2", "l3", "l4", "l5");
p1 += 8;
p2 += 8;
} while (--lines > 0);
}
static void
sparc_3(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2,
const unsigned long * __restrict p3)
{
int lines = bytes / (sizeof (long)) / 8;
do {
__asm__ __volatile__(
"ldd [%0 + 0x00], %%g2\n\t"
"ldd [%0 + 0x08], %%g4\n\t"
"ldd [%0 + 0x10], %%o0\n\t"
"ldd [%0 + 0x18], %%o2\n\t"
"ldd [%1 + 0x00], %%o4\n\t"
"ldd [%1 + 0x08], %%l0\n\t"
"ldd [%1 + 0x10], %%l2\n\t"
"ldd [%1 + 0x18], %%l4\n\t"
"xor %%g2, %%o4, %%g2\n\t"
"xor %%g3, %%o5, %%g3\n\t"
"ldd [%2 + 0x00], %%o4\n\t"
"xor %%g4, %%l0, %%g4\n\t"
"xor %%g5, %%l1, %%g5\n\t"
"ldd [%2 + 0x08], %%l0\n\t"
"xor %%o0, %%l2, %%o0\n\t"
"xor %%o1, %%l3, %%o1\n\t"
"ldd [%2 + 0x10], %%l2\n\t"
"xor %%o2, %%l4, %%o2\n\t"
"xor %%o3, %%l5, %%o3\n\t"
"ldd [%2 + 0x18], %%l4\n\t"
"xor %%g2, %%o4, %%g2\n\t"
"xor %%g3, %%o5, %%g3\n\t"
"xor %%g4, %%l0, %%g4\n\t"
"xor %%g5, %%l1, %%g5\n\t"
"xor %%o0, %%l2, %%o0\n\t"
"xor %%o1, %%l3, %%o1\n\t"
"xor %%o2, %%l4, %%o2\n\t"
"xor %%o3, %%l5, %%o3\n\t"
"std %%g2, [%0 + 0x00]\n\t"
"std %%g4, [%0 + 0x08]\n\t"
"std %%o0, [%0 + 0x10]\n\t"
"std %%o2, [%0 + 0x18]\n"
:
: "r" (p1), "r" (p2), "r" (p3)
: "g2", "g3", "g4", "g5",
"o0", "o1", "o2", "o3", "o4", "o5",
"l0", "l1", "l2", "l3", "l4", "l5");
p1 += 8;
p2 += 8;
p3 += 8;
} while (--lines > 0);
}
static void
sparc_4(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2,
const unsigned long * __restrict p3,
const unsigned long * __restrict p4)
{
int lines = bytes / (sizeof (long)) / 8;
do {
__asm__ __volatile__(
"ldd [%0 + 0x00], %%g2\n\t"
"ldd [%0 + 0x08], %%g4\n\t"
"ldd [%0 + 0x10], %%o0\n\t"
"ldd [%0 + 0x18], %%o2\n\t"
"ldd [%1 + 0x00], %%o4\n\t"
"ldd [%1 + 0x08], %%l0\n\t"
"ldd [%1 + 0x10], %%l2\n\t"
"ldd [%1 + 0x18], %%l4\n\t"
"xor %%g2, %%o4, %%g2\n\t"
"xor %%g3, %%o5, %%g3\n\t"
"ldd [%2 + 0x00], %%o4\n\t"
"xor %%g4, %%l0, %%g4\n\t"
"xor %%g5, %%l1, %%g5\n\t"
"ldd [%2 + 0x08], %%l0\n\t"
"xor %%o0, %%l2, %%o0\n\t"
"xor %%o1, %%l3, %%o1\n\t"
"ldd [%2 + 0x10], %%l2\n\t"
"xor %%o2, %%l4, %%o2\n\t"
"xor %%o3, %%l5, %%o3\n\t"
"ldd [%2 + 0x18], %%l4\n\t"
"xor %%g2, %%o4, %%g2\n\t"
"xor %%g3, %%o5, %%g3\n\t"
"ldd [%3 + 0x00], %%o4\n\t"
"xor %%g4, %%l0, %%g4\n\t"
"xor %%g5, %%l1, %%g5\n\t"
"ldd [%3 + 0x08], %%l0\n\t"
"xor %%o0, %%l2, %%o0\n\t"
"xor %%o1, %%l3, %%o1\n\t"
"ldd [%3 + 0x10], %%l2\n\t"
"xor %%o2, %%l4, %%o2\n\t"
"xor %%o3, %%l5, %%o3\n\t"
"ldd [%3 + 0x18], %%l4\n\t"
"xor %%g2, %%o4, %%g2\n\t"
"xor %%g3, %%o5, %%g3\n\t"
"xor %%g4, %%l0, %%g4\n\t"
"xor %%g5, %%l1, %%g5\n\t"
"xor %%o0, %%l2, %%o0\n\t"
"xor %%o1, %%l3, %%o1\n\t"
"xor %%o2, %%l4, %%o2\n\t"
"xor %%o3, %%l5, %%o3\n\t"
"std %%g2, [%0 + 0x00]\n\t"
"std %%g4, [%0 + 0x08]\n\t"
"std %%o0, [%0 + 0x10]\n\t"
"std %%o2, [%0 + 0x18]\n"
:
: "r" (p1), "r" (p2), "r" (p3), "r" (p4)
: "g2", "g3", "g4", "g5",
"o0", "o1", "o2", "o3", "o4", "o5",
"l0", "l1", "l2", "l3", "l4", "l5");
p1 += 8;
p2 += 8;
p3 += 8;
p4 += 8;
} while (--lines > 0);
}
static void
sparc_5(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2,
const unsigned long * __restrict p3,
const unsigned long * __restrict p4,
const unsigned long * __restrict p5)
{
int lines = bytes / (sizeof (long)) / 8;
do {
__asm__ __volatile__(
"ldd [%0 + 0x00], %%g2\n\t"
"ldd [%0 + 0x08], %%g4\n\t"
"ldd [%0 + 0x10], %%o0\n\t"
"ldd [%0 + 0x18], %%o2\n\t"
"ldd [%1 + 0x00], %%o4\n\t"
"ldd [%1 + 0x08], %%l0\n\t"
"ldd [%1 + 0x10], %%l2\n\t"
"ldd [%1 + 0x18], %%l4\n\t"
"xor %%g2, %%o4, %%g2\n\t"
"xor %%g3, %%o5, %%g3\n\t"
"ldd [%2 + 0x00], %%o4\n\t"
"xor %%g4, %%l0, %%g4\n\t"
"xor %%g5, %%l1, %%g5\n\t"
"ldd [%2 + 0x08], %%l0\n\t"
"xor %%o0, %%l2, %%o0\n\t"
"xor %%o1, %%l3, %%o1\n\t"
"ldd [%2 + 0x10], %%l2\n\t"
"xor %%o2, %%l4, %%o2\n\t"
"xor %%o3, %%l5, %%o3\n\t"
"ldd [%2 + 0x18], %%l4\n\t"
"xor %%g2, %%o4, %%g2\n\t"
"xor %%g3, %%o5, %%g3\n\t"
"ldd [%3 + 0x00], %%o4\n\t"
"xor %%g4, %%l0, %%g4\n\t"
"xor %%g5, %%l1, %%g5\n\t"
"ldd [%3 + 0x08], %%l0\n\t"
"xor %%o0, %%l2, %%o0\n\t"
"xor %%o1, %%l3, %%o1\n\t"
"ldd [%3 + 0x10], %%l2\n\t"
"xor %%o2, %%l4, %%o2\n\t"
"xor %%o3, %%l5, %%o3\n\t"
"ldd [%3 + 0x18], %%l4\n\t"
"xor %%g2, %%o4, %%g2\n\t"
"xor %%g3, %%o5, %%g3\n\t"
"ldd [%4 + 0x00], %%o4\n\t"
"xor %%g4, %%l0, %%g4\n\t"
"xor %%g5, %%l1, %%g5\n\t"
"ldd [%4 + 0x08], %%l0\n\t"
"xor %%o0, %%l2, %%o0\n\t"
"xor %%o1, %%l3, %%o1\n\t"
"ldd [%4 + 0x10], %%l2\n\t"
"xor %%o2, %%l4, %%o2\n\t"
"xor %%o3, %%l5, %%o3\n\t"
"ldd [%4 + 0x18], %%l4\n\t"
"xor %%g2, %%o4, %%g2\n\t"
"xor %%g3, %%o5, %%g3\n\t"
"xor %%g4, %%l0, %%g4\n\t"
"xor %%g5, %%l1, %%g5\n\t"
"xor %%o0, %%l2, %%o0\n\t"
"xor %%o1, %%l3, %%o1\n\t"
"xor %%o2, %%l4, %%o2\n\t"
"xor %%o3, %%l5, %%o3\n\t"
"std %%g2, [%0 + 0x00]\n\t"
"std %%g4, [%0 + 0x08]\n\t"
"std %%o0, [%0 + 0x10]\n\t"
"std %%o2, [%0 + 0x18]\n"
:
: "r" (p1), "r" (p2), "r" (p3), "r" (p4), "r" (p5)
: "g2", "g3", "g4", "g5",
"o0", "o1", "o2", "o3", "o4", "o5",
"l0", "l1", "l2", "l3", "l4", "l5");
p1 += 8;
p2 += 8;
p3 += 8;
p4 += 8;
p5 += 8;
} while (--lines > 0);
}
DO_XOR_BLOCKS(sparc32, sparc_2, sparc_3, sparc_4, sparc_5);
struct xor_block_template xor_block_SPARC = {
.name = "SPARC",
.xor_gen = xor_gen_sparc32,
};
linux-7.2.3/lib/raid/xor/sparc/xor-sparc64-glue.c
+59
View File
@@ -0,0 +1,59 @@
// SPDX-License-Identifier: GPL-2.0-or-later
/*
* High speed xor_block operation for RAID4/5 utilizing the
* UltraSparc Visual Instruction Set and Niagara block-init
* twin-load instructions.
*
* Copyright (C) 1997, 1999 Jakub Jelinek (jj@ultra.linux.cz)
* Copyright (C) 2006 David S. Miller <davem@davemloft.net>
*/
#include "xor_impl.h"
#include "xor_arch.h"
void xor_vis_2(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2);
void xor_vis_3(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2,
const unsigned long * __restrict p3);
void xor_vis_4(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2,
const unsigned long * __restrict p3,
const unsigned long * __restrict p4);
void xor_vis_5(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2,
const unsigned long * __restrict p3,
const unsigned long * __restrict p4,
const unsigned long * __restrict p5);
/* XXX Ugh, write cheetah versions... -DaveM */
DO_XOR_BLOCKS(vis, xor_vis_2, xor_vis_3, xor_vis_4, xor_vis_5);
struct xor_block_template xor_block_VIS = {
.name = "VIS",
.xor_gen = xor_gen_vis,
};
void xor_niagara_2(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2);
void xor_niagara_3(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2,
const unsigned long * __restrict p3);
void xor_niagara_4(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2,
const unsigned long * __restrict p3,
const unsigned long * __restrict p4);
void xor_niagara_5(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2,
const unsigned long * __restrict p3,
const unsigned long * __restrict p4,
const unsigned long * __restrict p5);
DO_XOR_BLOCKS(niagara, xor_niagara_2, xor_niagara_3, xor_niagara_4,
xor_niagara_5);
struct xor_block_template xor_block_niagara = {
.name = "Niagara",
.xor_gen = xor_gen_niagara,
};
linux-7.2.3/lib/raid/xor/sparc/xor-sparc64.S
+636
View File
@@ -0,0 +1,636 @@
/* SPDX-License-Identifier: GPL-2.0 */
/*
* High speed xor_block operation for RAID4/5 utilizing the
* UltraSparc Visual Instruction Set and Niagara store-init/twin-load.
*
* Copyright (C) 1997, 1999 Jakub Jelinek (jj@ultra.linux.cz)
* Copyright (C) 2006 David S. Miller <davem@davemloft.net>
*/
#include <linux/export.h>
#include <linux/linkage.h>
#include <asm/visasm.h>
#include <asm/asi.h>
#include <asm/dcu.h>
#include <asm/spitfire.h>
/*
* Requirements:
* !(((long)dest | (long)sourceN) & (64 - 1)) &&
* !(len & 127) && len >= 256
*/
.text
/* VIS versions. */
ENTRY(xor_vis_2)
rd %fprs, %o5
andcc %o5, FPRS_FEF|FPRS_DU, %g0
be,pt %icc, 0f
sethi %hi(VISenter), %g1
jmpl %g1 + %lo(VISenter), %g7
add %g7, 8, %g7
0: wr %g0, FPRS_FEF, %fprs
rd %asi, %g1
wr %g0, ASI_BLK_P, %asi
membar #LoadStore|#StoreLoad|#StoreStore
sub %o0, 128, %o0
ldda [%o1] %asi, %f0
ldda [%o2] %asi, %f16
2: ldda [%o1 + 64] %asi, %f32
fxor %f0, %f16, %f16
fxor %f2, %f18, %f18
fxor %f4, %f20, %f20
fxor %f6, %f22, %f22
fxor %f8, %f24, %f24
fxor %f10, %f26, %f26
fxor %f12, %f28, %f28
fxor %f14, %f30, %f30
stda %f16, [%o1] %asi
ldda [%o2 + 64] %asi, %f48
ldda [%o1 + 128] %asi, %f0
fxor %f32, %f48, %f48
fxor %f34, %f50, %f50
add %o1, 128, %o1
fxor %f36, %f52, %f52
add %o2, 128, %o2
fxor %f38, %f54, %f54
subcc %o0, 128, %o0
fxor %f40, %f56, %f56
fxor %f42, %f58, %f58
fxor %f44, %f60, %f60
fxor %f46, %f62, %f62
stda %f48, [%o1 - 64] %asi
bne,pt %xcc, 2b
ldda [%o2] %asi, %f16
ldda [%o1 + 64] %asi, %f32
fxor %f0, %f16, %f16
fxor %f2, %f18, %f18
fxor %f4, %f20, %f20
fxor %f6, %f22, %f22
fxor %f8, %f24, %f24
fxor %f10, %f26, %f26
fxor %f12, %f28, %f28
fxor %f14, %f30, %f30
stda %f16, [%o1] %asi
ldda [%o2 + 64] %asi, %f48
membar #Sync
fxor %f32, %f48, %f48
fxor %f34, %f50, %f50
fxor %f36, %f52, %f52
fxor %f38, %f54, %f54
fxor %f40, %f56, %f56
fxor %f42, %f58, %f58
fxor %f44, %f60, %f60
fxor %f46, %f62, %f62
stda %f48, [%o1 + 64] %asi
membar #Sync|#StoreStore|#StoreLoad
wr %g1, %g0, %asi
retl
wr %g0, 0, %fprs
ENDPROC(xor_vis_2)
ENTRY(xor_vis_3)
rd %fprs, %o5
andcc %o5, FPRS_FEF|FPRS_DU, %g0
be,pt %icc, 0f
sethi %hi(VISenter), %g1
jmpl %g1 + %lo(VISenter), %g7
add %g7, 8, %g7
0: wr %g0, FPRS_FEF, %fprs
rd %asi, %g1
wr %g0, ASI_BLK_P, %asi
membar #LoadStore|#StoreLoad|#StoreStore
sub %o0, 64, %o0
ldda [%o1] %asi, %f0
ldda [%o2] %asi, %f16
3: ldda [%o3] %asi, %f32
fxor %f0, %f16, %f48
fxor %f2, %f18, %f50
add %o1, 64, %o1
fxor %f4, %f20, %f52
fxor %f6, %f22, %f54
add %o2, 64, %o2
fxor %f8, %f24, %f56
fxor %f10, %f26, %f58
fxor %f12, %f28, %f60
fxor %f14, %f30, %f62
ldda [%o1] %asi, %f0
fxor %f48, %f32, %f48
fxor %f50, %f34, %f50
fxor %f52, %f36, %f52
fxor %f54, %f38, %f54
add %o3, 64, %o3
fxor %f56, %f40, %f56
fxor %f58, %f42, %f58
subcc %o0, 64, %o0
fxor %f60, %f44, %f60
fxor %f62, %f46, %f62
stda %f48, [%o1 - 64] %asi
bne,pt %xcc, 3b
ldda [%o2] %asi, %f16
ldda [%o3] %asi, %f32
fxor %f0, %f16, %f48
fxor %f2, %f18, %f50
fxor %f4, %f20, %f52
fxor %f6, %f22, %f54
fxor %f8, %f24, %f56
fxor %f10, %f26, %f58
fxor %f12, %f28, %f60
fxor %f14, %f30, %f62
membar #Sync
fxor %f48, %f32, %f48
fxor %f50, %f34, %f50
fxor %f52, %f36, %f52
fxor %f54, %f38, %f54
fxor %f56, %f40, %f56
fxor %f58, %f42, %f58
fxor %f60, %f44, %f60
fxor %f62, %f46, %f62
stda %f48, [%o1] %asi
membar #Sync|#StoreStore|#StoreLoad
wr %g1, %g0, %asi
retl
wr %g0, 0, %fprs
ENDPROC(xor_vis_3)
ENTRY(xor_vis_4)
rd %fprs, %o5
andcc %o5, FPRS_FEF|FPRS_DU, %g0
be,pt %icc, 0f
sethi %hi(VISenter), %g1
jmpl %g1 + %lo(VISenter), %g7
add %g7, 8, %g7
0: wr %g0, FPRS_FEF, %fprs
rd %asi, %g1
wr %g0, ASI_BLK_P, %asi
membar #LoadStore|#StoreLoad|#StoreStore
sub %o0, 64, %o0
ldda [%o1] %asi, %f0
ldda [%o2] %asi, %f16
4: ldda [%o3] %asi, %f32
fxor %f0, %f16, %f16
fxor %f2, %f18, %f18
add %o1, 64, %o1
fxor %f4, %f20, %f20
fxor %f6, %f22, %f22
add %o2, 64, %o2
fxor %f8, %f24, %f24
fxor %f10, %f26, %f26
fxor %f12, %f28, %f28
fxor %f14, %f30, %f30
ldda [%o4] %asi, %f48
fxor %f16, %f32, %f32
fxor %f18, %f34, %f34
fxor %f20, %f36, %f36
fxor %f22, %f38, %f38
add %o3, 64, %o3
fxor %f24, %f40, %f40
fxor %f26, %f42, %f42
fxor %f28, %f44, %f44
fxor %f30, %f46, %f46
ldda [%o1] %asi, %f0
fxor %f32, %f48, %f48
fxor %f34, %f50, %f50
fxor %f36, %f52, %f52
add %o4, 64, %o4
fxor %f38, %f54, %f54
fxor %f40, %f56, %f56
fxor %f42, %f58, %f58
subcc %o0, 64, %o0
fxor %f44, %f60, %f60
fxor %f46, %f62, %f62
stda %f48, [%o1 - 64] %asi
bne,pt %xcc, 4b
ldda [%o2] %asi, %f16
ldda [%o3] %asi, %f32
fxor %f0, %f16, %f16
fxor %f2, %f18, %f18
fxor %f4, %f20, %f20
fxor %f6, %f22, %f22
fxor %f8, %f24, %f24
fxor %f10, %f26, %f26
fxor %f12, %f28, %f28
fxor %f14, %f30, %f30
ldda [%o4] %asi, %f48
fxor %f16, %f32, %f32
fxor %f18, %f34, %f34
fxor %f20, %f36, %f36
fxor %f22, %f38, %f38
fxor %f24, %f40, %f40
fxor %f26, %f42, %f42
fxor %f28, %f44, %f44
fxor %f30, %f46, %f46
membar #Sync
fxor %f32, %f48, %f48
fxor %f34, %f50, %f50
fxor %f36, %f52, %f52
fxor %f38, %f54, %f54
fxor %f40, %f56, %f56
fxor %f42, %f58, %f58
fxor %f44, %f60, %f60
fxor %f46, %f62, %f62
stda %f48, [%o1] %asi
membar #Sync|#StoreStore|#StoreLoad
wr %g1, %g0, %asi
retl
wr %g0, 0, %fprs
ENDPROC(xor_vis_4)
ENTRY(xor_vis_5)
save %sp, -192, %sp
rd %fprs, %o5
andcc %o5, FPRS_FEF|FPRS_DU, %g0
be,pt %icc, 0f
sethi %hi(VISenter), %g1
jmpl %g1 + %lo(VISenter), %g7
add %g7, 8, %g7
0: wr %g0, FPRS_FEF, %fprs
rd %asi, %g1
wr %g0, ASI_BLK_P, %asi
membar #LoadStore|#StoreLoad|#StoreStore
sub %i0, 64, %i0
ldda [%i1] %asi, %f0
ldda [%i2] %asi, %f16
5: ldda [%i3] %asi, %f32
fxor %f0, %f16, %f48
fxor %f2, %f18, %f50
add %i1, 64, %i1
fxor %f4, %f20, %f52
fxor %f6, %f22, %f54
add %i2, 64, %i2
fxor %f8, %f24, %f56
fxor %f10, %f26, %f58
fxor %f12, %f28, %f60
fxor %f14, %f30, %f62
ldda [%i4] %asi, %f16
fxor %f48, %f32, %f48
fxor %f50, %f34, %f50
fxor %f52, %f36, %f52
fxor %f54, %f38, %f54
add %i3, 64, %i3
fxor %f56, %f40, %f56
fxor %f58, %f42, %f58
fxor %f60, %f44, %f60
fxor %f62, %f46, %f62
ldda [%i5] %asi, %f32
fxor %f48, %f16, %f48
fxor %f50, %f18, %f50
add %i4, 64, %i4
fxor %f52, %f20, %f52
fxor %f54, %f22, %f54
add %i5, 64, %i5
fxor %f56, %f24, %f56
fxor %f58, %f26, %f58
fxor %f60, %f28, %f60
fxor %f62, %f30, %f62
ldda [%i1] %asi, %f0
fxor %f48, %f32, %f48
fxor %f50, %f34, %f50
fxor %f52, %f36, %f52
fxor %f54, %f38, %f54
fxor %f56, %f40, %f56
fxor %f58, %f42, %f58
subcc %i0, 64, %i0
fxor %f60, %f44, %f60
fxor %f62, %f46, %f62
stda %f48, [%i1 - 64] %asi
bne,pt %xcc, 5b
ldda [%i2] %asi, %f16
ldda [%i3] %asi, %f32
fxor %f0, %f16, %f48
fxor %f2, %f18, %f50
fxor %f4, %f20, %f52
fxor %f6, %f22, %f54
fxor %f8, %f24, %f56
fxor %f10, %f26, %f58
fxor %f12, %f28, %f60
fxor %f14, %f30, %f62
ldda [%i4] %asi, %f16
fxor %f48, %f32, %f48
fxor %f50, %f34, %f50
fxor %f52, %f36, %f52
fxor %f54, %f38, %f54
fxor %f56, %f40, %f56
fxor %f58, %f42, %f58
fxor %f60, %f44, %f60
fxor %f62, %f46, %f62
ldda [%i5] %asi, %f32
fxor %f48, %f16, %f48
fxor %f50, %f18, %f50
fxor %f52, %f20, %f52
fxor %f54, %f22, %f54
fxor %f56, %f24, %f56
fxor %f58, %f26, %f58
fxor %f60, %f28, %f60
fxor %f62, %f30, %f62
membar #Sync
fxor %f48, %f32, %f48
fxor %f50, %f34, %f50
fxor %f52, %f36, %f52
fxor %f54, %f38, %f54
fxor %f56, %f40, %f56
fxor %f58, %f42, %f58
fxor %f60, %f44, %f60
fxor %f62, %f46, %f62
stda %f48, [%i1] %asi
membar #Sync|#StoreStore|#StoreLoad
wr %g1, %g0, %asi
wr %g0, 0, %fprs
ret
restore
ENDPROC(xor_vis_5)
/* Niagara versions. */
ENTRY(xor_niagara_2) /* %o0=bytes, %o1=dest, %o2=src */
save %sp, -192, %sp
prefetch [%i1], #n_writes
prefetch [%i2], #one_read
rd %asi, %g7
wr %g0, ASI_BLK_INIT_QUAD_LDD_P, %asi
srlx %i0, 6, %g1
mov %i1, %i0
mov %i2, %i1
1: ldda [%i1 + 0x00] %asi, %i2 /* %i2/%i3 = src + 0x00 */
ldda [%i1 + 0x10] %asi, %i4 /* %i4/%i5 = src + 0x10 */
ldda [%i1 + 0x20] %asi, %g2 /* %g2/%g3 = src + 0x20 */
ldda [%i1 + 0x30] %asi, %l0 /* %l0/%l1 = src + 0x30 */
prefetch [%i1 + 0x40], #one_read
ldda [%i0 + 0x00] %asi, %o0 /* %o0/%o1 = dest + 0x00 */
ldda [%i0 + 0x10] %asi, %o2 /* %o2/%o3 = dest + 0x10 */
ldda [%i0 + 0x20] %asi, %o4 /* %o4/%o5 = dest + 0x20 */
ldda [%i0 + 0x30] %asi, %l2 /* %l2/%l3 = dest + 0x30 */
prefetch [%i0 + 0x40], #n_writes
xor %o0, %i2, %o0
xor %o1, %i3, %o1
stxa %o0, [%i0 + 0x00] %asi
stxa %o1, [%i0 + 0x08] %asi
xor %o2, %i4, %o2
xor %o3, %i5, %o3
stxa %o2, [%i0 + 0x10] %asi
stxa %o3, [%i0 + 0x18] %asi
xor %o4, %g2, %o4
xor %o5, %g3, %o5
stxa %o4, [%i0 + 0x20] %asi
stxa %o5, [%i0 + 0x28] %asi
xor %l2, %l0, %l2
xor %l3, %l1, %l3
stxa %l2, [%i0 + 0x30] %asi
stxa %l3, [%i0 + 0x38] %asi
add %i0, 0x40, %i0
subcc %g1, 1, %g1
bne,pt %xcc, 1b
add %i1, 0x40, %i1
membar #Sync
wr %g7, 0x0, %asi
ret
restore
ENDPROC(xor_niagara_2)
ENTRY(xor_niagara_3) /* %o0=bytes, %o1=dest, %o2=src1, %o3=src2 */
save %sp, -192, %sp
prefetch [%i1], #n_writes
prefetch [%i2], #one_read
prefetch [%i3], #one_read
rd %asi, %g7
wr %g0, ASI_BLK_INIT_QUAD_LDD_P, %asi
srlx %i0, 6, %g1
mov %i1, %i0
mov %i2, %i1
mov %i3, %l7
1: ldda [%i1 + 0x00] %asi, %i2 /* %i2/%i3 = src1 + 0x00 */
ldda [%i1 + 0x10] %asi, %i4 /* %i4/%i5 = src1 + 0x10 */
ldda [%l7 + 0x00] %asi, %g2 /* %g2/%g3 = src2 + 0x00 */
ldda [%l7 + 0x10] %asi, %l0 /* %l0/%l1 = src2 + 0x10 */
ldda [%i0 + 0x00] %asi, %o0 /* %o0/%o1 = dest + 0x00 */
ldda [%i0 + 0x10] %asi, %o2 /* %o2/%o3 = dest + 0x10 */
xor %g2, %i2, %g2
xor %g3, %i3, %g3
xor %o0, %g2, %o0
xor %o1, %g3, %o1
stxa %o0, [%i0 + 0x00] %asi
stxa %o1, [%i0 + 0x08] %asi
ldda [%i1 + 0x20] %asi, %i2 /* %i2/%i3 = src1 + 0x20 */
ldda [%l7 + 0x20] %asi, %g2 /* %g2/%g3 = src2 + 0x20 */
ldda [%i0 + 0x20] %asi, %o0 /* %o0/%o1 = dest + 0x20 */
xor %l0, %i4, %l0
xor %l1, %i5, %l1
xor %o2, %l0, %o2
xor %o3, %l1, %o3
stxa %o2, [%i0 + 0x10] %asi
stxa %o3, [%i0 + 0x18] %asi
ldda [%i1 + 0x30] %asi, %i4 /* %i4/%i5 = src1 + 0x30 */
ldda [%l7 + 0x30] %asi, %l0 /* %l0/%l1 = src2 + 0x30 */
ldda [%i0 + 0x30] %asi, %o2 /* %o2/%o3 = dest + 0x30 */
prefetch [%i1 + 0x40], #one_read
prefetch [%l7 + 0x40], #one_read
prefetch [%i0 + 0x40], #n_writes
xor %g2, %i2, %g2
xor %g3, %i3, %g3
xor %o0, %g2, %o0
xor %o1, %g3, %o1
stxa %o0, [%i0 + 0x20] %asi
stxa %o1, [%i0 + 0x28] %asi
xor %l0, %i4, %l0
xor %l1, %i5, %l1
xor %o2, %l0, %o2
xor %o3, %l1, %o3
stxa %o2, [%i0 + 0x30] %asi
stxa %o3, [%i0 + 0x38] %asi
add %i0, 0x40, %i0
add %i1, 0x40, %i1
subcc %g1, 1, %g1
bne,pt %xcc, 1b
add %l7, 0x40, %l7
membar #Sync
wr %g7, 0x0, %asi
ret
restore
ENDPROC(xor_niagara_3)
ENTRY(xor_niagara_4) /* %o0=bytes, %o1=dest, %o2=src1, %o3=src2, %o4=src3 */
save %sp, -192, %sp
prefetch [%i1], #n_writes
prefetch [%i2], #one_read
prefetch [%i3], #one_read
prefetch [%i4], #one_read
rd %asi, %g7
wr %g0, ASI_BLK_INIT_QUAD_LDD_P, %asi
srlx %i0, 6, %g1
mov %i1, %i0
mov %i2, %i1
mov %i3, %l7
mov %i4, %l6
1: ldda [%i1 + 0x00] %asi, %i2 /* %i2/%i3 = src1 + 0x00 */
ldda [%l7 + 0x00] %asi, %i4 /* %i4/%i5 = src2 + 0x00 */
ldda [%l6 + 0x00] %asi, %g2 /* %g2/%g3 = src3 + 0x00 */
ldda [%i0 + 0x00] %asi, %l0 /* %l0/%l1 = dest + 0x00 */
xor %i4, %i2, %i4
xor %i5, %i3, %i5
ldda [%i1 + 0x10] %asi, %i2 /* %i2/%i3 = src1 + 0x10 */
xor %g2, %i4, %g2
xor %g3, %i5, %g3
ldda [%l7 + 0x10] %asi, %i4 /* %i4/%i5 = src2 + 0x10 */
xor %l0, %g2, %l0
xor %l1, %g3, %l1
stxa %l0, [%i0 + 0x00] %asi
stxa %l1, [%i0 + 0x08] %asi
ldda [%l6 + 0x10] %asi, %g2 /* %g2/%g3 = src3 + 0x10 */
ldda [%i0 + 0x10] %asi, %l0 /* %l0/%l1 = dest + 0x10 */
xor %i4, %i2, %i4
xor %i5, %i3, %i5
ldda [%i1 + 0x20] %asi, %i2 /* %i2/%i3 = src1 + 0x20 */
xor %g2, %i4, %g2
xor %g3, %i5, %g3
ldda [%l7 + 0x20] %asi, %i4 /* %i4/%i5 = src2 + 0x20 */
xor %l0, %g2, %l0
xor %l1, %g3, %l1
stxa %l0, [%i0 + 0x10] %asi
stxa %l1, [%i0 + 0x18] %asi
ldda [%l6 + 0x20] %asi, %g2 /* %g2/%g3 = src3 + 0x20 */
ldda [%i0 + 0x20] %asi, %l0 /* %l0/%l1 = dest + 0x20 */
xor %i4, %i2, %i4
xor %i5, %i3, %i5
ldda [%i1 + 0x30] %asi, %i2 /* %i2/%i3 = src1 + 0x30 */
xor %g2, %i4, %g2
xor %g3, %i5, %g3
ldda [%l7 + 0x30] %asi, %i4 /* %i4/%i5 = src2 + 0x30 */
xor %l0, %g2, %l0
xor %l1, %g3, %l1
stxa %l0, [%i0 + 0x20] %asi
stxa %l1, [%i0 + 0x28] %asi
ldda [%l6 + 0x30] %asi, %g2 /* %g2/%g3 = src3 + 0x30 */
ldda [%i0 + 0x30] %asi, %l0 /* %l0/%l1 = dest + 0x30 */
prefetch [%i1 + 0x40], #one_read
prefetch [%l7 + 0x40], #one_read
prefetch [%l6 + 0x40], #one_read
prefetch [%i0 + 0x40], #n_writes
xor %i4, %i2, %i4
xor %i5, %i3, %i5
xor %g2, %i4, %g2
xor %g3, %i5, %g3
xor %l0, %g2, %l0
xor %l1, %g3, %l1
stxa %l0, [%i0 + 0x30] %asi
stxa %l1, [%i0 + 0x38] %asi
add %i0, 0x40, %i0
add %i1, 0x40, %i1
add %l7, 0x40, %l7
subcc %g1, 1, %g1
bne,pt %xcc, 1b
add %l6, 0x40, %l6
membar #Sync
wr %g7, 0x0, %asi
ret
restore
ENDPROC(xor_niagara_4)
ENTRY(xor_niagara_5) /* %o0=bytes, %o1=dest, %o2=src1, %o3=src2, %o4=src3, %o5=src4 */
save %sp, -192, %sp
prefetch [%i1], #n_writes
prefetch [%i2], #one_read
prefetch [%i3], #one_read
prefetch [%i4], #one_read
prefetch [%i5], #one_read
rd %asi, %g7
wr %g0, ASI_BLK_INIT_QUAD_LDD_P, %asi
srlx %i0, 6, %g1
mov %i1, %i0
mov %i2, %i1
mov %i3, %l7
mov %i4, %l6
mov %i5, %l5
1: ldda [%i1 + 0x00] %asi, %i2 /* %i2/%i3 = src1 + 0x00 */
ldda [%l7 + 0x00] %asi, %i4 /* %i4/%i5 = src2 + 0x00 */
ldda [%l6 + 0x00] %asi, %g2 /* %g2/%g3 = src3 + 0x00 */
ldda [%l5 + 0x00] %asi, %l0 /* %l0/%l1 = src4 + 0x00 */
ldda [%i0 + 0x00] %asi, %l2 /* %l2/%l3 = dest + 0x00 */
xor %i4, %i2, %i4
xor %i5, %i3, %i5
ldda [%i1 + 0x10] %asi, %i2 /* %i2/%i3 = src1 + 0x10 */
xor %g2, %i4, %g2
xor %g3, %i5, %g3
ldda [%l7 + 0x10] %asi, %i4 /* %i4/%i5 = src2 + 0x10 */
xor %l0, %g2, %l0
xor %l1, %g3, %l1
ldda [%l6 + 0x10] %asi, %g2 /* %g2/%g3 = src3 + 0x10 */
xor %l2, %l0, %l2
xor %l3, %l1, %l3
stxa %l2, [%i0 + 0x00] %asi
stxa %l3, [%i0 + 0x08] %asi
ldda [%l5 + 0x10] %asi, %l0 /* %l0/%l1 = src4 + 0x10 */
ldda [%i0 + 0x10] %asi, %l2 /* %l2/%l3 = dest + 0x10 */
xor %i4, %i2, %i4
xor %i5, %i3, %i5
ldda [%i1 + 0x20] %asi, %i2 /* %i2/%i3 = src1 + 0x20 */
xor %g2, %i4, %g2
xor %g3, %i5, %g3
ldda [%l7 + 0x20] %asi, %i4 /* %i4/%i5 = src2 + 0x20 */
xor %l0, %g2, %l0
xor %l1, %g3, %l1
ldda [%l6 + 0x20] %asi, %g2 /* %g2/%g3 = src3 + 0x20 */
xor %l2, %l0, %l2
xor %l3, %l1, %l3
stxa %l2, [%i0 + 0x10] %asi
stxa %l3, [%i0 + 0x18] %asi
ldda [%l5 + 0x20] %asi, %l0 /* %l0/%l1 = src4 + 0x20 */
ldda [%i0 + 0x20] %asi, %l2 /* %l2/%l3 = dest + 0x20 */
xor %i4, %i2, %i4
xor %i5, %i3, %i5
ldda [%i1 + 0x30] %asi, %i2 /* %i2/%i3 = src1 + 0x30 */
xor %g2, %i4, %g2
xor %g3, %i5, %g3
ldda [%l7 + 0x30] %asi, %i4 /* %i4/%i5 = src2 + 0x30 */
xor %l0, %g2, %l0
xor %l1, %g3, %l1
ldda [%l6 + 0x30] %asi, %g2 /* %g2/%g3 = src3 + 0x30 */
xor %l2, %l0, %l2
xor %l3, %l1, %l3
stxa %l2, [%i0 + 0x20] %asi
stxa %l3, [%i0 + 0x28] %asi
ldda [%l5 + 0x30] %asi, %l0 /* %l0/%l1 = src4 + 0x30 */
ldda [%i0 + 0x30] %asi, %l2 /* %l2/%l3 = dest + 0x30 */
prefetch [%i1 + 0x40], #one_read
prefetch [%l7 + 0x40], #one_read
prefetch [%l6 + 0x40], #one_read
prefetch [%l5 + 0x40], #one_read
prefetch [%i0 + 0x40], #n_writes
xor %i4, %i2, %i4
xor %i5, %i3, %i5
xor %g2, %i4, %g2
xor %g3, %i5, %g3
xor %l0, %g2, %l0
xor %l1, %g3, %l1
xor %l2, %l0, %l2
xor %l3, %l1, %l3
stxa %l2, [%i0 + 0x30] %asi
stxa %l3, [%i0 + 0x38] %asi
add %i0, 0x40, %i0
add %i1, 0x40, %i1
add %l7, 0x40, %l7
add %l6, 0x40, %l6
subcc %g1, 1, %g1
bne,pt %xcc, 1b
add %l5, 0x40, %l5
membar #Sync
wr %g7, 0x0, %asi
ret
restore
ENDPROC(xor_niagara_5)
linux-7.2.3/lib/raid/xor/sparc/xor_arch.h
+35
View File
@@ -0,0 +1,35 @@
/* SPDX-License-Identifier: GPL-2.0 */
/*
* Copyright (C) 1997, 1999 Jakub Jelinek (jj@ultra.linux.cz)
* Copyright (C) 2006 David S. Miller <davem@davemloft.net>
*/
#if defined(__sparc__) && defined(__arch64__)
#include <asm/spitfire.h>
extern struct xor_block_template xor_block_VIS;
extern struct xor_block_template xor_block_niagara;
static __always_inline void __init arch_xor_init(void)
{
/* Force VIS for everything except Niagara. */
if (tlb_type == hypervisor &&
(sun4v_chip_type == SUN4V_CHIP_NIAGARA1 ||
sun4v_chip_type == SUN4V_CHIP_NIAGARA2 ||
sun4v_chip_type == SUN4V_CHIP_NIAGARA3 ||
sun4v_chip_type == SUN4V_CHIP_NIAGARA4 ||
sun4v_chip_type == SUN4V_CHIP_NIAGARA5))
xor_force(&xor_block_niagara);
else
xor_force(&xor_block_VIS);
}
#else /* sparc64 */
extern struct xor_block_template xor_block_SPARC;
static __always_inline void __init arch_xor_init(void)
{
xor_register(&xor_block_8regs);
xor_register(&xor_block_32regs);
xor_register(&xor_block_SPARC);
}
#endif /* !sparc64 */
linux-7.2.3/lib/raid/xor/tests/Makefile
+3
View File
@@ -0,0 +1,3 @@
# SPDX-License-Identifier: GPL-2.0-only
obj-$(CONFIG_XOR_KUNIT_TEST) += xor_kunit.o
linux-7.2.3/lib/raid/xor/tests/xor_kunit.c
+187
View File
@@ -0,0 +1,187 @@
// SPDX-License-Identifier: GPL-2.0-or-later
/*
* Unit test the XOR library functions.
*
* Copyright 2024 Google LLC
* Copyright 2026 Christoph Hellwig
*
* Based on the CRC tests by Eric Biggers <ebiggers@google.com>.
*/
#include <kunit/test.h>
#include <linux/prandom.h>
#include <linux/string_choices.h>
#include <linux/vmalloc.h>
#include <linux/raid/xor.h>
#define XOR_KUNIT_SEED 42
#define XOR_KUNIT_MAX_BYTES 16384
#define XOR_KUNIT_MAX_BUFFERS 64
#define XOR_KUNIT_NUM_TEST_ITERS 1000
static struct rnd_state rng;
static void *test_buffers[XOR_KUNIT_MAX_BUFFERS];
static void *test_dest;
static void *test_ref;
static size_t test_buflen;
static u32 rand32(void)
{
return prandom_u32_state(&rng);
}
/* Reference implementation using dumb byte-wise XOR */
static void xor_ref(void *dest, void **srcs, unsigned int src_cnt,
unsigned int bytes)
{
unsigned int off, idx;
u8 *d = dest;
for (off = 0; off < bytes; off++) {
for (idx = 0; idx < src_cnt; idx++) {
u8 *src = srcs[idx];
d[off] ^= src[off];
}
}
}
/* Generate a random length that is a multiple of 512. */
static unsigned int random_length(unsigned int max_length)
{
return round_up((rand32() % max_length) + 1, 512);
}
/* Generate a random alignment that is a multiple of 64. */
static unsigned int random_alignment(unsigned int max_alignment)
{
return ((rand32() % max_alignment) + 1) & ~63;
}
static void xor_generate_random_data(void)
{
int i;
prandom_bytes_state(&rng, test_dest, test_buflen);
memcpy(test_ref, test_dest, test_buflen);
for (i = 0; i < XOR_KUNIT_MAX_BUFFERS; i++)
prandom_bytes_state(&rng, test_buffers[i], test_buflen);
}
/* Test that xor_gen gives the same result as a reference implementation. */
static void xor_test(struct kunit *test)
{
void *aligned_buffers[XOR_KUNIT_MAX_BUFFERS];
size_t i;
for (i = 0; i < XOR_KUNIT_NUM_TEST_ITERS; i++) {
unsigned int nr_buffers =
(rand32() % XOR_KUNIT_MAX_BUFFERS) + 1;
unsigned int len = random_length(XOR_KUNIT_MAX_BYTES);
unsigned int max_alignment, align = 0;
void *buffers;
if (rand32() % 8 == 0)
/* Refresh the data occasionally. */
xor_generate_random_data();
/*
* If we're not using the entire buffer size, inject randomize
* alignment into the buffer.
*/
max_alignment = XOR_KUNIT_MAX_BYTES - len;
if (max_alignment == 0) {
buffers = test_buffers;
} else if (rand32() % 2 == 0) {
/* Use random alignments mod 64 */
int j;
for (j = 0; j < nr_buffers; j++)
aligned_buffers[j] = test_buffers[j] +
random_alignment(max_alignment);
buffers = aligned_buffers;
align = random_alignment(max_alignment);
} else {
/* Go up to the guard page, to catch buffer overreads */
int j;
align = test_buflen - len;
for (j = 0; j < nr_buffers; j++)
aligned_buffers[j] = test_buffers[j] + align;
buffers = aligned_buffers;
}
/*
* Compute the XOR, and verify that it equals the XOR computed
* by a simple byte-at-a-time reference implementation.
*/
xor_ref(test_ref + align, buffers, nr_buffers, len);
xor_gen(test_dest + align, buffers, nr_buffers, len);
KUNIT_EXPECT_MEMEQ_MSG(test, test_ref + align,
test_dest + align, len,
"Wrong result with buffers=%u, len=%u, unaligned=%s, at_end=%s",
nr_buffers, len,
str_yes_no(max_alignment),
str_yes_no(align + len == test_buflen));
}
}
static struct kunit_case xor_test_cases[] = {
KUNIT_CASE(xor_test),
{},
};
static int xor_suite_init(struct kunit_suite *suite)
{
int i;
/*
* Allocate the test buffer using vmalloc() with a page-aligned length
* so that it is immediately followed by a guard page. This allows
* buffer overreads to be detected, even in assembly code.
*/
test_buflen = round_up(XOR_KUNIT_MAX_BYTES, PAGE_SIZE);
test_ref = vmalloc(test_buflen);
if (!test_ref)
return -ENOMEM;
test_dest = vmalloc(test_buflen);
if (!test_dest)
goto out_free_ref;
for (i = 0; i < XOR_KUNIT_MAX_BUFFERS; i++) {
test_buffers[i] = vmalloc(test_buflen);
if (!test_buffers[i])
goto out_free_buffers;
}
prandom_seed_state(&rng, XOR_KUNIT_SEED);
xor_generate_random_data();
return 0;
out_free_buffers:
while (--i >= 0)
vfree(test_buffers[i]);
vfree(test_dest);
out_free_ref:
vfree(test_ref);
return -ENOMEM;
}
static void xor_suite_exit(struct kunit_suite *suite)
{
int i;
vfree(test_ref);
vfree(test_dest);
for (i = 0; i < XOR_KUNIT_MAX_BUFFERS; i++)
vfree(test_buffers[i]);
}
static struct kunit_suite xor_test_suite = {
.name = "xor",
.test_cases = xor_test_cases,
.suite_init = xor_suite_init,
.suite_exit = xor_suite_exit,
};
kunit_test_suite(xor_test_suite);
MODULE_DESCRIPTION("Unit test for the XOR library functions");
MODULE_LICENSE("GPL");
linux-7.2.3/lib/raid/xor/um/xor_arch.h
+2
View File
@@ -0,0 +1,2 @@
/* SPDX-License-Identifier: GPL-2.0 */
#include <../x86/xor_arch.h>
linux-7.2.3/lib/raid/xor/x86/xor-avx.c
+156
View File
@@ -0,0 +1,156 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Optimized XOR parity functions for AVX
*
* Copyright (C) 2012 Intel Corporation
* Author: Jim Kukunas <james.t.kukunas@linux.intel.com>
*
* Based on Ingo Molnar and Zach Brown's respective MMX and SSE routines
*/
#include <linux/compiler.h>
#include <asm/fpu/api.h>
#include "xor_impl.h"
#include "xor_arch.h"
#define BLOCK4(i) \
BLOCK(32 * i, 0) \
BLOCK(32 * (i + 1), 1) \
BLOCK(32 * (i + 2), 2) \
BLOCK(32 * (i + 3), 3)
#define BLOCK16() \
BLOCK4(0) \
BLOCK4(4) \
BLOCK4(8) \
BLOCK4(12)
static void xor_avx_2(unsigned long bytes, unsigned long * __restrict p0,
const unsigned long * __restrict p1)
{
unsigned long lines = bytes >> 9;
while (lines--) {
#undef BLOCK
#define BLOCK(i, reg) \
do { \
asm volatile("vmovdqa %0, %%ymm" #reg : : "m" (p1[i / sizeof(*p1)])); \
asm volatile("vxorps %0, %%ymm" #reg ", %%ymm" #reg : : \
"m" (p0[i / sizeof(*p0)])); \
asm volatile("vmovdqa %%ymm" #reg ", %0" : \
"=m" (p0[i / sizeof(*p0)])); \
} while (0);
BLOCK16()
p0 = (unsigned long *)((uintptr_t)p0 + 512);
p1 = (unsigned long *)((uintptr_t)p1 + 512);
}
}
static void xor_avx_3(unsigned long bytes, unsigned long * __restrict p0,
const unsigned long * __restrict p1,
const unsigned long * __restrict p2)
{
unsigned long lines = bytes >> 9;
while (lines--) {
#undef BLOCK
#define BLOCK(i, reg) \
do { \
asm volatile("vmovdqa %0, %%ymm" #reg : : "m" (p2[i / sizeof(*p2)])); \
asm volatile("vxorps %0, %%ymm" #reg ", %%ymm" #reg : : \
"m" (p1[i / sizeof(*p1)])); \
asm volatile("vxorps %0, %%ymm" #reg ", %%ymm" #reg : : \
"m" (p0[i / sizeof(*p0)])); \
asm volatile("vmovdqa %%ymm" #reg ", %0" : \
"=m" (p0[i / sizeof(*p0)])); \
} while (0);
BLOCK16()
p0 = (unsigned long *)((uintptr_t)p0 + 512);
p1 = (unsigned long *)((uintptr_t)p1 + 512);
p2 = (unsigned long *)((uintptr_t)p2 + 512);
}
}
static void xor_avx_4(unsigned long bytes, unsigned long * __restrict p0,
const unsigned long * __restrict p1,
const unsigned long * __restrict p2,
const unsigned long * __restrict p3)
{
unsigned long lines = bytes >> 9;
while (lines--) {
#undef BLOCK
#define BLOCK(i, reg) \
do { \
asm volatile("vmovdqa %0, %%ymm" #reg : : "m" (p3[i / sizeof(*p3)])); \
asm volatile("vxorps %0, %%ymm" #reg ", %%ymm" #reg : : \
"m" (p2[i / sizeof(*p2)])); \
asm volatile("vxorps %0, %%ymm" #reg ", %%ymm" #reg : : \
"m" (p1[i / sizeof(*p1)])); \
asm volatile("vxorps %0, %%ymm" #reg ", %%ymm" #reg : : \
"m" (p0[i / sizeof(*p0)])); \
asm volatile("vmovdqa %%ymm" #reg ", %0" : \
"=m" (p0[i / sizeof(*p0)])); \
} while (0);
BLOCK16();
p0 = (unsigned long *)((uintptr_t)p0 + 512);
p1 = (unsigned long *)((uintptr_t)p1 + 512);
p2 = (unsigned long *)((uintptr_t)p2 + 512);
p3 = (unsigned long *)((uintptr_t)p3 + 512);
}
}
static void xor_avx_5(unsigned long bytes, unsigned long * __restrict p0,
const unsigned long * __restrict p1,
const unsigned long * __restrict p2,
const unsigned long * __restrict p3,
const unsigned long * __restrict p4)
{
unsigned long lines = bytes >> 9;
while (lines--) {
#undef BLOCK
#define BLOCK(i, reg) \
do { \
asm volatile("vmovdqa %0, %%ymm" #reg : : "m" (p4[i / sizeof(*p4)])); \
asm volatile("vxorps %0, %%ymm" #reg ", %%ymm" #reg : : \
"m" (p3[i / sizeof(*p3)])); \
asm volatile("vxorps %0, %%ymm" #reg ", %%ymm" #reg : : \
"m" (p2[i / sizeof(*p2)])); \
asm volatile("vxorps %0, %%ymm" #reg ", %%ymm" #reg : : \
"m" (p1[i / sizeof(*p1)])); \
asm volatile("vxorps %0, %%ymm" #reg ", %%ymm" #reg : : \
"m" (p0[i / sizeof(*p0)])); \
asm volatile("vmovdqa %%ymm" #reg ", %0" : \
"=m" (p0[i / sizeof(*p0)])); \
} while (0);
BLOCK16()
p0 = (unsigned long *)((uintptr_t)p0 + 512);
p1 = (unsigned long *)((uintptr_t)p1 + 512);
p2 = (unsigned long *)((uintptr_t)p2 + 512);
p3 = (unsigned long *)((uintptr_t)p3 + 512);
p4 = (unsigned long *)((uintptr_t)p4 + 512);
}
}
DO_XOR_BLOCKS(avx_inner, xor_avx_2, xor_avx_3, xor_avx_4, xor_avx_5);
static void xor_gen_avx(void *dest, void **srcs, unsigned int src_cnt,
unsigned int bytes)
{
kernel_fpu_begin();
xor_gen_avx_inner(dest, srcs, src_cnt, bytes);
kernel_fpu_end();
}
struct xor_block_template xor_block_avx = {
.name = "avx",
.xor_gen = xor_gen_avx,
};
linux-7.2.3/lib/raid/xor/x86/xor-mmx.c
+515
View File
@@ -0,0 +1,515 @@
// SPDX-License-Identifier: GPL-2.0-or-later
/*
* Optimized XOR parity functions for MMX.
*
* Copyright (C) 1998 Ingo Molnar.
*/
#include <asm/fpu/api.h>
#include "xor_impl.h"
#include "xor_arch.h"
#define LD(x, y) " movq 8*("#x")(%1), %%mm"#y" ;\n"
#define ST(x, y) " movq %%mm"#y", 8*("#x")(%1) ;\n"
#define XO1(x, y) " pxor 8*("#x")(%2), %%mm"#y" ;\n"
#define XO2(x, y) " pxor 8*("#x")(%3), %%mm"#y" ;\n"
#define XO3(x, y) " pxor 8*("#x")(%4), %%mm"#y" ;\n"
#define XO4(x, y) " pxor 8*("#x")(%5), %%mm"#y" ;\n"
static void
xor_pII_mmx_2(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2)
{
unsigned long lines = bytes >> 7;
asm volatile(
#undef BLOCK
#define BLOCK(i) \
LD(i, 0) \
LD(i + 1, 1) \
LD(i + 2, 2) \
LD(i + 3, 3) \
XO1(i, 0) \
ST(i, 0) \
XO1(i+1, 1) \
ST(i+1, 1) \
XO1(i + 2, 2) \
ST(i + 2, 2) \
XO1(i + 3, 3) \
ST(i + 3, 3)
" .align 32 ;\n"
" 1: ;\n"
BLOCK(0)
BLOCK(4)
BLOCK(8)
BLOCK(12)
" addl $128, %1 ;\n"
" addl $128, %2 ;\n"
" decl %0 ;\n"
" jnz 1b ;\n"
: "+r" (lines),
"+r" (p1), "+r" (p2)
:
: "memory");
}
static void
xor_pII_mmx_3(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2,
const unsigned long * __restrict p3)
{
unsigned long lines = bytes >> 7;
asm volatile(
#undef BLOCK
#define BLOCK(i) \
LD(i, 0) \
LD(i + 1, 1) \
LD(i + 2, 2) \
LD(i + 3, 3) \
XO1(i, 0) \
XO1(i + 1, 1) \
XO1(i + 2, 2) \
XO1(i + 3, 3) \
XO2(i, 0) \
ST(i, 0) \
XO2(i + 1, 1) \
ST(i + 1, 1) \
XO2(i + 2, 2) \
ST(i + 2, 2) \
XO2(i + 3, 3) \
ST(i + 3, 3)
" .align 32 ;\n"
" 1: ;\n"
BLOCK(0)
BLOCK(4)
BLOCK(8)
BLOCK(12)
" addl $128, %1 ;\n"
" addl $128, %2 ;\n"
" addl $128, %3 ;\n"
" decl %0 ;\n"
" jnz 1b ;\n"
: "+r" (lines),
"+r" (p1), "+r" (p2), "+r" (p3)
:
: "memory");
}
static void
xor_pII_mmx_4(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2,
const unsigned long * __restrict p3,
const unsigned long * __restrict p4)
{
unsigned long lines = bytes >> 7;
asm volatile(
#undef BLOCK
#define BLOCK(i) \
LD(i, 0) \
LD(i + 1, 1) \
LD(i + 2, 2) \
LD(i + 3, 3) \
XO1(i, 0) \
XO1(i + 1, 1) \
XO1(i + 2, 2) \
XO1(i + 3, 3) \
XO2(i, 0) \
XO2(i + 1, 1) \
XO2(i + 2, 2) \
XO2(i + 3, 3) \
XO3(i, 0) \
ST(i, 0) \
XO3(i + 1, 1) \
ST(i + 1, 1) \
XO3(i + 2, 2) \
ST(i + 2, 2) \
XO3(i + 3, 3) \
ST(i + 3, 3)
" .align 32 ;\n"
" 1: ;\n"
BLOCK(0)
BLOCK(4)
BLOCK(8)
BLOCK(12)
" addl $128, %1 ;\n"
" addl $128, %2 ;\n"
" addl $128, %3 ;\n"
" addl $128, %4 ;\n"
" decl %0 ;\n"
" jnz 1b ;\n"
: "+r" (lines),
"+r" (p1), "+r" (p2), "+r" (p3), "+r" (p4)
:
: "memory");
}
static void
xor_pII_mmx_5(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2,
const unsigned long * __restrict p3,
const unsigned long * __restrict p4,
const unsigned long * __restrict p5)
{
unsigned long lines = bytes >> 7;
/* Make sure GCC forgets anything it knows about p4 or p5,
such that it won't pass to the asm volatile below a
register that is shared with any other variable. That's
because we modify p4 and p5 there, but we can't mark them
as read/write, otherwise we'd overflow the 10-asm-operands
limit of GCC < 3.1. */
asm("" : "+r" (p4), "+r" (p5));
asm volatile(
#undef BLOCK
#define BLOCK(i) \
LD(i, 0) \
LD(i + 1, 1) \
LD(i + 2, 2) \
LD(i + 3, 3) \
XO1(i, 0) \
XO1(i + 1, 1) \
XO1(i + 2, 2) \
XO1(i + 3, 3) \
XO2(i, 0) \
XO2(i + 1, 1) \
XO2(i + 2, 2) \
XO2(i + 3, 3) \
XO3(i, 0) \
XO3(i + 1, 1) \
XO3(i + 2, 2) \
XO3(i + 3, 3) \
XO4(i, 0) \
ST(i, 0) \
XO4(i + 1, 1) \
ST(i + 1, 1) \
XO4(i + 2, 2) \
ST(i + 2, 2) \
XO4(i + 3, 3) \
ST(i + 3, 3)
" .align 32 ;\n"
" 1: ;\n"
BLOCK(0)
BLOCK(4)
BLOCK(8)
BLOCK(12)
" addl $128, %1 ;\n"
" addl $128, %2 ;\n"
" addl $128, %3 ;\n"
" addl $128, %4 ;\n"
" addl $128, %5 ;\n"
" decl %0 ;\n"
" jnz 1b ;\n"
: "+r" (lines),
"+r" (p1), "+r" (p2), "+r" (p3)
: "r" (p4), "r" (p5)
: "memory");
/* p4 and p5 were modified, and now the variables are dead.
Clobber them just to be sure nobody does something stupid
like assuming they have some legal value. */
asm("" : "=r" (p4), "=r" (p5));
}
#undef LD
#undef XO1
#undef XO2
#undef XO3
#undef XO4
#undef ST
#undef BLOCK
static void
xor_p5_mmx_2(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2)
{
unsigned long lines = bytes >> 6;
asm volatile(
" .align 32 ;\n"
" 1: ;\n"
" movq (%1), %%mm0 ;\n"
" movq 8(%1), %%mm1 ;\n"
" pxor (%2), %%mm0 ;\n"
" movq 16(%1), %%mm2 ;\n"
" movq %%mm0, (%1) ;\n"
" pxor 8(%2), %%mm1 ;\n"
" movq 24(%1), %%mm3 ;\n"
" movq %%mm1, 8(%1) ;\n"
" pxor 16(%2), %%mm2 ;\n"
" movq 32(%1), %%mm4 ;\n"
" movq %%mm2, 16(%1) ;\n"
" pxor 24(%2), %%mm3 ;\n"
" movq 40(%1), %%mm5 ;\n"
" movq %%mm3, 24(%1) ;\n"
" pxor 32(%2), %%mm4 ;\n"
" movq 48(%1), %%mm6 ;\n"
" movq %%mm4, 32(%1) ;\n"
" pxor 40(%2), %%mm5 ;\n"
" movq 56(%1), %%mm7 ;\n"
" movq %%mm5, 40(%1) ;\n"
" pxor 48(%2), %%mm6 ;\n"
" pxor 56(%2), %%mm7 ;\n"
" movq %%mm6, 48(%1) ;\n"
" movq %%mm7, 56(%1) ;\n"
" addl $64, %1 ;\n"
" addl $64, %2 ;\n"
" decl %0 ;\n"
" jnz 1b ;\n"
: "+r" (lines),
"+r" (p1), "+r" (p2)
:
: "memory");
}
static void
xor_p5_mmx_3(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2,
const unsigned long * __restrict p3)
{
unsigned long lines = bytes >> 6;
asm volatile(
" .align 32,0x90 ;\n"
" 1: ;\n"
" movq (%1), %%mm0 ;\n"
" movq 8(%1), %%mm1 ;\n"
" pxor (%2), %%mm0 ;\n"
" movq 16(%1), %%mm2 ;\n"
" pxor 8(%2), %%mm1 ;\n"
" pxor (%3), %%mm0 ;\n"
" pxor 16(%2), %%mm2 ;\n"
" movq %%mm0, (%1) ;\n"
" pxor 8(%3), %%mm1 ;\n"
" pxor 16(%3), %%mm2 ;\n"
" movq 24(%1), %%mm3 ;\n"
" movq %%mm1, 8(%1) ;\n"
" movq 32(%1), %%mm4 ;\n"
" movq 40(%1), %%mm5 ;\n"
" pxor 24(%2), %%mm3 ;\n"
" movq %%mm2, 16(%1) ;\n"
" pxor 32(%2), %%mm4 ;\n"
" pxor 24(%3), %%mm3 ;\n"
" pxor 40(%2), %%mm5 ;\n"
" movq %%mm3, 24(%1) ;\n"
" pxor 32(%3), %%mm4 ;\n"
" pxor 40(%3), %%mm5 ;\n"
" movq 48(%1), %%mm6 ;\n"
" movq %%mm4, 32(%1) ;\n"
" movq 56(%1), %%mm7 ;\n"
" pxor 48(%2), %%mm6 ;\n"
" movq %%mm5, 40(%1) ;\n"
" pxor 56(%2), %%mm7 ;\n"
" pxor 48(%3), %%mm6 ;\n"
" pxor 56(%3), %%mm7 ;\n"
" movq %%mm6, 48(%1) ;\n"
" movq %%mm7, 56(%1) ;\n"
" addl $64, %1 ;\n"
" addl $64, %2 ;\n"
" addl $64, %3 ;\n"
" decl %0 ;\n"
" jnz 1b ;\n"
: "+r" (lines),
"+r" (p1), "+r" (p2), "+r" (p3)
:
: "memory" );
}
static void
xor_p5_mmx_4(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2,
const unsigned long * __restrict p3,
const unsigned long * __restrict p4)
{
unsigned long lines = bytes >> 6;
asm volatile(
" .align 32,0x90 ;\n"
" 1: ;\n"
" movq (%1), %%mm0 ;\n"
" movq 8(%1), %%mm1 ;\n"
" pxor (%2), %%mm0 ;\n"
" movq 16(%1), %%mm2 ;\n"
" pxor 8(%2), %%mm1 ;\n"
" pxor (%3), %%mm0 ;\n"
" pxor 16(%2), %%mm2 ;\n"
" pxor 8(%3), %%mm1 ;\n"
" pxor (%4), %%mm0 ;\n"
" movq 24(%1), %%mm3 ;\n"
" pxor 16(%3), %%mm2 ;\n"
" pxor 8(%4), %%mm1 ;\n"
" movq %%mm0, (%1) ;\n"
" movq 32(%1), %%mm4 ;\n"
" pxor 24(%2), %%mm3 ;\n"
" pxor 16(%4), %%mm2 ;\n"
" movq %%mm1, 8(%1) ;\n"
" movq 40(%1), %%mm5 ;\n"
" pxor 32(%2), %%mm4 ;\n"
" pxor 24(%3), %%mm3 ;\n"
" movq %%mm2, 16(%1) ;\n"
" pxor 40(%2), %%mm5 ;\n"
" pxor 32(%3), %%mm4 ;\n"
" pxor 24(%4), %%mm3 ;\n"
" movq %%mm3, 24(%1) ;\n"
" movq 56(%1), %%mm7 ;\n"
" movq 48(%1), %%mm6 ;\n"
" pxor 40(%3), %%mm5 ;\n"
" pxor 32(%4), %%mm4 ;\n"
" pxor 48(%2), %%mm6 ;\n"
" movq %%mm4, 32(%1) ;\n"
" pxor 56(%2), %%mm7 ;\n"
" pxor 40(%4), %%mm5 ;\n"
" pxor 48(%3), %%mm6 ;\n"
" pxor 56(%3), %%mm7 ;\n"
" movq %%mm5, 40(%1) ;\n"
" pxor 48(%4), %%mm6 ;\n"
" pxor 56(%4), %%mm7 ;\n"
" movq %%mm6, 48(%1) ;\n"
" movq %%mm7, 56(%1) ;\n"
" addl $64, %1 ;\n"
" addl $64, %2 ;\n"
" addl $64, %3 ;\n"
" addl $64, %4 ;\n"
" decl %0 ;\n"
" jnz 1b ;\n"
: "+r" (lines),
"+r" (p1), "+r" (p2), "+r" (p3), "+r" (p4)
:
: "memory");
}
static void
xor_p5_mmx_5(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2,
const unsigned long * __restrict p3,
const unsigned long * __restrict p4,
const unsigned long * __restrict p5)
{
unsigned long lines = bytes >> 6;
/* Make sure GCC forgets anything it knows about p4 or p5,
such that it won't pass to the asm volatile below a
register that is shared with any other variable. That's
because we modify p4 and p5 there, but we can't mark them
as read/write, otherwise we'd overflow the 10-asm-operands
limit of GCC < 3.1. */
asm("" : "+r" (p4), "+r" (p5));
asm volatile(
" .align 32,0x90 ;\n"
" 1: ;\n"
" movq (%1), %%mm0 ;\n"
" movq 8(%1), %%mm1 ;\n"
" pxor (%2), %%mm0 ;\n"
" pxor 8(%2), %%mm1 ;\n"
" movq 16(%1), %%mm2 ;\n"
" pxor (%3), %%mm0 ;\n"
" pxor 8(%3), %%mm1 ;\n"
" pxor 16(%2), %%mm2 ;\n"
" pxor (%4), %%mm0 ;\n"
" pxor 8(%4), %%mm1 ;\n"
" pxor 16(%3), %%mm2 ;\n"
" movq 24(%1), %%mm3 ;\n"
" pxor (%5), %%mm0 ;\n"
" pxor 8(%5), %%mm1 ;\n"
" movq %%mm0, (%1) ;\n"
" pxor 16(%4), %%mm2 ;\n"
" pxor 24(%2), %%mm3 ;\n"
" movq %%mm1, 8(%1) ;\n"
" pxor 16(%5), %%mm2 ;\n"
" pxor 24(%3), %%mm3 ;\n"
" movq 32(%1), %%mm4 ;\n"
" movq %%mm2, 16(%1) ;\n"
" pxor 24(%4), %%mm3 ;\n"
" pxor 32(%2), %%mm4 ;\n"
" movq 40(%1), %%mm5 ;\n"
" pxor 24(%5), %%mm3 ;\n"
" pxor 32(%3), %%mm4 ;\n"
" pxor 40(%2), %%mm5 ;\n"
" movq %%mm3, 24(%1) ;\n"
" pxor 32(%4), %%mm4 ;\n"
" pxor 40(%3), %%mm5 ;\n"
" movq 48(%1), %%mm6 ;\n"
" movq 56(%1), %%mm7 ;\n"
" pxor 32(%5), %%mm4 ;\n"
" pxor 40(%4), %%mm5 ;\n"
" pxor 48(%2), %%mm6 ;\n"
" pxor 56(%2), %%mm7 ;\n"
" movq %%mm4, 32(%1) ;\n"
" pxor 48(%3), %%mm6 ;\n"
" pxor 56(%3), %%mm7 ;\n"
" pxor 40(%5), %%mm5 ;\n"
" pxor 48(%4), %%mm6 ;\n"
" pxor 56(%4), %%mm7 ;\n"
" movq %%mm5, 40(%1) ;\n"
" pxor 48(%5), %%mm6 ;\n"
" pxor 56(%5), %%mm7 ;\n"
" movq %%mm6, 48(%1) ;\n"
" movq %%mm7, 56(%1) ;\n"
" addl $64, %1 ;\n"
" addl $64, %2 ;\n"
" addl $64, %3 ;\n"
" addl $64, %4 ;\n"
" addl $64, %5 ;\n"
" decl %0 ;\n"
" jnz 1b ;\n"
: "+r" (lines),
"+r" (p1), "+r" (p2), "+r" (p3)
: "r" (p4), "r" (p5)
: "memory");
/* p4 and p5 were modified, and now the variables are dead.
Clobber them just to be sure nobody does something stupid
like assuming they have some legal value. */
asm("" : "=r" (p4), "=r" (p5));
}
DO_XOR_BLOCKS(pII_mmx_inner, xor_pII_mmx_2, xor_pII_mmx_3, xor_pII_mmx_4,
xor_pII_mmx_5);
static void xor_gen_pII_mmx(void *dest, void **srcs, unsigned int src_cnt,
unsigned int bytes)
{
kernel_fpu_begin();
xor_gen_pII_mmx_inner(dest, srcs, src_cnt, bytes);
kernel_fpu_end();
}
struct xor_block_template xor_block_pII_mmx = {
.name = "pII_mmx",
.xor_gen = xor_gen_pII_mmx,
};
DO_XOR_BLOCKS(p5_mmx_inner, xor_p5_mmx_2, xor_p5_mmx_3, xor_p5_mmx_4,
xor_p5_mmx_5);
static void xor_gen_p5_mmx(void *dest, void **srcs, unsigned int src_cnt,
unsigned int bytes)
{
kernel_fpu_begin();
xor_gen_p5_mmx_inner(dest, srcs, src_cnt, bytes);
kernel_fpu_end();
}
struct xor_block_template xor_block_p5_mmx = {
.name = "p5_mmx",
.xor_gen = xor_gen_p5_mmx,
};
linux-7.2.3/lib/raid/xor/x86/xor-sse.c
+459
View File
@@ -0,0 +1,459 @@
// SPDX-License-Identifier: GPL-2.0-or-later
/*
* Optimized XOR parity functions for SSE.
*
* Cache avoiding checksumming functions utilizing KNI instructions
* Copyright (C) 1999 Zach Brown (with obvious credit due Ingo)
*
* Based on
* High-speed RAID5 checksumming functions utilizing SSE instructions.
* Copyright (C) 1998 Ingo Molnar.
*
* x86-64 changes / gcc fixes from Andi Kleen.
* Copyright 2002 Andi Kleen, SuSE Labs.
*/
#include <asm/fpu/api.h>
#include "xor_impl.h"
#include "xor_arch.h"
#ifdef CONFIG_X86_32
/* reduce register pressure */
# define XOR_CONSTANT_CONSTRAINT "i"
#else
# define XOR_CONSTANT_CONSTRAINT "re"
#endif
#define OFFS(x) "16*("#x")"
#define PF_OFFS(x) "256+16*("#x")"
#define PF0(x) " prefetchnta "PF_OFFS(x)"(%[p1]) ;\n"
#define LD(x, y) " movaps "OFFS(x)"(%[p1]), %%xmm"#y" ;\n"
#define ST(x, y) " movaps %%xmm"#y", "OFFS(x)"(%[p1]) ;\n"
#define PF1(x) " prefetchnta "PF_OFFS(x)"(%[p2]) ;\n"
#define PF2(x) " prefetchnta "PF_OFFS(x)"(%[p3]) ;\n"
#define PF3(x) " prefetchnta "PF_OFFS(x)"(%[p4]) ;\n"
#define PF4(x) " prefetchnta "PF_OFFS(x)"(%[p5]) ;\n"
#define XO1(x, y) " xorps "OFFS(x)"(%[p2]), %%xmm"#y" ;\n"
#define XO2(x, y) " xorps "OFFS(x)"(%[p3]), %%xmm"#y" ;\n"
#define XO3(x, y) " xorps "OFFS(x)"(%[p4]), %%xmm"#y" ;\n"
#define XO4(x, y) " xorps "OFFS(x)"(%[p5]), %%xmm"#y" ;\n"
#define NOP(x)
#define BLK64(pf, op, i) \
pf(i) \
op(i, 0) \
op(i + 1, 1) \
op(i + 2, 2) \
op(i + 3, 3)
static void
xor_sse_2(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2)
{
unsigned long lines = bytes >> 8;
asm volatile(
#undef BLOCK
#define BLOCK(i) \
LD(i, 0) \
LD(i + 1, 1) \
PF1(i) \
PF1(i + 2) \
LD(i + 2, 2) \
LD(i + 3, 3) \
PF0(i + 4) \
PF0(i + 6) \
XO1(i, 0) \
XO1(i + 1, 1) \
XO1(i + 2, 2) \
XO1(i + 3, 3) \
ST(i, 0) \
ST(i + 1, 1) \
ST(i + 2, 2) \
ST(i + 3, 3) \
PF0(0)
PF0(2)
" .align 32 ;\n"
" 1: ;\n"
BLOCK(0)
BLOCK(4)
BLOCK(8)
BLOCK(12)
" add %[inc], %[p1] ;\n"
" add %[inc], %[p2] ;\n"
" dec %[cnt] ;\n"
" jnz 1b ;\n"
: [cnt] "+r" (lines),
[p1] "+r" (p1), [p2] "+r" (p2)
: [inc] XOR_CONSTANT_CONSTRAINT (256UL)
: "memory");
}
static void
xor_sse_2_pf64(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2)
{
unsigned long lines = bytes >> 8;
asm volatile(
#undef BLOCK
#define BLOCK(i) \
BLK64(PF0, LD, i) \
BLK64(PF1, XO1, i) \
BLK64(NOP, ST, i) \
" .align 32 ;\n"
" 1: ;\n"
BLOCK(0)
BLOCK(4)
BLOCK(8)
BLOCK(12)
" add %[inc], %[p1] ;\n"
" add %[inc], %[p2] ;\n"
" dec %[cnt] ;\n"
" jnz 1b ;\n"
: [cnt] "+r" (lines),
[p1] "+r" (p1), [p2] "+r" (p2)
: [inc] XOR_CONSTANT_CONSTRAINT (256UL)
: "memory");
}
static void
xor_sse_3(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2,
const unsigned long * __restrict p3)
{
unsigned long lines = bytes >> 8;
asm volatile(
#undef BLOCK
#define BLOCK(i) \
PF1(i) \
PF1(i + 2) \
LD(i, 0) \
LD(i + 1, 1) \
LD(i + 2, 2) \
LD(i + 3, 3) \
PF2(i) \
PF2(i + 2) \
PF0(i + 4) \
PF0(i + 6) \
XO1(i, 0) \
XO1(i + 1, 1) \
XO1(i + 2, 2) \
XO1(i + 3, 3) \
XO2(i, 0) \
XO2(i + 1, 1) \
XO2(i + 2, 2) \
XO2(i + 3, 3) \
ST(i, 0) \
ST(i + 1, 1) \
ST(i + 2, 2) \
ST(i + 3, 3) \
PF0(0)
PF0(2)
" .align 32 ;\n"
" 1: ;\n"
BLOCK(0)
BLOCK(4)
BLOCK(8)
BLOCK(12)
" add %[inc], %[p1] ;\n"
" add %[inc], %[p2] ;\n"
" add %[inc], %[p3] ;\n"
" dec %[cnt] ;\n"
" jnz 1b ;\n"
: [cnt] "+r" (lines),
[p1] "+r" (p1), [p2] "+r" (p2), [p3] "+r" (p3)
: [inc] XOR_CONSTANT_CONSTRAINT (256UL)
: "memory");
}
static void
xor_sse_3_pf64(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2,
const unsigned long * __restrict p3)
{
unsigned long lines = bytes >> 8;
asm volatile(
#undef BLOCK
#define BLOCK(i) \
BLK64(PF0, LD, i) \
BLK64(PF1, XO1, i) \
BLK64(PF2, XO2, i) \
BLK64(NOP, ST, i) \
" .align 32 ;\n"
" 1: ;\n"
BLOCK(0)
BLOCK(4)
BLOCK(8)
BLOCK(12)
" add %[inc], %[p1] ;\n"
" add %[inc], %[p2] ;\n"
" add %[inc], %[p3] ;\n"
" dec %[cnt] ;\n"
" jnz 1b ;\n"
: [cnt] "+r" (lines),
[p1] "+r" (p1), [p2] "+r" (p2), [p3] "+r" (p3)
: [inc] XOR_CONSTANT_CONSTRAINT (256UL)
: "memory");
}
static void
xor_sse_4(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2,
const unsigned long * __restrict p3,
const unsigned long * __restrict p4)
{
unsigned long lines = bytes >> 8;
asm volatile(
#undef BLOCK
#define BLOCK(i) \
PF1(i) \
PF1(i + 2) \
LD(i, 0) \
LD(i + 1, 1) \
LD(i + 2, 2) \
LD(i + 3, 3) \
PF2(i) \
PF2(i + 2) \
XO1(i, 0) \
XO1(i + 1, 1) \
XO1(i + 2, 2) \
XO1(i + 3, 3) \
PF3(i) \
PF3(i + 2) \
PF0(i + 4) \
PF0(i + 6) \
XO2(i, 0) \
XO2(i + 1, 1) \
XO2(i + 2, 2) \
XO2(i + 3, 3) \
XO3(i, 0) \
XO3(i + 1, 1) \
XO3(i + 2, 2) \
XO3(i + 3, 3) \
ST(i, 0) \
ST(i + 1, 1) \
ST(i + 2, 2) \
ST(i + 3, 3) \
PF0(0)
PF0(2)
" .align 32 ;\n"
" 1: ;\n"
BLOCK(0)
BLOCK(4)
BLOCK(8)
BLOCK(12)
" add %[inc], %[p1] ;\n"
" add %[inc], %[p2] ;\n"
" add %[inc], %[p3] ;\n"
" add %[inc], %[p4] ;\n"
" dec %[cnt] ;\n"
" jnz 1b ;\n"
: [cnt] "+r" (lines), [p1] "+r" (p1),
[p2] "+r" (p2), [p3] "+r" (p3), [p4] "+r" (p4)
: [inc] XOR_CONSTANT_CONSTRAINT (256UL)
: "memory");
}
static void
xor_sse_4_pf64(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2,
const unsigned long * __restrict p3,
const unsigned long * __restrict p4)
{
unsigned long lines = bytes >> 8;
asm volatile(
#undef BLOCK
#define BLOCK(i) \
BLK64(PF0, LD, i) \
BLK64(PF1, XO1, i) \
BLK64(PF2, XO2, i) \
BLK64(PF3, XO3, i) \
BLK64(NOP, ST, i) \
" .align 32 ;\n"
" 1: ;\n"
BLOCK(0)
BLOCK(4)
BLOCK(8)
BLOCK(12)
" add %[inc], %[p1] ;\n"
" add %[inc], %[p2] ;\n"
" add %[inc], %[p3] ;\n"
" add %[inc], %[p4] ;\n"
" dec %[cnt] ;\n"
" jnz 1b ;\n"
: [cnt] "+r" (lines), [p1] "+r" (p1),
[p2] "+r" (p2), [p3] "+r" (p3), [p4] "+r" (p4)
: [inc] XOR_CONSTANT_CONSTRAINT (256UL)
: "memory");
}
static void
xor_sse_5(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2,
const unsigned long * __restrict p3,
const unsigned long * __restrict p4,
const unsigned long * __restrict p5)
{
unsigned long lines = bytes >> 8;
asm volatile(
#undef BLOCK
#define BLOCK(i) \
PF1(i) \
PF1(i + 2) \
LD(i, 0) \
LD(i + 1, 1) \
LD(i + 2, 2) \
LD(i + 3, 3) \
PF2(i) \
PF2(i + 2) \
XO1(i, 0) \
XO1(i + 1, 1) \
XO1(i + 2, 2) \
XO1(i + 3, 3) \
PF3(i) \
PF3(i + 2) \
XO2(i, 0) \
XO2(i + 1, 1) \
XO2(i + 2, 2) \
XO2(i + 3, 3) \
PF4(i) \
PF4(i + 2) \
PF0(i + 4) \
PF0(i + 6) \
XO3(i, 0) \
XO3(i + 1, 1) \
XO3(i + 2, 2) \
XO3(i + 3, 3) \
XO4(i, 0) \
XO4(i + 1, 1) \
XO4(i + 2, 2) \
XO4(i + 3, 3) \
ST(i, 0) \
ST(i + 1, 1) \
ST(i + 2, 2) \
ST(i + 3, 3) \
PF0(0)
PF0(2)
" .align 32 ;\n"
" 1: ;\n"
BLOCK(0)
BLOCK(4)
BLOCK(8)
BLOCK(12)
" add %[inc], %[p1] ;\n"
" add %[inc], %[p2] ;\n"
" add %[inc], %[p3] ;\n"
" add %[inc], %[p4] ;\n"
" add %[inc], %[p5] ;\n"
" dec %[cnt] ;\n"
" jnz 1b ;\n"
: [cnt] "+r" (lines), [p1] "+r" (p1), [p2] "+r" (p2),
[p3] "+r" (p3), [p4] "+r" (p4), [p5] "+r" (p5)
: [inc] XOR_CONSTANT_CONSTRAINT (256UL)
: "memory");
}
static void
xor_sse_5_pf64(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2,
const unsigned long * __restrict p3,
const unsigned long * __restrict p4,
const unsigned long * __restrict p5)
{
unsigned long lines = bytes >> 8;
asm volatile(
#undef BLOCK
#define BLOCK(i) \
BLK64(PF0, LD, i) \
BLK64(PF1, XO1, i) \
BLK64(PF2, XO2, i) \
BLK64(PF3, XO3, i) \
BLK64(PF4, XO4, i) \
BLK64(NOP, ST, i) \
" .align 32 ;\n"
" 1: ;\n"
BLOCK(0)
BLOCK(4)
BLOCK(8)
BLOCK(12)
" add %[inc], %[p1] ;\n"
" add %[inc], %[p2] ;\n"
" add %[inc], %[p3] ;\n"
" add %[inc], %[p4] ;\n"
" add %[inc], %[p5] ;\n"
" dec %[cnt] ;\n"
" jnz 1b ;\n"
: [cnt] "+r" (lines), [p1] "+r" (p1), [p2] "+r" (p2),
[p3] "+r" (p3), [p4] "+r" (p4), [p5] "+r" (p5)
: [inc] XOR_CONSTANT_CONSTRAINT (256UL)
: "memory");
}
DO_XOR_BLOCKS(sse_inner, xor_sse_2, xor_sse_3, xor_sse_4, xor_sse_5);
static void xor_gen_sse(void *dest, void **srcs, unsigned int src_cnt,
unsigned int bytes)
{
kernel_fpu_begin();
xor_gen_sse_inner(dest, srcs, src_cnt, bytes);
kernel_fpu_end();
}
struct xor_block_template xor_block_sse = {
.name = "sse",
.xor_gen = xor_gen_sse,
};
DO_XOR_BLOCKS(sse_pf64_inner, xor_sse_2_pf64, xor_sse_3_pf64, xor_sse_4_pf64,
xor_sse_5_pf64);
static void xor_gen_sse_pf64(void *dest, void **srcs, unsigned int src_cnt,
unsigned int bytes)
{
kernel_fpu_begin();
xor_gen_sse_pf64_inner(dest, srcs, src_cnt, bytes);
kernel_fpu_end();
}
struct xor_block_template xor_block_sse_pf64 = {
.name = "prefetch64-sse",
.xor_gen = xor_gen_sse_pf64,
};
linux-7.2.3/lib/raid/xor/x86/xor_arch.h
+36
View File
@@ -0,0 +1,36 @@
/* SPDX-License-Identifier: GPL-2.0-or-later */
#include <asm/cpufeature.h>
extern struct xor_block_template xor_block_pII_mmx;
extern struct xor_block_template xor_block_p5_mmx;
extern struct xor_block_template xor_block_sse;
extern struct xor_block_template xor_block_sse_pf64;
extern struct xor_block_template xor_block_avx;
/*
* When SSE is available, use it as it can write around L2. We may also be able
* to load into the L1 only depending on how the cpu deals with a load to a line
* that is being prefetched.
*
* When AVX2 is available, force using it as it is better by all measures.
*
* 32-bit without MMX can fall back to the generic routines.
*/
static __always_inline void __init arch_xor_init(void)
{
if (boot_cpu_has(X86_FEATURE_AVX) &&
boot_cpu_has(X86_FEATURE_OSXSAVE)) {
xor_force(&xor_block_avx);
} else if (IS_ENABLED(CONFIG_X86_64) || boot_cpu_has(X86_FEATURE_XMM)) {
xor_register(&xor_block_sse);
xor_register(&xor_block_sse_pf64);
} else if (boot_cpu_has(X86_FEATURE_MMX)) {
xor_register(&xor_block_pII_mmx);
xor_register(&xor_block_p5_mmx);
} else {
xor_register(&xor_block_8regs);
xor_register(&xor_block_8regs_p);
xor_register(&xor_block_32regs);
xor_register(&xor_block_32regs_p);
}
}
linux-7.2.3/lib/raid/xor/xor-32regs-prefetch.c
+267
View File
@@ -0,0 +1,267 @@
// SPDX-License-Identifier: GPL-2.0-or-later
#include <linux/prefetch.h>
#include "xor_impl.h"
static void
xor_32regs_p_2(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2)
{
long lines = bytes / (sizeof (long)) / 8 - 1;
prefetchw(p1);
prefetch(p2);
do {
register long d0, d1, d2, d3, d4, d5, d6, d7;
prefetchw(p1+8);
prefetch(p2+8);
once_more:
d0 = p1[0]; /* Pull the stuff into registers */
d1 = p1[1]; /* ... in bursts, if possible. */
d2 = p1[2];
d3 = p1[3];
d4 = p1[4];
d5 = p1[5];
d6 = p1[6];
d7 = p1[7];
d0 ^= p2[0];
d1 ^= p2[1];
d2 ^= p2[2];
d3 ^= p2[3];
d4 ^= p2[4];
d5 ^= p2[5];
d6 ^= p2[6];
d7 ^= p2[7];
p1[0] = d0; /* Store the result (in bursts) */
p1[1] = d1;
p1[2] = d2;
p1[3] = d3;
p1[4] = d4;
p1[5] = d5;
p1[6] = d6;
p1[7] = d7;
p1 += 8;
p2 += 8;
} while (--lines > 0);
if (lines == 0)
goto once_more;
}
static void
xor_32regs_p_3(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2,
const unsigned long * __restrict p3)
{
long lines = bytes / (sizeof (long)) / 8 - 1;
prefetchw(p1);
prefetch(p2);
prefetch(p3);
do {
register long d0, d1, d2, d3, d4, d5, d6, d7;
prefetchw(p1+8);
prefetch(p2+8);
prefetch(p3+8);
once_more:
d0 = p1[0]; /* Pull the stuff into registers */
d1 = p1[1]; /* ... in bursts, if possible. */
d2 = p1[2];
d3 = p1[3];
d4 = p1[4];
d5 = p1[5];
d6 = p1[6];
d7 = p1[7];
d0 ^= p2[0];
d1 ^= p2[1];
d2 ^= p2[2];
d3 ^= p2[3];
d4 ^= p2[4];
d5 ^= p2[5];
d6 ^= p2[6];
d7 ^= p2[7];
d0 ^= p3[0];
d1 ^= p3[1];
d2 ^= p3[2];
d3 ^= p3[3];
d4 ^= p3[4];
d5 ^= p3[5];
d6 ^= p3[6];
d7 ^= p3[7];
p1[0] = d0; /* Store the result (in bursts) */
p1[1] = d1;
p1[2] = d2;
p1[3] = d3;
p1[4] = d4;
p1[5] = d5;
p1[6] = d6;
p1[7] = d7;
p1 += 8;
p2 += 8;
p3 += 8;
} while (--lines > 0);
if (lines == 0)
goto once_more;
}
static void
xor_32regs_p_4(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2,
const unsigned long * __restrict p3,
const unsigned long * __restrict p4)
{
long lines = bytes / (sizeof (long)) / 8 - 1;
prefetchw(p1);
prefetch(p2);
prefetch(p3);
prefetch(p4);
do {
register long d0, d1, d2, d3, d4, d5, d6, d7;
prefetchw(p1+8);
prefetch(p2+8);
prefetch(p3+8);
prefetch(p4+8);
once_more:
d0 = p1[0]; /* Pull the stuff into registers */
d1 = p1[1]; /* ... in bursts, if possible. */
d2 = p1[2];
d3 = p1[3];
d4 = p1[4];
d5 = p1[5];
d6 = p1[6];
d7 = p1[7];
d0 ^= p2[0];
d1 ^= p2[1];
d2 ^= p2[2];
d3 ^= p2[3];
d4 ^= p2[4];
d5 ^= p2[5];
d6 ^= p2[6];
d7 ^= p2[7];
d0 ^= p3[0];
d1 ^= p3[1];
d2 ^= p3[2];
d3 ^= p3[3];
d4 ^= p3[4];
d5 ^= p3[5];
d6 ^= p3[6];
d7 ^= p3[7];
d0 ^= p4[0];
d1 ^= p4[1];
d2 ^= p4[2];
d3 ^= p4[3];
d4 ^= p4[4];
d5 ^= p4[5];
d6 ^= p4[6];
d7 ^= p4[7];
p1[0] = d0; /* Store the result (in bursts) */
p1[1] = d1;
p1[2] = d2;
p1[3] = d3;
p1[4] = d4;
p1[5] = d5;
p1[6] = d6;
p1[7] = d7;
p1 += 8;
p2 += 8;
p3 += 8;
p4 += 8;
} while (--lines > 0);
if (lines == 0)
goto once_more;
}
static void
xor_32regs_p_5(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2,
const unsigned long * __restrict p3,
const unsigned long * __restrict p4,
const unsigned long * __restrict p5)
{
long lines = bytes / (sizeof (long)) / 8 - 1;
prefetchw(p1);
prefetch(p2);
prefetch(p3);
prefetch(p4);
prefetch(p5);
do {
register long d0, d1, d2, d3, d4, d5, d6, d7;
prefetchw(p1+8);
prefetch(p2+8);
prefetch(p3+8);
prefetch(p4+8);
prefetch(p5+8);
once_more:
d0 = p1[0]; /* Pull the stuff into registers */
d1 = p1[1]; /* ... in bursts, if possible. */
d2 = p1[2];
d3 = p1[3];
d4 = p1[4];
d5 = p1[5];
d6 = p1[6];
d7 = p1[7];
d0 ^= p2[0];
d1 ^= p2[1];
d2 ^= p2[2];
d3 ^= p2[3];
d4 ^= p2[4];
d5 ^= p2[5];
d6 ^= p2[6];
d7 ^= p2[7];
d0 ^= p3[0];
d1 ^= p3[1];
d2 ^= p3[2];
d3 ^= p3[3];
d4 ^= p3[4];
d5 ^= p3[5];
d6 ^= p3[6];
d7 ^= p3[7];
d0 ^= p4[0];
d1 ^= p4[1];
d2 ^= p4[2];
d3 ^= p4[3];
d4 ^= p4[4];
d5 ^= p4[5];
d6 ^= p4[6];
d7 ^= p4[7];
d0 ^= p5[0];
d1 ^= p5[1];
d2 ^= p5[2];
d3 ^= p5[3];
d4 ^= p5[4];
d5 ^= p5[5];
d6 ^= p5[6];
d7 ^= p5[7];
p1[0] = d0; /* Store the result (in bursts) */
p1[1] = d1;
p1[2] = d2;
p1[3] = d3;
p1[4] = d4;
p1[5] = d5;
p1[6] = d6;
p1[7] = d7;
p1 += 8;
p2 += 8;
p3 += 8;
p4 += 8;
p5 += 8;
} while (--lines > 0);
if (lines == 0)
goto once_more;
}
DO_XOR_BLOCKS(32regs_p, xor_32regs_p_2, xor_32regs_p_3, xor_32regs_p_4,
xor_32regs_p_5);
struct xor_block_template xor_block_32regs_p = {
.name = "32regs_prefetch",
.xor_gen = xor_gen_32regs_p,
};
linux-7.2.3/lib/raid/xor/xor-32regs.c
+217
View File
@@ -0,0 +1,217 @@
// SPDX-License-Identifier: GPL-2.0-or-later
#include "xor_impl.h"
static void
xor_32regs_2(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2)
{
long lines = bytes / (sizeof (long)) / 8;
do {
register long d0, d1, d2, d3, d4, d5, d6, d7;
d0 = p1[0]; /* Pull the stuff into registers */
d1 = p1[1]; /* ... in bursts, if possible. */
d2 = p1[2];
d3 = p1[3];
d4 = p1[4];
d5 = p1[5];
d6 = p1[6];
d7 = p1[7];
d0 ^= p2[0];
d1 ^= p2[1];
d2 ^= p2[2];
d3 ^= p2[3];
d4 ^= p2[4];
d5 ^= p2[5];
d6 ^= p2[6];
d7 ^= p2[7];
p1[0] = d0; /* Store the result (in bursts) */
p1[1] = d1;
p1[2] = d2;
p1[3] = d3;
p1[4] = d4;
p1[5] = d5;
p1[6] = d6;
p1[7] = d7;
p1 += 8;
p2 += 8;
} while (--lines > 0);
}
static void
xor_32regs_3(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2,
const unsigned long * __restrict p3)
{
long lines = bytes / (sizeof (long)) / 8;
do {
register long d0, d1, d2, d3, d4, d5, d6, d7;
d0 = p1[0]; /* Pull the stuff into registers */
d1 = p1[1]; /* ... in bursts, if possible. */
d2 = p1[2];
d3 = p1[3];
d4 = p1[4];
d5 = p1[5];
d6 = p1[6];
d7 = p1[7];
d0 ^= p2[0];
d1 ^= p2[1];
d2 ^= p2[2];
d3 ^= p2[3];
d4 ^= p2[4];
d5 ^= p2[5];
d6 ^= p2[6];
d7 ^= p2[7];
d0 ^= p3[0];
d1 ^= p3[1];
d2 ^= p3[2];
d3 ^= p3[3];
d4 ^= p3[4];
d5 ^= p3[5];
d6 ^= p3[6];
d7 ^= p3[7];
p1[0] = d0; /* Store the result (in bursts) */
p1[1] = d1;
p1[2] = d2;
p1[3] = d3;
p1[4] = d4;
p1[5] = d5;
p1[6] = d6;
p1[7] = d7;
p1 += 8;
p2 += 8;
p3 += 8;
} while (--lines > 0);
}
static void
xor_32regs_4(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2,
const unsigned long * __restrict p3,
const unsigned long * __restrict p4)
{
long lines = bytes / (sizeof (long)) / 8;
do {
register long d0, d1, d2, d3, d4, d5, d6, d7;
d0 = p1[0]; /* Pull the stuff into registers */
d1 = p1[1]; /* ... in bursts, if possible. */
d2 = p1[2];
d3 = p1[3];
d4 = p1[4];
d5 = p1[5];
d6 = p1[6];
d7 = p1[7];
d0 ^= p2[0];
d1 ^= p2[1];
d2 ^= p2[2];
d3 ^= p2[3];
d4 ^= p2[4];
d5 ^= p2[5];
d6 ^= p2[6];
d7 ^= p2[7];
d0 ^= p3[0];
d1 ^= p3[1];
d2 ^= p3[2];
d3 ^= p3[3];
d4 ^= p3[4];
d5 ^= p3[5];
d6 ^= p3[6];
d7 ^= p3[7];
d0 ^= p4[0];
d1 ^= p4[1];
d2 ^= p4[2];
d3 ^= p4[3];
d4 ^= p4[4];
d5 ^= p4[5];
d6 ^= p4[6];
d7 ^= p4[7];
p1[0] = d0; /* Store the result (in bursts) */
p1[1] = d1;
p1[2] = d2;
p1[3] = d3;
p1[4] = d4;
p1[5] = d5;
p1[6] = d6;
p1[7] = d7;
p1 += 8;
p2 += 8;
p3 += 8;
p4 += 8;
} while (--lines > 0);
}
static void
xor_32regs_5(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2,
const unsigned long * __restrict p3,
const unsigned long * __restrict p4,
const unsigned long * __restrict p5)
{
long lines = bytes / (sizeof (long)) / 8;
do {
register long d0, d1, d2, d3, d4, d5, d6, d7;
d0 = p1[0]; /* Pull the stuff into registers */
d1 = p1[1]; /* ... in bursts, if possible. */
d2 = p1[2];
d3 = p1[3];
d4 = p1[4];
d5 = p1[5];
d6 = p1[6];
d7 = p1[7];
d0 ^= p2[0];
d1 ^= p2[1];
d2 ^= p2[2];
d3 ^= p2[3];
d4 ^= p2[4];
d5 ^= p2[5];
d6 ^= p2[6];
d7 ^= p2[7];
d0 ^= p3[0];
d1 ^= p3[1];
d2 ^= p3[2];
d3 ^= p3[3];
d4 ^= p3[4];
d5 ^= p3[5];
d6 ^= p3[6];
d7 ^= p3[7];
d0 ^= p4[0];
d1 ^= p4[1];
d2 ^= p4[2];
d3 ^= p4[3];
d4 ^= p4[4];
d5 ^= p4[5];
d6 ^= p4[6];
d7 ^= p4[7];
d0 ^= p5[0];
d1 ^= p5[1];
d2 ^= p5[2];
d3 ^= p5[3];
d4 ^= p5[4];
d5 ^= p5[5];
d6 ^= p5[6];
d7 ^= p5[7];
p1[0] = d0; /* Store the result (in bursts) */
p1[1] = d1;
p1[2] = d2;
p1[3] = d3;
p1[4] = d4;
p1[5] = d5;
p1[6] = d6;
p1[7] = d7;
p1 += 8;
p2 += 8;
p3 += 8;
p4 += 8;
p5 += 8;
} while (--lines > 0);
}
DO_XOR_BLOCKS(32regs, xor_32regs_2, xor_32regs_3, xor_32regs_4, xor_32regs_5);
struct xor_block_template xor_block_32regs = {
.name = "32regs",
.xor_gen = xor_gen_32regs,
};
linux-7.2.3/lib/raid/xor/xor-8regs-prefetch.c
+146
View File
@@ -0,0 +1,146 @@
// SPDX-License-Identifier: GPL-2.0-or-later
#include <linux/prefetch.h>
#include "xor_impl.h"
static void
xor_8regs_p_2(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2)
{
long lines = bytes / (sizeof (long)) / 8 - 1;
prefetchw(p1);
prefetch(p2);
do {
prefetchw(p1+8);
prefetch(p2+8);
once_more:
p1[0] ^= p2[0];
p1[1] ^= p2[1];
p1[2] ^= p2[2];
p1[3] ^= p2[3];
p1[4] ^= p2[4];
p1[5] ^= p2[5];
p1[6] ^= p2[6];
p1[7] ^= p2[7];
p1 += 8;
p2 += 8;
} while (--lines > 0);
if (lines == 0)
goto once_more;
}
static void
xor_8regs_p_3(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2,
const unsigned long * __restrict p3)
{
long lines = bytes / (sizeof (long)) / 8 - 1;
prefetchw(p1);
prefetch(p2);
prefetch(p3);
do {
prefetchw(p1+8);
prefetch(p2+8);
prefetch(p3+8);
once_more:
p1[0] ^= p2[0] ^ p3[0];
p1[1] ^= p2[1] ^ p3[1];
p1[2] ^= p2[2] ^ p3[2];
p1[3] ^= p2[3] ^ p3[3];
p1[4] ^= p2[4] ^ p3[4];
p1[5] ^= p2[5] ^ p3[5];
p1[6] ^= p2[6] ^ p3[6];
p1[7] ^= p2[7] ^ p3[7];
p1 += 8;
p2 += 8;
p3 += 8;
} while (--lines > 0);
if (lines == 0)
goto once_more;
}
static void
xor_8regs_p_4(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2,
const unsigned long * __restrict p3,
const unsigned long * __restrict p4)
{
long lines = bytes / (sizeof (long)) / 8 - 1;
prefetchw(p1);
prefetch(p2);
prefetch(p3);
prefetch(p4);
do {
prefetchw(p1+8);
prefetch(p2+8);
prefetch(p3+8);
prefetch(p4+8);
once_more:
p1[0] ^= p2[0] ^ p3[0] ^ p4[0];
p1[1] ^= p2[1] ^ p3[1] ^ p4[1];
p1[2] ^= p2[2] ^ p3[2] ^ p4[2];
p1[3] ^= p2[3] ^ p3[3] ^ p4[3];
p1[4] ^= p2[4] ^ p3[4] ^ p4[4];
p1[5] ^= p2[5] ^ p3[5] ^ p4[5];
p1[6] ^= p2[6] ^ p3[6] ^ p4[6];
p1[7] ^= p2[7] ^ p3[7] ^ p4[7];
p1 += 8;
p2 += 8;
p3 += 8;
p4 += 8;
} while (--lines > 0);
if (lines == 0)
goto once_more;
}
static void
xor_8regs_p_5(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2,
const unsigned long * __restrict p3,
const unsigned long * __restrict p4,
const unsigned long * __restrict p5)
{
long lines = bytes / (sizeof (long)) / 8 - 1;
prefetchw(p1);
prefetch(p2);
prefetch(p3);
prefetch(p4);
prefetch(p5);
do {
prefetchw(p1+8);
prefetch(p2+8);
prefetch(p3+8);
prefetch(p4+8);
prefetch(p5+8);
once_more:
p1[0] ^= p2[0] ^ p3[0] ^ p4[0] ^ p5[0];
p1[1] ^= p2[1] ^ p3[1] ^ p4[1] ^ p5[1];
p1[2] ^= p2[2] ^ p3[2] ^ p4[2] ^ p5[2];
p1[3] ^= p2[3] ^ p3[3] ^ p4[3] ^ p5[3];
p1[4] ^= p2[4] ^ p3[4] ^ p4[4] ^ p5[4];
p1[5] ^= p2[5] ^ p3[5] ^ p4[5] ^ p5[5];
p1[6] ^= p2[6] ^ p3[6] ^ p4[6] ^ p5[6];
p1[7] ^= p2[7] ^ p3[7] ^ p4[7] ^ p5[7];
p1 += 8;
p2 += 8;
p3 += 8;
p4 += 8;
p5 += 8;
} while (--lines > 0);
if (lines == 0)
goto once_more;
}
DO_XOR_BLOCKS(8regs_p, xor_8regs_p_2, xor_8regs_p_3, xor_8regs_p_4,
xor_8regs_p_5);
struct xor_block_template xor_block_8regs_p = {
.name = "8regs_prefetch",
.xor_gen = xor_gen_8regs_p,
};
linux-7.2.3/lib/raid/xor/xor-8regs.c
+101
View File
@@ -0,0 +1,101 @@
// SPDX-License-Identifier: GPL-2.0-or-later
#include "xor_impl.h"
static void
xor_8regs_2(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2)
{
long lines = bytes / (sizeof (long)) / 8;
do {
p1[0] ^= p2[0];
p1[1] ^= p2[1];
p1[2] ^= p2[2];
p1[3] ^= p2[3];
p1[4] ^= p2[4];
p1[5] ^= p2[5];
p1[6] ^= p2[6];
p1[7] ^= p2[7];
p1 += 8;
p2 += 8;
} while (--lines > 0);
}
static void
xor_8regs_3(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2,
const unsigned long * __restrict p3)
{
long lines = bytes / (sizeof (long)) / 8;
do {
p1[0] ^= p2[0] ^ p3[0];
p1[1] ^= p2[1] ^ p3[1];
p1[2] ^= p2[2] ^ p3[2];
p1[3] ^= p2[3] ^ p3[3];
p1[4] ^= p2[4] ^ p3[4];
p1[5] ^= p2[5] ^ p3[5];
p1[6] ^= p2[6] ^ p3[6];
p1[7] ^= p2[7] ^ p3[7];
p1 += 8;
p2 += 8;
p3 += 8;
} while (--lines > 0);
}
static void
xor_8regs_4(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2,
const unsigned long * __restrict p3,
const unsigned long * __restrict p4)
{
long lines = bytes / (sizeof (long)) / 8;
do {
p1[0] ^= p2[0] ^ p3[0] ^ p4[0];
p1[1] ^= p2[1] ^ p3[1] ^ p4[1];
p1[2] ^= p2[2] ^ p3[2] ^ p4[2];
p1[3] ^= p2[3] ^ p3[3] ^ p4[3];
p1[4] ^= p2[4] ^ p3[4] ^ p4[4];
p1[5] ^= p2[5] ^ p3[5] ^ p4[5];
p1[6] ^= p2[6] ^ p3[6] ^ p4[6];
p1[7] ^= p2[7] ^ p3[7] ^ p4[7];
p1 += 8;
p2 += 8;
p3 += 8;
p4 += 8;
} while (--lines > 0);
}
static void
xor_8regs_5(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2,
const unsigned long * __restrict p3,
const unsigned long * __restrict p4,
const unsigned long * __restrict p5)
{
long lines = bytes / (sizeof (long)) / 8;
do {
p1[0] ^= p2[0] ^ p3[0] ^ p4[0] ^ p5[0];
p1[1] ^= p2[1] ^ p3[1] ^ p4[1] ^ p5[1];
p1[2] ^= p2[2] ^ p3[2] ^ p4[2] ^ p5[2];
p1[3] ^= p2[3] ^ p3[3] ^ p4[3] ^ p5[3];
p1[4] ^= p2[4] ^ p3[4] ^ p4[4] ^ p5[4];
p1[5] ^= p2[5] ^ p3[5] ^ p4[5] ^ p5[5];
p1[6] ^= p2[6] ^ p3[6] ^ p4[6] ^ p5[6];
p1[7] ^= p2[7] ^ p3[7] ^ p4[7] ^ p5[7];
p1 += 8;
p2 += 8;
p3 += 8;
p4 += 8;
p5 += 8;
} while (--lines > 0);
}
DO_XOR_BLOCKS(8regs, xor_8regs_2, xor_8regs_3, xor_8regs_4, xor_8regs_5);
struct xor_block_template xor_block_8regs = {
.name = "8regs",
.xor_gen = xor_gen_8regs,
};
linux-7.2.3/lib/raid/xor/xor-core.c
+194
View File
@@ -0,0 +1,194 @@
// SPDX-License-Identifier: GPL-2.0-or-later
/*
* Copyright (C) 1996, 1997, 1998, 1999, 2000,
* Ingo Molnar, Matti Aarnio, Jakub Jelinek, Richard Henderson.
*
* Dispatch optimized XOR parity functions.
*/
#include <linux/module.h>
#include <linux/gfp.h>
#include <linux/slab.h>
#include <linux/raid/xor.h>
#include <linux/jiffies.h>
#include <linux/preempt.h>
#include <linux/static_call.h>
#include "xor_impl.h"
DEFINE_STATIC_CALL_NULL(xor_gen_impl, *xor_block_8regs.xor_gen);
/**
* xor_gen - generate RAID-style XOR information
* @dest: destination vector
* @srcs: source vectors
* @src_cnt: number of source vectors
* @bytes: length in bytes of each vector
*
* Performs bit-wise XOR operation into @dest for each of the @src_cnt vectors
* in @srcs for a length of @bytes bytes. @src_cnt must be non-zero, and the
* memory pointed to by @dest and each member of @srcs must be at least 64-byte
* aligned. @bytes must be non-zero and a multiple of 512.
*
* Note: for typical RAID uses, @dest either needs to be zeroed, or filled with
* the first disk, which then needs to be removed from @srcs.
*/
void xor_gen(void *dest, void **srcs, unsigned int src_cnt, unsigned int bytes)
{
WARN_ON_ONCE(!in_task() || irqs_disabled() || softirq_count());
WARN_ON_ONCE(bytes == 0);
WARN_ON_ONCE(bytes & 511);
static_call(xor_gen_impl)(dest, srcs, src_cnt, bytes);
}
EXPORT_SYMBOL(xor_gen);
/* Set of all registered templates. */
static struct xor_block_template *__initdata template_list;
static struct xor_block_template *forced_template;
/**
* xor_register - register a XOR template
* @tmpl: template to register
*
* Register a XOR implementation with the core. Registered implementations
* will be measured by a trivial benchmark, and the fastest one is chosen
* unless an implementation is forced using xor_force().
*/
void __init xor_register(struct xor_block_template *tmpl)
{
tmpl->next = template_list;
template_list = tmpl;
}
/**
* xor_force - force use of a XOR template
* @tmpl: template to register
*
* Register a XOR implementation with the core and force using it. Forcing
* an implementation will make the core ignore any template registered using
* xor_register(), or any previous implementation forced using xor_force().
*/
void __init xor_force(struct xor_block_template *tmpl)
{
forced_template = tmpl;
}
#define BENCH_SIZE 4096
#define REPS 800U
static void __init
do_xor_speed(struct xor_block_template *tmpl, void *b1, void *b2)
{
int speed;
unsigned long reps;
ktime_t min, start, t0;
void *srcs[1] = { b2 };
preempt_disable();
reps = 0;
t0 = ktime_get();
/* delay start until time has advanced */
while ((start = ktime_get()) == t0)
cpu_relax();
do {
mb(); /* prevent loop optimization */
tmpl->xor_gen(b1, srcs, 1, BENCH_SIZE);
mb();
} while (reps++ < REPS || (t0 = ktime_get()) == start);
min = ktime_sub(t0, start);
preempt_enable();
// bytes/ns == GB/s, multiply by 1000 to get MB/s [not MiB/s]
speed = (1000 * reps * BENCH_SIZE) / (unsigned int)ktime_to_ns(min);
tmpl->speed = speed;
pr_info(" %-16s: %5d MB/sec\n", tmpl->name, speed);
}
static int __init calibrate_xor_blocks(void)
{
void *b1, *b2;
struct xor_block_template *f, *fastest;
if (forced_template)
return 0;
b1 = kmalloc(PAGE_SIZE * 4, GFP_KERNEL);
if (!b1) {
pr_warn("xor: Yikes! No memory available.\n");
return -ENOMEM;
}
b2 = b1 + 2*PAGE_SIZE + BENCH_SIZE;
pr_info("xor: measuring software checksum speed\n");
fastest = template_list;
for (f = template_list; f; f = f->next) {
do_xor_speed(f, b1, b2);
if (f->speed > fastest->speed)
fastest = f;
}
static_call_update(xor_gen_impl, fastest->xor_gen);
pr_info("xor: using function: %s (%d MB/sec)\n",
fastest->name, fastest->speed);
kfree(b1);
return 0;
}
#ifdef CONFIG_XOR_BLOCKS_ARCH
#include "xor_arch.h" /* $SRCARCH/xor_arch.h */
#else
static void __init arch_xor_init(void)
{
xor_register(&xor_block_8regs);
xor_register(&xor_block_8regs_p);
xor_register(&xor_block_32regs);
xor_register(&xor_block_32regs_p);
}
#endif /* CONFIG_XOR_BLOCKS_ARCH */
static int __init xor_init(void)
{
arch_xor_init();
/*
* If this arch/cpu has a short-circuited selection, don't loop through
* all the possible functions, just use the best one.
*/
if (forced_template) {
pr_info("xor: automatically using best checksumming function %-10s\n",
forced_template->name);
static_call_update(xor_gen_impl, forced_template->xor_gen);
return 0;
}
#ifdef MODULE
return calibrate_xor_blocks();
#else
/*
* Pick the first template as the temporary default until calibration
* happens.
*/
static_call_update(xor_gen_impl, template_list->xor_gen);
return 0;
#endif
}
static __exit void xor_exit(void)
{
}
MODULE_DESCRIPTION("RAID-5 checksumming functions");
MODULE_LICENSE("GPL");
/*
* When built-in we must register the default template before md, but we don't
* want calibration to run that early as that would delay the boot process.
*/
#ifndef MODULE
__initcall(calibrate_xor_blocks);
#endif
core_initcall(xor_init);
module_exit(xor_exit);
linux-7.2.3/lib/raid/xor/xor-neon.c
+179
View File
@@ -0,0 +1,179 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Authors: Jackie Liu <liuyun01@kylinos.cn>
* Copyright (C) 2018,Tianjin KYLIN Information Technology Co., Ltd.
*/
#include "xor_impl.h"
#include "xor-neon.h"
#include <asm/neon-intrinsics.h>
static void __xor_neon_2(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2)
{
uint64_t *dp1 = (uint64_t *)p1;
uint64_t *dp2 = (uint64_t *)p2;
register uint64x2_t v0, v1, v2, v3;
long lines = bytes / (sizeof(uint64x2_t) * 4);
do {
/* p1 ^= p2 */
v0 = veorq_u64(vld1q_u64(dp1 + 0), vld1q_u64(dp2 + 0));
v1 = veorq_u64(vld1q_u64(dp1 + 2), vld1q_u64(dp2 + 2));
v2 = veorq_u64(vld1q_u64(dp1 + 4), vld1q_u64(dp2 + 4));
v3 = veorq_u64(vld1q_u64(dp1 + 6), vld1q_u64(dp2 + 6));
/* store */
vst1q_u64(dp1 + 0, v0);
vst1q_u64(dp1 + 2, v1);
vst1q_u64(dp1 + 4, v2);
vst1q_u64(dp1 + 6, v3);
dp1 += 8;
dp2 += 8;
} while (--lines > 0);
}
static void __xor_neon_3(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2,
const unsigned long * __restrict p3)
{
uint64_t *dp1 = (uint64_t *)p1;
uint64_t *dp2 = (uint64_t *)p2;
uint64_t *dp3 = (uint64_t *)p3;
register uint64x2_t v0, v1, v2, v3;
long lines = bytes / (sizeof(uint64x2_t) * 4);
do {
/* p1 ^= p2 */
v0 = veorq_u64(vld1q_u64(dp1 + 0), vld1q_u64(dp2 + 0));
v1 = veorq_u64(vld1q_u64(dp1 + 2), vld1q_u64(dp2 + 2));
v2 = veorq_u64(vld1q_u64(dp1 + 4), vld1q_u64(dp2 + 4));
v3 = veorq_u64(vld1q_u64(dp1 + 6), vld1q_u64(dp2 + 6));
/* p1 ^= p3 */
v0 = veorq_u64(v0, vld1q_u64(dp3 + 0));
v1 = veorq_u64(v1, vld1q_u64(dp3 + 2));
v2 = veorq_u64(v2, vld1q_u64(dp3 + 4));
v3 = veorq_u64(v3, vld1q_u64(dp3 + 6));
/* store */
vst1q_u64(dp1 + 0, v0);
vst1q_u64(dp1 + 2, v1);
vst1q_u64(dp1 + 4, v2);
vst1q_u64(dp1 + 6, v3);
dp1 += 8;
dp2 += 8;
dp3 += 8;
} while (--lines > 0);
}
static void __xor_neon_4(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2,
const unsigned long * __restrict p3,
const unsigned long * __restrict p4)
{
uint64_t *dp1 = (uint64_t *)p1;
uint64_t *dp2 = (uint64_t *)p2;
uint64_t *dp3 = (uint64_t *)p3;
uint64_t *dp4 = (uint64_t *)p4;
register uint64x2_t v0, v1, v2, v3;
long lines = bytes / (sizeof(uint64x2_t) * 4);
do {
/* p1 ^= p2 */
v0 = veorq_u64(vld1q_u64(dp1 + 0), vld1q_u64(dp2 + 0));
v1 = veorq_u64(vld1q_u64(dp1 + 2), vld1q_u64(dp2 + 2));
v2 = veorq_u64(vld1q_u64(dp1 + 4), vld1q_u64(dp2 + 4));
v3 = veorq_u64(vld1q_u64(dp1 + 6), vld1q_u64(dp2 + 6));
/* p1 ^= p3 */
v0 = veorq_u64(v0, vld1q_u64(dp3 + 0));
v1 = veorq_u64(v1, vld1q_u64(dp3 + 2));
v2 = veorq_u64(v2, vld1q_u64(dp3 + 4));
v3 = veorq_u64(v3, vld1q_u64(dp3 + 6));
/* p1 ^= p4 */
v0 = veorq_u64(v0, vld1q_u64(dp4 + 0));
v1 = veorq_u64(v1, vld1q_u64(dp4 + 2));
v2 = veorq_u64(v2, vld1q_u64(dp4 + 4));
v3 = veorq_u64(v3, vld1q_u64(dp4 + 6));
/* store */
vst1q_u64(dp1 + 0, v0);
vst1q_u64(dp1 + 2, v1);
vst1q_u64(dp1 + 4, v2);
vst1q_u64(dp1 + 6, v3);
dp1 += 8;
dp2 += 8;
dp3 += 8;
dp4 += 8;
} while (--lines > 0);
}
static void __xor_neon_5(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2,
const unsigned long * __restrict p3,
const unsigned long * __restrict p4,
const unsigned long * __restrict p5)
{
uint64_t *dp1 = (uint64_t *)p1;
uint64_t *dp2 = (uint64_t *)p2;
uint64_t *dp3 = (uint64_t *)p3;
uint64_t *dp4 = (uint64_t *)p4;
uint64_t *dp5 = (uint64_t *)p5;
register uint64x2_t v0, v1, v2, v3;
long lines = bytes / (sizeof(uint64x2_t) * 4);
do {
/* p1 ^= p2 */
v0 = veorq_u64(vld1q_u64(dp1 + 0), vld1q_u64(dp2 + 0));
v1 = veorq_u64(vld1q_u64(dp1 + 2), vld1q_u64(dp2 + 2));
v2 = veorq_u64(vld1q_u64(dp1 + 4), vld1q_u64(dp2 + 4));
v3 = veorq_u64(vld1q_u64(dp1 + 6), vld1q_u64(dp2 + 6));
/* p1 ^= p3 */
v0 = veorq_u64(v0, vld1q_u64(dp3 + 0));
v1 = veorq_u64(v1, vld1q_u64(dp3 + 2));
v2 = veorq_u64(v2, vld1q_u64(dp3 + 4));
v3 = veorq_u64(v3, vld1q_u64(dp3 + 6));
/* p1 ^= p4 */
v0 = veorq_u64(v0, vld1q_u64(dp4 + 0));
v1 = veorq_u64(v1, vld1q_u64(dp4 + 2));
v2 = veorq_u64(v2, vld1q_u64(dp4 + 4));
v3 = veorq_u64(v3, vld1q_u64(dp4 + 6));
/* p1 ^= p5 */
v0 = veorq_u64(v0, vld1q_u64(dp5 + 0));
v1 = veorq_u64(v1, vld1q_u64(dp5 + 2));
v2 = veorq_u64(v2, vld1q_u64(dp5 + 4));
v3 = veorq_u64(v3, vld1q_u64(dp5 + 6));
/* store */
vst1q_u64(dp1 + 0, v0);
vst1q_u64(dp1 + 2, v1);
vst1q_u64(dp1 + 4, v2);
vst1q_u64(dp1 + 6, v3);
dp1 += 8;
dp2 += 8;
dp3 += 8;
dp4 += 8;
dp5 += 8;
} while (--lines > 0);
}
__DO_XOR_BLOCKS(neon_inner, __xor_neon_2, __xor_neon_3, __xor_neon_4,
__xor_neon_5);
#ifdef CONFIG_ARM64
extern typeof(__xor_neon_2) __xor_eor3_2 __alias(__xor_neon_2);
#endif
linux-7.2.3/lib/raid/xor/xor_impl.h
+56
View File
@@ -0,0 +1,56 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _XOR_IMPL_H
#define _XOR_IMPL_H
#include <linux/init.h>
#include <linux/minmax.h>
struct xor_block_template {
struct xor_block_template *next;
const char *name;
int speed;
void (*xor_gen)(void *dest, void **srcs, unsigned int src_cnt,
unsigned int bytes);
};
#define __DO_XOR_BLOCKS(_name, _handle1, _handle2, _handle3, _handle4) \
void \
xor_gen_##_name(void *dest, void **srcs, unsigned int src_cnt, \
unsigned int bytes) \
{ \
unsigned int src_off = 0; \
\
while (src_cnt > 0) { \
unsigned int this_cnt = min(src_cnt, 4); \
\
if (this_cnt == 1) \
_handle1(bytes, dest, srcs[src_off]); \
else if (this_cnt == 2) \
_handle2(bytes, dest, srcs[src_off], \
srcs[src_off + 1]); \
else if (this_cnt == 3) \
_handle3(bytes, dest, srcs[src_off], \
srcs[src_off + 1], srcs[src_off + 2]); \
else \
_handle4(bytes, dest, srcs[src_off], \
srcs[src_off + 1], srcs[src_off + 2], \
srcs[src_off + 3]); \
\
src_cnt -= this_cnt; \
src_off += this_cnt; \
} \
}
#define DO_XOR_BLOCKS(_name, _handle1, _handle2, _handle3, _handle4) \
static __DO_XOR_BLOCKS(_name, _handle1, _handle2, _handle3, _handle4)
/* generic implementations */
extern struct xor_block_template xor_block_8regs;
extern struct xor_block_template xor_block_32regs;
extern struct xor_block_template xor_block_8regs_p;
extern struct xor_block_template xor_block_32regs_p;
void __init xor_register(struct xor_block_template *tmpl);
void __init xor_force(struct xor_block_template *tmpl);
#endif /* _XOR_IMPL_H */