37 lines
809 B
C
37 lines
809 B
C
/* SPDX-License-Identifier: GPL-2.0-only */
|
|
/*
|
|
* CRC64 using ARM PMULL instructions
|
|
*/
|
|
|
|
#include <asm/simd.h>
|
|
|
|
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_pmull);
|
|
|
|
u64 crc64_nvme_neon(u64 crc, const u8 *p, size_t len);
|
|
|
|
#define crc64_be_arch crc64_be_generic
|
|
|
|
static inline u64 crc64_nvme_arch(u64 crc, const u8 *p, size_t len)
|
|
{
|
|
if (len >= 128 && static_branch_likely(&have_pmull) &&
|
|
likely(may_use_simd())) {
|
|
do {
|
|
size_t chunk = min_t(size_t, len & ~15, SZ_4K);
|
|
|
|
scoped_ksimd()
|
|
crc = crc64_nvme_neon(crc, p, chunk);
|
|
|
|
p += chunk;
|
|
len -= chunk;
|
|
} while (len >= 128);
|
|
}
|
|
return crc64_nvme_generic(crc, p, len);
|
|
}
|
|
|
|
#define crc64_mod_init_arch crc64_mod_init_arch
|
|
static void crc64_mod_init_arch(void)
|
|
{
|
|
if (elf_hwcap2 & HWCAP2_PMULL)
|
|
static_branch_enable(&have_pmull);
|
|
}
|