lastdragon/linux-kernel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Resguardo inicial

Browse Source
This commit is contained in:
lastdragon
2026-09-06 12:53:21 -06:00
parent 662efe3c2a
commit ad2088d178
94853 changed files with 43200404 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
linux-7.2.3/drivers/crypto/cavium/Makefile
+6
View File
@@ -0,0 +1,6 @@
# SPDX-License-Identifier: GPL-2.0-only
#
# Makefile for Cavium crypto device drivers
#
obj-$(CONFIG_CRYPTO_DEV_CPT) += cpt/
obj-$(CONFIG_CRYPTO_DEV_NITROX) += nitrox/
linux-7.2.3/drivers/crypto/cavium/cpt/Kconfig
+18
View File
@@ -0,0 +1,18 @@
# SPDX-License-Identifier: GPL-2.0-only
#
# Cavium crypto device configuration
#
config CRYPTO_DEV_CPT
tristate
config CAVIUM_CPT
tristate "Cavium Cryptographic Accelerator driver"
depends on ARCH_THUNDER || COMPILE_TEST
depends on PCI_MSI && 64BIT
select CRYPTO_DEV_CPT
help
Support for Cavium CPT block found in octeon-tx series of
processors.
To compile this as a module, choose M here.
linux-7.2.3/drivers/crypto/cavium/cpt/Makefile
+4
View File
@@ -0,0 +1,4 @@
# SPDX-License-Identifier: GPL-2.0-only
obj-$(CONFIG_CAVIUM_CPT) += cptpf.o cptvf.o
cptpf-objs := cptpf_main.o cptpf_mbox.o
cptvf-objs := cptvf_main.o cptvf_reqmanager.o cptvf_mbox.o cptvf_algs.o
linux-7.2.3/drivers/crypto/cavium/cpt/cpt_common.h
+153
View File
@@ -0,0 +1,153 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (C) 2016 Cavium, Inc.
*/
#ifndef __CPT_COMMON_H
#define __CPT_COMMON_H
#include <asm/byteorder.h>
#include <linux/delay.h>
#include <linux/pci.h>
#include "cpt_hw_types.h"
/* Device ID */
#define CPT_81XX_PCI_PF_DEVICE_ID 0xa040
#define CPT_81XX_PCI_VF_DEVICE_ID 0xa041
/* flags to indicate the features supported */
#define CPT_FLAG_SRIOV_ENABLED BIT(1)
#define CPT_FLAG_VF_DRIVER BIT(2)
#define CPT_FLAG_DEVICE_READY BIT(3)
#define cpt_sriov_enabled(cpt) ((cpt)->flags & CPT_FLAG_SRIOV_ENABLED)
#define cpt_vf_driver(cpt) ((cpt)->flags & CPT_FLAG_VF_DRIVER)
#define cpt_device_ready(cpt) ((cpt)->flags & CPT_FLAG_DEVICE_READY)
#define CPT_MBOX_MSG_TYPE_ACK 1
#define CPT_MBOX_MSG_TYPE_NACK 2
#define CPT_MBOX_MSG_TIMEOUT 2000
#define VF_STATE_DOWN 0
#define VF_STATE_UP 1
/*
* CPT Registers map for 81xx
*/
/* PF registers */
#define CPTX_PF_CONSTANTS(a) (0x0ll + ((u64)(a) << 36))
#define CPTX_PF_RESET(a) (0x100ll + ((u64)(a) << 36))
#define CPTX_PF_DIAG(a) (0x120ll + ((u64)(a) << 36))
#define CPTX_PF_BIST_STATUS(a) (0x160ll + ((u64)(a) << 36))
#define CPTX_PF_ECC0_CTL(a) (0x200ll + ((u64)(a) << 36))
#define CPTX_PF_ECC0_FLIP(a) (0x210ll + ((u64)(a) << 36))
#define CPTX_PF_ECC0_INT(a) (0x220ll + ((u64)(a) << 36))
#define CPTX_PF_ECC0_INT_W1S(a) (0x230ll + ((u64)(a) << 36))
#define CPTX_PF_ECC0_ENA_W1S(a) (0x240ll + ((u64)(a) << 36))
#define CPTX_PF_ECC0_ENA_W1C(a) (0x250ll + ((u64)(a) << 36))
#define CPTX_PF_MBOX_INTX(a, b) \
(0x400ll + ((u64)(a) << 36) + ((b) << 3))
#define CPTX_PF_MBOX_INT_W1SX(a, b) \
(0x420ll + ((u64)(a) << 36) + ((b) << 3))
#define CPTX_PF_MBOX_ENA_W1CX(a, b) \
(0x440ll + ((u64)(a) << 36) + ((b) << 3))
#define CPTX_PF_MBOX_ENA_W1SX(a, b) \
(0x460ll + ((u64)(a) << 36) + ((b) << 3))
#define CPTX_PF_EXEC_INT(a) (0x500ll + 0x1000000000ll * ((a) & 0x1))
#define CPTX_PF_EXEC_INT_W1S(a) (0x520ll + ((u64)(a) << 36))
#define CPTX_PF_EXEC_ENA_W1C(a) (0x540ll + ((u64)(a) << 36))
#define CPTX_PF_EXEC_ENA_W1S(a) (0x560ll + ((u64)(a) << 36))
#define CPTX_PF_GX_EN(a, b) \
(0x600ll + ((u64)(a) << 36) + ((b) << 3))
#define CPTX_PF_EXEC_INFO(a) (0x700ll + ((u64)(a) << 36))
#define CPTX_PF_EXEC_BUSY(a) (0x800ll + ((u64)(a) << 36))
#define CPTX_PF_EXEC_INFO0(a) (0x900ll + ((u64)(a) << 36))
#define CPTX_PF_EXEC_INFO1(a) (0x910ll + ((u64)(a) << 36))
#define CPTX_PF_INST_REQ_PC(a) (0x10000ll + ((u64)(a) << 36))
#define CPTX_PF_INST_LATENCY_PC(a) \
(0x10020ll + ((u64)(a) << 36))
#define CPTX_PF_RD_REQ_PC(a) (0x10040ll + ((u64)(a) << 36))
#define CPTX_PF_RD_LATENCY_PC(a) (0x10060ll + ((u64)(a) << 36))
#define CPTX_PF_RD_UC_PC(a) (0x10080ll + ((u64)(a) << 36))
#define CPTX_PF_ACTIVE_CYCLES_PC(a) (0x10100ll + ((u64)(a) << 36))
#define CPTX_PF_EXE_CTL(a) (0x4000000ll + ((u64)(a) << 36))
#define CPTX_PF_EXE_STATUS(a) (0x4000008ll + ((u64)(a) << 36))
#define CPTX_PF_EXE_CLK(a) (0x4000010ll + ((u64)(a) << 36))
#define CPTX_PF_EXE_DBG_CTL(a) (0x4000018ll + ((u64)(a) << 36))
#define CPTX_PF_EXE_DBG_DATA(a) (0x4000020ll + ((u64)(a) << 36))
#define CPTX_PF_EXE_BIST_STATUS(a) (0x4000028ll + ((u64)(a) << 36))
#define CPTX_PF_EXE_REQ_TIMER(a) (0x4000030ll + ((u64)(a) << 36))
#define CPTX_PF_EXE_MEM_CTL(a) (0x4000038ll + ((u64)(a) << 36))
#define CPTX_PF_EXE_PERF_CTL(a) (0x4001000ll + ((u64)(a) << 36))
#define CPTX_PF_EXE_DBG_CNTX(a, b) \
(0x4001100ll + ((u64)(a) << 36) + ((b) << 3))
#define CPTX_PF_EXE_PERF_EVENT_CNT(a) (0x4001180ll + ((u64)(a) << 36))
#define CPTX_PF_EXE_EPCI_INBX_CNT(a, b) \
(0x4001200ll + ((u64)(a) << 36) + ((b) << 3))
#define CPTX_PF_EXE_EPCI_OUTBX_CNT(a, b) \
(0x4001240ll + ((u64)(a) << 36) + ((b) << 3))
#define CPTX_PF_ENGX_UCODE_BASE(a, b) \
(0x4002000ll + ((u64)(a) << 36) + ((b) << 3))
#define CPTX_PF_QX_CTL(a, b) \
(0x8000000ll + ((u64)(a) << 36) + ((b) << 20))
#define CPTX_PF_QX_GMCTL(a, b) \
(0x8000020ll + ((u64)(a) << 36) + ((b) << 20))
#define CPTX_PF_QX_CTL2(a, b) \
(0x8000100ll + ((u64)(a) << 36) + ((b) << 20))
#define CPTX_PF_VFX_MBOXX(a, b, c) \
(0x8001000ll + ((u64)(a) << 36) + ((b) << 20) + ((c) << 8))
/* VF registers */
#define CPTX_VQX_CTL(a, b) (0x100ll + ((u64)(a) << 36) + ((b) << 20))
#define CPTX_VQX_SADDR(a, b) (0x200ll + ((u64)(a) << 36) + ((b) << 20))
#define CPTX_VQX_DONE_WAIT(a, b) (0x400ll + ((u64)(a) << 36) + ((b) << 20))
#define CPTX_VQX_INPROG(a, b) (0x410ll + ((u64)(a) << 36) + ((b) << 20))
#define CPTX_VQX_DONE(a, b) (0x420ll + ((u64)(a) << 36) + ((b) << 20))
#define CPTX_VQX_DONE_ACK(a, b) (0x440ll + ((u64)(a) << 36) + ((b) << 20))
#define CPTX_VQX_DONE_INT_W1S(a, b) (0x460ll + ((u64)(a) << 36) + ((b) << 20))
#define CPTX_VQX_DONE_INT_W1C(a, b) (0x468ll + ((u64)(a) << 36) + ((b) << 20))
#define CPTX_VQX_DONE_ENA_W1S(a, b) (0x470ll + ((u64)(a) << 36) + ((b) << 20))
#define CPTX_VQX_DONE_ENA_W1C(a, b) (0x478ll + ((u64)(a) << 36) + ((b) << 20))
#define CPTX_VQX_MISC_INT(a, b) (0x500ll + ((u64)(a) << 36) + ((b) << 20))
#define CPTX_VQX_MISC_INT_W1S(a, b) (0x508ll + ((u64)(a) << 36) + ((b) << 20))
#define CPTX_VQX_MISC_ENA_W1S(a, b) (0x510ll + ((u64)(a) << 36) + ((b) << 20))
#define CPTX_VQX_MISC_ENA_W1C(a, b) (0x518ll + ((u64)(a) << 36) + ((b) << 20))
#define CPTX_VQX_DOORBELL(a, b) (0x600ll + ((u64)(a) << 36) + ((b) << 20))
#define CPTX_VFX_PF_MBOXX(a, b, c) \
(0x1000ll + ((u64)(a) << 36) + ((b) << 20) + ((c) << 3))
enum vftype {
AE_TYPES = 1,
SE_TYPES = 2,
BAD_CPT_TYPES,
};
/* Max CPT devices supported */
enum cpt_mbox_opcode {
CPT_MSG_VF_UP = 1,
CPT_MSG_VF_DOWN,
CPT_MSG_READY,
CPT_MSG_QLEN,
CPT_MSG_QBIND_GRP,
CPT_MSG_VQ_PRIORITY,
};
/* CPT mailbox structure */
struct cpt_mbox {
u64 msg; /* Message type MBOX[0] */
u64 data;/* Data MBOX[1] */
};
/* Register read/write APIs */
static inline void cpt_write_csr64(u8 __iomem *hw_addr, u64 offset,
u64 val)
{
writeq(val, hw_addr + offset);
}
static inline u64 cpt_read_csr64(u8 __iomem *hw_addr, u64 offset)
{
return readq(hw_addr + offset);
}
#endif /* __CPT_COMMON_H */
linux-7.2.3/drivers/crypto/cavium/cpt/cpt_hw_types.h
+655
View File
@@ -0,0 +1,655 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (C) 2016 Cavium, Inc.
*/
#ifndef __CPT_HW_TYPES_H
#define __CPT_HW_TYPES_H
#include "cpt_common.h"
/**
* Enumeration cpt_comp_e
*
* CPT Completion Enumeration
* Enumerates the values of CPT_RES_S[COMPCODE].
*/
enum cpt_comp_e {
CPT_COMP_E_NOTDONE = 0x00,
CPT_COMP_E_GOOD = 0x01,
CPT_COMP_E_FAULT = 0x02,
CPT_COMP_E_SWERR = 0x03,
CPT_COMP_E_LAST_ENTRY = 0xFF
};
/**
* Structure cpt_inst_s
*
* CPT Instruction Structure
* This structure specifies the instruction layout. Instructions are
* stored in memory as little-endian unless CPT()_PF_Q()_CTL[INST_BE] is set.
* cpt_inst_s_s
* Word 0
* doneint:1 Done interrupt.
* 0 = No interrupts related to this instruction.
* 1 = When the instruction completes, CPT()_VQ()_DONE[DONE] will be
* incremented,and based on the rules described there an interrupt may
* occur.
* Word 1
* res_addr [127: 64] Result IOVA.
* If nonzero, specifies where to write CPT_RES_S.
* If zero, no result structure will be written.
* Address must be 16-byte aligned.
* Bits <63:49> are ignored by hardware; software should use a
* sign-extended bit <48> for forward compatibility.
* Word 2
* grp:10 [171:162] If [WQ_PTR] is nonzero, the SSO guest-group to use when
* CPT submits work SSO.
* For the SSO to not discard the add-work request, FPA_PF_MAP() must map
* [GRP] and CPT()_PF_Q()_GMCTL[GMID] as valid.
* tt:2 [161:160] If [WQ_PTR] is nonzero, the SSO tag type to use when CPT
* submits work to SSO
* tag:32 [159:128] If [WQ_PTR] is nonzero, the SSO tag to use when CPT
* submits work to SSO.
* Word 3
* wq_ptr [255:192] If [WQ_PTR] is nonzero, it is a pointer to a
* work-queue entry that CPT submits work to SSO after all context,
* output data, and result write operations are visible to other
* CNXXXX units and the cores. Bits <2:0> must be zero.
* Bits <63:49> are ignored by hardware; software should
* use a sign-extended bit <48> for forward compatibility.
* Internal:
* Bits <63:49>, <2:0> are ignored by hardware, treated as always 0x0.
* Word 4
* ei0; [319:256] Engine instruction word 0. Passed to the AE/SE.
* Word 5
* ei1; [383:320] Engine instruction word 1. Passed to the AE/SE.
* Word 6
* ei2; [447:384] Engine instruction word 1. Passed to the AE/SE.
* Word 7
* ei3; [511:448] Engine instruction word 1. Passed to the AE/SE.
*
*/
union cpt_inst_s {
u64 u[8];
struct cpt_inst_s_s {
#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */
u64 reserved_17_63:47;
u64 doneint:1;
u64 reserved_0_1:16;
#else /* Word 0 - Little Endian */
u64 reserved_0_15:16;
u64 doneint:1;
u64 reserved_17_63:47;
#endif /* Word 0 - End */
u64 res_addr;
#if defined(__BIG_ENDIAN_BITFIELD) /* Word 2 - Big Endian */
u64 reserved_172_19:20;
u64 grp:10;
u64 tt:2;
u64 tag:32;
#else /* Word 2 - Little Endian */
u64 tag:32;
u64 tt:2;
u64 grp:10;
u64 reserved_172_191:20;
#endif /* Word 2 - End */
u64 wq_ptr;
u64 ei0;
u64 ei1;
u64 ei2;
u64 ei3;
} s;
};
/**
* Structure cpt_res_s
*
* CPT Result Structure
* The CPT coprocessor writes the result structure after it completes a
* CPT_INST_S instruction. The result structure is exactly 16 bytes, and
* each instruction completion produces exactly one result structure.
*
* This structure is stored in memory as little-endian unless
* CPT()_PF_Q()_CTL[INST_BE] is set.
* cpt_res_s_s
* Word 0
* doneint:1 [16:16] Done interrupt. This bit is copied from the
* corresponding instruction's CPT_INST_S[DONEINT].
* compcode:8 [7:0] Indicates completion/error status of the CPT coprocessor
* for the associated instruction, as enumerated by CPT_COMP_E.
* Core software may write the memory location containing [COMPCODE] to
* 0x0 before ringing the doorbell, and then poll for completion by
* checking for a nonzero value.
* Once the core observes a nonzero [COMPCODE] value in this case,the CPT
* coprocessor will have also completed L2/DRAM write operations.
* Word 1
* reserved
*
*/
union cpt_res_s {
u64 u[2];
struct cpt_res_s_s {
#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */
u64 reserved_17_63:47;
u64 doneint:1;
u64 reserved_8_15:8;
u64 compcode:8;
#else /* Word 0 - Little Endian */
u64 compcode:8;
u64 reserved_8_15:8;
u64 doneint:1;
u64 reserved_17_63:47;
#endif /* Word 0 - End */
u64 reserved_64_127;
} s;
};
/**
* Register (NCB) cpt#_pf_bist_status
*
* CPT PF Control Bist Status Register
* This register has the BIST status of memories. Each bit is the BIST result
* of an individual memory (per bit, 0 = pass and 1 = fail).
* cptx_pf_bist_status_s
* Word0
* bstatus [29:0](RO/H) BIST status. One bit per memory, enumerated by
* CPT_RAMS_E.
*/
union cptx_pf_bist_status {
u64 u;
struct cptx_pf_bist_status_s {
#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */
u64 reserved_30_63:34;
u64 bstatus:30;
#else /* Word 0 - Little Endian */
u64 bstatus:30;
u64 reserved_30_63:34;
#endif /* Word 0 - End */
} s;
};
/**
* Register (NCB) cpt#_pf_constants
*
* CPT PF Constants Register
* This register contains implementation-related parameters of CPT in CNXXXX.
* cptx_pf_constants_s
* Word 0
* reserved_40_63:24 [63:40] Reserved.
* epcis:8 [39:32](RO) Number of EPCI busses.
* grps:8 [31:24](RO) Number of engine groups implemented.
* ae:8 [23:16](RO/H) Number of AEs. In CNXXXX, for CPT0 returns 0x0,
* for CPT1 returns 0x18, or less if there are fuse-disables.
* se:8 [15:8](RO/H) Number of SEs. In CNXXXX, for CPT0 returns 0x30,
* or less if there are fuse-disables, for CPT1 returns 0x0.
* vq:8 [7:0](RO) Number of VQs.
*/
union cptx_pf_constants {
u64 u;
struct cptx_pf_constants_s {
#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */
u64 reserved_40_63:24;
u64 epcis:8;
u64 grps:8;
u64 ae:8;
u64 se:8;
u64 vq:8;
#else /* Word 0 - Little Endian */
u64 vq:8;
u64 se:8;
u64 ae:8;
u64 grps:8;
u64 epcis:8;
u64 reserved_40_63:24;
#endif /* Word 0 - End */
} s;
};
/**
* Register (NCB) cpt#_pf_exe_bist_status
*
* CPT PF Engine Bist Status Register
* This register has the BIST status of each engine. Each bit is the
* BIST result of an individual engine (per bit, 0 = pass and 1 = fail).
* cptx_pf_exe_bist_status_s
* Word0
* reserved_48_63:16 [63:48] reserved
* bstatus:48 [47:0](RO/H) BIST status. One bit per engine.
*
*/
union cptx_pf_exe_bist_status {
u64 u;
struct cptx_pf_exe_bist_status_s {
#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */
u64 reserved_48_63:16;
u64 bstatus:48;
#else /* Word 0 - Little Endian */
u64 bstatus:48;
u64 reserved_48_63:16;
#endif /* Word 0 - End */
} s;
};
/**
* Register (NCB) cpt#_pf_q#_ctl
*
* CPT Queue Control Register
* This register configures queues. This register should be changed only
* when quiescent (see CPT()_VQ()_INPROG[INFLIGHT]).
* cptx_pf_qx_ctl_s
* Word0
* reserved_60_63:4 [63:60] reserved.
* aura:12; [59:48](R/W) Guest-aura for returning this queue's
* instruction-chunk buffers to FPA. Only used when [INST_FREE] is set.
* For the FPA to not discard the request, FPA_PF_MAP() must map
* [AURA] and CPT()_PF_Q()_GMCTL[GMID] as valid.
* reserved_45_47:3 [47:45] reserved.
* size:13 [44:32](R/W) Command-buffer size, in number of 64-bit words per
* command buffer segment. Must be 8*n + 1, where n is the number of
* instructions per buffer segment.
* reserved_11_31:21 [31:11] Reserved.
* cont_err:1 [10:10](R/W) Continue on error.
* 0 = When CPT()_VQ()_MISC_INT[NWRP], CPT()_VQ()_MISC_INT[IRDE] or
* CPT()_VQ()_MISC_INT[DOVF] are set by hardware or software via
* CPT()_VQ()_MISC_INT_W1S, then CPT()_VQ()_CTL[ENA] is cleared. Due to
* pipelining, additional instructions may have been processed between the
* instruction causing the error and the next instruction in the disabled
* queue (the instruction at CPT()_VQ()_SADDR).
* 1 = Ignore errors and continue processing instructions.
* For diagnostic use only.
* inst_free:1 [9:9](R/W) Instruction FPA free. When set, when CPT reaches the
* end of an instruction chunk, that chunk will be freed to the FPA.
* inst_be:1 [8:8](R/W) Instruction big-endian control. When set, instructions,
* instruction next chunk pointers, and result structures are stored in
* big-endian format in memory.
* iqb_ldwb:1 [7:7](R/W) Instruction load don't write back.
* 0 = The hardware issues NCB transient load (LDT) towards the cache,
* which if the line hits and it is dirty will cause the line to be
* written back before being replaced.
* 1 = The hardware issues NCB LDWB read-and-invalidate command towards
* the cache when fetching the last word of instructions; as a result the
* line will not be written back when replaced. This improves
* performance, but software must not read the instructions after they are
* posted to the hardware. Reads that do not consume the last word of a
* cache line always use LDI.
* reserved_4_6:3 [6:4] Reserved.
* grp:3; [3:1](R/W) Engine group.
* pri:1; [0:0](R/W) Queue priority.
* 1 = This queue has higher priority. Round-robin between higher
* priority queues.
* 0 = This queue has lower priority. Round-robin between lower
* priority queues.
*/
union cptx_pf_qx_ctl {
u64 u;
struct cptx_pf_qx_ctl_s {
#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */
u64 reserved_60_63:4;
u64 aura:12;
u64 reserved_45_47:3;
u64 size:13;
u64 reserved_11_31:21;
u64 cont_err:1;
u64 inst_free:1;
u64 inst_be:1;
u64 iqb_ldwb:1;
u64 reserved_4_6:3;
u64 grp:3;
u64 pri:1;
#else /* Word 0 - Little Endian */
u64 pri:1;
u64 grp:3;
u64 reserved_4_6:3;
u64 iqb_ldwb:1;
u64 inst_be:1;
u64 inst_free:1;
u64 cont_err:1;
u64 reserved_11_31:21;
u64 size:13;
u64 reserved_45_47:3;
u64 aura:12;
u64 reserved_60_63:4;
#endif /* Word 0 - End */
} s;
};
/**
* Register (NCB) cpt#_vq#_saddr
*
* CPT Queue Starting Buffer Address Registers
* These registers set the instruction buffer starting address.
* cptx_vqx_saddr_s
* Word0
* reserved_49_63:15 [63:49] Reserved.
* ptr:43 [48:6](R/W/H) Instruction buffer IOVA <48:6> (64-byte aligned).
* When written, it is the initial buffer starting address; when read,
* it is the next read pointer to be requested from L2C. The PTR field
* is overwritten with the next pointer each time that the command buffer
* segment is exhausted. New commands will then be read from the newly
* specified command buffer pointer.
* reserved_0_5:6 [5:0] Reserved.
*
*/
union cptx_vqx_saddr {
u64 u;
struct cptx_vqx_saddr_s {
#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */
u64 reserved_49_63:15;
u64 ptr:43;
u64 reserved_0_5:6;
#else /* Word 0 - Little Endian */
u64 reserved_0_5:6;
u64 ptr:43;
u64 reserved_49_63:15;
#endif /* Word 0 - End */
} s;
};
/**
* Register (NCB) cpt#_vq#_misc_ena_w1s
*
* CPT Queue Misc Interrupt Enable Set Register
* This register sets interrupt enable bits.
* cptx_vqx_misc_ena_w1s_s
* Word0
* reserved_5_63:59 [63:5] Reserved.
* swerr:1 [4:4](R/W1S/H) Reads or sets enable for
* CPT(0..1)_VQ(0..63)_MISC_INT[SWERR].
* nwrp:1 [3:3](R/W1S/H) Reads or sets enable for
* CPT(0..1)_VQ(0..63)_MISC_INT[NWRP].
* irde:1 [2:2](R/W1S/H) Reads or sets enable for
* CPT(0..1)_VQ(0..63)_MISC_INT[IRDE].
* dovf:1 [1:1](R/W1S/H) Reads or sets enable for
* CPT(0..1)_VQ(0..63)_MISC_INT[DOVF].
* mbox:1 [0:0](R/W1S/H) Reads or sets enable for
* CPT(0..1)_VQ(0..63)_MISC_INT[MBOX].
*
*/
union cptx_vqx_misc_ena_w1s {
u64 u;
struct cptx_vqx_misc_ena_w1s_s {
#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */
u64 reserved_5_63:59;
u64 swerr:1;
u64 nwrp:1;
u64 irde:1;
u64 dovf:1;
u64 mbox:1;
#else /* Word 0 - Little Endian */
u64 mbox:1;
u64 dovf:1;
u64 irde:1;
u64 nwrp:1;
u64 swerr:1;
u64 reserved_5_63:59;
#endif /* Word 0 - End */
} s;
};
/**
* Register (NCB) cpt#_vq#_doorbell
*
* CPT Queue Doorbell Registers
* Doorbells for the CPT instruction queues.
* cptx_vqx_doorbell_s
* Word0
* reserved_20_63:44 [63:20] Reserved.
* dbell_cnt:20 [19:0](R/W/H) Number of instruction queue 64-bit words to add
* to the CPT instruction doorbell count. Readback value is the
* current number of pending doorbell requests. If counter overflows
* CPT()_VQ()_MISC_INT[DBELL_DOVF] is set. To reset the count back to
* zero, write one to clear CPT()_VQ()_MISC_INT_ENA_W1C[DBELL_DOVF],
* then write a value of 2^20 minus the read [DBELL_CNT], then write one
* to CPT()_VQ()_MISC_INT_W1C[DBELL_DOVF] and
* CPT()_VQ()_MISC_INT_ENA_W1S[DBELL_DOVF]. Must be a multiple of 8.
* All CPT instructions are 8 words and require a doorbell count of
* multiple of 8.
*/
union cptx_vqx_doorbell {
u64 u;
struct cptx_vqx_doorbell_s {
#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */
u64 reserved_20_63:44;
u64 dbell_cnt:20;
#else /* Word 0 - Little Endian */
u64 dbell_cnt:20;
u64 reserved_20_63:44;
#endif /* Word 0 - End */
} s;
};
/**
* Register (NCB) cpt#_vq#_inprog
*
* CPT Queue In Progress Count Registers
* These registers contain the per-queue instruction in flight registers.
* cptx_vqx_inprog_s
* Word0
* reserved_8_63:56 [63:8] Reserved.
* inflight:8 [7:0](RO/H) Inflight count. Counts the number of instructions
* for the VF for which CPT is fetching, executing or responding to
* instructions. However this does not include any interrupts that are
* awaiting software handling (CPT()_VQ()_DONE[DONE] != 0x0).
* A queue may not be reconfigured until:
* 1. CPT()_VQ()_CTL[ENA] is cleared by software.
* 2. [INFLIGHT] is polled until equals to zero.
*/
union cptx_vqx_inprog {
u64 u;
struct cptx_vqx_inprog_s {
#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */
u64 reserved_8_63:56;
u64 inflight:8;
#else /* Word 0 - Little Endian */
u64 inflight:8;
u64 reserved_8_63:56;
#endif /* Word 0 - End */
} s;
};
/**
* Register (NCB) cpt#_vq#_misc_int
*
* CPT Queue Misc Interrupt Register
* These registers contain the per-queue miscellaneous interrupts.
* cptx_vqx_misc_int_s
* Word 0
* reserved_5_63:59 [63:5] Reserved.
* swerr:1 [4:4](R/W1C/H) Software error from engines.
* nwrp:1 [3:3](R/W1C/H) NCB result write response error.
* irde:1 [2:2](R/W1C/H) Instruction NCB read response error.
* dovf:1 [1:1](R/W1C/H) Doorbell overflow.
* mbox:1 [0:0](R/W1C/H) PF to VF mailbox interrupt. Set when
* CPT()_VF()_PF_MBOX(0) is written.
*
*/
union cptx_vqx_misc_int {
u64 u;
struct cptx_vqx_misc_int_s {
#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */
u64 reserved_5_63:59;
u64 swerr:1;
u64 nwrp:1;
u64 irde:1;
u64 dovf:1;
u64 mbox:1;
#else /* Word 0 - Little Endian */
u64 mbox:1;
u64 dovf:1;
u64 irde:1;
u64 nwrp:1;
u64 swerr:1;
u64 reserved_5_63:59;
#endif /* Word 0 - End */
} s;
};
/**
* Register (NCB) cpt#_vq#_done_ack
*
* CPT Queue Done Count Ack Registers
* This register is written by software to acknowledge interrupts.
* cptx_vqx_done_ack_s
* Word0
* reserved_20_63:44 [63:20] Reserved.
* done_ack:20 [19:0](R/W/H) Number of decrements to CPT()_VQ()_DONE[DONE].
* Reads CPT()_VQ()_DONE[DONE]. Written by software to acknowledge
* interrupts. If CPT()_VQ()_DONE[DONE] is still nonzero the interrupt
* will be re-sent if the conditions described in CPT()_VQ()_DONE[DONE]
* are satisfied.
*
*/
union cptx_vqx_done_ack {
u64 u;
struct cptx_vqx_done_ack_s {
#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */
u64 reserved_20_63:44;
u64 done_ack:20;
#else /* Word 0 - Little Endian */
u64 done_ack:20;
u64 reserved_20_63:44;
#endif /* Word 0 - End */
} s;
};
/**
* Register (NCB) cpt#_vq#_done
*
* CPT Queue Done Count Registers
* These registers contain the per-queue instruction done count.
* cptx_vqx_done_s
* Word0
* reserved_20_63:44 [63:20] Reserved.
* done:20 [19:0](R/W/H) Done count. When CPT_INST_S[DONEINT] set and that
* instruction completes, CPT()_VQ()_DONE[DONE] is incremented when the
* instruction finishes. Write to this field are for diagnostic use only;
* instead software writes CPT()_VQ()_DONE_ACK with the number of
* decrements for this field.
* Interrupts are sent as follows:
* * When CPT()_VQ()_DONE[DONE] = 0, then no results are pending, the
* interrupt coalescing timer is held to zero, and an interrupt is not
* sent.
* * When CPT()_VQ()_DONE[DONE] != 0, then the interrupt coalescing timer
* counts. If the counter is >= CPT()_VQ()_DONE_WAIT[TIME_WAIT]*1024, or
* CPT()_VQ()_DONE[DONE] >= CPT()_VQ()_DONE_WAIT[NUM_WAIT], i.e. enough
* time has passed or enough results have arrived, then the interrupt is
* sent.
* * When CPT()_VQ()_DONE_ACK is written (or CPT()_VQ()_DONE is written
* but this is not typical), the interrupt coalescing timer restarts.
* Note after decrementing this interrupt equation is recomputed,
* for example if CPT()_VQ()_DONE[DONE] >= CPT()_VQ()_DONE_WAIT[NUM_WAIT]
* and because the timer is zero, the interrupt will be resent immediately.
* (This covers the race case between software acknowledging an interrupt
* and a result returning.)
* * When CPT()_VQ()_DONE_ENA_W1S[DONE] = 0, interrupts are not sent,
* but the counting described above still occurs.
* Since CPT instructions complete out-of-order, if software is using
* completion interrupts the suggested scheme is to request a DONEINT on
* each request, and when an interrupt arrives perform a "greedy" scan for
* completions; even if a later command is acknowledged first this will
* not result in missing a completion.
* Software is responsible for making sure [DONE] does not overflow;
* for example by insuring there are not more than 2^20-1 instructions in
* flight that may request interrupts.
*
*/
union cptx_vqx_done {
u64 u;
struct cptx_vqx_done_s {
#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */
u64 reserved_20_63:44;
u64 done:20;
#else /* Word 0 - Little Endian */
u64 done:20;
u64 reserved_20_63:44;
#endif /* Word 0 - End */
} s;
};
/**
* Register (NCB) cpt#_vq#_done_wait
*
* CPT Queue Done Interrupt Coalescing Wait Registers
* Specifies the per queue interrupt coalescing settings.
* cptx_vqx_done_wait_s
* Word0
* reserved_48_63:16 [63:48] Reserved.
* time_wait:16; [47:32](R/W) Time hold-off. When CPT()_VQ()_DONE[DONE] = 0
* or CPT()_VQ()_DONE_ACK is written a timer is cleared. When the timer
* reaches [TIME_WAIT]*1024 then interrupt coalescing ends.
* see CPT()_VQ()_DONE[DONE]. If 0x0, time coalescing is disabled.
* reserved_20_31:12 [31:20] Reserved.
* num_wait:20 [19:0](R/W) Number of messages hold-off.
* When CPT()_VQ()_DONE[DONE] >= [NUM_WAIT] then interrupt coalescing ends
* see CPT()_VQ()_DONE[DONE]. If 0x0, same behavior as 0x1.
*
*/
union cptx_vqx_done_wait {
u64 u;
struct cptx_vqx_done_wait_s {
#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */
u64 reserved_48_63:16;
u64 time_wait:16;
u64 reserved_20_31:12;
u64 num_wait:20;
#else /* Word 0 - Little Endian */
u64 num_wait:20;
u64 reserved_20_31:12;
u64 time_wait:16;
u64 reserved_48_63:16;
#endif /* Word 0 - End */
} s;
};
/**
* Register (NCB) cpt#_vq#_done_ena_w1s
*
* CPT Queue Done Interrupt Enable Set Registers
* Write 1 to these registers will enable the DONEINT interrupt for the queue.
* cptx_vqx_done_ena_w1s_s
* Word0
* reserved_1_63:63 [63:1] Reserved.
* done:1 [0:0](R/W1S/H) Write 1 will enable DONEINT for this queue.
* Write 0 has no effect. Read will return the enable bit.
*/
union cptx_vqx_done_ena_w1s {
u64 u;
struct cptx_vqx_done_ena_w1s_s {
#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */
u64 reserved_1_63:63;
u64 done:1;
#else /* Word 0 - Little Endian */
u64 done:1;
u64 reserved_1_63:63;
#endif /* Word 0 - End */
} s;
};
/**
* Register (NCB) cpt#_vq#_ctl
*
* CPT VF Queue Control Registers
* This register configures queues. This register should be changed (other than
* clearing [ENA]) only when quiescent (see CPT()_VQ()_INPROG[INFLIGHT]).
* cptx_vqx_ctl_s
* Word0
* reserved_1_63:63 [63:1] Reserved.
* ena:1 [0:0](R/W/H) Enables the logical instruction queue.
* See also CPT()_PF_Q()_CTL[CONT_ERR] and CPT()_VQ()_INPROG[INFLIGHT].
* 1 = Queue is enabled.
* 0 = Queue is disabled.
*/
union cptx_vqx_ctl {
u64 u;
struct cptx_vqx_ctl_s {
#if defined(__BIG_ENDIAN_BITFIELD) /* Word 0 - Big Endian */
u64 reserved_1_63:63;
u64 ena:1;
#else /* Word 0 - Little Endian */
u64 ena:1;
u64 reserved_1_63:63;
#endif /* Word 0 - End */
} s;
};
#endif /*__CPT_HW_TYPES_H*/
linux-7.2.3/drivers/crypto/cavium/cpt/cptpf.h
+61
View File
@@ -0,0 +1,61 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (C) 2016 Cavium, Inc.
*/
#ifndef __CPTPF_H
#define __CPTPF_H
#include "cpt_common.h"
#define CSR_DELAY 30
#define CPT_MAX_CORE_GROUPS 8
#define CPT_MAX_SE_CORES 10
#define CPT_MAX_AE_CORES 6
#define CPT_MAX_TOTAL_CORES (CPT_MAX_SE_CORES + CPT_MAX_AE_CORES)
#define CPT_MAX_VF_NUM 16
#define CPT_PF_MSIX_VECTORS 3
#define CPT_PF_INT_VEC_E_MBOXX(a) (0x02 + (a))
#define CPT_UCODE_VERSION_SZ 32
struct cpt_device;
struct microcode {
u8 is_mc_valid;
u8 is_ae;
u8 group;
u8 num_cores;
u32 code_size;
u64 core_mask;
u8 version[CPT_UCODE_VERSION_SZ];
/* Base info */
dma_addr_t phys_base;
void *code;
};
struct cpt_vf_info {
u8 state;
u8 priority;
u8 id;
u32 qlen;
};
/**
* cpt device structure
*/
struct cpt_device {
u16 flags; /* Flags to hold device status bits */
u8 num_vf_en; /* Number of VFs enabled (0...CPT_MAX_VF_NUM) */
struct cpt_vf_info vfinfo[CPT_MAX_VF_NUM]; /* Per VF info */
void __iomem *reg_base; /* Register start address */
struct pci_dev *pdev; /* pci device handle */
struct microcode mcode[CPT_MAX_CORE_GROUPS];
u8 next_mc_idx; /* next microcode index */
u8 next_group;
u8 max_se_cores;
u8 max_ae_cores;
};
void cpt_mbox_intr_handler(struct cpt_device *cpt, int mbx);
#endif /* __CPTPF_H */
linux-7.2.3/drivers/crypto/cavium/cpt/cptpf_main.c
+669
View File
@@ -0,0 +1,669 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (C) 2016 Cavium, Inc.
*/
#include <linux/device.h>
#include <linux/firmware.h>
#include <linux/interrupt.h>
#include <linux/module.h>
#include <linux/moduleparam.h>
#include <linux/pci.h>
#include <linux/printk.h>
#include "cptpf.h"
#define DRV_NAME "thunder-cpt"
#define DRV_VERSION "1.0"
static u32 num_vfs = 4; /* Default 4 VF enabled */
module_param(num_vfs, uint, 0444);
MODULE_PARM_DESC(num_vfs, "Number of VFs to enable(1-16)");
/*
* Disable cores specified by coremask
*/
static void cpt_disable_cores(struct cpt_device *cpt, u64 coremask,
u8 type, u8 grp)
{
u64 pf_exe_ctl;
u32 timeout = 100;
u64 grpmask = 0;
struct device *dev = &cpt->pdev->dev;
if (type == AE_TYPES)
coremask = (coremask << cpt->max_se_cores);
/* Disengage the cores from groups */
grpmask = cpt_read_csr64(cpt->reg_base, CPTX_PF_GX_EN(0, grp));
cpt_write_csr64(cpt->reg_base, CPTX_PF_GX_EN(0, grp),
(grpmask & ~coremask));
udelay(CSR_DELAY);
grp = cpt_read_csr64(cpt->reg_base, CPTX_PF_EXEC_BUSY(0));
while (grp & coremask) {
dev_err(dev, "Cores still busy %llx", coremask);
grp = cpt_read_csr64(cpt->reg_base,
CPTX_PF_EXEC_BUSY(0));
if (!timeout--)
break;
udelay(CSR_DELAY);
}
/* Disable the cores */
pf_exe_ctl = cpt_read_csr64(cpt->reg_base, CPTX_PF_EXE_CTL(0));
cpt_write_csr64(cpt->reg_base, CPTX_PF_EXE_CTL(0),
(pf_exe_ctl & ~coremask));
udelay(CSR_DELAY);
}
/*
* Enable cores specified by coremask
*/
static void cpt_enable_cores(struct cpt_device *cpt, u64 coremask,
u8 type)
{
u64 pf_exe_ctl;
if (type == AE_TYPES)
coremask = (coremask << cpt->max_se_cores);
pf_exe_ctl = cpt_read_csr64(cpt->reg_base, CPTX_PF_EXE_CTL(0));
cpt_write_csr64(cpt->reg_base, CPTX_PF_EXE_CTL(0),
(pf_exe_ctl | coremask));
udelay(CSR_DELAY);
}
static void cpt_configure_group(struct cpt_device *cpt, u8 grp,
u64 coremask, u8 type)
{
u64 pf_gx_en = 0;
if (type == AE_TYPES)
coremask = (coremask << cpt->max_se_cores);
pf_gx_en = cpt_read_csr64(cpt->reg_base, CPTX_PF_GX_EN(0, grp));
cpt_write_csr64(cpt->reg_base, CPTX_PF_GX_EN(0, grp),
(pf_gx_en | coremask));
udelay(CSR_DELAY);
}
static void cpt_disable_mbox_interrupts(struct cpt_device *cpt)
{
/* Clear mbox(0) interupts for all vfs */
cpt_write_csr64(cpt->reg_base, CPTX_PF_MBOX_ENA_W1CX(0, 0), ~0ull);
}
static void cpt_disable_ecc_interrupts(struct cpt_device *cpt)
{
/* Clear ecc(0) interupts for all vfs */
cpt_write_csr64(cpt->reg_base, CPTX_PF_ECC0_ENA_W1C(0), ~0ull);
}
static void cpt_disable_exec_interrupts(struct cpt_device *cpt)
{
/* Clear exec interupts for all vfs */
cpt_write_csr64(cpt->reg_base, CPTX_PF_EXEC_ENA_W1C(0), ~0ull);
}
static void cpt_disable_all_interrupts(struct cpt_device *cpt)
{
cpt_disable_mbox_interrupts(cpt);
cpt_disable_ecc_interrupts(cpt);
cpt_disable_exec_interrupts(cpt);
}
static void cpt_enable_mbox_interrupts(struct cpt_device *cpt)
{
/* Set mbox(0) interupts for all vfs */
cpt_write_csr64(cpt->reg_base, CPTX_PF_MBOX_ENA_W1SX(0, 0), ~0ull);
}
static int cpt_load_microcode(struct cpt_device *cpt, struct microcode *mcode)
{
int ret = 0, core = 0, shift = 0;
u32 total_cores = 0;
struct device *dev = &cpt->pdev->dev;
if (!mcode || !mcode->code) {
dev_err(dev, "Either the mcode is null or data is NULL\n");
return -EINVAL;
}
if (mcode->code_size == 0) {
dev_err(dev, "microcode size is 0\n");
return -EINVAL;
}
/* Assumes 0-9 are SE cores for UCODE_BASE registers and
* AE core bases follow
*/
if (mcode->is_ae) {
core = CPT_MAX_SE_CORES; /* start couting from 10 */
total_cores = CPT_MAX_TOTAL_CORES; /* upto 15 */
} else {
core = 0; /* start couting from 0 */
total_cores = CPT_MAX_SE_CORES; /* upto 9 */
}
/* Point to microcode for each core of the group */
for (; core < total_cores ; core++, shift++) {
if (mcode->core_mask & (1 << shift)) {
cpt_write_csr64(cpt->reg_base,
CPTX_PF_ENGX_UCODE_BASE(0, core),
(u64)mcode->phys_base);
}
}
return ret;
}
static int do_cpt_init(struct cpt_device *cpt, struct microcode *mcode)
{
int ret = 0;
struct device *dev = &cpt->pdev->dev;
/* Make device not ready */
cpt->flags &= ~CPT_FLAG_DEVICE_READY;
/* Disable All PF interrupts */
cpt_disable_all_interrupts(cpt);
/* Calculate mcode group and coremasks */
if (mcode->is_ae) {
if (mcode->num_cores > cpt->max_ae_cores) {
dev_err(dev, "Requested for more cores than available AE cores\n");
ret = -EINVAL;
goto cpt_init_fail;
}
if (cpt->next_group >= CPT_MAX_CORE_GROUPS) {
dev_err(dev, "Can't load, all eight microcode groups in use");
return -ENFILE;
}
mcode->group = cpt->next_group;
/* Convert requested cores to mask */
mcode->core_mask = GENMASK(mcode->num_cores, 0);
cpt_disable_cores(cpt, mcode->core_mask, AE_TYPES,
mcode->group);
/* Load microcode for AE engines */
ret = cpt_load_microcode(cpt, mcode);
if (ret) {
dev_err(dev, "Microcode load Failed for %s\n",
mcode->version);
goto cpt_init_fail;
}
cpt->next_group++;
/* Configure group mask for the mcode */
cpt_configure_group(cpt, mcode->group, mcode->core_mask,
AE_TYPES);
/* Enable AE cores for the group mask */
cpt_enable_cores(cpt, mcode->core_mask, AE_TYPES);
} else {
if (mcode->num_cores > cpt->max_se_cores) {
dev_err(dev, "Requested for more cores than available SE cores\n");
ret = -EINVAL;
goto cpt_init_fail;
}
if (cpt->next_group >= CPT_MAX_CORE_GROUPS) {
dev_err(dev, "Can't load, all eight microcode groups in use");
return -ENFILE;
}
mcode->group = cpt->next_group;
/* Covert requested cores to mask */
mcode->core_mask = GENMASK(mcode->num_cores, 0);
cpt_disable_cores(cpt, mcode->core_mask, SE_TYPES,
mcode->group);
/* Load microcode for SE engines */
ret = cpt_load_microcode(cpt, mcode);
if (ret) {
dev_err(dev, "Microcode load Failed for %s\n",
mcode->version);
goto cpt_init_fail;
}
cpt->next_group++;
/* Configure group mask for the mcode */
cpt_configure_group(cpt, mcode->group, mcode->core_mask,
SE_TYPES);
/* Enable SE cores for the group mask */
cpt_enable_cores(cpt, mcode->core_mask, SE_TYPES);
}
/* Enabled PF mailbox interrupts */
cpt_enable_mbox_interrupts(cpt);
cpt->flags |= CPT_FLAG_DEVICE_READY;
return ret;
cpt_init_fail:
/* Enabled PF mailbox interrupts */
cpt_enable_mbox_interrupts(cpt);
return ret;
}
struct ucode_header {
u8 version[CPT_UCODE_VERSION_SZ];
__be32 code_length;
u32 data_length;
u64 sram_address;
};
static int cpt_ucode_load_fw(struct cpt_device *cpt, const u8 *fw, bool is_ae)
{
const struct firmware *fw_entry;
struct device *dev = &cpt->pdev->dev;
struct ucode_header *ucode;
unsigned int code_length;
struct microcode *mcode;
int j, ret = 0;
ret = request_firmware(&fw_entry, fw, dev);
if (ret)
return ret;
ucode = (struct ucode_header *)fw_entry->data;
mcode = &cpt->mcode[cpt->next_mc_idx];
memcpy(mcode->version, (u8 *)fw_entry->data, CPT_UCODE_VERSION_SZ);
code_length = ntohl(ucode->code_length);
if (code_length == 0 || code_length >= INT_MAX / 2) {
ret = -EINVAL;
goto fw_release;
}
mcode->code_size = code_length * 2;
mcode->is_ae = is_ae;
mcode->core_mask = 0ULL;
mcode->num_cores = is_ae ? 6 : 10;
/* Allocate DMAable space */
mcode->code = dma_alloc_coherent(&cpt->pdev->dev, mcode->code_size,
&mcode->phys_base, GFP_KERNEL);
if (!mcode->code) {
dev_err(dev, "Unable to allocate space for microcode");
ret = -ENOMEM;
goto fw_release;
}
memcpy((void *)mcode->code, (void *)(fw_entry->data + sizeof(*ucode)),
mcode->code_size);
/* Byte swap 64-bit */
for (j = 0; j < (mcode->code_size / 8); j++)
((__be64 *)mcode->code)[j] = cpu_to_be64(((u64 *)mcode->code)[j]);
/* MC needs 16-bit swap */
for (j = 0; j < (mcode->code_size / 2); j++)
((__be16 *)mcode->code)[j] = cpu_to_be16(((u16 *)mcode->code)[j]);
dev_dbg(dev, "mcode->code_size = %u\n", mcode->code_size);
dev_dbg(dev, "mcode->is_ae = %u\n", mcode->is_ae);
dev_dbg(dev, "mcode->num_cores = %u\n", mcode->num_cores);
dev_dbg(dev, "mcode->code = %llx\n", (u64)mcode->code);
dev_dbg(dev, "mcode->phys_base = %llx\n", mcode->phys_base);
ret = do_cpt_init(cpt, mcode);
if (ret) {
dma_free_coherent(&cpt->pdev->dev, mcode->code_size,
mcode->code, mcode->phys_base);
dev_err(dev, "do_cpt_init failed with ret: %d\n", ret);
goto fw_release;
}
dev_info(dev, "Microcode Loaded %s\n", mcode->version);
mcode->is_mc_valid = 1;
cpt->next_mc_idx++;
fw_release:
release_firmware(fw_entry);
return ret;
}
static int cpt_ucode_load(struct cpt_device *cpt)
{
int ret = 0;
struct device *dev = &cpt->pdev->dev;
ret = cpt_ucode_load_fw(cpt, "cpt8x-mc-ae.out", true);
if (ret) {
dev_err(dev, "ae:cpt_ucode_load failed with ret: %d\n", ret);
return ret;
}
ret = cpt_ucode_load_fw(cpt, "cpt8x-mc-se.out", false);
if (ret) {
dev_err(dev, "se:cpt_ucode_load failed with ret: %d\n", ret);
return ret;
}
return ret;
}
static irqreturn_t cpt_mbx0_intr_handler(int irq, void *cpt_irq)
{
struct cpt_device *cpt = (struct cpt_device *)cpt_irq;
cpt_mbox_intr_handler(cpt, 0);
return IRQ_HANDLED;
}
static void cpt_reset(struct cpt_device *cpt)
{
cpt_write_csr64(cpt->reg_base, CPTX_PF_RESET(0), 1);
}
static void cpt_find_max_enabled_cores(struct cpt_device *cpt)
{
union cptx_pf_constants pf_cnsts = {0};
pf_cnsts.u = cpt_read_csr64(cpt->reg_base, CPTX_PF_CONSTANTS(0));
cpt->max_se_cores = pf_cnsts.s.se;
cpt->max_ae_cores = pf_cnsts.s.ae;
}
static u32 cpt_check_bist_status(struct cpt_device *cpt)
{
union cptx_pf_bist_status bist_sts = {0};
bist_sts.u = cpt_read_csr64(cpt->reg_base,
CPTX_PF_BIST_STATUS(0));
return bist_sts.u;
}
static u64 cpt_check_exe_bist_status(struct cpt_device *cpt)
{
union cptx_pf_exe_bist_status bist_sts = {0};
bist_sts.u = cpt_read_csr64(cpt->reg_base,
CPTX_PF_EXE_BIST_STATUS(0));
return bist_sts.u;
}
static void cpt_disable_all_cores(struct cpt_device *cpt)
{
u32 grp, timeout = 100;
struct device *dev = &cpt->pdev->dev;
/* Disengage the cores from groups */
for (grp = 0; grp < CPT_MAX_CORE_GROUPS; grp++) {
cpt_write_csr64(cpt->reg_base, CPTX_PF_GX_EN(0, grp), 0);
udelay(CSR_DELAY);
}
grp = cpt_read_csr64(cpt->reg_base, CPTX_PF_EXEC_BUSY(0));
while (grp) {
dev_err(dev, "Cores still busy");
grp = cpt_read_csr64(cpt->reg_base,
CPTX_PF_EXEC_BUSY(0));
if (!timeout--)
break;
udelay(CSR_DELAY);
}
/* Disable the cores */
cpt_write_csr64(cpt->reg_base, CPTX_PF_EXE_CTL(0), 0);
}
/*
* Ensure all cores are disengaged from all groups by
* calling cpt_disable_all_cores() before calling this
* function.
*/
static void cpt_unload_microcode(struct cpt_device *cpt)
{
u32 grp = 0, core;
/* Free microcode bases and reset group masks */
for (grp = 0; grp < CPT_MAX_CORE_GROUPS; grp++) {
struct microcode *mcode = &cpt->mcode[grp];
if (cpt->mcode[grp].code)
dma_free_coherent(&cpt->pdev->dev, mcode->code_size,
mcode->code, mcode->phys_base);
mcode->code = NULL;
}
/* Clear UCODE_BASE registers for all engines */
for (core = 0; core < CPT_MAX_TOTAL_CORES; core++)
cpt_write_csr64(cpt->reg_base,
CPTX_PF_ENGX_UCODE_BASE(0, core), 0ull);
}
static int cpt_device_init(struct cpt_device *cpt)
{
u64 bist;
struct device *dev = &cpt->pdev->dev;
/* Reset the PF when probed first */
cpt_reset(cpt);
msleep(100);
/*Check BIST status*/
bist = (u64)cpt_check_bist_status(cpt);
if (bist) {
dev_err(dev, "RAM BIST failed with code 0x%llx", bist);
return -ENODEV;
}
bist = cpt_check_exe_bist_status(cpt);
if (bist) {
dev_err(dev, "Engine BIST failed with code 0x%llx", bist);
return -ENODEV;
}
/*Get CLK frequency*/
/*Get max enabled cores */
cpt_find_max_enabled_cores(cpt);
/*Disable all cores*/
cpt_disable_all_cores(cpt);
/*Reset device parameters*/
cpt->next_mc_idx = 0;
cpt->next_group = 0;
/* PF is ready */
cpt->flags |= CPT_FLAG_DEVICE_READY;
return 0;
}
static int cpt_register_interrupts(struct cpt_device *cpt)
{
int ret;
struct device *dev = &cpt->pdev->dev;
/* Enable MSI-X */
ret = pci_alloc_irq_vectors(cpt->pdev, CPT_PF_MSIX_VECTORS,
CPT_PF_MSIX_VECTORS, PCI_IRQ_MSIX);
if (ret < 0) {
dev_err(&cpt->pdev->dev, "Request for #%d msix vectors failed\n",
CPT_PF_MSIX_VECTORS);
return ret;
}
/* Register mailbox interrupt handlers */
ret = request_irq(pci_irq_vector(cpt->pdev, CPT_PF_INT_VEC_E_MBOXX(0)),
cpt_mbx0_intr_handler, 0, "CPT Mbox0", cpt);
if (ret)
goto fail;
/* Enable mailbox interrupt */
cpt_enable_mbox_interrupts(cpt);
return 0;
fail:
dev_err(dev, "Request irq failed\n");
pci_disable_msix(cpt->pdev);
return ret;
}
static void cpt_unregister_interrupts(struct cpt_device *cpt)
{
free_irq(pci_irq_vector(cpt->pdev, CPT_PF_INT_VEC_E_MBOXX(0)), cpt);
pci_disable_msix(cpt->pdev);
}
static int cpt_sriov_init(struct cpt_device *cpt, int num_vfs)
{
int pos = 0;
int err;
u16 total_vf_cnt;
struct pci_dev *pdev = cpt->pdev;
pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_SRIOV);
if (!pos) {
dev_err(&pdev->dev, "SRIOV capability is not found in PCIe config space\n");
return -ENODEV;
}
cpt->num_vf_en = num_vfs; /* User requested VFs */
pci_read_config_word(pdev, (pos + PCI_SRIOV_TOTAL_VF), &total_vf_cnt);
if (total_vf_cnt < cpt->num_vf_en)
cpt->num_vf_en = total_vf_cnt;
if (!total_vf_cnt)
return 0;
/*Enabled the available VFs */
err = pci_enable_sriov(pdev, cpt->num_vf_en);
if (err) {
dev_err(&pdev->dev, "SRIOV enable failed, num VF is %d\n",
cpt->num_vf_en);
cpt->num_vf_en = 0;
return err;
}
/* TODO: Optionally enable static VQ priorities feature */
dev_info(&pdev->dev, "SRIOV enabled, number of VF available %d\n",
cpt->num_vf_en);
cpt->flags |= CPT_FLAG_SRIOV_ENABLED;
return 0;
}
static int cpt_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
{
struct device *dev = &pdev->dev;
struct cpt_device *cpt;
int err;
if (num_vfs > 16 || num_vfs < 4) {
dev_warn(dev, "Invalid vf count %d, Resetting it to 4(default)\n",
num_vfs);
num_vfs = 4;
}
cpt = devm_kzalloc(dev, sizeof(*cpt), GFP_KERNEL);
if (!cpt)
return -ENOMEM;
pci_set_drvdata(pdev, cpt);
cpt->pdev = pdev;
err = pci_enable_device(pdev);
if (err) {
dev_err(dev, "Failed to enable PCI device\n");
pci_set_drvdata(pdev, NULL);
return err;
}
err = pci_request_regions(pdev, DRV_NAME);
if (err) {
dev_err(dev, "PCI request regions failed 0x%x\n", err);
goto cpt_err_disable_device;
}
err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(48));
if (err) {
dev_err(dev, "Unable to get usable 48-bit DMA configuration\n");
goto cpt_err_release_regions;
}
/* MAP PF's configuration registers */
cpt->reg_base = pcim_iomap(pdev, 0, 0);
if (!cpt->reg_base) {
dev_err(dev, "Cannot map config register space, aborting\n");
err = -ENOMEM;
goto cpt_err_release_regions;
}
/* CPT device HW initialization */
cpt_device_init(cpt);
/* Register interrupts */
err = cpt_register_interrupts(cpt);
if (err)
goto cpt_err_release_regions;
err = cpt_ucode_load(cpt);
if (err)
goto cpt_err_unregister_interrupts;
/* Configure SRIOV */
err = cpt_sriov_init(cpt, num_vfs);
if (err)
goto cpt_err_unregister_interrupts;
return 0;
cpt_err_unregister_interrupts:
cpt_unregister_interrupts(cpt);
cpt_err_release_regions:
pci_release_regions(pdev);
cpt_err_disable_device:
pci_disable_device(pdev);
pci_set_drvdata(pdev, NULL);
return err;
}
static void cpt_remove(struct pci_dev *pdev)
{
struct cpt_device *cpt = pci_get_drvdata(pdev);
/* Disengage SE and AE cores from all groups*/
cpt_disable_all_cores(cpt);
/* Unload microcodes */
cpt_unload_microcode(cpt);
cpt_unregister_interrupts(cpt);
pci_disable_sriov(pdev);
pci_release_regions(pdev);
pci_disable_device(pdev);
pci_set_drvdata(pdev, NULL);
}
static void cpt_shutdown(struct pci_dev *pdev)
{
struct cpt_device *cpt = pci_get_drvdata(pdev);
if (!cpt)
return;
dev_info(&pdev->dev, "Shutdown device %x:%x.\n",
(u32)pdev->vendor, (u32)pdev->device);
cpt_unregister_interrupts(cpt);
pci_release_regions(pdev);
pci_disable_device(pdev);
pci_set_drvdata(pdev, NULL);
}
/* Supported devices */
static const struct pci_device_id cpt_id_table[] = {
{ PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, CPT_81XX_PCI_PF_DEVICE_ID) },
{ 0, } /* end of table */
};
MODULE_DEVICE_TABLE(pci, cpt_id_table);
static struct pci_driver cpt_pci_driver = {
.name = DRV_NAME,
.id_table = cpt_id_table,
.probe = cpt_probe,
.remove = cpt_remove,
.shutdown = cpt_shutdown,
};
module_pci_driver(cpt_pci_driver);
MODULE_AUTHOR("George Cherian <george.cherian@cavium.com>");
MODULE_DESCRIPTION("Cavium Thunder CPT Physical Function Driver");
MODULE_LICENSE("GPL v2");
MODULE_VERSION(DRV_VERSION);
linux-7.2.3/drivers/crypto/cavium/cpt/cptpf_mbox.c
+160
View File
@@ -0,0 +1,160 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (C) 2016 Cavium, Inc.
*/
#include <linux/module.h>
#include "cptpf.h"
static void cpt_send_msg_to_vf(struct cpt_device *cpt, int vf,
struct cpt_mbox *mbx)
{
/* Writing mbox(0) causes interrupt */
cpt_write_csr64(cpt->reg_base, CPTX_PF_VFX_MBOXX(0, vf, 1),
mbx->data);
cpt_write_csr64(cpt->reg_base, CPTX_PF_VFX_MBOXX(0, vf, 0), mbx->msg);
}
/* ACKs VF's mailbox message
* @vf: VF to which ACK to be sent
*/
static void cpt_mbox_send_ack(struct cpt_device *cpt, int vf,
struct cpt_mbox *mbx)
{
mbx->data = 0ull;
mbx->msg = CPT_MBOX_MSG_TYPE_ACK;
cpt_send_msg_to_vf(cpt, vf, mbx);
}
static void cpt_clear_mbox_intr(struct cpt_device *cpt, u32 vf)
{
/* W1C for the VF */
cpt_write_csr64(cpt->reg_base, CPTX_PF_MBOX_INTX(0, 0), (1 << vf));
}
/*
* Configure QLEN/Chunk sizes for VF
*/
static void cpt_cfg_qlen_for_vf(struct cpt_device *cpt, int vf, u32 size)
{
union cptx_pf_qx_ctl pf_qx_ctl;
pf_qx_ctl.u = cpt_read_csr64(cpt->reg_base, CPTX_PF_QX_CTL(0, vf));
pf_qx_ctl.s.size = size;
pf_qx_ctl.s.cont_err = true;
cpt_write_csr64(cpt->reg_base, CPTX_PF_QX_CTL(0, vf), pf_qx_ctl.u);
}
/*
* Configure VQ priority
*/
static void cpt_cfg_vq_priority(struct cpt_device *cpt, int vf, u32 pri)
{
union cptx_pf_qx_ctl pf_qx_ctl;
pf_qx_ctl.u = cpt_read_csr64(cpt->reg_base, CPTX_PF_QX_CTL(0, vf));
pf_qx_ctl.s.pri = pri;
cpt_write_csr64(cpt->reg_base, CPTX_PF_QX_CTL(0, vf), pf_qx_ctl.u);
}
static int cpt_bind_vq_to_grp(struct cpt_device *cpt, u8 q, u8 grp)
{
struct microcode *mcode = cpt->mcode;
union cptx_pf_qx_ctl pf_qx_ctl;
struct device *dev = &cpt->pdev->dev;
if (q >= CPT_MAX_VF_NUM) {
dev_err(dev, "Queues are more than cores in the group");
return -EINVAL;
}
if (grp >= CPT_MAX_CORE_GROUPS) {
dev_err(dev, "Request group is more than possible groups");
return -EINVAL;
}
if (grp >= cpt->next_mc_idx) {
dev_err(dev, "Request group is higher than available functional groups");
return -EINVAL;
}
pf_qx_ctl.u = cpt_read_csr64(cpt->reg_base, CPTX_PF_QX_CTL(0, q));
pf_qx_ctl.s.grp = mcode[grp].group;
cpt_write_csr64(cpt->reg_base, CPTX_PF_QX_CTL(0, q), pf_qx_ctl.u);
dev_dbg(dev, "VF %d TYPE %s", q, (mcode[grp].is_ae ? "AE" : "SE"));
return mcode[grp].is_ae ? AE_TYPES : SE_TYPES;
}
/* Interrupt handler to handle mailbox messages from VFs */
static void cpt_handle_mbox_intr(struct cpt_device *cpt, int vf)
{
struct cpt_vf_info *vfx = &cpt->vfinfo[vf];
struct cpt_mbox mbx = {};
int vftype;
struct device *dev = &cpt->pdev->dev;
/*
* MBOX[0] contains msg
* MBOX[1] contains data
*/
mbx.msg = cpt_read_csr64(cpt->reg_base, CPTX_PF_VFX_MBOXX(0, vf, 0));
mbx.data = cpt_read_csr64(cpt->reg_base, CPTX_PF_VFX_MBOXX(0, vf, 1));
dev_dbg(dev, "%s: Mailbox msg 0x%llx from VF%d", __func__, mbx.msg, vf);
switch (mbx.msg) {
case CPT_MSG_VF_UP:
vfx->state = VF_STATE_UP;
try_module_get(THIS_MODULE);
cpt_mbox_send_ack(cpt, vf, &mbx);
break;
case CPT_MSG_READY:
mbx.msg = CPT_MSG_READY;
mbx.data = vf;
cpt_send_msg_to_vf(cpt, vf, &mbx);
break;
case CPT_MSG_VF_DOWN:
/* First msg in VF teardown sequence */
vfx->state = VF_STATE_DOWN;
module_put(THIS_MODULE);
cpt_mbox_send_ack(cpt, vf, &mbx);
break;
case CPT_MSG_QLEN:
vfx->qlen = mbx.data;
cpt_cfg_qlen_for_vf(cpt, vf, vfx->qlen);
cpt_mbox_send_ack(cpt, vf, &mbx);
break;
case CPT_MSG_QBIND_GRP:
vftype = cpt_bind_vq_to_grp(cpt, vf, (u8)mbx.data);
if ((vftype != AE_TYPES) && (vftype != SE_TYPES))
dev_err(dev, "Queue %d binding to group %llu failed",
vf, mbx.data);
else {
dev_dbg(dev, "Queue %d binding to group %llu successful",
vf, mbx.data);
mbx.msg = CPT_MSG_QBIND_GRP;
mbx.data = vftype;
cpt_send_msg_to_vf(cpt, vf, &mbx);
}
break;
case CPT_MSG_VQ_PRIORITY:
vfx->priority = mbx.data;
cpt_cfg_vq_priority(cpt, vf, vfx->priority);
cpt_mbox_send_ack(cpt, vf, &mbx);
break;
default:
dev_err(&cpt->pdev->dev, "Invalid msg from VF%d, msg 0x%llx\n",
vf, mbx.msg);
break;
}
}
void cpt_mbox_intr_handler (struct cpt_device *cpt, int mbx)
{
u64 intr;
u8 vf;
intr = cpt_read_csr64(cpt->reg_base, CPTX_PF_MBOX_INTX(0, 0));
dev_dbg(&cpt->pdev->dev, "PF interrupt Mbox%d 0x%llx\n", mbx, intr);
for (vf = 0; vf < CPT_MAX_VF_NUM; vf++) {
if (intr & (1ULL << vf)) {
dev_dbg(&cpt->pdev->dev, "Intr from VF %d\n", vf);
cpt_handle_mbox_intr(cpt, vf);
cpt_clear_mbox_intr(cpt, vf);
}
}
}
linux-7.2.3/drivers/crypto/cavium/cpt/cptvf.h
+129
View File
@@ -0,0 +1,129 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (C) 2016 Cavium, Inc.
*/
#ifndef __CPTVF_H
#define __CPTVF_H
#include <linux/list.h>
#include "cpt_common.h"
/* Default command queue length */
#define CPT_CMD_QLEN 2046
#define CPT_CMD_QCHUNK_SIZE 1023
/* Default command timeout in seconds */
#define CPT_COMMAND_TIMEOUT 4
#define CPT_TIMER_THOLD 0xFFFF
#define CPT_NUM_QS_PER_VF 1
#define CPT_INST_SIZE 64
#define CPT_NEXT_CHUNK_PTR_SIZE 8
#define CPT_VF_MSIX_VECTORS 2
#define CPT_VF_INTR_MBOX_MASK BIT(0)
#define CPT_VF_INTR_DOVF_MASK BIT(1)
#define CPT_VF_INTR_IRDE_MASK BIT(2)
#define CPT_VF_INTR_NWRP_MASK BIT(3)
#define CPT_VF_INTR_SERR_MASK BIT(4)
#define DMA_DIRECT_DIRECT 0 /* Input DIRECT, Output DIRECT */
#define DMA_GATHER_SCATTER 1
#define FROM_DPTR 1
/**
* Enumeration cpt_vf_int_vec_e
*
* CPT VF MSI-X Vector Enumeration
* Enumerates the MSI-X interrupt vectors.
*/
enum cpt_vf_int_vec_e {
CPT_VF_INT_VEC_E_MISC = 0x00,
CPT_VF_INT_VEC_E_DONE = 0x01
};
struct command_chunk {
u8 *head;
dma_addr_t dma_addr;
u32 size; /* Chunk size, max CPT_INST_CHUNK_MAX_SIZE */
struct hlist_node nextchunk;
};
struct command_queue {
spinlock_t lock; /* command queue lock */
u32 idx; /* Command queue host write idx */
u32 nchunks; /* Number of command chunks */
struct command_chunk *qhead; /* Command queue head, instructions
* are inserted here
*/
struct hlist_head chead;
};
struct command_qinfo {
u32 cmd_size;
u32 qchunksize; /* Command queue chunk size */
struct command_queue queue[CPT_NUM_QS_PER_VF];
};
struct pending_entry {
u8 busy; /* Entry status (free/busy) */
volatile u64 *completion_addr; /* Completion address */
void *post_arg;
void (*callback)(int, void *); /* Kernel ASYNC request callabck */
void *callback_arg; /* Kernel ASYNC request callabck arg */
};
struct pending_queue {
struct pending_entry *head; /* head of the queue */
u32 front; /* Process work from here */
u32 rear; /* Append new work here */
atomic64_t pending_count;
spinlock_t lock; /* Queue lock */
};
struct pending_qinfo {
u32 nr_queues; /* Number of queues supported */
u32 qlen; /* Queue length */
struct pending_queue queue[CPT_NUM_QS_PER_VF];
};
#define for_each_pending_queue(qinfo, q, i) \
for (i = 0, q = &qinfo->queue[i]; i < qinfo->nr_queues; i++, \
q = &qinfo->queue[i])
struct cpt_vf {
u16 flags; /* Flags to hold device status bits */
u8 vfid; /* Device Index 0...CPT_MAX_VF_NUM */
u8 vftype; /* VF type of SE_TYPE(1) or AE_TYPE(1) */
u8 vfgrp; /* VF group (0 - 8) */
u8 node; /* Operating node: Bits (46:44) in BAR0 address */
u8 priority; /* VF priority ring: 1-High proirity round
* robin ring;0-Low priority round robin ring;
*/
struct pci_dev *pdev; /* pci device handle */
void __iomem *reg_base; /* Register start address */
void *wqe_info; /* BH worker info */
/* MSI-X */
cpumask_var_t affinity_mask[CPT_VF_MSIX_VECTORS];
/* Command and Pending queues */
u32 qsize;
u32 nr_queues;
struct command_qinfo cqinfo; /* Command queue information */
struct pending_qinfo pqinfo; /* Pending queue information */
/* VF-PF mailbox communication */
bool pf_acked;
bool pf_nacked;
};
int cptvf_send_vf_up(struct cpt_vf *cptvf);
int cptvf_send_vf_down(struct cpt_vf *cptvf);
int cptvf_send_vf_to_grp_msg(struct cpt_vf *cptvf);
int cptvf_send_vf_priority_msg(struct cpt_vf *cptvf);
int cptvf_send_vq_size_msg(struct cpt_vf *cptvf);
int cptvf_check_pf_ready(struct cpt_vf *cptvf);
void cptvf_handle_mbox_intr(struct cpt_vf *cptvf);
void cvm_crypto_exit(void);
int cvm_crypto_init(struct cpt_vf *cptvf);
void vq_post_process(struct cpt_vf *cptvf, u32 qno);
void cptvf_write_vq_doorbell(struct cpt_vf *cptvf, u32 val);
#endif /* __CPTVF_H */
linux-7.2.3/drivers/crypto/cavium/cpt/cptvf_algs.c
+462
View File
@@ -0,0 +1,462 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (C) 2016 Cavium, Inc.
*/
#include <crypto/aes.h>
#include <crypto/algapi.h>
#include <crypto/authenc.h>
#include <crypto/internal/des.h>
#include <crypto/xts.h>
#include <linux/crypto.h>
#include <linux/err.h>
#include <linux/list.h>
#include <linux/scatterlist.h>
#include "cptvf.h"
#include "cptvf_algs.h"
struct cpt_device_handle {
void *cdev[MAX_DEVICES];
u32 dev_count;
};
static struct cpt_device_handle dev_handle;
static void cvm_callback(u32 status, void *arg)
{
struct crypto_async_request *req = (struct crypto_async_request *)arg;
crypto_request_complete(req, !status);
}
static inline void update_input_iv(struct cpt_request_info *req_info,
u8 *iv, u32 enc_iv_len,
u32 *argcnt)
{
/* Setting the iv information */
req_info->in[*argcnt].vptr = (void *)iv;
req_info->in[*argcnt].size = enc_iv_len;
req_info->req.dlen += enc_iv_len;
++(*argcnt);
}
static inline void update_output_iv(struct cpt_request_info *req_info,
u8 *iv, u32 enc_iv_len,
u32 *argcnt)
{
/* Setting the iv information */
req_info->out[*argcnt].vptr = (void *)iv;
req_info->out[*argcnt].size = enc_iv_len;
req_info->rlen += enc_iv_len;
++(*argcnt);
}
static inline void update_input_data(struct cpt_request_info *req_info,
struct scatterlist *inp_sg,
u32 nbytes, u32 *argcnt)
{
req_info->req.dlen += nbytes;
while (nbytes) {
u32 len = min(nbytes, inp_sg->length);
u8 *ptr = sg_virt(inp_sg);
req_info->in[*argcnt].vptr = (void *)ptr;
req_info->in[*argcnt].size = len;
nbytes -= len;
++(*argcnt);
++inp_sg;
}
}
static inline void update_output_data(struct cpt_request_info *req_info,
struct scatterlist *outp_sg,
u32 nbytes, u32 *argcnt)
{
req_info->rlen += nbytes;
while (nbytes) {
u32 len = min(nbytes, outp_sg->length);
u8 *ptr = sg_virt(outp_sg);
req_info->out[*argcnt].vptr = (void *)ptr;
req_info->out[*argcnt].size = len;
nbytes -= len;
++(*argcnt);
++outp_sg;
}
}
static inline u32 create_ctx_hdr(struct skcipher_request *req, u32 enc,
u32 *argcnt)
{
struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
struct cvm_enc_ctx *ctx = crypto_skcipher_ctx(tfm);
struct cvm_req_ctx *rctx = skcipher_request_ctx_dma(req);
struct fc_context *fctx = &rctx->fctx;
u32 enc_iv_len = crypto_skcipher_ivsize(tfm);
struct cpt_request_info *req_info = &rctx->cpt_req;
__be64 *ctrl_flags = NULL;
__be64 *offset_control;
req_info->ctrl.s.grp = 0;
req_info->ctrl.s.dma_mode = DMA_GATHER_SCATTER;
req_info->ctrl.s.se_req = SE_CORE_REQ;
req_info->req.opcode.s.major = MAJOR_OP_FC |
DMA_MODE_FLAG(DMA_GATHER_SCATTER);
if (enc)
req_info->req.opcode.s.minor = 2;
else
req_info->req.opcode.s.minor = 3;
req_info->req.param1 = req->cryptlen; /* Encryption Data length */
req_info->req.param2 = 0; /*Auth data length */
fctx->enc.enc_ctrl.e.enc_cipher = ctx->cipher_type;
fctx->enc.enc_ctrl.e.aes_key = ctx->key_type;
fctx->enc.enc_ctrl.e.iv_source = FROM_DPTR;
if (ctx->cipher_type == AES_XTS)
memcpy(fctx->enc.encr_key, ctx->enc_key, ctx->key_len * 2);
else
memcpy(fctx->enc.encr_key, ctx->enc_key, ctx->key_len);
ctrl_flags = (__be64 *)&fctx->enc.enc_ctrl.flags;
*ctrl_flags = cpu_to_be64(fctx->enc.enc_ctrl.flags);
offset_control = (__be64 *)&rctx->control_word;
*offset_control = cpu_to_be64(((u64)(enc_iv_len) << 16));
/* Storing Packet Data Information in offset
* Control Word First 8 bytes
*/
req_info->in[*argcnt].vptr = (u8 *)offset_control;
req_info->in[*argcnt].size = CONTROL_WORD_LEN;
req_info->req.dlen += CONTROL_WORD_LEN;
++(*argcnt);
req_info->in[*argcnt].vptr = (u8 *)fctx;
req_info->in[*argcnt].size = sizeof(struct fc_context);
req_info->req.dlen += sizeof(struct fc_context);
++(*argcnt);
return 0;
}
static inline u32 create_input_list(struct skcipher_request *req, u32 enc,
u32 enc_iv_len)
{
struct cvm_req_ctx *rctx = skcipher_request_ctx_dma(req);
struct cpt_request_info *req_info = &rctx->cpt_req;
u32 argcnt = 0;
create_ctx_hdr(req, enc, &argcnt);
update_input_iv(req_info, req->iv, enc_iv_len, &argcnt);
update_input_data(req_info, req->src, req->cryptlen, &argcnt);
req_info->incnt = argcnt;
return 0;
}
static inline void store_cb_info(struct skcipher_request *req,
struct cpt_request_info *req_info)
{
req_info->callback = (void *)cvm_callback;
req_info->callback_arg = (void *)&req->base;
}
static inline void create_output_list(struct skcipher_request *req,
u32 enc_iv_len)
{
struct cvm_req_ctx *rctx = skcipher_request_ctx_dma(req);
struct cpt_request_info *req_info = &rctx->cpt_req;
u32 argcnt = 0;
/* OUTPUT Buffer Processing
* AES encryption/decryption output would be
* received in the following format
*
* ------IV--------|------ENCRYPTED/DECRYPTED DATA-----|
* [ 16 Bytes/ [ Request Enc/Dec/ DATA Len AES CBC ]
*/
/* Reading IV information */
update_output_iv(req_info, req->iv, enc_iv_len, &argcnt);
update_output_data(req_info, req->dst, req->cryptlen, &argcnt);
req_info->outcnt = argcnt;
}
static inline int cvm_enc_dec(struct skcipher_request *req, u32 enc)
{
struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
struct cvm_req_ctx *rctx = skcipher_request_ctx_dma(req);
u32 enc_iv_len = crypto_skcipher_ivsize(tfm);
struct fc_context *fctx = &rctx->fctx;
struct cpt_request_info *req_info = &rctx->cpt_req;
void *cdev = NULL;
int status;
memset(req_info, 0, sizeof(struct cpt_request_info));
req_info->may_sleep = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) != 0;
memset(fctx, 0, sizeof(struct fc_context));
create_input_list(req, enc, enc_iv_len);
create_output_list(req, enc_iv_len);
store_cb_info(req, req_info);
cdev = dev_handle.cdev[smp_processor_id()];
status = cptvf_do_request(cdev, req_info);
/* We perform an asynchronous send and once
* the request is completed the driver would
* intimate through registered call back functions
*/
if (status)
return status;
else
return -EINPROGRESS;
}
static int cvm_encrypt(struct skcipher_request *req)
{
return cvm_enc_dec(req, true);
}
static int cvm_decrypt(struct skcipher_request *req)
{
return cvm_enc_dec(req, false);
}
static int cvm_xts_setkey(struct crypto_skcipher *cipher, const u8 *key,
u32 keylen)
{
struct cvm_enc_ctx *ctx = crypto_skcipher_ctx(cipher);
int err;
const u8 *key1 = key;
const u8 *key2 = key + (keylen / 2);
err = xts_verify_key(cipher, key, keylen);
if (err)
return err;
ctx->key_len = keylen;
memcpy(ctx->enc_key, key1, keylen / 2);
memcpy(ctx->enc_key + KEY2_OFFSET, key2, keylen / 2);
ctx->cipher_type = AES_XTS;
switch (ctx->key_len) {
case 32:
ctx->key_type = AES_128_BIT;
break;
case 64:
ctx->key_type = AES_256_BIT;
break;
default:
return -EINVAL;
}
return 0;
}
static int cvm_validate_keylen(struct cvm_enc_ctx *ctx, u32 keylen)
{
if ((keylen == 16) || (keylen == 24) || (keylen == 32)) {
ctx->key_len = keylen;
switch (ctx->key_len) {
case 16:
ctx->key_type = AES_128_BIT;
break;
case 24:
ctx->key_type = AES_192_BIT;
break;
case 32:
ctx->key_type = AES_256_BIT;
break;
default:
return -EINVAL;
}
if (ctx->cipher_type == DES3_CBC)
ctx->key_type = 0;
return 0;
}
return -EINVAL;
}
static int cvm_setkey(struct crypto_skcipher *cipher, const u8 *key,
u32 keylen, u8 cipher_type)
{
struct cvm_enc_ctx *ctx = crypto_skcipher_ctx(cipher);
ctx->cipher_type = cipher_type;
if (!cvm_validate_keylen(ctx, keylen)) {
memcpy(ctx->enc_key, key, keylen);
return 0;
} else {
return -EINVAL;
}
}
static int cvm_cbc_aes_setkey(struct crypto_skcipher *cipher, const u8 *key,
u32 keylen)
{
return cvm_setkey(cipher, key, keylen, AES_CBC);
}
static int cvm_ecb_aes_setkey(struct crypto_skcipher *cipher, const u8 *key,
u32 keylen)
{
return cvm_setkey(cipher, key, keylen, AES_ECB);
}
static int cvm_cbc_des3_setkey(struct crypto_skcipher *cipher, const u8 *key,
u32 keylen)
{
return verify_skcipher_des3_key(cipher, key) ?:
cvm_setkey(cipher, key, keylen, DES3_CBC);
}
static int cvm_ecb_des3_setkey(struct crypto_skcipher *cipher, const u8 *key,
u32 keylen)
{
return verify_skcipher_des3_key(cipher, key) ?:
cvm_setkey(cipher, key, keylen, DES3_ECB);
}
static int cvm_enc_dec_init(struct crypto_skcipher *tfm)
{
crypto_skcipher_set_reqsize_dma(tfm, sizeof(struct cvm_req_ctx));
return 0;
}
static struct skcipher_alg algs[] = { {
.base.cra_flags = CRYPTO_ALG_ASYNC |
CRYPTO_ALG_ALLOCATES_MEMORY,
.base.cra_blocksize = AES_BLOCK_SIZE,
.base.cra_ctxsize = sizeof(struct cvm_enc_ctx),
.base.cra_alignmask = 7,
.base.cra_priority = 4001,
.base.cra_name = "xts(aes)",
.base.cra_driver_name = "cavium-xts-aes",
.base.cra_module = THIS_MODULE,
.ivsize = AES_BLOCK_SIZE,
.min_keysize = 2 * AES_MIN_KEY_SIZE,
.max_keysize = 2 * AES_MAX_KEY_SIZE,
.setkey = cvm_xts_setkey,
.encrypt = cvm_encrypt,
.decrypt = cvm_decrypt,
.init = cvm_enc_dec_init,
}, {
.base.cra_flags = CRYPTO_ALG_ASYNC |
CRYPTO_ALG_ALLOCATES_MEMORY,
.base.cra_blocksize = AES_BLOCK_SIZE,
.base.cra_ctxsize = sizeof(struct cvm_enc_ctx),
.base.cra_alignmask = 7,
.base.cra_priority = 4001,
.base.cra_name = "cbc(aes)",
.base.cra_driver_name = "cavium-cbc-aes",
.base.cra_module = THIS_MODULE,
.ivsize = AES_BLOCK_SIZE,
.min_keysize = AES_MIN_KEY_SIZE,
.max_keysize = AES_MAX_KEY_SIZE,
.setkey = cvm_cbc_aes_setkey,
.encrypt = cvm_encrypt,
.decrypt = cvm_decrypt,
.init = cvm_enc_dec_init,
}, {
.base.cra_flags = CRYPTO_ALG_ASYNC |
CRYPTO_ALG_ALLOCATES_MEMORY,
.base.cra_blocksize = AES_BLOCK_SIZE,
.base.cra_ctxsize = sizeof(struct cvm_enc_ctx),
.base.cra_alignmask = 7,
.base.cra_priority = 4001,
.base.cra_name = "ecb(aes)",
.base.cra_driver_name = "cavium-ecb-aes",
.base.cra_module = THIS_MODULE,
.min_keysize = AES_MIN_KEY_SIZE,
.max_keysize = AES_MAX_KEY_SIZE,
.setkey = cvm_ecb_aes_setkey,
.encrypt = cvm_encrypt,
.decrypt = cvm_decrypt,
.init = cvm_enc_dec_init,
}, {
.base.cra_flags = CRYPTO_ALG_ASYNC |
CRYPTO_ALG_ALLOCATES_MEMORY,
.base.cra_blocksize = DES3_EDE_BLOCK_SIZE,
.base.cra_ctxsize = sizeof(struct cvm_des3_ctx),
.base.cra_alignmask = 7,
.base.cra_priority = 4001,
.base.cra_name = "cbc(des3_ede)",
.base.cra_driver_name = "cavium-cbc-des3_ede",
.base.cra_module = THIS_MODULE,
.min_keysize = DES3_EDE_KEY_SIZE,
.max_keysize = DES3_EDE_KEY_SIZE,
.ivsize = DES_BLOCK_SIZE,
.setkey = cvm_cbc_des3_setkey,
.encrypt = cvm_encrypt,
.decrypt = cvm_decrypt,
.init = cvm_enc_dec_init,
}, {
.base.cra_flags = CRYPTO_ALG_ASYNC |
CRYPTO_ALG_ALLOCATES_MEMORY,
.base.cra_blocksize = DES3_EDE_BLOCK_SIZE,
.base.cra_ctxsize = sizeof(struct cvm_des3_ctx),
.base.cra_alignmask = 7,
.base.cra_priority = 4001,
.base.cra_name = "ecb(des3_ede)",
.base.cra_driver_name = "cavium-ecb-des3_ede",
.base.cra_module = THIS_MODULE,
.min_keysize = DES3_EDE_KEY_SIZE,
.max_keysize = DES3_EDE_KEY_SIZE,
.ivsize = DES_BLOCK_SIZE,
.setkey = cvm_ecb_des3_setkey,
.encrypt = cvm_encrypt,
.decrypt = cvm_decrypt,
.init = cvm_enc_dec_init,
} };
static inline int cav_register_algs(void)
{
return crypto_register_skciphers(algs, ARRAY_SIZE(algs));
}
static inline void cav_unregister_algs(void)
{
crypto_unregister_skciphers(algs, ARRAY_SIZE(algs));
}
int cvm_crypto_init(struct cpt_vf *cptvf)
{
struct pci_dev *pdev = cptvf->pdev;
u32 dev_count;
dev_count = dev_handle.dev_count;
dev_handle.cdev[dev_count] = cptvf;
dev_handle.dev_count++;
if (dev_count == 3) {
if (cav_register_algs()) {
dev_err(&pdev->dev, "Error in registering crypto algorithms\n");
return -EINVAL;
}
}
return 0;
}
void cvm_crypto_exit(void)
{
u32 dev_count;
dev_count = --dev_handle.dev_count;
if (!dev_count)
cav_unregister_algs();
}
linux-7.2.3/drivers/crypto/cavium/cpt/cptvf_algs.h
+117
View File
@@ -0,0 +1,117 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (C) 2016 Cavium, Inc.
*/
#ifndef _CPTVF_ALGS_H_
#define _CPTVF_ALGS_H_
#include "request_manager.h"
#define MAX_DEVICES 16
#define MAJOR_OP_FC 0x33
#define MAX_ENC_KEY_SIZE 32
#define MAX_HASH_KEY_SIZE 64
#define MAX_KEY_SIZE (MAX_ENC_KEY_SIZE + MAX_HASH_KEY_SIZE)
#define CONTROL_WORD_LEN 8
#define KEY2_OFFSET 48
#define DMA_MODE_FLAG(dma_mode) \
(((dma_mode) == DMA_GATHER_SCATTER) ? (1 << 7) : 0)
enum req_type {
AE_CORE_REQ,
SE_CORE_REQ,
};
enum cipher_type {
DES3_CBC = 0x1,
DES3_ECB = 0x2,
AES_CBC = 0x3,
AES_ECB = 0x4,
AES_CFB = 0x5,
AES_CTR = 0x6,
AES_GCM = 0x7,
AES_XTS = 0x8
};
enum aes_type {
AES_128_BIT = 0x1,
AES_192_BIT = 0x2,
AES_256_BIT = 0x3
};
union encr_ctrl {
u64 flags;
struct {
#if defined(__BIG_ENDIAN_BITFIELD)
u64 enc_cipher:4;
u64 reserved1:1;
u64 aes_key:2;
u64 iv_source:1;
u64 hash_type:4;
u64 reserved2:3;
u64 auth_input_type:1;
u64 mac_len:8;
u64 reserved3:8;
u64 encr_offset:16;
u64 iv_offset:8;
u64 auth_offset:8;
#else
u64 auth_offset:8;
u64 iv_offset:8;
u64 encr_offset:16;
u64 reserved3:8;
u64 mac_len:8;
u64 auth_input_type:1;
u64 reserved2:3;
u64 hash_type:4;
u64 iv_source:1;
u64 aes_key:2;
u64 reserved1:1;
u64 enc_cipher:4;
#endif
} e;
};
struct cvm_cipher {
const char *name;
u8 value;
};
struct enc_context {
union encr_ctrl enc_ctrl;
u8 encr_key[32];
u8 encr_iv[16];
};
struct fchmac_context {
u8 ipad[64];
u8 opad[64]; /* or OPAD */
};
struct fc_context {
struct enc_context enc;
struct fchmac_context hmac;
};
struct cvm_enc_ctx {
u32 key_len;
u8 enc_key[MAX_KEY_SIZE];
u8 cipher_type:4;
u8 key_type:2;
};
struct cvm_des3_ctx {
u32 key_len;
u8 des3_key[MAX_KEY_SIZE];
};
struct cvm_req_ctx {
struct cpt_request_info cpt_req;
u64 control_word;
struct fc_context fctx;
};
int cptvf_do_request(void *cptvf, struct cpt_request_info *req);
#endif /*_CPTVF_ALGS_H_*/
linux-7.2.3/drivers/crypto/cavium/cpt/cptvf_main.c
+856
View File
@@ -0,0 +1,856 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (C) 2016 Cavium, Inc.
*/
#include <linux/interrupt.h>
#include <linux/module.h>
#include "cptvf.h"
#define DRV_NAME "thunder-cptvf"
#define DRV_VERSION "1.0"
struct cptvf_wqe {
struct tasklet_struct twork;
void *cptvf;
u32 qno;
};
struct cptvf_wqe_info {
struct cptvf_wqe vq_wqe[CPT_NUM_QS_PER_VF];
};
static void vq_work_handler(unsigned long data)
{
struct cptvf_wqe_info *cwqe_info = (struct cptvf_wqe_info *)data;
struct cptvf_wqe *cwqe = &cwqe_info->vq_wqe[0];
vq_post_process(cwqe->cptvf, cwqe->qno);
}
static int init_worker_threads(struct cpt_vf *cptvf)
{
struct pci_dev *pdev = cptvf->pdev;
struct cptvf_wqe_info *cwqe_info;
int i;
cwqe_info = kzalloc_obj(*cwqe_info);
if (!cwqe_info)
return -ENOMEM;
if (cptvf->nr_queues) {
dev_info(&pdev->dev, "Creating VQ worker threads (%d)\n",
cptvf->nr_queues);
}
for (i = 0; i < cptvf->nr_queues; i++) {
tasklet_init(&cwqe_info->vq_wqe[i].twork, vq_work_handler,
(u64)cwqe_info);
cwqe_info->vq_wqe[i].qno = i;
cwqe_info->vq_wqe[i].cptvf = cptvf;
}
cptvf->wqe_info = cwqe_info;
return 0;
}
static void cleanup_worker_threads(struct cpt_vf *cptvf)
{
struct cptvf_wqe_info *cwqe_info;
struct pci_dev *pdev = cptvf->pdev;
int i;
cwqe_info = (struct cptvf_wqe_info *)cptvf->wqe_info;
if (!cwqe_info)
return;
if (cptvf->nr_queues) {
dev_info(&pdev->dev, "Cleaning VQ worker threads (%u)\n",
cptvf->nr_queues);
}
for (i = 0; i < cptvf->nr_queues; i++)
tasklet_kill(&cwqe_info->vq_wqe[i].twork);
kfree_sensitive(cwqe_info);
cptvf->wqe_info = NULL;
}
static void free_pending_queues(struct pending_qinfo *pqinfo)
{
int i;
struct pending_queue *queue;
for_each_pending_queue(pqinfo, queue, i) {
if (!queue->head)
continue;
/* free single queue */
kfree_sensitive((queue->head));
queue->front = 0;
queue->rear = 0;
return;
}
pqinfo->qlen = 0;
pqinfo->nr_queues = 0;
}
static int alloc_pending_queues(struct pending_qinfo *pqinfo, u32 qlen,
u32 nr_queues)
{
u32 i;
int ret;
struct pending_queue *queue = NULL;
pqinfo->nr_queues = nr_queues;
pqinfo->qlen = qlen;
for_each_pending_queue(pqinfo, queue, i) {
queue->head = kzalloc_objs(*queue->head, qlen);
if (!queue->head) {
ret = -ENOMEM;
goto pending_qfail;
}
queue->front = 0;
queue->rear = 0;
atomic64_set((&queue->pending_count), (0));
/* init queue spin lock */
spin_lock_init(&queue->lock);
}
return 0;
pending_qfail:
free_pending_queues(pqinfo);
return ret;
}
static int init_pending_queues(struct cpt_vf *cptvf, u32 qlen, u32 nr_queues)
{
struct pci_dev *pdev = cptvf->pdev;
int ret;
if (!nr_queues)
return 0;
ret = alloc_pending_queues(&cptvf->pqinfo, qlen, nr_queues);
if (ret) {
dev_err(&pdev->dev, "failed to setup pending queues (%u)\n",
nr_queues);
return ret;
}
return 0;
}
static void cleanup_pending_queues(struct cpt_vf *cptvf)
{
struct pci_dev *pdev = cptvf->pdev;
if (!cptvf->nr_queues)
return;
dev_info(&pdev->dev, "Cleaning VQ pending queue (%u)\n",
cptvf->nr_queues);
free_pending_queues(&cptvf->pqinfo);
}
static void free_command_queues(struct cpt_vf *cptvf,
struct command_qinfo *cqinfo)
{
int i;
struct command_queue *queue = NULL;
struct command_chunk *chunk = NULL;
struct pci_dev *pdev = cptvf->pdev;
struct hlist_node *node;
/* clean up for each queue */
for (i = 0; i < cptvf->nr_queues; i++) {
queue = &cqinfo->queue[i];
if (hlist_empty(&cqinfo->queue[i].chead))
continue;
hlist_for_each_entry_safe(chunk, node, &cqinfo->queue[i].chead,
nextchunk) {
dma_free_coherent(&pdev->dev,
chunk->size + CPT_NEXT_CHUNK_PTR_SIZE,
chunk->head,
chunk->dma_addr);
chunk->head = NULL;
chunk->dma_addr = 0;
hlist_del(&chunk->nextchunk);
kfree_sensitive(chunk);
}
queue->nchunks = 0;
queue->idx = 0;
}
/* common cleanup */
cqinfo->cmd_size = 0;
}
static int alloc_command_queues(struct cpt_vf *cptvf,
struct command_qinfo *cqinfo, size_t cmd_size,
u32 qlen)
{
int i;
size_t q_size;
struct command_queue *queue = NULL;
struct pci_dev *pdev = cptvf->pdev;
/* common init */
cqinfo->cmd_size = cmd_size;
/* Qsize in dwords, needed for SADDR config, 1-next chunk pointer */
cptvf->qsize = min(qlen, cqinfo->qchunksize) *
CPT_NEXT_CHUNK_PTR_SIZE + 1;
/* Qsize in bytes to create space for alignment */
q_size = qlen * cqinfo->cmd_size;
/* per queue initialization */
for (i = 0; i < cptvf->nr_queues; i++) {
size_t c_size = 0;
size_t rem_q_size = q_size;
struct command_chunk *curr = NULL, *first = NULL, *last = NULL;
u32 qcsize_bytes = cqinfo->qchunksize * cqinfo->cmd_size;
queue = &cqinfo->queue[i];
INIT_HLIST_HEAD(&cqinfo->queue[i].chead);
do {
curr = kzalloc_obj(*curr);
if (!curr)
goto cmd_qfail;
c_size = (rem_q_size > qcsize_bytes) ? qcsize_bytes :
rem_q_size;
curr->head = dma_alloc_coherent(&pdev->dev,
c_size + CPT_NEXT_CHUNK_PTR_SIZE,
&curr->dma_addr,
GFP_KERNEL);
if (!curr->head) {
dev_err(&pdev->dev, "Command Q (%d) chunk (%d) allocation failed\n",
i, queue->nchunks);
kfree(curr);
goto cmd_qfail;
}
curr->size = c_size;
if (queue->nchunks == 0) {
hlist_add_head(&curr->nextchunk,
&cqinfo->queue[i].chead);
first = curr;
} else {
hlist_add_behind(&curr->nextchunk,
&last->nextchunk);
}
queue->nchunks++;
rem_q_size -= c_size;
if (last)
*((u64 *)(&last->head[last->size])) = (u64)curr->dma_addr;
last = curr;
} while (rem_q_size);
/* Make the queue circular */
/* Tie back last chunk entry to head */
curr = first;
*((u64 *)(&last->head[last->size])) = (u64)curr->dma_addr;
queue->qhead = curr;
spin_lock_init(&queue->lock);
}
return 0;
cmd_qfail:
free_command_queues(cptvf, cqinfo);
return -ENOMEM;
}
static int init_command_queues(struct cpt_vf *cptvf, u32 qlen)
{
struct pci_dev *pdev = cptvf->pdev;
int ret;
/* setup AE command queues */
ret = alloc_command_queues(cptvf, &cptvf->cqinfo, CPT_INST_SIZE,
qlen);
if (ret) {
dev_err(&pdev->dev, "failed to allocate AE command queues (%u)\n",
cptvf->nr_queues);
return ret;
}
return ret;
}
static void cleanup_command_queues(struct cpt_vf *cptvf)
{
struct pci_dev *pdev = cptvf->pdev;
if (!cptvf->nr_queues)
return;
dev_info(&pdev->dev, "Cleaning VQ command queue (%u)\n",
cptvf->nr_queues);
free_command_queues(cptvf, &cptvf->cqinfo);
}
static void cptvf_sw_cleanup(struct cpt_vf *cptvf)
{
cleanup_worker_threads(cptvf);
cleanup_pending_queues(cptvf);
cleanup_command_queues(cptvf);
}
static int cptvf_sw_init(struct cpt_vf *cptvf, u32 qlen, u32 nr_queues)
{
struct pci_dev *pdev = cptvf->pdev;
int ret = 0;
u32 max_dev_queues = 0;
max_dev_queues = CPT_NUM_QS_PER_VF;
/* possible cpus */
nr_queues = min_t(u32, nr_queues, max_dev_queues);
cptvf->nr_queues = nr_queues;
ret = init_command_queues(cptvf, qlen);
if (ret) {
dev_err(&pdev->dev, "Failed to setup command queues (%u)\n",
nr_queues);
return ret;
}
ret = init_pending_queues(cptvf, qlen, nr_queues);
if (ret) {
dev_err(&pdev->dev, "Failed to setup pending queues (%u)\n",
nr_queues);
goto setup_pqfail;
}
/* Create worker threads for BH processing */
ret = init_worker_threads(cptvf);
if (ret) {
dev_err(&pdev->dev, "Failed to setup worker threads\n");
goto init_work_fail;
}
return 0;
init_work_fail:
cleanup_worker_threads(cptvf);
cleanup_pending_queues(cptvf);
setup_pqfail:
cleanup_command_queues(cptvf);
return ret;
}
static void cptvf_free_irq_affinity(struct cpt_vf *cptvf, int vec)
{
irq_set_affinity_hint(pci_irq_vector(cptvf->pdev, vec), NULL);
free_cpumask_var(cptvf->affinity_mask[vec]);
}
static void cptvf_write_vq_ctl(struct cpt_vf *cptvf, bool val)
{
union cptx_vqx_ctl vqx_ctl;
vqx_ctl.u = cpt_read_csr64(cptvf->reg_base, CPTX_VQX_CTL(0, 0));
vqx_ctl.s.ena = val;
cpt_write_csr64(cptvf->reg_base, CPTX_VQX_CTL(0, 0), vqx_ctl.u);
}
void cptvf_write_vq_doorbell(struct cpt_vf *cptvf, u32 val)
{
union cptx_vqx_doorbell vqx_dbell;
vqx_dbell.u = cpt_read_csr64(cptvf->reg_base,
CPTX_VQX_DOORBELL(0, 0));
vqx_dbell.s.dbell_cnt = val * 8; /* Num of Instructions * 8 words */
cpt_write_csr64(cptvf->reg_base, CPTX_VQX_DOORBELL(0, 0),
vqx_dbell.u);
}
static void cptvf_write_vq_inprog(struct cpt_vf *cptvf, u8 val)
{
union cptx_vqx_inprog vqx_inprg;
vqx_inprg.u = cpt_read_csr64(cptvf->reg_base, CPTX_VQX_INPROG(0, 0));
vqx_inprg.s.inflight = val;
cpt_write_csr64(cptvf->reg_base, CPTX_VQX_INPROG(0, 0), vqx_inprg.u);
}
static void cptvf_write_vq_done_numwait(struct cpt_vf *cptvf, u32 val)
{
union cptx_vqx_done_wait vqx_dwait;
vqx_dwait.u = cpt_read_csr64(cptvf->reg_base,
CPTX_VQX_DONE_WAIT(0, 0));
vqx_dwait.s.num_wait = val;
cpt_write_csr64(cptvf->reg_base, CPTX_VQX_DONE_WAIT(0, 0),
vqx_dwait.u);
}
static void cptvf_write_vq_done_timewait(struct cpt_vf *cptvf, u16 time)
{
union cptx_vqx_done_wait vqx_dwait;
vqx_dwait.u = cpt_read_csr64(cptvf->reg_base,
CPTX_VQX_DONE_WAIT(0, 0));
vqx_dwait.s.time_wait = time;
cpt_write_csr64(cptvf->reg_base, CPTX_VQX_DONE_WAIT(0, 0),
vqx_dwait.u);
}
static void cptvf_enable_swerr_interrupts(struct cpt_vf *cptvf)
{
union cptx_vqx_misc_ena_w1s vqx_misc_ena;
vqx_misc_ena.u = cpt_read_csr64(cptvf->reg_base,
CPTX_VQX_MISC_ENA_W1S(0, 0));
/* Set mbox(0) interupts for the requested vf */
vqx_misc_ena.s.swerr = 1;
cpt_write_csr64(cptvf->reg_base, CPTX_VQX_MISC_ENA_W1S(0, 0),
vqx_misc_ena.u);
}
static void cptvf_enable_mbox_interrupts(struct cpt_vf *cptvf)
{
union cptx_vqx_misc_ena_w1s vqx_misc_ena;
vqx_misc_ena.u = cpt_read_csr64(cptvf->reg_base,
CPTX_VQX_MISC_ENA_W1S(0, 0));
/* Set mbox(0) interupts for the requested vf */
vqx_misc_ena.s.mbox = 1;
cpt_write_csr64(cptvf->reg_base, CPTX_VQX_MISC_ENA_W1S(0, 0),
vqx_misc_ena.u);
}
static void cptvf_enable_done_interrupts(struct cpt_vf *cptvf)
{
union cptx_vqx_done_ena_w1s vqx_done_ena;
vqx_done_ena.u = cpt_read_csr64(cptvf->reg_base,
CPTX_VQX_DONE_ENA_W1S(0, 0));
/* Set DONE interrupt for the requested vf */
vqx_done_ena.s.done = 1;
cpt_write_csr64(cptvf->reg_base, CPTX_VQX_DONE_ENA_W1S(0, 0),
vqx_done_ena.u);
}
static void cptvf_clear_dovf_intr(struct cpt_vf *cptvf)
{
union cptx_vqx_misc_int vqx_misc_int;
vqx_misc_int.u = cpt_read_csr64(cptvf->reg_base,
CPTX_VQX_MISC_INT(0, 0));
/* W1C for the VF */
vqx_misc_int.s.dovf = 1;
cpt_write_csr64(cptvf->reg_base, CPTX_VQX_MISC_INT(0, 0),
vqx_misc_int.u);
}
static void cptvf_clear_irde_intr(struct cpt_vf *cptvf)
{
union cptx_vqx_misc_int vqx_misc_int;
vqx_misc_int.u = cpt_read_csr64(cptvf->reg_base,
CPTX_VQX_MISC_INT(0, 0));
/* W1C for the VF */
vqx_misc_int.s.irde = 1;
cpt_write_csr64(cptvf->reg_base, CPTX_VQX_MISC_INT(0, 0),
vqx_misc_int.u);
}
static void cptvf_clear_nwrp_intr(struct cpt_vf *cptvf)
{
union cptx_vqx_misc_int vqx_misc_int;
vqx_misc_int.u = cpt_read_csr64(cptvf->reg_base,
CPTX_VQX_MISC_INT(0, 0));
/* W1C for the VF */
vqx_misc_int.s.nwrp = 1;
cpt_write_csr64(cptvf->reg_base,
CPTX_VQX_MISC_INT(0, 0), vqx_misc_int.u);
}
static void cptvf_clear_mbox_intr(struct cpt_vf *cptvf)
{
union cptx_vqx_misc_int vqx_misc_int;
vqx_misc_int.u = cpt_read_csr64(cptvf->reg_base,
CPTX_VQX_MISC_INT(0, 0));
/* W1C for the VF */
vqx_misc_int.s.mbox = 1;
cpt_write_csr64(cptvf->reg_base, CPTX_VQX_MISC_INT(0, 0),
vqx_misc_int.u);
}
static void cptvf_clear_swerr_intr(struct cpt_vf *cptvf)
{
union cptx_vqx_misc_int vqx_misc_int;
vqx_misc_int.u = cpt_read_csr64(cptvf->reg_base,
CPTX_VQX_MISC_INT(0, 0));
/* W1C for the VF */
vqx_misc_int.s.swerr = 1;
cpt_write_csr64(cptvf->reg_base, CPTX_VQX_MISC_INT(0, 0),
vqx_misc_int.u);
}
static u64 cptvf_read_vf_misc_intr_status(struct cpt_vf *cptvf)
{
return cpt_read_csr64(cptvf->reg_base, CPTX_VQX_MISC_INT(0, 0));
}
static irqreturn_t cptvf_misc_intr_handler(int irq, void *cptvf_irq)
{
struct cpt_vf *cptvf = (struct cpt_vf *)cptvf_irq;
struct pci_dev *pdev = cptvf->pdev;
u64 intr;
intr = cptvf_read_vf_misc_intr_status(cptvf);
/*Check for MISC interrupt types*/
if (likely(intr & CPT_VF_INTR_MBOX_MASK)) {
dev_dbg(&pdev->dev, "Mailbox interrupt 0x%llx on CPT VF %d\n",
intr, cptvf->vfid);
cptvf_handle_mbox_intr(cptvf);
cptvf_clear_mbox_intr(cptvf);
} else if (unlikely(intr & CPT_VF_INTR_DOVF_MASK)) {
cptvf_clear_dovf_intr(cptvf);
/*Clear doorbell count*/
cptvf_write_vq_doorbell(cptvf, 0);
dev_err(&pdev->dev, "Doorbell overflow error interrupt 0x%llx on CPT VF %d\n",
intr, cptvf->vfid);
} else if (unlikely(intr & CPT_VF_INTR_IRDE_MASK)) {
cptvf_clear_irde_intr(cptvf);
dev_err(&pdev->dev, "Instruction NCB read error interrupt 0x%llx on CPT VF %d\n",
intr, cptvf->vfid);
} else if (unlikely(intr & CPT_VF_INTR_NWRP_MASK)) {
cptvf_clear_nwrp_intr(cptvf);
dev_err(&pdev->dev, "NCB response write error interrupt 0x%llx on CPT VF %d\n",
intr, cptvf->vfid);
} else if (unlikely(intr & CPT_VF_INTR_SERR_MASK)) {
cptvf_clear_swerr_intr(cptvf);
dev_err(&pdev->dev, "Software error interrupt 0x%llx on CPT VF %d\n",
intr, cptvf->vfid);
} else {
dev_err(&pdev->dev, "Unhandled interrupt in CPT VF %d\n",
cptvf->vfid);
}
return IRQ_HANDLED;
}
static inline struct cptvf_wqe *get_cptvf_vq_wqe(struct cpt_vf *cptvf,
int qno)
{
struct cptvf_wqe_info *nwqe_info;
if (unlikely(qno >= cptvf->nr_queues))
return NULL;
nwqe_info = (struct cptvf_wqe_info *)cptvf->wqe_info;
return &nwqe_info->vq_wqe[qno];
}
static inline u32 cptvf_read_vq_done_count(struct cpt_vf *cptvf)
{
union cptx_vqx_done vqx_done;
vqx_done.u = cpt_read_csr64(cptvf->reg_base, CPTX_VQX_DONE(0, 0));
return vqx_done.s.done;
}
static inline void cptvf_write_vq_done_ack(struct cpt_vf *cptvf,
u32 ackcnt)
{
union cptx_vqx_done_ack vqx_dack_cnt;
vqx_dack_cnt.u = cpt_read_csr64(cptvf->reg_base,
CPTX_VQX_DONE_ACK(0, 0));
vqx_dack_cnt.s.done_ack = ackcnt;
cpt_write_csr64(cptvf->reg_base, CPTX_VQX_DONE_ACK(0, 0),
vqx_dack_cnt.u);
}
static irqreturn_t cptvf_done_intr_handler(int irq, void *cptvf_irq)
{
struct cpt_vf *cptvf = (struct cpt_vf *)cptvf_irq;
struct pci_dev *pdev = cptvf->pdev;
/* Read the number of completions */
u32 intr = cptvf_read_vq_done_count(cptvf);
if (intr) {
struct cptvf_wqe *wqe;
/* Acknowledge the number of
* scheduled completions for processing
*/
cptvf_write_vq_done_ack(cptvf, intr);
wqe = get_cptvf_vq_wqe(cptvf, 0);
if (unlikely(!wqe)) {
dev_err(&pdev->dev, "No work to schedule for VF (%d)",
cptvf->vfid);
return IRQ_NONE;
}
tasklet_hi_schedule(&wqe->twork);
}
return IRQ_HANDLED;
}
static void cptvf_set_irq_affinity(struct cpt_vf *cptvf, int vec)
{
struct pci_dev *pdev = cptvf->pdev;
int cpu;
if (!zalloc_cpumask_var(&cptvf->affinity_mask[vec],
GFP_KERNEL)) {
dev_err(&pdev->dev, "Allocation failed for affinity_mask for VF %d",
cptvf->vfid);
return;
}
cpu = cptvf->vfid % num_online_cpus();
cpumask_set_cpu(cpumask_local_spread(cpu, cptvf->node),
cptvf->affinity_mask[vec]);
irq_set_affinity_hint(pci_irq_vector(pdev, vec),
cptvf->affinity_mask[vec]);
}
static void cptvf_write_vq_saddr(struct cpt_vf *cptvf, u64 val)
{
union cptx_vqx_saddr vqx_saddr;
vqx_saddr.u = val;
cpt_write_csr64(cptvf->reg_base, CPTX_VQX_SADDR(0, 0), vqx_saddr.u);
}
static void cptvf_device_init(struct cpt_vf *cptvf)
{
u64 base_addr = 0;
/* Disable the VQ */
cptvf_write_vq_ctl(cptvf, 0);
/* Reset the doorbell */
cptvf_write_vq_doorbell(cptvf, 0);
/* Clear inflight */
cptvf_write_vq_inprog(cptvf, 0);
/* Write VQ SADDR */
/* TODO: for now only one queue, so hard coded */
base_addr = (u64)(cptvf->cqinfo.queue[0].qhead->dma_addr);
cptvf_write_vq_saddr(cptvf, base_addr);
/* Configure timerhold / coalescence */
cptvf_write_vq_done_timewait(cptvf, CPT_TIMER_THOLD);
cptvf_write_vq_done_numwait(cptvf, 1);
/* Enable the VQ */
cptvf_write_vq_ctl(cptvf, 1);
/* Flag the VF ready */
cptvf->flags |= CPT_FLAG_DEVICE_READY;
}
static int cptvf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
{
struct device *dev = &pdev->dev;
struct cpt_vf *cptvf;
int err;
cptvf = devm_kzalloc(dev, sizeof(*cptvf), GFP_KERNEL);
if (!cptvf)
return -ENOMEM;
pci_set_drvdata(pdev, cptvf);
cptvf->pdev = pdev;
err = pci_enable_device(pdev);
if (err) {
dev_err(dev, "Failed to enable PCI device\n");
pci_set_drvdata(pdev, NULL);
return err;
}
err = pci_request_regions(pdev, DRV_NAME);
if (err) {
dev_err(dev, "PCI request regions failed 0x%x\n", err);
goto cptvf_err_disable_device;
}
/* Mark as VF driver */
cptvf->flags |= CPT_FLAG_VF_DRIVER;
err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(48));
if (err) {
dev_err(dev, "Unable to get usable 48-bit DMA configuration\n");
goto cptvf_err_release_regions;
}
/* MAP PF's configuration registers */
cptvf->reg_base = pcim_iomap(pdev, 0, 0);
if (!cptvf->reg_base) {
dev_err(dev, "Cannot map config register space, aborting\n");
err = -ENOMEM;
goto cptvf_err_release_regions;
}
cptvf->node = dev_to_node(&pdev->dev);
err = pci_alloc_irq_vectors(pdev, CPT_VF_MSIX_VECTORS,
CPT_VF_MSIX_VECTORS, PCI_IRQ_MSIX);
if (err < 0) {
dev_err(dev, "Request for #%d msix vectors failed\n",
CPT_VF_MSIX_VECTORS);
goto cptvf_err_release_regions;
}
err = request_irq(pci_irq_vector(pdev, CPT_VF_INT_VEC_E_MISC),
cptvf_misc_intr_handler, 0, "CPT VF misc intr",
cptvf);
if (err) {
dev_err(dev, "Request misc irq failed");
goto cptvf_free_vectors;
}
/* Enable mailbox interrupt */
cptvf_enable_mbox_interrupts(cptvf);
cptvf_enable_swerr_interrupts(cptvf);
/* Check ready with PF */
/* Gets chip ID / device Id from PF if ready */
err = cptvf_check_pf_ready(cptvf);
if (err) {
dev_err(dev, "PF not responding to READY msg");
goto cptvf_free_misc_irq;
}
/* CPT VF software resources initialization */
cptvf->cqinfo.qchunksize = CPT_CMD_QCHUNK_SIZE;
err = cptvf_sw_init(cptvf, CPT_CMD_QLEN, CPT_NUM_QS_PER_VF);
if (err) {
dev_err(dev, "cptvf_sw_init() failed");
goto cptvf_free_misc_irq;
}
/* Convey VQ LEN to PF */
err = cptvf_send_vq_size_msg(cptvf);
if (err) {
dev_err(dev, "PF not responding to QLEN msg");
goto cptvf_free_misc_irq;
}
/* CPT VF device initialization */
cptvf_device_init(cptvf);
/* Send msg to PF to assign currnet Q to required group */
cptvf->vfgrp = 1;
err = cptvf_send_vf_to_grp_msg(cptvf);
if (err) {
dev_err(dev, "PF not responding to VF_GRP msg");
goto cptvf_free_misc_irq;
}
cptvf->priority = 1;
err = cptvf_send_vf_priority_msg(cptvf);
if (err) {
dev_err(dev, "PF not responding to VF_PRIO msg");
goto cptvf_free_misc_irq;
}
err = request_irq(pci_irq_vector(pdev, CPT_VF_INT_VEC_E_DONE),
cptvf_done_intr_handler, 0, "CPT VF done intr",
cptvf);
if (err) {
dev_err(dev, "Request done irq failed\n");
goto cptvf_free_misc_irq;
}
/* Enable mailbox interrupt */
cptvf_enable_done_interrupts(cptvf);
/* Set irq affinity masks */
cptvf_set_irq_affinity(cptvf, CPT_VF_INT_VEC_E_MISC);
cptvf_set_irq_affinity(cptvf, CPT_VF_INT_VEC_E_DONE);
err = cptvf_send_vf_up(cptvf);
if (err) {
dev_err(dev, "PF not responding to UP msg");
goto cptvf_free_irq_affinity;
}
err = cvm_crypto_init(cptvf);
if (err) {
dev_err(dev, "Algorithm register failed\n");
goto cptvf_free_irq_affinity;
}
return 0;
cptvf_free_irq_affinity:
cptvf_free_irq_affinity(cptvf, CPT_VF_INT_VEC_E_DONE);
cptvf_free_irq_affinity(cptvf, CPT_VF_INT_VEC_E_MISC);
cptvf_free_misc_irq:
free_irq(pci_irq_vector(pdev, CPT_VF_INT_VEC_E_MISC), cptvf);
cptvf_free_vectors:
pci_free_irq_vectors(cptvf->pdev);
cptvf_err_release_regions:
pci_release_regions(pdev);
cptvf_err_disable_device:
pci_disable_device(pdev);
pci_set_drvdata(pdev, NULL);
return err;
}
static void cptvf_remove(struct pci_dev *pdev)
{
struct cpt_vf *cptvf = pci_get_drvdata(pdev);
if (!cptvf) {
dev_err(&pdev->dev, "Invalid CPT-VF device\n");
return;
}
/* Convey DOWN to PF */
if (cptvf_send_vf_down(cptvf)) {
dev_err(&pdev->dev, "PF not responding to DOWN msg");
} else {
cptvf_free_irq_affinity(cptvf, CPT_VF_INT_VEC_E_DONE);
cptvf_free_irq_affinity(cptvf, CPT_VF_INT_VEC_E_MISC);
free_irq(pci_irq_vector(pdev, CPT_VF_INT_VEC_E_DONE), cptvf);
free_irq(pci_irq_vector(pdev, CPT_VF_INT_VEC_E_MISC), cptvf);
pci_free_irq_vectors(cptvf->pdev);
cptvf_sw_cleanup(cptvf);
pci_set_drvdata(pdev, NULL);
pci_release_regions(pdev);
pci_disable_device(pdev);
cvm_crypto_exit();
}
}
static void cptvf_shutdown(struct pci_dev *pdev)
{
cptvf_remove(pdev);
}
/* Supported devices */
static const struct pci_device_id cptvf_id_table[] = {
{ PCI_VDEVICE(CAVIUM, CPT_81XX_PCI_VF_DEVICE_ID) },
{ } /* end of table */
};
MODULE_DEVICE_TABLE(pci, cptvf_id_table);
static struct pci_driver cptvf_pci_driver = {
.name = DRV_NAME,
.id_table = cptvf_id_table,
.probe = cptvf_probe,
.remove = cptvf_remove,
.shutdown = cptvf_shutdown,
};
module_pci_driver(cptvf_pci_driver);
MODULE_AUTHOR("George Cherian <george.cherian@cavium.com>");
MODULE_DESCRIPTION("Cavium Thunder CPT Virtual Function Driver");
MODULE_LICENSE("GPL v2");
MODULE_VERSION(DRV_VERSION);
linux-7.2.3/drivers/crypto/cavium/cpt/cptvf_mbox.c
+191
View File
@@ -0,0 +1,191 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (C) 2016 Cavium, Inc.
*/
#include "cptvf.h"
static void cptvf_send_msg_to_pf(struct cpt_vf *cptvf, struct cpt_mbox *mbx)
{
/* Writing mbox(1) causes interrupt */
cpt_write_csr64(cptvf->reg_base, CPTX_VFX_PF_MBOXX(0, 0, 0),
mbx->msg);
cpt_write_csr64(cptvf->reg_base, CPTX_VFX_PF_MBOXX(0, 0, 1),
mbx->data);
}
/* Interrupt handler to handle mailbox messages from VFs */
void cptvf_handle_mbox_intr(struct cpt_vf *cptvf)
{
struct cpt_mbox mbx = {};
/*
* MBOX[0] contains msg
* MBOX[1] contains data
*/
mbx.msg = cpt_read_csr64(cptvf->reg_base, CPTX_VFX_PF_MBOXX(0, 0, 0));
mbx.data = cpt_read_csr64(cptvf->reg_base, CPTX_VFX_PF_MBOXX(0, 0, 1));
dev_dbg(&cptvf->pdev->dev, "%s: Mailbox msg 0x%llx from PF\n",
__func__, mbx.msg);
switch (mbx.msg) {
case CPT_MSG_READY:
{
cptvf->pf_acked = true;
cptvf->vfid = mbx.data;
dev_dbg(&cptvf->pdev->dev, "Received VFID %d\n", cptvf->vfid);
break;
}
case CPT_MSG_QBIND_GRP:
cptvf->pf_acked = true;
cptvf->vftype = mbx.data;
dev_dbg(&cptvf->pdev->dev, "VF %d type %s group %d\n",
cptvf->vfid, ((mbx.data == SE_TYPES) ? "SE" : "AE"),
cptvf->vfgrp);
break;
case CPT_MBOX_MSG_TYPE_ACK:
cptvf->pf_acked = true;
break;
case CPT_MBOX_MSG_TYPE_NACK:
cptvf->pf_nacked = true;
break;
default:
dev_err(&cptvf->pdev->dev, "Invalid msg from PF, msg 0x%llx\n",
mbx.msg);
break;
}
}
static int cptvf_send_msg_to_pf_timeout(struct cpt_vf *cptvf,
struct cpt_mbox *mbx)
{
int timeout = CPT_MBOX_MSG_TIMEOUT;
int sleep = 10;
cptvf->pf_acked = false;
cptvf->pf_nacked = false;
cptvf_send_msg_to_pf(cptvf, mbx);
/* Wait for previous message to be acked, timeout 2sec */
while (!cptvf->pf_acked) {
if (cptvf->pf_nacked)
return -EINVAL;
msleep(sleep);
if (cptvf->pf_acked)
break;
timeout -= sleep;
if (!timeout) {
dev_err(&cptvf->pdev->dev, "PF didn't ack to mbox msg %llx from VF%u\n",
(mbx->msg & 0xFF), cptvf->vfid);
return -EBUSY;
}
}
return 0;
}
/*
* Checks if VF is able to comminicate with PF
* and also gets the CPT number this VF is associated to.
*/
int cptvf_check_pf_ready(struct cpt_vf *cptvf)
{
struct pci_dev *pdev = cptvf->pdev;
struct cpt_mbox mbx = {};
mbx.msg = CPT_MSG_READY;
if (cptvf_send_msg_to_pf_timeout(cptvf, &mbx)) {
dev_err(&pdev->dev, "PF didn't respond to READY msg\n");
return -EBUSY;
}
return 0;
}
/*
* Communicate VQs size to PF to program CPT(0)_PF_Q(0-15)_CTL of the VF.
* Must be ACKed.
*/
int cptvf_send_vq_size_msg(struct cpt_vf *cptvf)
{
struct pci_dev *pdev = cptvf->pdev;
struct cpt_mbox mbx = {};
mbx.msg = CPT_MSG_QLEN;
mbx.data = cptvf->qsize;
if (cptvf_send_msg_to_pf_timeout(cptvf, &mbx)) {
dev_err(&pdev->dev, "PF didn't respond to vq_size msg\n");
return -EBUSY;
}
return 0;
}
/*
* Communicate VF group required to PF and get the VQ binded to that group
*/
int cptvf_send_vf_to_grp_msg(struct cpt_vf *cptvf)
{
struct pci_dev *pdev = cptvf->pdev;
struct cpt_mbox mbx = {};
mbx.msg = CPT_MSG_QBIND_GRP;
/* Convey group of the VF */
mbx.data = cptvf->vfgrp;
if (cptvf_send_msg_to_pf_timeout(cptvf, &mbx)) {
dev_err(&pdev->dev, "PF didn't respond to vf_type msg\n");
return -EBUSY;
}
return 0;
}
/*
* Communicate VF group required to PF and get the VQ binded to that group
*/
int cptvf_send_vf_priority_msg(struct cpt_vf *cptvf)
{
struct pci_dev *pdev = cptvf->pdev;
struct cpt_mbox mbx = {};
mbx.msg = CPT_MSG_VQ_PRIORITY;
/* Convey group of the VF */
mbx.data = cptvf->priority;
if (cptvf_send_msg_to_pf_timeout(cptvf, &mbx)) {
dev_err(&pdev->dev, "PF didn't respond to vf_type msg\n");
return -EBUSY;
}
return 0;
}
/*
* Communicate to PF that VF is UP and running
*/
int cptvf_send_vf_up(struct cpt_vf *cptvf)
{
struct pci_dev *pdev = cptvf->pdev;
struct cpt_mbox mbx = {};
mbx.msg = CPT_MSG_VF_UP;
if (cptvf_send_msg_to_pf_timeout(cptvf, &mbx)) {
dev_err(&pdev->dev, "PF didn't respond to UP msg\n");
return -EBUSY;
}
return 0;
}
/*
* Communicate to PF that VF is DOWN and running
*/
int cptvf_send_vf_down(struct cpt_vf *cptvf)
{
struct pci_dev *pdev = cptvf->pdev;
struct cpt_mbox mbx = {};
mbx.msg = CPT_MSG_VF_DOWN;
if (cptvf_send_msg_to_pf_timeout(cptvf, &mbx)) {
dev_err(&pdev->dev, "PF didn't respond to DOWN msg\n");
return -EBUSY;
}
return 0;
}
linux-7.2.3/drivers/crypto/cavium/cpt/cptvf_reqmanager.c
+575
View File
@@ -0,0 +1,575 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (C) 2016 Cavium, Inc.
*/
#include "cptvf.h"
#include "cptvf_algs.h"
#include "request_manager.h"
/**
* get_free_pending_entry - get free entry from pending queue
* @q: pending queue
* @qlen: queue length
*/
static struct pending_entry *get_free_pending_entry(struct pending_queue *q,
int qlen)
{
struct pending_entry *ent = NULL;
ent = &q->head[q->rear];
if (unlikely(ent->busy)) {
ent = NULL;
goto no_free_entry;
}
q->rear++;
if (unlikely(q->rear == qlen))
q->rear = 0;
no_free_entry:
return ent;
}
static inline void pending_queue_inc_front(struct pending_qinfo *pqinfo,
int qno)
{
struct pending_queue *queue = &pqinfo->queue[qno];
queue->front++;
if (unlikely(queue->front == pqinfo->qlen))
queue->front = 0;
}
static int setup_sgio_components(struct cpt_vf *cptvf, struct buf_ptr *list,
int buf_count, u8 *buffer)
{
int ret = 0, i, j;
int components;
struct sglist_component *sg_ptr = NULL;
struct pci_dev *pdev = cptvf->pdev;
if (unlikely(!list)) {
dev_err(&pdev->dev, "Input List pointer is NULL\n");
return -EFAULT;
}
for (i = 0; i < buf_count; i++) {
if (likely(list[i].vptr)) {
list[i].dma_addr = dma_map_single(&pdev->dev,
list[i].vptr,
list[i].size,
DMA_BIDIRECTIONAL);
if (unlikely(dma_mapping_error(&pdev->dev,
list[i].dma_addr))) {
dev_err(&pdev->dev, "DMA map kernel buffer failed for component: %d\n",
i);
ret = -EIO;
goto sg_cleanup;
}
}
}
components = buf_count / 4;
sg_ptr = (struct sglist_component *)buffer;
for (i = 0; i < components; i++) {
sg_ptr->u.s.len0 = cpu_to_be16(list[i * 4 + 0].size);
sg_ptr->u.s.len1 = cpu_to_be16(list[i * 4 + 1].size);
sg_ptr->u.s.len2 = cpu_to_be16(list[i * 4 + 2].size);
sg_ptr->u.s.len3 = cpu_to_be16(list[i * 4 + 3].size);
sg_ptr->ptr0 = cpu_to_be64(list[i * 4 + 0].dma_addr);
sg_ptr->ptr1 = cpu_to_be64(list[i * 4 + 1].dma_addr);
sg_ptr->ptr2 = cpu_to_be64(list[i * 4 + 2].dma_addr);
sg_ptr->ptr3 = cpu_to_be64(list[i * 4 + 3].dma_addr);
sg_ptr++;
}
components = buf_count % 4;
switch (components) {
case 3:
sg_ptr->u.s.len2 = cpu_to_be16(list[i * 4 + 2].size);
sg_ptr->ptr2 = cpu_to_be64(list[i * 4 + 2].dma_addr);
fallthrough;
case 2:
sg_ptr->u.s.len1 = cpu_to_be16(list[i * 4 + 1].size);
sg_ptr->ptr1 = cpu_to_be64(list[i * 4 + 1].dma_addr);
fallthrough;
case 1:
sg_ptr->u.s.len0 = cpu_to_be16(list[i * 4 + 0].size);
sg_ptr->ptr0 = cpu_to_be64(list[i * 4 + 0].dma_addr);
break;
default:
break;
}
return ret;
sg_cleanup:
for (j = 0; j < i; j++) {
if (list[j].dma_addr) {
dma_unmap_single(&pdev->dev, list[j].dma_addr,
list[j].size, DMA_BIDIRECTIONAL);
}
list[j].dma_addr = 0;
}
return ret;
}
static inline int setup_sgio_list(struct cpt_vf *cptvf,
struct cpt_info_buffer *info,
struct cpt_request_info *req)
{
u16 g_sz_bytes = 0, s_sz_bytes = 0;
int ret = 0;
struct pci_dev *pdev = cptvf->pdev;
if (req->incnt > MAX_SG_IN_CNT || req->outcnt > MAX_SG_OUT_CNT) {
dev_err(&pdev->dev, "Request SG components are higher than supported\n");
ret = -EINVAL;
goto scatter_gather_clean;
}
/* Setup gather (input) components */
g_sz_bytes = ((req->incnt + 3) / 4) * sizeof(struct sglist_component);
info->gather_components = kzalloc(g_sz_bytes, req->may_sleep ? GFP_KERNEL : GFP_ATOMIC);
if (!info->gather_components) {
ret = -ENOMEM;
goto scatter_gather_clean;
}
ret = setup_sgio_components(cptvf, req->in,
req->incnt,
info->gather_components);
if (ret) {
dev_err(&pdev->dev, "Failed to setup gather list\n");
ret = -EFAULT;
goto scatter_gather_clean;
}
/* Setup scatter (output) components */
s_sz_bytes = ((req->outcnt + 3) / 4) * sizeof(struct sglist_component);
info->scatter_components = kzalloc(s_sz_bytes, req->may_sleep ? GFP_KERNEL : GFP_ATOMIC);
if (!info->scatter_components) {
ret = -ENOMEM;
goto scatter_gather_clean;
}
ret = setup_sgio_components(cptvf, req->out,
req->outcnt,
info->scatter_components);
if (ret) {
dev_err(&pdev->dev, "Failed to setup gather list\n");
ret = -EFAULT;
goto scatter_gather_clean;
}
/* Create and initialize DPTR */
info->dlen = g_sz_bytes + s_sz_bytes + SG_LIST_HDR_SIZE;
info->in_buffer = kzalloc(info->dlen, req->may_sleep ? GFP_KERNEL : GFP_ATOMIC);
if (!info->in_buffer) {
ret = -ENOMEM;
goto scatter_gather_clean;
}
((__be16 *)info->in_buffer)[0] = cpu_to_be16(req->outcnt);
((__be16 *)info->in_buffer)[1] = cpu_to_be16(req->incnt);
((__be16 *)info->in_buffer)[2] = 0;
((__be16 *)info->in_buffer)[3] = 0;
memcpy(&info->in_buffer[8], info->gather_components,
g_sz_bytes);
memcpy(&info->in_buffer[8 + g_sz_bytes],
info->scatter_components, s_sz_bytes);
info->dptr_baddr = dma_map_single(&pdev->dev,
(void *)info->in_buffer,
info->dlen,
DMA_BIDIRECTIONAL);
if (dma_mapping_error(&pdev->dev, info->dptr_baddr)) {
dev_err(&pdev->dev, "Mapping DPTR Failed %d\n", info->dlen);
ret = -EIO;
goto scatter_gather_clean;
}
/* Create and initialize RPTR */
info->out_buffer = kzalloc(COMPLETION_CODE_SIZE, req->may_sleep ? GFP_KERNEL : GFP_ATOMIC);
if (!info->out_buffer) {
ret = -ENOMEM;
goto scatter_gather_clean;
}
*((u64 *)info->out_buffer) = ~((u64)COMPLETION_CODE_INIT);
info->alternate_caddr = (u64 *)info->out_buffer;
info->rptr_baddr = dma_map_single(&pdev->dev,
(void *)info->out_buffer,
COMPLETION_CODE_SIZE,
DMA_BIDIRECTIONAL);
if (dma_mapping_error(&pdev->dev, info->rptr_baddr)) {
dev_err(&pdev->dev, "Mapping RPTR Failed %d\n",
COMPLETION_CODE_SIZE);
ret = -EIO;
goto scatter_gather_clean;
}
return 0;
scatter_gather_clean:
return ret;
}
static int send_cpt_command(struct cpt_vf *cptvf, union cpt_inst_s *cmd,
u32 qno)
{
struct pci_dev *pdev = cptvf->pdev;
struct command_qinfo *qinfo = NULL;
struct command_queue *queue;
struct command_chunk *chunk;
u8 *ent;
int ret = 0;
if (unlikely(qno >= cptvf->nr_queues)) {
dev_err(&pdev->dev, "Invalid queue (qno: %d, nr_queues: %d)\n",
qno, cptvf->nr_queues);
return -EINVAL;
}
qinfo = &cptvf->cqinfo;
queue = &qinfo->queue[qno];
/* lock command queue */
spin_lock(&queue->lock);
ent = &queue->qhead->head[queue->idx * qinfo->cmd_size];
memcpy(ent, (void *)cmd, qinfo->cmd_size);
if (++queue->idx >= queue->qhead->size / 64) {
hlist_for_each_entry(chunk, &queue->chead, nextchunk) {
if (chunk == queue->qhead) {
continue;
} else {
queue->qhead = chunk;
break;
}
}
queue->idx = 0;
}
/* make sure all memory stores are done before ringing doorbell */
smp_wmb();
cptvf_write_vq_doorbell(cptvf, 1);
/* unlock command queue */
spin_unlock(&queue->lock);
return ret;
}
static void do_request_cleanup(struct cpt_vf *cptvf,
struct cpt_info_buffer *info)
{
int i;
struct pci_dev *pdev = cptvf->pdev;
struct cpt_request_info *req;
if (info->dptr_baddr)
dma_unmap_single(&pdev->dev, info->dptr_baddr,
info->dlen, DMA_BIDIRECTIONAL);
if (info->rptr_baddr)
dma_unmap_single(&pdev->dev, info->rptr_baddr,
COMPLETION_CODE_SIZE, DMA_BIDIRECTIONAL);
if (info->comp_baddr)
dma_unmap_single(&pdev->dev, info->comp_baddr,
sizeof(union cpt_res_s), DMA_BIDIRECTIONAL);
if (info->req) {
req = info->req;
for (i = 0; i < req->outcnt; i++) {
if (req->out[i].dma_addr)
dma_unmap_single(&pdev->dev,
req->out[i].dma_addr,
req->out[i].size,
DMA_BIDIRECTIONAL);
}
for (i = 0; i < req->incnt; i++) {
if (req->in[i].dma_addr)
dma_unmap_single(&pdev->dev,
req->in[i].dma_addr,
req->in[i].size,
DMA_BIDIRECTIONAL);
}
}
kfree_sensitive(info->scatter_components);
kfree_sensitive(info->gather_components);
kfree_sensitive(info->out_buffer);
kfree_sensitive(info->in_buffer);
kfree_sensitive((void *)info->completion_addr);
kfree_sensitive(info);
}
static void do_post_process(struct cpt_vf *cptvf, struct cpt_info_buffer *info)
{
struct pci_dev *pdev = cptvf->pdev;
if (!info) {
dev_err(&pdev->dev, "incorrect cpt_info_buffer for post processing\n");
return;
}
do_request_cleanup(cptvf, info);
}
static inline void process_pending_queue(struct cpt_vf *cptvf,
struct pending_qinfo *pqinfo,
int qno)
{
struct pci_dev *pdev = cptvf->pdev;
struct pending_queue *pqueue = &pqinfo->queue[qno];
struct pending_entry *pentry = NULL;
struct cpt_info_buffer *info = NULL;
union cpt_res_s *status = NULL;
unsigned char ccode;
while (1) {
spin_lock_bh(&pqueue->lock);
pentry = &pqueue->head[pqueue->front];
if (unlikely(!pentry->busy)) {
spin_unlock_bh(&pqueue->lock);
break;
}
info = (struct cpt_info_buffer *)pentry->post_arg;
if (unlikely(!info)) {
dev_err(&pdev->dev, "Pending Entry post arg NULL\n");
pending_queue_inc_front(pqinfo, qno);
spin_unlock_bh(&pqueue->lock);
continue;
}
status = (union cpt_res_s *)pentry->completion_addr;
ccode = status->s.compcode;
if ((status->s.compcode == CPT_COMP_E_FAULT) ||
(status->s.compcode == CPT_COMP_E_SWERR)) {
dev_err(&pdev->dev, "Request failed with %s\n",
(status->s.compcode == CPT_COMP_E_FAULT) ?
"DMA Fault" : "Software error");
pentry->completion_addr = NULL;
pentry->busy = false;
atomic64_dec((&pqueue->pending_count));
pentry->post_arg = NULL;
pending_queue_inc_front(pqinfo, qno);
do_request_cleanup(cptvf, info);
spin_unlock_bh(&pqueue->lock);
break;
} else if (status->s.compcode == COMPLETION_CODE_INIT) {
/* check for timeout */
if (time_after_eq(jiffies,
(info->time_in +
(CPT_COMMAND_TIMEOUT * HZ)))) {
dev_err(&pdev->dev, "Request timed out");
pentry->completion_addr = NULL;
pentry->busy = false;
atomic64_dec((&pqueue->pending_count));
pentry->post_arg = NULL;
pending_queue_inc_front(pqinfo, qno);
do_request_cleanup(cptvf, info);
spin_unlock_bh(&pqueue->lock);
break;
} else if ((*info->alternate_caddr ==
(~COMPLETION_CODE_INIT)) &&
(info->extra_time < TIME_IN_RESET_COUNT)) {
info->time_in = jiffies;
info->extra_time++;
spin_unlock_bh(&pqueue->lock);
break;
}
}
pentry->completion_addr = NULL;
pentry->busy = false;
pentry->post_arg = NULL;
atomic64_dec((&pqueue->pending_count));
pending_queue_inc_front(pqinfo, qno);
spin_unlock_bh(&pqueue->lock);
do_post_process(info->cptvf, info);
/*
* Calling callback after we find
* that the request has been serviced
*/
pentry->callback(ccode, pentry->callback_arg);
}
}
int process_request(struct cpt_vf *cptvf, struct cpt_request_info *req)
{
int ret = 0, clear = 0, queue = 0;
struct cpt_info_buffer *info = NULL;
struct cptvf_request *cpt_req = NULL;
union ctrl_info *ctrl = NULL;
union cpt_res_s *result = NULL;
struct pending_entry *pentry = NULL;
struct pending_queue *pqueue = NULL;
struct pci_dev *pdev = cptvf->pdev;
u8 group = 0;
struct cpt_vq_command vq_cmd;
union cpt_inst_s cptinst;
info = kzalloc_obj(*info, req->may_sleep ? GFP_KERNEL : GFP_ATOMIC);
if (unlikely(!info)) {
dev_err(&pdev->dev, "Unable to allocate memory for info_buffer\n");
return -ENOMEM;
}
cpt_req = (struct cptvf_request *)&req->req;
ctrl = (union ctrl_info *)&req->ctrl;
info->cptvf = cptvf;
group = ctrl->s.grp;
ret = setup_sgio_list(cptvf, info, req);
if (ret) {
dev_err(&pdev->dev, "Setting up SG list failed");
goto request_cleanup;
}
cpt_req->dlen = info->dlen;
/*
* Get buffer for union cpt_res_s response
* structure and its physical address
*/
info->completion_addr = kzalloc(sizeof(union cpt_res_s), req->may_sleep ? GFP_KERNEL : GFP_ATOMIC);
if (unlikely(!info->completion_addr)) {
dev_err(&pdev->dev, "Unable to allocate memory for completion_addr\n");
ret = -ENOMEM;
goto request_cleanup;
}
result = (union cpt_res_s *)info->completion_addr;
result->s.compcode = COMPLETION_CODE_INIT;
info->comp_baddr = dma_map_single(&pdev->dev,
(void *)info->completion_addr,
sizeof(union cpt_res_s),
DMA_BIDIRECTIONAL);
if (dma_mapping_error(&pdev->dev, info->comp_baddr)) {
dev_err(&pdev->dev, "mapping compptr Failed %lu\n",
sizeof(union cpt_res_s));
ret = -EFAULT;
goto request_cleanup;
}
/* Fill the VQ command */
vq_cmd.cmd.u64 = 0;
vq_cmd.cmd.s.opcode = cpu_to_be16(cpt_req->opcode.flags);
vq_cmd.cmd.s.param1 = cpu_to_be16(cpt_req->param1);
vq_cmd.cmd.s.param2 = cpu_to_be16(cpt_req->param2);
vq_cmd.cmd.s.dlen = cpu_to_be16(cpt_req->dlen);
vq_cmd.dptr = info->dptr_baddr;
vq_cmd.rptr = info->rptr_baddr;
vq_cmd.cptr.u64 = 0;
vq_cmd.cptr.s.grp = group;
/* Get Pending Entry to submit command */
/* Always queue 0, because 1 queue per VF */
queue = 0;
pqueue = &cptvf->pqinfo.queue[queue];
if (atomic64_read(&pqueue->pending_count) > PENDING_THOLD) {
dev_err(&pdev->dev, "pending threshold reached\n");
process_pending_queue(cptvf, &cptvf->pqinfo, queue);
}
get_pending_entry:
spin_lock_bh(&pqueue->lock);
pentry = get_free_pending_entry(pqueue, cptvf->pqinfo.qlen);
if (unlikely(!pentry)) {
spin_unlock_bh(&pqueue->lock);
if (clear == 0) {
process_pending_queue(cptvf, &cptvf->pqinfo, queue);
clear = 1;
goto get_pending_entry;
}
dev_err(&pdev->dev, "Get free entry failed\n");
dev_err(&pdev->dev, "queue: %d, rear: %d, front: %d\n",
queue, pqueue->rear, pqueue->front);
ret = -EFAULT;
goto request_cleanup;
}
pentry->completion_addr = info->completion_addr;
pentry->post_arg = (void *)info;
pentry->callback = req->callback;
pentry->callback_arg = req->callback_arg;
info->pentry = pentry;
pentry->busy = true;
atomic64_inc(&pqueue->pending_count);
/* Send CPT command */
info->pentry = pentry;
info->time_in = jiffies;
info->req = req;
/* Create the CPT_INST_S type command for HW interpretation */
cptinst.s.doneint = true;
cptinst.s.res_addr = (u64)info->comp_baddr;
cptinst.s.tag = 0;
cptinst.s.grp = 0;
cptinst.s.wq_ptr = 0;
cptinst.s.ei0 = vq_cmd.cmd.u64;
cptinst.s.ei1 = vq_cmd.dptr;
cptinst.s.ei2 = vq_cmd.rptr;
cptinst.s.ei3 = vq_cmd.cptr.u64;
ret = send_cpt_command(cptvf, &cptinst, queue);
spin_unlock_bh(&pqueue->lock);
if (unlikely(ret)) {
dev_err(&pdev->dev, "Send command failed for AE\n");
ret = -EFAULT;
goto request_cleanup;
}
return 0;
request_cleanup:
dev_dbg(&pdev->dev, "Failed to submit CPT command\n");
do_request_cleanup(cptvf, info);
return ret;
}
void vq_post_process(struct cpt_vf *cptvf, u32 qno)
{
struct pci_dev *pdev = cptvf->pdev;
if (unlikely(qno > cptvf->nr_queues)) {
dev_err(&pdev->dev, "Request for post processing on invalid pending queue: %u\n",
qno);
return;
}
process_pending_queue(cptvf, &cptvf->pqinfo, qno);
}
int cptvf_do_request(void *vfdev, struct cpt_request_info *req)
{
struct cpt_vf *cptvf = (struct cpt_vf *)vfdev;
struct pci_dev *pdev = cptvf->pdev;
if (!cpt_device_ready(cptvf)) {
dev_err(&pdev->dev, "CPT Device is not ready");
return -ENODEV;
}
if ((cptvf->vftype == SE_TYPES) && (!req->ctrl.s.se_req)) {
dev_err(&pdev->dev, "CPTVF-%d of SE TYPE got AE request",
cptvf->vfid);
return -EINVAL;
} else if ((cptvf->vftype == AE_TYPES) && (req->ctrl.s.se_req)) {
dev_err(&pdev->dev, "CPTVF-%d of AE TYPE got SE request",
cptvf->vfid);
return -EINVAL;
}
return process_request(cptvf, req);
}
linux-7.2.3/drivers/crypto/cavium/cpt/request_manager.h
+146
View File
@@ -0,0 +1,146 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (C) 2016 Cavium, Inc.
*/
#ifndef __REQUEST_MANAGER_H
#define __REQUEST_MANAGER_H
#include "cpt_common.h"
#define TIME_IN_RESET_COUNT 5
#define COMPLETION_CODE_SIZE 8
#define COMPLETION_CODE_INIT 0
#define PENDING_THOLD 100
#define MAX_SG_IN_CNT 12
#define MAX_SG_OUT_CNT 13
#define SG_LIST_HDR_SIZE 8
#define MAX_BUF_CNT 16
union ctrl_info {
u32 flags;
struct {
#if defined(__BIG_ENDIAN_BITFIELD)
u32 reserved0:26;
u32 grp:3; /* Group bits */
u32 dma_mode:2; /* DMA mode */
u32 se_req:1;/* To SE core */
#else
u32 se_req:1; /* To SE core */
u32 dma_mode:2; /* DMA mode */
u32 grp:3; /* Group bits */
u32 reserved0:26;
#endif
} s;
};
union opcode_info {
u16 flags;
struct {
u8 major;
u8 minor;
} s;
};
struct cptvf_request {
union opcode_info opcode;
u16 param1;
u16 param2;
u16 dlen;
};
struct buf_ptr {
u8 *vptr;
dma_addr_t dma_addr;
u16 size;
};
struct cpt_request_info {
u8 incnt; /* Number of input buffers */
u8 outcnt; /* Number of output buffers */
u16 rlen; /* Output length */
union ctrl_info ctrl; /* User control information */
struct cptvf_request req; /* Request Information (Core specific) */
bool may_sleep;
struct buf_ptr in[MAX_BUF_CNT];
struct buf_ptr out[MAX_BUF_CNT];
void (*callback)(int, void *); /* Kernel ASYNC request callabck */
void *callback_arg; /* Kernel ASYNC request callabck arg */
};
struct sglist_component {
union {
u64 len;
struct {
__be16 len0;
__be16 len1;
__be16 len2;
__be16 len3;
} s;
} u;
__be64 ptr0;
__be64 ptr1;
__be64 ptr2;
__be64 ptr3;
};
struct cpt_info_buffer {
struct cpt_vf *cptvf;
unsigned long time_in;
u8 extra_time;
struct cpt_request_info *req;
dma_addr_t dptr_baddr;
u32 dlen;
dma_addr_t rptr_baddr;
dma_addr_t comp_baddr;
u8 *in_buffer;
u8 *out_buffer;
u8 *gather_components;
u8 *scatter_components;
struct pending_entry *pentry;
volatile u64 *completion_addr;
volatile u64 *alternate_caddr;
};
/*
* CPT_INST_S software command definitions
* Words EI (0-3)
*/
union vq_cmd_word0 {
u64 u64;
struct {
__be16 opcode;
__be16 param1;
__be16 param2;
__be16 dlen;
} s;
};
union vq_cmd_word3 {
u64 u64;
struct {
#if defined(__BIG_ENDIAN_BITFIELD)
u64 grp:3;
u64 cptr:61;
#else
u64 cptr:61;
u64 grp:3;
#endif
} s;
};
struct cpt_vq_command {
union vq_cmd_word0 cmd;
u64 dptr;
u64 rptr;
union vq_cmd_word3 cptr;
};
void vq_post_process(struct cpt_vf *cptvf, u32 qno);
int process_request(struct cpt_vf *cptvf, struct cpt_request_info *req);
#endif /* __REQUEST_MANAGER_H */
linux-7.2.3/drivers/crypto/cavium/nitrox/Kconfig
+21
View File
@@ -0,0 +1,21 @@
# SPDX-License-Identifier: GPL-2.0-only
#
# Cavium NITROX Crypto Device configuration
#
config CRYPTO_DEV_NITROX
tristate
select CRYPTO_SKCIPHER
select CRYPTO_AES
select CRYPTO_LIB_DES
select FW_LOADER
config CRYPTO_DEV_NITROX_CNN55XX
tristate "Support for Cavium CNN55XX driver"
depends on PCI_MSI && 64BIT
select CRYPTO_DEV_NITROX
help
Support for Cavium NITROX family CNN55XX driver
for accelerating crypto workloads.
To compile this as a module, choose M here: the module
will be called n5pf.
linux-7.2.3/drivers/crypto/cavium/nitrox/Makefile
+15
View File
@@ -0,0 +1,15 @@
# SPDX-License-Identifier: GPL-2.0
obj-$(CONFIG_CRYPTO_DEV_NITROX_CNN55XX) += n5pf.o
n5pf-objs := nitrox_main.o \
nitrox_isr.o \
nitrox_lib.o \
nitrox_hal.o \
nitrox_reqmgr.o \
nitrox_algs.o \
nitrox_mbx.o \
nitrox_skcipher.o \
nitrox_aead.o
n5pf-$(CONFIG_PCI_IOV) += nitrox_sriov.o
n5pf-$(CONFIG_DEBUG_FS) += nitrox_debugfs.o
linux-7.2.3/drivers/crypto/cavium/nitrox/nitrox_aead.c
+567
View File
@@ -0,0 +1,567 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/kernel.h>
#include <linux/printk.h>
#include <linux/crypto.h>
#include <linux/rtnetlink.h>
#include <crypto/aead.h>
#include <crypto/authenc.h>
#include <crypto/des.h>
#include <crypto/internal/aead.h>
#include <crypto/scatterwalk.h>
#include <crypto/gcm.h>
#include "nitrox_dev.h"
#include "nitrox_common.h"
#include "nitrox_req.h"
#define GCM_AES_SALT_SIZE 4
union gph_p3 {
struct {
#ifdef __BIG_ENDIAN_BITFIELD
u16 iv_offset : 8;
u16 auth_offset : 8;
#else
u16 auth_offset : 8;
u16 iv_offset : 8;
#endif
};
u16 param;
};
static int nitrox_aes_gcm_setkey(struct crypto_aead *aead, const u8 *key,
unsigned int keylen)
{
int aes_keylen;
struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
struct flexi_crypto_context *fctx;
union fc_ctx_flags flags;
aes_keylen = flexi_aes_keylen(keylen);
if (aes_keylen < 0)
return -EINVAL;
/* fill crypto context */
fctx = nctx->u.fctx;
flags.fu = be64_to_cpu(fctx->flags.f);
flags.w0.aes_keylen = aes_keylen;
fctx->flags.f = cpu_to_be64(flags.fu);
/* copy enc key to context */
memset(&fctx->crypto, 0, sizeof(fctx->crypto));
memcpy(fctx->crypto.u.key, key, keylen);
return 0;
}
static int nitrox_aead_setauthsize(struct crypto_aead *aead,
unsigned int authsize)
{
struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
struct flexi_crypto_context *fctx = nctx->u.fctx;
union fc_ctx_flags flags;
flags.fu = be64_to_cpu(fctx->flags.f);
flags.w0.mac_len = authsize;
fctx->flags.f = cpu_to_be64(flags.fu);
aead->authsize = authsize;
return 0;
}
static int nitrox_aes_gcm_setauthsize(struct crypto_aead *aead,
unsigned int authsize)
{
switch (authsize) {
case 4:
case 8:
case 12:
case 13:
case 14:
case 15:
case 16:
break;
default:
return -EINVAL;
}
return nitrox_aead_setauthsize(aead, authsize);
}
static int alloc_src_sglist(struct nitrox_kcrypt_request *nkreq,
struct scatterlist *src, char *iv, int ivsize,
int buflen)
{
int nents = sg_nents_for_len(src, buflen);
int ret;
if (nents < 0)
return nents;
/* IV entry */
nents += 1;
/* Allocate buffer to hold IV and input scatterlist array */
ret = alloc_src_req_buf(nkreq, nents, ivsize);
if (ret)
return ret;
nitrox_creq_copy_iv(nkreq->src, iv, ivsize);
nitrox_creq_set_src_sg(nkreq, nents, ivsize, src, buflen);
return 0;
}
static int alloc_dst_sglist(struct nitrox_kcrypt_request *nkreq,
struct scatterlist *dst, int ivsize, int buflen)
{
int nents = sg_nents_for_len(dst, buflen);
int ret;
if (nents < 0)
return nents;
/* IV, ORH, COMPLETION entries */
nents += 3;
/* Allocate buffer to hold ORH, COMPLETION and output scatterlist
* array
*/
ret = alloc_dst_req_buf(nkreq, nents);
if (ret)
return ret;
nitrox_creq_set_orh(nkreq);
nitrox_creq_set_comp(nkreq);
nitrox_creq_set_dst_sg(nkreq, nents, ivsize, dst, buflen);
return 0;
}
static void free_src_sglist(struct nitrox_kcrypt_request *nkreq)
{
kfree(nkreq->src);
}
static void free_dst_sglist(struct nitrox_kcrypt_request *nkreq)
{
kfree(nkreq->dst);
}
static int nitrox_set_creq(struct nitrox_aead_rctx *rctx)
{
struct se_crypto_request *creq = &rctx->nkreq.creq;
union gph_p3 param3;
int ret;
creq->flags = rctx->flags;
creq->gfp = (rctx->flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? GFP_KERNEL :
GFP_ATOMIC;
creq->ctrl.value = 0;
creq->opcode = FLEXI_CRYPTO_ENCRYPT_HMAC;
creq->ctrl.s.arg = rctx->ctrl_arg;
creq->gph.param0 = cpu_to_be16(rctx->cryptlen);
creq->gph.param1 = cpu_to_be16(rctx->cryptlen + rctx->assoclen);
creq->gph.param2 = cpu_to_be16(rctx->ivsize + rctx->assoclen);
param3.iv_offset = 0;
param3.auth_offset = rctx->ivsize;
creq->gph.param3 = cpu_to_be16(param3.param);
creq->ctx_handle = rctx->ctx_handle;
creq->ctrl.s.ctxl = sizeof(struct flexi_crypto_context);
ret = alloc_src_sglist(&rctx->nkreq, rctx->src, rctx->iv, rctx->ivsize,
rctx->srclen);
if (ret)
return ret;
ret = alloc_dst_sglist(&rctx->nkreq, rctx->dst, rctx->ivsize,
rctx->dstlen);
if (ret) {
free_src_sglist(&rctx->nkreq);
return ret;
}
return 0;
}
static void nitrox_aead_callback(void *arg, int err)
{
struct aead_request *areq = arg;
struct nitrox_aead_rctx *rctx = aead_request_ctx(areq);
free_src_sglist(&rctx->nkreq);
free_dst_sglist(&rctx->nkreq);
if (err) {
pr_err_ratelimited("request failed status 0x%0x\n", err);
err = -EINVAL;
}
aead_request_complete(areq, err);
}
static inline bool nitrox_aes_gcm_assoclen_supported(unsigned int assoclen)
{
if (assoclen <= 512)
return true;
return false;
}
static int nitrox_aes_gcm_enc(struct aead_request *areq)
{
struct crypto_aead *aead = crypto_aead_reqtfm(areq);
struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
struct nitrox_aead_rctx *rctx = aead_request_ctx(areq);
struct se_crypto_request *creq = &rctx->nkreq.creq;
struct flexi_crypto_context *fctx = nctx->u.fctx;
int ret;
if (!nitrox_aes_gcm_assoclen_supported(areq->assoclen))
return -EINVAL;
memcpy(fctx->crypto.iv, areq->iv, GCM_AES_SALT_SIZE);
rctx->cryptlen = areq->cryptlen;
rctx->assoclen = areq->assoclen;
rctx->srclen = areq->assoclen + areq->cryptlen;
rctx->dstlen = rctx->srclen + aead->authsize;
rctx->iv = &areq->iv[GCM_AES_SALT_SIZE];
rctx->ivsize = GCM_AES_IV_SIZE - GCM_AES_SALT_SIZE;
rctx->flags = areq->base.flags;
rctx->ctx_handle = nctx->u.ctx_handle;
rctx->src = areq->src;
rctx->dst = areq->dst;
rctx->ctrl_arg = ENCRYPT;
ret = nitrox_set_creq(rctx);
if (ret)
return ret;
/* send the crypto request */
return nitrox_process_se_request(nctx->ndev, creq, nitrox_aead_callback,
areq);
}
static int nitrox_aes_gcm_dec(struct aead_request *areq)
{
struct crypto_aead *aead = crypto_aead_reqtfm(areq);
struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
struct nitrox_aead_rctx *rctx = aead_request_ctx(areq);
struct se_crypto_request *creq = &rctx->nkreq.creq;
struct flexi_crypto_context *fctx = nctx->u.fctx;
int ret;
if (!nitrox_aes_gcm_assoclen_supported(areq->assoclen))
return -EINVAL;
memcpy(fctx->crypto.iv, areq->iv, GCM_AES_SALT_SIZE);
rctx->cryptlen = areq->cryptlen - aead->authsize;
rctx->assoclen = areq->assoclen;
rctx->srclen = areq->cryptlen + areq->assoclen;
rctx->dstlen = rctx->srclen - aead->authsize;
rctx->iv = &areq->iv[GCM_AES_SALT_SIZE];
rctx->ivsize = GCM_AES_IV_SIZE - GCM_AES_SALT_SIZE;
rctx->flags = areq->base.flags;
rctx->ctx_handle = nctx->u.ctx_handle;
rctx->src = areq->src;
rctx->dst = areq->dst;
rctx->ctrl_arg = DECRYPT;
ret = nitrox_set_creq(rctx);
if (ret)
return ret;
/* send the crypto request */
return nitrox_process_se_request(nctx->ndev, creq, nitrox_aead_callback,
areq);
}
static int nitrox_aead_init(struct crypto_aead *aead)
{
struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
struct crypto_ctx_hdr *chdr;
/* get the first device */
nctx->ndev = nitrox_get_first_device();
if (!nctx->ndev)
return -ENODEV;
/* allocate nitrox crypto context */
chdr = crypto_alloc_context(nctx->ndev);
if (!chdr) {
nitrox_put_device(nctx->ndev);
return -ENOMEM;
}
nctx->chdr = chdr;
nctx->u.ctx_handle = (uintptr_t)((u8 *)chdr->vaddr +
sizeof(struct ctx_hdr));
nctx->u.fctx->flags.f = 0;
return 0;
}
static int nitrox_gcm_common_init(struct crypto_aead *aead)
{
int ret;
struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
union fc_ctx_flags *flags;
ret = nitrox_aead_init(aead);
if (ret)
return ret;
flags = &nctx->u.fctx->flags;
flags->w0.cipher_type = CIPHER_AES_GCM;
flags->w0.hash_type = AUTH_NULL;
flags->w0.iv_source = IV_FROM_DPTR;
/* ask microcode to calculate ipad/opad */
flags->w0.auth_input_type = 1;
flags->f = cpu_to_be64(flags->fu);
return 0;
}
static int nitrox_aes_gcm_init(struct crypto_aead *aead)
{
int ret;
ret = nitrox_gcm_common_init(aead);
if (ret)
return ret;
crypto_aead_set_reqsize(aead,
sizeof(struct aead_request) +
sizeof(struct nitrox_aead_rctx));
return 0;
}
static void nitrox_aead_exit(struct crypto_aead *aead)
{
struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
/* free the nitrox crypto context */
if (nctx->u.ctx_handle) {
struct flexi_crypto_context *fctx = nctx->u.fctx;
memzero_explicit(&fctx->crypto, sizeof(struct crypto_keys));
memzero_explicit(&fctx->auth, sizeof(struct auth_keys));
crypto_free_context((void *)nctx->chdr);
}
nitrox_put_device(nctx->ndev);
nctx->u.ctx_handle = 0;
nctx->ndev = NULL;
}
static int nitrox_rfc4106_setkey(struct crypto_aead *aead, const u8 *key,
unsigned int keylen)
{
struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
struct flexi_crypto_context *fctx = nctx->u.fctx;
int ret;
if (keylen < GCM_AES_SALT_SIZE)
return -EINVAL;
keylen -= GCM_AES_SALT_SIZE;
ret = nitrox_aes_gcm_setkey(aead, key, keylen);
if (ret)
return ret;
memcpy(fctx->crypto.iv, key + keylen, GCM_AES_SALT_SIZE);
return 0;
}
static int nitrox_rfc4106_setauthsize(struct crypto_aead *aead,
unsigned int authsize)
{
switch (authsize) {
case 8:
case 12:
case 16:
break;
default:
return -EINVAL;
}
return nitrox_aead_setauthsize(aead, authsize);
}
static int nitrox_rfc4106_set_aead_rctx_sglist(struct aead_request *areq)
{
struct nitrox_rfc4106_rctx *rctx = aead_request_ctx_dma(areq);
struct nitrox_aead_rctx *aead_rctx = &rctx->base;
unsigned int assoclen = areq->assoclen - GCM_RFC4106_IV_SIZE;
struct scatterlist *sg;
if (areq->assoclen != 16 && areq->assoclen != 20)
return -EINVAL;
scatterwalk_map_and_copy(rctx->assoc, areq->src, 0, assoclen, 0);
sg_init_table(rctx->src, 3);
sg_set_buf(rctx->src, rctx->assoc, assoclen);
sg = scatterwalk_ffwd(rctx->src + 1, areq->src, areq->assoclen);
if (sg != rctx->src + 1)
sg_chain(rctx->src, 2, sg);
if (areq->src != areq->dst) {
sg_init_table(rctx->dst, 3);
sg_set_buf(rctx->dst, rctx->assoc, assoclen);
sg = scatterwalk_ffwd(rctx->dst + 1, areq->dst, areq->assoclen);
if (sg != rctx->dst + 1)
sg_chain(rctx->dst, 2, sg);
}
aead_rctx->src = rctx->src;
aead_rctx->dst = (areq->src == areq->dst) ? rctx->src : rctx->dst;
return 0;
}
static void nitrox_rfc4106_callback(void *arg, int err)
{
struct aead_request *areq = arg;
struct nitrox_rfc4106_rctx *rctx = aead_request_ctx_dma(areq);
struct nitrox_kcrypt_request *nkreq = &rctx->base.nkreq;
free_src_sglist(nkreq);
free_dst_sglist(nkreq);
if (err) {
pr_err_ratelimited("request failed status 0x%0x\n", err);
err = -EINVAL;
}
aead_request_complete(areq, err);
}
static int nitrox_rfc4106_enc(struct aead_request *areq)
{
struct crypto_aead *aead = crypto_aead_reqtfm(areq);
struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
struct nitrox_rfc4106_rctx *rctx = aead_request_ctx_dma(areq);
struct nitrox_aead_rctx *aead_rctx = &rctx->base;
struct se_crypto_request *creq = &aead_rctx->nkreq.creq;
int ret;
aead_rctx->cryptlen = areq->cryptlen;
aead_rctx->assoclen = areq->assoclen - GCM_RFC4106_IV_SIZE;
aead_rctx->srclen = aead_rctx->assoclen + aead_rctx->cryptlen;
aead_rctx->dstlen = aead_rctx->srclen + aead->authsize;
aead_rctx->iv = areq->iv;
aead_rctx->ivsize = GCM_RFC4106_IV_SIZE;
aead_rctx->flags = areq->base.flags;
aead_rctx->ctx_handle = nctx->u.ctx_handle;
aead_rctx->ctrl_arg = ENCRYPT;
ret = nitrox_rfc4106_set_aead_rctx_sglist(areq);
if (ret)
return ret;
ret = nitrox_set_creq(aead_rctx);
if (ret)
return ret;
/* send the crypto request */
return nitrox_process_se_request(nctx->ndev, creq,
nitrox_rfc4106_callback, areq);
}
static int nitrox_rfc4106_dec(struct aead_request *areq)
{
struct crypto_aead *aead = crypto_aead_reqtfm(areq);
struct nitrox_crypto_ctx *nctx = crypto_aead_ctx_dma(aead);
struct nitrox_rfc4106_rctx *rctx = aead_request_ctx(areq);
struct nitrox_aead_rctx *aead_rctx = &rctx->base;
struct se_crypto_request *creq = &aead_rctx->nkreq.creq;
int ret;
aead_rctx->cryptlen = areq->cryptlen - aead->authsize;
aead_rctx->assoclen = areq->assoclen - GCM_RFC4106_IV_SIZE;
aead_rctx->srclen =
areq->cryptlen - GCM_RFC4106_IV_SIZE + areq->assoclen;
aead_rctx->dstlen = aead_rctx->srclen - aead->authsize;
aead_rctx->iv = areq->iv;
aead_rctx->ivsize = GCM_RFC4106_IV_SIZE;
aead_rctx->flags = areq->base.flags;
aead_rctx->ctx_handle = nctx->u.ctx_handle;
aead_rctx->ctrl_arg = DECRYPT;
ret = nitrox_rfc4106_set_aead_rctx_sglist(areq);
if (ret)
return ret;
ret = nitrox_set_creq(aead_rctx);
if (ret)
return ret;
/* send the crypto request */
return nitrox_process_se_request(nctx->ndev, creq,
nitrox_rfc4106_callback, areq);
}
static int nitrox_rfc4106_init(struct crypto_aead *aead)
{
int ret;
ret = nitrox_gcm_common_init(aead);
if (ret)
return ret;
crypto_aead_set_reqsize_dma(aead, sizeof(struct aead_request) +
sizeof(struct nitrox_rfc4106_rctx));
return 0;
}
static struct aead_alg nitrox_aeads[] = { {
.base = {
.cra_name = "gcm(aes)",
.cra_driver_name = "n5_aes_gcm",
.cra_priority = PRIO,
.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,
.cra_blocksize = 1,
.cra_ctxsize = sizeof(struct nitrox_crypto_ctx),
.cra_alignmask = 0,
.cra_module = THIS_MODULE,
},
.setkey = nitrox_aes_gcm_setkey,
.setauthsize = nitrox_aes_gcm_setauthsize,
.encrypt = nitrox_aes_gcm_enc,
.decrypt = nitrox_aes_gcm_dec,
.init = nitrox_aes_gcm_init,
.exit = nitrox_aead_exit,
.ivsize = GCM_AES_IV_SIZE,
.maxauthsize = AES_BLOCK_SIZE,
}, {
.base = {
.cra_name = "rfc4106(gcm(aes))",
.cra_driver_name = "n5_rfc4106",
.cra_priority = PRIO,
.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,
.cra_blocksize = 1,
.cra_ctxsize = sizeof(struct nitrox_crypto_ctx),
.cra_alignmask = 0,
.cra_module = THIS_MODULE,
},
.setkey = nitrox_rfc4106_setkey,
.setauthsize = nitrox_rfc4106_setauthsize,
.encrypt = nitrox_rfc4106_enc,
.decrypt = nitrox_rfc4106_dec,
.init = nitrox_rfc4106_init,
.exit = nitrox_aead_exit,
.ivsize = GCM_RFC4106_IV_SIZE,
.maxauthsize = AES_BLOCK_SIZE,
} };
int nitrox_register_aeads(void)
{
return crypto_register_aeads(nitrox_aeads, ARRAY_SIZE(nitrox_aeads));
}
void nitrox_unregister_aeads(void)
{
crypto_unregister_aeads(nitrox_aeads, ARRAY_SIZE(nitrox_aeads));
}
linux-7.2.3/drivers/crypto/cavium/nitrox/nitrox_algs.c
+24
View File
@@ -0,0 +1,24 @@
#include "nitrox_common.h"
int nitrox_crypto_register(void)
{
int err;
err = nitrox_register_skciphers();
if (err)
return err;
err = nitrox_register_aeads();
if (err) {
nitrox_unregister_skciphers();
return err;
}
return 0;
}
void nitrox_crypto_unregister(void)
{
nitrox_unregister_aeads();
nitrox_unregister_skciphers();
}
linux-7.2.3/drivers/crypto/cavium/nitrox/nitrox_common.h
+30
View File
@@ -0,0 +1,30 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __NITROX_COMMON_H
#define __NITROX_COMMON_H
#include "nitrox_dev.h"
#include "nitrox_req.h"
int nitrox_crypto_register(void);
void nitrox_crypto_unregister(void);
int nitrox_register_aeads(void);
void nitrox_unregister_aeads(void);
int nitrox_register_skciphers(void);
void nitrox_unregister_skciphers(void);
void *crypto_alloc_context(struct nitrox_device *ndev);
void crypto_free_context(void *ctx);
struct nitrox_device *nitrox_get_first_device(void);
void nitrox_put_device(struct nitrox_device *ndev);
int nitrox_common_sw_init(struct nitrox_device *ndev);
void nitrox_common_sw_cleanup(struct nitrox_device *ndev);
void pkt_slc_resp_tasklet(unsigned long data);
int nitrox_process_se_request(struct nitrox_device *ndev,
struct se_crypto_request *req,
completion_t cb,
void *cb_arg);
void backlog_qflush_work(struct work_struct *work);
#endif /* __NITROX_COMMON_H */
linux-7.2.3/drivers/crypto/cavium/nitrox/nitrox_csr.h
+1439
View File
File diff suppressed because it is too large Load Diff
linux-7.2.3/drivers/crypto/cavium/nitrox/nitrox_debugfs.c
+70
View File
@@ -0,0 +1,70 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/seq_file.h>
#include <linux/debugfs.h>
#include "nitrox_csr.h"
#include "nitrox_debugfs.h"
#include "nitrox_dev.h"
static int firmware_show(struct seq_file *s, void *v)
{
struct nitrox_device *ndev = s->private;
seq_printf(s, "Version: %s\n", ndev->hw.fw_name[0]);
seq_printf(s, "Version: %s\n", ndev->hw.fw_name[1]);
return 0;
}
DEFINE_SHOW_ATTRIBUTE(firmware);
static int device_show(struct seq_file *s, void *v)
{
struct nitrox_device *ndev = s->private;
seq_printf(s, "NITROX [%d]\n", ndev->idx);
seq_printf(s, " Part Name: %s\n", ndev->hw.partname);
seq_printf(s, " Frequency: %d MHz\n", ndev->hw.freq);
seq_printf(s, " Device ID: 0x%0x\n", ndev->hw.device_id);
seq_printf(s, " Revision ID: 0x%0x\n", ndev->hw.revision_id);
seq_printf(s, " Cores: [AE=%u SE=%u ZIP=%u]\n",
ndev->hw.ae_cores, ndev->hw.se_cores, ndev->hw.zip_cores);
return 0;
}
DEFINE_SHOW_ATTRIBUTE(device);
static int stats_show(struct seq_file *s, void *v)
{
struct nitrox_device *ndev = s->private;
seq_printf(s, "NITROX [%d] Request Statistics\n", ndev->idx);
seq_printf(s, " Posted: %llu\n",
(u64)atomic64_read(&ndev->stats.posted));
seq_printf(s, " Completed: %llu\n",
(u64)atomic64_read(&ndev->stats.completed));
seq_printf(s, " Dropped: %llu\n",
(u64)atomic64_read(&ndev->stats.dropped));
return 0;
}
DEFINE_SHOW_ATTRIBUTE(stats);
void nitrox_debugfs_exit(struct nitrox_device *ndev)
{
debugfs_remove_recursive(ndev->debugfs_dir);
ndev->debugfs_dir = NULL;
}
void nitrox_debugfs_init(struct nitrox_device *ndev)
{
struct dentry *dir;
dir = debugfs_create_dir(KBUILD_MODNAME, NULL);
ndev->debugfs_dir = dir;
debugfs_create_file("firmware", 0400, dir, ndev, &firmware_fops);
debugfs_create_file("device", 0400, dir, ndev, &device_fops);
debugfs_create_file("stats", 0400, dir, ndev, &stats_fops);
}
linux-7.2.3/drivers/crypto/cavium/nitrox/nitrox_debugfs.h
+20
View File
@@ -0,0 +1,20 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __NITROX_DEBUGFS_H
#define __NITROX_DEBUGFS_H
#include "nitrox_dev.h"
#ifdef CONFIG_DEBUG_FS
void nitrox_debugfs_init(struct nitrox_device *ndev);
void nitrox_debugfs_exit(struct nitrox_device *ndev);
#else
static inline void nitrox_debugfs_init(struct nitrox_device *ndev)
{
}
static inline void nitrox_debugfs_exit(struct nitrox_device *ndev)
{
}
#endif /* !CONFIG_DEBUG_FS */
#endif /* __NITROX_DEBUGFS_H */
linux-7.2.3/drivers/crypto/cavium/nitrox/nitrox_dev.h
+302
View File
@@ -0,0 +1,302 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __NITROX_DEV_H
#define __NITROX_DEV_H
#include <linux/dma-mapping.h>
#include <linux/interrupt.h>
#include <linux/pci.h>
#include <linux/if.h>
#define VERSION_LEN 32
/* Maximum queues in PF mode */
#define MAX_PF_QUEUES 64
/* Maximum device queues */
#define MAX_DEV_QUEUES (MAX_PF_QUEUES)
/* Maximum UCD Blocks */
#define CNN55XX_MAX_UCD_BLOCKS 8
/**
* struct nitrox_cmdq - NITROX command queue
* @cmd_qlock: command queue lock
* @resp_qlock: response queue lock
* @backlog_qlock: backlog queue lock
* @ndev: NITROX device
* @response_head: submitted request list
* @backlog_head: backlog queue
* @dbell_csr_addr: doorbell register address for this queue
* @compl_cnt_csr_addr: completion count register address of the slc port
* @base: command queue base address
* @dma: dma address of the base
* @pending_count: request pending at device
* @backlog_count: backlog request count
* @write_idx: next write index for the command
* @instr_size: command size
* @qno: command queue number
* @qsize: command queue size
* @unalign_base: unaligned base address
* @unalign_dma: unaligned dma address
*/
struct nitrox_cmdq {
spinlock_t cmd_qlock;
spinlock_t resp_qlock;
spinlock_t backlog_qlock;
struct nitrox_device *ndev;
struct list_head response_head;
struct list_head backlog_head;
u8 __iomem *dbell_csr_addr;
u8 __iomem *compl_cnt_csr_addr;
u8 *base;
dma_addr_t dma;
struct work_struct backlog_qflush;
atomic_t pending_count;
atomic_t backlog_count;
int write_idx;
u8 instr_size;
u8 qno;
u32 qsize;
u8 *unalign_base;
dma_addr_t unalign_dma;
};
/**
* struct nitrox_hw - NITROX hardware information
* @partname: partname ex: CNN55xxx-xxx
* @fw_name: firmware version
* @freq: NITROX frequency
* @vendor_id: vendor ID
* @device_id: device ID
* @revision_id: revision ID
* @se_cores: number of symmetric cores
* @ae_cores: number of asymmetric cores
* @zip_cores: number of zip cores
*/
struct nitrox_hw {
char partname[IFNAMSIZ * 2];
char fw_name[CNN55XX_MAX_UCD_BLOCKS][VERSION_LEN];
int freq;
u16 vendor_id;
u16 device_id;
u8 revision_id;
u8 se_cores;
u8 ae_cores;
u8 zip_cores;
};
struct nitrox_stats {
atomic64_t posted;
atomic64_t completed;
atomic64_t dropped;
};
#define IRQ_NAMESZ 32
struct nitrox_q_vector {
char name[IRQ_NAMESZ];
bool valid;
int ring;
struct tasklet_struct resp_tasklet;
union {
struct nitrox_cmdq *cmdq;
struct nitrox_device *ndev;
};
};
enum mcode_type {
MCODE_TYPE_INVALID,
MCODE_TYPE_AE,
MCODE_TYPE_SE_SSL,
MCODE_TYPE_SE_IPSEC,
};
/**
* mbox_msg - Mailbox message data
* @type: message type
* @opcode: message opcode
* @data: message data
*/
union mbox_msg {
u64 value;
struct {
u64 type: 2;
u64 opcode: 6;
u64 data: 58;
};
struct {
u64 type: 2;
u64 opcode: 6;
u64 chipid: 8;
u64 vfid: 8;
} id;
struct {
u64 type: 2;
u64 opcode: 6;
u64 count: 4;
u64 info: 40;
u64 next_se_grp: 3;
u64 next_ae_grp: 3;
} mcode_info;
};
/**
* nitrox_vfdev - NITROX VF device instance in PF
* @state: VF device state
* @vfno: VF number
* @nr_queues: number of queues enabled in VF
* @ring: ring to communicate with VF
* @msg: Mailbox message data from VF
* @mbx_resp: Mailbox counters
*/
struct nitrox_vfdev {
atomic_t state;
int vfno;
int nr_queues;
int ring;
union mbox_msg msg;
atomic64_t mbx_resp;
};
/**
* struct nitrox_iov - SR-IOV information
* @num_vfs: number of VF(s) enabled
* @max_vf_queues: Maximum number of queues allowed for VF
* @vfdev: VF(s) devices
* @pf2vf_wq: workqueue for PF2VF communication
* @msix: MSI-X entry for PF in SR-IOV case
*/
struct nitrox_iov {
int num_vfs;
int max_vf_queues;
struct nitrox_vfdev *vfdev;
struct workqueue_struct *pf2vf_wq;
struct msix_entry msix;
};
/*
* NITROX Device states
*/
enum ndev_state {
__NDEV_NOT_READY,
__NDEV_READY,
__NDEV_IN_RESET,
};
/* NITROX support modes for VF(s) */
enum vf_mode {
__NDEV_MODE_PF,
__NDEV_MODE_VF16,
__NDEV_MODE_VF32,
__NDEV_MODE_VF64,
__NDEV_MODE_VF128,
};
#define __NDEV_SRIOV_BIT 0
/* command queue size */
#define DEFAULT_CMD_QLEN 2048
/* command timeout in milliseconds */
#define CMD_TIMEOUT 2000
#define DEV(ndev) ((struct device *)(&(ndev)->pdev->dev))
#define NITROX_CSR_ADDR(ndev, offset) \
((ndev)->bar_addr + (offset))
/**
* struct nitrox_device - NITROX Device Information.
* @list: pointer to linked list of devices
* @bar_addr: iomap address
* @pdev: PCI device information
* @state: NITROX device state
* @flags: flags to indicate device the features
* @timeout: Request timeout in jiffies
* @refcnt: Device usage count
* @idx: device index (0..N)
* @node: NUMA node id attached
* @qlen: Command queue length
* @nr_queues: Number of command queues
* @mode: Device mode PF/VF
* @ctx_pool: DMA pool for crypto context
* @pkt_inq: Packet input rings
* @aqmq: AQM command queues
* @qvec: MSI-X queue vectors information
* @iov: SR-IOV informatin
* @num_vecs: number of MSI-X vectors
* @stats: request statistics
* @hw: hardware information
* @debugfs_dir: debugfs directory
*/
struct nitrox_device {
struct list_head list;
u8 __iomem *bar_addr;
struct pci_dev *pdev;
atomic_t state;
unsigned long flags;
unsigned long timeout;
refcount_t refcnt;
u8 idx;
int node;
u16 qlen;
u16 nr_queues;
enum vf_mode mode;
struct dma_pool *ctx_pool;
struct nitrox_cmdq *pkt_inq;
struct nitrox_cmdq *aqmq[MAX_DEV_QUEUES] ____cacheline_aligned_in_smp;
struct nitrox_q_vector *qvec;
struct nitrox_iov iov;
int num_vecs;
struct nitrox_stats stats;
struct nitrox_hw hw;
#if IS_ENABLED(CONFIG_DEBUG_FS)
struct dentry *debugfs_dir;
#endif
};
/**
* nitrox_read_csr - Read from device register
* @ndev: NITROX device
* @offset: offset of the register to read
*
* Returns: value read
*/
static inline u64 nitrox_read_csr(struct nitrox_device *ndev, u64 offset)
{
return readq(ndev->bar_addr + offset);
}
/**
* nitrox_write_csr - Write to device register
* @ndev: NITROX device
* @offset: offset of the register to write
* @value: value to write
*/
static inline void nitrox_write_csr(struct nitrox_device *ndev, u64 offset,
u64 value)
{
writeq(value, (ndev->bar_addr + offset));
}
static inline bool nitrox_ready(struct nitrox_device *ndev)
{
return atomic_read(&ndev->state) == __NDEV_READY;
}
static inline bool nitrox_vfdev_ready(struct nitrox_vfdev *vfdev)
{
return atomic_read(&vfdev->state) == __NDEV_READY;
}
#endif /* __NITROX_DEV_H */
linux-7.2.3/drivers/crypto/cavium/nitrox/nitrox_hal.c
+679
View File
@@ -0,0 +1,679 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/delay.h>
#include "nitrox_dev.h"
#include "nitrox_csr.h"
#include "nitrox_hal.h"
#define PLL_REF_CLK 50
#define MAX_CSR_RETRIES 10
/**
* emu_enable_cores - Enable EMU cluster cores.
* @ndev: NITROX device
*/
static void emu_enable_cores(struct nitrox_device *ndev)
{
union emu_se_enable emu_se;
union emu_ae_enable emu_ae;
int i;
/* AE cores 20 per cluster */
emu_ae.value = 0;
emu_ae.s.enable = 0xfffff;
/* SE cores 16 per cluster */
emu_se.value = 0;
emu_se.s.enable = 0xffff;
/* enable per cluster cores */
for (i = 0; i < NR_CLUSTERS; i++) {
nitrox_write_csr(ndev, EMU_AE_ENABLEX(i), emu_ae.value);
nitrox_write_csr(ndev, EMU_SE_ENABLEX(i), emu_se.value);
}
}
/**
* nitrox_config_emu_unit - configure EMU unit.
* @ndev: NITROX device
*/
void nitrox_config_emu_unit(struct nitrox_device *ndev)
{
union emu_wd_int_ena_w1s emu_wd_int;
union emu_ge_int_ena_w1s emu_ge_int;
u64 offset;
int i;
/* enable cores */
emu_enable_cores(ndev);
/* enable general error and watch dog interrupts */
emu_ge_int.value = 0;
emu_ge_int.s.se_ge = 0xffff;
emu_ge_int.s.ae_ge = 0xfffff;
emu_wd_int.value = 0;
emu_wd_int.s.se_wd = 1;
for (i = 0; i < NR_CLUSTERS; i++) {
offset = EMU_WD_INT_ENA_W1SX(i);
nitrox_write_csr(ndev, offset, emu_wd_int.value);
offset = EMU_GE_INT_ENA_W1SX(i);
nitrox_write_csr(ndev, offset, emu_ge_int.value);
}
}
static void reset_pkt_input_ring(struct nitrox_device *ndev, int ring)
{
union nps_pkt_in_instr_ctl pkt_in_ctl;
union nps_pkt_in_done_cnts pkt_in_cnts;
int max_retries = MAX_CSR_RETRIES;
u64 offset;
/* step 1: disable the ring, clear enable bit */
offset = NPS_PKT_IN_INSTR_CTLX(ring);
pkt_in_ctl.value = nitrox_read_csr(ndev, offset);
pkt_in_ctl.s.enb = 0;
nitrox_write_csr(ndev, offset, pkt_in_ctl.value);
/* step 2: wait to clear [ENB] */
usleep_range(100, 150);
do {
pkt_in_ctl.value = nitrox_read_csr(ndev, offset);
if (!pkt_in_ctl.s.enb)
break;
udelay(50);
} while (max_retries--);
/* step 3: clear done counts */
offset = NPS_PKT_IN_DONE_CNTSX(ring);
pkt_in_cnts.value = nitrox_read_csr(ndev, offset);
nitrox_write_csr(ndev, offset, pkt_in_cnts.value);
usleep_range(50, 100);
}
void enable_pkt_input_ring(struct nitrox_device *ndev, int ring)
{
union nps_pkt_in_instr_ctl pkt_in_ctl;
int max_retries = MAX_CSR_RETRIES;
u64 offset;
/* 64-byte instruction size */
offset = NPS_PKT_IN_INSTR_CTLX(ring);
pkt_in_ctl.value = nitrox_read_csr(ndev, offset);
pkt_in_ctl.s.is64b = 1;
pkt_in_ctl.s.enb = 1;
nitrox_write_csr(ndev, offset, pkt_in_ctl.value);
/* wait for set [ENB] */
do {
pkt_in_ctl.value = nitrox_read_csr(ndev, offset);
if (pkt_in_ctl.s.enb)
break;
udelay(50);
} while (max_retries--);
}
/**
* nitrox_config_pkt_input_rings - configure Packet Input Rings
* @ndev: NITROX device
*/
void nitrox_config_pkt_input_rings(struct nitrox_device *ndev)
{
int i;
for (i = 0; i < ndev->nr_queues; i++) {
struct nitrox_cmdq *cmdq = &ndev->pkt_inq[i];
union nps_pkt_in_instr_rsize pkt_in_rsize;
union nps_pkt_in_instr_baoff_dbell pkt_in_dbell;
u64 offset;
reset_pkt_input_ring(ndev, i);
/**
* step 4:
* configure ring base address 16-byte aligned,
* size and interrupt threshold.
*/
offset = NPS_PKT_IN_INSTR_BADDRX(i);
nitrox_write_csr(ndev, offset, cmdq->dma);
/* configure ring size */
offset = NPS_PKT_IN_INSTR_RSIZEX(i);
pkt_in_rsize.value = 0;
pkt_in_rsize.s.rsize = ndev->qlen;
nitrox_write_csr(ndev, offset, pkt_in_rsize.value);
/* set high threshold for pkt input ring interrupts */
offset = NPS_PKT_IN_INT_LEVELSX(i);
nitrox_write_csr(ndev, offset, 0xffffffff);
/* step 5: clear off door bell counts */
offset = NPS_PKT_IN_INSTR_BAOFF_DBELLX(i);
pkt_in_dbell.value = 0;
pkt_in_dbell.s.dbell = 0xffffffff;
nitrox_write_csr(ndev, offset, pkt_in_dbell.value);
/* enable the ring */
enable_pkt_input_ring(ndev, i);
}
}
static void reset_pkt_solicit_port(struct nitrox_device *ndev, int port)
{
union nps_pkt_slc_ctl pkt_slc_ctl;
union nps_pkt_slc_cnts pkt_slc_cnts;
int max_retries = MAX_CSR_RETRIES;
u64 offset;
/* step 1: disable slc port */
offset = NPS_PKT_SLC_CTLX(port);
pkt_slc_ctl.value = nitrox_read_csr(ndev, offset);
pkt_slc_ctl.s.enb = 0;
nitrox_write_csr(ndev, offset, pkt_slc_ctl.value);
/* step 2 */
usleep_range(100, 150);
/* wait to clear [ENB] */
do {
pkt_slc_ctl.value = nitrox_read_csr(ndev, offset);
if (!pkt_slc_ctl.s.enb)
break;
udelay(50);
} while (max_retries--);
/* step 3: clear slc counters */
offset = NPS_PKT_SLC_CNTSX(port);
pkt_slc_cnts.value = nitrox_read_csr(ndev, offset);
nitrox_write_csr(ndev, offset, pkt_slc_cnts.value);
usleep_range(50, 100);
}
void enable_pkt_solicit_port(struct nitrox_device *ndev, int port)
{
union nps_pkt_slc_ctl pkt_slc_ctl;
int max_retries = MAX_CSR_RETRIES;
u64 offset;
offset = NPS_PKT_SLC_CTLX(port);
pkt_slc_ctl.value = 0;
pkt_slc_ctl.s.enb = 1;
/*
* 8 trailing 0x00 bytes will be added
* to the end of the outgoing packet.
*/
pkt_slc_ctl.s.z = 1;
/* enable response header */
pkt_slc_ctl.s.rh = 1;
nitrox_write_csr(ndev, offset, pkt_slc_ctl.value);
/* wait to set [ENB] */
do {
pkt_slc_ctl.value = nitrox_read_csr(ndev, offset);
if (pkt_slc_ctl.s.enb)
break;
udelay(50);
} while (max_retries--);
}
static void config_pkt_solicit_port(struct nitrox_device *ndev, int port)
{
union nps_pkt_slc_int_levels pkt_slc_int;
u64 offset;
reset_pkt_solicit_port(ndev, port);
/* step 4: configure interrupt levels */
offset = NPS_PKT_SLC_INT_LEVELSX(port);
pkt_slc_int.value = 0;
/* time interrupt threshold */
pkt_slc_int.s.timet = 0x3fffff;
nitrox_write_csr(ndev, offset, pkt_slc_int.value);
/* enable the solicit port */
enable_pkt_solicit_port(ndev, port);
}
void nitrox_config_pkt_solicit_ports(struct nitrox_device *ndev)
{
int i;
for (i = 0; i < ndev->nr_queues; i++)
config_pkt_solicit_port(ndev, i);
}
/**
* enable_nps_core_interrupts - enable NPS core interrutps
* @ndev: NITROX device.
*
* This includes NPS core interrupts.
*/
static void enable_nps_core_interrupts(struct nitrox_device *ndev)
{
union nps_core_int_ena_w1s core_int;
/* NPS core interrutps */
core_int.value = 0;
core_int.s.host_wr_err = 1;
core_int.s.host_wr_timeout = 1;
core_int.s.exec_wr_timeout = 1;
core_int.s.npco_dma_malform = 1;
core_int.s.host_nps_wr_err = 1;
nitrox_write_csr(ndev, NPS_CORE_INT_ENA_W1S, core_int.value);
}
void nitrox_config_nps_core_unit(struct nitrox_device *ndev)
{
union nps_core_gbl_vfcfg core_gbl_vfcfg;
/* endian control information */
nitrox_write_csr(ndev, NPS_CORE_CONTROL, 1ULL);
/* disable ILK interface */
core_gbl_vfcfg.value = 0;
core_gbl_vfcfg.s.ilk_disable = 1;
core_gbl_vfcfg.s.cfg = __NDEV_MODE_PF;
nitrox_write_csr(ndev, NPS_CORE_GBL_VFCFG, core_gbl_vfcfg.value);
/* enable nps core interrupts */
enable_nps_core_interrupts(ndev);
}
/**
* enable_nps_pkt_interrupts - enable NPS packet interrutps
* @ndev: NITROX device.
*
* This includes NPS packet in and slc interrupts.
*/
static void enable_nps_pkt_interrupts(struct nitrox_device *ndev)
{
/* NPS packet in ring interrupts */
nitrox_write_csr(ndev, NPS_PKT_IN_RERR_LO_ENA_W1S, (~0ULL));
nitrox_write_csr(ndev, NPS_PKT_IN_RERR_HI_ENA_W1S, (~0ULL));
nitrox_write_csr(ndev, NPS_PKT_IN_ERR_TYPE_ENA_W1S, (~0ULL));
/* NPS packet slc port interrupts */
nitrox_write_csr(ndev, NPS_PKT_SLC_RERR_HI_ENA_W1S, (~0ULL));
nitrox_write_csr(ndev, NPS_PKT_SLC_RERR_LO_ENA_W1S, (~0ULL));
nitrox_write_csr(ndev, NPS_PKT_SLC_ERR_TYPE_ENA_W1S, (~0uLL));
}
void nitrox_config_nps_pkt_unit(struct nitrox_device *ndev)
{
/* config input and solicit ports */
nitrox_config_pkt_input_rings(ndev);
nitrox_config_pkt_solicit_ports(ndev);
/* enable nps packet interrupts */
enable_nps_pkt_interrupts(ndev);
}
static void reset_aqm_ring(struct nitrox_device *ndev, int ring)
{
union aqmq_en aqmq_en_reg;
union aqmq_activity_stat activity_stat;
union aqmq_cmp_cnt cmp_cnt;
int max_retries = MAX_CSR_RETRIES;
u64 offset;
/* step 1: disable the queue */
offset = AQMQ_ENX(ring);
aqmq_en_reg.value = 0;
aqmq_en_reg.queue_enable = 0;
nitrox_write_csr(ndev, offset, aqmq_en_reg.value);
/* step 2: wait for AQMQ_ACTIVITY_STATX[QUEUE_ACTIVE] to clear */
usleep_range(100, 150);
offset = AQMQ_ACTIVITY_STATX(ring);
do {
activity_stat.value = nitrox_read_csr(ndev, offset);
if (!activity_stat.queue_active)
break;
udelay(50);
} while (max_retries--);
/* step 3: clear commands completed count */
offset = AQMQ_CMP_CNTX(ring);
cmp_cnt.value = nitrox_read_csr(ndev, offset);
nitrox_write_csr(ndev, offset, cmp_cnt.value);
usleep_range(50, 100);
}
void enable_aqm_ring(struct nitrox_device *ndev, int ring)
{
union aqmq_en aqmq_en_reg;
u64 offset;
offset = AQMQ_ENX(ring);
aqmq_en_reg.value = 0;
aqmq_en_reg.queue_enable = 1;
nitrox_write_csr(ndev, offset, aqmq_en_reg.value);
usleep_range(50, 100);
}
void nitrox_config_aqm_rings(struct nitrox_device *ndev)
{
int ring;
for (ring = 0; ring < ndev->nr_queues; ring++) {
struct nitrox_cmdq *cmdq = ndev->aqmq[ring];
union aqmq_drbl drbl;
union aqmq_qsz qsize;
union aqmq_cmp_thr cmp_thr;
u64 offset;
/* steps 1 - 3 */
reset_aqm_ring(ndev, ring);
/* step 4: clear doorbell count of ring */
offset = AQMQ_DRBLX(ring);
drbl.value = 0;
drbl.dbell_count = 0xFFFFFFFF;
nitrox_write_csr(ndev, offset, drbl.value);
/* step 5: configure host ring details */
/* set host address for next command of ring */
offset = AQMQ_NXT_CMDX(ring);
nitrox_write_csr(ndev, offset, 0ULL);
/* set host address of ring base */
offset = AQMQ_BADRX(ring);
nitrox_write_csr(ndev, offset, cmdq->dma);
/* set ring size */
offset = AQMQ_QSZX(ring);
qsize.value = 0;
qsize.host_queue_size = ndev->qlen;
nitrox_write_csr(ndev, offset, qsize.value);
/* set command completion threshold */
offset = AQMQ_CMP_THRX(ring);
cmp_thr.value = 0;
cmp_thr.commands_completed_threshold = 1;
nitrox_write_csr(ndev, offset, cmp_thr.value);
/* step 6: enable the queue */
enable_aqm_ring(ndev, ring);
}
}
static void enable_aqm_interrupts(struct nitrox_device *ndev)
{
/* clear interrupt enable bits */
nitrox_write_csr(ndev, AQM_DBELL_OVF_LO_ENA_W1S, (~0ULL));
nitrox_write_csr(ndev, AQM_DBELL_OVF_HI_ENA_W1S, (~0ULL));
nitrox_write_csr(ndev, AQM_DMA_RD_ERR_LO_ENA_W1S, (~0ULL));
nitrox_write_csr(ndev, AQM_DMA_RD_ERR_HI_ENA_W1S, (~0ULL));
nitrox_write_csr(ndev, AQM_EXEC_NA_LO_ENA_W1S, (~0ULL));
nitrox_write_csr(ndev, AQM_EXEC_NA_HI_ENA_W1S, (~0ULL));
nitrox_write_csr(ndev, AQM_EXEC_ERR_LO_ENA_W1S, (~0ULL));
nitrox_write_csr(ndev, AQM_EXEC_ERR_HI_ENA_W1S, (~0ULL));
}
void nitrox_config_aqm_unit(struct nitrox_device *ndev)
{
/* config aqm command queues */
nitrox_config_aqm_rings(ndev);
/* enable aqm interrupts */
enable_aqm_interrupts(ndev);
}
void nitrox_config_pom_unit(struct nitrox_device *ndev)
{
union pom_int_ena_w1s pom_int;
int i;
/* enable pom interrupts */
pom_int.value = 0;
pom_int.s.illegal_dport = 1;
nitrox_write_csr(ndev, POM_INT_ENA_W1S, pom_int.value);
/* enable perf counters */
for (i = 0; i < ndev->hw.se_cores; i++)
nitrox_write_csr(ndev, POM_PERF_CTL, BIT_ULL(i));
}
/**
* nitrox_config_rand_unit - enable NITROX random number unit
* @ndev: NITROX device
*/
void nitrox_config_rand_unit(struct nitrox_device *ndev)
{
union efl_rnm_ctl_status efl_rnm_ctl;
u64 offset;
offset = EFL_RNM_CTL_STATUS;
efl_rnm_ctl.value = nitrox_read_csr(ndev, offset);
efl_rnm_ctl.s.ent_en = 1;
efl_rnm_ctl.s.rng_en = 1;
nitrox_write_csr(ndev, offset, efl_rnm_ctl.value);
}
void nitrox_config_efl_unit(struct nitrox_device *ndev)
{
int i;
for (i = 0; i < NR_CLUSTERS; i++) {
union efl_core_int_ena_w1s efl_core_int;
u64 offset;
/* EFL core interrupts */
offset = EFL_CORE_INT_ENA_W1SX(i);
efl_core_int.value = 0;
efl_core_int.s.len_ovr = 1;
efl_core_int.s.d_left = 1;
efl_core_int.s.epci_decode_err = 1;
nitrox_write_csr(ndev, offset, efl_core_int.value);
offset = EFL_CORE_VF_ERR_INT0_ENA_W1SX(i);
nitrox_write_csr(ndev, offset, (~0ULL));
offset = EFL_CORE_VF_ERR_INT1_ENA_W1SX(i);
nitrox_write_csr(ndev, offset, (~0ULL));
}
}
void nitrox_config_bmi_unit(struct nitrox_device *ndev)
{
union bmi_ctl bmi_ctl;
union bmi_int_ena_w1s bmi_int_ena;
u64 offset;
/* no threshold limits for PCIe */
offset = BMI_CTL;
bmi_ctl.value = nitrox_read_csr(ndev, offset);
bmi_ctl.s.max_pkt_len = 0xff;
bmi_ctl.s.nps_free_thrsh = 0xff;
bmi_ctl.s.nps_hdrq_thrsh = 0x7a;
nitrox_write_csr(ndev, offset, bmi_ctl.value);
/* enable interrupts */
offset = BMI_INT_ENA_W1S;
bmi_int_ena.value = 0;
bmi_int_ena.s.max_len_err_nps = 1;
bmi_int_ena.s.pkt_rcv_err_nps = 1;
bmi_int_ena.s.fpf_undrrn = 1;
nitrox_write_csr(ndev, offset, bmi_int_ena.value);
}
void nitrox_config_bmo_unit(struct nitrox_device *ndev)
{
union bmo_ctl2 bmo_ctl2;
u64 offset;
/* no threshold limits for PCIe */
offset = BMO_CTL2;
bmo_ctl2.value = nitrox_read_csr(ndev, offset);
bmo_ctl2.s.nps_slc_buf_thrsh = 0xff;
nitrox_write_csr(ndev, offset, bmo_ctl2.value);
}
void invalidate_lbc(struct nitrox_device *ndev)
{
union lbc_inval_ctl lbc_ctl;
union lbc_inval_status lbc_stat;
int max_retries = MAX_CSR_RETRIES;
u64 offset;
/* invalidate LBC */
offset = LBC_INVAL_CTL;
lbc_ctl.value = nitrox_read_csr(ndev, offset);
lbc_ctl.s.cam_inval_start = 1;
nitrox_write_csr(ndev, offset, lbc_ctl.value);
offset = LBC_INVAL_STATUS;
do {
lbc_stat.value = nitrox_read_csr(ndev, offset);
if (lbc_stat.s.done)
break;
udelay(50);
} while (max_retries--);
}
void nitrox_config_lbc_unit(struct nitrox_device *ndev)
{
union lbc_int_ena_w1s lbc_int_ena;
u64 offset;
invalidate_lbc(ndev);
/* enable interrupts */
offset = LBC_INT_ENA_W1S;
lbc_int_ena.value = 0;
lbc_int_ena.s.dma_rd_err = 1;
lbc_int_ena.s.over_fetch_err = 1;
lbc_int_ena.s.cam_inval_abort = 1;
lbc_int_ena.s.cam_hard_err = 1;
nitrox_write_csr(ndev, offset, lbc_int_ena.value);
offset = LBC_PLM_VF1_64_INT_ENA_W1S;
nitrox_write_csr(ndev, offset, (~0ULL));
offset = LBC_PLM_VF65_128_INT_ENA_W1S;
nitrox_write_csr(ndev, offset, (~0ULL));
offset = LBC_ELM_VF1_64_INT_ENA_W1S;
nitrox_write_csr(ndev, offset, (~0ULL));
offset = LBC_ELM_VF65_128_INT_ENA_W1S;
nitrox_write_csr(ndev, offset, (~0ULL));
}
void config_nps_core_vfcfg_mode(struct nitrox_device *ndev, enum vf_mode mode)
{
union nps_core_gbl_vfcfg vfcfg;
vfcfg.value = nitrox_read_csr(ndev, NPS_CORE_GBL_VFCFG);
vfcfg.s.cfg = mode & 0x7;
nitrox_write_csr(ndev, NPS_CORE_GBL_VFCFG, vfcfg.value);
}
static const char *get_core_option(u8 se_cores, u8 ae_cores)
{
const char *option = "";
if (ae_cores == AE_MAX_CORES) {
switch (se_cores) {
case SE_MAX_CORES:
option = "60";
break;
case 40:
option = "60s";
break;
}
} else if (ae_cores == (AE_MAX_CORES / 2)) {
option = "30";
} else {
option = "60i";
}
return option;
}
static const char *get_feature_option(u8 zip_cores, int core_freq)
{
if (zip_cores == 0)
return "";
else if (zip_cores < ZIP_MAX_CORES)
return "-C15";
if (core_freq >= 850)
return "-C45";
else if (core_freq >= 750)
return "-C35";
else if (core_freq >= 550)
return "-C25";
return "";
}
void nitrox_get_hwinfo(struct nitrox_device *ndev)
{
union emu_fuse_map emu_fuse;
union rst_boot rst_boot;
union fus_dat1 fus_dat1;
unsigned char name[IFNAMSIZ * 2] = {};
int i, dead_cores;
u64 offset;
/* get core frequency */
offset = RST_BOOT;
rst_boot.value = nitrox_read_csr(ndev, offset);
ndev->hw.freq = (rst_boot.pnr_mul + 3) * PLL_REF_CLK;
for (i = 0; i < NR_CLUSTERS; i++) {
offset = EMU_FUSE_MAPX(i);
emu_fuse.value = nitrox_read_csr(ndev, offset);
if (emu_fuse.s.valid) {
dead_cores = hweight32(emu_fuse.s.ae_fuse);
ndev->hw.ae_cores += AE_CORES_PER_CLUSTER - dead_cores;
dead_cores = hweight16(emu_fuse.s.se_fuse);
ndev->hw.se_cores += SE_CORES_PER_CLUSTER - dead_cores;
}
}
/* find zip hardware availability */
offset = FUS_DAT1;
fus_dat1.value = nitrox_read_csr(ndev, offset);
if (!fus_dat1.nozip) {
dead_cores = hweight8(fus_dat1.zip_info);
ndev->hw.zip_cores = ZIP_MAX_CORES - dead_cores;
}
/* determine the partname
* CNN55<core option>-<freq><pincount>-<feature option>-<rev>
*/
snprintf(name, sizeof(name), "CNN55%s-%3dBG676%s-1.%u",
get_core_option(ndev->hw.se_cores, ndev->hw.ae_cores),
ndev->hw.freq,
get_feature_option(ndev->hw.zip_cores, ndev->hw.freq),
ndev->hw.revision_id);
/* copy partname */
strscpy(ndev->hw.partname, name, sizeof(ndev->hw.partname));
}
void enable_pf2vf_mbox_interrupts(struct nitrox_device *ndev)
{
u64 value = ~0ULL;
u64 reg_addr;
/* Mailbox interrupt low enable set register */
reg_addr = NPS_PKT_MBOX_INT_LO_ENA_W1S;
nitrox_write_csr(ndev, reg_addr, value);
/* Mailbox interrupt high enable set register */
reg_addr = NPS_PKT_MBOX_INT_HI_ENA_W1S;
nitrox_write_csr(ndev, reg_addr, value);
}
void disable_pf2vf_mbox_interrupts(struct nitrox_device *ndev)
{
u64 value = ~0ULL;
u64 reg_addr;
/* Mailbox interrupt low enable clear register */
reg_addr = NPS_PKT_MBOX_INT_LO_ENA_W1C;
nitrox_write_csr(ndev, reg_addr, value);
/* Mailbox interrupt high enable clear register */
reg_addr = NPS_PKT_MBOX_INT_HI_ENA_W1C;
nitrox_write_csr(ndev, reg_addr, value);
}
linux-7.2.3/drivers/crypto/cavium/nitrox/nitrox_hal.h
+29
View File
@@ -0,0 +1,29 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __NITROX_HAL_H
#define __NITROX_HAL_H
#include "nitrox_dev.h"
void nitrox_config_aqm_rings(struct nitrox_device *ndev);
void nitrox_config_aqm_unit(struct nitrox_device *ndev);
void nitrox_config_emu_unit(struct nitrox_device *ndev);
void nitrox_config_pkt_input_rings(struct nitrox_device *ndev);
void nitrox_config_pkt_solicit_ports(struct nitrox_device *ndev);
void nitrox_config_nps_core_unit(struct nitrox_device *ndev);
void nitrox_config_nps_pkt_unit(struct nitrox_device *ndev);
void nitrox_config_pom_unit(struct nitrox_device *ndev);
void nitrox_config_rand_unit(struct nitrox_device *ndev);
void nitrox_config_efl_unit(struct nitrox_device *ndev);
void nitrox_config_bmi_unit(struct nitrox_device *ndev);
void nitrox_config_bmo_unit(struct nitrox_device *ndev);
void nitrox_config_lbc_unit(struct nitrox_device *ndev);
void invalidate_lbc(struct nitrox_device *ndev);
void enable_aqm_ring(struct nitrox_device *ndev, int qno);
void enable_pkt_input_ring(struct nitrox_device *ndev, int ring);
void enable_pkt_solicit_port(struct nitrox_device *ndev, int port);
void config_nps_core_vfcfg_mode(struct nitrox_device *ndev, enum vf_mode mode);
void nitrox_get_hwinfo(struct nitrox_device *ndev);
void enable_pf2vf_mbox_interrupts(struct nitrox_device *ndev);
void disable_pf2vf_mbox_interrupts(struct nitrox_device *ndev);
#endif /* __NITROX_HAL_H */
linux-7.2.3/drivers/crypto/cavium/nitrox/nitrox_isr.c
+458
View File
@@ -0,0 +1,458 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/pci.h>
#include <linux/printk.h>
#include <linux/slab.h>
#include "nitrox_dev.h"
#include "nitrox_csr.h"
#include "nitrox_common.h"
#include "nitrox_hal.h"
#include "nitrox_isr.h"
#include "nitrox_mbx.h"
/*
* One vector for each type of ring
* - NPS packet ring, AQMQ ring and ZQMQ ring
*/
#define NR_RING_VECTORS 3
#define NR_NON_RING_VECTORS 1
/* base entry for packet ring/port */
#define PKT_RING_MSIX_BASE 0
#define NON_RING_MSIX_BASE 192
/**
* nps_pkt_slc_isr - IRQ handler for NPS solicit port
* @irq: irq number
* @data: argument
*/
static irqreturn_t nps_pkt_slc_isr(int irq, void *data)
{
struct nitrox_q_vector *qvec = data;
union nps_pkt_slc_cnts slc_cnts;
struct nitrox_cmdq *cmdq = qvec->cmdq;
slc_cnts.value = readq(cmdq->compl_cnt_csr_addr);
/* New packet on SLC output port */
if (slc_cnts.s.slc_int)
tasklet_hi_schedule(&qvec->resp_tasklet);
return IRQ_HANDLED;
}
static void clear_nps_core_err_intr(struct nitrox_device *ndev)
{
u64 value;
/* Write 1 to clear */
value = nitrox_read_csr(ndev, NPS_CORE_INT);
nitrox_write_csr(ndev, NPS_CORE_INT, value);
dev_err_ratelimited(DEV(ndev), "NSP_CORE_INT 0x%016llx\n", value);
}
static void clear_nps_pkt_err_intr(struct nitrox_device *ndev)
{
union nps_pkt_int pkt_int;
unsigned long value, offset;
int i;
pkt_int.value = nitrox_read_csr(ndev, NPS_PKT_INT);
dev_err_ratelimited(DEV(ndev), "NPS_PKT_INT 0x%016llx\n",
pkt_int.value);
if (pkt_int.s.slc_err) {
offset = NPS_PKT_SLC_ERR_TYPE;
value = nitrox_read_csr(ndev, offset);
nitrox_write_csr(ndev, offset, value);
dev_err_ratelimited(DEV(ndev),
"NPS_PKT_SLC_ERR_TYPE 0x%016lx\n", value);
offset = NPS_PKT_SLC_RERR_LO;
value = nitrox_read_csr(ndev, offset);
nitrox_write_csr(ndev, offset, value);
/* enable the solicit ports */
for_each_set_bit(i, &value, BITS_PER_LONG)
enable_pkt_solicit_port(ndev, i);
dev_err_ratelimited(DEV(ndev),
"NPS_PKT_SLC_RERR_LO 0x%016lx\n", value);
offset = NPS_PKT_SLC_RERR_HI;
value = nitrox_read_csr(ndev, offset);
nitrox_write_csr(ndev, offset, value);
dev_err_ratelimited(DEV(ndev),
"NPS_PKT_SLC_RERR_HI 0x%016lx\n", value);
}
if (pkt_int.s.in_err) {
offset = NPS_PKT_IN_ERR_TYPE;
value = nitrox_read_csr(ndev, offset);
nitrox_write_csr(ndev, offset, value);
dev_err_ratelimited(DEV(ndev),
"NPS_PKT_IN_ERR_TYPE 0x%016lx\n", value);
offset = NPS_PKT_IN_RERR_LO;
value = nitrox_read_csr(ndev, offset);
nitrox_write_csr(ndev, offset, value);
/* enable the input ring */
for_each_set_bit(i, &value, BITS_PER_LONG)
enable_pkt_input_ring(ndev, i);
dev_err_ratelimited(DEV(ndev),
"NPS_PKT_IN_RERR_LO 0x%016lx\n", value);
offset = NPS_PKT_IN_RERR_HI;
value = nitrox_read_csr(ndev, offset);
nitrox_write_csr(ndev, offset, value);
dev_err_ratelimited(DEV(ndev),
"NPS_PKT_IN_RERR_HI 0x%016lx\n", value);
}
}
static void clear_pom_err_intr(struct nitrox_device *ndev)
{
u64 value;
value = nitrox_read_csr(ndev, POM_INT);
nitrox_write_csr(ndev, POM_INT, value);
dev_err_ratelimited(DEV(ndev), "POM_INT 0x%016llx\n", value);
}
static void clear_pem_err_intr(struct nitrox_device *ndev)
{
u64 value;
value = nitrox_read_csr(ndev, PEM0_INT);
nitrox_write_csr(ndev, PEM0_INT, value);
dev_err_ratelimited(DEV(ndev), "PEM(0)_INT 0x%016llx\n", value);
}
static void clear_lbc_err_intr(struct nitrox_device *ndev)
{
union lbc_int lbc_int;
u64 value, offset;
int i;
lbc_int.value = nitrox_read_csr(ndev, LBC_INT);
dev_err_ratelimited(DEV(ndev), "LBC_INT 0x%016llx\n", lbc_int.value);
if (lbc_int.s.dma_rd_err) {
for (i = 0; i < NR_CLUSTERS; i++) {
offset = EFL_CORE_VF_ERR_INT0X(i);
value = nitrox_read_csr(ndev, offset);
nitrox_write_csr(ndev, offset, value);
offset = EFL_CORE_VF_ERR_INT1X(i);
value = nitrox_read_csr(ndev, offset);
nitrox_write_csr(ndev, offset, value);
}
}
if (lbc_int.s.cam_soft_err) {
dev_err_ratelimited(DEV(ndev), "CAM_SOFT_ERR, invalidating LBC\n");
invalidate_lbc(ndev);
}
if (lbc_int.s.pref_dat_len_mismatch_err) {
offset = LBC_PLM_VF1_64_INT;
value = nitrox_read_csr(ndev, offset);
nitrox_write_csr(ndev, offset, value);
offset = LBC_PLM_VF65_128_INT;
value = nitrox_read_csr(ndev, offset);
nitrox_write_csr(ndev, offset, value);
}
if (lbc_int.s.rd_dat_len_mismatch_err) {
offset = LBC_ELM_VF1_64_INT;
value = nitrox_read_csr(ndev, offset);
nitrox_write_csr(ndev, offset, value);
offset = LBC_ELM_VF65_128_INT;
value = nitrox_read_csr(ndev, offset);
nitrox_write_csr(ndev, offset, value);
}
nitrox_write_csr(ndev, LBC_INT, lbc_int.value);
}
static void clear_efl_err_intr(struct nitrox_device *ndev)
{
int i;
for (i = 0; i < NR_CLUSTERS; i++) {
union efl_core_int core_int;
u64 value, offset;
offset = EFL_CORE_INTX(i);
core_int.value = nitrox_read_csr(ndev, offset);
nitrox_write_csr(ndev, offset, core_int.value);
dev_err_ratelimited(DEV(ndev), "ELF_CORE(%d)_INT 0x%016llx\n",
i, core_int.value);
if (core_int.s.se_err) {
offset = EFL_CORE_SE_ERR_INTX(i);
value = nitrox_read_csr(ndev, offset);
nitrox_write_csr(ndev, offset, value);
}
}
}
static void clear_bmi_err_intr(struct nitrox_device *ndev)
{
u64 value;
value = nitrox_read_csr(ndev, BMI_INT);
nitrox_write_csr(ndev, BMI_INT, value);
dev_err_ratelimited(DEV(ndev), "BMI_INT 0x%016llx\n", value);
}
static void nps_core_int_tasklet(unsigned long data)
{
struct nitrox_q_vector *qvec = (void *)(uintptr_t)(data);
struct nitrox_device *ndev = qvec->ndev;
/* if pf mode do queue recovery */
if (ndev->mode == __NDEV_MODE_PF) {
} else {
/**
* if VF(s) enabled communicate the error information
* to VF(s)
*/
}
}
/*
* nps_core_int_isr - interrupt handler for NITROX errors and
* mailbox communication
*/
static irqreturn_t nps_core_int_isr(int irq, void *data)
{
struct nitrox_q_vector *qvec = data;
struct nitrox_device *ndev = qvec->ndev;
union nps_core_int_active core_int;
core_int.value = nitrox_read_csr(ndev, NPS_CORE_INT_ACTIVE);
if (core_int.s.nps_core)
clear_nps_core_err_intr(ndev);
if (core_int.s.nps_pkt)
clear_nps_pkt_err_intr(ndev);
if (core_int.s.pom)
clear_pom_err_intr(ndev);
if (core_int.s.pem)
clear_pem_err_intr(ndev);
if (core_int.s.lbc)
clear_lbc_err_intr(ndev);
if (core_int.s.efl)
clear_efl_err_intr(ndev);
if (core_int.s.bmi)
clear_bmi_err_intr(ndev);
/* Mailbox interrupt */
if (core_int.s.mbox)
nitrox_pf2vf_mbox_handler(ndev);
/* If more work callback the ISR, set resend */
core_int.s.resend = 1;
nitrox_write_csr(ndev, NPS_CORE_INT_ACTIVE, core_int.value);
return IRQ_HANDLED;
}
void nitrox_unregister_interrupts(struct nitrox_device *ndev)
{
struct pci_dev *pdev = ndev->pdev;
int i;
for (i = 0; i < ndev->num_vecs; i++) {
struct nitrox_q_vector *qvec;
int vec;
qvec = ndev->qvec + i;
if (!qvec->valid)
continue;
/* get the vector number */
vec = pci_irq_vector(pdev, i);
irq_set_affinity_hint(vec, NULL);
free_irq(vec, qvec);
tasklet_disable(&qvec->resp_tasklet);
tasklet_kill(&qvec->resp_tasklet);
qvec->valid = false;
}
kfree(ndev->qvec);
ndev->qvec = NULL;
pci_free_irq_vectors(pdev);
}
int nitrox_register_interrupts(struct nitrox_device *ndev)
{
struct pci_dev *pdev = ndev->pdev;
struct nitrox_q_vector *qvec;
int nr_vecs, vec, cpu;
int ret, i;
/*
* PF MSI-X vectors
*
* Entry 0: NPS PKT ring 0
* Entry 1: AQMQ ring 0
* Entry 2: ZQM ring 0
* Entry 3: NPS PKT ring 1
* Entry 4: AQMQ ring 1
* Entry 5: ZQM ring 1
* ....
* Entry 192: NPS_CORE_INT_ACTIVE
*/
nr_vecs = pci_msix_vec_count(pdev);
if (nr_vecs < 0) {
dev_err(DEV(ndev), "Error in getting vec count %d\n", nr_vecs);
return nr_vecs;
}
/* Enable MSI-X */
ret = pci_alloc_irq_vectors(pdev, nr_vecs, nr_vecs, PCI_IRQ_MSIX);
if (ret < 0) {
dev_err(DEV(ndev), "msix vectors %d alloc failed\n", nr_vecs);
return ret;
}
ndev->num_vecs = nr_vecs;
ndev->qvec = kzalloc_objs(*qvec, nr_vecs);
if (!ndev->qvec) {
pci_free_irq_vectors(pdev);
return -ENOMEM;
}
/* request irqs for packet rings/ports */
for (i = PKT_RING_MSIX_BASE; i < (nr_vecs - 1); i += NR_RING_VECTORS) {
qvec = &ndev->qvec[i];
qvec->ring = i / NR_RING_VECTORS;
if (qvec->ring >= ndev->nr_queues)
break;
qvec->cmdq = &ndev->pkt_inq[qvec->ring];
snprintf(qvec->name, IRQ_NAMESZ, "nitrox-pkt%d", qvec->ring);
/* get the vector number */
vec = pci_irq_vector(pdev, i);
ret = request_irq(vec, nps_pkt_slc_isr, 0, qvec->name, qvec);
if (ret) {
dev_err(DEV(ndev), "irq failed for pkt ring/port%d\n",
qvec->ring);
goto irq_fail;
}
cpu = qvec->ring % num_online_cpus();
irq_set_affinity_hint(vec, get_cpu_mask(cpu));
tasklet_init(&qvec->resp_tasklet, pkt_slc_resp_tasklet,
(unsigned long)qvec);
qvec->valid = true;
}
/* request irqs for non ring vectors */
i = NON_RING_MSIX_BASE;
qvec = &ndev->qvec[i];
qvec->ndev = ndev;
snprintf(qvec->name, IRQ_NAMESZ, "nitrox-core-int%d", i);
/* get the vector number */
vec = pci_irq_vector(pdev, i);
ret = request_irq(vec, nps_core_int_isr, 0, qvec->name, qvec);
if (ret) {
dev_err(DEV(ndev), "irq failed for nitrox-core-int%d\n", i);
goto irq_fail;
}
cpu = num_online_cpus();
irq_set_affinity_hint(vec, get_cpu_mask(cpu));
tasklet_init(&qvec->resp_tasklet, nps_core_int_tasklet,
(unsigned long)qvec);
qvec->valid = true;
return 0;
irq_fail:
nitrox_unregister_interrupts(ndev);
return ret;
}
void nitrox_sriov_unregister_interrupts(struct nitrox_device *ndev)
{
struct pci_dev *pdev = ndev->pdev;
int i;
for (i = 0; i < ndev->num_vecs; i++) {
struct nitrox_q_vector *qvec;
int vec;
qvec = ndev->qvec + i;
if (!qvec->valid)
continue;
vec = ndev->iov.msix.vector;
irq_set_affinity_hint(vec, NULL);
free_irq(vec, qvec);
tasklet_disable(&qvec->resp_tasklet);
tasklet_kill(&qvec->resp_tasklet);
qvec->valid = false;
}
kfree(ndev->qvec);
ndev->qvec = NULL;
pci_disable_msix(pdev);
}
int nitrox_sriov_register_interupts(struct nitrox_device *ndev)
{
struct pci_dev *pdev = ndev->pdev;
struct nitrox_q_vector *qvec;
int vec, cpu;
int ret;
/**
* only non ring vectors i.e Entry 192 is available
* for PF in SR-IOV mode.
*/
ndev->iov.msix.entry = NON_RING_MSIX_BASE;
ret = pci_enable_msix_exact(pdev, &ndev->iov.msix, NR_NON_RING_VECTORS);
if (ret) {
dev_err(DEV(ndev), "failed to allocate nps-core-int%d\n",
NON_RING_MSIX_BASE);
return ret;
}
qvec = kzalloc_objs(*qvec, NR_NON_RING_VECTORS);
if (!qvec) {
pci_disable_msix(pdev);
return -ENOMEM;
}
qvec->ndev = ndev;
ndev->qvec = qvec;
ndev->num_vecs = NR_NON_RING_VECTORS;
snprintf(qvec->name, IRQ_NAMESZ, "nitrox-core-int%d",
NON_RING_MSIX_BASE);
vec = ndev->iov.msix.vector;
ret = request_irq(vec, nps_core_int_isr, 0, qvec->name, qvec);
if (ret) {
dev_err(DEV(ndev), "irq failed for nitrox-core-int%d\n",
NON_RING_MSIX_BASE);
goto iov_irq_fail;
}
cpu = num_online_cpus();
irq_set_affinity_hint(vec, get_cpu_mask(cpu));
tasklet_init(&qvec->resp_tasklet, nps_core_int_tasklet,
(unsigned long)qvec);
qvec->valid = true;
return 0;
iov_irq_fail:
nitrox_sriov_unregister_interrupts(ndev);
return ret;
}
linux-7.2.3/drivers/crypto/cavium/nitrox/nitrox_isr.h
+21
View File
@@ -0,0 +1,21 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __NITROX_ISR_H
#define __NITROX_ISR_H
#include "nitrox_dev.h"
int nitrox_register_interrupts(struct nitrox_device *ndev);
void nitrox_unregister_interrupts(struct nitrox_device *ndev);
int nitrox_sriov_register_interupts(struct nitrox_device *ndev);
void nitrox_sriov_unregister_interrupts(struct nitrox_device *ndev);
#ifdef CONFIG_PCI_IOV
int nitrox_sriov_configure(struct pci_dev *pdev, int num_vfs);
#else
static inline int nitrox_sriov_configure(struct pci_dev *pdev, int num_vfs)
{
return 0;
}
#endif
#endif /* __NITROX_ISR_H */
linux-7.2.3/drivers/crypto/cavium/nitrox/nitrox_lib.c
+300
View File
@@ -0,0 +1,300 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/cpumask.h>
#include <linux/dma-mapping.h>
#include <linux/dmapool.h>
#include <linux/delay.h>
#include <linux/gfp.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/pci_regs.h>
#include <linux/vmalloc.h>
#include <linux/pci.h>
#include "nitrox_dev.h"
#include "nitrox_common.h"
#include "nitrox_req.h"
#include "nitrox_csr.h"
#define CRYPTO_CTX_SIZE 256
/* packet input ring alignments */
#define PKTIN_Q_ALIGN_BYTES 16
/* AQM Queue input alignments */
#define AQM_Q_ALIGN_BYTES 32
static int nitrox_cmdq_init(struct nitrox_cmdq *cmdq, int align_bytes)
{
struct nitrox_device *ndev = cmdq->ndev;
cmdq->qsize = (ndev->qlen * cmdq->instr_size) + align_bytes;
cmdq->unalign_base = dma_alloc_coherent(DEV(ndev), cmdq->qsize,
&cmdq->unalign_dma,
GFP_KERNEL);
if (!cmdq->unalign_base)
return -ENOMEM;
cmdq->dma = PTR_ALIGN(cmdq->unalign_dma, align_bytes);
cmdq->base = cmdq->unalign_base + (cmdq->dma - cmdq->unalign_dma);
cmdq->write_idx = 0;
spin_lock_init(&cmdq->cmd_qlock);
spin_lock_init(&cmdq->resp_qlock);
spin_lock_init(&cmdq->backlog_qlock);
INIT_LIST_HEAD(&cmdq->response_head);
INIT_LIST_HEAD(&cmdq->backlog_head);
INIT_WORK(&cmdq->backlog_qflush, backlog_qflush_work);
atomic_set(&cmdq->pending_count, 0);
atomic_set(&cmdq->backlog_count, 0);
return 0;
}
static void nitrox_cmdq_reset(struct nitrox_cmdq *cmdq)
{
cmdq->write_idx = 0;
atomic_set(&cmdq->pending_count, 0);
atomic_set(&cmdq->backlog_count, 0);
}
static void nitrox_cmdq_cleanup(struct nitrox_cmdq *cmdq)
{
struct nitrox_device *ndev;
if (!cmdq)
return;
if (!cmdq->unalign_base)
return;
ndev = cmdq->ndev;
cancel_work_sync(&cmdq->backlog_qflush);
dma_free_coherent(DEV(ndev), cmdq->qsize,
cmdq->unalign_base, cmdq->unalign_dma);
nitrox_cmdq_reset(cmdq);
cmdq->dbell_csr_addr = NULL;
cmdq->compl_cnt_csr_addr = NULL;
cmdq->unalign_base = NULL;
cmdq->base = NULL;
cmdq->unalign_dma = 0;
cmdq->dma = 0;
cmdq->qsize = 0;
cmdq->instr_size = 0;
}
static void nitrox_free_aqm_queues(struct nitrox_device *ndev)
{
int i;
for (i = 0; i < ndev->nr_queues; i++) {
nitrox_cmdq_cleanup(ndev->aqmq[i]);
kfree_sensitive(ndev->aqmq[i]);
ndev->aqmq[i] = NULL;
}
}
static int nitrox_alloc_aqm_queues(struct nitrox_device *ndev)
{
int i, err;
for (i = 0; i < ndev->nr_queues; i++) {
struct nitrox_cmdq *cmdq;
u64 offset;
cmdq = kzalloc_node(sizeof(*cmdq), GFP_KERNEL, ndev->node);
if (!cmdq) {
err = -ENOMEM;
goto aqmq_fail;
}
cmdq->ndev = ndev;
cmdq->qno = i;
cmdq->instr_size = sizeof(struct aqmq_command_s);
/* AQM Queue Doorbell Counter Register Address */
offset = AQMQ_DRBLX(i);
cmdq->dbell_csr_addr = NITROX_CSR_ADDR(ndev, offset);
/* AQM Queue Commands Completed Count Register Address */
offset = AQMQ_CMD_CNTX(i);
cmdq->compl_cnt_csr_addr = NITROX_CSR_ADDR(ndev, offset);
err = nitrox_cmdq_init(cmdq, AQM_Q_ALIGN_BYTES);
if (err) {
kfree_sensitive(cmdq);
goto aqmq_fail;
}
ndev->aqmq[i] = cmdq;
}
return 0;
aqmq_fail:
nitrox_free_aqm_queues(ndev);
return err;
}
static void nitrox_free_pktin_queues(struct nitrox_device *ndev)
{
int i;
for (i = 0; i < ndev->nr_queues; i++) {
struct nitrox_cmdq *cmdq = &ndev->pkt_inq[i];
nitrox_cmdq_cleanup(cmdq);
}
kfree(ndev->pkt_inq);
ndev->pkt_inq = NULL;
}
static int nitrox_alloc_pktin_queues(struct nitrox_device *ndev)
{
int i, err;
ndev->pkt_inq = kcalloc_node(ndev->nr_queues,
sizeof(struct nitrox_cmdq),
GFP_KERNEL, ndev->node);
if (!ndev->pkt_inq)
return -ENOMEM;
for (i = 0; i < ndev->nr_queues; i++) {
struct nitrox_cmdq *cmdq;
u64 offset;
cmdq = &ndev->pkt_inq[i];
cmdq->ndev = ndev;
cmdq->qno = i;
cmdq->instr_size = sizeof(struct nps_pkt_instr);
/* packet input ring doorbell address */
offset = NPS_PKT_IN_INSTR_BAOFF_DBELLX(i);
cmdq->dbell_csr_addr = NITROX_CSR_ADDR(ndev, offset);
/* packet solicit port completion count address */
offset = NPS_PKT_SLC_CNTSX(i);
cmdq->compl_cnt_csr_addr = NITROX_CSR_ADDR(ndev, offset);
err = nitrox_cmdq_init(cmdq, PKTIN_Q_ALIGN_BYTES);
if (err)
goto pktq_fail;
}
return 0;
pktq_fail:
nitrox_free_pktin_queues(ndev);
return err;
}
static int create_crypto_dma_pool(struct nitrox_device *ndev)
{
size_t size;
/* Crypto context pool, 16 byte aligned */
size = CRYPTO_CTX_SIZE + sizeof(struct ctx_hdr);
ndev->ctx_pool = dma_pool_create("nitrox-context",
DEV(ndev), size, 16, 0);
if (!ndev->ctx_pool)
return -ENOMEM;
return 0;
}
static void destroy_crypto_dma_pool(struct nitrox_device *ndev)
{
if (!ndev->ctx_pool)
return;
dma_pool_destroy(ndev->ctx_pool);
ndev->ctx_pool = NULL;
}
/*
* crypto_alloc_context - Allocate crypto context from pool
* @ndev: NITROX Device
*/
void *crypto_alloc_context(struct nitrox_device *ndev)
{
struct ctx_hdr *ctx;
struct crypto_ctx_hdr *chdr;
void *vaddr;
dma_addr_t dma;
chdr = kmalloc_obj(*chdr);
if (!chdr)
return NULL;
vaddr = dma_pool_zalloc(ndev->ctx_pool, GFP_KERNEL, &dma);
if (!vaddr) {
kfree(chdr);
return NULL;
}
/* fill meta data */
ctx = vaddr;
ctx->pool = ndev->ctx_pool;
ctx->dma = dma;
ctx->ctx_dma = dma + sizeof(struct ctx_hdr);
chdr->pool = ndev->ctx_pool;
chdr->dma = dma;
chdr->vaddr = vaddr;
return chdr;
}
/**
* crypto_free_context - Free crypto context to pool
* @ctx: context to free
*/
void crypto_free_context(void *ctx)
{
struct crypto_ctx_hdr *ctxp;
if (!ctx)
return;
ctxp = ctx;
dma_pool_free(ctxp->pool, ctxp->vaddr, ctxp->dma);
kfree(ctxp);
}
/**
* nitrox_common_sw_init - allocate software resources.
* @ndev: NITROX device
*
* Allocates crypto context pools and command queues etc.
*
* Return: 0 on success, or a negative error code on error.
*/
int nitrox_common_sw_init(struct nitrox_device *ndev)
{
int err = 0;
/* per device crypto context pool */
err = create_crypto_dma_pool(ndev);
if (err)
return err;
err = nitrox_alloc_pktin_queues(ndev);
if (err)
destroy_crypto_dma_pool(ndev);
err = nitrox_alloc_aqm_queues(ndev);
if (err) {
nitrox_free_pktin_queues(ndev);
destroy_crypto_dma_pool(ndev);
}
return err;
}
/**
* nitrox_common_sw_cleanup - free software resources.
* @ndev: NITROX device
*/
void nitrox_common_sw_cleanup(struct nitrox_device *ndev)
{
nitrox_free_aqm_queues(ndev);
nitrox_free_pktin_queues(ndev);
destroy_crypto_dma_pool(ndev);
}
linux-7.2.3/drivers/crypto/cavium/nitrox/nitrox_main.c
+582
View File
@@ -0,0 +1,582 @@
// SPDX-License-Identifier: GPL-2.0-only
#include <linux/delay.h>
#include <linux/firmware.h>
#include <linux/list.h>
#include <linux/module.h>
#include <linux/mutex.h>
#include <linux/pci.h>
#include <linux/pci_ids.h>
#include "nitrox_dev.h"
#include "nitrox_common.h"
#include "nitrox_csr.h"
#include "nitrox_hal.h"
#include "nitrox_isr.h"
#include "nitrox_debugfs.h"
#define CNN55XX_DEV_ID 0x12
#define UCODE_HLEN 48
#define DEFAULT_SE_GROUP 0
#define DEFAULT_AE_GROUP 0
#define DRIVER_VERSION "1.2"
#define CNN55XX_UCD_BLOCK_SIZE 32768
#define CNN55XX_MAX_UCODE_SIZE (CNN55XX_UCD_BLOCK_SIZE * 2)
#define FW_DIR "cavium/"
/* SE microcode */
#define SE_FW FW_DIR "cnn55xx_se.fw"
/* AE microcode */
#define AE_FW FW_DIR "cnn55xx_ae.fw"
static const char nitrox_driver_name[] = "CNN55XX";
static LIST_HEAD(ndevlist);
static DEFINE_MUTEX(devlist_lock);
static unsigned int num_devices;
/*
* nitrox_pci_tbl - PCI Device ID Table
*/
static const struct pci_device_id nitrox_pci_tbl[] = {
{ PCI_VDEVICE(CAVIUM, CNN55XX_DEV_ID) },
/* required last entry */
{ }
};
MODULE_DEVICE_TABLE(pci, nitrox_pci_tbl);
static unsigned int qlen = DEFAULT_CMD_QLEN;
module_param(qlen, uint, 0644);
MODULE_PARM_DESC(qlen, "Command queue length - default 2048");
/**
* struct ucode - Firmware Header
* @id: microcode ID
* @version: firmware version
* @code_size: code section size
* @raz: alignment
* @code: code section
*/
struct ucode {
u8 id;
char version[VERSION_LEN - 1];
__be32 code_size;
u8 raz[12];
u64 code[];
};
/*
* write_to_ucd_unit - Write Firmware to NITROX UCD unit
*/
static void write_to_ucd_unit(struct nitrox_device *ndev, u32 ucode_size,
u64 *ucode_data, int block_num)
{
u32 code_size;
u64 offset, data;
int i = 0;
/*
* UCD structure
*
* -------------
* | BLK 7 |
* -------------
* | BLK 6 |
* -------------
* | ... |
* -------------
* | BLK 0 |
* -------------
* Total of 8 blocks, each size 32KB
*/
/* set the block number */
offset = UCD_UCODE_LOAD_BLOCK_NUM;
nitrox_write_csr(ndev, offset, block_num);
code_size = roundup(ucode_size, 16);
while (code_size) {
data = ucode_data[i];
/* write 8 bytes at a time */
offset = UCD_UCODE_LOAD_IDX_DATAX(i);
nitrox_write_csr(ndev, offset, data);
code_size -= 8;
i++;
}
usleep_range(300, 400);
}
static int nitrox_load_fw(struct nitrox_device *ndev)
{
const struct firmware *fw;
const char *fw_name;
struct ucode *ucode;
u64 *ucode_data;
u64 offset;
union ucd_core_eid_ucode_block_num core_2_eid_val;
union aqm_grp_execmsk_lo aqm_grp_execmask_lo;
union aqm_grp_execmsk_hi aqm_grp_execmask_hi;
u32 ucode_size;
int ret, i = 0;
fw_name = SE_FW;
dev_info(DEV(ndev), "Loading firmware \"%s\"\n", fw_name);
ret = request_firmware(&fw, fw_name, DEV(ndev));
if (ret < 0) {
dev_err(DEV(ndev), "failed to get firmware %s\n", fw_name);
return ret;
}
ucode = (struct ucode *)fw->data;
ucode_size = be32_to_cpu(ucode->code_size) * 2;
if (!ucode_size || ucode_size > CNN55XX_MAX_UCODE_SIZE) {
dev_err(DEV(ndev), "Invalid ucode size: %u for firmware %s\n",
ucode_size, fw_name);
release_firmware(fw);
return -EINVAL;
}
ucode_data = ucode->code;
/* copy the firmware version */
memcpy(&ndev->hw.fw_name[0][0], ucode->version, (VERSION_LEN - 2));
ndev->hw.fw_name[0][VERSION_LEN - 1] = '\0';
/* Load SE Firmware on UCD Block 0 */
write_to_ucd_unit(ndev, ucode_size, ucode_data, 0);
release_firmware(fw);
/* put all SE cores in DEFAULT_SE_GROUP */
offset = POM_GRP_EXECMASKX(DEFAULT_SE_GROUP);
nitrox_write_csr(ndev, offset, (~0ULL));
/* write block number and firmware length
* bit:<2:0> block number
* bit:3 is set SE uses 32KB microcode
* bit:3 is clear SE uses 64KB microcode
*/
core_2_eid_val.value = 0ULL;
core_2_eid_val.ucode_blk = 0;
if (ucode_size <= CNN55XX_UCD_BLOCK_SIZE)
core_2_eid_val.ucode_len = 1;
else
core_2_eid_val.ucode_len = 0;
for (i = 0; i < ndev->hw.se_cores; i++) {
offset = UCD_SE_EID_UCODE_BLOCK_NUMX(i);
nitrox_write_csr(ndev, offset, core_2_eid_val.value);
}
fw_name = AE_FW;
dev_info(DEV(ndev), "Loading firmware \"%s\"\n", fw_name);
ret = request_firmware(&fw, fw_name, DEV(ndev));
if (ret < 0) {
dev_err(DEV(ndev), "failed to get firmware %s\n", fw_name);
return ret;
}
ucode = (struct ucode *)fw->data;
ucode_size = be32_to_cpu(ucode->code_size) * 2;
if (!ucode_size || ucode_size > CNN55XX_MAX_UCODE_SIZE) {
dev_err(DEV(ndev), "Invalid ucode size: %u for firmware %s\n",
ucode_size, fw_name);
release_firmware(fw);
return -EINVAL;
}
ucode_data = ucode->code;
/* copy the firmware version */
memcpy(&ndev->hw.fw_name[1][0], ucode->version, (VERSION_LEN - 2));
ndev->hw.fw_name[1][VERSION_LEN - 1] = '\0';
/* Load AE Firmware on UCD Block 2 */
write_to_ucd_unit(ndev, ucode_size, ucode_data, 2);
release_firmware(fw);
/* put all AE cores in DEFAULT_AE_GROUP */
offset = AQM_GRP_EXECMSK_LOX(DEFAULT_AE_GROUP);
aqm_grp_execmask_lo.exec_0_to_39 = 0xFFFFFFFFFFULL;
nitrox_write_csr(ndev, offset, aqm_grp_execmask_lo.value);
offset = AQM_GRP_EXECMSK_HIX(DEFAULT_AE_GROUP);
aqm_grp_execmask_hi.exec_40_to_79 = 0xFFFFFFFFFFULL;
nitrox_write_csr(ndev, offset, aqm_grp_execmask_hi.value);
/* write block number and firmware length
* bit:<2:0> block number
* bit:3 is set AE uses 32KB microcode
* bit:3 is clear AE uses 64KB microcode
*/
core_2_eid_val.value = 0ULL;
core_2_eid_val.ucode_blk = 2;
if (ucode_size <= CNN55XX_UCD_BLOCK_SIZE)
core_2_eid_val.ucode_len = 1;
else
core_2_eid_val.ucode_len = 0;
for (i = 0; i < ndev->hw.ae_cores; i++) {
offset = UCD_AE_EID_UCODE_BLOCK_NUMX(i);
nitrox_write_csr(ndev, offset, core_2_eid_val.value);
}
return 0;
}
/**
* nitrox_add_to_devlist - add NITROX device to global device list
* @ndev: NITROX device
*/
static int nitrox_add_to_devlist(struct nitrox_device *ndev)
{
struct nitrox_device *dev;
int ret = 0;
INIT_LIST_HEAD(&ndev->list);
refcount_set(&ndev->refcnt, 1);
mutex_lock(&devlist_lock);
list_for_each_entry(dev, &ndevlist, list) {
if (dev == ndev) {
ret = -EEXIST;
goto unlock;
}
}
ndev->idx = num_devices++;
list_add_tail(&ndev->list, &ndevlist);
unlock:
mutex_unlock(&devlist_lock);
return ret;
}
/**
* nitrox_remove_from_devlist - remove NITROX device from
* global device list
* @ndev: NITROX device
*/
static void nitrox_remove_from_devlist(struct nitrox_device *ndev)
{
mutex_lock(&devlist_lock);
list_del(&ndev->list);
num_devices--;
mutex_unlock(&devlist_lock);
}
struct nitrox_device *nitrox_get_first_device(void)
{
struct nitrox_device *ndev = NULL, *iter;
mutex_lock(&devlist_lock);
list_for_each_entry(iter, &ndevlist, list) {
if (nitrox_ready(iter)) {
ndev = iter;
break;
}
}
mutex_unlock(&devlist_lock);
if (!ndev)
return NULL;
refcount_inc(&ndev->refcnt);
/* barrier to sync with other cpus */
smp_mb__after_atomic();
return ndev;
}
void nitrox_put_device(struct nitrox_device *ndev)
{
if (!ndev)
return;
refcount_dec(&ndev->refcnt);
/* barrier to sync with other cpus */
smp_mb__after_atomic();
}
static int nitrox_device_flr(struct pci_dev *pdev)
{
int pos = 0;
pos = pci_save_state(pdev);
if (pos) {
dev_err(&pdev->dev, "Failed to save pci state\n");
return -ENOMEM;
}
pcie_reset_flr(pdev, PCI_RESET_DO_RESET);
pci_restore_state(pdev);
return 0;
}
static int nitrox_pf_sw_init(struct nitrox_device *ndev)
{
int err;
err = nitrox_common_sw_init(ndev);
if (err)
return err;
err = nitrox_register_interrupts(ndev);
if (err)
nitrox_common_sw_cleanup(ndev);
return err;
}
static void nitrox_pf_sw_cleanup(struct nitrox_device *ndev)
{
nitrox_unregister_interrupts(ndev);
nitrox_common_sw_cleanup(ndev);
}
/**
* nitrox_bist_check - Check NITROX BIST registers status
* @ndev: NITROX device
*/
static int nitrox_bist_check(struct nitrox_device *ndev)
{
u64 value = 0;
int i;
for (i = 0; i < NR_CLUSTERS; i++) {
value += nitrox_read_csr(ndev, EMU_BIST_STATUSX(i));
value += nitrox_read_csr(ndev, EFL_CORE_BIST_REGX(i));
}
value += nitrox_read_csr(ndev, UCD_BIST_STATUS);
value += nitrox_read_csr(ndev, NPS_CORE_BIST_REG);
value += nitrox_read_csr(ndev, NPS_CORE_NPC_BIST_REG);
value += nitrox_read_csr(ndev, NPS_PKT_SLC_BIST_REG);
value += nitrox_read_csr(ndev, NPS_PKT_IN_BIST_REG);
value += nitrox_read_csr(ndev, POM_BIST_REG);
value += nitrox_read_csr(ndev, BMI_BIST_REG);
value += nitrox_read_csr(ndev, EFL_TOP_BIST_STAT);
value += nitrox_read_csr(ndev, BMO_BIST_REG);
value += nitrox_read_csr(ndev, LBC_BIST_STATUS);
value += nitrox_read_csr(ndev, PEM_BIST_STATUSX(0));
if (value)
return -EIO;
return 0;
}
static int nitrox_pf_hw_init(struct nitrox_device *ndev)
{
int err;
err = nitrox_bist_check(ndev);
if (err) {
dev_err(&ndev->pdev->dev, "BIST check failed\n");
return err;
}
/* get cores information */
nitrox_get_hwinfo(ndev);
nitrox_config_nps_core_unit(ndev);
nitrox_config_aqm_unit(ndev);
nitrox_config_nps_pkt_unit(ndev);
nitrox_config_pom_unit(ndev);
nitrox_config_efl_unit(ndev);
/* configure IO units */
nitrox_config_bmi_unit(ndev);
nitrox_config_bmo_unit(ndev);
/* configure Local Buffer Cache */
nitrox_config_lbc_unit(ndev);
nitrox_config_rand_unit(ndev);
/* load firmware on cores */
err = nitrox_load_fw(ndev);
if (err)
return err;
nitrox_config_emu_unit(ndev);
return 0;
}
/**
* nitrox_probe - NITROX Initialization function.
* @pdev: PCI device information struct
* @id: entry in nitrox_pci_tbl
*
* Return: 0, if the driver is bound to the device, or
* a negative error if there is failure.
*/
static int nitrox_probe(struct pci_dev *pdev,
const struct pci_device_id *id)
{
struct nitrox_device *ndev;
int err;
dev_info_once(&pdev->dev, "%s driver version %s\n",
nitrox_driver_name, DRIVER_VERSION);
err = pci_enable_device_mem(pdev);
if (err)
return err;
/* do FLR */
err = nitrox_device_flr(pdev);
if (err) {
dev_err(&pdev->dev, "FLR failed\n");
goto flr_fail;
}
if (!dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64))) {
dev_dbg(&pdev->dev, "DMA to 64-BIT address\n");
} else {
err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
if (err) {
dev_err(&pdev->dev, "DMA configuration failed\n");
goto flr_fail;
}
}
err = pci_request_mem_regions(pdev, nitrox_driver_name);
if (err)
goto flr_fail;
pci_set_master(pdev);
ndev = kzalloc_obj(*ndev);
if (!ndev) {
err = -ENOMEM;
goto ndev_fail;
}
pci_set_drvdata(pdev, ndev);
ndev->pdev = pdev;
/* add to device list */
nitrox_add_to_devlist(ndev);
ndev->hw.vendor_id = pdev->vendor;
ndev->hw.device_id = pdev->device;
ndev->hw.revision_id = pdev->revision;
/* command timeout in jiffies */
ndev->timeout = msecs_to_jiffies(CMD_TIMEOUT);
ndev->node = dev_to_node(&pdev->dev);
if (ndev->node == NUMA_NO_NODE)
ndev->node = 0;
ndev->bar_addr = ioremap(pci_resource_start(pdev, 0),
pci_resource_len(pdev, 0));
if (!ndev->bar_addr) {
err = -EIO;
goto ioremap_err;
}
/* allocate command queus based on cpus, max queues are 64 */
ndev->nr_queues = min_t(u32, MAX_PF_QUEUES, num_online_cpus());
ndev->qlen = qlen;
err = nitrox_pf_sw_init(ndev);
if (err)
goto pf_sw_fail;
err = nitrox_pf_hw_init(ndev);
if (err)
goto pf_hw_fail;
nitrox_debugfs_init(ndev);
/* clear the statistics */
atomic64_set(&ndev->stats.posted, 0);
atomic64_set(&ndev->stats.completed, 0);
atomic64_set(&ndev->stats.dropped, 0);
atomic_set(&ndev->state, __NDEV_READY);
/* barrier to sync with other cpus */
smp_mb__after_atomic();
err = nitrox_crypto_register();
if (err)
goto crypto_fail;
return 0;
crypto_fail:
nitrox_debugfs_exit(ndev);
atomic_set(&ndev->state, __NDEV_NOT_READY);
/* barrier to sync with other cpus */
smp_mb__after_atomic();
pf_hw_fail:
nitrox_pf_sw_cleanup(ndev);
pf_sw_fail:
iounmap(ndev->bar_addr);
ioremap_err:
nitrox_remove_from_devlist(ndev);
kfree(ndev);
pci_set_drvdata(pdev, NULL);
ndev_fail:
pci_release_mem_regions(pdev);
flr_fail:
pci_disable_device(pdev);
return err;
}
/**
* nitrox_remove - Unbind the driver from the device.
* @pdev: PCI device information struct
*/
static void nitrox_remove(struct pci_dev *pdev)
{
struct nitrox_device *ndev = pci_get_drvdata(pdev);
if (!ndev)
return;
if (!refcount_dec_and_test(&ndev->refcnt)) {
dev_err(DEV(ndev), "Device refcnt not zero (%d)\n",
refcount_read(&ndev->refcnt));
return;
}
dev_info(DEV(ndev), "Removing Device %x:%x\n",
ndev->hw.vendor_id, ndev->hw.device_id);
atomic_set(&ndev->state, __NDEV_NOT_READY);
/* barrier to sync with other cpus */
smp_mb__after_atomic();
nitrox_remove_from_devlist(ndev);
/* disable SR-IOV */
nitrox_sriov_configure(pdev, 0);
nitrox_crypto_unregister();
nitrox_debugfs_exit(ndev);
nitrox_pf_sw_cleanup(ndev);
iounmap(ndev->bar_addr);
kfree(ndev);
pci_set_drvdata(pdev, NULL);
pci_release_mem_regions(pdev);
pci_disable_device(pdev);
}
static void nitrox_shutdown(struct pci_dev *pdev)
{
pci_set_drvdata(pdev, NULL);
pci_release_mem_regions(pdev);
pci_disable_device(pdev);
}
static struct pci_driver nitrox_driver = {
.name = nitrox_driver_name,
.id_table = nitrox_pci_tbl,
.probe = nitrox_probe,
.remove = nitrox_remove,
.shutdown = nitrox_shutdown,
.sriov_configure = nitrox_sriov_configure,
};
module_pci_driver(nitrox_driver);
MODULE_AUTHOR("Srikanth Jampala <Jampala.Srikanth@cavium.com>");
MODULE_DESCRIPTION("Cavium CNN55XX PF Driver" DRIVER_VERSION " ");
MODULE_LICENSE("GPL");
MODULE_VERSION(DRIVER_VERSION);
MODULE_FIRMWARE(SE_FW);
linux-7.2.3/drivers/crypto/cavium/nitrox/nitrox_mbx.c
+217
View File
@@ -0,0 +1,217 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/bitmap.h>
#include <linux/workqueue.h>
#include "nitrox_csr.h"
#include "nitrox_hal.h"
#include "nitrox_dev.h"
#include "nitrox_mbx.h"
#define RING_TO_VFNO(_x, _y) ((_x) / (_y))
/*
* mbx_msg_type - Mailbox message types
*/
enum mbx_msg_type {
MBX_MSG_TYPE_NOP,
MBX_MSG_TYPE_REQ,
MBX_MSG_TYPE_ACK,
MBX_MSG_TYPE_NACK,
};
/*
* mbx_msg_opcode - Mailbox message opcodes
*/
enum mbx_msg_opcode {
MSG_OP_VF_MODE = 1,
MSG_OP_VF_UP,
MSG_OP_VF_DOWN,
MSG_OP_CHIPID_VFID,
MSG_OP_MCODE_INFO = 11,
};
struct pf2vf_work {
struct nitrox_vfdev *vfdev;
struct nitrox_device *ndev;
struct work_struct pf2vf_resp;
};
static inline u64 pf2vf_read_mbox(struct nitrox_device *ndev, int ring)
{
u64 reg_addr;
reg_addr = NPS_PKT_MBOX_VF_PF_PFDATAX(ring);
return nitrox_read_csr(ndev, reg_addr);
}
static inline void pf2vf_write_mbox(struct nitrox_device *ndev, u64 value,
int ring)
{
u64 reg_addr;
reg_addr = NPS_PKT_MBOX_PF_VF_PFDATAX(ring);
nitrox_write_csr(ndev, reg_addr, value);
}
static void pf2vf_send_response(struct nitrox_device *ndev,
struct nitrox_vfdev *vfdev)
{
union mbox_msg msg;
msg.value = vfdev->msg.value;
switch (vfdev->msg.opcode) {
case MSG_OP_VF_MODE:
msg.data = ndev->mode;
break;
case MSG_OP_VF_UP:
vfdev->nr_queues = vfdev->msg.data;
atomic_set(&vfdev->state, __NDEV_READY);
break;
case MSG_OP_CHIPID_VFID:
msg.id.chipid = ndev->idx;
msg.id.vfid = vfdev->vfno;
break;
case MSG_OP_VF_DOWN:
vfdev->nr_queues = 0;
atomic_set(&vfdev->state, __NDEV_NOT_READY);
break;
case MSG_OP_MCODE_INFO:
msg.data = 0;
msg.mcode_info.count = 2;
msg.mcode_info.info = MCODE_TYPE_SE_SSL | (MCODE_TYPE_AE << 5);
msg.mcode_info.next_se_grp = 1;
msg.mcode_info.next_ae_grp = 1;
break;
default:
msg.type = MBX_MSG_TYPE_NOP;
break;
}
if (msg.type == MBX_MSG_TYPE_NOP)
return;
/* send ACK to VF */
msg.type = MBX_MSG_TYPE_ACK;
pf2vf_write_mbox(ndev, msg.value, vfdev->ring);
vfdev->msg.value = 0;
atomic64_inc(&vfdev->mbx_resp);
}
static void pf2vf_resp_handler(struct work_struct *work)
{
struct pf2vf_work *pf2vf_resp = container_of(work, struct pf2vf_work,
pf2vf_resp);
struct nitrox_vfdev *vfdev = pf2vf_resp->vfdev;
struct nitrox_device *ndev = pf2vf_resp->ndev;
switch (vfdev->msg.type) {
case MBX_MSG_TYPE_REQ:
/* process the request from VF */
pf2vf_send_response(ndev, vfdev);
break;
case MBX_MSG_TYPE_ACK:
case MBX_MSG_TYPE_NACK:
break;
}
kfree(pf2vf_resp);
}
void nitrox_pf2vf_mbox_handler(struct nitrox_device *ndev)
{
DECLARE_BITMAP(csr, BITS_PER_TYPE(u64));
struct nitrox_vfdev *vfdev;
struct pf2vf_work *pfwork;
u64 value, reg_addr;
u32 i;
int vfno;
/* loop for VF(0..63) */
reg_addr = NPS_PKT_MBOX_INT_LO;
value = nitrox_read_csr(ndev, reg_addr);
bitmap_from_u64(csr, value);
for_each_set_bit(i, csr, BITS_PER_TYPE(csr)) {
/* get the vfno from ring */
vfno = RING_TO_VFNO(i, ndev->iov.max_vf_queues);
vfdev = ndev->iov.vfdev + vfno;
vfdev->ring = i;
/* fill the vf mailbox data */
vfdev->msg.value = pf2vf_read_mbox(ndev, vfdev->ring);
pfwork = kzalloc_obj(*pfwork, GFP_ATOMIC);
if (!pfwork)
continue;
pfwork->vfdev = vfdev;
pfwork->ndev = ndev;
INIT_WORK(&pfwork->pf2vf_resp, pf2vf_resp_handler);
queue_work(ndev->iov.pf2vf_wq, &pfwork->pf2vf_resp);
/* clear the corresponding vf bit */
nitrox_write_csr(ndev, reg_addr, BIT_ULL(i));
}
/* loop for VF(64..127) */
reg_addr = NPS_PKT_MBOX_INT_HI;
value = nitrox_read_csr(ndev, reg_addr);
bitmap_from_u64(csr, value);
for_each_set_bit(i, csr, BITS_PER_TYPE(csr)) {
/* get the vfno from ring */
vfno = RING_TO_VFNO(i + 64, ndev->iov.max_vf_queues);
vfdev = ndev->iov.vfdev + vfno;
vfdev->ring = (i + 64);
/* fill the vf mailbox data */
vfdev->msg.value = pf2vf_read_mbox(ndev, vfdev->ring);
pfwork = kzalloc_obj(*pfwork, GFP_ATOMIC);
if (!pfwork)
continue;
pfwork->vfdev = vfdev;
pfwork->ndev = ndev;
INIT_WORK(&pfwork->pf2vf_resp, pf2vf_resp_handler);
queue_work(ndev->iov.pf2vf_wq, &pfwork->pf2vf_resp);
/* clear the corresponding vf bit */
nitrox_write_csr(ndev, reg_addr, BIT_ULL(i));
}
}
int nitrox_mbox_init(struct nitrox_device *ndev)
{
struct nitrox_vfdev *vfdev;
int i;
ndev->iov.vfdev = kzalloc_objs(struct nitrox_vfdev, ndev->iov.num_vfs);
if (!ndev->iov.vfdev)
return -ENOMEM;
for (i = 0; i < ndev->iov.num_vfs; i++) {
vfdev = ndev->iov.vfdev + i;
vfdev->vfno = i;
}
/* allocate pf2vf response workqueue */
ndev->iov.pf2vf_wq = alloc_workqueue("nitrox_pf2vf", WQ_PERCPU, 0);
if (!ndev->iov.pf2vf_wq) {
kfree(ndev->iov.vfdev);
ndev->iov.vfdev = NULL;
return -ENOMEM;
}
/* enable pf2vf mailbox interrupts */
enable_pf2vf_mbox_interrupts(ndev);
return 0;
}
void nitrox_mbox_cleanup(struct nitrox_device *ndev)
{
/* disable pf2vf mailbox interrupts */
disable_pf2vf_mbox_interrupts(ndev);
/* destroy workqueue */
if (ndev->iov.pf2vf_wq)
destroy_workqueue(ndev->iov.pf2vf_wq);
kfree(ndev->iov.vfdev);
ndev->iov.pf2vf_wq = NULL;
ndev->iov.vfdev = NULL;
}
linux-7.2.3/drivers/crypto/cavium/nitrox/nitrox_mbx.h
+9
View File
@@ -0,0 +1,9 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __NITROX_MBX_H
#define __NITROX_MBX_H
int nitrox_mbox_init(struct nitrox_device *ndev);
void nitrox_mbox_cleanup(struct nitrox_device *ndev);
void nitrox_pf2vf_mbox_handler(struct nitrox_device *ndev);
#endif /* __NITROX_MBX_H */
linux-7.2.3/drivers/crypto/cavium/nitrox/nitrox_req.h
+728
View File
@@ -0,0 +1,728 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __NITROX_REQ_H
#define __NITROX_REQ_H
#include <linux/dma-mapping.h>
#include <crypto/aes.h>
#include "nitrox_dev.h"
#define PENDING_SIG 0xFFFFFFFFFFFFFFFFUL
#define PRIO 4001
typedef void (*sereq_completion_t)(void *req, int err);
/**
* struct gphdr - General purpose Header
* @param0: first parameter.
* @param1: second parameter.
* @param2: third parameter.
* @param3: fourth parameter.
*
* Params tell the iv and enc/dec data offsets.
*/
struct gphdr {
__be16 param0;
__be16 param1;
__be16 param2;
__be16 param3;
};
/**
* struct se_req_ctrl - SE request information.
* @arg: Minor number of the opcode
* @ctxc: Context control.
* @unca: Uncertainity enabled.
* @info: Additional information for SE cores.
* @ctxl: Context length in bytes.
* @uddl: User defined data length
*/
union se_req_ctrl {
u64 value;
struct {
u64 raz : 22;
u64 arg : 8;
u64 ctxc : 2;
u64 unca : 1;
u64 info : 3;
u64 unc : 8;
u64 ctxl : 12;
u64 uddl : 8;
} s;
};
#define MAX_IV_LEN 16
/**
* struct se_crypto_request - SE crypto request structure.
* @opcode: Request opcode (enc/dec)
* @flags: flags from crypto subsystem
* @ctx_handle: Crypto context handle.
* @gph: GP Header
* @ctrl: Request Information.
* @orh: ORH address
* @comp: completion address
* @src: Input sglist
* @dst: Output sglist
*/
struct se_crypto_request {
u8 opcode;
gfp_t gfp;
u32 flags;
u64 ctx_handle;
struct gphdr gph;
union se_req_ctrl ctrl;
u64 *orh;
u64 *comp;
struct scatterlist *src;
struct scatterlist *dst;
};
/* Crypto opcodes */
#define FLEXI_CRYPTO_ENCRYPT_HMAC 0x33
#define ENCRYPT 0
#define DECRYPT 1
/* IV from context */
#define IV_FROM_CTX 0
/* IV from Input data */
#define IV_FROM_DPTR 1
/**
* cipher opcodes for firmware
*/
enum flexi_cipher {
CIPHER_NULL = 0,
CIPHER_3DES_CBC,
CIPHER_3DES_ECB,
CIPHER_AES_CBC,
CIPHER_AES_ECB,
CIPHER_AES_CFB,
CIPHER_AES_CTR,
CIPHER_AES_GCM,
CIPHER_AES_XTS,
CIPHER_AES_CCM,
CIPHER_AES_CBC_CTS,
CIPHER_AES_ECB_CTS,
CIPHER_INVALID
};
enum flexi_auth {
AUTH_NULL = 0,
AUTH_MD5,
AUTH_SHA1,
AUTH_SHA2_SHA224,
AUTH_SHA2_SHA256,
AUTH_SHA2_SHA384,
AUTH_SHA2_SHA512,
AUTH_GMAC,
AUTH_INVALID
};
/**
* struct crypto_keys - Crypto keys
* @key: Encryption key or KEY1 for AES-XTS
* @iv: Encryption IV or Tweak for AES-XTS
*/
struct crypto_keys {
union {
u8 key[AES_MAX_KEY_SIZE];
u8 key1[AES_MAX_KEY_SIZE];
} u;
u8 iv[AES_BLOCK_SIZE];
};
/**
* struct auth_keys - Authentication keys
* @ipad: IPAD or KEY2 for AES-XTS
* @opad: OPAD or AUTH KEY if auth_input_type = 1
*/
struct auth_keys {
union {
u8 ipad[64];
u8 key2[64];
} u;
u8 opad[64];
};
union fc_ctx_flags {
__be64 f;
u64 fu;
struct {
#if defined(__BIG_ENDIAN_BITFIELD)
u64 cipher_type : 4;
u64 reserved_59 : 1;
u64 aes_keylen : 2;
u64 iv_source : 1;
u64 hash_type : 4;
u64 reserved_49_51 : 3;
u64 auth_input_type: 1;
u64 mac_len : 8;
u64 reserved_0_39 : 40;
#else
u64 reserved_0_39 : 40;
u64 mac_len : 8;
u64 auth_input_type: 1;
u64 reserved_49_51 : 3;
u64 hash_type : 4;
u64 iv_source : 1;
u64 aes_keylen : 2;
u64 reserved_59 : 1;
u64 cipher_type : 4;
#endif
} w0;
};
/**
* struct flexi_crypto_context - Crypto context
* @cipher_type: Encryption cipher type
* @aes_keylen: AES key length
* @iv_source: Encryption IV source
* @hash_type: Authentication type
* @auth_input_type: Authentication input type
* 1 - Authentication IV and KEY, microcode calculates OPAD/IPAD
* 0 - Authentication OPAD/IPAD
* @mac_len: mac length
* @crypto: Crypto keys
* @auth: Authentication keys
*/
struct flexi_crypto_context {
union fc_ctx_flags flags;
struct crypto_keys crypto;
struct auth_keys auth;
};
struct crypto_ctx_hdr {
struct dma_pool *pool;
dma_addr_t dma;
void *vaddr;
};
struct nitrox_crypto_ctx {
struct nitrox_device *ndev;
union {
u64 ctx_handle;
struct flexi_crypto_context *fctx;
} u;
struct crypto_ctx_hdr *chdr;
sereq_completion_t callback;
};
struct nitrox_kcrypt_request {
struct se_crypto_request creq;
u8 *src;
u8 *dst;
u8 *iv_out;
};
/**
* struct nitrox_aead_rctx - AEAD request context
* @nkreq: Base request context
* @cryptlen: Encryption/Decryption data length
* @assoclen: AAD length
* @srclen: Input buffer length
* @dstlen: Output buffer length
* @iv: IV data
* @ivsize: IV data length
* @flags: AEAD req flags
* @ctx_handle: Device context handle
* @src: Source sglist
* @dst: Destination sglist
* @ctrl_arg: Identifies the request type (ENCRYPT/DECRYPT)
*/
struct nitrox_aead_rctx {
struct nitrox_kcrypt_request nkreq;
unsigned int cryptlen;
unsigned int assoclen;
unsigned int srclen;
unsigned int dstlen;
u8 *iv;
int ivsize;
u32 flags;
u64 ctx_handle;
struct scatterlist *src;
struct scatterlist *dst;
u8 ctrl_arg;
};
/**
* struct nitrox_rfc4106_rctx - rfc4106 cipher request context
* @base: AEAD request context
* @src: Source sglist
* @dst: Destination sglist
* @assoc: AAD
*/
struct nitrox_rfc4106_rctx {
struct nitrox_aead_rctx base;
struct scatterlist src[3];
struct scatterlist dst[3];
u8 assoc[20];
};
/**
* struct pkt_instr_hdr - Packet Instruction Header
* @g: Gather used
* When [G] is set and [GSZ] != 0, the instruction is
* indirect gather instruction.
* When [G] is set and [GSZ] = 0, the instruction is
* direct gather instruction.
* @gsz: Number of pointers in the indirect gather list
* @ihi: When set hardware duplicates the 1st 8 bytes of pkt_instr_hdr
* and adds them to the packet after the pkt_instr_hdr but before any UDD
* @ssz: Not used by the input hardware. But can become slc_store_int[SSZ]
* when [IHI] is set.
* @fsz: The number of front data bytes directly included in the
* PCIe instruction.
* @tlen: The length of the input packet in bytes, include:
* - 16B pkt_hdr
* - Inline context bytes if any,
* - UDD if any,
* - packet payload bytes
*/
union pkt_instr_hdr {
__be64 bev;
u64 value;
struct {
#if defined(__BIG_ENDIAN_BITFIELD)
u64 raz_48_63 : 16;
u64 g : 1;
u64 gsz : 7;
u64 ihi : 1;
u64 ssz : 7;
u64 raz_30_31 : 2;
u64 fsz : 6;
u64 raz_16_23 : 8;
u64 tlen : 16;
#else
u64 tlen : 16;
u64 raz_16_23 : 8;
u64 fsz : 6;
u64 raz_30_31 : 2;
u64 ssz : 7;
u64 ihi : 1;
u64 gsz : 7;
u64 g : 1;
u64 raz_48_63 : 16;
#endif
} s;
};
/**
* struct pkt_hdr - Packet Input Header
* @opcode: Request opcode (Major)
* @arg: Request opcode (Minor)
* @ctxc: Context control.
* @unca: When set [UNC] is the uncertainty count for an input packet.
* The hardware uses uncertainty counts to predict
* output buffer use and avoid deadlock.
* @info: Not used by input hardware. Available for use
* during SE processing.
* @destport: The expected destination port/ring/channel for the packet.
* @unc: Uncertainty count for an input packet.
* @grp: SE group that will process the input packet.
* @ctxl: Context Length in 64-bit words.
* @uddl: User-defined data (UDD) length in bytes.
* @ctxp: Context pointer. CTXP<63,2:0> must be zero in all cases.
*/
union pkt_hdr {
__be64 bev[2];
u64 value[2];
struct {
#if defined(__BIG_ENDIAN_BITFIELD)
u64 opcode : 8;
u64 arg : 8;
u64 ctxc : 2;
u64 unca : 1;
u64 raz_44 : 1;
u64 info : 3;
u64 destport : 9;
u64 unc : 8;
u64 raz_19_23 : 5;
u64 grp : 3;
u64 raz_15 : 1;
u64 ctxl : 7;
u64 uddl : 8;
#else
u64 uddl : 8;
u64 ctxl : 7;
u64 raz_15 : 1;
u64 grp : 3;
u64 raz_19_23 : 5;
u64 unc : 8;
u64 destport : 9;
u64 info : 3;
u64 raz_44 : 1;
u64 unca : 1;
u64 ctxc : 2;
u64 arg : 8;
u64 opcode : 8;
#endif
__be64 ctxp;
} s;
};
/**
* struct slc_store_info - Solicited Paceket Output Store Information.
* @ssz: The number of scatterlist pointers for the solicited output port
* packet.
* @rptr: The result pointer for the solicited output port packet.
* If [SSZ]=0, [RPTR] must point directly to a buffer on the remote
* host that is large enough to hold the entire output packet.
* If [SSZ]!=0, [RPTR] must point to an array of ([SSZ]+3)/4
* sglist components at [RPTR] on the remote host.
*/
union slc_store_info {
__be64 bev[2];
u64 value[2];
struct {
#if defined(__BIG_ENDIAN_BITFIELD)
u64 raz_39_63 : 25;
u64 ssz : 7;
u64 raz_0_31 : 32;
#else
u64 raz_0_31 : 32;
u64 ssz : 7;
u64 raz_39_63 : 25;
#endif
__be64 rptr;
} s;
};
/**
* struct nps_pkt_instr - NPS Packet Instruction of SE cores.
* @dptr0 : Input pointer points to buffer in remote host.
* @ih: Packet Instruction Header (8 bytes)
* @irh: Packet Input Header (16 bytes)
* @slc: Solicited Packet Output Store Information (16 bytes)
* @fdata: Front data
*
* 64-Byte Instruction Format
*/
struct nps_pkt_instr {
__be64 dptr0;
union pkt_instr_hdr ih;
union pkt_hdr irh;
union slc_store_info slc;
u64 fdata[2];
};
/**
* struct aqmq_command_s - The 32 byte command for AE processing.
* @opcode: Request opcode
* @param1: Request control parameter 1
* @param2: Request control parameter 2
* @dlen: Input length
* @dptr: Input pointer points to buffer in remote host
* @rptr: Result pointer points to buffer in remote host
* @grp: AQM Group (0..7)
* @cptr: Context pointer
*/
struct aqmq_command_s {
__be16 opcode;
__be16 param1;
__be16 param2;
__be16 dlen;
__be64 dptr;
__be64 rptr;
union {
__be64 word3;
#if defined(__BIG_ENDIAN_BITFIELD)
u64 grp : 3;
u64 cptr : 61;
#else
u64 cptr : 61;
u64 grp : 3;
#endif
};
};
/**
* struct ctx_hdr - Book keeping data about the crypto context
* @pool: Pool used to allocate crypto context
* @dma: Base DMA address of the crypto context
* @ctx_dma: Actual usable crypto context for NITROX
*/
struct ctx_hdr {
struct dma_pool *pool;
dma_addr_t dma;
dma_addr_t ctx_dma;
};
/*
* struct sglist_component - SG list component format
* @len0: The number of bytes at [PTR0] on the remote host.
* @len1: The number of bytes at [PTR1] on the remote host.
* @len2: The number of bytes at [PTR2] on the remote host.
* @len3: The number of bytes at [PTR3] on the remote host.
* @dma0: First pointer point to buffer in remote host.
* @dma1: Second pointer point to buffer in remote host.
* @dma2: Third pointer point to buffer in remote host.
* @dma3: Fourth pointer point to buffer in remote host.
*/
struct nitrox_sgcomp {
__be16 len[4];
__be64 dma[4];
};
/*
* strutct nitrox_sgtable - SG list information
* @sgmap_cnt: Number of buffers mapped
* @total_bytes: Total bytes in sglist.
* @sgcomp_len: Total sglist components length.
* @sgcomp_dma: DMA address of sglist component.
* @sg: crypto request buffer.
* @sgcomp: sglist component for NITROX.
*/
struct nitrox_sgtable {
u8 sgmap_cnt;
u16 total_bytes;
u32 sgcomp_len;
dma_addr_t sgcomp_dma;
struct scatterlist *sg;
struct nitrox_sgcomp *sgcomp;
};
/* Response Header Length */
#define ORH_HLEN 8
/* Completion bytes Length */
#define COMP_HLEN 8
struct resp_hdr {
u64 *orh;
u64 *completion;
};
typedef void (*completion_t)(void *arg, int err);
/**
* struct nitrox_softreq - Represents the NIROX Request.
* @response: response list entry
* @backlog: Backlog list entry
* @ndev: Device used to submit the request
* @cmdq: Command queue for submission
* @resp: Response headers
* @instr: 64B instruction
* @in: SG table for input
* @out SG table for output
* @tstamp: Request submitted time in jiffies
* @callback: callback after request completion/timeout
* @cb_arg: callback argument
*/
struct nitrox_softreq {
struct list_head response;
struct list_head backlog;
u32 flags;
gfp_t gfp;
atomic_t status;
struct nitrox_device *ndev;
struct nitrox_cmdq *cmdq;
struct nps_pkt_instr instr;
struct resp_hdr resp;
struct nitrox_sgtable in;
struct nitrox_sgtable out;
unsigned long tstamp;
completion_t callback;
void *cb_arg;
};
static inline int flexi_aes_keylen(int keylen)
{
int aes_keylen;
switch (keylen) {
case AES_KEYSIZE_128:
aes_keylen = 1;
break;
case AES_KEYSIZE_192:
aes_keylen = 2;
break;
case AES_KEYSIZE_256:
aes_keylen = 3;
break;
default:
aes_keylen = -EINVAL;
break;
}
return aes_keylen;
}
static inline void *alloc_req_buf(int nents, int extralen, gfp_t gfp)
{
size_t size;
size = sizeof(struct scatterlist) * nents;
size += extralen;
return kzalloc(size, gfp);
}
/**
* create_single_sg - Point SG entry to the data
* @sg: Destination SG list
* @buf: Data
* @buflen: Data length
*
* Returns next free entry in the destination SG list
**/
static inline struct scatterlist *create_single_sg(struct scatterlist *sg,
void *buf, int buflen)
{
sg_set_buf(sg, buf, buflen);
sg++;
return sg;
}
/**
* create_multi_sg - Create multiple sg entries with buflen data length from
* source sglist
* @to_sg: Destination SG list
* @from_sg: Source SG list
* @buflen: Data length
*
* Returns next free entry in the destination SG list
**/
static inline struct scatterlist *create_multi_sg(struct scatterlist *to_sg,
struct scatterlist *from_sg,
int buflen)
{
struct scatterlist *sg = to_sg;
unsigned int sglen;
for (; buflen && from_sg; buflen -= sglen) {
sglen = from_sg->length;
if (sglen > buflen)
sglen = buflen;
sg_set_buf(sg, sg_virt(from_sg), sglen);
from_sg = sg_next(from_sg);
sg++;
}
return sg;
}
static inline void set_orh_value(u64 *orh)
{
WRITE_ONCE(*orh, PENDING_SIG);
}
static inline void set_comp_value(u64 *comp)
{
WRITE_ONCE(*comp, PENDING_SIG);
}
static inline int alloc_src_req_buf(struct nitrox_kcrypt_request *nkreq,
int nents, int ivsize)
{
struct se_crypto_request *creq = &nkreq->creq;
nkreq->src = alloc_req_buf(nents, ivsize, creq->gfp);
if (!nkreq->src)
return -ENOMEM;
return 0;
}
static inline void nitrox_creq_copy_iv(char *dst, char *src, int size)
{
memcpy(dst, src, size);
}
static inline struct scatterlist *nitrox_creq_src_sg(char *iv, int ivsize)
{
return (struct scatterlist *)(iv + ivsize);
}
static inline void nitrox_creq_set_src_sg(struct nitrox_kcrypt_request *nkreq,
int nents, int ivsize,
struct scatterlist *src, int buflen)
{
char *iv = nkreq->src;
struct scatterlist *sg;
struct se_crypto_request *creq = &nkreq->creq;
creq->src = nitrox_creq_src_sg(iv, ivsize);
sg = creq->src;
sg_init_table(sg, nents);
/* Input format:
* +----+----------------+
* | IV | SRC sg entries |
* +----+----------------+
*/
/* IV */
sg = create_single_sg(sg, iv, ivsize);
/* SRC entries */
create_multi_sg(sg, src, buflen);
}
static inline int alloc_dst_req_buf(struct nitrox_kcrypt_request *nkreq,
int nents)
{
int extralen = ORH_HLEN + COMP_HLEN;
struct se_crypto_request *creq = &nkreq->creq;
nkreq->dst = alloc_req_buf(nents, extralen, creq->gfp);
if (!nkreq->dst)
return -ENOMEM;
return 0;
}
static inline void nitrox_creq_set_orh(struct nitrox_kcrypt_request *nkreq)
{
struct se_crypto_request *creq = &nkreq->creq;
creq->orh = (u64 *)(nkreq->dst);
set_orh_value(creq->orh);
}
static inline void nitrox_creq_set_comp(struct nitrox_kcrypt_request *nkreq)
{
struct se_crypto_request *creq = &nkreq->creq;
creq->comp = (u64 *)(nkreq->dst + ORH_HLEN);
set_comp_value(creq->comp);
}
static inline struct scatterlist *nitrox_creq_dst_sg(char *dst)
{
return (struct scatterlist *)(dst + ORH_HLEN + COMP_HLEN);
}
static inline void nitrox_creq_set_dst_sg(struct nitrox_kcrypt_request *nkreq,
int nents, int ivsize,
struct scatterlist *dst, int buflen)
{
struct se_crypto_request *creq = &nkreq->creq;
struct scatterlist *sg;
char *iv = nkreq->src;
creq->dst = nitrox_creq_dst_sg(nkreq->dst);
sg = creq->dst;
sg_init_table(sg, nents);
/* Output format:
* +-----+----+----------------+-----------------+
* | ORH | IV | DST sg entries | COMPLETION Bytes|
* +-----+----+----------------+-----------------+
*/
/* ORH */
sg = create_single_sg(sg, creq->orh, ORH_HLEN);
/* IV */
sg = create_single_sg(sg, iv, ivsize);
/* DST entries */
sg = create_multi_sg(sg, dst, buflen);
/* COMPLETION Bytes */
create_single_sg(sg, creq->comp, COMP_HLEN);
}
#endif /* __NITROX_REQ_H */
linux-7.2.3/drivers/crypto/cavium/nitrox/nitrox_reqmgr.c
+607
View File
@@ -0,0 +1,607 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/gfp.h>
#include <linux/workqueue.h>
#include <crypto/internal/skcipher.h>
#include "nitrox_common.h"
#include "nitrox_dev.h"
#include "nitrox_req.h"
#include "nitrox_csr.h"
/* SLC_STORE_INFO */
#define MIN_UDD_LEN 16
/* PKT_IN_HDR + SLC_STORE_INFO */
#define FDATA_SIZE 32
/* Base destination port for the solicited requests */
#define SOLICIT_BASE_DPORT 256
#define REQ_NOT_POSTED 1
#define REQ_BACKLOG 2
#define REQ_POSTED 3
/*
* Response codes from SE microcode
* 0x00 - Success
* Completion with no error
* 0x43 - ERR_GC_DATA_LEN_INVALID
* Invalid Data length if Encryption Data length is
* less than 16 bytes for AES-XTS and AES-CTS.
* 0x45 - ERR_GC_CTX_LEN_INVALID
* Invalid context length: CTXL != 23 words.
* 0x4F - ERR_GC_DOCSIS_CIPHER_INVALID
* DOCSIS support is enabled with other than
* AES/DES-CBC mode encryption.
* 0x50 - ERR_GC_DOCSIS_OFFSET_INVALID
* Authentication offset is other than 0 with
* Encryption IV source = 0.
* Authentication offset is other than 8 (DES)/16 (AES)
* with Encryption IV source = 1
* 0x51 - ERR_GC_CRC32_INVALID_SELECTION
* CRC32 is enabled for other than DOCSIS encryption.
* 0x52 - ERR_GC_AES_CCM_FLAG_INVALID
* Invalid flag options in AES-CCM IV.
*/
static inline int incr_index(int index, int count, int max)
{
if ((index + count) >= max)
index = index + count - max;
else
index += count;
return index;
}
static void softreq_unmap_sgbufs(struct nitrox_softreq *sr)
{
struct nitrox_device *ndev = sr->ndev;
struct device *dev = DEV(ndev);
dma_unmap_sg(dev, sr->in.sg, sg_nents(sr->in.sg),
DMA_BIDIRECTIONAL);
dma_unmap_single(dev, sr->in.sgcomp_dma, sr->in.sgcomp_len,
DMA_TO_DEVICE);
kfree(sr->in.sgcomp);
sr->in.sg = NULL;
sr->in.sgmap_cnt = 0;
dma_unmap_sg(dev, sr->out.sg, sg_nents(sr->out.sg),
DMA_BIDIRECTIONAL);
dma_unmap_single(dev, sr->out.sgcomp_dma, sr->out.sgcomp_len,
DMA_TO_DEVICE);
kfree(sr->out.sgcomp);
sr->out.sg = NULL;
sr->out.sgmap_cnt = 0;
}
static void softreq_destroy(struct nitrox_softreq *sr)
{
softreq_unmap_sgbufs(sr);
kfree(sr);
}
/**
* create_sg_component - create SG componets for N5 device.
* @sr: Request structure
* @sgtbl: SG table
* @map_nents: number of dma mapped entries
*
* Component structure
*
* 63 48 47 32 31 16 15 0
* --------------------------------------
* | LEN0 | LEN1 | LEN2 | LEN3 |
* |-------------------------------------
* | PTR0 |
* --------------------------------------
* | PTR1 |
* --------------------------------------
* | PTR2 |
* --------------------------------------
* | PTR3 |
* --------------------------------------
*
* Returns 0 if success or a negative errno code on error.
*/
static int create_sg_component(struct nitrox_softreq *sr,
struct nitrox_sgtable *sgtbl, int map_nents)
{
struct nitrox_device *ndev = sr->ndev;
struct nitrox_sgcomp *sgcomp;
struct scatterlist *sg;
dma_addr_t dma;
size_t sz_comp;
int i, j, nr_sgcomp;
nr_sgcomp = roundup(map_nents, 4) / 4;
/* each component holds 4 dma pointers */
sz_comp = nr_sgcomp * sizeof(*sgcomp);
sgcomp = kzalloc(sz_comp, sr->gfp);
if (!sgcomp)
return -ENOMEM;
sgtbl->sgcomp = sgcomp;
sg = sgtbl->sg;
/* populate device sg component */
for (i = 0; i < nr_sgcomp; i++) {
for (j = 0; j < 4 && sg; j++) {
sgcomp[i].len[j] = cpu_to_be16(sg_dma_len(sg));
sgcomp[i].dma[j] = cpu_to_be64(sg_dma_address(sg));
sg = sg_next(sg);
}
}
/* map the device sg component */
dma = dma_map_single(DEV(ndev), sgtbl->sgcomp, sz_comp, DMA_TO_DEVICE);
if (dma_mapping_error(DEV(ndev), dma)) {
kfree(sgtbl->sgcomp);
sgtbl->sgcomp = NULL;
return -ENOMEM;
}
sgtbl->sgcomp_dma = dma;
sgtbl->sgcomp_len = sz_comp;
return 0;
}
/**
* dma_map_inbufs - DMA map input sglist and creates sglist component
* for N5 device.
* @sr: Request structure
* @req: Crypto request structre
*
* Returns 0 if successful or a negative errno code on error.
*/
static int dma_map_inbufs(struct nitrox_softreq *sr,
struct se_crypto_request *req)
{
struct device *dev = DEV(sr->ndev);
struct scatterlist *sg;
int i, nents, ret = 0;
nents = dma_map_sg(dev, req->src, sg_nents(req->src),
DMA_BIDIRECTIONAL);
if (!nents)
return -EINVAL;
for_each_sg(req->src, sg, nents, i)
sr->in.total_bytes += sg_dma_len(sg);
sr->in.sg = req->src;
sr->in.sgmap_cnt = nents;
ret = create_sg_component(sr, &sr->in, sr->in.sgmap_cnt);
if (ret)
goto incomp_err;
return 0;
incomp_err:
dma_unmap_sg(dev, req->src, sg_nents(req->src), DMA_BIDIRECTIONAL);
sr->in.sgmap_cnt = 0;
return ret;
}
static int dma_map_outbufs(struct nitrox_softreq *sr,
struct se_crypto_request *req)
{
struct device *dev = DEV(sr->ndev);
int nents, ret = 0;
nents = dma_map_sg(dev, req->dst, sg_nents(req->dst),
DMA_BIDIRECTIONAL);
if (!nents)
return -EINVAL;
sr->out.sg = req->dst;
sr->out.sgmap_cnt = nents;
ret = create_sg_component(sr, &sr->out, sr->out.sgmap_cnt);
if (ret)
goto outcomp_map_err;
return 0;
outcomp_map_err:
dma_unmap_sg(dev, req->dst, sg_nents(req->dst), DMA_BIDIRECTIONAL);
sr->out.sgmap_cnt = 0;
sr->out.sg = NULL;
return ret;
}
static inline int softreq_map_iobuf(struct nitrox_softreq *sr,
struct se_crypto_request *creq)
{
int ret;
ret = dma_map_inbufs(sr, creq);
if (ret)
return ret;
ret = dma_map_outbufs(sr, creq);
if (ret)
softreq_unmap_sgbufs(sr);
return ret;
}
static inline void backlog_list_add(struct nitrox_softreq *sr,
struct nitrox_cmdq *cmdq)
{
INIT_LIST_HEAD(&sr->backlog);
spin_lock_bh(&cmdq->backlog_qlock);
list_add_tail(&sr->backlog, &cmdq->backlog_head);
atomic_inc(&cmdq->backlog_count);
atomic_set(&sr->status, REQ_BACKLOG);
spin_unlock_bh(&cmdq->backlog_qlock);
}
static inline void response_list_add(struct nitrox_softreq *sr,
struct nitrox_cmdq *cmdq)
{
INIT_LIST_HEAD(&sr->response);
spin_lock_bh(&cmdq->resp_qlock);
list_add_tail(&sr->response, &cmdq->response_head);
spin_unlock_bh(&cmdq->resp_qlock);
}
static inline void response_list_del(struct nitrox_softreq *sr,
struct nitrox_cmdq *cmdq)
{
spin_lock_bh(&cmdq->resp_qlock);
list_del(&sr->response);
spin_unlock_bh(&cmdq->resp_qlock);
}
static struct nitrox_softreq *
get_first_response_entry(struct nitrox_cmdq *cmdq)
{
return list_first_entry_or_null(&cmdq->response_head,
struct nitrox_softreq, response);
}
static inline bool cmdq_full(struct nitrox_cmdq *cmdq, int qlen)
{
if (atomic_inc_return(&cmdq->pending_count) > qlen) {
atomic_dec(&cmdq->pending_count);
/* sync with other cpus */
smp_mb__after_atomic();
return true;
}
/* sync with other cpus */
smp_mb__after_atomic();
return false;
}
/**
* post_se_instr - Post SE instruction to Packet Input ring
* @sr: Request structure
* @cmdq: Command queue structure
*
* Returns 0 if successful or a negative error code,
* if no space in ring.
*/
static void post_se_instr(struct nitrox_softreq *sr,
struct nitrox_cmdq *cmdq)
{
struct nitrox_device *ndev = sr->ndev;
int idx;
u8 *ent;
spin_lock_bh(&cmdq->cmd_qlock);
idx = cmdq->write_idx;
/* copy the instruction */
ent = cmdq->base + (idx * cmdq->instr_size);
memcpy(ent, &sr->instr, cmdq->instr_size);
atomic_set(&sr->status, REQ_POSTED);
response_list_add(sr, cmdq);
sr->tstamp = jiffies;
/* flush the command queue updates */
dma_wmb();
/* Ring doorbell with count 1 */
writeq(1, cmdq->dbell_csr_addr);
cmdq->write_idx = incr_index(idx, 1, ndev->qlen);
spin_unlock_bh(&cmdq->cmd_qlock);
/* increment the posted command count */
atomic64_inc(&ndev->stats.posted);
}
static int post_backlog_cmds(struct nitrox_cmdq *cmdq)
{
struct nitrox_device *ndev = cmdq->ndev;
struct nitrox_softreq *sr, *tmp;
int ret = 0;
if (!atomic_read(&cmdq->backlog_count))
return 0;
spin_lock_bh(&cmdq->backlog_qlock);
list_for_each_entry_safe(sr, tmp, &cmdq->backlog_head, backlog) {
/* submit until space available */
if (unlikely(cmdq_full(cmdq, ndev->qlen))) {
ret = -ENOSPC;
break;
}
/* delete from backlog list */
list_del(&sr->backlog);
atomic_dec(&cmdq->backlog_count);
/* sync with other cpus */
smp_mb__after_atomic();
/* post the command */
post_se_instr(sr, cmdq);
}
spin_unlock_bh(&cmdq->backlog_qlock);
return ret;
}
static int nitrox_enqueue_request(struct nitrox_softreq *sr)
{
struct nitrox_cmdq *cmdq = sr->cmdq;
struct nitrox_device *ndev = sr->ndev;
/* try to post backlog requests */
post_backlog_cmds(cmdq);
if (unlikely(cmdq_full(cmdq, ndev->qlen))) {
if (!(sr->flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
/* increment drop count */
atomic64_inc(&ndev->stats.dropped);
return -ENOSPC;
}
/* add to backlog list */
backlog_list_add(sr, cmdq);
return -EINPROGRESS;
}
post_se_instr(sr, cmdq);
return -EINPROGRESS;
}
/**
* nitrox_process_se_request - Send request to SE core
* @ndev: NITROX device
* @req: Crypto request
* @callback: Completion callback
* @cb_arg: Completion callback arguments
*
* Returns 0 on success, or a negative error code.
*/
int nitrox_process_se_request(struct nitrox_device *ndev,
struct se_crypto_request *req,
completion_t callback,
void *cb_arg)
{
struct nitrox_softreq *sr;
dma_addr_t ctx_handle = 0;
int qno, ret = 0;
if (!nitrox_ready(ndev))
return -ENODEV;
sr = kzalloc_obj(*sr, req->gfp);
if (!sr)
return -ENOMEM;
sr->ndev = ndev;
sr->flags = req->flags;
sr->gfp = req->gfp;
sr->callback = callback;
sr->cb_arg = cb_arg;
atomic_set(&sr->status, REQ_NOT_POSTED);
sr->resp.orh = req->orh;
sr->resp.completion = req->comp;
ret = softreq_map_iobuf(sr, req);
if (ret) {
kfree(sr);
return ret;
}
/* get the context handle */
if (req->ctx_handle) {
struct ctx_hdr *hdr;
u8 *ctx_ptr;
ctx_ptr = (u8 *)(uintptr_t)req->ctx_handle;
hdr = (struct ctx_hdr *)(ctx_ptr - sizeof(struct ctx_hdr));
ctx_handle = hdr->ctx_dma;
}
/* select the queue */
qno = smp_processor_id() % ndev->nr_queues;
sr->cmdq = &ndev->pkt_inq[qno];
/*
* 64-Byte Instruction Format
*
* ----------------------
* | DPTR0 | 8 bytes
* ----------------------
* | PKT_IN_INSTR_HDR | 8 bytes
* ----------------------
* | PKT_IN_HDR | 16 bytes
* ----------------------
* | SLC_INFO | 16 bytes
* ----------------------
* | Front data | 16 bytes
* ----------------------
*/
/* fill the packet instruction */
/* word 0 */
sr->instr.dptr0 = cpu_to_be64(sr->in.sgcomp_dma);
/* word 1 */
sr->instr.ih.value = 0;
sr->instr.ih.s.g = 1;
sr->instr.ih.s.gsz = sr->in.sgmap_cnt;
sr->instr.ih.s.ssz = sr->out.sgmap_cnt;
sr->instr.ih.s.fsz = FDATA_SIZE + sizeof(struct gphdr);
sr->instr.ih.s.tlen = sr->instr.ih.s.fsz + sr->in.total_bytes;
sr->instr.ih.bev = cpu_to_be64(sr->instr.ih.value);
/* word 2 */
sr->instr.irh.value[0] = 0;
sr->instr.irh.s.uddl = MIN_UDD_LEN;
/* context length in 64-bit words */
sr->instr.irh.s.ctxl = (req->ctrl.s.ctxl / 8);
/* offset from solicit base port 256 */
sr->instr.irh.s.destport = SOLICIT_BASE_DPORT + qno;
sr->instr.irh.s.ctxc = req->ctrl.s.ctxc;
sr->instr.irh.s.arg = req->ctrl.s.arg;
sr->instr.irh.s.opcode = req->opcode;
sr->instr.irh.bev[0] = cpu_to_be64(sr->instr.irh.value[0]);
/* word 3 */
sr->instr.irh.s.ctxp = cpu_to_be64(ctx_handle);
/* word 4 */
sr->instr.slc.value[0] = 0;
sr->instr.slc.s.ssz = sr->out.sgmap_cnt;
sr->instr.slc.bev[0] = cpu_to_be64(sr->instr.slc.value[0]);
/* word 5 */
sr->instr.slc.s.rptr = cpu_to_be64(sr->out.sgcomp_dma);
/*
* No conversion for front data,
* It goes into payload
* put GP Header in front data
*/
sr->instr.fdata[0] = *((u64 *)&req->gph);
sr->instr.fdata[1] = 0;
ret = nitrox_enqueue_request(sr);
if (ret == -ENOSPC)
goto send_fail;
return ret;
send_fail:
softreq_destroy(sr);
return ret;
}
static inline int cmd_timeout(unsigned long tstamp, unsigned long timeout)
{
return time_after_eq(jiffies, (tstamp + timeout));
}
void backlog_qflush_work(struct work_struct *work)
{
struct nitrox_cmdq *cmdq;
cmdq = container_of(work, struct nitrox_cmdq, backlog_qflush);
post_backlog_cmds(cmdq);
}
static bool sr_completed(struct nitrox_softreq *sr)
{
u64 orh = READ_ONCE(*sr->resp.orh);
unsigned long timeout = jiffies + msecs_to_jiffies(1);
if ((orh != PENDING_SIG) && (orh & 0xff))
return true;
while (READ_ONCE(*sr->resp.completion) == PENDING_SIG) {
if (time_after(jiffies, timeout)) {
pr_err("comp not done\n");
return false;
}
}
return true;
}
/**
* process_response_list - process completed requests
* @cmdq: Command queue structure
*
* Returns the number of responses processed.
*/
static void process_response_list(struct nitrox_cmdq *cmdq)
{
struct nitrox_device *ndev = cmdq->ndev;
struct nitrox_softreq *sr;
int req_completed = 0, err = 0, budget;
completion_t callback;
void *cb_arg;
/* check all pending requests */
budget = atomic_read(&cmdq->pending_count);
while (req_completed < budget) {
sr = get_first_response_entry(cmdq);
if (!sr)
break;
if (atomic_read(&sr->status) != REQ_POSTED)
break;
/* check orh and completion bytes updates */
if (!sr_completed(sr)) {
/* request not completed, check for timeout */
if (!cmd_timeout(sr->tstamp, ndev->timeout))
break;
dev_err_ratelimited(DEV(ndev),
"Request timeout, orh 0x%016llx\n",
READ_ONCE(*sr->resp.orh));
}
atomic_dec(&cmdq->pending_count);
atomic64_inc(&ndev->stats.completed);
/* sync with other cpus */
smp_mb__after_atomic();
/* remove from response list */
response_list_del(sr, cmdq);
/* ORH error code */
err = READ_ONCE(*sr->resp.orh) & 0xff;
callback = sr->callback;
cb_arg = sr->cb_arg;
softreq_destroy(sr);
if (callback)
callback(cb_arg, err);
req_completed++;
}
}
/*
* pkt_slc_resp_tasklet - post processing of SE responses
*/
void pkt_slc_resp_tasklet(unsigned long data)
{
struct nitrox_q_vector *qvec = (void *)(uintptr_t)(data);
struct nitrox_cmdq *cmdq = qvec->cmdq;
union nps_pkt_slc_cnts slc_cnts;
/* read completion count */
slc_cnts.value = readq(cmdq->compl_cnt_csr_addr);
/* resend the interrupt if more work to do */
slc_cnts.s.resend = 1;
process_response_list(cmdq);
/*
* clear the interrupt with resend bit enabled,
* MSI-X interrupt generates if Completion count > Threshold
*/
writeq(slc_cnts.value, cmdq->compl_cnt_csr_addr);
if (atomic_read(&cmdq->backlog_count))
schedule_work(&cmdq->backlog_qflush);
}
linux-7.2.3/drivers/crypto/cavium/nitrox/nitrox_skcipher.c
+531
View File
@@ -0,0 +1,531 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/crypto.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/printk.h>
#include <crypto/aes.h>
#include <crypto/skcipher.h>
#include <crypto/scatterwalk.h>
#include <crypto/ctr.h>
#include <crypto/internal/des.h>
#include <crypto/xts.h>
#include "nitrox_dev.h"
#include "nitrox_common.h"
#include "nitrox_req.h"
struct nitrox_cipher {
const char *name;
enum flexi_cipher value;
};
/*
* supported cipher list
*/
static const struct nitrox_cipher flexi_cipher_table[] = {
{ "null", CIPHER_NULL },
{ "cbc(des3_ede)", CIPHER_3DES_CBC },
{ "ecb(des3_ede)", CIPHER_3DES_ECB },
{ "cbc(aes)", CIPHER_AES_CBC },
{ "ecb(aes)", CIPHER_AES_ECB },
{ "cfb(aes)", CIPHER_AES_CFB },
{ "rfc3686(ctr(aes))", CIPHER_AES_CTR },
{ "xts(aes)", CIPHER_AES_XTS },
{ "cts(cbc(aes))", CIPHER_AES_CBC_CTS },
{ NULL, CIPHER_INVALID }
};
static enum flexi_cipher flexi_cipher_type(const char *name)
{
const struct nitrox_cipher *cipher = flexi_cipher_table;
while (cipher->name) {
if (!strcmp(cipher->name, name))
break;
cipher++;
}
return cipher->value;
}
static void free_src_sglist(struct skcipher_request *skreq)
{
struct nitrox_kcrypt_request *nkreq = skcipher_request_ctx(skreq);
kfree(nkreq->src);
}
static void free_dst_sglist(struct skcipher_request *skreq)
{
struct nitrox_kcrypt_request *nkreq = skcipher_request_ctx(skreq);
kfree(nkreq->dst);
}
static void nitrox_skcipher_callback(void *arg, int err)
{
struct skcipher_request *skreq = arg;
free_src_sglist(skreq);
free_dst_sglist(skreq);
if (err) {
pr_err_ratelimited("request failed status 0x%0x\n", err);
err = -EINVAL;
}
skcipher_request_complete(skreq, err);
}
static void nitrox_cbc_cipher_callback(void *arg, int err)
{
struct skcipher_request *skreq = arg;
struct nitrox_kcrypt_request *nkreq = skcipher_request_ctx(skreq);
struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(skreq);
int ivsize = crypto_skcipher_ivsize(cipher);
unsigned int start = skreq->cryptlen - ivsize;
if (err) {
nitrox_skcipher_callback(arg, err);
return;
}
if (nkreq->creq.ctrl.s.arg == ENCRYPT) {
scatterwalk_map_and_copy(skreq->iv, skreq->dst, start, ivsize,
0);
} else {
if (skreq->src != skreq->dst) {
scatterwalk_map_and_copy(skreq->iv, skreq->src, start,
ivsize, 0);
} else {
memcpy(skreq->iv, nkreq->iv_out, ivsize);
kfree(nkreq->iv_out);
}
}
nitrox_skcipher_callback(arg, err);
}
static int nitrox_skcipher_init(struct crypto_skcipher *tfm)
{
struct nitrox_crypto_ctx *nctx = crypto_skcipher_ctx(tfm);
struct crypto_ctx_hdr *chdr;
/* get the first device */
nctx->ndev = nitrox_get_first_device();
if (!nctx->ndev)
return -ENODEV;
/* allocate nitrox crypto context */
chdr = crypto_alloc_context(nctx->ndev);
if (!chdr) {
nitrox_put_device(nctx->ndev);
return -ENOMEM;
}
nctx->callback = nitrox_skcipher_callback;
nctx->chdr = chdr;
nctx->u.ctx_handle = (uintptr_t)((u8 *)chdr->vaddr +
sizeof(struct ctx_hdr));
crypto_skcipher_set_reqsize(tfm, crypto_skcipher_reqsize(tfm) +
sizeof(struct nitrox_kcrypt_request));
return 0;
}
static int nitrox_cbc_init(struct crypto_skcipher *tfm)
{
int err;
struct nitrox_crypto_ctx *nctx = crypto_skcipher_ctx(tfm);
err = nitrox_skcipher_init(tfm);
if (err)
return err;
nctx->callback = nitrox_cbc_cipher_callback;
return 0;
}
static void nitrox_skcipher_exit(struct crypto_skcipher *tfm)
{
struct nitrox_crypto_ctx *nctx = crypto_skcipher_ctx(tfm);
/* free the nitrox crypto context */
if (nctx->u.ctx_handle) {
struct flexi_crypto_context *fctx = nctx->u.fctx;
memzero_explicit(&fctx->crypto, sizeof(struct crypto_keys));
memzero_explicit(&fctx->auth, sizeof(struct auth_keys));
crypto_free_context((void *)nctx->chdr);
}
nitrox_put_device(nctx->ndev);
nctx->u.ctx_handle = 0;
nctx->ndev = NULL;
}
static inline int nitrox_skcipher_setkey(struct crypto_skcipher *cipher,
int aes_keylen, const u8 *key,
unsigned int keylen)
{
struct crypto_tfm *tfm = crypto_skcipher_tfm(cipher);
struct nitrox_crypto_ctx *nctx = crypto_tfm_ctx(tfm);
struct flexi_crypto_context *fctx;
union fc_ctx_flags *flags;
enum flexi_cipher cipher_type;
const char *name;
name = crypto_tfm_alg_name(tfm);
cipher_type = flexi_cipher_type(name);
if (unlikely(cipher_type == CIPHER_INVALID)) {
pr_err("unsupported cipher: %s\n", name);
return -EINVAL;
}
/* fill crypto context */
fctx = nctx->u.fctx;
flags = &fctx->flags;
flags->f = 0;
flags->w0.cipher_type = cipher_type;
flags->w0.aes_keylen = aes_keylen;
flags->w0.iv_source = IV_FROM_DPTR;
flags->f = cpu_to_be64(*(u64 *)&flags->w0);
/* copy the key to context */
memcpy(fctx->crypto.u.key, key, keylen);
return 0;
}
static int nitrox_aes_setkey(struct crypto_skcipher *cipher, const u8 *key,
unsigned int keylen)
{
int aes_keylen;
aes_keylen = flexi_aes_keylen(keylen);
if (aes_keylen < 0)
return -EINVAL;
return nitrox_skcipher_setkey(cipher, aes_keylen, key, keylen);
}
static int alloc_src_sglist(struct skcipher_request *skreq, int ivsize)
{
struct nitrox_kcrypt_request *nkreq = skcipher_request_ctx(skreq);
int nents = sg_nents(skreq->src) + 1;
int ret;
/* Allocate buffer to hold IV and input scatterlist array */
ret = alloc_src_req_buf(nkreq, nents, ivsize);
if (ret)
return ret;
nitrox_creq_copy_iv(nkreq->src, skreq->iv, ivsize);
nitrox_creq_set_src_sg(nkreq, nents, ivsize, skreq->src,
skreq->cryptlen);
return 0;
}
static int alloc_dst_sglist(struct skcipher_request *skreq, int ivsize)
{
struct nitrox_kcrypt_request *nkreq = skcipher_request_ctx(skreq);
int nents = sg_nents(skreq->dst) + 3;
int ret;
/* Allocate buffer to hold ORH, COMPLETION and output scatterlist
* array
*/
ret = alloc_dst_req_buf(nkreq, nents);
if (ret)
return ret;
nitrox_creq_set_orh(nkreq);
nitrox_creq_set_comp(nkreq);
nitrox_creq_set_dst_sg(nkreq, nents, ivsize, skreq->dst,
skreq->cryptlen);
return 0;
}
static int nitrox_skcipher_crypt(struct skcipher_request *skreq, bool enc)
{
struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(skreq);
struct nitrox_crypto_ctx *nctx = crypto_skcipher_ctx(cipher);
struct nitrox_kcrypt_request *nkreq = skcipher_request_ctx(skreq);
int ivsize = crypto_skcipher_ivsize(cipher);
struct se_crypto_request *creq;
int ret;
creq = &nkreq->creq;
creq->flags = skreq->base.flags;
creq->gfp = (skreq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
GFP_KERNEL : GFP_ATOMIC;
/* fill the request */
creq->ctrl.value = 0;
creq->opcode = FLEXI_CRYPTO_ENCRYPT_HMAC;
creq->ctrl.s.arg = (enc ? ENCRYPT : DECRYPT);
/* param0: length of the data to be encrypted */
creq->gph.param0 = cpu_to_be16(skreq->cryptlen);
creq->gph.param1 = 0;
/* param2: encryption data offset */
creq->gph.param2 = cpu_to_be16(ivsize);
creq->gph.param3 = 0;
creq->ctx_handle = nctx->u.ctx_handle;
creq->ctrl.s.ctxl = sizeof(struct flexi_crypto_context);
ret = alloc_src_sglist(skreq, ivsize);
if (ret)
return ret;
ret = alloc_dst_sglist(skreq, ivsize);
if (ret) {
free_src_sglist(skreq);
return ret;
}
/* send the crypto request */
return nitrox_process_se_request(nctx->ndev, creq, nctx->callback,
skreq);
}
static int nitrox_cbc_decrypt(struct skcipher_request *skreq)
{
struct nitrox_kcrypt_request *nkreq = skcipher_request_ctx(skreq);
struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(skreq);
int ivsize = crypto_skcipher_ivsize(cipher);
gfp_t flags = (skreq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
GFP_KERNEL : GFP_ATOMIC;
unsigned int start = skreq->cryptlen - ivsize;
if (skreq->src != skreq->dst)
return nitrox_skcipher_crypt(skreq, false);
nkreq->iv_out = kmalloc(ivsize, flags);
if (!nkreq->iv_out)
return -ENOMEM;
scatterwalk_map_and_copy(nkreq->iv_out, skreq->src, start, ivsize, 0);
return nitrox_skcipher_crypt(skreq, false);
}
static int nitrox_aes_encrypt(struct skcipher_request *skreq)
{
return nitrox_skcipher_crypt(skreq, true);
}
static int nitrox_aes_decrypt(struct skcipher_request *skreq)
{
return nitrox_skcipher_crypt(skreq, false);
}
static int nitrox_3des_setkey(struct crypto_skcipher *cipher,
const u8 *key, unsigned int keylen)
{
return verify_skcipher_des3_key(cipher, key) ?:
nitrox_skcipher_setkey(cipher, 0, key, keylen);
}
static int nitrox_3des_encrypt(struct skcipher_request *skreq)
{
return nitrox_skcipher_crypt(skreq, true);
}
static int nitrox_3des_decrypt(struct skcipher_request *skreq)
{
return nitrox_skcipher_crypt(skreq, false);
}
static int nitrox_aes_xts_setkey(struct crypto_skcipher *cipher,
const u8 *key, unsigned int keylen)
{
struct nitrox_crypto_ctx *nctx = crypto_skcipher_ctx(cipher);
struct flexi_crypto_context *fctx;
int aes_keylen, ret;
ret = xts_verify_key(cipher, key, keylen);
if (ret)
return ret;
keylen /= 2;
aes_keylen = flexi_aes_keylen(keylen);
if (aes_keylen < 0)
return -EINVAL;
fctx = nctx->u.fctx;
/* copy KEY2 */
memcpy(fctx->auth.u.key2, (key + keylen), keylen);
return nitrox_skcipher_setkey(cipher, aes_keylen, key, keylen);
}
static int nitrox_aes_ctr_rfc3686_setkey(struct crypto_skcipher *cipher,
const u8 *key, unsigned int keylen)
{
struct nitrox_crypto_ctx *nctx = crypto_skcipher_ctx(cipher);
struct flexi_crypto_context *fctx;
int aes_keylen;
if (keylen < CTR_RFC3686_NONCE_SIZE)
return -EINVAL;
fctx = nctx->u.fctx;
memcpy(fctx->crypto.iv, key + (keylen - CTR_RFC3686_NONCE_SIZE),
CTR_RFC3686_NONCE_SIZE);
keylen -= CTR_RFC3686_NONCE_SIZE;
aes_keylen = flexi_aes_keylen(keylen);
if (aes_keylen < 0)
return -EINVAL;
return nitrox_skcipher_setkey(cipher, aes_keylen, key, keylen);
}
static struct skcipher_alg nitrox_skciphers[] = { {
.base = {
.cra_name = "cbc(aes)",
.cra_driver_name = "n5_cbc(aes)",
.cra_priority = PRIO,
.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,
.cra_blocksize = AES_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct nitrox_crypto_ctx),
.cra_alignmask = 0,
.cra_module = THIS_MODULE,
},
.min_keysize = AES_MIN_KEY_SIZE,
.max_keysize = AES_MAX_KEY_SIZE,
.ivsize = AES_BLOCK_SIZE,
.setkey = nitrox_aes_setkey,
.encrypt = nitrox_aes_encrypt,
.decrypt = nitrox_cbc_decrypt,
.init = nitrox_cbc_init,
.exit = nitrox_skcipher_exit,
}, {
.base = {
.cra_name = "ecb(aes)",
.cra_driver_name = "n5_ecb(aes)",
.cra_priority = PRIO,
.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,
.cra_blocksize = AES_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct nitrox_crypto_ctx),
.cra_alignmask = 0,
.cra_module = THIS_MODULE,
},
.min_keysize = AES_MIN_KEY_SIZE,
.max_keysize = AES_MAX_KEY_SIZE,
.ivsize = AES_BLOCK_SIZE,
.setkey = nitrox_aes_setkey,
.encrypt = nitrox_aes_encrypt,
.decrypt = nitrox_aes_decrypt,
.init = nitrox_skcipher_init,
.exit = nitrox_skcipher_exit,
}, {
.base = {
.cra_name = "xts(aes)",
.cra_driver_name = "n5_xts(aes)",
.cra_priority = PRIO,
.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,
.cra_blocksize = AES_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct nitrox_crypto_ctx),
.cra_alignmask = 0,
.cra_module = THIS_MODULE,
},
.min_keysize = 2 * AES_MIN_KEY_SIZE,
.max_keysize = 2 * AES_MAX_KEY_SIZE,
.ivsize = AES_BLOCK_SIZE,
.setkey = nitrox_aes_xts_setkey,
.encrypt = nitrox_aes_encrypt,
.decrypt = nitrox_aes_decrypt,
.init = nitrox_skcipher_init,
.exit = nitrox_skcipher_exit,
}, {
.base = {
.cra_name = "rfc3686(ctr(aes))",
.cra_driver_name = "n5_rfc3686(ctr(aes))",
.cra_priority = PRIO,
.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,
.cra_blocksize = 1,
.cra_ctxsize = sizeof(struct nitrox_crypto_ctx),
.cra_alignmask = 0,
.cra_module = THIS_MODULE,
},
.min_keysize = AES_MIN_KEY_SIZE + CTR_RFC3686_NONCE_SIZE,
.max_keysize = AES_MAX_KEY_SIZE + CTR_RFC3686_NONCE_SIZE,
.ivsize = CTR_RFC3686_IV_SIZE,
.init = nitrox_skcipher_init,
.exit = nitrox_skcipher_exit,
.setkey = nitrox_aes_ctr_rfc3686_setkey,
.encrypt = nitrox_aes_encrypt,
.decrypt = nitrox_aes_decrypt,
}, {
.base = {
.cra_name = "cts(cbc(aes))",
.cra_driver_name = "n5_cts(cbc(aes))",
.cra_priority = PRIO,
.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,
.cra_blocksize = AES_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct nitrox_crypto_ctx),
.cra_alignmask = 0,
.cra_module = THIS_MODULE,
},
.min_keysize = AES_MIN_KEY_SIZE,
.max_keysize = AES_MAX_KEY_SIZE,
.ivsize = AES_BLOCK_SIZE,
.setkey = nitrox_aes_setkey,
.encrypt = nitrox_aes_encrypt,
.decrypt = nitrox_aes_decrypt,
.init = nitrox_skcipher_init,
.exit = nitrox_skcipher_exit,
}, {
.base = {
.cra_name = "cbc(des3_ede)",
.cra_driver_name = "n5_cbc(des3_ede)",
.cra_priority = PRIO,
.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,
.cra_blocksize = DES3_EDE_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct nitrox_crypto_ctx),
.cra_alignmask = 0,
.cra_module = THIS_MODULE,
},
.min_keysize = DES3_EDE_KEY_SIZE,
.max_keysize = DES3_EDE_KEY_SIZE,
.ivsize = DES3_EDE_BLOCK_SIZE,
.setkey = nitrox_3des_setkey,
.encrypt = nitrox_3des_encrypt,
.decrypt = nitrox_cbc_decrypt,
.init = nitrox_cbc_init,
.exit = nitrox_skcipher_exit,
}, {
.base = {
.cra_name = "ecb(des3_ede)",
.cra_driver_name = "n5_ecb(des3_ede)",
.cra_priority = PRIO,
.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,
.cra_blocksize = DES3_EDE_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct nitrox_crypto_ctx),
.cra_alignmask = 0,
.cra_module = THIS_MODULE,
},
.min_keysize = DES3_EDE_KEY_SIZE,
.max_keysize = DES3_EDE_KEY_SIZE,
.ivsize = DES3_EDE_BLOCK_SIZE,
.setkey = nitrox_3des_setkey,
.encrypt = nitrox_3des_encrypt,
.decrypt = nitrox_3des_decrypt,
.init = nitrox_skcipher_init,
.exit = nitrox_skcipher_exit,
}
};
int nitrox_register_skciphers(void)
{
return crypto_register_skciphers(nitrox_skciphers,
ARRAY_SIZE(nitrox_skciphers));
}
void nitrox_unregister_skciphers(void)
{
crypto_unregister_skciphers(nitrox_skciphers,
ARRAY_SIZE(nitrox_skciphers));
}
linux-7.2.3/drivers/crypto/cavium/nitrox/nitrox_sriov.c
+234
View File
@@ -0,0 +1,234 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/pci.h>
#include <linux/delay.h>
#include "nitrox_dev.h"
#include "nitrox_hal.h"
#include "nitrox_common.h"
#include "nitrox_isr.h"
#include "nitrox_mbx.h"
/**
* num_vfs_valid - validate VF count
* @num_vfs: number of VF(s)
*/
static inline bool num_vfs_valid(int num_vfs)
{
bool valid = false;
switch (num_vfs) {
case 16:
case 32:
case 64:
case 128:
valid = true;
break;
}
return valid;
}
static inline enum vf_mode num_vfs_to_mode(int num_vfs)
{
enum vf_mode mode = 0;
switch (num_vfs) {
case 0:
mode = __NDEV_MODE_PF;
break;
case 16:
mode = __NDEV_MODE_VF16;
break;
case 32:
mode = __NDEV_MODE_VF32;
break;
case 64:
mode = __NDEV_MODE_VF64;
break;
case 128:
mode = __NDEV_MODE_VF128;
break;
}
return mode;
}
static inline int vf_mode_to_nr_queues(enum vf_mode mode)
{
int nr_queues = 0;
switch (mode) {
case __NDEV_MODE_PF:
nr_queues = MAX_PF_QUEUES;
break;
case __NDEV_MODE_VF16:
nr_queues = 8;
break;
case __NDEV_MODE_VF32:
nr_queues = 4;
break;
case __NDEV_MODE_VF64:
nr_queues = 2;
break;
case __NDEV_MODE_VF128:
nr_queues = 1;
break;
}
return nr_queues;
}
static void nitrox_pf_cleanup(struct nitrox_device *ndev)
{
/* PF has no queues in SR-IOV mode */
atomic_set(&ndev->state, __NDEV_NOT_READY);
/* unregister crypto algorithms */
nitrox_crypto_unregister();
/* cleanup PF resources */
nitrox_unregister_interrupts(ndev);
nitrox_common_sw_cleanup(ndev);
}
/**
* nitrox_pf_reinit - re-initialize PF resources once SR-IOV is disabled
* @ndev: NITROX device
*/
static int nitrox_pf_reinit(struct nitrox_device *ndev)
{
int err;
/* allocate resources for PF */
err = nitrox_common_sw_init(ndev);
if (err)
return err;
err = nitrox_register_interrupts(ndev);
if (err) {
nitrox_common_sw_cleanup(ndev);
return err;
}
/* configure the AQM queues */
nitrox_config_aqm_rings(ndev);
/* configure the packet queues */
nitrox_config_pkt_input_rings(ndev);
nitrox_config_pkt_solicit_ports(ndev);
/* set device to ready state */
atomic_set(&ndev->state, __NDEV_READY);
/* register crypto algorithms */
return nitrox_crypto_register();
}
static void nitrox_sriov_cleanup(struct nitrox_device *ndev)
{
/* unregister interrupts for PF in SR-IOV */
nitrox_sriov_unregister_interrupts(ndev);
nitrox_mbox_cleanup(ndev);
}
static int nitrox_sriov_init(struct nitrox_device *ndev)
{
int ret;
/* register interrupts for PF in SR-IOV */
ret = nitrox_sriov_register_interupts(ndev);
if (ret)
return ret;
ret = nitrox_mbox_init(ndev);
if (ret)
goto sriov_init_fail;
return 0;
sriov_init_fail:
nitrox_sriov_cleanup(ndev);
return ret;
}
static int nitrox_sriov_enable(struct pci_dev *pdev, int num_vfs)
{
struct nitrox_device *ndev = pci_get_drvdata(pdev);
int err;
if (!num_vfs_valid(num_vfs)) {
dev_err(DEV(ndev), "Invalid num_vfs %d\n", num_vfs);
return -EINVAL;
}
if (pci_num_vf(pdev) == num_vfs)
return num_vfs;
err = pci_enable_sriov(pdev, num_vfs);
if (err) {
dev_err(DEV(ndev), "failed to enable PCI sriov %d\n", err);
return err;
}
dev_info(DEV(ndev), "Enabled VF(s) %d\n", num_vfs);
ndev->mode = num_vfs_to_mode(num_vfs);
ndev->iov.num_vfs = num_vfs;
ndev->iov.max_vf_queues = vf_mode_to_nr_queues(ndev->mode);
/* set bit in flags */
set_bit(__NDEV_SRIOV_BIT, &ndev->flags);
/* cleanup PF resources */
nitrox_pf_cleanup(ndev);
/* PF SR-IOV mode initialization */
err = nitrox_sriov_init(ndev);
if (err)
goto iov_fail;
config_nps_core_vfcfg_mode(ndev, ndev->mode);
return num_vfs;
iov_fail:
pci_disable_sriov(pdev);
/* clear bit in flags */
clear_bit(__NDEV_SRIOV_BIT, &ndev->flags);
ndev->iov.num_vfs = 0;
ndev->mode = __NDEV_MODE_PF;
/* reset back to working mode in PF */
nitrox_pf_reinit(ndev);
return err;
}
static int nitrox_sriov_disable(struct pci_dev *pdev)
{
struct nitrox_device *ndev = pci_get_drvdata(pdev);
if (!test_bit(__NDEV_SRIOV_BIT, &ndev->flags))
return 0;
if (pci_vfs_assigned(pdev)) {
dev_warn(DEV(ndev), "VFs are attached to VM. Can't disable SR-IOV\n");
return -EPERM;
}
pci_disable_sriov(pdev);
/* clear bit in flags */
clear_bit(__NDEV_SRIOV_BIT, &ndev->flags);
ndev->iov.num_vfs = 0;
ndev->iov.max_vf_queues = 0;
ndev->mode = __NDEV_MODE_PF;
/* cleanup PF SR-IOV resources */
nitrox_sriov_cleanup(ndev);
config_nps_core_vfcfg_mode(ndev, ndev->mode);
return nitrox_pf_reinit(ndev);
}
int nitrox_sriov_configure(struct pci_dev *pdev, int num_vfs)
{
if (!num_vfs)
return nitrox_sriov_disable(pdev);
return nitrox_sriov_enable(pdev, num_vfs);
}