lastdragon/linux-kernel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Resguardo inicial

Browse Source
This commit is contained in:
lastdragon
2026-09-06 12:53:21 -06:00
parent 662efe3c2a
commit ad2088d178
94853 changed files with 43200404 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
linux-7.2.3/drivers/infiniband/hw/irdma/Kconfig
+14
View File
@@ -0,0 +1,14 @@
# SPDX-License-Identifier: GPL-2.0-only
config INFINIBAND_IRDMA
tristate "Intel(R) Ethernet Protocol Driver for RDMA"
depends on INET
depends on IPV6 || !IPV6
depends on PCI
depends on IDPF && ICE && I40E
select GENERIC_ALLOCATOR
select AUXILIARY_BUS
select CRC32
help
This is an Intel(R) Ethernet Protocol Driver for RDMA that
supports IPU E2000 (RoCEv2), E810 (iWARP/RoCEv2) and X722 (iWARP)
network devices.
linux-7.2.3/drivers/infiniband/hw/irdma/Makefile
+31
View File
@@ -0,0 +1,31 @@
# SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
# Copyright (c) 2019, Intel Corporation.
#
# Makefile for the Intel(R) Ethernet Connection RDMA Linux Driver
#
obj-$(CONFIG_INFINIBAND_IRDMA) += irdma.o
irdma-objs := cm.o \
ctrl.o \
hmc.o \
hw.o \
i40iw_hw.o \
i40iw_if.o \
ig3rdma_if.o\
icrdma_if.o \
icrdma_hw.o \
ig3rdma_hw.o\
main.o \
pble.o \
puda.o \
trace.o \
uda.o \
uk.o \
utils.o \
verbs.o \
virtchnl.o \
ws.o \
CFLAGS_trace.o = -I$(src)
linux-7.2.3/drivers/infiniband/hw/irdma/cm.c
+4437
View File
File diff suppressed because it is too large Load Diff
linux-7.2.3/drivers/infiniband/hw/irdma/cm.h
+416
View File
@@ -0,0 +1,416 @@
/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
/* Copyright (c) 2015 - 2021 Intel Corporation */
#ifndef IRDMA_CM_H
#define IRDMA_CM_H
#define IRDMA_MPA_REQUEST_ACCEPT 1
#define IRDMA_MPA_REQUEST_REJECT 2
/* IETF MPA -- defines */
#define IEFT_MPA_KEY_REQ "MPA ID Req Frame"
#define IEFT_MPA_KEY_REP "MPA ID Rep Frame"
#define IETF_MPA_KEY_SIZE 16
#define IETF_MPA_VER 1
#define IETF_MAX_PRIV_DATA_LEN 512
#define IETF_MPA_FRAME_SIZE 20
#define IETF_RTR_MSG_SIZE 4
#define IETF_MPA_V2_FLAG 0x10
#define SNDMARKER_SEQNMASK 0x000001ff
#define IRDMA_MAX_IETF_SIZE 32
/* IETF RTR MSG Fields */
#define IETF_PEER_TO_PEER 0x8000
#define IETF_FLPDU_ZERO_LEN 0x4000
#define IETF_RDMA0_WRITE 0x8000
#define IETF_RDMA0_READ 0x4000
#define IETF_NO_IRD_ORD 0x3fff
#define MAX_PORTS 65536
#define IRDMA_PASSIVE_STATE_INDICATED 0
#define IRDMA_DO_NOT_SEND_RESET_EVENT 1
#define IRDMA_SEND_RESET_EVENT 2
#define MAX_IRDMA_IFS 4
#define SET_ACK 1
#define SET_SYN 2
#define SET_FIN 4
#define SET_RST 8
#define TCP_OPTIONS_PADDING 3
#define IRDMA_DEFAULT_RETRYS 64
#define IRDMA_DEFAULT_RETRANS 32
#define IRDMA_DEFAULT_TTL 0x40
#define IRDMA_DEFAULT_RTT_VAR 6
#define IRDMA_DEFAULT_SS_THRESH 0x3fffffff
#define IRDMA_DEFAULT_REXMIT_THRESH 8
#define IRDMA_RETRY_TIMEOUT HZ
#define IRDMA_SHORT_TIME 10
#define IRDMA_LONG_TIME (2 * HZ)
#define IRDMA_MAX_TIMEOUT ((unsigned long)(12 * HZ))
#define IRDMA_CM_HASHTABLE_SIZE 1024
#define IRDMA_CM_TCP_TIMER_INTERVAL 3000
#define IRDMA_CM_DEFAULT_MTU 1540
#define IRDMA_CM_DEFAULT_FRAME_CNT 10
#define IRDMA_CM_THREAD_STACK_SIZE 256
#define IRDMA_CM_DEFAULT_RCV_WND 64240
#define IRDMA_CM_DEFAULT_RCV_WND_SCALED 0x3FFFC
#define IRDMA_CM_DEFAULT_RCV_WND_SCALE 2
#define IRDMA_CM_DEFAULT_FREE_PKTS 10
#define IRDMA_CM_FREE_PKT_LO_WATERMARK 2
#define IRDMA_CM_DEFAULT_MSS 536
#define IRDMA_CM_DEFAULT_MPA_VER 2
#define IRDMA_CM_DEFAULT_SEQ 0x159bf75f
#define IRDMA_CM_DEFAULT_LOCAL_ID 0x3b47
#define IRDMA_CM_DEFAULT_SEQ2 0x18ed5740
#define IRDMA_CM_DEFAULT_LOCAL_ID2 0xb807
#define IRDMA_MAX_CM_BUF (IRDMA_MAX_IETF_SIZE + IETF_MAX_PRIV_DATA_LEN)
enum ietf_mpa_flags {
IETF_MPA_FLAGS_REJECT = 0x20,
IETF_MPA_FLAGS_CRC = 0x40,
IETF_MPA_FLAGS_MARKERS = 0x80,
};
enum irdma_timer_type {
IRDMA_TIMER_TYPE_SEND,
IRDMA_TIMER_TYPE_CLOSE,
};
enum option_nums {
OPTION_NUM_EOL,
OPTION_NUM_NONE,
OPTION_NUM_MSS,
OPTION_NUM_WINDOW_SCALE,
OPTION_NUM_SACK_PERM,
OPTION_NUM_SACK,
OPTION_NUM_WRITE0 = 0xbc,
};
/* cm node transition states */
enum irdma_cm_node_state {
IRDMA_CM_STATE_UNKNOWN,
IRDMA_CM_STATE_INITED,
IRDMA_CM_STATE_LISTENING,
IRDMA_CM_STATE_SYN_RCVD,
IRDMA_CM_STATE_SYN_SENT,
IRDMA_CM_STATE_ONE_SIDE_ESTABLISHED,
IRDMA_CM_STATE_ESTABLISHED,
IRDMA_CM_STATE_ACCEPTING,
IRDMA_CM_STATE_MPAREQ_SENT,
IRDMA_CM_STATE_MPAREQ_RCVD,
IRDMA_CM_STATE_MPAREJ_RCVD,
IRDMA_CM_STATE_OFFLOADED,
IRDMA_CM_STATE_FIN_WAIT1,
IRDMA_CM_STATE_FIN_WAIT2,
IRDMA_CM_STATE_CLOSE_WAIT,
IRDMA_CM_STATE_TIME_WAIT,
IRDMA_CM_STATE_LAST_ACK,
IRDMA_CM_STATE_CLOSING,
IRDMA_CM_STATE_LISTENER_DESTROYED,
IRDMA_CM_STATE_CLOSED,
};
enum mpa_frame_ver {
IETF_MPA_V1 = 1,
IETF_MPA_V2 = 2,
};
enum mpa_frame_key {
MPA_KEY_REQUEST,
MPA_KEY_REPLY,
};
enum send_rdma0 {
SEND_RDMA_READ_ZERO = 1,
SEND_RDMA_WRITE_ZERO = 2,
};
enum irdma_tcpip_pkt_type {
IRDMA_PKT_TYPE_UNKNOWN,
IRDMA_PKT_TYPE_SYN,
IRDMA_PKT_TYPE_SYNACK,
IRDMA_PKT_TYPE_ACK,
IRDMA_PKT_TYPE_FIN,
IRDMA_PKT_TYPE_RST,
};
enum irdma_cm_listener_state {
IRDMA_CM_LISTENER_PASSIVE_STATE = 1,
IRDMA_CM_LISTENER_ACTIVE_STATE = 2,
IRDMA_CM_LISTENER_EITHER_STATE = 3,
};
/* CM event codes */
enum irdma_cm_event_type {
IRDMA_CM_EVENT_UNKNOWN,
IRDMA_CM_EVENT_ESTABLISHED,
IRDMA_CM_EVENT_MPA_REQ,
IRDMA_CM_EVENT_MPA_CONNECT,
IRDMA_CM_EVENT_MPA_ACCEPT,
IRDMA_CM_EVENT_MPA_REJECT,
IRDMA_CM_EVENT_MPA_ESTABLISHED,
IRDMA_CM_EVENT_CONNECTED,
IRDMA_CM_EVENT_RESET,
IRDMA_CM_EVENT_ABORTED,
};
struct ietf_mpa_v1 {
u8 key[IETF_MPA_KEY_SIZE];
u8 flags;
u8 rev;
__be16 priv_data_len;
u8 priv_data[];
};
struct ietf_rtr_msg {
__be16 ctrl_ird;
__be16 ctrl_ord;
};
struct ietf_mpa_v2 {
u8 key[IETF_MPA_KEY_SIZE];
u8 flags;
u8 rev;
__be16 priv_data_len;
struct ietf_rtr_msg rtr_msg;
u8 priv_data[];
};
struct option_base {
u8 optionnum;
u8 len;
};
struct option_mss {
u8 optionnum;
u8 len;
__be16 mss;
};
struct option_windowscale {
u8 optionnum;
u8 len;
u8 shiftcount;
};
union all_known_options {
char eol;
struct option_base base;
struct option_mss mss;
struct option_windowscale windowscale;
};
struct irdma_timer_entry {
struct list_head list;
unsigned long timetosend; /* jiffies */
struct irdma_puda_buf *sqbuf;
u32 type;
u32 retrycount;
u32 retranscount;
u32 context;
u32 send_retrans;
int close_when_complete;
};
/* CM context params */
struct irdma_cm_tcp_context {
u8 client;
u32 loc_seq_num;
u32 loc_ack_num;
u32 rem_ack_num;
u32 rcv_nxt;
u32 loc_id;
u32 rem_id;
u32 snd_wnd;
u32 max_snd_wnd;
u32 rcv_wnd;
u32 mss;
u8 snd_wscale;
u8 rcv_wscale;
};
struct irdma_apbvt_entry {
struct hlist_node hlist;
u32 use_cnt;
u16 port;
};
struct irdma_cm_listener {
struct list_head list;
struct iw_cm_id *cm_id;
struct irdma_cm_core *cm_core;
struct irdma_device *iwdev;
struct list_head child_listen_list;
struct irdma_apbvt_entry *apbvt_entry;
enum irdma_cm_listener_state listener_state;
refcount_t refcnt;
atomic_t pend_accepts_cnt;
u32 loc_addr[4];
u32 reused_node;
int backlog;
u16 loc_port;
u16 vlan_id;
u8 loc_mac[ETH_ALEN];
u8 user_pri;
u8 tos;
bool qhash_set:1;
bool ipv4:1;
};
struct irdma_kmem_info {
void *addr;
u32 size;
};
struct irdma_mpa_priv_info {
const void *addr;
u32 size;
};
struct irdma_cm_node {
struct irdma_qp *iwqp;
struct irdma_device *iwdev;
struct irdma_sc_dev *dev;
struct irdma_cm_tcp_context tcp_cntxt;
struct irdma_cm_core *cm_core;
struct irdma_timer_entry *send_entry;
struct irdma_timer_entry *close_entry;
struct irdma_cm_listener *listener;
struct list_head timer_entry;
struct list_head reset_entry;
struct list_head teardown_entry;
struct irdma_apbvt_entry *apbvt_entry;
struct rcu_head rcu_head;
struct irdma_mpa_priv_info pdata;
struct irdma_sc_ah *ah;
union {
struct ietf_mpa_v1 mpa_frame;
struct ietf_mpa_v2 mpa_v2_frame;
};
struct irdma_kmem_info mpa_hdr;
struct iw_cm_id *cm_id;
struct hlist_node list;
struct completion establish_comp;
spinlock_t retrans_list_lock; /* protect CM node rexmit updates*/
atomic_t passive_state;
refcount_t refcnt;
enum irdma_cm_node_state state;
enum send_rdma0 send_rdma0_op;
enum mpa_frame_ver mpa_frame_rev;
u32 loc_addr[4], rem_addr[4];
u16 loc_port, rem_port;
int apbvt_set;
int accept_pend;
u16 vlan_id;
u16 ird_size;
u16 ord_size;
u16 mpav2_ird_ord;
u16 lsmm_size;
u8 pdata_buf[IETF_MAX_PRIV_DATA_LEN];
u8 loc_mac[ETH_ALEN];
u8 rem_mac[ETH_ALEN];
u8 user_pri;
u8 tos;
bool ack_rcvd:1;
bool qhash_set:1;
bool ipv4:1;
bool snd_mark_en:1;
bool rcv_mark_en:1;
bool do_lpb:1;
bool accelerated:1;
};
/* Used by internal CM APIs to pass CM information*/
struct irdma_cm_info {
struct iw_cm_id *cm_id;
u16 loc_port;
u16 rem_port;
u32 loc_addr[4];
u32 rem_addr[4];
u32 qh_qpid;
u16 vlan_id;
int backlog;
u8 user_pri;
u8 tos;
bool ipv4;
};
struct irdma_cm_event {
enum irdma_cm_event_type type;
struct irdma_cm_info cm_info;
struct work_struct event_work;
struct irdma_cm_node *cm_node;
};
struct irdma_cm_core {
struct irdma_device *iwdev;
struct irdma_sc_dev *dev;
struct list_head listen_list;
DECLARE_HASHTABLE(cm_hash_tbl, 8);
DECLARE_HASHTABLE(apbvt_hash_tbl, 8);
struct timer_list tcp_timer;
struct workqueue_struct *event_wq;
spinlock_t ht_lock; /* protect CM node (active side) list */
spinlock_t listen_list_lock; /* protect listener list */
spinlock_t apbvt_lock; /*serialize apbvt add/del entries*/
u64 stats_nodes_created;
u64 stats_nodes_destroyed;
u64 stats_listen_created;
u64 stats_listen_destroyed;
u64 stats_listen_nodes_created;
u64 stats_listen_nodes_destroyed;
u64 stats_lpbs;
u64 stats_accepts;
u64 stats_rejects;
u64 stats_connect_errs;
u64 stats_passive_errs;
u64 stats_pkt_retrans;
u64 stats_backlog_drops;
struct irdma_puda_buf *(*form_cm_frame)(struct irdma_cm_node *cm_node,
struct irdma_kmem_info *options,
struct irdma_kmem_info *hdr,
struct irdma_mpa_priv_info *pdata,
u8 flags);
int (*cm_create_ah)(struct irdma_cm_node *cm_node, bool wait);
void (*cm_free_ah)(struct irdma_cm_node *cm_node);
};
int irdma_schedule_cm_timer(struct irdma_cm_node *cm_node,
struct irdma_puda_buf *sqbuf,
enum irdma_timer_type type, int send_retrans,
int close_when_complete);
static inline u8 irdma_tos2dscp(u8 tos)
{
#define IRDMA_DSCP_VAL GENMASK(7, 2)
return (u8)FIELD_GET(IRDMA_DSCP_VAL, tos);
}
int irdma_accept(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param);
int irdma_reject(struct iw_cm_id *cm_id, const void *pdata, u8 pdata_len);
int irdma_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param);
int irdma_create_listen(struct iw_cm_id *cm_id, int backlog);
int irdma_destroy_listen(struct iw_cm_id *cm_id);
int irdma_add_arp(struct irdma_pci_f *rf, u32 *ip, bool ipv4, const u8 *mac);
void irdma_cm_teardown_connections(struct irdma_device *iwdev, u32 *ipaddr,
struct irdma_cm_info *nfo,
bool disconnect_all);
int irdma_cm_start(struct irdma_device *dev);
int irdma_cm_stop(struct irdma_device *dev);
bool irdma_ipv4_is_lpb(u32 loc_addr, u32 rem_addr);
bool irdma_ipv6_is_lpb(u32 *loc_addr, u32 *rem_addr);
int irdma_arp_table(struct irdma_pci_f *rf, u32 *ip_addr, bool ipv4,
const u8 *mac_addr, u32 action);
void irdma_if_notify(struct irdma_device *iwdev, struct net_device *netdev,
u32 *ipaddr, bool ipv4, bool ifup);
bool irdma_port_in_use(struct irdma_cm_core *cm_core, u16 port);
void irdma_send_ack(struct irdma_cm_node *cm_node);
void irdma_lpb_nop(struct irdma_sc_qp *qp);
void irdma_rem_ref_cm_node(struct irdma_cm_node *cm_node);
void irdma_add_conn_est_qh(struct irdma_cm_node *cm_node);
#endif /* IRDMA_CM_H */
linux-7.2.3/drivers/infiniband/hw/irdma/ctrl.c
+6627
View File
File diff suppressed because it is too large Load Diff
linux-7.2.3/drivers/infiniband/hw/irdma/defs.h
+1188
View File
File diff suppressed because it is too large Load Diff
linux-7.2.3/drivers/infiniband/hw/irdma/hmc.c
+709
View File
@@ -0,0 +1,709 @@
// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
/* Copyright (c) 2015 - 2021 Intel Corporation */
#include "osdep.h"
#include "hmc.h"
#include "defs.h"
#include "type.h"
#include "protos.h"
#include "virtchnl.h"
/**
* irdma_find_sd_index_limit - finds segment descriptor index limit
* @hmc_info: pointer to the HMC configuration information structure
* @type: type of HMC resources we're searching
* @idx: starting index for the object
* @cnt: number of objects we're trying to create
* @sd_idx: pointer to return index of the segment descriptor in question
* @sd_limit: pointer to return the maximum number of segment descriptors
*
* This function calculates the segment descriptor index and index limit
* for the resource defined by irdma_hmc_rsrc_type.
*/
static void irdma_find_sd_index_limit(struct irdma_hmc_info *hmc_info, u32 type,
u32 idx, u32 cnt, u32 *sd_idx,
u32 *sd_limit)
{
u64 fpm_addr, fpm_limit;
fpm_addr = hmc_info->hmc_obj[(type)].base +
hmc_info->hmc_obj[type].size * idx;
fpm_limit = fpm_addr + hmc_info->hmc_obj[type].size * cnt;
*sd_idx = (u32)(fpm_addr / IRDMA_HMC_DIRECT_BP_SIZE);
*sd_limit = (u32)((fpm_limit - 1) / IRDMA_HMC_DIRECT_BP_SIZE);
*sd_limit += 1;
}
/**
* irdma_find_pd_index_limit - finds page descriptor index limit
* @hmc_info: pointer to the HMC configuration information struct
* @type: HMC resource type we're examining
* @idx: starting index for the object
* @cnt: number of objects we're trying to create
* @pd_idx: pointer to return page descriptor index
* @pd_limit: pointer to return page descriptor index limit
*
* Calculates the page descriptor index and index limit for the resource
* defined by irdma_hmc_rsrc_type.
*/
static void irdma_find_pd_index_limit(struct irdma_hmc_info *hmc_info, u32 type,
u32 idx, u32 cnt, u32 *pd_idx,
u32 *pd_limit)
{
u64 fpm_adr, fpm_limit;
fpm_adr = hmc_info->hmc_obj[type].base +
hmc_info->hmc_obj[type].size * idx;
fpm_limit = fpm_adr + (hmc_info)->hmc_obj[(type)].size * (cnt);
*pd_idx = (u32)(fpm_adr / IRDMA_HMC_PAGED_BP_SIZE);
*pd_limit = (u32)((fpm_limit - 1) / IRDMA_HMC_PAGED_BP_SIZE);
*pd_limit += 1;
}
/**
* irdma_set_sd_entry - setup entry for sd programming
* @pa: physical addr
* @idx: sd index
* @type: paged or direct sd
* @entry: sd entry ptr
*/
static void irdma_set_sd_entry(u64 pa, u32 idx, enum irdma_sd_entry_type type,
struct irdma_update_sd_entry *entry)
{
entry->data = pa |
FIELD_PREP(IRDMA_PFHMC_SDDATALOW_PMSDBPCOUNT, IRDMA_HMC_MAX_BP_COUNT) |
FIELD_PREP(IRDMA_PFHMC_SDDATALOW_PMSDTYPE,
type == IRDMA_SD_TYPE_PAGED ? 0 : 1) |
FIELD_PREP(IRDMA_PFHMC_SDDATALOW_PMSDVALID, 1);
entry->cmd = idx | FIELD_PREP(IRDMA_PFHMC_SDCMD_PMSDWR, 1) | BIT(15);
}
/**
* irdma_clr_sd_entry - setup entry for sd clear
* @idx: sd index
* @type: paged or direct sd
* @entry: sd entry ptr
*/
static void irdma_clr_sd_entry(u32 idx, enum irdma_sd_entry_type type,
struct irdma_update_sd_entry *entry)
{
entry->data = FIELD_PREP(IRDMA_PFHMC_SDDATALOW_PMSDBPCOUNT, IRDMA_HMC_MAX_BP_COUNT) |
FIELD_PREP(IRDMA_PFHMC_SDDATALOW_PMSDTYPE,
type == IRDMA_SD_TYPE_PAGED ? 0 : 1);
entry->cmd = idx | FIELD_PREP(IRDMA_PFHMC_SDCMD_PMSDWR, 1) | BIT(15);
}
/**
* irdma_invalidate_pf_hmc_pd - Invalidates the pd cache in the hardware for PF
* @dev: pointer to our device struct
* @sd_idx: segment descriptor index
* @pd_idx: page descriptor index
*/
static inline void irdma_invalidate_pf_hmc_pd(struct irdma_sc_dev *dev, u32 sd_idx,
u32 pd_idx)
{
u32 val = FIELD_PREP(IRDMA_PFHMC_PDINV_PMSDIDX, sd_idx) |
FIELD_PREP(IRDMA_PFHMC_PDINV_PMSDPARTSEL, 1) |
FIELD_PREP(IRDMA_PFHMC_PDINV_PMPDIDX, pd_idx);
writel(val, dev->hw_regs[IRDMA_PFHMC_PDINV]);
}
/**
* irdma_hmc_sd_one - setup 1 sd entry for cqp
* @dev: pointer to the device structure
* @hmc_fn_id: hmc's function id
* @pa: physical addr
* @sd_idx: sd index
* @type: paged or direct sd
* @setsd: flag to set or clear sd
*/
int irdma_hmc_sd_one(struct irdma_sc_dev *dev, u8 hmc_fn_id, u64 pa, u32 sd_idx,
enum irdma_sd_entry_type type, bool setsd)
{
struct irdma_update_sds_info sdinfo;
sdinfo.cnt = 1;
sdinfo.hmc_fn_id = hmc_fn_id;
if (setsd)
irdma_set_sd_entry(pa, sd_idx, type, sdinfo.entry);
else
irdma_clr_sd_entry(sd_idx, type, sdinfo.entry);
return dev->cqp->process_cqp_sds(dev, &sdinfo);
}
/**
* irdma_hmc_sd_grp - setup group of sd entries for cqp
* @dev: pointer to the device structure
* @hmc_info: pointer to the HMC configuration information struct
* @sd_index: sd index
* @sd_cnt: number of sd entries
* @setsd: flag to set or clear sd
*/
static int irdma_hmc_sd_grp(struct irdma_sc_dev *dev,
struct irdma_hmc_info *hmc_info, u32 sd_index,
u32 sd_cnt, bool setsd)
{
struct irdma_hmc_sd_entry *sd_entry;
struct irdma_update_sds_info sdinfo = {};
u64 pa;
u32 i;
int ret_code = 0;
sdinfo.hmc_fn_id = hmc_info->hmc_fn_id;
for (i = sd_index; i < sd_index + sd_cnt; i++) {
sd_entry = &hmc_info->sd_table.sd_entry[i];
if (!sd_entry || (!sd_entry->valid && setsd) ||
(sd_entry->valid && !setsd))
continue;
if (setsd) {
pa = (sd_entry->entry_type == IRDMA_SD_TYPE_PAGED) ?
sd_entry->u.pd_table.pd_page_addr.pa :
sd_entry->u.bp.addr.pa;
irdma_set_sd_entry(pa, i, sd_entry->entry_type,
&sdinfo.entry[sdinfo.cnt]);
} else {
irdma_clr_sd_entry(i, sd_entry->entry_type,
&sdinfo.entry[sdinfo.cnt]);
}
sdinfo.cnt++;
if (sdinfo.cnt == IRDMA_MAX_SD_ENTRIES) {
ret_code = dev->cqp->process_cqp_sds(dev, &sdinfo);
if (ret_code) {
ibdev_dbg(to_ibdev(dev),
"HMC: sd_programming failed err=%d\n",
ret_code);
return ret_code;
}
sdinfo.cnt = 0;
}
}
if (sdinfo.cnt)
ret_code = dev->cqp->process_cqp_sds(dev, &sdinfo);
return ret_code;
}
/**
* irdma_hmc_finish_add_sd_reg - program sd entries for objects
* @dev: pointer to the device structure
* @info: create obj info
*/
static int irdma_hmc_finish_add_sd_reg(struct irdma_sc_dev *dev,
struct irdma_hmc_create_obj_info *info)
{
if (info->start_idx >= info->hmc_info->hmc_obj[info->rsrc_type].cnt)
return -EINVAL;
if ((info->start_idx + info->count) >
info->hmc_info->hmc_obj[info->rsrc_type].cnt)
return -EINVAL;
if (!info->add_sd_cnt)
return 0;
return irdma_hmc_sd_grp(dev, info->hmc_info,
info->hmc_info->sd_indexes[0], info->add_sd_cnt,
true);
}
/**
* irdma_sc_create_hmc_obj - allocate backing store for hmc objects
* @dev: pointer to the device structure
* @info: pointer to irdma_hmc_create_obj_info struct
*
* This will allocate memory for PDs and backing pages and populate
* the sd and pd entries.
*/
int irdma_sc_create_hmc_obj(struct irdma_sc_dev *dev,
struct irdma_hmc_create_obj_info *info)
{
struct irdma_hmc_sd_entry *sd_entry;
u32 sd_idx, sd_lmt;
u32 pd_idx = 0, pd_lmt = 0;
u32 pd_idx1 = 0, pd_lmt1 = 0;
u32 i, j;
bool pd_error = false;
int ret_code = 0;
if (dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_3 &&
dev->hmc_info->hmc_obj[info->rsrc_type].mem_loc == IRDMA_LOC_MEM)
return 0;
if (info->start_idx >= info->hmc_info->hmc_obj[info->rsrc_type].cnt)
return -EINVAL;
if ((info->start_idx + info->count) >
info->hmc_info->hmc_obj[info->rsrc_type].cnt) {
ibdev_dbg(to_ibdev(dev),
"HMC: error type %u, start = %u, req cnt %u, cnt = %u\n",
info->rsrc_type, info->start_idx, info->count,
info->hmc_info->hmc_obj[info->rsrc_type].cnt);
return -EINVAL;
}
irdma_find_sd_index_limit(info->hmc_info, info->rsrc_type,
info->start_idx, info->count, &sd_idx,
&sd_lmt);
if (sd_idx >= info->hmc_info->sd_table.sd_cnt ||
sd_lmt > info->hmc_info->sd_table.sd_cnt) {
return -EINVAL;
}
irdma_find_pd_index_limit(info->hmc_info, info->rsrc_type,
info->start_idx, info->count, &pd_idx,
&pd_lmt);
for (j = sd_idx; j < sd_lmt; j++) {
ret_code = irdma_add_sd_table_entry(dev->hw, info->hmc_info, j,
info->entry_type,
IRDMA_HMC_DIRECT_BP_SIZE);
if (ret_code)
goto exit_sd_error;
sd_entry = &info->hmc_info->sd_table.sd_entry[j];
if (sd_entry->entry_type == IRDMA_SD_TYPE_PAGED &&
(dev->hmc_info == info->hmc_info &&
info->rsrc_type != IRDMA_HMC_IW_PBLE)) {
pd_idx1 = max(pd_idx, (j * IRDMA_HMC_MAX_BP_COUNT));
pd_lmt1 = min(pd_lmt, (j + 1) * IRDMA_HMC_MAX_BP_COUNT);
for (i = pd_idx1; i < pd_lmt1; i++) {
/* update the pd table entry */
ret_code = irdma_add_pd_table_entry(dev,
info->hmc_info,
i, NULL);
if (ret_code) {
pd_error = true;
break;
}
}
if (pd_error) {
while (i && (i > pd_idx1)) {
irdma_remove_pd_bp(dev, info->hmc_info,
i - 1);
i--;
}
}
}
if (sd_entry->valid)
continue;
info->hmc_info->sd_indexes[info->add_sd_cnt] = (u16)j;
info->add_sd_cnt++;
sd_entry->valid = true;
}
return irdma_hmc_finish_add_sd_reg(dev, info);
exit_sd_error:
while (j && (j > sd_idx)) {
sd_entry = &info->hmc_info->sd_table.sd_entry[j - 1];
switch (sd_entry->entry_type) {
case IRDMA_SD_TYPE_PAGED:
pd_idx1 = max(pd_idx, (j - 1) * IRDMA_HMC_MAX_BP_COUNT);
pd_lmt1 = min(pd_lmt, (j * IRDMA_HMC_MAX_BP_COUNT));
for (i = pd_idx1; i < pd_lmt1; i++)
irdma_prep_remove_pd_page(info->hmc_info, i);
break;
case IRDMA_SD_TYPE_DIRECT:
irdma_prep_remove_pd_page(info->hmc_info, (j - 1));
break;
default:
ret_code = -EINVAL;
break;
}
j--;
}
return ret_code;
}
/**
* irdma_finish_del_sd_reg - delete sd entries for objects
* @dev: pointer to the device structure
* @info: dele obj info
* @reset: true if called before reset
*/
static int irdma_finish_del_sd_reg(struct irdma_sc_dev *dev,
struct irdma_hmc_del_obj_info *info,
bool reset)
{
struct irdma_hmc_sd_entry *sd_entry;
int ret_code = 0;
u32 i, sd_idx;
struct irdma_dma_mem *mem;
if (dev->privileged && !reset)
ret_code = irdma_hmc_sd_grp(dev, info->hmc_info,
info->hmc_info->sd_indexes[0],
info->del_sd_cnt, false);
if (ret_code)
ibdev_dbg(to_ibdev(dev), "HMC: error cqp sd sd_grp\n");
for (i = 0; i < info->del_sd_cnt; i++) {
sd_idx = info->hmc_info->sd_indexes[i];
sd_entry = &info->hmc_info->sd_table.sd_entry[sd_idx];
mem = (sd_entry->entry_type == IRDMA_SD_TYPE_PAGED) ?
&sd_entry->u.pd_table.pd_page_addr :
&sd_entry->u.bp.addr;
if (!mem || !mem->va) {
ibdev_dbg(to_ibdev(dev), "HMC: error cqp sd mem\n");
} else {
dma_free_coherent(dev->hw->device, mem->size, mem->va,
mem->pa);
mem->va = NULL;
}
}
return ret_code;
}
/**
* irdma_sc_del_hmc_obj - remove pe hmc objects
* @dev: pointer to the device structure
* @info: pointer to irdma_hmc_del_obj_info struct
* @reset: true if called before reset
*
* This will de-populate the SDs and PDs. It frees
* the memory for PDS and backing storage. After this function is returned,
* caller should deallocate memory allocated previously for
* book-keeping information about PDs and backing storage.
*/
int irdma_sc_del_hmc_obj(struct irdma_sc_dev *dev,
struct irdma_hmc_del_obj_info *info, bool reset)
{
struct irdma_hmc_pd_table *pd_table;
u32 sd_idx, sd_lmt;
u32 pd_idx, pd_lmt, rel_pd_idx;
u32 i, j;
int ret_code = 0;
if (dev->hmc_info->hmc_obj[info->rsrc_type].mem_loc == IRDMA_LOC_MEM)
return 0;
if (info->start_idx >= info->hmc_info->hmc_obj[info->rsrc_type].cnt) {
ibdev_dbg(to_ibdev(dev),
"HMC: error start_idx[%04d] >= [type %04d].cnt[%04d]\n",
info->start_idx, info->rsrc_type,
info->hmc_info->hmc_obj[info->rsrc_type].cnt);
return -EINVAL;
}
if ((info->start_idx + info->count) >
info->hmc_info->hmc_obj[info->rsrc_type].cnt) {
ibdev_dbg(to_ibdev(dev),
"HMC: error start_idx[%04d] + count %04d >= [type %04d].cnt[%04d]\n",
info->start_idx, info->count, info->rsrc_type,
info->hmc_info->hmc_obj[info->rsrc_type].cnt);
return -EINVAL;
}
irdma_find_pd_index_limit(info->hmc_info, info->rsrc_type,
info->start_idx, info->count, &pd_idx,
&pd_lmt);
for (j = pd_idx; j < pd_lmt; j++) {
sd_idx = j / IRDMA_HMC_PD_CNT_IN_SD;
if (!info->hmc_info->sd_table.sd_entry[sd_idx].valid)
continue;
if (info->hmc_info->sd_table.sd_entry[sd_idx].entry_type !=
IRDMA_SD_TYPE_PAGED)
continue;
rel_pd_idx = j % IRDMA_HMC_PD_CNT_IN_SD;
pd_table = &info->hmc_info->sd_table.sd_entry[sd_idx].u.pd_table;
if (pd_table->pd_entry &&
pd_table->pd_entry[rel_pd_idx].valid) {
ret_code = irdma_remove_pd_bp(dev, info->hmc_info, j);
if (ret_code) {
ibdev_dbg(to_ibdev(dev),
"HMC: remove_pd_bp error\n");
return ret_code;
}
}
}
irdma_find_sd_index_limit(info->hmc_info, info->rsrc_type,
info->start_idx, info->count, &sd_idx,
&sd_lmt);
if (sd_idx >= info->hmc_info->sd_table.sd_cnt ||
sd_lmt > info->hmc_info->sd_table.sd_cnt) {
ibdev_dbg(to_ibdev(dev), "HMC: invalid sd_idx\n");
return -EINVAL;
}
for (i = sd_idx; i < sd_lmt; i++) {
pd_table = &info->hmc_info->sd_table.sd_entry[i].u.pd_table;
if (!info->hmc_info->sd_table.sd_entry[i].valid)
continue;
switch (info->hmc_info->sd_table.sd_entry[i].entry_type) {
case IRDMA_SD_TYPE_DIRECT:
ret_code = irdma_prep_remove_sd_bp(info->hmc_info, i);
if (!ret_code) {
info->hmc_info->sd_indexes[info->del_sd_cnt] =
(u16)i;
info->del_sd_cnt++;
}
break;
case IRDMA_SD_TYPE_PAGED:
ret_code = irdma_prep_remove_pd_page(info->hmc_info, i);
if (ret_code)
break;
if (dev->hmc_info != info->hmc_info &&
info->rsrc_type == IRDMA_HMC_IW_PBLE &&
pd_table->pd_entry) {
kfree(pd_table->pd_entry_virt_mem.va);
pd_table->pd_entry = NULL;
}
info->hmc_info->sd_indexes[info->del_sd_cnt] = (u16)i;
info->del_sd_cnt++;
break;
default:
break;
}
}
return irdma_finish_del_sd_reg(dev, info, reset);
}
/**
* irdma_add_sd_table_entry - Adds a segment descriptor to the table
* @hw: pointer to our hw struct
* @hmc_info: pointer to the HMC configuration information struct
* @sd_index: segment descriptor index to manipulate
* @type: what type of segment descriptor we're manipulating
* @direct_mode_sz: size to alloc in direct mode
*/
int irdma_add_sd_table_entry(struct irdma_hw *hw,
struct irdma_hmc_info *hmc_info, u32 sd_index,
enum irdma_sd_entry_type type, u64 direct_mode_sz)
{
struct irdma_hmc_sd_entry *sd_entry;
struct irdma_dma_mem dma_mem;
u64 alloc_len;
sd_entry = &hmc_info->sd_table.sd_entry[sd_index];
if (!sd_entry->valid) {
if (type == IRDMA_SD_TYPE_PAGED)
alloc_len = IRDMA_HMC_PAGED_BP_SIZE;
else
alloc_len = direct_mode_sz;
/* allocate a 4K pd page or 2M backing page */
dma_mem.size = ALIGN(alloc_len, IRDMA_HMC_PD_BP_BUF_ALIGNMENT);
dma_mem.va = dma_alloc_coherent(hw->device, dma_mem.size,
&dma_mem.pa, GFP_KERNEL);
if (!dma_mem.va)
return -ENOMEM;
if (type == IRDMA_SD_TYPE_PAGED) {
struct irdma_virt_mem *vmem =
&sd_entry->u.pd_table.pd_entry_virt_mem;
vmem->size = sizeof(struct irdma_hmc_pd_entry) * 512;
vmem->va = kzalloc(vmem->size, GFP_KERNEL);
if (!vmem->va) {
dma_free_coherent(hw->device, dma_mem.size,
dma_mem.va, dma_mem.pa);
dma_mem.va = NULL;
return -ENOMEM;
}
sd_entry->u.pd_table.pd_entry = vmem->va;
memcpy(&sd_entry->u.pd_table.pd_page_addr, &dma_mem,
sizeof(sd_entry->u.pd_table.pd_page_addr));
} else {
memcpy(&sd_entry->u.bp.addr, &dma_mem,
sizeof(sd_entry->u.bp.addr));
sd_entry->u.bp.sd_pd_index = sd_index;
}
hmc_info->sd_table.sd_entry[sd_index].entry_type = type;
hmc_info->sd_table.use_cnt++;
}
if (sd_entry->entry_type == IRDMA_SD_TYPE_DIRECT)
sd_entry->u.bp.use_cnt++;
return 0;
}
/**
* irdma_add_pd_table_entry - Adds page descriptor to the specified table
* @dev: pointer to our device structure
* @hmc_info: pointer to the HMC configuration information structure
* @pd_index: which page descriptor index to manipulate
* @rsrc_pg: if not NULL, use preallocated page instead of allocating new one.
*
* This function:
* 1. Initializes the pd entry
* 2. Adds pd_entry in the pd_table
* 3. Mark the entry valid in irdma_hmc_pd_entry structure
* 4. Initializes the pd_entry's ref count to 1
* assumptions:
* 1. The memory for pd should be pinned down, physically contiguous and
* aligned on 4K boundary and zeroed memory.
* 2. It should be 4K in size.
*/
int irdma_add_pd_table_entry(struct irdma_sc_dev *dev,
struct irdma_hmc_info *hmc_info, u32 pd_index,
struct irdma_dma_mem *rsrc_pg)
{
struct irdma_hmc_pd_table *pd_table;
struct irdma_hmc_pd_entry *pd_entry;
struct irdma_dma_mem mem;
struct irdma_dma_mem *page = &mem;
u32 sd_idx, rel_pd_idx;
u64 *pd_addr;
u64 page_desc;
if (pd_index / IRDMA_HMC_PD_CNT_IN_SD >= hmc_info->sd_table.sd_cnt)
return -EINVAL;
sd_idx = (pd_index / IRDMA_HMC_PD_CNT_IN_SD);
if (hmc_info->sd_table.sd_entry[sd_idx].entry_type !=
IRDMA_SD_TYPE_PAGED)
return 0;
rel_pd_idx = (pd_index % IRDMA_HMC_PD_CNT_IN_SD);
pd_table = &hmc_info->sd_table.sd_entry[sd_idx].u.pd_table;
pd_entry = &pd_table->pd_entry[rel_pd_idx];
if (!pd_entry->valid) {
if (rsrc_pg) {
pd_entry->rsrc_pg = true;
page = rsrc_pg;
} else {
page->size = ALIGN(IRDMA_HMC_PAGED_BP_SIZE,
IRDMA_HMC_PD_BP_BUF_ALIGNMENT);
page->va = dma_alloc_coherent(dev->hw->device,
page->size, &page->pa,
GFP_KERNEL);
if (!page->va)
return -ENOMEM;
pd_entry->rsrc_pg = false;
}
memcpy(&pd_entry->bp.addr, page, sizeof(pd_entry->bp.addr));
pd_entry->bp.sd_pd_index = pd_index;
pd_entry->bp.entry_type = IRDMA_SD_TYPE_PAGED;
page_desc = page->pa | 0x1;
pd_addr = pd_table->pd_page_addr.va;
pd_addr += rel_pd_idx;
memcpy(pd_addr, &page_desc, sizeof(*pd_addr));
pd_entry->sd_index = sd_idx;
pd_entry->valid = true;
pd_table->use_cnt++;
if (hmc_info->hmc_fn_id < dev->hw_attrs.first_hw_vf_fpm_id &&
dev->privileged)
irdma_invalidate_pf_hmc_pd(dev, sd_idx, rel_pd_idx);
}
pd_entry->bp.use_cnt++;
return 0;
}
/**
* irdma_remove_pd_bp - remove a backing page from a page descriptor
* @dev: pointer to our HW structure
* @hmc_info: pointer to the HMC configuration information structure
* @idx: the page index
*
* This function:
* 1. Marks the entry in pd table (for paged address mode) or in sd table
* (for direct address mode) invalid.
* 2. Write to register PMPDINV to invalidate the backing page in FV cache
* 3. Decrement the ref count for the pd _entry
* assumptions:
* 1. Caller can deallocate the memory used by backing storage after this
* function returns.
*/
int irdma_remove_pd_bp(struct irdma_sc_dev *dev,
struct irdma_hmc_info *hmc_info, u32 idx)
{
struct irdma_hmc_pd_entry *pd_entry;
struct irdma_hmc_pd_table *pd_table;
struct irdma_hmc_sd_entry *sd_entry;
u32 sd_idx, rel_pd_idx;
struct irdma_dma_mem *mem;
u64 *pd_addr;
sd_idx = idx / IRDMA_HMC_PD_CNT_IN_SD;
rel_pd_idx = idx % IRDMA_HMC_PD_CNT_IN_SD;
if (sd_idx >= hmc_info->sd_table.sd_cnt)
return -EINVAL;
sd_entry = &hmc_info->sd_table.sd_entry[sd_idx];
if (sd_entry->entry_type != IRDMA_SD_TYPE_PAGED)
return -EINVAL;
pd_table = &hmc_info->sd_table.sd_entry[sd_idx].u.pd_table;
pd_entry = &pd_table->pd_entry[rel_pd_idx];
if (--pd_entry->bp.use_cnt)
return 0;
pd_entry->valid = false;
pd_table->use_cnt--;
pd_addr = pd_table->pd_page_addr.va;
pd_addr += rel_pd_idx;
memset(pd_addr, 0, sizeof(u64));
if (dev->privileged && dev->hmc_fn_id == hmc_info->hmc_fn_id)
irdma_invalidate_pf_hmc_pd(dev, sd_idx, idx);
if (!pd_entry->rsrc_pg) {
mem = &pd_entry->bp.addr;
if (!mem || !mem->va)
return -EINVAL;
dma_free_coherent(dev->hw->device, mem->size, mem->va,
mem->pa);
mem->va = NULL;
}
if (!pd_table->use_cnt)
kfree(pd_table->pd_entry_virt_mem.va);
return 0;
}
/**
* irdma_prep_remove_sd_bp - Prepares to remove a backing page from a sd entry
* @hmc_info: pointer to the HMC configuration information structure
* @idx: the page index
*/
int irdma_prep_remove_sd_bp(struct irdma_hmc_info *hmc_info, u32 idx)
{
struct irdma_hmc_sd_entry *sd_entry;
sd_entry = &hmc_info->sd_table.sd_entry[idx];
if (--sd_entry->u.bp.use_cnt)
return -EBUSY;
hmc_info->sd_table.use_cnt--;
sd_entry->valid = false;
return 0;
}
/**
* irdma_prep_remove_pd_page - Prepares to remove a PD page from sd entry.
* @hmc_info: pointer to the HMC configuration information structure
* @idx: segment descriptor index to find the relevant page descriptor
*/
int irdma_prep_remove_pd_page(struct irdma_hmc_info *hmc_info, u32 idx)
{
struct irdma_hmc_sd_entry *sd_entry;
sd_entry = &hmc_info->sd_table.sd_entry[idx];
if (sd_entry->u.pd_table.use_cnt)
return -EBUSY;
sd_entry->valid = false;
hmc_info->sd_table.use_cnt--;
return 0;
}
linux-7.2.3/drivers/infiniband/hw/irdma/hmc.h
+186
View File
@@ -0,0 +1,186 @@
/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
/* Copyright (c) 2015 - 2020 Intel Corporation */
#ifndef IRDMA_HMC_H
#define IRDMA_HMC_H
#include "defs.h"
#define IRDMA_HMC_MAX_BP_COUNT 512
#define IRDMA_MAX_SD_ENTRIES 11
#define IRDMA_HW_DBG_HMC_INVALID_BP_MARK 0xca
#define IRDMA_HMC_INFO_SIGNATURE 0x484d5347
#define IRDMA_HMC_PD_CNT_IN_SD 512
#define IRDMA_HMC_DIRECT_BP_SIZE 0x200000
#define IRDMA_HMC_MAX_SD_COUNT 8192
#define IRDMA_HMC_PAGED_BP_SIZE 4096
#define IRDMA_HMC_PD_BP_BUF_ALIGNMENT 4096
#define IRDMA_FIRST_VF_FPM_ID 8
#define FPM_MULTIPLIER 1024
#define IRDMA_OBJ_LOC_MEM_BIT 0x4
#define IRDMA_XF_MULTIPLIER 16
#define IRDMA_RRF_MULTIPLIER 8
#define IRDMA_MIN_PBLE_PAGES 3
#define IRDMA_HMC_PAGE_SIZE 2097152
#define IRDMA_MIN_MR_PER_QP 4
#define IRDMA_MIN_QP_CNT 64
#define IRDMA_FSIAV_CNT_MAX 1048576
#define IRDMA_MIN_IRD 8
#define IRDMA_HMC_MIN_RRF 16
enum irdma_hmc_rsrc_type {
IRDMA_HMC_IW_QP = 0,
IRDMA_HMC_IW_CQ = 1,
IRDMA_HMC_IW_SRQ = 2,
IRDMA_HMC_IW_HTE = 3,
IRDMA_HMC_IW_ARP = 4,
IRDMA_HMC_IW_APBVT_ENTRY = 5,
IRDMA_HMC_IW_MR = 6,
IRDMA_HMC_IW_XF = 7,
IRDMA_HMC_IW_XFFL = 8,
IRDMA_HMC_IW_Q1 = 9,
IRDMA_HMC_IW_Q1FL = 10,
IRDMA_HMC_IW_TIMER = 11,
IRDMA_HMC_IW_FSIMC = 12,
IRDMA_HMC_IW_FSIAV = 13,
IRDMA_HMC_IW_PBLE = 14,
IRDMA_HMC_IW_RRF = 15,
IRDMA_HMC_IW_RRFFL = 16,
IRDMA_HMC_IW_HDR = 17,
IRDMA_HMC_IW_MD = 18,
IRDMA_HMC_IW_OOISC = 19,
IRDMA_HMC_IW_OOISCFFL = 20,
IRDMA_HMC_IW_MAX, /* Must be last entry */
};
enum irdma_sd_entry_type {
IRDMA_SD_TYPE_INVALID = 0,
IRDMA_SD_TYPE_PAGED = 1,
IRDMA_SD_TYPE_DIRECT = 2,
};
enum irdma_hmc_obj_mem {
IRDMA_HOST_MEM = 0,
IRDMA_LOC_MEM = 1,
};
struct irdma_hmc_obj_info {
u64 base;
u32 max_cnt;
u32 cnt;
u64 size;
enum irdma_hmc_obj_mem mem_loc;
};
struct irdma_hmc_bp {
enum irdma_sd_entry_type entry_type;
struct irdma_dma_mem addr;
u32 sd_pd_index;
u32 use_cnt;
};
struct irdma_hmc_pd_entry {
struct irdma_hmc_bp bp;
u32 sd_index;
bool rsrc_pg:1;
bool valid:1;
};
struct irdma_hmc_pd_table {
struct irdma_dma_mem pd_page_addr;
struct irdma_hmc_pd_entry *pd_entry;
struct irdma_virt_mem pd_entry_virt_mem;
u32 use_cnt;
u32 sd_index;
};
struct irdma_hmc_sd_entry {
enum irdma_sd_entry_type entry_type;
bool valid;
union {
struct irdma_hmc_pd_table pd_table;
struct irdma_hmc_bp bp;
} u;
};
struct irdma_hmc_sd_table {
struct irdma_virt_mem addr;
u32 sd_cnt;
u32 use_cnt;
struct irdma_hmc_sd_entry *sd_entry;
};
struct irdma_hmc_info {
u32 signature;
u8 hmc_fn_id;
u16 first_sd_index;
struct irdma_hmc_obj_info *hmc_obj;
struct irdma_virt_mem hmc_obj_virt_mem;
struct irdma_hmc_sd_table sd_table;
u16 sd_indexes[IRDMA_HMC_MAX_SD_COUNT];
};
struct irdma_update_sd_entry {
u64 cmd;
u64 data;
};
struct irdma_update_sds_info {
u32 cnt;
u8 hmc_fn_id;
struct irdma_update_sd_entry entry[IRDMA_MAX_SD_ENTRIES];
};
struct irdma_ccq_cqe_info;
struct irdma_hmc_fcn_info {
u32 vf_id;
u8 protocol_used;
u8 free_fcn;
};
struct irdma_hmc_create_obj_info {
struct irdma_hmc_info *hmc_info;
struct irdma_virt_mem add_sd_virt_mem;
u32 rsrc_type;
u32 start_idx;
u32 count;
u32 add_sd_cnt;
enum irdma_sd_entry_type entry_type;
bool privileged;
};
struct irdma_hmc_del_obj_info {
struct irdma_hmc_info *hmc_info;
struct irdma_virt_mem del_sd_virt_mem;
u32 rsrc_type;
u32 start_idx;
u32 count;
u32 del_sd_cnt;
bool privileged;
};
int irdma_copy_dma_mem(struct irdma_hw *hw, void *dest_buf,
struct irdma_dma_mem *src_mem, u64 src_offset, u64 size);
int irdma_sc_create_hmc_obj(struct irdma_sc_dev *dev,
struct irdma_hmc_create_obj_info *info);
int irdma_sc_del_hmc_obj(struct irdma_sc_dev *dev,
struct irdma_hmc_del_obj_info *info, bool reset);
int irdma_hmc_sd_one(struct irdma_sc_dev *dev, u8 hmc_fn_id, u64 pa, u32 sd_idx,
enum irdma_sd_entry_type type,
bool setsd);
int irdma_update_sds_noccq(struct irdma_sc_dev *dev,
struct irdma_update_sds_info *info);
struct irdma_vfdev *irdma_vfdev_from_fpm(struct irdma_sc_dev *dev,
u8 hmc_fn_id);
struct irdma_hmc_info *irdma_vf_hmcinfo_from_fpm(struct irdma_sc_dev *dev,
u8 hmc_fn_id);
int irdma_add_sd_table_entry(struct irdma_hw *hw,
struct irdma_hmc_info *hmc_info, u32 sd_index,
enum irdma_sd_entry_type type, u64 direct_mode_sz);
int irdma_add_pd_table_entry(struct irdma_sc_dev *dev,
struct irdma_hmc_info *hmc_info, u32 pd_index,
struct irdma_dma_mem *rsrc_pg);
int irdma_remove_pd_bp(struct irdma_sc_dev *dev,
struct irdma_hmc_info *hmc_info, u32 idx);
int irdma_prep_remove_sd_bp(struct irdma_hmc_info *hmc_info, u32 idx);
int irdma_prep_remove_pd_page(struct irdma_hmc_info *hmc_info, u32 idx);
#endif /* IRDMA_HMC_H */
linux-7.2.3/drivers/infiniband/hw/irdma/hw.c
+2906
View File
File diff suppressed because it is too large Load Diff
linux-7.2.3/drivers/infiniband/hw/irdma/i40iw_hw.c
+261
View File
@@ -0,0 +1,261 @@
// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
/* Copyright (c) 2015 - 2021 Intel Corporation */
#include "osdep.h"
#include "type.h"
#include "i40iw_hw.h"
#include "protos.h"
static u32 i40iw_regs[IRDMA_MAX_REGS] = {
I40E_PFPE_CQPTAIL,
I40E_PFPE_CQPDB,
I40E_PFPE_CCQPSTATUS,
I40E_PFPE_CCQPHIGH,
I40E_PFPE_CCQPLOW,
I40E_PFPE_CQARM,
I40E_PFPE_CQACK,
I40E_PFPE_AEQALLOC,
I40E_PFPE_CQPERRCODES,
I40E_PFPE_WQEALLOC,
I40E_PFINT_DYN_CTLN(0),
I40IW_DB_ADDR_OFFSET,
I40E_GLPCI_LBARCTRL,
I40E_GLPE_CPUSTATUS0,
I40E_GLPE_CPUSTATUS1,
I40E_GLPE_CPUSTATUS2,
I40E_PFINT_AEQCTL,
I40E_PFINT_CEQCTL(0),
I40E_VSIQF_CTL(0),
I40E_PFHMC_PDINV,
I40E_GLHMC_VFPDINV(0),
I40E_GLPE_CRITERR,
0xffffffff /* PFINT_RATEN not used in FPK */
};
static u32 i40iw_stat_offsets[] = {
I40E_GLPES_PFIP4RXDISCARD(0),
I40E_GLPES_PFIP4RXTRUNC(0),
I40E_GLPES_PFIP4TXNOROUTE(0),
I40E_GLPES_PFIP6RXDISCARD(0),
I40E_GLPES_PFIP6RXTRUNC(0),
I40E_GLPES_PFIP6TXNOROUTE(0),
I40E_GLPES_PFTCPRTXSEG(0),
I40E_GLPES_PFTCPRXOPTERR(0),
I40E_GLPES_PFTCPRXPROTOERR(0),
I40E_GLPES_PFRXVLANERR(0),
I40E_GLPES_PFIP4RXOCTSLO(0),
I40E_GLPES_PFIP4RXPKTSLO(0),
I40E_GLPES_PFIP4RXFRAGSLO(0),
I40E_GLPES_PFIP4RXMCPKTSLO(0),
I40E_GLPES_PFIP4TXOCTSLO(0),
I40E_GLPES_PFIP4TXPKTSLO(0),
I40E_GLPES_PFIP4TXFRAGSLO(0),
I40E_GLPES_PFIP4TXMCPKTSLO(0),
I40E_GLPES_PFIP6RXOCTSLO(0),
I40E_GLPES_PFIP6RXPKTSLO(0),
I40E_GLPES_PFIP6RXFRAGSLO(0),
I40E_GLPES_PFIP6RXMCPKTSLO(0),
I40E_GLPES_PFIP6TXOCTSLO(0),
I40E_GLPES_PFIP6TXPKTSLO(0),
I40E_GLPES_PFIP6TXFRAGSLO(0),
I40E_GLPES_PFIP6TXMCPKTSLO(0),
I40E_GLPES_PFTCPRXSEGSLO(0),
I40E_GLPES_PFTCPTXSEGLO(0),
I40E_GLPES_PFRDMARXRDSLO(0),
I40E_GLPES_PFRDMARXSNDSLO(0),
I40E_GLPES_PFRDMARXWRSLO(0),
I40E_GLPES_PFRDMATXRDSLO(0),
I40E_GLPES_PFRDMATXSNDSLO(0),
I40E_GLPES_PFRDMATXWRSLO(0),
I40E_GLPES_PFRDMAVBNDLO(0),
I40E_GLPES_PFRDMAVINVLO(0),
I40E_GLPES_PFIP4RXMCOCTSLO(0),
I40E_GLPES_PFIP4TXMCOCTSLO(0),
I40E_GLPES_PFIP6RXMCOCTSLO(0),
I40E_GLPES_PFIP6TXMCOCTSLO(0),
I40E_GLPES_PFUDPRXPKTSLO(0),
I40E_GLPES_PFUDPTXPKTSLO(0)
};
static u64 i40iw_masks[IRDMA_MAX_MASKS] = {
I40E_PFPE_CCQPSTATUS_CCQP_DONE,
I40E_PFPE_CCQPSTATUS_CCQP_ERR,
I40E_CQPSQ_STAG_PDID,
I40E_CQPSQ_CQ_CEQID,
I40E_CQPSQ_CQ_CQID,
I40E_COMMIT_FPM_CQCNT,
I40E_CQPSQ_UPESD_HMCFNID,
};
static u64 i40iw_shifts[IRDMA_MAX_SHIFTS] = {
I40E_PFPE_CCQPSTATUS_CCQP_DONE_S,
I40E_PFPE_CCQPSTATUS_CCQP_ERR_S,
I40E_CQPSQ_STAG_PDID_S,
I40E_CQPSQ_CQ_CEQID_S,
I40E_CQPSQ_CQ_CQID_S,
I40E_COMMIT_FPM_CQCNT_S,
I40E_CQPSQ_UPESD_HMCFNID_S,
};
/**
* i40iw_config_ceq- Configure CEQ interrupt
* @dev: pointer to the device structure
* @ceq_id: Completion Event Queue ID
* @idx: vector index
* @enable: Enable CEQ interrupt when true
*/
static void i40iw_config_ceq(struct irdma_sc_dev *dev, u32 ceq_id, u32 idx,
bool enable)
{
u32 reg_val;
reg_val = FIELD_PREP(I40E_PFINT_LNKLSTN_FIRSTQ_INDX, ceq_id) |
FIELD_PREP(I40E_PFINT_LNKLSTN_FIRSTQ_TYPE, QUEUE_TYPE_CEQ);
wr32(dev->hw, I40E_PFINT_LNKLSTN(idx - 1), reg_val);
reg_val = FIELD_PREP(I40E_PFINT_DYN_CTLN_ITR_INDX, 0x3) |
FIELD_PREP(I40E_PFINT_DYN_CTLN_INTENA, 0x1);
wr32(dev->hw, I40E_PFINT_DYN_CTLN(idx - 1), reg_val);
reg_val = FIELD_PREP(IRDMA_GLINT_CEQCTL_CAUSE_ENA, enable) |
FIELD_PREP(IRDMA_GLINT_CEQCTL_MSIX_INDX, idx) |
FIELD_PREP(I40E_PFINT_CEQCTL_NEXTQ_INDX, NULL_QUEUE_INDEX) |
FIELD_PREP(IRDMA_GLINT_CEQCTL_ITR_INDX, 0x3);
wr32(dev->hw, i40iw_regs[IRDMA_GLINT_CEQCTL] + 4 * ceq_id, reg_val);
}
/**
* i40iw_ena_irq - Enable interrupt
* @dev: pointer to the device structure
* @idx: vector index
*/
static void i40iw_ena_irq(struct irdma_sc_dev *dev, u32 idx)
{
u32 val;
val = FIELD_PREP(IRDMA_GLINT_DYN_CTL_INTENA, 0x1) |
FIELD_PREP(IRDMA_GLINT_DYN_CTL_CLEARPBA, 0x1) |
FIELD_PREP(IRDMA_GLINT_DYN_CTL_ITR_INDX, 0x3);
wr32(dev->hw, i40iw_regs[IRDMA_GLINT_DYN_CTL] + 4 * (idx - 1), val);
}
/**
* i40iw_disable_irq - Disable interrupt
* @dev: pointer to the device structure
* @idx: vector index
*/
static void i40iw_disable_irq(struct irdma_sc_dev *dev, u32 idx)
{
wr32(dev->hw, i40iw_regs[IRDMA_GLINT_DYN_CTL] + 4 * (idx - 1), 0);
}
static const struct irdma_irq_ops i40iw_irq_ops = {
.irdma_cfg_aeq = irdma_cfg_aeq,
.irdma_cfg_ceq = i40iw_config_ceq,
.irdma_dis_irq = i40iw_disable_irq,
.irdma_en_irq = i40iw_ena_irq,
};
static const struct irdma_hw_stat_map i40iw_hw_stat_map[] = {
[IRDMA_HW_STAT_INDEX_RXVLANERR] = { 0, 0, IRDMA_MAX_STATS_24 },
[IRDMA_HW_STAT_INDEX_IP4RXOCTS] = { 8, 0, IRDMA_MAX_STATS_48 },
[IRDMA_HW_STAT_INDEX_IP4RXPKTS] = { 16, 0, IRDMA_MAX_STATS_48 },
[IRDMA_HW_STAT_INDEX_IP4RXDISCARD] = { 24, 0, IRDMA_MAX_STATS_32 },
[IRDMA_HW_STAT_INDEX_IP4RXTRUNC] = { 32, 0, IRDMA_MAX_STATS_32 },
[IRDMA_HW_STAT_INDEX_IP4RXFRAGS] = { 40, 0, IRDMA_MAX_STATS_48 },
[IRDMA_HW_STAT_INDEX_IP4RXMCPKTS] = { 48, 0, IRDMA_MAX_STATS_48 },
[IRDMA_HW_STAT_INDEX_IP6RXOCTS] = { 56, 0, IRDMA_MAX_STATS_48 },
[IRDMA_HW_STAT_INDEX_IP6RXPKTS] = { 64, 0, IRDMA_MAX_STATS_48 },
[IRDMA_HW_STAT_INDEX_IP6RXDISCARD] = { 72, 0, IRDMA_MAX_STATS_32 },
[IRDMA_HW_STAT_INDEX_IP6RXTRUNC] = { 80, 0, IRDMA_MAX_STATS_32 },
[IRDMA_HW_STAT_INDEX_IP6RXFRAGS] = { 88, 0, IRDMA_MAX_STATS_48 },
[IRDMA_HW_STAT_INDEX_IP6RXMCPKTS] = { 96, 0, IRDMA_MAX_STATS_48 },
[IRDMA_HW_STAT_INDEX_IP4TXOCTS] = { 104, 0, IRDMA_MAX_STATS_48 },
[IRDMA_HW_STAT_INDEX_IP4TXPKTS] = { 112, 0, IRDMA_MAX_STATS_48 },
[IRDMA_HW_STAT_INDEX_IP4TXFRAGS] = { 120, 0, IRDMA_MAX_STATS_48 },
[IRDMA_HW_STAT_INDEX_IP4TXMCPKTS] = { 128, 0, IRDMA_MAX_STATS_48 },
[IRDMA_HW_STAT_INDEX_IP6TXOCTS] = { 136, 0, IRDMA_MAX_STATS_48 },
[IRDMA_HW_STAT_INDEX_IP6TXPKTS] = { 144, 0, IRDMA_MAX_STATS_48 },
[IRDMA_HW_STAT_INDEX_IP6TXFRAGS] = { 152, 0, IRDMA_MAX_STATS_48 },
[IRDMA_HW_STAT_INDEX_IP6TXMCPKTS] = { 160, 0, IRDMA_MAX_STATS_48 },
[IRDMA_HW_STAT_INDEX_IP4TXNOROUTE] = { 168, 0, IRDMA_MAX_STATS_24 },
[IRDMA_HW_STAT_INDEX_IP6TXNOROUTE] = { 176, 0, IRDMA_MAX_STATS_24 },
[IRDMA_HW_STAT_INDEX_TCPRXSEGS] = { 184, 0, IRDMA_MAX_STATS_48 },
[IRDMA_HW_STAT_INDEX_TCPRXOPTERR] = { 192, 0, IRDMA_MAX_STATS_24 },
[IRDMA_HW_STAT_INDEX_TCPRXPROTOERR] = { 200, 0, IRDMA_MAX_STATS_24 },
[IRDMA_HW_STAT_INDEX_TCPTXSEG] = { 208, 0, IRDMA_MAX_STATS_48 },
[IRDMA_HW_STAT_INDEX_TCPRTXSEG] = { 216, 0, IRDMA_MAX_STATS_32 },
[IRDMA_HW_STAT_INDEX_RDMARXWRS] = { 224, 0, IRDMA_MAX_STATS_48 },
[IRDMA_HW_STAT_INDEX_RDMARXRDS] = { 232, 0, IRDMA_MAX_STATS_48 },
[IRDMA_HW_STAT_INDEX_RDMARXSNDS] = { 240, 0, IRDMA_MAX_STATS_48 },
[IRDMA_HW_STAT_INDEX_RDMATXWRS] = { 248, 0, IRDMA_MAX_STATS_48 },
[IRDMA_HW_STAT_INDEX_RDMATXRDS] = { 256, 0, IRDMA_MAX_STATS_48 },
[IRDMA_HW_STAT_INDEX_RDMATXSNDS] = { 264, 0, IRDMA_MAX_STATS_48 },
[IRDMA_HW_STAT_INDEX_RDMAVBND] = { 272, 0, IRDMA_MAX_STATS_48 },
[IRDMA_HW_STAT_INDEX_RDMAVINV] = { 280, 0, IRDMA_MAX_STATS_48 },
[IRDMA_HW_STAT_INDEX_IP4RXMCOCTS] = { 288, 0, IRDMA_MAX_STATS_48 },
[IRDMA_HW_STAT_INDEX_IP4TXMCOCTS] = { 296, 0, IRDMA_MAX_STATS_48 },
[IRDMA_HW_STAT_INDEX_IP6RXMCOCTS] = { 304, 0, IRDMA_MAX_STATS_48 },
[IRDMA_HW_STAT_INDEX_IP6TXMCOCTS] = { 312, 0, IRDMA_MAX_STATS_48 },
[IRDMA_HW_STAT_INDEX_UDPRXPKTS] = { 320, 0, IRDMA_MAX_STATS_48 },
[IRDMA_HW_STAT_INDEX_UDPTXPKTS] = { 328, 0, IRDMA_MAX_STATS_48 },
};
void i40iw_init_hw(struct irdma_sc_dev *dev)
{
int i;
u8 __iomem *hw_addr;
for (i = 0; i < IRDMA_MAX_REGS; ++i) {
hw_addr = dev->hw->hw_addr;
if (i == IRDMA_DB_ADDR_OFFSET)
hw_addr = NULL;
dev->hw_regs[i] = (u32 __iomem *)(i40iw_regs[i] + hw_addr);
}
for (i = 0; i < IRDMA_HW_STAT_INDEX_MAX_GEN_1; ++i)
dev->hw_stats_regs[i] = i40iw_stat_offsets[i];
dev->hw_attrs.first_hw_vf_fpm_id = I40IW_FIRST_VF_FPM_ID;
dev->hw_attrs.max_hw_vf_fpm_id = IRDMA_MAX_VF_FPM_ID;
for (i = 0; i < IRDMA_MAX_SHIFTS; ++i)
dev->hw_shifts[i] = i40iw_shifts[i];
for (i = 0; i < IRDMA_MAX_MASKS; ++i)
dev->hw_masks[i] = i40iw_masks[i];
dev->wqe_alloc_db = dev->hw_regs[IRDMA_WQEALLOC];
dev->cq_arm_db = dev->hw_regs[IRDMA_CQARM];
dev->aeq_alloc_db = dev->hw_regs[IRDMA_AEQALLOC];
dev->cqp_db = dev->hw_regs[IRDMA_CQPDB];
dev->cq_ack_db = dev->hw_regs[IRDMA_CQACK];
dev->ceq_itr_mask_db = NULL;
dev->aeq_itr_mask_db = NULL;
dev->irq_ops = &i40iw_irq_ops;
dev->hw_stats_map = i40iw_hw_stat_map;
/* Setup the hardware limits, hmc may limit further */
dev->hw_attrs.uk_attrs.max_hw_wq_frags = I40IW_MAX_WQ_FRAGMENT_COUNT;
dev->hw_attrs.uk_attrs.max_hw_read_sges = I40IW_MAX_SGE_RD;
dev->hw_attrs.max_hw_device_pages = I40IW_MAX_PUSH_PAGE_COUNT;
dev->hw_attrs.uk_attrs.max_hw_inline = I40IW_MAX_INLINE_DATA_SIZE;
dev->hw_attrs.page_size_cap = SZ_4K | SZ_2M;
dev->hw_attrs.max_hw_ird = I40IW_MAX_IRD_SIZE;
dev->hw_attrs.max_hw_ord = I40IW_MAX_ORD_SIZE;
dev->hw_attrs.max_hw_wqes = I40IW_MAX_WQ_ENTRIES;
dev->hw_attrs.uk_attrs.max_hw_rq_quanta = I40IW_QP_SW_MAX_RQ_QUANTA;
dev->hw_attrs.uk_attrs.max_hw_wq_quanta = I40IW_QP_SW_MAX_WQ_QUANTA;
dev->hw_attrs.uk_attrs.max_hw_sq_chunk = I40IW_MAX_QUANTA_PER_WR;
dev->hw_attrs.max_hw_pds = I40IW_MAX_PDS;
dev->hw_attrs.max_stat_inst = I40IW_MAX_STATS_COUNT;
dev->hw_attrs.max_stat_idx = IRDMA_HW_STAT_INDEX_MAX_GEN_1;
dev->hw_attrs.max_hw_outbound_msg_size = I40IW_MAX_OUTBOUND_MSG_SIZE;
dev->hw_attrs.max_hw_inbound_msg_size = I40IW_MAX_INBOUND_MSG_SIZE;
dev->hw_attrs.uk_attrs.min_hw_wq_size = I40IW_MIN_WQ_SIZE;
dev->hw_attrs.max_qp_wr = I40IW_MAX_QP_WRS;
}
linux-7.2.3/drivers/infiniband/hw/irdma/i40iw_hw.h
+162
View File
@@ -0,0 +1,162 @@
/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
/* Copyright (c) 2015 - 2021 Intel Corporation */
#ifndef I40IW_HW_H
#define I40IW_HW_H
#define I40E_VFPE_CQPTAIL1 0x0000A000 /* Reset: VFR */
#define I40E_VFPE_CQPDB1 0x0000BC00 /* Reset: VFR */
#define I40E_VFPE_CCQPSTATUS1 0x0000B800 /* Reset: VFR */
#define I40E_VFPE_CCQPHIGH1 0x00009800 /* Reset: VFR */
#define I40E_VFPE_CCQPLOW1 0x0000AC00 /* Reset: VFR */
#define I40E_VFPE_CQARM1 0x0000B400 /* Reset: VFR */
#define I40E_VFPE_CQACK1 0x0000B000 /* Reset: VFR */
#define I40E_VFPE_AEQALLOC1 0x0000A400 /* Reset: VFR */
#define I40E_VFPE_CQPERRCODES1 0x00009C00 /* Reset: VFR */
#define I40E_VFPE_WQEALLOC1 0x0000C000 /* Reset: VFR */
#define I40E_VFINT_DYN_CTLN(_INTVF) (0x00024800 + ((_INTVF) * 4)) /* _i=0...511 */ /* Reset: VFR */
#define I40E_PFPE_CQPTAIL 0x00008080 /* Reset: PFR */
#define I40E_PFPE_CQPDB 0x00008000 /* Reset: PFR */
#define I40E_PFPE_CCQPSTATUS 0x00008100 /* Reset: PFR */
#define I40E_PFPE_CCQPHIGH 0x00008200 /* Reset: PFR */
#define I40E_PFPE_CCQPLOW 0x00008180 /* Reset: PFR */
#define I40E_PFPE_CQARM 0x00131080 /* Reset: PFR */
#define I40E_PFPE_CQACK 0x00131100 /* Reset: PFR */
#define I40E_PFPE_AEQALLOC 0x00131180 /* Reset: PFR */
#define I40E_PFPE_CQPERRCODES 0x00008880 /* Reset: PFR */
#define I40E_PFPE_WQEALLOC 0x00138C00 /* Reset: PFR */
#define I40E_GLPCI_LBARCTRL 0x000BE484 /* Reset: POR */
#define I40E_GLPE_CPUSTATUS0 0x0000D040 /* Reset: PE_CORER */
#define I40E_GLPE_CPUSTATUS1 0x0000D044 /* Reset: PE_CORER */
#define I40E_GLPE_CPUSTATUS2 0x0000D048 /* Reset: PE_CORER */
#define I40E_GLPE_CRITERR 0x000B4000 /* Reset: PE_CORER */
#define I40E_PFHMC_PDINV 0x000C0300 /* Reset: PFR */
#define I40E_GLHMC_VFPDINV(_i) (0x000C8300 + ((_i) * 4)) /* _i=0...31 */ /* Reset: CORER */
#define I40E_PFINT_DYN_CTLN(_INTPF) (0x00034800 + ((_INTPF) * 4)) /* _i=0...511 */ /* Reset: PFR */
#define I40E_PFINT_AEQCTL 0x00038700 /* Reset: CORER */
#define I40E_GLPES_PFIP4RXDISCARD(_i) (0x00010600 + ((_i) * 4)) /* _i=0...15 */ /* Reset: PE_CORER */
#define I40E_GLPES_PFIP4RXTRUNC(_i) (0x00010700 + ((_i) * 4)) /* _i=0...15 */ /* Reset: PE_CORER */
#define I40E_GLPES_PFIP4TXNOROUTE(_i) (0x00012E00 + ((_i) * 4)) /* _i=0...15 */ /* Reset: PE_CORER */
#define I40E_GLPES_PFIP6RXDISCARD(_i) (0x00011200 + ((_i) * 4)) /* _i=0...15 */ /* Reset: PE_CORER */
#define I40E_GLPES_PFIP6RXTRUNC(_i) (0x00011300 + ((_i) * 4)) /* _i=0...15 */ /* Reset: PE_CORER */
#define I40E_GLPES_PFRDMAVBNDLO(_i) (0x00014800 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
#define I40E_GLPES_PFIP4TXMCOCTSLO(_i) (0x00012000 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
#define I40E_GLPES_PFIP6RXMCOCTSLO(_i) (0x00011600 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
#define I40E_GLPES_PFIP6TXMCOCTSLO(_i) (0x00012A00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
#define I40E_GLPES_PFUDPRXPKTSLO(_i) (0x00013800 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
#define I40E_GLPES_PFUDPTXPKTSLO(_i) (0x00013A00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
#define I40E_GLPES_PFIP6TXNOROUTE(_i) (0x00012F00 + ((_i) * 4)) /* _i=0...15 */ /* Reset: PE_CORER */
#define I40E_GLPES_PFTCPRTXSEG(_i) (0x00013600 + ((_i) * 4)) /* _i=0...15 */ /* Reset: PE_CORER */
#define I40E_GLPES_PFTCPRXOPTERR(_i) (0x00013200 + ((_i) * 4)) /* _i=0...15 */ /* Reset: PE_CORER */
#define I40E_GLPES_PFTCPRXPROTOERR(_i) (0x00013300 + ((_i) * 4)) /* _i=0...15 */ /* Reset: PE_CORER */
#define I40E_GLPES_PFRXVLANERR(_i) (0x00010000 + ((_i) * 4)) /* _i=0...15 */ /* Reset: PE_CORER */
#define I40E_GLPES_PFIP4RXOCTSLO(_i) (0x00010200 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
#define I40E_GLPES_PFIP4RXPKTSLO(_i) (0x00010400 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
#define I40E_GLPES_PFIP4RXFRAGSLO(_i) (0x00010800 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
#define I40E_GLPES_PFIP4RXMCPKTSLO(_i) (0x00010C00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
#define I40E_GLPES_PFIP4TXOCTSLO(_i) (0x00011A00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
#define I40E_GLPES_PFIP4TXPKTSLO(_i) (0x00011C00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
#define I40E_GLPES_PFIP4TXFRAGSLO(_i) (0x00011E00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
#define I40E_GLPES_PFIP4TXMCPKTSLO(_i) (0x00012200 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
#define I40E_GLPES_PFIP6RXOCTSLO(_i) (0x00010E00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
#define I40E_GLPES_PFIP6RXPKTSLO(_i) (0x00011000 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
#define I40E_GLPES_PFIP6RXFRAGSLO(_i) (0x00011400 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
#define I40E_GLPES_PFIP6TXOCTSLO(_i) (0x00012400 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
#define I40E_GLPES_PFIP6TXPKTSLO(_i) (0x00012600 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
#define I40E_GLPES_PFIP6TXFRAGSLO(_i) (0x00012800 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
#define I40E_GLPES_PFIP6TXMCPKTSLO(_i) (0x00012C00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
#define I40E_GLPES_PFTCPTXSEGLO(_i) (0x00013400 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
#define I40E_GLPES_PFRDMARXRDSLO(_i) (0x00013E00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
#define I40E_GLPES_PFRDMARXSNDSLO(_i) (0x00014000 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
#define I40E_GLPES_PFRDMARXWRSLO(_i) (0x00013C00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
#define I40E_GLPES_PFRDMATXRDSLO(_i) (0x00014400 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
#define I40E_GLPES_PFRDMATXSNDSLO(_i) (0x00014600 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
#define I40E_GLPES_PFRDMATXWRSLO(_i) (0x00014200 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
#define I40E_GLPES_PFIP4RXMCOCTSLO(_i) (0x00010A00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
#define I40E_GLPES_PFIP6RXMCPKTSLO(_i) (0x00011800 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
#define I40E_GLPES_PFTCPRXSEGSLO(_i) (0x00013000 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
#define I40E_GLPES_PFRDMAVINVLO(_i) (0x00014A00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
#define I40IW_DB_ADDR_OFFSET (4 * 1024 * 1024 - 64 * 1024)
#define I40IW_VF_DB_ADDR_OFFSET (64 * 1024)
#define I40E_PFINT_LNKLSTN(_INTPF) (0x00035000 + ((_INTPF) * 4)) /* _i=0...511 */ /* Reset: PFR */
#define I40E_PFINT_LNKLSTN_MAX_INDEX 511
#define I40E_PFINT_LNKLSTN_FIRSTQ_INDX GENMASK(10, 0)
#define I40E_PFINT_LNKLSTN_FIRSTQ_TYPE GENMASK(12, 11)
#define I40E_PFINT_CEQCTL(_INTPF) (0x00036800 + ((_INTPF) * 4)) /* _i=0...511 */ /* Reset: CORER */
#define I40E_PFINT_CEQCTL_MAX_INDEX 511
/* shifts/masks for FLD_[LS/RS]_64 macros used in device table */
#define I40E_PFINT_CEQCTL_MSIX_INDX_S 0
#define I40E_PFINT_CEQCTL_MSIX_INDX GENMASK(7, 0)
#define I40E_PFINT_CEQCTL_ITR_INDX_S 11
#define I40E_PFINT_CEQCTL_ITR_INDX GENMASK(12, 11)
#define I40E_PFINT_CEQCTL_MSIX0_INDX_S 13
#define I40E_PFINT_CEQCTL_MSIX0_INDX GENMASK(15, 13)
#define I40E_PFINT_CEQCTL_NEXTQ_INDX_S 16
#define I40E_PFINT_CEQCTL_NEXTQ_INDX GENMASK(26, 16)
#define I40E_PFINT_CEQCTL_NEXTQ_TYPE_S 27
#define I40E_PFINT_CEQCTL_NEXTQ_TYPE GENMASK(28, 27)
#define I40E_PFINT_CEQCTL_CAUSE_ENA_S 30
#define I40E_PFINT_CEQCTL_CAUSE_ENA BIT(30)
#define I40E_PFINT_CEQCTL_INTEVENT_S 31
#define I40E_PFINT_CEQCTL_INTEVENT BIT(31)
#define I40E_CQPSQ_STAG_PDID_S 48
#define I40E_CQPSQ_STAG_PDID GENMASK_ULL(62, 48)
#define I40E_PFPE_CCQPSTATUS_CCQP_DONE_S 0
#define I40E_PFPE_CCQPSTATUS_CCQP_DONE BIT_ULL(0)
#define I40E_PFPE_CCQPSTATUS_CCQP_ERR_S 31
#define I40E_PFPE_CCQPSTATUS_CCQP_ERR BIT_ULL(31)
#define I40E_PFINT_DYN_CTLN_ITR_INDX_S 3
#define I40E_PFINT_DYN_CTLN_ITR_INDX GENMASK(4, 3)
#define I40E_PFINT_DYN_CTLN_INTENA_S 0
#define I40E_PFINT_DYN_CTLN_INTENA BIT(0)
#define I40E_CQPSQ_CQ_CEQID_S 24
#define I40E_CQPSQ_CQ_CEQID GENMASK(30, 24)
#define I40E_CQPSQ_CQ_CQID_S 0
#define I40E_CQPSQ_CQ_CQID GENMASK_ULL(15, 0)
#define I40E_COMMIT_FPM_CQCNT_S 0
#define I40E_COMMIT_FPM_CQCNT GENMASK_ULL(17, 0)
#define I40E_CQPSQ_UPESD_HMCFNID_S 0
#define I40E_CQPSQ_UPESD_HMCFNID GENMASK_ULL(5, 0)
#define I40E_VSIQF_CTL(_VSI) (0x0020D800 + ((_VSI) * 4))
enum i40iw_device_caps_const {
I40IW_MAX_WQ_FRAGMENT_COUNT = 3,
I40IW_MAX_SGE_RD = 1,
I40IW_MAX_PUSH_PAGE_COUNT = 0,
I40IW_MAX_INLINE_DATA_SIZE = 48,
I40IW_MAX_IRD_SIZE = 63,
I40IW_MAX_ORD_SIZE = 127,
I40IW_MAX_WQ_ENTRIES = 2048,
I40IW_MAX_WQE_SIZE_RQ = 128,
I40IW_MAX_PDS = 32768,
I40IW_MAX_STATS_COUNT = 16,
I40IW_MAX_CQ_SIZE = 1048575,
I40IW_MAX_OUTBOUND_MSG_SIZE = 2147483647,
I40IW_MAX_INBOUND_MSG_SIZE = 2147483647,
I40IW_MIN_WQ_SIZE = 4 /* WQEs */,
};
#define I40IW_QP_WQE_MIN_SIZE 32
#define I40IW_QP_WQE_MAX_SIZE 128
#define I40IW_MAX_RQ_WQE_SHIFT 2
#define I40IW_MAX_QUANTA_PER_WR 2
#define I40IW_QP_SW_MAX_SQ_QUANTA 2048
#define I40IW_QP_SW_MAX_RQ_QUANTA 16384
#define I40IW_QP_SW_MAX_WQ_QUANTA 2048
#define I40IW_MAX_QP_WRS ((I40IW_QP_SW_MAX_SQ_QUANTA - IRDMA_SQ_RSVD) / I40IW_MAX_QUANTA_PER_WR)
#define I40IW_FIRST_VF_FPM_ID 16
#define QUEUE_TYPE_CEQ 2
#define NULL_QUEUE_INDEX 0x7FF
void i40iw_init_hw(struct irdma_sc_dev *dev);
#endif /* I40IW_HW_H */
linux-7.2.3/drivers/infiniband/hw/irdma/i40iw_if.c
+220
View File
@@ -0,0 +1,220 @@
// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
/* Copyright (c) 2015 - 2021 Intel Corporation */
#include "main.h"
#include "i40iw_hw.h"
#include <linux/net/intel/i40e_client.h>
static struct i40e_client i40iw_client;
/**
* i40iw_l2param_change - handle mss change
* @cdev_info: parent lan device information structure with data/ops
* @client: client for parameter change
* @params: new parameters from L2
*/
static void i40iw_l2param_change(struct i40e_info *cdev_info,
struct i40e_client *client,
struct i40e_params *params)
{
struct irdma_l2params l2params = {};
struct irdma_device *iwdev;
struct ib_device *ibdev;
ibdev = ib_device_get_by_netdev(cdev_info->netdev, RDMA_DRIVER_IRDMA);
if (!ibdev)
return;
iwdev = to_iwdev(ibdev);
if (iwdev->vsi.mtu != params->mtu) {
l2params.mtu_changed = true;
l2params.mtu = params->mtu;
}
irdma_change_l2params(&iwdev->vsi, &l2params);
ib_device_put(ibdev);
}
/**
* i40iw_close - client interface operation close for iwarp/uda device
* @cdev_info: parent lan device information structure with data/ops
* @client: client to close
* @reset: flag to indicate close on reset
*
* Called by the lan driver during the processing of client unregister
* Destroy and clean up the driver resources
*/
static void i40iw_close(struct i40e_info *cdev_info, struct i40e_client *client,
bool reset)
{
struct irdma_device *iwdev;
struct ib_device *ibdev;
ibdev = ib_device_get_by_netdev(cdev_info->netdev, RDMA_DRIVER_IRDMA);
if (WARN_ON(!ibdev))
return;
iwdev = to_iwdev(ibdev);
if (reset)
iwdev->rf->reset = true;
iwdev->iw_status = 0;
irdma_port_ibevent(iwdev);
ib_unregister_device_and_put(ibdev);
pr_debug("INIT: Gen1 PF[%d] close complete\n", PCI_FUNC(cdev_info->pcidev->devfn));
}
static void i40iw_request_reset(struct irdma_pci_f *rf)
{
struct i40e_info *cdev_info = rf->cdev;
cdev_info->ops->request_reset(cdev_info, &i40iw_client, 1);
}
static void i40iw_fill_device_info(struct irdma_device *iwdev, struct i40e_info *cdev_info)
{
struct irdma_pci_f *rf = iwdev->rf;
rf->rdma_ver = IRDMA_GEN_1;
rf->sc_dev.hw = &rf->hw;
rf->sc_dev.hw_attrs.uk_attrs.hw_rev = IRDMA_GEN_1;
rf->sc_dev.privileged = true;
rf->gen_ops.request_reset = i40iw_request_reset;
rf->pcidev = cdev_info->pcidev;
rf->pf_id = cdev_info->fid;
rf->hw.hw_addr = cdev_info->hw_addr;
rf->cdev = cdev_info;
rf->msix_count = cdev_info->msix_count;
rf->msix_entries = cdev_info->msix_entries;
rf->limits_sel = 5;
rf->protocol_used = IRDMA_IWARP_PROTOCOL_ONLY;
rf->iwdev = iwdev;
iwdev->init_state = INITIAL_STATE;
iwdev->rcv_wnd = IRDMA_CM_DEFAULT_RCV_WND_SCALED;
iwdev->rcv_wscale = IRDMA_CM_DEFAULT_RCV_WND_SCALE;
iwdev->netdev = cdev_info->netdev;
iwdev->vsi_num = 0;
}
/**
* i40iw_open - client interface operation open for iwarp/uda device
* @cdev_info: parent lan device information structure with data/ops
* @client: iwarp client information, provided during registration
*
* Called by the lan driver during the processing of client register
* Create device resources, set up queues, pble and hmc objects and
* register the device with the ib verbs interface
* Return 0 if successful, otherwise return error
*/
static int i40iw_open(struct i40e_info *cdev_info, struct i40e_client *client)
{
struct irdma_l2params l2params = {};
struct irdma_device *iwdev;
struct irdma_pci_f *rf;
int err = -EIO;
int i;
u16 qset;
u16 last_qset = IRDMA_NO_QSET;
iwdev = ib_alloc_device(irdma_device, ibdev);
if (!iwdev)
return -ENOMEM;
iwdev->rf = kzalloc_obj(*rf);
if (!iwdev->rf) {
ib_dealloc_device(&iwdev->ibdev);
return -ENOMEM;
}
i40iw_fill_device_info(iwdev, cdev_info);
rf = iwdev->rf;
if (irdma_ctrl_init_hw(rf)) {
err = -EIO;
goto err_ctrl_init;
}
l2params.mtu = (cdev_info->params.mtu) ? cdev_info->params.mtu : IRDMA_DEFAULT_MTU;
for (i = 0; i < I40E_CLIENT_MAX_USER_PRIORITY; i++) {
qset = cdev_info->params.qos.prio_qos[i].qs_handle;
l2params.up2tc[i] = cdev_info->params.qos.prio_qos[i].tc;
l2params.qs_handle_list[i] = qset;
if (last_qset == IRDMA_NO_QSET)
last_qset = qset;
else if ((qset != last_qset) && (qset != IRDMA_NO_QSET))
iwdev->dcb_vlan_mode = true;
}
if (irdma_rt_init_hw(iwdev, &l2params)) {
err = -EIO;
goto err_rt_init;
}
err = irdma_ib_register_device(iwdev);
if (err)
goto err_ibreg;
ibdev_dbg(&iwdev->ibdev, "INIT: Gen1 PF[%d] open success\n",
PCI_FUNC(rf->pcidev->devfn));
return 0;
err_ibreg:
irdma_rt_deinit_hw(iwdev);
err_rt_init:
irdma_ctrl_deinit_hw(rf);
err_ctrl_init:
kfree(iwdev->rf);
ib_dealloc_device(&iwdev->ibdev);
return err;
}
/* client interface functions */
static const struct i40e_client_ops i40e_ops = {
.open = i40iw_open,
.close = i40iw_close,
.l2_param_change = i40iw_l2param_change
};
static struct i40e_client i40iw_client = {
.ops = &i40e_ops,
.type = I40E_CLIENT_IWARP,
};
static int i40iw_probe(struct auxiliary_device *aux_dev, const struct auxiliary_device_id *id)
{
struct i40e_auxiliary_device *i40e_adev = container_of(aux_dev,
struct i40e_auxiliary_device,
aux_dev);
struct i40e_info *cdev_info = i40e_adev->ldev;
strscpy_pad(i40iw_client.name, "irdma", I40E_CLIENT_STR_LENGTH);
i40e_client_device_register(cdev_info, &i40iw_client);
return 0;
}
static void i40iw_remove(struct auxiliary_device *aux_dev)
{
struct i40e_auxiliary_device *i40e_adev = container_of(aux_dev,
struct i40e_auxiliary_device,
aux_dev);
struct i40e_info *cdev_info = i40e_adev->ldev;
i40e_client_device_unregister(cdev_info);
}
static const struct auxiliary_device_id i40iw_auxiliary_id_table[] = {
{.name = "i40e.iwarp", },
{},
};
MODULE_DEVICE_TABLE(auxiliary, i40iw_auxiliary_id_table);
struct auxiliary_driver i40iw_auxiliary_drv = {
.name = "gen_1",
.id_table = i40iw_auxiliary_id_table,
.probe = i40iw_probe,
.remove = i40iw_remove,
};
linux-7.2.3/drivers/infiniband/hw/irdma/icrdma_hw.c
+205
View File
@@ -0,0 +1,205 @@
// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
/* Copyright (c) 2017 - 2021 Intel Corporation */
#include "osdep.h"
#include "type.h"
#include "icrdma_hw.h"
static u32 icrdma_regs[IRDMA_MAX_REGS] = {
PFPE_CQPTAIL,
PFPE_CQPDB,
PFPE_CCQPSTATUS,
PFPE_CCQPHIGH,
PFPE_CCQPLOW,
PFPE_CQARM,
PFPE_CQACK,
PFPE_AEQALLOC,
PFPE_CQPERRCODES,
PFPE_WQEALLOC,
GLINT_DYN_CTL(0),
ICRDMA_DB_ADDR_OFFSET,
GLPCI_LBARCTRL,
GLPE_CPUSTATUS0,
GLPE_CPUSTATUS1,
GLPE_CPUSTATUS2,
PFINT_AEQCTL,
GLINT_CEQCTL(0),
VSIQF_PE_CTL1(0),
PFHMC_PDINV,
GLHMC_VFPDINV(0),
GLPE_CRITERR,
GLINT_RATE(0),
};
static u64 icrdma_masks[IRDMA_MAX_MASKS] = {
ICRDMA_CCQPSTATUS_CCQP_DONE,
ICRDMA_CCQPSTATUS_CCQP_ERR,
ICRDMA_CQPSQ_STAG_PDID,
ICRDMA_CQPSQ_CQ_CEQID,
ICRDMA_CQPSQ_CQ_CQID,
ICRDMA_COMMIT_FPM_CQCNT,
ICRDMA_CQPSQ_UPESD_HMCFNID,
};
static u64 icrdma_shifts[IRDMA_MAX_SHIFTS] = {
ICRDMA_CCQPSTATUS_CCQP_DONE_S,
ICRDMA_CCQPSTATUS_CCQP_ERR_S,
ICRDMA_CQPSQ_STAG_PDID_S,
ICRDMA_CQPSQ_CQ_CEQID_S,
ICRDMA_CQPSQ_CQ_CQID_S,
ICRDMA_COMMIT_FPM_CQCNT_S,
ICRDMA_CQPSQ_UPESD_HMCFNID_S,
};
/**
* icrdma_ena_irq - Enable interrupt
* @dev: pointer to the device structure
* @idx: vector index
*/
static void icrdma_ena_irq(struct irdma_sc_dev *dev, u32 idx)
{
u32 val;
u32 interval = 0;
if (dev->ceq_itr && dev->aeq->msix_idx != idx)
interval = dev->ceq_itr >> 1; /* 2 usec units */
val = FIELD_PREP(IRDMA_GLINT_DYN_CTL_ITR_INDX, 0) |
FIELD_PREP(IRDMA_GLINT_DYN_CTL_INTERVAL, interval) |
FIELD_PREP(IRDMA_GLINT_DYN_CTL_INTENA, 1) |
FIELD_PREP(IRDMA_GLINT_DYN_CTL_CLEARPBA, 1);
if (dev->hw_attrs.uk_attrs.hw_rev != IRDMA_GEN_1)
writel(val, dev->hw_regs[IRDMA_GLINT_DYN_CTL] + idx);
else
writel(val, dev->hw_regs[IRDMA_GLINT_DYN_CTL] + (idx - 1));
}
/**
* icrdma_disable_irq - Disable interrupt
* @dev: pointer to the device structure
* @idx: vector index
*/
static void icrdma_disable_irq(struct irdma_sc_dev *dev, u32 idx)
{
if (dev->hw_attrs.uk_attrs.hw_rev != IRDMA_GEN_1)
writel(0, dev->hw_regs[IRDMA_GLINT_DYN_CTL] + idx);
else
writel(0, dev->hw_regs[IRDMA_GLINT_DYN_CTL] + (idx - 1));
}
/**
* icrdma_cfg_ceq- Configure CEQ interrupt
* @dev: pointer to the device structure
* @ceq_id: Completion Event Queue ID
* @idx: vector index
* @enable: True to enable, False disables
*/
static void icrdma_cfg_ceq(struct irdma_sc_dev *dev, u32 ceq_id, u32 idx,
bool enable)
{
u32 reg_val;
reg_val = FIELD_PREP(IRDMA_GLINT_CEQCTL_CAUSE_ENA, enable) |
FIELD_PREP(IRDMA_GLINT_CEQCTL_MSIX_INDX, idx) |
FIELD_PREP(IRDMA_GLINT_CEQCTL_ITR_INDX, 3);
writel(reg_val, dev->hw_regs[IRDMA_GLINT_CEQCTL] + ceq_id);
}
static const struct irdma_irq_ops icrdma_irq_ops = {
.irdma_cfg_aeq = irdma_cfg_aeq,
.irdma_cfg_ceq = icrdma_cfg_ceq,
.irdma_dis_irq = icrdma_disable_irq,
.irdma_en_irq = icrdma_ena_irq,
};
static const struct irdma_hw_stat_map icrdma_hw_stat_map[] = {
[IRDMA_HW_STAT_INDEX_RXVLANERR] = { 0, 32, IRDMA_MAX_STATS_24 },
[IRDMA_HW_STAT_INDEX_IP4RXOCTS] = { 8, 0, IRDMA_MAX_STATS_48 },
[IRDMA_HW_STAT_INDEX_IP4RXPKTS] = { 16, 0, IRDMA_MAX_STATS_48 },
[IRDMA_HW_STAT_INDEX_IP4RXDISCARD] = { 24, 32, IRDMA_MAX_STATS_32 },
[IRDMA_HW_STAT_INDEX_IP4RXTRUNC] = { 24, 0, IRDMA_MAX_STATS_32 },
[IRDMA_HW_STAT_INDEX_IP4RXFRAGS] = { 32, 0, IRDMA_MAX_STATS_48 },
[IRDMA_HW_STAT_INDEX_IP4RXMCOCTS] = { 40, 0, IRDMA_MAX_STATS_48 },
[IRDMA_HW_STAT_INDEX_IP4RXMCPKTS] = { 48, 0, IRDMA_MAX_STATS_48 },
[IRDMA_HW_STAT_INDEX_IP6RXOCTS] = { 56, 0, IRDMA_MAX_STATS_48 },
[IRDMA_HW_STAT_INDEX_IP6RXPKTS] = { 64, 0, IRDMA_MAX_STATS_48 },
[IRDMA_HW_STAT_INDEX_IP6RXDISCARD] = { 72, 32, IRDMA_MAX_STATS_32 },
[IRDMA_HW_STAT_INDEX_IP6RXTRUNC] = { 72, 0, IRDMA_MAX_STATS_32 },
[IRDMA_HW_STAT_INDEX_IP6RXFRAGS] = { 80, 0, IRDMA_MAX_STATS_48 },
[IRDMA_HW_STAT_INDEX_IP6RXMCOCTS] = { 88, 0, IRDMA_MAX_STATS_48 },
[IRDMA_HW_STAT_INDEX_IP6RXMCPKTS] = { 96, 0, IRDMA_MAX_STATS_48 },
[IRDMA_HW_STAT_INDEX_IP4TXOCTS] = { 104, 0, IRDMA_MAX_STATS_48 },
[IRDMA_HW_STAT_INDEX_IP4TXPKTS] = { 112, 0, IRDMA_MAX_STATS_48 },
[IRDMA_HW_STAT_INDEX_IP4TXFRAGS] = { 120, 0, IRDMA_MAX_STATS_48 },
[IRDMA_HW_STAT_INDEX_IP4TXMCOCTS] = { 128, 0, IRDMA_MAX_STATS_48 },
[IRDMA_HW_STAT_INDEX_IP4TXMCPKTS] = { 136, 0, IRDMA_MAX_STATS_48 },
[IRDMA_HW_STAT_INDEX_IP6TXOCTS] = { 144, 0, IRDMA_MAX_STATS_48 },
[IRDMA_HW_STAT_INDEX_IP6TXPKTS] = { 152, 0, IRDMA_MAX_STATS_48 },
[IRDMA_HW_STAT_INDEX_IP6TXFRAGS] = { 160, 0, IRDMA_MAX_STATS_48 },
[IRDMA_HW_STAT_INDEX_IP6TXMCOCTS] = { 168, 0, IRDMA_MAX_STATS_48 },
[IRDMA_HW_STAT_INDEX_IP6TXMCPKTS] = { 176, 0, IRDMA_MAX_STATS_48 },
[IRDMA_HW_STAT_INDEX_IP4TXNOROUTE] = { 184, 32, IRDMA_MAX_STATS_24 },
[IRDMA_HW_STAT_INDEX_IP6TXNOROUTE] = { 184, 0, IRDMA_MAX_STATS_24 },
[IRDMA_HW_STAT_INDEX_TCPRXSEGS] = { 192, 32, IRDMA_MAX_STATS_48 },
[IRDMA_HW_STAT_INDEX_TCPRXOPTERR] = { 200, 32, IRDMA_MAX_STATS_24 },
[IRDMA_HW_STAT_INDEX_TCPRXPROTOERR] = { 200, 0, IRDMA_MAX_STATS_24 },
[IRDMA_HW_STAT_INDEX_TCPTXSEG] = { 208, 0, IRDMA_MAX_STATS_48 },
[IRDMA_HW_STAT_INDEX_TCPRTXSEG] = { 216, 32, IRDMA_MAX_STATS_32 },
[IRDMA_HW_STAT_INDEX_UDPRXPKTS] = { 224, 0, IRDMA_MAX_STATS_48 },
[IRDMA_HW_STAT_INDEX_UDPTXPKTS] = { 232, 0, IRDMA_MAX_STATS_48 },
[IRDMA_HW_STAT_INDEX_RDMARXWRS] = { 240, 0, IRDMA_MAX_STATS_48 },
[IRDMA_HW_STAT_INDEX_RDMARXRDS] = { 248, 0, IRDMA_MAX_STATS_48 },
[IRDMA_HW_STAT_INDEX_RDMARXSNDS] = { 256, 0, IRDMA_MAX_STATS_48 },
[IRDMA_HW_STAT_INDEX_RDMATXWRS] = { 264, 0, IRDMA_MAX_STATS_48 },
[IRDMA_HW_STAT_INDEX_RDMATXRDS] = { 272, 0, IRDMA_MAX_STATS_48 },
[IRDMA_HW_STAT_INDEX_RDMATXSNDS] = { 280, 0, IRDMA_MAX_STATS_48 },
[IRDMA_HW_STAT_INDEX_RDMAVBND] = { 288, 0, IRDMA_MAX_STATS_48 },
[IRDMA_HW_STAT_INDEX_RDMAVINV] = { 296, 0, IRDMA_MAX_STATS_48 },
[IRDMA_HW_STAT_INDEX_RXNPECNMARKEDPKTS] = { 304, 0, IRDMA_MAX_STATS_56 },
[IRDMA_HW_STAT_INDEX_RXRPCNPIGNORED] = { 312, 32, IRDMA_MAX_STATS_24 },
[IRDMA_HW_STAT_INDEX_RXRPCNPHANDLED] = { 312, 0, IRDMA_MAX_STATS_32 },
[IRDMA_HW_STAT_INDEX_TXNPCNPSENT] = { 320, 0, IRDMA_MAX_STATS_32 },
};
void icrdma_init_hw(struct irdma_sc_dev *dev)
{
int i;
u8 __iomem *hw_addr;
for (i = 0; i < IRDMA_MAX_REGS; ++i) {
hw_addr = dev->hw->hw_addr;
if (i == IRDMA_DB_ADDR_OFFSET)
hw_addr = NULL;
dev->hw_regs[i] = (u32 __iomem *)(hw_addr + icrdma_regs[i]);
}
dev->hw_attrs.max_hw_vf_fpm_id = IRDMA_MAX_VF_FPM_ID;
dev->hw_attrs.first_hw_vf_fpm_id = IRDMA_FIRST_VF_FPM_ID;
for (i = 0; i < IRDMA_MAX_SHIFTS; ++i)
dev->hw_shifts[i] = icrdma_shifts[i];
for (i = 0; i < IRDMA_MAX_MASKS; ++i)
dev->hw_masks[i] = icrdma_masks[i];
dev->wqe_alloc_db = dev->hw_regs[IRDMA_WQEALLOC];
dev->cq_arm_db = dev->hw_regs[IRDMA_CQARM];
dev->aeq_alloc_db = dev->hw_regs[IRDMA_AEQALLOC];
dev->cqp_db = dev->hw_regs[IRDMA_CQPDB];
dev->cq_ack_db = dev->hw_regs[IRDMA_CQACK];
dev->irq_ops = &icrdma_irq_ops;
dev->hw_attrs.page_size_cap = SZ_4K | SZ_2M | SZ_1G;
dev->hw_stats_map = icrdma_hw_stat_map;
dev->hw_attrs.max_hw_ird = ICRDMA_MAX_IRD_SIZE;
dev->hw_attrs.max_hw_ord = ICRDMA_MAX_ORD_SIZE;
dev->hw_attrs.max_stat_inst = ICRDMA_MAX_STATS_COUNT;
dev->hw_attrs.max_stat_idx = IRDMA_HW_STAT_INDEX_MAX_GEN_2;
dev->hw_attrs.max_hw_device_pages = ICRDMA_MAX_PUSH_PAGE_COUNT;
dev->hw_attrs.uk_attrs.min_hw_wq_size = ICRDMA_MIN_WQ_SIZE;
dev->hw_attrs.uk_attrs.max_hw_sq_chunk = IRDMA_MAX_QUANTA_PER_WR;
dev->hw_attrs.uk_attrs.feature_flags |= IRDMA_FEATURE_RTS_AE |
IRDMA_FEATURE_CQ_RESIZE;
}
linux-7.2.3/drivers/infiniband/hw/irdma/icrdma_hw.h
+73
View File
@@ -0,0 +1,73 @@
/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
/* Copyright (c) 2017 - 2021 Intel Corporation */
#ifndef ICRDMA_HW_H
#define ICRDMA_HW_H
#include "irdma.h"
#define VFPE_CQPTAIL1 0x0000a000
#define VFPE_CQPDB1 0x0000bc00
#define VFPE_CCQPSTATUS1 0x0000b800
#define VFPE_CCQPHIGH1 0x00009800
#define VFPE_CCQPLOW1 0x0000ac00
#define VFPE_CQARM1 0x0000b400
#define VFPE_CQARM1 0x0000b400
#define VFPE_CQACK1 0x0000b000
#define VFPE_AEQALLOC1 0x0000a400
#define VFPE_CQPERRCODES1 0x00009c00
#define VFPE_WQEALLOC1 0x0000c000
#define VFINT_DYN_CTLN(_i) (0x00003800 + ((_i) * 4)) /* _i=0...63 */
#define PFPE_CQPTAIL 0x00500880
#define PFPE_CQPDB 0x00500800
#define PFPE_CCQPSTATUS 0x0050a000
#define PFPE_CCQPHIGH 0x0050a100
#define PFPE_CCQPLOW 0x0050a080
#define PFPE_CQARM 0x00502c00
#define PFPE_CQACK 0x00502c80
#define PFPE_AEQALLOC 0x00502d00
#define GLINT_DYN_CTL(_INT) (0x00160000 + ((_INT) * 4)) /* _i=0...2047 */
#define GLPCI_LBARCTRL 0x0009de74
#define GLPE_CPUSTATUS0 0x0050ba5c
#define GLPE_CPUSTATUS1 0x0050ba60
#define GLPE_CPUSTATUS2 0x0050ba64
#define PFINT_AEQCTL 0x0016cb00
#define PFPE_CQPERRCODES 0x0050a200
#define PFPE_WQEALLOC 0x00504400
#define GLINT_CEQCTL(_INT) (0x0015c000 + ((_INT) * 4)) /* _i=0...2047 */
#define VSIQF_PE_CTL1(_VSI) (0x00414000 + ((_VSI) * 4)) /* _i=0...767 */
#define PFHMC_PDINV 0x00520300
#define GLHMC_VFPDINV(_i) (0x00528300 + ((_i) * 4)) /* _i=0...31 */
#define GLPE_CRITERR 0x00534000
#define GLINT_RATE(_INT) (0x0015A000 + ((_INT) * 4)) /* _i=0...2047 */ /* Reset Source: CORER */
#define ICRDMA_DB_ADDR_OFFSET (8 * 1024 * 1024 - 64 * 1024)
#define ICRDMA_VF_DB_ADDR_OFFSET (64 * 1024)
/* shifts/masks for FLD_[LS/RS]_64 macros used in device table */
#define ICRDMA_CCQPSTATUS_CCQP_DONE_S 0
#define ICRDMA_CCQPSTATUS_CCQP_DONE BIT_ULL(0)
#define ICRDMA_CCQPSTATUS_CCQP_ERR_S 31
#define ICRDMA_CCQPSTATUS_CCQP_ERR BIT_ULL(31)
#define ICRDMA_CQPSQ_STAG_PDID_S 46
#define ICRDMA_CQPSQ_STAG_PDID GENMASK_ULL(63, 46)
#define ICRDMA_CQPSQ_CQ_CEQID_S 22
#define ICRDMA_CQPSQ_CQ_CEQID GENMASK_ULL(31, 22)
#define ICRDMA_CQPSQ_CQ_CQID_S 0
#define ICRDMA_CQPSQ_CQ_CQID GENMASK_ULL(18, 0)
#define ICRDMA_COMMIT_FPM_CQCNT_S 0
#define ICRDMA_COMMIT_FPM_CQCNT GENMASK_ULL(19, 0)
#define ICRDMA_CQPSQ_UPESD_HMCFNID_S 0
#define ICRDMA_CQPSQ_UPESD_HMCFNID GENMASK_ULL(5, 0)
enum icrdma_device_caps_const {
ICRDMA_MAX_STATS_COUNT = 128,
ICRDMA_MAX_IRD_SIZE = 127,
ICRDMA_MAX_ORD_SIZE = 255,
ICRDMA_MIN_WQ_SIZE = 8 /* WQEs */,
ICRDMA_MAX_PUSH_PAGE_COUNT = 256,
};
void icrdma_init_hw(struct irdma_sc_dev *dev);
#endif /* ICRDMA_HW_H*/
linux-7.2.3/drivers/infiniband/hw/irdma/icrdma_if.c
+346
View File
@@ -0,0 +1,346 @@
// SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
/* Copyright (c) 2015 - 2024 Intel Corporation */
#include "main.h"
#include <linux/net/intel/iidc_rdma_ice.h>
static void icrdma_prep_tc_change(struct irdma_device *iwdev)
{
iwdev->vsi.tc_change_pending = true;
irdma_sc_suspend_resume_qps(&iwdev->vsi, IRDMA_OP_SUSPEND);
/* Wait for all qp's to suspend */
wait_event_timeout(iwdev->suspend_wq,
!atomic_read(&iwdev->vsi.qp_suspend_reqs),
msecs_to_jiffies(IRDMA_EVENT_TIMEOUT_MS));
irdma_ws_reset(&iwdev->vsi);
}
static void icrdma_fill_qos_info(struct irdma_l2params *l2params,
struct iidc_rdma_qos_params *qos_info)
{
int i;
l2params->num_tc = qos_info->num_tc;
l2params->vsi_prio_type = qos_info->vport_priority_type;
l2params->vsi_rel_bw = qos_info->vport_relative_bw;
for (i = 0; i < l2params->num_tc; i++) {
l2params->tc_info[i].egress_virt_up =
qos_info->tc_info[i].egress_virt_up;
l2params->tc_info[i].ingress_virt_up =
qos_info->tc_info[i].ingress_virt_up;
l2params->tc_info[i].prio_type = qos_info->tc_info[i].prio_type;
l2params->tc_info[i].rel_bw = qos_info->tc_info[i].rel_bw;
l2params->tc_info[i].tc_ctx = qos_info->tc_info[i].tc_ctx;
}
for (i = 0; i < IIDC_MAX_USER_PRIORITY; i++)
l2params->up2tc[i] = qos_info->up2tc[i];
if (qos_info->pfc_mode == IIDC_DSCP_PFC_MODE) {
l2params->dscp_mode = true;
memcpy(l2params->dscp_map, qos_info->dscp_map, sizeof(l2params->dscp_map));
}
}
static void icrdma_iidc_event_handler(struct iidc_rdma_core_dev_info *cdev_info,
struct iidc_rdma_event *event)
{
struct irdma_device *iwdev = dev_get_drvdata(&cdev_info->adev->dev);
struct irdma_l2params l2params = {};
if (*event->type & BIT(IIDC_RDMA_EVENT_AFTER_MTU_CHANGE)) {
ibdev_dbg(&iwdev->ibdev, "CLNT: new MTU = %d\n", iwdev->netdev->mtu);
if (iwdev->vsi.mtu != iwdev->netdev->mtu) {
l2params.mtu = iwdev->netdev->mtu;
l2params.mtu_changed = true;
irdma_log_invalid_mtu(l2params.mtu, &iwdev->rf->sc_dev);
irdma_change_l2params(&iwdev->vsi, &l2params);
}
} else if (*event->type & BIT(IIDC_RDMA_EVENT_BEFORE_TC_CHANGE)) {
if (iwdev->vsi.tc_change_pending)
return;
icrdma_prep_tc_change(iwdev);
} else if (*event->type & BIT(IIDC_RDMA_EVENT_AFTER_TC_CHANGE)) {
struct iidc_rdma_priv_dev_info *idc_priv = cdev_info->iidc_priv;
if (!iwdev->vsi.tc_change_pending)
return;
l2params.tc_changed = true;
ibdev_dbg(&iwdev->ibdev, "CLNT: TC Change\n");
icrdma_fill_qos_info(&l2params, &idc_priv->qos_info);
if (iwdev->rf->protocol_used != IRDMA_IWARP_PROTOCOL_ONLY)
iwdev->dcb_vlan_mode =
l2params.num_tc > 1 && !l2params.dscp_mode;
irdma_change_l2params(&iwdev->vsi, &l2params);
} else if (*event->type & BIT(IIDC_RDMA_EVENT_CRIT_ERR)) {
ibdev_warn(&iwdev->ibdev, "ICE OICR event notification: oicr = 0x%08x\n",
event->reg);
if (event->reg & IRDMAPFINT_OICR_PE_CRITERR_M) {
u32 pe_criterr;
pe_criterr = readl(iwdev->rf->sc_dev.hw_regs[IRDMA_GLPE_CRITERR]);
#define IRDMA_Q1_RESOURCE_ERR 0x0001024d
if (pe_criterr != IRDMA_Q1_RESOURCE_ERR) {
ibdev_err(&iwdev->ibdev, "critical PE Error, GLPE_CRITERR=0x%08x\n",
pe_criterr);
iwdev->rf->reset = true;
} else {
ibdev_warn(&iwdev->ibdev, "Q1 Resource Check\n");
}
}
if (event->reg & IRDMAPFINT_OICR_HMC_ERR_M) {
ibdev_err(&iwdev->ibdev, "HMC Error\n");
iwdev->rf->reset = true;
}
if (event->reg & IRDMAPFINT_OICR_PE_PUSH_M) {
ibdev_err(&iwdev->ibdev, "PE Push Error\n");
iwdev->rf->reset = true;
}
if (iwdev->rf->reset)
iwdev->rf->gen_ops.request_reset(iwdev->rf);
}
}
/**
* icrdma_lan_register_qset - Register qset with LAN driver
* @vsi: vsi structure
* @tc_node: Traffic class node
*/
static int icrdma_lan_register_qset(struct irdma_sc_vsi *vsi,
struct irdma_ws_node *tc_node)
{
struct irdma_device *iwdev = vsi->back_vsi;
struct iidc_rdma_core_dev_info *cdev_info = iwdev->rf->cdev;
struct iidc_rdma_qset_params qset = {};
int ret;
qset.qs_handle = tc_node->qs_handle;
qset.tc = tc_node->traffic_class;
qset.vport_id = vsi->vsi_idx;
ret = ice_add_rdma_qset(cdev_info, &qset);
if (ret) {
ibdev_dbg(&iwdev->ibdev, "WS: LAN alloc_res for rdma qset failed.\n");
return ret;
}
tc_node->l2_sched_node_id = qset.teid;
vsi->qos[tc_node->user_pri].l2_sched_node_id = qset.teid;
return 0;
}
/**
* icrdma_lan_unregister_qset - Unregister qset with LAN driver
* @vsi: vsi structure
* @tc_node: Traffic class node
*/
static void icrdma_lan_unregister_qset(struct irdma_sc_vsi *vsi,
struct irdma_ws_node *tc_node)
{
struct irdma_device *iwdev = vsi->back_vsi;
struct iidc_rdma_core_dev_info *cdev_info = iwdev->rf->cdev;
struct iidc_rdma_qset_params qset = {};
qset.qs_handle = tc_node->qs_handle;
qset.tc = tc_node->traffic_class;
qset.vport_id = vsi->vsi_idx;
qset.teid = tc_node->l2_sched_node_id;
if (ice_del_rdma_qset(cdev_info, &qset))
ibdev_dbg(&iwdev->ibdev, "WS: LAN free_res for rdma qset failed.\n");
}
/**
* icrdma_request_reset - Request a reset
* @rf: RDMA PCI function
*/
static void icrdma_request_reset(struct irdma_pci_f *rf)
{
ibdev_warn(&rf->iwdev->ibdev, "Requesting a reset\n");
ice_rdma_request_reset(rf->cdev, IIDC_FUNC_RESET);
}
static int icrdma_init_interrupts(struct irdma_pci_f *rf, struct iidc_rdma_core_dev_info *cdev)
{
int i;
rf->msix_count = num_online_cpus() + IRDMA_NUM_AEQ_MSIX;
rf->msix_entries = kzalloc_objs(*rf->msix_entries, rf->msix_count);
if (!rf->msix_entries)
return -ENOMEM;
for (i = 0; i < rf->msix_count; i++)
if (ice_alloc_rdma_qvector(cdev, &rf->msix_entries[i]))
break;
if (i < IRDMA_MIN_MSIX) {
while (--i >= 0)
ice_free_rdma_qvector(cdev, &rf->msix_entries[i]);
kfree(rf->msix_entries);
return -ENOMEM;
}
rf->msix_count = i;
return 0;
}
static void icrdma_deinit_interrupts(struct irdma_pci_f *rf, struct iidc_rdma_core_dev_info *cdev)
{
int i;
for (i = 0; i < rf->msix_count; i++)
ice_free_rdma_qvector(cdev, &rf->msix_entries[i]);
kfree(rf->msix_entries);
}
static void icrdma_fill_device_info(struct irdma_device *iwdev,
struct iidc_rdma_core_dev_info *cdev_info)
{
struct iidc_rdma_priv_dev_info *idc_priv = cdev_info->iidc_priv;
struct irdma_pci_f *rf = iwdev->rf;
rf->sc_dev.hw = &rf->hw;
rf->iwdev = iwdev;
rf->cdev = cdev_info;
rf->hw.hw_addr = idc_priv->hw_addr;
rf->pcidev = cdev_info->pdev;
rf->hw.device = &rf->pcidev->dev;
rf->pf_id = idc_priv->pf_id;
rf->rdma_ver = IRDMA_GEN_2;
rf->sc_dev.hw_attrs.uk_attrs.hw_rev = IRDMA_GEN_2;
rf->sc_dev.is_pf = true;
rf->sc_dev.privileged = true;
rf->gen_ops.register_qset = icrdma_lan_register_qset;
rf->gen_ops.unregister_qset = icrdma_lan_unregister_qset;
rf->default_vsi.vsi_idx = idc_priv->vport_id;
rf->protocol_used =
cdev_info->rdma_protocol == IIDC_RDMA_PROTOCOL_ROCEV2 ?
IRDMA_ROCE_PROTOCOL_ONLY : IRDMA_IWARP_PROTOCOL_ONLY;
rf->rsrc_profile = IRDMA_HMC_PROFILE_DEFAULT;
rf->rst_to = IRDMA_RST_TIMEOUT_HZ;
rf->gen_ops.request_reset = icrdma_request_reset;
rf->limits_sel = 7;
mutex_init(&rf->ah_tbl_lock);
iwdev->netdev = idc_priv->netdev;
iwdev->vsi_num = idc_priv->vport_id;
iwdev->init_state = INITIAL_STATE;
iwdev->roce_cwnd = IRDMA_ROCE_CWND_DEFAULT;
iwdev->roce_ackcreds = IRDMA_ROCE_ACKCREDS_DEFAULT;
iwdev->rcv_wnd = IRDMA_CM_DEFAULT_RCV_WND_SCALED;
iwdev->rcv_wscale = IRDMA_CM_DEFAULT_RCV_WND_SCALE;
if (iwdev->rf->protocol_used != IRDMA_IWARP_PROTOCOL_ONLY)
iwdev->roce_mode = true;
}
static int icrdma_probe(struct auxiliary_device *aux_dev, const struct auxiliary_device_id *id)
{
struct iidc_rdma_core_auxiliary_dev *iidc_adev;
struct iidc_rdma_core_dev_info *cdev_info;
struct iidc_rdma_priv_dev_info *idc_priv;
struct irdma_l2params l2params = {};
struct irdma_device *iwdev;
struct irdma_pci_f *rf;
int err;
iidc_adev = container_of(aux_dev, struct iidc_rdma_core_auxiliary_dev, adev);
cdev_info = iidc_adev->cdev_info;
idc_priv = cdev_info->iidc_priv;
iwdev = ib_alloc_device(irdma_device, ibdev);
if (!iwdev)
return -ENOMEM;
iwdev->rf = kzalloc_obj(*rf);
if (!iwdev->rf) {
ib_dealloc_device(&iwdev->ibdev);
return -ENOMEM;
}
icrdma_fill_device_info(iwdev, cdev_info);
rf = iwdev->rf;
err = icrdma_init_interrupts(rf, cdev_info);
if (err)
goto err_init_interrupts;
err = irdma_ctrl_init_hw(rf);
if (err)
goto err_ctrl_init;
l2params.mtu = iwdev->netdev->mtu;
icrdma_fill_qos_info(&l2params, &idc_priv->qos_info);
if (iwdev->rf->protocol_used != IRDMA_IWARP_PROTOCOL_ONLY)
iwdev->dcb_vlan_mode = l2params.num_tc > 1 && !l2params.dscp_mode;
err = irdma_rt_init_hw(iwdev, &l2params);
if (err)
goto err_rt_init;
err = irdma_ib_register_device(iwdev);
if (err)
goto err_ibreg;
ice_rdma_update_vsi_filter(cdev_info, iwdev->vsi_num, true);
ibdev_dbg(&iwdev->ibdev, "INIT: Gen2 PF[%d] device probe success\n", PCI_FUNC(rf->pcidev->devfn));
auxiliary_set_drvdata(aux_dev, iwdev);
return 0;
err_ibreg:
irdma_rt_deinit_hw(iwdev);
err_rt_init:
irdma_ctrl_deinit_hw(rf);
err_ctrl_init:
icrdma_deinit_interrupts(rf, cdev_info);
err_init_interrupts:
mutex_destroy(&rf->ah_tbl_lock);
kfree(rf);
ib_dealloc_device(&iwdev->ibdev);
return err;
}
static void icrdma_remove(struct auxiliary_device *aux_dev)
{
struct iidc_rdma_core_auxiliary_dev *idc_adev =
container_of(aux_dev, struct iidc_rdma_core_auxiliary_dev, adev);
struct iidc_rdma_core_dev_info *cdev_info = idc_adev->cdev_info;
struct irdma_device *iwdev = auxiliary_get_drvdata(aux_dev);
u8 rdma_ver = iwdev->rf->rdma_ver;
ice_rdma_update_vsi_filter(cdev_info, iwdev->vsi_num, false);
irdma_ib_unregister_device(iwdev);
icrdma_deinit_interrupts(iwdev->rf, cdev_info);
mutex_destroy(&iwdev->rf->ah_tbl_lock);
kfree(iwdev->rf);
pr_debug("INIT: Gen[%d] func[%d] device remove success\n",
rdma_ver, PCI_FUNC(cdev_info->pdev->devfn));
}
static const struct auxiliary_device_id icrdma_auxiliary_id_table[] = {
{.name = "ice.iwarp", },
{.name = "ice.roce", },
{},
};
MODULE_DEVICE_TABLE(auxiliary, icrdma_auxiliary_id_table);
struct iidc_rdma_core_auxiliary_drv icrdma_core_auxiliary_drv = {
.adrv = {
.name = "gen_2",
.id_table = icrdma_auxiliary_id_table,
.probe = icrdma_probe,
.remove = icrdma_remove,
},
.event_handler = icrdma_iidc_event_handler,
};
linux-7.2.3/drivers/infiniband/hw/irdma/ig3rdma_hw.c
+169
View File
@@ -0,0 +1,169 @@
// SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
/* Copyright (c) 2018 - 2024 Intel Corporation */
#include "osdep.h"
#include "type.h"
#include "protos.h"
#include "ig3rdma_hw.h"
/**
* ig3rdma_ena_irq - Enable interrupt
* @dev: pointer to the device structure
* @idx: vector index
*/
static void ig3rdma_ena_irq(struct irdma_sc_dev *dev, u32 idx)
{
u32 val;
u32 int_stride = 1; /* one u32 per register */
if (dev->is_pf)
int_stride = 0x400;
else
idx--; /* VFs use DYN_CTL_N */
val = FIELD_PREP(IRDMA_GLINT_DYN_CTL_INTENA, 1) |
FIELD_PREP(IRDMA_GLINT_DYN_CTL_CLEARPBA, 1);
writel(val, dev->hw_regs[IRDMA_GLINT_DYN_CTL] + (idx * int_stride));
}
/**
* ig3rdma_disable_irq - Disable interrupt
* @dev: pointer to the device structure
* @idx: vector index
*/
static void ig3rdma_disable_irq(struct irdma_sc_dev *dev, u32 idx)
{
u32 int_stride = 1; /* one u32 per register */
if (dev->is_pf)
int_stride = 0x400;
else
idx--; /* VFs use DYN_CTL_N */
writel(0, dev->hw_regs[IRDMA_GLINT_DYN_CTL] + (idx * int_stride));
}
static const struct irdma_irq_ops ig3rdma_irq_ops = {
.irdma_dis_irq = ig3rdma_disable_irq,
.irdma_en_irq = ig3rdma_ena_irq,
};
static const struct irdma_hw_stat_map ig3rdma_hw_stat_map[] = {
[IRDMA_HW_STAT_INDEX_RXVLANERR] = { 0, 0, 0 },
[IRDMA_HW_STAT_INDEX_IP4RXOCTS] = { 8, 0, 0 },
[IRDMA_HW_STAT_INDEX_IP4RXPKTS] = { 16, 0, 0 },
[IRDMA_HW_STAT_INDEX_IP4RXDISCARD] = { 24, 0, 0 },
[IRDMA_HW_STAT_INDEX_IP4RXTRUNC] = { 32, 0, 0 },
[IRDMA_HW_STAT_INDEX_IP4RXFRAGS] = { 40, 0, 0 },
[IRDMA_HW_STAT_INDEX_IP4RXMCOCTS] = { 48, 0, 0 },
[IRDMA_HW_STAT_INDEX_IP4RXMCPKTS] = { 56, 0, 0 },
[IRDMA_HW_STAT_INDEX_IP6RXOCTS] = { 64, 0, 0 },
[IRDMA_HW_STAT_INDEX_IP6RXPKTS] = { 72, 0, 0 },
[IRDMA_HW_STAT_INDEX_IP6RXDISCARD] = { 80, 0, 0 },
[IRDMA_HW_STAT_INDEX_IP6RXTRUNC] = { 88, 0, 0 },
[IRDMA_HW_STAT_INDEX_IP6RXFRAGS] = { 96, 0, 0 },
[IRDMA_HW_STAT_INDEX_IP6RXMCOCTS] = { 104, 0, 0 },
[IRDMA_HW_STAT_INDEX_IP6RXMCPKTS] = { 112, 0, 0 },
[IRDMA_HW_STAT_INDEX_IP4TXOCTS] = { 120, 0, 0 },
[IRDMA_HW_STAT_INDEX_IP4TXPKTS] = { 128, 0, 0 },
[IRDMA_HW_STAT_INDEX_IP4TXFRAGS] = { 136, 0, 0 },
[IRDMA_HW_STAT_INDEX_IP4TXMCOCTS] = { 144, 0, 0 },
[IRDMA_HW_STAT_INDEX_IP4TXMCPKTS] = { 152, 0, 0 },
[IRDMA_HW_STAT_INDEX_IP6TXOCTS] = { 160, 0, 0 },
[IRDMA_HW_STAT_INDEX_IP6TXPKTS] = { 168, 0, 0 },
[IRDMA_HW_STAT_INDEX_IP6TXFRAGS] = { 176, 0, 0 },
[IRDMA_HW_STAT_INDEX_IP6TXMCOCTS] = { 184, 0, 0 },
[IRDMA_HW_STAT_INDEX_IP6TXMCPKTS] = { 192, 0, 0 },
[IRDMA_HW_STAT_INDEX_IP4TXNOROUTE] = { 200, 0, 0 },
[IRDMA_HW_STAT_INDEX_IP6TXNOROUTE] = { 208, 0, 0 },
[IRDMA_HW_STAT_INDEX_TCPRTXSEG] = { 216, 0, 0 },
[IRDMA_HW_STAT_INDEX_TCPRXOPTERR] = { 224, 0, 0 },
[IRDMA_HW_STAT_INDEX_TCPRXPROTOERR] = { 232, 0, 0 },
[IRDMA_HW_STAT_INDEX_TCPTXSEG] = { 240, 0, 0 },
[IRDMA_HW_STAT_INDEX_TCPRXSEGS] = { 248, 0, 0 },
[IRDMA_HW_STAT_INDEX_UDPRXPKTS] = { 256, 0, 0 },
[IRDMA_HW_STAT_INDEX_UDPTXPKTS] = { 264, 0, 0 },
[IRDMA_HW_STAT_INDEX_RDMARXWRS] = { 272, 0, 0 },
[IRDMA_HW_STAT_INDEX_RDMARXRDS] = { 280, 0, 0 },
[IRDMA_HW_STAT_INDEX_RDMARXSNDS] = { 288, 0, 0 },
[IRDMA_HW_STAT_INDEX_RDMATXWRS] = { 296, 0, 0 },
[IRDMA_HW_STAT_INDEX_RDMATXRDS] = { 304, 0, 0 },
[IRDMA_HW_STAT_INDEX_RDMATXSNDS] = { 312, 0, 0 },
[IRDMA_HW_STAT_INDEX_RDMAVBND] = { 320, 0, 0 },
[IRDMA_HW_STAT_INDEX_RDMAVINV] = { 328, 0, 0 },
[IRDMA_HW_STAT_INDEX_RXNPECNMARKEDPKTS] = { 336, 0, 0 },
[IRDMA_HW_STAT_INDEX_RXRPCNPHANDLED] = { 344, 0, 0 },
[IRDMA_HW_STAT_INDEX_RXRPCNPIGNORED] = { 352, 0, 0 },
[IRDMA_HW_STAT_INDEX_TXNPCNPSENT] = { 360, 0, 0 },
[IRDMA_HW_STAT_INDEX_RNR_SENT] = { 368, 0, 0 },
[IRDMA_HW_STAT_INDEX_RNR_RCVD] = { 376, 0, 0 },
[IRDMA_HW_STAT_INDEX_RDMAORDLMTCNT] = { 384, 0, 0 },
[IRDMA_HW_STAT_INDEX_RDMAIRDLMTCNT] = { 392, 0, 0 },
[IRDMA_HW_STAT_INDEX_RDMARXATS] = { 408, 0, 0 },
[IRDMA_HW_STAT_INDEX_RDMATXATS] = { 416, 0, 0 },
[IRDMA_HW_STAT_INDEX_NAKSEQERR] = { 424, 0, 0 },
[IRDMA_HW_STAT_INDEX_NAKSEQERR_IMPLIED] = { 432, 0, 0 },
[IRDMA_HW_STAT_INDEX_RTO] = { 440, 0, 0 },
[IRDMA_HW_STAT_INDEX_RXOOOPKTS] = { 448, 0, 0 },
[IRDMA_HW_STAT_INDEX_ICRCERR] = { 456, 0, 0 },
};
void ig3rdma_init_hw(struct irdma_sc_dev *dev)
{
dev->irq_ops = &ig3rdma_irq_ops;
dev->hw_stats_map = ig3rdma_hw_stat_map;
dev->hw_attrs.uk_attrs.max_hw_wq_frags = IG3RDMA_MAX_WQ_FRAGMENT_COUNT;
dev->hw_attrs.uk_attrs.max_hw_read_sges = IG3RDMA_MAX_SGE_RD;
dev->hw_attrs.uk_attrs.max_hw_sq_chunk = IRDMA_MAX_QUANTA_PER_WR;
dev->hw_attrs.first_hw_vf_fpm_id = 0;
dev->hw_attrs.max_hw_vf_fpm_id = IG3_MAX_APFS + IG3_MAX_AVFS;
dev->hw_attrs.uk_attrs.feature_flags |= IRDMA_FEATURE_64_BYTE_CQE;
dev->hw_attrs.uk_attrs.feature_flags |= IRDMA_FEATURE_CQE_TIMESTAMPING;
dev->hw_attrs.uk_attrs.feature_flags |= IRDMA_FEATURE_SRQ;
dev->hw_attrs.uk_attrs.feature_flags |= IRDMA_FEATURE_RTS_AE |
IRDMA_FEATURE_CQ_RESIZE;
dev->hw_attrs.page_size_cap = SZ_4K | SZ_2M | SZ_1G;
dev->hw_attrs.max_hw_ird = IG3RDMA_MAX_IRD_SIZE;
dev->hw_attrs.max_hw_ord = IG3RDMA_MAX_ORD_SIZE;
dev->hw_attrs.max_stat_inst = IG3RDMA_MAX_STATS_COUNT;
dev->hw_attrs.max_stat_idx = IRDMA_HW_STAT_INDEX_MAX_GEN_3;
dev->hw_attrs.uk_attrs.min_hw_wq_size = IG3RDMA_MIN_WQ_SIZE;
dev->hw_attrs.uk_attrs.max_hw_srq_quanta = IRDMA_SRQ_MAX_QUANTA;
dev->hw_attrs.uk_attrs.max_hw_inline = IG3RDMA_MAX_INLINE_DATA_SIZE;
dev->hw_attrs.max_hw_device_pages =
dev->is_pf ? IG3RDMA_MAX_PF_PUSH_PAGE_COUNT : IG3RDMA_MAX_VF_PUSH_PAGE_COUNT;
}
static void __iomem *__ig3rdma_get_reg_addr(struct irdma_mmio_region *region, u64 reg_offset)
{
if (reg_offset >= region->offset &&
reg_offset < (region->offset + region->len)) {
reg_offset -= region->offset;
return region->addr + reg_offset;
}
return NULL;
}
void __iomem *ig3rdma_get_reg_addr(struct irdma_hw *hw, u64 reg_offset)
{
u8 __iomem *reg_addr;
int i;
reg_addr = __ig3rdma_get_reg_addr(&hw->rdma_reg, reg_offset);
if (reg_addr)
return reg_addr;
for (i = 0; i < hw->num_io_regions; i++) {
reg_addr = __ig3rdma_get_reg_addr(&hw->io_regs[i], reg_offset);
if (reg_addr)
return reg_addr;
}
WARN_ON_ONCE(1);
return NULL;
}
linux-7.2.3/drivers/infiniband/hw/irdma/ig3rdma_hw.h
+32
View File
@@ -0,0 +1,32 @@
/* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB */
/* Copyright (c) 2021 - 2024 Intel Corporation */
#ifndef IG3RDMA_HW_H
#define IG3RDMA_HW_H
#define IG3_MAX_APFS 1
#define IG3_MAX_AVFS 0
#define IG3_PF_RDMA_REGION_OFFSET 0xBC00000
#define IG3_PF_RDMA_REGION_LEN 0x401000
#define IG3_VF_RDMA_REGION_OFFSET 0x8C00
#define IG3_VF_RDMA_REGION_LEN 0x8400
enum ig3rdma_device_caps_const {
IG3RDMA_MAX_WQ_FRAGMENT_COUNT = 14,
IG3RDMA_MAX_SGE_RD = 14,
IG3RDMA_MAX_STATS_COUNT = 128,
IG3RDMA_MAX_IRD_SIZE = 64,
IG3RDMA_MAX_ORD_SIZE = 64,
IG3RDMA_MIN_WQ_SIZE = 16 /* WQEs */,
IG3RDMA_MAX_INLINE_DATA_SIZE = 216,
IG3RDMA_MAX_PF_PUSH_PAGE_COUNT = 8192,
IG3RDMA_MAX_VF_PUSH_PAGE_COUNT = 16,
};
void __iomem *ig3rdma_get_reg_addr(struct irdma_hw *hw, u64 reg_offset);
int ig3rdma_vchnl_send_sync(struct irdma_sc_dev *dev, u8 *msg, u16 len,
u8 *recv_msg, u16 *recv_len);
#endif /* IG3RDMA_HW_H*/
linux-7.2.3/drivers/infiniband/hw/irdma/ig3rdma_if.c
+235
View File
@@ -0,0 +1,235 @@
// SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
/* Copyright (c) 2023 - 2024 Intel Corporation */
#include "main.h"
#include <linux/net/intel/iidc_rdma_idpf.h>
#include "ig3rdma_hw.h"
static void ig3rdma_idc_core_event_handler(struct iidc_rdma_core_dev_info *cdev_info,
struct iidc_rdma_event *event)
{
struct irdma_pci_f *rf = auxiliary_get_drvdata(cdev_info->adev);
if (*event->type & BIT(IIDC_RDMA_EVENT_WARN_RESET)) {
rf->reset = true;
rf->sc_dev.vchnl_up = false;
}
}
int ig3rdma_vchnl_send_sync(struct irdma_sc_dev *dev, u8 *msg, u16 len,
u8 *recv_msg, u16 *recv_len)
{
struct iidc_rdma_core_dev_info *cdev_info = dev_to_rf(dev)->cdev;
int ret;
ret = idpf_idc_rdma_vc_send_sync(cdev_info, msg, len, recv_msg,
recv_len);
if (ret == -ETIMEDOUT) {
ibdev_err(&(dev_to_rf(dev)->iwdev->ibdev),
"Virtual channel Req <-> Resp completion timeout\n");
dev->vchnl_up = false;
}
return ret;
}
static int ig3rdma_vchnl_init(struct irdma_pci_f *rf,
struct iidc_rdma_core_dev_info *cdev_info,
u8 *rdma_ver)
{
struct iidc_rdma_priv_dev_info *idc_priv = cdev_info->iidc_priv;
struct irdma_vchnl_init_info virt_info;
u8 gen = rf->rdma_ver;
int ret;
rf->vchnl_wq = alloc_ordered_workqueue("irdma-virtchnl-wq", 0);
if (!rf->vchnl_wq)
return -ENOMEM;
mutex_init(&rf->sc_dev.vchnl_mutex);
virt_info.is_pf = !idc_priv->ftype;
virt_info.hw_rev = gen;
virt_info.privileged = gen == IRDMA_GEN_2;
virt_info.vchnl_wq = rf->vchnl_wq;
ret = irdma_sc_vchnl_init(&rf->sc_dev, &virt_info);
if (ret) {
destroy_workqueue(rf->vchnl_wq);
mutex_destroy(&rf->sc_dev.vchnl_mutex);
return ret;
}
*rdma_ver = rf->sc_dev.hw_attrs.uk_attrs.hw_rev;
return 0;
}
/**
* ig3rdma_request_reset - Request a reset
* @rf: RDMA PCI function
*/
static void ig3rdma_request_reset(struct irdma_pci_f *rf)
{
ibdev_warn(&rf->iwdev->ibdev, "Requesting a reset\n");
idpf_idc_request_reset(rf->cdev, IIDC_FUNC_RESET);
}
static int ig3rdma_cfg_regions(struct irdma_hw *hw,
struct iidc_rdma_core_dev_info *cdev_info)
{
struct iidc_rdma_priv_dev_info *idc_priv = cdev_info->iidc_priv;
struct pci_dev *pdev = cdev_info->pdev;
int i;
switch (idc_priv->ftype) {
case IIDC_FUNCTION_TYPE_PF:
hw->rdma_reg.len = IG3_PF_RDMA_REGION_LEN;
hw->rdma_reg.offset = IG3_PF_RDMA_REGION_OFFSET;
break;
case IIDC_FUNCTION_TYPE_VF:
hw->rdma_reg.len = IG3_VF_RDMA_REGION_LEN;
hw->rdma_reg.offset = IG3_VF_RDMA_REGION_OFFSET;
break;
default:
return -ENODEV;
}
hw->rdma_reg.addr = ioremap(pci_resource_start(pdev, 0) + hw->rdma_reg.offset,
hw->rdma_reg.len);
if (!hw->rdma_reg.addr)
return -ENOMEM;
hw->num_io_regions = le16_to_cpu(idc_priv->num_memory_regions);
hw->io_regs = kzalloc_objs(struct irdma_mmio_region, hw->num_io_regions);
if (!hw->io_regs) {
iounmap(hw->rdma_reg.addr);
return -ENOMEM;
}
for (i = 0; i < hw->num_io_regions; i++) {
hw->io_regs[i].addr =
idc_priv->mapped_mem_regions[i].region_addr;
hw->io_regs[i].len =
le64_to_cpu(idc_priv->mapped_mem_regions[i].size);
hw->io_regs[i].offset =
le64_to_cpu(idc_priv->mapped_mem_regions[i].start_offset);
}
return 0;
}
static void ig3rdma_decfg_rf(struct irdma_pci_f *rf)
{
struct irdma_hw *hw = &rf->hw;
mutex_destroy(&rf->ah_tbl_lock);
destroy_workqueue(rf->vchnl_wq);
mutex_destroy(&rf->sc_dev.vchnl_mutex);
kfree(hw->io_regs);
iounmap(hw->rdma_reg.addr);
}
static int ig3rdma_cfg_rf(struct irdma_pci_f *rf,
struct iidc_rdma_core_dev_info *cdev_info)
{
struct iidc_rdma_priv_dev_info *idc_priv = cdev_info->iidc_priv;
int err;
rf->sc_dev.hw = &rf->hw;
rf->cdev = cdev_info;
rf->pcidev = cdev_info->pdev;
rf->hw.device = &rf->pcidev->dev;
rf->msix_count = idc_priv->msix_count;
rf->msix_entries = idc_priv->msix_entries;
err = ig3rdma_vchnl_init(rf, cdev_info, &rf->rdma_ver);
if (err)
return err;
err = ig3rdma_cfg_regions(&rf->hw, cdev_info);
if (err) {
destroy_workqueue(rf->vchnl_wq);
mutex_destroy(&rf->sc_dev.vchnl_mutex);
return err;
}
rf->protocol_used = IRDMA_ROCE_PROTOCOL_ONLY;
rf->rsrc_profile = IRDMA_HMC_PROFILE_DEFAULT;
rf->rst_to = IRDMA_RST_TIMEOUT_HZ;
rf->gen_ops.request_reset = ig3rdma_request_reset;
rf->limits_sel = 7;
mutex_init(&rf->ah_tbl_lock);
return 0;
}
static int ig3rdma_core_probe(struct auxiliary_device *aux_dev,
const struct auxiliary_device_id *id)
{
struct iidc_rdma_core_auxiliary_dev *idc_adev =
container_of(aux_dev, struct iidc_rdma_core_auxiliary_dev, adev);
struct iidc_rdma_core_dev_info *cdev_info = idc_adev->cdev_info;
struct irdma_pci_f *rf;
int err;
rf = kzalloc_obj(*rf);
if (!rf)
return -ENOMEM;
err = ig3rdma_cfg_rf(rf, cdev_info);
if (err)
goto err_cfg_rf;
err = irdma_ctrl_init_hw(rf);
if (err)
goto err_ctrl_init;
auxiliary_set_drvdata(aux_dev, rf);
err = idpf_idc_vport_dev_ctrl(cdev_info, true);
if (err)
goto err_vport_ctrl;
return 0;
err_vport_ctrl:
irdma_ctrl_deinit_hw(rf);
err_ctrl_init:
ig3rdma_decfg_rf(rf);
err_cfg_rf:
kfree(rf);
return err;
}
static void ig3rdma_core_remove(struct auxiliary_device *aux_dev)
{
struct iidc_rdma_core_auxiliary_dev *idc_adev =
container_of(aux_dev, struct iidc_rdma_core_auxiliary_dev, adev);
struct iidc_rdma_core_dev_info *cdev_info = idc_adev->cdev_info;
struct irdma_pci_f *rf = auxiliary_get_drvdata(aux_dev);
idpf_idc_vport_dev_ctrl(cdev_info, false);
irdma_ctrl_deinit_hw(rf);
ig3rdma_decfg_rf(rf);
kfree(rf);
}
static const struct auxiliary_device_id ig3rdma_core_auxiliary_id_table[] = {
{.name = "idpf.8086.rdma.core", },
{},
};
MODULE_DEVICE_TABLE(auxiliary, ig3rdma_core_auxiliary_id_table);
struct iidc_rdma_core_auxiliary_drv ig3rdma_core_auxiliary_drv = {
.adrv = {
.name = "core",
.id_table = ig3rdma_core_auxiliary_id_table,
.probe = ig3rdma_core_probe,
.remove = ig3rdma_core_remove,
},
.event_handler = ig3rdma_idc_core_event_handler,
};
linux-7.2.3/drivers/infiniband/hw/irdma/irdma.h
+175
View File
@@ -0,0 +1,175 @@
/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
/* Copyright (c) 2017 - 2021 Intel Corporation */
#ifndef IRDMA_H
#define IRDMA_H
#define IRDMA_WQEALLOC_WQE_DESC_INDEX GENMASK(31, 20)
#define IRDMA_CQPTAIL_WQTAIL GENMASK(10, 0)
#define IRDMA_CQPTAIL_CQP_OP_ERR BIT(31)
#define IRDMA_CQPERRCODES_CQP_MINOR_CODE GENMASK(15, 0)
#define IRDMA_CQPERRCODES_CQP_MAJOR_CODE GENMASK(31, 16)
#define IRDMA_GLPCI_LBARCTRL_PE_DB_SIZE GENMASK(5, 4)
#define IRDMA_GLINT_RATE_INTERVAL GENMASK(5, 0)
#define IRDMA_GLINT_RATE_INTRL_ENA BIT(6)
#define IRDMA_GLINT_DYN_CTL_INTENA BIT(0)
#define IRDMA_GLINT_DYN_CTL_CLEARPBA BIT(1)
#define IRDMA_GLINT_DYN_CTL_ITR_INDX GENMASK(4, 3)
#define IRDMA_GLINT_DYN_CTL_INTERVAL GENMASK(16, 5)
#define IRDMA_GLINT_CEQCTL_ITR_INDX GENMASK(12, 11)
#define IRDMA_GLINT_CEQCTL_CAUSE_ENA BIT(30)
#define IRDMA_GLINT_CEQCTL_MSIX_INDX GENMASK(10, 0)
#define IRDMA_PFINT_AEQCTL_MSIX_INDX GENMASK(10, 0)
#define IRDMA_PFINT_AEQCTL_ITR_INDX GENMASK(12, 11)
#define IRDMA_PFINT_AEQCTL_CAUSE_ENA BIT(30)
#define IRDMA_PFHMC_PDINV_PMSDIDX GENMASK(11, 0)
#define IRDMA_PFHMC_PDINV_PMSDPARTSEL BIT(15)
#define IRDMA_PFHMC_PDINV_PMPDIDX GENMASK(24, 16)
#define IRDMA_PFHMC_SDDATALOW_PMSDVALID BIT(0)
#define IRDMA_PFHMC_SDDATALOW_PMSDTYPE BIT(1)
#define IRDMA_PFHMC_SDDATALOW_PMSDBPCOUNT GENMASK(11, 2)
#define IRDMA_PFHMC_SDDATALOW_PMSDDATALOW GENMASK(31, 12)
#define IRDMA_PFHMC_SDCMD_PMSDWR BIT(31)
#define IRDMA_INVALID_CQ_IDX 0xffffffff
#define IRDMA_Q_INVALID_IDX 0xffff
enum irdma_dyn_idx_t {
IRDMA_IDX_ITR0 = 0,
IRDMA_IDX_ITR1 = 1,
IRDMA_IDX_ITR2 = 2,
IRDMA_IDX_NOITR = 3,
};
enum irdma_registers {
IRDMA_CQPTAIL,
IRDMA_CQPDB,
IRDMA_CCQPSTATUS,
IRDMA_CCQPHIGH,
IRDMA_CCQPLOW,
IRDMA_CQARM,
IRDMA_CQACK,
IRDMA_AEQALLOC,
IRDMA_CQPERRCODES,
IRDMA_WQEALLOC,
IRDMA_GLINT_DYN_CTL,
IRDMA_DB_ADDR_OFFSET,
IRDMA_GLPCI_LBARCTRL,
IRDMA_GLPE_CPUSTATUS0,
IRDMA_GLPE_CPUSTATUS1,
IRDMA_GLPE_CPUSTATUS2,
IRDMA_PFINT_AEQCTL,
IRDMA_GLINT_CEQCTL,
IRDMA_VSIQF_PE_CTL1,
IRDMA_PFHMC_PDINV,
IRDMA_GLHMC_VFPDINV,
IRDMA_GLPE_CRITERR,
IRDMA_GLINT_RATE,
IRDMA_MAX_REGS, /* Must be last entry */
};
enum irdma_shifts {
IRDMA_CCQPSTATUS_CCQP_DONE_S,
IRDMA_CCQPSTATUS_CCQP_ERR_S,
IRDMA_CQPSQ_STAG_PDID_S,
IRDMA_CQPSQ_CQ_CEQID_S,
IRDMA_CQPSQ_CQ_CQID_S,
IRDMA_COMMIT_FPM_CQCNT_S,
IRDMA_CQPSQ_UPESD_HMCFNID_S,
IRDMA_MAX_SHIFTS,
};
enum irdma_masks {
IRDMA_CCQPSTATUS_CCQP_DONE_M,
IRDMA_CCQPSTATUS_CCQP_ERR_M,
IRDMA_CQPSQ_STAG_PDID_M,
IRDMA_CQPSQ_CQ_CEQID_M,
IRDMA_CQPSQ_CQ_CQID_M,
IRDMA_COMMIT_FPM_CQCNT_M,
IRDMA_CQPSQ_UPESD_HMCFNID_M,
IRDMA_MAX_MASKS, /* Must be last entry */
};
#define IRDMA_MAX_MGS_PER_CTX 8
struct irdma_mcast_grp_ctx_entry_info {
u32 qp_id;
bool valid_entry;
u16 dest_port;
u32 use_cnt;
};
struct irdma_mcast_grp_info {
u8 dest_mac_addr[ETH_ALEN];
u16 vlan_id;
u16 hmc_fcn_id;
bool ipv4_valid:1;
bool vlan_valid:1;
u16 mg_id;
u32 no_of_mgs;
u32 dest_ip_addr[4];
u16 qs_handle;
struct irdma_dma_mem dma_mem_mc;
struct irdma_mcast_grp_ctx_entry_info mg_ctx_info[IRDMA_MAX_MGS_PER_CTX];
};
enum irdma_vers {
IRDMA_GEN_RSVD,
IRDMA_GEN_1,
IRDMA_GEN_2,
IRDMA_GEN_3,
IRDMA_GEN_4,
IRDMA_GEN_NEXT,
IRDMA_GEN_MAX = IRDMA_GEN_NEXT-1
};
struct irdma_uk_attrs {
u64 feature_flags;
u32 max_hw_wq_frags;
u32 max_hw_read_sges;
u32 max_hw_inline;
u32 max_hw_rq_quanta;
u32 max_hw_wq_quanta;
u32 min_hw_cq_size;
u32 max_hw_cq_size;
u32 max_hw_srq_quanta;
u16 max_hw_sq_chunk;
u16 min_hw_wq_size;
u8 hw_rev;
};
struct irdma_hw_attrs {
struct irdma_uk_attrs uk_attrs;
u64 max_hw_outbound_msg_size;
u64 max_hw_inbound_msg_size;
u64 max_mr_size;
u64 page_size_cap;
u32 min_hw_qp_id;
u32 min_hw_aeq_size;
u32 max_hw_aeq_size;
u32 min_hw_ceq_size;
u32 max_hw_ceq_size;
u32 max_hw_device_pages;
u32 max_hw_vf_fpm_id;
u32 first_hw_vf_fpm_id;
u32 max_hw_ird;
u32 max_hw_ord;
u32 max_hw_wqes;
u32 max_hw_pds;
u32 max_hw_ena_vf_count;
u32 max_qp_wr;
u32 max_pe_ready_count;
u32 max_done_count;
u32 max_sleep_count;
u32 max_cqp_compl_wait_time_ms;
u32 min_hw_srq_id;
u16 max_stat_inst;
u16 max_stat_idx;
};
void i40iw_init_hw(struct irdma_sc_dev *dev);
void icrdma_init_hw(struct irdma_sc_dev *dev);
void ig3rdma_init_hw(struct irdma_sc_dev *dev);
void __iomem *ig3rdma_get_reg_addr(struct irdma_hw *hw, u64 reg_offset);
#endif /* IRDMA_H*/
linux-7.2.3/drivers/infiniband/hw/irdma/main.c
+211
View File
@@ -0,0 +1,211 @@
// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
/* Copyright (c) 2015 - 2021 Intel Corporation */
#include "main.h"
#include <linux/net/intel/iidc_rdma_idpf.h>
MODULE_ALIAS("i40iw");
MODULE_DESCRIPTION("Intel(R) Ethernet Protocol Driver for RDMA");
MODULE_LICENSE("Dual BSD/GPL");
static struct notifier_block irdma_inetaddr_notifier = {
.notifier_call = irdma_inetaddr_event
};
static struct notifier_block irdma_inetaddr6_notifier = {
.notifier_call = irdma_inet6addr_event
};
static struct notifier_block irdma_net_notifier = {
.notifier_call = irdma_net_event
};
static struct notifier_block irdma_netdevice_notifier = {
.notifier_call = irdma_netdevice_event
};
static void irdma_register_notifiers(void)
{
register_inetaddr_notifier(&irdma_inetaddr_notifier);
register_inet6addr_notifier(&irdma_inetaddr6_notifier);
register_netevent_notifier(&irdma_net_notifier);
register_netdevice_notifier(&irdma_netdevice_notifier);
}
static void irdma_unregister_notifiers(void)
{
unregister_netevent_notifier(&irdma_net_notifier);
unregister_inetaddr_notifier(&irdma_inetaddr_notifier);
unregister_inet6addr_notifier(&irdma_inetaddr6_notifier);
unregister_netdevice_notifier(&irdma_netdevice_notifier);
}
void irdma_log_invalid_mtu(u16 mtu, struct irdma_sc_dev *dev)
{
if (mtu < IRDMA_MIN_MTU_IPV4)
ibdev_warn(to_ibdev(dev), "MTU setting [%d] too low for RDMA traffic. Minimum MTU is 576 for IPv4\n", mtu);
else if (mtu < IRDMA_MIN_MTU_IPV6)
ibdev_warn(to_ibdev(dev), "MTU setting [%d] too low for RDMA traffic. Minimum MTU is 1280 for IPv6\\n", mtu);
}
static void ig3rdma_idc_vport_event_handler(struct iidc_rdma_vport_dev_info *cdev_info,
struct iidc_rdma_event *event)
{
struct irdma_device *iwdev = auxiliary_get_drvdata(cdev_info->adev);
struct irdma_l2params l2params = {};
if (*event->type & BIT(IIDC_RDMA_EVENT_AFTER_MTU_CHANGE)) {
ibdev_dbg(&iwdev->ibdev, "CLNT: new MTU = %d\n", iwdev->netdev->mtu);
if (iwdev->vsi.mtu != iwdev->netdev->mtu) {
l2params.mtu = iwdev->netdev->mtu;
l2params.mtu_changed = true;
irdma_log_invalid_mtu(l2params.mtu, &iwdev->rf->sc_dev);
irdma_change_l2params(&iwdev->vsi, &l2params);
}
}
}
static int ig3rdma_vport_probe(struct auxiliary_device *aux_dev,
const struct auxiliary_device_id *id)
{
struct iidc_rdma_vport_auxiliary_dev *idc_adev =
container_of(aux_dev, struct iidc_rdma_vport_auxiliary_dev, adev);
struct auxiliary_device *aux_core_dev = idc_adev->vdev_info->core_adev;
struct irdma_pci_f *rf = auxiliary_get_drvdata(aux_core_dev);
struct irdma_l2params l2params = {};
struct irdma_device *iwdev;
int err;
if (!rf) {
WARN_ON_ONCE(1);
return -ENOMEM;
}
iwdev = ib_alloc_device(irdma_device, ibdev);
/* Fill iwdev info */
iwdev->is_vport = true;
iwdev->rf = rf;
iwdev->vport_id = idc_adev->vdev_info->vport_id;
iwdev->netdev = idc_adev->vdev_info->netdev;
iwdev->init_state = INITIAL_STATE;
iwdev->roce_cwnd = IRDMA_ROCE_CWND_DEFAULT;
iwdev->roce_ackcreds = IRDMA_ROCE_ACKCREDS_DEFAULT;
iwdev->rcv_wnd = IRDMA_CM_DEFAULT_RCV_WND_SCALED;
iwdev->rcv_wscale = IRDMA_CM_DEFAULT_RCV_WND_SCALE;
iwdev->roce_mode = true;
iwdev->push_mode = false;
l2params.mtu = iwdev->netdev->mtu;
err = irdma_rt_init_hw(iwdev, &l2params);
if (err)
goto err_rt_init;
err = irdma_ib_register_device(iwdev);
if (err)
goto err_ibreg;
auxiliary_set_drvdata(aux_dev, iwdev);
ibdev_dbg(&iwdev->ibdev,
"INIT: Gen[%d] vport[%d] probe success. dev_name = %s, core_dev_name = %s, netdev=%s\n",
rf->rdma_ver, idc_adev->vdev_info->vport_id,
dev_name(&aux_dev->dev),
dev_name(&idc_adev->vdev_info->core_adev->dev),
netdev_name(idc_adev->vdev_info->netdev));
return 0;
err_ibreg:
irdma_rt_deinit_hw(iwdev);
err_rt_init:
ib_dealloc_device(&iwdev->ibdev);
return err;
}
static void ig3rdma_vport_remove(struct auxiliary_device *aux_dev)
{
struct iidc_rdma_vport_auxiliary_dev *idc_adev =
container_of(aux_dev, struct iidc_rdma_vport_auxiliary_dev, adev);
struct irdma_device *iwdev = auxiliary_get_drvdata(aux_dev);
ibdev_dbg(&iwdev->ibdev,
"INIT: Gen[%d] dev_name = %s, core_dev_name = %s, netdev=%s\n",
iwdev->rf->rdma_ver, dev_name(&aux_dev->dev),
dev_name(&idc_adev->vdev_info->core_adev->dev),
netdev_name(idc_adev->vdev_info->netdev));
irdma_ib_unregister_device(iwdev);
}
static const struct auxiliary_device_id ig3rdma_vport_auxiliary_id_table[] = {
{.name = "idpf.8086.rdma.vdev", },
{},
};
MODULE_DEVICE_TABLE(auxiliary, ig3rdma_vport_auxiliary_id_table);
static struct iidc_rdma_vport_auxiliary_drv ig3rdma_vport_auxiliary_drv = {
.adrv = {
.name = "vdev",
.id_table = ig3rdma_vport_auxiliary_id_table,
.probe = ig3rdma_vport_probe,
.remove = ig3rdma_vport_remove,
},
.event_handler = ig3rdma_idc_vport_event_handler,
};
static int __init irdma_init_module(void)
{
int ret;
ret = auxiliary_driver_register(&i40iw_auxiliary_drv);
if (ret) {
pr_err("Failed i40iw(gen_1) auxiliary_driver_register() ret=%d\n",
ret);
return ret;
}
ret = auxiliary_driver_register(&icrdma_core_auxiliary_drv.adrv);
if (ret) {
auxiliary_driver_unregister(&i40iw_auxiliary_drv);
pr_err("Failed icrdma(gen_2) auxiliary_driver_register() ret=%d\n",
ret);
return ret;
}
ret = auxiliary_driver_register(&ig3rdma_core_auxiliary_drv.adrv);
if (ret) {
auxiliary_driver_unregister(&icrdma_core_auxiliary_drv.adrv);
auxiliary_driver_unregister(&i40iw_auxiliary_drv);
pr_err("Failed ig3rdma(gen_3) core auxiliary_driver_register() ret=%d\n",
ret);
return ret;
}
ret = auxiliary_driver_register(&ig3rdma_vport_auxiliary_drv.adrv);
if (ret) {
auxiliary_driver_unregister(&ig3rdma_core_auxiliary_drv.adrv);
auxiliary_driver_unregister(&icrdma_core_auxiliary_drv.adrv);
auxiliary_driver_unregister(&i40iw_auxiliary_drv);
pr_err("Failed ig3rdma vport auxiliary_driver_register() ret=%d\n",
ret);
return ret;
}
irdma_register_notifiers();
return 0;
}
static void __exit irdma_exit_module(void)
{
irdma_unregister_notifiers();
auxiliary_driver_unregister(&icrdma_core_auxiliary_drv.adrv);
auxiliary_driver_unregister(&i40iw_auxiliary_drv);
auxiliary_driver_unregister(&ig3rdma_core_auxiliary_drv.adrv);
auxiliary_driver_unregister(&ig3rdma_vport_auxiliary_drv.adrv);
}
module_init(irdma_init_module);
module_exit(irdma_exit_module);
linux-7.2.3/drivers/infiniband/hw/irdma/main.h
+579
View File
@@ -0,0 +1,579 @@
/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
/* Copyright (c) 2015 - 2021 Intel Corporation */
#ifndef IRDMA_MAIN_H
#define IRDMA_MAIN_H
#include <linux/ip.h>
#include <linux/tcp.h>
#include <linux/if_vlan.h>
#include <net/addrconf.h>
#include <net/netevent.h>
#include <net/tcp.h>
#include <net/ip6_route.h>
#include <net/flow.h>
#include <net/secure_seq.h>
#include <linux/netdevice.h>
#include <linux/etherdevice.h>
#include <linux/inetdevice.h>
#include <linux/spinlock.h>
#include <linux/kernel.h>
#include <linux/delay.h>
#include <linux/pci.h>
#include <linux/dma-mapping.h>
#include <linux/workqueue.h>
#include <linux/slab.h>
#include <linux/io.h>
#include <linux/iopoll.h>
#include <linux/crc32c.h>
#include <linux/kthread.h>
#ifndef CONFIG_64BIT
#include <linux/io-64-nonatomic-lo-hi.h>
#endif
#include <linux/auxiliary_bus.h>
#include <linux/net/intel/iidc_rdma.h>
#include <rdma/ib_smi.h>
#include <rdma/ib_verbs.h>
#include <rdma/ib_pack.h>
#include <rdma/rdma_cm.h>
#include <rdma/iw_cm.h>
#include <rdma/ib_user_verbs.h>
#include <rdma/ib_cache.h>
#include <rdma/iter.h>
#include <rdma/uverbs_ioctl.h>
#include "osdep.h"
#include "defs.h"
#include "hmc.h"
#include "type.h"
#include "ws.h"
#include "protos.h"
#include "pble.h"
#include "cm.h"
#include <rdma/irdma-abi.h>
#include "verbs.h"
#include "user.h"
#include "puda.h"
extern struct auxiliary_driver i40iw_auxiliary_drv;
extern struct iidc_rdma_core_auxiliary_drv icrdma_core_auxiliary_drv;
extern struct iidc_rdma_core_auxiliary_drv ig3rdma_core_auxiliary_drv;
#define IRDMA_FW_VER_DEFAULT 2
#define IRDMA_HW_VER 2
#define IRDMA_ARP_ADD 1
#define IRDMA_ARP_DELETE 2
#define IRDMA_ARP_RESOLVE 3
#define IRDMA_MACIP_ADD 1
#define IRDMA_MACIP_DELETE 2
#define IW_GEN_3_CCQ_SIZE (2 * IRDMA_CQP_SW_SQSIZE_2048 + 2)
#define IW_CCQ_SIZE (IRDMA_CQP_SW_SQSIZE_2048 + 2)
#define IW_CEQ_SIZE 2048
#define IW_AEQ_SIZE 2048
#define RX_BUF_SIZE (1536 + 8)
#define IW_REG0_SIZE (4 * 1024)
#define IW_TX_TIMEOUT (6 * HZ)
#define IW_FIRST_QPN 1
#define IW_SW_CONTEXT_ALIGN 1024
#define MAX_DPC_ITERATIONS 128
#define IRDMA_EVENT_TIMEOUT_MS 5000
#define IRDMA_VCHNL_EVENT_TIMEOUT 100000
#define IRDMA_RST_TIMEOUT_HZ 4
#define IRDMA_NO_QSET 0xffff
#define IW_CFG_FPM_QP_COUNT 32768
#define IRDMA_MAX_PAGES_PER_FMR 262144
#define IRDMA_MIN_PAGES_PER_FMR 1
#define IRDMA_CQP_COMPL_RQ_WQE_FLUSHED 2
#define IRDMA_CQP_COMPL_SQ_WQE_FLUSHED 3
#define IRDMA_Q_TYPE_PE_AEQ 0x80
#define IRDMA_Q_INVALID_IDX 0xffff
#define IRDMA_REM_ENDPOINT_TRK_QPID 3
#define IRDMA_DRV_OPT_ENA_MPA_VER_0 0x00000001
#define IRDMA_DRV_OPT_DISABLE_MPA_CRC 0x00000002
#define IRDMA_DRV_OPT_DISABLE_FIRST_WRITE 0x00000004
#define IRDMA_DRV_OPT_DISABLE_INTF 0x00000008
#define IRDMA_DRV_OPT_ENA_MSI 0x00000010
#define IRDMA_DRV_OPT_DUAL_LOGICAL_PORT 0x00000020
#define IRDMA_DRV_OPT_NO_INLINE_DATA 0x00000080
#define IRDMA_DRV_OPT_DISABLE_INT_MOD 0x00000100
#define IRDMA_DRV_OPT_DISABLE_VIRT_WQ 0x00000200
#define IRDMA_DRV_OPT_ENA_PAU 0x00000400
#define IRDMA_DRV_OPT_MCAST_LOGPORT_MAP 0x00000800
#define IW_HMC_OBJ_TYPE_NUM ARRAY_SIZE(iw_hmc_obj_types)
#define IRDMA_ROCE_CWND_DEFAULT 0x400
#define IRDMA_ROCE_ACKCREDS_DEFAULT 0x1E
#define IRDMA_FLUSH_SQ BIT(0)
#define IRDMA_FLUSH_RQ BIT(1)
#define IRDMA_REFLUSH BIT(2)
#define IRDMA_FLUSH_WAIT BIT(3)
#define IRDMA_IRQ_NAME_STR_LEN (64)
#define IRDMA_NUM_AEQ_MSIX 1
#define IRDMA_MIN_MSIX 2
enum init_completion_state {
INVALID_STATE = 0,
INITIAL_STATE,
CQP_CREATED,
HMC_OBJS_CREATED,
HW_RSRC_INITIALIZED,
CCQ_CREATED,
CEQ0_CREATED,
CEQS_CREATED,
PBLE_CHUNK_MEM,
AEQ_CREATED,
ILQ_CREATED,
IEQ_CREATED, /* Last state of probe */
IP_ADDR_REGISTERED, /* Last state of open */
};
struct irdma_rsrc_limits {
u32 qplimit;
u32 mrlimit;
u32 cqlimit;
};
struct irdma_cqp_err_info {
u16 maj;
u16 min;
const char *desc;
};
struct irdma_cqp_compl_info {
u32 op_ret_val;
u16 maj_err_code;
u16 min_err_code;
bool error;
u8 op_code;
};
struct irdma_cqp_request {
struct cqp_cmds_info info;
struct completion comp;
struct list_head list;
refcount_t refcnt;
void (*callback_fcn)(struct irdma_cqp_request *cqp_request);
void *param;
struct irdma_cqp_compl_info compl_info;
bool waiting:1;
bool dynamic:1;
bool pending:1;
};
struct irdma_cqp {
struct irdma_sc_cqp sc_cqp;
spinlock_t req_lock; /* protect CQP request list */
spinlock_t compl_lock; /* protect CQP completion processing */
wait_queue_head_t waitq;
wait_queue_head_t remove_wq;
struct irdma_dma_mem sq;
struct irdma_dma_mem host_ctx;
u64 *scratch_array;
struct irdma_cqp_request *cqp_requests;
struct irdma_ooo_cqp_op *oop_op_array;
struct list_head cqp_avail_reqs;
struct list_head cqp_pending_reqs;
};
struct irdma_ccq {
struct irdma_sc_cq sc_cq;
struct irdma_dma_mem mem_cq;
struct irdma_dma_mem shadow_area;
};
struct irdma_ceq {
struct irdma_sc_ceq sc_ceq;
struct irdma_dma_mem mem;
u32 irq;
u32 msix_idx;
struct irdma_pci_f *rf;
struct tasklet_struct dpc_tasklet;
spinlock_t ce_lock; /* sync cq destroy with cq completion event notification */
};
struct irdma_aeq {
struct irdma_sc_aeq sc_aeq;
struct irdma_dma_mem mem;
struct irdma_pble_alloc palloc;
bool virtual_map;
};
struct irdma_arp_entry {
u32 ip_addr[4];
u8 mac_addr[ETH_ALEN];
};
struct irdma_msix_vector {
u32 idx;
u32 irq;
u32 cpu_affinity;
u32 ceq_id;
cpumask_t mask;
char name[IRDMA_IRQ_NAME_STR_LEN];
};
struct irdma_mc_table_info {
u32 mgn;
u32 dest_ip[4];
bool lan_fwd:1;
bool ipv4_valid:1;
};
struct mc_table_list {
struct list_head list;
struct irdma_mc_table_info mc_info;
struct irdma_mcast_grp_info mc_grp_ctx;
};
struct irdma_qv_info {
u32 v_idx; /* msix_vector */
u16 ceq_idx;
u16 aeq_idx;
u8 itr_idx;
};
struct irdma_qvlist_info {
u32 num_vectors;
struct irdma_qv_info qv_info[] __counted_by(num_vectors);
};
struct irdma_gen_ops {
void (*request_reset)(struct irdma_pci_f *rf);
int (*register_qset)(struct irdma_sc_vsi *vsi,
struct irdma_ws_node *tc_node);
void (*unregister_qset)(struct irdma_sc_vsi *vsi,
struct irdma_ws_node *tc_node);
};
struct irdma_pci_f {
bool reset:1;
bool rsrc_created:1;
bool msix_shared:1;
bool hwqp1_rsvd:1;
u8 rsrc_profile;
u8 *hmc_info_mem;
u8 *mem_rsrc;
u8 rdma_ver;
u8 rst_to;
u8 pf_id;
enum irdma_protocol_used protocol_used;
u32 sd_type;
u32 msix_count;
u32 max_mr;
u32 max_qp;
u32 max_cq;
u32 max_srq;
u32 next_srq;
u32 max_ah;
u32 next_ah;
u32 max_mcg;
u32 next_mcg;
u32 max_pd;
u32 next_qp;
u32 next_cq;
u32 next_pd;
u32 max_mr_size;
u32 max_cqe;
u32 mr_stagmask;
u32 used_pds;
u32 used_cqs;
u32 used_srqs;
u32 used_mrs;
u32 used_qps;
u32 arp_table_size;
u32 next_arp_index;
u32 ceqs_count;
u32 next_ws_node_id;
u32 max_ws_node_id;
u32 limits_sel;
unsigned long *allocated_ws_nodes;
unsigned long *allocated_qps;
unsigned long *allocated_cqs;
unsigned long *allocated_srqs;
unsigned long *allocated_mrs;
unsigned long *allocated_pds;
unsigned long *allocated_mcgs;
unsigned long *allocated_ahs;
unsigned long *allocated_arps;
enum init_completion_state init_state;
struct irdma_sc_dev sc_dev;
struct pci_dev *pcidev;
void *cdev;
struct irdma_hw hw;
struct irdma_cqp cqp;
struct irdma_ccq ccq;
struct irdma_aeq aeq;
struct irdma_ceq *ceqlist;
struct irdma_hmc_pble_rsrc *pble_rsrc;
struct irdma_arp_entry *arp_table;
spinlock_t arp_lock; /*protect ARP table access*/
spinlock_t rsrc_lock; /* protect HW resource array access */
spinlock_t qptable_lock; /*protect QP table access*/
spinlock_t cqtable_lock; /*protect CQ table access*/
struct irdma_qp **qp_table;
struct irdma_cq **cq_table;
spinlock_t qh_list_lock; /* protect mc_qht_list */
struct mc_table_list mc_qht_list;
struct irdma_msix_vector *iw_msixtbl;
struct irdma_qvlist_info *iw_qvlist;
struct tasklet_struct dpc_tasklet;
struct msix_entry *msix_entries;
struct irdma_dma_mem obj_mem;
struct irdma_dma_mem obj_next;
atomic_t vchnl_msgs;
wait_queue_head_t vchnl_waitq;
struct workqueue_struct *cqp_cmpl_wq;
struct work_struct cqp_cmpl_work;
struct workqueue_struct *vchnl_wq;
struct irdma_sc_vsi default_vsi;
void *back_fcn;
struct irdma_gen_ops gen_ops;
struct irdma_device *iwdev;
DECLARE_HASHTABLE(ah_hash_tbl, 8);
struct mutex ah_tbl_lock; /* protect AH hash table access */
};
struct irdma_device {
struct ib_device ibdev;
struct irdma_pci_f *rf;
struct net_device *netdev;
struct workqueue_struct *cleanup_wq;
struct irdma_sc_vsi vsi;
struct irdma_cm_core cm_core;
u32 roce_cwnd;
u32 roce_ackcreds;
u32 vendor_id;
u32 vendor_part_id;
u32 push_mode;
u32 rcv_wnd;
u16 mac_ip_table_idx;
u16 vsi_num;
u16 vport_id;
u8 rcv_wscale;
u8 iw_status;
bool roce_mode:1;
bool roce_dcqcn_en:1;
bool dcb_vlan_mode:1;
bool iw_ooo:1;
bool is_vport:1;
enum init_completion_state init_state;
wait_queue_head_t suspend_wq;
};
static inline struct irdma_device *to_iwdev(struct ib_device *ibdev)
{
return container_of(ibdev, struct irdma_device, ibdev);
}
static inline struct irdma_ucontext *to_ucontext(struct ib_ucontext *ibucontext)
{
return container_of(ibucontext, struct irdma_ucontext, ibucontext);
}
static inline struct irdma_user_mmap_entry *
to_irdma_mmap_entry(struct rdma_user_mmap_entry *rdma_entry)
{
return container_of(rdma_entry, struct irdma_user_mmap_entry,
rdma_entry);
}
static inline struct irdma_pd *to_iwpd(struct ib_pd *ibpd)
{
return container_of(ibpd, struct irdma_pd, ibpd);
}
static inline struct irdma_ah *to_iwah(struct ib_ah *ibah)
{
return container_of(ibah, struct irdma_ah, ibah);
}
static inline struct irdma_mr *to_iwmr(struct ib_mr *ibmr)
{
return container_of(ibmr, struct irdma_mr, ibmr);
}
static inline struct irdma_mr *to_iwmw(struct ib_mw *ibmw)
{
return container_of(ibmw, struct irdma_mr, ibmw);
}
static inline struct irdma_cq *to_iwcq(struct ib_cq *ibcq)
{
return container_of(ibcq, struct irdma_cq, ibcq);
}
static inline struct irdma_qp *to_iwqp(struct ib_qp *ibqp)
{
return container_of(ibqp, struct irdma_qp, ibqp);
}
static inline struct irdma_pci_f *dev_to_rf(struct irdma_sc_dev *dev)
{
return container_of(dev, struct irdma_pci_f, sc_dev);
}
static inline struct irdma_srq *to_iwsrq(struct ib_srq *ibsrq)
{
return container_of(ibsrq, struct irdma_srq, ibsrq);
}
/**
* irdma_alloc_resource - allocate a resource
* @iwdev: device pointer
* @resource_array: resource bit array:
* @max_resources: maximum resource number
* @req_resources_num: Allocated resource number
* @next: next free id
**/
static inline int irdma_alloc_rsrc(struct irdma_pci_f *rf,
unsigned long *rsrc_array, u32 max_rsrc,
u32 *req_rsrc_num, u32 *next)
{
u32 rsrc_num;
unsigned long flags;
spin_lock_irqsave(&rf->rsrc_lock, flags);
rsrc_num = find_next_zero_bit(rsrc_array, max_rsrc, *next);
if (rsrc_num >= max_rsrc) {
rsrc_num = find_first_zero_bit(rsrc_array, max_rsrc);
if (rsrc_num >= max_rsrc) {
spin_unlock_irqrestore(&rf->rsrc_lock, flags);
ibdev_dbg(&rf->iwdev->ibdev,
"ERR: resource [%d] allocation failed\n",
rsrc_num);
return -EOVERFLOW;
}
}
__set_bit(rsrc_num, rsrc_array);
*next = rsrc_num + 1;
if (*next == max_rsrc)
*next = 0;
*req_rsrc_num = rsrc_num;
spin_unlock_irqrestore(&rf->rsrc_lock, flags);
return 0;
}
/**
* irdma_free_resource - free a resource
* @iwdev: device pointer
* @resource_array: resource array for the resource_num
* @resource_num: resource number to free
**/
static inline void irdma_free_rsrc(struct irdma_pci_f *rf,
unsigned long *rsrc_array, u32 rsrc_num)
{
unsigned long flags;
spin_lock_irqsave(&rf->rsrc_lock, flags);
__clear_bit(rsrc_num, rsrc_array);
spin_unlock_irqrestore(&rf->rsrc_lock, flags);
}
int irdma_ctrl_init_hw(struct irdma_pci_f *rf);
void irdma_ctrl_deinit_hw(struct irdma_pci_f *rf);
int irdma_rt_init_hw(struct irdma_device *iwdev,
struct irdma_l2params *l2params);
void irdma_rt_deinit_hw(struct irdma_device *iwdev);
void irdma_qp_add_ref(struct ib_qp *ibqp);
void irdma_qp_rem_ref(struct ib_qp *ibqp);
void irdma_free_lsmm_rsrc(struct irdma_qp *iwqp);
struct ib_qp *irdma_get_qp(struct ib_device *ibdev, int qpn);
void irdma_flush_wqes(struct irdma_qp *iwqp, u32 flush_mask);
void irdma_manage_arp_cache(struct irdma_pci_f *rf,
const unsigned char *mac_addr,
u32 *ip_addr, bool ipv4, u32 action);
struct irdma_apbvt_entry *irdma_add_apbvt(struct irdma_device *iwdev, u16 port);
void irdma_del_apbvt(struct irdma_device *iwdev,
struct irdma_apbvt_entry *entry);
struct irdma_cqp_request *irdma_alloc_and_get_cqp_request(struct irdma_cqp *cqp,
bool wait);
void irdma_free_cqp_request(struct irdma_cqp *cqp,
struct irdma_cqp_request *cqp_request);
void irdma_put_cqp_request(struct irdma_cqp *cqp,
struct irdma_cqp_request *cqp_request);
int irdma_alloc_local_mac_entry(struct irdma_pci_f *rf, u16 *mac_tbl_idx);
int irdma_add_local_mac_entry(struct irdma_pci_f *rf, const u8 *mac_addr, u16 idx);
void irdma_del_local_mac_entry(struct irdma_pci_f *rf, u16 idx);
u32 irdma_initialize_hw_rsrc(struct irdma_pci_f *rf);
void irdma_port_ibevent(struct irdma_device *iwdev);
void irdma_cm_disconn(struct irdma_qp *qp);
bool irdma_cqp_crit_err(struct irdma_sc_dev *dev, u8 cqp_cmd,
u16 maj_err_code, u16 min_err_code);
int irdma_handle_cqp_op(struct irdma_pci_f *rf,
struct irdma_cqp_request *cqp_request);
int irdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask,
struct ib_udata *udata);
int irdma_modify_qp_roce(struct ib_qp *ibqp, struct ib_qp_attr *attr,
int attr_mask, struct ib_udata *udata);
void irdma_cq_add_ref(struct ib_cq *ibcq);
void irdma_cq_rem_ref(struct ib_cq *ibcq);
void irdma_cq_wq_destroy(struct irdma_pci_f *rf, struct irdma_sc_cq *cq);
void irdma_srq_event(struct irdma_sc_srq *srq);
void irdma_srq_wq_destroy(struct irdma_pci_f *rf, struct irdma_sc_srq *srq);
void irdma_cleanup_pending_cqp_op(struct irdma_pci_f *rf);
int irdma_get_timeout_threshold(struct irdma_sc_dev *dev);
int irdma_hw_modify_qp(struct irdma_device *iwdev, struct irdma_qp *iwqp,
struct irdma_modify_qp_info *info, bool wait);
int irdma_qp_suspend_resume(struct irdma_sc_qp *qp, bool suspend);
int irdma_manage_qhash(struct irdma_device *iwdev, struct irdma_cm_info *cminfo,
enum irdma_quad_entry_type etype,
enum irdma_quad_hash_manage_type mtype, void *cmnode,
bool wait);
void irdma_receive_ilq(struct irdma_sc_vsi *vsi, struct irdma_puda_buf *rbuf);
void irdma_free_sqbuf(struct irdma_sc_vsi *vsi, void *bufp);
void irdma_free_qp_rsrc(struct irdma_qp *iwqp);
int irdma_setup_cm_core(struct irdma_device *iwdev, u8 ver);
void irdma_cleanup_cm_core(struct irdma_cm_core *cm_core);
void irdma_next_iw_state(struct irdma_qp *iwqp, u8 state, u8 del_hash, u8 term,
u8 term_len);
int irdma_send_syn(struct irdma_cm_node *cm_node, u32 sendack);
int irdma_send_reset(struct irdma_cm_node *cm_node);
struct irdma_cm_node *irdma_find_node(struct irdma_cm_core *cm_core,
u16 rem_port, u32 *rem_addr, u16 loc_port,
u32 *loc_addr, u16 vlan_id);
int irdma_hw_flush_wqes(struct irdma_pci_f *rf, struct irdma_sc_qp *qp,
struct irdma_qp_flush_info *info, bool wait);
void irdma_gen_ae(struct irdma_pci_f *rf, struct irdma_sc_qp *qp,
struct irdma_gen_ae_info *info, bool wait);
void irdma_copy_ip_ntohl(u32 *dst, __be32 *src);
void irdma_copy_ip_htonl(__be32 *dst, u32 *src);
u16 irdma_get_vlan_ipv4(u32 *addr);
void irdma_get_vlan_mac_ipv6(u32 *addr, u16 *vlan_id, u8 *mac);
struct ib_mr *irdma_reg_phys_mr(struct ib_pd *ib_pd, u64 addr, u64 size,
int acc, u64 *iova_start, bool dma_mr);
int irdma_upload_qp_context(struct irdma_qp *iwqp, bool freeze, bool raw);
void irdma_cqp_ce_handler(struct irdma_pci_f *rf, struct irdma_sc_cq *cq);
int irdma_ah_cqp_op(struct irdma_pci_f *rf, struct irdma_sc_ah *sc_ah, u8 cmd,
bool wait,
void (*callback_fcn)(struct irdma_cqp_request *cqp_request),
void *cb_param);
void irdma_gsi_ud_qp_ah_cb(struct irdma_cqp_request *cqp_request);
int irdma_inetaddr_event(struct notifier_block *notifier, unsigned long event,
void *ptr);
int irdma_inet6addr_event(struct notifier_block *notifier, unsigned long event,
void *ptr);
int irdma_net_event(struct notifier_block *notifier, unsigned long event,
void *ptr);
int irdma_netdevice_event(struct notifier_block *notifier, unsigned long event,
void *ptr);
void irdma_add_ip(struct irdma_device *iwdev);
void cqp_compl_worker(struct work_struct *work);
void irdma_log_invalid_mtu(u16 mtu, struct irdma_sc_dev *dev);
#endif /* IRDMA_MAIN_H */
linux-7.2.3/drivers/infiniband/hw/irdma/osdep.h
+74
View File
@@ -0,0 +1,74 @@
/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
/* Copyright (c) 2015 - 2021 Intel Corporation */
#ifndef IRDMA_OSDEP_H
#define IRDMA_OSDEP_H
#include <linux/pci.h>
#include <linux/bitfield.h>
#include <rdma/ib_verbs.h>
#include <net/dscp.h>
#define STATS_TIMER_DELAY 60000
struct irdma_dma_info {
dma_addr_t *dmaaddrs;
};
struct irdma_dma_mem {
void *va;
dma_addr_t pa;
u32 size;
} __packed;
struct irdma_virt_mem {
void *va;
u32 size;
} __packed;
struct irdma_sc_vsi;
struct irdma_sc_dev;
struct irdma_sc_qp;
struct irdma_puda_buf;
struct irdma_puda_cmpl_info;
struct irdma_update_sds_info;
struct irdma_hmc_fcn_info;
struct irdma_manage_vf_pble_info;
struct irdma_hw;
struct irdma_pci_f;
struct ib_device *to_ibdev(struct irdma_sc_dev *dev);
void irdma_ieq_mpa_crc_ae(struct irdma_sc_dev *dev, struct irdma_sc_qp *qp);
enum irdma_status_code irdma_vf_wait_vchnl_resp(struct irdma_sc_dev *dev);
bool irdma_vf_clear_to_send(struct irdma_sc_dev *dev);
void irdma_add_dev_ref(struct irdma_sc_dev *dev);
void irdma_put_dev_ref(struct irdma_sc_dev *dev);
int irdma_ieq_check_mpacrc(const void *addr, u32 len, u32 val);
struct irdma_sc_qp *irdma_ieq_get_qp(struct irdma_sc_dev *dev,
struct irdma_puda_buf *buf);
void irdma_send_ieq_ack(struct irdma_sc_qp *qp);
void irdma_ieq_update_tcpip_info(struct irdma_puda_buf *buf, u16 len,
u32 seqnum);
int irdma_puda_get_tcpip_info(struct irdma_puda_cmpl_info *info,
struct irdma_puda_buf *buf);
int irdma_cqp_sds_cmd(struct irdma_sc_dev *dev,
struct irdma_update_sds_info *info);
int irdma_cqp_manage_hmc_fcn_cmd(struct irdma_sc_dev *dev,
struct irdma_hmc_fcn_info *hmcfcninfo,
u16 *pmf_idx);
int irdma_alloc_query_fpm_buf(struct irdma_sc_dev *dev,
struct irdma_dma_mem *mem);
void *irdma_remove_cqp_head(struct irdma_sc_dev *dev);
void irdma_term_modify_qp(struct irdma_sc_qp *qp, u8 next_state, u8 term,
u8 term_len);
void irdma_terminate_done(struct irdma_sc_qp *qp, int timeout_occurred);
void irdma_terminate_start_timer(struct irdma_sc_qp *qp);
void irdma_terminate_del_timer(struct irdma_sc_qp *qp);
void irdma_hw_stats_start_timer(struct irdma_sc_vsi *vsi);
void irdma_hw_stats_stop_timer(struct irdma_sc_vsi *vsi);
void wr32(struct irdma_hw *hw, u32 reg, u32 val);
u32 rd32(struct irdma_hw *hw, u32 reg);
u64 rd64(struct irdma_hw *hw, u32 reg);
int irdma_map_vm_page_list(struct irdma_hw *hw, void *va, dma_addr_t *pg_dma,
u32 pg_cnt);
void irdma_unmap_vm_page_list(struct irdma_hw *hw, dma_addr_t *pg_dma, u32 pg_cnt);
#endif /* IRDMA_OSDEP_H */
linux-7.2.3/drivers/infiniband/hw/irdma/pble.c
+519
View File
@@ -0,0 +1,519 @@
// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
/* Copyright (c) 2015 - 2021 Intel Corporation */
#include "osdep.h"
#include "hmc.h"
#include "defs.h"
#include "type.h"
#include "protos.h"
#include "pble.h"
static int add_pble_prm(struct irdma_hmc_pble_rsrc *pble_rsrc);
/**
* irdma_destroy_pble_prm - destroy prm during module unload
* @pble_rsrc: pble resources
*/
void irdma_destroy_pble_prm(struct irdma_hmc_pble_rsrc *pble_rsrc)
{
struct irdma_chunk *chunk;
struct irdma_pble_prm *pinfo = &pble_rsrc->pinfo;
while (!list_empty(&pinfo->clist)) {
chunk = (struct irdma_chunk *) pinfo->clist.next;
list_del(&chunk->list);
if (chunk->type == PBLE_SD_PAGED)
irdma_pble_free_paged_mem(chunk);
bitmap_free(chunk->bitmapbuf);
kfree(chunk->chunkmem.va);
}
}
/**
* irdma_hmc_init_pble - Initialize pble resources during module load
* @dev: irdma_sc_dev struct
* @pble_rsrc: pble resources
*/
int irdma_hmc_init_pble(struct irdma_sc_dev *dev,
struct irdma_hmc_pble_rsrc *pble_rsrc)
{
struct irdma_hmc_info *hmc_info;
u32 fpm_idx = 0;
int status = 0;
hmc_info = dev->hmc_info;
pble_rsrc->dev = dev;
pble_rsrc->fpm_base_addr = hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].base;
/* Start pble' on 4k boundary */
if (pble_rsrc->fpm_base_addr & 0xfff)
fpm_idx = (4096 - (pble_rsrc->fpm_base_addr & 0xfff)) >> 3;
pble_rsrc->unallocated_pble =
hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt - fpm_idx;
pble_rsrc->next_fpm_addr = pble_rsrc->fpm_base_addr + (fpm_idx << 3);
pble_rsrc->pinfo.pble_shift = PBLE_SHIFT;
mutex_init(&pble_rsrc->pble_mutex_lock);
spin_lock_init(&pble_rsrc->pinfo.prm_lock);
INIT_LIST_HEAD(&pble_rsrc->pinfo.clist);
if (add_pble_prm(pble_rsrc)) {
irdma_destroy_pble_prm(pble_rsrc);
status = -ENOMEM;
}
return status;
}
/**
* get_sd_pd_idx - Returns sd index, pd index and rel_pd_idx from fpm address
* @pble_rsrc: structure containing fpm address
* @idx: where to return indexes
*/
static void get_sd_pd_idx(struct irdma_hmc_pble_rsrc *pble_rsrc,
struct sd_pd_idx *idx)
{
idx->sd_idx = pble_rsrc->next_fpm_addr / IRDMA_HMC_DIRECT_BP_SIZE;
idx->pd_idx = (u32)(pble_rsrc->next_fpm_addr / IRDMA_HMC_PAGED_BP_SIZE);
idx->rel_pd_idx = (idx->pd_idx % IRDMA_HMC_PD_CNT_IN_SD);
}
/**
* add_sd_direct - add sd direct for pble
* @pble_rsrc: pble resource ptr
* @info: page info for sd
*/
static int add_sd_direct(struct irdma_hmc_pble_rsrc *pble_rsrc,
struct irdma_add_page_info *info)
{
struct irdma_sc_dev *dev = pble_rsrc->dev;
int ret_code = 0;
struct sd_pd_idx *idx = &info->idx;
struct irdma_chunk *chunk = info->chunk;
struct irdma_hmc_info *hmc_info = info->hmc_info;
struct irdma_hmc_sd_entry *sd_entry = info->sd_entry;
u32 offset = 0;
if (!sd_entry->valid) {
ret_code = irdma_add_sd_table_entry(dev->hw, hmc_info,
info->idx.sd_idx,
IRDMA_SD_TYPE_DIRECT,
IRDMA_HMC_DIRECT_BP_SIZE);
if (ret_code)
return ret_code;
chunk->type = PBLE_SD_CONTIGOUS;
}
offset = idx->rel_pd_idx << HMC_PAGED_BP_SHIFT;
chunk->size = info->pages << HMC_PAGED_BP_SHIFT;
chunk->vaddr = sd_entry->u.bp.addr.va + offset;
chunk->fpm_addr = pble_rsrc->next_fpm_addr;
ibdev_dbg(to_ibdev(dev),
"PBLE: chunk_size[%lld] = 0x%llx vaddr=0x%p fpm_addr = %llx\n",
chunk->size, chunk->size, chunk->vaddr, chunk->fpm_addr);
return 0;
}
/**
* fpm_to_idx - given fpm address, get pble index
* @pble_rsrc: pble resource management
* @addr: fpm address for index
*/
static u32 fpm_to_idx(struct irdma_hmc_pble_rsrc *pble_rsrc, u64 addr)
{
u64 idx;
idx = (addr - (pble_rsrc->fpm_base_addr)) >> 3;
return (u32)idx;
}
/**
* add_bp_pages - add backing pages for sd
* @pble_rsrc: pble resource management
* @info: page info for sd
*/
static int add_bp_pages(struct irdma_hmc_pble_rsrc *pble_rsrc,
struct irdma_add_page_info *info)
{
struct irdma_sc_dev *dev = pble_rsrc->dev;
u8 *addr;
struct irdma_dma_mem mem;
struct irdma_hmc_pd_entry *pd_entry;
struct irdma_hmc_sd_entry *sd_entry = info->sd_entry;
struct irdma_hmc_info *hmc_info = info->hmc_info;
struct irdma_chunk *chunk = info->chunk;
int status = 0;
u32 rel_pd_idx = info->idx.rel_pd_idx;
u32 pd_idx = info->idx.pd_idx;
u32 i;
if (irdma_pble_get_paged_mem(chunk, info->pages))
return -ENOMEM;
status = irdma_add_sd_table_entry(dev->hw, hmc_info, info->idx.sd_idx,
IRDMA_SD_TYPE_PAGED,
IRDMA_HMC_DIRECT_BP_SIZE);
if (status)
goto error;
addr = chunk->vaddr;
for (i = 0; i < info->pages; i++) {
mem.pa = (u64)chunk->dmainfo.dmaaddrs[i];
mem.size = 4096;
mem.va = addr;
pd_entry = &sd_entry->u.pd_table.pd_entry[rel_pd_idx++];
if (!pd_entry->valid) {
status = irdma_add_pd_table_entry(dev, hmc_info,
pd_idx++, &mem);
if (status)
goto error;
addr += 4096;
}
}
chunk->fpm_addr = pble_rsrc->next_fpm_addr;
return 0;
error:
irdma_pble_free_paged_mem(chunk);
return status;
}
/**
* irdma_get_type - add a sd entry type for sd
* @dev: irdma_sc_dev struct
* @idx: index of sd
* @pages: pages in the sd
*/
static enum irdma_sd_entry_type irdma_get_type(struct irdma_sc_dev *dev,
struct sd_pd_idx *idx, u32 pages)
{
enum irdma_sd_entry_type sd_entry_type;
if (dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_3)
sd_entry_type = (!idx->rel_pd_idx &&
pages == IRDMA_HMC_PD_CNT_IN_SD) ?
IRDMA_SD_TYPE_DIRECT : IRDMA_SD_TYPE_PAGED;
else
sd_entry_type = (!idx->rel_pd_idx &&
pages == IRDMA_HMC_PD_CNT_IN_SD &&
dev->privileged) ?
IRDMA_SD_TYPE_DIRECT : IRDMA_SD_TYPE_PAGED;
return sd_entry_type;
}
/**
* add_pble_prm - add a sd entry for pble resoure
* @pble_rsrc: pble resource management
*/
static int add_pble_prm(struct irdma_hmc_pble_rsrc *pble_rsrc)
{
struct irdma_sc_dev *dev = pble_rsrc->dev;
struct irdma_hmc_sd_entry *sd_entry;
struct irdma_hmc_info *hmc_info;
struct irdma_chunk *chunk;
struct irdma_add_page_info info;
struct sd_pd_idx *idx = &info.idx;
int ret_code = 0;
enum irdma_sd_entry_type sd_entry_type;
u64 sd_reg_val = 0;
struct irdma_virt_mem chunkmem;
u32 pages;
if (pble_rsrc->unallocated_pble < PBLE_PER_PAGE)
return -ENOMEM;
if (pble_rsrc->next_fpm_addr & 0xfff)
return -EINVAL;
chunkmem.size = sizeof(*chunk);
chunkmem.va = kzalloc(chunkmem.size, GFP_KERNEL);
if (!chunkmem.va)
return -ENOMEM;
chunk = chunkmem.va;
chunk->chunkmem = chunkmem;
hmc_info = dev->hmc_info;
chunk->dev = dev;
chunk->fpm_addr = pble_rsrc->next_fpm_addr;
get_sd_pd_idx(pble_rsrc, idx);
sd_entry = &hmc_info->sd_table.sd_entry[idx->sd_idx];
pages = (idx->rel_pd_idx) ? (IRDMA_HMC_PD_CNT_IN_SD - idx->rel_pd_idx) :
IRDMA_HMC_PD_CNT_IN_SD;
pages = min(pages, pble_rsrc->unallocated_pble >> PBLE_512_SHIFT);
info.chunk = chunk;
info.hmc_info = hmc_info;
info.pages = pages;
info.sd_entry = sd_entry;
if (!sd_entry->valid)
sd_entry_type = irdma_get_type(dev, idx, pages);
else
sd_entry_type = sd_entry->entry_type;
ibdev_dbg(to_ibdev(dev),
"PBLE: pages = %d, unallocated_pble[%d] current_fpm_addr = %llx\n",
pages, pble_rsrc->unallocated_pble,
pble_rsrc->next_fpm_addr);
ibdev_dbg(to_ibdev(dev), "PBLE: sd_entry_type = %d\n", sd_entry_type);
if (sd_entry_type == IRDMA_SD_TYPE_DIRECT)
ret_code = add_sd_direct(pble_rsrc, &info);
if (ret_code)
sd_entry_type = IRDMA_SD_TYPE_PAGED;
else
pble_rsrc->stats_direct_sds++;
if (sd_entry_type == IRDMA_SD_TYPE_PAGED) {
ret_code = add_bp_pages(pble_rsrc, &info);
if (ret_code)
goto error;
else
pble_rsrc->stats_paged_sds++;
}
ret_code = irdma_prm_add_pble_mem(&pble_rsrc->pinfo, chunk);
if (ret_code)
goto error;
pble_rsrc->next_fpm_addr += chunk->size;
ibdev_dbg(to_ibdev(dev),
"PBLE: next_fpm_addr = %llx chunk_size[%llu] = 0x%llx\n",
pble_rsrc->next_fpm_addr, chunk->size, chunk->size);
pble_rsrc->unallocated_pble -= (u32)(chunk->size >> 3);
sd_reg_val = (sd_entry_type == IRDMA_SD_TYPE_PAGED) ?
sd_entry->u.pd_table.pd_page_addr.pa :
sd_entry->u.bp.addr.pa;
if ((dev->privileged && !sd_entry->valid) ||
dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_3) {
ret_code = irdma_hmc_sd_one(dev, hmc_info->hmc_fn_id,
sd_reg_val, idx->sd_idx,
sd_entry->entry_type, true);
if (ret_code)
goto error;
}
list_add(&chunk->list, &pble_rsrc->pinfo.clist);
sd_entry->valid = true;
return 0;
error:
bitmap_free(chunk->bitmapbuf);
kfree(chunk->chunkmem.va);
return ret_code;
}
/**
* free_lvl2 - fee level 2 pble
* @pble_rsrc: pble resource management
* @palloc: level 2 pble allocation
*/
static void free_lvl2(struct irdma_hmc_pble_rsrc *pble_rsrc,
struct irdma_pble_alloc *palloc)
{
u32 i;
struct irdma_pble_level2 *lvl2 = &palloc->level2;
struct irdma_pble_info *root = &lvl2->root;
struct irdma_pble_info *leaf = lvl2->leaf;
for (i = 0; i < lvl2->leaf_cnt; i++, leaf++) {
if (leaf->addr)
irdma_prm_return_pbles(&pble_rsrc->pinfo,
&leaf->chunkinfo);
else
break;
}
if (root->addr)
irdma_prm_return_pbles(&pble_rsrc->pinfo, &root->chunkinfo);
kfree(lvl2->leafmem.va);
lvl2->leaf = NULL;
}
/**
* get_lvl2_pble - get level 2 pble resource
* @pble_rsrc: pble resource management
* @palloc: level 2 pble allocation
*/
static int get_lvl2_pble(struct irdma_hmc_pble_rsrc *pble_rsrc,
struct irdma_pble_alloc *palloc)
{
u32 lf4k, lflast, total, i;
u32 pblcnt = PBLE_PER_PAGE;
u64 *addr;
struct irdma_pble_level2 *lvl2 = &palloc->level2;
struct irdma_pble_info *root = &lvl2->root;
struct irdma_pble_info *leaf;
int ret_code;
u64 fpm_addr;
/* number of full 512 (4K) leafs) */
lf4k = palloc->total_cnt >> 9;
lflast = palloc->total_cnt % PBLE_PER_PAGE;
total = (lflast == 0) ? lf4k : lf4k + 1;
lvl2->leaf_cnt = total;
lvl2->leafmem.size = (sizeof(*leaf) * total);
lvl2->leafmem.va = kzalloc(lvl2->leafmem.size, GFP_KERNEL);
if (!lvl2->leafmem.va)
return -ENOMEM;
lvl2->leaf = lvl2->leafmem.va;
leaf = lvl2->leaf;
ret_code = irdma_prm_get_pbles(&pble_rsrc->pinfo, &root->chunkinfo,
total << 3, &root->addr, &fpm_addr);
if (ret_code) {
kfree(lvl2->leafmem.va);
lvl2->leaf = NULL;
return -ENOMEM;
}
root->idx = fpm_to_idx(pble_rsrc, fpm_addr);
root->cnt = total;
addr = root->addr;
for (i = 0; i < total; i++, leaf++) {
pblcnt = (lflast && ((i + 1) == total)) ?
lflast : PBLE_PER_PAGE;
ret_code = irdma_prm_get_pbles(&pble_rsrc->pinfo,
&leaf->chunkinfo, pblcnt << 3,
&leaf->addr, &fpm_addr);
if (ret_code)
goto error;
leaf->idx = fpm_to_idx(pble_rsrc, fpm_addr);
leaf->cnt = pblcnt;
*addr = (u64)leaf->idx;
addr++;
}
palloc->level = PBLE_LEVEL_2;
pble_rsrc->stats_lvl2++;
return 0;
error:
free_lvl2(pble_rsrc, palloc);
return -ENOMEM;
}
/**
* get_lvl1_pble - get level 1 pble resource
* @pble_rsrc: pble resource management
* @palloc: level 1 pble allocation
*/
static int get_lvl1_pble(struct irdma_hmc_pble_rsrc *pble_rsrc,
struct irdma_pble_alloc *palloc)
{
int ret_code;
u64 fpm_addr;
struct irdma_pble_info *lvl1 = &palloc->level1;
ret_code = irdma_prm_get_pbles(&pble_rsrc->pinfo, &lvl1->chunkinfo,
palloc->total_cnt << 3, &lvl1->addr,
&fpm_addr);
if (ret_code)
return -ENOMEM;
palloc->level = PBLE_LEVEL_1;
lvl1->idx = fpm_to_idx(pble_rsrc, fpm_addr);
lvl1->cnt = palloc->total_cnt;
pble_rsrc->stats_lvl1++;
return 0;
}
/**
* get_lvl1_lvl2_pble - calls get_lvl1 and get_lvl2 pble routine
* @pble_rsrc: pble resources
* @palloc: contains all inforamtion regarding pble (idx + pble addr)
* @lvl: Bitmask for requested pble level
*/
static int get_lvl1_lvl2_pble(struct irdma_hmc_pble_rsrc *pble_rsrc,
struct irdma_pble_alloc *palloc, u8 lvl)
{
int status = 0;
status = get_lvl1_pble(pble_rsrc, palloc);
if (!status || lvl == PBLE_LEVEL_1 || palloc->total_cnt <= PBLE_PER_PAGE)
return status;
status = get_lvl2_pble(pble_rsrc, palloc);
return status;
}
/**
* irdma_get_pble - allocate pbles from the prm
* @pble_rsrc: pble resources
* @palloc: contains all inforamtion regarding pble (idx + pble addr)
* @pble_cnt: #of pbles requested
* @lvl: requested pble level mask
*/
int irdma_get_pble(struct irdma_hmc_pble_rsrc *pble_rsrc,
struct irdma_pble_alloc *palloc, u32 pble_cnt,
u8 lvl)
{
int status = 0;
int max_sds = 0;
int i;
palloc->total_cnt = pble_cnt;
palloc->level = PBLE_LEVEL_0;
mutex_lock(&pble_rsrc->pble_mutex_lock);
/*check first to see if we can get pble's without acquiring
* additional sd's
*/
status = get_lvl1_lvl2_pble(pble_rsrc, palloc, lvl);
if (!status)
goto exit;
max_sds = (palloc->total_cnt >> 18) + 1;
for (i = 0; i < max_sds; i++) {
status = add_pble_prm(pble_rsrc);
if (status)
break;
status = get_lvl1_lvl2_pble(pble_rsrc, palloc, lvl);
/* if level1_only, only go through it once */
if (!status || lvl)
break;
}
exit:
if (!status) {
pble_rsrc->allocdpbles += pble_cnt;
pble_rsrc->stats_alloc_ok++;
} else {
pble_rsrc->stats_alloc_fail++;
}
mutex_unlock(&pble_rsrc->pble_mutex_lock);
return status;
}
/**
* irdma_free_pble - put pbles back into prm
* @pble_rsrc: pble resources
* @palloc: contains all information regarding pble resource being freed
*/
void irdma_free_pble(struct irdma_hmc_pble_rsrc *pble_rsrc,
struct irdma_pble_alloc *palloc)
{
if (palloc->level == PBLE_LEVEL_2)
free_lvl2(pble_rsrc, palloc);
else
irdma_prm_return_pbles(&pble_rsrc->pinfo,
&palloc->level1.chunkinfo);
mutex_lock(&pble_rsrc->pble_mutex_lock);
pble_rsrc->freedpbles += palloc->total_cnt;
pble_rsrc->stats_alloc_freed++;
mutex_unlock(&pble_rsrc->pble_mutex_lock);
}
linux-7.2.3/drivers/infiniband/hw/irdma/pble.h
+132
View File
@@ -0,0 +1,132 @@
/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
/* Copyright (c) 2015 - 2019 Intel Corporation */
#ifndef IRDMA_PBLE_H
#define IRDMA_PBLE_H
#define PBLE_SHIFT 6
#define PBLE_PER_PAGE 512
#define HMC_PAGED_BP_SHIFT 12
#define PBLE_512_SHIFT 9
#define PBLE_INVALID_IDX 0xffffffff
enum irdma_pble_level {
PBLE_LEVEL_0 = 0,
PBLE_LEVEL_1 = 1,
PBLE_LEVEL_2 = 2,
};
enum irdma_alloc_type {
PBLE_NO_ALLOC = 0,
PBLE_SD_CONTIGOUS = 1,
PBLE_SD_PAGED = 2,
};
struct irdma_chunk;
struct irdma_pble_chunkinfo {
struct irdma_chunk *pchunk;
u64 bit_idx;
u64 bits_used;
};
struct irdma_pble_info {
u64 *addr;
u32 idx;
u32 cnt;
struct irdma_pble_chunkinfo chunkinfo;
};
struct irdma_pble_level2 {
struct irdma_pble_info root;
struct irdma_pble_info *leaf;
struct irdma_virt_mem leafmem;
u32 leaf_cnt;
};
struct irdma_pble_alloc {
u32 total_cnt;
enum irdma_pble_level level;
union {
struct irdma_pble_info level1;
struct irdma_pble_level2 level2;
};
};
struct sd_pd_idx {
u32 sd_idx;
u32 pd_idx;
u32 rel_pd_idx;
};
struct irdma_add_page_info {
struct irdma_chunk *chunk;
struct irdma_hmc_sd_entry *sd_entry;
struct irdma_hmc_info *hmc_info;
struct sd_pd_idx idx;
u32 pages;
};
struct irdma_chunk {
struct list_head list;
struct irdma_dma_info dmainfo;
unsigned long *bitmapbuf;
u32 sizeofbitmap;
u64 size;
void *vaddr;
u64 fpm_addr;
u32 pg_cnt;
enum irdma_alloc_type type;
struct irdma_sc_dev *dev;
struct irdma_virt_mem chunkmem;
};
struct irdma_pble_prm {
struct list_head clist;
spinlock_t prm_lock; /* protect prm bitmap */
u64 total_pble_alloc;
u64 free_pble_cnt;
u8 pble_shift;
};
struct irdma_hmc_pble_rsrc {
u32 unallocated_pble;
struct mutex pble_mutex_lock; /* protect PBLE resource */
struct irdma_sc_dev *dev;
u64 fpm_base_addr;
u64 next_fpm_addr;
struct irdma_pble_prm pinfo;
u64 allocdpbles;
u64 freedpbles;
u32 stats_direct_sds;
u32 stats_paged_sds;
u64 stats_alloc_ok;
u64 stats_alloc_fail;
u64 stats_alloc_freed;
u64 stats_lvl1;
u64 stats_lvl2;
};
void irdma_destroy_pble_prm(struct irdma_hmc_pble_rsrc *pble_rsrc);
int irdma_hmc_init_pble(struct irdma_sc_dev *dev,
struct irdma_hmc_pble_rsrc *pble_rsrc);
void irdma_free_pble(struct irdma_hmc_pble_rsrc *pble_rsrc,
struct irdma_pble_alloc *palloc);
int irdma_get_pble(struct irdma_hmc_pble_rsrc *pble_rsrc,
struct irdma_pble_alloc *palloc, u32 pble_cnt,
u8 lvl);
int irdma_prm_add_pble_mem(struct irdma_pble_prm *pprm,
struct irdma_chunk *pchunk);
int irdma_prm_get_pbles(struct irdma_pble_prm *pprm,
struct irdma_pble_chunkinfo *chunkinfo, u64 mem_size,
u64 **vaddr, u64 *fpm_addr);
void irdma_prm_return_pbles(struct irdma_pble_prm *pprm,
struct irdma_pble_chunkinfo *chunkinfo);
void irdma_pble_acquire_lock(struct irdma_hmc_pble_rsrc *pble_rsrc,
unsigned long *flags);
void irdma_pble_release_lock(struct irdma_hmc_pble_rsrc *pble_rsrc,
unsigned long *flags);
void irdma_pble_free_paged_mem(struct irdma_chunk *chunk);
int irdma_pble_get_paged_mem(struct irdma_chunk *chunk, u32 pg_cnt);
void irdma_prm_rem_bitmapmem(struct irdma_hw *hw, struct irdma_chunk *chunk);
#endif /* IRDMA_PBLE_H */
linux-7.2.3/drivers/infiniband/hw/irdma/protos.h
+97
View File
@@ -0,0 +1,97 @@
/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
/* Copyright (c) 2016 - 2021 Intel Corporation */
#ifndef IRDMA_PROTOS_H
#define IRDMA_PROTOS_H
#define PAUSE_TIMER_VAL 0xffff
#define REFRESH_THRESHOLD 0x7fff
#define HIGH_THRESHOLD 0x800
#define LOW_THRESHOLD 0x200
#define ALL_TC2PFC 0xff
#define CQP_COMPL_WAIT_TIME_MS 10
#define CQP_TIMEOUT_THRESHOLD 500
#define CQP_DEF_CMPL_TIMEOUT_THRESHOLD 2500
/* init operations */
int irdma_sc_dev_init(enum irdma_vers ver, struct irdma_sc_dev *dev,
struct irdma_device_init_info *info);
void irdma_sc_rt_init(struct irdma_sc_dev *dev);
void irdma_sc_cqp_post_sq(struct irdma_sc_cqp *cqp);
__le64 *irdma_sc_cqp_get_next_send_wqe(struct irdma_sc_cqp *cqp, u64 scratch);
int irdma_sc_mr_fast_register(struct irdma_sc_qp *qp,
struct irdma_fast_reg_stag_info *info,
bool post_sq);
/* HMC/FPM functions */
int irdma_sc_init_iw_hmc(struct irdma_sc_dev *dev, u8 hmc_fn_id);
/* stats misc */
int irdma_cqp_gather_stats_cmd(struct irdma_sc_dev *dev,
struct irdma_vsi_pestat *pestat, bool wait);
void irdma_cqp_gather_stats_gen1(struct irdma_sc_dev *dev,
struct irdma_vsi_pestat *pestat);
void irdma_hw_stats_read_all(struct irdma_vsi_pestat *stats,
const u64 *hw_stats_regs);
int irdma_cqp_ws_node_cmd(struct irdma_sc_dev *dev, u8 cmd,
struct irdma_ws_node_info *node_info);
int irdma_cqp_ceq_cmd(struct irdma_sc_dev *dev, struct irdma_sc_ceq *sc_ceq,
u8 op);
int irdma_cqp_aeq_cmd(struct irdma_sc_dev *dev, struct irdma_sc_aeq *sc_aeq,
u8 op);
int irdma_cqp_stats_inst_cmd(struct irdma_sc_vsi *vsi, u8 cmd,
struct irdma_stats_inst_info *stats_info);
u16 irdma_alloc_ws_node_id(struct irdma_sc_dev *dev);
void irdma_free_ws_node_id(struct irdma_sc_dev *dev, u16 node_id);
void irdma_update_stats(struct irdma_dev_hw_stats *hw_stats,
struct irdma_gather_stats *gather_stats,
struct irdma_gather_stats *last_gather_stats,
const struct irdma_hw_stat_map *map, u16 max_stat_idx);
/* vsi functions */
int irdma_vsi_stats_init(struct irdma_sc_vsi *vsi,
struct irdma_vsi_stats_info *info);
void irdma_vsi_stats_free(struct irdma_sc_vsi *vsi);
void irdma_sc_vsi_init(struct irdma_sc_vsi *vsi,
struct irdma_vsi_init_info *info);
int irdma_sc_add_cq_ctx(struct irdma_sc_ceq *ceq, struct irdma_sc_cq *cq);
void irdma_sc_remove_cq_ctx(struct irdma_sc_ceq *ceq, struct irdma_sc_cq *cq);
/* misc L2 param change functions */
void irdma_change_l2params(struct irdma_sc_vsi *vsi,
struct irdma_l2params *l2params);
void irdma_sc_suspend_resume_qps(struct irdma_sc_vsi *vsi, u8 suspend);
int irdma_cqp_qp_suspend_resume(struct irdma_sc_qp *qp, u8 cmd);
void irdma_qp_add_qos(struct irdma_sc_qp *qp);
void irdma_qp_rem_qos(struct irdma_sc_qp *qp);
struct irdma_sc_qp *irdma_get_qp_from_list(struct list_head *head,
struct irdma_sc_qp *qp);
void irdma_reinitialize_ieq(struct irdma_sc_vsi *vsi);
/* terminate functions*/
void irdma_terminate_send_fin(struct irdma_sc_qp *qp);
void irdma_terminate_connection(struct irdma_sc_qp *qp,
struct irdma_aeqe_info *info);
void irdma_terminate_received(struct irdma_sc_qp *qp,
struct irdma_aeqe_info *info);
/* dynamic memory allocation */
/* misc */
u8 irdma_get_encoded_wqe_size(u32 wqsize, enum irdma_queue_type queue_type);
void irdma_modify_qp_to_err(struct irdma_sc_qp *sc_qp);
int irdma_sc_static_hmc_pages_allocated(struct irdma_sc_cqp *cqp, u64 scratch,
u8 hmc_fn_id, bool post_sq,
bool poll_registers);
int irdma_cfg_fpm_val(struct irdma_sc_dev *dev, u32 qp_count);
int irdma_get_rdma_features(struct irdma_sc_dev *dev);
void free_sd_mem(struct irdma_sc_dev *dev);
int irdma_process_cqp_cmd(struct irdma_sc_dev *dev,
struct cqp_cmds_info *pcmdinfo);
int irdma_process_bh(struct irdma_sc_dev *dev);
int irdma_cqp_sds_cmd(struct irdma_sc_dev *dev,
struct irdma_update_sds_info *info);
int irdma_alloc_query_fpm_buf(struct irdma_sc_dev *dev,
struct irdma_dma_mem *mem);
int irdma_cqp_manage_hmc_fcn_cmd(struct irdma_sc_dev *dev,
struct irdma_hmc_fcn_info *hmcfcninfo,
u16 *pmf_idx);
void irdma_add_dev_ref(struct irdma_sc_dev *dev);
void irdma_put_dev_ref(struct irdma_sc_dev *dev);
void *irdma_remove_cqp_head(struct irdma_sc_dev *dev);
#endif /* IRDMA_PROTOS_H */
linux-7.2.3/drivers/infiniband/hw/irdma/puda.c
+1732
View File
File diff suppressed because it is too large Load Diff
linux-7.2.3/drivers/infiniband/hw/irdma/puda.h
+182
View File
@@ -0,0 +1,182 @@
/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
/* Copyright (c) 2015 - 2020 Intel Corporation */
#ifndef IRDMA_PUDA_H
#define IRDMA_PUDA_H
#define IRDMA_IEQ_MPA_FRAMING 6
#define IRDMA_TCP_OFFSET 40
#define IRDMA_IPV4_PAD 20
#define IRDMA_MRK_BLK_SZ 512
enum puda_rsrc_type {
IRDMA_PUDA_RSRC_TYPE_ILQ = 1,
IRDMA_PUDA_RSRC_TYPE_IEQ,
IRDMA_PUDA_RSRC_TYPE_MAX, /* Must be last entry */
};
enum puda_rsrc_complete {
PUDA_CQ_CREATED = 1,
PUDA_QP_CREATED,
PUDA_TX_COMPLETE,
PUDA_RX_COMPLETE,
PUDA_HASH_CRC_COMPLETE,
};
struct irdma_sc_dev;
struct irdma_sc_qp;
struct irdma_sc_cq;
struct irdma_puda_cmpl_info {
struct irdma_qp_uk *qp;
u8 q_type;
u8 l3proto;
u8 l4proto;
u16 vlan;
u32 payload_len;
u32 compl_error; /* No_err=0, else major and minor err code */
u32 qp_id;
u32 wqe_idx;
bool ipv4:1;
bool smac_valid:1;
bool vlan_valid:1;
u8 smac[ETH_ALEN];
};
struct irdma_puda_send_info {
u64 paddr; /* Physical address */
u32 len;
u32 ah_id;
u8 tcplen;
u8 maclen;
bool ipv4:1;
bool do_lpb:1;
void *scratch;
};
struct irdma_puda_buf {
struct list_head list; /* MUST be first entry */
struct irdma_dma_mem mem; /* DMA memory for the buffer */
struct irdma_puda_buf *next; /* for alloclist in rsrc struct */
struct irdma_virt_mem buf_mem; /* Buffer memory for this buffer */
void *scratch;
u8 *iph;
u8 *tcph;
u8 *data;
u16 datalen;
u16 vlan_id;
u8 tcphlen; /* tcp length in bytes */
u8 maclen; /* mac length in bytes */
u32 totallen; /* machlen+iphlen+tcphlen+datalen */
refcount_t refcount;
u8 hdrlen;
bool ipv4:1;
bool vlan_valid:1;
bool do_lpb:1; /* Loopback buffer */
bool smac_valid:1;
u32 seqnum;
u32 ah_id;
u8 smac[ETH_ALEN];
struct irdma_sc_vsi *vsi;
};
struct irdma_puda_rsrc_info {
void (*receive)(struct irdma_sc_vsi *vsi, struct irdma_puda_buf *buf);
void (*xmit_complete)(struct irdma_sc_vsi *vsi, void *sqwrid);
enum puda_rsrc_type type; /* ILQ or IEQ */
u32 count;
u32 pd_id;
u32 cq_id;
u32 qp_id;
u32 sq_size;
u32 rq_size;
u32 tx_buf_cnt; /* total bufs allocated will be rq_size + tx_buf_cnt */
u16 buf_size;
u16 stats_idx;
bool stats_idx_valid:1;
int abi_ver;
};
struct irdma_puda_rsrc {
struct irdma_sc_cq cq;
struct irdma_sc_qp qp;
struct irdma_sc_pd sc_pd;
struct irdma_sc_dev *dev;
struct irdma_sc_vsi *vsi;
struct irdma_dma_mem cqmem;
struct irdma_dma_mem qpmem;
struct irdma_virt_mem ilq_mem;
enum puda_rsrc_complete cmpl;
enum puda_rsrc_type type;
u16 buf_size; /*buf must be max datalen + tcpip hdr + mac */
u32 cq_id;
u32 qp_id;
u32 sq_size;
u32 rq_size;
u32 cq_size;
struct irdma_sq_uk_wr_trk_info *sq_wrtrk_array;
u64 *rq_wrid_array;
u32 compl_rxwqe_idx;
u32 rx_wqe_idx;
u32 rxq_invalid_cnt;
u32 tx_wqe_avail_cnt;
struct list_head txpend;
struct list_head bufpool; /* free buffers pool list for recv and xmit */
u32 alloc_buf_count;
u32 avail_buf_count; /* snapshot of currently available buffers */
spinlock_t bufpool_lock;
struct irdma_puda_buf *alloclist;
void (*receive)(struct irdma_sc_vsi *vsi, struct irdma_puda_buf *buf);
void (*xmit_complete)(struct irdma_sc_vsi *vsi, void *sqwrid);
/* puda stats */
u64 stats_buf_alloc_fail;
u64 stats_pkt_rcvd;
u64 stats_pkt_sent;
u64 stats_rcvd_pkt_err;
u64 stats_sent_pkt_q;
u64 stats_bad_qp_id;
/* IEQ stats */
u64 fpdu_processed;
u64 bad_seq_num;
u64 crc_err;
u64 pmode_count;
u64 partials_handled;
u16 stats_idx;
bool check_crc:1;
bool stats_idx_valid:1;
};
struct irdma_puda_buf *irdma_puda_get_bufpool(struct irdma_puda_rsrc *rsrc);
void irdma_puda_ret_bufpool(struct irdma_puda_rsrc *rsrc,
struct irdma_puda_buf *buf);
void irdma_puda_send_buf(struct irdma_puda_rsrc *rsrc,
struct irdma_puda_buf *buf);
int irdma_puda_send(struct irdma_sc_qp *qp, struct irdma_puda_send_info *info);
int irdma_puda_create_rsrc(struct irdma_sc_vsi *vsi,
struct irdma_puda_rsrc_info *info);
void irdma_puda_dele_rsrc(struct irdma_sc_vsi *vsi, enum puda_rsrc_type type,
bool reset);
int irdma_puda_poll_cmpl(struct irdma_sc_dev *dev, struct irdma_sc_cq *cq,
u32 *compl_err);
struct irdma_sc_qp *irdma_ieq_get_qp(struct irdma_sc_dev *dev,
struct irdma_puda_buf *buf);
int irdma_puda_get_tcpip_info(struct irdma_puda_cmpl_info *info,
struct irdma_puda_buf *buf);
int irdma_ieq_check_mpacrc(const void *addr, u32 len, u32 val);
void irdma_ieq_mpa_crc_ae(struct irdma_sc_dev *dev, struct irdma_sc_qp *qp);
void irdma_ieq_update_tcpip_info(struct irdma_puda_buf *buf, u16 len, u32 seqnum);
int irdma_cqp_qp_create_cmd(struct irdma_sc_dev *dev, struct irdma_sc_qp *qp);
int irdma_cqp_cq_create_cmd(struct irdma_sc_dev *dev, struct irdma_sc_cq *cq);
int irdma_cqp_qp_destroy_cmd(struct irdma_sc_dev *dev, struct irdma_sc_qp *qp);
void irdma_cqp_cq_destroy_cmd(struct irdma_sc_dev *dev, struct irdma_sc_cq *cq);
void irdma_puda_ieq_get_ah_info(struct irdma_sc_qp *qp,
struct irdma_ah_info *ah_info);
int irdma_puda_create_ah(struct irdma_sc_dev *dev,
struct irdma_ah_info *ah_info, bool wait,
enum puda_rsrc_type type, void *cb_param,
struct irdma_sc_ah **ah);
void irdma_puda_free_ah(struct irdma_sc_dev *dev, struct irdma_sc_ah *ah);
void irdma_ieq_process_fpdus(struct irdma_sc_qp *qp,
struct irdma_puda_rsrc *ieq);
void irdma_ieq_cleanup_qp(struct irdma_puda_rsrc *ieq, struct irdma_sc_qp *qp);
#endif /*IRDMA_PROTOS_H */
linux-7.2.3/drivers/infiniband/hw/irdma/trace.c
+112
View File
@@ -0,0 +1,112 @@
// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
/* Copyright (c) 2019 Intel Corporation */
#define CREATE_TRACE_POINTS
#include "trace.h"
const char *print_ip_addr(struct trace_seq *p, u32 *addr, u16 port, bool ipv4)
{
const char *ret = trace_seq_buffer_ptr(p);
if (ipv4) {
__be32 myaddr = htonl(*addr);
trace_seq_printf(p, "%pI4:%d", &myaddr, htons(port));
} else {
trace_seq_printf(p, "%pI6:%d", addr, htons(port));
}
trace_seq_putc(p, 0);
return ret;
}
const char *parse_iw_event_type(enum iw_cm_event_type iw_type)
{
switch (iw_type) {
case IW_CM_EVENT_CONNECT_REQUEST:
return "IwRequest";
case IW_CM_EVENT_CONNECT_REPLY:
return "IwReply";
case IW_CM_EVENT_ESTABLISHED:
return "IwEstablished";
case IW_CM_EVENT_DISCONNECT:
return "IwDisconnect";
case IW_CM_EVENT_CLOSE:
return "IwClose";
}
return "Unknown";
}
const char *parse_cm_event_type(enum irdma_cm_event_type cm_type)
{
switch (cm_type) {
case IRDMA_CM_EVENT_ESTABLISHED:
return "CmEstablished";
case IRDMA_CM_EVENT_MPA_REQ:
return "CmMPA_REQ";
case IRDMA_CM_EVENT_MPA_CONNECT:
return "CmMPA_CONNECT";
case IRDMA_CM_EVENT_MPA_ACCEPT:
return "CmMPA_ACCEPT";
case IRDMA_CM_EVENT_MPA_REJECT:
return "CmMPA_REJECT";
case IRDMA_CM_EVENT_MPA_ESTABLISHED:
return "CmMPA_ESTABLISHED";
case IRDMA_CM_EVENT_CONNECTED:
return "CmConnected";
case IRDMA_CM_EVENT_RESET:
return "CmReset";
case IRDMA_CM_EVENT_ABORTED:
return "CmAborted";
case IRDMA_CM_EVENT_UNKNOWN:
return "none";
}
return "Unknown";
}
const char *parse_cm_state(enum irdma_cm_node_state state)
{
switch (state) {
case IRDMA_CM_STATE_UNKNOWN:
return "UNKNOWN";
case IRDMA_CM_STATE_INITED:
return "INITED";
case IRDMA_CM_STATE_LISTENING:
return "LISTENING";
case IRDMA_CM_STATE_SYN_RCVD:
return "SYN_RCVD";
case IRDMA_CM_STATE_SYN_SENT:
return "SYN_SENT";
case IRDMA_CM_STATE_ONE_SIDE_ESTABLISHED:
return "ONE_SIDE_ESTABLISHED";
case IRDMA_CM_STATE_ESTABLISHED:
return "ESTABLISHED";
case IRDMA_CM_STATE_ACCEPTING:
return "ACCEPTING";
case IRDMA_CM_STATE_MPAREQ_SENT:
return "MPAREQ_SENT";
case IRDMA_CM_STATE_MPAREQ_RCVD:
return "MPAREQ_RCVD";
case IRDMA_CM_STATE_MPAREJ_RCVD:
return "MPAREJ_RECVD";
case IRDMA_CM_STATE_OFFLOADED:
return "OFFLOADED";
case IRDMA_CM_STATE_FIN_WAIT1:
return "FIN_WAIT1";
case IRDMA_CM_STATE_FIN_WAIT2:
return "FIN_WAIT2";
case IRDMA_CM_STATE_CLOSE_WAIT:
return "CLOSE_WAIT";
case IRDMA_CM_STATE_TIME_WAIT:
return "TIME_WAIT";
case IRDMA_CM_STATE_LAST_ACK:
return "LAST_ACK";
case IRDMA_CM_STATE_CLOSING:
return "CLOSING";
case IRDMA_CM_STATE_LISTENER_DESTROYED:
return "LISTENER_DESTROYED";
case IRDMA_CM_STATE_CLOSED:
return "CLOSED";
}
return ("Bad state");
}
linux-7.2.3/drivers/infiniband/hw/irdma/trace.h
+3
View File
@@ -0,0 +1,3 @@
/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
/* Copyright (c) 2019 Intel Corporation */
#include "trace_cm.h"
linux-7.2.3/drivers/infiniband/hw/irdma/trace_cm.h
+460
View File
@@ -0,0 +1,460 @@
/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
/* Copyright (c) 2019 - 2021 Intel Corporation */
#if !defined(__TRACE_CM_H) || defined(TRACE_HEADER_MULTI_READ)
#define __TRACE_CM_H
#include <linux/tracepoint.h>
#include <linux/trace_seq.h>
#include "main.h"
const char *print_ip_addr(struct trace_seq *p, u32 *addr, u16 port, bool ivp4);
const char *parse_iw_event_type(enum iw_cm_event_type iw_type);
const char *parse_cm_event_type(enum irdma_cm_event_type cm_type);
const char *parse_cm_state(enum irdma_cm_node_state);
#define __print_ip_addr(addr, port, ipv4) print_ip_addr(p, addr, port, ipv4)
#undef TRACE_SYSTEM
#define TRACE_SYSTEM irdma_cm
TRACE_EVENT(irdma_create_listen,
TP_PROTO(struct irdma_device *iwdev, struct irdma_cm_info *cm_info),
TP_ARGS(iwdev, cm_info),
TP_STRUCT__entry(__field(struct irdma_device *, iwdev)
__dynamic_array(u32, laddr, 4)
__field(u16, lport)
__field(bool, ipv4)
),
TP_fast_assign(__entry->iwdev = iwdev;
__entry->lport = cm_info->loc_port;
__entry->ipv4 = cm_info->ipv4;
memcpy(__get_dynamic_array(laddr),
cm_info->loc_addr, 4);
),
TP_printk("iwdev=%p loc: %s",
__entry->iwdev,
__print_ip_addr(__get_dynamic_array(laddr),
__entry->lport, __entry->ipv4)
)
);
TRACE_EVENT(irdma_dec_refcnt_listen,
TP_PROTO(struct irdma_cm_listener *listener, void *caller),
TP_ARGS(listener, caller),
TP_STRUCT__entry(__field(struct irdma_device *, iwdev)
__field(u32, refcnt)
__dynamic_array(u32, laddr, 4)
__field(u16, lport)
__field(bool, ipv4)
__field(void *, caller)
),
TP_fast_assign(__entry->iwdev = listener->iwdev;
__entry->lport = listener->loc_port;
__entry->ipv4 = listener->ipv4;
memcpy(__get_dynamic_array(laddr),
listener->loc_addr, 4);
),
TP_printk("iwdev=%p caller=%pS loc: %s",
__entry->iwdev,
__entry->caller,
__print_ip_addr(__get_dynamic_array(laddr),
__entry->lport, __entry->ipv4)
)
);
DECLARE_EVENT_CLASS(listener_template,
TP_PROTO(struct irdma_cm_listener *listener),
TP_ARGS(listener),
TP_STRUCT__entry(__field(struct irdma_device *, iwdev)
__field(u16, lport)
__field(u16, vlan_id)
__field(bool, ipv4)
__field(enum irdma_cm_listener_state,
state)
__dynamic_array(u32, laddr, 4)
),
TP_fast_assign(__entry->iwdev = listener->iwdev;
__entry->lport = listener->loc_port;
__entry->vlan_id = listener->vlan_id;
__entry->ipv4 = listener->ipv4;
__entry->state = listener->listener_state;
memcpy(__get_dynamic_array(laddr),
listener->loc_addr, 4);
),
TP_printk("iwdev=%p vlan=%d loc: %s",
__entry->iwdev,
__entry->vlan_id,
__print_ip_addr(__get_dynamic_array(laddr),
__entry->lport, __entry->ipv4)
)
);
DEFINE_EVENT(listener_template, irdma_find_listener,
TP_PROTO(struct irdma_cm_listener *listener),
TP_ARGS(listener));
DEFINE_EVENT(listener_template, irdma_del_multiple_qhash,
TP_PROTO(struct irdma_cm_listener *listener),
TP_ARGS(listener));
TRACE_EVENT(irdma_negotiate_mpa_v2,
TP_PROTO(struct irdma_cm_node *cm_node),
TP_ARGS(cm_node),
TP_STRUCT__entry(__field(struct irdma_cm_node *, cm_node)
__field(u16, ord_size)
__field(u16, ird_size)
),
TP_fast_assign(__entry->cm_node = cm_node;
__entry->ord_size = cm_node->ord_size;
__entry->ird_size = cm_node->ird_size;
),
TP_printk("MPVA2 Negotiated cm_node=%p ORD:[%d], IRD:[%d]",
__entry->cm_node,
__entry->ord_size,
__entry->ird_size
)
);
DECLARE_EVENT_CLASS(tos_template,
TP_PROTO(struct irdma_device *iwdev, u8 tos, u8 user_pri),
TP_ARGS(iwdev, tos, user_pri),
TP_STRUCT__entry(__field(struct irdma_device *, iwdev)
__field(u8, tos)
__field(u8, user_pri)
),
TP_fast_assign(__entry->iwdev = iwdev;
__entry->tos = tos;
__entry->user_pri = user_pri;
),
TP_printk("iwdev=%p TOS:[%d] UP:[%d]",
__entry->iwdev,
__entry->tos,
__entry->user_pri
)
);
DEFINE_EVENT(tos_template, irdma_listener_tos,
TP_PROTO(struct irdma_device *iwdev, u8 tos, u8 user_pri),
TP_ARGS(iwdev, tos, user_pri));
DEFINE_EVENT(tos_template, irdma_dcb_tos,
TP_PROTO(struct irdma_device *iwdev, u8 tos, u8 user_pri),
TP_ARGS(iwdev, tos, user_pri));
DECLARE_EVENT_CLASS(qhash_template,
TP_PROTO(struct irdma_device *iwdev,
struct irdma_cm_listener *listener,
const char *dev_addr),
TP_ARGS(iwdev, listener, dev_addr),
TP_STRUCT__entry(__field(struct irdma_device *, iwdev)
__field(u16, lport)
__field(u16, vlan_id)
__field(bool, ipv4)
__dynamic_array(u32, laddr, 4)
__dynamic_array(u32, mac, ETH_ALEN)
),
TP_fast_assign(__entry->iwdev = iwdev;
__entry->lport = listener->loc_port;
__entry->vlan_id = listener->vlan_id;
__entry->ipv4 = listener->ipv4;
memcpy(__get_dynamic_array(laddr),
listener->loc_addr, 4);
ether_addr_copy(__get_dynamic_array(mac),
dev_addr);
),
TP_printk("iwdev=%p vlan=%d MAC=%6phC loc: %s",
__entry->iwdev,
__entry->vlan_id,
__get_dynamic_array(mac),
__print_ip_addr(__get_dynamic_array(laddr),
__entry->lport, __entry->ipv4)
)
);
DEFINE_EVENT(qhash_template, irdma_add_mqh_6,
TP_PROTO(struct irdma_device *iwdev,
struct irdma_cm_listener *listener,
const char *dev_addr),
TP_ARGS(iwdev, listener, dev_addr));
DEFINE_EVENT(qhash_template, irdma_add_mqh_4,
TP_PROTO(struct irdma_device *iwdev,
struct irdma_cm_listener *listener,
const char *dev_addr),
TP_ARGS(iwdev, listener, dev_addr));
TRACE_EVENT(irdma_addr_resolve,
TP_PROTO(struct irdma_device *iwdev, char *dev_addr),
TP_ARGS(iwdev, dev_addr),
TP_STRUCT__entry(__field(struct irdma_device *, iwdev)
__dynamic_array(u8, mac, ETH_ALEN)
),
TP_fast_assign(__entry->iwdev = iwdev;
ether_addr_copy(__get_dynamic_array(mac), dev_addr);
),
TP_printk("iwdev=%p MAC=%6phC", __entry->iwdev,
__get_dynamic_array(mac)
)
);
TRACE_EVENT(irdma_send_cm_event,
TP_PROTO(struct irdma_cm_node *cm_node, struct iw_cm_id *cm_id,
enum iw_cm_event_type type, int status, void *caller),
TP_ARGS(cm_node, cm_id, type, status, caller),
TP_STRUCT__entry(__field(struct irdma_device *, iwdev)
__field(struct irdma_cm_node *, cm_node)
__field(struct iw_cm_id *, cm_id)
__field(u32, refcount)
__field(u16, lport)
__field(u16, rport)
__field(enum irdma_cm_node_state, state)
__field(bool, ipv4)
__field(u16, vlan_id)
__field(int, accel)
__field(enum iw_cm_event_type, type)
__field(int, status)
__field(void *, caller)
__dynamic_array(u32, laddr, 4)
__dynamic_array(u32, raddr, 4)
),
TP_fast_assign(__entry->iwdev = cm_node->iwdev;
__entry->cm_node = cm_node;
__entry->cm_id = cm_id;
__entry->refcount = refcount_read(&cm_node->refcnt);
__entry->state = cm_node->state;
__entry->lport = cm_node->loc_port;
__entry->rport = cm_node->rem_port;
__entry->ipv4 = cm_node->ipv4;
__entry->vlan_id = cm_node->vlan_id;
__entry->accel = cm_node->accelerated;
__entry->type = type;
__entry->status = status;
__entry->caller = caller;
memcpy(__get_dynamic_array(laddr),
cm_node->loc_addr, 4);
memcpy(__get_dynamic_array(raddr),
cm_node->rem_addr, 4);
),
TP_printk("iwdev=%p caller=%pS cm_id=%p node=%p refcnt=%d vlan_id=%d accel=%d state=%s event_type=%s status=%d loc: %s rem: %s",
__entry->iwdev,
__entry->caller,
__entry->cm_id,
__entry->cm_node,
__entry->refcount,
__entry->vlan_id,
__entry->accel,
parse_cm_state(__entry->state),
parse_iw_event_type(__entry->type),
__entry->status,
__print_ip_addr(__get_dynamic_array(laddr),
__entry->lport, __entry->ipv4),
__print_ip_addr(__get_dynamic_array(raddr),
__entry->rport, __entry->ipv4)
)
);
TRACE_EVENT(irdma_send_cm_event_no_node,
TP_PROTO(struct iw_cm_id *cm_id, enum iw_cm_event_type type,
int status, void *caller),
TP_ARGS(cm_id, type, status, caller),
TP_STRUCT__entry(__field(struct iw_cm_id *, cm_id)
__field(enum iw_cm_event_type, type)
__field(int, status)
__field(void *, caller)
),
TP_fast_assign(__entry->cm_id = cm_id;
__entry->type = type;
__entry->status = status;
__entry->caller = caller;
),
TP_printk("cm_id=%p caller=%pS event_type=%s status=%d",
__entry->cm_id,
__entry->caller,
parse_iw_event_type(__entry->type),
__entry->status
)
);
DECLARE_EVENT_CLASS(cm_node_template,
TP_PROTO(struct irdma_cm_node *cm_node,
enum irdma_cm_event_type type, void *caller),
TP_ARGS(cm_node, type, caller),
TP_STRUCT__entry(__field(struct irdma_device *, iwdev)
__field(struct irdma_cm_node *, cm_node)
__field(u32, refcount)
__field(u16, lport)
__field(u16, rport)
__field(enum irdma_cm_node_state, state)
__field(bool, ipv4)
__field(u16, vlan_id)
__field(int, accel)
__field(enum irdma_cm_event_type, type)
__field(void *, caller)
__dynamic_array(u32, laddr, 4)
__dynamic_array(u32, raddr, 4)
),
TP_fast_assign(__entry->iwdev = cm_node->iwdev;
__entry->cm_node = cm_node;
__entry->refcount = refcount_read(&cm_node->refcnt);
__entry->state = cm_node->state;
__entry->lport = cm_node->loc_port;
__entry->rport = cm_node->rem_port;
__entry->ipv4 = cm_node->ipv4;
__entry->vlan_id = cm_node->vlan_id;
__entry->accel = cm_node->accelerated;
__entry->type = type;
__entry->caller = caller;
memcpy(__get_dynamic_array(laddr),
cm_node->loc_addr, 4);
memcpy(__get_dynamic_array(raddr),
cm_node->rem_addr, 4);
),
TP_printk("iwdev=%p caller=%pS node=%p refcnt=%d vlan_id=%d accel=%d state=%s event_type=%s loc: %s rem: %s",
__entry->iwdev,
__entry->caller,
__entry->cm_node,
__entry->refcount,
__entry->vlan_id,
__entry->accel,
parse_cm_state(__entry->state),
parse_cm_event_type(__entry->type),
__print_ip_addr(__get_dynamic_array(laddr),
__entry->lport, __entry->ipv4),
__print_ip_addr(__get_dynamic_array(raddr),
__entry->rport, __entry->ipv4)
)
);
DEFINE_EVENT(cm_node_template, irdma_create_event,
TP_PROTO(struct irdma_cm_node *cm_node,
enum irdma_cm_event_type type, void *caller),
TP_ARGS(cm_node, type, caller));
DEFINE_EVENT(cm_node_template, irdma_accept,
TP_PROTO(struct irdma_cm_node *cm_node,
enum irdma_cm_event_type type, void *caller),
TP_ARGS(cm_node, type, caller));
DEFINE_EVENT(cm_node_template, irdma_connect,
TP_PROTO(struct irdma_cm_node *cm_node,
enum irdma_cm_event_type type, void *caller),
TP_ARGS(cm_node, type, caller));
DEFINE_EVENT(cm_node_template, irdma_reject,
TP_PROTO(struct irdma_cm_node *cm_node,
enum irdma_cm_event_type type, void *caller),
TP_ARGS(cm_node, type, caller));
DEFINE_EVENT(cm_node_template, irdma_find_node,
TP_PROTO(struct irdma_cm_node *cm_node,
enum irdma_cm_event_type type, void *caller),
TP_ARGS(cm_node, type, caller));
DEFINE_EVENT(cm_node_template, irdma_send_reset,
TP_PROTO(struct irdma_cm_node *cm_node,
enum irdma_cm_event_type type, void *caller),
TP_ARGS(cm_node, type, caller));
DEFINE_EVENT(cm_node_template, irdma_rem_ref_cm_node,
TP_PROTO(struct irdma_cm_node *cm_node,
enum irdma_cm_event_type type, void *caller),
TP_ARGS(cm_node, type, caller));
DEFINE_EVENT(cm_node_template, irdma_cm_event_handler,
TP_PROTO(struct irdma_cm_node *cm_node,
enum irdma_cm_event_type type, void *caller),
TP_ARGS(cm_node, type, caller));
TRACE_EVENT(open_err_template,
TP_PROTO(struct irdma_cm_node *cm_node, bool reset, void *caller),
TP_ARGS(cm_node, reset, caller),
TP_STRUCT__entry(__field(struct irdma_device *, iwdev)
__field(struct irdma_cm_node *, cm_node)
__field(enum irdma_cm_node_state, state)
__field(bool, reset)
__field(void *, caller)
),
TP_fast_assign(__entry->iwdev = cm_node->iwdev;
__entry->cm_node = cm_node;
__entry->state = cm_node->state;
__entry->reset = reset;
__entry->caller = caller;
),
TP_printk("iwdev=%p caller=%pS node%p reset=%d state=%s",
__entry->iwdev,
__entry->caller,
__entry->cm_node,
__entry->reset,
parse_cm_state(__entry->state)
)
);
DEFINE_EVENT(open_err_template, irdma_active_open_err,
TP_PROTO(struct irdma_cm_node *cm_node, bool reset, void *caller),
TP_ARGS(cm_node, reset, caller));
DEFINE_EVENT(open_err_template, irdma_passive_open_err,
TP_PROTO(struct irdma_cm_node *cm_node, bool reset, void *caller),
TP_ARGS(cm_node, reset, caller));
DECLARE_EVENT_CLASS(cm_node_ah_template,
TP_PROTO(struct irdma_cm_node *cm_node),
TP_ARGS(cm_node),
TP_STRUCT__entry(__field(struct irdma_device *, iwdev)
__field(struct irdma_cm_node *, cm_node)
__field(struct irdma_sc_ah *, ah)
__field(u32, refcount)
__field(u16, lport)
__field(u16, rport)
__field(enum irdma_cm_node_state, state)
__field(bool, ipv4)
__field(u16, vlan_id)
__field(int, accel)
__dynamic_array(u32, laddr, 4)
__dynamic_array(u32, raddr, 4)
),
TP_fast_assign(__entry->iwdev = cm_node->iwdev;
__entry->cm_node = cm_node;
__entry->ah = cm_node->ah;
__entry->refcount = refcount_read(&cm_node->refcnt);
__entry->lport = cm_node->loc_port;
__entry->rport = cm_node->rem_port;
__entry->state = cm_node->state;
__entry->ipv4 = cm_node->ipv4;
__entry->vlan_id = cm_node->vlan_id;
__entry->accel = cm_node->accelerated;
memcpy(__get_dynamic_array(laddr),
cm_node->loc_addr, 4);
memcpy(__get_dynamic_array(raddr),
cm_node->rem_addr, 4);
),
TP_printk("iwdev=%p node=%p ah=%p refcnt=%d vlan_id=%d accel=%d state=%s loc: %s rem: %s",
__entry->iwdev,
__entry->cm_node,
__entry->ah,
__entry->refcount,
__entry->vlan_id,
__entry->accel,
parse_cm_state(__entry->state),
__print_ip_addr(__get_dynamic_array(laddr),
__entry->lport, __entry->ipv4),
__print_ip_addr(__get_dynamic_array(raddr),
__entry->rport, __entry->ipv4)
)
);
DEFINE_EVENT(cm_node_ah_template, irdma_cm_free_ah,
TP_PROTO(struct irdma_cm_node *cm_node),
TP_ARGS(cm_node));
DEFINE_EVENT(cm_node_ah_template, irdma_create_ah,
TP_PROTO(struct irdma_cm_node *cm_node),
TP_ARGS(cm_node));
#endif /* __TRACE_CM_H */
#undef TRACE_INCLUDE_PATH
#undef TRACE_INCLUDE_FILE
#define TRACE_INCLUDE_PATH .
#define TRACE_INCLUDE_FILE trace_cm
#include <trace/define_trace.h>
linux-7.2.3/drivers/infiniband/hw/irdma/type.h
+1692
View File
File diff suppressed because it is too large Load Diff
linux-7.2.3/drivers/infiniband/hw/irdma/uda.c
+265
View File
@@ -0,0 +1,265 @@
// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
/* Copyright (c) 2016 - 2021 Intel Corporation */
#include <linux/etherdevice.h>
#include "osdep.h"
#include "hmc.h"
#include "defs.h"
#include "type.h"
#include "protos.h"
#include "uda.h"
#include "uda_d.h"
/**
* irdma_sc_access_ah() - Create, modify or delete AH
* @cqp: struct for cqp hw
* @info: ah information
* @op: Operation
* @scratch: u64 saved to be used during cqp completion
*/
int irdma_sc_access_ah(struct irdma_sc_cqp *cqp, struct irdma_ah_info *info,
u32 op, u64 scratch)
{
__le64 *wqe;
u64 qw1, qw2;
wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
if (!wqe)
return -ENOMEM;
set_64bit_val(wqe, 0, ether_addr_to_u64(info->mac_addr) << 16);
qw1 = FIELD_PREP(IRDMA_UDA_CQPSQ_MAV_PDINDEXLO, info->pd_idx) |
FIELD_PREP(IRDMA_UDA_CQPSQ_MAV_TC, info->tc_tos) |
FIELD_PREP(IRDMA_UDAQPC_VLANTAG, info->vlan_tag);
qw2 = FIELD_PREP(IRDMA_UDA_CQPSQ_MAV_ARPINDEX, info->dst_arpindex) |
FIELD_PREP(IRDMA_UDA_CQPSQ_MAV_FLOWLABEL, info->flow_label) |
FIELD_PREP(IRDMA_UDA_CQPSQ_MAV_HOPLIMIT, info->hop_ttl) |
FIELD_PREP(IRDMA_UDA_CQPSQ_MAV_PDINDEXHI, info->pd_idx >> 16);
if (!info->ipv4_valid) {
set_64bit_val(wqe, 40,
FIELD_PREP(IRDMA_UDA_CQPSQ_MAV_ADDR0, info->dest_ip_addr[0]) |
FIELD_PREP(IRDMA_UDA_CQPSQ_MAV_ADDR1, info->dest_ip_addr[1]));
set_64bit_val(wqe, 32,
FIELD_PREP(IRDMA_UDA_CQPSQ_MAV_ADDR2, info->dest_ip_addr[2]) |
FIELD_PREP(IRDMA_UDA_CQPSQ_MAV_ADDR3, info->dest_ip_addr[3]));
set_64bit_val(wqe, 56,
FIELD_PREP(IRDMA_UDA_CQPSQ_MAV_ADDR0, info->src_ip_addr[0]) |
FIELD_PREP(IRDMA_UDA_CQPSQ_MAV_ADDR1, info->src_ip_addr[1]));
set_64bit_val(wqe, 48,
FIELD_PREP(IRDMA_UDA_CQPSQ_MAV_ADDR2, info->src_ip_addr[2]) |
FIELD_PREP(IRDMA_UDA_CQPSQ_MAV_ADDR3, info->src_ip_addr[3]));
} else {
set_64bit_val(wqe, 32,
FIELD_PREP(IRDMA_UDA_CQPSQ_MAV_ADDR3, info->dest_ip_addr[0]));
set_64bit_val(wqe, 48,
FIELD_PREP(IRDMA_UDA_CQPSQ_MAV_ADDR3, info->src_ip_addr[0]));
}
set_64bit_val(wqe, 8, qw1);
set_64bit_val(wqe, 16, qw2);
dma_wmb(); /* need write block before writing WQE header */
set_64bit_val(
wqe, 24,
FIELD_PREP(IRDMA_UDA_CQPSQ_MAV_WQEVALID, cqp->polarity) |
FIELD_PREP(IRDMA_UDA_CQPSQ_MAV_OPCODE, op) |
FIELD_PREP(IRDMA_UDA_CQPSQ_MAV_DOLOOPBACKK, info->do_lpbk) |
FIELD_PREP(IRDMA_UDA_CQPSQ_MAV_IPV4VALID, info->ipv4_valid) |
FIELD_PREP(IRDMA_UDA_CQPSQ_MAV_AVIDX, info->ah_idx) |
FIELD_PREP(IRDMA_UDA_CQPSQ_MAV_INSERTVLANTAG, info->insert_vlan_tag));
print_hex_dump_debug("WQE: MANAGE_AH WQE", DUMP_PREFIX_OFFSET, 16, 8,
wqe, IRDMA_CQP_WQE_SIZE * 8, false);
irdma_sc_cqp_post_sq(cqp);
return 0;
}
/**
* irdma_create_mg_ctx() - create a mcg context
* @info: multicast group context info
*/
static void irdma_create_mg_ctx(struct irdma_mcast_grp_info *info)
{
struct irdma_mcast_grp_ctx_entry_info *entry_info = NULL;
u8 idx = 0; /* index in the array */
u8 ctx_idx = 0; /* index in the MG context */
memset(info->dma_mem_mc.va, 0, IRDMA_MAX_MGS_PER_CTX * sizeof(u64));
for (idx = 0; idx < IRDMA_MAX_MGS_PER_CTX; idx++) {
entry_info = &info->mg_ctx_info[idx];
if (entry_info->valid_entry) {
set_64bit_val((__le64 *)info->dma_mem_mc.va,
ctx_idx * sizeof(u64),
FIELD_PREP(IRDMA_UDA_MGCTX_DESTPORT, entry_info->dest_port) |
FIELD_PREP(IRDMA_UDA_MGCTX_VALIDENT, entry_info->valid_entry) |
FIELD_PREP(IRDMA_UDA_MGCTX_QPID, entry_info->qp_id));
ctx_idx++;
}
}
}
/**
* irdma_access_mcast_grp() - Access mcast group based on op
* @cqp: Control QP
* @info: multicast group context info
* @op: operation to perform
* @scratch: u64 saved to be used during cqp completion
*/
int irdma_access_mcast_grp(struct irdma_sc_cqp *cqp,
struct irdma_mcast_grp_info *info, u32 op,
u64 scratch)
{
__le64 *wqe;
if (info->mg_id >= IRDMA_UDA_MAX_FSI_MGS) {
ibdev_dbg(to_ibdev(cqp->dev), "WQE: mg_id out of range\n");
return -EINVAL;
}
wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
if (!wqe) {
ibdev_dbg(to_ibdev(cqp->dev), "WQE: ring full\n");
return -ENOMEM;
}
irdma_create_mg_ctx(info);
set_64bit_val(wqe, 32, info->dma_mem_mc.pa);
set_64bit_val(wqe, 16,
FIELD_PREP(IRDMA_UDA_CQPSQ_MG_VLANID, info->vlan_id) |
FIELD_PREP(IRDMA_UDA_CQPSQ_QS_HANDLE, info->qs_handle));
set_64bit_val(wqe, 0, ether_addr_to_u64(info->dest_mac_addr));
set_64bit_val(wqe, 8,
FIELD_PREP(IRDMA_UDA_CQPSQ_MG_HMC_FCN_ID, info->hmc_fcn_id));
if (!info->ipv4_valid) {
set_64bit_val(wqe, 56,
FIELD_PREP(IRDMA_UDA_CQPSQ_MAV_ADDR0, info->dest_ip_addr[0]) |
FIELD_PREP(IRDMA_UDA_CQPSQ_MAV_ADDR1, info->dest_ip_addr[1]));
set_64bit_val(wqe, 48,
FIELD_PREP(IRDMA_UDA_CQPSQ_MAV_ADDR2, info->dest_ip_addr[2]) |
FIELD_PREP(IRDMA_UDA_CQPSQ_MAV_ADDR3, info->dest_ip_addr[3]));
} else {
set_64bit_val(wqe, 48,
FIELD_PREP(IRDMA_UDA_CQPSQ_MAV_ADDR3, info->dest_ip_addr[0]));
}
dma_wmb(); /* need write memory block before writing the WQE header. */
set_64bit_val(wqe, 24,
FIELD_PREP(IRDMA_UDA_CQPSQ_MG_WQEVALID, cqp->polarity) |
FIELD_PREP(IRDMA_UDA_CQPSQ_MG_OPCODE, op) |
FIELD_PREP(IRDMA_UDA_CQPSQ_MG_MGIDX, info->mg_id) |
FIELD_PREP(IRDMA_UDA_CQPSQ_MG_VLANVALID, info->vlan_valid) |
FIELD_PREP(IRDMA_UDA_CQPSQ_MG_IPV4VALID, info->ipv4_valid));
print_hex_dump_debug("WQE: MANAGE_MCG WQE", DUMP_PREFIX_OFFSET, 16, 8,
wqe, IRDMA_CQP_WQE_SIZE * 8, false);
print_hex_dump_debug("WQE: MCG_HOST CTX WQE", DUMP_PREFIX_OFFSET, 16,
8, info->dma_mem_mc.va,
IRDMA_MAX_MGS_PER_CTX * 8, false);
irdma_sc_cqp_post_sq(cqp);
return 0;
}
/**
* irdma_compare_mgs - Compares two multicast group structures
* @entry1: Multcast group info
* @entry2: Multcast group info in context
*/
static bool irdma_compare_mgs(struct irdma_mcast_grp_ctx_entry_info *entry1,
struct irdma_mcast_grp_ctx_entry_info *entry2)
{
if (entry1->dest_port == entry2->dest_port &&
entry1->qp_id == entry2->qp_id)
return true;
return false;
}
/**
* irdma_sc_add_mcast_grp - Allocates mcast group entry in ctx
* @ctx: Multcast group context
* @mg: Multcast group info
*/
int irdma_sc_add_mcast_grp(struct irdma_mcast_grp_info *ctx,
struct irdma_mcast_grp_ctx_entry_info *mg)
{
u32 idx;
bool free_entry_found = false;
u32 free_entry_idx = 0;
/* find either an identical or a free entry for a multicast group */
for (idx = 0; idx < IRDMA_MAX_MGS_PER_CTX; idx++) {
if (ctx->mg_ctx_info[idx].valid_entry) {
if (irdma_compare_mgs(&ctx->mg_ctx_info[idx], mg)) {
ctx->mg_ctx_info[idx].use_cnt++;
return 0;
}
continue;
}
if (!free_entry_found) {
free_entry_found = true;
free_entry_idx = idx;
}
}
if (free_entry_found) {
ctx->mg_ctx_info[free_entry_idx] = *mg;
ctx->mg_ctx_info[free_entry_idx].valid_entry = true;
ctx->mg_ctx_info[free_entry_idx].use_cnt = 1;
ctx->no_of_mgs++;
return 0;
}
return -ENOMEM;
}
/**
* irdma_sc_del_mcast_grp - Delete mcast group
* @ctx: Multcast group context
* @mg: Multcast group info
*
* Finds and removes a specific mulicast group from context, all
* parameters must match to remove a multicast group.
*/
int irdma_sc_del_mcast_grp(struct irdma_mcast_grp_info *ctx,
struct irdma_mcast_grp_ctx_entry_info *mg)
{
u32 idx;
/* find an entry in multicast group context */
for (idx = 0; idx < IRDMA_MAX_MGS_PER_CTX; idx++) {
if (!ctx->mg_ctx_info[idx].valid_entry)
continue;
if (irdma_compare_mgs(mg, &ctx->mg_ctx_info[idx])) {
ctx->mg_ctx_info[idx].use_cnt--;
if (!ctx->mg_ctx_info[idx].use_cnt) {
ctx->mg_ctx_info[idx].valid_entry = false;
ctx->no_of_mgs--;
/* Remove gap if element was not the last */
if (idx != ctx->no_of_mgs &&
ctx->no_of_mgs > 0) {
memcpy(&ctx->mg_ctx_info[idx],
&ctx->mg_ctx_info[ctx->no_of_mgs - 1],
sizeof(ctx->mg_ctx_info[idx]));
ctx->mg_ctx_info[ctx->no_of_mgs - 1].valid_entry = false;
}
}
return 0;
}
}
return -EINVAL;
}
linux-7.2.3/drivers/infiniband/hw/irdma/uda.h
+87
View File
@@ -0,0 +1,87 @@
/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
/* Copyright (c) 2016 - 2021 Intel Corporation */
#ifndef IRDMA_UDA_H
#define IRDMA_UDA_H
#define IRDMA_UDA_MAX_FSI_MGS 4096
#define IRDMA_UDA_MAX_PFS 16
#define IRDMA_UDA_MAX_VFS 128
struct irdma_sc_cqp;
struct irdma_ah_info {
struct irdma_sc_vsi *vsi;
u32 pd_idx;
u32 dst_arpindex;
u32 dest_ip_addr[4];
u32 src_ip_addr[4];
u32 flow_label;
u32 ah_idx;
u16 vlan_tag;
u8 insert_vlan_tag;
u8 tc_tos;
u8 hop_ttl;
u8 mac_addr[ETH_ALEN];
bool ah_valid:1;
bool ipv4_valid:1;
bool do_lpbk:1;
};
struct irdma_sc_ah {
struct irdma_sc_dev *dev;
struct irdma_ah_info ah_info;
};
int irdma_sc_add_mcast_grp(struct irdma_mcast_grp_info *ctx,
struct irdma_mcast_grp_ctx_entry_info *mg);
int irdma_sc_del_mcast_grp(struct irdma_mcast_grp_info *ctx,
struct irdma_mcast_grp_ctx_entry_info *mg);
int irdma_sc_access_ah(struct irdma_sc_cqp *cqp, struct irdma_ah_info *info,
u32 op, u64 scratch);
int irdma_access_mcast_grp(struct irdma_sc_cqp *cqp,
struct irdma_mcast_grp_info *info, u32 op,
u64 scratch);
static inline void irdma_sc_init_ah(struct irdma_sc_dev *dev, struct irdma_sc_ah *ah)
{
ah->dev = dev;
}
static inline int irdma_sc_create_ah(struct irdma_sc_cqp *cqp,
struct irdma_ah_info *info, u64 scratch)
{
return irdma_sc_access_ah(cqp, info, IRDMA_CQP_OP_CREATE_ADDR_HANDLE,
scratch);
}
static inline int irdma_sc_destroy_ah(struct irdma_sc_cqp *cqp,
struct irdma_ah_info *info, u64 scratch)
{
return irdma_sc_access_ah(cqp, info, IRDMA_CQP_OP_DESTROY_ADDR_HANDLE,
scratch);
}
static inline int irdma_sc_create_mcast_grp(struct irdma_sc_cqp *cqp,
struct irdma_mcast_grp_info *info,
u64 scratch)
{
return irdma_access_mcast_grp(cqp, info, IRDMA_CQP_OP_CREATE_MCAST_GRP,
scratch);
}
static inline int irdma_sc_modify_mcast_grp(struct irdma_sc_cqp *cqp,
struct irdma_mcast_grp_info *info,
u64 scratch)
{
return irdma_access_mcast_grp(cqp, info, IRDMA_CQP_OP_MODIFY_MCAST_GRP,
scratch);
}
static inline int irdma_sc_destroy_mcast_grp(struct irdma_sc_cqp *cqp,
struct irdma_mcast_grp_info *info,
u64 scratch)
{
return irdma_access_mcast_grp(cqp, info, IRDMA_CQP_OP_DESTROY_MCAST_GRP,
scratch);
}
#endif /* IRDMA_UDA_H */
linux-7.2.3/drivers/infiniband/hw/irdma/uda_d.h
+127
View File
@@ -0,0 +1,127 @@
/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
/* Copyright (c) 2016 - 2021 Intel Corporation */
#ifndef IRDMA_UDA_D_H
#define IRDMA_UDA_D_H
/* L4 packet type */
#define IRDMA_E_UDA_SQ_L4T_UNKNOWN 0
#define IRDMA_E_UDA_SQ_L4T_TCP 1
#define IRDMA_E_UDA_SQ_L4T_SCTP 2
#define IRDMA_E_UDA_SQ_L4T_UDP 3
/* Inner IP header type */
#define IRDMA_E_UDA_SQ_IIPT_UNKNOWN 0
#define IRDMA_E_UDA_SQ_IIPT_IPV6 1
#define IRDMA_E_UDA_SQ_IIPT_IPV4_NO_CSUM 2
#define IRDMA_E_UDA_SQ_IIPT_IPV4_CSUM 3
#define IRDMA_UDA_QPSQ_PUSHWQE BIT_ULL(56)
#define IRDMA_UDA_QPSQ_INLINEDATAFLAG BIT_ULL(57)
#define IRDMA_UDA_QPSQ_INLINEDATALEN GENMASK_ULL(55, 48)
#define IRDMA_UDA_QPSQ_ADDFRAGCNT GENMASK_ULL(41, 38)
#define IRDMA_UDA_QPSQ_IPFRAGFLAGS GENMASK_ULL(43, 42)
#define IRDMA_UDA_QPSQ_NOCHECKSUM BIT_ULL(45)
#define IRDMA_UDA_QPSQ_AHIDXVALID BIT_ULL(46)
#define IRDMA_UDA_QPSQ_LOCAL_FENCE BIT_ULL(61)
#define IRDMA_UDA_QPSQ_AHIDX GENMASK_ULL(16, 0)
#define IRDMA_UDA_QPSQ_PROTOCOL GENMASK_ULL(23, 16)
#define IRDMA_UDA_QPSQ_EXTHDRLEN GENMASK_ULL(40, 32)
#define IRDMA_UDA_QPSQ_MULTICAST BIT_ULL(63)
#define IRDMA_UDA_QPSQ_MACLEN GENMASK_ULL(62, 56)
#define IRDMA_UDA_QPSQ_MACLEN_LINE 2
#define IRDMA_UDA_QPSQ_IPLEN GENMASK_ULL(54, 48)
#define IRDMA_UDA_QPSQ_IPLEN_LINE 2
#define IRDMA_UDA_QPSQ_L4T GENMASK_ULL(31, 30)
#define IRDMA_UDA_QPSQ_L4T_LINE 2
#define IRDMA_UDA_QPSQ_IIPT GENMASK_ULL(29, 28)
#define IRDMA_UDA_QPSQ_IIPT_LINE 2
#define IRDMA_UDA_QPSQ_DO_LPB_LINE 3
#define IRDMA_UDA_QPSQ_FWD_PROG_CONFIRM BIT_ULL(45)
#define IRDMA_UDA_QPSQ_FWD_PROG_CONFIRM_LINE 3
#define IRDMA_UDA_QPSQ_IMMDATA GENMASK_ULL(63, 0)
/* Byte Offset 0 */
#define IRDMA_UDAQPC_IPV4_M BIT_ULL(3)
#define IRDMA_UDAQPC_INSERTVLANTAG BIT_ULL(5)
#define IRDMA_UDAQPC_ISQP1 BIT_ULL(6)
#define IRDMA_UDAQPC_ECNENABLE BIT_ULL(14)
#define IRDMA_UDAQPC_PDINDEXHI GENMASK_ULL(21, 20)
#define IRDMA_UDAQPC_DCTCPENABLE BIT_ULL(25)
#define IRDMA_UDAQPC_RCVTPHEN IRDMAQPC_RCVTPHEN
#define IRDMA_UDAQPC_XMITTPHEN IRDMAQPC_XMITTPHEN
#define IRDMA_UDAQPC_RQTPHEN IRDMAQPC_RQTPHEN
#define IRDMA_UDAQPC_SQTPHEN IRDMAQPC_SQTPHEN
#define IRDMA_UDAQPC_PPIDX IRDMAQPC_PPIDX
#define IRDMA_UDAQPC_PMENA IRDMAQPC_PMENA
#define IRDMA_UDAQPC_INSERTTAG2 BIT_ULL(11)
#define IRDMA_UDAQPC_INSERTTAG3 BIT_ULL(14)
#define IRDMA_UDAQPC_RQSIZE IRDMAQPC_RQSIZE
#define IRDMA_UDAQPC_SQSIZE IRDMAQPC_SQSIZE
#define IRDMA_UDAQPC_TXCQNUM IRDMAQPC_TXCQNUM
#define IRDMA_UDAQPC_RXCQNUM IRDMAQPC_RXCQNUM
#define IRDMA_UDAQPC_QPCOMPCTX IRDMAQPC_QPCOMPCTX
#define IRDMA_UDAQPC_SQTPHVAL IRDMAQPC_SQTPHVAL
#define IRDMA_UDAQPC_RQTPHVAL IRDMAQPC_RQTPHVAL
#define IRDMA_UDAQPC_QSHANDLE IRDMAQPC_QSHANDLE
#define IRDMA_UDAQPC_RQHDRRINGBUFSIZE GENMASK_ULL(49, 48)
#define IRDMA_UDAQPC_SQHDRRINGBUFSIZE GENMASK_ULL(33, 32)
#define IRDMA_UDAQPC_PRIVILEGEENABLE BIT_ULL(25)
#define IRDMA_UDAQPC_USE_STATISTICS_INSTANCE BIT_ULL(26)
#define IRDMA_UDAQPC_STATISTICS_INSTANCE_INDEX GENMASK_ULL(6, 0)
#define IRDMA_UDAQPC_PRIVHDRGENENABLE BIT_ULL(0)
#define IRDMA_UDAQPC_RQHDRSPLITENABLE BIT_ULL(3)
#define IRDMA_UDAQPC_RQHDRRINGBUFENABLE BIT_ULL(2)
#define IRDMA_UDAQPC_SQHDRRINGBUFENABLE BIT_ULL(1)
#define IRDMA_UDAQPC_IPID GENMASK_ULL(47, 32)
#define IRDMA_UDAQPC_SNDMSS GENMASK_ULL(29, 16)
#define IRDMA_UDAQPC_VLANTAG GENMASK_ULL(15, 0)
#define IRDMA_UDA_CQPSQ_MAV_PDINDEXHI GENMASK_ULL(27, 20)
#define IRDMA_UDA_CQPSQ_MAV_PDINDEXLO GENMASK_ULL(63, 48)
#define IRDMA_UDA_CQPSQ_MAV_SRCMACADDRINDEX GENMASK_ULL(29, 24)
#define IRDMA_UDA_CQPSQ_MAV_ARPINDEX GENMASK_ULL(63, 48)
#define IRDMA_UDA_CQPSQ_MAV_TC GENMASK_ULL(39, 32)
#define IRDMA_UDA_CQPSQ_MAV_HOPLIMIT GENMASK_ULL(39, 32)
#define IRDMA_UDA_CQPSQ_MAV_FLOWLABEL GENMASK_ULL(19, 0)
#define IRDMA_UDA_CQPSQ_MAV_ADDR0 GENMASK_ULL(63, 32)
#define IRDMA_UDA_CQPSQ_MAV_ADDR1 GENMASK_ULL(31, 0)
#define IRDMA_UDA_CQPSQ_MAV_ADDR2 GENMASK_ULL(63, 32)
#define IRDMA_UDA_CQPSQ_MAV_ADDR3 GENMASK_ULL(31, 0)
#define IRDMA_UDA_CQPSQ_MAV_WQEVALID BIT_ULL(63)
#define IRDMA_UDA_CQPSQ_MAV_OPCODE GENMASK_ULL(37, 32)
#define IRDMA_UDA_CQPSQ_MAV_DOLOOPBACKK BIT_ULL(62)
#define IRDMA_UDA_CQPSQ_MAV_IPV4VALID BIT_ULL(59)
#define IRDMA_UDA_CQPSQ_MAV_AVIDX GENMASK_ULL(23, 0)
#define IRDMA_UDA_CQPSQ_MAV_INSERTVLANTAG BIT_ULL(60)
#define IRDMA_UDA_MGCTX_VFFLAG BIT_ULL(29)
#define IRDMA_UDA_MGCTX_DESTPORT GENMASK_ULL(47, 32)
#define IRDMA_UDA_MGCTX_VFID GENMASK_ULL(28, 22)
#define IRDMA_UDA_MGCTX_VALIDENT BIT_ULL(31)
#define IRDMA_UDA_MGCTX_PFID GENMASK_ULL(21, 18)
#define IRDMA_UDA_MGCTX_FLAGIGNOREDPORT BIT_ULL(30)
#define IRDMA_UDA_MGCTX_QPID GENMASK_ULL(17, 0)
#define IRDMA_UDA_CQPSQ_MG_WQEVALID BIT_ULL(63)
#define IRDMA_UDA_CQPSQ_MG_OPCODE GENMASK_ULL(37, 32)
#define IRDMA_UDA_CQPSQ_MG_MGIDX GENMASK_ULL(12, 0)
#define IRDMA_UDA_CQPSQ_MG_IPV4VALID BIT_ULL(60)
#define IRDMA_UDA_CQPSQ_MG_VLANVALID BIT_ULL(59)
#define IRDMA_UDA_CQPSQ_MG_HMC_FCN_ID GENMASK_ULL(5, 0)
#define IRDMA_UDA_CQPSQ_MG_VLANID GENMASK_ULL(43, 32)
#define IRDMA_UDA_CQPSQ_QS_HANDLE GENMASK_ULL(9, 0)
#define IRDMA_UDA_CQPSQ_QHASH_QPN GENMASK_ULL(49, 32)
#define IRDMA_UDA_CQPSQ_QHASH_ BIT_ULL(0)
#define IRDMA_UDA_CQPSQ_QHASH_SRC_PORT GENMASK_ULL(31, 16)
#define IRDMA_UDA_CQPSQ_QHASH_DEST_PORT GENMASK_ULL(15, 0)
#define IRDMA_UDA_CQPSQ_QHASH_ADDR0 GENMASK_ULL(63, 32)
#define IRDMA_UDA_CQPSQ_QHASH_ADDR1 GENMASK_ULL(31, 0)
#define IRDMA_UDA_CQPSQ_QHASH_ADDR2 GENMASK_ULL(63, 32)
#define IRDMA_UDA_CQPSQ_QHASH_ADDR3 GENMASK_ULL(31, 0)
#define IRDMA_UDA_CQPSQ_QHASH_WQEVALID BIT_ULL(63)
#define IRDMA_UDA_CQPSQ_QHASH_OPCODE GENMASK_ULL(37, 32)
#define IRDMA_UDA_CQPSQ_QHASH_MANAGE GENMASK_ULL(62, 61)
#define IRDMA_UDA_CQPSQ_QHASH_IPV4VALID GENMASK_ULL(60, 60)
#define IRDMA_UDA_CQPSQ_QHASH_LANFWD GENMASK_ULL(59, 59)
#define IRDMA_UDA_CQPSQ_QHASH_ENTRYTYPE GENMASK_ULL(44, 42)
#endif /* IRDMA_UDA_D_H */
linux-7.2.3/drivers/infiniband/hw/irdma/uk.c
+1928
View File
File diff suppressed because it is too large Load Diff
linux-7.2.3/drivers/infiniband/hw/irdma/user.h
+675
View File
@@ -0,0 +1,675 @@
/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
/* Copyright (c) 2015 - 2020 Intel Corporation */
#ifndef IRDMA_USER_H
#define IRDMA_USER_H
#define irdma_handle void *
#define irdma_adapter_handle irdma_handle
#define irdma_qp_handle irdma_handle
#define irdma_cq_handle irdma_handle
#define irdma_pd_id irdma_handle
#define irdma_stag_handle irdma_handle
#define irdma_stag_index u32
#define irdma_stag u32
#define irdma_stag_key u8
#define irdma_tagged_offset u64
#define irdma_access_privileges u32
#define irdma_physical_fragment u64
#define irdma_address_list u64 *
#define IRDMA_MAX_MR_SIZE 0x200000000000ULL
#define IRDMA_ACCESS_FLAGS_LOCALREAD 0x01
#define IRDMA_ACCESS_FLAGS_LOCALWRITE 0x02
#define IRDMA_ACCESS_FLAGS_REMOTEREAD_ONLY 0x04
#define IRDMA_ACCESS_FLAGS_REMOTEREAD 0x05
#define IRDMA_ACCESS_FLAGS_REMOTEWRITE_ONLY 0x08
#define IRDMA_ACCESS_FLAGS_REMOTEWRITE 0x0a
#define IRDMA_ACCESS_FLAGS_BIND_WINDOW 0x10
#define IRDMA_ACCESS_FLAGS_ZERO_BASED 0x20
#define IRDMA_ACCESS_FLAGS_ALL 0x3f
#define IRDMA_OP_TYPE_RDMA_WRITE 0x00
#define IRDMA_OP_TYPE_RDMA_READ 0x01
#define IRDMA_OP_TYPE_SEND 0x03
#define IRDMA_OP_TYPE_SEND_INV 0x04
#define IRDMA_OP_TYPE_SEND_SOL 0x05
#define IRDMA_OP_TYPE_SEND_SOL_INV 0x06
#define IRDMA_OP_TYPE_RDMA_WRITE_SOL 0x0d
#define IRDMA_OP_TYPE_BIND_MW 0x08
#define IRDMA_OP_TYPE_FAST_REG_NSMR 0x09
#define IRDMA_OP_TYPE_INV_STAG 0x0a
#define IRDMA_OP_TYPE_RDMA_READ_INV_STAG 0x0b
#define IRDMA_OP_TYPE_NOP 0x0c
#define IRDMA_OP_TYPE_ATOMIC_FETCH_AND_ADD 0x0f
#define IRDMA_OP_TYPE_ATOMIC_COMPARE_AND_SWAP 0x11
#define IRDMA_OP_TYPE_REC 0x3e
#define IRDMA_OP_TYPE_REC_IMM 0x3f
#define IRDMA_FLUSH_MAJOR_ERR 1
#define IRDMA_SRQFLUSH_RSVD_MAJOR_ERR 0xfffe
/* Async Events codes */
#define IRDMA_AE_AMP_UNALLOCATED_STAG 0x0102
#define IRDMA_AE_AMP_INVALID_STAG 0x0103
#define IRDMA_AE_AMP_BAD_QP 0x0104
#define IRDMA_AE_AMP_BAD_PD 0x0105
#define IRDMA_AE_AMP_BAD_STAG_KEY 0x0106
#define IRDMA_AE_AMP_BAD_STAG_INDEX 0x0107
#define IRDMA_AE_AMP_BOUNDS_VIOLATION 0x0108
#define IRDMA_AE_AMP_RIGHTS_VIOLATION 0x0109
#define IRDMA_AE_AMP_TO_WRAP 0x010a
#define IRDMA_AE_AMP_FASTREG_VALID_STAG 0x010c
#define IRDMA_AE_AMP_FASTREG_MW_STAG 0x010d
#define IRDMA_AE_AMP_FASTREG_INVALID_RIGHTS 0x010e
#define IRDMA_AE_AMP_FASTREG_INVALID_LENGTH 0x0110
#define IRDMA_AE_AMP_INVALIDATE_SHARED 0x0111
#define IRDMA_AE_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS 0x0112
#define IRDMA_AE_AMP_INVALIDATE_MR_WITH_BOUND_WINDOWS 0x0113
#define IRDMA_AE_AMP_MWBIND_VALID_STAG 0x0114
#define IRDMA_AE_AMP_MWBIND_OF_MR_STAG 0x0115
#define IRDMA_AE_AMP_MWBIND_TO_ZERO_BASED_STAG 0x0116
#define IRDMA_AE_AMP_MWBIND_TO_MW_STAG 0x0117
#define IRDMA_AE_AMP_MWBIND_INVALID_RIGHTS 0x0118
#define IRDMA_AE_AMP_MWBIND_INVALID_BOUNDS 0x0119
#define IRDMA_AE_AMP_MWBIND_TO_INVALID_PARENT 0x011a
#define IRDMA_AE_AMP_MWBIND_BIND_DISABLED 0x011b
#define IRDMA_AE_PRIV_OPERATION_DENIED 0x011c
#define IRDMA_AE_AMP_INVALIDATE_TYPE1_MW 0x011d
#define IRDMA_AE_AMP_MWBIND_ZERO_BASED_TYPE1_MW 0x011e
#define IRDMA_AE_AMP_FASTREG_INVALID_PBL_HPS_CFG 0x011f
#define IRDMA_AE_AMP_MWBIND_WRONG_TYPE 0x0120
#define IRDMA_AE_AMP_FASTREG_PBLE_MISMATCH 0x0121
#define IRDMA_AE_UDA_XMIT_DGRAM_TOO_LONG 0x0132
#define IRDMA_AE_UDA_XMIT_BAD_PD 0x0133
#define IRDMA_AE_UDA_XMIT_DGRAM_TOO_SHORT 0x0134
#define IRDMA_AE_UDA_L4LEN_INVALID 0x0135
#define IRDMA_AE_BAD_CLOSE 0x0201
#define IRDMA_AE_RDMAP_ROE_BAD_LLP_CLOSE 0x0202
#define IRDMA_AE_CQ_OPERATION_ERROR 0x0203
#define IRDMA_AE_RDMA_READ_WHILE_ORD_ZERO 0x0205
#define IRDMA_AE_STAG_ZERO_INVALID 0x0206
#define IRDMA_AE_IB_RREQ_AND_Q1_FULL 0x0207
#define IRDMA_AE_IB_INVALID_REQUEST 0x0208
#define IRDMA_AE_SRQ_LIMIT 0x0209
#define IRDMA_AE_WQE_UNEXPECTED_OPCODE 0x020a
#define IRDMA_AE_WQE_INVALID_PARAMETER 0x020b
#define IRDMA_AE_WQE_INVALID_FRAG_DATA 0x020c
#define IRDMA_AE_IB_REMOTE_ACCESS_ERROR 0x020d
#define IRDMA_AE_IB_REMOTE_OP_ERROR 0x020e
#define IRDMA_AE_SRQ_CATASTROPHIC_ERROR 0x020f
#define IRDMA_AE_WQE_LSMM_TOO_LONG 0x0220
#define IRDMA_AE_ATOMIC_ALIGNMENT 0x0221
#define IRDMA_AE_ATOMIC_MASK 0x0222
#define IRDMA_AE_INVALID_REQUEST 0x0223
#define IRDMA_AE_PCIE_ATOMIC_DISABLE 0x0224
#define IRDMA_AE_DDP_INVALID_MSN_GAP_IN_MSN 0x0301
#define IRDMA_AE_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER 0x0303
#define IRDMA_AE_DDP_UBE_INVALID_DDP_VERSION 0x0304
#define IRDMA_AE_DDP_UBE_INVALID_MO 0x0305
#define IRDMA_AE_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE 0x0306
#define IRDMA_AE_DDP_UBE_INVALID_QN 0x0307
#define IRDMA_AE_DDP_NO_L_BIT 0x0308
#define IRDMA_AE_RDMAP_ROE_INVALID_RDMAP_VERSION 0x0311
#define IRDMA_AE_RDMAP_ROE_UNEXPECTED_OPCODE 0x0312
#define IRDMA_AE_ROE_INVALID_RDMA_READ_REQUEST 0x0313
#define IRDMA_AE_ROE_INVALID_RDMA_WRITE_OR_READ_RESP 0x0314
#define IRDMA_AE_ROCE_RSP_LENGTH_ERROR 0x0316
#define IRDMA_AE_ROCE_EMPTY_MCG 0x0380
#define IRDMA_AE_ROCE_BAD_MC_IP_ADDR 0x0381
#define IRDMA_AE_ROCE_BAD_MC_QPID 0x0382
#define IRDMA_AE_MCG_QP_PROTOCOL_MISMATCH 0x0383
#define IRDMA_AE_INVALID_ARP_ENTRY 0x0401
#define IRDMA_AE_INVALID_TCP_OPTION_RCVD 0x0402
#define IRDMA_AE_STALE_ARP_ENTRY 0x0403
#define IRDMA_AE_INVALID_AH_ENTRY 0x0406
#define IRDMA_AE_LLP_CLOSE_COMPLETE 0x0501
#define IRDMA_AE_LLP_CONNECTION_RESET 0x0502
#define IRDMA_AE_LLP_FIN_RECEIVED 0x0503
#define IRDMA_AE_LLP_RECEIVED_MARKER_AND_LENGTH_FIELDS_DONT_MATCH 0x0504
#define IRDMA_AE_LLP_RECEIVED_MPA_CRC_ERROR 0x0505
#define IRDMA_AE_LLP_SEGMENT_TOO_SMALL 0x0507
#define IRDMA_AE_LLP_SYN_RECEIVED 0x0508
#define IRDMA_AE_LLP_TERMINATE_RECEIVED 0x0509
#define IRDMA_AE_LLP_TOO_MANY_RETRIES 0x050a
#define IRDMA_AE_LLP_TOO_MANY_KEEPALIVE_RETRIES 0x050b
#define IRDMA_AE_LLP_DOUBT_REACHABILITY 0x050c
#define IRDMA_AE_LLP_CONNECTION_ESTABLISHED 0x050e
#define IRDMA_AE_LLP_TOO_MANY_RNRS 0x050f
#define IRDMA_AE_RESOURCE_EXHAUSTION 0x0520
#define IRDMA_AE_RESET_SENT 0x0601
#define IRDMA_AE_TERMINATE_SENT 0x0602
#define IRDMA_AE_RESET_NOT_SENT 0x0603
#define IRDMA_AE_LCE_QP_CATASTROPHIC 0x0700
#define IRDMA_AE_LCE_FUNCTION_CATASTROPHIC 0x0701
#define IRDMA_AE_LCE_CQ_CATASTROPHIC 0x0702
#define IRDMA_AE_REMOTE_QP_CATASTROPHIC 0x0703
#define IRDMA_AE_LOCAL_QP_CATASTROPHIC 0x0704
#define IRDMA_AE_RCE_QP_CATASTROPHIC 0x0705
#define IRDMA_AE_QP_SUSPEND_COMPLETE 0x0900
#define IRDMA_AE_CQP_DEFERRED_COMPLETE 0x0901
#define IRDMA_AE_ADAPTER_CATASTROPHIC 0x0B0B
enum irdma_device_caps_const {
IRDMA_WQE_SIZE = 4,
IRDMA_CQP_WQE_SIZE = 8,
IRDMA_CQE_SIZE = 4,
IRDMA_EXTENDED_CQE_SIZE = 8,
IRDMA_AEQE_SIZE = 2,
IRDMA_CEQE_SIZE = 1,
IRDMA_CQP_CTX_SIZE = 8,
IRDMA_SHADOW_AREA_SIZE = 8,
IRDMA_QUERY_FPM_BUF_SIZE = 200,
IRDMA_COMMIT_FPM_BUF_SIZE = 208,
IRDMA_GATHER_STATS_BUF_SIZE = 1024,
IRDMA_MIN_IW_QP_ID = 0,
IRDMA_MAX_IW_QP_ID = 262143,
IRDMA_MIN_IW_SRQ_ID = 0,
IRDMA_MIN_CEQID = 0,
IRDMA_MAX_CEQID = 1023,
IRDMA_CEQ_MAX_COUNT = IRDMA_MAX_CEQID + 1,
IRDMA_MIN_CQID = 0,
IRDMA_MAX_CQID = 524287,
IRDMA_MIN_AEQ_ENTRIES = 1,
IRDMA_MAX_AEQ_ENTRIES = 524287,
IRDMA_MAX_AEQ_ENTRIES_GEN_3 = 262144,
IRDMA_MIN_CEQ_ENTRIES = 1,
IRDMA_MAX_CEQ_ENTRIES = 262143,
IRDMA_MIN_CQ_SIZE = 1,
IRDMA_MAX_CQ_SIZE = 1048575,
IRDMA_DB_ID_ZERO = 0,
IRDMA_MAX_WQ_FRAGMENT_COUNT = 13,
IRDMA_MAX_SGE_RD = 13,
IRDMA_MAX_OUTBOUND_MSG_SIZE = 2147483647,
IRDMA_MAX_INBOUND_MSG_SIZE = 2147483647,
IRDMA_MAX_PUSH_PAGE_COUNT = 1024,
IRDMA_MAX_PE_ENA_VF_COUNT = 32,
IRDMA_MAX_VF_FPM_ID = 47,
IRDMA_MAX_SQ_PAYLOAD_SIZE = 2145386496,
IRDMA_MAX_INLINE_DATA_SIZE = 101,
IRDMA_MAX_WQ_ENTRIES = 32768,
IRDMA_Q2_BUF_SIZE = 256,
IRDMA_QP_CTX_SIZE = 256,
IRDMA_MAX_PDS = 262144,
IRDMA_MIN_WQ_SIZE_GEN2 = 8,
};
enum irdma_addressing_type {
IRDMA_ADDR_TYPE_ZERO_BASED = 0,
IRDMA_ADDR_TYPE_VA_BASED = 1,
};
enum irdma_flush_opcode {
FLUSH_INVALID = 0,
FLUSH_GENERAL_ERR,
FLUSH_PROT_ERR,
FLUSH_REM_ACCESS_ERR,
FLUSH_LOC_QP_OP_ERR,
FLUSH_REM_OP_ERR,
FLUSH_LOC_LEN_ERR,
FLUSH_FATAL_ERR,
FLUSH_RETRY_EXC_ERR,
FLUSH_MW_BIND_ERR,
FLUSH_REM_INV_REQ_ERR,
FLUSH_RNR_RETRY_EXC_ERR,
};
enum irdma_qp_event_type {
IRDMA_QP_EVENT_CATASTROPHIC,
IRDMA_QP_EVENT_ACCESS_ERR,
IRDMA_QP_EVENT_REQ_ERR,
};
enum irdma_cmpl_status {
IRDMA_COMPL_STATUS_SUCCESS = 0,
IRDMA_COMPL_STATUS_FLUSHED,
IRDMA_COMPL_STATUS_INVALID_WQE,
IRDMA_COMPL_STATUS_QP_CATASTROPHIC,
IRDMA_COMPL_STATUS_REMOTE_TERMINATION,
IRDMA_COMPL_STATUS_INVALID_STAG,
IRDMA_COMPL_STATUS_BASE_BOUND_VIOLATION,
IRDMA_COMPL_STATUS_ACCESS_VIOLATION,
IRDMA_COMPL_STATUS_INVALID_PD_ID,
IRDMA_COMPL_STATUS_WRAP_ERROR,
IRDMA_COMPL_STATUS_STAG_INVALID_PDID,
IRDMA_COMPL_STATUS_RDMA_READ_ZERO_ORD,
IRDMA_COMPL_STATUS_QP_NOT_PRIVLEDGED,
IRDMA_COMPL_STATUS_STAG_NOT_INVALID,
IRDMA_COMPL_STATUS_INVALID_PHYS_BUF_SIZE,
IRDMA_COMPL_STATUS_INVALID_PHYS_BUF_ENTRY,
IRDMA_COMPL_STATUS_INVALID_FBO,
IRDMA_COMPL_STATUS_INVALID_LEN,
IRDMA_COMPL_STATUS_INVALID_ACCESS,
IRDMA_COMPL_STATUS_PHYS_BUF_LIST_TOO_LONG,
IRDMA_COMPL_STATUS_INVALID_VIRT_ADDRESS,
IRDMA_COMPL_STATUS_INVALID_REGION,
IRDMA_COMPL_STATUS_INVALID_WINDOW,
IRDMA_COMPL_STATUS_INVALID_TOTAL_LEN,
IRDMA_COMPL_STATUS_UNKNOWN,
};
enum irdma_cmpl_notify {
IRDMA_CQ_COMPL_EVENT = 0,
IRDMA_CQ_COMPL_SOLICITED = 1,
};
enum irdma_qp_caps {
IRDMA_WRITE_WITH_IMM = 1,
IRDMA_SEND_WITH_IMM = 2,
IRDMA_ROCE = 4,
IRDMA_PUSH_MODE = 8,
};
struct irdma_srq_uk;
struct irdma_srq_uk_init_info;
struct irdma_qp_uk;
struct irdma_cq_uk;
struct irdma_qp_uk_init_info;
struct irdma_cq_uk_init_info;
struct irdma_ring {
u32 head;
u32 tail;
u32 size;
};
struct irdma_cqe {
__le64 buf[IRDMA_CQE_SIZE];
};
struct irdma_extended_cqe {
__le64 buf[IRDMA_EXTENDED_CQE_SIZE];
};
struct irdma_post_send {
struct ib_sge *sg_list;
u32 num_sges;
u32 qkey;
u32 dest_qp;
u32 ah_id;
};
struct irdma_post_rq_info {
u64 wr_id;
struct ib_sge *sg_list;
u32 num_sges;
};
struct irdma_rdma_write {
struct ib_sge *lo_sg_list;
u32 num_lo_sges;
struct ib_sge rem_addr;
};
struct irdma_rdma_read {
struct ib_sge *lo_sg_list;
u32 num_lo_sges;
struct ib_sge rem_addr;
};
struct irdma_bind_window {
irdma_stag mr_stag;
u64 bind_len;
void *va;
enum irdma_addressing_type addressing_type;
bool ena_reads:1;
bool ena_writes:1;
irdma_stag mw_stag;
bool mem_window_type_1:1;
bool remote_atomics_en:1;
};
struct irdma_atomic_fetch_add {
u64 tagged_offset;
u64 remote_tagged_offset;
u64 fetch_add_data_bytes;
u32 stag;
u32 remote_stag;
};
struct irdma_atomic_compare_swap {
u64 tagged_offset;
u64 remote_tagged_offset;
u64 swap_data_bytes;
u64 compare_data_bytes;
u32 stag;
u32 remote_stag;
};
struct irdma_inv_local_stag {
irdma_stag target_stag;
};
struct irdma_post_sq_info {
u64 wr_id;
u8 op_type;
u8 l4len;
bool signaled:1;
bool read_fence:1;
bool local_fence:1;
bool inline_data:1;
bool imm_data_valid:1;
bool report_rtt:1;
bool udp_hdr:1;
bool defer_flag:1;
bool remote_atomic_en:1;
u32 imm_data;
u32 stag_to_inv;
union {
struct irdma_post_send send;
struct irdma_rdma_write rdma_write;
struct irdma_rdma_read rdma_read;
struct irdma_bind_window bind_window;
struct irdma_inv_local_stag inv_local_stag;
struct irdma_atomic_fetch_add atomic_fetch_add;
struct irdma_atomic_compare_swap atomic_compare_swap;
} op;
};
struct irdma_cq_poll_info {
u64 wr_id;
irdma_qp_handle qp_handle;
u32 bytes_xfered;
u32 tcp_seq_num_rtt;
u32 qp_id;
u32 ud_src_qpn;
u32 imm_data;
irdma_stag inv_stag; /* or L_R_Key */
enum irdma_cmpl_status comp_status;
u16 major_err;
u16 minor_err;
u16 ud_vlan;
u8 ud_smac[6];
u8 op_type;
u8 q_type;
bool stag_invalid_set:1; /* or L_R_Key set */
bool error:1;
bool solicited_event:1;
bool ipv4:1;
bool ud_vlan_valid:1;
bool ud_smac_valid:1;
bool imm_valid:1;
};
struct qp_err_code {
enum irdma_flush_opcode flush_code;
enum irdma_qp_event_type event_type;
};
int irdma_uk_atomic_compare_swap(struct irdma_qp_uk *qp,
struct irdma_post_sq_info *info, bool post_sq);
int irdma_uk_atomic_fetch_add(struct irdma_qp_uk *qp,
struct irdma_post_sq_info *info, bool post_sq);
int irdma_uk_inline_rdma_write(struct irdma_qp_uk *qp,
struct irdma_post_sq_info *info, bool post_sq);
int irdma_uk_inline_send(struct irdma_qp_uk *qp,
struct irdma_post_sq_info *info, bool post_sq);
int irdma_uk_post_nop(struct irdma_qp_uk *qp, u64 wr_id, bool signaled,
bool post_sq);
int irdma_uk_post_receive(struct irdma_qp_uk *qp,
struct irdma_post_rq_info *info);
void irdma_uk_qp_post_wr(struct irdma_qp_uk *qp);
int irdma_uk_rdma_read(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info,
bool inv_stag, bool post_sq);
int irdma_uk_rdma_write(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info,
bool post_sq);
int irdma_uk_send(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info,
bool post_sq);
int irdma_uk_stag_local_invalidate(struct irdma_qp_uk *qp,
struct irdma_post_sq_info *info,
bool post_sq);
struct irdma_wqe_uk_ops {
void (*iw_copy_inline_data)(u8 *dest, struct ib_sge *sge_list,
u32 num_sges, u8 polarity);
u16 (*iw_inline_data_size_to_quanta)(u32 data_size);
void (*iw_set_fragment)(__le64 *wqe, u32 offset, struct ib_sge *sge,
u8 valid);
void (*iw_set_mw_bind_wqe)(__le64 *wqe,
struct irdma_bind_window *op_info);
};
bool irdma_uk_cq_empty(struct irdma_cq_uk *cq);
int irdma_uk_cq_poll_cmpl(struct irdma_cq_uk *cq,
struct irdma_cq_poll_info *info);
void irdma_uk_cq_request_notification(struct irdma_cq_uk *cq,
enum irdma_cmpl_notify cq_notify);
void irdma_uk_cq_resize(struct irdma_cq_uk *cq, void *cq_base, int size);
void irdma_uk_cq_set_resized_cnt(struct irdma_cq_uk *qp, u16 cnt);
void irdma_uk_cq_init(struct irdma_cq_uk *cq,
struct irdma_cq_uk_init_info *info);
int irdma_uk_qp_init(struct irdma_qp_uk *qp,
struct irdma_qp_uk_init_info *info);
void irdma_uk_calc_shift_wq(struct irdma_qp_uk_init_info *ukinfo, u8 *sq_shift,
u8 *rq_shift);
int irdma_uk_calc_depth_shift_sq(struct irdma_qp_uk_init_info *ukinfo,
u32 *sq_depth, u8 *sq_shift);
int irdma_uk_calc_depth_shift_rq(struct irdma_qp_uk_init_info *ukinfo,
u32 *rq_depth, u8 *rq_shift);
int irdma_uk_srq_init(struct irdma_srq_uk *srq,
struct irdma_srq_uk_init_info *info);
int irdma_uk_srq_post_receive(struct irdma_srq_uk *srq,
struct irdma_post_rq_info *info);
struct irdma_srq_uk {
u32 srq_caps;
struct irdma_qp_quanta *srq_base;
struct irdma_uk_attrs *uk_attrs;
__le64 *shadow_area;
struct irdma_ring srq_ring;
u32 srq_id;
u32 srq_size;
u32 max_srq_frag_cnt;
struct irdma_wqe_uk_ops wqe_ops;
u8 srwqe_polarity;
u8 wqe_size;
u8 wqe_size_multiplier;
u8 deferred_flag;
spinlock_t *lock;
};
struct irdma_srq_uk_init_info {
struct irdma_qp_quanta *srq;
struct irdma_uk_attrs *uk_attrs;
__le64 *shadow_area;
u64 *srq_wrid_array;
u32 srq_id;
u32 srq_caps;
u32 srq_size;
u32 max_srq_frag_cnt;
};
struct irdma_sq_uk_wr_trk_info {
u64 wrid;
u32 wr_len;
u16 quanta;
u8 signaled;
u8 reserved[1];
};
struct irdma_qp_quanta {
__le64 elem[IRDMA_WQE_SIZE];
};
struct irdma_qp_uk {
struct irdma_qp_quanta *sq_base;
struct irdma_qp_quanta *rq_base;
struct irdma_uk_attrs *uk_attrs;
u32 __iomem *wqe_alloc_db;
struct irdma_sq_uk_wr_trk_info *sq_wrtrk_array;
u64 *rq_wrid_array;
__le64 *shadow_area;
struct irdma_ring sq_ring;
struct irdma_ring rq_ring;
struct irdma_ring initial_ring;
u32 qp_id;
u32 qp_caps;
u32 sq_size;
u32 rq_size;
u32 max_sq_frag_cnt;
u32 max_rq_frag_cnt;
u32 max_inline_data;
struct irdma_wqe_uk_ops wqe_ops;
u16 conn_wqes;
u8 qp_type;
u8 swqe_polarity;
u8 swqe_polarity_deferred;
u8 rwqe_polarity;
u8 rq_wqe_size;
u8 rq_wqe_size_multiplier;
bool deferred_flag:1;
bool first_sq_wq:1;
bool sq_flush_complete:1; /* Indicates flush was seen and SQ was empty after the flush */
bool rq_flush_complete:1; /* Indicates flush was seen and RQ was empty after the flush */
bool destroy_pending:1; /* Indicates the QP is being destroyed */
void *back_qp;
u8 dbg_rq_flushed;
struct irdma_srq_uk *srq_uk;
u8 sq_flush_seen;
u8 rq_flush_seen;
};
struct irdma_cq_uk {
struct irdma_cqe *cq_base;
u32 __iomem *cqe_alloc_db;
u32 __iomem *cq_ack_db;
__le64 *shadow_area;
u32 cq_id;
u32 cq_size;
struct irdma_ring cq_ring;
u8 polarity;
bool avoid_mem_cflct:1;
};
struct irdma_qp_uk_init_info {
struct irdma_qp_quanta *sq;
struct irdma_qp_quanta *rq;
struct irdma_uk_attrs *uk_attrs;
u32 __iomem *wqe_alloc_db;
__le64 *shadow_area;
struct irdma_sq_uk_wr_trk_info *sq_wrtrk_array;
u64 *rq_wrid_array;
u32 qp_id;
u32 qp_caps;
u32 sq_size;
u32 rq_size;
u32 max_sq_frag_cnt;
u32 max_rq_frag_cnt;
u32 max_inline_data;
u32 sq_depth;
u32 rq_depth;
u8 first_sq_wq;
u8 type;
u8 sq_shift;
u8 rq_shift;
int abi_ver;
struct irdma_srq_uk *srq_uk;
};
struct irdma_cq_uk_init_info {
u32 __iomem *cqe_alloc_db;
u32 __iomem *cq_ack_db;
struct irdma_cqe *cq_base;
__le64 *shadow_area;
u32 cq_size;
u32 cq_id;
bool avoid_mem_cflct;
};
__le64 *irdma_qp_get_next_send_wqe(struct irdma_qp_uk *qp, u32 *wqe_idx,
u16 quanta, u32 total_size,
struct irdma_post_sq_info *info);
__le64 *irdma_srq_get_next_recv_wqe(struct irdma_srq_uk *srq, u32 *wqe_idx);
__le64 *irdma_qp_get_next_recv_wqe(struct irdma_qp_uk *qp, u32 *wqe_idx);
void irdma_uk_clean_cq(void *q, struct irdma_cq_uk *cq);
int irdma_nop(struct irdma_qp_uk *qp, u64 wr_id, bool signaled, bool post_sq);
int irdma_fragcnt_to_quanta_sq(u32 frag_cnt, u16 *quanta);
int irdma_fragcnt_to_wqesize_rq(u32 frag_cnt, u16 *wqe_size);
void irdma_get_wqe_shift(struct irdma_uk_attrs *uk_attrs, u32 sge,
u32 inline_data, u8 *shift);
int irdma_get_sqdepth(struct irdma_uk_attrs *uk_attrs, u32 sq_size, u8 shift,
u32 *wqdepth);
int irdma_get_rqdepth(struct irdma_uk_attrs *uk_attrs, u32 rq_size, u8 shift,
u32 *wqdepth);
int irdma_get_srqdepth(struct irdma_uk_attrs *uk_attrs, u32 srq_size, u8 shift,
u32 *srqdepth);
void irdma_clr_wqes(struct irdma_qp_uk *qp, u32 qp_wqe_idx);
static inline struct qp_err_code irdma_ae_to_qp_err_code(u16 ae_id)
{
struct qp_err_code qp_err = {};
switch (ae_id) {
case IRDMA_AE_AMP_BOUNDS_VIOLATION:
case IRDMA_AE_AMP_INVALID_STAG:
case IRDMA_AE_AMP_RIGHTS_VIOLATION:
case IRDMA_AE_AMP_UNALLOCATED_STAG:
case IRDMA_AE_AMP_BAD_PD:
case IRDMA_AE_AMP_BAD_QP:
case IRDMA_AE_AMP_BAD_STAG_KEY:
case IRDMA_AE_AMP_BAD_STAG_INDEX:
case IRDMA_AE_AMP_TO_WRAP:
case IRDMA_AE_PRIV_OPERATION_DENIED:
qp_err.flush_code = FLUSH_PROT_ERR;
qp_err.event_type = IRDMA_QP_EVENT_ACCESS_ERR;
break;
case IRDMA_AE_UDA_XMIT_BAD_PD:
case IRDMA_AE_WQE_UNEXPECTED_OPCODE:
qp_err.flush_code = FLUSH_LOC_QP_OP_ERR;
qp_err.event_type = IRDMA_QP_EVENT_CATASTROPHIC;
break;
case IRDMA_AE_UDA_XMIT_DGRAM_TOO_SHORT:
case IRDMA_AE_UDA_XMIT_DGRAM_TOO_LONG:
case IRDMA_AE_UDA_L4LEN_INVALID:
case IRDMA_AE_DDP_UBE_INVALID_MO:
case IRDMA_AE_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER:
qp_err.flush_code = FLUSH_LOC_LEN_ERR;
qp_err.event_type = IRDMA_QP_EVENT_CATASTROPHIC;
break;
case IRDMA_AE_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS:
case IRDMA_AE_IB_REMOTE_ACCESS_ERROR:
qp_err.flush_code = FLUSH_REM_ACCESS_ERR;
qp_err.event_type = IRDMA_QP_EVENT_ACCESS_ERR;
break;
case IRDMA_AE_AMP_MWBIND_INVALID_RIGHTS:
case IRDMA_AE_AMP_MWBIND_BIND_DISABLED:
case IRDMA_AE_AMP_MWBIND_INVALID_BOUNDS:
case IRDMA_AE_AMP_MWBIND_VALID_STAG:
qp_err.flush_code = FLUSH_MW_BIND_ERR;
qp_err.event_type = IRDMA_QP_EVENT_ACCESS_ERR;
break;
case IRDMA_AE_LLP_TOO_MANY_RETRIES:
qp_err.flush_code = FLUSH_RETRY_EXC_ERR;
qp_err.event_type = IRDMA_QP_EVENT_CATASTROPHIC;
break;
case IRDMA_AE_IB_INVALID_REQUEST:
qp_err.flush_code = FLUSH_REM_INV_REQ_ERR;
qp_err.event_type = IRDMA_QP_EVENT_REQ_ERR;
break;
case IRDMA_AE_LLP_SEGMENT_TOO_SMALL:
case IRDMA_AE_LLP_RECEIVED_MPA_CRC_ERROR:
case IRDMA_AE_ROCE_RSP_LENGTH_ERROR:
case IRDMA_AE_IB_REMOTE_OP_ERROR:
qp_err.flush_code = FLUSH_REM_OP_ERR;
qp_err.event_type = IRDMA_QP_EVENT_CATASTROPHIC;
break;
case IRDMA_AE_LLP_TOO_MANY_RNRS:
qp_err.flush_code = FLUSH_RNR_RETRY_EXC_ERR;
qp_err.event_type = IRDMA_QP_EVENT_CATASTROPHIC;
break;
case IRDMA_AE_LCE_QP_CATASTROPHIC:
case IRDMA_AE_REMOTE_QP_CATASTROPHIC:
case IRDMA_AE_LOCAL_QP_CATASTROPHIC:
case IRDMA_AE_RCE_QP_CATASTROPHIC:
qp_err.flush_code = FLUSH_FATAL_ERR;
qp_err.event_type = IRDMA_QP_EVENT_CATASTROPHIC;
break;
default:
qp_err.flush_code = FLUSH_GENERAL_ERR;
qp_err.event_type = IRDMA_QP_EVENT_CATASTROPHIC;
break;
}
return qp_err;
}
#endif /* IRDMA_USER_H */
linux-7.2.3/drivers/infiniband/hw/irdma/utils.c
+2506
View File
File diff suppressed because it is too large Load Diff
linux-7.2.3/drivers/infiniband/hw/irdma/verbs.c
+5599
View File
File diff suppressed because it is too large Load Diff
linux-7.2.3/drivers/infiniband/hw/irdma/verbs.h
+340
View File
@@ -0,0 +1,340 @@
/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
/* Copyright (c) 2015 - 2021 Intel Corporation */
#ifndef IRDMA_VERBS_H
#define IRDMA_VERBS_H
#define IRDMA_MAX_SAVED_PHY_PGADDR 4
#define IRDMA_FLUSH_DELAY_MS 20
#define IRDMA_PKEY_TBL_SZ 1
#define IRDMA_DEFAULT_PKEY 0xFFFF
#define IRDMA_SHADOW_PGCNT 1
struct irdma_ucontext {
struct ib_ucontext ibucontext;
struct irdma_device *iwdev;
struct rdma_user_mmap_entry *db_mmap_entry;
struct list_head cq_reg_mem_list;
spinlock_t cq_reg_mem_list_lock; /* protect CQ memory list */
struct list_head qp_reg_mem_list;
spinlock_t qp_reg_mem_list_lock; /* protect QP memory list */
struct list_head srq_reg_mem_list;
spinlock_t srq_reg_mem_list_lock; /* protect SRQ memory list */
int abi_ver;
u8 legacy_mode : 1;
u8 use_raw_attrs : 1;
};
struct irdma_pd {
struct ib_pd ibpd;
struct irdma_sc_pd sc_pd;
};
union irdma_sockaddr {
struct sockaddr_in saddr_in;
struct sockaddr_in6 saddr_in6;
};
struct irdma_av {
u8 macaddr[16];
struct rdma_ah_attr attrs;
union irdma_sockaddr sgid_addr;
union irdma_sockaddr dgid_addr;
u8 net_type;
};
struct irdma_ah {
struct ib_ah ibah;
struct irdma_sc_ah sc_ah;
struct irdma_pd *pd;
struct irdma_av av;
u8 sgid_index;
union ib_gid dgid;
struct hlist_node list;
refcount_t refcnt;
struct irdma_ah *parent_ah; /* AH from cached list */
};
struct irdma_hmc_pble {
union {
u32 idx;
dma_addr_t addr;
};
};
struct irdma_cq_mr {
struct irdma_hmc_pble cq_pbl;
dma_addr_t shadow;
};
struct irdma_srq_mr {
struct irdma_hmc_pble srq_pbl;
dma_addr_t shadow;
};
struct irdma_qp_mr {
struct irdma_hmc_pble sq_pbl;
struct irdma_hmc_pble rq_pbl;
dma_addr_t shadow;
dma_addr_t rq_pa;
struct page *sq_page;
};
struct irdma_cq_buf {
struct irdma_dma_mem kmem_buf;
struct irdma_cq_uk cq_uk;
struct irdma_hw *hw;
struct list_head list;
struct work_struct work;
};
struct irdma_pbl {
struct list_head list;
union {
struct irdma_qp_mr qp_mr;
struct irdma_cq_mr cq_mr;
struct irdma_srq_mr srq_mr;
};
bool pbl_allocated:1;
bool on_list:1;
u64 user_base;
struct irdma_pble_alloc pble_alloc;
struct irdma_mr *iwmr;
};
struct irdma_mr {
union {
struct ib_mr ibmr;
struct ib_mw ibmw;
};
struct ib_umem *region;
int access;
bool is_hwreg:1;
bool dma_mr:1;
u16 type;
u32 page_cnt;
u64 page_size;
u32 npages;
u32 stag;
u64 len;
u64 pgaddrmem[IRDMA_MAX_SAVED_PHY_PGADDR];
struct irdma_pbl iwpbl;
};
struct irdma_srq {
struct ib_srq ibsrq;
struct irdma_sc_srq sc_srq __aligned(64);
struct irdma_dma_mem kmem;
u64 *srq_wrid_mem;
refcount_t refcnt;
spinlock_t lock; /* for poll srq */
struct irdma_pbl *iwpbl;
struct irdma_sge *sg_list;
u16 srq_head;
u32 srq_num;
u32 max_wr;
bool user_mode:1;
};
struct irdma_cq {
struct ib_cq ibcq;
struct irdma_sc_cq sc_cq;
u32 cq_num;
bool user_mode;
atomic_t armed;
enum irdma_cmpl_notify last_notify;
struct irdma_dma_mem kmem;
struct irdma_dma_mem kmem_shadow;
struct completion free_cq;
refcount_t refcnt;
spinlock_t lock; /* for poll cq */
struct list_head resize_list;
struct irdma_cq_poll_info cur_cqe;
struct list_head cmpl_generated;
};
struct irdma_cmpl_gen {
struct list_head list;
struct irdma_cq_poll_info cpi;
};
struct disconn_work {
struct work_struct work;
struct irdma_qp *iwqp;
};
struct iw_cm_id;
struct irdma_qp_kmode {
struct irdma_dma_mem dma_mem;
struct irdma_sq_uk_wr_trk_info *sq_wrid_mem;
u64 *rq_wrid_mem;
};
struct irdma_qp {
struct ib_qp ibqp;
struct irdma_sc_qp sc_qp;
struct irdma_device *iwdev;
struct irdma_cq *iwscq;
struct irdma_cq *iwrcq;
struct irdma_pd *iwpd;
struct rdma_user_mmap_entry *push_wqe_mmap_entry;
struct rdma_user_mmap_entry *push_db_mmap_entry;
struct irdma_qp_host_ctx_info ctx_info;
union {
struct irdma_iwarp_offload_info iwarp_info;
struct irdma_roce_offload_info roce_info;
};
union {
struct irdma_tcp_offload_info tcp_info;
struct irdma_udp_offload_info udp_info;
};
struct irdma_ah roce_ah;
struct list_head teardown_entry;
refcount_t refcnt;
struct iw_cm_id *cm_id;
struct irdma_cm_node *cm_node;
struct delayed_work dwork_flush;
struct ib_mr *lsmm_mr;
atomic_t hw_mod_qp_pend;
enum ib_qp_state ibqp_state;
u32 qp_mem_size;
u32 last_aeq;
int max_send_wr;
int max_recv_wr;
atomic_t close_timer_started;
spinlock_t lock; /* serialize posting WRs to SQ/RQ */
struct irdma_qp_context *iwqp_context;
void *pbl_vbase;
dma_addr_t pbl_pbase;
struct page *page;
u8 active_conn : 1;
u8 user_mode : 1;
u8 hte_added : 1;
u8 flush_issued : 1;
u8 sig_all : 1;
u8 pau_mode : 1;
u8 suspend_pending : 1;
u8 rsvd : 1;
u8 iwarp_state;
u16 term_sq_flush_code;
u16 term_rq_flush_code;
u8 hw_iwarp_state;
u8 hw_tcp_state;
struct irdma_qp_kmode kqp;
struct irdma_dma_mem host_ctx;
struct timer_list terminate_timer;
struct irdma_pbl *iwpbl;
struct irdma_dma_mem q2_ctx_mem;
struct irdma_dma_mem ietf_mem;
struct completion free_qp;
wait_queue_head_t waitq;
wait_queue_head_t mod_qp_waitq;
u8 rts_ae_rcvd;
};
enum irdma_mmap_flag {
IRDMA_MMAP_IO_NC,
IRDMA_MMAP_IO_WC,
};
struct irdma_user_mmap_entry {
struct rdma_user_mmap_entry rdma_entry;
u64 bar_offset;
u8 mmap_flag;
};
static inline u16 irdma_fw_major_ver(struct irdma_sc_dev *dev)
{
return (u16)FIELD_GET(IRDMA_FW_VER_MAJOR, dev->feature_info[IRDMA_FEATURE_FW_INFO]);
}
static inline u16 irdma_fw_minor_ver(struct irdma_sc_dev *dev)
{
return (u16)FIELD_GET(IRDMA_FW_VER_MINOR, dev->feature_info[IRDMA_FEATURE_FW_INFO]);
}
static inline void set_ib_wc_op_sq(struct irdma_cq_poll_info *cq_poll_info,
struct ib_wc *entry)
{
switch (cq_poll_info->op_type) {
case IRDMA_OP_TYPE_RDMA_WRITE:
case IRDMA_OP_TYPE_RDMA_WRITE_SOL:
entry->opcode = IB_WC_RDMA_WRITE;
break;
case IRDMA_OP_TYPE_RDMA_READ_INV_STAG:
case IRDMA_OP_TYPE_RDMA_READ:
entry->opcode = IB_WC_RDMA_READ;
break;
case IRDMA_OP_TYPE_SEND_SOL:
case IRDMA_OP_TYPE_SEND_SOL_INV:
case IRDMA_OP_TYPE_SEND_INV:
case IRDMA_OP_TYPE_SEND:
entry->opcode = IB_WC_SEND;
break;
case IRDMA_OP_TYPE_FAST_REG_NSMR:
entry->opcode = IB_WC_REG_MR;
break;
case IRDMA_OP_TYPE_ATOMIC_COMPARE_AND_SWAP:
entry->opcode = IB_WC_COMP_SWAP;
break;
case IRDMA_OP_TYPE_ATOMIC_FETCH_AND_ADD:
entry->opcode = IB_WC_FETCH_ADD;
break;
case IRDMA_OP_TYPE_INV_STAG:
entry->opcode = IB_WC_LOCAL_INV;
break;
default:
entry->status = IB_WC_GENERAL_ERR;
}
}
static inline void set_ib_wc_op_rq_gen_3(struct irdma_cq_poll_info *info,
struct ib_wc *entry)
{
switch (info->op_type) {
case IRDMA_OP_TYPE_RDMA_WRITE:
case IRDMA_OP_TYPE_RDMA_WRITE_SOL:
entry->opcode = IB_WC_RECV_RDMA_WITH_IMM;
break;
default:
entry->opcode = IB_WC_RECV;
}
}
static inline void set_ib_wc_op_rq(struct irdma_cq_poll_info *cq_poll_info,
struct ib_wc *entry, bool send_imm_support)
{
/**
* iWARP does not support sendImm, so the presence of Imm data
* must be WriteImm.
*/
if (!send_imm_support) {
entry->opcode = cq_poll_info->imm_valid ?
IB_WC_RECV_RDMA_WITH_IMM :
IB_WC_RECV;
return;
}
switch (cq_poll_info->op_type) {
case IB_OPCODE_RDMA_WRITE_ONLY_WITH_IMMEDIATE:
case IB_OPCODE_RDMA_WRITE_LAST_WITH_IMMEDIATE:
entry->opcode = IB_WC_RECV_RDMA_WITH_IMM;
break;
default:
entry->opcode = IB_WC_RECV;
}
}
void irdma_mcast_mac(u32 *ip_addr, u8 *mac, bool ipv4);
int irdma_ib_register_device(struct irdma_device *iwdev);
void irdma_ib_unregister_device(struct irdma_device *iwdev);
void irdma_ib_dealloc_device(struct ib_device *ibdev);
void irdma_ib_qp_event(struct irdma_qp *iwqp, enum irdma_qp_event_type event);
void irdma_generate_flush_completions(struct irdma_qp *iwqp);
void irdma_remove_cmpls_list(struct irdma_cq *iwcq);
int irdma_generated_cmpls(struct irdma_cq *iwcq, struct irdma_cq_poll_info *cq_poll_info);
#endif /* IRDMA_VERBS_H */
linux-7.2.3/drivers/infiniband/hw/irdma/virtchnl.c
+618
View File
@@ -0,0 +1,618 @@
// SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
/* Copyright (c) 2015 - 2024 Intel Corporation */
#include "osdep.h"
#include "hmc.h"
#include "defs.h"
#include "type.h"
#include "protos.h"
#include "virtchnl.h"
#include "ws.h"
#include "i40iw_hw.h"
#include "ig3rdma_hw.h"
struct vchnl_reg_map_elem {
u16 reg_id;
u16 reg_idx;
bool pg_rel;
};
struct vchnl_regfld_map_elem {
u16 regfld_id;
u16 regfld_idx;
};
static struct vchnl_reg_map_elem vchnl_reg_map[] = {
{IRDMA_VCHNL_REG_ID_CQPTAIL, IRDMA_CQPTAIL, false},
{IRDMA_VCHNL_REG_ID_CQPDB, IRDMA_CQPDB, false},
{IRDMA_VCHNL_REG_ID_CCQPSTATUS, IRDMA_CCQPSTATUS, false},
{IRDMA_VCHNL_REG_ID_CCQPHIGH, IRDMA_CCQPHIGH, false},
{IRDMA_VCHNL_REG_ID_CCQPLOW, IRDMA_CCQPLOW, false},
{IRDMA_VCHNL_REG_ID_CQARM, IRDMA_CQARM, false},
{IRDMA_VCHNL_REG_ID_CQACK, IRDMA_CQACK, false},
{IRDMA_VCHNL_REG_ID_AEQALLOC, IRDMA_AEQALLOC, false},
{IRDMA_VCHNL_REG_ID_CQPERRCODES, IRDMA_CQPERRCODES, false},
{IRDMA_VCHNL_REG_ID_WQEALLOC, IRDMA_WQEALLOC, false},
{IRDMA_VCHNL_REG_ID_DB_ADDR_OFFSET, IRDMA_DB_ADDR_OFFSET, false },
{IRDMA_VCHNL_REG_ID_DYN_CTL, IRDMA_GLINT_DYN_CTL, false },
{IRDMA_VCHNL_REG_INV_ID, IRDMA_VCHNL_REG_INV_ID, false }
};
static struct vchnl_regfld_map_elem vchnl_regfld_map[] = {
{IRDMA_VCHNL_REGFLD_ID_CCQPSTATUS_CQP_OP_ERR, IRDMA_CCQPSTATUS_CCQP_ERR_M},
{IRDMA_VCHNL_REGFLD_ID_CCQPSTATUS_CCQP_DONE, IRDMA_CCQPSTATUS_CCQP_DONE_M},
{IRDMA_VCHNL_REGFLD_ID_CQPSQ_STAG_PDID, IRDMA_CQPSQ_STAG_PDID_M},
{IRDMA_VCHNL_REGFLD_ID_CQPSQ_CQ_CEQID, IRDMA_CQPSQ_CQ_CEQID_M},
{IRDMA_VCHNL_REGFLD_ID_CQPSQ_CQ_CQID, IRDMA_CQPSQ_CQ_CQID_M},
{IRDMA_VCHNL_REGFLD_ID_COMMIT_FPM_CQCNT, IRDMA_COMMIT_FPM_CQCNT_M},
{IRDMA_VCHNL_REGFLD_ID_UPESD_HMCN_ID, IRDMA_CQPSQ_UPESD_HMCFNID_M},
{IRDMA_VCHNL_REGFLD_INV_ID, IRDMA_VCHNL_REGFLD_INV_ID}
};
#define IRDMA_VCHNL_REG_COUNT ARRAY_SIZE(vchnl_reg_map)
#define IRDMA_VCHNL_REGFLD_COUNT ARRAY_SIZE(vchnl_regfld_map)
#define IRDMA_VCHNL_REGFLD_BUF_SIZE \
(IRDMA_VCHNL_REG_COUNT * sizeof(struct irdma_vchnl_reg_info) + \
IRDMA_VCHNL_REGFLD_COUNT * sizeof(struct irdma_vchnl_reg_field_info))
#define IRDMA_REGMAP_RESP_BUF_SIZE (IRDMA_VCHNL_RESP_MIN_SIZE + IRDMA_VCHNL_REGFLD_BUF_SIZE)
/**
* irdma_sc_vchnl_init - Initialize dev virtchannel and get hw_rev
* @dev: dev structure to update
* @info: virtchannel info parameters to fill into the dev structure
*/
int irdma_sc_vchnl_init(struct irdma_sc_dev *dev,
struct irdma_vchnl_init_info *info)
{
dev->vchnl_up = true;
dev->privileged = info->privileged;
dev->is_pf = info->is_pf;
dev->hw_attrs.uk_attrs.hw_rev = info->hw_rev;
if (!dev->privileged) {
int ret = irdma_vchnl_req_get_ver(dev, IRDMA_VCHNL_CHNL_VER_MAX,
&dev->vchnl_ver);
ibdev_dbg(to_ibdev(dev),
"DEV: Get Channel version ret = %d, version is %u\n",
ret, dev->vchnl_ver);
if (ret)
return ret;
ret = irdma_vchnl_req_get_caps(dev);
if (ret)
return ret;
dev->hw_attrs.uk_attrs.hw_rev = dev->vc_caps.hw_rev;
}
return 0;
}
/**
* irdma_vchnl_req_verify_resp - Verify requested response size
* @vchnl_req: vchnl message requested
* @resp_len: response length sent from vchnl peer
*/
static int irdma_vchnl_req_verify_resp(struct irdma_vchnl_req *vchnl_req,
u16 resp_len)
{
switch (vchnl_req->vchnl_msg->op_code) {
case IRDMA_VCHNL_OP_GET_VER:
case IRDMA_VCHNL_OP_GET_HMC_FCN:
case IRDMA_VCHNL_OP_PUT_HMC_FCN:
if (resp_len != vchnl_req->parm_len)
return -EBADMSG;
break;
case IRDMA_VCHNL_OP_GET_RDMA_CAPS:
if (resp_len < IRDMA_VCHNL_OP_GET_RDMA_CAPS_MIN_SIZE)
return -EBADMSG;
break;
case IRDMA_VCHNL_OP_GET_REG_LAYOUT:
case IRDMA_VCHNL_OP_QUEUE_VECTOR_MAP:
case IRDMA_VCHNL_OP_QUEUE_VECTOR_UNMAP:
case IRDMA_VCHNL_OP_ADD_VPORT:
case IRDMA_VCHNL_OP_DEL_VPORT:
break;
default:
return -EOPNOTSUPP;
}
return 0;
}
static void irdma_free_vchnl_req_msg(struct irdma_vchnl_req *vchnl_req)
{
kfree(vchnl_req->vchnl_msg);
}
static int irdma_alloc_vchnl_req_msg(struct irdma_vchnl_req *vchnl_req,
struct irdma_vchnl_req_init_info *info)
{
struct irdma_vchnl_op_buf *vchnl_msg;
vchnl_msg = kzalloc(IRDMA_VCHNL_MAX_MSG_SIZE, GFP_KERNEL);
if (!vchnl_msg)
return -ENOMEM;
vchnl_msg->op_ctx = (uintptr_t)vchnl_req;
vchnl_msg->buf_len = sizeof(*vchnl_msg) + info->req_parm_len;
if (info->req_parm_len)
memcpy(vchnl_msg->buf, info->req_parm, info->req_parm_len);
vchnl_msg->op_code = info->op_code;
vchnl_msg->op_ver = info->op_ver;
vchnl_req->vchnl_msg = vchnl_msg;
vchnl_req->parm = info->resp_parm;
vchnl_req->parm_len = info->resp_parm_len;
return 0;
}
static int irdma_vchnl_req_send_sync(struct irdma_sc_dev *dev,
struct irdma_vchnl_req_init_info *info)
{
u16 resp_len = sizeof(dev->vc_recv_buf);
struct irdma_vchnl_req vchnl_req = {};
u16 msg_len;
u8 *msg;
int ret;
ret = irdma_alloc_vchnl_req_msg(&vchnl_req, info);
if (ret)
return ret;
msg_len = vchnl_req.vchnl_msg->buf_len;
msg = (u8 *)vchnl_req.vchnl_msg;
mutex_lock(&dev->vchnl_mutex);
ret = ig3rdma_vchnl_send_sync(dev, msg, msg_len, dev->vc_recv_buf,
&resp_len);
dev->vc_recv_len = resp_len;
if (ret)
goto exit;
ret = irdma_vchnl_req_get_resp(dev, &vchnl_req);
exit:
mutex_unlock(&dev->vchnl_mutex);
ibdev_dbg(to_ibdev(dev),
"VIRT: virtual channel send %s caller: %pS ret=%d op=%u op_ver=%u req_len=%u parm_len=%u resp_len=%u\n",
!ret ? "SUCCEEDS" : "FAILS", __builtin_return_address(0),
ret, vchnl_req.vchnl_msg->op_code,
vchnl_req.vchnl_msg->op_ver, vchnl_req.vchnl_msg->buf_len,
vchnl_req.parm_len, vchnl_req.resp_len);
irdma_free_vchnl_req_msg(&vchnl_req);
return ret;
}
/**
* irdma_vchnl_req_get_reg_layout - Get Register Layout
* @dev: RDMA device pointer
*/
int irdma_vchnl_req_get_reg_layout(struct irdma_sc_dev *dev)
{
u16 reg_idx, reg_id, tmp_reg_id, regfld_idx, regfld_id, tmp_regfld_id;
struct irdma_vchnl_reg_field_info *regfld_array = NULL;
u8 resp_buffer[IRDMA_REGMAP_RESP_BUF_SIZE] = {};
struct vchnl_regfld_map_elem *regfld_map_array;
struct irdma_vchnl_req_init_info info = {};
struct vchnl_reg_map_elem *reg_map_array;
struct irdma_vchnl_reg_info *reg_array;
u8 num_bits, shift_cnt;
u16 buf_len = 0;
u64 bitmask;
u32 rindex;
int ret;
if (!dev->vchnl_up)
return -EBUSY;
info.op_code = IRDMA_VCHNL_OP_GET_REG_LAYOUT;
info.op_ver = IRDMA_VCHNL_OP_GET_REG_LAYOUT_V0;
info.resp_parm = resp_buffer;
info.resp_parm_len = sizeof(resp_buffer);
ret = irdma_vchnl_req_send_sync(dev, &info);
if (ret)
return ret;
/* parse the response buffer and update reg info*/
/* Parse registers till invalid */
/* Parse register fields till invalid */
reg_array = (struct irdma_vchnl_reg_info *)resp_buffer;
for (rindex = 0; rindex < IRDMA_VCHNL_REG_COUNT; rindex++) {
buf_len += sizeof(struct irdma_vchnl_reg_info);
if (buf_len >= sizeof(resp_buffer))
return -ENOMEM;
regfld_array =
(struct irdma_vchnl_reg_field_info *)&reg_array[rindex + 1];
reg_id = reg_array[rindex].reg_id;
if (reg_id == IRDMA_VCHNL_REG_INV_ID)
break;
reg_id &= ~IRDMA_VCHNL_REG_PAGE_REL;
if (reg_id >= IRDMA_VCHNL_REG_COUNT)
return -EINVAL;
/* search regmap for register index in hw_regs.*/
reg_map_array = vchnl_reg_map;
do {
tmp_reg_id = reg_map_array->reg_id;
if (tmp_reg_id == reg_id)
break;
reg_map_array++;
} while (tmp_reg_id != IRDMA_VCHNL_REG_INV_ID);
if (tmp_reg_id != reg_id)
continue;
reg_idx = reg_map_array->reg_idx;
/* Page relative, DB Offset do not need bar offset */
if (reg_idx == IRDMA_DB_ADDR_OFFSET ||
(reg_array[rindex].reg_id & IRDMA_VCHNL_REG_PAGE_REL)) {
dev->hw_regs[reg_idx] =
(u32 __iomem *)(uintptr_t)reg_array[rindex].reg_offset;
continue;
}
/* Update the local HW struct */
dev->hw_regs[reg_idx] = ig3rdma_get_reg_addr(dev->hw,
reg_array[rindex].reg_offset);
if (!dev->hw_regs[reg_idx])
return -EINVAL;
}
if (!regfld_array)
return -ENOMEM;
/* set up doorbell variables using mapped DB page */
dev->wqe_alloc_db = dev->hw_regs[IRDMA_WQEALLOC];
dev->cq_arm_db = dev->hw_regs[IRDMA_CQARM];
dev->aeq_alloc_db = dev->hw_regs[IRDMA_AEQALLOC];
dev->cqp_db = dev->hw_regs[IRDMA_CQPDB];
dev->cq_ack_db = dev->hw_regs[IRDMA_CQACK];
for (rindex = 0; rindex < IRDMA_VCHNL_REGFLD_COUNT; rindex++) {
buf_len += sizeof(struct irdma_vchnl_reg_field_info);
if ((buf_len - 1) > sizeof(resp_buffer))
break;
if (regfld_array[rindex].fld_id == IRDMA_VCHNL_REGFLD_INV_ID)
break;
regfld_id = regfld_array[rindex].fld_id;
regfld_map_array = vchnl_regfld_map;
do {
tmp_regfld_id = regfld_map_array->regfld_id;
if (tmp_regfld_id == regfld_id)
break;
regfld_map_array++;
} while (tmp_regfld_id != IRDMA_VCHNL_REGFLD_INV_ID);
if (tmp_regfld_id != regfld_id)
continue;
regfld_idx = regfld_map_array->regfld_idx;
num_bits = regfld_array[rindex].fld_bits;
shift_cnt = regfld_array[rindex].fld_shift;
if ((num_bits + shift_cnt > 64) || !num_bits) {
ibdev_dbg(to_ibdev(dev),
"ERR: Invalid field mask id %d bits %d shift %d",
regfld_id, num_bits, shift_cnt);
continue;
}
bitmask = (1ULL << num_bits) - 1;
dev->hw_masks[regfld_idx] = bitmask << shift_cnt;
dev->hw_shifts[regfld_idx] = shift_cnt;
}
return 0;
}
int irdma_vchnl_req_add_vport(struct irdma_sc_dev *dev, u16 vport_id,
u32 qp1_id, struct irdma_qos *qos)
{
struct irdma_vchnl_resp_vport_info resp_vport = { 0 };
struct irdma_vchnl_req_vport_info req_vport = { 0 };
struct irdma_vchnl_req_init_info info = { 0 };
int ret, i;
if (!dev->vchnl_up)
return -EBUSY;
info.op_code = IRDMA_VCHNL_OP_ADD_VPORT;
info.op_ver = IRDMA_VCHNL_OP_ADD_VPORT_V0;
req_vport.vport_id = vport_id;
req_vport.qp1_id = qp1_id;
info.req_parm_len = sizeof(req_vport);
info.req_parm = &req_vport;
info.resp_parm = &resp_vport;
info.resp_parm_len = sizeof(resp_vport);
ret = irdma_vchnl_req_send_sync(dev, &info);
if (ret)
return ret;
for (i = 0; i < IRDMA_MAX_USER_PRIORITY; i++) {
qos[i].qs_handle = resp_vport.qs_handle[i];
qos[i].valid = true;
}
return 0;
}
int irdma_vchnl_req_del_vport(struct irdma_sc_dev *dev, u16 vport_id, u32 qp1_id)
{
struct irdma_vchnl_req_init_info info = { 0 };
struct irdma_vchnl_req_vport_info req_vport = { 0 };
if (!dev->vchnl_up)
return -EBUSY;
info.op_code = IRDMA_VCHNL_OP_DEL_VPORT;
info.op_ver = IRDMA_VCHNL_OP_DEL_VPORT_V0;
req_vport.vport_id = vport_id;
req_vport.qp1_id = qp1_id;
info.req_parm_len = sizeof(req_vport);
info.req_parm = &req_vport;
return irdma_vchnl_req_send_sync(dev, &info);
}
/**
* irdma_vchnl_req_aeq_vec_map - Map AEQ to vector on this function
* @dev: RDMA device pointer
* @v_idx: vector index
*/
int irdma_vchnl_req_aeq_vec_map(struct irdma_sc_dev *dev, u32 v_idx)
{
struct irdma_vchnl_req_init_info info = {};
struct irdma_vchnl_qvlist_info *qvl;
struct irdma_vchnl_qv_info *qv;
u16 qvl_size, num_vectors = 1;
int ret;
if (!dev->vchnl_up)
return -EBUSY;
qvl_size = struct_size(qvl, qv_info, num_vectors);
qvl = kzalloc(qvl_size, GFP_KERNEL);
if (!qvl)
return -ENOMEM;
qvl->num_vectors = 1;
qv = qvl->qv_info;
qv->ceq_idx = IRDMA_Q_INVALID_IDX;
qv->v_idx = v_idx;
qv->itr_idx = IRDMA_IDX_ITR0;
info.op_code = IRDMA_VCHNL_OP_QUEUE_VECTOR_MAP;
info.op_ver = IRDMA_VCHNL_OP_QUEUE_VECTOR_MAP_V0;
info.req_parm = qvl;
info.req_parm_len = qvl_size;
ret = irdma_vchnl_req_send_sync(dev, &info);
kfree(qvl);
return ret;
}
/**
* irdma_vchnl_req_ceq_vec_map - Map CEQ to vector on this function
* @dev: RDMA device pointer
* @ceq_id: CEQ index
* @v_idx: vector index
*/
int irdma_vchnl_req_ceq_vec_map(struct irdma_sc_dev *dev, u16 ceq_id, u32 v_idx)
{
struct irdma_vchnl_req_init_info info = {};
struct irdma_vchnl_qvlist_info *qvl;
struct irdma_vchnl_qv_info *qv;
u16 qvl_size, num_vectors = 1;
int ret;
if (!dev->vchnl_up)
return -EBUSY;
qvl_size = struct_size(qvl, qv_info, num_vectors);
qvl = kzalloc(qvl_size, GFP_KERNEL);
if (!qvl)
return -ENOMEM;
qvl->num_vectors = num_vectors;
qv = qvl->qv_info;
qv->aeq_idx = IRDMA_Q_INVALID_IDX;
qv->ceq_idx = ceq_id;
qv->v_idx = v_idx;
qv->itr_idx = IRDMA_IDX_ITR0;
info.op_code = IRDMA_VCHNL_OP_QUEUE_VECTOR_MAP;
info.op_ver = IRDMA_VCHNL_OP_QUEUE_VECTOR_MAP_V0;
info.req_parm = qvl;
info.req_parm_len = qvl_size;
ret = irdma_vchnl_req_send_sync(dev, &info);
kfree(qvl);
return ret;
}
/**
* irdma_vchnl_req_get_ver - Request Channel version
* @dev: RDMA device pointer
* @ver_req: Virtual channel version requested
* @ver_res: Virtual channel version response
*/
int irdma_vchnl_req_get_ver(struct irdma_sc_dev *dev, u16 ver_req, u32 *ver_res)
{
struct irdma_vchnl_req_init_info info = {};
int ret;
if (!dev->vchnl_up)
return -EBUSY;
info.op_code = IRDMA_VCHNL_OP_GET_VER;
info.op_ver = ver_req;
info.resp_parm = ver_res;
info.resp_parm_len = sizeof(*ver_res);
ret = irdma_vchnl_req_send_sync(dev, &info);
if (ret)
return ret;
if (*ver_res < IRDMA_VCHNL_CHNL_VER_MIN) {
ibdev_dbg(to_ibdev(dev),
"VIRT: %s unsupported vchnl version 0x%0x\n",
__func__, *ver_res);
return -EOPNOTSUPP;
}
return 0;
}
/**
* irdma_vchnl_req_get_hmc_fcn - Request VF HMC Function
* @dev: RDMA device pointer
*/
int irdma_vchnl_req_get_hmc_fcn(struct irdma_sc_dev *dev)
{
struct irdma_vchnl_req_hmc_info req_hmc = {};
struct irdma_vchnl_resp_hmc_info resp_hmc = {};
struct irdma_vchnl_req_init_info info = {};
int ret;
if (!dev->vchnl_up)
return -EBUSY;
info.op_code = IRDMA_VCHNL_OP_GET_HMC_FCN;
if (dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_3) {
info.op_ver = IRDMA_VCHNL_OP_GET_HMC_FCN_V2;
req_hmc.protocol_used = dev->protocol_used;
info.req_parm_len = sizeof(req_hmc);
info.req_parm = &req_hmc;
info.resp_parm = &resp_hmc;
info.resp_parm_len = sizeof(resp_hmc);
}
ret = irdma_vchnl_req_send_sync(dev, &info);
if (ret)
return ret;
if (dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_3) {
int i;
dev->hmc_fn_id = resp_hmc.hmc_func;
for (i = 0; i < IRDMA_MAX_USER_PRIORITY; i++) {
dev->qos[i].qs_handle = resp_hmc.qs_handle[i];
dev->qos[i].valid = true;
}
}
return 0;
}
/**
* irdma_vchnl_req_put_hmc_fcn - Free VF HMC Function
* @dev: RDMA device pointer
*/
int irdma_vchnl_req_put_hmc_fcn(struct irdma_sc_dev *dev)
{
struct irdma_vchnl_req_init_info info = {};
if (!dev->vchnl_up)
return -EBUSY;
info.op_code = IRDMA_VCHNL_OP_PUT_HMC_FCN;
info.op_ver = IRDMA_VCHNL_OP_PUT_HMC_FCN_V0;
return irdma_vchnl_req_send_sync(dev, &info);
}
/**
* irdma_vchnl_req_get_caps - Request RDMA capabilities
* @dev: RDMA device pointer
*/
int irdma_vchnl_req_get_caps(struct irdma_sc_dev *dev)
{
struct irdma_vchnl_req_init_info info = {};
int ret;
if (!dev->vchnl_up)
return -EBUSY;
info.op_code = IRDMA_VCHNL_OP_GET_RDMA_CAPS;
info.op_ver = IRDMA_VCHNL_OP_GET_RDMA_CAPS_V0;
info.resp_parm = &dev->vc_caps;
info.resp_parm_len = sizeof(dev->vc_caps);
ret = irdma_vchnl_req_send_sync(dev, &info);
if (ret)
return ret;
if (dev->vc_caps.hw_rev > IRDMA_GEN_MAX ||
dev->vc_caps.hw_rev < IRDMA_GEN_2) {
ibdev_dbg(to_ibdev(dev),
"ERR: %s unsupported hw_rev version 0x%0x\n",
__func__, dev->vc_caps.hw_rev);
return -EOPNOTSUPP;
}
return 0;
}
/**
* irdma_vchnl_req_get_resp - Receive the inbound vchnl response.
* @dev: Dev pointer
* @vchnl_req: Vchannel request
*/
int irdma_vchnl_req_get_resp(struct irdma_sc_dev *dev,
struct irdma_vchnl_req *vchnl_req)
{
struct irdma_vchnl_resp_buf *vchnl_msg_resp =
(struct irdma_vchnl_resp_buf *)dev->vc_recv_buf;
u16 resp_len;
int ret;
if ((uintptr_t)vchnl_req != (uintptr_t)vchnl_msg_resp->op_ctx) {
ibdev_dbg(to_ibdev(dev),
"VIRT: error vchnl context value does not match\n");
return -EBADMSG;
}
resp_len = dev->vc_recv_len - sizeof(*vchnl_msg_resp);
resp_len = min(resp_len, vchnl_req->parm_len);
ret = irdma_vchnl_req_verify_resp(vchnl_req, resp_len);
if (ret)
return ret;
ret = (int)vchnl_msg_resp->op_ret;
if (ret)
return ret;
vchnl_req->resp_len = 0;
if (vchnl_req->parm_len && vchnl_req->parm && resp_len) {
memcpy(vchnl_req->parm, vchnl_msg_resp->buf, resp_len);
vchnl_req->resp_len = resp_len;
ibdev_dbg(to_ibdev(dev), "VIRT: Got response, data size %u\n",
resp_len);
}
return 0;
}
linux-7.2.3/drivers/infiniband/hw/irdma/virtchnl.h
+176
View File
@@ -0,0 +1,176 @@
/* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB */
/* Copyright (c) 2015 - 2024 Intel Corporation */
#ifndef IRDMA_VIRTCHNL_H
#define IRDMA_VIRTCHNL_H
#include "hmc.h"
#include "irdma.h"
/* IRDMA_VCHNL_CHNL_VER_V0 is for legacy hw, no longer supported. */
#define IRDMA_VCHNL_CHNL_VER_V2 2
#define IRDMA_VCHNL_CHNL_VER_MIN IRDMA_VCHNL_CHNL_VER_V2
#define IRDMA_VCHNL_CHNL_VER_MAX IRDMA_VCHNL_CHNL_VER_V2
#define IRDMA_VCHNL_OP_GET_HMC_FCN_V0 0
#define IRDMA_VCHNL_OP_GET_HMC_FCN_V1 1
#define IRDMA_VCHNL_OP_GET_HMC_FCN_V2 2
#define IRDMA_VCHNL_OP_PUT_HMC_FCN_V0 0
#define IRDMA_VCHNL_OP_GET_REG_LAYOUT_V0 0
#define IRDMA_VCHNL_OP_QUEUE_VECTOR_MAP_V0 0
#define IRDMA_VCHNL_OP_QUEUE_VECTOR_UNMAP_V0 0
#define IRDMA_VCHNL_OP_ADD_VPORT_V0 0
#define IRDMA_VCHNL_OP_DEL_VPORT_V0 0
#define IRDMA_VCHNL_OP_GET_RDMA_CAPS_V0 0
#define IRDMA_VCHNL_OP_GET_RDMA_CAPS_MIN_SIZE 1
#define IRDMA_VCHNL_REG_ID_CQPTAIL 0
#define IRDMA_VCHNL_REG_ID_CQPDB 1
#define IRDMA_VCHNL_REG_ID_CCQPSTATUS 2
#define IRDMA_VCHNL_REG_ID_CCQPHIGH 3
#define IRDMA_VCHNL_REG_ID_CCQPLOW 4
#define IRDMA_VCHNL_REG_ID_CQARM 5
#define IRDMA_VCHNL_REG_ID_CQACK 6
#define IRDMA_VCHNL_REG_ID_AEQALLOC 7
#define IRDMA_VCHNL_REG_ID_CQPERRCODES 8
#define IRDMA_VCHNL_REG_ID_WQEALLOC 9
#define IRDMA_VCHNL_REG_ID_IPCONFIG0 10
#define IRDMA_VCHNL_REG_ID_DB_ADDR_OFFSET 11
#define IRDMA_VCHNL_REG_ID_DYN_CTL 12
#define IRDMA_VCHNL_REG_ID_AEQITRMASK 13
#define IRDMA_VCHNL_REG_ID_CEQITRMASK 14
#define IRDMA_VCHNL_REG_INV_ID 0xFFFF
#define IRDMA_VCHNL_REG_PAGE_REL 0x8000
#define IRDMA_VCHNL_REGFLD_ID_CCQPSTATUS_CQP_OP_ERR 2
#define IRDMA_VCHNL_REGFLD_ID_CCQPSTATUS_CCQP_DONE 5
#define IRDMA_VCHNL_REGFLD_ID_CQPSQ_STAG_PDID 6
#define IRDMA_VCHNL_REGFLD_ID_CQPSQ_CQ_CEQID 7
#define IRDMA_VCHNL_REGFLD_ID_CQPSQ_CQ_CQID 8
#define IRDMA_VCHNL_REGFLD_ID_COMMIT_FPM_CQCNT 9
#define IRDMA_VCHNL_REGFLD_ID_UPESD_HMCN_ID 10
#define IRDMA_VCHNL_REGFLD_INV_ID 0xFFFF
#define IRDMA_VCHNL_RESP_MIN_SIZE (sizeof(struct irdma_vchnl_resp_buf))
enum irdma_vchnl_ops {
IRDMA_VCHNL_OP_GET_VER = 0,
IRDMA_VCHNL_OP_GET_HMC_FCN = 1,
IRDMA_VCHNL_OP_PUT_HMC_FCN = 2,
IRDMA_VCHNL_OP_GET_REG_LAYOUT = 11,
IRDMA_VCHNL_OP_GET_RDMA_CAPS = 13,
IRDMA_VCHNL_OP_QUEUE_VECTOR_MAP = 14,
IRDMA_VCHNL_OP_QUEUE_VECTOR_UNMAP = 15,
IRDMA_VCHNL_OP_ADD_VPORT = 16,
IRDMA_VCHNL_OP_DEL_VPORT = 17,
};
struct irdma_vchnl_req_hmc_info {
u8 protocol_used;
u8 disable_qos;
} __packed;
struct irdma_vchnl_resp_hmc_info {
u16 hmc_func;
u16 qs_handle[IRDMA_MAX_USER_PRIORITY];
} __packed;
struct irdma_vchnl_qv_info {
u32 v_idx;
u16 ceq_idx;
u16 aeq_idx;
u8 itr_idx;
};
struct irdma_vchnl_qvlist_info {
u32 num_vectors;
struct irdma_vchnl_qv_info qv_info[];
};
struct irdma_vchnl_req_vport_info {
u16 vport_id;
u32 qp1_id;
};
struct irdma_vchnl_resp_vport_info {
u16 qs_handle[IRDMA_MAX_USER_PRIORITY];
};
struct irdma_vchnl_op_buf {
u16 op_code;
u16 op_ver;
u16 buf_len;
u16 rsvd;
u64 op_ctx;
u8 buf[];
} __packed;
struct irdma_vchnl_resp_buf {
u64 op_ctx;
u16 buf_len;
s16 op_ret;
u16 rsvd[2];
u8 buf[];
} __packed;
struct irdma_vchnl_rdma_caps {
u8 hw_rev;
u16 cqp_timeout_s;
u16 cqp_def_timeout_s;
u16 max_hw_push_len;
} __packed;
struct irdma_vchnl_init_info {
struct workqueue_struct *vchnl_wq;
enum irdma_vers hw_rev;
bool privileged;
bool is_pf;
};
struct irdma_vchnl_reg_info {
u32 reg_offset;
u16 field_cnt;
u16 reg_id; /* High bit of reg_id: bar or page relative */
};
struct irdma_vchnl_reg_field_info {
u8 fld_shift;
u8 fld_bits;
u16 fld_id;
};
struct irdma_vchnl_req {
struct irdma_vchnl_op_buf *vchnl_msg;
void *parm;
u32 vf_id;
u16 parm_len;
u16 resp_len;
};
struct irdma_vchnl_req_init_info {
void *req_parm;
void *resp_parm;
u16 req_parm_len;
u16 resp_parm_len;
u16 op_code;
u16 op_ver;
} __packed;
struct irdma_qos;
int irdma_sc_vchnl_init(struct irdma_sc_dev *dev,
struct irdma_vchnl_init_info *info);
int irdma_vchnl_req_get_ver(struct irdma_sc_dev *dev, u16 ver_req,
u32 *ver_res);
int irdma_vchnl_req_get_hmc_fcn(struct irdma_sc_dev *dev);
int irdma_vchnl_req_put_hmc_fcn(struct irdma_sc_dev *dev);
int irdma_vchnl_req_get_caps(struct irdma_sc_dev *dev);
int irdma_vchnl_req_get_resp(struct irdma_sc_dev *dev,
struct irdma_vchnl_req *vc_req);
int irdma_vchnl_req_get_reg_layout(struct irdma_sc_dev *dev);
int irdma_vchnl_req_aeq_vec_map(struct irdma_sc_dev *dev, u32 v_idx);
int irdma_vchnl_req_ceq_vec_map(struct irdma_sc_dev *dev, u16 ceq_id,
u32 v_idx);
int irdma_vchnl_req_add_vport(struct irdma_sc_dev *dev, u16 vport_id,
u32 qp1_id, struct irdma_qos *qos);
int irdma_vchnl_req_del_vport(struct irdma_sc_dev *dev, u16 vport_id,
u32 qp1_id);
#endif /* IRDMA_VIRTCHNL_H */
linux-7.2.3/drivers/infiniband/hw/irdma/ws.c
+406
View File
@@ -0,0 +1,406 @@
// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
/* Copyright (c) 2017 - 2021 Intel Corporation */
#include "osdep.h"
#include "hmc.h"
#include "defs.h"
#include "type.h"
#include "protos.h"
#include "ws.h"
/**
* irdma_alloc_node - Allocate a WS node and init
* @vsi: vsi pointer
* @user_pri: user priority
* @node_type: Type of node, leaf or parent
* @parent: parent node pointer
*/
static struct irdma_ws_node *irdma_alloc_node(struct irdma_sc_vsi *vsi,
u8 user_pri,
enum irdma_ws_node_type node_type,
struct irdma_ws_node *parent)
{
struct irdma_virt_mem ws_mem;
struct irdma_ws_node *node;
u16 node_index = 0;
ws_mem.size = sizeof(struct irdma_ws_node);
ws_mem.va = kzalloc(ws_mem.size, GFP_KERNEL);
if (!ws_mem.va)
return NULL;
if (parent) {
node_index = irdma_alloc_ws_node_id(vsi->dev);
if (node_index == IRDMA_WS_NODE_INVALID) {
kfree(ws_mem.va);
return NULL;
}
}
node = ws_mem.va;
node->index = node_index;
node->vsi_index = vsi->vsi_idx;
INIT_LIST_HEAD(&node->child_list_head);
if (node_type == WS_NODE_TYPE_LEAF) {
node->type_leaf = true;
node->traffic_class = vsi->qos[user_pri].traffic_class;
node->user_pri = user_pri;
node->rel_bw = vsi->qos[user_pri].rel_bw;
if (!node->rel_bw)
node->rel_bw = 1;
node->lan_qs_handle = vsi->qos[user_pri].lan_qos_handle;
node->prio_type = IRDMA_PRIO_WEIGHTED_RR;
} else {
node->rel_bw = 1;
node->prio_type = IRDMA_PRIO_WEIGHTED_RR;
node->enable = true;
}
node->parent = parent;
return node;
}
/**
* irdma_free_node - Free a WS node
* @vsi: VSI stricture of device
* @node: Pointer to node to free
*/
static void irdma_free_node(struct irdma_sc_vsi *vsi,
struct irdma_ws_node *node)
{
struct irdma_virt_mem ws_mem;
if (node->index)
irdma_free_ws_node_id(vsi->dev, node->index);
ws_mem.va = node;
ws_mem.size = sizeof(struct irdma_ws_node);
kfree(ws_mem.va);
}
/**
* irdma_ws_cqp_cmd - Post CQP work scheduler node cmd
* @vsi: vsi pointer
* @node: pointer to node
* @cmd: add, remove or modify
*/
static int irdma_ws_cqp_cmd(struct irdma_sc_vsi *vsi,
struct irdma_ws_node *node, u8 cmd)
{
struct irdma_ws_node_info node_info = {};
node_info.id = node->index;
node_info.vsi = node->vsi_index;
if (node->parent)
node_info.parent_id = node->parent->index;
else
node_info.parent_id = node_info.id;
node_info.weight = node->rel_bw;
node_info.tc = node->traffic_class;
node_info.prio_type = node->prio_type;
node_info.type_leaf = node->type_leaf;
node_info.enable = node->enable;
if (irdma_cqp_ws_node_cmd(vsi->dev, cmd, &node_info)) {
ibdev_dbg(to_ibdev(vsi->dev), "WS: CQP WS CMD failed\n");
return -ENOMEM;
}
if (node->type_leaf && cmd == IRDMA_OP_WS_ADD_NODE) {
node->qs_handle = node_info.qs_handle;
vsi->qos[node->user_pri].qs_handle = node_info.qs_handle;
}
return 0;
}
/**
* ws_find_node - Find SC WS node based on VSI id or TC
* @parent: parent node of First VSI or TC node
* @match_val: value to match
* @type: match type VSI/TC
*/
static struct irdma_ws_node *ws_find_node(struct irdma_ws_node *parent,
u16 match_val,
enum irdma_ws_match_type type)
{
struct irdma_ws_node *node;
switch (type) {
case WS_MATCH_TYPE_VSI:
list_for_each_entry(node, &parent->child_list_head, siblings) {
if (node->vsi_index == match_val)
return node;
}
break;
case WS_MATCH_TYPE_TC:
list_for_each_entry(node, &parent->child_list_head, siblings) {
if (node->traffic_class == match_val)
return node;
}
break;
default:
break;
}
return NULL;
}
/**
* irdma_tc_in_use - Checks to see if a leaf node is in use
* @vsi: vsi pointer
* @user_pri: user priority
*/
static bool irdma_tc_in_use(struct irdma_sc_vsi *vsi, u8 user_pri)
{
int i;
mutex_lock(&vsi->qos[user_pri].qos_mutex);
if (!list_empty(&vsi->qos[user_pri].qplist)) {
mutex_unlock(&vsi->qos[user_pri].qos_mutex);
return true;
}
/* Check if the traffic class associated with the given user priority
* is in use by any other user priority. If so, nothing left to do
*/
for (i = 0; i < IRDMA_MAX_USER_PRIORITY; i++) {
if (vsi->qos[i].traffic_class == vsi->qos[user_pri].traffic_class &&
!list_empty(&vsi->qos[i].qplist)) {
mutex_unlock(&vsi->qos[user_pri].qos_mutex);
return true;
}
}
mutex_unlock(&vsi->qos[user_pri].qos_mutex);
return false;
}
/**
* irdma_remove_leaf - Remove leaf node unconditionally
* @vsi: vsi pointer
* @user_pri: user priority
*/
static void irdma_remove_leaf(struct irdma_sc_vsi *vsi, u8 user_pri)
{
struct irdma_ws_node *ws_tree_root, *vsi_node, *tc_node;
int i;
u16 traffic_class;
traffic_class = vsi->qos[user_pri].traffic_class;
for (i = 0; i < IRDMA_MAX_USER_PRIORITY; i++)
if (vsi->qos[i].traffic_class == traffic_class)
vsi->qos[i].valid = false;
ws_tree_root = vsi->dev->ws_tree_root;
if (!ws_tree_root)
return;
vsi_node = ws_find_node(ws_tree_root, vsi->vsi_idx,
WS_MATCH_TYPE_VSI);
if (!vsi_node)
return;
tc_node = ws_find_node(vsi_node,
vsi->qos[user_pri].traffic_class,
WS_MATCH_TYPE_TC);
if (!tc_node)
return;
irdma_ws_cqp_cmd(vsi, tc_node, IRDMA_OP_WS_DELETE_NODE);
vsi->unregister_qset(vsi, tc_node);
list_del(&tc_node->siblings);
irdma_free_node(vsi, tc_node);
/* Check if VSI node can be freed */
if (list_empty(&vsi_node->child_list_head)) {
irdma_ws_cqp_cmd(vsi, vsi_node, IRDMA_OP_WS_DELETE_NODE);
list_del(&vsi_node->siblings);
irdma_free_node(vsi, vsi_node);
/* Free head node there are no remaining VSI nodes */
if (list_empty(&ws_tree_root->child_list_head)) {
irdma_ws_cqp_cmd(vsi, ws_tree_root,
IRDMA_OP_WS_DELETE_NODE);
irdma_free_node(vsi, ws_tree_root);
vsi->dev->ws_tree_root = NULL;
}
}
}
/**
* irdma_ws_add - Build work scheduler tree, set RDMA qs_handle
* @vsi: vsi pointer
* @user_pri: user priority
*/
int irdma_ws_add(struct irdma_sc_vsi *vsi, u8 user_pri)
{
struct irdma_ws_node *ws_tree_root;
struct irdma_ws_node *vsi_node;
struct irdma_ws_node *tc_node;
u16 traffic_class;
int ret = 0;
int i;
mutex_lock(&vsi->dev->ws_mutex);
if (vsi->tc_change_pending) {
ret = -EBUSY;
goto exit;
}
if (vsi->qos[user_pri].valid)
goto exit;
ws_tree_root = vsi->dev->ws_tree_root;
if (!ws_tree_root) {
ibdev_dbg(to_ibdev(vsi->dev), "WS: Creating root node\n");
ws_tree_root = irdma_alloc_node(vsi, user_pri,
WS_NODE_TYPE_PARENT, NULL);
if (!ws_tree_root) {
ret = -ENOMEM;
goto exit;
}
ret = irdma_ws_cqp_cmd(vsi, ws_tree_root, IRDMA_OP_WS_ADD_NODE);
if (ret) {
irdma_free_node(vsi, ws_tree_root);
goto exit;
}
vsi->dev->ws_tree_root = ws_tree_root;
}
/* Find a second tier node that matches the VSI */
vsi_node = ws_find_node(ws_tree_root, vsi->vsi_idx,
WS_MATCH_TYPE_VSI);
/* If VSI node doesn't exist, add one */
if (!vsi_node) {
ibdev_dbg(to_ibdev(vsi->dev),
"WS: Node not found matching VSI %d\n",
vsi->vsi_idx);
vsi_node = irdma_alloc_node(vsi, user_pri, WS_NODE_TYPE_PARENT,
ws_tree_root);
if (!vsi_node) {
ret = -ENOMEM;
goto vsi_add_err;
}
ret = irdma_ws_cqp_cmd(vsi, vsi_node, IRDMA_OP_WS_ADD_NODE);
if (ret) {
irdma_free_node(vsi, vsi_node);
goto vsi_add_err;
}
list_add(&vsi_node->siblings, &ws_tree_root->child_list_head);
}
ibdev_dbg(to_ibdev(vsi->dev),
"WS: Using node %d which represents VSI %d\n",
vsi_node->index, vsi->vsi_idx);
traffic_class = vsi->qos[user_pri].traffic_class;
tc_node = ws_find_node(vsi_node, traffic_class,
WS_MATCH_TYPE_TC);
if (!tc_node) {
/* Add leaf node */
ibdev_dbg(to_ibdev(vsi->dev),
"WS: Node not found matching VSI %d and TC %d\n",
vsi->vsi_idx, traffic_class);
tc_node = irdma_alloc_node(vsi, user_pri, WS_NODE_TYPE_LEAF,
vsi_node);
if (!tc_node) {
ret = -ENOMEM;
goto leaf_add_err;
}
ret = irdma_ws_cqp_cmd(vsi, tc_node, IRDMA_OP_WS_ADD_NODE);
if (ret) {
irdma_free_node(vsi, tc_node);
goto leaf_add_err;
}
list_add(&tc_node->siblings, &vsi_node->child_list_head);
/*
* callback to LAN to update the LAN tree with our node
*/
ret = vsi->register_qset(vsi, tc_node);
if (ret)
goto reg_err;
tc_node->enable = true;
ret = irdma_ws_cqp_cmd(vsi, tc_node, IRDMA_OP_WS_MODIFY_NODE);
if (ret) {
vsi->unregister_qset(vsi, tc_node);
goto reg_err;
}
}
ibdev_dbg(to_ibdev(vsi->dev),
"WS: Using node %d which represents VSI %d TC %d\n",
tc_node->index, vsi->vsi_idx, traffic_class);
/*
* Iterate through other UPs and update the QS handle if they have
* a matching traffic class.
*/
for (i = 0; i < IRDMA_MAX_USER_PRIORITY; i++) {
if (vsi->qos[i].traffic_class == traffic_class) {
vsi->qos[i].qs_handle = tc_node->qs_handle;
vsi->qos[i].lan_qos_handle = tc_node->lan_qs_handle;
vsi->qos[i].l2_sched_node_id = tc_node->l2_sched_node_id;
vsi->qos[i].valid = true;
}
}
goto exit;
reg_err:
irdma_ws_cqp_cmd(vsi, tc_node, IRDMA_OP_WS_DELETE_NODE);
list_del(&tc_node->siblings);
irdma_free_node(vsi, tc_node);
leaf_add_err:
if (list_empty(&vsi_node->child_list_head)) {
if (irdma_ws_cqp_cmd(vsi, vsi_node, IRDMA_OP_WS_DELETE_NODE))
goto exit;
list_del(&vsi_node->siblings);
irdma_free_node(vsi, vsi_node);
}
vsi_add_err:
/* Free head node there are no remaining VSI nodes */
if (list_empty(&ws_tree_root->child_list_head)) {
irdma_ws_cqp_cmd(vsi, ws_tree_root, IRDMA_OP_WS_DELETE_NODE);
vsi->dev->ws_tree_root = NULL;
irdma_free_node(vsi, ws_tree_root);
}
exit:
mutex_unlock(&vsi->dev->ws_mutex);
return ret;
}
/**
* irdma_ws_remove - Free WS scheduler node, update WS tree
* @vsi: vsi pointer
* @user_pri: user priority
*/
void irdma_ws_remove(struct irdma_sc_vsi *vsi, u8 user_pri)
{
mutex_lock(&vsi->dev->ws_mutex);
if (irdma_tc_in_use(vsi, user_pri))
goto exit;
irdma_remove_leaf(vsi, user_pri);
exit:
mutex_unlock(&vsi->dev->ws_mutex);
}
/**
* irdma_ws_reset - Reset entire WS tree
* @vsi: vsi pointer
*/
void irdma_ws_reset(struct irdma_sc_vsi *vsi)
{
u8 i;
mutex_lock(&vsi->dev->ws_mutex);
for (i = 0; i < IRDMA_MAX_USER_PRIORITY; ++i)
irdma_remove_leaf(vsi, i);
mutex_unlock(&vsi->dev->ws_mutex);
}
linux-7.2.3/drivers/infiniband/hw/irdma/ws.h
+41
View File
@@ -0,0 +1,41 @@
/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
/* Copyright (c) 2015 - 2020 Intel Corporation */
#ifndef IRDMA_WS_H
#define IRDMA_WS_H
#include "osdep.h"
enum irdma_ws_node_type {
WS_NODE_TYPE_PARENT,
WS_NODE_TYPE_LEAF,
};
enum irdma_ws_match_type {
WS_MATCH_TYPE_VSI,
WS_MATCH_TYPE_TC,
};
struct irdma_ws_node {
struct list_head siblings;
struct list_head child_list_head;
struct irdma_ws_node *parent;
u64 lan_qs_handle; /* opaque handle used by LAN */
u32 l2_sched_node_id;
u16 index;
u16 qs_handle;
u16 vsi_index;
u8 traffic_class;
u8 user_pri;
u8 rel_bw;
u8 abstraction_layer; /* used for splitting a TC */
u8 prio_type;
bool type_leaf:1;
bool enable:1;
};
struct irdma_sc_vsi;
int irdma_ws_add(struct irdma_sc_vsi *vsi, u8 user_pri);
void irdma_ws_remove(struct irdma_sc_vsi *vsi, u8 user_pri);
void irdma_ws_reset(struct irdma_sc_vsi *vsi);
#endif /* IRDMA_WS_H */