RDMA ibverbs_API功能说明

设备管理

获取当前活动网卡

返回当前rdma设备列表

struct ibv_device **ibv_get_device_list(int *num_devices);

//使用
struct ibv_device **dev_list = ibv_get_device_list(NULL);

获取网卡名

返回网卡名字字符串：如"mlx5_0"，一般通过网卡名字确定将要使用的网卡

const char *ibv_get_device_name(struct ibv_device *device);

打开网卡

初始化一个网卡，返回该网卡的ibv_context上下文，后续对于该网卡的操作都基于这个上下文。

//初始化使用网卡
struct ibv_context *ibv_open_device(struct ibv_device *device);

//释放网卡
int ibv_close_device(struct ibv_context *context);

资源初始化

所有基于网卡的配置，都是以该网卡的ibv_context上下文进行操作

申请PD保护域

PD保护域用来注册内存区域(MR)、创建队列等

struct ibv_pd *ibv_alloc_pd(struct ibv_context *context);

注册内存区域

注册可以本地/远程读写的内存区域

这个内存区域归属于PD保护域

struct ibv_mr *ibv_reg_mr(struct ibv_pd *pd, void *addr, size_t length, int access);

enum ibv_access_flags {
	IBV_ACCESS_LOCAL_WRITE		= 1,
	IBV_ACCESS_REMOTE_WRITE		= (1<<1),
	IBV_ACCESS_REMOTE_READ		= (1<<2),
	IBV_ACCESS_REMOTE_ATOMIC	= (1<<3),
	IBV_ACCESS_MW_BIND		= (1<<4),
	IBV_ACCESS_ZERO_BASED		= (1<<5),
	IBV_ACCESS_ON_DEMAND		= (1<<6),
	IBV_ACCESS_HUGETLB		= (1<<7),
	IBV_ACCESS_RELAXED_ORDERING	= IBV_ACCESS_OPTIONAL_FIRST,
};

参数addr, 本地申请好的内存地址， length空间大小， access内存读写flag

创建完成通道

完成通道用于存放完成队列，I/O可由event事件驱动

struct ibv_comp_channel *ibv_create_comp_channel(struct ibv_context *context);

创建完成队列

完成队列绑定到完成通道，

参数cqe是队列大小

/**
 * ibv_create_cq - Create a completion queue
 * @context - Context CQ will be attached to
 * @cqe - Minimum number of entries required for CQ
 * @cq_context - Consumer-supplied context returned for completion events
 * @channel - Completion channel where completion events will be queued.
 *     May be NULL if completion events will not be used.
 * @comp_vector - Completion vector used to signal completion events.
 *     Must be >= 0 and < context->num_comp_vectors.
 */
struct ibv_cq *ibv_create_cq(struct ibv_context *context, int cqe,
			     void *cq_context,
			     struct ibv_comp_channel *channel,
			     int comp_vector);

// 一般用法
struct ibv_cq *cq = ibv_create_cq(context, num, NULL, channel, 0);

//还有ibv_create_cq_ex扩展完成队列，具备更多功能 To Do.

创建QueuePair

根据qp_init_attr参数设置QP，返回ibv_qp指针。

struct ibv_qp *ibv_create_qp(struct ibv_pd *pd,
			     struct ibv_qp_init_attr *qp_init_attr);

// 查询QP属性，创建后可用其查询确认相关属性
int ibv_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
		 int attr_mask,
		 struct ibv_qp_init_attr *init_attr);

qp_init_attr属性

其中发送/接收完成队列提前设置为已创建的完成队列。

struct ibv_qp_init_attr {
	void		       *qp_context;  
	struct ibv_cq	       *send_cq;  // 发送完成队列
	struct ibv_cq	       *recv_cq;  // 接收完成队列
	struct ibv_srq	       *srq;      // 共享接收队列, 多个的QP共享一个队列
	struct ibv_qp_cap	cap;		  // QP容量属性
	enum ibv_qp_type	qp_type;      // QP类型，可靠RC，不可靠UC,UD等
	int			sq_sig_all;
};

struct ibv_qp_cap {
	uint32_t		max_send_wr;   //最大发送工作请求数量
	uint32_t		max_recv_wr;   //最大接收工作请求数量
	uint32_t		max_send_sge;  //一次发送最大的数据块数量
	uint32_t		max_recv_sge;  //一次接收最大的数据块数量
	uint32_t		max_inline_data; //内联数据大小，小数据和控制信令使用
};

收发API

建链

CM建链，使用CM VERBS，也是基于ibverbs 特殊的控制QP实现。

Socket建链，使用TCP/IP Socket建立链接

建链的本质是交换彼此的 LID, QPN，PSN，GID。

这个可以单独开一篇文档描述(To Do).

修改QueuePair

修改QP的相关状态属性

int ibv_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
		  int attr_mask);

在创建QP以后，通过该方法初始化QP

qp_state设置为INIT， pkey_index，access_flags设为0

struct ibv_qp_attr attr = {
    .qp_state        = IBV_QPS_INIT,
    .pkey_index      = 0,
    .port_num        = 1,
    .qp_access_flags = 0
};

if (ibv_modify_qp(ctx->qp, &attr,
                  IBV_QP_STATE              |
                  IBV_QP_PKEY_INDEX         |
                  IBV_QP_PORT               |
                  IBV_QP_ACCESS_FLAGS))

在建链交互之后，获取到对端QPN， PSN，LID, GID，通过该函数设置这些参数，建立QP连接。

第一步先设置远端参数，进入RTR状态

RTR状态以后可以接收数据

	struct ibv_qp_attr attr = {
		.qp_state		= IBV_QPS_RTR,
		.path_mtu		= mtu,
		.dest_qp_num		= dest_qpn,
		.rq_psn			= dest_psn,
		.max_dest_rd_atomic	= 1,
		.min_rnr_timer		= 12,
		.ah_attr		= {
			.is_global	= 0,
			.dlid		= dest_lid,
			.sl		= sl,
			.src_path_bits	= 0,
			.port_num	= port
		}
	};

	if (dest->gid.global.interface_id) {
		attr.ah_attr.is_global = 1;
		attr.ah_attr.grh.hop_limit = 1;
		attr.ah_attr.grh.dgid = dest->gid;
		attr.ah_attr.grh.sgid_index = sgid_idx;
	}
	if (ibv_modify_qp(ctx->qp, &attr,
			  IBV_QP_STATE              |
			  IBV_QP_AV                 |
			  IBV_QP_PATH_MTU           |
			  IBV_QP_DEST_QPN           |
			  IBV_QP_RQ_PSN             |
			  IBV_QP_MAX_DEST_RD_ATOMIC |
			  IBV_QP_MIN_RNR_TIMER)) {
		fprintf(stderr, "Failed to modify QP to RTR\n");
		return 1;
	}

第二步设置RTS状态

RTS状态后可以发送数据

	attr.qp_state	    = IBV_QPS_RTS;
	attr.timeout	    = 14;
	attr.retry_cnt	    = 7;
	attr.rnr_retry	    = 7;
	attr.sq_psn	    = my_psn;
	attr.max_rd_atomic  = 1;
	if (ibv_modify_qp(ctx->qp, &attr,
			  IBV_QP_STATE              |
			  IBV_QP_TIMEOUT            |
			  IBV_QP_RETRY_CNT          |
			  IBV_QP_RNR_RETRY          |
			  IBV_QP_SQ_PSN             |
			  IBV_QP_MAX_QP_RD_ATOMIC)) {
		fprintf(stderr, "Failed to modify QP to RTS\n");
		return 1;
	}

设置接收工作请求

将工作请求放入接收队列，用于接收数据

参数 wr是工作请求链表的头，每个工作请求包含接收内存区域，数据长度等

参数bad_wr用于返回发生错误的工作请求

/**
 * ibv_post_recv - Post a list of work requests to a receive queue.
 */
static inline int ibv_post_recv(struct ibv_qp *qp, struct ibv_recv_wr *wr,
				struct ibv_recv_wr **bad_wr)
{
	return qp->context->ops.post_recv(qp, wr, bad_wr);
}

用法示例：

// 数据块列表
struct ibv_sge list = {
    .addr	= buf,      // 接收数据内存地址
    .length = size,     // 数据大小
    .lkey	= mr->lkey  // 内存注册区域的lkey
};
// 接收工作请求实例
struct ibv_recv_wr wr = {
    .wr_id	    = 1,     //工作请求id
    .sg_list    = &list, //使用数据块列表
    .num_sge    = 1,     //列表中数据块个数
};
struct ibv_recv_wr *bad_wr; // 接收错误请求

// 一般可以重复执行ibv_post_recv，从而塞入一列工作请求
ibv_post_recv(qp, &wr, &bad_wr);

设置发送工作请求

设置一组工作请求wr，工作请求中包含传输数据块

/**
 * ibv_post_send - Post a list of work requests to a send queue.
 *
 * If IBV_SEND_INLINE flag is set, the data buffers can be reused
 * immediately after the call returns.
 */
static inline int ibv_post_send(struct ibv_qp *qp, struct ibv_send_wr *wr,
				struct ibv_send_wr **bad_wr)
{
	return qp->context->ops.post_send(qp, wr, bad_wr);
}

发送示例

struct ibv_sge list = {
    .addr	= buf,
    .length = size,
    .lkey	= lkey
};
struct ibv_send_wr wr = {
    .wr_id	    = 2,
    .sg_list    = &list,
    .num_sge    = 1,
    .opcode     = IBV_WR_SEND,
    .send_flags = ctx->send_flags,
};
struct ibv_send_wr *bad_wr;
ibv_post_send(ctx->qp, &wr, &bad_wr);

设置CQ通知

设置完成队列请求通知，当有完成消息到达完成队列，会触发event事件。

/**
 * ibv_req_notify_cq - Request completion notification on a CQ.  An
 *   event will be added to the completion channel associated with the
 *   CQ when an entry is added to the CQ.
 * @cq: The completion queue to request notification for.
 * @solicited_only: If non-zero, an event will be generated only for
 *   the next solicited CQ entry.  If zero, any CQ entry, solicited or
 *   not, will generate an event.
 */
static inline int ibv_req_notify_cq(struct ibv_cq *cq, int solicited_only)
{
	return cq->context->ops.req_notify_cq(cq, solicited_only);
}

获取CQ事件

获取完成通道内的完成事件，后2参数返回CQ和CQ上下文

该函数是阻塞的，直到有完成事件才能调用成功

/**
 * ibv_get_cq_event - Read next CQ event
 * @channel: Channel to get next event from.
 * @cq: Used to return pointer to CQ.
 * @cq_context: Used to return consumer-supplied CQ context.
 *
 * All completion events returned by ibv_get_cq_event() must
 * eventually be acknowledged with ibv_ack_cq_events().
 */
int ibv_get_cq_event(struct ibv_comp_channel *channel,
		     struct ibv_cq **cq, void **cq_context);

获取接口属性

获取接口属性，判断接口工作状态

/**
 * ibv_query_port - Get port properties
 */
int ibv_query_port(struct ibv_context *context, uint8_t port_num,
		   struct _compat_ibv_port_attr *port_attr);

//获取的属性
struct ibv_port_attr {
	enum ibv_port_state	state;
	enum ibv_mtu		max_mtu;
	enum ibv_mtu		active_mtu;
	int			gid_tbl_len;
	uint32_t		port_cap_flags;
	uint32_t		max_msg_sz;
	uint32_t		bad_pkey_cntr;
	uint32_t		qkey_viol_cntr;
	uint16_t		pkey_tbl_len;
	uint16_t		lid;
	uint16_t		sm_lid;
	uint8_t			lmc;
	uint8_t			max_vl_num;
	uint8_t			sm_sl;
	uint8_t			subnet_timeout;
	uint8_t			init_type_reply;
	uint8_t			active_width;
	uint8_t			active_speed;
	uint8_t			phys_state;
	uint8_t			link_layer;
	uint8_t			flags;
	uint16_t		port_cap_flags2;
};

获取接口GID

获取的是接口的gid表项，其中index可以为0,1,2

对于CX5网卡

index 0 时， GID为本地IPv6地址

index 1 时， GID为本地IPv4地址构成

具体可因网卡有所不同，如Soft-Roce网卡，index 2 对应的是IPv4地址构成的GID

/**
 * ibv_query_gid - Get a GID table entry
 */
int ibv_query_gid(struct ibv_context *context, uint8_t port_num,
		  int index, union ibv_gid *gid);

获取CQ消息

在获得CQ事件通知之后，执行该函数获取完成消息

/**
 * ibv_poll_cq - Poll a CQ for work completions
 * @cq:the CQ being polled
 * @num_entries:maximum number of completions to return
 * @wc:array of at least @num_entries of &struct ibv_wc where completions
 *   will be returned
 *
 * Poll a CQ for (possibly multiple) completions.  If the return value
 * is < 0, an error occurred.  If the return value is >= 0, it is the
 * number of completions returned.  If the return value is
 * non-negative and strictly less than num_entries, then the CQ was
 * emptied.
 */
static inline int ibv_poll_cq(struct ibv_cq *cq, int num_entries, struct ibv_wc *wc)
{
	return cq->context->ops.poll_cq(cq, num_entries, wc);
}

根据返回的wc中的信息可以根据wr_id关联到设置发送/接收的工作请求


struct ibv_wc {
	uint64_t		wr_id;
	enum ibv_wc_status	status;
	enum ibv_wc_opcode	opcode;
	uint32_t		vendor_err;
	uint32_t		byte_len;
	/* When (wc_flags & IBV_WC_WITH_IMM): Immediate data in network byte order.
	 * When (wc_flags & IBV_WC_WITH_INV): Stores the invalidated rkey.
	 */
	union {
		__be32		imm_data;
		uint32_t	invalidated_rkey;
	};
	uint32_t		qp_num;
	uint32_t		src_qp;
	unsigned int		wc_flags;
	uint16_t		pkey_index;
	uint16_t		slid;
	uint8_t			sl;
	uint8_t			dlid_path_bits;
};