（1）虚拟环境：VMware® Workstation 16 Pro
网上随便下载一个也行
（2）操作系统：ubuntu-22.04-beta-desktop-amd64.iso
下载地址：oldubuntu-releases-releases-22.04安装包下载_开源镜像站-阿里云
（3）DPDK版本：22.07
下载地址：http://fast.dpdk.org/rel/dpdk-22.07.tar.xz
（4）使用VMware安装ubuntu22.04，怎么安装去网上找
安装后的配置，添加两个网络适配器，用于DPDK通信
（5）DPDK的编译
访问链接：Ubuntu22.04 虚拟机中搭建 DPDK 开发环境
缺啥就安装啥
先安装一些依赖的软件包：
apt-get install meson
apt install python3-pyelftools
apt-get install pkg-configninja安装：Linux(15)Ubuntu安装ninja构建工具_一歲抬頭的博客-CSDN博客
再编译安装 DPDK：
wget https://fast.dpdk.org/rel/dpdk-22.07.tar.xz
tar xf dpdk-22.07.tar.xz
cd dpdk-22.07
meson build
cd build
ninja
ninja install执行完之后所有的库都安装在 /usr/local/lib/x86_64-linux-gnu/ 目录。
可执行程序和脚本都安装在 /usr/local/bin/ 目录。
要卸载只需执行 ninja uninstall 即可。
编译 igb uio 驱动
git clone http://dpdk.org/git/dpdk-kmods
cd dpdk-kmods/linux/igb_uio
make
编译出来 igb_uio.ko
为了验证我们的环境是没问题的，先编译出 l2fwd 程序：
cd examples/l2fwd
make
接着加载 igb_uio 驱动：
cd dpdk-kmods/linux/igb_uio
modprobe uio
insmod igb_uio.ko intr_mode=legacy
注意： 加载驱动时要带着参数intr_mode=legacy，如果不加参数，将会有问题！
分配一些大页内存【这里是1G】：
echo 512 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages绑定两个网卡【虚拟机有 3 个网卡，最后两个网卡供 DPDK 使用】：
ifconfig ens34 down
ifconfig ens35 down
dpdk-devbind.py -b igb_uio ens34 ens35
运行 l2fwd 程序：
cd examples/l2fwd/build/
./l2fwd -l 0-1 -- -p 0x3 -T 1如果看到以下信息，说明 DPDK 环境没问题！
EAL: Detected CPU lcores: 8
EAL: Detected NUMA nodes: 1
EAL: Detected shared linkage of DPDK
EAL: Multi-process socket /var/run/dpdk/rte/mp_socket
EAL: Selected IOVA mode 'PA'
EAL: VFIO support initialized
EAL: Probe PCI driver: net_e1000_em (8086:100f) device: 0000:02:02.0 (socket 0)
EAL: Error reading from file descriptor 20: Input/output error
EAL: Probe PCI driver: net_e1000_em (8086:100f) device: 0000:02:03.0 (socket 0)
EAL: Error reading from file descriptor 6: Input/output error
TELEMETRY: No legacy callbacks, legacy socket not created
MAC updating enabled
Lcore 0: RX port 0 TX port 1
Lcore 1: RX port 1 TX port 0
Initializing port 0... EAL: Error enabling interrupts for fd 20 (Input/output error)
done:
Port 0, MAC address: 00:0C:29:0C:53:91Initializing port 1... EAL: Error enabling interrupts for fd 6 (Input/output error)
done:
Port 1, MAC address: 00:0C:29:0C:53:9BChecking link statusdone
Port 0 Link up at 1 Gbps FDX Autoneg
Port 1 Link up at 1 Gbps FDX Autoneg
L2FWD: entering main loop on lcore 1
L2FWD: -- lcoreid=1 portid=1
L2FWD: entering main loop on lcore 0
L2FWD: -- lcoreid=0 portid=0Port statistics ====================================
Statistics for port 0 ------------------------------
Packets sent: 0
Packets received: 0
Packets dropped: 0
Statistics for port 1 ------------------------------
Packets sent: 0
Packets received: 0
Packets dropped: 0
Aggregate statistics ===============================
Total packets sent: 0
Total packets received: 0
Total packets dropped: 0
====================================================
^CSignal 2 received, preparing to exit...
EAL: Error disabling interrupts for fd 20 (Input/output error)
Closing port 0... Done
EAL: Error disabling interrupts for fd 6 (Input/output error)
Closing port 1... Done
Bye...
EAL: Error enabling interrupts for fd 20 (Input/output error) 等错误可以忽略。
解绑网卡
两个网卡 ens34 和 ens35 被 DPDK 占用后，ifconfig 里是没有的，要恢复请进行如下操作。
首先查看两个网卡 pci 设备号：
root@ubuntu2204:~# lspci | grep Eth
02:00.0 Ethernet controller: Intel Corporation 82545EM Gigabit Ethernet Controller (Copper) (rev 01)
02:01.0 Ethernet controller: Intel Corporation 82545EM Gigabit Ethernet Controller (Copper) (rev 01)
02:02.0 Ethernet controller: Intel Corporation 82545EM Gigabit Ethernet Controller (Copper) (rev 01)
02:03.0 Ethernet controller: Intel Corporation 82545EM Gigabit Ethernet Controller (Copper) (rev 01)
可看到最后两个网卡设备号是 02:02.0 和 02.03.0。
然后解绑两个网卡的 igb_uio 驱动，绑定 e1000 驱动：
dpdk-devbind.py -u 02:02.0 02:03.0
dpdk-devbind.py -b e1000 02:02.0 02:03.0
最后将网卡up起来：
ifconfig ens34 up
ifconfig ens35 up（6）DPDK间的发送与接收
上面环境配置完后，克隆出另一个环境，这样就有两个DPDK环境
DPDK接收数据代码：
#include <stdio.h>
#include <string.h>
#include <rte_eal.h>
#include <rte_ethdev.h>
#include <rte_ip.h>
#include <rte_tcp.h>
#include <rte_udp.h>
#include <rte_mbuf.h>
#include <rte_mempool.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <stdlib.h>
#define NB_MBUF 512
#define MAX_PKT_BURST 32
#define SELF_PROTO_TYPE 0x0888
static struct rte_eth_conf port_conf = {
.rxmode = {
.split_hdr_size = 0
}
};
// cat /proc/interrupts
void init_port(struct rte_mempool *mbuf_pool){
uint16_t nb_ports = 0;
int ret = 0;
int portid = 0;
struct rte_eth_dev_info dev_info;
struct rte_ether_addr addr;
const int num_rx_queues = 1;
const int num_tx_queues = 0;
nb_ports = rte_eth_dev_count_avail();
if(nb_ports == 0){
rte_exit(EXIT_FAILURE, "No support eth found\n");
}
for(portid = 0; portid < nb_ports; portid++){
ret = rte_eth_macaddr_get(portid, &addr);
if (ret != 0){
rte_exit(EXIT_FAILURE, "macaddr get failed\n");
}
printf("Port %u MAC: %02"PRIx8" %02"PRIx8" %02"PRIx8
" %02"PRIx8" %02"PRIx8" %02"PRIx8"\n",
portid, RTE_ETHER_ADDR_BYTES(&addr));
ret = rte_eth_dev_info_get(portid, &dev_info);
ret = rte_eth_dev_configure(portid, num_rx_queues, num_tx_queues, &port_conf);
ret = rte_eth_rx_queue_setup(portid, 0, 128, rte_eth_dev_socket_id(portid), NULL, mbuf_pool);
ret = rte_eth_dev_start(portid);
if (ret < 0) {
rte_exit(EXIT_FAILURE, "rte_eth_dev_start:err=%d, port=%u\n", ret, portid);
}
}
}
/*
发送方注意设置LP信息
*/
int
main(int argc, char **argv)
{
int ret;
unsigned lcore_id;
int i = 0;
int portid = 0;
int nb_rx = 0;
/* 初始化环境 */
ret = rte_eal_init(argc, argv);
if (ret < 0)
rte_panic("Cannot init EAL\n");
/* 创建内存池 */
struct rte_mempool *mbuf_pool = rte_pktmbuf_pool_create("my pool", NB_MBUF, 32, 0,
RTE_MBUF_DEFAULT_BUF_SIZE, rte_socket_id());
if (mbuf_pool == NULL){
return -1;
}
init_port(mbuf_pool);
while(1){
struct rte_mbuf* pkts_burst[MAX_PKT_BURST];
nb_rx = rte_eth_rx_burst(portid, 0, pkts_burst, MAX_PKT_BURST);
if(nb_rx == 0){
sleep(1);
continue;
}
printf("recv data start : %d \n", nb_rx);
for(i = 0; i < nb_rx; i++){
// ether
struct rte_ether_hdr *hdr = rte_pktmbuf_mtod(pkts_burst[i], struct rte_ether_hdr *);
// ip
printf("ether_type = %x \n", hdr->ether_type);
if(hdr->ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4)){
struct rte_ipv4_hdr *iphdr = rte_pktmbuf_mtod_offset(pkts_burst[i], struct rte_ipv4_hdr *, sizeof(struct rte_ether_hdr));
// upd
if(iphdr->next_proto_id == IPPROTO_UDP){
// (struct rte_udp_hdr *)RTE_PTR_ADD(iphdr, sizeof(struct rte_ipv4_hdr));
struct rte_udp_hdr* udphdr = (struct rte_udp_hdr*)(iphdr + 1);
uint16_t length = ntohs(udphdr->dgram_len);
*(char*)(udphdr + length) = '\0';
struct in_addr addr;
addr.s_addr = iphdr->src_addr;
printf("src:%s:%d \n", inet_ntoa(addr), ntohs(udphdr->src_port));
addr.s_addr = iphdr->dst_addr;
printf("dst:%s:%d, %s \n", inet_ntoa(addr), ntohs(udphdr->dst_port), (char*)(udphdr+1));
}
}else if(hdr->ether_type == rte_cpu_to_be_16(SELF_PROTO_TYPE)){
char *data = rte_pktmbuf_mtod_offset(pkts_burst[i], char *, sizeof(struct rte_ether_hdr));
printf("recv data: %s \n", data);
}
rte_pktmbuf_free(pkts_burst[i]);
}
}
return 0;
}makefile文件
# SPDX-License-Identifier: BSD-3-Clause
# Copyright(c) 2010-2014 Intel Corporation
# binary name
APP = demo1
# all source are stored in SRCS-y
SRCS-y := main.c
PKGCONF ?= pkg-config
# Build using pkg-config variables if possible
ifneq ($(shell $(PKGCONF) --exists libdpdk && echo 0),0)
$(error "no installation of DPDK found")
endif
all: shared
.PHONY: shared static
shared: build/$(APP)-shared
ln -sf $(APP)-shared build/$(APP)
static: build/$(APP)-static
ln -sf $(APP)-static build/$(APP)
PC_FILE := $(shell $(PKGCONF) --path libdpdk 2>/dev/null)
CFLAGS += -O3 $(shell $(PKGCONF) --cflags libdpdk)
# Add flag to allow experimental API as l2fwd uses rte_ethdev_set_ptype API
CFLAGS += -DALLOW_EXPERIMENTAL_API
LDFLAGS_SHARED = $(shell $(PKGCONF) --libs libdpdk)
LDFLAGS_STATIC = $(shell $(PKGCONF) --static --libs libdpdk)
ifeq ($(MAKECMDGOALS),static)
# check for broken pkg-config
ifeq ($(shell echo $(LDFLAGS_STATIC) | grep 'whole-archive.*l:lib.*no-whole-archive'),)
$(warning "pkg-config output list does not contain drivers between 'whole-archive'/'no-whole-archive' flags.")
$(error "Cannot generate statically-linked binaries with this version of pkg-config")
endif
endif
build/$(APP)-shared: $(SRCS-y) Makefile $(PC_FILE) | build
$(CC) $(CFLAGS) $(SRCS-y) -o $@ $(LDFLAGS) $(LDFLAGS_SHARED)
build/$(APP)-static: $(SRCS-y) Makefile $(PC_FILE) | build
$(CC) $(CFLAGS) $(SRCS-y) -o $@ $(LDFLAGS) $(LDFLAGS_STATIC)
build:
@mkdir -p $@
.PHONY: clean
clean:
rm -f build/$(APP) build/$(APP)-static build/$(APP)-shared
test -d build && rmdir -p build || true
DPDK发送数据代码
#include <stdio.h>
#include <string.h>
#include <rte_eal.h>
#include <rte_ethdev.h>
#include <rte_ip.h>
#include <rte_tcp.h>
#include <rte_udp.h>
#include <rte_mbuf.h>
#include <rte_mempool.h>
#include <rte_memcpy.h>
#include <rte_cycles.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <stdlib.h>
#define NB_MBUF 512
#define MAX_PKT_BURST 32
#define PKG_LEN 128
#define DST_MAC "005056224E38"
#define SELF_PROTO_TYPE 0x0888
static struct rte_eth_conf port_conf;
// cat /proc/interrupts
void init_port(struct rte_mempool *mbuf_pool){
uint16_t nb_ports = 0;
int ret = 0;
int portid = 0;
struct rte_eth_dev_info dev_info;
struct rte_ether_addr addr;
const int num_rx_queues = 0;
const int num_tx_queues = 1;
nb_ports = rte_eth_dev_count_avail();
if(nb_ports == 0){
rte_exit(EXIT_FAILURE, "No support eth found\n");
}
for(portid = 0; portid < nb_ports; portid++){
ret = rte_eth_macaddr_get(portid, &addr);
if (ret != 0){
rte_exit(EXIT_FAILURE, "macaddr get failed\n");
}
printf("Port %u MAC: %02"PRIx8" %02"PRIx8" %02"PRIx8
" %02"PRIx8" %02"PRIx8" %02"PRIx8"\n",
portid, RTE_ETHER_ADDR_BYTES(&addr));
ret = rte_eth_dev_info_get(portid, &dev_info);
ret = rte_eth_dev_configure(portid, num_rx_queues, num_tx_queues, &port_conf);
ret = rte_eth_tx_queue_setup(portid, 0, 128, rte_eth_dev_socket_id(portid), NULL);
ret = rte_eth_dev_start(portid);
if (ret < 0) {
rte_exit(EXIT_FAILURE, "rte_eth_dev_start:err=%d, port=%u\n", ret, portid);
}
ret = rte_eth_promiscuous_enable(portid);
if (ret != 0) {
rte_exit(EXIT_FAILURE, "promiscuous mode enable failed: %s\n", rte_strerror(-ret));
return;
}
}
}
void sendData(uint16_t portId, struct rte_mempool *mbuf_pool){
struct rte_ether_addr addr_src;
struct rte_ether_addr addr_dst;
int ret = 0;
int queue_id = 0;
int nb_prep = 0;
int nb_send = 0;
struct rte_mbuf *pkt = NULL;
ret = rte_eth_macaddr_get(portId, &addr_src);
addr_dst.addr_bytes[0] = 0x00;
addr_dst.addr_bytes[1] = 0x50;
addr_dst.addr_bytes[2] = 0x56;
addr_dst.addr_bytes[3] = 0x22;
addr_dst.addr_bytes[4] = 0x4E;
addr_dst.addr_bytes[5] = 0x38;
pkt = rte_pktmbuf_alloc(mbuf_pool);
if(!pkt){
printf("rte_pktmbuf_alloc faild \n");
return;
}
pkt->data_len = PKG_LEN;
struct rte_ether_hdr *hdr = rte_pktmbuf_mtod(pkt, struct rte_ether_hdr *);
// 源地址
rte_ether_addr_copy(&addr_src, &(hdr->src_addr));
// 目的地址
rte_ether_addr_copy(&addr_dst, &(hdr->dst_addr));
// 帧协议
hdr->ether_type = rte_cpu_to_be_16(SELF_PROTO_TYPE);
char* data = rte_pktmbuf_mtod_offset(pkt, char *, sizeof(struct rte_ether_hdr));
rte_memcpy(data, "hello", strlen("hello") + 1);
nb_prep = rte_eth_tx_prepare(portId, queue_id, &pkt, 1); //准备数据
nb_send = rte_eth_tx_burst(portId, queue_id, &pkt, 1); //发送数据
// 释放
rte_pktmbuf_free(pkt);
}
int
main(int argc, char **argv)
{
int ret;
unsigned lcore_id;
int i = 0;
int portid = 0;
int nb_rx = 0;
/* 初始化环境 */
ret = rte_eal_init(argc, argv);
if (ret < 0)
rte_panic("Cannot init EAL\n");
/* 创建内存池 */
struct rte_mempool *mbuf_pool = rte_pktmbuf_pool_create("my pool", NB_MBUF, 32, 0,
RTE_MBUF_DEFAULT_BUF_SIZE, rte_socket_id());
if (mbuf_pool == NULL){
return -1;
}
init_port(mbuf_pool);
while(1){
sendData(portid, mbuf_pool);
// 1000000微秒 = 1秒
rte_delay_us(1000000);
}
return 0;
}发送数据代码中，目标MAC地址需要改为接收虚拟机中的MAC
这个MAC在接收数据代码里也会打印，复制即可
makefile文件
# SPDX-License-Identifier: BSD-3-Clause
# Copyright(c) 2010-2014 Intel Corporation
# binary name
APP = send
# all source are stored in SRCS-y
SRCS-y := main.c
PKGCONF ?= pkg-config
# Build using pkg-config variables if possible
ifneq ($(shell $(PKGCONF) --exists libdpdk && echo 0),0)
$(error "no installation of DPDK found")
endif
all: shared
.PHONY: shared static
shared: build/$(APP)-shared
ln -sf $(APP)-shared build/$(APP)
static: build/$(APP)-static
ln -sf $(APP)-static build/$(APP)
PC_FILE := $(shell $(PKGCONF) --path libdpdk 2>/dev/null)
CFLAGS += -O3 $(shell $(PKGCONF) --cflags libdpdk)
# Add flag to allow experimental API as l2fwd uses rte_ethdev_set_ptype API
CFLAGS += -DALLOW_EXPERIMENTAL_API
LDFLAGS_SHARED = $(shell $(PKGCONF) --libs libdpdk)
LDFLAGS_STATIC = $(shell $(PKGCONF) --static --libs libdpdk)
ifeq ($(MAKECMDGOALS),static)
# check for broken pkg-config
ifeq ($(shell echo $(LDFLAGS_STATIC) | grep 'whole-archive.*l:lib.*no-whole-archive'),)
$(warning "pkg-config output list does not contain drivers between 'whole-archive'/'no-whole-archive' flags.")
$(error "Cannot generate statically-linked binaries with this version of pkg-config")
endif
endif
build/$(APP)-shared: $(SRCS-y) Makefile $(PC_FILE) | build
$(CC) $(CFLAGS) $(SRCS-y) -o $@ $(LDFLAGS) $(LDFLAGS_SHARED)
build/$(APP)-static: $(SRCS-y) Makefile $(PC_FILE) | build
$(CC) $(CFLAGS) $(SRCS-y) -o $@ $(LDFLAGS) $(LDFLAGS_STATIC)
build:
@mkdir -p $@
.PHONY: clean
clean:
rm -f build/$(APP) build/$(APP)-static build/$(APP)-shared
test -d build && rmdir -p build || true
测试
自己编译就行，make一下
发送数据：
接收数据：
![[Linux打怪升级之路]-缓冲区](https://img-blog.csdnimg.cn/297efc010473415ea72a4abd03e9ea42.png)
