操作系统
root@ubuntu22:~# lsb_release -a
No LSB modules are available.
Distributor ID: Ubuntu
Description: Ubuntu 22.04.3 LTS
Release: 22.04
Codename: jammy
root@ubuntu22:~#
前置软件准备
apt install git
apt install meson
apt install gcc
apt install pkg-config
apt install libnuma-dev
apt install libssl-dev
apt install automake autoconf
apt install libtool
apt install libpopt-dev
DPDK编译安装
下载DPVS
git clone https://github.com/iqiyi/dpvs.git
cd dpvs
下载DPDK20.11
在dpvs目录下下载dpdk，方便后续操作
root@ubuntu22:~/dpvs# wget https://fast.dpdk.org/rel/dpdk-20.11.10.tar.xz
--2025-02-20 15:16:23-- https://fast.dpdk.org/rel/dpdk-20.11.10.tar.xz
Resolving fast.dpdk.org (fast.dpdk.org)... 151.101.130.49, 151.101.66.49, 151.101.2.49, ...
Connecting to fast.dpdk.org (fast.dpdk.org)|151.101.130.49|:443... connected.
HTTP request sent, awaiting response... 200 OK
Length: 14047916 (13M) [application/octet-stream]
Saving to: ‘dpdk-20.11.10.tar.xz’
dpdk-20.11.10.tar.xz 100%[============================================================>] 13.40M 6.96MB/s in 1.9s
2025-02-20 15:16:26 (6.96 MB/s) - ‘dpdk-20.11.10.tar.xz’ saved [14047916/14047916]
root@ubuntu22:~/dpvs#
root@ubuntu22:~/dpvs# tar xf dpdk-20.11.10.tar.xz
打PATCH
root@ubuntu22:~/dpvs# cp patch/dpdk-stable-20.11.10/* dpdk-stable-20.11.10/
root@ubuntu22:~/dpvs/dpdk-stable-20.11.10# patch -p1 < 0001-kni-use-netlink-event-for-multicast-driver-part.patch
patching file kernel/linux/kni/kni_net.c
root@ubuntu22:~/dpvs/dpdk-stable-20.11.10# patch -p1 < 0002-pdump-change-dpdk-pdump-tool-for-dpvs.patch
patching file app/pdump/main.c
patching file lib/librte_pdump/rte_pdump.c
patching file lib/librte_pdump/rte_pdump.h
root@ubuntu22:~/dpvs/dpdk-stable-20.11.10# patch -p1 < 0003-debug-enable-dpdk-eal-memory-debug.patch
patching file lib/librte_eal/common/rte_malloc.c
patching file lib/librte_eal/include/rte_malloc.h
root@ubuntu22:~/dpvs/dpdk-stable-20.11.10# patch -p1 < 0004-ixgbe_flow-patch-ixgbe-fdir-rte_flow-for-dpvs.patch
patching file drivers/net/ixgbe/ixgbe_flow.c
root@ubuntu22:~/dpvs/dpdk-stable-20.11.10# patch -p1 < 0005
-bash: 0005: No such file or directory
root@ubuntu22:~/dpvs/dpdk-stable-20.11.10# patch -p1 < 0005-bonding-allow-slaves-from-different-numa-nodes.patch
patching file drivers/net/bonding/rte_eth_bond_pmd.c
root@ubuntu22:~/dpvs/dpdk-stable-20.11.10# patch -p1 < 0006-bonding-fix-problem-in-mode-4-dropping-multicast-pac.patch
patching file drivers/net/bonding/rte_eth_bond_pmd.c
root@ubuntu22:~/dpvs/dpdk-stable-20.11.10# patch -p1 < 0007-bonding-device-sends-packets-with-user-specified-sal.patch
patching file drivers/net/bonding/rte_eth_bond_pmd.c
patching file lib/librte_mbuf/rte_mbuf.h
编译
在dpdkbuild中编译，安装在dpdklib中，后续指定 pkg-config目录到
root@ubuntu22:~/dpvs/dpdk-stable-20.11.10# mkdir dpdklib
root@ubuntu22:~/dpvs/dpdk-stable-20.11.10# mkdir dpdkbuild
root@ubuntu22:~/dpvs/dpdk-stable-20.11.10# meson -Denable_kmods=true -Dprefix=/root/dpvs/dpdk-stable-20.11.10/dpdklib dpdkbuild
root@ubuntu22:~/dpvs/dpdk-stable-20.11.10# ninja -C dpdkbuild/
ninja: Entering directory `dpdkbuild/'
[2419/2421] Generating kernel/linux/kni/rte_kni with a custom command
make: Entering directory '/usr/src/linux-headers-6.2.0-26-generic'
warning: the compiler differs from the one used to build the kernel
The kernel was built by: x86_64-linux-gnu-gcc-11 (Ubuntu 11.3.0-1ubuntu1~22.04.1) 11.3.0
You are using: gcc-11 (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0
CC [M] /root/dpvs/dpdk-stable-20.11.10/dpdkbuild/kernel/linux/kni/kni_misc.o
CC [M] /root/dpvs/dpdk-stable-20.11.10/dpdkbuild/kernel/linux/kni/kni_net.o
LD [M] /root/dpvs/dpdk-stable-20.11.10/dpdkbuild/kernel/linux/kni/rte_kni.o
MODPOST /root/dpvs/dpdk-stable-20.11.10/dpdkbuild/kernel/linux/kni/Module.symvers
CC [M] /root/dpvs/dpdk-stable-20.11.10/dpdkbuild/kernel/linux/kni/rte_kni.mod.o
LD [M] /root/dpvs/dpdk-stable-20.11.10/dpdkbuild/kernel/linux/kni/rte_kni.ko
BTF [M] /root/dpvs/dpdk-stable-20.11.10/dpdkbuild/kernel/linux/kni/rte_kni.ko
Skipping BTF generation for /root/dpvs/dpdk-stable-20.11.10/dpdkbuild/kernel/linux/kni/rte_kni.ko due to unavailability of vmlinux
make: Leaving directory '/usr/src/linux-headers-6.2.0-26-generic'
[2421/2421] Linking target app/test/dpdk-test
root@ubuntu22:~/dpvs/dpdk-stable-20.11.10# cd dpdkbuild
root@ubuntu22:~/dpvs/dpdk-stable-20.11.10/dpdkbuild# ninja install
root@ubuntu22:~/dpvs/dpdk-stable-20.11.10#
设置pkg-cofnig
这里的pkgconfig目录与centos(/lib64/pkgconfig)不同，需要根据实际路径设定
root@ubuntu22:~/dpvs/dpdk-stable-20.11.10/dpdklib# export PKG_CONFIG_PATH=/root/dpvs/dpdk-stable-20.11.10/dpdklib/lib/x86_64-linux-gnu/pkgconfig
root@ubuntu22:~/dpvs/dpdk-stable-20.11.10/dpdklib#
验证libdpdk
root@ubuntu22:~/dpvs/dpdk-stable-20.11.10/dpdklib# pkg-config --libs libdpdk
-L/root/dpvs/dpdk-stable-20.11.10/dpdklib/lib/x86_64-linux-gnu -Wl,--as-needed -lrte_node -lrte_graph -lrte_bpf -lrte_flow_classify -lrte_pipeline -lrte_table -lrte_port -lrte_fib -lrte_ipsec -lrte_vhost -lrte_stack -lrte_security -lrte_sched -lrte_reorder -lrte_rib -lrte_regexdev -lrte_rawdev -lrte_pdump -lrte_power -lrte_member -lrte_lpm -lrte_latencystats -lrte_kni -lrte_jobstats -lrte_ip_frag -lrte_gso -lrte_gro -lrte_eventdev -lrte_efd -lrte_distributor -lrte_cryptodev -lrte_compressdev -lrte_cfgfile -lrte_bitratestats -lrte_bbdev -lrte_acl -lrte_timer -lrte_hash -lrte_metrics -lrte_cmdline -lrte_pci -lrte_ethdev -lrte_meter -lrte_net -lrte_mbuf -lrte_mempool -lrte_rcu -lrte_ring -lrte_eal -lrte_telemetry -lrte_kvargs
设置大页
echo "vm.nr_hugepages=1024" >> /etc/sysctl.conf
sysctl -p
root@ubuntu22:/etc/apt# cat /proc/meminfo | grep Huge
AnonHugePages: 0 kB
ShmemHugePages: 0 kB
FileHugePages: 0 kB
HugePages_Total: 1024
HugePages_Free: 1024
HugePages_Rsvd: 0
HugePages_Surp: 0
Hugepagesize: 2048 kB
Hugetlb: 2097152 kB
安装golang
heathcheck dpvs-agent 需要golang 1.20版本
wget https://golang.google.cn/dl/go1.20.linux-amd64.tar.gz
rm -rf /usr/local/go && tar -C /usr/local -xzf go1.20.linux-amd64.tar.gz
export PATH=$PATH:/usr/local/go/bin
go env -w GOPROXY=https://goproxy.cn,direct
编译DPVS
修改编译选项
开启编译healthcheck ipvs-agent，config.mk中修改,
export CONFIG_DPVS_AGENT=y
编译安装
root@ubuntu22:~# make
root@ubuntu22:~# make install
root@ubuntu22:~# cd dpvs
root@ubuntu22:~/dpvs# ls bin
dpip dpvs dpvs-agent healthcheck ipvsadm keepalived
报错处理
报错
/root/dpvs/src/ipvs/ip_vs_synproxy.c: In function ‘cookie_hash’:
/root/dpvs/src/ipvs/ip_vs_synproxy.c:216:5: error: ‘MD5’ is deprecated: Since OpenSSL 3.0 [-Werror=deprecated-declarations]
216 | MD5((unsigned char *)data, sizeof(data), hash);
| ^~~
libssl3.0对 MD5有一个废弃告警，这里会把告警变成报错。
src/Makefile中，去掉-Werror
50
#CFLAGS += -Wall -Werror -Wstrict-prototypes -Wmissing-prototypes -mcmodel=medium
51 CFLAGS += -Wall -Wstrict-prototypes -Wmissing-prototypes -mcmodel=medium
52
报错
lldp.c: In function ‘lldp_do_cmd’:
lldp.c:101:27: error: format not a string literal and no format arguments [-Werror=format-security]
101 | printf(message->message);
tools/dpip/lldp.c中修改101行
//printf(message->message);
printf("%s", message->message);
运行DPVS
加载驱动
insmod ./dpdk-stable-20.11.10/dpdkbuild/kernel/linux/kni/rte_kni.ko
modprobe uio_pci_generic
修改网卡驱动
root@ubuntu22:~# lspci | grep Eth
03:00.0 Ethernet controller: Intel Corporation 82574L Gigabit Network Connection
0b:00.0 Ethernet controller: VMware VMXNET3 Ethernet Controller (rev 01)
13:00.0 Ethernet controller: VMware VMXNET3 Ethernet Controller (rev 01)
dpdk-devbind.py -u 0000:0b:00.0
dpdk-devbind.py -b uio_pci_generic 0000:0b:00.0
修改配置文件
cp conf/dpvs.conf.single-nic.sample /etc/dpvs.conf
cat /etc/dpvs.conf
netif_defs {
<init> pktpool_size 524287
<init> pktpool_cache 256
<init> device dpdk0 {
rx {
queue_number 1 # 虚拟机网卡，改为1个RX队列
descriptor_number 1024
rss all
}
tx {
queue_number 1 # 虚拟机网卡，改为1个TX队列
descriptor_number 1024
}
! mtu 1500
! promisc_mode
! allmulticast
kni_name dpdk0.kni
}
}
对应的worker写改为1个rx worker 1个tx worker
worker_defs {
<init> worker cpu0 {
type master
cpu_id 0
}
<init> worker cpu1 {
type slave
cpu_id 1
port dpdk0 {
rx_queue_ids 0
tx_queue_ids 0
! isol_rx_cpu_ids 9
! isol_rxq_ring_sz 1048576
}
}
<init> worker cpu2 {
type slave
cpu_id 2
port dpdk0 {
rx_queue_ids 1
tx_queue_ids 1
! isol_rx_cpu_ids 10
! isol_rxq_ring_sz 1048576
}
}
}
...
sa_pool {
pool_hash_size 16
flow_enable off # 虚拟机VMXNET3网卡不支持flow, 所以off掉
}
运行报错
dpvs – -a 0000:0b:00.0 -l 0-2 , 0 master 1 ，2 worker
root@ubuntu22:~/dpvs/bin# dpvs -- -a 0000:0b:00.0 -l 0-2
Command 'dpvs' not found, did you mean:
command 'pvs' from deb lvm2 (2.03.11-2.1ubuntu4)
Try: apt install <deb name>
root@ubuntu22:~/dpvs/bin# ./dpvs -- -a 0000:0b:00.0 -l 0-2
current thread affinity is set to 3F
EAL: Detected 6 lcore(s)
EAL: Detected 1 NUMA nodes
EAL: Detected static linkage of DPDK
EAL: Multi-process socket /var/run/dpdk/rte/mp_socket
EAL: Selected IOVA mode 'PA'
EAL: No available hugepages reported in hugepages-1048576kB
EAL: Probing VFIO support...
EAL: Probe PCI driver: net_vmxnet3 (15ad:07b0) device: 0000:0b:00.0 (socket 0)
EAL: No legacy callbacks, legacy socket not created
DPVS: dpvs version: 1.9-8, build on 2025.02.21.01:02:16
DPVS: dpvs-conf-file: /etc/dpvs.conf
DPVS: dpvs-pid-file: /var/run/dpvs.pid
DPVS: dpvs-ipc-file: /var/run/dpvs.ipc
CFG_FILE: Opening configuration file '/etc/dpvs.conf'.
CFG_FILE: log_level = WARNING
NETIF: dpdk0:rx_queue_number = 1
NETIF: worker cpu1:dpdk0 rx_queue_id += 0
NETIF: worker cpu1:dpdk0 tx_queue_id += 0
NETIF: worker cpu2:dpdk0 rx_queue_id += 1
NETIF: worker cpu2:dpdk0 tx_queue_id += 1
NETIF: dpdk0: rte_eth_dev_set_mc_addr_list failed -- Operation not supported,enable all multicast
Segmentation fault (core dumped)
直接段错误，重新以DEBUG模式编译DPDK 和 DPVS ，GDB跟踪
Thread 1 "dpvs" received signal SIGSEGV, Segmentation fault.
0x000055555601d5f4 in vmxnet3_dev_set_rxmode (set=1, feature=8, hw=0x1003ad940) at ../drivers/net/vmxnet3/vmxnet3_ethdev.c:1293
1293 rxConf->rxMode = rxConf->rxMode | feature;
(gdb) bt
#0 0x000055555601d5f4 in vmxnet3_dev_set_rxmode (set=1, feature=8, hw=0x1003ad940) at ../drivers/net/vmxnet3/vmxnet3_ethdev.c:1293
#1 vmxnet3_dev_allmulticast_enable (dev=0x555556b6c980 <rte_eth_devices>) at ../drivers/net/vmxnet3/vmxnet3_ethdev.c:1341
#2 0x000055555626f3e2 in rte_eth_allmulticast_enable (port_id=0) at ../lib/librte_ethdev/rte_ethdev.c:2595
#3 0x0000555555c51033 in dpdk_set_mc_list (dev=0x10af20ec0) at /root/dpvs/src/netif.c:3356
#4 0x0000555555c1bc8c in __netif_set_mc_list (dev=0x10af20ec0) at /root/dpvs/src/netif_addr.c:189
#5 0x0000555555c1bd3b in netif_mc_add (dev=0x10af20ec0, addr=0x5555562d3500 <LLDP_ETHER_ADDR_DST>) at /root/dpvs/src/netif_addr.c:210
#6 0x0000555555be0150 in lldp_ether_addr_filter (add=true) at /root/dpvs/src/lldp.c:1512
#7 0x0000555555be0241 in lldp_xmit_start () at /root/dpvs/src/lldp.c:1530
#8 0x0000555555be2198 in dpvs_lldp_init () at /root/dpvs/src/lldp.c:1857
#9 0x0000555555aa0e8e in inet_init () at /root/dpvs/src/inet.c:103
#10 0x0000555555bef475 in modules_init () at /root/dpvs/src/main.c:138
#11 0x0000555555befe07 in main (argc=5, argv=0x7fffffffe2f0) at /root/dpvs/src/main.c:339
(gdb) f 0
#0 0x000055555601d5f4 in vmxnet3_dev_set_rxmode (set=1, feature=8, hw=0x1003ad940) at ../drivers/net/vmxnet3/vmxnet3_ethdev.c:1293
1293 rxConf->rxMode = rxConf->rxMode | feature;
(gdb) p rxConf
$1 = (struct Vmxnet3_RxFilterConf *) 0x78
(gdb) p hw->shared->devRead
Cannot access memory at address 0x8
(gdb) p hw->shared
$2 = (Vmxnet3_DriverShared *) 0x0
(gdb) p *hw
$3 = {hw_addr0 = 0x1100800000 "", hw_addr1 = 0x1100801000 "", back = 0x0, device_id = 1968, vendor_id = 5549, subsystem_device_id = 0, subsystem_vendor_id = 0,
adapter_stopped = false, perm_addr = "\000\f)\264\277<", num_tx_queues = 1 '\001', num_rx_queues = 1 '\001', bufs_per_pkt = 1 '\001', version = 1 '\001',
txdata_desc_size = 128, rxdata_desc_size = 0, num_intrs = 0 '\000', tqd_start = 0x0, rqd_start = 0x0, shared = 0x0, sharedPA = 0
参考代码
static int
vmxnet3_dev_allmulticast_enable(struct rte_eth_dev *dev)
{
struct vmxnet3_hw *hw = dev->data->dev_private;
vmxnet3_dev_set_rxmode(hw, VMXNET3_RXM_ALL_MULTI, 1);
return 0;
}
/* Updating rxmode through Vmxnet3_DriverShared structure in adapter */
static void
vmxnet3_dev_set_rxmode(struct vmxnet3_hw *hw, uint32_t feature, int set)
{
struct Vmxnet3_RxFilterConf *rxConf = &hw->shared->devRead.rxFilterConf;
if (set)
rxConf->rxMode = rxConf->rxMode | feature;
else
rxConf->rxMode = rxConf->rxMode & (~feature);
VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, VMXNET3_CMD_UPDATE_RX_MODE);
}
hw下shared为0 ，rxConf则更是一个野指针，导致coredump.
猜测当前VMware虚拟机的vmxnet3网卡或驱动的问题
物理机测试
将上述编译安装流程都走一遍，配置文件不变, 运行成功
root@r750-132:~/dpvs/bin# ./dpvs -- -a 98:00.0 -l 0-9
current thread affinity is set to FFFFFFFF
EAL: Detected 32 lcore(s)
EAL: Detected 2 NUMA nodes
EAL: Detected static linkage of DPDK
EAL: Multi-process socket /var/run/dpdk/rte/mp_socket
EAL: Selected IOVA mode 'PA'
EAL: No available hugepages reported in hugepages-1048576kB
EAL: Probing VFIO support...
EAL: VFIO support initialized
EAL: Probe PCI driver: mlx5_pci (15b3:1017) device: 0000:98:00.0 (socket 1)
EAL: No legacy callbacks, legacy socket not created
DPVS: dpvs version: 1.9-8, build on 2025.02.20.15:37:58
DPVS: dpvs-conf-file: /etc/dpvs.conf
DPVS: dpvs-pid-file: /var/run/dpvs.pid
DPVS: dpvs-ipc-file: /var/run/dpvs.ipc
CFG_FILE: Opening configuration file '/etc/dpvs.conf'.
CFG_FILE: log_level = WARNING
NETIF: dpdk0:rx_queue_number = 8
NETIF: worker cpu1:dpdk0 rx_queue_id += 0
NETIF: worker cpu1:dpdk0 tx_queue_id += 0
NETIF: worker cpu2:dpdk0 rx_queue_id += 1
NETIF: worker cpu2:dpdk0 tx_queue_id += 1
NETIF: worker cpu3:dpdk0 rx_queue_id += 2
NETIF: worker cpu3:dpdk0 tx_queue_id += 2
NETIF: worker cpu4:dpdk0 rx_queue_id += 3
NETIF: worker cpu4:dpdk0 tx_queue_id += 3
NETIF: worker cpu5:dpdk0 rx_queue_id += 4
NETIF: worker cpu5:dpdk0 tx_queue_id += 4
NETIF: worker cpu6:dpdk0 rx_queue_id += 5
NETIF: worker cpu6:dpdk0 tx_queue_id += 5
NETIF: worker cpu7:dpdk0 rx_queue_id += 6
NETIF: worker cpu7:dpdk0 tx_queue_id += 6
NETIF: worker cpu8:dpdk0 rx_queue_id += 7
NETIF: worker cpu8:dpdk0 tx_queue_id += 7
SAPOOL: sapool_filter_enable = on
IPVS: dp_vs_conn_init: lcore 9: nothing to do.
NETIF: Ethdev port_id=0 invalid tx_offload: 0x1000e, valid value: 0xc96af
配置接口/查看接口
root@r750-132:~/dpvs/bin# ./dpip link show
1: dpdk0: socket 1 mtu 1500 rx-queue 8 tx-queue 8
UP 100000 Mbps full-duplex auto-nego lldp
addr E8:EB:D3:A3:83:76 OF_RX_IP_CSUM OF_TX_IP_CSUM OF_TX_TCP_CSUM OF_TX_UDP_CSUM
root@r750-132:~/dpvs/bin# ./dpip addr show
inet 10.1.1.132/24 scope global dpdk0
valid_lft forever preferred_lft forever
inet6 fe80::eaeb:d3ff:fea3:8376/64 scope link dpdk0
valid_lft forever preferred_lft forever
