宿主机配置
虚拟机配置文件
<domain type='kvm'> //如果是Xen，则type=‘xen’
<name>redflag1</name> //虚拟机名称，同一物理机唯一
<uuid>44748c15-7c00-4817-8724-675a27c3f821</uuid> //同一物理机唯一，可用uuidgen生成
<memory>67108864</memory>
<currentMemory>67108864</currentMemory> //memory这两个值最好设成一样
<vcpu>64</vcpu>
<cpu mode="host-passthrough"/>
<os>
<type arch='aarch64' machine='virt'>hvm</type>
<loader readonly='yes' type='pflash'>/usr/share/edk2/aarch64/QEMU_EFI-pflash.raw</loader>
<nvram template='/usr/share/edk2/aarch64/vars-template-pflash.raw'>/usr/share/edk2/aarch64/QEMU_VARS.fd</nvram>
</os>
<features>
<acpi/>
<apic/>
<pae/>
</features>
<clock offset='localtime'/> //虚拟机时钟设置，这里表示本地本机时间
<on_poweroff>destroy</on_poweroff> //突发事件动作
<on_reboot>restart</on_reboot>
<on_crash>restart</on_crash>
<devices> //设备配置
<emulator>/usr/bin/qemu-kvm</emulator> //如果是Xen则是/usr/lib/xen/binqemu-dm
<controller type="scsi" index="0" model="virtio-scsi"/>
<controller type='usb' index='0' model='qemu-xhci' ports='15'/>
<input type='keyboard' bus='usb'/>
<input type='mouse' bus='usb'/>
<input type='tablet' bus='usb'/>
<disk type='file' device='cdrom'>//光盘
<driver name='qemu' type='raw'/>
<source file='/home/yeqiang/RedFlag-Asianux-Server-7.5-aarch64-for-Phytium-dvddisc-20210701.iso'/>
<target dev='hda' bus='scsi'/>
<readonly/>
</disk>
<disk type='file' device='disk'> //硬盘
<driver name='qemu' type='qcow2'/>
<source file='/home/yeqiang/qemu-virtual-machine/redflag.qcow2'/>
<target dev='vda' bus='virtio'/>
</disk>
<graphics type='vnc' port='5900' autoport='yes' listen='0.0.0.0' keymap='en-us'> //配置vnc，windows下可以使用vncviewer登录，获取vnc端口号：virsh vncdisplay vm0
<listen type='address' address='0.0.0.0'/>
</graphics>
<interface type='bridge'>
<source bridge='virbr0'/>
</interface>
</devices>
</domain>
测试程序
#include <errno.h>
#include <iostream>
#include <pthread.h>
#include <signal.h>
#include <stdio.h>
#include <stdlib.h>
#include <string>
#include <sys/stat.h>
#include <sys/time.h>
#include <sys/types.h>
#include <unistd.h>
using namespace std;
std::string sub_cgroup_dir("/sys/fs/cgroup/cpu/test");
// common lib
bool is_dir(const std::string &path) {
struct stat statbuf;
if (stat(path.c_str(), &statbuf) == 0) {
if (0 != S_ISDIR(statbuf.st_mode)) {
return true;
}
}
return false;
}
bool write_file(const std::string &file_path, int num) {
// std::cout << file_path << " op:" << num << std::endl;
FILE *fp = fopen(file_path.c_str(), "w");
if (fp == NULL) {
return false;
}
// std::cout << file_path << " op:" << num << std::endl;
std::string write_data = to_string(num);
fputs(write_data.c_str(), fp);
fclose(fp);
return true;
}
// ms
long get_ms_timestamp() {
timeval tv;
gettimeofday(&tv, NULL);
return (tv.tv_sec * 1000 + tv.tv_usec / 1000);
}
// cgroup
bool create_cgroup() {
if (is_dir(sub_cgroup_dir) == false) {
if (mkdir(sub_cgroup_dir.c_str(), S_IRWXU | S_IRGRP) != 0) {
cout << "mkdir cgroup dir fail" << endl;
return false;
}
}
int pid = getpid();
cout << "pid is " << pid << endl;
std::string procs_path = sub_cgroup_dir + "/cgroup.procs";
return write_file(procs_path, pid);
}
bool set_period(int period) {
std::string period_path = sub_cgroup_dir + "/cpu.cfs_period_us";
return write_file(period_path, period);
}
bool set_quota(int quota) {
std::string quota_path = sub_cgroup_dir + "/cpu.cfs_quota_us";
return write_file(quota_path, quota);
}
bool set_cpuOnline(int cpuId, int online) {
std::string cpuPath = std::string("/sys/devices/system/cpu/cpu") +
std::to_string(cpuId) + std::string("/online");
return write_file(cpuPath, online);
}
// thread
// param: ms interval
void *thread_func(void *param) {
int i = 0;
int interval = (long)param;
long last = get_ms_timestamp();
while (true) {
i++;
if (i % 100000 != 0) {
continue;
}
long current = get_ms_timestamp();
if ((current - last) >= interval) {
last = current;
}
}
pthread_exit(NULL);
}
void test_thread() {
const int k_thread_num = 320;
pthread_t pthreads[k_thread_num];
for (int i = 0; i < k_thread_num; i++) {
if (pthread_create(&pthreads[i], NULL, thread_func, (void *)(i + 1)) != 0) {
cout << "create thread fail" << endl;
} else {
cout << "create thread success,tid is " << pthreads[i] << endl;
}
}
}
void *thread_cpu_online_ctl(void *param) {
int online, cpu;
bool b;
while (true) {
online = rand() % 2;
cpu = rand() % 64;
b = set_cpuOnline(cpu, online);
// std::cout << cpu << " -> " << online << "result:" << b << std::endl;
}
pthread_exit(NULL);
}
void cpu_ctl_thread() {
const int k_thread_num = 64;
pthread_t pthreads[k_thread_num];
for (int i = 0; i < k_thread_num; i++) {
if (pthread_create(&pthreads[i], NULL, thread_cpu_online_ctl,
(void *)(i + 1)) != 0) {
cout << "create thread fail" << endl;
} else {
cout << "create thread success,tid is " << pthreads[i] << endl;
}
}
}
int main(int argc, char *argv[]) {
int period = 100000;
int quota = 4000000;
cout << "period is " << period << endl;
cout << "quota is " << quota << endl;
srand(time(nullptr));
test_thread();
cpu_ctl_thread();
if (create_cgroup() == false) {
cout << "create cgroup fail" << endl;
return -1;
}
set_period(period);
set_quota(quota);
while (true) {
sleep(10000);
}
return 0;
}
编译
g++ -std=c++11 -lpthread tcti.cpp -o tcti执行
./tcti &vmstat监控
vmstat 1一段时间后
此时，bash上敲击回车能换行,但是无信息输出
ctrl+C 可退出程序（又一次测试未退出，卡主了）
tcti进程 cpu 0%
kill -9 无法杀死
reboot，卡死
此时键盘输入有信息打印
卡死后，放置约14小时后，再次vnc登录，光标已不再闪烁，键盘输入也没有给出反馈，彻底死机。
此时在宿主机查看kvm进程CPU状态，几乎没有消耗
virsh 导出内核转储文件分析(64G内存，时间较长)
virsh dump --memory-only --format=kdump-zlib redflag1 redflag1-coredump-0428-stucked.zlib重启虚拟机，分析coredump
crash调试
bt -a
ps
原本期望通过不断设置CPU在线状态，迫使CFS触发_nohz_idle_balance。实际上的故障与期望不一致，产生了新问题。
重新测试
刚开始测试时，100.0 id 数值明显不对，应该是操作CPU在线状态导致
约6分钟后，测试进程没有CPU消耗了
STAT：SI
S //处于休眠状态；
l //多线程，克隆线程（使用 CLONE_THREAD, 类似 NPTL pthreads）；
参考：Linux进程状态(ps stat)详解_smartvxworks的博客-CSDN博客
说明此时线程sleep后没有再被调度器分配CPU了 。
当前CPU online状态
gdb attach上去看看
挂不上去，且无法退出
正确指令应该是 gdb attach 2034(虽然敲错了指令，但日志已显示Attatching to process 2034)
在kvm外，导出coredump
virsh dump --memory-only --format=kdump-zlib redflag1 redflag1-coredump-0428-stucked2.zlib
从目前看，应该是触发了bug，当前任务被调度出去，一直没能调度回来。该进程程序本身没有什么问题。
查看几个子线程，连号
都在这个位置
复测，待续。。。
10分51秒后，tcti CPU占用率掉到了0%
尝试手动调整cgroup限制，新开ssh窗口登录上去，方便观察top信息
cat /dev/null > /sys/fs/cgroup/cpu/test/cgroup.procs
cat /dev/null > /sys/fs/cgroup/cpu/test/tasks 查看发现无法清空
设置所有CPU online=1
ls /sys/devices/system/cpu/ | grep cpu | grep -v freq | grep -v idle | xargs -i echo "echo 1 > /sys/devices/system/cpu/"{}"/online" | bash当前进程卡死了。。。
推测引发了内核层bug
mesg当前时段没有信息输出
openEuler复测
openEuler 20.03 LTS SP3 aarch64
运行两小时，正常，说明是内核的bug
