目录
- 前言
- 运行Yolov5模型
- 第一步：修改CMA_SIZE
- 第二步：修改yolov5项目代码
- 第三步：运行程序
前言
买了luckfox pico的rv1103开发板，摄像头是SC3336 3MP Camera (A)摄像头，参考RV1103 Luckfox Pico使用SPI NAND Flash烧录镜像焊了nand flash，烧录的buildroot系统。
官网中有一些调用摄像头进行检测的实例：RKMPI-example，其中retinaface的两个例子luckfox_pico_rtsp_retinaface和luckfox_pico_rtsp_retinaface_osd均在开发板上能跑起来会有人脸的识别结果打印出来，但是rtsp推流时会显示下方的问题，在局域网下的 PC 使用 VLC 软件打开网络串流rtsp://172.32.0.93/live/0并不能获取图像，后续再看啥问题。
运行Yolov5模型
以官网实例中的luckfox_pico_rtsp_yolov5为基础，实现在rv1103上调用摄像头进行实时检测。
受内存限制，
luckfox pico开发板是无法运行该yolov5例子的。
官网上实现的是RKNN推理图像rtsp推流实例，并且为了最大化提高rtsp推流的帧率，实例使用VI组件来实现摄像头图像捕获。在RKNN推理结果的标注上，采用两种方式来实现：1）将标注好的图像上传到 VENC 组件进行编码传输
2）在图像上传到 VENC 组件后再使用 RGN 模块以打 OSD 的形式标注结果
其中yolov5例子采用的是第1种方法，工作流程为：
要在rv1103上跑起来yolov5需要将rtsp推流部分去掉，直接以输出打印的方式查看yolov5检测结果，从该流程中可以知道需要将VENC组件和rtsp部分去掉，像下面图中所示，直接VI输入VPSS然后输入rknn，其余都删除。
具体操作如下：
第一步：修改CMA_SIZE
rknn和摄像头部分都有用到CMA内存，系统烧录时默认的是24M，不太够，需要增加到30M。参考RV1103 Luckfox Pico使用SPI NAND Flash烧录镜像，在第2）步中的BoardConfig-SPI_NAND-Buildroot-RV1103_Luckfox_Pico_Pro-IPC.mk中需要调整这句代码：export RK_BOOTARGS_CMA_SIZE="30M"，其他都一样的流程进行烧录。
adb连接上开发板后，可使用下面命令查看CMA的大小是否正确
[root@luckfox ]# grep -i cma /proc/meminfo
CmaTotal: 30720 kB
CmaAllocated: 12 kB
CmaReleased: 30708 kB
CmaFree: 0 kB
第二步：修改yolov5项目代码
源码：https://github.com/luckfox-eng29/luckfox_pico_rtsp_yolov5
很简单，将main.cc代码中与VENC和rstp两部分的代码注释掉就行，注释完后代码为：
#include <assert.h>
#include <errno.h>
#include <fcntl.h>
#include <getopt.h>
#include <pthread.h>
#include <signal.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/poll.h>
#include <time.h>
#include <unistd.h>
#include <vector>
#include "rtsp_demo.h"
#include "luckfox_mpi.h"
#include "yolov5.h"
#include <opencv2/core/core.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <opencv2/imgproc/imgproc.hpp>
// disp size
int width = 720;
int height = 480;
// model size
int model_width = 640;
int model_height = 640;
float scale ;
int leftPadding ;
int topPadding ;
bool quit = false;
static void sigterm_handler(int sig) {
fprintf(stderr, "signal %d\n", sig);
quit = true;
}
cv::Mat letterbox(cv::Mat input)
{
float scaleX = (float)model_width / (float)width; //0.888
float scaleY = (float)model_height / (float)height; //1.125
scale = scaleX < scaleY ? scaleX : scaleY;
int inputWidth = (int)((float)width * scale);
int inputHeight = (int)((float)height * scale);
leftPadding = (model_width - inputWidth) / 2;
topPadding = (model_height - inputHeight) / 2;
cv::Mat inputScale;
cv::resize(input, inputScale, cv::Size(inputWidth,inputHeight), 0, 0, cv::INTER_LINEAR);
cv::Mat letterboxImage(640, 640, CV_8UC3,cv::Scalar(0, 0, 0));
cv::Rect roi(leftPadding, topPadding, inputWidth, inputHeight);
inputScale.copyTo(letterboxImage(roi));
return letterboxImage;
}
void mapCoordinates(int *x, int *y) {
int mx = *x - leftPadding;
int my = *y - topPadding;
*x = (int)((float)mx / scale);
*y = (int)((float)my / scale);
}
int main(int argc, char *argv[]) {
RK_S32 s32Ret = 0;
int sX,sY,eX,eY;
// Ctrl-c quit
signal(SIGINT, sigterm_handler);
// Rknn model
char text[16];
rknn_app_context_t rknn_app_ctx;
object_detect_result_list od_results;
int ret;
const char *model_path = "./model/yolov5.rknn";
memset(&rknn_app_ctx, 0, sizeof(rknn_app_context_t));
init_yolov5_model(model_path, &rknn_app_ctx);
printf("init rknn model success!\n");
init_post_process();
//h264_frame
// VENC_STREAM_S stFrame;
// stFrame.pstPack = (VENC_PACK_S *)malloc(sizeof(VENC_PACK_S));
// VIDEO_FRAME_INFO_S h264_frame;
VIDEO_FRAME_INFO_S stVpssFrame;
// rkaiq init
RK_BOOL multi_sensor = RK_FALSE;
const char *iq_dir = "/etc/iqfiles";
rk_aiq_working_mode_t hdr_mode = RK_AIQ_WORKING_MODE_NORMAL;
//hdr_mode = RK_AIQ_WORKING_MODE_ISP_HDR2;
SAMPLE_COMM_ISP_Init(0, hdr_mode, multi_sensor, iq_dir);
SAMPLE_COMM_ISP_Run(0);
// rkmpi init
if (RK_MPI_SYS_Init() != RK_SUCCESS) {
RK_LOGE("rk mpi sys init fail!");
return -1;
}
// rtsp init
// rtsp_demo_handle g_rtsplive = NULL;
// rtsp_session_handle g_rtsp_session;
// g_rtsplive = create_rtsp_demo(554);
// g_rtsp_session = rtsp_new_session(g_rtsplive, "/live/0");
// rtsp_set_video(g_rtsp_session, RTSP_CODEC_ID_VIDEO_H264, NULL, 0);
// rtsp_sync_video_ts(g_rtsp_session, rtsp_get_reltime(), rtsp_get_ntptime());
// vi init
vi_dev_init();
vi_chn_init(0, width, height);
// vpss init
vpss_init(0, width, height);
// bind vi to vpss
MPP_CHN_S stSrcChn, stvpssChn;
stSrcChn.enModId = RK_ID_VI;
stSrcChn.s32DevId = 0;
stSrcChn.s32ChnId = 0;
stvpssChn.enModId = RK_ID_VPSS;
stvpssChn.s32DevId = 0;
stvpssChn.s32ChnId = 0;
printf("====RK_MPI_SYS_Bind vi0 to vpss0====\n");
s32Ret = RK_MPI_SYS_Bind(&stSrcChn, &stvpssChn);
if (s32Ret != RK_SUCCESS) {
RK_LOGE("bind 0 ch venc failed");
return -1;
}
// venc init
// RK_CODEC_ID_E enCodecType = RK_VIDEO_ID_AVC;
// venc_init(0, width, height, enCodecType);
// printf("venc init success\n");
while(!quit)
{
// get vpss frame
s32Ret = RK_MPI_VPSS_GetChnFrame(0,0, &stVpssFrame,-1);
if(s32Ret == RK_SUCCESS)
{
void *data = RK_MPI_MB_Handle2VirAddr(stVpssFrame.stVFrame.pMbBlk);
//opencv
cv::Mat frame(height,width,CV_8UC3,data);
//cv::Mat frame640;
//cv::resize(frame, frame640, cv::Size(640,640), 0, 0, cv::INTER_LINEAR);
//letterbox
cv::Mat letterboxImage = letterbox(frame);
memcpy(rknn_app_ctx.input_mems[0]->virt_addr, letterboxImage.data, model_width*model_height*3);
inference_yolov5_model(&rknn_app_ctx, &od_results);
for(int i = 0; i < od_results.count; i++)
{
//获取框的四个坐标
if(od_results.count >= 1)
{
object_detect_result *det_result = &(od_results.results[i]);
printf("%s @ (%d %d %d %d) %.3f\n", coco_cls_to_name(det_result->cls_id),
det_result->box.left, det_result->box.top,
det_result->box.right, det_result->box.bottom,
det_result->prop);
sX = (int)(det_result->box.left );
sY = (int)(det_result->box.top );
eX = (int)(det_result->box.right );
eY = (int)(det_result->box.bottom );
mapCoordinates(&sX,&sY);
mapCoordinates(&eX,&eY);
cv::rectangle(frame,cv::Point(sX ,sY),
cv::Point(eX ,eY),
cv::Scalar(0,255,0),3);
sprintf(text, "%s %.1f%%", coco_cls_to_name(det_result->cls_id), det_result->prop * 100);
cv::putText(frame,text,cv::Point(sX, sY - 8),
cv::FONT_HERSHEY_SIMPLEX,1,
cv::Scalar(0,255,0),2);
}
}
memcpy(data, frame.data, width * height * 3);
}
// send stream
// encode H264
// RK_MPI_VENC_SendFrame(0, &stVpssFrame,-1);
// rtsp
// s32Ret = RK_MPI_VENC_GetStream(0, &stFrame, -1);
// if(s32Ret == RK_SUCCESS)
// {
// if(g_rtsplive && g_rtsp_session)
// {
// //printf("len = %d PTS = %d \n",stFrame.pstPack->u32Len, stFrame.pstPack->u64PTS);
// void *pData = RK_MPI_MB_Handle2VirAddr(stFrame.pstPack->pMbBlk);
// rtsp_tx_video(g_rtsp_session, (uint8_t *)pData, stFrame.pstPack->u32Len,
// stFrame.pstPack->u64PTS);
// rtsp_do_event(g_rtsplive);
// }
// }
// release frame
s32Ret = RK_MPI_VPSS_ReleaseChnFrame(0, 0, &stVpssFrame);
if (s32Ret != RK_SUCCESS) {
RK_LOGE("RK_MPI_VI_ReleaseChnFrame fail %x", s32Ret);
}
// s32Ret = RK_MPI_VENC_ReleaseStream(0, &stFrame);
// if (s32Ret != RK_SUCCESS) {
// RK_LOGE("RK_MPI_VENC_ReleaseStream fail %x", s32Ret);
// }
// memset(text,0,8);
}
printf("Release\n");
RK_MPI_SYS_UnBind(&stSrcChn, &stvpssChn);
RK_MPI_VI_DisableChn(0, 0);
RK_MPI_VI_DisableDev(0);
RK_MPI_VPSS_StopGrp(0);
RK_MPI_VPSS_DestroyGrp(0);
// RK_MPI_VENC_StopRecvFrame(0);
// RK_MPI_VENC_DestroyChn(0);
// free(stFrame.pstPack);
// if (g_rtsplive)
// rtsp_del_demo(g_rtsplive);
SAMPLE_COMM_ISP_Stop(0);
RK_MPI_SYS_Exit();
// Release rknn model
release_yolov5_model(&rknn_app_ctx);
deinit_post_process();
return 0;
}
前提需要下载luckfox pico的sdk：https://github.com/LuckfoxTECH/luckfox-pico或者https://gitee.com/LuckfoxTECH/luckfox-pico，编译脚本：
export LUCKFOX_SDK_PATH=< 你的 Luckfox-pico SDK 路径>
cd /home/cw/zhouying/RV1103/luckfox_pico_rtsp_yolov5/
mkdir build
cd build
cmake ..
make && make install
# 在确定能adb连接开发板后将程序push进开发板
adb push ../luckfox_rtsp_yolov5_demo/ /
第三步：运行程序
adb shell连接开发板
top或者ps输出自启动的程序有哪些，需要kill掉一些程序释放出内存才能跑起来yolov5
[root@luckfox ]# top
Mem: 26820K used, 920K free, 228K shrd, 0K buff, 1604K cached
CPU: 17% usr 79% sys 0% nic 2% idle 0% io 0% irq 0% sirq
Load average: 11.21 3.00 1.02 2/106 426
PID PPID USER STAT VSZ %VSZ %CPU COMMAND
251 1 root S 92616 334% 51% rkipc -a /oem/usr/share/iqfiles
401 399 root D 31148 112% 8% smbd -D
425 420 root R 1796 6% 8% top
36 2 root SW 0 0% 8% [kswapd0]
424 423 root S 1800 6% 5% {default.script} /bin/sh /usr/share/ud
190 2 root SW 0 0% 5% [vcodec_thread_0]
193 1 root S 5368 19% 3% /usr/sbin/ntpd -g -p /var/run/ntpd.pid
355 1 root S 14392 52% 0% /usr/bin/adbd
314 1 root S 4852 17% 0% sshd: /usr/sbin/sshd [listener] 0 of 1
420 355 root S 1804 6% 0% /bin/sh -l
65 1 root S 1800 6% 0% {rcS} /bin/sh /etc/init.d/rcS
1 0 root S 1796 6% 0% init
423 251 root S 1792 6% 0% udhcpc -i usb0 -T 1 -A 0 -b -q
399 65 root S 1792 6% 0% {S91smb} /bin/sh /etc/init.d/S91smb st
71 1 root S 1788 6% 0% /sbin/syslogd -n
320 1 root S 1788 6% 0% /usr/sbin/telnetd -F
75 1 root S 1784 6% 0% /sbin/klogd -n
86 1 root S 1628 6% 0% /sbin/udevd -d
426 424 root D 1012 4% 0% /sbin/ifconfig usb0 up
41 2 root IW 0 0% 0% [kworker/u2:1-fl]
[root@luckfox ]# killall rkipc
[root@luckfox ]# killall smbd
[root@luckfox ]# killall sshd
[root@luckfox ]# killall ntpd
[root@luckfox ]# killall udhcpc
[root@luckfox ]# killall nmbd
然后就能运行程序了
[root@luckfox ]# cd luckfox_rtsp_yolov5_demo/
[root@luckfox luckfox_rtsp_yolov5_demo]# ./luckfox_rtsp_yolov5
index=0, name=images, n_dims=4, dims=[1, 640, 640, 3], n_elems=1228800, size=1228800, fmt=NHWC, type=INT8, qnt_type=AFFINE, zp=-128, scale=0.003922
index=0, name=output0, n_dims=4, dims=[1, 80, 80, 255], n_elems=1632000, size=1632000, fmt=NHWC, type=INT8, qnt_type=AFFINE, zp=-128, scale=0.003922
index=1, name=286, n_dims=4, dims=[1, 40, 40, 255], n_elems=408000, size=408000, fmt=NHWC, type=INT8, qnt_type=AFFINE, zp=-128, scale=0.003922
index=2, name=288, n_dims=4, dims=[1, 20, 20, 255], n_elems=102000, size=102000, fmt=NHWC, type=INT8, qnt_type=AFFINE, zp=-128, scale=0.003922
model is NHWC input fmt
model input height=640, width=640, channel=3
init rknn model success!
load lable ./model/coco_80_labels_list.txt
rkaiq log level ff0
ID: 0, sensor_name is m00_b_sc3336 4-0030, iqfiles is /etc/iqfiles
rk_aiq_uapi2_sysctl_init/prepare succeed
rk_aiq_uapi2_sysctl_start succeed
rockit log path (null), log_size = 0, can use export rt_log_path=, export rt_log_size= change
log_file = (nil)
RTVersion 00:02:18-532 {dump :064} ---------------------------------------------------------
RTVersion 00:02:18-534 {dump :065} rockit version: git-8cb4d25b8 Tue Feb 28 11:12:39 2023 +0800
RTVersion 00:02:18-534 {dump :066} rockit building: built- 2023-02-28 15:23:19
RTVersion 00:02:18-535 {dump :067} ---------------------------------------------------------
(null) 00:02:18-535 {log_level_init :203}
please use echo name=level > /tmp/rt_log_level set log level
name: all cmpi mb sys vdec venc rgn vpss vgs tde avs wbc vo vi ai ao aenc adec
log_level: 0 1 2 3 4 5 6
rockit default level 4, can use export rt_log_level=x, x=0,1,2,3,4,5,6 change
(null) 00:02:18-536 {read_log_level :093} text is all=4
(null) 00:02:18-536 {read_log_level :095} module is all, log_level is 4
(null) 00:02:18-541 {monitor_log_level :144} #Start monitor_log_level thread, arg:(nil)
RTIsp3x 00:02:18-567 {ispInitDevice :208} sensor name = m00_b_sc3336 4-0030
RTIsp3x 00:02:18-567 {ispInitDevice :211} sensor_index = 0
RTIsp3x 00:02:18-573 {ispInitDevice :208} sensor name = m00_b_sc3336 4-0030
RTIsp3x 00:02:18-573 {ispInitDevice :211} sensor_index = 0
vi_dev_init
RKViDev 00:02:18-587 {vi_set_dev_attr :440} VI_DEV_ATTR_S all parameter reserved
=== VI ATTRS INFO: ===
devId : 0
chnId : 0
buffcnt: 0
buffSize: 0
width: 0
height: 0
Maxwidth: 0
Maxwidth: 0
streaming: 0
RKViChn 00:02:18-593 {prepareRuntime :334} ---------------------------------------------------------
RKViChn 00:02:18-593 {prepareRuntime :336} vi version: 1.86, name:vvi
RKViChn 00:02:18-594 {prepareRuntime :337} rockit-ko version: vmpi:fb2eed2be49e
RKViChn 00:02:18-594 {prepareRuntime :338} rockit-ko building: -2023-02-09-11:04:49
RKViChn 00:02:18-594 {prepareRuntime :339} ---------------------------------------------------------
RKViChn 00:02:18-600 {prepareRuntime :358} mb pool create success, MBCnt= 2
cmpi 00:02:18-601 {createRuntime :546} [non-WRAP MODE]: buff size = 518400
RTIsp3x 00:02:18-607 {ispInitDevice :208} sensor name = m00_b_sc3336 4-0030
RTIsp3x 00:02:18-607 {ispInitDevice :211} sensor_index = 0
RTDeviceV4L2 00:02:18-611 {open :138} open video name(/dev/video11)
RKViChn 00:02:18-612 {setFrameRate :1153} [vi] dev(0) ch(0) illegal param s32SrcFrameRate(0) s32DstFrameRate(0)
RTDeviceV4L2 00:02:18-614 {ispCameraInfo :549} current device:/dev/video11 isn't compatible(cap:0x84201000) device,memoryType:4, retry:0
RTDeviceV4L2 00:02:18-615 {ispInitFormat :726} ioctl VIDIOC_S_FMT OK
RKViChn 00:02:18-616 {start :813} =========== vi Start startRuntime ===========
RTDeviceV4L2 00:02:18-616 {ispStreamOn :440} do ispStreamOn start
RTDeviceV4L2 00:02:18-638 {ispStreamOn :493} do ispStreamOn done
vpss_init
=== 0 ===
=== 1 ===
rga_api version 1.10.0_[2]
====RK_MPI_SYS_Bind vi0 to vpss0====
cmpi 00:02:18-721 {mb_get_buffer_by_i:422} allocated buffer(this=0x2e78b0, data=(nil), size=0, id=-1)
traffic light @ (27 239 81 297) 0.266
person @ (185 285 488 532) 0.465
laptop @ (533 380 619 500) 0.465
person @ (120 234 356 501) 0.294
laptop @ (464 344 535 430) 0.286
person @ (206 272 506 528) 0.424
person @ (577 297 640 513) 0.251
person @ (215 285 535 527) 0.592
person @ (190 291 427 523) 0.441
laptop @ (529 385 592 455) 0.364
person @ (190 294 423 522) 0.599
bottle @ (524 380 585 456) 0.263
person @ (229 351 445 529) 0.421
^Csignal 2
Release
RTDeviceV4L2 00:02:25-737 {ispStreamOff :503} do ispStreamOff start
RTDeviceV4L2 00:02:25-802 {ispStreamOff :514} do ispStreamOff done
RTDeviceV4L2 00:02:25-803 {close :365} do RTDeviceV4L2 close
rk_aiq_uapi2_sysctl_stop enter
rk_aiq_uapi2_sysctl_deinit enter
MessageParser process loop exit!
rk_aiq_uapi2_sysctl_deinit exit
RKSockServer 00:02:26-568 {start :162} accept failed
Release success
