目录
- 引言
- 前期准备
- Step1 pt格式转onnx格式
- Step2 创建工程文件并安装NuGet Package
- Step3 预处理API部署YOLO模型
- Step4 前处理+推理+后处理
- Step5 异步推理
- 参考博客
引言
YOLO（You Only Look Once）作为一个目标检测算法，其调用的接口为Python语言，通常情况下需要将YOLO模型的pt格式转换为其他格式的模型供其他编程语言调用，常见的onnx格式可以支持C++/C#/Python等多种语言的调用，本文教程主要介绍如何将YOLO模型的pt格式转换为onnx格式，并在Window环境下用C#调用模型并推理。
前期准备
开始前建议先下载参考项目源码，在Anaconda中重新配置一个Python3.10的环境，安装的PackageList如下：
Step1 pt格式转onnx格式
from ultralytics import YOLO
import os
# Model Path
save_model_name = os.path.normpath(os.path.join(os.getcwd(), './yolov10-main/runs/detect/mytrain_0806/weights/best.pt'))
# Load model
model = YOLO(os.path.normpath(save_model_name))
# Create 'best.onnx'
model.export(format="onnx", dynamic=False, simplify=True)
save_model_name替换成pt文件所在的路径
生成的onnx文件也在save_model_name路径下
Step2 创建工程文件并安装NuGet Package
Step3 预处理API部署YOLO模型
将OpenVINO部署方式封装为接口，只需要实例化Predictor类即可加载模型
using OpenCvSharp.Dnn;
using OpenCvSharp;
using OpenVinoSharp;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.InteropServices;
using System.Text;
using System.Threading.Tasks;
using OpenVinoSharp.preprocess;
using System.Diagnostics;
using OpenVinoSharp.Extensions;
using OpenVinoSharp.Extensions.model;
namespace DeepLearningDotNet
{
public class Predictor : IDisposable
{
/// <summary>
/// OpenVINO Runtime Core
/// </summary>
private Core core;
/// <summary>
/// 加载前处理API的模型
/// </summary>
private Model model;
/// <summary>
/// 加载到设备中的模型
/// </summary>
private CompiledModel compiled;
/// <summary>
/// OpenVINO推理接口
/// </summary>
public InferRequest openvino_api_infer;
/// <summary>
/// OpenCV推理接口
/// </summary>
public Net opencv_infer;
/// <summary>
/// 接口类型
/// </summary>
private string engine;
/// <summary>
/// 模型图片尺寸
/// </summary>
private int[] input_size;
public Predictor() { }
/// <summary>
/// 实例化推理器
/// </summary>
/// <param name="model_path"></param>
/// <param name="engine"></param>
/// <param name="device"></param>
/// <exception cref="ArgumentNullException"></exception>
public Predictor(string model_path, string engine = "OpenVINO", string device = "AUTO", int[] input_size = null)
{
//判断路径是否合法
if (model_path == null || model_path == "" || !File.Exists(model_path))
{
throw new ArgumentNullException(nameof(model_path));
}
this.engine = engine;
if (engine == "OpenVINO")
{
// -------- Step 1. Initialize OpenVINO Runtime Core --------
core = new Core();
//判断设备是否可用
if (!core.get_available_devices().Contains(device))
{
throw new ArgumentNullException(nameof(device));
}
// -------- Step 2. Read inference model --------
Model tempModel = core.read_model(model_path);
OvExtensions.printf_model_info(tempModel);
PrePostProcessor processor = new PrePostProcessor(tempModel);
this.input_size = input_size ?? new int[] { 1, 3, 640, 640 };
Tensor input_tensor_pro = new Tensor(new OvType(ElementType.U8), new Shape(this.input_size[0], this.input_size[2], this.input_size[3], this.input_size[1]));
InputInfo input_info = processor.input(0);
InputTensorInfo input_tensor_info = input_info.tensor();
input_tensor_info.set_from(input_tensor_pro).set_layout(new Layout("NHWC")).set_color_format(ColorFormat.BGR);
PreProcessSteps process_steps = input_info.preprocess();
process_steps.convert_color(ColorFormat.RGB).resize(ResizeAlgorithm.RESIZE_LINEAR)
.convert_element_type(new OvType(ElementType.F32)).scale(255.0f).convert_layout(new Layout("NCHW"));
model = processor.build();
// -------- Step 3. Loading a model to the device --------
compiled = core.compile_model(model, device);
// -------- Step 4. Create an infer request --------
openvino_api_infer = compiled.create_infer_request();
}
if (engine == "OpenCv")
{
opencv_infer = CvDnn.ReadNetFromOnnx(model_path);
}
}
public void Dispose()
{
openvino_api_infer.Dispose();
opencv_infer.Dispose();
compiled.Dispose();
model.Dispose();
core.Dispose();
GC.Collect();
}
/// <summary>
/// OpenVINO推理方法
/// </summary>
/// <param name="input_data"></param>
/// <param name="input_names"></param>
/// <param name="input_size"></param>
/// <param name="output_names"></param>
/// <param name="output_sizes"></param>
/// <returns></returns>
public List<float[]> OpenVinoInfer(Mat img, List<string> input_names, int[] input_size, List<string> output_names, List<int[]> output_sizes)
{
List<float[]> returns = new List<float[]>();
try
{
// -------- Step 6. Set up input data --------
if (set_input_tensor_data(img))
{
// -------- Step 7. Do inference synchronously --------
openvino_api_infer.infer();
// -------- Step 8. Get infer result data --------
Tensor output_tensor = openvino_api_infer.get_output_tensor();
int output_length = (int)output_tensor.get_size();
float[] output_data = output_tensor.get_data<float>(output_length);
returns.Add(output_data);
}
return returns;
}
catch { return returns; }
}
/// <summary>
/// 加载数据到推理器中
/// 该函数不能放在继承类中会内存泄露
/// </summary>
/// <param name="img">输入前处理后的img</param>
/// <returns></returns>
public bool set_input_tensor_data(Mat img)
{
try
{
//输入的图片未经过前处理则不加载到推理器中
if (Math.Max(img.Size().Width, img.Size().Height) != input_size[2]
&& Math.Max(img.Size().Width, img.Size().Height) != input_size[3])
return false;
// 从推理器中获取输入数据格式
Tensor input_tensor = openvino_api_infer.get_input_tensor();
Shape input_shape = input_tensor.get_shape();
byte[] input_data = new byte[input_shape[1] * input_shape[2] * input_shape[3]];
// 通过非托管指针img数据写入到input_data数组
Marshal.Copy(img.Ptr(0), input_data, 0, input_data.Length);
IntPtr destination = input_tensor.data();
// 把input_tensor的指针指向input_data数组
Marshal.Copy(input_data, 0, destination, input_data.Length);
return true;
}
catch { return false; }
}
/// <summary>
/// OpenCv推理方法
/// </summary>
/// <param name="input_data"></param>
/// <param name="input_names"></param>
/// <param name="input_size"></param>
/// <param name="output_names"></param>
/// <param name="output_sizes"></param>
/// <returns></returns>
public List<float[]> OpenCvInfer(Mat img, List<string> input_names, int[] input_size, List<string> output_names, List<int[]> output_sizes)
{
List<float[]> returns = new List<float[]>();
float[] input_data = OpenVinoSharp.Extensions.process.Permute.run(img);
var input_tensor = openvino_api_infer.get_input_tensor();
input_tensor.set_data(input_data);
openvino_api_infer.infer();
foreach (var name in output_names)
{
var output_tensor = openvino_api_infer.get_tensor(name);
returns.Add(output_tensor.get_data<float>((int)output_tensor.get_size()));
}
return returns;
}
}
}
OpenVINO提供了预处理C#API可以将前处理加载到模型中使用GPU进行前处理加快运行速度
只需要实例化PrePostProcessor将输入和输出数据的格式指定后将build的新模型加载到推理器中即可
Step4 前处理+推理+后处理
将OpenVINO推理方式封装为接口，只需要实例化DeepLearning类即可推理
using OpenCvSharp;
using OpenCvSharp.Dnn;
using OpenVinoSharp;
using OpenVinoSharp.Extensions.result;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Runtime.InteropServices;
using System.Text;
using System.Threading.Tasks;
using static OpenCvSharp.FileStorage;
namespace DeepLearningDotNet
{
public class OpenVinoDet : DeepLearning
{
public OpenVinoDet(string model_path, string engine, string device, int categ_nums, float det_thresh, float det_nms_thresh, int[] input_size)
: base(model_path, engine, device, categ_nums, det_thresh, det_nms_thresh, new int[] { 1, 3, input_size[0], input_size[1] },
new List<string> { "images" }, new List<int[]> { new int[] { 1, 4 + categ_nums, 8400 } }, new List<string> { "output0" })
{
}
/// <summary>
/// OpenVino推理接口
/// </summary>
/// <param name="img"></param>
/// <returns></returns>
public BaseResult predict(Mat img)
{
if (img.Cols == (int)0)
return new BaseResult();
List<float[]> result_data;
//前处理
Mat afterPreImage = preprocess(img);
//推理
result_data = m_predictor.OpenVinoInfer(afterPreImage, m_input_names, m_input_size, m_output_names, m_output_sizes);
//后处理
BaseResult re = postprocess(result_data);
return re;
}
/// <summary>
/// API前处理方法
/// </summary>
/// <param name="img"></param>
/// <returns></returns>
public override Mat preprocess(Mat img)
{
m_image_size = new List<int> { (int)img.Size().Width, (int)img.Size().Height };
//OpenCv图像的BGR通道转换对齐到模型图片的RGB通道【利用API前处理会自动对齐】
//Cv2.CvtColor(img, img, ColorConversionCodes.BGR2RGB);
不变形缩放【保留图片的宽高比，缺少的部分用灰色图像填充】
img = OpenVinoSharp.Extensions.process.Resize.letterbox_img(img, (int)m_input_size[2], out m_factor);
//将像素值从0-255映射成0-1归一化【利用API前处理会自动归一化】
//img = OpenVinoSharp.Extensions.process.Normalize.run(img, true);
return img;
}
/// <summary>
/// 后处理方法
/// </summary>
/// <param name="results"></param>
/// <returns></returns>
public override BaseResult postprocess(List<float[]> results)
{
List<Rect> positionBoxes = new List<Rect>();
List<int> classIds = new List<int>();
List<float> confidences = new List<float>();
//图像前处理后推理将结果还原到原图上
for (int i = 0; i < results[0].Length / 6; i++)
{
int s = 6 * i;
if ((float)results[0][s + 4] > m_det_thresh)
{
float cx = results[0][s + 0];
float cy = results[0][s + 1];
float dx = results[0][s + 2];
float dy = results[0][s + 3];
//坐标缩放
int x = (int)((cx) * m_factor);
int y = (int)((cy) * m_factor);
//尺寸缩放
int width = (int)((dx - cx) * m_factor);
int height = (int)((dy - cy) * m_factor);
Rect box = new Rect();
box.X = x;
box.Y = y;
box.Width = width;
box.Height = height;
positionBoxes.Add(box);
classIds.Add((int)results[0][s + 5]);
confidences.Add((float)results[0][s + 4]);
}
}
DetResult re = new DetResult();
int[] indices;
//利用OpenCv非极大值抑制来筛选检测结果
CvDnn.NMSBoxes(positionBoxes, confidences, m_det_thresh, m_det_nms_thresh, out indices);
foreach (int i in indices)
{
re.add(classIds[i], confidences[i], positionBoxes[i]);
}
return re;
}
/// <summary>
/// 将结果绘制到图上
/// </summary>
/// <param name="bresult"></param>
/// <param name="image"></param>
/// <returns></returns>
public static Mat draw_det_result(BaseResult bresult, Mat image)
{
Mat mat = new Mat();
image.CopyTo(mat);
DetResult detResult = bresult as DetResult;
for (int i = 0; i < detResult.count; i++)
{
Cv2.Rectangle(mat, detResult.datas[i].box, new Scalar(0.0, 0.0, 255.0), 2);
Cv2.Rectangle(mat, new Point(detResult.datas[i].box.TopLeft.X, detResult.datas[i].box.TopLeft.Y + 30), new Point(detResult.datas[i].box.BottomRight.X, detResult.datas[i].box.TopLeft.Y), new Scalar(0.0, 255.0, 255.0), -1);
Cv2.PutText(mat, detResult.datas[i].lable + "-" + detResult.datas[i].score.ToString("0.00"), new Point(detResult.datas[i].box.X, detResult.datas[i].box.Y + 25), HersheyFonts.HersheySimplex, 0.8, new Scalar(0.0, 0.0, 0.0), 2);
}
return mat;
}
}
}
已经提前将预处理C#API加载到模型中不需要再进行通道对齐和归一化处理
通过分数和NMS来后处理筛选最终检测结果
Step5 异步推理
从下图可以看出使用Async异步推理的方式可以极大的提高系统的效率
/// <summary>
/// 实时检测
/// </summary>
public void video_api_predict()
{
//目标检测
string model_type = "YOLOv10Det";
string engine_type = "OpenVINO";
string device = "CPU";
float score = 0.2f;
float nms = 0.5f;
int categ_num = 80;
int[] input_size = { 640, 640 };
//目前还未能使用OpenVINO调用iGPU进行处理
DeepLearning curr_predictor = DeepLearning.GetYOLO(model_type, model_path, engine_type, device, categ_num, score, nms, input_size);
DeepLearning next_predictor = DeepLearning.GetYOLO(model_type, model_path, engine_type, device, categ_num, score, nms, input_size);
Mat frame_curr = CaptureVideo();
/*先加载首帧图片进行推理*/
if (frame_curr.Cols != 0)
{
try
{
//使用API前处理图片
Mat img = ((OpenVinoDet)curr_predictor).preprocess(frame_curr);
//数据加载到推理器中
((OpenVinoDet)curr_predictor).m_predictor.set_input_tensor_data(img);
//异步启动推理器进行推理
((OpenVinoDet)curr_predictor).m_predictor.openvino_api_infer.start_async();
}
catch (Exception e)
{
MessageBox.Show(e.Message);
}
}
while (true)
{
if (!isCaptureVideo)
{
Thread.Sleep(50);
continue;
}
/*这个过程相当于提前把下一帧的图片丢进去推理与首帧的图片后处理并行*/
Mat frame_next = CaptureVideo();
sw.Restart();
if (frame_next.Cols != 0)
{
try
{
//使用API前处理图片
Mat img = ((OpenVinoDet)next_predictor).preprocess(frame_next);
//数据加载到推理器中
((OpenVinoDet)next_predictor).m_predictor.set_input_tensor_data(img);
//异步启动推理器进行推理
((OpenVinoDet)next_predictor).m_predictor.openvino_api_infer.start_async();
}
catch (Exception e){ }
}
//等待首帧处理完毕
((OpenVinoDet)curr_predictor).m_predictor.openvino_api_infer.wait();
List<float[]> result_data = new List<float[]>();
foreach (var name in ((OpenVinoDet)curr_predictor).m_output_names)
{
var output_tensor = ((OpenVinoDet)curr_predictor).m_predictor.openvino_api_infer.get_tensor(name);
result_data.Add(output_tensor.get_data<float>((int)output_tensor.get_size()));
}
//异步进行绘制会出现内存无法访问的情况
//this.BeginInvoke(() =>
/*首帧图片已经处理完成进行绘制*/
this.Invoke(() =>
{
//后处理首帧图片
BaseResult result = ((OpenVinoDet)curr_predictor).postprocess(result_data);
result.update_lable(class_names);
Mat re_img = new Mat();
re_img = OpenVinoDet.draw_det_result(result, frame_curr);
if (re_img.Cols != 0)
{
sw.Stop();
if(sw.ElapsedMilliseconds != 0)
PredictTimes.Add(1000 / sw.ElapsedMilliseconds);
pictureBox2.Image = OpenCvSharp.Extensions.BitmapConverter.ToBitmap(re_img);
DetResult detResult = result as DetResult;
textBox8.Text = detResult.count.ToString();
for (int i = 0; i < detResult.count; i++)
{
textBox1.Text = detResult.datas[i].lable;
textBox2.Text = detResult.datas[i].score.ToString("0.00");
int X = detResult.datas[i].box.TopLeft.X;
int Y = detResult.datas[i].box.TopLeft.Y;
int W = detResult.datas[i].box.Width;
int H = detResult.datas[i].box.Height;
textBox3.Text = X.ToString();
textBox4.Text = Y.ToString();
textBox5.Text = W.ToString();
textBox6.Text = H.ToString();
}
if (PredictTimes.Count > 10)
{
textBox9.Text = (PredictTimes.Average()).ToString("00");
PredictTimes.Clear();
}
}
});
//首帧处理完毕后交换到下一帧（包括推理器和图片）
DeepLearning tempInferRequest;
tempInferRequest = (OpenVinoDet)curr_predictor;
curr_predictor = next_predictor;
next_predictor = tempInferRequest;
frame_next.CopyTo(frame_curr);
}
}
/// <summary>
/// 模拟实时取图
/// </summary>
/// <returns></returns>
public Mat CaptureVideo()
{
// 生成一个范围在 0 到 39 之间的随机整数用于获取图片模拟实时取图
Random random = new Random();
int randomNumber = random.Next(0, 40);
string image_path = string.Format("D:\\MyCode\\yolo_train\\script\\source\\images\\{0}.png", randomNumber);
Mat img = new Mat();
double startTime = Convert.ToDouble(DateTime.Now.ToString("mmss.ff")) * 1000;
img = Cv2.ImRead(image_path);
double endTime = Convert.ToDouble(DateTime.Now.ToString("mmss.ff")) * 1000;
this.Invoke(new Action(() =>
{
CaptureTimes.Add(endTime - startTime);
if(CaptureTimes.Count > 10)
{
textBox10.Text = (CaptureTimes.Average()).ToString("00");
CaptureTimes.Clear();
}
}));
return img;
}
OpenVINO提供了推理请求(Infer Request)机制，来实现在指定的推理设备上以同步或异步方式运行AI模型
创建infer_request对象
将数据传入模型的指定输入节点set_tensor(input_node, input_tensor)
通过非阻塞(non-blocking)的方式启动推理计算start_async()
阻塞等待推理计算结束wait()
从模型的指定输出节点获取推理结果get_tensor(output_node)
OpenVINO推理YOLOv10模型
参考博客
1.用 OpenVINO™ C# API 在intel平台部署 YOLOv10 目标检测模型
2.【OpenVINO™】使用 OpenVINO™ C# 异步推理接口部署YOLOv8 ——在Intel IGPU 上速度依旧飞起！！
3.在蝰蛇峡谷上实现YOLOv5模型的OpenVINO异步推理程序
