Halcon深度学习，异常值缺陷检测

前言

halcon深度学习分为常见的4大类。分类，语义分割，异常值检测，深度OCR。本篇主要针对halcon的异常值检测，如何训练和部署，并通过图像预处理的方式实现对异常值缺陷检测的精准实现。
异常值检测不同于语义分割的项目，异常值检测可以仅训练OK图像，不训练NG图像，但是使用NG图像训练可以显著提高准确度

1.总结程序

ImageDir := 'E:/New/图像文件/异常值检测单块'
ImageSubDirs := ['OK','NG']
OutputDir := 'E:/New/图像文件/异常值检测单块/样品'
ExampleSpecificPreprocessing := true

ImageWidth := 160
ImageHeight := 160

Complexity := 15

*******************1.拆分数据集***************
create_dict (GenParamDataset)
set_dict_tuple (GenParamDataset, 'image_sub_dirs', ImageSubDirs)
read_dl_dataset_anomaly (ImageDir, [], [], [], GenParamDataset, DLDataset)
* 

split_dl_dataset (DLDataset, 50, 10, [])
* 
* 加载异常值检测模型
read_dl_model ('initial_dl_anomaly_medium.hdl', DLModelHandle)
set_dl_model_param (DLModelHandle, 'image_width', ImageWidth)
set_dl_model_param (DLModelHandle, 'image_height', ImageHeight)
set_dl_model_param (DLModelHandle, 'complexity', Complexity)
* 校验GPU
query_available_dl_devices (['runtime','id'], ['gpu',0], DLDevice)
set_dl_model_param (DLModelHandle, 'device', DLDevice)
* 
* 设置参数和预处理模型
create_dict (PreprocessSettings)
set_dict_tuple (PreprocessSettings, 'overwrite_files', true)
create_dl_preprocess_param ('anomaly_detection', ImageWidth, ImageHeight, 3, [], [], 'constant_values', 'full_domain', [], [], [], [], DLPreprocessParam)
preprocess_dl_dataset (DLDataset, OutputDir, DLPreprocessParam, PreprocessSettings, DLDatasetFileName)
* 
get_dict_tuple (DLDataset, 'samples', DatasetSamples)
if (ExampleSpecificPreprocessing)
    read_dl_samples (DLDataset, [0:|DatasetSamples| - 1], DLSampleBatch)
    preprocess_dl_samples_bottle (DLSampleBatch)
    write_dl_samples (DLDataset, [0:|DatasetSamples| - 1], DLSampleBatch, [], [])
endif
*随机抽取检查标注情况
create_dict (WindowDict)
for Index := 0 to 9 by 1
    SampleIndex := int(rand(1) * |DatasetSamples|)
    read_dl_samples (DLDataset, SampleIndex, DLSample)
    dev_display_dl_data (DLSample, [], DLDataset, 'anomaly_ground_truth', [], WindowDict)
    dev_disp_text ('Press Run (F5) to continue', 'window', 'bottom', 'right', 'black', [], [])
    * 
    get_dict_tuple (WindowDict, 'anomaly_ground_truth', WindowHandles)
    dev_set_window (WindowHandles[0])
    dev_disp_text ('Preprocessed image', 'window', 'top', 'left', 'black', [], [])
    * 
    stop ()
endfor
dev_close_window_dict (WindowDict)
* ***   2.) 训练   ***
* 
* 创建训练参数
* 
* C显示训练进度
EnableDisplay := true
* 
*设置最小训练误差和最大迭代次数
ErrorThreshold := 0.001
MaxNumEpochs := 30
* 
*控制每张图像的大小训练参数，越大可以增加准确率，但是训练时间会显著增加
DomainRatio := 0.25
* 
* 设置稳健性，当训练失败时，可以调高参数值
RegularizationNoise := 0.01
* 
create_dict (TrainParamAnomaly)
set_dict_tuple (TrainParamAnomaly, 'regularization_noise', RegularizationNoise)
set_dict_tuple (TrainParamAnomaly, 'error_threshold', ErrorThreshold)
set_dict_tuple (TrainParamAnomaly, 'domain_ratio', DomainRatio)
create_dl_train_param (DLModelHandle, MaxNumEpochs, [], EnableDisplay, 73, 'anomaly', TrainParamAnomaly, TrainParam)
* 
* 调用数据集，并开始训练
train_dl_model (DLDataset, DLModelHandle, TrainParam, 0, TrainResults, TrainInfos, EvaluationInfos)
dev_disp_text ('Press Run (F5) to continue', 'window', 'bottom', 'right', 'black', [], [])
stop ()
* 
dev_close_window ()
* ***   3.评估模型   ***
* 
* 计算异常评分系数，当缺陷过小时，可以提高系数，使得评估更准确
StandardDeviationFactor := 1.0
set_dl_model_param (DLModelHandle, 'standard_deviation_factor', StandardDeviationFactor)
* 
*预估阀值，用于分割缺陷和背景
create_dict (GenParamThreshold)
set_dict_tuple (GenParamThreshold, 'enable_display', 'true')
compute_dl_anomaly_thresholds (DLModelHandle, DLDataset, GenParamThreshold, AnomalySegmentationThreshold, AnomalyClassificationThresholds)
dev_disp_text ('Press Run (F5) to continue', 'window', 'bottom', 'right', 'black', [], [])
stop ()
* 
dev_close_window ()
* 
* 设置评估参数跟评估模型
create_dict (GenParamEvaluation)
set_dict_tuple (GenParamEvaluation, 'measures', 'all')
set_dict_tuple (GenParamEvaluation, 'anomaly_classification_thresholds', AnomalyClassificationThresholds)
evaluate_dl_model (DLDataset, DLModelHandle, 'split', 'test', GenParamEvaluation, EvaluationResult, EvalParams)
* 
* 设置显示参数
create_dict (GenParamDisplay)
*显示评估模型
set_dict_tuple (GenParamDisplay, 'display_mode', ['score_histogram','score_legend'])
create_dict (WindowDict)
dev_display_anomaly_detection_evaluation (EvaluationResult, EvalParams, GenParamDisplay, WindowDict)
dev_disp_text ('Press Run (F5) to continue', 'window', 'bottom', 'right', 'black', 'box', 'true')
stop ()
* 
dev_close_window_dict (WindowDict)
* 
* 可视化评估参数
set_dict_tuple (GenParamDisplay, 'display_mode', ['pie_charts_precision','pie_charts_recall','absolute_confusion_matrix'])
* 
ClassificationThresholdIndex := |AnomalyClassificationThresholds| - 1
set_dict_tuple (GenParamDisplay, 'classification_threshold_index', 0)
create_dict (WindowDict)
dev_display_anomaly_detection_evaluation (EvaluationResult, EvalParams, GenParamDisplay, WindowDict)
dev_disp_text ('Press Run (F5) to continue', 'window', 'bottom', 'right', 'black', [], [])
stop ()
* 
dev_close_window_dict (WindowDict)
* 
* 写出模型
write_dl_model (DLModelHandle, 'E:/New/图像文件/异常值检测/model_final.hdl')


* ***   4.推理   ***

InferenceClassificationThreshold := AnomalyClassificationThresholds[0]

InferenceSegmentationThreshold := AnomalySegmentationThreshold
create_dict (DLDatasetInfo)
set_dict_tuple (DLDatasetInfo, 'class_names', ['OK','NG'])
set_dict_tuple (DLDatasetInfo, 'class_ids', [0,1])
create_dict (WindowDict)

* Image Acquisition 01: Code generated by Image Acquisition 01
list_files ('E:/New/图像文件/异常值检测单块/OK', ['files','follow_links'], ImageFiles)
tuple_regexp_select (ImageFiles, ['\\.(tif|tiff|gif|bmp|jpg|jpeg|jp2|png|pcx|pgm|ppm|pbm|xwd|ima|hobj)$','ignore_case'], ImageFiles)
for Index := 0 to |ImageFiles| - 1 by 1
    read_image (Image, ImageFiles[Index])
    * Image Acquisition 01: Do something
    dev_set_color ('red')
    *预处理图像，将所要检测的区域提取出
    threshold (Image, Regions, 29, 156)
    closing_circle (Regions, RegionClosing, 20)
    fill_up (RegionClosing, RegionFillUp)
    reduce_domain (Image, RegionFillUp, ImageReduced)
    *缩放图像，将图像缩放到与训练预处理一致，方便筛选
    get_image_size (ImageReduced, Width, Height)
    zoom_image_size (Image, ImageZoom,160,160, 'constant')
    zoom_region (RegionFillUp, RegionZoom,0.89,0.89)
    *将图像加载进行推理
    gen_dl_samples_from_images (Image, DLSample)
    preprocess_dl_samples (DLSample, DLPreprocessParam)
    apply_dl_model (DLModelHandle, DLSample, [], DLResult)

      read_dict_object (AnomalyImage, DLResult, 'anomaly_image')
    read_dict_tuple (DLResult, 'anomaly_score', AnomalyScore)
    * 
    *使用二值化选择所需要的区域
    threshold (AnomalyImage, AnomalyRegion,0.2,0.6)
    *对图像取交集，提取所检测的区域
    intersection (AnomalyRegion, RegionZoom, RegionIntersection)
    connection (RegionIntersection, ConnectedRegions)
    select_shape (ConnectedRegions, SelectedRegions, 'area', 'and', 100, 50000000)
     gen_contour_region_xld (SelectedRegions, Contours, 'border')
     *统计NG的区域个数
     count_obj (Contours, Number)
     dev_get_window (WindowHandle)
     if (Number>0)
         disp_message (WindowHandle, 'NG', 'window', 12, 12, 'black', 'true')
         stop ()
     else
         disp_message (WindowHandle, 'OK', 'window', 12, 12, 'black', 'true')
     endif
     
     reduce_domain (ImageZoom, RegionZoom, ImageReduced1)
     dev_display (ImageReduced1)
     dev_set_color ('green')
     dev_display (Contours)
endfor
dev_close_window_dict (WindowDict)

2.程序分析

2.1预处理和选择路径

*图像输入
ImageDir := 'E:/New/图像文件/异常值检测单块'
*检测数组
ImageSubDirs := ['OK','NG']
*预处理输出路径
OutputDir := 'E:/New/图像文件/异常值检测单块/样品'
ExampleSpecificPreprocessing := true

ImageWidth := 160
ImageHeight := 160

Complexity := 15

*******************1.拆分数据集***************
create_dict (GenParamDataset)
set_dict_tuple (GenParamDataset, 'image_sub_dirs', ImageSubDirs)
read_dl_dataset_anomaly (ImageDir, [], [], [], GenParamDataset, DLDataset)

注意事项：1.选择输入的预处理图像时，需要满足图像的大小是20的进制倍数。当我们训练的图像非常小时，比如我要训练的图像200190 。那么应该选取的是160160的预处理模型。预处理模型只支持：40，80,160,320,640，4中预处理格式，如果设置的格式错误是预处理将无法进行下去

2.2设置训练参数

 ***   2.) 训练   ***
* 
* 创建训练参数
* 
* 显示训练进度
EnableDisplay := true
* 
*设置最小训练误差和最大迭代次数
ErrorThreshold := 0.001
MaxNumEpochs := 30
* 
*控制每张图像的大小训练参数，越大可以增加准确率，但是训练时间会显著增加
DomainRatio := 0.25
* 
* 设置稳健性，当训练失败时，可以调高参数值
RegularizationNoise := 0.01
* 
create_dict (TrainParamAnomaly)
set_dict_tuple (TrainParamAnomaly, 'regularization_noise', RegularizationNoise)
set_dict_tuple (TrainParamAnomaly, 'error_threshold', ErrorThreshold)
set_dict_tuple (TrainParamAnomaly, 'domain_ratio', DomainRatio)
create_dl_train_param (DLModelHandle, MaxNumEpochs, [], EnableDisplay, 73, 'anomaly', TrainParamAnomaly, TrainParam)
* 
* 调用数据集，并开始训练
train_dl_model (DLDataset, DLModelHandle, TrainParam, 0, TrainResults, TrainInfos, EvaluationInfos)
dev_disp_text ('Press Run (F5) to continue', 'window', 'bottom', 'right', 'black', [], [])
stop ()

2.3显示评估混淆矩阵

 ***   3.评估模型   ***
* 
* 计算异常评分系数，当缺陷过小时，可以提高系数，使得评估更准确
StandardDeviationFactor := 1.0
set_dl_model_param (DLModelHandle, 'standard_deviation_factor', StandardDeviationFactor)
* 
*预估阀值，用于分割缺陷和背景
create_dict (GenParamThreshold)
set_dict_tuple (GenParamThreshold, 'enable_display', 'true')
compute_dl_anomaly_thresholds (DLModelHandle, DLDataset, GenParamThreshold, AnomalySegmentationThreshold, AnomalyClassificationThresholds)
dev_disp_text ('Press Run (F5) to continue', 'window', 'bottom', 'right', 'black', [], [])
stop ()
* 
dev_close_window ()
* 
* 设置评估参数跟评估模型
create_dict (GenParamEvaluation)
set_dict_tuple (GenParamEvaluation, 'measures', 'all')
set_dict_tuple (GenParamEvaluation, 'anomaly_classification_thresholds', AnomalyClassificationThresholds)
evaluate_dl_model (DLDataset, DLModelHandle, 'split', 'test', GenParamEvaluation, EvaluationResult, EvalParams)
* 
* 设置显示参数
create_dict (GenParamDisplay)
*显示评估模型
set_dict_tuple (GenParamDisplay, 'display_mode', ['score_histogram','score_legend'])
create_dict (WindowDict)
dev_display_anomaly_detection_evaluation (EvaluationResult, EvalParams, GenParamDisplay, WindowDict)
dev_disp_text ('Press Run (F5) to continue', 'window', 'bottom', 'right', 'black', 'box', 'true')
stop ()
* 
dev_close_window_dict (WindowDict)
* 
* 可视化评估参数
set_dict_tuple (GenParamDisplay, 'display_mode', ['pie_charts_precision','pie_charts_recall','absolute_confusion_matrix'])
* 
ClassificationThresholdIndex := |AnomalyClassificationThresholds| - 1
set_dict_tuple (GenParamDisplay, 'classification_threshold_index', 0)
create_dict (WindowDict)
dev_display_anomaly_detection_evaluation (EvaluationResult, EvalParams, GenParamDisplay, WindowDict)
dev_disp_text ('Press Run (F5) to continue', 'window', 'bottom', 'right', 'black', [], [])
stop ()
* 
dev_close_window_dict (WindowDict)
* 
* 写出模型

2.3.1评估分数表

在这里插入图片描述
上图是二值化区域表，纵轴错误分类的图像比例，横轴代表匹配分数。通常来看OK与NG的交叉点是最小匹配分数的阈值。一般选取稍大的部分可以有效防止误判情况。以上图而言，可以选取0.15-0.2之间。

3.应用推理


* ***   4.推理   ***

InferenceClassificationThreshold := AnomalyClassificationThresholds[0]

InferenceSegmentationThreshold := AnomalySegmentationThreshold
create_dict (DLDatasetInfo)
set_dict_tuple (DLDatasetInfo, 'class_names', ['OK','NG'])
set_dict_tuple (DLDatasetInfo, 'class_ids', [0,1])
create_dict (WindowDict)

* Image Acquisition 01: Code generated by Image Acquisition 01
list_files ('E:/New/图像文件/异常值检测单块/OK', ['files','follow_links'], ImageFiles)
tuple_regexp_select (ImageFiles, ['\\.(tif|tiff|gif|bmp|jpg|jpeg|jp2|png|pcx|pgm|ppm|pbm|xwd|ima|hobj)$','ignore_case'], ImageFiles)
for Index := 0 to |ImageFiles| - 1 by 1
    read_image (Image, ImageFiles[Index])
    * Image Acquisition 01: Do something
    dev_set_color ('red')
    *预处理图像，将所要检测的区域提取出
    threshold (Image, Regions, 29, 156)
    closing_circle (Regions, RegionClosing, 20)
    fill_up (RegionClosing, RegionFillUp)
    reduce_domain (Image, RegionFillUp, ImageReduced)
    *缩放图像，将图像缩放到与训练预处理一致，方便筛选
    get_image_size (ImageReduced, Width, Height)
    zoom_image_size (Image, ImageZoom,160,160, 'constant')
    zoom_region (RegionFillUp, RegionZoom,0.89,0.89)
    *将图像加载进行推理
    gen_dl_samples_from_images (Image, DLSample)
    preprocess_dl_samples (DLSample, DLPreprocessParam)
    apply_dl_model (DLModelHandle, DLSample, [], DLResult)

      read_dict_object (AnomalyImage, DLResult, 'anomaly_image')
    read_dict_tuple (DLResult, 'anomaly_score', AnomalyScore)
    * 
    *使用二值化选择所需要的区域
    threshold (AnomalyImage, AnomalyRegion,0.2,0.6)
    *对图像取交集，提取所检测的区域
    intersection (AnomalyRegion, RegionZoom, RegionIntersection)
    connection (RegionIntersection, ConnectedRegions)
    select_shape (ConnectedRegions, SelectedRegions, 'area', 'and', 100, 50000000)
     gen_contour_region_xld (SelectedRegions, Contours, 'border')
     *统计NG的区域个数
     count_obj (Contours, Number)
     dev_get_window (WindowHandle)
     if (Number>0)
         disp_message (WindowHandle, 'NG', 'window', 12, 12, 'black', 'true')
         stop ()
     else
         disp_message (WindowHandle, 'OK', 'window', 12, 12, 'black', 'true')
     endif
     
     reduce_domain (ImageZoom, RegionZoom, ImageReduced1)
     dev_display (ImageReduced1)
     dev_set_color ('green')
     dev_display (Contours)
endfor
dev_close_window_dict (WindowDict)

3.1输入的图像

在这里插入图片描述
输入一张NG图像进行测试。如上图所示，在110的字符中心存在漏印的部分，

3.2将区域提取出来

在这里插入图片描述

*预处理图像，将所要检测的区域提取出
    threshold (Image, Regions, 29, 156)
    closing_circle (Regions, RegionClosing, 20)
    fill_up (RegionClosing, RegionFillUp)
    reduce_domain (Image, RegionFillUp, ImageReduced)
    *缩放图像，将图像缩放到与训练预处理一致，方便筛选
    get_image_size (ImageReduced, Width, Height)
    zoom_image_size (Image, ImageZoom,160,160, 'constant')
    zoom_region (RegionFillUp, RegionZoom,0.89,0.89)

在推理的过程中，由于异常值深度学习的特性，会导致将大量的背景定义为异常区域，所以在推理前需要将图像三角标志提取出来。

3.3匹配异常区域

 intersection (AnomalyRegion, RegionZoom, RegionIntersection)
    connection (RegionIntersection, ConnectedRegions)
    select_shape (ConnectedRegions, SelectedRegions, 'area', 'and', 100, 50000000)
     gen_contour_region_xld (SelectedRegions, Contours, 'border')
     *统计NG的区域个数
     count_obj (Contours, Number)