FGSM方法生成交通信号牌的对抗图像样本

背景：

生成对抗样本，即扰动图像，让原本是“停车”的信号牌识别为“禁止驶入”

实验准备

模型：找一个训练好的，识别交通信号牌的CNN模型，灰度图像

模型地址：GitHub - Daulettulegenov/TSR_CNN: Traffic sign recognition

数据：Chinese Traffic Sign Database(CTSDB)

当下最受欢迎的国内交通标志数据集之一，该数据集容纳6164个交通标志图像，其中有58类交通标志。图像分为4170张图像的训练数据库和包含1994张图像的测试数据库两个子交通标志数据库。

官网链接：Traffic Sign Recogntion Database

数据结构：

模型在TSR_CNN-main下
8张目标表图像在 target下
stop下是停止图像
no_entry下是禁止通行图像

对抗攻击

1、对抗步骤：

加载CNN模型
预测原始的输出类型，可以看到并不能正确的分类，因为是中文字幕“停”而不是 STOP。
需要迁移训练，让其识别中文的“停”
测试是否可以识别中文的停
预测新的输出类型，可以看到能正确的分类，即便是中文的“停”
生成扰动图像，原始图像增加扰动，让其能识别识别为no entry
保存扰动图像

2、代码如下：

ART-Adversarial Robustness Toolbox检测AI模型及对抗攻击的工具

import os
import cv2
import numpy as np
import tensorflow as tf
from art.estimators.classification import TensorFlowV2Classifier
from art.attacks.evasion import FastGradientMethod
from tensorflow.keras.models import load_model
from tensorflow.keras.models import Model
from tensorflow.keras.layers import Dense
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.utils import to_categorical
from sklearn.model_selection import train_test_split

ID_CLASS_MAP = {
    0: 'Speed Limit 20 km/h',
    1: 'Speed Limit 30 km/h',
    2: 'Speed Limit 50 km/h',
    3: 'Speed Limit 60 km/h',
    4: 'Speed Limit 70 km/h',
    5: 'Speed Limit 80 km/h',
    6: 'End of Speed Limit 80 km/h',
    7: 'Speed Limit 100 km/h',
    8: 'Speed Limit 120 km/h',
    9: 'No passing',
    10: 'No passing for vechiles over 3.5 metric tons',
    11: 'Right-of-way at the next intersection',
    12: 'Priority road',
    13: 'Yield',
    14: 'Stop',
    15: 'No vechiles',
    16: 'Vechiles over 3.5 metric tons prohibited',
    17: 'No entry',
    18: 'General caution',
    19: 'Dangerous curve to the left',
    20: 'Dangerous curve to the right',
    21: 'Double curve',
    22: 'Bumpy road',
    23: 'Slippery road',
    24: 'Road narrows on the right',
    25: 'Road work',
    26: 'Traffic signals',
    27: 'Pedestrians',
    28: 'Children crossing',
    29: 'Bicycles crossing',
    30: 'Beware of ice/snow',
    31: 'Wild animals crossing',
    32: 'End of all speed and passing limits',
    33: 'Turn right ahead',
    34: 'Turn left ahead',
    35: 'Ahead only',
    36: 'Go straight or right',
    37: 'Go straight or left',
    38: 'Keep right',
    39: 'Keep left',
    40: 'Roundabout mandatory',
    41: 'End of no passing',
    42: 'End of no passing by vechiles over 3.5 metric tons'
}


def get_class_name(class_no):
    """
    根据类别编号返回对应的交通信号牌的名称
    :param class_no: 类别编号
    :return: 交通信号牌的名称
    """
    return ID_CLASS_MAP.get(class_no)


def grayscale(img):
    """
    将图像转换为灰度图像
    """
    img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    return img


def equalize(img):
    """
    进行直方图均衡化
    """
    new_img = cv2.equalizeHist(img)
    return new_img


def preprocessing(img):
    """
    归一化处理
    """
    img = equalize(img)
    img = img / 255
    return img


def read_imgs(image_dir, label=0):
    """
    读取图片
    :param image_dir:
    :param label:
    :return:
    """
    image_files = os.listdir(image_dir)
    images = []
    labels = []
    for image_file in image_files:
        image_path = os.path.join(image_dir, image_file)
        image = cv2.imread(image_path, cv2.IMREAD_GRAYSCALE)  # 以灰度模式读取图片
        image = cv2.resize(image, (30, 30))
        img = preprocessing(image)
        images.append(img)
        labels.append(label)
    return images, labels


def my_predict(model):
    """
    读取指定目录下的所有图像，然后使用模型进行预测，并打印出预测结果。
    """
    print('************my_predict*************')
    # 读取图片
    image_dir = r'D:\MyPython\adversarial\target'
    image_files = os.listdir(image_dir)
    images = []
    for image_file in image_files:
        image_path = os.path.join(image_dir, image_file)
        image = cv2.imread(image_path, cv2.IMREAD_GRAYSCALE)  # 以灰度模式读取图片
        image = cv2.resize(image, (30, 30))
        img = preprocessing(image)
        img = img.reshape(1, 30, 30, 1)
        images.append(img)
        # PREDICT IMAGE
        predictions = model.predict(img)
        predict_x = model.predict(img)
        classIndex = np.argmax(predict_x)
        probabilityValue = np.amax(predictions)
        print("img path:", image_file, " ==> ", str(classIndex) + " " + str(get_class_name(classIndex)))
        print(str(round(probabilityValue * 100, 2)) + "%")


def reset_model(model):
    """
    修改模型的结构，移除最后的分类层，然后添加一个新的分类层。
    """
    base_model = model
    # 移除最后的分类层
    base_model = Model(inputs=base_model.input, outputs=base_model.layers[-2].output)
    # 添加一个新的分类层
    output = Dense(2, activation='softmax', name='new_dense')(base_model.output)
    model = Model(inputs=base_model.input, outputs=output)
    # 编译模型
    model.compile(optimizer=Adam(), loss='categorical_crossentropy', metrics=['accuracy'])
    return model


def retrain_with2label(model):
    """
    取两个目录下的所有图像，然后使用这些图像和对应的标签来训练模型。
    """
    print('************retrain_with2label*************')
    image_dir1 = r'D:\MyPython\adversarial\stop'
    image_dir2 = r'D:\MyPython\adversarial\no_entry'
    images1, labels1 = read_imgs(image_dir1, 0)
    images2, labels2 = read_imgs(image_dir2, 1)

    # 合并图片和标签
    images = images1 + images2
    labels = labels1 + labels2

    images = np.array(images, dtype='float32')
    # 如果模型的输入形状是(30, 30, 1)，那么我们需要增加一个维度
    if model.input_shape[-1] == 1:
        images = np.expand_dims(images, axis=-1)

    labels = np.array(labels)
    labels = to_categorical(labels, num_classes=2)

    # 划分训练集和测试集
    train_images, test_images, train_labels, test_labels = train_test_split(images, labels, test_size=0.2)
    # 训练模型
    model.fit(train_images, train_labels, validation_data=(test_images, test_labels), epochs=10)


def my_predict2(model):
    """
    读取指定目录下的所有图像，然后使用模型进行预测，并返回预测结果和图像数据。
    """
    # 选择stop的图像，扰动前的
    images, _ = read_imgs(r'D:\MyPython\adversarial\target')
    if model.input_shape[-1] == 1:
        images = np.expand_dims(images, axis=-1)
    preds = model.predict(images)
    print('Predicted before:', preds.argmax(axis=1))
    return images


def run_art(images):
    """
    使用对抗性攻击来生成对抗样本，并保存对抗样本和平均扰动。
    """
    # 创建一个目标标签（我们希望模型将0 stop识别为1 no entry）
    target_label = to_categorical(1, num_classes=2)
    target_label = np.tile(target_label, (len(images), 1))

    # 创建ART分类器
    classifier = TensorFlowV2Classifier(
        model=model,
        nb_classes=2,
        input_shape=(30, 30, 1),
        loss_object=tf.keras.losses.CategoricalCrossentropy(from_logits=True),
        clip_values=(0, 1)
    )
    # 创建FGSM实例
    attack = FastGradientMethod(estimator=classifier, targeted=True)

    # 初始化对抗样本为原始图像
    adv_images = np.copy(images)

    for i in range(100):  # 最多迭代100次
        # 生成对抗样本的扰动
        perturbations = attack.generate(x=adv_images, y=target_label) - adv_images

        # 计算所有样本的平均扰动
        avg_perturbation = np.mean(perturbations, axis=0)

        # 将平均扰动添加到所有对抗样本上
        adv_images += avg_perturbation

        # 使用模型对对抗样本进行预测
        preds = model.predict(adv_images)
        print('Iteration:', i, 'Predicted after:', preds.argmax(axis=1))

        # 如果所有的预测结果都为1，那么停止迭代
        if np.all(preds.argmax(axis=1) == 1):
            break

    # 保存对抗样本
    for i in range(len(adv_images)):
        # 将图像的数据类型转换为uint8，并且将图像的值范围调整到[0, 255]
        img = (adv_images[i] * 255).astype(np.uint8)
        # 保存图像
        print(f'save adversarial picture: adversarial_image_{i}.png')
        cv2.imwrite(f'adversarial_image_{i}.png', img)

    # 归一化平均扰动并保存为图像
    avg_perturbation = (avg_perturbation - np.min(avg_perturbation)) / (
                np.max(avg_perturbation) - np.min(avg_perturbation))
    # 将平均扰动的值范围调整到[0, 255]，并转换为uint8类型
    avg_perturbation = (avg_perturbation * 255).astype(np.uint8)
    # 将灰度图像转换为RGB图像
    avg_perturbation_rgb = cv2.cvtColor(avg_perturbation, cv2.COLOR_GRAY2RGB)
    # 保存图像
    print(f'save perturbation picture: perturbation_image.png')
    cv2.imwrite('perturbation_image.png', avg_perturbation_rgb)


if __name__ == "__main__":
    # 加载CNN模型
    model = load_model(r'D:\MyPython\adversarial\TSR_CNN-main\CNN_model_3.h5', compile=False)
    # 预测原始的输出类型，可以看到并不能正确的分类，因为是中文字幕“停”而不是 STOP。
    my_predict(model)
    # 需要迁移训练，让其识别中文的“停”
    model = reset_model(model)
    # 测试是否可以识别中文的停
    retrain_with2label(model)
    # 预测新的输出类型，可以看到能正确的分类，即便是中文的“停”
    images = my_predict2(model)
    # 生成扰动图像，原始图像增加扰动，让其能识别识别为no entry,保存扰动图像
    run_art(images)

3、实验结果：

3.1运行日志

************my_predict*************
img path: 052_0001_j.png  ==>  28 Children crossing
99.83%
img path: 052_0005_j.png  ==>  3 Speed Limit 60 km/h
50.46%
img path: 052_0011.png  ==>  3 Speed Limit 60 km/h
52.22%
img path: 052_0013.png  ==>  8 Speed Limit 120 km/h
93.85%
img path: 052_1_0002_1_j.png  ==>  35 Ahead only
26.76%
img path: capture_20240114141426681.bmp  ==>  40 Roundabout mandatory
89.58%
img path: capture_20240114141515221.bmp  ==>  17 No entry
34.44%
img path: capture_20240114141530130.bmp  ==>  33 Turn right ahead
42.66%
************retrain_with2label*************
Epoch 1/10
3/3 [==============================] - 1s 107ms/step - loss: 1.2774 - accuracy: 0.4857 - val_loss: 0.2057 - val_accuracy: 1.0000
Epoch 2/10
3/3 [==============================] - 0s 32ms/step - loss: 0.2968 - accuracy: 0.8714 - val_loss: 0.1363 - val_accuracy: 1.0000
Epoch 3/10
3/3 [==============================] - 0s 33ms/step - loss: 0.1908 - accuracy: 0.9286 - val_loss: 0.1200 - val_accuracy: 1.0000
Epoch 4/10
3/3 [==============================] - 0s 31ms/step - loss: 0.1482 - accuracy: 0.9429 - val_loss: 0.1365 - val_accuracy: 1.0000
Epoch 5/10
3/3 [==============================] - 0s 31ms/step - loss: 0.0665 - accuracy: 1.0000 - val_loss: 0.1404 - val_accuracy: 1.0000
Epoch 6/10
3/3 [==============================] - 0s 32ms/step - loss: 0.0638 - accuracy: 0.9857 - val_loss: 0.1384 - val_accuracy: 1.0000
Epoch 7/10
3/3 [==============================] - 0s 31ms/step - loss: 0.0347 - accuracy: 0.9857 - val_loss: 0.1278 - val_accuracy: 1.0000
Epoch 8/10
3/3 [==============================] - 0s 30ms/step - loss: 0.0228 - accuracy: 0.9857 - val_loss: 0.1143 - val_accuracy: 0.8889
Epoch 9/10
3/3 [==============================] - 0s 32ms/step - loss: 0.0056 - accuracy: 1.0000 - val_loss: 0.1030 - val_accuracy: 0.8889
Epoch 10/10
3/3 [==============================] - 0s 32ms/step - loss: 0.0112 - accuracy: 1.0000 - val_loss: 0.0898 - val_accuracy: 0.8889
Predicted before: [0 0 0 0 0 0 0 0]
Iteration: 0 Predicted after: [0 0 0 0 1 1 0 0]
Iteration: 1 Predicted after: [1 1 1 1 1 1 1 1]
save adversarial picture: adversarial_image_0.png
save adversarial picture: adversarial_image_1.png
save adversarial picture: adversarial_image_2.png
save adversarial picture: adversarial_image_3.png
save adversarial picture: adversarial_image_4.png
save adversarial picture: adversarial_image_5.png
save adversarial picture: adversarial_image_6.png
save adversarial picture: adversarial_image_7.png
save perturbation picture: perturbation_image.png