【实验八】前馈神经网络（4）优化问题

1 参数初始化

模型构建

模型训练

优化

完整代码

2 梯度消失问题

模型构建

模型训练

完整代码

3 死亡Relu问题

模型构建

模型训练

优化

完整代码

1 参数初始化

实现一个神经网络前，需要先初始化模型参数。如果对每一层的权重和偏置都用0初始化，那么通过第一遍前向计算，所有隐藏层神经元的激活值都相同；在反向传播时，所有权重的更新也都相同，这样会导致隐藏层神经元没有差异性，出现对称权重现象。

导入需要的库：

import torch.nn as nn
import torch.nn.functional as F
from torch.nn.init import constant_, normal_, uniform_
import torch
from data import make_moons
from nndl import accuracy
from Runner2_2 import RunnerV2_2
import matplotlib
matplotlib.use('TkAgg')
import matplotlib.pyplot as plt

这次实验又认识到一个pytorch新的模块torch.nn.init ，是 PyTorch 中一个用于初始化神经网络模型参数的模块。

pytorch 笔记：torch.nn.init

总结常用的几个有：

常数初始化：将权重或偏置初始化为固定值torch.nn.init.constant_(tensor, value)

正态分布初始化：从正态分布中随机生成权重torch.nn.init.normal_(tensor, mean, std)

均匀分布初始化：从均匀分布中随机生成权重。torch.nn.init.uniform_(tensor, a, b)

零初始化：将权重或偏置初始化为零torch.nn.init.zeros_(tensor)

模型构建

将模型参数全都初始化为0

class Model_MLP_L2_V4(nn.Module):
    def __init__(self, input_size, hidden_size, output_size):
        super(Model_MLP_L2_V4, self).__init__()
        # 定义第一个线性层，输入特征数为 input_size，输出特征数为 hidden_size
        self.fc1 = nn.Linear(input_size, hidden_size)
        '''
         weight为权重参数属性,bias为偏置参数属性，这里使用'torch.nn.init.constant_'进行常量初始化
        '''
        # 初始化第一个线性层的权重和偏置为 0
        constant_(self.fc1.weight, 0.0)
        constant_(self.fc1.bias, 0.0)
        # 定义第二个线性层，输入特征数为 hidden_size，输出特征数为 output_size
        self.fc2 = nn.Linear(hidden_size, output_size)
        # 初始化第二个线性层的权重和偏置为 0
        constant_(self.fc2.weight, 0.0)
        constant_(self.fc2.bias, 0.0)
        self.act_fn = F.sigmoid

    # 前向计算
    def forward(self, inputs):
        z1 = self.fc1(inputs)
        a1 = self.act_fn(z1)
        z2 = self.fc2(a1)
        a2 = self.act_fn(z2)
        return a2

设置打印权重变化的函数：

def print_weight(runner):
    print('The weights of the Layers：')
    # 通过 enumerate() 可以同时获取参数的索引 i 和参数的内容 item
    for i, item in enumerate(runner.model.named_parameters()):
        print(item)
        print('=========================')

模型训练

利用runner类训练模型：

# ================================训练模型===========================
input_size = 2
hidden_size = 5
output_size = 1
model = Model_MLP_L2_V4(input_size=input_size, hidden_size=hidden_size, output_size=output_size)

# 设置损失函数
loss_fn = F.binary_cross_entropy

# 设置优化器
learning_rate = 0.2
optimizer = torch.optim.SGD(params=model.parameters(), lr=learning_rate)

# 设置评价指标
metric = accuracy

# 其他参数
epoch = 2000
saved_path = 'best_model.pdparams'

# 实例化RunnerV2_2类，并传入训练配置
runner = RunnerV2_2(model, optimizer, metric, loss_fn)

runner.train([X_train, y_train], [X_dev, y_dev], num_epochs=5, log_epochs=50, save_path="best_model.pdparams",
             custom_print_log=print_weight)

输出结果：

The weights of the Layers：
('fc1.weight', Parameter containing:
tensor([[0., 0.],
        [0., 0.],
        [0., 0.],
        [0., 0.],
        [0., 0.]], requires_grad=True))
=========================
('fc1.bias', Parameter containing:
tensor([0., 0., 0., 0., 0.], requires_grad=True))
=========================
('fc2.weight', Parameter containing:
tensor([[0., 0., 0., 0., 0.]], requires_grad=True))
=========================
('fc2.bias', Parameter containing:
tensor([0.], requires_grad=True))
=========================
[Evaluate] best accuracy performence has been updated: 0.00000 --> 0.47500
[Train] epoch: 0/5, loss: 0.6931473016738892
The weights of the Layers：
('fc1.weight', Parameter containing:
tensor([[0., 0.],
        [0., 0.],
        [0., 0.],
        [0., 0.],
        [0., 0.]], requires_grad=True))
=========================
('fc1.bias', Parameter containing:
tensor([0., 0., 0., 0., 0.], requires_grad=True))
=========================
('fc2.weight', Parameter containing:
tensor([[0.0008, 0.0008, 0.0008, 0.0008, 0.0008]], requires_grad=True))
=========================
('fc2.bias', Parameter containing:
tensor([0.0016], requires_grad=True))
=========================
The weights of the Layers：
('fc1.weight', Parameter containing:
tensor([[ 9.3081e-06, -7.6568e-06],
        [ 9.3081e-06, -7.6568e-06],
        [ 9.3081e-06, -7.6568e-06],
        [ 9.3081e-06, -7.6568e-06],
        [ 9.3081e-06, -7.6568e-06]], requires_grad=True))
=========================
('fc1.bias', Parameter containing:
tensor([2.7084e-07, 2.7084e-07, 2.7084e-07, 2.7084e-07, 2.7084e-07],
       requires_grad=True))
=========================
('fc2.weight', Parameter containing:
tensor([[0.0015, 0.0015, 0.0015, 0.0015, 0.0015]], requires_grad=True))
=========================
('fc2.bias', Parameter containing:
tensor([0.0029], requires_grad=True))
=========================
The weights of the Layers：
('fc1.weight', Parameter containing:
tensor([[ 2.6847e-05, -2.2122e-05],
        [ 2.6847e-05, -2.2122e-05],
        [ 2.6847e-05, -2.2122e-05],
        [ 2.6847e-05, -2.2122e-05],
        [ 2.6847e-05, -2.2122e-05]], requires_grad=True))
=========================
('fc1.bias', Parameter containing:
tensor([7.2455e-07, 7.2455e-07, 7.2455e-07, 7.2455e-07, 7.2455e-07],
       requires_grad=True))
=========================
('fc2.weight', Parameter containing:
tensor([[0.0021, 0.0021, 0.0021, 0.0021, 0.0021]], requires_grad=True))
=========================
('fc2.bias', Parameter containing:
tensor([0.0042], requires_grad=True))
=========================
The weights of the Layers：
('fc1.weight', Parameter containing:
tensor([[ 5.1669e-05, -4.2643e-05],
        [ 5.1669e-05, -4.2643e-05],
        [ 5.1669e-05, -4.2643e-05],
        [ 5.1669e-05, -4.2643e-05],
        [ 5.1669e-05, -4.2643e-05]], requires_grad=True))
=========================
('fc1.bias', Parameter containing:
tensor([1.2953e-06, 1.2953e-06, 1.2953e-06, 1.2953e-06, 1.2953e-06],
       requires_grad=True))
=========================
('fc2.weight', Parameter containing:
tensor([[0.0026, 0.0026, 0.0026, 0.0026, 0.0026]], requires_grad=True))
=========================
('fc2.bias', Parameter containing:
tensor([0.0053], requires_grad=True))

所有权重的更新都相同，即出现了对称权重现象

可视化权重变化：

# ===========可视化函数===============
def plot(runner, fig_name):
    plt.figure(figsize=(10, 5))
    epochs = [i for i in range(0, len(runner.train_scores))]
    plt.subplot(1, 2, 1)
    plt.plot(epochs, runner.train_loss, color='#e4007f', label="Train loss")
    plt.plot(epochs, runner.dev_loss, color='#f19ec2', linestyle='--', label="Dev loss")
    # 绘制坐标轴和图例
    plt.ylabel("loss", fontsize='large')
    plt.xlabel("epoch", fontsize='large')
    plt.legend(loc='upper right', fontsize='x-large')
    plt.subplot(1, 2, 2)
    plt.plot(epochs, runner.train_scores, color='#e4007f', label="Train accuracy")
    plt.plot(epochs, runner.dev_scores, color='#f19ec2', linestyle='--', label="Dev accuracy")
    # 绘制坐标轴和图例
    plt.ylabel("score", fontsize='large')
    plt.xlabel("epoch", fontsize='large')
    plt.legend(loc='lower right', fontsize='x-large')
    plt.savefig(fig_name)
    plt.show()
plot(runner, 'fw-acc.pdf')

从图像可以看出，二分类score为50%左右，说明模型没有学到任何内容。训练和验证的loss几乎没有怎么下降。

优化

为了避免对称权重现象，可以使用高斯分布或均匀分布初始化神经网络的参数。

高斯分布和均匀分布采样的实现和可视化代码如下：

import matplotlib
matplotlib.use('TkAgg')
import matplotlib.pyplot as plt
import torch

# 使用'torch.normal'实现高斯分布采样，其中'mean'为高斯分布的均值，'std'为高斯分布的标准差，'shape'为输出形状
gausian_weights = torch.normal(mean=0.0, std=1.0, size=[10000])
# 使用'torch.uniform'实现在[min,max)范围内的均匀分布采样，其中'shape'为输出形状
uniform_weights = torch.Tensor(10000)
uniform_weights.uniform_(-1,1)
gausian_weights=gausian_weights.numpy()
uniform_weights=uniform_weights.numpy()
print(uniform_weights)
# 绘制两种参数分布
plt.figure()
plt.subplot(1,2,1)
plt.title('Gausian Distribution')
plt.hist(gausian_weights, bins=200, density=True, color='#f19ec2')
plt.subplot(1,2,2)
plt.title('Uniform Distribution')
plt.hist(uniform_weights, bins=200, density=True, color='#e4007f')
plt.savefig('fw-gausian-uniform.pdf')
plt.show()

完整代码

'''
@author: lxy
@function: The Impact of Zero Weight Initialization
@date: 2024/10/31
'''
import torch.nn as nn
import torch.nn.functional as F
from torch.nn.init import constant_, normal_, uniform_
import torch
from data import make_moons
from nndl import accuracy
from Runner2_2 import RunnerV2_2
import matplotlib
matplotlib.use('TkAgg')
import matplotlib.pyplot as plt


class Model_MLP_L2_V4(nn.Module):
    def __init__(self, input_size, hidden_size, output_size):
        super(Model_MLP_L2_V4, self).__init__()
        # 定义第一个线性层，输入特征数为 input_size，输出特征数为 hidden_size
        self.fc1 = nn.Linear(input_size, hidden_size)
        '''
         weight为权重参数属性,bias为偏置参数属性，这里使用'torch.nn.init.constant_'进行常量初始化
        '''
        # 初始化第一个线性层的权重和偏置为 0
        constant_(self.fc1.weight, 0.0)
        constant_(self.fc1.bias, 0.0)
        # 定义第二个线性层，输入特征数为 hidden_size，输出特征数为 output_size
        self.fc2 = nn.Linear(hidden_size, output_size)
        # 初始化第二个线性层的权重和偏置为 0
        constant_(self.fc2.weight, 0.0)
        constant_(self.fc2.bias, 0.0)

        self.act_fn = F.sigmoid

    # 前向计算
    def forward(self, inputs):
        z1 = self.fc1(inputs)
        a1 = self.act_fn(z1)
        z2 = self.fc2(a1)
        a2 = self.act_fn(z2)
        return a2

def print_weight(runner):
    print('The weights of the Layers：')
    # 通过 enumerate() 可以同时获取参数的索引 i 和参数的内容 item
    for i, item in enumerate(runner.model.named_parameters()):
        print(item)
        print('=========================')

# =============================数据集=======================
# 数据集构建
n_samples = 1000
X, y = make_moons(n_samples=n_samples, shuffle=True, noise=0.2)
# 划分数据集
num_train = 640  # 训练集样本数量
num_dev = 160    # 验证集样本数量
num_test = 200   # 测试集样本数量
# 根据指定数量划分数据集
X_train, y_train = X[:num_train], y[:num_train]  # 训练集
X_dev, y_dev = X[num_train:num_train + num_dev], y[num_train:num_train + num_dev]  # 验证集
X_test, y_test = X[num_train + num_dev:], y[num_train + num_dev:]  # 测试集
# 调整标签的形状，将其转换为[N, 1]的格式
y_train = y_train.reshape([-1, 1])
y_dev = y_dev.reshape([-1, 1])
y_test = y_test.reshape([-1, 1])
# ================================训练模型===========================
input_size = 2
hidden_size = 5
output_size = 1
model = Model_MLP_L2_V4(input_size=input_size, hidden_size=hidden_size, output_size=output_size)

# 设置损失函数
loss_fn = F.binary_cross_entropy

# 设置优化器
learning_rate = 0.2
optimizer = torch.optim.SGD(params=model.parameters(), lr=learning_rate)

# 设置评价指标
metric = accuracy

# 其他参数
epoch = 2000
saved_path = 'best_model.pdparams'

# 实例化RunnerV2_2类，并传入训练配置
runner = RunnerV2_2(model, optimizer, metric, loss_fn)

runner.train([X_train, y_train], [X_dev, y_dev], num_epochs=5, log_epochs=50, save_path="best_model.pdparams",
             custom_print_log=print_weight)

# ===========可视化函数===============
def plot(runner, fig_name):
    plt.figure(figsize=(10, 5))
    epochs = [i for i in range(0, len(runner.train_scores))]
    plt.subplot(1, 2, 1)
    plt.plot(epochs, runner.train_loss, color='#e4007f', label="Train loss")
    plt.plot(epochs, runner.dev_loss, color='#f19ec2', linestyle='--', label="Dev loss")
    # 绘制坐标轴和图例
    plt.ylabel("loss", fontsize='large')
    plt.xlabel("epoch", fontsize='large')
    plt.legend(loc='upper right', fontsize='x-large')
    plt.subplot(1, 2, 2)
    plt.plot(epochs, runner.train_scores, color='#e4007f', label="Train accuracy")
    plt.plot(epochs, runner.dev_scores, color='#f19ec2', linestyle='--', label="Dev accuracy")
    # 绘制坐标轴和图例
    plt.ylabel("score", fontsize='large')
    plt.xlabel("epoch", fontsize='large')
    plt.legend(loc='lower right', fontsize='x-large')
    plt.savefig(fig_name)
    plt.show()
plot(runner, 'fw-acc.pdf')

2 梯度消失问题

由于Sigmoid型函数的饱和性，饱和区的导数更接近于0，误差经过每一层传递都会不断衰减。当网络层数很深时，梯度就会不停衰减，甚至消失，使得整个网络很难训练，这就是所谓的梯度消失问题。减轻梯度消失问题的方法有很多种，一种简单有效的方式就是使用导数比较大的激活函数，如：ReLU。

定义一个前馈神经网络，包含4个隐藏层和1个输出层，分别使用ReLU函数和sigmod函数作为激活函数，观察梯度变化。

模型构建

class Model_MLP_L5(nn.Module):
    def __init__(self, input_size, output_size, act='sigmoid', w_init=nn.init.normal_, b_init=nn.init.constant_):
        super(Model_MLP_L5, self).__init__()
        self.fc1 = nn.Linear(input_size, 3)
        self.fc2 = nn.Linear(3, 3)
        self.fc3 = nn.Linear(3, 3)
        self.fc4 = nn.Linear(3, 3)
        self.fc5 = nn.Linear(3, output_size)

        # 定义激活函数
        if act == 'sigmoid':
            self.act = F.sigmoid
        elif act == 'relu':
            self.act = F.relu
        elif act == 'lrelu':
            self.act = F.leaky_relu
        else:
            raise ValueError("Please enter sigmoid, relu or lrelu!")

        # 初始化权重和偏置
        self.init_weights(w_init, b_init)

    # 初始化线性层权重和偏置参数
    def init_weights(self, w_init, b_init):
        for m in self.children():
            if isinstance(m, nn.Linear):
                w_init(m.weight, mean=0.0, std=0.01)  # 对权重进行初始化
                b_init(m.bias, 1.0)  # 对偏置进行初始化

    def forward(self, inputs):
        outputs = self.fc1(inputs)
        outputs = self.act(outputs)
        outputs = self.fc2(outputs)
        outputs = self.act(outputs)
        outputs = self.fc3(outputs)
        outputs = self.act(outputs)
        outputs = self.fc4(outputs)
        outputs = self.act(outputs)
        outputs = self.fc5(outputs)
        outputs = F.sigmoid(outputs)
        return outputs

设置打印梯度的L 2范数的函数

def print_grads(runner, grad_norms):
    """ 打印模型每一层的梯度并计算其L2范数。 """
    print("The gradient of the Layers:")
    for name, param in runner.model.named_parameters():
        if param.requires_grad and param.grad is not None:
            grad_norm = param.grad.data.norm(2).item()  # 计算L2范数
            grad_norms[name].append(grad_norm)  # 记录L2范数
            print(f'Layer: {name}, Gradient Norm: {grad_norm}')

这里为什么要打印梯度范数？

当梯度过大时，它可能导致模型训练过程中的数值不稳定，进而影响模型的性能。

打印范数可以帮助我们了解梯度的幅度大小。范数可以衡量向量的大小，因此通过打印梯度的范数，我们可以直观地看到梯度的幅度是否过大或过小。简单说，就是范数可以反应梯度的大小，打印范数我们可以及时知道梯度的情况。

参考连接：

梯度爆炸实验

模型训练

分别使用sigmod函数和relu函数

# =====================使用sigmoid激活函数训练=====================
torch.manual_seed(111)
lr = 0.01
model = Model_MLP_L5(input_size=2, output_size=1, act='sigmoid')
optimizer = torch.optim.SGD(params=model.parameters(), lr=lr)
loss_fn = F.binary_cross_entropy
metric = accuracy

# 初始化L2范数记录字典
grad_norms_sigmoid = {name: [] for name, _ in model.named_parameters()}

# 实例化Runner类
runner = RunnerV2_2(model, optimizer, metric, loss_fn)
print("使用sigmoid函数为激活函数时：")
runner.train([X_train, y_train], [X_dev, y_dev],
             num_epochs=1, log_epochs=None,
             save_path="best_model.pdparams",
             custom_print_log=lambda runner: print_grads(runner, grad_norms_sigmoid))

# =====================使用ReLU激活函数训练=====================
torch.manual_seed(102)
model = Model_MLP_L5(input_size=2, output_size=1, act='relu')
optimizer = torch.optim.SGD(params=model.parameters(), lr=lr)
loss_fn = F.binary_cross_entropy

# 初始化L2范数记录字典
grad_norms_relu = {name: [] for name, _ in model.named_parameters()}

# 实例化Runner类
runner = RunnerV2_2(model, optimizer, metric, loss_fn)
print("使用ReLU函数为激活函数时：")
runner.train([X_train, y_train], [X_dev, y_dev],
             num_epochs=1, log_epochs=None,
             save_path="best_model.pdparams",
             custom_print_log=lambda runner: print_grads(runner, grad_norms_relu))

运行结果输出：

使用sigmoid函数为激活函数时：
The gradient of the Layers:
Layer: fc1.weight, Gradient Norm: 2.4828878536498067e-11
Layer: fc1.bias, Gradient Norm: 1.8694254477757966e-11
Layer: fc2.weight, Gradient Norm: 1.2134693250231976e-08
Layer: fc2.bias, Gradient Norm: 9.58359702707412e-09
Layer: fc3.weight, Gradient Norm: 5.372268333303509e-06
Layer: fc3.bias, Gradient Norm: 4.236671884427778e-06
Layer: fc4.weight, Gradient Norm: 0.001065725926309824
Layer: fc4.bias, Gradient Norm: 0.0008412969764322042
Layer: fc5.weight, Gradient Norm: 0.27612796425819397
Layer: fc5.bias, Gradient Norm: 0.21845529973506927
[Evaluate] best accuracy performence has been updated: 0.00000 --> 0.54375
使用ReLU函数为激活函数时：
The gradient of the Layers:
Layer: fc1.weight, Gradient Norm: 1.736074395353171e-08
Layer: fc1.bias, Gradient Norm: 1.370427327174184e-08
Layer: fc2.weight, Gradient Norm: 1.4403226487047505e-06
Layer: fc2.bias, Gradient Norm: 8.300839340336097e-07
Layer: fc3.weight, Gradient Norm: 0.00011438350338721648
Layer: fc3.bias, Gradient Norm: 6.653369928244501e-05
Layer: fc4.weight, Gradient Norm: 0.009503044188022614
Layer: fc4.bias, Gradient Norm: 0.005468158517032862
Layer: fc5.weight, Gradient Norm: 0.3917791247367859
Layer: fc5.bias, Gradient Norm: 0.22893022000789642
[Evaluate] best accuracy performence has been updated: 0.00000 --> 0.54375

可视化梯度范数的变化情况：

# 可视化梯度L2范数
def plot_grad_norms(grad_norms_sigmoid, grad_norms_relu):
    layers = list(grad_norms_sigmoid.keys())
    sigmoid_norms = [np.mean(grad_norms_sigmoid[layer]) for layer in layers]
    relu_norms = [np.mean(grad_norms_relu[layer]) for layer in layers]

    x = np.arange(len(layers))

    plt.figure(figsize=(10, 6))
    plt.plot(x, sigmoid_norms, marker='o', label='Sigmoid', color='b')
    plt.plot(x, relu_norms, marker='o', label='ReLU', color='r')

    plt.ylabel('Gradient L2 Norm')
    plt.title('Gradient L2 Norm by different Activation Function')
    plt.xticks(x, layers)
    plt.legend()

    # 设置 y 轴为对数坐标
    plt.yscale('log')

    # 设置 y 轴的范围
    plt.ylim(1e-8, 1)  # 设置下限为 1e-8，上限为 1

    # 设置 y 轴的刻度
    plt.yticks([1, 1e-1, 1e-2, 1e-3, 1e-4, 1e-5, 1e-6, 1e-7, 1e-8,1e-9,1e-10,1e-11])

    plt.grid()
    plt.tight_layout()
    plt.show()

图中展示了使用不同激活函数时，网络每层梯度值的ℓ2范数情况。从结果可以看到，5层的全连接前馈神经网络使用Sigmoid型函数作为激活函数时，梯度经过每一个神经层的传递都会不断衰减，最终传递到第一个神经层时，梯度几乎完全消失。改为ReLU激活函数后，梯度消失现象得到了缓解

完整代码

'''
@author: lxy
@function: Exploration and Optimization of the Gradient Vanishing Problem
@date: 2024/10/31
'''
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.nn.init import constant_, normal_
import numpy as np
import matplotlib
matplotlib.use('TkAgg')
import matplotlib.pyplot as plt
from data import make_moons
from nndl import accuracy
from Runner2_2 import RunnerV2_2

class Model_MLP_L5(nn.Module):
    def __init__(self, input_size, output_size, act='sigmoid', w_init=nn.init.normal_, b_init=nn.init.constant_):
        super(Model_MLP_L5, self).__init__()
        self.fc1 = nn.Linear(input_size, 3)
        self.fc2 = nn.Linear(3, 3)
        self.fc3 = nn.Linear(3, 3)
        self.fc4 = nn.Linear(3, 3)
        self.fc5 = nn.Linear(3, output_size)

        # 定义激活函数
        if act == 'sigmoid':
            self.act = F.sigmoid
        elif act == 'relu':
            self.act = F.relu
        elif act == 'lrelu':
            self.act = F.leaky_relu
        else:
            raise ValueError("Please enter sigmoid, relu or lrelu!")

        # 初始化权重和偏置
        self.init_weights(w_init, b_init)

    # 初始化线性层权重和偏置参数
    def init_weights(self, w_init, b_init):
        for m in self.children():
            if isinstance(m, nn.Linear):
                w_init(m.weight, mean=0.0, std=0.01)  # 对权重进行初始化
                b_init(m.bias, 1.0)  # 对偏置进行初始化

    def forward(self, inputs):
        outputs = self.fc1(inputs)
        outputs = self.act(outputs)
        outputs = self.fc2(outputs)
        outputs = self.act(outputs)
        outputs = self.fc3(outputs)
        outputs = self.act(outputs)
        outputs = self.fc4(outputs)
        outputs = self.act(outputs)
        outputs = self.fc5(outputs)
        outputs = F.sigmoid(outputs)
        return outputs


def print_grads(runner, grad_norms):
    """ 打印模型每一层的梯度并计算其L2范数。 """
    print("The gradient of the Layers:")
    for name, param in runner.model.named_parameters():
        if param.requires_grad and param.grad is not None:
            grad_norm = param.grad.data.norm(2).item()  # 计算L2范数
            grad_norms[name].append(grad_norm)  # 记录L2范数
            print(f'Layer: {name}, Gradient Norm: {grad_norm}')



# 可视化梯度L2范数
def plot_grad_norms(grad_norms_sigmoid, grad_norms_relu):
    layers = list(grad_norms_sigmoid.keys())
    sigmoid_norms = [np.mean(grad_norms_sigmoid[layer]) for layer in layers]
    relu_norms = [np.mean(grad_norms_relu[layer]) for layer in layers]

    x = np.arange(len(layers))

    plt.figure(figsize=(10, 6))
    plt.plot(x, sigmoid_norms, marker='o', label='Sigmoid', color='b')
    plt.plot(x, relu_norms, marker='o', label='ReLU', color='r')

    plt.ylabel('Gradient L2 Norm')
    plt.title('Gradient L2 Norm by different Activation Function')
    plt.xticks(x, layers)
    plt.legend()
    # 设置 y 轴为对数坐标
    plt.yscale('log')
    # 设置 y 轴的范围
    plt.ylim(1e-8, 1)  # 设置下限为 1e-8，上限为 1
    # 设置 y 轴的刻度
    plt.yticks([1, 1e-1, 1e-2, 1e-3, 1e-4, 1e-5, 1e-6, 1e-7, 1e-8,1e-9,1e-10,1e-11])

    plt.grid()
    plt.tight_layout()
    plt.show()


# =============================数据集=======================
# 数据集构建
n_samples = 1000
X, y = make_moons(n_samples=n_samples, shuffle=True, noise=0.2)
# 划分数据集
num_train = 640  # 训练集样本数量
num_dev = 160  # 验证集样本数量
num_test = 200  # 测试集样本数量
# 根据指定数量划分数据集
X_train, y_train = X[:num_train], y[:num_train]  # 训练集
X_dev, y_dev = X[num_train:num_train + num_dev], y[num_train:num_train + num_dev]  # 验证集
X_test, y_test = X[num_train + num_dev:], y[num_train + num_dev:]  # 测试集
# 调整标签的形状，将其转换为[N, 1]的格式
y_train = y_train.reshape([-1, 1])
y_dev = y_dev.reshape([-1, 1])
y_test = y_test.reshape([-1, 1])
# =====================使用sigmoid激活函数训练=====================
torch.manual_seed(111)
lr = 0.01
model = Model_MLP_L5(input_size=2, output_size=1, act='sigmoid')
optimizer = torch.optim.SGD(params=model.parameters(), lr=lr)
loss_fn = F.binary_cross_entropy
metric = accuracy

# 初始化L2范数记录字典
grad_norms_sigmoid = {name: [] for name, _ in model.named_parameters()}

# 实例化Runner类
runner = RunnerV2_2(model, optimizer, metric, loss_fn)
print("使用sigmoid函数为激活函数时：")
runner.train([X_train, y_train], [X_dev, y_dev],
             num_epochs=1, log_epochs=None,
             save_path="best_model.pdparams",
             custom_print_log=lambda runner: print_grads(runner, grad_norms_sigmoid))

# =====================使用ReLU激活函数训练=====================
torch.manual_seed(102)
model = Model_MLP_L5(input_size=2, output_size=1, act='relu')
optimizer = torch.optim.SGD(params=model.parameters(), lr=lr)
loss_fn = F.binary_cross_entropy

# 初始化L2范数记录字典
grad_norms_relu = {name: [] for name, _ in model.named_parameters()}

# 实例化Runner类
runner = RunnerV2_2(model, optimizer, metric, loss_fn)
print("使用ReLU函数为激活函数时：")
runner.train([X_train, y_train], [X_dev, y_dev],
             num_epochs=1, log_epochs=None,
             save_path="best_model.pdparams",
             custom_print_log=lambda runner: print_grads(runner, grad_norms_relu))

# 绘制梯度范数
plot_grad_norms(grad_norms_sigmoid, grad_norms_relu)

3 死亡Relu问题

ReLU激活函数可以一定程度上改善梯度消失问题，但是ReLU函数在某些情况下容易出现死亡 ReLU问题，使得网络难以训练。

这是由于激活前神经元通常也包含偏置项，如果偏置项是一个过小的负数当x<0时，ReLU函数的输出恒为0。在训练过程中，如果参数在一次不恰当的更新后，某个ReLU神经元在所有训练数据上都不能被激活（即输出为0），那么这个神经元自身参数的梯度永远都会是0，在以后的训练过程中永远都不能被激活。

模型构建

当神经层的偏置被初始化为一个相对于权重较大的负值时，可以想像，输入经过神经层的处理，最终的输出会为负值，从而导致死亡ReLU现象。这里我们初始化偏置为-8.0

class Model_MLP_L5(nn.Module):
    def __init__(self, input_size, output_size, act='sigmoid', w_init=nn.init.normal_, b_init=-8.0):
        super(Model_MLP_L5, self).__init__()
        self.fc1 = nn.Linear(input_size, 3)
        self.fc2 = nn.Linear(3, 3)
        self.fc3 = nn.Linear(3, 3)
        self.fc4 = nn.Linear(3, 3)
        self.fc5 = nn.Linear(3, output_size)

        # 定义激活函数
        if act == 'sigmoid':
            self.act = F.sigmoid
        elif act == 'relu':
            self.act = F.relu
        elif act == 'lrelu':
            self.act = F.leaky_relu
        else:
            raise ValueError("Please enter sigmoid, relu or lrelu!")

        # 初始化权重和偏置
        self.init_weights(w_init, b_init)

    # 初始化线性层权重和偏置参数
    def init_weights(self, w_init, b_init):
        for m in self.children():
            if isinstance(m, nn.Linear):
                w_init(m.weight, mean=0.0, std=0.01)  # 对权重进行初始化
                constant_(m.bias, b_init)

    def forward(self, inputs):
        outputs = self.fc1(inputs)
        outputs = self.act(outputs)
        outputs = self.fc2(outputs)
        outputs = self.act(outputs)
        outputs = self.fc3(outputs)
        outputs = self.act(outputs)
        outputs = self.fc4(outputs)
        outputs = self.act(outputs)
        outputs = self.fc5(outputs)
        outputs = F.sigmoid(outputs)
        return outputs

设置打印梯度范数的函数：

def print_grads(runner, grad_norms):
    """ 打印模型每一层的梯度并计算其L2范数。 """
    print("The gradient of the Layers:")
    for name, param in runner.model.named_parameters():
        if param.requires_grad and param.grad is not None:
            grad_norm = param.grad.data.norm(2).item()  # 计算L2范数
            grad_norms[name].append(grad_norm)  # 记录L2范数
            print(f'Layer: {name}, Gradient Norm: {grad_norm}')

模型训练

使用relu函数-观察梯度变化

# 定义网络，并使用较大的负值来初始化偏置
model =  Model_MLP_L5(input_size=2, output_size=1, act='relu')
#model =  Model_MLP_L5(input_size=2, output_size=1, act='lrelu')
torch.manual_seed(111)
lr = 0.01
optimizer = torch.optim.SGD(params=model.parameters(), lr=lr)
loss_fn = F.binary_cross_entropy
metric = accuracy
# 初始化L2范数记录字典
grad_norms = {name: [] for name, _ in model.named_parameters()}
# 实例化Runner类
runner = RunnerV2_2(model, optimizer, metric, loss_fn)

# 启动训练
runner.train([X_train, y_train], [X_dev, y_dev],
             num_epochs=1, log_epochs=0,
             save_path="best_model.pdparams",
             custom_print_log=lambda runner: print_grads(runner, grad_norms))

运行结果：

The gradient of the Layers:
Layer: fc1.weight, Gradient Norm: 0.0
Layer: fc1.bias, Gradient Norm: 0.0
Layer: fc2.weight, Gradient Norm: 0.0
Layer: fc2.bias, Gradient Norm: 0.0
Layer: fc3.weight, Gradient Norm: 0.0
Layer: fc3.bias, Gradient Norm: 0.0
Layer: fc4.weight, Gradient Norm: 0.0
Layer: fc4.bias, Gradient Norm: 0.0
Layer: fc5.weight, Gradient Norm: 0.0
Layer: fc5.bias, Gradient Norm: 0.4887271523475647
[Evaluate] best accuracy performence has been updated: 0.00000 --> 0.50000

可以看出梯度反向传播时以及变为0了，出现了死亡relu问题

可视化梯度变化：

# 可视化梯度L2范数
def plot_grad_norms(grad_norms):
    layers = list(grad_norms.keys())
    norms = [np.mean(grad_norms[layer]) for layer in layers]
    x = np.arange(len(layers))  # x轴为层数
    plt.figure(figsize=(10, 6))
    plt.plot(x, norms, marker='o', label='ReLU', color='r')
    plt.ylabel('Gradient L2 Norm')
    plt.title('Gradient L2 Norm --Relu')
    plt.xticks(x, layers)
    plt.legend()
    # 设置 y 轴为对数坐标
    plt.yscale('log')
    # 设置 y 轴的范围
    plt.ylim(1e-8, 1)  # 设置下限为 1e-8，上限为 1
    # 设置 y 轴的刻度
    plt.yticks([1, 1e-1, 1e-2, 1e-3, 1e-4, 1e-5, 1e-6, 1e-7, 1e-8, 1e-9, 1e-12, 1e-17]
               )
    plt.grid()
    plt.tight_layout()
    plt.show()

从输出结果以及可视化的图像可以发现，使用 ReLU 作为激活函数，当满足条件时，会发生死亡ReLU问题，网络训练过程中 ReLU 神经元的梯度始终为0，参数无法更新。

优化

针对死亡ReLU问题，一种简单有效的优化方式就是将激活函数更换为Leaky ReLU、ELU等ReLU 的变种。接下来，观察将激活函数更换为 Leaky ReLU时的梯度情况。

model =  Model_MLP_L5(input_size=2, output_size=1, act='lrelu')

The gradient of the Layers:
Layer: fc1.weight, Gradient Norm: 1.6563501643453269e-16
Layer: fc1.bias, Gradient Norm: 1.6535552203171837e-16
Layer: fc2.weight, Gradient Norm: 1.4167183051694288e-13
Layer: fc2.bias, Gradient Norm: 1.0233488318897588e-12
Layer: fc3.weight, Gradient Norm: 6.822118980842617e-10
Layer: fc3.bias, Gradient Norm: 4.9233230825507235e-09
Layer: fc4.weight, Gradient Norm: 6.337210834317375e-06
Layer: fc4.bias, Gradient Norm: 4.57389687653631e-05
Layer: fc5.weight, Gradient Norm: 0.07076060771942139
Layer: fc5.bias, Gradient Norm: 0.510601818561554
[Evaluate] best accuracy performence has been updated: 0.00000 --> 0.55625

完整代码

'''
@author: lxy
@function: Exploration and Optimization of the Dead ReLU Problem
@date: 2024/10/31
'''
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.nn.init import constant_, normal_
import numpy as np
import matplotlib

matplotlib.use('TkAgg')
import matplotlib.pyplot as plt
from data import make_moons
from nndl import accuracy
from Runner2_2 import RunnerV2_2


# 定义模型

class Model_MLP_L5(nn.Module):
    def __init__(self, input_size, output_size, act='sigmoid', w_init=nn.init.normal_, b_init=-8.0):
        super(Model_MLP_L5, self).__init__()
        self.fc1 = nn.Linear(input_size, 3)
        self.fc2 = nn.Linear(3, 3)
        self.fc3 = nn.Linear(3, 3)
        self.fc4 = nn.Linear(3, 3)
        self.fc5 = nn.Linear(3, output_size)

        # 定义激活函数
        if act == 'sigmoid':
            self.act = F.sigmoid
        elif act == 'relu':
            self.act = F.relu
        elif act == 'lrelu':
            self.act = F.leaky_relu
        else:
            raise ValueError("Please enter sigmoid, relu or lrelu!")

        # 初始化权重和偏置
        self.init_weights(w_init, b_init)

    # 初始化线性层权重和偏置参数
    def init_weights(self, w_init, b_init):
        for m in self.children():
            if isinstance(m, nn.Linear):
                w_init(m.weight, mean=0.0, std=0.01)  # 对权重进行初始化
                constant_(m.bias, b_init)

    def forward(self, inputs):
        outputs = self.fc1(inputs)
        outputs = self.act(outputs)
        outputs = self.fc2(outputs)
        outputs = self.act(outputs)
        outputs = self.fc3(outputs)
        outputs = self.act(outputs)
        outputs = self.fc4(outputs)
        outputs = self.act(outputs)
        outputs = self.fc5(outputs)
        outputs = F.sigmoid(outputs)
        return outputs



def print_grads(runner, grad_norms):
    """ 打印模型每一层的梯度并计算其L2范数。 """
    print("The gradient of the Layers:")
    for name, param in runner.model.named_parameters():
        if param.requires_grad and param.grad is not None:
            grad_norm = param.grad.data.norm(2).item()  # 计算L2范数
            grad_norms[name].append(grad_norm)  # 记录L2范数
            print(f'Layer: {name}, Gradient Norm: {grad_norm}')

# 可视化梯度L2范数
def plot_grad_norms(grad_norms):
    layers = list(grad_norms.keys())
    norms = [np.mean(grad_norms[layer]) for layer in layers]
    x = np.arange(len(layers))  # x轴为层数
    plt.figure(figsize=(10, 6))
    plt.plot(x, norms, marker='o', label='ReLU', color='r')
    plt.ylabel('Gradient L2 Norm')
    plt.title('Gradient L2 Norm --Relu')
    plt.xticks(x, layers)
    plt.legend()
    # 设置 y 轴为对数坐标
    plt.yscale('log')
    # 设置 y 轴的范围
    plt.ylim(1e-8, 1)  # 设置下限为 1e-8，上限为 1
    # 设置 y 轴的刻度
    plt.yticks([1, 1e-1, 1e-2, 1e-3, 1e-4, 1e-5, 1e-6, 1e-7, 1e-8, 1e-9, 1e-12, 1e-17]
               )
    plt.grid()
    plt.tight_layout()
    plt.show()
# =============================数据集=======================
# 数据集构建
n_samples = 1000
X, y = make_moons(n_samples=n_samples, shuffle=True, noise=0.2)
# 划分数据集
num_train = 640  # 训练集样本数量
num_dev = 160  # 验证集样本数量
num_test = 200  # 测试集样本数量
# 根据指定数量划分数据集
X_train, y_train = X[:num_train], y[:num_train]  # 训练集
X_dev, y_dev = X[num_train:num_train + num_dev], y[num_train:num_train + num_dev]  # 验证集
X_test, y_test = X[num_train + num_dev:], y[num_train + num_dev:]  # 测试集
# 调整标签的形状，将其转换为[N, 1]的格式
y_train = y_train.reshape([-1, 1])
y_dev = y_dev.reshape([-1, 1])
y_test = y_test.reshape([-1, 1])

# ===============================模型训练=================
model =  Model_MLP_L5(input_size=2, output_size=1, act='relu')
#model =  Model_MLP_L5(input_size=2, output_size=1, act='lrelu')
torch.manual_seed(111)
lr = 0.01
optimizer = torch.optim.SGD(params=model.parameters(), lr=lr)
loss_fn = F.binary_cross_entropy
metric = accuracy
# 初始化L2范数记录字典
grad_norms = {name: [] for name, _ in model.named_parameters()}
# 实例化Runner类
runner = RunnerV2_2(model, optimizer, metric, loss_fn)

# 启动训练
runner.train([X_train, y_train], [X_dev, y_dev],
             num_epochs=1, log_epochs=0,
             save_path="best_model.pdparams",
             custom_print_log=lambda runner: print_grads(runner, grad_norms))
# 绘制梯度范数
plot_grad_norms(grad_norms)