全连接卷积神经网络（FCN）

​ FCN是用来深度网络来做语义分割的奠基性工作，用转置卷积层来替换CNN最后的全连接层，从而可以实现对每个像素的预测

​ CNN(卷积神经网络)可以认为是一个预训练好的模型。CNN的最后一层是全局平均池化层，无论是什么形状的输入，最后输出都是1*1的，这对像素预测不太好

​ $1\times 1$卷积层用来变换输出通道，降低通道数，转置卷积层用于把图像放大。

1.代码实现

1.1FCN网络的实现(使用预训练模型)

import torch
import torchvision
from torch import nn
from torch.nn import functional as F
from d2l import torch as d2l

pretrained_net = torchvision.models.resnet18(weights=torchvision.models.ResNet18_Weights.IMAGENET1K_V1)
print(list(pretrained_net.children())[-3:]) # 看一下最后两层:全局平均池化层和全连接层

net = nn.Sequential(*list(pretrained_net.children())[:-2]) # 丢掉后面的几层

X = torch.rand(size=(1, 3, 320, 480))
print(net(X).shape) #发现高宽均减少至原来的1/32,即10和15,


num_classes = 21
# 1 * 1卷积层，变换输出通道为类别数
net.add_module('final_conv', nn.Conv2d(512, num_classes, kernel_size=1))
# 转置卷积层，我们想让高宽变为32倍，如果步幅为s，填充为s/2(假设是整数)，且卷积核的高和宽为2s，那么转置后的矩阵会放大s倍
net.add_module('transpose_conv', nn.ConvTranspose2d(num_classes, num_classes,
                                    kernel_size=64, padding=16, stride=32))

1.2 初始化转置卷积层

​ 有时我们需要将图像放大，即上采样(unsampling)。双线性插值(bilinear interpolation)是常用的上采样方式之一，它也经常用于初始化转置卷积层。

​ 双线性插值

​ 给定输入图像，需要计算上采样输出图像上的每个像素

1. 将输出图像的坐标(x,y)映射到输入图像的坐标$(x^{\prime },y^{\prime })$上。例如，根据输入与输出额尺寸之比来映射，请注意，映射后的$x^{\prime },y^{\prime }$仍然是实数
2. 在输入图像上找到离坐标$(x^{\prime },y^{\prime })$最近的4个像素
3. 输出图像在坐标$(x,y)$上的像素一句输入图像上这4个像素及其与$(x^{\prime },y^{\prime })$的相对距离来计算

'''初始化转置卷积层'''
#
def bilinear_kernel(in_channels, out_channels, kernel_size):
    # 对于奇数尺寸的核，中心位置为factor-1，对于偶数尺寸的核，中心位置为factor-0.5
    factor = (kernel_size + 1) // 2
    if kernel_size % 2 == 1:
        center = factor - 1
    else:
        center = factor - 0.5
    # 生成用于计算双线性核的网格
    #og 是一个包含两个张量的元组，一个张量是行向量，一个张量是列向量。
    og = (torch.arange(kernel_size).reshape(-1, 1),
          torch.arange(kernel_size).reshape(1, -1))

    # 计算双线性插值的过滤器,点到中心距离，比并且归一化到[0,1]的区间 形状为kernel_size 8* kernel_size
    filt = (1 - torch.abs(og[0] - center) / factor) * \
           (1 - torch.abs(og[1] - center) / factor)

    # 初始化权重张量
    weight = torch.zeros((in_channels, out_channels,
                          kernel_size, kernel_size))

    # 将过滤器应用到权重张量的每个输入通道和输出通道对
    weight[range(in_channels), range(out_channels), :, :] = filt
    return weight

conv_trans = nn.ConvTranspose2d(3, 3, kernel_size=4, padding=1, stride=2,
                                bias=False)
# 卷积核用双线性插值初始化
conv_trans.weight.data.copy_(bilinear_kernel(3, 3, 4))

img = torchvision.transforms.ToTensor()(d2l.Image.open('../img/catdog.jpg'))
X = img.unsqueeze(0)
Y = conv_trans(X)
out_img = Y[0].permute(1, 2, 0).detach()

d2l.set_figsize()
print('input image shape:', img.permute(1, 2, 0).shape)
d2l.plt.imshow(img.permute(1, 2, 0));
print('output image shape:', out_img.shape)
#input image shape: torch.Size([2160, 3840, 3])
#output image shape: torch.Size([4320, 7680, 3])
d2l.plt.imshow(out_img);
d2l.plt.show()

W = bilinear_kernel(num_classes, num_classes, 64)
net.transpose_conv.weight.data.copy_(W);

# filt 示例
(1 - torch.abs(og[0] - center) / factor) = tensor([[0.3333],
                                                   [0.6667],
                                                   [1.0000],
                                                   [0.6667],
                                                   [0.3333]])

(1 - torch.abs(og[1] - center) / factor) = tensor([[0.3333, 0.6667, 1.0000, 0.6667, 0.3333]])

filt = (1 - torch.abs(og[0] - center) / factor) * (1 - torch.abs(og[1] - center) / factor)
filt = tensor([[0.1111, 0.2222, 0.3333, 0.2222, 0.1111],
               [0.2222, 0.4444, 0.6667, 0.4444, 0.2222],
               [0.3333, 0.6667, 1.0000, 0.6667, 0.3333],
               [0.2222, 0.4444, 0.6667, 0.4444, 0.2222],
               [0.1111, 0.2222, 0.3333, 0.2222, 0.1111]])

1.3 训练

# crop_size 设定训练图片的大小
batch_size, crop_size = 32, (320, 480)
train_iter, test_iter = d2l.load_data_voc(batch_size, crop_size)

#使用转置卷积层的通道来预测像素的类别，所以需要在损失计算中指定通道维。
def loss(inputs, targets):
    # 对高做平均，对宽做平均，做两次均值，得到一个值
    return F.cross_entropy(inputs, targets, reduction='none').mean(1).mean(1)

num_epochs, lr, wd, devices = 5, 0.001, 1e-3, d2l.try_all_gpus()
trainer = torch.optim.SGD(net.parameters(), lr=lr, weight_decay=wd)
d2l.train_ch13(net, train_iter, test_iter, loss, trainer, num_epochs, devices)
d2l.plt.show()
def predict(img):
    X = test_iter.dataset.normalize_image(img).unsqueeze(0)
    pred = net(X.to(devices[0])).argmax(dim=1)
    return pred.reshape(pred.shape[1], pred.shape[2])

def label2image(pred):
    colormap = torch.tensor(d2l.VOC_COLORMAP, device=devices[0])
    X = pred.long()
    return colormap[X, :]

voc_dir = d2l.download_extract('voc2012', 'VOCdevkit/VOC2012')
test_images, test_labels = d2l.read_voc_images(voc_dir, False)
n, imgs = 4, []
for i in range(n):
    crop_rect = (0, 0, 320, 480)
    X = torchvision.transforms.functional.crop(test_images[i], *crop_rect)
    pred = label2image(predict(X))
    imgs += [X.permute(1,2,0), pred.cpu(),
             torchvision.transforms.functional.crop(
                 test_labels[i], *crop_rect).permute(1,2,0)]
d2l.show_images(imgs[::3] + imgs[1::3] + imgs[2::3], 3, n, scale=2);
d2l.plt.show()

loss 0.413, train acc 0.870, test acc 0.853