调用库API实现以及手写实现
import torch
import torch.nn as nn
batch_szie=2 #sample
time_steps=3
embedding_dim=4 #channel
num_group=2
inputx=torch.randn(batch_szie,time_steps,embedding_dim)# N*L*C-
批归一化 BatchNorm1d
## 1. 批归一化 实现batch_norm并验证API ---- per channel
# NLP: [N,L,C] -> [C]
# CV: [N,C,H,W] -> [C]
batch_norm_op = torch.nn.BatchNorm1d(embedding_dim, affine=False)
bn_y = batch_norm_op(inputx.transpose(-1,-2)).transpose(-1,-2)
#手写batch_norm
bn_mean=inputx.mean(dim=(0,1),keepdim=True) #.unsqueeze(0).unsqueeze(0).repeat(batch_szie,time_steps,1) #C扩维成 N L C
bn_std=inputx.std(dim=(0,1),unbiased=False,keepdim=True) #.unsqueeze(0).unsqueeze(0).repeat(batch_szie,time_steps,1)
verify_bn_y=(inputx-bn_mean)/(bn_std+1e-5)
print("bn_y:",bn_y)
print("verify_bn_y:",verify_bn_y)
-
层归一化 LayerNorm
## 2. 层归一化 实现layer_normal并验证API ---- per sample per layer
# NLP: [N,L,C] -> [N,L]
# CV: [N,C,H,W] -> [N,H,W]
layer_norm_op=torch.nn.LayerNorm(embedding_dim,elementwise_affine=False)
ln_y=layer_norm_op(inputx)
#手写layer_norm
ln_mean=inputx.mean(dim=-1,keepdim=True)
ln_std=inputx.std(dim=-1,unbiased=False,keepdim=True)
verify_ln_y=(inputx-ln_mean)/(ln_std+1e-5)
print("ln_y:",ln_y)
print("verify_ln_y:",verify_ln_y) 
-
实例归一化 InstanceNorm1d
## 3. 实例归一化 实现instance_norm并验证API ---- per sample per channel 把实时不变的消掉了用于风格迁移
# NLP: [N,L,C] -> [N,C]
# CV: [N,C,H,W] -> [N,C]
ins_norm_op=torch.nn.InstanceNorm1d(embedding_dim)
in_y=ins_norm_op(inputx.transpose(-1,-2)).transpose(-1,-2)
#手写 ins_norm
in_mean=inputx.mean(dim=1,keepdim=True)
in_std=inputx.std(dim=1,unbiased=False,keepdim=True)
verify_in_y=(inputx-in_mean)/(in_std+1e-5)
print("in_y:",in_y)
print("verify_in_y:",verify_in_y) 
-
群归一化 GroupNorm
## 4. 群归一化 实现group_norm并验证API ------- per sample per group
# NLP: [N,G,L,C//G] -> [N,G]
# CV: [N,G,C//G,H,W] -> [N,G]
group_norm_op=torch.nn.GroupNorm(num_group,embedding_dim,affine=False)
gn_y=group_norm_op(inputx.transpose(-1,-2)).transpose(-1,-2)
#手写 group_norm
group_inputxs=torch.split(inputx,split_size_or_sections=embedding_dim // num_group,dim=-1) #tensor最后一维切分
results=[]
for g_inputx in group_inputxs:
gn_mean=g_inputx.mean(dim=(1,2),keepdim=True)
gn_std=g_inputx.std(dim=(1,2),unbiased=False,keepdim=True)
gn_result=(g_inputx-gn_mean)/(gn_std+1e-5)
results.append(gn_result)
verify_gn_y=torch.cat(results,dim=-1)
print("gn_y:",gn_y)
print("verify_gn_y:",verify_gn_y) 
-
权重归一化 weight_norm
### 5. 权重归一化 实现weight_norm并验证API
linear=nn.Linear(embedding_dim,3,bias=False)
wn_linear=torch.nn.utils.weight_norm(linear)
wn_linear_output=wn_linear(inputx)
print(wn_linear_output.shape)
#手写 weight_norm
weight_direction=linear.weight/(linear.weight.norm(dim=1,keepdim=True))
weight_magnitude=wn_linear.weight_g
print(weight_direction.shape)
print(weight_magnitude.shape)
verify_wn_linear_output=inputx@(weight_direction.transpose(-1,-2)) * (weight_magnitude.transpose(-1,-2))
print("wn_linear_output:",wn_linear_output)
print("verify_wn_linear_output:",verify_wn_linear_output) 
