李沐深度学习-权重衰退简洁实现_李牧深度学习

import torch
import torch.nn as nn
import torch.utils.data as Data
import numpy as np
import sys

sys.path.append("路径")
import d2lzh_pytorch as d2l

'''
---------------------------------------------------设计初始化样本数据
'''
batch_size, lr, num_inputs, n_train, n_test = 1, 0.001, 200, 20, 100
# 设计初始化模型参数，为了得到对应标签值y
true_w, true_b = torch.ones((num_inputs, 1)) * 0.01, 0.05
features = torch.randn((n_train + n_test, num_inputs))
train_features, test_features = features[:n_train, :], features[n_train:, :]
labels = torch.matmul(features, true_w) + true_b
labels += torch.tensor(np.random.normal(0, 0.01, size=labels.size()), dtype=torch.float)
train_labels, test_labels = labels[:n_train], labels[n_train:]
'''
---------------------------------------------------导入数据集
'''
dataset = Data.TensorDataset(train_features, train_labels)
train_iter = Data.DataLoader(dataset, batch_size, shuffle=True)
'''
---------------------------------------------------初始化模型及参数
# 这里的net[0]指的是 容器中第一个网络层
'''
net = nn.Sequential(
    nn.Linear(num_inputs, 1)
)
nn.init.normal_(net[0].weight, mean=0, std=0.01)
nn.init.normal_(net[0].bias, mean=0, std=0.01)

'''
--------------------------------------------------定义损失函数
'''
loss = nn.MSELoss()
'''
--------------------------------------------------训练模型
'''
num_epochs = 100


def train(wd, name):
    # net是整个容器的实例化对象，访问对象里面的网络层需要用索引，因为这里只有一个layer，所以用net[0]就是指第一个网络层
    optimizer_w = torch.optim.SGD(params=[net[0].weight], lr=lr, weight_decay=wd)  # 对权重参数衰退
    optimizer_b = torch.optim.SGD(params=[net[0].bias], lr=lr)  # 不对偏差衰退
    train_l, test_l = [], []
    for epochs in range(num_epochs):
        for X, y in train_iter:
            l = loss(net(X), y).mean()
            optimizer_w.zero_grad()
            optimizer_b.zero_grad()
            l.backward()
            optimizer_w.step()
            optimizer_b.step()
        train_l.append(loss(net(train_features), train_labels).mean().item())
        test_l.append(loss(net(test_features), test_labels).mean().item())
    d2l.semilogy(range(1, num_epochs + 1), train_l, 'epoch', 'loss', name,
                 range(1, num_epochs + 1), test_l, ['train', 'test'])
    print(f'L2 norm of w:', net[0].weight.data.norm().item())


'''
------------------------------------------------------'权重衰退不参与'
'''
# train(0, '权重衰退不参与')
'''
--------------------------------------------------------权重衰退参与模拟
'''
train(3, '权重衰退参与模拟')

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74