李沐深度学习-权重衰退从零开始实验_李沐权重衰退

import torch
import torch.nn as nn
import numpy as np
import sys

sys.path.append("路径")
import d2lzh_pytorch as d2l

'''
该实验为了验证权重衰退的作用，特地设置样本数小于权重数量
样本特征维度=p  x1,x2....xp
模型： y=0.05+Σ(i=1,p) 0.01Xi +ε   噪声服从均值0，标准差0.01 正态分布
为了观察过拟合，因该设置训练数据集<模型复杂度（模型参数/特征数）
'''
n_train, n_test, num_inputs = 20, 100, 200
true_w, true_b = torch.ones((num_inputs, 1)) * 0.01, 0.05  # 不是分类问题，所以w只有一列，行数=特征数即可

features = torch.randn((n_train + n_test, num_inputs))
labels = torch.matmul(features, true_w) + true_b
labels += torch.tensor(np.random.normal(0, 0.01, labels.size()), dtype=torch.float)
train_features, test_features = features[:n_train, :], features[n_train:, :]
train_labels, test_labels = labels[:n_train], labels[n_train:]
x = torch.tensor([[1, 2, 3], [1, 2, 3]])
'''
            从零开始实现权重衰减：通过在目标函数后添加L2范数惩罚项来实现权重衰减
---------------------------------------------------------------------------初始化模型参数
'''


# 定义随机初始化模型参数的函数
def init_params():
    w = torch.randn((num_inputs, 1), requires_grad=True)
    b = torch.zeros(1, requires_grad=True)
    return [w, b]


'''
-----------------------------------------------------------------------定义范数惩罚项
'''


def l2_penalty(w):
    return (w ** 2).sum() / 2  # 为什么要进行sum()操作？


'''
----------------------------------------------------------定义训练和测试
'''
# 定义在训练集和测试集上分别训练和测试模型
batch_size, num_epochs, lr = 1, 100, 0.003
net, loss = d2l.linreg, d2l.square_loss
dataset = torch.utils.data.TensorDataset(train_features, train_labels)
train_iter = torch.utils.data.DataLoader(dataset, batch_size, shuffle=True)


def fit_and_plot(lambd, name):
    train_l, test_l = [], []
    w, b = init_params()
    for epoch in range(num_epochs):
        for X, y in train_iter:
            y_hat = net(X, w, b)  # net计算预测值
            # 添加L2惩罚项
            l = loss(y_hat, y) + lambd * l2_penalty(w)  # 计算预测值和实际值之间的损失
            l = l.sum()
            if w.grad is not None:  # 对参数的梯度清零
                w.grad.data.zero_()
                b.grad.data.zero_()
            l.backward()  # 这一步计算了参数的梯度，然后将梯度存储在参数中
            d2l.sgd([w, b], lr, batch_size)  # 这一步对参数中存储的梯度进行了计算，实现梯度下降
        train_l.append((loss(net(train_features, w, b), train_labels)).mean().item())
        test_l.append(loss(net(test_features, w, b), test_labels).mean().item())
    d2l.semilogy(range(1, num_epochs + 1), train_l, 'epoch', 'loss', name,
                 range(1, num_epochs + 1), test_l, ['train_l', 'test_l'])
    print(f'L2 norm of w:', w.norm().item())


'''
-----------------------------------------------------无权重衰退参与实验
'''
fit_and_plot(0, "无L2范数惩罚项过拟合观察")

# 权重衰退起作用就是使得权重参数w接近于0，减小了模型复杂度
'''
----------------------------------------------------权重衰退参与实验
'''
fit_and_plot(10, "L2范数惩罚项参与过拟合观察")

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87