devbox
繁依Fanyi0
这个屌丝很懒，什么也没留下！
关注作者
热门标签
热门文章
当前位置:   article > 正文

DataLoader与Dataset_if (epoch + 1) % val_interval == 0:

作者：繁依Fanyi0 | 2024-03-27 19:02:52

赞

踩

if (epoch + 1) % val_interval == 0:

1、DataLoader

torch.utils.data.DataLoader

功能：构建可迭代的数据装载器

  • dataset: Dataset类，决定数据从哪读取及如何读取
  • batchsize : 批大小
  • num _works: 是否多进程读取数据(设置多线程，但是一般机器如果没有更多GPU，会出错，尽量设置num_works=0)
  • shuffle: 每个epo ch是否乱序(random.shuffle有什么区别呢？)
  • drop_last：当样本数不能被batchsize整除时，是否舍弃最后一批数据

 Epoch：所有训练样本都已经输入到模型中，称为一个Epoch

Iteration：一批样本输入到模型中，称之为一个Iteration

Batchsize：批大小，决定一个Epoch有多少个Iteration

样本总数：80，Batchsize=8，1 Epoch = 10 Iteration

当样本总数为：87，Batchsize=8，一般默认都是drop_last=False

drop_last=True1 Epoch = 10 Iteration------>正确的！！！多余的7个被丢弃
drop_last=False1 Epoch = 10 Iteration------>错误的！！！

 

 

 

2、Dataset

torch.utils.data.Dataset

功能：Dataset抽象类，所有自定义的Dataset需要继承它，并且复写:__getitem__()

__getitem__():接收一个索引，返回一个样本

3、人民币二分类

数据集主要分为：一元和100元：每一个类单独文件夹存放！

程序主要按照下面步骤进行：

1、数据集处理

对数据集进行划分：原始图像每个类100张

train80
valid10
test10
  1. import torch
  2. import os
  3. # shutil：高级的 文件、文件夹、压缩包 处理模块
  4. import shutil
  5. import random
  6. # 显示当前路径
  7. # BASE_DIR = os.path.dirname(os.path.abspath('__file__'))
  8. # print(BASE_DIR)
  9.  
  10. # 创建新的目录
  11. def makedir(new_dir):
  12.     if not os.path.exists(new_dir):
  13.         os.makedirs(new_dir)
  14.  
  15. # 当模块被直接运行时，以下代码块将被运行，当模块是被导入时，代码块不被运行
  16. if __name__ == '__main__':
  17.     dataset_dir = os.path.join("data","RMB_data")
  18.     split_dir = os.path.join("data","rmb_split")
  19.     # 将每个类别进行划分Train/valid/test三个部分
  20.     train_dir = os.path.join(split_dir,"train")
  21.     valid_dir = os.path.join(split_dir,"valid")
  22.     test_dir = os.path.join(split_dir,"test")
  23.     
  24.     # 判断目录是否存在，不要也可以
  25. #     if not os.path.exists(dataset_dir):
  26. #         raise Exception("\n{} 不存在重新下载放到 {}下，并解压即可".format(
  27. #             dataset_dir, os.path.dirname(dataset_dir)))
  28.     
  29.     # 数据集划分比例
  30.     train_pct = 0.8
  31.     valid_pct = 0.1
  32.     test_pct = 0.1
  33.     
  34.     for root,dirs,files in os.walk(dataset_dir):
  35.         for sub_dir in dirs:
  36.             imgs = os.listdir(os.path.join(root,sub_dir))
  37.             imgs = list(filter(lambda x: x.endswith('.jpg'),imgs))
  38.             random.shuffle(imgs)
  39.             img_count = len(imgs)
  40.             
  41.             train_point = int(img_count*train_pct)
  42.             valid_point = int(img_count*(train_pct + valid_pct))
  43.             
  44.             for i in range(img_count):
  45.                 if i < train_point:
  46.                     out_dir = os.path.join(train_dir,sub_dir)
  47.                 elif i < valid_point:
  48.                     out_dir = os.path.join(valid_dir,sub_dir)
  49.                 else:
  50.                     out_dir = os.path.join(test_dir,sub_dir)
  51.                 
  52.                 makedir(out_dir)
  53.                 
  54.                 target_path = os.path.join(out_dir,imgs[i])
  55.                 src_path = os.path.join(dataset_dir,sub_dir,imgs[i])
  56.                 
  57.                 # 复制文件从源文件到目标文件
  58.                 shutil.copy(src_path,target_path)
  59.             print('Class:{}, train:{}, valid:{}, test:{}'.format(sub_dir, train_point, valid_point-train_point,
  60.                                                                  img_count-valid_point))
  61.             print("已在 {} 创建划分好的数据\n".format(out_dir))
Class:1, train:80, valid:10, test:10
已在 data\rmb_split\test\1 创建划分好的数据

Class:100, train:80, valid:10, test:10
已在 data\rmb_split\test\100 创建划分好的数据

2、开始训练：没有对数据集进行增强操作！！！

  1. import os
  2. # BASE_DIR = os.path.dirname(os.path.abspath('__file__'))/
  3. import numpy as np
  4. import torch
  5. import torch.nn as nn
  6. from torch.utils.data import DataLoader
  7. import torchvision.transforms as transforms
  8. import torch.optim as optim
  9. from matplotlib import pyplot as plt
  10.  
  11. # lenet存放在model文件夹中
  12. path_lenet = os.path.join("model","lenet.py")
  13. # print(path_lenet)
  14. # common_tools.py存放路径
  15. path_tools = os.path.join("tools","common_tools.py")
  16. # print(path_tools)
  17. # 下面是进行判断的操作
  18. # 不添加也可以
  19. # assert os.path.exists(path_lenet), "{}不存在，请将lenet.py文件放到 {}".format(path_lenet, os.path.dirname(path_lenet))
  20. # assert os.path.exists(path_tools), "{}不存在，请将common_tools.py文件放到 {}".format(path_tools, os.path.dirname(path_tools))
  21.  
  22. # import sys
  23. # hello_pytorch_DIR = os.path.abspath(os.path.dirname('__file__')+os.path.sep+".."+os.path.sep+"..")
  24. # print(hello_pytorch_DIR)
  25. # sys.path.append(hello_pytorch_DIR)
  26.  
  27. from model.lenet import LeNet
  28. from tools.my_dataset import RMBDataset
  29. from tools.common_tools import set_seed
  30.  
  31. set_seed()
  32. rmb_label = {"1":0,"100":1}
  33.  
  34. # 参数设置
  35. MAX_EPOCH = 10
  36. BATCH_SIZE = 16
  37. LR = 0.01
  38. log_interval = 10
  39. val_interval = 1
  40.  
  41. # 简单的拼接地址
  42. split_dir = os.path.join("data","rmb_split")
  43. # print(split_dir)
  44. # 训练集地址
  45. train_dir = os.path.join(split_dir,"train")
  46. # 验证集地址
  47. valid_dir = os.path.join(split_dir,"valid")
  48. # print(train_dir)
  49. # print(valid_dir)
  50.  
  51. # 数据增强
  52. norm_mean = [0.485,0.456,0.406]
  53. norm_std = [0.229,0.224,0.225]
  54.  
  55. train_transform = transforms.Compose([
  56.     transforms.Resize((32,32)),
  57.     transforms.RandomCrop(32,padding=4),
  58.     transforms.ToTensor(),
  59.     transforms.Normalize(norm_mean,norm_std),
  60. ])
  61.  
  62. valid_transform = transforms.Compose([
  63.     transforms.Resize((32,32)),
  64.     transforms.ToTensor(),
  65.     transforms.Normalize(norm_mean,norm_std),
  66. ])
  67.  
  68. # 构建MyDataset实例
  69. train_data = RMBDataset(data_dir=train_dir,transform=train_transform)
  70. valid_data = RMBDataset(data_dir=valid_dir,transform=valid_transform)
  71.  
  72. # 构建DataLoader
  73. train_loader = DataLoader(dataset=train_data,batch_size=BATCH_SIZE,shuffle=True)
  74. valid_loader = DataLoader(dataset=valid_data,batch_size=BATCH_SIZE)
  75.  
  76. # 模型
  77. net = LeNet(classes=2)
  78. # 初始化权重参数
  79. net.initialize_weights()
  80.  
  81. # 损失函数
  82. criterion = nn.CrossEntropyLoss()
  83. # 优化器
  84. optimizer = optim.SGD(net.parameters(),lr=LR,momentum=0.9)
  85. # 设置学习率下降策略
  86. scheduler = torch.optim.lr_scheduler.StepLR(optimizer,step_size=10,gamma=0.1)
  87.  
  88. train_curve = list()
  89. valid_curve = list()
  90.  
  91. for epoch in range(MAX_EPOCH):
  92.     loss_mean = 0.
  93.     correct = 0.
  94.     total = 0.
  95.     net.train()
  96.     
  97.     for i,data in enumerate(train_loader):
  98.         #forward
  99.         inputs,labels = data
  100.         outputs = net(inputs)
  101.         
  102.         # backward
  103.         optimizer.zero_grad()
  104.         loss = criterion(outputs,labels)
  105.         loss.backward()
  106.         
  107.         # update权重
  108.         optimizer.step()
  109.         
  110.         # 统计分类情况
  111.         _,predicted = torch.max(outputs.data,1)
  112.         total += labels.size(0)
  113.         correct += (predicted==labels).squeeze().sum().numpy()
  114.         
  115.         # 打印训练信息
  116.         loss_mean += loss.item()
  117.         train_curve.append(loss.item())
  118.         if (i+1) % log_interval == 0:
  119.             loss_mean = loss_mean / log_interval
  120.             print("Training:Epoch[{:0>3}/{:0>3}] Iteration[{:0>3}/{:0>3}] Loss: {:.4f} Acc:{:.2%}".format(
  121.                 epoch, MAX_EPOCH, i+1, len(train_loader), loss_mean, correct / total))
  122.             loss_mean = 0.
  123.     # 更新学习率
  124.     scheduler.step()
  125.     
  126.     # 验证模型
  127.     if (epoch+1) % val_interval == 0:
  128.         correct_val = 0.
  129.         total_val = 0.
  130.         loss_val = 0.
  131.         net.eval()
  132.         # 测试的话就不需要对梯度进行更新了
  133.         with torch.no_grad():
  134.             for j,data in enumerate(valid_loader):
  135.                 inputs,labels = data
  136.                 outputs = net(inputs)
  137.                 loss = criterion(outputs,labels)
  138.                 
  139.                 _,predicted = torch.max(outputs.data,1)
  140.                 total_val += labels.size(0)
  141.                 correct_val += (predicted==labels).squeeze().sum().numpy()
  142.                 
  143.                 loss_val += loss.item()
  144.                 
  145.             loss_val_epoch = loss_val / len(valid_loader)
  146.             valid_curve.append(loss_val_epoch)
  147.             print("Valid:\t Epoch[{:0>3}/{:0>3}] Iteration[{:0>3}/{:0>3}] Loss: {:.4f} Acc:{:.2%}".format(
  148.                 epoch, MAX_EPOCH, j+1, len(valid_loader), loss_val_epoch, correct_val / total_val))
  149.             
  150. train_x = range(len(train_curve))
  151. train_y = train_curve
  152.  
  153. train_iters = len(train_loader)
  154. valid_x = np.arange(1, len(valid_curve)+1) * train_iters*val_interval # 由于valid中记录的是epochloss，需要对记录点进行转换到iterations
  155. valid_y = valid_curve
  156.  
  157. plt.plot(train_x, train_y, label='Train')
  158. plt.plot(valid_x, valid_y, label='Valid')
  159.  
  160. plt.legend(loc='upper right')
  161. plt.ylabel('loss value')
  162. plt.xlabel('Iteration')
  163. plt.show()
Training:Epoch[000/010] Iteration[010/013] Loss: 0.6107 Acc:69.38%
Valid:	 Epoch[000/010] Iteration[003/003] Loss: 0.7187 Acc:57.89%
Training:Epoch[001/010] Iteration[010/013] Loss: 0.7202 Acc:65.00%
Valid:	 Epoch[001/010] Iteration[003/003] Loss: 0.3015 Acc:100.00%
Training:Epoch[002/010] Iteration[010/013] Loss: 0.1356 Acc:100.00%
Valid:	 Epoch[002/010] Iteration[003/003] Loss: 0.0002 Acc:100.00%
Training:Epoch[003/010] Iteration[010/013] Loss: 0.0214 Acc:99.38%
Valid:	 Epoch[003/010] Iteration[003/003] Loss: 0.0001 Acc:100.00%
Training:Epoch[004/010] Iteration[010/013] Loss: 0.0008 Acc:100.00%
Valid:	 Epoch[004/010] Iteration[003/003] Loss: 0.0000 Acc:100.00%
Training:Epoch[005/010] Iteration[010/013] Loss: 0.0000 Acc:100.00%
Valid:	 Epoch[005/010] Iteration[003/003] Loss: 0.0000 Acc:100.00%
Training:Epoch[006/010] Iteration[010/013] Loss: 0.0001 Acc:100.00%
Valid:	 Epoch[006/010] Iteration[003/003] Loss: 0.0000 Acc:100.00%
Training:Epoch[007/010] Iteration[010/013] Loss: 0.0001 Acc:100.00%
Valid:	 Epoch[007/010] Iteration[003/003] Loss: 0.0000 Acc:100.00%
Training:Epoch[008/010] Iteration[010/013] Loss: 0.0001 Acc:100.00%
Valid:	 Epoch[008/010] Iteration[003/003] Loss: 0.0000 Acc:100.00%
Training:Epoch[009/010] Iteration[010/013] Loss: 0.0000 Acc:100.00%
Valid:	 Epoch[009/010] Iteration[003/003] Loss: 0.0000 Acc:100.00%

 

3、测试训练的模型 

  1. # 使用一张图像进行测试
  2. # BASE_DIR = os.path.dirname(os.path.abspath('__file__'))
  3. test_dir = os.path.join("test_data")
  4.  
  5. test_data = RMBDataset(data_dir=test_dir, transform=valid_transform)
  6. valid_loader = DataLoader(dataset=test_data, batch_size=1)
  7.  
  8. # 进行测试
  9. for i, data in enumerate(valid_loader):
  10.     # forward
  11.     inputs, labels = data
  12.     outputs = net(inputs)
  13.     _, predicted = torch.max(outputs.data, 1)
  14.  
  15.     rmb = 1 if predicted.numpy()[0] == 0 else 100
  16.     print("模型获得{}元".format(rmb))

 结果：模型获得100元

附加文件：

(1)、lenet.py存放再model文件夹中

  1. import torch.nn as nn
  2. import torch.nn.functional as F
  3.  
  4. class LeNet(nn.Module):
  5.     def __init__(self, classes):
  6.         super(LeNet, self).__init__()
  7.         self.conv1 = nn.Conv2d(3, 6, 5)
  8.         self.conv2 = nn.Conv2d(6, 16, 5)
  9.         self.fc1 = nn.Linear(16*5*5, 120)
  10.         self.fc2 = nn.Linear(120, 84)
  11.         self.fc3 = nn.Linear(84, classes)
  12.  
  13.     def forward(self, x):
  14.         out = F.relu(self.conv1(x))
  15.         out = F.max_pool2d(out, 2)
  16.         out = F.relu(self.conv2(out))
  17.         out = F.max_pool2d(out, 2)
  18.         out = out.view(out.size(0), -1)
  19.         out = F.relu(self.fc1(out))
  20.         out = F.relu(self.fc2(out))
  21.         out = self.fc3(out)
  22.         return out
  23.  
  24.     def initialize_weights(self):
  25.         for m in self.modules():
  26.             if isinstance(m, nn.Conv2d):
  27.                 nn.init.xavier_normal_(m.weight.data)
  28.                 if m.bias is not None:
  29.                     m.bias.data.zero_()
  30.             elif isinstance(m, nn.BatchNorm2d):
  31.                 m.weight.data.fill_(1)
  32.                 m.bias.data.zero_()
  33.             elif isinstance(m, nn.Linear):
  34.                 nn.init.normal_(m.weight.data, 0, 0.1)
  35.                 m.bias.data.zero_()
  36.  
  37.  
  38. class LeNet2(nn.Module):
  39.     def __init__(self, classes):
  40.         super(LeNet2, self).__init__()
  41.         self.features = nn.Sequential(
  42.             nn.Conv2d(3, 6, 5),
  43.             nn.ReLU(),
  44.             nn.MaxPool2d(2, 2),
  45.             nn.Conv2d(6, 16, 5),
  46.             nn.ReLU(),
  47.             nn.MaxPool2d(2, 2)
  48.         )
  49.         self.classifier = nn.Sequential(
  50.             nn.Linear(16*5*5, 120),
  51.             nn.ReLU(),
  52.             nn.Linear(120, 84),
  53.             nn.ReLU(),
  54.             nn.Linear(84, classes)
  55.         )
  56.  
  57.     def forward(self, x):
  58.         x = self.features(x)
  59.         x = x.view(x.size()[0], -1)
  60.         x = self.classifier(x)
  61.         return x
  62.  
  63. class LeNet_bn(nn.Module):
  64.     def __init__(self, classes):
  65.         super(LeNet_bn, self).__init__()
  66.         self.conv1 = nn.Conv2d(3, 6, 5)
  67.         self.bn1 = nn.BatchNorm2d(num_features=6)
  68.  
  69.         self.conv2 = nn.Conv2d(6, 16, 5)
  70.         self.bn2 = nn.BatchNorm2d(num_features=16)
  71.  
  72.         self.fc1 = nn.Linear(16 * 5 * 5, 120)
  73.         self.bn3 = nn.BatchNorm1d(num_features=120)
  74.  
  75.         self.fc2 = nn.Linear(120, 84)
  76.         self.fc3 = nn.Linear(84, classes)
  77.  
  78.     def forward(self, x):
  79.         out = self.conv1(x)
  80.         out = self.bn1(out)
  81.         out = F.relu(out)
  82.  
  83.         out = F.max_pool2d(out, 2)
  84.  
  85.         out = self.conv2(out)
  86.         out = self.bn2(out)
  87.         out = F.relu(out)
  88.  
  89.         out = F.max_pool2d(out, 2)
  90.  
  91.         out = out.view(out.size(0), -1)
  92.  
  93.         out = self.fc1(out)
  94.         out = self.bn3(out)
  95.         out = F.relu(out)
  96.  
  97.         out = F.relu(self.fc2(out))
  98.         out = self.fc3(out)
  99.         return out
  100.  
  101.     def initialize_weights(self):
  102.         for m in self.modules():
  103.             if isinstance(m, nn.Conv2d):
  104.                 nn.init.xavier_normal_(m.weight.data)
  105.                 if m.bias is not None:
  106.                     m.bias.data.zero_()
  107.             elif isinstance(m, nn.BatchNorm2d):
  108.                 m.weight.data.fill_(1)
  109.                 m.bias.data.zero_()
  110.             elif isinstance(m, nn.Linear):
  111.                 nn.init.normal_(m.weight.data, 0, 1)
  112.                 m.bias.data.zero_()

(2)、common_tool.py存放在tools中

  1. import torch
  2. import random
  3. import psutil
  4. import numpy as np
  5. from PIL import Image
  6. import torchvision.transforms as transforms
  7.  
  8.  
  9. def transform_invert(img_, transform_train):
  10.     """
  11.     将data 进行反transfrom操作
  12.     :param img_: tensor
  13.     :param transform_train: torchvision.transforms
  14.     :return: PIL image
  15.     """
  16.     if 'Normalize' in str(transform_train):
  17.         norm_transform = list(filter(lambda x: isinstance(x, transforms.Normalize), transform_train.transforms))
  18.         mean = torch.tensor(norm_transform[0].mean, dtype=img_.dtype, device=img_.device)
  19.         std = torch.tensor(norm_transform[0].std, dtype=img_.dtype, device=img_.device)
  20.         img_.mul_(std[:, None, None]).add_(mean[:, None, None])
  21.  
  22.     img_ = img_.transpose(0, 2).transpose(0, 1)  # C*H*W --> H*W*C
  23.     if 'ToTensor' in str(transform_train):
  24.         img_ = img_.detach().numpy() * 255
  25.  
  26.     if img_.shape[2] == 3:
  27.         img_ = Image.fromarray(img_.astype('uint8')).convert('RGB')
  28.     elif img_.shape[2] == 1:
  29.         img_ = Image.fromarray(img_.astype('uint8').squeeze())
  30.     else:
  31.         raise Exception("Invalid img shape, expected 1 or 3 in axis 2, but got {}!".format(img_.shape[2]) )
  32.  
  33.     return img_
  34.  
  35.  
  36. def set_seed(seed=1):
  37.     random.seed(seed)
  38.     np.random.seed(seed)
  39.     torch.manual_seed(seed)
  40.     torch.cuda.manual_seed(seed)
  41.  
  42.  
  43. def get_memory_info():
  44.     virtual_memory = psutil.virtual_memory()
  45.     used_memory = virtual_memory.used/1024/1024/1024
  46.     free_memory = virtual_memory.free/1024/1024/1024
  47.     memory_percent = virtual_memory.percent
  48.     memory_info = "Usage Memory：{:.2f} G，Percentage: {:.1f}%，Free Memory：{:.2f} G".format(
  49.         used_memory, memory_percent, free_memory)
  50.     return memory_info

 

 

 

 

 

 

 

 

 

 

 

 

 

 

 

 

 

 

声明：本文内容由网友自发贡献，不代表【wpsshop博客】立场，版权归原作者所有，本站不承担相应法律责任。如您发现有侵权的内容，请联系我们。转载请注明出处：https://www.wpsshop.cn/w/繁依Fanyi0/article/detail/326375
推荐阅读
相关标签

Copyright © 2003-2013 www.wpsshop.cn 版权所有，并保留所有权利。

  

闽ICP备14008679号