devbox
2023面试高手
这个屌丝很懒，什么也没留下！
关注作者
热门标签
热门文章
当前位置:   article > 正文

【日常】手写卷积神经网络（conv+relu+maxpooling+linear+relu+linear+softmax+交叉熵损失+正则）_convolution relu max pooling

作者：2023面试高手 | 2024-02-10 18:47:12

赞

踩

convolution relu max pooling

第二次课程作业，手写CNN进行手写数字识别。据说还要手写一次RNN（这酸爽还有谁）。

作业坑点在于Assignment给定的这个架构与默认参数效果非常差，结果怎么调都调不好，一开始一直是以为写错了，但是检查了很久都发现不了哪里写错了。后来用keras实现了同样的架构，发现确实效果差得惊人（跟瞎猜没有区别），然后随便改了改（加一个卷积层或者加一个dense层，甚至只要修改一下优化函数，SGD的效果比adagrad及adam都要差太多），就能得到比较好的效果。这作业做完之后觉得神经网络真的是科学中的玄学，玄学中的佛学。

代码架构如下所示：

  1. ../
  2. 	train_small.py														 # 主文件
  3. 	test_grad.py														 # 用于测试layers.py的层前向及反向传播是否正确
  4. 	dataset.py															 # 读取数据
  5. 	check_gradient.py													 # test_grad.py常用的函数
  6. ../nn/
  7. 		cnn.py															 # CNN架构
  8. 		layers.py														 # 各个层的类
  9. 		loss.py															 # 损失函数
  10. 		optimizer.py													 # 优化器
  11. 		utils.py														 # 其他
  12. ../cifar-10-batches-py
  13. 		# 数据文件, 通过下面的方法下载
  14. 		# wget http://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz
  15. 		# tar -xzvf cifar-10-python.tar.gz
  16. 		# rm cifar-10-python.tar.gz

代码下面自取：

train_small.py

  1. #-*- coding:UTF-8 -*-
  2. import os
  3. import sys
  4. sys.path.append("{}\\nn".format(os.getcwd()))							 # 最近python老是犯病
  5. import numpy as np
  6. from nn.cnn import CNN
  7. from nn.optimizer import SGD
  8. from nn.utils import accuracy
  9. from dataset import get_cifar10_data
  10. from matplotlib import pyplot as plt
  11. from collections import Counter
  12.  
  13. def train(model,X_train,y_train,X_val,y_val,batch_size,n_epochs,
  14. 	lr=1e-2,
  15. 	lr_decay=0.8,
  16. 	momentum=0.0,
  17. 	wd=0.0,
  18. 	verbose=True,
  19. 	print_level=1,
  20. ):																		 # 模型训练
  21. 	print("开始训练...")
  22. 	n_train = X_train.shape[0]											 # 获取训练集样本数量
  23. 	iterations_per_epoch = max(n_train//batch_size,1)					 # 每个epoch的迭代次数
  24. 	n_iterations = n_epochs*iterations_per_epoch						 # 总的迭代次数
  25. 	loss_hist = []														 # 储存每个epoch的损失函数值
  26. 	opt_params = {"lr":lr,"weight_decay":wd,"momentum":momentum}		 # 设置最优参数, 预设了学习率
  27. 	print("训练样本数：{}".format(n_train))
  28. 	print("每阶段迭代次数：{}".format(iterations_per_epoch))
  29. 	print("阶段数：{}".format(n_epochs))
  30. 	print("总迭代次数：{}".format(n_iterations))
  31. 	count = 0
  32. 	for epoch in range(n_epochs):										 # 遍历所有epoch
  33. 		for t in range(iterations_per_epoch):							 # 遍历总的迭代次数
  34. 			count += 1
  35. 			batch_mask = np.random.choice(n_train,batch_size)			 # 从训练集中随机取batch_size个出来
  36. 			X_batch = X_train[batch_mask]								 # 输入采样
  37. 			y_batch = y_train[batch_mask]								 # 输出采样
  38. 			loss,score = model.oracle(X_batch,y_batch)					 # 评估函数值与梯度
  39. 			loss_hist.append(loss)										 # 记录损失值								
  40. 			sgd = SGD(model.param_groups,**opt_params)					 # 定义优化器
  41. 			sgd.step()													 # 进行随机梯度下降
  42. 			if verbose and t%print_level==0:							 # 输出训练损失				
  43. 				train_acc = accuracy(score,y_batch)						 # 计算精度
  44. 				print("(Iteration {}/{},epoch {})loss:{},accu:{}".format(
  45. 					count,n_iterations,epoch,loss_hist[-1],train_acc))
  46. 			if t==iterations_per_epoch-1: opt_params["lr"] *= lr_decay
  47. 	
  48. 	""" 实验绘图 """
  49. 	plt.close()
  50. 	plt.figure()
  51. 	plt.plot(loss_hist,label="training loss")
  52. 	plt.legend(loc="best")
  53. 	plt.show()
  54.  
  55. if __name__ == "__main__":
  56. 	model = CNN()														 # 初始化卷积神经网络
  57. 	data = get_cifar10_data()											 # 读取数据
  58. 	num_train = 100													 # 训练集数量
  59. 	data = {															 # 把data转为字典形式
  60. 		"X_train": data["X_train"][:num_train],
  61. 		"y_train": data["y_train"][:num_train],
  62. 		"X_val": data["X_val"],
  63. 		"y_val": data["y_val"],
  64. 	}
  65. 	X_train,y_train,X_val,y_val = data["X_train"],data["y_train"],data["X_val"],data["y_val"]
  66. 	train(model,X_train,y_train,X_val,y_val,50,50)

test_grad.py

  1. #-*- coding:UTF-8 -*-
  2. import os
  3. import sys
  4. sys.path.append("{}\\nn".format(os.getcwd()))							 # 最近python老是犯病
  5. import numpy as np
  6. from nn.cnn import CNN
  7. from check_gradient import *
  8. from nn.layers import Conv,MaxPool,Linear,Relu
  9.  
  10. def rel_error(x,y):														 # 计算相对误差
  11. 	return np.nanmax(np.abs(x-y)/(np.maximum(1e-8,np.abs(x)+np.abs(y))))
  12.  
  13. """ relu激活层就不管了, 反正错了也不是我的锅 """
  14. check_conv_forward = True
  15. #check_conv_forward = False
  16. check_conv_backward = True
  17. #check_conv_backward = False
  18. check_linear_forward = True
  19. #check_linear_forward = False
  20. check_linear_backward = True
  21. #check_linear_backward = False
  22. check_pool_forward = True
  23. #check_pool_forward = False
  24. check_pool_backward = True
  25. #check_pool_backward = False
  26.  
  27. if check_conv_forward:													 # 检查卷积层前向传播是否正确
  28. 	x_shape = (2,3,4,4)													 # 2个样本, 3个轨道, 4×4像素
  29. 	w_shape = (3,3,4,4)													 # 3个输入轨道, 3个输出轨道, 4×4卷积核
  30. 	x = np.linspace(-0.1,0.5,num=np.prod(x_shape)).reshape(x_shape)		 # 2×3×4×4的样本
  31. 	w = np.linspace(-0.2,0.3,num=np.prod(w_shape)).reshape(w_shape)		 # 3×3×4×4的卷积过滤器
  32. 	b = np.linspace(-0.1,0.2,num=3)										 # 3个偏差
  33. 	conv = Conv(4,4,3,3,2,1)											 # 卷积层: 4×4卷积核, 3个输入轨道, 3个输出轨道, 步长2, padding值为1
  34. 	conv.params["w"]["param"] = w										 # 设置卷积过滤器
  35. 	conv.params["b"]["param"] = b										 # 设置偏差值
  36. 	out = conv(x)														 # Layer类是可以被调用的, 返回前向传播的结果
  37. 	correct_out = np.array([											 # 事先计算好的正确输出维度为: 2×3×2×2
  38. 		[
  39. 			[[-0.08759809,-0.10987781],[-0.18387192,-0.2109216]],
  40. 			[[0.21027089,0.21661097],[0.22847626,0.23004637]],
  41. 			[[0.50813986,0.54309974],[0.64082444,0.67101435]]
  42. 		],
  43. 		[
  44. 			[[-0.98053589,-1.03143541],[-1.19128892,-1.24695841]],
  45. 			[[0.69108355,0.66880383],[0.59480972,0.56776003]],
  46. 			[[2.36270298,2.36904306],[2.38090835,2.38247847]]
  47. 		]
  48. 	])
  49. 	print("Testing convolutional forward...")
  50. 	print("difference: {}".format(rel_error(out,correct_out)))
  51.  
  52. if check_conv_backward:													 # 检查卷积层反向传播是否正确
  53. 	np.random.seed(231)													 # 初始化随机种子
  54. 	x = np.random.randn(2,3,16,16)										 # 2个样本, 3个轨道, 16×16像素
  55. 	w = np.random.randn(3,3,3,3)										 # 3个输入轨道, 3个输出轨道, 3×3的卷积核
  56. 	b = np.random.randn(3,)												 # 3个偏差
  57. 	dout = np.random.randn(2,3,14,14)									 # 随机给定一个输出dout: 这个应该是被假设为下一层反向传回当层的输入梯度值
  58. 	conv = Conv(3,3,3,3,1,0)											 # 初始化一个3个输入轨道, 3个输出轨道, 3×3的卷积核, 步长为1且不padding
  59. 	conv.params["w"]["param"] = w										 # 设置卷积过滤器
  60. 	conv.params["b"]["param"] = b										 # 设置偏差
  61. 	out = conv(x)														 # 计算卷积层在输入x后的输出结果
  62. 	dx = conv.backward(dout,x)											 # 计算对应dout输入的反向传播输出
  63. 	dx_num = eval_numerical_gradient_array(conv,x,dout)					 # dx_num维度与x,dx完全相同
  64. 	params = conv.params												 # 获取卷积层的参数
  65.  
  66. 	def fw(v):															 # 计算用v过滤器后的输出结果
  67. 		tmp = params["w"]["param"]								
  68. 		params["w"]["param"] = v							
  69. 		f_w = conv(x)										
  70. 		params["w"]["param"] = tmp							
  71. 		return f_w								
  72.  
  73. 	def fb(v):															 # 计算用v偏差后的输出结果
  74. 		tmp = params["b"]["param"]
  75. 		params["b"]["param"] = v
  76. 		f_b = conv(x)
  77. 		params["b"]["param"] = tmp
  78. 		return f_b
  79.  
  80. 	dw = params["w"]["grad"]											 # 卷积过滤器的梯度
  81. 	dw_num = eval_numerical_gradient_array(fw,w,dout)					 # dw_num维度与w,dw完全相同
  82. 	db = params["b"]["grad"]											 # db_num维度与b,db完全相同
  83. 	db_num = eval_numerical_gradient_array(fb,b,dout)
  84. 	
  85. 	print("Testing convolutional backward")
  86. 	print("dx error: {}".format(rel_error(dx_num,dx)))
  87. 	print("dw error: {}".format(rel_error(dw_num,dw)))
  88. 	print("db error: {}".format(rel_error(db_num,db)))
  89.  
  90. if check_linear_forward:												 # 检查线性层前向传播是否正确
  91. 	x_shape = (2,3,4,4)													 # 2个样本, 3个轨道, 4×4像素
  92. 	w_shape = (3*4*4,64)												 # 从48维映射到64维
  93. 	b_shape = (1,64)
  94. 	x = np.linspace(-0.1,0.5,num=np.prod(x_shape)).reshape(x_shape)
  95. 	w = np.linspace(-0.2,0.3,num=np.prod(w_shape)).reshape(w_shape)	
  96. 	b = np.linspace(-0.1,0.2,num=64).reshape(b_shape)					 # 64个偏差
  97. 	linear = Linear(3*4*4,64)
  98. 	linear.params["w"]["param"] = w										 # 设置卷积过滤器
  99. 	linear.params["b"]["param"] = b										 # 设置偏差值
  100. 	out = linear(x)														 # Layer类是可以被调用的, 返回前向传播的结果
  101.  
  102. 	correct_out = np.dot(x.reshape(2,48),w)+b							 # 单纯的全连接层
  103. 	print("Testing linear forward...")
  104. 	print("difference: {}".format(rel_error(out,correct_out)))		
  105.  
  106. if check_linear_backward:												 # 检查线性层反向传播是否正确
  107. 	np.random.seed(231)													 # 初始化随机种子
  108. 	x = np.random.randn(2,3,4,4)										 # 2个样本, 3个轨道, 16×16像素
  109. 	w = np.random.randn(3*4*4,64)										 # 3个输入轨道, 3个输出轨道, 3×3的卷积核
  110. 	b = np.random.randn(1,64)											 # 3个偏差
  111. 	dout = np.random.randn(2,64)										 # 随机给定一个输出dout: 这个应该是被假设为下一层反向传回当层的输入梯度值
  112. 	linear = Linear(3*4*4,64)											 # 初始化一个3个输入轨道, 3个输出轨道, 3×3的卷积核, 步长为1且不padding
  113. 	linear.params["w"]["param"] = w										 # 设置卷积过滤器
  114. 	linear.params["b"]["param"] = b										 # 设置偏差
  115. 	out = linear(x)														 # 计算卷积层在输入x后的输出结果
  116. 	dx = linear.backward(dout,x)										 # 计算对应dout输入的反向传播输出
  117. 	dx_num = eval_numerical_gradient_array(linear,x,dout)				 # dx_num维度与x,dx完全相同
  118. 	dx_num = dx_num.reshape(dx_num.shape[0],-1)							 # 调整dx_num维度为二维张量
  119. 	params = linear.params												 # 获取卷积层的参数
  120.  
  121. 	def fw(v):															 # 计算用v过滤器后的输出结果
  122. 		tmp = params["w"]["param"]								
  123. 		params["w"]["param"] = v							
  124. 		f_w = linear(x)										
  125. 		params["w"]["param"] = tmp							
  126. 		return f_w								
  127.  
  128. 	def fb(v):															 # 计算用v偏差后的输出结果
  129. 		tmp = params["b"]["param"]
  130. 		params["b"]["param"] = v
  131. 		f_b = linear(x)
  132. 		params["b"]["param"] = tmp
  133. 		return f_b
  134.  
  135. 	dw = params["w"]["grad"]											 # 卷积过滤器的梯度
  136. 	dw_num = eval_numerical_gradient_array(fw,w,dout)					 # dw_num维度与w,dw完全相同
  137. 	db = params["b"]["grad"]											 # db_num维度与b,db完全相同
  138. 	db_num = eval_numerical_gradient_array(fb,b,dout)
  139. 	
  140. 	print("Testing linear backward")
  141. 	print("dx error: {}".format(rel_error(dx_num,dx)))
  142. 	print("dw error: {}".format(rel_error(dw_num,dw)))
  143. 	print("db error: {}".format(rel_error(db_num,db)))
  144.  
  145. if check_pool_forward:													 # 检查池化层前向传播是否正确
  146. 	x_shape = (2,3,4,4)													 # 2个样本, 3个轨道, 4×4像素
  147. 	x = np.linspace(-0.1,0.5,num=np.prod(x_shape)).reshape(x_shape)		 # 2×3×4×4的样本
  148. 	pool = MaxPool(kernel_size=2,stride=2,padding=0)					 # 2×2的池化层, 步长2且不padding
  149. 	out = pool(x)
  150. 	out_shape = (2,3,2,2)
  151. 	correct_out = np.zeros(out_shape)
  152. 	for i in range(out_shape[0]):
  153. 		for j in range(out_shape[1]):
  154. 			for k in range(out_shape[2]):
  155. 				for l in range(out_shape[3]):
  156. 					correct_out[i,j,k,l] = x[i,j,2*k+1,2*l+1]			 # 因为是按顺序排列的, 每个窗口的右下角恰好为最大值
  157. 	print("Testing pooling forward...")
  158. 	print("difference: {}".format(rel_error(out,correct_out)))	
  159.  
  160. if check_pool_backward:													 # 检查池化层反向传播是否正确
  161. 	np.random.seed(231)													 # 初始化随机种子
  162. 	x = np.random.randn(3,2,8,8)										 # 随机给定一个输入x
  163. 	dout = np.random.randn(3,2,4,4)										 # 随机给定一个输出dout: 这个应该是被假设为下一层反向传回当层的输入梯度值
  164. 	pool = MaxPool(kernel_size=2,stride=2,padding=0)					 # 初始化一个2×2的池化核, 不padding且步长为2, 其输出恰好为8×8-->4×4
  165. 	out = pool(x)														 # 得出一个对应x的前向输出
  166. 	dx = pool.backward(dout,x)											 # 得出一个对应dout的反向输出的梯度
  167. 	dx_num = eval_numerical_gradient_array(pool,x,dout)					 # 调用手动计算梯度的函数: dx_num的维度与x的维度完全相同(3,2,8,8)
  168. 	print("Testing pooling backward:")
  169. 	print("dx error: ",rel_error(dx,dx_num))							 # 你的误差应该在1e-12左右
  170.  
  171.

check_gradient.py

  1. #-*- coding:UTF-8 -*-
  2. import sys
  3. import numpy as np
  4. from random import randrange
  5.  
  6. if sys.version_info>=(3,0):												 # 判断python版本, 对于python3.x需要重写xrange生成器
  7. 	def xrange(*args,**kwargs):
  8. 		return iter(range(*args,**kwargs))
  9.  
  10. def eval_numerical_gradient(f,x,
  11. 	verbose=True,
  12. 	h=1e-5
  13. ):																		 # 求函数f在点x处的梯度的一个非常幼稚的实施方案: f为只有一个参数的函数, x为需要求梯度的点或者是数组
  14. 	fx = f(x)															 # 求函数在给定点的函数值
  15. 	grad = np.zeros_like(x)												 # 预设梯度为与x形状相同的玩意儿
  16. 	it = np.nditer(x,flags=["multi_index"],op_flags=["readwrite"])		 # 把x做成numpy生成器		 
  17. 	while not it.finished:												 # 遍历x中的所有元素, 一个个求偏导
  18. 		ix = it.multi_index												 # 获取生成器index
  19. 		oldval = x[ix]													 # 获取对应index的数值
  20. 		x[ix] = oldval+h 												 # 右移一段
  21. 		fxph = f(x)														 # 计算右函数值
  22. 		x[ix] = oldval-h												 # 左移一段
  23. 		fxmh = f(x)														 # 计算左函数值
  24. 		x[ix] = oldval													 # 还原该index上原先的值
  25. 		grad[ix] = (fxph-fxmh)/(2*h)									 # 计算偏导数 
  26. 		if verbose: print(ix,grad[ix])									 # 输出梯度
  27. 		it.iternext()													 # 步入下一次
  28. 	return grad
  29.  
  30. def eval_numerical_gradient_array(f,x,df,
  31. 	h=1e-5
  32. ):																		 # 对于一个接收了一个数组并返回数组的函数来计算数值的梯度
  33. 	grad = np.zeros_like(x)												 # 这里的输入x的维度为N_samples×n_Channels×Height×Width
  34. 	it = np.nditer(x,flags=["multi_index"],op_flags=["readwrite"])		 # 把x做成一个迭代生成器
  35. 	while not it.finished:
  36. 		ix = it.multi_index												 # 获取生成器index
  37. 		oldval = x[ix]													 # 获取对应index的数值
  38. 		x[ix] = oldval+h												 # 右移一段
  39. 		pos = f(x).copy()												 # 计算右函数值
  40. 		x[ix] = oldval-h												 # 左移一段
  41. 		neg = f(x).copy()												 # 计算左函数值
  42. 		x[ix] = oldval													 # 还原该index上原先的值
  43. 		grad[ix] = np.sum((pos-neg)*df)/(2*h)							 # 这个乘以df是什么操作?
  44. 		it.iternext()
  45. 	return grad
  46.  
  47. def eval_numerical_gradient_blobs(f,inputs,output,
  48. 	h=1e-5
  49. ):																		 # 对一个操作输入与输出斑点的函数计算数值梯度: f为函数(f接收几个输入斑点作为参数,然后跟随一个斑点用于写入输出==>y=f(x,w,out),x与w为输入斑点,f的结果将被写入out),inputs为输入的斑点,output为输出斑点,h为步长
  50. 	numeric_diffs = []
  51. 	for input_blob in inputs:
  52. 		diff = np.zeros_like(input_blob.diffs)
  53. 		it = np.nditer(
  54. 			input_blob.vals,
  55. 			flags=["multi_index"],
  56. 			op_flags=["readwrite"]
  57. 		)
  58. 		while not it.finished:
  59. 			idx = it.multi_index
  60. 			orig = input_blob.vals[idx]
  61. 			input_blob.vals[idx] = orig+h
  62. 			f(*(inputs + (output,)))
  63. 			pos = np.copy(output.vals)
  64. 			input_blob.vals[idx] = orig-h
  65. 			f(*(inputs + (output,)))
  66. 			neg = np.copy(output.vals)
  67. 			input_blob.vals[idx] = orig
  68. 			diff[idx] = np.sum((pos-neg)*output.diffs)/(2.0*h)
  69. 			it.iternext()
  70. 		numeric_diffs.append(diff)
  71. 	return numeric_diffs
  72.  
  73. def eval_numerical_gradient_net(net,inputs,output,
  74. 	h=1e-5
  75. ):
  76. 	result = eval_numerical_gradient_blobs(
  77. 		lambda *args: net.forward(),inputs,output,h=h
  78. 	)
  79. 	return result
  80.  
  81. def grad_check_sparse(f,x,analytic_grad,
  82. 	num_checks=10,
  83. 	h=1e-5
  84. ):																		 # 通过采样来检查梯度的稀疏性
  85. 	for i in range(num_checks):											 # 检查num_checks个点
  86. 		ix = tuple([randrange(m) for m in x.shape])						 # 随机生成一个位置
  87. 		oldval = x[ix]													 # 获取对应index的数值
  88. 		x[ix] = oldval+h 												 # 右移一段
  89. 		fxph = f(x)														 # 计算左函数值
  90. 		x[ix] = oldval-h												 # 左移一段
  91. 		fxmh = f(x)														 # 计算右函数值
  92. 		x[ix] = oldval													 # 还原该index上原先的值
  93. 		grad[ix] = (fxph-fxmh)/(2*h)									 # 计算偏导数 
  94. 		grad_analytic = analytic_grad[ix]
  95. 		error = abs(grad_numerical-grad_analytic)
  96. 		total = abs(grad_numerical)+abs(grad_analytic)
  97. 		rel_error = error/total
  98. 		print("numerical: %f analytic: %f,relative error: %e" % (
  99. 			grad_numerical,
  100. 			grad_analytic,
  101. 			rel_error
  102. 		))

dataset.py

  1. #-*- coding:UTF-8 -*-
  2. import os
  3. import pickle
  4. import numpy as np
  5.  
  6. def load_cifar_batch(filename):											 # 导入cifar-10数据集的一个batch
  7. 	with open(filename,"rb") as f:
  8. 		datadict = pickle.load(f,encoding="latin1")	
  9. 		X = datadict["data"]											 # 获取输入字段数据
  10. 		Y = datadict["labels"]											 # 获取输出标签数据
  11. 		X = X.reshape(10000,3,32,32).transpose(0,2,3,1).astype("float")	 # 每个batch里面都是10000个样本, 改变维度顺序把RGB轨道调到最后
  12. 		Y = np.array(Y)
  13. 		return X,Y
  14.  
  15. def load_cifar10():														 # 导入全部cifar-10数据集: 5个训练集batch与1个测试集test_batch
  16. 	xs = []
  17. 	ys = []
  18. 	for b in range(1,6):
  19. 		f = os.path.join("cifar-10-batches-py","data_batch_%d" % (b,))
  20. 		X,Y = load_cifar_batch(f)
  21. 		xs.append(X)
  22. 		ys.append(Y)
  23. 	Xtr = np.concatenate(xs)											 # 把5个分文件的输入字段拼接
  24. 	Ytr = np.concatenate(ys)											 # 把5个分文件的输出标签拼接
  25. 	del X,Y
  26. 	Xte,Yte = load_cifar_batch(
  27. 		os.path.join("cifar-10-batches-py","test_batch")
  28. 	)
  29. 	"""
  30. 		Xtr: 训练集(50000,32,32,3)
  31. 		Ytr: 训练集(50000,)
  32. 		Xte: 训练集(10000,32,32,3)
  33. 		Yte: 训练集(10000,)
  34. 	"""
  35. 	return Xtr,Ytr,Xte,Yte
  36.  
  37. def get_cifar10_data(
  38. 	n_train=49000,
  39. 	n_val=1000,
  40. 	n_test=10000,
  41. 	subtract_mean=True
  42. ):																		 # 从磁盘导入cifar-10数据集并进行预处理, 这与SVM的预处理操作相同, 我们将它打包为一个函数
  43. 	X_train,y_train,X_test,y_test = load_cifar10()						 # 导入全部数据
  44. 	mask = list(range(n_train,n_train + n_val))							 # 用于划分验证集
  45. 	X_val = X_train[mask]
  46. 	y_val = y_train[mask]
  47. 	mask = list(range(n_train))											 # 用于划分训练集
  48. 	X_train = X_train[mask]
  49. 	y_train = y_train[mask]
  50. 	mask = list(range(n_test))											 # 用于划分测试集
  51. 	X_test = X_test[mask]
  52. 	y_test = y_test[mask]
  53. 	if subtract_mean:													 # 数据标准化, 减去平均图片
  54. 		mean_image = np.mean(X_train,axis=0)
  55. 		X_train -= mean_image
  56. 		X_val -= mean_image
  57. 		X_test -= mean_image
  58. 	""" 调整输入特征维度的顺序从而使得channel处于最前面的位置 """
  59. 	X_train = X_train.transpose(0,3,1,2).copy()
  60. 	X_val = X_val.transpose(0,3,1,2).copy()
  61. 	X_test = X_test.transpose(0,3,1,2).copy()
  62. 	return {															 # 将数据打包为一个字典
  63. 	  "X_train": X_train,
  64. 	  "y_train": y_train,
  65. 	  "X_val": X_val,
  66. 	  "y_val": y_val,
  67. 	  "X_test": X_test,
  68. 	  "y_test": y_test,
  69. 	}
  70.  
  71. if __name__ == "__main__":
  72. 	Xtr,ytr,Xte,yte = load_cifar10()
  73. 	print(Xtr.shape)
  74. 	print(ytr.shape)
  75. 	print(Xte.shape)
  76. 	print(yte.shape)
  77. 	print(set(ytr.tolist()))
  78. 	data = get_cifar10_data()
  79. 	for key,value in data.items():
  80. 		print(key,value.shape)

nn/cnn.py

  1. #-*- coding:UTF-8 -*-
  2. import math
  3. import time
  4. import numpy as np
  5. from loss import SoftmaxCE,softmax
  6. from layers import Conv,Relu,MaxPool,Linear
  7.  
  8. class CNN(object):														 # 卷积神经网络架构类: 卷积+激活+池化+线性+激活+线性+软大
  9. 	def __init__(self,
  10. 		image_size=(3,32,32),
  11. 		channels=3,
  12. 		conv_kernel=7,
  13. 		pool_kernel=2,
  14. 		hidden_units=100,
  15. 		n_classes=10,
  16. 	):																	 # 构造函数: 初始化神经网络,定义网络层
  17. 		""" 类构造参数 """												
  18. 		self.image_size = image_size									 # 图片形状3×H×W
  19. 		self.channels = channels										 # 卷积层的轨道数
  20. 		self.conv_kernel = conv_kernel									 # 卷积层核维度
  21. 		self.pool_kernel = pool_kernel									 # 池化层核维度
  22. 		self.hidden_units = hidden_units								 # 线性传播中隐层单元数量
  23. 		self.n_classes = n_classes										 # 也许是总层数吧
  24. 		""" 类常用参数 """
  25. 		channel,height,width = self.image_size							 # 这三个变量将记录卷积部分的输入轨道与维度
  26. 		self.conv_stride = 1											 # 卷积核移动步长
  27. 		self.conv_padding = 0											 # 卷积层对输入padding数量
  28. 		self.pool_stride = 2											 # 池化层窗口移动步长
  29. 		self.pool_padding = 0											 # 池化层对输入padding数量
  30. 		self.conv = Conv(												 # 卷积层: 3×32×32-->3×(32-7+1)×(32-7+1)-->3×26×26
  31. 			height=self.conv_kernel,
  32. 			width=self.conv_kernel,
  33. 			in_channels=self.image_size[0],
  34. 			out_channels=self.channels,
  35. 			stride=self.conv_stride,
  36. 			padding=self.conv_padding,
  37. 			init_scale=1e-2,
  38. 		)
  39. 		""" 经过卷积层后轨道与维度的变化 """
  40. 		channel = self.channels
  41. 		height += (2*self.conv_padding-self.conv_kernel)
  42. 		height /= self.conv_stride
  43. 		height = int(height)+1
  44. 		width += (2*self.conv_padding-self.conv_kernel)
  45. 		width /= self.conv_stride
  46. 		width = int(width)+1										
  47. 		self.relu1 = Relu()												 # 激活层 A: 3×26×26-->3×26×26
  48. 		self.pool = MaxPool(											 # 池化层: 3×26×26-->3×13×13
  49. 			kernel_size=self.pool_kernel,
  50. 			stride=self.pool_stride,
  51. 			padding=self.pool_padding,
  52. 		)
  53. 		""" 经过池化层后轨道与维度的变化 """
  54. 		channel = channel
  55. 		height += (2*self.pool_padding-self.pool_kernel)
  56. 		height /= self.pool_stride
  57. 		height = int(height)+1
  58. 		width += (2*self.pool_padding-self.pool_kernel)
  59. 		width /= self.pool_stride
  60. 		width = int(width)+1
  61. 		self.linear1 = Linear(											 # 线性层 A: 3×13×13-->507-->100
  62. 			in_features=channel*height*width,
  63. 			out_features=self.hidden_units,
  64. 			init_scale=1e-2,
  65. 		)											
  66. 		self.relu2 = Relu()												 # 激活层 B: 100-->100 							
  67. 		self.linear2 = Linear(											 # 线性层 B: 100-->10
  68. 			in_features=self.hidden_units,
  69. 			out_features=10,											 # 最后一层应该是输出对每个字段的预测概率
  70. 			init_scale=1e-2,
  71. 		)
  72. 		""" 类初始化 """
  73. 		self.softmaxce = SoftmaxCE()
  74. 		self.param_groups = [											 # 卷积层与线性层有参数
  75. 			{
  76. 				"w": {
  77. 					"param": self.conv.params["w"]["param"],
  78. 					"grad": self.conv.params["w"]["grad"],
  79. 					"pregrad": np.zeros_like(self.conv.params["w"]["grad"])
  80. 				},
  81. 				"b": {
  82. 					"param": self.conv.params["b"]["param"],
  83. 					"grad": self.conv.params["b"]["grad"],
  84. 					"pregrad": np.zeros_like(self.conv.params["b"]["grad"])
  85. 				},
  86. 			},
  87. 			{
  88. 				"w": {
  89. 					"param": self.linear1.params["w"]["param"],
  90. 					"grad": self.linear1.params["w"]["grad"],
  91. 					"pregrad": np.zeros_like(self.linear1.params["w"]["grad"])
  92. 				},
  93. 				"b": {
  94. 					"param": self.linear1.params["b"]["param"],
  95. 					"grad": self.linear1.params["b"]["grad"],
  96. 					"pregrad": np.zeros_like(self.linear1.params["b"]["grad"])
  97. 				},
  98. 			},
  99. 			{
  100. 				"w": {
  101. 					"param": self.linear2.params["w"]["param"],
  102. 					"grad": self.linear2.params["w"]["grad"],
  103. 					"pregrad": np.zeros_like(self.linear2.params["w"]["grad"])
  104. 				},
  105. 				"b": {
  106. 					"param": self.linear2.params["b"]["param"],
  107. 					"grad": self.linear2.params["b"]["grad"],
  108. 					"pregrad": np.zeros_like(self.linear2.params["b"]["grad"])
  109. 				},
  110. 			},
  111. 		]
  112. 		
  113. 	def oracle(self,x,y):												 # 计算损失函数值,输出得分,损失函数梯度: x为一个N_samples×N_channels×Height×Width的张量,y为类别标签
  114. 		""" 前向传播 """
  115. 		""" 卷积层 """
  116. 		conv_out = self.conv.forward(x)
  117.  
  118. 		""" 激活层 """
  119. 		relu1_out = self.relu1.forward(conv_out)
  120.  
  121. 		""" 池化层 """
  122. 		pool_out = self.pool.forward(relu1_out)
  123.  
  124. 		""" 线性层 """
  125. 		linear1_out = self.linear1.forward(pool_out)
  126.  
  127. 		""" 激活层 """
  128. 		relu2_out = self.relu2.forward(linear1_out)
  129.  
  130. 		""" 线性层 """
  131. 		linear2_out = self.linear2.forward(relu2_out)
  132.  
  133. 		""" 软大交叉熵 """
  134. 		fx,g,s = self.softmaxce(linear2_out,y)							 # 损失函数值&梯度(是算在最后一层上面的梯度)&得分
  135. 		""" 反向传播 """
  136. 		linear2_back = self.linear2.backward(g,relu2_out)
  137. 		self.update_param()
  138. 		relu2_back = self.relu2.backward(linear2_back,linear1_out)
  139. 		self.update_param()
  140. 		linear1_back = self.linear1.backward(relu2_back,pool_out)
  141. 		self.update_param()
  142. 		pool_back = self.pool.backward(linear1_back,relu1_out)
  143. 		self.update_param()
  144. 		relu1_back = self.relu1.backward(pool_back,conv_out)
  145. 		self.update_param()
  146. 		conv_back = self.conv.backward(relu1_back,x)
  147. 		self.update_param()
  148. 		return fx,s
  149.  
  150. 	def oracle_time(self,x,y):											 # 计算损失函数值,输出得分,损失函数梯度: x为一个N_samples×N_channels×Height×Width的张量,y为类别标签
  151. 		""" 前向传播 """
  152. 		""" 卷积层 """
  153. 		t = time.time()
  154. 		conv_out = self.conv.forward(x)
  155. 		conv_out_time = time.time()-t
  156.  
  157. 		""" 激活层 """
  158. 		t = time.time()
  159. 		relu1_out = self.relu1.forward(conv_out)
  160. 		relu1_out_time = time.time()-t
  161.  
  162. 		""" 池化层 """
  163. 		t = time.time()
  164. 		pool_out = self.pool.forward(relu1_out)
  165. 		pool_out_time = time.time()-t
  166.  
  167. 		""" 线性层 """
  168. 		t = time.time()
  169. 		linear1_out = self.linear1.forward(pool_out)
  170. 		linear1_out_time = time.time()-t
  171.  
  172. 		""" 激活层 """
  173. 		t = time.time()
  174. 		relu2_out = self.relu2.forward(linear1_out)
  175. 		relu2_out_time = time.time()-t
  176.  
  177. 		""" 线性层 """
  178. 		t = time.time()
  179. 		linear2_out = self.linear2.forward(relu2_out)
  180. 		linear2_out_time = time.time()-t
  181.  
  182. 		""" 软大交叉熵 """
  183. 		t = time.time()
  184. 		fx,g,s = self.softmaxce(linear2_out,y)							 # 损失函数值&梯度(是算在最后一层上面的梯度)&得分
  185. 		sf_out_time = time.time()-t
  186. 		""" 反向传播 """
  187. 		t = time.time()
  188. 		linear2_back = self.linear2.backward(g,relu2_out)
  189. 		self.update_param()
  190. 		linear2_back_time = time.time()-t
  191. 		t = time.time()
  192. 		relu2_back = self.relu2.backward(linear2_back,linear1_out)
  193. 		self.update_param()
  194. 		relu2_back_time = time.time()-t
  195. 		t = time.time()
  196. 		linear1_back = self.linear1.backward(relu2_back,pool_out)
  197. 		self.update_param()
  198. 		linear1_back_time = time.time()-t
  199. 		t = time.time()
  200. 		pool_back = self.pool.backward(linear1_back,relu1_out)
  201. 		self.update_param()
  202. 		pool_back_time = time.time()-t
  203. 		t = time.time()
  204. 		relu1_back = self.relu1.backward(pool_back,conv_out)
  205. 		self.update_param()
  206. 		relu1_back_time = time.time()-t
  207. 		t = time.time()
  208. 		conv_back = self.conv.backward(relu1_back,x)
  209. 		self.update_param()
  210. 		conv_back_time = time.time()-t
  211.  
  212. 		timedict = dict(
  213. 			conv_out_time=conv_out_time,
  214. 			relu1_out_time=relu1_out_time,
  215. 			pool_out_time=pool_out_time,
  216. 			linear1_out_time=linear1_out_time,
  217. 			relu2_out_time=relu2_out_time,
  218. 			linear2_out_time=linear2_out_time,
  219. 			conv_back_time=conv_back_time,
  220. 			relu1_back_time=relu1_back_time,
  221. 			pool_back_time=pool_back_time,
  222. 			linear1_back_time=linear1_back_time,
  223. 			relu2_back_time=relu2_back_time,
  224. 			linear2_back_time=linear2_back_time,
  225. 		)
  226. 		return fx,s,timedict
  227.  
  228. 	def score(self,x):													 # 预测的得分,除了oracle函数外还需要一个另外的得分函数,这在检查精度时是有用的: x为输入特征
  229. 		conv_out = self.conv(x)
  230. 		relu1_out = self.relu1(conv_out)
  231. 		pool_out = self.pool(relu1_out)
  232. 		linear1_out = self.linear1(pool_out)
  233. 		relu2_out = self.relu2(linear1_out)
  234. 		linear2_out = self.linear2(relu2_out)
  235. 		s = softmax(linear2_out)
  236. 		return s
  237.  
  238. 	def update_param(self,):											 # 更新参数及梯度
  239. 		self.param_groups[0]["w"]["param"] = self.conv.params["w"]["param"]
  240. 		self.param_groups[0]["w"]["grad"] = self.conv.params["w"]["grad"]
  241. 		self.param_groups[0]["b"]["param"] = self.conv.params["b"]["param"]
  242. 		self.param_groups[0]["b"]["grad"] = self.conv.params["b"]["grad"]
  243. 		self.param_groups[1]["w"]["param"] = self.linear1.params["w"]["param"]
  244. 		self.param_groups[1]["w"]["grad"] = self.linear1.params["w"]["grad"]
  245. 		self.param_groups[1]["b"]["param"] = self.linear1.params["b"]["param"]
  246. 		self.param_groups[1]["b"]["grad"] = self.linear1.params["b"]["grad"]
  247. 		self.param_groups[2]["w"]["param"] = self.linear2.params["w"]["param"]
  248. 		self.param_groups[2]["w"]["grad"] = self.linear2.params["w"]["grad"]
  249. 		self.param_groups[2]["b"]["param"] = self.linear2.params["b"]["param"]
  250. 		self.param_groups[2]["b"]["grad"] = self.linear2.params["b"]["grad"]
  251.  
  252. if __name__ == "__main__":
  253. 	cnn = CNN()

nn/layers.py

  1. #-*- coding:UTF-8 -*-
  2. import abc
  3. import numpy as np
  4. from numba import jit
  5.  
  6. class Layer(object):													 # 神经网络层的基类,该类包含抽象方法,因此不可以被实例化
  7. 	def __init__(self,):												 # 抽象类构造函数
  8. 		self.params = dict()											 # 参数字典: 其中键为参数名称的字符串,键为字典,键字典里面包含"param"与"grad"两个字段,分别记录参数值与梯度
  9.  
  10. 	@abc.abstractmethod
  11. 	def forward(self,x):												 # 评估输入特征与返回输出: x为输入特征, 返回值f(x)为输出特征
  12. 		pass
  13.  
  14. 	@abc.abstractmethod
  15. 	def backward(self,grad_in,x):										 # 计算梯度并将梯度反向传播, 为了未来可以参考, 更新过的梯度应该被储存在self.params的对应区域中: grad_in为从反向传播得到的梯度, x为输入特征, 返回值grad_x为反向传播到下一层的梯度(分别为w.r.t x)
  16. 		pass
  17.  
  18. 	def __call__(self,*args,**kwargs):									 # 使得Layer类型的变量可调用
  19. 		return self.forward(*args,**kwargs)
  20.  
  21.  
  22. class Conv(Layer):														 # 卷积层类, 参数w为卷积过滤器, 参数b为偏差
  23. 	def __init__(self,height,width,
  24. 		in_channels=3,
  25. 		out_channels=3,
  26. 		stride=1,
  27. 		padding=0,
  28. 		init_scale=1e-2,
  29. 	):																	 # 构造函数
  30. 		super(Conv,self).__init__()
  31. 		""" 类构造参数 """
  32. 		self.height = height											 # 卷积核的高度
  33. 		self.width = width												 # 卷积核的宽度
  34. 		self.in_channels = in_channels									 # 输入轨道数
  35. 		self.out_channels = out_channels								 # 输出轨道数
  36. 		self.stride = stride											 # 卷积核移动步长
  37. 		self.padding = padding											 # 是否需要在周围补充0
  38. 		self.init_scale = init_scale									 # 初始规模
  39. 		""" 父类参数: 我不在类初始化时设定grad参数, 因为如果重复调用同一对象的backward方法可能会导致梯度重复更新而错误 """
  40. 		self.params["w"] = {											 # 我理解的情况是这样的: 每个卷积过滤器的轨道数应与输入特征的轨道数相同, 每个卷积过滤器可以生成一个输出层轨道, 输出轨道数应当与卷积过滤器的数量相一致
  41. 			"param": self.init_scale*np.random.random((self.out_channels,
  42. 							self.in_channels,self.height,self.width)),
  43. 			"grad": None,
  44. 		}
  45. 		self.params["b"] = {											 # 偏差值
  46. 			"param": self.init_scale*np.random.random((self.out_channels,)),
  47. 			"grad": None,
  48. 		}
  49.  
  50. 	@jit(nopython=False,parallel=True)
  51. 	def forward(self,x):												 # 前向传播: x为一只四维张量, N_samples×n_Channels×Height×Width, 返回值out为卷积核的输出
  52. 		nSamples,nChannels,height,width = x.shape						 # 获取输入张量x的四个维度值
  53. 		assert nChannels==self.in_channels								 # 断言
  54. 		outshape = (													 # 计算输出的形状
  55. 			nSamples,self.out_channels,									 # 样本数, 输出轨道数
  56. 			int((2*self.padding+height-self.height)/self.stride)+1,		 # 输出高度
  57. 			int((2*self.padding+width-self.width)/self.stride)+1		 # 输出宽度
  58. 		)
  59. 		out = np.zeros(outshape)										 # 初始化输出
  60. 		if self.padding:												 # 如果需要padding
  61. 			x_ = np.zeros((
  62. 				nSamples,nChannels,
  63. 				height+2*self.padding,
  64. 				width+2*self.padding
  65. 			))
  66. 			x_[:,:,self.padding:-self.padding,
  67. 				self.padding:-self.padding] = x
  68. 		else: x_ = x.copy()
  69. 		for i in range(outshape[0]):									 # 遍历样本
  70. 			for j in range(outshape[1]):								 # 遍历输出轨道
  71. 				for k in range(outshape[2]):							 # 遍历像素点
  72. 					for l in range(outshape[3]):						 # 虽然很蠢, 但是用迭代生成器也不是很方便操作感觉
  73. 						x1,y1 = k*self.stride,l*self.stride
  74. 						x2,y2 = x1+self.height,y1+self.width
  75. 						total = 0
  76. 						for m in range(nChannels):						 # 遍历输入轨道计算哈氏积并累和: 这里天坑在于check_gradient.py的正确结果是默认w参数第一个维度是输出轨道, 第二个维度是输入轨道得到的, 而参数表里是先输入轨道后输出轨道的
  77. 							t1 = x_[i,m,x1:x2,y1:y2]					 # 输入对应区域
  78. 							t2 = self.params["w"]["param"][j,m,:,:]		 # 卷积过滤器对应区域
  79. 							total += np.nansum(t1*t2)
  80. 						out[i,j,k,l] = total+self.params["b"]["param"][j]# 最后不要忘了加上偏差值
  81. 		return out
  82. 	
  83. 	@jit(nopython=False,parallel=True)
  84. 	def backward(self,grad_in,x):										 # 卷积层的反向传播: grad_in的维度与卷积层forward中输出的维度相同
  85. 		self.params["w"]["grad"] = np.zeros((
  86. 			self.out_channels,self.in_channels,self.height,self.width))	 # 选择在反向传播时再重定义grad参数
  87. 		self.params["b"]["grad"] = np.zeros((self.out_channels,))		 # 选择在反向传播时再重定义grad参数
  88. 		nSamples,nChannels,height,width = x.shape	
  89. 		outshape = (													 # 计算输出的形状: 也是grad_in的形状
  90. 			nSamples,self.out_channels,									 # 样本数, 输出轨道数
  91. 			int((2*self.padding+height-self.height)/self.stride)+1,		 # 输出高度
  92. 			int((2*self.padding+width-self.width)/self.stride)+1		 # 输出宽度
  93. 		)
  94. 		assert outshape==grad_in.shape									 # 断言
  95. 		x_ = np.zeros(( 												 # 复现padding后的输入
  96. 			nSamples,nChannels,
  97. 			height+2*self.padding,
  98. 			width+2*self.padding
  99. 		))
  100. 		if self.padding:												 # 如果需要padding
  101. 			x_[:,:,self.padding:-self.padding,
  102. 				self.padding:-self.padding] = x
  103. 		else: x_ = x.copy()
  104. 		grad_x = np.zeros_like(x_)										 # 先设法对padding后的x求梯度, 然后只需取grad_x中间部分即可
  105. 		"""
  106. 			https://www.cnblogs.com/pinard/p/6494810.html
  107. 			上面链接中给出了步长为1且不padding情况下卷积层反向传播极其简洁的表达式;
  108. 			很可惜我只能用最愚蠢的方法来一个个填写梯度了;
  109. 		"""
  110. 		for i in range(outshape[0]):									 # 遍历样本
  111. 			for j in range(outshape[1]):								 # 遍历输出轨道
  112. 				self.params["b"]["grad"][j] += np.nansum(grad_in[i,j,:,:])
  113. 				for k in range(outshape[2]):							 # 遍历像素点
  114. 					for l in range(outshape[3]):						 # grad_in的维度必然为outshape, 通过遍历前向传播中outshape中每个位置上元素表达式来反向求导
  115. 						x1,y1 = k*self.stride,l*self.stride
  116. 						x2,y2 = x1+self.height,y1+self.width
  117. 						for m in range(nChannels):
  118. 							grad_x[i,m,x1:x2,y1:y2] += grad_in[i,j,
  119. 								k,l]*self.params["w"]["param"][j,m,:,:]
  120. 							self.params["w"]["grad"][j,m,:,:] += grad_in[
  121. 								i,j,k,l]*x_[i,m,x1:x2,y1:y2]		
  122. 		grad_x = grad_x[:,:,self.padding:-self.padding,
  123. 			self.padding:-self.padding] if self.padding else grad_x		 # 取出grad_x中不是padding的部分作为将要传递下去的梯度
  124. 		return grad_x
  125.  
  126. class Linear(Layer):													 # 线性层类, 用于对特征应用线性变换: w为n_in×n_out的矩阵, b为1×n_out的向量
  127. 	def __init__(self,in_features,out_features,
  128. 		init_scale=1e-2
  129. 	):																	 # 构造函数
  130. 		super(Linear,self).__init__()
  131. 		""" 类构造参数 """
  132. 		self.in_features = in_features									 # 输入特征数量
  133. 		self.out_features = out_features								 # 输出特征数量
  134. 		self.init_scale = 1e-2											 # 初始规模
  135. 		""" 父类参数 """
  136. 		self.params["w"] = {											 # 线性变换矩阵
  137. 			"param": self.init_scale*np.random.random((
  138. 						self.in_features,self.out_features)),
  139. 			"grad": None,												 # 不多说, 选择放在backward方法中初始化
  140. 		}
  141. 		self.params["b"] = {											 # 常数项偏差
  142. 			"param": self.init_scale*np.random.random((1,self.out_features)),
  143. 			"grad": None,
  144. 		}
  145. 		
  146. 	def forward(self,x):												 # 前向传播: x为维度为[n,d1,d2,...,dm]的输入特征, 返回值out为输出特征
  147. 		w = self.params["w"]["param"]						
  148. 		b = self.params["b"]["param"]
  149. 		x_ = x.reshape(x.shape[0],-1)									 # 全连接层我们需要以把每个输入样本压成一条向量处理
  150. 		out = np.dot(x_,w)+b											 # 这里两个维度分别为n_Sample×n_out与n_out×1, 但是它们还是可以相加的, 结果为每个sample加上b
  151. 		return out
  152.  
  153. 	def backward(self,grad_in,x):										 # 线性层的反向传播: 这里的grad_in维度与forward中out维度相同
  154. 		"""
  155. 			out = np.dot(x,w) + b;
  156. 			x.shape = (n_Sample,in_features);
  157. 			w.shape = (in_features,out_features);
  158. 			out.shape = grad_in.shape = (n_Sample,out_features);
  159. 			b.shape = (1,out_features)
  160. 			b的梯度应该为(n_Sample,out_features)
  161. 		"""
  162. 		x_ = x.reshape(x.shape[0],-1)
  163. 		self.params["w"]["grad"] = np.dot(x_.T,grad_in)					 # 凑维度凑出来的
  164. 		self.params["b"]["grad"] = np.nansum(grad_in,axis=0)			 # 每个样本都被b搞了一下
  165. 		grad_x = np.dot(grad_in,self.params["w"]["param"].T)			 # 凑维度凑出来的
  166. 		return grad_x
  167. 		
  168. class Relu(Layer):														 # 激活层类
  169. 	def __init__(self):													 # 构造函数
  170. 		super(Relu,self).__init__()
  171.  
  172. 	def forward(self,x):												 # 前向传播
  173. 		return np.maximum(x,0)											 # 正数返回, 负数变零
  174.  
  175. 	def backward(self,grad_in,x):										 # 反向传播
  176. 		return grad_in*(x>0)											 # 返回输入的梯度乘上relu的梯度
  177.  
  178. class MaxPool(Layer):													 # 池化层类
  179. 	def __init__(self,kernel_size,
  180. 		stride=2,
  181. 		padding=0
  182. 	):																	 # 构造函数
  183. 		super(MaxPool,self).__init__()
  184. 		""" 类构造参数 """
  185. 		self.kernel_size = kernel_size
  186. 		self.stride = stride
  187. 		self.padding = padding
  188. 		""" 父类参数 """
  189. 		self.params = dict()											 # 池化层应该是没有参数
  190.  
  191. 	def forward(self,x):												 # 前向传播: x为一只四维张量, N_samples×n_Channels×Height×Width, 返回值out为池化层的输出
  192. 		nSamples,nChannels,height,width = x.shape						 # 获取输入张量x的四个维度值
  193. 		outshape = (													 # 计算输出的形状
  194. 			nSamples,nChannels,											 # 样本数, 输出轨道数
  195. 			int((2*self.padding+height-self.kernel_size)/self.stride)+1, # 输出高度
  196. 			int((2*self.padding+width-self.kernel_size)/self.stride)+1	 # 输出宽度
  197. 		)
  198. 		out = np.zeros(outshape)										 # 初始化输出
  199. 		if self.padding:												 # 如果需要padding
  200. 			x_ = np.zeros((
  201. 				nSamples,nChannels,
  202. 				height+2*self.padding,
  203. 				width+2*self.padding
  204. 			))
  205. 			x_[:,:,self.padding:-self.padding,
  206. 				self.padding:-self.padding] = x
  207. 		else: x_ = x.copy()
  208. 		for i in range(outshape[0]):									 # 遍历样本
  209. 			for j in range(outshape[1]):								 # 遍历输出轨道
  210. 				for k in range(outshape[2]):							 # 遍历像素点
  211. 					for l in range(outshape[3]):						 # 虽然很蠢, 但是用迭代生成器也不是很方便操作感觉
  212. 						x1,y1 = k*self.stride,l*self.stride
  213. 						x2,y2 = x1+self.kernel_size,y1+self.kernel_size
  214. 						out[i,j,k,l] = np.nanmax(x_[i,j,x1:x2,y1:y2])	 # 取窗口最大值
  215. 		return out
  216.  
  217. 	def backward(self,grad_in,x):										 # 反向传播: 此事grad_in的维度恰与MaxPool的输出相同, 因此需要找出grad_in中每个元素对应了x中哪块区域
  218. 		"""
  219. 			最大值池化层平均值池化层都没有需要学习的参数;
  220. 			只需将误差项传递到上一层, 没有梯度的计算;
  221. 			最大值池化层：下一层的误差项的值会原封不动的传递到上一层对应区块中的最大值所对应的神经元, 而其他神经元的误差项的值都是0;
  222. 			平均值池化层: 下一层的误差项的值会平均分配到上一层对应区块中的所有神经元;
  223. 		"""
  224. 		nSamples,nChannels,height,width = x.shape						 # 获取输入的形状
  225. 		outshape = (													 # 计算输出的形状: 也是grad_in的形状
  226. 			nSamples,nChannels,											 # 样本数, 输出轨道数
  227. 			int((2*self.padding+height-self.kernel_size)/self.stride)+1, # 输出高度
  228. 			int((2*self.padding+width-self.kernel_size)/self.stride)+1	 # 输出宽度
  229. 		)
  230. 		grad_in_reshape = grad_in.reshape(outshape)
  231. 		x_ = np.zeros(( 												 # 复现padding后的输入
  232. 			nSamples,nChannels,
  233. 			height+2*self.padding,
  234. 			width+2*self.padding
  235. 		))
  236. 		if self.padding:												 # 如果需要padding
  237. 			x_[:,:,self.padding:-self.padding,
  238. 				self.padding:-self.padding] = x
  239. 		else: x_ = x.copy()
  240. 		grad_x = np.zeros_like(x_)										 # 先设法对padding后的x求梯度, 然后只需取grad_x中间部分即可
  241.  
  242. 		for i in range(outshape[0]):									 # 遍历样本
  243. 			for j in range(outshape[1]):								 # 遍历输出轨道
  244. 				for k in range(outshape[2]):							 # 遍历像素点
  245. 					for l in range(outshape[3]):						 # grad_in的维度必然为outshape, 通过遍历前向传播中outshape中每个位置上元素表达式来反向求导
  246. 						x1,y1 = k*self.stride,l*self.stride
  247. 						x2,y2 = x1+self.kernel_size,y1+self.kernel_size
  248. 						maxgrid = np.nanmax(x_[i,j,x1:x2,y1:y2])		 # 找出对应该grad_in格子的原区域中最大值
  249. 						grad_x[i,j,x1:x2,y1:y2] += grad_in_reshape[i,j,
  250. 							k,l]*(x_[i,j,x1:x2,y1:y2]==maxgrid)
  251.  
  252. 		grad_x = grad_x[:,:,self.padding:-self.padding,
  253. 			self.padding:-self.padding] if self.padding else grad_x		 # 取出grad_x中不是padding的部分作为将要传递下去的梯度
  254. 		return grad_x

nn/optimizer.py

  1. #-*- coding:UTF-8 -*-
  2. import abc
  3. import numpy as np
  4.  
  5. class Optimizer(object):												 # 抽象优化器类, 为神经网络优化的一次算法(SGD算一次,Newton算二次): params_groups是神经网络模型中所有的参数构成的列表, configs是优化超参数
  6. 	def __init__(self,param_groups):									 # 构造函数
  7. 		self.param_groups = param_groups								 # 参数组应该是一个列表, 一般包含w, b两个字典, 再下一层是param与grad两个字段
  8.  
  9. 	@abc.abstractmethod
  10. 	def step(self,isNestrov):											 # 步长问题
  11. 		pass
  12.  
  13. class SGD(Optimizer):													 # 随机梯度下降类
  14. 	def __init__(self,param_groups,
  15. 		lr=1e-2,
  16. 		weight_decay=0.0,
  17. 		momentum=0.0
  18. 	):																	 # 构造函数, 参数含义可以参考HW2
  19. 		super(SGD,self).__init__(param_groups)
  20. 		self.configs = dict(											 # 配置字典中包含学习率与权重衰减等超参数
  21. 			lr=lr,
  22. 			weight_decay=weight_decay,
  23. 			momentum=momentum,
  24. 		)			
  25.  
  26. 	def step(self):														 # 步长问题
  27. 		lr = self.configs["lr"]											 # 获取学习率
  28. 		weight_decay = self.configs["weight_decay"]						 # 获取权重衰减系数
  29. 		momentum = self.configs["momentum"]								 # 获取动量系数
  30. 		""" 实现传统动量: 我觉得可能Nesterov's方法实现起来比较麻烦, 徒增计算量 """
  31. 		count = 0
  32. 		for group in self.param_groups:									 # 遍历每个参数
  33. 			count += 1
  34. 			for k,p in group.items():									 # 获取每个参数的信息
  35. 				grad = p["grad"]										 # 获取参数梯度值
  36. 				pregrad = p["pregrad"]									 # 获取上一次参数梯度值
  37. 				if k=="w":												 # 如果这是一个矩阵参数那么需要考虑权重衰减系数
  38. 					v = momentum*pregrad - grad - weight_decay*p["param"]
  39. 					p["param"] += lr*v
  40. 				else:
  41. 					v = momentum*pregrad - grad
  42. 					p["param"] += lr*v									 # 对于偏差参数不需要考虑权重衰减
  43. 				p["pregrad"] = v										 # 记录本次下降方向

nn/loss.py

  1. #-*- coding:UTF-8 -*-
  2. import numpy as np
  3.  
  4. def softmax(x):															 # 给定一个矩阵, 计算每行减去该行最大值后的softmax输出
  5. 	x_bar = x-np.nanmax(x,axis=1,keepdims=True)							 # 减去该行最大值: 防止数值溢出
  6. 	z = np.nansum(np.exp(x_bar),axis=1,keepdims=True)					 # 计算自然底数幂和
  7. 	return np.exp(x_bar)/z												 # 返回softmax输出
  8.  
  9. class SoftmaxCE(object):												 # 利用softmax转换计算交叉熵损失值的抽象类
  10. 	def __init__(self):													 # 构造函数
  11. 		pass
  12.  
  13. 	@staticmethod
  14. 	def __call__(x,y):													 # 调用方法: x为n_samples×n_features的矩阵, y看起来像是某个样本的标签0~9
  15. 		sf = softmax(x)													 # 计算x的softmax输出
  16. 		n = x.shape[0]													 # 获取样本数
  17. 		sf_log = -np.log(sf[range(n),y])								 # 对应y的位置的概率即为预测准确的概率
  18. 		loss = np.mean(sf_log)											 # 用这部分概率来计算交叉熵值
  19. 		g = sf.copy()									
  20. 		g[range(n),y] -= 1												 # softmax交叉熵求导的形式极其简洁
  21. 		g /= n								
  22. 		return loss,g,sf												 # 返回交叉熵损失值, 梯度及softmax层的输出结果

nn/utils.py

  1. #-*- coding:UTF-8 -*-
  2. import numpy as np
  3.  
  4. def accuracy(score,y):													 # 计算分类精度, 返回一个0~1的标量: score是n×n类别得分矩阵, y为n×1的标签
  5. 	acc = np.mean(score.argmax(axis=1)==y)
  6. 	return acc

另外附一份用keras复现上面的架构，以及在该架构的基础上调参的测试代码：

  1. #-*- coding:UTF-8 -*-
  2. import time
  3. import numpy as np
  4. import tensorflow as tf
  5. from dataset import get_cifar10_data
  6. from matplotlib import pyplot as plt
  7.  
  8. from keras.optimizers import SGD
  9. from keras.utils import to_categorical
  10. from keras.models import Sequential,Model
  11. from keras.layers import Dense,Dropout,Activation,Flatten,Conv2D,MaxPooling2D,Embedding,LSTM,Input
  12.  
  13. def read_data(num_train=100):											 # 读取数据
  14. 	data = get_cifar10_data()					
  15. 	data = {															 # 把data转为字典形式
  16. 		"X_train": data["X_train"][:num_train].transpose(0,2,3,1),		 # 把轨道放到最后一个维度上
  17. 		"y_train": to_categorical(data["y_train"][:num_train],10),		 # 标签一热化
  18. 		"X_val": data["X_val"].transpose(0,2,3,1),						 # 把轨道放到最后一个维度上
  19. 		"y_val": to_categorical(data["y_val"],10),						 # 标签一热化
  20. 	}
  21. 	return data
  22.  
  23. def model_1():															 # conv+relu+maxpool+linear+relu+linear+softmax									
  24. 	model = Sequential()
  25. 	model.add(Conv2D(3,(7,7),activation="relu",input_shape=(32,32,3)))
  26. 	model.add(MaxPooling2D(pool_size=(2,2)))
  27. 	model.add(Flatten())
  28. 	model.add(Dense(100,activation="relu"))
  29. 	model.add(Dense(10,activation="softmax"))
  30. 	sgd = SGD(lr=0.01,decay=1e-6,momentum=0.9,nesterov=False)
  31. 	model.compile(loss="categorical_crossentropy",optimizer=sgd,metrics=["accuracy"])
  32. 	model.summary()
  33. 	return model
  34.  
  35. def model_2():
  36. 	model = Sequential()
  37. 	model.add(Conv2D(3,(7,7),activation="relu",input_shape=(32,32,3)))
  38. 	model.add(MaxPooling2D(pool_size=(2,2)))
  39. 	model.add(Conv2D(3,(3,3),activation="relu",input_shape=(13,13,3)))
  40. 	model.add(Flatten())
  41. 	model.add(Dense(100,activation="sigmoid"))
  42. 	model.add(Dense(10,activation="softmax"))
  43. 	sgd = SGD(lr=0.01,decay=1e-6,momentum=0.9,nesterov=False)
  44. 	model.compile(loss="categorical_crossentropy",optimizer=sgd,metrics=["accuracy"])
  45. 	model.summary()
  46. 	return model
  47.  
  48. def model_3():
  49. 	model = Sequential()
  50. 	model.add(Conv2D(3,(7,7),activation="relu",input_shape=(32,32,3)))
  51. 	model.add(MaxPooling2D(pool_size=(2,2)))
  52. 	model.add(Flatten())
  53. 	model.add(Dense(256,activation="sigmoid"))
  54. 	model.add(Dense(128,activation="sigmoid"))
  55. 	model.add(Dense(10,activation="softmax"))
  56. 	sgd = SGD(lr=0.01,decay=1e-6,momentum=0.9,nesterov=False)
  57. 	model.compile(loss="categorical_crossentropy",optimizer=sgd,metrics=["accuracy"])
  58. 	model.summary()
  59. 	return model
  60.  
  61. def model_4():
  62. 	model = Sequential()
  63. 	model.add(Conv2D(3,(7,7),activation="relu",input_shape=(32,32,3)))
  64. 	model.add(MaxPooling2D(pool_size=(2,2)))
  65. 	model.add(Conv2D(3,(3,3),activation="relu",input_shape=(13,13,3)))
  66. 	model.add(Flatten())
  67. 	model.add(Dense(256,activation="sigmoid"))
  68. 	model.add(Dense(128,activation="sigmoid"))
  69. 	model.add(Dense(10,activation="softmax"))
  70. 	sgd = SGD(lr=0.01,decay=1e-6,momentum=0.9,nesterov=False)
  71. 	model.compile(loss="categorical_crossentropy",optimizer=sgd,metrics=["accuracy"])
  72. 	model.summary()
  73. 	return model
  74.  
  75. def model_adagrad():
  76. 	model = Sequential()
  77. 	model.add(Conv2D(3,(7,7),activation="relu",input_shape=(32,32,3)))
  78. 	model.add(MaxPooling2D(pool_size=(2,2)))
  79. 	model.add(Flatten())
  80. 	model.add(Dense(256,activation="sigmoid"))
  81. 	model.add(Dense(128,activation="sigmoid"))
  82. 	model.add(Dense(10,activation="softmax"))
  83. 	model.compile(loss="categorical_crossentropy",optimizer="adagrad",metrics=["accuracy"])
  84. 	model.summary()
  85. 	return model
  86.  
  87. def model_adam():
  88. 	model = Sequential()
  89. 	model.add(Conv2D(3,(7,7),activation="relu",input_shape=(32,32,3)))
  90. 	model.add(MaxPooling2D(pool_size=(2,2)))
  91. 	model.add(Flatten())
  92. 	model.add(Dense(256,activation="sigmoid"))
  93. 	model.add(Dense(128,activation="sigmoid"))
  94. 	model.add(Dense(10,activation="softmax"))
  95. 	model.compile(loss="categorical_crossentropy",optimizer="adam",metrics=["accuracy"])
  96. 	model.summary()
  97. 	return model
  98.  
  99. if __name__ == "__main__":
  100. 	""" P3Q1 """
  101. 	model = model_1()
  102. 	#model = model_2()
  103. 	#model = model_3()
  104. 	#model = model_4()
  105. 	data = read_data(49000)
  106. 	t = time.time()
  107. 	model.fit(data["X_train"],data["y_train"],batch_size=50,epochs=10)
  108. 	print("训练耗时：{}".format(time.time()-t))
  109. 	score = model.evaluate(data["X_val"],data["y_val"],batch_size=32)
  110. 	print(score)
  111.  
  112. 	""" P3Q2 """
  113. 	#model = model_adagrad()
  114. 	model = model_adam()
  115. 	data = read_data(49000)
  116. 	t = time.time()
  117. 	model.fit(data["X_train"],data["y_train"],batch_size=50,epochs=10)
  118. 	print("训练耗时：{}".format(time.time()-t))
  119. 	score = model.evaluate(data["X_val"],data["y_val"],batch_size=32)
  120. 	print(score)

分享学习，共同进步！

 

 

声明：本文内容由网友自发贡献，不代表【wpsshop博客】立场，版权归原作者所有，本站不承担相应法律责任。如您发现有侵权的内容，请联系我们。转载请注明出处：https://www.wpsshop.cn/w/2023面试高手/article/detail/75043
推荐阅读
相关标签

Copyright © 2003-2013 www.wpsshop.cn 版权所有，并保留所有权利。

  

闽ICP备14008679号