深度学习入门2：自制框架DeZero_深度学习入门2:自制框架 pdf

待更新…

第一阶段 自动微分

步骤1 作为箱子的变量

步骤3 函数的连续调用

步骤4 数值微分

步骤6 手动进行反向传播

import numpy as np


class Varialbe:
    def __init__(self, data):
        self.data = data
        self.grad = None

class Fuction:
    def __call__(self, input):
        x = input.data
        y = self.forward(x)
        output = Varialbe(y)
        self.input = input
        return output
    
    def forward(self, x):
        raise NotImplementedError()
    
    def backward(self, gy):
        raise NotImplementedError()

class Square(Fuction):
    def forward(self, x):
        y = x ** 2
        return y
    def backward(self, gy):
        x = self.input.data
        gx = 2 * x * gy
        return gx

class Exp(Fuction):
    def forward(self, x):
        y = np.exp(x)
        return y
    
    def backward(self, gy):
        x = self.input.data
        gx = np.exp(x) * gy
        return gx

def numerical_deff(f, x, eps=1e-4):
    x0 = Varialbe(x.data - eps)
    x1 = Varialbe(x.data + eps)
    y0 = f(x0)
    y1 = f(x1)
    return (y1.data - y0.data) / (2 * eps)


A = Square()
B = Exp()
C = Square()

x = Varialbe(np.array(0.5))
a = A(x)
b = B(a)
c = C(b)

y.grad = np.array(1.0)
b.grad = C.backward(y.grad)
a.grad = B.backward(b.grad)
x.garad = A.backward(a.grad)
print(x.grad)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63

步骤7 反向传播的自动化

import numpy as np


class Varialbe:
    def __init__(self, data):
        self.data = data
        self.grad = None
        self.creator = None
    
    def set_creator(self, func):
        self.creator = func
        
class Fuction:
    def __call__(self, input):
        x = input.data
        y = self.forward(x)
        output = Varialbe(y)
        output.set_creator(self)
        self.input = input
        self.output = output
        return output
    
    def forward(self, x):
        raise NotImplementedError()
    
    def backward(self, gy):
        raise NotImplementedError()

class Square(Fuction):
    def forward(self, x):
        y = x ** 2
        return y
    def backward(self, gy):
        x = self.input.data
        gx = 2 * x * gy
        return gx

class Exp(Fuction):
    def forward(self, x):
        y = np.exp(x)
        return y
    
    def backward(self, gy):
        x = self.input.data
        gx = np.exp(x) * gy
        return gx

def numerical_deff(f, x, eps=1e-4):
    x0 = Varialbe(x.data - eps)
    x1 = Varialbe(x.data + eps)
    y0 = f(x0)
    y1 = f(x1)
    return (y1.data - y0.data) / (2 * eps)


A = Square()
B = Exp()
C = Square()

x = Varialbe(np.array(0.5))
a = A(x)
b = B(a)
y = C(b)

# assert y.creator == C
# assert y.creator.input == b 
# assert y.creator.input.creator == B
# assert y.creator.input.creator.input == a
# assert y.creator.input.creator.input.creator == A
# assert y.creator.input.creator.input.creator.input == x

y.grad =  np.array(1.0)
C = y.creator
b = C.input
b.grad = C.backward(y.grad)

B = b.creator
a = B.input
a.grad = B.backward(b.grad)

A = a.creator
x = A.input
x.grad = A.backward(a.grad)
print(x.grad)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85

import numpy as np

# 调用一个Function时，Function会记录左右的input和output，同时output会记录生成它的Func
class Varialbe:
    def __init__(self, data):
        self.data = data
        self.grad = None
        self.creator = None
    
    def set_creator(self, func):
        self.creator = func
    
    def backward(self):
        f = self.creator
        if f is not None:
            x = f.input
            x.grad = f.backward(self.grad)
            x.backward() 
    
        
class Fuction:
    def __call__(self, input):
        x = input.data
        y = self.forward(x)
        output = Varialbe(y)
        output.set_creator(self)
        self.input = input
        self.output = output
        return output
    
    def forward(self, x):
        raise NotImplementedError()
    
    def backward(self, gy):
        raise NotImplementedError()

class Square(Fuction):
    def forward(self, x):
        y = x ** 2
        return y
    def backward(self, gy):
        x = self.input.data
        gx = 2 * x * gy
        return gx

class Exp(Fuction):
    def forward(self, x):
        y = np.exp(x)
        return y
    
    def backward(self, gy):
        x = self.input.data
        gx = np.exp(x) * gy
        return gx

def numerical_deff(f, x, eps=1e-4):
    x0 = Varialbe(x.data - eps)
    x1 = Varialbe(x.data + eps)
    y0 = f(x0)
    y1 = f(x1)
    return (y1.data - y0.data) / (2 * eps)


A = Square()
B = Exp()
C = Square()

x = Varialbe(np.array(0.5))
a = A(x)
b = B(a)
y = C(b)

# assert y.creator == C
# assert y.creator.input == b 
# assert y.creator.input.creator == B
# assert y.creator.input.creator.input == a
# assert y.creator.input.creator.input.creator == A
# assert y.creator.input.creator.input.creator.input == x

y.grad =  np.array(1.0)
y.backward()
print(x.grad)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82

import numpy as np

# 调用一个Function时，Function会记录左右的input和output，同时output会记录生成它的Func
class Varialbe:
    def __init__(self, data):
        self.data = data
        self.grad = None
        self.creator = None
    
    def set_creator(self, func):
        self.creator = func
    
    # def backward(self):
    #     f = self.creator
    #     if f is not None:
    #         x = f.input
    #         x.grad = f.backward(self.grad)
    #         x.backward() 
    
    def backward(self):
        funcs = [self.creator]
        while funcs:
            f = funcs.pop()
            x, y = f.input, f.output
            x.grad = f.backward(y.grad)
            
            if x.creator  is not None:
                funcs.append(x.creator)
    
        
class Fuction:
    def __call__(self, input):
        x = input.data
        y = self.forward(x)
        output = Varialbe(y)
        output.set_creator(self)
        self.input = input
        self.output = output
        return output
    
    def forward(self, x):
        raise NotImplementedError()
    
    def backward(self, gy):
        raise NotImplementedError()

class Square(Fuction):
    def forward(self, x):
        y = x ** 2
        return y
    def backward(self, gy):
        x = self.input.data
        gx = 2 * x * gy
        return gx

class Exp(Fuction):
    def forward(self, x):
        y = np.exp(x)
        return y
    
    def backward(self, gy):
        x = self.input.data
        gx = np.exp(x) * gy
        return gx

def numerical_deff(f, x, eps=1e-4):
    x0 = Varialbe(x.data - eps)
    x1 = Varialbe(x.data + eps)
    y0 = f(x0)
    y1 = f(x1)
    return (y1.data - y0.data) / (2 * eps)


A = Square()
B = Exp()
C = Square()

x = Varialbe(np.array(0.5))
a = A(x)
b = B(a)
y = C(b)

# assert y.creator == C
# assert y.creator.input == b 
# assert y.creator.input.creator == B
# assert y.creator.input.creator.input == a
# assert y.creator.input.creator.input.creator == A
# assert y.creator.input.creator.input.creator.input == x

# y.grad =  np.array(1.0)
# C1 = y.creator
# b = C1.input
# b.grad = C1.backward(y.grad)

# B1 = b.creator
# a = B1.input
# a.grad = B1.backward(b.grad)

# A1 = a.creator
# x = A1.input
# x.grad = A1.backward(a.grad)
# print(x.grad)

y.grad = np.array(1.0)
y.backward()
print(x.grad)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106

import numpy as np

# 调用一个Function时，Function会记录左右的input和output，同时output会记录生成它的Func
class Varialbe:
    def __init__(self, data):
        self.data = data
        self.grad = None
        self.creator = None
    
    def set_creator(self, func):
        self.creator = func
    
    # def backward(self):
    #     f = self.creator
    #     if f is not None:
    #         x = f.input
    #         x.grad = f.backward(self.grad)
    #         x.backward() 
    
    def backward(self):
        if self.grad is None:
            self.grad = np.ones_like(self.data)
            
        funcs = [self.creator]
        while funcs:
            f = funcs.pop()
            x, y = f.input, f.output
            x.grad = f.backward(y.grad)
            
            if x.creator  is not None:
                funcs.append(x.creator)
    
        
class Fuction:
    def __call__(self, input):
        x = input.data
        y = self.forward(x)
        output = Varialbe(y)
        output.set_creator(self)
        self.input = input
        self.output = output
        return output
    
    def forward(self, x):
        raise NotImplementedError()
    
    def backward(self, gy):
        raise NotImplementedError()

class Square(Fuction):
    def forward(self, x):
        y = x ** 2
        return y
    def backward(self, gy):
        x = self.input.data
        gx = 2 * x * gy
        return gx

class Exp(Fuction):
    def forward(self, x):
        y = np.exp(x)
        return y
    
    def backward(self, gy):
        x = self.input.data
        gx = np.exp(x) * gy
        return gx

def numerical_deff(f, x, eps=1e-4):
    x0 = Varialbe(x.data - eps)
    x1 = Varialbe(x.data + eps)
    y0 = f(x0)
    y1 = f(x1)
    return (y1.data - y0.data) / (2 * eps)


def square(x):
    f = Square()
    return f(x)

def exp(x):
    f =  Exp()
    return f(x)



x = Varialbe(np.array(0.5))
y = square(exp(square(x)))
y.backward()
print(x.grad)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90

import numpy as np
import unittest

class SquareTest(unittest.TestCase):
    def test_forward(self):
        x = Varialbe(np.array(2.0))
        y = square(x)
        expected = np.array(4.0)
        self.assertEqual(y.data, expected)
    
    def test_backward(self):
        x = Varialbe(np.array(3.))
        y = square(x)
        y.backward()
        expected = np.array(6.0)
        self.assertEqual(x.grad, expected)
    
    def test_gradient_check(self):
        x = Varialbe(np.random.rand(1))
        y = square(x)
        y.backward()
        num_grad = numerical_deff(square, x)
        flg = np.allclose(x.grad, num_grad)
        self.assertTrue(flg)
        
        
# 调用一个Function时，Function会记录左右的input和output，同时output会记录生成它的Func
class Varialbe:
    def __init__(self, data):
        if data is not None:
            if not isinstance(data, np.ndarray):
                raise TypeError('{} is not supported'.format(type(data)))
        self.data = data
        self.grad = None
        self.creator = None
    
    def set_creator(self, func):
        self.creator = func
    
    # def backward(self):
    #     f = self.creator
    #     if f is not None:
    #         x = f.input
    #         x.grad = f.backward(self.grad)
    #         x.backward() 
    
    def backward(self):
        if self.grad is None:
            self.grad = np.ones_like(self.data)
            
        funcs = [self.creator]
        while funcs:
            f = funcs.pop()
            x, y = f.input, f.output
            x.grad = f.backward(y.grad)
            
            if x.creator  is not None:
                funcs.append(x.creator)
    
        
class Fuction:
    def __call__(self, input):
        x = input.data
        y = self.forward(x)
        output = Varialbe(as_array(y))
        output.set_creator(self)
        self.input = input
        self.output = output
        return output
    
    def forward(self, x):
        raise NotImplementedError()
    
    def backward(self, gy):
        raise NotImplementedError()

class Square(Fuction):
    def forward(self, x):
        y = x ** 2
        return y
    def backward(self, gy):
        x = self.input.data
        gx = 2 * x * gy
        return gx

class Exp(Fuction):
    def forward(self, x):
        y = np.exp(x)
        return y
    
    def backward(self, gy):
        x = self.input.data
        gx = np.exp(x) * gy
        return gx

def numerical_deff(f, x, eps=1e-4):
    x0 = Varialbe(x.data - eps)
    x1 = Varialbe(x.data + eps)
    y0 = f(x0)
    y1 = f(x1)
    return (y1.data - y0.data) / (2 * eps)


def square(x):
    f = Square()
    return f(x)

def exp(x):
    f =  Exp()
    return f(x)

def as_array(x):
    if np.isscalar(x):
        return np.array(x)
    return x

x = Varialbe(np.array(0.5))
y = square(exp(square(x)))
y.backward()
print(x.grad)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120

#步骤11 可变长参数

import numpy as np
import unittest

class SquareTest(unittest.TestCase):
    def test_forward(self):
        x = Varialbe(np.array(2.0))
        y = square(x)
        expected = np.array(4.0)
        self.assertEqual(y.data, expected)
    
    def test_backward(self):
        x = Varialbe(np.array(3.))
        y = square(x)
        y.backward()
        expected = np.array(6.0)
        self.assertEqual(x.grad, expected)
    
    def test_gradient_check(self):
        x = Varialbe(np.random.rand(1))
        y = square(x)
        y.backward()
        num_grad = numerical_deff(square, x)
        flg = np.allclose(x.grad, num_grad)
        self.assertTrue(flg)
        
        
# 调用一个Function时，Function会记录左右的input和output，同时output会记录生成它的Func
class Varialbe:
    def __init__(self, data):
        if data is not None:
            if not isinstance(data, np.ndarray):
                raise TypeError('{} is not supported'.format(type(data)))
        self.data = data
        self.grad = None
        self.creator = None
        self.generation = 0
    
    def set_creator(self, func):
        self.creator = func
        self.generation = func.generation + 1
    # def backward(self):
    #     f = self.creator
    #     if f is not None:
    #         x = f.input
    #         x.grad = f.backward(self.grad)
    #         x.backward() 
    
    def backward(self):
        if self.grad is None:
            self.grad = np.ones_like(self.data)
            
        funcs = []
        seen_set = set()
        
        # 反向传播的时候，按照generation
        def add_func(f):
            if f not in seen_set:
                funcs.append(f)
                seen_set.add(f)
                funcs.sort(key=lambda x: x.generation)
        add_func(self.creator)
            
        funcs = [self.creator]
        while funcs:
            f = funcs.pop()
            gys = [output.grad for output in f.outputs]
            gxs = f.backward(*gys)
            if not isinstance(gxs, tuple):
                gxs = (gxs, )
            for x, gx in zip(f.inputs, gxs):
                if x.grad is None: #为了节省内存，同一个Variable求完梯度后grad并不会清空，所以当新的公式也需要对该变量求导后，会导致grad累加，结果错误
                    x.grad = gx
                else:   
                    # 为了解决相同变量作为输入的问题x+x = y
                    x.grad = x.grad + gx
                
                if x.creator is not None:
                    # funcs.append(x.creator)
                    add_func(x.creator)
    def cleargrad(self):
        self.grad = None
        
class Fuction:
    def __call__(self, *inputs):

        xs = [x.data for x in inputs]
        ys = self.forward(*xs)
        if not isinstance(ys, tuple): 
            ys = (ys, )
        outputs = [Varialbe(as_array(y)) for y in ys]
       
        self.generation = max([x.generation for x in inputs])
        for output in outputs:
            output.set_creator(self)

        self.inputs = inputs
        self.outputs = outputs
        # print(type(self.inputs))
        # print(type(self.outputs))
        # print(type(ys))
        return  outputs if len(outputs) > 1 else outputs[0]
    
    def forward(self, x):
        raise NotImplementedError()
    
    def backward(self, gy):
        raise NotImplementedError()

class Square(Fuction):
    def forward(self, x):
        print(type(x))
        y = x ** 2
        return y
    def backward(self, gy):
        x = self.inputs[0].data # inputs是tuple类型变量，对于输入只有一个元素的tuple（x,)
        gx = 2 * x * gy
        return gx

class Exp(Fuction):
    def forward(self, x):
        y = np.exp(x)
        return y
    
    def backward(self, gy):
        x = self.input.data
        gx = np.exp(x) * gy
        return 
    
class Add(Fuction):
    def forward(self, x0, x1):
        y = x0 + x1
        return y
    def backward(self, gy):
        return gy, gy
    
def numerical_deff(f, x, eps=1e-4):
    x0 = Varialbe(x.data - eps)
    x1 = Varialbe(x.data + eps)
    y0 = f(x0)
    y1 = f(x1)
    return (y1.data - y0.data) / (2 * eps)


def square(x):
    f = Square()
    z = f(x)
    return z

def exp(x):
    f =  Exp()
    return f(x)

def as_array(x):
    if np.isscalar(x):
        return np.array(x)
    return x

def add(x0, x1):
    return Add()(x0, x1)


x = Varialbe(np.array(2.0))
a = square(x)
y = add(square(a), square(a))
y.backward()

print(y.data)
print(x.grad)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170