简单的PPO算法笔记_ppo算法流程图

---

学习了莫烦讲解的PPO，写了点自己的理解笔记，希望能帮到你们。

---

代码

代码可以去上面的链接自己下载跑一下，这边也给出我参考莫烦自己学的，基本是一样的：

import gym
import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt


EP_MAX = 1000
EP_LEN = 200
BATCH = 32
GAMMA = 0.9
C_LR = 0.0002
A_LR = 0.0001
A_UPDATE_STEPS = 10
C_UPDATE_STEPS = 10
METHOD = [
    dict(name='kl_pen', kl_target=0.01, lam=0.5),   # KL penalty
    dict(name='clip', epsilon=0.2),                 # Clipped surrogate objective, find this is better
][1]


class PPO:
    def __init__(self):
        self.sess = tf.Session()
        self.tfs = tf.placeholder(tf.float32, [None, S_DIM], 'state')
        self._build_anet('Critic')
        with tf.variable_scope('closs'):
            self.tfdc_r = tf.placeholder(tf.float32, [None, 1], name='discounted_r')
            self.adv = self.tfdc_r - self.v
            closs = tf.reduce_mean(tf.square(self.adv))
            self.ctrain = tf.train.AdamOptimizer(C_LR).minimize(closs)

        pi, pi_params = self._build_anet('pi', trainable=True)
        oldpi, oldpi_params = self._build_anet('oldpi', trainable=False)
        with tf.variable_scope('sample_action'):
            self.sample_op = tf.squeeze(pi.sample(1), axis=0)
        with tf.variable_scope('update_oldpi'):
            self.update_oldpi_op = [oldp.assign(p) for p, oldp in zip(pi_params, oldpi_params)]

        with tf.variable_scope('aloss'):
            self.tfa = tf.placeholder(tf.float32, [None, A_DIM], 'action')
            self.tfadv = tf.placeholder(tf.float32, [None, 1], 'advantage')
            with tf.variable_scope('surrogate'):
                ratio = pi.prob(self.tfa) / oldpi.prob(self.tfa)
                surr = ratio * self.tfadv
                if METHOD['name'] == 'kl_pen':
                    self.tflam = tf.placeholder(tf.float32, None, 'lambda')
                    kl = tf.distributions.kl_divergence(oldpi, pi)
                    self.kl_mean = tf.reduce_mean(kl)
                    self.aloss = -(tf.reduce_mean(surr - self.tflam * kl))
                else:  # clipping method, find this is better
                    self.aloss = -tf.reduce_mean(tf.minimum(
                        surr,
                        tf.clip_by_value(ratio, 1. - METHOD['epsilon'], 1. + METHOD['epsilon']) * self.tfadv))
            self.atrain = tf.train.AdamOptimizer(A_LR).minimize(self.aloss)
        tf.summary.FileWriter('log/', self.sess.graph)
        self.sess.run(tf.global_variables_initializer())

    def _build_anet(self, name, trainable=True):
        if name == 'Critic':
            with tf.variable_scope(name):
                # self.s_Critic = tf.placeholder(tf.float32, [None, S_DIM], 'state')
                l1_Critic = tf.layers.dense(self.tfs, 100, tf.nn.relu, trainable=trainable, name='l1')
                self.v = tf.layers.dense(l1_Critic, 1, trainable=trainable, name='value_predict')

        else:
            with tf.variable_scope(name):
                # self.s_Actor = tf.placeholder(tf.float32, [None, S_DIM], 'state')
                l1_Actor = tf.layers.dense(self.tfs, 100, tf.nn.relu, trainable=trainable, name='l1')
                mu = 2 * tf.layers.dense(l1_Actor, A_DIM, tf.nn.tanh, trainable=trainable, name='mu')
                sigma = tf.layers.dense(l1_Actor, A_DIM, tf.nn.softplus, trainable=trainable, name='sigma')
                norm_list = tf.distributions.Normal(loc=mu, scale=sigma)
                params = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope=name)
            return norm_list, params

    def update(self, s, a, r):
        self.sess.run(self.update_oldpi_op)
        adv = self.sess.run(self.adv, {self.tfdc_r: r, self.tfs: s})
        if METHOD['name'] == 'kl_pen':
            for _ in range(A_UPDATE_STEPS):
                _, kl = self.sess.run([self.atrain, self.kl_mean], {self.tfa: a, self.tfadv: adv, self.tfs: s, self.tflam: METHOD['lam']})
                if kl > 4 * METHOD['kl_target']:  # this in in google's paper
                    break
            if kl < METHOD['kl_target'] / 1.5:  # adaptive lambda, this is in OpenAI's paper
                METHOD['lam'] /= 2
            elif kl > METHOD['kl_target'] * 1.5:
                METHOD['lam'] *= 2
            METHOD['lam'] = np.clip(METHOD['lam'], 1e-4, 10)    # sometimes explode, this clipping is my solution
        else:
            [self.sess.run(self.atrain, {self.tfs: s, self.tfa: a, self.tfadv: adv}) for _ in range(A_UPDATE_STEPS)]

        [self.sess.run(self.ctrain, {self.tfs: s, self.tfdc_r: r}) for _ in range(C_UPDATE_STEPS)]

    def choose_action(self, s):
        s = s[np.newaxis, :]
        a = self.sess.run(self.sample_op, {self.tfs: s})[0]
        return np.clip(a, -2, 2)

    def get_v(self, s):
        if s.ndim < 2:
            s = s[np.newaxis, :]
        return self.sess.run(self.v, {self.tfs: s})


env = gym.make('Pendulum-v0').unwrapped
S_DIM = env.observation_space.shape[0]
A_DIM = env.action_space.shape[0]
ppo = PPO()
all_ep_r = []

for ep in range(EP_MAX):
    s = env.reset()
    buffer_s, buffer_a, buffer_r = [], [], []
    ep_r = 0
    for t in range(EP_LEN):
        env.render()
        a = ppo.choose_action(s)
        s_, r, done, _ = env.step(a)
        buffer_s.append(s)
        buffer_a.append(a)
        buffer_r.append((r+8)/8)
        s = s_
        ep_r += r

        if (t+1) % BATCH == 0 or t == EP_LEN - 1:
            v_s_ = ppo.get_v(s_)
            discounted_r = []
            for r in buffer_r[::-1]:
                v_s_ = r + GAMMA*v_s_
                discounted_r.append(v_s_)
            discounted_r.reverse()
            bs, ba, br = np.vstack(buffer_s), np.vstack(buffer_a), np.vstack(discounted_r)
            buffer_s, buffer_a, buffer_r = [], [], []
            ppo.update(bs, ba, br)

    if ep == 0:
        all_ep_r.append(ep_r)
    else:
        all_ep_r.append(all_ep_r[-1]*0.9 + ep_r*0.1)
    print('Ep:%d | Ep_r:%f' % (ep, ep_r))

plt.plot(np.arange(len(all_ep_r)), all_ep_r)
plt.xlabel('Episode')
plt.ylabel('Moving averaged episode reward')
plt.show()

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145

理解流程图

PPO算法本质上是一个AC算法，有Actor和Critic神经网络，其中，Critic网络的更新方式和AC算法差不多，Actor网络我感觉和Q-Learning一样有新旧神经网络，并周期性的更新旧神经网络。Critci网络就不多说了，不懂的可以参考一下莫烦的教程和我之前写得一篇理解，adv相当于AC中的TD_error。Actor网络主要作用就是决定策略$\pi$(pi)，程序中实现的时候假设策略是一个正态分布，所以神经网络主要是预测合适的$\mu$(mu)和$\sigma$(sigma)。然后根据这个分布选择动作，作用于环境，环境反馈下个状态等等信息。程序更新神经网络实现的时候，会存储32个动作及其环境输出的信息来更新网络，就是上面流程图中提到的batch和buffer，其中缓存下来的reward还需要做一个discounted的转换（就是一个累计的reward）。

程序讲解

大致思路

程序主要分为两部分，一部分是PPO类，还有一部分就是主程序。实现的思路莫烦老师已经已经讲的很清楚了，这边就不赘述了，截图蹭页数

主程序讲的就是环境env和算法交互的内容，输入环境内容，输出算法决策并更新算法参数。

实现注意点

几个我自己看程序的疑惑点记录一下，主要是在PPO类中：

1、tf.squeeze

在__init__sample_action中self.sample_op = tf.squeeze(pi.sample(1), axis=0)，这边pi是一个正态分布，sample(1)就是采一个点（就是选一个动作），我们调整一下程序调试一下：

 # self.sample_op = tf.squeeze(pi.sample(1), axis=0)
 self.sample_op = pi.sample(1)
 …………
 a = self.sess.run(self.sample_op, {self.tfs: s})
1
2
3
4

看下输出结果：

再调整一下程序：

 self.sample_op = tf.squeeze(pi.sample(1), axis=0)
 # self.sample_op = pi.sample(1)
 …………
 a = self.sess.run(self.sample_op, {self.tfs: s})
1
2
3
4

输出结果：

明显压缩了一维。写一段程序帮助理解一下：

简单的来说，squeeze是改变shape的，里面的内容是不变的：把所有一维的抹去（参数axis是锁定要抹去的维数的，下面一段程序axis=0，就是指抹去shape中的第一个1）。

2、Normal().prob

ratio = pi.prob(self.tfa) / oldpi.prob(self.tfa)这个函数是用来求对应点的概率密度的。