update

codes/PPO/README.md

@@ -0,0 +1,141 @@
+## 原理简介
+PPO是一种off-policy算法，具有较好的性能，其前身是TRPO算法，也是policy gradient算法的一种，它是现在 OpenAI 默认的强化学习算法，具体原理可参考[PPO算法讲解](https://datawhalechina.github.io/easy-rl/#/chapter5/chapter5)。PPO算法主要有两个变种，一个是结合KL penalty的，一个是用了clip方法，本文实现的是后者即```PPO-clip```。
+## 伪代码
+要实现必先了解伪代码，伪代码如下：
+![在这里插入图片描述](assets/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L0pvaG5KaW0w,size_16,color_FFFFFF,t_70.png)
+这是谷歌找到的一张比较适合的图，本人比较懒就没有修改，上面的```k```就是第```k```个episode，第六步是用随机梯度下降的方法优化，这里的损失函数(即```argmax```后面的部分)可能有点难理解，可参考[PPO paper](https://arxiv.org/abs/1707.06347)，如下：
+![在这里插入图片描述](assets/20210323154236878.png)
+第七步就是一个平方损失函数，即实际回报与期望回报的差平方。
+## 代码实战
+[点击查看完整代码](https://github.com/JohnJim0816/rl-tutorials/tree/master/PPO)
+### PPOmemory
+首先第三步需要搜集一条轨迹信息，我们可以定义一个```PPOmemory```来存储相关信息：
+```python
+class PPOMemory:
+    def __init__(self, batch_size):
+        self.states = []
+        self.probs = []
+        self.vals = []
+        self.actions = []
+        self.rewards = []
+        self.dones = []
+        self.batch_size = batch_size
+    def sample(self):
+        batch_step = np.arange(0, len(self.states), self.batch_size)
+        indices = np.arange(len(self.states), dtype=np.int64)
+        np.random.shuffle(indices)
+        batches = [indices[i:i+self.batch_size] for i in batch_step]
+        return np.array(self.states),\
+                np.array(self.actions),\
+                np.array(self.probs),\
+                np.array(self.vals),\
+                np.array(self.rewards),\
+                np.array(self.dones),\
+                batches
+
+    def push(self, state, action, probs, vals, reward, done):
+        self.states.append(state)
+        self.actions.append(action)
+        self.probs.append(probs)
+        self.vals.append(vals)
+        self.rewards.append(reward)
+        self.dones.append(done)
+
+    def clear(self):
+        self.states = []
+        self.probs = []
+        self.actions = []
+        self.rewards = []
+        self.dones = []
+        self.vals = []
+```
+这里的push函数就是将得到的相关量放入memory中，sample就是随机采样出来，方便第六步的随机梯度下降。
+### PPO model
+model就是actor和critic两个网络了：
+```python
+import torch.nn as nn
+from torch.distributions.categorical import Categorical
+class Actor(nn.Module):
+    def __init__(self,state_dim, action_dim,
+            hidden_dim=256):
+        super(Actor, self).__init__()
+
+        self.actor = nn.Sequential(
+                nn.Linear(state_dim, hidden_dim),
+                nn.ReLU(),
+                nn.Linear(hidden_dim, hidden_dim),
+                nn.ReLU(),
+                nn.Linear(hidden_dim, action_dim),
+                nn.Softmax(dim=-1)
+        )
+    def forward(self, state):
+        dist = self.actor(state)
+        dist = Categorical(dist)
+        return dist
+
+class Critic(nn.Module):
+    def __init__(self, state_dim,hidden_dim=256):
+        super(Critic, self).__init__()
+        self.critic = nn.Sequential(
+                nn.Linear(state_dim, hidden_dim),
+                nn.ReLU(),
+                nn.Linear(hidden_dim, hidden_dim),
+                nn.ReLU(),
+                nn.Linear(hidden_dim, 1)
+        )
+    def forward(self, state):
+        value = self.critic(state)
+        return value
+```
+这里Actor就是得到一个概率分布(Categorica，也可以是别的分布，可以搜索torch distributionsl)，critc根据当前状态得到一个值，这里的输入维度可以是```state_dim+action_dim```，即将action信息也纳入critic网络中，这样会更好一些，感兴趣的小伙伴可以试试。
+
+### PPO update
+定义一个update函数主要实现伪代码中的第六步和第七步：
+```python
+def update(self):
+    for _ in range(self.n_epochs):
+        state_arr, action_arr, old_prob_arr, vals_arr,\
+        reward_arr, dones_arr, batches = \
+                self.memory.sample()
+        values = vals_arr
+        ### compute advantage ###
+        advantage = np.zeros(len(reward_arr), dtype=np.float32)
+        for t in range(len(reward_arr)-1):
+            discount = 1
+            a_t = 0
+            for k in range(t, len(reward_arr)-1):
+                a_t += discount*(reward_arr[k] + self.gamma*values[k+1]*\
+                        (1-int(dones_arr[k])) - values[k])
+                discount *= self.gamma*self.gae_lambda
+            advantage[t] = a_t
+        advantage = torch.tensor(advantage).to(self.device)
+        ### SGD ###
+        values = torch.tensor(values).to(self.device)
+        for batch in batches:
+            states = torch.tensor(state_arr[batch], dtype=torch.float).to(self.device)
+            old_probs = torch.tensor(old_prob_arr[batch]).to(self.device)
+            actions = torch.tensor(action_arr[batch]).to(self.device)
+            dist = self.actor(states)
+            critic_value = self.critic(states)
+            critic_value = torch.squeeze(critic_value)
+            new_probs = dist.log_prob(actions)
+            prob_ratio = new_probs.exp() / old_probs.exp()
+            weighted_probs = advantage[batch] * prob_ratio
+            weighted_clipped_probs = torch.clamp(prob_ratio, 1-self.policy_clip,
+                    1+self.policy_clip)*advantage[batch]
+            actor_loss = -torch.min(weighted_probs, weighted_clipped_probs).mean()
+            returns = advantage[batch] + values[batch]
+            critic_loss = (returns-critic_value)**2
+            critic_loss = critic_loss.mean()
+            total_loss = actor_loss + 0.5*critic_loss
+            self.actor_optimizer.zero_grad()
+            self.critic_optimizer.zero_grad()
+            total_loss.backward()
+            self.actor_optimizer.step()
+            self.critic_optimizer.step()
+    self.memory.clear()
+```
+该部分首先从memory中提取搜集到的轨迹信息，然后计算gae，即advantage，接着使用随机梯度下降更新网络，最后清除memory以便搜集下一条轨迹信息。
+
+最后实现效果如下：
+![在这里插入图片描述](assets/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L0pvaG5KaW0w,size_16,color_FFFFFF,t_70-20210405110725113.png)

codes/PPO/agent.py

@@ -5,7 +5,7 @@ Author: John
 Email: johnjim0816@gmail.com
 Date: 2021-03-23 15:17:42
 LastEditor: John
-LastEditTime: 2021-03-23 15:52:34
+LastEditTime: 2021-04-11 01:24:24
 Discription: 
 Environment: 
 '''
@@ -17,16 +17,18 @@ from PPO.model import Actor,Critic
 from PPO.memory import PPOMemory
 class PPO:
     def __init__(self, state_dim, action_dim,cfg):
+        self.env = cfg.env
         self.gamma = cfg.gamma
         self.policy_clip = cfg.policy_clip
         self.n_epochs = cfg.n_epochs
         self.gae_lambda = cfg.gae_lambda
         self.device = cfg.device
-        self.actor = Actor(state_dim, action_dim).to(self.device)
-        self.critic = Critic(state_dim).to(self.device)
-        self.actor_optimizer = optim.Adam(self.actor.parameters(), lr=cfg.lr)
-        self.critic_optimizer = optim.Adam(self.critic.parameters(), lr=cfg.lr)
+        self.actor = Actor(state_dim, action_dim,cfg.hidden_dim).to(self.device)
+        self.critic = Critic(state_dim,cfg.hidden_dim).to(self.device)
+        self.actor_optimizer = optim.Adam(self.actor.parameters(), lr=cfg.actor_lr)
+        self.critic_optimizer = optim.Adam(self.critic.parameters(), lr=cfg.critic_lr)
         self.memory = PPOMemory(cfg.batch_size)
+        self.loss = 0
 
     def choose_action(self, observation):
         state = torch.tensor([observation], dtype=torch.float).to(self.device)
@@ -74,6 +76,7 @@ class PPO:
                 critic_loss = (returns-critic_value)**2
                 critic_loss = critic_loss.mean()
                 total_loss = actor_loss + 0.5*critic_loss
+                self.loss  = total_loss
                 self.actor_optimizer.zero_grad()
                 self.critic_optimizer.zero_grad()
                 total_loss.backward()
@@ -81,13 +84,13 @@ class PPO:
                 self.critic_optimizer.step()
         self.memory.clear()  
     def save(self,path):
-        actor_checkpoint = os.path.join(path, 'actor_torch_ppo.pt')
-        critic_checkpoint= os.path.join(path, 'critic_torch_ppo.pt')
+        actor_checkpoint = os.path.join(path, self.env+'_actor.pt')
+        critic_checkpoint= os.path.join(path, self.env+'_critic.pt')
         torch.save(self.actor.state_dict(), actor_checkpoint)
         torch.save(self.critic.state_dict(), critic_checkpoint)
     def load(self,path):
-        actor_checkpoint = os.path.join(path, 'actor_torch_ppo.pt')
-        critic_checkpoint= os.path.join(path, 'critic_torch_ppo.pt')
+        actor_checkpoint = os.path.join(path, self.env+'_actor.pt')
+        critic_checkpoint= os.path.join(path, self.env+'_critic.pt')
         self.actor.load_state_dict(torch.load(actor_checkpoint)) 
         self.critic.load_state_dict(torch.load(critic_checkpoint))  
 

codes/PPO/main.py

@@ -5,12 +5,14 @@ Author: John
 Email: johnjim0816@gmail.com
 Date: 2021-03-22 16:18:10
 LastEditor: John
-LastEditTime: 2021-03-23 15:52:52
+LastEditTime: 2021-04-11 01:24:41
 Discription: 
 Environment: 
 '''
 import sys,os
-sys.path.append(os.getcwd()) # add current terminal path to sys.path
+curr_path = os.path.dirname(__file__)
+parent_path=os.path.dirname(curr_path) 
+sys.path.append(parent_path) # add current terminal path to sys.path
 import gym
 import numpy as np
 import torch
@@ -33,15 +35,18 @@ if not os.path.exists(RESULT_PATH): # 检测是否存在文件夹
 
 class PPOConfig:
     def __init__(self) -> None:
+        self.env = 'CartPole-v0'
         self.algo = 'PPO'
         self.batch_size = 5
         self.gamma=0.99
         self.n_epochs = 4
-        self.lr = 0.0003
+        self.actor_lr = 0.0003
+        self.critic_lr = 0.0003
         self.gae_lambda=0.95
         self.policy_clip=0.2
+        self.hidden_dim = 256
         self.update_fre = 20 # frequency of agent update
-        self.train_eps = 250 # max training episodes
+        self.train_eps = 300 # max training episodes
         self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # check gpu
         
 def train(cfg,env,agent):
@@ -70,7 +75,8 @@ def train(cfg,env,agent):
         else:
             ma_rewards.append(ep_reward)
         avg_reward = np.mean(rewards[-100:])
-        if avg_reward > best_reward:
+        if avg_rewardself.actor_lr = 0.002
+        self.critic_lr = 0.005 > best_reward:
             best_reward = avg_reward
             agent.save(path=SAVED_MODEL_PATH)
         print('Episode:{}/{}, Reward:{:.1f}, avg reward:{:.1f}, Done:{}'.format(i_episode+1,cfg.train_eps,ep_reward,avg_reward,done))
@@ -78,7 +84,7 @@ def train(cfg,env,agent):
 
 if __name__ == '__main__':
     cfg = PPOConfig()
-    env = gym.make('CartPole-v0')
+    env = gym.make(cfg.env)
     env.seed(1)
     state_dim=env.observation_space.shape[0]
     action_dim=env.action_space.n

codes/PPO/model.py

@@ -5,7 +5,7 @@ Author: John
 Email: johnjim0816@gmail.com
 Date: 2021-03-23 15:29:24
 LastEditor: John
-LastEditTime: 2021-03-23 15:29:52
+LastEditTime: 2021-04-08 22:36:43
 Discription: 
 Environment: 
 '''
@@ -13,7 +13,7 @@ import torch.nn as nn
 from torch.distributions.categorical import Categorical
 class Actor(nn.Module):
     def __init__(self,state_dim, action_dim,
-            hidden_dim=256):
+            hidden_dim):
         super(Actor, self).__init__()
 
         self.actor = nn.Sequential(
@@ -30,7 +30,7 @@ class Actor(nn.Module):
         return dist
 
 class Critic(nn.Module):
-    def __init__(self, state_dim,hidden_dim=256):
+    def __init__(self, state_dim,hidden_dim):
         super(Critic, self).__init__()
         self.critic = nn.Sequential(
                 nn.Linear(state_dim, hidden_dim),

codes/PPO/task1.py

@@ -0,0 +1,97 @@
+#!/usr/bin/env python
+# coding=utf-8
+'''
+Author: John
+Email: johnjim0816@gmail.com
+Date: 2021-03-22 16:18:10
+LastEditor: John
+LastEditTime: 2021-04-11 01:25:43
+Discription: 
+Environment: 
+'''
+import sys,os
+curr_path = os.path.dirname(__file__)
+parent_path=os.path.dirname(curr_path) 
+sys.path.append(parent_path) # add current terminal path to sys.path
+import gym
+import numpy as np
+import torch
+import datetime
+from PPO.agent import PPO
+from common.plot import plot_rewards
+from common.utils import save_results
+
+SEQUENCE = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") # 获取当前时间
+SAVED_MODEL_PATH = os.path.split(os.path.abspath(__file__))[0]+"/saved_model/"+SEQUENCE+'/' # 生成保存的模型路径
+if not os.path.exists(os.path.split(os.path.abspath(__file__))[0]+"/saved_model/"): # 检测是否存在文件夹
+    os.mkdir(os.path.split(os.path.abspath(__file__))[0]+"/saved_model/")
+if not os.path.exists(SAVED_MODEL_PATH): # 检测是否存在文件夹
+    os.mkdir(SAVED_MODEL_PATH)
+RESULT_PATH = os.path.split(os.path.abspath(__file__))[0]+"/results/"+SEQUENCE+'/' # 存储reward的路径
+if not os.path.exists(os.path.split(os.path.abspath(__file__))[0]+"/results/"): # 检测是否存在文件夹
+    os.mkdir(os.path.split(os.path.abspath(__file__))[0]+"/results/")
+if not os.path.exists(RESULT_PATH): # 检测是否存在文件夹
+    os.mkdir(RESULT_PATH)
+
+class PPOConfig:
+    def __init__(self) -> None:
+        self.env = 'LunarLander-v2'
+        self.algo = 'PPO'
+        self.batch_size = 128
+        self.gamma=0.95
+        self.n_epochs = 4
+        self.actor_lr = 0.002
+        self.critic_lr = 0.005
+        self.gae_lambda=0.95
+        self.policy_clip=0.2
+        self.hidden_dim = 256
+        self.update_fre = 20 # frequency of agent update
+        self.train_eps = 300 # max training episodes
+        self.train_steps = 1000
+        self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # check gpu
+        
+def train(cfg,env,agent):
+    best_reward = env.reward_range[0]
+    rewards= []
+    ma_rewards = [] # moving average rewards
+    avg_reward = 0
+    running_steps = 0
+    for i_episode in range(cfg.train_eps):
+        state = env.reset()
+        done = False
+        ep_reward = 0
+        # for i_step in range(cfg.train_steps):
+        while not done:
+            action, prob, val = agent.choose_action(state)
+            state_, reward, done, _ = env.step(action)
+            running_steps += 1
+            ep_reward += reward
+            agent.memory.push(state, action, prob, val, reward, done)
+            if running_steps % cfg.update_fre == 0:
+                agent.update()
+            state = state_
+            # if done:
+            #     break
+        rewards.append(ep_reward)
+        if ma_rewards:
+            ma_rewards.append(
+                0.9*ma_rewards[-1]+0.1*ep_reward)
+        else:
+            ma_rewards.append(ep_reward)
+        avg_reward = np.mean(rewards[-100:])
+        if avg_reward > best_reward:
+            best_reward = avg_reward
+            agent.save(path=SAVED_MODEL_PATH)
+        print('Episode:{}/{}, Reward:{:.1f}, avg reward:{:.1f}, Loss:{}'.format(i_episode+1,cfg.train_eps,ep_reward,avg_reward,agent.loss))
+    return rewards,ma_rewards
+
+if __name__ == '__main__':
+    cfg = PPOConfig()
+    env = gym.make(cfg.env)
+    env.seed(1)
+    state_dim=env.observation_space.shape[0]
+    action_dim=env.action_space.n
+    agent = PPO(state_dim,action_dim,cfg)
+    rewards,ma_rewards = train(cfg,env,agent)
+    save_results(rewards,ma_rewards,tag='train',path=RESULT_PATH)
+    plot_rewards(rewards,ma_rewards,tag="train",algo = cfg.algo,path=RESULT_PATH)