update codes

codes/DQN-series/DQN/agent.py

@@ -12,9 +12,6 @@ LastEditTime: 2021-09-15 13:35:36
 '''off-policy
 '''
 
-
-
-
 import torch
 import torch.nn as nn
 import torch.optim as optim
@@ -24,9 +21,9 @@ import numpy as np
 from common.memory import ReplayBuffer
 from common.model import MLP
 class DQN:
-    def __init__(self, state_dim, action_dim, cfg):
+    def __init__(self, n_states, n_actions, cfg):
 
-        self.action_dim = action_dim  # 总的动作个数
+        self.n_actions = n_actions  # 总的动作个数
         self.device = cfg.device  # 设备，cpu或gpu等
         self.gamma = cfg.gamma  # 奖励的折扣因子
         # e-greedy策略相关参数
@@ -35,15 +32,15 @@ class DQN:
             (cfg.epsilon_start - cfg.epsilon_end) * \
             math.exp(-1. * frame_idx / cfg.epsilon_decay)
         self.batch_size = cfg.batch_size
-        self.policy_net = MLP(state_dim, action_dim,hidden_dim=cfg.hidden_dim).to(self.device)
-        self.target_net = MLP(state_dim, action_dim,hidden_dim=cfg.hidden_dim).to(self.device)
+        self.policy_net = MLP(n_states, n_actions,hidden_dim=cfg.hidden_dim).to(self.device)
+        self.target_net = MLP(n_states, n_actions,hidden_dim=cfg.hidden_dim).to(self.device)
         for target_param, param in zip(self.target_net.parameters(),self.policy_net.parameters()): # 复制参数到目标网路targe_net
             target_param.data.copy_(param.data)
         self.optimizer = optim.Adam(self.policy_net.parameters(), lr=cfg.lr) # 优化器
-        self.memory = ReplayBuffer(cfg.memory_capacity)
+        self.memory = ReplayBuffer(cfg.memory_capacity) # 经验回放
 
     def choose_action(self, state):
-        '''选择动作
+        ''' 选择动作
         '''
         self.frame_idx += 1
         if random.random() > self.epsilon(self.frame_idx):
@@ -52,13 +49,7 @@ class DQN:
                 q_values = self.policy_net(state)
                 action = q_values.max(1)[1].item() # 选择Q值最大的动作
         else:
-            action = random.randrange(self.action_dim)
-        return action
-    def predict(self,state):
-        with torch.no_grad():
-            state = torch.tensor([state], device=self.device, dtype=torch.float32)
-            q_values = self.policy_net(state)
-            action = q_values.max(1)[1].item()
+            action = random.randrange(self.n_actions)
         return action
     def update(self):
         if len(self.memory) < self.batch_size: # 当memory中不满足一个批量时，不更新策略
@@ -67,16 +58,11 @@ class DQN:
         state_batch, action_batch, reward_batch, next_state_batch, done_batch = self.memory.sample(
             self.batch_size)
         # 转为张量
-        state_batch = torch.tensor(
-            state_batch, device=self.device, dtype=torch.float)
-        action_batch = torch.tensor(action_batch, device=self.device).unsqueeze(
-            1)  
-        reward_batch = torch.tensor(
-            reward_batch, device=self.device, dtype=torch.float)  
-        next_state_batch = torch.tensor(
-            next_state_batch, device=self.device, dtype=torch.float)
-        done_batch = torch.tensor(np.float32(
-            done_batch), device=self.device)
+        state_batch = torch.tensor(state_batch, device=self.device, dtype=torch.float)
+        action_batch = torch.tensor(action_batch, device=self.device).unsqueeze(1)  
+        reward_batch = torch.tensor(reward_batch, device=self.device, dtype=torch.float)  
+        next_state_batch = torch.tensor(next_state_batch, device=self.device, dtype=torch.float)
+        done_batch = torch.tensor(np.float32(done_batch), device=self.device)
         q_values = self.policy_net(state_batch).gather(dim=1, index=action_batch) # 计算当前状态(s_t,a)对应的Q(s_t, a)
         next_q_values = self.target_net(next_state_batch).max(1)[0].detach() # 计算下一时刻的状态(s_t_,a)对应的Q值
         # 计算期望的Q值，对于终止状态，此时done_batch[0]=1, 对应的expected_q_value等于reward

codes/DQN-series/DQN/task0_train.py

@@ -19,19 +19,14 @@ import torch
 import datetime
 
 from common.utils import save_results, make_dir
-from common.plot import plot_rewards,plot_rewards_cn
+from common.plot import plot_rewards
 from DQN.agent import DQN
 
 curr_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")  # 获取当前时间
-
 class DQNConfig:
     def __init__(self):
         self.algo = "DQN"  # 算法名称
         self.env = 'CartPole-v0' # 环境名称
-        self.result_path = curr_path+"/outputs/" + self.env + \
-            '/'+curr_time+'/results/'  # 保存结果的路径
-        self.model_path = curr_path+"/outputs/" + self.env + \
-            '/'+curr_time+'/models/'  # 保存模型的路径
         self.train_eps = 200 # 训练的回合数
         self.eval_eps = 30 # 测试的回合数
         self.gamma = 0.95 # 强化学习中的折扣因子
@@ -42,42 +37,53 @@ class DQNConfig:
         self.memory_capacity = 100000  # 经验回放的容量
         self.batch_size = 64 # mini-batch SGD中的批量大小
         self.target_update = 4 # 目标网络的更新频率
-        self.device = torch.device(
-            "cuda" if torch.cuda.is_available() else "cpu")  # 检测GPU
-        self.hidden_dim = 256  # hidden size of net
-        
+        self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")  # 检测GPU
+        self.hidden_dim = 256  # 网络隐藏层
+class PlotConfig:
+    def __init__(self) -> None:
+        self.algo = "DQN"  # 算法名称
+        self.env = 'CartPole-v0' # 环境名称
+        self.result_path = curr_path+"/outputs/" + self.env + \
+            '/'+curr_time+'/results/'  # 保存结果的路径
+        self.model_path = curr_path+"/outputs/" + self.env + \
+            '/'+curr_time+'/models/'  # 保存模型的路径
+        self.save = True # 是否保存图片
+        self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")  # 检测GPU
+
 def env_agent_config(cfg,seed=1):
-    env = gym.make(cfg.env)  
-    env.seed(seed)
-    n_states = env.observation_space.shape[0]
-    n_actions = env.action_space.n
-    agent = DQN(n_states,n_actions,cfg)
+    ''' 创建环境和智能体
+    '''
+    env = gym.make(cfg.env)  # 创建环境
+    env.seed(seed) # 设置随机种子
+    n_states = env.observation_space.shape[0] # 状态数
+    n_actions = env.action_space.n # 动作数
+    agent = DQN(n_states,n_actions,cfg) # 创建智能体
     return env,agent
     
 def train(cfg, env, agent):
+    ''' 训练
+    '''
     print('开始训练!')
     print(f'环境：{cfg.env}, 算法：{cfg.algo}, 设备：{cfg.device}')
-    rewards = [] # 记录奖励
-    ma_rewards = []  # 记录滑动平均奖励
+    rewards = [] # 记录所有回合的奖励
+    ma_rewards = []  # 记录所有回合的滑动平均奖励
     for i_ep in range(cfg.train_eps):
-        state = env.reset()
-        done = False
-        ep_reward = 0
+        ep_reward = 0 # 记录一回合内的奖励
+        state = env.reset() # 重置环境，返回初始状态
         while True:
-            action = agent.choose_action(state)
-            next_state, reward, done, _ = env.step(action)
-            ep_reward += reward
-            agent.memory.push(state, action, reward, next_state, done)
-            state = next_state
-            agent.update()
+            action = agent.choose_action(state) # 选择动作
+            next_state, reward, done, _ = env.step(action) # 更新环境，返回transition
+            agent.memory.push(state, action, reward, next_state, done) # 保存transition
+            state = next_state # 更新下一个状态
+            agent.update() # 更新智能体
+            ep_reward += reward # 累加奖励
             if done:
                 break
-        if (i_ep+1) % cfg.target_update == 0:
+        if (i_ep+1) % cfg.target_update == 0: # 智能体目标网络更新
             agent.target_net.load_state_dict(agent.policy_net.state_dict())
-        if (i_ep+1)%10 == 0:
+        if (i_ep+1)%10 == 0: 
             print('回合：{}/{}, 奖励：{}'.format(i_ep+1, cfg.train_eps, ep_reward))
         rewards.append(ep_reward)
-        # save ma_rewards
         if ma_rewards:
             ma_rewards.append(0.9*ma_rewards[-1]+0.1*ep_reward)
         else:
@@ -88,16 +94,19 @@ def train(cfg, env, agent):
 def eval(cfg,env,agent):
     print('开始测试!')
     print(f'环境：{cfg.env}, 算法：{cfg.algo}, 设备：{cfg.device}')
-    rewards = []  
-    ma_rewards = [] # moving average rewards
+    # 由于测试不需要使用epsilon-greedy策略，所以相应的值设置为0
+    cfg.epsilon_start = 0.0 # e-greedy策略中初始epsilon
+    cfg.epsilon_end = 0.0 # e-greedy策略中的终止epsilon
+    rewards = [] # 记录所有回合的奖励
+    ma_rewards = []  # 记录所有回合的滑动平均奖励
     for i_ep in range(cfg.eval_eps):
-        ep_reward = 0  # reward per episode
-        state = env.reset()  
+        ep_reward = 0 # 记录一回合内的奖励
+        state = env.reset() # 重置环境，返回初始状态
         while True:
-            action = agent.predict(state) 
-            next_state, reward, done, _ = env.step(action)  
-            state = next_state  
-            ep_reward += reward
+            action = agent.choose_action(state) # 选择动作
+            next_state, reward, done, _ = env.step(action) # 更新环境，返回transition
+            state = next_state # 更新下一个状态
+            ep_reward += reward # 累加奖励
             if done:
                 break
         rewards.append(ep_reward)
@@ -111,17 +120,17 @@ def eval(cfg,env,agent):
 
 if __name__ == "__main__":
     cfg = DQNConfig()
+    plot_cfg = PlotConfig()
     # 训练
     env,agent = env_agent_config(cfg,seed=1)
     rewards, ma_rewards = train(cfg, env, agent)
-    make_dir(cfg.result_path, cfg.model_path)
-    agent.save(path=cfg.model_path)
-    save_results(rewards, ma_rewards, tag='train', path=cfg.result_path)
-    plot_rewards_cn(rewards, ma_rewards, tag="train",
-                 algo=cfg.algo, path=cfg.result_path)
+    make_dir(plot_cfg.result_path, plot_cfg.model_path) # 创建保存结果和模型路径的文件夹
+    agent.save(path=plot_cfg.model_path) # 保存模型
+    save_results(rewards, ma_rewards, tag='train', path=plot_cfg.result_path) # 保存结果
+    plot_rewards(rewards, ma_rewards, plot_cfg, tag="train") # 画出结果
     # 测试
     env,agent = env_agent_config(cfg,seed=10)
-    agent.load(path=cfg.model_path)
+    agent.load(path=plot_cfg.model_path) # 导入模型
     rewards,ma_rewards = eval(cfg,env,agent)
-    save_results(rewards,ma_rewards,tag='eval',path=cfg.result_path)
-    plot_rewards_cn(rewards,ma_rewards,tag="eval",env=cfg.env,algo = cfg.algo,path=cfg.result_path)
+    save_results(rewards,ma_rewards,tag='eval',path=plot_cfg.result_path) # 保存结果
+    plot_rewards(rewards,ma_rewards, plot_cfg, tag="eval") # 画出结果

codes/DQN-series/NoisyDQN/task0_train.ipynb

@@ -0,0 +1,25 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import sys\n",
+    "from pathlib import Path\n",
+    "curr_path = str(Path().absolute()) # 当前路径\n",
+    "parent_path = str(Path().absolute().parent) # 父路径\n",
+    "sys.path.append(parent_path) # 添加路径到系统路径"
+   ]
+  }
+ ],
+ "metadata": {
+  "language_info": {
+   "name": "python"
+  },
+  "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

codes/DQN-series/README.md

@@ -0,0 +1,3 @@
+
+本目录下汇总了基础的DQN及其变种或升级，如下
+

codes/PPO/task0_train.py

@@ -100,7 +100,7 @@ def eval(cfg,env,agent):
                 0.9*ma_rewards[-1]+0.1*ep_reward)
         else:
             ma_rewards.append(ep_reward)
-        print(f"Episode:{i_ep+1}/{cfg.train_eps}, Reward:{ep_reward:.3f}")
+        print(f"Episode:{i_ep+1}/{cfg.eval_eps}, Reward:{ep_reward:.3f}")
     print('Complete evaling！')
     return rewards,ma_rewards
     

codes/PolicyGradient/README.md

@@ -8,12 +8,16 @@ Policy-based方法是强化学习中与Value-based(比如Q-learning)相对的方
 
 结合REINFORCE原理，其伪代码如下：
 
+<img src="assets/image-20211016004808604.png" alt="image-20211016004808604" style="zoom:50%;" />
+
+https://pytorch.org/docs/stable/distributions.html
+
+加负号的原因是，在公式中应该是实现的梯度上升算法，而loss一般使用随机梯度下降的，所以加个负号保持一致性。
+
 ![img](assets/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L0pvaG5KaW0w,size_16,color_FFFFFF,t_70-20210428001336032.png)
 
 ## 实现
 
-
-
 ## 参考
 
 [REINFORCE和Reparameterization Trick](https://blog.csdn.net/JohnJim0/article/details/110230703)

codes/PolicyGradient/agent.py

@@ -5,7 +5,7 @@ Author: John
 Email: johnjim0816@gmail.com
 Date: 2020-11-22 23:27:44
 LastEditor: John
-LastEditTime: 2021-05-05 17:33:10
+LastEditTime: 2021-10-16 00:43:52
 Discription: 
 Environment: 
 '''
@@ -56,7 +56,6 @@ class PolicyGradient:
             state = state_pool[i]
             action = Variable(torch.FloatTensor([action_pool[i]]))
             reward = reward_pool[i]
-
             state = Variable(torch.from_numpy(state).float())
             probs = self.policy_net(state)
             m = Bernoulli(probs)

codes/PolicyGradient/task0_train.py

@@ -5,14 +5,14 @@ Author: John
 Email: johnjim0816@gmail.com
 Date: 2020-11-22 23:21:53
 LastEditor: John
-LastEditTime: 2021-05-05 17:35:20
+LastEditTime: 2021-10-16 00:34:13
 Discription: 
 Environment: 
 '''
 import sys,os
-curr_path = os.path.dirname(__file__)
-parent_path = os.path.dirname(curr_path)
-sys.path.append(parent_path)  # add current terminal path to sys.path
+curr_path = os.path.dirname(os.path.abspath(__file__)) # 当前文件所在绝对路径
+parent_path = os.path.dirname(curr_path) # 父路径
+sys.path.append(parent_path) # 添加父路径到系统路径sys.path
 
 import gym
 import torch
@@ -23,21 +23,20 @@ from PolicyGradient.agent import PolicyGradient
 from common.plot import plot_rewards
 from common.utils import save_results,make_dir
 
-curr_time = datetime.datetime.now().strftime(
-    "%Y%m%d-%H%M%S")  # obtain current time
+curr_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")  # 获取当前时间
 
 class PGConfig:
     def __init__(self):
-        self.algo = "PolicyGradient"  # name of algo
-        self.env = 'CartPole-v0'
+        self.algo = "PolicyGradient"  # 算法名称
+        self.env = 'CartPole-v0' # 环境名称
         self.result_path = curr_path+"/outputs/" + self.env + \
-            '/'+curr_time+'/results/'  # path to save results
+            '/'+curr_time+'/results/'  # 保存结果的路径
         self.model_path = curr_path+"/outputs/" + self.env + \
-            '/'+curr_time+'/models/'  # path to save models
-        self.train_eps = 300 # 训练的episode数目
-        self.eval_eps = 50
+            '/'+curr_time+'/models/'  # 保存模型的路径
+        self.train_eps = 300 # 训练的回合数
+        self.eval_eps = 30 # 测试的回合数
         self.batch_size = 8
-        self.lr = 0.01 # learning rate
+        self.lr = 0.01 # 学习率
         self.gamma = 0.99
         self.hidden_dim = 36 # dimmension of hidden layer
         self.device = torch.device(
@@ -59,7 +58,7 @@ def train(cfg,env,agent):
     reward_pool = [] 
     rewards = []
     ma_rewards = []
-    for i_episode in range(cfg.train_eps):
+    for i_ep in range(cfg.train_eps):
         state = env.reset()
         ep_reward = 0
         for _ in count():
@@ -73,9 +72,9 @@ def train(cfg,env,agent):
             reward_pool.append(reward)
             state = next_state
             if done:
-                print('Episode:', i_episode, ' Reward:',  ep_reward)
+                print('Episode:', i_ep, ' Reward:',  ep_reward)
                 break
-        if i_episode > 0 and i_episode % cfg.batch_size == 0:
+        if i_ep > 0 and i_ep % cfg.batch_size == 0:
             agent.update(reward_pool,state_pool,action_pool)
             state_pool = [] # 每个episode的state
             action_pool = []
@@ -95,7 +94,7 @@ def eval(cfg,env,agent):
     print(f'Env:{cfg.env}, Algorithm:{cfg.algo}, Device:{cfg.device}')
     rewards = []
     ma_rewards = []
-    for i_episode in range(cfg.eval_eps):
+    for i_ep in range(cfg.eval_eps):
         state = env.reset()
         ep_reward = 0
         for _ in count():
@@ -106,7 +105,7 @@ def eval(cfg,env,agent):
                 reward = 0
             state = next_state
             if done:
-                print('Episode:', i_episode, ' Reward:',  ep_reward)
+                print('Episode:', i_ep, ' Reward:',  ep_reward)
                 break
         rewards.append(ep_reward)
         if ma_rewards:
@@ -116,6 +115,7 @@ def eval(cfg,env,agent):
             ma_rewards.append(ep_reward)
     print('complete evaling！')
     return rewards, ma_rewards
+    
 if __name__ == "__main__":
     cfg = PGConfig()
 

codes/README.md

@@ -18,14 +18,14 @@
 
 ## 运行环境
 
-python 3.7、pytorch 1.6.0-1.7.1、gym 0.17.0-0.19.0
+python 3.7、pytorch 1.6.0-1.8.1、gym 0.17.0-0.19.0
 
 ## 使用说明
 
 运行带有```train```的py文件或ipynb文件进行训练，如果前面带有```task```如```task0_train.py```，表示对task0任务训练，
 类似的带有```eval```即为测试。
 
-## 算法进度
+## 内容导航
 
 |                 算法名称                 |                         相关论文材料                         | 环境                                      |                备注                |
 | :--------------------------------------: | :----------------------------------------------------------: | ----------------------------------------- | :--------------------------------: |

codes/common/model.py

@@ -15,15 +15,15 @@ import torch.nn.functional as F
 from torch.distributions import Categorical
 
 class MLP(nn.Module):
-    def __init__(self, input_dim,output_dim,hidden_dim=128):
+    def __init__(self, n_states,n_actions,hidden_dim=128):
         """ 初始化q网络，为全连接网络
-            input_dim: 输入的特征数即环境的状态数
-            output_dim: 输出的动作维度
+            n_states: 输入的特征数即环境的状态数
+            n_actions: 输出的动作维度
         """
         super(MLP, self).__init__()
-        self.fc1 = nn.Linear(input_dim, hidden_dim) # 输入层
+        self.fc1 = nn.Linear(n_states, hidden_dim) # 输入层
         self.fc2 = nn.Linear(hidden_dim,hidden_dim) # 隐藏层
-        self.fc3 = nn.Linear(hidden_dim, output_dim) # 输出层
+        self.fc3 = nn.Linear(hidden_dim, n_actions) # 输出层
         
     def forward(self, x):
         # 各层对应的激活函数
@@ -32,10 +32,10 @@ class MLP(nn.Module):
         return self.fc3(x)
 
 class Critic(nn.Module):
-    def __init__(self, n_obs, output_dim, hidden_size, init_w=3e-3):
+    def __init__(self, n_obs, n_actions, hidden_size, init_w=3e-3):
         super(Critic, self).__init__()
         
-        self.linear1 = nn.Linear(n_obs + output_dim, hidden_size)
+        self.linear1 = nn.Linear(n_obs + n_actions, hidden_size)
         self.linear2 = nn.Linear(hidden_size, hidden_size)
         self.linear3 = nn.Linear(hidden_size, 1)
         # 随机初始化为较小的值
@@ -51,11 +51,11 @@ class Critic(nn.Module):
         return x
 
 class Actor(nn.Module):
-    def __init__(self, n_obs, output_dim, hidden_size, init_w=3e-3):
+    def __init__(self, n_obs, n_actions, hidden_size, init_w=3e-3):
         super(Actor, self).__init__()  
         self.linear1 = nn.Linear(n_obs, hidden_size)
         self.linear2 = nn.Linear(hidden_size, hidden_size)
-        self.linear3 = nn.Linear(hidden_size, output_dim)
+        self.linear3 = nn.Linear(hidden_size, n_actions)
         
         self.linear3.weight.data.uniform_(-init_w, init_w)
         self.linear3.bias.data.uniform_(-init_w, init_w)
@@ -67,18 +67,18 @@ class Actor(nn.Module):
         return x
 
 class ActorCritic(nn.Module):
-    def __init__(self, input_dim, output_dim, hidden_dim=256):
+    def __init__(self, n_states, n_actions, hidden_dim=256):
         super(ActorCritic, self).__init__()
         self.critic = nn.Sequential(
-            nn.Linear(input_dim, hidden_dim),
+            nn.Linear(n_states, hidden_dim),
             nn.ReLU(),
             nn.Linear(hidden_dim, 1)
         )
         
         self.actor = nn.Sequential(
-            nn.Linear(input_dim, hidden_dim),
+            nn.Linear(n_states, hidden_dim),
             nn.ReLU(),
-            nn.Linear(hidden_dim, output_dim),
+            nn.Linear(hidden_dim, n_actions),
             nn.Softmax(dim=1),
         )
         

codes/common/plot.py

@@ -11,36 +11,52 @@ Environment:
 '''
 import matplotlib.pyplot as plt
 import seaborn as sns
-from matplotlib.font_manager import FontProperties
-def chinese_font():  
-    return FontProperties(fname='/System/Library/Fonts/STHeiti Light.ttc',size=15)  # 系统字体路径，此处是mac的
-def plot_rewards(rewards,ma_rewards,tag="train",env='CartPole-v0',algo = "DQN",save=True,path='./'):
-    sns.set()
-    plt.title("average learning curve of {} for {}".format(algo,env))
+# from matplotlib.font_manager import FontProperties # 导入字体模块
+
+# def chinese_font():  
+#     ''' 设置中文字体
+#     '''
+#     return FontProperties(fname='/System/Library/Fonts/STHeiti Light.ttc',size=15)  # fname系统字体路径，此处是mac的
+# def plot_rewards_cn(rewards,ma_rewards,tag="train",env='CartPole-v0',algo = "DQN",save=True,path='./'):
+#     ''' 中文画图
+#     '''
+#     sns.set()
+#     plt.figure()
+#     plt.title(u"{}环境下{}算法的学习曲线".format(env,algo),fontproperties=chinese_font())
+#     plt.xlabel(u'回合数',fontproperties=chinese_font())
+#     plt.plot(rewards)
+#     plt.plot(ma_rewards)
+#     plt.legend((u'奖励',u'滑动平均奖励',),loc="best",prop=chinese_font())
+#     if save:
+#         plt.savefig(path+f"{tag}_rewards_curve_cn")
+#     # plt.show()
+
+def plot_rewards(rewards,ma_rewards,plot_cfg,tag='train'):
+    sns.set() 
+    plt.figure() # 创建一个图形实例，方便同时多画几个图
+    plt.title("learning curve on {} of {} for {}".format(plot_cfg.device, plot_cfg.algo, plot_cfg.env))
     plt.xlabel('epsiodes')
     plt.plot(rewards,label='rewards')
     plt.plot(ma_rewards,label='ma rewards')
     plt.legend()
-    if save:
-        plt.savefig(path+"{}_rewards_curve".format(tag))
+    if plot_cfg.save:
+        plt.savefig(plot_cfg.result_path+"{}_rewards_curve".format(tag))
     plt.show()
-
-def plot_rewards_cn(rewards,ma_rewards,tag="train",env='CartPole-v0',algo = "DQN",save=True,path='./'):
-    ''' 中文画图
-    '''
-    sns.set()
-    plt.figure()
-    plt.title(u"{}环境下{}算法的学习曲线".format(env,algo),fontproperties=chinese_font())
-    plt.xlabel(u'回合数',fontproperties=chinese_font())
-    plt.plot(rewards)
-    plt.plot(ma_rewards)
-    plt.legend((u'奖励',u'滑动平均奖励',),loc="best",prop=chinese_font())
-    if save:
-        plt.savefig(path+f"{tag}_rewards_curve_cn")
-    # plt.show()
+# def plot_rewards(rewards,ma_rewards,tag="train",env='CartPole-v0',algo = "DQN",save=True,path='./'):
+#     sns.set() 
+#     plt.figure() # 创建一个图形实例，方便同时多画几个图
+#     plt.title("average learning curve of {} for {}".format(algo,env))
+#     plt.xlabel('epsiodes')
+#     plt.plot(rewards,label='rewards')
+#     plt.plot(ma_rewards,label='ma rewards')
+#     plt.legend()
+#     if save:
+#         plt.savefig(path+"{}_rewards_curve".format(tag))
+#     plt.show()
 
 def plot_losses(losses,algo = "DQN",save=True,path='./'):
     sns.set()
+    plt.figure()
     plt.title("loss curve of {}".format(algo))
     plt.xlabel('epsiodes')
     plt.plot(losses,label='rewards')

codes/envs/README.md

@@ -0,0 +1,6 @@
+## 环境汇总
+
+[OpenAI Gym](./gym_info.md)  
+[MuJoCo](./mujoco_info.md)  
+
+