update codes

codes/A2C/agent.py

@@ -12,10 +12,10 @@ Environment:
 import torch.optim as optim
 from A2C.model import ActorCritic
 class A2C:
-    def __init__(self,state_dim,action_dim,cfg) -> None:
+    def __init__(self,n_states,n_actions,cfg) -> None:
         self.gamma = cfg.gamma
         self.device = cfg.device
-        self.model = ActorCritic(state_dim, action_dim, cfg.hidden_size).to(self.device)
+        self.model = ActorCritic(n_states, n_actions, cfg.hidden_size).to(self.device)
         self.optimizer = optim.Adam(self.model.parameters())
 
     def compute_returns(self,next_value, rewards, masks):

codes/A2C/model.py

@@ -13,19 +13,19 @@ import torch.nn as nn
 import torch.nn.functional as F
 from torch.distributions import Categorical
 class ActorCritic(nn.Module):
-    def __init__(self, num_inputs, num_outputs, hidden_size, std=0.0):
+    def __init__(self, n_states, n_actions, hidden_dim):
         super(ActorCritic, self).__init__()
         
         self.critic = nn.Sequential(
-            nn.Linear(num_inputs, hidden_size),
+            nn.Linear(n_states, hidden_dim),
             nn.ReLU(),
-            nn.Linear(hidden_size, 1)
+            nn.Linear(hidden_dim, 1)
         )
         
         self.actor = nn.Sequential(
-            nn.Linear(num_inputs, hidden_size),
+            nn.Linear(n_states, hidden_dim),
             nn.ReLU(),
-            nn.Linear(hidden_size, num_outputs),
+            nn.Linear(hidden_dim, n_actions),
             nn.Softmax(dim=1),
         )
         

codes/A2C/task0_train.py

@@ -1,8 +1,7 @@
 import sys,os
-curr_path = os.path.dirname(__file__)
-parent_path = os.path.dirname(curr_path)
-sys.path.append(parent_path)  # add current terminal path to sys.path
-
+curr_path = os.path.dirname(os.path.abspath(__file__)) # 当前文件所在绝对路径
+parent_path = os.path.dirname(curr_path) # 父路径
+sys.path.append(parent_path) # 添加路径到系统路径sys.path
 
 import gym
 import numpy as np
@@ -17,17 +16,28 @@ from common.plot import plot_rewards
 curr_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") # obtain current time
 class A2CConfig:
     def __init__(self) -> None:
-        self.algo='A2C'
-        self.env= 'CartPole-v0'
-        self.result_path = curr_path+"/outputs/" +self.env+'/'+curr_time+'/results/'  # path to save results
-        self.model_path = curr_path+"/outputs/" +self.env+'/'+curr_time+'/models/'  # path to save models
-        self.n_envs = 8
-        self.gamma = 0.99
-        self.hidden_size = 256
+        self.algo='A2C' # 算法名称
+        self.env_name= 'CartPole-v0' # 环境名称
+        self.n_envs = 8 # 异步的环境数目
+        self.gamma = 0.99 # 强化学习中的折扣因子
+        self.hidden_dim = 256
         self.lr = 1e-3 # learning rate
         self.max_frames = 30000
         self.n_steps = 5
         self.device  = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+class PlotConfig:
+    def __init__(self) -> None:
+        self.algo = "DQN"  # 算法名称
+        self.env_name = 'CartPole-v0' # 环境名称
+        self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")  # 检测GPU
+
+        self.result_path = curr_path+"/outputs/" + self.env_name + \
+            '/'+curr_time+'/results/'  # 保存结果的路径
+        self.model_path = curr_path+"/outputs/" + self.env_name + \
+            '/'+curr_time+'/models/'  # 保存模型的路径
+        self.save = True # 是否保存图片
+        
+
 def make_envs(env_name):
     def _thunk():
         env = gym.make(env_name)
@@ -57,11 +67,11 @@ def compute_returns(next_value, rewards, masks, gamma=0.99):
 
 
 def train(cfg,envs):
-    env = gym.make(cfg.env) # a single env
+    env = gym.make(cfg.env_name) # a single env
     env.seed(10)
     state_dim  = envs.observation_space.shape[0]
     action_dim = envs.action_space.n
-    model = ActorCritic(state_dim, action_dim, cfg.hidden_size).to(cfg.device)
+    model = ActorCritic(state_dim, action_dim, cfg.hidden_dim).to(cfg.device)
     optimizer = optim.Adam(model.parameters())
     frame_idx    = 0
     test_rewards = []
@@ -112,9 +122,11 @@ def train(cfg,envs):
     return test_rewards, test_ma_rewards
 if __name__ == "__main__":
     cfg = A2CConfig()
-    envs = [make_envs(cfg.env) for i in range(cfg.n_envs)]
-    envs = SubprocVecEnv(envs) # 8 env
+    plot_cfg = PlotConfig()
+    envs = [make_envs(cfg.env_name) for i in range(cfg.n_envs)]
+    envs = SubprocVecEnv(envs) 
+    # 训练
     rewards,ma_rewards = train(cfg,envs)
-    make_dir(cfg.result_path,cfg.model_path)
-    save_results(rewards,ma_rewards,tag='train',path=cfg.result_path)
-    plot_rewards(rewards,ma_rewards,tag="train",env=cfg.env,algo = cfg.algo,path=cfg.result_path)
+    make_dir(plot_cfg.result_path,plot_cfg.model_path)
+    save_results(rewards, ma_rewards, tag='train', path=plot_cfg.result_path) # 保存结果
+    plot_rewards(rewards, ma_rewards, plot_cfg, tag="train") # 画出结果

codes/DDPG/task0_train.py

@@ -12,7 +12,7 @@ LastEditTime: 2021-09-16 01:31:33
 import sys,os
 curr_path = os.path.dirname(os.path.abspath(__file__)) # 当前文件所在绝对路径
 parent_path = os.path.dirname(curr_path) # 父路径
-sys.path.append(parent_path) # 添加父路径到系统路径sys.path
+sys.path.append(parent_path) # 添加路径到系统路径sys.path
 
 import datetime
 import gym
@@ -21,44 +21,51 @@ import torch
 from DDPG.env import NormalizedActions, OUNoise
 from DDPG.agent import DDPG
 from common.utils import save_results,make_dir
-from common.plot import plot_rewards, plot_rewards_cn
+from common.plot import plot_rewards
 
 curr_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")  # 获取当前时间
 
 class DDPGConfig:
     def __init__(self):
         self.algo = 'DDPG' # 算法名称
-        self.env = 'Pendulum-v0' # 环境名称
-        self.result_path = curr_path+"/outputs/" + self.env + \
-            '/'+curr_time+'/results/'  # 保存结果的路径
-        self.model_path = curr_path+"/outputs/" + self.env + \
-            '/'+curr_time+'/models/'  # 保存模型的路径
+        self.env_name = 'Pendulum-v0' # 环境名称
+        self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # 检测GPU
         self.train_eps = 300 # 训练的回合数
         self.eval_eps = 50 # 测试的回合数
         self.gamma = 0.99 # 折扣因子
         self.critic_lr = 1e-3 # 评论家网络的学习率
         self.actor_lr = 1e-4 # 演员网络的学习率
-        self.memory_capacity = 8000 
-        self.batch_size = 128
-        self.target_update = 2
-        self.hidden_dim = 256
+        self.memory_capacity = 8000 # 经验回放的容量
+        self.batch_size = 128 # mini-batch SGD中的批量大小
+        self.target_update = 2 # 目标网络的更新频率
+        self.hidden_dim = 256 # 网络隐藏层维度
         self.soft_tau = 1e-2 # 软更新参数
-        self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+        
+class PlotConfig:
+    def __init__(self) -> None:
+        self.algo = "DQN"  # 算法名称
+        self.env_name = 'CartPole-v0' # 环境名称
+        self.result_path = curr_path+"/outputs/" + self.env_name + \
+            '/'+curr_time+'/results/'  # 保存结果的路径
+        self.model_path = curr_path+"/outputs/" + self.env_name + \
+            '/'+curr_time+'/models/'  # 保存模型的路径
+        self.save = True # 是否保存图片
+        self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")  # 检测GPU
 
 def env_agent_config(cfg,seed=1):
-    env = NormalizedActions(gym.make(cfg.env)) 
+    env = NormalizedActions(gym.make(cfg.env_name)) # 装饰action噪声
     env.seed(seed) # 随机种子
-    state_dim = env.observation_space.shape[0]
-    action_dim = env.action_space.shape[0]
-    agent = DDPG(state_dim,action_dim,cfg)
+    n_states = env.observation_space.shape[0]
+    n_actions = env.action_space.shape[0]
+    agent = DDPG(n_states,n_actions,cfg)
     return env,agent
 
 def train(cfg, env, agent):
     print('开始训练！')
-    print(f'环境：{cfg.env}，算法：{cfg.algo}，设备：{cfg.device}')
+    print(f'环境：{cfg.env_name}，算法：{cfg.algo}，设备：{cfg.device}')
     ou_noise = OUNoise(env.action_space)  # 动作噪声
-    rewards = [] # 记录奖励
-    ma_rewards = []  # 记录滑动平均奖励
+    rewards = [] # 记录所有回合的奖励
+    ma_rewards = []  # 记录所有回合的滑动平均奖励
     for i_ep in range(cfg.train_eps):
         state = env.reset()
         ou_noise.reset()
@@ -86,9 +93,9 @@ def train(cfg, env, agent):
 
 def eval(cfg, env, agent):
     print('开始测试！')
-    print(f'环境：{cfg.env}, 算法：{cfg.algo}, 设备：{cfg.device}')
-    rewards = [] # 记录奖励
-    ma_rewards = []  # 记录滑动平均奖励
+    print(f'环境：{cfg.env_name}, 算法：{cfg.algo}, 设备：{cfg.device}')
+    rewards = [] # 记录所有回合的奖励
+    ma_rewards = []  # 记录所有回合的滑动平均奖励
     for i_ep in range(cfg.eval_eps):
         state = env.reset() 
         done = False
@@ -112,17 +119,18 @@ def eval(cfg, env, agent):
 
 if __name__ == "__main__":
     cfg = DDPGConfig()
+    plot_cfg = PlotConfig()
     # 训练
     env,agent = env_agent_config(cfg,seed=1)
     rewards, ma_rewards = train(cfg, env, agent)
-    make_dir(cfg.result_path, cfg.model_path)
-    agent.save(path=cfg.model_path)
-    save_results(rewards, ma_rewards, tag='train', path=cfg.result_path)
-    plot_rewards_cn(rewards, ma_rewards, tag="train", env = cfg.env, algo=cfg.algo, path=cfg.result_path)
+    make_dir(plot_cfg.result_path, plot_cfg.model_path)
+    agent.save(path=plot_cfg.model_path)
+    save_results(rewards, ma_rewards, tag='train', path=plot_cfg.result_path)
+    plot_rewards(rewards, ma_rewards, plot_cfg, tag="train")
     # 测试
     env,agent = env_agent_config(cfg,seed=10)
-    agent.load(path=cfg.model_path)
-    rewards,ma_rewards = eval(cfg,env,agent)
+    agent.load(path=plot_cfg.model_path)
+    rewards,ma_rewards = eval(plot_cfg,env,agent)
     save_results(rewards,ma_rewards,tag = 'eval',path = cfg.result_path)
-    plot_rewards_cn(rewards,ma_rewards,tag = "eval",env = cfg.env,algo = cfg.algo,path=cfg.result_path)
+    plot_rewards(rewards,ma_rewards,plot_cfg,tag = "eval")
     

codes/DQN-series/README.md

@@ -1,3 +0,0 @@
-
-本目录下汇总了基础的DQN及其变种或升级，如下
-

codes/DQN/task0_train.py

@@ -12,7 +12,7 @@ LastEditTime: 2021-09-15 15:34:13
 import sys,os
 curr_path = os.path.dirname(os.path.abspath(__file__)) # 当前文件所在绝对路径
 parent_path = os.path.dirname(curr_path) # 父路径
-sys.path.append(parent_path) # 添加父路径到系统路径sys.path
+sys.path.append(parent_path) # 添加路径到系统路径
 
 import gym
 import torch
@@ -26,9 +26,11 @@ curr_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")  # 获取当前时
 class DQNConfig:
     def __init__(self):
         self.algo = "DQN"  # 算法名称
-        self.env = 'CartPole-v0' # 环境名称
+        self.env_name = 'CartPole-v0' # 环境名称
+        self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")  # 检测GPU
         self.train_eps = 200 # 训练的回合数
         self.eval_eps = 30 # 测试的回合数
+        # 超参数
         self.gamma = 0.95 # 强化学习中的折扣因子
         self.epsilon_start = 0.90 # e-greedy策略中初始epsilon
         self.epsilon_end = 0.01 # e-greedy策略中的终止epsilon
@@ -37,23 +39,22 @@ class DQNConfig:
         self.memory_capacity = 100000  # 经验回放的容量
         self.batch_size = 64 # mini-batch SGD中的批量大小
         self.target_update = 4 # 目标网络的更新频率
-        self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")  # 检测GPU
         self.hidden_dim = 256  # 网络隐藏层
 class PlotConfig:
     def __init__(self) -> None:
         self.algo = "DQN"  # 算法名称
-        self.env = 'CartPole-v0' # 环境名称
-        self.result_path = curr_path+"/outputs/" + self.env + \
+        self.env_name = 'CartPole-v0' # 环境名称
+        self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")  # 检测GPU
+        self.result_path = curr_path+"/outputs/" + self.env_name + \
             '/'+curr_time+'/results/'  # 保存结果的路径
-        self.model_path = curr_path+"/outputs/" + self.env + \
+        self.model_path = curr_path+"/outputs/" + self.env_name + \
             '/'+curr_time+'/models/'  # 保存模型的路径
         self.save = True # 是否保存图片
-        self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")  # 检测GPU
-
+        
 def env_agent_config(cfg,seed=1):
     ''' 创建环境和智能体
     '''
-    env = gym.make(cfg.env)  # 创建环境
+    env = gym.make(cfg.env_name)  # 创建环境
     env.seed(seed) # 设置随机种子
     n_states = env.observation_space.shape[0] # 状态数
     n_actions = env.action_space.n # 动作数
@@ -64,7 +65,7 @@ def train(cfg, env, agent):
     ''' 训练
     '''
     print('开始训练!')
-    print(f'环境：{cfg.env}, 算法：{cfg.algo}, 设备：{cfg.device}')
+    print(f'环境：{cfg.env_name}, 算法：{cfg.algo}, 设备：{cfg.device}')
     rewards = [] # 记录所有回合的奖励
     ma_rewards = []  # 记录所有回合的滑动平均奖励
     for i_ep in range(cfg.train_eps):
@@ -93,7 +94,7 @@ def train(cfg, env, agent):
 
 def eval(cfg,env,agent):
     print('开始测试!')
-    print(f'环境：{cfg.env}, 算法：{cfg.algo}, 设备：{cfg.device}')
+    print(f'环境：{cfg.env_name}, 算法：{cfg.algo}, 设备：{cfg.device}')
     # 由于测试不需要使用epsilon-greedy策略，所以相应的值设置为0
     cfg.epsilon_start = 0.0 # e-greedy策略中初始epsilon
     cfg.epsilon_end = 0.0 # e-greedy策略中的终止epsilon

codes/PPO/agent.py

@@ -29,13 +29,16 @@ class PPO:
         self.memory = PPOMemory(cfg.batch_size)
         self.loss = 0
 
-    def choose_action(self, observation):
-        state = torch.tensor([observation], dtype=torch.float).to(self.device)
+    def choose_action(self, state,continuous=False):
+        state = torch.tensor([state], dtype=torch.float).to(self.device)
         dist = self.actor(state)
         value = self.critic(state)
         action = dist.sample()
         probs = torch.squeeze(dist.log_prob(action)).item()
-        action = torch.squeeze(action).item()
+        if continuous:
+            action = torch.tanh(action)
+        else:
+            action = torch.squeeze(action).item()
         value = torch.squeeze(value).item()
         return action, probs, value
 

codes/PPO/task0_train.py

@@ -10,14 +10,13 @@ Discription:
 Environment: 
 '''
 import sys,os
-curr_path = os.path.dirname(__file__)
-parent_path=os.path.dirname(curr_path) 
-sys.path.append(parent_path) # add current terminal path to sys.path
+curr_path = os.path.dirname(os.path.abspath(__file__)) # 当前文件所在绝对路径
+parent_path = os.path.dirname(curr_path) # 父路径
+sys.path.append(parent_path) # 添加路径到系统路径
 
 import gym
 import torch
 import datetime
-import tqdm
 from PPO.agent import PPO
 from common.plot import plot_rewards
 from common.utils import save_results,make_dir
@@ -26,12 +25,12 @@ curr_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") # obtain current t
 
 class PPOConfig:
     def __init__(self) -> None:
-        self.env = 'CartPole-v0'
-        self.algo = 'PPO'
-        self.result_path = curr_path+"/results/" +self.env+'/'+curr_time+'/results/'  # path to save results
-        self.model_path = curr_path+"/results/" +self.env+'/'+curr_time+'/models/'  # path to save models
-        self.train_eps = 200 # max training episodes
-        self.eval_eps = 50
+        self.algo = "DQN"  # 算法名称
+        self.env_name = 'CartPole-v0' # 环境名称
+        self.continuous = False # 环境是否为连续动作
+        self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")  # 检测GPU
+        self.train_eps = 200 # 训练的回合数
+        self.eval_eps = 20 # 测试的回合数
         self.batch_size = 5
         self.gamma=0.99
         self.n_epochs = 4
@@ -41,10 +40,20 @@ class PPOConfig:
         self.policy_clip=0.2
         self.hidden_dim = 256
         self.update_fre = 20 # frequency of agent update
-        self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # check gpu
 
+class PlotConfig:
+    def __init__(self) -> None:
+        self.algo = "DQN"  # 算法名称
+        self.env_name = 'CartPole-v0' # 环境名称
+        self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")  # 检测GPU
+        self.result_path = curr_path+"/outputs/" + self.env_name + \
+            '/'+curr_time+'/results/'  # 保存结果的路径
+        self.model_path = curr_path+"/outputs/" + self.env_name + \
+            '/'+curr_time+'/models/'  # 保存模型的路径
+        self.save = True # 是否保存图片
+        
 def env_agent_config(cfg,seed=1):
-    env = gym.make(cfg.env)  
+    env = gym.make(cfg.env_name)  
     env.seed(seed)
     state_dim = env.observation_space.shape[0]
     action_dim = env.action_space.n
@@ -53,44 +62,44 @@ def env_agent_config(cfg,seed=1):
             
 def train(cfg,env,agent):
     print('开始训练！')
-    print(f'Env:{cfg.env}, Algorithm:{cfg.algo}, Device:{cfg.device}')
-    rewards= []
-    ma_rewards = [] # moving average rewards
-    running_steps = 0
+    print(f'环境：{cfg.env_name}, 算法：{cfg.algo}, 设备：{cfg.device}')
+    rewards = [] # 记录所有回合的奖励
+    ma_rewards = []  # 记录所有回合的滑动平均奖励
+    steps = 0
     for i_ep in range(cfg.train_eps):
         state = env.reset()
         done = False
         ep_reward = 0
         while not done:
-            action, prob, val = agent.choose_action(state)
+            action, prob, val = agent.choose_action(state,continuous=cfg.continuous)
             state_, reward, done, _ = env.step(action)
-            running_steps += 1
+            steps += 1
             ep_reward += reward
             agent.memory.push(state, action, prob, val, reward, done)
-            if running_steps % cfg.update_fre == 0:
+            if steps % cfg.update_fre == 0:
                 agent.update()
             state = state_
         rewards.append(ep_reward)
         if ma_rewards:
-            ma_rewards.append(
-                0.9*ma_rewards[-1]+0.1*ep_reward)
+            ma_rewards.append(0.9*ma_rewards[-1]+0.1*ep_reward)
         else:
             ma_rewards.append(ep_reward)
-        print(f"回合：{i_ep+1}/{cfg.train_eps}，奖励：{ep_reward:.2f}")
-    print('Complete training！')
+        if (i_ep+1)%10 == 0: 
+            print(f"回合：{i_ep+1}/{cfg.train_eps}，奖励：{ep_reward:.2f}")
+    print('完成训练！')
     return rewards,ma_rewards
 
 def eval(cfg,env,agent):
-    print('Start to eval !')
-    print(f'Env:{cfg.env}, Algorithm:{cfg.algo}, Device:{cfg.device}')
-    rewards= []
-    ma_rewards = [] # moving average rewards
+    print('开始测试!')
+    print(f'环境：{cfg.env_name}, 算法：{cfg.algo}, 设备：{cfg.device}')
+    rewards = [] # 记录所有回合的奖励
+    ma_rewards = []  # 记录所有回合的滑动平均奖励
     for i_ep in range(cfg.eval_eps):
         state = env.reset()
         done = False
         ep_reward = 0
         while not done:
-            action, prob, val = agent.choose_action(state)
+            action, prob, val = agent.choose_action(state,cfg.continuous)
             state_, reward, done, _ = env.step(action)
             ep_reward += reward
             state = state_
@@ -100,23 +109,23 @@ def eval(cfg,env,agent):
                 0.9*ma_rewards[-1]+0.1*ep_reward)
         else:
             ma_rewards.append(ep_reward)
-        print(f"Episode:{i_ep+1}/{cfg.eval_eps}, Reward:{ep_reward:.3f}")
-    print('Complete evaling！')
+        print('回合：{}/{}, 奖励：{}'.format(i_ep+1, cfg.eval_eps, ep_reward))
+    print('完成训练！')
     return rewards,ma_rewards
     
 if __name__ == '__main__':
     cfg  = PPOConfig()
-    # train
+    plot_cfg = PlotConfig()
+    # 训练
     env,agent = env_agent_config(cfg,seed=1)
     rewards, ma_rewards = train(cfg, env, agent)
-    make_dir(cfg.result_path, cfg.model_path)
-    agent.save(path=cfg.model_path)
-    save_results(rewards, ma_rewards, tag='train', path=cfg.result_path)
-    plot_rewards(rewards, ma_rewards, tag="train",
-                 algo=cfg.algo, path=cfg.result_path)
-    # eval
+    make_dir(plot_cfg.result_path, plot_cfg.model_path) # 创建保存结果和模型路径的文件夹
+    agent.save(path=plot_cfg.model_path)
+    save_results(rewards, ma_rewards, tag='train', path=plot_cfg.result_path)
+    plot_rewards(rewards, ma_rewards, plot_cfg, tag="train")
+    # 测试
     env,agent = env_agent_config(cfg,seed=10)
-    agent.load(path=cfg.model_path)
+    agent.load(path=plot_cfg.model_path)
     rewards,ma_rewards = eval(cfg,env,agent)
-    save_results(rewards,ma_rewards,tag='eval',path=cfg.result_path)
-    plot_rewards(rewards,ma_rewards,tag="eval",env=cfg.env,algo = cfg.algo,path=cfg.result_path)
+    save_results(rewards,ma_rewards,tag='eval',path=plot_cfg.result_path)
+    plot_rewards(rewards,ma_rewards,plot_cfg,tag="eval")

codes/PPO/task1_train.py

@@ -0,0 +1,132 @@
+#!/usr/bin/env python
+# coding=utf-8
+'''
+Author: John
+Email: johnjim0816@gmail.com
+Date: 2021-03-22 16:18:10
+LastEditor: John
+LastEditTime: 2021-09-26 22:05:00
+Discription: 
+Environment: 
+'''
+import sys,os
+curr_path = os.path.dirname(os.path.abspath(__file__)) # 当前文件所在绝对路径
+parent_path = os.path.dirname(curr_path) # 父路径
+sys.path.append(parent_path) # 添加路径到系统路径
+
+import gym
+import torch
+import datetime
+from PPO.agent import PPO
+from common.plot import plot_rewards
+from common.utils import save_results,make_dir
+
+curr_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") # obtain current time
+
+class PPOConfig:
+    def __init__(self) -> None:
+        self.algo = "PPO"  # 算法名称
+        self.env_name = 'Pendulum-v1' # 环境名称
+        self.continuous = True # 环境是否为连续动作
+        self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")  # 检测GPU
+        self.train_eps = 200 # 训练的回合数
+        self.eval_eps = 20 # 测试的回合数
+        self.batch_size = 5
+        self.gamma=0.99
+        self.n_epochs = 4
+        self.actor_lr = 0.0003
+        self.critic_lr = 0.0003
+        self.gae_lambda=0.95
+        self.policy_clip=0.2
+        self.hidden_dim = 256
+        self.update_fre = 20 # frequency of agent update
+
+class PlotConfig:
+    def __init__(self) -> None:
+        self.algo = "PPO"  # 算法名称
+        self.env_name = 'Pendulum-v1' # 环境名称
+        self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")  # 检测GPU
+        self.result_path = curr_path+"/outputs/" + self.env_name + \
+            '/'+curr_time+'/results/'  # 保存结果的路径
+        self.model_path = curr_path+"/outputs/" + self.env_name + \
+            '/'+curr_time+'/models/'  # 保存模型的路径
+        self.save = True # 是否保存图片
+        
+def env_agent_config(cfg,seed=1):
+    env = gym.make(cfg.env_name)  
+    env.seed(seed)
+    state_dim = env.observation_space.shape[0]
+    action_dim = env.action_space.shape[0]
+    agent = PPO(state_dim,action_dim,cfg)
+    return env,agent
+            
+def train(cfg,env,agent):
+    print('开始训练！')
+    print(f'环境：{cfg.env_name}, 算法：{cfg.algo}, 设备：{cfg.device}')
+    rewards = [] # 记录所有回合的奖励
+    ma_rewards = []  # 记录所有回合的滑动平均奖励
+    steps = 0
+    for i_ep in range(cfg.train_eps):
+        state = env.reset()
+        done = False
+        ep_reward = 0
+        while not done:
+            action, prob, val = agent.choose_action(state,continuous=cfg.continuous)
+            print(action)
+            state_, reward, done, _ = env.step(action)
+            steps += 1
+            ep_reward += reward
+            agent.memory.push(state, action, prob, val, reward, done)
+            if steps % cfg.update_fre == 0:
+                agent.update()
+            state = state_
+        rewards.append(ep_reward)
+        if ma_rewards:
+            ma_rewards.append(0.9*ma_rewards[-1]+0.1*ep_reward)
+        else:
+            ma_rewards.append(ep_reward)
+        if (i_ep+1)%10 == 0: 
+            print(f"回合：{i_ep+1}/{cfg.train_eps}，奖励：{ep_reward:.2f}")
+    print('完成训练！')
+    return rewards,ma_rewards
+
+def eval(cfg,env,agent):
+    print('开始测试!')
+    print(f'环境：{cfg.env_name}, 算法：{cfg.algo}, 设备：{cfg.device}')
+    rewards = [] # 记录所有回合的奖励
+    ma_rewards = []  # 记录所有回合的滑动平均奖励
+    for i_ep in range(cfg.eval_eps):
+        state = env.reset()
+        done = False
+        ep_reward = 0
+        while not done:
+            action, prob, val = agent.choose_action(state,continuous=False)
+            state_, reward, done, _ = env.step(action)
+            ep_reward += reward
+            state = state_
+        rewards.append(ep_reward)
+        if ma_rewards:
+            ma_rewards.append(
+                0.9*ma_rewards[-1]+0.1*ep_reward)
+        else:
+            ma_rewards.append(ep_reward)
+        print('回合：{}/{}, 奖励：{}'.format(i_ep+1, cfg.eval_eps, ep_reward))
+    print('完成训练！')
+    return rewards,ma_rewards
+    
+if __name__ == '__main__':
+    cfg  = PPOConfig()
+    plot_cfg = PlotConfig()
+    # 训练
+    env,agent = env_agent_config(cfg,seed=1)
+    rewards, ma_rewards = train(cfg, env, agent)
+    make_dir(plot_cfg.result_path, plot_cfg.model_path) # 创建保存结果和模型路径的文件夹
+    agent.save(path=plot_cfg.model_path)
+    save_results(rewards, ma_rewards, tag='train', path=plot_cfg.result_path)
+    plot_rewards(rewards, ma_rewards, plot_cfg, tag="train")
+    # 测试
+    env,agent = env_agent_config(cfg,seed=10)
+    agent.load(path=plot_cfg.model_path)
+    rewards,ma_rewards = eval(cfg,env,agent)
+    save_results(rewards,ma_rewards,tag='eval',path=plot_cfg.result_path)
+    plot_rewards(rewards,ma_rewards,plot_cfg,tag="eval")

codes/SAC/task0_train.ipynb

@@ -1,30 +1,4 @@
 {
- "metadata": {
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.7.10"
-  },
-  "orig_nbformat": 2,
-  "kernelspec": {
-   "name": "python3710jvsc74a57bd0fd81e6a9e450d5c245c1a0b5da0b03c89c450f614a13afa2acb1654375922756",
-   "display_name": "Python 3.7.10 64-bit ('mujoco': conda)"
-  },
-  "metadata": {
-   "interpreter": {
-    "hash": "fd81e6a9e450d5c245c1a0b5da0b03c89c450f614a13afa2acb1654375922756"
-   }
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 2,
  "cells": [
   {
    "cell_type": "code",
@@ -170,9 +144,29 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 7,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "ename": "DeprecatedEnv",
+     "evalue": "Env Pendulum-v0 not found (valid versions include ['Pendulum-v1'])",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
+      "\u001b[0;32m~/anaconda3/envs/py37/lib/python3.7/site-packages/gym/envs/registration.py\u001b[0m in \u001b[0;36mspec\u001b[0;34m(self, path)\u001b[0m\n\u001b[1;32m    157\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 158\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0menv_specs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mid\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    159\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;31mKeyError\u001b[0m: 'Pendulum-v0'",
+      "\nDuring handling of the above exception, another exception occurred:\n",
+      "\u001b[0;31mDeprecatedEnv\u001b[0m                             Traceback (most recent call last)",
+      "\u001b[0;32m<ipython-input-7-91b1038013e4>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m     \u001b[0;31m# train\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m     \u001b[0menv\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0magent\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0menv_agent_config\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcfg\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mseed\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      6\u001b[0m     \u001b[0mrewards\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mma_rewards\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcfg\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0menv\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0magent\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m     \u001b[0mmake_dir\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcfg\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresult_path\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcfg\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel_path\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m<ipython-input-4-040773221550>\u001b[0m in \u001b[0;36menv_agent_config\u001b[0;34m(cfg, seed)\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0menv_agent_config\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcfg\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mseed\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m     \u001b[0menv\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mNormalizedActions\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgym\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmake\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Pendulum-v0\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m     \u001b[0menv\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mseed\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mseed\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m     \u001b[0maction_dim\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0menv\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maction_space\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m     \u001b[0mstate_dim\u001b[0m  \u001b[0;34m=\u001b[0m \u001b[0menv\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mobservation_space\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m~/anaconda3/envs/py37/lib/python3.7/site-packages/gym/envs/registration.py\u001b[0m in \u001b[0;36mmake\u001b[0;34m(id, **kwargs)\u001b[0m\n\u001b[1;32m    233\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    234\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mmake\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mid\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 235\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mregistry\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmake\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mid\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    236\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    237\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m~/anaconda3/envs/py37/lib/python3.7/site-packages/gym/envs/registration.py\u001b[0m in \u001b[0;36mmake\u001b[0;34m(self, path, **kwargs)\u001b[0m\n\u001b[1;32m    126\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    127\u001b[0m             \u001b[0mlogger\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minfo\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Making new env: %s\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpath\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 128\u001b[0;31m         \u001b[0mspec\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mspec\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    129\u001b[0m         \u001b[0menv\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mspec\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmake\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    130\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0menv\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m~/anaconda3/envs/py37/lib/python3.7/site-packages/gym/envs/registration.py\u001b[0m in \u001b[0;36mspec\u001b[0;34m(self, path)\u001b[0m\n\u001b[1;32m    185\u001b[0m                 raise error.DeprecatedEnv(\n\u001b[1;32m    186\u001b[0m                     \"Env {} not found (valid versions include {})\".format(\n\u001b[0;32m--> 187\u001b[0;31m                         \u001b[0mid\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmatching_envs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    188\u001b[0m                     )\n\u001b[1;32m    189\u001b[0m                 )\n",
+      "\u001b[0;31mDeprecatedEnv\u001b[0m: Env Pendulum-v0 not found (valid versions include ['Pendulum-v1'])"
+     ]
+    }
+   ],
    "source": [
     "if __name__ == \"__main__\":\n",
     "    cfg=SACConfig()\n",
@@ -193,5 +187,35 @@
     "    plot_rewards(rewards,ma_rewards,tag=\"eval\",env=cfg.env,algo = cfg.algo,path=cfg.result_path)\n"
    ]
   }
- ]
-}
+ ],
+ "metadata": {
+  "interpreter": {
+   "hash": "fe38df673a99c62a9fea33a7aceda74c9b65b12ee9d076c5851d98b692a4989a"
+  },
+  "kernelspec": {
+   "display_name": "Python 3.7.10 64-bit ('mujoco': conda)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.7.10"
+  },
+  "metadata": {
+   "interpreter": {
+    "hash": "fd81e6a9e450d5c245c1a0b5da0b03c89c450f614a13afa2acb1654375922756"
+   }
+  },
+  "orig_nbformat": 2
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}