update

codes/DQN/dqn.py

@@ -5,7 +5,7 @@
 @Email: johnjim0816@gmail.com
 @Date: 2020-06-12 00:50:49
 @LastEditor: John
-LastEditTime: 2022-03-02 11:05:11
+LastEditTime: 2022-07-13 00:08:18
 @Discription: 
 @Environment: python 3.7.7
 '''
@@ -20,7 +20,22 @@ import random
 import math
 import numpy as np
 
-
+class MLP(nn.Module):
+    def __init__(self, n_states,n_actions,hidden_dim=128):
+        """ 初始化q网络，为全连接网络
+            n_states: 输入的特征数即环境的状态维度
+            n_actions: 输出的动作维度
+        """
+        super(MLP, self).__init__()
+        self.fc1 = nn.Linear(n_states, hidden_dim) # 输入层
+        self.fc2 = nn.Linear(hidden_dim,hidden_dim) # 隐藏层
+        self.fc3 = nn.Linear(hidden_dim, n_actions) # 输出层
+        
+    def forward(self, x):
+        # 各层对应的激活函数
+        x = F.relu(self.fc1(x)) 
+        x = F.relu(self.fc2(x))
+        return self.fc3(x)
 
 class ReplayBuffer:
     def __init__(self, capacity):
@@ -47,7 +62,7 @@ class ReplayBuffer:
         return len(self.buffer)
 
 class DQN:
-    def __init__(self, n_actions,model,cfg):
+    def __init__(self, n_states,n_actions,cfg):
 
         self.n_actions = n_actions  # 总的动作个数
         self.device = cfg.device  # 设备，cpu或gpu等
@@ -58,8 +73,8 @@ class DQN:
             (cfg.epsilon_start - cfg.epsilon_end) * \
             math.exp(-1. * frame_idx / cfg.epsilon_decay)
         self.batch_size = cfg.batch_size
-        self.policy_net = model.to(self.device)
-        self.target_net = model.to(self.device)
+        self.policy_net = MLP(n_states,n_actions).to(self.device)
+        self.target_net = MLP(n_states,n_actions).to(self.device)
         for target_param, param in zip(self.target_net.parameters(),self.policy_net.parameters()): # 复制参数到目标网路targe_net
             target_param.data.copy_(param.data)
         self.optimizer = optim.Adam(self.policy_net.parameters(), lr=cfg.lr) # 优化器

codes/DQN/outputs/CartPole-v0/20220713-211653/results/params.txt

@@ -0,0 +1,19 @@
+------------------ start ------------------
+algo_name : DQN
+env_name : CartPole-v0
+train_eps : 200
+test_eps : 20
+gamma : 0.95
+epsilon_start : 0.95
+epsilon_end : 0.01
+epsilon_decay : 500
+lr : 0.0001
+memory_capacity : 100000
+batch_size : 64
+target_update : 4
+hidden_dim : 256
+result_path : C:\Users\24438\Desktop\rl-tutorials\codes\DQN/outputs/CartPole-v0/20220713-211653/results/
+model_path : C:\Users\24438\Desktop\rl-tutorials\codes\DQN/outputs/CartPole-v0/20220713-211653/models/
+save_fig : True
+device : cuda
+------------------- end -------------------

codes/DQN/task0.py

@@ -1,5 +1,7 @@
+from lib2to3.pytree import type_repr
 import sys
 import os
+from parso import parse
 import torch.nn as nn
 import torch.nn.functional as F
 curr_path = os.path.dirname(os.path.abspath(__file__))  # 当前文件所在绝对路径
@@ -10,86 +12,58 @@ import gym
 import torch
 import datetime
 import numpy as np
+import argparse
 from common.utils import save_results_1, make_dir
-from common.utils import plot_rewards
+from common.utils import plot_rewards,save_args
 from dqn import DQN
 
-curr_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")  # 获取当前时间
+def get_args():
+    """ Hyperparameters
+    """
+    curr_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")  # Obtain current time
+    parser = argparse.ArgumentParser(description="hyperparameters")      
+    parser.add_argument('--algo_name',default='DQN',type=str,help="name of algorithm")
+    parser.add_argument('--env_name',default='CartPole-v0',type=str,help="name of environment")
+    parser.add_argument('--train_eps',default=200,type=int,help="episodes of training")
+    parser.add_argument('--test_eps',default=20,type=int,help="episodes of testing")
+    parser.add_argument('--gamma',default=0.95,type=float,help="discounted factor")
+    parser.add_argument('--epsilon_start',default=0.95,type=float,help="initial value of epsilon")
+    parser.add_argument('--epsilon_end',default=0.01,type=float,help="final value of epsilon")
+    parser.add_argument('--epsilon_decay',default=500,type=int,help="decay rate of epsilon")
+    parser.add_argument('--lr',default=0.0001,type=float,help="learning rate")
+    parser.add_argument('--memory_capacity',default=100000,type=int,help="memory capacity")
+    parser.add_argument('--batch_size',default=64,type=int)
+    parser.add_argument('--target_update',default=4,type=int)
+    parser.add_argument('--hidden_dim',default=256,type=int)
+    parser.add_argument('--result_path',default=curr_path + "/outputs/" + parser.parse_args().env_name + \
+            '/' + curr_time + '/results/' )
+    parser.add_argument('--model_path',default=curr_path + "/outputs/" + parser.parse_args().env_name + \
+            '/' + curr_time + '/models/' ) # path to save models
+    parser.add_argument('--save_fig',default=True,type=bool,help="if save figure or not")           
+    args = parser.parse_args()    
+    args.device = torch.device(
+            "cuda" if torch.cuda.is_available() else "cpu")  # check GPU                        
+    return args
 
-class MLP(nn.Module):
-    def __init__(self, n_states,n_actions,hidden_dim=128):
-        """ 初始化q网络，为全连接网络
-            n_states: 输入的特征数即环境的状态维度
-            n_actions: 输出的动作维度
-        """
-        super(MLP, self).__init__()
-        self.fc1 = nn.Linear(n_states, hidden_dim) # 输入层
-        self.fc2 = nn.Linear(hidden_dim,hidden_dim) # 隐藏层
-        self.fc3 = nn.Linear(hidden_dim, n_actions) # 输出层
-        
-    def forward(self, x):
-        # 各层对应的激活函数
-        x = F.relu(self.fc1(x)) 
-        x = F.relu(self.fc2(x))
-        return self.fc3(x)
-
-class Config:
-    '''超参数
-    '''
-
-    def __init__(self):
-        ############################### hyperparameters ################################
-        self.algo_name = 'DQN'  # algorithm name
-        self.env_name = 'CartPole-v0'  # environment name
-        self.device = torch.device(
-            "cuda" if torch.cuda.is_available() else "cpu")  # check GPU
-        self.seed = 10 # 随机种子，置0则不设置随机种子
-        self.train_eps = 200  # 训练的回合数
-        self.test_eps = 20  # 测试的回合数
-        ################################################################################
-        
-        ################################## 算法超参数 ###################################
-        self.gamma = 0.95  # 强化学习中的折扣因子
-        self.epsilon_start = 0.90  # e-greedy策略中初始epsilon
-        self.epsilon_end = 0.01  # e-greedy策略中的终止epsilon
-        self.epsilon_decay = 500  # e-greedy策略中epsilon的衰减率
-        self.lr = 0.0001  # 学习率
-        self.memory_capacity = 100000  # 经验回放的容量
-        self.batch_size = 64  # mini-batch SGD中的批量大小
-        self.target_update = 4  # 目标网络的更新频率
-        self.hidden_dim = 256  # 网络隐藏层
-        ################################################################################
-        
-        ################################# 保存结果相关参数 ################################
-        self.result_path = curr_path + "/outputs/" + self.env_name + \
-            '/' + curr_time + '/results/'  # 保存结果的路径
-        self.model_path = curr_path + "/outputs/" + self.env_name + \
-            '/' + curr_time + '/models/'  # 保存模型的路径
-        self.save = True # 是否保存图片
-        ################################################################################
-
-
-def env_agent_config(cfg):
+def env_agent_config(cfg,seed=1):
     ''' 创建环境和智能体
     '''
     env = gym.make(cfg.env_name)  # 创建环境
     n_states = env.observation_space.shape[0]  # 状态维度
     n_actions = env.action_space.n  # 动作维度
     print(f"n states: {n_states}, n actions: {n_actions}")
-    model = MLP(n_states,n_actions)
-    agent = DQN(n_actions, model, cfg)  # 创建智能体
-    if cfg.seed !=0: # 设置随机种子
-        torch.manual_seed(cfg.seed)
-        env.seed(cfg.seed)
-        np.random.seed(cfg.seed)
+    agent = DQN(n_states,n_actions, cfg)  # 创建智能体
+    if seed !=0: # 设置随机种子
+        torch.manual_seed(seed)
+        env.seed(seed)
+        np.random.seed(seed)
     return env, agent
 
-
 def train(cfg, env, agent):
-    ''' 训练
+    ''' Training
     '''
-    print('开始训练!')
-    print(f'环境：{cfg.env_name}, 算法：{cfg.algo_name}, 设备：{cfg.device}')
+    print('Start training!')
+    print(f'Env:{cfg.env_name}, A{cfg.algo_name}, 设备：{cfg.device}')
     rewards = []  # 记录所有回合的奖励
     ma_rewards = []  # 记录所有回合的滑动平均奖励
     steps = []
@@ -117,7 +91,7 @@ def train(cfg, env, agent):
         else:
             ma_rewards.append(ep_reward)
         if (i_ep + 1) % 1 == 0:
-            print(f'Episode：{i_ep+1}/{cfg.test_eps}, Reward:{ep_reward:.2f}, Step:{ep_step:.2f} Epislon:{agent.epsilon(agent.frame_idx):.3f}')
+            print(f'Episode：{i_ep+1}/{cfg.train_eps}, Reward:{ep_reward:.2f}, Step:{ep_step:.2f} Epislon:{agent.epsilon(agent.frame_idx):.3f}')
     print('Finish training!')
     env.close()
     res_dic = {'rewards':rewards,'ma_rewards':ma_rewards,'steps':steps}
@@ -152,18 +126,19 @@ def test(cfg, env, agent):
             ma_rewards.append(ma_rewards[-1] * 0.9 + ep_reward * 0.1)
         else:
             ma_rewards.append(ep_reward)
-        print(f'Episode：{i_ep+1}/{cfg.train_eps}, Reward:{ep_reward:.2f}, Step:{ep_step:.2f}')
+        print(f'Episode：{i_ep+1}/{cfg.test_eps}, Reward:{ep_reward:.2f}, Step:{ep_step:.2f}')
     print('完成测试！')
     env.close()
     return {'rewards':rewards,'ma_rewards':ma_rewards,'steps':steps}
 
 
 if __name__ == "__main__":
-    cfg = Config()
+    cfg = get_args()
     # 训练
     env, agent = env_agent_config(cfg)
     res_dic = train(cfg, env, agent)
     make_dir(cfg.result_path, cfg.model_path)  # 创建保存结果和模型路径的文件夹
+    save_args(cfg)
     agent.save(path=cfg.model_path)  # 保存模型
     save_results_1(res_dic, tag='train',
                  path=cfg.result_path)  # 保存结果

codes/DQN/task1.py

@@ -1,168 +0,0 @@
-#!/usr/bin/env python
-# coding=utf-8
-'''
-Author: JiangJi
-Email: johnjim0816@gmail.com
-Date: 2021-12-22 11:14:17
-LastEditor: JiangJi
-LastEditTime: 2022-06-18 20:12:20
-Discription: 使用 Nature DQN 训练 CartPole-v1
-'''
-import sys
-import os
-curr_path = os.path.dirname(os.path.abspath(__file__))  # 当前文件所在绝对路径
-parent_path = os.path.dirname(curr_path)  # 父路径
-sys.path.append(parent_path)  # 添加路径到系统路径
-
-import gym
-import torch
-import datetime
-import torch.nn as nn
-import torch.nn.functional as F
-
-from common.utils import save_results, make_dir
-from common.utils import plot_rewards, plot_rewards_cn
-from dqn import DQN
-
-curr_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")  # 获取当前时间
-algo_name = "DQN"  # 算法名称
-env_name = 'CartPole-v1'  # 环境名称
-class DQNConfig:
-    ''' 算法相关参数设置
-    '''
-
-    def __init__(self):
-        self.algo_name = algo_name  # 算法名称
-        self.env_name = env_name  # 环境名称
-        self.device = torch.device(
-            "cuda" if torch.cuda.is_available() else "cpu")  # 检测GPU
-        self.train_eps = 300  # 训练的回合数
-        self.test_eps = 20  # 测试的回合数
-        # 超参数
-        self.gamma = 0.99  # 强化学习中的折扣因子
-        self.epsilon_start = 0.99  # e-greedy策略中初始epsilon
-        self.epsilon_end = 0.005  # e-greedy策略中的终止epsilon
-        self.epsilon_decay = 500  # e-greedy策略中epsilon的衰减率
-        self.lr = 0.0001  # 学习率
-        self.memory_capacity = 100000  # 经验回放的容量
-        self.batch_size = 128  # mini-batch SGD中的批量大小
-        self.target_update = 4  # 目标网络的更新频率
-        self.hidden_dim = 512  # 网络隐藏层
-class PlotConfig:
-    ''' 绘图相关参数设置
-    '''
-
-    def __init__(self) -> None:
-        self.algo_name = algo_name  # 算法名称
-        self.env_name = env_name  # 环境名称
-        self.device = torch.device(
-            "cuda" if torch.cuda.is_available() else "cpu")  # 检测GPU
-        self.result_path = curr_path + "/outputs/" + self.env_name + \
-            '/' + curr_time + '/results/'  # 保存结果的路径
-        self.model_path = curr_path + "/outputs/" + self.env_name + \
-            '/' + curr_time + '/models/'  # 保存模型的路径
-        self.save = True  # 是否保存图片
-
-class MLP(nn.Module):
-    def __init__(self, n_states,n_actions,hidden_dim=128):
-        """ 初始化q网络，为全连接网络
-            n_states: 输入的特征数即环境的状态维度
-            n_actions: 输出的动作维度
-        """
-        super(MLP, self).__init__()
-        self.fc1 = nn.Linear(n_states, hidden_dim) # 输入层
-        self.fc2 = nn.Linear(hidden_dim,hidden_dim) # 隐藏层
-        self.fc3 = nn.Linear(hidden_dim, n_actions) # 输出层
-        
-    def forward(self, x):
-        # 各层对应的激活函数
-        x = F.relu(self.fc1(x)) 
-        x = F.relu(self.fc2(x))
-        return self.fc3(x)
-        
-def env_agent_config(cfg, seed=1):
-    ''' 创建环境和智能体
-    '''
-    env = gym.make(cfg.env_name)  # 创建环境
-    env.seed(seed)  # 设置随机种子
-    n_states = env.observation_space.shape[0]  # 状态维度
-    n_actions = env.action_space.n  # 动作维度
-    model = MLP(n_states,n_actions)
-    agent = DQN(n_actions,model,cfg)  # 创建智能体
-    return env, agent
-
-def train(cfg, env, agent):
-    ''' 训练
-    '''
-    print('开始训练!')
-    print(f'环境：{cfg.env_name}, 算法：{cfg.algo_name}, 设备：{cfg.device}')
-    rewards = [] # 记录所有回合的奖励
-    ma_rewards = []  # 记录所有回合的滑动平均奖励
-    for i_ep in range(cfg.train_eps):
-        ep_reward = 0 # 记录一回合内的奖励
-        state = env.reset() # 重置环境，返回初始状态
-        while True:
-            action = agent.choose_action(state) # 选择动作
-            next_state, reward, done, _ = env.step(action) # 更新环境，返回transition
-            agent.memory.push(state, action, reward, next_state, done) # 保存transition
-            state = next_state # 更新下一个状态
-            agent.update() # 更新智能体
-            ep_reward += reward # 累加奖励
-            if done:
-                break
-        if (i_ep+1) % cfg.target_update == 0: # 智能体目标网络更新
-            agent.target_net.load_state_dict(agent.policy_net.state_dict())
-        rewards.append(ep_reward)
-        if ma_rewards:
-            ma_rewards.append(0.9*ma_rewards[-1]+0.1*ep_reward)
-        else:
-            ma_rewards.append(ep_reward)
-        if (i_ep+1)%10 == 0: 
-            print('回合：{}/{}, 奖励：{}'.format(i_ep+1, cfg.train_eps, ep_reward))
-    print('完成训练！')
-    return rewards, ma_rewards
-
-def test(cfg,env,agent):
-    print('开始测试!')
-    print(f'环境：{cfg.env_name}, 算法：{cfg.algo_name}, 设备：{cfg.device}')
-    # 由于测试不需要使用epsilon-greedy策略，所以相应的值设置为0
-    cfg.epsilon_start = 0.0 # e-greedy策略中初始epsilon
-    cfg.epsilon_end = 0.0 # e-greedy策略中的终止epsilon
-    rewards = [] # 记录所有回合的奖励
-    ma_rewards = []  # 记录所有回合的滑动平均奖励
-    for i_ep in range(cfg.test_eps):
-        ep_reward = 0 # 记录一回合内的奖励
-        state = env.reset() # 重置环境，返回初始状态
-        while True:
-            action = agent.choose_action(state) # 选择动作
-            next_state, reward, done, _ = env.step(action) # 更新环境，返回transition
-            state = next_state # 更新下一个状态
-            ep_reward += reward # 累加奖励
-            if done:
-                break
-        rewards.append(ep_reward)
-        if ma_rewards:
-            ma_rewards.append(ma_rewards[-1]*0.9+ep_reward*0.1)
-        else:
-            ma_rewards.append(ep_reward)
-        print(f"回合：{i_ep+1}/{cfg.test_eps}，奖励：{ep_reward:.1f}")
-    print('完成测试！')
-    return rewards,ma_rewards
-if __name__ == "__main__":
-    cfg = DQNConfig()
-    plot_cfg = PlotConfig()
-    # 训练
-    env, agent = env_agent_config(cfg, seed=1)
-    rewards, ma_rewards = train(cfg, env, agent)
-    make_dir(plot_cfg.result_path, plot_cfg.model_path)  # 创建保存结果和模型路径的文件夹
-    agent.save(path=plot_cfg.model_path)  # 保存模型
-    save_results(rewards, ma_rewards, tag='train',
-                path=plot_cfg.result_path)  # 保存结果
-    plot_rewards_cn(rewards, ma_rewards, plot_cfg, tag="train")  # 画出结果
-    # 测试
-    env, agent = env_agent_config(cfg, seed=10)
-    agent.load(path=plot_cfg.model_path)  # 导入模型
-    rewards, ma_rewards = test(cfg, env, agent)
-    save_results(rewards, ma_rewards, tag='test',
-                path=plot_cfg.result_path)  # 保存结果
-    plot_rewards_cn(rewards, ma_rewards, plot_cfg, tag="test")  # 画出结果

codes/DQN/task2.py

@@ -1,150 +0,0 @@
-#!/usr/bin/env python
-# coding=utf-8
-'''
-Author: JiangJi
-Email: johnjim0816@gmail.com
-Date: 2021-12-22 11:14:17
-LastEditor: JiangJi
-LastEditTime: 2022-02-10 06:17:46
-Discription: 使用 DQN-cnn  训练 PongNoFrameskip-v4
-'''
-import sys
-import os
-curr_path = os.path.dirname(os.path.abspath(__file__))  # 当前文件所在绝对路径
-parent_path = os.path.dirname(curr_path)  # 父路径
-sys.path.append(parent_path)  # 添加路径到系统路径
-
-import gym
-import torch
-import datetime
-from common.utils import save_results, make_dir
-from common.utils import plot_rewards, plot_rewards_cn
-from common.atari_wrappers import make_atari, wrap_deepmind
-from dqn import DQN
-
-curr_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")  # 获取当前时间
-algo_name = 'DQN-cnn'  # 算法名称
-env_name = 'PongNoFrameskip-v4'  # 环境名称
-device = torch.device("cuda" if torch.cuda.is_available() else "cpu")  # 检测GPU
-class DQNConfig:
-    ''' 算法相关参数设置
-    '''
-
-    def __init__(self):
-        self.algo_name = algo_name  # 算法名称
-        self.env_name = env_name  # 环境名称
-        self.device = device # 检测GPU
-        self.train_eps = 500  # 训练的回合数
-        self.test_eps = 30  # 测试的回合数
-        # 超参数
-        self.gamma = 0.95  # 强化学习中的折扣因子
-        self.epsilon_start = 0.90  # e-greedy策略中初始epsilon
-        self.epsilon_end = 0.01  # e-greedy策略中的终止epsilon
-        self.epsilon_decay = 500  # e-greedy策略中epsilon的衰减率
-        self.lr = 0.0001  # 学习率
-        self.memory_capacity = 100000  # 经验回放的容量
-        self.batch_size = 64  # mini-batch SGD中的批量大小
-        self.target_update = 4  # 目标网络的更新频率
-        self.hidden_dim = 256  # 网络隐藏层
-class PlotConfig:
-    ''' 绘图相关参数设置
-    '''
-
-    def __init__(self) -> None:
-        self.algo_name = algo_name  # 算法名称
-        self.env_name = env_name  # 环境名称
-        self.device = device  # 检测GPU
-        self.result_path = curr_path + "/outputs/" + self.env_name + \
-            '/' + curr_time + '/results/'  # 保存结果的路径
-        self.model_path = curr_path + "/outputs/" + self.env_name + \
-            '/' + curr_time + '/models/'  # 保存模型的路径
-        self.save = True  # 是否保存图片
-
-
-def env_agent_config(cfg, seed=1):
-    ''' 创建环境和智能体
-    '''
-    env    = make_atari(cfg.env_name) # 创建环境
-    # env    = wrap_deepmind(env)
-    # env    = wrap_pytorch(env) 
-    env.seed(seed)  # 设置随机种子
-    n_states = env.observation_space.shape[0]  # 状态维度
-    n_actions = env.action_space.n  # 动作维度
-    agent = DQN(n_states, n_actions, cfg)  # 创建智能体
-    return env, agent
-
-def train(cfg, env, agent):
-    ''' 训练
-    '''
-    print('开始训练!')
-    print(f'环境：{cfg.env_name}, 算法：{cfg.algo_name}, 设备：{cfg.device}')
-    rewards = [] # 记录所有回合的奖励
-    ma_rewards = []  # 记录所有回合的滑动平均奖励
-    for i_ep in range(cfg.train_eps):
-        ep_reward = 0 # 记录一回合内的奖励
-        state = env.reset() # 重置环境，返回初始状态
-        while True:
-            action = agent.choose_action(state) # 选择动作
-            next_state, reward, done, _ = env.step(action) # 更新环境，返回transition
-            agent.memory.push(state, action, reward, next_state, done) # 保存transition
-            state = next_state # 更新下一个状态
-            agent.update() # 更新智能体
-            ep_reward += reward # 累加奖励
-            if done:
-                break
-        if (i_ep+1) % cfg.target_update == 0: # 智能体目标网络更新
-            agent.target_net.load_state_dict(agent.policy_net.state_dict())
-        rewards.append(ep_reward)
-        if ma_rewards:
-            ma_rewards.append(0.9*ma_rewards[-1]+0.1*ep_reward)
-        else:
-            ma_rewards.append(ep_reward)
-        if (i_ep+1)%10 == 0: 
-            print('回合：{}/{}, 奖励：{}'.format(i_ep+1, cfg.train_eps, ep_reward))
-    print('完成训练！')
-    return rewards, ma_rewards
-
-def test(cfg,env,agent):
-    print('开始测试!')
-    print(f'环境：{cfg.env_name}, 算法：{cfg.algo_name}, 设备：{cfg.device}')
-    # 由于测试不需要使用epsilon-greedy策略，所以相应的值设置为0
-    cfg.epsilon_start = 0.0 # e-greedy策略中初始epsilon
-    cfg.epsilon_end = 0.0 # e-greedy策略中的终止epsilon
-    rewards = [] # 记录所有回合的奖励
-    ma_rewards = []  # 记录所有回合的滑动平均奖励
-    for i_ep in range(cfg.test_eps):
-        ep_reward = 0 # 记录一回合内的奖励
-        state = env.reset() # 重置环境，返回初始状态
-        while True:
-            action = agent.choose_action(state) # 选择动作
-            next_state, reward, done, _ = env.step(action) # 更新环境，返回transition
-            state = next_state # 更新下一个状态
-            ep_reward += reward # 累加奖励
-            if done:
-                break
-        rewards.append(ep_reward)
-        if ma_rewards:
-            ma_rewards.append(ma_rewards[-1]*0.9+ep_reward*0.1)
-        else:
-            ma_rewards.append(ep_reward)
-        print(f"回合：{i_ep+1}/{cfg.test_eps}，奖励：{ep_reward:.1f}")
-    print('完成测试！')
-    return rewards,ma_rewards
-if __name__ == "__main__":
-    cfg = DQNConfig()
-    plot_cfg = PlotConfig()
-    # 训练
-    env, agent = env_agent_config(cfg, seed=1)
-    rewards, ma_rewards = train(cfg, env, agent)
-    make_dir(plot_cfg.result_path, plot_cfg.model_path)  # 创建保存结果和模型路径的文件夹
-    agent.save(path=plot_cfg.model_path)  # 保存模型
-    save_results(rewards, ma_rewards, tag='train',
-                path=plot_cfg.result_path)  # 保存结果
-    plot_rewards_cn(rewards, ma_rewards, plot_cfg, tag="train")  # 画出结果
-    # 测试
-    env, agent = env_agent_config(cfg, seed=10)
-    agent.load(path=plot_cfg.model_path)  # 导入模型
-    rewards, ma_rewards = test(cfg, env, agent)
-    save_results(rewards, ma_rewards, tag='test',
-                path=plot_cfg.result_path)  # 保存结果
-    plot_rewards_cn(rewards, ma_rewards, plot_cfg, tag="test")  # 画出结果

codes/DQN/task4.py

@@ -1,180 +0,0 @@
-import sys
-import os
-import torch.nn as nn
-import torch.nn.functional as F
-curr_path = os.path.dirname(os.path.abspath(__file__))  # 当前文件所在绝对路径
-parent_path = os.path.dirname(curr_path)  # 父路径
-sys.path.append(parent_path)  # 添加路径到系统路径
-
-import gym
-import torch
-import datetime
-import numpy as np
-from common.utils import save_results_1, make_dir
-from common.utils import plot_rewards
-from dqn_1 import DQN
-
-curr_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")  # 获取当前时间
-
-class MLP(nn.Module):
-    def __init__(self, n_states,n_actions,hidden_dim=256):
-        """ 初始化q网络，为全连接网络
-            n_states: 输入的特征数即环境的状态维度
-            n_actions: 输出的动作维度
-        """
-        super(MLP, self).__init__()
-        self.fc1 = nn.Linear(n_states, hidden_dim) # 输入层
-        self.fc2 = nn.Linear(hidden_dim,hidden_dim) # 隐藏层
-        self.fc3 = nn.Linear(hidden_dim,hidden_dim) # 隐藏层
-        self.fc4 = nn.Linear(hidden_dim, n_actions) # 输出层
-        
-    def forward(self, x):
-        # 各层对应的激活函数
-        x = F.relu(self.fc1(x)) 
-        x = F.relu(self.fc2(x))
-        x = F.relu(self.fc3(x))
-        return self.fc4(x)
-
-class Config:
-    '''超参数
-    '''
-
-    def __init__(self):
-        ################################## 环境超参数 ###################################
-        self.algo_name = 'DQN'  # 算法名称
-        # self.env_name = 'Breakout-ram-v0'  # 环境名称
-        self.env_name = 'ALE/Pong-ram-v5'
-        self.device = torch.device(
-            "cuda" if torch.cuda.is_available() else "cpu")  # 检测GPUgjgjlkhfsf风刀霜的撒发十
-        self.seed = 10 # 随机种子，置0则不设置随机种子
-        self.train_eps = 5  # 训练的回合数
-        self.test_eps = 30  # 测试的回合数
-        ################################################################################
-        
-        ################################## 算法超参数 ###################################
-        self.gamma = 0.99  # 强化学习中的折扣因子
-        self.epsilon_start = 0.95  # e-greedy策略中初始epsilon
-        self.epsilon_end = 0.01  # e-greedy策略中的终止epsilon
-        self.epsilon_decay = 500000  # e-greedy策略中epsilon的衰减率
-        self.lr = 0.00025 # 学习率
-        self.memory_capacity = int(5e4)  # 经验回放的容量
-        self.batch_size = 32  # mini-batch SGD中的批量大小
-        self.target_update = 4  # 目标网络的更新频率
-        self.hidden_dim = 512  # 网络隐藏层
-        ################################################################################
-        
-        ################################# 保存结果相关参数 ################################
-        self.result_path = curr_path + "/outputs/" + self.env_name + \
-            '/' + curr_time + '/results/'  # 保存结果的路径
-        self.model_path = curr_path + "/outputs/" + self.env_name + \
-            '/' + curr_time + '/models/'  # 保存模型的路径
-        self.save = True # 是否保存图片
-        ################################################################################
-
-
-def env_agent_config(cfg):
-    ''' 创建环境和智能体
-    '''
-    env = gym.make(cfg.env_name)  # 创建环境
-    n_states = env.observation_space.shape[0]  # 状态维度
-    n_actions = env.action_space.n  # 动作维度
-    print(f"n states: {n_states}, n actions: {n_actions}")
-    model = MLP(n_states,n_actions)
-    agent = DQN(n_states, n_actions, model, cfg)  # 创建智能体
-    if cfg.seed !=0: # 设置随机种子
-        torch.manual_seed(cfg.seed)
-        env.seed(cfg.seed)
-        np.random.seed(cfg.seed)
-    return env, agent
-
-
-def train(cfg, env, agent):
-    ''' 训练
-    '''
-    print('开始训练!')
-    print(f'环境：{cfg.env_name}, 算法：{cfg.algo_name}, 设备：{cfg.device}')
-    rewards = []  # 记录所有回合的奖励
-    ma_rewards = []  # 记录所有回合的滑动平均奖励
-    steps = []
-    for i_ep in range(cfg.train_eps):
-        ep_reward = 0  # 记录一回合内的奖励
-        state = env.reset()  # 重置环境，返回初始状态
-        ep_step = 0
-        while True:
-            ep_step+=1
-            action = agent.choose_action(state)  # 选择动作
-            next_state, reward, done, _ = env.step(action)  # 更新环境，返回transition
-            agent.memory.push(state, action, reward,
-                              next_state, done)  # 保存transition
-            state = next_state  # 更新下一个状态
-            agent.update()  # 更新智能体
-            ep_reward += reward  # 累加奖励
-            if done:
-                break
-        if (i_ep + 1) % cfg.target_update == 0:  # 智能体目标网络更新
-            agent.target_net.load_state_dict(agent.policy_net.state_dict())
-        steps.append(ep_step)
-        rewards.append(ep_reward)
-        if ma_rewards:
-            ma_rewards.append(0.9 * ma_rewards[-1] + 0.1 * ep_reward)
-        else:
-            ma_rewards.append(ep_reward)
-        if (i_ep + 1) % 1 == 0:
-            print(f'Episode：{i_ep+1}/{cfg.train_eps}, Reward:{ep_reward:.2f}, Epislon:{agent.epsilon(agent.frame_idx):.3f}')
-    print('完成训练！')
-    env.close()
-    res_dic = {'rewards':rewards,'ma_rewards':ma_rewards,'steps':steps}
-    return res_dic
-
-
-def test(cfg, env, agent):
-    print('开始测试!')
-    print(f'环境：{cfg.env_name}, 算法：{cfg.algo_name}, 设备：{cfg.device}')
-    ############# 由于测试不需要使用epsilon-greedy策略，所以相应的值设置为0 ###############
-    cfg.epsilon_start = 0.0  # e-greedy策略中初始epsilon
-    cfg.epsilon_end = 0.0  # e-greedy策略中的终止epsilon
-    ################################################################################
-    rewards = []  # 记录所有回合的奖励
-    ma_rewards = []  # 记录所有回合的滑动平均奖励
-    steps = []
-    for i_ep in range(cfg.test_eps):
-        ep_reward = 0  # 记录一回合内的奖励
-        ep_step = 0
-        state = env.reset()  # 重置环境，返回初始状态
-        while True:
-            ep_step+=1
-            action = agent.choose_action(state)  # 选择动作
-            next_state, reward, done, _ = env.step(action)  # 更新环境，返回transition
-            state = next_state  # 更新下一个状态
-            ep_reward += reward  # 累加奖励
-            if done:
-                break
-        steps.append(ep_step)
-        rewards.append(ep_reward)
-        if ma_rewards:
-            ma_rewards.append(ma_rewards[-1] * 0.9 + ep_reward * 0.1)
-        else:
-            ma_rewards.append(ep_reward)
-        print(f"回合：{i_ep+1}/{cfg.test_eps}，奖励：{ep_reward:.1f}")
-    print('完成测试！')
-    env.close()
-    return {'rewards':rewards,'ma_rewards':ma_rewards,'steps':steps}
-
-
-if __name__ == "__main__":
-    cfg = Config()
-    # 训练
-    env, agent = env_agent_config(cfg)
-    res_dic = train(cfg, env, agent)
-    make_dir(cfg.result_path, cfg.model_path)  # 创建保存结果和模型路径的文件夹
-    agent.save(path=cfg.model_path)  # 保存模型
-    save_results_1(res_dic, tag='train',
-                 path=cfg.result_path)  # 保存结果
-    plot_rewards(res_dic['rewards'], res_dic['ma_rewards'], cfg, tag="train")  # 画出结果
-    # 测试
-    env, agent = env_agent_config(cfg)
-    agent.load(path=cfg.model_path)  # 导入模型
-    res_dic = test(cfg, env, agent)
-    save_results_1(res_dic, tag='test',
-                 path=cfg.result_path)  # 保存结果
-    plot_rewards(res_dic['rewards'], res_dic['ma_rewards'],cfg, tag="test")  # 画出结果

codes/DQN/task5.py

@@ -1,149 +0,0 @@
-import sys
-import os
-curr_path = os.path.dirname(os.path.abspath(__file__))  # 当前文件所在绝对路径
-parent_path = os.path.dirname(curr_path)  # 父路径
-sys.path.append(parent_path)  # 添加路径到系统路径
-
-import gym
-import torch
-import datetime
-import numpy as np
-from common.utils import save_results, make_dir
-from common.utils import plot_rewards
-from dqn import DQN
-
-curr_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")  # 获取当前时间
-
-
-class Config:
-    '''超参数
-    '''
-
-    def __init__(self):
-        ################################## 环境超参数 ###################################
-        self.algo_name = 'DQN'  # 算法名称
-        self.env_name = 'SpaceInvaders-ram-v0'  # 环境名称
-        self.device = torch.device(
-            "cuda" if torch.cuda.is_available() else "cpu")  # 检测GPUgjgjlkhfsf风刀霜的撒发十
-        self.seed = 10 # 随机种子，置0则不设置随机种子
-        self.train_eps = 200  # 训练的回合数
-        self.test_eps = 30  # 测试的回合数
-        ################################################################################
-        
-        ################################## 算法超参数 ###################################
-        self.gamma = 0.99  # 强化学习中的折扣因子
-        self.epsilon_start = 0.95  # e-greedy策略中初始epsilon
-        self.epsilon_end = 0.01  # e-greedy策略中的终止epsilon
-        self.epsilon_decay = 20000  # e-greedy策略中epsilon的衰减率
-        self.lr = 2e-4  # 学习率
-        self.memory_capacity = int(1e5)  # 经验回放的容量
-        self.batch_size = 32  # mini-batch SGD中的批量大小
-        self.target_update = 4  # 目标网络的更新频率
-        self.hidden_dim = 512  # 网络隐藏层
-        ################################################################################
-        
-        ################################# 保存结果相关参数 ################################
-        self.result_path = curr_path + "/outputs/" + self.env_name + \
-            '/' + curr_time + '/results/'  # 保存结果的路径
-        self.model_path = curr_path + "/outputs/" + self.env_name + \
-            '/' + curr_time + '/models/'  # 保存模型的路径
-        self.save = True # 是否保存图片
-        ################################################################################
-
-
-def env_agent_config(cfg):
-    ''' 创建环境和智能体
-    '''
-    env = gym.make(cfg.env_name)  # 创建环境
-    n_states = env.observation_space.shape[0]  # 状态维度
-    n_actions = env.action_space.n  # 动作维度
-    print(f"n states: {n_states}, n actions: {n_actions}")
-    agent = DQN(n_states, n_actions, cfg)  # 创建智能体
-    if cfg.seed !=0: # 设置随机种子
-        torch.manual_seed(cfg.seed)
-        env.seed(cfg.seed)
-        np.random.seed(cfg.seed)
-    return env, agent
-
-
-def train(cfg, env, agent):
-    ''' 训练
-    '''
-    print('开始训练!')
-    print(f'环境：{cfg.env_name}, 算法：{cfg.algo_name}, 设备：{cfg.device}')
-    rewards = []  # 记录所有回合的奖励
-    ma_rewards = []  # 记录所有回合的滑动平均奖励
-    for i_ep in range(cfg.train_eps):
-        ep_reward = 0  # 记录一回合内的奖励
-        state = env.reset()  # 重置环境，返回初始状态
-        while True:
-            action = agent.choose_action(state)  # 选择动作
-            next_state, reward, done, _ = env.step(action)  # 更新环境，返回transition
-            agent.memory.push(state, action, reward,
-                              next_state, done)  # 保存transition
-            state = next_state  # 更新下一个状态
-            agent.update()  # 更新智能体
-            ep_reward += reward  # 累加奖励
-            if done:
-                break
-        if (i_ep + 1) % cfg.target_update == 0:  # 智能体目标网络更新
-            agent.target_net.load_state_dict(agent.policy_net.state_dict())
-        rewards.append(ep_reward)
-        if ma_rewards:
-            ma_rewards.append(0.9 * ma_rewards[-1] + 0.1 * ep_reward)
-        else:
-            ma_rewards.append(ep_reward)
-        if (i_ep + 1) % 1 == 0:
-            print(f'Episode：{i_ep+1}/{cfg.train_eps}, Reward:{ep_reward:.2f}, Epislon:{agent.epsilon(agent.frame_idx):.3f}')
-    print('完成训练！')
-    env.close()
-    return rewards, ma_rewards
-
-
-def test(cfg, env, agent):
-    print('开始测试!')
-    print(f'环境：{cfg.env_name}, 算法：{cfg.algo_name}, 设备：{cfg.device}')
-    ############# 由于测试不需要使用epsilon-greedy策略，所以相应的值设置为0 ###############
-    cfg.epsilon_start = 0.0  # e-greedy策略中初始epsilon
-    cfg.epsilon_end = 0.0  # e-greedy策略中的终止epsilon
-    ################################################################################
-    rewards = []  # 记录所有回合的奖励
-    ma_rewards = []  # 记录所有回合的滑动平均奖励
-    for i_ep in range(cfg.test_eps):
-        ep_reward = 0  # 记录一回合内的奖励
-        state = env.reset()  # 重置环境，返回初始状态
-        while True:
-            action = agent.choose_action(state)  # 选择动作
-            next_state, reward, done, _ = env.step(action)  # 更新环境，返回transition
-            state = next_state  # 更新下一个状态
-            ep_reward += reward  # 累加奖励
-            if done:
-                break
-        rewards.append(ep_reward)
-        if ma_rewards:
-            ma_rewards.append(ma_rewards[-1] * 0.9 + ep_reward * 0.1)
-        else:
-            ma_rewards.append(ep_reward)
-        print(f"回合：{i_ep+1}/{cfg.test_eps}，奖励：{ep_reward:.1f}")
-    print('完成测试！')
-    env.close()
-    return rewards, ma_rewards
-
-
-if __name__ == "__main__":
-    cfg = Config()
-    # 训练
-    env, agent = env_agent_config(cfg)
-    rewards, ma_rewards = train(cfg, env, agent)
-    make_dir(cfg.result_path, cfg.model_path)  # 创建保存结果和模型路径的文件夹
-    agent.save(path=cfg.model_path)  # 保存模型
-    save_results(rewards, ma_rewards, tag='train',
-                 path=cfg.result_path)  # 保存结果
-    plot_rewards(rewards, ma_rewards, cfg, tag="train")  # 画出结果
-    # 测试
-    env, agent = env_agent_config(cfg)
-    agent.load(path=cfg.model_path)  # 导入模型
-    rewards, ma_rewards = test(cfg, env, agent)
-    save_results(rewards, ma_rewards, tag='test',
-                 path=cfg.result_path)  # 保存结果
-    plot_rewards(rewards, ma_rewards, cfg, tag="test")  # 画出结果