update projects

projects/codes/DoubleDQN/double_dqn.py

@@ -63,18 +63,18 @@ class MLP(nn.Module):
         return self.fc3(x)
         
 class DoubleDQN:
-    def __init__(self, n_states, n_actions, cfg):
+    def __init__(self, n_states, n_actions, model, memory, cfg):
         self.n_actions = n_actions  # 总的动作个数
         self.device = torch.device(cfg.device)  # 设备，cpu或gpu等
         self.gamma = cfg.gamma
         # e-greedy策略相关参数
-        self.actions_count = 0
+        self.sample_count = 0
         self.epsilon_start = cfg.epsilon_start
         self.epsilon_end = cfg.epsilon_end
         self.epsilon_decay = cfg.epsilon_decay
         self.batch_size = cfg.batch_size
-        self.policy_net = MLP(n_states, n_actions,hidden_dim=cfg.hidden_dim).to(self.device)
-        self.target_net = MLP(n_states, n_actions,hidden_dim=cfg.hidden_dim).to(self.device)
+        self.policy_net = model.to(self.device)
+        self.target_net = model.to(self.device)
         # target_net copy from policy_net
         for target_param, param in zip(self.target_net.parameters(), self.policy_net.parameters()):
             target_param.data.copy_(param.data)
@@ -82,13 +82,13 @@ class DoubleDQN:
         # 可查parameters()与state_dict()的区别，前者require_grad=True
         self.optimizer = optim.Adam(self.policy_net.parameters(), lr=cfg.lr)
         self.loss = 0
-        self.memory = ReplayBuffer(cfg.memory_capacity)
+        self.memory = memory
         
-    def choose_action(self, state):
+    def sample(self, state):
         '''选择动作
         '''
-        self.actions_count += 1
-        self.epsilon = self.epsilon_end + (self.epsilon_start - self.epsilon_end) * math.exp(-1. * self.actions_count / self.epsilon_decay)
+        self.sample_count += 1
+        self.epsilon = self.epsilon_end + (self.epsilon_start - self.epsilon_end) * math.exp(-1. * self.sample_count / self.epsilon_decay)
         if random.random() > self.epsilon:
             with torch.no_grad():
                 # 先转为张量便于丢给神经网络,state元素数据原本为float64
@@ -104,9 +104,16 @@ class DoubleDQN:
         else:
             action = random.randrange(self.n_actions)
         return action
+    def predict(self, state):
+        '''选择动作
+        '''
+        with torch.no_grad():
+            state = torch.tensor([state], device=self.device, dtype=torch.float32)
+            q_value = self.policy_net(state)
+            action = q_value.max(1)[1].item()  
+        return action
     def update(self):
-
-        if len(self.memory) < self.batch_size:
+        if len(self.memory) < self.batch_size: # 只有memory满了才会更新
             return
         # 从memory中随机采样transition
         state_batch, action_batch, reward_batch, next_state_batch, done_batch = self.memory.sample(
@@ -150,7 +157,7 @@ class DoubleDQN:
         for param in self.policy_net.parameters():  # clip防止梯度爆炸
             param.grad.data.clamp_(-1, 1)
         self.optimizer.step()  # 更新模型
-
+    
     def save(self,path):
         torch.save(self.target_net.state_dict(), path+'checkpoint.pth')
 

projects/codes/DoubleDQN/outputs/CartPole-v0/20220721-215416/results/params.json

@@ -1,19 +0,0 @@
-{
-    "algo_name": "DoubleDQN",
-    "env_name": "CartPole-v0",
-    "train_eps": 200,
-    "test_eps": 20,
-    "gamma": 0.99,
-    "epsilon_start": 0.95,
-    "epsilon_end": 0.01,
-    "epsilon_decay": 500,
-    "lr": 0.0001,
-    "memory_capacity": 100000,
-    "batch_size": 64,
-    "target_update": 2,
-    "hidden_dim": 256,
-    "device": "cuda",
-    "result_path": "C:\\Users\\24438\\Desktop\\rl-tutorials\\codes\\DoubleDQN/outputs/CartPole-v0/20220721-215416/results/",
-    "model_path": "C:\\Users\\24438\\Desktop\\rl-tutorials\\codes\\DoubleDQN/outputs/CartPole-v0/20220721-215416/models/",
-    "save_fig": true
-}

projects/codes/DoubleDQN/outputs/CartPole-v0/20220803-104127/results/params.json

@@ -0,0 +1 @@
+{"algo_name": "DoubleDQN", "env_name": "CartPole-v0", "train_eps": 200, "test_eps": 20, "gamma": 0.95, "epsilon_start": 0.95, "epsilon_end": 0.01, "epsilon_decay": 500, "lr": 0.0001, "memory_capacity": 100000, "batch_size": 64, "target_update": 4, "hidden_dim": 256, "device": "cpu", "result_path": "/root/Desktop/rl-tutorials/codes/DoubleDQN/outputs/CartPole-v0/20220803-104127/results/", "model_path": "/root/Desktop/rl-tutorials/codes/DoubleDQN/outputs/CartPole-v0/20220803-104127/models/", "save_fig": true}

projects/codes/DoubleDQN/task0.py

@@ -20,31 +20,33 @@ import argparse
 
 from common.utils import save_results,make_dir
 from common.utils import plot_rewards,save_args
+from common.models import MLP
+from common.memories import ReplayBuffer
 from DoubleDQN.double_dqn import DoubleDQN
 
 def get_args():
-    """ Hyperparameters
+    """ 超参数
     """
-    curr_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")  # Obtain current time
+    curr_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")  # 获取当前时间
     parser = argparse.ArgumentParser(description="hyperparameters")      
     parser.add_argument('--algo_name',default='DoubleDQN',type=str,help="name of algorithm")
     parser.add_argument('--env_name',default='CartPole-v0',type=str,help="name of environment")
     parser.add_argument('--train_eps',default=200,type=int,help="episodes of training")
     parser.add_argument('--test_eps',default=20,type=int,help="episodes of testing")
-    parser.add_argument('--gamma',default=0.99,type=float,help="discounted factor")
+    parser.add_argument('--gamma',default=0.95,type=float,help="discounted factor")
     parser.add_argument('--epsilon_start',default=0.95,type=float,help="initial value of epsilon")
     parser.add_argument('--epsilon_end',default=0.01,type=float,help="final value of epsilon")
     parser.add_argument('--epsilon_decay',default=500,type=int,help="decay rate of epsilon")
     parser.add_argument('--lr',default=0.0001,type=float,help="learning rate")
     parser.add_argument('--memory_capacity',default=100000,type=int,help="memory capacity")
     parser.add_argument('--batch_size',default=64,type=int)
-    parser.add_argument('--target_update',default=2,type=int)
+    parser.add_argument('--target_update',default=4,type=int)
     parser.add_argument('--hidden_dim',default=256,type=int)
     parser.add_argument('--device',default='cpu',type=str,help="cpu or cuda") 
     parser.add_argument('--result_path',default=curr_path + "/outputs/" + parser.parse_args().env_name + \
             '/' + curr_time + '/results/' )
     parser.add_argument('--model_path',default=curr_path + "/outputs/" + parser.parse_args().env_name + \
-            '/' + curr_time + '/models/' ) # path to save models
+            '/' + curr_time + '/models/' ) # 保存模型的路径
     parser.add_argument('--save_fig',default=True,type=bool,help="if save figure or not")           
     args = parser.parse_args()                          
     return args
@@ -55,19 +57,20 @@ def env_agent_config(cfg,seed=1):
     env.seed(seed)
     n_states = env.observation_space.shape[0]
     n_actions = env.action_space.n
-    agent = DoubleDQN(n_states,n_actions,cfg)
+    model = MLP(n_states, n_actions,hidden_dim=cfg.hidden_dim)
+    memory = ReplayBuffer(cfg.memory_capacity)
+    agent = DoubleDQN(n_states,n_actions,model,memory,cfg)
     return env,agent
 
 def train(cfg,env,agent):
-    print('Start training!')
-    print(f'Env:{cfg.env_name}, Algorithm:{cfg.algo_name}, Device:{cfg.device}')
+    print("开始训练！")
+    print(f"回合：{cfg.env_name}, 算法：{cfg.algo_name}, 设备：{cfg.device}")
     rewards = [] # 记录所有回合的奖励
-    ma_rewards = []  # 记录所有回合的滑动平均奖励
     for i_ep in range(cfg.train_eps):
         ep_reward = 0 # 记录一回合内的奖励
         state = env.reset() # 重置环境，返回初始状态
         while True:
-            action = agent.choose_action(state) 
+            action = agent.sample(state) 
             next_state, reward, done, _ = env.step(action)
             ep_reward += reward
             agent.memory.push(state, action, reward, next_state, done) 
@@ -78,61 +81,45 @@ def train(cfg,env,agent):
         if i_ep % cfg.target_update == 0:
             agent.target_net.load_state_dict(agent.policy_net.state_dict())
         if (i_ep+1)%10 == 0: 
-            print(f'Env:{i_ep+1}/{cfg.train_eps}, Reward:{ep_reward:.2f}')
-        rewards.append(ep_reward)
-        if ma_rewards:
-            ma_rewards.append(
-                0.9*ma_rewards[-1]+0.1*ep_reward)
-        else:
-            ma_rewards.append(ep_reward)   
-    print('Finish training!')
-    return {'rewards':rewards,'ma_rewards':ma_rewards}
+            print(f'回合：{i_ep+1}/{cfg.train_eps}，奖励：{ep_reward:.2f}，Epislon：{agent.epsilon:.3f}')
+        rewards.append(ep_reward) 
+    print("完成训练！")
+    return {'rewards':rewards}
 
 def test(cfg,env,agent):
-    print('Start testing')
-    print(f'Env:{cfg.env_name}, Algorithm:{cfg.algo_name}, Device:{cfg.device}')
-    ############# 由于测试不需要使用epsilon-greedy策略，所以相应的值设置为0 ###############
-    cfg.epsilon_start = 0.0  # e-greedy策略中初始epsilon
-    cfg.epsilon_end = 0.0  # e-greedy策略中的终止epsilon
-    ################################################################################
+    print("开始测试！")
+    print(f"回合：{cfg.env_name}, 算法：{cfg.algo_name}, 设备：{cfg.device}")
     rewards = [] # 记录所有回合的奖励
-    ma_rewards = []  # 记录所有回合的滑动平均奖励
-    
     for i_ep in range(cfg.test_eps):
         state = env.reset() 
         ep_reward = 0   
         while True:
-            action = agent.choose_action(state) 
+            action = agent.predict(state) 
             next_state, reward, done, _ = env.step(action)  
             state = next_state  
             ep_reward += reward
             if done:
                 break
         rewards.append(ep_reward)
-        if ma_rewards:
-            ma_rewards.append(ma_rewards[-1]*0.9+ep_reward*0.1)
-        else:
-            ma_rewards.append(ep_reward)
-        print(f"Epside:{i_ep+1}/{cfg.test_eps}, Reward:{ep_reward:.1f}")
-    print('Finish testing!')
-    return {'rewards':rewards,'ma_rewards':ma_rewards}
+        print(f'回合：{i_ep+1}/{cfg.test_eps}，奖励：{ep_reward:.2f}')
+    print("完成测试！")
+    return {'rewards':rewards}
 
 if __name__ == "__main__":
     cfg = get_args()
-    print(cfg.device)
-    # training
-    env,agent = env_agent_config(cfg,seed=1)
+    # 训练
+    env, agent = env_agent_config(cfg,seed=1)
     res_dic = train(cfg, env, agent)
-    make_dir(cfg.result_path, cfg.model_path)
-    save_args(cfg)
-    agent.save(path=cfg.model_path)
+    make_dir(cfg.result_path, cfg.model_path)  
+    save_args(cfg) # 保存参数
+    agent.save(path=cfg.model_path)  # 保存模型
     save_results(res_dic, tag='train',
                  path=cfg.result_path)  
-    plot_rewards(res_dic['rewards'], res_dic['ma_rewards'], cfg, tag="train")  
-    # testing
-    env,agent = env_agent_config(cfg,seed=10)
-    agent.load(path=cfg.model_path)
-    res_dic = test(cfg,env,agent)
+    plot_rewards(res_dic['rewards'], cfg, tag="train")  
+    # 测试
+    env, agent = env_agent_config(cfg,seed=1)
+    agent.load(path=cfg.model_path)  # 导入模型
+    res_dic = test(cfg, env, agent)
     save_results(res_dic, tag='test',
-                 path=cfg.result_path)  
-    plot_rewards(res_dic['rewards'], res_dic['ma_rewards'], cfg, tag="test") 
+                 path=cfg.result_path)  # 保存结果
+    plot_rewards(res_dic['rewards'], cfg, tag="test")  # 画出结果