update

codes/DQN/README.md

@@ -1,5 +1,5 @@
 # DQN
-
+#TODO
 ## 原理简介
 DQN是Q-leanning算法的优化和延伸，Q-leaning中使用有限的Q表存储值的信息，而DQN中则用神经网络替代Q表存储信息，这样更适用于高维的情况，相关知识基础可参考[datawhale李宏毅笔记-Q学习](https://datawhalechina.github.io/easy-rl/#/chapter6/chapter6)。
 

codes/DQN/agent.py

@@ -5,7 +5,7 @@
 @Email: johnjim0816@gmail.com
 @Date: 2020-06-12 00:50:49
 @LastEditor: John
-LastEditTime: 2021-03-30 17:01:26
+LastEditTime: 2021-04-29 22:19:18
 @Discription: 
 @Environment: python 3.7.7
 '''
@@ -39,6 +39,8 @@ class DQN:
                               hidden_dim=cfg.hidden_dim).to(self.device)
         self.target_net = MLP(state_dim, action_dim,
                               hidden_dim=cfg.hidden_dim).to(self.device)
+        for target_param, param in zip(self.target_net.parameters(), self.policy_net.parameters()):
+            target_param.data.copy_(param.data)
         self.optimizer = optim.Adam(self.policy_net.parameters(), lr=cfg.lr)
         self.loss = 0
         self.memory = ReplayBuffer(cfg.memory_capacity)
@@ -48,21 +50,16 @@ class DQN:
         '''
         self.frame_idx += 1
         if random.random() > self.epsilon(self.frame_idx):
-            with torch.no_grad():
-                # 先转为张量便于丢给神经网络,state元素数据原本为float64
-                # 注意state=torch.tensor(state).unsqueeze(0)跟state=torch.tensor([state])等价
-                state = torch.tensor(
-                    [state], device=self.device, dtype=torch.float32)
-                # 如tensor([[-0.0798, -0.0079]], grad_fn=<AddmmBackward>)
-                q_value = self.policy_net(state)
-                # tensor.max(1)返回每行的最大值以及对应的下标，
-                # 如torch.return_types.max(values=tensor([10.3587]),indices=tensor([0]))
-                # 所以tensor.max(1)[1]返回最大值对应的下标，即action
-                action = q_value.max(1)[1].item()
+            action = self.predict(state)
         else:
             action = random.randrange(self.action_dim)
         return action
-
+    def predict(self,state):
+        with torch.no_grad():
+            state = torch.tensor([state], device=self.device, dtype=torch.float32)
+            q_values = self.policy_net(state)
+            action = q_values.max(1)[1].item()
+        return action
     def update(self):
 
         if len(self.memory) < self.batch_size:
@@ -109,3 +106,5 @@ class DQN:
 
     def load(self, path):
         self.target_net.load_state_dict(torch.load(path+'dqn_checkpoint.pth'))
+        for target_param, param in zip(self.target_net.parameters(), self.policy_net.parameters()):
+            param.data.copy_(target_param.data)

codes/DQN/task0_train.py

@@ -5,7 +5,7 @@
 @Email: johnjim0816@gmail.com
 @Date: 2020-06-12 00:48:57
 @LastEditor: John
-LastEditTime: 2021-04-29 02:02:12
+LastEditTime: 2021-04-29 22:23:38
 @Discription: 
 @Environment: python 3.7.7
 '''
@@ -36,21 +36,28 @@ class DQNConfig:
             '/'+curr_time+'/results/'  # path to save results
         self.model_path = curr_path+"/outputs/" + self.env + \
             '/'+curr_time+'/models/'  # path to save results
+        self.train_eps = 300  # 训练的episode数目
+        self.eval_eps = 50 # number of episodes for evaluating
         self.gamma = 0.95
-        self.epsilon_start = 1  # e-greedy策略的初始epsilon
+        self.epsilon_start = 0.90  # e-greedy策略的初始epsilon
         self.epsilon_end = 0.01
         self.epsilon_decay = 500
         self.lr = 0.0001  # learning rate
-        self.memory_capacity = 10000  # Replay Memory容量
-        self.batch_size = 32
-        self.train_eps = 300  # 训练的episode数目
+        self.memory_capacity = 100000  # Replay Memory容量
+        self.batch_size = 64
         self.target_update = 2  # target net的更新频率
-        self.eval_eps = 20  # 测试的episode数目
         self.device = torch.device(
             "cuda" if torch.cuda.is_available() else "cpu")  # 检测gpu
         self.hidden_dim = 256  # 神经网络隐藏层维度
 
-
+def env_agent_config(cfg,seed=1):
+    env = gym.make(cfg.env)  
+    env.seed(seed)
+    state_dim = env.observation_space.shape[0]
+    action_dim = env.action_space.n
+    agent = DQN(state_dim,action_dim,cfg)
+    return env,agent
+    
 def train(cfg, env, agent):
     print('Start to train !')
     print(f'Env:{cfg.env}, Algorithm:{cfg.algo}, Device:{cfg.device}')
@@ -60,13 +67,15 @@ def train(cfg, env, agent):
         state = env.reset()
         done = False
         ep_reward = 0
-        while not done:
+        while True:
             action = agent.choose_action(state)
             next_state, reward, done, _ = env.step(action)
             ep_reward += reward
             agent.memory.push(state, action, reward, next_state, done)
             state = next_state
             agent.update()
+            if done:
+                break
         if i_episode % cfg.target_update == 0:
             agent.target_net.load_state_dict(agent.policy_net.state_dict())
         print('Episode:{}/{}, Reward:{}'.format(i_episode+1, cfg.train_eps, ep_reward))
@@ -79,17 +88,39 @@ def train(cfg, env, agent):
     print('Complete training！')
     return rewards, ma_rewards
 
+def eval(cfg,env,agent):
+    rewards = []  # 记录所有episode的reward
+    ma_rewards = [] # 滑动平均的reward
+    for i_ep in range(cfg.eval_eps):
+        ep_reward = 0  # 记录每个episode的reward
+        state = env.reset()  # 重置环境, 重新开一局（即开始新的一个episode）
+        while True:
+            action = agent.predict(state)  # 根据算法选择一个动作
+            next_state, reward, done, _ = env.step(action)  # 与环境进行一个交互
+            state = next_state  # 存储上一个观察值
+            ep_reward += reward
+            if done:
+                break
+        rewards.append(ep_reward)
+        if ma_rewards:
+            ma_rewards.append(ma_rewards[-1]*0.9+ep_reward*0.1)
+        else:
+            ma_rewards.append(ep_reward)
+        print(f"Episode:{i_ep+1}/{cfg.eval_eps}, reward:{ep_reward:.1f}")
+    return rewards,ma_rewards
 
 if __name__ == "__main__":
     cfg = DQNConfig()
-    env = gym.make(cfg.env)
-    env.seed(1)
-    state_dim = env.observation_space.shape[0]
-    action_dim = env.action_space.n
-    agent = DQN(state_dim, action_dim, cfg)
+    env,agent = env_agent_config(cfg,seed=1)
     rewards, ma_rewards = train(cfg, env, agent)
     make_dir(cfg.result_path, cfg.model_path)
     agent.save(path=cfg.model_path)
     save_results(rewards, ma_rewards, tag='train', path=cfg.result_path)
     plot_rewards(rewards, ma_rewards, tag="train",
                  algo=cfg.algo, path=cfg.result_path)
+
+    env,agent = env_agent_config(cfg,seed=10)
+    agent.load(path=cfg.model_path)
+    rewards,ma_rewards = eval(cfg,env,agent)
+    save_results(rewards,ma_rewards,tag='eval',path=cfg.result_path)
+    plot_rewards(rewards,ma_rewards,tag="eval",env=cfg.env,algo = cfg.algo,path=cfg.result_path)