update

codes/QLearning/agent.py

@@ -5,7 +5,7 @@ Author: John
 Email: johnjim0816@gmail.com
 Date: 2020-09-11 23:03:00
 LastEditor: John
-LastEditTime: 2021-03-12 16:48:25
+LastEditTime: 2021-03-26 16:51:01
 Discription: 
 Environment: 
 '''
@@ -16,39 +16,35 @@ from collections import defaultdict
 
 class QLearning(object):
     def __init__(self,
-                 n_actions,cfg):
-        self.n_actions = n_actions  # number of actions
+                 action_dim,cfg):
+        self.action_dim = action_dim  # dimension of acgtion
         self.lr = cfg.lr  # learning rate
         self.gamma = cfg.gamma  
         self.epsilon = 0 
-        self.sample_count = 0  # epsilon随训练的也就是采样次数逐渐衰减，所以需要计数
+        self.sample_count = 0  
         self.epsilon_start = cfg.epsilon_start
         self.epsilon_end = cfg.epsilon_end
         self.epsilon_decay = cfg.epsilon_decay
-        self.Q_table  = defaultdict(lambda: np.zeros(n_actions)) # 使用字典存储Q表，个人比较喜欢这种，也可以用下面一行的二维数组表示，但是需要额外更改代码
-        # self.Q_table = np.zeros((n_states, n_actions))  # Q表
+        self.Q_table  = defaultdict(lambda: np.zeros(action_dim)) # A nested dictionary that maps state -> (action -> action-value)
     def choose_action(self, state):
         self.sample_count += 1
         self.epsilon = self.epsilon_end + (self.epsilon_start - self.epsilon_end) * \
             math.exp(-1. * self.sample_count / self.epsilon_decay)
-        # 随机选取0-1之间的值，如果大于epsilon就按照贪心策略选取action，否则随机选取
+        # e-greedy policy
         if np.random.uniform(0, 1) > self.epsilon:
-            action = np.argmax(self.Q_table[state])
+            action = np.argmax(self.Q_table[str(state)])
         else:
-            action = np.random.choice(self.n_actions)  # 有一定概率随机探索选取一个动作
+            action = np.random.choice(self.action_dim) 
         return action
             
     def update(self, state, action, reward, next_state, done):
-        Q_predict = self.Q_table[state][action]
+        Q_predict = self.Q_table[str(state)][action]
         if done:
             Q_target = reward  # terminal state
         else:
-            Q_target = reward + self.gamma * np.max(
-                self.Q_table[next_state])  # Q_table-learning
-        self.Q_table[state][action] += self.lr * (Q_target - Q_predict)
+            Q_target = reward + self.gamma * np.max(self.Q_table[str(next_state)]) 
+        self.Q_table[str(state)][action] += self.lr * (Q_target - Q_predict)
     def save(self,path):
-        '''把 Q表格 的数据保存到文件中
-        '''
         import dill
         torch.save(
             obj=self.Q_table,
@@ -56,7 +52,5 @@ class QLearning(object):
             pickle_module=dill
         )
     def load(self, path):
-        '''从文件中读取数据到 Q表格
-        '''
         import dill
         self.Q_table =torch.load(f=path+'Qleaning_model.pkl',pickle_module=dill)

codes/QLearning/main.py

@@ -5,7 +5,7 @@ Author: John
 Email: johnjim0816@gmail.com
 Date: 2020-09-11 23:03:00
 LastEditor: John
-LastEditTime: 2021-03-12 21:16:50
+LastEditTime: 2021-03-26 17:16:07
 Discription: 
 Environment: 
 '''
@@ -35,20 +35,18 @@ if not os.path.exists(RESULT_PATH): # 检测是否存在文件夹
 class QlearningConfig:
     '''训练相关参数'''
     def __init__(self):
-        self.n_episodes = 200 # 训练的episode数目
+        self.train_eps = 200 # 训练的episode数目
         self.gamma = 0.9 # reward的衰减率
         self.epsilon_start = 0.99 # e-greedy策略中初始epsilon
         self.epsilon_end = 0.01 # e-greedy策略中的终止epsilon
         self.epsilon_decay = 200 # e-greedy策略中epsilon的衰减率
-        self.lr = 0.1 # 学习率
+        self.lr = 0.1 # learning rate
 
 def train(cfg,env,agent):
-    # env = gym.make("FrozenLake-v0", is_slippery=False)  # 0 left, 1 down, 2 right, 3 up
-    # env = FrozenLakeWapper(env)
-    rewards = []  # 记录所有episode的reward
-    ma_rewards = [] # 滑动平均的reward
+    rewards = []  
+    ma_rewards = [] # moving average reward
     steps = []  # 记录所有episode的steps
-    for i_episode in range(cfg.n_episodes):
+    for i_episode in range(cfg.train_eps):
         ep_reward = 0  # 记录每个episode的reward
         ep_steps = 0  # 记录每个episode走了多少step
         state = env.reset()  # 重置环境, 重新开一局（即开始新的一个episode）
@@ -63,12 +61,11 @@ def train(cfg,env,agent):
                 break
         steps.append(ep_steps)
         rewards.append(ep_reward)
-        # 计算滑动平均的reward
         if ma_rewards:
             ma_rewards.append(ma_rewards[-1]*0.9+ep_reward*0.1)
         else:
             ma_rewards.append(ep_reward)
-        print("Episode:{}/{}: reward:{:.1f}".format(i_episode+1, cfg.n_episodes,ep_reward))
+        print("Episode:{}/{}: reward:{:.1f}".format(i_episode+1, cfg.train_eps,ep_reward))
     return rewards,ma_rewards
 
 def eval(cfg,env,agent):
@@ -77,7 +74,7 @@ def eval(cfg,env,agent):
     rewards = []  # 记录所有episode的reward
     ma_rewards = [] # 滑动平均的reward
     steps = []  # 记录所有episode的steps
-    for i_episode in range(cfg.n_episodes):
+    for i_episode in range(cfg.train_eps):
         ep_reward = 0  # 记录每个episode的reward
         ep_steps = 0  # 记录每个episode走了多少step
         state = env.reset()  # 重置环境, 重新开一局（即开始新的一个episode）
@@ -96,15 +93,15 @@ def eval(cfg,env,agent):
             ma_rewards.append(rewards[-1]*0.9+ep_reward*0.1)
         else:
             ma_rewards.append(ep_reward)
-        print("Episode:{}/{}: reward:{:.1f}".format(i_episode+1, cfg.n_episodes,ep_reward))
+        print("Episode:{}/{}: reward:{:.1f}".format(i_episode+1, cfg.train_eps,ep_reward))
     return rewards,ma_rewards
     
 if __name__ == "__main__":
     cfg = QlearningConfig()
     env = gym.make("CliffWalking-v0")  # 0 up, 1 right, 2 down, 3 left
     env = CliffWalkingWapper(env)
-    n_actions = env.action_space.n
-    agent = QLearning(n_actions,cfg)
+    action_dim = env.action_space.n
+    agent = QLearning(action_dim,cfg)
     rewards,ma_rewards = train(cfg,env,agent)
     agent.save(path=SAVED_MODEL_PATH)
     save_results(rewards,ma_rewards,tag='train',path=RESULT_PATH)