update sarsa

codes/Sarsa/agent.py

@@ -5,30 +5,37 @@ Author: John
 Email: johnjim0816@gmail.com
 Date: 2021-03-12 16:58:16
 LastEditor: John
-LastEditTime: 2021-03-13 11:02:50
+LastEditTime: 2022-04-24 21:14:23
 Discription: 
 Environment: 
 '''
 import numpy as np
 from collections import defaultdict
 import torch
+import math
 class Sarsa(object):
     def __init__(self,
-                 action_dim,sarsa_cfg,):
+                 n_actions,cfg,):
-        self.action_dim = action_dim  # number of actions
+        self.n_actions = n_actions  # number of actions
-        self.lr = sarsa_cfg.lr  # learning rate
+        self.lr = cfg.lr  # learning rate
-        self.gamma = sarsa_cfg.gamma  
+        self.gamma = cfg.gamma  
-        self.epsilon = sarsa_cfg.epsilon  
+        self.sample_count = 0 
-        self.Q  = defaultdict(lambda: np.zeros(action_dim))
+        self.epsilon_start = cfg.epsilon_start
-        # self.Q = np.zeros((state_dim, action_dim))  # Q表
+        self.epsilon_end = cfg.epsilon_end
+        self.epsilon_decay = cfg.epsilon_decay 
+        self.Q  = defaultdict(lambda: np.zeros(n_actions))
+        # self.Q = np.zeros((state_dim, n_actions))  # Q表
     def choose_action(self, state):
+        self.sample_count += 1
+        self.epsilon = self.epsilon_end + (self.epsilon_start - self.epsilon_end) * \
+            math.exp(-1. * self.sample_count / self.epsilon_decay) # The probability to select a random action, is is log decayed
         best_action = np.argmax(self.Q[state])
-        # action = best_action
+        action_probs = np.ones(self.n_actions, dtype=float) * self.epsilon / self.n_actions
-        action_probs = np.ones(self.action_dim, dtype=float) * self.epsilon / self.action_dim
         action_probs[best_action] += (1.0 - self.epsilon)
         action = np.random.choice(np.arange(len(action_probs)), p=action_probs) 
         return action
-            
+    def predict_action(self,state):
+        return np.argmax(self.Q[state])
     def update(self, state, action, reward, next_state, next_action,done):
         Q_predict = self.Q[state][action]
         if done:

codes/Sarsa/task0_train.py

@@ -5,61 +5,63 @@ Author: John
 Email: johnjim0816@gmail.com
 Date: 2021-03-11 17:59:16
 LastEditor: John
-LastEditTime: 2021-05-06 17:12:37
+LastEditTime: 2022-04-24 22:17:05
 Discription: 
 Environment: 
 '''
 import sys,os
-curr_path = os.path.dirname(__file__)
+curr_path = os.path.dirname(os.path.abspath(__file__)) # current path of file
 parent_path = os.path.dirname(curr_path)
 sys.path.append(parent_path)  # add current terminal path to sys.path
 
 import datetime
+import torch
 from envs.racetrack_env import RacetrackEnv
 from Sarsa.agent import Sarsa
-from common.plot import plot_rewards
+from common.utils import save_results,make_dir,plot_rewards
-from common.utils import save_results,make_dir
 
 curr_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") # obtain current time
 
-class SarsaConfig:
+class Config:
     ''' parameters for Sarsa
     '''
     def __init__(self):
-        self.algo = 'Qlearning'
+        self.algo_name = 'Qlearning'
-        self.env = 'CliffWalking-v0' # 0 up, 1 right, 2 down, 3 left
+        self.env_name = 'CliffWalking-v0' # 0 up, 1 right, 2 down, 3 left
-        self.result_path = curr_path+"/outputs/" +self.env+'/'+curr_time+'/results/'  # path to save results
+        self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # check GPU
-        self.model_path = curr_path+"/outputs/" +self.env+'/'+curr_time+'/models/'  # path to save models
+        self.result_path = curr_path+"/outputs/" +self.env_name+'/'+curr_time+'/results/'  # path to save results
-        self.train_eps = 200
+        self.model_path = curr_path+"/outputs/" +self.env_name+'/'+curr_time+'/models/'  # path to save models
-        self.test_eps = 50
+        self.train_eps = 300
-        self.epsilon = 0.15 # epsilon: The probability to select a random action . 
+        self.test_eps = 20
-        self.gamma = 0.9 # gamma: Gamma discount factor.
+        self.epsilon_start = 0.90  # start value of epsilon
+        self.epsilon_end = 0.01  # end value of epsilon
+        self.epsilon_decay = 200  # decay rate of epsilon
+        self.gamma = 0.99 # gamma: Gamma discount factor.
         self.lr = 0.2 # learning rate: step size parameter
-        self.n_steps = 2000
+        self.n_steps = 200
+        self.save = True # if save figures
 
 def env_agent_config(cfg,seed=1):
     env = RacetrackEnv()
-    action_dim=9
+    action_dim = 9
     agent = Sarsa(action_dim,cfg)
     return env,agent
         
 def train(cfg,env,agent):
     rewards = []
     ma_rewards = []
-    for i_episode in range(cfg.train_eps):
+    for i_ep in range(cfg.train_eps):
-        # Print out which episode we're on, useful for debugging.
-        # Generate an episode.
-        # An episode is an array of (state, action, reward) tuples
         state = env.reset()
+        action = agent.choose_action(state)
         ep_reward = 0
-        while True:
+        # while True:
-        # for t in range(cfg.n_steps):
+        for _ in range(cfg.n_steps):
-            action = agent.choose_action(state)
             next_state, reward, done = env.step(action)
             ep_reward+=reward
             next_action = agent.choose_action(next_state)
             agent.update(state, action, reward, next_state, next_action,done)
             state = next_state
+            action = next_action
             if done:
                 break  
         if ma_rewards:
@@ -67,22 +69,22 @@ def train(cfg,env,agent):
         else:
             ma_rewards.append(ep_reward)
         rewards.append(ep_reward)
-        if (i_episode+1)%10==0:
+        if (i_ep+1)%2==0:
-            print("Episode:{}/{}: Reward:{}".format(i_episode+1, cfg.train_eps,ep_reward))
+            print(f"Episode:{i_ep+1}, Reward:{ep_reward}, Epsilon:{agent.epsilon}")
     return rewards,ma_rewards
 
 def eval(cfg,env,agent):
     rewards = []
     ma_rewards = []
-    for i_episode in range(cfg.test_eps):
+    for i_ep in range(cfg.test_eps):
         # Print out which episode we're on, useful for debugging.
         # Generate an episode.
         # An episode is an array of (state, action, reward) tuples
         state = env.reset()
         ep_reward = 0
         while True:
-        # for t in range(cfg.n_steps):
+        # for _ in range(cfg.n_steps):
-            action = agent.choose_action(state)
+            action = agent.predict_action(state)
             next_state, reward, done = env.step(action)
             ep_reward+=reward
             state = next_state
@@ -93,25 +95,25 @@ def eval(cfg,env,agent):
         else:
             ma_rewards.append(ep_reward)
         rewards.append(ep_reward)
-        if (i_episode+1)%10==0:
+        if (i_ep+1)%1==0:
-            print("Episode:{}/{}: Reward:{}".format(i_episode+1, cfg.test_eps,ep_reward))
+            print("Episode:{}/{}: Reward:{}".format(i_ep+1, cfg.test_eps,ep_reward))
     print('Complete evaling！')
     return rewards,ma_rewards
         
 if __name__ == "__main__":
-    cfg = SarsaConfig()
+    cfg = Config()
     env,agent = env_agent_config(cfg,seed=1)
     rewards,ma_rewards = train(cfg,env,agent)
     make_dir(cfg.result_path,cfg.model_path)
     agent.save(path=cfg.model_path)
     save_results(rewards,ma_rewards,tag='train',path=cfg.result_path)
-    plot_rewards(rewards,ma_rewards,tag="train",env=cfg.env,algo = cfg.algo,path=cfg.result_path)
+    plot_rewards(rewards, ma_rewards, cfg, tag="train")
 
     env,agent = env_agent_config(cfg,seed=10)
     agent.load(path=cfg.model_path)
     rewards,ma_rewards = eval(cfg,env,agent)
-    save_results(rewards,ma_rewards,tag='eval',path=cfg.result_path)
+    save_results(rewards,ma_rewards,tag='test',path=cfg.result_path)
-    plot_rewards(rewards,ma_rewards,tag="eval",env=cfg.env,algo = cfg.algo,path=cfg.result_path)
+    plot_rewards(rewards, ma_rewards, cfg, tag="test")