diff --git a/codes/Sarsa/agent.py b/codes/Sarsa/agent.py index 020f6da..4ed885b 100644 --- a/codes/Sarsa/agent.py +++ b/codes/Sarsa/agent.py @@ -5,30 +5,37 @@ Author: John Email: johnjim0816@gmail.com Date: 2021-03-12 16:58:16 LastEditor: John -LastEditTime: 2021-03-13 11:02:50 +LastEditTime: 2022-04-24 21:14:23 Discription: Environment: ''' import numpy as np from collections import defaultdict import torch +import math class Sarsa(object): def __init__(self, - action_dim,sarsa_cfg,): - self.action_dim = action_dim # number of actions - self.lr = sarsa_cfg.lr # learning rate - self.gamma = sarsa_cfg.gamma - self.epsilon = sarsa_cfg.epsilon - self.Q = defaultdict(lambda: np.zeros(action_dim)) - # self.Q = np.zeros((state_dim, action_dim)) # Q表 + n_actions,cfg,): + self.n_actions = n_actions # number of actions + self.lr = cfg.lr # learning rate + self.gamma = cfg.gamma + self.sample_count = 0 + self.epsilon_start = cfg.epsilon_start + self.epsilon_end = cfg.epsilon_end + self.epsilon_decay = cfg.epsilon_decay + self.Q = defaultdict(lambda: np.zeros(n_actions)) + # self.Q = np.zeros((state_dim, n_actions)) # Q表 def choose_action(self, state): + self.sample_count += 1 + self.epsilon = self.epsilon_end + (self.epsilon_start - self.epsilon_end) * \ + math.exp(-1. * self.sample_count / self.epsilon_decay) # The probability to select a random action, is is log decayed best_action = np.argmax(self.Q[state]) - # action = best_action - action_probs = np.ones(self.action_dim, dtype=float) * self.epsilon / self.action_dim + action_probs = np.ones(self.n_actions, dtype=float) * self.epsilon / self.n_actions action_probs[best_action] += (1.0 - self.epsilon) action = np.random.choice(np.arange(len(action_probs)), p=action_probs) return action - + def predict_action(self,state): + return np.argmax(self.Q[state]) def update(self, state, action, reward, next_state, next_action,done): Q_predict = self.Q[state][action] if done: diff --git a/codes/Sarsa/outputs/CliffWalking-v0/20210506-171245/models/sarsa_model.pkl b/codes/Sarsa/outputs/CliffWalking-v0/20210506-171245/models/sarsa_model.pkl deleted file mode 100644 index ff25fd5..0000000 Binary files a/codes/Sarsa/outputs/CliffWalking-v0/20210506-171245/models/sarsa_model.pkl and /dev/null differ diff --git a/codes/Sarsa/outputs/CliffWalking-v0/20210506-171245/results/eval_ma_rewards.npy b/codes/Sarsa/outputs/CliffWalking-v0/20210506-171245/results/eval_ma_rewards.npy deleted file mode 100644 index d7d62e3..0000000 Binary files a/codes/Sarsa/outputs/CliffWalking-v0/20210506-171245/results/eval_ma_rewards.npy and /dev/null differ diff --git a/codes/Sarsa/outputs/CliffWalking-v0/20210506-171245/results/eval_rewards.npy b/codes/Sarsa/outputs/CliffWalking-v0/20210506-171245/results/eval_rewards.npy deleted file mode 100644 index de0a816..0000000 Binary files a/codes/Sarsa/outputs/CliffWalking-v0/20210506-171245/results/eval_rewards.npy and /dev/null differ diff --git a/codes/Sarsa/outputs/CliffWalking-v0/20210506-171245/results/eval_rewards_curve.png b/codes/Sarsa/outputs/CliffWalking-v0/20210506-171245/results/eval_rewards_curve.png deleted file mode 100644 index 3de2db7..0000000 Binary files a/codes/Sarsa/outputs/CliffWalking-v0/20210506-171245/results/eval_rewards_curve.png and /dev/null differ diff --git a/codes/Sarsa/outputs/CliffWalking-v0/20210506-171245/results/train_ma_rewards.npy b/codes/Sarsa/outputs/CliffWalking-v0/20210506-171245/results/train_ma_rewards.npy deleted file mode 100644 index 3f9bf83..0000000 Binary files a/codes/Sarsa/outputs/CliffWalking-v0/20210506-171245/results/train_ma_rewards.npy and /dev/null differ diff --git a/codes/Sarsa/outputs/CliffWalking-v0/20210506-171245/results/train_rewards.npy b/codes/Sarsa/outputs/CliffWalking-v0/20210506-171245/results/train_rewards.npy deleted file mode 100644 index e0fd7e9..0000000 Binary files a/codes/Sarsa/outputs/CliffWalking-v0/20210506-171245/results/train_rewards.npy and /dev/null differ diff --git a/codes/Sarsa/outputs/CliffWalking-v0/20210506-171245/results/train_rewards_curve.png b/codes/Sarsa/outputs/CliffWalking-v0/20210506-171245/results/train_rewards_curve.png deleted file mode 100644 index 0a8cd37..0000000 Binary files a/codes/Sarsa/outputs/CliffWalking-v0/20210506-171245/results/train_rewards_curve.png and /dev/null differ diff --git a/codes/Sarsa/outputs/CliffWalking-v0/20220424-221748/models/sarsa_model.pkl b/codes/Sarsa/outputs/CliffWalking-v0/20220424-221748/models/sarsa_model.pkl new file mode 100644 index 0000000..9973dc4 Binary files /dev/null and b/codes/Sarsa/outputs/CliffWalking-v0/20220424-221748/models/sarsa_model.pkl differ diff --git a/codes/Sarsa/outputs/CliffWalking-v0/20220424-221748/results/test_ma_rewards.npy b/codes/Sarsa/outputs/CliffWalking-v0/20220424-221748/results/test_ma_rewards.npy new file mode 100644 index 0000000..a77a41f Binary files /dev/null and b/codes/Sarsa/outputs/CliffWalking-v0/20220424-221748/results/test_ma_rewards.npy differ diff --git a/codes/Sarsa/outputs/CliffWalking-v0/20220424-221748/results/test_rewards.npy b/codes/Sarsa/outputs/CliffWalking-v0/20220424-221748/results/test_rewards.npy new file mode 100644 index 0000000..068a9de Binary files /dev/null and b/codes/Sarsa/outputs/CliffWalking-v0/20220424-221748/results/test_rewards.npy differ diff --git a/codes/Sarsa/outputs/CliffWalking-v0/20220424-221748/results/test_rewards_curve.png b/codes/Sarsa/outputs/CliffWalking-v0/20220424-221748/results/test_rewards_curve.png new file mode 100644 index 0000000..373d627 Binary files /dev/null and b/codes/Sarsa/outputs/CliffWalking-v0/20220424-221748/results/test_rewards_curve.png differ diff --git a/codes/Sarsa/outputs/CliffWalking-v0/20220424-221748/results/train_ma_rewards.npy b/codes/Sarsa/outputs/CliffWalking-v0/20220424-221748/results/train_ma_rewards.npy new file mode 100644 index 0000000..1aa168b Binary files /dev/null and b/codes/Sarsa/outputs/CliffWalking-v0/20220424-221748/results/train_ma_rewards.npy differ diff --git a/codes/Sarsa/outputs/CliffWalking-v0/20220424-221748/results/train_rewards.npy b/codes/Sarsa/outputs/CliffWalking-v0/20220424-221748/results/train_rewards.npy new file mode 100644 index 0000000..9924002 Binary files /dev/null and b/codes/Sarsa/outputs/CliffWalking-v0/20220424-221748/results/train_rewards.npy differ diff --git a/codes/Sarsa/outputs/CliffWalking-v0/20220424-221748/results/train_rewards_curve.png b/codes/Sarsa/outputs/CliffWalking-v0/20220424-221748/results/train_rewards_curve.png new file mode 100644 index 0000000..ad53c8d Binary files /dev/null and b/codes/Sarsa/outputs/CliffWalking-v0/20220424-221748/results/train_rewards_curve.png differ diff --git a/codes/Sarsa/task0_train.py b/codes/Sarsa/task0_train.py index e477afa..7e426df 100644 --- a/codes/Sarsa/task0_train.py +++ b/codes/Sarsa/task0_train.py @@ -5,61 +5,63 @@ Author: John Email: johnjim0816@gmail.com Date: 2021-03-11 17:59:16 LastEditor: John -LastEditTime: 2021-05-06 17:12:37 +LastEditTime: 2022-04-24 22:17:05 Discription: Environment: ''' import sys,os -curr_path = os.path.dirname(__file__) +curr_path = os.path.dirname(os.path.abspath(__file__)) # current path of file parent_path = os.path.dirname(curr_path) sys.path.append(parent_path) # add current terminal path to sys.path import datetime +import torch from envs.racetrack_env import RacetrackEnv from Sarsa.agent import Sarsa -from common.plot import plot_rewards -from common.utils import save_results,make_dir +from common.utils import save_results,make_dir,plot_rewards curr_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") # obtain current time -class SarsaConfig: +class Config: ''' parameters for Sarsa ''' def __init__(self): - self.algo = 'Qlearning' - self.env = 'CliffWalking-v0' # 0 up, 1 right, 2 down, 3 left - self.result_path = curr_path+"/outputs/" +self.env+'/'+curr_time+'/results/' # path to save results - self.model_path = curr_path+"/outputs/" +self.env+'/'+curr_time+'/models/' # path to save models - self.train_eps = 200 - self.test_eps = 50 - self.epsilon = 0.15 # epsilon: The probability to select a random action . - self.gamma = 0.9 # gamma: Gamma discount factor. + self.algo_name = 'Qlearning' + self.env_name = 'CliffWalking-v0' # 0 up, 1 right, 2 down, 3 left + self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # check GPU + self.result_path = curr_path+"/outputs/" +self.env_name+'/'+curr_time+'/results/' # path to save results + self.model_path = curr_path+"/outputs/" +self.env_name+'/'+curr_time+'/models/' # path to save models + self.train_eps = 300 + self.test_eps = 20 + self.epsilon_start = 0.90 # start value of epsilon + self.epsilon_end = 0.01 # end value of epsilon + self.epsilon_decay = 200 # decay rate of epsilon + self.gamma = 0.99 # gamma: Gamma discount factor. self.lr = 0.2 # learning rate: step size parameter - self.n_steps = 2000 + self.n_steps = 200 + self.save = True # if save figures def env_agent_config(cfg,seed=1): env = RacetrackEnv() - action_dim=9 + action_dim = 9 agent = Sarsa(action_dim,cfg) return env,agent def train(cfg,env,agent): rewards = [] ma_rewards = [] - for i_episode in range(cfg.train_eps): - # Print out which episode we're on, useful for debugging. - # Generate an episode. - # An episode is an array of (state, action, reward) tuples + for i_ep in range(cfg.train_eps): state = env.reset() + action = agent.choose_action(state) ep_reward = 0 - while True: - # for t in range(cfg.n_steps): - action = agent.choose_action(state) + # while True: + for _ in range(cfg.n_steps): next_state, reward, done = env.step(action) ep_reward+=reward next_action = agent.choose_action(next_state) agent.update(state, action, reward, next_state, next_action,done) state = next_state + action = next_action if done: break if ma_rewards: @@ -67,22 +69,22 @@ def train(cfg,env,agent): else: ma_rewards.append(ep_reward) rewards.append(ep_reward) - if (i_episode+1)%10==0: - print("Episode:{}/{}: Reward:{}".format(i_episode+1, cfg.train_eps,ep_reward)) + if (i_ep+1)%2==0: + print(f"Episode:{i_ep+1}, Reward:{ep_reward}, Epsilon:{agent.epsilon}") return rewards,ma_rewards def eval(cfg,env,agent): rewards = [] ma_rewards = [] - for i_episode in range(cfg.test_eps): + for i_ep in range(cfg.test_eps): # Print out which episode we're on, useful for debugging. # Generate an episode. # An episode is an array of (state, action, reward) tuples state = env.reset() ep_reward = 0 while True: - # for t in range(cfg.n_steps): - action = agent.choose_action(state) + # for _ in range(cfg.n_steps): + action = agent.predict_action(state) next_state, reward, done = env.step(action) ep_reward+=reward state = next_state @@ -93,25 +95,25 @@ def eval(cfg,env,agent): else: ma_rewards.append(ep_reward) rewards.append(ep_reward) - if (i_episode+1)%10==0: - print("Episode:{}/{}: Reward:{}".format(i_episode+1, cfg.test_eps,ep_reward)) + if (i_ep+1)%1==0: + print("Episode:{}/{}: Reward:{}".format(i_ep+1, cfg.test_eps,ep_reward)) print('Complete evaling！') return rewards,ma_rewards if __name__ == "__main__": - cfg = SarsaConfig() + cfg = Config() env,agent = env_agent_config(cfg,seed=1) rewards,ma_rewards = train(cfg,env,agent) make_dir(cfg.result_path,cfg.model_path) agent.save(path=cfg.model_path) save_results(rewards,ma_rewards,tag='train',path=cfg.result_path) - plot_rewards(rewards,ma_rewards,tag="train",env=cfg.env,algo = cfg.algo,path=cfg.result_path) + plot_rewards(rewards, ma_rewards, cfg, tag="train") env,agent = env_agent_config(cfg,seed=10) agent.load(path=cfg.model_path) rewards,ma_rewards = eval(cfg,env,agent) - save_results(rewards,ma_rewards,tag='eval',path=cfg.result_path) - plot_rewards(rewards,ma_rewards,tag="eval",env=cfg.env,algo = cfg.algo,path=cfg.result_path) + save_results(rewards,ma_rewards,tag='test',path=cfg.result_path) + plot_rewards(rewards, ma_rewards, cfg, tag="test")