diff --git a/codes/QLearning/task0_train.py b/codes/QLearning/task0_train.py index 73fedae..f036162 100644 --- a/codes/QLearning/task0_train.py +++ b/codes/QLearning/task0_train.py @@ -5,7 +5,7 @@ Author: John Email: johnjim0816@gmail.com Date: 2020-09-11 23:03:00 LastEditor: John -LastEditTime: 2021-04-29 17:01:08 +LastEditTime: 2021-05-06 17:04:38 Discription: Environment: ''' @@ -15,6 +15,7 @@ parent_path=os.path.dirname(curr_path) sys.path.append(parent_path) # add current terminal path to sys.path import gym +import torch import datetime from envs.gridworld_env import CliffWalkingWapper @@ -37,6 +38,8 @@ class QlearningConfig: self.epsilon_end = 0.01 # e-greedy策略中的终止epsilon self.epsilon_decay = 200 # e-greedy策略中epsilon的衰减率 self.lr = 0.1 # learning rate + self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # check gpu + def env_agent_config(cfg,seed=1): env = gym.make(cfg.env) @@ -48,6 +51,8 @@ def env_agent_config(cfg,seed=1): return env,agent def train(cfg,env,agent): + print('Start to train !') + print(f'Env:{cfg.env}, Algorithm:{cfg.algo}, Device:{cfg.device}') rewards = [] ma_rewards = [] # moving average reward for i_ep in range(cfg.train_eps): @@ -67,11 +72,14 @@ def train(cfg,env,agent): else: ma_rewards.append(ep_reward) print("Episode:{}/{}: reward:{:.1f}".format(i_ep+1, cfg.train_eps,ep_reward)) + print('Complete training！') return rewards,ma_rewards def eval(cfg,env,agent): # env = gym.make("FrozenLake-v0", is_slippery=False) # 0 left, 1 down, 2 right, 3 up # env = FrozenLakeWapper(env) + print('Start to eval !') + print(f'Env:{cfg.env}, Algorithm:{cfg.algo}, Device:{cfg.device}') rewards = [] # 记录所有episode的reward ma_rewards = [] # 滑动平均的reward for i_ep in range(cfg.eval_eps): @@ -90,6 +98,7 @@ def eval(cfg,env,agent): else: ma_rewards.append(ep_reward) print(f"Episode:{i_ep+1}/{cfg.eval_eps}, reward:{ep_reward:.1f}") + print('Complete evaling！') return rewards,ma_rewards if __name__ == "__main__": diff --git a/codes/SAC/task0_train.py b/codes/SAC/task0_train.py index 8e2e025..1996b01 100644 --- a/codes/SAC/task0_train.py +++ b/codes/SAC/task0_train.py @@ -5,12 +5,10 @@ Author: JiangJi Email: johnjim0816@gmail.com Date: 2021-04-29 12:59:22 LastEditor: JiangJi -LastEditTime: 2021-05-06 01:47:36 +LastEditTime: 2021-05-06 16:58:01 Discription: Environment: ''' - - import sys,os curr_path = os.path.dirname(__file__) parent_path = os.path.dirname(curr_path) diff --git a/codes/Sarsa/main.py b/codes/Sarsa/main.py deleted file mode 100644 index a2363ed..0000000 --- a/codes/Sarsa/main.py +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/env python -# coding=utf-8 -''' -Author: John -Email: johnjim0816@gmail.com -Date: 2021-03-11 17:59:16 -LastEditor: John -LastEditTime: 2021-03-12 17:01:43 -Discription: -Environment: -''' -import sys,os -sys.path.append(os.getcwd()) -import datetime -from envs.racetrack_env import RacetrackEnv -from Sarsa.agent import Sarsa -from common.plot import plot_rewards -from common.utils import save_results - -SEQUENCE = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") # 获取当前时间 -SAVED_MODEL_PATH = os.path.split(os.path.abspath(__file__))[0]+"/saved_model/"+SEQUENCE+'/' # 生成保存的模型路径 -if not os.path.exists(os.path.split(os.path.abspath(__file__))[0]+"/saved_model/"): # 检测是否存在文件夹 - os.mkdir(os.path.split(os.path.abspath(__file__))[0]+"/saved_model/") -if not os.path.exists(SAVED_MODEL_PATH): # 检测是否存在文件夹 - os.mkdir(SAVED_MODEL_PATH) -RESULT_PATH = os.path.split(os.path.abspath(__file__))[0]+"/results/"+SEQUENCE+'/' # 存储reward的路径 -if not os.path.exists(os.path.split(os.path.abspath(__file__))[0]+"/results/"): # 检测是否存在文件夹 - os.mkdir(os.path.split(os.path.abspath(__file__))[0]+"/results/") -if not os.path.exists(RESULT_PATH): # 检测是否存在文件夹 - os.mkdir(RESULT_PATH) - -class SarsaConfig: - ''' parameters for Sarsa - ''' - def __init__(self): - self.epsilon = 0.15 # epsilon: The probability to select a random action . - self.gamma = 0.9 # gamma: Gamma discount factor. - self.lr = 0.2 # learning rate: step size parameter - self.n_episodes = 150 - self.n_steps = 2000 - -def sarsa_train(cfg,env,agent): - rewards = [] - ma_rewards = [] - for i_episode in range(cfg.n_episodes): - # Print out which episode we're on, useful for debugging. - # Generate an episode. - # An episode is an array of (state, action, reward) tuples - state = env.reset() - ep_reward = 0 - while True: - # for t in range(cfg.n_steps): - action = agent.choose_action(state) - next_state, reward, done = env.step(action) - ep_reward+=reward - next_action = agent.choose_action(next_state) - agent.update(state, action, reward, next_state, next_action,done) - state = next_state - if done: - break - if ma_rewards: - ma_rewards.append(ma_rewards[-1]*0.9+ep_reward*0.1) - else: - ma_rewards.append(ep_reward) - rewards.append(ep_reward) - # if (i_episode+1)%10==0: - # print("Episode:{}/{}: Reward:{}".format(i_episode+1, cfg.n_episodes,ep_reward)) - return rewards,ma_rewards - -if __name__ == "__main__": - sarsa_cfg = SarsaConfig() - env = RacetrackEnv() - action_dim=9 - agent = Sarsa(action_dim,sarsa_cfg) - rewards,ma_rewards = sarsa_train(sarsa_cfg,env,agent) - agent.save(path=SAVED_MODEL_PATH) - save_results(rewards,ma_rewards,tag='train',path=RESULT_PATH) - plot_rewards(rewards,ma_rewards,tag="train",algo = "On-Policy First-Visit MC Control",path=RESULT_PATH) - - diff --git a/codes/Sarsa/outputs/CliffWalking-v0/20210506-171245/models/sarsa_model.pkl b/codes/Sarsa/outputs/CliffWalking-v0/20210506-171245/models/sarsa_model.pkl new file mode 100644 index 0000000..ff25fd5 Binary files /dev/null and b/codes/Sarsa/outputs/CliffWalking-v0/20210506-171245/models/sarsa_model.pkl differ diff --git a/codes/Sarsa/outputs/CliffWalking-v0/20210506-171245/results/eval_ma_rewards.npy b/codes/Sarsa/outputs/CliffWalking-v0/20210506-171245/results/eval_ma_rewards.npy new file mode 100644 index 0000000..d7d62e3 Binary files /dev/null and b/codes/Sarsa/outputs/CliffWalking-v0/20210506-171245/results/eval_ma_rewards.npy differ diff --git a/codes/Sarsa/outputs/CliffWalking-v0/20210506-171245/results/eval_rewards.npy b/codes/Sarsa/outputs/CliffWalking-v0/20210506-171245/results/eval_rewards.npy new file mode 100644 index 0000000..de0a816 Binary files /dev/null and b/codes/Sarsa/outputs/CliffWalking-v0/20210506-171245/results/eval_rewards.npy differ diff --git a/codes/Sarsa/outputs/CliffWalking-v0/20210506-171245/results/eval_rewards_curve.png b/codes/Sarsa/outputs/CliffWalking-v0/20210506-171245/results/eval_rewards_curve.png new file mode 100644 index 0000000..3de2db7 Binary files /dev/null and b/codes/Sarsa/outputs/CliffWalking-v0/20210506-171245/results/eval_rewards_curve.png differ diff --git a/codes/Sarsa/outputs/CliffWalking-v0/20210506-171245/results/train_ma_rewards.npy b/codes/Sarsa/outputs/CliffWalking-v0/20210506-171245/results/train_ma_rewards.npy new file mode 100644 index 0000000..3f9bf83 Binary files /dev/null and b/codes/Sarsa/outputs/CliffWalking-v0/20210506-171245/results/train_ma_rewards.npy differ diff --git a/codes/Sarsa/outputs/CliffWalking-v0/20210506-171245/results/train_rewards.npy b/codes/Sarsa/outputs/CliffWalking-v0/20210506-171245/results/train_rewards.npy new file mode 100644 index 0000000..e0fd7e9 Binary files /dev/null and b/codes/Sarsa/outputs/CliffWalking-v0/20210506-171245/results/train_rewards.npy differ diff --git a/codes/Sarsa/outputs/CliffWalking-v0/20210506-171245/results/train_rewards_curve.png b/codes/Sarsa/outputs/CliffWalking-v0/20210506-171245/results/train_rewards_curve.png new file mode 100644 index 0000000..0a8cd37 Binary files /dev/null and b/codes/Sarsa/outputs/CliffWalking-v0/20210506-171245/results/train_rewards_curve.png differ diff --git a/codes/Sarsa/results/20210313-110256/ma_rewards_train.npy b/codes/Sarsa/results/20210313-110256/ma_rewards_train.npy deleted file mode 100644 index 943a6d4..0000000 Binary files a/codes/Sarsa/results/20210313-110256/ma_rewards_train.npy and /dev/null differ diff --git a/codes/Sarsa/results/20210313-110256/rewards_curve_train.png b/codes/Sarsa/results/20210313-110256/rewards_curve_train.png deleted file mode 100644 index ea31886..0000000 Binary files a/codes/Sarsa/results/20210313-110256/rewards_curve_train.png and /dev/null differ diff --git a/codes/Sarsa/results/20210313-110256/rewards_train.npy b/codes/Sarsa/results/20210313-110256/rewards_train.npy deleted file mode 100644 index d0702e8..0000000 Binary files a/codes/Sarsa/results/20210313-110256/rewards_train.npy and /dev/null differ diff --git a/codes/Sarsa/saved_model/20210313-110256/sarsa_model.pkl b/codes/Sarsa/saved_model/20210313-110256/sarsa_model.pkl deleted file mode 100644 index d19971c..0000000 Binary files a/codes/Sarsa/saved_model/20210313-110256/sarsa_model.pkl and /dev/null differ diff --git a/codes/Sarsa/task0_train.py b/codes/Sarsa/task0_train.py new file mode 100644 index 0000000..d21db17 --- /dev/null +++ b/codes/Sarsa/task0_train.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python +# coding=utf-8 +''' +Author: John +Email: johnjim0816@gmail.com +Date: 2021-03-11 17:59:16 +LastEditor: John +LastEditTime: 2021-05-06 17:12:37 +Discription: +Environment: +''' +import sys,os +curr_path = os.path.dirname(__file__) +parent_path = os.path.dirname(curr_path) +sys.path.append(parent_path) # add current terminal path to sys.path + +import datetime +from envs.racetrack_env import RacetrackEnv +from Sarsa.agent import Sarsa +from common.plot import plot_rewards +from common.utils import save_results,make_dir + +curr_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") # obtain current time + +class SarsaConfig: + ''' parameters for Sarsa + ''' + def __init__(self): + self.algo = 'Qlearning' + self.env = 'CliffWalking-v0' # 0 up, 1 right, 2 down, 3 left + self.result_path = curr_path+"/outputs/" +self.env+'/'+curr_time+'/results/' # path to save results + self.model_path = curr_path+"/outputs/" +self.env+'/'+curr_time+'/models/' # path to save models + self.train_eps = 200 + self.eval_eps = 50 + self.epsilon = 0.15 # epsilon: The probability to select a random action . + self.gamma = 0.9 # gamma: Gamma discount factor. + self.lr = 0.2 # learning rate: step size parameter + self.n_steps = 2000 + +def env_agent_config(cfg,seed=1): + env = RacetrackEnv() + action_dim=9 + agent = Sarsa(action_dim,cfg) + return env,agent + +def train(cfg,env,agent): + rewards = [] + ma_rewards = [] + for i_episode in range(cfg.train_eps): + # Print out which episode we're on, useful for debugging. + # Generate an episode. + # An episode is an array of (state, action, reward) tuples + state = env.reset() + ep_reward = 0 + while True: + # for t in range(cfg.n_steps): + action = agent.choose_action(state) + next_state, reward, done = env.step(action) + ep_reward+=reward + next_action = agent.choose_action(next_state) + agent.update(state, action, reward, next_state, next_action,done) + state = next_state + if done: + break + if ma_rewards: + ma_rewards.append(ma_rewards[-1]*0.9+ep_reward*0.1) + else: + ma_rewards.append(ep_reward) + rewards.append(ep_reward) + if (i_episode+1)%10==0: + print("Episode:{}/{}: Reward:{}".format(i_episode+1, cfg.train_eps,ep_reward)) + return rewards,ma_rewards + +def eval(cfg,env,agent): + rewards = [] + ma_rewards = [] + for i_episode in range(cfg.eval_eps): + # Print out which episode we're on, useful for debugging. + # Generate an episode. + # An episode is an array of (state, action, reward) tuples + state = env.reset() + ep_reward = 0 + while True: + # for t in range(cfg.n_steps): + action = agent.choose_action(state) + next_state, reward, done = env.step(action) + ep_reward+=reward + state = next_state + if done: + break + if ma_rewards: + ma_rewards.append(ma_rewards[-1]*0.9+ep_reward*0.1) + else: + ma_rewards.append(ep_reward) + rewards.append(ep_reward) + if (i_episode+1)%10==0: + print("Episode:{}/{}: Reward:{}".format(i_episode+1, cfg.eval_eps,ep_reward)) + print('Complete evaling！') + return rewards,ma_rewards + +if __name__ == "__main__": + cfg = SarsaConfig() + env,agent = env_agent_config(cfg,seed=1) + rewards,ma_rewards = train(cfg,env,agent) + make_dir(cfg.result_path,cfg.model_path) + agent.save(path=cfg.model_path) + save_results(rewards,ma_rewards,tag='train',path=cfg.result_path) + plot_rewards(rewards,ma_rewards,tag="train",env=cfg.env,algo = cfg.algo,path=cfg.result_path) + + env,agent = env_agent_config(cfg,seed=10) + agent.load(path=cfg.model_path) + rewards,ma_rewards = eval(cfg,env,agent) + save_results(rewards,ma_rewards,tag='eval',path=cfg.result_path) + plot_rewards(rewards,ma_rewards,tag="eval",env=cfg.env,algo = cfg.algo,path=cfg.result_path) + + +