update projects

projects/codes/Sarsa/outputs/CliffWalking-v0/20220803-142740/results/params.json

@@ -0,0 +1 @@
+{"algo_name": "Sarsa", "env_name": "CliffWalking-v0", "train_eps": 300, "test_eps": 20, "ep_max_steps": 200, "gamma": 0.99, "epsilon_start": 0.9, "epsilon_end": 0.01, "epsilon_decay": 200, "lr": 0.2, "device": "cpu", "result_path": "/Users/jj/Desktop/rl-tutorials/codes/Sarsa/outputs/CliffWalking-v0/20220803-142740/results/", "model_path": "/Users/jj/Desktop/rl-tutorials/codes/Sarsa/outputs/CliffWalking-v0/20220803-142740/models/", "save_fig": true}

projects/codes/Sarsa/outputs/CliffWalking-v0/20220804-223029/results/params.json

@@ -0,0 +1,15 @@
+{
+    "algo_name": "Sarsa",
+    "env_name": "CliffWalking-v0",
+    "train_eps": 400,
+    "test_eps": 20,
+    "gamma": 0.9,
+    "epsilon_start": 0.95,
+    "epsilon_end": 0.01,
+    "epsilon_decay": 300,
+    "lr": 0.1,
+    "device": "cpu",
+    "result_path": "c:\\Users\\24438\\Desktop\\rl-tutorials\\codes\\Sarsa/outputs/CliffWalking-v0/20220804-223029/results/",
+    "model_path": "c:\\Users\\24438\\Desktop\\rl-tutorials\\codes\\Sarsa/outputs/CliffWalking-v0/20220804-223029/models/",
+    "save_fig": true
+}

projects/codes/Sarsa/sarsa.py

@@ -5,7 +5,7 @@ Author: John
 Email: johnjim0816@gmail.com
 Date: 2021-03-12 16:58:16
 LastEditor: John
-LastEditTime: 2022-04-29 20:12:57
+LastEditTime: 2022-08-04 22:22:16
 Discription: 
 Environment: 
 '''
@@ -15,7 +15,7 @@ import torch
 import math
 class Sarsa(object):
     def __init__(self,
-                 n_actions,cfg,):
+                 n_actions,cfg):
         self.n_actions = n_actions  
         self.lr = cfg.lr  
         self.gamma = cfg.gamma  
@@ -24,7 +24,7 @@ class Sarsa(object):
         self.epsilon_end = cfg.epsilon_end
         self.epsilon_decay = cfg.epsilon_decay 
         self.Q  = defaultdict(lambda: np.zeros(n_actions)) # Q table
-    def choose_action(self, state):
+    def sample(self, state):
         self.sample_count += 1
         self.epsilon = self.epsilon_end + (self.epsilon_start - self.epsilon_end) * \
             math.exp(-1. * self.sample_count / self.epsilon_decay) # The probability to select a random action, is is log decayed
@@ -33,14 +33,14 @@ class Sarsa(object):
         action_probs[best_action] += (1.0 - self.epsilon)
         action = np.random.choice(np.arange(len(action_probs)), p=action_probs) 
         return action
-    def predict_action(self,state):
+    def predict(self,state):
         return np.argmax(self.Q[state])
     def update(self, state, action, reward, next_state, next_action,done):
         Q_predict = self.Q[state][action]
         if done:
-            Q_target = reward  # terminal state
+            Q_target = reward  # 终止状态
         else:
-            Q_target = reward + self.gamma * self.Q[next_state][next_action] 
+            Q_target = reward + self.gamma * self.Q[next_state][next_action] # 与Q learning不同，Sarsa是拿下一步动作对应的Q值去更新
         self.Q[state][action] += self.lr * (Q_target - Q_predict) 
     def save(self,path):
         '''把 Q表格 的数据保存到文件中

projects/codes/Sarsa/task0.py

@@ -5,115 +5,114 @@ Author: John
 Email: johnjim0816@gmail.com
 Date: 2021-03-11 17:59:16
 LastEditor: John
-LastEditTime: 2022-04-29 20:18:13
+LastEditTime: 2022-08-04 22:28:51
 Discription: 
 Environment: 
 '''
 import sys,os
-curr_path = os.path.dirname(os.path.abspath(__file__)) # current path of file
-parent_path = os.path.dirname(curr_path)
-sys.path.append(parent_path)  # add current terminal path to sys.path
+curr_path = os.path.dirname(os.path.abspath(__file__))  # 当前文件所在绝对路径
+parent_path = os.path.dirname(curr_path)  # 父路径
+sys.path.append(parent_path)  # 添加路径到系统路径
 
 import datetime
-import torch
+import argparse
 from envs.racetrack_env import RacetrackEnv
 from Sarsa.sarsa import Sarsa
-from common.utils import save_results,make_dir,plot_rewards
+from common.utils import save_results,make_dir,plot_rewards,save_args
 
-curr_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") # obtain current time
+def get_args():
+    """ 超参数
+    """
+    curr_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")  # 获取当前时间
+    parser = argparse.ArgumentParser(description="hyperparameters")      
+    parser.add_argument('--algo_name',default='Sarsa',type=str,help="name of algorithm")
+    parser.add_argument('--env_name',default='CliffWalking-v0',type=str,help="name of environment")
+    parser.add_argument('--train_eps',default=300,type=int,help="episodes of training") # 训练的回合数
+    parser.add_argument('--test_eps',default=20,type=int,help="episodes of testing") # 测试的回合数
+    parser.add_argument('--ep_max_steps',default=200,type=int) # 每回合最大的部署
+    parser.add_argument('--gamma',default=0.99,type=float,help="discounted factor") # 折扣因子
+    parser.add_argument('--epsilon_start',default=0.90,type=float,help="initial value of epsilon") #  e-greedy策略中初始epsilon
+    parser.add_argument('--epsilon_end',default=0.01,type=float,help="final value of epsilon") # e-greedy策略中的终止epsilon
+    parser.add_argument('--epsilon_decay',default=200,type=int,help="decay rate of epsilon") # e-greedy策略中epsilon的衰减率
+    parser.add_argument('--lr',default=0.2,type=float,help="learning rate")
+    parser.add_argument('--device',default='cpu',type=str,help="cpu or cuda") 
+    parser.add_argument('--result_path',default=curr_path + "/outputs/" + parser.parse_args().env_name + \
+            '/' + curr_time + '/results/' )
+    parser.add_argument('--model_path',default=curr_path + "/outputs/" + parser.parse_args().env_name + \
+            '/' + curr_time + '/models/' ) # path to save models
+    parser.add_argument('--save_fig',default=True,type=bool,help="if save figure or not")           
+    args = parser.parse_args()                          
+    return args
 
-class Config:
-    ''' parameters for Sarsa
-    '''
-    def __init__(self):
-        self.algo_name = 'Qlearning'
-        self.env_name = 'CliffWalking-v0' # 0 up, 1 right, 2 down, 3 left
-        self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # check GPU
-        self.result_path = curr_path+"/outputs/" +self.env_name+'/'+curr_time+'/results/'  # path to save results
-        self.model_path = curr_path+"/outputs/" +self.env_name+'/'+curr_time+'/models/'  # path to save models
-        self.train_eps = 300 # training episodes
-        self.test_eps = 20 # testing episodes
-        self.n_steps = 200 # maximum steps per episode 
-        self.epsilon_start = 0.90  # start value of epsilon
-        self.epsilon_end = 0.01  # end value of epsilon
-        self.epsilon_decay = 200  # decay rate of epsilon
-        self.gamma = 0.99 # gamma: Gamma discount factor.
-        self.lr = 0.2 # learning rate: step size parameter 
-        self.save = True # if save figures
 
 def env_agent_config(cfg,seed=1):
     env = RacetrackEnv()
-    n_states = 9 # number of actions
-    agent = Sarsa(n_states,cfg)
+    n_actions = 9 # 动作数
+    agent = Sarsa(n_actions,cfg)
     return env,agent
         
 def train(cfg,env,agent):
-    rewards = []
-    ma_rewards = []
+    print('开始训练！')
+    print(f'环境:{cfg.env_name}, 算法:{cfg.algo_name}, 设备:{cfg.device}')
+    rewards = []  # 记录奖励
     for i_ep in range(cfg.train_eps):
         state = env.reset()
-        action = agent.choose_action(state)
+        action = agent.sample(state)
         ep_reward = 0
         # while True:
-        for _ in range(cfg.n_steps):
+        for _ in range(cfg.ep_max_steps):
             next_state, reward, done = env.step(action)
             ep_reward+=reward
-            next_action = agent.choose_action(next_state)
+            next_action = agent.sample(next_state)
             agent.update(state, action, reward, next_state, next_action,done)
             state = next_state
             action = next_action
             if done:
                 break  
-        if ma_rewards:
-            ma_rewards.append(ma_rewards[-1]*0.9+ep_reward*0.1)
-        else:
-            ma_rewards.append(ep_reward)
         rewards.append(ep_reward)
         if (i_ep+1)%2==0:
-            print(f"Episode:{i_ep+1}, Reward:{ep_reward}, Epsilon:{agent.epsilon}")
-    return rewards,ma_rewards
+            print(f"回合：{i_ep+1}/{cfg.train_eps}，奖励：{ep_reward:.1f}，Epsilon：{agent.epsilon}")
+    print('完成训练！')
+    return {"rewards":rewards}
 
 def test(cfg,env,agent):
+    print('开始测试！')
+    print(f'环境：{cfg.env_name}, 算法：{cfg.algo_name}, 设备：{cfg.device}')
     rewards = []
-    ma_rewards = []
     for i_ep in range(cfg.test_eps):
-        # Print out which episode we're on, useful for debugging.
-        # Generate an episode.
-        # An episode is an array of (state, action, reward) tuples
         state = env.reset()
         ep_reward = 0
-        while True:
-        # for _ in range(cfg.n_steps):
-            action = agent.predict_action(state)
+        # while True:
+        for _ in range(cfg.ep_max_steps):
+            action = agent.predict(state)
             next_state, reward, done = env.step(action)
             ep_reward+=reward
             state = next_state
             if done:
                 break  
-        if ma_rewards:
-            ma_rewards.append(ma_rewards[-1]*0.9+ep_reward*0.1)
-        else:
-            ma_rewards.append(ep_reward)
         rewards.append(ep_reward)
-        if (i_ep+1)%1==0:
-            print("Episode:{}/{}: Reward:{}".format(i_ep+1, cfg.test_eps,ep_reward))
-    print('Complete testing！')
-    return rewards,ma_rewards
+        print(f"回合数：{i_ep+1}/{cfg.test_eps}, 奖励：{ep_reward:.1f}")
+    print('完成测试！')
+    return {"rewards":rewards}
         
 if __name__ == "__main__":
-    cfg = Config()
-    env,agent = env_agent_config(cfg,seed=1)
-    rewards,ma_rewards = train(cfg,env,agent)
-    make_dir(cfg.result_path,cfg.model_path)
-    agent.save(path=cfg.model_path)
-    save_results(rewards,ma_rewards,tag='train',path=cfg.result_path)
-    plot_rewards(rewards, ma_rewards, cfg, tag="train")
-
-    env,agent = env_agent_config(cfg,seed=10)
-    agent.load(path=cfg.model_path)
-    rewards,ma_rewards = test(cfg,env,agent)
-    save_results(rewards,ma_rewards,tag='test',path=cfg.result_path)
-    plot_rewards(rewards, ma_rewards, cfg, tag="test")
+    cfg = get_args()
+    # 训练
+    env, agent = env_agent_config(cfg)
+    res_dic = train(cfg, env, agent)
+    make_dir(cfg.result_path, cfg.model_path)  
+    save_args(cfg) # save parameters
+    agent.save(path=cfg.model_path)  # save model
+    save_results(res_dic, tag='train',
+                 path=cfg.result_path)  
+    plot_rewards(res_dic['rewards'], cfg, tag="train")  
+    # 测试
+    env, agent = env_agent_config(cfg)
+    agent.load(path=cfg.model_path)  # 导入模型
+    res_dic = test(cfg, env, agent)
+    save_results(res_dic, tag='test',
+                 path=cfg.result_path)  # 保存结果
+    plot_rewards(res_dic['rewards'], cfg, tag="test")  # 画出结果
     
     
 

projects/codes/Sarsa/task1.py

@@ -0,0 +1,131 @@
+#!/usr/bin/env python
+# coding=utf-8
+'''
+Author: John
+Email: johnjim0816@gmail.com
+Date: 2020-09-11 23:03:00
+LastEditor: John
+LastEditTime: 2022-08-04 22:44:00
+Discription: 
+Environment: 
+'''
+import sys
+import os
+curr_path = os.path.dirname(os.path.abspath(__file__))  # 当前文件所在绝对路径
+parent_path = os.path.dirname(curr_path)  # 父路径
+sys.path.append(parent_path)  # 添加路径到系统路径
+
+import gym
+import torch
+import datetime
+import argparse
+from envs.gridworld_env import CliffWalkingWapper
+from Sarsa.sarsa import Sarsa
+from common.utils import plot_rewards,save_args
+from common.utils import save_results,make_dir
+
+
+def get_args():
+    """ 
+    """
+    curr_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")  # 获取当前时间
+    parser = argparse.ArgumentParser(description="hyperparameters")      
+    parser.add_argument('--algo_name',default='Sarsa',type=str,help="name of algorithm")
+    parser.add_argument('--env_name',default='CliffWalking-v0',type=str,help="name of environment")
+    parser.add_argument('--train_eps',default=400,type=int,help="episodes of training") # 训练的回合数
+    parser.add_argument('--test_eps',default=20,type=int,help="episodes of testing") # 测试的回合数
+    parser.add_argument('--gamma',default=0.90,type=float,help="discounted factor") # 折扣因子
+    parser.add_argument('--epsilon_start',default=0.95,type=float,help="initial value of epsilon") #  e-greedy策略中初始epsilon
+    parser.add_argument('--epsilon_end',default=0.01,type=float,help="final value of epsilon") # e-greedy策略中的终止epsilon
+    parser.add_argument('--epsilon_decay',default=300,type=int,help="decay rate of epsilon") # e-greedy策略中epsilon的衰减率
+    parser.add_argument('--lr',default=0.1,type=float,help="learning rate")
+    parser.add_argument('--device',default='cpu',type=str,help="cpu or cuda") 
+    parser.add_argument('--result_path',default=curr_path + "/outputs/" + parser.parse_args().env_name + \
+            '/' + curr_time + '/results/' )
+    parser.add_argument('--model_path',default=curr_path + "/outputs/" + parser.parse_args().env_name + \
+            '/' + curr_time + '/models/' ) # path to save models
+    parser.add_argument('--save_fig',default=True,type=bool,help="if save figure or not")           
+    args = parser.parse_args([])                          
+    return args
+curr_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") # 获取当前时间
+      
+def train(cfg,env,agent):
+    print('开始训练！')
+    print(f'环境:{cfg.env_name}, 算法:{cfg.algo_name}, 设备:{cfg.device}')
+    rewards = []  # 记录奖励
+    for i_ep in range(cfg.train_eps):
+        ep_reward = 0  # 记录每个回合的奖励
+        state = env.reset()  # 重置环境,即开始新的回合
+        action = agent.sample(state)
+        while True:
+            action = agent.sample(state)  # 根据算法采样一个动作
+            next_state, reward, done, _ = env.step(action)  # 与环境进行一次动作交互
+            next_action = agent.sample(next_state)
+            agent.update(state, action, reward, next_state, next_action,done) # 算法更新
+            state = next_state # 更新状态
+            action = next_action
+            ep_reward += reward
+            if done:
+                break
+        rewards.append(ep_reward)
+        print(f"回合：{i_ep+1}/{cfg.train_eps}，奖励：{ep_reward:.1f}，Epsilon：{agent.epsilon}")
+    print('完成训练！')
+    return {"rewards":rewards}
+    
+def test(cfg,env,agent):
+    print('开始测试！')
+    print(f'环境：{cfg.env_name}, 算法：{cfg.algo_name}, 设备：{cfg.device}')
+    rewards = []  # 记录所有回合的奖励
+    for i_ep in range(cfg.test_eps):
+        ep_reward = 0  # 记录每个episode的reward
+        state = env.reset()  # 重置环境, 重新开一局（即开始新的一个回合）
+        while True:
+            action = agent.predict(state)  # 根据算法选择一个动作
+            next_state, reward, done, _ = env.step(action)  # 与环境进行一个交互
+            state = next_state  # 更新状态
+            ep_reward += reward
+            if done:
+                break
+        rewards.append(ep_reward)
+        print(f"回合数：{i_ep+1}/{cfg.test_eps}, 奖励：{ep_reward:.1f}")
+    print('完成测试！')
+    return {"rewards":rewards}
+        
+def env_agent_config(cfg,seed=1):
+    '''创建环境和智能体
+    Args:
+        cfg ([type]): [description]
+        seed (int, optional): 随机种子. Defaults to 1.
+    Returns:
+        env [type]: 环境
+        agent : 智能体
+    '''    
+    env = gym.make(cfg.env_name)  
+    env = CliffWalkingWapper(env)
+    env.seed(seed) # 设置随机种子
+    n_states = env.observation_space.n # 状态维度
+    n_actions = env.action_space.n # 动作维度
+    print(f"状态数：{n_states}，动作数：{n_actions}")
+    agent = Sarsa(n_actions,cfg)
+    return env,agent
+if __name__ == "__main__":
+    cfg = get_args()
+    # 训练
+    env, agent = env_agent_config(cfg)
+    res_dic = train(cfg, env, agent)
+    make_dir(cfg.result_path, cfg.model_path)  
+    save_args(cfg) # save parameters
+    agent.save(path=cfg.model_path)  # save model
+    save_results(res_dic, tag='train',
+                 path=cfg.result_path)  
+    plot_rewards(res_dic['rewards'], cfg, tag="train")  
+    # 测试
+    env, agent = env_agent_config(cfg)
+    agent.load(path=cfg.model_path)  # 导入模型
+    res_dic = test(cfg, env, agent)
+    save_results(res_dic, tag='test',
+                 path=cfg.result_path)  # 保存结果
+    plot_rewards(res_dic['rewards'], cfg, tag="test")  # 画出结果
+
+        
+