update projects

projects/codes/Sarsa/README.md

@@ -0,0 +1,19 @@
+# Sarsa
+
+## 使用说明
+
+运行```main.py```即可
+
+## 环境说明
+
+见[环境说明](https://github.com/JohnJim0816/reinforcement-learning-tutorials/blob/master/env_info.md)中的The Racetrack
+
+## 算法伪代码
+
+![sarsa_algo](assets/sarsa_algo.png)
+
+## 其他说明
+
+### 与Q-learning区别
+
+算法上区别很小，只在更新公式上，但Q-learning是Off-policy，而Sarsa是On-policy，可参考[知乎：强化学习中sarsa算法是不是比q-learning算法收敛速度更慢？](https://www.zhihu.com/question/268461866)

projects/codes/Sarsa/sarsa.py

@@ -0,0 +1,58 @@
+#!/usr/bin/env python
+# coding=utf-8
+'''
+Author: John
+Email: johnjim0816@gmail.com
+Date: 2021-03-12 16:58:16
+LastEditor: John
+LastEditTime: 2022-04-29 20:12:57
+Discription: 
+Environment: 
+'''
+import numpy as np
+from collections import defaultdict
+import torch
+import math
+class Sarsa(object):
+    def __init__(self,
+                 n_actions,cfg,):
+        self.n_actions = n_actions  
+        self.lr = cfg.lr  
+        self.gamma = cfg.gamma  
+        self.sample_count = 0 
+        self.epsilon_start = cfg.epsilon_start
+        self.epsilon_end = cfg.epsilon_end
+        self.epsilon_decay = cfg.epsilon_decay 
+        self.Q  = defaultdict(lambda: np.zeros(n_actions)) # Q table
+    def choose_action(self, state):
+        self.sample_count += 1
+        self.epsilon = self.epsilon_end + (self.epsilon_start - self.epsilon_end) * \
+            math.exp(-1. * self.sample_count / self.epsilon_decay) # The probability to select a random action, is is log decayed
+        best_action = np.argmax(self.Q[state])
+        action_probs = np.ones(self.n_actions, dtype=float) * self.epsilon / self.n_actions
+        action_probs[best_action] += (1.0 - self.epsilon)
+        action = np.random.choice(np.arange(len(action_probs)), p=action_probs) 
+        return action
+    def predict_action(self,state):
+        return np.argmax(self.Q[state])
+    def update(self, state, action, reward, next_state, next_action,done):
+        Q_predict = self.Q[state][action]
+        if done:
+            Q_target = reward  # terminal state
+        else:
+            Q_target = reward + self.gamma * self.Q[next_state][next_action] 
+        self.Q[state][action] += self.lr * (Q_target - Q_predict) 
+    def save(self,path):
+        '''把 Q表格 的数据保存到文件中
+        '''
+        import dill
+        torch.save(
+            obj=self.Q,
+            f=path+"sarsa_model.pkl",
+            pickle_module=dill
+        )
+    def load(self, path):
+        '''从文件中读取数据到 Q表格
+        '''
+        import dill
+        self.Q =torch.load(f=path+'sarsa_model.pkl',pickle_module=dill)

projects/codes/Sarsa/task0.py

@@ -0,0 +1,119 @@
+#!/usr/bin/env python
+# coding=utf-8
+'''
+Author: John
+Email: johnjim0816@gmail.com
+Date: 2021-03-11 17:59:16
+LastEditor: John
+LastEditTime: 2022-04-29 20:18:13
+Discription: 
+Environment: 
+'''
+import sys,os
+curr_path = os.path.dirname(os.path.abspath(__file__)) # current path of file
+parent_path = os.path.dirname(curr_path)
+sys.path.append(parent_path)  # add current terminal path to sys.path
+
+import datetime
+import torch
+from envs.racetrack_env import RacetrackEnv
+from Sarsa.sarsa import Sarsa
+from common.utils import save_results,make_dir,plot_rewards
+
+curr_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") # obtain current time
+
+class Config:
+    ''' parameters for Sarsa
+    '''
+    def __init__(self):
+        self.algo_name = 'Qlearning'
+        self.env_name = 'CliffWalking-v0' # 0 up, 1 right, 2 down, 3 left
+        self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # check GPU
+        self.result_path = curr_path+"/outputs/" +self.env_name+'/'+curr_time+'/results/'  # path to save results
+        self.model_path = curr_path+"/outputs/" +self.env_name+'/'+curr_time+'/models/'  # path to save models
+        self.train_eps = 300 # training episodes
+        self.test_eps = 20 # testing episodes
+        self.n_steps = 200 # maximum steps per episode 
+        self.epsilon_start = 0.90  # start value of epsilon
+        self.epsilon_end = 0.01  # end value of epsilon
+        self.epsilon_decay = 200  # decay rate of epsilon
+        self.gamma = 0.99 # gamma: Gamma discount factor.
+        self.lr = 0.2 # learning rate: step size parameter 
+        self.save = True # if save figures
+
+def env_agent_config(cfg,seed=1):
+    env = RacetrackEnv()
+    n_states = 9 # number of actions
+    agent = Sarsa(n_states,cfg)
+    return env,agent
+        
+def train(cfg,env,agent):
+    rewards = []
+    ma_rewards = []
+    for i_ep in range(cfg.train_eps):
+        state = env.reset()
+        action = agent.choose_action(state)
+        ep_reward = 0
+        # while True:
+        for _ in range(cfg.n_steps):
+            next_state, reward, done = env.step(action)
+            ep_reward+=reward
+            next_action = agent.choose_action(next_state)
+            agent.update(state, action, reward, next_state, next_action,done)
+            state = next_state
+            action = next_action
+            if done:
+                break  
+        if ma_rewards:
+            ma_rewards.append(ma_rewards[-1]*0.9+ep_reward*0.1)
+        else:
+            ma_rewards.append(ep_reward)
+        rewards.append(ep_reward)
+        if (i_ep+1)%2==0:
+            print(f"Episode:{i_ep+1}, Reward:{ep_reward}, Epsilon:{agent.epsilon}")
+    return rewards,ma_rewards
+
+def test(cfg,env,agent):
+    rewards = []
+    ma_rewards = []
+    for i_ep in range(cfg.test_eps):
+        # Print out which episode we're on, useful for debugging.
+        # Generate an episode.
+        # An episode is an array of (state, action, reward) tuples
+        state = env.reset()
+        ep_reward = 0
+        while True:
+        # for _ in range(cfg.n_steps):
+            action = agent.predict_action(state)
+            next_state, reward, done = env.step(action)
+            ep_reward+=reward
+            state = next_state
+            if done:
+                break  
+        if ma_rewards:
+            ma_rewards.append(ma_rewards[-1]*0.9+ep_reward*0.1)
+        else:
+            ma_rewards.append(ep_reward)
+        rewards.append(ep_reward)
+        if (i_ep+1)%1==0:
+            print("Episode:{}/{}: Reward:{}".format(i_ep+1, cfg.test_eps,ep_reward))
+    print('Complete testing！')
+    return rewards,ma_rewards
+        
+if __name__ == "__main__":
+    cfg = Config()
+    env,agent = env_agent_config(cfg,seed=1)
+    rewards,ma_rewards = train(cfg,env,agent)
+    make_dir(cfg.result_path,cfg.model_path)
+    agent.save(path=cfg.model_path)
+    save_results(rewards,ma_rewards,tag='train',path=cfg.result_path)
+    plot_rewards(rewards, ma_rewards, cfg, tag="train")
+
+    env,agent = env_agent_config(cfg,seed=10)
+    agent.load(path=cfg.model_path)
+    rewards,ma_rewards = test(cfg,env,agent)
+    save_results(rewards,ma_rewards,tag='test',path=cfg.result_path)
+    plot_rewards(rewards, ma_rewards, cfg, tag="test")
+    
+    
+