update projects

projects/codes/MonteCarlo/README.md

@@ -0,0 +1,5 @@
+# *On-Policy First-Visit MC Control*
+
+### 伪代码
+
+![mc_control_algo](assets/mc_control_algo.png)

projects/codes/MonteCarlo/agent.py

@@ -0,0 +1,65 @@
+#!/usr/bin/env python
+# coding=utf-8
+'''
+Author: John
+Email: johnjim0816@gmail.com
+Date: 2021-03-12 16:14:34
+LastEditor: John
+LastEditTime: 2021-05-05 16:58:39
+Discription: 
+Environment: 
+'''
+import numpy as np
+from collections import defaultdict
+import torch
+import dill
+
+class FisrtVisitMC:
+    ''' On-Policy First-Visit MC Control
+    '''
+    def __init__(self,n_actions,cfg):
+        self.n_actions = n_actions
+        self.epsilon = cfg.epsilon
+        self.gamma = cfg.gamma 
+        self.Q_table = defaultdict(lambda: np.zeros(n_actions))
+        self.returns_sum = defaultdict(float) # sum of returns
+        self.returns_count = defaultdict(float)
+        
+    def choose_action(self,state):
+        ''' e-greed policy '''
+        if state in self.Q_table.keys():
+            best_action = np.argmax(self.Q_table[state])
+            action_probs = np.ones(self.n_actions, dtype=float) * self.epsilon / self.n_actions
+            action_probs[best_action] += (1.0 - self.epsilon)
+            action = np.random.choice(np.arange(len(action_probs)), p=action_probs)
+        else:
+            action = np.random.randint(0,self.n_actions)
+        return action
+    def update(self,one_ep_transition):
+        # Find all (state, action) pairs we've visited in this one_ep_transition
+        # We convert each state to a tuple so that we can use it as a dict key
+        sa_in_episode = set([(tuple(x[0]), x[1]) for x in one_ep_transition])
+        for state, action in sa_in_episode:
+            sa_pair = (state, action)
+            # Find the first occurence of the (state, action) pair in the one_ep_transition
+            first_occurence_idx = next(i for i,x in enumerate(one_ep_transition)
+                                       if x[0] == state and x[1] == action)
+            # Sum up all rewards since the first occurance
+            G = sum([x[2]*(self.gamma**i) for i,x in enumerate(one_ep_transition[first_occurence_idx:])])
+            # Calculate average return for this state over all sampled episodes
+            self.returns_sum[sa_pair] += G
+            self.returns_count[sa_pair] += 1.0
+            self.Q_table[state][action] = self.returns_sum[sa_pair] / self.returns_count[sa_pair]
+    def save(self,path):
+        '''把 Q表格 的数据保存到文件中
+        '''
+        torch.save(
+            obj=self.Q_table,
+            f=path+"Q_table",
+            pickle_module=dill
+        )
+
+    def load(self, path):
+        '''从文件中读取数据到 Q表格
+        '''
+        self.Q_table =torch.load(f=path+"Q_table",pickle_module=dill)

projects/codes/MonteCarlo/task0_train.py

@@ -0,0 +1,118 @@
+#!/usr/bin/env python
+# coding=utf-8
+'''
+Author: John
+Email: johnjim0816@gmail.com
+Date: 2021-03-11 14:26:44
+LastEditor: John
+LastEditTime: 2021-05-05 17:27:50
+Discription: 
+Environment: 
+'''
+
+import sys,os
+curr_path = os.path.dirname(__file__)
+parent_path = os.path.dirname(curr_path)
+sys.path.append(parent_path)  # add current terminal path to sys.path
+
+import torch
+import datetime
+
+from common.utils import save_results,make_dir
+from common.plot import plot_rewards
+from MonteCarlo.agent import FisrtVisitMC
+from envs.racetrack_env import RacetrackEnv
+
+curr_time = datetime.datetime.now().strftime(
+    "%Y%m%d-%H%M%S")  # obtain current time
+
+class MCConfig:
+    def __init__(self):
+        self.algo = "MC"  # name of algo
+        self.env = 'Racetrack'
+        self.result_path = curr_path+"/outputs/" + self.env + \
+            '/'+curr_time+'/results/'  # path to save results
+        self.model_path = curr_path+"/outputs/" + self.env + \
+            '/'+curr_time+'/models/'  # path to save models
+        # epsilon: The probability to select a random action .
+        self.epsilon = 0.15
+        self.gamma = 0.9  # gamma: Gamma discount factor.
+        self.train_eps = 200
+        self.device = torch.device(
+            "cuda" if torch.cuda.is_available() else "cpu")  # check gpu
+
+def env_agent_config(cfg,seed=1):
+    env = RacetrackEnv()
+    n_actions = 9
+    agent = FisrtVisitMC(n_actions, cfg)
+    return env,agent
+    
+def train(cfg, env, agent):
+    print('Start to eval !')
+    print(f'Env:{cfg.env}, Algorithm:{cfg.algo}, Device:{cfg.device}')
+    rewards = []
+    ma_rewards = []  # moving average rewards
+    for i_ep in range(cfg.train_eps):
+        state = env.reset()
+        ep_reward = 0
+        one_ep_transition = []
+        while True:
+            action = agent.choose_action(state)
+            next_state, reward, done = env.step(action)
+            ep_reward += reward
+            one_ep_transition.append((state, action, reward))
+            state = next_state
+            if done:
+                break
+        rewards.append(ep_reward)
+        if ma_rewards:
+            ma_rewards.append(ma_rewards[-1]*0.9+ep_reward*0.1)
+        else:
+            ma_rewards.append(ep_reward)
+        agent.update(one_ep_transition)
+        if (i_ep+1) % 10 == 0:
+            print(f"Episode:{i_ep+1}/{cfg.train_eps}: Reward:{ep_reward}")
+    print('Complete training！')
+    return rewards, ma_rewards
+
+def eval(cfg, env, agent):
+    print('Start to eval !')
+    print(f'Env:{cfg.env}, Algorithm:{cfg.algo}, Device:{cfg.device}')
+    rewards = []
+    ma_rewards = []  # moving average rewards
+    for i_ep in range(cfg.train_eps):
+        state = env.reset()
+        ep_reward = 0
+        while True:
+            action = agent.choose_action(state)
+            next_state, reward, done = env.step(action)
+            ep_reward += reward
+            state = next_state
+            if done:
+                break
+        rewards.append(ep_reward)
+        if ma_rewards:
+            ma_rewards.append(ma_rewards[-1]*0.9+ep_reward*0.1)
+        else:
+            ma_rewards.append(ep_reward)
+        if (i_ep+1) % 10 == 0:
+            print(f"Episode:{i_ep+1}/{cfg.train_eps}: Reward:{ep_reward}")
+    return rewards, ma_rewards
+    
+if __name__ == "__main__":
+    cfg = MCConfig()
+    
+    # train
+    env,agent = env_agent_config(cfg,seed=1)
+    rewards, ma_rewards = train(cfg, env, agent)
+    make_dir(cfg.result_path, cfg.model_path)
+    agent.save(path=cfg.model_path)
+    save_results(rewards, ma_rewards, tag='train', path=cfg.result_path)
+    plot_rewards(rewards, ma_rewards, tag="train",
+                 algo=cfg.algo, path=cfg.result_path)
+    # eval
+    env,agent = env_agent_config(cfg,seed=10)
+    agent.load(path=cfg.model_path)
+    rewards,ma_rewards = eval(cfg,env,agent)
+    save_results(rewards,ma_rewards,tag='eval',path=cfg.result_path)
+    plot_rewards(rewards,ma_rewards,tag="eval",env=cfg.env,algo = cfg.algo,path=cfg.result_path)