bacow/easy-rl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update

Browse Source
This commit is contained in:
JohnJim0816
2021-04-04 16:59:03 +08:00
parent 14b66bfb16
commit 312b57fdff
8 changed files with 221 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
codes/DQN/main.py
View File

@@ -5,13 +5,11 @@
@Email: johnjim0816@gmail.com @Email: johnjim0816@gmail.com
@Date: 2020-06-12 00:48:57 @Date: 2020-06-12 00:48:57
@LastEditor: John @LastEditor: John
LastEditTime: 2021-03-30 16:59:19 LastEditTime: 2021-04-04 00:26:47
@Discription: @Discription:
@Environment: python 3.7.7 @Environment: python 3.7.7
''' '''
import sys,os import sys,os
from pathlib import Path
import sys,os
curr_path = os.path.dirname(__file__) curr_path = os.path.dirname(__file__)
parent_path=os.path.dirname(curr_path) parent_path=os.path.dirname(curr_path)
sys.path.append(parent_path) # add current terminal path to sys.path sys.path.append(parent_path) # add current terminal path to sys.path
@@ -21,19 +19,13 @@ import torch
import datetime import datetime
from DQN.agent import DQN from DQN.agent import DQN
from common.plot import plot_rewards from common.plot import plot_rewards
from common.utils import save_results from common.utils import save_results,make_dir,del_empty_dir
SEQUENCE = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") # obtain current time SEQUENCE = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") # obtain current time
SAVED_MODEL_PATH = curr_path+"/saved_model/"+SEQUENCE+'/' # path to save model SAVED_MODEL_PATH = curr_path+"/saved_model/"+SEQUENCE+'/' # path to save model
if not os.path.exists(curr_path+"/saved_model/"):
os.mkdir(curr_path+"/saved_model/")
if not os.path.exists(SAVED_MODEL_PATH):
os.mkdir(SAVED_MODEL_PATH)
RESULT_PATH = curr_path+"/results/"+SEQUENCE+'/' # path to save rewards RESULT_PATH = curr_path+"/results/"+SEQUENCE+'/' # path to save rewards
if not os.path.exists(curr_path+"/results/"): make_dir(curr_path+"/saved_model/",curr_path+"/results/")
os.mkdir(curr_path+"/results/") del_empty_dir(curr_path+"/saved_model/",curr_path+"/results/")
if not os.path.exists(RESULT_PATH):
os.mkdir(RESULT_PATH)
class DQNConfig: class DQNConfig:
def __init__(self): def __init__(self):
@@ -72,8 +64,7 @@ def train(cfg,env,agent):
rewards.append(ep_reward) rewards.append(ep_reward)
# 计算滑动窗口的reward # 计算滑动窗口的reward
if ma_rewards: if ma_rewards:
ma_rewards.append( ma_rewards.append(0.9*ma_rewards[-1]+0.1*ep_reward)
0.9*ma_rewards[-1]+0.1*ep_reward)
else: else:
ma_rewards.append(ep_reward) ma_rewards.append(ep_reward)
print('Complete training！') print('Complete training！')
@@ -87,6 +78,8 @@ if __name__ == "__main__":
action_dim = env.action_space.n action_dim = env.action_space.n
agent = DQN(state_dim,action_dim,cfg) agent = DQN(state_dim,action_dim,cfg)
rewards,ma_rewards = train(cfg,env,agent) rewards,ma_rewards = train(cfg,env,agent)
make_dir(SAVED_MODEL_PATH,RESULT_PATH)
agent.save(path=SAVED_MODEL_PATH) agent.save(path=SAVED_MODEL_PATH)
save_results(rewards,ma_rewards,tag='train',path=RESULT_PATH) save_results(rewards,ma_rewards,tag='train',path=RESULT_PATH)
plot_rewards(rewards,ma_rewards,tag="train",algo = cfg.algo,path=RESULT_PATH) plot_rewards(rewards,ma_rewards,tag="train",algo = cfg.algo,path=RESULT_PATH)
del_empty_dir(SAVED_MODEL_PATH,RESULT_PATH)

152
codes/QLearning/main.ipynb Normal file
View File

File diff suppressed because one or more lines are too long

2
codes/QLearning/main.py
View File

@@ -5,7 +5,7 @@ Author: John
Email: johnjim0816@gmail.com Email: johnjim0816@gmail.com
Date: 2020-09-11 23:03:00 Date: 2020-09-11 23:03:00
LastEditor: John LastEditor: John
LastEditTime: 2021-03-31 18:21:00 LastEditTime: 2021-03-31 18:14:59
Discription: Discription:
Environment: Environment:
''' '''

6
codes/README.md
View File

@@ -32,14 +32,14 @@ python 3.7、pytorch 1.6.0-1.7.1、gym 0.17.0-0.18.0
| [Sarsa](./Sarsa) | | [Racetrack](./envs/racetrack_env.md) | | | [Sarsa](./Sarsa) | | [Racetrack](./envs/racetrack_env.md) | |
| [DQN](./DQN) | [DQN Paper](https://www.cs.toronto.edu/~vmnih/docs/dqn.pdf) | [CartPole-v0](./envs/gym_info.md) | | | [DQN](./DQN) | [DQN Paper](https://www.cs.toronto.edu/~vmnih/docs/dqn.pdf) | [CartPole-v0](./envs/gym_info.md) | |
| [DQN-cnn](./DQN_cnn) | [DQN Paper](https://www.cs.toronto.edu/~vmnih/docs/dqn.pdf) | [CartPole-v0](./envs/gym_info.md) | 与DQN相比使用了CNN而不是全链接网络 | | [DQN-cnn](./DQN_cnn) | [DQN Paper](https://www.cs.toronto.edu/~vmnih/docs/dqn.pdf) | [CartPole-v0](./envs/gym_info.md) | 与DQN相比使用了CNN而不是全链接网络 |
| [DoubleDQN](./DoubleDQN) | | [CartPole-v0](./envs/gym_info.md) | 效果不好，待改进 | | [DoubleDQN](./DoubleDQN) | | [CartPole-v0](./envs/gym_info.md) | |
| Hierarchical DQN | [H-DQN Paper](https://arxiv.org/abs/1604.06057) | | | | [Hierarchical DQN](HierarchicalDQN) | [H-DQN Paper](https://arxiv.org/abs/1604.06057) | [CartPole-v0](./envs/gym_info.md) | |
| [PolicyGradient](./PolicyGradient) | | [CartPole-v0](./envs/gym_info.md) | | | [PolicyGradient](./PolicyGradient) | | [CartPole-v0](./envs/gym_info.md) | |
| A2C | | [CartPole-v0](./envs/gym_info.md) | | | A2C | | [CartPole-v0](./envs/gym_info.md) | |
| A3C | | | | | A3C | | | |
| SAC | | | | | SAC | | | |
| [PPO](./PPO) | [PPO paper](https://arxiv.org/abs/1707.06347) | [CartPole-v0](./envs/gym_info.md) | | | [PPO](./PPO) | [PPO paper](https://arxiv.org/abs/1707.06347) | [CartPole-v0](./envs/gym_info.md) | |
| DDPG | [DDPG Paper](https://arxiv.org/abs/1509.02971) | [Pendulum-v0](./envs/gym_info.md) | | | [DDPG](./DDPG) | [DDPG Paper](https://arxiv.org/abs/1509.02971) | [Pendulum-v0](./envs/gym_info.md) | |
| TD3 | [TD3 Paper](https://arxiv.org/abs/1802.09477) | | | | TD3 | [TD3 Paper](https://arxiv.org/abs/1802.09477) | | |
| GAIL | | | | | GAIL | | | |

36
codes/README_en.md
View File

@@ -15,8 +15,6 @@ The code structure mainly contains several scripts as following：
* ```agent.py``` core algorithms, include a python Class with functions(choose action, update) * ```agent.py``` core algorithms, include a python Class with functions(choose action, update)
* ```main.py``` main function * ```main.py``` main function
Note that ```model.py```,```memory.py```,```plot.py``` shall be utilized in different algorithms，thus they are put into ```common``` folder。 Note that ```model.py```,```memory.py```,```plot.py``` shall be utilized in different algorithms，thus they are put into ```common``` folder。
## Runnig Environment ## Runnig Environment
@@ -28,23 +26,23 @@ run ```main.py``` or ```main.ipynb```
## Schedule ## Schedule
| Name | Related materials | Used Envs | Notes | | Name | Related materials | Used Envs | Notes |
| :----------------------------------------------------------: | :---------------------------------------------------------: | ------------------------------------------------------------ | :----------------------------------------------------------: | | :--------------------------------------: | :---------------------------------------------------------: | ------------------------------------- | :------: |
| [On-Policy First-Visit MC](./MonteCarlo) | | [Racetrack](./envs/racetrack_env.md) | | | [On-Policy First-Visit MC](./MonteCarlo) | | [Racetrack](./envs/racetrack_env.md) | |
| [Q-Learning](./QLearning) | | [CliffWalking-v0](./envs/gym_info.md) | | | [Q-Learning](./QLearning) | | [CliffWalking-v0](./envs/gym_info.md) | |
| [Sarsa](./Sarsa) | | [Racetrack](./envs/racetrack_env.md) | | | [Sarsa](./Sarsa) | | [Racetrack](./envs/racetrack_env.md) | |
| [DQN](./DQN) | [DQN-paper](https://www.cs.toronto.edu/~vmnih/docs/dqn.pdf) | [CartPole-v0](./envs/gym_info.md) | | | [DQN](./DQN) | [DQN-paper](https://www.cs.toronto.edu/~vmnih/docs/dqn.pdf) | [CartPole-v0](./envs/gym_info.md) | |
| [DQN-cnn](./DQN_cnn) | [DQN-paper](https://www.cs.toronto.edu/~vmnih/docs/dqn.pdf) | [CartPole-v0](./envs/gym_info.md) | | | [DQN-cnn](./DQN_cnn) | [DQN-paper](https://www.cs.toronto.edu/~vmnih/docs/dqn.pdf) | [CartPole-v0](./envs/gym_info.md) | |
| [DoubleDQN](./DoubleDQN) | | [CartPole-v0](./envs/gym_info.md) | not well | | [DoubleDQN](./DoubleDQN) | | [CartPole-v0](./envs/gym_info.md) | not well |
| Hierarchical DQN | [Hierarchical DQN](https://arxiv.org/abs/1604.06057) | | | | [Hierarchical DQN](HierarchicalDQN) | [Hierarchical DQN](https://arxiv.org/abs/1604.06057) | [CartPole-v0](./envs/gym_info.md) | |
| [PolicyGradient](./PolicyGradient) | | [CartPole-v0](./envs/gym_info.md) | | | [PolicyGradient](./PolicyGradient) | | [CartPole-v0](./envs/gym_info.md) | |
| A2C | | [CartPole-v0](./envs/gym_info.md) | | | A2C | | [CartPole-v0](./envs/gym_info.md) | |
| A3C | | | | | A3C | | | |
| SAC | | | | | SAC | | | |
| [PPO](./PPO) | [PPO paper](https://arxiv.org/abs/1707.06347) | [CartPole-v0](./envs/gym_info.md) | | | [PPO](./PPO) | [PPO paper](https://arxiv.org/abs/1707.06347) | [CartPole-v0](./envs/gym_info.md) | |
| DDPG | [DDPG Paper](https://arxiv.org/abs/1509.02971) | [Pendulum-v0](./envs/gym_info.md) | | | [DDPG](./DDPG) | [DDPG Paper](https://arxiv.org/abs/1509.02971) | [Pendulum-v0](./envs/gym_info.md) | |
| TD3 | [Twin Dueling DDPG Paper](https://arxiv.org/abs/1802.09477) | | | | TD3 | [Twin Dueling DDPG Paper](https://arxiv.org/abs/1802.09477) | | |
| GAIL | | | | | GAIL | | | |
## Refs ## Refs

12
codes/common/plot.py
View File

@@ -5,28 +5,30 @@ Author: John
Email: johnjim0816@gmail.com Email: johnjim0816@gmail.com
Date: 2020-10-07 20:57:11 Date: 2020-10-07 20:57:11
LastEditor: John LastEditor: John
LastEditTime: 2021-03-31 14:05:52 LastEditTime: 2021-03-31 18:47:28
Discription: Discription:
Environment: Environment:
''' '''
import matplotlib.pyplot as plt import matplotlib.pyplot as plt
import seaborn as sns import seaborn as sns
def plot_rewards(rewards,ma_rewards,tag="train",algo = "DQN",path='./'): def plot_rewards(rewards,ma_rewards,tag="train",algo = "DQN",save=True,path='./'):
sns.set() sns.set()
plt.title("average learning curve of {}".format(algo)) plt.title("average learning curve of {}".format(algo))
plt.xlabel('epsiodes') plt.xlabel('epsiodes')
plt.plot(rewards,label='rewards') plt.plot(rewards,label='rewards')
plt.plot(ma_rewards,label='moving average rewards') plt.plot(ma_rewards,label='moving average rewards')
plt.legend() plt.legend()
plt.savefig(path+"rewards_curve_{}".format(tag)) if save:
plt.savefig(path+"rewards_curve_{}".format(tag))
plt.show() plt.show()
def plot_losses(losses,algo = "DQN",path='./'): def plot_losses(losses,algo = "DQN",save=True,path='./'):
sns.set() sns.set()
plt.title("loss curve of {}".format(algo)) plt.title("loss curve of {}".format(algo))
plt.xlabel('epsiodes') plt.xlabel('epsiodes')
plt.plot(losses,label='rewards') plt.plot(losses,label='rewards')
plt.legend() plt.legend()
plt.savefig(path+"losses_curve") if save:
plt.savefig(path+"losses_curve")
plt.show() plt.show()

17
codes/common/utils.py
View File

@@ -5,7 +5,7 @@ Author: John
Email: johnjim0816@gmail.com Email: johnjim0816@gmail.com
Date: 2021-03-12 16:02:24 Date: 2021-03-12 16:02:24
LastEditor: John LastEditor: John
LastEditTime: 2021-03-12 16:10:28 LastEditTime: 2021-04-03 21:42:13
Discription: Discription:
Environment: Environment:
''' '''
@@ -18,4 +18,17 @@ def save_results(rewards,ma_rewards,tag='train',path='./results'):
''' '''
np.save(path+'rewards_'+tag+'.npy', rewards) np.save(path+'rewards_'+tag+'.npy', rewards)
np.save(path+'ma_rewards_'+tag+'.npy', ma_rewards) np.save(path+'ma_rewards_'+tag+'.npy', ma_rewards)
print('results saved!') print('results saved!')
def make_dir(*paths):
for path in paths:
if not os.path.exists(path): # check if exists
os.mkdir(path)
def del_empty_dir(*paths):
'''del_empty_dir delete empty folders unders "paths"
'''
for path in paths:
dirs = os.listdir(path)
for dir in dirs:
if not os.listdir(os.path.join(path, dir)):
os.removedirs(os.path.join(path, dir))

19
codes/test.py Normal file
View File

@@ -0,0 +1,19 @@
#!/usr/bin/env python
# coding=utf-8
'''
Author: John
Email: johnjim0816@gmail.com
Date: 2021-03-25 23:25:15
LastEditor: John
LastEditTime: 2021-03-26 16:46:52
Discription:
Environment:
'''
from collections import defaultdict
import numpy as np
action_dim = 2
Q_table = defaultdict(lambda: np.zeros(action_dim))
Q_table[str(0)] = 1
print(Q_table[str(0)])
Q_table[str(21)] = 3
print(Q_table[str(21)])