update projects
This commit is contained in:
johnjim0816
142
projects/codes/PPO/README.md
Normal file
142
projects/codes/PPO/README.md
Normal file
@@ -0,0 +1,142 @@
|
||||
## 原理简介
|
||||
|
||||
PPO是一种on-policy算法，具有较好的性能，其前身是TRPO算法，也是policy gradient算法的一种，它是现在 OpenAI 默认的强化学习算法，具体原理可参考[PPO算法讲解](https://datawhalechina.github.io/easy-rl/#/chapter5/chapter5)。PPO算法主要有两个变种，一个是结合KL penalty的，一个是用了clip方法，本文实现的是后者即```PPO-clip```。
|
||||
## 伪代码
|
||||
要实现必先了解伪代码，伪代码如下：
|
||||
|
||||
这是谷歌找到的一张比较适合的图，本人比较懒就没有修改，上面的```k```就是第```k```个episode，第六步是用随机梯度下降的方法优化，这里的损失函数(即```argmax```后面的部分)可能有点难理解，可参考[PPO paper](https://arxiv.org/abs/1707.06347)，如下：
|
||||
|
||||
第七步就是一个平方损失函数，即实际回报与期望回报的差平方。
|
||||
## 代码实战
|
||||
[点击查看完整代码](https://github.com/JohnJim0816/rl-tutorials/tree/master/PPO)
|
||||
### PPOmemory
|
||||
首先第三步需要搜集一条轨迹信息，我们可以定义一个```PPOmemory```来存储相关信息：
|
||||
```python
|
||||
class PPOMemory:
|
||||
def __init__(self, batch_size):
|
||||
self.states = []
|
||||
self.probs = []
|
||||
self.vals = []
|
||||
self.actions = []
|
||||
self.rewards = []
|
||||
self.dones = []
|
||||
self.batch_size = batch_size
|
||||
def sample(self):
|
||||
batch_step = np.arange(0, len(self.states), self.batch_size)
|
||||
indices = np.arange(len(self.states), dtype=np.int64)
|
||||
np.random.shuffle(indices)
|
||||
batches = [indices[i:i+self.batch_size] for i in batch_step]
|
||||
return np.array(self.states),\
|
||||
np.array(self.actions),\
|
||||
np.array(self.probs),\
|
||||
np.array(self.vals),\
|
||||
np.array(self.rewards),\
|
||||
np.array(self.dones),\
|
||||
batches
|
||||
|
||||
def push(self, state, action, probs, vals, reward, done):
|
||||
self.states.append(state)
|
||||
self.actions.append(action)
|
||||
self.probs.append(probs)
|
||||
self.vals.append(vals)
|
||||
self.rewards.append(reward)
|
||||
self.dones.append(done)
|
||||
|
||||
def clear(self):
|
||||
self.states = []
|
||||
self.probs = []
|
||||
self.actions = []
|
||||
self.rewards = []
|
||||
self.dones = []
|
||||
self.vals = []
|
||||
```
|
||||
这里的push函数就是将得到的相关量放入memory中，sample就是随机采样出来，方便第六步的随机梯度下降。
|
||||
### PPO model
|
||||
model就是actor和critic两个网络了：
|
||||
```python
|
||||
import torch.nn as nn
|
||||
from torch.distributions.categorical import Categorical
|
||||
class Actor(nn.Module):
|
||||
def __init__(self,n_states, n_actions,
|
||||
hidden_dim=256):
|
||||
super(Actor, self).__init__()
|
||||
|
||||
self.actor = nn.Sequential(
|
||||
nn.Linear(n_states, hidden_dim),
|
||||
nn.ReLU(),
|
||||
nn.Linear(hidden_dim, hidden_dim),
|
||||
nn.ReLU(),
|
||||
nn.Linear(hidden_dim, n_actions),
|
||||
nn.Softmax(dim=-1)
|
||||
)
|
||||
def forward(self, state):
|
||||
dist = self.actor(state)
|
||||
dist = Categorical(dist)
|
||||
return dist
|
||||
|
||||
class Critic(nn.Module):
|
||||
def __init__(self, n_states,hidden_dim=256):
|
||||
super(Critic, self).__init__()
|
||||
self.critic = nn.Sequential(
|
||||
nn.Linear(n_states, hidden_dim),
|
||||
nn.ReLU(),
|
||||
nn.Linear(hidden_dim, hidden_dim),
|
||||
nn.ReLU(),
|
||||
nn.Linear(hidden_dim, 1)
|
||||
)
|
||||
def forward(self, state):
|
||||
value = self.critic(state)
|
||||
return value
|
||||
```
|
||||
这里Actor就是得到一个概率分布(Categorica，也可以是别的分布，可以搜索torch distributionsl)，critc根据当前状态得到一个值，这里的输入维度可以是```n_states+n_actions```，即将action信息也纳入critic网络中，这样会更好一些，感兴趣的小伙伴可以试试。
|
||||
|
||||
### PPO update
|
||||
定义一个update函数主要实现伪代码中的第六步和第七步：
|
||||
```python
|
||||
def update(self):
|
||||
for _ in range(self.n_epochs):
|
||||
state_arr, action_arr, old_prob_arr, vals_arr,\
|
||||
reward_arr, dones_arr, batches = \
|
||||
self.memory.sample()
|
||||
values = vals_arr
|
||||
### compute advantage ###
|
||||
advantage = np.zeros(len(reward_arr), dtype=np.float32)
|
||||
for t in range(len(reward_arr)-1):
|
||||
discount = 1
|
||||
a_t = 0
|
||||
for k in range(t, len(reward_arr)-1):
|
||||
a_t += discount*(reward_arr[k] + self.gamma*values[k+1]*\
|
||||
(1-int(dones_arr[k])) - values[k])
|
||||
discount *= self.gamma*self.gae_lambda
|
||||
advantage[t] = a_t
|
||||
advantage = torch.tensor(advantage).to(self.device)
|
||||
### SGD ###
|
||||
values = torch.tensor(values).to(self.device)
|
||||
for batch in batches:
|
||||
states = torch.tensor(state_arr[batch], dtype=torch.float).to(self.device)
|
||||
old_probs = torch.tensor(old_prob_arr[batch]).to(self.device)
|
||||
actions = torch.tensor(action_arr[batch]).to(self.device)
|
||||
dist = self.actor(states)
|
||||
critic_value = self.critic(states)
|
||||
critic_value = torch.squeeze(critic_value)
|
||||
new_probs = dist.log_prob(actions)
|
||||
prob_ratio = new_probs.exp() / old_probs.exp()
|
||||
weighted_probs = advantage[batch] * prob_ratio
|
||||
weighted_clipped_probs = torch.clamp(prob_ratio, 1-self.policy_clip,
|
||||
1+self.policy_clip)*advantage[batch]
|
||||
actor_loss = -torch.min(weighted_probs, weighted_clipped_probs).mean()
|
||||
returns = advantage[batch] + values[batch]
|
||||
critic_loss = (returns-critic_value)**2
|
||||
critic_loss = critic_loss.mean()
|
||||
total_loss = actor_loss + 0.5*critic_loss
|
||||
self.actor_optimizer.zero_grad()
|
||||
self.critic_optimizer.zero_grad()
|
||||
total_loss.backward()
|
||||
self.actor_optimizer.step()
|
||||
self.critic_optimizer.step()
|
||||
self.memory.clear()
|
||||
```
|
||||
该部分首先从memory中提取搜集到的轨迹信息，然后计算gae，即advantage，接着使用随机梯度下降更新网络，最后清除memory以便搜集下一条轨迹信息。
|
||||
|
||||
最后实现效果如下：
|
||||
|
||||
BIN
projects/codes/PPO/assets/20210323154236878.png
Normal file
BIN
projects/codes/PPO/assets/20210323154236878.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 13 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 75 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 37 KiB |
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"algo_name": "PPO",
|
||||
"env_name": "CartPole-v0",
|
||||
"continuous": false,
|
||||
"train_eps": 200,
|
||||
"test_eps": 20,
|
||||
"gamma": 0.99,
|
||||
"batch_size": 5,
|
||||
"n_epochs": 4,
|
||||
"actor_lr": 0.0003,
|
||||
"critic_lr": 0.0003,
|
||||
"gae_lambda": 0.95,
|
||||
"policy_clip": 0.2,
|
||||
"update_fre": 20,
|
||||
"hidden_dim": 256,
|
||||
"device": "cpu",
|
||||
"result_path": "C:\\Users\\24438\\Desktop\\rl-tutorials\\codes\\PPO/outputs/CartPole-v0/20220731-233512/results/",
|
||||
"model_path": "C:\\Users\\24438\\Desktop\\rl-tutorials\\codes\\PPO/outputs/CartPole-v0/20220731-233512/models/",
|
||||
"save_fig": true
|
||||
}
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
|
After Width: | Height: | Size: 27 KiB |
Binary file not shown.
Binary file not shown.
Binary file not shown.
|
After Width: | Height: | Size: 65 KiB |
162
projects/codes/PPO/ppo2.py
Normal file
162
projects/codes/PPO/ppo2.py
Normal file
@@ -0,0 +1,162 @@
|
||||
#!/usr/bin/env python
|
||||
# coding=utf-8
|
||||
'''
|
||||
Author: John
|
||||
Email: johnjim0816@gmail.com
|
||||
Date: 2021-03-23 15:17:42
|
||||
LastEditor: John
|
||||
LastEditTime: 2021-12-31 19:38:33
|
||||
Discription:
|
||||
Environment:
|
||||
'''
|
||||
import os
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.optim as optim
|
||||
import torch.nn as nn
|
||||
from torch.distributions.categorical import Categorical
|
||||
class PPOMemory:
|
||||
def __init__(self, batch_size):
|
||||
self.states = []
|
||||
self.probs = []
|
||||
self.vals = []
|
||||
self.actions = []
|
||||
self.rewards = []
|
||||
self.dones = []
|
||||
self.batch_size = batch_size
|
||||
def sample(self):
|
||||
batch_step = np.arange(0, len(self.states), self.batch_size)
|
||||
indices = np.arange(len(self.states), dtype=np.int64)
|
||||
np.random.shuffle(indices)
|
||||
batches = [indices[i:i+self.batch_size] for i in batch_step]
|
||||
return np.array(self.states),np.array(self.actions),np.array(self.probs),\
|
||||
np.array(self.vals),np.array(self.rewards),np.array(self.dones),batches
|
||||
|
||||
def push(self, state, action, probs, vals, reward, done):
|
||||
self.states.append(state)
|
||||
self.actions.append(action)
|
||||
self.probs.append(probs)
|
||||
self.vals.append(vals)
|
||||
self.rewards.append(reward)
|
||||
self.dones.append(done)
|
||||
|
||||
def clear(self):
|
||||
self.states = []
|
||||
self.probs = []
|
||||
self.actions = []
|
||||
self.rewards = []
|
||||
self.dones = []
|
||||
self.vals = []
|
||||
class Actor(nn.Module):
|
||||
def __init__(self,n_states, n_actions,
|
||||
hidden_dim):
|
||||
super(Actor, self).__init__()
|
||||
|
||||
self.actor = nn.Sequential(
|
||||
nn.Linear(n_states, hidden_dim),
|
||||
nn.ReLU(),
|
||||
nn.Linear(hidden_dim, hidden_dim),
|
||||
nn.ReLU(),
|
||||
nn.Linear(hidden_dim, n_actions),
|
||||
nn.Softmax(dim=-1)
|
||||
)
|
||||
def forward(self, state):
|
||||
dist = self.actor(state)
|
||||
dist = Categorical(dist)
|
||||
return dist
|
||||
|
||||
class Critic(nn.Module):
|
||||
def __init__(self, n_states,hidden_dim):
|
||||
super(Critic, self).__init__()
|
||||
self.critic = nn.Sequential(
|
||||
nn.Linear(n_states, hidden_dim),
|
||||
nn.ReLU(),
|
||||
nn.Linear(hidden_dim, hidden_dim),
|
||||
nn.ReLU(),
|
||||
nn.Linear(hidden_dim, 1)
|
||||
)
|
||||
def forward(self, state):
|
||||
value = self.critic(state)
|
||||
return value
|
||||
class PPO:
|
||||
def __init__(self, n_states, n_actions,cfg):
|
||||
self.gamma = cfg.gamma
|
||||
self.continuous = cfg.continuous
|
||||
self.policy_clip = cfg.policy_clip
|
||||
self.n_epochs = cfg.n_epochs
|
||||
self.gae_lambda = cfg.gae_lambda
|
||||
self.device = cfg.device
|
||||
self.actor = Actor(n_states, n_actions,cfg.hidden_dim).to(self.device)
|
||||
self.critic = Critic(n_states,cfg.hidden_dim).to(self.device)
|
||||
self.actor_optimizer = optim.Adam(self.actor.parameters(), lr=cfg.actor_lr)
|
||||
self.critic_optimizer = optim.Adam(self.critic.parameters(), lr=cfg.critic_lr)
|
||||
self.memory = PPOMemory(cfg.batch_size)
|
||||
self.loss = 0
|
||||
|
||||
def choose_action(self, state):
|
||||
state = np.array([state]) # 先转成数组再转tensor更高效
|
||||
state = torch.tensor(state, dtype=torch.float).to(self.device)
|
||||
dist = self.actor(state)
|
||||
value = self.critic(state)
|
||||
action = dist.sample()
|
||||
probs = torch.squeeze(dist.log_prob(action)).item()
|
||||
if self.continuous:
|
||||
action = torch.tanh(action)
|
||||
else:
|
||||
action = torch.squeeze(action).item()
|
||||
value = torch.squeeze(value).item()
|
||||
return action, probs, value
|
||||
|
||||
def update(self):
|
||||
for _ in range(self.n_epochs):
|
||||
state_arr, action_arr, old_prob_arr, vals_arr,reward_arr, dones_arr, batches = self.memory.sample()
|
||||
values = vals_arr[:]
|
||||
### compute advantage ###
|
||||
advantage = np.zeros(len(reward_arr), dtype=np.float32)
|
||||
for t in range(len(reward_arr)-1):
|
||||
discount = 1
|
||||
a_t = 0
|
||||
for k in range(t, len(reward_arr)-1):
|
||||
a_t += discount*(reward_arr[k] + self.gamma*values[k+1]*\
|
||||
(1-int(dones_arr[k])) - values[k])
|
||||
discount *= self.gamma*self.gae_lambda
|
||||
advantage[t] = a_t
|
||||
advantage = torch.tensor(advantage).to(self.device)
|
||||
### SGD ###
|
||||
values = torch.tensor(values).to(self.device)
|
||||
for batch in batches:
|
||||
states = torch.tensor(state_arr[batch], dtype=torch.float).to(self.device)
|
||||
old_probs = torch.tensor(old_prob_arr[batch]).to(self.device)
|
||||
actions = torch.tensor(action_arr[batch]).to(self.device)
|
||||
dist = self.actor(states)
|
||||
critic_value = self.critic(states)
|
||||
critic_value = torch.squeeze(critic_value)
|
||||
new_probs = dist.log_prob(actions)
|
||||
prob_ratio = new_probs.exp() / old_probs.exp()
|
||||
weighted_probs = advantage[batch] * prob_ratio
|
||||
weighted_clipped_probs = torch.clamp(prob_ratio, 1-self.policy_clip,
|
||||
1+self.policy_clip)*advantage[batch]
|
||||
actor_loss = -torch.min(weighted_probs, weighted_clipped_probs).mean()
|
||||
returns = advantage[batch] + values[batch]
|
||||
critic_loss = (returns-critic_value)**2
|
||||
critic_loss = critic_loss.mean()
|
||||
total_loss = actor_loss + 0.5*critic_loss
|
||||
self.loss = total_loss
|
||||
self.actor_optimizer.zero_grad()
|
||||
self.critic_optimizer.zero_grad()
|
||||
total_loss.backward()
|
||||
self.actor_optimizer.step()
|
||||
self.critic_optimizer.step()
|
||||
self.memory.clear()
|
||||
def save(self,path):
|
||||
actor_checkpoint = os.path.join(path, 'ppo_actor.pt')
|
||||
critic_checkpoint= os.path.join(path, 'ppo_critic.pt')
|
||||
torch.save(self.actor.state_dict(), actor_checkpoint)
|
||||
torch.save(self.critic.state_dict(), critic_checkpoint)
|
||||
def load(self,path):
|
||||
actor_checkpoint = os.path.join(path, 'ppo_actor.pt')
|
||||
critic_checkpoint= os.path.join(path, 'ppo_critic.pt')
|
||||
self.actor.load_state_dict(torch.load(actor_checkpoint))
|
||||
self.critic.load_state_dict(torch.load(critic_checkpoint))
|
||||
|
||||
|
||||
132
projects/codes/PPO/task0.py
Normal file
132
projects/codes/PPO/task0.py
Normal file
@@ -0,0 +1,132 @@
|
||||
import sys,os
|
||||
curr_path = os.path.dirname(os.path.abspath(__file__)) # 当前文件所在绝对路径
|
||||
parent_path = os.path.dirname(curr_path) # 父路径
|
||||
sys.path.append(parent_path) # 添加路径到系统路径
|
||||
|
||||
import gym
|
||||
import torch
|
||||
import numpy as np
|
||||
import datetime
|
||||
import argparse
|
||||
from common.utils import plot_rewards,save_args,save_results,make_dir
|
||||
from ppo2 import PPO
|
||||
|
||||
def get_args():
|
||||
""" Hyperparameters
|
||||
"""
|
||||
curr_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") # 获取当前时间
|
||||
parser = argparse.ArgumentParser(description="hyperparameters")
|
||||
parser.add_argument('--algo_name',default='PPO',type=str,help="name of algorithm")
|
||||
parser.add_argument('--env_name',default='CartPole-v0',type=str,help="name of environment")
|
||||
parser.add_argument('--continuous',default=False,type=bool,help="if PPO is continous") # PPO既可适用于连续动作空间，也可以适用于离散动作空间
|
||||
parser.add_argument('--train_eps',default=200,type=int,help="episodes of training")
|
||||
parser.add_argument('--test_eps',default=20,type=int,help="episodes of testing")
|
||||
parser.add_argument('--gamma',default=0.99,type=float,help="discounted factor")
|
||||
parser.add_argument('--batch_size',default=5,type=int) # mini-batch SGD中的批量大小
|
||||
parser.add_argument('--n_epochs',default=4,type=int)
|
||||
parser.add_argument('--actor_lr',default=0.0003,type=float,help="learning rate of actor net")
|
||||
parser.add_argument('--critic_lr',default=0.0003,type=float,help="learning rate of critic net")
|
||||
parser.add_argument('--gae_lambda',default=0.95,type=float)
|
||||
parser.add_argument('--policy_clip',default=0.2,type=float) # PPO-clip中的clip参数，一般是0.1~0.2左右
|
||||
parser.add_argument('--update_fre',default=20,type=int)
|
||||
parser.add_argument('--hidden_dim',default=256,type=int)
|
||||
parser.add_argument('--device',default='cpu',type=str,help="cpu or cuda")
|
||||
parser.add_argument('--result_path',default=curr_path + "/outputs/" + parser.parse_args().env_name + \
|
||||
'/' + curr_time + '/results/' )
|
||||
parser.add_argument('--model_path',default=curr_path + "/outputs/" + parser.parse_args().env_name + \
|
||||
'/' + curr_time + '/models/' ) # path to save models
|
||||
parser.add_argument('--save_fig',default=True,type=bool,help="if save figure or not")
|
||||
args = parser.parse_args()
|
||||
return args
|
||||
|
||||
def env_agent_config(cfg,seed = 1):
|
||||
''' 创建环境和智能体
|
||||
'''
|
||||
env = gym.make(cfg.env_name) # 创建环境
|
||||
n_states = env.observation_space.shape[0] # 状态维度
|
||||
if cfg.continuous:
|
||||
n_actions = env.action_space.shape[0] # 动作维度
|
||||
else:
|
||||
n_actions = env.action_space.n # 动作维度
|
||||
agent = PPO(n_states, n_actions, cfg) # 创建智能体
|
||||
if seed !=0: # 设置随机种子
|
||||
torch.manual_seed(seed)
|
||||
env.seed(seed)
|
||||
np.random.seed(seed)
|
||||
return env, agent
|
||||
|
||||
def train(cfg,env,agent):
|
||||
print('开始训练！')
|
||||
print(f'环境：{cfg.env_name}, 算法：{cfg.algo_name}, 设备：{cfg.device}')
|
||||
rewards = [] # 记录所有回合的奖励
|
||||
ma_rewards = [] # 记录所有回合的滑动平均奖励
|
||||
steps = 0
|
||||
for i_ep in range(cfg.train_eps):
|
||||
state = env.reset()
|
||||
done = False
|
||||
ep_reward = 0
|
||||
while not done:
|
||||
action, prob, val = agent.choose_action(state)
|
||||
state_, reward, done, _ = env.step(action)
|
||||
steps += 1
|
||||
ep_reward += reward
|
||||
agent.memory.push(state, action, prob, val, reward, done)
|
||||
if steps % cfg.update_fre == 0:
|
||||
agent.update()
|
||||
state = state_
|
||||
rewards.append(ep_reward)
|
||||
if ma_rewards:
|
||||
ma_rewards.append(0.9*ma_rewards[-1]+0.1*ep_reward)
|
||||
else:
|
||||
ma_rewards.append(ep_reward)
|
||||
if (i_ep+1)%10 == 0:
|
||||
print(f"回合：{i_ep+1}/{cfg.train_eps}，奖励：{ep_reward:.2f}")
|
||||
print('完成训练！')
|
||||
env.close()
|
||||
res_dic = {'rewards':rewards,'ma_rewards':ma_rewards}
|
||||
return res_dic
|
||||
|
||||
def test(cfg,env,agent):
|
||||
print('开始测试!')
|
||||
print(f'环境：{cfg.env_name}, 算法：{cfg.algo_name}, 设备：{cfg.device}')
|
||||
rewards = [] # 记录所有回合的奖励
|
||||
ma_rewards = [] # 记录所有回合的滑动平均奖励
|
||||
for i_ep in range(cfg.test_eps):
|
||||
state = env.reset()
|
||||
done = False
|
||||
ep_reward = 0
|
||||
while not done:
|
||||
action, prob, val = agent.choose_action(state)
|
||||
state_, reward, done, _ = env.step(action)
|
||||
ep_reward += reward
|
||||
state = state_
|
||||
rewards.append(ep_reward)
|
||||
if ma_rewards:
|
||||
ma_rewards.append(
|
||||
0.9*ma_rewards[-1]+0.1*ep_reward)
|
||||
else:
|
||||
ma_rewards.append(ep_reward)
|
||||
print('回合：{}/{}, 奖励：{}'.format(i_ep+1, cfg.test_eps, ep_reward))
|
||||
print('完成训练！')
|
||||
env.close()
|
||||
res_dic = {'rewards':rewards,'ma_rewards':ma_rewards}
|
||||
return res_dic
|
||||
|
||||
if __name__ == "__main__":
|
||||
cfg = get_args()
|
||||
# 训练
|
||||
env, agent = env_agent_config(cfg)
|
||||
res_dic = train(cfg, env, agent)
|
||||
make_dir(cfg.result_path, cfg.model_path)
|
||||
save_args(cfg) # 保存参数
|
||||
agent.save(path=cfg.model_path) # save model
|
||||
save_results(res_dic, tag='train',
|
||||
path=cfg.result_path)
|
||||
plot_rewards(res_dic['rewards'], res_dic['ma_rewards'], cfg, tag="train")
|
||||
# 测试
|
||||
env, agent = env_agent_config(cfg)
|
||||
agent.load(path=cfg.model_path) # 导入模型
|
||||
res_dic = test(cfg, env, agent)
|
||||
save_results(res_dic, tag='test',
|
||||
path=cfg.result_path) # 保存结果
|
||||
plot_rewards(res_dic['rewards'], res_dic['ma_rewards'],cfg, tag="test") # 画出结果
|
||||
67
projects/codes/PPO/task1.py
Normal file
67
projects/codes/PPO/task1.py
Normal file
@@ -0,0 +1,67 @@
|
||||
import sys,os
|
||||
curr_path = os.path.dirname(os.path.abspath(__file__)) # 当前文件所在绝对路径
|
||||
parent_path = os.path.dirname(curr_path) # 父路径
|
||||
sys.path.append(parent_path) # 添加路径到系统路径
|
||||
|
||||
import gym
|
||||
import torch
|
||||
import datetime
|
||||
from common.utils import plot_rewards
|
||||
from common.utils import save_results,make_dir
|
||||
from ppo2 import PPO
|
||||
|
||||
curr_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") # 获取当前时间
|
||||
|
||||
class PPOConfig:
|
||||
def __init__(self) -> None:
|
||||
self.algo = "PPO" # 算法名称
|
||||
self.env_name = 'Pendulum-v1' # 环境名称
|
||||
self.continuous = True # 环境是否为连续动作
|
||||
self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # 检测GPU
|
||||
self.train_eps = 200 # 训练的回合数
|
||||
self.test_eps = 20 # 测试的回合数
|
||||
self.batch_size = 5
|
||||
self.gamma=0.99
|
||||
self.n_epochs = 4
|
||||
self.actor_lr = 0.0003
|
||||
self.critic_lr = 0.0003
|
||||
self.gae_lambda=0.95
|
||||
self.policy_clip=0.2
|
||||
self.hidden_dim = 256
|
||||
self.update_fre = 20 # frequency of agent update
|
||||
|
||||
class PlotConfig:
|
||||
def __init__(self) -> None:
|
||||
self.algo = "PPO" # 算法名称
|
||||
self.env_name = 'Pendulum-v1' # 环境名称
|
||||
self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # 检测GPU
|
||||
self.result_path = curr_path+"/outputs/" + self.env_name + \
|
||||
'/'+curr_time+'/results/' # 保存结果的路径
|
||||
self.model_path = curr_path+"/outputs/" + self.env_name + \
|
||||
'/'+curr_time+'/models/' # 保存模型的路径
|
||||
self.save = True # 是否保存图片
|
||||
|
||||
def env_agent_config(cfg,seed=1):
|
||||
env = gym.make(cfg.env_name)
|
||||
env.seed(seed)
|
||||
n_states = env.observation_space.shape[0]
|
||||
n_actions = env.action_space.shape[0]
|
||||
agent = PPO(n_states,n_actions,cfg)
|
||||
return env,agent
|
||||
|
||||
|
||||
cfg = PPOConfig()
|
||||
plot_cfg = PlotConfig()
|
||||
# 训练
|
||||
env,agent = env_agent_config(cfg,seed=1)
|
||||
rewards, ma_rewards = train(cfg, env, agent)
|
||||
make_dir(plot_cfg.result_path, plot_cfg.model_path) # 创建保存结果和模型路径的文件夹
|
||||
agent.save(path=plot_cfg.model_path)
|
||||
save_results(rewards, ma_rewards, tag='train', path=plot_cfg.result_path)
|
||||
plot_rewards(rewards, ma_rewards, plot_cfg, tag="train")
|
||||
# 测试
|
||||
env,agent = env_agent_config(cfg,seed=10)
|
||||
agent.load(path=plot_cfg.model_path)
|
||||
rewards,ma_rewards = eval(cfg,env,agent)
|
||||
save_results(rewards,ma_rewards,tag='eval',path=plot_cfg.result_path)
|
||||
plot_rewards(rewards,ma_rewards,plot_cfg,tag="eval")
|
||||
Reference in New Issue
Block a user