Merge branch 'master' of github.com:datawhalechina/easy-rl
This commit is contained in:
qiwang067
@@ -10,12 +10,40 @@ Discription:
|
||||
Environment:
|
||||
'''
|
||||
import torch.optim as optim
|
||||
from A2C.model import ActorCritic
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
from torch.distributions import Categorical
|
||||
|
||||
class ActorCritic(nn.Module):
|
||||
''' A2C网络模型，包含一个Actor和Critic
|
||||
'''
|
||||
def __init__(self, input_dim, output_dim, hidden_dim):
|
||||
super(ActorCritic, self).__init__()
|
||||
self.critic = nn.Sequential(
|
||||
nn.Linear(input_dim, hidden_dim),
|
||||
nn.ReLU(),
|
||||
nn.Linear(hidden_dim, 1)
|
||||
)
|
||||
|
||||
self.actor = nn.Sequential(
|
||||
nn.Linear(input_dim, hidden_dim),
|
||||
nn.ReLU(),
|
||||
nn.Linear(hidden_dim, output_dim),
|
||||
nn.Softmax(dim=1),
|
||||
)
|
||||
|
||||
def forward(self, x):
|
||||
value = self.critic(x)
|
||||
probs = self.actor(x)
|
||||
dist = Categorical(probs)
|
||||
return dist, value
|
||||
class A2C:
|
||||
def __init__(self,n_states,n_actions,cfg) -> None:
|
||||
''' A2C算法
|
||||
'''
|
||||
def __init__(self,state_dim,action_dim,cfg) -> None:
|
||||
self.gamma = cfg.gamma
|
||||
self.device = cfg.device
|
||||
self.model = ActorCritic(n_states, n_actions, cfg.hidden_size).to(self.device)
|
||||
self.model = ActorCritic(state_dim, action_dim, cfg.hidden_size).to(self.device)
|
||||
self.optimizer = optim.Adam(self.model.parameters())
|
||||
|
||||
def compute_returns(self,next_value, rewards, masks):
|
||||
|
||||
@@ -1,36 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
# coding=utf-8
|
||||
'''
|
||||
Author: JiangJi
|
||||
Email: johnjim0816@gmail.com
|
||||
Date: 2021-05-03 21:38:54
|
||||
LastEditor: JiangJi
|
||||
LastEditTime: 2021-05-03 21:40:06
|
||||
Discription:
|
||||
Environment:
|
||||
'''
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
from torch.distributions import Categorical
|
||||
class ActorCritic(nn.Module):
|
||||
def __init__(self, n_states, n_actions, hidden_dim):
|
||||
super(ActorCritic, self).__init__()
|
||||
|
||||
self.critic = nn.Sequential(
|
||||
nn.Linear(n_states, hidden_dim),
|
||||
nn.ReLU(),
|
||||
nn.Linear(hidden_dim, 1)
|
||||
)
|
||||
|
||||
self.actor = nn.Sequential(
|
||||
nn.Linear(n_states, hidden_dim),
|
||||
nn.ReLU(),
|
||||
nn.Linear(hidden_dim, n_actions),
|
||||
nn.Softmax(dim=1),
|
||||
)
|
||||
|
||||
def forward(self, x):
|
||||
value = self.critic(x)
|
||||
probs = self.actor(x)
|
||||
dist = Categorical(probs)
|
||||
return dist, value
|
||||
Binary file not shown.
Binary file not shown.
@@ -1,7 +1,8 @@
|
||||
import sys,os
|
||||
curr_path = os.path.dirname(os.path.abspath(__file__)) # 当前文件所在绝对路径
|
||||
parent_path = os.path.dirname(curr_path) # 父路径
|
||||
sys.path.append(parent_path) # 添加路径到系统路径sys.path
|
||||
import sys
|
||||
import os
|
||||
curr_path = os.path.dirname(os.path.abspath(__file__)) # 当前文件所在绝对路径
|
||||
parent_path = os.path.dirname(curr_path) # 父路径
|
||||
sys.path.append(parent_path) # 添加路径到系统路径
|
||||
|
||||
import gym
|
||||
import numpy as np
|
||||
@@ -9,15 +10,18 @@ import torch
|
||||
import torch.optim as optim
|
||||
import datetime
|
||||
from common.multiprocessing_env import SubprocVecEnv
|
||||
from A2C.model import ActorCritic
|
||||
from A2C.agent import ActorCritic
|
||||
from common.utils import save_results, make_dir
|
||||
from common.plot import plot_rewards
|
||||
from common.utils import plot_rewards
|
||||
|
||||
curr_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") # 获取当前时间
|
||||
algo_name = 'A2C' # 算法名称
|
||||
env_name = 'CartPole-v0' # 环境名称
|
||||
|
||||
curr_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") # obtain current time
|
||||
class A2CConfig:
|
||||
def __init__(self) -> None:
|
||||
self.algo='A2C' # 算法名称
|
||||
self.env_name= 'CartPole-v0' # 环境名称
|
||||
self.algo_name = algo_name# 算法名称
|
||||
self.env_name = env_name # 环境名称
|
||||
self.n_envs = 8 # 异步的环境数目
|
||||
self.gamma = 0.99 # 强化学习中的折扣因子
|
||||
self.hidden_dim = 256
|
||||
@@ -27,10 +31,9 @@ class A2CConfig:
|
||||
self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
||||
class PlotConfig:
|
||||
def __init__(self) -> None:
|
||||
self.algo = "DQN" # 算法名称
|
||||
self.env_name = 'CartPole-v0' # 环境名称
|
||||
self.algo_name = algo_name # 算法名称
|
||||
self.env_name = env_name # 环境名称
|
||||
self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # 检测GPU
|
||||
|
||||
self.result_path = curr_path+"/outputs/" + self.env_name + \
|
||||
'/'+curr_time+'/results/' # 保存结果的路径
|
||||
self.model_path = curr_path+"/outputs/" + self.env_name + \
|
||||
@@ -67,6 +70,8 @@ def compute_returns(next_value, rewards, masks, gamma=0.99):
|
||||
|
||||
|
||||
def train(cfg,envs):
|
||||
print('开始训练!')
|
||||
print(f'环境：{cfg.env_name}, 算法：{cfg.algo}, 设备：{cfg.device}')
|
||||
env = gym.make(cfg.env_name) # a single env
|
||||
env.seed(10)
|
||||
state_dim = envs.observation_space.shape[0]
|
||||
@@ -119,6 +124,7 @@ def train(cfg,envs):
|
||||
optimizer.zero_grad()
|
||||
loss.backward()
|
||||
optimizer.step()
|
||||
print('完成训练！')
|
||||
return test_rewards, test_ma_rewards
|
||||
if __name__ == "__main__":
|
||||
cfg = A2CConfig()
|
||||
@@ -9,15 +9,68 @@ LastEditTime: 2021-09-16 00:55:30
|
||||
@Discription:
|
||||
@Environment: python 3.7.7
|
||||
'''
|
||||
import random
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.optim as optim
|
||||
import torch.nn.functional as F
|
||||
class ReplayBuffer:
|
||||
def __init__(self, capacity):
|
||||
self.capacity = capacity # 经验回放的容量
|
||||
self.buffer = [] # 缓冲区
|
||||
self.position = 0
|
||||
|
||||
from common.model import Actor, Critic
|
||||
from common.memory import ReplayBuffer
|
||||
def push(self, state, action, reward, next_state, done):
|
||||
''' 缓冲区是一个队列，容量超出时去掉开始存入的转移(transition)
|
||||
'''
|
||||
if len(self.buffer) < self.capacity:
|
||||
self.buffer.append(None)
|
||||
self.buffer[self.position] = (state, action, reward, next_state, done)
|
||||
self.position = (self.position + 1) % self.capacity
|
||||
|
||||
def sample(self, batch_size):
|
||||
batch = random.sample(self.buffer, batch_size) # 随机采出小批量转移
|
||||
state, action, reward, next_state, done = zip(*batch) # 解压成状态，动作等
|
||||
return state, action, reward, next_state, done
|
||||
|
||||
def __len__(self):
|
||||
''' 返回当前存储的量
|
||||
'''
|
||||
return len(self.buffer)
|
||||
class Actor(nn.Module):
|
||||
def __init__(self, state_dim, action_dim, hidden_dim, init_w=3e-3):
|
||||
super(Actor, self).__init__()
|
||||
self.linear1 = nn.Linear(state_dim, hidden_dim)
|
||||
self.linear2 = nn.Linear(hidden_dim, hidden_dim)
|
||||
self.linear3 = nn.Linear(hidden_dim, action_dim)
|
||||
|
||||
self.linear3.weight.data.uniform_(-init_w, init_w)
|
||||
self.linear3.bias.data.uniform_(-init_w, init_w)
|
||||
|
||||
def forward(self, x):
|
||||
x = F.relu(self.linear1(x))
|
||||
x = F.relu(self.linear2(x))
|
||||
x = torch.tanh(self.linear3(x))
|
||||
return x
|
||||
class Critic(nn.Module):
|
||||
def __init__(self, state_dim, action_dim, hidden_dim, init_w=3e-3):
|
||||
super(Critic, self).__init__()
|
||||
|
||||
self.linear1 = nn.Linear(state_dim + action_dim, hidden_dim)
|
||||
self.linear2 = nn.Linear(hidden_dim, hidden_dim)
|
||||
self.linear3 = nn.Linear(hidden_dim, 1)
|
||||
# 随机初始化为较小的值
|
||||
self.linear3.weight.data.uniform_(-init_w, init_w)
|
||||
self.linear3.bias.data.uniform_(-init_w, init_w)
|
||||
|
||||
def forward(self, state, action):
|
||||
# 按维数1拼接
|
||||
x = torch.cat([state, action], 1)
|
||||
x = F.relu(self.linear1(x))
|
||||
x = F.relu(self.linear2(x))
|
||||
x = self.linear3(x)
|
||||
return x
|
||||
class DDPG:
|
||||
def __init__(self, state_dim, action_dim, cfg):
|
||||
self.device = cfg.device
|
||||
|
||||
@@ -16,12 +16,10 @@ class NormalizedActions(gym.ActionWrapper):
|
||||
''' 将action范围重定在[0.1]之间
|
||||
'''
|
||||
def action(self, action):
|
||||
|
||||
low_bound = self.action_space.low
|
||||
upper_bound = self.action_space.high
|
||||
action = low_bound + (action + 1.0) * 0.5 * (upper_bound - low_bound)
|
||||
action = np.clip(action, low_bound, upper_bound)
|
||||
|
||||
return action
|
||||
|
||||
def reverse_action(self, action):
|
||||
|
||||
81
codes/DDPG/task0.py
Normal file
81
codes/DDPG/task0.py
Normal file
@@ -0,0 +1,81 @@
|
||||
#!/usr/bin/env python
|
||||
# coding=utf-8
|
||||
'''
|
||||
@Author: John
|
||||
@Email: johnjim0816@gmail.com
|
||||
@Date: 2020-06-11 20:58:21
|
||||
@LastEditor: John
|
||||
LastEditTime: 2021-09-16 01:31:33
|
||||
@Discription:
|
||||
@Environment: python 3.7.7
|
||||
'''
|
||||
import sys,os
|
||||
curr_path = os.path.dirname(os.path.abspath(__file__)) # 当前文件所在绝对路径
|
||||
parent_path = os.path.dirname(curr_path) # 父路径
|
||||
sys.path.append(parent_path) # 添加路径到系统路径sys.path
|
||||
|
||||
import datetime
|
||||
import gym
|
||||
import torch
|
||||
|
||||
from DDPG.env import NormalizedActions
|
||||
from DDPG.agent import DDPG
|
||||
from DDPG.train import train,test
|
||||
from common.utils import save_results,make_dir
|
||||
from common.utils import plot_rewards
|
||||
|
||||
curr_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") # 获取当前时间
|
||||
algo_name = 'DDPG' # 算法名称
|
||||
env_name = 'Pendulum-v1' # 环境名称，gym新版本（约0.21.0之后）中Pendulum-v0改为Pendulum-v1
|
||||
|
||||
class DDPGConfig:
|
||||
def __init__(self):
|
||||
self.algo_name = algo_name # 算法名称
|
||||
self.env_name = env_name # 环境名称
|
||||
self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # 检测GPU
|
||||
self.train_eps = 300 # 训练的回合数
|
||||
self.test_eps = 50 # 测试的回合数
|
||||
self.gamma = 0.99 # 折扣因子
|
||||
self.critic_lr = 1e-3 # 评论家网络的学习率
|
||||
self.actor_lr = 1e-4 # 演员网络的学习率
|
||||
self.memory_capacity = 8000 # 经验回放的容量
|
||||
self.batch_size = 128 # mini-batch SGD中的批量大小
|
||||
self.target_update = 2 # 目标网络的更新频率
|
||||
self.hidden_dim = 256 # 网络隐藏层维度
|
||||
self.soft_tau = 1e-2 # 软更新参数
|
||||
|
||||
class PlotConfig:
|
||||
def __init__(self) -> None:
|
||||
self.algo_name = algo_name # 算法名称
|
||||
self.env_name = env_name # 环境名称
|
||||
self.result_path = curr_path+"/outputs/" + self.env_name + \
|
||||
'/'+curr_time+'/results/' # 保存结果的路径
|
||||
self.model_path = curr_path+"/outputs/" + self.env_name + \
|
||||
'/'+curr_time+'/models/' # 保存模型的路径
|
||||
self.save = True # 是否保存图片
|
||||
self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # 检测GPU
|
||||
|
||||
def env_agent_config(cfg,seed=1):
|
||||
env = NormalizedActions(gym.make(cfg.env_name)) # 装饰action噪声
|
||||
env.seed(seed) # 随机种子
|
||||
state_dim = env.observation_space.shape[0]
|
||||
action_dim = env.action_space.shape[0]
|
||||
agent = DDPG(state_dim,action_dim,cfg)
|
||||
return env,agent
|
||||
|
||||
cfg = DDPGConfig()
|
||||
plot_cfg = PlotConfig()
|
||||
# 训练
|
||||
env,agent = env_agent_config(cfg,seed=1)
|
||||
rewards, ma_rewards = train(cfg, env, agent)
|
||||
make_dir(plot_cfg.result_path, plot_cfg.model_path)
|
||||
agent.save(path=plot_cfg.model_path)
|
||||
save_results(rewards, ma_rewards, tag='train', path=plot_cfg.result_path)
|
||||
plot_rewards(rewards, ma_rewards, plot_cfg, tag="train") # 画出结果
|
||||
# 测试
|
||||
env,agent = env_agent_config(cfg,seed=10)
|
||||
agent.load(path=plot_cfg.model_path)
|
||||
rewards,ma_rewards = test(plot_cfg,env,agent)
|
||||
save_results(rewards,ma_rewards,tag = 'test',path = cfg.result_path)
|
||||
plot_rewards(rewards, ma_rewards, plot_cfg, tag="test") # 画出结果
|
||||
|
||||
@@ -1,136 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
# coding=utf-8
|
||||
'''
|
||||
@Author: John
|
||||
@Email: johnjim0816@gmail.com
|
||||
@Date: 2020-06-11 20:58:21
|
||||
@LastEditor: John
|
||||
LastEditTime: 2021-09-16 01:31:33
|
||||
@Discription:
|
||||
@Environment: python 3.7.7
|
||||
'''
|
||||
import sys,os
|
||||
curr_path = os.path.dirname(os.path.abspath(__file__)) # 当前文件所在绝对路径
|
||||
parent_path = os.path.dirname(curr_path) # 父路径
|
||||
sys.path.append(parent_path) # 添加路径到系统路径sys.path
|
||||
|
||||
import datetime
|
||||
import gym
|
||||
import torch
|
||||
|
||||
from DDPG.env import NormalizedActions, OUNoise
|
||||
from DDPG.agent import DDPG
|
||||
from common.utils import save_results,make_dir
|
||||
from common.plot import plot_rewards
|
||||
|
||||
curr_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") # 获取当前时间
|
||||
|
||||
class DDPGConfig:
|
||||
def __init__(self):
|
||||
self.algo = 'DDPG' # 算法名称
|
||||
self.env_name = 'Pendulum-v0' # 环境名称
|
||||
self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # 检测GPU
|
||||
self.train_eps = 300 # 训练的回合数
|
||||
self.eval_eps = 50 # 测试的回合数
|
||||
self.gamma = 0.99 # 折扣因子
|
||||
self.critic_lr = 1e-3 # 评论家网络的学习率
|
||||
self.actor_lr = 1e-4 # 演员网络的学习率
|
||||
self.memory_capacity = 8000 # 经验回放的容量
|
||||
self.batch_size = 128 # mini-batch SGD中的批量大小
|
||||
self.target_update = 2 # 目标网络的更新频率
|
||||
self.hidden_dim = 256 # 网络隐藏层维度
|
||||
self.soft_tau = 1e-2 # 软更新参数
|
||||
|
||||
class PlotConfig:
|
||||
def __init__(self) -> None:
|
||||
self.algo = "DQN" # 算法名称
|
||||
self.env_name = 'CartPole-v0' # 环境名称
|
||||
self.result_path = curr_path+"/outputs/" + self.env_name + \
|
||||
'/'+curr_time+'/results/' # 保存结果的路径
|
||||
self.model_path = curr_path+"/outputs/" + self.env_name + \
|
||||
'/'+curr_time+'/models/' # 保存模型的路径
|
||||
self.save = True # 是否保存图片
|
||||
self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # 检测GPU
|
||||
|
||||
def env_agent_config(cfg,seed=1):
|
||||
env = NormalizedActions(gym.make(cfg.env_name)) # 装饰action噪声
|
||||
env.seed(seed) # 随机种子
|
||||
n_states = env.observation_space.shape[0]
|
||||
n_actions = env.action_space.shape[0]
|
||||
agent = DDPG(n_states,n_actions,cfg)
|
||||
return env,agent
|
||||
|
||||
def train(cfg, env, agent):
|
||||
print('开始训练！')
|
||||
print(f'环境：{cfg.env_name}，算法：{cfg.algo}，设备：{cfg.device}')
|
||||
ou_noise = OUNoise(env.action_space) # 动作噪声
|
||||
rewards = [] # 记录所有回合的奖励
|
||||
ma_rewards = [] # 记录所有回合的滑动平均奖励
|
||||
for i_ep in range(cfg.train_eps):
|
||||
state = env.reset()
|
||||
ou_noise.reset()
|
||||
done = False
|
||||
ep_reward = 0
|
||||
i_step = 0
|
||||
while not done:
|
||||
i_step += 1
|
||||
action = agent.choose_action(state)
|
||||
action = ou_noise.get_action(action, i_step)
|
||||
next_state, reward, done, _ = env.step(action)
|
||||
ep_reward += reward
|
||||
agent.memory.push(state, action, reward, next_state, done)
|
||||
agent.update()
|
||||
state = next_state
|
||||
if (i_ep+1)%10 == 0:
|
||||
print('回合：{}/{}，奖励：{:.2f}'.format(i_ep+1, cfg.train_eps, ep_reward))
|
||||
rewards.append(ep_reward)
|
||||
if ma_rewards:
|
||||
ma_rewards.append(0.9*ma_rewards[-1]+0.1*ep_reward)
|
||||
else:
|
||||
ma_rewards.append(ep_reward)
|
||||
print('完成训练！')
|
||||
return rewards, ma_rewards
|
||||
|
||||
def eval(cfg, env, agent):
|
||||
print('开始测试！')
|
||||
print(f'环境：{cfg.env_name}, 算法：{cfg.algo}, 设备：{cfg.device}')
|
||||
rewards = [] # 记录所有回合的奖励
|
||||
ma_rewards = [] # 记录所有回合的滑动平均奖励
|
||||
for i_ep in range(cfg.eval_eps):
|
||||
state = env.reset()
|
||||
done = False
|
||||
ep_reward = 0
|
||||
i_step = 0
|
||||
while not done:
|
||||
i_step += 1
|
||||
action = agent.choose_action(state)
|
||||
next_state, reward, done, _ = env.step(action)
|
||||
ep_reward += reward
|
||||
state = next_state
|
||||
print('回合：{}/{}, 奖励：{}'.format(i_ep+1, cfg.train_eps, ep_reward))
|
||||
rewards.append(ep_reward)
|
||||
if ma_rewards:
|
||||
ma_rewards.append(0.9*ma_rewards[-1]+0.1*ep_reward)
|
||||
else:
|
||||
ma_rewards.append(ep_reward)
|
||||
print('完成测试！')
|
||||
return rewards, ma_rewards
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
cfg = DDPGConfig()
|
||||
plot_cfg = PlotConfig()
|
||||
# 训练
|
||||
env,agent = env_agent_config(cfg,seed=1)
|
||||
rewards, ma_rewards = train(cfg, env, agent)
|
||||
make_dir(plot_cfg.result_path, plot_cfg.model_path)
|
||||
agent.save(path=plot_cfg.model_path)
|
||||
save_results(rewards, ma_rewards, tag='train', path=plot_cfg.result_path)
|
||||
plot_rewards(rewards, ma_rewards, plot_cfg, tag="train")
|
||||
# 测试
|
||||
env,agent = env_agent_config(cfg,seed=10)
|
||||
agent.load(path=plot_cfg.model_path)
|
||||
rewards,ma_rewards = eval(plot_cfg,env,agent)
|
||||
save_results(rewards,ma_rewards,tag = 'eval',path = cfg.result_path)
|
||||
plot_rewards(rewards,ma_rewards,plot_cfg,tag = "eval")
|
||||
|
||||
64
codes/DDPG/train.py
Normal file
64
codes/DDPG/train.py
Normal file
@@ -0,0 +1,64 @@
|
||||
import sys
|
||||
import os
|
||||
curr_path = os.path.dirname(os.path.abspath(__file__)) # 当前文件所在绝对路径
|
||||
parent_path = os.path.dirname(curr_path) # 父路径
|
||||
sys.path.append(parent_path) # 添加路径到系统路径
|
||||
|
||||
from DDPG.env import OUNoise
|
||||
|
||||
def train(cfg, env, agent):
|
||||
print('开始训练！')
|
||||
print(f'环境：{cfg.env_name}，算法：{cfg.algo}，设备：{cfg.device}')
|
||||
ou_noise = OUNoise(env.action_space) # 动作噪声
|
||||
rewards = [] # 记录所有回合的奖励
|
||||
ma_rewards = [] # 记录所有回合的滑动平均奖励
|
||||
for i_ep in range(cfg.train_eps):
|
||||
state = env.reset()
|
||||
ou_noise.reset()
|
||||
done = False
|
||||
ep_reward = 0
|
||||
i_step = 0
|
||||
while not done:
|
||||
i_step += 1
|
||||
action = agent.choose_action(state)
|
||||
action = ou_noise.get_action(action, i_step)
|
||||
next_state, reward, done, _ = env.step(action)
|
||||
ep_reward += reward
|
||||
agent.memory.push(state, action, reward, next_state, done)
|
||||
agent.update()
|
||||
state = next_state
|
||||
if (i_ep+1)%10 == 0:
|
||||
print('回合：{}/{}，奖励：{:.2f}'.format(i_ep+1, cfg.train_eps, ep_reward))
|
||||
rewards.append(ep_reward)
|
||||
if ma_rewards:
|
||||
ma_rewards.append(0.9*ma_rewards[-1]+0.1*ep_reward)
|
||||
else:
|
||||
ma_rewards.append(ep_reward)
|
||||
print('完成训练！')
|
||||
return rewards, ma_rewards
|
||||
|
||||
def test(cfg, env, agent):
|
||||
print('开始测试！')
|
||||
print(f'环境：{cfg.env_name}, 算法：{cfg.algo}, 设备：{cfg.device}')
|
||||
rewards = [] # 记录所有回合的奖励
|
||||
ma_rewards = [] # 记录所有回合的滑动平均奖励
|
||||
for i_ep in range(cfg.test_eps):
|
||||
state = env.reset()
|
||||
done = False
|
||||
ep_reward = 0
|
||||
i_step = 0
|
||||
while not done:
|
||||
i_step += 1
|
||||
action = agent.choose_action(state)
|
||||
next_state, reward, done, _ = env.step(action)
|
||||
ep_reward += reward
|
||||
state = next_state
|
||||
print('回合：{}/{}, 奖励：{}'.format(i_ep+1, cfg.train_eps, ep_reward))
|
||||
rewards.append(ep_reward)
|
||||
if ma_rewards:
|
||||
ma_rewards.append(0.9*ma_rewards[-1]+0.1*ep_reward)
|
||||
else:
|
||||
ma_rewards.append(ep_reward)
|
||||
print(f"回合：{i_ep+1}/{cfg.test_eps}，奖励：{ep_reward:.1f}")
|
||||
print('完成测试！')
|
||||
return rewards, ma_rewards
|
||||
@@ -5,7 +5,7 @@
|
||||
@Email: johnjim0816@gmail.com
|
||||
@Date: 2020-06-12 00:50:49
|
||||
@LastEditor: John
|
||||
LastEditTime: 2021-09-15 13:35:36
|
||||
LastEditTime: 2021-12-22 14:01:37
|
||||
@Discription:
|
||||
@Environment: python 3.7.7
|
||||
'''
|
||||
@@ -14,16 +14,57 @@ LastEditTime: 2021-09-15 13:35:36
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
import torch.optim as optim
|
||||
import random
|
||||
import math
|
||||
import numpy as np
|
||||
from common.memory import ReplayBuffer
|
||||
from common.model import MLP
|
||||
class DQN:
|
||||
def __init__(self, n_states, n_actions, cfg):
|
||||
|
||||
self.n_actions = n_actions # 总的动作个数
|
||||
class MLP(nn.Module):
|
||||
def __init__(self, state_dim,action_dim,hidden_dim=128):
|
||||
""" 初始化q网络，为全连接网络
|
||||
state_dim: 输入的特征数即环境的状态维度
|
||||
action_dim: 输出的动作维度
|
||||
"""
|
||||
super(MLP, self).__init__()
|
||||
self.fc1 = nn.Linear(state_dim, hidden_dim) # 输入层
|
||||
self.fc2 = nn.Linear(hidden_dim,hidden_dim) # 隐藏层
|
||||
self.fc3 = nn.Linear(hidden_dim, action_dim) # 输出层
|
||||
|
||||
def forward(self, x):
|
||||
# 各层对应的激活函数
|
||||
x = F.relu(self.fc1(x))
|
||||
x = F.relu(self.fc2(x))
|
||||
return self.fc3(x)
|
||||
|
||||
class ReplayBuffer:
|
||||
def __init__(self, capacity):
|
||||
self.capacity = capacity # 经验回放的容量
|
||||
self.buffer = [] # 缓冲区
|
||||
self.position = 0
|
||||
|
||||
def push(self, state, action, reward, next_state, done):
|
||||
''' 缓冲区是一个队列，容量超出时去掉开始存入的转移(transition)
|
||||
'''
|
||||
if len(self.buffer) < self.capacity:
|
||||
self.buffer.append(None)
|
||||
self.buffer[self.position] = (state, action, reward, next_state, done)
|
||||
self.position = (self.position + 1) % self.capacity
|
||||
|
||||
def sample(self, batch_size):
|
||||
batch = random.sample(self.buffer, batch_size) # 随机采出小批量转移
|
||||
state, action, reward, next_state, done = zip(*batch) # 解压成状态，动作等
|
||||
return state, action, reward, next_state, done
|
||||
|
||||
def __len__(self):
|
||||
''' 返回当前存储的量
|
||||
'''
|
||||
return len(self.buffer)
|
||||
|
||||
class DQN:
|
||||
def __init__(self, state_dim, action_dim, cfg):
|
||||
|
||||
self.action_dim = action_dim # 总的动作个数
|
||||
self.device = cfg.device # 设备，cpu或gpu等
|
||||
self.gamma = cfg.gamma # 奖励的折扣因子
|
||||
# e-greedy策略相关参数
|
||||
@@ -32,8 +73,8 @@ class DQN:
|
||||
(cfg.epsilon_start - cfg.epsilon_end) * \
|
||||
math.exp(-1. * frame_idx / cfg.epsilon_decay)
|
||||
self.batch_size = cfg.batch_size
|
||||
self.policy_net = MLP(n_states, n_actions,hidden_dim=cfg.hidden_dim).to(self.device)
|
||||
self.target_net = MLP(n_states, n_actions,hidden_dim=cfg.hidden_dim).to(self.device)
|
||||
self.policy_net = MLP(state_dim, action_dim,hidden_dim=cfg.hidden_dim).to(self.device)
|
||||
self.target_net = MLP(state_dim, action_dim,hidden_dim=cfg.hidden_dim).to(self.device)
|
||||
for target_param, param in zip(self.target_net.parameters(),self.policy_net.parameters()): # 复制参数到目标网路targe_net
|
||||
target_param.data.copy_(param.data)
|
||||
self.optimizer = optim.Adam(self.policy_net.parameters(), lr=cfg.lr) # 优化器
|
||||
@@ -49,7 +90,7 @@ class DQN:
|
||||
q_values = self.policy_net(state)
|
||||
action = q_values.max(1)[1].item() # 选择Q值最大的动作
|
||||
else:
|
||||
action = random.randrange(self.n_actions)
|
||||
action = random.randrange(self.action_dim)
|
||||
return action
|
||||
def update(self):
|
||||
if len(self.memory) < self.batch_size: # 当memory中不满足一个批量时，不更新策略
|
||||
133
codes/DQN/dqn_cnn.py
Normal file
133
codes/DQN/dqn_cnn.py
Normal file
@@ -0,0 +1,133 @@
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.optim as optim
|
||||
import torch.autograd as autograd
|
||||
import random
|
||||
import math
|
||||
class CNN(nn.Module):
|
||||
def __init__(self, input_dim, output_dim):
|
||||
super(CNN, self).__init__()
|
||||
|
||||
self.input_dim = input_dim
|
||||
self.output_dim = output_dim
|
||||
|
||||
self.features = nn.Sequential(
|
||||
nn.Conv2d(input_dim[0], 32, kernel_size=8, stride=4),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(32, 64, kernel_size=4, stride=2),
|
||||
nn.ReLU(),
|
||||
nn.Conv2d(64, 64, kernel_size=3, stride=1),
|
||||
nn.ReLU()
|
||||
)
|
||||
|
||||
self.fc = nn.Sequential(
|
||||
nn.Linear(self.feature_size(), 512),
|
||||
nn.ReLU(),
|
||||
nn.Linear(512, self.output_dim)
|
||||
)
|
||||
|
||||
def forward(self, x):
|
||||
x = self.features(x)
|
||||
x = x.view(x.size(0), -1)
|
||||
x = self.fc(x)
|
||||
return x
|
||||
|
||||
def feature_size(self):
|
||||
return self.features(autograd.Variable(torch.zeros(1, *self.input_dim))).view(1, -1).size(1)
|
||||
|
||||
|
||||
def act(self, state, epsilon):
|
||||
if random.random() > epsilon:
|
||||
state = Variable(torch.FloatTensor(np.float32(state)).unsqueeze(0), volatile=True)
|
||||
q_value = self.forward(state)
|
||||
action = q_value.max(1)[1].data[0]
|
||||
else:
|
||||
action = random.randrange(env.action_space.n)
|
||||
return action
|
||||
|
||||
class ReplayBuffer:
|
||||
def __init__(self, capacity):
|
||||
self.capacity = capacity # 经验回放的容量
|
||||
self.buffer = [] # 缓冲区
|
||||
self.position = 0
|
||||
|
||||
def push(self, state, action, reward, next_state, done):
|
||||
''' 缓冲区是一个队列，容量超出时去掉开始存入的转移(transition)
|
||||
'''
|
||||
if len(self.buffer) < self.capacity:
|
||||
self.buffer.append(None)
|
||||
self.buffer[self.position] = (state, action, reward, next_state, done)
|
||||
self.position = (self.position + 1) % self.capacity
|
||||
|
||||
def sample(self, batch_size):
|
||||
batch = random.sample(self.buffer, batch_size) # 随机采出小批量转移
|
||||
state, action, reward, next_state, done = zip(*batch) # 解压成状态，动作等
|
||||
return state, action, reward, next_state, done
|
||||
|
||||
def __len__(self):
|
||||
''' 返回当前存储的量
|
||||
'''
|
||||
return len(self.buffer)
|
||||
|
||||
class DQN:
|
||||
def __init__(self, state_dim, action_dim, cfg):
|
||||
|
||||
self.action_dim = action_dim # 总的动作个数
|
||||
self.device = cfg.device # 设备，cpu或gpu等
|
||||
self.gamma = cfg.gamma # 奖励的折扣因子
|
||||
# e-greedy策略相关参数
|
||||
self.frame_idx = 0 # 用于epsilon的衰减计数
|
||||
self.epsilon = lambda frame_idx: cfg.epsilon_end + \
|
||||
(cfg.epsilon_start - cfg.epsilon_end) * \
|
||||
math.exp(-1. * frame_idx / cfg.epsilon_decay)
|
||||
self.batch_size = cfg.batch_size
|
||||
self.policy_net = CNN(state_dim, action_dim).to(self.device)
|
||||
self.target_net = CNN(state_dim, action_dim).to(self.device)
|
||||
for target_param, param in zip(self.target_net.parameters(),self.policy_net.parameters()): # 复制参数到目标网路targe_net
|
||||
target_param.data.copy_(param.data)
|
||||
self.optimizer = optim.Adam(self.policy_net.parameters(), lr=cfg.lr) # 优化器
|
||||
self.memory = ReplayBuffer(cfg.memory_capacity) # 经验回放
|
||||
|
||||
def choose_action(self, state):
|
||||
''' 选择动作
|
||||
'''
|
||||
self.frame_idx += 1
|
||||
if random.random() > self.epsilon(self.frame_idx):
|
||||
with torch.no_grad():
|
||||
state = torch.tensor([state], device=self.device, dtype=torch.float32)
|
||||
q_values = self.policy_net(state)
|
||||
action = q_values.max(1)[1].item() # 选择Q值最大的动作
|
||||
else:
|
||||
action = random.randrange(self.action_dim)
|
||||
return action
|
||||
def update(self):
|
||||
if len(self.memory) < self.batch_size: # 当memory中不满足一个批量时，不更新策略
|
||||
return
|
||||
# 从经验回放中(replay memory)中随机采样一个批量的转移(transition)
|
||||
state_batch, action_batch, reward_batch, next_state_batch, done_batch = self.memory.sample(
|
||||
self.batch_size)
|
||||
# 转为张量
|
||||
state_batch = torch.tensor(state_batch, device=self.device, dtype=torch.float)
|
||||
action_batch = torch.tensor(action_batch, device=self.device).unsqueeze(1)
|
||||
reward_batch = torch.tensor(reward_batch, device=self.device, dtype=torch.float)
|
||||
next_state_batch = torch.tensor(next_state_batch, device=self.device, dtype=torch.float)
|
||||
done_batch = torch.tensor(np.float32(done_batch), device=self.device)
|
||||
q_values = self.policy_net(state_batch).gather(dim=1, index=action_batch) # 计算当前状态(s_t,a)对应的Q(s_t, a)
|
||||
next_q_values = self.target_net(next_state_batch).max(1)[0].detach() # 计算下一时刻的状态(s_t_,a)对应的Q值
|
||||
# 计算期望的Q值，对于终止状态，此时done_batch[0]=1, 对应的expected_q_value等于reward
|
||||
expected_q_values = reward_batch + self.gamma * next_q_values * (1-done_batch)
|
||||
loss = nn.MSELoss()(q_values, expected_q_values.unsqueeze(1)) # 计算均方根损失
|
||||
# 优化更新模型
|
||||
self.optimizer.zero_grad()
|
||||
loss.backward()
|
||||
for param in self.policy_net.parameters(): # clip防止梯度爆炸
|
||||
param.grad.data.clamp_(-1, 1)
|
||||
self.optimizer.step()
|
||||
|
||||
def save(self, path):
|
||||
torch.save(self.target_net.state_dict(), path+'dqn_checkpoint.pth')
|
||||
|
||||
def load(self, path):
|
||||
self.target_net.load_state_dict(torch.load(path+'dqn_checkpoint.pth'))
|
||||
for target_param, param in zip(self.target_net.parameters(), self.policy_net.parameters()):
|
||||
param.data.copy_(target_param.data)
|
||||
Binary file not shown.
Binary file not shown.
|
Before Width: | Height: | Size: 27 KiB |
Binary file not shown.
Binary file not shown.
|
Before Width: | Height: | Size: 40 KiB |
Binary file not shown.
Binary file not shown.
|
After Width: | Height: | Size: 26 KiB |
Binary file not shown.
Binary file not shown.
Binary file not shown.
|
After Width: | Height: | Size: 40 KiB |
148
codes/DQN/task0.py
Normal file
148
codes/DQN/task0.py
Normal file
@@ -0,0 +1,148 @@
|
||||
import sys
|
||||
import os
|
||||
curr_path = os.path.dirname(os.path.abspath(__file__)) # 当前文件所在绝对路径
|
||||
parent_path = os.path.dirname(curr_path) # 父路径
|
||||
sys.path.append(parent_path) # 添加路径到系统路径
|
||||
|
||||
import gym
|
||||
import torch
|
||||
import datetime
|
||||
import numpy as np
|
||||
from common.utils import save_results, make_dir
|
||||
from common.utils import plot_rewards
|
||||
from DQN.dqn import DQN
|
||||
|
||||
curr_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") # 获取当前时间
|
||||
|
||||
|
||||
class Config:
|
||||
'''超参数
|
||||
'''
|
||||
|
||||
def __init__(self):
|
||||
################################## 环境超参数 ###################################
|
||||
self.algo_name = 'DQN' # 算法名称
|
||||
self.env_name = 'CartPole-v0' # 环境名称
|
||||
self.device = torch.device(
|
||||
"cuda" if torch.cuda.is_available() else "cpu") # 检测GPUgjgjlkhfsf风刀霜的撒发十
|
||||
self.seed = 10 # 随机种子，置0则不设置随机种子
|
||||
self.train_eps = 200 # 训练的回合数
|
||||
self.test_eps = 30 # 测试的回合数
|
||||
################################################################################
|
||||
|
||||
################################## 算法超参数 ###################################
|
||||
self.gamma = 0.95 # 强化学习中的折扣因子
|
||||
self.epsilon_start = 0.90 # e-greedy策略中初始epsilon
|
||||
self.epsilon_end = 0.01 # e-greedy策略中的终止epsilon
|
||||
self.epsilon_decay = 500 # e-greedy策略中epsilon的衰减率
|
||||
self.lr = 0.0001 # 学习率
|
||||
self.memory_capacity = 100000 # 经验回放的容量
|
||||
self.batch_size = 64 # mini-batch SGD中的批量大小
|
||||
self.target_update = 4 # 目标网络的更新频率
|
||||
self.hidden_dim = 256 # 网络隐藏层
|
||||
################################################################################
|
||||
|
||||
################################# 保存结果相关参数 ##############################
|
||||
self.result_path = curr_path + "/outputs/" + self.env_name + \
|
||||
'/' + curr_time + '/results/' # 保存结果的路径
|
||||
self.model_path = curr_path + "/outputs/" + self.env_name + \
|
||||
'/' + curr_time + '/models/' # 保存模型的路径
|
||||
self.save = True # 是否保存图片
|
||||
################################################################################
|
||||
|
||||
|
||||
def env_agent_config(cfg):
|
||||
''' 创建环境和智能体
|
||||
'''
|
||||
env = gym.make(cfg.env_name) # 创建环境
|
||||
state_dim = env.observation_space.shape[0] # 状态维度
|
||||
action_dim = env.action_space.n # 动作维度
|
||||
agent = DQN(state_dim, action_dim, cfg) # 创建智能体
|
||||
if cfg.seed !=0: # 设置随机种子
|
||||
torch.manual_seed(cfg.seed)
|
||||
env.seed(cfg.seed)
|
||||
np.random.seed(cfg.seed)
|
||||
return env, agent
|
||||
|
||||
|
||||
def train(cfg, env, agent):
|
||||
''' 训练
|
||||
'''
|
||||
print('开始训练!')
|
||||
print(f'环境：{cfg.env_name}, 算法：{cfg.algo_name}, 设备：{cfg.device}')
|
||||
rewards = [] # 记录所有回合的奖励
|
||||
ma_rewards = [] # 记录所有回合的滑动平均奖励
|
||||
for i_ep in range(cfg.train_eps):
|
||||
ep_reward = 0 # 记录一回合内的奖励
|
||||
state = env.reset() # 重置环境，返回初始状态
|
||||
while True:
|
||||
action = agent.choose_action(state) # 选择动作
|
||||
next_state, reward, done, _ = env.step(action) # 更新环境，返回transition
|
||||
agent.memory.push(state, action, reward,
|
||||
next_state, done) # 保存transition
|
||||
state = next_state # 更新下一个状态
|
||||
agent.update() # 更新智能体
|
||||
ep_reward += reward # 累加奖励
|
||||
if done:
|
||||
break
|
||||
if (i_ep + 1) % cfg.target_update == 0: # 智能体目标网络更新
|
||||
agent.target_net.load_state_dict(agent.policy_net.state_dict())
|
||||
rewards.append(ep_reward)
|
||||
if ma_rewards:
|
||||
ma_rewards.append(0.9 * ma_rewards[-1] + 0.1 * ep_reward)
|
||||
else:
|
||||
ma_rewards.append(ep_reward)
|
||||
if (i_ep + 1) % 10 == 0:
|
||||
print('回合：{}/{}, 奖励：{}'.format(i_ep + 1, cfg.train_eps, ep_reward))
|
||||
print('完成训练！')
|
||||
env.close()
|
||||
return rewards, ma_rewards
|
||||
|
||||
|
||||
def test(cfg, env, agent):
|
||||
print('开始测试!')
|
||||
print(f'环境：{cfg.env_name}, 算法：{cfg.algo_name}, 设备：{cfg.device}')
|
||||
############# 由于测试不需要使用epsilon-greedy策略，所以相应的值设置为0 ###############
|
||||
cfg.epsilon_start = 0.0 # e-greedy策略中初始epsilon
|
||||
cfg.epsilon_end = 0.0 # e-greedy策略中的终止epsilon
|
||||
################################################################################
|
||||
rewards = [] # 记录所有回合的奖励
|
||||
ma_rewards = [] # 记录所有回合的滑动平均奖励
|
||||
for i_ep in range(cfg.test_eps):
|
||||
ep_reward = 0 # 记录一回合内的奖励
|
||||
state = env.reset() # 重置环境，返回初始状态
|
||||
while True:
|
||||
action = agent.choose_action(state) # 选择动作
|
||||
next_state, reward, done, _ = env.step(action) # 更新环境，返回transition
|
||||
state = next_state # 更新下一个状态
|
||||
ep_reward += reward # 累加奖励
|
||||
if done:
|
||||
break
|
||||
rewards.append(ep_reward)
|
||||
if ma_rewards:
|
||||
ma_rewards.append(ma_rewards[-1] * 0.9 + ep_reward * 0.1)
|
||||
else:
|
||||
ma_rewards.append(ep_reward)
|
||||
print(f"回合：{i_ep+1}/{cfg.test_eps}，奖励：{ep_reward:.1f}")
|
||||
print('完成测试！')
|
||||
env.close()
|
||||
return rewards, ma_rewards
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
cfg = Config()
|
||||
# 训练
|
||||
env, agent = env_agent_config(cfg)
|
||||
rewards, ma_rewards = train(cfg, env, agent)
|
||||
make_dir(cfg.result_path, cfg.model_path) # 创建保存结果和模型路径的文件夹
|
||||
agent.save(path=cfg.model_path) # 保存模型
|
||||
save_results(rewards, ma_rewards, tag='train',
|
||||
path=cfg.result_path) # 保存结果
|
||||
plot_rewards(rewards, ma_rewards, cfg, tag="train") # 画出结果
|
||||
# 测试
|
||||
env, agent = env_agent_config(cfg)
|
||||
agent.load(path=cfg.model_path) # 导入模型
|
||||
rewards, ma_rewards = test(cfg, env, agent)
|
||||
save_results(rewards, ma_rewards, tag='test',
|
||||
path=cfg.result_path) # 保存结果
|
||||
plot_rewards(rewards, ma_rewards, cfg, tag="test") # 画出结果
|
||||
File diff suppressed because one or more lines are too long
@@ -1,137 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
# coding=utf-8
|
||||
'''
|
||||
@Author: John
|
||||
@Email: johnjim0816@gmail.com
|
||||
@Date: 2020-06-12 00:48:57
|
||||
@LastEditor: John
|
||||
LastEditTime: 2021-09-15 15:34:13
|
||||
@Discription:
|
||||
@Environment: python 3.7.7
|
||||
'''
|
||||
import sys,os
|
||||
curr_path = os.path.dirname(os.path.abspath(__file__)) # 当前文件所在绝对路径
|
||||
parent_path = os.path.dirname(curr_path) # 父路径
|
||||
sys.path.append(parent_path) # 添加路径到系统路径
|
||||
|
||||
import gym
|
||||
import torch
|
||||
import datetime
|
||||
|
||||
from common.utils import save_results, make_dir
|
||||
from common.plot import plot_rewards
|
||||
from DQN.agent import DQN
|
||||
|
||||
curr_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") # 获取当前时间
|
||||
class DQNConfig:
|
||||
def __init__(self):
|
||||
self.algo = "DQN" # 算法名称
|
||||
self.env_name = 'CartPole-v0' # 环境名称
|
||||
self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # 检测GPU
|
||||
self.train_eps = 200 # 训练的回合数
|
||||
self.eval_eps = 30 # 测试的回合数
|
||||
# 超参数
|
||||
self.gamma = 0.95 # 强化学习中的折扣因子
|
||||
self.epsilon_start = 0.90 # e-greedy策略中初始epsilon
|
||||
self.epsilon_end = 0.01 # e-greedy策略中的终止epsilon
|
||||
self.epsilon_decay = 500 # e-greedy策略中epsilon的衰减率
|
||||
self.lr = 0.0001 # 学习率
|
||||
self.memory_capacity = 100000 # 经验回放的容量
|
||||
self.batch_size = 64 # mini-batch SGD中的批量大小
|
||||
self.target_update = 4 # 目标网络的更新频率
|
||||
self.hidden_dim = 256 # 网络隐藏层
|
||||
class PlotConfig:
|
||||
def __init__(self) -> None:
|
||||
self.algo = "DQN" # 算法名称
|
||||
self.env_name = 'CartPole-v0' # 环境名称
|
||||
self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # 检测GPU
|
||||
self.result_path = curr_path+"/outputs/" + self.env_name + \
|
||||
'/'+curr_time+'/results/' # 保存结果的路径
|
||||
self.model_path = curr_path+"/outputs/" + self.env_name + \
|
||||
'/'+curr_time+'/models/' # 保存模型的路径
|
||||
self.save = True # 是否保存图片
|
||||
|
||||
def env_agent_config(cfg,seed=1):
|
||||
''' 创建环境和智能体
|
||||
'''
|
||||
env = gym.make(cfg.env_name) # 创建环境
|
||||
env.seed(seed) # 设置随机种子
|
||||
n_states = env.observation_space.shape[0] # 状态数
|
||||
n_actions = env.action_space.n # 动作数
|
||||
agent = DQN(n_states,n_actions,cfg) # 创建智能体
|
||||
return env,agent
|
||||
|
||||
def train(cfg, env, agent):
|
||||
''' 训练
|
||||
'''
|
||||
print('开始训练!')
|
||||
print(f'环境：{cfg.env_name}, 算法：{cfg.algo}, 设备：{cfg.device}')
|
||||
rewards = [] # 记录所有回合的奖励
|
||||
ma_rewards = [] # 记录所有回合的滑动平均奖励
|
||||
for i_ep in range(cfg.train_eps):
|
||||
ep_reward = 0 # 记录一回合内的奖励
|
||||
state = env.reset() # 重置环境，返回初始状态
|
||||
while True:
|
||||
action = agent.choose_action(state) # 选择动作
|
||||
next_state, reward, done, _ = env.step(action) # 更新环境，返回transition
|
||||
agent.memory.push(state, action, reward, next_state, done) # 保存transition
|
||||
state = next_state # 更新下一个状态
|
||||
agent.update() # 更新智能体
|
||||
ep_reward += reward # 累加奖励
|
||||
if done:
|
||||
break
|
||||
if (i_ep+1) % cfg.target_update == 0: # 智能体目标网络更新
|
||||
agent.target_net.load_state_dict(agent.policy_net.state_dict())
|
||||
if (i_ep+1)%10 == 0:
|
||||
print('回合：{}/{}, 奖励：{}'.format(i_ep+1, cfg.train_eps, ep_reward))
|
||||
rewards.append(ep_reward)
|
||||
if ma_rewards:
|
||||
ma_rewards.append(0.9*ma_rewards[-1]+0.1*ep_reward)
|
||||
else:
|
||||
ma_rewards.append(ep_reward)
|
||||
print('完成训练！')
|
||||
return rewards, ma_rewards
|
||||
|
||||
def eval(cfg,env,agent):
|
||||
print('开始测试!')
|
||||
print(f'环境：{cfg.env_name}, 算法：{cfg.algo}, 设备：{cfg.device}')
|
||||
# 由于测试不需要使用epsilon-greedy策略，所以相应的值设置为0
|
||||
cfg.epsilon_start = 0.0 # e-greedy策略中初始epsilon
|
||||
cfg.epsilon_end = 0.0 # e-greedy策略中的终止epsilon
|
||||
rewards = [] # 记录所有回合的奖励
|
||||
ma_rewards = [] # 记录所有回合的滑动平均奖励
|
||||
for i_ep in range(cfg.eval_eps):
|
||||
ep_reward = 0 # 记录一回合内的奖励
|
||||
state = env.reset() # 重置环境，返回初始状态
|
||||
while True:
|
||||
action = agent.choose_action(state) # 选择动作
|
||||
next_state, reward, done, _ = env.step(action) # 更新环境，返回transition
|
||||
state = next_state # 更新下一个状态
|
||||
ep_reward += reward # 累加奖励
|
||||
if done:
|
||||
break
|
||||
rewards.append(ep_reward)
|
||||
if ma_rewards:
|
||||
ma_rewards.append(ma_rewards[-1]*0.9+ep_reward*0.1)
|
||||
else:
|
||||
ma_rewards.append(ep_reward)
|
||||
print(f"回合：{i_ep+1}/{cfg.eval_eps}, 奖励：{ep_reward:.1f}")
|
||||
print('完成测试！')
|
||||
return rewards,ma_rewards
|
||||
|
||||
if __name__ == "__main__":
|
||||
cfg = DQNConfig()
|
||||
plot_cfg = PlotConfig()
|
||||
# 训练
|
||||
env,agent = env_agent_config(cfg,seed=1)
|
||||
rewards, ma_rewards = train(cfg, env, agent)
|
||||
make_dir(plot_cfg.result_path, plot_cfg.model_path) # 创建保存结果和模型路径的文件夹
|
||||
agent.save(path=plot_cfg.model_path) # 保存模型
|
||||
save_results(rewards, ma_rewards, tag='train', path=plot_cfg.result_path) # 保存结果
|
||||
plot_rewards(rewards, ma_rewards, plot_cfg, tag="train") # 画出结果
|
||||
# 测试
|
||||
env,agent = env_agent_config(cfg,seed=10)
|
||||
agent.load(path=plot_cfg.model_path) # 导入模型
|
||||
rewards,ma_rewards = eval(cfg,env,agent)
|
||||
save_results(rewards,ma_rewards,tag='eval',path=plot_cfg.result_path) # 保存结果
|
||||
plot_rewards(rewards,ma_rewards, plot_cfg, tag="eval") # 画出结果
|
||||
148
codes/DQN/task1.py
Normal file
148
codes/DQN/task1.py
Normal file
@@ -0,0 +1,148 @@
|
||||
#!/usr/bin/env python
|
||||
# coding=utf-8
|
||||
'''
|
||||
Author: JiangJi
|
||||
Email: johnjim0816@gmail.com
|
||||
Date: 2021-12-22 11:14:17
|
||||
LastEditor: JiangJi
|
||||
LastEditTime: 2021-12-22 11:40:44
|
||||
Discription: 使用 Nature DQN 训练 CartPole-v1
|
||||
'''
|
||||
import sys
|
||||
import os
|
||||
curr_path = os.path.dirname(os.path.abspath(__file__)) # 当前文件所在绝对路径
|
||||
parent_path = os.path.dirname(curr_path) # 父路径
|
||||
sys.path.append(parent_path) # 添加路径到系统路径
|
||||
|
||||
import gym
|
||||
import torch
|
||||
import datetime
|
||||
from common.utils import save_results, make_dir
|
||||
from common.utils import plot_rewards, plot_rewards_cn
|
||||
from DQN.dqn import DQN
|
||||
|
||||
curr_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") # 获取当前时间
|
||||
algo_name = "DQN" # 算法名称
|
||||
env_name = 'CartPole-v1' # 环境名称
|
||||
class DQNConfig:
|
||||
''' 算法相关参数设置
|
||||
'''
|
||||
|
||||
def __init__(self):
|
||||
self.algo_name = algo_name # 算法名称
|
||||
self.env_name = env_name # 环境名称
|
||||
self.device = torch.device(
|
||||
"cuda" if torch.cuda.is_available() else "cpu") # 检测GPU
|
||||
self.train_eps = 200 # 训练的回合数
|
||||
self.test_eps = 30 # 测试的回合数
|
||||
# 超参数
|
||||
self.gamma = 0.95 # 强化学习中的折扣因子
|
||||
self.epsilon_start = 0.90 # e-greedy策略中初始epsilon
|
||||
self.epsilon_end = 0.01 # e-greedy策略中的终止epsilon
|
||||
self.epsilon_decay = 500 # e-greedy策略中epsilon的衰减率
|
||||
self.lr = 0.0001 # 学习率
|
||||
self.memory_capacity = 100000 # 经验回放的容量
|
||||
self.batch_size = 64 # mini-batch SGD中的批量大小
|
||||
self.target_update = 4 # 目标网络的更新频率
|
||||
self.hidden_dim = 256 # 网络隐藏层
|
||||
class PlotConfig:
|
||||
''' 绘图相关参数设置
|
||||
'''
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.algo_name = algo_name # 算法名称
|
||||
self.env_name = env_name # 环境名称
|
||||
self.device = torch.device(
|
||||
"cuda" if torch.cuda.is_available() else "cpu") # 检测GPU
|
||||
self.result_path = curr_path + "/outputs/" + self.env_name + \
|
||||
'/' + curr_time + '/results/' # 保存结果的路径
|
||||
self.model_path = curr_path + "/outputs/" + self.env_name + \
|
||||
'/' + curr_time + '/models/' # 保存模型的路径
|
||||
self.save = True # 是否保存图片
|
||||
|
||||
|
||||
def env_agent_config(cfg, seed=1):
|
||||
''' 创建环境和智能体
|
||||
'''
|
||||
env = gym.make(cfg.env_name) # 创建环境
|
||||
env.seed(seed) # 设置随机种子
|
||||
state_dim = env.observation_space.shape[0] # 状态维度
|
||||
action_dim = env.action_space.n # 动作维度
|
||||
agent = DQN(state_dim, action_dim, cfg) # 创建智能体
|
||||
return env, agent
|
||||
|
||||
def train(cfg, env, agent):
|
||||
''' 训练
|
||||
'''
|
||||
print('开始训练!')
|
||||
print(f'环境：{cfg.env_name}, 算法：{cfg.algo_name}, 设备：{cfg.device}')
|
||||
rewards = [] # 记录所有回合的奖励
|
||||
ma_rewards = [] # 记录所有回合的滑动平均奖励
|
||||
for i_ep in range(cfg.train_eps):
|
||||
ep_reward = 0 # 记录一回合内的奖励
|
||||
state = env.reset() # 重置环境，返回初始状态
|
||||
while True:
|
||||
action = agent.choose_action(state) # 选择动作
|
||||
next_state, reward, done, _ = env.step(action) # 更新环境，返回transition
|
||||
agent.memory.push(state, action, reward, next_state, done) # 保存transition
|
||||
state = next_state # 更新下一个状态
|
||||
agent.update() # 更新智能体
|
||||
ep_reward += reward # 累加奖励
|
||||
if done:
|
||||
break
|
||||
if (i_ep+1) % cfg.target_update == 0: # 智能体目标网络更新
|
||||
agent.target_net.load_state_dict(agent.policy_net.state_dict())
|
||||
rewards.append(ep_reward)
|
||||
if ma_rewards:
|
||||
ma_rewards.append(0.9*ma_rewards[-1]+0.1*ep_reward)
|
||||
else:
|
||||
ma_rewards.append(ep_reward)
|
||||
if (i_ep+1)%10 == 0:
|
||||
print('回合：{}/{}, 奖励：{}'.format(i_ep+1, cfg.train_eps, ep_reward))
|
||||
print('完成训练！')
|
||||
return rewards, ma_rewards
|
||||
|
||||
def test(cfg,env,agent):
|
||||
print('开始测试!')
|
||||
print(f'环境：{cfg.env_name}, 算法：{cfg.algo_name}, 设备：{cfg.device}')
|
||||
# 由于测试不需要使用epsilon-greedy策略，所以相应的值设置为0
|
||||
cfg.epsilon_start = 0.0 # e-greedy策略中初始epsilon
|
||||
cfg.epsilon_end = 0.0 # e-greedy策略中的终止epsilon
|
||||
rewards = [] # 记录所有回合的奖励
|
||||
ma_rewards = [] # 记录所有回合的滑动平均奖励
|
||||
for i_ep in range(cfg.test_eps):
|
||||
ep_reward = 0 # 记录一回合内的奖励
|
||||
state = env.reset() # 重置环境，返回初始状态
|
||||
while True:
|
||||
action = agent.choose_action(state) # 选择动作
|
||||
next_state, reward, done, _ = env.step(action) # 更新环境，返回transition
|
||||
state = next_state # 更新下一个状态
|
||||
ep_reward += reward # 累加奖励
|
||||
if done:
|
||||
break
|
||||
rewards.append(ep_reward)
|
||||
if ma_rewards:
|
||||
ma_rewards.append(ma_rewards[-1]*0.9+ep_reward*0.1)
|
||||
else:
|
||||
ma_rewards.append(ep_reward)
|
||||
print(f"回合：{i_ep+1}/{cfg.test_eps}，奖励：{ep_reward:.1f}")
|
||||
print('完成测试！')
|
||||
return rewards,ma_rewards
|
||||
if __name__ == "__main__":
|
||||
cfg = DQNConfig()
|
||||
plot_cfg = PlotConfig()
|
||||
# 训练
|
||||
env, agent = env_agent_config(cfg, seed=1)
|
||||
rewards, ma_rewards = train(cfg, env, agent)
|
||||
make_dir(plot_cfg.result_path, plot_cfg.model_path) # 创建保存结果和模型路径的文件夹
|
||||
agent.save(path=plot_cfg.model_path) # 保存模型
|
||||
save_results(rewards, ma_rewards, tag='train',
|
||||
path=plot_cfg.result_path) # 保存结果
|
||||
plot_rewards_cn(rewards, ma_rewards, plot_cfg, tag="train") # 画出结果
|
||||
# 测试
|
||||
env, agent = env_agent_config(cfg, seed=10)
|
||||
agent.load(path=plot_cfg.model_path) # 导入模型
|
||||
rewards, ma_rewards = test(cfg, env, agent)
|
||||
save_results(rewards, ma_rewards, tag='test',
|
||||
path=plot_cfg.result_path) # 保存结果
|
||||
plot_rewards_cn(rewards, ma_rewards, plot_cfg, tag="test") # 画出结果
|
||||
150
codes/DQN/task2.py
Normal file
150
codes/DQN/task2.py
Normal file
@@ -0,0 +1,150 @@
|
||||
#!/usr/bin/env python
|
||||
# coding=utf-8
|
||||
'''
|
||||
Author: JiangJi
|
||||
Email: johnjim0816@gmail.com
|
||||
Date: 2021-12-22 11:14:17
|
||||
LastEditor: JiangJi
|
||||
LastEditTime: 2021-12-22 15:27:48
|
||||
Discription: 使用 DQN-cnn 训练 PongNoFrameskip-v4
|
||||
'''
|
||||
import sys
|
||||
import os
|
||||
curr_path = os.path.dirname(os.path.abspath(__file__)) # 当前文件所在绝对路径
|
||||
parent_path = os.path.dirname(curr_path) # 父路径
|
||||
sys.path.append(parent_path) # 添加路径到系统路径
|
||||
|
||||
import gym
|
||||
import torch
|
||||
import datetime
|
||||
from common.utils import save_results, make_dir
|
||||
from common.utils import plot_rewards, plot_rewards_cn
|
||||
from common.atari_wrappers import make_atari, wrap_deepmind
|
||||
from DQN.dqn import DQN
|
||||
|
||||
curr_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") # 获取当前时间
|
||||
algo_name = 'DQN-cnn' # 算法名称
|
||||
env_name = 'PongNoFrameskip-v4' # 环境名称
|
||||
device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # 检测GPU
|
||||
class DQNConfig:
|
||||
''' 算法相关参数设置
|
||||
'''
|
||||
|
||||
def __init__(self):
|
||||
self.algo_name = algo_name # 算法名称
|
||||
self.env_name = env_name # 环境名称
|
||||
self.device = device # 检测GPU
|
||||
self.train_eps = 500 # 训练的回合数
|
||||
self.test_eps = 30 # 测试的回合数
|
||||
# 超参数
|
||||
self.gamma = 0.95 # 强化学习中的折扣因子
|
||||
self.epsilon_start = 0.90 # e-greedy策略中初始epsilon
|
||||
self.epsilon_end = 0.01 # e-greedy策略中的终止epsilon
|
||||
self.epsilon_decay = 500 # e-greedy策略中epsilon的衰减率
|
||||
self.lr = 0.0001 # 学习率
|
||||
self.memory_capacity = 100000 # 经验回放的容量
|
||||
self.batch_size = 64 # mini-batch SGD中的批量大小
|
||||
self.target_update = 4 # 目标网络的更新频率
|
||||
self.hidden_dim = 256 # 网络隐藏层
|
||||
class PlotConfig:
|
||||
''' 绘图相关参数设置
|
||||
'''
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.algo_name = algo_name # 算法名称
|
||||
self.env_name = env_name # 环境名称
|
||||
self.device = device # 检测GPU
|
||||
self.result_path = curr_path + "/outputs/" + self.env_name + \
|
||||
'/' + curr_time + '/results/' # 保存结果的路径
|
||||
self.model_path = curr_path + "/outputs/" + self.env_name + \
|
||||
'/' + curr_time + '/models/' # 保存模型的路径
|
||||
self.save = True # 是否保存图片
|
||||
|
||||
|
||||
def env_agent_config(cfg, seed=1):
|
||||
''' 创建环境和智能体
|
||||
'''
|
||||
env = make_atari(cfg.env_name) # 创建环境
|
||||
# env = wrap_deepmind(env)
|
||||
# env = wrap_pytorch(env)
|
||||
env.seed(seed) # 设置随机种子
|
||||
state_dim = env.observation_space.shape[0] # 状态维度
|
||||
action_dim = env.action_space.n # 动作维度
|
||||
agent = DQN(state_dim, action_dim, cfg) # 创建智能体
|
||||
return env, agent
|
||||
|
||||
def train(cfg, env, agent):
|
||||
''' 训练
|
||||
'''
|
||||
print('开始训练!')
|
||||
print(f'环境：{cfg.env_name}, 算法：{cfg.algo_name}, 设备：{cfg.device}')
|
||||
rewards = [] # 记录所有回合的奖励
|
||||
ma_rewards = [] # 记录所有回合的滑动平均奖励
|
||||
for i_ep in range(cfg.train_eps):
|
||||
ep_reward = 0 # 记录一回合内的奖励
|
||||
state = env.reset() # 重置环境，返回初始状态
|
||||
while True:
|
||||
action = agent.choose_action(state) # 选择动作
|
||||
next_state, reward, done, _ = env.step(action) # 更新环境，返回transition
|
||||
agent.memory.push(state, action, reward, next_state, done) # 保存transition
|
||||
state = next_state # 更新下一个状态
|
||||
agent.update() # 更新智能体
|
||||
ep_reward += reward # 累加奖励
|
||||
if done:
|
||||
break
|
||||
if (i_ep+1) % cfg.target_update == 0: # 智能体目标网络更新
|
||||
agent.target_net.load_state_dict(agent.policy_net.state_dict())
|
||||
rewards.append(ep_reward)
|
||||
if ma_rewards:
|
||||
ma_rewards.append(0.9*ma_rewards[-1]+0.1*ep_reward)
|
||||
else:
|
||||
ma_rewards.append(ep_reward)
|
||||
if (i_ep+1)%10 == 0:
|
||||
print('回合：{}/{}, 奖励：{}'.format(i_ep+1, cfg.train_eps, ep_reward))
|
||||
print('完成训练！')
|
||||
return rewards, ma_rewards
|
||||
|
||||
def test(cfg,env,agent):
|
||||
print('开始测试!')
|
||||
print(f'环境：{cfg.env_name}, 算法：{cfg.algo_name}, 设备：{cfg.device}')
|
||||
# 由于测试不需要使用epsilon-greedy策略，所以相应的值设置为0
|
||||
cfg.epsilon_start = 0.0 # e-greedy策略中初始epsilon
|
||||
cfg.epsilon_end = 0.0 # e-greedy策略中的终止epsilon
|
||||
rewards = [] # 记录所有回合的奖励
|
||||
ma_rewards = [] # 记录所有回合的滑动平均奖励
|
||||
for i_ep in range(cfg.test_eps):
|
||||
ep_reward = 0 # 记录一回合内的奖励
|
||||
state = env.reset() # 重置环境，返回初始状态
|
||||
while True:
|
||||
action = agent.choose_action(state) # 选择动作
|
||||
next_state, reward, done, _ = env.step(action) # 更新环境，返回transition
|
||||
state = next_state # 更新下一个状态
|
||||
ep_reward += reward # 累加奖励
|
||||
if done:
|
||||
break
|
||||
rewards.append(ep_reward)
|
||||
if ma_rewards:
|
||||
ma_rewards.append(ma_rewards[-1]*0.9+ep_reward*0.1)
|
||||
else:
|
||||
ma_rewards.append(ep_reward)
|
||||
print(f"回合：{i_ep+1}/{cfg.test_eps}，奖励：{ep_reward:.1f}")
|
||||
print('完成测试！')
|
||||
return rewards,ma_rewards
|
||||
if __name__ == "__main__":
|
||||
cfg = DQNConfig()
|
||||
plot_cfg = PlotConfig()
|
||||
# 训练
|
||||
env, agent = env_agent_config(cfg, seed=1)
|
||||
rewards, ma_rewards = train(cfg, env, agent)
|
||||
make_dir(plot_cfg.result_path, plot_cfg.model_path) # 创建保存结果和模型路径的文件夹
|
||||
agent.save(path=plot_cfg.model_path) # 保存模型
|
||||
save_results(rewards, ma_rewards, tag='train',
|
||||
path=plot_cfg.result_path) # 保存结果
|
||||
plot_rewards_cn(rewards, ma_rewards, plot_cfg, tag="train") # 画出结果
|
||||
# 测试
|
||||
env, agent = env_agent_config(cfg, seed=10)
|
||||
agent.load(path=plot_cfg.model_path) # 导入模型
|
||||
rewards, ma_rewards = test(cfg, env, agent)
|
||||
save_results(rewards, ma_rewards, tag='test',
|
||||
path=plot_cfg.result_path) # 保存结果
|
||||
plot_rewards_cn(rewards, ma_rewards, plot_cfg, tag="test") # 画出结果
|
||||
@@ -6,7 +6,7 @@
|
||||
|
||||
<img src="../../easy_rl_book/res/ch12/assets/pendulum_1.png" alt="image-20210915161550713" style="zoom:50%;" />
|
||||
|
||||
该环境的状态数有三个，设摆针竖直方向上的顺时针旋转角为$\theta$，$\theta$设在$[-\pi,\pi]$之间，则相应的状态为$[cos\theta,sin\theta,\dot{\theta}]$，即表示角度和角速度，我们的动作则是一个-2到2之间的力矩，它是一个连续量，因而该环境不能用离散动作的算法比如 DQN 来解决。关于奖励是根据相关的物理原理而计算出的等式，如下：
|
||||
该环境的状态维度有三个，设摆针竖直方向上的顺时针旋转角为$\theta$，$\theta$设在$[-\pi,\pi]$之间，则相应的状态为$[cos\theta,sin\theta,\dot{\theta}]$，即表示角度和角速度，我们的动作则是一个-2到2之间的力矩，它是一个连续量，因而该环境不能用离散动作的算法比如 DQN 来解决。关于奖励是根据相关的物理原理而计算出的等式，如下：
|
||||
$$
|
||||
-\left(\theta^{2}+0.1 * \hat{\theta}^{2}+0.001 * \text { action }^{2}\right)
|
||||
$$
|
||||
|
||||
@@ -14,21 +14,21 @@ CartPole-v0是一个经典的入门环境，如下图，它通过向左(动作=0
|
||||
import gym
|
||||
env = gym.make('CartPole-v0') # 建立环境
|
||||
env.seed(1) # 随机种子
|
||||
n_states = env.observation_space.shape[0] # 状态数
|
||||
n_actions = env.action_space.n # 动作数
|
||||
state_dim = env.observation_space.shape[0] # 状态维度
|
||||
action_dim = env.action_space.n # 动作维度
|
||||
state = env.reset() # 初始化环境
|
||||
print(f"状态数：{n_states}，动作数：{n_actions}")
|
||||
print(f"状态维度：{state_dim}，动作维度：{action_dim}")
|
||||
print(f"初始状态：{state}")
|
||||
```
|
||||
|
||||
可以得到结果：
|
||||
|
||||
```bash
|
||||
状态数：4，动作数：2
|
||||
状态维度：4，动作维度：2
|
||||
初始状态：[ 0.03073904 0.00145001 -0.03088818 -0.03131252]
|
||||
```
|
||||
|
||||
该环境状态数是四个，分别为车的位置、车的速度、杆的角度以及杆顶部的速度，动作数为两个，并且是离散的向左或者向右。理论上达到最优化算法的情况下，推车杆是一直能保持平衡的，也就是每回合的步数是无限，但是这不方便训练，所以环境内部设置了每回合的最大步数为200，也就是说理想情况下，只需要我们每回合的奖励达到200就算训练完成。
|
||||
该环境状态维度是四个，分别为车的位置、车的速度、杆的角度以及杆顶部的速度，动作维度为两个，并且是离散的向左或者向右。理论上达到最优化算法的情况下，推车杆是一直能保持平衡的，也就是每回合的步数是无限，但是这不方便训练，所以环境内部设置了每回合的最大步数为200，也就是说理想情况下，只需要我们每回合的奖励达到200就算训练完成。
|
||||
|
||||
## DQN基本接口
|
||||
|
||||
@@ -125,7 +125,7 @@ class ReplayBuffer:
|
||||
class MLP(nn.Module):
|
||||
def __init__(self, input_dim,output_dim,hidden_dim=128):
|
||||
""" 初始化q网络，为全连接网络
|
||||
input_dim: 输入的特征数即环境的状态数
|
||||
input_dim: 输入的特征数即环境的状态维度
|
||||
output_dim: 输出的动作维度
|
||||
"""
|
||||
super(MLP, self).__init__()
|
||||
|
||||
@@ -27,21 +27,21 @@ env = gym.make('CliffWalking-v0') # 定义环境
|
||||
env = CliffWalkingWapper(env) # 装饰环境
|
||||
```
|
||||
|
||||
这里我们在程序中使用了一个装饰器重新定义环境，但不影响对环境的理解，感兴趣的同学具体看相关代码。可以由于gym环境封装得比较好，所以我们想要使用这个环境只需要使用gym.make命令输入函数名即可，然后我们可以查看环境的状态和动作数目：
|
||||
这里我们在程序中使用了一个装饰器重新定义环境，但不影响对环境的理解，感兴趣的同学具体看相关代码。可以由于gym环境封装得比较好，所以我们想要使用这个环境只需要使用gym.make命令输入函数名即可，然后我们可以查看环境的状态和动作维度目：
|
||||
|
||||
```python
|
||||
n_states = env.observation_space.n # 状态数
|
||||
n_actions = env.action_space.n # 动作数
|
||||
print(f"状态数：{n_states}，动作数：{n_actions}")
|
||||
state_dim = env.observation_space.n # 状态维度
|
||||
action_dim = env.action_space.n # 动作维度
|
||||
print(f"状态维度：{state_dim}，动作维度：{action_dim}")
|
||||
```
|
||||
|
||||
打印出来的结果如下：
|
||||
|
||||
```bash
|
||||
状态数：48，动作数：4
|
||||
状态维度：48，动作维度：4
|
||||
```
|
||||
|
||||
我们的状态数是48个，这里我们设置的是智能体当前所在网格的编号，而动作数是4，这表示有0，1，2，3对应着上下左右四个动作。另外我们也可以初始化环境并打印当前所在的状态：
|
||||
我们的状态维度是48个，这里我们设置的是智能体当前所在网格的编号，而动作维度是4，这表示有0，1，2，3对应着上下左右四个动作。另外我们也可以初始化环境并打印当前所在的状态：
|
||||
|
||||
```python
|
||||
state = env.reset()
|
||||
@@ -72,9 +72,9 @@ print(state)
|
||||
env = gym.make('CliffWalking-v0') # 定义环境
|
||||
env = CliffWalkingWapper(env) # 装饰环境
|
||||
env.seed(1) # 设置随机种子
|
||||
n_states = env.observation_space.n # 状态数
|
||||
n_actions = env.action_space.n # 动作数
|
||||
agent = QLearning(n_states,n_actions,cfg) # cfg存储算法相关参数
|
||||
state_dim = env.observation_space.n # 状态维度
|
||||
action_dim = env.action_space.n # 动作维度
|
||||
agent = QLearning(state_dim,action_dim,cfg) # cfg存储算法相关参数
|
||||
for i_ep in range(cfg.train_eps): # cfg.train_eps表示最大训练的回合数
|
||||
ep_reward = 0 # 记录每个回合的奖励
|
||||
state = env.reset() # 重置环境
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
@Email: johnjim0816@gmail.com
|
||||
@Date: 2020-06-12 00:50:49
|
||||
@LastEditor: John
|
||||
LastEditTime: 2021-05-04 22:28:06
|
||||
LastEditTime: 2021-11-19 18:07:09
|
||||
@Discription:
|
||||
@Environment: python 3.7.7
|
||||
'''
|
||||
@@ -16,14 +16,54 @@ LastEditTime: 2021-05-04 22:28:06
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.optim as optim
|
||||
import torch.nn.functional as F
|
||||
import random
|
||||
import math
|
||||
import numpy as np
|
||||
from common.memory import ReplayBuffer
|
||||
from common.model import MLP
|
||||
|
||||
class ReplayBuffer:
|
||||
def __init__(self, capacity):
|
||||
self.capacity = capacity # 经验回放的容量
|
||||
self.buffer = [] # 缓冲区
|
||||
self.position = 0
|
||||
|
||||
def push(self, state, action, reward, next_state, done):
|
||||
''' 缓冲区是一个队列，容量超出时去掉开始存入的转移(transition)
|
||||
'''
|
||||
if len(self.buffer) < self.capacity:
|
||||
self.buffer.append(None)
|
||||
self.buffer[self.position] = (state, action, reward, next_state, done)
|
||||
self.position = (self.position + 1) % self.capacity
|
||||
|
||||
def sample(self, batch_size):
|
||||
batch = random.sample(self.buffer, batch_size) # 随机采出小批量转移
|
||||
state, action, reward, next_state, done = zip(*batch) # 解压成状态，动作等
|
||||
return state, action, reward, next_state, done
|
||||
|
||||
def __len__(self):
|
||||
''' 返回当前存储的量
|
||||
'''
|
||||
return len(self.buffer)
|
||||
|
||||
class MLP(nn.Module):
|
||||
def __init__(self, state_dim,action_dim,hidden_dim=128):
|
||||
""" 初始化q网络，为全连接网络
|
||||
state_dim: 输入的特征数即环境的状态维度
|
||||
action_dim: 输出的动作维度
|
||||
"""
|
||||
super(MLP, self).__init__()
|
||||
self.fc1 = nn.Linear(state_dim, hidden_dim) # 输入层
|
||||
self.fc2 = nn.Linear(hidden_dim,hidden_dim) # 隐藏层
|
||||
self.fc3 = nn.Linear(hidden_dim, action_dim) # 输出层
|
||||
|
||||
def forward(self, x):
|
||||
# 各层对应的激活函数
|
||||
x = F.relu(self.fc1(x))
|
||||
x = F.relu(self.fc2(x))
|
||||
return self.fc3(x)
|
||||
|
||||
class DoubleDQN:
|
||||
def __init__(self, state_dim, action_dim, cfg):
|
||||
|
||||
self.action_dim = action_dim # 总的动作个数
|
||||
self.device = cfg.device # 设备，cpu或gpu等
|
||||
self.gamma = cfg.gamma
|
||||
@@ -43,8 +83,15 @@ class DoubleDQN:
|
||||
self.optimizer = optim.Adam(self.policy_net.parameters(), lr=cfg.lr)
|
||||
self.loss = 0
|
||||
self.memory = ReplayBuffer(cfg.memory_capacity)
|
||||
def predict(self,state):
|
||||
with torch.no_grad():
|
||||
|
||||
def choose_action(self, state):
|
||||
'''选择动作
|
||||
'''
|
||||
self.actions_count += 1
|
||||
self.epsilon = self.epsilon_end + (self.epsilon_start - self.epsilon_end) * \
|
||||
math.exp(-1. * self.actions_count / self.epsilon_decay)
|
||||
if random.random() > self.epsilon:
|
||||
with torch.no_grad():
|
||||
# 先转为张量便于丢给神经网络,state元素数据原本为float64
|
||||
# 注意state=torch.tensor(state).unsqueeze(0)跟state=torch.tensor([state])等价
|
||||
state = torch.tensor(
|
||||
@@ -55,15 +102,6 @@ class DoubleDQN:
|
||||
# 如torch.return_types.max(values=tensor([10.3587]),indices=tensor([0]))
|
||||
# 所以tensor.max(1)[1]返回最大值对应的下标，即action
|
||||
action = q_value.max(1)[1].item()
|
||||
return action
|
||||
def choose_action(self, state):
|
||||
'''选择动作
|
||||
'''
|
||||
self.actions_count += 1
|
||||
self.epsilon = self.epsilon_end + (self.epsilon_start - self.epsilon_end) * \
|
||||
math.exp(-1. * self.actions_count / self.epsilon_decay)
|
||||
if random.random() > self.epsilon:
|
||||
action = self.predict(state)
|
||||
else:
|
||||
action = random.randrange(self.action_dim)
|
||||
return action
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
|
Before Width: | Height: | Size: 47 KiB |
Binary file not shown.
Binary file not shown.
Binary file not shown.
|
Before Width: | Height: | Size: 57 KiB |
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
|
After Width: | Height: | Size: 37 KiB |
Binary file not shown.
Binary file not shown.
Binary file not shown.
|
After Width: | Height: | Size: 39 KiB |
144
codes/DoubleDQN/task0.py
Normal file
144
codes/DoubleDQN/task0.py
Normal file
@@ -0,0 +1,144 @@
|
||||
#!/usr/bin/env python
|
||||
# coding=utf-8
|
||||
'''
|
||||
Author: JiangJi
|
||||
Email: johnjim0816@gmail.com
|
||||
Date: 2021-11-07 18:10:37
|
||||
LastEditor: JiangJi
|
||||
LastEditTime: 2021-12-29 15:02:30
|
||||
Discription:
|
||||
'''
|
||||
|
||||
import sys,os
|
||||
curr_path = os.path.dirname(os.path.abspath(__file__)) # 当前文件所在绝对路径
|
||||
parent_path = os.path.dirname(curr_path) # 父路径
|
||||
sys.path.append(parent_path) # 添加路径到系统路径
|
||||
|
||||
import gym
|
||||
import torch
|
||||
import datetime
|
||||
|
||||
from common.utils import save_results, make_dir
|
||||
from common.utils import plot_rewards
|
||||
from DoubleDQN.double_dqn import DoubleDQN
|
||||
|
||||
curr_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") # 获取当前时间
|
||||
|
||||
class Config:
|
||||
def __init__(self):
|
||||
################################## 环境超参数 ###################################
|
||||
self.algo_name = 'DoubleDQN' # 算法名称
|
||||
self.env_name = 'CartPole-v0' # 环境名称
|
||||
self.device = torch.device(
|
||||
"cuda" if torch.cuda.is_available() else "cpu") # 检测GPU
|
||||
self.train_eps = 200 # 训练的回合数
|
||||
self.test_eps = 30 # 测试的回合数
|
||||
################################################################################
|
||||
|
||||
################################## 算法超参数 ###################################
|
||||
self.gamma = 0.95 # 强化学习中的折扣因子
|
||||
self.epsilon_start = 0.95 # e-greedy策略中初始epsilon
|
||||
self.epsilon_end = 0.01 # e-greedy策略中的终止epsilon
|
||||
self.epsilon_decay = 500 # e-greedy策略中epsilon的衰减率
|
||||
self.lr = 0.0001 # 学习率
|
||||
self.memory_capacity = 100000 # 经验回放的容量
|
||||
self.batch_size = 64 # mini-batch SGD中的批量大小
|
||||
self.target_update = 2 # 目标网络的更新频率
|
||||
self.hidden_dim = 256 # 网络隐藏层
|
||||
################################################################################
|
||||
|
||||
################################# 保存结果相关参数 ##############################
|
||||
self.result_path = curr_path + "/outputs/" + self.env_name + \
|
||||
'/' + curr_time + '/results/' # 保存结果的路径
|
||||
self.model_path = curr_path + "/outputs/" + self.env_name + \
|
||||
'/' + curr_time + '/models/' # 保存模型的路径
|
||||
self.save = True # 是否保存图片
|
||||
################################################################################
|
||||
|
||||
|
||||
def env_agent_config(cfg,seed=1):
|
||||
env = gym.make(cfg.env_name)
|
||||
env.seed(seed)
|
||||
state_dim = env.observation_space.shape[0]
|
||||
action_dim = env.action_space.n
|
||||
agent = DoubleDQN(state_dim,action_dim,cfg)
|
||||
return env,agent
|
||||
|
||||
def train(cfg,env,agent):
|
||||
print('开始训练!')
|
||||
print(f'环境：{cfg.env_name}, 算法：{cfg.algo_name}, 设备：{cfg.device}')
|
||||
rewards = [] # 记录所有回合的奖励
|
||||
ma_rewards = [] # 记录所有回合的滑动平均奖励
|
||||
for i_ep in range(cfg.train_eps):
|
||||
ep_reward = 0 # 记录一回合内的奖励
|
||||
state = env.reset() # 重置环境，返回初始状态
|
||||
while True:
|
||||
action = agent.choose_action(state)
|
||||
next_state, reward, done, _ = env.step(action)
|
||||
ep_reward += reward
|
||||
agent.memory.push(state, action, reward, next_state, done)
|
||||
state = next_state
|
||||
agent.update()
|
||||
if done:
|
||||
break
|
||||
if i_ep % cfg.target_update == 0:
|
||||
agent.target_net.load_state_dict(agent.policy_net.state_dict())
|
||||
if (i_ep+1)%10 == 0:
|
||||
print(f'回合：{i_ep+1}/{cfg.train_eps}，奖励：{ep_reward}')
|
||||
rewards.append(ep_reward)
|
||||
if ma_rewards:
|
||||
ma_rewards.append(
|
||||
0.9*ma_rewards[-1]+0.1*ep_reward)
|
||||
else:
|
||||
ma_rewards.append(ep_reward)
|
||||
print('完成训练！')
|
||||
env.close()
|
||||
return rewards,ma_rewards
|
||||
|
||||
def test(cfg,env,agent):
|
||||
print('开始测试!')
|
||||
print(f'环境：{cfg.env_name}, 算法：{cfg.algo_name}, 设备：{cfg.device}')
|
||||
############# 由于测试不需要使用epsilon-greedy策略，所以相应的值设置为0 ###############
|
||||
cfg.epsilon_start = 0.0 # e-greedy策略中初始epsilon
|
||||
cfg.epsilon_end = 0.0 # e-greedy策略中的终止epsilon
|
||||
################################################################################
|
||||
rewards = [] # 记录所有回合的奖励
|
||||
ma_rewards = [] # 记录所有回合的滑动平均奖励
|
||||
|
||||
for i_ep in range(cfg.test_eps):
|
||||
state = env.reset()
|
||||
ep_reward = 0
|
||||
while True:
|
||||
action = agent.choose_action(state)
|
||||
next_state, reward, done, _ = env.step(action)
|
||||
state = next_state
|
||||
ep_reward += reward
|
||||
if done:
|
||||
break
|
||||
rewards.append(ep_reward)
|
||||
if ma_rewards:
|
||||
ma_rewards.append(ma_rewards[-1]*0.9+ep_reward*0.1)
|
||||
else:
|
||||
ma_rewards.append(ep_reward)
|
||||
print(f"回合：{i_ep+1}/{cfg.test_eps}，奖励：{ep_reward:.1f}")
|
||||
print('完成测试！')
|
||||
env.close()
|
||||
return rewards,ma_rewards
|
||||
|
||||
if __name__ == "__main__":
|
||||
cfg = Config()
|
||||
# 训练
|
||||
env, agent = env_agent_config(cfg)
|
||||
rewards, ma_rewards = train(cfg, env, agent)
|
||||
make_dir(cfg.result_path, cfg.model_path) # 创建保存结果和模型路径的文件夹
|
||||
agent.save(path=cfg.model_path) # 保存模型
|
||||
save_results(rewards, ma_rewards, tag='train',
|
||||
path=cfg.result_path) # 保存结果
|
||||
plot_rewards(rewards, ma_rewards, cfg, tag="train") # 画出结果
|
||||
# 测试
|
||||
env, agent = env_agent_config(cfg)
|
||||
agent.load(path=cfg.model_path) # 导入模型
|
||||
rewards, ma_rewards = test(cfg, env, agent)
|
||||
save_results(rewards, ma_rewards, tag='test',
|
||||
path=cfg.result_path) # 保存结果
|
||||
plot_rewards(rewards, ma_rewards, cfg, tag="test") # 画出结果
|
||||
@@ -1,194 +0,0 @@
|
||||
{
|
||||
"metadata": {
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.7.10"
|
||||
},
|
||||
"orig_nbformat": 2,
|
||||
"kernelspec": {
|
||||
"name": "python3710jvsc74a57bd0366e1054dee9d4501b0eb8f87335afd3c67fc62db6ee611bbc7f8f5a1fefe232",
|
||||
"display_name": "Python 3.7.10 64-bit ('py37': conda)"
|
||||
},
|
||||
"metadata": {
|
||||
"interpreter": {
|
||||
"hash": "366e1054dee9d4501b0eb8f87335afd3c67fc62db6ee611bbc7f8f5a1fefe232"
|
||||
}
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2,
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import sys\n",
|
||||
"from pathlib import Path\n",
|
||||
"curr_path = str(Path().absolute())\n",
|
||||
"parent_path = str(Path().absolute().parent)\n",
|
||||
"sys.path.append(parent_path) # add current terminal path to sys.path"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import gym\n",
|
||||
"import torch\n",
|
||||
"import datetime\n",
|
||||
"from DoubleDQN.agent import DoubleDQN\n",
|
||||
"from common.plot import plot_rewards\n",
|
||||
"from common.utils import save_results, make_dir\n",
|
||||
"\n",
|
||||
"curr_time = datetime.datetime.now().strftime(\n",
|
||||
" \"%Y%m%d-%H%M%S\") # obtain current time"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"class DoubleDQNConfig:\n",
|
||||
" def __init__(self):\n",
|
||||
" self.algo = \"DoubleDQN\" # name of algo\n",
|
||||
" self.env = 'CartPole-v0' # env name\n",
|
||||
" self.result_path = curr_path+\"/outputs/\" + self.env + \\\n",
|
||||
" '/'+curr_time+'/results/' # path to save results\n",
|
||||
" self.model_path = curr_path+\"/outputs/\" + self.env + \\\n",
|
||||
" '/'+curr_time+'/models/' # path to save models\n",
|
||||
" self.train_eps = 200 # max tranng episodes\n",
|
||||
" self.eval_eps = 50 # max evaling episodes\n",
|
||||
" self.gamma = 0.95\n",
|
||||
" self.epsilon_start = 1 # start epsilon of e-greedy policy\n",
|
||||
" self.epsilon_end = 0.01 \n",
|
||||
" self.epsilon_decay = 500\n",
|
||||
" self.lr = 0.001 # learning rate\n",
|
||||
" self.memory_capacity = 100000 # capacity of Replay Memory\n",
|
||||
" self.batch_size = 64\n",
|
||||
" self.target_update = 2 # update frequency of target net\n",
|
||||
" self.device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\") # check gpu\n",
|
||||
" self.hidden_dim = 256 # hidden size of net"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def env_agent_config(cfg,seed=1):\n",
|
||||
" env = gym.make(cfg.env) \n",
|
||||
" env.seed(seed)\n",
|
||||
" state_dim = env.observation_space.shape[0]\n",
|
||||
" action_dim = env.action_space.n\n",
|
||||
" agent = DoubleDQN(state_dim,action_dim,cfg)\n",
|
||||
" return env,agent"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def train(cfg,env,agent):\n",
|
||||
" print('Start to train !')\n",
|
||||
" rewards,ma_rewards = [],[]\n",
|
||||
" for i_ep in range(cfg.train_eps):\n",
|
||||
" state = env.reset() \n",
|
||||
" ep_reward = 0\n",
|
||||
" while True:\n",
|
||||
" action = agent.choose_action(state) \n",
|
||||
" next_state, reward, done, _ = env.step(action)\n",
|
||||
" ep_reward += reward\n",
|
||||
" agent.memory.push(state, action, reward, next_state, done) \n",
|
||||
" state = next_state \n",
|
||||
" agent.update() \n",
|
||||
" if done:\n",
|
||||
" break\n",
|
||||
" if i_ep % cfg.target_update == 0:\n",
|
||||
" agent.target_net.load_state_dict(agent.policy_net.state_dict())\n",
|
||||
" if (i_ep+1)%10 == 0:\n",
|
||||
" print(f'Episode:{i_ep+1}/{cfg.train_eps}, Reward:{ep_reward}')\n",
|
||||
" rewards.append(ep_reward)\n",
|
||||
" if ma_rewards:\n",
|
||||
" ma_rewards.append(\n",
|
||||
" 0.9*ma_rewards[-1]+0.1*ep_reward)\n",
|
||||
" else:\n",
|
||||
" ma_rewards.append(ep_reward) \n",
|
||||
" print('Complete training！')\n",
|
||||
" return rewards,ma_rewards"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def eval(cfg,env,agent):\n",
|
||||
" print('Start to eval !')\n",
|
||||
" print(f'Env:{cfg.env}, Algorithm:{cfg.algo}, Device:{cfg.device}')\n",
|
||||
" rewards = [] \n",
|
||||
" ma_rewards = []\n",
|
||||
" for i_ep in range(cfg.eval_eps):\n",
|
||||
" state = env.reset() \n",
|
||||
" ep_reward = 0 \n",
|
||||
" while True:\n",
|
||||
" action = agent.predict(state) \n",
|
||||
" next_state, reward, done, _ = env.step(action) \n",
|
||||
" state = next_state \n",
|
||||
" ep_reward += reward\n",
|
||||
" if done:\n",
|
||||
" break\n",
|
||||
" rewards.append(ep_reward)\n",
|
||||
" if ma_rewards:\n",
|
||||
" ma_rewards.append(ma_rewards[-1]*0.9+ep_reward*0.1)\n",
|
||||
" else:\n",
|
||||
" ma_rewards.append(ep_reward)\n",
|
||||
" print(f\"Episode:{i_ep+1}/{cfg.eval_eps}, reward:{ep_reward:.1f}\")\n",
|
||||
" print('Complete evaling！')\n",
|
||||
" return rewards,ma_rewards "
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"if __name__ == \"__main__\":\n",
|
||||
" cfg = DoubleDQNConfig()\n",
|
||||
" # train\n",
|
||||
" env,agent = env_agent_config(cfg,seed=1)\n",
|
||||
" rewards, ma_rewards = train(cfg, env, agent)\n",
|
||||
" make_dir(cfg.result_path, cfg.model_path)\n",
|
||||
" agent.save(path=cfg.model_path)\n",
|
||||
" save_results(rewards, ma_rewards, tag='train', path=cfg.result_path)\n",
|
||||
" plot_rewards(rewards, ma_rewards, tag=\"train\",\n",
|
||||
" algo=cfg.algo, path=cfg.result_path)\n",
|
||||
"\n",
|
||||
" # eval\n",
|
||||
" env,agent = env_agent_config(cfg,seed=10)\n",
|
||||
" agent.load(path=cfg.model_path)\n",
|
||||
" rewards,ma_rewards = eval(cfg,env,agent)\n",
|
||||
" save_results(rewards,ma_rewards,tag='eval',path=cfg.result_path)\n",
|
||||
" plot_rewards(rewards,ma_rewards,tag=\"eval\",env=cfg.env,algo = cfg.algo,path=cfg.result_path)"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,123 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
# coding=utf-8
|
||||
'''
|
||||
@Author: John
|
||||
@Email: johnjim0816@gmail.com
|
||||
@Date: 2020-06-12 00:48:57
|
||||
@LastEditor: John
|
||||
LastEditTime: 2021-09-10 15:26:05
|
||||
@Discription:
|
||||
@Environment: python 3.7.7
|
||||
'''
|
||||
import sys,os
|
||||
curr_path = os.path.dirname(__file__)
|
||||
parent_path = os.path.dirname(curr_path)
|
||||
sys.path.append(parent_path) # add current terminal path to sys.path
|
||||
|
||||
import gym
|
||||
import torch
|
||||
import datetime
|
||||
from DoubleDQN.agent import DoubleDQN
|
||||
from common.plot import plot_rewards
|
||||
from common.utils import save_results, make_dir
|
||||
|
||||
curr_time = datetime.datetime.now().strftime(
|
||||
"%Y%m%d-%H%M%S") # obtain current time
|
||||
|
||||
class DoubleDQNConfig:
|
||||
def __init__(self):
|
||||
self.algo = "DoubleDQN" # name of algo
|
||||
self.env = 'CartPole-v0' # env name
|
||||
self.result_path = curr_path+"/outputs/" + self.env + \
|
||||
'/'+curr_time+'/results/' # path to save results
|
||||
self.model_path = curr_path+"/outputs/" + self.env + \
|
||||
'/'+curr_time+'/models/' # path to save models
|
||||
self.train_eps = 200 # max tranng episodes
|
||||
self.eval_eps = 50 # max evaling episodes
|
||||
self.gamma = 0.95
|
||||
self.epsilon_start = 1 # start epsilon of e-greedy policy
|
||||
self.epsilon_end = 0.01
|
||||
self.epsilon_decay = 500
|
||||
self.lr = 0.001 # learning rate
|
||||
self.memory_capacity = 100000 # capacity of Replay Memory
|
||||
self.batch_size = 64
|
||||
self.target_update = 2 # update frequency of target net
|
||||
self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # check gpu
|
||||
self.hidden_dim = 256 # hidden size of net
|
||||
|
||||
def env_agent_config(cfg,seed=1):
|
||||
env = gym.make(cfg.env)
|
||||
env.seed(seed)
|
||||
state_dim = env.observation_space.shape[0]
|
||||
action_dim = env.action_space.n
|
||||
agent = DoubleDQN(state_dim,action_dim,cfg)
|
||||
return env,agent
|
||||
|
||||
def train(cfg,env,agent):
|
||||
print('Start to train !')
|
||||
rewards,ma_rewards = [],[]
|
||||
for i_ep in range(cfg.train_eps):
|
||||
state = env.reset()
|
||||
ep_reward = 0
|
||||
while True:
|
||||
action = agent.choose_action(state)
|
||||
next_state, reward, done, _ = env.step(action)
|
||||
ep_reward += reward
|
||||
agent.memory.push(state, action, reward, next_state, done)
|
||||
state = next_state
|
||||
agent.update()
|
||||
if done:
|
||||
break
|
||||
if i_ep % cfg.target_update == 0:
|
||||
agent.target_net.load_state_dict(agent.policy_net.state_dict())
|
||||
print(f'Episode:{i_ep+1}/{cfg.train_eps}, Reward:{ep_reward},Epsilon:{agent.epsilon:.2f}')
|
||||
rewards.append(ep_reward)
|
||||
if ma_rewards:
|
||||
ma_rewards.append(
|
||||
0.9*ma_rewards[-1]+0.1*ep_reward)
|
||||
else:
|
||||
ma_rewards.append(ep_reward)
|
||||
print('Complete training！')
|
||||
return rewards,ma_rewards
|
||||
|
||||
def eval(cfg,env,agent):
|
||||
print('Start to eval !')
|
||||
print(f'Env:{cfg.env}, Algorithm:{cfg.algo}, Device:{cfg.device}')
|
||||
rewards = []
|
||||
ma_rewards = []
|
||||
for i_ep in range(cfg.eval_eps):
|
||||
state = env.reset()
|
||||
ep_reward = 0
|
||||
while True:
|
||||
action = agent.predict(state)
|
||||
next_state, reward, done, _ = env.step(action)
|
||||
state = next_state
|
||||
ep_reward += reward
|
||||
if done:
|
||||
break
|
||||
rewards.append(ep_reward)
|
||||
if ma_rewards:
|
||||
ma_rewards.append(ma_rewards[-1]*0.9+ep_reward*0.1)
|
||||
else:
|
||||
ma_rewards.append(ep_reward)
|
||||
print(f"Episode:{i_ep+1}/{cfg.eval_eps}, reward:{ep_reward:.1f}")
|
||||
print('Complete evaling！')
|
||||
return rewards,ma_rewards
|
||||
|
||||
if __name__ == "__main__":
|
||||
cfg = DoubleDQNConfig()
|
||||
# 训练
|
||||
env,agent = env_agent_config(cfg,seed=1)
|
||||
rewards, ma_rewards = train(cfg, env, agent)
|
||||
make_dir(cfg.result_path, cfg.model_path)
|
||||
agent.save(path=cfg.model_path)
|
||||
save_results(rewards, ma_rewards, tag='train', path=cfg.result_path)
|
||||
plot_rewards(rewards, ma_rewards, tag="train",
|
||||
algo=cfg.algo, path=cfg.result_path)
|
||||
|
||||
# 测试
|
||||
env,agent = env_agent_config(cfg,seed=10)
|
||||
agent.load(path=cfg.model_path)
|
||||
rewards,ma_rewards = eval(cfg,env,agent)
|
||||
save_results(rewards,ma_rewards,tag='eval',path=cfg.result_path)
|
||||
plot_rewards(rewards,ma_rewards,tag="eval",env=cfg.env,algo = cfg.algo,path=cfg.result_path)
|
||||
@@ -136,12 +136,12 @@
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"class DuelingNet(nn.Module):\n",
|
||||
" def __init__(self, n_states, n_actions,hidden_size=128):\n",
|
||||
" def __init__(self, state_dim, action_dim,hidden_size=128):\n",
|
||||
" super(DuelingNet, self).__init__()\n",
|
||||
" \n",
|
||||
" # 隐藏层\n",
|
||||
" self.hidden = nn.Sequential(\n",
|
||||
" nn.Linear(n_states, hidden_size),\n",
|
||||
" nn.Linear(state_dim, hidden_size),\n",
|
||||
" nn.ReLU()\n",
|
||||
" )\n",
|
||||
" \n",
|
||||
@@ -149,7 +149,7 @@
|
||||
" self.advantage = nn.Sequential(\n",
|
||||
" nn.Linear(hidden_size, hidden_size),\n",
|
||||
" nn.ReLU(),\n",
|
||||
" nn.Linear(hidden_size, n_actions)\n",
|
||||
" nn.Linear(hidden_size, action_dim)\n",
|
||||
" )\n",
|
||||
" \n",
|
||||
" # 价值函数\n",
|
||||
@@ -192,7 +192,7 @@
|
||||
],
|
||||
"source": [
|
||||
"class DuelingDQN:\n",
|
||||
" def __init__(self,n_states,n_actions,cfg) -> None:\n",
|
||||
" def __init__(self,state_dim,action_dim,cfg) -> None:\n",
|
||||
" self.batch_size = cfg.batch_size\n",
|
||||
" self.device = cfg.device\n",
|
||||
" self.loss_history = [] # 记录loss的变化\n",
|
||||
@@ -200,8 +200,8 @@
|
||||
" self.epsilon = lambda frame_idx: cfg.epsilon_end + \\\n",
|
||||
" (cfg.epsilon_start - cfg.epsilon_end) * \\\n",
|
||||
" math.exp(-1. * frame_idx / cfg.epsilon_decay)\n",
|
||||
" self.policy_net = DuelingNet(n_states, n_actions,hidden_dim=cfg.hidden_dim).to(self.device)\n",
|
||||
" self.target_net = DuelingNet(n_states, n_actions,hidden_dim=cfg.hidden_dim).to(self.device)\n",
|
||||
" self.policy_net = DuelingNet(state_dim, action_dim,hidden_dim=cfg.hidden_dim).to(self.device)\n",
|
||||
" self.target_net = DuelingNet(state_dim, action_dim,hidden_dim=cfg.hidden_dim).to(self.device)\n",
|
||||
" for target_param, param in zip(self.target_net.parameters(),self.policy_net.parameters()): # 复制参数到目标网络targe_net\n",
|
||||
" target_param.data.copy_(param.data)\n",
|
||||
" self.optimizer = optim.Adam(self.policy_net.parameters(), lr=cfg.lr) # 优化器\n",
|
||||
|
||||
@@ -11,11 +11,50 @@ Environment:
|
||||
'''
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.optim as optim
|
||||
import torch.nn.functional as F
|
||||
import numpy as np
|
||||
import random,math
|
||||
import torch.optim as optim
|
||||
from common.model import MLP
|
||||
from common.memory import ReplayBuffer
|
||||
|
||||
class ReplayBuffer:
|
||||
def __init__(self, capacity):
|
||||
self.capacity = capacity # 经验回放的容量
|
||||
self.buffer = [] # 缓冲区
|
||||
self.position = 0
|
||||
|
||||
def push(self, state, action, reward, next_state, done):
|
||||
''' 缓冲区是一个队列，容量超出时去掉开始存入的转移(transition)
|
||||
'''
|
||||
if len(self.buffer) < self.capacity:
|
||||
self.buffer.append(None)
|
||||
self.buffer[self.position] = (state, action, reward, next_state, done)
|
||||
self.position = (self.position + 1) % self.capacity
|
||||
|
||||
def sample(self, batch_size):
|
||||
batch = random.sample(self.buffer, batch_size) # 随机采出小批量转移
|
||||
state, action, reward, next_state, done = zip(*batch) # 解压成状态，动作等
|
||||
return state, action, reward, next_state, done
|
||||
|
||||
def __len__(self):
|
||||
''' 返回当前存储的量
|
||||
'''
|
||||
return len(self.buffer)
|
||||
class MLP(nn.Module):
|
||||
def __init__(self, input_dim,output_dim,hidden_dim=128):
|
||||
""" 初始化q网络，为全连接网络
|
||||
input_dim: 输入的特征数即环境的状态维度
|
||||
output_dim: 输出的动作维度
|
||||
"""
|
||||
super(MLP, self).__init__()
|
||||
self.fc1 = nn.Linear(input_dim, hidden_dim) # 输入层
|
||||
self.fc2 = nn.Linear(hidden_dim,hidden_dim) # 隐藏层
|
||||
self.fc3 = nn.Linear(hidden_dim, output_dim) # 输出层
|
||||
|
||||
def forward(self, x):
|
||||
# 各层对应的激活函数
|
||||
x = F.relu(self.fc1(x))
|
||||
x = F.relu(self.fc2(x))
|
||||
return self.fc3(x)
|
||||
|
||||
class HierarchicalDQN:
|
||||
def __init__(self,state_dim,action_dim,cfg):
|
||||
@@ -24,7 +63,7 @@ class HierarchicalDQN:
|
||||
self.gamma = cfg.gamma
|
||||
self.device = cfg.device
|
||||
self.batch_size = cfg.batch_size
|
||||
self.frame_idx = 0
|
||||
self.frame_idx = 0 # 用于epsilon的衰减计数
|
||||
self.epsilon = lambda frame_idx: cfg.epsilon_end + (cfg.epsilon_start - cfg.epsilon_end ) * math.exp(-1. * frame_idx / cfg.epsilon_decay)
|
||||
self.policy_net = MLP(2*state_dim, action_dim,cfg.hidden_dim).to(self.device)
|
||||
self.meta_policy_net = MLP(state_dim, state_dim,cfg.hidden_dim).to(self.device)
|
||||
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
|
After Width: | Height: | Size: 62 KiB |
Binary file not shown.
Binary file not shown.
Binary file not shown.
|
After Width: | Height: | Size: 77 KiB |
Binary file not shown.
Binary file not shown.
|
Before Width: | Height: | Size: 73 KiB |
Binary file not shown.
Binary file not shown.
|
Before Width: | Height: | Size: 21 KiB |
Binary file not shown.
Binary file not shown.
|
Before Width: | Height: | Size: 62 KiB |
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
88
codes/HierarchicalDQN/task0.py
Normal file
88
codes/HierarchicalDQN/task0.py
Normal file
@@ -0,0 +1,88 @@
|
||||
#!/usr/bin/env python
|
||||
# coding=utf-8
|
||||
'''
|
||||
Author: John
|
||||
Email: johnjim0816@gmail.com
|
||||
Date: 2021-03-29 10:37:32
|
||||
LastEditor: John
|
||||
LastEditTime: 2021-05-04 22:35:56
|
||||
Discription:
|
||||
Environment:
|
||||
'''
|
||||
import sys
|
||||
import os
|
||||
curr_path = os.path.dirname(os.path.abspath(__file__)) # 当前文件所在绝对路径
|
||||
parent_path = os.path.dirname(curr_path) # 父路径
|
||||
sys.path.append(parent_path) # 添加路径到系统路径
|
||||
|
||||
import datetime
|
||||
import numpy as np
|
||||
import torch
|
||||
import gym
|
||||
|
||||
from common.utils import save_results,make_dir
|
||||
from common.utils import plot_rewards
|
||||
from HierarchicalDQN.agent import HierarchicalDQN
|
||||
from HierarchicalDQN.train import train,test
|
||||
|
||||
curr_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") # 获取当前时间
|
||||
algo_name = "Hierarchical DQN" # 算法名称
|
||||
env_name = 'CartPole-v0' # 环境名称
|
||||
class HierarchicalDQNConfig:
|
||||
def __init__(self):
|
||||
self.algo_name = algo_name # 算法名称
|
||||
self.env_name = env_name # 环境名称
|
||||
self.device = torch.device(
|
||||
"cuda" if torch.cuda.is_available() else "cpu") # 检测GPU
|
||||
self.train_eps = 300 # 训练的episode数目
|
||||
self.test_eps = 50 # 测试的episode数目
|
||||
self.gamma = 0.99
|
||||
self.epsilon_start = 1 # start epsilon of e-greedy policy
|
||||
self.epsilon_end = 0.01
|
||||
self.epsilon_decay = 200
|
||||
self.lr = 0.0001 # learning rate
|
||||
self.memory_capacity = 10000 # Replay Memory capacity
|
||||
self.batch_size = 32
|
||||
self.target_update = 2 # 目标网络的更新频率
|
||||
self.hidden_dim = 256 # 网络隐藏层
|
||||
class PlotConfig:
|
||||
''' 绘图相关参数设置
|
||||
'''
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.algo_name = algo_name # 算法名称
|
||||
self.env_name = env_name # 环境名称
|
||||
self.device = torch.device(
|
||||
"cuda" if torch.cuda.is_available() else "cpu") # 检测GPU
|
||||
self.result_path = curr_path + "/outputs/" + self.env_name + \
|
||||
'/' + curr_time + '/results/' # 保存结果的路径
|
||||
self.model_path = curr_path + "/outputs/" + self.env_name + \
|
||||
'/' + curr_time + '/models/' # 保存模型的路径
|
||||
self.save = True # 是否保存图片
|
||||
|
||||
def env_agent_config(cfg,seed=1):
|
||||
env = gym.make(cfg.env_name)
|
||||
env.seed(seed)
|
||||
state_dim = env.observation_space.shape[0]
|
||||
action_dim = env.action_space.n
|
||||
agent = HierarchicalDQN(state_dim,action_dim,cfg)
|
||||
return env,agent
|
||||
|
||||
if __name__ == "__main__":
|
||||
cfg = HierarchicalDQNConfig()
|
||||
plot_cfg = PlotConfig()
|
||||
# 训练
|
||||
env, agent = env_agent_config(cfg, seed=1)
|
||||
rewards, ma_rewards = train(cfg, env, agent)
|
||||
make_dir(plot_cfg.result_path, plot_cfg.model_path) # 创建保存结果和模型路径的文件夹
|
||||
agent.save(path=plot_cfg.model_path) # 保存模型
|
||||
save_results(rewards, ma_rewards, tag='train',
|
||||
path=plot_cfg.result_path) # 保存结果
|
||||
plot_rewards(rewards, ma_rewards, plot_cfg, tag="train") # 画出结果
|
||||
# 测试
|
||||
env, agent = env_agent_config(cfg, seed=10)
|
||||
agent.load(path=plot_cfg.model_path) # 导入模型
|
||||
rewards, ma_rewards = test(cfg, env, agent)
|
||||
save_results(rewards, ma_rewards, tag='test', path=plot_cfg.result_path) # 保存结果
|
||||
plot_rewards(rewards, ma_rewards, plot_cfg, tag="test") # 画出结果
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -1,146 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
# coding=utf-8
|
||||
'''
|
||||
Author: John
|
||||
Email: johnjim0816@gmail.com
|
||||
Date: 2021-03-29 10:37:32
|
||||
LastEditor: John
|
||||
LastEditTime: 2021-05-04 22:35:56
|
||||
Discription:
|
||||
Environment:
|
||||
'''
|
||||
|
||||
|
||||
import sys,os
|
||||
curr_path = os.path.dirname(__file__)
|
||||
parent_path = os.path.dirname(curr_path)
|
||||
sys.path.append(parent_path) # add current terminal path to sys.path
|
||||
|
||||
import datetime
|
||||
import numpy as np
|
||||
import torch
|
||||
import gym
|
||||
|
||||
from common.utils import save_results,make_dir
|
||||
from common.plot import plot_rewards
|
||||
from HierarchicalDQN.agent import HierarchicalDQN
|
||||
|
||||
curr_time = datetime.datetime.now().strftime(
|
||||
"%Y%m%d-%H%M%S") # obtain current time
|
||||
|
||||
class HierarchicalDQNConfig:
|
||||
def __init__(self):
|
||||
self.algo = "H-DQN" # name of algo
|
||||
self.env = 'CartPole-v0'
|
||||
self.result_path = curr_path+"/outputs/" + self.env + \
|
||||
'/'+curr_time+'/results/' # path to save results
|
||||
self.model_path = curr_path+"/outputs/" + self.env + \
|
||||
'/'+curr_time+'/models/' # path to save models
|
||||
self.train_eps = 300 # 训练的episode数目
|
||||
self.eval_eps = 50 # 测试的episode数目
|
||||
self.gamma = 0.99
|
||||
self.epsilon_start = 1 # start epsilon of e-greedy policy
|
||||
self.epsilon_end = 0.01
|
||||
self.epsilon_decay = 200
|
||||
self.lr = 0.0001 # learning rate
|
||||
self.memory_capacity = 10000 # Replay Memory capacity
|
||||
self.batch_size = 32
|
||||
self.target_update = 2 # target net的更新频率
|
||||
self.device = torch.device(
|
||||
"cuda" if torch.cuda.is_available() else "cpu") # 检测gpu
|
||||
self.hidden_dim = 256 # dimension of hidden layer
|
||||
|
||||
def env_agent_config(cfg,seed=1):
|
||||
env = gym.make(cfg.env)
|
||||
env.seed(seed)
|
||||
state_dim = env.observation_space.shape[0]
|
||||
action_dim = env.action_space.n
|
||||
agent = HierarchicalDQN(state_dim,action_dim,cfg)
|
||||
return env,agent
|
||||
|
||||
def train(cfg, env, agent):
|
||||
print('Start to train !')
|
||||
print(f'Env:{cfg.env}, Algorithm:{cfg.algo}, Device:{cfg.device}')
|
||||
rewards = []
|
||||
ma_rewards = [] # moveing average reward
|
||||
for i_ep in range(cfg.train_eps):
|
||||
state = env.reset()
|
||||
done = False
|
||||
ep_reward = 0
|
||||
while not done:
|
||||
goal = agent.set_goal(state)
|
||||
onehot_goal = agent.to_onehot(goal)
|
||||
meta_state = state
|
||||
extrinsic_reward = 0
|
||||
while not done and goal != np.argmax(state):
|
||||
goal_state = np.concatenate([state, onehot_goal])
|
||||
action = agent.choose_action(goal_state)
|
||||
next_state, reward, done, _ = env.step(action)
|
||||
ep_reward += reward
|
||||
extrinsic_reward += reward
|
||||
intrinsic_reward = 1.0 if goal == np.argmax(
|
||||
next_state) else 0.0
|
||||
agent.memory.push(goal_state, action, intrinsic_reward, np.concatenate(
|
||||
[next_state, onehot_goal]), done)
|
||||
state = next_state
|
||||
agent.update()
|
||||
agent.meta_memory.push(meta_state, goal, extrinsic_reward, state, done)
|
||||
print('Episode:{}/{}, Reward:{}, Loss:{:.2f}, Meta_Loss:{:.2f}'.format(i_ep+1, cfg.train_eps, ep_reward,agent.loss_numpy ,agent.meta_loss_numpy ))
|
||||
rewards.append(ep_reward)
|
||||
if ma_rewards:
|
||||
ma_rewards.append(
|
||||
0.9*ma_rewards[-1]+0.1*ep_reward)
|
||||
else:
|
||||
ma_rewards.append(ep_reward)
|
||||
print('Complete training！')
|
||||
return rewards, ma_rewards
|
||||
|
||||
def eval(cfg, env, agent):
|
||||
print('Start to eval !')
|
||||
print(f'Env:{cfg.env}, Algorithm:{cfg.algo}, Device:{cfg.device}')
|
||||
rewards = []
|
||||
ma_rewards = [] # moveing average reward
|
||||
for i_ep in range(cfg.train_eps):
|
||||
state = env.reset()
|
||||
done = False
|
||||
ep_reward = 0
|
||||
while not done:
|
||||
goal = agent.set_goal(state)
|
||||
onehot_goal = agent.to_onehot(goal)
|
||||
extrinsic_reward = 0
|
||||
while not done and goal != np.argmax(state):
|
||||
goal_state = np.concatenate([state, onehot_goal])
|
||||
action = agent.choose_action(goal_state)
|
||||
next_state, reward, done, _ = env.step(action)
|
||||
ep_reward += reward
|
||||
extrinsic_reward += reward
|
||||
state = next_state
|
||||
agent.update()
|
||||
print(f'Episode:{i_ep+1}/{cfg.train_eps}, Reward:{ep_reward}, Loss:{agent.loss_numpy:.2f}, Meta_Loss:{agent.meta_loss_numpy:.2f}')
|
||||
rewards.append(ep_reward)
|
||||
if ma_rewards:
|
||||
ma_rewards.append(
|
||||
0.9*ma_rewards[-1]+0.1*ep_reward)
|
||||
else:
|
||||
ma_rewards.append(ep_reward)
|
||||
print('Complete training！')
|
||||
return rewards, ma_rewards
|
||||
|
||||
if __name__ == "__main__":
|
||||
cfg = HierarchicalDQNConfig()
|
||||
|
||||
# train
|
||||
env,agent = env_agent_config(cfg,seed=1)
|
||||
rewards, ma_rewards = train(cfg, env, agent)
|
||||
make_dir(cfg.result_path, cfg.model_path)
|
||||
agent.save(path=cfg.model_path)
|
||||
save_results(rewards, ma_rewards, tag='train', path=cfg.result_path)
|
||||
plot_rewards(rewards, ma_rewards, tag="train",
|
||||
algo=cfg.algo, path=cfg.result_path)
|
||||
# eval
|
||||
env,agent = env_agent_config(cfg,seed=10)
|
||||
agent.load(path=cfg.model_path)
|
||||
rewards,ma_rewards = eval(cfg,env,agent)
|
||||
save_results(rewards,ma_rewards,tag='eval',path=cfg.result_path)
|
||||
plot_rewards(rewards,ma_rewards,tag="eval",env=cfg.env,algo = cfg.algo,path=cfg.result_path)
|
||||
|
||||
77
codes/HierarchicalDQN/train.py
Normal file
77
codes/HierarchicalDQN/train.py
Normal file
@@ -0,0 +1,77 @@
|
||||
import sys
|
||||
import os
|
||||
curr_path = os.path.dirname(os.path.abspath(__file__)) # 当前文件所在绝对路径
|
||||
parent_path = os.path.dirname(curr_path) # 父路径
|
||||
sys.path.append(parent_path) # 添加路径到系统路径
|
||||
|
||||
import numpy as np
|
||||
|
||||
def train(cfg, env, agent):
|
||||
print('开始训练!')
|
||||
print(f'环境：{cfg.env_name}, 算法：{cfg.algo_name}, 设备：{cfg.device}')
|
||||
rewards = [] # 记录所有回合的奖励
|
||||
ma_rewards = [] # 记录所有回合的滑动平均奖励
|
||||
for i_ep in range(cfg.train_eps):
|
||||
state = env.reset()
|
||||
done = False
|
||||
ep_reward = 0
|
||||
while not done:
|
||||
goal = agent.set_goal(state)
|
||||
onehot_goal = agent.to_onehot(goal)
|
||||
meta_state = state
|
||||
extrinsic_reward = 0
|
||||
while not done and goal != np.argmax(state):
|
||||
goal_state = np.concatenate([state, onehot_goal])
|
||||
action = agent.choose_action(goal_state)
|
||||
next_state, reward, done, _ = env.step(action)
|
||||
ep_reward += reward
|
||||
extrinsic_reward += reward
|
||||
intrinsic_reward = 1.0 if goal == np.argmax(
|
||||
next_state) else 0.0
|
||||
agent.memory.push(goal_state, action, intrinsic_reward, np.concatenate(
|
||||
[next_state, onehot_goal]), done)
|
||||
state = next_state
|
||||
agent.update()
|
||||
if (i_ep+1)%10 == 0:
|
||||
print(f'回合：{i_ep+1}/{cfg.train_eps}，奖励：{ep_reward}，Loss:{agent.loss_numpy:.2f}， Meta_Loss:{agent.meta_loss_numpy:.2f}')
|
||||
agent.meta_memory.push(meta_state, goal, extrinsic_reward, state, done)
|
||||
rewards.append(ep_reward)
|
||||
if ma_rewards:
|
||||
ma_rewards.append(
|
||||
0.9*ma_rewards[-1]+0.1*ep_reward)
|
||||
else:
|
||||
ma_rewards.append(ep_reward)
|
||||
print('完成训练！')
|
||||
return rewards, ma_rewards
|
||||
|
||||
def test(cfg, env, agent):
|
||||
print('开始测试!')
|
||||
print(f'环境：{cfg.env_name}, 算法：{cfg.algo_name}, 设备：{cfg.device}')
|
||||
rewards = [] # 记录所有回合的奖励
|
||||
ma_rewards = [] # 记录所有回合的滑动平均奖励
|
||||
for i_ep in range(cfg.train_eps):
|
||||
state = env.reset()
|
||||
done = False
|
||||
ep_reward = 0
|
||||
while not done:
|
||||
goal = agent.set_goal(state)
|
||||
onehot_goal = agent.to_onehot(goal)
|
||||
extrinsic_reward = 0
|
||||
while not done and goal != np.argmax(state):
|
||||
goal_state = np.concatenate([state, onehot_goal])
|
||||
action = agent.choose_action(goal_state)
|
||||
next_state, reward, done, _ = env.step(action)
|
||||
ep_reward += reward
|
||||
extrinsic_reward += reward
|
||||
state = next_state
|
||||
agent.update()
|
||||
if (i_ep+1)%10 == 0:
|
||||
print(f'回合：{i_ep+1}/{cfg.train_eps}，奖励：{ep_reward}，Loss:{agent.loss_numpy:.2f}， Meta_Loss:{agent.meta_loss_numpy:.2f}')
|
||||
rewards.append(ep_reward)
|
||||
if ma_rewards:
|
||||
ma_rewards.append(
|
||||
0.9*ma_rewards[-1]+0.1*ep_reward)
|
||||
else:
|
||||
ma_rewards.append(ep_reward)
|
||||
print('完成训练！')
|
||||
return rewards, ma_rewards
|
||||
7
codes/Logs.md
Normal file
7
codes/Logs.md
Normal file
@@ -0,0 +1,7 @@
|
||||
## 记录笔者更新的日志
|
||||
|
||||
**2021.12.28-1**：将```task.py```中的两个Config类合并为一个，并加以注释便于阅读，从DQN算法开始更新
|
||||
|
||||
**2021.12.22-3**：将```agent.py```更改为对应的算法名称，便于区分如```dqn```与```dqn_cnn```的情况
|
||||
**2021.12.22-2**：简化了代码结构，将原来的```train.py```和```task.py```等合并到```task.py```中
|
||||
**2021.12.22-1**：简化了代码结构，将原来的```model.py```和```memory.py```等合并到```agent.py```中，```plot.py```的内容合并到```common.utils.py```中
|
||||
52
codes/NoisyDQN/noisy_dqn.py
Normal file
52
codes/NoisyDQN/noisy_dqn.py
Normal file
@@ -0,0 +1,52 @@
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
class NoisyLinear(nn.Module):
|
||||
def __init__(self, input_dim, output_dim, std_init=0.4):
|
||||
super(NoisyLinear, self).__init__()
|
||||
|
||||
self.input_dim = input_dim
|
||||
self.output_dim = output_dim
|
||||
self.std_init = std_init
|
||||
|
||||
self.weight_mu = nn.Parameter(torch.FloatTensor(output_dim, input_dim))
|
||||
self.weight_sigma = nn.Parameter(torch.FloatTensor(output_dim, input_dim))
|
||||
self.register_buffer('weight_epsilon', torch.FloatTensor(output_dim, input_dim))
|
||||
|
||||
self.bias_mu = nn.Parameter(torch.FloatTensor(output_dim))
|
||||
self.bias_sigma = nn.Parameter(torch.FloatTensor(output_dim))
|
||||
self.register_buffer('bias_epsilon', torch.FloatTensor(output_dim))
|
||||
|
||||
self.reset_parameters()
|
||||
self.reset_noise()
|
||||
|
||||
def forward(self, x):
|
||||
if self.training:
|
||||
weight = self.weight_mu + self.weight_sigma.mul( (self.weight_epsilon))
|
||||
bias = self.bias_mu + self.bias_sigma.mul(Variable(self.bias_epsilon))
|
||||
else:
|
||||
weight = self.weight_mu
|
||||
bias = self.bias_mu
|
||||
|
||||
return F.linear(x, weight, bias)
|
||||
|
||||
def reset_parameters(self):
|
||||
mu_range = 1 / math.sqrt(self.weight_mu.size(1))
|
||||
|
||||
self.weight_mu.data.uniform_(-mu_range, mu_range)
|
||||
self.weight_sigma.data.fill_(self.std_init / math.sqrt(self.weight_sigma.size(1)))
|
||||
|
||||
self.bias_mu.data.uniform_(-mu_range, mu_range)
|
||||
self.bias_sigma.data.fill_(self.std_init / math.sqrt(self.bias_sigma.size(0)))
|
||||
|
||||
def reset_noise(self):
|
||||
epsilon_in = self._scale_noise(self.input_dim)
|
||||
epsilon_out = self._scale_noise(self.output_dim)
|
||||
|
||||
self.weight_epsilon.copy_(epsilon_out.ger(epsilon_in))
|
||||
self.bias_epsilon.copy_(self._scale_noise(self.output_dim))
|
||||
|
||||
def _scale_noise(self, size):
|
||||
x = torch.randn(size)
|
||||
x = x.sign().mul(x.abs().sqrt())
|
||||
return x
|
||||
@@ -20,7 +20,7 @@ class PPOConfig:
|
||||
self.continuous = False # 环境是否为连续动作
|
||||
self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # 检测GPU
|
||||
self.train_eps = 200 # 训练的回合数
|
||||
self.eval_eps = 20 # 测试的回合数
|
||||
self.test_eps = 20 # 测试的回合数
|
||||
self.batch_size = 5
|
||||
self.gamma=0.99
|
||||
self.n_epochs = 4
|
||||
|
||||
@@ -20,7 +20,7 @@ class PPOConfig:
|
||||
self.continuous = True # 环境是否为连续动作
|
||||
self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # 检测GPU
|
||||
self.train_eps = 200 # 训练的回合数
|
||||
self.eval_eps = 20 # 测试的回合数
|
||||
self.test_eps = 20 # 测试的回合数
|
||||
self.batch_size = 5
|
||||
self.gamma=0.99
|
||||
self.n_epochs = 4
|
||||
|
||||
@@ -68,7 +68,7 @@
|
||||
" self.result_path = curr_path+\"/results/\" +self.env+'/'+curr_time+'/results/' # path to save results\n",
|
||||
" self.model_path = curr_path+\"/results/\" +self.env+'/'+curr_time+'/models/' # path to save models\n",
|
||||
" self.train_eps = 200 # max training episodes\n",
|
||||
" self.eval_eps = 50\n",
|
||||
" self.test_eps = 50\n",
|
||||
" self.batch_size = 5\n",
|
||||
" self.gamma=0.99\n",
|
||||
" self.n_epochs = 4\n",
|
||||
@@ -144,7 +144,7 @@
|
||||
" print(f'Env:{cfg.env}, Algorithm:{cfg.algo}, Device:{cfg.device}')\n",
|
||||
" rewards= []\n",
|
||||
" ma_rewards = [] # moving average rewards\n",
|
||||
" for i_ep in range(cfg.eval_eps):\n",
|
||||
" for i_ep in range(cfg.test_eps):\n",
|
||||
" state = env.reset()\n",
|
||||
" done = False\n",
|
||||
" ep_reward = 0\n",
|
||||
|
||||
@@ -32,7 +32,7 @@ def eval(cfg,env,agent):
|
||||
print(f'环境：{cfg.env_name}, 算法：{cfg.algo}, 设备：{cfg.device}')
|
||||
rewards = [] # 记录所有回合的奖励
|
||||
ma_rewards = [] # 记录所有回合的滑动平均奖励
|
||||
for i_ep in range(cfg.eval_eps):
|
||||
for i_ep in range(cfg.test_eps):
|
||||
state = env.reset()
|
||||
done = False
|
||||
ep_reward = 0
|
||||
@@ -47,7 +47,7 @@ def eval(cfg,env,agent):
|
||||
0.9*ma_rewards[-1]+0.1*ep_reward)
|
||||
else:
|
||||
ma_rewards.append(ep_reward)
|
||||
print('回合：{}/{}, 奖励：{}'.format(i_ep+1, cfg.eval_eps, ep_reward))
|
||||
print('回合：{}/{}, 奖励：{}'.format(i_ep+1, cfg.test_eps, ep_reward))
|
||||
print('完成训练！')
|
||||
return rewards,ma_rewards
|
||||
|
||||
@@ -74,7 +74,7 @@ if __name__ == '__main__':
|
||||
self.continuous = False # 环境是否为连续动作
|
||||
self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # 检测GPU
|
||||
self.train_eps = 200 # 训练的回合数
|
||||
self.eval_eps = 20 # 测试的回合数
|
||||
self.test_eps = 20 # 测试的回合数
|
||||
self.batch_size = 5
|
||||
self.gamma=0.99
|
||||
self.n_epochs = 4
|
||||
|
||||
@@ -5,21 +5,22 @@ Author: John
|
||||
Email: johnjim0816@gmail.com
|
||||
Date: 2021-03-23 16:35:58
|
||||
LastEditor: John
|
||||
LastEditTime: 2021-03-23 16:36:20
|
||||
LastEditTime: 2021-12-21 23:21:26
|
||||
Discription:
|
||||
Environment:
|
||||
'''
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
class MLP(nn.Module):
|
||||
|
||||
''' 多层感知机
|
||||
输入：state维度
|
||||
输出：概率
|
||||
'''
|
||||
def __init__(self,state_dim,hidden_dim = 36):
|
||||
def __init__(self,input_dim,hidden_dim = 36):
|
||||
super(MLP, self).__init__()
|
||||
# 24和36为hidden layer的层数，可根据state_dim, action_dim的情况来改变
|
||||
self.fc1 = nn.Linear(state_dim, hidden_dim)
|
||||
# 24和36为hidden layer的层数，可根据input_dim, action_dim的情况来改变
|
||||
self.fc1 = nn.Linear(input_dim, hidden_dim)
|
||||
self.fc2 = nn.Linear(hidden_dim,hidden_dim)
|
||||
self.fc3 = nn.Linear(hidden_dim, 1) # Prob of Left
|
||||
|
||||
|
||||
@@ -34,7 +34,7 @@ class PGConfig:
|
||||
self.model_path = curr_path+"/outputs/" + self.env + \
|
||||
'/'+curr_time+'/models/' # 保存模型的路径
|
||||
self.train_eps = 300 # 训练的回合数
|
||||
self.eval_eps = 30 # 测试的回合数
|
||||
self.test_eps = 30 # 测试的回合数
|
||||
self.batch_size = 8
|
||||
self.lr = 0.01 # 学习率
|
||||
self.gamma = 0.99
|
||||
@@ -94,7 +94,7 @@ def eval(cfg,env,agent):
|
||||
print(f'Env:{cfg.env}, Algorithm:{cfg.algo}, Device:{cfg.device}')
|
||||
rewards = []
|
||||
ma_rewards = []
|
||||
for i_ep in range(cfg.eval_eps):
|
||||
for i_ep in range(cfg.test_eps):
|
||||
state = env.reset()
|
||||
ep_reward = 0
|
||||
for _ in count():
|
||||
|
||||
@@ -5,7 +5,7 @@ Author: John
|
||||
Email: johnjim0816@gmail.com
|
||||
Date: 2020-09-11 23:03:00
|
||||
LastEditor: John
|
||||
LastEditTime: 2021-09-19 23:05:45
|
||||
LastEditTime: 2021-12-22 10:54:57
|
||||
Discription: use defaultdict to define Q table
|
||||
Environment:
|
||||
'''
|
||||
@@ -17,15 +17,15 @@ from collections import defaultdict
|
||||
class QLearning(object):
|
||||
def __init__(self,state_dim,
|
||||
action_dim,cfg):
|
||||
self.action_dim = action_dim # dimension of acgtion
|
||||
self.lr = cfg.lr # learning rate
|
||||
self.action_dim = action_dim
|
||||
self.lr = cfg.lr # 学习率
|
||||
self.gamma = cfg.gamma
|
||||
self.epsilon = 0
|
||||
self.sample_count = 0
|
||||
self.epsilon_start = cfg.epsilon_start
|
||||
self.epsilon_end = cfg.epsilon_end
|
||||
self.epsilon_decay = cfg.epsilon_decay
|
||||
self.Q_table = defaultdict(lambda: np.zeros(action_dim)) # A nested dictionary that maps state -> (action -> action-value)
|
||||
self.Q_table = defaultdict(lambda: np.zeros(action_dim)) # 用嵌套字典存放状态->动作->状态-动作值（Q值）的映射，即Q表
|
||||
def choose_action(self, state):
|
||||
self.sample_count += 1
|
||||
self.epsilon = self.epsilon_end + (self.epsilon_start - self.epsilon_end) * \
|
||||
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
|
Before Width: | Height: | Size: 49 KiB |
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
|
Before Width: | Height: | Size: 48 KiB |
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
|
Before Width: | Height: | Size: 33 KiB |
Binary file not shown.
Binary file not shown.
|
Before Width: | Height: | Size: 31 KiB |
Binary file not shown.
Binary file not shown.
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user