bacow/easy-rl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update codes

Browse Source
This commit is contained in:
johnjim0816
2021-12-28 18:46:52 +08:00
parent 41fb561d25
commit bd51b5a7ad
52 changed files with 305 additions and 292 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
codes/A2C/agent.py
View File

@@ -40,10 +40,10 @@ class ActorCritic(nn.Module):
class A2C: class A2C:
''' A2C算法 ''' A2C算法
''' '''
def __init__(self,n_states,n_actions,cfg) -> None: def __init__(self,state_dim,action_dim,cfg) -> None:
self.gamma = cfg.gamma self.gamma = cfg.gamma
self.device = cfg.device self.device = cfg.device
self.model = ActorCritic(n_states, n_actions, cfg.hidden_size).to(self.device) self.model = ActorCritic(state_dim, action_dim, cfg.hidden_size).to(self.device)
self.optimizer = optim.Adam(self.model.parameters()) self.optimizer = optim.Adam(self.model.parameters())
def compute_returns(self,next_value, rewards, masks): def compute_returns(self,next_value, rewards, masks):

6
codes/A2C/task0.py
View File

@@ -74,9 +74,9 @@ def train(cfg,envs):
print(f'环境：{cfg.env_name}, 算法：{cfg.algo}, 设备：{cfg.device}') print(f'环境：{cfg.env_name}, 算法：{cfg.algo}, 设备：{cfg.device}')
env = gym.make(cfg.env_name) # a single env env = gym.make(cfg.env_name) # a single env
env.seed(10) env.seed(10)
n_states = envs.observation_space.shape[0] state_dim = envs.observation_space.shape[0]
n_actions = envs.action_space.n action_dim = envs.action_space.n
model = ActorCritic(n_states, n_actions, cfg.hidden_dim).to(cfg.device) model = ActorCritic(state_dim, action_dim, cfg.hidden_dim).to(cfg.device)
optimizer = optim.Adam(model.parameters()) optimizer = optim.Adam(model.parameters())
frame_idx = 0 frame_idx = 0
test_rewards = [] test_rewards = []

20
codes/DDPG/agent.py
View File

@@ -39,11 +39,11 @@ class ReplayBuffer:
''' '''
return len(self.buffer) return len(self.buffer)
class Actor(nn.Module): class Actor(nn.Module):
def __init__(self, n_states, n_actions, hidden_dim, init_w=3e-3): def __init__(self, state_dim, action_dim, hidden_dim, init_w=3e-3):
super(Actor, self).__init__() super(Actor, self).__init__()
self.linear1 = nn.Linear(n_states, hidden_dim) self.linear1 = nn.Linear(state_dim, hidden_dim)
self.linear2 = nn.Linear(hidden_dim, hidden_dim) self.linear2 = nn.Linear(hidden_dim, hidden_dim)
self.linear3 = nn.Linear(hidden_dim, n_actions) self.linear3 = nn.Linear(hidden_dim, action_dim)
self.linear3.weight.data.uniform_(-init_w, init_w) self.linear3.weight.data.uniform_(-init_w, init_w)
self.linear3.bias.data.uniform_(-init_w, init_w) self.linear3.bias.data.uniform_(-init_w, init_w)
@@ -54,10 +54,10 @@ class Actor(nn.Module):
x = torch.tanh(self.linear3(x)) x = torch.tanh(self.linear3(x))
return x return x
class Critic(nn.Module): class Critic(nn.Module):
def __init__(self, n_states, n_actions, hidden_dim, init_w=3e-3): def __init__(self, state_dim, action_dim, hidden_dim, init_w=3e-3):
super(Critic, self).__init__() super(Critic, self).__init__()
self.linear1 = nn.Linear(n_states + n_actions, hidden_dim) self.linear1 = nn.Linear(state_dim + action_dim, hidden_dim)
self.linear2 = nn.Linear(hidden_dim, hidden_dim) self.linear2 = nn.Linear(hidden_dim, hidden_dim)
self.linear3 = nn.Linear(hidden_dim, 1) self.linear3 = nn.Linear(hidden_dim, 1)
# 随机初始化为较小的值 # 随机初始化为较小的值
@@ -72,12 +72,12 @@ class Critic(nn.Module):
x = self.linear3(x) x = self.linear3(x)
return x return x
class DDPG: class DDPG:
def __init__(self, n_states, n_actions, cfg): def __init__(self, state_dim, action_dim, cfg):
self.device = cfg.device self.device = cfg.device
self.critic = Critic(n_states, n_actions, cfg.hidden_dim).to(cfg.device) self.critic = Critic(state_dim, action_dim, cfg.hidden_dim).to(cfg.device)
self.actor = Actor(n_states, n_actions, cfg.hidden_dim).to(cfg.device) self.actor = Actor(state_dim, action_dim, cfg.hidden_dim).to(cfg.device)
self.target_critic = Critic(n_states, n_actions, cfg.hidden_dim).to(cfg.device) self.target_critic = Critic(state_dim, action_dim, cfg.hidden_dim).to(cfg.device)
self.target_actor = Actor(n_states, n_actions, cfg.hidden_dim).to(cfg.device) self.target_actor = Actor(state_dim, action_dim, cfg.hidden_dim).to(cfg.device)
# 复制参数到目标网络 # 复制参数到目标网络
for target_param, param in zip(self.target_critic.parameters(), self.critic.parameters()): for target_param, param in zip(self.target_critic.parameters(), self.critic.parameters()):

6
codes/DDPG/env.py
View File

@@ -39,15 +39,15 @@ class OUNoise(object):
self.max_sigma = max_sigma self.max_sigma = max_sigma
self.min_sigma = min_sigma self.min_sigma = min_sigma
self.decay_period = decay_period self.decay_period = decay_period
self.n_actions = action_space.shape[0] self.action_dim = action_space.shape[0]
self.low = action_space.low self.low = action_space.low
self.high = action_space.high self.high = action_space.high
self.reset() self.reset()
def reset(self): def reset(self):
self.obs = np.ones(self.n_actions) * self.mu self.obs = np.ones(self.action_dim) * self.mu
def evolve_obs(self): def evolve_obs(self):
x = self.obs x = self.obs
dx = self.theta * (self.mu - x) + self.sigma * np.random.randn(self.n_actions) dx = self.theta * (self.mu - x) + self.sigma * np.random.randn(self.action_dim)
self.obs = x + dx self.obs = x + dx
return self.obs return self.obs
def get_action(self, action, t=0): def get_action(self, action, t=0):

6
codes/DDPG/task0.py
View File

@@ -58,9 +58,9 @@ class PlotConfig:
def env_agent_config(cfg,seed=1): def env_agent_config(cfg,seed=1):
env = NormalizedActions(gym.make(cfg.env_name)) # 装饰action噪声 env = NormalizedActions(gym.make(cfg.env_name)) # 装饰action噪声
env.seed(seed) # 随机种子 env.seed(seed) # 随机种子
n_states = env.observation_space.shape[0] state_dim = env.observation_space.shape[0]
n_actions = env.action_space.shape[0] action_dim = env.action_space.shape[0]
agent = DDPG(n_states,n_actions,cfg) agent = DDPG(state_dim,action_dim,cfg)
return env,agent return env,agent
cfg = DDPGConfig() cfg = DDPGConfig()

18
codes/DQN/README.md
View File

@@ -50,15 +50,15 @@ import torch.nn as nn
import torch.nn.functional as F import torch.nn.functional as F
class FCN(nn.Module): class FCN(nn.Module):
def __init__(self, n_states=4, n_actions=18): def __init__(self, state_dim=4, action_dim=18):
""" 初始化q网络，为全连接网络 """ 初始化q网络，为全连接网络
n_states: 输入的feature即环境的state数目 state_dim: 输入的feature即环境的state数目
n_actions: 输出的action总个数 action_dim: 输出的action总个数
""" """
super(FCN, self).__init__() super(FCN, self).__init__()
self.fc1 = nn.Linear(n_states, 128) # 输入层 self.fc1 = nn.Linear(state_dim, 128) # 输入层
self.fc2 = nn.Linear(128, 128) # 隐藏层 self.fc2 = nn.Linear(128, 128) # 隐藏层
self.fc3 = nn.Linear(128, n_actions) # 输出层 self.fc3 = nn.Linear(128, action_dim) # 输出层
def forward(self, x): def forward(self, x):
# 各层对应的激活函数 # 各层对应的激活函数
@@ -66,7 +66,7 @@ class FCN(nn.Module):
x = F.relu(self.fc2(x)) x = F.relu(self.fc2(x))
return self.fc3(x) return self.fc3(x)
``` ```
输入为n_states，输出为n_actions，包含一个128维度的隐藏层，这里根据需要可增加隐藏层维度和数量，然后一般使用relu激活函数，这里跟深度学习的网路设置是一样的。 输入为state_dim，输出为action_dim，包含一个128维度的隐藏层，这里根据需要可增加隐藏层维度和数量，然后一般使用relu激活函数，这里跟深度学习的网路设置是一样的。
### Replay Buffer ### Replay Buffer
@@ -107,8 +107,8 @@ class ReplayBuffer:
在类中建立两个网络，以及optimizer和memory， 在类中建立两个网络，以及optimizer和memory，
```python ```python
self.policy_net = MLP(n_states, n_actions,hidden_dim=cfg.hidden_dim).to(self.device) self.policy_net = MLP(state_dim, action_dim,hidden_dim=cfg.hidden_dim).to(self.device)
self.target_net = MLP(n_states, n_actions,hidden_dim=cfg.hidden_dim).to(self.device) self.target_net = MLP(state_dim, action_dim,hidden_dim=cfg.hidden_dim).to(self.device)
for target_param, param in zip(self.target_net.parameters(),self.policy_net.parameters()): # copy params from policy net for target_param, param in zip(self.target_net.parameters(),self.policy_net.parameters()): # copy params from policy net
target_param.data.copy_(param.data) target_param.data.copy_(param.data)
self.optimizer = optim.Adam(self.policy_net.parameters(), lr=cfg.lr) self.optimizer = optim.Adam(self.policy_net.parameters(), lr=cfg.lr)
@@ -124,7 +124,7 @@ def choose_action(self, state):
if random.random() > self.epsilon(self.frame_idx): if random.random() > self.epsilon(self.frame_idx):
action = self.predict(state) action = self.predict(state)
else: else:
action = random.randrange(self.n_actions) action = random.randrange(self.action_dim)
return action return action
``` ```

20
codes/DQN/dqn.py
View File

@@ -21,15 +21,15 @@ import math
import numpy as np import numpy as np
class MLP(nn.Module): class MLP(nn.Module):
def __init__(self, n_states,n_actions,hidden_dim=128): def __init__(self, state_dim,action_dim,hidden_dim=128):
""" 初始化q网络，为全连接网络 """ 初始化q网络，为全连接网络
n_states: 输入的特征数即环境的状态数 state_dim: 输入的特征数即环境的状态维度
n_actions: 输出的动作维度 action_dim: 输出的动作维度
""" """
super(MLP, self).__init__() super(MLP, self).__init__()
self.fc1 = nn.Linear(n_states, hidden_dim) # 输入层 self.fc1 = nn.Linear(state_dim, hidden_dim) # 输入层
self.fc2 = nn.Linear(hidden_dim,hidden_dim) # 隐藏层 self.fc2 = nn.Linear(hidden_dim,hidden_dim) # 隐藏层
self.fc3 = nn.Linear(hidden_dim, n_actions) # 输出层 self.fc3 = nn.Linear(hidden_dim, action_dim) # 输出层
def forward(self, x): def forward(self, x):
# 各层对应的激活函数 # 各层对应的激活函数
@@ -62,9 +62,9 @@ class ReplayBuffer:
return len(self.buffer) return len(self.buffer)
class DQN: class DQN:
def __init__(self, n_states, n_actions, cfg): def __init__(self, state_dim, action_dim, cfg):
self.n_actions = n_actions # 总的动作个数 self.action_dim = action_dim # 总的动作个数
self.device = cfg.device # 设备，cpu或gpu等 self.device = cfg.device # 设备，cpu或gpu等
self.gamma = cfg.gamma # 奖励的折扣因子 self.gamma = cfg.gamma # 奖励的折扣因子
# e-greedy策略相关参数 # e-greedy策略相关参数
@@ -73,8 +73,8 @@ class DQN:
(cfg.epsilon_start - cfg.epsilon_end) * \ (cfg.epsilon_start - cfg.epsilon_end) * \
math.exp(-1. * frame_idx / cfg.epsilon_decay) math.exp(-1. * frame_idx / cfg.epsilon_decay)
self.batch_size = cfg.batch_size self.batch_size = cfg.batch_size
self.policy_net = MLP(n_states, n_actions,hidden_dim=cfg.hidden_dim).to(self.device) self.policy_net = MLP(state_dim, action_dim,hidden_dim=cfg.hidden_dim).to(self.device)
self.target_net = MLP(n_states, n_actions,hidden_dim=cfg.hidden_dim).to(self.device) self.target_net = MLP(state_dim, action_dim,hidden_dim=cfg.hidden_dim).to(self.device)
for target_param, param in zip(self.target_net.parameters(),self.policy_net.parameters()): # 复制参数到目标网路targe_net for target_param, param in zip(self.target_net.parameters(),self.policy_net.parameters()): # 复制参数到目标网路targe_net
target_param.data.copy_(param.data) target_param.data.copy_(param.data)
self.optimizer = optim.Adam(self.policy_net.parameters(), lr=cfg.lr) # 优化器 self.optimizer = optim.Adam(self.policy_net.parameters(), lr=cfg.lr) # 优化器
@@ -90,7 +90,7 @@ class DQN:
q_values = self.policy_net(state) q_values = self.policy_net(state)
action = q_values.max(1)[1].item() # 选择Q值最大的动作 action = q_values.max(1)[1].item() # 选择Q值最大的动作
else: else:
action = random.randrange(self.n_actions) action = random.randrange(self.action_dim)
return action return action
def update(self): def update(self):
if len(self.memory) < self.batch_size: # 当memory中不满足一个批量时，不更新策略 if len(self.memory) < self.batch_size: # 当memory中不满足一个批量时，不更新策略

10
codes/DQN/dqn_cnn.py
View File

@@ -70,9 +70,9 @@ class ReplayBuffer:
return len(self.buffer) return len(self.buffer)
class DQN: class DQN:
def __init__(self, n_states, n_actions, cfg): def __init__(self, state_dim, action_dim, cfg):
self.n_actions = n_actions # 总的动作个数 self.action_dim = action_dim # 总的动作个数
self.device = cfg.device # 设备，cpu或gpu等 self.device = cfg.device # 设备，cpu或gpu等
self.gamma = cfg.gamma # 奖励的折扣因子 self.gamma = cfg.gamma # 奖励的折扣因子
# e-greedy策略相关参数 # e-greedy策略相关参数
@@ -81,8 +81,8 @@ class DQN:
(cfg.epsilon_start - cfg.epsilon_end) * \ (cfg.epsilon_start - cfg.epsilon_end) * \
math.exp(-1. * frame_idx / cfg.epsilon_decay) math.exp(-1. * frame_idx / cfg.epsilon_decay)
self.batch_size = cfg.batch_size self.batch_size = cfg.batch_size
self.policy_net = CNN(n_states, n_actions).to(self.device) self.policy_net = CNN(state_dim, action_dim).to(self.device)
self.target_net = CNN(n_states, n_actions).to(self.device) self.target_net = CNN(state_dim, action_dim).to(self.device)
for target_param, param in zip(self.target_net.parameters(),self.policy_net.parameters()): # 复制参数到目标网路targe_net for target_param, param in zip(self.target_net.parameters(),self.policy_net.parameters()): # 复制参数到目标网路targe_net
target_param.data.copy_(param.data) target_param.data.copy_(param.data)
self.optimizer = optim.Adam(self.policy_net.parameters(), lr=cfg.lr) # 优化器 self.optimizer = optim.Adam(self.policy_net.parameters(), lr=cfg.lr) # 优化器
@@ -98,7 +98,7 @@ class DQN:
q_values = self.policy_net(state) q_values = self.policy_net(state)
action = q_values.max(1)[1].item() # 选择Q值最大的动作 action = q_values.max(1)[1].item() # 选择Q值最大的动作
else: else:
action = random.randrange(self.n_actions) action = random.randrange(self.action_dim)
return action return action
def update(self): def update(self):
if len(self.memory) < self.batch_size: # 当memory中不满足一个批量时，不更新策略 if len(self.memory) < self.batch_size: # 当memory中不满足一个批量时，不更新策略

117
codes/DQN/task0.py
View File

@@ -7,23 +7,29 @@ sys.path.append(parent_path) # 添加路径到系统路径
import gym import gym
import torch import torch
import datetime import datetime
import numpy as np
from common.utils import save_results, make_dir from common.utils import save_results, make_dir
from common.utils import plot_rewards from common.utils import plot_rewards
from DQN.dqn import DQN from DQN.dqn import DQN
curr_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") # 获取当前时间 curr_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") # 获取当前时间
algo_name = 'DQN' # 算法名称
env_name = 'CartPole-v0' # 环境名称
class DQNConfig:
class Config:
'''超参数
'''
def __init__(self): def __init__(self):
self.algo_name = algo_name # 算法名称 ################################## 环境超参数 ###################################
self.env_name = env_name # 环境名称 self.algo_name = 'DQN' # 算法名称
self.env_name = 'CartPole-v0' # 环境名称
self.device = torch.device( self.device = torch.device(
"cuda" if torch.cuda.is_available() else "cpu") # 检测GPU "cuda" if torch.cuda.is_available() else "cpu") # 检测GPUgjgjlkhfsf风刀霜的撒发十
self.train_eps = 200 # 训练的回合数 self.train_eps = 200 # 训练的回合数
self.test_eps = 30 # 测试的回合数 self.test_eps = 30 # 测试的回合数
# 超参数 ################################################################################
################################## 算法超参数 ###################################
self.gamma = 0.95 # 强化学习中的折扣因子 self.gamma = 0.95 # 强化学习中的折扣因子
self.epsilon_start = 0.90 # e-greedy策略中初始epsilon self.epsilon_start = 0.90 # e-greedy策略中初始epsilon
self.epsilon_end = 0.01 # e-greedy策略中的终止epsilon self.epsilon_end = 0.01 # e-greedy策略中的终止epsilon
@@ -33,99 +39,106 @@ class DQNConfig:
self.batch_size = 64 # mini-batch SGD中的批量大小 self.batch_size = 64 # mini-batch SGD中的批量大小
self.target_update = 4 # 目标网络的更新频率 self.target_update = 4 # 目标网络的更新频率
self.hidden_dim = 256 # 网络隐藏层 self.hidden_dim = 256 # 网络隐藏层
class PlotConfig: ################################################################################
def __init__(self) -> None:
self.algo = algo_name # 算法名称 ################################# 保存结果相关参数 ################################
self.env_name = env_name # 环境名称
self.device = torch.device(
"cuda" if torch.cuda.is_available() else "cpu") # 检测GPU
self.result_path = curr_path + "/outputs/" + self.env_name + \ self.result_path = curr_path + "/outputs/" + self.env_name + \
'/' + curr_time + '/results/' # 保存结果的路径 '/' + curr_time + '/results/' # 保存结果的路径
self.model_path = curr_path + "/outputs/" + self.env_name + \ self.model_path = curr_path + "/outputs/" + self.env_name + \
'/' + curr_time + '/models/' # 保存模型的路径 '/' + curr_time + '/models/' # 保存模型的路径
self.save = True # 是否保存图片 self.save = True # 是否保存图片
################################################################################
def env_agent_config(cfg, seed=1): def env_agent_config(cfg, seed=1):
''' 创建环境和智能体 ''' 创建环境和智能体
''' '''
env = gym.make(cfg.env_name) # 创建环境 env = gym.make(cfg.env_name) # 创建环境
env.seed(seed) # 设置随机种子 state_dim = env.observation_space.shape[0] # 状态维度
n_states = env.observation_space.shape[0] # 状态数 action_dim = env.action_space.n # 动作维度
n_actions = env.action_space.n # 动作数 agent = DQN(state_dim, action_dim, cfg) # 创建智能体
agent = DQN(n_states, n_actions, cfg) # 创建智能体 if seed !=0: # 设置随机种子
torch.manual_seed(seed)
env.seed(seed)
np.random.seed(seed)
return env, agent return env, agent
def train(cfg, env, agent): def train(cfg, env, agent):
''' 训练 ''' 训练
''' '''
print('开始训练!') print('开始训练!')
print(f'环境：{cfg.env_name}, 算法：{cfg.algo_name}, 设备：{cfg.device}') print(f'环境：{cfg.env_name}, 算法：{cfg.algo_name}, 设备：{cfg.device}')
rewards = [] # 记录所有回合的奖励 rewards = [] # 记录所有回合的奖励
ma_rewards = [] # 记录所有回合的滑动平均奖励 ma_rewards = [] # 记录所有回合的滑动平均奖励
for i_ep in range(cfg.train_eps): for i_ep in range(cfg.train_eps):
ep_reward = 0 # 记录一回合内的奖励 ep_reward = 0 # 记录一回合内的奖励
state = env.reset() # 重置环境，返回初始状态 state = env.reset() # 重置环境，返回初始状态
while True: while True:
action = agent.choose_action(state) # 选择动作 action = agent.choose_action(state) # 选择动作
next_state, reward, done, _ = env.step(action) # 更新环境，返回transition next_state, reward, done, _ = env.step(action) # 更新环境，返回transition
agent.memory.push(state, action, reward, next_state, done) # 保存transition agent.memory.push(state, action, reward,
state = next_state # 更新下一个状态 next_state, done) # 保存transition
agent.update() # 更新智能体 state = next_state # 更新下一个状态
ep_reward += reward # 累加奖励 agent.update() # 更新智能体
ep_reward += reward # 累加奖励
if done: if done:
break break
if (i_ep+1) % cfg.target_update == 0: # 智能体目标网络更新 if (i_ep + 1) % cfg.target_update == 0: # 智能体目标网络更新
agent.target_net.load_state_dict(agent.policy_net.state_dict()) agent.target_net.load_state_dict(agent.policy_net.state_dict())
rewards.append(ep_reward) rewards.append(ep_reward)
if ma_rewards: if ma_rewards:
ma_rewards.append(0.9*ma_rewards[-1]+0.1*ep_reward) ma_rewards.append(0.9 * ma_rewards[-1] + 0.1 * ep_reward)
else: else:
ma_rewards.append(ep_reward) ma_rewards.append(ep_reward)
if (i_ep+1)%10 == 0: if (i_ep + 1) % 10 == 0:
print('回合：{}/{}, 奖励：{}'.format(i_ep+1, cfg.train_eps, ep_reward)) print('回合：{}/{}, 奖励：{}'.format(i_ep + 1, cfg.train_eps, ep_reward))
print('完成训练！') print('完成训练！')
return rewards, ma_rewards return rewards, ma_rewards
def test(cfg,env,agent):
def test(cfg, env, agent):
print('开始测试!') print('开始测试!')
print(f'环境：{cfg.env_name}, 算法：{cfg.algo_name}, 设备：{cfg.device}') print(f'环境：{cfg.env_name}, 算法：{cfg.algo_name}, 设备：{cfg.device}')
# 由于测试不需要使用epsilon-greedy策略，所以相应的值设置为0 # 由于测试不需要使用epsilon-greedy策略，所以相应的值设置为0
cfg.epsilon_start = 0.0 # e-greedy策略中初始epsilon cfg.epsilon_start = 0.0 # e-greedy策略中初始epsilon
cfg.epsilon_end = 0.0 # e-greedy策略中的终止epsilon cfg.epsilon_end = 0.0 # e-greedy策略中的终止epsilon
rewards = [] # 记录所有回合的奖励 rewards = [] # 记录所有回合的奖励
ma_rewards = [] # 记录所有回合的滑动平均奖励 ma_rewards = [] # 记录所有回合的滑动平均奖励
for i_ep in range(cfg.test_eps): for i_ep in range(cfg.test_eps):
ep_reward = 0 # 记录一回合内的奖励 ep_reward = 0 # 记录一回合内的奖励
state = env.reset() # 重置环境，返回初始状态 state = env.reset() # 重置环境，返回初始状态
while True: while True:
action = agent.choose_action(state) # 选择动作 action = agent.choose_action(state) # 选择动作
next_state, reward, done, _ = env.step(action) # 更新环境，返回transition next_state, reward, done, _ = env.step(action) # 更新环境，返回transition
state = next_state # 更新下一个状态 state = next_state # 更新下一个状态
ep_reward += reward # 累加奖励 ep_reward += reward # 累加奖励
if done: if done:
break break
rewards.append(ep_reward) rewards.append(ep_reward)
if ma_rewards: if ma_rewards:
ma_rewards.append(ma_rewards[-1]*0.9+ep_reward*0.1) ma_rewards.append(ma_rewards[-1] * 0.9 + ep_reward * 0.1)
else: else:
ma_rewards.append(ep_reward) ma_rewards.append(ep_reward)
print(f"回合：{i_ep+1}/{cfg.test_eps}，奖励：{ep_reward:.1f}") print(f"回合：{i_ep+1}/{cfg.test_eps}，奖励：{ep_reward:.1f}")
print('完成测试！') print('完成测试！')
return rewards,ma_rewards return rewards, ma_rewards
if __name__ == "__main__": if __name__ == "__main__":
cfg = DQNConfig() cfg = Config()
plot_cfg = PlotConfig()
# 训练 # 训练
env, agent = env_agent_config(cfg, seed=1) env, agent = env_agent_config(cfg, seed=1)
rewards, ma_rewards = train(cfg, env, agent) rewards, ma_rewards = train(cfg, env, agent)
make_dir(plot_cfg.result_path, plot_cfg.model_path) # 创建保存结果和模型路径的文件夹 make_dir(cfg.result_path, cfg.model_path) # 创建保存结果和模型路径的文件夹
agent.save(path=plot_cfg.model_path) # 保存模型 agent.save(path=cfg.model_path) # 保存模型
save_results(rewards, ma_rewards, tag='train', save_results(rewards, ma_rewards, tag='train',
path=plot_cfg.result_path) # 保存结果 path=cfg.result_path) # 保存结果
plot_rewards(rewards, ma_rewards, plot_cfg, tag="train") # 画出结果 plot_rewards(rewards, ma_rewards, cfg, tag="train") # 画出结果
# 测试 # 测试
env, agent = env_agent_config(cfg, seed=10) env, agent = env_agent_config(cfg, seed=10)
agent.load(path=plot_cfg.model_path) # 导入模型 agent.load(path=cfg.model_path) # 导入模型
rewards, ma_rewards = test(cfg, env, agent) rewards, ma_rewards = test(cfg, env, agent)
save_results(rewards, ma_rewards, tag='test', path=plot_cfg.result_path) # 保存结果 save_results(rewards, ma_rewards, tag='test',
plot_rewards(rewards, ma_rewards, plot_cfg, tag="test") # 画出结果 path=cfg.result_path) # 保存结果
plot_rewards(rewards, ma_rewards, cfg, tag="test") # 画出结果

6
codes/DQN/task1.py
View File

@@ -66,9 +66,9 @@ def env_agent_config(cfg, seed=1):
''' '''
env = gym.make(cfg.env_name) # 创建环境 env = gym.make(cfg.env_name) # 创建环境
env.seed(seed) # 设置随机种子 env.seed(seed) # 设置随机种子
n_states = env.observation_space.shape[0] # 状态数 state_dim = env.observation_space.shape[0] # 状态维度
n_actions = env.action_space.n # 动作数 action_dim = env.action_space.n # 动作维度
agent = DQN(n_states, n_actions, cfg) # 创建智能体 agent = DQN(state_dim, action_dim, cfg) # 创建智能体
return env, agent return env, agent
def train(cfg, env, agent): def train(cfg, env, agent):

6
codes/DQN/task2.py
View File

@@ -68,9 +68,9 @@ def env_agent_config(cfg, seed=1):
# env = wrap_deepmind(env) # env = wrap_deepmind(env)
# env = wrap_pytorch(env) # env = wrap_pytorch(env)
env.seed(seed) # 设置随机种子 env.seed(seed) # 设置随机种子
n_states = env.observation_space.shape[0] # 状态数 state_dim = env.observation_space.shape[0] # 状态维度
n_actions = env.action_space.n # 动作数 action_dim = env.action_space.n # 动作维度
agent = DQN(n_states, n_actions, cfg) # 创建智能体 agent = DQN(state_dim, action_dim, cfg) # 创建智能体
return env, agent return env, agent
def train(cfg, env, agent): def train(cfg, env, agent):

8
codes/Docs/使用DDPG解决倒立摆问题.md
View File

@@ -6,7 +6,7 @@
<img src="../../easy_rl_book/res/ch12/assets/pendulum_1.png" alt="image-20210915161550713" style="zoom:50%;" /> <img src="../../easy_rl_book/res/ch12/assets/pendulum_1.png" alt="image-20210915161550713" style="zoom:50%;" />
该环境的状态数有三个，设摆针竖直方向上的顺时针旋转角为$\theta$，$\theta$设在$[-\pi,\pi]$之间，则相应的状态为$[cos\theta,sin\theta,\dot{\theta}]$，即表示角度和角速度，我们的动作则是一个-2到2之间的力矩，它是一个连续量，因而该环境不能用离散动作的算法比如 DQN 来解决。关于奖励是根据相关的物理原理而计算出的等式，如下： 该环境的状态维度有三个，设摆针竖直方向上的顺时针旋转角为$\theta$，$\theta$设在$[-\pi,\pi]$之间，则相应的状态为$[cos\theta,sin\theta,\dot{\theta}]$，即表示角度和角速度，我们的动作则是一个-2到2之间的力矩，它是一个连续量，因而该环境不能用离散动作的算法比如 DQN 来解决。关于奖励是根据相关的物理原理而计算出的等式，如下：
$$ $$
-\left(\theta^{2}+0.1 * \hat{\theta}^{2}+0.001 * \text { action }^{2}\right) -\left(\theta^{2}+0.1 * \hat{\theta}^{2}+0.001 * \text { action }^{2}\right)
$$ $$
@@ -90,15 +90,15 @@ class OUNoise(object):
self.max_sigma = max_sigma self.max_sigma = max_sigma
self.min_sigma = min_sigma self.min_sigma = min_sigma
self.decay_period = decay_period self.decay_period = decay_period
self.n_actions = action_space.shape[0] self.action_dim = action_space.shape[0]
self.low = action_space.low self.low = action_space.low
self.high = action_space.high self.high = action_space.high
self.reset() self.reset()
def reset(self): def reset(self):
self.obs = np.ones(self.n_actions) * self.mu self.obs = np.ones(self.action_dim) * self.mu
def evolve_obs(self): def evolve_obs(self):
x = self.obs x = self.obs
dx = self.theta * (self.mu - x) + self.sigma * np.random.randn(self.n_actions) dx = self.theta * (self.mu - x) + self.sigma * np.random.randn(self.action_dim)
self.obs = x + dx self.obs = x + dx
return self.obs return self.obs
def get_action(self, action, t=0): def get_action(self, action, t=0):

14
codes/Docs/使用DQN解决推车杆问题.md
View File

@@ -14,21 +14,21 @@ CartPole-v0是一个经典的入门环境，如下图，它通过向左(动作=0
import gym import gym
env = gym.make('CartPole-v0') # 建立环境 env = gym.make('CartPole-v0') # 建立环境
env.seed(1) # 随机种子 env.seed(1) # 随机种子
n_states = env.observation_space.shape[0] # 状态数 state_dim = env.observation_space.shape[0] # 状态维度
n_actions = env.action_space.n # 动作数 action_dim = env.action_space.n # 动作维度
state = env.reset() # 初始化环境 state = env.reset() # 初始化环境
print(f"状态数：{n_states}，动作数：{n_actions}") print(f"状态维度：{state_dim}，动作维度：{action_dim}")
print(f"初始状态：{state}") print(f"初始状态：{state}")
``` ```
可以得到结果： 可以得到结果：
```bash ```bash
状态数：4，动作数：2 状态维度：4，动作维度：2
初始状态：[ 0.03073904 0.00145001 -0.03088818 -0.03131252] 初始状态：[ 0.03073904 0.00145001 -0.03088818 -0.03131252]
``` ```
该环境状态数是四个，分别为车的位置、车的速度、杆的角度以及杆顶部的速度，动作数为两个，并且是离散的向左或者向右。理论上达到最优化算法的情况下，推车杆是一直能保持平衡的，也就是每回合的步数是无限，但是这不方便训练，所以环境内部设置了每回合的最大步数为200，也就是说理想情况下，只需要我们每回合的奖励达到200就算训练完成。 该环境状态维度是四个，分别为车的位置、车的速度、杆的角度以及杆顶部的速度，动作维度为两个，并且是离散的向左或者向右。理论上达到最优化算法的情况下，推车杆是一直能保持平衡的，也就是每回合的步数是无限，但是这不方便训练，所以环境内部设置了每回合的最大步数为200，也就是说理想情况下，只需要我们每回合的奖励达到200就算训练完成。
## DQN基本接口 ## DQN基本接口
@@ -125,7 +125,7 @@ class ReplayBuffer:
class MLP(nn.Module): class MLP(nn.Module):
def __init__(self, input_dim,output_dim,hidden_dim=128): def __init__(self, input_dim,output_dim,hidden_dim=128):
""" 初始化q网络，为全连接网络 """ 初始化q网络，为全连接网络
input_dim: 输入的特征数即环境的状态数 input_dim: 输入的特征数即环境的状态维度
output_dim: 输出的动作维度 output_dim: 输出的动作维度
""" """
super(MLP, self).__init__() super(MLP, self).__init__()
@@ -157,7 +157,7 @@ def choose_action(self, state):
q_values = self.policy_net(state) q_values = self.policy_net(state)
action = q_values.max(1)[1].item() # 选择Q值最大的动作 action = q_values.max(1)[1].item() # 选择Q值最大的动作
else: else:
action = random.randrange(self.n_actions) action = random.randrange(self.action_dim)
``` ```
可以看到跟Q学习算法其实是一样的，都是用的$\epsilon-greedy$策略，只是使用神经网络的话我们需要通过Torch或者Tensorflow工具来处理相应的数据。 可以看到跟Q学习算法其实是一样的，都是用的$\epsilon-greedy$策略，只是使用神经网络的话我们需要通过Torch或者Tensorflow工具来处理相应的数据。

20
codes/Docs/使用Q-learning解决悬崖寻路问题.md
View File

@@ -27,21 +27,21 @@ env = gym.make('CliffWalking-v0') # 定义环境
env = CliffWalkingWapper(env) # 装饰环境 env = CliffWalkingWapper(env) # 装饰环境
``` ```
这里我们在程序中使用了一个装饰器重新定义环境，但不影响对环境的理解，感兴趣的同学具体看相关代码。可以由于gym环境封装得比较好，所以我们想要使用这个环境只需要使用gym.make命令输入函数名即可，然后我们可以查看环境的状态和动作数目： 这里我们在程序中使用了一个装饰器重新定义环境，但不影响对环境的理解，感兴趣的同学具体看相关代码。可以由于gym环境封装得比较好，所以我们想要使用这个环境只需要使用gym.make命令输入函数名即可，然后我们可以查看环境的状态和动作维度目：
```python ```python
n_states = env.observation_space.n # 状态数 state_dim = env.observation_space.n # 状态维度
n_actions = env.action_space.n # 动作数 action_dim = env.action_space.n # 动作维度
print(f"状态数：{n_states}，动作数：{n_actions}") print(f"状态维度：{state_dim}，动作维度：{action_dim}")
``` ```
打印出来的结果如下： 打印出来的结果如下：
```bash ```bash
状态数：48，动作数：4 状态维度：48，动作维度：4
``` ```
我们的状态数是48个，这里我们设置的是智能体当前所在网格的编号，而动作数是4，这表示有0，1，2，3对应着上下左右四个动作。另外我们也可以初始化环境并打印当前所在的状态： 我们的状态维度是48个，这里我们设置的是智能体当前所在网格的编号，而动作维度是4，这表示有0，1，2，3对应着上下左右四个动作。另外我们也可以初始化环境并打印当前所在的状态：
```python ```python
state = env.reset() state = env.reset()
@@ -72,9 +72,9 @@ print(state)
env = gym.make('CliffWalking-v0') # 定义环境 env = gym.make('CliffWalking-v0') # 定义环境
env = CliffWalkingWapper(env) # 装饰环境 env = CliffWalkingWapper(env) # 装饰环境
env.seed(1) # 设置随机种子 env.seed(1) # 设置随机种子
n_states = env.observation_space.n # 状态数 state_dim = env.observation_space.n # 状态维度
n_actions = env.action_space.n # 动作数 action_dim = env.action_space.n # 动作维度
agent = QLearning(n_states,n_actions,cfg) # cfg存储算法相关参数 agent = QLearning(state_dim,action_dim,cfg) # cfg存储算法相关参数
for i_ep in range(cfg.train_eps): # cfg.train_eps表示最大训练的回合数 for i_ep in range(cfg.train_eps): # cfg.train_eps表示最大训练的回合数
ep_reward = 0 # 记录每个回合的奖励 ep_reward = 0 # 记录每个回合的奖励
state = env.reset() # 重置环境 state = env.reset() # 重置环境
@@ -126,7 +126,7 @@ def choose_action(self, state):
if np.random.uniform(0, 1) > self.epsilon: if np.random.uniform(0, 1) > self.epsilon:
action = np.argmax(self.Q_table[str(state)]) # 选择Q(s,a)最大对应的动作 action = np.argmax(self.Q_table[str(state)]) # 选择Q(s,a)最大对应的动作
else: else:
action = np.random.choice(self.n_actions) # 随机选择动作 action = np.random.choice(self.action_dim) # 随机选择动作
return action return action
``` ```

20
codes/DoubleDQN/agent.py
View File

@@ -46,15 +46,15 @@ class ReplayBuffer:
return len(self.buffer) return len(self.buffer)
class MLP(nn.Module): class MLP(nn.Module):
def __init__(self, n_states,n_actions,hidden_dim=128): def __init__(self, state_dim,action_dim,hidden_dim=128):
""" 初始化q网络，为全连接网络 """ 初始化q网络，为全连接网络
n_states: 输入的特征数即环境的状态数 state_dim: 输入的特征数即环境的状态维度
n_actions: 输出的动作维度 action_dim: 输出的动作维度
""" """
super(MLP, self).__init__() super(MLP, self).__init__()
self.fc1 = nn.Linear(n_states, hidden_dim) # 输入层 self.fc1 = nn.Linear(state_dim, hidden_dim) # 输入层
self.fc2 = nn.Linear(hidden_dim,hidden_dim) # 隐藏层 self.fc2 = nn.Linear(hidden_dim,hidden_dim) # 隐藏层
self.fc3 = nn.Linear(hidden_dim, n_actions) # 输出层 self.fc3 = nn.Linear(hidden_dim, action_dim) # 输出层
def forward(self, x): def forward(self, x):
# 各层对应的激活函数 # 各层对应的激活函数
@@ -63,8 +63,8 @@ class MLP(nn.Module):
return self.fc3(x) return self.fc3(x)
class DoubleDQN: class DoubleDQN:
def __init__(self, n_states, n_actions, cfg): def __init__(self, state_dim, action_dim, cfg):
self.n_actions = n_actions # 总的动作个数 self.action_dim = action_dim # 总的动作个数
self.device = cfg.device # 设备，cpu或gpu等 self.device = cfg.device # 设备，cpu或gpu等
self.gamma = cfg.gamma self.gamma = cfg.gamma
# e-greedy策略相关参数 # e-greedy策略相关参数
@@ -73,8 +73,8 @@ class DoubleDQN:
self.epsilon_end = cfg.epsilon_end self.epsilon_end = cfg.epsilon_end
self.epsilon_decay = cfg.epsilon_decay self.epsilon_decay = cfg.epsilon_decay
self.batch_size = cfg.batch_size self.batch_size = cfg.batch_size
self.policy_net = MLP(n_states, n_actions,hidden_dim=cfg.hidden_dim).to(self.device) self.policy_net = MLP(state_dim, action_dim,hidden_dim=cfg.hidden_dim).to(self.device)
self.target_net = MLP(n_states, n_actions,hidden_dim=cfg.hidden_dim).to(self.device) self.target_net = MLP(state_dim, action_dim,hidden_dim=cfg.hidden_dim).to(self.device)
# target_net copy from policy_net # target_net copy from policy_net
for target_param, param in zip(self.target_net.parameters(), self.policy_net.parameters()): for target_param, param in zip(self.target_net.parameters(), self.policy_net.parameters()):
target_param.data.copy_(param.data) target_param.data.copy_(param.data)
@@ -103,7 +103,7 @@ class DoubleDQN:
# 所以tensor.max(1)[1]返回最大值对应的下标，即action # 所以tensor.max(1)[1]返回最大值对应的下标，即action
action = q_value.max(1)[1].item() action = q_value.max(1)[1].item()
else: else:
action = random.randrange(self.n_actions) action = random.randrange(self.action_dim)
return action return action
def update(self): def update(self):

6
codes/DoubleDQN/task0.py
View File

@@ -61,9 +61,9 @@ class PlotConfig:
def env_agent_config(cfg,seed=1): def env_agent_config(cfg,seed=1):
env = gym.make(cfg.env_name) env = gym.make(cfg.env_name)
env.seed(seed) env.seed(seed)
n_states = env.observation_space.shape[0] state_dim = env.observation_space.shape[0]
n_actions = env.action_space.n action_dim = env.action_space.n
agent = DoubleDQN(n_states,n_actions,cfg) agent = DoubleDQN(state_dim,action_dim,cfg)
return env,agent return env,agent
cfg = DoubleDQNConfig() cfg = DoubleDQNConfig()

14
codes/DuelingDQN/task0_train.ipynb
View File

@@ -136,12 +136,12 @@
"outputs": [], "outputs": [],
"source": [ "source": [
"class DuelingNet(nn.Module):\n", "class DuelingNet(nn.Module):\n",
" def __init__(self, n_states, n_actions,hidden_size=128):\n", " def __init__(self, state_dim, action_dim,hidden_size=128):\n",
" super(DuelingNet, self).__init__()\n", " super(DuelingNet, self).__init__()\n",
" \n", " \n",
" # 隐藏层\n", " # 隐藏层\n",
" self.hidden = nn.Sequential(\n", " self.hidden = nn.Sequential(\n",
" nn.Linear(n_states, hidden_size),\n", " nn.Linear(state_dim, hidden_size),\n",
" nn.ReLU()\n", " nn.ReLU()\n",
" )\n", " )\n",
" \n", " \n",
@@ -149,7 +149,7 @@
" self.advantage = nn.Sequential(\n", " self.advantage = nn.Sequential(\n",
" nn.Linear(hidden_size, hidden_size),\n", " nn.Linear(hidden_size, hidden_size),\n",
" nn.ReLU(),\n", " nn.ReLU(),\n",
" nn.Linear(hidden_size, n_actions)\n", " nn.Linear(hidden_size, action_dim)\n",
" )\n", " )\n",
" \n", " \n",
" # 价值函数\n", " # 价值函数\n",
@@ -192,7 +192,7 @@
], ],
"source": [ "source": [
"class DuelingDQN:\n", "class DuelingDQN:\n",
" def __init__(self,n_states,n_actions,cfg) -> None:\n", " def __init__(self,state_dim,action_dim,cfg) -> None:\n",
" self.batch_size = cfg.batch_size\n", " self.batch_size = cfg.batch_size\n",
" self.device = cfg.device\n", " self.device = cfg.device\n",
" self.loss_history = [] # 记录loss的变化\n", " self.loss_history = [] # 记录loss的变化\n",
@@ -200,8 +200,8 @@
" self.epsilon = lambda frame_idx: cfg.epsilon_end + \\\n", " self.epsilon = lambda frame_idx: cfg.epsilon_end + \\\n",
" (cfg.epsilon_start - cfg.epsilon_end) * \\\n", " (cfg.epsilon_start - cfg.epsilon_end) * \\\n",
" math.exp(-1. * frame_idx / cfg.epsilon_decay)\n", " math.exp(-1. * frame_idx / cfg.epsilon_decay)\n",
" self.policy_net = DuelingNet(n_states, n_actions,hidden_dim=cfg.hidden_dim).to(self.device)\n", " self.policy_net = DuelingNet(state_dim, action_dim,hidden_dim=cfg.hidden_dim).to(self.device)\n",
" self.target_net = DuelingNet(n_states, n_actions,hidden_dim=cfg.hidden_dim).to(self.device)\n", " self.target_net = DuelingNet(state_dim, action_dim,hidden_dim=cfg.hidden_dim).to(self.device)\n",
" for target_param, param in zip(self.target_net.parameters(),self.policy_net.parameters()): # 复制参数到目标网络targe_net\n", " for target_param, param in zip(self.target_net.parameters(),self.policy_net.parameters()): # 复制参数到目标网络targe_net\n",
" target_param.data.copy_(param.data)\n", " target_param.data.copy_(param.data)\n",
" self.optimizer = optim.Adam(self.policy_net.parameters(), lr=cfg.lr) # 优化器\n", " self.optimizer = optim.Adam(self.policy_net.parameters(), lr=cfg.lr) # 优化器\n",
@@ -214,7 +214,7 @@
" q_values = self.policy_net(state)\n", " q_values = self.policy_net(state)\n",
" action = q_values.max(1)[1].item() # 选择Q值最大的动作\n", " action = q_values.max(1)[1].item() # 选择Q值最大的动作\n",
" else:\n", " else:\n",
" action = random.randrange(self.n_actions)\n", " action = random.randrange(self.action_dim)\n",
" return action\n", " return action\n",
" def update(self):\n", " def update(self):\n",
" if len(self.memory) < self.batch_size: # 当memory中不满足一个批量时，不更新策略\n", " if len(self.memory) < self.batch_size: # 当memory中不满足一个批量时，不更新策略\n",

18
codes/HierarchicalDQN/agent.py
View File

@@ -42,7 +42,7 @@ class ReplayBuffer:
class MLP(nn.Module): class MLP(nn.Module):
def __init__(self, input_dim,output_dim,hidden_dim=128): def __init__(self, input_dim,output_dim,hidden_dim=128):
""" 初始化q网络，为全连接网络 """ 初始化q网络，为全连接网络
input_dim: 输入的特征数即环境的状态数 input_dim: 输入的特征数即环境的状态维度
output_dim: 输出的动作维度 output_dim: 输出的动作维度
""" """
super(MLP, self).__init__() super(MLP, self).__init__()
@@ -57,16 +57,16 @@ class MLP(nn.Module):
return self.fc3(x) return self.fc3(x)
class HierarchicalDQN: class HierarchicalDQN:
def __init__(self,n_states,n_actions,cfg): def __init__(self,state_dim,action_dim,cfg):
self.n_states = n_states self.state_dim = state_dim
self.n_actions = n_actions self.action_dim = action_dim
self.gamma = cfg.gamma self.gamma = cfg.gamma
self.device = cfg.device self.device = cfg.device
self.batch_size = cfg.batch_size self.batch_size = cfg.batch_size
self.frame_idx = 0 # 用于epsilon的衰减计数 self.frame_idx = 0 # 用于epsilon的衰减计数
self.epsilon = lambda frame_idx: cfg.epsilon_end + (cfg.epsilon_start - cfg.epsilon_end ) * math.exp(-1. * frame_idx / cfg.epsilon_decay) self.epsilon = lambda frame_idx: cfg.epsilon_end + (cfg.epsilon_start - cfg.epsilon_end ) * math.exp(-1. * frame_idx / cfg.epsilon_decay)
self.policy_net = MLP(2*n_states, n_actions,cfg.hidden_dim).to(self.device) self.policy_net = MLP(2*state_dim, action_dim,cfg.hidden_dim).to(self.device)
self.meta_policy_net = MLP(n_states, n_states,cfg.hidden_dim).to(self.device) self.meta_policy_net = MLP(state_dim, state_dim,cfg.hidden_dim).to(self.device)
self.optimizer = optim.Adam(self.policy_net.parameters(),lr=cfg.lr) self.optimizer = optim.Adam(self.policy_net.parameters(),lr=cfg.lr)
self.meta_optimizer = optim.Adam(self.meta_policy_net.parameters(),lr=cfg.lr) self.meta_optimizer = optim.Adam(self.meta_policy_net.parameters(),lr=cfg.lr)
self.memory = ReplayBuffer(cfg.memory_capacity) self.memory = ReplayBuffer(cfg.memory_capacity)
@@ -76,7 +76,7 @@ class HierarchicalDQN:
self.losses = [] self.losses = []
self.meta_losses = [] self.meta_losses = []
def to_onehot(self,x): def to_onehot(self,x):
oh = np.zeros(self.n_states) oh = np.zeros(self.state_dim)
oh[x - 1] = 1. oh[x - 1] = 1.
return oh return oh
def set_goal(self,state): def set_goal(self,state):
@@ -85,7 +85,7 @@ class HierarchicalDQN:
state = torch.tensor(state, device=self.device, dtype=torch.float32).unsqueeze(0) state = torch.tensor(state, device=self.device, dtype=torch.float32).unsqueeze(0)
goal = self.meta_policy_net(state).max(1)[1].item() goal = self.meta_policy_net(state).max(1)[1].item()
else: else:
goal = random.randrange(self.n_states) goal = random.randrange(self.state_dim)
return goal return goal
def choose_action(self,state): def choose_action(self,state):
self.frame_idx += 1 self.frame_idx += 1
@@ -95,7 +95,7 @@ class HierarchicalDQN:
q_value = self.policy_net(state) q_value = self.policy_net(state)
action = q_value.max(1)[1].item() action = q_value.max(1)[1].item()
else: else:
action = random.randrange(self.n_actions) action = random.randrange(self.action_dim)
return action return action
def update(self): def update(self):
self.update_policy() self.update_policy()

6
codes/HierarchicalDQN/task0.py
View File

@@ -63,9 +63,9 @@ class PlotConfig:
def env_agent_config(cfg,seed=1): def env_agent_config(cfg,seed=1):
env = gym.make(cfg.env_name) env = gym.make(cfg.env_name)
env.seed(seed) env.seed(seed)
n_states = env.observation_space.shape[0] state_dim = env.observation_space.shape[0]
n_actions = env.action_space.n action_dim = env.action_space.n
agent = HierarchicalDQN(n_states,n_actions,cfg) agent = HierarchicalDQN(state_dim,action_dim,cfg)
return env,agent return env,agent
if __name__ == "__main__": if __name__ == "__main__":

10
codes/MonteCarlo/agent.py
View File

@@ -17,11 +17,11 @@ import dill
class FisrtVisitMC: class FisrtVisitMC:
''' On-Policy First-Visit MC Control ''' On-Policy First-Visit MC Control
''' '''
def __init__(self,n_actions,cfg): def __init__(self,action_dim,cfg):
self.n_actions = n_actions self.action_dim = action_dim
self.epsilon = cfg.epsilon self.epsilon = cfg.epsilon
self.gamma = cfg.gamma self.gamma = cfg.gamma
self.Q_table = defaultdict(lambda: np.zeros(n_actions)) self.Q_table = defaultdict(lambda: np.zeros(action_dim))
self.returns_sum = defaultdict(float) # sum of returns self.returns_sum = defaultdict(float) # sum of returns
self.returns_count = defaultdict(float) self.returns_count = defaultdict(float)
@@ -29,11 +29,11 @@ class FisrtVisitMC:
''' e-greed policy ''' ''' e-greed policy '''
if state in self.Q_table.keys(): if state in self.Q_table.keys():
best_action = np.argmax(self.Q_table[state]) best_action = np.argmax(self.Q_table[state])
action_probs = np.ones(self.n_actions, dtype=float) * self.epsilon / self.n_actions action_probs = np.ones(self.action_dim, dtype=float) * self.epsilon / self.action_dim
action_probs[best_action] += (1.0 - self.epsilon) action_probs[best_action] += (1.0 - self.epsilon)
action = np.random.choice(np.arange(len(action_probs)), p=action_probs) action = np.random.choice(np.arange(len(action_probs)), p=action_probs)
else: else:
action = np.random.randint(0,self.n_actions) action = np.random.randint(0,self.action_dim)
return action return action
def update(self,one_ep_transition): def update(self,one_ep_transition):
# Find all (state, action) pairs we've visited in this one_ep_transition # Find all (state, action) pairs we've visited in this one_ep_transition

4
codes/MonteCarlo/task0_train.py
View File

@@ -43,8 +43,8 @@ class MCConfig:
def env_agent_config(cfg,seed=1): def env_agent_config(cfg,seed=1):
env = RacetrackEnv() env = RacetrackEnv()
n_actions = 9 action_dim = 9
agent = FisrtVisitMC(n_actions, cfg) agent = FisrtVisitMC(action_dim, cfg)
return env,agent return env,agent
def train(cfg, env, agent): def train(cfg, env, agent):

12
codes/PPO/README.md
View File

@@ -57,16 +57,16 @@ model就是actor和critic两个网络了：
import torch.nn as nn import torch.nn as nn
from torch.distributions.categorical import Categorical from torch.distributions.categorical import Categorical
class Actor(nn.Module): class Actor(nn.Module):
def __init__(self,n_states, n_actions, def __init__(self,state_dim, action_dim,
hidden_dim=256): hidden_dim=256):
super(Actor, self).__init__() super(Actor, self).__init__()
self.actor = nn.Sequential( self.actor = nn.Sequential(
nn.Linear(n_states, hidden_dim), nn.Linear(state_dim, hidden_dim),
nn.ReLU(), nn.ReLU(),
nn.Linear(hidden_dim, hidden_dim), nn.Linear(hidden_dim, hidden_dim),
nn.ReLU(), nn.ReLU(),
nn.Linear(hidden_dim, n_actions), nn.Linear(hidden_dim, action_dim),
nn.Softmax(dim=-1) nn.Softmax(dim=-1)
) )
def forward(self, state): def forward(self, state):
@@ -75,10 +75,10 @@ class Actor(nn.Module):
return dist return dist
class Critic(nn.Module): class Critic(nn.Module):
def __init__(self, n_states,hidden_dim=256): def __init__(self, state_dim,hidden_dim=256):
super(Critic, self).__init__() super(Critic, self).__init__()
self.critic = nn.Sequential( self.critic = nn.Sequential(
nn.Linear(n_states, hidden_dim), nn.Linear(state_dim, hidden_dim),
nn.ReLU(), nn.ReLU(),
nn.Linear(hidden_dim, hidden_dim), nn.Linear(hidden_dim, hidden_dim),
nn.ReLU(), nn.ReLU(),
@@ -88,7 +88,7 @@ class Critic(nn.Module):
value = self.critic(state) value = self.critic(state)
return value return value
``` ```
这里Actor就是得到一个概率分布(Categorica，也可以是别的分布，可以搜索torch distributionsl)，critc根据当前状态得到一个值，这里的输入维度可以是```n_states+n_actions```，即将action信息也纳入critic网络中，这样会更好一些，感兴趣的小伙伴可以试试。 这里Actor就是得到一个概率分布(Categorica，也可以是别的分布，可以搜索torch distributionsl)，critc根据当前状态得到一个值，这里的输入维度可以是```state_dim+action_dim```，即将action信息也纳入critic网络中，这样会更好一些，感兴趣的小伙伴可以试试。
### PPO update ### PPO update
定义一个update函数主要实现伪代码中的第六步和第七步： 定义一个update函数主要实现伪代码中的第六步和第七步：

6
codes/PPO/agent.py
View File

@@ -16,15 +16,15 @@ import torch.optim as optim
from PPO.model import Actor,Critic from PPO.model import Actor,Critic
from PPO.memory import PPOMemory from PPO.memory import PPOMemory
class PPO: class PPO:
def __init__(self, n_states, n_actions,cfg): def __init__(self, state_dim, action_dim,cfg):
self.gamma = cfg.gamma self.gamma = cfg.gamma
self.continuous = cfg.continuous self.continuous = cfg.continuous
self.policy_clip = cfg.policy_clip self.policy_clip = cfg.policy_clip
self.n_epochs = cfg.n_epochs self.n_epochs = cfg.n_epochs
self.gae_lambda = cfg.gae_lambda self.gae_lambda = cfg.gae_lambda
self.device = cfg.device self.device = cfg.device
self.actor = Actor(n_states, n_actions,cfg.hidden_dim).to(self.device) self.actor = Actor(state_dim, action_dim,cfg.hidden_dim).to(self.device)
self.critic = Critic(n_states,cfg.hidden_dim).to(self.device) self.critic = Critic(state_dim,cfg.hidden_dim).to(self.device)
self.actor_optimizer = optim.Adam(self.actor.parameters(), lr=cfg.actor_lr) self.actor_optimizer = optim.Adam(self.actor.parameters(), lr=cfg.actor_lr)
self.critic_optimizer = optim.Adam(self.critic.parameters(), lr=cfg.critic_lr) self.critic_optimizer = optim.Adam(self.critic.parameters(), lr=cfg.critic_lr)
self.memory = PPOMemory(cfg.batch_size) self.memory = PPOMemory(cfg.batch_size)

10
codes/PPO/model.py
View File

@@ -12,16 +12,16 @@ Environment:
import torch.nn as nn import torch.nn as nn
from torch.distributions.categorical import Categorical from torch.distributions.categorical import Categorical
class Actor(nn.Module): class Actor(nn.Module):
def __init__(self,n_states, n_actions, def __init__(self,state_dim, action_dim,
hidden_dim): hidden_dim):
super(Actor, self).__init__() super(Actor, self).__init__()
self.actor = nn.Sequential( self.actor = nn.Sequential(
nn.Linear(n_states, hidden_dim), nn.Linear(state_dim, hidden_dim),
nn.ReLU(), nn.ReLU(),
nn.Linear(hidden_dim, hidden_dim), nn.Linear(hidden_dim, hidden_dim),
nn.ReLU(), nn.ReLU(),
nn.Linear(hidden_dim, n_actions), nn.Linear(hidden_dim, action_dim),
nn.Softmax(dim=-1) nn.Softmax(dim=-1)
) )
def forward(self, state): def forward(self, state):
@@ -30,10 +30,10 @@ class Actor(nn.Module):
return dist return dist
class Critic(nn.Module): class Critic(nn.Module):
def __init__(self, n_states,hidden_dim): def __init__(self, state_dim,hidden_dim):
super(Critic, self).__init__() super(Critic, self).__init__()
self.critic = nn.Sequential( self.critic = nn.Sequential(
nn.Linear(n_states, hidden_dim), nn.Linear(state_dim, hidden_dim),
nn.ReLU(), nn.ReLU(),
nn.Linear(hidden_dim, hidden_dim), nn.Linear(hidden_dim, hidden_dim),
nn.ReLU(), nn.ReLU(),

6
codes/PPO/task0.py
View File

@@ -45,9 +45,9 @@ class PlotConfig:
def env_agent_config(cfg,seed=1): def env_agent_config(cfg,seed=1):
env = gym.make(cfg.env_name) env = gym.make(cfg.env_name)
env.seed(seed) env.seed(seed)
n_states = env.observation_space.shape[0] state_dim = env.observation_space.shape[0]
n_actions = env.action_space.n action_dim = env.action_space.n
agent = PPO(n_states,n_actions,cfg) agent = PPO(state_dim,action_dim,cfg)
return env,agent return env,agent
cfg = PPOConfig() cfg = PPOConfig()

6
codes/PPO/task1.py
View File

@@ -45,9 +45,9 @@ class PlotConfig:
def env_agent_config(cfg,seed=1): def env_agent_config(cfg,seed=1):
env = gym.make(cfg.env_name) env = gym.make(cfg.env_name)
env.seed(seed) env.seed(seed)
n_states = env.observation_space.shape[0] state_dim = env.observation_space.shape[0]
n_actions = env.action_space.shape[0] action_dim = env.action_space.shape[0]
agent = PPO(n_states,n_actions,cfg) agent = PPO(state_dim,action_dim,cfg)
return env,agent return env,agent

6
codes/PPO/train.ipynb
View File

@@ -90,9 +90,9 @@
"def env_agent_config(cfg,seed=1):\n", "def env_agent_config(cfg,seed=1):\n",
" env = gym.make(cfg.env) \n", " env = gym.make(cfg.env) \n",
" env.seed(seed)\n", " env.seed(seed)\n",
" n_states = env.observation_space.shape[0]\n", " state_dim = env.observation_space.shape[0]\n",
" n_actions = env.action_space.n\n", " action_dim = env.action_space.n\n",
" agent = PPO(n_states,n_actions,cfg)\n", " agent = PPO(state_dim,action_dim,cfg)\n",
" return env,agent" " return env,agent"
] ]
}, },

6
codes/PPO/train.py
View File

@@ -99,9 +99,9 @@ if __name__ == '__main__':
def env_agent_config(cfg,seed=1): def env_agent_config(cfg,seed=1):
env = gym.make(cfg.env_name) env = gym.make(cfg.env_name)
env.seed(seed) env.seed(seed)
n_states = env.observation_space.shape[0] state_dim = env.observation_space.shape[0]
n_actions = env.action_space.n action_dim = env.action_space.n
agent = PPO(n_states,n_actions,cfg) agent = PPO(state_dim,action_dim,cfg)
return env,agent return env,agent
cfg = PPOConfig() cfg = PPOConfig()

4
codes/PolicyGradient/agent.py
View File

@@ -17,9 +17,9 @@ from PolicyGradient.model import MLP
class PolicyGradient: class PolicyGradient:
def __init__(self, n_states,cfg): def __init__(self, state_dim,cfg):
self.gamma = cfg.gamma self.gamma = cfg.gamma
self.policy_net = MLP(n_states,hidden_dim=cfg.hidden_dim) self.policy_net = MLP(state_dim,hidden_dim=cfg.hidden_dim)
self.optimizer = torch.optim.RMSprop(self.policy_net.parameters(), lr=cfg.lr) self.optimizer = torch.optim.RMSprop(self.policy_net.parameters(), lr=cfg.lr)
self.batch_size = cfg.batch_size self.batch_size = cfg.batch_size

2
codes/PolicyGradient/model.py
View File

@@ -19,7 +19,7 @@ class MLP(nn.Module):
''' '''
def __init__(self,input_dim,hidden_dim = 36): def __init__(self,input_dim,hidden_dim = 36):
super(MLP, self).__init__() super(MLP, self).__init__()
# 24和36为hidden layer的层数，可根据input_dim, n_actions的情况来改变 # 24和36为hidden layer的层数，可根据input_dim, action_dim的情况来改变
self.fc1 = nn.Linear(input_dim, hidden_dim) self.fc1 = nn.Linear(input_dim, hidden_dim)
self.fc2 = nn.Linear(hidden_dim,hidden_dim) self.fc2 = nn.Linear(hidden_dim,hidden_dim)
self.fc3 = nn.Linear(hidden_dim, 1) # Prob of Left self.fc3 = nn.Linear(hidden_dim, 1) # Prob of Left

4
codes/PolicyGradient/task0_train.py
View File

@@ -46,8 +46,8 @@ class PGConfig:
def env_agent_config(cfg,seed=1): def env_agent_config(cfg,seed=1):
env = gym.make(cfg.env) env = gym.make(cfg.env)
env.seed(seed) env.seed(seed)
n_states = env.observation_space.shape[0] state_dim = env.observation_space.shape[0]
agent = PolicyGradient(n_states,cfg) agent = PolicyGradient(state_dim,cfg)
return env,agent return env,agent
def train(cfg,env,agent): def train(cfg,env,agent):

10
codes/QLearning/agent.py
View File

@@ -15,9 +15,9 @@ import torch
from collections import defaultdict from collections import defaultdict
class QLearning(object): class QLearning(object):
def __init__(self,n_states, def __init__(self,state_dim,
n_actions,cfg): action_dim,cfg):
self.n_actions = n_actions self.action_dim = action_dim
self.lr = cfg.lr # 学习率 self.lr = cfg.lr # 学习率
self.gamma = cfg.gamma self.gamma = cfg.gamma
self.epsilon = 0 self.epsilon = 0
@@ -25,7 +25,7 @@ class QLearning(object):
self.epsilon_start = cfg.epsilon_start self.epsilon_start = cfg.epsilon_start
self.epsilon_end = cfg.epsilon_end self.epsilon_end = cfg.epsilon_end
self.epsilon_decay = cfg.epsilon_decay self.epsilon_decay = cfg.epsilon_decay
self.Q_table = defaultdict(lambda: np.zeros(n_actions)) # 用嵌套字典存放状态->动作->状态-动作值（Q值）的映射，即Q表 self.Q_table = defaultdict(lambda: np.zeros(action_dim)) # 用嵌套字典存放状态->动作->状态-动作值（Q值）的映射，即Q表
def choose_action(self, state): def choose_action(self, state):
self.sample_count += 1 self.sample_count += 1
self.epsilon = self.epsilon_end + (self.epsilon_start - self.epsilon_end) * \ self.epsilon = self.epsilon_end + (self.epsilon_start - self.epsilon_end) * \
@@ -34,7 +34,7 @@ class QLearning(object):
if np.random.uniform(0, 1) > self.epsilon: if np.random.uniform(0, 1) > self.epsilon:
action = np.argmax(self.Q_table[str(state)]) # 选择Q(s,a)最大对应的动作 action = np.argmax(self.Q_table[str(state)]) # 选择Q(s,a)最大对应的动作
else: else:
action = np.random.choice(self.n_actions) # 随机选择动作 action = np.random.choice(self.action_dim) # 随机选择动作
return action return action
def predict(self,state): def predict(self,state):
action = np.argmax(self.Q_table[str(state)]) action = np.argmax(self.Q_table[str(state)])

16
codes/QLearning/task0.ipynb
View File

@@ -38,9 +38,9 @@
"outputs": [], "outputs": [],
"source": [ "source": [
"class QLearning(object):\n", "class QLearning(object):\n",
" def __init__(self,n_states,\n", " def __init__(self,state_dim,\n",
" n_actions,cfg):\n", " action_dim,cfg):\n",
" self.n_actions = n_actions \n", " self.action_dim = action_dim \n",
" self.lr = cfg.lr # 学习率\n", " self.lr = cfg.lr # 学习率\n",
" self.gamma = cfg.gamma \n", " self.gamma = cfg.gamma \n",
" self.epsilon = 0 \n", " self.epsilon = 0 \n",
@@ -48,7 +48,7 @@
" self.epsilon_start = cfg.epsilon_start\n", " self.epsilon_start = cfg.epsilon_start\n",
" self.epsilon_end = cfg.epsilon_end\n", " self.epsilon_end = cfg.epsilon_end\n",
" self.epsilon_decay = cfg.epsilon_decay\n", " self.epsilon_decay = cfg.epsilon_decay\n",
" self.Q_table = defaultdict(lambda: np.zeros(n_actions)) # 用嵌套字典存放状态->动作->状态-动作值（Q值）的映射，即Q表\n", " self.Q_table = defaultdict(lambda: np.zeros(action_dim)) # 用嵌套字典存放状态->动作->状态-动作值（Q值）的映射，即Q表\n",
" def choose_action(self, state):\n", " def choose_action(self, state):\n",
" self.sample_count += 1\n", " self.sample_count += 1\n",
" self.epsilon = self.epsilon_end + (self.epsilon_start - self.epsilon_end) * \\\n", " self.epsilon = self.epsilon_end + (self.epsilon_start - self.epsilon_end) * \\\n",
@@ -57,7 +57,7 @@
" if np.random.uniform(0, 1) > self.epsilon:\n", " if np.random.uniform(0, 1) > self.epsilon:\n",
" action = np.argmax(self.Q_table[str(state)]) # 选择Q(s,a)最大对应的动作\n", " action = np.argmax(self.Q_table[str(state)]) # 选择Q(s,a)最大对应的动作\n",
" else:\n", " else:\n",
" action = np.random.choice(self.n_actions) # 随机选择动作\n", " action = np.random.choice(self.action_dim) # 随机选择动作\n",
" return action\n", " return action\n",
" def predict(self,state):\n", " def predict(self,state):\n",
" action = np.argmax(self.Q_table[str(state)])\n", " action = np.argmax(self.Q_table[str(state)])\n",
@@ -238,9 +238,9 @@
" env = gym.make(cfg.env_name) \n", " env = gym.make(cfg.env_name) \n",
" env = CliffWalkingWapper(env)\n", " env = CliffWalkingWapper(env)\n",
" env.seed(seed) # 设置随机种子\n", " env.seed(seed) # 设置随机种子\n",
" n_states = env.observation_space.n # 状态维度\n", " state_dim = env.observation_space.n # 状态维度\n",
" n_actions = env.action_space.n # 动作维度\n", " action_dim = env.action_space.n # 动作维度\n",
" agent = QLearning(n_states,n_actions,cfg)\n", " agent = QLearning(state_dim,action_dim,cfg)\n",
" return env,agent" " return env,agent"
] ]
}, },

6
codes/QLearning/task0.py
View File

@@ -68,9 +68,9 @@ def env_agent_config(cfg,seed=1):
env = gym.make(cfg.env_name) env = gym.make(cfg.env_name)
env = CliffWalkingWapper(env) env = CliffWalkingWapper(env)
env.seed(seed) # 设置随机种子 env.seed(seed) # 设置随机种子
n_states = env.observation_space.n # 状态维度 state_dim = env.observation_space.n # 状态维度
n_actions = env.action_space.n # 动作维度 action_dim = env.action_space.n # 动作维度
agent = QLearning(n_states,n_actions,cfg) agent = QLearning(state_dim,action_dim,cfg)
return env,agent return env,agent
cfg = QlearningConfig() cfg = QlearningConfig()

10
codes/Sarsa/agent.py
View File

@@ -14,17 +14,17 @@ from collections import defaultdict
import torch import torch
class Sarsa(object): class Sarsa(object):
def __init__(self, def __init__(self,
n_actions,sarsa_cfg,): action_dim,sarsa_cfg,):
self.n_actions = n_actions # number of actions self.action_dim = action_dim # number of actions
self.lr = sarsa_cfg.lr # learning rate self.lr = sarsa_cfg.lr # learning rate
self.gamma = sarsa_cfg.gamma self.gamma = sarsa_cfg.gamma
self.epsilon = sarsa_cfg.epsilon self.epsilon = sarsa_cfg.epsilon
self.Q = defaultdict(lambda: np.zeros(n_actions)) self.Q = defaultdict(lambda: np.zeros(action_dim))
# self.Q = np.zeros((n_states, n_actions)) # Q表 # self.Q = np.zeros((state_dim, action_dim)) # Q表
def choose_action(self, state): def choose_action(self, state):
best_action = np.argmax(self.Q[state]) best_action = np.argmax(self.Q[state])
# action = best_action # action = best_action
action_probs = np.ones(self.n_actions, dtype=float) * self.epsilon / self.n_actions action_probs = np.ones(self.action_dim, dtype=float) * self.epsilon / self.action_dim
action_probs[best_action] += (1.0 - self.epsilon) action_probs[best_action] += (1.0 - self.epsilon)
action = np.random.choice(np.arange(len(action_probs)), p=action_probs) action = np.random.choice(np.arange(len(action_probs)), p=action_probs)
return action return action

4
codes/Sarsa/task0_train.py
View File

@@ -39,8 +39,8 @@ class SarsaConfig:
def env_agent_config(cfg,seed=1): def env_agent_config(cfg,seed=1):
env = RacetrackEnv() env = RacetrackEnv()
n_actions=9 action_dim=9
agent = Sarsa(n_actions,cfg) agent = Sarsa(action_dim,cfg)
return env,agent return env,agent
def train(cfg,env,agent): def train(cfg,env,agent):

16
codes/SoftActorCritic/model.py
View File

@@ -17,10 +17,10 @@ from torch.distributions import Normal
device=torch.device("cuda" if torch.cuda.is_available() else "cpu") device=torch.device("cuda" if torch.cuda.is_available() else "cpu")
class ValueNet(nn.Module): class ValueNet(nn.Module):
def __init__(self, n_states, hidden_dim, init_w=3e-3): def __init__(self, state_dim, hidden_dim, init_w=3e-3):
super(ValueNet, self).__init__() super(ValueNet, self).__init__()
self.linear1 = nn.Linear(n_states, hidden_dim) self.linear1 = nn.Linear(state_dim, hidden_dim)
self.linear2 = nn.Linear(hidden_dim, hidden_dim) self.linear2 = nn.Linear(hidden_dim, hidden_dim)
self.linear3 = nn.Linear(hidden_dim, 1) self.linear3 = nn.Linear(hidden_dim, 1)
@@ -35,10 +35,10 @@ class ValueNet(nn.Module):
class SoftQNet(nn.Module): class SoftQNet(nn.Module):
def __init__(self, n_states, n_actions, hidden_dim, init_w=3e-3): def __init__(self, state_dim, action_dim, hidden_dim, init_w=3e-3):
super(SoftQNet, self).__init__() super(SoftQNet, self).__init__()
self.linear1 = nn.Linear(n_states + n_actions, hidden_dim) self.linear1 = nn.Linear(state_dim + action_dim, hidden_dim)
self.linear2 = nn.Linear(hidden_dim, hidden_dim) self.linear2 = nn.Linear(hidden_dim, hidden_dim)
self.linear3 = nn.Linear(hidden_dim, 1) self.linear3 = nn.Linear(hidden_dim, 1)
@@ -54,20 +54,20 @@ class SoftQNet(nn.Module):
class PolicyNet(nn.Module): class PolicyNet(nn.Module):
def __init__(self, n_states, n_actions, hidden_dim, init_w=3e-3, log_std_min=-20, log_std_max=2): def __init__(self, state_dim, action_dim, hidden_dim, init_w=3e-3, log_std_min=-20, log_std_max=2):
super(PolicyNet, self).__init__() super(PolicyNet, self).__init__()
self.log_std_min = log_std_min self.log_std_min = log_std_min
self.log_std_max = log_std_max self.log_std_max = log_std_max
self.linear1 = nn.Linear(n_states, hidden_dim) self.linear1 = nn.Linear(state_dim, hidden_dim)
self.linear2 = nn.Linear(hidden_dim, hidden_dim) self.linear2 = nn.Linear(hidden_dim, hidden_dim)
self.mean_linear = nn.Linear(hidden_dim, n_actions) self.mean_linear = nn.Linear(hidden_dim, action_dim)
self.mean_linear.weight.data.uniform_(-init_w, init_w) self.mean_linear.weight.data.uniform_(-init_w, init_w)
self.mean_linear.bias.data.uniform_(-init_w, init_w) self.mean_linear.bias.data.uniform_(-init_w, init_w)
self.log_std_linear = nn.Linear(hidden_dim, n_actions) self.log_std_linear = nn.Linear(hidden_dim, action_dim)
self.log_std_linear.weight.data.uniform_(-init_w, init_w) self.log_std_linear.weight.data.uniform_(-init_w, init_w)
self.log_std_linear.bias.data.uniform_(-init_w, init_w) self.log_std_linear.bias.data.uniform_(-init_w, init_w)

26
codes/SoftActorCritic/sac.py
View File

@@ -43,10 +43,10 @@ class ReplayBuffer:
return len(self.buffer) return len(self.buffer)
class ValueNet(nn.Module): class ValueNet(nn.Module):
def __init__(self, n_states, hidden_dim, init_w=3e-3): def __init__(self, state_dim, hidden_dim, init_w=3e-3):
super(ValueNet, self).__init__() super(ValueNet, self).__init__()
self.linear1 = nn.Linear(n_states, hidden_dim) self.linear1 = nn.Linear(state_dim, hidden_dim)
self.linear2 = nn.Linear(hidden_dim, hidden_dim) self.linear2 = nn.Linear(hidden_dim, hidden_dim)
self.linear3 = nn.Linear(hidden_dim, 1) self.linear3 = nn.Linear(hidden_dim, 1)
@@ -61,10 +61,10 @@ class ValueNet(nn.Module):
class SoftQNet(nn.Module): class SoftQNet(nn.Module):
def __init__(self, n_states, n_actions, hidden_dim, init_w=3e-3): def __init__(self, state_dim, action_dim, hidden_dim, init_w=3e-3):
super(SoftQNet, self).__init__() super(SoftQNet, self).__init__()
self.linear1 = nn.Linear(n_states + n_actions, hidden_dim) self.linear1 = nn.Linear(state_dim + action_dim, hidden_dim)
self.linear2 = nn.Linear(hidden_dim, hidden_dim) self.linear2 = nn.Linear(hidden_dim, hidden_dim)
self.linear3 = nn.Linear(hidden_dim, 1) self.linear3 = nn.Linear(hidden_dim, 1)
@@ -80,20 +80,20 @@ class SoftQNet(nn.Module):
class PolicyNet(nn.Module): class PolicyNet(nn.Module):
def __init__(self, n_states, n_actions, hidden_dim, init_w=3e-3, log_std_min=-20, log_std_max=2): def __init__(self, state_dim, action_dim, hidden_dim, init_w=3e-3, log_std_min=-20, log_std_max=2):
super(PolicyNet, self).__init__() super(PolicyNet, self).__init__()
self.log_std_min = log_std_min self.log_std_min = log_std_min
self.log_std_max = log_std_max self.log_std_max = log_std_max
self.linear1 = nn.Linear(n_states, hidden_dim) self.linear1 = nn.Linear(state_dim, hidden_dim)
self.linear2 = nn.Linear(hidden_dim, hidden_dim) self.linear2 = nn.Linear(hidden_dim, hidden_dim)
self.mean_linear = nn.Linear(hidden_dim, n_actions) self.mean_linear = nn.Linear(hidden_dim, action_dim)
self.mean_linear.weight.data.uniform_(-init_w, init_w) self.mean_linear.weight.data.uniform_(-init_w, init_w)
self.mean_linear.bias.data.uniform_(-init_w, init_w) self.mean_linear.bias.data.uniform_(-init_w, init_w)
self.log_std_linear = nn.Linear(hidden_dim, n_actions) self.log_std_linear = nn.Linear(hidden_dim, action_dim)
self.log_std_linear.weight.data.uniform_(-init_w, init_w) self.log_std_linear.weight.data.uniform_(-init_w, init_w)
self.log_std_linear.bias.data.uniform_(-init_w, init_w) self.log_std_linear.bias.data.uniform_(-init_w, init_w)
@@ -134,14 +134,14 @@ class PolicyNet(nn.Module):
return action[0] return action[0]
class SAC: class SAC:
def __init__(self,n_states,n_actions,cfg) -> None: def __init__(self,state_dim,action_dim,cfg) -> None:
self.batch_size = cfg.batch_size self.batch_size = cfg.batch_size
self.memory = ReplayBuffer(cfg.capacity) self.memory = ReplayBuffer(cfg.capacity)
self.device = cfg.device self.device = cfg.device
self.value_net = ValueNet(n_states, cfg.hidden_dim).to(self.device) self.value_net = ValueNet(state_dim, cfg.hidden_dim).to(self.device)
self.target_value_net = ValueNet(n_states, cfg.hidden_dim).to(self.device) self.target_value_net = ValueNet(state_dim, cfg.hidden_dim).to(self.device)
self.soft_q_net = SoftQNet(n_states, n_actions, cfg.hidden_dim).to(self.device) self.soft_q_net = SoftQNet(state_dim, action_dim, cfg.hidden_dim).to(self.device)
self.policy_net = PolicyNet(n_states, n_actions, cfg.hidden_dim).to(self.device) self.policy_net = PolicyNet(state_dim, action_dim, cfg.hidden_dim).to(self.device)
self.value_optimizer = optim.Adam(self.value_net.parameters(), lr=cfg.value_lr) self.value_optimizer = optim.Adam(self.value_net.parameters(), lr=cfg.value_lr)
self.soft_q_optimizer = optim.Adam(self.soft_q_net.parameters(), lr=cfg.soft_q_lr) self.soft_q_optimizer = optim.Adam(self.soft_q_net.parameters(), lr=cfg.soft_q_lr)
self.policy_optimizer = optim.Adam(self.policy_net.parameters(), lr=cfg.policy_lr) self.policy_optimizer = optim.Adam(self.policy_net.parameters(), lr=cfg.policy_lr)

6
codes/SoftActorCritic/task0.py
View File

@@ -63,9 +63,9 @@ class PlotConfig:
def env_agent_config(cfg,seed=1): def env_agent_config(cfg,seed=1):
env = NormalizedActions(gym.make(cfg.env_name)) env = NormalizedActions(gym.make(cfg.env_name))
env.seed(seed) env.seed(seed)
n_actions = env.action_space.shape[0] action_dim = env.action_space.shape[0]
n_states = env.observation_space.shape[0] state_dim = env.observation_space.shape[0]
agent = SAC(n_states,n_actions,cfg) agent = SAC(state_dim,action_dim,cfg)
return env,agent return env,agent
def train(cfg,env,agent): def train(cfg,env,agent):

8
codes/SoftActorCritic/task0_train.ipynb
View File

@@ -70,9 +70,9 @@
"def env_agent_config(cfg,seed=1):\n", "def env_agent_config(cfg,seed=1):\n",
" env = NormalizedActions(gym.make(\"Pendulum-v0\"))\n", " env = NormalizedActions(gym.make(\"Pendulum-v0\"))\n",
" env.seed(seed)\n", " env.seed(seed)\n",
" n_actions = env.action_space.shape[0]\n", " action_dim = env.action_space.shape[0]\n",
" n_states = env.observation_space.shape[0]\n", " state_dim = env.observation_space.shape[0]\n",
" agent = SAC(n_states,n_actions,cfg)\n", " agent = SAC(state_dim,action_dim,cfg)\n",
" return env,agent" " return env,agent"
] ]
}, },
@@ -159,7 +159,7 @@
"\nDuring handling of the above exception, another exception occurred:\n", "\nDuring handling of the above exception, another exception occurred:\n",
"\u001b[0;31mDeprecatedEnv\u001b[0m Traceback (most recent call last)", "\u001b[0;31mDeprecatedEnv\u001b[0m Traceback (most recent call last)",
"\u001b[0;32m<ipython-input-7-91b1038013e4>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m 3\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 4\u001b[0m \u001b[0;31m# train\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m \u001b[0menv\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0magent\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0menv_agent_config\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcfg\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mseed\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 6\u001b[0m \u001b[0mrewards\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mma_rewards\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcfg\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0menv\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0magent\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 7\u001b[0m \u001b[0mmake_dir\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcfg\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresult_path\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcfg\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel_path\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m<ipython-input-7-91b1038013e4>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m 3\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 4\u001b[0m \u001b[0;31m# train\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m \u001b[0menv\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0magent\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0menv_agent_config\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcfg\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mseed\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 6\u001b[0m \u001b[0mrewards\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mma_rewards\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcfg\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0menv\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0magent\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 7\u001b[0m \u001b[0mmake_dir\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcfg\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresult_path\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcfg\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel_path\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m<ipython-input-4-040773221550>\u001b[0m in \u001b[0;36menv_agent_config\u001b[0;34m(cfg, seed)\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0menv_agent_config\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcfg\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mseed\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0menv\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mNormalizedActions\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgym\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmake\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Pendulum-v0\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 3\u001b[0m \u001b[0menv\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mseed\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mseed\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 4\u001b[0m \u001b[0mn_actions\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0menv\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maction_space\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[0mn_states\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0menv\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mobservation_space\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m<ipython-input-4-040773221550>\u001b[0m in \u001b[0;36menv_agent_config\u001b[0;34m(cfg, seed)\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0menv_agent_config\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcfg\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mseed\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0menv\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mNormalizedActions\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgym\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmake\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Pendulum-v0\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 3\u001b[0m \u001b[0menv\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mseed\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mseed\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 4\u001b[0m \u001b[0maction_dim\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0menv\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maction_space\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[0mstate_dim\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0menv\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mobservation_space\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/anaconda3/envs/py37/lib/python3.7/site-packages/gym/envs/registration.py\u001b[0m in \u001b[0;36mmake\u001b[0;34m(id, **kwargs)\u001b[0m\n\u001b[1;32m 233\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 234\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mmake\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mid\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 235\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mregistry\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmake\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mid\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 236\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 237\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m~/anaconda3/envs/py37/lib/python3.7/site-packages/gym/envs/registration.py\u001b[0m in \u001b[0;36mmake\u001b[0;34m(id, **kwargs)\u001b[0m\n\u001b[1;32m 233\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 234\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mmake\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mid\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 235\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mregistry\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmake\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mid\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 236\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 237\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/anaconda3/envs/py37/lib/python3.7/site-packages/gym/envs/registration.py\u001b[0m in \u001b[0;36mmake\u001b[0;34m(self, path, **kwargs)\u001b[0m\n\u001b[1;32m 126\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 127\u001b[0m \u001b[0mlogger\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minfo\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Making new env: %s\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpath\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 128\u001b[0;31m \u001b[0mspec\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mspec\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 129\u001b[0m \u001b[0menv\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mspec\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmake\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 130\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0menv\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m~/anaconda3/envs/py37/lib/python3.7/site-packages/gym/envs/registration.py\u001b[0m in \u001b[0;36mmake\u001b[0;34m(self, path, **kwargs)\u001b[0m\n\u001b[1;32m 126\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 127\u001b[0m \u001b[0mlogger\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minfo\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Making new env: %s\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpath\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 128\u001b[0;31m \u001b[0mspec\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mspec\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 129\u001b[0m \u001b[0menv\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mspec\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmake\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 130\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0menv\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/anaconda3/envs/py37/lib/python3.7/site-packages/gym/envs/registration.py\u001b[0m in \u001b[0;36mspec\u001b[0;34m(self, path)\u001b[0m\n\u001b[1;32m 185\u001b[0m raise error.DeprecatedEnv(\n\u001b[1;32m 186\u001b[0m \"Env {} not found (valid versions include {})\".format(\n\u001b[0;32m--> 187\u001b[0;31m \u001b[0mid\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmatching_envs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 188\u001b[0m )\n\u001b[1;32m 189\u001b[0m )\n", "\u001b[0;32m~/anaconda3/envs/py37/lib/python3.7/site-packages/gym/envs/registration.py\u001b[0m in \u001b[0;36mspec\u001b[0;34m(self, path)\u001b[0m\n\u001b[1;32m 185\u001b[0m raise error.DeprecatedEnv(\n\u001b[1;32m 186\u001b[0m \"Env {} not found (valid versions include {})\".format(\n\u001b[0;32m--> 187\u001b[0;31m \u001b[0mid\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmatching_envs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 188\u001b[0m )\n\u001b[1;32m 189\u001b[0m )\n",

4
codes/TD3/agent.py
View File

@@ -21,8 +21,8 @@ class Actor(nn.Module):
'''[summary] '''[summary]
Args: Args:
input_dim (int): 输入维度，这里等于n_states input_dim (int): 输入维度，这里等于state_dim
output_dim (int): 输出维度，这里等于n_actions output_dim (int): 输出维度，这里等于action_dim
max_action (int): action的最大值 max_action (int): action的最大值
''' '''
super(Actor, self).__init__() super(Actor, self).__init__()

8
codes/TD3/memory.py
View File

@@ -14,13 +14,13 @@ import torch
class ReplayBuffer(object): class ReplayBuffer(object):
def __init__(self, n_states, n_actions, max_size=int(1e6)): def __init__(self, state_dim, action_dim, max_size=int(1e6)):
self.max_size = max_size self.max_size = max_size
self.ptr = 0 self.ptr = 0
self.size = 0 self.size = 0
self.state = np.zeros((max_size, n_states)) self.state = np.zeros((max_size, state_dim))
self.action = np.zeros((max_size, n_actions)) self.action = np.zeros((max_size, action_dim))
self.next_state = np.zeros((max_size, n_states)) self.next_state = np.zeros((max_size, state_dim))
self.reward = np.zeros((max_size, 1)) self.reward = np.zeros((max_size, 1))
self.not_done = np.zeros((max_size, 1)) self.not_done = np.zeros((max_size, 1))
self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

6
codes/TD3/task0_eval.py
View File

@@ -74,10 +74,10 @@ if __name__ == "__main__":
env.seed(cfg.seed) # Set seeds env.seed(cfg.seed) # Set seeds
torch.manual_seed(cfg.seed) torch.manual_seed(cfg.seed)
np.random.seed(cfg.seed) np.random.seed(cfg.seed)
n_states = env.observation_space.shape[0] state_dim = env.observation_space.shape[0]
n_actions = env.action_space.shape[0] action_dim = env.action_space.shape[0]
max_action = float(env.action_space.high[0]) max_action = float(env.action_space.high[0])
td3= TD3(n_states,n_actions,max_action,cfg) td3= TD3(state_dim,action_dim,max_action,cfg)
cfg.model_path = './TD3/results/HalfCheetah-v2/20210416-130341/models/' cfg.model_path = './TD3/results/HalfCheetah-v2/20210416-130341/models/'
td3.load(cfg.model_path) td3.load(cfg.model_path)
td3_rewards,td3_ma_rewards = eval(cfg.env,td3,cfg.seed) td3_rewards,td3_ma_rewards = eval(cfg.env,td3,cfg.seed)

12
codes/TD3/task0_train.py
View File

@@ -72,7 +72,7 @@ def train(cfg,env,agent):
else: else:
action = ( action = (
agent.choose_action(np.array(state)) agent.choose_action(np.array(state))
+ np.random.normal(0, max_action * cfg.expl_noise, size=n_actions) + np.random.normal(0, max_action * cfg.expl_noise, size=action_dim)
).clip(-max_action, max_action) ).clip(-max_action, max_action)
# Perform action # Perform action
next_state, reward, done, _ = env.step(action) next_state, reward, done, _ = env.step(action)
@@ -121,11 +121,11 @@ def train(cfg,env,agent):
# else: # else:
# action = ( # action = (
# agent.choose_action(np.array(state)) # agent.choose_action(np.array(state))
# + np.random.normal(0, max_action * cfg.expl_noise, size=n_actions) # + np.random.normal(0, max_action * cfg.expl_noise, size=action_dim)
# ).clip(-max_action, max_action) # ).clip(-max_action, max_action)
# # action = ( # # action = (
# # agent.choose_action(np.array(state)) # # agent.choose_action(np.array(state))
# # + np.random.normal(0, max_action * cfg.expl_noise, size=n_actions) # # + np.random.normal(0, max_action * cfg.expl_noise, size=action_dim)
# # ).clip(-max_action, max_action) # # ).clip(-max_action, max_action)
# # Perform action # # Perform action
# next_state, reward, done, _ = env.step(action) # next_state, reward, done, _ = env.step(action)
@@ -157,10 +157,10 @@ if __name__ == "__main__":
env.seed(cfg.seed) # Set seeds env.seed(cfg.seed) # Set seeds
torch.manual_seed(cfg.seed) torch.manual_seed(cfg.seed)
np.random.seed(cfg.seed) np.random.seed(cfg.seed)
n_states = env.observation_space.shape[0] state_dim = env.observation_space.shape[0]
n_actions = env.action_space.shape[0] action_dim = env.action_space.shape[0]
max_action = float(env.action_space.high[0]) max_action = float(env.action_space.high[0])
agent = TD3(n_states,n_actions,max_action,cfg) agent = TD3(state_dim,action_dim,max_action,cfg)
rewards,ma_rewards = train(cfg,env,agent) rewards,ma_rewards = train(cfg,env,agent)
make_dir(cfg.result_path,cfg.model_path) make_dir(cfg.result_path,cfg.model_path)
agent.save(path=cfg.model_path) agent.save(path=cfg.model_path)

6
codes/TD3/task1_eval.py
View File

@@ -70,10 +70,10 @@ if __name__ == "__main__":
env.seed(cfg.seed) # Set seeds env.seed(cfg.seed) # Set seeds
torch.manual_seed(cfg.seed) torch.manual_seed(cfg.seed)
np.random.seed(cfg.seed) np.random.seed(cfg.seed)
n_states = env.observation_space.shape[0] state_dim = env.observation_space.shape[0]
n_actions = env.action_space.shape[0] action_dim = env.action_space.shape[0]
max_action = float(env.action_space.high[0]) max_action = float(env.action_space.high[0])
td3= TD3(n_states,n_actions,max_action,cfg) td3= TD3(state_dim,action_dim,max_action,cfg)
cfg.model_path = './TD3/results/Pendulum-v0/20210428-092059/models/' cfg.model_path = './TD3/results/Pendulum-v0/20210428-092059/models/'
cfg.result_path = './TD3/results/Pendulum-v0/20210428-092059/results/' cfg.result_path = './TD3/results/Pendulum-v0/20210428-092059/results/'
td3.load(cfg.model_path) td3.load(cfg.model_path)

8
codes/TD3/task1_train.py
View File

@@ -79,7 +79,7 @@ def train(cfg,env,agent):
else: else:
action = ( action = (
agent.choose_action(np.array(state)) agent.choose_action(np.array(state))
+ np.random.normal(0, max_action * cfg.expl_noise, size=n_actions) + np.random.normal(0, max_action * cfg.expl_noise, size=action_dim)
).clip(-max_action, max_action) ).clip(-max_action, max_action)
# Perform action # Perform action
next_state, reward, done, _ = env.step(action) next_state, reward, done, _ = env.step(action)
@@ -109,10 +109,10 @@ if __name__ == "__main__":
env.seed(1) # 随机种子 env.seed(1) # 随机种子
torch.manual_seed(1) torch.manual_seed(1)
np.random.seed(1) np.random.seed(1)
n_states = env.observation_space.shape[0] state_dim = env.observation_space.shape[0]
n_actions = env.action_space.shape[0] action_dim = env.action_space.shape[0]
max_action = float(env.action_space.high[0]) max_action = float(env.action_space.high[0])
agent = TD3(n_states,n_actions,max_action,cfg) agent = TD3(state_dim,action_dim,max_action,cfg)
rewards,ma_rewards = train(cfg,env,agent) rewards,ma_rewards = train(cfg,env,agent)
make_dir(plot_cfg.result_path,plot_cfg.model_path) make_dir(plot_cfg.result_path,plot_cfg.model_path)
agent.save(path=plot_cfg.model_path) agent.save(path=plot_cfg.model_path)

18
codes/common/model.py
View File

@@ -17,7 +17,7 @@ from torch.distributions import Categorical
class MLP(nn.Module): class MLP(nn.Module):
def __init__(self, input_dim,output_dim,hidden_dim=128): def __init__(self, input_dim,output_dim,hidden_dim=128):
""" 初始化q网络，为全连接网络 """ 初始化q网络，为全连接网络
input_dim: 输入的特征数即环境的状态数 input_dim: 输入的特征数即环境的状态维度
output_dim: 输出的动作维度 output_dim: 输出的动作维度
""" """
super(MLP, self).__init__() super(MLP, self).__init__()
@@ -32,10 +32,10 @@ class MLP(nn.Module):
return self.fc3(x) return self.fc3(x)
class Critic(nn.Module): class Critic(nn.Module):
def __init__(self, n_obs, n_actions, hidden_size, init_w=3e-3): def __init__(self, n_obs, action_dim, hidden_size, init_w=3e-3):
super(Critic, self).__init__() super(Critic, self).__init__()
self.linear1 = nn.Linear(n_obs + n_actions, hidden_size) self.linear1 = nn.Linear(n_obs + action_dim, hidden_size)
self.linear2 = nn.Linear(hidden_size, hidden_size) self.linear2 = nn.Linear(hidden_size, hidden_size)
self.linear3 = nn.Linear(hidden_size, 1) self.linear3 = nn.Linear(hidden_size, 1)
# 随机初始化为较小的值 # 随机初始化为较小的值
@@ -51,11 +51,11 @@ class Critic(nn.Module):
return x return x
class Actor(nn.Module): class Actor(nn.Module):
def __init__(self, n_obs, n_actions, hidden_size, init_w=3e-3): def __init__(self, n_obs, action_dim, hidden_size, init_w=3e-3):
super(Actor, self).__init__() super(Actor, self).__init__()
self.linear1 = nn.Linear(n_obs, hidden_size) self.linear1 = nn.Linear(n_obs, hidden_size)
self.linear2 = nn.Linear(hidden_size, hidden_size) self.linear2 = nn.Linear(hidden_size, hidden_size)
self.linear3 = nn.Linear(hidden_size, n_actions) self.linear3 = nn.Linear(hidden_size, action_dim)
self.linear3.weight.data.uniform_(-init_w, init_w) self.linear3.weight.data.uniform_(-init_w, init_w)
self.linear3.bias.data.uniform_(-init_w, init_w) self.linear3.bias.data.uniform_(-init_w, init_w)
@@ -67,18 +67,18 @@ class Actor(nn.Module):
return x return x
class ActorCritic(nn.Module): class ActorCritic(nn.Module):
def __init__(self, n_states, n_actions, hidden_dim=256): def __init__(self, state_dim, action_dim, hidden_dim=256):
super(ActorCritic, self).__init__() super(ActorCritic, self).__init__()
self.critic = nn.Sequential( self.critic = nn.Sequential(
nn.Linear(n_states, hidden_dim), nn.Linear(state_dim, hidden_dim),
nn.ReLU(), nn.ReLU(),
nn.Linear(hidden_dim, 1) nn.Linear(hidden_dim, 1)
) )
self.actor = nn.Sequential( self.actor = nn.Sequential(
nn.Linear(n_states, hidden_dim), nn.Linear(state_dim, hidden_dim),
nn.ReLU(), nn.ReLU(),
nn.Linear(hidden_dim, n_actions), nn.Linear(hidden_dim, action_dim),
nn.Softmax(dim=1), nn.Softmax(dim=1),
) )

2
codes/envs/blackjack.py
View File

@@ -77,7 +77,7 @@ class BlackjackEnv(gym.Env):
self.natural = natural self.natural = natural
# Start the first game # Start the first game
self._reset() # Number of self._reset() # Number of
self.n_actions = 2 self.action_dim = 2
def reset(self): def reset(self):
return self._reset() return self._reset()

6
codes/envs/cliff_walking.py
View File

@@ -31,7 +31,7 @@ class CliffWalkingEnv(discrete.DiscreteEnv):
self.shape = (4, 12) self.shape = (4, 12)
nS = np.prod(self.shape) nS = np.prod(self.shape)
n_actions = 4 action_dim = 4
# Cliff Location # Cliff Location
self._cliff = np.zeros(self.shape, dtype=np.bool) self._cliff = np.zeros(self.shape, dtype=np.bool)
@@ -41,7 +41,7 @@ class CliffWalkingEnv(discrete.DiscreteEnv):
P = {} P = {}
for s in range(nS): for s in range(nS):
position = np.unravel_index(s, self.shape) position = np.unravel_index(s, self.shape)
P[s] = { a : [] for a in range(n_actions) } P[s] = { a : [] for a in range(action_dim) }
P[s][UP] = self._calculate_transition_prob(position, [-1, 0]) P[s][UP] = self._calculate_transition_prob(position, [-1, 0])
P[s][RIGHT] = self._calculate_transition_prob(position, [0, 1]) P[s][RIGHT] = self._calculate_transition_prob(position, [0, 1])
P[s][DOWN] = self._calculate_transition_prob(position, [1, 0]) P[s][DOWN] = self._calculate_transition_prob(position, [1, 0])
@@ -51,7 +51,7 @@ class CliffWalkingEnv(discrete.DiscreteEnv):
isd = np.zeros(nS) isd = np.zeros(nS)
isd[np.ravel_multi_index((3,0), self.shape)] = 1.0 isd[np.ravel_multi_index((3,0), self.shape)] = 1.0
super(CliffWalkingEnv, self).__init__(nS, n_actions, P, isd) super(CliffWalkingEnv, self).__init__(nS, action_dim, P, isd)
def render(self, mode='human', close=False): def render(self, mode='human', close=False):
self._render(mode, close) self._render(mode, close)

6
codes/envs/gridworld.py
View File

@@ -37,7 +37,7 @@ class GridworldEnv(discrete.DiscreteEnv):
self.shape = shape self.shape = shape
nS = np.prod(shape) nS = np.prod(shape)
n_actions = 4 action_dim = 4
MAX_Y = shape[0] MAX_Y = shape[0]
MAX_X = shape[1] MAX_X = shape[1]
@@ -51,7 +51,7 @@ class GridworldEnv(discrete.DiscreteEnv):
y, x = it.multi_index y, x = it.multi_index
# P[s][a] = (prob, next_state, reward, is_done) # P[s][a] = (prob, next_state, reward, is_done)
P[s] = {a : [] for a in range(n_actions)} P[s] = {a : [] for a in range(action_dim)}
is_done = lambda s: s == 0 or s == (nS - 1) is_done = lambda s: s == 0 or s == (nS - 1)
reward = 0.0 if is_done(s) else -1.0 reward = 0.0 if is_done(s) else -1.0
@@ -82,7 +82,7 @@ class GridworldEnv(discrete.DiscreteEnv):
# This should not be used in any model-free learning algorithm # This should not be used in any model-free learning algorithm
self.P = P self.P = P
super(GridworldEnv, self).__init__(nS, n_actions, P, isd) super(GridworldEnv, self).__init__(nS, action_dim, P, isd)
def _render(self, mode='human', close=False): def _render(self, mode='human', close=False):
""" Renders the current gridworld layout """ Renders the current gridworld layout

12
codes/envs/stochastic_mdp.py
View File

@@ -17,31 +17,31 @@ class StochasticMDP:
def __init__(self): def __init__(self):
self.end = False self.end = False
self.curr_state = 2 self.curr_state = 2
self.n_actions = 2 self.action_dim = 2
self.n_states = 6 self.state_dim = 6
self.p_right = 0.5 self.p_right = 0.5
def reset(self): def reset(self):
self.end = False self.end = False
self.curr_state = 2 self.curr_state = 2
state = np.zeros(self.n_states) state = np.zeros(self.state_dim)
state[self.curr_state - 1] = 1. state[self.curr_state - 1] = 1.
return state return state
def step(self, action): def step(self, action):
if self.curr_state != 1: if self.curr_state != 1:
if action == 1: if action == 1:
if random.random() < self.p_right and self.curr_state < self.n_states: if random.random() < self.p_right and self.curr_state < self.state_dim:
self.curr_state += 1 self.curr_state += 1
else: else:
self.curr_state -= 1 self.curr_state -= 1
if action == 0: if action == 0:
self.curr_state -= 1 self.curr_state -= 1
if self.curr_state == self.n_states: if self.curr_state == self.state_dim:
self.end = True self.end = True
state = np.zeros(self.n_states) state = np.zeros(self.state_dim)
state[self.curr_state - 1] = 1. state[self.curr_state - 1] = 1.
if self.curr_state == 1: if self.curr_state == 1:

6
codes/envs/windy_gridworld.py
View File

@@ -30,7 +30,7 @@ class WindyGridworldEnv(discrete.DiscreteEnv):
self.shape = (7, 10) self.shape = (7, 10)
nS = np.prod(self.shape) nS = np.prod(self.shape)
n_actions = 4 action_dim = 4
# Wind strength # Wind strength
winds = np.zeros(self.shape) winds = np.zeros(self.shape)
@@ -41,7 +41,7 @@ class WindyGridworldEnv(discrete.DiscreteEnv):
P = {} P = {}
for s in range(nS): for s in range(nS):
position = np.unravel_index(s, self.shape) position = np.unravel_index(s, self.shape)
P[s] = { a : [] for a in range(n_actions) } P[s] = { a : [] for a in range(action_dim) }
P[s][UP] = self._calculate_transition_prob(position, [-1, 0], winds) P[s][UP] = self._calculate_transition_prob(position, [-1, 0], winds)
P[s][RIGHT] = self._calculate_transition_prob(position, [0, 1], winds) P[s][RIGHT] = self._calculate_transition_prob(position, [0, 1], winds)
P[s][DOWN] = self._calculate_transition_prob(position, [1, 0], winds) P[s][DOWN] = self._calculate_transition_prob(position, [1, 0], winds)
@@ -51,7 +51,7 @@ class WindyGridworldEnv(discrete.DiscreteEnv):
isd = np.zeros(nS) isd = np.zeros(nS)
isd[np.ravel_multi_index((3,0), self.shape)] = 1.0 isd[np.ravel_multi_index((3,0), self.shape)] = 1.0
super(WindyGridworldEnv, self).__init__(nS, n_actions, P, isd) super(WindyGridworldEnv, self).__init__(nS, action_dim, P, isd)
def render(self, mode='human', close=False): def render(self, mode='human', close=False):
self._render(mode, close) self._render(mode, close)