diff --git a/codes/A2C/agent.py b/codes/A2C/a2c.py similarity index 90% rename from codes/A2C/agent.py rename to codes/A2C/a2c.py index 997401b..bd26785 100644 --- a/codes/A2C/agent.py +++ b/codes/A2C/a2c.py @@ -40,10 +40,10 @@ class ActorCritic(nn.Module): class A2C: ''' A2C算法 ''' - def __init__(self,state_dim,action_dim,cfg) -> None: + def __init__(self,n_states,n_actions,cfg) -> None: self.gamma = cfg.gamma self.device = cfg.device - self.model = ActorCritic(state_dim, action_dim, cfg.hidden_size).to(self.device) + self.model = ActorCritic(n_states, n_actions, cfg.hidden_size).to(self.device) self.optimizer = optim.Adam(self.model.parameters()) def compute_returns(self,next_value, rewards, masks): diff --git a/codes/A2C/task0.py b/codes/A2C/task0.py index fd54d87..8e3cd0f 100644 --- a/codes/A2C/task0.py +++ b/codes/A2C/task0.py @@ -10,7 +10,7 @@ import torch import torch.optim as optim import datetime from common.multiprocessing_env import SubprocVecEnv -from A2C.agent import ActorCritic +from a2c import ActorCritic from common.utils import save_results, make_dir from common.utils import plot_rewards @@ -74,9 +74,9 @@ def train(cfg,envs): print(f'环境：{cfg.env_name}, 算法：{cfg.algo}, 设备：{cfg.device}') env = gym.make(cfg.env_name) # a single env env.seed(10) - state_dim = envs.observation_space.shape[0] - action_dim = envs.action_space.n - model = ActorCritic(state_dim, action_dim, cfg.hidden_dim).to(cfg.device) + n_states = envs.observation_space.shape[0] + n_actions = envs.action_space.n + model = ActorCritic(n_states, n_actions, cfg.hidden_dim).to(cfg.device) optimizer = optim.Adam(model.parameters()) frame_idx = 0 test_rewards = [] diff --git a/codes/DDPG/outputs/Pendulum-v0/20210916-013138/models/checkpoint.pt b/codes/DDPG/assets/outputs/Pendulum-v0/20210916-013138/models/checkpoint.pt similarity index 100% rename from codes/DDPG/outputs/Pendulum-v0/20210916-013138/models/checkpoint.pt rename to codes/DDPG/assets/outputs/Pendulum-v0/20210916-013138/models/checkpoint.pt diff --git a/codes/DDPG/outputs/Pendulum-v0/20210916-013138/results/eval_ma_rewards.npy b/codes/DDPG/assets/outputs/Pendulum-v0/20210916-013138/results/eval_ma_rewards.npy similarity index 100% rename from codes/DDPG/outputs/Pendulum-v0/20210916-013138/results/eval_ma_rewards.npy rename to codes/DDPG/assets/outputs/Pendulum-v0/20210916-013138/results/eval_ma_rewards.npy diff --git a/codes/DDPG/outputs/Pendulum-v0/20210916-013138/results/eval_rewards.npy b/codes/DDPG/assets/outputs/Pendulum-v0/20210916-013138/results/eval_rewards.npy similarity index 100% rename from codes/DDPG/outputs/Pendulum-v0/20210916-013138/results/eval_rewards.npy rename to codes/DDPG/assets/outputs/Pendulum-v0/20210916-013138/results/eval_rewards.npy diff --git a/codes/DDPG/outputs/Pendulum-v0/20210916-013138/results/eval_rewards_curve_cn.png b/codes/DDPG/assets/outputs/Pendulum-v0/20210916-013138/results/eval_rewards_curve_cn.png similarity index 100% rename from codes/DDPG/outputs/Pendulum-v0/20210916-013138/results/eval_rewards_curve_cn.png rename to codes/DDPG/assets/outputs/Pendulum-v0/20210916-013138/results/eval_rewards_curve_cn.png diff --git a/codes/DDPG/outputs/Pendulum-v0/20210916-013138/results/train_ma_rewards.npy b/codes/DDPG/assets/outputs/Pendulum-v0/20210916-013138/results/train_ma_rewards.npy similarity index 100% rename from codes/DDPG/outputs/Pendulum-v0/20210916-013138/results/train_ma_rewards.npy rename to codes/DDPG/assets/outputs/Pendulum-v0/20210916-013138/results/train_ma_rewards.npy diff --git a/codes/DDPG/outputs/Pendulum-v0/20210916-013138/results/train_rewards.npy b/codes/DDPG/assets/outputs/Pendulum-v0/20210916-013138/results/train_rewards.npy similarity index 100% rename from codes/DDPG/outputs/Pendulum-v0/20210916-013138/results/train_rewards.npy rename to codes/DDPG/assets/outputs/Pendulum-v0/20210916-013138/results/train_rewards.npy diff --git a/codes/DDPG/outputs/Pendulum-v0/20210916-013138/results/train_rewards_curve_cn.png b/codes/DDPG/assets/outputs/Pendulum-v0/20210916-013138/results/train_rewards_curve_cn.png similarity index 100% rename from codes/DDPG/outputs/Pendulum-v0/20210916-013138/results/train_rewards_curve_cn.png rename to codes/DDPG/assets/outputs/Pendulum-v0/20210916-013138/results/train_rewards_curve_cn.png diff --git a/codes/DDPG/agent.py b/codes/DDPG/ddpg.py similarity index 88% rename from codes/DDPG/agent.py rename to codes/DDPG/ddpg.py index 6ec2eef..01ded1c 100644 --- a/codes/DDPG/agent.py +++ b/codes/DDPG/ddpg.py @@ -39,11 +39,11 @@ class ReplayBuffer: ''' return len(self.buffer) class Actor(nn.Module): - def __init__(self, state_dim, action_dim, hidden_dim, init_w=3e-3): + def __init__(self, n_states, n_actions, hidden_dim, init_w=3e-3): super(Actor, self).__init__() - self.linear1 = nn.Linear(state_dim, hidden_dim) + self.linear1 = nn.Linear(n_states, hidden_dim) self.linear2 = nn.Linear(hidden_dim, hidden_dim) - self.linear3 = nn.Linear(hidden_dim, action_dim) + self.linear3 = nn.Linear(hidden_dim, n_actions) self.linear3.weight.data.uniform_(-init_w, init_w) self.linear3.bias.data.uniform_(-init_w, init_w) @@ -54,10 +54,10 @@ class Actor(nn.Module): x = torch.tanh(self.linear3(x)) return x class Critic(nn.Module): - def __init__(self, state_dim, action_dim, hidden_dim, init_w=3e-3): + def __init__(self, n_states, n_actions, hidden_dim, init_w=3e-3): super(Critic, self).__init__() - self.linear1 = nn.Linear(state_dim + action_dim, hidden_dim) + self.linear1 = nn.Linear(n_states + n_actions, hidden_dim) self.linear2 = nn.Linear(hidden_dim, hidden_dim) self.linear3 = nn.Linear(hidden_dim, 1) # 随机初始化为较小的值 @@ -72,12 +72,12 @@ class Critic(nn.Module): x = self.linear3(x) return x class DDPG: - def __init__(self, state_dim, action_dim, cfg): + def __init__(self, n_states, n_actions, cfg): self.device = cfg.device - self.critic = Critic(state_dim, action_dim, cfg.hidden_dim).to(cfg.device) - self.actor = Actor(state_dim, action_dim, cfg.hidden_dim).to(cfg.device) - self.target_critic = Critic(state_dim, action_dim, cfg.hidden_dim).to(cfg.device) - self.target_actor = Actor(state_dim, action_dim, cfg.hidden_dim).to(cfg.device) + self.critic = Critic(n_states, n_actions, cfg.hidden_dim).to(cfg.device) + self.actor = Actor(n_states, n_actions, cfg.hidden_dim).to(cfg.device) + self.target_critic = Critic(n_states, n_actions, cfg.hidden_dim).to(cfg.device) + self.target_actor = Actor(n_states, n_actions, cfg.hidden_dim).to(cfg.device) # 复制参数到目标网络 for target_param, param in zip(self.target_critic.parameters(), self.critic.parameters()): diff --git a/codes/DDPG/env.py b/codes/DDPG/env.py index 92fe482..89445cf 100644 --- a/codes/DDPG/env.py +++ b/codes/DDPG/env.py @@ -39,15 +39,15 @@ class OUNoise(object): self.max_sigma = max_sigma self.min_sigma = min_sigma self.decay_period = decay_period - self.action_dim = action_space.shape[0] + self.n_actions = action_space.shape[0] self.low = action_space.low self.high = action_space.high self.reset() def reset(self): - self.obs = np.ones(self.action_dim) * self.mu + self.obs = np.ones(self.n_actions) * self.mu def evolve_obs(self): x = self.obs - dx = self.theta * (self.mu - x) + self.sigma * np.random.randn(self.action_dim) + dx = self.theta * (self.mu - x) + self.sigma * np.random.randn(self.n_actions) self.obs = x + dx return self.obs def get_action(self, action, t=0): diff --git a/codes/DDPG/task0.py b/codes/DDPG/task0.py index 81fa9a6..04da4a9 100644 --- a/codes/DDPG/task0.py +++ b/codes/DDPG/task0.py @@ -5,7 +5,7 @@ @Email: johnjim0816@gmail.com @Date: 2020-06-11 20:58:21 @LastEditor: John -LastEditTime: 2021-09-16 01:31:33 +LastEditTime: 2022-02-10 06:23:27 @Discription: @Environment: python 3.7.7 ''' @@ -18,23 +18,29 @@ import datetime import gym import torch -from DDPG.env import NormalizedActions -from DDPG.agent import DDPG +from env import NormalizedActions,OUNoise +from ddpg import DDPG from DDPG.train import train,test from common.utils import save_results,make_dir from common.utils import plot_rewards curr_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") # 获取当前时间 -algo_name = 'DDPG' # 算法名称 -env_name = 'Pendulum-v1' # 环境名称，gym新版本（约0.21.0之后）中Pendulum-v0改为Pendulum-v1 +class Config: + '''超参数 + ''' -class DDPGConfig: def __init__(self): - self.algo_name = algo_name # 算法名称 - self.env_name = env_name # 环境名称 - self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # 检测GPU + ################################## 环境超参数 ################################### + self.algo_name = 'DDPG' # 算法名称 + self.env_name = 'Pendulum-v1' # 环境名称，gym新版本（约0.21.0之后）中Pendulum-v0改为Pendulum-v1 + self.device = torch.device( + "cuda" if torch.cuda.is_available() else "cpu") # 检测GPUgjgjlkhfsf风刀霜的撒发十 + self.seed = 10 # 随机种子，置0则不设置随机种子 self.train_eps = 300 # 训练的回合数 self.test_eps = 50 # 测试的回合数 + ################################################################################ + + ################################## 算法超参数 ################################### self.gamma = 0.99 # 折扣因子 self.critic_lr = 1e-3 # 评论家网络的学习率 self.actor_lr = 1e-4 # 演员网络的学习率 @@ -43,39 +49,92 @@ class DDPGConfig: self.target_update = 2 # 目标网络的更新频率 self.hidden_dim = 256 # 网络隐藏层维度 self.soft_tau = 1e-2 # 软更新参数 + ################################################################################ -class PlotConfig: - def __init__(self) -> None: - self.algo_name = algo_name # 算法名称 - self.env_name = env_name # 环境名称 - self.result_path = curr_path+"/outputs/" + self.env_name + \ - '/'+curr_time+'/results/' # 保存结果的路径 - self.model_path = curr_path+"/outputs/" + self.env_name + \ - '/'+curr_time+'/models/' # 保存模型的路径 + ################################# 保存结果相关参数 ################################ + self.result_path = curr_path + "/outputs/" + self.env_name + \ + '/' + curr_time + '/results/' # 保存结果的路径 + self.model_path = curr_path + "/outputs/" + self.env_name + \ + '/' + curr_time + '/models/' # 保存模型的路径 self.save = True # 是否保存图片 - self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # 检测GPU + ################################################################################ def env_agent_config(cfg,seed=1): env = NormalizedActions(gym.make(cfg.env_name)) # 装饰action噪声 env.seed(seed) # 随机种子 - state_dim = env.observation_space.shape[0] - action_dim = env.action_space.shape[0] - agent = DDPG(state_dim,action_dim,cfg) + n_states = env.observation_space.shape[0] + n_actions = env.action_space.shape[0] + agent = DDPG(n_states,n_actions,cfg) return env,agent +def train(cfg, env, agent): + print('开始训练！') + print(f'环境：{cfg.env_name}，算法：{cfg.algo}，设备：{cfg.device}') + ou_noise = OUNoise(env.action_space) # 动作噪声 + rewards = [] # 记录所有回合的奖励 + ma_rewards = [] # 记录所有回合的滑动平均奖励 + for i_ep in range(cfg.train_eps): + state = env.reset() + ou_noise.reset() + done = False + ep_reward = 0 + i_step = 0 + while not done: + i_step += 1 + action = agent.choose_action(state) + action = ou_noise.get_action(action, i_step) + next_state, reward, done, _ = env.step(action) + ep_reward += reward + agent.memory.push(state, action, reward, next_state, done) + agent.update() + state = next_state + if (i_ep+1)%10 == 0: + print('回合：{}/{}，奖励：{:.2f}'.format(i_ep+1, cfg.train_eps, ep_reward)) + rewards.append(ep_reward) + if ma_rewards: + ma_rewards.append(0.9*ma_rewards[-1]+0.1*ep_reward) + else: + ma_rewards.append(ep_reward) + print('完成训练！') + return rewards, ma_rewards -cfg = DDPGConfig() -plot_cfg = PlotConfig() -# 训练 -env,agent = env_agent_config(cfg,seed=1) -rewards, ma_rewards = train(cfg, env, agent) -make_dir(plot_cfg.result_path, plot_cfg.model_path) -agent.save(path=plot_cfg.model_path) -save_results(rewards, ma_rewards, tag='train', path=plot_cfg.result_path) -plot_rewards(rewards, ma_rewards, plot_cfg, tag="train") # 画出结果 -# 测试 -env,agent = env_agent_config(cfg,seed=10) -agent.load(path=plot_cfg.model_path) -rewards,ma_rewards = test(plot_cfg,env,agent) -save_results(rewards,ma_rewards,tag = 'test',path = cfg.result_path) -plot_rewards(rewards, ma_rewards, plot_cfg, tag="test") # 画出结果 +def test(cfg, env, agent): + print('开始测试！') + print(f'环境：{cfg.env_name}, 算法：{cfg.algo}, 设备：{cfg.device}') + rewards = [] # 记录所有回合的奖励 + ma_rewards = [] # 记录所有回合的滑动平均奖励 + for i_ep in range(cfg.test_eps): + state = env.reset() + done = False + ep_reward = 0 + i_step = 0 + while not done: + i_step += 1 + action = agent.choose_action(state) + next_state, reward, done, _ = env.step(action) + ep_reward += reward + state = next_state + print('回合：{}/{}, 奖励：{}'.format(i_ep+1, cfg.train_eps, ep_reward)) + rewards.append(ep_reward) + if ma_rewards: + ma_rewards.append(0.9*ma_rewards[-1]+0.1*ep_reward) + else: + ma_rewards.append(ep_reward) + print(f"回合：{i_ep+1}/{cfg.test_eps}，奖励：{ep_reward:.1f}") + print('完成测试！') + return rewards, ma_rewards +if __name__ == "__main__": + cfg = Config() + # 训练 + env,agent = env_agent_config(cfg,seed=1) + rewards, ma_rewards = train(cfg, env, agent) + make_dir(cfg.result_path, cfg.model_path) + agent.save(path=cfg.model_path) + save_results(rewards, ma_rewards, tag='train', path=cfg.result_path) + plot_rewards(rewards, ma_rewards, cfg, tag="train") # 画出结果 + # 测试 + env,agent = env_agent_config(cfg,seed=10) + agent.load(path=cfg.model_path) + rewards,ma_rewards = test(cfg,env,agent) + save_results(rewards,ma_rewards,tag = 'test',path = cfg.result_path) + plot_rewards(rewards, ma_rewards, cfg, tag="test") # 画出结果 diff --git a/codes/DDPG/train.py b/codes/DDPG/train.py deleted file mode 100644 index 4cdfa9d..0000000 --- a/codes/DDPG/train.py +++ /dev/null @@ -1,64 +0,0 @@ -import sys -import os -curr_path = os.path.dirname(os.path.abspath(__file__)) # 当前文件所在绝对路径 -parent_path = os.path.dirname(curr_path) # 父路径 -sys.path.append(parent_path) # 添加路径到系统路径 - -from DDPG.env import OUNoise - -def train(cfg, env, agent): - print('开始训练！') - print(f'环境：{cfg.env_name}，算法：{cfg.algo}，设备：{cfg.device}') - ou_noise = OUNoise(env.action_space) # 动作噪声 - rewards = [] # 记录所有回合的奖励 - ma_rewards = [] # 记录所有回合的滑动平均奖励 - for i_ep in range(cfg.train_eps): - state = env.reset() - ou_noise.reset() - done = False - ep_reward = 0 - i_step = 0 - while not done: - i_step += 1 - action = agent.choose_action(state) - action = ou_noise.get_action(action, i_step) - next_state, reward, done, _ = env.step(action) - ep_reward += reward - agent.memory.push(state, action, reward, next_state, done) - agent.update() - state = next_state - if (i_ep+1)%10 == 0: - print('回合：{}/{}，奖励：{:.2f}'.format(i_ep+1, cfg.train_eps, ep_reward)) - rewards.append(ep_reward) - if ma_rewards: - ma_rewards.append(0.9*ma_rewards[-1]+0.1*ep_reward) - else: - ma_rewards.append(ep_reward) - print('完成训练！') - return rewards, ma_rewards - -def test(cfg, env, agent): - print('开始测试！') - print(f'环境：{cfg.env_name}, 算法：{cfg.algo}, 设备：{cfg.device}') - rewards = [] # 记录所有回合的奖励 - ma_rewards = [] # 记录所有回合的滑动平均奖励 - for i_ep in range(cfg.test_eps): - state = env.reset() - done = False - ep_reward = 0 - i_step = 0 - while not done: - i_step += 1 - action = agent.choose_action(state) - next_state, reward, done, _ = env.step(action) - ep_reward += reward - state = next_state - print('回合：{}/{}, 奖励：{}'.format(i_ep+1, cfg.train_eps, ep_reward)) - rewards.append(ep_reward) - if ma_rewards: - ma_rewards.append(0.9*ma_rewards[-1]+0.1*ep_reward) - else: - ma_rewards.append(ep_reward) - print(f"回合：{i_ep+1}/{cfg.test_eps}，奖励：{ep_reward:.1f}") - print('完成测试！') - return rewards, ma_rewards \ No newline at end of file diff --git a/codes/DQN/README.md b/codes/DQN/README.md index fc82fe6..33e7397 100644 --- a/codes/DQN/README.md +++ b/codes/DQN/README.md @@ -50,15 +50,15 @@ import torch.nn as nn import torch.nn.functional as F class FCN(nn.Module): - def __init__(self, state_dim=4, action_dim=18): + def __init__(self, n_states=4, n_actions=18): """ 初始化q网络，为全连接网络 - state_dim: 输入的feature即环境的state数目 - action_dim: 输出的action总个数 + n_states: 输入的feature即环境的state数目 + n_actions: 输出的action总个数 """ super(FCN, self).__init__() - self.fc1 = nn.Linear(state_dim, 128) # 输入层 + self.fc1 = nn.Linear(n_states, 128) # 输入层 self.fc2 = nn.Linear(128, 128) # 隐藏层 - self.fc3 = nn.Linear(128, action_dim) # 输出层 + self.fc3 = nn.Linear(128, n_actions) # 输出层 def forward(self, x): # 各层对应的激活函数 @@ -66,7 +66,7 @@ class FCN(nn.Module): x = F.relu(self.fc2(x)) return self.fc3(x) ``` -输入为state_dim，输出为action_dim，包含一个128维度的隐藏层，这里根据需要可增加隐藏层维度和数量，然后一般使用relu激活函数，这里跟深度学习的网路设置是一样的。 +输入为n_states，输出为n_actions，包含一个128维度的隐藏层，这里根据需要可增加隐藏层维度和数量，然后一般使用relu激活函数，这里跟深度学习的网路设置是一样的。 ### Replay Buffer @@ -107,8 +107,8 @@ class ReplayBuffer: 在类中建立两个网络，以及optimizer和memory， ```python -self.policy_net = MLP(state_dim, action_dim,hidden_dim=cfg.hidden_dim).to(self.device) -self.target_net = MLP(state_dim, action_dim,hidden_dim=cfg.hidden_dim).to(self.device) +self.policy_net = MLP(n_states, n_actions,hidden_dim=cfg.hidden_dim).to(self.device) +self.target_net = MLP(n_states, n_actions,hidden_dim=cfg.hidden_dim).to(self.device) for target_param, param in zip(self.target_net.parameters(),self.policy_net.parameters()): # copy params from policy net target_param.data.copy_(param.data) self.optimizer = optim.Adam(self.policy_net.parameters(), lr=cfg.lr) @@ -124,7 +124,7 @@ def choose_action(self, state): if random.random() > self.epsilon(self.frame_idx): action = self.predict(state) else: - action = random.randrange(self.action_dim) + action = random.randrange(self.n_actions) return action ``` diff --git a/codes/DQN/dqn.py b/codes/DQN/dqn.py index 4a4dfc4..8e74e37 100644 --- a/codes/DQN/dqn.py +++ b/codes/DQN/dqn.py @@ -5,7 +5,7 @@ @Email: johnjim0816@gmail.com @Date: 2020-06-12 00:50:49 @LastEditor: John -LastEditTime: 2021-12-22 14:01:37 +LastEditTime: 2022-03-02 11:05:11 @Discription: @Environment: python 3.7.7 ''' @@ -20,22 +20,7 @@ import random import math import numpy as np -class MLP(nn.Module): - def __init__(self, state_dim,action_dim,hidden_dim=128): - """ 初始化q网络，为全连接网络 - state_dim: 输入的特征数即环境的状态维度 - action_dim: 输出的动作维度 - """ - super(MLP, self).__init__() - self.fc1 = nn.Linear(state_dim, hidden_dim) # 输入层 - self.fc2 = nn.Linear(hidden_dim,hidden_dim) # 隐藏层 - self.fc3 = nn.Linear(hidden_dim, action_dim) # 输出层 - - def forward(self, x): - # 各层对应的激活函数 - x = F.relu(self.fc1(x)) - x = F.relu(self.fc2(x)) - return self.fc3(x) + class ReplayBuffer: def __init__(self, capacity): @@ -62,9 +47,9 @@ class ReplayBuffer: return len(self.buffer) class DQN: - def __init__(self, state_dim, action_dim, cfg): + def __init__(self, n_actions,model,cfg): - self.action_dim = action_dim # 总的动作个数 + self.n_actions = n_actions # 总的动作个数 self.device = cfg.device # 设备，cpu或gpu等 self.gamma = cfg.gamma # 奖励的折扣因子 # e-greedy策略相关参数 @@ -73,8 +58,8 @@ class DQN: (cfg.epsilon_start - cfg.epsilon_end) * \ math.exp(-1. * frame_idx / cfg.epsilon_decay) self.batch_size = cfg.batch_size - self.policy_net = MLP(state_dim, action_dim,hidden_dim=cfg.hidden_dim).to(self.device) - self.target_net = MLP(state_dim, action_dim,hidden_dim=cfg.hidden_dim).to(self.device) + self.policy_net = model.to(self.device) + self.target_net = model.to(self.device) for target_param, param in zip(self.target_net.parameters(),self.policy_net.parameters()): # 复制参数到目标网路targe_net target_param.data.copy_(param.data) self.optimizer = optim.Adam(self.policy_net.parameters(), lr=cfg.lr) # 优化器 @@ -86,23 +71,24 @@ class DQN: self.frame_idx += 1 if random.random() > self.epsilon(self.frame_idx): with torch.no_grad(): - state = torch.tensor([state], device=self.device, dtype=torch.float32) + state = torch.tensor(state, device=self.device, dtype=torch.float32).unsqueeze(dim=0) q_values = self.policy_net(state) action = q_values.max(1)[1].item() # 选择Q值最大的动作 else: - action = random.randrange(self.action_dim) + action = random.randrange(self.n_actions) return action def update(self): if len(self.memory) < self.batch_size: # 当memory中不满足一个批量时，不更新策略 return # 从经验回放中(replay memory)中随机采样一个批量的转移(transition) + # print('updating') + state_batch, action_batch, reward_batch, next_state_batch, done_batch = self.memory.sample( self.batch_size) - # 转为张量 - state_batch = torch.tensor(state_batch, device=self.device, dtype=torch.float) + state_batch = torch.tensor(np.array(state_batch), device=self.device, dtype=torch.float) action_batch = torch.tensor(action_batch, device=self.device).unsqueeze(1) reward_batch = torch.tensor(reward_batch, device=self.device, dtype=torch.float) - next_state_batch = torch.tensor(next_state_batch, device=self.device, dtype=torch.float) + next_state_batch = torch.tensor(np.array(next_state_batch), device=self.device, dtype=torch.float) done_batch = torch.tensor(np.float32(done_batch), device=self.device) q_values = self.policy_net(state_batch).gather(dim=1, index=action_batch) # 计算当前状态(s_t,a)对应的Q(s_t, a) next_q_values = self.target_net(next_state_batch).max(1)[0].detach() # 计算下一时刻的状态(s_t_,a)对应的Q值 diff --git a/codes/DQN/dqn_cnn.py b/codes/DQN/dqn_cnn.py index c14118f..4c086b2 100644 --- a/codes/DQN/dqn_cnn.py +++ b/codes/DQN/dqn_cnn.py @@ -70,9 +70,9 @@ class ReplayBuffer: return len(self.buffer) class DQN: - def __init__(self, state_dim, action_dim, cfg): + def __init__(self, n_states, n_actions, cfg): - self.action_dim = action_dim # 总的动作个数 + self.n_actions = n_actions # 总的动作个数 self.device = cfg.device # 设备，cpu或gpu等 self.gamma = cfg.gamma # 奖励的折扣因子 # e-greedy策略相关参数 @@ -81,8 +81,8 @@ class DQN: (cfg.epsilon_start - cfg.epsilon_end) * \ math.exp(-1. * frame_idx / cfg.epsilon_decay) self.batch_size = cfg.batch_size - self.policy_net = CNN(state_dim, action_dim).to(self.device) - self.target_net = CNN(state_dim, action_dim).to(self.device) + self.policy_net = CNN(n_states, n_actions).to(self.device) + self.target_net = CNN(n_states, n_actions).to(self.device) for target_param, param in zip(self.target_net.parameters(),self.policy_net.parameters()): # 复制参数到目标网路targe_net target_param.data.copy_(param.data) self.optimizer = optim.Adam(self.policy_net.parameters(), lr=cfg.lr) # 优化器 @@ -94,11 +94,12 @@ class DQN: self.frame_idx += 1 if random.random() > self.epsilon(self.frame_idx): with torch.no_grad(): + print(type(state)) state = torch.tensor([state], device=self.device, dtype=torch.float32) q_values = self.policy_net(state) action = q_values.max(1)[1].item() # 选择Q值最大的动作 else: - action = random.randrange(self.action_dim) + action = random.randrange(self.n_actions) return action def update(self): if len(self.memory) < self.batch_size: # 当memory中不满足一个批量时，不更新策略 diff --git a/codes/DQN/dqn_cnn2.py b/codes/DQN/dqn_cnn2.py new file mode 100644 index 0000000..67b7fd8 --- /dev/null +++ b/codes/DQN/dqn_cnn2.py @@ -0,0 +1,142 @@ +import torch +import torch.nn as nn +import torch.optim as optim +import torch.autograd as autograd +import random +import math +import numpy as np +class CNN(nn.Module): + def __init__(self, n_frames, n_actions): + super(CNN,self).__init__() + self.n_frames = n_frames + self.n_actions = n_actions + + # Layers + self.conv1 = nn.Conv2d( + in_channels=n_frames, + out_channels=16, + kernel_size=8, + stride=4, + padding=2 + ) + self.conv2 = nn.Conv2d( + in_channels=16, + out_channels=32, + kernel_size=4, + stride=2, + padding=1 + ) + self.fc1 = nn.Linear( + in_features=3200, + out_features=256, + ) + self.fc2 = nn.Linear( + in_features=256, + out_features=n_actions, + ) + + # Activation Functions + self.relu = nn.ReLU() + + def flatten(self, x): + batch_size = x.size()[0] + x = x.view(batch_size, -1) + return x + + def forward(self, x): + + # Forward pass + x = self.relu(self.conv1(x)) # In: (80, 80, 4) Out: (20, 20, 16) + x = self.relu(self.conv2(x)) # In: (20, 20, 16) Out: (10, 10, 32) + x = self.flatten(x) # In: (10, 10, 32) Out: (3200,) + x = self.relu(self.fc1(x)) # In: (3200,) Out: (256,) + x = self.fc2(x) # In: (256,) Out: (4,) + + return x + +class ReplayBuffer: + def __init__(self, capacity): + self.capacity = capacity # 经验回放的容量 + self.buffer = [] # 缓冲区 + self.position = 0 + + def push(self, state, action, reward, next_state, done): + ''' 缓冲区是一个队列，容量超出时去掉开始存入的转移(transition) + ''' + if len(self.buffer) < self.capacity: + self.buffer.append(None) + self.buffer[self.position] = (state, action, reward, next_state, done) + self.position = (self.position + 1) % self.capacity + + def sample(self, batch_size): + batch = random.sample(self.buffer, batch_size) # 随机采出小批量转移 + state, action, reward, next_state, done = zip(*batch) # 解压成状态，动作等 + return state, action, reward, next_state, done + + def __len__(self): + ''' 返回当前存储的量 + ''' + return len(self.buffer) + +class DQN: + def __init__(self, n_states, n_actions, cfg): + + self.n_actions = n_actions # 总的动作个数 + self.device = cfg.device # 设备，cpu或gpu等 + self.gamma = cfg.gamma # 奖励的折扣因子 + # e-greedy策略相关参数 + self.frame_idx = 0 # 用于epsilon的衰减计数 + self.epsilon = lambda frame_idx: cfg.epsilon_end + \ + (cfg.epsilon_start - cfg.epsilon_end) * \ + math.exp(-1. * frame_idx / cfg.epsilon_decay) + self.batch_size = cfg.batch_size + self.policy_net = CNN(n_states, n_actions).to(self.device) + self.target_net = CNN(n_states, n_actions).to(self.device) + for target_param, param in zip(self.target_net.parameters(),self.policy_net.parameters()): # 复制参数到目标网路targe_net + target_param.data.copy_(param.data) + self.optimizer = optim.Adam(self.policy_net.parameters(), lr=cfg.lr) # 优化器 + self.memory = ReplayBuffer(cfg.memory_capacity) # 经验回放 + + def choose_action(self, state): + ''' 选择动作 + ''' + self.frame_idx += 1 + if random.random() > self.epsilon(self.frame_idx): + with torch.no_grad(): + state = torch.tensor([state], device=self.device, dtype=torch.float32) + q_values = self.policy_net(state) + action = q_values.max(1)[1].item() # 选择Q值最大的动作 + else: + action = random.randrange(self.n_actions) + return action + def update(self): + if len(self.memory) < self.batch_size: # 当memory中不满足一个批量时，不更新策略 + return + # 从经验回放中(replay memory)中随机采样一个批量的转移(transition) + state_batch, action_batch, reward_batch, next_state_batch, done_batch = self.memory.sample( + self.batch_size) + # 转为张量 + state_batch = torch.tensor(state_batch, device=self.device, dtype=torch.float) + action_batch = torch.tensor(action_batch, device=self.device).unsqueeze(1) + reward_batch = torch.tensor(reward_batch, device=self.device, dtype=torch.float) + next_state_batch = torch.tensor(next_state_batch, device=self.device, dtype=torch.float) + done_batch = torch.tensor(np.float32(done_batch), device=self.device) + q_values = self.policy_net(state_batch).gather(dim=1, index=action_batch) # 计算当前状态(s_t,a)对应的Q(s_t, a) + next_q_values = self.target_net(next_state_batch).max(1)[0].detach() # 计算下一时刻的状态(s_t_,a)对应的Q值 + # 计算期望的Q值，对于终止状态，此时done_batch[0]=1, 对应的expected_q_value等于reward + expected_q_values = reward_batch + self.gamma * next_q_values * (1-done_batch) + loss = nn.MSELoss()(q_values, expected_q_values.unsqueeze(1)) # 计算均方根损失 + # 优化更新模型 + self.optimizer.zero_grad() + loss.backward() + for param in self.policy_net.parameters(): # clip防止梯度爆炸 + param.grad.data.clamp_(-1, 1) + self.optimizer.step() + + def save(self, path): + torch.save(self.target_net.state_dict(), path+'dqn_checkpoint.pth') + + def load(self, path): + self.target_net.load_state_dict(torch.load(path+'dqn_checkpoint.pth')) + for target_param, param in zip(self.target_net.parameters(), self.policy_net.parameters()): + param.data.copy_(target_param.data) \ No newline at end of file diff --git a/codes/DQN/outputs/CartPole-v0/20220302-111332/models/dqn_checkpoint.pth b/codes/DQN/outputs/CartPole-v0/20220302-111332/models/dqn_checkpoint.pth new file mode 100644 index 0000000..6eb0130 Binary files /dev/null and b/codes/DQN/outputs/CartPole-v0/20220302-111332/models/dqn_checkpoint.pth differ diff --git a/codes/PPO/outputs/CartPole-v0/20211117-184614/results/eval_ma_rewards.npy b/codes/DQN/outputs/CartPole-v0/20220302-111332/results/test_ma_rewards.npy similarity index 100% rename from codes/PPO/outputs/CartPole-v0/20211117-184614/results/eval_ma_rewards.npy rename to codes/DQN/outputs/CartPole-v0/20220302-111332/results/test_ma_rewards.npy diff --git a/codes/PPO/outputs/CartPole-v0/20211117-184614/results/eval_rewards.npy b/codes/DQN/outputs/CartPole-v0/20220302-111332/results/test_rewards.npy similarity index 100% rename from codes/PPO/outputs/CartPole-v0/20211117-184614/results/eval_rewards.npy rename to codes/DQN/outputs/CartPole-v0/20220302-111332/results/test_rewards.npy diff --git a/codes/DQN/outputs/CartPole-v0/20220302-111332/results/test_rewards_curve.png b/codes/DQN/outputs/CartPole-v0/20220302-111332/results/test_rewards_curve.png new file mode 100644 index 0000000..76f8a18 Binary files /dev/null and b/codes/DQN/outputs/CartPole-v0/20220302-111332/results/test_rewards_curve.png differ diff --git a/codes/Sarsa/outputs/CliffWalking-v0/20220424-221748/results/test_rewards.npy b/codes/DQN/outputs/CartPole-v0/20220302-111332/results/test_steps.npy similarity index 61% rename from codes/Sarsa/outputs/CliffWalking-v0/20220424-221748/results/test_rewards.npy rename to codes/DQN/outputs/CartPole-v0/20220302-111332/results/test_steps.npy index 068a9de..db9c3fd 100644 Binary files a/codes/Sarsa/outputs/CliffWalking-v0/20220424-221748/results/test_rewards.npy and b/codes/DQN/outputs/CartPole-v0/20220302-111332/results/test_steps.npy differ diff --git a/codes/DQN/outputs/CartPole-v0/20220302-111332/results/train_ma_rewards.npy b/codes/DQN/outputs/CartPole-v0/20220302-111332/results/train_ma_rewards.npy new file mode 100644 index 0000000..d43b263 Binary files /dev/null and b/codes/DQN/outputs/CartPole-v0/20220302-111332/results/train_ma_rewards.npy differ diff --git a/codes/DQN/outputs/CartPole-v0/20220302-111332/results/train_rewards.npy b/codes/DQN/outputs/CartPole-v0/20220302-111332/results/train_rewards.npy new file mode 100644 index 0000000..303e570 Binary files /dev/null and b/codes/DQN/outputs/CartPole-v0/20220302-111332/results/train_rewards.npy differ diff --git a/codes/DQN/outputs/CartPole-v0/20220302-111332/results/train_rewards_curve.png b/codes/DQN/outputs/CartPole-v0/20220302-111332/results/train_rewards_curve.png new file mode 100644 index 0000000..012be04 Binary files /dev/null and b/codes/DQN/outputs/CartPole-v0/20220302-111332/results/train_rewards_curve.png differ diff --git a/codes/DQN/outputs/CartPole-v0/20220302-111332/results/train_steps.npy b/codes/DQN/outputs/CartPole-v0/20220302-111332/results/train_steps.npy new file mode 100644 index 0000000..3d25f8f Binary files /dev/null and b/codes/DQN/outputs/CartPole-v0/20220302-111332/results/train_steps.npy differ diff --git a/codes/DQN/task0.py b/codes/DQN/task0.py index c7cd5da..49a97a4 100644 --- a/codes/DQN/task0.py +++ b/codes/DQN/task0.py @@ -1,5 +1,7 @@ import sys import os +import torch.nn as nn +import torch.nn.functional as F curr_path = os.path.dirname(os.path.abspath(__file__)) # 当前文件所在绝对路径 parent_path = os.path.dirname(curr_path) # 父路径 sys.path.append(parent_path) # 添加路径到系统路径 @@ -8,26 +10,42 @@ import gym import torch import datetime import numpy as np -from common.utils import save_results, make_dir +from common.utils import save_results_1, make_dir from common.utils import plot_rewards -from DQN.dqn import DQN +from dqn import DQN curr_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") # 获取当前时间 +class MLP(nn.Module): + def __init__(self, n_states,n_actions,hidden_dim=128): + """ 初始化q网络，为全连接网络 + n_states: 输入的特征数即环境的状态维度 + n_actions: 输出的动作维度 + """ + super(MLP, self).__init__() + self.fc1 = nn.Linear(n_states, hidden_dim) # 输入层 + self.fc2 = nn.Linear(hidden_dim,hidden_dim) # 隐藏层 + self.fc3 = nn.Linear(hidden_dim, n_actions) # 输出层 + + def forward(self, x): + # 各层对应的激活函数 + x = F.relu(self.fc1(x)) + x = F.relu(self.fc2(x)) + return self.fc3(x) class Config: '''超参数 ''' def __init__(self): - ################################## 环境超参数 ################################### - self.algo_name = 'DQN' # 算法名称 - self.env_name = 'CartPole-v0' # 环境名称 + ############################### hyperparameters ################################ + self.algo_name = 'DQN' # algorithm name + self.env_name = 'CartPole-v0' # environment name self.device = torch.device( - "cuda" if torch.cuda.is_available() else "cpu") # 检测GPUgjgjlkhfsf风刀霜的撒发十 + "cuda" if torch.cuda.is_available() else "cpu") # check GPU self.seed = 10 # 随机种子，置0则不设置随机种子 self.train_eps = 200 # 训练的回合数 - self.test_eps = 30 # 测试的回合数 + self.test_eps = 20 # 测试的回合数 ################################################################################ ################################## 算法超参数 ################################### @@ -41,8 +59,8 @@ class Config: self.target_update = 4 # 目标网络的更新频率 self.hidden_dim = 256 # 网络隐藏层 ################################################################################ - - ################################# 保存结果相关参数 ############################## + + ################################# 保存结果相关参数 ################################ self.result_path = curr_path + "/outputs/" + self.env_name + \ '/' + curr_time + '/results/' # 保存结果的路径 self.model_path = curr_path + "/outputs/" + self.env_name + \ @@ -55,9 +73,11 @@ def env_agent_config(cfg): ''' 创建环境和智能体 ''' env = gym.make(cfg.env_name) # 创建环境 - state_dim = env.observation_space.shape[0] # 状态维度 - action_dim = env.action_space.n # 动作维度 - agent = DQN(state_dim, action_dim, cfg) # 创建智能体 + n_states = env.observation_space.shape[0] # 状态维度 + n_actions = env.action_space.n # 动作维度 + print(f"n states: {n_states}, n actions: {n_actions}") + model = MLP(n_states,n_actions) + agent = DQN(n_actions, model, cfg) # 创建智能体 if cfg.seed !=0: # 设置随机种子 torch.manual_seed(cfg.seed) env.seed(cfg.seed) @@ -72,10 +92,13 @@ def train(cfg, env, agent): print(f'环境：{cfg.env_name}, 算法：{cfg.algo_name}, 设备：{cfg.device}') rewards = [] # 记录所有回合的奖励 ma_rewards = [] # 记录所有回合的滑动平均奖励 + steps = [] for i_ep in range(cfg.train_eps): ep_reward = 0 # 记录一回合内的奖励 + ep_step = 0 state = env.reset() # 重置环境，返回初始状态 while True: + ep_step += 1 action = agent.choose_action(state) # 选择动作 next_state, reward, done, _ = env.step(action) # 更新环境，返回transition agent.memory.push(state, action, reward, @@ -87,16 +110,18 @@ def train(cfg, env, agent): break if (i_ep + 1) % cfg.target_update == 0: # 智能体目标网络更新 agent.target_net.load_state_dict(agent.policy_net.state_dict()) + steps.append(ep_step) rewards.append(ep_reward) if ma_rewards: ma_rewards.append(0.9 * ma_rewards[-1] + 0.1 * ep_reward) else: ma_rewards.append(ep_reward) - if (i_ep + 1) % 10 == 0: - print('回合：{}/{}, 奖励：{}'.format(i_ep + 1, cfg.train_eps, ep_reward)) - print('完成训练！') + if (i_ep + 1) % 1 == 0: + print(f'Episode：{i_ep+1}/{cfg.test_eps}, Reward:{ep_reward:.2f}, Step:{ep_step:.2f} Epislon:{agent.epsilon(agent.frame_idx):.3f}') + print('Finish training!') env.close() - return rewards, ma_rewards + res_dic = {'rewards':rewards,'ma_rewards':ma_rewards,'steps':steps} + return res_dic def test(cfg, env, agent): @@ -108,41 +133,45 @@ def test(cfg, env, agent): ################################################################################ rewards = [] # 记录所有回合的奖励 ma_rewards = [] # 记录所有回合的滑动平均奖励 + steps = [] for i_ep in range(cfg.test_eps): ep_reward = 0 # 记录一回合内的奖励 + ep_step = 0 state = env.reset() # 重置环境，返回初始状态 while True: + ep_step+=1 action = agent.choose_action(state) # 选择动作 next_state, reward, done, _ = env.step(action) # 更新环境，返回transition state = next_state # 更新下一个状态 ep_reward += reward # 累加奖励 if done: break + steps.append(ep_step) rewards.append(ep_reward) if ma_rewards: ma_rewards.append(ma_rewards[-1] * 0.9 + ep_reward * 0.1) else: ma_rewards.append(ep_reward) - print(f"回合：{i_ep+1}/{cfg.test_eps}，奖励：{ep_reward:.1f}") + print(f'Episode：{i_ep+1}/{cfg.train_eps}, Reward:{ep_reward:.2f}, Step:{ep_step:.2f}') print('完成测试！') env.close() - return rewards, ma_rewards + return {'rewards':rewards,'ma_rewards':ma_rewards,'steps':steps} if __name__ == "__main__": cfg = Config() # 训练 env, agent = env_agent_config(cfg) - rewards, ma_rewards = train(cfg, env, agent) + res_dic = train(cfg, env, agent) make_dir(cfg.result_path, cfg.model_path) # 创建保存结果和模型路径的文件夹 agent.save(path=cfg.model_path) # 保存模型 - save_results(rewards, ma_rewards, tag='train', + save_results_1(res_dic, tag='train', path=cfg.result_path) # 保存结果 - plot_rewards(rewards, ma_rewards, cfg, tag="train") # 画出结果 + plot_rewards(res_dic['rewards'], res_dic['ma_rewards'], cfg, tag="train") # 画出结果 # 测试 env, agent = env_agent_config(cfg) agent.load(path=cfg.model_path) # 导入模型 - rewards, ma_rewards = test(cfg, env, agent) - save_results(rewards, ma_rewards, tag='test', + res_dic = test(cfg, env, agent) + save_results_1(res_dic, tag='test', path=cfg.result_path) # 保存结果 - plot_rewards(rewards, ma_rewards, cfg, tag="test") # 画出结果 + plot_rewards(res_dic['rewards'], res_dic['ma_rewards'],cfg, tag="test") # 画出结果 diff --git a/codes/DQN/task1.py b/codes/DQN/task1.py index 078aa4c..75bff3a 100644 --- a/codes/DQN/task1.py +++ b/codes/DQN/task1.py @@ -5,7 +5,7 @@ Author: JiangJi Email: johnjim0816@gmail.com Date: 2021-12-22 11:14:17 LastEditor: JiangJi -LastEditTime: 2021-12-22 11:40:44 +LastEditTime: 2022-02-10 06:17:41 Discription: 使用 Nature DQN 训练 CartPole-v1 ''' import sys @@ -19,7 +19,7 @@ import torch import datetime from common.utils import save_results, make_dir from common.utils import plot_rewards, plot_rewards_cn -from DQN.dqn import DQN +from dqn import DQN curr_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") # 获取当前时间 algo_name = "DQN" # 算法名称 @@ -66,9 +66,9 @@ def env_agent_config(cfg, seed=1): ''' env = gym.make(cfg.env_name) # 创建环境 env.seed(seed) # 设置随机种子 - state_dim = env.observation_space.shape[0] # 状态维度 - action_dim = env.action_space.n # 动作维度 - agent = DQN(state_dim, action_dim, cfg) # 创建智能体 + n_states = env.observation_space.shape[0] # 状态维度 + n_actions = env.action_space.n # 动作维度 + agent = DQN(n_states, n_actions, cfg) # 创建智能体 return env, agent def train(cfg, env, agent): diff --git a/codes/DQN/task2.py b/codes/DQN/task2.py index 16571b2..9e0f8c2 100644 --- a/codes/DQN/task2.py +++ b/codes/DQN/task2.py @@ -5,7 +5,7 @@ Author: JiangJi Email: johnjim0816@gmail.com Date: 2021-12-22 11:14:17 LastEditor: JiangJi -LastEditTime: 2021-12-22 15:27:48 +LastEditTime: 2022-02-10 06:17:46 Discription: 使用 DQN-cnn 训练 PongNoFrameskip-v4 ''' import sys @@ -20,7 +20,7 @@ import datetime from common.utils import save_results, make_dir from common.utils import plot_rewards, plot_rewards_cn from common.atari_wrappers import make_atari, wrap_deepmind -from DQN.dqn import DQN +from dqn import DQN curr_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") # 获取当前时间 algo_name = 'DQN-cnn' # 算法名称 @@ -68,9 +68,9 @@ def env_agent_config(cfg, seed=1): # env = wrap_deepmind(env) # env = wrap_pytorch(env) env.seed(seed) # 设置随机种子 - state_dim = env.observation_space.shape[0] # 状态维度 - action_dim = env.action_space.n # 动作维度 - agent = DQN(state_dim, action_dim, cfg) # 创建智能体 + n_states = env.observation_space.shape[0] # 状态维度 + n_actions = env.action_space.n # 动作维度 + agent = DQN(n_states, n_actions, cfg) # 创建智能体 return env, agent def train(cfg, env, agent): diff --git a/codes/DQN/task4.py b/codes/DQN/task4.py new file mode 100644 index 0000000..436b36b --- /dev/null +++ b/codes/DQN/task4.py @@ -0,0 +1,180 @@ +import sys +import os +import torch.nn as nn +import torch.nn.functional as F +curr_path = os.path.dirname(os.path.abspath(__file__)) # 当前文件所在绝对路径 +parent_path = os.path.dirname(curr_path) # 父路径 +sys.path.append(parent_path) # 添加路径到系统路径 + +import gym +import torch +import datetime +import numpy as np +from common.utils import save_results_1, make_dir +from common.utils import plot_rewards +from dqn_1 import DQN + +curr_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") # 获取当前时间 + +class MLP(nn.Module): + def __init__(self, n_states,n_actions,hidden_dim=256): + """ 初始化q网络，为全连接网络 + n_states: 输入的特征数即环境的状态维度 + n_actions: 输出的动作维度 + """ + super(MLP, self).__init__() + self.fc1 = nn.Linear(n_states, hidden_dim) # 输入层 + self.fc2 = nn.Linear(hidden_dim,hidden_dim) # 隐藏层 + self.fc3 = nn.Linear(hidden_dim,hidden_dim) # 隐藏层 + self.fc4 = nn.Linear(hidden_dim, n_actions) # 输出层 + + def forward(self, x): + # 各层对应的激活函数 + x = F.relu(self.fc1(x)) + x = F.relu(self.fc2(x)) + x = F.relu(self.fc3(x)) + return self.fc4(x) + +class Config: + '''超参数 + ''' + + def __init__(self): + ################################## 环境超参数 ################################### + self.algo_name = 'DQN' # 算法名称 + # self.env_name = 'Breakout-ram-v0' # 环境名称 + self.env_name = 'ALE/Pong-ram-v5' + self.device = torch.device( + "cuda" if torch.cuda.is_available() else "cpu") # 检测GPUgjgjlkhfsf风刀霜的撒发十 + self.seed = 10 # 随机种子，置0则不设置随机种子 + self.train_eps = 5 # 训练的回合数 + self.test_eps = 30 # 测试的回合数 + ################################################################################ + + ################################## 算法超参数 ################################### + self.gamma = 0.99 # 强化学习中的折扣因子 + self.epsilon_start = 0.95 # e-greedy策略中初始epsilon + self.epsilon_end = 0.01 # e-greedy策略中的终止epsilon + self.epsilon_decay = 500000 # e-greedy策略中epsilon的衰减率 + self.lr = 0.00025 # 学习率 + self.memory_capacity = int(5e4) # 经验回放的容量 + self.batch_size = 32 # mini-batch SGD中的批量大小 + self.target_update = 4 # 目标网络的更新频率 + self.hidden_dim = 512 # 网络隐藏层 + ################################################################################ + + ################################# 保存结果相关参数 ################################ + self.result_path = curr_path + "/outputs/" + self.env_name + \ + '/' + curr_time + '/results/' # 保存结果的路径 + self.model_path = curr_path + "/outputs/" + self.env_name + \ + '/' + curr_time + '/models/' # 保存模型的路径 + self.save = True # 是否保存图片 + ################################################################################ + + +def env_agent_config(cfg): + ''' 创建环境和智能体 + ''' + env = gym.make(cfg.env_name) # 创建环境 + n_states = env.observation_space.shape[0] # 状态维度 + n_actions = env.action_space.n # 动作维度 + print(f"n states: {n_states}, n actions: {n_actions}") + model = MLP(n_states,n_actions) + agent = DQN(n_states, n_actions, model, cfg) # 创建智能体 + if cfg.seed !=0: # 设置随机种子 + torch.manual_seed(cfg.seed) + env.seed(cfg.seed) + np.random.seed(cfg.seed) + return env, agent + + +def train(cfg, env, agent): + ''' 训练 + ''' + print('开始训练!') + print(f'环境：{cfg.env_name}, 算法：{cfg.algo_name}, 设备：{cfg.device}') + rewards = [] # 记录所有回合的奖励 + ma_rewards = [] # 记录所有回合的滑动平均奖励 + steps = [] + for i_ep in range(cfg.train_eps): + ep_reward = 0 # 记录一回合内的奖励 + state = env.reset() # 重置环境，返回初始状态 + ep_step = 0 + while True: + ep_step+=1 + action = agent.choose_action(state) # 选择动作 + next_state, reward, done, _ = env.step(action) # 更新环境，返回transition + agent.memory.push(state, action, reward, + next_state, done) # 保存transition + state = next_state # 更新下一个状态 + agent.update() # 更新智能体 + ep_reward += reward # 累加奖励 + if done: + break + if (i_ep + 1) % cfg.target_update == 0: # 智能体目标网络更新 + agent.target_net.load_state_dict(agent.policy_net.state_dict()) + steps.append(ep_step) + rewards.append(ep_reward) + if ma_rewards: + ma_rewards.append(0.9 * ma_rewards[-1] + 0.1 * ep_reward) + else: + ma_rewards.append(ep_reward) + if (i_ep + 1) % 1 == 0: + print(f'Episode：{i_ep+1}/{cfg.train_eps}, Reward:{ep_reward:.2f}, Epislon:{agent.epsilon(agent.frame_idx):.3f}') + print('完成训练！') + env.close() + res_dic = {'rewards':rewards,'ma_rewards':ma_rewards,'steps':steps} + return res_dic + + +def test(cfg, env, agent): + print('开始测试!') + print(f'环境：{cfg.env_name}, 算法：{cfg.algo_name}, 设备：{cfg.device}') + ############# 由于测试不需要使用epsilon-greedy策略，所以相应的值设置为0 ############### + cfg.epsilon_start = 0.0 # e-greedy策略中初始epsilon + cfg.epsilon_end = 0.0 # e-greedy策略中的终止epsilon + ################################################################################ + rewards = [] # 记录所有回合的奖励 + ma_rewards = [] # 记录所有回合的滑动平均奖励 + steps = [] + for i_ep in range(cfg.test_eps): + ep_reward = 0 # 记录一回合内的奖励 + ep_step = 0 + state = env.reset() # 重置环境，返回初始状态 + while True: + ep_step+=1 + action = agent.choose_action(state) # 选择动作 + next_state, reward, done, _ = env.step(action) # 更新环境，返回transition + state = next_state # 更新下一个状态 + ep_reward += reward # 累加奖励 + if done: + break + steps.append(ep_step) + rewards.append(ep_reward) + if ma_rewards: + ma_rewards.append(ma_rewards[-1] * 0.9 + ep_reward * 0.1) + else: + ma_rewards.append(ep_reward) + print(f"回合：{i_ep+1}/{cfg.test_eps}，奖励：{ep_reward:.1f}") + print('完成测试！') + env.close() + return {'rewards':rewards,'ma_rewards':ma_rewards,'steps':steps} + + +if __name__ == "__main__": + cfg = Config() + # 训练 + env, agent = env_agent_config(cfg) + res_dic = train(cfg, env, agent) + make_dir(cfg.result_path, cfg.model_path) # 创建保存结果和模型路径的文件夹 + agent.save(path=cfg.model_path) # 保存模型 + save_results_1(res_dic, tag='train', + path=cfg.result_path) # 保存结果 + plot_rewards(res_dic['rewards'], res_dic['ma_rewards'], cfg, tag="train") # 画出结果 + # 测试 + env, agent = env_agent_config(cfg) + agent.load(path=cfg.model_path) # 导入模型 + res_dic = test(cfg, env, agent) + save_results_1(res_dic, tag='test', + path=cfg.result_path) # 保存结果 + plot_rewards(res_dic['rewards'], res_dic['ma_rewards'],cfg, tag="test") # 画出结果 diff --git a/codes/DQN/task5.py b/codes/DQN/task5.py new file mode 100644 index 0000000..519a8f6 --- /dev/null +++ b/codes/DQN/task5.py @@ -0,0 +1,149 @@ +import sys +import os +curr_path = os.path.dirname(os.path.abspath(__file__)) # 当前文件所在绝对路径 +parent_path = os.path.dirname(curr_path) # 父路径 +sys.path.append(parent_path) # 添加路径到系统路径 + +import gym +import torch +import datetime +import numpy as np +from common.utils import save_results, make_dir +from common.utils import plot_rewards +from dqn import DQN + +curr_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") # 获取当前时间 + + +class Config: + '''超参数 + ''' + + def __init__(self): + ################################## 环境超参数 ################################### + self.algo_name = 'DQN' # 算法名称 + self.env_name = 'SpaceInvaders-ram-v0' # 环境名称 + self.device = torch.device( + "cuda" if torch.cuda.is_available() else "cpu") # 检测GPUgjgjlkhfsf风刀霜的撒发十 + self.seed = 10 # 随机种子，置0则不设置随机种子 + self.train_eps = 200 # 训练的回合数 + self.test_eps = 30 # 测试的回合数 + ################################################################################ + + ################################## 算法超参数 ################################### + self.gamma = 0.99 # 强化学习中的折扣因子 + self.epsilon_start = 0.95 # e-greedy策略中初始epsilon + self.epsilon_end = 0.01 # e-greedy策略中的终止epsilon + self.epsilon_decay = 20000 # e-greedy策略中epsilon的衰减率 + self.lr = 2e-4 # 学习率 + self.memory_capacity = int(1e5) # 经验回放的容量 + self.batch_size = 32 # mini-batch SGD中的批量大小 + self.target_update = 4 # 目标网络的更新频率 + self.hidden_dim = 512 # 网络隐藏层 + ################################################################################ + + ################################# 保存结果相关参数 ################################ + self.result_path = curr_path + "/outputs/" + self.env_name + \ + '/' + curr_time + '/results/' # 保存结果的路径 + self.model_path = curr_path + "/outputs/" + self.env_name + \ + '/' + curr_time + '/models/' # 保存模型的路径 + self.save = True # 是否保存图片 + ################################################################################ + + +def env_agent_config(cfg): + ''' 创建环境和智能体 + ''' + env = gym.make(cfg.env_name) # 创建环境 + n_states = env.observation_space.shape[0] # 状态维度 + n_actions = env.action_space.n # 动作维度 + print(f"n states: {n_states}, n actions: {n_actions}") + agent = DQN(n_states, n_actions, cfg) # 创建智能体 + if cfg.seed !=0: # 设置随机种子 + torch.manual_seed(cfg.seed) + env.seed(cfg.seed) + np.random.seed(cfg.seed) + return env, agent + + +def train(cfg, env, agent): + ''' 训练 + ''' + print('开始训练!') + print(f'环境：{cfg.env_name}, 算法：{cfg.algo_name}, 设备：{cfg.device}') + rewards = [] # 记录所有回合的奖励 + ma_rewards = [] # 记录所有回合的滑动平均奖励 + for i_ep in range(cfg.train_eps): + ep_reward = 0 # 记录一回合内的奖励 + state = env.reset() # 重置环境，返回初始状态 + while True: + action = agent.choose_action(state) # 选择动作 + next_state, reward, done, _ = env.step(action) # 更新环境，返回transition + agent.memory.push(state, action, reward, + next_state, done) # 保存transition + state = next_state # 更新下一个状态 + agent.update() # 更新智能体 + ep_reward += reward # 累加奖励 + if done: + break + if (i_ep + 1) % cfg.target_update == 0: # 智能体目标网络更新 + agent.target_net.load_state_dict(agent.policy_net.state_dict()) + rewards.append(ep_reward) + if ma_rewards: + ma_rewards.append(0.9 * ma_rewards[-1] + 0.1 * ep_reward) + else: + ma_rewards.append(ep_reward) + if (i_ep + 1) % 1 == 0: + print(f'Episode：{i_ep+1}/{cfg.train_eps}, Reward:{ep_reward:.2f}, Epislon:{agent.epsilon(agent.frame_idx):.3f}') + print('完成训练！') + env.close() + return rewards, ma_rewards + + +def test(cfg, env, agent): + print('开始测试!') + print(f'环境：{cfg.env_name}, 算法：{cfg.algo_name}, 设备：{cfg.device}') + ############# 由于测试不需要使用epsilon-greedy策略，所以相应的值设置为0 ############### + cfg.epsilon_start = 0.0 # e-greedy策略中初始epsilon + cfg.epsilon_end = 0.0 # e-greedy策略中的终止epsilon + ################################################################################ + rewards = [] # 记录所有回合的奖励 + ma_rewards = [] # 记录所有回合的滑动平均奖励 + for i_ep in range(cfg.test_eps): + ep_reward = 0 # 记录一回合内的奖励 + state = env.reset() # 重置环境，返回初始状态 + while True: + action = agent.choose_action(state) # 选择动作 + next_state, reward, done, _ = env.step(action) # 更新环境，返回transition + state = next_state # 更新下一个状态 + ep_reward += reward # 累加奖励 + if done: + break + rewards.append(ep_reward) + if ma_rewards: + ma_rewards.append(ma_rewards[-1] * 0.9 + ep_reward * 0.1) + else: + ma_rewards.append(ep_reward) + print(f"回合：{i_ep+1}/{cfg.test_eps}，奖励：{ep_reward:.1f}") + print('完成测试！') + env.close() + return rewards, ma_rewards + + +if __name__ == "__main__": + cfg = Config() + # 训练 + env, agent = env_agent_config(cfg) + rewards, ma_rewards = train(cfg, env, agent) + make_dir(cfg.result_path, cfg.model_path) # 创建保存结果和模型路径的文件夹 + agent.save(path=cfg.model_path) # 保存模型 + save_results(rewards, ma_rewards, tag='train', + path=cfg.result_path) # 保存结果 + plot_rewards(rewards, ma_rewards, cfg, tag="train") # 画出结果 + # 测试 + env, agent = env_agent_config(cfg) + agent.load(path=cfg.model_path) # 导入模型 + rewards, ma_rewards = test(cfg, env, agent) + save_results(rewards, ma_rewards, tag='test', + path=cfg.result_path) # 保存结果 + plot_rewards(rewards, ma_rewards, cfg, tag="test") # 画出结果 diff --git a/codes/DQN/test copy.py b/codes/DQN/test copy.py new file mode 100644 index 0000000..f4b0b04 --- /dev/null +++ b/codes/DQN/test copy.py @@ -0,0 +1,184 @@ +import random +import numpy as np +import pandas as pd +import tensorflow as tf +import os +import gym +import time +from collections import deque +from tensorflow.keras import optimizers +from keras.models import Sequential +from keras.layers import Dense, Dropout +from keras.layers import Activation, Flatten, Conv1D, MaxPooling1D,Reshape +import matplotlib.pyplot as plt + +class DQN: + def __init__(self, env): + self.env = env + self.memory = deque(maxlen=400000) + self.gamma = 0.99 + self.epsilon = 1.0 + self.epsilon_min = 0.01 + self.epsilon_decay = self.epsilon_min / 500000 + + self.batch_size = 32 + self.train_start = 1000 + self.state_size = self.env.observation_space.shape[0]*4 + self.action_size = self.env.action_space.n + self.learning_rate = 0.00025 + + self.evaluation_model = self.create_model() + self.target_model = self.create_model() + + def create_model(self): + model = Sequential() + model.add(Dense(128*2, input_dim=self.state_size,activation='relu')) + model.add(Dense(128*2, activation='relu')) + model.add(Dense(128*2, activation='relu')) + model.add(Dense(self.env.action_space.n, activation='linear')) + model.compile(loss='mean_squared_error', optimizer=optimizers.RMSprop(lr=self.learning_rate,decay=0.99,epsilon=1e-6)) + return model + + def choose_action(self, state, steps): + if steps > 50000: + if self.epsilon > self.epsilon_min: + self.epsilon -= self.epsilon_decay + if np.random.random() < self.epsilon: + return self.env.action_space.sample() + return np.argmax(self.evaluation_model.predict(state)[0]) + + def remember(self, cur_state, action, reward, new_state, done): + if not hasattr(self, 'memory_counter'): + self.memory_counter = 0 + + transition = (cur_state, action, reward, new_state, done) + self.memory.extend([transition]) + + self.memory_counter += 1 + + def replay(self): + if len(self.memory) < self.train_start: + return + + mini_batch = random.sample(self.memory, self.batch_size) + + update_input = np.zeros((self.batch_size, self.state_size)) + update_target = np.zeros((self.batch_size, self.action_size)) + + for i in range(self.batch_size): + state, action, reward, new_state, done = mini_batch[i] + target = self.evaluation_model.predict(state)[0] + + if done: + target[action] = reward + else: + target[action] = reward + self.gamma * np.amax(self.target_model.predict(new_state)[0]) + + update_input[i] = state + update_target[i] = target + + self.evaluation_model.fit(update_input, update_target, batch_size=self.batch_size, epochs=1, verbose=0) + + def target_train(self): + self.target_model.set_weights(self.evaluation_model.get_weights()) + return + + def visualize(self, reward, episode): + plt.plot(episode, reward, 'ob-') + plt.title('Average reward each 100 episode') + plt.ylabel('Reward') + plt.xlabel('Episodes') + plt.grid() + plt.show() + + def transform(self,state): + if state.shape[1]==512: + return state + a=[np.binary_repr(x,width=8) for x in state[0]] + res=[] + for x in a: + res.extend([x[:2],x[2:4],x[4:6],x[6:]]) + res=[int(x,2) for x in res] + return np.array(res) + +os.environ["CUDA_VISIBLE_DEVICES"] = "0" +def main(): + # env = gym.make('Breakout-ram-v0') + env = gym.make('Breakout-ram-v0') + env = env.unwrapped + + print(env.action_space) + print(env.observation_space.shape[0]) + print(env.observation_space.high) + print(env.observation_space.low) + + #print(env.observation_space.shape) + + + episodes = 5000 + trial_len = 10000 + + tmp_reward=0 + sum_rewards = 0 + n_success = 0 + total_steps = 0 + + graph_reward = [] + graph_episodes = [] + time_record = [] + + dqn_agent = DQN(env=env) + for i_episode in range(episodes): + start_time = time.time() + total_reward = 0 + cur_state = env.reset().reshape(1,128) + cur_state=dqn_agent.transform(cur_state).reshape(1,128*4)/4 + i_step=0 + for step in range(trial_len): + #env.render() + i_step+=1 + action = dqn_agent.choose_action(cur_state, total_steps) + new_state, reward, done, _ = env.step(action) + new_state = new_state.reshape(1, 128) + new_state = dqn_agent.transform(new_state).reshape(1,128*4)/4 + total_reward += reward + sum_rewards += reward + tmp_reward += reward + if reward>0: #Testing whether it is good. + reward=1 + + dqn_agent.remember(cur_state, action, reward, new_state, done) + if total_steps > 10000: + if total_steps%4 == 0: + dqn_agent.replay() + if total_steps%5000 == 0: + dqn_agent.target_train() + + cur_state = new_state + total_steps += 1 + if done: + env.reset() + break + if (i_episode+1) % 100 == 0: + graph_reward.append(sum_rewards/100) + graph_episodes.append(i_episode+1) + sum_rewards = 0 + print("Episode ",i_episode+1," Reward: ") + print(graph_reward[-1]) + end_time = time.time() + time_record.append(end_time-start_time) + print("NOW in episode: " + str(i_episode)) + print("Time cost: " + str(end_time-start_time)) + print("Reward: ",tmp_reward) + print("Step:", i_step) + tmp_reward=0 + print("Reward: ") + print(graph_reward) + print("Episode: ") + print(graph_episodes) + print("Average_time: ") + print(sum(time_record)/5000) + dqn_agent.visualize(graph_reward, graph_episodes) + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/codes/Docs/使用DDPG解决倒立摆问题.md b/codes/Docs/使用DDPG解决倒立摆问题.md index fd625f5..d0c8505 100644 --- a/codes/Docs/使用DDPG解决倒立摆问题.md +++ b/codes/Docs/使用DDPG解决倒立摆问题.md @@ -90,15 +90,15 @@ class OUNoise(object): self.max_sigma = max_sigma self.min_sigma = min_sigma self.decay_period = decay_period - self.action_dim = action_space.shape[0] + self.n_actions = action_space.shape[0] self.low = action_space.low self.high = action_space.high self.reset() def reset(self): - self.obs = np.ones(self.action_dim) * self.mu + self.obs = np.ones(self.n_actions) * self.mu def evolve_obs(self): x = self.obs - dx = self.theta * (self.mu - x) + self.sigma * np.random.randn(self.action_dim) + dx = self.theta * (self.mu - x) + self.sigma * np.random.randn(self.n_actions) self.obs = x + dx return self.obs def get_action(self, action, t=0): diff --git a/codes/Docs/使用DQN解决推车杆问题.md b/codes/Docs/使用DQN解决推车杆问题.md index 393c52d..a5f5a58 100644 --- a/codes/Docs/使用DQN解决推车杆问题.md +++ b/codes/Docs/使用DQN解决推车杆问题.md @@ -14,10 +14,10 @@ CartPole-v0是一个经典的入门环境，如下图，它通过向左(动作=0 import gym env = gym.make('CartPole-v0') # 建立环境 env.seed(1) # 随机种子 -state_dim = env.observation_space.shape[0] # 状态维度 -action_dim = env.action_space.n # 动作维度 +n_states = env.observation_space.shape[0] # 状态维度 +n_actions = env.action_space.n # 动作维度 state = env.reset() # 初始化环境 -print(f"状态维度：{state_dim}，动作维度：{action_dim}") +print(f"状态维度：{n_states}，动作维度：{n_actions}") print(f"初始状态：{state}") ``` @@ -157,7 +157,7 @@ def choose_action(self, state): q_values = self.policy_net(state) action = q_values.max(1)[1].item() # 选择Q值最大的动作 else: - action = random.randrange(self.action_dim) + action = random.randrange(self.n_actions) ``` 可以看到跟Q学习算法其实是一样的，都是用的$\epsilon-greedy$策略，只是使用神经网络的话我们需要通过Torch或者Tensorflow工具来处理相应的数据。 diff --git a/codes/Docs/使用Q-learning解决悬崖寻路问题.md b/codes/Docs/使用Q-learning解决悬崖寻路问题.md index 44e5b6c..a57d044 100644 --- a/codes/Docs/使用Q-learning解决悬崖寻路问题.md +++ b/codes/Docs/使用Q-learning解决悬崖寻路问题.md @@ -30,9 +30,9 @@ env = CliffWalkingWapper(env) # 装饰环境 这里我们在程序中使用了一个装饰器重新定义环境，但不影响对环境的理解，感兴趣的同学具体看相关代码。可以由于gym环境封装得比较好，所以我们想要使用这个环境只需要使用gym.make命令输入函数名即可，然后我们可以查看环境的状态和动作维度目： ```python -state_dim = env.observation_space.n # 状态维度 -action_dim = env.action_space.n # 动作维度 -print(f"状态维度：{state_dim}，动作维度：{action_dim}") +n_states = env.observation_space.n # 状态维度 +n_actions = env.action_space.n # 动作维度 +print(f"状态维度：{n_states}，动作维度：{n_actions}") ``` 打印出来的结果如下： @@ -72,9 +72,9 @@ print(state) env = gym.make('CliffWalking-v0') # 定义环境 env = CliffWalkingWapper(env) # 装饰环境 env.seed(1) # 设置随机种子 -state_dim = env.observation_space.n # 状态维度 -action_dim = env.action_space.n # 动作维度 -agent = QLearning(state_dim,action_dim,cfg) # cfg存储算法相关参数 +n_states = env.observation_space.n # 状态维度 +n_actions = env.action_space.n # 动作维度 +agent = QLearning(n_states,n_actions,cfg) # cfg存储算法相关参数 for i_ep in range(cfg.train_eps): # cfg.train_eps表示最大训练的回合数 ep_reward = 0 # 记录每个回合的奖励 state = env.reset() # 重置环境 @@ -126,7 +126,7 @@ def choose_action(self, state): if np.random.uniform(0, 1) > self.epsilon: action = np.argmax(self.Q_table[str(state)]) # 选择Q(s,a)最大对应的动作 else: - action = np.random.choice(self.action_dim) # 随机选择动作 + action = np.random.choice(self.n_actions) # 随机选择动作 return action ``` diff --git a/codes/DoubleDQN/double_dqn.py b/codes/DoubleDQN/double_dqn.py index e712edb..8dbdc52 100644 --- a/codes/DoubleDQN/double_dqn.py +++ b/codes/DoubleDQN/double_dqn.py @@ -46,15 +46,15 @@ class ReplayBuffer: return len(self.buffer) class MLP(nn.Module): - def __init__(self, state_dim,action_dim,hidden_dim=128): + def __init__(self, n_states,n_actions,hidden_dim=128): """ 初始化q网络，为全连接网络 - state_dim: 输入的特征数即环境的状态维度 - action_dim: 输出的动作维度 + n_states: 输入的特征数即环境的状态维度 + n_actions: 输出的动作维度 """ super(MLP, self).__init__() - self.fc1 = nn.Linear(state_dim, hidden_dim) # 输入层 + self.fc1 = nn.Linear(n_states, hidden_dim) # 输入层 self.fc2 = nn.Linear(hidden_dim,hidden_dim) # 隐藏层 - self.fc3 = nn.Linear(hidden_dim, action_dim) # 输出层 + self.fc3 = nn.Linear(hidden_dim, n_actions) # 输出层 def forward(self, x): # 各层对应的激活函数 @@ -63,8 +63,8 @@ class MLP(nn.Module): return self.fc3(x) class DoubleDQN: - def __init__(self, state_dim, action_dim, cfg): - self.action_dim = action_dim # 总的动作个数 + def __init__(self, n_states, n_actions, cfg): + self.n_actions = n_actions # 总的动作个数 self.device = cfg.device # 设备，cpu或gpu等 self.gamma = cfg.gamma # e-greedy策略相关参数 @@ -73,8 +73,8 @@ class DoubleDQN: self.epsilon_end = cfg.epsilon_end self.epsilon_decay = cfg.epsilon_decay self.batch_size = cfg.batch_size - self.policy_net = MLP(state_dim, action_dim,hidden_dim=cfg.hidden_dim).to(self.device) - self.target_net = MLP(state_dim, action_dim,hidden_dim=cfg.hidden_dim).to(self.device) + self.policy_net = MLP(n_states, n_actions,hidden_dim=cfg.hidden_dim).to(self.device) + self.target_net = MLP(n_states, n_actions,hidden_dim=cfg.hidden_dim).to(self.device) # target_net copy from policy_net for target_param, param in zip(self.target_net.parameters(), self.policy_net.parameters()): target_param.data.copy_(param.data) @@ -103,7 +103,7 @@ class DoubleDQN: # 所以tensor.max(1)[1]返回最大值对应的下标，即action action = q_value.max(1)[1].item() else: - action = random.randrange(self.action_dim) + action = random.randrange(self.n_actions) return action def update(self): diff --git a/codes/DoubleDQN/task0.py b/codes/DoubleDQN/task0.py index 7657a88..2f91e1e 100644 --- a/codes/DoubleDQN/task0.py +++ b/codes/DoubleDQN/task0.py @@ -59,9 +59,9 @@ class Config: def env_agent_config(cfg,seed=1): env = gym.make(cfg.env_name) env.seed(seed) - state_dim = env.observation_space.shape[0] - action_dim = env.action_space.n - agent = DoubleDQN(state_dim,action_dim,cfg) + n_states = env.observation_space.shape[0] + n_actions = env.action_space.n + agent = DoubleDQN(n_states,n_actions,cfg) return env,agent def train(cfg,env,agent): diff --git a/codes/DuelingDQN/task0_train.ipynb b/codes/DuelingDQN/task0_train.ipynb index 7e38218..efa485f 100644 --- a/codes/DuelingDQN/task0_train.ipynb +++ b/codes/DuelingDQN/task0_train.ipynb @@ -136,12 +136,12 @@ "outputs": [], "source": [ "class DuelingNet(nn.Module):\n", - " def __init__(self, state_dim, action_dim,hidden_size=128):\n", + " def __init__(self, n_states, n_actions,hidden_size=128):\n", " super(DuelingNet, self).__init__()\n", " \n", " # 隐藏层\n", " self.hidden = nn.Sequential(\n", - " nn.Linear(state_dim, hidden_size),\n", + " nn.Linear(n_states, hidden_size),\n", " nn.ReLU()\n", " )\n", " \n", @@ -149,7 +149,7 @@ " self.advantage = nn.Sequential(\n", " nn.Linear(hidden_size, hidden_size),\n", " nn.ReLU(),\n", - " nn.Linear(hidden_size, action_dim)\n", + " nn.Linear(hidden_size, n_actions)\n", " )\n", " \n", " # 价值函数\n", @@ -192,7 +192,7 @@ ], "source": [ "class DuelingDQN:\n", - " def __init__(self,state_dim,action_dim,cfg) -> None:\n", + " def __init__(self,n_states,n_actions,cfg) -> None:\n", " self.batch_size = cfg.batch_size\n", " self.device = cfg.device\n", " self.loss_history = [] # 记录loss的变化\n", @@ -200,8 +200,8 @@ " self.epsilon = lambda frame_idx: cfg.epsilon_end + \\\n", " (cfg.epsilon_start - cfg.epsilon_end) * \\\n", " math.exp(-1. * frame_idx / cfg.epsilon_decay)\n", - " self.policy_net = DuelingNet(state_dim, action_dim,hidden_dim=cfg.hidden_dim).to(self.device)\n", - " self.target_net = DuelingNet(state_dim, action_dim,hidden_dim=cfg.hidden_dim).to(self.device)\n", + " self.policy_net = DuelingNet(n_states, n_actions,hidden_dim=cfg.hidden_dim).to(self.device)\n", + " self.target_net = DuelingNet(n_states, n_actions,hidden_dim=cfg.hidden_dim).to(self.device)\n", " for target_param, param in zip(self.target_net.parameters(),self.policy_net.parameters()): # 复制参数到目标网络targe_net\n", " target_param.data.copy_(param.data)\n", " self.optimizer = optim.Adam(self.policy_net.parameters(), lr=cfg.lr) # 优化器\n", @@ -214,7 +214,7 @@ " q_values = self.policy_net(state)\n", " action = q_values.max(1)[1].item() # 选择Q值最大的动作\n", " else:\n", - " action = random.randrange(self.action_dim)\n", + " action = random.randrange(self.n_actions)\n", " return action\n", " def update(self):\n", " if len(self.memory) < self.batch_size: # 当memory中不满足一个批量时，不更新策略\n", diff --git a/codes/HierarchicalDQN/agent.py b/codes/HierarchicalDQN/agent.py index ce0cd1f..91428cc 100644 --- a/codes/HierarchicalDQN/agent.py +++ b/codes/HierarchicalDQN/agent.py @@ -57,16 +57,16 @@ class MLP(nn.Module): return self.fc3(x) class HierarchicalDQN: - def __init__(self,state_dim,action_dim,cfg): - self.state_dim = state_dim - self.action_dim = action_dim + def __init__(self,n_states,n_actions,cfg): + self.n_states = n_states + self.n_actions = n_actions self.gamma = cfg.gamma self.device = cfg.device self.batch_size = cfg.batch_size self.frame_idx = 0 # 用于epsilon的衰减计数 self.epsilon = lambda frame_idx: cfg.epsilon_end + (cfg.epsilon_start - cfg.epsilon_end ) * math.exp(-1. * frame_idx / cfg.epsilon_decay) - self.policy_net = MLP(2*state_dim, action_dim,cfg.hidden_dim).to(self.device) - self.meta_policy_net = MLP(state_dim, state_dim,cfg.hidden_dim).to(self.device) + self.policy_net = MLP(2*n_states, n_actions,cfg.hidden_dim).to(self.device) + self.meta_policy_net = MLP(n_states, n_states,cfg.hidden_dim).to(self.device) self.optimizer = optim.Adam(self.policy_net.parameters(),lr=cfg.lr) self.meta_optimizer = optim.Adam(self.meta_policy_net.parameters(),lr=cfg.lr) self.memory = ReplayBuffer(cfg.memory_capacity) @@ -76,7 +76,7 @@ class HierarchicalDQN: self.losses = [] self.meta_losses = [] def to_onehot(self,x): - oh = np.zeros(self.state_dim) + oh = np.zeros(self.n_states) oh[x - 1] = 1. return oh def set_goal(self,state): @@ -85,7 +85,7 @@ class HierarchicalDQN: state = torch.tensor(state, device=self.device, dtype=torch.float32).unsqueeze(0) goal = self.meta_policy_net(state).max(1)[1].item() else: - goal = random.randrange(self.state_dim) + goal = random.randrange(self.n_states) return goal def choose_action(self,state): self.frame_idx += 1 @@ -95,7 +95,7 @@ class HierarchicalDQN: q_value = self.policy_net(state) action = q_value.max(1)[1].item() else: - action = random.randrange(self.action_dim) + action = random.randrange(self.n_actions) return action def update(self): self.update_policy() diff --git a/codes/HierarchicalDQN/task0.py b/codes/HierarchicalDQN/task0.py index 3eceefd..b2cf312 100644 --- a/codes/HierarchicalDQN/task0.py +++ b/codes/HierarchicalDQN/task0.py @@ -63,9 +63,9 @@ class PlotConfig: def env_agent_config(cfg,seed=1): env = gym.make(cfg.env_name) env.seed(seed) - state_dim = env.observation_space.shape[0] - action_dim = env.action_space.n - agent = HierarchicalDQN(state_dim,action_dim,cfg) + n_states = env.observation_space.shape[0] + n_actions = env.action_space.n + agent = HierarchicalDQN(n_states,n_actions,cfg) return env,agent if __name__ == "__main__": diff --git a/codes/LICENSE b/codes/LICENSE deleted file mode 100644 index 673d927..0000000 --- a/codes/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2020 John Jim - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/codes/Logs.md b/codes/Logs.md deleted file mode 100644 index 7dc6497..0000000 --- a/codes/Logs.md +++ /dev/null @@ -1,7 +0,0 @@ -## 记录笔者更新的日志 - -**2021.12.28-1**：将```task.py```中的两个Config类合并为一个，并加以注释便于阅读，从DQN算法开始更新 - -**2021.12.22-3**：将```agent.py```更改为对应的算法名称，便于区分如```dqn```与```dqn_cnn```的情况 -**2021.12.22-2**：简化了代码结构，将原来的```train.py```和```task.py```等合并到```task.py```中 -**2021.12.22-1**：简化了代码结构，将原来的```model.py```和```memory.py```等合并到```agent.py```中，```plot.py```的内容合并到```common.utils.py```中 \ No newline at end of file diff --git a/codes/MonteCarlo/agent.py b/codes/MonteCarlo/agent.py index 44af71d..bfe6940 100644 --- a/codes/MonteCarlo/agent.py +++ b/codes/MonteCarlo/agent.py @@ -17,11 +17,11 @@ import dill class FisrtVisitMC: ''' On-Policy First-Visit MC Control ''' - def __init__(self,action_dim,cfg): - self.action_dim = action_dim + def __init__(self,n_actions,cfg): + self.n_actions = n_actions self.epsilon = cfg.epsilon self.gamma = cfg.gamma - self.Q_table = defaultdict(lambda: np.zeros(action_dim)) + self.Q_table = defaultdict(lambda: np.zeros(n_actions)) self.returns_sum = defaultdict(float) # sum of returns self.returns_count = defaultdict(float) @@ -29,11 +29,11 @@ class FisrtVisitMC: ''' e-greed policy ''' if state in self.Q_table.keys(): best_action = np.argmax(self.Q_table[state]) - action_probs = np.ones(self.action_dim, dtype=float) * self.epsilon / self.action_dim + action_probs = np.ones(self.n_actions, dtype=float) * self.epsilon / self.n_actions action_probs[best_action] += (1.0 - self.epsilon) action = np.random.choice(np.arange(len(action_probs)), p=action_probs) else: - action = np.random.randint(0,self.action_dim) + action = np.random.randint(0,self.n_actions) return action def update(self,one_ep_transition): # Find all (state, action) pairs we've visited in this one_ep_transition diff --git a/codes/MonteCarlo/task0_train.py b/codes/MonteCarlo/task0_train.py index dae0c95..51858f8 100644 --- a/codes/MonteCarlo/task0_train.py +++ b/codes/MonteCarlo/task0_train.py @@ -43,8 +43,8 @@ class MCConfig: def env_agent_config(cfg,seed=1): env = RacetrackEnv() - action_dim = 9 - agent = FisrtVisitMC(action_dim, cfg) + n_actions = 9 + agent = FisrtVisitMC(n_actions, cfg) return env,agent def train(cfg, env, agent): diff --git a/codes/PPO/README.md b/codes/PPO/README.md index 66825c9..125ef51 100644 --- a/codes/PPO/README.md +++ b/codes/PPO/README.md @@ -57,16 +57,16 @@ model就是actor和critic两个网络了： import torch.nn as nn from torch.distributions.categorical import Categorical class Actor(nn.Module): - def __init__(self,state_dim, action_dim, + def __init__(self,n_states, n_actions, hidden_dim=256): super(Actor, self).__init__() self.actor = nn.Sequential( - nn.Linear(state_dim, hidden_dim), + nn.Linear(n_states, hidden_dim), nn.ReLU(), nn.Linear(hidden_dim, hidden_dim), nn.ReLU(), - nn.Linear(hidden_dim, action_dim), + nn.Linear(hidden_dim, n_actions), nn.Softmax(dim=-1) ) def forward(self, state): @@ -75,10 +75,10 @@ class Actor(nn.Module): return dist class Critic(nn.Module): - def __init__(self, state_dim,hidden_dim=256): + def __init__(self, n_states,hidden_dim=256): super(Critic, self).__init__() self.critic = nn.Sequential( - nn.Linear(state_dim, hidden_dim), + nn.Linear(n_states, hidden_dim), nn.ReLU(), nn.Linear(hidden_dim, hidden_dim), nn.ReLU(), @@ -88,7 +88,7 @@ class Critic(nn.Module): value = self.critic(state) return value ``` -这里Actor就是得到一个概率分布(Categorica，也可以是别的分布，可以搜索torch distributionsl)，critc根据当前状态得到一个值，这里的输入维度可以是```state_dim+action_dim```，即将action信息也纳入critic网络中，这样会更好一些，感兴趣的小伙伴可以试试。 +这里Actor就是得到一个概率分布(Categorica，也可以是别的分布，可以搜索torch distributionsl)，critc根据当前状态得到一个值，这里的输入维度可以是```n_states+n_actions```，即将action信息也纳入critic网络中，这样会更好一些，感兴趣的小伙伴可以试试。 ### PPO update 定义一个update函数主要实现伪代码中的第六步和第七步： diff --git a/codes/PPO/memory.py b/codes/PPO/memory.py deleted file mode 100644 index c47fbc8..0000000 --- a/codes/PPO/memory.py +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env python -# coding=utf-8 -''' -Author: John -Email: johnjim0816@gmail.com -Date: 2021-03-23 15:30:46 -LastEditor: John -LastEditTime: 2021-09-26 22:00:07 -Discription: -Environment: -''' -import numpy as np -class PPOMemory: - def __init__(self, batch_size): - self.states = [] - self.probs = [] - self.vals = [] - self.actions = [] - self.rewards = [] - self.dones = [] - self.batch_size = batch_size - def sample(self): - batch_step = np.arange(0, len(self.states), self.batch_size) - indices = np.arange(len(self.states), dtype=np.int64) - np.random.shuffle(indices) - batches = [indices[i:i+self.batch_size] for i in batch_step] - return np.array(self.states),np.array(self.actions),np.array(self.probs),\ - np.array(self.vals),np.array(self.rewards),np.array(self.dones),batches - - def push(self, state, action, probs, vals, reward, done): - self.states.append(state) - self.actions.append(action) - self.probs.append(probs) - self.vals.append(vals) - self.rewards.append(reward) - self.dones.append(done) - - def clear(self): - self.states = [] - self.probs = [] - self.actions = [] - self.rewards = [] - self.dones = [] - self.vals = [] \ No newline at end of file diff --git a/codes/PPO/model.py b/codes/PPO/model.py deleted file mode 100644 index fc182d5..0000000 --- a/codes/PPO/model.py +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env python -# coding=utf-8 -''' -Author: John -Email: johnjim0816@gmail.com -Date: 2021-03-23 15:29:24 -LastEditor: John -LastEditTime: 2021-04-08 22:36:43 -Discription: -Environment: -''' -import torch.nn as nn -from torch.distributions.categorical import Categorical -class Actor(nn.Module): - def __init__(self,state_dim, action_dim, - hidden_dim): - super(Actor, self).__init__() - - self.actor = nn.Sequential( - nn.Linear(state_dim, hidden_dim), - nn.ReLU(), - nn.Linear(hidden_dim, hidden_dim), - nn.ReLU(), - nn.Linear(hidden_dim, action_dim), - nn.Softmax(dim=-1) - ) - def forward(self, state): - dist = self.actor(state) - dist = Categorical(dist) - return dist - -class Critic(nn.Module): - def __init__(self, state_dim,hidden_dim): - super(Critic, self).__init__() - self.critic = nn.Sequential( - nn.Linear(state_dim, hidden_dim), - nn.ReLU(), - nn.Linear(hidden_dim, hidden_dim), - nn.ReLU(), - nn.Linear(hidden_dim, 1) - ) - def forward(self, state): - value = self.critic(state) - return value \ No newline at end of file diff --git a/codes/PPO/outputs/CartPole-v0/20211117-184614/models/ppo_actor.pt b/codes/PPO/outputs/CartPole-v0/20211117-184614/models/ppo_actor.pt deleted file mode 100644 index 6d7edc6..0000000 Binary files a/codes/PPO/outputs/CartPole-v0/20211117-184614/models/ppo_actor.pt and /dev/null differ diff --git a/codes/PPO/outputs/CartPole-v0/20211117-184614/models/ppo_critic.pt b/codes/PPO/outputs/CartPole-v0/20211117-184614/models/ppo_critic.pt deleted file mode 100644 index 63c35a8..0000000 Binary files a/codes/PPO/outputs/CartPole-v0/20211117-184614/models/ppo_critic.pt and /dev/null differ diff --git a/codes/PPO/outputs/CartPole-v0/20211117-184614/results/eval_rewards_curve.png b/codes/PPO/outputs/CartPole-v0/20211117-184614/results/eval_rewards_curve.png deleted file mode 100644 index 59eb91a..0000000 Binary files a/codes/PPO/outputs/CartPole-v0/20211117-184614/results/eval_rewards_curve.png and /dev/null differ diff --git a/codes/PPO/outputs/CartPole-v0/20211117-184614/results/train_ma_rewards.npy b/codes/PPO/outputs/CartPole-v0/20211117-184614/results/train_ma_rewards.npy deleted file mode 100644 index 9db0ffe..0000000 Binary files a/codes/PPO/outputs/CartPole-v0/20211117-184614/results/train_ma_rewards.npy and /dev/null differ diff --git a/codes/PPO/outputs/CartPole-v0/20211117-184614/results/train_rewards.npy b/codes/PPO/outputs/CartPole-v0/20211117-184614/results/train_rewards.npy deleted file mode 100644 index 5800e79..0000000 Binary files a/codes/PPO/outputs/CartPole-v0/20211117-184614/results/train_rewards.npy and /dev/null differ diff --git a/codes/PPO/outputs/CartPole-v0/20211117-184614/results/train_rewards_curve.png b/codes/PPO/outputs/CartPole-v0/20211117-184614/results/train_rewards_curve.png deleted file mode 100644 index b4a5cfe..0000000 Binary files a/codes/PPO/outputs/CartPole-v0/20211117-184614/results/train_rewards_curve.png and /dev/null differ diff --git a/codes/PPO/outputs/CartPole-v0/20211231-193837/models/ppo_actor.pt b/codes/PPO/outputs/CartPole-v0/20211231-193837/models/ppo_actor.pt new file mode 100644 index 0000000..36fa194 Binary files /dev/null and b/codes/PPO/outputs/CartPole-v0/20211231-193837/models/ppo_actor.pt differ diff --git a/codes/PPO/outputs/CartPole-v0/20211231-193837/models/ppo_critic.pt b/codes/PPO/outputs/CartPole-v0/20211231-193837/models/ppo_critic.pt new file mode 100644 index 0000000..eaf611a Binary files /dev/null and b/codes/PPO/outputs/CartPole-v0/20211231-193837/models/ppo_critic.pt differ diff --git a/codes/PPO/outputs/CartPole-v0/20211231-193837/results/test_ma_rewards.npy b/codes/PPO/outputs/CartPole-v0/20211231-193837/results/test_ma_rewards.npy new file mode 100644 index 0000000..14bca8b Binary files /dev/null and b/codes/PPO/outputs/CartPole-v0/20211231-193837/results/test_ma_rewards.npy differ diff --git a/codes/PPO/outputs/CartPole-v0/20211231-193837/results/test_rewards.npy b/codes/PPO/outputs/CartPole-v0/20211231-193837/results/test_rewards.npy new file mode 100644 index 0000000..14bca8b Binary files /dev/null and b/codes/PPO/outputs/CartPole-v0/20211231-193837/results/test_rewards.npy differ diff --git a/codes/PPO/outputs/CartPole-v0/20211231-193837/results/test_rewards_curve.png b/codes/PPO/outputs/CartPole-v0/20211231-193837/results/test_rewards_curve.png new file mode 100644 index 0000000..961f15d Binary files /dev/null and b/codes/PPO/outputs/CartPole-v0/20211231-193837/results/test_rewards_curve.png differ diff --git a/codes/PPO/outputs/CartPole-v0/20211231-193837/results/train_ma_rewards.npy b/codes/PPO/outputs/CartPole-v0/20211231-193837/results/train_ma_rewards.npy new file mode 100644 index 0000000..b2254f0 Binary files /dev/null and b/codes/PPO/outputs/CartPole-v0/20211231-193837/results/train_ma_rewards.npy differ diff --git a/codes/PPO/outputs/CartPole-v0/20211231-193837/results/train_rewards.npy b/codes/PPO/outputs/CartPole-v0/20211231-193837/results/train_rewards.npy new file mode 100644 index 0000000..c67c7d7 Binary files /dev/null and b/codes/PPO/outputs/CartPole-v0/20211231-193837/results/train_rewards.npy differ diff --git a/codes/PPO/outputs/CartPole-v0/20211231-193837/results/train_rewards_curve.png b/codes/PPO/outputs/CartPole-v0/20211231-193837/results/train_rewards_curve.png new file mode 100644 index 0000000..cf01ae0 Binary files /dev/null and b/codes/PPO/outputs/CartPole-v0/20211231-193837/results/train_rewards_curve.png differ diff --git a/codes/PPO/agent.py b/codes/PPO/ppo2.py similarity index 60% rename from codes/PPO/agent.py rename to codes/PPO/ppo2.py index 0a7edd9..13cfab7 100644 --- a/codes/PPO/agent.py +++ b/codes/PPO/ppo2.py @@ -5,7 +5,7 @@ Author: John Email: johnjim0816@gmail.com Date: 2021-03-23 15:17:42 LastEditor: John -LastEditTime: 2021-09-26 22:02:00 +LastEditTime: 2021-12-31 19:38:33 Discription: Environment: ''' @@ -13,25 +13,89 @@ import os import numpy as np import torch import torch.optim as optim -from PPO.model import Actor,Critic -from PPO.memory import PPOMemory +import torch.nn as nn +from torch.distributions.categorical import Categorical +class PPOMemory: + def __init__(self, batch_size): + self.states = [] + self.probs = [] + self.vals = [] + self.actions = [] + self.rewards = [] + self.dones = [] + self.batch_size = batch_size + def sample(self): + batch_step = np.arange(0, len(self.states), self.batch_size) + indices = np.arange(len(self.states), dtype=np.int64) + np.random.shuffle(indices) + batches = [indices[i:i+self.batch_size] for i in batch_step] + return np.array(self.states),np.array(self.actions),np.array(self.probs),\ + np.array(self.vals),np.array(self.rewards),np.array(self.dones),batches + + def push(self, state, action, probs, vals, reward, done): + self.states.append(state) + self.actions.append(action) + self.probs.append(probs) + self.vals.append(vals) + self.rewards.append(reward) + self.dones.append(done) + + def clear(self): + self.states = [] + self.probs = [] + self.actions = [] + self.rewards = [] + self.dones = [] + self.vals = [] +class Actor(nn.Module): + def __init__(self,n_states, n_actions, + hidden_dim): + super(Actor, self).__init__() + + self.actor = nn.Sequential( + nn.Linear(n_states, hidden_dim), + nn.ReLU(), + nn.Linear(hidden_dim, hidden_dim), + nn.ReLU(), + nn.Linear(hidden_dim, n_actions), + nn.Softmax(dim=-1) + ) + def forward(self, state): + dist = self.actor(state) + dist = Categorical(dist) + return dist + +class Critic(nn.Module): + def __init__(self, n_states,hidden_dim): + super(Critic, self).__init__() + self.critic = nn.Sequential( + nn.Linear(n_states, hidden_dim), + nn.ReLU(), + nn.Linear(hidden_dim, hidden_dim), + nn.ReLU(), + nn.Linear(hidden_dim, 1) + ) + def forward(self, state): + value = self.critic(state) + return value class PPO: - def __init__(self, state_dim, action_dim,cfg): + def __init__(self, n_states, n_actions,cfg): self.gamma = cfg.gamma self.continuous = cfg.continuous self.policy_clip = cfg.policy_clip self.n_epochs = cfg.n_epochs self.gae_lambda = cfg.gae_lambda self.device = cfg.device - self.actor = Actor(state_dim, action_dim,cfg.hidden_dim).to(self.device) - self.critic = Critic(state_dim,cfg.hidden_dim).to(self.device) + self.actor = Actor(n_states, n_actions,cfg.hidden_dim).to(self.device) + self.critic = Critic(n_states,cfg.hidden_dim).to(self.device) self.actor_optimizer = optim.Adam(self.actor.parameters(), lr=cfg.actor_lr) self.critic_optimizer = optim.Adam(self.critic.parameters(), lr=cfg.critic_lr) self.memory = PPOMemory(cfg.batch_size) self.loss = 0 def choose_action(self, state): - state = torch.tensor([state], dtype=torch.float).to(self.device) + state = np.array([state]) # 先转成数组再转tensor更高效 + state = torch.tensor(state, dtype=torch.float).to(self.device) dist = self.actor(state) value = self.critic(state) action = dist.sample() diff --git a/codes/PPO/task0.py b/codes/PPO/task0.py index 8e0d92a..2d40944 100644 --- a/codes/PPO/task0.py +++ b/codes/PPO/task0.py @@ -5,63 +5,127 @@ sys.path.append(parent_path) # 添加路径到系统路径 import gym import torch +import numpy as np import datetime -from common.plot import plot_rewards +from common.utils import plot_rewards from common.utils import save_results,make_dir -from PPO.agent import PPO -from PPO.train import train +from ppo2 import PPO curr_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") # 获取当前时间 -class PPOConfig: +class Config: def __init__(self) -> None: - self.algo = "DQN" # 算法名称 + ################################## 环境超参数 ################################### + self.algo_name = "DQN" # 算法名称 self.env_name = 'CartPole-v0' # 环境名称 self.continuous = False # 环境是否为连续动作 self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # 检测GPU + self.seed = 10 # 随机种子，置0则不设置随机种子 self.train_eps = 200 # 训练的回合数 self.test_eps = 20 # 测试的回合数 - self.batch_size = 5 - self.gamma=0.99 + ################################################################################ + + ################################## 算法超参数 #################################### + self.batch_size = 5 # mini-batch SGD中的批量大小 + self.gamma = 0.95 # 强化学习中的折扣因子 self.n_epochs = 4 - self.actor_lr = 0.0003 - self.critic_lr = 0.0003 - self.gae_lambda=0.95 - self.policy_clip=0.2 + self.actor_lr = 0.0003 # actor的学习率 + self.critic_lr = 0.0003 # critic的学习率 + self.gae_lambda = 0.95 + self.policy_clip = 0.2 self.hidden_dim = 256 - self.update_fre = 20 # frequency of agent update - -class PlotConfig: - def __init__(self) -> None: - self.algo = "DQN" # 算法名称 - self.env_name = 'CartPole-v0' # 环境名称 - self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # 检测GPU + self.update_fre = 20 # 策略更新频率 + ################################################################################ + + ################################# 保存结果相关参数 ################################ self.result_path = curr_path+"/outputs/" + self.env_name + \ '/'+curr_time+'/results/' # 保存结果的路径 self.model_path = curr_path+"/outputs/" + self.env_name + \ '/'+curr_time+'/models/' # 保存模型的路径 self.save = True # 是否保存图片 + ################################################################################ + +def env_agent_config(cfg): + ''' 创建环境和智能体 + ''' + env = gym.make(cfg.env_name) # 创建环境 + n_states = env.observation_space.shape[0] # 状态维度 + if cfg.continuous: + n_actions = env.action_space.shape[0] # 动作维度 + else: + n_actions = env.action_space.n # 动作维度 + agent = PPO(n_states, n_actions, cfg) # 创建智能体 + if cfg.seed !=0: # 设置随机种子 + torch.manual_seed(cfg.seed) + env.seed(cfg.seed) + np.random.seed(cfg.seed) + return env, agent -def env_agent_config(cfg,seed=1): - env = gym.make(cfg.env_name) - env.seed(seed) - state_dim = env.observation_space.shape[0] - action_dim = env.action_space.n - agent = PPO(state_dim,action_dim,cfg) - return env,agent +def train(cfg,env,agent): + print('开始训练！') + print(f'环境：{cfg.env_name}, 算法：{cfg.algo_name}, 设备：{cfg.device}') + rewards = [] # 记录所有回合的奖励 + ma_rewards = [] # 记录所有回合的滑动平均奖励 + steps = 0 + for i_ep in range(cfg.train_eps): + state = env.reset() + done = False + ep_reward = 0 + while not done: + action, prob, val = agent.choose_action(state) + state_, reward, done, _ = env.step(action) + steps += 1 + ep_reward += reward + agent.memory.push(state, action, prob, val, reward, done) + if steps % cfg.update_fre == 0: + agent.update() + state = state_ + rewards.append(ep_reward) + if ma_rewards: + ma_rewards.append(0.9*ma_rewards[-1]+0.1*ep_reward) + else: + ma_rewards.append(ep_reward) + if (i_ep+1)%10 == 0: + print(f"回合：{i_ep+1}/{cfg.train_eps}，奖励：{ep_reward:.2f}") + print('完成训练！') + return rewards,ma_rewards -cfg = PPOConfig() -plot_cfg = PlotConfig() -# 训练 -env,agent = env_agent_config(cfg,seed=1) -rewards, ma_rewards = train(cfg, env, agent) -make_dir(plot_cfg.result_path, plot_cfg.model_path) # 创建保存结果和模型路径的文件夹 -agent.save(path=plot_cfg.model_path) -save_results(rewards, ma_rewards, tag='train', path=plot_cfg.result_path) -plot_rewards(rewards, ma_rewards, plot_cfg, tag="train") -# 测试 -env,agent = env_agent_config(cfg,seed=10) -agent.load(path=plot_cfg.model_path) -rewards,ma_rewards = eval(cfg,env,agent) -save_results(rewards,ma_rewards,tag='eval',path=plot_cfg.result_path) -plot_rewards(rewards,ma_rewards,plot_cfg,tag="eval") \ No newline at end of file +def test(cfg,env,agent): + print('开始测试!') + print(f'环境：{cfg.env_name}, 算法：{cfg.algo_name}, 设备：{cfg.device}') + rewards = [] # 记录所有回合的奖励 + ma_rewards = [] # 记录所有回合的滑动平均奖励 + for i_ep in range(cfg.test_eps): + state = env.reset() + done = False + ep_reward = 0 + while not done: + action, prob, val = agent.choose_action(state) + state_, reward, done, _ = env.step(action) + ep_reward += reward + state = state_ + rewards.append(ep_reward) + if ma_rewards: + ma_rewards.append( + 0.9*ma_rewards[-1]+0.1*ep_reward) + else: + ma_rewards.append(ep_reward) + print('回合：{}/{}, 奖励：{}'.format(i_ep+1, cfg.test_eps, ep_reward)) + print('完成训练！') + return rewards,ma_rewards + +if __name__ == "__main__": + cfg = Config() + # 训练 + env,agent = env_agent_config(cfg) + rewards, ma_rewards = train(cfg, env, agent) + make_dir(cfg.result_path, cfg.model_path) # 创建保存结果和模型路径的文件夹 + agent.save(path=cfg.model_path) + save_results(rewards, ma_rewards, tag='train', path=cfg.result_path) + plot_rewards(rewards, ma_rewards, cfg, tag="train") + # 测试 + env,agent = env_agent_config(cfg) + agent.load(path=cfg.model_path) + rewards,ma_rewards = test(cfg,env,agent) + save_results(rewards,ma_rewards,tag='test',path=cfg.result_path) + plot_rewards(rewards,ma_rewards,cfg,tag="test") \ No newline at end of file diff --git a/codes/PPO/task1.py b/codes/PPO/task1.py index 38d9152..04726cb 100644 --- a/codes/PPO/task1.py +++ b/codes/PPO/task1.py @@ -6,10 +6,9 @@ sys.path.append(parent_path) # 添加路径到系统路径 import gym import torch import datetime -from common.plot import plot_rewards +from common.utils import plot_rewards from common.utils import save_results,make_dir -from PPO.agent import PPO -from PPO.train import train +from ppo2 import PPO curr_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") # 获取当前时间 @@ -45,9 +44,9 @@ class PlotConfig: def env_agent_config(cfg,seed=1): env = gym.make(cfg.env_name) env.seed(seed) - state_dim = env.observation_space.shape[0] - action_dim = env.action_space.shape[0] - agent = PPO(state_dim,action_dim,cfg) + n_states = env.observation_space.shape[0] + n_actions = env.action_space.shape[0] + agent = PPO(n_states,n_actions,cfg) return env,agent diff --git a/codes/PPO/train.ipynb b/codes/PPO/train.ipynb deleted file mode 100644 index b2dc91a..0000000 --- a/codes/PPO/train.ipynb +++ /dev/null @@ -1,257 +0,0 @@ -{ - "metadata": { - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.10" - }, - "orig_nbformat": 2, - "kernelspec": { - "name": "python3710jvsc74a57bd0366e1054dee9d4501b0eb8f87335afd3c67fc62db6ee611bbc7f8f5a1fefe232", - "display_name": "Python 3.7.10 64-bit ('py37': conda)" - }, - "metadata": { - "interpreter": { - "hash": "366e1054dee9d4501b0eb8f87335afd3c67fc62db6ee611bbc7f8f5a1fefe232" - } - } - }, - "nbformat": 4, - "nbformat_minor": 2, - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "import sys\n", - "from pathlib import Path\n", - "curr_path = str(Path().absolute())\n", - "parent_path = str(Path().absolute().parent)\n", - "sys.path.append(parent_path) # add current terminal path to sys.path" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "import gym\n", - "import torch\n", - "import datetime\n", - "from PPO.agent import PPO\n", - "from common.plot import plot_rewards\n", - "from common.utils import save_results,make_dir\n", - "\n", - "curr_time = datetime.datetime.now().strftime(\"%Y%m%d-%H%M%S\") # obtain current time" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "class PPOConfig:\n", - " def __init__(self) -> None:\n", - " self.env = 'CartPole-v0'\n", - " self.algo = 'PPO'\n", - " self.result_path = curr_path+\"/results/\" +self.env+'/'+curr_time+'/results/' # path to save results\n", - " self.model_path = curr_path+\"/results/\" +self.env+'/'+curr_time+'/models/' # path to save models\n", - " self.train_eps = 200 # max training episodes\n", - " self.test_eps = 50\n", - " self.batch_size = 5\n", - " self.gamma=0.99\n", - " self.n_epochs = 4\n", - " self.actor_lr = 0.0003\n", - " self.critic_lr = 0.0003\n", - " self.gae_lambda=0.95\n", - " self.policy_clip=0.2\n", - " self.hidden_dim = 256\n", - " self.update_fre = 20 # frequency of agent update\n", - " self.device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\") # check gpu" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "def env_agent_config(cfg,seed=1):\n", - " env = gym.make(cfg.env) \n", - " env.seed(seed)\n", - " state_dim = env.observation_space.shape[0]\n", - " action_dim = env.action_space.n\n", - " agent = PPO(state_dim,action_dim,cfg)\n", - " return env,agent" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "def train(cfg,env,agent):\n", - " print('Start to train !')\n", - " print(f'Env:{cfg.env}, Algorithm:{cfg.algo}, Device:{cfg.device}')\n", - " rewards= []\n", - " ma_rewards = [] # moving average rewards\n", - " running_steps = 0\n", - " for i_ep in range(cfg.train_eps):\n", - " state = env.reset()\n", - " done = False\n", - " ep_reward = 0\n", - " while not done:\n", - " action, prob, val = agent.choose_action(state)\n", - " state_, reward, done, _ = env.step(action)\n", - " running_steps += 1\n", - " ep_reward += reward\n", - " agent.memory.push(state, action, prob, val, reward, done)\n", - " if running_steps % cfg.update_fre == 0:\n", - " agent.update()\n", - " state = state_\n", - " rewards.append(ep_reward)\n", - " if ma_rewards:\n", - " ma_rewards.append(\n", - " 0.9*ma_rewards[-1]+0.1*ep_reward)\n", - " else:\n", - " ma_rewards.append(ep_reward)\n", - " if (i_ep+1)%10==0:\n", - " print(f\"Episode:{i_ep+1}/{cfg.train_eps}, Reward:{ep_reward:.3f}\")\n", - " print('Complete training！')\n", - " return rewards,ma_rewards" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - "def eval(cfg,env,agent):\n", - " print('Start to eval !')\n", - " print(f'Env:{cfg.env}, Algorithm:{cfg.algo}, Device:{cfg.device}')\n", - " rewards= []\n", - " ma_rewards = [] # moving average rewards\n", - " for i_ep in range(cfg.test_eps):\n", - " state = env.reset()\n", - " done = False\n", - " ep_reward = 0\n", - " while not done:\n", - " action, prob, val = agent.choose_action(state)\n", - " state_, reward, done, _ = env.step(action)\n", - " ep_reward += reward\n", - " state = state_\n", - " rewards.append(ep_reward)\n", - " if ma_rewards:\n", - " ma_rewards.append(\n", - " 0.9*ma_rewards[-1]+0.1*ep_reward)\n", - " else:\n", - " ma_rewards.append(ep_reward)\n", - " if (i_ep+1)%10==0:\n", - " print(f\"Episode:{i_ep+1}/{cfg.train_eps}, Reward:{ep_reward:.3f}\")\n", - " print('Complete evaling！')\n", - " return rewards,ma_rewards" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "Start to train !\n", - "Env:CartPole-v0, Algorithm:PPO, Device:cuda\n", - "Episode:10/200, Reward:15.000\n", - "Episode:20/200, Reward:9.000\n", - "Episode:30/200, Reward:20.000\n", - "Episode:40/200, Reward:17.000\n", - "Episode:50/200, Reward:64.000\n", - "Episode:60/200, Reward:90.000\n", - "Episode:70/200, Reward:23.000\n", - "Episode:80/200, Reward:138.000\n", - "Episode:90/200, Reward:150.000\n", - "Episode:100/200, Reward:200.000\n", - "Episode:110/200, Reward:200.000\n", - "Episode:120/200, Reward:200.000\n", - "Episode:130/200, Reward:200.000\n", - "Episode:140/200, Reward:200.000\n", - "Episode:150/200, Reward:200.000\n", - "Episode:160/200, Reward:200.000\n", - "Episode:170/200, Reward:200.000\n", - "Episode:180/200, Reward:200.000\n", - "Episode:190/200, Reward:200.000\n", - "Episode:200/200, Reward:200.000\n", - "Complete training！\n", - "results saved!\n" - ] - }, - { - "output_type": "display_data", - "data": { - "text/plain": "
", - "image/svg+xml": "\n\n\n \n \n \n \n 2021-05-06T01:36:50.188726\n image/svg+xml\n \n \n Matplotlib v3.4.1, https://matplotlib.org/\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n\n", - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXsAAAEcCAYAAAAmzxTpAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Z1A+gAAAACXBIWXMAAAsTAAALEwEAmpwYAACCHklEQVR4nO2deZwU1bm/n6rqdfaVYVgERUEUhYEBUVQU911jVC5xTUxiFpcYTExiNEFNgjEm0UuuMSbm+tOrMXELbmjcd0EBBRUB2WeYfe/pparO74/qqu6e6Znp2aeZ83w+MN3VVeec2t5663ve8x5FCCGQSCQSyT6NOtwNkEgkEsngI429RCKRjAKksZdIJJJRgDT2EolEMgqQxl4ikUhGAdLYSyQSyShAGvtRwD333MPSpUuHpe4zzjiD999/f1jqHmkEg0Guuuoq5syZwzXXXDPczeHDDz/k5JNPpqysjP/85z/D3ZxBZ9q0aezYsWO4mzFsSGMvGVSeffZZjjjiiOFuxojghRdeoLa2lvfff5+777670+/33HMPhx56KGVlZZSXl7N48WLWrl0LwBNPPMH06dMpKytj9uzZnHPOObz66qvOts3Nzdxyyy0sWLCAmTNnctZZZ/H444932567776br33ta6xdu5YTTzxxQPbx448/5pvf/Cbl5eXMmzePr371qz22ozsuueQS/vnPfyYsmzZtGrNmzaKsrIxjjjmGX//61xiG0d+m95nPPvuMr3zlK8ycOZOvfOUrfPbZZ8PWlu6Qxn4Y0HV9uJswIOwL+zGU+1BRUcHkyZNxuVxdrnPaaaexdu1a3n33XWbPns3VV1+NPe5x1qxZrF27ljVr1vDVr36V6667jqamJsLhMJdffjkVFRU8+uijrFmzhhtuuIHf/e53PPDAA92256CDDurTviQ7bmvXruWyyy5j7ty5vPjii7z//vv84he/4I033uh1+UIITNPs8venn36atWvX8ve//51nnnmGxx57rNd1DAThcJjvfve7nH322axevZpzzz2X7373u4TD4WFpT3dIY9+B++67jxNPPJGysjJOP/10XnrpJcA6qeXl5XzxxRfOuvX19Rx++OHU1dUB8Oqrr3LOOec4Xtnnn3/urLto0SLuu+8+zjrrLGbNmoWu613WBWAYBr/5zW844ogjWLRoEQ899BDTpk1zbrKWlhZ++tOfcvTRR3PMMcfw+9//PmXvZt26dSxevJjy8nLOPvvsBJnl8ccf57TTTqOsrIwTTjiBRx991Pnt/fff59hjj+W+++5jwYIF/OQnP+Gee+7h2muv5Uc/+hFlZWWcccYZfPLJJwn7/c477wD0uO7GjRs599xzKSsr45prruG6667j97//fZf78dhjjzltPf3009m4cSPQ+XX9xhtvdMpJtg+nnXZagpes6zrz5893yuvueHVk69atXHLJJZSXl3PGGWfw8ssvA5YX/ac//Ynnn3+esrKyTt5qR9xuN+eddx41NTU0NDQk/KaqKueffz7BYJCdO3fy9NNPU1lZyR//+EcmTpyI2+3m2GOP5aabbuLuu++mtbW1U/knnngiu3bt4qqrrqKsrIxwOExVVRVXXXUV8+bN46STTkowoPfccw/XXHMNS5cuZfbs2Tz55JOdyrzjjjs499xz+da3vkVBQQGKojBjxgz++Mc/AtDU1MS3v/1t5s+fz9y5c/n2t7/N3r17ne0vueQSfv/737N48WJmzpzJDTfcwJo1a1i2bBllZWUsW7asU51Tpkxhzpw5bN68GbCuiZNOOol58+Zx1VVXUVVVlfT4hsNhli9fznHHHcdRRx3FzTffTDAYTLpud9fHBx98gK7rXHbZZXg8Hi699FKEELz33ntJyxpWhCSB5557Tuzdu1cYhiGeffZZMXPmTFFVVSWEEOLGG28Ud911l7PuQw89JL7+9a8LIYTYuHGjmD9/vli3bp3QdV088cQT4vjjjxehUEgIIcTxxx8vzj77bFFRUSHa29t7rOv//u//xGmnnSYqKytFY2OjuOyyy8TUqVNFJBIRQgjx3e9+V/z85z8XbW1tora2Vpx//vnikUceSbpPd999t/jhD38ohBBi7969Yt68eeK1114ThmGIt956S8ybN0/U1dUJIYR49dVXxY4dO4RpmuL9998Xhx9+uNiwYYMQQoj33ntPTJ8+Xdxxxx0iFAqJ9vZ2cffdd4sZM2aI1157Tei6Lu68805xwQUXOHUff/zx4u2333ba0dW6oVBIHHfcceLvf/+7CIfDYtWqVeLQQw9NON4dz9PRRx8t1q9fL0zTFNu3bxe7d+8WQggxdepUsX37dmfdH//4x045yfbhnnvuEddff72z/quvvipOPfXUlI5XPOFwWJx44onif/7nf0QoFBLvvPOOmDVrlti6dWun89DTeQqFQuI3v/mNWLhwoRBCiMcff1wsXrxYCCFEJBIRf//738WsWbNEc3OzuO6668SPfvSjTuVFIhExffp08cYbbyStL/7cCCHEkiVLxC233CKCwaD49NNPxRFHHCHeeecdp22HHHKIeOmll4RhGM41bBMIBMTBBx8s3n333S73r76+XrzwwgsiEAiIlpYWcfXVV4vvfOc7zu8XX3yxWLhwofjiiy9EJBIR4XBYXHzxxeKxxx5LKCf+/G7evFkcddRR4rHHHhPvvPOOmDdvntiwYYMIhUJi2bJlYsmSJUm3u/3228W3v/1t0dDQIFpaWsS3v/1tceeddyZtd3fXxwMPPCC+8Y1vJKz/rW99S/z1r3/t8jgMF9Kz78Bpp51GSUkJqqpy+umnM2nSJD7++GMAzjrrLJ599lln3ZUrV3LWWWcB8I9//IOLLrqImTNnomka5513Hm63m3Xr1jnrX3LJJZSWluLz+Xqs6/nnn+fSSy9l7Nix5Obm8q1vfcspp7a2ltdff52f/vSnZGRkUFhYyOWXX57Qtq54+umnOfbYY1m4cCGqqrJgwQJmzJjB66+/DsBxxx3Hfvvth6IozJs3jwULFrBmzRpne1VVueaaa/B4PM5+zJkzh4ULF6JpGuecc07CG01Hulp3/fr16LrOpZdeitvt5uSTT+awww7rspx//etfXHnllRx++OEoisKkSZMYP358j/ufbB/OOussXnnlFdrb2wHrvJ5xxhkpHa941q9fTyAQ4Fvf+hYej4cjjzyS448/PqXzYvPCCy9QXl7OwoUL2bhxI//93/+dUH55eTkLFizg2WefZcWKFWRnZ9PQ0EBxcXGnslwuF/n5+Z3eDJJRWVnJRx99xNKlS/F6vUyfPp0LLriAp59+2lln1qxZnHjiiaiq6px7m+bmZkzTTNoOm/z8fE455RT8fj9ZWVl85zvfYfXq1QnrnHfeeRx00EG4XC7cbneXZZ133nnMnTuXq666iq9+9aucf/75rFy5kvPPP59DDz0Uj8fD9ddfz7p169i9e3fCtkIIHnvsMX7605+Sl5dHVlYW3/72t7s8T91dH21tbWRnZyesn5WVRVtbW5dtHy66Fg9HKU899RQPPPAAe/bsASAQCDg3yxFHHEEwGGT9+vUUFhby+eefOx1bFRUVPPXUUzz00ENOWZFIhOrqaud7aWlpynVVV1cnrD927Fjnc0VFBbquc/TRRzvLTNPsVH4yKioqeOGFFzq9ltqdqK+//jorVqxg+/btmKZJMBhk6tSpzrr5+fl4vd6EMouKipzPPp+PUCiErutJtemu1q2urqakpARFUZzfu9ufyspK9ttvvx73Nxkd92HSpElMmTKFV199leOPP55XXnmFp556Cuj5eMVTXV3N2LFjUdWYDzVu3LgupYRknHrqqdx5551Jf5s5cyaPPPJI0v2pqanptFzXdRoaGsjPz++x3urqanJzc8nKykpo+4YNG5zv8ddgR3JyclBVlZqaGqZMmZJ0nfb2dn7961/z5ptv0tTUBFjG0jAMNE0Duj/n8Tz55JNMmjSp0z4ceuihzvfMzEzy8vKoqqpiwoQJzvL6+nra29v5yle+4iwTcX0EV155JR9++CEAv/zlLzn77LO7vD4yMzM7yWRtbW1kZmamtB9DiTT2cezZs4ebbrqJv//975SVlTnep42maZx66qk888wzFBUVcdxxxzk3R2lpKVdddRXf+c53uiw/3pD1VFdxcXGCnhn/eezYsXg8Ht57771uO/uSUVpayjnnnMNtt93W6bdwOMw111zD8uXLOeGEE3C73Xz3u991Ogg77sNAUlxcTFVVFUIIp47KykomTpzY5X7s3Lkz6W9+v9/xwgBqamooKSlxvifbhzPPPJNnnnkG0zQ58MADHUPS3fHqyJgxY9i7dy+maToGv7KyksmTJ/e4bX846qijuOuuuwgEAmRkZDjLX3zxRTweD7NmzeqxjDFjxtDU1ERra6tzTVdWVvZ43Gz8fj+zZs3ixRdfZP78+UnX+dvf/sa2bdt47LHHKC4u5rPPPuPcc88dsOtrzJgxjuMElvPU2NiYsA9gPRx9Ph/PPvtsp98A7r///k7Luro+DjzwQP72t78lXLebNm1iyZIlfd6PwULKOHG0t7ejKAoFBQWA1Vlpd/zYnHXWWTz//POsXLmSM88801l+wQUX8Oijj7J+/XqEEAQCAV577bWknWOp1HXaaafx4IMPUlVVRXNzM3/5y1+c38aMGcOCBQv4zW9+Q2trK6ZpsnPnTj744IMe9/Hss8/m1Vdf5c0338QwDEKhEO+//z579+4lHA4TDocpKCjA5XLx+uuv8/bbb6d+APvBrFmz0DSNhx56CF3X+c9//pPQeduRr371q/ztb39jw4YNCCHYsWOHc6MffPDBPPPMMxiGwRtvvNFJKkjG6aefzttvv80jjzyScF67O14dOfzww/H5fNx///1EIhHef/99XnnlFU4//fQ+HJHUOeeccxg7dizXXnstu3fvJhKJ8Oabb3Lbbbfx/e9/v5PMkIzS0lLKysq46667CIVCfP755/zrX//i7LPPTrkdN9xwA08++ST333+/84b6+eef84Mf/ACwPF6v10tOTg6NjY0JElVXFBUVsWvXrpTqP/PMM3niiSf47LPPCIfD3HXXXRx++OEJXj1YMt4FF1zAr371Kye4oqqqijfffLPLsru6PubNm4emaTz44IOEw2Hnzb6rB95wIo19HAceeCBf//rXWbx4MUcddRRffPEFs2fPTlhn5syZ+P1+qqurOfbYY53lhx12GLfeeivLli1j7ty5nHzyyTzxxBN9ruvCCy9kwYIFnH322Zx77rksXLgQl8vlvO7ecccdRCIRTj/9dObOncs111yT9FW+I6WlpfzpT3/iz3/+M0ceeSQLFy7kr3/9K6ZpkpWVxU033cR1113H3LlzeeaZZ1i0aFFvD2Of8Hg83HPPPfzrX/9i7ty5/Pvf/+a4447D4/EkXf+0007jqquu4oc//CGzZ8/me9/7niMN/OxnP+PVV1+lvLyclStXphRDPmbMGCe0Md44d3e8ku3DvffeyxtvvMH8+fP55S9/yR133NGlrDFQeDweHnjgAUpLS7nwwguZM2cOv/nNb/jBD37AlVdemXI5d911F3v27OGYY47h+9//PldffTVHHXVUytvPnj2b//3f/+W9997jxBNPZN68efz85z9n4cKFAFx22WWEQiHmz5/PRRddxDHHHNNjmZdeeimrVq1i7ty5Pb5dHXXUUVx77bVcffXVHH300ezatavLaK4bbriBSZMmceGFFzJ79mwuv/xytm3b1mXZXV0fHo+HFStW8PTTT1NeXs7jjz/OihUrurxuhxNFCDl5STrw+uuv84tf/CJBO97XueCCC1i8eDHnn3/+cDdFIkl7pGc/QgkGg7z++uvouk5VVRUrVqwYsFGOI5UPPviAmpoadF3nySefZNOmTSl5fxKJpGdkB+0IRQjB3XffzXXXXYfP5+O4447j2muvHe5mDSrbtm3juuuuo729nQkTJnD33XczZsyY4W6WRLJPIGUciUQiGQVIGUcikUhGAdLYSyQSyShAGnuJRCIZBYzoDtqGhjZMs/ddCoWFWdTVJR/MNJzIdvWekdo22a7eMVLbBSO3bX1pl6oq5OcnT9Uwoo29aYo+GXt725GIbFfvGaltk+3qHSO1XTBy2zaQ7ZIyjkQikYwCpLGXSCSSUcCIlnE6IoSgoaGGcDgIdP16U12tdjul2XCxb7dLwePxkZ9fPGiZMSUSSd/p0dg3NDTwox/9iJ07d+LxeJg0aRLLli2joKCAdevWcfPNNxMKhRg/fjy//e1vKSwsBOj2t77S2tqEoiiUlExAUbp+KXG5VHR95BnVfbldQpg0NtbS2tpEdnbewDRMIpEMGD3KOIqicOWVV7Jq1SpWrlzJxIkTufPOOzFNkxtuuIGbb76ZVatWUV5e7ky60N1v/aG9vZXs7LxuDb1keFAUlezsfNrbR15Ug0QiScHY5+XlJczKM2vWLCoqKtiwYQNer5fy8nIAFi9ezAsvvADQ7W/9wTQNNC2tlKdRhaa5MM3UJj0fDIQQmEIkTIZhRpf1Zt2Oy+PXd/6Zotv1O5aXyr+u2tKresze1Znqvy6PQ6r/BqldA9L2XratP8eiq+w0qazTX3plOU3T5JFHHmHRokVUVlYybtw457eCggJM06SxsbHb3/Ly8vrVYKkHjxyEEFTUtlGQ48PvdfXp3NQ3B/nVQx/y4yWzKc7zd7vu259U8sb6Cn5y8Rw++qKGf7+1jZuvmIsarfd3/1jHp9sbyPS5+M1VR1LfHOK2B9cQ0U1OmD2Br51sTa/Y1Bbmp/e9R3tI55DJ+SxdXMYz72zniTe+dOqaOCaLX359nvN92d/XsKOqpcu2+Twat115BAU51tysj768mRdXpzbphk1Rro9ff3s+G7c1cPe/Pk76kPJ5NG7/5nzys61pFf/6zKe8vaHzRCoDicetcsvlc8n0uflJ9LiNHARuDHxKBK8SSfhrfdYZf2gZp518BH9//nPe/Liy3zVecvJUjp89gbseW8/GbfW92vbwKYVcd8FMXv5wN+u31HL9RbN4c30Ff3/+c6cXckJxFsu+Ma/bcvpCr4z9rbfeSkZGBhdffDEvvfTSgDemI4WFWQnfq6tVXK7UJJxU1xtqBqtdy5bdEp0kenGftu9LuwxTENFNdFM426uqSnFxzzMj2extDlHfHCJo0OV29vK61h18WdFMcXE29Wsr2FndSm5uBj6vdRnvrQ/g0hTagjour5uwCBHRTVyaSk1z0Cmnob2B9pDOxJIsPt3egOJ28d6nVUwuzeGow8exYWstH2+ppbAwC1VVaA/p7KhqYfbBYzh4UkGn9lXXB/jP6p2EUZw6qhrbKc73c9K8SZ3WT8bW3Y28v3EvXr+X2pYQphAsOXkaxD1AK2pbee3D3eDSKC7OZntlM29v2MsRh45lyoS8lI95b6hpCPDSBzvRFRXF7aI9pHPsrPFMKEn9HPcJIdCMIN5wA95QI55QA55IM269DVekDXek1fqst6GZkW6Lqtu9jeLiE6lsaGd8cSYLZyef6jIexYzgCTfjCTfhCTfhDTfhiTRRtWcvkVqF4uLpVNUHOGB8LvNnpDZv7oefV7GtsoXi4mx21rSxrTJ6LbdF0DSVC0+0nJEJxVnOddSbe6knUjb2y5cvZ8eOHdx7772oqkppaSkVFRXO7/X19aiqSl5eXre/9Ya6utaEQQWmaabUkTgcHaFdTbAdz0C1K1ldIvo62pfy+9ou3bC2MYzYeTFNk5qarj3gjjQ0BACoa2hLul1xcbazvLUthGEKqqqbaW6x5pitrGomy++22qObeN0auqFTW9dGY6NVts+j0R6MOOXU1Fr9CkcfVsojVZt5dNVnVNS28bWTpnJC2Tgi4Qgfb6llT2UjPo+LPTXW+nOnFnPEISWd2rWtspn/rN7J3qoWxuZYHndbIEJxro8Ty2JvuN2R53fx/sa9bNvVQGVNKxleFyfOHp+wzoZtdbz24W7q6tqo8bt48JmN+DwaXzvxIOcYxLerP4hwO0bdTqrbPifDv4nGmmnoIcvwzDqwkFkHFvVQQiLdtcsMNGHW7cRs2IPZsAejYQ9mQyVE2hNX1FwovhwUfzZKdj6KfxKKL9v65/GhuP0Q/au4fSgePy8/+TTzI++x9/ONBNojjCvMSDgnwtTJNeqo3fo5ZmMlZkMFZmMloqWWThF/3kyKXAF21a2npuZ4dMNkQlFmyue4rS3Eph0N7K1qoqa+jbBu3SvNrUF8Hi2hnJqalj6dS1VVOjnJNikZ+7vuuosNGzZw3333OdNtzZgxg2AwyJo1aygvL+fRRx/l1FNP7fG3fYmjjy7niiu+ybvvvs0RRxzJkiWXcM89v2fr1s2Ew2HKysq5+uofsGfPLn760x/x6KP/Qtd1zjjjBC677BssWXIpL7/8Em+++Rq/+MXtPPLIQ7z88osYho7H42Xp0hs56KBpSes699zzue22W6irq2Xs2FJngmuAp59+gsce+z/cbg9CmCxb9hsmTZo8aMehPxKjLVUYRs+F2A9+6+FifbYfOFZZoKlKtE3CaZdLUxKchkh0m0kl2RTl+vjPmt0AzJxiRYv53NbUj6Gwgc/joq45CEBhri9pu/zRN4v2cEzeiOgmmb7UX5xzMq37qrktTEsgTHZm52nttKiXb5gm1Y3trPm8mtOPnOQY+r4ihEA0V6FXfI5R+TlG9TZEcxUA2cAiP+yt+wIz25o2U+2HlCqEsAz63s0YVZsxqrYgmqud3xV/Dmr+eNwHHYmaMwYlqxA1uwglq9Ay6r2s+wvfTMojHxDZ9CbhyP5kaWEiX67GqN6KWbUVo3Y7rUb0zUBzoeaWohXvj3rQUag5xSiZBaiZBSiZ+ShuL5vv/QFeIxDdF1B70ZycDOs8tQYiNAci6LqJEJaD5tIGX57u8WrcvHkzf/7zn5k8eTKLF1sSwYQJE1ixYgV33HEHt9xyS0J4JViv8l39NpC8/UklbyXR4BSlfwYI4OjDS1lwWM+vZ16vl/vvfxCA3/zmVmbNms2NN/4c0zT55S9v4tln/83ZZ59HINBGbW0Nu3fvYf/9p7BmzWqWLLmUDz/8gPLyuQCceuoZ/Nd/XQzA6tXv89vf/pr77vt70rp+9rMbmDmzjK9//Vvs2bObyy9fwhFHHAnAn/70Rx5++HGKiooIh8ODF9svOn3ofRG2sU+hjba91g3hGPn4NxIhBJrLMtR2pxuAS1PR44y9vY3bpTJzShEvf7Sb8UWZFEX7DLyeqLGPWJ3NtU2WsS/qythH1w/Gadlh3cAdfWikQnbUEDQHwjS3hR3DEI8atSymgPqmIAI4ZFJ+ynXEI0wdo/IL9O0fou9Yh2i1Jt5W/LloJQeiTl2AVjSZynAGea/chhaoxYgeQ7WXip8wIrRtXkPw43fRd61HtFmTkSu+bLSxB6FNPx61eH/UgvGovoGVh0xPJluU/Tn4i7e4TFnPxL01BPcKy7AXTcZ9yCLyDzyUNu9YlKwilB52LiB85BptVtlC9Orhk51hPcCb2qxzLIhKoYaJewhk5x6N/UEHHcSmTZuS/jZ79mxWrlzZ69/2JU47LTbT/FtvvcFnn23k0UcfBqypBceMsV77Z88uZ/XqD9izZw/nnPMVHn74QSKRCGvWfMDFF18OwKZNn/H//t8DNDc3oaoqu3bt7LKujz76kOuuuwGA8eMnOA8Mq6653H77LSxYcAxHHnk048dPGJR9t81nfx6s9rZGCjlA7IeWbpiO8Y506dnH3gTcLjXhzSFiG3tN5fADC3n5o90cPiU2BsTrtm6LYDhm7F2a4njfHfE5nn0sEimim7i11G9gu+yWtjDNgQilBRkJvwsjQsbetZzrX4OIHILpstbXelEHgFG3k8imN9E3v4sItYLmxjVhBtqsM3CNm46SOzbBgCnVrbSZHtztdUSix1NLwcAJITCrtxL54m0iX35Aa6gN3D5cE2bgmng4Wuk0lJwxgx5w4XVrfGAeysGR7SjCZHP+AmYdexxq0WSUaGRfVnE27SnKJW34KDYaAesa603z7XPc2Bqmrd16m4joZtSzHwHGfiSz4LDk3vdQavZ+f/xNKfjVr+5MalznzJnLmjWWsb/55ltZt+4j/vOfVQgB48aNJxKJ8POf/5j//u+/MG3awdTW1nDuuad1U1fX/OpXv+Wzzzby4YdruOaaq1i69CcceeSC/uxmF4i4//tYQq9kHOtvvGcf0eONvXCMfXyYnNuloseVb2/rcqlMn5TPqfP24/g4fdzXwbOvawpSmOPrUr7wuFRURUmIUumtt5bhdaGpCk1Rz37axDwARLCV8KcvE9n4MnntzRzvh8r6LwkVWfKemoKOIEwTffsawuufx6zZBqoL1+TZuA48AteEGSgub5fbKqpCnZlNQXud82bVXZ3C1NG3fkD44+cx63aB5sG1/xyKyk+gNXMSitY/yam3eNwqn0fGkPnd+7jrt29w+qH7MadkSp/LC+DHa1j9kaKXnr1t7Ctq25x7JmKYvXYM+kpaG/uRxoIFx/LQQ//L0qU3omkajY2NBAJtjBs3njlz5vLnP/83ubn5jBlTQnn5PP785xXMnWuNYQiHQxiG4bwJPPHEP7uta86ccp599t9cfvmVVFTsYc2a1ZSXz4tOUL6XQw6ZwSGHzKCiYjebN28aFGMf8+z7bu5tA56KjGPEafaGLePEGXFhCsdDMuM0e7emEgrHIjbiPXuXpnLhogMT6vHGafZgefZd6fVghQP7vRrBUJxnHzHx9MLYK4r15tDYYnl9+Z4IwXf+j8jnr4EeRpt4OI1Fs8he+yCuQDXtphW50Z1+LkwTffPbhNY9i2jai5Jbgveor+E+8EgUX/JOvI6oCtQaWZQE6503pWQGTpgGkc9fJ7z2GURbPWreOLzHXI57yhEoHj8Zxdm0DUDHcW/xuDTCERNTKJhC4HGlLq0lI4AfjwghDB2zl559dqSWya4a9tSOxUsYr6Kj6yb6SJFxJKlz7bU/5E9/upvLL/8vFEXB7fZwzTU/ZNy48YwZU0JGRiaHHz4LsDz9qqq9zJ5tDTzLzMziG9/4Nt/85qXk5ORy/PEn9FDXUm677Rb+859VlJaOo6xsDmBJHbff/gtaW1tQFJWSkhKuuur7g7PDosPfvhThaPY9F2Kvq5vCMfJddtCaMRnHpakJ5cd79smwNXtbxqlrDjLrwO5Tffg8rsQO2l7cwCISIvzR03zbs4b3auax0LeXBV8+RsQM4TrwKDwzT0MrmEDt3maCHz2Cu63G2TetCy9b3/sFobcfwqzbiVo4Ce+J38U1ubxHTbojWtSz94R2YRp6pzqFEBi71hN67zHMxgq0sVPxHHMZ2sTDRsRId7dbJaybhCPWOe/NAzgZAayHvgi2IIRIqbNaBFsJffBPzE1v8IMcwdaKzzg9rxKBQiRyLBFDSBlnpPPWW2sSvmdkZLJ06U+6XP+xx5505KWCgkLefHN1wu9f+9plfO1rlznfL7nkii7rKi4ewx//+D9J6/nTn+5PbQf6yQDYekcaSEXGsQ22bpgxGcfo0EEb59k7HbSuRGMf79knI76DNhwxaG4LU5jTtWcP4PdqjowjhDX+IBVjr+9YR/Dt/4dorSNXyeA8/RnIgLacqeSccBlaQUxeUlWVaiOHwkBNXGdporExAi20v3Iv+pb3UDIL8J3wXVwHzO2zNq4qCrVGNoowUdsbE+o0A00E33gAY+c6lNwSfCdfg2tS2Yga+Oh1aeiGSTD6IPb0otM8GQGsTnzL2Hfv2Qsh0LetJvT2Q4hgG+5DT+SV9dUcpX5MtZlDqauJQMWnRHQXmT4XZmsdwbceRDRVkXHhr/vVzmRIYy/pN/3roE3ds48P03QMf7xmbwo0LRZ6acbJOMlCL92u5HeqE3oZMZywy6Lc7kf3+rwu501AN2LRPl0h9BDBNx9E3/w2av54fGf/lH+uDmJueYc6I4vzTjuTsQV5CduoqkKNkc3YQI1zLOKNvV7xGbtf+wtGoAnP7LPxzDwDxd21Hp8KqqpQa1oRMlqg1lqmKES2fUjozb8jIu14j7gI94yTnA7PkYRt3NuCtrHvnwfdrkSNfXtzt5q9CLYSfOMB9O0fohZNxn/6DWiFE1m9aTUrqw5BR+XWvH/i2vUhujGXKcZW2v75ZxAG3iOXDMoDc+SdHUn6IOw//Qi9jP5NKfQyiWdv/xXCaoXL9jrNWI4Rl6Yk9ey7enV2PPuwQV1T9zH2Nn6Pi5ZAOKF8dxf6sNm4l/aX/huzYQ+e2efgKTsLRXORlbmFF0KWFp+dJPRSUxWqzRzcoR0I3apLVSxtPrzmCcLrnsVdWIr35GvRilIbudsTqqpQZ1r6vqu9DhU/mR//g+D2tywjdvw30fLH91DK8GE/cFuj0S/91ewdYx9swRTJ+0yMup20v3g3oq0R7xEX4j7sFBTVqjc7080OrHP7SWQ/5u39mIV6O3NbN6AWH4D/hKtQc8b0q41dIY29pM+ImLXvM7GBUqkPqtLjRuxGHGNvrROTcTqEXsY9TCKGFerWlffkcakoWJp9bXP3MfY2fq9GdaPl2Yf1rvVhfcda2l/5M4rqwn/6D3FNmOH8lp0ZM/DJwjwVVaHayEVBoAWsuHjNjBB86X70HWtxTzuG8WdfRV1T9+kDeoOqKDSaGZiKhqd5D1dl78C7vRLPzNPxzP0KijqyTYjtyTvGvt+evRURJ9pbEMLdScaJbHmP4Ot/Q/FlknH2T9HGHJDwe05G7Lx+FJrMfO8W5rKBLzJmM/vs7w7q29HIPlOStGAgZBw9BRnHsNc1hLO+PZLWljUSQy+t7Tp10OqiSwkHrGgTj0cjFDFobAmhALlZyWPsbXwelzOoKqInl3Eim98h+Nr9qEWT8J/0fdSsxE5f2xBoqkKGt/OtqSkK1UYOAJ5ANRmKjveNu9HrtuE96mt4ZpyE6vEBA2jsVQWBSsiTR/7e98h2qQTnXEz2nBMHrI7BxI6sao2+dfXXsw/ixUSJevb5CQ5DeP3zhN7/B9rYqfhO/B5qRm6n7eON/WZ9LM3jj+T57X5chXOZM8gy2PB3l0vSFjEQMo4zqKpnGUf0IONAx3QJcR208YOqjJ7jmn1ujWDYoLU9QobPhdZDFIvfqznROOEkxj786asEX/0LWuk0Ms74USdDDzFvPifTk/Stw9bsAbIbv+C6nBdQGnbiO/G7eGac1G37+ootU7T5SzE0H//TciLmAUcNSl2DQUcZx91Pz15VVUKKP6rZx9IlhNY9S+j9f+A6YB7+M36U1NBD3DnOcGOiUnXQV1gfmYxb699DKBWkZy/pN0OVGycWjSOc9W0Zx35WOHH2Zqzsjh20um72mOXT69EIRwyCYUFmCrln/B4X4YiJEZeszzY04Q0vEXrnYbT9ZuI/8XsoruRvCbbXl0yvB8vYh/AQdmdRVLOGNsWDWHQd7gNmJF1/ILCfcZvHn0WW38OWF7f1KzfOUGN30La2Ww9ibz89e0WBoJJBVntL9LtCaO0zhFf/C9eU+fiO/6ajzyfDPrcFOT6aAxEiumnJit28aQ4U0rOXDCu9SpcQ9xbQ0bM34zpj7e/xmr0gpuH3xrNva4+klGjMTpkQDBsJMk5ky7uE3nkY1+Q5+E+6uktDDzFDEP+qH4/91tKcdQBt/lJ+13wG6tipPbatP9h1RlQvuuZNWJYO2Ma9tT0q4/TTs1cUhaDiRwQtYz+h/n3L0B94ZI+GHmKevT3vgR5N6icHVUmGldtv/wUHHzyd88+/KOnvtkzSrxG0vQm9TJBxEkMvYzKO6nyP1+ztOlRVsXLcp+DZhyIGgaDeo16vb1/LnA0PsNt7MO3B+YR1q6M2q3k7wdX3o5UejG/Rt3vsfMt2PPvk9dke9eb9zgdFoW7PlkE3vLacZJqi2xG0I5WYjKNHvw+EZ+9HtDcxw72Tg/a+hmvyHHzHfTOlAWv2g7wgmgo7FDEwxdAMqpKe/QhG14duRqD+1DUQuXHiR8J2RbyMo5t2NI7dQWutozmePQmhl9b2sbeBnjx7b5xmn+nr2rM36vfQ/uqfcZlBLsj8AFY/QkQ3yVdbyV/7N9TsYvwnd+/R27hdKuOKMtmvJHkqA9uWGHH7m0punP5gl2/EZRFNJ8++YzSOdwA8+4CSgWir59Kst2jxj8O36Fspj0wuzvPh92ocMM7qaLcH4knPvgciX7xNZNMbnZYritLveRzd047FPbX7fDJHH13ON7/5Hd5883Wampr48Y9/xpo1H/D++++g6zq33rqcyZP3p66ull/84mcEAm2EQmGOOmoB3/3utV2W2Zsc+Q899Fi/c+S/997bzJvX+xz5RSVWfPWAZL1MJfTS0fdNZ31HxrFTI6iq892RceI8eyCl0a1ej0Z9S4jWYIRMf/LbRITbaX/pHhSXh4rya/jypX9y7LY30XJn8vWs11FMA/8p16J4M3vcN5vbrjyiy9+0uDEE9iN2sPVzVVFQSAxlHewHzEDiROM4Mk7/PHs1KuNgRAgLH59P/i/Gd5NIriMZPjcrfrDQGdEbiBp7mS4hDcjKyub++x/klVf+w09+8kN+8YtfcdVV3+fhh/+XBx/8GzfffCtZWdksX/57cnKyCAbDXH/993nvvXeYPz95VEPvcuTXsndvRb9y5D/wwEPoutnrHPkx+9yfaBxbxunNoKrO+ewdGUdLDL1UiMXeG73U7NvaI4TCRpeaffCdhxDNVfjP+DEeUcxz7WUsyNrFhE/+guYKEZr7TdS81KasS4V4ScVmKAyvqirOTGjQuwk7hpv4aBxF6f9biapAk5ILisbfW49ltid51E2q7bKT50nPvgfcUxck9b6HMsXxCSecDMC0aQcDCgsWHBP9Pp3XX38VsJKT/elPf2TDho8RQlBXV8fmzV90aex7kyP/ww8/oLKyYlhy5Nuvxv2LxrH+9iZdQrLQS0fG6RB6qaqKs8yJ4NHNpHHs8Xg9Gk1tljeYzNi3fvo2+hdv45l9Dq5xB+OrbaNdeKideDwl257lreBUjpg0p8d96g3xk5fYD7ehMvaWjJPYjnTA9uTbQwZej9bv/gZFUdjsmspJF57Klv/5iDl9LE5TVRQlTsaRnv3Ix56mUVVVPJ6YUVBVFcOwntr/+MfDtLQ089e/PoimuVm+/HbC4VCXZfYmR/6HH66momJ4cuQfNmue1cIhCr2Mj6bRO4Ve2p59YuhlvDcX38Hbo2bvib3udzT2ZnsztavuRy3eH8/ss4HYbFV7Co+gScnhiTUqx/RTMuiIqigoSuzBqCiDL+PY9cbP/JVOoZfxHrN3ALxnRQGBAlFprj8PD7dLHVIZJ6Uali9fzqJFi5g2bRpffPEFALt37+acc85x/i1atIh58+Y52yxatIhTTz3V+f3NN98cnD1IA1paWigsLMLr9VJTU81bb72e8rZ2jnz7wdHY2EhFxR7AMvbvv/8uLS0tTo78v/71z45H3tcc+YCTIx+sztuKij0ccsgMLrnkcubNm8/mzZscK9+/Dlrrb2+icex0tRDLZ+90xsaNoBWmZZhinYyxCU96em32xRnqjnH2oXf+DzMYwLfwG06onTMPbQSqsw/GQBsUb80xvGZq6XUHpE7VqrOrTJsjGVVRnLQV/dXrIdYfaF+u/TkFbk0deR20J5xwApdeeilf+9rXnGUTJkzg6aefdr7ffvvtjkGyufvuu5k6dXDjgNOBCy5YzM9//mOWLLmAoqIxzJkzt+eNovScIz9j2HLkxyR70etZe2Kbpq7Z28YmFDf9ny3XmR00exFNcayoirMsPkVyT56UN84wZMVF4+h7PkXf+h55x1yIURB724rlwNcdIzwYN7CmWpNwIIYuKkZV7Idn+hl7sM5DOMWU0z1hHYs4Ga0f1t4VZ+xHTAdteXl5t7+Hw2FWrlzJX//61wFpVLoQn2O+tHQczz77svN99uxy/vrX/wfA2LGl/OUvD6bUl9DbHPmPPPKE83moc+TbWR77Q2+icewbLBhJnCQE4jX7uERo0ckl7Aid3kbj2NgyjjB1Qu88jJJdRN6R51LXGNt/VVHweTTaQwYet4qmKoNiFG0v2/48FKiqgilixy+dZBywPPq2oD5wnr1pOtdtvzx7l0p7unXQvvLKK5SUlHDooYcmLF+6dClCCObMmcP1119PTk7OQFQnGSHEa/W9nXw5tl3qg6qSevYdc+NoMX3eknHiYsXjOmh79OyTGPvIZ69hNuzBd/LVqG4vkPiw83vt2apcg3bzqkosXfOQefaqgmmaCXP8phO2kR8wzT7Os++PZu/SVJqjQQBp00H7+OOPc/755ycse/jhhyktLSUcDnP77bezbNky7rzzzl6VW1iYOLikulrtceSjTarrDTX7UrviPUvNZXuzKsXF2SmX4Y+OKFRUpcvtnOV26GHcDaZE6wtEDXl+dJKRjEwPnrYwmqZSkG91Qufk+ikuzkY3BLk5vm7bWVLUBlge1/hxuYhIkF3rVuLb71BKyxcmtitKVoYbgYLLreHzuHp1HFLF5VLxel0IrM7oZHUMdL1uzQo+8PncqN2cp54YjOORChk+y8xlZnh6vsZ6wOtxYQid/AKrgzYnu/vrqDv8PhdVDYFo/VmDfi77beyrqqpYvXo1d9xxR8Ly0lIrvtjj8bBkyRK+853v9LrsurrWhJhi0zSJRIwen6ZDGXrZG/a1diWkDY4YmFEPsKabiaX/+4lPKCnwc8Fx1iTfLa1WVFIopCfdrrg421lut9HeBiDQHqGmpoW6Oss4B6LSUktLyPnc2mLlpK+ra6PKZ01TFwknr88mGB2Ek+lzUVvbSuijf2O0NeE98Wpqa1sT2mXj1lQam9tBmGgq3ZbfVxSgLRC23qToXEeydvUXISDQHkZTrOimvpQ/GO1KFccnEaLHa6wndN0kHDaorW0FoK0t1Pf9ErG349bmdmpqEmWmvhwzVVU6Ock2/Tb2Tz75JAsXLiQ/P99ZFggEMAyD7OxshBA899xzTJ8+vb9VoaoahqHjcvWcmEoyBMTpOAIwDB21h0RQFbVtHYrofW6cbmWcDjNVxYdeGqaJYU82rnXvMNgdtFl+NyIcIPzx87gmlaGVHNjlNll+N02tYTL9blz9zMHSFXbMu/15KNBUOwIovVIl2Ng57L0DotnbYzhi3/tKvNQ3FG/8KRn72267jRdffJHa2lquuOIK8vLyePbZZwHL2P/sZz9LWL+uro6rr74awzAwTZMpU6Zwyy239Luxfn8WLS2N5OUVjoiZ6yUxTFPQ1tqA35/cq3DWi8szD7HnRW8mL7HnetVUJUnWy7hEaJ1CL0WPUwba2Jp9ps9NeOPLEG7HM+fcbrfJ9rvZXdNKge5NOkvVQKAqCsK0ZhAYKsOrRCOAhjLccyCxz8XAROMoCZPZ9+d4uOMcjhGj2d90003cdNNNSX9btWpVp2UTJ07kqaee6lfDkpGVlUtDQw1VVbvpLrpbVVXMFEL5hpp9rV2BoO4MCqk2vWT4M8jK6n74eHzOGojz7FNIhGaH/tl5RXwezTHedvMTZ6qyjFNsoJVwEqe5e/Ds7Tj7PB9EPnkRbeLhPc7rmp3hoSUQGbAwv2RoqmI99MTQRcU4nn10RHK6YXfQDkw0TscO2r6XFR8kMGI8+5GCoigUFPQ8Ge9w6oPdsa+1699vbeOpt6wBXr/8+jwKCrr36iE2MbhNb1Ic2+sEI5Zn7/O4uo7GsUMv1dgDQDcFkWj64VRSHAMcrq9DBFvwlJ3VY/uyM9xEdJPWQAS/d3BkHCVqeIUYwtBLJZb1Mi09e7ea8Lc/2IOqYjJO/0bQOp9HyghaiSQZCR20KXjmkJg9Efo3gtbn1ZwRtJ1knKhBVJTE3Di649n3MILWozFeq+eQprdwTZ6Da+xBPbYvKzr5SH1LcNA8tXgve+gGVSkIEZsPIN2wNfv+zj8LMc/eHAjPPt7YS89eMpIx47T3VAZFQazjtGMZ8TLOWx9XUtUQ4PyFUxLW61iDz6NR2zE3TryME9WY45d1NRl4PHrlJsLrnuUHhV+CKxvvsZentG/2pCMtgcigeWq2lw1DO6jKMK0RtNKzV6L9Ttb3/mn2VnsUhqb/RXr2kj5j9smzj+UVgeSe/cdf1rH6s+ou67LxeVxxKY6tZXaysJiME9dBa5gpGfvQB//EqN6Kr/QAsk65GtWXWqxz/NyxA6EPJyM+T82Q5sYRAiMqi6UbA+nZqwM5qCp6Dbpc6pDM/iU9e0mfiffs9RQ7eDt69slCLw3D7CTrJJuMxufRHC05/rVajdNVVSUxn73uhF4mt1pG/W7Mqi145y/Gc/ipKe2TTfx0goPl2WvR1AV2+uahYDiSrw0kjmc/ICNoO2j2/SjLvkaGIi8OSGMv6Qfx9l1PVcYRidE4dhnxMpDluSY+PJJp+r5oJ6phmLFQOFWxXrWjxklRFDQlWehl8hss8tlroLpw9TBLWTKy47JjugdAMkiGZXjNIQ29VJ1+gvRLggZxnv0AvG2pHTT7/hwP9wCGhKZCGr6USUYKZgdvPLVt6CDjxDpYHS9fiE7GPdmLg89j+SoRPVFDVdXYDWmFXsYGWkW68eyFHiay+R1cB5SnLN0ktkdzyh00zT7q2ZtD2FlqZ70cyjoHksHU7Aci9LKnMOCBQhp7SZ9JkHF64dknG1QFMe/dMEQnjd7sQsax6jad9RVFcW5IO/QyflCV3o1nr+9YB+EA7mnHprQvHVEUxdHtBzXO3tHsB6WKpHXaxl5LSxlnoOPs42XD/nv2gzXauiPS2Ev6TJ9CL+NmPAIwiX87EM46HUfUJu+gjTP2zmt1TGO2Qy9d8R200XYm87wjm99GycxHKz04pX1Jhm3sB2sErSNRDWHopaLG6lTS0bN3Daxmb4r4gIC+lyU9e0nakBB6mUKcvL1NsmgcqwzLEBtCdArltMuPl18cGccwE2UcW1eNdig6nn03oZdmezPGrg24DzwSpR8hJ3YnbU/pGPpKvJetDlFojPXwJG07aPcvzeGQyfmMK8rsd1mqkxtn4Dx7qdlLRjymKRxvSTdM3v6kktrG9i7Xt6MYRJy1Twjf7EbGsW8ub5zuao9S1fVEGccOFRTCkjrsCU2sQVXJNXt96wcgDFwHHdmLI9CZwZZx1DgZZ6g8e1s6Std89gU5PpYuLiPT1/8Eikp0gNnAaPZK9K809pIRjimEY9TaQzp/ffYz3vqkstv1rb+xZfH6fbyM01Hbtz37eN3V9ux1IzbgSlVinWh2h2JCB20Xnr3+5Qeo+RPQCiamvP/JyPbbnv1gRuPYkUaDUkXSOoUzSG1o6hypxOaglZ69ZBRhmsIxvs1tEaD7jlo7oqbrDtqojGN7+PFvANEV7RtDUxXncySug1ZVLRnHNAUmUU9fUVCwc+N0NvZmoBFj72ZcB6Q+N3BXDEUHrS2FDUvWy1Fu7e0Be84ctP0oa6jj7KWxl/SZeBmnqc2aUKS7jtqYZ995UBXEPPuORt+uC2Ix05qmODeJrpsJnpaqKo5cZNsmTbM8Yrt98YZS3/YhIHAd0P1cy6mQk+mJtnOQOmjjR9AOuYyTnnH2A4nKwCVCG+ww3U71DUktkn0SU0SHeoMzl2a3xt7sLOPEf7Y1e3u9jgOtIBYr7VJV5ybRDTPmaSmgEAu9tKNHLK3bRAjrjSD+JtW3rUHNK0XLH9/rY9ARe2DVYIdeiiFNhEbsAZOGHbQDiaImpkuQg6okowI77lrTVJoDPcs49g2SLF0CxAZmxWSc2IPD0ezt2GRNweWybjRLxrHWswdV2VKHbZw0VbVG0BqJk42b7c0YlZ/j2r//Xj3A+OJMfB6NMdF5bwea4UhdYL0ppW8++4EkNnmJ9X0gBlXJdAmSEY9987s0xfHsuxtJa98gCekSknTCGh08fIhp+3YfgaapMRnHSOwws1PyxmdpdDxiUyR4Uvr2j0CIATP2Y/Iz+NP1CwekrGTYD7KhzWcflXGkZ59k8pL08exTMvbLly9n1apV7Nmzh5UrVzJ16lQAFi1ahMfjwev1ArB06VKOOeYYANatW8fNN99MKBRi/Pjx/Pa3v6WwsHCQdkMyHNi6sUtTaW23PPtICjJOfKdsshG0ZgejH//ZNvYuTYnJOLqZ8Fody+USi1jRVAXDsHLuxA9i0betQckZg1q4X6/3fziIyThDnBtHID174qNx7O99L2tEdtCecMIJPPzww4wf31nTvPvuu3n66ad5+umnHUNvmiY33HADN998M6tWraK8vJw777xzYFsuGXY6hjZC93ntHWOeoowTP4rW7CTjxDz7iGE6N19C6GWc92s/ACK66QxPF8FWjD2f4d6/fEhSzA4Eqqoi7NDLoU6EJqNxOkfjpJFnn1It5eXllJaWplzohg0b8Hq9lJdbr8aLFy/mhRde6FsLJSMW+7XeFTeSs7sO2uSafez37mQc+wHhyDiq6uQD75gbx+5QFEI4RlyLdtDqhoi9EexYaw2kGiAJZyhwpggcwjw1if0EQ1LliCU+fTYMlGY/NAe135r90qVLEUIwZ84crr/+enJycqisrGTcuHHOOgUFBZimSWNjI3l5eSmXXVjY85ymXVFc3PushUPBvtQuzWV51/Z8rQCqpnZZlqlF11MUZx1X3CCprGwfxcXZzsMgJ9fvtK2yKQhAbrYPAL/PxdiSHAC8Po8jaRQXZ+Nxu/B4XCiKQobPTXFxNl6PhtvtImRE8PtcFBdns/e1j9Fyiig55PA+efbDcS6zMr0IQCDIyvImbcNAtysz04PACvvM8Hv6XP5IvfYh9bZlRo9/do51HRbkZ/Z5v1zREb15uf4uyxjIY9YvY//www9TWlpKOBzm9ttvZ9myZQMq19TVtSZNgNUT+9rE3oNNX9sVCukoHi1hAodAe6TLsmobAoAl19jrhEK683t9fYCamhYnoqe2tpVJY3OoqWmh3t5Wt9YXQtDU0AZAY1PAyUXTUN+KYZq0ByNEdINwRKempgUhoK09TGtbGE1VqK5qJLBtA+4p86itbe31vg/XuQyFIlY6CSEIBTsf68FoVzioW0nkIiaRsN6n8kfqtQ+9a1t7IIwwBY3RtCCNjQFq+ji5fChi4HapqEIkrb8vx0xVlS6d5H6JRba04/F4WLJkCR999JGzvKKiwlmvvr4eVVV75dVLRj52HHu8Zt/djFVdReM4E4I7g6l6HlTlUpU4GUd0lnGEwDRJiMYxTUEoYuJ1a5g12yDSjjb+0L4fgGEgITJmyEbQ0mncwmhFUbDerAZAs/e6NW698giOmjF2gFrXPX029oFAgJYW66kjhOC5555j+vTpAMyYMYNgMMiaNWsAePTRRzn11N5N8SYZ+ZgmaIqSEE3QfbqE5NE49vZ2si379+TGPtZBa08mrhsmdnYcNT70klg0jp1ALBgx8Ho09D2fAqCNn96PIzD0JHSWDpFmbz0oo+MqRrmxj58bAej3w29Mnn9kxdnfdtttvPjii9TW1nLFFVeQl5fHvffey9VXX41hGJimyZQpU7jlllsAK2Lgjjvu4JZbbkkIvZTsWzihl/GpB1IaQZsYjeN2qYQiRqdsl/Ex+7ERtHboZczoR/T43Dhx+dfjDKJLVTAMk3DEwOvWMPZsRC2c1KcZqYYTe1APDGHoZbROOYI2Fldvv32m07MvJWN/0003cdNNN3Va/tRTT3W5zezZs1m5cmWfGyYZ+diTXmupevais2dvCuFEI+immdSbj9/WSYSmJcozsbhnBRWcWOj40EvDFATDBhmagVGxBc9hp/Rxz4ePeAM/lIOqAGnsiRl3I042TBdkugRJn7Fv/vjX0O5H0Cbz7DvIOEkGUsV/9nbw7G0jHq+h2lJH59BLQThiMMaoBNNAG5deEg4kGvghM/bRenTDlHH2SixdtvV9OFvTO6Sxl/QZu3PV9swVepJx7L+dZRwgOsI19puexLOPJUKLGnFNSXhIKPagKkiIC9c0lXDEenMoDu0CRUErObDvOz9MJBj7IcyNA9ZI5SGaHGvEYh9yU3r2ktGENYFGTMbJynD3QcZJ9Oy7lHE6pTiO5bW3ZRyFuBTHZsdEaAqBkJXSIb99J2rhJBSPvz+7PyzEG/ih1OzBikIZ7TKOQmIHbToZ0HRqq2SEYeVKiY0AzM309LmDFqxOr65kHPutwO1OzAFuhSKaTv8B2EPaE7M0aqpCIKijYZDVthutdFq/9n24GBbNfhjqHKmo0rOXjEbseVDtdAk5mR7Hs7/36Q2s/rw6Yf1kqY2tDtp4GScurXF8NI49B23c5CXWX9UJ2XTCLBXFyV9iZSk00RRBIKQz0VWHKnS0sVMH7DgMJcOi2Svxn9PHuA0GTjSOkJq9ZBRhpxDu6NkLIfjoixo2725MWD/ZNINC4GSh7CjjJJN0bM/e1UHGEXEDqFQlKuNElwVfvIeTmv4FwuRAV5W13diDBuw4DCXxBn4oJxwf6jpHKk6cvSESvqcDMp+9pM/YoY22t2NPyacbZsKo1tj6nROgWfKLiqJ0L+MYHTR7+wHjjCiNG91pyzhCCDRFoO/eyFgjzFn+j5jp2UkkswTVnzOQh2LI0IahgzZ+4JAcQWv9laGXklGFEefZK0B2hmXsAyEDoLOxN+M/2zKOdQNpqtopGiepZ+9SufD4A5k3vQSInx81Fnnj5F83BdnhajDCBNUMTvB/il8J0zbrvwb0OAwl8bZlqDtorc9DUuWIJTaoKv1kHOnZS/qM3QE6b3oJORkeR1ppjyY3i/fkO353PPvoA8MOoUw272z8tqqicOoRsYlG7InEO8o4pmklUMgL7gHg3aLz8ex8n9eDB/P9kvSUcCDRwCtD5Kolyjij2z/sGHqZTn0Y0thL+oydjGv/0hz2L83h1Y92AzFjb3Tw7EUS42179q7oTFIJ0xQmSZfQ0Zu1B1WZxAZQ2YOqAHLb96D4c2jyj+fVwHwAfO6+ZSkcCajDYHilZx/DPhZyUJVkVNExGZft2QeCuvN7wvpJonEEwkloZnRIl5CsQ7ejfdOU2NyytiFSlFj2zdz23WhjpiSkdIjPv59uJBpeGXo51Nh778g4pM/xkMZe0mc6ptl1jH0Xnn38V/uzsDV7TUU3RYI3n8zwdzQ2towT30GrKtZbQoYSJCNch1oyJeGNwJvGnv1wRMZIYx9DpkuQjDpMIaIjKmPL7Nj3mGafuE18DH0s9DI6CteWcXrooO3ozarRNwJ7tKzZWk9Z08tg6uznqrPaNWaKI3nEh4qmIwme/RDdvTLOPoYTjTMA+eyHGmnsJX0imYbu7kHGEWb8Z+GsY0XjJJNxEh8Odt6beDrKOJEt7zG17UPGs5eJWtTYF012PFKvR0urcLmODGcitKGsc6QS0+ytazOdLiVp7CV9wskyGS8rdJBxutPs42UcO01y96GXyb0oTVNjMo6iYNZuB2ActUx01dPuLUTx+J3EaV53el/yCTLOEE44nuzzaMSOgErHQVXpfeVLhg3b6U7U7KMyTrALzb5DtkuIeuwosXj5hMlLEmWcZBp1LMVx9HPdDgAmajVM0OpoyyhNaGc66/WQOKhpeDz7IalyxBKf2x+kZy8ZBRhJNPSOHbTdxdnHp0tQozJOx8lLOkbjJBu9GT+oyqtEEE1WOoQDXXsp1NoIZIy31tNiMk46oynDbezTyLoNAp0HVaXP8Ugpzn758uWsWrWKPXv2sHLlSqZOnUpDQwM/+tGP2LlzJx6Ph0mTJrFs2TIKCgoAmDZtGlOnTkWNugJ33HEH06alZ6ZBSWfMJDJOp0FVqUTjROPjXUlknI6dtclkC3sAlWkKSqkFoMY3ieKg5eG3Z0WNvbJvePbDkwhNyjg29t470TjD15Rek5Jnf8IJJ/Dwww8zfvx4Z5miKFx55ZWsWrWKlStXMnHiRO68886E7R599FGefvppnn76aWno9zGSRcfYMk4gaOWNT0nGSeig7RiNk9hBm8y42SNvhYCximXsd+XOdn4PZpZG17Mu9XT37GUitOEl3rNXSC/PPiVjX15eTmlpacKyvLw8jjjiCOf7rFmzqKioGNjWSUYsyTz7WAdtV7lxOnvtsQ5aKxpHT0hxnLh+UmMfJ+OMpRYlI4/azAMxBdQaWZjuTGc9SH/PfjgSoUnPPoaTzz5uyst0YUDSJZimySOPPMKiRYsSll9yySUYhsGxxx7L1Vdfjcfj6VW5hYVZfW5TcXF2n7cdTPaZdrmsSyc3x+9sq0dDFUK6Zew1TU0oNyMzdv7z8zMpLs5GURQy/B4yQgZhPUhmphcAj0vF5XY5bfN4XLg6lAeQmeGx0iS7XZRQi3/cFHyZ2ewwiqgychkTbV9urjUrVW62b8DOwXCcy+bogxSgqDAraRsGul31gYjzOS8/o8/lj9RrH1JvW151G2Bd26o6+Ps0kOUPiLG/9dZbycjI4OKLL3aWvfbaa5SWltLa2soNN9zAihUr+MEPftCrcuvqWjt5h6lQXJxNTU1Lr7cbbPaldtU2tgPQ1hZ0tm1uDgLQGjUOwbCeUG5zSzC2fV0rPtVKhxwMRRCGSXtIp6nJKtftUgm0hwGoqWkhEAijIDq1MxzW0Q2TcLCdAtGAnr2AYCDCiuaTMVG4Itq+QFsIAGGaA3IOhutc2sfH+hygxpXoXQ5Gu5qbY3W2tgT7VP5Ivfahd22zj0UwpAPKoO5TX46ZqipdOsn9jsZZvnw5O3bs4A9/+IPTGQs4sk9WVhYXXHABH330UX+rkowg7BGEWhIZJ9hVB22CZm//FagouF0qkYjpTDLudqmJkTlJNHthGsyt+hcz1C3k6HWoCNSCCSiKQgQXBrEBVHY70zkJGsgO2uHG0ewNM+2SwvXL2N91111s2LCBFStWJEg0TU1NBIOWF6frOqtWrWL69On9a6lkRJGsg9aecUp0WMfZJj4ap0M+e7dLJWLEJi/xuLROWS87GprIZ68xtu0L5ri2UqBbnbNqwYSkw/udDtp0N/bx6SmGRbMfkipHLLY/a5jJQ4FHMinJOLfddhsvvvgitbW1XHHFFeTl5fGHP/yBP//5z0yePJnFixcDMGHCBFasWMGXX37JzTffjKIo6LpOWVkZ11577aDuiGRocYx9Es/eWadDnH2yFMf2ROFul0o4YiRMP2iIrjtoRaiN8JonAZisVRMySjBQUXNLUJVdznr2NrbB8qS5sZcTjg8v8dE46XYoUjL2N910EzfddFOn5Zs2bUq6fllZGStXruxfyyQjGttuJwu9tOmc9TKZjJPo2euOZ68mRuOIREMT+fwNRKiVnQXz2a/+Pabom2lQC8hTXYmjTKMf7UFVvn0o9FIa+6HHdmdMU6RVemOQI2glfSRZIjRVSbz8O8o4yScct0LY3C6NSMR0pBu3S+s0wCpetjDqdqBkFVJROA+AbNFCnVoYbUeszo6affrLONLYDyeOZx9NzJdOSGMv6RO2sU6YjFpRcLlil1T3nn1nzV4AEd20Zq7SlMS4/A7pEsyGCtT88ei+AlpMHwANWhGQvEPRMfZp7tkPSz77JH0go5X4aQnTLc5eGntJn+gqv3y8lNNZs++8vRBWx6sn+pAIhg00VXHy1NsYcYnQhGliNlai5o9D1VS+1McAUK8VA4mjGu0ONZ/HUiyz/O6+7fAIQVU7P8gGvU4ZjeOQzpq9NPaSPmEkkXGs77FLqvtpCWPLLBnHNvY6mqriUtXOI2ijN5poqQEjgpY/HlVV2BIpwRQKDS7L2CcziPuXZvPDi2Zx0ITcfu33cDMckopMlxAjfg7adPPs5YTjkj7h5LPvcL27Xd0Y+6SavVWGPfFJMGxY6RNUpctoHKNhj1V3/ji0NoW3QtOo8uyHT8vq1Cb7hlQUhUP3L+jz/o4U4j3roTK8CVLdKDf2zkxVptTsJaOELueEjfveUzSO/cBQFAW3O4mMYyQ+HJy8JA1WDiY1bxyaqmCiskfPSzDsNvuabRqW0Mt9+Hj2loREaGlm7aWxl/SJZInQIJbmOH4d53sHz97+WVHArVkdp7axtxOjxW9rGzqzYQ9KZgGKx+/UrxtmLKZ+H44eGQ7NXso4MZxBVYaZdv0X0thL+kSyQVXQwdj3EI1jxnn2HndMs7dlnK6iccyGPagF0Tz18cY+2pRkMs6+gn28h9Loyg7aGHZwsSlDLyWjBWdawi6icRSS5bNP/Byv+8dr9lrU2Otx29uTl4hIyIrEyRsHxDqEdSP2Wr0vGyd7LMNQvrHsy29KvSVes0+3a0sae0mfsL3yjh6mK26SkHhdPn4bsD1767OaRLPXOkXjWIYm8ukrYERwHzC3U/22IYr35pV98ApXo30aQ1dfYt2jmcRonGFuTC/ZB28FyVDQU5y9PXgp2byz9ueEDtroQyIUF43TUcZxKzrhj59HG38oWsmBVv1JUiMkGKd0uyNTQFWVId2vfflNqbckRuOk17GQxl7SJ5KNoIWYZ28PYko2OxXY0TjWZ3sErV2upqrRQVWJMs7U0EZEezOeOec4y+M9+9Eg44C1T0Op2SuK4hi50e7Z29eYndMpnZDGXtInuhpUZRt7f9SzT8yHQ9znDh20rlgag2TROMIUFEUqUTILcI2dmrCujW3YE2ScNLshU2GoZRyIHdt98eHZG+J3P92OhTT2kj4Rk3ESl3fMLtnRs7dlnvjQS1VJHIyl2pp9B88+x2hAzS1JqC+x8zDZsvS6IVPB7sAeSmJhrUNa7YhDTWNHYpSfOklf6Sr00t1RxukwYYkdPSNMEjz7eGOvaZYxEyJ+khNBlt7Z2CeXcei0bF9CVYZ+cNNwhHyORJQ0vraksZf0CWdQVYcLvqNn3zGtcSqevRanSdtSjtdsxyuCSYx93BtBUs2+jzs4ghlOGSfdDNxAk879QdLYS/pEsnz2EN9Bm0TGEXGjYDtE42hqYieg/dCwwy/zabJ+yxmbUF+yEaWJWS/T64ZMBSudxNDeupoj4+x7x7M3pHN/UI9XzPLly1m0aBHTpk3jiy++cJZv27aNiy66iFNOOYWLLrqI7du3p/SbZN/AtuE9ReN07GS1py6Ml2gUJbGT1pJxooOlousURI290q2MY/3d10MvFWUYNHvFGii3Lx7P3rBPyzgnnHACDz/8MOPHj09Yfsstt7BkyRJWrVrFkiVLuPnmm1P6TbJv0LVn36GDNiEChy5kHGuZLeUcEv6Yqdsfw0PEmbmqQGlGoKDmFCfUl2xQVTp3oqWCpipDLk8pwyAdjUTSOclej8a+vLyc0tLShGV1dXV8+umnnHnmmQCceeaZfPrpp9TX13f7m2TfwehyUFVnGWfj9noq69qi0Th2B22ijAMxYz8ttJHChk/4TvZ/0IMBAAqVJtpdOSha4uQjPco4+6C1Hw7NXpPGHkjvzv8+5bOvrKykpKQETbNfuzXGjBlDZWUlQogufyso6F0+8cLCrL40D4Di4uw+bzuY7CvtysjwADCmOBufN3YZ5WRbUwQWRc9dbq6fux5bT9nUYlRNxRuVdzIzveQXZEbX8VEcLUdrCZBv1BDMncSkxp20PPUb9rv4ZorUZkK+ok7tjMTNepuZ6aG4OJv8xmDCfhXk+Hq1b6kyXOfS49bwuLUu6x+MdrlcGpqq96vskXrtQ+ptc/s8zmePp+tzMFAMZPkjevKSurrWTpkTU6G4OJuampZBaFH/2Jfa1dxiGdT6+raESJr8DDcTijPRwxEA6uraCIZ0mlqChMO64803tQSprWsFoLUlRE1NC5oC47QGNEx2lyzgP7v35xs1b7Lz/h8xRm2iyrV/p3Y2NbY7n4PBCDU1LbQ0x5Y11LdhhCK92rdUGM5zKUyBaZpJ6x+0dkVzwfS17JF67UPv2tYSCDufdT35ORgo+nLMVFXp0knuk7EvLS2lqqoKwzDQNA3DMKiurqa0tBQhRJe/SfYdYvnsE5eXHzyG8oPHsPaLGsCSe3TDJKKbVuilPY9shw5asGScca46AEI5E/g44sJ18rWYa57Aq+i0+jtfQ2pCB+3oGEGrDHFuHBj6fDwjlX1as09GYWEh06dP55lnngHgmWeeYfr06RQUFHT7m2TfoatEaDZqXIilbggiholp4kTjmGZcB210XbemMlGrI6T6Ef5CAMS4w/Cfews3NVxAVV5Zp3oS0yXYf/f90MvhGEG7Lx7L3pLY+Z9ex6NHz/62227jxRdfpLa2liuuuIK8vDyeffZZfvGLX3DjjTfypz/9iZycHJYvX+5s091vkn0De/KGri54Z75YU2AYJrrt2UejcQSJHbTB9x7ljPAX+Nx1NHrGomp2nnoTU1FoEX5nWTya1nlQlbKPh14W5vjwuIY2zl5VpLGHjrlxhq8dfaFHY3/TTTdx0003dVo+ZcoU/vnPfybdprvfJPsGRtw0gclwpgvUTQTEZJwknr072EDkkxeZLEzQYJP3UNzxDwszscyEehJy14+O0Mtvn33okNepqvvmg7O3pLNnL0fQSvqEMLu/+bXob6GIAUSNvSkYa1RSojZamn3U2hfseQtQWJ+5AIAG/34JI2i7k4x6lHHS7IZMheFKl7AvHsveks6DqkZ0NI5k5GIK0a3BsX8L65Zbbmn2goWtz3FgZgYVogwhwKeEyd37Aa4p89gUWMCju8dzzJQDmBC9kXTTJBgNgHAnkS6Sx9kn/13Sd4ajn2Akks6d/9Kzl/SJ7ubgFEJQ+MEKjvR+QTjOs3eJEFlmM/tpdVb4oBDs76pBNUK4Dz4Wt0slIHyomhrryDUEO6qs8LOJYzqHlNlvABCX9TLJA0DSPxRV6ZQaYzQi89lLRh3defZmwx48dZuZ790S8+x1kwKzEQC/GsEXrEUIKFCtWHs1dyxuOzeOqjpepG6abKtoRlFgUknnASb2BNzW59gymzS7H0cslowz3K0YftL52pLGXtInwhEjwauOx9j9CQD7abWY7ZZXHtZNioilzMgJ7kYIQYHailA0lIxcR6aJH5pvGIJtlc2ML8py5rXtiDPRuJoo4yikn646UpEyjkU6a/bS2Ev6xK7qVsYXZSb9Td+9EaF5UBXIbNwMWJ59sdKAgUZQuMgJVEQ9+zZ0Xz6KojrhhGqcYdENk22VzRwwruth4/ZDR+0g40i9fuCwJkyRx1Nq9pJRRThisLu6jf1Lczr9JvQwRuUmwpOPpM30UNz4Cd/Kepn5rk8ppoEWdyG7jSJyghWYQlCgtWL484FYB6xLVRzNfnd1K21BnQPG5XbZHifXegcZJ908r5GM1+PC08Wb1WgjXefjldE4kl6zs6oVUwgOSGLsjcpNYEQQpTPY9Ol2Zrd+wRgPTHLVYqLS5Nmf3W0qB4Q+JWBEKFBbMfxTgJixj/fsP91mpU9I9mCxiQ2mSpRxpGM/cFy46EAn3fRoR1EAIT17ySjgy8pmAPYfl2iAzZYagm89CJ4MxJipfBA+kDYtl2cCZWSpIXLUdlo9xew2ilExcTdsJ0cNYmRYqTTijb0t6Xz4eTUet8q4oowu22O/BXT0uGT0yMAxJs9PaWFy2W60oUjPXjJa2FbZTEGOl7wsr7NMmCaBZ+5AhNrIOP2HhNwePouM599jZvNezV7meLZR6mqkzVvMDtPKSe/fsxoA059o7F2qQn62l8tPOxgDhcIsd8Jcsx3ROnr0SUbSSiQDhe1DpNvlJT17Sa/ZVtHcSVYRLdWIlhq8R1yINmaKY5zDERNQeDU4HUMotPhKaVP8NLuLyKhaB+B49va0hKqqoCgKx84cx4UnTuXwKUXdtqejRx/7OyC7K5EkkCy7ajogjb2kV7S2R6hubO9k7I26XQBoRZOBuBG00UFV74cP5BeN5xP25KGqCrW+SaiGNTTWzLAyXLriZJzeoHXw5J3QyzS7GSXpgSI9e8looL7ZmrRkTJ4/YblZvxsUBTV/HBAzwLaxB4VmkYGqWka4xrsfAIZQwJ8H4Oj0rm4km2TYoZd2tksZeikZTKRnLxkVtLRbsz5lZyTOBWvW70LNHYvisqZts+1sSE+M4LBHYlZ7JiJQaDAzUaLG3d1Hz76jRi9lHMlgEgvxHd529BZp7CW9wp6WLTvDk7DcqNuFWjDR+d5RxnGWK5YeH1T9hLLHU23kOMY5fgRtb9A6RuFIGUcyiEjPXjIqaAl09uxFuB3RUoNaMMFZZhvsSAfP3p5ST5iCikMv5//aFjjGeUJxFifPncjBk/J71SZHxumio1YiGUiS5WBKB/oVerl7926+973vOd9bWlpobW3lgw8+YNGiRXg8HrxeKzxv6dKlHHPMMf1rrWTIee/TvYTCBgtnjQcsY68okOmPGXuzYQ8AWmFnzz7UwbPXVAVFsRKpRTw5tAi/Y6RdmsriEw7qdRtjGn3y7xLJQKJ0eINMF/pl7CdMmMDTTz/tfL/99tsxjNjNfffddzN16tT+VCEZZt5YV0EgpDvGvjUQJsvvTvBq7EicBBlHsWWcJJq9qiAECGdZ/9qoKdKzlwwd6SoTDpjvEw6HWblyJeeff/5AFSkZAQSCOrohnO8tgUgnvd6s3QGeDJSsQmeZolgefGfN3vrNFIlz0PaHjiNo0/VmlKQHo9Kzj+eVV16hpKSEQw+NzY+5dOlShBDMmTOH66+/npycrvObSEYmgZCe4CE3B8Jk+xMjcYzqLWhjDuhkXDVVSXhQgK3Zgxk3LWF/b5qOso0SzXEvQy8lg8Go1OzjefzxxxO8+ocffpjS0lLC4TC33347y5Yt48477+xVmYWFnWcmSpXi4q5T4g4nI6FdtY3tvLF2N+cdd6BjoLtqVzBs4PVozu+BkMHk0hznuxlqp6V+D/mHLiC/QxmapqIbiZ59brYPt1vD7dbIyvIBUFSYRXFx1+e6p2Pm91kPn/y8DGddVVXwuLVBPd4j4VwmQ7ar9/SmbVp0pHdGhmfQ92kgyx8QY19VVcXq1au54447nGWlpaUAeDwelixZwne+851el1tX1+pMNt0biouzqalp6fV2g81Iadcj/9nMS2t2MXVcDiUFGV22yxSCtqAVfWP/3tgSxDMh1/mu7/kUEASzJnQqw/Z71KhsA9AWCGMagmAwQlNTOwANDW24SX6eUzlmhm49UJqbg866iqJgGOagHe+Rci47ItvVe3rbNmFa/VDBYGRQ96kvx0xVlS6d5AHR7J988kkWLlxIfr4VMhcIBGhpsRophOC5555j+vTpA1GVZAD4eGstAFUNgW7XC4UNhLAmCwcwTJO2oJ4QdmlUbQFAG3NAp+3t8Eu/V0tYpipYHbRR+97f1+HYoKq4ZXKyDckgMao1+yeffJKf/exnzve6ujquvvpqDMPANE2mTJnCLbfcMhBVSfrJ3voAVQ3t0c/tHD6l63UDQR0APRor39pufY/voDWqt6LmjUPxdk5/axthl6bi0lR0w0RRLN1+QDtonayXcbMIqYoMvZQMCqM6xfGqVasSvk+cOJGnnnpqIIqWDDDrt1hevUtTqKrv3rNvD1nG3TAFphBxo2ctz14IgVm1FW1SWdLtY8Zewe2yjL2dLsEUwhFuBq6DNlaQVU963YyS9CCW4ji9ri+Zz36U8fHWOsYXZeJxa+yNM/Y7q1rweV2MyfOzp6YVVVUIRI09WN69M3o2Go0j2psQoVa0ov2S1mUbW01TcbsE7SHLIKuKFWdv98cMlGcfb9ztEE+JZKBJ19xL8kV3lLG1oomDJ+UztsCfoNnf/8yn/ONla3Lwvz33OQ+9+IUj44A18XfHvDiirREAJasgaV1anIzjjouFV1QF04zJOP0eVKV21lCV6BuERDLQpOs4Dmns0wwhBF/sanQMZW+I6CbhiElOpoeS/Azqm0POoKe2oE5No6XlVzcEqG8JEQhFErbtmBdHBBoAUDOS57LpKOPYy9ToftiBVv2dPlBVEwdVOfVIay8ZBNK1g1Ya+zRjW2ULv3n4I7ZWNPd6W1uDz/C6KCmw5nStjnbWhsIGdc1B2kM6bUGdxtYQ7aFYjHwkzrO38+KYtmef2YWxj8t54xj7aLoEUxDn2Q+QjKNKGUcy+KSrZy81+zQjEI17t//2hnhjPzZq7G3dPhQxMEzBrupW63vYoL4l6Gwb0U1a2iNk+ly4opKMCDQCCoo/+choR8ZRFXA8e+smEabphF72956RMo5kKFHSVLOXxj7NsFMGd0wdnAp2h6vf52JMvjXTVFVDgIhuYkQ1lU27Gp31K2tjmr5uCNraIwnZLkVbA4o/B0WNxdHHY3vamqaiqjEv3o7GcdIl0F8ZJ1kHrdJveUgiSYaMxpEMCfYAJ/tvb7A7XDO8LvxeFzmZHmoa2wmGYx2xm3Y2OJ8r46J1bL3f544ZdjPQ2KWEA4lx9nagpT2ZePygqoHy7BNkHDX94qAl6UG6avbS2KcZutF/zz7DZ532TJ+LQMhw5B2ALXuanM81UT3frjcUMfDEGXsRaEDJjGW67EhMs1ewkyc4mn18NE4/PfCuZZw0uxslaYGj2ffzjXSokR20aYZt5PU+GPt4zR7A73URDOkE44x9OGKSZXfACuFczhHDJKwbeNyxS0a0NaJm5nVZX0LopdNBa5l9EZf1sr9qS5cyTnrdi5I0wYmzTzOZUBr7NMNOGRwxeh96acs4ftvYezTawzrBcGJmyv1KspxOWFujt2UcTzTjnzB0RLAFpYuwS+gm9NJJl2CtNxiDqvxezdlPiWQgkTKOZEiIddAaPazZmUDImlLQ57EMts/rorE17Hj8Xo9GKGxQlOujuqGd2qYguZkeWtsj6LpJOBLz7K1IHFBS8Oy1aG4ciNfsxYDls9eicfbx5Xz7nBl4XdKXkQw86ZrPXt4NaYbdMdtxUpBUCAR1MrwuxzPxe1y0h3VCUc9+fJGVzKww109uljVKNifT49Qb1k1Hs7eNfVcDqiB2U7g7jKC1Jy8ZKM8+WW6cMXl+crO8/SpXIklGunr20tinGf0NvbQ7ZwF8Xo32uA7acVFjX5TjIy/TMpS2sbc9e29UxjHbrKid7jx7e2Sr1kHGUZSO6RL6d9fYbxv224NEMpjIQVWSIUF3PPs+dNAGdTK8sTh5v8fqoLWN/YTobFFFeT7Hs8+N8+xDEbOzjJOR12V9yTto7QnHrXQJA3G/HDG9hIJsr9OxLJEMJunq2Utjn2bo/fTs4ycS8XtdCKChJQTA/ENKyPS5mDI+l893NgIxzz4cMdGNDjKOqqH4up5OMFkHraJY/+x89gOhe/q9Lg6fUtTvciSSVFCkZi8ZCiL9jLPP8MW8X1/U8Nc3W2kRMnwuFhxWiqoo5EWNfE40w6WdFM327PXKTdakJUrXl5AtobvUmGavxac4FiLtvCOJxDby6XbpSmOfZtiefV9kHLuD1sbvsT43NAed2aRs8rITNfu2aNimx6VhNFRgVm3BPfWobutzPHuX2kGzxwm9TDfdUyJJ1zj7fss4ixYtwuPx4PVaxmHp0qUcc8wxrFu3jptvvplQKMT48eP57W9/S2Fh16MtJanRH8++vUMHrS3p1DUHnXBMm4P3y+f8hQcwfVIemqo4Mfoet0pk0xugaLgOWtBtffGJ0BLi7J10CQMj40gkQ0msg3Z429FbBkSzv/vuu5k6darz3TRNbrjhBn79619TXl7On/70J+68805+/etfD0R1oxonGqeXnr1hmgTDRsJAI1/Us69vCuJ1J77kuV0qZxw52fnc1m7JOF5NoH/xNq5Js1C7yHZpE58IbeaBRZx3zP4U5/njonHS74aRSGIdtOl18Q6KjLNhwwa8Xi/l5eUALF68mBdeeGEwqhp19NWzt3PTJ8g40c+NLUG8nq6f+y5NdWScgoZPEcEW3NMX9lhnfG6cLL+bsxbs3yEaR6TdDSORjGrPfunSpQghmDNnDtdffz2VlZWMGzfO+b2goADTNGlsbCQvL28gqhy19FWz75gEDax0CWANcPK6k6cphqhnH82fn1fxNkpuCdqEGT3WGR96GY9iD6oy0y8nuEQSy2efXhdvv439ww8/TGlpKeFwmNtvv51ly5Zx0kknDUTbKCzsOqyvJ4qLswekDQNNf9ulRAcqCUXpVVnNUc9+7JhsZztfZmyEaXamJ2l5emsDxe4glUEvE7Q6vI3bKTjpCnLH5PZYZ2a0/IL8jISyM6MRPl6fC1VVe9yPffVcDhayXb2nN23zRyPacnP9g75PA1l+v419aWkpAB6PhyVLlvCd73yHSy+9lIqKCmed+vp6VFXttVdfV9eKafY+LUBxcTY1NS293m6wGYh2BaLaeTAY6VVZe/Za0xjqodh28W8HKnQqTwhB4PFfcAW1/Dp4FqdnrENoHkLj56ZUd8ieVastlLB+KKhjmCaB9ggKotuy9uVzORjIdvWe3rYtHJ3/oaU5OKj71JdjpqpKl05yvzT7QCBAS4vVGCEEzz33HNOnT2fGjBkEg0HWrFkDwKOPPsqpp57an6okUfqaz75jxkuw5BVPNEqmYzQOgLH7E8z63WQQ5Ic5z3CoZw+RGWejeDJSqjO+gzYeR8aRmr0kDRmV6RLq6uq4+uqrMQwD0zSZMmUKt9xyC6qqcscdd3DLLbckhF5K+k9fo3HsQVEZHdL++rwuwno4YVISm/DHL6Bk5vNmaDpH6++wJrQ/s6afmHKdWtwI2nhUVUFEc+Ok2f0ikTiTlqRbf1O/jP3EiRN56qmnkv42e/ZsVq5c2Z/iJUlwsl72NRrHl3jK/R6N5rZEz16YBuGPX8DY8yneIy7kkw0lrK/P4Eu9hHlJ3gC6wvbs3V149qaZft6RRBLtNku7OY7lCNo0Q+/jHLTNbWE0VcGXxLOHxGic4H9WEP7gn7gmz8Z9yAm43Bpf6OPQ0ZzJS1LBjlboKOOo0Xz21qCqXu2GRDLsxKJxhrkhvUQmQkszYimOe9dxXdccpCDH2ylczA6/tD17ffcG9O0f4Sn/Cp6ys1AUxfHMFaWzJNMdXck4imLNVGXKdAmSNERNU81eevZpRnyKYzsffCrUNrVTlOvvtNzusPV6NIQwCb3/T5TsIjwzT3MuZjvVgcet9eoCd3LjqB09e2S6BEnakq4pjqWxTyNMIdAN4XjKvZmtqq4pSGGOr9Nyf5yME17/AmbdDrzlX0HRYtkxbc++t9P8xaYl7NBBG71LZNZLSToi0yVIBh0j6tXbBjrV8MuIbtLYGqYoN4mxj6ZJKGzcSPiDx3AdMA/XgfMT1nHFefa9wXkz6NhBG30IGIYMvZSkH04+++FtRq9Jt/aOamzjbhvoVDtp7Xz1hUmMvc+r4SHC2C1PoI6Zgu+4KzvlqLfTHfTW2GtdxNnbmqdhSs9ekn6oSM9eMshEorKNPxo+mWr4ZW3U2Cf17L0ujvBuQYsE8M1fjOLydFrH0ex7KeMU5frI8Lo6hXvaMo5ummmXE1wiGdWJ0CRDg23c7YFRqXr2dU1Rz76DZi+EIFMzON73GXrB/mhjD0q6fV89+1kHFfHHa49GUzvG2Uc1e1M4A1QkknQhXTV7aezTiEgHzT5lz74piKoo5OfEEp8J0yTw1C+ZXbsDNDBnXNLl9rFonN559oqioCW5IRwZx5Bx9pL0w/Zd0u2tdNQZ+9rGdgpzfSPuqdzUGsLvdXXrPdvG3Z5hKnXPvp38bE+Ch61/+QFm7Q7chyzCO+5A2H9ul9vHonF659l3hX3sLc1+ZJ0HiaQnZOhlGlDVEODH977L5zsbh7spCQgh+MUDq3n+/Z3drud49p7UonF0w6S1PWKFXcbF2AthEl67EjV/PN4FFzPmiFO6nTi8r559V9gekWGaaXfDSCRONE6aXbyjyrOvbwoiiEWnjBSCYYOmtjA1je3drudE46So2b+4ehdPvvElqqpQPm2Ms1zfsQ6zYQ++RVd1a+Rt7Lj+3mr2XaHEyTiuXnb6SiTDjSo9+5GPPbVee3TWppFCY2sIgNZornqbcMRI+N5Rs+/Js6+qD2CagohuUloYS0usb34HxZ+D64B5KbXP6aAdIBnHvlkMOahKkoY40ThpFlywzxr7ZJOe2FPzDYWxN8zUE5U1tYYBnEm9ATZuq+fqP75Jc1vYWeZE46QYetkW1BlXlMkvrpjLSXMnAiAiIfSdH+Pav9yZ9aonBlrGiffspWYvSTcUpGc/Yti8q4Grfvc6tU2Jsog9j6qd7newqG1q5zu/e4Mte5pSWr+xrbNnv6e2jYhuJkg7dl6cVEMv29ojZPpc7FeS7WS11HetByOM64CuO2Q74tb6FmffFY5nb5r75gUo2adJV81+n7zXdu5tQTdM9tYHEpa3tUc9+/DgevY7ovVvq2xOaf3GFst7jzf2LQFrWXMg5tnbso0vGo2Timef6XcnLNO/XI3iz0EbOy2ltkFiIrSBQEbjSNIZqdmPIGxD2dKWqIEHHM9+cI29/ZCxBzP1RFPUsw8EdcxoJsuWQCThLyTR7Hvy7IMRMn0xYy/0EPrO9b2ScKDvg6q6wq7aMEXaxSpLJKNyWsKGhgZ+9KMfsXPnTjweD5MmTWLZsmUUFBQwbdo0pk6dihq9s++44w6mTUvdm+wPts4d7xVDfAft4Mo4VfWW9FKXYtSPrdkLLIOf5Xc7D6ymZJp9N4Oq/rNmF598Wc8PLpxpGXt/7BTrOz8GvXcSDsQlQhtgGScSkaGXkvTDdlDSzVHpl7FXFIUrr7ySI444AoDly5dz55138qtf/QqwJhrPzMzsfyt7ie0Nx3duwhB69g2WZ1+bomdvR+OApbNn+d20RCWdlrh9sHPjdKfZb97dxKfb6wlHDMIRM8Gz74uEA3Q7KXlfsD2iQEhnv5LsASlTIhkqRuWgqry8PMfQA8yaNYuKiop+N6q/tPTo2Q+usa/utYwTdjpQbd3eeWDF7YPdQevrZlBVa3sEwxRUNVhvF7Zmb0k463BNntMrCQdg4pgsLjllGocdUNir7boi3iOaOWVgypRIhop0lXEGTLM3TZNHHnmERYsWOcsuueQSzjnnHH73u98RDoe72XpgcTT7QKJm70Tj9LKDdsueJn77yNqU8scHghGaA5Z33toeIRTuWTJqbA0xvth6A7KNfavdQdvWuYPW7VJxaWpSz97efk9tKwCZdpjmzvV9knDAuqiPLxs/cB200b8ZXhdTxucOSJkSyVBhh16mW4fngI2gvfXWW8nIyODiiy8G4LXXXqO0tJTW1lZuuOEGVqxYwQ9+8INelVlYmNWnttjGPhA2KC6OyQS2Vh/ssLwnXl5XwWc7GjBUlXE9bPfFzgYAyqaN4c11ezA1NaGujvUGwzrtIYMpE/L4sqIZ1a2RX5DpvIXE74PH60JTFUpKcvC4VdxuV6fy7LEEjQHr7/ixORTmedj94RO48scy9vByFLWz0e7N8egveXutB9Gc6SWUju3Z2A9l23qDbFfvGKntgt61LTvbyh5bWJRFcX5GD2v3j4E8ZgNi7JcvX86OHTu49957nQ7Z0tJSALKysrjgggt44IEHel1uXV1r0sFRPWHLOPVN7dTUtADWFHht7REUoD2oU1XdnHKcbEWVVcbWHfX4enicf/5lLQAHlmbz5jrYvL0OfzTdQHFxttMem+qovl+YZeWRr6xqYfvOesCa/KOhOehs09QcZIyrje0rruY8Tw5tdSY1NZOdsoQQTmfv5h1WGXpIZ89zf0dv2Iv/zBuprUsMR+2qXYNJS4slMR08IbfHeoe6baki29U7Rmq7oPdtC0Sj5xrq21D0wQv26MsxU1WlSye5328id911Fxs2bGDFihV4PJbBampqIhi09Gpd11m1ahXTp0/vb1Up0+x00EacSbmDIR0B5GV7EZCSvGJjd6CmklOnqj6AosD0SflAz7p9Y9Q4jy3MRFGgNRhx5Kcx+X5aAxHngRcxTOZ4v8RsrKBM28zxe/9O2+O3EPlyNUIIQhHD0fX31LahYpLz6eNENryI+5BFuMYdnPI+DyZTJ+Zx8tyJzJ5aPNxNkUh6Tbpq9v3y7Ddv3syf//xnJk+ezOLFiwGYMGECV155JTfffDOKoqDrOmVlZVx77bUD0uCesKJQDEczD4YN/F6XI4sU5vhoaAnRHtKdePWesA1yKqGUe+sDFOb4KMj14dIUJyJn7Rc1vPrP9Ri6yYXHH8iksdnRsq0HSUG2l0yfm9Z23ZGhxhdlUlkXoKU9Qm6mh4huMl3bhVp8AHdXH8sxObuZb3xK8D8r0EqnEZj7dSa7ashT2vi8YRzfzHoD15YK3DNOxjv/wt4dyEEk0+dm8QnJJ0qRSEY6tpFPs8jL/hn7gw46iE2bNiX9beXKlf0pus/YHZSlhRls3t1EcyCM3+siEDX2Rbk+tuxp6lVEjj3oqb4l1O16Ed1k47Z6ZhxQiKooFOT4nAfES2t2sbumjbb2CGs31zjG3pZdcrM8ZEYfUHbY5fhCP2uwZKncTA+ucDMT1Bpck47BrPfzmWcmi85dQmTTG4Te+T/cz93CD3Ks+sJCw4WB95jL8Uw/LuV9lUgk3XPo/gUcP3s82Zmdp/AcyaRbh3KPxIy9Fd1iR7O0RiNx7Em3Ux1YFa+DNzQHqW8O8ujLmx25JJ6Pt9bSFtQ5asZYq64cH7VNQSK6ydaKZhbNnUhBjo/quHw3G7fXk5/txbvnQ8rc29ADLbS0hTnL/yHHf/4rfpDzHGLj8xgNeygNbAHANbkMl6aiGyaKquKZfhwZ5/yMcOZYngnM4u/tx7NNH8M/I8dLQy+RDDBjCzK45ORpaZcbZ5/LZ29njhxXZBt763u8Zw+ph1/acetgefbvbtzLi6t3MXtqMVMn5iWs+/Yne8nN9HDIZEuvnzQ2m5dW7+LjrXVEdJMZ+xeye9tuahvaACu+fsOX9Xz94DpCrz7DaYAZUQhuyCfDX0+k9HCUXXvI2/wsgc3PMh9oIpus/Am4XdUJoaBa0SS2zLiKl7Z9yv6l2fypcmJCWmOJRDK62eeMfWvUqI+LGjpb/27r5NmnZuxtvT4/20t9c4gvK6zkZruqW5lQnMmLq3dxxpGTaQ/rfPJlHSeVT0RVFPTKTRzn/QLD8yWbXt3GhRl7KX39eS5rq6HazCO8sYUtu0Oc4vucGdUb0cYfyouhWahVn3GYUstbkf05+YRvc9fdb3HpMWM4qrCO1e+tY4cYxxJFwaWp7Klp5cFVmzjn6P3JzfTQGu3YHV+cxbbKloTRsxKJZHSz7xn7qGc/Nmrs7RGoMc/emp6vJ2Nf3djOxm31FEcfDgeMy+HDTTXOlIa7qq1Y8X+/vZ0p43MJhQ0MUzDn4GLCa/9NeM2T+IEzo851yOfGN3YWG/2H4a/4kNDb/4+pwEF+cI2dhv/E7xJ8q4LXvnSze2whe8JtnOd3o6kKdbofz8ELeXNNltM5dMjkfPbWB3h93R5UBS4+eRot7REUBcZFJSx7QJVEIpHsc9bAlnFyMz1k+lyOZt8WjDDW1Uzmy7dzfkYWakMGwhyLomoIIXjz40oOGJfDhGIrRvU/q3fxnw93c87R+6MgmF5oshaTrEg909z1NFWBCGQwQatjV1UzwYggUw1RuvNFwh8/i+ugo/DOX8wr62t46Y1PmDlzGtdeMI8P39jCHz8bz/VnTOCRFzZw3NGHceKRUwHI8rsJ6ybVDe1k+92oikJWhpu6piBb9jSxZU8zx84aB8AZR07mjCMn878vfM4b6ys448jJtEbz6uRlWx1HHdMbSySS0cs+Z+xb2yN4PRpul0Z2hodNuxp59t3t1NS18bWsd6CliaO8lbg2b6L1y7+hFu1Hm5aLa3sVuxUIjS1h0txjMHdvpczTyNZPg3wn+02mfV7J7HwFTYkO8gq/AXVALrRtfIM2NYvj8qoxPjZwHTgf38JvoKga8w738dqGWsoPsQaZFef5ESi8tiVMhZHP1ANKnbYfMC4HBdhZ3cqcaAz6QRPyeO/TKtZuqaUgx8tXFx6QsL9nzJ/EWx9X8tx7O2gNhC1jn+kFkDKORCJx2OeMfVt7hIIMhfDHq/hK1hbqaxvwrYkwXw2yn7sa37FX8bOVTZw+OcD8sUHM6i8xqrdSoKl43RqZNR/T/sKHnAMQHYgWcWkYM87k1dXbaBGZHDynjI1r1gKCEF6OyNyF2whRl1nGnNPPQSuY6LQnJ8PDbVfGksUV51ky0rrNtWR4Xc6bBMAhkwv48ddm838vfeF0/n7rrEMoLcjgrU8queqcGWR0MOBFeX7mTh/DOxv2Mr44k2y/m7zsqLH373OnVyKR9JF9zhq0BsKc53qT0HubmaZ5UAoy0FUvTW1hNmXOZc6UIxDed9jqnkhOYTGTDsnmtr99wKwDizhp7gR+9vf3uXC6wfubG1A0D+OooilzEtfMP53n3nqNCWOyWHjQwdz7thU+OW/6GP778wMQAr46c0qCoU9Glt9NhtdFIKQzdWJep5zYUyfm8YuvxyYCd2kq5x17AOcde0DHohzmTB3Dexur+HJPM7MOKqIwx8uYfD+TZPpgiUQSZZ8z9vu1fcw0czOeuefjLTvLWZ4HTIp+9nldrNtSy9sb9jrx6kcdNpb9SrLJyvLzzy90dKOEk2ZN5KU1u5g+Nh9VVdh/XA6HTMpnXFEmmqqQ5XdTPm0MH3xWbdU9JrXEbcV5fnZUtXQK3ewrh0zOx6Up6IYgy+/C7dL4zbePHJCyJRLJvsE+N6hqCruoyZ6GZ9YZXa5jp08oyvUxoTiT0sIMpu+Xj6ooHH5AIbphkpPhdgZH5UaTlP304jmce8wBuF0qB03IpWxqMRNLYgZ+YqrGPt+Scqbtl9fHvey8P9OiD44sf3qN6pNIJEPDPufZz7j0xxSPyaWurq3LdfzRyT/OOmoyRx9eiilic6HOPLCINz+uZP/SHCaOyaIwx5ugq9ss/a8y57PXo+F1qeRmeVNq4/6l2Wzd08R+JX1L4ZyMww8sYuP2BrJkBI5EIknCPmfsVc3lpFnuiqI8HyUFGRw5YyyKoqDFDXs+ZHI+mT4X0ydZ0s2vvjUfTetcXvxQ6YMm5OLrxcQep8zbjxPnTEDr5YxR3VF2YBH/fHUrYwvkqFmJRNKZfc7Yp8J/nXAQumHiSmLEfR4Xy686yplv1e3q2Yh//7zDejUfpaooqCmU2xuK8vz84eoFKWfylEgko4tRaRlcmprU0Ntk9HLk6UBN19dfOoZlSiQSic0+10ErkUgkks5IYy+RSCSjAGnsJRKJZBQwqMZ+27ZtXHTRRZxyyilcdNFFbN++fTCrk0gkEkkXDKqxv+WWW1iyZAmrVq1iyZIl3HzzzYNZnUQikUi6YNCMfV1dHZ9++ilnnnkmAGeeeSaffvop9fX1g1WlRCKRSLpg0EIvKysrKSkpQdOssERN0xgzZgyVlZUUFBSkVEbHJGG9oT/bDiayXb1npLZNtqt3jNR2wchtW2/b1d36IzrOPj8/s8/bFhYOXCqCgUS2q/eM1LbJdvWOkdouGLltG8h2DZqMU1paSlVVFYZhAGAYBtXV1ZSWlvawpUQikUgGmkEz9oWFhUyfPp1nnnkGgGeeeYbp06enLOFIJBKJZOBQhBBisArfunUrN954I83NzeTk5LB8+XIOOKDrSTgkEolEMjgMqrGXSCQSychAjqCVSCSSUYA09hKJRDIKkMZeIpFIRgHS2EskEskoQBp7iUQiGQWM6BG0vWXbtm3ceOONNDY2kpeXx/Lly5k8efKQt6OhoYEf/ehH7Ny5E4/Hw6RJk1i2bBkFBQVMmzaNqVOnOvPk3nHHHUybNm3I2rZo0SI8Hg9erzU5+tKlSznmmGNYt24dN998M6FQiPHjx/Pb3/6WwsLCIWvX7t27+d73vud8b2lpobW1lQ8++KDLNg8Wy5cvZ9WqVezZs4eVK1cydepUoPvrayiuvWTt6u5aA4bkeuvqeHV33obiekvWru6us57aPFB0d866Oy79PmZiH+KSSy4RTz31lBBCiKeeekpccsklw9KOhoYG8d577znff/Ob34if/OQnQgghpk6dKlpbW4elXUIIcfzxx4tNmzYlLDMMQ5x44oli9erVQgghVqxYIW688cbhaJ7DbbfdJn75y18KIZK3eTBZvXq1qKio6FRvd9fXUFx7ydrV3bUmxNBcb10dr67O21Bdb121K57466y7Ng8kXZ2z7o7LQByzfUbGGUlZNvPy8jjiiCOc77NmzaKiomLI25EqGzZswOv1Ul5eDsDixYt54YUXhq094XCYlStXcv755w9L/eXl5Z3SenR3fQ3VtZesXSPhWkvWru4Yquutp3YN13XW1Tnr7rgMxDHbZ2ScgciyORiYpskjjzzCokWLnGWXXHIJhmFw7LHHcvXVV+PxeIa0TUuXLkUIwZw5c7j++uuprKxk3Lhxzu8FBQWYpulIEkPNK6+8QklJCYceemiXbc7JyRnSNnV3fQkhRsS1l+xag+G93pKdt5FyvSW7zrpq82ARf866Oy4Dccz2Gc9+pHLrrbeSkZHBxRdfDMBrr73GE088wcMPP8yWLVtYsWLFkLbn4Ycf5t///jePP/44QgiWLVs2pPWnwuOPP57gbaVDm0cCHa81GN7rbaSft47XGQx9m5Ods8FinzH2IzHL5vLly9mxYwd/+MMfnA4yuz1ZWVlccMEFfPTRR0PaJrt+j8fDkiVL+OijjygtLU149a+vr0dV1WHx6quqqli9ejVnnXWWsyxZm4ea7q6vkXDtJbvW7HbD8FxvXZ23kXC9JbvOumvzYNDxnHV3XAbimO0zxn6kZdm866672LBhAytWrHBem5uamggGgwDous6qVauYPn36kLUpEAjQ0tICgBCC5557junTpzNjxgyCwSBr1qwB4NFHH+XUU08dsnbF8+STT7Jw4ULy8/O7bfNQ0931NdzXXrJrDYb3euvuvI2E663jddZTmweaZOesu+MyEMdsn0qENlKybG7evJkzzzyTyZMn4/P5AJgwYQJXXnklN998M4qioOs6ZWVl/PSnPyUzs++TtPSGXbt2cfXVV2MYBqZpMmXKFG666SbGjBnDRx99xC233JIQ1lVUVDQk7YrnlFNO4Wc/+xnHHntsj20eLG677TZefPFFamtryc/PJy8vj2effbbb62sorr1k7frDH/6Q9FpbsWIFa9euHZLrLVm77r333m7P21Bcb12dR+h8ncHQXWtd2YcVK1Z0e1z6e8z2KWMvkUgkkuTsMzKORCKRSLpGGnuJRCIZBUhjL5FIJKMAaewlEolkFCCNvUQikYwCpLGXSLrh3nvv5Wc/+1mftr3xxhv5/e9/P8Atkkj6xj6TG0ciGQyuuuqq4W6CRDIgSM9eIpFIRgHS2Ev2Kaqqqrj66quZP38+ixYt4sEHHwTgnnvu4ZprruG6666jrKyM8847j88//9zZ7r777uOYY46hrKyMU045hXfffdfZbunSpc56L7/8MmeccQbl5eVccsklbN261fnt008/5bzzzqOsrIzrrruOUCiU0LZXX32Vc845h/LychYvXpxS/RLJgNG39PsSycjDMAxx3nnniXvuuUeEQiGxc+dOsWjRIvHGG2+Iu+++WxxyyCHi+eefF+FwWNx///3i+OOPF+FwWGzdulUce+yxYu/evUIIIXbt2iV27NghhBDi7rvvFj/84Q+FEEJ8+eWXYubMmeKtt94S4XBY3HfffeLEE08UoVBIhEIhcdxxx4kHHnhAhMNh8fzzz4tDDjlE3HXXXUIIITZu3Cjmz58v1q1bJ3RdF0888YQ4/vjjRSgU6rZ+iWSgkJ69ZJ/hk08+ob6+nu9///t4PB4mTpzIhRdeyHPPPQfAoYceyqmnnorb7eaKK64gHA6zfv16NE0jHA6zdetWIpEIEyZMYL/99utU/nPPPcfChQtZsGABbrebb3zjGwSDQdauXcv69euJRCJcdtlluN1uTj31VA477DBn23/84x9cdNFFzJw5E03TOO+883C73axbty7l+iWS/iA7aCX7DHv27KG6utqZzQesdMPl5eWMGzeOsWPHOstVVaWkpMRZ/6c//Sn33HMPW7Zs4eijj+bGG2+kpKQkofzq6uqECSTstLRVVVVomkZJSQmKoji/x69bUVHBU089xUMPPeQsi0QiVFdXM2/evJTql0j6g/TsJfsMpaWlTJgwgTVr1jj/1q5dy1/+8hcA9u7d66xrmiZVVVVORsOzzjqLRx55hFdffRVFUbjzzjs7lT9mzJiEnOJCCGcGq+LiYqqqqhBxeQXj1y0tLeWqq65KaNv69eudqQxTqV8i6Q/S2Ev2GQ4//HAyMzO57777CAaDGIbBF198wccffwzAxo0befHFF9F1nf/93//F4/Ewc+ZMvvzyS959913C4TAejwev15swAYjNaaedxuuvv867775LJBLhb3/7Gx6Ph7KyMmbNmoXL5eLBBx8kEonw4osv8sknnzjbXnDBBTz66KOsX78eIQSBQIDXXnuN1tbWlOuXSPqDlHEk+wyapnHvvfeyfPlyTjjhBMLhMPvvvz/XXXcdACeccALPPfccP/7xj5k0aRL33HMPbrebcDjM7373O7Zu3Yrb7aasrCzpdHQHHHAAv/3tb7n11lupqqpi+vTp3Hvvvc7kE/fccw8///nP+cMf/sDChQs56aSTnG0PO+wwbr31VpYtW8aOHTvw+XzMnj2b8vLylOuXSPqDzGcvGRXcc8897NixQ8ojklGLfFeUSCSSUYA09hKJRDIKkDKORCKRjAKkZy+RSCSjAGnsJRKJZBQgjb1EIpGMAqSxl0gkklGANPYSiUQyCpDGXiKRSEYB/x9S2bqM2BzdzgAAAABJRU5ErkJggg==\n" - }, - "metadata": {} - }, - { - "output_type": "stream", - "name": "stdout", - "text": [ - "Start to eval !\n", - "Env:CartPole-v0, Algorithm:PPO, Device:cuda\n", - "Episode:10/200, Reward:200.000\n", - "Episode:20/200, Reward:183.000\n", - "Episode:30/200, Reward:157.000\n", - "Episode:40/200, Reward:200.000\n", - "Episode:50/200, Reward:113.000\n", - "Complete evaling！\n", - "results saved!\n" - ] - }, - { - "output_type": "display_data", - "data": { - "text/plain": "
", - "image/svg+xml": "\n\n\n \n \n \n \n 2021-05-06T01:36:55.923900\n image/svg+xml\n \n \n Matplotlib v3.4.1, https://matplotlib.org/\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n\n", - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXsAAAEcCAYAAAAmzxTpAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Z1A+gAAAACXBIWXMAAAsTAAALEwEAmpwYAAB9qUlEQVR4nO2dd5QUVfr3vxU7TJ5hEhkRcECFgSEoSlZAYUBdBVnDmll/BtbV1V1RMK6oq64svurq6rq6pjUCKhgQxICggJKRJGFynk6V7vtHT9X0zHSo7ume0H0/53AO01VddW9X9dNPfe8TGEIIAYVCoVDiGrazB0ChUCiU2EONPYVCoSQA1NhTKBRKAkCNPYVCoSQA1NhTKBRKAkCNPYVCoSQA1NgnAMuXL8ftt9/eKec+//zzsWnTpk45d1fD7XZj4cKFGDVqFG655ZbOHg5++OEHnHvuuSgsLMRnn33W2cOJOUOGDMGRI0c6exidBjX2lJiyevVqjB07trOH0SX45JNPUFlZiU2bNuHpp59us3358uUYNmwYCgsLUVRUhPnz52Pr1q0AgHfffRcFBQUoLCzEyJEjMWfOHKxbt854b319PZYsWYLx48dj+PDhmD17Nt55552g43n66afx29/+Flu3bsW0adOiMseffvoJ1113HYqKijBmzBj85je/CTmOYFx++eV4++23W7w2ZMgQjBgxAoWFhTj77LPx17/+FaqqtnfoEbN7925ceOGFGD58OC688ELs3r2708YSDGrsOwFFUTp7CFEhHubRkXM4ceIE+vfvD57nA+4zc+ZMbN26Fd9++y1GjhyJm2++GXre44gRI7B161Zs2bIFv/nNb7Bo0SLU1dVBkiT87ne/w4kTJ/DGG29gy5YtuOOOO/C3v/0NL730UtDxDBo0KKK5+Pvctm7diiuvvBKjR4/G2rVrsWnTJixduhQbNmwI+/iEEGiaFnD7Bx98gK1bt+Lll1/GqlWr8NZbb4V9jmggSRJuvPFGFBcXY/PmzZg7dy5uvPFGSJLUKeMJBjX2rXj++ecxbdo0FBYW4rzzzsOnn34KwHtRi4qKsG/fPmPf6upqnH766aiqqgIArFu3DnPmzDG8sj179hj7TpkyBc8//zxmz56NESNGQFGUgOcCAFVV8cgjj2Ds2LGYMmUKXn31VQwZMsT4kjU0NOAvf/kLzjrrLJx99tl48sknTXs327Ztw/z581FUVITi4uIWMss777yDmTNnorCwEFOnTsUbb7xhbNu0aRMmTJiA559/HuPHj8ef//xnLF++HLfeeiv+9Kc/obCwEOeffz5+/vnnFvP+5ptvACDkvjt37sTcuXNRWFiIW265BYsWLcKTTz4ZcB5vvfWWMdbzzjsPO3fuBND2cf2uu+4yjuNvDjNnzmzhJSuKgnHjxhnHC/Z5tebAgQO4/PLLUVRUhPPPPx+ff/45AK8X/cwzz+Djjz9GYWFhG2+1NYIg4IILLkBFRQVqampabGNZFhdddBHcbjd+/fVXfPDBBygpKcHf//539OnTB4IgYMKECVi8eDGefvppNDY2tjn+tGnTcPToUSxcuBCFhYWQJAllZWVYuHAhxowZg3POOaeFAV2+fDluueUW3H777Rg5ciTee++9Nsd89NFHMXfuXFx//fXIzMwEwzA49dRT8fe//x0AUFdXhxtuuAHjxo3D6NGjccMNN6C0tNR4/+WXX44nn3wS8+fPx/Dhw3HHHXdgy5YtuP/++1FYWIj777+/zTkHDhyIUaNGYf/+/QC898Q555yDMWPGYOHChSgrK/P7+UqShGXLlmHSpEk488wzce+998LtdvvdN9j98f3330NRFFx55ZUQRRFXXHEFCCH47rvv/B6rUyGUFnz00UektLSUqKpKVq9eTYYPH07KysoIIYTcdddd5IknnjD2ffXVV8nVV19NCCFk586dZNy4cWTbtm1EURTy7rvvksmTJxOPx0MIIWTy5MmkuLiYnDhxgrhcrpDn+u9//0tmzpxJSkpKSG1tLbnyyivJ4MGDiSzLhBBCbrzxRnLPPfcQh8NBKisryUUXXURef/11v3N6+umnyR//+EdCCCGlpaVkzJgx5MsvvySqqpKNGzeSMWPGkKqqKkIIIevWrSNHjhwhmqaRTZs2kdNPP53s2LGDEELId999RwoKCsijjz5KPB4Pcblc5Omnnyannnoq+fLLL4miKOTxxx8nF198sXHuyZMnk6+//toYR6B9PR4PmTRpEnn55ZeJJElkzZo1ZNiwYS0+79bX6ayzziLbt28nmqaRw4cPk2PHjhFCCBk8eDA5fPiwse+dd95pHMffHJYvX05uu+02Y/9169aRGTNmmPq8fJEkiUybNo38v//3/4jH4yHffPMNGTFiBDlw4ECb6xDqOnk8HvLII4+QiRMnEkIIeeedd8j8+fMJIYTIskxefvllMmLECFJfX08WLVpE/vSnP7U5nizLpKCggGzYsMHv+XyvDSGELFiwgCxZsoS43W6ya9cuMnbsWPLNN98YYxs6dCj59NNPiaqqxj2s43Q6ySmnnEK+/fbbgPOrrq4mn3zyCXE6naShoYHcfPPN5Pe//72x/bLLLiMTJ04k+/btI7IsE0mSyGWXXUbeeuutFsfxvb779+8nZ555JnnrrbfIN998Q8aMGUN27NhBPB4Puf/++8mCBQv8vu+hhx4iN9xwA6mpqSENDQ3khhtuII8//rjfcQe7P1566SVyzTXXtNj/+uuvJy+++GLAz6GzoJ59K2bOnInc3FywLIvzzjsP/fr1w08//QQAmD17NlavXm3su3LlSsyePRsA8Oabb2LevHkYPnw4OI7DBRdcAEEQsG3bNmP/yy+/HPn5+bBarSHP9fHHH+OKK65AXl4e0tLScP311xvHqaysxPr16/GXv/wFdrsdWVlZ+N3vftdibIH44IMPMGHCBEycOBEsy2L8+PE49dRTsX79egDApEmT0LdvXzAMgzFjxmD8+PHYsmWL8X6WZXHLLbdAFEVjHqNGjcLEiRPBcRzmzJnT4ommNYH23b59OxRFwRVXXAFBEHDuuefitNNOC3ic//3vf7j22mtx+umng2EY9OvXD7169Qo5f39zmD17Nr744gu4XC4A3ut6/vnnm/q8fNm+fTucTieuv/56iKKIM844A5MnTzZ1XXQ++eQTFBUVYeLEidi5cyf+8Y9/tDh+UVERxo8fj9WrV2PFihVISUlBTU0NsrOz2xyL53lkZGS0eTLwR0lJCX788UfcfvvtsFgsKCgowMUXX4wPPvjA2GfEiBGYNm0aWJY1rr1OfX09NE3zOw6djIwMTJ8+HTabDcnJyfj973+PzZs3t9jnggsuwKBBg8DzPARBCHisCy64AKNHj8bChQvxm9/8BhdddBFWrlyJiy66CMOGDYMoirjtttuwbds2HDt2rMV7CSF466238Je//AXp6elITk7GDTfcEPA6Bbs/HA4HUlJSWuyfnJwMh8MRcOydRWDxMEF5//338dJLL+H48eMAAKfTaXxZxo4dC7fbje3btyMrKwt79uwxFrZOnDiB999/H6+++qpxLFmWUV5ebvydn59v+lzl5eUt9s/LyzP+f+LECSiKgrPOOst4TdO0Nsf3x4kTJ/DJJ5+0eSzVF1HXr1+PFStW4PDhw9A0DW63G4MHDzb2zcjIgMViaXHMHj16GP+3Wq3weDxQFMWvNh1o3/LycuTm5oJhGGN7sPmUlJSgb9++Iefrj9Zz6NevHwYOHIh169Zh8uTJ+OKLL/D+++8DCP15+VJeXo68vDywbLMP1bNnz4BSgj9mzJiBxx9/3O+24cOH4/XXX/c7n4qKijavK4qCmpoaZGRkhDxveXk50tLSkJyc3GLsO3bsMP72vQdbk5qaCpZlUVFRgYEDB/rdx+Vy4a9//Su++uor1NXVAfAaS1VVwXEcgODX3Jf33nsP/fr1azOHYcOGGX8nJSUhPT0dZWVl6N27t/F6dXU1XC4XLrzwQuM14rNGcO211+KHH34AANx3330oLi4OeH8kJSW1kckcDgeSkpJMzaMjocbeh+PHj2Px4sV4+eWXUVhYaHifOhzHYcaMGVi1ahV69OiBSZMmGV+O/Px8LFy4EL///e8DHt/XkIU6V3Z2dgs90/f/eXl5EEUR3333XdDFPn/k5+djzpw5ePDBB9tskyQJt9xyC5YtW4apU6dCEATceOONxgJh6zlEk+zsbJSVlYEQYpyjpKQEffr0CTiPX3/91e82m81meGEAUFFRgdzcXONvf3OYNWsWVq1aBU3TcPLJJxuGJNjn1ZqcnByUlpZC0zTD4JeUlKB///4h39sezjzzTDzxxBNwOp2w2+3G62vXroUoihgxYkTIY+Tk5KCurg6NjY3GPV1SUhLyc9Ox2WwYMWIE1q5di3Hjxvnd51//+hcOHTqEt956C9nZ2di9ezfmzp0btfsrJyfHcJwAr/NUW1vbYg6A98fRarVi9erVbbYBwAsvvNDmtUD3x8knn4x//etfLe7bvXv3YsGCBRHPI1ZQGccHl8sFhmGQmZkJwLtYqS/86MyePRsff/wxVq5ciVmzZhmvX3zxxXjjjTewfft2EELgdDrx5Zdf+l0cM3OumTNn4pVXXkFZWRnq6+vxz3/+09iWk5OD8ePH45FHHkFjYyM0TcOvv/6K77//PuQci4uLsW7dOnz11VdQVRUejwebNm1CaWkpJEmCJEnIzMwEz/NYv349vv76a/MfYDsYMWIEOI7Dq6++CkVR8Nlnn7VYvG3Nb37zG/zrX//Cjh07QAjBkSNHjC/6KaecglWrVkFVVWzYsKGNVOCP8847D19//TVef/31Ftc12OfVmtNPPx1WqxUvvPACZFnGpk2b8MUXX+C8886L4BMxz5w5c5CXl4dbb70Vx44dgyzL+Oqrr/Dggw/ipptuaiMz+CM/Px+FhYV44okn4PF4sGfPHvzvf/9DcXGx6XHccccdeO+99/DCCy8YT6h79uzBH/7wBwBej9disSA1NRW1tbUtJKpA9OjRA0ePHjV1/lmzZuHdd9/F7t27IUkSnnjiCZx++uktvHrAK+NdfPHFePjhh43girKyMnz11VcBjx3o/hgzZgw4jsMrr7wCSZKMJ/tAP3idCTX2Ppx88sm4+uqrMX/+fJx55pnYt28fRo4c2WKf4cOHw2azoby8HBMmTDBeP+200/DAAw/g/vvvx+jRo3Huuefi3Xffjfhcl1xyCcaPH4/i4mLMnTsXEydOBM/zxuPuo48+ClmWcd5552H06NG45ZZb/D7KtyY/Px/PPPMMnnvuOZxxxhmYOHEiXnzxRWiahuTkZCxevBiLFi3C6NGjsWrVKkyZMiXcjzEiRFHE8uXL8b///Q+jR4/Ghx9+iEmTJkEURb/7z5w5EwsXLsQf//hHjBw5Ev/3f/9nSAN333031q1bh6KiIqxcudJUDHlOTo4R2uhrnIN9Xv7m8Oyzz2LDhg0YN24c7rvvPjz66KMBZY1oIYoiXnrpJeTn5+OSSy7BqFGj8Mgjj+APf/gDrr32WtPHeeKJJ3D8+HGcffbZuOmmm3DzzTfjzDPPNP3+kSNH4t///je+++47TJs2DWPGjME999yDiRMnAgCuvPJKeDwejBs3DvPmzcPZZ58d8phXXHEF1qxZg9GjR4d8ujrzzDNx66234uabb8ZZZ52Fo0ePBozmuuOOO9CvXz9ccsklGDlyJH73u9/h0KFDAY8d6P4QRRErVqzABx98gKKiIrzzzjtYsWJFwPu2M2EIoc1LugPr16/H0qVLW2jH8c7FF1+M+fPn46KLLursoVAo3R7q2XdR3G431q9fD0VRUFZWhhUrVkQty7Gr8v3336OiogKKouC9997D3r17TXl/FAolNHSBtotCCMHTTz+NRYsWwWq1YtKkSbj11ls7e1gx5dChQ1i0aBFcLhd69+6Np59+Gjk5OZ09LAolLqAyDoVCoSQAVMahUCiUBIAaewqFQkkAqLGnUCiUBKBLL9DW1DigaeEvKWRlJaOqyn8yUzxD551Y0HknFmbmzbIMMjL8l2ro0sZe00hExl5/byJC551Y0HknFu2ZN5VxKBQKJQGgxp5CoVASAGrsKRQKJQEIaexrampw3XXXYfr06Zg9ezZuuukmVFdXA/C2aysuLsb06dNx9dVXGxXkQm2jUCgUSscS0tgzDINrr70Wa9aswcqVK9GnTx88/vjj0DQNd9xxB+69916sWbMGRUVFRtOFYNsoFAqF0vGENPbp6ektuvKMGDECJ06cwI4dO2CxWFBUVAQAmD9/Pj755BMACLot1miaBlVVoWpt/wXrVt9ZaISE/a870RHz07TwzxHrz7Yzr19UP9sYf4YkkusUwfWO1ry7c3WZsEIvNU3D66+/jilTpqCkpAQ9e/Y0tmVmZkLTNNTW1gbdlp6eHrXB+2PHf5/EAKf/phdlXB4GXv3XdnXDIYTgwVe24JyiPhg3LHCbNjPsP1aLR/+7FWqY4VQLpg3CtCL/HZxa892uUny6+SgWX1EUsy5TgfhxXwWeeW9HWF82BsC1s4bijFPNfbYrvzmM9zYcjHCEbblyxhBMHGGul20gDpyow7LXfoSitp03xzK47ZLhKOif2a5zBOJ4pQMPvLwZktLWsWEZBv93wakoHBy4T6wvH2w8hA82Bq7x7g+BZ7H4iiL0yUkOvTOA5z7cie93l4fesZ1MKuyFy84dDDbEd0AjBK9/th9f/HAM/u5am4XDQ9eNQ3qyxc/Wtjz19nac0jcDM8ZG1kIzmoRl7B944AHY7XZcdtll+PTTT2M1JoOsLHM3jC89z5iBY7vbVkpsLD+OU5Q9SHEega1/4EbWoXC6ZRwqaUBlg4Ts7NAdgILx/d4KqBrBb6YMgihwpt7zwYYDKKtzBzx369dLaw/jUEkD0jOSTJ8jWtRuPwGNEFx67hDTPzQffXMIO47UoHjyIFP7bz9Qhd45yZhQ2Dv0ziF478tfUFob+LM1y9aD1VBUgrkTB8JubW6arWkEb3y6F+UNEia08xw6rcd6sKwRkqJh5hn9kZHasin4G5/uRVm9x/T8fi1vRI90G84d2y/0zgA8koJ31v2CsnoPRg4z10t2z6+1GNI3A6MK2rYHjBalVQ58seUoOJ7FzZcUgmP934uaRvDMO9vx+Q/HMGlUb/Ts0dL+lFc78dnmX+HW2n7ugdh7tBYWC9/ue0qnPccxbeyXLVuGI0eO4NlnnwXLssjPz8eJEyeM7dXV1WBZFunp6UG3hUNVVWPYSQQ9Bg5DwbhxqKhoaPH6W5/uQt+DB1Hx7WrYkvqHdUxfKmu9vU3r6t1tzhEuJRXebLjpRb3AseYCozb8eBTVtS6/587OTmnzelW1EwBw9EQtUu0d2z2nptYFjmVwzkjznvKhY7XYcaAS5eX1IX8gXB4Fh07UYd60IZhW2DPovmbY8ONRlFc52n1dy5qu6zkje8FmafkVe3/9LzhWWt/ucwD+r3dltQMAcOawXPTq0TKT8qOvD+JEWYPpc5fXONEnO8n0Z6sRgpVfHcQvR6pRcXJWyP0bXTLqHRJmjOkb1vXzN+9QpFh5fLDxENwuGVedVwC2lcHXCMErn+zBhu0lOP+Mfrhwwklt7r9fyxrw2eZfcbykDnmpoT17WVHhkVSUVbb/ngLMzZtlmYBOsikL88QTT2DHjh0t2m2deuqpcLvd2LJlCwDgjTfewIwZM0Ju6yx4ixXfewZCOfQjNGdtxMdpdMsAAFlV2z2mBqeEJCtv2tADgN0qwOlWTO/v9Hj39UjtH2+4eCQVljCfJgb3SUODU0ZZjSvkvgeO14EQYNhJ0ZFEUuwiGpxyu4/j9MhgGMAqtp17RooFtQ2edp8jEJLslW9Evu09lZ5sQW2j+XPXNnhMyxWAVybKybCjrMnBCEVZjXe/vEx7iD3bz5yzBmDuWQPw9Y5S/Ouj3S2cSI0Q/Ptjr6GfdWZ/v4YeAJJt3qe0Rpe5e6TR5f3uVcfweodDSM9+//79eO6559C/f3/Mnz8fANC7d2+sWLECjz76KJYsWQKPx4NevXrhscceA+Bt6BtoW2dhEVhscA/BJOtuyHs2wDLSfCNlXxqbjIE/TTTsY7lkJIfpbdstPCrr3Kb3dzT9MLg7w9jLKkQhvFSOQb3TAQD7jtaGNAL7jtWCZRgM6ZeJxvrQPw6hSLELOFzafg/M4VZgt/B+DUZGigU1YRjccJEV73UOZOzN3juyosHhVpCeHN79mZdpw9EKh6l99R+F3ExbWOeIlOKzBgAM8P5Xh0AIcM35BQADvPzxHmz8qQSzz+yPuWcPCPhEGa6xb3BKxv4eOXzHJ9qENPaDBg3C3r17/W4bOXIkVq5cGfa2zkAUOFRoqUBeAeTdX0IcMQtMGB61jn6h5SgY+wanjBS7EHpHH5KsPH4tN+99OpueRNyS+aeBaCEpWtg3eH6WHck2AfuP1WLC8OCP9vuP1qFvbjJsFh7RKIuVaheNL2h7cLoVJFn9X9eMZAt2Halp9zkCoTshAt/2c09PseCX43WmjlPX9IMUjmcPALmZdmzdXwlF1cBzwb9fpdUusAyD7PSOMfYAUDx+ABgA7311CAABx7LY+HMJisf3x5yzAht6wGtDRIE1bewdPvvVNniQ2wFPMMFImAxa3ejIJ50N4qiG+uv2iI7TEFVjLyHFFp6xt1n5uJZxGIbBoN5p2H80uFGSFQ0HS+oxuE96O0bYkpQkES6PanjHkeJ0K7BZ/ftR6SkW1DVKMSvkpRt7f09U6UkiGl0yFDX0vVvb6P3RSwvT2Odl2qFqBFUmniDKqp3okW4N+aMQbWaPH4ALJ5yEb3eWGYZ+7tn+pZvWJNsE4+k+FA0+xr663vzTeKzo0lUvo4l+83tyT4VoT4e0+wvw/QvDPo4jmsbeJeOknqlhvSfJKsAtqVA1zZTW3+kyjh/dOhSDeqdj6/5K1DYG1oyPlDVAVjQM6p3W3mEapDY9ZTU4ZWSmRv7I7XTLSApg7DNSLNAIQb1TCttrNoOsqGAY+I04SU/xnq+uUUJWmrXNdl9qDc8+XBnH672WVjtDerKl1c4O0ev9MevM/kixC9A0gskjzUdyJduEiDz7rqDbJ5xn71EB4ZSJUI/ugFYffnyv/msttdP7I4Sg0SkjJVzNvsmIuDyhz69qmuHRuzpFxlFh8aMdh0L31vcfC+zd7z9aC6BZ448GerRSfTulHKdHaRFy6UtGk4GvidGXX5I1iDzn10vVDbeZRVrD2KeEL+MAXkMeDI0QlNV0nrEHgIkjeoVl6IHwjH0jNfadgx5jLskahIJJAMNA3v1l2MeJlmfv8qhQNWIs+pjF3hTK53CHvuF85Z5O8ewlLaLY/r65yRAFFvuaDLo/9h2tRW6mHalJ0QsnTWk6Vr2jfRE5+gKtP3TjGauIHFnRIAT4gdWfJMwZewkcy4R9fybbBCTbhJARObUNHkiy1uk6driEY+wbXDKsIodkm4CaLiDjJIyxNzx7WQWblAG+XyHkvV+BqOF9saO1QNvg8nqP4S/Qevc3o9v77tMZmr0kq7BEIOPwHIuBPdOw/1it3+0aIfjleB0GR1HCAXxlnMg9e0JISBkHQMwiciQlcARUmmHsQ8+vttGDtGQxZMapP3IzbSE9e/3HIC+j4xZno0G4Mk6yTUBmioV69h2JHoomyV6jJwydDOJugHJwc1jH0Rdn2m3sm46TbItMxjFl7D2d7NnLKkQ/USFmGNQ7DUfLG+HytJ3niUoHHG4lqhIOAENSa4+MIysaFJUY16k1qXavAY2VjOP17P1/5il2ASzDmPLs64Ksl4QiL8MeMk+itGl7d/TsnW7F1AJ7Q5Oxz0ixoLqeGvsOQ/cwPU3Gnus1FExqLuRd68I6jp5U1d44e/1HI1zPXjciZmQc3306J/Qy8tjiQX3SQYg3cao1upY/uE90PXuryEHg2XYlVukL4oE0e5ZlkJYsxliz9/+1Zhnvuc3KOJEa+9xMO2oaPEHvubJqJ0SBDXtNoLNJtgkgMPn9c8lItgvITLWipoHKOB2G7mHqGYYMw0IcOglq2X6o1UdNH6fZs2+fp6xLBfEq4xBC4JE0WMTIbrGBPVPBMgz2+ZFy9h+tRVqSGPX4bIZhkGIX0OCI3LPXn6YCyThAU2JVrDx7NbCxB/QsWnMyTriRODr6omt5EO++tNqJ3Ax7RDJRZxJOYlWDs9mzd7gVw9HsLBLG2Ptq9jrC4LMBjjft3XtkFZKigWHaL+PoN0tKuDJO08Kf04+80Rrd2CdZ+Q6XcRTVWxI2Us/eKvLom5vsN95+/7FaDOqTHpMqnil2EfXt8Oz1JLZAC7RAU8mEGGn2sqwGXKAFvBE5oc4tySocbiXsGHudPBMROZ0Zdtkeku3mjb3DLSPZKiCzqY5OZ8faJ4yx1xetJB9jz1iTwZ80FvL+b6A1VIQ8hh6Jk5Ykeo1ZOxJjGpwyRJ4NewFTFFhwLBOWjJOZau1wGUf/UY1Uswe8IZgHS+pb/LBW1blRVe+J+uKsTqpdbJdmH0rGAbzhlzGTcYJo9oDXs68L4dnXNT3ZROrZ5zQtugaKyFFUDZW17m6h1xNNgVK6D54t78HxwYPos/5enMyXhkysUlQNLo/qlXFSvDkNsbrmZkkYY89zXiPpkVt65JbCWQDDwrn6cWjO4Fmb+q95RtPFk01kIgaiwSkZXkI4MAyDJCsPl8kFWp7zShMd7dnrP6qRROPoDOqdBlnRcKSsuV6NHqET7cVZnVS70K5oHJfbnIzjllS/i8/tRVZCyTjeLNpgT6a6558RoWcvChyyUi0BPfuKWhc0QpDXQTVxwkWrK4W041M4P3kSjf++Ca4PH4a09UNA0wDRjutSvoBaEbzOv+4YJtsEZBiefeca+4TJoAW8N6HUSjdj0/Nhn3kbnKsfhevjx2GfdRcYS5Lf9+vGPjPFgkMl3i9WpDJFg0sOW8LRsVkFw4MMhtPtTe6xirzhrXUUhmcfZiE0X3SDvv9oLU7u5fXk9x2rg1XkTDfHCJeUJBH1DhmEkIhkIv1pKlC5BMAn1r7R06YEcnuRFBVCkM9cX3Sta/SgR4A1D13Tb0+Gb26mHaXV/jX7suquFYlDNAVq6X4ov26HcmQbSF0pAIBJzYFw8hngeg8D37MAjCUJrtpK1Ly+FCfteRnq0N7gMvzXb2r0MfaZerhtJy/SJpSxtwis30USLvdk2M69Ba5PnoTzkydhP+8OMELbG73Zs/dua49u3+CUI/LsAa/X6DSZVGW38LCKHNwmMm6jib4QbmmHjJOaJCI30459R2sxc5y3gYZu+FvXI48WqXYRiqrBLakRGWJ9nSSoZu+TRZuf5d+xiBQphGfvG2sf0Ng3yQ1pEco4gNeQb9pZ5vdHU/f4czM6x9gTokGrK4VWdgDKsZ1Qjv4ESE6A5cH1PAX8sKng+w4Hm9q2CZI1LQvPOc/BbeKncH30GOzFd4NN6dFmP19jL/DexKrOjrVPKGMvClzAkEm+96mwTlkI9+fPwPXZP2A791YwXMuPp9ElQ4CCk3AEdkZpV8mERpcUcWlXu4U3mUHrTe6xilznafbtkHEAYHDvNPy4rwIaIXC6FRyvdGDM0Nh1NdKjo+qdUmTG3qPAInBBi3sZiVUx+PLLcijNPnTJhFqHJ6LsWV/yMuxwehQ0uOQ2TXPKapxGpm1HQCQX1PIDUMsOQC3/BWr5QcDjLcPM2FLB9x8Fvt9w8L2GgRGDfycZhoHHkoWvsi7GObVvwvnRY7DP/gtYe8s1pEaXDDvjRmbFj1DtA5CZGrt1GrMklLG3CFzQEEThpNEg0u/g2fAS3Oueh3XKQjAs633MO7YLffZ/hgczdsF6SEGx7WTIytkRj6XBGbmMY7fyKK8NXb/d4VaQbPfKOB0d9mVo9u2s4T2odzq++qkEJyodqKz1PgbHanEWgFF+ocEhIzcj/Pc73HLAhCodXxkn2oTy7M2cu7bBG2PfnmgnXaIpq3a2MfalVbGPxCGK5JVlfvkWyq8/AZoCgAGb0QvCgCJwOQPB5p4MNj0PDBOe1JhsE3BcTYV9xh/gXP0YXB//DfZZd4KxJIHIbiiHf0TOtvV4MH0fuK0Ezq3AXPEUrK0bHZvJmiThjH0ob1w8ZSIgOeH57k24WQ6MaINycDOIuwGZrBXblJNwai6DwqqDaHS7AYSvHcuKVyaIXMYx163K6VGQm2mHReSgqMRUjfFo4YmSsdcTp/Yfq0NlU5vDAfnhVQoNB90wRbpI661lH/xrZRE42C181D09jXivcbDQy2SbAI5lgsbatyfGXicvqzn8svViemmNE6cNCN22MFyIpkEt2QN5/7dQDm0BZBcYWxqEYVPB9zkdXM5JIT13M+glE7i802E792a41jwF58dPgE3OgnJkG6BKsAmp+NI9FDMungvm+E/ov/UjXEN+gedHB8TTZ4DhO7ZFKJBgxl4MoNm32e/0mSBuB6RtqwBOBN9vBPiTx+HN7Rz2nXCg30k8cqv3wnV8G9BnWtjjMGLsIzT29qaa9qEWEb0LtLzRHs8tqUi2dayxb88CLQBkp9uQlixi/9FaVNS5MCA/NaaN031lnEhwBimC5kssEqtko5Z94M/HTBZtbaMHPdu5ltAj1QqOZYzFWB2XR0FdY+QSpj+0+grIezdA3rcRxFEDCFbwA0ZBOPlMcD0LImpSFIxkm4DjlV4ZiO9zGqyTb4D78/8HUl8OYchZ4AeOxUe7GKzbdgJzew4Ceg7CRudJEH9+DyO2vAt57wZYxs0H339UTHJFApFQxt4icKZT4cXRF4HvNwJsRi/DG6jftA0pdgEkeyCq1CTYj30PIHxjb2TPRqhZ2q08NEKCLiKSJo1bX6AFALdH6TCd1Figbadh9jYzScfuX2vQ6JRx7ug+0RheQJrr40SWWOVwK+gRolY84JVTYmXsg3n2gB5rH/jcdY0ShvZrX19flmWQk2FrE2uvZ9W2V8YhqgLlyFbIe9ZDPbYTYACu92kQxl0Kvt+ImHrOyXYRja5a429h4BhwuSeDsaeCYb3fx8YfdrX4rtl75OOfjZPwyJx0JO14F+5P/wH+5HGwTVkYs3G2JqGMvShwprVrhmHA5Z7c4rXGploXgsBjszQQ06t/htZYDTY5vC9Gg+HZR6jZW/Sa9kpAY++WVGiEIKkp9BIA3B2o20dLxgG8Gv2WPd7eA4Oi2JnKHwLPwmbhIi6Z4PLIsFtDS3sZyRYcq4hGM8VmJCORLbSxD5Tw5JFVOD0K0lPabyzzMu1tYu2NSJwIjb1WX4Gqn9+HY9sXIK56MEmZEEfNgTDkbLDJ0ZeG/JFs49HokqERYpR7aG0DGpuKoOno4ZcVtv7Ivug+SJvfgbT9IyiDxoPvc1qHjDuhjL1FYNvE2YdDo0tGfpYdAs/he89AzLD9BPmXb2AZMSus40RaF0dHr4/jcCvIDCBfGyGArWScjiJaMg7QMoFKj7ePJSntyKJ1NElnochIsaDeIZnuOGYGQ8YJEe6alixi76/+++DqHn9aUvsLlOVm2vHzwWpoGjFCZcuqnWAA5IRR14gQArVsP+Sf1kA58iMABny/ERBOmQiu92lRl2lCkWwVQIjX2QrUa7i1sc9I9T7tVde7wbCZEIsugHxwMzzfvQmu17AOmUNCGXuR59pk0IZDo0tGkk2AwLOo0lLgSOkPdu9GiMPPD0t7ay5vHLmMAyBorL1eO6eFjNOB4ZeSrIHnmKgYsj45ybCKHLLSrB0iQ3kbj4cv46iad+E9kAHwJSPFAkK8kklmamjZxwxSGDKOw61AktU2+r6RUBUlz15RNVTXu42Y/tJqJzJTrabWXYiqQDn4PaQdn0KrOARYkiAOPw95Z89BjafjFzh1fOvjBDP2vq0fjdyKpixahhNgGXsJ3J+tgLx3A8SCSbEdNBLM2FvEthm0ZtETbVJsgvGYXJE1AkmH34dWcRBczkDTx2p0ymAYmDIK/jBT017/IfDG2Xv378jKlx458vLGrWFZBnPPPsloLhJrUuyCqdDW1uitIs0s0Kb7NDGJnrE39zSlR9rUOaQ2lUObe89GwbNvqpFTWuNsYexDlUkgshvyri8g/bwWxFkLNi0PlrOugDBoPBjBAj41BahoCHqMWGJUvnQGDs9t7dkLPItUe8vEKn5AEbjcQZC2vAth4NioRAoFI6TbtWzZMkyZMgVDhgzBvn37jNfXrVuHuXPnYs6cOSguLsbatWuNbYcOHcK8efMwffp0zJs3D4cPH47J4MNF5Fmomjc8LVxaZsR5P7by1GEAJ0De93VYx2po8ggizQK1+8g4gXD6FOSydJKME82omXNH98G4YXlRO14wUpPEiDR7PdHNlIyjZ7JGcZFWlnXPPvjnnhGkPWE0SiXo5DVF9OgROcToO+s/0odITnh+/BCO/94Oz6a3wGb0hG3GbbBf8jDEoVP8ZrV3BnrDoUCVLzXNGxzR+ik0I9WKap+SCQzDwHLGfBBXPaTtH8VuwE2EvCunTp2KK664Ar/97W+N1wgh+NOf/oTXXnsNgwcPxp49e3DppZdi2rRpYFkWS5YswYIFCzBnzhx88MEHuPfee/HKK6/EdCJmsPj0oQ033lyvcpfk49m7iQC+/yjIBzbBcsalYDhznmejU4pIryeKB/K+r2FL85YOCFbm2OGj2XeGsfcnEXQXUuwiGlxyC63ZDL7rJKGIRRatZGj2we/tYO0Jaxs94Dk2ZK6AGVLtAmwWzliUrXfKcHnUNmGXxN0IacenkHZ8CkhOcH2HwzKyOKyn5Y4k2dYUcRPA2DvcMgjayrSZKZY2T4xczkDwA8dC+mkNhILJYQd7hEPIK1pUVOT3dZZl0dDgfZRqaGhATk4OWJZFVVUVdu3ahZdeegkAMGvWLDzwwAOorq5GZmbsJmIG0aemvZkvpC/N9ecF44dCVjQIg8dDOfAdlCPbIJxkLkOuwSmHFYlDNAXyng2QfvwQxFkLiHb05KYG1+x9vEzjx6mDNXtLFBZnO4NUu3cBrtHdNtU/GM39A0L/kCfbvclN0exFqzfUCa3ZN5VM8PNDoydURSP+m2EY5GbYjcgfo+9sUyQOkZyQtn/sNfKyG3z/URBHzgbXo3+7zx1LQnn2viqAL5kpVuz5tbbN/pYxv4Fy+Ad4Nr8D2+TrojtYHyL6+WYYBk899RRuvPFG2O12OBwOPP/88wCAkpIS5ObmguO8hpXjOOTk5KCkpKTTjb3h2UdQ00a/gEk2AQzDQOBZSIrmXUm3p0Pet9G8sXfJyDcRekaIBuXAJni2vAdSXw4udxCEM38Lz7f/xY0pn+H7hl4ATvL7XqdHAQPAZuHBwJtM09EyTrQ0+45CT1IzSiY4wzT2HvOePcsw3q5RsfDsQ3zuRhato+2569rRjtAfeZl2/NLUWtIIu0wTIP28BtKPK0E8jeBPGuM18pmxzaGIFjYLB45lQht7e2sZxwKXR2kTMs2mZEM89VxI2z+Ceto5Mfuxi8jYK4qC5557Ds888wxGjRqFH374AYsWLcLq1aujOrisrMjL2GZnp7R5rUeW90kkKdnqd3tQfqkCAPTvk4GsNBtEgQMvcMjJTUPV8Emo++5DZNhU8MnpIQ/ldCvIzkoKOAZCCJy//ICaL/8LqfwIxJx+yJzxF9hOHgmGYSANHIzGZ+/EGSX/RYalEHxqc9U9/Zgaw8BuE5CulEOqPgGblQfDseHPO0I0AqQkWTrsfO05D9FUNGz7HDUb3oSl1yD0HXIhAIAT+LCOyzbdI316pgesKOlLTqYdDo/arrH7vtdirQYA5OWkIjsj+Pmz0qxwy1qbcze4ZPTLS414TEpdBVhrMliL9/wn9U7Hpt1lSEu3o8ElYbT1MJI//wSe2nLYBpyOzMmXw5Lv32EJRkfdV4FIsYtQiP9xHCjz5k/0yU9vsb1fr3QAAOPnvtKmzcev+zdC++Ft5P72voBPVu2Zd0TGfvfu3SgvL8eoUaMAAKNGjYLNZsOBAwfQq1cvlJWVQVVVcBwHVVVRXl6O/Pz8sM9TVdUYUTeo7OwUVPhZrfe4vBplaXkDksOUGErLvcfzOD2okLxNQeob3KioaIDaezRA3kfZps8gnj496HE0QlDvkMAzxO8Y1ZoT8HzzGtTjO8Gk5sA6ZSH4gWPgYFg4KvUknDS8y8/GPPUDHHtlCWzFfwFrS20xb0dlBS6xfIXjL70IABghTkNNXZbfc8aCRpeEJCvfIecLdL3NoBzfBc+3r0OrPgo2ewCcB7bDfmgHxllG4NfjQ5GXZt7LLWu6Pm6HBxVyaMksycrjaFlDxGNvPe/qGm8Kf0O9E1CCnz/FJqC00tHm3FV1Lgzpkx72mLS6Mri/fR3qr9sAeKtJsml5GKylYJpFw/5PFQzd+Rkm2MuhcX1hO+928L1PRT0QdmRNe653tLBbeVTWOP2O40RpPQBA9kgttntblQO/HKmC1Y/5EQqL4f76VZRu+Qp8/8I2283Mm2WZgE5yRMY+Ly8PpaWlOHjwIE466SQcOHAAVVVV6Nu3L9LT01FQUIBVq1Zhzpw5WLVqFQoKCjpdwgFaavbh0uiSYRE4I9JB4FgjiYXL6AU2ewDk/RtDGnunW4FGSJuKl0RywvPDB5B3fAYIIixnLIAwbIqRft1mPLZ8rJRn4aLGlXB99DfYZ98JIAVElSHv+BTnlb8PjlUgDj8PyvGdOL9yI1a7B4U970iRZLVLa/ZaXRk8m96EcvhHMCk9YJ32f+AHFIHUl6Pxyxdxqfwt6raXQ+u9EGxqtqljOt0KOJbxG/pINA3y7i8g/fQJuKy+EE6ZiIwkK35q9ETcKKU1ZpOqAG+0TUmr7FaPpMLlUcMqgkZkN6StqyD99AnA8RBHzgF4AaSuHFp9GdLq9mOWvR7YuhUSkrEheQbOu/CSsCtNdjWSrXzA1oSNbv+avbEoH6BjlVAwCfKOz+DZ/I5fY99eQhr7Bx98EGvXrkVlZSWuuuoqpKenY/Xq1Vi6dCluvfVW4yZ9+OGHkZ6eDgBYunQp7rrrLjzzzDNITU3FsmXLoj7wSGiOxonM2Our8EDb2vjC4PHwfP0q1KpfwWX1DXic1tmzhGhQ9m6EZ/P/QFwNEE6ZAHH0RWBtwSs72q0CDjhzm6ru/R2uT56CY/wcOD59BaS+DMfZ/vjONgHXj50CvvoY5P/di6L6zwCMDXvukeCRI+/iFUuI7G76UV0LcALE0b+BeNq5Ri0VJi0XSbPuxMtPP48L2a1w/O9uWMZcDGHY1JAGytlU3ri14VbLD8K98RVolYfB5pwEtewXKId/xDlCGjiuP5zVBUjKan+NfiOpysSPbHqyBbuPtMyi1TV8M5o9IcS7nrTpTRBHDfhBZ8Iy5mKwSS0Dz10eBX948nNcMiYDb22px9TB/bu9oQe89XHKavyXnGh0yuA5ps39n5FiAQMEbGLCsDysk66FfHBztIcLwISxX7x4MRYvXtzm9eLiYhQXF/t9z8CBA/H222+3f3RRRve4IvXsk3yTJHw8ewAQBo6D59vXIW3/GNZJ14Jh/Rs6I3vWLkCtPAz3V/+GVnEIbO7JsM24DVx2f1PjsVu9DUz4vmNhneKtulf2zmNg0/JgnXkb3l7jQp7NuwjMZfbGFnEcxkrfQD7wPYSBY8Kef7hEO84+Gii/bod74ysgjVXgB58Ny5iLwNrT2+zHcRx+4k5Fat4onM9thOeb16Ac2QbbubcEjfV2NLWB1CEeBzzf/w/y7i/B2NNgnfp78CeNATQVypGtaNyyFjPl7VDf+QnOvqdDPPUc74J/hF6+rHizllkT709PEeH0KC0W0vXFYj3hKxBq9XF4Nv4bauk+sD36wTr1RvB5/p8abRYetuQkbCnlIWks8jqpO1W0SbbxOHA88AJtclMghy88xyI1SUR1feD2hFzuyW1qckWLxMqg9YmzD5dGl9yiSqUgtDT2jDUZwtCpkHeshbOuFNaJ14DL7O33OAIU5Bz6GM4DX4CxpsA6+XrwJ58R1pfcbuGN6A9h4BiA5ZDEuuHpPRYMx8Ph3tgiKmRP8hj0rf4FPTe+Ai5/SJvOOtGEENKl4uw1Zx083/4XyoFN3kSd4rvBBTBOOql2EWWSDbYL/gB595fwfP0KXB//DbYZfwiY6ej0eKuMEkKg7P8anu/eBPE0Qjj1HFiKLmh+H8dDOGk0GoRB+Nvr67GosAHppZvh+uhxsNkDIBbOAt+vMGwPWFLUkGGXOnrtm7pGD3KaDLCRUJXkX8YhqgJp22pIWz8EI9hgOft3EIZMCFnXJS/Djn1HvRE5XaXvbHtJaqpp70+Ca50960ssSlubJaGMfXs1e9/Sta09ewCwnHEpuNyT4fn6P3C+uwTiyGKII85vobtrZftwR9oq2H6phzDkbFjGzQ/Y4DwYSVYekqwZDUmEAaOQ5rOA4/S0LMhlsQh4X52I38vvwbPxFVjPuSlmtbQVVQMh6HTNnhACZe9XcG96E5A9EEddAHHEeaaS31LsAuqdEhiGgTh0MhhLEtxfPAfn6kdhn/lHMH4qWzrdMnJEN1wfPQ71+E7v09r428H16Of3HOkpFlRrKTiUOxpnTZkHef83kLaugnvtcrAZvb1G/6QxpotkyUrwloQtz623J5QMY68XQfPn2asVh+Be/yK06mPgB46D5cwFIaVGndxMO/YerQXQ/tLGXYUUmwhV819mPJixz0y1tqkE2lEklLHXjU9Emn2rNoIiz7bpA8swjLe2dc9T4PnmNUhb3oNyaAusE68Fm5oDz/f/w5Ddn6MKyRCm3wZrv9MjnosuFzjdihEXrqOoGiRZayEpWEUex6RUiGdcCOn7t6Ac2ATh5HERnz8YnijVsm8PWn053BtegnpiN7i8wbBM+B249J6m35+aJOJIqU8kxcAxYDgBrs9WwLlqGWzn39HC2BFCMNC9E9Olb6C6GFjGXw5h6OSg3nmGT3ITwwkQT5kIYfBZUA5sgrRtFdxfPAtmy3uwnuGt0R4KSQ7ektCXdD8lE2obJQg826K2D1EkSD+8D+mnj8HY0mA799awFw91A2+z8BFXeu1qJPlk0foz9r16+HfgMlIs2H2kOubj80dCGXueY8EAYVe+VDUNTo9iXGDAm6XY2rPXYW2psE39PeSBY+D56hU437sPjDUFxFWPQ2mj8eLxIfh7Oww90Jy443DLbYy9w8jkbB6vt+m4CvH0GVAO/wD31/8B1/MUv5p1e9ELrnWGjEMIgbJvI9zfvAaA8UoNp0wIWxLxljlu+WPO9y+EbcYiuNY8DdfKR7wGPykDmrMW7g0vYTazHZViH/S74GawqTkhzyHwHJJtAmp8yhYwLAdh0JngTx4H5fBWSJvfgeuL55D82ydCFsqSw5Bx0v2UTGidPauW7odr/YsgdaUQTpkAy9h5ET2F6uUR8jJtHdqZKZak+GTRti4mF9yzt8DlUYP2oogV3X9ZPAwYhoEYQeVLh8trPH1LHAg8F9DYG/v0H4Wkix8CP2g8mKR02Ir/gk32SbDY2v8omxSk8qVRKsHnZvL2odWgEsA26VpAkeDe8DIICT+PIRR6hnJHe/bE3Qj3ZyvgXv8iuB79kPSbByAWTIoo+iPVLsDlUdoUzeN7nwrbeX+E5qiGc+VfIe38HI6374Z6fBfec47Gj/2uNGXodQJl0TKMV5qzTr4OkF2Q92wIeSxvs3Fzn3mSlQfPsa08ew/Ski0gmgbPjx/AufJhQFNgO+8OWCdcHZGhB5o9+3jR6wGfypetsmg1QuBwKQH7S2emNNe172gSytgDgIUPv4FJc6kEc569L4w1GbZJ1yDpwvvA5w1Cg0uOyqOs3dIk4/gphuZb8VLHKHMsq2DT82EZfRHUX7fB/cVzIHJ0b7xodqkyi3J8Fxzv3APl8FaIYy6G7fw7wab0CP3GAKT4lExoDZ8/BPbz7wBxN8Lz9X/ApuWBL16CL90FLT5zM4RasOOyB4DLHwJpx1oQLfh9KyuaqbBLwOv4pCeLLdoT1jZK6GnzwLV6GaQt74EfOA5JFz0Avvcwc5MJQHa6DUlWHv3zYtcovqPxrWnvi8vjzaNJDnAfxKIAnlkSSsYB9NaE4ck4/gobiU21ccIl3HorgfCVcVrjr0ZLcx9ab3MN4bTpIKoMacu7cFb9Cus5N4HLMK9pB6NZxom9L0FUGVWf/RuuTR+CTcuDbe49psNXg6Ffo3qHZHxBfeFyBsI+525o5QfBDxrfJMXsC7vAXkaKiCNNGZeBEE+bAdfav0M5uDnoOoukBK5HpFYfh2fTm2BsKeD7jgDf+1TvU4WPjNPLtQ+za76BygHWSddBGDw+rLkEgudY/PWGM2CzdI3orGjgW9Pel0B1cXQyU733UqBY+1iScMbeIoQv4zRXvPSVccx59m2O5Qy8eBMOwWQch0/jEp3W3aoYhoGlcDa4nIFwf/7/4HzvPlgnXg1hYPuTrjpqgVZrqITrsxXQKg5BKJgMyxnzwfDRKeKlG/uGIO0JuYxe4DJ6AQiv4qUv6ckW1DtlI6rK73n6DQeTlgfp5zXgB44NqHvLstYmMxsA5F++g3vDv8BwIggIlH1fAyyHi7je2OnqDbU2F65ta3CFbT0aLD2RUXwL2LTo9g7oqEb3HYW9qcBga8++MUQXuvTkpsSqTpBxEs7YiwIHT5hVLwPJOIqqtWg6bIYGV2S17FsTrFuVfxmnydi3+qHjew2F/aL74fpsBdyf/z+opfu94aBc5LeG1AEyjnJsJ9yf/z8QTUXub/4EZ+bQqB5fv0Zme9HqP7C2sD375qiYHmn+F2AZhoV42rnwbHwFauk+8PlD/O4nKVqLBVqiKvB89zrknZ+DyxsM69Tfg7GlejN4j2xFyu7vcQ6+hvMtb/Odz13DkDtuPnpG2dDHIyzLGLH2vjSrAP6f3nmORWqySD37jsAisJDCLPUbyLMHAEXRTEedeGQVkqxFxcsReA4CzwY39hZfz977f39ljtmkDNhn3wXPprch/7wGasUh2KbfajqOujVGs3Ex+saeEAJp+2pIm98Bm94T9nNuRtKgQXBGuTBWiiHjmOtF6/QTAWWGjKYFu9oGKaCxB7zlOKTN70L+6ZOAxl5WVCP0UmusguuzZ6CVH4Bw2nRYxl5s5Hvw+UPA5w/BVjIOG77ahiXTk1BGMvDhJ/W4PbX9T52JQnBjH/g+yEyxdopmn3ALtF7PPnzNnufYFhq0HvUQjm6vP+KF07gkGHYrD6fHj2bvViDybAsvz1ez9wfD8rCecSms026EVnkY0o8fRjwuw7M3GQZoFiK54P70H5C+/x/4AaNhn3sP2PTYeKE2CweeY4LKOL40r5OEv0ALIGQTE4a3QBg2BcqRbdBqS/3uIykaBIGDcmwnnO8uhVZzHNZp/wfrGZf6LaiXnmxBhZaKhr7jUSb2Nl6jmCMlqLEPfB9kplhoNE5HIEai2Tu9ETS+WqluSMPR7RuaSiynREm/tFt4v31onR65jZygtyb0hCi9K5w0BvxJoyHv+xpEjsz70DX7aMbZq1VH4XzvPihHtsJyxqVeSUKITqNufzAM0xRrb1bGafs0ZYZwojOEoVMBjoP08xq/2yVFxZBGb9kFxpaKpAuWBG2oo2fK1jZ4UNsQvd6ziUJyAGPPMkzQGPqMVAuqGzwxCXsORmLKOBEs0LZeeGs29uaP1RBlzz7JKgRYoFXajDeYjNMaYegUKL98B/mXbyEWTAp7XNHS7ImqQDnyI+TdX0I9vguMLRW2WXcGlDGiTapd9Bt66Q9/uQ1mMOLdTRh71p4GYdCZkPdthFh0AYDmRhZEU1HMf4uhVXvB9x8F6+TrQzbo1mvg1DZKqG30QOTZuIqYiTVJNh5Hytoa+2Rb28qnvmSmWI1y0uFGb7WHhDP2kYZettbgIvHsm2WcKHn2Vh51fppGO91Km5vIGkbTcS53ENiM3pB3rYNwysSwsx49sjeTM5xm3b5o9eWQ96yHvPcrEFc9mOQsiEUXQiiYFPE6QiSkJAmod5iUcdwKbBYu7DkzDIOMFNF0L1rhtBmQ92yAvGsd0PcyAF55y/nZCpxl3Ytfs87E0HOuNZVIpnv2dY0e1Dm87QjjJcO1I0ixiX49++QQzlxz+KUbdj81lmJFwhn7SEMve+e0vCiRaPa6/hsoBjdc7FYeJyodbV53uhWktWpAIfIsGMZc03GGYSAMnQzP1/+BVnEQXM7AsMblkVXTNVp8Ie5GuNe/COXIVoBhwPcdAaFgErjep5kuBhZNUu0iSirNFa1yuBUj0S1cMpLNV0LkMnqC63M65F2fQ5t2iXch9pMnodWcwBuOcegzfAaGmcwYtlt4CDzr9ewbPGE1LaF4PXtZ0VqUiW50ykgO4a3rWbQ1DR70zu44Y594mn1TMpQWhl7mr9ZFZJq9V88L91E/EEkWb0p/axxuuU1UCMMwRn0cMwiDzgQEK6RdX4Q9LknWjDUCs2iOGjhX/hXK0Z8hjpyDpEv/Btv0W8H3Hd4phh7QZRzJlLbq8ihhR+LopKeE13hcPH0GiKse1V+8Cuf7D0BrqII2+WZ86xkcViKbnkVb2+jx1sUJUcee0hJdjvVNrGp0h/bs9XWajl6kTThjrxsh2aSUoxEChztKxt4pI9netqlBpNisvNHm0BeXx7+XaRV508aeEW0QTj4DyoHvQdyNod/gg6+nYwattgTODx6E1lgF23l/hKXoArDJnd/GMiVJgNTkuYXC0dSlKhIyUiyoaTS/YMf1LACb1Rf1m1cDLAf7nLsh9zgFAEwXQtNJS7Y0GXvJqHFPMYe+LuYr5fiTfFuTniKCYYDqAO0JY0XCGXtdfjFb097pVkBI21CqyIx9dBKqdJKsPAgAt493rxHiV7MHEJZnD3gXaqHKkPdtDGtcXhnHnLFXKw7B+eHDgCLBPvsu8D0LwjpXLDFKJphYpHW26lIVDhnJFsiK5jeyyh8Mw8ByxqVIGjoe9rn3gMvsbciJZj93nfRkC0qrnfDIqlHjnmKOlFb1cQghaHS27GjnD45lkR6GdBctEs7Yh9uHtjmhquUF1DVpKYxonNbdrtqLLgf5RuS4PQoI/Cf3WATOqFtjBi6rD9jckyHtWgdCzP+omW02rhzfBeeqZYBggX3O3eB69Dd9jo5Af0xvMLFIq3epioT0CIpj8T0LkHvBbUaJ6uZm4+F9pb0yDg27jISkVpUv3ZIKVSN+S1a0JiPFguoGKuPElHD70DaXSmjl2Tf9aIQv40TPe9I9SV+PUDf8/tL2vZ69Oe9RRxw6BaS+DOrxXabf45G1kNmz8sHNcH38BNjkHrAX3x31WizRIJySCe2RcXwX7CJFdzrMVr3UyfAx8NTYh0frMsf+yqoEIpxF+WiRcMbe8OxNGulAGXECF0HoZZTKG+s0F0NrlhkcQQpyhaPZ6/ADisBYkr2hfiaRQmj28v5v4P78GbDZ/WEv/jPYpIywxtRRNBdDCy7j6J3BIl+g1ePd22PsI5NxfKO2aDROeOjXu7WxN+PZp9gFOFzmcjiiRcIZe6MPrUmjZ1Sxa2Wk9ScEs8Ze0wgc0ZZxdGPvo9kbaft+JAWrJXzPnuFFCKdMgHJkKzRHjan3BNPs5V++g/vLf4LLPwX28++IuCFGR2B49iFkHH+F58JB96jb4+npAQfhLtCmU88+YniOhc3CG0beYaJUgk6yXUSjq21wRSwJeWcsW7YMU6ZMwZAhQ7Bv3z7jdY/HgyVLluDcc8/F7Nmzcc899xjbDh06hHnz5mH69OmYN28eDh8+HJPBR0KzZx+ejNO6GYHu2YdzHILoZc8CvjXtfWUcucU2X6xhavY6QsEkgBDIu780tb8kq35DL+WD38O97nlweYNhm74oauWIY4UocLCKXEgZxxHkMzcDz7HokWbFlr3lEV0foPk+DF+z914DS9NcKeHhWx+nIUQt+9bv04MpOoqQd8bUqVPx2muvoVevXi1ef+yxx2CxWLBmzRqsXLkSt956q7FtyZIlWLBgAdasWYMFCxbg3nvvjf7II6RZszcv43As0yaNPNxoHP1GiKaMY3Sr8rlhoi3jAACbmgOuz6mQ96wH0ULfnB5Za7NAKx/+Ae7PnwOXMxC2GX8ImcrfVTBTMiHY05RZrpg+BCcqHXjxo90R1UzR78NIPXvf3rMU8/hWvjRTBE0nUKerWBLyzigqKkJ+fn6L1xwOB95//33ceuutxg3So4e3BVxVVRV27dqFWbNmAQBmzZqFXbt2obq6czqqtyaSaJwkW9vYeIZhwHPmG5g06tmzUZRxrBYODIMWlS+bJQX/C7SSokHVwm+6IhZMAXHWQjmyLeh+hBBIrWQc5chWuD/zavS2mbfFtIBZtElJEkJWvoy0cYkvp56Uhd9MGogte8rx0XdHwn5/s7EPzzu3WTiIPIs0KuFERIpdMKTeRqcMBuZ+9FMCdLqKJRFp9kePHkV6ejr+8Y9/4MILL8Tll1+OLVu2AABKSkqQm5sLjvPedBzHIScnByUlJdEbdTswNPswjH0gnV0Mo1tVtIugATCycVvIOB5vlq6/R3Kj8qUUvrHn+g4Hk9ID0uZ3QJTA2rKkaCA+51J+/QmuT1eAzeoL+3l/BCMGrtneFUm1iyFr2gf7gQ2HGWP6YkxBDt5dfxA/HagK673GAm2Y0TgMwyA30468zO51XboKSVYfz97tdQzN1EfSPXu9Em5HENHdqaoqjh49iqFDh+LOO+/E9u3bsXDhQnz66adRHVxWVuR1I7KzU/y+npzq/WIKohBwH188iob0VKvffS0iB07gTB0H+ysBAP16pyMrSJOKcElJEqERxhgDYVgk2QTk5LQtGJad5V0MTUqxokd6+GNwzf4/lPz3fjBb30H2eTf43UdvYJ2VYUey4zBKP10OMacv8hcsAWeLXR0QU9cgAnKyknC4tCHo8VnBe2379EpHZmr7nlruuGI07ly+Ef9cuRNPLJqIniFqp+jjEpu8yZ55aWGXln7o9+MhClzIZKCuRKyud7hkZ9mx7ZcKZGenQFYJ0pJFU2Mj+hMYZ9J+6Odrx7wjMvb5+fnged6QaoYPH46MjAwcOnQIPXv2RFlZGVRVBcdxUFUV5eXlbaQgM1RVNULTwtcvs7NTUBGgc5F+vOpaZ8B9fKmtdyMv0+53X45lUN/gMXWcE+XefSSXhIowI2KCYRE4VNe5UFHhNUhVNU7YRM7vmOQmbfl4SR1IiLr2fkkeAHH4TDRs/QhyjyEQBoxqs0tlnQsAwFb/ipJv/gk2NRfiubehupEAjdHtJqUT7Hq3F4EF6hollJXXB2w/WVbhLSfhdrhR4aeZTLgsLB6K+/+9BUv/+S0WX1EUsDa677xral1gANTWOCLS3hWPDGdjxzfUiIRYXu9w4QC4PCpOlNShqtYFm8ibGpsuI5eUN5iei5l5sywT0EmOSMbJzMzE2LFj8fXX3t6Vhw4dQlVVFfr164esrCwUFBRg1apVAIBVq1ahoKAAmZmdX+sE8H4YAm++pn2DK3D6s7fpuPkQTpuFD9hUOlKSWnWrcgQolQCEV+Y4EGLRhWB79Id7w7/8hmJ6ZA3prAMD9/4bjGjzavQdWMY12qTYxZBRE06PAoFnw9bLA9Ej3YbfzxmGsmoXXli1y1R4ntzUf5YusnYsvolVDj8FEwNhEbxrJV1Ks3/wwQcxYcIElJaW4qqrrsL5558PALjvvvvw3HPPYfbs2bjtttvw6KOPIjXVKx0sXboUr776KqZPn45XX30V9913X2xnESYiz5rS7AkhQS+gyHNhReNEM8Zex27hWxgipydwJqfV0Owjf7JgOB62KQsBVYZ73fMgrRZ7ZWcDFiZ/DlaVYJt5W5coaNYejPo4QWLtnW45apVMdQr6Z+KSKSdj6/5KfGxiwVZS1LAjcSjtJ8XH2DeEYewBr27fpTT7xYsXY/HixW1e79OnD/7zn//4fc/AgQPx9ttvt390McIicpBMhF66PN5aF4EuoNBULtkMjVEugqZjtwptyiXoTaxbE063qmCw6XmwnnkZ3Bv+Bemnj2EZ4XUAiCojedM/wXH1qC5ciPTMPu06T1dAv2beiBz/CWCBCs+1l3OKeuPHfRXYtKsM55/RP+i+UhiN7ynRIylCzx5oamvYlTz7eETkOVOefaM7eNyswLOQVfPRONEMu9SxW1t59u7AddWjIePo8EPOBj+gCNLmd6FWHAIhGtxfvgBLzQH813EmSF7HtA6MNWYqXwaTztoDwzDITrP67VnQGl3GoXQs+ne6ut4NSdFM1cXRSbGLRv5NR5CQd4fZblWBSiXoCDxrui5+g0uOatilTpKVb6rN4p2Pt2OS/xtOD4d0h9mpyx8Mw8A64Sow9jS4Pn8Wnm/+C+XAJpQPmIkfpJPa3X+2q5CSZEbGadvzN1pYLTxcntDXS4qwOxilfejGvqzG29EsnO94CvXsY48omNPsQ2XEiSY9e0KI0bgk2uj1WJweBR5ZhaJqJhZooxMNxFiSYJ18PUh9OeSdn0EYOhWluWcDaH+z8a5Cso0HAwRNrAq2TtJebBYOLkkJmVXr9ezj4zPvTui2obTKa+zD+dFPtgnUs481FoEzpbU7jCp2QTR7Ez8abslrhGOi2Vua6+Po4w1UkMsicGAAuE14imbhe54Cy/jfQhg2DZYzfwtP0+caL8aeY715C8FKJjiDPE21F5uFByEIucYkKRr17DsBgWdhETmUVuuefXgLtC6PAsWkFNxeEq7hOOA1RDUmysnqv7qBQy/NReOEU/o0XHzLHOslGQJp9gzDwCKaW68IB3HYNOP/+rHDzeTsyqQmiQGLoTV3BouNjGMTmyubBuvrKytqt0qKiieSrQLKarz5JeFcA92JdLjkDilXET/fyDAQBXMeeaNLBsMEToMXTJZLaAih/bcHm2HsFThcodP2LRE0MAkHvWpjPEWGZKZYUNbkubXG7VFB0L4iaMGwWsxJb17PPn4+8+5Esl0w7EA44dXJJvslRIuENPYWgTNV9bLRJSPJKgTMnBRNhl7qem8sZBxdI3S6FTQ2xez6azauE2nlS7PockKgz6w7MqRvOo5VOFDnZ5FWLykdaeOSUOiefahFWlmmMk5n4bumF1Y0jk2vj0ONfcwQzUbjhIibFXgWiqqFXDwL1Mc2GjTXtJeb26IFMTzhNh0PF4+sxpVXDwBD+3sTw3Yfblu51ShvHCsZx6Ib+1CePU2q6ix0G2Gz8OBY89ego8scJ+TdIQpe3TqkkXZKQaUXszXtY1HxUsdoOu5pXqD1139WxyrE1thLkrlm492JfrkpSLLy2OnH2DuiVPEyEGaNvUxlnE5DN/bhOnPNZY47Jos2vr6VJrEILAgBFDWUR6606VDlix7qFir8ssElgef8lx1uLzzHwiJwTTJOUzROEP04kqbj4RCPnj3LMijol4Fdh2vaOAixl3G8n6XLhGYfbrNxSnTQjX24C+RJVMaJPWZr2jvcwWPjdY00VFicHq0RqyJVehatwyXDInJBi61ZLbHX7OMl7NKXoQMyUdPgMULsdKJVyz4Q+lNaMM2eENLk2Sfk17nTMTz7MNfk9B62dIE2hpjpVmUkQoXQ7IHQnr3LowQsUxsN7Fbeq9k7QxfksoqR9aE1i0eKP88eAIY16fY7D7WUcgwZJ8iieHswEuGCyDiRtiSkRAfDs49g3SbFLlDNPpY096ENbPQkWYOiauaMfYgnBJdHNR7HY0FSU+XLRpcUUk6wxFiz98hqXHr22ek2ZKdbsetwy7LOTo8ChmkOkYw2HMtCFNigMo7RpYpq9p2CbiMiqX3lLZlANfuYYeF1zz6wR66XHu0enr3QtEAbOpPT2pRUZaZGeiR4ZZz4vK2G9c/Enl9rWmQ86uWNYxlqagtRH8fw7OP0c+/qGMY+gtDqjiyZkJB3hyiG1uz1BKVgxl408aMBeBfXYi3jON0yGl1SyBBAvcxxrKQcjxSfnj3gDcF0SyoOlzR3C4pVeWNfbCIfNBpHamqgI0S5MQ7FHOnJIhjGm3wXLslUxokthmcfpMuUnkCTGiRcMjzPPnYG0G7ljdDLUDJONMsc+yMeo3F0TumXAQZoEYLp9MSuVIKOXgwtEHrl1Xj93Ls6ackW3HNlEcYOzQ37vSk2EQ1OOWQYeDRISGNvaPZSYCOtl7RNTTKj2Ycy9qqRCRkL7E2P+fUOKWiMPRD9ypetkeJUswe8T3n981NaGHtHDLpUtcYq8kGL1+nOBl2g7Tz656VG1HI0panUgplmSu0lIe8OIxoniGevF75KTWqfZ68RAneMNXs9CsAtqSEjAqLVrcofGiFNHZPi97Ya2j8TB4/XG7JKsGYx0cL7Yx5ExtGLz1Fj3+1INmLtY79Im5B3h5k4+3qHBFFgDePo9zhGnH3g43gkb6GsWGv2xv9NLNDq44o2+hNOvHr2gNfYa4Rg76+1ADpGs7eGknFoNE63pSNLJiSksdejRYI9OtU7pKB6PWAug1b3yGKt2fv7vz8sMdTsm8sbx6/ROblXGkSeNaQcRwzLG+t4F2iDhAnTOPtui172vCM6ViXk3WHGs69zSEgLIuEA5jR7V5NR7QgZp/X//RFLzV7/POPZsxd4FoP7pGPX4WrIircpTaxlHJuFh9sTuFuVLkfGs3wWryQbDe2psY8JPMeCY5mg8kuDUwqq1wPmNPtmzz62C7TG/0Mu0IbW7H8+WGWqyXVr9M8zWJONeGBo/0yUVDlxrMIBIHa17HVsFh4Ega+Z7mxQz777oZdY6IhYe1N3x7JlyzBlyhQMGTIE+/bta7P9H//4R5tt27ZtQ3FxMaZPn46rr74aVVVV0Rt1FNArXwai3mHe2Af70dDT3GMajROGjBMq9LKyzoUn39qO73aWhj0OvUdAvC8UDhvgLZ2wZU85gNiVN9ZpbmDi/5rRDNrui60pIa+xqyzQTp06Fa+99hp69erVZtvOnTuxbdu2Fts0TcMdd9yBe++9F2vWrEFRUREef/zx6I06Cni7Vfn3yDWNoMElhyxJzDIMeI4J6tk7O1izDyXjWELIOEfLGwFE5mkkgowDAL2yk5BqF7DZMPaxj8YBApc5prVxui8swyDZxncdzb6oqAj5+fltXpckCffffz+WLl3a4vUdO3bAYrGgqKgIADB//nx88skn7R9tFLEEaWDS4JJBCEJq9kBTH9ogmr27AzR7i8CBY73p+qEkBZZhgtbH0aUJKuMEhmUYDO2fico6N4DYG3urGNzYU82+e5NsFztExmnXXfr3v/8dxcXF6N27d4vXS0pK0LNnT+PvzMxMaJqG2tpapKenmz5+VlZyxGPLzk4Jut1uFQCW8btfo1wHAOidnxryOBaRAydwAfdjmx6t+/RKj+njfrJdgMMlo1fPtJCllO1WHgzH+h1zVX1TI3bW//ZgWI7XAwByc1LCfm976ejzjT0tH9/tKgMA9OmZjuzsyO/VUOQ3eh/xRZvYZp7Z2SkQRB4cyyAvNy1mY+hqdPT1jiUZqVa4Zc3UnNoz74iN/datW7Fjxw7cfvvtEZ88FFVVjdC08NOIs7NTUFHREHQfjgUaGj1+9ztyrNb7H0UNeRyeZVDf4P84AFBZ7QADoKHeBUeD28zwI8LaJJ1UVjaG3FfgWdTWu/2O+UDT3KtrXSHn3pqKKu+5nQ1uVHAd14PWzPWONn2y7Mb/3Q4PKhC7dHdPk55bVt6Aikyb8bo+79o6NwSe7fDPoLPojOsdS6w8i5JqZ8g5mZk3yzIBneSIn/s2b96MAwcOYOrUqZgyZQpKS0txzTXXYOPGjcjPz8eJEyeMfaurq8GybFhefawReQ6eAO0Em0slmJFx2JCavdXCxbwBt90qmC6xahU5v/XRFVUzmnNEJuM0LRTGuYwDAJmpVuQ3GfyOKIQGNK//tEZW1LhfFI9nUuwdU+Y44rv0+uuvx/XXX2/8PWXKFDz77LMYPHgwNE2D2+3Gli1bUFRUhDfeeAMzZsyIyoCjhUXgjJ6trdGLoJnT7Nmg9ezdHjVoFm606JFmhWLyKcgq+u9WVVLlhNp0jFBt8PwhJcgCrc7wgT3gdJdGVBMlHPTF/UANTCRFMxL8KN0Pb+VLBRohMXUKTVmhBx98EGvXrkVlZSWuuuoqpKenY/Xq1QH3Z1kWjz76KJYsWQKPx4NevXrhsccei9qgo4EosIE9e6e3Z6yZRdVQnr3LE7rGfDS4csYpyMpKgsvhCbmvVeSMHzRfjlV4ZZi8THvQjM1A6NE4iRIVcsGEAThndJ+Yn8dYoA0SekkXZ7svyTYRGiFweZSIul2ZxZQVWrx4MRYvXhx0ny+++KLF3yNHjsTKlSsjH1mMCRqN0xRjb6ZnrMhzRpyzP1ySErMuRr7YrTyS7aJpY19W03buxysc4FgGA/JTsKep9ks4eMsbszGXrLoKAs8hIyX215ZlGVhELnDopawmzA9sPJJia86ijaWxT9g7RAxi7OucUsgYex2BZ404Z3/EuktVJFhFzm+c/bGKRuRn2ZFsEwPqw8HwyPHZbLwrYAti7CVFowlV3RijGFqMY+0T1thbgmTQ1puoi6MT2tjHtpZ9JFhF3m/Vy+MVjeidnQybxduUPNxIqHiuZd/Z2Cx8QBlHVjTq2XdjmksmxHaRNmHvEFFgoagEqtbWUJupeKnjNfaB9e2u6NlbBK8x9y2s5XQrqKr3oFd2kjHecIulxXOXqs5GL4bmD4lG43Rr9Cg66tnHCKOBSavsV40QNDhlU2GXgLcOTCjNPpalEiLBauFA0LLq5/Gm+Pxe2cmGsQ9XyvHIatw2G+9sgsk4sqJBoD+y3RajzHGMs2gT9pspGsa+pVfudCtQNWLa2AscByWAsVdUb7uxrubZ+6t8ebypTEJvX88+zIgciWr2McMaRMaRZI169t0YUWAh8GzMyxwn7B2ie6Ctdfs6E71nfRGEwJ69URenq2n2ej1/H+NxrKIRVpFDVqrVeBKJxLOnMk5ssAVpTUiTqro3DMMg2SZQzT5W6NELrWUcPXs2zaxmz3kXaP01luiIWvaR4K/M8bEKB3plJ4FhmvMLws2ilaixjxneblWBo3F4auy7Nd4sWurZxwS9MmNrz77BRKNxX/RkFsVPYlVHtCSMhNbdqgghRiQO0PwkEm4WLdXsY4fN4q1UqvlxKmQaetntSbEJVLOPFYGahdeFURcH8Hr2APyGX3ZZz97SUrOvbZTgcCvNxt7w7Klm31XQ11lah8yqmgZVI1TG6eZ0RJnjhL1Dmj37tjIOyzBIMllUTI+C8Kfbd0T/2UjQDbJu7I83lUnonZ0EIHSzjEBQzT526MXWWl8To3EJfaLq1iTbBLpAGyv0x97WMk69Q0KKXTCd8t8tPftWMo7esKRXk2evlzwIx9hrGoGsUM8+VujXrPU1oS0J44MUmwCXR/ErB0eLhDX2lgChl2Z6z/qia/b+PPvm/rNd64vYOvTyWEUj0pJFI7nDu0gbOK7bH4nSkrCzMKS1VjIObTYeH+hZtIEq8UaDhL1DxAChl/XO8Iy97tn7i7V3dlHP3iI2zd3H2Pdu1WkpWKifPwwPk8oJMaE596G1Z9/UkpAa+25NclP0Xyx1+4S9QywBtPZwSiUAzVqp5KdkgltSwbFMl/O6OJaFyLPe6A6N4ESl09DrdbzG3vwCLfXsY4v+dNg696G52Tj93LszHVEyoWtZoQ5E4FkwaBndQAhBnUM2XQQNCK7ZO5vq4pgpldzR6JUvy2qcUFQNvXq08uyDpOf7Q5KosY8lNkvbrGeAPlHFC0aZY+rZRx+GYbxljn08cpdHhaJqYWr2gaNx3B7FWFjralhFHm5ZbS6TkOPPsw9Ds9flBGrsY0KgRDe9SxqVcbo3zWWOY5dFm9B3iCiwLUIvmxOqzDcQ0CUaf5q9y6N2SJeqSLCIHNweFccqGsEwQM+sVsbeyodVLqHZs0/oWypmWEJE41AZp3uTTD372NK6W5WRUBWOZs8H1uxdHsVIYOpq6DLOsQoHcjLsbTxym8V/n9pA6D+a1LOPDSzDwCpybdZRZCP0MqG/yt0enmNhs/BUs48VYqsGJvVhZs8CzfHNgeLsu6pnrzcdP17RiN49ktps12ux+Kv54w+6QBt7vA1M/Efj0KSq7k+sSyYk9B1iEdgWhdDqw6yLA/h69v4yaDum/2wkWEQODU4J5TUu9Mr2Y+wtHFSNBK3V74tEjX3M8dfAhCZVxQ/JdsGQkmNBQht7kW/r2TNoTnAwQyjNvqvF2OtYRQ5V9R4QoE2MPRB+yQT9c6RRIbHDX4QUTaqKH7xljqlnHxMsYkvNvt4hIckmgGPNfyyBPHtCiLclYRerZa/jGyXUO6etsbdGaOypZx87/DUwoUlV8UOXkHGWLVuGKVOmYMiQIdi3bx8AoKamBtdddx2mT5+O2bNn46abbkJ1dbXxnm3btqG4uBjTp0/H1VdfjaqqqtjMoB2IPNvCs68Lo9G4Dssw4DmmjWYvK95qhF2tvLGOXjJB4FnkpNvabA+38qVH1sCAepixxF84bHNSFf3cuzspdrHzF2inTp2K1157Db169TJeYxgG1157LdasWYOVK1eiT58+ePzxxwEAmqbhjjvuwL333os1a9agqKjI2NaV8EbjtNTsw9HrdQSebRON01UrXuronn3PrCSwbNukr3BlHL1xSVdMIIsX/Mk4kqJ5EwTp597tSbYLkBStTQmXaGHK2BcVFSE/P7/Fa+np6Rg7dqzx94gRI3DixAkAwI4dO2CxWFBUVAQAmD9/Pj755JNojTlq+IvGiczYt+1D21UrXuroxr51mQSdcLtVSbRxScyx+ZFxZNp/Nm4wYu1jtEgbFUukaRpef/11TJkyBQBQUlKCnj17GtszMzOhaRpqa2uRnp5u+rhZWW21ZLNkZ6eE3Cc9zQZZ1Yx9G10ycrOSTL3XF6vIgeW5Fu+rdXuNZF52StjHaw9mz5Xd9NkOGZDl9z0a5/0x4EXe3DE5Fjar0KFz9aWzztuR9MiwwyOpyMxKBtf0NMbyLCxmr1EcEY/z7ZWXCgAQLGLA+bVn3lEx9g888ADsdjsuu+yyaBzOoKqqEZpmLs7bl+zsFFRUNITcT5UVeCQVZeX1kBUNLo8KgYWp9/rCsQwaGj0t3neitB4AILmlsI8XKWbnDQCyx6sNptt4v+9xub3byysbTR2zocEDnmM6bK6+hDPv7ozWJBUeO14De9MPa0OjBzzbOZ97ZxGv11uTvQ7i0RO1SLO2XeszM2+WZQI6ye029suWLcORI0fw7LPPgm2KYsnPzzckHQCorq4Gy7JhefUdgR45Iitac0JVGNmzOnrTcV/0hc2uKuMM65+JiycNxCn9Mvxu16OIzJZM8MgqjfWOMVafRXO71fvILykaTaiKE1JiXOa4XXfJE088gR07dmDFihUQxWYjeeqpp8LtdmPLli0AgDfeeAMzZsxo30hjgJ7a75HViLJndQSBhdx6gbaLa/YWkcPMcf3Ac/5vAZZlYBE40yUTaLPx2GMsmvtk0UqKalRepXRvYl3m2JQlevDBB7F27VpUVlbiqquuQnp6Op566ik899xz6N+/P+bPnw8A6N27N1asWAGWZfHoo49iyZIl8Hg86NWrFx577LGYTKA9GF2m2mvsObZNnL3+heyqxt4MNgsXlmefkWyJ8YgSGz0b2+0TDksXaOMHu5UHwwANrk5coF28eDEWL17c5vW9e/cGfM/IkSOxcuXKyEfWAVgMz15DXdMKeLhx9oD3CUH/sdDRPfuuWuLYDOGUOZZkjRZBizH+pDVJ0Yxm5JTuDcswSLYJMfPsE9olEH360OrGOiVKmr3bo0Lk2YAySXfAXy2WQHhlHGrsY4nVaGDSfE1kRaWefRwRy5IJCX2XWFoZe7uFjygTURDaJlXpXaq6MzYLD6fJDFqJGvuY4y/RTW5KqqLEB7HMok3ou6S56bg3GiclAgkHCODZS123lr1ZwpFxPLIKUUzo2ynmWI0GJs0/wJKi0SioOCKW9XES+tvZwrN3ykgLo9qlLyLPtTH2To8Cexeti2MWu4VrUz/dH6qmQVEJLNToxBSLyIGBH8+eRkHFDbEsc5zQd0nr0MtIInEAb20cf5q9tYtWvDSLVTTn2Uu0S1WHwDJMU+XLlqGXVLOPH5JtAhpdCjSTTYPCoXtbo3bSWrNP7e8/wSgU3kJoGgghRkEql0dBWpI9amPtDOwWHpKsQVG1oAvNRnnjbhx51F2wWTgj9JIQAlnWaP/ZOOK0k7JQWedGLMraJbSx1z0ih1uB06O0y7MHAEUlEPgmY9+Fu1SZxWZEf6hItgU29npPAOphxh6bz9OWomogoJ97PDG4TzoG90mPybET+i7RPfvKOjeAyBKqgOYvm28WrSsOonH0H6tQUo7ebJxG48Qe3z60RpN3auwpJkjou4RlGfAci4paFwAgLYIYe6DZs9d1e40QuD1ql+1SZRazNe2pjNNxWC2cEY2jP1EJ9EeWYoKENvaAt+l4ZZ3X2Ecu4zRp/03G3iOpIOjepRIA8zXtaUvCjsNXxqHyGSUcEv4uEQUO1fUeAIg8zr6VZ99cBK17Gz+zrQkl2my8w7D5hMPqP7I0qYpihoS/SywCB7WpZn6kMo4Y0NhTz54SXbwlLFrKODSpimKGhDf2ujdqEbiINec2nn0X7z9rFn38oSpfSnSBtsOwiTw8sgpV04zPnSZVUcyQ8HeJbqBSkyLLngWajb1eHydePHs9A9gdIovWI+kyDjX2scbqEw7roZo9JQwS/i4RDWMfmYQDNC/QtpFxunl0Cs+x4FgmtGev6DJOwt9OMcfmEw5LZRxKOCT8t9Pw7CPU64H41ewZhmkqhhZ8gdYjq2AYdOtyzt0FPZzX7VGbQy+pZ08xQfe2RlFA1+wjaVqi0zYaJz40e8Abax9ygVbSYBE4o1QEJXb4rqPoob5UxqGYoftbo3ZiiYqM01azZxAfSUZmyhxLikr1+g7C5tPARH/goklVFDMkvLHX9c5oGPvmaBxvLXs2Djxdm4UzFXpJ9fqOoVmzVyE3FUaknj3FDAl/l1iaGm60T7Nvu0Db3ROqdMx49vUOCcm2yD8/inn0stm+C7RUs6eYIeHvklh49m6PGhd6PQBTC7SVdW5kp1s7aESJjVGvSFKMhXGO7f5PkJTYk/DGPhqaPcsy4FjGWDBzSUq3L4KmYwvRwEQjBFV1bmSlUWPfEYgCC4bxyjiS7G1JSBfGKWYIaeyXLVuGKVOmYMiQIdi3b5/x+qFDhzBv3jxMnz4d8+bNw+HDh01t62oM6ZuOUYOz0aOdxsq3W1U8lDfWsVm9tVhIgM45dY0SVI2gR5qtg0eWmDAMY/wAS7JKJRyKaULeKVOnTsVrr72GXr16tXh9yZIlWLBgAdasWYMFCxbg3nvvNbWtq9E3NwX/d+Fp7Y4RF3nWqGfv8qhxpdkT0lz/pjV6xdD2/lhSzOPtVuWVcWjxOYpZQt4pRUVFyM/Pb/FaVVUVdu3ahVmzZgEAZs2ahV27dqG6ujrotngmbj37EJUvK2u9jV+ose84vA1M1CbPPj6cCkrsicgilZSUIDc3FxznvdE4jkNOTg5KSkpACAm4LTMzM6zzZGUlRzI8AEB2dkrE740Eq0UAw7HIzk6BW1aRmW7v8DEA0Z93bg/vNbDaRb/HdiknAABDBmZ3aiG0zvisO4uUJAtUQiArGuxWPqHmrpOIcwbaN+8u7X5WVTVC08Lvsp6dnYKKioYYjCgwLAM0OiSUlNZ5C4OpaoePIRbzlpsWZ4+X1sPGtV0IPHKiDmlJIuprnVE9bzh0xvXuTHiWQV2DB0wqAxZIqLkDiXe9dczMm2WZgE5yRMY+Pz8fZWVlUFUVHMdBVVWUl5cjPz8fhJCA2+IZXbN3N1WAtMaJjKOH+rkDRORU1rmphNPB2CwcymtV2OkCLSUMIrpTsrKyUFBQgFWrVgEAVq1ahYKCAmRmZgbdFs/omr0epmiPE2OvNx0PVPmShl12PN4GJt5oHFqmgmKWkMb+wQcfxIQJE1BaWoqrrroK559/PgBg6dKlePXVVzF9+nS8+uqruO+++4z3BNsWrwg8B8nH2FvjJM4+WNNxTSOoqnfTsMsORg+99Mga9ewppglpkRYvXozFixe3eX3gwIF4++23/b4n2LZ4ReBZyKqvZx8fHlewaJzaRk9TjD317DsSm6XZsaB1cShmoXdKlBB5FrKsGS0J40Wzt4gcGPj37CvrmsIuaamEDkW/t2obPDT0kmIaauyjBN/Gs48PY88yDKwBKl82J1RRGacj0UtxKKpGPXuKaeidEiVEnoUkq0bUSrx49oCexBPYs89KtXT0kBIa3+xs2mycYhZ6p0QJXbN3xplmDwSufFlZ60ZaskilhA7GNzub9p+lmIUa+ygh8JxXs/eo4FgmrvqxBqp8WVnnoouznUBLYx8/9xklttA7JUoIPAsCoNElwWbh46rsrM3C+42z9yZUUb2+o7H6tLukoZcUs9A7JUroHla9Q46bipc6epVFX1RNQ02Dh3r2nYDv4j9NqqKYhRr7KKF7WHUOKW4qXur4a01Y2yDRGPtOwnfxn3r2FLPEl1XqRATDs5fizgB6ZZyWC7Q07LLzEHkWLMNAIyRmmr2qKqipqYCiSDE5fnsoL2ehaVpnD6PD8Z03z4vIyMgGx5k34dTYRwnD2Dsl9MmJvDRzV8Rm4aGoGmSlOT3fSKiKsx+27gDDMLBZODjcSswioWpqKmC12pGUlNfl1p94noWiJJ6x1+dNCIHDUY+amgr06GG+wCR9BowSegicrGhxp9n7NrnWqaxzgwGQmUqNfWegS4Wx8uwVRUJSUmqXM/QU7499UlJq2E9d1NhHCV/tNN40ez36w1e3r6xzIT3FQjXjTkIvtBfLz58a+q5LJNeGflOjhBjHxt5f5Uta2rhz0ZP26I9tx/PQQ0vxzjtvdvYwwobeKVHCVzuNN2Pvr/JlRS1tWtKZ6BE5iRh6qSj+eyt093PFmviySp1ICxlHjK8voK2VZ09j7DufWGv2XY2zzirCVVddh2+//RpnnHEm5s+/DMuXP4kDB/ZDkiQUFhbh5pv/gOPHj+Ivf/kTXn31LSiKgvPPn4orr7wGCxZcgc8//xRfffUlli59CK+//io+/3wtVFWBKFpw++13YdCgIW3ONXbsGZg79yI8+OASVFVVIi8vHyzb/Jl/8MG7eOut/0IQRBCi4f77H0G/fv075TMKBTX2USKeZRx9wVk39jX1HmiE0LDLTkR3KDpKxvn65xJs/KkkJsc+6/R8jD8tdFSJxWLBCy+8Ap5n8eCD92HEiJG46657oGka7rtvMVav/hDFxRfA6XSgsrISpaUnMGDAQGzZshkLFlyBH374HkVFowEAM2acj0svvQwAsHnzJjz22F/x/PMvtzkXANx99x0YPrwQV199PY4fP4bf/W4Bxo49AwDwzDN/x2uvvYMePXpAkqQuHRIaX1apE/H90sVTxUugrWdvVLuknn2n0ezZx9dTZDBmzpxl/H/jxg3YvXsn3njjNQCA2+1GTk4uAGDkyCL88MP3KCk5gTlzLsRrr70CWZaxZcv3uOyy3wEA9u7djf/85yXU19eBZVkcPfprwHP9+OMPWLToDgBAr169jR8M77lG46GHlmD8+LNxxhlnoVev3jGZezSIL6vUifga+3ipZa8TyNhnU2PfafTskYQeadYOK3E8/jRz3ncssdnsPn8RPPzw436N66hRo/HDD5tx4sRx3HvvA9i27Ud89tkaEAL07NkLsizjnnvuxD/+8U8MGXIKKisrMHfuzCDnCszDDz+G3bt34ocftuCWWxbi9tv/jDPOGN+eacaMxBD8OoAWnn2cafY8x0LgWWOBtrLORWPsO5nxp+XjpXung03Q8Mjx4yfg1Vf/DVX13pO1tbU4ceI4AK+x37TpWzQ0NCAnJxdFRWPw4ovPGR65JHmgqqrxJPDuu8FbqI4aVYTVqz8EAJw4cRxbtmwG4F28PXHiOIYOPRWXX/47jBkzDvv3743JfKNBfLmgnUg8e/ZAywYmlXVupKdY4qqMM6V7ceutf8QzzzyN3/3uUjAMA0EQccstf0TPnr2Qk5MLu92O008fAcBr/MvKSjFyZBEAICkpGddccwOuu+4KpKamYfLkqSHOdTsefHAJPvtsDfLze6KwcBQAQNM0PPTQUjQ2NoBhWOTm5mLhwptiOu/2wBBCSGcPIhBVVY3QtPCHl52dgoqKhhiMKDjXPboOqkbw9K1nI9kmdPj5YznvPz//HfrlJmPhnFPxyGs/ghCCP182KibnCpfOut6dTSznXVp6BHl5/WJy7PaS6OUSdPxdI5ZlkJXlv1xLu12zdevWYe7cuZgzZw6Ki4uxdu1aAMChQ4cwb948TJ8+HfPmzcPhw4fbe6ouj+7dx1u5BMAb/aHXtK+iTUsolG5Hu/QGQgj+9Kc/4bXXXsPgwYOxZ88eXHrppZg2bRqWLFmCBQsWYM6cOfjggw9w77334pVXXonWuLskAs9CIwQcG3/yhl7mWFE1VDd4kEXDLimUbkW7rRLLsmho8D5KehdEclBTU4Ndu3Zh1ixv+NKsWbOwa9cuVFdXt/d0XRqRZ+Muxl7HbuHh9qioafCAEBqJQ6F0N9plmRiGwVNPPYUbb7wRdrsdDocDzz//PEpKSpCbmwuO88oZHMchJycHJSUlyMzMjMrAuyI8z6HjlfqOwWrxyjiVtXode2rsKZTuRLuMvaIoeO655/DMM89g1KhR+OGHH7Bo0SI8+uijURlcoIUGM2Rnp0RlDOFgt/IQeLZTzq0Tq3NnpdvhlirhaVofGnxSD2RnJcXkXJHQmZ95ZxKreZeXs+C7cCmGrjy2WOI7b5YNz9a0y9jv3r0b5eXlGDXKG5UxatQo2Gw2WCwWlJWVQVVVcBwHVVVRXl6O/PzwkjK6WzSOhWdhFflOiwyJ6bw1DS6PgoPHasAwAJGVLhMBQ6Nxoo+maV024oVG43jRNK3N9Y9ZNE5eXh5KS0tx8OBBAMCBAwdQVVWFfv36oaCgAKtWrQIArFq1CgUFBXEt4QDA1ecV4LJzB3f2MGKCvhZxrNyBDBpjT6F0O9rl2WdnZ2Pp0qW49dZbjWL6Dz/8MNLT07F06VLcddddeOaZZ5Camoply5ZFZcBdmR7p8Ruhohv7o+WN6EEzZymUqPLQQ0txyikFuOiieTE7R7tDR4qLi1FcXNzm9YEDB+Ltt4OnIVO6D7qxr6p3Y0jf9M4dDIUSZRRFAc93TCRdR57Ll/iME6REHd9EMRqJQ+lozjqrCNdd93t89dV61NfX4U9/uhtbtnyPTZu+gaIoeOCBZejffwCqqiqxdOndcDgckCQJZ545HjfeeGvAY/rWrV+w4PK4rpFPjT3FFDax+VahpY0TD3nf15D3bojJsYUhEyAMDl0pMjk5BS+88ArWr/8cf/7zH7F06cNYuPAmvPbav/HKK//Cvfc+gOTkFCxb9iTsdjsURcFtt92E7777BuPGnen3mL516x955IG4rpFPjT3FFL7JYrRpCaUzmDr1XADAkCGnAGAwfvzZTX8XYP36dQC8ESrPPPN3/PzzTwAIqqqqsH//voDGPpFq5FNjTzFFS2NPPftEQxg83pT3HUtEUQQAsCwHUWxOX2RZ1ih1/Oabr6GhoR7PP/8yLBYLli17CJLkCXjMRKqRT+PnKKbQyzazDIPMVEsnj4ZC8U9DQwOysnrAYrGgoqIcGzeuN/3eeK+RTz17iilEgQXLMMhIscRloTdKfHDxxfNxzz134vLLL0F2di5GjRod+k1NxHuNfFrPPo6I9bxvfmoDemcn487fjozZOSKBXu/oQ+vZdz06vZ49JXHIybChb25i1qChULo7VMahmOZPC0aCYxOz5ymF0t2hxp5iGosQfx24KJREgco4FArFL114OS/hieTaUGNPoVDawPMiHI56avC7IIQQOBz14HkxrPdRGYdCobQhIyMbNTUVaGys7eyhtIFl2aiVEOhO+M6b50VkZGSH9X5q7CkUShs4jkePHuE1G+ooaKhtZFAZh0KhUBIAauwpFAolAejSMg7bjpju9ry3O0PnnVjQeScWoeYdbHuXLpdAoVAolOhAZRwKhUJJAKixp1AolASAGnsKhUJJAKixp1AolASAGnsKhUJJAKixp1AolASAGnsKhUJJAKixp1AolASAGnsKhUJJAOLK2B86dAjz5s3D9OnTMW/ePBw+fLizhxQTli1bhilTpmDIkCHYt2+f8Xq8z7+mpgbXXXcdpk+fjtmzZ+Omm25CdXU1AGDbtm0oLi7G9OnTcfXVV6OqqqqTRxtdbrzxRhQXF2Pu3LlYsGABdu/eDSD+r7nOP/7xjxb3e7xf7ylTpmDGjBmYM2cO5syZg6+++gpAO+dN4ojLL7+cvP/++4QQQt5//31y+eWXd/KIYsPmzZvJiRMnyOTJk8nevXuN1+N9/jU1NeS7774z/n7kkUfIn//8Z6KqKpk2bRrZvHkzIYSQFStWkLvuuquzhhkT6uvrjf9/+umnZO7cuYSQ+L/mhBCyY8cOcs011xj3eyJc79bfbUJIu+cdN559VVUVdu3ahVmzZgEAZs2ahV27dhmeXzxRVFSE/PyWtcYTYf7p6ekYO3as8feIESNw4sQJ7NixAxaLBUVFRQCA+fPn45NPPumsYcaElJQU4/+NjY1gGCYhrrkkSbj//vuxdOlS47VEuN7+aO+8u3TVy3AoKSlBbm4uOM7bFJvjOOTk5KCkpASZmZmdPLrYk2jz1zQNr7/+OqZMmYKSkhL07NnT2JaZmQlN01BbW4v09PTOG2SUufvuu/H111+DEIIXXnghIa753//+dxQXF6N3797Ga4lyvW+//XYQQjBq1Cjcdttt7Z533Hj2lMTigQcegN1ux2WXXdbZQ+kwHnroIXz55Zf4wx/+gEcffbSzhxNztm7dih07dmDBggWdPZQO57XXXsOHH36Id955B4QQ3H///e0+ZtwY+/z8fJSVlUFVVQCAqqooLy9vI3fEK4k0/2XLluHIkSN46qmnwLIs8vPzceLECWN7dXU1WJaNKy/Pl7lz52LTpk3Iy8uL62u+efNmHDhwAFOnTsWUKVNQWlqKa665BkeOHIn7661fQ1EUsWDBAvz444/tvs/jxthnZWWhoKAAq1atAgCsWrUKBQUFcfM4G4pEmf8TTzyBHTt2YMWKFRBFEQBw6qmnwu12Y8uWLQCAN954AzNmzOjMYUYVh8OBkpIS4+8vvvgCaWlpcX/Nr7/+emzcuBFffPEFvvjiC+Tl5eHFF1/EtddeG9fX2+l0oqHB22uWEIKPPvoIBQUF7b7P46p5yYEDB3DXXXehvr4eqampWLZsGU466aTOHlbUefDBB7F27VpUVlYiIyMD6enpWL16ddzPf//+/Zg1axb69+8Pq9UKAOjduzdWrFiBH3/8EUuWLIHH40GvXr3w2GOPoUePHp084uhQWVmJG2+8ES6XCyzLIi0tDXfeeSeGDRsW99fclylTpuDZZ5/F4MGD4/p6Hz16FDfffDNUVYWmaRg4cCAWL16MnJycds07row9hUKhUPwTNzIOhUKhUAJDjT2FQqEkANTYUygUSgJAjT2FQqEkANTYUygUSgJAjT2FEoRnn30Wd999d0Tvveuuu/Dkk09GeUQUSmTETW0cCiUWLFy4sLOHQKFEBerZUygUSgJAjT0lrigrK8PNN9+McePGYcqUKXjllVcAAMuXL8ctt9yCRYsWobCwEBdccAH27NljvO/555/H2WefjcLCQkyfPh3ffvut8b7bb7/d2O/zzz/H+eefj6KiIlx++eU4cOCAsW3Xrl244IILUFhYiEWLFsHj8bQY27p16zBnzhwUFRVh/vz5ps5PoUSNqFTap1C6AKqqkgsuuIAsX76ceDwe8uuvv5IpU6aQDRs2kKeffpoMHTqUfPzxx0SSJPLCCy+QyZMnE0mSyIEDB8iECRNIaWkpIYSQo0ePkiNHjhBCCHn66afJH//4R0IIIQcPHiTDhw8nGzduJJIkkeeff55MmzaNeDwe4vF4yKRJk8hLL71EJEkiH3/8MRk6dCh54oknCCGE7Ny5k4wbN45s27aNKIpC3n33XTJ58mTi8XiCnp9CiRbUs6fEDT///DOqq6tx0003QRRF9OnTB5dccgk++ugjAMCwYcMwY8YMCIKAq666CpIkYfv27eA4DpIk4cCBA5BlGb1790bfvn3bHP+jjz7CxIkTMX78eAiCgGuuuQZutxtbt27F9u3bIcsyrrzySgiCgBkzZuC0004z3vvmm29i3rx5GD58ODiOwwUXXABBELBt2zbT56dQ2gNdoKXEDcePH0d5ebnRyQfwlv0tKipCz549kZeXZ7zOsixyc3ON/f/yl79g+fLl+OWXX3DWWWfhrrvuQm5ubovjl5eXt2geoZdXLisrA8dxyM3NBcMwxnbffU+cOIH3338fr776qvGaLMsoLy/HmDFjTJ2fQmkP1LOnxA35+fno3bs3tmzZYvzbunUr/vnPfwIASktLjX01TUNZWRlycnIAALNnz8brr7+OdevWgWEYPP74422On5OT06KeOCHE6BaVnZ2NsrIyEJ+6gr775ufnY+HChS3Gtn37dqOloJnzUyjtgRp7Stxw+umnIykpCc8//zzcbjdUVcW+ffvw008/AQB27tyJtWvXQlEU/Pvf/4Yoihg+fDgOHjyIb7/9FpIkQRRFWCwWsGzbr8bMmTOxfv16fPvtt5BlGf/6178giiIKCwsxYsQI8DyPV155BbIsY+3atfj555+N91588cV44403sH37dhBC4HQ68eWXX6KxsdH0+SmU9kBlHErcwHEcnn32WSxbtgxTp06FJEkYMGAAFi1aBACYOnUqPvroI9x5553o168fli9fDkEQIEkS/va3v+HAgQMQBAGFhYV+28CddNJJeOyxx/DAAw+grKwMBQUFePbZZ40mKsuXL8c999yDp556ChMnTsQ555xjvPe0007DAw88gPvvvx9HjhyB1WrFyJEjUVRUZPr8FEp7oPXsKQnB8uXLceTIESqPUBIW+qxIoVAoCQA19hQKhZIAUBmHQqFQEgDq2VMoFEoCQI09hUKhJADU2FMoFEoCQI09hUKhJADU2FMoFEoCQI09hUKhJAD/HyGIMZHomwikAAAAAElFTkSuQmCC\n" - }, - "metadata": {} - } - ], - "source": [ - "if __name__ == '__main__':\n", - " cfg = PPOConfig()\n", - " # train\n", - " env,agent = env_agent_config(cfg,seed=1)\n", - " rewards, ma_rewards = train(cfg, env, agent)\n", - " make_dir(cfg.result_path, cfg.model_path)\n", - " agent.save(path=cfg.model_path)\n", - " save_results(rewards, ma_rewards, tag='train', path=cfg.result_path)\n", - " plot_rewards(rewards, ma_rewards, tag=\"train\",\n", - " algo=cfg.algo, path=cfg.result_path)\n", - " # eval\n", - " env,agent = env_agent_config(cfg,seed=10)\n", - " agent.load(path=cfg.model_path)\n", - " rewards,ma_rewards = eval(cfg,env,agent)\n", - " save_results(rewards,ma_rewards,tag='eval',path=cfg.result_path)\n", - " plot_rewards(rewards,ma_rewards,tag=\"eval\",env=cfg.env,algo = cfg.algo,path=cfg.result_path)" - ] - } - ] -} \ No newline at end of file diff --git a/codes/PPO/train.py b/codes/PPO/train.py deleted file mode 100644 index e642df0..0000000 --- a/codes/PPO/train.py +++ /dev/null @@ -1,121 +0,0 @@ -def train(cfg,env,agent): - print('开始训练！') - print(f'环境：{cfg.env_name}, 算法：{cfg.algo}, 设备：{cfg.device}') - rewards = [] # 记录所有回合的奖励 - ma_rewards = [] # 记录所有回合的滑动平均奖励 - steps = 0 - for i_ep in range(cfg.train_eps): - state = env.reset() - done = False - ep_reward = 0 - while not done: - action, prob, val = agent.choose_action(state) - state_, reward, done, _ = env.step(action) - steps += 1 - ep_reward += reward - agent.memory.push(state, action, prob, val, reward, done) - if steps % cfg.update_fre == 0: - agent.update() - state = state_ - rewards.append(ep_reward) - if ma_rewards: - ma_rewards.append(0.9*ma_rewards[-1]+0.1*ep_reward) - else: - ma_rewards.append(ep_reward) - if (i_ep+1)%10 == 0: - print(f"回合：{i_ep+1}/{cfg.train_eps}，奖励：{ep_reward:.2f}") - print('完成训练！') - return rewards,ma_rewards - -def eval(cfg,env,agent): - print('开始测试!') - print(f'环境：{cfg.env_name}, 算法：{cfg.algo}, 设备：{cfg.device}') - rewards = [] # 记录所有回合的奖励 - ma_rewards = [] # 记录所有回合的滑动平均奖励 - for i_ep in range(cfg.test_eps): - state = env.reset() - done = False - ep_reward = 0 - while not done: - action, prob, val = agent.choose_action(state) - state_, reward, done, _ = env.step(action) - ep_reward += reward - state = state_ - rewards.append(ep_reward) - if ma_rewards: - ma_rewards.append( - 0.9*ma_rewards[-1]+0.1*ep_reward) - else: - ma_rewards.append(ep_reward) - print('回合：{}/{}, 奖励：{}'.format(i_ep+1, cfg.test_eps, ep_reward)) - print('完成训练！') - return rewards,ma_rewards - -if __name__ == '__main__': - import sys,os - curr_path = os.path.dirname(os.path.abspath(__file__)) # 当前文件所在绝对路径 - parent_path = os.path.dirname(curr_path) # 父路径 - sys.path.append(parent_path) # 添加路径到系统路径 - - import gym - import torch - import datetime - from common.plot import plot_rewards - from common.utils import save_results,make_dir - from PPO.agent import PPO - from PPO.train import train - - curr_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") # 获取当前时间 - - class PPOConfig: - def __init__(self) -> None: - self.algo = "DQN" # 算法名称 - self.env_name = 'CartPole-v0' # 环境名称 - self.continuous = False # 环境是否为连续动作 - self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # 检测GPU - self.train_eps = 200 # 训练的回合数 - self.test_eps = 20 # 测试的回合数 - self.batch_size = 5 - self.gamma=0.99 - self.n_epochs = 4 - self.actor_lr = 0.0003 - self.critic_lr = 0.0003 - self.gae_lambda=0.95 - self.policy_clip=0.2 - self.hidden_dim = 256 - self.update_fre = 20 # frequency of agent update - - class PlotConfig: - def __init__(self) -> None: - self.algo = "DQN" # 算法名称 - self.env_name = 'CartPole-v0' # 环境名称 - self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # 检测GPU - self.result_path = curr_path+"/outputs/" + self.env_name + \ - '/'+curr_time+'/results/' # 保存结果的路径 - self.model_path = curr_path+"/outputs/" + self.env_name + \ - '/'+curr_time+'/models/' # 保存模型的路径 - self.save = True # 是否保存图片 - - def env_agent_config(cfg,seed=1): - env = gym.make(cfg.env_name) - env.seed(seed) - state_dim = env.observation_space.shape[0] - action_dim = env.action_space.n - agent = PPO(state_dim,action_dim,cfg) - return env,agent - - cfg = PPOConfig() - plot_cfg = PlotConfig() - # 训练 - env,agent = env_agent_config(cfg,seed=1) - rewards, ma_rewards = train(cfg, env, agent) - make_dir(plot_cfg.result_path, plot_cfg.model_path) # 创建保存结果和模型路径的文件夹 - agent.save(path=plot_cfg.model_path) - save_results(rewards, ma_rewards, tag='train', path=plot_cfg.result_path) - plot_rewards(rewards, ma_rewards, plot_cfg, tag="train") - # 测试 - env,agent = env_agent_config(cfg,seed=10) - agent.load(path=plot_cfg.model_path) - rewards,ma_rewards = eval(cfg,env,agent) - save_results(rewards,ma_rewards,tag='eval',path=plot_cfg.result_path) - plot_rewards(rewards,ma_rewards,plot_cfg,tag="eval") \ No newline at end of file diff --git a/codes/PolicyGradient/model.py b/codes/PolicyGradient/model.py deleted file mode 100644 index 6d9bc64..0000000 --- a/codes/PolicyGradient/model.py +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env python -# coding=utf-8 -''' -Author: John -Email: johnjim0816@gmail.com -Date: 2021-03-23 16:35:58 -LastEditor: John -LastEditTime: 2021-12-21 23:21:26 -Discription: -Environment: -''' -import torch.nn as nn -import torch.nn.functional as F -class MLP(nn.Module): - - ''' 多层感知机 - 输入：state维度 - 输出：概率 - ''' - def __init__(self,input_dim,hidden_dim = 36): - super(MLP, self).__init__() - # 24和36为hidden layer的层数，可根据input_dim, action_dim的情况来改变 - self.fc1 = nn.Linear(input_dim, hidden_dim) - self.fc2 = nn.Linear(hidden_dim,hidden_dim) - self.fc3 = nn.Linear(hidden_dim, 1) # Prob of Left - - def forward(self, x): - x = F.relu(self.fc1(x)) - x = F.relu(self.fc2(x)) - x = F.sigmoid(self.fc3(x)) - return x \ No newline at end of file diff --git a/codes/PolicyGradient/outputs/CartPole-v0/20210505-173524/models/pg_checkpoint.pt b/codes/PolicyGradient/outputs/CartPole-v0/20210505-173524/models/pg_checkpoint.pt deleted file mode 100644 index 2ea029d..0000000 Binary files a/codes/PolicyGradient/outputs/CartPole-v0/20210505-173524/models/pg_checkpoint.pt and /dev/null differ diff --git a/codes/PolicyGradient/outputs/CartPole-v0/20210505-173524/results/eval_ma_rewards.npy b/codes/PolicyGradient/outputs/CartPole-v0/20210505-173524/results/eval_ma_rewards.npy deleted file mode 100644 index a8a5243..0000000 Binary files a/codes/PolicyGradient/outputs/CartPole-v0/20210505-173524/results/eval_ma_rewards.npy and /dev/null differ diff --git a/codes/PolicyGradient/outputs/CartPole-v0/20210505-173524/results/eval_rewards.npy b/codes/PolicyGradient/outputs/CartPole-v0/20210505-173524/results/eval_rewards.npy deleted file mode 100644 index a8a5243..0000000 Binary files a/codes/PolicyGradient/outputs/CartPole-v0/20210505-173524/results/eval_rewards.npy and /dev/null differ diff --git a/codes/PolicyGradient/outputs/CartPole-v0/20210505-173524/results/eval_rewards_curve.png b/codes/PolicyGradient/outputs/CartPole-v0/20210505-173524/results/eval_rewards_curve.png deleted file mode 100644 index 2c19fd2..0000000 Binary files a/codes/PolicyGradient/outputs/CartPole-v0/20210505-173524/results/eval_rewards_curve.png and /dev/null differ diff --git a/codes/PolicyGradient/outputs/CartPole-v0/20210505-173524/results/train_ma_rewards.npy b/codes/PolicyGradient/outputs/CartPole-v0/20210505-173524/results/train_ma_rewards.npy deleted file mode 100644 index 3238411..0000000 Binary files a/codes/PolicyGradient/outputs/CartPole-v0/20210505-173524/results/train_ma_rewards.npy and /dev/null differ diff --git a/codes/PolicyGradient/outputs/CartPole-v0/20210505-173524/results/train_rewards.npy b/codes/PolicyGradient/outputs/CartPole-v0/20210505-173524/results/train_rewards.npy deleted file mode 100644 index 3450bf8..0000000 Binary files a/codes/PolicyGradient/outputs/CartPole-v0/20210505-173524/results/train_rewards.npy and /dev/null differ diff --git a/codes/PolicyGradient/outputs/CartPole-v0/20210505-173524/results/train_rewards_curve.png b/codes/PolicyGradient/outputs/CartPole-v0/20210505-173524/results/train_rewards_curve.png deleted file mode 100644 index 5fee65a..0000000 Binary files a/codes/PolicyGradient/outputs/CartPole-v0/20210505-173524/results/train_rewards_curve.png and /dev/null differ diff --git a/codes/PolicyGradient/outputs/CartPole-v0/20220210-061325/models/pg_checkpoint.pt b/codes/PolicyGradient/outputs/CartPole-v0/20220210-061325/models/pg_checkpoint.pt new file mode 100644 index 0000000..64c6702 Binary files /dev/null and b/codes/PolicyGradient/outputs/CartPole-v0/20220210-061325/models/pg_checkpoint.pt differ diff --git a/codes/PolicyGradient/outputs/CartPole-v0/20220210-061325/results/test_ma_rewards.npy b/codes/PolicyGradient/outputs/CartPole-v0/20220210-061325/results/test_ma_rewards.npy new file mode 100644 index 0000000..343fcc6 Binary files /dev/null and b/codes/PolicyGradient/outputs/CartPole-v0/20220210-061325/results/test_ma_rewards.npy differ diff --git a/codes/PolicyGradient/outputs/CartPole-v0/20220210-061325/results/test_rewards.npy b/codes/PolicyGradient/outputs/CartPole-v0/20220210-061325/results/test_rewards.npy new file mode 100644 index 0000000..343fcc6 Binary files /dev/null and b/codes/PolicyGradient/outputs/CartPole-v0/20220210-061325/results/test_rewards.npy differ diff --git a/codes/PolicyGradient/outputs/CartPole-v0/20220210-061325/results/test_rewards_curve.png b/codes/PolicyGradient/outputs/CartPole-v0/20220210-061325/results/test_rewards_curve.png new file mode 100644 index 0000000..7ff5198 Binary files /dev/null and b/codes/PolicyGradient/outputs/CartPole-v0/20220210-061325/results/test_rewards_curve.png differ diff --git a/codes/PolicyGradient/outputs/CartPole-v0/20220210-061325/results/train_ma_rewards.npy b/codes/PolicyGradient/outputs/CartPole-v0/20220210-061325/results/train_ma_rewards.npy new file mode 100644 index 0000000..8aea751 Binary files /dev/null and b/codes/PolicyGradient/outputs/CartPole-v0/20220210-061325/results/train_ma_rewards.npy differ diff --git a/codes/PolicyGradient/outputs/CartPole-v0/20220210-061325/results/train_rewards.npy b/codes/PolicyGradient/outputs/CartPole-v0/20220210-061325/results/train_rewards.npy new file mode 100644 index 0000000..2198ca9 Binary files /dev/null and b/codes/PolicyGradient/outputs/CartPole-v0/20220210-061325/results/train_rewards.npy differ diff --git a/codes/PolicyGradient/outputs/CartPole-v0/20220210-061325/results/train_rewards_curve.png b/codes/PolicyGradient/outputs/CartPole-v0/20220210-061325/results/train_rewards_curve.png new file mode 100644 index 0000000..03b4c24 Binary files /dev/null and b/codes/PolicyGradient/outputs/CartPole-v0/20220210-061325/results/train_rewards_curve.png differ diff --git a/codes/PolicyGradient/agent.py b/codes/PolicyGradient/pg.py similarity index 72% rename from codes/PolicyGradient/agent.py rename to codes/PolicyGradient/pg.py index 8f349b5..688895f 100644 --- a/codes/PolicyGradient/agent.py +++ b/codes/PolicyGradient/pg.py @@ -5,21 +5,41 @@ Author: John Email: johnjim0816@gmail.com Date: 2020-11-22 23:27:44 LastEditor: John -LastEditTime: 2021-10-16 00:43:52 +LastEditTime: 2022-02-10 01:25:27 Discription: Environment: ''' import torch +import torch.nn as nn +import torch.nn.functional as F from torch.distributions import Bernoulli from torch.autograd import Variable import numpy as np -from PolicyGradient.model import MLP +class MLP(nn.Module): + + ''' 多层感知机 + 输入：state维度 + 输出：概率 + ''' + def __init__(self,input_dim,hidden_dim = 36): + super(MLP, self).__init__() + # 24和36为hidden layer的层数，可根据input_dim, n_actions的情况来改变 + self.fc1 = nn.Linear(input_dim, hidden_dim) + self.fc2 = nn.Linear(hidden_dim,hidden_dim) + self.fc3 = nn.Linear(hidden_dim, 1) # Prob of Left + + def forward(self, x): + x = F.relu(self.fc1(x)) + x = F.relu(self.fc2(x)) + x = F.sigmoid(self.fc3(x)) + return x + class PolicyGradient: - def __init__(self, state_dim,cfg): + def __init__(self, n_states,cfg): self.gamma = cfg.gamma - self.policy_net = MLP(state_dim,hidden_dim=cfg.hidden_dim) + self.policy_net = MLP(n_states,hidden_dim=cfg.hidden_dim) self.optimizer = torch.optim.RMSprop(self.policy_net.parameters(), lr=cfg.lr) self.batch_size = cfg.batch_size diff --git a/codes/PolicyGradient/task0.py b/codes/PolicyGradient/task0.py new file mode 100644 index 0000000..c676fe3 --- /dev/null +++ b/codes/PolicyGradient/task0.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python +# coding=utf-8 +''' +Author: John +Email: johnjim0816@gmail.com +Date: 2020-11-22 23:21:53 +LastEditor: John +LastEditTime: 2022-02-10 06:13:21 +Discription: +Environment: +''' +import sys +import os +curr_path = os.path.dirname(os.path.abspath(__file__)) # 当前文件所在绝对路径 +parent_path = os.path.dirname(curr_path) # 父路径 +sys.path.append(parent_path) # 添加路径到系统路径 + +import gym +import torch +import datetime +from itertools import count + +from pg import PolicyGradient +from common.utils import save_results, make_dir +from common.utils import plot_rewards + +curr_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") # 获取当前时间 + +class Config: + '''超参数 + ''' + + def __init__(self): + ################################## 环境超参数 ################################### + self.algo_name = "PolicyGradient" # 算法名称 + self.env_name = 'CartPole-v0' # 环境名称 + self.device = torch.device( + "cuda" if torch.cuda.is_available() else "cpu") # 检测GPUgjgjlkhfsf风刀霜的撒发十 + self.seed = 10 # 随机种子，置0则不设置随机种子 + self.train_eps = 300 # 训练的回合数 + self.test_eps = 30 # 测试的回合数 + ################################################################################ + + ################################## 算法超参数 ################################### + self.batch_size = 8 # mini-batch SGD中的批量大小 + self.lr = 0.01 # 学习率 + self.gamma = 0.99 # 强化学习中的折扣因子 + self.hidden_dim = 36 # 网络隐藏层 + ################################################################################ + + ################################# 保存结果相关参数 ################################ + self.result_path = curr_path + "/outputs/" + self.env_name + \ + '/' + curr_time + '/results/' # 保存结果的路径 + self.model_path = curr_path + "/outputs/" + self.env_name + \ + '/' + curr_time + '/models/' # 保存模型的路径 + self.save = True # 是否保存图片 + ################################################################################ + + +def env_agent_config(cfg,seed=1): + env = gym.make(cfg.env_name) + env.seed(seed) + n_states = env.observation_space.shape[0] + agent = PolicyGradient(n_states,cfg) + return env,agent + +def train(cfg,env,agent): + print('开始训练!') + print(f'环境：{cfg.env_name}, 算法：{cfg.algo_name}, 设备：{cfg.device}') + state_pool = [] # 存放每batch_size个episode的state序列 + action_pool = [] + reward_pool = [] + rewards = [] + ma_rewards = [] + for i_ep in range(cfg.train_eps): + state = env.reset() + ep_reward = 0 + for _ in count(): + action = agent.choose_action(state) # 根据当前环境state选择action + next_state, reward, done, _ = env.step(action) + ep_reward += reward + if done: + reward = 0 + state_pool.append(state) + action_pool.append(float(action)) + reward_pool.append(reward) + state = next_state + if done: + print('回合：{}/{}, 奖励：{}'.format(i_ep + 1, cfg.train_eps, ep_reward)) + break + if i_ep > 0 and i_ep % cfg.batch_size == 0: + agent.update(reward_pool,state_pool,action_pool) + state_pool = [] # 每个episode的state + action_pool = [] + reward_pool = [] + rewards.append(ep_reward) + if ma_rewards: + ma_rewards.append( + 0.9*ma_rewards[-1]+0.1*ep_reward) + else: + ma_rewards.append(ep_reward) + print('完成训练！') + env.close() + return rewards, ma_rewards + + +def test(cfg,env,agent): + print('开始测试!') + print(f'环境：{cfg.env_name}, 算法：{cfg.algo_name}, 设备：{cfg.device}') + rewards = [] + ma_rewards = [] + for i_ep in range(cfg.test_eps): + state = env.reset() + ep_reward = 0 + for _ in count(): + action = agent.choose_action(state) # 根据当前环境state选择action + next_state, reward, done, _ = env.step(action) + ep_reward += reward + if done: + reward = 0 + state = next_state + if done: + print('回合：{}/{}, 奖励：{}'.format(i_ep + 1, cfg.train_eps, ep_reward)) + break + rewards.append(ep_reward) + if ma_rewards: + ma_rewards.append( + 0.9*ma_rewards[-1]+0.1*ep_reward) + else: + ma_rewards.append(ep_reward) + print('完成测试！') + env.close() + return rewards, ma_rewards + +if __name__ == "__main__": + cfg = Config() + # 训练 + env, agent = env_agent_config(cfg) + rewards, ma_rewards = train(cfg, env, agent) + make_dir(cfg.result_path, cfg.model_path) # 创建保存结果和模型路径的文件夹 + agent.save(path=cfg.model_path) # 保存模型 + save_results(rewards, ma_rewards, tag='train', + path=cfg.result_path) # 保存结果 + plot_rewards(rewards, ma_rewards, cfg, tag="train") # 画出结果 + # 测试 + env, agent = env_agent_config(cfg) + agent.load(path=cfg.model_path) # 导入模型 + rewards, ma_rewards = test(cfg, env, agent) + save_results(rewards, ma_rewards, tag='test', + path=cfg.result_path) # 保存结果 + plot_rewards(rewards, ma_rewards, cfg, tag="test") # 画出结果 + diff --git a/codes/PolicyGradient/task0_train.py b/codes/PolicyGradient/task0_train.py deleted file mode 100644 index b6866f0..0000000 --- a/codes/PolicyGradient/task0_train.py +++ /dev/null @@ -1,136 +0,0 @@ -#!/usr/bin/env python -# coding=utf-8 -''' -Author: John -Email: johnjim0816@gmail.com -Date: 2020-11-22 23:21:53 -LastEditor: John -LastEditTime: 2021-10-16 00:34:13 -Discription: -Environment: -''' -import sys,os -curr_path = os.path.dirname(os.path.abspath(__file__)) # 当前文件所在绝对路径 -parent_path = os.path.dirname(curr_path) # 父路径 -sys.path.append(parent_path) # 添加父路径到系统路径sys.path - -import gym -import torch -import datetime -from itertools import count - -from PolicyGradient.agent import PolicyGradient -from common.plot import plot_rewards -from common.utils import save_results,make_dir - -curr_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") # 获取当前时间 - -class PGConfig: - def __init__(self): - self.algo = "PolicyGradient" # 算法名称 - self.env = 'CartPole-v0' # 环境名称 - self.result_path = curr_path+"/outputs/" + self.env + \ - '/'+curr_time+'/results/' # 保存结果的路径 - self.model_path = curr_path+"/outputs/" + self.env + \ - '/'+curr_time+'/models/' # 保存模型的路径 - self.train_eps = 300 # 训练的回合数 - self.test_eps = 30 # 测试的回合数 - self.batch_size = 8 - self.lr = 0.01 # 学习率 - self.gamma = 0.99 - self.hidden_dim = 36 # dimmension of hidden layer - self.device = torch.device( - "cuda" if torch.cuda.is_available() else "cpu") # check gpu - - -def env_agent_config(cfg,seed=1): - env = gym.make(cfg.env) - env.seed(seed) - state_dim = env.observation_space.shape[0] - agent = PolicyGradient(state_dim,cfg) - return env,agent - -def train(cfg,env,agent): - print('Start to eval !') - print(f'Env:{cfg.env}, Algorithm:{cfg.algo}, Device:{cfg.device}') - state_pool = [] # 存放每batch_size个episode的state序列 - action_pool = [] - reward_pool = [] - rewards = [] - ma_rewards = [] - for i_ep in range(cfg.train_eps): - state = env.reset() - ep_reward = 0 - for _ in count(): - action = agent.choose_action(state) # 根据当前环境state选择action - next_state, reward, done, _ = env.step(action) - ep_reward += reward - if done: - reward = 0 - state_pool.append(state) - action_pool.append(float(action)) - reward_pool.append(reward) - state = next_state - if done: - print('Episode:', i_ep, ' Reward:', ep_reward) - break - if i_ep > 0 and i_ep % cfg.batch_size == 0: - agent.update(reward_pool,state_pool,action_pool) - state_pool = [] # 每个episode的state - action_pool = [] - reward_pool = [] - rewards.append(ep_reward) - if ma_rewards: - ma_rewards.append( - 0.9*ma_rewards[-1]+0.1*ep_reward) - else: - ma_rewards.append(ep_reward) - print('complete training！') - return rewards, ma_rewards - - -def eval(cfg,env,agent): - print('Start to eval !') - print(f'Env:{cfg.env}, Algorithm:{cfg.algo}, Device:{cfg.device}') - rewards = [] - ma_rewards = [] - for i_ep in range(cfg.test_eps): - state = env.reset() - ep_reward = 0 - for _ in count(): - action = agent.choose_action(state) # 根据当前环境state选择action - next_state, reward, done, _ = env.step(action) - ep_reward += reward - if done: - reward = 0 - state = next_state - if done: - print('Episode:', i_ep, ' Reward:', ep_reward) - break - rewards.append(ep_reward) - if ma_rewards: - ma_rewards.append( - 0.9*ma_rewards[-1]+0.1*ep_reward) - else: - ma_rewards.append(ep_reward) - print('complete evaling！') - return rewards, ma_rewards - -if __name__ == "__main__": - cfg = PGConfig() - - # train - env,agent = env_agent_config(cfg,seed=1) - rewards, ma_rewards = train(cfg, env, agent) - make_dir(cfg.result_path, cfg.model_path) - agent.save(path=cfg.model_path) - save_results(rewards, ma_rewards, tag='train', path=cfg.result_path) - plot_rewards(rewards, ma_rewards, tag="train", - algo=cfg.algo, path=cfg.result_path) - # eval - env,agent = env_agent_config(cfg,seed=10) - agent.load(path=cfg.model_path) - rewards,ma_rewards = eval(cfg,env,agent) - save_results(rewards,ma_rewards,tag='eval',path=cfg.result_path) - plot_rewards(rewards,ma_rewards,tag="eval",env=cfg.env,algo = cfg.algo,path=cfg.result_path) - diff --git a/codes/envs/gridworld_env.py b/codes/QLearning/env/gridworld_env.py similarity index 100% rename from codes/envs/gridworld_env.py rename to codes/QLearning/env/gridworld_env.py diff --git a/codes/QLearning/outputs/CliffWalking-v0/20211222-110223/models/Qleaning_model.pkl b/codes/QLearning/outputs/CliffWalking-v0/20211222-110223/models/Qleaning_model.pkl deleted file mode 100644 index dc89386..0000000 Binary files a/codes/QLearning/outputs/CliffWalking-v0/20211222-110223/models/Qleaning_model.pkl and /dev/null differ diff --git a/codes/QLearning/outputs/CliffWalking-v0/20211222-110223/results/test_rewards_curve.png b/codes/QLearning/outputs/CliffWalking-v0/20211222-110223/results/test_rewards_curve.png deleted file mode 100644 index d745634..0000000 Binary files a/codes/QLearning/outputs/CliffWalking-v0/20211222-110223/results/test_rewards_curve.png and /dev/null differ diff --git a/codes/QLearning/outputs/CliffWalking-v0/20211222-110223/results/train_ma_rewards.npy b/codes/QLearning/outputs/CliffWalking-v0/20211222-110223/results/train_ma_rewards.npy deleted file mode 100644 index 23e7c95..0000000 Binary files a/codes/QLearning/outputs/CliffWalking-v0/20211222-110223/results/train_ma_rewards.npy and /dev/null differ diff --git a/codes/QLearning/outputs/CliffWalking-v0/20211222-110223/results/train_rewards.npy b/codes/QLearning/outputs/CliffWalking-v0/20211222-110223/results/train_rewards.npy deleted file mode 100644 index 0ceb153..0000000 Binary files a/codes/QLearning/outputs/CliffWalking-v0/20211222-110223/results/train_rewards.npy and /dev/null differ diff --git a/codes/QLearning/outputs/CliffWalking-v0/20211222-110223/results/train_rewards_curve.png b/codes/QLearning/outputs/CliffWalking-v0/20211222-110223/results/train_rewards_curve.png deleted file mode 100644 index a15bd2a..0000000 Binary files a/codes/QLearning/outputs/CliffWalking-v0/20211222-110223/results/train_rewards_curve.png and /dev/null differ diff --git a/codes/QLearning/outputs/CliffWalking-v0/20211222-111747/models/Qleaning_model.pkl b/codes/QLearning/outputs/CliffWalking-v0/20211222-111747/models/Qleaning_model.pkl deleted file mode 100644 index c362dbd..0000000 Binary files a/codes/QLearning/outputs/CliffWalking-v0/20211222-111747/models/Qleaning_model.pkl and /dev/null differ diff --git a/codes/QLearning/outputs/CliffWalking-v0/20211222-111747/results/test_ma_rewards.npy b/codes/QLearning/outputs/CliffWalking-v0/20211222-111747/results/test_ma_rewards.npy deleted file mode 100644 index 9bee5e4..0000000 Binary files a/codes/QLearning/outputs/CliffWalking-v0/20211222-111747/results/test_ma_rewards.npy and /dev/null differ diff --git a/codes/QLearning/outputs/CliffWalking-v0/20211222-111747/results/test_rewards.npy b/codes/QLearning/outputs/CliffWalking-v0/20211222-111747/results/test_rewards.npy deleted file mode 100644 index 8aeb5dd..0000000 Binary files a/codes/QLearning/outputs/CliffWalking-v0/20211222-111747/results/test_rewards.npy and /dev/null differ diff --git a/codes/QLearning/outputs/CliffWalking-v0/20211222-111747/results/test_rewards_curve.png b/codes/QLearning/outputs/CliffWalking-v0/20211222-111747/results/test_rewards_curve.png deleted file mode 100644 index 5f3ffb5..0000000 Binary files a/codes/QLearning/outputs/CliffWalking-v0/20211222-111747/results/test_rewards_curve.png and /dev/null differ diff --git a/codes/QLearning/outputs/CliffWalking-v0/20211222-111747/results/train_ma_rewards.npy b/codes/QLearning/outputs/CliffWalking-v0/20211222-111747/results/train_ma_rewards.npy deleted file mode 100644 index 261a3d5..0000000 Binary files a/codes/QLearning/outputs/CliffWalking-v0/20211222-111747/results/train_ma_rewards.npy and /dev/null differ diff --git a/codes/QLearning/outputs/CliffWalking-v0/20211222-111747/results/train_rewards.npy b/codes/QLearning/outputs/CliffWalking-v0/20211222-111747/results/train_rewards.npy deleted file mode 100644 index b1a0f23..0000000 Binary files a/codes/QLearning/outputs/CliffWalking-v0/20211222-111747/results/train_rewards.npy and /dev/null differ diff --git a/codes/QLearning/outputs/CliffWalking-v0/20211222-111747/results/train_rewards_curve.png b/codes/QLearning/outputs/CliffWalking-v0/20211222-111747/results/train_rewards_curve.png deleted file mode 100644 index 9a9d6ad..0000000 Binary files a/codes/QLearning/outputs/CliffWalking-v0/20211222-111747/results/train_rewards_curve.png and /dev/null differ diff --git a/codes/QLearning/outputs/CliffWalking-v0/20220210-005501/models/Qleaning_model.pkl b/codes/QLearning/outputs/CliffWalking-v0/20220210-005501/models/Qleaning_model.pkl new file mode 100644 index 0000000..9053e52 Binary files /dev/null and b/codes/QLearning/outputs/CliffWalking-v0/20220210-005501/models/Qleaning_model.pkl differ diff --git a/codes/QLearning/outputs/CliffWalking-v0/20211222-110223/results/test_ma_rewards.npy b/codes/QLearning/outputs/CliffWalking-v0/20220210-005501/results/test_ma_rewards.npy similarity index 100% rename from codes/QLearning/outputs/CliffWalking-v0/20211222-110223/results/test_ma_rewards.npy rename to codes/QLearning/outputs/CliffWalking-v0/20220210-005501/results/test_ma_rewards.npy diff --git a/codes/QLearning/outputs/CliffWalking-v0/20211222-110223/results/test_rewards.npy b/codes/QLearning/outputs/CliffWalking-v0/20220210-005501/results/test_rewards.npy similarity index 100% rename from codes/QLearning/outputs/CliffWalking-v0/20211222-110223/results/test_rewards.npy rename to codes/QLearning/outputs/CliffWalking-v0/20220210-005501/results/test_rewards.npy diff --git a/codes/QLearning/outputs/CliffWalking-v0/20220210-005501/results/test_rewards_curve.png b/codes/QLearning/outputs/CliffWalking-v0/20220210-005501/results/test_rewards_curve.png new file mode 100644 index 0000000..f7cee1b Binary files /dev/null and b/codes/QLearning/outputs/CliffWalking-v0/20220210-005501/results/test_rewards_curve.png differ diff --git a/codes/QLearning/outputs/CliffWalking-v0/20220210-005501/results/train_ma_rewards.npy b/codes/QLearning/outputs/CliffWalking-v0/20220210-005501/results/train_ma_rewards.npy new file mode 100644 index 0000000..5050935 Binary files /dev/null and b/codes/QLearning/outputs/CliffWalking-v0/20220210-005501/results/train_ma_rewards.npy differ diff --git a/codes/QLearning/outputs/CliffWalking-v0/20220210-005501/results/train_rewards.npy b/codes/QLearning/outputs/CliffWalking-v0/20220210-005501/results/train_rewards.npy new file mode 100644 index 0000000..12c27d8 Binary files /dev/null and b/codes/QLearning/outputs/CliffWalking-v0/20220210-005501/results/train_rewards.npy differ diff --git a/codes/QLearning/outputs/CliffWalking-v0/20220210-005501/results/train_rewards_curve.png b/codes/QLearning/outputs/CliffWalking-v0/20220210-005501/results/train_rewards_curve.png new file mode 100644 index 0000000..b7d33a6 Binary files /dev/null and b/codes/QLearning/outputs/CliffWalking-v0/20220210-005501/results/train_rewards_curve.png differ diff --git a/codes/QLearning/agent.py b/codes/QLearning/qlearning.py similarity index 85% rename from codes/QLearning/agent.py rename to codes/QLearning/qlearning.py index b72de22..be57831 100644 --- a/codes/QLearning/agent.py +++ b/codes/QLearning/qlearning.py @@ -15,9 +15,9 @@ import torch from collections import defaultdict class QLearning(object): - def __init__(self,state_dim, - action_dim,cfg): - self.action_dim = action_dim + def __init__(self,n_states, + n_actions,cfg): + self.n_actions = n_actions self.lr = cfg.lr # 学习率 self.gamma = cfg.gamma self.epsilon = 0 @@ -25,7 +25,7 @@ class QLearning(object): self.epsilon_start = cfg.epsilon_start self.epsilon_end = cfg.epsilon_end self.epsilon_decay = cfg.epsilon_decay - self.Q_table = defaultdict(lambda: np.zeros(action_dim)) # 用嵌套字典存放状态->动作->状态-动作值（Q值）的映射，即Q表 + self.Q_table = defaultdict(lambda: np.zeros(n_actions)) # 用嵌套字典存放状态->动作->状态-动作值（Q值）的映射，即Q表 def choose_action(self, state): self.sample_count += 1 self.epsilon = self.epsilon_end + (self.epsilon_start - self.epsilon_end) * \ @@ -34,7 +34,7 @@ class QLearning(object): if np.random.uniform(0, 1) > self.epsilon: action = np.argmax(self.Q_table[str(state)]) # 选择Q(s,a)最大对应的动作 else: - action = np.random.choice(self.action_dim) # 随机选择动作 + action = np.random.choice(self.n_actions) # 随机选择动作 return action def predict(self,state): action = np.argmax(self.Q_table[str(state)]) diff --git a/codes/QLearning/task0.ipynb b/codes/QLearning/task0.ipynb deleted file mode 100644 index dc447ce..0000000 --- a/codes/QLearning/task0.ipynb +++ /dev/null @@ -1,386 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 36, - "metadata": {}, - "outputs": [], - "source": [ - "import sys\n", - "from pathlib import Path\n", - "curr_path = str(Path().absolute())\n", - "parent_path = str(Path().absolute().parent)\n", - "sys.path.append(parent_path) # 添加路径到系统路径\n", - "\n", - "import gym\n", - "import torch\n", - "import math\n", - "import datetime\n", - "import numpy as np\n", - "from collections import defaultdict\n", - "from envs.gridworld_env import CliffWalkingWapper\n", - "from QLearning.agent import QLearning\n", - "from common.utils import plot_rewards\n", - "from common.utils import save_results,make_dir\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## QLearning算法" - ] - }, - { - "cell_type": "code", - "execution_count": 37, - "metadata": {}, - "outputs": [], - "source": [ - "class QLearning(object):\n", - " def __init__(self,state_dim,\n", - " action_dim,cfg):\n", - " self.action_dim = action_dim \n", - " self.lr = cfg.lr # 学习率\n", - " self.gamma = cfg.gamma \n", - " self.epsilon = 0 \n", - " self.sample_count = 0 \n", - " self.epsilon_start = cfg.epsilon_start\n", - " self.epsilon_end = cfg.epsilon_end\n", - " self.epsilon_decay = cfg.epsilon_decay\n", - " self.Q_table = defaultdict(lambda: np.zeros(action_dim)) # 用嵌套字典存放状态->动作->状态-动作值（Q值）的映射，即Q表\n", - " def choose_action(self, state):\n", - " self.sample_count += 1\n", - " self.epsilon = self.epsilon_end + (self.epsilon_start - self.epsilon_end) * \\\n", - " math.exp(-1. * self.sample_count / self.epsilon_decay) # epsilon是会递减的，这里选择指数递减\n", - " # e-greedy 策略\n", - " if np.random.uniform(0, 1) > self.epsilon:\n", - " action = np.argmax(self.Q_table[str(state)]) # 选择Q(s,a)最大对应的动作\n", - " else:\n", - " action = np.random.choice(self.action_dim) # 随机选择动作\n", - " return action\n", - " def predict(self,state):\n", - " action = np.argmax(self.Q_table[str(state)])\n", - " return action\n", - " def update(self, state, action, reward, next_state, done):\n", - " Q_predict = self.Q_table[str(state)][action] \n", - " if done: # 终止状态\n", - " Q_target = reward \n", - " else:\n", - " Q_target = reward + self.gamma * np.max(self.Q_table[str(next_state)]) \n", - " self.Q_table[str(state)][action] += self.lr * (Q_target - Q_predict)\n", - " def save(self,path):\n", - " import dill\n", - " torch.save(\n", - " obj=self.Q_table,\n", - " f=path+\"Qleaning_model.pkl\",\n", - " pickle_module=dill\n", - " )\n", - " print(\"保存模型成功！\")\n", - " def load(self, path):\n", - " import dill\n", - " self.Q_table =torch.load(f=path+'Qleaning_model.pkl',pickle_module=dill)\n", - " print(\"加载模型成功！\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 训练" - ] - }, - { - "cell_type": "code", - "execution_count": 38, - "metadata": {}, - "outputs": [], - "source": [ - "def train(cfg,env,agent):\n", - " print('开始训练!')\n", - " print(f'环境：{cfg.env_name}, 算法：{cfg.algo_name}, 设备：{cfg.device}')\n", - " rewards = [] # 记录奖励\n", - " ma_rewards = [] # 记录滑动平均奖励\n", - " for i_ep in range(cfg.train_eps):\n", - " ep_reward = 0 # 记录每个episode的reward\n", - " state = env.reset() # 重置环境, 重新开一局（即开始新的一个episode）\n", - " while True:\n", - " action = agent.choose_action(state) # 根据算法选择一个动作\n", - " next_state, reward, done, _ = env.step(action) # 与环境进行一次动作交互\n", - " agent.update(state, action, reward, next_state, done) # Q-learning算法更新\n", - " state = next_state # 存储上一个观察值\n", - " ep_reward += reward\n", - " if done:\n", - " break\n", - " rewards.append(ep_reward)\n", - " if ma_rewards:\n", - " ma_rewards.append(ma_rewards[-1]*0.9+ep_reward*0.1)\n", - " else:\n", - " ma_rewards.append(ep_reward)\n", - " if (i_ep+1)%20 == 0: \n", - " print('回合：{}/{}, 奖励：{}'.format(i_ep+1, cfg.train_eps, ep_reward))\n", - " print('完成训练！')\n", - " return rewards,ma_rewards" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 测试" - ] - }, - { - "cell_type": "code", - "execution_count": 39, - "metadata": {}, - "outputs": [], - "source": [ - "def test(cfg,env,agent):\n", - " # env = gym.make(\"FrozenLake-v0\", is_slippery=False) # 0 left, 1 down, 2 right, 3 up\n", - " # env = FrozenLakeWapper(env)\n", - " print('开始测试!')\n", - " print(f'环境：{cfg.env_name}, 算法：{cfg.algo_name}, 设备：{cfg.device}')\n", - " # 由于测试不需要使用epsilon-greedy策略，所以相应的值设置为0\n", - " cfg.epsilon_start = 0.0 # e-greedy策略中初始epsilon\n", - " cfg.epsilon_end = 0.0 # e-greedy策略中的终止epsilon\n", - " rewards = [] # 记录所有回合的奖励\n", - " ma_rewards = [] # 记录所有回合的滑动平均奖励\n", - " rewards = [] # 记录所有episode的reward\n", - " ma_rewards = [] # 滑动平均的reward\n", - " for i_ep in range(cfg.test_eps):\n", - " ep_reward = 0 # 记录每个episode的reward\n", - " state = env.reset() # 重置环境, 重新开一局（即开始新的一个episode）\n", - " while True:\n", - " action = agent.predict(state) # 根据算法选择一个动作\n", - " next_state, reward, done, _ = env.step(action) # 与环境进行一个交互\n", - " state = next_state # 存储上一个观察值\n", - " ep_reward += reward\n", - " if done:\n", - " break\n", - " rewards.append(ep_reward)\n", - " if ma_rewards:\n", - " ma_rewards.append(ma_rewards[-1]*0.9+ep_reward*0.1)\n", - " else:\n", - " ma_rewards.append(ep_reward)\n", - " print(f\"回合：{i_ep+1}/{cfg.test_eps}，奖励：{ep_reward:.1f}\")\n", - " print('完成测试！')\n", - " return rewards,ma_rewards" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 设置参数" - ] - }, - { - "cell_type": "code", - "execution_count": 40, - "metadata": {}, - "outputs": [], - "source": [ - "curr_time = datetime.datetime.now().strftime(\"%Y%m%d-%H%M%S\") # 获取当前时间\n", - "algo_name = 'Q-learning' # 算法名称\n", - "env_name = 'CliffWalking-v0' # 环境名称\n", - "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\") # 检测GPU\n", - "class QlearningConfig:\n", - " '''训练相关参数'''\n", - " def __init__(self):\n", - " self.algo_name = algo_name # 算法名称\n", - " self.env_name = env_name # 环境名称\n", - " self.device = device # 检测GPU\n", - " self.train_eps = 400 # 训练的回合数\n", - " self.test_eps = 20 # 测试的回合数\n", - " self.gamma = 0.9 # reward的衰减率\n", - " self.epsilon_start = 0.95 # e-greedy策略中初始epsilon\n", - " self.epsilon_end = 0.01 # e-greedy策略中的终止epsilon\n", - " self.epsilon_decay = 300 # e-greedy策略中epsilon的衰减率\n", - " self.lr = 0.1 # 学习率 \n", - "class PlotConfig:\n", - " ''' 绘图相关参数设置\n", - " '''\n", - "\n", - " def __init__(self) -> None:\n", - " self.algo_name = algo_name # 算法名称\n", - " self.env_name = env_name # 环境名称\n", - " self.device = device # 检测GPU\n", - " self.result_path = curr_path + \"/outputs/\" + self.env_name + \\\n", - " '/' + curr_time + '/results/' # 保存结果的路径\n", - " self.model_path = curr_path + \"/outputs/\" + self.env_name + \\\n", - " '/' + curr_time + '/models/' # 保存模型的路径\n", - " self.save = True # 是否保存图片" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 创建环境和智能体" - ] - }, - { - "cell_type": "code", - "execution_count": 41, - "metadata": {}, - "outputs": [], - "source": [ - "def env_agent_config(cfg,seed=1):\n", - " '''创建环境和智能体\n", - " Args:\n", - " cfg ([type]): [description]\n", - " seed (int, optional): 随机种子. Defaults to 1.\n", - " Returns:\n", - " env [type]: 环境\n", - " agent : 智能体\n", - " ''' \n", - " env = gym.make(cfg.env_name) \n", - " env = CliffWalkingWapper(env)\n", - " env.seed(seed) # 设置随机种子\n", - " state_dim = env.observation_space.n # 状态维度\n", - " action_dim = env.action_space.n # 动作维度\n", - " agent = QLearning(state_dim,action_dim,cfg)\n", - " return env,agent" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 执行训练并输出结果" - ] - }, - { - "cell_type": "code", - "execution_count": 42, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "开始训练!\n", - "环境：CliffWalking-v0, 算法：Q-learning, 设备：cuda\n", - "回合：20/400, 奖励：-82\n", - "回合：40/400, 奖励：-51\n", - "回合：60/400, 奖励：-50\n", - "回合：80/400, 奖励：-53\n", - "回合：100/400, 奖励：-21\n", - "回合：120/400, 奖励：-35\n", - "回合：140/400, 奖励：-44\n", - "回合：160/400, 奖励：-28\n", - "回合：180/400, 奖励：-28\n", - "回合：200/400, 奖励：-17\n", - "回合：220/400, 奖励：-18\n", - "回合：240/400, 奖励：-22\n", - "回合：260/400, 奖励：-19\n", - "回合：280/400, 奖励：-15\n", - "回合：300/400, 奖励：-14\n", - "回合：320/400, 奖励：-13\n", - "回合：340/400, 奖励：-13\n", - "回合：360/400, 奖励：-13\n", - "回合：380/400, 奖励：-13\n", - "回合：400/400, 奖励：-13\n", - "完成训练！\n", - "保存模型成功！\n", - "结果保存完毕！\n" - ] - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYwAAAEcCAYAAADUX4MJAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAABEx0lEQVR4nO3dd3gU1frA8e/MlvROOh2kiUIgFAFRmoB0G1wUG4j+VBAVFRuoXK+ABYWLIopwVa6oV4qASBEQFEQiVRCkE0nvPZvdnd8fIUuybJINaSu8n+fhYbNnyjuzs/vOOWdmjqJpmoYQQghRCbW+AxBCCPH3IAlDCCGEUyRhCCGEcIokDCGEEE6RhCGEEMIpkjCEEEI45YpOGH379mXnzp11vt6YmBgGDhxY5+sVF/3111+0bt0as9lc48vetGkTN910E1FRURw5cqTGlnslHq8pKSncfffdREVFMWvWrFpZh73WrVtz9uxZAKZPn86CBQtsZf/973/p0aMHUVFRpKen89tvv3HLLbcQFRXF5s2baywG++Nv3LhxfP311w6nnTBhAitXrqyxddcmfX0HcCWKjo5mw4YN9R2GqCWzZ8/m5Zdfpn///g7LNU1j8eLFfPXVVyQkJBAYGMjw4cN5/PHHMRqNdRxt5WrzeP3yyy8JCAhg7969KIpSI8tMSkri3XffZfv27eTm5hIaGsqtt97KhAkT8PT0LDPta6+9ZntdVFTErFmz+Oqrr2jTpg0A8+bN4+677+a+++5j7dq1DB48mPXr19vmeeCBB0hISLjkvRtuuIGJEyfWyPZ8/PHHNbKcqvrrr794/vnnOXjwIOHh4UyfPp0ePXpUOM8VXcOoLRaLpb5DqLYrYRvqS1xcHNdcc0255f/85z/56quvmD17Nnv37uWjjz5i586dPPXUU3UY5UX1+VnHxcXRokWLy0oWjmqHGRkZjBkzhsLCQpYvX86+fftYsmQJWVlZnDt3rsLlpaamUlhYSMuWLcvEV/JZdunShVOnTpGWlmZb/9GjRyksLCzz3v79+4mOjq7y9riap59+mnbt2rF7926efPJJJk+ebNvO8lw1CcNqtbJo0SL69+9Pt27deOKJJ8jIyLCVT548mZ49e9K5c2fuvvtujh8/biubNm0aM2bM4KGHHqJjx47s3r2bvn37snjxYoYNG0bnzp2ZMmUKhYWFAOzevZvevXvb5q9oWoCPPvqIXr160atXL77++usyVWp7GRkZPP/88/Tq1YsuXbrw6KOPArBixQr+8Y9/lJm29HLst2Hx4sX07NmzzI/Jpk2bGDZsmFP7y95XX33FgAED6Nq1K4888giJiYll4vjiiy+45ZZbiI6O5tVXX6W8BwxYLBYWLlxI//79iYqK4rbbbiM+Pt5hE1Ppar7FYmH27Nl069aNfv368eOPP5ZZ7jfffMPgwYOJioqiX79+LF++vNxtsVqtvP/++/Tp04cbbriBZ599luzsbEwmE1FRUVgsFkaMGOGwhnHmzBn++9//8tZbbxEVFYVer+eaa65h/vz5bNu2jV9//bXc9drH8Hc/XqdNm8aqVatYvHgxUVFR7Ny5E5PJxOuvv26b//XXX8dkMpWJY9GiRfTs2ZPnn3/+kmUuWbIELy8v3nzzTRo2bAhAeHg4L730kq3WYB/D3LlzOX36NIMGDQKKE8O9995L//79iY2N5ZFHHiEqKoqAgAAaNWrEnj17ADhy5AgtW7akS5cuZd6zWq1cd911bNu2jZEjR9KpUyduuukm5s+f79Rnm5SUxLBhw2w1i9LHccn3ePbs2XTp0oW+ffuWOZZjY2NtTXz3338/r776KlOnTnW4ngkTJvD555+XeW/48OFs3LiR06dPc/jwYSZNmoS7uzsDBw6kVatWldY0r5qE8dlnn7F582Y+//xzduzYgZ+fX5nqau/evdmwYQO7du2iXbt2l3wIa9eu5ZFHHmHv3r107twZgPXr1/Pxxx/zww8/cOzYMVasWFHu+subdvv27SxdupQlS5awadMmdu/eXeF2PPvss+Tn57Nu3Tp27tzJ/fff7/Q+KL0N9913Hx4eHvzyyy+28jVr1tgSRmX7q7Rdu3bx9ttv8+677/LTTz8RGRl5ydn0tm3b+N///se3337L+vXr2bFjh8NlLVmyhHXr1rFo0SL27t3Lv/71L9zd3Svdtq+++oqtW7eyatUqvvnmG77//vsy5UFBQXz44Yfs3buXN954gzfeeIPDhw87XNaKFStYuXIln376KZs3byYvL4/XXnsNo9HIvn37AFi9erXDNu9du3YRFhbG9ddfX+b98PBwOnbsyM8//1zptsCVcbzOmjWLYcOGMX78ePbt20ePHj344IMPOHDgAKtXr+bbb7/l0KFDvP/++7Z5UlJSyMzMZOvWrcycOfOSZe7atYsBAwagqlX76WrWrBlr164FYM+ePbbPNiIigoULF7Jv3z6MRmOZ5LBnzx6io6Pp3Llzmfc6dOiAwWDAw8OD2bNnExMTw4cffsgXX3xRaT9IbGws48aN45577mHChAkOpzl48CDNmjXjl19+YcKECbz44ou2E6ypU6dy/fXXs3v3bh5//HFWr15d7rqGDh1q22aAEydOEBcXx80338yJEydo1KgR3t7etvI2bdpw4sSJCuO/ahLG8uXLefLJJwkLC8NoNPL444+zYcMG2xnrHXfcgbe3N0ajkUmTJnH06FGys7Nt8/fr14/OnTujqipubm5A8ZlBaGgo/v7+9OnThz/++KPc9Zc37fr167ntttu45ppr8PDwYNKkSeUuIykpie3bt/Pqq6/i5+eHwWCga9euTu8D+20YMmSI7YDKyclh+/btDBkyxKn9VdqaNWu4/fbbufbaazEajTz11FPs37+fv/76yzbNQw89hK+vLxEREXTr1o2jR486jPHrr7/miSeeoHnz5iiKQps2bQgICKh029avX899991HeHg4/v7+PPzww2XKb775Zho3boyiKHTt2pWePXsSExPjcFlr1qzh/vvvp1GjRnh5efHUU0/x3XffOdWBnp6eTnBwsMOy4ODgSqv8Ja6E49WRNWvW8NhjjxEUFERgYCCPPfYY3377ra1cVVUmT56M0Wh0eKKQkZFR7v6tCV26dLEdFzExMbaEUfq9ku9ct27daN26Naqq0qZNG4YMGVJhDfLEiRPcd999TJo0idGjR5c7XUREBHfddRc6nY5Ro0aRnJxMSkoKcXFxHDp0yLZ/oqOj6du3b7nL6d+/P0ePHuX8+fNA8b4fMGAARqOR3NxcfHx8ykzv4+NDbm5uhfvnqkkYcXFxPPbYY0RHRxMdHc2tt96KqqqkpqZisVh466236N+/P506dbJ9COnp6bb5w8PDL1lm6QPXw8ODvLy8ctdf3rRJSUmEhYVVuJ4SCQkJ+Pn54efn58QWX8p+2cOGDWPTpk2YTCY2bdpEu3btiIyMBCreX/aSkpJs8wF4eXnh7+9fplnKfvvLOzATEhJo3LhxlbctKSmpzPZFRESUKf/xxx+566676Nq1K9HR0Wzfvr3M51vR9kRGRmI2mx1uu72AgACSk5MdliUnJ9uS34QJE4iKiiIqKqrMD2aJK+F4dSQpKanMZxMREUFSUpLt74CAAFuCc8Tf37/c/VsTunTpwrFjx8jMzOTAgQN07NiRFi1akJycTGZmJnv37rX1Xxw4cIBx48bRvXt3OnfuzPLly8s9pqD4BzskJKTSK9IaNGhge+3h4QFAXl4eSUlJ+Pn52d6Dsvt/+vTptmNq4cKFeHt7c9NNN7Fu3TqguNY5fPhwoPg7mpOTU2a9OTk5eHl5VRjbVZMwwsLC+Oijj4iJibH9O3ToEKGhoaxZs4YffviBJUuW8Ntvv7FlyxaActvZa1JISEiZH9b4+PgKtyEzM5OsrKxLyjw8PCgoKLD97cyXqmXLlkRERLB9+3bWrl3L0KFDy6yrvP3laBtKzmKg+ODOyMhwOG1lwsLCHHZellz9Ut42BgcHl9l3pV+bTCYmT57Mgw8+yM8//0xMTAy9e/cu9/O13564uDj0ej1BQUGVxt+9e3fi4+M5ePBgmffj4+PZv3+/7ez0448/Zt++fezbt8/2JS7tSjhey5s/Li6uzPwhISG2vyvrHL/hhhvYtGkTVqu1Sut1VqNGjQgJCeHLL78kPDzc9gPasWNHvvzyS3Jzc+nYsSNQ3Glc0l/222+/MWbMmAo/g8cff5yAgACefvrpy7oQITg4mMzMTPLz823vld7/r732mu2YeuSRR4DiZql169axb98+CgsL6datG1D83Y+NjS2TNI4ePVrmggBHrpqE8Y9//IN3333X9kOQlpZma2/Mzc3FaDQSEBBAfn4+77zzTp3FNWjQIFasWMHJkyfJz88v055rLyQkhN69e/Pqq6+SmZlJUVGRrW21TZs2HD9+nD/++IPCwkKnO+CGDh3Kf/7zH/bs2WPrFISK95ejZaxYsYI//vgDk8nEO++8w/XXX2/rlKyKO++8k/fee48zZ86gaRpHjx4lPT2dwMBAQkNDWb16NRaLhf/973/Exsba5hs8eDCfffYZCQkJZGZmsmjRIluZyWTCZDIRGBiIXq/nxx9/rLAvoWSfxMbGkpuby9y5cxk8eDB6feVXoTdr1owxY8YwdepU9u/fj8Vi4fjx40yaNImoqKhKL1sscSUcr44MGTKEDz74gLS0NNLS0liwYIGt38wZDzzwALm5uTz33HO2fZOYmMgbb7xRbjNnVUVHR7N06dIyV0J17tyZpUuX0r59e1tTWW5uLn5+fri5uXHw4MEy/QWOGAwG3nvvPfLz83n22WernPQiIyNp37498+fPx2QysW/fPrZu3VrhPDfddBNxcXHMmzfPVkuF4uO0bdu2LFiwgMLCQjZt2sSxY8cqrf1cNQnj3nvvpW/fvjz44INERUVx11132c4CR44cSUREBDfeeCNDhgyxnUHUhZtuuolx48Zx7733MmDAADp06ABQ7vX6c+bMQa/XM3jwYHr06MF//vMfoPgAeOyxx7j//vu55ZZbbB2dlRk6dCh79uyhe/fuBAYG2t6vaH/Z69GjB0888QSTJk2iV69exMbGMnfu3KrsBpsHHniAwYMH8+CDD9KpUydefPFF2xU6M2fOZPHixXTr1o0TJ04QFRVlm++uu+6iV69ejBgxglGjRnHLLbfYyry9vXnppZeYMmUKXbp0Ye3atRW2/d5+++0MHz6ce+65h379+mE0Gnn55Zed3obp06dzxx138Mwzz9ChQweGDh1KREQE77//vtOdtVfK8Wrv0UcfpX379gwfPpzhw4dz7bXX2q70c4a/vz9ffPEFer2eu+66i6ioKO677z58fHxo0qTJZW2TvS5dupCamlrmOxQdHU1qaipdunSxvTdjxgzmzZtHVFQUCxYsYPDgwZUu22g08u9//5vU1FReeOGFKieNt956i/3799OtWzfeffddbr311gr3vdFoZMCAAezcubNMCwLAO++8w++//06XLl146623mDdvXpnfAEcUGUDJtZw8eZKhQ4dy6NAhp85oheubN28emzZtYtmyZfj6+tZ3ODVKjtf6NWXKFJo3b87kyZPrZH1XTQ3DlZV0PGdmZvLmm2/Sp08f+fJdQSZPnszo0aPZv39/fYdSI+R4rT8HDx7k3LlzWK1Wtm/fzg8//FDuEwdqg9QwXMD48ePZv38/Op2OLl26MGPGjDIdgUK4Ejle68+WLVt49dVXycjIICwsjIkTJ3L77bfX2folYQghhHCKNEkJIYRwiiQMIYQQTpGEIYQQwilX9KUN6em5WK1V76IJCvImNTWn8gnrmMRVda4am8RVNRJX1VxuXKqqEBBQ/uNBruiEYbVql5UwSuZ1RRJX1blqbBJX1UhcVVMbcUmTlBBCCKdIwhBCCOEUl04Yp0+fZvTo0QwcOJDRo0dz5syZ+g5JCCGuWi6dMGbMmMHYsWPZsGEDY8eOZfr06fUdkhBCXLVcNmGkpqZy5MgR2xMWhw4dypEjR5wesUwIIUTNctmrpOLj4wkNDUWn0wGg0+kICQkhPj6+0kfwCvF3oWlapYMG1cc6KppH0zQ0QHVQbv+koZJllJ6novkBrJpGSUnJ0kpf8WMttQ7lwjqsFTzhSC2nvCSWyuZXSsVxSaxWrcJ5y1seFSyzoumrOm9Nc9mEUROCgrwrn6gcwcE+lU9UD2ojrj/PpXPkdCojerewfcFPx2USGuiJp7sBAIvFSlJ6PuENvGzl5xKyCfR1x8/f02Fc5f3oHI9NR0GhZSN/2zQFhWay84rY/OtZht3YHG/PS5/xf+xsGmfisxjYvantvZN/ZfDnuXT6dmmMm0HH6bhMNu4+y40dI8nMKUTT68gxWWke6YemaZz4K4MTsRl0ahNKZk4hft5uhAZ6cjouE00Ds8XKsg1HaRHpx7jBbTl9PpO5/43h+pZBtG8eyJGTydx3a1vOnM8gKT2PrJwCtv96nNv6tMLfx4PUzDyycwpISskiKysbPWbM6GnWOJhrmgSz7dfTeBmt+LjByTPJpKZm0qqhD+1bNMDfz5OdhxI4k5DD8N7X4O2pJyunkJ/3/4WbHho18CCv0EJ8SjbtmgdzIi6X1JR0GgbowGImP78QBQ2jXsFdr2EtKkJVNHLNejzd9VzT0JfjsRnk5ptw06tYLBbcDCqR3QfwzoYUIoI88FaLwJSLu2JmdP8W/LwvltT0bPyMVgxaEe1bhbL9t1hM+Tk0CtCjsxbh56UnPTOPXI8ItmVEkptfRKSfSlFmCuFeZryNGnk5uRithYT6qGhF+VBUSJCXimYuRK9YMeoUDHoFk8mMyVSEh1FF06yYiywoaKgXtgvNima1lnoPjHqVgkIzoKHDioqGqljRoWHCwK8BQ/jlbBH+ah7+ai4+SgEeqgl/oxWdJR93nRXNYkaHFT1WdIoFHRo6xYIeKwZVQ9EsxeWKFR1WdFhQFSgetPdiggNQ0Mq8dlR28b2L06rl5PRszZPtHv34KzkXf10BvvpCDJYC3BQzbkoRbooZd6UII2b0igWzYsD64EuEN4xwvMBqcNmHD6ampjJw4EB2796NTqfDYrHQrVs3Nm7c6HQNIzU157KuRQ4O9iE5ObvK8zkjLiUXP28jXhd+iDVNIye/CB8HP5DlxVXyI1tktmLQX9qqeDYhm58PxZOQnkeX1iEY9Crdrw0jt6AIs9mKn7cbSRn5xCXn0r55IDM++ZX41DwmDm9H93ZhpGTm8+wHu2jT2J9nx3YiJ7+IrfvOs3L7KV4c1xk/LyPPLtxlW98dfa+hcQNP3N30NArxJjkjn6+2HOdcbBIN/XWE+Orp1jqI5uFeHDqexPpdp9BhpV9UOHuPJtAkxIvEbDOFman4KvmoaPh5G/BxU/Fy05GZZ0KzWsnOyUd34QusVzT8PFVycgsufIGteBgVMoqMuGmFGBQLesWCm2JBhwWjrvjLarVYURXN9oOjoqHXKVgtljLv2/6v3ZP/WmPRFMyKHk0Dd6UIq1b6Z01BUxSsmoKKFTMq2VYPAnW56Kj60KcWrfhnUa9YOW8OIEDNxVM1lTu9FQWraqBQ02NGj6bqMZk1LBoYDHp0qo6sfDMoCn7e7qCoKDodGTkm8k1WAv08UFUdmqKQnlNEYZGVAB83DAY9mnLhU1VUCs3gm3mcIJ3jG9jMih6z6o5Z0YOqB0WHVdGhlfrfgorJqqA3GMq8b1VUQEFv0GMuspRNA8qFfWxbk1Lm/0KzFVVV0OtUu3L7moNCVp6JJjkH8FfLjr9uVXSYVSMW1YhFdbvwvwGrosfq5kPHux8nu6DqPQ6qqlR4ou2yCQNg3Lhx3HHHHYwYMYLVq1fzv//9j88++8zp+WsrYeQXmjkVn8W1TZ1LXL8dS6JpmC8BPm5MmLOVxqHevPJA8djOOw7EsWT9UV55oAuNQrzZfiCO7QfiGNW7OU3DfNl1OIEz8Vn8ciSRtyb3Jjsrnze/2EdksDd/xmYwuFtjRt7YjOy8Ilb9dBp3o47Dp9NISs/HUmrbn7jjehatOQxFhQzs0pADh0+jL8igdYsIfj+VSiM1Cc+AQBpG9WLx+j8BULES6mWlIL/4bCZMl4G/wYxRK8RPyaWZMRV/LZM8zQ0APRYMqhVVs2BQLBiU6o27bNEUrCgogAUVi6ZgQYemqBRZVSwouLm5odPryTFpmMwanloeeZobit6Il7cnqblW3N3dyC8qbgYJ8vPEy8NISlYhbm4GTsfnYNEg0M+TgiIriqLStlkQ51PySM4sJKfATJe2YRw5m4HZCoVmDUVVaRruR2J6Pka9SlCDQPYdT+aaCB/CGniDolKkqRjdPckxwdm4NM7+lYqPO9zSvTmZBRASGkhWnoXwEH9OJ+aRmJ5HUZGZTi0DWbXjFGfOp9O3c0OSMgoZ0KUJnp5uZBVYOHk+E4tVIzE1l8hAd6LaNUR18yQpy0SeScOiKTQK8cHNrfiE5PCZNP48l0GnVsGkZRVQZLHSte3Fsdb/tWgzvU3bQdHRpUs7dL5BHDqbS2CgH6cS82jdJIiQBr4oRk+Ssi38cTqZjtcE4x/oj2L0wIye3UeSOJ+UhdeRlYSoWTRq0YzA8EhU70DydD4kZ5tp1jgYxeiJYvQAnbHSJrI/YzNwM+hoElZccw0O9iExMYucgiJ8nTi5AkjLKuBfCzcQbTxNcFgwN/Voj+IdiOLhVxyLrvoNLLV5cglw+HQai77aRWN9Kjd0akH36FYo7r4oBrdaietvnTBOnjzJtGnTyMrKwtfXl9mzZ9O8eXOn56+thLFg5SF+O5bM24/1JMCn7Af3y+EEdv6ewFOjO7LjYBy/HE7kj7PpALz+UDde/Gg3AJ9M68sfZ9NZueMUJ/7KBKBhsDd/JV88G2oc6s25xIt/D+jamD9Op9mm0WNBxUrjAAUtO5VANQcfNZ9CzcCNIZkYss6TYfXARy1ARcNdNeOjXBxA3pEktyYczzTS3j8Xr4JE9DgerF7TGdGHtuBUjjupyWnFP1A6AxZNRVP1+Hh70qRpJOcyLDSNDOJsUj4nEnKIDPHjupYhfLrpJAnpBUwYfh2/n0rlz7PJPHRnT1Rvf+LTCmgc5ouiKMSl5FJYZCEs0JPT8Vm0auSPqirEHE1i3/EUxg9paztTO3Aihff+d5CnRnegXZNAVFWp9LO0WK0oilJue7qzKusnKDJb0OtU2zQVxVVktlBYZMXbw1CtmJwxe9lejsVmEBLgwayHb7jsH5rtB+JYur54TO2Z47sSGXz5zcGOXE5cpiILj7z9IwD9Ozdk7IBWNRrT5cZVFbFJOcz45FcAJg5rR/drw2o1rsoShkv3YbRo0YKvv/66vsOg0GRhxfZTjOrdDHejnrMJxR9EZm7hJQlj0ZojAJxLzGbJd2UHpd99JNH2euX2U6zZeaZMeelkUbyMkr81gtQcfv/tAIFqDtFeyTTVJdBYn3axCcFu5M+iIl/O6gIxajmcNwdgRSU8xB/3Ro3Y8XsSbVo1xuIewKafj6IqVrr17sEfO7YwUovBx92Ad2AL1KDr2HmmiLQcC94+XnTu1pETKVYahATSsnEwiqIQkJLLqu+O8tCwtoQGeF6y7/wv/H/dtXBdqfdvv6MFCel5RDYLIrI1DLBqqBcacZuEX9ynEQ0uPtemXakaXde2oWXOlAE6tGzA/Ck32pr7nKFzcoztylR2xmzQ65xelkGvq9L01eHnXXy27uxZe3lKz+/jVb1l1RSjQYebQUdhkQVvz9pPvrXBr9S+9HWB/erSCcNV/LD3LzbFxOLlrmd4r2a2s9H07EKaXkj4FquVzzYcs82z4ddzlyyndIKwTxYARoOKqag4AQSq2bTQJ9FUn0wH33R8TMm26TRFR6ohDK15f77dm0KbaxrSoUMrVO8GPPHhXjwUE08+2J9dO04TcyyZZ/4RRXJGPs3ahuJm1DGiR/FyCk0WVh03MqxnU0IDPVm4uS27ClvSODKIF4ZGA9Cn+4XW2ZIz42ZlY45s4MV7T99c5bOZBv4eNPD3sP2tltfjV0VVSRYC/LyKk7NPNX9QS+ZXFPB2oc/Ax9NAYabFqT5CV+TtYUBRQNPKJo/6IgnDCWZz8Y/41n3ncXfT2/qo0rIK0TSN7PwijpxJY/uBeNs8uw4nXrIcTYMGfu6kZBYAEOTrTmpW8ev+nRsS4WXm1O5t9An4iwamOAAsOjeMwS2xRvQluciTBsGB+DRqja+++OC5vbsZd+PFj7FH52vYFBNLaKAXjUJ92Hc8hSahPrRtEnBJPG5GHS+M63whNg2jXsVkNuDrdfEMv7Yv+RT1y/9CDcPNUL0aTUmtwsfDUGPJvyZ4exhIySzApw6a92qDqir4ehrJzDVJDeNv48Lxn5lrYvkPxwn2dwdg2aY/+f1UKgdOpjqcrXRyaN3In2OxGYzu2xKzRWPtrjP0jYrks41/0qOpgdvcd1J0eDudvayovk3QNx6BvkVXVP9wFKW42aSZg3bJ0skCYHS/ltzZpwWqqnBLdCM6tAjC073yj1lRFIL83IlPzav22ab4+yg5867qvQT2fC8cM67SHFWipCmqLvqDaouvl5HsvCK8XGAbJGE4w+67lJxRYHtdXrIAaBTibUsY44e25dDJVDq1Km7779YulFPHjjPcI4a+WUcoytZhaNcXfYtu6EJbXvaZvaooqLried2MOhqHOn/fxsWE4VpfelF7dBeOleo+CdvNoMOoV6vdF1LTfDwu1Hz+xidBfl5GsvNM1b4woyZIwqhEzNEkdh25tHmpIte3CAKKm5n2HU8BoIGfB306NQSKm3+Kft9I8C9f0c/DAs264tX9TlSf4JoNvoo83YoPh7/zl0tUTeSFiwraOWiyrApFUQjwdb/kIpD6VnIsO7oR9O+iS5sQ2+XF9U0SRiXeX/V7lefx93bj/sFtANDrFErfmGNJjaVg+ydYk0+jbxKFscttqAENXaKvoKQdW+9CbdCidjUO9eGtR3vUyA/947ddZzvpcBURDbzw9TTg7eFacVXFjR1q/o7ty/X33Yv1pFm4L0F+7gzu1pijZ9P5etvJS6ZxN17sQHxv8o2219bMBPK+fR1F74b7TePRt+rlEomiRKBvcd+MK3VaitpX8rlXV2SD8of2rC+9rg/nhmtDa+zy6audJAwHzBYrz36wkztubnFJ2eBujYluEwJAgI8bX287ybAeTctcJls6YXhcOOOypP1F/vdzQdXhOWo6qndQ7W7EZbi1e2OMepWe14XXdyhC1AhVUVDr6J6Wq4GkXQcKCs2kZBbw4erDl5SVvrTN39uNT6b1pdf1ZX9g7a9csqScJW/1P8FqwfPWqS6ZLKD4hrHB3ZuUesaNEEJcJDWMCji6cMRRh7D9e+5uF89oNHMh+ZsXoBg98BzxkssmCyGEqIycSjpQ0RWGji45tb/pqaRJStM0CmNWomUl4d5noiQLIcTfmtQwHHB0D5Ovp4HWjQMc3gRn33Gtv9DBVnRoA0UHv8fQujf6iLa1EqsQQtQVSRgO2D/Ad3C3xozo1QxjBY9PePy26/jv5j9JyyrEqmloBTkU7l2NrtH1uPW+v5YjFkKI2idNUk5oGOJdYbIA6NQqmFaN/IHiGofp4HowFeDW7U7boz2EEOLvTGoYDtg3Sbk7+WC20X2vwd2o5/pQDdPPm9C37I4usFEtRCiEEHVPTn0d0Oy6vY1G5xKGn5eRewe2xvLL56CquHW9ozbCE0KIeiEJw5HLrGEAmM8fwRJ7CLfOI+SqKCHEFUUShgP2F0lVZawA08HvUTz9MbTrV7NBCSFEPZOE4YD9VVJuTjZJWXPTsfz1O4ZWvVD0f9+nYwohhCOSMJzgTA1DKyokb+0sUFQMrXvVQVRCCFG3JGE4YH+VlDM1DPOZ39AyE/EY8BiqX1gtRSaEEPVHEoYD9gnDqK94N2maRtGxHSjeQegad6jFyIQQov7IfRgOlFxW27FlA4wGtdIxK8ynfsUS9wdu3cfITXpCiCuWJAwHSmoYUa0acOP1lY92ZTq0ETUgAkP7W2o5MiGEqD9yOuxAyVVSCpWPPGfNScOadBJ9i+4oMqqXEOIKJr9wFXBm9FTzmd8A0DePruVohBCifknCcMDR483LYz4dgxoQgc7fdQZqF0KI2iAJw4GSTu/KahjW/CwsCX+ibya1CyHElU8ShiMXahiV9WGYz+wFTZOEIYS4KtTqVVLTpk1j586dBAQEADBo0CD+7//+D4CUlBSeffZZzp8/j5ubGzNnzqRDhw6VltUFW4tUJTUM8+kYFN8QVHmEuRDiKlDrl9VOnDiRe+6555L33377baKjo/nkk0+IiYnhmWeeYcOGDSiKUmFZXbh4lVQF05hNWOKOYri2X53FJYQQ9anemqS+//57xowZA0B0dDRGo5FDhw5VWlYXbJ3eFeQBS9JJsJrRR8pY3UKIq0OtJ4wlS5YwbNgwHn30UU6ePAlAeno6mqYRGBhomy48PJyEhIQKy+paRX0YlrijoCjowlrVYURCCFF/qtUkNWrUKOLi4hyW7dy5kyeffJLg4GBUVWXVqlVMmDCBzZs3V2eVVRIU5H1Z851LyALAz9eD4GAfh9PEpRzHLaw5IZGhlx3f5SgvnvrmqnGB68YmcVWNxFU1tRFXtRLGypUrKywPDb34Yzpy5EjeeOMNEhISiIyMBCAtLc1Wk4iPjycsLMzWQe6orKpSU3OwWqtwU8UFJXNkZeeTnJx9abnZRMH5PzFc299heW0JDvap0/U5y1XjAteNTeKqGomrai43LlVVKjzRrtUmqcTERNvrHTt2oKqqLYkMGjSI5cuXAxATE0NBQQHt27evtKxOlFxWW05ntiXpJFjM6MPb1F1MQghRz2r1KqnnnnuO1NRUFEXB29ubDz74AL2+eJVPP/00zzzzDKtWrcLNzY05c+agXngWU0VldaGyPm9LYnFfjC7smjqJRwghXEGtJoylS5eWWxYcHFxueUVldcF+iFZ71tRzKD7BKG5edRSREELUP7nTuwLl3V5hST2HLqhx3QYjhBD1TBKGA1oFt3prRYVomYmokjCEEFcZSRgOlDRJqQ5qGNa0WEBDDZLHgQghri6SMByo6E5vS2osADpJGEKIq4wkDAdsjzd3kDGsqefA6IHi3aCuwxJCiHolCcOBCmsYabHoghrLAweFEFcdSRgVsE8JmmbFmhorHd5CiKuSJAwHbI83t8sYWlYSmAvRyfgXQoirkCQMBy7etlc2Y1hSzwGgNpAahhDi6iMJwxHbs6TKvm1NjQVFRfWPqPuYhBCinknCcECzjeldliX1HKp/OIreWOcxCSFEfZOE4YCG44xhzUxA9Q+v+4CEEMIFSMJw4GIN42LG0KxmtKwUVL+qj8shhBBXAkkYFSlVw9CyU0CzoPpLwhBCXJ0kYThgu6y21HvWzOIxxVXfuh2SVQghXIUkDAcc3ehtzSwePVDxk4QhhLg6ScJwxJYxLqYMa1ZS8TOk3F1zwHchhKhtkjAcuPjwwYusWcmoPiHyDCkhxFVLEoYDmoMb97TsZFTf4PoJSAghXIAkDEfshvTWNCvW7GQUH0kYQoirlyQMB2xNUheqGFpeJljMUsMQQlzVJGE4YLVrkrJmJQGg+obUU0RCCFH/JGE4Ynent5adDIAqTVJCiKuYJAwH7J8lZc1KBkVB8Q6qv6CEEKKeScJwwP5ptdasJBSvQBSdvt5iEkKI+iYJoyIlNYzsZOm/EEJc9SRhOHDxWVIX+jCykqX/Qghx1ZOE4UDpJ4No5kK0/EwUuaRWCHGVk4ThSKkb96xZKYBcISWEENVOGKtXr2bYsGG0a9eOzz//vExZfn4+U6ZMYcCAAQwaNIitW7dWu6wu2JqkFNCy5R4MIYQAqPZlP23btmXu3LksWrTokrLFixfj7e3Npk2bOHPmDHfffTcbN27Ey8vrssvqwsXHmyvFl9SCNEkJIa561a5htGrVipYtW6Kqly5q/fr1jB49GoCmTZvSvn17tm/fXq2yuqCVbpLKTgaDO4qbd52tXwghXFGt9mHExcURGRlp+zs8PJyEhIRqldWNi01S1qwkVN9geay5EOKqV2mT1KhRo4iLi3NYtnPnTnQ6XY0HVVOCgi6vVnA8PhuAwEAvdAXp6APDCA52jYGTXCUOe64aF7hubBJX1UhcVVMbcVWaMFauXHnZC4+IiOD8+fMEBgYCEB8fT7du3apVVhWpqTlYrVrlE9opmSM9PQ//zFS0Bi1JTs6u8nJqWnCwj0vEYc9V4wLXjU3iqhqJq2ouNy5VVSo80a7VJqlBgwbx5ZdfAnDmzBkOHTrEjTfeWK2yOlHyaBBLERTmonj61926hRDCRVU7Yaxdu5bevXvz/fff895779G7d29OnDgBwPjx48nKymLAgAE8/PDDvPbaa3h7e1errC7YxsMozARA9Qqos3ULIYSrqvZltUOHDmXo0KEOyzw9PZk3b16NltWFkqukdAXFCUORhCGEEHKntyMlN+6pJQlDmqSEEEIShiMlNQw1P6P4f6lhCCGEJAxHbHd652eAzghGz/oMRwghXIIkDEdKmqTy01B9guSmPSGEQBKGQ7YaRl6aDMsqhBAXSMJwwNaHkZuG6t2gfoMRQggXIQnDIQ0jRSimHBQfqWEIIQRIwnBI0yBAzQVA9ZEahhBCgCQMhzQNAnTFCUP6MIQQopgkDIc0/JQ8AFRPuQdDCCFAEoZDmga+agEAiqdfPUcjhBCuQRKGAxrgq+ahGTxQ9Mb6DkcIIVyCJAwHNA181ALwkNqFEEKUkIThUHEfhubuW9+BCCGEy5CE4YDUMIQQ4lLVHg/jSnP0px9I3rebJsZ8kBqGEELYSMKwk/fXn/R0+xMAq9QwhBDCRpqk7BTpSj3KXGoYQghhIwnDTpHOw/Zaca+7ccSFEMLVScKwUzph4O5Tf4EIIYSLkYRhx1SqSUpxkxqGEEKUkIRhx6Iv1YfhIX0YQghRQhKGnTJNUnq3+gtECCFcjCQMO+ZSNQxVlbG8hRCihCQMe6rO9lLShRBCXCQJw46ilE4TkjKEEKKE3OltR1VgVV5nsq0ePCD5QgghbKSGYUdRFLYWXEuMqXl9hyKEEC5FEoad0i1SitQwhBDCptoJY/Xq1QwbNox27drx+eeflymbNm0avXv3ZsSIEYwYMYIPPvjAVpaSksKDDz7IwIEDGT58OAcOHHCqrLaV7sNQpA9DCCFsqt2H0bZtW+bOncuiRYsclk+cOJF77rnnkvfffvttoqOj+eSTT4iJieGZZ55hw4YNKIpSYVltU6XPWwghHKp2DaNVq1a0bNkSVa3aor7//nvGjBkDQHR0NEajkUOHDlVaVpckXwghxEW13oexZMkShg0bxqOPPsrJkycBSE9PR9M0AgMDbdOFh4eTkJBQYVldUEs3SUnGEEIIm0qbpEaNGkVcXJzDsp07d6LT6RyWATz55JMEBwejqiqrVq1iwoQJbN68+fKjraKgoKo/PNDb2932OjjYB4O+/O2rD8HBrvkEXVeNC1w3NomraiSuqqmNuCpNGCtXrrzshYeGhtpejxw5kjfeeIOEhAQiIyMBSEtLs9Uk4uPjCQsLIyAgoNyyqkpNzcFq1ao0T15eoe11SkoOep3rXEgWHOxDcnJ2fYdxCVeNC1w3NomraiSuqrncuFRVqfBEu1Z/DRMTE22vd+zYgaqqtiQyaNAgli9fDkBMTAwFBQW0b9++0rLapkiTlBBCOFTtq6TWrl3LnDlzyMrK4ocffmDRokV88skntGzZkueee47U1FQURcHb25sPPvgAvb54lU8//TTPPPMMq1atws3NjTlz5tg6zisqq21l7sOQbm8hhLCpdsIYOnQoQ4cOdVi2dOnScucLDg4ut7yistqmls0YQgghLnCdBnoXIflCCCEck4Rhp2wfhqQMIYQoIQnDjuQIIYRwTBKGHVUyhhBCOCQJw46kCyGEcEwShj3JGEII4ZAkDDvSJCWEEI5JwrAj+UIIIRyThGFHLqUVQgjHJGHYkXwhhBCOScKwI30YQgjhmCQMO9IkJYQQjknCsCPpQgghHJOEYUdqGEII4ZgkDDuq5AshhHBIEoYdqWEIIYRjkjDsSL4QQgjHJGHYkRqGEEI4JgnDjuQLIYRwTBKGHUkYQgjhmCQMO9IkJYQQjknCsCM7RAghHJPfRztSwxBCCMckYdiRfCGEEI5JwrAjNQwhhHBMEoYdeby5EEI4JgnDjuQLIYRwTBKGHWmSEkIIxyRh2JF8IYQQjlU7Ybz66qsMGjSI4cOHM2bMGA4dOmQrS0lJ4cEHH2TgwIEMHz6cAwcOVLustkkfhhBCOFbthNG7d2/WrFnDt99+y8MPP8yTTz5pK3v77beJjo5mw4YNTJ8+nWeeeQZN06pVVtskXwghhGPVThh9+vTBYDAA0LFjRxISErBarQB8//33jBkzBoDo6GiMRqOtBnK5ZUIIIepHjfZhLFu2jJtvvhlVVUlPT0fTNAIDA23l4eHhJCQkXHZZXZAmKSGEcExf2QSjRo0iLi7OYdnOnTvR6XQArFu3jjVr1rBs2bKajbAagoK8qzxPnuVi01dwsE9NhlMjXDEmcN24wHVjk7iqRuKqmtqIq9KEsXLlykoXsmnTJubOncvSpUtp0KABAAEBAQCkpaXZagvx8fGEhYVddllVpabmYLVWre8jIz3P9jo5ObvK66xNwcE+LhcTuG5c4LqxSVxVI3FVzeXGpapKhSfa1W6S2rp1K2+88QaLFy+mYcOGZcoGDRrE8uXLAYiJiaGgoID27dtXq6y2SYuUEEI4VmkNozLPP/88BoOByZMn295bunQpAQEBPP300zzzzDOsWrUKNzc35syZg6oW56jLLatt0ochhBCOVTth/PLLL+WWBQcHs3Tp0hotq22SL4QQwjG509uOPBpECCEck4RhR/KFEEI4JgnDjvRhCCGEY5Iw7EiTlBBCOCYJw47kCyGEcEwShh3JF0II4ZgkDDvSJCWEEI5JwrAj+UIIIRyThGFHahhCCOGYJAw7quQLIYRwSBKGHalhCCGEY5Iw7Ei+EEIIx6r98MErjdQwhKgZFouZ9PRkzGZTtZeVlKTahn52JX/XuPR6IwEBweh0VUsBkjDsSB+GEDUjPT0Zd3dPvLzCqn0iptermM2u98P8d4xL0zRyc7NIT0+mQYPwKi1XmqTsSA1DiJphNpvw8vKV75SLURQFLy/fy6r5ScKwI8e2EDVHkoVrutzPRRKGHTnAhRC17fXXX+Gbb76s7zCqTBKGHUkXQlz5zGbzFbmu2iad3nakhiHElalXr2geeOAhdu36mW7dbmDs2HHMnz+XkyePYzKZiIqKZtKkJzl/PpYXXniWzz//CrPZzJAh/bjvvvGMHXsvP/ywiR07tvHKK6/zxRefs2XLRsxmM0ajG1OnTuOaa1o7XNfIkbfzz3/OIDU1hbCwcFT14rn66tUr+Oqr/2IwGNE0K6+9NosmTZrWyz6qjCQMIUSt+/lQPD8djL/s+RUFNM1xWa/rw+l5nXNX+7i5ufHxx58CMGvWTDp27MS0aS9jtVp59dWXWLfuW4YPH0VeXi4pKSkkJMTRrFkLYmL2MHbsvfz2269ER3cBYNCgIYwbdy9ms5U9e3bz5ptvsGjRUofrevHFZ+jQIYoHH5zI+fN/cf/9Y+nW7QYA3n//PZYt+4YGDRpgMplc8jLdEpIwhBBXjcGDh9pe//TTdv744zDLly8DoKCggJCQUAA6dYrmt99+JT4+jhEjbmPZsk8pKioiJuZX7rnnfgCOHfuDl19eQmZmJqqqEht7rtx17d37G1OmPANAZGRDW9IpXlcXXn99Bj173sgNN/QiMrJhrWx7TZCEIYSodT2vc74W4EhN3e/g4eFZ6i+Nf/3rLYc/0J07d+G33/YQF3ee6dNnsn//XjZv3oCmQUREJEVFRbz88nN88MHHtGzZmpSUZEaOHFzBusr3r3+9yR9/HOa332KYPPkRpk59nhtu6Fmdzaw10ukthLgq9ezZm88//w8WiwWAjIwM4uLOA8UJY/fuXWRnZxMSEkp0dFcWL/7QVjMwmQqxWCyEhoYBsGLF1xWuq3PnaNat+xaAuLjzxMTsAYo7xOPiztOuXXvGjbufrl27c/z4sVrZ3pogNQwhxFXpiSee5v3353H//f9AURQMBiOTJz9NREQkISGheHp6cv31HYHiBJKYmECnTtEAeHl5M378wzzwwD34+vrRp0+/StY1lX/+cwabN28gPDyCqKjOAFitVl5//RVycrJRFJXQ0FAeeeTxWt3u6lA0rbyupL+/1NQcrNaqb96Ds7YA8Mm0vjUdUrUEB/uQnJxd32FcwlXjAteN7WqIKyHhLGFhTWpkWX/HR3DUJ2ficvT5qKpCUJB3ufNIk5QQQginSMIQQgjhFEkYQgghnCIJQwghhFNq5CqpV199lV27dmE0GvH09OTFF1/kuuuuA2DcuHHExcXh7V3ckXLvvfdy++23A3D69GmmTZtGRkYG/v7+zJ49m6ZNm1ZaJoQQou7VSA2jd+/erFmzhm+//ZaHH36YJ598skz5Sy+9xOrVq1m9erUtWQDMmDGDsWPHsmHDBsaOHcv06dOdKhNCCFH3aiRh9OnTB4PBAEDHjh1JSEio9HkoqampHDlyhKFDi2+fHzp0KEeOHCEtLa3CMiGEEPWjxvswli1bxs0331zmaYxz5sxh2LBhTJ06lcTERADi4+MJDQ1Fp9MBoNPpCAkJIT4+vsIyIYQQl6qLMTac6sMYNWoUcXFxDst27txp+2Fft24da9asYdmyZbbyOXPmEB4ejsVi4cMPP2TKlCl88cUXNRB65Sq6AcUZwcE+NRRJzXHFmMB14wLXje1KjyspSUWvr7lz0ppcVk0qictsNqPX183DMxytS1EUVFWxxVPZ/lJVtcqftVNbt3Llykqn2bRpE3PnzmXp0qU0aNDA9n54ePEDx3Q6Hffeey///ve/sVqthIeHk5iYiMViQafTYbFYSEpKIjw8HE3Tyi2risu907uEq92JezXcHVzTXDW2qyEuq9VaY3dB18Qd1b16RfPQQ//Hjh0/kpmZyXPPvUhMzK/s3r0Ts9nMzJmzadq0GampKbzyyovk5uZiMpno0aMnjz76RLnLrOkxNn74YSMWy+WNsWG1apjNVtauXckXXyyrcIwNq9V6yWdd2Z3eNZIOt27dyhtvvMGSJUto2PDikx/NZjMZGRm2BLJu3TpatWqFqqoEBQXRtm1b1q5dy4gRI1i7di1t27YlMDAQoMIyIcTfS9GfP1N0bPtlz68oCuU9xcjQujeGVs493dXb24ePP/6ULVs28/zzT/PKK//ikUceZ9my//Dpp58wffpMvL19mD17Lp6enpjNZp566nF++WUn3bv3cLjMmh5j4x//uAegWmNszJ//HsuW/a/Gx9iokYTx/PPPYzAYmDx5su29pUuX4ubmxsSJEykqKgIgJCSEd955xzbNK6+8wrRp03j//ffx9fVl9uzZTpUJIcTl6NfvFgBat24DKPTseeOFv9vy449bgeIz7/fff49Dhw4CGqmpqRw//me5CaOmx9j47LMlZGVVb4yN6OjaGWOjRhLGL7/8Um7ZihUryi1r0aIFX3/t+LHAFZUJIf5eDK16Ol0LcKSmHvJnNBqB4vZ7o9Fge19VVdtjzr/8chnZ2VksWlR80jt79uuYTIXlLrOmx9j4978/onXrNtUaY2PWrLc4dOhQjY+x4Zq9SEIIUU+ys7MJCmqAm5sbyclJ/PTTj07PWxNjbJTUSKozxsb583/VyhgbMh6GEEKUcuedY3j55ecYN+4ugoND6dy5S+UzXVATY2w89NC91R5jY+bMGWRn1/wYGzIehgMyHkbVuGpc4LqxXQ1xyXgY9UfGw6hjIf4e9R2CEEK4FGmScuCbWUNJTc2p7zCEEMKlSA3DAaNBh14nu0YIIUqTX0UhRK25grtI/9Yu93ORhCGEqBV6vZHc3CxJGi5G0zRyc7PQ641Vnlf6MIQQtSIgIJj09GRycjKqvazi5yS53tVIf9e49HojAQHBVV6uJAwhRK3Q6fQ0aFC1B4aW52q4DLkm1VZc0iQlhBDCKZIwhBBCOOWKbpJSVaVe5q1NElfVuWpsElfVSFxVczlxVTbPFf1oECGEEDVHmqSEEEI4RRKGEEIIp0jCEEII4RRJGEIIIZwiCUMIIYRTJGEIIYRwiiQMIYQQTpGEIYQQwimSMIQQQjhFEkYpp0+fZvTo0QwcOJDRo0dz5syZeoulb9++DBo0iBEjRjBixAh27NgBwP79+xk+fDgDBw7kwQcfJDU1tVbjmD17Nn379qV169b8+eeftvcr2ld1sR/Li6u8/QZ1s+/S09N56KGHGDhwIMOGDePxxx8nLS2t0vXXdmwVxdW6dWuGDRtm22fHjh2zzbdlyxYGDRrEgAEDmDJlCvn5+TUaF8Cjjz7K8OHDGTlyJGPHjuWPP/4A6v8Yqyi2+j7OAP7973+XOf7r5PjShM24ceO0VatWaZqmaatWrdLGjRtXb7H06dNHO3bsWJn3LBaL1r9/f23Pnj2apmnaggULtGnTptVqHHv27NHi4uIuiaeifVUX+7G8uBztN02ru32Xnp6u/fLLL7a/Z82apT3//PMVrr8uYisvLk3TtFatWmk5OTmXzJOTk6P16NFDO336tKZpmvbCCy9o8+fPr9G4NE3TsrKybK83bdqkjRw5UtO0+j/GKoqtvo+z33//XRs/frwtjro6viRhXJCSkqJ17txZM5vNmqZpmtls1jp37qylpqbWSzyODsgDBw5oQ4YMsf2dmpqqdezYsc7jqWhf1fV+dDZh1Ne++/7777X77ruvwvXXR2wlcWla+Qnju+++0yZOnGj7++DBg9qtt95aq3GtXLlSGzVqlEsdY/axaVr9HmeFhYXaXXfdpcXGxtriqKvj64p+Wm1VxMfHExoaik6nA0Cn0xESEkJ8fDyBgYH1EtPUqVPRNI3OnTvz1FNPER8fT0REhK08MDAQq9VKRkYG/v7+dRZXRftK07R634/2+83X17de9p3VauWLL76gb9++Fa6/rmMrHVeJcePGYbFY6N27N5MmTcJoNF4SV0REBPHx8TUeD8CLL77Izz//jKZpfPzxxy51jNnHVqK+jrP33nuP4cOH07BhQ9t7dXV8SR+Gi1q2bBnffvst33zzDZqm8dprr9V3SH8LrrTfZs6ciaenJ/fcc0+9xeCIfVzbtm1jxYoVLFu2jBMnTrBgwYI6j+n1119n27ZtPPnkk8yZM6fO118RR7HV13G2b98+fv/9d8aOHVsn67MnCeOC8PBwEhMTsVgsAFgsFpKSkggPr5khJi8nHgCj0cjYsWPZu3cv4eHhxMXF2aZJS0tDVdU6rV2UxFbevqrv/ehov5W8X5f7bvbs2Zw9e5Z3330XVVUrXH9dxmYfF1zcZ97e3tx5553l7rO4uLha/xxHjhzJ7t27CQsLc7ljrCS29PT0ejvO9uzZw8mTJ+nXrx99+/YlISGB8ePHc/bs2To5viRhXBAUFETbtm1Zu3YtAGvXrqVt27b10hyVl5dHdnbxeLyapvHdd9/Rtm1b2rdvT0FBATExMQAsX76cQYMG1Xl8Fe2r+tyP5e03oE733TvvvMPvv//OggULMBqNla6/rmJzFFdmZiYFBQUAmM1mNmzYYNtnN954I4cOHbJdgbR8+XIGDx5cozHl5uaWaebasmULfn5+LnGMlRebm5tbvR1nEydO5KeffmLLli1s2bKFsLAwFi9ezIQJE+rk+JIBlEo5efIk06ZNIysrC19fX2bPnk3z5s3rPI7Y2FgmTZqExWLBarXSokULXnrpJUJCQti7dy8zZsygsLCQyMhI3nzzTRo0aFBrsfzzn/9k48aNpKSkEBAQgL+/P+vWratwX9XFfnQU18KFC8vdb0Cd7Lvjx48zdOhQmjZtiru7OwANGzZkwYIFFa6/tmMrL64JEyYwffp0FEXBbDYTFRXFCy+8gJeXFwCbN2/mzTffxGq10rZtW2bNmoWnp2eNxZWSksKjjz5Kfn4+qqri5+fHc889x7XXXlvvx1h5sfn6+tb7cVaib9++LFy4kFatWtXJ8SUJQwghhFOkSUoIIYRTJGEIIYRwiiQMIYQQTpGEIYQQwimSMIQQQjhFEoYQtWjhwoW8+OKLlzXvtGnTmDt3bg1HJMTlk2dJCVGLHnnkkfoOQYgaIzUMIYQQTpGEIUQpiYmJTJo0ie7du9O3b18+/fRTAObPn8/kyZOZMmUKUVFRjBo1iqNHj9rmW7RoETfeeCNRUVEMHDiQXbt22eabOnWqbboffviBIUOGEB0dzbhx4zh58qSt7MiRI4waNYqoqCimTJlCYWFhmdi2bt3KiBEjiI6OZsyYMU6tX4gadVkPRRfiCmSxWLRRo0Zp8+fP1woLC7Vz585pffv21bZv367NmzdPa9eunbZ+/XrNZDJpH3/8sdanTx/NZDJpJ0+e1Hr37q0lJCRomqZpsbGx2tmzZzVN07R58+ZpTz/9tKZpmnbq1CmtQ4cO2k8//aSZTCZt0aJFWv/+/bXCwkKtsLBQu/nmm7UlS5ZoJpNJW79+vdauXTvtnXfe0TRN0w4fPqx1795d279/v2Y2m7UVK1Zoffr00QoLCytcvxA1SWoYQlxw6NAh0tLSePzxxzEajTRq1Ii77rqL7777DoBrr72WQYMGYTAYeOCBBzCZTBw4cACdTofJZOLkyZMUFRXRsGFDGjdufMnyv/vuO2666SZ69uyJwWBg/PjxFBQUsG/fPg4cOEBRURH33XcfBoOBQYMGcd1119nm/fLLLxk9ejQdOnRAp9MxatQoDAYD+/fvd3r9QlSXdHoLccH58+dJSkoiOjra9p7FYiE6OpqIiAjCwsJs76uqSmhoqG36F154gfnz53PixAl69erFtGnTCA0NLbP8pKSkMgPZlDz2PDExEZ1OR2hoKIqi2MpLTxsXF8eqVav4/PPPbe8VFRWRlJRE165dnVq/ENUlNQwhLggPD6dhw4bExMTY/u3bt4+PPvoIgISEBNu0VquVxMRE2xNKhw0bxhdffMHWrVtRFIW33nrrkuWHhISUGZdA0zTbyHLBwcEkJiailXoWaOlpw8PDeeSRR8rEduDAAYYOHer0+oWoLkkYQlxw/fXX4+XlxaJFiygoKMBisfDnn39y8OBBAA4fPszGjRsxm8385z//wWg00qFDB06dOsWuXbswmUwYjUbc3NxsgxOVNnjwYH788Ud27dpFUVERn3zyCUajkaioKDp27Iher+fTTz+lqKiIjRs3cujQIdu8d955J8uXL+fAgQNomkZeXh7btm0jJyfH6fULUV3SJCXEBTqdjoULFzJ79mz69euHyWSiWbNmTJkyBYB+/frx3Xff8dxzz9GkSRPmz5+PwWDAZDLx9ttvc/LkSQwGA1FRUQ6H7GzevDlvvvkmM2fOJDExkbZt27Jw4ULbYEbz58/n5Zdf5t133+Wmm25iwIABtnmvu+46Zs6cyWuvvcbZs2dxd3enU6dOREdHO71+IapLxsMQwgnz58/n7Nmz0tQjrmpSbxVCCOEUSRhCCCGcIk1SQgghnCI1DCGEEE6RhCGEEMIpkjCEEEI4RRKGEEIIp0jCEEII4RRJGEIIIZzy/62D+zOTOmZTAAAAAElFTkSuQmCC", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "加载模型成功！\n", - "开始测试!\n", - "环境：CliffWalking-v0, 算法：Q-learning, 设备：cuda\n", - "回合：1/20，奖励：-13.0\n", - "回合：2/20，奖励：-13.0\n", - "回合：3/20，奖励：-13.0\n", - "回合：4/20，奖励：-13.0\n", - "回合：5/20，奖励：-13.0\n", - "回合：6/20，奖励：-13.0\n", - "回合：7/20，奖励：-13.0\n", - "回合：8/20，奖励：-13.0\n", - "回合：9/20，奖励：-13.0\n", - "回合：10/20，奖励：-13.0\n", - "回合：11/20，奖励：-13.0\n", - "回合：12/20，奖励：-13.0\n", - "回合：13/20，奖励：-13.0\n", - "回合：14/20，奖励：-13.0\n", - "回合：15/20，奖励：-13.0\n", - "回合：16/20，奖励：-13.0\n", - "回合：17/20，奖励：-13.0\n", - "回合：18/20，奖励：-13.0\n", - "回合：19/20，奖励：-13.0\n", - "回合：20/20，奖励：-13.0\n", - "完成测试！\n", - "结果保存完毕！\n" - ] - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEcCAYAAADdtCNzAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAzDUlEQVR4nO3de1xM+f8H8NfM1GDVJlQKu4sWtS4NJURuodSo+LLs5s6ua8vqS2QvYpG9hNbl27ov67LfRQpr01qxWffwtfGTXbutUZHQhSYz5/eHr/kana6nKZvX8/HweMzM+ZzzeZ8zn+k155xxjkwQBAFERETPkFd3AURE9HxiQBARkSgGBBERiWJAEBGRKAYEERGJYkAQEZGoGhUQvXv3RlJSUpX3e/r0afTv37/K+6X/+euvv9CqVSs8evSo0pcdHx+PHj16QKVS4ddff6205dbE8Xr79m28/fbbUKlUWLJkiUn6eFarVq3wxx9/AAA+/PBDrFy50jDtm2++QdeuXaFSqZCdnY0zZ86gX79+UKlUOHToUKXV8Oz4GzFiBL799lvRtuPHj8fu3bsrrW9TMqvuAmoCV1dXHDx4sLrLIBOJiIjABx98AC8vL9HpgiBg3bp12LlzJ9LT01G/fn0MHDgQU6dOhVKprOJqS2fK8bpjxw5YW1vj7NmzkMlklbLMzMxMLFu2DImJicjLy4OdnR0GDBiA8ePH46WXXjJqGx4ebnhcWFiIJUuWYOfOnWjdujUAYMWKFXj77bcxatQoxMXFwcfHBwcOHDDMM2bMGKSnpxd5rUuXLnjnnXcqZX3Wrl1bKcspr7/++gtz5szBhQsXYG9vjw8//BBdu3YtcZ4atQdhKjqdrrpLkKwmrEN10Wg0eP3114udvnDhQuzcuRMRERE4e/YsvvrqKyQlJeH999+vwir/pzrfa41GgxYtWlQoHMT2/u7evYthw4ahoKAA27dvx7lz57Bhwwbcv38ff/75Z4nLy8rKQkFBARwdHY3qe/Jeurm54bfffsOdO3cM/V++fBkFBQVGryUnJ8PV1bXc6/O8mTlzJpydnXHixAnMmDEDwcHBhvUsTo0NCL1ej+joaHh5ecHd3R3vvfce7t69a5geHBwMDw8PdOzYEW+//TauXr1qmBYaGoqPPvoIEyZMgIuLC06cOIHevXtj3bp1UKvV6NixI6ZPn46CggIAwIkTJ+Dp6WmYv6S2APDVV1+hW7du6NatG7799lujXeRn3b17F3PmzEG3bt3g5uaGyZMnAwB27dqF4cOHG7V9ejnPrsO6devg4eFh9McjPj4earW6TNvrWTt37kTfvn3RqVMnTJw4ERkZGUZ1bNu2Df369YOrqyvmz5+P4v7Dvk6nw5o1a+Dl5QWVSoVBgwbh5s2booeMnt5t1+l0iIiIgLu7O/r06YMjR44YLfe7776Dj48PVCoV+vTpg+3btxe7Lnq9HqtWrUKvXr3QpUsXzJo1Czk5OdBqtVCpVNDpdPD39xfdg7h+/Tq++eYbfPbZZ1CpVDAzM8Prr7+OqKgo/PTTTzh58mSx/T5bw999vIaGhmLPnj1Yt24dVCoVkpKSoNVq8cknnxjm/+STT6DVao3qiI6OhoeHB+bMmVNkmRs2bEDdunXx6aefokmTJgAAe3t7zJs3z7BX8GwNkZGR+P333+Ht7Q3gcRCMHDkSXl5eSEtLw8SJE6FSqWBtbY2mTZvi1KlTAIBff/0Vjo6OcHNzM3pNr9ejbdu2+OmnnxAQEIAOHTqgR48eiIqKKtN7m5mZCbVabdhzeHocP/kcR0REwM3NDb179zYay2lpaYZDdqNHj8b8+fMREhIi2s/48eOxZcsWo9cGDhyIH374Ab///jsuXbqEadOmoXbt2ujfvz9atmxZ6p5kjQ2Ir7/+GocOHcKWLVtw9OhRWFlZGe1+enp64uDBgzh+/DicnZ2LbPS4uDhMnDgRZ8+eRceOHQEABw4cwNq1a5GQkIArV65g165dxfZfXNvExERs3LgRGzZsQHx8PE6cOFHiesyaNQsPHjzAvn37kJSUhNGjR5d5Gzy9DqNGjUKdOnXwyy+/GKbHxsYaAqK07fW048eP4/PPP8eyZctw7NgxNG7cuMi35Z9++gn//ve/sXfvXhw4cABHjx4VXdaGDRuwb98+REdH4+zZs1i0aBFq165d6rrt3LkThw8fxp49e/Ddd9/h+++/N5reoEED/Otf/8LZs2exePFiLF68GJcuXRJd1q5du7B7925s3rwZhw4dQn5+PsLDw6FUKnHu3DkAQExMjOgx6+PHj6NRo0Zo166d0ev29vZwcXHBzz//XOq6ADVjvC5ZsgRqtRrjxo3DuXPn0LVrV6xevRrnz59HTEwM9u7di4sXL2LVqlWGeW7fvo179+7h8OHDWLBgQZFlHj9+HH379oVcXr4/Vc2aNUNcXBwA4NSpU4b31sHBAWvWrMG5c+egVCqNwuDUqVNwdXVFx44djV5r3749zM3NUadOHUREROD06dP417/+hW3btpV6HiMtLQ0jRoxAUFAQxo8fL9rmwoULaNasGX755ReMHz8eYWFhhi9UISEhaNeuHU6cOIGpU6ciJiam2L78/PwM6wwAqamp0Gg06NmzJ1JTU9G0aVNYWFgYprdu3Rqpqakl1l9jA2L79u2YMWMGGjVqBKVSialTp+LgwYOGb6T/+Mc/YGFhAaVSiWnTpuHy5cvIyckxzN+nTx907NgRcrkctWrVAvA4+e3s7FCvXj306tULKSkpxfZfXNsDBw5g0KBBeP3111GnTh1Mmzat2GVkZmYiMTER8+fPh5WVFczNzdGpU6cyb4Nn18HX19cwgHJzc5GYmAhfX98yba+nxcbGYvDgwXjjjTegVCrx/vvvIzk5GX/99ZehzYQJE/Dyyy/DwcEB7u7uuHz5smiN3377Ld577z00b94cMpkMrVu3hrW1danrduDAAYwaNQr29vaoV68e3n33XaPpPXv2xCuvvAKZTIZOnTrBw8MDp0+fFl1WbGwsRo8ejaZNm6Ju3bp4//33sX///jKd8M7OzoaNjY3oNBsbm1J34Z+oCeNVTGxsLKZMmYIGDRqgfv36mDJlCvbu3WuYLpfLERwcDKVSKfrF4O7du8Vu38rg5uZmGBenT582BMTTrz35zLm7u6NVq1aQy+Vo3bo1fH19S9xDTE1NxahRozBt2jS8+eabxbZzcHDA0KFDoVAoEBgYiFu3buH27dvQaDS4ePGiYfu4urqid+/exS7Hy8sLly9fxo0bNwA83vZ9+/aFUqlEXl4eLC0tjdpbWloiLy+vxO1TYwNCo9FgypQpcHV1haurKwYMGAC5XI6srCzodDp89tln8PLyQocOHQwbPTs72zC/vb19kWU+PVDr1KmD/Pz8Yvsvrm1mZiYaNWpUYj9PpKenw8rKClZWVmVY46KeXbZarUZ8fDy0Wi3i4+Ph7OyMxo0bAyh5ez0rMzPTMB8A1K1bF/Xq1TM6zPTs+hc3ENPT0/HKK6+Ue90yMzON1s/BwcFo+pEjRzB06FB06tQJrq6uSExMNHp/S1qfxo0b49GjR6Lr/ixra2vcunVLdNqtW7cMYTd+/HioVCqoVCqjP5BP1ITxKiYzM9PovXFwcEBmZqbhubW1tSHQxNSrV6/Y7VsZ3NzccOXKFdy7dw/nz5+Hi4sLWrRogVu3buHevXs4e/as4fzD+fPnMWLECHTu3BkdO3bE9u3bix1TwOM/0La2tqX+Yqxhw4aGx3Xq1AEA5OfnIzMzE1ZWVobXAOPt/+GHHxrG1Jo1a2BhYYEePXpg3759AB7vVQ4cOBDA489obm6uUb+5ubmoW7duibXV2IBo1KgRvvrqK5w+fdrw7+LFi7Czs0NsbCwSEhKwYcMGnDlzBj/++CMAFHucvDLZ2toa/SG9efNmietw79493L9/v8i0OnXq4OHDh4bnZfkQOTo6wsHBAYmJiYiLi4Ofn59RX8VtL7F1ePItBXg8mO/evSvatjSNGjUSPdn45Ncpxa2jjY2N0bZ7+rFWq0VwcDDGjh2Ln3/+GadPn4anp2ex7++z66PRaGBmZoYGDRqUWn/nzp1x8+ZNXLhwwej1mzdvIjk52fDtc+3atTh37hzOnTtn+NA+rSaM1+Lm12g0RvPb2toanpd2MrtLly6Ij4+HXq8vV79l1bRpU9ja2mLHjh2wt7c3/MF0cXHBjh07kJeXBxcXFwCPT/I+Od915swZDBs2rMT3YOrUqbC2tsbMmTMr9MMBGxsb3Lt3Dw8ePDC89vT2Dw8PN4ypiRMnAnh8mGnfvn04d+4cCgoK4O7uDuDxZz8tLc0oJC5fvmx0Al9MjQ2I4cOHY9myZYYP/p07dwzHC/Py8qBUKmFtbY0HDx7giy++qLK6vL29sWvXLly7dg0PHjwwOh77LFtbW3h6emL+/Pm4d+8eCgsLDcdGW7dujatXryIlJQUFBQVlPmHm5+eHTZs24dSpU4aTeEDJ20tsGbt27UJKSgq0Wi2++OILtGvXznASsTyGDBmC5cuX4/r16xAEAZcvX0Z2djbq168POzs7xMTEQKfT4d///jfS0tIM8/n4+ODrr79Geno67t27h+joaMM0rVYLrVaL+vXrw8zMDEeOHCnxXMCTbZKWloa8vDxERkbCx8cHZmal/wq8WbNmGDZsGEJCQpCcnAydToerV69i2rRpUKlUpf6M8ImaMF7F+Pr6YvXq1bhz5w7u3LmDlStXGs57lcWYMWOQl5eH2bNnG7ZNRkYGFi9eXOxhy/JydXXFxo0bjX6p1LFjR2zcuBFt2rQxHPrKy8uDlZUVatWqhQsXLhgd7xdjbm6O5cuX48GDB5g1a1a5Q65x48Zo06YNoqKioNVqce7cORw+fLjEeXr06AGNRoMVK1YY9kKBx+PUyckJK1euREFBAeLj43HlypVS925qbECMHDkSvXv3xtixY6FSqTB06FDDt7yAgAA4ODige/fu8PX1NXxDqAo9evTAiBEjMHLkSPTt2xft27cHgGJ/L7906VKYmZnBx8cHXbt2xaZNmwA8fsOnTJmC0aNHo1+/foYTk6Xx8/PDqVOn0LlzZ9SvX9/weknb61ldu3bFe++9h2nTpqFbt25IS0tDZGRkeTaDwZgxY+Dj44OxY8eiQ4cOCAsLM/yCZsGCBVi3bh3c3d2RmpoKlUplmG/o0KHo1q0b/P39ERgYiH79+hmmWVhYYN68eZg+fTrc3NwQFxdX4rHbwYMHY+DAgQgKCkKfPn2gVCrxwQcflHkdPvzwQ/zjH//AP//5T7Rv3x5+fn5wcHDAqlWrynxytaaM12dNnjwZbdq0wcCBAzFw4EC88cYbhl/ilUW9evWwbds2mJmZYejQoVCpVBg1ahQsLS3x6quvVmidnuXm5oasrCyjz5CrqyuysrLg5uZmeO2jjz7CihUroFKpsHLlSvj4+JS6bKVSiS+//BJZWVmYO3duuUPis88+Q3JyMtzd3bFs2TIMGDCgxG2vVCrRt29fJCUlGR0hAIAvvvgC//nPf+Dm5obPPvsMK1asMPobIEbGGwZVr2vXrsHPzw8XL14s0zdWev6tWLEC8fHx2Lp1K15++eXqLqdScbxWr+nTp6N58+YIDg6ukv5q7B7E8+zJieJ79+7h008/Ra9evfhhq0GCg4Px5ptvIjk5ubpLqRQcr9XnwoUL+PPPP6HX65GYmIiEhIRi/0e/KXAPohqMGzcOycnJUCgUcHNzw0cffWR04o7oecLxWn1+/PFHzJ8/H3fv3kWjRo3wzjvvYPDgwVXWPwOCiIhE8RATERGJYkAQEZEoBgQREYmqUT9FyM7Og15fsVMqDRpYICsrt/SG1YT1ScP6pGF90jyv9cnlMlhbF3+5jRoVEHq9UOGAeDL/84z1ScP6pGF90jzv9YnhISYiIhLFgCAiIlE16hATEVUfQRCQnX0LWu1DAFV7OCUzU26yK75WhuquT6Ewg4VFPdSpU/LlvZ/FgCCiSpGbew8ymQx2dk0gk1XtwQkzMzkePXp+A6I66xMEAYWFWty9+/hy+eUJCR5iIqJK8eBBLiwt61V5OFDJZDIZlMpaqFfPBrm5d8s1L99JIqoUer0OCgUPSjyvzM2V0OlKv43u0xgQRFRpSrtDHFWfirw3DAgiIhMLD/8I3323o7rLKDcGBBHVeI8ele/Qyt+lL1PjAUMiqpG6dXPFmDETcPz4z3B374K33hqBqKhIXLt2FVqtFiqVK6ZNm4EbN9Iwd+4sbNmyE48ePYKvbx+MGjUOb701EgkJ8Th69Cd8/PEn2LZtCxISfoBO9whKZS2EhITi9ddbifYVEDAYCxd+hKys22jUyB4Kxf++i8fE7MLOnd/A3FwJQdAjPHwJXn31tWrZRqVhQBBRpfv54k0cu3DTJMvu1s4eHm3ty9S2Vq1aWLt2MwBgyZIFcHHpgNDQD6DX6zF//jzs27cXAwcGIj8/D7dv30Z6ugbNmrXA6dOn8NZbI3HmzEm4uj6+L7W3ty+GDw8CAJw6dQKffroY0dEbRfsKC/sn2rdXYezYd3Djxl8YM+YtdOrUBQCwatVybN36HRo2bAitVvtc//+NSgmImJgYrF27FteuXcPcuXMRFBRkmDZ//nwcP34cSqUSL730EsLCwtC2bdtil3Xnzh34+fnB1dUVK1asqIzyiOgF5ePjZ3h87FgiUlIuYfv2rQCAhw8fwtbWDgDQoYMrzpw5iZs3NfD3H4StWzejsLAQp0+fRFDQaADAlSsp+PrrDbh//x7kcjnS0v4stq+zZ89g+vR/AgAaN24CV9dOhmkdOrjhk08+godHd3Tp0g2NGzcxybpXhkoJCCcnJ0RGRiI6OrrINE9PT8ydOxfm5uY4fPgwZsyYgUOHDhW7rI8//hg9evRAXl5eZZRGRNXAo23Zv+WbUp06Lz31TMCiRZ+J/kHu2NENZ86cgkZzAx9+uADJyWdx6NBBCALg4NAYhYWF+OCD2fjyy6/QqlVr3L59CwEBPiX0VbxFiz5FSsolnDlzGsHBExESMgddunhIWU2TqZST1C1btoSjoyPk8qKL69WrF8zNzQEALi4uSE9PL3aXau/evWjYsCHc3NwqoywiIgMPD09s2bIJOp0OAHD37l1oNDcAPA6IEyeOIycnB7a2dnB17YR16/5lOLyk1RZAp9MZ9jh27fq2xL46dnTFvn17AQAazQ2cPn0SwOMT2BrNDTg7t8GIEaPRqVNnXL16xSTrWxmq9BzE1q1b0bNnT9EgycjIwMaNG/H111/j4MGDFVp+gwYWkuqzsbGUNL+psT5pWJ80pdWXmSmHmVn1/TBSrG8zs//V9P77/8SXXy7HmDFvQSaTwdzcHNOnh+CVV5rCwcEedevWhYuLC8zM5OjUyR3h4R/Aza0TzMzksLJ6GRMmTMSECSNhZWWF3r29ivRp3NcshId/gKCgg3BwaAyVqiPkchnkcmDRoo+Rm5v738uS2GHq1OAq225yubxc40wmCEKpV9UKDAyERqMRnZaUlASFQgEACA0NRZs2bYzOQTyxb98+rFixAlu3bkXDhg2LTH/nnXcwZswYdOnSBbt27cJPP/1U7nMQWVm5Fb7muo2NJW7dyqnQvFWB9UnD+qQpS33p6X+gUaNXq6giY7wWU9k8+x7J5bISv1iXaQ9i9+7dkoqKj49HZGQkNm7cKBoOAJCcnIywsDAAQF5eHgoKCjBhwgR89dVXkvomIqKKMfkhpsOHD2Px4sXYsGEDmjQp/mz9yZMnDY8rugdBRESVp1IOfMXFxcHT0xPff/89li9fDk9PT6SmpgIA5syZg8LCQgQHB8Pf3x/+/v7Izs4GAISFhSEhIaEySiAiokpWpnMQfxc8B1F9WJ80NaE+noMo3vNSX3nPQfBaTEREJIoBQUREohgQREQkigFBRPQ39MknH5v8HhMMCCKiCngR7jHBy30TUaUr/L+fUXgl0STLNm/lCfOWpV/crls3V0yYMAlHjx7BvXv3MHt2GE6fPokTJ5Lw6NEjLFgQgddea4asrNv4+OMw5OXlQavVomtXD0ye/F6xy6zIPSb69/fCqFFjK/UeE09fsshU95hgQBBRjWVhYYm1azfjxx8PYc6cmfj440WYOHEqtm7dhM2b1+PDDxfAwsISERGReOmll/Do0SO8//5U/PJLEjp37iq6zIrcY6J58+aVfo+J0aPfgru7ae8xwYAgokpn3tKjTN/yTa1Pn34AgFatWgOQwcOj+3+fO+HIkcMAAL1ej1WrluPixQsABGRlZeHq1f8rNiAqco+JgIDB+PrrTZV8j4n/XfXaVPeYYEAQUY2lVCoBPL6KqVJpbnhdLpcbLvu9Y8dW5OTcR3T0RtSqVQsREZ9Aqy0odpkVucdEePgnOHv2zN/uHhM8SU1EL7ScnBw0aNAQtWrVwq1bmTh27EiZ563ee0ycAmDae0xwD4KIXmhDhgzDBx/MxogRQ2FjY4eOHct+w7L33puJVatWYPTo4f+9x4QSwcEz4eDQGLa2dnjppZfQrp0LgMeBkZGRjg4dXAEAdetaYNy4dzFhwki8/LIVevXqU0pfIVi48CMcOnQQ9vYOUKk6Anh8iOyTTz5Gbm4OZDI57OzsMHHi1IptjGfwWkz/VROuhVOdWJ80NaE+XoupeM9LfbwWExERVQoGBBERiWJAEFGlqUFHrGucirw3DAgiqhRyuQI6XfVcEoJKV1iohUJRvt8lMSCIqFLUqWOBnJy7EITqPxlL/yMIArTaAty9ewsWFvXKNS9/5kpElcLCwgrZ2beQkfEXgKo91CSXyyvt8hKmUN31KRRmsLS0Rp06dcs1HwOCiCqFTCZD/fq21dJ3TfiZ8POIh5iIiEgUA4KIiERJDoiYmBio1Wo4Oztjy5YtRtPmz58Pb29vDBw4EMOGDcPFixeLXc7x48cxaNAg+Pr6wtfXF5cvX5ZaGhERSSD5HISTkxMiIyMRHR1dZJqnpyfmzp0Lc3NzHD58GDNmzMChQ4eKtMvIyEBYWBjWrl2L5s2b4+HDh9V2ByUiInpMckC0bNkSAIzubvREr169DI9dXFyQnp4OvV5fpO0333wDf39/NG/eHABQu3ZtqWUREZFEVfYrpq1bt6Jnz56iQZKamorGjRtj5MiRuH//Ptzd3TFz5kzDtdyJiKjqlRoQgYGB0Gg0otOSkpKgUChK7WTfvn2IjY3F1q1bRafrdDqcPXsWGzZsQK1atRASEoLo6GhMnVq+S9aWdFXCsrCxsZQ0v6mxPmlYnzSsT5rnvT4xpQbE7t27JXUQHx+PyMhIbNy4EQ0bNhRt4+DggDZt2sDS8vEG9Pb2RkxMTLn74uW+qw/rk4b1ScP6KqZaL/d9+PBhLF68GOvWrUOTJsXfI9XPzw8nTpyAVquFIAg4duwYWrdubcrSiIioFJIDIi4uDp6envj++++xfPlyeHp6IjU1FQAwZ84cFBYWIjg4GP7+/vD390d2djYAICwsDAkJCQCADh06oHv37ggICMDAgQOh0+nw7rvvSi2NiIgk4B3l/ut53QV8gvVJw/qkYX3SPK/18Y5yRERUIQwIIiISxYAgIiJRDAgiIhLFgCAiIlEMCCIiEsWAICIiUQwIIiISxYAgIiJRDAgiIhLFgCAiIlEMCCIiEsWAICIiUQwIIiISxYAgIiJRDAgiIhLFgCAiIlEMCCIiEsWAICIiUQwIIiISxYAgIiJRDAgiIhIlOSBiYmKgVqvh7OyMLVu2GE2bP38+vL29MXDgQAwbNgwXL14UXcaDBw8wc+ZM+Pn5wdfXF9OnT0dubq7U0oiISALJAeHk5ITIyEj4+fkVmebp6YnY2Fjs3bsX7777LmbMmCG6jB07dqCwsBCxsbGIi4uDXq/Htm3bpJZGREQSmEldQMuWLQEAcnnRrOnVq5fhsYuLC9LT06HX64u0lclkePjwIQoLCwEA+fn5aNSokdTSiIhIAskBUVZbt25Fz549RYNk2LBhSE5OhoeHBwCgW7duUKvV5e6jQQMLSTXa2FhKmt/UWJ80rE8a1ifN816fmFIDIjAwEBqNRnRaUlISFApFqZ3s27cPsbGx2Lp1a7HLAYBjx44BAGbOnIl169Zh3LhxpS77aVlZudDrhXLN84SNjSVu3cqp0LxVgfVJw/qkYX3SPK/1yeWyEr9YlxoQu3fvllRAfHw8IiMjsXHjRjRs2FC0zfbt2+Hv749atWoBAAYMGIA9e/aUOyCIiKjymPRnrocPH8bixYuxbt06NGnSpNh2TZo0wbFjxyAIAvR6PY4ePYrXX3/dlKUREVEpJAdEXFwcPD098f3332P58uXw9PREamoqAGDOnDkoLCxEcHAw/P394e/vj+zsbABAWFgYEhISAABTpkzB/fv34efnB7VaDa1Wi0mTJkktjYiIJJAJglCxg/bPIZ6DqD6sTxrWJw3rq5jSzkHwf1ITEZEoBgQREYliQBARkSgGBBERiWJAEBGRKAYEERGJYkAQEZEoBgQREYliQBARkSgGBBERiWJAEBGRKAYEERGJYkAQEZEoBgQREYliQBARkSgGBBERiWJAEBGRKAYEERGJYkAQEZEoBgQREYliQBARkSjJARETEwO1Wg1nZ2ds2bLFaNrq1auhVqsREBAAf39/7N+/v9jl7Ny5E3379oWXlxfCw8Oh1+ullkZERBKYSV2Ak5MTIiMjER0dXWRaUFAQJk2aBADIyMiAj48PPDw8YGVlZdQuLS0NX375Jfbs2YN69ephwoQJ2Lt3LwICAqSWR0REFSR5D6Jly5ZwdHSEXF50UZaWlobH+fn5kMlkonsGBw8ehJeXF+rXrw+5XI4hQ4aUuLdBRESmJ3kPojTbtm3Dpk2bkJ6ejkWLFsHa2rpIm5s3b8LBwcHw3MHBATdv3jR1aQaXftwHs+vHIeiFKuuzvK7IZaxPAtYnDeuTxtT16Zp3xRu9fSt9uaUGRGBgIDQajei0pKQkKBSKEucfPnw4hg8fjitXriAkJARdunQRDYnK0KCBRYXmq1PbHIUAZHJZ5RZUyVifNKxPGtYnjSnrq1PbHDY2lqU3LKdSA2L37t2V0lGrVq1ga2uLkydPon///kbT7O3tjUJIo9HA3t6+3H1kZeVCX4GUbt61H2z8B+PWrZxyz1tVbGwsWZ8ErE8a1idNVdRXkeXL5bISv1ib9GeuqamphsdpaWlISUmBo6NjkXb9+/fHoUOHcOfOHej1enz77bfw8fExZWlERFQKyecg4uLisHTpUty/fx8JCQmIjo7G+vXr4ejoiKioKKSmpsLMzAwKhQLz5s1DixYtAADLly+Hra0thg8fjqZNm2Ly5MkYOnQoAMDDwwMDBw6UWhoREUkgEwTh+T2zU04VPcQEcBdVKtYnDeuThvVVTLUeYiIior8vBgQREYliQBARkSgGBBERiWJAEBGRKAYEERGJYkAQEZEoBgQREYliQBARkSgGBBERiWJAEBGRKAYEERGJYkAQEZEoBgQREYliQBARkSgGBBERiWJAEBGRKAYEERGJYkAQEZEoBgQREYliQBARkSjJARETEwO1Wg1nZ2ds2bLFaNrq1auhVqsREBAAf39/7N+/X3QZhw4dwqBBg+Dn5wdfX1+sX79eallERCSRmdQFODk5ITIyEtHR0UWmBQUFYdKkSQCAjIwM+Pj4wMPDA1ZWVkbtbGxssHr1atjZ2SEnJweDBg1Cu3bt4OrqKrU8IiKqIMkB0bJlSwCAXF50Z8TS0tLwOD8/HzKZDHq9vki79u3bG83TokUL3LhxgwFBRFSNTH4OYtu2bfD29kZgYCAWLFgAa2vrEttfu3YNycnJ6Ny5s6lLIyKiEsgEQRBKahAYGAiNRiM6LSkpCQqFAgAQGhqKNm3aICgoSLTtlStXEBISgs2bNxcbEpmZmRgxYgSmT58OHx+f8qwHERFVslIPMe3evbtSOmrVqhVsbW1x8uRJ9O/fv8j0rKwsjBkzBuPHj69wOGRl5UKvLzHvimVjY4lbt3IqNG9VYH3SsD5pWJ80z2t9crkMDRpYFD/dlJ2npqYaHqelpSElJQWOjo5F2mVnZ2PMmDF4++23MWTIEFOWREREZST5JHVcXByWLl2K+/fvIyEhAdHR0Vi/fj0cHR0RFRWF1NRUmJmZQaFQYN68eWjRogUAYPny5bC1tcXw4cMRHR2N69evY8eOHdixYwcAYOTIkRg8eLDU8oiIqIJKPQfxd8JDTNWH9UnD+qRhfRVTrYeYiIjo74sBQUREohgQREQkigFBRESiGBBERCSKAUFERKIYEEREJIoBQUREohgQREQkigFBRESiGBBERCSKAUFERKIYEEREJIoBQUREohgQREQkigFBRESiGBBERCSKAUFERKIYEEREJIoBQUREohgQREQkigFBRESiJAdETEwM1Go1nJ2dsWXLFqNpq1evhlqtRkBAAPz9/bF///4Sl1VQUABfX18MGjRIallERCSRmdQFODk5ITIyEtHR0UWmBQUFYdKkSQCAjIwM+Pj4wMPDA1ZWVqLLioyMRPv27XH58mWpZRERkUSS9yBatmwJR0dHyOVFF2VpaWl4nJ+fD5lMBr1eL7qc06dP4/r16/D395daEhERVQLJexCl2bZtGzZt2oT09HQsWrQI1tbWRdrk5+dj0aJFWL16Na5fv17hvho0sJBQKWBjY1l6o2rE+qRhfdKwPmme9/rElBoQgYGB0Gg0otOSkpKgUChKnH/48OEYPnw4rly5gpCQEHTp0qVISCxduhRvvfUW7OzsJAVEVlYu9HqhQvPa2Fji1q2cCvdtaqxPGtYnDeuT5nmtTy6XlfjFutSA2L17d6UU0qpVK9ja2uLkyZPo37+/0bQzZ84gMTERq1atQkFBAe7duwe1Wo3Y2NhK6ZuIiMrPpIeYUlNT4ejoCABIS0tDSkqK4fnTng6CEydOICIiArt27TJlaUREVArJAREXF4elS5fi/v37SEhIQHR0NNavXw9HR0dERUUhNTUVZmZmUCgUmDdvHlq0aAEAWL58OWxtbTF8+HDJK0FERJVPJghCxQ7aP4d4DqL6sD5pWJ80rK9iSjsHwf9JTUREohgQREQkigFBRESiGBBERCSKAUFERKIYEEREJIoBQUREohgQREQkigFBRESiGBBERCSKAUFERKIYEEREJIoBQUREohgQREQkigFBRESiGBBERCSKAUFERKIYEEREJIoBQUREohgQREQkigFBRESiJAdETEwM1Go1nJ2dsWXLFqNpq1evhlqtRkBAAPz9/bF///5il5OSkoK3334bAwYMwIABA3DkyBGppRERkQRmUhfg5OSEyMhIREdHF5kWFBSESZMmAQAyMjLg4+MDDw8PWFlZGbXLz8/H1KlT8fnnn8PFxQWPHj1CTk6O1NKIiEgCyQHRsmVLAIBcXnRnxNLS0vA4Pz8fMpkMer2+SLu4uDh07NgRLi4uj4syM4O1tbXU0oiISALJAVGabdu2YdOmTUhPT8eiRYtE//CnpqbCzMwMEyZMQGZmJt544w3Mnj27yJ4GERFVHZkgCEJJDQIDA6HRaESnJSUlQaFQAABCQ0PRpk0bBAUFiba9cuUKQkJCsHnz5iIhsXDhQvz444/Yvn07GjZsiMWLFyM3NxeLFy+uyDoREVElKHUPYvfu3ZXSUatWrWBra4uTJ0+if//+RtPs7e3h7u4OW1tbAIBarcbcuXPL3UdWVi70+hLzrlg2Npa4dev5Pe/B+qRhfdKwPmme1/rkchkaNLAofropO09NTTU8TktLQ0pKChwdHYu08/HxwYULF5CbmwsASExMRKtWrUxZGhERlULyOYi4uDgsXboU9+/fR0JCAqKjo7F+/Xo4OjoiKirKcH5BoVBg3rx5aNGiBQBg+fLlsLW1xfDhw+Hg4IAJEyZg2LBhkMlkaNKkCRYsWCB55YiIqOJKPQfxd8JDTNWH9UnD+qRhfRVTrYeYiIjo74sBQUREohgQREQkigFBRESiGBBERCSKAUFERKIYEEREJIoBQUREohgQREQkigFBRESiGBBERCTK5DcMqkpyuaxa5zc11icN65OG9UnzPNZXWk016mJ9RERUeXiIiYiIRDEgiIhIFAOCiIhEMSCIiEgUA4KIiEQxIIiISBQDgoiIRDEgiIhIFAOCiIhE1ahLbZTm999/R2hoKO7evYt69eohIiICr732mlEbnU6HhQsX4ujRo5DJZHjnnXcwZMgQk9eWnZ2NWbNm4c8//4RSqcSrr76K8PBw1K9f36hdaGgokpKSYG1tDQDw9vbGpEmTTF4fAPTu3RtKpRK1atUCAISEhKB79+5GbR48eIA5c+bg0qVLUCgUmD17Nnr16mXy2v766y9MmTLF8DwnJwe5ubk4efKkUbuoqCh88803sLW1BQB06NABH330kUlqioiIwMGDB3Hjxg3ExsaiZcuWAMo2DgHTj0Wx+so6DgHTj8Xitl9ZxiFg+rEoVl9ZxyFQtWOxwoQXyIgRI4Q9e/YIgiAIe/bsEUaMGFGkze7du4WxY8cKOp1OyMrKErp37y6kpaWZvLbs7Gzhl19+MTxfsmSJMGfOnCLtZs+eLXz99dcmr0dMr169hCtXrpTYJioqSggLCxMEQRB+//13oWvXrkJubm5VlGdk4cKFwvz584u8vmLFCmHJkiVVUsOpU6cEjUZTZLuVZRwKgunHolh9ZR2HgmD6sVjc9ivLOBQE04/F4up7WnHjUBCqdixW1AtziCkrKwu//vor/Pz8AAB+fn749ddfcefOHaN2+/fvx5AhQyCXy1G/fn14eXnh+++/N3l99erVg7u7u+G5i4sLNBqNyfutbAcOHMCbb74JAHjttdfQpk0bJCYmVmkNWq0WsbGxGDx4cJX2+yxXV1fY29sbvVbWcQiYfiyK1fc8jUOx+srD1GOxtPqel3EoxQsTEDdv3oSdnR0UCgUAQKFQwNbWFjdv3izSzsHBwfDc3t4e6enpVVqrXq/Htm3b0Lt3b9HpGzZsgFqtxuTJk3Ht2rUqrS0kJARqtRoff/wx7t+/X2S6RqNB48aNDc+rY/v9+OOPsLOzwxtvvCE6fd++fVCr1Rg7dizOnTtXpbWVdRw+aVudY7G0cQhU31gsbRwC1T8WSxuHQPWOxbJ4YQLi72TBggV46aWXEBQUVGTajBkzEB8fj9jYWPTr1w/jx4+HTqerkrq2bt2KvXv34rvvvoMgCAgPD6+Sfsvru+++K/Zb27Bhw5CQkIDY2FiMGzcOkydPRnZ2dhVX+PdQ0jgEqm8s1oRxCPw9xuILExD29vbIyMgwDGCdTofMzMwiu4j29vZGu9Q3b95Eo0aNqqzOiIgI/PHHH1i2bBnk8qJvj52dneH1gIAA5OfnV9m3oifbSqlU4q233sLZs2eLtHFwcMCNGzcMz6t6+2VkZODUqVNQq9Wi021sbGBubg4A8PDwgL29Pa5evVpl9ZV1HD5pW11jsbRxCFTfWCzLOASqdyyWNg6B6h+LZfHCBESDBg3g5OSEuLg4AEBcXBycnJyK/DrD29sb3377LfR6Pe7cuYNDhw6hf//+VVLjF198gf/85z9YuXIllEqlaJuMjAzD46NHj0Iul8POzs7kteXn5yMnJwcAIAgC9u/fDycnpyLtvL29sWPHDgDA9evXcfHiRdFfmJjK7t270aNHD8Mva5719PZLSUnBjRs30KxZs6oqr8zjEKi+sViWcQhUz1gs6zgEqncsljYOgeofi2XxQt0w6Nq1awgNDcX9+/fx8ssvIyIiAs2bN8eECRMQHByMtm3bQqfTITw8HD///DMAYMKECYYTXaZ09epV+Pn54bXXXkPt2rUBAE2aNMHKlSvh7++P6Oho2NnZYfTo0cjKyoJMJoOFhQVmzZoFFxcXk9eXlpaGadOmQafTQa/Xo0WLFpg3bx5sbW2N6svPz0doaChSUlIgl8vxz3/+E15eXiav74n+/fsjLCwMnp6ehteefn9nz56NS5cuQS6Xw9zcHMHBwejRo4dJalm4cCF++OEH3L59G9bW1qhXrx727dtX7Dh8tlZTj0Wx+pYtW1bsOARQpWNRrL41a9YUOw6frc/UY7G49xcQH4dA9Y3FinqhAoKIiMruhTnERERE5cOAICIiUQwIIiISxYAgIiJRDAgiIhLFgCCqZGvWrEFYWFiF5g0NDUVkZGQlV0RUMS/U5b6JqsLEiROruwSiSsE9CCIiEsWAoBdeRkYGpk2bhs6dO6N3797YvHkzgMc3dAkODsb06dOhUqkQGBiIy5cvG+aLjo5G9+7doVKp0L9/fxw/ftwwX0hIiKFdQkICfH194erqihEjRhhd9fTXX39FYGAgVCoVpk+fjoKCAqPaDh8+DH9/f7i6umLYsGFl6p+o0lTfrSiIqp9OpxMCAwOFqKgooaCgQPjzzz+F3r17C4mJicKKFSsEZ2dn4cCBA4JWqxXWrl0r9OrVS9BqtcK1a9cET09PIT09XRAEQUhLSxP++OMPQRAe3whm5syZgiAIwm+//Sa0b99eOHbsmKDVaoXo6GjBy8tLKCgoEAoKCoSePXsKGzZsELRarXDgwAHB2dlZ+OKLLwRBEIRLly4JnTt3FpKTk4VHjx4Ju3btEnr16iUUFBSU2D9RZeEeBL3QLl68iDt37mDq1KlQKpVo2rQphg4div379wMA3njjDXh7e8Pc3BxjxoyBVqvF+fPnoVAooNVqce3aNRQWFqJJkyZ45ZVXiix///796NGjBzw8PGBubo5x48bh4cOHOHfuHM6fP4/CwkKMGjUK5ubm8Pb2Rtu2bQ3z7tixA2+++Sbat28PhUKBwMBAmJubIzk5ucz9E0nBk9T0Qrtx4wYyMzPh6upqeE2n08HV1RUODg5Gl4d+crXSJ+3nzp2LqKgopKamolu3bggNDS1yNdPMzEyjm/7I5XLDJb8VCgXs7Owgk8kM059uq9FosGfPHmzZssXwWmFhITIzM9GpU6cy9U8kBfcg6IVmb2+PJk2a4PTp04Z/586dw1dffQUARvc30Ov1yMjIMFw5VK1WY9u2bTh8+DBkMhk+++yzIsu3tbU1uqeDIAiGu8rZ2NggIyMDwlPXy3y6rb29PSZOnGhU2/nz5w23Ky1L/0RSMCDohdauXTvUrVsX0dHRePjwIXQ6Hf7v//4PFy5cAABcunQJP/zwAx49eoRNmzZBqVSiffv2+O2333D8+HFotVoolUrUqlVL9MY6Pj4+OHLkCI4fP47CwkKsX78eSqUSKpUKLi4uMDMzw+bNm1FYWIgffvgBFy9eNMw7ZMgQbN++HefPn4cgCMjPz8dPP/2E3NzcMvdPJAUPMdELTaFQYM2aNYiIiECfPn2g1WrRrFkzTJ8+HQDQp08f7N+/H7Nnz8arr76KqKgomJubQ6vV4vPPP8e1a9dgbm4OlUoleuvL5s2b49NPP8WCBQuQkZEBJycnrFmzxnAjnqioKHzwwQdYtmwZevTogb59+xrmbdu2LRYsWIDw8HD88ccfqF27Njp06ABXV9cy908kBe8HQVSMqKgo/PHHHzx0Qy8s7pMSEZEoBgQREYniISYiIhLFPQgiIhLFgCAiIlEMCCIiEsWAICIiUQwIIiISxYAgIiJR/w9MhqhgxjIp/gAAAABJRU5ErkJggg==", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "cfg = QlearningConfig()\n", - "plot_cfg = PlotConfig()\n", - "# 训练\n", - "env, agent = env_agent_config(cfg, seed=1)\n", - "rewards, ma_rewards = train(cfg, env, agent)\n", - "make_dir(plot_cfg.result_path, plot_cfg.model_path) # 创建保存结果和模型路径的文件夹\n", - "agent.save(path=plot_cfg.model_path) # 保存模型\n", - "save_results(rewards, ma_rewards, tag='train',\n", - " path=plot_cfg.result_path) # 保存结果\n", - "plot_rewards(rewards, ma_rewards, plot_cfg, tag=\"train\") # 画出结果\n", - "# 测试\n", - "env, agent = env_agent_config(cfg, seed=10)\n", - "agent.load(path=plot_cfg.model_path) # 导入模型\n", - "rewards, ma_rewards = test(cfg, env, agent)\n", - "save_results(rewards, ma_rewards, tag='test', path=plot_cfg.result_path) # 保存结果\n", - "plot_rewards(rewards, ma_rewards, plot_cfg, tag=\"test\") # 画出结果" - ] - } - ], - "metadata": { - "interpreter": { - "hash": "fbea1422c2cf61ed9c0cfc03f38f71cc9083cc288606edc4170b5309b352ce27" - }, - "kernelspec": { - "display_name": "Python 3.7.11 64-bit ('py37': conda)", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.10" - }, - "orig_nbformat": 2 - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/codes/QLearning/task0.py b/codes/QLearning/task0.py index 59a1668..607cefa 100644 --- a/codes/QLearning/task0.py +++ b/codes/QLearning/task0.py @@ -5,7 +5,7 @@ Author: John Email: johnjim0816@gmail.com Date: 2020-09-11 23:03:00 LastEditor: John -LastEditTime: 2021-12-22 11:13:23 +LastEditTime: 2022-02-10 00:54:02 Discription: Environment: ''' @@ -19,42 +19,93 @@ import gym import torch import datetime -from envs.gridworld_env import CliffWalkingWapper -from QLearning.agent import QLearning -from QLearning.train import train,test -from common.utils import plot_rewards,plot_rewards_cn +from env.gridworld_env import CliffWalkingWapper +from qlearning import QLearning +from common.utils import plot_rewards from common.utils import save_results,make_dir curr_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") # 获取当前时间 -algo_name = 'Q-learning' # 算法名称 -env_name = 'CliffWalking-v0' # 环境名称 -device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # 检测GPU -class QlearningConfig: - '''训练相关参数''' - def __init__(self): - self.algo_name = algo_name # 算法名称 - self.env_name = env_name # 环境名称 - self.device = device # 检测GPU - self.train_eps = 400 # 训练的回合数 - self.test_eps = 30 # 测试的回合数 - self.gamma = 0.9 # reward的衰减率 - self.epsilon_start = 0.95 # e-greedy策略中初始epsilon - self.epsilon_end = 0.01 # e-greedy策略中的终止epsilon - self.epsilon_decay = 300 # e-greedy策略中epsilon的衰减率 - self.lr = 0.1 # 学习率 -class PlotConfig: - ''' 绘图相关参数设置 +class Config: + '''超参数 ''' - def __init__(self) -> None: - self.algo_name = algo_name # 算法名称 - self.env_name = env_name # 环境名称 - self.device = device # 检测GPU + def __init__(self): + ################################## 环境超参数 ################################### + self.algo_name = 'Q-learning' # 算法名称 + self.env_name = 'CliffWalking-v0' # 环境名称 + self.device = torch.device( + "cuda" if torch.cuda.is_available() else "cpu") # 检测GPUgjgjlkhfsf风刀霜的撒发十 + self.seed = 10 # 随机种子，置0则不设置随机种子 + self.train_eps = 400 # 训练的回合数 + self.test_eps = 30 # 测试的回合数 + ################################################################################ + + ################################## 算法超参数 ################################### + self.gamma = 0.90 # 强化学习中的折扣因子 + self.epsilon_start = 0.95 # e-greedy策略中初始epsilon + self.epsilon_end = 0.01 # e-greedy策略中的终止epsilon + self.epsilon_decay = 300 # e-greedy策略中epsilon的衰减率 + self.lr = 0.1 # 学习率 + ################################################################################ + + ################################# 保存结果相关参数 ################################ self.result_path = curr_path + "/outputs/" + self.env_name + \ '/' + curr_time + '/results/' # 保存结果的路径 self.model_path = curr_path + "/outputs/" + self.env_name + \ '/' + curr_time + '/models/' # 保存模型的路径 - self.save = True # 是否保存图片 + self.save = True # 是否保存图片 + ################################################################################ + +def train(cfg,env,agent): + print('开始训练！') + print(f'环境:{cfg.env_name}, 算法:{cfg.algo_name}, 设备:{cfg.device}') + rewards = [] # 记录奖励 + ma_rewards = [] # 记录滑动平均奖励 + for i_ep in range(cfg.train_eps): + ep_reward = 0 # 记录每个回合的奖励 + state = env.reset() # 重置环境,即开始新的回合 + while True: + action = agent.choose_action(state) # 根据算法选择一个动作 + next_state, reward, done, _ = env.step(action) # 与环境进行一次动作交互 + agent.update(state, action, reward, next_state, done) # Q学习算法更新 + state = next_state # 更新状态 + ep_reward += reward + if done: + break + rewards.append(ep_reward) + if ma_rewards: + ma_rewards.append(ma_rewards[-1]*0.9+ep_reward*0.1) + else: + ma_rewards.append(ep_reward) + print("回合数：{}/{}，奖励{:.1f}".format(i_ep+1, cfg.train_eps,ep_reward)) + print('完成训练！') + return rewards,ma_rewards + +def test(cfg,env,agent): + print('开始测试！') + print(f'环境：{cfg.env_name}, 算法：{cfg.algo_name}, 设备：{cfg.device}') + for item in agent.Q_table.items(): + print(item) + rewards = [] # 记录所有回合的奖励 + ma_rewards = [] # 滑动平均的奖励 + for i_ep in range(cfg.test_eps): + ep_reward = 0 # 记录每个episode的reward + state = env.reset() # 重置环境, 重新开一局（即开始新的一个回合） + while True: + action = agent.predict(state) # 根据算法选择一个动作 + next_state, reward, done, _ = env.step(action) # 与环境进行一个交互 + state = next_state # 更新状态 + ep_reward += reward + if done: + break + rewards.append(ep_reward) + if ma_rewards: + ma_rewards.append(ma_rewards[-1]*0.9+ep_reward*0.1) + else: + ma_rewards.append(ep_reward) + print(f"回合数：{i_ep+1}/{cfg.test_eps}, 奖励：{ep_reward:.1f}") + print('完成测试！') + return rewards,ma_rewards def env_agent_config(cfg,seed=1): '''创建环境和智能体 @@ -68,26 +119,25 @@ def env_agent_config(cfg,seed=1): env = gym.make(cfg.env_name) env = CliffWalkingWapper(env) env.seed(seed) # 设置随机种子 - state_dim = env.observation_space.n # 状态维度 - action_dim = env.action_space.n # 动作维度 - agent = QLearning(state_dim,action_dim,cfg) + n_states = env.observation_space.n # 状态维度 + n_actions = env.action_space.n # 动作维度 + agent = QLearning(n_states,n_actions,cfg) return env,agent - -cfg = QlearningConfig() -plot_cfg = PlotConfig() -# 训练 -env, agent = env_agent_config(cfg, seed=1) -rewards, ma_rewards = train(cfg, env, agent) -make_dir(plot_cfg.result_path, plot_cfg.model_path) # 创建保存结果和模型路径的文件夹 -agent.save(path=plot_cfg.model_path) # 保存模型 -save_results(rewards, ma_rewards, tag='train', - path=plot_cfg.result_path) # 保存结果 -plot_rewards(rewards, ma_rewards, plot_cfg, tag="train") # 画出结果 -# 测试 -env, agent = env_agent_config(cfg, seed=10) -agent.load(path=plot_cfg.model_path) # 导入模型 -rewards, ma_rewards = test(cfg, env, agent) -save_results(rewards, ma_rewards, tag='test', path=plot_cfg.result_path) # 保存结果 -plot_rewards(rewards, ma_rewards, plot_cfg, tag="test") # 画出结果 +if __name__ == "__main__": + cfg = Config() + # 训练 + env, agent = env_agent_config(cfg, seed=1) + rewards, ma_rewards = train(cfg, env, agent) + make_dir(cfg.result_path, cfg.model_path) # 创建保存结果和模型路径的文件夹 + agent.save(path=cfg.model_path) # 保存模型 + save_results(rewards, ma_rewards, tag='train', + path=cfg.result_path) # 保存结果 + plot_rewards(rewards, ma_rewards, cfg, tag="train") # 画出结果 + # 测试 + env, agent = env_agent_config(cfg, seed=10) + agent.load(path=cfg.model_path) # 导入模型 + rewards, ma_rewards = test(cfg, env, agent) + save_results(rewards, ma_rewards, tag='test', path=cfg.result_path) # 保存结果 + plot_rewards(rewards, ma_rewards, cfg, tag="test") # 画出结果 diff --git a/codes/QLearning/train.py b/codes/QLearning/train.py deleted file mode 100644 index 2c4aa09..0000000 --- a/codes/QLearning/train.py +++ /dev/null @@ -1,50 +0,0 @@ -def train(cfg,env,agent): - print('开始训练！') - print(f'环境:{cfg.env_name}, 算法:{cfg.algo_name}, 设备:{cfg.device}') - rewards = [] # 记录奖励 - ma_rewards = [] # 记录滑动平均奖励 - for i_ep in range(cfg.train_eps): - ep_reward = 0 # 记录每个回合的奖励 - state = env.reset() # 重置环境,即开始新的回合 - while True: - action = agent.choose_action(state) # 根据算法选择一个动作 - next_state, reward, done, _ = env.step(action) # 与环境进行一次动作交互 - agent.update(state, action, reward, next_state, done) # Q学习算法更新 - state = next_state # 更新状态 - ep_reward += reward - if done: - break - rewards.append(ep_reward) - if ma_rewards: - ma_rewards.append(ma_rewards[-1]*0.9+ep_reward*0.1) - else: - ma_rewards.append(ep_reward) - print("回合数：{}/{}，奖励{:.1f}".format(i_ep+1, cfg.train_eps,ep_reward)) - print('完成训练！') - return rewards,ma_rewards - -def test(cfg,env,agent): - print('开始测试！') - print(f'环境：{cfg.env_name}, 算法：{cfg.algo_name}, 设备：{cfg.device}') - for item in agent.Q_table.items(): - print(item) - rewards = [] # 记录所有回合的奖励 - ma_rewards = [] # 滑动平均的奖励 - for i_ep in range(cfg.test_eps): - ep_reward = 0 # 记录每个episode的reward - state = env.reset() # 重置环境, 重新开一局（即开始新的一个回合） - while True: - action = agent.predict(state) # 根据算法选择一个动作 - next_state, reward, done, _ = env.step(action) # 与环境进行一个交互 - state = next_state # 更新状态 - ep_reward += reward - if done: - break - rewards.append(ep_reward) - if ma_rewards: - ma_rewards.append(ma_rewards[-1]*0.9+ep_reward*0.1) - else: - ma_rewards.append(ep_reward) - print(f"回合数：{i_ep+1}/{cfg.test_eps}, 奖励：{ep_reward:.1f}") - print('完成测试！') - return rewards,ma_rewards \ No newline at end of file diff --git a/codes/README.md b/codes/README.md index 3896fbb..18ebd7c 100644 --- a/codes/README.md +++ b/codes/README.md @@ -1,3 +1,4 @@ +中文｜[English](./README_en.md) ## 写在前面 本项目用于学习RL基础算法，尽量做到: **注释详细**，**结构清晰**。 diff --git a/codes/README_en.md b/codes/README_en.md new file mode 100644 index 0000000..430fca9 --- /dev/null +++ b/codes/README_en.md @@ -0,0 +1 @@ +English｜[中文](./README.md) \ No newline at end of file diff --git a/codes/RainbowDQN/rainbow_dqn.py b/codes/RainbowDQN/rainbow_dqn.py new file mode 100644 index 0000000..0d7f783 --- /dev/null +++ b/codes/RainbowDQN/rainbow_dqn.py @@ -0,0 +1,215 @@ +import math +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.optim as optim +from torch.autograd import Variable +import random +class ReplayBuffer: + def __init__(self, capacity): + self.capacity = capacity # 经验回放的容量 + self.buffer = [] # 缓冲区 + self.position = 0 + + def push(self, state, action, reward, next_state, done): + ''' 缓冲区是一个队列，容量超出时去掉开始存入的转移(transition) + ''' + if len(self.buffer) < self.capacity: + self.buffer.append(None) + self.buffer[self.position] = (state, action, reward, next_state, done) + self.position = (self.position + 1) % self.capacity + + def sample(self, batch_size): + batch = random.sample(self.buffer, batch_size) # 随机采出小批量转移 + state, action, reward, next_state, done = zip(*batch) # 解压成状态，动作等 + return state, action, reward, next_state, done + + def __len__(self): + ''' 返回当前存储的量 + ''' + return len(self.buffer) +class NoisyLinear(nn.Module): + def __init__(self, input_dim, output_dim, device, std_init=0.4): + super(NoisyLinear, self).__init__() + + self.device = device + self.input_dim = input_dim + self.output_dim = output_dim + self.std_init = std_init + + self.weight_mu = nn.Parameter(torch.FloatTensor(output_dim, input_dim)) + self.weight_sigma = nn.Parameter(torch.FloatTensor(output_dim, input_dim)) + self.register_buffer('weight_epsilon', torch.FloatTensor(output_dim, input_dim)) + + self.bias_mu = nn.Parameter(torch.FloatTensor(output_dim)) + self.bias_sigma = nn.Parameter(torch.FloatTensor(output_dim)) + self.register_buffer('bias_epsilon', torch.FloatTensor(output_dim)) + + self.reset_parameters() + self.reset_noise() + + def forward(self, x): + if self.device: + weight_epsilon = self.weight_epsilon.cuda() + bias_epsilon = self.bias_epsilon.cuda() + else: + weight_epsilon = self.weight_epsilon + bias_epsilon = self.bias_epsilon + + if self.training: + weight = self.weight_mu + self.weight_sigma.mul(Variable(weight_epsilon)) + bias = self.bias_mu + self.bias_sigma.mul(Variable(bias_epsilon)) + else: + weight = self.weight_mu + bias = self.bias_mu + + return F.linear(x, weight, bias) + + def reset_parameters(self): + mu_range = 1 / math.sqrt(self.weight_mu.size(1)) + + self.weight_mu.data.uniform_(-mu_range, mu_range) + self.weight_sigma.data.fill_(self.std_init / math.sqrt(self.weight_sigma.size(1))) + + self.bias_mu.data.uniform_(-mu_range, mu_range) + self.bias_sigma.data.fill_(self.std_init / math.sqrt(self.bias_sigma.size(0))) + + def reset_noise(self): + epsilon_in = self._scale_noise(self.input_dim) + epsilon_out = self._scale_noise(self.output_dim) + + self.weight_epsilon.copy_(epsilon_out.ger(epsilon_in)) + self.bias_epsilon.copy_(self._scale_noise(self.output_dim)) + + def _scale_noise(self, size): + x = torch.randn(size) + x = x.sign().mul(x.abs().sqrt()) + return x + +class RainbowModel(nn.Module): + def __init__(self, n_states, n_actions, n_atoms, Vmin, Vmax): + super(RainbowModel, self).__init__() + + self.n_states = n_states + self.n_actions = n_actions + self.n_atoms = n_atoms + self.Vmin = Vmin + self.Vmax = Vmax + + self.linear1 = nn.Linear(n_states, 32) + self.linear2 = nn.Linear(32, 64) + + self.noisy_value1 = NoisyLinear(64, 64, device=device) + self.noisy_value2 = NoisyLinear(64, self.n_atoms, device=device) + + self.noisy_advantage1 = NoisyLinear(64, 64, device=device) + self.noisy_advantage2 = NoisyLinear(64, self.n_atoms * self.n_actions, device=device) + + def forward(self, x): + batch_size = x.size(0) + + x = F.relu(self.linear1(x)) + x = F.relu(self.linear2(x)) + + value = F.relu(self.noisy_value1(x)) + value = self.noisy_value2(value) + + advantage = F.relu(self.noisy_advantage1(x)) + advantage = self.noisy_advantage2(advantage) + + value = value.view(batch_size, 1, self.n_atoms) + advantage = advantage.view(batch_size, self.n_actions, self.n_atoms) + + x = value + advantage - advantage.mean(1, keepdim=True) + x = F.softmax(x.view(-1, self.n_atoms)).view(-1, self.n_actions, self.n_atoms) + + return x + + def reset_noise(self): + self.noisy_value1.reset_noise() + self.noisy_value2.reset_noise() + self.noisy_advantage1.reset_noise() + self.noisy_advantage2.reset_noise() + + def act(self, state): + state = Variable(torch.FloatTensor(state).unsqueeze(0), volatile=True) + dist = self.forward(state).data.cpu() + dist = dist * torch.linspace(self.Vmin, self.Vmax, self.n_atoms) + action = dist.sum(2).max(1)[1].numpy()[0] + return action + +class RainbowDQN(nn.Module): + def __init__(self, n_states, n_actions, n_atoms, Vmin, Vmax,cfg): + super(RainbowDQN, self).__init__() + self.n_states = n_states + self.n_actions = n_actions + self.n_atoms = cfg.n_atoms + self.Vmin = cfg.Vmin + self.Vmax = cfg.Vmax + self.policy_model = RainbowModel(n_states, n_actions, n_atoms, Vmin, Vmax) + self.target_model = RainbowModel(n_states, n_actions, n_atoms, Vmin, Vmax) + self.batch_size = cfg.batch_size + self.memory = ReplayBuffer(cfg.memory_capacity) # 经验回放 + self.optimizer = optim.Adam(self.policy_model.parameters(), 0.001) + def choose_action(self,state): + state = Variable(torch.FloatTensor(state).unsqueeze(0), volatile=True) + dist = self.policy_model(state).data.cpu() + dist = dist * torch.linspace(self.Vmin, self.Vmax, self.n_atoms) + action = dist.sum(2).max(1)[1].numpy()[0] + return action + def projection_distribution(self,next_state, rewards, dones): + + + delta_z = float(self.Vmax - self.Vmin) / (self.n_atoms - 1) + support = torch.linspace(self.Vmin, self.Vmax, self.n_atoms) + + next_dist = self.target_model(next_state).data.cpu() * support + next_action = next_dist.sum(2).max(1)[1] + next_action = next_action.unsqueeze(1).unsqueeze(1).expand(next_dist.size(0), 1, next_dist.size(2)) + next_dist = next_dist.gather(1, next_action).squeeze(1) + + rewards = rewards.unsqueeze(1).expand_as(next_dist) + dones = dones.unsqueeze(1).expand_as(next_dist) + support = support.unsqueeze(0).expand_as(next_dist) + + Tz = rewards + (1 - dones) * 0.99 * support + Tz = Tz.clamp(min=self.Vmin, max=self.Vmax) + b = (Tz - self.Vmin) / delta_z + l = b.floor().long() + u = b.ceil().long() + + offset = torch.linspace(0, (self.batch_size - 1) * self.n_atoms, self.batch_size).long()\ + .unsqueeze(1).expand(self.batch_size, self.n_atoms) + + proj_dist = torch.zeros(next_dist.size()) + proj_dist.view(-1).index_add_(0, (l + offset).view(-1), (next_dist * (u.float() - b)).view(-1)) + proj_dist.view(-1).index_add_(0, (u + offset).view(-1), (next_dist * (b - l.float())).view(-1)) + + return proj_dist + def update(self): + if len(self.memory) < self.batch_size: # 当memory中不满足一个批量时，不更新策略 + return + state, action, reward, next_state, done = self.memory.sample(self.batch_size) + + state = Variable(torch.FloatTensor(np.float32(state))) + next_state = Variable(torch.FloatTensor(np.float32(next_state)), volatile=True) + action = Variable(torch.LongTensor(action)) + reward = torch.FloatTensor(reward) + done = torch.FloatTensor(np.float32(done)) + + proj_dist = self.projection_distribution(next_state, reward, done) + + dist = self.policy_model(state) + action = action.unsqueeze(1).unsqueeze(1).expand(self.batch_size, 1, self.n_atoms) + dist = dist.gather(1, action).squeeze(1) + dist.data.clamp_(0.01, 0.99) + loss = -(Variable(proj_dist) * dist.log()).sum(1) + loss = loss.mean() + + self.optimizer.zero_grad() + loss.backward() + self.optimizer.step() + + self.policy_model.reset_noise() + self.target_model.reset_noise() + \ No newline at end of file diff --git a/codes/RainbowDQN/task0.py b/codes/RainbowDQN/task0.py new file mode 100644 index 0000000..49a97a4 --- /dev/null +++ b/codes/RainbowDQN/task0.py @@ -0,0 +1,177 @@ +import sys +import os +import torch.nn as nn +import torch.nn.functional as F +curr_path = os.path.dirname(os.path.abspath(__file__)) # 当前文件所在绝对路径 +parent_path = os.path.dirname(curr_path) # 父路径 +sys.path.append(parent_path) # 添加路径到系统路径 + +import gym +import torch +import datetime +import numpy as np +from common.utils import save_results_1, make_dir +from common.utils import plot_rewards +from dqn import DQN + +curr_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") # 获取当前时间 + +class MLP(nn.Module): + def __init__(self, n_states,n_actions,hidden_dim=128): + """ 初始化q网络，为全连接网络 + n_states: 输入的特征数即环境的状态维度 + n_actions: 输出的动作维度 + """ + super(MLP, self).__init__() + self.fc1 = nn.Linear(n_states, hidden_dim) # 输入层 + self.fc2 = nn.Linear(hidden_dim,hidden_dim) # 隐藏层 + self.fc3 = nn.Linear(hidden_dim, n_actions) # 输出层 + + def forward(self, x): + # 各层对应的激活函数 + x = F.relu(self.fc1(x)) + x = F.relu(self.fc2(x)) + return self.fc3(x) + +class Config: + '''超参数 + ''' + + def __init__(self): + ############################### hyperparameters ################################ + self.algo_name = 'DQN' # algorithm name + self.env_name = 'CartPole-v0' # environment name + self.device = torch.device( + "cuda" if torch.cuda.is_available() else "cpu") # check GPU + self.seed = 10 # 随机种子，置0则不设置随机种子 + self.train_eps = 200 # 训练的回合数 + self.test_eps = 20 # 测试的回合数 + ################################################################################ + + ################################## 算法超参数 ################################### + self.gamma = 0.95 # 强化学习中的折扣因子 + self.epsilon_start = 0.90 # e-greedy策略中初始epsilon + self.epsilon_end = 0.01 # e-greedy策略中的终止epsilon + self.epsilon_decay = 500 # e-greedy策略中epsilon的衰减率 + self.lr = 0.0001 # 学习率 + self.memory_capacity = 100000 # 经验回放的容量 + self.batch_size = 64 # mini-batch SGD中的批量大小 + self.target_update = 4 # 目标网络的更新频率 + self.hidden_dim = 256 # 网络隐藏层 + ################################################################################ + + ################################# 保存结果相关参数 ################################ + self.result_path = curr_path + "/outputs/" + self.env_name + \ + '/' + curr_time + '/results/' # 保存结果的路径 + self.model_path = curr_path + "/outputs/" + self.env_name + \ + '/' + curr_time + '/models/' # 保存模型的路径 + self.save = True # 是否保存图片 + ################################################################################ + + +def env_agent_config(cfg): + ''' 创建环境和智能体 + ''' + env = gym.make(cfg.env_name) # 创建环境 + n_states = env.observation_space.shape[0] # 状态维度 + n_actions = env.action_space.n # 动作维度 + print(f"n states: {n_states}, n actions: {n_actions}") + model = MLP(n_states,n_actions) + agent = DQN(n_actions, model, cfg) # 创建智能体 + if cfg.seed !=0: # 设置随机种子 + torch.manual_seed(cfg.seed) + env.seed(cfg.seed) + np.random.seed(cfg.seed) + return env, agent + + +def train(cfg, env, agent): + ''' 训练 + ''' + print('开始训练!') + print(f'环境：{cfg.env_name}, 算法：{cfg.algo_name}, 设备：{cfg.device}') + rewards = [] # 记录所有回合的奖励 + ma_rewards = [] # 记录所有回合的滑动平均奖励 + steps = [] + for i_ep in range(cfg.train_eps): + ep_reward = 0 # 记录一回合内的奖励 + ep_step = 0 + state = env.reset() # 重置环境，返回初始状态 + while True: + ep_step += 1 + action = agent.choose_action(state) # 选择动作 + next_state, reward, done, _ = env.step(action) # 更新环境，返回transition + agent.memory.push(state, action, reward, + next_state, done) # 保存transition + state = next_state # 更新下一个状态 + agent.update() # 更新智能体 + ep_reward += reward # 累加奖励 + if done: + break + if (i_ep + 1) % cfg.target_update == 0: # 智能体目标网络更新 + agent.target_net.load_state_dict(agent.policy_net.state_dict()) + steps.append(ep_step) + rewards.append(ep_reward) + if ma_rewards: + ma_rewards.append(0.9 * ma_rewards[-1] + 0.1 * ep_reward) + else: + ma_rewards.append(ep_reward) + if (i_ep + 1) % 1 == 0: + print(f'Episode：{i_ep+1}/{cfg.test_eps}, Reward:{ep_reward:.2f}, Step:{ep_step:.2f} Epislon:{agent.epsilon(agent.frame_idx):.3f}') + print('Finish training!') + env.close() + res_dic = {'rewards':rewards,'ma_rewards':ma_rewards,'steps':steps} + return res_dic + + +def test(cfg, env, agent): + print('开始测试!') + print(f'环境：{cfg.env_name}, 算法：{cfg.algo_name}, 设备：{cfg.device}') + ############# 由于测试不需要使用epsilon-greedy策略，所以相应的值设置为0 ############### + cfg.epsilon_start = 0.0 # e-greedy策略中初始epsilon + cfg.epsilon_end = 0.0 # e-greedy策略中的终止epsilon + ################################################################################ + rewards = [] # 记录所有回合的奖励 + ma_rewards = [] # 记录所有回合的滑动平均奖励 + steps = [] + for i_ep in range(cfg.test_eps): + ep_reward = 0 # 记录一回合内的奖励 + ep_step = 0 + state = env.reset() # 重置环境，返回初始状态 + while True: + ep_step+=1 + action = agent.choose_action(state) # 选择动作 + next_state, reward, done, _ = env.step(action) # 更新环境，返回transition + state = next_state # 更新下一个状态 + ep_reward += reward # 累加奖励 + if done: + break + steps.append(ep_step) + rewards.append(ep_reward) + if ma_rewards: + ma_rewards.append(ma_rewards[-1] * 0.9 + ep_reward * 0.1) + else: + ma_rewards.append(ep_reward) + print(f'Episode：{i_ep+1}/{cfg.train_eps}, Reward:{ep_reward:.2f}, Step:{ep_step:.2f}') + print('完成测试！') + env.close() + return {'rewards':rewards,'ma_rewards':ma_rewards,'steps':steps} + + +if __name__ == "__main__": + cfg = Config() + # 训练 + env, agent = env_agent_config(cfg) + res_dic = train(cfg, env, agent) + make_dir(cfg.result_path, cfg.model_path) # 创建保存结果和模型路径的文件夹 + agent.save(path=cfg.model_path) # 保存模型 + save_results_1(res_dic, tag='train', + path=cfg.result_path) # 保存结果 + plot_rewards(res_dic['rewards'], res_dic['ma_rewards'], cfg, tag="train") # 画出结果 + # 测试 + env, agent = env_agent_config(cfg) + agent.load(path=cfg.model_path) # 导入模型 + res_dic = test(cfg, env, agent) + save_results_1(res_dic, tag='test', + path=cfg.result_path) # 保存结果 + plot_rewards(res_dic['rewards'], res_dic['ma_rewards'],cfg, tag="test") # 画出结果 diff --git a/codes/Sarsa/outputs/CliffWalking-v0/20220424-221748/models/sarsa_model.pkl b/codes/Sarsa/outputs/CliffWalking-v0/20220424-221748/models/sarsa_model.pkl deleted file mode 100644 index 9973dc4..0000000 Binary files a/codes/Sarsa/outputs/CliffWalking-v0/20220424-221748/models/sarsa_model.pkl and /dev/null differ diff --git a/codes/Sarsa/outputs/CliffWalking-v0/20220424-221748/results/test_ma_rewards.npy b/codes/Sarsa/outputs/CliffWalking-v0/20220424-221748/results/test_ma_rewards.npy deleted file mode 100644 index a77a41f..0000000 Binary files a/codes/Sarsa/outputs/CliffWalking-v0/20220424-221748/results/test_ma_rewards.npy and /dev/null differ diff --git a/codes/Sarsa/outputs/CliffWalking-v0/20220424-221748/results/test_rewards_curve.png b/codes/Sarsa/outputs/CliffWalking-v0/20220424-221748/results/test_rewards_curve.png deleted file mode 100644 index 373d627..0000000 Binary files a/codes/Sarsa/outputs/CliffWalking-v0/20220424-221748/results/test_rewards_curve.png and /dev/null differ diff --git a/codes/Sarsa/outputs/CliffWalking-v0/20220424-221748/results/train_ma_rewards.npy b/codes/Sarsa/outputs/CliffWalking-v0/20220424-221748/results/train_ma_rewards.npy deleted file mode 100644 index 1aa168b..0000000 Binary files a/codes/Sarsa/outputs/CliffWalking-v0/20220424-221748/results/train_ma_rewards.npy and /dev/null differ diff --git a/codes/Sarsa/outputs/CliffWalking-v0/20220424-221748/results/train_rewards.npy b/codes/Sarsa/outputs/CliffWalking-v0/20220424-221748/results/train_rewards.npy deleted file mode 100644 index 9924002..0000000 Binary files a/codes/Sarsa/outputs/CliffWalking-v0/20220424-221748/results/train_rewards.npy and /dev/null differ diff --git a/codes/Sarsa/outputs/CliffWalking-v0/20220424-221748/results/train_rewards_curve.png b/codes/Sarsa/outputs/CliffWalking-v0/20220424-221748/results/train_rewards_curve.png deleted file mode 100644 index ad53c8d..0000000 Binary files a/codes/Sarsa/outputs/CliffWalking-v0/20220424-221748/results/train_rewards_curve.png and /dev/null differ diff --git a/codes/Sarsa/outputs/CliffWalking-v0/20220429-202317/models/sarsa_model.pkl b/codes/Sarsa/outputs/CliffWalking-v0/20220429-202317/models/sarsa_model.pkl new file mode 100644 index 0000000..71c5339 Binary files /dev/null and b/codes/Sarsa/outputs/CliffWalking-v0/20220429-202317/models/sarsa_model.pkl differ diff --git a/codes/Sarsa/outputs/CliffWalking-v0/20220429-202317/results/test_ma_rewards.npy b/codes/Sarsa/outputs/CliffWalking-v0/20220429-202317/results/test_ma_rewards.npy new file mode 100644 index 0000000..980eabe Binary files /dev/null and b/codes/Sarsa/outputs/CliffWalking-v0/20220429-202317/results/test_ma_rewards.npy differ diff --git a/codes/Sarsa/outputs/CliffWalking-v0/20220429-202317/results/test_rewards.npy b/codes/Sarsa/outputs/CliffWalking-v0/20220429-202317/results/test_rewards.npy new file mode 100644 index 0000000..5c08614 Binary files /dev/null and b/codes/Sarsa/outputs/CliffWalking-v0/20220429-202317/results/test_rewards.npy differ diff --git a/codes/Sarsa/outputs/CliffWalking-v0/20220429-202317/results/test_rewards_curve.png b/codes/Sarsa/outputs/CliffWalking-v0/20220429-202317/results/test_rewards_curve.png new file mode 100644 index 0000000..b53212b Binary files /dev/null and b/codes/Sarsa/outputs/CliffWalking-v0/20220429-202317/results/test_rewards_curve.png differ diff --git a/codes/Sarsa/outputs/CliffWalking-v0/20220429-202317/results/train_ma_rewards.npy b/codes/Sarsa/outputs/CliffWalking-v0/20220429-202317/results/train_ma_rewards.npy new file mode 100644 index 0000000..d12b47a Binary files /dev/null and b/codes/Sarsa/outputs/CliffWalking-v0/20220429-202317/results/train_ma_rewards.npy differ diff --git a/codes/Sarsa/outputs/CliffWalking-v0/20220429-202317/results/train_rewards.npy b/codes/Sarsa/outputs/CliffWalking-v0/20220429-202317/results/train_rewards.npy new file mode 100644 index 0000000..5da3ce1 Binary files /dev/null and b/codes/Sarsa/outputs/CliffWalking-v0/20220429-202317/results/train_rewards.npy differ diff --git a/codes/Sarsa/outputs/CliffWalking-v0/20220429-202317/results/train_rewards_curve.png b/codes/Sarsa/outputs/CliffWalking-v0/20220429-202317/results/train_rewards_curve.png new file mode 100644 index 0000000..d18775f Binary files /dev/null and b/codes/Sarsa/outputs/CliffWalking-v0/20220429-202317/results/train_rewards_curve.png differ diff --git a/codes/Sarsa/sarsa.py b/codes/Sarsa/sarsa.py index 4ed885b..477ab14 100644 --- a/codes/Sarsa/sarsa.py +++ b/codes/Sarsa/sarsa.py @@ -5,7 +5,7 @@ Author: John Email: johnjim0816@gmail.com Date: 2021-03-12 16:58:16 LastEditor: John -LastEditTime: 2022-04-24 21:14:23 +LastEditTime: 2022-04-29 20:12:57 Discription: Environment: ''' @@ -16,15 +16,14 @@ import math class Sarsa(object): def __init__(self, n_actions,cfg,): - self.n_actions = n_actions # number of actions - self.lr = cfg.lr # learning rate + self.n_actions = n_actions + self.lr = cfg.lr self.gamma = cfg.gamma self.sample_count = 0 self.epsilon_start = cfg.epsilon_start self.epsilon_end = cfg.epsilon_end self.epsilon_decay = cfg.epsilon_decay - self.Q = defaultdict(lambda: np.zeros(n_actions)) - # self.Q = np.zeros((state_dim, n_actions)) # Q表 + self.Q = defaultdict(lambda: np.zeros(n_actions)) # Q table def choose_action(self, state): self.sample_count += 1 self.epsilon = self.epsilon_end + (self.epsilon_start - self.epsilon_end) * \ diff --git a/codes/Sarsa/task0.py b/codes/Sarsa/task0.py index a4c7335..d60969f 100644 --- a/codes/Sarsa/task0.py +++ b/codes/Sarsa/task0.py @@ -5,7 +5,7 @@ Author: John Email: johnjim0816@gmail.com Date: 2021-03-11 17:59:16 LastEditor: John -LastEditTime: 2022-04-24 23:03:51 +LastEditTime: 2022-04-29 20:18:13 Discription: Environment: ''' @@ -31,20 +31,20 @@ class Config: self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # check GPU self.result_path = curr_path+"/outputs/" +self.env_name+'/'+curr_time+'/results/' # path to save results self.model_path = curr_path+"/outputs/" +self.env_name+'/'+curr_time+'/models/' # path to save models - self.train_eps = 300 - self.test_eps = 20 + self.train_eps = 300 # training episodes + self.test_eps = 20 # testing episodes + self.n_steps = 200 # maximum steps per episode self.epsilon_start = 0.90 # start value of epsilon self.epsilon_end = 0.01 # end value of epsilon self.epsilon_decay = 200 # decay rate of epsilon self.gamma = 0.99 # gamma: Gamma discount factor. - self.lr = 0.2 # learning rate: step size parameter - self.n_steps = 200 + self.lr = 0.2 # learning rate: step size parameter self.save = True # if save figures def env_agent_config(cfg,seed=1): env = RacetrackEnv() - action_dim = 9 - agent = Sarsa(action_dim,cfg) + n_states = 9 # number of actions + agent = Sarsa(n_states,cfg) return env,agent def train(cfg,env,agent): @@ -73,7 +73,7 @@ def train(cfg,env,agent): print(f"Episode:{i_ep+1}, Reward:{ep_reward}, Epsilon:{agent.epsilon}") return rewards,ma_rewards -def eval(cfg,env,agent): +def test(cfg,env,agent): rewards = [] ma_rewards = [] for i_ep in range(cfg.test_eps): @@ -97,7 +97,7 @@ def eval(cfg,env,agent): rewards.append(ep_reward) if (i_ep+1)%1==0: print("Episode:{}/{}: Reward:{}".format(i_ep+1, cfg.test_eps,ep_reward)) - print('Complete evaling！') + print('Complete testing！') return rewards,ma_rewards if __name__ == "__main__": @@ -111,7 +111,7 @@ if __name__ == "__main__": env,agent = env_agent_config(cfg,seed=10) agent.load(path=cfg.model_path) - rewards,ma_rewards = eval(cfg,env,agent) + rewards,ma_rewards = test(cfg,env,agent) save_results(rewards,ma_rewards,tag='test',path=cfg.result_path) plot_rewards(rewards, ma_rewards, cfg, tag="test") diff --git a/codes/SoftActorCritic/model.py b/codes/SoftActorCritic/model.py index 85bbfcd..ba04737 100644 --- a/codes/SoftActorCritic/model.py +++ b/codes/SoftActorCritic/model.py @@ -17,10 +17,10 @@ from torch.distributions import Normal device=torch.device("cuda" if torch.cuda.is_available() else "cpu") class ValueNet(nn.Module): - def __init__(self, state_dim, hidden_dim, init_w=3e-3): + def __init__(self, n_states, hidden_dim, init_w=3e-3): super(ValueNet, self).__init__() - self.linear1 = nn.Linear(state_dim, hidden_dim) + self.linear1 = nn.Linear(n_states, hidden_dim) self.linear2 = nn.Linear(hidden_dim, hidden_dim) self.linear3 = nn.Linear(hidden_dim, 1) @@ -35,10 +35,10 @@ class ValueNet(nn.Module): class SoftQNet(nn.Module): - def __init__(self, state_dim, action_dim, hidden_dim, init_w=3e-3): + def __init__(self, n_states, n_actions, hidden_dim, init_w=3e-3): super(SoftQNet, self).__init__() - self.linear1 = nn.Linear(state_dim + action_dim, hidden_dim) + self.linear1 = nn.Linear(n_states + n_actions, hidden_dim) self.linear2 = nn.Linear(hidden_dim, hidden_dim) self.linear3 = nn.Linear(hidden_dim, 1) @@ -54,20 +54,20 @@ class SoftQNet(nn.Module): class PolicyNet(nn.Module): - def __init__(self, state_dim, action_dim, hidden_dim, init_w=3e-3, log_std_min=-20, log_std_max=2): + def __init__(self, n_states, n_actions, hidden_dim, init_w=3e-3, log_std_min=-20, log_std_max=2): super(PolicyNet, self).__init__() self.log_std_min = log_std_min self.log_std_max = log_std_max - self.linear1 = nn.Linear(state_dim, hidden_dim) + self.linear1 = nn.Linear(n_states, hidden_dim) self.linear2 = nn.Linear(hidden_dim, hidden_dim) - self.mean_linear = nn.Linear(hidden_dim, action_dim) + self.mean_linear = nn.Linear(hidden_dim, n_actions) self.mean_linear.weight.data.uniform_(-init_w, init_w) self.mean_linear.bias.data.uniform_(-init_w, init_w) - self.log_std_linear = nn.Linear(hidden_dim, action_dim) + self.log_std_linear = nn.Linear(hidden_dim, n_actions) self.log_std_linear.weight.data.uniform_(-init_w, init_w) self.log_std_linear.bias.data.uniform_(-init_w, init_w) diff --git a/codes/SoftActorCritic/sac.py b/codes/SoftActorCritic/sac.py index d565db5..c67257f 100644 --- a/codes/SoftActorCritic/sac.py +++ b/codes/SoftActorCritic/sac.py @@ -43,10 +43,10 @@ class ReplayBuffer: return len(self.buffer) class ValueNet(nn.Module): - def __init__(self, state_dim, hidden_dim, init_w=3e-3): + def __init__(self, n_states, hidden_dim, init_w=3e-3): super(ValueNet, self).__init__() - self.linear1 = nn.Linear(state_dim, hidden_dim) + self.linear1 = nn.Linear(n_states, hidden_dim) self.linear2 = nn.Linear(hidden_dim, hidden_dim) self.linear3 = nn.Linear(hidden_dim, 1) @@ -61,10 +61,10 @@ class ValueNet(nn.Module): class SoftQNet(nn.Module): - def __init__(self, state_dim, action_dim, hidden_dim, init_w=3e-3): + def __init__(self, n_states, n_actions, hidden_dim, init_w=3e-3): super(SoftQNet, self).__init__() - self.linear1 = nn.Linear(state_dim + action_dim, hidden_dim) + self.linear1 = nn.Linear(n_states + n_actions, hidden_dim) self.linear2 = nn.Linear(hidden_dim, hidden_dim) self.linear3 = nn.Linear(hidden_dim, 1) @@ -80,20 +80,20 @@ class SoftQNet(nn.Module): class PolicyNet(nn.Module): - def __init__(self, state_dim, action_dim, hidden_dim, init_w=3e-3, log_std_min=-20, log_std_max=2): + def __init__(self, n_states, n_actions, hidden_dim, init_w=3e-3, log_std_min=-20, log_std_max=2): super(PolicyNet, self).__init__() self.log_std_min = log_std_min self.log_std_max = log_std_max - self.linear1 = nn.Linear(state_dim, hidden_dim) + self.linear1 = nn.Linear(n_states, hidden_dim) self.linear2 = nn.Linear(hidden_dim, hidden_dim) - self.mean_linear = nn.Linear(hidden_dim, action_dim) + self.mean_linear = nn.Linear(hidden_dim, n_actions) self.mean_linear.weight.data.uniform_(-init_w, init_w) self.mean_linear.bias.data.uniform_(-init_w, init_w) - self.log_std_linear = nn.Linear(hidden_dim, action_dim) + self.log_std_linear = nn.Linear(hidden_dim, n_actions) self.log_std_linear.weight.data.uniform_(-init_w, init_w) self.log_std_linear.bias.data.uniform_(-init_w, init_w) @@ -134,14 +134,14 @@ class PolicyNet(nn.Module): return action[0] class SAC: - def __init__(self,state_dim,action_dim,cfg) -> None: + def __init__(self,n_states,n_actions,cfg) -> None: self.batch_size = cfg.batch_size self.memory = ReplayBuffer(cfg.capacity) self.device = cfg.device - self.value_net = ValueNet(state_dim, cfg.hidden_dim).to(self.device) - self.target_value_net = ValueNet(state_dim, cfg.hidden_dim).to(self.device) - self.soft_q_net = SoftQNet(state_dim, action_dim, cfg.hidden_dim).to(self.device) - self.policy_net = PolicyNet(state_dim, action_dim, cfg.hidden_dim).to(self.device) + self.value_net = ValueNet(n_states, cfg.hidden_dim).to(self.device) + self.target_value_net = ValueNet(n_states, cfg.hidden_dim).to(self.device) + self.soft_q_net = SoftQNet(n_states, n_actions, cfg.hidden_dim).to(self.device) + self.policy_net = PolicyNet(n_states, n_actions, cfg.hidden_dim).to(self.device) self.value_optimizer = optim.Adam(self.value_net.parameters(), lr=cfg.value_lr) self.soft_q_optimizer = optim.Adam(self.soft_q_net.parameters(), lr=cfg.soft_q_lr) self.policy_optimizer = optim.Adam(self.policy_net.parameters(), lr=cfg.policy_lr) diff --git a/codes/SoftActorCritic/task0.py b/codes/SoftActorCritic/task0.py index e910749..668d289 100644 --- a/codes/SoftActorCritic/task0.py +++ b/codes/SoftActorCritic/task0.py @@ -63,9 +63,9 @@ class PlotConfig: def env_agent_config(cfg,seed=1): env = NormalizedActions(gym.make(cfg.env_name)) env.seed(seed) - action_dim = env.action_space.shape[0] - state_dim = env.observation_space.shape[0] - agent = SAC(state_dim,action_dim,cfg) + n_actions = env.action_space.shape[0] + n_states = env.observation_space.shape[0] + agent = SAC(n_states,n_actions,cfg) return env,agent def train(cfg,env,agent): diff --git a/codes/SoftActorCritic/task0_train.ipynb b/codes/SoftActorCritic/task0_train.ipynb index 14be84e..3be10c6 100644 --- a/codes/SoftActorCritic/task0_train.ipynb +++ b/codes/SoftActorCritic/task0_train.ipynb @@ -70,9 +70,9 @@ "def env_agent_config(cfg,seed=1):\n", " env = NormalizedActions(gym.make(\"Pendulum-v0\"))\n", " env.seed(seed)\n", - " action_dim = env.action_space.shape[0]\n", - " state_dim = env.observation_space.shape[0]\n", - " agent = SAC(state_dim,action_dim,cfg)\n", + " n_actions = env.action_space.shape[0]\n", + " n_states = env.observation_space.shape[0]\n", + " agent = SAC(n_states,n_actions,cfg)\n", " return env,agent" ] }, @@ -159,7 +159,7 @@ "\nDuring handling of the above exception, another exception occurred:\n", "\u001b[0;31mDeprecatedEnv\u001b[0m Traceback (most recent call last)", "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 3\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 4\u001b[0m \u001b[0;31m# train\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m \u001b[0menv\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0magent\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0menv_agent_config\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcfg\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mseed\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 6\u001b[0m \u001b[0mrewards\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mma_rewards\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcfg\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0menv\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0magent\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 7\u001b[0m \u001b[0mmake_dir\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcfg\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresult_path\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcfg\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel_path\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m\u001b[0m in \u001b[0;36menv_agent_config\u001b[0;34m(cfg, seed)\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0menv_agent_config\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcfg\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mseed\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0menv\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mNormalizedActions\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgym\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmake\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Pendulum-v0\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 3\u001b[0m \u001b[0menv\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mseed\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mseed\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 4\u001b[0m \u001b[0maction_dim\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0menv\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maction_space\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[0mstate_dim\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0menv\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mobservation_space\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m\u001b[0m in \u001b[0;36menv_agent_config\u001b[0;34m(cfg, seed)\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0menv_agent_config\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcfg\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mseed\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0menv\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mNormalizedActions\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgym\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmake\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Pendulum-v0\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 3\u001b[0m \u001b[0menv\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mseed\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mseed\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 4\u001b[0m \u001b[0mn_actions\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0menv\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maction_space\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[0mn_states\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0menv\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mobservation_space\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m~/anaconda3/envs/py37/lib/python3.7/site-packages/gym/envs/registration.py\u001b[0m in \u001b[0;36mmake\u001b[0;34m(id, **kwargs)\u001b[0m\n\u001b[1;32m 233\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 234\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mmake\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mid\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 235\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mregistry\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmake\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mid\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 236\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 237\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m~/anaconda3/envs/py37/lib/python3.7/site-packages/gym/envs/registration.py\u001b[0m in \u001b[0;36mmake\u001b[0;34m(self, path, **kwargs)\u001b[0m\n\u001b[1;32m 126\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 127\u001b[0m \u001b[0mlogger\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minfo\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Making new env: %s\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpath\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 128\u001b[0;31m \u001b[0mspec\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mspec\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 129\u001b[0m \u001b[0menv\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mspec\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmake\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 130\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0menv\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m~/anaconda3/envs/py37/lib/python3.7/site-packages/gym/envs/registration.py\u001b[0m in \u001b[0;36mspec\u001b[0;34m(self, path)\u001b[0m\n\u001b[1;32m 185\u001b[0m raise error.DeprecatedEnv(\n\u001b[1;32m 186\u001b[0m \"Env {} not found (valid versions include {})\".format(\n\u001b[0;32m--> 187\u001b[0;31m \u001b[0mid\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmatching_envs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 188\u001b[0m )\n\u001b[1;32m 189\u001b[0m )\n", diff --git a/codes/TD3/agent.py b/codes/TD3/agent.py index 91939a6..f77a912 100644 --- a/codes/TD3/agent.py +++ b/codes/TD3/agent.py @@ -21,8 +21,8 @@ class Actor(nn.Module): '''[summary] Args: - input_dim (int): 输入维度，这里等于state_dim - output_dim (int): 输出维度，这里等于action_dim + input_dim (int): 输入维度，这里等于n_states + output_dim (int): 输出维度，这里等于n_actions max_action (int): action的最大值 ''' super(Actor, self).__init__() diff --git a/codes/TD3/memory.py b/codes/TD3/memory.py index 7e2671c..bcf38bb 100644 --- a/codes/TD3/memory.py +++ b/codes/TD3/memory.py @@ -14,13 +14,13 @@ import torch class ReplayBuffer(object): - def __init__(self, state_dim, action_dim, max_size=int(1e6)): + def __init__(self, n_states, n_actions, max_size=int(1e6)): self.max_size = max_size self.ptr = 0 self.size = 0 - self.state = np.zeros((max_size, state_dim)) - self.action = np.zeros((max_size, action_dim)) - self.next_state = np.zeros((max_size, state_dim)) + self.state = np.zeros((max_size, n_states)) + self.action = np.zeros((max_size, n_actions)) + self.next_state = np.zeros((max_size, n_states)) self.reward = np.zeros((max_size, 1)) self.not_done = np.zeros((max_size, 1)) self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") diff --git a/codes/TD3/task0_eval.py b/codes/TD3/task0_eval.py index 0420dce..cb977b4 100644 --- a/codes/TD3/task0_eval.py +++ b/codes/TD3/task0_eval.py @@ -74,10 +74,10 @@ if __name__ == "__main__": env.seed(cfg.seed) # Set seeds torch.manual_seed(cfg.seed) np.random.seed(cfg.seed) - state_dim = env.observation_space.shape[0] - action_dim = env.action_space.shape[0] + n_states = env.observation_space.shape[0] + n_actions = env.action_space.shape[0] max_action = float(env.action_space.high[0]) - td3= TD3(state_dim,action_dim,max_action,cfg) + td3= TD3(n_states,n_actions,max_action,cfg) cfg.model_path = './TD3/results/HalfCheetah-v2/20210416-130341/models/' td3.load(cfg.model_path) td3_rewards,td3_ma_rewards = eval(cfg.env,td3,cfg.seed) diff --git a/codes/TD3/task0_train.py b/codes/TD3/task0_train.py index 11e2adf..58e4af9 100644 --- a/codes/TD3/task0_train.py +++ b/codes/TD3/task0_train.py @@ -72,7 +72,7 @@ def train(cfg,env,agent): else: action = ( agent.choose_action(np.array(state)) - + np.random.normal(0, max_action * cfg.expl_noise, size=action_dim) + + np.random.normal(0, max_action * cfg.expl_noise, size=n_actions) ).clip(-max_action, max_action) # Perform action next_state, reward, done, _ = env.step(action) @@ -121,11 +121,11 @@ def train(cfg,env,agent): # else: # action = ( # agent.choose_action(np.array(state)) -# + np.random.normal(0, max_action * cfg.expl_noise, size=action_dim) +# + np.random.normal(0, max_action * cfg.expl_noise, size=n_actions) # ).clip(-max_action, max_action) # # action = ( # # agent.choose_action(np.array(state)) -# # + np.random.normal(0, max_action * cfg.expl_noise, size=action_dim) +# # + np.random.normal(0, max_action * cfg.expl_noise, size=n_actions) # # ).clip(-max_action, max_action) # # Perform action # next_state, reward, done, _ = env.step(action) @@ -157,10 +157,10 @@ if __name__ == "__main__": env.seed(cfg.seed) # Set seeds torch.manual_seed(cfg.seed) np.random.seed(cfg.seed) - state_dim = env.observation_space.shape[0] - action_dim = env.action_space.shape[0] + n_states = env.observation_space.shape[0] + n_actions = env.action_space.shape[0] max_action = float(env.action_space.high[0]) - agent = TD3(state_dim,action_dim,max_action,cfg) + agent = TD3(n_states,n_actions,max_action,cfg) rewards,ma_rewards = train(cfg,env,agent) make_dir(cfg.result_path,cfg.model_path) agent.save(path=cfg.model_path) diff --git a/codes/TD3/task1_eval.py b/codes/TD3/task1_eval.py index ae17681..0d28c48 100644 --- a/codes/TD3/task1_eval.py +++ b/codes/TD3/task1_eval.py @@ -70,10 +70,10 @@ if __name__ == "__main__": env.seed(cfg.seed) # Set seeds torch.manual_seed(cfg.seed) np.random.seed(cfg.seed) - state_dim = env.observation_space.shape[0] - action_dim = env.action_space.shape[0] + n_states = env.observation_space.shape[0] + n_actions = env.action_space.shape[0] max_action = float(env.action_space.high[0]) - td3= TD3(state_dim,action_dim,max_action,cfg) + td3= TD3(n_states,n_actions,max_action,cfg) cfg.model_path = './TD3/results/Pendulum-v0/20210428-092059/models/' cfg.result_path = './TD3/results/Pendulum-v0/20210428-092059/results/' td3.load(cfg.model_path) diff --git a/codes/TD3/task1_train.py b/codes/TD3/task1_train.py index 9780f76..868f686 100644 --- a/codes/TD3/task1_train.py +++ b/codes/TD3/task1_train.py @@ -79,7 +79,7 @@ def train(cfg,env,agent): else: action = ( agent.choose_action(np.array(state)) - + np.random.normal(0, max_action * cfg.expl_noise, size=action_dim) + + np.random.normal(0, max_action * cfg.expl_noise, size=n_actions) ).clip(-max_action, max_action) # Perform action next_state, reward, done, _ = env.step(action) @@ -109,10 +109,10 @@ if __name__ == "__main__": env.seed(1) # 随机种子 torch.manual_seed(1) np.random.seed(1) - state_dim = env.observation_space.shape[0] - action_dim = env.action_space.shape[0] + n_states = env.observation_space.shape[0] + n_actions = env.action_space.shape[0] max_action = float(env.action_space.high[0]) - agent = TD3(state_dim,action_dim,max_action,cfg) + agent = TD3(n_states,n_actions,max_action,cfg) rewards,ma_rewards = train(cfg,env,agent) make_dir(plot_cfg.result_path,plot_cfg.model_path) agent.save(path=plot_cfg.model_path) diff --git a/codes/common/model.py b/codes/common/model.py index 4ab0b8b..1e7bbaa 100644 --- a/codes/common/model.py +++ b/codes/common/model.py @@ -32,10 +32,10 @@ class MLP(nn.Module): return self.fc3(x) class Critic(nn.Module): - def __init__(self, n_obs, action_dim, hidden_size, init_w=3e-3): + def __init__(self, n_obs, n_actions, hidden_size, init_w=3e-3): super(Critic, self).__init__() - self.linear1 = nn.Linear(n_obs + action_dim, hidden_size) + self.linear1 = nn.Linear(n_obs + n_actions, hidden_size) self.linear2 = nn.Linear(hidden_size, hidden_size) self.linear3 = nn.Linear(hidden_size, 1) # 随机初始化为较小的值 @@ -51,11 +51,11 @@ class Critic(nn.Module): return x class Actor(nn.Module): - def __init__(self, n_obs, action_dim, hidden_size, init_w=3e-3): + def __init__(self, n_obs, n_actions, hidden_size, init_w=3e-3): super(Actor, self).__init__() self.linear1 = nn.Linear(n_obs, hidden_size) self.linear2 = nn.Linear(hidden_size, hidden_size) - self.linear3 = nn.Linear(hidden_size, action_dim) + self.linear3 = nn.Linear(hidden_size, n_actions) self.linear3.weight.data.uniform_(-init_w, init_w) self.linear3.bias.data.uniform_(-init_w, init_w) @@ -67,18 +67,18 @@ class Actor(nn.Module): return x class ActorCritic(nn.Module): - def __init__(self, state_dim, action_dim, hidden_dim=256): + def __init__(self, n_states, n_actions, hidden_dim=256): super(ActorCritic, self).__init__() self.critic = nn.Sequential( - nn.Linear(state_dim, hidden_dim), + nn.Linear(n_states, hidden_dim), nn.ReLU(), nn.Linear(hidden_dim, 1) ) self.actor = nn.Sequential( - nn.Linear(state_dim, hidden_dim), + nn.Linear(n_states, hidden_dim), nn.ReLU(), - nn.Linear(hidden_dim, action_dim), + nn.Linear(hidden_dim, n_actions), nn.Softmax(dim=1), ) diff --git a/codes/common/utils.py b/codes/common/utils.py index 6027804..612163f 100644 --- a/codes/common/utils.py +++ b/codes/common/utils.py @@ -5,7 +5,7 @@ Author: John Email: johnjim0816@gmail.com Date: 2021-03-12 16:02:24 LastEditor: John -LastEditTime: 2021-11-30 18:39:19 +LastEditTime: 2022-02-28 11:50:11 Discription: Environment: ''' @@ -68,7 +68,13 @@ def plot_losses(losses, algo="DQN", save=True, path='./'): plt.savefig(path+"losses_curve") plt.show() - +def save_results_1(dic, tag='train', path='./results'): + ''' 保存奖励 + ''' + for key,value in dic.items(): + np.save(path+'{}_{}.npy'.format(tag,key),value) + print('Results saved！') + def save_results(rewards, ma_rewards, tag='train', path='./results'): ''' 保存奖励 ''' diff --git a/codes/envs/blackjack.py b/codes/envs/blackjack.py index 6946895..87f02d2 100644 --- a/codes/envs/blackjack.py +++ b/codes/envs/blackjack.py @@ -77,7 +77,7 @@ class BlackjackEnv(gym.Env): self.natural = natural # Start the first game self._reset() # Number of - self.action_dim = 2 + self.n_actions = 2 def reset(self): return self._reset() diff --git a/codes/envs/cliff_walking.py b/codes/envs/cliff_walking.py index 73e33c7..05b9b2e 100644 --- a/codes/envs/cliff_walking.py +++ b/codes/envs/cliff_walking.py @@ -31,7 +31,7 @@ class CliffWalkingEnv(discrete.DiscreteEnv): self.shape = (4, 12) nS = np.prod(self.shape) - action_dim = 4 + n_actions = 4 # Cliff Location self._cliff = np.zeros(self.shape, dtype=np.bool) @@ -41,7 +41,7 @@ class CliffWalkingEnv(discrete.DiscreteEnv): P = {} for s in range(nS): position = np.unravel_index(s, self.shape) - P[s] = { a : [] for a in range(action_dim) } + P[s] = { a : [] for a in range(n_actions) } P[s][UP] = self._calculate_transition_prob(position, [-1, 0]) P[s][RIGHT] = self._calculate_transition_prob(position, [0, 1]) P[s][DOWN] = self._calculate_transition_prob(position, [1, 0]) @@ -51,7 +51,7 @@ class CliffWalkingEnv(discrete.DiscreteEnv): isd = np.zeros(nS) isd[np.ravel_multi_index((3,0), self.shape)] = 1.0 - super(CliffWalkingEnv, self).__init__(nS, action_dim, P, isd) + super(CliffWalkingEnv, self).__init__(nS, n_actions, P, isd) def render(self, mode='human', close=False): self._render(mode, close) diff --git a/codes/envs/gridworld.py b/codes/envs/gridworld.py index c4fd512..cf3aec2 100644 --- a/codes/envs/gridworld.py +++ b/codes/envs/gridworld.py @@ -37,7 +37,7 @@ class GridworldEnv(discrete.DiscreteEnv): self.shape = shape nS = np.prod(shape) - action_dim = 4 + n_actions = 4 MAX_Y = shape[0] MAX_X = shape[1] @@ -51,7 +51,7 @@ class GridworldEnv(discrete.DiscreteEnv): y, x = it.multi_index # P[s][a] = (prob, next_state, reward, is_done) - P[s] = {a : [] for a in range(action_dim)} + P[s] = {a : [] for a in range(n_actions)} is_done = lambda s: s == 0 or s == (nS - 1) reward = 0.0 if is_done(s) else -1.0 @@ -82,7 +82,7 @@ class GridworldEnv(discrete.DiscreteEnv): # This should not be used in any model-free learning algorithm self.P = P - super(GridworldEnv, self).__init__(nS, action_dim, P, isd) + super(GridworldEnv, self).__init__(nS, n_actions, P, isd) def _render(self, mode='human', close=False): """ Renders the current gridworld layout diff --git a/codes/envs/snake/checkpoint.npy b/codes/envs/snake/checkpoint.npy new file mode 100644 index 0000000..591d49e Binary files /dev/null and b/codes/envs/snake/checkpoint.npy differ diff --git a/codes/envs/snake/checkpoint1.npy b/codes/envs/snake/checkpoint1.npy new file mode 100644 index 0000000..84b54ca Binary files /dev/null and b/codes/envs/snake/checkpoint1.npy differ diff --git a/codes/envs/snake/checkpoint2.npy b/codes/envs/snake/checkpoint2.npy new file mode 100644 index 0000000..4614eb7 Binary files /dev/null and b/codes/envs/snake/checkpoint2.npy differ diff --git a/codes/envs/snake/checkpoint3.npy b/codes/envs/snake/checkpoint3.npy new file mode 100644 index 0000000..8737b4c Binary files /dev/null and b/codes/envs/snake/checkpoint3.npy differ diff --git a/codes/envs/snake/q_agent.npy b/codes/envs/snake/q_agent.npy new file mode 100644 index 0000000..75ef415 Binary files /dev/null and b/codes/envs/snake/q_agent.npy differ diff --git a/codes/envs/stochastic_mdp.py b/codes/envs/stochastic_mdp.py index 5770fa5..3c1ad4d 100644 --- a/codes/envs/stochastic_mdp.py +++ b/codes/envs/stochastic_mdp.py @@ -17,31 +17,31 @@ class StochasticMDP: def __init__(self): self.end = False self.curr_state = 2 - self.action_dim = 2 - self.state_dim = 6 + self.n_actions = 2 + self.n_states = 6 self.p_right = 0.5 def reset(self): self.end = False self.curr_state = 2 - state = np.zeros(self.state_dim) + state = np.zeros(self.n_states) state[self.curr_state - 1] = 1. return state def step(self, action): if self.curr_state != 1: if action == 1: - if random.random() < self.p_right and self.curr_state < self.state_dim: + if random.random() < self.p_right and self.curr_state < self.n_states: self.curr_state += 1 else: self.curr_state -= 1 if action == 0: self.curr_state -= 1 - if self.curr_state == self.state_dim: + if self.curr_state == self.n_states: self.end = True - state = np.zeros(self.state_dim) + state = np.zeros(self.n_states) state[self.curr_state - 1] = 1. if self.curr_state == 1: diff --git a/codes/envs/windy_gridworld.py b/codes/envs/windy_gridworld.py index ac9c66a..2a9d4a4 100644 --- a/codes/envs/windy_gridworld.py +++ b/codes/envs/windy_gridworld.py @@ -30,7 +30,7 @@ class WindyGridworldEnv(discrete.DiscreteEnv): self.shape = (7, 10) nS = np.prod(self.shape) - action_dim = 4 + n_actions = 4 # Wind strength winds = np.zeros(self.shape) @@ -41,7 +41,7 @@ class WindyGridworldEnv(discrete.DiscreteEnv): P = {} for s in range(nS): position = np.unravel_index(s, self.shape) - P[s] = { a : [] for a in range(action_dim) } + P[s] = { a : [] for a in range(n_actions) } P[s][UP] = self._calculate_transition_prob(position, [-1, 0], winds) P[s][RIGHT] = self._calculate_transition_prob(position, [0, 1], winds) P[s][DOWN] = self._calculate_transition_prob(position, [1, 0], winds) @@ -51,7 +51,7 @@ class WindyGridworldEnv(discrete.DiscreteEnv): isd = np.zeros(nS) isd[np.ravel_multi_index((3,0), self.shape)] = 1.0 - super(WindyGridworldEnv, self).__init__(nS, action_dim, P, isd) + super(WindyGridworldEnv, self).__init__(nS, n_actions, P, isd) def render(self, mode='human', close=False): self._render(mode, close)