diff --git a/codes/A2C/agent.py b/codes/A2C/agent.py index e095bc5..997401b 100644 --- a/codes/A2C/agent.py +++ b/codes/A2C/agent.py @@ -10,12 +10,40 @@ Discription: Environment: ''' import torch.optim as optim -from A2C.model import ActorCritic +import torch.nn as nn +import torch.nn.functional as F +from torch.distributions import Categorical + +class ActorCritic(nn.Module): + ''' A2C网络模型，包含一个Actor和Critic + ''' + def __init__(self, input_dim, output_dim, hidden_dim): + super(ActorCritic, self).__init__() + self.critic = nn.Sequential( + nn.Linear(input_dim, hidden_dim), + nn.ReLU(), + nn.Linear(hidden_dim, 1) + ) + + self.actor = nn.Sequential( + nn.Linear(input_dim, hidden_dim), + nn.ReLU(), + nn.Linear(hidden_dim, output_dim), + nn.Softmax(dim=1), + ) + + def forward(self, x): + value = self.critic(x) + probs = self.actor(x) + dist = Categorical(probs) + return dist, value class A2C: - def __init__(self,n_states,n_actions,cfg) -> None: + ''' A2C算法 + ''' + def __init__(self,state_dim,action_dim,cfg) -> None: self.gamma = cfg.gamma self.device = cfg.device - self.model = ActorCritic(n_states, n_actions, cfg.hidden_size).to(self.device) + self.model = ActorCritic(state_dim, action_dim, cfg.hidden_size).to(self.device) self.optimizer = optim.Adam(self.model.parameters()) def compute_returns(self,next_value, rewards, masks): diff --git a/codes/A2C/model.py b/codes/A2C/model.py deleted file mode 100644 index 473bcb2..0000000 --- a/codes/A2C/model.py +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env python -# coding=utf-8 -''' -Author: JiangJi -Email: johnjim0816@gmail.com -Date: 2021-05-03 21:38:54 -LastEditor: JiangJi -LastEditTime: 2021-05-03 21:40:06 -Discription: -Environment: -''' -import torch.nn as nn -import torch.nn.functional as F -from torch.distributions import Categorical -class ActorCritic(nn.Module): - def __init__(self, n_states, n_actions, hidden_dim): - super(ActorCritic, self).__init__() - - self.critic = nn.Sequential( - nn.Linear(n_states, hidden_dim), - nn.ReLU(), - nn.Linear(hidden_dim, 1) - ) - - self.actor = nn.Sequential( - nn.Linear(n_states, hidden_dim), - nn.ReLU(), - nn.Linear(hidden_dim, n_actions), - nn.Softmax(dim=1), - ) - - def forward(self, x): - value = self.critic(x) - probs = self.actor(x) - dist = Categorical(probs) - return dist, value \ No newline at end of file diff --git a/codes/A2C/outputs/CartPole-v0/20211221-165620/results/train_ma_rewards.npy b/codes/A2C/outputs/CartPole-v0/20211221-165620/results/train_ma_rewards.npy new file mode 100644 index 0000000..6537afd Binary files /dev/null and b/codes/A2C/outputs/CartPole-v0/20211221-165620/results/train_ma_rewards.npy differ diff --git a/codes/A2C/outputs/CartPole-v0/20211221-165620/results/train_rewards.npy b/codes/A2C/outputs/CartPole-v0/20211221-165620/results/train_rewards.npy new file mode 100644 index 0000000..56f779b Binary files /dev/null and b/codes/A2C/outputs/CartPole-v0/20211221-165620/results/train_rewards.npy differ diff --git a/codes/A2C/task0_train.ipynb b/codes/A2C/task0.ipynb similarity index 100% rename from codes/A2C/task0_train.ipynb rename to codes/A2C/task0.ipynb diff --git a/codes/A2C/task0_train.py b/codes/A2C/task0.py similarity index 86% rename from codes/A2C/task0_train.py rename to codes/A2C/task0.py index 5927048..fd54d87 100644 --- a/codes/A2C/task0_train.py +++ b/codes/A2C/task0.py @@ -1,7 +1,8 @@ -import sys,os -curr_path = os.path.dirname(os.path.abspath(__file__)) # 当前文件所在绝对路径 -parent_path = os.path.dirname(curr_path) # 父路径 -sys.path.append(parent_path) # 添加路径到系统路径sys.path +import sys +import os +curr_path = os.path.dirname(os.path.abspath(__file__)) # 当前文件所在绝对路径 +parent_path = os.path.dirname(curr_path) # 父路径 +sys.path.append(parent_path) # 添加路径到系统路径 import gym import numpy as np @@ -9,15 +10,18 @@ import torch import torch.optim as optim import datetime from common.multiprocessing_env import SubprocVecEnv -from A2C.model import ActorCritic +from A2C.agent import ActorCritic from common.utils import save_results, make_dir -from common.plot import plot_rewards +from common.utils import plot_rewards + +curr_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") # 获取当前时间 +algo_name = 'A2C' # 算法名称 +env_name = 'CartPole-v0' # 环境名称 -curr_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") # obtain current time class A2CConfig: def __init__(self) -> None: - self.algo='A2C' # 算法名称 - self.env_name= 'CartPole-v0' # 环境名称 + self.algo_name = algo_name# 算法名称 + self.env_name = env_name # 环境名称 self.n_envs = 8 # 异步的环境数目 self.gamma = 0.99 # 强化学习中的折扣因子 self.hidden_dim = 256 @@ -27,10 +31,9 @@ class A2CConfig: self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") class PlotConfig: def __init__(self) -> None: - self.algo = "DQN" # 算法名称 - self.env_name = 'CartPole-v0' # 环境名称 + self.algo_name = algo_name # 算法名称 + self.env_name = env_name # 环境名称 self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # 检测GPU - self.result_path = curr_path+"/outputs/" + self.env_name + \ '/'+curr_time+'/results/' # 保存结果的路径 self.model_path = curr_path+"/outputs/" + self.env_name + \ @@ -67,6 +70,8 @@ def compute_returns(next_value, rewards, masks, gamma=0.99): def train(cfg,envs): + print('开始训练!') + print(f'环境：{cfg.env_name}, 算法：{cfg.algo}, 设备：{cfg.device}') env = gym.make(cfg.env_name) # a single env env.seed(10) state_dim = envs.observation_space.shape[0] @@ -119,6 +124,7 @@ def train(cfg,envs): optimizer.zero_grad() loss.backward() optimizer.step() + print('完成训练！') return test_rewards, test_ma_rewards if __name__ == "__main__": cfg = A2CConfig() diff --git a/codes/DDPG/agent.py b/codes/DDPG/agent.py index 528872e..01ded1c 100644 --- a/codes/DDPG/agent.py +++ b/codes/DDPG/agent.py @@ -9,22 +9,75 @@ LastEditTime: 2021-09-16 00:55:30 @Discription: @Environment: python 3.7.7 ''' +import random import numpy as np import torch import torch.nn as nn import torch.optim as optim - -from common.model import Actor, Critic -from common.memory import ReplayBuffer - - +import torch.nn.functional as F +class ReplayBuffer: + def __init__(self, capacity): + self.capacity = capacity # 经验回放的容量 + self.buffer = [] # 缓冲区 + self.position = 0 + + def push(self, state, action, reward, next_state, done): + ''' 缓冲区是一个队列，容量超出时去掉开始存入的转移(transition) + ''' + if len(self.buffer) < self.capacity: + self.buffer.append(None) + self.buffer[self.position] = (state, action, reward, next_state, done) + self.position = (self.position + 1) % self.capacity + + def sample(self, batch_size): + batch = random.sample(self.buffer, batch_size) # 随机采出小批量转移 + state, action, reward, next_state, done = zip(*batch) # 解压成状态，动作等 + return state, action, reward, next_state, done + + def __len__(self): + ''' 返回当前存储的量 + ''' + return len(self.buffer) +class Actor(nn.Module): + def __init__(self, n_states, n_actions, hidden_dim, init_w=3e-3): + super(Actor, self).__init__() + self.linear1 = nn.Linear(n_states, hidden_dim) + self.linear2 = nn.Linear(hidden_dim, hidden_dim) + self.linear3 = nn.Linear(hidden_dim, n_actions) + + self.linear3.weight.data.uniform_(-init_w, init_w) + self.linear3.bias.data.uniform_(-init_w, init_w) + + def forward(self, x): + x = F.relu(self.linear1(x)) + x = F.relu(self.linear2(x)) + x = torch.tanh(self.linear3(x)) + return x +class Critic(nn.Module): + def __init__(self, n_states, n_actions, hidden_dim, init_w=3e-3): + super(Critic, self).__init__() + + self.linear1 = nn.Linear(n_states + n_actions, hidden_dim) + self.linear2 = nn.Linear(hidden_dim, hidden_dim) + self.linear3 = nn.Linear(hidden_dim, 1) + # 随机初始化为较小的值 + self.linear3.weight.data.uniform_(-init_w, init_w) + self.linear3.bias.data.uniform_(-init_w, init_w) + + def forward(self, state, action): + # 按维数1拼接 + x = torch.cat([state, action], 1) + x = F.relu(self.linear1(x)) + x = F.relu(self.linear2(x)) + x = self.linear3(x) + return x class DDPG: - def __init__(self, state_dim, action_dim, cfg): + def __init__(self, n_states, n_actions, cfg): self.device = cfg.device - self.critic = Critic(state_dim, action_dim, cfg.hidden_dim).to(cfg.device) - self.actor = Actor(state_dim, action_dim, cfg.hidden_dim).to(cfg.device) - self.target_critic = Critic(state_dim, action_dim, cfg.hidden_dim).to(cfg.device) - self.target_actor = Actor(state_dim, action_dim, cfg.hidden_dim).to(cfg.device) + self.critic = Critic(n_states, n_actions, cfg.hidden_dim).to(cfg.device) + self.actor = Actor(n_states, n_actions, cfg.hidden_dim).to(cfg.device) + self.target_critic = Critic(n_states, n_actions, cfg.hidden_dim).to(cfg.device) + self.target_actor = Actor(n_states, n_actions, cfg.hidden_dim).to(cfg.device) # 复制参数到目标网络 for target_param, param in zip(self.target_critic.parameters(), self.critic.parameters()): diff --git a/codes/DDPG/env.py b/codes/DDPG/env.py index 99da3c5..92fe482 100644 --- a/codes/DDPG/env.py +++ b/codes/DDPG/env.py @@ -16,12 +16,10 @@ class NormalizedActions(gym.ActionWrapper): ''' 将action范围重定在[0.1]之间 ''' def action(self, action): - low_bound = self.action_space.low upper_bound = self.action_space.high action = low_bound + (action + 1.0) * 0.5 * (upper_bound - low_bound) action = np.clip(action, low_bound, upper_bound) - return action def reverse_action(self, action): diff --git a/codes/DDPG/task0.py b/codes/DDPG/task0.py new file mode 100644 index 0000000..33872f4 --- /dev/null +++ b/codes/DDPG/task0.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python +# coding=utf-8 +''' +@Author: John +@Email: johnjim0816@gmail.com +@Date: 2020-06-11 20:58:21 +@LastEditor: John +LastEditTime: 2021-09-16 01:31:33 +@Discription: +@Environment: python 3.7.7 +''' +import sys,os +curr_path = os.path.dirname(os.path.abspath(__file__)) # 当前文件所在绝对路径 +parent_path = os.path.dirname(curr_path) # 父路径 +sys.path.append(parent_path) # 添加路径到系统路径sys.path + +import datetime +import gym +import torch + +from DDPG.env import NormalizedActions +from DDPG.agent import DDPG +from DDPG.train import train,test +from common.utils import save_results,make_dir +from common.utils import plot_rewards + +curr_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") # 获取当前时间 +algo_name = 'DDPG' # 算法名称 +env_name = 'Pendulum-v1' # 环境名称，gym新版本（约0.21.0之后）中Pendulum-v0改为Pendulum-v1 + +class DDPGConfig: + def __init__(self): + self.algo_name = algo_name # 算法名称 + self.env_name = env_name # 环境名称 + self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # 检测GPU + self.train_eps = 300 # 训练的回合数 + self.eval_eps = 50 # 测试的回合数 + self.gamma = 0.99 # 折扣因子 + self.critic_lr = 1e-3 # 评论家网络的学习率 + self.actor_lr = 1e-4 # 演员网络的学习率 + self.memory_capacity = 8000 # 经验回放的容量 + self.batch_size = 128 # mini-batch SGD中的批量大小 + self.target_update = 2 # 目标网络的更新频率 + self.hidden_dim = 256 # 网络隐藏层维度 + self.soft_tau = 1e-2 # 软更新参数 + +class PlotConfig: + def __init__(self) -> None: + self.algo_name = algo_name # 算法名称 + self.env_name = env_name # 环境名称 + self.result_path = curr_path+"/outputs/" + self.env_name + \ + '/'+curr_time+'/results/' # 保存结果的路径 + self.model_path = curr_path+"/outputs/" + self.env_name + \ + '/'+curr_time+'/models/' # 保存模型的路径 + self.save = True # 是否保存图片 + self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # 检测GPU + +def env_agent_config(cfg,seed=1): + env = NormalizedActions(gym.make(cfg.env_name)) # 装饰action噪声 + env.seed(seed) # 随机种子 + n_states = env.observation_space.shape[0] + n_actions = env.action_space.shape[0] + agent = DDPG(n_states,n_actions,cfg) + return env,agent + +cfg = DDPGConfig() +plot_cfg = PlotConfig() +# 训练 +env,agent = env_agent_config(cfg,seed=1) +rewards, ma_rewards = train(cfg, env, agent) +make_dir(plot_cfg.result_path, plot_cfg.model_path) +agent.save(path=plot_cfg.model_path) +save_results(rewards, ma_rewards, tag='train', path=plot_cfg.result_path) +plot_rewards(rewards, ma_rewards, plot_cfg, tag="train") # 画出结果 +# 测试 +env,agent = env_agent_config(cfg,seed=10) +agent.load(path=plot_cfg.model_path) +rewards,ma_rewards = test(plot_cfg,env,agent) +save_results(rewards,ma_rewards,tag = 'test',path = cfg.result_path) +plot_rewards(rewards, ma_rewards, plot_cfg, tag="test") # 画出结果 + diff --git a/codes/DDPG/task0_train.py b/codes/DDPG/task0_train.py deleted file mode 100644 index ea76661..0000000 --- a/codes/DDPG/task0_train.py +++ /dev/null @@ -1,136 +0,0 @@ -#!/usr/bin/env python -# coding=utf-8 -''' -@Author: John -@Email: johnjim0816@gmail.com -@Date: 2020-06-11 20:58:21 -@LastEditor: John -LastEditTime: 2021-09-16 01:31:33 -@Discription: -@Environment: python 3.7.7 -''' -import sys,os -curr_path = os.path.dirname(os.path.abspath(__file__)) # 当前文件所在绝对路径 -parent_path = os.path.dirname(curr_path) # 父路径 -sys.path.append(parent_path) # 添加路径到系统路径sys.path - -import datetime -import gym -import torch - -from DDPG.env import NormalizedActions, OUNoise -from DDPG.agent import DDPG -from common.utils import save_results,make_dir -from common.plot import plot_rewards - -curr_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") # 获取当前时间 - -class DDPGConfig: - def __init__(self): - self.algo = 'DDPG' # 算法名称 - self.env_name = 'Pendulum-v0' # 环境名称 - self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # 检测GPU - self.train_eps = 300 # 训练的回合数 - self.eval_eps = 50 # 测试的回合数 - self.gamma = 0.99 # 折扣因子 - self.critic_lr = 1e-3 # 评论家网络的学习率 - self.actor_lr = 1e-4 # 演员网络的学习率 - self.memory_capacity = 8000 # 经验回放的容量 - self.batch_size = 128 # mini-batch SGD中的批量大小 - self.target_update = 2 # 目标网络的更新频率 - self.hidden_dim = 256 # 网络隐藏层维度 - self.soft_tau = 1e-2 # 软更新参数 - -class PlotConfig: - def __init__(self) -> None: - self.algo = "DQN" # 算法名称 - self.env_name = 'CartPole-v0' # 环境名称 - self.result_path = curr_path+"/outputs/" + self.env_name + \ - '/'+curr_time+'/results/' # 保存结果的路径 - self.model_path = curr_path+"/outputs/" + self.env_name + \ - '/'+curr_time+'/models/' # 保存模型的路径 - self.save = True # 是否保存图片 - self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # 检测GPU - -def env_agent_config(cfg,seed=1): - env = NormalizedActions(gym.make(cfg.env_name)) # 装饰action噪声 - env.seed(seed) # 随机种子 - n_states = env.observation_space.shape[0] - n_actions = env.action_space.shape[0] - agent = DDPG(n_states,n_actions,cfg) - return env,agent - -def train(cfg, env, agent): - print('开始训练！') - print(f'环境：{cfg.env_name}，算法：{cfg.algo}，设备：{cfg.device}') - ou_noise = OUNoise(env.action_space) # 动作噪声 - rewards = [] # 记录所有回合的奖励 - ma_rewards = [] # 记录所有回合的滑动平均奖励 - for i_ep in range(cfg.train_eps): - state = env.reset() - ou_noise.reset() - done = False - ep_reward = 0 - i_step = 0 - while not done: - i_step += 1 - action = agent.choose_action(state) - action = ou_noise.get_action(action, i_step) - next_state, reward, done, _ = env.step(action) - ep_reward += reward - agent.memory.push(state, action, reward, next_state, done) - agent.update() - state = next_state - if (i_ep+1)%10 == 0: - print('回合：{}/{}，奖励：{:.2f}'.format(i_ep+1, cfg.train_eps, ep_reward)) - rewards.append(ep_reward) - if ma_rewards: - ma_rewards.append(0.9*ma_rewards[-1]+0.1*ep_reward) - else: - ma_rewards.append(ep_reward) - print('完成训练！') - return rewards, ma_rewards - -def eval(cfg, env, agent): - print('开始测试！') - print(f'环境：{cfg.env_name}, 算法：{cfg.algo}, 设备：{cfg.device}') - rewards = [] # 记录所有回合的奖励 - ma_rewards = [] # 记录所有回合的滑动平均奖励 - for i_ep in range(cfg.eval_eps): - state = env.reset() - done = False - ep_reward = 0 - i_step = 0 - while not done: - i_step += 1 - action = agent.choose_action(state) - next_state, reward, done, _ = env.step(action) - ep_reward += reward - state = next_state - print('回合：{}/{}, 奖励：{}'.format(i_ep+1, cfg.train_eps, ep_reward)) - rewards.append(ep_reward) - if ma_rewards: - ma_rewards.append(0.9*ma_rewards[-1]+0.1*ep_reward) - else: - ma_rewards.append(ep_reward) - print('完成测试！') - return rewards, ma_rewards - - -if __name__ == "__main__": - cfg = DDPGConfig() - plot_cfg = PlotConfig() - # 训练 - env,agent = env_agent_config(cfg,seed=1) - rewards, ma_rewards = train(cfg, env, agent) - make_dir(plot_cfg.result_path, plot_cfg.model_path) - agent.save(path=plot_cfg.model_path) - save_results(rewards, ma_rewards, tag='train', path=plot_cfg.result_path) - plot_rewards(rewards, ma_rewards, plot_cfg, tag="train") - # 测试 - env,agent = env_agent_config(cfg,seed=10) - agent.load(path=plot_cfg.model_path) - rewards,ma_rewards = eval(plot_cfg,env,agent) - save_results(rewards,ma_rewards,tag = 'eval',path = cfg.result_path) - plot_rewards(rewards,ma_rewards,plot_cfg,tag = "eval") - diff --git a/codes/DDPG/train.py b/codes/DDPG/train.py new file mode 100644 index 0000000..8554cd0 --- /dev/null +++ b/codes/DDPG/train.py @@ -0,0 +1,64 @@ +import sys +import os +curr_path = os.path.dirname(os.path.abspath(__file__)) # 当前文件所在绝对路径 +parent_path = os.path.dirname(curr_path) # 父路径 +sys.path.append(parent_path) # 添加路径到系统路径 + +from DDPG.env import OUNoise + +def train(cfg, env, agent): + print('开始训练！') + print(f'环境：{cfg.env_name}，算法：{cfg.algo}，设备：{cfg.device}') + ou_noise = OUNoise(env.action_space) # 动作噪声 + rewards = [] # 记录所有回合的奖励 + ma_rewards = [] # 记录所有回合的滑动平均奖励 + for i_ep in range(cfg.train_eps): + state = env.reset() + ou_noise.reset() + done = False + ep_reward = 0 + i_step = 0 + while not done: + i_step += 1 + action = agent.choose_action(state) + action = ou_noise.get_action(action, i_step) + next_state, reward, done, _ = env.step(action) + ep_reward += reward + agent.memory.push(state, action, reward, next_state, done) + agent.update() + state = next_state + if (i_ep+1)%10 == 0: + print('回合：{}/{}，奖励：{:.2f}'.format(i_ep+1, cfg.train_eps, ep_reward)) + rewards.append(ep_reward) + if ma_rewards: + ma_rewards.append(0.9*ma_rewards[-1]+0.1*ep_reward) + else: + ma_rewards.append(ep_reward) + print('完成训练！') + return rewards, ma_rewards + +def test(cfg, env, agent): + print('开始测试！') + print(f'环境：{cfg.env_name}, 算法：{cfg.algo}, 设备：{cfg.device}') + rewards = [] # 记录所有回合的奖励 + ma_rewards = [] # 记录所有回合的滑动平均奖励 + for i_ep in range(cfg.eval_eps): + state = env.reset() + done = False + ep_reward = 0 + i_step = 0 + while not done: + i_step += 1 + action = agent.choose_action(state) + next_state, reward, done, _ = env.step(action) + ep_reward += reward + state = next_state + print('回合：{}/{}, 奖励：{}'.format(i_ep+1, cfg.train_eps, ep_reward)) + rewards.append(ep_reward) + if ma_rewards: + ma_rewards.append(0.9*ma_rewards[-1]+0.1*ep_reward) + else: + ma_rewards.append(ep_reward) + print(f"回合：{i_ep+1}/{cfg.eval_eps}，奖励：{ep_reward:.1f}") + print('完成测试！') + return rewards, ma_rewards \ No newline at end of file diff --git a/codes/DQN/agent.py b/codes/DQN/agent.py index 27845d2..2e1e5de 100644 --- a/codes/DQN/agent.py +++ b/codes/DQN/agent.py @@ -14,16 +14,57 @@ LastEditTime: 2021-09-15 13:35:36 import torch import torch.nn as nn +import torch.nn.functional as F import torch.optim as optim import random import math import numpy as np -from common.memory import ReplayBuffer -from common.model import MLP -class DQN: - def __init__(self, n_states, n_actions, cfg): - self.n_actions = n_actions # 总的动作个数 +class MLP(nn.Module): + def __init__(self, state_dim,action_dim,hidden_dim=128): + """ 初始化q网络，为全连接网络 + state_dim: 输入的特征数即环境的状态数 + action_dim: 输出的动作维度 + """ + super(MLP, self).__init__() + self.fc1 = nn.Linear(state_dim, hidden_dim) # 输入层 + self.fc2 = nn.Linear(hidden_dim,hidden_dim) # 隐藏层 + self.fc3 = nn.Linear(hidden_dim, action_dim) # 输出层 + + def forward(self, x): + # 各层对应的激活函数 + x = F.relu(self.fc1(x)) + x = F.relu(self.fc2(x)) + return self.fc3(x) + +class ReplayBuffer: + def __init__(self, capacity): + self.capacity = capacity # 经验回放的容量 + self.buffer = [] # 缓冲区 + self.position = 0 + + def push(self, state, action, reward, next_state, done): + ''' 缓冲区是一个队列，容量超出时去掉开始存入的转移(transition) + ''' + if len(self.buffer) < self.capacity: + self.buffer.append(None) + self.buffer[self.position] = (state, action, reward, next_state, done) + self.position = (self.position + 1) % self.capacity + + def sample(self, batch_size): + batch = random.sample(self.buffer, batch_size) # 随机采出小批量转移 + state, action, reward, next_state, done = zip(*batch) # 解压成状态，动作等 + return state, action, reward, next_state, done + + def __len__(self): + ''' 返回当前存储的量 + ''' + return len(self.buffer) + +class DQN: + def __init__(self, state_dim, action_dim, cfg): + + self.action_dim = action_dim # 总的动作个数 self.device = cfg.device # 设备，cpu或gpu等 self.gamma = cfg.gamma # 奖励的折扣因子 # e-greedy策略相关参数 @@ -32,8 +73,8 @@ class DQN: (cfg.epsilon_start - cfg.epsilon_end) * \ math.exp(-1. * frame_idx / cfg.epsilon_decay) self.batch_size = cfg.batch_size - self.policy_net = MLP(n_states, n_actions,hidden_dim=cfg.hidden_dim).to(self.device) - self.target_net = MLP(n_states, n_actions,hidden_dim=cfg.hidden_dim).to(self.device) + self.policy_net = MLP(state_dim, action_dim,hidden_dim=cfg.hidden_dim).to(self.device) + self.target_net = MLP(state_dim, action_dim,hidden_dim=cfg.hidden_dim).to(self.device) for target_param, param in zip(self.target_net.parameters(),self.policy_net.parameters()): # 复制参数到目标网路targe_net target_param.data.copy_(param.data) self.optimizer = optim.Adam(self.policy_net.parameters(), lr=cfg.lr) # 优化器 @@ -49,7 +90,7 @@ class DQN: q_values = self.policy_net(state) action = q_values.max(1)[1].item() # 选择Q值最大的动作 else: - action = random.randrange(self.n_actions) + action = random.randrange(self.action_dim) return action def update(self): if len(self.memory) < self.batch_size: # 当memory中不满足一个批量时，不更新策略 diff --git a/codes/DQN/task0.py b/codes/DQN/task0.py new file mode 100644 index 0000000..e4c326e --- /dev/null +++ b/codes/DQN/task0.py @@ -0,0 +1,75 @@ +import sys +import os +curr_path = os.path.dirname(os.path.abspath(__file__)) # 当前文件所在绝对路径 +parent_path = os.path.dirname(curr_path) # 父路径 +sys.path.append(parent_path) # 添加路径到系统路径 + +import gym +import torch +import datetime +from common.utils import save_results, make_dir +from common.utils import plot_rewards +from DQN.agent import DQN +from DQN.train import train,test + +curr_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") # 获取当前时间 +algo_name = "DQN" # 算法名称 +env_name = 'CartPole-v0' # 环境名称 + +class DQNConfig: + def __init__(self): + self.algo_name = algo_name # 算法名称 + self.env_name = env_name # 环境名称 + self.device = torch.device( + "cuda" if torch.cuda.is_available() else "cpu") # 检测GPU + self.train_eps = 200 # 训练的回合数 + self.eval_eps = 30 # 测试的回合数 + # 超参数 + self.gamma = 0.95 # 强化学习中的折扣因子 + self.epsilon_start = 0.90 # e-greedy策略中初始epsilon + self.epsilon_end = 0.01 # e-greedy策略中的终止epsilon + self.epsilon_decay = 500 # e-greedy策略中epsilon的衰减率 + self.lr = 0.0001 # 学习率 + self.memory_capacity = 100000 # 经验回放的容量 + self.batch_size = 64 # mini-batch SGD中的批量大小 + self.target_update = 4 # 目标网络的更新频率 + self.hidden_dim = 256 # 网络隐藏层 +class PlotConfig: + def __init__(self) -> None: + self.algo = algo_name # 算法名称 + self.env_name = env_name # 环境名称 + self.device = torch.device( + "cuda" if torch.cuda.is_available() else "cpu") # 检测GPU + self.result_path = curr_path + "/outputs/" + self.env_name + \ + '/' + curr_time + '/results/' # 保存结果的路径 + self.model_path = curr_path + "/outputs/" + self.env_name + \ + '/' + curr_time + '/models/' # 保存模型的路径 + self.save = True # 是否保存图片 + +def env_agent_config(cfg, seed=1): + ''' 创建环境和智能体 + ''' + env = gym.make(cfg.env_name) # 创建环境 + env.seed(seed) # 设置随机种子 + state_dim = env.observation_space.shape[0] # 状态数 + action_dim = env.action_space.n # 动作数 + agent = DQN(state_dim, action_dim, cfg) # 创建智能体 + return env, agent + + +cfg = DQNConfig() +plot_cfg = PlotConfig() +# 训练 +env, agent = env_agent_config(cfg, seed=1) +rewards, ma_rewards = train(cfg, env, agent) +make_dir(plot_cfg.result_path, plot_cfg.model_path) # 创建保存结果和模型路径的文件夹 +agent.save(path=plot_cfg.model_path) # 保存模型 +save_results(rewards, ma_rewards, tag='train', + path=plot_cfg.result_path) # 保存结果 +plot_rewards(rewards, ma_rewards, plot_cfg, tag="train") # 画出结果 +# 测试 +env, agent = env_agent_config(cfg, seed=10) +agent.load(path=plot_cfg.model_path) # 导入模型 +rewards, ma_rewards = test(cfg, env, agent) +save_results(rewards, ma_rewards, tag='test', path=plot_cfg.result_path) # 保存结果 +plot_rewards(rewards, ma_rewards, plot_cfg, tag="test") # 画出结果 diff --git a/codes/DQN/task1.py b/codes/DQN/task1.py new file mode 100644 index 0000000..d85a2ef --- /dev/null +++ b/codes/DQN/task1.py @@ -0,0 +1,83 @@ +import sys +import os +curr_path = os.path.dirname(os.path.abspath(__file__)) # 当前文件所在绝对路径 +parent_path = os.path.dirname(curr_path) # 父路径 +sys.path.append(parent_path) # 添加路径到系统路径 + +import gym +import torch +import datetime +from common.utils import save_results, make_dir +from common.utils import plot_rewards, plot_rewards_cn +from DQN.agent import DQN +from DQN.train import train,test + + +curr_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") # 获取当前时间 +algo_name = "DQN" # 算法名称 +env_name = 'CartPole-v1' # 环境名称 +class DQNConfig: + ''' 算法相关参数设置 + ''' + + def __init__(self): + self.algo_name = algo_name # 算法名称 + self.env_name = env_name # 环境名称 + self.device = torch.device( + "cuda" if torch.cuda.is_available() else "cpu") # 检测GPU + self.train_eps = 200 # 训练的回合数 + self.eval_eps = 30 # 测试的回合数 + # 超参数 + self.gamma = 0.95 # 强化学习中的折扣因子 + self.epsilon_start = 0.90 # e-greedy策略中初始epsilon + self.epsilon_end = 0.01 # e-greedy策略中的终止epsilon + self.epsilon_decay = 500 # e-greedy策略中epsilon的衰减率 + self.lr = 0.0001 # 学习率 + self.memory_capacity = 100000 # 经验回放的容量 + self.batch_size = 64 # mini-batch SGD中的批量大小 + self.target_update = 4 # 目标网络的更新频率 + self.hidden_dim = 256 # 网络隐藏层 +class PlotConfig: + ''' 绘图相关参数设置 + ''' + + def __init__(self) -> None: + self.algo_name = algo_name # 算法名称 + self.env_name = env_name # 环境名称 + self.device = torch.device( + "cuda" if torch.cuda.is_available() else "cpu") # 检测GPU + self.result_path = curr_path + "/outputs/" + self.env_name + \ + '/' + curr_time + '/results/' # 保存结果的路径 + self.model_path = curr_path + "/outputs/" + self.env_name + \ + '/' + curr_time + '/models/' # 保存模型的路径 + self.save = True # 是否保存图片 + + +def env_agent_config(cfg, seed=1): + ''' 创建环境和智能体 + ''' + env = gym.make(cfg.env_name) # 创建环境 + env.seed(seed) # 设置随机种子 + state_dim = env.observation_space.shape[0] # 状态数 + action_dim = env.action_space.n # 动作数 + agent = DQN(state_dim, action_dim, cfg) # 创建智能体 + return env, agent + + +cfg = DQNConfig() +plot_cfg = PlotConfig() +# 训练 +env, agent = env_agent_config(cfg, seed=1) +rewards, ma_rewards = train(cfg, env, agent) +make_dir(plot_cfg.result_path, plot_cfg.model_path) # 创建保存结果和模型路径的文件夹 +agent.save(path=plot_cfg.model_path) # 保存模型 +save_results(rewards, ma_rewards, tag='train', + path=plot_cfg.result_path) # 保存结果 +plot_rewards_cn(rewards, ma_rewards, plot_cfg, tag="train") # 画出结果 +# 测试 +env, agent = env_agent_config(cfg, seed=10) +agent.load(path=plot_cfg.model_path) # 导入模型 +rewards, ma_rewards = test(cfg, env, agent) +save_results(rewards, ma_rewards, tag='test', + path=plot_cfg.result_path) # 保存结果 +plot_rewards_cn(rewards, ma_rewards, plot_cfg, tag="test") # 画出结果 diff --git a/codes/DQN/task0_train.ipynb b/codes/DQN/train.ipynb similarity index 98% rename from codes/DQN/task0_train.ipynb rename to codes/DQN/train.ipynb index 464e216..ba4308e 100644 --- a/codes/DQN/task0_train.ipynb +++ b/codes/DQN/train.ipynb @@ -38,15 +38,15 @@ "outputs": [], "source": [ "class MLP(nn.Module):\n", - " def __init__(self, n_states,n_actions,hidden_dim=128):\n", + " def __init__(self, state_dim,action_dim,hidden_dim=128):\n", " \"\"\" 初始化q网络，为全连接网络\n", - " n_states: 输入的特征数即环境的状态数\n", - " n_actions: 输出的动作维度\n", + " state_dim: 输入的特征数即环境的状态数\n", + " action_dim: 输出的动作维度\n", " \"\"\"\n", " super(MLP, self).__init__()\n", - " self.fc1 = nn.Linear(n_states, hidden_dim) # 输入层\n", + " self.fc1 = nn.Linear(state_dim, hidden_dim) # 输入层\n", " self.fc2 = nn.Linear(hidden_dim,hidden_dim) # 隐藏层\n", - " self.fc3 = nn.Linear(hidden_dim, n_actions) # 输出层\n", + " self.fc3 = nn.Linear(hidden_dim, action_dim) # 输出层\n", " \n", " def forward(self, x):\n", " # 各层对应的激活函数\n", @@ -107,9 +107,9 @@ "outputs": [], "source": [ "class DQN:\n", - " def __init__(self, n_states, n_actions, cfg):\n", + " def __init__(self, state_dim, action_dim, cfg):\n", "\n", - " self.n_actions = n_actions # 总的动作个数\n", + " self.action_dim = action_dim # 总的动作个数\n", " self.device = cfg.device # 设备，cpu或gpu等\n", " self.gamma = cfg.gamma # 奖励的折扣因子\n", " # e-greedy策略相关参数\n", @@ -118,8 +118,8 @@ " (cfg.epsilon_start - cfg.epsilon_end) * \\\n", " math.exp(-1. * frame_idx / cfg.epsilon_decay)\n", " self.batch_size = cfg.batch_size\n", - " self.policy_net = MLP(n_states, n_actions,hidden_dim=cfg.hidden_dim).to(self.device)\n", - " self.target_net = MLP(n_states, n_actions,hidden_dim=cfg.hidden_dim).to(self.device)\n", + " self.policy_net = MLP(state_dim, action_dim,hidden_dim=cfg.hidden_dim).to(self.device)\n", + " self.target_net = MLP(state_dim, action_dim,hidden_dim=cfg.hidden_dim).to(self.device)\n", " for target_param, param in zip(self.target_net.parameters(),self.policy_net.parameters()): # 复制参数到目标网路targe_net\n", " target_param.data.copy_(param.data)\n", " self.optimizer = optim.Adam(self.policy_net.parameters(), lr=cfg.lr) # 优化器\n", @@ -135,7 +135,7 @@ " q_values = self.policy_net(state)\n", " action = q_values.max(1)[1].item() # 选择Q值最大的动作\n", " else:\n", - " action = random.randrange(self.n_actions)\n", + " action = random.randrange(self.action_dim)\n", " return action\n", " def update(self):\n", " if len(self.memory) < self.batch_size: # 当memory中不满足一个批量时，不更新策略\n", @@ -211,9 +211,9 @@ " '''\n", " env = gym.make(cfg.env) # 创建环境\n", " env.seed(seed) # 设置随机种子\n", - " n_states = env.observation_space.shape[0] # 状态数\n", - " n_actions = env.action_space.n # 动作数\n", - " agent = DQN(n_states,n_actions,cfg) # 创建智能体\n", + " state_dim = env.observation_space.shape[0] # 状态数\n", + " action_dim = env.action_space.n # 动作数\n", + " agent = DQN(state_dim,action_dim,cfg) # 创建智能体\n", " return env,agent" ] }, diff --git a/codes/DQN/task0_train.py b/codes/DQN/train.py similarity index 52% rename from codes/DQN/task0_train.py rename to codes/DQN/train.py index 5fd0ccd..4f8510e 100644 --- a/codes/DQN/task0_train.py +++ b/codes/DQN/train.py @@ -9,63 +9,11 @@ LastEditTime: 2021-09-15 15:34:13 @Discription: @Environment: python 3.7.7 ''' -import sys,os -curr_path = os.path.dirname(os.path.abspath(__file__)) # 当前文件所在绝对路径 -parent_path = os.path.dirname(curr_path) # 父路径 -sys.path.append(parent_path) # 添加路径到系统路径 - -import gym -import torch -import datetime - -from common.utils import save_results, make_dir -from common.plot import plot_rewards -from DQN.agent import DQN - -curr_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") # 获取当前时间 -class DQNConfig: - def __init__(self): - self.algo = "DQN" # 算法名称 - self.env_name = 'CartPole-v0' # 环境名称 - self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # 检测GPU - self.train_eps = 200 # 训练的回合数 - self.eval_eps = 30 # 测试的回合数 - # 超参数 - self.gamma = 0.95 # 强化学习中的折扣因子 - self.epsilon_start = 0.90 # e-greedy策略中初始epsilon - self.epsilon_end = 0.01 # e-greedy策略中的终止epsilon - self.epsilon_decay = 500 # e-greedy策略中epsilon的衰减率 - self.lr = 0.0001 # 学习率 - self.memory_capacity = 100000 # 经验回放的容量 - self.batch_size = 64 # mini-batch SGD中的批量大小 - self.target_update = 4 # 目标网络的更新频率 - self.hidden_dim = 256 # 网络隐藏层 -class PlotConfig: - def __init__(self) -> None: - self.algo = "DQN" # 算法名称 - self.env_name = 'CartPole-v0' # 环境名称 - self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # 检测GPU - self.result_path = curr_path+"/outputs/" + self.env_name + \ - '/'+curr_time+'/results/' # 保存结果的路径 - self.model_path = curr_path+"/outputs/" + self.env_name + \ - '/'+curr_time+'/models/' # 保存模型的路径 - self.save = True # 是否保存图片 - -def env_agent_config(cfg,seed=1): - ''' 创建环境和智能体 - ''' - env = gym.make(cfg.env_name) # 创建环境 - env.seed(seed) # 设置随机种子 - n_states = env.observation_space.shape[0] # 状态数 - n_actions = env.action_space.n # 动作数 - agent = DQN(n_states,n_actions,cfg) # 创建智能体 - return env,agent - def train(cfg, env, agent): ''' 训练 ''' print('开始训练!') - print(f'环境：{cfg.env_name}, 算法：{cfg.algo}, 设备：{cfg.device}') + print(f'环境：{cfg.env_name}, 算法：{cfg.algo_name}, 设备：{cfg.device}') rewards = [] # 记录所有回合的奖励 ma_rewards = [] # 记录所有回合的滑动平均奖励 for i_ep in range(cfg.train_eps): @@ -92,9 +40,9 @@ def train(cfg, env, agent): print('完成训练！') return rewards, ma_rewards -def eval(cfg,env,agent): +def test(cfg,env,agent): print('开始测试!') - print(f'环境：{cfg.env_name}, 算法：{cfg.algo}, 设备：{cfg.device}') + print(f'环境：{cfg.env_name}, 算法：{cfg.algo_name}, 设备：{cfg.device}') # 由于测试不需要使用epsilon-greedy策略，所以相应的值设置为0 cfg.epsilon_start = 0.0 # e-greedy策略中初始epsilon cfg.epsilon_end = 0.0 # e-greedy策略中的终止epsilon @@ -115,11 +63,64 @@ def eval(cfg,env,agent): ma_rewards.append(ma_rewards[-1]*0.9+ep_reward*0.1) else: ma_rewards.append(ep_reward) - print(f"回合：{i_ep+1}/{cfg.eval_eps}, 奖励：{ep_reward:.1f}") + print(f"回合：{i_ep+1}/{cfg.eval_eps}，奖励：{ep_reward:.1f}") print('完成测试！') return rewards,ma_rewards if __name__ == "__main__": + import sys,os + curr_path = os.path.dirname(os.path.abspath(__file__)) # 当前文件所在绝对路径 + parent_path = os.path.dirname(curr_path) # 父路径 + sys.path.append(parent_path) # 添加路径到系统路径 + + import gym + import torch + import datetime + + from common.utils import save_results, make_dir + from common.utils import plot_rewards + from DQN.agent import DQN + from DQN.train import train + + curr_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") # 获取当前时间 + class DQNConfig: + def __init__(self): + self.algo = "DQN" # 算法名称 + self.env_name = 'CartPole-v0' # 环境名称 + self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # 检测GPU + self.train_eps = 200 # 训练的回合数 + self.eval_eps = 30 # 测试的回合数 + # 超参数 + self.gamma = 0.95 # 强化学习中的折扣因子 + self.epsilon_start = 0.90 # e-greedy策略中初始epsilon + self.epsilon_end = 0.01 # e-greedy策略中的终止epsilon + self.epsilon_decay = 500 # e-greedy策略中epsilon的衰减率 + self.lr = 0.0001 # 学习率 + self.memory_capacity = 100000 # 经验回放的容量 + self.batch_size = 64 # mini-batch SGD中的批量大小 + self.target_update = 4 # 目标网络的更新频率 + self.hidden_dim = 256 # 网络隐藏层 + class PlotConfig: + def __init__(self) -> None: + self.algo = "DQN" # 算法名称 + self.env_name = 'CartPole-v0' # 环境名称 + self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # 检测GPU + self.result_path = curr_path+"/outputs/" + self.env_name + \ + '/'+curr_time+'/results/' # 保存结果的路径 + self.model_path = curr_path+"/outputs/" + self.env_name + \ + '/'+curr_time+'/models/' # 保存模型的路径 + self.save = True # 是否保存图片 + + def env_agent_config(cfg,seed=1): + ''' 创建环境和智能体 + ''' + env = gym.make(cfg.env_name) # 创建环境 + env.seed(seed) # 设置随机种子 + state_dim = env.observation_space.shape[0] # 状态数 + action_dim = env.action_space.n # 动作数 + agent = DQN(state_dim,action_dim,cfg) # 创建智能体 + return env,agent + cfg = DQNConfig() plot_cfg = PlotConfig() # 训练 @@ -132,6 +133,6 @@ if __name__ == "__main__": # 测试 env,agent = env_agent_config(cfg,seed=10) agent.load(path=plot_cfg.model_path) # 导入模型 - rewards,ma_rewards = eval(cfg,env,agent) - save_results(rewards,ma_rewards,tag='eval',path=plot_cfg.result_path) # 保存结果 - plot_rewards(rewards,ma_rewards, plot_cfg, tag="eval") # 画出结果 + rewards,ma_rewards = test(cfg,env,agent) + save_results(rewards,ma_rewards,tag='test',path=plot_cfg.result_path) # 保存结果 + plot_rewards(rewards,ma_rewards, plot_cfg, tag="test") # 画出结果 \ No newline at end of file diff --git a/codes/Docs/使用DQN解决推车杆问题.md b/codes/Docs/使用DQN解决推车杆问题.md index 5889165..ac56ac6 100644 --- a/codes/Docs/使用DQN解决推车杆问题.md +++ b/codes/Docs/使用DQN解决推车杆问题.md @@ -14,10 +14,10 @@ CartPole-v0是一个经典的入门环境，如下图，它通过向左(动作=0 import gym env = gym.make('CartPole-v0') # 建立环境 env.seed(1) # 随机种子 -n_states = env.observation_space.shape[0] # 状态数 -n_actions = env.action_space.n # 动作数 +state_dim = env.observation_space.shape[0] # 状态数 +action_dim = env.action_space.n # 动作数 state = env.reset() # 初始化环境 -print(f"状态数：{n_states}，动作数：{n_actions}") +print(f"状态数：{state_dim}，动作数：{action_dim}") print(f"初始状态：{state}") ``` diff --git a/codes/Docs/使用Q-learning解决悬崖寻路问题.md b/codes/Docs/使用Q-learning解决悬崖寻路问题.md index 244d85b..ac25945 100644 --- a/codes/Docs/使用Q-learning解决悬崖寻路问题.md +++ b/codes/Docs/使用Q-learning解决悬崖寻路问题.md @@ -30,9 +30,9 @@ env = CliffWalkingWapper(env) # 装饰环境 这里我们在程序中使用了一个装饰器重新定义环境，但不影响对环境的理解，感兴趣的同学具体看相关代码。可以由于gym环境封装得比较好，所以我们想要使用这个环境只需要使用gym.make命令输入函数名即可，然后我们可以查看环境的状态和动作数目： ```python -n_states = env.observation_space.n # 状态数 -n_actions = env.action_space.n # 动作数 -print(f"状态数：{n_states}，动作数：{n_actions}") +state_dim = env.observation_space.n # 状态数 +action_dim = env.action_space.n # 动作数 +print(f"状态数：{state_dim}，动作数：{action_dim}") ``` 打印出来的结果如下： @@ -72,9 +72,9 @@ print(state) env = gym.make('CliffWalking-v0') # 定义环境 env = CliffWalkingWapper(env) # 装饰环境 env.seed(1) # 设置随机种子 -n_states = env.observation_space.n # 状态数 -n_actions = env.action_space.n # 动作数 -agent = QLearning(n_states,n_actions,cfg) # cfg存储算法相关参数 +state_dim = env.observation_space.n # 状态数 +action_dim = env.action_space.n # 动作数 +agent = QLearning(state_dim,action_dim,cfg) # cfg存储算法相关参数 for i_ep in range(cfg.train_eps): # cfg.train_eps表示最大训练的回合数 ep_reward = 0 # 记录每个回合的奖励 state = env.reset() # 重置环境 diff --git a/codes/DoubleDQN/agent.py b/codes/DoubleDQN/agent.py index 1ade5f8..7b26fa1 100644 --- a/codes/DoubleDQN/agent.py +++ b/codes/DoubleDQN/agent.py @@ -5,7 +5,7 @@ @Email: johnjim0816@gmail.com @Date: 2020-06-12 00:50:49 @LastEditor: John -LastEditTime: 2021-05-04 22:28:06 +LastEditTime: 2021-11-19 18:07:09 @Discription: @Environment: python 3.7.7 ''' @@ -16,15 +16,55 @@ LastEditTime: 2021-05-04 22:28:06 import torch import torch.nn as nn import torch.optim as optim +import torch.nn.functional as F import random import math import numpy as np -from common.memory import ReplayBuffer -from common.model import MLP -class DoubleDQN: - def __init__(self, state_dim, action_dim, cfg): + +class ReplayBuffer: + def __init__(self, capacity): + self.capacity = capacity # 经验回放的容量 + self.buffer = [] # 缓冲区 + self.position = 0 + + def push(self, state, action, reward, next_state, done): + ''' 缓冲区是一个队列，容量超出时去掉开始存入的转移(transition) + ''' + if len(self.buffer) < self.capacity: + self.buffer.append(None) + self.buffer[self.position] = (state, action, reward, next_state, done) + self.position = (self.position + 1) % self.capacity + + def sample(self, batch_size): + batch = random.sample(self.buffer, batch_size) # 随机采出小批量转移 + state, action, reward, next_state, done = zip(*batch) # 解压成状态，动作等 + return state, action, reward, next_state, done + + def __len__(self): + ''' 返回当前存储的量 + ''' + return len(self.buffer) + +class MLP(nn.Module): + def __init__(self, n_states,n_actions,hidden_dim=128): + """ 初始化q网络，为全连接网络 + n_states: 输入的特征数即环境的状态数 + n_actions: 输出的动作维度 + """ + super(MLP, self).__init__() + self.fc1 = nn.Linear(n_states, hidden_dim) # 输入层 + self.fc2 = nn.Linear(hidden_dim,hidden_dim) # 隐藏层 + self.fc3 = nn.Linear(hidden_dim, n_actions) # 输出层 - self.action_dim = action_dim # 总的动作个数 + def forward(self, x): + # 各层对应的激活函数 + x = F.relu(self.fc1(x)) + x = F.relu(self.fc2(x)) + return self.fc3(x) + +class DoubleDQN: + def __init__(self, n_states, n_actions, cfg): + self.n_actions = n_actions # 总的动作个数 self.device = cfg.device # 设备，cpu或gpu等 self.gamma = cfg.gamma # e-greedy策略相关参数 @@ -33,8 +73,8 @@ class DoubleDQN: self.epsilon_end = cfg.epsilon_end self.epsilon_decay = cfg.epsilon_decay self.batch_size = cfg.batch_size - self.policy_net = MLP(state_dim, action_dim,hidden_dim=cfg.hidden_dim).to(self.device) - self.target_net = MLP(state_dim, action_dim,hidden_dim=cfg.hidden_dim).to(self.device) + self.policy_net = MLP(n_states, n_actions,hidden_dim=cfg.hidden_dim).to(self.device) + self.target_net = MLP(n_states, n_actions,hidden_dim=cfg.hidden_dim).to(self.device) # target_net copy from policy_net for target_param, param in zip(self.target_net.parameters(), self.policy_net.parameters()): target_param.data.copy_(param.data) @@ -43,8 +83,15 @@ class DoubleDQN: self.optimizer = optim.Adam(self.policy_net.parameters(), lr=cfg.lr) self.loss = 0 self.memory = ReplayBuffer(cfg.memory_capacity) - def predict(self,state): - with torch.no_grad(): + + def choose_action(self, state): + '''选择动作 + ''' + self.actions_count += 1 + self.epsilon = self.epsilon_end + (self.epsilon_start - self.epsilon_end) * \ + math.exp(-1. * self.actions_count / self.epsilon_decay) + if random.random() > self.epsilon: + with torch.no_grad(): # 先转为张量便于丢给神经网络,state元素数据原本为float64 # 注意state=torch.tensor(state).unsqueeze(0)跟state=torch.tensor([state])等价 state = torch.tensor( @@ -55,17 +102,8 @@ class DoubleDQN: # 如torch.return_types.max(values=tensor([10.3587]),indices=tensor([0])) # 所以tensor.max(1)[1]返回最大值对应的下标，即action action = q_value.max(1)[1].item() - return action - def choose_action(self, state): - '''选择动作 - ''' - self.actions_count += 1 - self.epsilon = self.epsilon_end + (self.epsilon_start - self.epsilon_end) * \ - math.exp(-1. * self.actions_count / self.epsilon_decay) - if random.random() > self.epsilon: - action = self.predict(state) else: - action = random.randrange(self.action_dim) + action = random.randrange(self.n_actions) return action def update(self): diff --git a/codes/DoubleDQN/outputs/CartPole-v0/20210504-150900/models/checkpoint.pth b/codes/DoubleDQN/outputs/CartPole-v0/20210504-150900/models/checkpoint.pth deleted file mode 100644 index 8c4b561..0000000 Binary files a/codes/DoubleDQN/outputs/CartPole-v0/20210504-150900/models/checkpoint.pth and /dev/null differ diff --git a/codes/DoubleDQN/outputs/CartPole-v0/20210504-150900/results/eval_ma_rewards.npy b/codes/DoubleDQN/outputs/CartPole-v0/20210504-150900/results/eval_ma_rewards.npy deleted file mode 100644 index 0f77696..0000000 Binary files a/codes/DoubleDQN/outputs/CartPole-v0/20210504-150900/results/eval_ma_rewards.npy and /dev/null differ diff --git a/codes/DoubleDQN/outputs/CartPole-v0/20210504-150900/results/eval_rewards.npy b/codes/DoubleDQN/outputs/CartPole-v0/20210504-150900/results/eval_rewards.npy deleted file mode 100644 index 57f8759..0000000 Binary files a/codes/DoubleDQN/outputs/CartPole-v0/20210504-150900/results/eval_rewards.npy and /dev/null differ diff --git a/codes/DoubleDQN/outputs/CartPole-v0/20210504-150900/results/eval_rewards_curve.png b/codes/DoubleDQN/outputs/CartPole-v0/20210504-150900/results/eval_rewards_curve.png deleted file mode 100644 index 038e031..0000000 Binary files a/codes/DoubleDQN/outputs/CartPole-v0/20210504-150900/results/eval_rewards_curve.png and /dev/null differ diff --git a/codes/DoubleDQN/outputs/CartPole-v0/20210504-150900/results/train_ma_rewards.npy b/codes/DoubleDQN/outputs/CartPole-v0/20210504-150900/results/train_ma_rewards.npy deleted file mode 100644 index 63d10e7..0000000 Binary files a/codes/DoubleDQN/outputs/CartPole-v0/20210504-150900/results/train_ma_rewards.npy and /dev/null differ diff --git a/codes/DoubleDQN/outputs/CartPole-v0/20210504-150900/results/train_rewards.npy b/codes/DoubleDQN/outputs/CartPole-v0/20210504-150900/results/train_rewards.npy deleted file mode 100644 index d486ad9..0000000 Binary files a/codes/DoubleDQN/outputs/CartPole-v0/20210504-150900/results/train_rewards.npy and /dev/null differ diff --git a/codes/DoubleDQN/outputs/CartPole-v0/20210504-150900/results/train_rewards_curve.png b/codes/DoubleDQN/outputs/CartPole-v0/20210504-150900/results/train_rewards_curve.png deleted file mode 100644 index f91bc4d..0000000 Binary files a/codes/DoubleDQN/outputs/CartPole-v0/20210504-150900/results/train_rewards_curve.png and /dev/null differ diff --git a/codes/DoubleDQN/outputs/CartPole-v0/20211221-185355/models/checkpoint.pth b/codes/DoubleDQN/outputs/CartPole-v0/20211221-185355/models/checkpoint.pth new file mode 100644 index 0000000..fc1ca66 Binary files /dev/null and b/codes/DoubleDQN/outputs/CartPole-v0/20211221-185355/models/checkpoint.pth differ diff --git a/codes/DoubleDQN/outputs/CartPole-v0/20211221-185355/results/test_ma_rewards.npy b/codes/DoubleDQN/outputs/CartPole-v0/20211221-185355/results/test_ma_rewards.npy new file mode 100644 index 0000000..b32c0a8 Binary files /dev/null and b/codes/DoubleDQN/outputs/CartPole-v0/20211221-185355/results/test_ma_rewards.npy differ diff --git a/codes/DoubleDQN/outputs/CartPole-v0/20211221-185355/results/test_rewards.npy b/codes/DoubleDQN/outputs/CartPole-v0/20211221-185355/results/test_rewards.npy new file mode 100644 index 0000000..9ccf4e9 Binary files /dev/null and b/codes/DoubleDQN/outputs/CartPole-v0/20211221-185355/results/test_rewards.npy differ diff --git a/codes/DoubleDQN/outputs/CartPole-v0/20211221-185355/results/test_rewards_curve.png b/codes/DoubleDQN/outputs/CartPole-v0/20211221-185355/results/test_rewards_curve.png new file mode 100644 index 0000000..3580ef9 Binary files /dev/null and b/codes/DoubleDQN/outputs/CartPole-v0/20211221-185355/results/test_rewards_curve.png differ diff --git a/codes/DoubleDQN/outputs/CartPole-v0/20211221-185355/results/train_ma_rewards.npy b/codes/DoubleDQN/outputs/CartPole-v0/20211221-185355/results/train_ma_rewards.npy new file mode 100644 index 0000000..b0838ab Binary files /dev/null and b/codes/DoubleDQN/outputs/CartPole-v0/20211221-185355/results/train_ma_rewards.npy differ diff --git a/codes/DoubleDQN/outputs/CartPole-v0/20211221-185355/results/train_rewards.npy b/codes/DoubleDQN/outputs/CartPole-v0/20211221-185355/results/train_rewards.npy new file mode 100644 index 0000000..12e1347 Binary files /dev/null and b/codes/DoubleDQN/outputs/CartPole-v0/20211221-185355/results/train_rewards.npy differ diff --git a/codes/DoubleDQN/outputs/CartPole-v0/20211221-185355/results/train_rewards_curve.png b/codes/DoubleDQN/outputs/CartPole-v0/20211221-185355/results/train_rewards_curve.png new file mode 100644 index 0000000..d612a6a Binary files /dev/null and b/codes/DoubleDQN/outputs/CartPole-v0/20211221-185355/results/train_rewards_curve.png differ diff --git a/codes/DoubleDQN/task0.py b/codes/DoubleDQN/task0.py new file mode 100644 index 0000000..4fe9579 --- /dev/null +++ b/codes/DoubleDQN/task0.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python +# coding=utf-8 +''' +Author: JiangJi +Email: johnjim0816@gmail.com +Date: 2021-11-07 18:10:37 +LastEditor: JiangJi +LastEditTime: 2021-11-19 18:34:05 +Discription: +''' + +import sys,os +curr_path = os.path.dirname(os.path.abspath(__file__)) # 当前文件所在绝对路径 +parent_path = os.path.dirname(curr_path) # 父路径 +sys.path.append(parent_path) # 添加路径到系统路径 + +import gym +import torch +import datetime + +from common.utils import save_results, make_dir +from common.utils import plot_rewards +from DoubleDQN.agent import DoubleDQN +from DoubleDQN.train import train,test + +curr_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") # 获取当前时间 +algo_name = 'DoubleDQN' # 算法名称 +env_name = 'CartPole-v0' # 环境名称 +class DoubleDQNConfig: + def __init__(self): + self.algo_name = algo_name # 算法名称 + self.env_name = env_name # 环境名称 + self.device = torch.device( + "cuda" if torch.cuda.is_available() else "cpu") # 检测GPU + self.train_eps = 200 # 训练的回合数 + self.test_eps = 30 # 测试的回合数 + self.gamma = 0.95 # 强化学习中的折扣因子 + self.epsilon_start = 0.95 # e-greedy策略中初始epsilon + self.epsilon_end = 0.01 # e-greedy策略中的终止epsilon + self.epsilon_decay = 500 # e-greedy策略中epsilon的衰减率 + self.lr = 0.0001 # 学习率 + self.memory_capacity = 100000 # 经验回放的容量 + self.batch_size = 64 # mini-batch SGD中的批量大小 + self.target_update = 2 # 目标网络的更新频率 + self.hidden_dim = 256 # 网络隐藏层 +class PlotConfig: + ''' 绘图相关参数设置 + ''' + + def __init__(self) -> None: + self.algo_name = algo_name # 算法名称 + self.env_name = env_name # 环境名称 + self.device = torch.device( + "cuda" if torch.cuda.is_available() else "cpu") # 检测GPU + self.result_path = curr_path + "/outputs/" + self.env_name + \ + '/' + curr_time + '/results/' # 保存结果的路径 + self.model_path = curr_path + "/outputs/" + self.env_name + \ + '/' + curr_time + '/models/' # 保存模型的路径 + self.save = True # 是否保存图片 + +def env_agent_config(cfg,seed=1): + env = gym.make(cfg.env_name) + env.seed(seed) + n_states = env.observation_space.shape[0] + n_actions = env.action_space.n + agent = DoubleDQN(n_states,n_actions,cfg) + return env,agent + +cfg = DoubleDQNConfig() +plot_cfg = PlotConfig() +# 训练 +env,agent = env_agent_config(cfg,seed=1) +rewards, ma_rewards = train(cfg, env, agent) +make_dir(plot_cfg.result_path, plot_cfg.model_path) # 创建保存结果和模型路径的文件夹 +agent.save(path=plot_cfg.model_path) # 保存模型 +save_results(rewards, ma_rewards, tag='train', path=plot_cfg.result_path) # 保存结果 +plot_rewards(rewards, ma_rewards, plot_cfg, tag="train") # 画出结果 +# 测试 +env,agent = env_agent_config(cfg,seed=10) +agent.load(path=plot_cfg.model_path) # 导入模型 +rewards,ma_rewards = test(cfg,env,agent) +save_results(rewards, ma_rewards, tag='test', path=plot_cfg.result_path) # 保存结果 +plot_rewards(rewards, ma_rewards, plot_cfg, tag="test") # 画出结果 diff --git a/codes/DoubleDQN/task0_train.ipynb b/codes/DoubleDQN/task0_train.ipynb deleted file mode 100644 index ee2e5d4..0000000 --- a/codes/DoubleDQN/task0_train.ipynb +++ /dev/null @@ -1,194 +0,0 @@ -{ - "metadata": { - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.10" - }, - "orig_nbformat": 2, - "kernelspec": { - "name": "python3710jvsc74a57bd0366e1054dee9d4501b0eb8f87335afd3c67fc62db6ee611bbc7f8f5a1fefe232", - "display_name": "Python 3.7.10 64-bit ('py37': conda)" - }, - "metadata": { - "interpreter": { - "hash": "366e1054dee9d4501b0eb8f87335afd3c67fc62db6ee611bbc7f8f5a1fefe232" - } - } - }, - "nbformat": 4, - "nbformat_minor": 2, - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import sys\n", - "from pathlib import Path\n", - "curr_path = str(Path().absolute())\n", - "parent_path = str(Path().absolute().parent)\n", - "sys.path.append(parent_path) # add current terminal path to sys.path" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import gym\n", - "import torch\n", - "import datetime\n", - "from DoubleDQN.agent import DoubleDQN\n", - "from common.plot import plot_rewards\n", - "from common.utils import save_results, make_dir\n", - "\n", - "curr_time = datetime.datetime.now().strftime(\n", - " \"%Y%m%d-%H%M%S\") # obtain current time" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "class DoubleDQNConfig:\n", - " def __init__(self):\n", - " self.algo = \"DoubleDQN\" # name of algo\n", - " self.env = 'CartPole-v0' # env name\n", - " self.result_path = curr_path+\"/outputs/\" + self.env + \\\n", - " '/'+curr_time+'/results/' # path to save results\n", - " self.model_path = curr_path+\"/outputs/\" + self.env + \\\n", - " '/'+curr_time+'/models/' # path to save models\n", - " self.train_eps = 200 # max tranng episodes\n", - " self.eval_eps = 50 # max evaling episodes\n", - " self.gamma = 0.95\n", - " self.epsilon_start = 1 # start epsilon of e-greedy policy\n", - " self.epsilon_end = 0.01 \n", - " self.epsilon_decay = 500\n", - " self.lr = 0.001 # learning rate\n", - " self.memory_capacity = 100000 # capacity of Replay Memory\n", - " self.batch_size = 64\n", - " self.target_update = 2 # update frequency of target net\n", - " self.device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\") # check gpu\n", - " self.hidden_dim = 256 # hidden size of net" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def env_agent_config(cfg,seed=1):\n", - " env = gym.make(cfg.env) \n", - " env.seed(seed)\n", - " state_dim = env.observation_space.shape[0]\n", - " action_dim = env.action_space.n\n", - " agent = DoubleDQN(state_dim,action_dim,cfg)\n", - " return env,agent" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def train(cfg,env,agent):\n", - " print('Start to train !')\n", - " rewards,ma_rewards = [],[]\n", - " for i_ep in range(cfg.train_eps):\n", - " state = env.reset() \n", - " ep_reward = 0\n", - " while True:\n", - " action = agent.choose_action(state) \n", - " next_state, reward, done, _ = env.step(action)\n", - " ep_reward += reward\n", - " agent.memory.push(state, action, reward, next_state, done) \n", - " state = next_state \n", - " agent.update() \n", - " if done:\n", - " break\n", - " if i_ep % cfg.target_update == 0:\n", - " agent.target_net.load_state_dict(agent.policy_net.state_dict())\n", - " if (i_ep+1)%10 == 0:\n", - " print(f'Episode:{i_ep+1}/{cfg.train_eps}, Reward:{ep_reward}')\n", - " rewards.append(ep_reward)\n", - " if ma_rewards:\n", - " ma_rewards.append(\n", - " 0.9*ma_rewards[-1]+0.1*ep_reward)\n", - " else:\n", - " ma_rewards.append(ep_reward) \n", - " print('Complete training！')\n", - " return rewards,ma_rewards" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def eval(cfg,env,agent):\n", - " print('Start to eval !')\n", - " print(f'Env:{cfg.env}, Algorithm:{cfg.algo}, Device:{cfg.device}')\n", - " rewards = [] \n", - " ma_rewards = []\n", - " for i_ep in range(cfg.eval_eps):\n", - " state = env.reset() \n", - " ep_reward = 0 \n", - " while True:\n", - " action = agent.predict(state) \n", - " next_state, reward, done, _ = env.step(action) \n", - " state = next_state \n", - " ep_reward += reward\n", - " if done:\n", - " break\n", - " rewards.append(ep_reward)\n", - " if ma_rewards:\n", - " ma_rewards.append(ma_rewards[-1]*0.9+ep_reward*0.1)\n", - " else:\n", - " ma_rewards.append(ep_reward)\n", - " print(f\"Episode:{i_ep+1}/{cfg.eval_eps}, reward:{ep_reward:.1f}\")\n", - " print('Complete evaling！')\n", - " return rewards,ma_rewards " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "if __name__ == \"__main__\":\n", - " cfg = DoubleDQNConfig()\n", - " # train\n", - " env,agent = env_agent_config(cfg,seed=1)\n", - " rewards, ma_rewards = train(cfg, env, agent)\n", - " make_dir(cfg.result_path, cfg.model_path)\n", - " agent.save(path=cfg.model_path)\n", - " save_results(rewards, ma_rewards, tag='train', path=cfg.result_path)\n", - " plot_rewards(rewards, ma_rewards, tag=\"train\",\n", - " algo=cfg.algo, path=cfg.result_path)\n", - "\n", - " # eval\n", - " env,agent = env_agent_config(cfg,seed=10)\n", - " agent.load(path=cfg.model_path)\n", - " rewards,ma_rewards = eval(cfg,env,agent)\n", - " save_results(rewards,ma_rewards,tag='eval',path=cfg.result_path)\n", - " plot_rewards(rewards,ma_rewards,tag=\"eval\",env=cfg.env,algo = cfg.algo,path=cfg.result_path)" - ] - } - ] -} \ No newline at end of file diff --git a/codes/DoubleDQN/task0_train.py b/codes/DoubleDQN/task0_train.py deleted file mode 100644 index 0148ea2..0000000 --- a/codes/DoubleDQN/task0_train.py +++ /dev/null @@ -1,123 +0,0 @@ -#!/usr/bin/env python -# coding=utf-8 -''' -@Author: John -@Email: johnjim0816@gmail.com -@Date: 2020-06-12 00:48:57 -@LastEditor: John -LastEditTime: 2021-09-10 15:26:05 -@Discription: -@Environment: python 3.7.7 -''' -import sys,os -curr_path = os.path.dirname(__file__) -parent_path = os.path.dirname(curr_path) -sys.path.append(parent_path) # add current terminal path to sys.path - -import gym -import torch -import datetime -from DoubleDQN.agent import DoubleDQN -from common.plot import plot_rewards -from common.utils import save_results, make_dir - -curr_time = datetime.datetime.now().strftime( - "%Y%m%d-%H%M%S") # obtain current time - -class DoubleDQNConfig: - def __init__(self): - self.algo = "DoubleDQN" # name of algo - self.env = 'CartPole-v0' # env name - self.result_path = curr_path+"/outputs/" + self.env + \ - '/'+curr_time+'/results/' # path to save results - self.model_path = curr_path+"/outputs/" + self.env + \ - '/'+curr_time+'/models/' # path to save models - self.train_eps = 200 # max tranng episodes - self.eval_eps = 50 # max evaling episodes - self.gamma = 0.95 - self.epsilon_start = 1 # start epsilon of e-greedy policy - self.epsilon_end = 0.01 - self.epsilon_decay = 500 - self.lr = 0.001 # learning rate - self.memory_capacity = 100000 # capacity of Replay Memory - self.batch_size = 64 - self.target_update = 2 # update frequency of target net - self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # check gpu - self.hidden_dim = 256 # hidden size of net - -def env_agent_config(cfg,seed=1): - env = gym.make(cfg.env) - env.seed(seed) - state_dim = env.observation_space.shape[0] - action_dim = env.action_space.n - agent = DoubleDQN(state_dim,action_dim,cfg) - return env,agent - -def train(cfg,env,agent): - print('Start to train !') - rewards,ma_rewards = [],[] - for i_ep in range(cfg.train_eps): - state = env.reset() - ep_reward = 0 - while True: - action = agent.choose_action(state) - next_state, reward, done, _ = env.step(action) - ep_reward += reward - agent.memory.push(state, action, reward, next_state, done) - state = next_state - agent.update() - if done: - break - if i_ep % cfg.target_update == 0: - agent.target_net.load_state_dict(agent.policy_net.state_dict()) - print(f'Episode:{i_ep+1}/{cfg.train_eps}, Reward:{ep_reward},Epsilon:{agent.epsilon:.2f}') - rewards.append(ep_reward) - if ma_rewards: - ma_rewards.append( - 0.9*ma_rewards[-1]+0.1*ep_reward) - else: - ma_rewards.append(ep_reward) - print('Complete training！') - return rewards,ma_rewards - -def eval(cfg,env,agent): - print('Start to eval !') - print(f'Env:{cfg.env}, Algorithm:{cfg.algo}, Device:{cfg.device}') - rewards = [] - ma_rewards = [] - for i_ep in range(cfg.eval_eps): - state = env.reset() - ep_reward = 0 - while True: - action = agent.predict(state) - next_state, reward, done, _ = env.step(action) - state = next_state - ep_reward += reward - if done: - break - rewards.append(ep_reward) - if ma_rewards: - ma_rewards.append(ma_rewards[-1]*0.9+ep_reward*0.1) - else: - ma_rewards.append(ep_reward) - print(f"Episode:{i_ep+1}/{cfg.eval_eps}, reward:{ep_reward:.1f}") - print('Complete evaling！') - return rewards,ma_rewards - -if __name__ == "__main__": - cfg = DoubleDQNConfig() - # 训练 - env,agent = env_agent_config(cfg,seed=1) - rewards, ma_rewards = train(cfg, env, agent) - make_dir(cfg.result_path, cfg.model_path) - agent.save(path=cfg.model_path) - save_results(rewards, ma_rewards, tag='train', path=cfg.result_path) - plot_rewards(rewards, ma_rewards, tag="train", - algo=cfg.algo, path=cfg.result_path) - - # 测试 - env,agent = env_agent_config(cfg,seed=10) - agent.load(path=cfg.model_path) - rewards,ma_rewards = eval(cfg,env,agent) - save_results(rewards,ma_rewards,tag='eval',path=cfg.result_path) - plot_rewards(rewards,ma_rewards,tag="eval",env=cfg.env,algo = cfg.algo,path=cfg.result_path) diff --git a/codes/DoubleDQN/train.py b/codes/DoubleDQN/train.py new file mode 100644 index 0000000..ff0a786 --- /dev/null +++ b/codes/DoubleDQN/train.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python +# coding=utf-8 +''' +Author: JiangJi +Email: johnjim0816@gmail.com +Date: 2021-11-07 18:10:37 +LastEditor: JiangJi +LastEditTime: 2021-11-19 18:34:05 +Discription: +''' + +import sys,os +curr_path = os.path.dirname(os.path.abspath(__file__)) # 当前文件所在绝对路径 +parent_path = os.path.dirname(curr_path) # 父路径 +sys.path.append(parent_path) # 添加路径到系统路径 + +def train(cfg,env,agent): + print('开始训练!') + print(f'环境：{cfg.env_name}, 算法：{cfg.algo_name}, 设备：{cfg.device}') + rewards = [] # 记录所有回合的奖励 + ma_rewards = [] # 记录所有回合的滑动平均奖励 + for i_ep in range(cfg.train_eps): + ep_reward = 0 # 记录一回合内的奖励 + state = env.reset() # 重置环境，返回初始状态 + while True: + action = agent.choose_action(state) + next_state, reward, done, _ = env.step(action) + ep_reward += reward + agent.memory.push(state, action, reward, next_state, done) + state = next_state + agent.update() + if done: + break + if i_ep % cfg.target_update == 0: + agent.target_net.load_state_dict(agent.policy_net.state_dict()) + if (i_ep+1)%10 == 0: + print(f'回合：{i_ep+1}/{cfg.train_eps}，奖励：{ep_reward}') + rewards.append(ep_reward) + if ma_rewards: + ma_rewards.append( + 0.9*ma_rewards[-1]+0.1*ep_reward) + else: + ma_rewards.append(ep_reward) + print('完成训练！') + return rewards,ma_rewards + +def test(cfg,env,agent): + print('开始测试!') + print(f'环境：{cfg.env_name}, 算法：{cfg.algo_name}, 设备：{cfg.device}') + # 由于测试不需要使用epsilon-greedy策略，所以相应的值设置为0 + cfg.epsilon_start = 0.0 # e-greedy策略中初始epsilon + cfg.epsilon_end = 0.0 # e-greedy策略中的终止epsilon + rewards = [] # 记录所有回合的奖励 + ma_rewards = [] # 记录所有回合的滑动平均奖励 + for i_ep in range(cfg.test_eps): + state = env.reset() + ep_reward = 0 + while True: + action = agent.choose_action(state) + next_state, reward, done, _ = env.step(action) + state = next_state + ep_reward += reward + if done: + break + rewards.append(ep_reward) + if ma_rewards: + ma_rewards.append(ma_rewards[-1]*0.9+ep_reward*0.1) + else: + ma_rewards.append(ep_reward) + print(f"回合：{i_ep+1}/{cfg.test_eps}，奖励：{ep_reward:.1f}") + print('完成测试！') + return rewards,ma_rewards + diff --git a/codes/DuelingDQN/task0_train.ipynb b/codes/DuelingDQN/task0_train.ipynb index c2cd1c3..7e38218 100644 --- a/codes/DuelingDQN/task0_train.ipynb +++ b/codes/DuelingDQN/task0_train.ipynb @@ -136,12 +136,12 @@ "outputs": [], "source": [ "class DuelingNet(nn.Module):\n", - " def __init__(self, n_states, n_actions,hidden_size=128):\n", + " def __init__(self, state_dim, action_dim,hidden_size=128):\n", " super(DuelingNet, self).__init__()\n", " \n", " # 隐藏层\n", " self.hidden = nn.Sequential(\n", - " nn.Linear(n_states, hidden_size),\n", + " nn.Linear(state_dim, hidden_size),\n", " nn.ReLU()\n", " )\n", " \n", @@ -149,7 +149,7 @@ " self.advantage = nn.Sequential(\n", " nn.Linear(hidden_size, hidden_size),\n", " nn.ReLU(),\n", - " nn.Linear(hidden_size, n_actions)\n", + " nn.Linear(hidden_size, action_dim)\n", " )\n", " \n", " # 价值函数\n", @@ -192,7 +192,7 @@ ], "source": [ "class DuelingDQN:\n", - " def __init__(self,n_states,n_actions,cfg) -> None:\n", + " def __init__(self,state_dim,action_dim,cfg) -> None:\n", " self.batch_size = cfg.batch_size\n", " self.device = cfg.device\n", " self.loss_history = [] # 记录loss的变化\n", @@ -200,8 +200,8 @@ " self.epsilon = lambda frame_idx: cfg.epsilon_end + \\\n", " (cfg.epsilon_start - cfg.epsilon_end) * \\\n", " math.exp(-1. * frame_idx / cfg.epsilon_decay)\n", - " self.policy_net = DuelingNet(n_states, n_actions,hidden_dim=cfg.hidden_dim).to(self.device)\n", - " self.target_net = DuelingNet(n_states, n_actions,hidden_dim=cfg.hidden_dim).to(self.device)\n", + " self.policy_net = DuelingNet(state_dim, action_dim,hidden_dim=cfg.hidden_dim).to(self.device)\n", + " self.target_net = DuelingNet(state_dim, action_dim,hidden_dim=cfg.hidden_dim).to(self.device)\n", " for target_param, param in zip(self.target_net.parameters(),self.policy_net.parameters()): # 复制参数到目标网络targe_net\n", " target_param.data.copy_(param.data)\n", " self.optimizer = optim.Adam(self.policy_net.parameters(), lr=cfg.lr) # 优化器\n", diff --git a/codes/HierarchicalDQN/agent.py b/codes/HierarchicalDQN/agent.py index 3760643..62c539a 100644 --- a/codes/HierarchicalDQN/agent.py +++ b/codes/HierarchicalDQN/agent.py @@ -11,23 +11,62 @@ Environment: ''' import torch import torch.nn as nn +import torch.optim as optim +import torch.nn.functional as F import numpy as np import random,math -import torch.optim as optim -from common.model import MLP -from common.memory import ReplayBuffer +class ReplayBuffer: + def __init__(self, capacity): + self.capacity = capacity # 经验回放的容量 + self.buffer = [] # 缓冲区 + self.position = 0 + + def push(self, state, action, reward, next_state, done): + ''' 缓冲区是一个队列，容量超出时去掉开始存入的转移(transition) + ''' + if len(self.buffer) < self.capacity: + self.buffer.append(None) + self.buffer[self.position] = (state, action, reward, next_state, done) + self.position = (self.position + 1) % self.capacity + + def sample(self, batch_size): + batch = random.sample(self.buffer, batch_size) # 随机采出小批量转移 + state, action, reward, next_state, done = zip(*batch) # 解压成状态，动作等 + return state, action, reward, next_state, done + + def __len__(self): + ''' 返回当前存储的量 + ''' + return len(self.buffer) +class MLP(nn.Module): + def __init__(self, input_dim,output_dim,hidden_dim=128): + """ 初始化q网络，为全连接网络 + input_dim: 输入的特征数即环境的状态数 + output_dim: 输出的动作维度 + """ + super(MLP, self).__init__() + self.fc1 = nn.Linear(input_dim, hidden_dim) # 输入层 + self.fc2 = nn.Linear(hidden_dim,hidden_dim) # 隐藏层 + self.fc3 = nn.Linear(hidden_dim, output_dim) # 输出层 + + def forward(self, x): + # 各层对应的激活函数 + x = F.relu(self.fc1(x)) + x = F.relu(self.fc2(x)) + return self.fc3(x) + class HierarchicalDQN: - def __init__(self,state_dim,action_dim,cfg): - self.state_dim = state_dim - self.action_dim = action_dim + def __init__(self,n_states,n_actions,cfg): + self.n_states = n_states + self.n_actions = n_actions self.gamma = cfg.gamma self.device = cfg.device self.batch_size = cfg.batch_size - self.frame_idx = 0 + self.frame_idx = 0 # 用于epsilon的衰减计数 self.epsilon = lambda frame_idx: cfg.epsilon_end + (cfg.epsilon_start - cfg.epsilon_end ) * math.exp(-1. * frame_idx / cfg.epsilon_decay) - self.policy_net = MLP(2*state_dim, action_dim,cfg.hidden_dim).to(self.device) - self.meta_policy_net = MLP(state_dim, state_dim,cfg.hidden_dim).to(self.device) + self.policy_net = MLP(2*n_states, n_actions,cfg.hidden_dim).to(self.device) + self.meta_policy_net = MLP(n_states, n_states,cfg.hidden_dim).to(self.device) self.optimizer = optim.Adam(self.policy_net.parameters(),lr=cfg.lr) self.meta_optimizer = optim.Adam(self.meta_policy_net.parameters(),lr=cfg.lr) self.memory = ReplayBuffer(cfg.memory_capacity) @@ -37,7 +76,7 @@ class HierarchicalDQN: self.losses = [] self.meta_losses = [] def to_onehot(self,x): - oh = np.zeros(self.state_dim) + oh = np.zeros(self.n_states) oh[x - 1] = 1. return oh def set_goal(self,state): @@ -46,7 +85,7 @@ class HierarchicalDQN: state = torch.tensor(state, device=self.device, dtype=torch.float32).unsqueeze(0) goal = self.meta_policy_net(state).max(1)[1].item() else: - goal = random.randrange(self.state_dim) + goal = random.randrange(self.n_states) return goal def choose_action(self,state): self.frame_idx += 1 @@ -56,7 +95,7 @@ class HierarchicalDQN: q_value = self.policy_net(state) action = q_value.max(1)[1].item() else: - action = random.randrange(self.action_dim) + action = random.randrange(self.n_actions) return action def update(self): self.update_policy() diff --git a/codes/HierarchicalDQN/outputs/CartPole-v0/20211221-200119/models/meta_checkpoint.pth b/codes/HierarchicalDQN/outputs/CartPole-v0/20211221-200119/models/meta_checkpoint.pth new file mode 100644 index 0000000..02f3f7c Binary files /dev/null and b/codes/HierarchicalDQN/outputs/CartPole-v0/20211221-200119/models/meta_checkpoint.pth differ diff --git a/codes/HierarchicalDQN/outputs/CartPole-v0/20211221-200119/models/policy_checkpoint.pth b/codes/HierarchicalDQN/outputs/CartPole-v0/20211221-200119/models/policy_checkpoint.pth new file mode 100644 index 0000000..9d906ea Binary files /dev/null and b/codes/HierarchicalDQN/outputs/CartPole-v0/20211221-200119/models/policy_checkpoint.pth differ diff --git a/codes/HierarchicalDQN/outputs/CartPole-v0/20211221-200119/results/test_ma_rewards.npy b/codes/HierarchicalDQN/outputs/CartPole-v0/20211221-200119/results/test_ma_rewards.npy new file mode 100644 index 0000000..14dd955 Binary files /dev/null and b/codes/HierarchicalDQN/outputs/CartPole-v0/20211221-200119/results/test_ma_rewards.npy differ diff --git a/codes/HierarchicalDQN/outputs/CartPole-v0/20211221-200119/results/test_rewards.npy b/codes/HierarchicalDQN/outputs/CartPole-v0/20211221-200119/results/test_rewards.npy new file mode 100644 index 0000000..e815222 Binary files /dev/null and b/codes/HierarchicalDQN/outputs/CartPole-v0/20211221-200119/results/test_rewards.npy differ diff --git a/codes/HierarchicalDQN/outputs/CartPole-v0/20211221-200119/results/test_rewards_curve.png b/codes/HierarchicalDQN/outputs/CartPole-v0/20211221-200119/results/test_rewards_curve.png new file mode 100644 index 0000000..645b21a Binary files /dev/null and b/codes/HierarchicalDQN/outputs/CartPole-v0/20211221-200119/results/test_rewards_curve.png differ diff --git a/codes/HierarchicalDQN/outputs/CartPole-v0/20211221-200119/results/train_ma_rewards.npy b/codes/HierarchicalDQN/outputs/CartPole-v0/20211221-200119/results/train_ma_rewards.npy new file mode 100644 index 0000000..bf58391 Binary files /dev/null and b/codes/HierarchicalDQN/outputs/CartPole-v0/20211221-200119/results/train_ma_rewards.npy differ diff --git a/codes/HierarchicalDQN/outputs/CartPole-v0/20211221-200119/results/train_rewards.npy b/codes/HierarchicalDQN/outputs/CartPole-v0/20211221-200119/results/train_rewards.npy new file mode 100644 index 0000000..f4d20ff Binary files /dev/null and b/codes/HierarchicalDQN/outputs/CartPole-v0/20211221-200119/results/train_rewards.npy differ diff --git a/codes/HierarchicalDQN/outputs/CartPole-v0/20211221-200119/results/train_rewards_curve.png b/codes/HierarchicalDQN/outputs/CartPole-v0/20211221-200119/results/train_rewards_curve.png new file mode 100644 index 0000000..20ccbc5 Binary files /dev/null and b/codes/HierarchicalDQN/outputs/CartPole-v0/20211221-200119/results/train_rewards_curve.png differ diff --git a/codes/HierarchicalDQN/results/20210331-134559/ma_rewards_train.npy b/codes/HierarchicalDQN/results/20210331-134559/ma_rewards_train.npy deleted file mode 100644 index daab87d..0000000 Binary files a/codes/HierarchicalDQN/results/20210331-134559/ma_rewards_train.npy and /dev/null differ diff --git a/codes/HierarchicalDQN/results/20210331-134559/rewards_curve_train.png b/codes/HierarchicalDQN/results/20210331-134559/rewards_curve_train.png deleted file mode 100644 index 77555ad..0000000 Binary files a/codes/HierarchicalDQN/results/20210331-134559/rewards_curve_train.png and /dev/null differ diff --git a/codes/HierarchicalDQN/results/20210331-134559/rewards_train.npy b/codes/HierarchicalDQN/results/20210331-134559/rewards_train.npy deleted file mode 100644 index 5a1ad82..0000000 Binary files a/codes/HierarchicalDQN/results/20210331-134559/rewards_train.npy and /dev/null differ diff --git a/codes/HierarchicalDQN/results/20210331-145852/losses_curve.png b/codes/HierarchicalDQN/results/20210331-145852/losses_curve.png deleted file mode 100644 index 4f962ea..0000000 Binary files a/codes/HierarchicalDQN/results/20210331-145852/losses_curve.png and /dev/null differ diff --git a/codes/HierarchicalDQN/results/20210331-145852/ma_rewards_train.npy b/codes/HierarchicalDQN/results/20210331-145852/ma_rewards_train.npy deleted file mode 100644 index 523bdb4..0000000 Binary files a/codes/HierarchicalDQN/results/20210331-145852/ma_rewards_train.npy and /dev/null differ diff --git a/codes/HierarchicalDQN/results/20210331-145852/rewards_curve_train.png b/codes/HierarchicalDQN/results/20210331-145852/rewards_curve_train.png deleted file mode 100644 index 97443e5..0000000 Binary files a/codes/HierarchicalDQN/results/20210331-145852/rewards_curve_train.png and /dev/null differ diff --git a/codes/HierarchicalDQN/results/20210331-145852/rewards_train.npy b/codes/HierarchicalDQN/results/20210331-145852/rewards_train.npy deleted file mode 100644 index 99cf87a..0000000 Binary files a/codes/HierarchicalDQN/results/20210331-145852/rewards_train.npy and /dev/null differ diff --git a/codes/HierarchicalDQN/saved_model/20210331-134559/meta_checkpoint.pth b/codes/HierarchicalDQN/saved_model/20210331-134559/meta_checkpoint.pth deleted file mode 100644 index 873b3ef..0000000 Binary files a/codes/HierarchicalDQN/saved_model/20210331-134559/meta_checkpoint.pth and /dev/null differ diff --git a/codes/HierarchicalDQN/saved_model/20210331-134559/policy_checkpoint.pth b/codes/HierarchicalDQN/saved_model/20210331-134559/policy_checkpoint.pth deleted file mode 100644 index be8ea8a..0000000 Binary files a/codes/HierarchicalDQN/saved_model/20210331-134559/policy_checkpoint.pth and /dev/null differ diff --git a/codes/HierarchicalDQN/saved_model/20210331-145852/meta_checkpoint.pth b/codes/HierarchicalDQN/saved_model/20210331-145852/meta_checkpoint.pth deleted file mode 100644 index e3f7c38..0000000 Binary files a/codes/HierarchicalDQN/saved_model/20210331-145852/meta_checkpoint.pth and /dev/null differ diff --git a/codes/HierarchicalDQN/saved_model/20210331-145852/policy_checkpoint.pth b/codes/HierarchicalDQN/saved_model/20210331-145852/policy_checkpoint.pth deleted file mode 100644 index 6be6ea3..0000000 Binary files a/codes/HierarchicalDQN/saved_model/20210331-145852/policy_checkpoint.pth and /dev/null differ diff --git a/codes/HierarchicalDQN/task0.py b/codes/HierarchicalDQN/task0.py new file mode 100644 index 0000000..b2cf312 --- /dev/null +++ b/codes/HierarchicalDQN/task0.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python +# coding=utf-8 +''' +Author: John +Email: johnjim0816@gmail.com +Date: 2021-03-29 10:37:32 +LastEditor: John +LastEditTime: 2021-05-04 22:35:56 +Discription: +Environment: +''' +import sys +import os +curr_path = os.path.dirname(os.path.abspath(__file__)) # 当前文件所在绝对路径 +parent_path = os.path.dirname(curr_path) # 父路径 +sys.path.append(parent_path) # 添加路径到系统路径 + +import datetime +import numpy as np +import torch +import gym + +from common.utils import save_results,make_dir +from common.utils import plot_rewards +from HierarchicalDQN.agent import HierarchicalDQN +from HierarchicalDQN.train import train,test + +curr_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") # 获取当前时间 +algo_name = "Hierarchical DQN" # 算法名称 +env_name = 'CartPole-v0' # 环境名称 +class HierarchicalDQNConfig: + def __init__(self): + self.algo_name = algo_name # 算法名称 + self.env_name = env_name # 环境名称 + self.device = torch.device( + "cuda" if torch.cuda.is_available() else "cpu") # 检测GPU + self.train_eps = 300 # 训练的episode数目 + self.test_eps = 50 # 测试的episode数目 + self.gamma = 0.99 + self.epsilon_start = 1 # start epsilon of e-greedy policy + self.epsilon_end = 0.01 + self.epsilon_decay = 200 + self.lr = 0.0001 # learning rate + self.memory_capacity = 10000 # Replay Memory capacity + self.batch_size = 32 + self.target_update = 2 # 目标网络的更新频率 + self.hidden_dim = 256 # 网络隐藏层 +class PlotConfig: + ''' 绘图相关参数设置 + ''' + + def __init__(self) -> None: + self.algo_name = algo_name # 算法名称 + self.env_name = env_name # 环境名称 + self.device = torch.device( + "cuda" if torch.cuda.is_available() else "cpu") # 检测GPU + self.result_path = curr_path + "/outputs/" + self.env_name + \ + '/' + curr_time + '/results/' # 保存结果的路径 + self.model_path = curr_path + "/outputs/" + self.env_name + \ + '/' + curr_time + '/models/' # 保存模型的路径 + self.save = True # 是否保存图片 + +def env_agent_config(cfg,seed=1): + env = gym.make(cfg.env_name) + env.seed(seed) + n_states = env.observation_space.shape[0] + n_actions = env.action_space.n + agent = HierarchicalDQN(n_states,n_actions,cfg) + return env,agent + +if __name__ == "__main__": + cfg = HierarchicalDQNConfig() + plot_cfg = PlotConfig() + # 训练 + env, agent = env_agent_config(cfg, seed=1) + rewards, ma_rewards = train(cfg, env, agent) + make_dir(plot_cfg.result_path, plot_cfg.model_path) # 创建保存结果和模型路径的文件夹 + agent.save(path=plot_cfg.model_path) # 保存模型 + save_results(rewards, ma_rewards, tag='train', + path=plot_cfg.result_path) # 保存结果 + plot_rewards(rewards, ma_rewards, plot_cfg, tag="train") # 画出结果 + # 测试 + env, agent = env_agent_config(cfg, seed=10) + agent.load(path=plot_cfg.model_path) # 导入模型 + rewards, ma_rewards = test(cfg, env, agent) + save_results(rewards, ma_rewards, tag='test', path=plot_cfg.result_path) # 保存结果 + plot_rewards(rewards, ma_rewards, plot_cfg, tag="test") # 画出结果 + diff --git a/codes/HierarchicalDQN/task0_train.ipynb b/codes/HierarchicalDQN/task0_train.ipynb deleted file mode 100644 index c63e950..0000000 --- a/codes/HierarchicalDQN/task0_train.ipynb +++ /dev/null @@ -1,477 +0,0 @@ -{ - "metadata": { - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.10-final" - }, - "orig_nbformat": 2, - "kernelspec": { - "name": "python3", - "display_name": "Python 3.7.10 64-bit ('py37': conda)", - "metadata": { - "interpreter": { - "hash": "fbea1422c2cf61ed9c0cfc03f38f71cc9083cc288606edc4170b5309b352ce27" - } - } - } - }, - "nbformat": 4, - "nbformat_minor": 2, - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "import sys,os\n", - "from pathlib import Path\n", - "curr_path = str(Path().absolute())\n", - "parent_path = str(Path().absolute().parent)\n", - "sys.path.append(parent_path) # add current terminal path to sys.path\n", - "\n", - "import gym\n", - "import torch\n", - "import numpy as np\n", - "import datetime\n", - "\n", - "from HierarchicalDQN.agent import HierarchicalDQN\n", - "from common.plot import plot_rewards\n", - "from common.utils import save_results" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "SEQUENCE = datetime.datetime.now().strftime(\n", - " \"%Y%m%d-%H%M%S\") # obtain current time\n", - "SAVED_MODEL_PATH = curr_path+\"/saved_model/\"+SEQUENCE+'/' # path to save model\n", - "if not os.path.exists(curr_path+\"/saved_model/\"):\n", - " os.mkdir(curr_path+\"/saved_model/\")\n", - "if not os.path.exists(SAVED_MODEL_PATH):\n", - " os.mkdir(SAVED_MODEL_PATH)\n", - "RESULT_PATH = curr_path+\"/results/\"+SEQUENCE+'/' # path to save rewards\n", - "if not os.path.exists(curr_path+\"/results/\"):\n", - " os.mkdir(curr_path+\"/results/\")\n", - "if not os.path.exists(RESULT_PATH):\n", - " os.mkdir(RESULT_PATH)" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "class HierarchicalDQNConfig:\n", - " def __init__(self):\n", - " self.algo = \"H-DQN\" # name of algo\n", - " self.gamma = 0.95\n", - " self.epsilon_start = 1 # start epsilon of e-greedy policy\n", - " self.epsilon_end = 0.01\n", - " self.epsilon_decay = 500\n", - " self.lr = 0.0001 # learning rate\n", - " self.memory_capacity = 20000 # Replay Memory capacity\n", - " self.batch_size = 64\n", - " self.train_eps = 300 # 训练的episode数目\n", - " self.target_update = 2 # target net的更新频率\n", - " self.eval_eps = 20 # 测试的episode数目\n", - " self.device = torch.device(\n", - " \"cuda\" if torch.cuda.is_available() else \"cpu\") # 检测gpu\n", - " self.hidden_dim = 256 # dimension of hidden layer" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "def train(cfg, env, agent):\n", - " print('Start to train !')\n", - " rewards = []\n", - " ma_rewards = [] # moveing average reward\n", - " for i_episode in range(cfg.train_eps):\n", - " state = env.reset()\n", - " done = False\n", - " ep_reward = 0\n", - " while not done:\n", - " goal = agent.set_goal(state)\n", - " onehot_goal = agent.to_onehot(goal)\n", - " meta_state = state\n", - " extrinsic_reward = 0\n", - " while not done and goal != np.argmax(state):\n", - " goal_state = np.concatenate([state, onehot_goal])\n", - " action = agent.choose_action(goal_state)\n", - " next_state, reward, done, _ = env.step(action)\n", - " ep_reward += reward\n", - " extrinsic_reward += reward\n", - " intrinsic_reward = 1.0 if goal == np.argmax(\n", - " next_state) else 0.0\n", - " agent.memory.push(goal_state, action, intrinsic_reward, np.concatenate(\n", - " [next_state, onehot_goal]), done)\n", - " state = next_state\n", - " agent.update()\n", - " agent.meta_memory.push(meta_state, goal, extrinsic_reward, state, done)\n", - " print('Episode:{}/{}, Reward:{}'.format(i_episode+1, cfg.train_eps, ep_reward))\n", - " rewards.append(ep_reward)\n", - " if ma_rewards:\n", - " ma_rewards.append(\n", - " 0.9*ma_rewards[-1]+0.1*ep_reward)\n", - " else:\n", - " ma_rewards.append(ep_reward)\n", - " print('Complete training！')\n", - " return rewards, ma_rewards" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "Start to train !\n", - "Episode:1/300, Reward:25.0\n", - "Episode:2/300, Reward:26.0\n", - "Episode:3/300, Reward:23.0\n", - "Episode:4/300, Reward:19.0\n", - "Episode:5/300, Reward:23.0\n", - "Episode:6/300, Reward:21.0\n", - "Episode:7/300, Reward:21.0\n", - "Episode:8/300, Reward:22.0\n", - "Episode:9/300, Reward:15.0\n", - "Episode:10/300, Reward:12.0\n", - "Episode:11/300, Reward:39.0\n", - "Episode:12/300, Reward:42.0\n", - "Episode:13/300, Reward:79.0\n", - "Episode:14/300, Reward:54.0\n", - "Episode:15/300, Reward:28.0\n", - "Episode:16/300, Reward:85.0\n", - "Episode:17/300, Reward:46.0\n", - "Episode:18/300, Reward:37.0\n", - "Episode:19/300, Reward:45.0\n", - "Episode:20/300, Reward:79.0\n", - "Episode:21/300, Reward:80.0\n", - "Episode:22/300, Reward:154.0\n", - "Episode:23/300, Reward:74.0\n", - "Episode:24/300, Reward:129.0\n", - "Episode:25/300, Reward:185.0\n", - "Episode:26/300, Reward:200.0\n", - "Episode:27/300, Reward:115.0\n", - "Episode:28/300, Reward:104.0\n", - "Episode:29/300, Reward:200.0\n", - "Episode:30/300, Reward:118.0\n", - "Episode:31/300, Reward:200.0\n", - "Episode:32/300, Reward:200.0\n", - "Episode:33/300, Reward:83.0\n", - "Episode:34/300, Reward:75.0\n", - "Episode:35/300, Reward:46.0\n", - "Episode:36/300, Reward:96.0\n", - "Episode:37/300, Reward:78.0\n", - "Episode:38/300, Reward:150.0\n", - "Episode:39/300, Reward:147.0\n", - "Episode:40/300, Reward:74.0\n", - "Episode:41/300, Reward:137.0\n", - "Episode:42/300, Reward:182.0\n", - "Episode:43/300, Reward:200.0\n", - "Episode:44/300, Reward:200.0\n", - "Episode:45/300, Reward:200.0\n", - "Episode:46/300, Reward:184.0\n", - "Episode:47/300, Reward:200.0\n", - "Episode:48/300, Reward:200.0\n", - "Episode:49/300, Reward:200.0\n", - "Episode:50/300, Reward:61.0\n", - "Episode:51/300, Reward:9.0\n", - "Episode:52/300, Reward:9.0\n", - "Episode:53/300, Reward:200.0\n", - "Episode:54/300, Reward:200.0\n", - "Episode:55/300, Reward:200.0\n", - "Episode:56/300, Reward:200.0\n", - "Episode:57/300, Reward:200.0\n", - "Episode:58/300, Reward:200.0\n", - "Episode:59/300, Reward:200.0\n", - "Episode:60/300, Reward:167.0\n", - "Episode:61/300, Reward:200.0\n", - "Episode:62/300, Reward:200.0\n", - "Episode:63/300, Reward:200.0\n", - "Episode:64/300, Reward:200.0\n", - "Episode:65/300, Reward:200.0\n", - "Episode:66/300, Reward:200.0\n", - "Episode:67/300, Reward:200.0\n", - "Episode:68/300, Reward:200.0\n", - "Episode:69/300, Reward:197.0\n", - "Episode:70/300, Reward:200.0\n", - "Episode:71/300, Reward:200.0\n", - "Episode:72/300, Reward:200.0\n", - "Episode:73/300, Reward:200.0\n", - "Episode:74/300, Reward:200.0\n", - "Episode:75/300, Reward:200.0\n", - "Episode:76/300, Reward:200.0\n", - "Episode:77/300, Reward:200.0\n", - "Episode:78/300, Reward:200.0\n", - "Episode:79/300, Reward:200.0\n", - "Episode:80/300, Reward:200.0\n", - "Episode:81/300, Reward:181.0\n", - "Episode:82/300, Reward:200.0\n", - "Episode:83/300, Reward:200.0\n", - "Episode:84/300, Reward:200.0\n", - "Episode:85/300, Reward:200.0\n", - "Episode:86/300, Reward:200.0\n", - "Episode:87/300, Reward:200.0\n", - "Episode:88/300, Reward:200.0\n", - "Episode:89/300, Reward:200.0\n", - "Episode:90/300, Reward:200.0\n", - "Episode:91/300, Reward:200.0\n", - "Episode:92/300, Reward:200.0\n", - "Episode:93/300, Reward:200.0\n", - "Episode:94/300, Reward:200.0\n", - "Episode:95/300, Reward:200.0\n", - "Episode:96/300, Reward:200.0\n", - "Episode:97/300, Reward:200.0\n", - "Episode:98/300, Reward:200.0\n", - "Episode:99/300, Reward:192.0\n", - "Episode:100/300, Reward:183.0\n", - "Episode:101/300, Reward:200.0\n", - "Episode:102/300, Reward:200.0\n", - "Episode:103/300, Reward:200.0\n", - "Episode:104/300, Reward:200.0\n", - "Episode:105/300, Reward:200.0\n", - "Episode:106/300, Reward:200.0\n", - "Episode:107/300, Reward:200.0\n", - "Episode:108/300, Reward:200.0\n", - "Episode:109/300, Reward:200.0\n", - "Episode:110/300, Reward:200.0\n", - "Episode:111/300, Reward:200.0\n", - "Episode:112/300, Reward:200.0\n", - "Episode:113/300, Reward:200.0\n", - "Episode:114/300, Reward:200.0\n", - "Episode:115/300, Reward:200.0\n", - "Episode:116/300, Reward:200.0\n", - "Episode:117/300, Reward:200.0\n", - "Episode:118/300, Reward:200.0\n", - "Episode:119/300, Reward:200.0\n", - "Episode:120/300, Reward:196.0\n", - "Episode:121/300, Reward:200.0\n", - "Episode:122/300, Reward:200.0\n", - "Episode:123/300, Reward:200.0\n", - "Episode:124/300, Reward:200.0\n", - "Episode:125/300, Reward:200.0\n", - "Episode:126/300, Reward:189.0\n", - "Episode:127/300, Reward:193.0\n", - "Episode:128/300, Reward:200.0\n", - "Episode:129/300, Reward:200.0\n", - "Episode:130/300, Reward:193.0\n", - "Episode:131/300, Reward:183.0\n", - "Episode:132/300, Reward:183.0\n", - "Episode:133/300, Reward:200.0\n", - "Episode:134/300, Reward:200.0\n", - "Episode:135/300, Reward:200.0\n", - "Episode:136/300, Reward:200.0\n", - "Episode:137/300, Reward:200.0\n", - "Episode:138/300, Reward:200.0\n", - "Episode:139/300, Reward:100.0\n", - "Episode:140/300, Reward:118.0\n", - "Episode:141/300, Reward:99.0\n", - "Episode:142/300, Reward:185.0\n", - "Episode:143/300, Reward:41.0\n", - "Episode:144/300, Reward:11.0\n", - "Episode:145/300, Reward:9.0\n", - "Episode:146/300, Reward:152.0\n", - "Episode:147/300, Reward:155.0\n", - "Episode:148/300, Reward:181.0\n", - "Episode:149/300, Reward:197.0\n", - "Episode:150/300, Reward:200.0\n", - "Episode:151/300, Reward:200.0\n", - "Episode:152/300, Reward:200.0\n", - "Episode:153/300, Reward:200.0\n", - "Episode:154/300, Reward:200.0\n", - "Episode:155/300, Reward:200.0\n", - "Episode:156/300, Reward:123.0\n", - "Episode:157/300, Reward:11.0\n", - "Episode:158/300, Reward:8.0\n", - "Episode:159/300, Reward:9.0\n", - "Episode:160/300, Reward:10.0\n", - "Episode:161/300, Reward:9.0\n", - "Episode:162/300, Reward:10.0\n", - "Episode:163/300, Reward:9.0\n", - "Episode:164/300, Reward:9.0\n", - "Episode:165/300, Reward:10.0\n", - "Episode:166/300, Reward:9.0\n", - "Episode:167/300, Reward:9.0\n", - "Episode:168/300, Reward:9.0\n", - "Episode:169/300, Reward:9.0\n", - "Episode:170/300, Reward:10.0\n", - "Episode:171/300, Reward:9.0\n", - "Episode:172/300, Reward:9.0\n", - "Episode:173/300, Reward:11.0\n", - "Episode:174/300, Reward:11.0\n", - "Episode:175/300, Reward:10.0\n", - "Episode:176/300, Reward:9.0\n", - "Episode:177/300, Reward:10.0\n", - "Episode:178/300, Reward:8.0\n", - "Episode:179/300, Reward:9.0\n", - "Episode:180/300, Reward:9.0\n", - "Episode:181/300, Reward:10.0\n", - "Episode:182/300, Reward:10.0\n", - "Episode:183/300, Reward:9.0\n", - "Episode:184/300, Reward:10.0\n", - "Episode:185/300, Reward:10.0\n", - "Episode:186/300, Reward:13.0\n", - "Episode:187/300, Reward:16.0\n", - "Episode:188/300, Reward:117.0\n", - "Episode:189/300, Reward:13.0\n", - "Episode:190/300, Reward:16.0\n", - "Episode:191/300, Reward:11.0\n", - "Episode:192/300, Reward:11.0\n", - "Episode:193/300, Reward:13.0\n", - "Episode:194/300, Reward:13.0\n", - "Episode:195/300, Reward:9.0\n", - "Episode:196/300, Reward:20.0\n", - "Episode:197/300, Reward:12.0\n", - "Episode:198/300, Reward:10.0\n", - "Episode:199/300, Reward:14.0\n", - "Episode:200/300, Reward:12.0\n", - "Episode:201/300, Reward:14.0\n", - "Episode:202/300, Reward:12.0\n", - "Episode:203/300, Reward:11.0\n", - "Episode:204/300, Reward:10.0\n", - "Episode:205/300, Reward:13.0\n", - "Episode:206/300, Reward:10.0\n", - "Episode:207/300, Reward:10.0\n", - "Episode:208/300, Reward:13.0\n", - "Episode:209/300, Reward:9.0\n", - "Episode:210/300, Reward:11.0\n", - "Episode:211/300, Reward:14.0\n", - "Episode:212/300, Reward:10.0\n", - "Episode:213/300, Reward:20.0\n", - "Episode:214/300, Reward:12.0\n", - "Episode:215/300, Reward:13.0\n", - "Episode:216/300, Reward:17.0\n", - "Episode:217/300, Reward:17.0\n", - "Episode:218/300, Reward:11.0\n", - "Episode:219/300, Reward:15.0\n", - "Episode:220/300, Reward:26.0\n", - "Episode:221/300, Reward:73.0\n", - "Episode:222/300, Reward:44.0\n", - "Episode:223/300, Reward:48.0\n", - "Episode:224/300, Reward:102.0\n", - "Episode:225/300, Reward:162.0\n", - "Episode:226/300, Reward:123.0\n", - "Episode:227/300, Reward:200.0\n", - "Episode:228/300, Reward:200.0\n", - "Episode:229/300, Reward:120.0\n", - "Episode:230/300, Reward:173.0\n", - "Episode:231/300, Reward:138.0\n", - "Episode:232/300, Reward:106.0\n", - "Episode:233/300, Reward:193.0\n", - "Episode:234/300, Reward:117.0\n", - "Episode:235/300, Reward:120.0\n", - "Episode:236/300, Reward:98.0\n", - "Episode:237/300, Reward:98.0\n", - "Episode:238/300, Reward:200.0\n", - "Episode:239/300, Reward:96.0\n", - "Episode:240/300, Reward:170.0\n", - "Episode:241/300, Reward:107.0\n", - "Episode:242/300, Reward:107.0\n", - "Episode:243/300, Reward:200.0\n", - "Episode:244/300, Reward:128.0\n", - "Episode:245/300, Reward:165.0\n", - "Episode:246/300, Reward:168.0\n", - "Episode:247/300, Reward:200.0\n", - "Episode:248/300, Reward:200.0\n", - "Episode:249/300, Reward:200.0\n", - "Episode:250/300, Reward:200.0\n", - "Episode:251/300, Reward:200.0\n", - "Episode:252/300, Reward:200.0\n", - "Episode:253/300, Reward:200.0\n", - "Episode:254/300, Reward:200.0\n", - "Episode:255/300, Reward:200.0\n", - "Episode:256/300, Reward:200.0\n", - "Episode:257/300, Reward:164.0\n", - "Episode:258/300, Reward:200.0\n", - "Episode:259/300, Reward:190.0\n", - "Episode:260/300, Reward:185.0\n", - "Episode:261/300, Reward:200.0\n", - "Episode:262/300, Reward:200.0\n", - "Episode:263/300, Reward:200.0\n", - "Episode:264/300, Reward:200.0\n", - "Episode:265/300, Reward:168.0\n", - "Episode:266/300, Reward:200.0\n", - "Episode:267/300, Reward:200.0\n", - "Episode:268/300, Reward:200.0\n", - "Episode:269/300, Reward:200.0\n", - "Episode:270/300, Reward:200.0\n", - "Episode:271/300, Reward:200.0\n", - "Episode:272/300, Reward:200.0\n", - "Episode:273/300, Reward:200.0\n", - "Episode:274/300, Reward:200.0\n", - "Episode:275/300, Reward:188.0\n", - "Episode:276/300, Reward:200.0\n", - "Episode:277/300, Reward:177.0\n", - "Episode:278/300, Reward:200.0\n", - "Episode:279/300, Reward:200.0\n", - "Episode:280/300, Reward:200.0\n", - "Episode:281/300, Reward:200.0\n", - "Episode:282/300, Reward:200.0\n", - "Episode:283/300, Reward:200.0\n", - "Episode:284/300, Reward:189.0\n", - "Episode:285/300, Reward:200.0\n", - "Episode:286/300, Reward:200.0\n", - "Episode:287/300, Reward:200.0\n", - "Episode:288/300, Reward:200.0\n", - "Episode:289/300, Reward:200.0\n", - "Episode:290/300, Reward:200.0\n", - "Episode:291/300, Reward:200.0\n", - "Episode:292/300, Reward:200.0\n", - "Episode:293/300, Reward:200.0\n", - "Episode:294/300, Reward:200.0\n", - "Episode:295/300, Reward:200.0\n", - "Episode:296/300, Reward:200.0\n", - "Episode:297/300, Reward:200.0\n", - "Episode:298/300, Reward:200.0\n", - "Episode:299/300, Reward:200.0\n", - "Episode:300/300, Reward:200.0\n", - "Complete training！\n", - "results saved!\n" - ] - }, - { - "output_type": "display_data", - "data": { - "text/plain": "
", - "image/svg+xml": "\n\n\n\n \n \n \n \n 2021-03-31T14:01:15.395751\n image/svg+xml\n \n \n Matplotlib v3.3.4, https://matplotlib.org/\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n\n", - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXkAAAEXCAYAAABI/TQXAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAACOjUlEQVR4nO2dd5jc1NWHX0nTt3qbe8HdBlwwxaaZ5oIL1XRwgBBIQiCQfHyAMYQ4MSHAB4EQICFAEkqAmBIghB4IYKoBG4Mxxr1ur7NTpfv9oZFGMzuzO9ub3ufx41mNpLlX5ejod889RxJCCGxsbGxs+iRydzfAxsbGxqbzsI28jY2NTR/GNvI2NjY2fRjbyNvY2Nj0YWwjb2NjY9OHsY28jY2NTR/GNvI23crvf/97VqxY0SW/9YMf/IDvvvuuS36ru7n//vs55phjuP766xOW79q1i+nTpzdZv7nzcN1113HUUUdx8sknc/LJJ7NgwQJ+/vOfU15enrDe008/zWmnncaiRYtYuHAh11xzDbt3707Yz/z582lsbEzYbvr06ezatautXbVpAUd3N8DGpqt48MEHu7sJXcaqVau44447OPjggztkfxdeeCHf//73ARBC8Mc//pFLLrmEZ599FkVRuP3221m3bh333XcfgwYNQtM0XnjhBc466yz+8Y9/MHjwYAB2797NypUrWblyZYe0y6ZlbCPfB9A0jVtuuYW1a9fi9/sRQvDrX/+a8ePHM3v2bF599VWKi4sBOPPMM7n88suZNWsWd9xxB5988gmqqjJ58mSWL19OdnY2xx13HFOmTGHjxo387Gc/w+Fw8Mc//pFwOExVVRWnnHIKV111FQB/+tOfWLVqFVlZWRx88MG8+eabvPXWW4TD4bT7T0dpaSkrVqxg7969RCIRFi5cyA9/+EMAHnjgAd544w1CoRCBQIBrr72WOXPm8Pvf/54vvviCsrIyJkyYwMiRI9m9ezfl5eXs3r2bgoIC7rrrLgYOHMhxxx3H3XffTWNjI3fddRfDhw9n06ZNhMNhbrrpJmbOnElVVRXXX389O3bsID8/n+LiYsaNG8cVV1yR0Fa/38+vf/1rPvvsMxRF4YQTTuDqq6/m+uuvZ9y4caZBvO6668y/rcf1iiuu4P777+fFF18EoK6ujuOPP5433niDYDCY9jhY2bdvHzfffDO7d+9GCMEpp5zCJZdcwlVXXUVpaSk33HADP/3pT1mwYEG7rzErkiTxwx/+kOeee47333+fiRMn8vjjj/Paa69RUlICgCzLnHLKKaxfv54//vGP3HzzzQAsXbqUf/7zn7z66qvMmzevQ9tlkxpbrukDrF27lrKyMp566ilefvllTj31VB588EFycnKYM2cOL7zwAgCbN2+mvLyco446ij/96U8oisKzzz7LCy+8QElJCXfccYe5z3HjxvHvf/+bE044gYcffphbb72VZ599lqeeeoo//elPVFVV8e677/Lss8+yatUqnn32Wfx+v7l9S/tPxTXXXMPpp59u7nP16tW8/PLL7N69m9WrV/PYY4/x4osvcvXVV3PPPfeY2+3evZvnnnvO3P+nn37K3XffzSuvvEJubi5PPfVUk99at24dF198Mc8//zxLlizh3nvvBeDXv/41Y8eO5d///jd33303n332Wcq23nPPPYRCIV5++WWef/55PvvsMz7++OMWz5VxXE888UT8fj9ffvklAC+99BKzZ88mLy8v7XFI5n/+53847LDDePHFF/n73//OCy+8wL/+9S9+97vfmcc7lYEPBoOm9GL8e/LJJ1tsezITJkzg22+/Ze3atQwfPtw08FaOOOKIhGNYUFDArbfeyk033cTevXtb/Zs2rcf25PsA06dPJy8vjyeffJKdO3fy0UcfkZWVBcAZZ5zBL3/5S77//e/zzDPPcNpppyHLMm+//Tb19fWsXr0agEgkQmFhoblP4zVfkiQeeOAB3n77bV566SU2b96MEIJAIMA777zD/Pnzyc3NBeC8887jww8/BGhx/8k0NjbyySefUFtby913320u++abb1iwYAG//e1vefHFF9m+fbv5xmIwbdo0HI74pXzooYeabwyTJ0+mtra2ye8NGTKESZMmmes899xzALzzzjvm55KSEubPn5+yvatXr+b6669HURQUReGxxx4DMLdNh/W4LlmyhOeee44DDzyQZ599lmuuuabF42A9Xp999hkPP/wwADk5OZx22mn897//ZeHChc22wePx8M9//jNh2e9//3uqq6ub3S4ZSZLwer0trqdpWsLfRx55JKeeeirXXHMNf/vb31r1mzatxzbyfYC3336blStXctFFF3H88cczevRo03s/+OCDiUajrFu3jpdeesn02DRNY9myZcyePRvQ5YdQKGTu0+fzAboxOfXUUznhhBM4+OCDOf3003njjTcQQuBwOLCmPlIUxfzc0v6T0TQNIQRPPvmkaTiqqqpwu9189dVX/PjHP+bCCy/kiCOO4JBDDuGXv/xlk7YaeDwe87MkSaRKz5RuneQ+yXLql12Hw4EkSebfe/fuxePxNPm9SCSSsJ21raeffjqnnHIKZ5xxBvX19Rx22GE0NDSkPQ6pjlfysmg0mrK9mVJaWsqll15q/v2nP/0p5XpCCL766ivOP/98Ro4cyc6dOykrKzO9+dLSUgYOHMiHH37ItGnTmmz/s5/9jLPOOosHHnigXe21aRlbrukDvP/++xx77LGce+65HHjggbzxxhuoqmp+f8YZZ/CrX/2KCRMmMGTIEED3ph5//HHC4TCapnHjjTdy5513Ntn39u3baWho4KqrruK4447j448/NreZPXs2r732GvX19YA+2GeQ6f4NsrOzmTZtGo888giga9TnnHMOb775Jp988gkHHHAAF110EYceeihvvvlmQv86ktmzZ5v9qK6u5o033kgw5gazZs3iueeeQ9M0wuEwV155JZ988gkDBgxg/fr1gG6cP/3007S/NXDgQKZOncpNN93EkiVLgOaPg5Xs7GymTp3K448/DkB9fT3PP/88hx9+eLv6P3DgQP75z3+a/wYOHNhkHVVV+cMf/sCAAQM45JBDKCkp4YILLuDnP/85paWlVFdX88Mf/pDLL7+cf/3rX1x22WVN9uFyufi///s/Hn74YYLBYLvabNM8tpHvA5x99tl88sknLF68mLPOOovhw4eza9cu8zX5lFNOYcOGDZxxxhnmNj/+8Y8ZOnQop556KgsWLEAIwXXXXddk3xMmTOCYY47hxBNP5NRTT+Wtt95i7NixbN++nVmzZnHmmWdy1llncdppp1FfX296n5nu38odd9zB2rVrWbx4MWeccQaLFi3ipJNOYtGiRVRXV7NgwQJOO+00fD4ftbW1NDQ0dOBR1Ln++uvZsmULixcv5sorr2TIkCEJXr/BT37yE5xOJyeffDKnnHIKs2fPZu7cuVxwwQWUl5czb948rrnmGg499NBmf++MM85gw4YNnHrqqS0eh2TuuOMOPvjgAxYvXsySJUuYO3cup512WvsPQgr+8pe/mH095ZRT2LNnT4KX//Of/5zFixfzox/9iPPPP988N0VFRbz22msp9zl69GiuvfbaJnKOTcci2amGbdrKl19+yeeff87SpUsBeOSRR1i7di2/+93vurdh7eDxxx9n8uTJTJ8+nXA4zLnnnssVV1xhyk42rSMYDPLhhx9yzDHHdHdT+i22kbdpMw0NDSxbtowtW7YgSRKDBw/mV7/6VcpX/N7CRx99xG9/+1s0TSMSiTB//vwm4ZM2Nr0J28jb2NjY9GFsTd7GxsamD2MbeRsbG5s+jG3kbWxsbPowtpG3sbGx6cP0uBmv1dV+NK31Y8GFhdlUVnZ83HR3YPelZ2L3pWfS3/siyxIDBmSl/b7HGXlNE20y8sa2fQW7Lz0Tuy89E7sv6bHlGhsbG5s+jG3kbWxsbPowtpG3sbGx6cNkZOTvvfdeFi5cyMKFC7ntttsAPZ/24sWLmTt3LnfddZe57oYNGzj99NOZN28eN9xwQ7tTn9rY2NjYtJ0Wjfzq1at57733eO6553j++ef56quveOmll1i2bBn33XcfL7/8MuvXr+edd94B9Oo+N954I6+++ipCCJ5++ulO74SNjY2NTWpaNPLFxcVcd911uFwunE4nY8aMYdu2bYwcOZLhw4fjcDhYvHgxr7zyCrt37yYYDJpFAk477TReeeWVzu5DtyKEQGtl+h8tto2RNij57+T1kv8lb5fun7V9mfxr7fottQ9i0VIduM907U53LtL1pzWY+9DS99MmNR19nFKezxTXWEvbZXoPZXJPZrqvTO6/zqDFEMpx48aZn7dt28bLL7/MBRdcYBaGBr1MWmlpKWVlZQnLi4uLKS0t7eAmdw+aJrjktv9wzvHjmHPIcG574jMOP2Aw767bw6ZdtZx13FjmHTqixf08+eYmXvtkJwDTxxUxalAOz727FYCJI/L533MPAuDddXv4y8vfkOrU52e7uGjBJO5ZtQ41TbiVyyFzyaLJ/PlfXxOOtJyvW5LgRycfwDPvbKa0OtDi+s0xc/+BXLp4f5797xZeWr0tw60EXilCjhQgVw6QLQfxSmE8UgSPFMGpSEw85DBefH8rbhHCK4fxSWEUWeKoM87Bl53D8j9/SCSq8b/nHMTYYXmEIirXPfABtf5wYl+BSxZPZtb+gzJq2X3Pr2fNxvImyw+bPJDLTto/w/71P77aWsX/PfUFWR4Ht//4cDyu9OamtiHEr//2KT87axqDC/WY72A4yi8f+YTvL5zM2GF5RFWN6//4IZV1LRcZWXLMGD78ah/DS3L4weLJPPji13z4ddwWHT11MB6Xw7wX0zFqUA4nHDyM/3y2m+MPHsaDL35Nsj3O9Tm5eOEkfv/Ml2nvx+aQ0fApGleeexjFxTmt3r4lMo6T37RpE5dddhnXXnstDoeDrVu3Jnyfrsxaqqo6zVFYmN2q9a10xgEyqG/UDcUL72/l3AWT2bKnjgmjCtlVrtcarWmMZPT7e6oaKSnwkeNzsqvCj8fjZECOm8J8L/uqA+Y+6gJRJAnOmTsxYfvNu2r46Kt97KkOoGqC044Zi8eddBqF4Kk3vuWvr3xDNKpx9pwJyHLz5+GJV79hT3WA0uoAB00oYeKogmbXl7Qo3mA5vsY9eAMVuENVuMPVKP4KKvbkkSffwN6qRgrzPMybOcpslytciy+wF0+wAk+wAm+wAneoGmekHlmkrvYkkNAEKGvX8sMUl0fovWpqZ12GHPYzTKljX1UDs6YPY2+Fn1p/mMOnDGbU4LyEvtYFoxmdr+r6IJ9/W86MiSVMGBk/Jv/9fBdlNYFOveY6m85uu//bCv3/YBS3101xgS/tupWNESrrQgTVeLv2VfoprQ5QF1IpLs6hvjFMZV2wyblI5oX/bqaiPsSucj+7yv0su/gwdpY3MGpwLodPGcLba3ayrzqAz+2kKN/L3MNGptzP+s0VfLm5gp0VjWzeU8fUujCSJHHO3AnmOrtK6/nvF7v5ekctqiY44/hxOB1K050JgStcQ1bjXryBfbhD1XhCVbhCNbjDtUho5HtHA8M6/LxkZOTXrFnDlVdeybJly1i4cCEff/wxFRUV5vdGbceBAwcmLC8vL09Zwb05Kisb2jQZoLg4h/Ly+lZvlynlNbp363YplJfXowmB3x+CmK/d2BjO6PfrGkIMHuCleICX97/cS3VtgKI8D8OKsiit9FNeXk9xcQ6NjWEUReaE6UMStvc6JD76ah+VVY0AHDttMFkeZ5PfWbepnPVbq5g4Ip+5M4a22K5n/rOJPWV6+w8YNYBjkn5XRMOo+75F3bOB6O4NaJU7QIsNqksKUk4hcn4xOxzFDKj+ml1/u4mAdiaTCiMc7fgSdc83qGVbEIF4UW3Jk4OUNxB50GRkXz6SNw/Jl6v/781BcvmQXD40h4tr7/kPJepeigpz+d5JByG5smjQnPzlwX9wIe8x4N/LuGWA/pDwf/IB+4b9D7vD+QDMGFvE9PH6G6YQGm+/8SGBen9G5+s/n+9GE3Dy4aOYvv9gc5uNWyvZV9XYqddcZ9LZ9wtAbV38jbC8oh6pmZKN1bHruaYmfkwrqhvN/ZSX11Ndr9cI3n9k4vWZ3Jf/fLqDhoZ4PeHy8nrq/GFmjC/mhOlD2LClgvKaIEITFOW6m9xjBpKmse67CrbuqgGgtLIBt1NJWH/jjmr++8VuNu2oRpEl5h08DDnm8GpVO1F3byC6ZwNq6SYIxYvPS758pJwi5IETkXOKkHKKiOaNNNvbGmRZatY5btHI7927l8svv5y77rqLWbNmATB16lS2bt3K9u3bGTZsGC+99BKnn346Q4cOxe12s2bNGmbMmMHzzz/P0Ucf3aoG91Qag7pB88ZeOTVN1+CM17NMNbVgSMVb6MDndhAIqTQEohTmupEkEl4DhdBlhWScDn0YpTGkt8eppB5WOXhiCeu3VnHIxMwesh6XQk3sJvK4dE9EaCrq7q+JbFpNdNtnEA2BpKCUjMZ5wByUopHIRSOQcwciyfo25Wv38PfX3uPnyr9ZKv6KNxgi9AFIuQNRhk5GKRmDUjwKOW8QkieztzYFGDm8hM83yZw4cgRKwXAAsoVgXXQ/Vo+azKCqNXxTDnJWPker7xN8834aZ/wUAK9LQS3fRmTjf4lu+4xr82r4tqoBmJD+R2N89m05Awt8DC1OnDYuy1Kn6qh9AWFx1lry2zRT17ZsH/tsOH0RVZcdjXsgHQ5FJqomjtU0BqP4Ys6Q0yETVTUiUQmPz5V2Pzk+ff3dFbpxrm+M4HYm/nZuliu2TgM5PieiZg+h7z4ksvkjRF0ZoF/7zv1mIBeNQikcgVwwDMnZtKRkZ9GikX/ooYcIhULceuut5rKzzz6bW2+9lSuuuIJQKMTs2bOZP38+oNedXL58OX6/n8mTJ5ul4Xo7/mAEiBtAfbBEN/ZAE50uHYFwFI/bYV5wVXVBhpdkNZG7BKmtvGHUAzEj70hzwR82eSC1/jCzDmhZdxZCY4KrlD0NRQB45SjhtS8T/vI1RGMNuHw4x87EMeoglEHjkVzetPsqzPOwUy2idtxiyr75nFDJZGaecAJybnHabTJhwogBfL6pggkj8s1lsiSR43OyN5rH19IR1OdEGFzo47ldTs6vfpnsr//JdJfCwI/+Q2PtTlCcOEZMpX7LWrzhqox+t6I2yMiB2U1kR1mW+tRU+s7AenhaOlbGtZ88aG/9PxrN3MgbDwTQ5SJVE2R5dHPnVGQiUQ1FkZrdV47XaW4PumTrdsalGK2hkuyvXubKnC/5PDyKw9y7aPzHHpAklCGTcUxbiGPYAcjZhc22t7Np0cgvX76c5cuXp/zuhRdeaLJs4sSJZrX7voThyXvcjvhouqalvDibIxBS8boUfDEdvTEUxed2pvHkm1p546IMhKIosoScZszD7VRYfPioFtsT3bWe0EdPcx47+Eodzgb3IEZ9uIpQpBFlyCScR5yPY8RUJKWpJJSKojzdQ9leMIsnGguZO2hkuw08wOEHDCIQijI5aawgN8tFnT9MWXWAUYNzKMn38uJXRXzvyBPI//oNLswGTRuI+4gLcI6dieTOYtv9P8cdzeyVOBCMmOfKiizRpkG2/oRqKdDd0luPluKN2PhsHOeIYeTTvL0aOBXJfCAA7Il54j7DyDv0h4ASbcHIZ1m9fEGdP0KWx4EI+QmteZ7IV28BUKS4WJL1MfXk4J55Fo6xhyP78lLvtBvocQnKeiqGJ+91KfFXSy1+cWZi4yNRjaiq4XE78FoMh9fjIBxRde/dQir7bco1wWiLHk1ziGADwdWPEf3uQ6ScYjY6JrI/37C/cyeRvPFkH3UOSvF+rd5vQa4HCX0MIxhWU44XtIVsr5OTj2zantwsF9UNISpqgxw6uYSBBT4EUDt+IfUVjfx7s5OLzzsXV5bb3KZO+BgazSzTX2MomnCuDGQ5daCBTZzWePKGSdZSePLGcc5YrnHIBEJx/d8w8sa16HDonrxDkdI+MLTGGnI/+Qs35X1HqZpLkVLPPf7FHOqrxP/Uo4hgA86Js3FNX8TKv63H11DKsAn7c/GUA5ptW3dgG/kMsXryhoOianGznIk+Gwwbur5iehUAPrdu5BNuCiFSGnmHEtfkHS14NOmI7t1I8I0/IIJ+XDNOxTVtIauf/YrvdnvYow7gvNOXoBSmT13aHA5FJj/HbeqY1n52Brk+F19t1aWXknwfJQN0Kam0VmVn8RzWfrMVb9KDpl748EQrW9x3JKoSVUXKPsiS1KLO3N+xavLW26O2IcRjr3/LxQsmmQ/Q+BsxTbZp4slnINcEw/GB15SevGHkU+wrumcDwTfvRwoHiOJlrLMUBypXZL3EwHAdUslovAv+B6VIHyj1ZmWxrb6YSdldp7O3BtvIZ4ihyzkdsnlBWl9HM3HqAmHdu/C6HQkSgM/joM4fTvQMBaQS5a1yjcuZIlSrBcIb3ib03qNIucX4FvwPSqEe2+/xOHgtOEX/nMJzbQ2FuR52xiJ1fB3kyacjz/JKPbwkm/xs/e+q+iCBkIrLITd5GNYLH26tEaFFkeT0fTUe7CnlGluTbxGr42OVtrbsqWPNxnJOPGwko4fk6uvGbiUtYbA2UcIxjHy6cSgDpyITDFs8+cokI6/oA6/hFHJN+MvXCH34d+S8QXgXXcs9f91ENBTkVN+nHOr6jrXemRxx0g/MQAOIX4O5WekHcbsT28hnSGNMrhFafHaaqja9IJsjGBss9bgcTTz5Jpo8uu6bjDW6JsvbOgMa+uIlwh+vQhl+IN7jfojkjnvrXstElXYb+TwP3+3WQyU7Sq5Jh3FjSZJu5I3PNQ3htFJLg6THa4vGWqRmBsWMCCZvGk/e1uSbR01hsPXPTZel8uTjsmjrNHldronnzIp78vHoGoBQWDU/CyEIf/YC4TXP4Rg1A88xlyC5vGT7drIvqPGUfyavBqYwYcgYjpQTnatcX8828nYWygwxbnh9GrW+LKq20pM3jIY7Ua7xehyx6Jr4uun0XsMrFaLli91K6IuXCX+8CsfYmXjnXZVg4CEeNSRJ+mzZ9jAgJ65/d7pck6XfuIMKfMiyhCxL5Ga5qGkIEUhj5OvR+y4aa5rdd9yTb/qgsjX5lkmlrwOWwIWmD4GECLOkh0EkFmffslwjJXjyNQ36REYzusYyWcm4h8JrntMN/Pgj8ZzwYzOCzAij1JCp0rITomsMcm1Pvm9gyDWahjlAGk1x4TaHMRjkdTsSPOe4J28NoUw9W9h6gbf02moQ+fZ9wh8/jWPMTDzHJL5qGhhG3uNytHqWcjJWI9/at43WYszkHWwZQ8jPdlPboMtfKT15dE9e89fQnOBlPNhTR9fYcfItISzZNBJCI1MY+WSDbv2+tZp8qhmnEpjXgnV7p0Mm8s1/CX/2As4JR+M++kIkKf59TlIcfSqJNDf2IMhrJua+O7E9+Qwx5BojIRKQEKaVmSZvePIOZFkyDavP01SuQTQfXZP8OR1q+VaC7z6CMmQSnmMvSWngIS7ReN2t1/mTKehCT/7A0YVMGVPIuSfEcyzlZ7mojXnyvhT9aTA9+epm921OgEsl18h2CGVLJMg1KT6njqSJb29KOMabs2nkm79GHUr8xjEcDrdLMcONrW/ABYFtBN/9K8rQ/XEftTTBwEPckzdI5cnvP7qQGeOLGdhM2obuxDbyGWJ68rFJUADRVsQBQ1yT91qMO8Q8eZImQwmRcsardRCxJblGC9QReO33SN48PMf/qNlBRqsn314G5MSjDDpbk8/yOLnqjKkU5MZ/My/bTU1DKK0mH5B8aMgIf/NGPtCcJy9LWE6/TQpS6fDW5VrSm2uTbdJ58krzb5rW+2LsUD1e3Srf+MLlDJRryJKCjNv6D+S8gXjnXJ7y/pg6togjpwxGib0xul1NjfzQoiwuP+3AdoU0dyY9s1U9kEZTromnFk2cOt3yPozoGsNr9pnec8yTx+K9QEpX3noBN3dRCaERfPvPiGAd3rlXIntzm22bIR95UlzEraUg1+LJd7Jck4r8bBf1jRH8gUhKIy9JEg1KHlps2nk6TLkmzcCrrck3T7rompSRNCk8+eTY+dakNTAYMzRxUpIWrGe/L//IFbmvcU7WahQ1gOf4HyK5Unvh08YWcfGCSaZMk8qT7+nYRj4DjNwXQCyVgRFd0zpPPhCKIkuSObDpcztwOfUQP+NV0tiLSCPXyLJkehXNefKRL19D3bkO98yzzXje5vDEZA1vBxj5XJ8LJdbO9g7itoX8bDcCqGtMbeRlWaJByUer1VPPrtlYxt5Kf5P1GoPRtH2QJTuEsiVSDaxaP2spNPsEnT4prYEZQplBdI3B8KScQ6EP/o4SDZAjBznQtYuKkXPNMOLmMHLWuJy9z2T2vhZ3A5GoFh/914TpbSQnQWqJYEjF61bMgU1rvLxp0EV8f+leSo2LON3Aq1q1i9DH/8Ax6iCck49vsV0Ql2k6Qq6RZYn8bFfsDaV9g7htIS87PgCWzpOvUwag1ZUhhOCvr2zkP5/tbrJeYygaGy9p2gdZlhBknpiuP5LKYENquSZVdE2TOHlVw6HILV5TCW+7ToUZ44tZdPgooju/JLppNfX7ncCHoTGsDw+jbuQxGfXF3Ys9eTu6JgOS4301U65p/cCr1YhOGDHAjD4xLlzNIteku5idikwINa0nH/r4H+Bw4z76ooyNbFyT75iLeECOh1p/qOUVO4H87LhclNKTl6BOyYdAEBGoJapqqClOYGMw9ZsAxKN6NE0gt6AR91dSGXFo6qGDNbrGur3+v1WTz0T3tg68OhWZy087EKGp+P+xDDlvEIHxc/n7p2sBiZ85MzOBtpHv4yRHCSQnToJMQyijCdEr8w+LvyYattjcTRq5BuKaZOop2d+g7liL69AzkT2ZFx8wNfl2ToQymDRyAFUZVPDpDAYO8JLlcRAMq01e10GXWurkfAC0urJYCbem+9GTx6Ux8ub5sj35dFjlmFTGu7k8NdD0YRDN1Mhbw4xjBj/y7XuI2lI8c3+K0+XGmE2e6WCpy2Ub+T5NoiePRa7REpa3RCiipr1IDI/bmtUynX9oePDJ2qQQgtDHTyNlDcB1wAktN8iCocl3lCd/6tGjO2Q/bcHncfL7q9LXMZAkiVpZH5ATtaW6MUphrAMxuSYVhievaoKuH1ruHSQOvDYdv0rw5FNskyqtQSYTAK3rOBwyQo0Q/uwF5OLRKCOn4axsTPg+E0xPvoPuj67E1uQzIHmA1fQsEuSalq18VBVpB42SPXn9vzRyTRpPXt2xFq1sC64ZpyA5Wjcxw+tyUDLAy7Ditpdf7C3IEtRLuSDJaLWlsSLPTdfT00KnNvKKIa/ZYZRp0TQRDyiwDrImhUYa60Iaj99SNCQzuSYxzDiy4W1EQyXuQ05DkhLz1WQ6a9ww8m3JF9Xd2J58BiSHesU9i9SaYzpUTcOTzpPHiK6Jh5Klk2scaYx8+MtXkbIKcI4/ssW2JCPLErdeNqvV2/VGJFlCRUbKLkBrqEDTClI+pMMRNe1NLcmJYyg2TdGEwKFIhKMiyUOPf2+Qqi5DcsRN5pq8sY7AsfZZQhteRxkyCWWoXnS9tRMKIR5dk1wZqjdgG/kMSNbezRSorRx4jaoCxZP6IpGbePKpUw2DVa6Jr6BW7UTdswHXoWemndVqoyNLuoGRswvR6itj8xOarheOamlD5mQpPvBqkxpN099cw1EtTZ4ay7pJXrv1c2sHXp0O/dwc6/kaacManBOPxn3YWaYk2jYj3w80+YaGBs4++2weeOABNm/ezJ133ml+V1paytSpU/njH//IvffeyzPPPENurj755swzz+S8887r+JZ3IU0GXo1BIss6mcg1qqqZMe5NSNLk9ep/Lck18Qsu8uXroLhwTewbNXU7EzmWDE7KLkTbswGgScEWiHnyaabQy7Yn3yKaJvS3zlDzM1khneFPXJapJu8JVVIgN3Cidy3yiGm4j0rMR5M4oTAzo92bJ0NlZOTXrl3L8uXL2bZtGwCzZ89m9uzZAJSXl3POOedw/fXXA7B+/XruvPNOpk+f3jkt7gaSB15T3diZOHSqJlBa0OS1uI3POLpGC9QR+W41zvFHZVwcuz8jxXLBy9mFRBtrkNFSe/KR9J68ItuefEtoIv62mTKtQaqJTymyUKoWTT5dtJO5TTTMwI/uZlleEKek4T7opCb5aBxt0OSL873kZrl6bOqC5sioxU8//TS/+MUvKCkpafLdbbfdxtlnn82oUaMA3cg/+OCDLF68mBUrVhAKdU+sdEeS/KqZbBCUDNPORlUtQWKxIqeIrkln5R1Jck1002pQozhbGVHTX5GJRS/lFIEQ5Mv+JucvquoT4NJq8uZD2Tby6dA0DYccc0RaSlBmODfWh0HSeno1p+ZNVnTHF8iRRpySxs5oIY6SplFesmSZNe7IbI7DsdOH8ptLZ3bL5L72kpEnv3LlypTLt23bxscff2x+7/f7mTRpEtdeey1Dhw7luuuu47777uPqq6/OuEGFhW33RIuLM48Lbw1VjRHzs6zI5OV5E753OGQURW7x9wUS2T53yvVyYkm9Cgr0/rtcDhxp9pkdy1tdOCCL4uIcdm37BPfgMQwaP7F1HesiOuu8tBW324kmBAOGDmcfUCD7cbmcCe30B/RzXpDvTVhufM6PXQP5+VkUF7WtVGJ309nnRXEouI08TT6X+Xter6vJMl8sTa/XGz8PWbFJbca9pQn92k/VbmPZvrc+At8A7ts3g0ZHHvel6aPLqRAIRRk8KK/HGe6OPi/tGnh96qmnOPfcc3G59BOUlZXFgw8+aH5/8cUXs2zZslYZ+crKhja9AhcX51BeXt/q7TKhskrPa+JQZMLhKFXViXlOFEkiHFFb/P1wRCUSiaZczx+bHVpR2UB+jptgKIqmaSnX1WI5PAKNYUo3fUt432bcM8/ptP63h848L20lGlWJqBr1qp6U6orc1/ioGsrLx5vr1DTo5yMcip8va1/8DfHz5RC9L46yK85LKBRFinnhdfVBvttWSV6Wi4bYtV5XHzTbUN+gT5xr8IfMZbWxyXSh2DkIhqJoatN7wuiLFqijcfPnhMefwMZdg8h2OtP20aFIOBSZiorMCrp3FW05L7IsNesct0tgevPNN1mwYIH59549e1i1apX5txACh6P3B/AYZf6cDhlNa6rDKoqUUXSNqsZfX5OJx8lbJkOlk2tMTV4i8t2HgIRj7GEZ9MQG9OgaIQRSdoG5bL/AVwnrhCN6xtB0CdZkW5NvEU0I81pdv6WS//nD+9Q0hJotGpIqn7yRciLaQpx89LsP9YD8UTOB5iNnnA65V+rrbaHNvayqqiIYDDJ8+HBzmcfj4fbbb2fnzp0IIXj88ceZM2dOhzS0OzEGfpyKhKDpxJmMNXlNoKTR5M38ZMZuRLqpUPHBIqcsEfnuA5Shk5B9+S3+vo2OJOm54CWHCzH6cADCkjthnXDsbSldNIWvfifjHXtsI98MmiZwxB6GVfUhVE1Q3xgxJ0alLNqdKv2wNYSyucyrm95HLt4PpXAoQNrxL9DvIdvIt8CuXbsYNGhQwrKCggJWrFjBj370I+bPn48QgosuuqjdjexuTCPvkBPyyRsospxZdI2qpTfyyQOvtBxd4/XvRtSV4RzbPyYxdRTW+qxi5vfYGBmMWwskrBOONJ+7fNCmVZyT9YE98NoMVk/eSBNszeiqJXjt8f+NmeRNEpQ148lrDZVoFdtxjj7UEpjQgiffihrJvZlWaSlvvfWW+XnKlCk8/fTTTdaZN28e8+bNa3/LehBG3g2HIqcModTlmuZvdiEEqipalGsSU66mi66JpSou/xIkGceogzLtig36UbUaGr/mZrBWm7COKdek8OS1ujLc/n24FagK1gE9a2C5p6BpcUNrFPyIqvGJUam89pqGED++87/877nTE2a8CiGanQwV3bEWAMfIaQjjTbcFI29NFd6X6R+PsnaiWTx5oYkm+UoykWs0oU+3SefJx0MoMf9vyZN3l21AGTgWyd07ozu6C92T1z9rmsAv3LhEkicf8zxTxclHt31uflaqd3ZeQ3s5qibMUMVoSk/eoskTN/JRVaOyNpiwXtQyLpaK6PYvkHIHIuUNwhELi2wu+Zgt19gkkCDXiKZyjUORW0xUZVykLSYoS7EsGadDIVdqRKndhTJiSovtt0lEkqQEA+IXbtwiiLCcRMOTd6eYERnd8QURbyGaAKV6e9c0uhcihJ6gTJakuFyjamlSGOj/Ry1evnVmrFGKMdUMVS0cRN3zNY4RU5EkCUXWK601J9d43I4Oy7ja0+n9oS9dgGHkdblGNJVrZCnltPiEfRhGPk1aAzNBmcX4pEtr4HEpTHLu0fc33DbyrUWPrtE/a0LQqLn1Ix1uhNiM4XBUN/LOJE9eREOo+zYRGH4kjVs+J792Rxe2vHehCYEkS8hy3MhHLXlsUiUoM/JBqVo8wEETenQOxAtzWwls+xLUKI6R08xlDofUbMHvs44bmzCTvS9jG/kMSAyhTCHXZBBCGY1t1FJag4T9pLlGD5lYwtgtDUj+fOSC4alXskmLZKnPasg1ACLYYKaFMAZek3PXqPs2gRYlXDSBvZu2Uugv78KW9y4MuUaWIfbMjHny6XPXGG+81jdmTdP45JsyCnPd7De46fhH46ZPwelBGRSf5+BU5GY9+cGF/UfitOWaDDAuQGPgtYlcI8stRlkYD4qMo2tEPDNlMh6nRE7tJhzDD+xxs/V6A9boGk0QN/Kh+MSYeAhl4i0S3fUVyA6iBaNp1NzIkUZsUqPnk4+PN4GuyRshkQk55mO3T9TqycfWC4ZVvtpaxYwJJU2udyEEjd99hmPYAUhK3Gd1KHLGBUH6OvZRyADjFdL05FNG12S2j3RZKJskKEsTXSOEILLxXQgHUGyppk1IkiWKSRP4tbgnb5Auukbd/bU+2O30EBBO5EjALgGYBqsmb6APvOqfm8tMqVnkmoZABFUTDCrwNf2NulLUhiozV7yB0yGnncjW37DlmgyIT4aSY1WEmsbJt3SjRy26firkJL0mXZx89Nv3CL37F6TcgTiGHdCKXtgYyLHJUBAfeAUQwfh08nBURZISH8pqYx1a5XZcB5+GLEs0CjeSUCEaBmfiZCob3WExNHmDqKrF9fcUM16tnrz1rRZS3zvRvRsBUIZMSFh+/tzxCQXd+zO2kc8Ac+DVjK5J/N6hSC3OfDQ8+UzL/yFSFw2J7liLlF1I1pkrkWT79LUFWYoPlCcYeatcE9FwOZUEeSCwfT0AjqGTkQU0CldsOz+SbeRNvttVy0sfbCMS1esnWF9e04VQNtHkU7wxO1JkjFT3fYvsy0XOG5ywfMqYog7pS1/Afp/JAKsnnzJ3jZzBwKuhyacT2kksQqGJpkVDhBCo+75FGTzBNvDtQJIsaWw1CAonKjIiGE88F45quJNe9wNb14HTi1y8H7IkEdBiRj6cmLCuv/Pd7lrWba7EH4jock2SJ5+crgBSRddoJOd9SzW5Sd37LZ7hk+yxqWawjXwGJIdQNsknr8gthlBGzVmzqS/GrJrvOMbzdbPRNaJ2HyJQhzI48dXUpnUkTIaKjX2EJHcTTT5Zjw/u+Bpl8HgkWYnJNYYnbw++WjFmiAtoYuStmrzaWk8+ychrDVWI+nK8IyZ3dBf6FLaRzwBTanFIqQdeZanF3DXx6JrUh3zo2j9yqu9TRMx9EUI0OTmG/uiwhIrZtJ6EyVBGBAeeBI88HFETZkSKkJ9I5W6UkjGAbrwaTZnH9uStWN90ZTlp4DVdWoOkwVg1AyOv7vsWAM9w28g3h23kM0CLRQnoxjzNZKgWQyhjD4oUco2wvpdGgrFlNBl51cq2gDsLKS8xMZxN67BOhjLOW0jyJBhrvYh33JNXy7cCoMQqDcmyRCDmyWMb+QTUBCOfGEIZTZegLMlL0oRoItckvwWrezeC04Nr4MgOannfxDbyGaCqwvRIhGh6QSqK3HIIpZbek9dq9pmfpWAdEIuuSd5HxVaU4v1s/bGdJEyGip23IO5EIx9REzR5tWwLAErxfoD+oLDlmtRYnSBZkpDkJE8+VVrhpPsnpSfvaOrJK4PGIcn9Iz1BW7GNfAaosTzwhkeSPB3aIbccXdPcwKtW+p35WQrEsiEmRdeIaAitardpZGzaTnJ0DUBQcjfvyZdtxlk41EwGJ8sSQeFCINkDr0moTeSa+HfWyVBaioFXg1Qpva0DryLciFa9G2XguI5sep/ENvIZoGoCxeKRJKcozSitQTMhlIaXCCAHdSOvQYLHrlXsAKEh20a+3ejRNfpnYdXkrdE1kbiRF0KglW3BPTRuUPQHhYSquG1NPglVTfTk0w28pgqhNPfRgiavVug5g5TiUR3V7D6LbeTTIITgXx9so6ouiGZ68vp3alLyGofS8mSoeIROiljfyu2EcvUcNFLQ8OST1jE0YdvIt5vEtAaGkXdBJIDQVAKhKPuqGinK04uri/oKRLAez5BxCfsAUB1eW65JInngVbFq8qo1J016uSZVjiirXKNV6Nk/5UJbj28JO9g6DfWNEZ55ZwselwNV0/TXTuPGTvbkZQlB83VZo2nSGghNRavaSXjYEVC7FzlkaPIi4TVXLd+K5MtHzhrQQT3sv1jTGhiGJEjMoIcbWbvFT1TVmDGhGNClGgD3kPGEYvswroWo4rU9+STUBE2eRE0+qjabhdIgZQildfZxxTakrAHIvrwObXtfJGNPvqGhgUWLFrFr1y4Arr/+eubOncvJJ5/MySefzOuvvw7A6tWrWbx4MXPnzuWuu+7qnFZ3AdYcGnpFp7gmH7W4GBKWgh8Z7K9JrG/NPlCjRHKHUad5TbkmObpGK99qe/EdhDGADnFDEyA2YzXo59NvysnLdjEmltZWLdsCihNXyYiEfQBEFQ8ibHvyVpoLoYyqlhTCKdIaGKgpNPkET75yu+3FZ0hGnvzatWtZvnw527ZtM5etX7+exx57jJKSEnNZMBhk2bJlPProowwePJjLLruMd955h9mzZ3d4wzsba34NVQizEAEkavKyLFlSEoi0lT7SJSjTKvXXzmjuUOo0L7mGJy8sxb3DjWi1+3CMO7xD+tbfSU41DBC0pDbYVdbAhOH55vnWyrciF41MyHJofBeRvRCq6srm93jUpPvDWvEyEo3XOU5V/s/6d7KEYwy8ikgIrWYvrv0O6eCW900y8uSffvppfvGLX5gGvbGxkT179nDjjTeyePFi7rnnHjRNY926dYwcOZLhw4fjcDhYvHgxr7zySqd2oLMwp14LEQ+hTCHXSJJkSjTJGuLGHdVs3qN75tE0k6HUyh2gOFGzB1Kn+Uy5BuLSj1q+Td82FqNt0z5kCVNeMzz5Rgwj34gWe6iDPodBrdqJUpToNRqGK6p4bLkmiYToGinZk9eahK9CU7lGFaKJ4TfegrWqnSAEStGoDm553yQjT37lypUJf1dWVjJz5kxWrFiBz+fjsssuY9WqVfh8PoqLi831SkpKKC0tbVWDCguzW7W+leLijiuoHIldmF6fC4dTwe1SyM3VdVunpWyYokjk5OjLC4uycVvC7i6+VS98/uL/nYzHq8dUDxqYi9cdP+x76nYjl4xkQGEOG4UXR3hfbL8ybreD4uIcar7bSwAomXAAiq/3FY3uyPPSEWRnG+crh+xs/aEaUbygQbZbRZYlvF4nxcU5RGpKaYgEyRupD7oafTFSEePOQtQ2UlSU3evmL3TWebHeH7m5HtyW610TIMeMtaLIZhscSSkknE4HipJo5AcPykWSJGq376MRKJ6wP45cffuedo21h47uS5sGXocPH84f/vAH8+8LLriA559/nvnz5zdZt7UXfmVlQ4sx56koLs6hvLy+5RUzpKJS987q64MEAhE977hfH3Zr8IfM9SSgsVH/u7ysHneKupFffL2X2jq9UHR1lZ8Gh+ElCoJ7t+Dc72BqawP4NTdSNIhQo0QiGuFwlPLyegI7NyNlDaDKD/g7ro9dQUefl44gEAgDUFZWR22tfl4aok4A6soriKo+QqEI5eX1RLZ9A0Cjq5hcMPtiDKQ3qk5Qo5Tvq0Ry9J5MlJ15XhoDkfhnfxg1Gn/FDYajpmQZjF3fAKFYDdf4PsIJOr1Dkaio0HMLBbdtRPLkUBV0IoXqe+Q11lba0hdZlpp1jtsUQrlx40ZeffVV828hBA6Hg4EDB1JRUWEuLysrS9DsexPWV8omk6GS4oClpAySBtle3XB8urHcMuM1/tAT/ioI+ZELRyBZZlCqgYZYdE1sv9V7kAcM7Yxu9ktkyxiKKddYZq/qQyuxY1+1E5CaHH9DugsrHnM7G50mA69JcfJmnvgU5f+s+7DuJzlGXi4a2evenLqLNhl5IQS33HILtbW1RCIRnnrqKebMmcPUqVPZunUr27dvR1VVXnrpJY4++uiObnOXYI0AMEMozeia1DP6kiMEcny6kf9udy1R1citnTTBCVCKRiJLkpnXXAvUW3KraGg1tpHvSGTLGIr5MEcGpxcRaohVNNLX1Sp3IuWWIDk9TfYhAWHJMPK2Lm9gBBkATcv/qanzyaeKrrF+bxh5oUbRqnehFI7AJjPaJNdMnDiRSy+9lHPOOYdoNMrcuXNZtGgRALfeeitXXHEFoVCI2bNnp5RwegPWWF5Ni0XXxB6JyRexWZ81KYjS8N5VVUNVRZPIGrVyByAhFwyDirBZhk6NGXlJ0ifiEA0jDxjSGd3sl5heusWTF4DkyUKE/DGDY3jyu1AKhqXcjyxLhGXbyCeTECeflNYgak013EJ0jdVTNyYRanWloKn6PWOTEa0y8m+99Zb5+bzzzuO8885rss6sWbN44YUX2t+ybsaaRCmqCZxK6hBKyRJCmepC1felx9YnR9ZoFduQ8wYiOT3IUsRMXasF6jGi7rXq3QAotiffYSTKNfHPkssw8ronL6IhtLpSHGMOS7kfSZIISzEd3pZrTBLkmqS0BqomTCep2egaTSSEXpqRNTV79f3m205PpthpDdJg9eQNL9yc5WiJlbQuT37ltD4okj15IQRq6XfIA8cCutduyDWGJy9LEmrMyNuefMdhzMDURGKorOnJE4ulr94DQiAXDk+5H1km7snbScpMmpsMBXryN0jW5Jvuw7ofI7e/Vr1H32++nW47U2wjn4b4jFdick0zA6+GXJNi8Ah0Yx9VtYS8NaK2FBGsRxmkh+ZJkmTKNVqgAUMx0Kr3IGUVILmaVqq3aRtyKrlGoGeYjMk1kqTr8QBKQWojr8gSIWy5JhnrmJWSNPAK8fDTxELeTePkRQpNXqvZq98PSWMkNumxjXwarAmsjIFXc3JSgiZvkWtSeCMQe0XVRGKEQOkmABSLJx/GgSYpMU9eIKHLNbYX37FYB8qFZjHyFrlGkiTUqp3gcCHlFqfZj0QIO6d8MunkGlfMGw/FjHxzWSiTZ7yaRr52H3J+YtFum+axjXwaEnLXaAJFiQ+8Wj0VyeLhN/HkDb1XE2Z0jbn/0k3gzjIvWP0BIqE6fWiNepysbuT32pE1HUx8hrJFkycu12jGA7ZqF3LBMCQp9W0iSRIaErjsJGVWrB66JMUHXo05JMl5g/TPTfeRGF2jZw7VavbaTk8rsY18GrQET745ucaSYybFhQr6q2cTT756D0rBcNOAGPtQHVmogXo0ATlaLah2ZE1HEx9DSSHXaCouIkjo6SSUZpJgGeUgJXeWbeQtJGrycXnMnTSrNSFBWYqgBesyhyIj/NUQCdqefCuxjXwarAmsTE0+xcCrVcZp8srZzMCrVrM3YfDIkHxUh1ePrhGCvKg+scyOrOlYjLOgWXRfTQiIVX3yECInWgGRgCmnpUKOVQSTXFl2JkoLVk9ekePFdjyu9EY+5cCriIdOOh0yWo0x6Gob+dZgG/k0GHbc8OTlZgZeDRkn/cCrPg3eCKEUwQZ9pqulILcZnun06Zo8kK/q2Q1tuaZjsUZDaXG9xizt5yFEQSgWuloyJv1+pFg8t9tne/IW0iUo87gSI7ZbCqHUhDDvGYciW8InbSPfGmwjnwZrUQlVM/LJ699ZK0MlRtck7cN8G9DM1AigDx4BCUY+7sn79OgaIchRq5G8eUgub4f3rz8THyhPipOPGXkvIfKDu8GdhZQ3sJn9xOUaO04+jmatt2B5A27ek08t1zhNIy/pRt7lRfLmdVbT+yS2kU9DYpx8YmWoxHzyiUYj5T4sDwqwTuiwGnmjCIVPz10jBNlqDXJu78z905ORpaaevCZAcutJnrKkIIWBbSglY5rNj6Jr8vp2ItTQ+Q3vJViNtyLFy/8lJ+9reeA1nuvJGfPk5fwhds6aVmIb+TRY9XQjv7gZQpnmddRq4zURT3JgpEYwHhJabSlIClJOkbm+cdlGHT7QoriIkK3WpA3fs2k7idE1lslQbn0uwqHOTfgiNTjHH9nsfmRZQtUEkjcHEWxACK3Z9fsLCfeHLGEEJ3maG3hNMZ4lhMAhx9MS60belmpai23k06AmePKxLJRm0RDNNOySnHoyVHLVGzXByO9Dyi1CkuMaZdyT16WZbOHHp9bbnnwnYJ4HkRimZ3jyE5178LsKcex3cPP7kSSEJpA8OSA0W7KJkc4JauLJN5O7xtDkjYFXH0FEY41t5NuAbeTTYI2uSQ6hjKpxfV2RJEu0RtPtIXbBasJ8bdXqypsYb+MNNOrQvclBUoVeP9Y28h1O4mQo/bMmAIcLYg/erQVHIMnN3x6yEULp1Ys8aMG6ZtfvL1ivfUnGosknDrwKwBrCmrwPzRJ2XBKJzT4eNL6TWt13sY18GuI5TYgbeSMLpRZPUWDV6hM8eZH4KprgyTdUIOckyjCGJx926J78EMr1/efYck1Hk1aukSQkdxZVahZl+Qe2uB9Zisk1Ht3Ii0DfKFzRXpJDKFN58kY4sXmfpcjgKiyafHFwOzjcKCX7dWrb+yK2kU+DcfNHYikM5GRPPmbx9VTD+jYijSdvyAKKLOnx1CE/skWPN/YD+sArwFCpDADJ9uQ7HHMMBZHwMAdwHXwaT/gPNz36Zvcj6+fWNPJB28hDqrQG+mdrdI3DUh0teRvjb03Eo2sKG7ehDJ6QIHHaZIZt5NNgXHORWMa85IIfhodh1eRTRQs4FMlMryrLElqd7qFLSUY+WZMfLFUSlZxI3twO7plNPDW0RS6IeZKOCUezKTqYTAI4EjR5bE/eIG2cvGXg1Yw0i8ll6YqGKIpMntRIVrgCx5BJndvwPopt5NNgeBZGLU9Fls2Ze4A56i9LqStDGRe6Q5HNEEpFltAa9FmsTWSY2D7CMSPvlFT8jnw7XKwTSExrQOyz/r9h9DM57rIRQum1PXkDQ5o0sMqZVrnG0NqtAQ5WtJhc41Akxjljxe2HTu7UtvdVMjbyDQ0NLFq0iF27dgHw1FNPsWjRIhYvXsz1119POKwXR7733ns59thjOfnkkzn55JN5/PHHO6flnYxh5BM9+fj3hiefLtWwlmDkda9EliVEXWojH5cQZORYKJ/fkd/BvbKBpMlQplyTKNtk8mg1NXnFCU6PbeRp6pFbZU5r7hqz0lO6gVcRH3gd79xLVPGmzetv0zwZCVxr165l+fLlbNu2DYCtW7fy0EMP8eyzz5KVlcV1113HE088wYUXXsj69eu58847mT59eme2u9NRkz15JTEvtuGJyNbKUCmMvNMhJxQe0erLwekx86QYWHV92ZuDFmrE7xjQKX3r7yRMhkoyMsL05DPYjyyZYzaSJ8c28iRKNRCr8Rq7bxRFwumQiUTjKT60NJo86Pfe4CyVqdkVKIMmpc0GatM8GR21p59+ml/84heUlOiDgC6Xi5tvvpns7GwkSWL8+PHs2aMnD1q/fj0PPvggixcvZsWKFYRCoc5rfSdi3OyGJ59c4cbQFK2FipMnQ0HcY4mqGrIso9VXIOcUNZEDjEBMIQSKV4/XbnTmd3CvbCAxukYkDbwapzC5mlEqZFkyt5e8ObYmT+IbLCTeN4okmcvN/PBpomsAnFoj83b9AU+kFs+oKZ3e9r5KRkZ+5cqVHHxwfGLI0KFDOfzwwwGoqqri8ccf5/jjj8fv9zNp0iSuvfZannvuOerq6rjvvvs6p+WdjJpSrrEOvFo9+eblGohF5EgSor5p+CRYJQTdkwfwO21PvjNIrPGaLNe0QpOPyTVge/IGRl4nt7PpmJWiyGYZP1OusaT+SKZQ1KKg4j78fJwTj+7klvdd2hWPVFpayiWXXMLpp5/OYYfpxY4ffPBB8/uLL76YZcuWcfXVV2e8z8LC7Da3p7g4p83bJuP16RV/jAIh+Xk+iovjbfN6HLH/XRRqpeRJfnLzvGYbgrGL1ut2AnooZlaWE7G7At/YqRQltTUYigKQleVGicYG8rKLO7RP3UVP60NZvT5+lJPrxRmboCOAoqJsArHzkJ3tTtlu6zKPx4E/FKW4OIey/EIC1Tt7XF+bozPaWtugv7l73Q78Qf3Y5O7RH36FhVn64Ks/fl8MGJBFcYEvhR8P+ZK+XfEBB+Mqbj4pWW867i3R0X1ps5HfvHkzP/jBDzj//PO5+OKLAdizZw+rV69myZIlgO4VORyt+4nKyoaU+lxLFBfnUF7ecZ5UfX0QiNejbPSHqK6Kp5M1XtPD4Siud+7jZF8R1dVHmG2oqDQSVlk0x8Y6RCRIyJHbpK1GSbSGhiCyLxdNSFSrWR3ap+6go89LR1BXFwCgpqaRQCBiLi8rrzcfto3+UJN2J/clGlEJh1XKy+sJSx5Ufx1lZXW9IiKqs86LYeSNN9jqKj+Njfqy+rpAXNqMue7lFfVIqoqmaUhSouQ5QNbvt5qIF6mZtvbEa6yttKUvsiw16xy3aSSjoaGB73//+/z0pz81DTyAx+Ph9ttvZ+fOnQghePzxx5kzZ05bfqLbMV7joxa5xhpCadatJIoSbmCMsyyx+nySXAPgU/Vp71IKuUa2DN7mHTyfJ0NHI2SlyXo27Scx1bDFqoi4Jp+pXGPKO54c0KIQCXZwa3sXhnw1ZUwhx0wbQrbPGdfkZTmeOthhDLzq2wmBOcHQoED2E5a9SE53F7W+b9ImI79q1SoqKip4+OGHzVDJu+++m4KCAlasWMGPfvQj5s+fjxCCiy66qKPb3CWYkzRifydr8lJswDVL6E/dfLkRKVAV3z528zuV+DZZ0RogdaoCa056Z8EQ1kZHI2UUyGfTWmRz4LXpzGQzhDKDQy/FcteAHStvYBj54SXZLJ0/MaGQtyxLOB365+SBV82styAY79iDgsoA2U/A2bxMY9MyrdJS3nrrLQAuvPBCLrzwwpTrzJs3j3nz5rW7Yd1NsmSUHF1jTNf2afGb2l2zFRgX215fluDJm0Y+cbYrWEMojWiDzAxNawgE/DQ01KCq0Y7dcTOUlckJRSR6Ak5N8LNTh5LrqWfeNB/HHqBX3iov099Af3bqULI8Efbt256wXXJf5kzxEFVd7Nu3HeEbiDj2Shrq65EaA13an7bQWedFVfXjl+0NmMdvaJ7GmEFuHLJkevJG7pqn//MdPzrlAAR69M1QpYrLc9/gw9BYBsgNBJ2D0v2UTYbYiSDSkDwDT4+Tj/9tTNf2aQ2x9Q0jH9vekGsc8Y280Vq92lCKSk/J1aX0hFkd0hVAN/D19dXk5xfjdLq6TDd2OGRT8uophCIqUdlP4QAvDY0Rc7B14MAcNCEISw0U5HrIzXIlbJfcF6UmQCiiMqg4GxEJodXsQc4daOal78l01nkJR1Qisp/ifC9ZXidCCBoDAc44WiPLpZr3g+H8rNtcybrNlYB+jw0X+tvwTPd3AOxwTezwNvY37NkFaUie1KHIchO5RpIlfKruyW+JluCp2WZ+H5drLEY+XJM2q6SxZ/NXBR0q1zQ01JCfX4zL5e4VA4OdidH7hkDEDJFNu1JLmIH1sYRbmtqepvU5JEkiy+dj1LBhBAJ1CeX8DMwQS1liiKOakHDwbUT34IP2XJF2Yxv5NDTx5OXEGa+yLDF9bBGD3CE0VxbfRgbj8u/Ts0ySmLvGwBuuSpsfPjnWvqPlGlWN4nS6Wl6xHxEIRs0Zzcm0+tBLsUHyLq4OVd8YTqg53N3EB64TlzudLlQ1anry/mBcMgyEVHKkAJc7n2GWexP7tAH8reEovgwPoyZ7dBe1vO9iG/k0iCaefFNN/tKT9qfIFUB48tkSLUFCoJbqr5nmjNfYRS2j4Qk3X7NVkqzRBqINlqZ5+rsHb9KBh8E0arKs77gLPfmoqlFZG6Qx2HVjLC2SJvrZuPaMyVCTRsYn+gXCUUY6yimRqnFJKmWigHrh5c8NxxH0pS+kbpMZtpFPQ5McHJYcNWAZKPVXo/kGsD1ahEBG3bcJsIZQ6isOkP1IaMh56S9aa0ie6GC5xibOOWedQnnZvibLhTWGMgOaPDNlBUQXyjVJGTR7AkZ6gnQOhfFmO3CAj99fdRSgTwTMluPpTxqkeMx3JuklbJrHNvJpSCXXSJbcG2a62oYqhDefME5CvhLUSr1MmfGQMDTIYiUWI9+CJ2/9Wfv67npaby8tW8hyl2ryPci2x2mhUYYnL8vxrJSBsEp+bOLTF+ERrJfjg63222f7saNr0pAcQmmEfOVmOamqCyEhIaJhRKgB4dNfPcPeQnx1pUBc7jEeCkWyPkDbnCcvSZLpCWkdHF3T0/jss0+5//57UFWNwYMH4/X62LJlM5qmcd55SznuuDmcfPJ8nn76eXy+LH70o4s54oijOf/8C3njjVf54ovP+dGPfsJvfvMrysvLqKgoZ9q06SxfvoLPP19j7nv06DFceeXPWLHiRsrKShk1ajThsO417ti+mYf+eCeaquJ0uvjFTTczZOgIoBXjrtbLRFZSJ2HpNHqemTflqzTfG06PHHOYFFkiGIoyUGqkAR+PNBzDsOIswB9br9Ob3OexjXwakjMrGAnJcn0uqupCyLIu1QAQM/IhTxHa3o0ITTM9+Ull/0byNqJIGprsRPKmn9whYTEawljSObz/5V7eW7e3U/Z95JTBHHHg4BbX27lzB6tWvcSjjz5CUVExy5f/Er+/gR/+8GImTz6AGTMO5vPPP2P69Bns3buXL774jPPPv5APP1zN8cfPYfXq9xg3bjy//vVviUQinH/+GWzc+E3CvrOzs7nzzt8yfvxE7rjjHr744jPeeut1AP790ioWLD6Dw2Ydw4fv/4evvlrPkKGxnOWZTIZKWkmSFIQWSbN259HzTD1pj5/hyRuzxz0uJebJN+KPyTTWYAXJtvLtxjbyaUilyQNm7LQsSWj+2AxX3wCgmpCnELQooqFSL12GyqDqL8h3S+yIFhH2FDb7+ilZNXn6vhczfPhIsrOz+fTTjwmFgvzrXy8AEAwG2bp1C7NmHcmaNR8jyxJz557Im2++RjQaZe3aL7jmmmW43W6+/no9Tz/9BNu2baW2tpZAoDFh3wCff76Gm2++BYBp0w5iyBB98tP0g2byl4fuYd3nnzBtxkyOP2GupXVtOPiyAppqFgXvbOLhtj3HzJtpHtIcP6snD+ByKgRCUd3Iy7qzZJ1bYmvy7cc28mlIjq5xJBl5SZIQDfokDtPIuwsB0OpK0bQiRjkqUESELBkmOPdQkzer2d80NHnRBTftEQdm5m13Jm63npNE01RuvPFXTJiga7FVVZXk5uZRX1/Pk08+jqI4mDHjEHbs2MZLLz3P6NGjcbvdrFr1JG+//RYnnXQqS5Ycytatm81jZ+wb9HNlnd2pKLoWfOis2YwdP5nPP/uQV//1DN9t+IyfX7Ms8w6kGnhF6GGUUhfkHeo5tr0paWyzGW0mQXTnl1zuXMW7jceRJzdSJetSmXVuSV93dLoCe+A1DakGXgHyTE8etCS5Jugu0LetLUUTgnHOvQjixUUaBjZfLUuSpKT8Kf3jCj/ooEN4/vlVAFRUVPC9751Daek+BgwYgNvt5v33/8uUKdM46KBD+MtfHuLww/WojE8++YiTTjqNuXNPBCQ2bfo25VT9gw8+lNde+zcAGzZ8xe7degnL39+5gs3ffcPxcxaz5OyL2LRpY6vbnnCVGAnl+vGEqJbKJxoG3FO+gcBrv6eYauaH/kWWHKZR1t+8nA5brulIbCOfhrRyTSzPfGMoivBXgTsLOZYlL+zMBYcLrXYfqiYY5yglkDWEMjWHSjWLcP6oZn9TNjz52N/9xMZz8cU/IBQKccEFZ/LTn/6QH//4SoYOHQbArFlHkJ2dg8/nY8aMQ6ioKOfww48E4Mwzz+WRR/7ExRefx513/pYDDpjC3r17muz/+9+/jN27d3H++Wfy2GN/YfBgXa456bTzeOG5J7jhfy/lib89wI8vv8o8+JkcegkSrLzUxUZeJP3fE2ipLQ6HzBCligFrHkTOH8yzniXIsQlkAUVP8pboyfeTm6ATseWaNDT15GMDrzFPviEQQWuoQs4qiJeTA5SBY4lsfA/3xMmMcFRQnzeTJ/bqF+/JSgbPVGHVNfsuBx10MAcdpFcby8rK5qabfpVyvQsvvIQLL7wEgDFjxvLee5+a382YcQh///uzafdvkJWVzS233G7+rWoaO0v1nEO/uvV+c/mw4uwm571VxIy80NQ+fe4yIZ1tdioyw5VKJATeOZdT/epe/l57LGc436baqUeeWTV528a3H9uTT0P6EMqYkW+MIPzVSFkDLOXkwDP7+0iywuiNf8MlqYTzRrI1WsLWaElCWoRUSJKERlyusa/wziHdoGCrzbvUzXJND3Tl4+NJaSZDOSTyZD1Lp5Q1ALdT4cvAIG6oOZN6tz6HJNvrNNf3uZ0p92OTObYnn4YmRl5JNPKRQAOaVIazZL+EvDNydiHOSbMRX/wLgOiAkcBufR8tGG1DrjHuWluO7HrMs55RCGXCFiB1bWoDY05FD7LxJuku9Tyfi3y5EeHORlKcuJ1KTBqVzOCGwlwPK75/KKGIyn6Dc7uu0X0U25NPQ3KcvJw08DpXvAfREM4JR1tywev/O0YfCkC95oGsoib7SIcRQtmG6oc2HUL8wGf8fE2YoSyZYZRdS8+5YFpqycSRA5i5nwslWw9WcLviUUjG27IsSwwrzmbMkDxbk+8AbCOfBqsnb1SBAvB5HEhoTHNtxTn5WJSS0XFNPraNXDgCv2cg30UG4nBYL+LmD7ceQinimrx9gXcKHXdYU+xIVvp3uuEWBq4lScIZrkeKRaQZqQ0g/ras2K+wHUpGRr6hoYFFixaxa5ceerZ69WoWL17M3Llzueuuu8z1NmzYwOmnn868efO44YYbiEZ7UHa8VmIdgLMaZ1mSWHnBAcgI5LzB5jIgwTivG30RT/gPTwgHy0iTF8RvFPta71IsSllGmDUALNeK1JWevEj4r0cgMrh4RWM1clYqTz4eQ2/TcbRo5NeuXcs555zDtm3bAH024rJly7jvvvt4+eWXWb9+Pe+88w4A11xzDTfeeCOvvvoqQgiefvrpTm18Z2INoUz2LIo9+tR1yaenKIgXho6vE1E8hHEmTNFuyUORJEDEHzB2FsquR7TFyluRFb2gdxcgmnzoAbTgyQs1igjUIWWl8OQtco1Nx9GikX/66af5xS9+QUmJPvK9bt06Ro4cyfDhw3E4HCxevJhXXnmF3bt3EwwGmTZtGgCnnXYar7zySqc2vjMRzRh5EagF4kbe9OQtd5ualGoYMvDkkbpktmt/J60MJhJWatvOFQcIDdHFxUN6Ci0NXIvGGv1r08hbnCDFmDhoG/mOpEUjv3LlSg4+OB5zXFZWRnFxvIRdSUkJpaWlTZYXFxdTWlrawc3tOlSLsU02zqJRN/KyN9GTt9pnLUVlqJblGv1twNiP7dC0jvfee4c///mBjNdXmpm30JpDnxhGGQtYU/t+yuGXX36RlStvTlzYkicfmyUu+5qRa+wLv0NpdQhlKk/TmlgreXlrKSzMbnmlNBQX57R522SsF5rLKSfsu+a7IEGgePhQZLeXSFS/oX0+l7mex+tClqC4KL5dcVE2xcXp++d0KLjdDvNYZud4OqxPZWVywiSTrqSrfveYY47lmGOOzWjdocXZuJwKW/fUmssURUaKHXtFSX28rMsMo+RwxOv/ak4nEUCRNORO7rcciXu+bTnG7T0vslFjIUUaAqezae4eWYLoB4+CJFM0ZizO/BxKChvM73Ny9Jnj+XneVl/3HXnvdzcd3ZdWG/mBAwdSUVFh/l1WVkZJSUmT5eXl5abE0xoqKxuaxKhnQnFxDuXl9a3eLh3hSOLrtnXfwfIycLioqI0gSfE6ofX1QXO9hoYQsixRU+M3t6upacTVjN+laRqBYMRcw98Q6rA+aZpG1FK0OvLt+0Q2/rdD9p2Mc8LROMcfAeiGJJqiWPZnn33K3/72MELAnj27OOaY48nKyuLdd99BCMEdd9xNQUEh77//Lg8+eD9CaAwZMpRrrlnG11+v54UXnuO2234HwDPPPMXOnTsYP34in3++hhtuuJklSxYzb94CPv74AwKBIMuX/5KJEyexZct3rFz5S1RVZerUaXzwwWr+/MjTVNUFiaoaQhPs3LGV23/9B0KhINXVVZx99vmceuoSlixZxMMPP05BQSF1dbWcf/6Z3PmHv/P+e+/x8MN/JBqNMnjQYP7nku+RnxPmzLOWMHnyAWzatJH77vszTz/9d9as+YS6ujry8/NZufI2CguLePPN13nooQfweDyMHz8RVVW54Yab2bDhK+65505CoSB5eflcc80yM4MmgKpq/PoXV5Ofl8fuXdtYseI3VFZW8tBDD+htGTyUa6+9gZdffonq6ip+/OMr+eSTD1m27H95/fW3AZnzzz+De+55gC+++Jwnn3yMUChEKBTiuuuWM23aQfzkJ5eSm5vH1q2bWbHiN2ze/B1//etDZGVlM2jQILxeH9Goxr33/o5PPvkIgcS0GbP42ZVXNDnnaihIuGwH3nlXURPxQXk94WA8NXM4pI9lNLTyuu/oe787aUtfZFlq1jlu9aN86tSpbN26le3bt6OqKi+99BJHH300Q4cOxe12s2bNGgCef/55jj766NbuvseQkLUwhSYvefPMNxWzSlSSXCNLiXVhW5oMReyNKJ6grB0d6AV8/fVXLFt2E48++jTPP7+K/PwBPPTQo4wdO4433niN6uoqbr/9Fn7zmzv461+f5MADp3Lnnbcxc+YRbNz4DXV1erWtN954NZakLJG8vDwefPBvnHLKaTz66MMA/PrXN3PJJZfxl788wZAhQ9E0NWHcBODtN//FuedfxJ///DfuuecB/vSn+3A4HBx33Bz+85839HXefouZhx9No7+BP/7xXv7v/+7lkUee4NDDZvHHRx8z5ZqZMw/n739/Fr/fz44d23jggYd58slnGTp0GK+99grV1dXcc8//cffd9/PnPz9q9ikSiXDrrb/mF79YycMPP87ZZ5/Pb3+7MuVxHLnfGP7+92cpKirhgQcsbTl0Jvff/3sOP/xI1qz5BIBPP/0Ej8fDxo3fsGfPbrxeH/n5A/jnP5/httt+x1//+nfOP/97PPHEo+b+x4wZy9///iwDBhRw//338Ic/PMgDDzxMY6Oe1nnfvr18+OFq/vrXv3Pb/91P6d7dhEKhJu0UagQpuxDHyGnxfQ/N44xjx3DGMWOYMDwfsEMoO5pWe/Jut5tbb72VK664glAoxOzZs5k/fz4Ad9xxB8uXL8fv9zN58mSWLl3a4Q3uKjRNv9hUTSAnxbeLQJ056Apx/dGIirn7H2tZu7kSj0tJkH1a0hrNBGVdECfvHH+E6W13F6NHj2HgwEEA5OXlc/DB+iSygQMHUV9fx9dff8WkSfszePAQAE466TQeffQvOBwOZs8+lnfeeYtDDjmM2tpaJk8+gG3btibs/7DDDo/9zljeeec/1NXVsm/fXmbN0hOcLVx4Mv/4x5MJ2wjgvKU/YsfmdTz66CN8990mM0f9iScu5M47b+f008/ijTde5Zzzf8Dm7zZQWlrKlVf+ENDTJud43WaEzeTJBwAwbNhwfvKTq3nxxefZsWM7X331JUOHDmPdus854IADKS4uMX/jv/99m507t7Nnzy6uu+5nZtv8fj9WDJ9i/ITJAHz99XpKS/cltCU3N4+RI0fh9zdQV1fHunWfc/rpZ/L552twuTwcfviRyLLMLbfczvvvv8uOHdv5/PM1Cde80Ycvv1zLAQdMoaBAT6k9d+6JrFnzCUVFxbjdbn70o4uZdtAszjz3+wmpnk3UKErJmIRFTofMiYeNBGBHqe7B2ka+Y8nYyL/11lvm51mzZvHCCy80WWfixImsWrWqY1rWzWhC4HDIqGHVnG5tIBprkfMGmX9LkpRQ1WntZj3PfBNPPsMZr/0lwMbhSLz8jDzvBskRKkII1JiHPHfuAv785/upr69jzpz5KffvcrkStpVlpfnopdhXv79rBUWF+Rx91GyOP34ub775GgCTJk2mvr6ODRu+oqysjP33P5A9+97kwAOncttt+nyRUChEw97NiJiRN4zdN99s4Oabb+Dss8/l2GOPR1HkWJvklPKkqury1F/+8kTsb5Xq6qqU7XW54nn5p0yZym9/G2+L4W0fdtgs/vvf/wAShx9+JA899ABCSHz/+5fR2NjIJZcsZd68BUydOp0xY8byzDPx8GejD3pefkvUWex8ORwO/vSnv/DFF5/xn7ff4RfXX8599z3IiBEj401VoyA0lEHj0h7+YSXZnHP8OCbvV5B2HZvWY894TYOmCTPlqdU4C01Da6xJ8OQhsT6rgSxLCdtmEl2jpxq2c9eA7kF+/fWXZvrgF154loMOmgHAAQccSEVFBa+++nJKqSYV2dnZDBs2jA8+eB+A119/JeXb0pfr1vC9Cy/lqKOO4YsvPgMwHy5z5szn9ttv4YRYFakxYyfx1VdfsmPHdgD+8pc/c/8jf20SXfPFF2uYPn0Gp5yyhFGjRvPxxx+haRoHHDCVb775moqKCoQQvPHGa0iSxMiRo6irq2Pt2s8B+Ne/XuDmm29I3TERP17JbbnvvrsBmDXrSB599BGmTJnGuHET2Lp1Kzt3bmfChIns3LkDWZZZuvRiZsw4hA8/XJ0yL/+UKdP4+usvKS8vQ9M0s4zit99+w09+cilTp07nwksuZ+jwkWYbzCZGdfkm2ZO3IksScw4ZnhA7b9N+7ARlaVA1YWq1VuMc+vDvEPKjDJ6QsL5hoK3IsoRV6WnRkyexaEifF+VboKCgkGuuuYFly/6HSCTKoEGDuO66m8zvjz9+Dh999IGZez4Tbrjhl/zmNyt48MH7GDNmnO6lJs1zOO2M73HVTy8jNyeH4cNHMnjwEPbu3cOoUSOZN28Bf/7zA2Y5wfwBBVx77Y3cdNP1aJpKcfFAlv/sqiYToo4/fi7Lll3D9753NoriYMyYsezdu4cBAwZw1VX/w9VX/xiXy83gwYNxuXJxuVz86le3cvfddxAOh/H5sli+/JcJ+0z2/wsLi7juupsS2nLTTSsAmD59BpWVFUyfPgNJkhg/fgI5ObqjMnbsOMaOHc+55y7B4/EwbdpB7NvXtP5vQUEhV111DVdd9WM8Hi+jRu0HwPjxEznggCksXXoWDqeLEaPGMXPm4Ykbq/rxkPMHJe/WppORRA+bfdNTomuu+N1/8bodVNQGGTM0lxsuOBitZi/+p6/Huf/xeI64IGH9S29/mzkHD+OMY8dy8a26tDUgx81vfziLS29/G4D7fz67WS/l5oc/piDXw1XnHsSFK15j6bwJHDN9aNr1W8O+fdsZNGhkyyt2MOmia7qLRx55kMWLT6WoqIh33nmL1177N8tv+g2lVY0MKvShCSiLffa4En2g5L7UN4aprA0yrCQ7YT6E1liD8FcjF45EaiFfUW1tDatWPcVFF/0AWZb53e9uZ9iw4SxZcnaLfWkIRKioCZDldVKc723Vceis81JeEyAUVhlWkhjtoTVUsm/fDoaMnd7hv2lH1zQfXWN78mnQhDBvXCMqJvzNOyApuKYvbrK+LKfw5KXEt4AWB5SSv+7fjnynMHDgIK6++sc4HA5ycnK57robE1foiJQSxoQoLQqyq9lVjVq2S5eehaIojB8/kcWLT83wh3qUfwbEDl+qQ6dGY6mYbboa28inwSrXGINk0W/fxzFyGrIvv8n6Rn3W5GWyMShLpgnKBIYcak/v7ngWLFjMggWJD+lAqJ25ZpJsrSQ79EUZJCqTJImrrvqfdv1uzzL1IqWNF5pqG/luwj7qadC0eEoCWZYg5EcE61EGj0+5vpxCkzeMvizrhr4lox2v8dqzbtt+Q0LumrZtBkAs6kSonZuorCcmKBMiTeivFm1RurLpHGxPPg2aJszp2oosIYK6TiZ5Uk85TpVcLBTWPblMc3GY6SFayP/RNiSE0JBsbyotrbWVlpirxC/MMoBdlWq751j5VC0RQugPPPva6xbso54CvTqTMOPjFVlCM4y8N3U5MjP80WLow7GBreRQynSYRUPMvzvOzLtcHmpqKohGI3amyySsR7lVRybN+ZEkuUsrRPWosykSD4sQgmgkSG0ggLPLHno2VmxPPgVmGT+rJx9owZOP6enWPPQRw8hLEiKDx6keQmmd8drWHjRlwIBiGhpqqaoqRevCykX6ZJ+eE12TikhUo8EfRlZdCCFoaIxQrrmbpDtI7ksorNIQiFCaYl3NXwtSLXLSLNWOJBBS8QcjhBsVRKh1Ba8767zUNoQBkCLxAWdJjSJ/8w75k3tvmpPeTL838pomePnD7Rw/Yxhet344DC3dEdMQFUVuUa6RZSk2I7OpX6XIUkazWFOlLO4oJEkiJyefnJz8jt95M/SG8LbvdtVy53Nr+NlZUwmEVO5/fj0rvn8og5Iyhib35eMNpTzwz6/41SWHMagoK2HdwGv3oNXuI+uMWzqt3W+u2cXjr29jyphCrjpjaqu27azz8vCja3A6ZK45Jx4qGfn2PYLbP0Y+7JQO/z2blun3cs3Osgae/e8Wvt4WnzKuJRX8kCUJEdATR0nedJ68ngteTeEdJYdSpsNMa2DOeLWja7oEy8O1NXmDjHVSptnOKkBrqOpUacxwRpKjuroTVdPM4h/msn3fgsuHnGtPhOoO+r2Rj8aMckSNG2ezqpMh1yixgVenB0lJ/Vosxwx0NMVELn3ma8tGQ04qGmLHyXcN8Teo+GzjTMbK5WbevOScYogEIdR5co1RvawH2XhUTTTJthrdvQHHkIl2dE030e+PuuG1W2UWw/tyKInRNemkGgCXQyYYVlPKNZkPvBoJyowJOTZdQbwQewd68rlFAGh1ZR3VzCYY/kRPGkhXNZFQcUurL0fUl6MMmdSNrerf9HsjbxjlVJ68NUGZCNSnlWoACvM8VNYFUdVUco2UkfQSj9Ax/rbNfFcgJcg1icsy2S6VXCLn6KmDtfqKJt91FIZxb0sakM5CVUWCQ6Pu2wSAMnhidzWp32MbedHUkzfuGcWSoKwlT74w10NFbTAhusYgY08efSJUZ0TX2KTHSGFghM7qyzLYzvIGkIycE/Pk6zvTk+95co2mJRp5sx5y7HjYdD390sgLIfj823I95NHw5C3Jmp777xYAM+mTQ5ZjRj51jDxAUZ6HOn+YQLhpLLCSoSavyzU9LO65HxD3yK3LMhlDidV1TSXXuLxInhxEXed58kZ7e9zAq9XIh/wgKeD0dGOr+jf90shv3l3H75/9kk07a+KafGwAtqouyH/X7uH4g4YxbazufSgyiEA9cjNyTVGe/kAoqw40+U6WWqfJG1bejq7pGuIG3eLJt3PgFUDKKUarL29/A9MgeqAnH9VEQnSNCDUgebJs6bEbaXOc/D/+8Q8ee+wx8+9du3Zx8sknEwgEWLNmDV6vbvR+8pOfMGfOnPa3tAMJxrztxlDUTAZmePKhiD5RaMywXNMwO0VEz73RnFyTp3sqViO/32Dd85ekxLzy6TDCMHuSZ9YfSKXJZzaGEvPk02jick4RasW2jmhiSozf7UnXi5ZULlOE/EjurGa2sOls2mzkzzjjDM444wwANm3axOWXX85PfvITvve97/HYY49RUlLSYY3saIxBVqtEY2jpxjKnIps3sQe9hFpzA69FMSNfWq2ve/WZUzlwtF4LM1O5xgjDNLCdn67BWqO3NZEq1tDLVMi5JUS3rkFoUSS54+cdGj/bo6JrkgZeRbABbCPfrXSIXHPzzTdz9dVX4/F42LNnDzfeeCOLFy/mnnvu6ZFT2qMxHT4UUU1vyDDuxgPA6ZBNw+zRYka+GU8+P9uNIkuUxjz5xLJ/mRcntqNruh7rAGpr8gbJzQy8QqwKklA7TZePT4bqlN23CTV54NX25LuddrsXq1evJhgMcuKJJ7Jz505mzpzJihUr8Pl8XHbZZaxatYozzzwz4/01V+GkJYqL0xthK94dNQC43U48sVQGLpeD4uIcSuv0WpRFBdkUF+ltyXPq8s6AwYPwNPMbhXkequv17QsLssz2uF0OFEVusX0ejwPFL5s3b16eN+M+9WR6eh9CMSOZk+NBCUYAKC7KZkBu08FCa1+MayU3zXkKhsewB8gWNWQVpy9g3Va8Xj0/jCK3fG2lojPOiyYEOdluc9+BSCOe/NGdfg309GusNXR0X9pt5J988kkuuugiAIYPH84f/vAH87sLLriA559/vlVGvivK/1XFJJXKmkayPPoM1np/iPLyeioq9RmKDQ1BGuqcuJ0Kroie0qA2KFPfzG84FJlaf1DfX33QbI/P7UCWpRbbFwmrRKKq6U7W1QV6fN6XlugNuWtqYtdDbV3ALCBSVeUnGookrJfcl9pa/a2turoxZR+F0MdkanZsoXFAx8eJN/j1h0w4orb6GHfkeYlEVRoCUQbkuImqGqFQxNx3NNBASLg79RroDddYpnRG+b92yTXhcJhPPvmE4447DoCNGzfy6quvmt8LIXA4el4ONGPCUjiimVq8Ue/S1OQdMi6nwm9/OIv9CvTXz+ZCKI1tjBzy1lfW7y+cxMULWr7JzclQZD7r0qYDSJHWIJNA+bhck9opkdxZSN5ctJqmRbE7gnh0TffqNW+u2c1ND31kFqFXYgOvQo1CJGjLNd1Mu4z8xo0bGTVqFD6fD9AvtltuuYXa2loikQhPPfVUj4usAYjENPmwRZM3cs5E1fjAK0BulgtCDaC4kJzuZvfrdMQPp9XIe92OJkWhUxFPaxD7O8P+2LQPqyZvVvPKJLomdrqbe/GU8wej1exrdxtTYQx3dXd0TX1jGH8wSjiSWCRHhBoAbCPfzbTLzd65cyeDBsUzy02cOJFLL72Uc845h2g0yty5c1m0aFG7G9nRRK2efOxzKk/eoKWUBgYuq5FXWv/8NEIo7RmvXYtxpoQgXpWrAzx5ADlvMNFta9rXwDSIHjLwagQyBGNvsQ7TyOvSp+Rp+zibTftpl5FfsGABCxYsSFh23nnncd5557WrUZ2NYdBDUdVMa2AYfiO6xmEx0i2lNDBwORTzsyPDaBor8VTD8b9tOh9rorF4crjMcg1B8560nD8IEaxHBBs63NhpPUSuMSYSGuMZSrKRtz35bqVfzniNmJ68Ra5Rm/Hkg5l58o40ck2mSHryGkTsNdw28V2DGe9O3CvOKEEZzYdQgi7XAJ2iy/eUOHljXCuYXNM4aBh525PvTvqlkU+Uawwjn2jsEzz5QF2Gnnw75Rr0EoKicyp526TBnLlqOfYZyTVyBnJNJxp5M06+m6eiGPdM0PDkY9e+rcn3DPqpkY8NvEbVeHRNE0/eOmsvU7mmfZ68bKca7hYsqWssmnzmck1zjrSUXQSyA6224wdfzeiabk5pl+zJK8kDr7Ym3630UyOfIoQyjScvIiGIhjOSa5xWTV5poyaPXTSkq7Fq8vHomsy3a06Tl2QZOW9g53jyRnRNN4+8GveOkYHVNPJBvx6CZGeg7Fb6p5GPNqPJqxoORYrfwLHKPnJOcYv7TQyhbMOhNePkzT9tugBrquHWvEXJGQy8Ash5gzpJk+8ZWSiNCLWmnrye0sB+I+1e+qWRNwdeo5pZ49Uq11iNtVZXCoCcO7DF/SZq8m2Va0SrJAOb9mOqNa1MUNZS7hpzvfzBaHXlCDXS/IqtpKcU8m4i11g0eVuq6X76pZG3avLJnnxUFQmDrlptzJPPazmrptPZ3ugaKWFCjm3juwZTrqG1qYb1/1t6MMiFw0GoaNV72tPMJsSjazp0t63G8ORThlDag67dTj818uk1+UhUTQyfrCtF8uQguXwt7teIk5clqU1eeHLREJuuwVr8Iz5HoeXt4vnkm19PKRypr1e5o40tTE3PiZNPHUIpgnYGyp5AvzTyEYsmnxxd09STL0XKa1mqgbgm3xapBiwzXu3cNV1KyslQHejJS7kl4HCjdriRN/7v5oFX08jrnrzDEl1jx8h3P/3SyKsWTV61GHeIafJWI19XlpEeDxYj3wapBvQ3AE0TrZqQY9Nx6FJZ5sddtsg8zSHJMnLh8A735IVZGapDd9tqzIHXUIqBV1uT73b6pZE3EpRB/BUz7slr5sxVLViP8Fch52ZW5crVTiOvyJL+ZmHOhbKtfFcQN9a6J5/pcc8khNJAKRyBWrGjQ6UVU67pZiuvJoVQyrJkZ6DsQfRLI28YdIgPFqmaHiNt9eTDnzwDkoxjv4Mz2m9crmnbYVUUCVXTWjXr0qb9JE9qytyTj22XgZGVC0dAJICo77gqUcKUazpsl20imiK6xs5b03OwjXzsFRP0186IqodQipCfyIZ3cE4+FqVgaEb7NSZDtd2Tl1HV1uU0t2k/yZOhMh0LkWTDk295XaVwBABq5fa2NTIFPWbgNUWcvG3kew790shHopr5im68YoKuy0ejGg5FRvNXAQJl8ISM99teucYhS3qSLC3zTIg27Sd5MlSmpy+TVMPmugXDQJI7VJfvaXHy1hBKO6VBz6FfGvmoquHz6FmWjQsT9ElSpiffWAuA5M3LeL8dIddAPPrHlmu6hnjumlh0TYbHPZPcNea6Dhdy/mDUio4z8j0lTj6aasarnYGyx9BPjbywGHmrXGN48hIioNd1ldtg5NuSSx7i8cXGjFzbyHcN1spQQmQeumq8aWXqScuFIzrWk7foRN3pzatJIZS6Jm9noOwp9FMjr5EVM/JWfd705BWLJ+/L3Mi7OkCTh7jGacs1XYNxlI0apRnLNbG7J1P7qhSNQPir0GIORHuxykTdqcsb0TXmbGFZQgsYb8ItJ/az6VzaVRlq6dKlVFZWmsW6V6xYwY4dO7j//vuJRCJceOGFPbJKlC7XOJssV1WNaCx3jRaoBcXZqgx6RlqDNss1si3XdAeJnnzrQygzNbByyRgAtNLNyKOmt6GliVgHfDUN2njZtZto0pRfhywh6ir0meJ2Bspup81GXgjBli1bePvtt00jX1paytVXX82zzz6Ly+Xi7LPP5rDDDmPs2LEd1uD2IoQgqgrTk7cSiWpEYjNeRWMtki+vVbNOjdDLts54Nbazvl3YdA2SFM9d09oQykylEqVoFEgKatl3ODrAyPc0T95AkSW0+nKknKJuapGNlTYb+S1btiBJEj/4wQ+orKzkzDPPJCsri5kzZ5Kfnw/AvHnzeOWVV/jJT37SUe1tN8bMVqsnH6u6h6oJMwulaKxD8ua2at8uZ/s0ecOTj5qavO3KdxVyLG+QRitCKM3JUJn9huRwIReNQC3d3NZmJmB9uHSXJi+EMDV5A1mW0OorUIpGdkubbBJps5Gvq6tj1qxZ3HzzzQSDQZYuXcqJJ55IcXE873pJSQnr1q1r1X4LC9s+Gl9c3LL+1xjU070WDYgnHPO4FQIhlaxsD1FVIzfXg1JbjyN/YEb7tKLIEh6Ps9XbAQzI19tkyDWFBVlt2k9Pozf0QZIkvF4XmiShKFLaNluXG8bN53Nl3Edp5CTq175JUaEPSVZa3qAZHEp8+8LC7JQSZHN0xHlJ9dZZUpxNWUMlWfvPorCLzn1vuMYypaP70mYjP336dKZP1185fT4fS5Ys4Te/+Q0//OEPE9ZrrTdaWdnQpko3xcU5lJfXt7heXWNYb5dFR3Q6dCNfGts+EooSqatGFI7OaJ9WnA4ZTdVavR1Aoz8ExIuaVFf78Tl6tzef6XnpCfj9IQKhKEKQss3JfTEkkoaGUMZ9jOSPQkRClH69DmVg+2TMkGWOR3l5fauMfEedl1BEbbKsevdu0KIEldwuOfe96Rpribb0RZalZp3jNg/VfPrpp3zwwQfm30IIhg4dSkVFfNp2WVkZJSWZ5X3pKgz90OuOP9/cMZnFiJl3yAIRbGhVjLyB0yG3K3cNxEMo7ZHXrkOOafKtSVAWTzWcuVOiDJ4IQHT3161tYhOsCk13pTZQkzx5l0NGaawCMqumZtP5tNnI19fXc9tttxEKhWhoaOC5557j9ttv54MPPqCqqopAIMBrr73G0Ucf3ZHtbTeGAXU6ZHOGqtupv/YakzmytAZAtCp80sDVDiMvm5p85nVGbToGI5e/Hl3Tmu1aV0hb9uYiFw5H3bOh9Y1Moido8tGkp8uYoXnQoDt6tpHvGbRZrjn22GNZu3Ytp5xyCpqmce655zJjxgyuvvpqli5dSiQSYcmSJUyZMqUj29tuDClEUSRcToVwVMMX8+rrY1JOfqM+YUWJhby1hhEDcxhS1LYJIEacvB1d0/VIlvq6rZEY5Vg1r9agDJlM5Os3EdEwksPVuo0tJETXdJMrnxxZM2FEPpp/NwBS9oDuaJJNEu2Kk7/qqqu46qqrEpYtXryYxYsXt2e3nYphQJ2KrEfDBKAgzwO7atlT0QhAfsMWcGfpZdtayRWnt/2hZoZQRu3omq5GL9iie/KteYOSJKnVXrRj6GQiX76Kum8TjmH7t7KlcbQeKNdMHDEAsbsBHC4kh7t7GmWTQL+b8WrINQ5FNmeo+twOfG4H3+2uAQQ59VtwDJ6IJHXt4XHYk6G6DQldlG9NWgOIafmtfPFSBk/Q4+V3f9XKVibSE+LkjQgj4+11v8G5iGA9kqfvRLv0dtrlyfdGDC/Z4ZDNuHZFlhmQ62Z3uZ9CuQElUI0ydFKXty1ZrrFtfNdhyjWtNJZt8eQlpwelZDTRPRtoj6/bE3LXGNfqSUeMYsaEYhRZJhK0y/71JPqdJ1/ToOvuPrcDlzOea2ZAjn67TXTt05cNmdzlbTOzUNrRNV2OJEloGLlrWuHJy23LAqkMnYxWvg0RbGj9xjGsv9tdE14NT16RZdNJEcEGO8VwD6LfGfnPvi0n1+dk5MAc3JbC2wUxIz/ZW47kzUPOH9zlbTNnvMbeNvrdyelGdNlFxAZeM99OQmqTVOIYfiAgiO76stXbGugFTuKfuwPTyFtSeehG3pZregr9yo6EIirrNldy0IQSZFkyPXlZkhiQ4wEE+8l7UIZM6pZBz6Zx8l3ehH6L0yETiWp6CGUrzr0xYNta5JLRSN5cots+b/W2BkKIeNHsbvLko+YYl9XI19uefA+iX2nyW3bXEoqoTBurJ04yjHyWWsuEiv8yJGcfWaIRx/ADuqV9cpInb6ca7jqcDoWIqrVqMhQY8fWt/z1JknGMmEpk66cINYqktP5W1IQuk0RVtU2zxDsCI4TSlGo0FcKNtpHvQfQrT74xViAkP1uPTXY5ZLKkIDO2/pn86q8YKNeyteRYHGMP75b2GSmKo3bRkC7H6ZAJR1rvycty2+QaAMeoGRAOEN3ZuvxOBlqCJ9/Nco3RDqO2q23kewz9ysiHo7qRNzx4l1NhjKMMV9RP5MgfcXPtEsITT0SSu+ewOOwslN2G0yHrMlmrPfm2x6grww/UJZuN77Zpe6EJUwvvrjj5qCUkGXSpBrA1+R5Ev5JrjPhzI52Byykz1FGFQKJwzGRuvijMsJLu80DiA6/dXLSzH+JyyEQiKi6H3CqZTG5DCKWBJCs4xh1B5MvX0AJ1yK1Mba2JuBbeYzz5oFH2z/bkewr9y5OPJHnyDoWhShVBTxGSw82IgTmtCp/raAy5xq7x2vU4Yp58a8r/gRFf33YD65xwFAiV6KbVrd5WH3iNedDd7sknGXlbrukx9C8jH40nJ4OYJ69U0+gb0p3NMkkOobTlmq7D5VDM6JrWjHe3JXeNFWXAEOSS0UQ2vtvqh4UmsMg13R1Cacs1PZX+ZeRjnrzTIaPV7uOwr26jQPETzOr6mPhUJIdQ2ia+63A6ZMJRrdUJytrryQM4JxyNVr0brfS7Vm1nHXjtNiOvJsk1sSLltiffc+hfRj5W2k+WJMJfvo4z6icqZBrze0YN2iY1Xm0r32XE4+RbK9dI7R70dI6dCS4f4fWvtWq7BLmmmxKXGkW8zYHXen0yYXuya9p0LP3KyEciGi6HjAgHiGx6n5qSg/h59fmEc4d1d9OA+HR6c8arLdd0GW2fDNX2EEpzH04PrknHEN36KVp9RcsbxNC0HiDXJHnyWn0FUq6dR74n0a+MfDiq4nTIRPdsgEgQ/5DDgLie2N1IkoQiS3Y++W7AlWDkM99ObkcIpRXn/scDEuGv3sh4GyGEGXbb7dE1xsOmvhw5p6hb2mKTmp5h3bqIcFTD5VTQKncCEqJgBNCzPGarke9Bzerz6Jq8qs94bWUIZUcYWDm7EMd+BxP55h1EOJDRNomafLub0CaMfPIOWUZoUURDpV0RqofRrjj5e++9l3//+98AzJ49m//93//l+uuvZ82aNXi9XgB+8pOfMGfOnPa3tAMIx+KgtcodSHkluLw+IDHvRnejKBLhkJ3WoKtxKjJC6Ear1ZOhOsjCuqbMJ7rlY8LrX8d90Ektri+EJaqlm8v/KYqEaKgEIWwj38Nos5FfvXo17733Hs899xySJHHJJZfw+uuvs379eh577LEeV8Ab9MlQLqeCWrUTpXAE+w3O5azjxjJpZM8pU6YPpOlRQLYn33U4YwVkQlHNrPmbCW3NXZMKpWQ0jpHTCa/9N67JxzUboWJo8KPVLWyVsrpRk4+V05Ql1Nh4gq3J9yzaLNcUFxdz3XXX4XK5cDqdjBkzhj179rBnzx5uvPFGFi9ezD333IOmdY2+/J81O7ln1ToisdQFqQhHVLKUKKKuDLlwBLIsMe/QEebkqJ5AW4uA27QPY+5EOKK2Krqmo+QaA9chp0MkSOiLfzW7nhCCfNnP3MYXWZ73PKKL7rNkVE2XjCRJQqsvB7A1+R5Gm438uHHjmDZtGgDbtm3j5Zdf5qijjmLmzJnccsstPP3003z66aesWrWqo9qalm3rv8T7r5s4teJ+Kl68C7ViOyJFTFk4qjGUMgCUNtRv7Qqsebl70lhBX8dlMfKtQdKrBnYYSsEwHGNnEvnqDbSGyrTraRrky3oysBw5SP63L3WLZOMPRvG6dUFA3f01uLxIWQVd3g6b9LQ7d82mTZu47LLLuPbaaxk9ejR/+MMfzO8uuOACnn/+ec4888yM91dY2PpJFKERA9lZMo6tpbXsX/kdjc/+AsnlwTd6Gt4xB+EZOg5X8Qg0ITg48hFKVj4DpxyK7PK0+rc6G+tbRVFxNh5X708vVFzc82c/FhToNUojqsDtdqRtc/Jyl0vB4VA6tI+ReUvZ9afP0D56nJKzlqUM6QxFVPIkfYB2S6SY0TvfwfF1MQXHnJPx73REm0NRjQG5HgZ4Veq3riF3xjyKBua3e7+tpTdcY5nS0X1plwVZs2YNV155JcuWLWPhwoVs3LiRbdu2MW/ePCAW4uVo3U9UVja0eiDLnVvCrB8u47d/+ZgXv93Bz2dp7Nm0gXE7vsH/zYcAuKYtYngwTLGyG8eh36OyNgJEWvU7XYHVGausaOhRUlJbKC7Ooby8vrub0SLBRr0sZDAcJRpRU7Y5VV/UqEYoFOngPnpxHXwagQ/+zt4PXsM5rmnq62A4Sq6sG/mHG47hmknb4P1VhLKH4hg5vcVf6KjzUl7VSJZbYd/ql0GLou53ZJef795yjWVCW/oiy1KzznGb5Zq9e/dy+eWXc8cdd7Bw4UJAN+q33HILtbW1RCIRnnrqqS6NrJk6tojKkJOHNhVz7+5pvDfmKqqPWcZm74GEv3iJU+Q3qXcU4Jx4dJe1qbU4LIKwnbum64hr8lrrJkPJ7Z/xmrI9+89BHjiW4OrH0fzVTb4XAnLlABoyDcJD2bhTkItGEfjPn1Br9nR8g9JQ5w+Tn+UksuFtlCGTuqVspk3ztNnIP/TQQ4RCIW699VZOPvlkTj75ZD7//HMuvfRSzjnnHBYuXMikSZNYtGhRR7a3WcYPzwNg8249f8ZLH+7g5md3cc/uafhHHIlLUtlQeByS3HO9YyXByHdjQ/oZhiYPrXu4ynRO+KIky3iOvhjUKIFXftckdl4Tgly5kaDiQyChSU68cy5Hkh00/nOlPuGvC6htDDOGHYiGSpyTj+uS37RpHW2Wa5YvX87y5ctTfnfeeee1uUHtIcfnYmhxFrvL/UwbW4Q/GKEhEGFvZSPfDjqRl9YP5NDxk7qlbZmi9KCY/f6EM8HIZ75dR+SuSYcyYAjeEy4n8OrvCLx5H955P0WS9VtWCMiTAoQU/TVdEwI5pwTfKTcSeOV3BP51B57jLuWN8kFMGDGA0UNal6veyvotlVTWBZk9bWjC8lBEJRRWGRbZBk4PjlEty0Q2XU+fm/E6YXg+AMcfPIzrz5/Biu8fiiJL7KsKUBHNwuXs2V1WLFWp7OiariPByLdiu47IQtkcjhFTcB+5FHXnlwTfecSMGtM0Qa4cIOTQB+mMOHk5twTfKctRBo4h+J8/UfDpn9ix+pV2teGNNbt4/t2tTZbX+fVxjNxIBXLBMPMBZNOz6NkWrw0cceBgpowpZNxQXbpRZJmifC+7yvViBi5Hz5VqIClO3rbxXYazrXKN3HGTodLhmnQMroNPJbrpfYJv3IeIhBBCN/Jhh+7JWyOGJZcP77yfIgZNpkSpZUbVywTfeQgRDbfp92sbwtT6w03CS3UjL/AGS1EGDE29sU230+cevfsNzuWqM6YmLBs4wMuOUn3Eusd78ha5xrbxXUd75JquiE93TT8JyeEm9NFTNL5QhjjwVHLkIKXORE/ebJc7i5pDLuXX6z9iScHXHLnxXSJb1+CceDSuA+ZAK8L0avwhACrrgjQGo7z35V5GDsohz+ciRwqiRBqRC3pGJlebpvQ5I5+KkgFe1m3WJ5ZYb+aeiFWusaNrug7rG15ri4Z0RUoBSZJwTZmPnD+EwH/+iPL23QCEnfob6xtrdjF5VAEDctzmNnWNYQQyz9VO4YTz5hHd8BaRL18lsu4Vdg+bgLz/iSgjpjbbX00TpixTWRvk9U938eWWShyKxDnHj2Owokf+yLYn32Pp2Ravgxg4wGd+7i1yjW3eu5a2evJyJw68psIxYgrZ591J+dQLebzhcAKDpwGwfV89//l8d8K6hnGOqhqNuaPwHv8jss68FdehZ6LWVRJ49Xc0PvdLIt++nzJME2IPilj/ymoCfLurBpdDJqoKvttdyxBHDWAb+Z5Mv/DkhxVnmZ97vFxjaPK2le9S2qzJd5FcY0VyuHm3ehBrhcSpYwfDm9sBTEnSoM4fn+xXURtk6956igdkU559KGNOOQHP9o8IffocwbcfBEnBMXYmztEHIxeNQs4awLZ9dXyzvUb/TQSffVsO4QDnT/Szfdtu6rftYq73S+TC4Ujetkfv2HQu/cLIjx2WZ37u8Z58TJO3I2u6Fj3JFm0o/6fnkelKVE3js2/LmTq2MOF63ravHiEEH35dSlVdkJ1lDeZ3+6oa+durGynI9VBRE2BwURY3X3gkWeMOZ/s3G8na8xGerR8S3fQ+QpJxjpzG9u1hGholrsgpZZSjgtpqHzkDArjKVKbGXo6DkhfvCT+xpcUeTL8w8oosU5TnoaI22OM9ZDsLZfcgSRIuh0Ko1QnKJESHpihrmW931FDfGOHgCSVkeRxIEgwtymJXuZ/Pvq3gwRe/NtfN9jppCET45JsyIlGN0qpGAHaVNfDxN6UcOnEgv3uzilzf/tx8/pnc+8hrHChv5qCy7UxU65nqiRAWDj4Ij8dDGMWbzZGLT+LXz2wFfxXHHDOD2XkDu7T/Nq2jXxh5gPPnjud3/1jHIIs+3xMxBl5tx6jrcTpkQhG1VW9RhvffVgKhKB99XYrHrXDoxIHIGTzkP91Yjsspc+CYQlxOhYeuPY5Nu2r4zWOf8ddXviHb62RwoY9Nu2opzvfidsp8ubkSCRhU6GP/UQVs3lvHo69upLo+RJ0/TJ0/zKdbGviiJo8vOIi9o4fxxqe78EohNCFx7GFjeX39Pm4882CUXA85RQ2sr1MYu5+dxqCn02+M/JQxRTx07bE9/rXSkGt6ejv7InlZLhoCkVa97fncDmobQkRVjTp/mKr6EIW5HipqA4wblk+dP8yeCj8TLYVpGoMRtuytY/9RBbzw/lZe/XgnAF9vrea8ueP5bnctk0cOQJL0UpCffVuOpgmmjy/my82VrNlYxpQxRQnFTUaU5JDlcdAQiHDWcWOJqhqbdtUiSzDnkBE8+eYmhpdk88uLDwVAcjq49vfv8o//bMahyERVjcde+9bc31trdjN2WB7f7aoF4Mxjx7LkmDHmA3DiiAFU1gYZUtiznSabfmTkoXcYTrNmZ3cV7ezHHD9jGH97dSMbtqWONEnFlDFFvLtuL+98sYcXV2+j3h8m2+ekvjHCSUeM4sOvSimrCbBg5kiGFPkQAl7/ZCc7yho4ZtoQVn+1j8MmD6Qk38uLq7fx5ZZKav1hLl4wCa/bwQdf7dMHPNEfQrWxiJlZ+ydKJG6Xwv9dfgThqEa218nm3bpx3lfVyDHThvD6JzuZOjZezKMo38v/njud25/8gnFD86isC7JhezUHjC5g085aQhGVBTNH8vKH26mo0fPmWN9wFswcyYmHjegV91R/p18Z+d5AXrYe55yf7W5hTZuO5sgpg/nbqxuZNi7zykYHji7A7VR4/PVvyfU5GTssj72VjUwckc8L72/D61bYf9QAXv5wu7mN0yEzdUwhb3+xB4cicdIRoxhU4EMA//pgG7k+J4/8e4MpA51xzBg8LoXHXv+Wk44YxXEzhpHrczVpi8upmKmpRw7SJzsdNWUILqfCLZfObJIXqSDXw8ofHAZCj/Wvb4yQ43Pyxxe+oqY+xNQxhUwZXZj2zcY28L0DSXRXBeA0tCWfPPSdnNKaEAhFIRwImxV3ejO97byEIyoOh5xSl0/Xl0de3sAX31Xwv+dMZ3BRll5L2CFTURsk2+vE41KorA2ak6ayvE6yPE6q6oI4HHKCwQ6Go2zYVs29z37JkmPGcPgBg8wHfzAcbVURmbb0BfS4eiGEWfe2p9PbrrHm6Ix88raR74HYfemZpOtLZxjFQCjaqQ/5/nBeeiOdYeR7v6toY9PNOJSOn2DXF97ibHoGPXv6p42NjY1Nu7CNvI2NjU0fplOM/IsvvsiCBQuYM2cOjz/+eGf8hI2NjY1NBnS48FdaWspdd93Fs88+i8vl4uyzz+awww5j7NixHf1TNjY2NjYt0OGe/OrVq5k5cyb5+fn4fD7mzZvHK6+0r/yYjY2NjU3b6HAjX1ZWRnFxsfl3SUkJpaWlHf0zNjY2NjYZ0OFyTaqw+9bMjGsu3rMliltR0qynY/elZ2L3pWdi9yU9He7JDxw4kIqKCvPvsrIySkpKOvpnbGxsbGwyoMON/OGHH84HH3xAVVUVgUCA1157jaOPPrqjf8bGxsbGJgM6XK4ZOHAgV199NUuXLiUSibBkyRKmTJnS0T9jY2NjY5MBPS53jY2NjY1Nx2HPeLWxsbHpw9hG3sbGxqYPYxt5Gxsbmz6MbeRtbGxs+jC2kbexsbHpw9hG3sbGxqYP0yeMfG9Pbbx06VIWLlzIySefzMknn8zatWt7VZ8aGhpYtGgRu3btAvQkdYsXL2bu3Lncdddd5nobNmzg9NNPZ968edxwww1Eo9HuanJakvty/fXXM3fuXPPcvP7660D6PvYU7r33XhYuXMjChQu57bbbgN57XlL1pbeel7vvvpsFCxawcOFCHnnkEaALzovo5ezbt08ce+yxorq6Wvj9frF48WKxadOm7m5WxmiaJo444ggRiUTMZb2pT1988YVYtGiR2H///cXOnTtFIBAQs2fPFjt27BCRSERcfPHF4u233xZCCLFw4ULx+eefCyGEuP7668Xjjz/ejS1vSnJfhBBi0aJForS0NGG95vrYE3j//ffFWWedJUKhkAiHw2Lp0qXixRdf7JXnJVVfXnvttV55Xj766CNx9tlni0gkIgKBgDj22GPFhg0bOv289HpPvrenNt6yZQuSJPGDH/yAk046iccee6xX9enpp5/mF7/4hZmfaN26dYwcOZLhw4fjcDhYvHgxr7zyCrt37yYYDDJt2jQATjvttB7Xp+S+NDY2smfPHm688UYWL17MPffcg6ZpafvYUyguLua6667D5XLhdDoZM2YM27Zt65XnJVVf9uzZ0yvPy6GHHsrf/vY3HA4HlZWVqKpKXV1dp5+XXl8tOFVq43Xr1nVji1pHXV0ds2bN4uabbyYYDLJ06VJOPPHEXtOnlStXJvydLtV08vLi4uIel4I6uS+VlZXMnDmTFStW4PP5uOyyy1i1ahU+n69Hp9MeN26c+Xnbtm28/PLLXHDBBb3yvKTqyxNPPMHHH3/c684LgNPp5J577uHhhx9m/vz5XXK/9HpPXrQztXF3M336dG677TZ8Ph8FBQUsWbKEe+65p8l6vaVP6c5HbzxPw4cP5w9/+AOFhYV4vV4uuOAC3nnnnV7Tl02bNnHxxRdz7bXXMmLEiCbf96bzYu3L6NGje/V5ufLKK/nggw/Yu3cv27Zta/J9R5+XXm/ke3tq408//ZQPPvjA/FsIwdChQ3ttn9Kdj+Tl5eXlPb5PGzdu5NVXXzX/FkLgcDh6xTW3Zs0aLrzwQn7+859z6qmn9urzktyX3npeNm/ezIYNGwDwer3MnTuXjz76qNPPS6838r09tXF9fT233XYboVCIhoYGnnvuOW6//fZe26epU6eydetWtm/fjqqqvPTSSxx99NEMHToUt9vNmjVrAHj++ed7fJ+EENxyyy3U1tYSiUR46qmnmDNnTto+9hT27t3L5Zdfzh133MHChQuB3nteUvWlt56XXbt2sXz5csLhMOFwmDfffJOzzz67089Lr9fke3tq42OPPZa1a9dyyimnoGka5557LjNmzOi1fXK73dx6661cccUVhEIhZs+ezfz58wG44447WL58OX6/n8mTJ7N06dJubm3zTJw4kUsvvZRzzjmHaDTK3LlzWbRoEUDaPvYEHnroIUKhELfeequ57Oyzz+6V5yVdX3rjeZk9e7Z5ryuKwty5c1m4cCEFBQWdel7sVMM2NjY2fZheL9fY2NjY2KTHNvI2NjY2fRjbyNvY2Nj0YWwjb2NjY9OHsY28jY2NTR/GNvI2Nin4wQ9+wHfffdeqbS677DKeffbZTmqRjU3b6PVx8jY2ncGDDz7Y3U2wsekQbCNv06d46623uP/++4lEIng8Hq699lree+89Nm3aREVFBZWVlUycOJGVK1eSnZ3NE088wZNPPonT6cTtdrNixQrGjh3Lcccdx913382BBx7IU089xaOPPoosyxQVFXHjjTey3377UVpaynXXXUdZWRlDhgyhsrLSbMfmzZtZuXIlNTU1qKrKBRdcwJIlS/D7/Vx//fVs374dWZbZf//9WbFiBbJsv1TbdBJtz45sY9Oz2Lp1q1i0aJGoqqoSQgjx7bffiiOOOELceuut4uijjxbl5eVCVVXxs5/9TNx6660iGo2K/fff38xL/txzz4knn3xSCCHEscceK9atWydWr14tTjjhBFFZWSmEEOKZZ54RJ554otA0Tfz4xz8Wd911lxBCiG3btolp06aJZ555RkQiEbFgwQKxfv16IYQQdXV14sQTTxSff/65eO6558TFF18shBAiGo2KG264QWzbtq0rD5NNP8P25G36DO+//z5lZWVceOGF5jJJktixYwfz58+nqKgIgCVLlnDLLbdw7bXXMn/+fM4++2yOOeYYjjjiCBYvXpywz3fffZcFCxZQUFAA6Hm9V65cya5du1i9ejXXXnstACNHjuSwww4D9JS4O3bsYNmyZeZ+gsEgX3/9NUcddRR33XUXF1xwAYcffjjf+973GDlyZGceFpt+jm3kbfoMmqYxa9Ysfve735nL9u7dy1NPPUU4HE5Yz5BH7rjjDr799ltWr17Ngw8+yKpVq7j//vvNdUWKrB9CCKLRaJOUsA6Hfjupqkpubi7//Oc/ze8qKirIycnB7Xbz+uuv89FHH/Hhhx9y0UUXsXz58h6VY8Wmb2ELgTZ9hpkzZ/L++++zefNmAN555x1OOukkQqEQb775JvX19WiaxtNPP82xxx5LVVUVs2fPJj8/nwsvvJCrrrqKjRs3JuzzyCOP5OWXX6aqqgqAZ555hvz8fEaOHMlRRx3FU089BcCePXv46KOPANhvv/1wu92mkd+7dy+LFi1i/fr1PPHEE1x//fUceeSRXHPNNRx55JFs2rSpqw6RTT/ETlBm06f497//zQMPPGDmGF+2bBkffPABH374IaqqUl1dzSGHHMLy5cvxeDw8+eST/O1vf8Pj8aAoCldffTWHH354wsDr448/zpNPPommaRQUFHDTTTcxbtw4qqqquP7669mxYweDBg0iGo1y6qmnctppp/HNN9+YA6/RaJSlS5dyzjnn0NjYyLJly9i4cSNer5chQ4awcuVK8vLyuvvQ2fRRbCNv0+f5/e9/T3V1NTfddFN3N8XGpsux5RobGxubPoztydvY2Nj0YWxP3sbGxqYPYxt5Gxsbmz6MbeRtbGxs+jC2kbexsbHpw9hG3sbGxqYPYxt5Gxsbmz7M/wMcC4nelkepKAAAAABJRU5ErkJggg==\n" - }, - "metadata": {} - } - ], - "source": [ - "env = gym.make('CartPole-v0')\n", - "env.seed(1)\n", - "cfg = HierarchicalDQNConfig()\n", - "state_dim = env.observation_space.shape[0]\n", - "action_dim = env.action_space.n\n", - "agent = HierarchicalDQN(state_dim, action_dim, cfg)\n", - "rewards, ma_rewards = train(cfg, env, agent)\n", - "agent.save(path=SAVED_MODEL_PATH)\n", - "save_results(rewards, ma_rewards, tag='train', path=RESULT_PATH)\n", - "plot_rewards(rewards, ma_rewards, tag=\"train\",\n", - " algo=cfg.algo, path=RESULT_PATH)" - ] - } - ] -} \ No newline at end of file diff --git a/codes/HierarchicalDQN/task0_train.py b/codes/HierarchicalDQN/task0_train.py deleted file mode 100644 index 2676094..0000000 --- a/codes/HierarchicalDQN/task0_train.py +++ /dev/null @@ -1,146 +0,0 @@ -#!/usr/bin/env python -# coding=utf-8 -''' -Author: John -Email: johnjim0816@gmail.com -Date: 2021-03-29 10:37:32 -LastEditor: John -LastEditTime: 2021-05-04 22:35:56 -Discription: -Environment: -''' - - -import sys,os -curr_path = os.path.dirname(__file__) -parent_path = os.path.dirname(curr_path) -sys.path.append(parent_path) # add current terminal path to sys.path - -import datetime -import numpy as np -import torch -import gym - -from common.utils import save_results,make_dir -from common.plot import plot_rewards -from HierarchicalDQN.agent import HierarchicalDQN - -curr_time = datetime.datetime.now().strftime( - "%Y%m%d-%H%M%S") # obtain current time - -class HierarchicalDQNConfig: - def __init__(self): - self.algo = "H-DQN" # name of algo - self.env = 'CartPole-v0' - self.result_path = curr_path+"/outputs/" + self.env + \ - '/'+curr_time+'/results/' # path to save results - self.model_path = curr_path+"/outputs/" + self.env + \ - '/'+curr_time+'/models/' # path to save models - self.train_eps = 300 # 训练的episode数目 - self.eval_eps = 50 # 测试的episode数目 - self.gamma = 0.99 - self.epsilon_start = 1 # start epsilon of e-greedy policy - self.epsilon_end = 0.01 - self.epsilon_decay = 200 - self.lr = 0.0001 # learning rate - self.memory_capacity = 10000 # Replay Memory capacity - self.batch_size = 32 - self.target_update = 2 # target net的更新频率 - self.device = torch.device( - "cuda" if torch.cuda.is_available() else "cpu") # 检测gpu - self.hidden_dim = 256 # dimension of hidden layer - -def env_agent_config(cfg,seed=1): - env = gym.make(cfg.env) - env.seed(seed) - state_dim = env.observation_space.shape[0] - action_dim = env.action_space.n - agent = HierarchicalDQN(state_dim,action_dim,cfg) - return env,agent - -def train(cfg, env, agent): - print('Start to train !') - print(f'Env:{cfg.env}, Algorithm:{cfg.algo}, Device:{cfg.device}') - rewards = [] - ma_rewards = [] # moveing average reward - for i_ep in range(cfg.train_eps): - state = env.reset() - done = False - ep_reward = 0 - while not done: - goal = agent.set_goal(state) - onehot_goal = agent.to_onehot(goal) - meta_state = state - extrinsic_reward = 0 - while not done and goal != np.argmax(state): - goal_state = np.concatenate([state, onehot_goal]) - action = agent.choose_action(goal_state) - next_state, reward, done, _ = env.step(action) - ep_reward += reward - extrinsic_reward += reward - intrinsic_reward = 1.0 if goal == np.argmax( - next_state) else 0.0 - agent.memory.push(goal_state, action, intrinsic_reward, np.concatenate( - [next_state, onehot_goal]), done) - state = next_state - agent.update() - agent.meta_memory.push(meta_state, goal, extrinsic_reward, state, done) - print('Episode:{}/{}, Reward:{}, Loss:{:.2f}, Meta_Loss:{:.2f}'.format(i_ep+1, cfg.train_eps, ep_reward,agent.loss_numpy ,agent.meta_loss_numpy )) - rewards.append(ep_reward) - if ma_rewards: - ma_rewards.append( - 0.9*ma_rewards[-1]+0.1*ep_reward) - else: - ma_rewards.append(ep_reward) - print('Complete training！') - return rewards, ma_rewards - -def eval(cfg, env, agent): - print('Start to eval !') - print(f'Env:{cfg.env}, Algorithm:{cfg.algo}, Device:{cfg.device}') - rewards = [] - ma_rewards = [] # moveing average reward - for i_ep in range(cfg.train_eps): - state = env.reset() - done = False - ep_reward = 0 - while not done: - goal = agent.set_goal(state) - onehot_goal = agent.to_onehot(goal) - extrinsic_reward = 0 - while not done and goal != np.argmax(state): - goal_state = np.concatenate([state, onehot_goal]) - action = agent.choose_action(goal_state) - next_state, reward, done, _ = env.step(action) - ep_reward += reward - extrinsic_reward += reward - state = next_state - agent.update() - print(f'Episode:{i_ep+1}/{cfg.train_eps}, Reward:{ep_reward}, Loss:{agent.loss_numpy:.2f}, Meta_Loss:{agent.meta_loss_numpy:.2f}') - rewards.append(ep_reward) - if ma_rewards: - ma_rewards.append( - 0.9*ma_rewards[-1]+0.1*ep_reward) - else: - ma_rewards.append(ep_reward) - print('Complete training！') - return rewards, ma_rewards - -if __name__ == "__main__": - cfg = HierarchicalDQNConfig() - - # train - env,agent = env_agent_config(cfg,seed=1) - rewards, ma_rewards = train(cfg, env, agent) - make_dir(cfg.result_path, cfg.model_path) - agent.save(path=cfg.model_path) - save_results(rewards, ma_rewards, tag='train', path=cfg.result_path) - plot_rewards(rewards, ma_rewards, tag="train", - algo=cfg.algo, path=cfg.result_path) - # eval - env,agent = env_agent_config(cfg,seed=10) - agent.load(path=cfg.model_path) - rewards,ma_rewards = eval(cfg,env,agent) - save_results(rewards,ma_rewards,tag='eval',path=cfg.result_path) - plot_rewards(rewards,ma_rewards,tag="eval",env=cfg.env,algo = cfg.algo,path=cfg.result_path) - diff --git a/codes/HierarchicalDQN/train.py b/codes/HierarchicalDQN/train.py new file mode 100644 index 0000000..3dc8aa3 --- /dev/null +++ b/codes/HierarchicalDQN/train.py @@ -0,0 +1,77 @@ +import sys +import os +curr_path = os.path.dirname(os.path.abspath(__file__)) # 当前文件所在绝对路径 +parent_path = os.path.dirname(curr_path) # 父路径 +sys.path.append(parent_path) # 添加路径到系统路径 + +import numpy as np + +def train(cfg, env, agent): + print('开始训练!') + print(f'环境：{cfg.env_name}, 算法：{cfg.algo_name}, 设备：{cfg.device}') + rewards = [] # 记录所有回合的奖励 + ma_rewards = [] # 记录所有回合的滑动平均奖励 + for i_ep in range(cfg.train_eps): + state = env.reset() + done = False + ep_reward = 0 + while not done: + goal = agent.set_goal(state) + onehot_goal = agent.to_onehot(goal) + meta_state = state + extrinsic_reward = 0 + while not done and goal != np.argmax(state): + goal_state = np.concatenate([state, onehot_goal]) + action = agent.choose_action(goal_state) + next_state, reward, done, _ = env.step(action) + ep_reward += reward + extrinsic_reward += reward + intrinsic_reward = 1.0 if goal == np.argmax( + next_state) else 0.0 + agent.memory.push(goal_state, action, intrinsic_reward, np.concatenate( + [next_state, onehot_goal]), done) + state = next_state + agent.update() + if (i_ep+1)%10 == 0: + print(f'回合：{i_ep+1}/{cfg.train_eps}，奖励：{ep_reward}，Loss:{agent.loss_numpy:.2f}， Meta_Loss:{agent.meta_loss_numpy:.2f}') + agent.meta_memory.push(meta_state, goal, extrinsic_reward, state, done) + rewards.append(ep_reward) + if ma_rewards: + ma_rewards.append( + 0.9*ma_rewards[-1]+0.1*ep_reward) + else: + ma_rewards.append(ep_reward) + print('完成训练！') + return rewards, ma_rewards + +def test(cfg, env, agent): + print('开始测试!') + print(f'环境：{cfg.env_name}, 算法：{cfg.algo_name}, 设备：{cfg.device}') + rewards = [] # 记录所有回合的奖励 + ma_rewards = [] # 记录所有回合的滑动平均奖励 + for i_ep in range(cfg.train_eps): + state = env.reset() + done = False + ep_reward = 0 + while not done: + goal = agent.set_goal(state) + onehot_goal = agent.to_onehot(goal) + extrinsic_reward = 0 + while not done and goal != np.argmax(state): + goal_state = np.concatenate([state, onehot_goal]) + action = agent.choose_action(goal_state) + next_state, reward, done, _ = env.step(action) + ep_reward += reward + extrinsic_reward += reward + state = next_state + agent.update() + if (i_ep+1)%10 == 0: + print(f'回合：{i_ep+1}/{cfg.train_eps}，奖励：{ep_reward}，Loss:{agent.loss_numpy:.2f}， Meta_Loss:{agent.meta_loss_numpy:.2f}') + rewards.append(ep_reward) + if ma_rewards: + ma_rewards.append( + 0.9*ma_rewards[-1]+0.1*ep_reward) + else: + ma_rewards.append(ep_reward) + print('完成训练！') + return rewards, ma_rewards \ No newline at end of file diff --git a/codes/QLearning/task0_train.py b/codes/QLearning/task0_train.py index 6e616ab..2a9e0ea 100644 --- a/codes/QLearning/task0_train.py +++ b/codes/QLearning/task0_train.py @@ -45,9 +45,9 @@ def env_agent_config(cfg,seed=1): env = gym.make(cfg.env) env = CliffWalkingWapper(env) env.seed(seed) # 设置随机种子 - n_states = env.observation_space.n # 状态维度 - n_actions = env.action_space.n # 动作维度 - agent = QLearning(n_states,n_actions,cfg) + state_dim = env.observation_space.n # 状态维度 + action_dim = env.action_space.n # 动作维度 + agent = QLearning(state_dim,action_dim,cfg) return env,agent def train(cfg,env,agent): diff --git a/codes/README.md b/codes/README.md index fdee344..49f6ac7 100644 --- a/codes/README.md +++ b/codes/README.md @@ -45,4 +45,6 @@ python 3.7、pytorch 1.6.0-1.8.1、gym 0.17.0-0.19.0 [RL-Adventure-2](https://github.com/higgsfield/RL-Adventure-2) -[RL-Adventure](https://github.com/higgsfield/RL-Adventure) \ No newline at end of file +[RL-Adventure](https://github.com/higgsfield/RL-Adventure) + +[Google 开源项目风格指南——中文版](https://zh-google-styleguide.readthedocs.io/en/latest/google-python-styleguide/python_style_rules/#comments) \ No newline at end of file diff --git a/codes/SAC/model.py b/codes/SAC/model.py index 146db0d..85bbfcd 100644 --- a/codes/SAC/model.py +++ b/codes/SAC/model.py @@ -5,7 +5,7 @@ Author: JiangJi Email: johnjim0816@gmail.com Date: 2021-04-29 12:53:58 LastEditor: JiangJi -LastEditTime: 2021-04-29 12:57:29 +LastEditTime: 2021-11-19 18:04:19 Discription: Environment: ''' @@ -35,12 +35,12 @@ class ValueNet(nn.Module): class SoftQNet(nn.Module): - def __init__(self, num_inputs, num_actions, hidden_size, init_w=3e-3): + def __init__(self, state_dim, action_dim, hidden_dim, init_w=3e-3): super(SoftQNet, self).__init__() - self.linear1 = nn.Linear(num_inputs + num_actions, hidden_size) - self.linear2 = nn.Linear(hidden_size, hidden_size) - self.linear3 = nn.Linear(hidden_size, 1) + self.linear1 = nn.Linear(state_dim + action_dim, hidden_dim) + self.linear2 = nn.Linear(hidden_dim, hidden_dim) + self.linear3 = nn.Linear(hidden_dim, 1) self.linear3.weight.data.uniform_(-init_w, init_w) self.linear3.bias.data.uniform_(-init_w, init_w) @@ -54,20 +54,20 @@ class SoftQNet(nn.Module): class PolicyNet(nn.Module): - def __init__(self, num_inputs, num_actions, hidden_size, init_w=3e-3, log_std_min=-20, log_std_max=2): + def __init__(self, state_dim, action_dim, hidden_dim, init_w=3e-3, log_std_min=-20, log_std_max=2): super(PolicyNet, self).__init__() self.log_std_min = log_std_min self.log_std_max = log_std_max - self.linear1 = nn.Linear(num_inputs, hidden_size) - self.linear2 = nn.Linear(hidden_size, hidden_size) + self.linear1 = nn.Linear(state_dim, hidden_dim) + self.linear2 = nn.Linear(hidden_dim, hidden_dim) - self.mean_linear = nn.Linear(hidden_size, num_actions) + self.mean_linear = nn.Linear(hidden_dim, action_dim) self.mean_linear.weight.data.uniform_(-init_w, init_w) self.mean_linear.bias.data.uniform_(-init_w, init_w) - self.log_std_linear = nn.Linear(hidden_size, num_actions) + self.log_std_linear = nn.Linear(hidden_dim, action_dim) self.log_std_linear.weight.data.uniform_(-init_w, init_w) self.log_std_linear.bias.data.uniform_(-init_w, init_w) diff --git a/codes/common/model.py b/codes/common/model.py index be03368..27e5e4e 100644 --- a/codes/common/model.py +++ b/codes/common/model.py @@ -15,15 +15,15 @@ import torch.nn.functional as F from torch.distributions import Categorical class MLP(nn.Module): - def __init__(self, n_states,n_actions,hidden_dim=128): + def __init__(self, input_dim,output_dim,hidden_dim=128): """ 初始化q网络，为全连接网络 - n_states: 输入的特征数即环境的状态数 - n_actions: 输出的动作维度 + input_dim: 输入的特征数即环境的状态数 + output_dim: 输出的动作维度 """ super(MLP, self).__init__() - self.fc1 = nn.Linear(n_states, hidden_dim) # 输入层 + self.fc1 = nn.Linear(input_dim, hidden_dim) # 输入层 self.fc2 = nn.Linear(hidden_dim,hidden_dim) # 隐藏层 - self.fc3 = nn.Linear(hidden_dim, n_actions) # 输出层 + self.fc3 = nn.Linear(hidden_dim, output_dim) # 输出层 def forward(self, x): # 各层对应的激活函数 @@ -32,10 +32,10 @@ class MLP(nn.Module): return self.fc3(x) class Critic(nn.Module): - def __init__(self, n_obs, n_actions, hidden_size, init_w=3e-3): + def __init__(self, n_obs, action_dim, hidden_size, init_w=3e-3): super(Critic, self).__init__() - self.linear1 = nn.Linear(n_obs + n_actions, hidden_size) + self.linear1 = nn.Linear(n_obs + action_dim, hidden_size) self.linear2 = nn.Linear(hidden_size, hidden_size) self.linear3 = nn.Linear(hidden_size, 1) # 随机初始化为较小的值 @@ -51,11 +51,11 @@ class Critic(nn.Module): return x class Actor(nn.Module): - def __init__(self, n_obs, n_actions, hidden_size, init_w=3e-3): + def __init__(self, n_obs, action_dim, hidden_size, init_w=3e-3): super(Actor, self).__init__() self.linear1 = nn.Linear(n_obs, hidden_size) self.linear2 = nn.Linear(hidden_size, hidden_size) - self.linear3 = nn.Linear(hidden_size, n_actions) + self.linear3 = nn.Linear(hidden_size, action_dim) self.linear3.weight.data.uniform_(-init_w, init_w) self.linear3.bias.data.uniform_(-init_w, init_w) @@ -67,18 +67,18 @@ class Actor(nn.Module): return x class ActorCritic(nn.Module): - def __init__(self, n_states, n_actions, hidden_dim=256): + def __init__(self, state_dim, action_dim, hidden_dim=256): super(ActorCritic, self).__init__() self.critic = nn.Sequential( - nn.Linear(n_states, hidden_dim), + nn.Linear(state_dim, hidden_dim), nn.ReLU(), nn.Linear(hidden_dim, 1) ) self.actor = nn.Sequential( - nn.Linear(n_states, hidden_dim), + nn.Linear(state_dim, hidden_dim), nn.ReLU(), - nn.Linear(hidden_dim, n_actions), + nn.Linear(hidden_dim, action_dim), nn.Softmax(dim=1), ) diff --git a/codes/common/multiprocessing_env.py b/codes/common/multiprocessing_env.py index 04b4e3c..28c8aba 100644 --- a/codes/common/multiprocessing_env.py +++ b/codes/common/multiprocessing_env.py @@ -1,5 +1,5 @@ -#This code is from openai baseline -#https://github.com/openai/baselines/tree/master/baselines/common/vec_env +# 该代码来自 openai baseline，用于多线程环境 +# https://github.com/openai/baselines/tree/master/baselines/common/vec_env import numpy as np from multiprocessing import Process, Pipe diff --git a/codes/common/plot.py b/codes/common/plot.py deleted file mode 100644 index bc9c1dd..0000000 --- a/codes/common/plot.py +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/env python -# coding=utf-8 -''' -Author: John -Email: johnjim0816@gmail.com -Date: 2020-10-07 20:57:11 -LastEditor: John -LastEditTime: 2021-09-23 12:23:01 -Discription: -Environment: -''' -import matplotlib.pyplot as plt -import seaborn as sns -from matplotlib.font_manager import FontProperties # 导入字体模块 - -def plot_rewards(rewards,ma_rewards,plot_cfg,tag='train'): - sns.set() - plt.figure() # 创建一个图形实例，方便同时多画几个图 - plt.title("learning curve on {} of {} for {}".format(plot_cfg.device, plot_cfg.algo, plot_cfg.env_name)) - plt.xlabel('epsiodes') - plt.plot(rewards,label='rewards') - plt.plot(ma_rewards,label='ma rewards') - plt.legend() - if plot_cfg.save: - plt.savefig(plot_cfg.result_path+"{}_rewards_curve".format(tag)) - plt.show() - -def plot_losses(losses,algo = "DQN",save=True,path='./'): - sns.set() - plt.figure() - plt.title("loss curve of {}".format(algo)) - plt.xlabel('epsiodes') - plt.plot(losses,label='rewards') - plt.legend() - if save: - plt.savefig(path+"losses_curve") - plt.show() - diff --git a/codes/common/utils.py b/codes/common/utils.py index a3ca7be..6027804 100644 --- a/codes/common/utils.py +++ b/codes/common/utils.py @@ -5,29 +5,90 @@ Author: John Email: johnjim0816@gmail.com Date: 2021-03-12 16:02:24 LastEditor: John -LastEditTime: 2021-09-11 21:48:49 +LastEditTime: 2021-11-30 18:39:19 Discription: Environment: ''' import os import numpy as np from pathlib import Path +import matplotlib.pyplot as plt +import seaborn as sns -def save_results(rewards,ma_rewards,tag='train',path='./results'): - '''save rewards and ma_rewards +from matplotlib.font_manager import FontProperties # 导入字体模块 + +def chinese_font(): + ''' 设置中文字体，注意需要根据自己电脑情况更改字体路径，否则还是默认的字体 + ''' + try: + font = FontProperties( + fname='/System/Library/Fonts/STHeiti Light.ttc', size=15) # fname系统字体路径，此处是mac的 + except: + font = None + return font + +def plot_rewards_cn(rewards, ma_rewards, plot_cfg, tag='train'): + ''' 中文画图 + ''' + sns.set() + plt.figure() + plt.title(u"{}环境下{}算法的学习曲线".format(plot_cfg.env_name, + plot_cfg.algo_name), fontproperties=chinese_font()) + plt.xlabel(u'回合数', fontproperties=chinese_font()) + plt.plot(rewards) + plt.plot(ma_rewards) + plt.legend((u'奖励', u'滑动平均奖励',), loc="best", prop=chinese_font()) + if plot_cfg.save: + plt.savefig(plot_cfg.result_path+f"{tag}_rewards_curve_cn") + # plt.show() + + +def plot_rewards(rewards, ma_rewards, plot_cfg, tag='train'): + sns.set() + plt.figure() # 创建一个图形实例，方便同时多画几个图 + plt.title("learning curve on {} of {} for {}".format( + plot_cfg.device, plot_cfg.algo_name, plot_cfg.env_name)) + plt.xlabel('epsiodes') + plt.plot(rewards, label='rewards') + plt.plot(ma_rewards, label='ma rewards') + plt.legend() + if plot_cfg.save: + plt.savefig(plot_cfg.result_path+"{}_rewards_curve".format(tag)) + plt.show() + + +def plot_losses(losses, algo="DQN", save=True, path='./'): + sns.set() + plt.figure() + plt.title("loss curve of {}".format(algo)) + plt.xlabel('epsiodes') + plt.plot(losses, label='rewards') + plt.legend() + if save: + plt.savefig(path+"losses_curve") + plt.show() + + +def save_results(rewards, ma_rewards, tag='train', path='./results'): + ''' 保存奖励 ''' np.save(path+'{}_rewards.npy'.format(tag), rewards) np.save(path+'{}_ma_rewards.npy'.format(tag), ma_rewards) print('结果保存完毕！') + def make_dir(*paths): + ''' 创建文件夹 + ''' for path in paths: Path(path).mkdir(parents=True, exist_ok=True) + + def del_empty_dir(*paths): - '''del_empty_dir delete empty folders unders "paths" + ''' 删除目录下所有空文件夹 ''' for path in paths: dirs = os.listdir(path) for dir in dirs: if not os.listdir(os.path.join(path, dir)): - os.removedirs(os.path.join(path, dir)) \ No newline at end of file + os.removedirs(os.path.join(path, dir)) diff --git a/codes/envs/assets/gym_info_20211130180023.png b/codes/envs/assets/gym_info_20211130180023.png new file mode 100644 index 0000000..723b67f Binary files /dev/null and b/codes/envs/assets/gym_info_20211130180023.png differ diff --git a/codes/envs/gym_info.md b/codes/envs/gym_info.md index dd4268a..49da18f 100644 --- a/codes/envs/gym_info.md +++ b/codes/envs/gym_info.md @@ -1,4 +1,5 @@ -## 环境说明 +# OpenAi Gym 环境说明 +## 基础控制 ### [CartPole v0](https://github.com/openai/gym/wiki/CartPole-v0) @@ -6,6 +7,17 @@ 通过向左或向右推车能够实现平衡，所以动作空间由两个动作组成。每进行一个step就会给一个reward，如果无法保持平衡那么done等于true，本次episode失败。理想状态下，每个episode至少能进行200个step，也就是说每个episode的reward总和至少为200，step数目至少为200 +### CartPole-v1 + +```CartPole v1```环境其实跟```CartPole v0```是一模一样的，区别在于每回合最大步数（max_episode_steps）以及奖励阈值（reward_threshold），如下是相关源码： + +![](assets/gym_info_20211130180023.png) + +这里先解释一下奖励阈值（reward_threshold），即Gym设置的一个合格标准，比如对于```CartPole v0```如果算法能够将奖励收敛到195以上，说明该算法合格。但实际上```CartPole v0```的每回合最大步数（max_episode_steps）是200，每步的奖励最大是1，也就是每回合最大奖励是200，比Gym设置的奖励阈值高。笔者猜测这是Gym可能是给算法学习者们设置的一个参考线，而实际中在写算法时并不会用到这个算法阈值，所以可以忽略。 + +再看每回合最大步数，可以看到```CartPole v1```的步数更长，相应的奖励要求更高，可以理解为```v1```是```v0```的难度升级版。 + + ### [Pendulum-v0](https://github.com/openai/gym/wiki/Pendulum-v0) 注：gym 0.18.0之后版本中Pendulum-v0已经改为Pendulum-v1 @@ -31,4 +43,8 @@ image-20201007211858925 -由于从起点到终点最少需要13步，每步得到-1的reward，因此最佳训练算法下，每个episode下reward总和应该为-13。 \ No newline at end of file +由于从起点到终点最少需要13步，每步得到-1的reward，因此最佳训练算法下，每个episode下reward总和应该为-13。 + +## 参考 + +[Gym环境相关源码](https://github.com/openai/gym/tree/master/gym/envs) \ No newline at end of file