diff --git a/codes/DDPG/agent.py b/codes/DDPG/agent.py index b080c15..0a8fd30 100644 --- a/codes/DDPG/agent.py +++ b/codes/DDPG/agent.py @@ -5,7 +5,7 @@ @Email: johnjim0816@gmail.com @Date: 2020-06-09 20:25:52 @LastEditor: John -LastEditTime: 2021-03-31 00:56:32 +LastEditTime: 2021-05-04 14:50:17 @Discription: @Environment: python 3.7.7 ''' @@ -26,6 +26,7 @@ class DDPG: self.target_critic = Critic(state_dim, action_dim, cfg.hidden_dim).to(cfg.device) self.target_actor = Actor(state_dim, action_dim, cfg.hidden_dim).to(cfg.device) + # copy parameters to target net for target_param, param in zip(self.target_critic.parameters(), self.critic.parameters()): target_param.data.copy_(param.data) for target_param, param in zip(self.target_actor.parameters(), self.actor.parameters()): @@ -42,7 +43,6 @@ class DDPG: def choose_action(self, state): state = torch.FloatTensor(state).unsqueeze(0).to(self.device) action = self.actor(state) - # torch.detach()用于切断反向传播 return action.detach().cpu().numpy()[0, 0] def update(self): @@ -50,13 +50,13 @@ class DDPG: return state, action, reward, next_state, done = self.memory.sample( self.batch_size) - # 将所有变量转为张量 + # convert variables to Tensor state = torch.FloatTensor(state).to(self.device) next_state = torch.FloatTensor(next_state).to(self.device) action = torch.FloatTensor(action).to(self.device) reward = torch.FloatTensor(reward).unsqueeze(1).to(self.device) done = torch.FloatTensor(np.float32(done)).unsqueeze(1).to(self.device) - # 注意critic将(s_t,a)作为输入 + policy_loss = self.critic(state, self.actor(state)) policy_loss = -policy_loss.mean() next_action = self.target_actor(next_state) diff --git a/codes/DDPG/main.py b/codes/DDPG/main.py deleted file mode 100644 index 736178b..0000000 --- a/codes/DDPG/main.py +++ /dev/null @@ -1,94 +0,0 @@ -#!/usr/bin/env python -# coding=utf-8 -''' -@Author: John -@Email: johnjim0816@gmail.com -@Date: 2020-06-11 20:58:21 -@LastEditor: John -LastEditTime: 2021-04-29 01:58:50 -@Discription: -@Environment: python 3.7.7 -''' -import sys,os -from pathlib import Path -import sys,os -curr_path = os.path.dirname(__file__) -parent_path=os.path.dirname(curr_path) -sys.path.append(parent_path) # add current terminal path to sys.path - -import torch -import gym -import numpy as np -import datetime -from DDPG.agent import DDPG -from DDPG.env import NormalizedActions,OUNoise -from common.plot import plot_rewards -from common.utils import save_results - -SEQUENCE = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") # obtain current time -SAVED_MODEL_PATH = curr_path+"/saved_model/"+SEQUENCE+'/' # path to save model -if not os.path.exists(curr_path+"/saved_model/"): os.mkdir(curr_path+"/saved_model/") -if not os.path.exists(SAVED_MODEL_PATH): os.mkdir(SAVED_MODEL_PATH) -RESULT_PATH = curr_path+"/results/"+SEQUENCE+'/' # path to save rewards -if not os.path.exists(curr_path+"/results/"): os.mkdir(curr_path+"/results/") -if not os.path.exists(RESULT_PATH): os.mkdir(RESULT_PATH) - -class DDPGConfig: - def __init__(self): - self.env = 'Pendulum-v0' - self.algo = 'DDPG' - self.gamma = 0.99 - self.critic_lr = 1e-3 - self.actor_lr = 1e-4 - self.memory_capacity = 10000 - self.batch_size = 128 - self.train_eps =300 - self.eval_eps = 200 - self.eval_steps = 200 - self.target_update = 4 - self.hidden_dim = 30 - self.soft_tau=1e-2 - self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") -def train(cfg,env,agent): - print('Start to train ! ') - ou_noise = OUNoise(env.action_space) # action noise - rewards = [] - ma_rewards = [] # moving average rewards - ep_steps = [] - for i_episode in range(cfg.train_eps): - state = env.reset() - ou_noise.reset() - done = False - ep_reward = 0 - i_step = 0 - while not done: - i_step += 1 - action = agent.choose_action(state) - action = ou_noise.get_action(action, i_step) # 即paper中的random process - next_state, reward, done, _ = env.step(action) - ep_reward += reward - agent.memory.push(state, action, reward, next_state, done) - agent.update() - state = next_state - print('Episode:{}/{}, Reward:{}'.format(i_episode+1,cfg.train_eps,ep_reward)) - ep_steps.append(i_step) - rewards.append(ep_reward) - if ma_rewards: - ma_rewards.append(0.9*ma_rewards[-1]+0.1*ep_reward) - else: - ma_rewards.append(ep_reward) - print('Complete training！') - return rewards,ma_rewards - -if __name__ == "__main__": - cfg = DDPGConfig() - env = NormalizedActions(gym.make("Pendulum-v0")) - env.seed(1) # 设置env随机种子 - state_dim = env.observation_space.shape[0] - action_dim = env.action_space.shape[0] - agent = DDPG(state_dim,action_dim,cfg) - rewards,ma_rewards = train(cfg,env,agent) - agent.save(path=SAVED_MODEL_PATH) - save_results(rewards,ma_rewards,tag='train',path=RESULT_PATH) - plot_rewards(rewards,ma_rewards,tag="train",algo = cfg.algo,path=RESULT_PATH) - \ No newline at end of file diff --git a/codes/DDPG/outputs/Pendulum-v0/20210504-024530/models/checkpoint.pt b/codes/DDPG/outputs/Pendulum-v0/20210504-024530/models/checkpoint.pt new file mode 100644 index 0000000..be79646 Binary files /dev/null and b/codes/DDPG/outputs/Pendulum-v0/20210504-024530/models/checkpoint.pt differ diff --git a/codes/DDPG/outputs/Pendulum-v0/20210504-024530/results/eval_ma_rewards.npy b/codes/DDPG/outputs/Pendulum-v0/20210504-024530/results/eval_ma_rewards.npy new file mode 100644 index 0000000..7062ae6 Binary files /dev/null and b/codes/DDPG/outputs/Pendulum-v0/20210504-024530/results/eval_ma_rewards.npy differ diff --git a/codes/DDPG/outputs/Pendulum-v0/20210504-024530/results/eval_rewards.npy b/codes/DDPG/outputs/Pendulum-v0/20210504-024530/results/eval_rewards.npy new file mode 100644 index 0000000..f5156f8 Binary files /dev/null and b/codes/DDPG/outputs/Pendulum-v0/20210504-024530/results/eval_rewards.npy differ diff --git a/codes/DDPG/outputs/Pendulum-v0/20210504-024530/results/eval_rewards_curve.png b/codes/DDPG/outputs/Pendulum-v0/20210504-024530/results/eval_rewards_curve.png new file mode 100644 index 0000000..53589b0 Binary files /dev/null and b/codes/DDPG/outputs/Pendulum-v0/20210504-024530/results/eval_rewards_curve.png differ diff --git a/codes/DDPG/outputs/Pendulum-v0/20210504-024530/results/train_ma_rewards.npy b/codes/DDPG/outputs/Pendulum-v0/20210504-024530/results/train_ma_rewards.npy new file mode 100644 index 0000000..e2d734b Binary files /dev/null and b/codes/DDPG/outputs/Pendulum-v0/20210504-024530/results/train_ma_rewards.npy differ diff --git a/codes/DDPG/outputs/Pendulum-v0/20210504-024530/results/train_rewards.npy b/codes/DDPG/outputs/Pendulum-v0/20210504-024530/results/train_rewards.npy new file mode 100644 index 0000000..092936c Binary files /dev/null and b/codes/DDPG/outputs/Pendulum-v0/20210504-024530/results/train_rewards.npy differ diff --git a/codes/DDPG/outputs/Pendulum-v0/20210504-024530/results/train_rewards_curve.png b/codes/DDPG/outputs/Pendulum-v0/20210504-024530/results/train_rewards_curve.png new file mode 100644 index 0000000..60e508a Binary files /dev/null and b/codes/DDPG/outputs/Pendulum-v0/20210504-024530/results/train_rewards_curve.png differ diff --git a/codes/DDPG/results/20210331-010047/ma_rewards_train.npy b/codes/DDPG/results/20210331-010047/ma_rewards_train.npy deleted file mode 100644 index 6d3572e..0000000 Binary files a/codes/DDPG/results/20210331-010047/ma_rewards_train.npy and /dev/null differ diff --git a/codes/DDPG/results/20210331-010047/rewards_curve_train.png b/codes/DDPG/results/20210331-010047/rewards_curve_train.png deleted file mode 100644 index f2046a5..0000000 Binary files a/codes/DDPG/results/20210331-010047/rewards_curve_train.png and /dev/null differ diff --git a/codes/DDPG/results/20210331-010047/rewards_train.npy b/codes/DDPG/results/20210331-010047/rewards_train.npy deleted file mode 100644 index 72a95cc..0000000 Binary files a/codes/DDPG/results/20210331-010047/rewards_train.npy and /dev/null differ diff --git a/codes/DDPG/saved_model/20210331-010047/checkpoint.pt b/codes/DDPG/saved_model/20210331-010047/checkpoint.pt deleted file mode 100644 index 85ddc28..0000000 Binary files a/codes/DDPG/saved_model/20210331-010047/checkpoint.pt and /dev/null differ diff --git a/codes/DDPG/task0_train.py b/codes/DDPG/task0_train.py new file mode 100644 index 0000000..50e2723 --- /dev/null +++ b/codes/DDPG/task0_train.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python +# coding=utf-8 +''' +@Author: John +@Email: johnjim0816@gmail.com +@Date: 2020-06-11 20:58:21 +@LastEditor: John +LastEditTime: 2021-05-04 14:49:45 +@Discription: +@Environment: python 3.7.7 +''' +import sys,os +curr_path = os.path.dirname(__file__) +parent_path = os.path.dirname(curr_path) +sys.path.append(parent_path) # add current terminal path to sys.path + +import datetime +import gym +import torch + +from DDPG.env import NormalizedActions, OUNoise +from DDPG.agent import DDPG +from common.utils import save_results,make_dir +from common.plot import plot_rewards + +curr_time = datetime.datetime.now().strftime( + "%Y%m%d-%H%M%S") # obtain current time + + +class DDPGConfig: + def __init__(self): + self.algo = 'DDPG' + self.env = 'Pendulum-v0' # env name + self.result_path = curr_path+"/outputs/" + self.env + \ + '/'+curr_time+'/results/' # path to save results + self.model_path = curr_path+"/outputs/" + self.env + \ + '/'+curr_time+'/models/' # path to save results + self.gamma = 0.99 + self.critic_lr = 1e-3 + self.actor_lr = 1e-4 + self.memory_capacity = 10000 + self.batch_size = 128 + self.train_eps = 300 + self.eval_eps = 50 + self.eval_steps = 200 + self.target_update = 4 + self.hidden_dim = 30 + self.soft_tau = 1e-2 + self.device = torch.device( + "cuda" if torch.cuda.is_available() else "cpu") + +def env_agent_config(cfg,seed=1): + env = NormalizedActions(gym.make(cfg.env)) + env.seed(seed) + state_dim = env.observation_space.shape[0] + action_dim = env.action_space.shape[0] + agent = DDPG(state_dim,action_dim,cfg) + return env,agent + +def train(cfg, env, agent): + print('Start to train ! ') + print(f'Env:{cfg.env}, Algorithm:{cfg.algo}, Device:{cfg.device}') + ou_noise = OUNoise(env.action_space) # action noise + rewards = [] + ma_rewards = [] # moving average rewards + for i_episode in range(cfg.train_eps): + state = env.reset() + ou_noise.reset() + done = False + ep_reward = 0 + i_step = 0 + while not done: + i_step += 1 + action = agent.choose_action(state) + action = ou_noise.get_action( + action, i_step) # 即paper中的random process + next_state, reward, done, _ = env.step(action) + ep_reward += reward + agent.memory.push(state, action, reward, next_state, done) + agent.update() + state = next_state + print('Episode:{}/{}, Reward:{}'.format(i_episode+1, cfg.train_eps, ep_reward)) + rewards.append(ep_reward) + if ma_rewards: + ma_rewards.append(0.9*ma_rewards[-1]+0.1*ep_reward) + else: + ma_rewards.append(ep_reward) + print('Complete training！') + return rewards, ma_rewards + +def eval(cfg, env, agent): + print('Start to Eval ! ') + print(f'Env:{cfg.env}, Algorithm:{cfg.algo}, Device:{cfg.device}') + rewards = [] + ma_rewards = [] # moving average rewards + for i_episode in range(cfg.eval_eps): + state = env.reset() + done = False + ep_reward = 0 + i_step = 0 + while not done: + i_step += 1 + action = agent.choose_action(state) + next_state, reward, done, _ = env.step(action) + ep_reward += reward + state = next_state + print('Episode:{}/{}, Reward:{}'.format(i_episode+1, cfg.train_eps, ep_reward)) + rewards.append(ep_reward) + if ma_rewards: + ma_rewards.append(0.9*ma_rewards[-1]+0.1*ep_reward) + else: + ma_rewards.append(ep_reward) + print('Complete Eval！') + return rewards, ma_rewards + + +if __name__ == "__main__": + cfg = DDPGConfig() + + # train + env,agent = env_agent_config(cfg,seed=1) + rewards, ma_rewards = train(cfg, env, agent) + make_dir(cfg.result_path, cfg.model_path) + agent.save(path=cfg.model_path) + save_results(rewards, ma_rewards, tag='train', path=cfg.result_path) + plot_rewards(rewards, ma_rewards, tag="train", + algo=cfg.algo, path=cfg.result_path) + + # eval + env,agent = env_agent_config(cfg,seed=10) + agent.load(path=cfg.model_path) + rewards,ma_rewards = eval(cfg,env,agent) + save_results(rewards,ma_rewards,tag='eval',path=cfg.result_path) + plot_rewards(rewards,ma_rewards,tag="eval",env=cfg.env,algo = cfg.algo,path=cfg.result_path) + diff --git a/codes/DQN/README.md b/codes/DQN/README.md index 45612be..fc82fe6 100644 --- a/codes/DQN/README.md +++ b/codes/DQN/README.md @@ -1,6 +1,7 @@ # DQN -#TODO + ## 原理简介 + DQN是Q-leanning算法的优化和延伸，Q-leaning中使用有限的Q表存储值的信息，而DQN中则用神经网络替代Q表存储信息，这样更适用于高维的情况，相关知识基础可参考[datawhale李宏毅笔记-Q学习](https://datawhalechina.github.io/easy-rl/#/chapter6/chapter6)。 论文方面主要可以参考两篇，一篇就是2013年谷歌DeepMind团队的[Playing Atari with Deep Reinforcement Learning](https://www.cs.toronto.edu/~vmnih/docs/dqn.pdf)，一篇是也是他们团队后来在Nature杂志上发表的[Human-level control through deep reinforcement learning](https://web.stanford.edu/class/psych209/Readings/MnihEtAlHassibis15NatureControlDeepRL.pdf)。后者在算法层面增加target q-net，也可以叫做Nature DQN。 @@ -15,7 +16,7 @@ https://blog.csdn.net/JohnJim0/article/details/109557173) img -## 代码实战 +## 代码实现 ### RL接口 @@ -24,23 +25,26 @@ https://blog.csdn.net/JohnJim0/article/details/109557173) for i_episode in range(MAX_EPISODES): state = env.reset() # reset环境状态 for i_step in range(MAX_STEPS): - action = agent.choose_action(state) # 根据当前环境state选择action - next_state, reward, done, _ = env.step(action) # 更新环境参数 - agent.memory.push(state, action, reward, next_state, done) # 将state等这些transition存入memory - agent.update() # 每步更新网络 - state = next_state # 跳转到下一个状态 - if done: - break + action = agent.choose_action(state) # 根据当前环境state选择action + next_state, reward, done, _ = env.step(action) # 更新环境参数 + agent.memory.push(state, action, reward, next_state, done) # 将state等这些transition存入memory + agent.update() # 每步更新网络 + state = next_state # 跳转到下一个状态 + if done: + break ``` -如上，首先需要循环多个episode训练，在每个episode中，首先需要重置环境，然后开始探索，每个episode加一个MAX_STEPS(也可以使用while not done, 加这个max_steps有时是因为比如gym环境训练目标就是在200个step下达到200的reward)，接下来的流程如下： +每个episode加一个MAX_STEPS，也可以使用while not done, 加这个max_steps有时是因为比如gym环境训练目标就是在200个step下达到200的reward，或者是当完成一个episode的步数较多时也可以设置，基本流程跟所有伪代码一致，如下： 1. agent选择动作 -2. 环境根据agent的动作反馈出新的state和reward +2. 环境根据agent的动作反馈出next_state和reward 3. agent进行更新，如有memory就会将transition(包含state，reward，action等)存入memory中 4. 跳转到下一个状态 -如果提前done了，就跳出for循环，进行下一个episode的训练。 +5. 如果done了，就跳出循环，进行下一个episode的训练。 + +想要实现完整的算法还需要创建Qnet，Replaybuffer等类 ### 两个Q网络 -前面讲了Nature DQN中有两个Q网络，一个是policy_net，一个是延时更新的target_net，两个网络的结构是一模一样的，如下(见```model.py```)： + +上文讲了Nature DQN中有两个Q网络，一个是policy_net，一个是延时更新的target_net，两个网络的结构是一模一样的，如下(见```model.py```)，注意DQN使用的Qnet就是全连接网络即FCH： ```python import torch.nn as nn import torch.nn.functional as F @@ -62,30 +66,12 @@ class FCN(nn.Module): x = F.relu(self.fc2(x)) return self.fc3(x) ``` -输入为state，输出为action，注意根据state和action的维度调整隐藏层的层数，这里设为128 +输入为state_dim，输出为action_dim，包含一个128维度的隐藏层，这里根据需要可增加隐藏层维度和数量，然后一般使用relu激活函数，这里跟深度学习的网路设置是一样的。 + +### Replay Buffer + +然后就是Replay Memory了，其作用主要是是克服经验数据的相关性（correlated data）和非平稳分布（non-stationary distribution）问题，实现如下(见```memory.py```)： -在```agent.py```中我们定义强化学习算法，包括```choose_action```和```update```两个主要函数，初始化中： -```python -self.policy_net = FCN(state_dim, action_dim).to(self.device) -self.target_net = FCN(state_dim, action_dim).to(self.device) -# target_net的初始模型参数完全复制policy_net -self.target_net.load_state_dict(self.policy_net.state_dict()) -self.target_net.eval() # 不启用 BatchNormalization 和 Dropout -# 可查parameters()与state_dict()的区别，前者require_grad=True -``` -可以看到policy_net跟target_net结构和初始参数一样，但在更新的时候target是每隔一段episode更新的，如下(见```main.py```)： -```python -# 更新target network，复制DQN中的所有weights and biases -if i_episode % cfg.target_update == 0: - agent.target_net.load_state_dict(agent.policy_net.state_dict()) -``` -可以调整```cfg.target_update```，注意该变量不要调得太大，否则会收敛很慢，我们最后保存的模型也是这个target_net，如下(见```agent.py```)： -```python -def save_model(self,path): - torch.save(self.target_net.state_dict(), path) -``` -### Replay Memory -然后就是Replay Memory了，如下(见```memory.py```)： ```python import random import numpy as np @@ -111,11 +97,120 @@ class ReplayBuffer: def __len__(self): return len(self.buffer) ``` -其实比较简单，主要包括push和sample两个步骤，push是将transitions放到memory中，sample是从memory随机抽取一些transition。 -最后结果如下： +参数capacity表示buffer的容量，主要包括push和sample两个步骤，push是将transitions放到memory中，sample是从memory随机抽取一些transition。 -![rewards_curve_train](assets/rewards_curve_train.png) +### Agent类 + +在```agent.py```中我们定义强化学习算法类，包括```choose_action```(选择动作，使用e-greedy策略时会多一个```predict```函数，下面会将到)和```update```(更新)等函数。 + +在类中建立两个网络，以及optimizer和memory， + +```python +self.policy_net = MLP(state_dim, action_dim,hidden_dim=cfg.hidden_dim).to(self.device) +self.target_net = MLP(state_dim, action_dim,hidden_dim=cfg.hidden_dim).to(self.device) +for target_param, param in zip(self.target_net.parameters(),self.policy_net.parameters()): # copy params from policy net + target_param.data.copy_(param.data) +self.optimizer = optim.Adam(self.policy_net.parameters(), lr=cfg.lr) +self.memory = ReplayBuffer(cfg.memory_capacity) +``` +然后是选择action： + +```python +def choose_action(self, state): + '''选择动作 + ''' + self.frame_idx += 1 + if random.random() > self.epsilon(self.frame_idx): + action = self.predict(state) + else: + action = random.randrange(self.action_dim) + return action +``` + +这里使用e-greedy策略，即设置一个参数epsilon，如果生成的随机数大于epsilon，就根据网络预测的选择action，否则还是随机选择action，这个epsilon是会逐渐减小的，可以使用线性或者指数减小的方式，但不会减小到零，这样在训练稳定时还能保持一定的探索，这部分可以学习探索与利用(exploration and exploition)相关知识。 + +上面讲到的预测函数其实就是根据state选取q值最大的action，如下： + +```python +def predict(self,state): + with torch.no_grad(): + state = torch.tensor([state], device=self.device, dtype=torch.float32) + q_values = self.policy_net(state) + action = q_values.max(1)[1].item() +``` + +然后是更新函数了： + +```python +def update(self): + + if len(self.memory) < self.batch_size: + return + # 从memory中随机采样transition + state_batch, action_batch, reward_batch, next_state_batch, done_batch = self.memory.sample( + self.batch_size) + '''转为张量 + 例如tensor([[-4.5543e-02, -2.3910e-01, 1.8344e-02, 2.3158e-01],...,[-1.8615e-02, -2.3921e-01, -1.1791e-02, 2.3400e-01]])''' + state_batch = torch.tensor( + state_batch, device=self.device, dtype=torch.float) + action_batch = torch.tensor(action_batch, device=self.device).unsqueeze( + 1) # 例如tensor([[1],...,[0]]) + reward_batch = torch.tensor( + reward_batch, device=self.device, dtype=torch.float) # tensor([1., 1.,...,1]) + next_state_batch = torch.tensor( + next_state_batch, device=self.device, dtype=torch.float) + done_batch = torch.tensor(np.float32( + done_batch), device=self.device) + + '''计算当前(s_t,a)对应的Q(s_t, a)''' + '''torch.gather:对于a=torch.Tensor([[1,2],[3,4]]),那么a.gather(1,torch.Tensor([[0],[1]]))=torch.Tensor([[1],[3]])''' + q_values = self.policy_net(state_batch).gather( + dim=1, index=action_batch) # 等价于self.forward + # 计算所有next states的V(s_{t+1})，即通过target_net中选取reward最大的对应states + next_q_values = self.target_net(next_state_batch).max( + 1)[0].detach() # 比如tensor([ 0.0060, -0.0171,...,]) + # 计算 expected_q_value + # 对于终止状态，此时done_batch[0]=1, 对应的expected_q_value等于reward + expected_q_values = reward_batch + \ + self.gamma * next_q_values * (1-done_batch) + # self.loss = F.smooth_l1_loss(q_values,expected_q_values.unsqueeze(1)) # 计算 Huber loss + loss = nn.MSELoss()(q_values, expected_q_values.unsqueeze(1)) # 计算 均方误差loss + # 优化模型 + self.optimizer.zero_grad() # zero_grad清除上一步所有旧的gradients from the last step + # loss.backward()使用backpropagation计算loss相对于所有parameters(需要gradients)的微分 + loss.backward() + # for param in self.policy_net.parameters(): # clip防止梯度爆炸 + # param.grad.data.clamp_(-1, 1) + self.optimizer.step() # 更新模型 +``` + +更新遵循伪代码的以下部分： + +image-20210507162813393 + +首先从replay buffer中选取一个batch的数据，计算loss，然后进行minibatch SGD。 + +然后是保存与加载模型的部分，如下： + +```python +def save(self, path): + torch.save(self.target_net.state_dict(), path+'dqn_checkpoint.pth') +def load(self, path): + self.target_net.load_state_dict(torch.load(path+'dqn_checkpoint.pth')) + for target_param, param in zip(self.target_net.parameters(), self.policy_net.parameters()): + param.data.copy_(target_param.data) +``` + + + +### 实验结果 + +训练结果如下： + +train_rewards_curve + +eval_rewards_curve ## 参考 diff --git a/codes/DQN/agent.py b/codes/DQN/agent.py index 669295f..514709a 100644 --- a/codes/DQN/agent.py +++ b/codes/DQN/agent.py @@ -5,7 +5,7 @@ @Email: johnjim0816@gmail.com @Date: 2020-06-12 00:50:49 @LastEditor: John -LastEditTime: 2021-04-29 22:19:18 +LastEditTime: 2021-05-07 16:30:05 @Discription: @Environment: python 3.7.7 ''' @@ -35,15 +35,13 @@ class DQN: (cfg.epsilon_start - cfg.epsilon_end) * \ math.exp(-1. * frame_idx / cfg.epsilon_decay) self.batch_size = cfg.batch_size - self.policy_net = MLP(state_dim, action_dim, - hidden_dim=cfg.hidden_dim).to(self.device) - self.target_net = MLP(state_dim, action_dim, - hidden_dim=cfg.hidden_dim).to(self.device) - for target_param, param in zip(self.target_net.parameters(), self.policy_net.parameters()): + self.policy_net = MLP(state_dim, action_dim,hidden_dim=cfg.hidden_dim).to(self.device) + self.target_net = MLP(state_dim, action_dim,hidden_dim=cfg.hidden_dim).to(self.device) + for target_param, param in zip(self.target_net.parameters(),self.policy_net.parameters()): # copy params from policy net target_param.data.copy_(param.data) self.optimizer = optim.Adam(self.policy_net.parameters(), lr=cfg.lr) - self.loss = 0 self.memory = ReplayBuffer(cfg.memory_capacity) + def choose_action(self, state): '''选择动作 @@ -92,11 +90,11 @@ class DQN: expected_q_values = reward_batch + \ self.gamma * next_q_values * (1-done_batch) # self.loss = F.smooth_l1_loss(q_values,expected_q_values.unsqueeze(1)) # 计算 Huber loss - self.loss = nn.MSELoss()(q_values, expected_q_values.unsqueeze(1)) # 计算 均方误差loss + loss = nn.MSELoss()(q_values, expected_q_values.unsqueeze(1)) # 计算 均方误差loss # 优化模型 self.optimizer.zero_grad() # zero_grad清除上一步所有旧的gradients from the last step # loss.backward()使用backpropagation计算loss相对于所有parameters(需要gradients)的微分 - self.loss.backward() + loss.backward() # for param in self.policy_net.parameters(): # clip防止梯度爆炸 # param.grad.data.clamp_(-1, 1) self.optimizer.step() # 更新模型 diff --git a/codes/DQN/assets/eval_rewards_curve.png b/codes/DQN/assets/eval_rewards_curve.png new file mode 100644 index 0000000..0327b47 Binary files /dev/null and b/codes/DQN/assets/eval_rewards_curve.png differ diff --git a/codes/DQN/assets/image-20210507162813393.png b/codes/DQN/assets/image-20210507162813393.png new file mode 100644 index 0000000..3c4ae33 Binary files /dev/null and b/codes/DQN/assets/image-20210507162813393.png differ diff --git a/codes/DQN/assets/train_rewards_curve.png b/codes/DQN/assets/train_rewards_curve.png new file mode 100644 index 0000000..b9667f1 Binary files /dev/null and b/codes/DQN/assets/train_rewards_curve.png differ diff --git a/codes/DQN/main.ipynb b/codes/DQN/main.ipynb deleted file mode 100644 index e21c74c..0000000 --- a/codes/DQN/main.ipynb +++ /dev/null @@ -1,467 +0,0 @@ -{ - "metadata": { - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.10-final" - }, - "orig_nbformat": 2, - "kernelspec": { - "name": "python3", - "display_name": "Python 3", - "language": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 2, - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "import sys,os\n", - "from pathlib import Path\n", - "curr_path = str(Path().absolute())\n", - "parent_path = str(Path().absolute().parent)\n", - "sys.path.append(parent_path) # add current terminal path to sys.path\n", - "import gym\n", - "import torch\n", - "import datetime\n", - "from DQN.agent import DQN\n", - "from common.plot import plot_rewards\n", - "from common.utils import save_results" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "SEQUENCE = datetime.datetime.now().strftime(\"%Y%m%d-%H%M%S\") # 获取当前时间\n", - "SAVED_MODEL_PATH = curr_path+\"/saved_model/\"+SEQUENCE+'/' # 生成保存的模型路径\n", - "if not os.path.exists(curr_path+\"/saved_model/\"): # 检测是否存在文件夹\n", - " os.mkdir(curr_path+\"/saved_model/\")\n", - "if not os.path.exists(SAVED_MODEL_PATH): # 检测是否存在文件夹\n", - " os.mkdir(SAVED_MODEL_PATH)\n", - "RESULT_PATH = curr_path+\"/results/\"+SEQUENCE+'/' # 存储reward的路径\n", - "if not os.path.exists(curr_path+\"/results/\"): # 检测是否存在文件夹\n", - " os.mkdir(curr_path+\"/results/\")\n", - "if not os.path.exists(RESULT_PATH): # 检测是否存在文件夹\n", - " os.mkdir(RESULT_PATH)" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "class DQNConfig:\n", - " def __init__(self):\n", - " self.algo = \"DQN\" # 算法名称\n", - " self.gamma = 0.99\n", - " self.epsilon_start = 0.95 # e-greedy策略的初始epsilon\n", - " self.epsilon_end = 0.01\n", - " self.epsilon_decay = 200\n", - " self.lr = 0.01 # 学习率\n", - " self.memory_capacity = 800 # Replay Memory容量\n", - " self.batch_size = 64\n", - " self.train_eps = 300 # 训练的episode数目\n", - " self.train_steps = 200 # 训练每个episode的最大长度\n", - " self.target_update = 2 # target net的更新频率\n", - " self.eval_eps = 20 # 测试的episode数目\n", - " self.eval_steps = 200 # 测试每个episode的最大长度\n", - " self.device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\") # 检测gpu\n", - " self.hidden_dim = 128 # 神经网络隐藏层维度" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "def train(cfg,env,agent):\n", - " print('Start to train !')\n", - " rewards = []\n", - " ma_rewards = [] # 滑动平均的reward\n", - " ep_steps = []\n", - " for i_episode in range(cfg.train_eps):\n", - " state = env.reset() # reset环境状态\n", - " ep_reward = 0\n", - " for i_step in range(cfg.train_steps):\n", - " action = agent.choose_action(state) # 根据当前环境state选择action\n", - " next_state, reward, done, _ = env.step(action) # 更新环境参数\n", - " ep_reward += reward\n", - " agent.memory.push(state, action, reward, next_state, done) # 将state等这些transition存入memory\n", - " state = next_state # 跳转到下一个状态\n", - " agent.update() # 每步更新网络\n", - " if done:\n", - " break\n", - " # 更新target network，复制DQN中的所有weights and biases\n", - " if i_episode % cfg.target_update == 0:\n", - " agent.target_net.load_state_dict(agent.policy_net.state_dict())\n", - " print('Episode:{}/{}, Reward:{}, Steps:{}, Done:{}'.format(i_episode+1,cfg.train_eps,ep_reward,i_step+1,done))\n", - " ep_steps.append(i_step)\n", - " rewards.append(ep_reward)\n", - " # 计算滑动窗口的reward\n", - " if ma_rewards:\n", - " ma_rewards.append(\n", - " 0.9*ma_rewards[-1]+0.1*ep_reward)\n", - " else:\n", - " ma_rewards.append(ep_reward) \n", - " print('Complete training！')\n", - " return rewards,ma_rewards" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": { - "tags": [] - }, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "Start to train !\n", - "Episode:1/300, Reward:41.0, Steps:41, Done:True\n", - "Episode:2/300, Reward:23.0, Steps:23, Done:True\n", - "Episode:3/300, Reward:19.0, Steps:19, Done:True\n", - "Episode:4/300, Reward:17.0, Steps:17, Done:True\n", - "Episode:5/300, Reward:14.0, Steps:14, Done:True\n", - "Episode:6/300, Reward:15.0, Steps:15, Done:True\n", - "Episode:7/300, Reward:10.0, Steps:10, Done:True\n", - "Episode:8/300, Reward:23.0, Steps:23, Done:True\n", - "Episode:9/300, Reward:14.0, Steps:14, Done:True\n", - "Episode:10/300, Reward:9.0, Steps:9, Done:True\n", - "Episode:11/300, Reward:9.0, Steps:9, Done:True\n", - "Episode:12/300, Reward:9.0, Steps:9, Done:True\n", - "Episode:13/300, Reward:9.0, Steps:9, Done:True\n", - "Episode:14/300, Reward:10.0, Steps:10, Done:True\n", - "Episode:15/300, Reward:10.0, Steps:10, Done:True\n", - "Episode:16/300, Reward:12.0, Steps:12, Done:True\n", - "Episode:17/300, Reward:10.0, Steps:10, Done:True\n", - "Episode:18/300, Reward:10.0, Steps:10, Done:True\n", - "Episode:19/300, Reward:9.0, Steps:9, Done:True\n", - "Episode:20/300, Reward:10.0, Steps:10, Done:True\n", - "Episode:21/300, Reward:8.0, Steps:8, Done:True\n", - "Episode:22/300, Reward:10.0, Steps:10, Done:True\n", - "Episode:23/300, Reward:10.0, Steps:10, Done:True\n", - "Episode:24/300, Reward:13.0, Steps:13, Done:True\n", - "Episode:25/300, Reward:10.0, Steps:10, Done:True\n", - "Episode:26/300, Reward:9.0, Steps:9, Done:True\n", - "Episode:27/300, Reward:9.0, Steps:9, Done:True\n", - "Episode:28/300, Reward:9.0, Steps:9, Done:True\n", - "Episode:29/300, Reward:12.0, Steps:12, Done:True\n", - "Episode:30/300, Reward:9.0, Steps:9, Done:True\n", - "Episode:31/300, Reward:9.0, Steps:9, Done:True\n", - "Episode:32/300, Reward:10.0, Steps:10, Done:True\n", - "Episode:33/300, Reward:11.0, Steps:11, Done:True\n", - "Episode:34/300, Reward:12.0, Steps:12, Done:True\n", - "Episode:35/300, Reward:8.0, Steps:8, Done:True\n", - "Episode:36/300, Reward:10.0, Steps:10, Done:True\n", - "Episode:37/300, Reward:9.0, Steps:9, Done:True\n", - "Episode:38/300, Reward:10.0, Steps:10, Done:True\n", - "Episode:39/300, Reward:10.0, Steps:10, Done:True\n", - "Episode:40/300, Reward:10.0, Steps:10, Done:True\n", - "Episode:41/300, Reward:9.0, Steps:9, Done:True\n", - "Episode:42/300, Reward:10.0, Steps:10, Done:True\n", - "Episode:43/300, Reward:10.0, Steps:10, Done:True\n", - "Episode:44/300, Reward:10.0, Steps:10, Done:True\n", - "Episode:45/300, Reward:9.0, Steps:9, Done:True\n", - "Episode:46/300, Reward:22.0, Steps:22, Done:True\n", - "Episode:47/300, Reward:74.0, Steps:74, Done:True\n", - "Episode:48/300, Reward:13.0, Steps:13, Done:True\n", - "Episode:49/300, Reward:29.0, Steps:29, Done:True\n", - "Episode:50/300, Reward:56.0, Steps:56, Done:True\n", - "Episode:51/300, Reward:74.0, Steps:74, Done:True\n", - "Episode:52/300, Reward:85.0, Steps:85, Done:True\n", - "Episode:53/300, Reward:72.0, Steps:72, Done:True\n", - "Episode:54/300, Reward:114.0, Steps:114, Done:True\n", - "Episode:55/300, Reward:97.0, Steps:97, Done:True\n", - "Episode:56/300, Reward:101.0, Steps:101, Done:True\n", - "Episode:57/300, Reward:104.0, Steps:104, Done:True\n", - "Episode:58/300, Reward:58.0, Steps:58, Done:True\n", - "Episode:59/300, Reward:11.0, Steps:11, Done:True\n", - "Episode:60/300, Reward:56.0, Steps:56, Done:True\n", - "Episode:61/300, Reward:74.0, Steps:74, Done:True\n", - "Episode:62/300, Reward:51.0, Steps:51, Done:True\n", - "Episode:63/300, Reward:113.0, Steps:113, Done:True\n", - "Episode:64/300, Reward:48.0, Steps:48, Done:True\n", - "Episode:65/300, Reward:97.0, Steps:97, Done:True\n", - "Episode:66/300, Reward:59.0, Steps:59, Done:True\n", - "Episode:67/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:68/300, Reward:67.0, Steps:67, Done:True\n", - "Episode:69/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:70/300, Reward:45.0, Steps:45, Done:True\n", - "Episode:71/300, Reward:48.0, Steps:48, Done:True\n", - "Episode:72/300, Reward:90.0, Steps:90, Done:True\n", - "Episode:73/300, Reward:47.0, Steps:47, Done:True\n", - "Episode:74/300, Reward:94.0, Steps:94, Done:True\n", - "Episode:75/300, Reward:107.0, Steps:107, Done:True\n", - "Episode:76/300, Reward:12.0, Steps:12, Done:True\n", - "Episode:77/300, Reward:30.0, Steps:30, Done:True\n", - "Episode:78/300, Reward:62.0, Steps:62, Done:True\n", - "Episode:79/300, Reward:64.0, Steps:64, Done:True\n", - "Episode:80/300, Reward:41.0, Steps:41, Done:True\n", - "Episode:81/300, Reward:67.0, Steps:67, Done:True\n", - "Episode:82/300, Reward:45.0, Steps:45, Done:True\n", - "Episode:83/300, Reward:130.0, Steps:130, Done:True\n", - "Episode:84/300, Reward:50.0, Steps:50, Done:True\n", - "Episode:85/300, Reward:51.0, Steps:51, Done:True\n", - "Episode:86/300, Reward:67.0, Steps:67, Done:True\n", - "Episode:87/300, Reward:37.0, Steps:37, Done:True\n", - "Episode:88/300, Reward:41.0, Steps:41, Done:True\n", - "Episode:89/300, Reward:54.0, Steps:54, Done:True\n", - "Episode:90/300, Reward:93.0, Steps:93, Done:True\n", - "Episode:91/300, Reward:71.0, Steps:71, Done:True\n", - "Episode:92/300, Reward:102.0, Steps:102, Done:True\n", - "Episode:93/300, Reward:55.0, Steps:55, Done:True\n", - "Episode:94/300, Reward:73.0, Steps:73, Done:True\n", - "Episode:95/300, Reward:61.0, Steps:61, Done:True\n", - "Episode:96/300, Reward:16.0, Steps:16, Done:True\n", - "Episode:97/300, Reward:61.0, Steps:61, Done:True\n", - "Episode:98/300, Reward:79.0, Steps:79, Done:True\n", - "Episode:99/300, Reward:76.0, Steps:76, Done:True\n", - "Episode:100/300, Reward:32.0, Steps:32, Done:True\n", - "Episode:101/300, Reward:95.0, Steps:95, Done:True\n", - "Episode:102/300, Reward:83.0, Steps:83, Done:True\n", - "Episode:103/300, Reward:41.0, Steps:41, Done:True\n", - "Episode:104/300, Reward:30.0, Steps:30, Done:True\n", - "Episode:105/300, Reward:83.0, Steps:83, Done:True\n", - "Episode:106/300, Reward:95.0, Steps:95, Done:True\n", - "Episode:107/300, Reward:104.0, Steps:104, Done:True\n", - "Episode:108/300, Reward:98.0, Steps:98, Done:True\n", - "Episode:109/300, Reward:109.0, Steps:109, Done:True\n", - "Episode:110/300, Reward:63.0, Steps:63, Done:True\n", - "Episode:111/300, Reward:98.0, Steps:98, Done:True\n", - "Episode:112/300, Reward:105.0, Steps:105, Done:True\n", - "Episode:113/300, Reward:99.0, Steps:99, Done:True\n", - "Episode:114/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:115/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:116/300, Reward:47.0, Steps:47, Done:True\n", - "Episode:117/300, Reward:98.0, Steps:98, Done:True\n", - "Episode:118/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:119/300, Reward:52.0, Steps:52, Done:True\n", - "Episode:120/300, Reward:55.0, Steps:55, Done:True\n", - "Episode:121/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:122/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:123/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:124/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:125/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:126/300, Reward:40.0, Steps:40, Done:True\n", - "Episode:127/300, Reward:42.0, Steps:42, Done:True\n", - "Episode:128/300, Reward:101.0, Steps:101, Done:True\n", - "Episode:129/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:130/300, Reward:70.0, Steps:70, Done:True\n", - "Episode:131/300, Reward:175.0, Steps:175, Done:True\n", - "Episode:132/300, Reward:90.0, Steps:90, Done:True\n", - "Episode:133/300, Reward:81.0, Steps:81, Done:True\n", - "Episode:134/300, Reward:61.0, Steps:61, Done:True\n", - "Episode:135/300, Reward:74.0, Steps:74, Done:True\n", - "Episode:136/300, Reward:68.0, Steps:68, Done:True\n", - "Episode:137/300, Reward:50.0, Steps:50, Done:True\n", - "Episode:138/300, Reward:51.0, Steps:51, Done:True\n", - "Episode:139/300, Reward:99.0, Steps:99, Done:True\n", - "Episode:140/300, Reward:87.0, Steps:87, Done:True\n", - "Episode:141/300, Reward:94.0, Steps:94, Done:True\n", - "Episode:142/300, Reward:51.0, Steps:51, Done:True\n", - "Episode:143/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:144/300, Reward:55.0, Steps:55, Done:True\n", - "Episode:145/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:146/300, Reward:57.0, Steps:57, Done:True\n", - "Episode:147/300, Reward:129.0, Steps:129, Done:True\n", - "Episode:148/300, Reward:74.0, Steps:74, Done:True\n", - "Episode:149/300, Reward:108.0, Steps:108, Done:True\n", - "Episode:150/300, Reward:63.0, Steps:63, Done:True\n", - "Episode:151/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:152/300, Reward:103.0, Steps:103, Done:True\n", - "Episode:153/300, Reward:129.0, Steps:129, Done:True\n", - "Episode:154/300, Reward:77.0, Steps:77, Done:True\n", - "Episode:155/300, Reward:129.0, Steps:129, Done:True\n", - "Episode:156/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:157/300, Reward:181.0, Steps:181, Done:True\n", - "Episode:158/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:159/300, Reward:136.0, Steps:136, Done:True\n", - "Episode:160/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:161/300, Reward:181.0, Steps:181, Done:True\n", - "Episode:162/300, Reward:120.0, Steps:120, Done:True\n", - "Episode:163/300, Reward:190.0, Steps:190, Done:True\n", - "Episode:164/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:165/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:166/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:167/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:168/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:169/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:170/300, Reward:89.0, Steps:89, Done:True\n", - "Episode:171/300, Reward:74.0, Steps:74, Done:True\n", - "Episode:172/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:173/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:174/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:175/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:176/300, Reward:93.0, Steps:93, Done:True\n", - "Episode:177/300, Reward:139.0, Steps:139, Done:True\n", - "Episode:178/300, Reward:78.0, Steps:78, Done:True\n", - "Episode:179/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:180/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:181/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:182/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:183/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:184/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:185/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:186/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:187/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:188/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:189/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:190/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:191/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:192/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:193/300, Reward:190.0, Steps:190, Done:True\n", - "Episode:194/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:195/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:196/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:197/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:198/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:199/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:200/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:201/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:202/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:203/300, Reward:67.0, Steps:67, Done:True\n", - "Episode:204/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:205/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:206/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:207/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:208/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:209/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:210/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:211/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:212/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:213/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:214/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:215/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:216/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:217/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:218/300, Reward:44.0, Steps:44, Done:True\n", - "Episode:219/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:220/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:221/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:222/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:223/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:224/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:225/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:226/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:227/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:228/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:229/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:230/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:231/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:232/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:233/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:234/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:235/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:236/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:237/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:238/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:239/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:240/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:241/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:242/300, Reward:126.0, Steps:126, Done:True\n", - "Episode:243/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:244/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:245/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:246/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:247/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:248/300, Reward:118.0, Steps:118, Done:True\n", - "Episode:249/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:250/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:251/300, Reward:99.0, Steps:99, Done:True\n", - "Episode:252/300, Reward:145.0, Steps:145, Done:True\n", - "Episode:253/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:254/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:255/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:256/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:257/300, Reward:130.0, Steps:130, Done:True\n", - "Episode:258/300, Reward:170.0, Steps:170, Done:True\n", - "Episode:259/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:260/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:261/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:262/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:263/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:264/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:265/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:266/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:267/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:268/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:269/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:270/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:271/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:272/300, Reward:135.0, Steps:135, Done:True\n", - "Episode:273/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:274/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:275/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:276/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:277/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:278/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:279/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:280/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:281/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:282/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:283/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:284/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:285/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:286/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:287/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:288/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:289/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:290/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:291/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:292/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:293/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:294/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:295/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:296/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:297/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:298/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:299/300, Reward:200.0, Steps:200, Done:True\n", - "Episode:300/300, Reward:200.0, Steps:200, Done:True\n", - "Complete training！\n", - "results saved!\n" - ] - }, - { - "output_type": "display_data", - "data": { - "text/plain": "
", - "image/svg+xml": "\n\n\n \n \n \n \n 2021-03-29T19:53:51.889101\n image/svg+xml\n \n \n Matplotlib v3.4.0, https://matplotlib.org/\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n\n", - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXsAAAEcCAYAAAAmzxTpAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8+yak3AAAACXBIWXMAAAsTAAALEwEAmpwYAACVh0lEQVR4nO2dd5wU9f3/nzOz5XrlgKMXBRGUdtgAC2gsWOPXhK+JRo1JTKKYGE2MEjWWRNSYRGK+amLMz0jUJFZs2EssICooooB0uON63z4zvz9mZ3Zmd/Zud2+PK8zr8YC9nfIps7uvec/r/f6834KqqioOHDhw4GBQQ+zrAThw4MCBg96HQ/YOHDhwcADAIXsHDhw4OADgkL0DBw4cHABwyN6BAwcODgA4ZO/AgQMHBwAcsnfQL7F8+XKuvvrqPul70aJFrF69uk/67m8IBAJcdtllzJ49myVLlvT1cBz0AK6+HoADB/0Nzz//fF8Pod/gpZdeoqGhgdWrV+NyJdLF8uXLue+++/B4PAAMHTqUuXPnctlllzF06FDjuLa2Nn73u9/x6quv0tHRwZgxY7jkkks455xzjGMWLFiA3+/ntddeIy8vD4B///vfPPvss/zjH//o5ZkOfjiWvYOUEIlE+noIWcFgmMf+nEN1dTXjxo2zJXodp556Kp988glr1qzhT3/6Ew0NDXz961+nrq4OgFAoxEUXXUR1dTWPPfYYa9eu5ZprruHOO+/k4YcftrSlKErCNgfZgUP2gwAPPPAAJ554IjNnzuS0007jlVdeAbQfWVVVFZs3bzaObWpq4vDDD6exsRGAN954g7POOouqqioWL17Ml19+aRy7YMECHnjgAc444wxmzJhBJBJJ2heALMvcfvvtHHnkkSxYsIBHHnmEyZMnG+TU3t7Oddddx7x585g/fz6///3vkWU5pTmuW7eOxYsXU1VVxZlnnmmRWZ544glOPfVUZs6cycKFC3nssceMfatXr+bYY4/lgQceYO7cufzyl79k+fLlXHnllfz85z9n5syZLFq0iM8++8wy7/feew+g22M///xzzj77bGbOnMmSJUv4yU9+wu9///uk8/jXv/5ljPW0007j888/B2Dy5Mns3LnTOO7aa6812rGbw6mnnsobb7xhHB+JRDjqqKOM9rq6XvHYunUrF1xwAVVVVSxatIjXXnsNgHvuuYc///nPvPjii8ycOZN///vfXXxC4Ha7Ofjgg/n9739PWVkZDz30EADPPPMMNTU1/PGPf2T06NG43W6OPfZYli5dyh/+8Ac6OzuNNr773e/yt7/9jba2ti77cpA+HLIfBBg9ejQrVqzgo48+4vLLL+eaa66hrq4Oj8fDSSedZJElXnzxRebMmUN5eTkbN27kuuuu4+abb2b16tV885vf5Ec/+hGhUMg4/vnnn+eBBx5g7dq1uFyupH2BRmRvv/02zzzzDE899RSvvvqqZZzXXnstLpeLl19+maeffpp33323WwIBqK2t5Qc/+AE//OEPWbNmDb/4xS9YsmQJTU1NAJSXl3P//ffz8ccf89vf/pbf/va3BukBNDQ00NrayhtvvMEtt9wCwOuvv86iRYtYu3YtCxYsMLbbIdmxoVCIyy+/nHPOOYc1a9Zw+umnJ8zZjBdffJHly5ezbNkyPv74Y/7v//6PkpKSbudvN4dFixbx3HPPGfv/+9//UlpaytSpU7u9XmaEw2Euu+wy5s6dy3vvvcfSpUu5+uqr2bZtG0uWLOEHP/iBYbmfd955KY1VkiQWLlzI2rVrAXjvvfc49thjDWlGx9e+9jUCgQDr1q0ztk2bNo0jjjiCBx98MKW+HKQOh+wHAU499VSGDRuGKIqcdtppjB07lk8//RSAM844w0L2K1eu5IwzzgDg8ccf55vf/CbTp09HkiTOOecc3G635cd3wQUXUFlZSU5OTrd9vfjii1x44YUMHz6c4uJivv/97xvtNDQ08NZbb3HdddeRl5dHeXk5F110UUr6+DPPPMOxxx7LcccdhyiKzJ07l2nTpvHWW28BcPzxxzNmzBgEQeCII45g7ty5BtEAiKLIkiVL8Hg8xjxmz57NcccdhyRJnHXWWZYnmngkO3b9+vVEIhEuvPBC3G43X/va1zjssMOStvOf//yHSy+9lMMPPxxBEBg7diwjR47sdv52czjjjDN4/fXX8fv9gPa5Llq0KKXrZcb69evx+Xx8//vfx+PxcPTRR3PCCSf02G8xdOhQWltbAWhubqaioiLhGJfLRWlpacJNaMmSJTzyyCO2NycHmcNx0A4CPP300zz00EPs3bsXAJ/PR3NzMwBHHnkkgUCA9evXU15ezpdffsmJJ54IaHrs008/zSOPPGK0FQ6HDUsdoLKyMuW+6urqLMcPHz7c+Lu6uppIJMK8efOMbYqiJLRvh+rqal566aUE2eLII48E4K233uLee+9lx44dKIpCIBBg0qRJxrGlpaV4vV5Lm0OGDDH+zsnJIRgMEolEbLXpZMfW1dUxbNgwBEEw9nc1n5qaGsaMGdPtfO0QP4exY8cyceJE3njjDU444QRef/11nn76aaD762VGXV0dw4cPRxRjdt+IESOora3NaJw6amtrKS4uNsZeX1+fcEwkEqG5uZnS0lLL9kmTJnH88cfzwAMPMHHixB6Nw0EMDtkPcOzdu5elS5fy97//nZkzZxrWpw5JkjjllFN47rnnGDJkCMcffzwFBQWARkyXXXYZP/zhD5O2byay7vqqqKhg3759xnvz38OHD8fj8fDBBx906eyzQ2VlJWeddRa33nprwr5QKMSSJUtYtmwZCxcuxO1286Mf/QhzMlfzHLKJiooKamtrUVXV6KOmpobRo0cnnceuXbts9+Xm5hpWOkB9fT3Dhg0z3tvN4fTTT+e5555DURQOOuggxo4da/ST7HrFY+jQoezbtw9FUQzCr6mpYdy4cd2emwyKovDGG29wzDHHAHDMMcdw99134/P5LFLOyy+/jNvtZvr06QltLFmyhHPOOYdLLrkk43E4sMKRcQY4/H4/giBQVlYGaM7KLVu2WI4544wzePHFF1m5ciWnn366sf28887jscceY/369aiqis/n480336SjoyOjvk499VQefvhhamtraWtr4y9/+YuxTw/Ju/322+no6EBRFHbt2sWaNWu6neOZZ57JG2+8wTvvvIMsywSDQVavXs2+ffsIhUKEQiHKyspwuVy89dZbvPvuu6lfwB5gxowZSJLEI488QiQS4dVXX7U4b+PxP//zP/ztb39jw4YNqKrKzp07jSekQw45hOeeew5Zlnn77bf58MMPu+3/tNNO49133+XRRx+1fK5dXa94HH744eTk5PDXv/6VcDjM6tWref311znttNPSvh6RSIStW7dy1VVX0dDQwEUXXQTAWWedxfDhw7nyyivZs2cP4XCYd955h1tvvZXvfve7FBYWJrQ1duxYTjvtNCfkMotwyH6A46CDDuKSSy5h8eLFHHPMMWzevJlZs2ZZjpk+fTq5ubnU1dVx7LHHGtsPO+wwbrnlFm6++WbmzJnD1772NZ588smM+/rGN77B3LlzOfPMMzn77LM57rjjcLlcSJIEwB133EE4HOa0005jzpw5LFmyxPbxPh6VlZX8+c9/5v777+foo4/muOOO48EHH0RRFAoKCli6dCk/+clPmDNnDs899xwLFixI9zJmBI/Hw/Lly/nPf/7DnDlzePbZZzn++OONmPN4nHrqqVx22WX87Gc/Y9asWfz4xz82dO3rr7+eN954g6qqKlauXGlIbV1h6NChzJgxg08++cRCzl1dL7s53Hfffbz99tscddRR/PrXv+aOO+5ISz7Ro3Wqqqr44Q9/SElJCU8++aTxZOLxeHjooYeorKzkG9/4BtOnT+fSSy/lO9/5DpdffnnSdn/84x/j8/lSHoeDriE4xUsc9BbeeustbrrpJot2PNhx3nnnsXjxYs4999y+Hkq/RTgc5nvf+x7Dhg3j9ttv7zWZzYEVjmXvIGsIBAK89dZbRCIRamtruffee1OyUAcy1qxZQ319PZFIhKeeeopNmzYxf/78vh5Wv4bb7Wb58uWMHj2abdu29fVwDhg4lr2DrMHv9/Ptb3+bbdu2kZOTw/HHH8/1119vOIQHIx5//HH++Mc/4vf7GTVqFD/72c84/vjj+3pYDhwkwCF7Bw4cODgA4Mg4Dhw4cHAAwCF7Bw4cODgA4JC9AwcOHBwA6NcraJubO1GU9F0K5eUFNDbaLwwaaHDm0j/hzKX/YbDMAzKfiygKlJbm2+7r12SvKGpGZK+fO1jgzKV/wplL/8NgmQdkfy6OjOPAgQMHBwAcsnfgwIGDAwAO2Ttw4MDBAYBuyb65uZnvfe97nHzyyZxxxhlcfvnlRlGBdevWceaZZ3LyySdzySWXGKXuutvnwIEDBw72L7ole0EQuPTSS1m1ahUrV65k9OjR3HXXXSiKwjXXXMMNN9zAqlWrqKqq4q677gLocp8DBw4cONj/6JbsS0pKLBVuZsyYQXV1NRs2bMDr9VJVVQXA4sWLeemllwC63OfAQapQVRWli2weSnS//q8/Z/5Q48aqxM0tfn/8XOzO7e6f3ka3x0ajPpKNsbt/PTm3J/Mwbzdfr2TXMZMx9vbcuppvtpFW6KWiKDz66KMsWLCAmpoaRowYYewrKytDURRaWlq63JdqgWUHPcfz7+9gd10HE0YUs2lXM1ece3hfDyktPLByI6s31nJS1Wj+98SDefz1LUQiKt/62iSeeGsrz7+/03L8lLGlXPO/MwF497Ma3vm0hmu/Ncuu6aTY1+Tjd499wmVnTeO+Zzbwi2/NYkhxrrHfH4zw64c+5BffmUNprvbzCUcUfnDXm3z7a5N45OXN/O+JB/Poq1s4Z/54zpg7HoDbV3zMlj2tCf2ddtRY/uf4ifz9xS9559MaY/vRU4fzvTMOBWDlu9t56p3tac0DoCjfw8WnHsKfnvwMuYswPq9HYsm5h/OnJz/FH5TT7uc7p0zm3Q37+MpmftnAqIoCTjt6DH95diPmWZQUeLj4tCn8/l/ryfVKPPSrk9m5r53b/vEREVnL3e91S/z6kjm4JJGlf11NIJT6/ATgkkVTePWjPezc157dSRl9qOQJQQqEAAVigAIhSE5xKUt/eUHW+0qL7G+55Rby8vL49re/zSuvvJL1wcSjvDzzbIkVFYnVbwYqMp1LXWuQ3fWdFBZ42VXX0S+uSTpjqG3WyvQ1tgepqCikutFPRFaoqCikoS1ISaGX047RyHT15zXUNvuN9hs7drK9pi3tOe+o76SxLciuBh+NbUHCCJY29tZ3UNfiZ09dO5OqtHqyrR1BAFa8shmAZ/6rEfNT72znkrMPR1FUtlW3cdjEIRx2UKye7QvvbTfmVt8aYHh5HguqxvDmR7upbwuY5hKiMM/NGfNTLyiyp7adt9ftZcPOZmRF5ZsnTkKSEh/kG1r8vLx6J/taAviDMsfOHMmooalfs3+9uolWf4R9jT4mjSmhasrw7k9KFarK9k1f8dXefbT5R6IC5598CACbdzWz9ota6tu1a+8PyrR2BAkqEJEVTjtmHKGwwqsf7kIRJWRRIBCSWVA1muHl9ouO4vHYy1/SFohQ3dDJlHFlzJw8NON5eEKt5PuqyfXX4Q024Q02kRNswhNqQ8BaVCbiLQcuyPrvNWWyX7ZsGTt37uS+++5DFEUqKyuprq429jc1NSGKIiUlJV3uSweNjR0ZLSyoqCikvr537sT7Gz2Ziz8QJhKR8fvDRGSlz69JunMJRzQrLBgMU1/fTjAUQY7OIxiMUJTr5sSZ2hPk3to2Glr8Rvs+XwhFUdOec0uLdoNp7wgA0Nzss7TR2NgJgKJgbG/3hQDQn74lMVaMo76+nbbOELKicviEMhbOjD3x/nfdHgIBbW6hsMyQohxOnDmCjVsbaGoPGO0HAmHyc2JzTQWbd7fw9rq9bNrRhMct8rXZI22LhGytbuXl1Ttpb9fmO2NiOTNMN6Tu8MQbW/D5tPmNqShIa4zxUFUVpWkP8p4NyPs2I+/bwpHBDnwFbv7bPgVREIz2Xais/aKW1taAcb6iqjS1aJ/P3KnDaPeFefXDXTQ2dxrHHDG5gkmjS1Iaz79f20xHRxBFUZlQWZjy3FRVRWnei7z3cyJ7NyLXfgXB2BiE3GKEogrE4YciFpQj5BYh5BRqr7mFiAXa9c/k9yqKQlIjOSWyv/vuu9mwYQMPPPCAUXJt2rRpBAIB1q5dS1VVFY899hinnHJKt/sc7D+oqoqqaq/0Xzk7KQwtWX+vqsbf5iLfoD1ym6VORVHJRPrUe4jp2HFjUmNjiR+nDjPZA7RELf/ifGu5QkEQbOcjCHFzUbVt6aC4QOurusHHkOKcpNWgxOh2XeYR0+xHNMaqZlxxSm7aTWTL+4S3f4TaVguAUDQMaewMmnZtoyhQjTvUhmh6MBGjA42YSi0qioosq8Z+/XNQFNWYn9uVerS5KArI0XPjP1M7KC01hLe8R3jLe6gdWvShUDwM97jZiBXjkIaMRSwdieDOSXkM2US3ZL9lyxbuv/9+xo0bx+LFiwEYNWoU9957L3fccQc33ngjwWCQkSNHcueddwIgimLSfQ72I9Q4wh9g0Icce405r1SsBCgIQpwzLrM5q3FkrsbdJQ2noIng4x8+XXFySUuHZvmXFHgt20Uh1o+Z0AVBSHDeimkSaUm+15iHTvx2cDdt5czcj0CuNPpOBWrIT/iLN/hZ7kvsaTsORR2S1g1JVVUiOz8h/Nkq5JpNIIhII6bgOvwUXGNnIOaXArDt5deZseNhvIFGRCE2D72vSMT6meufiyQKxg1BUVTCEe2mkA7ZS4JgaP9iErJXVRV593pC615A3rcZBAFp5FRcs87ENXIqYmHqT0m9jW7J/uCDD2bTpk22+2bNmsXKlSvT3udg/0Dz7lst4oEEczQEaNKJ2bK2kn28NazNOf4JoDvofclJLHtU63HasVbNNV4b1zX9kjjSFQXBciPTCV2Mm4uagWXv9UjkeiX8QZniuJsMgNy8l+AHj1O8+1MW5sJHHYcCud32oyoRwhvfIPjR0xDspEKEzuAeSIPsI9VfEFz9L5T67QiFQ/Ae+Q1ck+cj5iRq1KGcMgByQs2IYswfUNCxm3neL4nIMWlFUWMWvCSKxvXUyF6TBFMle1VVGeeqQw0VRdtLnFxk32aC7/0TpWEHQkE53iO/ievgoxHzSlLqY3+jXydCc9AzmK3bAWjYJ4zdYtmrxMk4Vss+ZplrEk/qfUZfFesY4tvt0rJPJuMUJMo4ZrnIbNnHhwxmIpEU53vxB32Wm4wqRwh9/Ayhdc+D20tg3HxydryDK+JDI/vk/cgNOwm88QBK816kkVPxzjmXnU/+AZcc0MbYzZVWg50EP3iM8KZ3EArKyTn2ElyT5iKIUtJzwp4SIqpIbqgJUahEjQQJfvAvDtr4Ggflw4uhmcaxZrnGbNnLikooatl7XMn7MtrxtRB462/8wPspn7cexRtMslj2ashH4N1HiGx5DyG/jJzjvovr4KMRxP5Np/17dA56BDVq3SoDVMZR4ghXVcEkcidY9mbEzknPLDb7B7TX+HZJ2J6g2UtxZN8ZIj/HhTuOaLSnkdg4daIVRcHSZiaWPWhPEvuafIZ8pLTV4X/1XpSGnbgmzcV71GJam0Lk7HgHt+wDym0X3qiqSvjTFwl++ARCbhG5X7sSaewMBEEggBePEujWryDXb8f/8nJUXwueGYvwzDoLwZVcXtIhShJNSj75kRZGSE10PnEjaus+AvmV5HTWkBNqAWKSlUH2kknGUVOXcSK71hF480HUcPRpLLQPmISk+zZqv8L/+n2oHU14ZpyOZ+YZCO7EJ6f+CIfsBzE0GSOm2w806OKTxUFrIluLFSrEE3C0jTTnHS/jxOtfhgO3SwdtvIwTStDrQbfgY+OMWfaJklQmlr3eZ3G+h8i+zQRW3YOKSs5JV+AeP1vry9VOWBVxR/zGmMxQ5QiBt/9GZMt7uMZXkTP/IoScWLRHQPWQrwToykEb3vwugXceQsgtJu/sXyFVjE95DqIADXIh44O7+b53K0QKyV30c7bViUz48HZywy3AMG2sJs1eFKwO2kgKZB/69EWCHzyOWD6G3IWXseaxBxkr12vtiQLhrz4g8OZfEPJLyTvzOqRhB6U8j/4Ah+wHMcxEPxBV+/iIGP1JRf/bTC2ijfRhfk25zziZJv5842ajdEH28ZZ9R9DWSSoKsZuKmdAFrA5aRVXTjpKBmGxU6duC/7mHEQqHkH/KTxCLY9q3IIp0qt6oZW+1ztVwEP+qPyBXf4Gn6uuaFRtH6H68uJXmpE8foc9fI/juP5BGHkrOwh/a6vJdQRIFGpRCDlWrqWYIB599PWJ+KZH2fciqQJ6J7DUZRyN1lyQY10xRYzKO22atgaqqmrz02SrthnbC9xFcHhqEMqYpW/EQZkTdewQ+fhGpcjK5X1uC4E0tVr8/wSH7QQxzJM5ArOlgJnmIWvOmfRbNXrAa4clkmO471V5iJGzdHe80Nh8b3wZoq2tbO4IMG12a0JVZm1fVWNijKBJ340o9SsaMkgIvk13VDPnsTcTy0eSd+jOLVa6NATqVHNyy1bJX5Qj+V5Yj13xJzvHfwz1prm0fATx4lIAt2Yc2vELwvRW4xs4k58QfIUjutOcgiALvBSdRWFTES76p/CYapSNKEi1KPvlyK6BSLPg0GUc2OWhNmn04olh0fMs4P3yC8GercE89Ee/R5yNEn8yaBM05fHbeWkbv2WK5EQxEOGQ/iKE7NC1a9wCC2ckKcZY91pjweAetaj4wDSSEXsY/GcTdDMzH6gjLsegcfyiCLyiTl5P4UxOTaPZa6KV5TGramr3Suo85Tc9xRPEniCXDbYle76tT9VKqxCx7VVUIvPEA8p4NeI+9OCnRAwTJwa0EUVXF4qAN7/hYI/pxs8k58YcZOy9FQaBGLuU993giQsCyvVEpoEhu5ezcjzgu5wuCnUcYn4sgxOQ0RVEJRWQ87kSrPvTZy4TWPYd7yvF4j/mW5abaJJSCCnNzttBaMpmRCy/r907YrjBwR+6gW2hhl4mJogYKEix7JUaIqqoaFhhErco4ndv8mioMkk/ySGDW2I1z4o7V9WGAQEhGUe0X5ZhJ3WwZm+PvY/vScDKH/Phf+gNuXwvS6KnkHHuxLdFrY4BOxcswud0YU2jdC0S2rcFzxHl4Djmuy778eBBRcBMxxi837iLw+v2IFRPIWfCDHhGkft0ismK5hqIo0KQUMFHexvDcaE6htjpkJQ9JFBAEs4yjfSbxEk5kzwaC7z+Ka9xsvHMvTLjGrWIJQdnN3nAJnZPOZ9QAJnpwyH7QQ7fsB6KMo8RJKRr/WXV8A/EOWhtSTgdykpuF3U0kwbI3k30wgqKotvKBKAqW9nTL2Bx/r81BTbnKkKqqBN75O0pbLbmLfo5rxJQuj9cte69SB4CnaSuhtU/gmnAEnumnddtfMBoJkyuENFkqHMD/8nIEbx65Jy/pseShx8pHZMVyDUVBoFEuQEKhVi5imNQGvmYUJde4KZgXVYUiiiUaSvG1EHjjAcTSEeQs+L7FcNAhSBJ/jZzL9naRiwdIxE1XcCpVDWJYV88OPLY3RBuTtGKWdMz8meCgNVIt9NRBGzcmuzj7uDupWcYJhGSUJMvtBcwyTkyWio+zT4g86gLhTW8T2boaT9W53RI9aATQqXrxqAHyhQBFn/w/hKKh2tNACn0GomSfJ4YQgODqf6G2N2jO2CwsLjLSIshqHNnDxvBItgrjuL99obbR10xEUQwHefwKWj0SR5Op/oIaCpCz8EcILnsil0SBBrmQMK6kK2gHEhyyH8TQ4+tVBuaiqsQ4+/h0Ccl/gIpqT9bdIT7aJmk0ThehlxbLPhSxyE9mWEIvsebGscbZ2z8ZxEPxtxFc/S+kysl4ZnRvletj6FRyEFE5L381YrCd3IU/QvDkdn8yEESz3POEEGWd2whvfB33YV/DNXxSSud3B4PsI4olZYQoCuyRy3lSPJVGpRCf4kbwNUfz2Gi0Jqgy09y7kGWZcETBEyX78KZ3kPd+jveY85HKRnbZt/5ZppIbp7/DIftBDD0KZ8DG2cdF1Ghzie2zRuPE55PRz8ls4slkIDvLXo47yGzZ+4IRVOzJwizjWDX7eBkntUVVwQ8eg3AA77zvIAip/bQFATpUzbKd6dlJcPxxSEPGpnQuQFDQknoVin4mVa9EKB6Od865KZ/fHXSCD8fJOIIQs/gBWpV8RH+zVTL74lW+V/gmRe1bCUdk3C4RNdBBaM1/kIZPwt2NPyKV3DgDCQ7ZD2JY4uwHINvHiDqma6sm4hcsMo5xmOXcdKedIOPEyUA6jafqoPUFItr4bB201puSORrH3G8q6RIi1V8Q2fIenumnIZWmnmZY1+wBWpRc/IecmvK5AAFBO/eknM/IDbeQM+/CrIYm6lJ6RE607PXtAC1qLoKvBVnWJDPF34a8/nkA8n01Uc1eJLj2KdRgB9653+72mjqWvYMBA5UY0Q88qk+MfFEVNbaqVlUTMrGY55g0dDLFPpPeLOJkHu1v6yHmsEyD7G24oqs4e3Ob3aUiUFVVS2WQX4Zn5hnJD7SBKEC9XISMyJOdRyB68tI6Pxgl+1GuZpoKD8I18tC0zu9+fGbNPnG7fq11y15PRxz6+BmIBPEpbvICtYQjChVCM+EvXsd96AKk8jHd9i2JgvHkkG7W0f4Ih+wHMVRVI/yYJDGwKD8WjROTc8zGvtky03+MBnkmy1rZHaInyHF9x3Ynbu+q5J9O9vEpFLQxY/lsLJa9RZLqOsWxvPdzlNqvtBWuaVrVgiDQqBTyYP5lrA+PTTuePxzV7BUVdgw/Kb2TU4Al9NI0OCneslfyEIPtKHKYIWIb4Y1v4p5yPNsjQ8kP1BKWFWaFPgTJg2f22Sn1bXEIO5a9g/6MeAlnIHG93dOI2UGbYO0K+jEY+/Vz0oHhH0ijeElXffiCYcDeshcFwXJTMufGSYizT9K+qqoEP3oaIb8M9+T5SceRDHqfYVWKvk+P1FRBpF0oZE3oIPx5lWn33x2E6IULR+I1e+3VTPYCKt5IO0cLn4Io4Jl1JjVKGfnBBorDjYwLbsJ96Akpp2wwSzeOjOOgXyOemDJ1VvYFLJq4nYOWRAetdoxVvkl3znYO2O7265b9rZcemVDSLxwtrpFMszeP18h6aRNnn4yELVa9lP6yGb1d3cmcrmUvCgL/KTifxzuPyigzZ3eQbHR689/69d0dKQdgdGAL09iE++C5iHkl1ChliCiczhuoiHgOT71inp2PYCDDIftBjO6Iqz/DznI2rwSOj1CJ/ykms8y7Q7xmnyjjWF/Nx9jlXtEjc5KtoLVzOMdHFnWl2Yc+exkhryQjq17rK9qHkWYgTVITIKB6URDTqhuQKuIXUsX/LUev7165DNlTwCzff3Eh4z78ZAC2KiPwuwoZKdazs3BGWrH/g03GSckUWLZsGatWrWLv3r2sXLmSSZMmsWfPHn784x8bx7S3t9PR0cGaNWsAWLBgAR6PB69Xc+BcffXVzJ+f2RfSQYYwJIno2wFl2ZtN29g2O40bYqRl3Bjs2kml3+hrJouqBBuy1yNzBBuyEE1RN3aVqvQ5JtPslbZ65N2f4Zl1ZkZWPWCs2tW/I+lzfSzlQ6Y1aLtCMuta/1uNHqOo4C+fTEHNR2wTxzG9RItICgi5vFt5AblbX8dfPo/pafQ92GSclL4hCxcu5MILL+Rb3/qWsW3UqFE888wzxvvbbrsNWZYt591zzz1MmpSdxRUO0of+I0yWwbE/Q7GxnM0RKvE6dsxBG92fsWVvtegTK1VZxwSx6yuZcqjr0MvhSTZEaF48FR9nDxhVtuJLMBptf/kWCHQbL94VjHw2GVr25jw+vSHjJLfsY8dIkoASUfFVTKWg5iPWuWcZpC6JAp1SMc91HMGinOKs9D1QkRLZV1VVdbk/FAqxcuVKHnzwwawMykF2kFBoYyCRfVylJu3VLOOoKTlog2GZprYAZUU5KfVrnJ+0+ImNZa/GdPl4UgjLsX3xsBYvsa6g1bcRPcauqEh409u4xsxALChLaW52MOq06nNItwGhBxJQCrCEWyYhX5ckEI5Ax9DpvJzzbdpdZabjIBRRUEmv2Hh8fweMZd8dXn/9dYYNG8bUqVMt26+++mpUVWX27NlcddVVFBUVpdVuebl9pr5UUFGRXpGE/oxM5yLqy8aj39Oy8nzyctLPKZ5NpDqXTn/Y+FuUBCoqCg2ir6goRBRFcnM8RnuFBRqZ63PU86O8/0Ud/11fzaO3nJoSGeXna6GEuuySn++1jLmwWssOqaiqsT0vzxsdVwF5ebHrK4qCcRMqKc5NmHtengdBiM4NyM/T5lNYqM2lvLwAt0tCFAVyc9yW8zs2vkuHv40hR51GXg++68Gw9uShX5ryIQVUlKYea+9xS8bJhYU5Wf/ddYRjj3O5uaZr4IpRl9ulFVZXEGhzl5PrdRnHuVwSanR8ZSWJn0FXyM+LhbEOGVKw3zkl2/1lheyfeOIJzj3XukR6xYoVVFZWEgqFuO2227j55pu566670mq3sbEjI+diRUUh9fXtaZ/XH9GTuUSiEkIo+oNuaOgg19t3iU7TmUuHiewjEYX6+nYjXXN9fTsRWSYUChvtdXZqNUP1Oepzrm/y0ekPU1fXnpKTrT1aHFw/v709YBlzS6uW911VMba3tWmFP1qafYRDEeNYtyTij8bZd3YGE+YeDIaJyNG5KSqBgDYfny8EQF1dOx63RCSiEApFLOf7PnodIb+MjsKJdPbgu66vENXn29zUiRCRuzrFAllWDAmosyOQ9d9dS4vP+DtsugbN7UFju5HKWFEJBCN4XaJpHCpt0WODgXBa4wuZPsuWFh/e/WjcZ/q7F0UhqZHc42ic2tpaPvzwQ844w7pyr7JSi7n1eDycf/75fPzxxz3tykGaiEkS9vpzf0ZiAjKr30F7mxh6GXOgatv1aI1UQzDjHbDplCUUhdjTFGiygU6mdpqviHUFrTnO3tJXnGSlhvzIezbgmjDHNjVvOuhxNA6CybmbfTZM5iS1Six6NktVy40TF7WjP72kK+NYFnENAs2+x2T/1FNPcdxxx1FaWmps8/l8tLdrdyVVVXnhhReYMqX7dKsOsov44tkD2UEb7zCNT3FsECT6a3TuaYafJoZexu+30+y1V1EULKRgIXubp4qCcCMFdBrtmvPZm8egqtabRWTXOlAiuMd37UtLBfFpB9KVpnvdQWsTbqn3q8Ml6RFFqpYuQbLeIIKhzMj+gAy9vPXWW3n55ZdpaGjg4osvpqSkhOef15IMPfXUU1x//fWW4xsbG7niiiuQZRlFUZg4cSI33nhj9kfvoEvYWaEDBfElBuPj2+NzvAvE7dct8yRpD7rrN/miKr1/czROzHo3k4LbJRKIEk08WaiqyoK993NUrgdVPSWasjk6l/jUD3ELyCLb1iLklSAOm5jSnLpCNuLse9NBaw5ZFZJZ9tEKVIqKkRvHfFwgatl7TMVLUsEBGXq5dOlSli5dartv1apVCdtGjx7N008/3aOBOeg54q3TgbSCNiaNaEHndnHvlkVV8TJOfNhpXLKyZFDjzkuQceIjnMxjFa2hl26XSLtPT5dgJQulYQeg5YHXW9KPGbfnBf43vw5FnW+MyXhyCQeI7P4U9yHHpZzGuCvEVh7r79M/37DsezyaRNjlw4G4aBy9SImqItuUL/RF/T8uybHsHQxyZLqatC+hj1WShIQ89pCoY8fr3IkSVoqWffS1uxW0yWQci2UviUlT5EZ2xHxY+lgFAVRFpqLpE3LdcSUL9eyPu9aDHMY1YU5K80kFQg+sc5HeDr1MIuNYLPvYjd5cvEQ/JxSN6ElbxklyoxmocMh+EMMgPLs1/v0c5th11aLZY7za5caJl2HSJnu9n6SJ0BK3y4qWbtlOxklW/CKya12szVDAmINctw2XHKBEBCUSBjxRzT563s5PEHKLkIYdnNJ8UoEgCBnnxsGUobPXF1UlIX5DxlHUhHq/Wk56TcZxSekNMJlDeKDCyY0ziBGfwXEgSfdGvhkb/Tq6wV6zx9gNmOSYFCcfL//EnxXfrj42nQzMBGGWDcxcofhaURp30+4Zqr3vaNTmIIC8+9PYce2NRvvaAiwFee9GpJFTexyFY4ZotuzTFGNEwbz6NmtDsrQf+9tMvrHtMRkHWwetnpO+JzLOYLDsHbIfxOgujLA/Qx+qVrovPuQyMTmYXbgipH+j6y5dgmqzXTZZk2bStzr4Yj81ufoLAKpLZmh9dTRp5woCkT0bkMXoYp6OemNOggBK015Uf1vWC4QIgmCSYjI4V9f7e0G1Txp6mcSyT3DQCvY333T77g2Jan/DIftBDDsrdKBAJx8pTsaBWNoE888vIYIlXrNP8xokS5eQLM4+3rKP1+/NvlR570bw5NFQOFk7P2rBu2UfSv0OmodM1w7ssFr28t7PtT56gexjoZfpk1qmN4pUYHl6E63b9T26Za9p9okOWh3pyjiDzUHrkP0gRjy9DaRoHJ2sRVFAxSrD6KGYXYZexkfjpDj3eMds8sVdcWRvZKyMvdpZpaqqEtn7Oa4RUwh5SlBUAbVTs+xLO7YCKi1Dq4ioInQ0GHMRBIjs3YhYPByxoDyluaQKu/UK6Zwb0+z3n4PWvM9ltuxlq4PW8hlkKOMIwuBIhOaQ/SBGgmwzcLjeIOuYZR/bp1ei7Sr00rDs015BG+0/WYrjuONAc4BLNpa9nZygttejdjQijZyCIIq0KrmGjFPS/hWCtwB/4SialXzo1MleRUJBrvkSaaQ1/1Q2YOfoTufc3lxU1VWsu1HsxaTZJzhoTae4M5RxBoNeDw7ZD2okRJL0zTAygk7WGtnbLbKyLouPX0GbUL82ZdHeen58cURbB20Szd5OBojs3QiAa+RUBEGgRcmHTk2uKe7YgTTiEARRokEuRGytNiSr0uBeiISQRmVXwoGeWfYImadHTgXJVtBCzEkrioLm3Fa07JYuy3U3WfkZyjiDQcIBh+wHNZJJEAMB1tBLbDR76/GWtMDYyTjp9WtnwVv2xztoo/0ns+z1v+W9nyPklyIUD0cQ0Cx4XzPFgg9vqAVp+MGIgsBn4dGI7ftQar9CVaHCvx0EAVflIalNJA3YFW5PFaJgyo2TzUFFYR5OPOmai72IghBb0yCZ5xM73pVmBJMeCeZY9g76PRIIbuBwfWxRlSgmyjh2ln2UamIkrcs46YVeJiP3+P3x+eylFCx7VVWRazYhjZiCEI3Jb1byoLOJ8a467bxhByMI8GFwAqo7l9Dnr2mWvX83YvkYBG9+SvNIB3YVv9JBb8bZC0KsRkByshcQTGSfzLHqdmVo2Q8CvR4csh/ksBLVQHLQWqNxbGQc0g29TJHsk8g2sff2DlohzgqMd9CKgoDa2Yzqb0MaOkEbMwLNSj6CEuEwz24UwYVYPkZb9Ymb8LBpyPs2g6pQHKhBGtrzXDh2sPN9pArRErbZO6RoyDVJHLRaOchYmuakDto0LXu7tRMDGQ7ZD2IkZmzsm3FkAnOcvYqaYEkn1qCNd9Bq29NfQRu/wX6/NSsnSSx7K+nIDdu1v4eM044V0DR7YJpnN50FoxAklzEXOa8c1dfMcLEJlxrqNbLXSTQTShOE2CXqLUpMpp2bLW9BEGyLu5uPSVd7j/9MBzocsh/ESEziNXBgrKA1LPvYvljoZWxbgmUfny4h5URoXT8N2WXFNC+qSqbZC4KAUr8DBBGxfIyxrTlK9jlCBF+Btl0/TcktBVXlcPcure3oE0G2oV+7nlrmvWbZm7R56/boqyhYNXsbsk83xt7Sr0P2Dvo7kkkQAwHmOHuIJ13VJhGavWWftozTzdNQvE9A7yNe301cQSsg129HLBuJ4PIYc9PJHqCzcKxlLpHcUgBmeHYSFnMQioelNId0EV/7Nh3YRURlG8lCIGPXXPsXsSN7QSf79KlOivtMBzocsh/ESFZlaSDArNmb30NUQlGty/P1v4zQyzhtPeXiJfHvU7DsFUVNiNxIWEErgNKw05Bw9G2dqhdV1OrW+gqjln30VynnaIWzh0lttOWNzEpKYzvow8zEMrd7uso2jJtRDxy0GVn2jmbvYKBgIFv2uuqi/+DiE4915aC1s+JTnXt30pddbQBFVQ0iMuvLFgvT14waaEesGG8aswAIyLkl1MmFKJ4C03aI5JQYx7bnjkpp/JlAMFnI6Z+c6DfJNgzLPt5Ba5JZRAHb0Evj3J5Y9g7ZO+jv6E6S6M9Q4yx7Od6yJ3mKYztiT70sYddrE+Jr3Optx0sNkihYHv/Fph3adpNlr3NIx9hjecV/WELKBUV0QW4RAO15I1MafyaIyTiZ6NqmdrI1oPg+kpCu+QYriLFUxvH57CH91bPm/g4oy37ZsmUsWLCAyZMns3nzZmP7ggULOOWUUzjrrLM466yzeOedd4x969at48wzz+Tkk0/mkksuobGxMfujd9AlBpIlHw9zugRIlE0gzrKPvqqqvTM29ayXqb23liVUE5x58Za92rQLBAmxLGah6+TaNmoua0IH2RYcF/K1PDgd+b1n2Ytx/WaK3nPQ6q/xln1su9hNNE66q2fN7RxQlv3ChQtZsWIFI0cmWhf33HMPzzzzDM888wzz52tl1BRF4ZprruGGG25g1apVVFVVcdddd2V35A66hC51mDGg4uzjHbS2ZG9j2ZPEss/YQWtv2Vvj/hOjcaS4UD+1cQdi2SjDOauN2X4+RqlARUUoHUl1pATZnf3FVLFx9MSy730HbbehlyJWGcdm5XImDtr4tRMDHSldgaqqKiorK1NudMOGDXi9XqqqqgBYvHgxL730UmYjdJAR7KhtAHG9JTcOmKptmf62/Aa70ewzlnHi24mL8gFrugRzcq4YSagoDTuQKsZZ2tKJMr74h2jMRUWYs5g/tX+tV3LFx8ZhfU0Hdk9X2Ua6K2izHo0zSMi+x2UJr776alRVZfbs2Vx11VUUFRVRU1PDiBEjjGPKyspQFIWWlhZKSkpSbru8vCDjcVVUFGZ8bn9DJnOxy2FfUpLb59cl1f4L97UDkJerWcIFBTnGvtKSPADy871Ge6UNPgBKSvJsvzeFhTkp9e3NcVvfe92W8/LytPEoimpsFyWRHK+LiopCGqMFxnNz3BQV5QIwROqAYCfF4w+hyNRWcXGLZW4lxXlUVBRS1x4CoKg4lyHDS+lUc1Ief2aIkVq6feSYrldpaX6vjNHjkbT2476/+vaCghzcLolwdAVtWVlsHAUFXgByc1xpj60lEAEgJ+47sL+Q7T57RPYrVqygsrKSUCjEbbfdxs0335xVuaaxsSPtohOgXaT6+vasjaMvkelc9NqnZjQ1+6jPc9scvX+QzlxaWvwAhEPaD6652Wfsq2/oAMDvCxnttbZqxzc1d+IREr8zzS3+lPr2+0OW9z5/yHJeZ2cQ0Cx8fXsoFMEjidTXt9MWHUckIuP3aceOcWkpjH05lQRNbXW0B6Jj0+bW3u63tNHc7KMuR/uJ+jqDvfad1v2ZqmlOqSIUjBh/t7b6emWMOge0twcs7SvR77jfH0JRFEOzb2+LfdaBgPZ5qoqa9tj0z0GOyPudTzL93YuikNRI7lE0ji7teDwezj//fD7++GNje3V1tXFcU1MToiimZdU76BnsJJuB5LCN1+yt0TiJmr1oknF6MvfuNPv4+H19bHr/5mgc/e8xrkYQXYhlVp+XkRYh6lGOxZPrffVukrH4cWQm4/R+6GV8lFLidqKFxfXQy8RonExkHCfFcRQ+n4/2du3Oo6oqL7zwAlOmTAFg2rRpBAIB1q5dC8Bjjz3GKaeckoXhOkgVduQ2gLg+lvVSr0IUt2IVkiTwUu31+dQTodmPI/69dQVtIjGYc7GMlho156xkfaqKdz7HNHtzGGnc/HoBYg8ctFg+gywNKA7JQiDN1zyZZh9z0DrROCnJOLfeeisvv/wyDQ0NXHzxxZSUlHDfffdxxRVXIMsyiqIwceJEbrzxRgBEUeSOO+7gxhtvJBgMMnLkSO68885enYgDK+wdtAOH7Y3cOHHWr3mf5ScoxPZl1UGbxNKPT8yWvAatykipEanimIS+9PHraZiFOAtWT/imt9dbiA/5TAfWOPvetey7WkGbLBonFnrZgzj7QZIuISWyX7p0KUuXLk3Y/vTTTyc9Z9asWaxcuTLjgTnoGQa+ZZ889NKuMpJoOs9unqnXoI0bR5KUxwnpEuKsQF3GGSK2kyuELCtndRgyTlx0kXU1sHVbb6AnoZfmW26v58ZJkuJYj8aJP958jDsTy95JhOZgIMBWt97/w8gYuiFvhF7Kprh2vTKSJfQy9sZWxskw62XyRVXmtmOLqswEUVa3liO8W7XtppWzOnQO6SrO3rjp9aqMY33N5FzYD4uqEiz72Hbz9bHT7Htk2Q8Ssu9x6KWD/gl7y37g0H28ZW+Os+/KQWt2apqR8oKyFBdVJTho4yx7LwFGfvUEI3MhokqIpYkLEnVZImmcPbGbS2/STU8s+55WuUoFKaVLMO1y2Vj2Tpy9Y9kPWthJ1AOH6rteQRtPjhYk0+xTlnFSs+zVbjT7oeFYNFotZQhSol2lc0i8Zm+X52f/OGgzOHk/Omi7i8bRYbbiDQdtBoQtDDLL3iH7AwgDy7LXXl1x1i/YW/aCquAmgkISCSvDGrQJoZckWvZmGUcnnSHBPaiCRFB1sRf7PPSGXKPf2IztsXaVuG29gZiDNv1OREua6d510MZXFbSuoI1ttxYcj5K9y8ln75D9IIV9mt8+GEiG6MqyV20s+6ItL/DTohejxcntLPvMxpFUs09ScFwykX2waBR3tS7iv8Ic27YT0yVYbxjBsIw/uoqzNwnHeKLI6Fz7v7OJpJa9OfLJtM9exsngRjbIHLSOZj9IYb+waP+PI1MYcfZdyTim490tOxkutbBHUXoUZx9/XNKyhEkctKIoIKJQHKyhc/h86rYXM0LMwQ6JidD07dof/3x1CwW5bsu+3kBPLHurjNPbln0SB218NI6tjJO5Ze/IOA76NQa6g1YnQP0HHukm66XUWYckqBDqzKqMk7g/UU6K1+zLxA4kZOTC4dockpBgshW0+o8yHFFobg9a9vUGslaWMFsDikN3ZQkF0RoVZJsILQMZx1lB62BAYOBb9sllnJhmHz025EcKtgHgCrb1SMZJFldv914foxaNgzHeCklbWS7nD41us+/LcNDqN7YuZIPetOyTpSNIF701RiGZjBOX9VKHbcHxDAjbIXsHAwK2lv0AiseJL15iV6nKWGnaVmvsE4OtPVpUFX9YV9E5eux+fMHx4e5oGpGiCssc4mGOp9feW7eb0buavfU1HVjz2ffOGJOFQJpXHJu1eUtIbk9W0EYdv46M46BfY7BY9vaavXWFlNKyz/hbCrbvl3QJYC5mHhunIAicfIgX3LmIOVpJwWSWoeGgTUiXkHjs/rDs+2vBcbM2b4b5JqDvkuIeo4yyhBnIOABet4TXLWV0bn+D46AdpBjwmn2X0Tjaq2HZt9aiIiCgIgXbLAuw4tvrDt1nvbTu06N/zERUEGlGLR6GGCWe5Jq99iqr3Vv2vavZW1/TO9nczn520Iqx/bHslvY3hEyt8599cwZDS3MzOre/wbHsBynssgMMIK7vRsaxkqPSug8lt4ROxYMUbDNuBpb2slSpyiyFKUrsmupEpKoqSksNYvGwbokmXsbpanFTr8o4pqeSdLE/HLRJyxKaI6AE+2ut3xAyWUELMHFkMYV5nu4PHABwyH6Qwl6zHzhQleSWfXxcutrRiJI/hDYlFynUQwdtt+kSTH+jWpyrqqrgf/4O1I5GpIpx3Tr4dCJKSJfQZw7a9M/dv3H2cX0b1zfWd7w235N0CYMNzhUYrLDV7AcO3Rsyjq5rd5HPXmmvR80ro03NxRVs65GElZZmr8QWcEmigOprRa7+Ave0k3BPO6nbFZj6itOEdAk2NvL+kXF6puP0moyTLPTS7KBNYtlLSeSdAxEO2Q9SZGMF7a7adm7++4cEQpHuD84yFFWjEUOXt8tnLwiochi1swUlr5xWJQ9XqC1J1ssUyT7+fVfROKo15l9prwfANfpwBNHVrWVv3KwSNPvEY3szIES/uWTShSXrZXaGY9OHvcxkCb1MYsHrln6mDtrBBOcKDFLYR+Okx/a76zrYsa+dlo5Q9wdnGWp0oZLhxLRbVAWoHU2Aipo/JEr27Sg2on2mDtp2f5hbH15LQ7Qeqfka3v7IR2yr1uL7RVFAbdPIXizUQy67dtDGl1yMT5dgRv+17M3t9JGDVhRiZSHjLPgJI4pYvPBgJo0u6ZWxDSSkFI2zbNkyVq1axd69e1m5ciWTJk2iubmZn//85+zatQuPx8PYsWO5+eabKSsrA2Dy5MlMmjTJiEi44447mDx5cu/NxIEF2dDsdVLNpOh7T6GoKoKQ6MQ0/61Z03UAqPlltCh5CKqMGEgs1JxpPvt9jT5kRWVPfSdDinMtN4PaZj9bq1sBTT7QLHsBobDc2GZ+jUeigxbLq/XY1MafCYyVqBn0YY2zz9aIrEi6gtZGxolPi+CSRL42Z3TvDGyAISWyX7hwIRdeeCHf+ta3jG2CIHDppZdy5JFHAtoN4a677uI3v/mNccxjjz1Gfn5+lofsIBVkw7K3K669v6Cq1myGZsveHKqotDdoGwuG0KJo3zXB38J875fMz/mShzvmc5hnNyElMZ+8bb9x7/V+dV09/lqEwnqRawGluR4hv9SoNdutg1afmxxXcLzPQi/T78PqoO2dMQqmEEszzOkSYg5aR5tPhpRknKqqKiorKy3bSkpKDKIHmDFjBtXV1fGnOugj2NFzupyt82ufWPaKiiAIMSemrWUPansDCBLkldCs5AEgBZoZ56pnmNTGotx1nJL7KbnhppT6TXZD1Ek+fncoLGt9igJqez1i4RBjX3fFL+LLEhrROLYraFMafkboajFXCifb/ZlVSEmePARby94h+2TIyqIqRVF49NFHWbBggWX7BRdcgCzLHHvssVxxxRV4PIMjXnUgIBs1aA0Zp68se5PFZs2No71qMk4DQkEZgiAalr0UaKFM7ABgslszQCp824Gju+032X1Nt77jr2soopG9SxRR2uuRRhxq7EuWmtfYHze3rhKS7Z/iJen3sT8ctHMOGUau12XjoNVfYykSMkmLcKAgK2R/yy23kJeXx7e//W1j25tvvkllZSUdHR1cc8013Hvvvfz0pz9Nq93y8oKMx1RRUZjxuf0NmcylNSgnbCso8KbVVl6+dnMuLs7L2vVMtR1vjgtRFCkpiVrrrtiS9dwcTSYpLclD2tSMq3w4ueUFdKpeFNFNntqBW+rUzhM0Ih0e2plS38myI+ZHr53bY/3JCKI2rtJCN2pnCwXDR1IW7Scc0W4Q+Xke275l3YEbJagh5QVUlOXZ9l9amr3PIB46h3o9rrT7KCiIpW8eMqSQkkJvNocGaN+ZWVMrE7YXFmp9l5fnkxdd+JSb6x40v/1sz6PHZL9s2TJ27tzJfffdZzhjAUP2KSgo4LzzzuOhhx5Ku+3Gxo6MJISKikLq6xOddAMRmc6lqakzYVtbeyCtttrbAgA0NnVSn9tzuyCdufh8IQSgrU2LggkEwsa+tg4t7W9rq5/ypn24xkyntakTEAh5ilBa6igSfLG2FDdlvh3U1bYidJPXPBxKvEkCNLf4qa9vJxi0hqG2d2pjCTXsAVQC7jJjjvoTUSgUtp13czTCJxBts7mpE1G277+11d9r32ndsg9H5LT78EXnD9DU1EE4sP8itwJ+ra/WFh/BoPb9UCLKoPjtZ/q7F0UhqZHco2eeu+++mw0bNnDvvfdaJJrW1lYCAY0oIpEIq1atYsqUKT3pykEWkL6DNvraJ9E4RKNxtPdmzd5IkqaEUf2tCIVDjMf4sKeY3LadFnnhg+DBeNQgStPulPq1g95/gowT1exzAtGwy5KYBao7mMUkN5jYGgKrZm+H3ozG6UluHGvB8f2rl9umS3ActEmRkrl266238vLLL9PQ0MDFF19MSUkJf/jDH7j//vsZN24cixcvBmDUqFHce++9bNu2jRtuuAFBEIhEIsycOZMrr7yyVyfiwIpsLKqKOSX7QrNXLVqsnYPWFWwBQCwcYhBVyFNMYds2ALZHhjBCauG94CQW5G5E3rcFacjYrvtNEqCqdEf2fi0qSCwZbtkvmWLA4xE/t67Icn+UJeyv+eyT9xcjeyMax3HQJkVKZL906VKWLl2asH3Tpk22x8+cOZOVK1f2bGQOegTb0Ms029AJTu6zaJyY088uN4470AiAUFhh/PDbCydQ3vAJAP/qPApZFalXCukUC3Ht2wzTTuy6424ctPGXIhTV5T2+OoSCcgSXVbMWRSGF0EtrnL0d9kdunJ6voN3Plr3JAR7Leuk4aJPBuTKDFNlIcdzncfZiEss+Oh6XvxmwWvaNZdON42rlYmqVEkCgzj0Sed/mbq9BsrnKSZ5ydLJ3+2otEo6Oo6cO59CxZbZtGouq1L627K3jSe/c3g+9TAbbaBzHsk8Kh+wHKbJRvCS2gjYLA0oTiqplqI8PTzT/7Qo0gehCyCs2bEpVlNg2+SJe809FJhbBU++qRPW1oPpauuw32TXSrW+7OHsBFXdnPWLJiITzvnPKIcw4eEjCdrBLl5B8XP21Bq01n312xpMqzIvWeprK+ECAU7xkkCI7lr3+2jfpEsx+TbuyhC5/syadCGIs3bGq0loyiWf91jtUk6SlMFCa9yLmlybtN9lMDQdt3PZQRKFQ8CPIIcTiod1PzATD+Ry3grarY3sDPbHsrfns9y/bjxtexKTRJRTkuh3LPgU4t8FBimxY9vrNQe0DzV5VNfKJlzoglgFTCrYaxK1zjqra39SahCjZN+3tpt8kMk4XDtpiUQvzFPPLu2w7Hjo5xtI5d3HsfllUlf65Qh9a9hNGFHHtt2bhdomOZp8CnCszSJEVy74PV9AqimpxvOnWr7ZPexWDrQj5JQAWyz6+yhWAT8hDyClEac6U7PUVtFZSC4UVSqJkLxQkf2KwQ2Lxkq40+7SaTgs90uwt7fSdVe3kxukeDtkPEGzZ00J9iz/l423LEqbZZ8xBm+aJWYAazXqps4mcEI2jIgXaEPJKAKtlr9+kzD98RVURy0YhN+/ppl/77ebQS/NNRFHVGNnnpUf28Vkv+0qz71mlqr5z0NqNw5FxksMh+wGC3z7yMb+47/0uj9nX5GNPvZYTJju5cbRXuQ88tIpqjZ82P12oqkquEEJQwjEZR9+HadGVmZQVFbF0JEpzdZdPON05aPVxmVEs+lBFCSE3veXt8aGXfabZ96AGbV86aM2w+8wdWOGQ/SDC469t4ZFV0bUPtnH2mYVe2hXw7m0Yi6rssl6qKsWi9pSjW9P2Mo5oaU8qHwPhAEpLTfJ+k1wjOYllD2iWfW4JgpDezyk+62VfafY9WUFrzWffd0Rr+D0csk8Kh+wHEUIRhWDEfvEPZL6Ctq80e3O6BDVOximO5r7RNXsdZgetGGfZS5Va8Ry55suk/Sa17A2yt95EAErETkhTwgG7dAldaPZpt57+ODKJphGS/L2/oRskDtknh0P2gwiyrCRd1p9sW1fQCbYvcuOoWFdGxqdLMCJgopq9fpyKtQi4cY4KQtFQhLwS5Br7ld+QgoOWmGU/QmpGRKFY9KWt10Ni+ua+jrPPLJ99Yjt9gfgC9Q4S4ZD9IIKsqiayT9zfny37+hY/nabMlnrxEh2yEiNZjex1GadEe7WEXmp/xztSBUFAqjwEuWZTUlLv3kGrWY9jpAZ+UbySed5NlIg+hC5i95Mhndw4varZJ/yROnqSaiGbiJV27OuR9F84ZD8AkKplrShq0njwZNu6bi/6uh8M+1/c9z63/L+1xns1uqjKbP1+I+99Lsh/B0WFItGH4s5DcGnZVmNkH7vhxTtoAaTKSdpK2rZa23F0FWe/t6ETVZH5mrSGr+d9CMC8nE14BNlSoSodWOLUuzxuf2j2Gcg4PTg3m3A0++7hkP0AgDkapivClhWTZW+zP/7UcESmKZqz3g6GZd/LbK+3X9ccCy1VoouqzDLOWKmeQ9zVKLJMsehHzSk2jo85aE2RGaYFNoa0U3kIAJEkUk6yme5r8vGrv65Gbd7LMeJ6xrvr6VC8DJPaiKgi7vGzM5i51TLuqzj7WEKx9M/Vdf6+Nqj1r6jD9cnhkP0AQESOUZCeeMsO3Vr2ce/vfWoDV//5veTt7Sey7/CHE7bp0TiYLPtC0U+BGMSr+LRwx9zixPMw//DNln10W0klQm4RcrW9k1a/bOViOzcV/4cKsQ2A1g6tUEZRVD56yX84D3UcB8DHoXG4Cu2TnXWHroqMW47rRRZLdQz251pf+wqG38Nh+6RwyH4AIGJaPeqPq5RkhqzEnJP2mr1146dbtRTByTT5/bWCtq3TWt3IF4gQCMmIgsl5iEyBoD2FlMoNFAs+1JwS4xzDQZtExtHnLggC0vBJSXV7fdsoqYlSyWfUsA1E89brFbBWBw9ia2QYz/pm8VJgVsYyhpgiWfbXrJf05NwswtHsu4dD9gMAZsveH4wQCEUsNwAdiknGsSPoZAZ6MmlofyRC8wcjNLVrpe3yvFpevt89/gk79rUjCALufRs4yLWPQjEmN5XJ9RSJfotlb3HQYg3bFAXBMgepcjJqZxNqZ3PCePTDdAv+IHct5+atxh3utGxvU3JREXgtMI12IfNayakuaNov+ewHsoPWicbpFk7WywEAc14YXzDC9X9ZzZSxpVzzvzOtx5lkHHsks+DBLn/U/gi9/PVDHxpMkZ/rQlFUdtVqq4BFATwbnuGMvBBPdB5hnDNS3oMkqERyS4xtlkVVSlyOc0mwzEGMVqtSmnYjFljlF31RVWGU1Gd6dgLQouTzWmAaRaIfv+ohYkqf3JN8LLGc7F0f15sZJWNPT5nIOD1YfZtFxBy0fTqMfo1uL82yZctYsGABkydPZvPmzcb27du3881vfpOTTz6Zb37zm+zYsSOlfQ7SR8REVF/taQXgi52JVqmidm3ZJw0rTGrZx1IEdIVMbwbBkExdi99wzObluGlqC1jSCYvBNiqlFoqicfWdiodRSjSZWZ5VsxeIOWhFUzlASYyz7MtGASDb1KTVp6KHduqY7tlJseCjWPTjE/It+3piTcYcnH1p2Wfeh35KXxvUjozTPbol+4ULF7JixQpGjhxp2X7jjTdy/vnns2rVKs4//3xuuOGGlPY5SB9myeadT7Wl/mOHJeZhkRXVVFEpsZ1kZJ9UxknBsm9qC3DZ795k5772pMckQ3NH0PLe6xKpNSV7a2zxQbATrxBhvEsr6P1FeCRuNL+FarLsARB0B601iZpLEi0FWARPHkJBOUqjTVI0VfuvUDCNQy5grKuRm0v/w3TPLiKeQu5cMp9jp2vFSqQepNVN1cG5P2rQDmgHre6Udxy0SdHtt7SqqorKSmu5tcbGRjZu3Mjpp58OwOmnn87GjRtpamrqcp+DzCCbNPvqBk07zstJVODMmn13cfbmv5PlOTM0+y7IvrEtQERWaWhNPSOnjua4sE9ZUS3hl77WFoSorDLZXYMKfB4eFTshLhpHFARDxjHnwo+37AHEstEoNpb9GLGWP5b9g6mevWwOD2eV/zD+0nEC9XLs5hqQCjhkbBket2i0nynEfqDZ94SwjRtFH6v2qmPZd4uMTJKamhqGDRuGJGm6pSRJDB06lJqami73OcgMds5YO21ellNfQRsKm2L3k2n5KaygDUdDQTMpSq47ZnVEZJXaJh+uqAZeYHLKjnY14SOXHZGK6JhAsAu9VLXxukTB+HK7pESyl8pHo7TUoMrWsM+DpGrj70a5gBf8M6mRS7m19Rw+D2lPtwFJc8jqJN8Tsk81VUF/LUvYb2QcU7SVA3v0awdteXnmUQ4VFemlm+3PKCjMsbzP9UoIopAwRxXNQVtRUUhBQaKmn5PrNs6pa/IZ20tL8yku8CYc73JpdJmT4056PfPqo08a+d6Urrn5mJBiNQAEUaDVF2ZkRQE797VbpBTQCpA0Kfn4yCGiwsiKEipKc439oiiQm+tGFQRcLhFPNLrH43YhByOWvjvGT6buk5Xs3baVG56o5u83fI3y4lzCaszx2qbG2gbYI5cxlb24XNoxhQXa5+L1SBl/3/QbmyiKXbYxdGhhr1VhEoV9AOTnpfYZmlFSp33+3Y2/tzFyaCGfbGlgZGXRoPntZ3seGZF9ZWUltbW1yLKMJEnIskxdXR2VlZWoqpp0X7pobOzIyPlXUVFIfX36GnJ/REVFIQ1NnZZtEyqL6AhEEuaoh2jW1rXR1hbNHSPELHqfL2Scs8uksdc3dBDyW2PdAYLRmP7OzlDS69nQqI2tudnX7TWP/1z27GsjP8dFWFYIhRUCwQh7atsZWprLt0+aRGGtAGvhw+AExrnq2eoZDwjsUitxyT7ymjshElt3IACdvhC+aI6dSDQ2HlQismLpW/EOA2Djh2uBEXywfi9HTBlmxPKDFl5pxuZwJSfnfkYbmhES1HP5qGT8fYs9cKhdttHY0NFrerRuDfsDyT/nZGht02/IXY+/t3H6UWM4dEI5w4u8g+K3nymHiaKQ1EjOyFQoLy9nypQpPPfccwA899xzTJkyhbKysi73OcgMcpyM43FLCdsgpq2vfHcHD0fz2scnA9PRbiL3nsTZh+UeyDhtAcqKcrj8nMOYOKIIWVHwhyLk5biYOLKYEpc2xid8c7i19RzWeo8G4Gn5OB7sOD6xQQFQtbG4RLNmLyYYDULBEARvAcPUOkALaQUoMD1NmK18gK8iw7mj9XR2FGohr3rIZY80+24WJUli5hJLqohJSel3Yow/mwPKAB63xPwZI7s/8ABGt2R/6623cuyxx7Jv3z4uvvhiFi1aBMBNN93EI488wsknn8wjjzzCr3/9a+OcrvY5SB+6xX7ucRO46eI5SJKYQK6qGoux37Sr2fjbYg2aTjGnKEj29JRKuoSeaPbN7UFKC71Mm1BOZXk+EVklHFFwR+UK1d+KKkr4VS3ZmS5jBFQ3PtWboHML0cVTihIXemmj2QuCgFgxjvJwLRIy/kCU7E0hl/vkkoQx75XLEMSoP0rPKZMFzT4Zz6bqwO0JehR62U/i7B10j25lnKVLl7J06dKE7RMnTuTf//637Tld7XOQPnQH7YyDKxg5JB+XJCQ4bc1cFjblzzFba+ZjOnwmsk+W7jdK4HIKDlo7J7LRvqJy49/WcOGiQzm4MqZDNrUHmTCiCNC0a1nR5Bad1BV/O3gL0e1GY3sSZ5z+TlZURFE09ts5aAGkivEU79nA3WUr+GLfKcgtHoqEAJ+FRrGicy5+NdGPofUbPT+6gqcni6rMq3ztYC7N2FvoSURNf3HQOugeznqzAQA99NJlkg3iLWnze3OyNIuMYzLtzZZ9d4ut1B5a9oGQzN6GTu5a8ZFpvAod/jBF+Z7oOEVkWSEcUXG7YpY9ObGbgz7/ZMU+dMveXsZJHJdrfBVteaPZFq5gSt1L+P71S0a6mmlXcpISPcRIMSbjZP4z6i5VgWTK/NlbcCz7AwMO2Q8A6FazK0oqLkm0xN6DVWoxk30sfa1gkXHMxyRPhJZ8f4c/zO8eX2fE13dl2euhneYnjg6/JpsU5kXJXhKIyFbLXvW3QW6RcY5h2Scp9qGvoNVlHH23SxJs/RLSkLF8MuG7LG8/mb+2H29sb49G4SSjL6PdLIZeJrOqRdNNq7cgmL8j6Z7bTxZVOege/Tr00oEGPV2C2bKPJ1ezZR2OyMbfoklXNhOe+fzuNfvEfbvrOvh8exPtnZrnP/7mk2xsOtp9mvO1MM+tzUkSCEWjZ1yuGNkLRbEoLt2Sjsk41jYFk4NW0+yTO2h1RGQFBZHPwqORVQFJUGlXtJBK0eYJSuvHqtVnQ8aJn8vPFs/AF4jwz1c293qO9qzE2fe5i9ZBd3As+wEAnZj1ZfmSmOigNVvf5gVTMSeiNQ2a+fzuZBw7y16PBuqIhh92JePYEW171GegW/YuUTTG55ZEVFVNsOylqLWu33ziCUYQBJToWgPJZNlLorYO1866j41boEHRJCPdsk/meI3X7F09kXGSWNVTx5Ux55ChSFLvW/bdRQR1hZ7cKBzsXziW/QBAvGbvikoelmMsln2ig1ZPJfDlzma+3NVsscQzyWevh1zqjt7uHLTxsLPsdbhdIoT9oEQQcsxkLyIKMWs7mWWvGGQfu176PKS4k8zXrVYuZpjUhhq1gZKRvfHEkIXQy+4qPWnZOzNuPuVRdDWGLs90ZJwBA8eyHwAwNHspFv0hx2krVs3eJOMIAoWCn0qpBUWFjzbVs2rNbkucfnIZJ/l+/WYRSsFBay/jxFn2ptWhLklA9WkVogSzZZ+Cg1YPQTVb9nk52g3FF0gs/GK+6f2n80jWBsezRdYSnCXTsM1PDNCz0MtU4uwHhGXvyDj9Hg7ZDwAYMk70V+kSRcMRqcNM3mZDXBQFrix6iZ/kPgWKSkRRiMhKSjJOV/nsw/E+gxQte30uumVfkOuyzA2iWSoDNmQfJb6uQi8VVYv0MTs2y4s1Db45LhcPQMR002xV8/hH53wionYDSsbh5igf6Jlm744mU+sqzr5fa/aOZT9g4JD9AIAcJ0vo5GK27pPFwkuiQIWkLbsuCDcaRG9x0GaQzz5etomkaNnrIZ/tvjD5Oa6Y7m2y7N0uEdXXCmCRcVyiaCG+xEVVAGpUxomV4hhSpJF9fOI1sHcsd5fgzLDssyDj5EefOpI9RewXyz566TOLs3dCLwcKHLIfADCHI0LMojTr9smkGFEU8CkaoZSH9ibIL5AC2du0neAz6CIax3y+Lt+0+0KGhAOJlr3q1yx7MS9OxjFxSoJlLwhRy94q45QVaTHzdpa9fsMcVpZnbDP8HN2QfSz0MvOfkV6KsSvLfn8tqsrknuVY9gMHDtn3c7y+djfbq9sMJyOYLXuTjJOM7AXBWCBUHtprWOTBUEzXTxqN04WDNjH0M7mMI1vIPhR9DRvOWUh00OpkL+TGFlV1Z+UmC70sKfAiCgLN7YGEc2RZZWhJLj/7xnRTO92QPVaLvicyjl6XILlmL/a61dyz4iWOZT9Q4ETj9HP83xPrCYRkY6UpxCSPVGLlRVEgX9Qs2iGhvYZFHgjL3Z7bVSK0BLLvyrJXbSx7f5jhJmvaHL6oW/aCtwBRin1FpTj9Ol76ENCLl1gXVUmSQEmhh6Y2O81eRZIES7Upw/HanYPWCIXtCdm7u9wv7QfLvmcraKOv2RuOg16CY9n3YwRCEQJRC9xi2Ud/nWaCTWbZuwWFHCGMX/VQLDfhiWgWcyhstuy7Dr20S5cQL+N0FXppb9mHklv2OtnnFVkISJJEi65sF3qpkpj1UhQESgu99jKOrCCJguX6mhei2SFe5ulJnH1+1LI3P2lZ+hJ7P13CYChL6KB7OGTfj9HWGUtDbCYUg+yV7i37/Gh+9o3CwQCMCO1ERCEYljku50t+UfQsShIJRr8J2N1IIpF4GSc1zV5PJewPRgy9WpuTybJ3CRrZ5xRZCCjeyk3U7KPZP2WrZS8IAqWFOfYOWkVFkkSLTyS28lYPK7Qim6GXuoyjX5d47B/LvgeaveOgHTBwyL4fo60zlqzMbPnGZJzuLfs8tNw1uxhJSPBwQuAVri1+llAozNfz1jDC1YIQaLU9t8toHCV1sjfv08csy6pFOjFb1p5QG3LDTsSSSgvRmjV7O2rRZBxt3KLFsoeyQi/N7YGEpxhZVnAlWPbR1+j5emI2o5+4m0E2onECIXuyF0Wh12PYzTfFjM/N4ngc9A4csu/HaDVb9jaasplEk1n2uWiWvU/IpdozDoBhUhvT2Rxrr7PB9tyuEqFFIqnLOPHrARRFS43mMpGk+WaWs+VlUGQ800+1WvaS2CUxmWUcs74viALFBR6tGlacXKIfa36yEONkjUSyj40nfuzpQpdx4mUxHZIo0AOVKCUYlzIjzd6x7AcKHLLvx2jzxcjeLBXYOWiTxdnnRi17v5DLB/kn8S/xLBQVzsj50DhG8iUh+y5SHKdi2a//qoFgSE6w7HX5yfK0EmU0L2FcO97HPekYxKKhlvYkMWbj2nGLvoI2Pl2CKAjkeKLaeNhK9pGojGO+vmKcxZ7Mss9G1svUHLT7R7PPKOul0UYWB+SgV+CQfT+GWbM357uxC71MatmrmmUfIJeA4GU7I9gpDyFXDLNaPRxZFXAlI/uuQi/jNPt4y761I8gf//Mpf/zPeusKWkUxrFizNa3P6TDPLgQ5jGvy/IQ+LTJOMsveZgWtIECOR6sulWDZy2qCZR6vxSe17LMYZ58M+yPFsfEkk8G5joN24KBHoZd79uzhxz/+sfG+vb2djo4O1qxZw4IFC/B4PHi9Woz31Vdfzfz5iT9gB8lhJnu/yYGnW5TmFAV2oY+HuvdwdOAtAIKClwI0Ul4dPAgBeN9zBOPD28nrxrK38992lYgNoDOah+bLXS0snD3aNE7FaNfuaaXKuw0KhiANOyihTy0DpPZ3V5a9Ls0YpRkFgRy3RvbxUS+aZm8l61i0jfbe7ZKir2LcTbfnMo7uoE2GipLchBtUttEzzd6RcQYKekT2o0aN4plnnjHe33bbbchy7It5zz33MGnSpJ50cUDDTPbmH7xOMuYUBXYyyum5n2AkDo7m05FlhfeDk3g/OInxlV4aAkVM8DcmnKuqqrHYyjYap5s4e7/J4binvsN0nmoca5Y/8je/yMKcWia7apAOOhNBSLSWXaYFRrZkT6x4iSSKxtytlr3VEarfGCztxMXZ6zVxvW6JcEQxnlTyvC5yvS6GRHPvZAJ9XMnwjRMSb3rZRlZy42RxPA56B1lbVBUKhVi5ciUPPvhgtpo84NHqC5HrlfAHZQImy16yseztpBa/qunBH4QnI3i0BGJmi9wtidQrhRzi34aqqhbrzNycXRx+Qm6cuPeBYOzmtHFHk+U4/eahW8RqoIPcTS9yZh7IqoB36gkJ/enzTslBGxd6KQoCOVG5JN5K1hdVxfdjftVlHI9bBH8sOsnrkfjjknlZqVTVlzCnwU4XjmU/cJA1sn/99dcZNmwYU6dONbZdffXVqKrK7NmzueqqqygqKuqihUSUlxdkPJ6KisLuD+pn2NfYSWlRDt6o5NAZiDB6WCGbd7WgEptTa5RI8wtyqKgopLUjiF3gXr4QYlfOJFb65zHcLeHxuCw3hbw8D/vqSxDlEKVuH+7S4cY+s1whiELC9RSlRMvbfIy7pt34W38C8bhEJJdEcYm2cra0OJeKikI6vtiAbvuvCR/Mt8aNsb0+ZWX5sTTPNmNyuSRcLgkVKCrwIkZJeujQQhRJu6ai20UYgREV2ndLBfLzPJa2vFFnrjd6g8iPLv7Ky3HT1BYkN+pUzfZ3rK++s3uaNCd+UfTzSAfhqE3v8Uj94jfXH8aQLWR7Llkj+yeeeIJzzz3XeL9ixQoqKysJhULcdttt3Hzzzdx1111ptdnY2JHU8dgVKioKqa9v7/7AfgRFVbni929zzvwJnDRH07hb2oNMGVfG5l0tAMac2qN1X5ubfdTXt/P7f63ns22JUkyeGKRW1dIsyBGFQDBsSYCGorIzMkRr+8tPcR+Ub+wyr7ANh+WE6+kzFSx3RUsKfr6ljoriHARBoNZ0vH6s1yPh84WMtjo7g9TXtxPY+CGqO4ef156D5PHwtSSfXUd7ILaqVyVhTIqs4Ato0pc/ECYQ7behoQNfVBLT00/87sdzKS30Eg7LCfPTn5jk6LXK80gIQHG+hz1AW4e2OCub37Fcr9Rn31ndou9oD6Q9huZmHwARm+/I/sZA/N0nQ6ZzEUUhqZGclWic2tpaPvzwQ8444wxjW2WlVjvU4/Fw/vnn8/HHH2ejq0ELWdZiwPUUwIqqEghGqCjJTThWlx106cQuDQCo5AtBQmIugmCKVDHJLS6XSI1cgiK6kWu3Ws42PwF0lxsnx+OizRfml/e9z2fbNMnGLOPoNxivWyKiqDEZR9TKD0Z2f4Y67BBCuHG5ktsfkkWaSdwvCEKsqpf5WDEWeqnLOPsaO4HYoiq9fXPbugN5WGkev7t8LpNHl2jXIwMDpCvc+9Nj+d2P52a1zXSgu0cy0+wdGWegICtk/9RTT3HcccdRWloKgM/no71duyupqsoLL7zAlClTstHVoIWupevEGgzJqMTisEdVxKxu3UGrk6bd6ksPEVyCQlDULG09lUC8Zq8g4isYhVy3zXK+FoGjki8Ekkbj6Bah7mRUgfqWaFy/aUz6U4LXIyGbNXtRQKn9CrWzCXHMTMC6eCwe5nqstuQixG5CltBLwOuxtlvTpFmkeroEfTzmVyMkURQoKfAa27NN9rlel3Ez6gs4cfYHBrLyDXvqqae4/vrrjfeNjY1cccUVyLKMoihMnDiRG2+8MRtdDVroBBgjcI0g83Jc/O7Hcy1RG7olqhOb3yavip7pMiTmIaD9oOPDJXXHY2fBaApq3iVS/QWuEdpNWVFVZnp2cFHBO/w1cmZC+xFZobzYS31LgNJCLw2tWjy/nugsEJQ1h6ZqtuxdCYuqwl99AJIbccws4EMj8sUOUjfROKIQu2nqDlrtn4AkCHhcojGW6obO6DzUhLh6Ie59/P5k+f8HKsQurml3cCz7gYOskP2qVass70ePHs3TTz+djaYPGOgkr1uNOoHn5bgoLfRajjVb9qqq2sZh5wk62edohEdixIxO9nXDjqHS/xX+5+/CM/1UPHPORVFVxrvqATha+BT4uuXciKwwdngRl3/9cD78so4te1qBWApjfyhCrsdFKKIYc/F6JPyBcCz0UoDI9g9xjZmOlKs5bV2ursjetILW9gjBkKlcokCux0WuyWLO8UgJZC8rSkLFqfislvHRKl3lARrYyCQax/rqoP/CWUHbT6CTvE6Eugxit5zeZWj2KuGIYks++TrZS7lRGUewRNhAjOxDrkLyzrwe14Q5hNY9h1K/HVVRKRI0SWaKuBO5pdpyrlY9S2BUicTUfc9SLkadx1HL3h+MkON1WcISvR4patlr483x7UP1t+EaO8Mg3C4te6nr0EtRiEX+iKLAwtmjuO6C2Zb+dVQ3RmUcWTUWVSVY8nHa/WC37DOrVJX56lsH+xcO2fcTGBEgUSLRHZy5NsvpzSmO/UlWV+aJGumGxFxEQfshJ5B9lFhVVUXw5pMz7wIQXYS3rkZRoUJqY1eknBAugh88jtIeW2kbkVVckkhk5yeMaFnHL4uf4dqiZxnfugZVVfAHZXI9kiWbpNctISuxm1Ne81fafEYciihoVrvLlZw2JFNu92QraM2afa7XxYghMV+HWRdv6wzR7guhEnN4x1vy8aTfW5p9XyPmoO2JZe/QfX+HQ/b9BDEZx6rD5+cmWvbmGrSBJHnQdcseT56x3D+ZjKNzl+DNxzX6MAJfvsO7D/2eSqmFHcpw3gwfjrxrPZ2PXk3zzi1GWy5JJLL3C60tQSFHCDEv9A6R7R9pMo7XFYuLR2Ze63OMlPcYmr23aQtC8XDEgjJNV5fEbiz7rrNeImC7OleHbtnrVb/0rKL6sa54zT6e9G2yjQ4G6Os63O706cDR7AcOHLLvJ4jEOWgNGcfOstcTocmKJerFDJ3sTzpmCj84c2qXMo7ZUnVPO4mI4Oa4nC+RBJUminkjdBg7plyErAoEt3ygjVdWcYkg7/2cxpKp/Kr5f7i59euEVQm59isCwQg5HsnwLxzj3cwY30bm8ZGWfAwZb9NXuEYeaplXV9E4Ghl34aAllo3TLjmZ7uSuKNHSG3RGw1ylZDJOHPkPVst+XGURS/7ncKaMKU37XCcaZ+DAIft+Al3G0YlEl3HsEmWJgiZnyIqKP2iVcSRRwE2EcqmDgOqivKyAscMLEexkHFdMxtHhGnkoqyf9xHjfTAmKAnV5B7E5XIm3Zl00hFNhjO9z1M4m2osPok3NQ0GkWi5FbtiJPyhT6gpwAc9wiHsvJ+d+hiJIjBdrEDobmOTehyCHcI2ZHutb7JrsLTnq7VRiU5y9nWWvyzj62gV9TYPLkHFE4/pq76MkH+egHWxkLwgCMw4aklHFLSc3zsCBQ/b9BPGhl7rFbqfZg0ZQso2MI4kC5+e/x1Her3ATuxGY9WwdumSic9cbn+zl5//3Hr6QzGOdRwHQIJYjKyq+QJh14bG4/I0o9dsZTQ3Ta59FHDqB9iGHG23ujpQhN+wkFAoxr+VZRlPD4rz3KRQDbBp+CooKhfs+4jD3blTJgzQitv5CksSEdMKWuUmphF7GNPt46JkvK4o1sm/XLfu4OHujUlWchT9YZZyewJFxBg4csu8nsAu99LhFS+k+MyRJIKIkyjguUWWGZycArwamGdu7s+wjssI/Vm2ioTVAc3uQ94OT+EnTBYSkPBRVxReMsC40FkX0EPz8VaZ7diELEnmLfg7ePKPNPXIZQtjPAvFDyoN7aBGKKJV8hFSJ+vJZ7FYqyG/YyDTPbuThhyK4PLHxSCKebkIvdRMyGbdEurTsNbIfkiDjJAm9jLPw9aesAhs/yoEKJ/Ry4KDvlu05sECJt+yDsiVGPB6SKCLLVhnnpJzPmOLdhyioPNh+PJ+Gx/A/0X0CAvH2aMxBq/LRpnpjux4rryLglrTUyL5AhIDqoXnobMq3ruH4nAhNBZMocedYpJc9kTIAFuR8Tkv+eN4PH8KpoRfZKo9AdHvYGBrJ2M515IoQHHeUZTwXnjKZsrg1BdY5m2Qc26yXsTh7W7L3amQ/NKmMYyX9ePKfNr6MC0+ezFFThyUd44EGp+D4wIFj2fcTGKGXphQIOV1UMZIkQQu9NMk4p+d9wkSpBkUV2BIZbjne7rdodtA2RJOrQSxWXu8HYsVIaofNBTWak6dEk2DMxLpHLmfPiIWsDk5k1/iz2OsZR71cyCfywbglkQ2hkQDUy4UII2PyD8BhE8oZWZE802l36RJEAeOGZivjJNHsdQetFOeQjY/GEQSB42eO7NPUBv0NjmY/cOCQfT9BooyjxanbIfjJSha7XqPMt8OQcVzIhvb+ZXgEftVjOccu70kszt4q8eiWPcRy1ejE2OEqQTnrNzzvm0HrEM25qpOlxy2iIrC5ZB7/7JyLWjAM1eXl1tZz+JIJuFwie+UymooPYaVvFqLUdeGOeIhC14nQzHc0O8v+qEOH8a2TJlFaqOW56fBZZZwEh6zoWK3dwZFxBg4csu8nsAu9tHPOqqpC6OOVHCpuZ1rbO0bUzjCpBVGA/8gLuL9jQUp9ukwyTlKyjxKerm+HIwoRbwkvBw5H8Gjaty6D5MYV9ZYkwbI61SVpN4MNYxazPjw27XJ+giDYR+EY+2N/24VelhXlsHD2KARBIMcj0RHQHbRWkjcSg4l6Ww6TJYPjoB04cJ5H+wn0kMFY6GWEQpv0xmp7A8ghgrgpDdcRCIbI87oYobYAUE85IHDVN6cblitYiVBPd+yKLlKKJ3tz1I7uIG43yF42Ytn1JwPdAs7xumjtDBlk75JE40YgmcIqjf09CPWze1Ix3wi6CyPM9bro8EeMcZrPUYnVrtW2pz3MAwZOnP3AgUP2/QT6qlI9XYI/KNta9nLDDgA+Ew6lSl1Pjq+OI/NrWKS+j6JCi1iCQJCp48os1pb57xyPVupQTz+gqhCOC8vUYWTG1MleVohEYlkrIRYBkxd1gIZCMTKXTESqP0kEQ8kXPnWHLmvQWiz7rtknx+OiJVqExHDI6mUSVWsbmaT+PVDgWPYDB47N0k9gxNnLMQetXTSO0rATRIkvpEMAGO7/ikW8iVtQaFIKEETJ4sjUYX6rL4/X49YVRbPs7QgylodHG1c4ohjkrlvE+lOBUSDEIuPELHt3lEzNMk+6iFmSdpZ9DN1b9pLJQWuVcfRFZmOHF3H8jBFMGFGc9jgPFDgO2oEDx7LvJ1DiHLSBkIzHk3gvlht2IpaOpKWzglDEQ1V4NSLwcMc89skliGWCrcVsJketxmoIlyggijEZpyDXbeSL0RG/ojUUUQyZxyB7Obra1xuv2YtWyz76dygiG9vSRVfkYp5jd5a9+akpXsbRHd05HokLTzkk7TEeSHActAMHjmXfT2BeQatnhvS6rNEqqqqiNOxEGjIWSRLZ4joYH3l8lDuPj0IT2CuXIQmCLdGZt+grSS0yTpTs4+GKs77DFrLX9s08uIIjDx3G/554MGCVcVwm52dMxpERhJ7JI+XFOYlzTEvGiV1bw0EbJf3RQws4YspQRg/NvOD9gQJHxhk4cCz7fgIjN46qEgpHHaBxWQjVzmbUQDvikLG4agRec52AX5GpLMgDtEVRoijYyiMWGcdjI+PIipa4TBQs6QDiV/CGI4pRLEVvx+uW+MGZU7XjRYFgWDHOddlZ9mE5I70eYPs+LW9+1SFDbeaYnoNWR3ycfa5H4rKzptme58AKIe7VQf9Fj8l+wYIFeDwevF5t5ePVV1/N/PnzWbduHTfccAPBYJCRI0dy5513Ul5e3uMBD1aYQy91DdwTZ9krDTsBkIaMQ5I6iSgRrWC2iZDNUS9miHEOWtAsWjEajROJaO143RI+00Kt+LbCEcXQugvt0i9LgqHZu6TYU4Y1GkfJSK8H7akAYPrEIQn70rHszf6QeEesk/smdTiW/cBBViz7e+65h0mTJhnvFUXhmmuu4be//S1VVVX8+c9/5q677uK3v/1tNroblIiFXiqGph2fJ0Zu2AGCgFg2GknchCwr0VTDVovWluhMmzxRGUfT7GPROHleF15PPNkLliZCEdkgezvZRxJFo8C4S7Rq9m6zZZ8hOSxeeDCNrQHbbKDmFruVcbymmr5xZQkdsk8Peq1fB/0bvaLZb9iwAa/XS1VVFQCLFy/mpZde6o2uBg2M0EslJuPopGwc07ATsaQSwe1FkkRkRSUSraF64uxRLJg1EikJ2esx6GK08DaYZJyog9btEhP6ND8RjKjIJxK17EVBsA0NdUmCYX1r+elNlr2u2YfljC37r80ZbfgGEuaYhoyjh5RKokBxgfZUevIRoyku8DD9oMSnBgfJISA4ZD8AkBXL/uqrr0ZVVWbPns1VV11FTU0NI0aMMPaXlZWhKAotLS2UlJSk3G55eeYOsoqKwozP7Qvk5MbSG+QXas7HIeVaST19Ljubd5E3dhoVFYXk53pQ0WLCC/K9/PBcLXXBr+57D09QTph/btQKd7lECvI1chs+rAiXJOLxuFCi7eSbFmIBTD2ogi92tXDFeTN49OUv8QUiRFSt2tPQoUUJ83C7JPxBLX59aEUhxdWaxp7jdRvEH5ZV3C4p7c+ou+NzTfV6hw4t6jI7ZdXUStZ8UcfPvjWbUdHQyoqKQh75dWXWxjOQ0JO5iCLk5nr6xfXoD2PIFrI9lx6T/YoVK6isrCQUCnHbbbdx8803c9JJJ2VjbDQ2dmRUKKKiopD6+vasjGF/ob09AGiaeF107H6fRpr19e0o/jbk9ibCBSOor28nEokQCsuEIzKhUMSYb0SWQVUT5h8MxmLKw2FNpmlq7ABU/P4wgWAEJSITb3CX5bv59cVzAFCjee0bmn3keiXba6zllNc+s9YWHwF/yBiXrtn7gxHyvK60P6Pujg+a0j23NHXiT5JbCKCiwMONF81JqV3b8wfgdywZsjGXYCDc59fD+Uy0J9pkRnKPZZzKSs0S8ng8nH/++Xz88cdUVlZSXV1tHNPU1IQoimlZ9QcazInQDBnH5KDVnbPikLGApo1HZBU5Wvhbx4TKIiaOTFwEpEschXluTUsXBSPXjKJq0Thul2gsuNIhWrR+UXPQ+sK2zlmwRu+Yo3HMoZehHsg4XcEsJWQSw+8gMwhC4iI+B/0PPbLsfT4fsixTWFiIqqq88MILTJkyhWnTphEIBFi7di1VVVU89thjnHLKKdka86CE7qCVFTXmoDWFXsp6JE75GECvVBV10JqI8+z5E2zb11Mhzzx4CCOHFDBhhCbBmBdVuWzIXrfSQcuFE47IdATCRk74eJidxS4pFgZqdtDKippWcrGJI4oSfAl2SMdB6yB7cBy0AwM9IvvGxkauuOIKZFlGURQmTpzIjTfeiCiK3HHHHdx4442W0EsHyaEnF1NMoZdui2W/A6FoKIJX0/F1y15R1ZRi1j/d2gho8ekTRxQz73Dticy8qMrOQWtOkOZ2S9FFVWEmjkjU68GaAkESY1kv40NC0yHj6y+sSuk4IwwQx7LfnxDoOhupg/6BHpH96NGjefrpp233zZo1i5UrV/ak+QMK5hW0eroBryn0Um7cbVj1oFnNIVM8e3dYOHsUL63exYRKK0kL0cLlEVnBLYnGQqn5h1fyzqc1DDGtVHVLIqGIQjiikJ9ExjETuksSLatTXS4z2Wc/EEy/Cg7R7184lv3AgLOCtp/ALOMYln3UylYVGbW9HnHCHON4SYqVGUyFOL9xwkH8z/ETE7RVUYzdNDTNXmtrwaxRXHzaFMuxHrdorJ4tzLUWRzHGJcZZ9nqcvRBn2feKZm+Nl3ewfzCkOMdiFDjon3DIvp9Aj7OH2CpRPR5e7WwCVUEoqjCOcYnpE6ddLhpRIEb2Ukyzd9sU/nabyDo/1/6rYzhkow5gS9ZLV2YyTqowct07ZL9f8etLjujrIThIAYOO7Jv21eAWIwy0qZlDTPVSg7qDVmlvAEAsjJG9meDt0iOkCkGI5bJxu0S8imTp2wwzWXdn2ccnFxN7oNmnCp3s7VbXOug9OJE4AwODLutl9XP/x9aHfmXkJB8oMC/RD0QLi+jyjGqQfWxlp1m6yaTikw5REIzoH5dLpLw4B69bIj8nUZM3k31Rvj3Z64RuOGYtlapibrzeIPud+zoAOH7GyKy37cDBQMegI/vN4kGUBmuI7Py4r4eSFmTZatmbLWulvR4EAaGgzNhmiXrpgf4tilg0+6pDhnLnj46xTYVgJvtk6X9jFr32qt+IxKisU5Dnju7P/ldv1FAtUmnh7FFZb9uBg4GOQUf2O/Km0SyUEPrwKVTVvtRef4TFsg/J1rDLtnqE/DIE0VRwQ+wFGUeSEAUhaZoBfZFXWZHXVtOHmMVuWPgmDR9iTwS9YdlfePJk/rBknu2NyoGDAx2DjuxzvG7+K85Bad5DZOuavh5OyoiYHLSBYMSS8VJtb7BIOBC3UrWHMk7QZNl3Bb1ObXz4phnx5B5fBaooz2N5n024XZLRvgMHDqwYhGTv4pPQOMSyUYQ+ehpVGRjWvVXGkY3FTaqqorTVIhRai3VYQhx7YNlbonG6IXs9D/6cKcOSHhNz0FpJX38t7kXL3oEDB8kx6J53cz0ufAEZz6wzCbz6ZyLb1uA+6Ki+Hla3SJRxovVdm6pR/W1IwyZajrcuXsqcOIXoClronuznHDKUYaV5jB2ePBtfTL6xavd62GdMxulbO0OWIzQ31xOJhLo/2AZ1dSLKADEkusNgmctgmQd0PxeXy0NpaQWSlDqFDz6y90r4gxHEcbMRS0cQ+mQlrolHIAj9+yFGsZB9hIIcbZFKYMcGAFwjrIWvrYuXemDZm9pxd/OEIAhCl0QP1ugbSK7Zy338o2xuricnJ4/8/OEZhQ66XCKRyOAglsEyl8EyD+h6Lqqq0tnZRnNzPUOGpJ6Su38zYAbIiZabC4VVPDPPQGneS2T7R308qu5hJj9/UDZWz/p3bkDIL0Uoskon1jj7nmj2sb+7s+xTgR5yGe+gjdfsO/1hm7P3HyKREPn5RU6MuIMBB0EQyM8vSvupdNCRfW603Jw/GME14UiE4uGEPnm238fdmzX7QCjmoA3s2ohUeUgCKcXnoMkU5nZdWSD7+JDLeM1et+w7AhGbs/cvHKJ3MFCRyXd3EJK9Ztn7QzKCKOKdeTpK427knev6dmDdQFZUYwWoqmolCRV/G3JnC9KQcQnHx+egyRTpyDipwJBtoq9ul4hA7KlBd9D2tWXvIDluu+0mnnji8b4ehoMsY9CSfSCav9110FEIhRUE+7l1H1FUS3pht0tEad4LgFiWuCLUmlSsJ9E4JrLPhmUfly4h1+viyvOmc8y04UDMsu/sB5Z9f0Uksv+uzf7sy0HfYvA5aD26Za99iQXRhWfm6QTffgh592e4xhzel8NLCllW8LhEIwma1yWhNEWrU5Ulrgg1W/M9SZdgfhrMpmVvTtR2+MRy4+/C6AraZOkWDlTMm1fFxRd/j/fff5cjjzya88+/gOXLf8/WrVsIhULMnFnFFVf8lL17d3PddT/nkUf+RSQSYdGihXznO9/l/PMv5LXXXuGdd97kpptu49FHH+G1115GliN4PF6uvvpaDj54sm1f5557Hr/+9a9obGxg+PBKRNNn98wzT/Kvf/0Tt9uDqircfPPtjB07rk+ukYOeYdCRfU5Usw8EZWOb++C5hD5+lsA7fyfn+EtxjTy0r4aXFLKiWhZSud2aZS/mFCDkJpYZzFa6YNFU8MMu+Vm6iLfs4+GSRK74+mHdRvXsT7z7WQ3//bQmrXMEAVJ5UJx3eCVzD0stYsLr9fLXvz4MwO2338KMGbO49tpfoSgKv/71Up5//lnOPPMcfL5OGhoa2LevmvHjJ7J27Yecf/6FfPTRGqqqtDTYp5yyiP/9328D8OGHq7nzzt/ywAN/t+1r6dKfM336TC655Pvs3buHiy46nyOPPBqAP//5j6xY8QRDhgwhFAoNmtDGAxE9Ivvm5mZ+/vOfs2vXLjweD2PHjuXmm2+mrKyMyZMnM2nSJMNKuOOOO5g8eXJWBt0VDMs+GHs8FSQXuSddTuCNB/Cv+iP5/3MLYtHQZE30CWRFxWta5p/rkVAa9uKpGG3rjJGyli5Be83xurLisNQjg7oa08xJFUn3Hcg49dTTjb//+9+3+eKLz3nssRUABAIBhg7VIrJmzario4/WUFNTzVlnfZ0VKx4mHA6zdu0avv3tiwDYtOkL/vGPh2hra0UURXbv3pW0r48/XsuVV14NwMiRo4wbhtbXHG677Ubmzp3P0UfPY+RIJ+/QQEWPyF4QBC699FKOPPJIAJYtW8Zdd93Fb37zGwAee+wx8vPzez7KNGBE44Rky3apYjy5p11N57+XEnjrQXJPv7ZfRWMoimrRzPO8LuTmPeRPOxY7AzK+/F+m0B20ed7ua7ymgviVswMBcw9L3frW0Rsx3bm5eaZ3Kr/5zV225Dp79hw++uhDqqv3csMNt7Bu3ce8+uoqVBVGjBhJOBzmV7/6BX/601+YPPkQGhrqOfvsU7voKzl+85s7+eKLz/noo7UsWXIZV1/9S44+em5Ppumgj9Cj5/aSkhKD6AFmzJhBdXV1jwfVE+hx9oFgouNJLCjHe+R5yDWbiGz/cH8PrUvIimKRcYpog5Afz7BxtsdLWQq91GWcnCwlD4stpho4ZN8fMXfusTzyyP9DljWjpaWlhepqzWE/e/YcVq9+n/b2doYOHUZV1RE8+OD9hkUeCgWRZdl4EnjyyX932dfs2XN4/vlnAaiu3svatdpvIxKJUF29l0MPncYFF1zEEUccxZYtm3plvg56H1nT7BVF4dFHH2XBggXGtgsuuABZljn22GO54oor8Hh63yknigI5Hslw0MbDfcjxhDe+QfCDx3GNmYHg6h+OQlm2WvYloToAPMPGE7Q53pr1sucOWl3+6iniQy8dZIYrr/wZf/7zPVx00f8iCAJut4clS37GiBEjGTp0GHl5eRx++AxAI+va2n3MmqUVZs/PL+C73/0B3/vehRQVFXPCCQu77OunP72GX//6V7z66ioqK0cwc+ZsQPtN33bbTXR0tCMIIsOGDeOyyy7v1Xk76D1kjexvueUW8vLy+Pa3NafQm2++SWVlJR0dHVxzzTXce++9/PSnP02rzfJy+5zp3SEvxwWiSEWFvRPQf9ql1DxyI+6vXqd0/nkZ9ZFtyIpKQb7XeF+uNoAg4hk6hgq3N+F4l6m4yLBhxRnLJrnRilNFhd6k1ysdlJa2A1CYb99eNvrIBurqxB4vIsvGIjQdH3xgrb9QVFTItdden/T4f//7aePvoUMreP996yrx73znYr7znYuN9xdf/N2kfQ0dOpR7773ftp8HHvhbt2PvT8jmZ9LX6G4uYhccZ9teTwcEmla/c+dO7rvvPsMhW1mpaaAFBQWcd955PPTQQ2m329jYYckZkypyvW6aWvzU17dbtr/7WQ2+QIST5ozFNWEOze/+h+DQaUg2oY37E6qqaonQzJEODTsQSyoR3d6EeQB0RBclCQI0NXZk3Hco+gQkgW0/6cLXETTajW+voqIwK31kA4qi9EhzP1DysAwkDJZ5QGpzURQl4fckikJSI7nHt8G7776bDRs2cO+99xoyTWtrK4FAANB0v1WrVjFlypSedpUyKofks6c+kQD/+2kNr328BwDv3AsQPHkEXr8PNcPMh9mCnvEyJuOouNr2IJaPSXpOKlEvqUB/IMjNkoM2Ni5Hs3fgoD+hR5b9li1buP/++xk3bhyLFy8GYNSoUVx66aXccMMNCIJAJBJh5syZXHnllVkZcCqYNqGctV/U0toZMpbng2YNN7cHUVUVMbeInOMuxf/S3QRXP07O3Av22/jioRcPyfNq0ky52IHgb0UadlDSc/RMlz2NejEctFnS7GO5cAbP47QDB4MBPfqFH3zwwWzaZO+dX7lyZU+a7hGmRVdsbt7dwpxDYvH0Hf4w4YhCZyBCQa4b15jDcR92MuHPViHkl+KdcXqyJnsVHT5Nkikq0G5Mk9zaAp+uFn9JWbLsdWSrlJ8Ul8/egQMH/QOD0vyaOKoEr1viy53NxjZVVQ2du6ktYGz3HvlNXAcdRWjNfwh9+mKf5M/Rx6U/hUxy1WhpjYuHJz1HFAREQeixZa8/VeR6siXj6E8cg/Kr5cDBgMWg/EW6JJHDJpTxwcZafNGEW4GQbGjjze2xYEZBFMk5/nu4xlcR/OBxAq/8CTXYuV/Hq5N9qStAmdjBZHcN0ohDu1305ZKEHlvQerHxbMfZ9ySFgwMHDrKPQUn2AIuOHoc/GOHVj3YD0G5KqdvUbo1cF0SJnBN/hPfIbxLZuY7Op36N3LBzv421wx9GQmbMB8u4seRJ3IKM57CvdXueJAk9jmc3LPsskX22HMcOHDjILgbtL3Ls8EKmjS/jnfU1WhkvE9k3twcSjhcEEc/0U8k741qIhPA9cyuhL9/aL7JOhz/MUKkNUdZuQn/vOA5pyNhuz5NEsecyTii7Ms5ATJdwIOCii84nGEz83jvoH6ipqWbRoq4Xv/UUg5bsAY6YMozGtgAvfLCTD7+oM7Y3t9mtSdUgDT+YvHNvRhp2EMG3H8L/3DKU9oZeHWeHP8xwVxsAd7Qu4vNwanH/kiT02ILWLftsyTilBV4mjChi7LD+sXjKgYa///2feL05fT2MlKGnidhfUBRlv/nr9mdfZgy6FMdmzDh4CJIo8MRb24xtBbnuBBknHmJuEbmnXUP4y7cIrn6czn9fj3vyPNyTj0UsGorgyc3qODv8YUZ7tMURdXJiOuNkcInZ0Oyza9l7PRJLL6zKSlsHEubNq+J73/sh77zzFq2trfziF9ezdu0aVq9+j0gkwi23LGPcuPEAPPLI31m16gUApkyZyk9+cg2iKHLuuYtYseIJSkpKAPjTn/5AXl4el1zyfebNq+Lll98mLy+P//mfMzjllEV8+OFqGhsb+Na3LuCcc74BwPr1n/C7392OIAjMnFnFO++8yZ13/oEJE6xhwFu3fsXvfnc7gYCfUCjEmWeewze+cT779u3j+9//Dk8++Twul0YvS5f+nLlzj+XUU0/n/ff/y8MP/41gMITb7eaKK65i2rTD+Pjjtfzxj3cxefIUNm/exPe+90M6Ozv5978fJRLRnsp//OOfUFV1RNJx3n33PYwdO4Fdu3bwxz/eTWtrC+FwmG98439ZtOjMhGv+4IP3s337Njo7O6it3cd99z3E559/aju+G2+8juOOW8CCBSeyYsX/4+GH/8YLL7yOJEl8+9vn8Zvf3EV+fj433XQ9nZ2dhEIhjjlmLj/60ZVJ+3rllZf417/+SX5+PkcfPc8YV3NzE7/+9VKamhoBqKo6giVLftbj79igJvuCXDfzDq/krXWx5GzjhhdS09i9A1YQRTyHnoBr1FSCHz1N+Is3CX/+GiqiFrI5eT6uMdMRpJ5fwg5fmMnudoSCcsJNqbcnSWKPo14MGSdLlv1ARHjzu4Q3vZ3WOYIgpGSduScfi3tSalkiCwoK+etfH+b111/ll7/8GTfd9Bsuu+xyg1xuuOEW3n//XVateoH77vsbeXn53Hrrjfz973/lRz9awvz5x/PKKy9x3nmLiUQivPLKS9x3n326g0AgwP33P0RNTTUXXvhNTj75dFwuFzfddD033XQb06fP5K233uA//3nM9vzKykr+8Ic/4/F48Pl8fP/73+GII45m3LjxjB8/kQ8+eJd5846jtbWFTz75iOuv/zV79+7h739/kLvvXk5+fgHbtm3l6quX8OSTzwOwffs2rrnmOqZN0woMtba2cNJJJyMIArt27eDKK3/EU0+9QCgUSjrOSCTCTTct5cYbb2Xs2HH4fJ1897sXMG3a4bZFVzZu3MDf/raCkpKSLsdXVXUEH320hgULTuSjjz5k/PiJfPHFRoYPr8Tn8zFmzFiCwSDLlv2evLw8IpEIV111OR988B5HHXVMQl9ffbWFhx/+Gw89tIKysnLuuut2Y0wvv/wio0aN4g9/+DMAbW1tKX1/usOg/4V/55RDOHxiOcuf+AyAQ8aWsmF7E+2+EIV51iRoDS1+9jX7mDY+VllJLBpK7gnfRznyG7z3yhs07PiKE2q3Ie9ah5BTiDh2Fvv8bsYcXoU0/GAEUSIUlvlsWxMzJw2xlP1Lhg5/mKFCC2LpCNgF58wfn9LcpCxa9tlaVOUgcyxcqDnlJ08+BBCYO3d+9P0U3nrrDQDWrl3DwoVfIz9fWxJ/5plf549/vAuAU089gz/+8U7OO28xH3zwHmPHjqOycoRtXyeeqPVVWTmCwsIi6uvrCIfDeL1epk+fCcBxx51AQYG9HBcIBPjTn27nq682IwgiDQ31fPXVZsaNG89pp53OCy88x7x5x/HKKy8xd+6x5Obmsnr1++zdu4cf//j7RjuyLBsW7KhRow2iB9i7dw833XQ99fX1uFwumpoaaWxsoLm5Oek4d+/exc6d27nxxuuMdsLhMDt2bLcl+6OPnms8CXU1vtmz5/DII38nFApRV1fH+edfwNq1qxk+vNJIQKcoCn/+8x/57LNPAZXGxka2bNlskL25r08++YhjjplHWZnGNWeddQ5vvPEKAFOnHsa//vUo9977R2bMmGUUkukpDohfuFk/Hh+tkLSztt1C6gB/ff4LvtrTym3fP5JhpdZ830JuMc9WD6PBX4x8+NmcNbGT8Kb/EtzyPuVyCP+uV8Gdg1A0jNqWAJ6gzK5thzFywgSE3GKEvCLtNbcQQbRe9qDPR4najFg8k79du4BU4ZLEHkfjLDp6LE+8tY2cLMk4AxHuSXNTtr519EYeFj3diCiKeDyxRHeiKKakYU+fPgOfz8fWrV/x4osrOe20M7rtK9Z+erVo77//XsrKyvnb31bgcrn46U9/TCikpR057rgFLF+uySgvvPAcV16pSRCqqnLkkUfzq1/dnNDejh3bE3Ls33TT9Vx++U859tjjURSFE0+cZ/SRDKqqUlxcwt///s+U5mHus6vxgVZz4rXXXmbatMOYPXsOt956I8OHVzJ7tpZa+vHHV9De3sYDD/wdr9fLsmW3EQrFJONUawhMm3Y4Dz/8T95//31WrXqBRx75O//3fw+mdG5XGNQOWh2lhbGskWN0st8XSyDU1BZg5Xs72Ly7BUVV+ecrW/hiZzMd/jBf7W2lMxDmxdW7aGgN4HaJfLipAWn0dHJPupw/e3/Atc2Leb/8bKSJR7OrVaDe70JBpGz3mwTeehD/S3fje/ImOlf8lI6/Xkr7/7uchhW/YO9jt7Dribs5J/wMLmRc42enNa/hZXkML03tC5QMi44ex9+uXWAUMXHQv1FVdQSvv/4KPl8nqqry3HNPM2dOrKbEKacs4rHHHmH9+k84/vj0ojvGjBlLIBDg00/XAfDOO2/S0WGfuK6jQ8ul73K52LbtK9avX2fsy8nJYd6847jvvnvx+ToNC/yII45i9er32bZtq3HsF198nnQ8HR0dxpPJ888/axB9V+McM2YsOTk5vPTS80Y7O3fuoLOz+2SB3Y1v9uyqaN2AIxg2bDhtba2sWfOB4Udob2+nvHwIXq+X+vo6/vvft5L2NXPmbN5//12am5sAeO65Z4x91dV7yc/P58QTT+aKK37Kpk1fZqUc5AFh2QuCQGV5HooK+TluKkpyWL+1kdJCL7Ks8uTb22jtDFFW5GXGQUN4/eO9fLatkeJ8D62dISaPLmHT7ha8Holz5o3nsde/4uFVmxg3vJCv9rbikrw8tyefz+XxfFY3motOPYR3v6xj68568vAxIi+CEGynUAxQJPgpl0N45U6KhDbyxTpKxQiflp/MvMr0yjb+8OxpvXTFHPRXHH30XLZu3cIPfqClLz7kkEP5zndi6YtPOeV0vvGNMznttDPIyUkv+sbj8XDjjbdy112/RRAEZsyYRWlpmSEZmfGd73yXW265geeff4bRo8cwY8ZMy/5TTz2DH//4Ui699DJj2+jRY7jhhlu4/fZbCAaDRCJhDjtsOlOmTLUdz5IlV3HddVdTWFjIkUceQ3FxcZfjLCgowOVysWzZ77nnnt/x6KP/QJYVysrKuPnm2237MKO78elFXnRL/rDDZvDRR2uoqNBSspx33mJ+9atfcMEF36CiYphxnB0OOuhgLrjgYn74w++Sl5dvqf71yScf8ctf/hNBEFFVhWuu+aWlCHymENS+iAFKEZmmOLZLpasoKghamoEHn9/Iu5/tM/YV5Xu44tzDGFVRgNsl0toRYsUrm/l8h3bXDYZkjp85kvOOn0iOR+Kfr27htY+07JmiIPDNhQfx6KtbEIALTp7M8TNH8vz7O3jirW0U5Xto94VYdPQ4Tpg5kk27mnnw+S84fsZITjt6LNtr2vjTk59x0amHcOz0RH21P6UF7in601z27dvJ8OHdr2VIhsGaTtfn6yQvTysl+vHHa7nttpv497+fzQrZZBN243zqqecYLPXQU/l+2X2Hu0pxfMCQvRkRWbHkxynO9+KN06xVVSUYlnlp9S5WrdnNb75/lEUOamoLEJEVcjwuivI9NLUFcEkiRdH8Njv3tXPz3z/k8q8fxpRxpRYHaCAUSXjvdUu26RH6E0H2FP1pLg7Zx2CeywsvrOTxx/+Jqip4PF6WLLnKqIjVn2A3zlmzZg3KzyQZHLInu6SiqCq+aJbMdNHhD2d0nhn9iSB7iv40F4fsYxgscxks84DeIfv+9WzWDyEKQsaE3VOid+DAgYNswSF7Bwcs+vFDrQMHXSKT765D9g4OSLhcHjo72xzCdzDgoKoqnZ1tuFye7g82oVdDL7dv3861115LS0sLJSUlLFu2jHHjxvVmlw4cpITS0gqam+vp6GjJ6HxRFLMS+9wfMFjmMljmAd3PxeXyUFpakVabvUr2N954I+effz5nnXUWzzzzDDfccAMPP/xwb3bpwEFKkCQXQ4ZUZnx+f3I29xSDZS6DZR7QO3PpNRmnsbGRjRs3cvrpWl3X008/nY0bN9LU1NRbXTpw4MCBgyToNbKvqalh2LBhSJIWvy5JEkOHDqWmpqa3unTgwIEDB0nQr9MlJIsXTQUVFYOneIYzl/4JZy79D4NlHpD9ufSaZV9ZWUltba2RrU+WZerq6qiszFwndeDAgQMHmaHXyL68vJwpU6bw3HPPAfDcc88xZcoUysrKeqtLBw4cOHCQBL2aLmHr1q1ce+21tLW1UVRUxLJly5gwYUJvdefAgQMHDpKgX+fGceDAgQMH2YGzgtaBAwcODgA4ZO/AgQMHBwAcsnfgwIGDAwAO2Ttw4MDBAQCH7B04cODgAIBD9g4cOHBwAKBfp0tIFwM9pfKCBQvweDx4vVqt26uvvpr58+ezbt06brjhBoLBICNHjuTOO++kvLy8j0cbw7Jly1i1ahV79+5l5cqVTJo0Cej68+ivn1WyuST7bIB++/k0Nzfz85//nF27duHxeBg7diw333wzZWVlXY65P86nq7lMnjyZSZMmGUXR77jjDiZPngzA66+/zh133IEsy0ydOpXf/va35Obm9uVUAPjRj37Enj17EEWRvLw8fvWrXzFlypTe/c2ogwgXXHCB+vTTT6uqqqpPP/20esEFF/TxiNLDCSecoG7atMmyTZZl9cQTT1Q//PBDVVVV9d5771WvvfbavhheUnz44YdqdXV1wvi7+jz662eVbC52n42q9u/Pp7m5Wf3ggw+M97fffrv6y1/+sssx99f5JJuLqqrqpEmT1I6OjoRzOjo61GOOOUbdvn27qqqqet1116nLly/fL+PtDm1tbcbfr7zyinr22Werqtq7v5lBI+MM1pTKGzZswOv1UlVVBcDixYt56aWX+nhUVlRVVSXkPOrq8+jPn5XdXLpCf/58SkpKOPLII433M2bMoLq6ussx99f5JJtLV3j77beZNm2aYf0uXryYF198sTeHmTIKC2NJzjo6OhAEodd/M4NGxukqpfJAysdz9dVXo6oqs2fP5qqrrqKmpoYRI0YY+8vKylAUxXiU66/o6vNQVXVAflbxn01RUdGA+XwUReHRRx9lwYIFXY55IMzHPBcdF1xwAbIsc+yxx3LFFVfg8XgS5jJixIh+lWL9+uuv591330VVVf7617/2+m9m0Fj2gwErVqzg2Wef5YknnkBVVW6++ea+HpKDKAb6Z3PLLbeQl5fHt7/97b4eSo8RP5c333yTJ598khUrVvDVV19x77339vEIU8Ntt93Gm2++yU9/+lPuuOOOXu9v0JD9YEiprI/V4/Fw/vnn8/HHH1NZWWl5XG1qakIUxX5jZSVDV5/HQPys7D4bfXt//3yWLVvGzp07+cMf/oAoil2Oub/PJ34uEPtsCgoKOO+885J+NtXV1f3yO3b22WezevVqhg8f3qu/mUFD9gM9pbLP56O9Xas5qaoqL7zwAlOmTGHatGkEAgHWrl0LwGOPPcYpp5zSl0NNCV19HgPts0r22QD9/vO5++672bBhA/feey8ejwfoesz9eT52c2ltbSUQCAAQiURYtWqV8dnMnz+fzz77jB07dgDaXE499dQ+GbsZnZ2dFjnp9ddfp7i4uNd/M4Mq6+VATqm8e/durrjiCmRZRlEUJk6cyNKlSxk6dCgff/wxN954oyUUbsiQIX09ZAO33norL7/8Mg0NDZSWllJSUsLzzz/f5efRXz8ru7ncd999ST8boN9+Plu2bOH0009n3Lhx5OTkADBq1CjuvffeLsfcH+eTbC6XXnopN9xwA4IgEIlEmDlzJtdddx35+fkAvPrqq9x5550oisKUKVO4/fbbycvL68up0NDQwI9+9CP8fj+iKFJcXMwvfvELpk6d2qu/mUFF9g4cOHDgwB6DRsZx4MCBAwfJ4ZC9AwcOHBwAcMjegQMHDg4AOGTvwIEDBwcAHLJ34MCBgwMADtk7cNAF7rvvPq6//vqMzr322mv5/e9/n+UROXCQGQZNbhwHDnoDl112WV8PwYGDrMCx7B04cODgAIBD9g4GFWpra7niiis46qijWLBgAQ8//DAAy5cvZ8mSJfzkJz9h5syZnHPOOXz55ZfGeQ888ADz589n5syZnHzyybz//vvGeVdffbVx3GuvvcaiRYuoqqriggsuYOvWrca+jRs3cs455zBz5kx+8pOfEAwGLWN74403OOuss6iqqmLx4sUp9e/AQdaQQd59Bw76JWRZVs855xx1+fLlajAYVHft2qUuWLBAffvtt9V77rlHPfTQQ9UXX3xRDYVC6l//+lf1hBNOUEOhkLp161b12GOPVfft26eqqqru3r1b3blzp6qqqnrPPfeoP/vZz1RVVdVt27ap06dPV//73/+qoVBIfeCBB9QTTzxRDQaDajAYVI8//nj1oYceUkOhkPriiy+qhx56qHr33Xerqqqqn3/+uXrUUUep69atUyORiPrkk0+qJ5xwghoMBrvs34GDbMGx7B0MGnz22Wc0NTVx+eWX4/F4GD16NN/4xjd44YUXAJg6dSqnnHIKbrebiy++mFAoxPr165EkiVAoxNatWwmHw4waNYoxY8YktP/CCy9w3HHHMXfuXNxuN9/97ncJBAJ88sknrF+/nnA4zHe+8x3cbjennHIKhx12mHHu448/zje/+U2mT5+OJEmcc845uN1u1q1bl3L/Dhz0BI6D1sGgwd69e6mrqzOqLIGWCraqqooRI0YwfPhwY7soigwbNsw4/rrrrmP58uV89dVXzJs3j2uvvZZhw4ZZ2q+rq7MUw9DTBdfW1iJJEsOGDUMQBGO/+djq6mqefvppHnnkEWNbOBymrq6OI444IqX+HTjoCRzL3sGgQWVlJaNGjWLt2rXGv08++YS//OUvAOzbt884VlEUamtrjcyVZ5xxBo8++ihvvPEGgiBw1113JbQ/dOhQS350VVWN6kIVFRXU1taimvIKmo+trKzksssus4xt/fr1Rpm5VPp34KAncMjewaDB4YcfTn5+Pg888ACBQABZltm8eTOffvopAJ9//jkvv/wykUiE//f//h8ej4fp06ezbds23n//fUKhEB6PB6/XaxTGMOPUU0/lrbfe4v333yccDvO3v/0Nj8fDzJkzmTFjBi6Xi4cffphwOMzLL7/MZ599Zpx73nnn8dhjj7F+/XpUVcXn8/Hmm2/S0dGRcv8OHPQEjozjYNBAkiTuu+8+li1bxsKFCwmFQowfP56f/OQnACxcuJAXXniBX/ziF4wdO5bly5fjdrsJhUL87ne/Y+vWrbjdbmbOnGlbdnDChAnceeed3HLLLdTW1jJlyhTuu+8+o5DG8uXL+dWvfsUf/vAHjjvuOE466STj3MMOO4xbbrmFm2++mZ07d5KTk8OsWbOoqqpKuX8HDnoCJ5+9gwMCy5cvZ+fOnY484uCAhfOs6MCBAwcHAByyd+DAgYMDAI6M48CBAwcHABzL3oEDBw4OADhk78CBAwcHAByyd+DAgYMDAA7ZO3DgwMEBAIfsHThw4OAAgEP2Dhw4cHAA4P8Dwcog7naoDvgAAAAASUVORK5CYII=\n" - }, - "metadata": {} - } - ], - "source": [ - "cfg = DQNConfig()\n", - "env = gym.make('CartPole-v0')\n", - "env.seed(1)\n", - "state_dim = env.observation_space.shape[0]\n", - "action_dim = env.action_space.n\n", - "agent = DQN(state_dim,action_dim,cfg)\n", - "rewards,ma_rewards = train(cfg,env,agent)\n", - "agent.save(path=SAVED_MODEL_PATH)\n", - "save_results(rewards,ma_rewards,tag='train',path=RESULT_PATH)\n", - "plot_rewards(rewards,ma_rewards,tag=\"train\",algo = cfg.algo,path=RESULT_PATH)" - ] - } - ] -} \ No newline at end of file diff --git a/codes/DQN/outputs/CartPole-v0/20210429-222132/models/dqn_checkpoint.pth b/codes/DQN/outputs/CartPole-v0/20210429-222132/models/dqn_checkpoint.pth deleted file mode 100644 index 2b2200e..0000000 Binary files a/codes/DQN/outputs/CartPole-v0/20210429-222132/models/dqn_checkpoint.pth and /dev/null differ diff --git a/codes/DQN/outputs/CartPole-v0/20210429-222132/results/eval_ma_rewards.npy b/codes/DQN/outputs/CartPole-v0/20210429-222132/results/eval_ma_rewards.npy deleted file mode 100644 index e25eb51..0000000 Binary files a/codes/DQN/outputs/CartPole-v0/20210429-222132/results/eval_ma_rewards.npy and /dev/null differ diff --git a/codes/DQN/outputs/CartPole-v0/20210429-222132/results/eval_rewards.npy b/codes/DQN/outputs/CartPole-v0/20210429-222132/results/eval_rewards.npy deleted file mode 100644 index 2fc0e4e..0000000 Binary files a/codes/DQN/outputs/CartPole-v0/20210429-222132/results/eval_rewards.npy and /dev/null differ diff --git a/codes/DQN/outputs/CartPole-v0/20210429-222132/results/eval_rewards_curve.png b/codes/DQN/outputs/CartPole-v0/20210429-222132/results/eval_rewards_curve.png deleted file mode 100644 index 295fdac..0000000 Binary files a/codes/DQN/outputs/CartPole-v0/20210429-222132/results/eval_rewards_curve.png and /dev/null differ diff --git a/codes/DQN/outputs/CartPole-v0/20210429-222132/results/train_ma_rewards.npy b/codes/DQN/outputs/CartPole-v0/20210429-222132/results/train_ma_rewards.npy deleted file mode 100644 index f71e613..0000000 Binary files a/codes/DQN/outputs/CartPole-v0/20210429-222132/results/train_ma_rewards.npy and /dev/null differ diff --git a/codes/DQN/outputs/CartPole-v0/20210429-222132/results/train_rewards.npy b/codes/DQN/outputs/CartPole-v0/20210429-222132/results/train_rewards.npy deleted file mode 100644 index fa9ffc3..0000000 Binary files a/codes/DQN/outputs/CartPole-v0/20210429-222132/results/train_rewards.npy and /dev/null differ diff --git a/codes/DQN/outputs/CartPole-v0/20210429-222132/results/train_rewards_curve.png b/codes/DQN/outputs/CartPole-v0/20210429-222132/results/train_rewards_curve.png deleted file mode 100644 index a6857d3..0000000 Binary files a/codes/DQN/outputs/CartPole-v0/20210429-222132/results/train_rewards_curve.png and /dev/null differ diff --git a/codes/DQN/outputs/CartPole-v0/20210504-190229/models/dqn_checkpoint.pth b/codes/DQN/outputs/CartPole-v0/20210504-190229/models/dqn_checkpoint.pth new file mode 100644 index 0000000..76ec7d6 Binary files /dev/null and b/codes/DQN/outputs/CartPole-v0/20210504-190229/models/dqn_checkpoint.pth differ diff --git a/codes/DQN/outputs/CartPole-v0/20210504-190229/results/eval_ma_rewards.npy b/codes/DQN/outputs/CartPole-v0/20210504-190229/results/eval_ma_rewards.npy new file mode 100644 index 0000000..f51fdef Binary files /dev/null and b/codes/DQN/outputs/CartPole-v0/20210504-190229/results/eval_ma_rewards.npy differ diff --git a/codes/DQN/outputs/CartPole-v0/20210504-190229/results/eval_rewards.npy b/codes/DQN/outputs/CartPole-v0/20210504-190229/results/eval_rewards.npy new file mode 100644 index 0000000..6533c74 Binary files /dev/null and b/codes/DQN/outputs/CartPole-v0/20210504-190229/results/eval_rewards.npy differ diff --git a/codes/DQN/outputs/CartPole-v0/20210504-190229/results/eval_rewards_curve.png b/codes/DQN/outputs/CartPole-v0/20210504-190229/results/eval_rewards_curve.png new file mode 100644 index 0000000..dd3e728 Binary files /dev/null and b/codes/DQN/outputs/CartPole-v0/20210504-190229/results/eval_rewards_curve.png differ diff --git a/codes/DQN/outputs/CartPole-v0/20210504-190229/results/train_ma_rewards.npy b/codes/DQN/outputs/CartPole-v0/20210504-190229/results/train_ma_rewards.npy new file mode 100644 index 0000000..25efa3d Binary files /dev/null and b/codes/DQN/outputs/CartPole-v0/20210504-190229/results/train_ma_rewards.npy differ diff --git a/codes/DQN/outputs/CartPole-v0/20210504-190229/results/train_rewards.npy b/codes/DQN/outputs/CartPole-v0/20210504-190229/results/train_rewards.npy new file mode 100644 index 0000000..c45be6e Binary files /dev/null and b/codes/DQN/outputs/CartPole-v0/20210504-190229/results/train_rewards.npy differ diff --git a/codes/DQN/outputs/CartPole-v0/20210504-190229/results/train_rewards_curve.png b/codes/DQN/outputs/CartPole-v0/20210504-190229/results/train_rewards_curve.png new file mode 100644 index 0000000..4a18b49 Binary files /dev/null and b/codes/DQN/outputs/CartPole-v0/20210504-190229/results/train_rewards_curve.png differ diff --git a/codes/DQN/task0_train.ipynb b/codes/DQN/task0_train.ipynb new file mode 100644 index 0000000..94ebd60 --- /dev/null +++ b/codes/DQN/task0_train.ipynb @@ -0,0 +1,270 @@ +{ + "metadata": { + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.10" + }, + "orig_nbformat": 2, + "kernelspec": { + "name": "python3710jvsc74a57bd0366e1054dee9d4501b0eb8f87335afd3c67fc62db6ee611bbc7f8f5a1fefe232", + "display_name": "Python 3.7.10 64-bit ('py37': conda)" + }, + "metadata": { + "interpreter": { + "hash": "366e1054dee9d4501b0eb8f87335afd3c67fc62db6ee611bbc7f8f5a1fefe232" + } + } + }, + "nbformat": 4, + "nbformat_minor": 2, + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "from pathlib import Path\n", + "curr_path = str(Path().absolute())\n", + "parent_path = str(Path().absolute().parent)\n", + "sys.path.append(parent_path) # add current terminal path to sys.path" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "import gym\n", + "import torch\n", + "import datetime\n", + "\n", + "from common.utils import save_results, make_dir\n", + "from common.plot import plot_rewards\n", + "from DQN.agent import DQN\n", + "\n", + "curr_time = datetime.datetime.now().strftime(\n", + " \"%Y%m%d-%H%M%S\") # obtain current time" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "class DQNConfig:\n", + " def __init__(self):\n", + " self.algo = \"DQN\" # name of algo\n", + " self.env = 'CartPole-v0'\n", + " self.result_path = curr_path+\"/outputs/\" + self.env + \\\n", + " '/'+curr_time+'/results/' # path to save results\n", + " self.model_path = curr_path+\"/outputs/\" + self.env + \\\n", + " '/'+curr_time+'/models/' # path to save results\n", + " self.train_eps = 300 # max trainng episodes\n", + " self.eval_eps = 50 # number of episodes for evaluating\n", + " self.gamma = 0.95\n", + " self.epsilon_start = 0.90 # start epsilon of e-greedy policy\n", + " self.epsilon_end = 0.01\n", + " self.epsilon_decay = 500\n", + " self.lr = 0.0001 # learning rate\n", + " self.memory_capacity = 100000 # capacity of Replay Memory\n", + " self.batch_size = 64\n", + " self.target_update = 2 # update frequency of target net\n", + " self.device = torch.device(\n", + " \"cuda\" if torch.cuda.is_available() else \"cpu\") # check gpu\n", + " self.hidden_dim = 256 # hidden size of net" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "def env_agent_config(cfg,seed=1):\n", + " env = gym.make(cfg.env) \n", + " env.seed(seed)\n", + " state_dim = env.observation_space.shape[0]\n", + " action_dim = env.action_space.n\n", + " agent = DQN(state_dim,action_dim,cfg)\n", + " return env,agent" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "def train(cfg, env, agent):\n", + " print('Start to train !')\n", + " print(f'Env:{cfg.env}, Algorithm:{cfg.algo}, Device:{cfg.device}')\n", + " rewards = []\n", + " ma_rewards = [] # moveing average reward\n", + " for i_ep in range(cfg.train_eps):\n", + " state = env.reset()\n", + " done = False\n", + " ep_reward = 0\n", + " while True:\n", + " action = agent.choose_action(state)\n", + " next_state, reward, done, _ = env.step(action)\n", + " ep_reward += reward\n", + " agent.memory.push(state, action, reward, next_state, done)\n", + " state = next_state\n", + " agent.update()\n", + " if done:\n", + " break\n", + " if i_ep % cfg.target_update == 0:\n", + " agent.target_net.load_state_dict(agent.policy_net.state_dict())\n", + " if (i_ep+1)%10 == 0:\n", + " print('Episode:{}/{}, Reward:{}'.format(i_ep+1, cfg.train_eps, ep_reward))\n", + " rewards.append(ep_reward)\n", + " # save ma rewards\n", + " if ma_rewards:\n", + " ma_rewards.append(0.9*ma_rewards[-1]+0.1*ep_reward)\n", + " else:\n", + " ma_rewards.append(ep_reward)\n", + " print('Complete training！')\n", + " return rewards, ma_rewards" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "def eval(cfg,env,agent):\n", + " rewards = [] \n", + " ma_rewards = [] # moving average rewards\n", + " for i_ep in range(cfg.eval_eps):\n", + " ep_reward = 0 # reward per episode\n", + " state = env.reset() \n", + " while True:\n", + " action = agent.predict(state) \n", + " next_state, reward, done, _ = env.step(action) \n", + " state = next_state \n", + " ep_reward += reward\n", + " if done:\n", + " break\n", + " rewards.append(ep_reward)\n", + " if ma_rewards:\n", + " ma_rewards.append(ma_rewards[-1]*0.9+ep_reward*0.1)\n", + " else:\n", + " ma_rewards.append(ep_reward)\n", + " if (i_ep+1)%10==0:\n", + " print(f\"Episode:{i_ep+1}/{cfg.eval_eps}, reward:{ep_reward:.1f}\")\n", + " return rewards,ma_rewards" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "tags": [] + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Start to train !\n", + "Env:CartPole-v0, Algorithm:DQN, Device:cuda\n", + "Episode:10/300, Reward:13.0\n", + "Episode:20/300, Reward:14.0\n", + "Episode:30/300, Reward:14.0\n", + "Episode:40/300, Reward:12.0\n", + "Episode:50/300, Reward:125.0\n", + "Episode:60/300, Reward:98.0\n", + "Episode:70/300, Reward:200.0\n", + "Episode:80/300, Reward:160.0\n", + "Episode:90/300, Reward:200.0\n", + "Episode:100/300, Reward:200.0\n", + "Episode:110/300, Reward:200.0\n", + "Episode:120/300, Reward:198.0\n", + "Episode:130/300, Reward:200.0\n", + "Episode:140/300, Reward:200.0\n", + "Episode:150/300, Reward:200.0\n", + "Episode:160/300, Reward:200.0\n", + "Episode:170/300, Reward:200.0\n", + "Episode:180/300, Reward:200.0\n", + "Episode:190/300, Reward:200.0\n", + "Episode:200/300, Reward:200.0\n", + "Episode:210/300, Reward:200.0\n", + "Episode:220/300, Reward:200.0\n", + "Episode:230/300, Reward:188.0\n", + "Episode:240/300, Reward:200.0\n", + "Episode:250/300, Reward:200.0\n", + "Episode:260/300, Reward:193.0\n", + "Episode:270/300, Reward:200.0\n", + "Episode:280/300, Reward:200.0\n", + "Episode:290/300, Reward:200.0\n", + "Episode:300/300, Reward:200.0\n", + "Complete training！\n", + "results saved!\n" + ] + }, + { + "output_type": "display_data", + "data": { + "text/plain": "
", + "image/svg+xml": "\n\n\n \n \n \n \n 2021-05-04T19:04:03.044086\n image/svg+xml\n \n \n Matplotlib v3.4.1, https://matplotlib.org/\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n\n", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXsAAAEcCAYAAAAmzxTpAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Z1A+gAAAACXBIWXMAAAsTAAALEwEAmpwYAABhrElEQVR4nO2dd5hU5dn/P+dM3dned1l6WaQILCxNpC0WUGwxKCFi7D8bVjREERKUN4K8aiAYJSYmRiLGVwWxgVHATlFAihRBadsL26ee5/fH7MzObK/s7Ozzua69dubU+zlnzvfc537ucz+KEEIgkUgkkqBG7WgDJBKJRNL+SLGXSCSSLoAUe4lEIukCSLGXSCSSLoAUe4lEIukCSLGXSCSSLoAU+yBm1apVzJ8/v0P2ffnll7N9+/YO2XegYbVaufPOOxk1ahT33XdfR5tTi48//pjJkyeTlpbGwYMHO9qcduX06dMMHDgQp9PZ0aacc6TYS9qF999/n7Fjx3a0GQHBRx99RH5+Ptu3b2flypW15q9atYohQ4aQlpZGWloal156KUuWLCE3N9dvuZKSEhYvXsyECRMYPnw4V1xxBe+8847fMhkZGYwfP56KigrvtDfffJO5c+fWa9+yZct44okn2L17N4MHD25la918/vnn/PrXvyYtLY1x48Zxww038Mknn7R4exkZGXz11Vfe7x7R9hyzjIwM1qxZ0xamt5ivv/6a6dOnM3z4cObOncuZM2c61J6aSLE/hwSLNxEM7TiXbcjMzKR3797o9fp6l5kxYwa7d+9mx44d/PnPfyY/P59f/OIXXsG32+3cdNNNZGZmsm7dOnbt2sUjjzzCM888w6uvvuq3LU3Tak1rzL4BAwa0qG0ul6vWtI8++oj777+fq6++ms8++4yvvvqK++67jy1btjR7+42dp507d7J7927+93//l9WrV/PZZ581ex9tQWFhIffeey/3338/O3bsYOjQoTz44IMdYkt9SLGvYs2aNVx00UWkpaVx2WWX8fHHHwPuiyw9PZ0jR454ly0sLGTYsGEUFBQAsGXLFq666irS09OZPXs2hw4d8i7r8TiuuOIKRowYgdPprHdf4L54nn76acaOHUtGRgavvfaa32NnaWkpjz32GBdeeCETJ07kueeeq/OCq4s9e/Ywe/Zs0tPTufLKK/3CLG+99RYzZswgLS2NadOmsW7dOu+87du3M2nSJNasWcOECRP43e9+x6pVq7j//vt59NFHSUtL4/LLL2ffvn1+7fZ4Yo0te+DAAa6++mrS0tK47777eOCBB3juuefqbcd//vMfr62XXXYZBw4cAGDgwIGcOHHCu9yCBQu826mrDTNmzPATIKfTybhx47zba+h41eTYsWPMnTuX9PR0Lr/8cq8Xu3LlSl544QU+/PBD0tLSePPNNxs4Q2AwGBgwYADPPfccMTExvPLKKwBs2LCBrKws/vSnP9GjRw8MBgOTJk1i4cKFPP/885SXl3u3ceutt/L3v/+dkpKSBvdlt9tJS0vD5XJx1VVXcdFFFzXYFs8xXbx4MbfffjsjRoyodUyEEDz99NPcfffdzJo1i/DwcFRVZcyYMTz11FMAnDx5khtvvJGxY8cyduxYHn74YT9ba14zDz30EJmZmdx5552kpaXx17/+tVZb0tLS6N+/P0ePHkXTNF544QWmTp3K+PHjefTRRyktLa3zGDT1empMBz7++GMGDBjAjBkzMJlMzJs3j0OHDnHs2LEGz8E5RUiEEEJ88MEHIjs7W7hcLvH++++L4cOHi5ycHCGEEAsWLBDPPvusd9nXXntN3HLLLUIIIQ4cOCDGjRsn9uzZI5xOp3j77bfF1KlThc1mE0IIMXXqVHHllVeKzMxMUVlZ2ei+/v3vf4sZM2aIrKwscfbsWfGb3/xGpKamCofDIYQQ4u677xZPPPGEKC8vF/n5+eLaa68Vr7/+ep1tWrlypXj44YeFEEJkZ2eLMWPGiK1btwqXyyW++OILMWbMGFFQUCCEEGLLli3ixIkTQtM0sX37djFs2DCxf/9+IYQQ33zzjRg0aJBYvny5sNlsorKyUqxcuVIMHTpUbN26VTidTrFixQoxa9Ys776nTp0qvvzyS68d9S1rs9nElClTxD/+8Q9ht9vFpk2bxJAhQ/yOd83zdOGFF4q9e/cKTdPEzz//LE6fPi2EECI1NVX8/PPP3mV/+9vferdTVxtWrVolHnroIe/yW7ZsEdOnT2/S8fLFbreLiy66SPzlL38RNptNfPXVV2LEiBHi2LFjtc5DY+fJl+eff1788pe/FEII8cADD4hHH3201jIOh0MMGjRIfPHFF37H/Z577vG2/T//+Y+44YYb6t2/73FrrC2//e1vxciRI8WuXbuEy+USVqvVb1s//vijSE1NFSdPnqx3fz///LP44osvhM1mEwUFBWLOnDniqaee8s6v65rx/T0JIcSpU6e814WmaWLXrl1i2LBh4quvvhJvvvmmuOiii8TJkydFWVmZuOeee8T8+fNrrSdE866nhnTgySefFIsWLfJb/vLLLxcfffRRvcfhXCM9+ypmzJhBYmIiqqpy2WWX0atXL77//nsArrjiCt5//33vshs3buSKK64A4I033uD6669n+PDh6HQ6rrnmGgwGA3v27PEuP3fuXJKTkzGbzY3u68MPP+TGG28kKSmJyMhI7rjjDu928vPz2bZtG4899hgWi4XY2FhuuukmP9vqY8OGDUyaNInJkyejqioTJkxg6NChbNu2DYApU6bQs2dPFEVhzJgxTJgwgV27dnnXV1WV++67D6PR6G3HqFGjmDx5MjqdjquuusrviaYm9S27d+9enE4nN954IwaDgUsuuYTzzz+/3u383//9H7fddhvDhg1DURR69epFSkpKo+2vqw1XXHEFn376KZWVlYD7vF5++eVNOl6+7N27l4qKCu644w6MRiPjx49n6tSpTTovDZGQkEBxcTEARUVFxMfH11pGr9cTHR1NYWGh3/T77ruP1157rdb0xmhKW6ZNm8aoUaNQVRWTyeS3/tmzZ72210evXr2YMGECRqORmJgYbr75Znbu3Om3TM1rpj7GjRvHmDFjWLhwIQ8//DDjx49n48aN3HTTTfTo0YPQ0FAeeughPvjgg1ohoeZeTw3pQEVFBeHh4X7Lh4WF+T1xdTT1BxG7GOvXr+eVV17xdqpUVFRQVFQEwNixY7Farezdu5fY2FgOHTrkfeTNzMxk/fr1vPbaa95tORwOv8615OTkJu8rNzfXb/mkpCTv58zMTJxOJxdeeKF3mqZptbZfF5mZmXz00Ue1whaeTtRt27axevVqfv75ZzRNw2q1kpqa6l02Ojq61oUdFxfn/Ww2m7HZbDidzjpj0/Utm5ubS2JiIoqieOc31J6srCx69uzZaHvromYbevXqRb9+/diyZQtTp07l008/Zf369UDjx8uX3NxckpKSUNVq36lbt27k5OS0yE4POTk5REZGem3Py8urtYzT6aSoqIjo6Gi/6ampqUyZMoU1a9bQr1+/Ju+zKW1p6PxERUV5t9OjR486l8nPz2fp0qXs2rWL8vJyhBBERET4LdOU3zTAN998U+v3lpub6+cApKSk4HQ6vWFXD41dT5dffjmZmZkA/PWvf21QBywWC2VlZX7bLy8vJzQ0tEntOBdIsQfOnDnDwoUL+cc//kFaWprX+/Sg0+mYPn067733HnFxcUyZMoWwsDDA/aO88847ueuuu+rdvq+QNbav+Ph4srOzvd99PyclJWE0Guv8gTdGcnIyV111lTdu6ovdbue+++5j2bJlTJs2DYPBwN13343wKYjq24a2JD4+npycHIQQ3n1kZWXVKxTJycmcPHmyznkhISFeLx0gLy+PxMRE7/e62jBz5kzee+89NE2jf//+9OrVy7uf+o5XTRISEsjOzkbTNK9IZmVl0bt370bXrQ9N09iyZQsXXHABABdccAHPPvssFRUVWCwW73KbN2/GYDAwfPjwWtu47777uOaaa7jllluavN/WtqVv374kJyezefNmbr311jqXefbZZ1EUhY0bNxIVFcV///tflixZ4rdMa35vCQkJfpkwmZmZ6PV6YmNjm3U91eXh16cDAwYM8MuMqqio4OTJk/Tv37/F7WhrZBgHqKysRFEUYmJiAHdn5dGjR/2WueKKK/jwww/ZuHEjM2fO9E6fNWsW69atY+/evQghqKioYOvWrbXu8k3d14wZM3j11VfJycmhpKTErzMqISGBCRMm8PTTT1NWVoamaZw8eZIdO3Y02sYrr7ySLVu28Pnnn+NyubDZbGzfvp3s7Gzsdjt2u52YmBj0ej3btm3jyy+/bPoBbAUjRoxAp9Px2muv4XQ6+e9//+vXeVuTX/7yl/z9739n//79CCE4ceKE98I+77zzeO+993C5XHz22We1QgN1cdlll/Hll1/y+uuv+53Xho5XTYYNG4bZbObll1/G4XCwfft2Pv30Uy677LJmHw+n08mxY8d46KGHyM/P56abbgLgqquuIikpifvvv5/Tp0/jcDj4/PPPeeqpp7j11ltrhRDA/eRy2WWX8a9//avJ+29tWxRFYcGCBbzwwgu89dZb3t/prl27eOKJJwC3x2uxWAgPDycnJ4eXX3650e3GxcVx6tSpJtkwc+ZM/vnPf3Lq1CnKy8t57rnnmDFjRi1Bb8n1VJ8OXHzxxRw9epRNmzZhs9lYvXo1AwcObNZTVXsjxR7o378/t9xyC7Nnz+aCCy7gyJEjjBw50m+Z4cOHExISQm5uLpMmTfJOP//883nyySdZsmQJo0eP5pJLLuHtt99u8b6uu+46JkyYwJVXXsnVV1/N5MmT0ev16HQ6AJYvX47D4eCyyy5j9OjR3HfffXU+3tckOTmZF154gZdeeonx48czefJk/va3v6FpGmFhYSxcuJAHHniA0aNH895775GRkdHcw9gijEYjq1at4v/+7/8YPXo07777LlOmTMFoNNa5/IwZM7jzzjt5+OGHGTlyJPfcc483rv3444+zZcsW0tPT2bhxo/cRuyESEhIYMWIEu3fv9hO0ho5XXW148cUX+eyzzxg3bhx/+MMfWL58ebMudE+2Tnp6OnfddRdRUVG8/fbb3icTo9HIK6+8QnJyMtdddx3Dhw/ntttu4ze/+Q333ntvvdu95557/HLuG6Mt2jJ9+nSee+453nrrLSZOnMgFF1zAn/70J6ZNmwbAvffey8GDB0lPT+eOO+7gkksuaXSbd9xxB3/5y19IT0/nb3/7W4PLXnvttVx55ZXccMMNTJs2DaPR6L3R1KS511N9OhATE8OqVat47rnnGD16NN9//z3PPvtso+06lyhCyMFLAplt27bx+9//vkU5yp2VWbNmMXv2bK699tqONiVgcTgc3H777SQmJvL000+3W5hNEjxIzz7AsFqtbNu2DafTSU5ODqtXr26Sh9qZ2bFjB3l5eTidTt555x0OHz7MxIkTO9qsgMZgMLBq1Sp69OjB8ePHO9ocSSdAevYBRmVlJTfccAPHjx/HbDYzZcoUHn/8cW9HUDDyxhtv8Kc//YnKykq6d+/Oww8/zJQpUzraLIkkqJBiL5FIJF0AGcaRSCSSLoAUe4lEIukCSLGXSCSSLkBAv0FbVFSOpjW/SyE2NoyCgrpfaupsyLYEJrItgUewtANa3hZVVYiOrrtEQ0CLvaaJFom9Z91gQbYlMJFtCTyCpR3Q9m2RYRyJRCLpAkixl0gkki6AFHuJRCLpAjQq9kVFRdx+++1ceumlXHHFFdx7773eARH27NnDlVdeyaWXXsott9ziVy+6oXkSiUQiObc0KvaKonDbbbexadMmNm7cSI8ePVixYgWapvHII4+waNEiNm3aRHp6OitWrABocJ5EIpFIzj2Nin1UVJTf6DwjRowgMzOT/fv3YzKZSE9PB2D27Nl89NFHAA3OkwQmmhBoQlBX9Qzf6Z7lPH/gHmS65vQ6/7S6pzdrGw3YUt/0pm6rsfV97fR8bsz2prStqe33PTfNaXejx0Vrmj2+iCYs39D6zT1PTWqTVvdvuDFbm/v7q28bNfdX047GzmNd89uSZqVeaprG66+/TkZGBllZWXTr1s07LyYmBk3TOHv2bIPzPMOWSdqPf3x4iOOZxSy5dSwfbT/J8cxiYiLMfLkvi1UPTOLLfVl8/n0WC37trqO/81AuL27YjxAQHW7i+oz+vLjhACvvn0hWQTl/fO07Qkx67r5mKCv/73scTndNd6NB5f5rh7Hq7X1Y7a4W2/vri1P59nAuh06ebdLyw/rFEh1uYtueTL/p3eND6ZUQxs4DpwlTbYQqVu//UMWOUXFgUpwYFaf7Pw4UnYH4sdP5z9afCNELLpk+iTXv7idEVBCmWAlXrYQqNnQ6hQkzr+DP6w9hc7j4/e3j+NO63QzuHc3+44UUl9vrtPVX0wbw5b4sTub65kwLLIqdcKWScNXKecMGU+Qw8uU+/4FRDDgJVysJr7IjPKkHt/86gw+/OcGbW495l0uOtXDVhX146d0D1KUTYSEGbps5mD+//T1OV+0FFDRCFZu7rTonl10+mb99eIRKq71q35WEqzYuuXwaw89LRgjB43/dTnahu07+jZcO5Mv9WRw7U9LgeZs2sjuKAi4h0KsqH+9q2mAkADpchKtWIpUKwlUrakgYUy6ezOp39uGqkaIYYtLx1G3jiA43kVVQzh/+sRO7Q0OvU5l37fm8uOEAlTb3eLQ6VWHetefzyoeHKC6r+xzW5Lqp/Tl4opD9x6vH980YmcLpvHKSYkLQqSpbdp/hgqFJxESYKSi2Eh9l5t0vfwZgUK9oHvlVGp98e5q1Hx+ptf3u8WH8ZcG0Jh+bptIssX/yySexWCzccMMNfPzxx21uTE1iY1te6TE+vvbIPZ2V5rbls71uEYyJCSW32MrJ3DJ2Hc7zbqug7ATHM0u82y389jRCQPqgRHb9kMPmXacByCuzk1/qvgAqbU5+zCzF4dSYeWEfnC7BR1//zM955VjtLjLSe5AU2/zxNt/ZepT8Uhtn8ssZ2DOaUYMSG1w+OyuPvft/5qxi47regt6hFYRU5mGyFiAqz2LOdHBdTO0BRjxoioqmGtFUI3b0qLZSLN/+mYeqDrFr20csi6x7/ZN7HODoQ3ddKd8fyaWo1MaX+7JJiLEw58K+7oWEwOAoxVKZTfaPh8nZfpR+TjvX9NKIVUoxWQswOkpQRfXNseLE53xDGjfGl9M33IbZWoDRXoxO8xcfe5mOSCWdL/Znk5pkIqOPRunp4xzJsXM6LwGdqnDdRQNRXXZCrLmEVOYgCk+Tk1PEoSMWQrVyrk83E2HLxWQrdP/ZizDZilGobrN126f8zqwQZrHiWyU/+6iN+IkPUFZhJ7uwgvRBiRw4XkBOsZUT2aUM6RvL8AH+g6LrnRVYKrI4c+QQZTkVnFKScboEZqOO5LhQpo6qHn5SddkJqcwhxJpHSGUeZmseZlshBkcJBmftAVgOHHQxQGdj+lAjodY8zNZ8DNYinHYbtuMK8Rddzq6j+dgdGjMn9OaTrw7z1YEcKm1OLhubQpLIY9+eHziT3Z3iMjvjhibRNyXKu32d01p1HN1/Zmse9qIcsk+O5nRubwb0iGL04CQOHjnDd/tPgtPKsVNmend3j0J3Kq+cMquTrIJybC6NuEgzcVEhnMwtIz4+nKyiSsJCDFw5yX9gmO7xbt1raw1rstgvW7aMEydO8OKLL6KqKsnJyd7BeAEKCwtRVZWoqKgG5zWHgoKyFr1YEB8fTl5eabPXC0Ra05bvD+VQUWnH6aq+kLOyiykvt+PSNO92i85WYjLoGNo7ml0/5GAxuUfF+ulUEflnrd51T+e4Pbcpw5KptDn56Oufycl3e6xjz4tnQPeoZrfl4+0/U1JqpdLmom9yOBeldfObL6xlODN/wJX5A66sQ2hFmVzl2U0JUKpDjUxETexGGQMoculI6RaPYg5HNUeghISjmMNQTGFgMKPoqn/yZ8tsLFz9KUOMZ9CZLQiXiwRXNqbQMC66cLB3/TJh5uv/vMqEgu08Hb0dVYF9Z6xACkN6R3PThGjCSw/hyjyEK+swotI9ctZ5AMaqP1uo2874gahhMSiWSJSQSD7encPwws1kKF/h0vQY1CTUpJ4oYbEolgjUkEiUkAg+P1DAoBPrOPnK49zqgHitFA672zEmDL7/qZJZYVbGH/sUrfA0UBVuUPX0MmnoTh5lRrSAqtL3SkgESng8asJAwhK7UYkFJSSCf3/yIym2YziFjoljz8MQHo0SEsGO99czOPsbsg9+T67iHjx+ZL8olJwj5GUbcboEQ3pFkzHAjCvzIM7MQ7iyDiFK8wEYBNgr9XyqjeGUM5ZcQzf6JZqZEp2DM/Mgrpwf0fJPgucmqLjPqxKfhBp6HoolCsUShWqJ5EiewLr9DYbkfMCQcOA0KOZwTPHdKVcTyTl2jN7b/87PP21FVCZyR8QZhvx8loujS/jpVCKTIuyk/FiMIjSGh8Hhn6zAEEb2sjA85CSurMO4co6iFZ7xHkd0BtSoJPJUGFL0KT+4xjNOb6Pvj1mMKT6DEuZeLtcVzoelk7nYfIbKyihOMojiUhsWk57k2FB6J4Vz+EQRObkl5BdVEBth9vvNC4cV4XTf5Fty3auqUq+T3CSxf/bZZ9m/fz9r1qzxDhc3dOhQrFYru3btIj09nXXr1jF9+vRG50nan9gIEwUlNk7klCIEfo/2lTZnVVzQHS9UFQWbw4XJoKJT3X5ceIj7HBeW2CgsrRb7whIbACEmvffRubTCAYBRr2uRrcn6Ypw2I06XhtHg3oawV+I8sRvHj9/gOn3ALQB6E7qkARj7jSPHasKKgX6Dz0ONSEBR3T/jECC+gX3VJDLUCMZQdtr6MjQlBrNBx8bDeVw8pAfG8wZ4l7M4XWyoGEloiIEzFQYGGc4woOgrZltSGG3LR//hWWyAEhqNLmUwuoS+qDE9KDUm8OTfv6R/rwTuvn5snTaUn/qRp45bMGPjoolDuPyCPnUuZ88+yd8OTOWX4SfILasgbtQ0LMn92H/WQuW2VxjGdzh0epSQVIwjr0SN7YkuOoVyQzTPrH6f8aYfsZqi+MXVU9DF9EAxhni3HeNzE7YeCOXfB5NIjrUwfew47zL/dZ7mfOM7VLzzB6zm3sy26Dlv51sMoZys4jhCTX0Zfnwb5XtPAKCYwtAlD0Q3OAM1tifv7y5idOa/ma5+hcug8KOrG/3ycqjc7ASdEV1CX4zDZ6Am9EEXlYISEec9rzXRq2d5uWwqY8JzKCGUe2++BDUkwu0tZxezcvcn3DUon4GOg5xX+jXFhkj03Yfy+Y+V9FWOUSzCMI64EF18X758712Gs4c7w7JI/SYHq3CBwYwusT/GvqPRxfZCje6GEhaHoqr841/b+FXlv5gd8gWuEj1KynkY+qTz7jdn0JxOLg35nt8o74EFNKGwy5bLCV0oB4rPY4Iln4GnD5EcZqOyYjylFXbCLQaEy4nzxG6cR7/CeXo/iikU8UDj4/I2l0bF/ujRo7z00kv07t2b2bNnA9C9e3dWr17N8uXLWbx4MTabjZSUFJ555hkAVFWtd56k/YmPCnGLfXapt9NHwe2jVNqc1Z1bmkDVVYm9UYde5+6vd1WNs1pQYqWo1EZCVAi5ZyspLHU/1puNOuxVcfuyyiqxNzT9lQ2haTiP78C+bzO3asc5UjyAnYwnXJRg/fp1HIe2gcOKEhqD4fxLMPQeiZrQx3vx92ybw4SiKHSLC+XHM8V0iw0lPiqEXYfzGNgzym85g16Hpjfzn/KxlFudHHYk86DhQ0YYT+CMHkJo/+HoUwajRCT4DQ8YDVx1yUj6JEfUa0NkmAmb0GHDQlS4ud7lLGY9x5yJvKWlUqy3MXHMBQCYtSL+UjaJboYSQhJ68ujlY/zWCxOCQjWG9ZXppMZGok9KbfCY9EoMZ/vBHHon+YcQSgxxvB/5a5xZh5gpdpNm0uFKHM7uAjMjK77kl6H52LQkjOm/QN8rDTUmBUWp/k24jh9jxQ+XEa5UcmnI93TTFZEdncaAcVPQJQ/0e+JqjLAQA1Zh5LOSHnSPD0MNqT6+ep1KaGgI3xvTGDbzeu579lOmpvdm1tT+HC78nnVHz6dHQhijRruP0yfOM5xvPEWcrpTy3pNIGDYBNb4vilr379kVEs0/SmdSUVHJuEnjuGiM++Y8ICKPTTtOsjo7kRi1jNMkcrP5E8boDzAmFK4WuzCc1bCZohliLMLx9VqcFX0Zb/iR8rV/Q1hLUUKjMQyagiF1QrsMM9noER4wYACHDx+uc97IkSPZuHFjs+dJ2hePJ/9zTinhIQaEALNJT6XNSYXNiScy5tIEeh3Y7C5Mhmqxt1V1tuafraS43E73+DByz1ZSXGYnxKRHURSMeveyHrE3GZrm2TtP7sW24020wtMokUmc0vWgv+tHZlkURh06hgOBvu8YDEMy0CX29xOM9iA51uIW+7hQRg2M52yZjfP7xtRaLsSkp6SqE/YMiayyz+JkmYEnrr0AY3z9fUuTR6Q0uP+oMKPPZ1O9y1lMBgCyCytIiK72ykNDDDjQc8IRQ5ql9s1CURSiw03kFlUSHVH/zcRDryqR75XoL/Ymg47jtmiOWYfwtW0ATqFjxdzJHP/0Rz7LDseGgd9Mn0pcz+g6txtqNnBWC+Usobxc5h7Mfu64gQzq3vDxqYswi8H7Odzns4focBNFpTYOnzyL1aV62+Q5br7Hr8IQzR+tv6awQmPx+WPQJTYcJzebdOwsi8SlRTDFXH08R6bGU1RqY+3pYn4ige7xYTyXNx2j4iLddJxk3VnMgyZhTjmPU5tf5aLjX/Cg+iW6MoHacwTGwRnoug+t9ybTFgR0ITRJy/CkgpVVOAgzGxBCEGLSucXe6u/ZAz6evdub8GTWZBdWYHO4GH1eAnt/dD8ZeOL5Hk/eG8ZpROy1imJy3n6Jyh++RolIxDztLvR9R/PFf75m1tm/cYHpCEVxo+h58RzUsNi2PSAN0C3O3ancLTaUULOBayf3q3M5S5XYq4pCZKiB46UhCMDcxJtcfUSGVot9pI/w1yTU7L5Uyyod9OsWUWu6+3Nt4QOI8Yh9eP03Ew8DukcyfUxPxg727yg3GXWcLXOH8azCiKoohIUYCA0xcMLlDp6FhtS9//psC29g+Ybwa3Md24gJN3HwRBE/nikmPsrMkD7um3dCtKXqf7XYmww6ckoFoGAyNn4uQ4zVIcwQk//yYT62JEaHcDrPiE3AFusQAG6OH4AlxMDGypGkpQ3ix7270Q+5mCkZ4zgXSLEPQjwiXp3vC2ajHrC5Y/ZV812+Yu/j2XvEvtzqTk+LiTB7nwxCTO6fjE5V0esUyj1hHH39HonzzEGsn/wF4ajEOPpajMNneEMyTmMEL5ReTIUwctWkC+l9DoUeYPR5CeQXW+md3LBHZ6kSmBCTDrNRjxBu4WuKQDRElI8AN+jZ+whcuKX6puArdqEhdV/OHpFvitjrdSrXZfSvNd1s0HHaJzUx3GJAVZQm3Wzqs60ur7wp6FQVi0lPhc3pJ7AeosNN2OwuFGDxTeleuzwin1gl+uA+f54uraY8nZp9BN5s8m+T7xNHQkwINQm3GKvOo8Kx0BH8u9zCTbFtFZRsHCn2QYjHs/d9ScNcJUpuz75qOY/Y211EWIzVnr3DP2c+JtyEperJIMTnB27U66hwuW8IhnrE3r7/Y2xfv44amUTKjU9STJTffKNBxzFnond755qYCDO/vrjhODbgbbfFrPcTeHNrxT7ULcB6nb9w1sRXSH1F0qh333SdLoGlHrGNruoLiGmC2NeHyajzy2f33HB899lU+2tuoyWEWQwNij1Az6RwrzcP0L9bJBeen8ywftUOhe+TWVPOZYhRX+dngDCzr2dvoSbhFoP3d5RT5E4lbenTTUuQhdCCEM9FqWkgNIEmIMQj9lXZOL7L1eygtdmdfl5gVLjJ68X4iX1VKMdoUOvsULLtegfbV2vR9xyO5eonMMb3qLWMb/inOZ285xqLR+xNBq8HqCqK95i1FJNRh9moIzLU2GCnXH2evaIoXiENq0dsqz37xmP29dpZw+uNDHXv0+OxG/Rqg6G8um4ELfXsoTpkUpfYe45j/5RIv+kmo45bLh/k9wTle+Nuimfv+/tvKIyTEFWXZ2/w/o5yiyqrprX8htdcAvfqkrSYqmSaKs/eHc7xXIiVfmLvXtBmd2GuEcYx6FR+f/No0gbEkRIX6v2R+4t9Vfy+Do/c9u167N9tQJ96IeaL5/ml+vniG/5pLO7fkfiHcdx2moy6NsmaiAwzEdlACAfcXqdata+aIukJ5dQXMx/aN4ZRqfGkxDf/pTcPNcNVEVV9DR5vtiGvvqZtCqAoDcf4G8MjrHV5xuMGJzKifxxXTujd6HY8Am/Qq6hq4+fS7PdU10AYJ7ruMI7Xs696+7g1N7zmIsM4QUjNei++tTvqDOM43Dnuuqowjs3uQheu0DMxnHnXDgOqPduaYRwAUw2P3H5oG/Zv17uFftItDWYY+Hn2DcT9OxqvZ282eJ9AWhvC8TBhaFKj21IUBYtZT1mlo7bYVwltfTHzxGgL9/zi/FbZWLMj2iP2jd1oatoIEBNhwu7UvDevlhDWwH5jIszc98thTdqO5ybW1Gyyhjx7k0HnvWlE+HS8h4UYsNqdmKucA5NBR47Xs5diL2kFnvCMqBrW0ffFqooaHbSaENgdLsxGHYYqz15ArfBEtWfv89hr9IRxqqe5so9i++JVdN2HYp50c6OpZL43itZ2drYn3pi9Se/t22iqQDTGzAt6N2m5arH3f/T3iHx9HbRtge+5SRsQx9A+sVX7bvhG48Gg12E0qOhVlYhQEzZHy2spQcNhnObguYk19cbtCYcq1H3+w0IM3vCeyaDD5nCRFGOhoMTqfQoMMbmn61TF7+bR3kixD0KqO2hBw+3pe7z9SpvT+6aspgkcDg2B+4fr8ewBr6B58BU7Dx7P3iP2WkUxlR+vQgmLI2TaXShq4xeQv2cfuGLvCeNYzHo8Dum5vjl5hLVm6MIj8o0JbmvwiGGISed92nPvu2lhHPcyBvQ6hRED4rC3ldi30jM2+YTkmoKn78pc9b5JXXZ5wkEWsx6nS2P0eQnknq30LmMxGzhbZic2wtwuL0/VhxT7IMQ39VJ4PPuqeRVWp/cCdWnC62H5dtBCHZ6992Kv/sl4PBtTVfjF9uW/EPYKLJc/imJqWnzY1zuqGQ4KJHzDWJ4bZ2tz7JttQ5WY1+vZt6PYm7wesL9khJj0KDQt/u4R+yua+CTTEAO6R9InOYLoRvo6GsNzE2vquazrCdeXoX1jvOGp0Cqxv3i0f2KC51pK7RHVEpNbjBT7IKRm6qVvze0Km9PrpWpCeNMsffPsoYEwjs/FXp2No8NxfCfOn3ZhHPNLdDHdm2xrZ+mg9X2y8YTJzrVnbzHpMerVWvvtnxLJsTPF9QpQW2Cq42YP7oykwX1iamW+1MV5vaK8T5WtZWDPaJ74TXqrt+N1WJoZxqmZdulh1pTqdxQsJr23HLgvRVUvp9Usy9HeSLEPQjSf1EtNwz9mb3WihVfF7F0Cu91X7KsvRF09YZy6OmjDdHZsX76GGtcb47AZzbLVI/CqorSZELQHvmEcT12gtuqgbSp9u0V467D7kn5eAunnJbTrvj0efV1tfvj6EU3axpyLGn+f4VzT3A7a6jBO48v3S4n0ltjwxVNQcKD07CWtxSv2viP6VE2z2p3ekE6DYZwaHauWOh5fPZ79CNtORGUpITMeblKc3pfGcvUDhYRoC6FmPSnxoWTlu9Pm2qqDtqlcOqYnl445d29c+uJpa0gAd6K3hIZuYnVh1KuoilKvZ+/LrKm130QGuHRMD/676zRxdeTitydS7IMQzSe1Uqsh9prw9fwFjqpa9yaDzs+z1tdIg+wWF4pRrxLv8wM1GnREKuWkVuxGP+ACdHG9mm1rzU7eQCUy1MiqByYB1Z5ZIGcPtTXe2PY5zB45F1SHcZrWLkVR3O9atOI4XJ8xgOszBjS+YBsTXGdOAvi8QVv1UhW4h4LzTPN4+y7hH8ZRFMX76n3NbJxeSeG8OH+K97tw2ulXvJ2k0CMoQsOUfnWLbPVcbIGcY18Tj8if6zBOR1Lt2QeXZDS3gxYgKcZCUh21bwKd4DpzEqC6g1aI6hesXC6f3Hsfz9/qE8YB0OlUnC5XrTBOTRwHt5Ca+zEY4UTUGIaGN2fYkGo8YZxzHRJpDR5h6Ew2t5Zqzz642tzcDlqABTeMDOiQY310HndK0mSET8Eql1YzjCP8bgA2n2wcAH1VKKdmGMd/+y7s+zdTZknhw4rhnEqa0mJbq8M4neenWO3Zdx1fydRIFkpnxdzMDlpwV91szdu/HUXnucIkTca3OqGrKibv8svQqX6D1mavIfZVIq9vIDPG+dO3iLICcrtP5SPrcHTmlg8M39xsiEAgItRdIbQ1VSQ7GxaTu9pnbGTLi6kFIuEWAwa9SkxE8J/LJt2mly1bxqZNmzhz5gwbN24kNTWV06dPc88993iXKS0tpaysjB07dgCQkZGB0WjEZHIfxPnz5zNx4sR2aIKkJp6xZTUhcLqqPXrPf28Yp6pUAlSXPvCEb+qr5iiEwP79RygRidgShgA/tKpz1ROrD/QOWl8iLEZefvxinDZHR5tyzjAadDx9x7hWv7EaaFjMBp7+f+MbHDgmWGiS2E+bNo0bb7yRX//6195p3bt3Z8OGDd7vS5cuxeXyfwV65cqVpKYGXm5tMCOqBisxGBTsDoGzhmfv+9mlabg09/i0Oq/Iuz36mnn2HrTcY2h5xzFNuAGjwX3htyYE46mD35k6aAFiI0PIy6ud8x7MNFaZs7PSlEFdgoEmXWHp6ekkJyfXO99ut7Nx40auvfbaNjNM0jI8HrzHQ68O31S/yecpbaxpApcm/Eq7ejx6Qz2evePoV6AzYhgwoTpHvhU1bRRFwWhouBa6RCJpPW3S2/Lpp5+SmJjIkCFD/KbPnz8fIQSjRo3ioYceIiIiop4tSNoKj6Z7PHRvGMcvjl/dQetyiRoF0NwCXpdnL1xOHMe2o+89EsUYQliI27NtTV1ycBePas+6LhKJpI3E/q233qrl1a9du5bk5GTsdjtLly5lyZIlrFixolnbjY1tecdffHzDY4p2JprTFmvV6/RuT9nh9ejdwZoqqj5aQk0Yy+3odap3H+aqsgBRkSG19lt+eAdltnLi0jOwxIcTHx/OH++ewJC+sU1ORaurLUv+3wVENWEAj0Cjq/7GAplgaQe0fVtaLfY5OTns3LmT5cuX+033hH2MRiNz5szhrrvuava2CwrK/DzSphIfH05eXmmz1wtEmtsWT+0Uj/Z6Dp/TWd2f4snAKS6ppKzMhqoo3n140jatlY5a+6389hMUczhl4f0or5qXGGEiP7+sVW2x6BTslXbyKmvXEQlUuvJvLFAJlnZAy9uiqkq9TnKre8XeeecdJk+eTHR0tHdaRUUFpaVV4iEEH3zwAYMGDWrtriRNwBOjr5lNU3cHrXtwE9+YvaEqfFPzDVphK8d5Yg/6/uOaXf9GIpF0PE3y7J966ik2b95Mfn4+N998M1FRUbz//vuAW+wff/xxv+ULCgqYN28eLpcLTdPo168fixcvbnvrJbXwdtDWEHvfoQk9GTqaJnBqwq8mjk5Xd+qlbdfboDkxpE5oF7slEkn70iSxX7hwIQsXLqxz3qZNm2pN69GjB+vXr2+VYZKWIbyevb9n7qqjg1bTqjpo68jG8e2gdWYewnHgEwzDpqOL691epkskknakcyU3Sxql3jCOy8ezr+q0dVZVxfQXe/dn39RL50+7QGfElP6LdrNbIpG0L1Lsg4z6wjj1e/aaN3Tju57vNOfp/ei6nYeiD/63DCWSYEWKfZDhyV6q7w1Y8O+gdWnCr6iTvkYHrVaShyjORt/j/PYyWSKRnAOk2AcZHge+rhLFNeXf8watTle7g9YTxnGe3ufeXvehbW+sRCI5Z0ixDzK0ejpooba3r1WlXur8Ui/9wziuU/tQwuNQIpPay2SJRHIOkGIfZHjFvo7CYroacXxnVSE0/9TL6jCOcDlxZv6Avvv5nXKwBolEUo0U+yCjZiE0X2rWqPd20Ppm43iqX6oqrpwfwWFF10OGcCSSzo4U+yDD1ZBnX5fY10y99AxeoldxntgNqh59t8HtaLFEIjkXSLEPMqo9+7pi9rXTMd1VL31TL6vCOAo4f/4WXcpgFGPnG1xZIpH4I8U+yBDeEsdN8+y1mqmXVWEcXWkmojQffZ9R7WesRCI5Z0ixDzI8A5PUlWdfU+xddaRe9kuJZEifGMx5BwDQ90prR2slEsm5Qop9kOHNs6/Ls68jjFOzEFrfbhE8fP0IROZB1NheqCFywBmJJBiQYh9kNJhnX2cYR6s1XThsuHJ+RJciO2YlkmBBin2QUV9tHGggjFMjTdOVdRg0F/ru/sNMSiSSzosU+yBDq6fqJdTxBq2oPeA4gCvnKCgquqQB7WeoRCI5p0ixDzIaDuPUl3pZQ+zzT6BGd0PRd64xYSUSSf1IsQ8ymhXGcWnu2jg+qZdCCLS8n1Dj+rSvoRKJ5JzSpJGqli1bxqZNmzhz5gwbN24kNTUVgIyMDIxGIyaT2wOcP38+EydOBGDPnj0sWrQIm81GSkoKzzzzDLGxse3UDIkHVwMljusqhFYz9VKUFyKspejie7ernRKJ5NzSJM9+2rRprF27lpSUlFrzVq5cyYYNG9iwYYNX6DVN45FHHmHRokVs2rSJ9PR0VqxY0baWS+qk4do4NcI4QuDSNL+YvSvrMIAUe4kkyGiS2Kenp5OcnNzkje7fvx+TyUR6ejoAs2fP5qOPPmqZhZJm0dw3aH2zcVwFp7B+8SpKZBJqbM92t1UikZw7mhTGaYj58+cjhGDUqFE89NBDREREkJWVRbdu3bzLxMTEoGkaZ8+eJSoqqsnbjo0Na7Fd8fHhLV430GhOWyyhRQDExobWmhdqqTGsoKIgBESEm4mPD6dw/3dUuJz0+M2T6MNjWmVzfXTV8xLoBEtbgqUd0PZtaZXYr127luTkZOx2O0uXLmXJkiVtGq4pKCjzZpc0h/j4cPLyStvMjo6kuW0pLqkEoLzUWmuew+Hy+15hdQJgrbSTl1dKxenjqFFJFFkNYG3749eVz0sgEyxtCZZ2QMvboqpKvU5yq7JxPKEdo9HInDlz+O6777zTMzMzvcsVFhaiqmqzvHpJy2h48BL/MI7D6fKbrhWcQo3p0c4WSiSSjqDFYl9RUUFpqfvOI4Tggw8+YNCgQQAMHToUq9XKrl27AFi3bh3Tp09vA3MljeHpoK0Zn4faZY/tjqqiaYqCsJUjygul2EskQUqTwjhPPfUUmzdvJj8/n5tvvpmoqChefPFF5s2bh8vlQtM0+vXrx+LFiwFQVZXly5ezePFiv9RLSfvT4Bu0NbJxHC5PhUwVV+Fp9+eY7u1soUQi6QiaJPYLFy5k4cKFtaavX7++3nVGjhzJxo0bW2yYpGU0Z8BxR5Vnr6oKWuEp92cp9hJJUCLfoA0yGixx7BPGURXFL+Sjnc0CgxklNPqc2CmRSM4tUuyDDM/gJY0VQvP1/N1in40alYyi1H4ikEgknR8p9kFGtWffcCE035uBTlXQirNRI5Pa3T6JRNIxSLEPMkSDHbR1e/Z6nIiyAtQoKfYSSbAixT7I8BZCqyP10i+M45OHb7IWAKBGNr0khkQi6VxIsQ8yNOEeU7au2LtvGMdsrE7EMlvzAKRnL5EEMVLsgwxNEyiKQh1FL/28fYu5WuxNlfkAqJGJ7W6fRCLpGKTYBxmaEKiqO7WyJr5hnDCzwfvZWJmLEhYrR6aSSIIYKfZBhqbhDePUlPv6PHtDeZ7MxJFIghwp9kGGpgmvV19zIHHV5wYQ6vXsBfqKXBmvl0iCHCn2QYY7jOOW9JqdtKqqeOeFVnn2EUolqtMmPXuJJMiRYh9kuPw8e/95ilJ9A/CEcRJ0Je5lo2TapUQSzEixDzIa9Ox9snRCQ9xhHK/YS89eIglqpNgHGcLXs68h9oqieKd5wjjJuiKEzogS1j7DEEokksBAin2Q4apKvQSo+RKtQvUNwKDXAdBdV4gW1QNFkT8FiSSYkVd4kKFpArVK7Wtm47hftvJ4/aCgkaIvQsjRqSSSoEeKfZDhTr10f64ZxlGVam9fURTi1VJMihMlpuc5tlIikZxrmjRS1bJly9i0aRNnzpxh48aNpKamUlRUxKOPPsrJkycxGo306tWLJUuWEBPjjv0OHDiQ1NRUr5e5fPlyBg4c2H4tkQDuEsde770Oz16pmmbOO8DjURvcy8X2OrdGSiSSc06TPPtp06axdu1aUlJSvNMUReG2225j06ZNbNy4kR49erBixQq/9datW8eGDRvYsGGDFPpzhKYJdEp1qMYXRXF7+9FqGZF7XgOgXDOiRnc712ZKJJJzTJPEPj09neRk/zzsqKgoxo4d6/0+YsQIMjMz29a6LsyJ7FLKKh3NXk8Twuu910y99GTjTDQdRnHZWXr2Kp4s/gV6g6GuTUkkkiCiSWGcxtA0jddff52MjAy/6XPnzsXlcjFp0iTmzZuH0Whs1nZjY8NabFN8fHiL1w0E7l/5BZdd0Js+PWOa1Ra9XofZqCc+PtybceMhOsqCwaASp5VCRAK5BZEAJCZEYDTo6tpcm9PZz4svsi2BR7C0A9q+LW0i9k8++SQWi4UbbrjBO23r1q0kJydTVlbGI488wurVq3nwwQebtd2CgjI0zzh7zSA+Ppy8vNJmrxdIVNocFJ6tBGhWWyqtDlwujby8Uu+A4h5KSioRmiBaLcdpSvBOLyosrxXfbw+C4bx4kG0JPIKlHdDytqiqUq+T3OpsnGXLlnHixAmef/55b2cs4A37hIWFMWvWLL777rvW7qpLoWm06EYnhG82jv88RXH/GKLVckRINOOHJHqnSySS4KZVnv2zzz7L/v37WbNmjV+Ipri4GJPJhNlsxul0smnTJgYNGtRqY7sKQgi0qr/m4s6zrz8bx4iTcNWKNTSWWy4exK8vHljnqFYSiSS4aJLYP/XUU2zevJn8/HxuvvlmoqKieP7553nppZfo3bs3s2fPBqB79+6sXr2a48ePs2jRIhRFwel0kpaWxv3339+uDQkmPCLfErF3CYHe81JVlYjrVKWqQBpEKOUgAEs0OlXFYpavWkgkXYEmif3ChQtZuHBhremHDx+uc/m0tDQ2btzYOsu6MJrm+d8Sz94nz96nrr1LE5hz9jFLe8+9YGhsm9gqkUg6B23SQStpWzwi3+IwjuJJvawasKQqShN14A1Uyt1fQmXhM4mkKyGf4QMQbxhHa9m6vh69qlbn22umCO9yiiW69YZKJJJOgxT7AMRV5dmLlnj2QnjHmlWrXqJyfxWolUUUKtF8WDEcRScf6iSSroQU+wCktWEcxbeypaKgoBCq2FCdVg7qh/CRdfg5yauXSCSBgxT7AKQ6jNMysdepNcM4EKe6X9Ao1UUBMrdeIulqSLEPQKo9++av6/IpceyphaMoCnE6t9iXqFFA7fLHEokkuJFiH4B4xb7Fb9BWe/aeAUs8nn2ZPso7TyKRdB2k2Acgrla8VOVbz15RPHF7iNGVoZkjEKrBO08ikXQdZEpGANKaDlqXb7mEqsFKVEUhkkqEOareUawkEklwIz37AMQj9qKFHbRqjdRLRYFItQIREunn9Uskkq6DFPsAxNWKDlrfN2hVVakaWFwhQq1EhERVz5NqL5F0KaTYByCe6E2LUi99O2ir8uz1iosw1YYIiax3FCuJRBLcSLEPQFytealK1MyzV4hQ3IOgEOITs5dnXiLpUshLPgBprMTx1t1n2H0kr+51NYFSdVbHDUniorRE4pQiABS/mL307CWSroQU+wCkOs++7vmvbjrMqrf31bMuXs9+RP84Lih8l9nKh+6ZlmgZs5dIuihS7AOQloZxPCNc+Qq56+Qe72fFEuVX8lgikXQdpNgHIB6Rb27qpefe4Cv2SkRi9WdzKKoqvXqJpCvSqNgvW7aMjIwMBg4cyJEjR7zTf/rpJ66//nouvfRSrr/+en7++ecmzZM0TlNfqqo53/PdrxSCw9056xIKiqrz1sqRSCRdi0bFftq0aaxdu5aUlBS/6YsXL2bOnDls2rSJOXPmsGjRoibNkzROUwuhVVidft894R+P2AvNhags5RtlJL8t+hWKgvuNWvk8J5F0ORq97NPT00lOTvabVlBQwMGDB5k5cyYAM2fO5ODBgxQWFjY4T9I0mloIrbjcXud6njCNsJYCgjIlDAd6nzdqpWcvkXQ1WlQbJysri8TERHQ6HQA6nY6EhASysrIQQtQ7LyameeOexsaGtcQ8AOLjw1u8bkcTmuWuUOkRZd+2+N4AVIPOb15ZhVv8IyLMxMeHY8vOoxyw6d3HMT4ujKH94ykqt3fY8enM56Umsi2BR7C0A9q+LQFdCK2goKxFb5HGx4eTl1faDhadG86erQDA4XQB+LXFMw3g5JliXHYnOlUhOTaU0iqxryi3kZdXivNMFgClIgSAgoJyhveJZnif6A45Pp39vPgi2xJ4BEs7oOVtUVWlXie5RWKfnJxMTk4OLpcLnU6Hy+UiNzeX5ORkhBD1zpM0jYY6aB3O6mkl5Xa2fHcas0nPA7OGe2P8njx7UVkMQIUSCsjiZxJJV6ZFXXWxsbEMGjSI9957D4D33nuPQYMGERMT0+A8SdNoaFhCp8+bViUVdqwOF1a7y295T/0brcIt9pWqBZAplxJJV6ZRz/6pp55i8+bN5Ofnc/PNNxMVFcX777/P73//exYsWMALL7xAREQEy5Yt867T0DxJ4zRU9dLprBb74nI7TpfA5XJP84i9TvHx7I0haHLAEomky9Oo2C9cuJCFCxfWmt6vXz/efPPNOtdpaJ6kcXyrXuYUVmCzOQkxuU+V01Ut9mUVDpwuDYfLreIunzx74bDiPLUPNTwe1Srr4UgkXR2ZcR2AeDx7IQSP/+VL3vvqZ+88h0v4fNZwuTRcLkFZpYPSqlRMVVGwfbsBUZyDafyvfAYgP2dNkEgkAUZAZ+N0VXw7aEsr7JRWOrzzfMM4LpeG0yVwujT+8eEhTueWAW7PXis4iZrQB323QSjKbvd0qfYSSZdFevYBiG/VS4+Ye/DtoHW4NJxVfyXlds6W24CqME5lMWpIJFDt0Uutl0i6LlLsAxDfqpeapvl5857PBr2K0ymqbgYCh1PD4XDPUxUFUVmCEhIBuGP1CjJmL5F0ZWQYJwARPqmXLs0t5h48n0OMOq9X73QpOF0anqVURUNYS33EXgq9RNLVkZ59AOLx7D3/HS7/0A2A2ajH4dTcNwNN+C2jd1aCEF6xlzXsJRKJFPsApObLVDU7ZQHMJh1Wh8s73zeub3SWA1R79kjPXiLp6kixD0Bqlklw1uHZhxj1WO3uEscuzR2z96BzurNyfGP2qtR6iaRLI8U+AHHV8Ox9QzTOqto4ZqMOu6N6uqdkAoDBXlPspWcvkXR1pNgHILU9e98O2irP3uTft+7n2Tuq8u2rUi9lzF4ikUixD0AaitlXd9Dq6l1f5ygHRQWTuwCa9OwlEokU+wCkoTCOq8rLN5vqz5rV2UpRzOEoivv0ypi9RCKRefYBiND8v3tCN2s3H2HvsXzAnWdfHzpHmTdeD9Kzl0gkUuwDElc92ThfHcii0ubuiDUZ6hd71V6OYq4erUbG7CUSiQzjBCA1Y/YOp6DS5vQKvV6notfXf+qUGmIvPXuJRCLFPgCp1UHr0igqtXm/G/QKel39p061l6OYfMVeevYSSVenVWGc06dPc88993i/l5aWUlZWxo4dO8jIyMBoNGIymQCYP38+EydObJ21XYSaHbQA+cWV3s86VcVQj9grCKjDs5fljSWSrk2rxL579+5s2LDB+33p0qW4XNUv96xcuZLU1NTW7KJLIuoYaDy3qFrs9Tql3jCOWbGjIPzEfsygRJJjQ9veUIlE0mlosw5au93Oxo0b+dvf/tZWm+yy1OXZ556tFnuHU0NfTy5lmOIO9/iGcc7vG8v5fWPb2EqJRNKZaDOx//TTT0lMTGTIkCHeafPnz0cIwahRo3jooYeIiIhoYAsSDzXfoAXI8/HsrXZXvZ69xSP25vD2MU4ikXRK2kzs33rrLa699lrv97Vr15KcnIzdbmfp0qUsWbKEFStWNGubsbFhjS9UD/HxnVfs9PraaZWFZdUdtC5NEFdPWCZUdS8XnZSAOQCPQWc+LzWRbQk8gqUd0PZtaROxz8nJYefOnSxfvtw7LTk5GQCj0cicOXO46667mr3dgoKyWpkpTSE+Ppy8vNJmrxcoWK2OWtOy8suJsBgoqXDPK/fJzvEltMqzL7aqlAbYMejs58UX2ZbAI1jaAS1vi6oq9TrJbZJ6+c477zB58mSio6MBqKiooLTUbagQgg8++IBBgwa1xa66BDVfqgJ3MbRucdXevF5fd8ze49krJtkhK5FIqmkTz/6dd97h8ccf934vKChg3rx5uFwuNE2jX79+LF68uC121SWo72kmMcbCoZNnAdCrdd+nQxWbuwia0dJe5kkkkk5Im4j9pk2b/L736NGD9evXt8WmuyT1iX1cpNn7ub4O2lDFCqYw+casRCLxQ75BG4DUJ/ahZoP3s15XfxhHMcsQjkQi8UcWQgtA6uuTtpj1GA0qg3vF1PsGbYxajhKa1I7WSSSSzogU+wDEpQn0OtVv7Flwe/YvPjwFgPKqjB3/5QQJumJ0UWnn0FqJRNIZkGGcAETTBIY6sm0s5up7s6eD1qhXvXVvIpVKzIoTNbrbuTFUIpF0GqTYByCaEOjqyLbxE/uqm4Fep3jj94m6YgDUqORzYKVEIulMyDBOAOL27GuLvW8HrU5VURR3Vo7eJehONheaDwNS7CUSSW2k2AcgmhB1ZttYaow7a9Cp6FUVvc7Fr0O+Ik4tAUAJiTwndkokks6DDOMEIJ4O2pqoNSpd6nUqOp1Ckq7EK/TZWpTMsZdIJLWQnn0AomnCO8ZsXVk5HvQ6hf7KGS7WbwFgRekV2PXh/PGcWSqRSDoLUuwDEHcYx+3ZmwwNiL1eZYb2CaFKBWdIJFeJw6TWPxC5RCLpukixD0DcYRx3KEbXwFizMbpyQqlgjzKEb9R0DDoVXT2Dmkgkkq6NjNkHIMInZt+QePdRcwA4oB+KzRCOTqcgtV4ikdSF9OwDEJdP6mV9nr1WVsBg5Tg2TJSZEjCr7swc2TcrkUjqQop9AKKJao9er1N47IZRRIeb/Jap/Og5+mqncXYbweyxAwFY/fa+c26rRCLpHEixD0B8X6rSqSr9u/vnzWvWUrTC0xiGTScs/RdE642AO3OnrvFrJRKJRIp9AOLSqssl1PVylZb9o3terzSUKqEH0OkUqDtxRyKRdHFkB20A4tK0BmP2zuwjoOrRxffxm67XVRdFk0gkEl9a7dlnZGRgNBoxmdwx5fnz5zNx4kT27NnDokWLsNlspKSk8MwzzxAbG9tqg4MdTRMI4a5mCXVn47hyjqLG9/bz6sEt9i6XDONIJJLatEkYZ+XKlaSmpnq/a5rGI488wh//+EfS09N54YUXWLFiBX/8o3y3szFcmjsOYzB4wjj+nr0Qwh2vT72w1rp6nYJD5l5KJJI6aJcwzv79+zGZTKSnpwMwe/ZsPvroo/bYVdDhrPLMPSNR6WrE7EVZATisqNEptda9ZHQPZozt2f5GSiSSTkebePbz589HCMGoUaN46KGHyMrKolu36gE0YmJi0DSNs2fPEhUV1Ra7DFpcVWMSGj21cWrUtdeKzgCgxtQW+2H94trZOolE0llptdivXbuW5ORk7HY7S5cuZcmSJVx88cVtYRuxsWEtXjc+PrxNbDjX6EqsAERHhri/6xS/tpw9VkAlkNA/FV1I52tjZz0vdSHbEngESzug7dvSarFPTnYPlGE0GpkzZw533XUXN954I5mZmd5lCgsLUVW12V59QUEZWn2jbzdAfHw4eXmlzV4vECgodou9zeYeY1anqn5tqTx1HCUkksIyoKxztbEzn5eayLYEHsHSDmh5W1RVqddJblXMvqKigtJSt0FCCD744AMGDRrE0KFDsVqt7Nq1C4B169Yxffr01uyqy+Cs6qD1ZuPUiNlrhafrDOFIJBJJQ7TKsy8oKGDevHm4XC40TaNfv34sXrwYVVVZvnw5ixcv9ku9lDSOJ3VSr1NR8M/G0c5moeX/jHH0LzvIOolE0llpldj36NGD9evX1zlv5MiRbNy4sTWb75J4atfrVBVVVfw8e/sPW0HRYRg4sYOsk0gknRX5Bm2A4cnG0esUFEXxZuMIIXD++A363mmoFjnGrEQiaR5S7AMMTxhHp1NQ1eqYvSjJQVQWo+s+tCPNk0gknRQp9gGGJ4yjV911bjzlElxZRwDQJQ3oMNskEknnRYp9gFEdxlHpkxxBn27ukI0z+wiKKQw1qltDq0skEkmdyBLHAYa3g1an8Miv0rz5tq6sw+iSBqDIqpYSiaQFSM8+wPDUxvGtdqmV5CJK89ClDO4osyQSSSdHin2A4al66VvH3nn6gHta9yEdYpNEIun8SLEPMKpfqqr27F2n96OExqBGJneUWRKJpJMjY/YBhqdcgie/XrNV4jxzAEO/MTJeLzlnuFxOiorycDrtHW1Kk8nNVdG04BiXs7G26PVGoqPj0emaLuFS7AMM3zx7gNJ9W8FhxXDe5I4zStLlKCrKw2y2EBqa1GmcDL1exekMDrFvqC1CCMrLSygqyiMurulP+zKME2B48+yrYvaluzejxvdBje/bkWZJuhhOp53Q0IhOI/RdCUVRCA2NaPZTlxT7AMOTZ69TFYTTjj33JPqew+VFJznnyN9c4NKScxO0Yu9wurxecmei2rNX0EpyAFAjkzrSJIlE4sPSpb/nrbfe6Ggzmk1Qin1uYQX/b8U2XvngUEeb0myqPXsV7Ww2AGqUFHuJxIPT6QzKfbU3QdlB+/y63QDsO17QwZY0H6dLoCjuEWccxVWefURiB1slkXQsF16Yzs03387XX3/J2LHjmTNnLqtWPcexY0ex2+2kpaXz4IMPc/LkCR577FFee+0/OJ1OLr98Gr/5za3MmXMjn3zyMZ9/vpXf/34pr7/+Gp98shmXy4nRaGL+/AUMGDCwzn1dffW1PPXUYgoK8klKSkb1GRd6w4a3+c9//o3BYEQIjSVLnqZXr94dcowaI+jEXgjBsTNnAYgKM3asMS3A5dK8nbNacTa6sGgUY0gHWyXp6ny5L4svvs9ql21fOCyZCec3nlViMpl4+eVXAXj66ScZMWIkCxY8gaZp/OEPC9m4cQMzZ15NRUU5+fn5ZGdn0qdPP3bt2smcOTfy7bc7SE8fDcD06Zfzq1/dAMDOndt55pk/smbNP+rc1+OPP8Lw4WnccssdnDlzmptumsPYseMBeOGFP7F27VvExcVht9sDOvUz6MS+tNJBhdX96FVh63yPYC5NeEslaMXZGGNl4TOJBGDGjJnez1988Rk//HCAdevWAmC1WklKcj8BjxyZzrff7iArK5OrrvoFa9e+isPhYNeuHdxww00AHD78A//61yuUlBSjqiqnTp2sd1/fffctDzzwCAApKd29Nwz3vkazdOliJkyYyPjxF5KS0r1d2t4WtErsi4qKePTRRzl58iRGo5FevXqxZMkSYmJiGDhwIKmpqd5HnuXLlzNw4MA2MbohcosqAUiMsVBcZmv3/bU1zirPXtjK0QpPYxg6qaNNkkiYcH7TvO/2JCTE4vNN8D//s8JPXD256aNGjebbb3eSmXmGRYueZM+e7/jvfzchBHTrloLD4eCJJ37Ln//8VwYOPI/8/DyuvnpGA/uqn//5n2f44YcDfPvtLu67707mz/8d48dPaIvmtjmt6qBVFIXbbruNTZs2sXHjRnr06MGKFSu889etW8eGDRvYsGHDORF6gNyiCgB6J4VjtbvQqjo8Owsez962/Q1w2okYeXFHmySRBBwTJkzitdf+icvlAuDs2bNkZp4BYNSo0Wzf/jWlpaUkJCSSnj6Gv/3tJa9HbrfbcLlcJCS4nwTefvvNBvc1alQ677//LgCZmWfYtWsn4O68zcw8w+DBQ5k79ybGjBnH0aOH26W9bUGrPPuoqCjGjh3r/T5ixAhef/31VhvVGnKLKlEVGBhRyRG1jEq7k807TqEJwbWT+7Vq20dOneWtbceYPzsNg759Epncnj04jnyFYeBETEl9Ia+0XfYlkXRW7r//YV54YSU33fQrFEXBYDDy4IPzSUhIJiEhEYvFwrBhIwC3+OfkZDNyZDoAoaFh3Hrr/+P2228kIiKSqVOnNbKv+Tz11GL++99NJCd3Iy1tFACaprF06e8pKytFUVQSExO5885727XdrUERQrSJ66tpGrfccgsZGRnceOONDBw4kCFDhuByuZg0aRLz5s3DaGxeh2lBQVmzPfM1G/aTmPM5U5WdVLj06C95iNWflYKAxTePbnwDDfD+1z/z1rbjLL9zPHFR7dNpuubdA2Rm5fKw8k9M439FSsYvyQsSsffU5g8Ggr0t2dknSErq1UEWtYyuUi7BQ13nSFUVYmPD6t5mWxn35JNPYrFYuOEGdw/31q1bSU5OpqysjEceeYTVq1fz4IMPNmub9RndEOcXbmYo32NNHEblmePEff5nBtjTKVRiiI8Pb/b2/KjqfzCGGFu/rXrQGXREG2zghMhE92Nme+2rI5BtCUxqtiU3V0XfTk+v7UlntLk+GmuLqqrN+g22idgvW7aMEydO8OKLL3o7ZJOT3Z05YWFhzJo1i1deeaXZ222JZx/dbyjWyFHkR57PPw59xmPmzVzBVio1I+s39uOsZuHyC/o02xaAvEJ3f8CZrGIiTLoWbaMxKirshGruTuZSh5EwCGoPsrMS7G3RNK3TecldzbPXNK3WeWvIs2/1bfDZZ59l//79rF692humKS4uxmq1Au5OjE2bNjFo0KDW7qpJnDfpEgZfNBOL2UCeFsHx4feypjQDFY1h3z/DgH1/Rqt6WcnDnqP5vP3Z8Ua3XWF1AFBubb+UTpcmCFPcYq+ERLbbfiQSSdeiVZ790aNHeemll+jduzezZ88GoHv37tx2220sWrQIRVFwOp2kpaVx//33t4nBTcVidjct02bhgKM7L5VOY3RENsPEIcrf+QMh0+5C3+N8AL45mM3eYwX8YlLDlSU9efvtmb/vdGmEqVZwgWKJaLf9SCSSrkWrxH7AgAEcPlx3qtHGjRtbs+lWE2JyN82Td3/MmciJ4iQ2i74sitlJ5aY/ETL9QfTdh1BSbsdmd+FwujDo6w/PeDz6inb07J0uQRgVoKgoptB2249EIulaBE9vRg1CqmLqeWcrvdOcLkGhFk5O+t2oUUlUbnoOx9GvKKlwAILSCkeD26wO4zS8XGtwaRqhVKKERKAoQXt6JBLJOSboyiV40KkqZqOOnKLKWvPyK1T6z1xA5cd/xrplDbdp4YhIQUV2d2Ii6n/5q+IcefYWKlBCgifTQyKRdDxB7TqGmPQUldYumZBfXIliDiPksvnoB0+jVDNhUFxEbH2ayk1/QisvqrWOJoRX5NvVs3cJQrQK2TkrkXQhzkWN/KD17MHdSVuX2BeUuDOFFJ0e24jreO6LFCKUCuadX0jCmS9w/ucxDAMnou81gkIRxg+nyhidNoBYtYR0408UVaa3m80uTcOiVKCEyM5ZieRc4XQ60evPjRyey335EtRiHxdh5kxeea3p+cVW7+eScvc4jiXCwrG4YfS54GJsu97BcfATHPs3EwKMBCqPJXBveBnRugq0ku+pePdzdD1HYBgwHjU0us1s1lxOzGoFqiWqzbYpkXR2Lrwwndtvv4vPP99GcXExv/3t4+zatYPt27/C6XTy5JPL6N+/HwUF+fz+949TXl6O3W7nggsmcPfddWcCNqVG/rx5D3LmzKmgqJEf1GKflhrP3mP+A5j0SQ7nxzPFlJTbiQg1esUe4FhmCQnRSYy46G6EvYJTB77nwy17CVEcTNCdJkxx8PfyDFJDzzLJUYB9x3+w73gTIpMp0czEpg7H0GckanT3Fo/fGaflo1NdqLE9W9V2iaQtcRz5Esfhz9pl24aBkzCkNl4pMiwsnJdffpVPP/0vv/vdw/z+9//DnXfey9q1/+TVV//OkiVLCQsLZ9my57BYLDidTh566F6++eYrxo27oM5tNlYj//333+XKK6855zXy33jjbaKiYtu0Rn5wi/2AOP7xof+0ayb15bk39rJpx0lmTe1PsY/Y7zyUy3dH8vjzA5MoLBU894UTxTiE6HATW7MGo6KRGBvGwWIr0ReeR3oyuH7awekf9mMtLiDi23dwfPsOSng8+l4j0CX0RY3thRqZhKLW3z2y+0gevZMjiA430U24X/jSJQ1ol2MikXRWpk27BICBA88DFCZMmFj1fRDbtm0B3G+VvvDCn9i373tAUFBQwNGjR+oV+8Zq5HsqY57rGvlLlizmggsubNMa+UEt9uEWI/1SIoiPDOG7o3nYHRr9ukUy6rwEPtubyTWT+lJS4Rb72AgzBSVWXJrg6wPZvPvlTwghmP+rkRw5WcRPWaVoqMRFhpBVUMGadw+yb0gS12dMZ8XWSGx2F1POC2X2wEqcP3+H44ctOPZ/7DZEp8ehD8MQGkG5MBPecyCmnkPRJfbjyOlSVr39PYmRRh69YQw9lBwqdeGEhcZ04JGTSPwxpE5okvfdnnje0FdVFaPR4J2uqqq31PEbb6yltLSENWv+gclkYtmypdjt9Y9r0ViNfA/nukb+kSM/sGPHjjatkR/UYg/w+Fx3Z+qDf/4Cl8uB2ahj/OBEdh3K5Yt9WfxwogijXkXzKf75r82HCTHqWXDDSFLiQomLNPOvzUcA0Ovc4ZnIMCNfH8jm+2P5OJ0aqd0j2f5TGamp55F20UT2HcqCkhxCKrIoz/6Z8rNFRFbYCRX5WIqOUrl3IxjMhDpNrIguw6C4KH/zLQbqyiky9yOhhWEgiaQrU1paSmxsHCaTiby8XL74YhtXX31tk9b11MifP38BOp2Os2fPUlFRTrduKYwaNZoXX/wzUVHR3hr5L720mtGj3SXeW1oj/6abbvPWyE9PH4PT6SQnJ5shQ4YycOBgMjNPc/ToYSn2zcFi0iM0gaIoDOkTg8mo49WP3G//9k4K5+dsd0GhCIuBkgoHM8b1pHu8u6CQyaDj8vG9eP/rE4w+L4HdR/N59Fdp7Pghlw++OcGdVw3BYjbwzOu7WfPuwRp9ARYUBjN2SCLfHs4jMtSIo7SUP1weQeWJg/x45BTJ3YeQUw6msjNECYXsyPNJ7YiDJJF0cmbNms0TT/yWuXOvIz4+kVGjml7WvK4a+ffd9zDduqWc8xr55eVlgNKmNfLbrJ59e9CSqpdQdxW/p17dRaXNydLbxwHwz48Ose94AfN+MYxucaH8a/Nhvvg+i6sv7MOW3Wf4nzvGeUsuePAMGej0GRTc93NZpYPPv8/k/7Ye47qp/Rk72H2XN+hVQs0GnC6NvLOVLPrbDvp2i0ABsgorWH7nBez+MY817x4EYPa0AVwyuke9bemsyLYEJrKefeAR0PXsA52E6BCsNpf3+w2XuH1nXVXH6U0zzmPuJQPR6RQuHdsTk6F2jRyPqHv+1/wcFmJgxtheZKR1x2Sse/3k2FDuuHIIL204gCYE103tj8moY3i/OMJCDAzoHknGyJS2abREIpFU0WXE/uYZg4DqpwRdjewYVVFQ9e44eV1C3xzqEnpfRp+XQGr3SCpsThJj3J02ISY9y+4cj9moa3HapkQikdRHlxH79hoztqVEhpmIDDP5TasZNpJIJJK2IrAUUCKRBAwB3J3X5WnJuZFiL5FIaqHXGykvL5GCH4AIISgvL0GvNzZrPRk3kEgktYiOjqeoKI+ysrMdbUqTUVW1zUoLdDSNtUWvNxIdHd+sbbar2P/0008sWLCAs2fPEhUVxbJly+jdu3d77lIikbQBOp2euLjkjjajWQR7OmxradcwzuLFi5kzZw6bNm1izpw5LFq0qD13J5FIJJJ6aDexLygo4ODBg8yc6S7+M3PmTA4ePEhhYWF77VIikUgk9dBuYZysrCwSExPR6dw55zqdjoSEBLKysoiJaVqRL1Vteb55a9YNNGRbAhPZlsAjWNoBLWtLQ+sEdAdtdHRoi9et75XhzohsS2Ai2xJ4BEs7oO3b0m5hnOTkZHJycrylR10uF7m5uSQnd65OH4lEIgkG2k3sY2NjGTRoEO+99x4A7733HoMGDWpyCEcikUgkbUe7Vr08duwYCxYsoKSkhIiICJYtW0bfvn3ba3cSiUQiqYeALnEskUgkkrZBlkuQSCSSLoAUe4lEIukCSLGXSCSSLoAUe4lEIukCSLGXSCSSLkBAv0HbXDp7lc2MjAyMRiMmk3sEq/nz5zNx4kT27NnDokWLsNlspKSk8MwzzxAbG9vB1lazbNkyNm3axJkzZ9i4cSOpqe7xfRs6H4F6ruprS33nBgjY81NUVMSjjz7KyZMnMRqN9OrViyVLlhATE9OgzYHYnobaMnDgQFJTU1Grhhpdvnw5AwcOBODTTz9l+fLluFwuhgwZwh//+EdCQkI6sikA3H333Zw+fRpVVbFYLDzxxBMMGjSofa8ZEUTMnTtXrF+/XgghxPr168XcuXM72KLmMXXqVHH48GG/aS6XS1x00UVi586dQgghVq9eLRYsWNAR5tXLzp07RWZmZi37GzofgXqu6mtLXedGiMA+P0VFReKbb77xfn/66afF7373uwZtDtT21NcWIYRITU0VZWVltdYpKysTF1xwgfjpp5+EEEI89thjYtWqVefE3sYoKSnxfv7444/F1VdfLYRo32smaMI4wVplc//+/ZhMJtLT0wGYPXs2H330UQdb5U96enqtMhgNnY9APld1taUhAvn8REVFMXbsWO/3ESNGkJmZ2aDNgdqe+trSEJ999hlDhw71er+zZ8/mww8/bE8zm0x4eLj3c1lZGYqitPs1EzRhnLaoshkIzJ8/HyEEo0aN4qGHHiIrK4tu3bp558fExKBpmvdRLlBp6HwIITrluap5biIiIjrN+dE0jddff52MjIwGbe4M7fFti4e5c+ficrmYNGkS8+bNw2g01mpLt27dyMrK6giT6+Txxx/nyy+/RAjByy+/3O7XTNB49sHA2rVreffdd3nrrbcQQrBkyZKONklSRWc/N08++SQWi4Ubbriho01pNTXbsnXrVt5++23Wrl3Ljz/+yOrVqzvYwqaxdOlStm7dyoMPPsjy5cvbfX9BI/bBUGXTY6vRaGTOnDl89913JCcn+z2uFhYWoqpqwHhZ9dHQ+eiM56quc+OZHujnZ9myZZw4cYLnn38eVVUbtDnQ21OzLVB9bsLCwpg1a1a95yYzMzMgf2NXX30127dvJykpqV2vmaAR+85eZbOiooLSUveYk0IIPvjgAwYNGsTQoUOxWq3s2rULgHXr1jF9+vSONLVJNHQ+Otu5qu/cAAF/fp599ln279/P6tWrMRqNQMM2B3J76mpLcXExVqsVAKfTyaZNm7znZuLEiezbt4+ff/4ZcLdlxowZHWK7L+Xl5X7hpE8//ZTIyMh2v2aCqhBaZ66yeerUKebNm4fL5ULTNPr168fChQtJSEjgu+++Y/HixX6pcHFxcR1tspennnqKzZs3k5+fT3R0NFFRUbz//vsNno9APVd1teXFF1+s99wAAXt+jh49ysyZM+nduzdmsxmA7t27s3r16gZtDsT21NeW2267jUWLFqEoCk6nk7S0NB577DFCQ90DH/33v//lmWeeQdM0Bg0axNNPP43FYunIppCfn8/dd99NZWUlqqoSGRnJb3/7W4YMGdKu10xQib1EIpFI6iZowjgSiUQiqR8p9hKJRNIFkGIvkUgkXQAp9hKJRNIFkGIvkUgkXQAp9hJJA7z44os8/vjjLVp3wYIFPPfcc21skUTSMoKmNo5E0h7ceeedHW2CRNImSM9eIpFIugBS7CVBRU5ODvPmzWPcuHFkZGTw6quvArBq1Sruu+8+HnjgAdLS0rjmmms4dOiQd701a9YwceJE0tLSuPTSS/n666+9682fP9+73CeffMLll19Oeno6c+fO5dixY955Bw8e5JprriEtLY0HHngAm83mZ9uWLVu46qqrSE9PZ/bs2U3av0TSZrSg7r5EEpC4XC5xzTXXiFWrVgmbzSZOnjwpMjIyxGeffSZWrlwpBg8eLD788ENht9vFyy+/LKZOnSrsdrs4duyYmDRpksjOzhZCCHHq1Clx4sQJIYQQK1euFA8//LAQQojjx4+L4cOHiy+++ELY7XaxZs0acdFFFwmbzSZsNpuYMmWKeOWVV4TdbhcffvihGDx4sHj22WeFEEIcOHBAjBs3TuzZs0c4nU7x9ttvi6lTpwqbzdbg/iWStkJ69pKgYd++fRQWFnLvvfdiNBrp0aMH1113HR988AEAQ4YMYfr06RgMBm6++Wbsdjt79+5Fp9Nht9s5duwYDoeD7t2707Nnz1rb/+CDD5g8eTITJkzAYDBw6623YrVa2b17N3v37sXhcPCb3/wGg8HA9OnTOf/8873rvvHGG1x//fUMHz4cnU7HNddcg8FgYM+ePU3ev0TSGmQHrSRoOHPmDLm5ud5RlsBdCjY9PZ1u3bqRlJTkna6qKomJid7lH3vsMVatWsWPP/7IhRdeyIIFC0hMTPTbfm5urt9gGJ5ywTk5Oeh0OhITE1EUxTvfd9nMzEzWr1/Pa6+95p3mcDjIzc1lzJgxTdq/RNIapGcvCRqSk5Pp3r07u3bt8v7t3r2bv/71rwBkZ2d7l9U0jZycHG/lyiuuuILXX3+dLVu2oCgKK1asqLX9hIQEv/roQgjv6ELx8fHk5OQgfOoK+i6bnJzMnXfe6Wfb3r17vcPMNWX/EklrkGIvCRqGDRtGaGgoa9aswWq14nK5OHLkCN9//z0ABw4cYPPmzTidTv75z39iNBoZPnw4x48f5+uvv8Zut2M0GjGZTN6BMXyZMWMG27Zt4+uvv8bhcPD3v/8do9FIWloaI0aMQK/X8+qrr+JwONi8eTP79u3zrjtr1izWrVvH3r17EUJQUVHB1q1bKSsra/L+JZLWIMM4kqBBp9Px4osvsmzZMqZNm4bdbqdPnz488MADAEybNo0PPviA3/72t/Tq1YtVq1ZhMBiw2+387//+L8eOHcNgMJCWllbnsIN9+/blmWee4cknnyQnJ4dBgwbx4osvegfSWLVqFU888QTPP/88kydP5uKLL/aue/755/Pkk0+yZMkSTpw4gdlsZuTIkaSnpzd5/xJJa5D17CVdglWrVnHixAkZHpF0WeSzokQikXQBpNhLJBJJF0CGcSQSiaQLID17iUQi6QJIsZdIJJIugBR7iUQi6QJIsZdIJJIugBR7iUQi6QJIsZdIJJIuwP8HKbDPCynjUs0AAAAASUVORK5CYII=\n" + }, + "metadata": {} + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Episode:10/50, reward:188.0\n", + "Episode:20/50, reward:200.0\n", + "Episode:30/50, reward:200.0\n", + "Episode:40/50, reward:200.0\n", + "Episode:50/50, reward:171.0\n", + "results saved!\n" + ] + }, + { + "output_type": "display_data", + "data": { + "text/plain": "
", + "image/svg+xml": "\n\n\n \n \n \n \n 2021-05-04T19:04:05.465993\n image/svg+xml\n \n \n Matplotlib v3.4.1, https://matplotlib.org/\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n\n", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXsAAAEcCAYAAAAmzxTpAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Z1A+gAAAACXBIWXMAAAsTAAALEwEAmpwYAABfK0lEQVR4nO2dd5wU9f3/XzOzM7u31/sdRy+HiJSDo55SDiO9xagExdh/VvSrRElEIZTEUwMKgSiRmBiIGqNCKAoqoGChd6QdcHC91y2zM/P5/bG3c233tu8dt5/n48GD293PzHw+OzPvfc/r8/683wwhhIBCoVAoHRq2rTtAoVAoFP9DjT2FQqEEAdTYUygUShBAjT2FQqEEAdTYUygUShBAjT2FQqEEAdTYd2DWrFmDBQsWtMmxp06digMHDrTJsdsbJpMJjz/+OIYOHYr58+e3dXda8NVXX2Hs2LFIS0vD2bNn27o7fiU3Nxd9+/aFJElt3ZWAQ409xS9s374dI0aMaOtutAu+/PJLlJaW4sCBA1i9enWLz9esWYP+/fsjLS0NaWlpmDhxIpYuXYri4uIm7aqrq7F48WJkZGRg0KBBmD59Oj7//PMmbTIzMzFq1CgYDAb1vU8++QTz5s1z2L+srCy88sorOHbsGG6++WYvR2tl3759uPfee5GWloaRI0fivvvuwzfffOPx/jIzM/HDDz+or21G2/adZWZmYv369b7ousf8+OOPmDRpEgYNGoR58+YhLy+vTfvTHGrsA0hH8SY6wjgCOYb8/Hx0794dGo3GYZvJkyfj2LFjOHjwIP7yl7+gtLQUv/zlL1WDL4oiHnjgAeTn5+Ojjz7C4cOH8dvf/hZvvPEGPvjggyb7UhSlxXvO+tenTx+PxibLcov3vvzySzz77LOYNWsWvvvuO/zwww+YP38+9uzZ4/b+nZ2nQ4cO4dixY/jzn/+MtWvX4rvvvnP7GL6gvLwcTz/9NJ599lkcPHgQt9xyC/7v//6vTfriCGrs61m/fj1uv/12pKWlYcqUKfjqq68AWG+y9PR0XLhwQW1bXl6OgQMHoqysDACwZ88ezJw5E+np6ZgzZw7OnTuntrV5HNOnT8fgwYMhSZLDYwHWm+e1117DiBEjkJmZiY0bNzZ57KypqcHvf/973HrrrbjtttuwatUquzecPY4fP445c+YgPT0dM2bMaCKzfPrpp5g8eTLS0tIwYcIEfPTRR+pnBw4cwJgxY7B+/XpkZGTgd7/7HdasWYNnn30WL774ItLS0jB16lScOnWqybhtnpiztmfOnMGsWbOQlpaG+fPn47nnnsOqVascjuM///mP2tcpU6bgzJkzAIC+ffsiJydHbbdw4UJ1P/bGMHny5CYGSJIkjBw5Ut1fa99Xc7KzszFv3jykp6dj6tSpqhe7evVqrFu3Dl988QXS0tLwySeftHKGAJ7n0adPH6xatQoxMTF4//33AQBbtmxBQUEB3n77bXTp0gU8z2PMmDFYtGgR3nrrLdTV1an7ePjhh/H3v/8d1dXVrR5LFEWkpaVBlmXMnDkTt99+e6tjsX2nixcvxqOPPorBgwe3+E4IIXjttdfw5JNP4q677kJ4eDhYlsXw4cOxfPlyAMC1a9dw//33Y8SIERgxYgReeOGFJn1tfs88//zzyM/Px+OPP460tDT87W9/azGWtLQ09O7dGxcvXoSiKFi3bh3Gjx+PUaNG4cUXX0RNTY3d78DV+8mZHfjqq6/Qp08fTJ48GVqtFs888wzOnTuH7OzsVs9BQCEUQgghO3bsIIWFhUSWZbJ9+3YyaNAgUlRURAghZOHChWTlypVq240bN5KHHnqIEELImTNnyMiRI8nx48eJJEnks88+I+PHjydms5kQQsj48ePJjBkzSH5+PjEajU6P9e9//5tMnjyZFBQUkMrKSvKb3/yGpKamEovFQggh5MknnySvvPIKqaurI6WlpeTOO+8kH374od0xrV69mrzwwguEEEIKCwvJ8OHDyd69e4ksy2T//v1k+PDhpKysjBBCyJ49e0hOTg5RFIUcOHCADBw4kJw+fZoQQshPP/1E+vXrR15//XViNpuJ0Wgkq1evJrfccgvZu3cvkSSJvPnmm+Suu+5Sjz1+/Hjy/fffq/1w1NZsNpNx48aRf/zjH0QURbJz507Sv3//Jt938/N06623khMnThBFUcjVq1dJbm4uIYSQ1NRUcvXqVbXtSy+9pO7H3hjWrFlDnn/+ebX9nj17yKRJk1z6vhojiiK5/fbbyV//+ldiNpvJDz/8QAYPHkyys7NbnAdn56kxb731FvnVr35FCCHkueeeIy+++GKLNhaLhfTr14/s37+/yff+1FNPqWP/z3/+Q+677z6Hx2/8vTkby0svvUSGDBlCDh8+TGRZJiaTqcm+Ll26RFJTU8m1a9ccHu/q1atk//79xGw2k7KyMjJ37lyyfPly9XN790zj64kQQq5fv67eF4qikMOHD5OBAweSH374gXzyySfk9ttvJ9euXSO1tbXkqaeeIgsWLGixHSHu3U+t2YFly5aRV199tUn7qVOnki+//NLh9xBoqGdfz+TJk5GYmAiWZTFlyhR069YNJ0+eBABMnz4d27dvV9tu3boV06dPBwB8/PHHuOeeezBo0CBwHIfZs2eD53kcP35cbT9v3jwkJydDp9M5PdYXX3yB+++/H0lJSYiMjMRjjz2m7qe0tBTffvstfv/730Ov1yM2NhYPPPBAk745YsuWLRgzZgzGjh0LlmWRkZGBW265Bd9++y0AYNy4cejatSsYhsHw4cORkZGBw4cPq9uzLIv58+dDEAR1HEOHDsXYsWPBcRxmzpzZ5ImmOY7anjhxApIk4f777wfP87jjjjswYMAAh/v573//i0ceeQQDBw4EwzDo1q0bUlJSnI7f3himT5+O3bt3w2g0ArCe16lTp7r0fTXmxIkTMBgMeOyxxyAIAkaNGoXx48e7dF5aIyEhAVVVVQCAiooKxMfHt2ij0WgQHR2N8vLyJu/Pnz8fGzdubPG+M1wZy4QJEzB06FCwLAutVttk+8rKSrXvjujWrRsyMjIgCAJiYmLw4IMP4tChQ03aNL9nHDFy5EgMHz4cixYtwgsvvIBRo0Zh69ateOCBB9ClSxeEhobi+eefx44dO1pIQu7eT63ZAYPBgPDw8Cbtw8LCmjxxtTWORcQgY/PmzXj//ffVSRWDwYCKigoAwIgRI2AymXDixAnExsbi3Llz6iNvfn4+Nm/ejI0bN6r7slgsTSbXkpOTXT5WcXFxk/ZJSUnq3/n5+ZAkCbfeeqv6nqIoLfZvj/z8fHz55ZctZAvbJOq3336LtWvX4urVq1AUBSaTCampqWrb6OjoFjd2XFyc+rdOp4PZbIYkSXa1aUdti4uLkZiYCIZh1M9bG09BQQG6du3qdLz2aD6Gbt26oVevXtizZw/Gjx+P3bt3Y/PmzQCcf1+NKS4uRlJSEli2wXfq1KkTioqKPOqnjaKiIkRGRqp9LykpadFGkiRUVFQgOjq6yfupqakYN24c1q9fj169erl8TFfG0tr5iYqKUvfTpUsXu21KS0uxYsUKHD58GHV1dSCEICIiokkbV65pAPjpp59aXG/FxcVNHICUlBRIkqTKrjac3U9Tp05Ffn4+AOBvf/tbq3ZAr9ejtra2yf7r6uoQGhrq0jgCATX2APLy8rBo0SL84x//QFpamup92uA4DpMmTcK2bdsQFxeHcePGISwsDID1onz88cfxxBNPONx/Y0Pm7Fjx8fEoLCxUXzf+OykpCYIg2L3AnZGcnIyZM2equmljRFHE/PnzkZWVhQkTJoDneTz55JMgjRKiNh6DL4mPj0dRUREIIeoxCgoKHBqK5ORkXLt2ze5nISEhqpcOACUlJUhMTFRf2xvDtGnTsG3bNiiKgt69e6Nbt27qcRx9X81JSEhAYWEhFEVRjWRBQQG6d+/udFtHKIqCPXv2YPTo0QCA0aNHY+XKlTAYDNDr9Wq7Xbt2ged5DBo0qMU+5s+fj9mzZ+Ohhx5y+bjejqVnz55ITk7Grl278PDDD9tts3LlSjAMg61btyIqKgpff/01li5d2qSNN9dbQkJCk0iY/Px8aDQaxMbGunU/2fPwHdmBPn36NImMMhgMuHbtGnr37u3xOHwNlXEAGI1GMAyDmJgYANbJyosXLzZpM336dHzxxRfYunUrpk2bpr5/11134aOPPsKJEydACIHBYMDevXtb/Mq7eqzJkyfjgw8+QFFREaqrq5tMRiUkJCAjIwOvvfYaamtroSgKrl27hoMHDzod44wZM7Bnzx7s27cPsizDbDbjwIEDKCwshCiKEEURMTEx0Gg0+Pbbb/H999+7/gV6weDBg8FxHDZu3AhJkvD11183mbxtzq9+9Sv8/e9/x+nTp0EIQU5Ojnpj33TTTdi2bRtkWcZ3333XQhqwx5QpU/D999/jww8/bHJeW/u+mjNw4EDodDq89957sFgsOHDgAHbv3o0pU6a4/X1IkoTs7Gw8//zzKC0txQMPPAAAmDlzJpKSkvDss88iNzcXFosF+/btw/Lly/Hwww+3kBAA65PLlClT8K9//cvl43s7FoZhsHDhQqxbtw6ffvqpep0ePnwYr7zyCgCrx6vX6xEeHo6ioiK89957TvcbFxeH69evu9SHadOm4Z///CeuX7+Ouro6rFq1CpMnT25h0D25nxzZgV/84he4ePEidu7cCbPZjLVr16Jv375uPVX5G2rsAfTu3RsPPfQQ5syZg9GjR+PChQsYMmRIkzaDBg1CSEgIiouLMWbMGPX9AQMGYNmyZVi6dCmGDRuGO+64A5999pnHx7r77ruRkZGBGTNmYNasWRg7diw0Gg04jgMAvP7667BYLJgyZQqGDRuG+fPn2328b05ycjLWrVuHd999F6NGjcLYsWOxYcMGKIqCsLAwLFq0CM899xyGDRuGbdu2ITMz092v0SMEQcCaNWvw3//+F8OGDcP//vc/jBs3DoIg2G0/efJkPP7443jhhRcwZMgQPPXUU6qu/fLLL2PPnj1IT0/H1q1b1Ufs1khISMDgwYNx7NixJgatte/L3hjeeecdfPfddxg5ciT+8Ic/4PXXX3frRrdF66Snp+OJJ55AVFQUPvvsM/XJRBAEvP/++0hOTsbdd9+NQYMG4ZFHHsFvfvMbPP300w73+9RTTzWJuXeGL8YyadIkrFq1Cp9++iluu+02jB49Gm+//TYmTJgAAHj66adx9uxZpKen47HHHsMdd9zhdJ+PPfYY/vrXvyI9PR0bNmxote2dd96JGTNm4L777sOECRMgCIL6Q9Mcd+8nR3YgJiYGa9aswapVqzBs2DCcPHkSK1eudDquQMIQQouXtGe+/fZbLFmyxKMY5RuVu+66C3PmzMGdd97Z1l1pt1gsFjz66KNITEzEa6+95jeZjdJxoJ59O8NkMuHbb7+FJEkoKirC2rVrXfJQb2QOHjyIkpISSJKEzz//HOfPn8dtt93W1t1q1/A8jzVr1qBLly64fPlyW3eHcgNAPft2htFoxH333YfLly9Dp9Nh3LhxePnll9WJoI7Ixx9/jLfffhtGoxGdO3fGCy+8gHHjxrV1tyiUDgU19hQKhRIEUBmHQqFQggBq7CkUCiUIoMaeQqFQgoB2vYK2oqIOiuL+lEJsbBjKyuwvaurI0HEHF3TcwYUr42ZZBtHR9lM0tGtjryjEI2Nv2zYYoeMOLui4gwtvxk1lHAqFQgkCqLGnUCiUIIAaewqFQgkCnBr7iooKPProo5g4cSKmT5+Op59+Wi2IcPz4ccyYMQMTJ07EQw891CRfdGufUSgUCiWwODX2DMPgkUcewc6dO7F161Z06dIFb775JhRFwW9/+1u8+uqr2LlzJ9LT0/Hmm28CQKufUSgUCiXwODX2UVFRTarzDB48GPn5+Th9+jS0Wi3S09MBAHPmzMGXX34JAK1+5m8URYEsy5CVlv/spaf1JYQQKG7+82W2CkVx//gd4p+Dcbv73Xpy/trjuFv7529a+w7dxZfjdvdfIK6dQGeqcSv0UlEUfPjhh8jMzERBQQE6deqkfhYTEwNFUVBZWdnqZ7ayZf7i9L9XoYfBfvGLYjYRPR/+ExjG+VRFVa0Zyz84jOfuHoyUONdKi/19+8/4/nTL4hatEarT4E//bxTCQni3tmvOpq8u4JsjuV7to6MRrufx2v8bhRCt88ucEIJX/34QeSXtp2aor2EZBk//cgAG94lz3tgDTmaX4Z0tp2ES5RafMQAenNIPtw50rdzgFz/l4JO92T7uoeskx+qx7JERYF1IHW0wWfC79T+hxmBx6xjpfePx5GzH9ZZ9jVvGftmyZdDr9bjvvvvw1Vdf+atPKrGx7md67DRqEnJ/blnsuLq4ADdLZxFadRGhfdKd7qe4RkRZtRlVRgmD41tWAbJHbmkduiSG47bBrhXAzi2uwXfH8kA4DvEuHsPhvkrq0CkuFOOG2i/nF2wUlNZiz5FcmBSgqwvfbXWdiLySOqT3S0Rq12in7W80ZEXBx19dQJVJ8vpas8e5nHL8dctpJMeGYvTATi0+//LHKzh7rQKzJ6Ta2bolJ6+Ut9n1fL2oBvuO58GsAF2TnH9Xh84WosZgweRR3REd0XqBdBvfn8hDQbnR7XPhzblz2dhnZWUhJycH77zzDliWRXJyslqMFwDKy8vBsiyioqJa/cwdyspq3V5EENerP/qNHImSkpom73/yzXkkX8wBt+8zGKL6Ot1PUf32RSU1LfbliOo6Ef27x+D2tJYXuz3OXC3Hd8fyUFBUjXDBu8CoihoTbnLj2B2J+PjwFucot8Rq7C9cKUN0iPPL/EpBNQBgVL8EpKXG+6WfvsbeuB2hEIKPv7qAikqDy9u4Sl5pHV7beASRegHP/mogIkNbVhm7mleJU5fLUFxc7bTQimiRcel6Je4Y1sXu9ezOuD2hsNyAfcfzcPBUPkI45579kbOF4FgGM0Z3g5bnXDpGTl4lzl2rdGscroybZRmHTrJLFmblypU4ffo01q5dq5aLu+WWW2AymXD48GEAwEcffYRJkyY5/aytELQC9hr7QS44D7nYebEHc/2jqMEsuXwMo1lySTKwoROsF4ZJdP0YjqgzWhCht1/KLxiJi7R6WKVVRictrZRUWtvFRYX4rU9tCcsw4DUsRMm381bl1Sas/Pg4NByL5+cMtmvoAaB3SiRqDBYUVzo/H1cLayArBL1TIn3aV1dJjA5BWAiPS3lVLrW/lFuFrolhLht6AOA1LCyyf+cQm+PUMl28eBHvvvsuunfvjjlz5gAAOnfujLVr1+L111/H4sWLYTabkZKSgjfeeAMAwLKsw8/aCi3P4UdzH8yOOgPx5BcIuf2pVtubLe4Ze0UhMIky9Dp3jL21rT2N0x1kRUGdSUKEgxstGNEJGoTreZRUmlxqX1plbWf7kbiRIZIZxFQLYqoBMdZY/zfVYJLuHBSj7yScWqMFf/74OEyihJfmDkFCKz+UvTtbDfel3CokRutb3W92vZHt1bltjD3DMOidEolLedVO20qygisF1RjronRrQ6NhYfHxD6/TYzpr0KdPH5w/f97uZ0OGDMHWrVvd/qwt0AoczOCh9BkL6eedUKqLwUa01PZt2Ayw0eyaITbWe+d6Nzz7ENWz987Y15msxw6nxr4J8VEhqsfujJJKI8JCeLeezAIJUSQQQzWIoRKKoRLEUIlyGGEqKwExVkMxVoPU/4PF/g/c7QKQXQEAQ73uj1mU8dYnJ1BSacIL9wxC18TWf0Q6xYUiRMvhUl4VMga0Pkl7MbcKidEhbfqk2rtzJI5fKkW1QWy1H9eLayFKivpj5ip8ezT2HQVd/SOWuecYhJz7CuKpndBlzHPYXrR59ibXZtgN9QbXIxnHDanIHrX1UQDUs29KXKRO1eKdUVppRHxU23j1RLaA1JZBqS0HqauAUlf/f/1rYqgAMdYAaDp/ZQYDRhcGJiQCTEgE2Pge9X9HggkJB6Oz/mN14WB0Yfjxn2+hr+EsiCSC0Xh+rUiygrWbT+FKQTWemj0AfV2Y0GYZBr06RTqVRgghuJRXhUG9Yj3uny+wSUjZeVVI6+N4DudSblWT9q7CcywkWQEhJGDF4oPG2Av1xt7EhSO892hYzu+DduhsMDr7kxkmNzV7Y307d2QcrY88+1qj1diHU82+CfFRIThyvgSyooBjW5+eKqkyoZsT79RTiCKB1JRBqS6GUlMCUlMKpaYUSm0pSE0ZiLGlAWS0YWDCosGExoCL7w5GHwVGHwVWHwUm1Pp3QpdOKC137ckFAM5y/TBAuQQp5zj4XsM9Hs8Ppwtx+nI57p/UF0PcmMzu3TkSW/ZdgcFkgV5nP9S4uMKIWqPFbU/Z13RPCgfHMrjkzNjnVSE2QofocK1b++c11utRkhXwGte1fm8IGmOva2RYhYGTIF3YB/HsN9AOmWm3vU2zN7po7D3x7DmWhaBhfWfsqWffhPhIHWSFoKLa3OrEq6IQlFWZkN7XsaznDKtBL4VSWQilqv5fdQmU6mKQ2jKANHpkZzVgwmLBhseC6zoQTHgc2LA4MGExYENjwIRGu+R5M5x7t28h3xW1Yji4C/u9MvZFFQZwLIOxg9yL/OqdEgkCIDu/GgN62vfcbZ5/W03O2hB4Dt2SwlXP3R62p5DULlFu799m4C0SNfY+x+ZFixYZXEwKuC4DYTn9NYSBk+3eWOoErclFY292X7MHrD9C3kbj2Ix9RKgASN79cNxoEEIgVZdByr0IpTIfSmUBlIp8KJX5GCSakanrj5KKga0a+4oaM2SFIM4FGYdIZigVBVAq8qBU5EGuyAepN+wgjb57bSjYiERwCb3A9h4JNiIBTESC9X99pEsL+3yNwHM4J/dFeu4RKHUVYEM9W09QWSMiKkxwW37okRwBhrFKH46M/cXcKui1GiS7uJDRn/ROicTuo3mQZAUaruX5Kq82o6LG3OoPEyEEpK684bqsvzbTS3MRGhYLi5ThzyE0IXiMPd9UMhEGTYZxWxYsF3+A0G9ci/Zm0T3P3tYuxA0ZB7BGjfjKs4/QC6ipdv2x/kaDWExQynMhl+dCKbsOpSIXctl11IqGhkZCCNioTuC6DIJUU4GZ8lEYv8+HrH8EXGJvu/u1hWfGRzb8IBBFAakuhlx2DUrZNesxK/JAakqhauesBmxkEtiYztD0SAcblWR9HZnkUB5sS3gNh1NMX6STw5Au/Qhh0BSP9lNZa0ZUmHuyBWB96u2SENaqbp+dV4VeKZEurVz1B0SRoFQUgIgG9EkOwy5ZQU5RDXp1amnQmz+FENFovU7Kr0Opv0bl8lzA0uie1IaCi+oEUReHgdI1mKvLgLDArI0JHmNf79nbPHYu+Sawcd0hnvwS/E1jWnha7mr2ticAzzx77409r2GhFTj4b6lJYCGmWsilOZBLc6CUXoVcehWkurihAa8DG9MZfK/hiOjSCwY+Fmx0J+vkZL2h4GUFf3n7X7hXcxiGLSvA3zwe2mF3gtE29RpLymvQmStDUtkhmHILIJddh1J+HZBEawOGAxuVBC6+B9jUW8FGdwIbkwI2IhEMG5hHcF8g8CwK5QhwiX1gubAf/MDJHk0OVtaa0SnWM8+7d0okvj9VaHcexWCyIK+0DsP7eS6nuQOxmK0/5GU5UErr/y/PAxTrvdyH4/F4eBwMRwshCxlgY7uodkIx1aDq4jFMCr2A+JNnULsnp94RqEcIARfTBXyfUWBjOoON6mS9PnXhYBgG54+cRu8jbwLXjgKdqLH3KWo0Tr2xZxgGwqDJMH3zV+uEVfchAADFUAWl+DIG1X6PIWHF+J9hiMPHuMaonr3WvZvfVzJOWAgfsFl9X0MkEUppDuTibOu/kqsgNSXq50xYLLi47mBTM8DGdAEX0wVMeKx640XGh0O0s7JQw7HID+mDrTEDMTfhLCxnvoZ05Qi0w38FMAzk4iuQS67gltIcDIqUgaOARdCDi+0C/qax4GK7go3tar1JOe9yF7UHBA0Ls0WGpu+tMH/3PpSSK+ASerq9n8paETd3i/GoDzZpJLe4Dt2apSLIzq9W2/gaoihQKvMgF1+GUnwZcsllq2Gvn0thtGFg47qBv+UX4OK6gdFoIeWfReypw0jI3wXDZ7vUNkpVIUhtGUYBgBZQyhOsjsBNY8HFdLH+KITGtHo/kvBE5EnRSMg9CmCaz8drj6Ax9jYZx9zIi9b0SAcTHgfx0KeQsg9CLs5WjcwgsJB5BpFhBhiNExEe1vrKSoNZglbgnEZ9NEen1aCqTnRzNE2pNVi8TqQWSJS6ivqVzNmQi7KhlOUASv2PcFgsuISeYPuNBRfXHVxcd68kkbhIHQpqZOhm3Qu+TwZM+96H6dsN1g81WnDx3XFRn4bT1RH4zb2TwITH37A/ms7gNZx1QrDnMJi/3wTLhf1uG3uzRYbRLCEq3LNgAHVxVV5VC2N/MbcKLMOgR6cIj/bdGCIarddX4QXIhRchl1xpWH8g6MEl9IQweDDY+B5W427HOGu6p+GbssHIvZqLRRPDIOedhVJ+DVxCLyg3jceaPdXoP2QQpo/v73b/eA2L42I3TCk/4dX8iTsEjbHneRYMGjx7AGBYDsKgqTDv/yeIaACX0Atc/0ywCb2xYmsxomvO48Hw72A5sR3I+FWr+zeYJbclHMDq2RdVeCnjmNq3sVdqSiAXnIeUfx5y4fkGOYYTwCX0gDBgItjEXtbJTH2UT48dHxWCE9nWwjlcfHfoZ70KOf9nMPposFHJYFgWX/3rCDSRTKuL7DoCAm9Nl8AIemh6DIEl+wC0I+e4FXNfVWsGAI80ewCIjdAhKkzApbwqTBjaucln2XlV6JIQpq4sdwfFVAM5/1y9cb8ApewaQAjAMGBju4FPzbDe3/E9wUQmuvyD3jslEj+dKUJVwhDEpzZMpv58tRwXLMcxtVuS230FrHH2x8VumKo/DunKEQi33O7RftwhaIw9yzAQ7OjjfL9x4HsNb6Hj1kklKGF747D5Ooae3QE5NR1cfHeH+zea3DP2RDJDzj+PfuZTuCh6l9mv1mBB54T2MyGoGKsh552FlHsGcv5Za+ghAGhDoUlKBXdzJrjkvtbHXda/l2BcVAiq60SYLTK0PAeG5aDpfEuTNiVVRgzo0baLeAKBoOHUxYJ86q2QLv0E6dpx8D1dD8OsrLU+hXpq7NVUBM1CGmVFweX8atzqZHWtDWIxQy48DynvLOS8n63GHcTqQCT2gpA2A1xSH3AJvcAInuc7sklKl/KqEN8ooss2OdvLw6cQXsOiWImEGJoE7sohaux9jZbnmnj2gPXig7blZJNJlBEdrsV/y4ZjcEQ5THvehf6Xf3DoBRnMUquROIQQa/TI9dOQck9DLjwPyBKGALiqjAIwxuNx1RotCG9Dz57IFuujcu5pSLlnrLIMYDXunfqBGzjZatxjUgIecmhbFVtaZbJbl0C0yKiqFV0Ku7zRETQsZMVa/IPrdDOY0BhYzu9309jbPHvP13T0TonE4fMlqKgxq4uRcovrYLbI6NXZvvEkhEApz4V07QTyi87ClHveKv2xGnCJvSGkz4Ym5Waw8d196kCkxIdCK1jTPIzq3+DFX8qrRkpcqMPFYc6wLaqqiRsAIedrKIYqsHr/ri0IKmOvs2PsHWG2yIgK1yKvVIvC1LvQ+dR7MB/8BLrR99ptbzBLLTL+EUm0erg5RyFdOwliqAQAsNEp4G+eAE2XAcjb+wnuUI5CNtaAC3F/BadCCOpMFoQG2NgrxmrI105AyjkOKe+MVQ9lOHBJvSGk/xKazreAjesOxs05DF9jC6csqTTaNfa2BGiNwy47KjxvPReiJEMnaMD3GQ3xxA4ohkqX5bPKmnpj7+aK0cb07mw91qW8Kgy7KUH9GwD6pDT0g1jM1vvn2glI10+C1FlrXwuJVumPS7kZXFIfMBrP++IMjmXRq1NEkycRhRBk51VhmBdRQzZjXxHTH7E5X0G6egTCzZle97c1gsrYawWuyQStIxSFwCIpqtdRqu+JHv0nwHL6K2i6pUGTcnOLbYxmCckxeqt2mHMcUs4xSLmnreF7vA6azreA6zIAms4DwIY1RDJc7VyHweffgfHQ5wgbc7/bYzKYJBCCgHj2cmU+pKvHIOUcg1KUDYCA0UeB7z0Smq6DwCXf5NUjsz+wLaYqdZAQTY2x76CpjRsjaGwLCxXoBKuUIx7fBunijxAGTXZpH5W1IngN69H8lI2uiWHgNSwu5TY19tHhWkQLFljOHYTlymHIeWetYZC8DpqU/tAMnQWu60Akduvi13z2zemdEomtP1xVU5gXlNbBYJa8ihri66P7aoV4sFHJkC4fosbel2h512Labd5/dL0uaTRL0I64G3LuGZj2vofQXy1rovErxmrcYjmF0RW5qPvXdYAQMKEx4FNvhab7EHDJfR2G7imRnfGDuQ8yzu+FfMsEcDHupUq1Lajy1wStUlkAy+WDkLIPQamwlj1k47pBGDIDmm5pYOO6tevolQg9D4FnVQ++ObYUyMEi4wBWzx6AdRFYYu/6mPtJLp1H64Iq11fPEkWGnP8zpKtHAYYFGx4HJjwewxJMuJ5bDKAPFEMlIvN+xP/TX0Pdxg0AUcCExYK/OROaboPBJaW6nRrCl/ROiQQhwOWCavTvHuOTlA4NuXEIND3SIR7fBsVYDTbE+0gkRwSXsRc4lzJM2n4QosIEMLBKNIxGC934x2DYshymHzZBN/peSFePwpJ9AHLeWcwUFNSSWAhp06HpNsRlI6gTOHxmHIyM8Osw//hvhExZ4JbxVI293nfGXqkqhCX7IKTLh6yLi8CAS+oD7eh7oek+tMmTSXuHYRjERzpOdVxSaQSvYR0W3ehI2GScxql1+dRbYd73DyilV8HF93C6D1dWzxKiQC68CCn7AKTLh0BMNQCvA8Coq0nvqW9b84+NIKIJUzkCIxsLYdAU62rkduRE9OwUCQbWNA/9u8fgUm4VwvU8EqI9fxq0GXuLpEDTaxjEY1shXT1qdzW/rwgqY6/jOXWCqTVsEQs6rQY6LaeuouUSekJImw7x6BbUZh8AFBlMeDzYAZPwp30cRt86FFPSu7vXJ4FDHdHB0HcKQk9/CinnmLrAyxV85dkTcx0s2Qdhufg9lKJLAAAusd7A90gPSBywv4iL1DksYlJaZUJcpK7dGBZ/om0k49jgew6D+YdNMB/8L0J+8RQYofXCIhW1Iro6iPySy3NhubAfUvZBq77OCdB0GwxN7xHQdB4AcDxgroNSW4rL5y/jp0Nn8IuuITAwIdhwUodH7r0dCXbSErQ1ep0GKfGhqkd/Ka8KvVMivbpmVGMvK2BjuoGJSLRKOdTY+waBd02zt3n2Wp6DXquBsVEyNGHIdJC6CkAIAd9rBNj4HqisFZG/93uPZuZtMcWVyaMQnvcjzD9+CE2XAS6v2LTlsvfE2BNFgZx3Gpbz+yHlHAVkCWx0CrQj7oam1wiwYR0jHDE+KgTnr1fazR1uzWPf8fV6oOkErQ1GGwrtqF/D/P1G1H32B4Tc/iS4uG4O91FZa8aAng1PdkQ0wHLpACzn90EpuWydpO9yC/gRd0HTLQ0M30we04WB04UhSd8Je/YBMZE9UV1nQRmThy5+SjHtC3qnROLAz0WoqhNRVGHEGDczfjbHtiLfIilgGAZ8z2EQT+wAMdX6La9SUBl7neBaNI6tjVbgEKLVNMmPw7Aa6MY+1KR9Q6oEzxZVAYBJArSj5sK4402Ip3ZBO3iqS9t74tkrteWwnNsLy7nvrBFC2lDwN40Fn3qrNYKmg3m5cVEhMImyNUS1Uc5/QghKqoxtnjs9UKgTtM0qJAk3Z4KN6QzT1+tg2LIM2tH3gb9pbIvrwGiWYBZlRIfykPLPwXL+O0iXDwOyaHUSRv4amj6jXNKdw/UCkmL0uJRbhWqDiB7JEU5TkrQlvTtHYu/xfHx3PE997Q0Mw0DDNVSr0vS06vaWq0cg3DTW6/7aI6iMvavRODZjr7N59k50foMHhUtsNC46rulzCzTd0iAe2wo+NcOlcLhaowUcy6j7cQQhBHLeWVjO7oaUcwwgBFyXAeAz7oOm66AOkfvFEfGRDbH2jY19nUmC0SwHjWcv2Dx7Ow6PJikV+juXwrT7XZj3/QNywXnobvuN6pkTSUTtxWO4R/8jhp37HMZTNQAfAj51NPi+Y6xVstx0EnqnROLohRKYLTImjejq/QD9iG0y9psjueBYBt2TvH8KERqVJmRju4EJj4d05TA19r5Ay3MQJQWKQsCyji9Mc2MZR8ejvLr1otVGD3PZAy2LjmtHzoH0ycswH/wEIeMedbq9syRoRDTAcn4/xLO7QaoKwejCIQycBL7feLARrlcZupGxGfOSSiN6JDd4nbawy7ggiLEHmhbMsAcbEoGQyS9APL4V4uHNMJTmgO+fCTn3DKTcM9DJIoZoeVhibkb4zSOg6THUqxj33p0jsf9UAQCgVxsXK3FGfFQIIvQ8qg0W9OoU4ZOCI7yGhUWuT8Rmk3JO7gQx17VY0e8Lgs7YA1bPvTXJRTW8AoeQRhO0jvCkSpUNXbPShGxkIoQBd0A8sQNSzxHQdB3Y6vY2Y98cpa4C4qldsPy8B7CYwCb2hnbIY9bkb17UH70RsYVVNo/IsU3atlXt2UCjhl5aHBe6ZlgW2iEzwSX2gWn3OzB/v9EaRtz3NlxEN6zeb8bSX40G72GK48Y0Dl1s68pUzmAYBr07R+HohRKfyX7Ni45reqRb7/urR8H3vc0nx2hMUBl7neCasW+s2eu1vFMZx5P6sza0jWQcG0LadEjXT8G4823oxj0Mvs9oh9s3N/ZyRR7EE19CuvQDQBRoeg6HMHByq3l9Ojo6QYOwEL5FrL1toVWwePa2OsyiC9XMNCk3I/TuP1lX10Z1AsMwyDtwDTIueZwXpzlJsXqE6jSICBXadSI/GzbZyVc/TM2NPRvfA0xYLCxXDlNj7y320hzbQzX2PIcQndWzb60KvMGLCVqWYazx/436xAgh0M/4HYw7V8O0Zz2IocrhopdaowXJMXrIRZdQuOdLGC4eBjgBfL/x1mySQSLVOCM+KqTFKtqSKhNCdRqPfqRvRFzx7BvDaEPBNZITKmvN0PKc0/khV2EZBjMyetww3//wfgm4VlyDm7v7Zp0JzzU19gzDQNNzGCznvvPJ/ptzY3zLPqJ5tSpHmEQZDKw3h17LgxDre46MudEsgWMZ9WZyF3sFTBhBj5ApL8C0528wH/gYSl0FtKPmtEgkFmnKxx21p2DYchmsPgLC0Nng+2eC1bXfMLa2ID5Kh6sFTZfYl1QaW61N29FoWMjjWUptd1fPusIvhnmX8TWQxETo8Nh093PXO6KxZm9Dm/5LtxLTuUNQGntnKRNEiwytwIFhGLXylC0vhj0MJutnnt4EjurQMhwP3YTHYdZHwXJ6F4ihErrxj4LheMilOTAf/hyPC8chWkKgHXE3ksfMRFmVxaM+dHTiIkNw5HxJk8n50kpju47t9jUajgXHMi1CL12lssaz2rMU+zSXcQCA0QgeVQ9zBZeMfVZWFnbu3Im8vDxs3boVqampAIC9e/fi7bffhiRJiIyMxJ/+9Cd06WL9pc7MzIQgCNBqrRfHggULcNttvteh3EHLu+7Z29raFkoZzBIcPbwZzJJXj6Kt1aFlGBbaUb8GGxoN84GPYTRWgdGFQ7pyGBD02GYYjLjhU/GLQX3ACjoA1NjbIz5KB1khqKgxIzZSB0UhKKs2YUhqcMlcvIZ1WcZpTmWt6JMqUhQrGg2LOqN3JUndOp4rjSZMmID7778f997bkN63qqoKL730Ej766CP06NEDW7ZswZIlS7Bhwwa1zerVq9UfhvaAzg3N3vYU0Nizd0RrXr8rhDjJ2WOrl8voI2HauwHQ8BCGzEBV17H46u8n8HB4+ylc0l6JaxR+GRupQ2WtGZJMgkrGAWyx3e7LOIQQVcah+Ibmmr2/cclCpaent3gvJycHcXFx6NHDmjxp7NixePHFF1FeXo6YmPaZKMtVzd4syuoPg15b79mbHBtjg5tVqpqjEzROY/kBgO8zGlxCTzDaMDC6MNTWF2gOdC77GxHbwqqSKiNuQrQahhksYZc2BJ6D2QPP3miWIEoKlXF8iD3N3p94vD65R48eKC0txcmTJwEAW7duBQAUFBSobRYsWIDp06djyZIlqK6u9rKr3mOTZpxp9maLDEGwyThWI95arL3RjzJOc9jIJDV3hi1VQltWqbpRiInQgWGA0vrY+mAqWtIY3kPPvsLLcoSUlvAaFpKHk+We4LGFCg8Px6pVq/CnP/0JZrMZY8aMQUREBDjOaiQ3bdqE5ORkiKKIFStWYOnSpXjzzTfdOkZsrOfyRHx8y4m38EirweYFjd3PbciEICJUi/j4cGjqNXuOd7yNySIjJjKk1X22RlRkCMRrlW5vz+RUAgC6pkQhPt76XXnahxsdV8YdHxWCGpOE+Phw1Il5YBigb684n6yGbCvcPd/6EB5gWbe3y6+w/jh27xzVLq6x9tAHb4kI00FW3BuLN+P2Khpn9OjRGD3auuCntLQUGzZsQNeu1hwXycnWwsGCIGDu3Ll44okn3N5/WVktFIW4vV18fLjdSjaEEDAAyioMrVa6qTNYa7qWlNSoXlBxaa3DbWqNFjCEeFw9h8gyDCaL29sXFFvbiyYRJSU1Dsfd0XF13DHhWuQW1aCkpAY5+dbKSJUVhgD00D94cr5ZALV1Zre3u5pXYf1Dltv8Guso17lkkWG2SC6PxZVxsyzj0En2Ks1cSUkJAEBRFKxcuRJz5syBXq+HwWBATY21U4QQ7NixA/369fPmUD6BYRgILmS+bByNw2s4aDjW4QStrCgwi7LXmr0oKZAV9/S7WqMIhvFsMVcwEhcVgpL6fDilVcagWTnbGMFOuJ8rVNpknFAq4/gKe6GX/sQlK7F8+XLs2rULpaWlePDBBxEVFYXt27fjrbfewtGjR2GxWJCRkYEFCxYAAMrKyvDMM89AlmUoioJevXph8eLFfh2Iq7hSdLxxNA5g1e0dafZGs3VfIV5q9oB1Ylivc/33t9YoISyEB9vBUhL7i/hIHapqRYgWGSWVRvT30UrIGwlew6Ha4H54bmWNGSFarsl9QfEOXsNCkgkUQgJyD7tkoRYtWoRFixa1eH/FihV223fp0gWbN2/2qmP+wpU0x2ZLQzQOYPWcHXn2Bi8yXtponAzNnQIotQbxhsgp0l6whVkWlhtQWSsGXdglYE1zbC/FsTNcKUdIcQ+1Dq2kqHmL/En7rRbgJ5wVHZcVBRZJUWUcwGrIHYVe2qpYeSvjAIDRxYgcG7VGCw27dANbquNzORX1r4Mr7BKwFjDxZAVtZa1Ijb2P4Tlb5bDASDnBZ+ydaPZmUVHb2dBrOaeevTe6uc5O5ktXqDVKNOzSDWyx9mfrjX0wavY876lmTxdU+ZrGRccDQdAZe2eafeP0xjZCdLxDzd7m8XsbZw84j/9vTq1RpJ69G0SEChA0LM5frwSAoKlQ1RithnNbxmlYPUs9e1/SuOh4IAg6Y691UnS8cXpjG3ot51jG8YlnX1+tyuz6TUgIoZ69mzAMg7ioEJhFGRqORWQQeqq8hoUoKSDE9ZDmOpMESSbU2PsY6tn7GecyTkP9WRutFTDxpv6sDZ3WfRnHbJEhyQqdoHWTuHopJy5SF5RRTLY6tJIb3mRljRkAEBVOjb0vsWn2EjX2/qF5oZDm2JdxrHHw9m4Q1bMXvPfsnYWENsaWKoEae/ewpUeIC8LJWcA6QQvArfw4lbX1xj4In4T8CfXs/YzWiWZvEu3JOI7z4xhMEnQC12oBc2d4otlTY+8ZtgicYNTrAesELeCegamkeXH8grfFZNwl6Iy9judgkRSHaRjsefY2Y2+0o9t7mwQNsK5qZBj3ZBybsacTtO5hi60PtgRoNtTShG4YGOrZ+wdbTiY6QesnnKU5tqfZh7Tm2XuZyx6wThzqBI1bE7S19asgw/XU2LtDSnwoGAbokhicNQBsMo47BUwqa80I1Wlu6IRx7ZFAyzhBl1SlcZpje0ba9iMgNEuXANg39kazd7nsbbiT5hignr2nJEbr8eenMoJWkrBN0Lrn2Yt0ctYPUM3ez9g8e0exxjYpxZ5nb0/GsdWf9RZ7Rcdbo9ZoAQMg1EsJKRgJVkMPNJIO3PTsg/k78xe2aBxq7P2EzkkBE7NFAcM0/OoCTiZozRavNXvAcdFxR9Qarcfl2KA7hRQv8MyzNyMqlOr1voYuqvIzgguavZbnwDSKwVY1e7sTtPblIHfxRMahkTgUd3FXs1cIQRWVcfwClXH8jFp03JGxt0gt0rjqtBwYtCw6Tgjxuv6segwPZBxq7CnuIrhpYGoNFsgKXT3rD6ix9zPaRrnj7WG2KE30egBgGQYh2pY57UWLAoWQNpNxqLGnuIstla7ZRRmHhl36D45lwIAae7/hrOi4uVGVqsbYy2nvi4yXNnRaKuNQ/I/qTboo4zQYe+rZ+xqGYazVqqhm7x+cxdmbxJYyDlBfrcpk39i3lYxDwy4p7uLuoiq6eta/BLI0YdAZe+eavWLX2Nvz7H1RuKRxvySZuJSgSrTIEC0KXVBFcRubZ+/qBK0tCVowZggNBBpq7P0Hr2HBoLXQS/syjt6OZq/KOD7S7NFKvxpDF1RRPIVhGLeKjlfWiQjX89BwQWcqAgLPUWPvNxiGgVZwXMDBLEotJmgBq2ffUsaxGl1fyTgAYHKQSrkxNmNPc9lTPMGa095FGaeGLqjyJ1Sz9zOtpTl2JOPodXZknPpcNj4x9lr3PXs6QUvxBIHnXJdx6OpZv8JrWJrP3p+0lubY5CQaR2lU4cdgsqifeYs7aY6psad4g+COZ09rz/oV6wQtTXHsN3QOShPKirVAiV3PXqsBQdP4fINZgoZjmqRW8LhPbhQdp8ae4g28hnNJJ1YUgqo6kXr2foRq9n7GUWlCs2j90u1p9mrmy0a6vdEsQ6/VNEmt4Cl0gpYSKLQ861LR8WqDCEJoOUJ/wms4iNTY+w8tb1+zt5fe2IZawKSRbm8wWXwi4QANnr3RFc/eYEGIlqMREhSPsBUddwZdPet/2tUEbVZWFjIzM9G3b19cuHBBfX/v3r2YPXs2pk+fjvvuuw/Xr19XP7ty5QruueceTJw4Effccw+uXr3ql857iqNoHHvpjW2E2MlpbzTLPkmVALip2ZssCNVRr57iGQLvmjdZWUMXVPmbdrWoasKECdi0aRNSUlLU96qqqvDSSy9h5cqV2Lp1K+666y4sWbJE/Xzx4sWYO3cudu7ciblz5+LVV1/1S+c9RefAs7dFKDjS7IGmxt5g9qVn74aMY7DQBVUUjxE0rsk4NFWC/2lXmn16ejqSk5ObvJeTk4O4uDj06NEDADB27Fjs378f5eXlKCsrw9mzZzFt2jQAwLRp03D27FmUl5f7ofueITjQ7G2evaNFVUDTAiY2zd4X8BoWHMu4PEFL9XqKp7g6QVtZawYDICKUXmv+ol159vbo0aMHSktLcfLkSQDA1q1bAQAFBQUoKChAYmIiOM5qMDmOQ0JCAgoKCnzUZe/ROQi9tFds3Ia9OrS+1OwB13Pa1xotdEEVxWME3nXNPjxUoAVy/EggNXuPLFV4eDhWrVqFP/3pTzCbzRgzZgwiIiLAcRwkyfVkXs6IjfW8KHR8fLjDz2Ki9bBICmJiQsE1muTU5lcDAJITI1psHxVtNcKMhlU/M4kyYqP1rR7LHUJDeIBhnO6vziQhPibUbjtf9eVGg47bdSIjdLBIitNtLQoQE6Frl99te+yTJ0RG6CC5cC5seDNuj93S0aNHY/To0QCA0tJSbNiwAV27doXRaERRURFkWQbHcZBlGcXFxS2kIFcoK6uFohDnDZsRHx+OkpIah59L9VJJbn5VkwnW4tJaAIChxoQSrmU4Ja9hUVpmQElJDSRZsXrhitLqsdyB17CorDa13ndZgdEsgQNp0c7ZuDsqdNzuIVtkiBbZ6baV1SbwHNPuvtuOdL4tZgmyQlBYVOX0CcqVcbMs49BJ9vj5rKSkBACgKApWrlyJOXPmQK/XIzY2Fv369cO2bdsAANu2bUO/fv0QExPj6aF8jqM0x7YFU/ZkHKBpMjSb3OIrzR6wyjiOVvbaoAuqKN7Ca1jIivMMq76qwkZxDF9fE1iS3Hdq3cXpmVy+fDl27dqF0tJSPPjgg4iKisL27dvx1ltv4ejRo7BYLMjIyMCCBQvUbZYsWYKFCxdi3bp1iIiIQFZWll8H4S6O0hzbXuscGPvG1ap8mSpB7ZfQMv9Oc+iCKoq32OrQWiSl1bUaBrMFyXH6QHUrKOG5hqLjWti3O77CqaVatGgRFi1a1OL9FStWONymV69e+OSTT7zrmR+xRds0T5lgtshgGDi8ARonQ1MLl/gozh6w/shU1OcPd0StgWa8pHiHwNsKmCgIaSWq0pfRZhT7BLIObVBOszuScUyiDJ3AOUx/oG+U5tiXhUtsuFKtinr2FG9pKE3oWDIkhMBgknz65EppSYOx938ytKA29s3DHEUHhUtsNK5W5cv6szZ0ggYms2uafbieLmGneIb6ZNuKNylaFCiE+PTJldISvpGk5m+C09jXX+zNVxE6Sm9sQ69rpNn7ScYxiTIIcTxZ0zBBS29Cime44k36w5mhtKSxZu9vgtLY2yZom3v2ZlF2GIkDNK1W5S8ZRyGk1V/5WqMFWp5TPQIKxV1sE7StFTBRnRlq7P0K1ez9jMPQS4tsNwmaDb1WA0lWYJFkGMwSGDRUmPIFruTHqTVaqFdP8YqGCVrH15nqzFAZx69QY+9ntK2EXtpLb2yjIWWC1djrtBxYH+Syt+FKAROaF4fiLWropQuePZVx/As19n6G17BgmJYetEl04tnrGnLaG82+X3DiSppjmheH4i02A9NafhyD2To3RGUc/6Jq9tTY+weGYayrVe1F47ji2Zskv4SluSrjUM+e4g2qjNNK6KU/5qQoLVE9ezpB6z8EO5kvnUbjaAPl2duXcQghqK4TadglxSvUCdpWPXuq2QcCKuMEAHtpjs1OPPvGBUwMZgl6H1eLcibjVBssMIkyEqJCfHpcSnDhioExmCVoOIZGffkZDTX2/kfbTMaRZAWSTFzW7K0yjm9vBGcyTlG5AQCQFEvzlVA8xyUZh6ZKCAhUsw8A2maeve3Cd7aCFrBq9lYZx8eeff2Ph8lBMrRCm7GPocae4jkca62K1qqM4+PCPBT7UM0+AGibVYUyOUlvDKA+bw5QZ7LAaJYRovO1Z9+6jFNYZoCGYxEbofPpcSnBh7ValRPPnur1fodjGTAMzY3jV5pr9q2VJLTBMAz0Wg0qa83WvCE+9uw5lgWvYR0b+3IDEmNCwLK+i+2nBCeChnOygtZCZZwAwDBMwOrQBq2x1/JNNXuzCzIOYJVyyqpM9X/7fvKqtcyXBeUGJEVTCYfiPVYD07pnT2WcwMBz1Nj7FW2zqlA2w9/aBC1gjcgpq7Yae19H4wCOi45LsoLSSiOdnKX4BIHnnGr2VMYJDNSz9zPNNfsGGaf1CzxEq0F5tbn+b3949hq7xr6k0ghZIXRyluITBA3rNBEa9ewDA69h6QStP9HyHCRZgaxYv2R1gpZv/SvR6zSQ64ug+1qzBxzLOEXlRgA0EofiG4RWZBxJViBaFKrZBwhew1HP3p+odWhFpf5/5xO0QNPEUP7y7I12PPtCGmNP8SF8KzKOkSZBCyhUs/czzdMcNxQbb/0Cb+ztBFKzLyyvQ7ieR6gfjkkJPqwyjn3P3khTJQQUqtn7meZpjhuicZzLOOrfAYzGKSwzUAmH4jNam6Cl6Y0DC9Xs/Yzq2dd70SZRBssw0HCtfyW2G0DDsX7JG+JograwnBp7iu9ozZukGS8DC/Xs/UxDaULrhW1LgsY4KUZiuwH89YirEziIogylUR1ag8mCaoOFGnuKz2hNxqGefWDhNSwkauz9h6Bq9g0TtM4kHKDhBvDXjaDTciBomqSqgObEofgYV2QcqtkHBurZ+xmdHc3eWYw90HAD+OsR117mS5rtkuJrhHoDQxo9QdpokHFoMEAg4LnAaPYuWaysrCzs3LkTeXl52Lp1K1JTUwEAe/bswdtvvw1CCAghePrpp3HHHXcAADIzMyEIArRaLQBgwYIFuO222/w0DPfRNisU4qwkoQ2bR++PyVmgsbzUYOwLyw1gGQbxNI89xUc0zmkvNLvuDWYJDBqysFL8S6A8e5eM/YQJE3D//ffj3nvvVd8jhODFF1/Epk2bkJqainPnzuHXv/41br/9drCs9UJavXq1+sPQ3rBF49hWEYoW12Qcm2cf4qcQSHvVqgrLDIiP0jmdPKZQXMVm4EUHxl6n1YB1Mn9F8Q3tytinp6fbfZ9lWdTU1AAAampqkJCQoBr69k5zo2oSZZfK/fnds7f1y9zUs6d6PcWXCJpGBUya1TQ2miS/Xd+UlrQrY28PhmHw1ltv4cknn4Rer0ddXR3Wr1/fpM2CBQtACMHQoUPx/PPPIyIiwusO+woNx4JlmCaafZyT1bNAo2gcP+mZOm1TzV4hBEUVRvTvEeOX41GCE1sdWntGxpoXh+r1gYLnWCiEQFYUcH50lj029pIk4d1338W6deswdOhQHDlyBM899xy2b9+O0NBQbNq0CcnJyRBFEStWrMDSpUvx5ptvunWM2NgwT7uH+Phwp210Wg6shkN8fDgsMkFkuNal7e6a0Acj+ie51NZdTPX3Hq/jER8fjqJyAyySgj7dYlw6nj/6dCNAx+0ecbHWJ/LQcF2LfUgKXL4X2or23Dd3iYq0PrVHRoU6jfLzZtweG/uff/4ZxcXFGDp0KABg6NChCAkJQXZ2NgYOHIjk5GQAgCAImDt3Lp544gm3j1FWVgtFaRkt4Iz4+HCUlNQ4bSdoWFRWGVFSUgOjyQLIxKXtJg/rAgAutXUXY501o2ZJaS1KSmpw9nIZACCUZ50ez9VxdzTouN3HaBABAEXFNQhrNldVVWtCTLiu3X6nHe18i2YLAKCgsKpVKdmVcbMs49BJ9viZISkpCYWFhbh8+TIAIDs7G2VlZejatSsMBoOq5RNCsGPHDvTr18/TQ/kNbaPVqiZRdpoELRA0L01IY+wp/kDV7O3JOCbJL0n+KPZpHBnlT1zy7JcvX45du3ahtLQUDz74IKKiorB9+3YsWbIEzz77rLrq9I9//COioqJw/fp1PPPMM5BlGYqioFevXli8eLFfB+IJWt6a09ua6pi4FI3j9z41mzguKjcgRMshItT55DGF4ipqNI6dVbRGs0Rj7AMIzwWm6LhLxn7RokVYtGhRi/dnzJiBGTNmtHi/S5cu2Lx5s9ed8zc63pp0zNXCJYGAZRho+YbMl7ZIHGdpHCgUdxAceJOEEOsELV09GzAC5dm3vSvbhmgFDcwWuaEkYTuQcYCmmS9p2CXFH/C8TcZp6tmbRBmE0CRogURDjb3/0fIszBZF9eyFdiDjAA057c2ijPJqMzX2FJ9jC71sXpqQ5rIPPNSzDwBagYNZlFTJRMe3jwvclua4qMKWEye0jXtE6WgIvP0JWprxMvComj019v5Dx2tUDxpwXpIwUOgEDiaz1FCKkHr2FB/ToNk3lXFUz54a+4BBPfsAIAhNZRytC4nQAoFNxikssxr7hGiaAI3iWzQcCwYNKb5tGEzUsw80qrH3czROUBt7Hc9BkhX1Am83nr3W+sRRWGFAbIS23fwIUToODMOA51nHnj3V7AMG7+Apy9cEtbG3hVpW168mdCXFcSCwRePQurMUfyJoWhYwoZp94KGafQCwLaKqrrMa+3bj2dtknHIDkmLo5CzFPwh8y9KEDZp9+7gXgoF2tYK2o2Iz7lU2Y99uPHuN6nHR6lQUf8FruBYGxmCSwGtY8Jr2cS8EA1SzDwC2UMvqOhEcy0DDtY9Vqo0Xd1EZh+IvrEXHW8o4VMIJLDQaJwA0lnEEnms3KQmosacEAoFnW6ygtebFocY+kHAsC45lqLH3J7YJ2qo6sd2kSgAaio4LGhbREdo27g2lo2J3gtZEPfu2QBOAalVBbuytBr7aILYbvR5o6FdCtJ7WAaX4DV7DwmInXQINuww8PMdSzd6f2GQcQtrP5CwAhNQbezo5S/EnAs+1kHGoZt82BKIObVAbe12jlMbtJewSaOgX1esp/sTRBC3V7AMPr2EhUWPvPxp78+1Js48KE8CxDHomt58C7ZSOh6Cxs4LWRI19WxAIzz6oz6qGY8AyDBRC1Mo97YHIMC1WPp2BsBBaLYjiPwSeg7mRgZFkBaKk0MIlbQDV7P0MwzCqfNNeUiXYCNcL7SYUlNIxaT5Ba6AZL9sMqtkHAJt80540ewolEAgaFgohkOo9SqOJGvu2ghr7AGCTb9pTNA6FEggaio5bjYyaBI3KOAGH56ix9zs2+YZ69pRgo3kBEyrjtB28hmr2fqe9avYUir+xJTuzraKlMk7bwduJjPI11NhTz54SpKh1aC1NPXu6qCrwUM0+ANiMPNXsKcGG0Nyzp1Wq2gyea5lu2tcEvbGnmj0lWOH5pql1DSYJDOi90Ba0C88+KysLmZmZ6Nu3Ly5cuKC+v2fPHsyaNQszZ87EjBkzsGvXLvWzK1eu4J577sHEiRNxzz334OrVq37pvC/Q0mgcSpCitXn29TKOsT4vDk2+F3hsWS8JIX47hlNjP2HCBGzatAkpKSnqe4QQvPjii3j99dexZcsWvP7663jppZegKNZfpsWLF2Pu3LnYuXMn5s6di1dffdVvA/AWdYKWejOUIMNWNMMm4xhoxss2g9ewIABkpQ2NfXp6OpKTk1tuyLKoqakBANTU1CAhIQEsy6KsrAxnz57FtGnTAADTpk3D2bNnUV5e7uOu+waq2VOCFXWC1hZ6SXPZtxmBKDru0ZllGAZvvfUWnnzySej1etTV1WH9+vUAgIKCAiQmJoLjrMaT4zgkJCSgoKAAMTExvuu5j6AyDiVYUSdoLQ0TtDTssm1oXJowxE/1ijw6s5Ik4d1338W6deswdOhQHDlyBM899xy2b9/u087FxoZ5vG18fLhL7UYPTkFuaR1694iFhrvx56tdHXdHg47bfYQQAQCg1fGIjw+HKCtIiNbfEN/ljdBHd4iJtqYzD48MQXy049Tm3ozbI2P/888/o7i4GEOHDgUADB06FCEhIcjOzkZKSgqKioogyzI4joMsyyguLrYrBTmjrKwWigcaVnx8OEpKalxqG8azeGjyTagor3P7OO0Nd8bdkaDj9gyTaA21LK80oKSkBjV1IpJj9O3+u+yI59tkFAEARcU1YBwsrnJl3CzLOHSSPXJlk5KSUFhYiMuXLwMAsrOzUVZWhq5duyI2Nhb9+vXDtm3bAADbtm1Dv3792qWEQ6EEM81lHKrZtx3tQrNfvnw5du3ahdLSUjz44IOIiorC9u3bsWTJEjz77LNqGt4//vGPiIqKAgAsWbIECxcuxLp16xAREYGsrCy/DYBCoXgGyzLQcAxESYZCCIwi1ezbisaavb9wemYXLVqERYsWtXh/xowZmDFjht1tevXqhU8++cT73lEoFL/CazhYLArMogxCaKqEtqJ5Ujp/cOPPSFIoFI8ReBaiJNNUCW2MLSmdPzNfUmNPoQQxgoaFKCkw0IyXbUogZBxq7CmUIEaol3Fo4ZK2RUONPYVC8ScCz8IsybRwSRtDPXsKheJXbBO0tHBJ26KGXlLNnkKh+ANVs6cyTptCPXsKheJXBJ6DSGWcNsdm7CVq7CkUij8QNKxVxjFLEDRsh8gPdSMSiBW09MxSKEEMr7HG2dNUCW0LyzLgWIZq9hQKxT8IPAex3rOnC6raFn+XJqTGnkIJYhpP0FLPvm2hxp5CofgNXsNCkhXUGS10craNocaeQqH4DVuFtqo6kco4bQzPsVSzp1Ao/sEW8lddJ1IZp43hNSxEC816SaFQ/IBQ79nLCqEyThvDa6hnT6FQ/IQtjzpAc9m3NTzH0kVVFArFP9jyqAM0l31bQydoKRSK3xB46tm3F3gNR409hULxD41lHKrZty0aqtlTKBR/YZugBahn39bwHJVxKBSKn+Abe/ZUs29TqGZPoVD8RmPPnso4bQs19hQKxW/Q0Mv2A42zp1AofsNm7BkG0Amck9YUfyLUe/aEEL/snxp7CiWIsck4eq0GDMO0cW+CG7ValewfY+/Sc1tWVhZ27tyJvLw8bN26FampqcjNzcVTTz2ltqmpqUFtbS0OHjwIAMjMzIQgCNBqtQCABQsW4LbbbvPDECgUiqdwLAOGoRJOe6BxtarGE+e+wqUzPGHCBNx///2499571fc6d+6MLVu2qK9XrFgBWW6axGf16tVITU31UVcpFIqvYRgGgoajk7PtALXouJ90e5fOcHp6equfi6KIrVu3YsOGDT7plCNkWUJFRQkkSWy1XXExC0Xx30RHe6Wtx63RCIiOjgfHUcNxI8FrWBp22Q7Q2Iy95J/Mlz45w7t370ZiYiL69+/f5P0FCxaAEIKhQ4fi+eefR0REhFfHqagogU6nR2hoUqv6okbj34RC7ZW2HDchBHV11aioKEFcXHKb9IHiGVqepTJOO0D17P10D/vkDH/66ae48847m7y3adMmJCcnQxRFrFixAkuXLsWbb77p1n5jY8OavC4uvo7IyCiXJpI0ftC8bgTactyRkVEwGKoRHx8e8GO3xTHbA74Y9+hBKeiSEHZDfYc3Ul9dJS6mBgAQHhHicHzejNtrY19UVIRDhw7h9ddfb/J+crLVuxMEAXPnzsUTTzzh9r7LymqhKA0z04qiQJYJgNZnq6ln33YoioKSkpqAHjM+Pjzgx2wP+GrcszO6A8AN8x121PNtqLPK00UlNQjjWzptroybZZkWTrL6mbcd/PzzzzF27FhER0er7xkMBtTUWDtFCMGOHTvQr18/bw9FaYUVK5bgk08+autuUCgUD1FDL9tSxlm+fDl27dqF0tJSPPjgg4iKisL27dsBWI39yy+/3KR9WVkZnnnmGciyDEVR0KtXLyxevNj3vW9HSJIEjSYwumcgj0WhUAJDu9DsFy1ahEWLFtn9bOfOnS3e69KlCzZv3uxVx24Ebr01HQ8++Ch+/PF7jBgxCnPnzsOaNauQnX0RoigiLS0dzzzzf8jLu47f//5FbNz4H0iShKlTJ+A3v3kYc+fej2+++Qr79u3FkiUr8OGHG/HNN7sgyxIEQYsFCxaiT5++do81a9adWL58McrKSpGUlAyWbXhI27LlM/znP/8GzwsgRMHSpa+hW7fubfIdUSgU12gcZ+8Pbmj38PtTBdh/sqDF+wwDeLvi+NaBycgY4DyqRKvV4r33PgAAvPbaMgwePAQLF74CRVHwhz8swvbt/8OMGbNhMNShtLQUhYX56NGjFw4fPoS5c+/HkSMHkZ4+DAAwadJU/PrX9wEADh06gDfe+BPWr/+H3WO9/PJvMWhQGh566DHk5eXigQfmYtSo0QCAdevexqZNnyIuLg6iKAZlGCqFcqPRLuLsKY6ZPHma+vf+/d/h55/P4KOPNgEATCYTEhISAQBDhqTjyJGDKCjIx8yZv8SmTR/AYrHg8OGDuO++BwAA58//jH/9631UV1eBZVlcv37N4bGOHj2C5577LQAgJaWz+oNhPdYwrFixGBkZt2HUqFuRktLZL2OnUCi+o13IOO2VjAH2ve9ARqWEhOgbvSL44x/ftGtchw4dhiNHDiE/Pw+vvroMx48fxddf7wQhQKdOKbBYLHjllZfwl7/8DX373oTS0hLMmjW5lWM55o9/fAM//3wGR44cxvz5j2PBgt9h1KgMb4ZJoVD8jM3Yi36yXcEZjO4nMjLGYOPGf6ppIyorK5GfnwfAauwPHPgRNTU1SEhIRHr6cGzY8K7qkYuiGbIsq08Cn332SavHGjo0Hdu3/w8AkJ+fh8OHDwGwTt7m5+fh5ptvwbx5D2D48JG4ePG8X8ZLoVB8B/XsbyCeffYFrFu3Gg888GswDAOeFzB//gvo1CkFCQmJ0Ov1GDhwMACr8S8qKsSQIdZUFKGhYXj44f+HRx+9HxERkRg/foKTYy3A8uWL8fXXO5Gc3AlpaUMBWOPcV6xYgtraGjAMi8TERDz++NN+HTeFQvGehgla/6RLYIi/kif7gOaLqgoLc5CU1M3pdu1hcVFb0B7G7eo58iUddZGNM+i4OxYKIXgkaw9mZHTHrNt6tvi8zRdVUSgUCsV7WIaBhmP8Fo1DjT2FQqG0E/xZh5YaewqFQmkn8Jz/pFhq7CkUCqWdQD17CoVCCQI0Go5q9hQKhdLR4Tnq2VMoFEqHR+Cpsae0AStWLMGnn37c1t2gUIIG6tkHKZIkdchjUSgU+/Aalma9bI/cems6Hn30Cezb9y2qqqrw0ksv4/Dhgzhw4AdIkoRly7LQvXsPlJWVYsmSl1FXVwdRFDF6dAaefPJZh/v0NEf+gw8+gjlz5tEc+RTKDQqvYWGpo8a+BZYL38Ny/rsW7zMMA2+zQPB9x4BPdZ4pMiwsHO+99wF27/4av/vdC1iy5I94/PGnsWnTP/HBB3/Hq68uQ1hYOLKyVkGv10OSJDz//NP46acfMHLkaLv79DxH/kHMmTPPpznyR4wYBYDmyKdQAoE/Qy9vaGPfHpgw4Q4AQN++NwFgkJFxW/3rfvj22z0ArMnJ1q17G6dOnQRAUFZWhosXLzg09p7myP/3v2mOfArlRsafmv0Nbez51Ay73ncgE4IJggAAYFkWgsCr77Msq6Y6/vjjTaipqcb69f+AVqtFVtYKiKLZ4T49zZF/8uQxmiOfQrmB8admTydoA0BNTQ1iY+Og1WpRUlKM/fu/dXlbd3LkDxs2gubIp1BuYDRUxrmxueuuOXjllZcwb97diI9PxNChw5xvVA/NkU+hBA/+1OxpPvsORHsYN81nHzjouDsem/ddxv++v4oNL40HwzBNPqP57CkUCqWDEBupQ1gID3+44FTGoVAolHZCxoBkDLspASzLOG/sJtSzp1AolHYCyzDQCf7xwW84Y9+OpxiCHnpuKJT2i9OfkKysLOzcuRN5eXnYunUrUlNTkZubi6eeekptU1NTg9raWhw8eBAAcOXKFSxcuBCVlZWIiopCVlYWunfv7n1nNQLq6qoRGhrRYvKC0rYQQlBXVw2NRmjrrlAoFDs4NfYTJkzA/fffj3vvvVd9r3PnztiyZYv6esWKFWocOAAsXrwYc+fOxcyZM7Flyxa8+uqr+OCDD7zubHR0PCoqSlBbW9lqO5Zlg3I5f1uPW6MREB0d32bHp1AojnFq7NPT01v9XBRFbN26FRs2bAAAlJWV4ezZs3j//fcBANOmTcOyZctQXl6OmJgYrzrLcRrExSU7bdeRQ7NaI1jHTaFQnOP1TMDu3buRmJiI/v37AwAKCgqQmJgIjuMAABzHISEhAQUFBW4be0fxoq4QHx/u8bY3MnTcwQUdd3Dhzbi9Nvaffvop7rzzTm93Y5fmi6pcJVg9XDru4IKOO7jwdlGVV8a+qKgIhw4dwuuvv66+l5ycjKKiIsiyDI7jIMsyiouLkZzsXH5pjjexpv6IU70RoOMOLui4gwtn427tc6+M/eeff46xY8ciOjpafS82Nhb9+vXDtm3bMHPmTGzbtg39+vXzSK+Pjg71uG/eSEA3MnTcwQUdd3Dhzbid5sZZvnw5du3ahdLSUkRHRyMqKgrbt28HAEycOBEvv/wyxowZ02Sb7OxsLFy4ENXV1YiIiEBWVhZ69uzpcScpFAqF4h3tOhEahUKhUHzDDbeClkKhUCjuQ409hUKhBAHU2FMoFEoQQI09hUKhBAHU2FMoFEoQQI09hUKhBAHU2FMoFEoQ0KGM/ZUrV3DPPfdg4sSJuOeee3D16tW27pJfyMrKQmZmJvr27YsLFy6o73f08VdUVODRRx/FxIkTMX36dDz99NMoLy8HABw/fhwzZszAxIkT8dBDD6GsrKyNe+tbnnzyScyYMQOzZs3C3Llz8fPPPwPo+Ofcxl/+8pcm13tHP9+ZmZmYNGkSZs6ciZkzZ2Lfvn0AvBw36UDMmzePbN68mRBCyObNm8m8efPauEf+4dChQyQ/P5+MHz+enD9/Xn2/o4+/oqKC/PTTT+rr1157jfzud78jsiyT22+/nRw6dIgQQsjatWvJwoUL26qbfqG6ulr9+6uvviKzZs0ihHT8c04IIadPnyYPP/ywer0Hw/lufm8TQrwed4fx7G159KdNmwbAmkf/7NmzqufXkUhPT2+RWC4Yxh8VFYURI0aorwcPHoz8/HycPn0aWq1Wrb0wZ84cfPnll23VTb8QHt6Q2ra2thYMwwTFORdFEUuXLsWSJUvU94LhfNvD23H7p7JtG+DLPPo3IsE2fkVR8OGHHyIzMxMFBQXo1KmT+llMTAwURVHLYnYUXn75ZXz//fcghOC9994LinP+9ttvY8aMGejcubP6XrCc7wULFoAQgqFDh+L555/3etwdxrOnBBfLli2DXq/Hfffd19ZdCRgrVqzA3r178X//939N0op3VI4dO4bTp09j7ty5bd2VgLNp0yb873//w6effgpCCJYuXer1PjuMsW+cRx+AV3n0b0SCafxZWVnIycnBW2+9BZZlkZycjPz8fPXz8vJysCzboby8xsyaNQsHDhxAUlJShz7nhw4dQnZ2NiZMmIDMzEwUFhbi4YcfRk5OToc/37ZzKAgC5s6di6NHj3p9nXcYY984jz4Ar/Lo34gEy/hXrlyJ06dPY+3atRAEAQBwyy23wGQy4fDhwwCAjz76CJMmTWrLbvqUuro6FBQUqK93796NyMjIDn/OH3vsMezfvx+7d+/G7t27kZSUhA0bNuCRRx7p0OfbYDCgpsZakYoQgh07dqBfv35eX+cdKsVxsOTRd1RjoKOP/+LFi5g2bRq6d+8OnU4HAOjcuTPWrl2Lo0ePYvHixTCbzUhJScEbb7yBuLi4Nu6xbygtLcWTTz4Jo9EIlmURGRmJl156Cf379+/w57wxmZmZeOedd5Camtqhz/f169fxzDPPQJZlKIqCXr16YdGiRUhISPBq3B3K2FMoFArFPh1GxqFQKBSKY6ixp1AolCCAGnsKhUIJAqixp1AolCCAGnsKhUIJAqixp1Ba4Z133sHLL7/s0bYLFy7EqlWrfNwjCsUzOkxuHArFHzz++ONt3QUKxSdQz55CoVCCAGrsKR2KoqIiPPPMMxg5ciQyMzPxwQcfAADWrFmD+fPn47nnnkNaWhpmz56Nc+fOqdutX78et912G9LS0jBx4kT8+OOP6nYLFixQ233zzTeYOnUq0tPTMW/ePGRnZ6ufnT17FrNnz0ZaWhqee+45mM3mJn3bs2cPZs6cifT0dMyZM8el41MoPsMnmfYplHaALMtk9uzZZM2aNcRsNpNr166RzMxM8t1335HVq1eTm2++mXzxxRdEFEXy3nvvkfHjxxNRFEl2djYZM2YMKSwsJIQQcv36dZKTk0MIIWT16tXkhRdeIIQQcvnyZTJo0CCyf/9+IooiWb9+Pbn99tuJ2WwmZrOZjBs3jrz//vtEFEXyxRdfkJtvvpmsXLmSEELImTNnyMiRI8nx48eJJEnks88+I+PHjydms7nV41MovoJ69pQOw6lTp1BeXo6nn34agiCgS5cuuPvuu7Fjxw4AQP/+/TFp0iTwPI8HH3wQoijixIkT4DgOoigiOzsbFosFnTt3RteuXVvsf8eOHRg7diwyMjLA8zwefvhhmEwmHDt2DCdOnIDFYsFvfvMb8DyPSZMmYcCAAeq2H3/8Me655x4MGjQIHMdh9uzZ4Hkex48fd/n4FIo30AlaSochLy8PxcXFaiUfwJr2Nz09HZ06dUJSUpL6PsuySExMVNv//ve/x5o1a3Dp0iXceuutWLhwIRITE5vsv7i4uEnxCFt65aKiInAch8TERDAMo37euG1+fj42b96MjRs3qu9ZLBYUFxdj+PDhLh2fQvEG6tlTOgzJycno3LkzDh8+rP47duwY/va3vwEACgsL1baKoqCoqAgJCQkAgOnTp+PDDz/Enj17wDAM3nzzzRb7T0hIaJJPnBCiVouKj49HUVERSKO8go3bJicn4/HHH2/StxMnTqglBV05PoXiDdTYUzoMAwcORGhoKNavXw+TyQRZlnHhwgWcPHkSAHDmzBns2rULkiThn//8JwRBwKBBg3D58mX8+OOPEEURgiBAq9WCZVveGpMnT8a3336LH3/8ERaLBX//+98hCALS0tIwePBgaDQafPDBB7BYLNi1axdOnTqlbnvXXXfho48+wokTJ0AIgcFgwN69e1FbW+vy8SkUb6AyDqXDwHEc3nnnHWRlZWHChAkQRRE9evTAc889BwCYMGECduzYgZdeegndunXDmjVrwPM8RFHEn//8Z2RnZ4PneaSlpdktA9ezZ0+88cYbWLZsGYqKitCvXz+88847ahGVNWvW4JVXXsFbb72FsWPH4he/+IW67YABA7Bs2TIsXboUOTk50Ol0GDJkCNLT010+PoXiDTSfPSUoWLNmDXJycqg8Qgla6LMihUKhBAHU2FMoFEoQQGUcCoVCCQKoZ0+hUChBADX2FAqFEgRQY0+hUChBADX2FAqFEgRQY0+hUChBADX2FAqFEgT8f/Agx2ffIGCBAAAAAElFTkSuQmCC\n" + }, + "metadata": {} + } + ], + "source": [ + "if __name__ == \"__main__\":\n", + " cfg = DQNConfig()\n", + "\n", + " # train\n", + " env,agent = env_agent_config(cfg,seed=1)\n", + " rewards, ma_rewards = train(cfg, env, agent)\n", + " make_dir(cfg.result_path, cfg.model_path)\n", + " agent.save(path=cfg.model_path)\n", + " save_results(rewards, ma_rewards, tag='train', path=cfg.result_path)\n", + " plot_rewards(rewards, ma_rewards, tag=\"train\",\n", + " algo=cfg.algo, path=cfg.result_path)\n", + " # eval\n", + " env,agent = env_agent_config(cfg,seed=10)\n", + " agent.load(path=cfg.model_path)\n", + " rewards,ma_rewards = eval(cfg,env,agent)\n", + " save_results(rewards,ma_rewards,tag='eval',path=cfg.result_path)\n", + " plot_rewards(rewards,ma_rewards,tag=\"eval\",env=cfg.env,algo = cfg.algo,path=cfg.result_path)" + ] + } + ] +} \ No newline at end of file diff --git a/codes/DQN/task0_train.py b/codes/DQN/task0_train.py index fc13983..e6f5e4a 100644 --- a/codes/DQN/task0_train.py +++ b/codes/DQN/task0_train.py @@ -5,7 +5,7 @@ @Email: johnjim0816@gmail.com @Date: 2020-06-12 00:48:57 @LastEditor: John -LastEditTime: 2021-04-29 22:23:38 +LastEditTime: 2021-05-05 16:49:15 @Discription: @Environment: python 3.7.7 ''' @@ -14,20 +14,17 @@ curr_path = os.path.dirname(__file__) parent_path = os.path.dirname(curr_path) sys.path.append(parent_path) # add current terminal path to sys.path -import datetime -import torch import gym +import torch +import datetime -from common.utils import save_results, make_dir, del_empty_dir +from common.utils import save_results, make_dir from common.plot import plot_rewards from DQN.agent import DQN - - curr_time = datetime.datetime.now().strftime( "%Y%m%d-%H%M%S") # obtain current time - class DQNConfig: def __init__(self): self.algo = "DQN" # name of algo @@ -35,21 +32,21 @@ class DQNConfig: self.result_path = curr_path+"/outputs/" + self.env + \ '/'+curr_time+'/results/' # path to save results self.model_path = curr_path+"/outputs/" + self.env + \ - '/'+curr_time+'/models/' # path to save results - self.train_eps = 300 # 训练的episode数目 + '/'+curr_time+'/models/' # path to save models + self.train_eps = 300 # max trainng episodes self.eval_eps = 50 # number of episodes for evaluating self.gamma = 0.95 - self.epsilon_start = 0.90 # e-greedy策略的初始epsilon + self.epsilon_start = 0.90 # start epsilon of e-greedy policy self.epsilon_end = 0.01 self.epsilon_decay = 500 self.lr = 0.0001 # learning rate - self.memory_capacity = 100000 # Replay Memory容量 + self.memory_capacity = 100000 # capacity of Replay Memory self.batch_size = 64 - self.target_update = 2 # target net的更新频率 + self.target_update = 4 # update frequency of target net self.device = torch.device( - "cuda" if torch.cuda.is_available() else "cpu") # 检测gpu - self.hidden_dim = 256 # 神经网络隐藏层维度 - + "cuda" if torch.cuda.is_available() else "cpu") # check gpu + self.hidden_dim = 256 # hidden size of net + def env_agent_config(cfg,seed=1): env = gym.make(cfg.env) env.seed(seed) @@ -63,7 +60,7 @@ def train(cfg, env, agent): print(f'Env:{cfg.env}, Algorithm:{cfg.algo}, Device:{cfg.device}') rewards = [] ma_rewards = [] # moveing average reward - for i_episode in range(cfg.train_eps): + for i_ep in range(cfg.train_eps): state = env.reset() done = False ep_reward = 0 @@ -76,11 +73,12 @@ def train(cfg, env, agent): agent.update() if done: break - if i_episode % cfg.target_update == 0: + if (i_ep+1) % cfg.target_update == 0: agent.target_net.load_state_dict(agent.policy_net.state_dict()) - print('Episode:{}/{}, Reward:{}'.format(i_episode+1, cfg.train_eps, ep_reward)) + if (i_ep+1)%10 == 0: + print('Episode:{}/{}, Reward:{}'.format(i_ep+1, cfg.train_eps, ep_reward)) rewards.append(ep_reward) - # 计算滑动窗口的reward + # save ma rewards if ma_rewards: ma_rewards.append(0.9*ma_rewards[-1]+0.1*ep_reward) else: @@ -89,15 +87,17 @@ def train(cfg, env, agent): return rewards, ma_rewards def eval(cfg,env,agent): - rewards = [] # 记录所有episode的reward - ma_rewards = [] # 滑动平均的reward + print('Start to eval !') + print(f'Env:{cfg.env}, Algorithm:{cfg.algo}, Device:{cfg.device}') + rewards = [] + ma_rewards = [] # moving average rewards for i_ep in range(cfg.eval_eps): - ep_reward = 0 # 记录每个episode的reward - state = env.reset() # 重置环境, 重新开一局（即开始新的一个episode） + ep_reward = 0 # reward per episode + state = env.reset() while True: - action = agent.predict(state) # 根据算法选择一个动作 - next_state, reward, done, _ = env.step(action) # 与环境进行一个交互 - state = next_state # 存储上一个观察值 + action = agent.predict(state) + next_state, reward, done, _ = env.step(action) + state = next_state ep_reward += reward if done: break @@ -106,11 +106,15 @@ def eval(cfg,env,agent): ma_rewards.append(ma_rewards[-1]*0.9+ep_reward*0.1) else: ma_rewards.append(ep_reward) - print(f"Episode:{i_ep+1}/{cfg.eval_eps}, reward:{ep_reward:.1f}") + if (i_ep+1)%10 == 10: + print(f"Episode:{i_ep+1}/{cfg.eval_eps}, reward:{ep_reward:.1f}") + print('Complete evaling！') return rewards,ma_rewards if __name__ == "__main__": cfg = DQNConfig() + + # train env,agent = env_agent_config(cfg,seed=1) rewards, ma_rewards = train(cfg, env, agent) make_dir(cfg.result_path, cfg.model_path) @@ -118,7 +122,7 @@ if __name__ == "__main__": save_results(rewards, ma_rewards, tag='train', path=cfg.result_path) plot_rewards(rewards, ma_rewards, tag="train", algo=cfg.algo, path=cfg.result_path) - + # eval env,agent = env_agent_config(cfg,seed=10) agent.load(path=cfg.model_path) rewards,ma_rewards = eval(cfg,env,agent) diff --git a/codes/DQN_cnn/README.md b/codes/DQN_cnn/README.md deleted file mode 100644 index 4d1be2a..0000000 --- a/codes/DQN_cnn/README.md +++ /dev/null @@ -1,2 +0,0 @@ -# DQN with cnn -原理与[DQN](../DQN)相同，只是将神经网络换成卷积神经网络，用于二维观测信息(state或obervation) \ No newline at end of file diff --git a/codes/DQN_cnn/agent.py b/codes/DQN_cnn/agent.py deleted file mode 100644 index de2021c..0000000 --- a/codes/DQN_cnn/agent.py +++ /dev/null @@ -1,107 +0,0 @@ -import random -import math -import torch -import torch.optim as optim -import torch.nn.functional as F -from DQN_cnn.memory import ReplayBuffer -from DQN_cnn.model import CNN - - -class DQNcnn: - def __init__(self, screen_height,screen_width, action_dim, cfg): - - self.device = cfg.device - self.action_dim = action_dim - self.gamma = cfg.gamma - # e-greedy策略相关参数 - self.actions_count = 0 - self.epsilon = 0 - self.epsilon_start = cfg.epsilon_start - self.epsilon_end = cfg.epsilon_end - self.epsilon_decay = cfg.epsilon_decay - self.batch_size = cfg.batch_size - self.policy_net = CNN(screen_height, screen_width, - action_dim).to(self.device) - self.target_net = CNN(screen_height, screen_width, - action_dim).to(self.device) - self.target_net.load_state_dict(self.policy_net.state_dict()) # target_net的初始模型参数完全复制policy_net - self.target_net.eval() # 不启用 BatchNormalization 和 Dropout - self.optimizer = optim.RMSprop(self.policy_net.parameters(),lr = cfg.lr) # 可查parameters()与state_dict()的区别，前者require_grad=True - self.loss = 0 - self.memory = ReplayBuffer(cfg.memory_capacity) - - - def choose_action(self, state): - '''选择动作 - Args: - state [array]: [description] - Returns: - action [array]: [description] - ''' - self.epsilon = self.epsilon_end + (self.epsilon_start - self.epsilon_end) * \ - math.exp(-1. * self.actions_count / self.epsilon_decay) - self.actions_count += 1 - if random.random() > self.epsilon: - with torch.no_grad(): - q_value = self.policy_net(state) # q_value比如tensor([[-0.2522, 0.3887]]) - # tensor.max(1)返回每行的最大值以及对应的下标， - # 如torch.return_types.max(values=tensor([10.3587]),indices=tensor([0])) - # 所以tensor.max(1)[1]返回最大值对应的下标，即action - action = q_value.max(1)[1].view(1, 1) # 注意这里action是个张量，如tensor([1]) - return action - else: - return torch.tensor([[random.randrange(self.action_dim)]], device=self.device, dtype=torch.long) - - def update(self): - if len(self.memory) < self.batch_size: - return - transitions = self.memory.sample(self.batch_size) - # Transpose the batch (see https://stackoverflow.com/a/19343/3343043 for - # detailed explanation). This converts batch-array of Transitions - # to Transition of batch-arrays. - batch = self.memory.Transition(*zip(*transitions)) - - # Compute a mask of non-final states and concatenate the batch elements - # (a final state would've been the one after which simulation ended) - non_final_mask = torch.tensor(tuple(map(lambda s: s is not None, - batch.state_)), device=self.device, dtype=torch.bool) - - non_final_state_s = torch.cat([s for s in batch.state_ - if s is not None]) - state_batch = torch.cat(batch.state) - action_batch = torch.cat(batch.action) - reward_batch = torch.cat(batch.reward) # tensor([1., 1.,...,]) - - - # Compute Q(s_t, a) - the model computes Q(s_t), then we select the - # columns of actions taken. These are the actions which would've been taken - # for each batch state according to policy_net - state_action_values = self.policy_net( - state_batch).gather(1, action_batch) #tensor([[ 1.1217],...,[ 0.8314]]) - - # Compute V(s_{t+1}) for all next states. - # Expected values of actions for non_final_state_s are computed based - # on the "older" target_net; selecting their best reward with max(1)[0]. - # This is merged based on the mask, such that we'll have either the expected - # state value or 0 in case the state was final. - state__values = torch.zeros(self.batch_size, device=self.device) - - state__values[non_final_mask] = self.target_net( - non_final_state_s).max(1)[0].detach() - - # Compute the expected Q values - expected_state_action_values = (state__values * self.gamma) + reward_batch # tensor([0.9685, 0.9683,...,]) - - # Compute Huber loss - self.loss = F.smooth_l1_loss( - state_action_values, expected_state_action_values.unsqueeze(1)) # .unsqueeze增加一个维度 - # Optimize the model - self.optimizer.zero_grad() # zero_grad clears old gradients from the last step (otherwise you’d just accumulate the gradients from all loss.backward() calls). - self.loss.backward() # loss.backward() computes the derivative of the loss w.r.t. the parameters (or anything requiring gradients) using backpropagation. - for param in self.policy_net.parameters(): # clip防止梯度爆炸 - param.grad.data.clamp_(-1, 1) - self.optimizer.step() # causes the optimizer to take a step based on the gradients of the parameters. - - -if __name__ == "__main__": - dqn = DQN() diff --git a/codes/DQN_cnn/env.py b/codes/DQN_cnn/env.py deleted file mode 100644 index 402eead..0000000 --- a/codes/DQN_cnn/env.py +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/env python -# coding=utf-8 -''' -@Author: John -@Email: johnjim0816@gmail.com -@Date: 2020-06-11 10:02:35 -@LastEditor: John -@LastEditTime: 2020-06-11 16:57:34 -@Discription: -@Environment: python 3.7.7 -''' - -import numpy as np -import torch -import torchvision.transforms as T -from PIL import Image - -resize = T.Compose([T.ToPILImage(), - T.Resize(40, interpolation=Image.CUBIC), - T.ToTensor()]) - - -def get_cart_location(env,screen_width): - world_width = env.x_threshold * 2 - scale = screen_width / world_width - return int(env.state[0] * scale + screen_width / 2.0) # MIDDLE OF CART - -def get_screen(env,device): - # Returned screen requested by gym is 400x600x3, but is sometimes larger - # such as 800x1200x3. Transpose it into torch order (CHW). - screen = env.render(mode='rgb_array').transpose((2, 0, 1)) - # Cart is in the lower half, so strip off the top and bottom of the screen - _, screen_height, screen_width = screen.shape - screen = screen[:, int(screen_height*0.4):int(screen_height * 0.8)] - view_width = int(screen_width * 0.6) - cart_location = get_cart_location(env,screen_width) - if cart_location < view_width // 2: - slice_range = slice(view_width) - elif cart_location > (screen_width - view_width // 2): - slice_range = slice(-view_width, None) - else: - slice_range = slice(cart_location - view_width // 2, - cart_location + view_width // 2) - # Strip off the edges, so that we have a square image centered on a cart - screen = screen[:, :, slice_range] - # Convert to float, rescale, convert to torch tensor - # (this doesn't require a copy) - screen = np.ascontiguousarray(screen, dtype=np.float32) / 255 - screen = torch.from_numpy(screen) - # Resize, and add a batch dimension (BCHW) - return resize(screen).unsqueeze(0).to(device) - -if __name__ == "__main__": - - import gym - env = gym.make('CartPole-v0').unwrapped - # if gpu is to be used - device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - env.reset() - import matplotlib.pyplot as plt - - plt.figure() - plt.imshow(get_screen(env,device).cpu().squeeze(0).permute(1, 2, 0).numpy(), - interpolation='none') - plt.title('Example extracted screen') - plt.show() \ No newline at end of file diff --git a/codes/DQN_cnn/main.py b/codes/DQN_cnn/main.py deleted file mode 100644 index 89f9d77..0000000 --- a/codes/DQN_cnn/main.py +++ /dev/null @@ -1,112 +0,0 @@ -#!/usr/bin/env python -# coding=utf-8 -''' -@Author: John -@Email: johnjim0816@gmail.com -@Date: 2020-06-11 10:01:09 -@LastEditor: John -LastEditTime: 2021-04-05 11:06:23 -@Discription: -@Environment: python 3.7.7 -''' -import sys,os -curr_path = os.path.dirname(__file__) -parent_path=os.path.dirname(curr_path) -sys.path.append(parent_path) # add current terminal path to sys.path - -import gym -import torch -import datetime -from DQN_cnn.env import get_screen -from DQN_cnn.agent import DQNcnn -from common.plot import plot_rewards -from common.utils import save_results - -SEQUENCE = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") # obtain current time -SAVED_MODEL_PATH = curr_path+"/saved_model/"+SEQUENCE+'/' # path to save model -if not os.path.exists(curr_path+"/saved_model/"): - os.mkdir(curr_path+"/saved_model/") -if not os.path.exists(SAVED_MODEL_PATH): - os.mkdir(SAVED_MODEL_PATH) -RESULT_PATH = curr_path+"/results/"+SEQUENCE+'/' # path to save rewards -if not os.path.exists(curr_path+"/results/"): - os.mkdir(curr_path+"/results/") -if not os.path.exists(RESULT_PATH): - os.mkdir(RESULT_PATH) - -class DQNcnnConfig: - def __init__(self) -> None: - self.algo = "DQN_cnn" # name of algo - self.gamma = 0.99 - self.epsilon_start = 0.95 # e-greedy策略的初始epsilon - self.epsilon_end = 0.05 - self.epsilon_decay = 200 - self.lr = 0.01 # leanring rate - self.memory_capacity = 10000 # Replay Memory容量 - self.batch_size = 64 - self.train_eps = 250 # 训练的episode数目 - self.train_steps = 200 # 训练每个episode的最大长度 - self.target_update = 4 # target net的更新频率 - self.eval_eps = 20 # 测试的episode数目 - self.eval_steps = 200 # 测试每个episode的最大长度 - self.hidden_dim = 128 # 神经网络隐藏层维度 - self.device = torch.device( - "cuda" if torch.cuda.is_available() else "cpu") # if gpu is to be used - -def train(cfg, env, agent): - rewards = [] - ma_rewards = [] - for i_episode in range(cfg.train_eps): - # Initialize the environment and state - env.reset() - last_screen = get_screen(env, cfg.device) - current_screen = get_screen(env, cfg.device) - state = current_screen - last_screen - ep_reward = 0 - for i_step in range(cfg.train_steps+1): - # Select and perform an action - action = agent.choose_action(state) - _, reward, done, _ = env.step(action.item()) - ep_reward += reward - reward = torch.tensor([reward], device=cfg.device) - # Observe new state - last_screen = current_screen - current_screen = get_screen(env, cfg.device) - if done: - break - state_ = current_screen - last_screen - # Store the transition in memory - agent.memory.push(state, action, state_, reward) - # Move to the next state - state = state_ - # Perform one step of the optimization (on the target network) - agent.update() - # Update the target network, copying all weights and biases in DQN - if i_episode % cfg.target_update == 0: - agent.target_net.load_state_dict(agent.policy_net.state_dict()) - print('Episode:{}/{}, Reward:{}, Steps:{}, Explore:{:.2f}, Done:{}'.format(i_episode+1,cfg.train_eps,ep_reward,i_step+1,agent.epsilon,done)) - rewards.append(ep_reward) - if ma_rewards: - ma_rewards.append(0.9*ma_rewards[-1]+0.1*ep_reward) - else: - ma_rewards.append(ep_reward) - return rewards,ma_rewards - - -if __name__ == "__main__": - cfg = DQNcnnConfig() - # Get screen size so that we can initialize layers correctly based on shape - # returned from AI gym. Typical dimensions at this point are close to 3x40x90 - # which is the result of a clamped and down-scaled render buffer in get_screen(env,device) - # 因为这里环境的state需要从默认的向量改为图像，所以要unwrapped更改state - env = gym.make('CartPole-v0').unwrapped - env.reset() - init_screen = get_screen(env, cfg.device) - _, _, screen_height, screen_width = init_screen.shape - # Get number of actions from gym action space - action_dim = env.action_space.n - agent = DQNcnn(screen_height, screen_width, - action_dim, cfg) - rewards,ma_rewards = train(cfg,env,agent) - save_results(rewards,ma_rewards,tag='train',path=RESULT_PATH) - plot_rewards(rewards,ma_rewards,tag="train",algo = cfg.algo,path=RESULT_PATH) diff --git a/codes/DQN_cnn/memory.py b/codes/DQN_cnn/memory.py deleted file mode 100644 index 7359a0c..0000000 --- a/codes/DQN_cnn/memory.py +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env python -# coding=utf-8 -''' -@Author: John -@Email: johnjim0816@gmail.com -@Date: 2020-06-11 09:42:44 -@LastEditor: John -LastEditTime: 2021-03-23 20:38:41 -@Discription: -@Environment: python 3.7.7 -''' -from collections import namedtuple -import random - -class ReplayBuffer(object): - - def __init__(self, capacity): - self.capacity = capacity - self.buffer = [] - self.position = 0 - self.Transition = namedtuple('Transition', - ('state', 'action', 'state_', 'reward')) - - def push(self, *args): - """Saves a transition.""" - if len(self.buffer) < self.capacity: - self.buffer.append(None) - self.buffer[self.position] = self.Transition(*args) - self.position = (self.position + 1) % self.capacity - - def sample(self, batch_size): - return random.sample(self.buffer, batch_size) - - def __len__(self): - return len(self.buffer) diff --git a/codes/DQN_cnn/model.py b/codes/DQN_cnn/model.py deleted file mode 100644 index 71e67ca..0000000 --- a/codes/DQN_cnn/model.py +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env python -# coding=utf-8 -''' -@Author: John -@Email: johnjim0816@gmail.com -@Date: 2020-06-11 12:18:12 -@LastEditor: John -@LastEditTime: 2020-06-11 17:23:45 -@Discription: -@Environment: python 3.7.7 -''' -import torch.nn as nn -import torch.nn.functional as F - -class CNN(nn.Module): - - def __init__(self, h, w, n_outputs): - super(CNN, self).__init__() - self.conv1 = nn.Conv2d(3, 16, kernel_size=5, stride=2) - self.bn1 = nn.BatchNorm2d(16) - self.conv2 = nn.Conv2d(16, 32, kernel_size=5, stride=2) - self.bn2 = nn.BatchNorm2d(32) - self.conv3 = nn.Conv2d(32, 32, kernel_size=5, stride=2) - self.bn3 = nn.BatchNorm2d(32) - - # Number of Linear input connections depends on output of conv2d layers - # and therefore the input image size, so compute it. - def conv2d_size_out(size, kernel_size = 5, stride = 2): - return (size - (kernel_size - 1) - 1) // stride + 1 - convw = conv2d_size_out(conv2d_size_out(conv2d_size_out(w))) - convh = conv2d_size_out(conv2d_size_out(conv2d_size_out(h))) - linear_input_size = convw * convh * 32 - self.head = nn.Linear(linear_input_size, n_outputs) - - # Called with either one element to determine next action, or a batch - # during optimization. Returns tensor([[left0exp,right0exp]...]). - def forward(self, x): - x = F.relu(self.bn1(self.conv1(x))) - x = F.relu(self.bn2(self.conv2(x))) - x = F.relu(self.bn3(self.conv3(x))) - return self.head(x.view(x.size(0), -1)) \ No newline at end of file diff --git a/codes/DoubleDQN/agent.py b/codes/DoubleDQN/agent.py index 34774c4..1ade5f8 100644 --- a/codes/DoubleDQN/agent.py +++ b/codes/DoubleDQN/agent.py @@ -5,7 +5,7 @@ @Email: johnjim0816@gmail.com @Date: 2020-06-12 00:50:49 @LastEditor: John -LastEditTime: 2021-03-28 11:07:35 +LastEditTime: 2021-05-04 22:28:06 @Discription: @Environment: python 3.7.7 ''' @@ -35,22 +35,16 @@ class DoubleDQN: self.batch_size = cfg.batch_size self.policy_net = MLP(state_dim, action_dim,hidden_dim=cfg.hidden_dim).to(self.device) self.target_net = MLP(state_dim, action_dim,hidden_dim=cfg.hidden_dim).to(self.device) - # target_net的初始模型参数完全复制policy_net - self.target_net.load_state_dict(self.policy_net.state_dict()) - self.target_net.eval() # 不启用 BatchNormalization 和 Dropout + # target_net copy from policy_net + for target_param, param in zip(self.target_net.parameters(), self.policy_net.parameters()): + target_param.data.copy_(param.data) + # self.target_net.eval() # 不启用 BatchNormalization 和 Dropout # 可查parameters()与state_dict()的区别，前者require_grad=True self.optimizer = optim.Adam(self.policy_net.parameters(), lr=cfg.lr) self.loss = 0 self.memory = ReplayBuffer(cfg.memory_capacity) - - def choose_action(self, state): - '''选择动作 - ''' - self.epsilon = self.epsilon_end + (self.epsilon_start - self.epsilon_end) * \ - math.exp(-1. * self.actions_count / self.epsilon_decay) - self.actions_count += 1 - if random.random() > self.epsilon: - with torch.no_grad(): + def predict(self,state): + with torch.no_grad(): # 先转为张量便于丢给神经网络,state元素数据原本为float64 # 注意state=torch.tensor(state).unsqueeze(0)跟state=torch.tensor([state])等价 state = torch.tensor( @@ -61,6 +55,15 @@ class DoubleDQN: # 如torch.return_types.max(values=tensor([10.3587]),indices=tensor([0])) # 所以tensor.max(1)[1]返回最大值对应的下标，即action action = q_value.max(1)[1].item() + return action + def choose_action(self, state): + '''选择动作 + ''' + self.actions_count += 1 + self.epsilon = self.epsilon_end + (self.epsilon_start - self.epsilon_end) * \ + math.exp(-1. * self.actions_count / self.epsilon_decay) + if random.random() > self.epsilon: + action = self.predict(state) else: action = random.randrange(self.action_dim) return action @@ -71,7 +74,7 @@ class DoubleDQN: # 从memory中随机采样transition state_batch, action_batch, reward_batch, next_state_batch, done_batch = self.memory.sample( self.batch_size) - ### 转为张量 ### + # convert to tensor state_batch = torch.tensor( state_batch, device=self.device, dtype=torch.float) action_batch = torch.tensor(action_batch, device=self.device).unsqueeze( @@ -82,8 +85,7 @@ class DoubleDQN: next_state_batch, device=self.device, dtype=torch.float) done_batch = torch.tensor(np.float32( - done_batch), device=self.device).unsqueeze(1) # 将bool转为float然后转为张量 - + done_batch), device=self.device) # 将bool转为float然后转为张量 # 计算当前(s_t,a)对应的Q(s_t, a) q_values = self.policy_net(state_batch) next_q_values = self.policy_net(next_state_batch) @@ -102,7 +104,7 @@ class DoubleDQN: next_state_batch) # 选出Q(s_t‘, a)对应的action，代入到next_target_values获得target net对应的next_q_value，即Q’(s_t|a=argmax Q(s_t‘, a)) next_target_q_value = next_target_values.gather(1, torch.max(next_q_values, 1)[1].unsqueeze(1)).squeeze(1) - q_target = reward_batch + self.gamma * next_target_q_value * (1-done_batch[0]) + q_target = reward_batch + self.gamma * next_target_q_value * (1-done_batch) self.loss = nn.MSELoss()(q_value, q_target.unsqueeze(1)) # 计算 均方误差loss # 优化模型 self.optimizer.zero_grad() # zero_grad清除上一步所有旧的gradients from the last step @@ -113,7 +115,9 @@ class DoubleDQN: self.optimizer.step() # 更新模型 def save(self,path): - torch.save(self.target_net.state_dict(), path+'DoubleDQN_checkpoint.pth') + torch.save(self.target_net.state_dict(), path+'checkpoint.pth') def load(self,path): - self.target_net.load_state_dict(torch.load(path+'DoubleDQN_checkpoint.pth')) + self.target_net.load_state_dict(torch.load(path+'checkpoint.pth')) + for target_param, param in zip(self.target_net.parameters(), self.policy_net.parameters()): + param.data.copy_(target_param.data) diff --git a/codes/DoubleDQN/main.py b/codes/DoubleDQN/main.py deleted file mode 100644 index 57c9f9c..0000000 --- a/codes/DoubleDQN/main.py +++ /dev/null @@ -1,93 +0,0 @@ -#!/usr/bin/env python -# coding=utf-8 -''' -@Author: John -@Email: johnjim0816@gmail.com -@Date: 2020-06-12 00:48:57 -@LastEditor: John -LastEditTime: 2021-03-28 11:05:14 -@Discription: -@Environment: python 3.7.7 -''' -import sys,os -sys.path.append(os.getcwd()) # add current terminal path -import gym -import torch -import datetime -from DoubleDQN.agent import DoubleDQN -from common.plot import plot_rewards -from common.utils import save_results - -SEQUENCE = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") # 获取当前时间 -SAVED_MODEL_PATH = os.path.split(os.path.abspath(__file__))[0]+"/saved_model/"+SEQUENCE+'/' # 生成保存的模型路径 -if not os.path.exists(os.path.split(os.path.abspath(__file__))[0]+"/saved_model/"): - os.mkdir(os.path.split(os.path.abspath(__file__))[0]+"/saved_model/") -if not os.path.exists(SAVED_MODEL_PATH): - os.mkdir(SAVED_MODEL_PATH) -RESULT_PATH = os.path.split(os.path.abspath(__file__))[0]+"/results/"+SEQUENCE+'/' # 存储reward的路径 -if not os.path.exists(os.path.split(os.path.abspath(__file__))[0]+"/results/"): - os.mkdir(os.path.split(os.path.abspath(__file__))[0]+"/results/") -if not os.path.exists(RESULT_PATH): - os.mkdir(RESULT_PATH) - -class DoubleDQNConfig: - def __init__(self): - self.algo = "Double DQN" # name of algo - self.gamma = 0.99 - self.epsilon_start = 0.9 # e-greedy策略的初始epsilon - self.epsilon_end = 0.01 - self.epsilon_decay = 200 - self.lr = 0.01 # 学习率 - self.memory_capacity = 10000 # Replay Memory容量 - self.batch_size = 128 - self.train_eps = 300 # 训练的episode数目 - self.train_steps = 200 # 训练每个episode的最大长度 - self.target_update = 2 # target net的更新频率 - self.eval_eps = 20 # 测试的episode数目 - self.eval_steps = 200 # 测试每个episode的最大长度 - self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # 检测gpu - self.hidden_dim = 128 # 神经网络隐藏层维度 - - -def train(cfg,env,agent): - print('Start to train !') - rewards,ma_rewards = [],[] - ep_steps = [] - for i_episode in range(cfg.train_eps): - state = env.reset() # reset环境状态 - ep_reward = 0 - for i_step in range(cfg.train_steps): - action = agent.choose_action(state) # 根据当前环境state选择action - next_state, reward, done, _ = env.step(action) # 更新环境参数 - ep_reward += reward - agent.memory.push(state, action, reward, next_state, done) # 将state等这些transition存入memory - state = next_state # 跳转到下一个状态 - agent.update() # 每步更新网络 - if done: - break - # 更新target network，复制DQN中的所有weights and biases - if i_episode % cfg.target_update == 0: - agent.target_net.load_state_dict(agent.policy_net.state_dict()) - print('Episode:{}/{}, Reward:{}, Steps:{}, Done:{}'.format(i_episode+1,cfg.train_eps,ep_reward,i_step,done)) - ep_steps.append(i_step) - rewards.append(ep_reward) - # 计算滑动窗口的reward - if ma_rewards: - ma_rewards.append( - 0.9*ma_rewards[-1]+0.1*ep_reward) - else: - ma_rewards.append(ep_reward) - print('Complete training！') - return rewards,ma_rewards - -if __name__ == "__main__": - cfg = DoubleDQNConfig() - env = gym.make('CartPole-v0').unwrapped # 可google为什么unwrapped gym，此处一般不需要 - env.seed(1) # 设置env随机种子 - state_dim = env.observation_space.shape[0] - action_dim = env.action_space.n - agent = DoubleDQN(state_dim,action_dim,cfg) - rewards,ma_rewards = train(cfg,env,agent) - agent.save(path=SAVED_MODEL_PATH) - save_results(rewards,ma_rewards,tag='train',path=RESULT_PATH) - plot_rewards(rewards,ma_rewards,tag="train",algo = cfg.algo,path=RESULT_PATH) diff --git a/codes/DoubleDQN/outputs/CartPole-v0/20210504-150900/models/checkpoint.pth b/codes/DoubleDQN/outputs/CartPole-v0/20210504-150900/models/checkpoint.pth new file mode 100644 index 0000000..8c4b561 Binary files /dev/null and b/codes/DoubleDQN/outputs/CartPole-v0/20210504-150900/models/checkpoint.pth differ diff --git a/codes/DoubleDQN/outputs/CartPole-v0/20210504-150900/results/eval_ma_rewards.npy b/codes/DoubleDQN/outputs/CartPole-v0/20210504-150900/results/eval_ma_rewards.npy new file mode 100644 index 0000000..0f77696 Binary files /dev/null and b/codes/DoubleDQN/outputs/CartPole-v0/20210504-150900/results/eval_ma_rewards.npy differ diff --git a/codes/DoubleDQN/outputs/CartPole-v0/20210504-150900/results/eval_rewards.npy b/codes/DoubleDQN/outputs/CartPole-v0/20210504-150900/results/eval_rewards.npy new file mode 100644 index 0000000..57f8759 Binary files /dev/null and b/codes/DoubleDQN/outputs/CartPole-v0/20210504-150900/results/eval_rewards.npy differ diff --git a/codes/DoubleDQN/outputs/CartPole-v0/20210504-150900/results/eval_rewards_curve.png b/codes/DoubleDQN/outputs/CartPole-v0/20210504-150900/results/eval_rewards_curve.png new file mode 100644 index 0000000..038e031 Binary files /dev/null and b/codes/DoubleDQN/outputs/CartPole-v0/20210504-150900/results/eval_rewards_curve.png differ diff --git a/codes/DoubleDQN/outputs/CartPole-v0/20210504-150900/results/train_ma_rewards.npy b/codes/DoubleDQN/outputs/CartPole-v0/20210504-150900/results/train_ma_rewards.npy new file mode 100644 index 0000000..63d10e7 Binary files /dev/null and b/codes/DoubleDQN/outputs/CartPole-v0/20210504-150900/results/train_ma_rewards.npy differ diff --git a/codes/DoubleDQN/outputs/CartPole-v0/20210504-150900/results/train_rewards.npy b/codes/DoubleDQN/outputs/CartPole-v0/20210504-150900/results/train_rewards.npy new file mode 100644 index 0000000..d486ad9 Binary files /dev/null and b/codes/DoubleDQN/outputs/CartPole-v0/20210504-150900/results/train_rewards.npy differ diff --git a/codes/DoubleDQN/outputs/CartPole-v0/20210504-150900/results/train_rewards_curve.png b/codes/DoubleDQN/outputs/CartPole-v0/20210504-150900/results/train_rewards_curve.png new file mode 100644 index 0000000..f91bc4d Binary files /dev/null and b/codes/DoubleDQN/outputs/CartPole-v0/20210504-150900/results/train_rewards_curve.png differ diff --git a/codes/DoubleDQN/results/20210328-110516/ma_rewards_train.npy b/codes/DoubleDQN/results/20210328-110516/ma_rewards_train.npy deleted file mode 100644 index 1c4be2b..0000000 Binary files a/codes/DoubleDQN/results/20210328-110516/ma_rewards_train.npy and /dev/null differ diff --git a/codes/DoubleDQN/results/20210328-110516/rewards_curve_train.png b/codes/DoubleDQN/results/20210328-110516/rewards_curve_train.png deleted file mode 100644 index 2817223..0000000 Binary files a/codes/DoubleDQN/results/20210328-110516/rewards_curve_train.png and /dev/null differ diff --git a/codes/DoubleDQN/results/20210328-110516/rewards_train.npy b/codes/DoubleDQN/results/20210328-110516/rewards_train.npy deleted file mode 100644 index 73acfde..0000000 Binary files a/codes/DoubleDQN/results/20210328-110516/rewards_train.npy and /dev/null differ diff --git a/codes/DoubleDQN/saved_model/20210328-110516/DoubleDQN_checkpoint.pth b/codes/DoubleDQN/saved_model/20210328-110516/DoubleDQN_checkpoint.pth deleted file mode 100644 index 69f5fce..0000000 Binary files a/codes/DoubleDQN/saved_model/20210328-110516/DoubleDQN_checkpoint.pth and /dev/null differ diff --git a/codes/DoubleDQN/task0_train.ipynb b/codes/DoubleDQN/task0_train.ipynb new file mode 100644 index 0000000..ee2e5d4 --- /dev/null +++ b/codes/DoubleDQN/task0_train.ipynb @@ -0,0 +1,194 @@ +{ + "metadata": { + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.10" + }, + "orig_nbformat": 2, + "kernelspec": { + "name": "python3710jvsc74a57bd0366e1054dee9d4501b0eb8f87335afd3c67fc62db6ee611bbc7f8f5a1fefe232", + "display_name": "Python 3.7.10 64-bit ('py37': conda)" + }, + "metadata": { + "interpreter": { + "hash": "366e1054dee9d4501b0eb8f87335afd3c67fc62db6ee611bbc7f8f5a1fefe232" + } + } + }, + "nbformat": 4, + "nbformat_minor": 2, + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "from pathlib import Path\n", + "curr_path = str(Path().absolute())\n", + "parent_path = str(Path().absolute().parent)\n", + "sys.path.append(parent_path) # add current terminal path to sys.path" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import gym\n", + "import torch\n", + "import datetime\n", + "from DoubleDQN.agent import DoubleDQN\n", + "from common.plot import plot_rewards\n", + "from common.utils import save_results, make_dir\n", + "\n", + "curr_time = datetime.datetime.now().strftime(\n", + " \"%Y%m%d-%H%M%S\") # obtain current time" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "class DoubleDQNConfig:\n", + " def __init__(self):\n", + " self.algo = \"DoubleDQN\" # name of algo\n", + " self.env = 'CartPole-v0' # env name\n", + " self.result_path = curr_path+\"/outputs/\" + self.env + \\\n", + " '/'+curr_time+'/results/' # path to save results\n", + " self.model_path = curr_path+\"/outputs/\" + self.env + \\\n", + " '/'+curr_time+'/models/' # path to save models\n", + " self.train_eps = 200 # max tranng episodes\n", + " self.eval_eps = 50 # max evaling episodes\n", + " self.gamma = 0.95\n", + " self.epsilon_start = 1 # start epsilon of e-greedy policy\n", + " self.epsilon_end = 0.01 \n", + " self.epsilon_decay = 500\n", + " self.lr = 0.001 # learning rate\n", + " self.memory_capacity = 100000 # capacity of Replay Memory\n", + " self.batch_size = 64\n", + " self.target_update = 2 # update frequency of target net\n", + " self.device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\") # check gpu\n", + " self.hidden_dim = 256 # hidden size of net" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def env_agent_config(cfg,seed=1):\n", + " env = gym.make(cfg.env) \n", + " env.seed(seed)\n", + " state_dim = env.observation_space.shape[0]\n", + " action_dim = env.action_space.n\n", + " agent = DoubleDQN(state_dim,action_dim,cfg)\n", + " return env,agent" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def train(cfg,env,agent):\n", + " print('Start to train !')\n", + " rewards,ma_rewards = [],[]\n", + " for i_ep in range(cfg.train_eps):\n", + " state = env.reset() \n", + " ep_reward = 0\n", + " while True:\n", + " action = agent.choose_action(state) \n", + " next_state, reward, done, _ = env.step(action)\n", + " ep_reward += reward\n", + " agent.memory.push(state, action, reward, next_state, done) \n", + " state = next_state \n", + " agent.update() \n", + " if done:\n", + " break\n", + " if i_ep % cfg.target_update == 0:\n", + " agent.target_net.load_state_dict(agent.policy_net.state_dict())\n", + " if (i_ep+1)%10 == 0:\n", + " print(f'Episode:{i_ep+1}/{cfg.train_eps}, Reward:{ep_reward}')\n", + " rewards.append(ep_reward)\n", + " if ma_rewards:\n", + " ma_rewards.append(\n", + " 0.9*ma_rewards[-1]+0.1*ep_reward)\n", + " else:\n", + " ma_rewards.append(ep_reward) \n", + " print('Complete training！')\n", + " return rewards,ma_rewards" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def eval(cfg,env,agent):\n", + " print('Start to eval !')\n", + " print(f'Env:{cfg.env}, Algorithm:{cfg.algo}, Device:{cfg.device}')\n", + " rewards = [] \n", + " ma_rewards = []\n", + " for i_ep in range(cfg.eval_eps):\n", + " state = env.reset() \n", + " ep_reward = 0 \n", + " while True:\n", + " action = agent.predict(state) \n", + " next_state, reward, done, _ = env.step(action) \n", + " state = next_state \n", + " ep_reward += reward\n", + " if done:\n", + " break\n", + " rewards.append(ep_reward)\n", + " if ma_rewards:\n", + " ma_rewards.append(ma_rewards[-1]*0.9+ep_reward*0.1)\n", + " else:\n", + " ma_rewards.append(ep_reward)\n", + " print(f\"Episode:{i_ep+1}/{cfg.eval_eps}, reward:{ep_reward:.1f}\")\n", + " print('Complete evaling！')\n", + " return rewards,ma_rewards " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "if __name__ == \"__main__\":\n", + " cfg = DoubleDQNConfig()\n", + " # train\n", + " env,agent = env_agent_config(cfg,seed=1)\n", + " rewards, ma_rewards = train(cfg, env, agent)\n", + " make_dir(cfg.result_path, cfg.model_path)\n", + " agent.save(path=cfg.model_path)\n", + " save_results(rewards, ma_rewards, tag='train', path=cfg.result_path)\n", + " plot_rewards(rewards, ma_rewards, tag=\"train\",\n", + " algo=cfg.algo, path=cfg.result_path)\n", + "\n", + " # eval\n", + " env,agent = env_agent_config(cfg,seed=10)\n", + " agent.load(path=cfg.model_path)\n", + " rewards,ma_rewards = eval(cfg,env,agent)\n", + " save_results(rewards,ma_rewards,tag='eval',path=cfg.result_path)\n", + " plot_rewards(rewards,ma_rewards,tag=\"eval\",env=cfg.env,algo = cfg.algo,path=cfg.result_path)" + ] + } + ] +} \ No newline at end of file diff --git a/codes/DoubleDQN/task0_train.py b/codes/DoubleDQN/task0_train.py new file mode 100644 index 0000000..8d0f842 --- /dev/null +++ b/codes/DoubleDQN/task0_train.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python +# coding=utf-8 +''' +@Author: John +@Email: johnjim0816@gmail.com +@Date: 2020-06-12 00:48:57 +@LastEditor: John +LastEditTime: 2021-05-04 22:26:59 +@Discription: +@Environment: python 3.7.7 +''' +import sys,os +curr_path = os.path.dirname(__file__) +parent_path = os.path.dirname(curr_path) +sys.path.append(parent_path) # add current terminal path to sys.path + +import gym +import torch +import datetime +from DoubleDQN.agent import DoubleDQN +from common.plot import plot_rewards +from common.utils import save_results, make_dir + +curr_time = datetime.datetime.now().strftime( + "%Y%m%d-%H%M%S") # obtain current time + +class DoubleDQNConfig: + def __init__(self): + self.algo = "DoubleDQN" # name of algo + self.env = 'CartPole-v0' # env name + self.result_path = curr_path+"/outputs/" + self.env + \ + '/'+curr_time+'/results/' # path to save results + self.model_path = curr_path+"/outputs/" + self.env + \ + '/'+curr_time+'/models/' # path to save models + self.train_eps = 200 # max tranng episodes + self.eval_eps = 50 # max evaling episodes + self.gamma = 0.95 + self.epsilon_start = 1 # start epsilon of e-greedy policy + self.epsilon_end = 0.01 + self.epsilon_decay = 500 + self.lr = 0.001 # learning rate + self.memory_capacity = 100000 # capacity of Replay Memory + self.batch_size = 64 + self.target_update = 2 # update frequency of target net + self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # check gpu + self.hidden_dim = 256 # hidden size of net + +def env_agent_config(cfg,seed=1): + env = gym.make(cfg.env) + env.seed(seed) + state_dim = env.observation_space.shape[0] + action_dim = env.action_space.n + agent = DoubleDQN(state_dim,action_dim,cfg) + return env,agent + +def train(cfg,env,agent): + print('Start to train !') + rewards,ma_rewards = [],[] + for i_ep in range(cfg.train_eps): + state = env.reset() + ep_reward = 0 + while True: + action = agent.choose_action(state) + next_state, reward, done, _ = env.step(action) + ep_reward += reward + agent.memory.push(state, action, reward, next_state, done) + state = next_state + agent.update() + if done: + break + if i_ep % cfg.target_update == 0: + agent.target_net.load_state_dict(agent.policy_net.state_dict()) + print(f'Episode:{i_ep+1}/{cfg.train_eps}, Reward:{ep_reward},Epsilon:{agent.epsilon:.2f}') + rewards.append(ep_reward) + if ma_rewards: + ma_rewards.append( + 0.9*ma_rewards[-1]+0.1*ep_reward) + else: + ma_rewards.append(ep_reward) + print('Complete training！') + return rewards,ma_rewards + +def eval(cfg,env,agent): + print('Start to eval !') + print(f'Env:{cfg.env}, Algorithm:{cfg.algo}, Device:{cfg.device}') + rewards = [] + ma_rewards = [] + for i_ep in range(cfg.eval_eps): + state = env.reset() + ep_reward = 0 + while True: + action = agent.predict(state) + next_state, reward, done, _ = env.step(action) + state = next_state + ep_reward += reward + if done: + break + rewards.append(ep_reward) + if ma_rewards: + ma_rewards.append(ma_rewards[-1]*0.9+ep_reward*0.1) + else: + ma_rewards.append(ep_reward) + print(f"Episode:{i_ep+1}/{cfg.eval_eps}, reward:{ep_reward:.1f}") + print('Complete evaling！') + return rewards,ma_rewards + +if __name__ == "__main__": + cfg = DoubleDQNConfig() + # train + env,agent = env_agent_config(cfg,seed=1) + rewards, ma_rewards = train(cfg, env, agent) + make_dir(cfg.result_path, cfg.model_path) + agent.save(path=cfg.model_path) + save_results(rewards, ma_rewards, tag='train', path=cfg.result_path) + plot_rewards(rewards, ma_rewards, tag="train", + algo=cfg.algo, path=cfg.result_path) + + # eval + env,agent = env_agent_config(cfg,seed=10) + agent.load(path=cfg.model_path) + rewards,ma_rewards = eval(cfg,env,agent) + save_results(rewards,ma_rewards,tag='eval',path=cfg.result_path) + plot_rewards(rewards,ma_rewards,tag="eval",env=cfg.env,algo = cfg.algo,path=cfg.result_path) diff --git a/codes/DoubleDQN/utils.py b/codes/DoubleDQN/utils.py deleted file mode 100644 index c5f5305..0000000 --- a/codes/DoubleDQN/utils.py +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env python -# coding=utf-8 -''' -Author: John -Email: johnjim0816@gmail.com -Date: 2020-10-15 21:28:00 -LastEditor: John -LastEditTime: 2020-10-15 21:50:30 -Discription: -Environment: -''' -import os -import numpy as np - - -def save_results(rewards,moving_average_rewards,ep_steps,tag='train',result_path='./results'): - if not os.path.exists(result_path): # 检测是否存在文件夹 - os.mkdir(result_path) - np.save(result_path+'rewards_'+tag+'.npy', rewards) - np.save(result_path+'moving_average_rewards_'+tag+'.npy', moving_average_rewards) - np.save(result_path+'steps_'+tag+'.npy',ep_steps ) \ No newline at end of file diff --git a/codes/HierarchicalDQN/agent.py b/codes/HierarchicalDQN/agent.py index bcfe1fa..3760643 100644 --- a/codes/HierarchicalDQN/agent.py +++ b/codes/HierarchicalDQN/agent.py @@ -5,7 +5,7 @@ Author: John Email: johnjim0816@gmail.com Date: 2021-03-24 22:18:18 LastEditor: John -LastEditTime: 2021-03-31 14:51:09 +LastEditTime: 2021-05-04 22:39:34 Discription: Environment: ''' @@ -65,11 +65,11 @@ class HierarchicalDQN: if self.batch_size > len(self.memory): return state_batch, action_batch, reward_batch, next_state_batch, done_batch = self.memory.sample(self.batch_size) - state_batch = torch.tensor(state_batch,dtype=torch.float) - action_batch = torch.tensor(action_batch,dtype=torch.int64).unsqueeze(1) - reward_batch = torch.tensor(reward_batch,dtype=torch.float) - next_state_batch = torch.tensor(next_state_batch, dtype=torch.float) - done_batch = torch.tensor(np.float32(done_batch)) + state_batch = torch.tensor(state_batch,device=self.device,dtype=torch.float) + action_batch = torch.tensor(action_batch,device=self.device,dtype=torch.int64).unsqueeze(1) + reward_batch = torch.tensor(reward_batch,device=self.device,dtype=torch.float) + next_state_batch = torch.tensor(next_state_batch,device=self.device, dtype=torch.float) + done_batch = torch.tensor(np.float32(done_batch),device=self.device) q_values = self.policy_net(state_batch).gather(dim=1, index=action_batch).squeeze(1) next_state_values = self.policy_net(next_state_batch).max(1)[0].detach() expected_q_values = reward_batch + 0.99 * next_state_values * (1-done_batch) @@ -79,17 +79,17 @@ class HierarchicalDQN: for param in self.policy_net.parameters(): # clip防止梯度爆炸 param.grad.data.clamp_(-1, 1) self.optimizer.step() - self.loss_numpy = loss.detach().numpy() + self.loss_numpy = loss.detach().cpu().numpy() self.losses.append(self.loss_numpy) def update_meta(self): if self.batch_size > len(self.meta_memory): return state_batch, action_batch, reward_batch, next_state_batch, done_batch = self.meta_memory.sample(self.batch_size) - state_batch = torch.tensor(state_batch,dtype=torch.float) - action_batch = torch.tensor(action_batch,dtype=torch.int64).unsqueeze(1) - reward_batch = torch.tensor(reward_batch,dtype=torch.float) - next_state_batch = torch.tensor(next_state_batch, dtype=torch.float) - done_batch = torch.tensor(np.float32(done_batch)) + state_batch = torch.tensor(state_batch,device=self.device,dtype=torch.float) + action_batch = torch.tensor(action_batch,device=self.device,dtype=torch.int64).unsqueeze(1) + reward_batch = torch.tensor(reward_batch,device=self.device,dtype=torch.float) + next_state_batch = torch.tensor(next_state_batch,device=self.device, dtype=torch.float) + done_batch = torch.tensor(np.float32(done_batch),device=self.device) q_values = self.meta_policy_net(state_batch).gather(dim=1, index=action_batch).squeeze(1) next_state_values = self.meta_policy_net(next_state_batch).max(1)[0].detach() expected_q_values = reward_batch + 0.99 * next_state_values * (1-done_batch) @@ -99,7 +99,7 @@ class HierarchicalDQN: for param in self.meta_policy_net.parameters(): # clip防止梯度爆炸 param.grad.data.clamp_(-1, 1) self.meta_optimizer.step() - self.meta_loss_numpy = meta_loss.detach().numpy() + self.meta_loss_numpy = meta_loss.detach().cpu().numpy() self.meta_losses.append(self.meta_loss_numpy) def save(self, path): diff --git a/codes/HierarchicalDQN/main.ipynb b/codes/HierarchicalDQN/task0_train.ipynb similarity index 100% rename from codes/HierarchicalDQN/main.ipynb rename to codes/HierarchicalDQN/task0_train.ipynb diff --git a/codes/HierarchicalDQN/main.py b/codes/HierarchicalDQN/task0_train.py similarity index 51% rename from codes/HierarchicalDQN/main.py rename to codes/HierarchicalDQN/task0_train.py index ea6dfdc..2676094 100644 --- a/codes/HierarchicalDQN/main.py +++ b/codes/HierarchicalDQN/task0_train.py @@ -5,7 +5,7 @@ Author: John Email: johnjim0816@gmail.com Date: 2021-03-29 10:37:32 LastEditor: John -LastEditTime: 2021-03-31 14:58:49 +LastEditTime: 2021-05-04 22:35:56 Discription: Environment: ''' @@ -21,27 +21,23 @@ import numpy as np import torch import gym -from common.utils import save_results -from common.plot import plot_rewards,plot_losses +from common.utils import save_results,make_dir +from common.plot import plot_rewards from HierarchicalDQN.agent import HierarchicalDQN -SEQUENCE = datetime.datetime.now().strftime( +curr_time = datetime.datetime.now().strftime( "%Y%m%d-%H%M%S") # obtain current time -SAVED_MODEL_PATH = curr_path+"/saved_model/"+SEQUENCE+'/' # path to save model -if not os.path.exists(curr_path+"/saved_model/"): - os.mkdir(curr_path+"/saved_model/") -if not os.path.exists(SAVED_MODEL_PATH): - os.mkdir(SAVED_MODEL_PATH) -RESULT_PATH = curr_path+"/results/"+SEQUENCE+'/' # path to save rewards -if not os.path.exists(curr_path+"/results/"): - os.mkdir(curr_path+"/results/") -if not os.path.exists(RESULT_PATH): - os.mkdir(RESULT_PATH) - class HierarchicalDQNConfig: def __init__(self): self.algo = "H-DQN" # name of algo + self.env = 'CartPole-v0' + self.result_path = curr_path+"/outputs/" + self.env + \ + '/'+curr_time+'/results/' # path to save results + self.model_path = curr_path+"/outputs/" + self.env + \ + '/'+curr_time+'/models/' # path to save models + self.train_eps = 300 # 训练的episode数目 + self.eval_eps = 50 # 测试的episode数目 self.gamma = 0.99 self.epsilon_start = 1 # start epsilon of e-greedy policy self.epsilon_end = 0.01 @@ -49,19 +45,25 @@ class HierarchicalDQNConfig: self.lr = 0.0001 # learning rate self.memory_capacity = 10000 # Replay Memory capacity self.batch_size = 32 - self.train_eps = 300 # 训练的episode数目 self.target_update = 2 # target net的更新频率 - self.eval_eps = 20 # 测试的episode数目 self.device = torch.device( "cuda" if torch.cuda.is_available() else "cpu") # 检测gpu self.hidden_dim = 256 # dimension of hidden layer +def env_agent_config(cfg,seed=1): + env = gym.make(cfg.env) + env.seed(seed) + state_dim = env.observation_space.shape[0] + action_dim = env.action_space.n + agent = HierarchicalDQN(state_dim,action_dim,cfg) + return env,agent def train(cfg, env, agent): print('Start to train !') + print(f'Env:{cfg.env}, Algorithm:{cfg.algo}, Device:{cfg.device}') rewards = [] ma_rewards = [] # moveing average reward - for i_episode in range(cfg.train_eps): + for i_ep in range(cfg.train_eps): state = env.reset() done = False ep_reward = 0 @@ -83,7 +85,7 @@ def train(cfg, env, agent): state = next_state agent.update() agent.meta_memory.push(meta_state, goal, extrinsic_reward, state, done) - print('Episode:{}/{}, Reward:{}, Loss:{:.2f}, Meta_Loss:{:.2f}'.format(i_episode+1, cfg.train_eps, ep_reward,agent.loss_numpy ,agent.meta_loss_numpy )) + print('Episode:{}/{}, Reward:{}, Loss:{:.2f}, Meta_Loss:{:.2f}'.format(i_ep+1, cfg.train_eps, ep_reward,agent.loss_numpy ,agent.meta_loss_numpy )) rewards.append(ep_reward) if ma_rewards: ma_rewards.append( @@ -93,18 +95,52 @@ def train(cfg, env, agent): print('Complete training！') return rewards, ma_rewards +def eval(cfg, env, agent): + print('Start to eval !') + print(f'Env:{cfg.env}, Algorithm:{cfg.algo}, Device:{cfg.device}') + rewards = [] + ma_rewards = [] # moveing average reward + for i_ep in range(cfg.train_eps): + state = env.reset() + done = False + ep_reward = 0 + while not done: + goal = agent.set_goal(state) + onehot_goal = agent.to_onehot(goal) + extrinsic_reward = 0 + while not done and goal != np.argmax(state): + goal_state = np.concatenate([state, onehot_goal]) + action = agent.choose_action(goal_state) + next_state, reward, done, _ = env.step(action) + ep_reward += reward + extrinsic_reward += reward + state = next_state + agent.update() + print(f'Episode:{i_ep+1}/{cfg.train_eps}, Reward:{ep_reward}, Loss:{agent.loss_numpy:.2f}, Meta_Loss:{agent.meta_loss_numpy:.2f}') + rewards.append(ep_reward) + if ma_rewards: + ma_rewards.append( + 0.9*ma_rewards[-1]+0.1*ep_reward) + else: + ma_rewards.append(ep_reward) + print('Complete training！') + return rewards, ma_rewards if __name__ == "__main__": - env = gym.make('CartPole-v0') - env.seed(1) cfg = HierarchicalDQNConfig() - state_dim = env.observation_space.shape[0] - action_dim = env.action_space.n - agent = HierarchicalDQN(state_dim, action_dim, cfg) - rewards, ma_rewards = train(cfg, env, agent) - agent.save(path=SAVED_MODEL_PATH) - save_results(rewards, ma_rewards, tag='train', path=RESULT_PATH) - plot_rewards(rewards, ma_rewards, tag="train", - algo=cfg.algo, path=RESULT_PATH) - plot_losses(agent.losses,algo=cfg.algo, path=RESULT_PATH) + + # train + env,agent = env_agent_config(cfg,seed=1) + rewards, ma_rewards = train(cfg, env, agent) + make_dir(cfg.result_path, cfg.model_path) + agent.save(path=cfg.model_path) + save_results(rewards, ma_rewards, tag='train', path=cfg.result_path) + plot_rewards(rewards, ma_rewards, tag="train", + algo=cfg.algo, path=cfg.result_path) + # eval + env,agent = env_agent_config(cfg,seed=10) + agent.load(path=cfg.model_path) + rewards,ma_rewards = eval(cfg,env,agent) + save_results(rewards,ma_rewards,tag='eval',path=cfg.result_path) + plot_rewards(rewards,ma_rewards,tag="eval",env=cfg.env,algo = cfg.algo,path=cfg.result_path) diff --git a/codes/MonteCarlo/agent.py b/codes/MonteCarlo/agent.py index 3ec4d7a..44af71d 100644 --- a/codes/MonteCarlo/agent.py +++ b/codes/MonteCarlo/agent.py @@ -5,13 +5,14 @@ Author: John Email: johnjim0816@gmail.com Date: 2021-03-12 16:14:34 LastEditor: John -LastEditTime: 2021-03-17 12:35:06 +LastEditTime: 2021-05-05 16:58:39 Discription: Environment: ''' import numpy as np from collections import defaultdict import torch +import dill class FisrtVisitMC: ''' On-Policy First-Visit MC Control @@ -20,14 +21,14 @@ class FisrtVisitMC: self.action_dim = action_dim self.epsilon = cfg.epsilon self.gamma = cfg.gamma - self.Q = defaultdict(lambda: np.zeros(action_dim)) + self.Q_table = defaultdict(lambda: np.zeros(action_dim)) self.returns_sum = defaultdict(float) # sum of returns self.returns_count = defaultdict(float) def choose_action(self,state): ''' e-greed policy ''' - if state in self.Q.keys(): - best_action = np.argmax(self.Q[state]) + if state in self.Q_table.keys(): + best_action = np.argmax(self.Q_table[state]) action_probs = np.ones(self.action_dim, dtype=float) * self.epsilon / self.action_dim action_probs[best_action] += (1.0 - self.epsilon) action = np.random.choice(np.arange(len(action_probs)), p=action_probs) @@ -48,19 +49,17 @@ class FisrtVisitMC: # Calculate average return for this state over all sampled episodes self.returns_sum[sa_pair] += G self.returns_count[sa_pair] += 1.0 - self.Q[state][action] = self.returns_sum[sa_pair] / self.returns_count[sa_pair] + self.Q_table[state][action] = self.returns_sum[sa_pair] / self.returns_count[sa_pair] def save(self,path): '''把 Q表格 的数据保存到文件中 ''' - import dill torch.save( - obj=self.Q, - f=path, + obj=self.Q_table, + f=path+"Q_table", pickle_module=dill ) def load(self, path): '''从文件中读取数据到 Q表格 ''' - import dill - self.Q =torch.load(f=path,pickle_module=dill) \ No newline at end of file + self.Q_table =torch.load(f=path+"Q_table",pickle_module=dill) \ No newline at end of file diff --git a/codes/MonteCarlo/main.py b/codes/MonteCarlo/main.py deleted file mode 100644 index c984475..0000000 --- a/codes/MonteCarlo/main.py +++ /dev/null @@ -1,88 +0,0 @@ -#!/usr/bin/env python -# coding=utf-8 -''' -Author: John -Email: johnjim0816@gmail.com -Date: 2021-03-11 14:26:44 -LastEditor: John -LastEditTime: 2021-03-17 12:35:36 -Discription: -Environment: -''' -import sys,os -sys.path.append(os.getcwd()) -import argparse -import datetime - -from envs.racetrack_env import RacetrackEnv -from MonteCarlo.agent import FisrtVisitMC -from common.plot import plot_rewards -from common.utils import save_results - -SEQUENCE = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") # 获取当前时间 -SAVED_MODEL_PATH = os.path.split(os.path.abspath(__file__))[0]+"/saved_model/"+SEQUENCE+'/' # 生成保存的模型路径 -if not os.path.exists(os.path.split(os.path.abspath(__file__))[0]+"/saved_model/"): # 检测是否存在文件夹 - os.mkdir(os.path.split(os.path.abspath(__file__))[0]+"/saved_model/") -if not os.path.exists(SAVED_MODEL_PATH): # 检测是否存在文件夹 - os.mkdir(SAVED_MODEL_PATH) -RESULT_PATH = os.path.split(os.path.abspath(__file__))[0]+"/results/"+SEQUENCE+'/' # 存储reward的路径 -if not os.path.exists(os.path.split(os.path.abspath(__file__))[0]+"/results/"): # 检测是否存在文件夹 - os.mkdir(os.path.split(os.path.abspath(__file__))[0]+"/results/") -if not os.path.exists(RESULT_PATH): # 检测是否存在文件夹 - os.mkdir(RESULT_PATH) - -class MCConfig: - def __init__(self): - self.epsilon = 0.15 # epsilon: The probability to select a random action . - self.gamma = 0.9 # gamma: Gamma discount factor. - self.n_episodes = 150 - self.n_steps = 2000 - -def get_mc_args(): - '''set parameters - ''' - parser = argparse.ArgumentParser() - parser.add_argument("--epsilon", default=0.15, type=float) # epsilon: The probability to select a random action . float between 0 and 1. - parser.add_argument("--gamma", default=0.9, type=float) # gamma: Gamma discount factor. - parser.add_argument("--n_episodes", default=150, type=int) - parser.add_argument("--n_steps", default=2000, type=int) - mc_cfg = parser.parse_args() - return mc_cfg - - - -def mc_train(cfg,env,agent): - rewards = [] - ma_rewards = [] # moving average rewards - for i_episode in range(cfg.n_episodes): - one_ep_transition = [] - state = env.reset() - ep_reward = 0 - while True: - # for t in range(cfg.n_steps): - action = agent.choose_action(state) - next_state, reward, done = env.step(action) - ep_reward+=reward - one_ep_transition.append((state, action, reward)) - state = next_state - if done: - break - rewards.append(ep_reward) - if ma_rewards: - ma_rewards.append(ma_rewards[-1]*0.9+ep_reward*0.1) - else: - ma_rewards.append(ep_reward) - agent.update(one_ep_transition) - if (i_episode+1)%10==0: - print("Episode:{}/{}: Reward:{}".format(i_episode+1, mc_cfg.n_episodes,ep_reward)) - return rewards,ma_rewards -if __name__ == "__main__": - mc_cfg = MCConfig() - env = RacetrackEnv() - action_dim=9 - agent = FisrtVisitMC(action_dim,mc_cfg) - rewards,ma_rewards= mc_train(mc_cfg,env,agent) - save_results(rewards,ma_rewards,tag='train',path=RESULT_PATH) - plot_rewards(rewards,ma_rewards,tag="train",algo = "On-Policy First-Visit MC Control",path=RESULT_PATH) - - diff --git a/codes/MonteCarlo/outputs/Racetrack/20210505-165945/models/Q_table b/codes/MonteCarlo/outputs/Racetrack/20210505-165945/models/Q_table new file mode 100644 index 0000000..6205ee5 Binary files /dev/null and b/codes/MonteCarlo/outputs/Racetrack/20210505-165945/models/Q_table differ diff --git a/codes/MonteCarlo/outputs/Racetrack/20210505-165945/results/eval_ma_rewards.npy b/codes/MonteCarlo/outputs/Racetrack/20210505-165945/results/eval_ma_rewards.npy new file mode 100644 index 0000000..5cc42f1 Binary files /dev/null and b/codes/MonteCarlo/outputs/Racetrack/20210505-165945/results/eval_ma_rewards.npy differ diff --git a/codes/MonteCarlo/outputs/Racetrack/20210505-165945/results/eval_rewards.npy b/codes/MonteCarlo/outputs/Racetrack/20210505-165945/results/eval_rewards.npy new file mode 100644 index 0000000..19bb2f1 Binary files /dev/null and b/codes/MonteCarlo/outputs/Racetrack/20210505-165945/results/eval_rewards.npy differ diff --git a/codes/MonteCarlo/outputs/Racetrack/20210505-165945/results/eval_rewards_curve.png b/codes/MonteCarlo/outputs/Racetrack/20210505-165945/results/eval_rewards_curve.png new file mode 100644 index 0000000..0738ac5 Binary files /dev/null and b/codes/MonteCarlo/outputs/Racetrack/20210505-165945/results/eval_rewards_curve.png differ diff --git a/codes/MonteCarlo/outputs/Racetrack/20210505-165945/results/train_ma_rewards.npy b/codes/MonteCarlo/outputs/Racetrack/20210505-165945/results/train_ma_rewards.npy new file mode 100644 index 0000000..f52b398 Binary files /dev/null and b/codes/MonteCarlo/outputs/Racetrack/20210505-165945/results/train_ma_rewards.npy differ diff --git a/codes/MonteCarlo/outputs/Racetrack/20210505-165945/results/train_rewards.npy b/codes/MonteCarlo/outputs/Racetrack/20210505-165945/results/train_rewards.npy new file mode 100644 index 0000000..fe83b3c Binary files /dev/null and b/codes/MonteCarlo/outputs/Racetrack/20210505-165945/results/train_rewards.npy differ diff --git a/codes/MonteCarlo/outputs/Racetrack/20210505-165945/results/train_rewards_curve.png b/codes/MonteCarlo/outputs/Racetrack/20210505-165945/results/train_rewards_curve.png new file mode 100644 index 0000000..b10b7ab Binary files /dev/null and b/codes/MonteCarlo/outputs/Racetrack/20210505-165945/results/train_rewards_curve.png differ diff --git a/codes/MonteCarlo/results/20210317-123623/ma_rewards_train.npy b/codes/MonteCarlo/results/20210317-123623/ma_rewards_train.npy deleted file mode 100644 index 5b5a10a..0000000 Binary files a/codes/MonteCarlo/results/20210317-123623/ma_rewards_train.npy and /dev/null differ diff --git a/codes/MonteCarlo/results/20210317-123623/rewards_curve_train.png b/codes/MonteCarlo/results/20210317-123623/rewards_curve_train.png deleted file mode 100644 index 08498fe..0000000 Binary files a/codes/MonteCarlo/results/20210317-123623/rewards_curve_train.png and /dev/null differ diff --git a/codes/MonteCarlo/results/20210317-123623/rewards_train.npy b/codes/MonteCarlo/results/20210317-123623/rewards_train.npy deleted file mode 100644 index 1c8d034..0000000 Binary files a/codes/MonteCarlo/results/20210317-123623/rewards_train.npy and /dev/null differ diff --git a/codes/MonteCarlo/task0_train.py b/codes/MonteCarlo/task0_train.py new file mode 100644 index 0000000..dae0c95 --- /dev/null +++ b/codes/MonteCarlo/task0_train.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python +# coding=utf-8 +''' +Author: John +Email: johnjim0816@gmail.com +Date: 2021-03-11 14:26:44 +LastEditor: John +LastEditTime: 2021-05-05 17:27:50 +Discription: +Environment: +''' + +import sys,os +curr_path = os.path.dirname(__file__) +parent_path = os.path.dirname(curr_path) +sys.path.append(parent_path) # add current terminal path to sys.path + +import torch +import datetime + +from common.utils import save_results,make_dir +from common.plot import plot_rewards +from MonteCarlo.agent import FisrtVisitMC +from envs.racetrack_env import RacetrackEnv + +curr_time = datetime.datetime.now().strftime( + "%Y%m%d-%H%M%S") # obtain current time + +class MCConfig: + def __init__(self): + self.algo = "MC" # name of algo + self.env = 'Racetrack' + self.result_path = curr_path+"/outputs/" + self.env + \ + '/'+curr_time+'/results/' # path to save results + self.model_path = curr_path+"/outputs/" + self.env + \ + '/'+curr_time+'/models/' # path to save models + # epsilon: The probability to select a random action . + self.epsilon = 0.15 + self.gamma = 0.9 # gamma: Gamma discount factor. + self.train_eps = 200 + self.device = torch.device( + "cuda" if torch.cuda.is_available() else "cpu") # check gpu + +def env_agent_config(cfg,seed=1): + env = RacetrackEnv() + action_dim = 9 + agent = FisrtVisitMC(action_dim, cfg) + return env,agent + +def train(cfg, env, agent): + print('Start to eval !') + print(f'Env:{cfg.env}, Algorithm:{cfg.algo}, Device:{cfg.device}') + rewards = [] + ma_rewards = [] # moving average rewards + for i_ep in range(cfg.train_eps): + state = env.reset() + ep_reward = 0 + one_ep_transition = [] + while True: + action = agent.choose_action(state) + next_state, reward, done = env.step(action) + ep_reward += reward + one_ep_transition.append((state, action, reward)) + state = next_state + if done: + break + rewards.append(ep_reward) + if ma_rewards: + ma_rewards.append(ma_rewards[-1]*0.9+ep_reward*0.1) + else: + ma_rewards.append(ep_reward) + agent.update(one_ep_transition) + if (i_ep+1) % 10 == 0: + print(f"Episode:{i_ep+1}/{cfg.train_eps}: Reward:{ep_reward}") + print('Complete training！') + return rewards, ma_rewards + +def eval(cfg, env, agent): + print('Start to eval !') + print(f'Env:{cfg.env}, Algorithm:{cfg.algo}, Device:{cfg.device}') + rewards = [] + ma_rewards = [] # moving average rewards + for i_ep in range(cfg.train_eps): + state = env.reset() + ep_reward = 0 + while True: + action = agent.choose_action(state) + next_state, reward, done = env.step(action) + ep_reward += reward + state = next_state + if done: + break + rewards.append(ep_reward) + if ma_rewards: + ma_rewards.append(ma_rewards[-1]*0.9+ep_reward*0.1) + else: + ma_rewards.append(ep_reward) + if (i_ep+1) % 10 == 0: + print(f"Episode:{i_ep+1}/{cfg.train_eps}: Reward:{ep_reward}") + return rewards, ma_rewards + +if __name__ == "__main__": + cfg = MCConfig() + + # train + env,agent = env_agent_config(cfg,seed=1) + rewards, ma_rewards = train(cfg, env, agent) + make_dir(cfg.result_path, cfg.model_path) + agent.save(path=cfg.model_path) + save_results(rewards, ma_rewards, tag='train', path=cfg.result_path) + plot_rewards(rewards, ma_rewards, tag="train", + algo=cfg.algo, path=cfg.result_path) + # eval + env,agent = env_agent_config(cfg,seed=10) + agent.load(path=cfg.model_path) + rewards,ma_rewards = eval(cfg,env,agent) + save_results(rewards,ma_rewards,tag='eval',path=cfg.result_path) + plot_rewards(rewards,ma_rewards,tag="eval",env=cfg.env,algo = cfg.algo,path=cfg.result_path) diff --git a/codes/PPO/results/CartPole-v0/20210428-101400/models/ppo_actor.pt b/codes/PPO/results/CartPole-v0/20210428-101400/models/ppo_actor.pt deleted file mode 100644 index 252815d..0000000 Binary files a/codes/PPO/results/CartPole-v0/20210428-101400/models/ppo_actor.pt and /dev/null differ diff --git a/codes/PPO/results/CartPole-v0/20210428-101400/models/ppo_critic.pt b/codes/PPO/results/CartPole-v0/20210428-101400/models/ppo_critic.pt deleted file mode 100644 index a67f3eb..0000000 Binary files a/codes/PPO/results/CartPole-v0/20210428-101400/models/ppo_critic.pt and /dev/null differ diff --git a/codes/PPO/results/CartPole-v0/20210428-101400/results/ma_rewards_train.npy b/codes/PPO/results/CartPole-v0/20210428-101400/results/ma_rewards_train.npy deleted file mode 100644 index 3772867..0000000 Binary files a/codes/PPO/results/CartPole-v0/20210428-101400/results/ma_rewards_train.npy and /dev/null differ diff --git a/codes/PPO/results/CartPole-v0/20210428-101400/results/rewards_curve_train.png b/codes/PPO/results/CartPole-v0/20210428-101400/results/rewards_curve_train.png deleted file mode 100644 index 378779d..0000000 Binary files a/codes/PPO/results/CartPole-v0/20210428-101400/results/rewards_curve_train.png and /dev/null differ diff --git a/codes/PPO/results/CartPole-v0/20210428-101400/results/rewards_train.npy b/codes/PPO/results/CartPole-v0/20210428-101400/results/rewards_train.npy deleted file mode 100644 index af131b9..0000000 Binary files a/codes/PPO/results/CartPole-v0/20210428-101400/results/rewards_train.npy and /dev/null differ diff --git a/codes/PPO/results/CartPole-v0/20210428-101634/models/ppo_actor.pt b/codes/PPO/results/CartPole-v0/20210428-101634/models/ppo_actor.pt deleted file mode 100644 index 516e740..0000000 Binary files a/codes/PPO/results/CartPole-v0/20210428-101634/models/ppo_actor.pt and /dev/null differ diff --git a/codes/PPO/results/CartPole-v0/20210428-101634/models/ppo_critic.pt b/codes/PPO/results/CartPole-v0/20210428-101634/models/ppo_critic.pt deleted file mode 100644 index 489e43d..0000000 Binary files a/codes/PPO/results/CartPole-v0/20210428-101634/models/ppo_critic.pt and /dev/null differ diff --git a/codes/PPO/results/CartPole-v0/20210428-101634/results/ma_rewards_train.npy b/codes/PPO/results/CartPole-v0/20210428-101634/results/ma_rewards_train.npy deleted file mode 100644 index 70dc625..0000000 Binary files a/codes/PPO/results/CartPole-v0/20210428-101634/results/ma_rewards_train.npy and /dev/null differ diff --git a/codes/PPO/results/CartPole-v0/20210428-101634/results/rewards_curve_train.png b/codes/PPO/results/CartPole-v0/20210428-101634/results/rewards_curve_train.png deleted file mode 100644 index 9c31971..0000000 Binary files a/codes/PPO/results/CartPole-v0/20210428-101634/results/rewards_curve_train.png and /dev/null differ diff --git a/codes/PPO/results/CartPole-v0/20210428-101634/results/rewards_train.npy b/codes/PPO/results/CartPole-v0/20210428-101634/results/rewards_train.npy deleted file mode 100644 index 477be61..0000000 Binary files a/codes/PPO/results/CartPole-v0/20210428-101634/results/rewards_train.npy and /dev/null differ diff --git a/codes/PPO/results/CartPole-v0/20210506-004345/models/ppo_actor.pt b/codes/PPO/results/CartPole-v0/20210506-004345/models/ppo_actor.pt new file mode 100644 index 0000000..652ec59 Binary files /dev/null and b/codes/PPO/results/CartPole-v0/20210506-004345/models/ppo_actor.pt differ diff --git a/codes/PPO/results/CartPole-v0/20210506-004345/models/ppo_critic.pt b/codes/PPO/results/CartPole-v0/20210506-004345/models/ppo_critic.pt new file mode 100644 index 0000000..9c71cfb Binary files /dev/null and b/codes/PPO/results/CartPole-v0/20210506-004345/models/ppo_critic.pt differ diff --git a/codes/PPO/results/CartPole-v0/20210506-004345/results/eval_ma_rewards.npy b/codes/PPO/results/CartPole-v0/20210506-004345/results/eval_ma_rewards.npy new file mode 100644 index 0000000..a8a5243 Binary files /dev/null and b/codes/PPO/results/CartPole-v0/20210506-004345/results/eval_ma_rewards.npy differ diff --git a/codes/PPO/results/CartPole-v0/20210506-004345/results/eval_rewards.npy b/codes/PPO/results/CartPole-v0/20210506-004345/results/eval_rewards.npy new file mode 100644 index 0000000..a8a5243 Binary files /dev/null and b/codes/PPO/results/CartPole-v0/20210506-004345/results/eval_rewards.npy differ diff --git a/codes/PPO/results/CartPole-v0/20210506-004345/results/eval_rewards_curve.png b/codes/PPO/results/CartPole-v0/20210506-004345/results/eval_rewards_curve.png new file mode 100644 index 0000000..624437a Binary files /dev/null and b/codes/PPO/results/CartPole-v0/20210506-004345/results/eval_rewards_curve.png differ diff --git a/codes/PPO/results/CartPole-v0/20210506-004345/results/train_ma_rewards.npy b/codes/PPO/results/CartPole-v0/20210506-004345/results/train_ma_rewards.npy new file mode 100644 index 0000000..b232547 Binary files /dev/null and b/codes/PPO/results/CartPole-v0/20210506-004345/results/train_ma_rewards.npy differ diff --git a/codes/PPO/results/CartPole-v0/20210506-004345/results/train_rewards.npy b/codes/PPO/results/CartPole-v0/20210506-004345/results/train_rewards.npy new file mode 100644 index 0000000..d6c6cd5 Binary files /dev/null and b/codes/PPO/results/CartPole-v0/20210506-004345/results/train_rewards.npy differ diff --git a/codes/PPO/results/CartPole-v0/20210506-004345/results/train_rewards_curve.png b/codes/PPO/results/CartPole-v0/20210506-004345/results/train_rewards_curve.png new file mode 100644 index 0000000..67d24f9 Binary files /dev/null and b/codes/PPO/results/CartPole-v0/20210506-004345/results/train_rewards_curve.png differ diff --git a/codes/PPO/results/CartPole-v0/20210506-013522/models/ppo_actor.pt b/codes/PPO/results/CartPole-v0/20210506-013522/models/ppo_actor.pt new file mode 100644 index 0000000..fb5fb41 Binary files /dev/null and b/codes/PPO/results/CartPole-v0/20210506-013522/models/ppo_actor.pt differ diff --git a/codes/PPO/results/CartPole-v0/20210506-013522/models/ppo_critic.pt b/codes/PPO/results/CartPole-v0/20210506-013522/models/ppo_critic.pt new file mode 100644 index 0000000..f9eb037 Binary files /dev/null and b/codes/PPO/results/CartPole-v0/20210506-013522/models/ppo_critic.pt differ diff --git a/codes/PPO/results/CartPole-v0/20210506-013522/results/eval_ma_rewards.npy b/codes/PPO/results/CartPole-v0/20210506-013522/results/eval_ma_rewards.npy new file mode 100644 index 0000000..54f966e Binary files /dev/null and b/codes/PPO/results/CartPole-v0/20210506-013522/results/eval_ma_rewards.npy differ diff --git a/codes/PPO/results/CartPole-v0/20210506-013522/results/eval_rewards.npy b/codes/PPO/results/CartPole-v0/20210506-013522/results/eval_rewards.npy new file mode 100644 index 0000000..a44c265 Binary files /dev/null and b/codes/PPO/results/CartPole-v0/20210506-013522/results/eval_rewards.npy differ diff --git a/codes/PPO/results/CartPole-v0/20210506-013522/results/eval_rewards_curve.png b/codes/PPO/results/CartPole-v0/20210506-013522/results/eval_rewards_curve.png new file mode 100644 index 0000000..18f5f0b Binary files /dev/null and b/codes/PPO/results/CartPole-v0/20210506-013522/results/eval_rewards_curve.png differ diff --git a/codes/PPO/results/CartPole-v0/20210506-013522/results/train_ma_rewards.npy b/codes/PPO/results/CartPole-v0/20210506-013522/results/train_ma_rewards.npy new file mode 100644 index 0000000..8bf7615 Binary files /dev/null and b/codes/PPO/results/CartPole-v0/20210506-013522/results/train_ma_rewards.npy differ diff --git a/codes/PPO/results/CartPole-v0/20210506-013522/results/train_rewards.npy b/codes/PPO/results/CartPole-v0/20210506-013522/results/train_rewards.npy new file mode 100644 index 0000000..cde0ab6 Binary files /dev/null and b/codes/PPO/results/CartPole-v0/20210506-013522/results/train_rewards.npy differ diff --git a/codes/PPO/results/CartPole-v0/20210506-013522/results/train_rewards_curve.png b/codes/PPO/results/CartPole-v0/20210506-013522/results/train_rewards_curve.png new file mode 100644 index 0000000..6c0db9b Binary files /dev/null and b/codes/PPO/results/CartPole-v0/20210506-013522/results/train_rewards_curve.png differ diff --git a/codes/PPO/task0_train.ipynb b/codes/PPO/task0_train.ipynb new file mode 100644 index 0000000..9c74585 --- /dev/null +++ b/codes/PPO/task0_train.ipynb @@ -0,0 +1,257 @@ +{ + "metadata": { + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.10" + }, + "orig_nbformat": 2, + "kernelspec": { + "name": "python3710jvsc74a57bd0366e1054dee9d4501b0eb8f87335afd3c67fc62db6ee611bbc7f8f5a1fefe232", + "display_name": "Python 3.7.10 64-bit ('py37': conda)" + }, + "metadata": { + "interpreter": { + "hash": "366e1054dee9d4501b0eb8f87335afd3c67fc62db6ee611bbc7f8f5a1fefe232" + } + } + }, + "nbformat": 4, + "nbformat_minor": 2, + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "from pathlib import Path\n", + "curr_path = str(Path().absolute())\n", + "parent_path = str(Path().absolute().parent)\n", + "sys.path.append(parent_path) # add current terminal path to sys.path" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "import gym\n", + "import torch\n", + "import datetime\n", + "from PPO.agent import PPO\n", + "from common.plot import plot_rewards\n", + "from common.utils import save_results,make_dir\n", + "\n", + "curr_time = datetime.datetime.now().strftime(\"%Y%m%d-%H%M%S\") # obtain current time" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "class PPOConfig:\n", + " def __init__(self) -> None:\n", + " self.env = 'CartPole-v0'\n", + " self.algo = 'PPO'\n", + " self.result_path = curr_path+\"/results/\" +self.env+'/'+curr_time+'/results/' # path to save results\n", + " self.model_path = curr_path+\"/results/\" +self.env+'/'+curr_time+'/models/' # path to save models\n", + " self.train_eps = 200 # max training episodes\n", + " self.eval_eps = 50\n", + " self.batch_size = 5\n", + " self.gamma=0.99\n", + " self.n_epochs = 4\n", + " self.actor_lr = 0.0003\n", + " self.critic_lr = 0.0003\n", + " self.gae_lambda=0.95\n", + " self.policy_clip=0.2\n", + " self.hidden_dim = 256\n", + " self.update_fre = 20 # frequency of agent update\n", + " self.device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\") # check gpu" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "def env_agent_config(cfg,seed=1):\n", + " env = gym.make(cfg.env) \n", + " env.seed(seed)\n", + " state_dim = env.observation_space.shape[0]\n", + " action_dim = env.action_space.n\n", + " agent = PPO(state_dim,action_dim,cfg)\n", + " return env,agent" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "def train(cfg,env,agent):\n", + " print('Start to train !')\n", + " print(f'Env:{cfg.env}, Algorithm:{cfg.algo}, Device:{cfg.device}')\n", + " rewards= []\n", + " ma_rewards = [] # moving average rewards\n", + " running_steps = 0\n", + " for i_ep in range(cfg.train_eps):\n", + " state = env.reset()\n", + " done = False\n", + " ep_reward = 0\n", + " while not done:\n", + " action, prob, val = agent.choose_action(state)\n", + " state_, reward, done, _ = env.step(action)\n", + " running_steps += 1\n", + " ep_reward += reward\n", + " agent.memory.push(state, action, prob, val, reward, done)\n", + " if running_steps % cfg.update_fre == 0:\n", + " agent.update()\n", + " state = state_\n", + " rewards.append(ep_reward)\n", + " if ma_rewards:\n", + " ma_rewards.append(\n", + " 0.9*ma_rewards[-1]+0.1*ep_reward)\n", + " else:\n", + " ma_rewards.append(ep_reward)\n", + " if (i_ep+1)%10==0:\n", + " print(f\"Episode:{i_ep+1}/{cfg.train_eps}, Reward:{ep_reward:.3f}\")\n", + " print('Complete training！')\n", + " return rewards,ma_rewards" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "def eval(cfg,env,agent):\n", + " print('Start to eval !')\n", + " print(f'Env:{cfg.env}, Algorithm:{cfg.algo}, Device:{cfg.device}')\n", + " rewards= []\n", + " ma_rewards = [] # moving average rewards\n", + " for i_ep in range(cfg.eval_eps):\n", + " state = env.reset()\n", + " done = False\n", + " ep_reward = 0\n", + " while not done:\n", + " action, prob, val = agent.choose_action(state)\n", + " state_, reward, done, _ = env.step(action)\n", + " ep_reward += reward\n", + " state = state_\n", + " rewards.append(ep_reward)\n", + " if ma_rewards:\n", + " ma_rewards.append(\n", + " 0.9*ma_rewards[-1]+0.1*ep_reward)\n", + " else:\n", + " ma_rewards.append(ep_reward)\n", + " if (i_ep+1)%10==0:\n", + " print(f\"Episode:{i_ep+1}/{cfg.train_eps}, Reward:{ep_reward:.3f}\")\n", + " print('Complete evaling！')\n", + " return rewards,ma_rewards" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Start to train !\n", + "Env:CartPole-v0, Algorithm:PPO, Device:cuda\n", + "Episode:10/200, Reward:15.000\n", + "Episode:20/200, Reward:9.000\n", + "Episode:30/200, Reward:20.000\n", + "Episode:40/200, Reward:17.000\n", + "Episode:50/200, Reward:64.000\n", + "Episode:60/200, Reward:90.000\n", + "Episode:70/200, Reward:23.000\n", + "Episode:80/200, Reward:138.000\n", + "Episode:90/200, Reward:150.000\n", + "Episode:100/200, Reward:200.000\n", + "Episode:110/200, Reward:200.000\n", + "Episode:120/200, Reward:200.000\n", + "Episode:130/200, Reward:200.000\n", + "Episode:140/200, Reward:200.000\n", + "Episode:150/200, Reward:200.000\n", + "Episode:160/200, Reward:200.000\n", + "Episode:170/200, Reward:200.000\n", + "Episode:180/200, Reward:200.000\n", + "Episode:190/200, Reward:200.000\n", + "Episode:200/200, Reward:200.000\n", + "Complete training！\n", + "results saved!\n" + ] + }, + { + "output_type": "display_data", + "data": { + "text/plain": "
", + "image/svg+xml": "\n\n\n \n \n \n \n 2021-05-06T01:36:50.188726\n image/svg+xml\n \n \n Matplotlib v3.4.1, https://matplotlib.org/\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n\n", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXsAAAEcCAYAAAAmzxTpAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Z1A+gAAAACXBIWXMAAAsTAAALEwEAmpwYAACCHklEQVR4nO2deZwU1bm/n6rqdfaVYVgERUEUhYEBUVQU911jVC5xTUxiFpcYTExiNEFNgjEm0UuuMSbm+tOrMXELbmjcd0EBBRUB2WeYfe/pparO74/qqu6e6Znp2aeZ83w+MN3VVeec2t5663ve8x5FCCGQSCQSyT6NOtwNkEgkEsngI429RCKRjAKksZdIJJJRgDT2EolEMgqQxl4ikUhGAdLYSyQSyShAGvtRwD333MPSpUuHpe4zzjiD999/f1jqHmkEg0Guuuoq5syZwzXXXDPczeHDDz/k5JNPpqysjP/85z/D3ZxBZ9q0aezYsWO4mzFsSGMvGVSeffZZjjjiiOFuxojghRdeoLa2lvfff5+777670+/33HMPhx56KGVlZZSXl7N48WLWrl0LwBNPPMH06dMpKytj9uzZnHPOObz66qvOts3Nzdxyyy0sWLCAmTNnctZZZ/H444932567776br33ta6xdu5YTTzxxQPbx448/5pvf/Cbl5eXMmzePr371qz22ozsuueQS/vnPfyYsmzZtGrNmzaKsrIxjjjmGX//61xiG0d+m95nPPvuMr3zlK8ycOZOvfOUrfPbZZ8PWlu6Qxn4Y0HV9uJswIOwL+zGU+1BRUcHkyZNxuVxdrnPaaaexdu1a3n33XWbPns3VV1+NPe5x1qxZrF27ljVr1vDVr36V6667jqamJsLhMJdffjkVFRU8+uijrFmzhhtuuIHf/e53PPDAA92256CDDurTviQ7bmvXruWyyy5j7ty5vPjii7z//vv84he/4I033uh1+UIITNPs8venn36atWvX8ve//51nnnmGxx57rNd1DAThcJjvfve7nH322axevZpzzz2X7373u4TD4WFpT3dIY9+B++67jxNPPJGysjJOP/10XnrpJcA6qeXl5XzxxRfOuvX19Rx++OHU1dUB8Oqrr3LOOec4Xtnnn3/urLto0SLuu+8+zjrrLGbNmoWu613WBWAYBr/5zW844ogjWLRoEQ899BDTpk1zbrKWlhZ++tOfcvTRR3PMMcfw+9//PmXvZt26dSxevJjy8nLOPvvsBJnl8ccf57TTTqOsrIwTTjiBRx991Pnt/fff59hjj+W+++5jwYIF/OQnP+Gee+7h2muv5Uc/+hFlZWWcccYZfPLJJwn7/c477wD0uO7GjRs599xzKSsr45prruG6667j97//fZf78dhjjzltPf3009m4cSPQ+XX9xhtvdMpJtg+nnXZagpes6zrz5893yuvueHVk69atXHLJJZSXl3PGGWfw8ssvA5YX/ac//Ynnn3+esrKyTt5qR9xuN+eddx41NTU0NDQk/KaqKueffz7BYJCdO3fy9NNPU1lZyR//+EcmTpyI2+3m2GOP5aabbuLuu++mtbW1U/knnngiu3bt4qqrrqKsrIxwOExVVRVXXXUV8+bN46STTkowoPfccw/XXHMNS5cuZfbs2Tz55JOdyrzjjjs499xz+da3vkVBQQGKojBjxgz++Mc/AtDU1MS3v/1t5s+fz9y5c/n2t7/N3r17ne0vueQSfv/737N48WJmzpzJDTfcwJo1a1i2bBllZWUsW7asU51Tpkxhzpw5bN68GbCuiZNOOol58+Zx1VVXUVVVlfT4hsNhli9fznHHHcdRRx3FzTffTDAYTLpud9fHBx98gK7rXHbZZXg8Hi699FKEELz33ntJyxpWhCSB5557Tuzdu1cYhiGeffZZMXPmTFFVVSWEEOLGG28Ud911l7PuQw89JL7+9a8LIYTYuHGjmD9/vli3bp3QdV088cQT4vjjjxehUEgIIcTxxx8vzj77bFFRUSHa29t7rOv//u//xGmnnSYqKytFY2OjuOyyy8TUqVNFJBIRQgjx3e9+V/z85z8XbW1tora2Vpx//vnikUceSbpPd999t/jhD38ohBBi7969Yt68eeK1114ThmGIt956S8ybN0/U1dUJIYR49dVXxY4dO4RpmuL9998Xhx9+uNiwYYMQQoj33ntPTJ8+Xdxxxx0iFAqJ9vZ2cffdd4sZM2aI1157Tei6Lu68805xwQUXOHUff/zx4u2333ba0dW6oVBIHHfcceLvf/+7CIfDYtWqVeLQQw9NON4dz9PRRx8t1q9fL0zTFNu3bxe7d+8WQggxdepUsX37dmfdH//4x045yfbhnnvuEddff72z/quvvipOPfXUlI5XPOFwWJx44onif/7nf0QoFBLvvPOOmDVrlti6dWun89DTeQqFQuI3v/mNWLhwoRBCiMcff1wsXrxYCCFEJBIRf//738WsWbNEc3OzuO6668SPfvSjTuVFIhExffp08cYbbyStL/7cCCHEkiVLxC233CKCwaD49NNPxRFHHCHeeecdp22HHHKIeOmll4RhGM41bBMIBMTBBx8s3n333S73r76+XrzwwgsiEAiIlpYWcfXVV4vvfOc7zu8XX3yxWLhwofjiiy9EJBIR4XBYXHzxxeKxxx5LKCf+/G7evFkcddRR4rHHHhPvvPOOmDdvntiwYYMIhUJi2bJlYsmSJUm3u/3228W3v/1t0dDQIFpaWsS3v/1tceeddyZtd3fXxwMPPCC+8Y1vJKz/rW99S/z1r3/t8jgMF9Kz78Bpp51GSUkJqqpy+umnM2nSJD7++GMAzjrrLJ599lln3ZUrV3LWWWcB8I9//IOLLrqImTNnomka5513Hm63m3Xr1jnrX3LJJZSWluLz+Xqs6/nnn+fSSy9l7Nix5Obm8q1vfcspp7a2ltdff52f/vSnZGRkUFhYyOWXX57Qtq54+umnOfbYY1m4cCGqqrJgwQJmzJjB66+/DsBxxx3Hfvvth6IozJs3jwULFrBmzRpne1VVueaaa/B4PM5+zJkzh4ULF6JpGuecc07CG01Hulp3/fr16LrOpZdeitvt5uSTT+awww7rspx//etfXHnllRx++OEoisKkSZMYP358j/ufbB/OOussXnnlFdrb2wHrvJ5xxhkpHa941q9fTyAQ4Fvf+hYej4cjjzyS448/PqXzYvPCCy9QXl7OwoUL2bhxI//93/+dUH55eTkLFizg2WefZcWKFWRnZ9PQ0EBxcXGnslwuF/n5+Z3eDJJRWVnJRx99xNKlS/F6vUyfPp0LLriAp59+2lln1qxZnHjiiaiq6px7m+bmZkzTTNoOm/z8fE455RT8fj9ZWVl85zvfYfXq1QnrnHfeeRx00EG4XC7cbneXZZ133nnMnTuXq666iq9+9aucf/75rFy5kvPPP59DDz0Uj8fD9ddfz7p169i9e3fCtkIIHnvsMX7605+Sl5dHVlYW3/72t7s8T91dH21tbWRnZyesn5WVRVtbW5dtHy66Fg9HKU899RQPPPAAe/bsASAQCDg3yxFHHEEwGGT9+vUUFhby+eefOx1bFRUVPPXUUzz00ENOWZFIhOrqaud7aWlpynVVV1cnrD927Fjnc0VFBbquc/TRRzvLTNPsVH4yKioqeOGFFzq9ltqdqK+//jorVqxg+/btmKZJMBhk6tSpzrr5+fl4vd6EMouKipzPPp+PUCiErutJtemu1q2urqakpARFUZzfu9ufyspK9ttvvx73Nxkd92HSpElMmTKFV199leOPP55XXnmFp556Cuj5eMVTXV3N2LFjUdWYDzVu3LgupYRknHrqqdx5551Jf5s5cyaPPPJI0v2pqanptFzXdRoaGsjPz++x3urqanJzc8nKykpo+4YNG5zv8ddgR3JyclBVlZqaGqZMmZJ0nfb2dn7961/z5ptv0tTUBFjG0jAMNE0Duj/n8Tz55JNMmjSp0z4ceuihzvfMzEzy8vKoqqpiwoQJzvL6+nra29v5yle+4iwTcX0EV155JR9++CEAv/zlLzn77LO7vD4yMzM7yWRtbW1kZmamtB9DiTT2cezZs4ebbrqJv//975SVlTnep42maZx66qk888wzFBUVcdxxxzk3R2lpKVdddRXf+c53uiw/3pD1VFdxcXGCnhn/eezYsXg8Ht57771uO/uSUVpayjnnnMNtt93W6bdwOMw111zD8uXLOeGEE3C73Xz3u991Ogg77sNAUlxcTFVVFUIIp47KykomTpzY5X7s3Lkz6W9+v9/xwgBqamooKSlxvifbhzPPPJNnnnkG0zQ58MADHUPS3fHqyJgxY9i7dy+maToGv7KyksmTJ/e4bX846qijuOuuuwgEAmRkZDjLX3zxRTweD7NmzeqxjDFjxtDU1ERra6tzTVdWVvZ43Gz8fj+zZs3ixRdfZP78+UnX+dvf/sa2bdt47LHHKC4u5rPPPuPcc88dsOtrzJgxjuMElvPU2NiYsA9gPRx9Ph/PPvtsp98A7r///k7Luro+DjzwQP72t78lXLebNm1iyZIlfd6PwULKOHG0t7ejKAoFBQWA1Vlpd/zYnHXWWTz//POsXLmSM88801l+wQUX8Oijj7J+/XqEEAQCAV577bWknWOp1HXaaafx4IMPUlVVRXNzM3/5y1+c38aMGcOCBQv4zW9+Q2trK6ZpsnPnTj744IMe9/Hss8/m1Vdf5c0338QwDEKhEO+//z579+4lHA4TDocpKCjA5XLx+uuv8/bbb6d+APvBrFmz0DSNhx56CF3X+c9//pPQeduRr371q/ztb39jw4YNCCHYsWOHc6MffPDBPPPMMxiGwRtvvNFJKkjG6aefzttvv80jjzyScF67O14dOfzww/H5fNx///1EIhHef/99XnnlFU4//fQ+HJHUOeeccxg7dizXXnstu3fvJhKJ8Oabb3Lbbbfx/e9/v5PMkIzS0lLKysq46667CIVCfP755/zrX//i7LPPTrkdN9xwA08++ST333+/84b6+eef84Mf/ACwPF6v10tOTg6NjY0JElVXFBUVsWvXrpTqP/PMM3niiSf47LPPCIfD3HXXXRx++OEJXj1YMt4FF1zAr371Kye4oqqqijfffLPLsru6PubNm4emaTz44IOEw2Hnzb6rB95wIo19HAceeCBf//rXWbx4MUcddRRffPEFs2fPTlhn5syZ+P1+qqurOfbYY53lhx12GLfeeivLli1j7ty5nHzyyTzxxBN9ruvCCy9kwYIFnH322Zx77rksXLgQl8vlvO7ecccdRCIRTj/9dObOncs111yT9FW+I6WlpfzpT3/iz3/+M0ceeSQLFy7kr3/9K6ZpkpWVxU033cR1113H3LlzeeaZZ1i0aFFvD2Of8Hg83HPPPfzrX/9i7ty5/Pvf/+a4447D4/EkXf+0007jqquu4oc//CGzZ8/me9/7niMN/OxnP+PVV1+lvLyclStXphRDPmbMGCe0Md44d3e8ku3DvffeyxtvvMH8+fP55S9/yR133NGlrDFQeDweHnjgAUpLS7nwwguZM2cOv/nNb/jBD37AlVdemXI5d911F3v27OGYY47h+9//PldffTVHHXVUytvPnj2b//3f/+W9997jxBNPZN68efz85z9n4cKFAFx22WWEQiHmz5/PRRddxDHHHNNjmZdeeimrVq1i7ty5Pb5dHXXUUVx77bVcffXVHH300ezatavLaK4bbriBSZMmceGFFzJ79mwuv/xytm3b1mXZXV0fHo+HFStW8PTTT1NeXs7jjz/OihUrurxuhxNFCDl5STrw+uuv84tf/CJBO97XueCCC1i8eDHnn3/+cDdFIkl7pGc/QgkGg7z++uvouk5VVRUrVqwYsFGOI5UPPviAmpoadF3nySefZNOmTSl5fxKJpGdkB+0IRQjB3XffzXXXXYfP5+O4447j2muvHe5mDSrbtm3juuuuo729nQkTJnD33XczZsyY4W6WRLJPIGUciUQiGQVIGUcikUhGAdLYSyQSyShAGnuJRCIZBYzoDtqGhjZMs/ddCoWFWdTVJR/MNJzIdvWekdo22a7eMVLbBSO3bX1pl6oq5OcnT9Uwoo29aYo+GXt725GIbFfvGaltk+3qHSO1XTBy2zaQ7ZIyjkQikYwCpLGXSCSSUcCIlnE6IoSgoaGGcDgIdP16U12tdjul2XCxb7dLwePxkZ9fPGiZMSUSSd/p0dg3NDTwox/9iJ07d+LxeJg0aRLLli2joKCAdevWcfPNNxMKhRg/fjy//e1vKSwsBOj2t77S2tqEoiiUlExAUbp+KXG5VHR95BnVfbldQpg0NtbS2tpEdnbewDRMIpEMGD3KOIqicOWVV7Jq1SpWrlzJxIkTufPOOzFNkxtuuIGbb76ZVatWUV5e7ky60N1v/aG9vZXs7LxuDb1keFAUlezsfNrbR15Ug0QiScHY5+XlJczKM2vWLCoqKtiwYQNer5fy8nIAFi9ezAsvvADQ7W/9wTQNNC2tlKdRhaa5MM3UJj0fDIQQmEIkTIZhRpf1Zt2Oy+PXd/6Zotv1O5aXyr+u2tKresze1Znqvy6PQ6r/BqldA9L2XratP8eiq+w0qazTX3plOU3T5JFHHmHRokVUVlYybtw457eCggJM06SxsbHb3/Ly8vrVYKkHjxyEEFTUtlGQ48PvdfXp3NQ3B/nVQx/y4yWzKc7zd7vu259U8sb6Cn5y8Rw++qKGf7+1jZuvmIsarfd3/1jHp9sbyPS5+M1VR1LfHOK2B9cQ0U1OmD2Br51sTa/Y1Bbmp/e9R3tI55DJ+SxdXMYz72zniTe+dOqaOCaLX359nvN92d/XsKOqpcu2+Twat115BAU51tysj768mRdXpzbphk1Rro9ff3s+G7c1cPe/Pk76kPJ5NG7/5nzys61pFf/6zKe8vaHzRCoDicetcsvlc8n0uflJ9LiNHARuDHxKBK8SSfhrfdYZf2gZp518BH9//nPe/Liy3zVecvJUjp89gbseW8/GbfW92vbwKYVcd8FMXv5wN+u31HL9RbN4c30Ff3/+c6cXckJxFsu+Ma/bcvpCr4z9rbfeSkZGBhdffDEvvfTSgDemI4WFWQnfq6tVXK7UJJxU1xtqBqtdy5bdEp0kenGftu9LuwxTENFNdFM426uqSnFxzzMj2extDlHfHCJo0OV29vK61h18WdFMcXE29Wsr2FndSm5uBj6vdRnvrQ/g0hTagjour5uwCBHRTVyaSk1z0Cmnob2B9pDOxJIsPt3egOJ28d6nVUwuzeGow8exYWstH2+ppbAwC1VVaA/p7KhqYfbBYzh4UkGn9lXXB/jP6p2EUZw6qhrbKc73c9K8SZ3WT8bW3Y28v3EvXr+X2pYQphAsOXkaxD1AK2pbee3D3eDSKC7OZntlM29v2MsRh45lyoS8lI95b6hpCPDSBzvRFRXF7aI9pHPsrPFMKEn9HPcJIdCMIN5wA95QI55QA55IM269DVekDXek1fqst6GZkW6Lqtu9jeLiE6lsaGd8cSYLZyef6jIexYzgCTfjCTfhCTfhDTfhiTRRtWcvkVqF4uLpVNUHOGB8LvNnpDZv7oefV7GtsoXi4mx21rSxrTJ6LbdF0DSVC0+0nJEJxVnOddSbe6knUjb2y5cvZ8eOHdx7772oqkppaSkVFRXO7/X19aiqSl5eXre/9Ya6utaEQQWmaabUkTgcHaFdTbAdz0C1K1ldIvo62pfy+9ou3bC2MYzYeTFNk5qarj3gjjQ0BACoa2hLul1xcbazvLUthGEKqqqbaW6x5pitrGomy++22qObeN0auqFTW9dGY6NVts+j0R6MOOXU1Fr9CkcfVsojVZt5dNVnVNS28bWTpnJC2Tgi4Qgfb6llT2UjPo+LPTXW+nOnFnPEISWd2rWtspn/rN7J3qoWxuZYHndbIEJxro8Ty2JvuN2R53fx/sa9bNvVQGVNKxleFyfOHp+wzoZtdbz24W7q6tqo8bt48JmN+DwaXzvxIOcYxLerP4hwO0bdTqrbPifDv4nGmmnoIcvwzDqwkFkHFvVQQiLdtcsMNGHW7cRs2IPZsAejYQ9mQyVE2hNX1FwovhwUfzZKdj6KfxKKL9v65/GhuP0Q/au4fSgePy8/+TTzI++x9/ONBNojjCvMSDgnwtTJNeqo3fo5ZmMlZkMFZmMloqWWThF/3kyKXAF21a2npuZ4dMNkQlFmyue4rS3Eph0N7K1qoqa+jbBu3SvNrUF8Hi2hnJqalj6dS1VVOjnJNikZ+7vuuosNGzZw3333OdNtzZgxg2AwyJo1aygvL+fRRx/l1FNP7fG3fYmjjy7niiu+ybvvvs0RRxzJkiWXcM89v2fr1s2Ew2HKysq5+uofsGfPLn760x/x6KP/Qtd1zjjjBC677BssWXIpL7/8Em+++Rq/+MXtPPLIQ7z88osYho7H42Xp0hs56KBpSes699zzue22W6irq2Xs2FJngmuAp59+gsce+z/cbg9CmCxb9hsmTZo8aMehPxKjLVUYRs+F2A9+6+FifbYfOFZZoKlKtE3CaZdLUxKchkh0m0kl2RTl+vjPmt0AzJxiRYv53NbUj6Gwgc/joq45CEBhri9pu/zRN4v2cEzeiOgmmb7UX5xzMq37qrktTEsgTHZm52nttKiXb5gm1Y3trPm8mtOPnOQY+r4ihEA0V6FXfI5R+TlG9TZEcxUA2cAiP+yt+wIz25o2U+2HlCqEsAz63s0YVZsxqrYgmqud3xV/Dmr+eNwHHYmaMwYlqxA1uwglq9Ay6r2s+wvfTMojHxDZ9CbhyP5kaWEiX67GqN6KWbUVo3Y7rUb0zUBzoeaWohXvj3rQUag5xSiZBaiZBSiZ+ShuL5vv/QFeIxDdF1B70ZycDOs8tQYiNAci6LqJEJaD5tIGX57u8WrcvHkzf/7zn5k8eTKLF1sSwYQJE1ixYgV33HEHt9xyS0J4JViv8l39NpC8/UklbyXR4BSlfwYI4OjDS1lwWM+vZ16vl/vvfxCA3/zmVmbNms2NN/4c0zT55S9v4tln/83ZZ59HINBGbW0Nu3fvYf/9p7BmzWqWLLmUDz/8gPLyuQCceuoZ/Nd/XQzA6tXv89vf/pr77vt70rp+9rMbmDmzjK9//Vvs2bObyy9fwhFHHAnAn/70Rx5++HGKiooIh8ODF9svOn3ofRG2sU+hjba91g3hGPn4NxIhBJrLMtR2pxuAS1PR44y9vY3bpTJzShEvf7Sb8UWZFEX7DLyeqLGPWJ3NtU2WsS/qythH1w/Gadlh3cAdfWikQnbUEDQHwjS3hR3DEI8atSymgPqmIAI4ZFJ+ynXEI0wdo/IL9O0fou9Yh2i1Jt5W/LloJQeiTl2AVjSZynAGea/chhaoxYgeQ7WXip8wIrRtXkPw43fRd61HtFmTkSu+bLSxB6FNPx61eH/UgvGovoGVh0xPJluU/Tn4i7e4TFnPxL01BPcKy7AXTcZ9yCLyDzyUNu9YlKwilB52LiB85BptVtlC9Orhk51hPcCb2qxzLIhKoYaJewhk5x6N/UEHHcSmTZuS/jZ79mxWrlzZ69/2JU47LTbT/FtvvcFnn23k0UcfBqypBceMsV77Z88uZ/XqD9izZw/nnPMVHn74QSKRCGvWfMDFF18OwKZNn/H//t8DNDc3oaoqu3bt7LKujz76kOuuuwGA8eMnOA8Mq6653H77LSxYcAxHHnk048dPGJR9t81nfx6s9rZGCjlA7IeWbpiO8Y506dnH3gTcLjXhzSFiG3tN5fADC3n5o90cPiU2BsTrtm6LYDhm7F2a4njfHfE5nn0sEimim7i11G9gu+yWtjDNgQilBRkJvwsjQsbetZzrX4OIHILpstbXelEHgFG3k8imN9E3v4sItYLmxjVhBtqsM3CNm46SOzbBgCnVrbSZHtztdUSix1NLwcAJITCrtxL54m0iX35Aa6gN3D5cE2bgmng4Wuk0lJwxgx5w4XVrfGAeysGR7SjCZHP+AmYdexxq0WSUaGRfVnE27SnKJW34KDYaAesa603z7XPc2Bqmrd16m4joZtSzHwHGfiSz4LDk3vdQavZ+f/xNKfjVr+5MalznzJnLmjWWsb/55ltZt+4j/vOfVQgB48aNJxKJ8POf/5j//u+/MG3awdTW1nDuuad1U1fX/OpXv+Wzzzby4YdruOaaq1i69CcceeSC/uxmF4i4//tYQq9kHOtvvGcf0eONvXCMfXyYnNuloseVb2/rcqlMn5TPqfP24/g4fdzXwbOvawpSmOPrUr7wuFRURUmIUumtt5bhdaGpCk1Rz37axDwARLCV8KcvE9n4MnntzRzvh8r6LwkVWfKemoKOIEwTffsawuufx6zZBqoL1+TZuA48AteEGSgub5fbKqpCnZlNQXud82bVXZ3C1NG3fkD44+cx63aB5sG1/xyKyk+gNXMSitY/yam3eNwqn0fGkPnd+7jrt29w+qH7MadkSp/LC+DHa1j9kaKXnr1t7Ctq25x7JmKYvXYM+kpaG/uRxoIFx/LQQ//L0qU3omkajY2NBAJtjBs3njlz5vLnP/83ubn5jBlTQnn5PP785xXMnWuNYQiHQxiG4bwJPPHEP7uta86ccp599t9cfvmVVFTsYc2a1ZSXz4tOUL6XQw6ZwSGHzKCiYjebN28aFGMf8+z7bu5tA56KjGPEafaGLePEGXFhCsdDMuM0e7emEgrHIjbiPXuXpnLhogMT6vHGafZgefZd6fVghQP7vRrBUJxnHzHx9MLYK4r15tDYYnl9+Z4IwXf+j8jnr4EeRpt4OI1Fs8he+yCuQDXtphW50Z1+LkwTffPbhNY9i2jai5Jbgveor+E+8EgUX/JOvI6oCtQaWZQE6503pWQGTpgGkc9fJ7z2GURbPWreOLzHXI57yhEoHj8Zxdm0DUDHcW/xuDTCERNTKJhC4HGlLq0lI4AfjwghDB2zl559dqSWya4a9tSOxUsYr6Kj6yb6SJFxJKlz7bU/5E9/upvLL/8vFEXB7fZwzTU/ZNy48YwZU0JGRiaHHz4LsDz9qqq9zJ5tDTzLzMziG9/4Nt/85qXk5ORy/PEn9FDXUm677Rb+859VlJaOo6xsDmBJHbff/gtaW1tQFJWSkhKuuur7g7PDosPfvhThaPY9F2Kvq5vCMfJddtCaMRnHpakJ5cd79smwNXtbxqlrDjLrwO5Tffg8rsQO2l7cwCISIvzR03zbs4b3auax0LeXBV8+RsQM4TrwKDwzT0MrmEDt3maCHz2Cu63G2TetCy9b3/sFobcfwqzbiVo4Ce+J38U1ubxHTbojWtSz94R2YRp6pzqFEBi71hN67zHMxgq0sVPxHHMZ2sTDRsRId7dbJaybhCPWOe/NAzgZAayHvgi2IIRIqbNaBFsJffBPzE1v8IMcwdaKzzg9rxKBQiRyLBFDSBlnpPPWW2sSvmdkZLJ06U+6XP+xx5505KWCgkLefHN1wu9f+9plfO1rlznfL7nkii7rKi4ewx//+D9J6/nTn+5PbQf6yQDYekcaSEXGsQ22bpgxGcfo0EEb59k7HbSuRGMf79knI76DNhwxaG4LU5jTtWcP4PdqjowjhDX+IBVjr+9YR/Dt/4dorSNXyeA8/RnIgLacqeSccBlaQUxeUlWVaiOHwkBNXGdporExAi20v3Iv+pb3UDIL8J3wXVwHzO2zNq4qCrVGNoowUdsbE+o0A00E33gAY+c6lNwSfCdfg2tS2Yga+Oh1aeiGSTD6IPb0otM8GQGsTnzL2Hfv2Qsh0LetJvT2Q4hgG+5DT+SV9dUcpX5MtZlDqauJQMWnRHQXmT4XZmsdwbceRDRVkXHhr/vVzmRIYy/pN/3roE3ds48P03QMf7xmbwo0LRZ6acbJOMlCL92u5HeqE3oZMZywy6Lc7kf3+rwu501AN2LRPl0h9BDBNx9E3/w2av54fGf/lH+uDmJueYc6I4vzTjuTsQV5CduoqkKNkc3YQI1zLOKNvV7xGbtf+wtGoAnP7LPxzDwDxd21Hp8KqqpQa1oRMlqg1lqmKES2fUjozb8jIu14j7gI94yTnA7PkYRt3NuCtrHvnwfdrkSNfXtzt5q9CLYSfOMB9O0fohZNxn/6DWiFE1m9aTUrqw5BR+XWvH/i2vUhujGXKcZW2v75ZxAG3iOXDMoDc+SdHUn6IOw//Qi9jP5NKfQyiWdv/xXCaoXL9jrNWI4Rl6Yk9ey7enV2PPuwQV1T9zH2Nn6Pi5ZAOKF8dxf6sNm4l/aX/huzYQ+e2efgKTsLRXORlbmFF0KWFp+dJPRSUxWqzRzcoR0I3apLVSxtPrzmCcLrnsVdWIr35GvRilIbudsTqqpQZ1r6vqu9DhU/mR//g+D2tywjdvw30fLH91DK8GE/cFuj0S/91ewdYx9swRTJ+0yMup20v3g3oq0R7xEX4j7sFBTVqjc7080OrHP7SWQ/5u39mIV6O3NbN6AWH4D/hKtQc8b0q41dIY29pM+ImLXvM7GBUqkPqtLjRuxGHGNvrROTcTqEXsY9TCKGFerWlffkcakoWJp9bXP3MfY2fq9GdaPl2Yf1rvVhfcda2l/5M4rqwn/6D3FNmOH8lp0ZM/DJwjwVVaHayEVBoAWsuHjNjBB86X70HWtxTzuG8WdfRV1T9+kDeoOqKDSaGZiKhqd5D1dl78C7vRLPzNPxzP0KijqyTYjtyTvGvt+evRURJ9pbEMLdScaJbHmP4Ot/Q/FlknH2T9HGHJDwe05G7Lx+FJrMfO8W5rKBLzJmM/vs7w7q29HIPlOStGAgZBw9BRnHsNc1hLO+PZLWljUSQy+t7Tp10OqiSwkHrGgTj0cjFDFobAmhALlZyWPsbXwelzOoKqInl3Eim98h+Nr9qEWT8J/0fdSsxE5f2xBoqkKGt/OtqSkK1UYOAJ5ANRmKjveNu9HrtuE96mt4ZpyE6vEBA2jsVQWBSsiTR/7e98h2qQTnXEz2nBMHrI7BxI6sao2+dfXXsw/ixUSJevb5CQ5DeP3zhN7/B9rYqfhO/B5qRm6n7eON/WZ9LM3jj+T57X5chXOZM8gy2PB3l0vSFjEQMo4zqKpnGUf0IONAx3QJcR208YOqjJ7jmn1ujWDYoLU9QobPhdZDFIvfqznROOEkxj786asEX/0LWuk0Ms74USdDDzFvPifTk/Stw9bsAbIbv+C6nBdQGnbiO/G7eGac1G37+ootU7T5SzE0H//TciLmAUcNSl2DQUcZx91Pz15VVUKKP6rZx9IlhNY9S+j9f+A6YB7+M36U1NBD3DnOcGOiUnXQV1gfmYxb699DKBWkZy/pN0OVGycWjSOc9W0Zx35WOHH2Zqzsjh20um72mOXT69EIRwyCYUFmCrln/B4X4YiJEZeszzY04Q0vEXrnYbT9ZuI/8XsoruRvCbbXl0yvB8vYh/AQdmdRVLOGNsWDWHQd7gNmJF1/ILCfcZvHn0WW38OWF7f1KzfOUGN30La2Ww9ibz89e0WBoJJBVntL9LtCaO0zhFf/C9eU+fiO/6ajzyfDPrcFOT6aAxEiumnJit28aQ4U0rOXDCu9SpcQ9xbQ0bM34zpj7e/xmr0gpuH3xrNva4+klGjMTpkQDBsJMk5ky7uE3nkY1+Q5+E+6uktDDzFDEP+qH4/91tKcdQBt/lJ+13wG6tipPbatP9h1RlQvuuZNWJYO2Ma9tT0q4/TTs1cUhaDiRwQtYz+h/n3L0B94ZI+GHmKevT3vgR5N6icHVUmGldtv/wUHHzyd88+/KOnvtkzSrxG0vQm9TJBxEkMvYzKO6nyP1+ztOlRVsXLcp+DZhyIGgaDeo16vb1/LnA0PsNt7MO3B+YR1q6M2q3k7wdX3o5UejG/Rt3vsfMt2PPvk9dke9eb9zgdFoW7PlkE3vLacZJqi2xG0I5WYjKNHvw+EZ+9HtDcxw72Tg/a+hmvyHHzHfTOlAWv2g7wgmgo7FDEwxdAMqpKe/QhG14duRqD+1DUQuXHiR8J2RbyMo5t2NI7dQWutozmePQmhl9b2sbeBnjx7b5xmn+nr2rM36vfQ/uqfcZlBLsj8AFY/QkQ3yVdbyV/7N9TsYvwnd+/R27hdKuOKMtmvJHkqA9uWGHH7m0punP5gl2/EZRFNJ8++YzSOdwA8+4CSgWir59Kst2jxj8O36Fspj0wuzvPh92ocMM7qaLcH4knPvgciX7xNZNMbnZYritLveRzd047FPbX7fDJHH13ON7/5Hd5883Wampr48Y9/xpo1H/D++++g6zq33rqcyZP3p66ull/84mcEAm2EQmGOOmoB3/3utV2W2Zsc+Q899Fi/c+S/997bzJvX+xz5RSVWfPWAZL1MJfTS0fdNZ31HxrFTI6iq892RceI8eyCl0a1ej0Z9S4jWYIRMf/LbRITbaX/pHhSXh4rya/jypX9y7LY30XJn8vWs11FMA/8p16J4M3vcN5vbrjyiy9+0uDEE9iN2sPVzVVFQSAxlHewHzEDiROM4Mk7/PHs1KuNgRAgLH59P/i/Gd5NIriMZPjcrfrDQGdEbiBp7mS4hDcjKyub++x/klVf+w09+8kN+8YtfcdVV3+fhh/+XBx/8GzfffCtZWdksX/57cnKyCAbDXH/993nvvXeYPz95VEPvcuTXsndvRb9y5D/wwEPoutnrHPkx+9yfaBxbxunNoKrO+ewdGUdLDL1UiMXeG73U7NvaI4TCRpeaffCdhxDNVfjP+DEeUcxz7WUsyNrFhE/+guYKEZr7TdS81KasS4V4ScVmKAyvqirOTGjQuwk7hpv4aBxF6f9biapAk5ILisbfW49ltid51E2q7bKT50nPvgfcUxck9b6HMsXxCSecDMC0aQcDCgsWHBP9Pp3XX38VsJKT/elPf2TDho8RQlBXV8fmzV90aex7kyP/ww8/oLKyYlhy5Nuvxv2LxrH+9iZdQrLQS0fG6RB6qaqKs8yJ4NHNpHHs8Xg9Gk1tljeYzNi3fvo2+hdv45l9Dq5xB+OrbaNdeKideDwl257lreBUjpg0p8d96g3xk5fYD7ehMvaWjJPYjnTA9uTbQwZej9bv/gZFUdjsmspJF57Klv/5iDl9LE5TVRQlTsaRnv3Ix56mUVVVPJ6YUVBVFcOwntr/+MfDtLQ089e/PoimuVm+/HbC4VCXZfYmR/6HH66momJ4cuQfNmue1cIhCr2Mj6bRO4Ve2p59YuhlvDcX38Hbo2bvib3udzT2ZnsztavuRy3eH8/ss4HYbFV7Co+gScnhiTUqx/RTMuiIqigoSuzBqCiDL+PY9cbP/JVOoZfxHrN3ALxnRQGBAlFprj8PD7dLHVIZJ6Uali9fzqJFi5g2bRpffPEFALt37+acc85x/i1atIh58+Y52yxatIhTTz3V+f3NN98cnD1IA1paWigsLMLr9VJTU81bb72e8rZ2jnz7wdHY2EhFxR7AMvbvv/8uLS0tTo78v/71z45H3tcc+YCTIx+sztuKij0ccsgMLrnkcubNm8/mzZscK9+/Dlrrb2+icex0tRDLZ+90xsaNoBWmZZhinYyxCU96em32xRnqjnH2oXf+DzMYwLfwG06onTMPbQSqsw/GQBsUb80xvGZq6XUHpE7VqrOrTJsjGVVRnLQV/dXrIdYfaF+u/TkFbk0deR20J5xwApdeeilf+9rXnGUTJkzg6aefdr7ffvvtjkGyufvuu5k6dXDjgNOBCy5YzM9//mOWLLmAoqIxzJkzt+eNovScIz9j2HLkxyR70etZe2Kbpq7Z28YmFDf9ny3XmR00exFNcayoirMsPkVyT56UN84wZMVF4+h7PkXf+h55x1yIURB724rlwNcdIzwYN7CmWpNwIIYuKkZV7Idn+hl7sM5DOMWU0z1hHYs4Ga0f1t4VZ+xHTAdteXl5t7+Hw2FWrlzJX//61wFpVLoQn2O+tHQczz77svN99uxy/vrX/wfA2LGl/OUvD6bUl9DbHPmPPPKE83moc+TbWR77Q2+icewbLBhJnCQE4jX7uERo0ckl7Aid3kbj2NgyjjB1Qu88jJJdRN6R51LXGNt/VVHweTTaQwYet4qmKoNiFG0v2/48FKiqgilixy+dZBywPPq2oD5wnr1pOtdtvzx7l0p7unXQvvLKK5SUlHDooYcmLF+6dClCCObMmcP1119PTk7OQFQnGSHEa/W9nXw5tl3qg6qSevYdc+NoMX3eknHiYsXjOmh79OyTGPvIZ69hNuzBd/LVqG4vkPiw83vt2apcg3bzqkosXfOQefaqgmmaCXP8phO2kR8wzT7Os++PZu/SVJqjQQBp00H7+OOPc/755ycse/jhhyktLSUcDnP77bezbNky7rzzzl6VW1iYOLikulrtceSjTarrDTX7UrviPUvNZXuzKsXF2SmX4Y+OKFRUpcvtnOV26GHcDaZE6wtEDXl+dJKRjEwPnrYwmqZSkG91Qufk+ikuzkY3BLk5vm7bWVLUBlge1/hxuYhIkF3rVuLb71BKyxcmtitKVoYbgYLLreHzuHp1HFLF5VLxel0IrM7oZHUMdL1uzQo+8PncqN2cp54YjOORChk+y8xlZnh6vsZ6wOtxYQid/AKrgzYnu/vrqDv8PhdVDYFo/VmDfi77beyrqqpYvXo1d9xxR8Ly0lIrvtjj8bBkyRK+853v9LrsurrWhJhi0zSJRIwen6ZDGXrZG/a1diWkDY4YmFEPsKabiaX/+4lPKCnwc8Fx1iTfLa1WVFIopCfdrrg421lut9HeBiDQHqGmpoW6Oss4B6LSUktLyPnc2mLlpK+ra6PKZ01TFwknr88mGB2Ek+lzUVvbSuijf2O0NeE98Wpqa1sT2mXj1lQam9tBmGgq3ZbfVxSgLRC23qToXEeydvUXISDQHkZTrOimvpQ/GO1KFccnEaLHa6wndN0kHDaorW0FoK0t1Pf9ErG349bmdmpqEmWmvhwzVVU6Ock2/Tb2Tz75JAsXLiQ/P99ZFggEMAyD7OxshBA899xzTJ8+vb9VoaoahqHjcvWcmEoyBMTpOAIwDB21h0RQFbVtHYrofW6cbmWcDjNVxYdeGqaJYU82rnXvMNgdtFl+NyIcIPzx87gmlaGVHNjlNll+N02tYTL9blz9zMHSFXbMu/15KNBUOwIovVIl2Ng57L0DotnbYzhi3/tKvNQ3FG/8KRn72267jRdffJHa2lquuOIK8vLyePbZZwHL2P/sZz9LWL+uro6rr74awzAwTZMpU6Zwyy239Luxfn8WLS2N5OUVjoiZ6yUxTFPQ1tqA35/cq3DWi8szD7HnRW8mL7HnetVUJUnWy7hEaJ1CL0WPUwba2Jp9ps9NeOPLEG7HM+fcbrfJ9rvZXdNKge5NOkvVQKAqCsK0ZhAYKsOrRCOAhjLccyCxz8XAROMoCZPZ9+d4uOMcjhGj2d90003cdNNNSX9btWpVp2UTJ07kqaee6lfDkpGVlUtDQw1VVbvpLrpbVVXMFEL5hpp9rV2BoO4MCqk2vWT4M8jK6n74eHzOGojz7FNIhGaH/tl5RXwezTHedvMTZ6qyjFNsoJVwEqe5e/Ds7Tj7PB9EPnkRbeLhPc7rmp3hoSUQGbAwv2RoqmI99MTQRcU4nn10RHK6YXfQDkw0TscO2r6XFR8kMGI8+5GCoigUFPQ8Ge9w6oPdsa+1699vbeOpt6wBXr/8+jwKCrr36iE2MbhNb1Ic2+sEI5Zn7/O4uo7GsUMv1dgDQDcFkWj64VRSHAMcrq9DBFvwlJ3VY/uyM9xEdJPWQAS/d3BkHCVqeIUYwtBLJZb1Mi09e7ea8Lc/2IOqYjJO/0bQOp9HyghaiSQZCR20KXjmkJg9Efo3gtbn1ZwRtJ1knKhBVJTE3Di649n3MILWozFeq+eQprdwTZ6Da+xBPbYvKzr5SH1LcNA8tXgve+gGVSkIEZsPIN2wNfv+zj8LMc/eHAjPPt7YS89eMpIx47T3VAZFQazjtGMZ8TLOWx9XUtUQ4PyFUxLW61iDz6NR2zE3TryME9WY45d1NRl4PHrlJsLrnuUHhV+CKxvvsZentG/2pCMtgcigeWq2lw1DO6jKMK0RtNKzV6L9Ttb3/mn2VnsUhqb/RXr2kj5j9smzj+UVgeSe/cdf1rH6s+ou67LxeVxxKY6tZXaysJiME9dBa5gpGfvQB//EqN6Kr/QAsk65GtWXWqxz/NyxA6EPJyM+T82Q5sYRAiMqi6UbA+nZqwM5qCp6Dbpc6pDM/iU9e0mfiffs9RQ7eDt69slCLw3D7CTrJJuMxufRHC05/rVajdNVVSUxn73uhF4mt1pG/W7Mqi145y/Gc/ipKe2TTfx0goPl2WvR1AV2+uahYDiSrw0kjmc/ICNoO2j2/SjLvkaGIi8OSGMv6Qfx9l1PVcYRidE4dhnxMpDluSY+PJJp+r5oJ6phmLFQOFWxXrWjxklRFDQlWehl8hss8tlroLpw9TBLWTKy47JjugdAMkiGZXjNIQ29VJ1+gvRLggZxnv0AvG2pHTT7/hwP9wCGhKZCGr6USUYKZgdvPLVt6CDjxDpYHS9fiE7GPdmLg89j+SoRPVFDVdXYDWmFXsYGWkW68eyFHiay+R1cB5SnLN0ktkdzyh00zT7q2ZtD2FlqZ70cyjoHksHU7Aci9LKnMOCBQhp7SZ9JkHF64dknG1QFMe/dMEQnjd7sQsax6jad9RVFcW5IO/QyflCV3o1nr+9YB+EA7mnHprQvHVEUxdHtBzXO3tHsB6WKpHXaxl5LSxlnoOPs42XD/nv2gzXauiPS2Ev6TJ9CL+NmPAIwiX87EM46HUfUJu+gjTP2zmt1TGO2Qy9d8R200XYm87wjm99GycxHKz04pX1Jhm3sB2sErSNRDWHopaLG6lTS0bN3Daxmb4r4gIC+lyU9e0nakBB6mUKcvL1NsmgcqwzLEBtCdArltMuPl18cGccwE2UcW1eNdig6nn03oZdmezPGrg24DzwSpR8hJ3YnbU/pGPpKvJetDlFojPXwJG07aPcvzeGQyfmMK8rsd1mqkxtn4Dx7qdlLRjymKRxvSTdM3v6kktrG9i7Xt6MYRJy1Twjf7EbGsW8ub5zuao9S1fVEGccOFRTCkjrsCU2sQVXJNXt96wcgDFwHHdmLI9CZwZZx1DgZZ6g8e1s6Std89gU5PpYuLiPT1/8Eikp0gNnAaPZK9K809pIRjimEY9TaQzp/ffYz3vqkstv1rb+xZfH6fbyM01Hbtz37eN3V9ux1IzbgSlVinWh2h2JCB20Xnr3+5Qeo+RPQCiamvP/JyPbbnv1gRuPYkUaDUkXSOoUzSG1o6hypxOaglZ69ZBRhmsIxvs1tEaD7jlo7oqbrDtqojGN7+PFvANEV7RtDUxXncySug1ZVLRnHNAUmUU9fUVCwc+N0NvZmoBFj72ZcB6Q+N3BXDEUHrS2FDUvWy1Fu7e0Be84ctP0oa6jj7KWxl/SZeBmnqc2aUKS7jtqYZ995UBXEPPuORt+uC2Ix05qmODeJrpsJnpaqKo5cZNsmTbM8Yrt98YZS3/YhIHAd0P1cy6mQk+mJtnOQOmjjR9AOuYyTnnH2A4nKwCVCG+ww3U71DUktkn0SU0SHeoMzl2a3xt7sLOPEf7Y1e3u9jgOtIBYr7VJV5ybRDTPmaSmgEAu9tKNHLK3bRAjrjSD+JtW3rUHNK0XLH9/rY9ARe2DVYIdeiiFNhEbsAZOGHbQDiaImpkuQg6okowI77lrTVJoDPcs49g2SLF0CxAZmxWSc2IPD0ezt2GRNweWybjRLxrHWswdV2VKHbZw0VbVG0BqJk42b7c0YlZ/j2r//Xj3A+OJMfB6NMdF5bwea4UhdYL0ppW8++4EkNnmJ9X0gBlXJdAmSEY9987s0xfHsuxtJa98gCekSknTCGh08fIhp+3YfgaapMRnHSOwws1PyxmdpdDxiUyR4Uvr2j0CIATP2Y/Iz+NP1CwekrGTYD7KhzWcflXGkZ59k8pL08exTMvbLly9n1apV7Nmzh5UrVzJ16lQAFi1ahMfjwev1ArB06VKOOeYYANatW8fNN99MKBRi/Pjx/Pa3v6WwsHCQdkMyHNi6sUtTaW23PPtICjJOfKdsshG0ZgejH//ZNvYuTYnJOLqZ8Fody+USi1jRVAXDsHLuxA9i0betQckZg1q4X6/3fziIyThDnBtHID174qNx7O99L2tEdtCecMIJPPzww4wf31nTvPvuu3n66ad5+umnHUNvmiY33HADN998M6tWraK8vJw777xzYFsuGXY6hjZC93ntHWOeoowTP4rW7CTjxDz7iGE6N19C6GWc92s/ACK66QxPF8FWjD2f4d6/fEhSzA4Eqqoi7NDLoU6EJqNxOkfjpJFnn1It5eXllJaWplzohg0b8Hq9lJdbr8aLFy/mhRde6FsLJSMW+7XeFTeSs7sO2uSafez37mQc+wHhyDiq6uQD75gbx+5QFEI4RlyLdtDqhoi9EexYaw2kGiAJZyhwpggcwjw1if0EQ1LliCU+fTYMlGY/NAe135r90qVLEUIwZ84crr/+enJycqisrGTcuHHOOgUFBZimSWNjI3l5eSmXXVjY85ymXVFc3PushUPBvtQuzWV51/Z8rQCqpnZZlqlF11MUZx1X3CCprGwfxcXZzsMgJ9fvtK2yKQhAbrYPAL/PxdiSHAC8Po8jaRQXZ+Nxu/B4XCiKQobPTXFxNl6PhtvtImRE8PtcFBdns/e1j9Fyiig55PA+efbDcS6zMr0IQCDIyvImbcNAtysz04PACvvM8Hv6XP5IvfYh9bZlRo9/do51HRbkZ/Z5v1zREb15uf4uyxjIY9YvY//www9TWlpKOBzm9ttvZ9myZQMq19TVtSZNgNUT+9rE3oNNX9sVCukoHi1hAodAe6TLsmobAoAl19jrhEK683t9fYCamhYnoqe2tpVJY3OoqWmh3t5Wt9YXQtDU0AZAY1PAyUXTUN+KYZq0ByNEdINwRKempgUhoK09TGtbGE1VqK5qJLBtA+4p86itbe31vg/XuQyFIlY6CSEIBTsf68FoVzioW0nkIiaRsN6n8kfqtQ+9a1t7IIwwBY3RtCCNjQFq+ji5fChi4HapqEIkrb8vx0xVlS6d5H6JRba04/F4WLJkCR999JGzvKKiwlmvvr4eVVV75dVLRj52HHu8Zt/djFVdReM4E4I7g6l6HlTlUpU4GUd0lnGEwDRJiMYxTUEoYuJ1a5g12yDSjjb+0L4fgGEgITJmyEbQ0mncwmhFUbDerAZAs/e6NW698giOmjF2gFrXPX029oFAgJYW66kjhOC5555j+vTpAMyYMYNgMMiaNWsAePTRRzn11N5N8SYZ+ZgmaIqSEE3QfbqE5NE49vZ2si379+TGPtZBa08mrhsmdnYcNT70klg0jp1ALBgx8Ho09D2fAqCNn96PIzD0JHSWDpFmbz0oo+MqRrmxj58bAej3w29Mnn9kxdnfdtttvPjii9TW1nLFFVeQl5fHvffey9VXX41hGJimyZQpU7jlllsAK2Lgjjvu4JZbbkkIvZTsWzihl/GpB1IaQZsYjeN2qYQiRqdsl/Ex+7ERtHboZczoR/T43Dhx+dfjDKJLVTAMk3DEwOvWMPZsRC2c1KcZqYYTe1APDGHoZbROOYI2Fldvv32m07MvJWN/0003cdNNN3Va/tRTT3W5zezZs1m5cmWfGyYZ+diTXmupevais2dvCuFEI+immdSbj9/WSYSmJcozsbhnBRWcWOj40EvDFATDBhmagVGxBc9hp/Rxz4ePeAM/lIOqAGnsiRl3I042TBdkugRJn7Fv/vjX0O5H0Cbz7DvIOEkGUsV/9nbw7G0jHq+h2lJH59BLQThiMMaoBNNAG5deEg4kGvghM/bRenTDlHH2SixdtvV9OFvTO6Sxl/QZu3PV9swVepJx7L+dZRwgOsI19puexLOPJUKLGnFNSXhIKPagKkiIC9c0lXDEenMoDu0CRUErObDvOz9MJBj7IcyNA9ZI5SGaHGvEYh9yU3r2ktGENYFGTMbJynD3QcZJ9Oy7lHE6pTiO5bW3ZRyFuBTHZsdEaAqBkJXSIb99J2rhJBSPvz+7PyzEG/ih1OzBikIZ7TKOQmIHbToZ0HRqq2SEYeVKiY0AzM309LmDFqxOr65kHPutwO1OzAFuhSKaTv8B2EPaE7M0aqpCIKijYZDVthutdFq/9n24GBbNfhjqHKmo0rOXjEbseVDtdAk5mR7Hs7/36Q2s/rw6Yf1kqY2tDtp4GScurXF8NI49B23c5CXWX9UJ2XTCLBXFyV9iZSk00RRBIKQz0VWHKnS0sVMH7DgMJcOi2Svxn9PHuA0GTjSOkJq9ZBRhpxDu6NkLIfjoixo2725MWD/ZNINC4GSh7CjjJJN0bM/e1UHGEXEDqFQlKuNElwVfvIeTmv4FwuRAV5W13diDBuw4DCXxBn4oJxwf6jpHKk6cvSESvqcDMp+9pM/YoY22t2NPyacbZsKo1tj6nROgWfKLiqJ0L+MYHTR7+wHjjCiNG91pyzhCCDRFoO/eyFgjzFn+j5jp2UkkswTVnzOQh2LI0IahgzZ+4JAcQWv9laGXklGFEefZK0B2hmXsAyEDoLOxN+M/2zKOdQNpqtopGiepZ+9SufD4A5k3vQSInx81Fnnj5F83BdnhajDCBNUMTvB/il8J0zbrvwb0OAwl8bZlqDtorc9DUuWIJTaoKv1kHOnZS/qM3QE6b3oJORkeR1ppjyY3i/fkO353PPvoA8MOoUw272z8tqqicOoRsYlG7InEO8o4pmklUMgL7gHg3aLz8ex8n9eDB/P9kvSUcCDRwCtD5Kolyjij2z/sGHqZTn0Y0thL+oydjGv/0hz2L83h1Y92AzFjb3Tw7EUS42179q7oTFIJ0xQmSZfQ0Zu1B1WZxAZQ2YOqAHLb96D4c2jyj+fVwHwAfO6+ZSkcCajDYHilZx/DPhZyUJVkVNExGZft2QeCuvN7wvpJonEEwkloZnRIl5CsQ7ejfdOU2NyytiFSlFj2zdz23WhjpiSkdIjPv59uJBpeGXo51Nh778g4pM/xkMZe0mc6ptl1jH0Xnn38V/uzsDV7TUU3RYI3n8zwdzQ2towT30GrKtZbQoYSJCNch1oyJeGNwJvGnv1wRMZIYx9DpkuQjDpMIaIjKmPL7Nj3mGafuE18DH0s9DI6CteWcXrooO3ozarRNwJ7tKzZWk9Z08tg6uznqrPaNWaKI3nEh4qmIwme/RDdvTLOPoYTjTMA+eyHGmnsJX0imYbu7kHGEWb8Z+GsY0XjJJNxEh8Odt6beDrKOJEt7zG17UPGs5eJWtTYF012PFKvR0urcLmODGcitKGsc6QS0+ytazOdLiVp7CV9wskyGS8rdJBxutPs42UcO01y96GXyb0oTVNjMo6iYNZuB2ActUx01dPuLUTx+J3EaV53el/yCTLOEE44nuzzaMSOgErHQVXpfeVLhg3b6U7U7KMyTrALzb5DtkuIeuwosXj5hMlLEmWcZBp1LMVx9HPdDgAmajVM0OpoyyhNaGc66/WQOKhpeDz7IalyxBKf2x+kZy8ZBRhJNPSOHbTdxdnHp0tQozJOx8lLOkbjJBu9GT+oyqtEEE1WOoQDXXsp1NoIZIy31tNiMk46oynDbezTyLoNAp0HVaXP8Ugpzn758uWsWrWKPXv2sHLlSqZOnUpDQwM/+tGP2LlzJx6Ph0mTJrFs2TIKCgoAmDZtGlOnTkWNugJ33HEH06alZ6ZBSWfMJDJOp0FVqUTjROPjXUlknI6dtclkC3sAlWkKSqkFoMY3ieKg5eG3Z0WNvbJvePbDkwhNyjg29t470TjD15Rek5Jnf8IJJ/Dwww8zfvx4Z5miKFx55ZWsWrWKlStXMnHiRO68886E7R599FGefvppnn76aWno9zGSRcfYMk4gaOWNT0nGSeig7RiNk9hBm8y42SNvhYCximXsd+XOdn4PZpZG17Mu9XT37GUitOEl3rNXSC/PPiVjX15eTmlpacKyvLw8jjjiCOf7rFmzqKioGNjWSUYsyTz7WAdtV7lxOnvtsQ5aKxpHT0hxnLh+UmMfJ+OMpRYlI4/azAMxBdQaWZjuTGc9SH/PfjgSoUnPPoaTzz5uyst0YUDSJZimySOPPMKiRYsSll9yySUYhsGxxx7L1Vdfjcfj6VW5hYVZfW5TcXF2n7cdTPaZdrmsSyc3x+9sq0dDFUK6Zew1TU0oNyMzdv7z8zMpLs5GURQy/B4yQgZhPUhmphcAj0vF5XY5bfN4XLg6lAeQmeGx0iS7XZRQi3/cFHyZ2ewwiqgychkTbV9urjUrVW62b8DOwXCcy+bogxSgqDAraRsGul31gYjzOS8/o8/lj9RrH1JvW151G2Bd26o6+Ps0kOUPiLG/9dZbycjI4OKLL3aWvfbaa5SWltLa2soNN9zAihUr+MEPftCrcuvqWjt5h6lQXJxNTU1Lr7cbbPaldtU2tgPQ1hZ0tm1uDgLQGjUOwbCeUG5zSzC2fV0rPtVKhxwMRRCGSXtIp6nJKtftUgm0hwGoqWkhEAijIDq1MxzW0Q2TcLCdAtGAnr2AYCDCiuaTMVG4Itq+QFsIAGGaA3IOhutc2sfH+hygxpXoXQ5Gu5qbY3W2tgT7VP5Ivfahd22zj0UwpAPKoO5TX46ZqipdOsn9jsZZvnw5O3bs4A9/+IPTGQs4sk9WVhYXXHABH330UX+rkowg7BGEWhIZJ9hVB22CZm//FagouF0qkYjpTDLudqmJkTlJNHthGsyt+hcz1C3k6HWoCNSCCSiKQgQXBrEBVHY70zkJGsgO2uHG0ewNM+2SwvXL2N91111s2LCBFStWJEg0TU1NBIOWF6frOqtWrWL69On9a6lkRJGsg9aecUp0WMfZJj4ap0M+e7dLJWLEJi/xuLROWS87GprIZ68xtu0L5ri2UqBbnbNqwYSkw/udDtp0N/bx6SmGRbMfkipHLLY/a5jJQ4FHMinJOLfddhsvvvgitbW1XHHFFeTl5fGHP/yBP//5z0yePJnFixcDMGHCBFasWMGXX37JzTffjKIo6LpOWVkZ11577aDuiGRocYx9Es/eWadDnH2yFMf2ROFul0o4YiRMP2iIrjtoRaiN8JonAZisVRMySjBQUXNLUJVdznr2NrbB8qS5sZcTjg8v8dE46XYoUjL2N910EzfddFOn5Zs2bUq6fllZGStXruxfyyQjGttuJwu9tOmc9TKZjJPo2euOZ68mRuOIREMT+fwNRKiVnQXz2a/+Pabom2lQC8hTXYmjTKMf7UFVvn0o9FIa+6HHdmdMU6RVemOQI2glfSRZIjRVSbz8O8o4yScct0LY3C6NSMR0pBu3S+s0wCpetjDqdqBkFVJROA+AbNFCnVoYbUeszo6affrLONLYDyeOZx9NzJdOSGMv6RO2sU6YjFpRcLlil1T3nn1nzV4AEd20Zq7SlMS4/A7pEsyGCtT88ei+AlpMHwANWhGQvEPRMfZp7tkPSz77JH0go5X4aQnTLc5eGntJn+gqv3y8lNNZs++8vRBWx6sn+pAIhg00VXHy1NsYcYnQhGliNlai5o9D1VS+1McAUK8VA4mjGu0ONZ/HUiyz/O6+7fAIQVU7P8gGvU4ZjeOQzpq9NPaSPmEkkXGs77FLqvtpCWPLLBnHNvY6mqriUtXOI2ijN5poqQEjgpY/HlVV2BIpwRQKDS7L2CcziPuXZvPDi2Zx0ITcfu33cDMckopMlxAjfg7adPPs5YTjkj7h5LPvcL27Xd0Y+6SavVWGPfFJMGxY6RNUpctoHKNhj1V3/ji0NoW3QtOo8uyHT8vq1Cb7hlQUhUP3L+jz/o4U4j3roTK8CVLdKDf2zkxVptTsJaOELueEjfveUzSO/cBQFAW3O4mMYyQ+HJy8JA1WDiY1bxyaqmCiskfPSzDsNvuabRqW0Mt9+Hj2loREaGlm7aWxl/SJZInQIJbmOH4d53sHz97+WVHArVkdp7axtxOjxW9rGzqzYQ9KZgGKx+/UrxtmLKZ+H44eGQ7NXso4MZxBVYaZdv0X0thL+kSyQVXQwdj3EI1jxnn2HndMs7dlnK6iccyGPagF0Tz18cY+2pRkMs6+gn28h9Loyg7aGHZwsSlDLyWjBWdawi6icRSS5bNP/Byv+8dr9lrU2Otx29uTl4hIyIrEyRsHxDqEdSP2Wr0vGyd7LMNQvrHsy29KvSVes0+3a0sae0mfsL3yjh6mK26SkHhdPn4bsD1767OaRLPXOkXjWIYm8ukrYERwHzC3U/22IYr35pV98ApXo30aQ1dfYt2jmcRonGFuTC/ZB28FyVDQU5y9PXgp2byz9ueEDtroQyIUF43TUcZxKzrhj59HG38oWsmBVv1JUiMkGKd0uyNTQFWVId2vfflNqbckRuOk17GQxl7SJ5KNoIWYZ28PYko2OxXY0TjWZ3sErV2upqrRQVWJMs7U0EZEezOeOec4y+M9+9Eg44C1T0Op2SuK4hi50e7Z29eYndMpnZDGXtInuhpUZRt7f9SzT8yHQ9znDh20rlgag2TROMIUFEUqUTILcI2dmrCujW3YE2ScNLshU2GoZRyIHdt98eHZG+J3P92OhTT2kj4Rk3ESl3fMLtnRs7dlnvjQS1VJHIyl2pp9B88+x2hAzS1JqC+x8zDZsvS6IVPB7sAeSmJhrUNa7YhDTWNHYpSfOklf6Sr00t1RxukwYYkdPSNMEjz7eGOvaZYxEyJ+khNBlt7Z2CeXcei0bF9CVYZ+cNNwhHyORJQ0vraksZf0CWdQVYcLvqNn3zGtcSqevRanSdtSjtdsxyuCSYx93BtBUs2+jzs4ghlOGSfdDNxAk879QdLYS/pEsnz2EN9Bm0TGEXGjYDtE42hqYieg/dCwwy/zabJ+yxmbUF+yEaWJWS/T64ZMBSudxNDeupoj4+x7x7M3pHN/UI9XzPLly1m0aBHTpk3jiy++cJZv27aNiy66iFNOOYWLLrqI7du3p/SbZN/AtuE9ReN07GS1py6Ml2gUJbGT1pJxooOlousURI290q2MY/3d10MvFWUYNHvFGii3Lx7P3rBPyzgnnHACDz/8MOPHj09Yfsstt7BkyRJWrVrFkiVLuPnmm1P6TbJv0LVn36GDNiEChy5kHGuZLeUcEv6Yqdsfw0PEmbmqQGlGoKDmFCfUl2xQVTp3oqWCpipDLk8pwyAdjUTSOclej8a+vLyc0tLShGV1dXV8+umnnHnmmQCceeaZfPrpp9TX13f7m2TfwehyUFVnGWfj9noq69qi0Th2B22ijAMxYz8ttJHChk/4TvZ/0IMBAAqVJtpdOSha4uQjPco4+6C1Hw7NXpPGHkjvzv8+5bOvrKykpKQETbNfuzXGjBlDZWUlQogufyso6F0+8cLCrL40D4Di4uw+bzuY7CvtysjwADCmOBufN3YZ5WRbUwQWRc9dbq6fux5bT9nUYlRNxRuVdzIzveQXZEbX8VEcLUdrCZBv1BDMncSkxp20PPUb9rv4ZorUZkK+ok7tjMTNepuZ6aG4OJv8xmDCfhXk+Hq1b6kyXOfS49bwuLUu6x+MdrlcGpqq96vskXrtQ+ptc/s8zmePp+tzMFAMZPkjevKSurrWTpkTU6G4OJuampZBaFH/2Jfa1dxiGdT6+raESJr8DDcTijPRwxEA6uraCIZ0mlqChMO64803tQSprWsFoLUlRE1NC5oC47QGNEx2lyzgP7v35xs1b7Lz/h8xRm2iyrV/p3Y2NbY7n4PBCDU1LbQ0x5Y11LdhhCK92rdUGM5zKUyBaZpJ6x+0dkVzwfS17JF67UPv2tYSCDufdT35ORgo+nLMVFXp0knuk7EvLS2lqqoKwzDQNA3DMKiurqa0tBQhRJe/SfYdYvnsE5eXHzyG8oPHsPaLGsCSe3TDJKKbVuilPY9shw5asGScca46AEI5E/g44sJ18rWYa57Aq+i0+jtfQ2pCB+3oGEGrDHFuHBj6fDwjlX1as09GYWEh06dP55lnngHgmWeeYfr06RQUFHT7m2TfoatEaDZqXIilbggiholp4kTjmGZcB210XbemMlGrI6T6Ef5CAMS4w/Cfews3NVxAVV5Zp3oS0yXYf/f90MvhGEG7Lx7L3pLY+Z9ex6NHz/62227jxRdfpLa2liuuuIK8vDyeffZZfvGLX3DjjTfypz/9iZycHJYvX+5s091vkn0De/KGri54Z75YU2AYJrrt2UejcQSJHbTB9x7ljPAX+Nx1NHrGomp2nnoTU1FoEX5nWTya1nlQlbKPh14W5vjwuIY2zl5VpLGHjrlxhq8dfaFHY3/TTTdx0003dVo+ZcoU/vnPfybdprvfJPsGRtw0gclwpgvUTQTEZJwknr072EDkkxeZLEzQYJP3UNzxDwszscyEehJy14+O0Mtvn33okNepqvvmg7O3pLNnL0fQSvqEMLu/+bXob6GIAUSNvSkYa1RSojZamn3U2hfseQtQWJ+5AIAG/34JI2i7k4x6lHHS7IZMheFKl7AvHsveks6DqkZ0NI5k5GIK0a3BsX8L65Zbbmn2goWtz3FgZgYVogwhwKeEyd37Aa4p89gUWMCju8dzzJQDmBC9kXTTJBgNgHAnkS6Sx9kn/13Sd4ajn2Akks6d/9Kzl/SJ7ubgFEJQ+MEKjvR+QTjOs3eJEFlmM/tpdVb4oBDs76pBNUK4Dz4Wt0slIHyomhrryDUEO6qs8LOJYzqHlNlvABCX9TLJA0DSPxRV6ZQaYzQi89lLRh3defZmwx48dZuZ790S8+x1kwKzEQC/GsEXrEUIKFCtWHs1dyxuOzeOqjpepG6abKtoRlFgUknnASb2BNzW59gymzS7H0cslowz3K0YftL52pLGXtInwhEjwauOx9j9CQD7abWY7ZZXHtZNioilzMgJ7kYIQYHailA0lIxcR6aJH5pvGIJtlc2ML8py5rXtiDPRuJoo4yikn646UpEyjkU6a/bS2Ev6xK7qVsYXZSb9Td+9EaF5UBXIbNwMWJ59sdKAgUZQuMgJVEQ9+zZ0Xz6KojrhhGqcYdENk22VzRwwruth4/ZDR+0g40i9fuCwJkyRx1Nq9pJRRThisLu6jf1Lczr9JvQwRuUmwpOPpM30UNz4Cd/Kepn5rk8ppoEWdyG7jSJyghWYQlCgtWL484FYB6xLVRzNfnd1K21BnQPG5XbZHifXegcZJ908r5GM1+PC08Wb1WgjXefjldE4kl6zs6oVUwgOSGLsjcpNYEQQpTPY9Ol2Zrd+wRgPTHLVYqLS5Nmf3W0qB4Q+JWBEKFBbMfxTgJixj/fsP91mpU9I9mCxiQ2mSpRxpGM/cFy46EAn3fRoR1EAIT17ySjgy8pmAPYfl2iAzZYagm89CJ4MxJipfBA+kDYtl2cCZWSpIXLUdlo9xew2ilExcTdsJ0cNYmRYqTTijb0t6Xz4eTUet8q4oowu22O/BXT0uGT0yMAxJs9PaWFy2W60oUjPXjJa2FbZTEGOl7wsr7NMmCaBZ+5AhNrIOP2HhNwePouM599jZvNezV7meLZR6mqkzVvMDtPKSe/fsxoA059o7F2qQn62l8tPOxgDhcIsd8Jcsx3ROnr0SUbSSiQDhe1DpNvlJT17Sa/ZVtHcSVYRLdWIlhq8R1yINmaKY5zDERNQeDU4HUMotPhKaVP8NLuLyKhaB+B49va0hKqqoCgKx84cx4UnTuXwKUXdtqejRx/7OyC7K5EkkCy7ajogjb2kV7S2R6hubO9k7I26XQBoRZOBuBG00UFV74cP5BeN5xP25KGqCrW+SaiGNTTWzLAyXLriZJzeoHXw5J3QyzS7GSXpgSI9e8looL7ZmrRkTJ4/YblZvxsUBTV/HBAzwLaxB4VmkYGqWka4xrsfAIZQwJ8H4Oj0rm4km2TYoZd2tksZeikZTKRnLxkVtLRbsz5lZyTOBWvW70LNHYvisqZts+1sSE+M4LBHYlZ7JiJQaDAzUaLG3d1Hz76jRi9lHMlgEgvxHd529BZp7CW9wp6WLTvDk7DcqNuFWjDR+d5RxnGWK5YeH1T9hLLHU23kOMY5fgRtb9A6RuFIGUcyiEjPXjIqaAl09uxFuB3RUoNaMMFZZhvsSAfP3p5ST5iCikMv5//aFjjGeUJxFifPncjBk/J71SZHxumio1YiGUiS5WBKB/oVerl7926+973vOd9bWlpobW3lgw8+YNGiRXg8HrxeKzxv6dKlHHPMMf1rrWTIee/TvYTCBgtnjQcsY68okOmPGXuzYQ8AWmFnzz7UwbPXVAVFsRKpRTw5tAi/Y6RdmsriEw7qdRtjGn3y7xLJQKJ0eINMF/pl7CdMmMDTTz/tfL/99tsxjNjNfffddzN16tT+VCEZZt5YV0EgpDvGvjUQJsvvTvBq7EicBBlHsWWcJJq9qiAECGdZ/9qoKdKzlwwd6SoTDpjvEw6HWblyJeeff/5AFSkZAQSCOrohnO8tgUgnvd6s3QGeDJSsQmeZolgefGfN3vrNFIlz0PaHjiNo0/VmlKQHo9Kzj+eVV16hpKSEQw+NzY+5dOlShBDMmTOH66+/npycrvObSEYmgZCe4CE3B8Jk+xMjcYzqLWhjDuhkXDVVSXhQgK3Zgxk3LWF/b5qOso0SzXEvQy8lg8Go1OzjefzxxxO8+ocffpjS0lLC4TC33347y5Yt48477+xVmYWFnWcmSpXi4q5T4g4nI6FdtY3tvLF2N+cdd6BjoLtqVzBs4PVozu+BkMHk0hznuxlqp6V+D/mHLiC/QxmapqIbiZ59brYPt1vD7dbIyvIBUFSYRXFx1+e6p2Pm91kPn/y8DGddVVXwuLVBPd4j4VwmQ7ar9/SmbVp0pHdGhmfQ92kgyx8QY19VVcXq1au54447nGWlpaUAeDwelixZwne+851el1tX1+pMNt0biouzqalp6fV2g81Iadcj/9nMS2t2MXVcDiUFGV22yxSCtqAVfWP/3tgSxDMh1/mu7/kUEASzJnQqw/Z71KhsA9AWCGMagmAwQlNTOwANDW24SX6eUzlmhm49UJqbg866iqJgGOagHe+Rci47ItvVe3rbNmFa/VDBYGRQ96kvx0xVlS6d5AHR7J988kkWLlxIfr4VMhcIBGhpsRophOC5555j+vTpA1GVZAD4eGstAFUNgW7XC4UNhLAmCwcwTJO2oJ4QdmlUbQFAG3NAp+3t8Eu/V0tYpipYHbRR+97f1+HYoKq4ZXKyDckgMao1+yeffJKf/exnzve6ujquvvpqDMPANE2mTJnCLbfcMhBVSfrJ3voAVQ3t0c/tHD6l63UDQR0APRor39pufY/voDWqt6LmjUPxdk5/axthl6bi0lR0w0RRLN1+QDtonayXcbMIqYoMvZQMCqM6xfGqVasSvk+cOJGnnnpqIIqWDDDrt1hevUtTqKrv3rNvD1nG3TAFphBxo2ctz14IgVm1FW1SWdLtY8Zewe2yjL2dLsEUwhFuBq6DNlaQVU963YyS9CCW4ji9ri+Zz36U8fHWOsYXZeJxa+yNM/Y7q1rweV2MyfOzp6YVVVUIRI09WN69M3o2Go0j2psQoVa0ov2S1mUbW01TcbsE7SHLIKuKFWdv98cMlGcfb9ztEE+JZKBJ19xL8kV3lLG1oomDJ+UztsCfoNnf/8yn/ONla3Lwvz33OQ+9+IUj44A18XfHvDiirREAJasgaV1anIzjjouFV1QF04zJOP0eVKV21lCV6BuERDLQpOs4Dmns0wwhBF/sanQMZW+I6CbhiElOpoeS/Azqm0POoKe2oE5No6XlVzcEqG8JEQhFErbtmBdHBBoAUDOS57LpKOPYy9ToftiBVv2dPlBVEwdVOfVIay8ZBNK1g1Ya+zRjW2ULv3n4I7ZWNPd6W1uDz/C6KCmw5nStjnbWhsIGdc1B2kM6bUGdxtYQ7aFYjHwkzrO38+KYtmef2YWxj8t54xj7aLoEUxDn2Q+QjKNKGUcy+KSrZy81+zQjEI17t//2hnhjPzZq7G3dPhQxMEzBrupW63vYoL4l6Gwb0U1a2iNk+ly4opKMCDQCCoo/+choR8ZRFXA8e+smEabphF72956RMo5kKFHSVLOXxj7NsFMGd0wdnAp2h6vf52JMvjXTVFVDgIhuYkQ1lU27Gp31K2tjmr5uCNraIwnZLkVbA4o/B0WNxdHHY3vamqaiqjEv3o7GcdIl0F8ZJ1kHrdJveUgiSYaMxpEMCfYAJ/tvb7A7XDO8LvxeFzmZHmoa2wmGYx2xm3Y2OJ8r46J1bL3f544ZdjPQ2KWEA4lx9nagpT2ZePygqoHy7BNkHDX94qAl6UG6avbS2KcZutF/zz7DZ532TJ+LQMhw5B2ALXuanM81UT3frjcUMfDEGXsRaEDJjGW67EhMs1ewkyc4mn18NE4/PfCuZZw0uxslaYGj2ffzjXSokR20aYZt5PU+GPt4zR7A73URDOkE44x9OGKSZXfACuFczhHDJKwbeNyxS0a0NaJm5nVZX0LopdNBa5l9EZf1sr9qS5cyTnrdi5I0wYmzTzOZUBr7NMNOGRwxeh96acs4ftvYezTawzrBcGJmyv1KspxOWFujt2UcTzTjnzB0RLAFpYuwS+gm9NJJl2CtNxiDqvxezdlPiWQgkTKOZEiIddAaPazZmUDImlLQ57EMts/rorE17Hj8Xo9GKGxQlOujuqGd2qYguZkeWtsj6LpJOBLz7K1IHFBS8Oy1aG4ciNfsxYDls9eicfbx5Xz7nBl4XdKXkQw86ZrPXt4NaYbdMdtxUpBUCAR1MrwuxzPxe1y0h3VCUc9+fJGVzKww109uljVKNifT49Qb1k1Hs7eNfVcDqiB2U7g7jKC1Jy8ZKM8+WW6cMXl+crO8/SpXIklGunr20tinGf0NvbQ7ZwF8Xo32uA7acVFjX5TjIy/TMpS2sbc9e29UxjHbrKid7jx7e2Sr1kHGUZSO6RL6d9fYbxv224NEMpjIQVWSIUF3PPs+dNAGdTK8sTh5v8fqoLWN/YTobFFFeT7Hs8+N8+xDEbOzjJOR12V9yTto7QnHrXQJA3G/HDG9hIJsr9OxLJEMJunq2Utjn2bo/fTs4ycS8XtdCKChJQTA/ENKyPS5mDI+l893NgIxzz4cMdGNDjKOqqH4up5OMFkHraJY/+x89gOhe/q9Lg6fUtTvciSSVFCkZi8ZCiL9jLPP8MW8X1/U8Nc3W2kRMnwuFhxWiqoo5EWNfE40w6WdFM327PXKTdakJUrXl5AtobvUmGavxac4FiLtvCOJxDby6XbpSmOfZtiefV9kHLuD1sbvsT43NAed2aRs8rITNfu2aNimx6VhNFRgVm3BPfWobutzPHuX2kGzxwm9TDfdUyJJ1zj7fss4ixYtwuPx4PVaxmHp0qUcc8wxrFu3jptvvplQKMT48eP57W9/S2Fh16MtJanRH8++vUMHrS3p1DUHnXBMm4P3y+f8hQcwfVIemqo4Mfoet0pk0xugaLgOWtBtffGJ0BLi7J10CQMj40gkQ0msg3Z429FbBkSzv/vuu5k6darz3TRNbrjhBn79619TXl7On/70J+68805+/etfD0R1oxonGqeXnr1hmgTDRsJAI1/Us69vCuJ1J77kuV0qZxw52fnc1m7JOF5NoH/xNq5Js1C7yHZpE58IbeaBRZx3zP4U5/njonHS74aRSGIdtOl18Q6KjLNhwwa8Xi/l5eUALF68mBdeeGEwqhp19NWzt3PTJ8g40c+NLUG8nq6f+y5NdWScgoZPEcEW3NMX9lhnfG6cLL+bsxbs3yEaR6TdDSORjGrPfunSpQghmDNnDtdffz2VlZWMGzfO+b2goADTNGlsbCQvL28gqhy19FWz75gEDax0CWANcPK6k6cphqhnH82fn1fxNkpuCdqEGT3WGR96GY9iD6oy0y8nuEQSy2efXhdvv439ww8/TGlpKeFwmNtvv51ly5Zx0kknDUTbKCzsOqyvJ4qLswekDQNNf9ulRAcqCUXpVVnNUc9+7JhsZztfZmyEaXamJ2l5emsDxe4glUEvE7Q6vI3bKTjpCnLH5PZYZ2a0/IL8jISyM6MRPl6fC1VVe9yPffVcDhayXb2nN23zRyPacnP9g75PA1l+v419aWkpAB6PhyVLlvCd73yHSy+9lIqKCmed+vp6VFXttVdfV9eKafY+LUBxcTY1NS293m6wGYh2BaLaeTAY6VVZe/Za0xjqodh28W8HKnQqTwhB4PFfcAW1/Dp4FqdnrENoHkLj56ZUd8ieVastlLB+KKhjmCaB9ggKotuy9uVzORjIdvWe3rYtHJ3/oaU5OKj71JdjpqpKl05yvzT7QCBAS4vVGCEEzz33HNOnT2fGjBkEg0HWrFkDwKOPPsqpp57an6okUfqaz75jxkuw5BVPNEqmYzQOgLH7E8z63WQQ5Ic5z3CoZw+RGWejeDJSqjO+gzYeR8aRmr0kDRmV6RLq6uq4+uqrMQwD0zSZMmUKt9xyC6qqcscdd3DLLbckhF5K+k9fo3HsQVEZHdL++rwuwno4YVISm/DHL6Bk5vNmaDpH6++wJrQ/s6afmHKdWtwI2nhUVUFEc+Ok2f0ikTiTlqRbf1O/jP3EiRN56qmnkv42e/ZsVq5c2Z/iJUlwsl72NRrHl3jK/R6N5rZEz16YBuGPX8DY8yneIy7kkw0lrK/P4Eu9hHlJ3gC6wvbs3V149qaZft6RRBLtNku7OY7lCNo0Q+/jHLTNbWE0VcGXxLOHxGic4H9WEP7gn7gmz8Z9yAm43Bpf6OPQ0ZzJS1LBjlboKOOo0Xz21qCqXu2GRDLsxKJxhrkhvUQmQkszYimOe9dxXdccpCDH2ylczA6/tD17ffcG9O0f4Sn/Cp6ys1AUxfHMFaWzJNMdXck4imLNVGXKdAmSNERNU81eevZpRnyKYzsffCrUNrVTlOvvtNzusPV6NIQwCb3/T5TsIjwzT3MuZjvVgcet9eoCd3LjqB09e2S6BEnakq4pjqWxTyNMIdAN4XjKvZmtqq4pSGGOr9Nyf5yME17/AmbdDrzlX0HRYtkxbc++t9P8xaYl7NBBG71LZNZLSToi0yVIBh0j6tXbBjrV8MuIbtLYGqYoN4mxj6ZJKGzcSPiDx3AdMA/XgfMT1nHFefa9wXkz6NhBG30IGIYMvZSkH04+++FtRq9Jt/aOamzjbhvoVDtp7Xz1hUmMvc+r4SHC2C1PoI6Zgu+4KzvlqLfTHfTW2GtdxNnbmqdhSs9ekn6oSM9eMshEorKNPxo+mWr4ZW3U2Cf17L0ujvBuQYsE8M1fjOLydFrH0ex7KeMU5frI8Lo6hXvaMo5ummmXE1wiGdWJ0CRDg23c7YFRqXr2dU1Rz76DZi+EIFMzON73GXrB/mhjD0q6fV89+1kHFfHHa49GUzvG2Uc1e1M4A1QkknQhXTV7aezTiEgHzT5lz74piKoo5OfEEp8J0yTw1C+ZXbsDNDBnXNLl9rFonN559oqioCW5IRwZx5Bx9pL0w/Zd0u2tdNQZ+9rGdgpzfSPuqdzUGsLvdXXrPdvG3Z5hKnXPvp38bE+Ch61/+QFm7Q7chyzCO+5A2H9ul9vHonF659l3hX3sLc1+ZJ0HiaQnZOhlGlDVEODH977L5zsbh7spCQgh+MUDq3n+/Z3drud49p7UonF0w6S1PWKFXcbF2AthEl67EjV/PN4FFzPmiFO6nTi8r559V9gekWGaaXfDSCRONE6aXbyjyrOvbwoiiEWnjBSCYYOmtjA1je3drudE46So2b+4ehdPvvElqqpQPm2Ms1zfsQ6zYQ++RVd1a+Rt7Lj+3mr2XaHEyTiuXnb6SiTDjSo9+5GPPbVee3TWppFCY2sIgNZornqbcMRI+N5Rs+/Js6+qD2CagohuUloYS0usb34HxZ+D64B5KbXP6aAdIBnHvlkMOahKkoY40ThpFlywzxr7ZJOe2FPzDYWxN8zUE5U1tYYBnEm9ATZuq+fqP75Jc1vYWeZE46QYetkW1BlXlMkvrpjLSXMnAiAiIfSdH+Pav9yZ9aonBlrGiffspWYvSTcUpGc/Yti8q4Grfvc6tU2Jsog9j6qd7newqG1q5zu/e4Mte5pSWr+xrbNnv6e2jYhuJkg7dl6cVEMv29ojZPpc7FeS7WS11HetByOM64CuO2Q74tb6FmffFY5nb5r75gUo2adJV81+n7zXdu5tQTdM9tYHEpa3tUc9+/DgevY7ovVvq2xOaf3GFst7jzf2LQFrWXMg5tnbso0vGo2Timef6XcnLNO/XI3iz0EbOy2ltkFiIrSBQEbjSNIZqdmPIGxD2dKWqIEHHM9+cI29/ZCxBzP1RFPUsw8EdcxoJsuWQCThLyTR7Hvy7IMRMn0xYy/0EPrO9b2ScKDvg6q6wq7aMEXaxSpLJKNyWsKGhgZ+9KMfsXPnTjweD5MmTWLZsmUUFBQwbdo0pk6dihq9s++44w6mTUvdm+wPts4d7xVDfAft4Mo4VfWW9FKXYtSPrdkLLIOf5Xc7D6ymZJp9N4Oq/rNmF598Wc8PLpxpGXt/7BTrOz8GvXcSDsQlQhtgGScSkaGXkvTDdlDSzVHpl7FXFIUrr7ySI444AoDly5dz55138qtf/QqwJhrPzMzsfyt7ie0Nx3duwhB69g2WZ1+bomdvR+OApbNn+d20RCWdlrh9sHPjdKfZb97dxKfb6wlHDMIRM8Gz74uEA3Q7KXlfsD2iQEhnv5LsASlTIhkqRuWgqry8PMfQA8yaNYuKiop+N6q/tPTo2Q+usa/utYwTdjpQbd3eeWDF7YPdQevrZlBVa3sEwxRUNVhvF7Zmb0k463BNntMrCQdg4pgsLjllGocdUNir7boi3iOaOWVgypRIhop0lXEGTLM3TZNHHnmERYsWOcsuueQSzjnnHH73u98RDoe72XpgcTT7QKJm70Tj9LKDdsueJn77yNqU8scHghGaA5Z33toeIRTuWTJqbA0xvth6A7KNfavdQdvWuYPW7VJxaWpSz97efk9tKwCZdpjmzvV9knDAuqiPLxs/cB200b8ZXhdTxucOSJkSyVBhh16mW4fngI2gvfXWW8nIyODiiy8G4LXXXqO0tJTW1lZuuOEGVqxYwQ9+8INelVlYmNWnttjGPhA2KC6OyQS2Vh/ssLwnXl5XwWc7GjBUlXE9bPfFzgYAyqaN4c11ezA1NaGujvUGwzrtIYMpE/L4sqIZ1a2RX5DpvIXE74PH60JTFUpKcvC4VdxuV6fy7LEEjQHr7/ixORTmedj94RO48scy9vByFLWz0e7N8egveXutB9Gc6SWUju3Z2A9l23qDbFfvGKntgt61LTvbyh5bWJRFcX5GD2v3j4E8ZgNi7JcvX86OHTu49957nQ7Z0tJSALKysrjgggt44IEHel1uXV1r0sFRPWHLOPVN7dTUtADWFHht7REUoD2oU1XdnHKcbEWVVcbWHfX4enicf/5lLQAHlmbz5jrYvL0OfzTdQHFxttMem+qovl+YZeWRr6xqYfvOesCa/KOhOehs09QcZIyrje0rruY8Tw5tdSY1NZOdsoQQTmfv5h1WGXpIZ89zf0dv2Iv/zBuprUsMR+2qXYNJS4slMR08IbfHeoe6baki29U7Rmq7oPdtC0Sj5xrq21D0wQv26MsxU1WlSye5328id911Fxs2bGDFihV4PJbBampqIhi09Gpd11m1ahXTp0/vb1Up0+x00EacSbmDIR0B5GV7EZCSvGJjd6CmklOnqj6AosD0SflAz7p9Y9Q4jy3MRFGgNRhx5Kcx+X5aAxHngRcxTOZ4v8RsrKBM28zxe/9O2+O3EPlyNUIIQhHD0fX31LahYpLz6eNENryI+5BFuMYdnPI+DyZTJ+Zx8tyJzJ5aPNxNkUh6Tbpq9v3y7Ddv3syf//xnJk+ezOLFiwGYMGECV155JTfffDOKoqDrOmVlZVx77bUD0uCesKJQDEczD4YN/F6XI4sU5vhoaAnRHtKdePWesA1yKqGUe+sDFOb4KMj14dIUJyJn7Rc1vPrP9Ri6yYXHH8iksdnRsq0HSUG2l0yfm9Z23ZGhxhdlUlkXoKU9Qm6mh4huMl3bhVp8AHdXH8sxObuZb3xK8D8r0EqnEZj7dSa7ashT2vi8YRzfzHoD15YK3DNOxjv/wt4dyEEk0+dm8QnJJ0qRSEY6tpFPs8jL/hn7gw46iE2bNiX9beXKlf0pus/YHZSlhRls3t1EcyCM3+siEDX2Rbk+tuxp6lVEjj3oqb4l1O16Ed1k47Z6ZhxQiKooFOT4nAfES2t2sbumjbb2CGs31zjG3pZdcrM8ZEYfUHbY5fhCP2uwZKncTA+ucDMT1Bpck47BrPfzmWcmi85dQmTTG4Te+T/cz93CD3Ks+sJCw4WB95jL8Uw/LuV9lUgk3XPo/gUcP3s82Zmdp/AcyaRbh3KPxIy9Fd1iR7O0RiNx7Em3Ux1YFa+DNzQHqW8O8ujLmx25JJ6Pt9bSFtQ5asZYq64cH7VNQSK6ydaKZhbNnUhBjo/quHw3G7fXk5/txbvnQ8rc29ADLbS0hTnL/yHHf/4rfpDzHGLj8xgNeygNbAHANbkMl6aiGyaKquKZfhwZ5/yMcOZYngnM4u/tx7NNH8M/I8dLQy+RDDBjCzK45ORpaZcbZ5/LZ29njhxXZBt763u8Zw+ph1/acetgefbvbtzLi6t3MXtqMVMn5iWs+/Yne8nN9HDIZEuvnzQ2m5dW7+LjrXVEdJMZ+xeye9tuahvaACu+fsOX9Xz94DpCrz7DaYAZUQhuyCfDX0+k9HCUXXvI2/wsgc3PMh9oIpus/Am4XdUJoaBa0SS2zLiKl7Z9yv6l2fypcmJCWmOJRDK62eeMfWvUqI+LGjpb/27r5NmnZuxtvT4/20t9c4gvK6zkZruqW5lQnMmLq3dxxpGTaQ/rfPJlHSeVT0RVFPTKTRzn/QLD8yWbXt3GhRl7KX39eS5rq6HazCO8sYUtu0Oc4vucGdUb0cYfyouhWahVn3GYUstbkf05+YRvc9fdb3HpMWM4qrCO1e+tY4cYxxJFwaWp7Klp5cFVmzjn6P3JzfTQGu3YHV+cxbbKloTRsxKJZHSz7xn7qGc/Nmrs7RGoMc/emp6vJ2Nf3djOxm31FEcfDgeMy+HDTTXOlIa7qq1Y8X+/vZ0p43MJhQ0MUzDn4GLCa/9NeM2T+IEzo851yOfGN3YWG/2H4a/4kNDb/4+pwEF+cI2dhv/E7xJ8q4LXvnSze2whe8JtnOd3o6kKdbofz8ELeXNNltM5dMjkfPbWB3h93R5UBS4+eRot7REUBcZFJSx7QJVEIpHsc9bAlnFyMz1k+lyOZt8WjDDW1Uzmy7dzfkYWakMGwhyLomoIIXjz40oOGJfDhGIrRvU/q3fxnw93c87R+6MgmF5oshaTrEg909z1NFWBCGQwQatjV1UzwYggUw1RuvNFwh8/i+ugo/DOX8wr62t46Y1PmDlzGtdeMI8P39jCHz8bz/VnTOCRFzZw3NGHceKRUwHI8rsJ6ybVDe1k+92oikJWhpu6piBb9jSxZU8zx84aB8AZR07mjCMn878vfM4b6ys448jJtEbz6uRlWx1HHdMbSySS0cs+Z+xb2yN4PRpul0Z2hodNuxp59t3t1NS18bWsd6CliaO8lbg2b6L1y7+hFu1Hm5aLa3sVuxUIjS1h0txjMHdvpczTyNZPg3wn+02mfV7J7HwFTYkO8gq/AXVALrRtfIM2NYvj8qoxPjZwHTgf38JvoKga8w738dqGWsoPsQaZFef5ESi8tiVMhZHP1ANKnbYfMC4HBdhZ3cqcaAz6QRPyeO/TKtZuqaUgx8tXFx6QsL9nzJ/EWx9X8tx7O2gNhC1jn+kFkDKORCJx2OeMfVt7hIIMhfDHq/hK1hbqaxvwrYkwXw2yn7sa37FX8bOVTZw+OcD8sUHM6i8xqrdSoKl43RqZNR/T/sKHnAMQHYgWcWkYM87k1dXbaBGZHDynjI1r1gKCEF6OyNyF2whRl1nGnNPPQSuY6LQnJ8PDbVfGksUV51ky0rrNtWR4Xc6bBMAhkwv48ddm838vfeF0/n7rrEMoLcjgrU8queqcGWR0MOBFeX7mTh/DOxv2Mr44k2y/m7zsqLH373OnVyKR9JF9zhq0BsKc53qT0HubmaZ5UAoy0FUvTW1hNmXOZc6UIxDed9jqnkhOYTGTDsnmtr99wKwDizhp7gR+9vf3uXC6wfubG1A0D+OooilzEtfMP53n3nqNCWOyWHjQwdz7thU+OW/6GP778wMQAr46c0qCoU9Glt9NhtdFIKQzdWJep5zYUyfm8YuvxyYCd2kq5x17AOcde0DHohzmTB3Dexur+HJPM7MOKqIwx8uYfD+TZPpgiUQSZZ8z9vu1fcw0czOeuefjLTvLWZ4HTIp+9nldrNtSy9sb9jrx6kcdNpb9SrLJyvLzzy90dKOEk2ZN5KU1u5g+Nh9VVdh/XA6HTMpnXFEmmqqQ5XdTPm0MH3xWbdU9JrXEbcV5fnZUtXQK3ewrh0zOx6Up6IYgy+/C7dL4zbePHJCyJRLJvsE+N6hqCruoyZ6GZ9YZXa5jp08oyvUxoTiT0sIMpu+Xj6ooHH5AIbphkpPhdgZH5UaTlP304jmce8wBuF0qB03IpWxqMRNLYgZ+YqrGPt+Scqbtl9fHvey8P9OiD44sf3qN6pNIJEPDPufZz7j0xxSPyaWurq3LdfzRyT/OOmoyRx9eiilic6HOPLCINz+uZP/SHCaOyaIwx5ugq9ss/a8y57PXo+F1qeRmeVNq4/6l2Wzd08R+JX1L4ZyMww8sYuP2BrJkBI5EIknCPmfsVc3lpFnuiqI8HyUFGRw5YyyKoqDFDXs+ZHI+mT4X0ydZ0s2vvjUfTetcXvxQ6YMm5OLrxcQep8zbjxPnTEDr5YxR3VF2YBH/fHUrYwvkqFmJRNKZfc7Yp8J/nXAQumHiSmLEfR4Xy686yplv1e3q2Yh//7zDejUfpaooqCmU2xuK8vz84eoFKWfylEgko4tRaRlcmprU0Ntk9HLk6UBN19dfOoZlSiQSic0+10ErkUgkks5IYy+RSCSjAGnsJRKJZBQwqMZ+27ZtXHTRRZxyyilcdNFFbN++fTCrk0gkEkkXDKqxv+WWW1iyZAmrVq1iyZIl3HzzzYNZnUQikUi6YNCMfV1dHZ9++ilnnnkmAGeeeSaffvop9fX1g1WlRCKRSLpg0EIvKysrKSkpQdOssERN0xgzZgyVlZUUFBSkVEbHJGG9oT/bDiayXb1npLZNtqt3jNR2wchtW2/b1d36IzrOPj8/s8/bFhYOXCqCgUS2q/eM1LbJdvWOkdouGLltG8h2DZqMU1paSlVVFYZhAGAYBtXV1ZSWlvawpUQikUgGmkEz9oWFhUyfPp1nnnkGgGeeeYbp06enLOFIJBKJZOBQhBBisArfunUrN954I83NzeTk5LB8+XIOOKDrSTgkEolEMjgMqrGXSCQSychAjqCVSCSSUYA09hKJRDIKkMZeIpFIRgHS2EskEskoQBp7iUQiGQWM6BG0vWXbtm3ceOONNDY2kpeXx/Lly5k8efKQt6OhoYEf/ehH7Ny5E4/Hw6RJk1i2bBkFBQVMmzaNqVOnOvPk3nHHHUybNm3I2rZo0SI8Hg9erzU5+tKlSznmmGNYt24dN998M6FQiPHjx/Pb3/6WwsLCIWvX7t27+d73vud8b2lpobW1lQ8++KDLNg8Wy5cvZ9WqVezZs4eVK1cydepUoPvrayiuvWTt6u5aA4bkeuvqeHV33obiekvWru6us57aPFB0d866Oy79PmZiH+KSSy4RTz31lBBCiKeeekpccsklw9KOhoYG8d577znff/Ob34if/OQnQgghpk6dKlpbW4elXUIIcfzxx4tNmzYlLDMMQ5x44oli9erVQgghVqxYIW688cbhaJ7DbbfdJn75y18KIZK3eTBZvXq1qKio6FRvd9fXUFx7ydrV3bUmxNBcb10dr67O21Bdb121K57466y7Ng8kXZ2z7o7LQByzfUbGGUlZNvPy8jjiiCOc77NmzaKiomLI25EqGzZswOv1Ul5eDsDixYt54YUXhq094XCYlStXcv755w9L/eXl5Z3SenR3fQ3VtZesXSPhWkvWru4Yquutp3YN13XW1Tnr7rgMxDHbZ2ScgciyORiYpskjjzzCokWLnGWXXHIJhmFw7LHHcvXVV+PxeIa0TUuXLkUIwZw5c7j++uuprKxk3Lhxzu8FBQWYpulIEkPNK6+8QklJCYceemiXbc7JyRnSNnV3fQkhRsS1l+xag+G93pKdt5FyvSW7zrpq82ARf866Oy4Dccz2Gc9+pHLrrbeSkZHBxRdfDMBrr73GE088wcMPP8yWLVtYsWLFkLbn4Ycf5t///jePP/44QgiWLVs2pPWnwuOPP57gbaVDm0cCHa81GN7rbaSft47XGQx9m5Ods8FinzH2IzHL5vLly9mxYwd/+MMfnA4yuz1ZWVlccMEFfPTRR0PaJrt+j8fDkiVL+OijjygtLU149a+vr0dV1WHx6quqqli9ejVnnXWWsyxZm4ea7q6vkXDtJbvW7HbD8FxvXZ23kXC9JbvOumvzYNDxnHV3XAbimO0zxn6kZdm866672LBhAytWrHBem5uamggGgwDous6qVauYPn36kLUpEAjQ0tICgBCC5557junTpzNjxgyCwSBr1qwB4NFHH+XUU08dsnbF8+STT7Jw4ULy8/O7bfNQ0931NdzXXrJrDYb3euvuvI2E663jddZTmweaZOesu+MyEMdsn0qENlKybG7evJkzzzyTyZMn4/P5AJgwYQJXXnklN998M4qioOs6ZWVl/PSnPyUzs++TtPSGXbt2cfXVV2MYBqZpMmXKFG666SbGjBnDRx99xC233JIQ1lVUVDQk7YrnlFNO4Wc/+xnHHntsj20eLG677TZefPFFamtryc/PJy8vj2effbbb62sorr1k7frDH/6Q9FpbsWIFa9euHZLrLVm77r333m7P21Bcb12dR+h8ncHQXWtd2YcVK1Z0e1z6e8z2KWMvkUgkkuTsMzKORCKRSLpGGnuJRCIZBUhjL5FIJKMAaewlEolkFCCNvUQikYwCpLGXSLrh3nvv5Wc/+1mftr3xxhv5/e9/P8Atkkj6xj6TG0ciGQyuuuqq4W6CRDIgSM9eIpFIRgHS2Ev2Kaqqqrj66quZP38+ixYt4sEHHwTgnnvu4ZprruG6666jrKyM8847j88//9zZ7r777uOYY46hrKyMU045hXfffdfZbunSpc56L7/8MmeccQbl5eVccsklbN261fnt008/5bzzzqOsrIzrrruOUCiU0LZXX32Vc845h/LychYvXpxS/RLJgNG39PsSycjDMAxx3nnniXvuuUeEQiGxc+dOsWjRIvHGG2+Iu+++WxxyyCHi+eefF+FwWNx///3i+OOPF+FwWGzdulUce+yxYu/evUIIIXbt2iV27NghhBDi7rvvFj/84Q+FEEJ8+eWXYubMmeKtt94S4XBY3HfffeLEE08UoVBIhEIhcdxxx4kHHnhAhMNh8fzzz4tDDjlE3HXXXUIIITZu3Cjmz58v1q1bJ3RdF0888YQ4/vjjRSgU6rZ+iWSgkJ69ZJ/hk08+ob6+nu9///t4PB4mTpzIhRdeyHPPPQfAoYceyqmnnorb7eaKK64gHA6zfv16NE0jHA6zdetWIpEIEyZMYL/99utU/nPPPcfChQtZsGABbrebb3zjGwSDQdauXcv69euJRCJcdtlluN1uTj31VA477DBn23/84x9cdNFFzJw5E03TOO+883C73axbty7l+iWS/iA7aCX7DHv27KG6utqZzQesdMPl5eWMGzeOsWPHOstVVaWkpMRZ/6c//Sn33HMPW7Zs4eijj+bGG2+kpKQkofzq6uqECSTstLRVVVVomkZJSQmKoji/x69bUVHBU089xUMPPeQsi0QiVFdXM2/evJTql0j6g/TsJfsMpaWlTJgwgTVr1jj/1q5dy1/+8hcA9u7d66xrmiZVVVVORsOzzjqLRx55hFdffRVFUbjzzjs7lT9mzJiEnOJCCGcGq+LiYqqqqhBxeQXj1y0tLeWqq65KaNv69eudqQxTqV8i6Q/S2Ev2GQ4//HAyMzO57777CAaDGIbBF198wccffwzAxo0befHFF9F1nf/93//F4/Ewc+ZMvvzyS959913C4TAejwev15swAYjNaaedxuuvv867775LJBLhb3/7Gx6Ph7KyMmbNmoXL5eLBBx8kEonw4osv8sknnzjbXnDBBTz66KOsX78eIQSBQIDXXnuN1tbWlOuXSPqDlHEk+wyapnHvvfeyfPlyTjjhBMLhMPvvvz/XXXcdACeccALPPfccP/7xj5k0aRL33HMPbrebcDjM7373O7Zu3Yrb7aasrCzpdHQHHHAAv/3tb7n11lupqqpi+vTp3Hvvvc7kE/fccw8///nP+cMf/sDChQs56aSTnG0PO+wwbr31VpYtW8aOHTvw+XzMnj2b8vLylOuXSPqDzGcvGRXcc8897NixQ8ojklGLfFeUSCSSUYA09hKJRDIKkDKORCKRjAKkZy+RSCSjAGnsJRKJZBQgjb1EIpGMAqSxl0gkklGANPYSiUQyCpDGXiKRSEYB/x9S2bqM2BzdzgAAAABJRU5ErkJggg==\n" + }, + "metadata": {} + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Start to eval !\n", + "Env:CartPole-v0, Algorithm:PPO, Device:cuda\n", + "Episode:10/200, Reward:200.000\n", + "Episode:20/200, Reward:183.000\n", + "Episode:30/200, Reward:157.000\n", + "Episode:40/200, Reward:200.000\n", + "Episode:50/200, Reward:113.000\n", + "Complete evaling！\n", + "results saved!\n" + ] + }, + { + "output_type": "display_data", + "data": { + "text/plain": "
", + "image/svg+xml": "\n\n\n \n \n \n \n 2021-05-06T01:36:55.923900\n image/svg+xml\n \n \n Matplotlib v3.4.1, https://matplotlib.org/\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n\n", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXsAAAEcCAYAAAAmzxTpAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Z1A+gAAAACXBIWXMAAAsTAAALEwEAmpwYAAB9qUlEQVR4nO2dd5QUVfr3vxU7TJ5hEhkRcECFgSEoSlZAYUBdBVnDmll/BtbV1V1RMK6oq64svurq6rq6pjUCKhgQxICggJKRJGFynk6V7vtHT9X0zHSo7ume0H0/53AO01VddW9X9dNPfe8TGEIIAYVCoVDiGrazB0ChUCiU2EONPYVCoSQA1NhTKBRKAkCNPYVCoSQA1NhTKBRKAkCNPYVCoSQA1NgnAMuXL8ftt9/eKec+//zzsWnTpk45d1fD7XZj4cKFGDVqFG655ZbOHg5++OEHnHvuuSgsLMRnn33W2cOJOUOGDMGRI0c6exidBjX2lJiyevVqjB07trOH0SX45JNPUFlZiU2bNuHpp59us3358uUYNmwYCgsLUVRUhPnz52Pr1q0AgHfffRcFBQUoLCzEyJEjMWfOHKxbt854b319PZYsWYLx48dj+PDhmD17Nt55552g43n66afx29/+Flu3bsW0adOiMseffvoJ1113HYqKijBmzBj85je/CTmOYFx++eV4++23W7w2ZMgQjBgxAoWFhTj77LPx17/+FaqqtnfoEbN7925ceOGFGD58OC688ELs3r2708YSDGrsOwFFUTp7CFEhHubRkXM4ceIE+vfvD57nA+4zc+ZMbN26Fd9++y1GjhyJm2++GXre44gRI7B161Zs2bIFv/nNb7Bo0SLU1dVBkiT87ne/w4kTJ/DGG29gy5YtuOOOO/C3v/0NL730UtDxDBo0KKK5+Pvctm7diiuvvBKjR4/G2rVrsWnTJixduhQbNmwI+/iEEGiaFnD7Bx98gK1bt+Lll1/GqlWr8NZbb4V9jmggSRJuvPFGFBcXY/PmzZg7dy5uvPFGSJLUKeMJBjX2rXj++ecxbdo0FBYW4rzzzsOnn34KwHtRi4qKsG/fPmPf6upqnH766aiqqgIArFu3DnPmzDG8sj179hj7TpkyBc8//zxmz56NESNGQFGUgOcCAFVV8cgjj2Ds2LGYMmUKXn31VQwZMsT4kjU0NOAvf/kLzjrrLJx99tl48sknTXs327Ztw/z581FUVITi4uIWMss777yDmTNnorCwEFOnTsUbb7xhbNu0aRMmTJiA559/HuPHj8ef//xnLF++HLfeeiv+9Kc/obCwEOeffz5+/vnnFvP+5ptvACDkvjt37sTcuXNRWFiIW265BYsWLcKTTz4ZcB5vvfWWMdbzzjsPO3fuBND2cf2uu+4yjuNvDjNnzmzhJSuKgnHjxhnHC/Z5tebAgQO4/PLLUVRUhPPPPx+ff/45AK8X/cwzz+Djjz9GYWFhG2+1NYIg4IILLkBFRQVqampabGNZFhdddBHcbjd+/fVXfPDBBygpKcHf//539OnTB4IgYMKECVi8eDGefvppNDY2tjn+tGnTcPToUSxcuBCFhYWQJAllZWVYuHAhxowZg3POOaeFAV2+fDluueUW3H777Rg5ciTee++9Nsd89NFHMXfuXFx//fXIzMwEwzA49dRT8fe//x0AUFdXhxtuuAHjxo3D6NGjccMNN6C0tNR4/+WXX44nn3wS8+fPx/Dhw3HHHXdgy5YtuP/++1FYWIj777+/zTkHDhyIUaNGYf/+/QC898Q555yDMWPGYOHChSgrK/P7+UqShGXLlmHSpEk488wzce+998LtdvvdN9j98f3330NRFFx55ZUQRRFXXHEFCCH47rvv/B6rUyGUFnz00UektLSUqKpKVq9eTYYPH07KysoIIYTcdddd5IknnjD2ffXVV8nVV19NCCFk586dZNy4cWTbtm1EURTy7rvvksmTJxOPx0MIIWTy5MmkuLiYnDhxgrhcrpDn+u9//0tmzpxJSkpKSG1tLbnyyivJ4MGDiSzLhBBCbrzxRnLPPfcQh8NBKisryUUXXURef/11v3N6+umnyR//+EdCCCGlpaVkzJgx5MsvvySqqpKNGzeSMWPGkKqqKkIIIevWrSNHjhwhmqaRTZs2kdNPP53s2LGDEELId999RwoKCsijjz5KPB4Pcblc5Omnnyannnoq+fLLL4miKOTxxx8nF198sXHuyZMnk6+//toYR6B9PR4PmTRpEnn55ZeJJElkzZo1ZNiwYS0+79bX6ayzziLbt28nmqaRw4cPk2PHjhFCCBk8eDA5fPiwse+dd95pHMffHJYvX05uu+02Y/9169aRGTNmmPq8fJEkiUybNo38v//3/4jH4yHffPMNGTFiBDlw4ECb6xDqOnk8HvLII4+QiRMnEkIIeeedd8j8+fMJIYTIskxefvllMmLECFJfX08WLVpE/vSnP7U5nizLpKCggGzYsMHv+XyvDSGELFiwgCxZsoS43W6ya9cuMnbsWPLNN98YYxs6dCj59NNPiaqqxj2s43Q6ySmnnEK+/fbbgPOrrq4mn3zyCXE6naShoYHcfPPN5Pe//72x/bLLLiMTJ04k+/btI7IsE0mSyGWXXUbeeuutFsfxvb779+8nZ555JnnrrbfIN998Q8aMGUN27NhBPB4Puf/++8mCBQv8vu+hhx4iN9xwA6mpqSENDQ3khhtuII8//rjfcQe7P1566SVyzTXXtNj/+uuvJy+++GLAz6GzoJ59K2bOnInc3FywLIvzzjsP/fr1w08//QQAmD17NlavXm3su3LlSsyePRsA8Oabb2LevHkYPnw4OI7DBRdcAEEQsG3bNmP/yy+/HPn5+bBarSHP9fHHH+OKK65AXl4e0tLScP311xvHqaysxPr16/GXv/wFdrsdWVlZ+N3vftdibIH44IMPMGHCBEycOBEsy2L8+PE49dRTsX79egDApEmT0LdvXzAMgzFjxmD8+PHYsmWL8X6WZXHLLbdAFEVjHqNGjcLEiRPBcRzmzJnT4ommNYH23b59OxRFwRVXXAFBEHDuuefitNNOC3ic//3vf7j22mtx+umng2EY9OvXD7169Qo5f39zmD17Nr744gu4XC4A3ut6/vnnm/q8fNm+fTucTieuv/56iKKIM844A5MnTzZ1XXQ++eQTFBUVYeLEidi5cyf+8Y9/tDh+UVERxo8fj9WrV2PFihVISUlBTU0NsrOz2xyL53lkZGS0eTLwR0lJCX788UfcfvvtsFgsKCgowMUXX4wPPvjA2GfEiBGYNm0aWJY1rr1OfX09NE3zOw6djIwMTJ8+HTabDcnJyfj973+PzZs3t9jnggsuwKBBg8DzPARBCHisCy64AKNHj8bChQvxm9/8BhdddBFWrlyJiy66CMOGDYMoirjtttuwbds2HDt2rMV7CSF466238Je//AXp6elITk7GDTfcEPA6Bbs/HA4HUlJSWuyfnJwMh8MRcOydRWDxMEF5//338dJLL+H48eMAAKfTaXxZxo4dC7fbje3btyMrKwt79uwxFrZOnDiB999/H6+++qpxLFmWUV5ebvydn59v+lzl5eUt9s/LyzP+f+LECSiKgrPOOst4TdO0Nsf3x4kTJ/DJJ5+0eSzVF1HXr1+PFStW4PDhw9A0DW63G4MHDzb2zcjIgMViaXHMHj16GP+3Wq3weDxQFMWvNh1o3/LycuTm5oJhGGN7sPmUlJSgb9++Iefrj9Zz6NevHwYOHIh169Zh8uTJ+OKLL/D+++8DCP15+VJeXo68vDywbLMP1bNnz4BSgj9mzJiBxx9/3O+24cOH4/XXX/c7n4qKijavK4qCmpoaZGRkhDxveXk50tLSkJyc3GLsO3bsMP72vQdbk5qaCpZlUVFRgYEDB/rdx+Vy4a9//Su++uor1NXVAfAaS1VVwXEcgODX3Jf33nsP/fr1azOHYcOGGX8nJSUhPT0dZWVl6N27t/F6dXU1XC4XLrzwQuM14rNGcO211+KHH34AANx3330oLi4OeH8kJSW1kckcDgeSkpJMzaMjocbeh+PHj2Px4sV4+eWXUVhYaHifOhzHYcaMGVi1ahV69OiBSZMmGV+O/Px8LFy4EL///e8DHt/XkIU6V3Z2dgs90/f/eXl5EEUR3333XdDFPn/k5+djzpw5ePDBB9tskyQJt9xyC5YtW4apU6dCEATceOONxgJh6zlEk+zsbJSVlYEQYpyjpKQEffr0CTiPX3/91e82m81meGEAUFFRgdzcXONvf3OYNWsWVq1aBU3TcPLJJxuGJNjn1ZqcnByUlpZC0zTD4JeUlKB///4h39sezjzzTDzxxBNwOp2w2+3G62vXroUoihgxYkTIY+Tk5KCurg6NjY3GPV1SUhLyc9Ox2WwYMWIE1q5di3Hjxvnd51//+hcOHTqEt956C9nZ2di9ezfmzp0btfsrJyfHcJwAr/NUW1vbYg6A98fRarVi9erVbbYBwAsvvNDmtUD3x8knn4x//etfLe7bvXv3YsGCBRHPI1ZQGccHl8sFhmGQmZkJwLtYqS/86MyePRsff/wxVq5ciVmzZhmvX3zxxXjjjTewfft2EELgdDrx5Zdf+l0cM3OumTNn4pVXXkFZWRnq6+vxz3/+09iWk5OD8ePH45FHHkFjYyM0TcOvv/6K77//PuQci4uLsW7dOnz11VdQVRUejwebNm1CaWkpJEmCJEnIzMwEz/NYv349vv76a/MfYDsYMWIEOI7Dq6++CkVR8Nlnn7VYvG3Nb37zG/zrX//Cjh07QAjBkSNHjC/6KaecglWrVkFVVWzYsKGNVOCP8847D19//TVef/31Ftc12OfVmtNPPx1WqxUvvPACZFnGpk2b8MUXX+C8886L4BMxz5w5c5CXl4dbb70Vx44dgyzL+Oqrr/Dggw/ipptuaiMz+CM/Px+FhYV44okn4PF4sGfPHvzvf/9DcXGx6XHccccdeO+99/DCCy8YT6h79uzBH/7wBwBej9disSA1NRW1tbUtJKpA9OjRA0ePHjV1/lmzZuHdd9/F7t27IUkSnnjiCZx++uktvHrAK+NdfPHFePjhh43girKyMnz11VcBjx3o/hgzZgw4jsMrr7wCSZKMJ/tAP3idCTX2Ppx88sm4+uqrMX/+fJx55pnYt28fRo4c2WKf4cOHw2azoby8HBMmTDBeP+200/DAAw/g/vvvx+jRo3Huuefi3Xffjfhcl1xyCcaPH4/i4mLMnTsXEydOBM/zxuPuo48+ClmWcd5552H06NG45ZZb/D7KtyY/Px/PPPMMnnvuOZxxxhmYOHEiXnzxRWiahuTkZCxevBiLFi3C6NGjsWrVKkyZMiXcjzEiRFHE8uXL8b///Q+jR4/Ghx9+iEmTJkEURb/7z5w5EwsXLsQf//hHjBw5Ev/3f/9nSAN333031q1bh6KiIqxcudJUDHlOTo4R2uhrnIN9Xv7m8Oyzz2LDhg0YN24c7rvvPjz66KMBZY1oIYoiXnrpJeTn5+OSSy7BqFGj8Mgjj+APf/gDrr32WtPHeeKJJ3D8+HGcffbZuOmmm3DzzTfjzDPPNP3+kSNH4t///je+++47TJs2DWPGjME999yDiRMnAgCuvPJKeDwejBs3DvPmzcPZZ58d8phXXHEF1qxZg9GjR4d8ujrzzDNx66234uabb8ZZZ52Fo0ePBozmuuOOO9CvXz9ccsklGDlyJH73u9/h0KFDAY8d6P4QRRErVqzABx98gKKiIrzzzjtYsWJFwPu2M2EIoc1LugPr16/H0qVLW2jH8c7FF1+M+fPn46KLLursoVAo3R7q2XdR3G431q9fD0VRUFZWhhUrVkQty7Gr8v3336OiogKKouC9997D3r17TXl/FAolNHSBtotCCMHTTz+NRYsWwWq1YtKkSbj11ls7e1gx5dChQ1i0aBFcLhd69+6Np59+Gjk5OZ09LAolLqAyDoVCoSQAVMahUCiUBIAaewqFQkkAqLGnUCiUBKBLL9DW1DigaeEvKWRlJaOqyn8yUzxD551Y0HknFmbmzbIMMjL8l2ro0sZe00hExl5/byJC551Y0HknFu2ZN5VxKBQKJQGgxp5CoVASAGrsKRQKJQEIaexrampw3XXXYfr06Zg9ezZuuukmVFdXA/C2aysuLsb06dNx9dVXGxXkQm2jUCgUSscS0tgzDINrr70Wa9aswcqVK9GnTx88/vjj0DQNd9xxB+69916sWbMGRUVFRtOFYNsoFAqF0vGENPbp6ektuvKMGDECJ06cwI4dO2CxWFBUVAQAmD9/Pj755BMACLot1miaBlVVoWpt/wXrVt9ZaISE/a870RHz07TwzxHrz7Yzr19UP9sYf4YkkusUwfWO1ry7c3WZsEIvNU3D66+/jilTpqCkpAQ9e/Y0tmVmZkLTNNTW1gbdlp6eHrXB+2PHf5/EAKf/phdlXB4GXv3XdnXDIYTgwVe24JyiPhg3LHCbNjPsP1aLR/+7FWqY4VQLpg3CtCL/HZxa892uUny6+SgWX1EUsy5TgfhxXwWeeW9HWF82BsC1s4bijFPNfbYrvzmM9zYcjHCEbblyxhBMHGGul20gDpyow7LXfoSitp03xzK47ZLhKOif2a5zBOJ4pQMPvLwZktLWsWEZBv93wakoHBy4T6wvH2w8hA82Bq7x7g+BZ7H4iiL0yUkOvTOA5z7cie93l4fesZ1MKuyFy84dDDbEd0AjBK9/th9f/HAM/u5am4XDQ9eNQ3qyxc/Wtjz19nac0jcDM8ZG1kIzmoRl7B944AHY7XZcdtll+PTTT2M1JoOsLHM3jC89z5iBY7vbVkpsLD+OU5Q9SHEega1/4EbWoXC6ZRwqaUBlg4Ts7NAdgILx/d4KqBrBb6YMgihwpt7zwYYDKKtzBzx369dLaw/jUEkD0jOSTJ8jWtRuPwGNEFx67hDTPzQffXMIO47UoHjyIFP7bz9Qhd45yZhQ2Dv0ziF478tfUFob+LM1y9aD1VBUgrkTB8JubW6arWkEb3y6F+UNEia08xw6rcd6sKwRkqJh5hn9kZHasin4G5/uRVm9x/T8fi1vRI90G84d2y/0zgA8koJ31v2CsnoPRg4z10t2z6+1GNI3A6MK2rYHjBalVQ58seUoOJ7FzZcUgmP934uaRvDMO9vx+Q/HMGlUb/Ts0dL+lFc78dnmX+HW2n7ugdh7tBYWC9/ue0qnPccxbeyXLVuGI0eO4NlnnwXLssjPz8eJEyeM7dXV1WBZFunp6UG3hUNVVWPYSQQ9Bg5DwbhxqKhoaPH6W5/uQt+DB1Hx7WrYkvqHdUxfKmu9vU3r6t1tzhEuJRXebLjpRb3AseYCozb8eBTVtS6/587OTmnzelW1EwBw9EQtUu0d2z2nptYFjmVwzkjznvKhY7XYcaAS5eX1IX8gXB4Fh07UYd60IZhW2DPovmbY8ONRlFc52n1dy5qu6zkje8FmafkVe3/9LzhWWt/ucwD+r3dltQMAcOawXPTq0TKT8qOvD+JEWYPpc5fXONEnO8n0Z6sRgpVfHcQvR6pRcXJWyP0bXTLqHRJmjOkb1vXzN+9QpFh5fLDxENwuGVedVwC2lcHXCMErn+zBhu0lOP+Mfrhwwklt7r9fyxrw2eZfcbykDnmpoT17WVHhkVSUVbb/ngLMzZtlmYBOsikL88QTT2DHjh0t2m2deuqpcLvd2LJlCwDgjTfewIwZM0Ju6yx4ixXfewZCOfQjNGdtxMdpdMsAAFlV2z2mBqeEJCtv2tADgN0qwOlWTO/v9Hj39UjtH2+4eCQVljCfJgb3SUODU0ZZjSvkvgeO14EQYNhJ0ZFEUuwiGpxyu4/j9MhgGMAqtp17RooFtQ2edp8jEJLslW9Evu09lZ5sQW2j+XPXNnhMyxWAVybKybCjrMnBCEVZjXe/vEx7iD3bz5yzBmDuWQPw9Y5S/Ouj3S2cSI0Q/Ptjr6GfdWZ/v4YeAJJt3qe0Rpe5e6TR5f3uVcfweodDSM9+//79eO6559C/f3/Mnz8fANC7d2+sWLECjz76KJYsWQKPx4NevXrhscceA+Bt6BtoW2dhEVhscA/BJOtuyHs2wDLSfCNlXxqbjIE/TTTsY7lkJIfpbdstPCrr3Kb3dzT9MLg7w9jLKkQhvFSOQb3TAQD7jtaGNAL7jtWCZRgM6ZeJxvrQPw6hSLELOFzafg/M4VZgt/B+DUZGigU1YRjccJEV73UOZOzN3juyosHhVpCeHN79mZdpw9EKh6l99R+F3ExbWOeIlOKzBgAM8P5Xh0AIcM35BQADvPzxHmz8qQSzz+yPuWcPCPhEGa6xb3BKxv4eOXzHJ9qENPaDBg3C3r17/W4bOXIkVq5cGfa2zkAUOFRoqUBeAeTdX0IcMQtMGB61jn6h5SgY+wanjBS7EHpHH5KsPH4tN+99OpueRNyS+aeBaCEpWtg3eH6WHck2AfuP1WLC8OCP9vuP1qFvbjJsFh7RKIuVaheNL2h7cLoVJFn9X9eMZAt2Halp9zkCoTshAt/2c09PseCX43WmjlPX9IMUjmcPALmZdmzdXwlF1cBzwb9fpdUusAyD7PSOMfYAUDx+ABgA7311CAABx7LY+HMJisf3x5yzAht6wGtDRIE1bewdPvvVNniQ2wFPMMFImAxa3ejIJ50N4qiG+uv2iI7TEFVjLyHFFp6xt1n5uJZxGIbBoN5p2H80uFGSFQ0HS+oxuE96O0bYkpQkES6PanjHkeJ0K7BZ/ftR6SkW1DVKMSvkpRt7f09U6UkiGl0yFDX0vVvb6P3RSwvT2Odl2qFqBFUmniDKqp3okW4N+aMQbWaPH4ALJ5yEb3eWGYZ+7tn+pZvWJNsE4+k+FA0+xr663vzTeKzo0lUvo4l+83tyT4VoT4e0+wvw/QvDPo4jmsbeJeOknqlhvSfJKsAtqVA1zZTW3+kyjh/dOhSDeqdj6/5K1DYG1oyPlDVAVjQM6p3W3mEapDY9ZTU4ZWSmRv7I7XTLSApg7DNSLNAIQb1TCttrNoOsqGAY+I04SU/xnq+uUUJWmrXNdl9qDc8+XBnH672WVjtDerKl1c4O0ev9MevM/kixC9A0gskjzUdyJduEiDz7rqDbJ5xn71EB4ZSJUI/ugFYffnyv/msttdP7I4Sg0SkjJVzNvsmIuDyhz69qmuHRuzpFxlFh8aMdh0L31vcfC+zd7z9aC6BZ448GerRSfTulHKdHaRFy6UtGk4GvidGXX5I1iDzn10vVDbeZRVrD2KeEL+MAXkMeDI0QlNV0nrEHgIkjeoVl6IHwjH0jNfadgx5jLskahIJJAMNA3v1l2MeJlmfv8qhQNWIs+pjF3hTK53CHvuF85Z5O8ewlLaLY/r65yRAFFvuaDLo/9h2tRW6mHalJ0QsnTWk6Vr2jfRE5+gKtP3TjGauIHFnRIAT4gdWfJMwZewkcy4R9fybbBCTbhJARObUNHkiy1uk6driEY+wbXDKsIodkm4CaLiDjJIyxNzx7WQWblAG+XyHkvV+BqOF9saO1QNvg8nqP4S/Qevc3o9v77tMZmr0kq7BEIOPwHIuBPdOw/1it3+0aIfjleB0GR1HCAXxlnMg9e0JISBkHQMwiciQlcARUmmHsQ8+vttGDtGQxZMapP3IzbSE9e/3HIC+j4xZno0G4Mk6yTUBmioV69h2JHoomyV6jJwydDOJugHJwc1jH0Rdn2m3sm46TbItMxjFl7D2d7NnLKkQ/USFmGNQ7DUfLG+HytJ3niUoHHG4lqhIOAENSa4+MIysaFJUY16k1qXavAY2VjOP17P1/5il2ASzDmPLs64Ksl4QiL8MeMk+itGl7d/TsnW7F1AJ7Q5Oxz0ixoLqeGvsOQ/cwPU3Gnus1FExqLuRd68I6jp5U1d44e/1HI1zPXjciZmQc3306J/Qy8tjiQX3SQYg3cao1upY/uE90PXuryEHg2XYlVukL4oE0e5ZlkJYsxliz9/+1Zhnvuc3KOJEa+9xMO2oaPEHvubJqJ0SBDXtNoLNJtgkgMPn9c8lItgvITLWipoHKOB2G7mHqGYYMw0IcOglq2X6o1UdNH6fZs2+fp6xLBfEq4xBC4JE0WMTIbrGBPVPBMgz2+ZFy9h+tRVqSGPX4bIZhkGIX0OCI3LPXn6YCyThAU2JVrDx7NbCxB/QsWnMyTriRODr6omt5EO++tNqJ3Ax7RDJRZxJOYlWDs9mzd7gVw9HsLBLG2Ptq9jrC4LMBjjft3XtkFZKigWHaL+PoN0tKuDJO08Kf04+80Rrd2CdZ+Q6XcRTVWxI2Us/eKvLom5vsN95+/7FaDOqTHpMqnil2EfXt8Oz1JLZAC7RAU8mEGGn2sqwGXKAFvBE5oc4tySocbiXsGHudPBMROZ0Zdtkeku3mjb3DLSPZKiCzqY5OZ8faJ4yx1xetJB9jz1iTwZ80FvL+b6A1VIQ8hh6Jk5Ykeo1ZOxJjGpwyRJ4NewFTFFhwLBOWjJOZau1wGUf/UY1Uswe8IZgHS+pb/LBW1blRVe+J+uKsTqpdbJdmH0rGAbzhlzGTcYJo9oDXs68L4dnXNT3ZROrZ5zQtugaKyFFUDZW17m6h1xNNgVK6D54t78HxwYPos/5enMyXhkysUlQNLo/qlXFSvDkNsbrmZkkYY89zXiPpkVt65JbCWQDDwrn6cWjO4Fmb+q95RtPFk01kIgaiwSkZXkI4MAyDJCsPl8kFWp7zShMd7dnrP6qRROPoDOqdBlnRcKSsuV6NHqET7cVZnVS70K5oHJfbnIzjllS/i8/tRVZCyTjeLNpgT6a6558RoWcvChyyUi0BPfuKWhc0QpDXQTVxwkWrK4W041M4P3kSjf++Ca4PH4a09UNA0wDRjutSvoBaEbzOv+4YJtsEZBiefeca+4TJoAW8N6HUSjdj0/Nhn3kbnKsfhevjx2GfdRcYS5Lf9+vGPjPFgkMl3i9WpDJFg0sOW8LRsVkFw4MMhtPtTe6xirzhrXUUhmcfZiE0X3SDvv9oLU7u5fXk9x2rg1XkTDfHCJeUJBH1DhmEkIhkIv1pKlC5BMAn1r7R06YEcnuRFBVCkM9cX3Sta/SgR4A1D13Tb0+Gb26mHaXV/jX7suquFYlDNAVq6X4ov26HcmQbSF0pAIBJzYFw8hngeg8D37MAjCUJrtpK1Ly+FCfteRnq0N7gMvzXb2r0MfaZerhtJy/SJpSxtwis30USLvdk2M69Ba5PnoTzkydhP+8OMELbG73Zs/dua49u3+CUI/LsAa/X6DSZVGW38LCKHNwmMm6jib4QbmmHjJOaJCI30459R2sxc5y3gYZu+FvXI48WqXYRiqrBLakRGWJ9nSSoZu+TRZuf5d+xiBQphGfvG2sf0Ng3yQ1pEco4gNeQb9pZ5vdHU/f4czM6x9gTokGrK4VWdgDKsZ1Qjv4ESE6A5cH1PAX8sKng+w4Hm9q2CZI1LQvPOc/BbeKncH30GOzFd4NN6dFmP19jL/DexKrOjrVPKGMvClzAkEm+96mwTlkI9+fPwPXZP2A791YwXMuPp9ElQ4CCk3AEdkZpV8mERpcUcWlXu4U3mUHrTe6xilznafbtkHEAYHDvNPy4rwIaIXC6FRyvdGDM0Nh1NdKjo+qdUmTG3qPAInBBi3sZiVUx+PLLcijNPnTJhFqHJ6LsWV/yMuxwehQ0uOQ2TXPKapxGpm1HQCQX1PIDUMsOQC3/BWr5QcDjLcPM2FLB9x8Fvt9w8L2GgRGDfycZhoHHkoWvsi7GObVvwvnRY7DP/gtYe8s1pEaXDDvjRmbFj1DtA5CZGrt1GrMklLG3CFzQEEThpNEg0u/g2fAS3Oueh3XKQjAs633MO7YLffZ/hgczdsF6SEGx7WTIytkRj6XBGbmMY7fyKK8NXb/d4VaQbPfKOB0d9mVo9u2s4T2odzq++qkEJyodqKz1PgbHanEWgFF+ocEhIzcj/Pc73HLAhCodXxkn2oTy7M2cu7bBG2PfnmgnXaIpq3a2MfalVbGPxCGK5JVlfvkWyq8/AZoCgAGb0QvCgCJwOQPB5p4MNj0PDBOe1JhsE3BcTYV9xh/gXP0YXB//DfZZd4KxJIHIbiiHf0TOtvV4MH0fuK0Ezq3AXPEUrK0bHZvJmiThjH0ob1w8ZSIgOeH57k24WQ6MaINycDOIuwGZrBXblJNwai6DwqqDaHS7AYSvHcuKVyaIXMYx163K6VGQm2mHReSgqMRUjfFo4YmSsdcTp/Yfq0NlU5vDAfnhVQoNB90wRbpI661lH/xrZRE42C181D09jXivcbDQy2SbAI5lgsbatyfGXicvqzn8svViemmNE6cNCN22MFyIpkEt2QN5/7dQDm0BZBcYWxqEYVPB9zkdXM5JIT13M+glE7i802E792a41jwF58dPgE3OgnJkG6BKsAmp+NI9FDMungvm+E/ov/UjXEN+gedHB8TTZ4DhO7ZFKJBgxl4MoNm32e/0mSBuB6RtqwBOBN9vBPiTx+HN7Rz2nXCg30k8cqv3wnV8G9BnWtjjMGLsIzT29qaa9qEWEb0LtLzRHs8tqUi2dayxb88CLQBkp9uQlixi/9FaVNS5MCA/NaaN031lnEhwBimC5kssEqtko5Z94M/HTBZtbaMHPdu5ltAj1QqOZYzFWB2XR0FdY+QSpj+0+grIezdA3rcRxFEDCFbwA0ZBOPlMcD0LImpSFIxkm4DjlV4ZiO9zGqyTb4D78/8HUl8OYchZ4AeOxUe7GKzbdgJzew4Ceg7CRudJEH9+DyO2vAt57wZYxs0H339UTHJFApFQxt4icKZT4cXRF4HvNwJsRi/DG6jftA0pdgEkeyCq1CTYj30PIHxjb2TPRqhZ2q08NEKCLiKSJo1bX6AFALdH6TCd1Figbadh9jYzScfuX2vQ6JRx7ug+0RheQJrr40SWWOVwK+gRolY84JVTYmXsg3n2gB5rH/jcdY0ShvZrX19flmWQk2FrE2uvZ9W2V8YhqgLlyFbIe9ZDPbYTYACu92kQxl0Kvt+ImHrOyXYRja5a429h4BhwuSeDsaeCYb3fx8YfdrX4rtl75OOfjZPwyJx0JO14F+5P/wH+5HGwTVkYs3G2JqGMvShwprVrhmHA5Z7c4rXGploXgsBjszQQ06t/htZYDTY5vC9Gg+HZR6jZW/Sa9kpAY++WVGiEIKkp9BIA3B2o20dLxgG8Gv2WPd7eA4Oi2JnKHwLPwmbhIi6Z4PLIsFtDS3sZyRYcq4hGM8VmJCORLbSxD5Tw5JFVOD0K0lPabyzzMu1tYu2NSJwIjb1WX4Gqn9+HY9sXIK56MEmZEEfNgTDkbLDJ0ZeG/JFs49HokqERYpR7aG0DGpuKoOno4ZcVtv7Ivug+SJvfgbT9IyiDxoPvc1qHjDuhjL1FYNvE2YdDo0tGfpYdAs/he89AzLD9BPmXb2AZMSus40RaF0dHr4/jcCvIDCBfGyGArWScjiJaMg7QMoFKj7ePJSntyKJ1NElnochIsaDeIZnuOGYGQ8YJEe6alixi76/+++DqHn9aUvsLlOVm2vHzwWpoGjFCZcuqnWAA5IRR14gQArVsP+Sf1kA58iMABny/ERBOmQiu92lRl2lCkWwVQIjX2QrUa7i1sc9I9T7tVde7wbCZEIsugHxwMzzfvQmu17AOmUNCGXuR59pk0IZDo0tGkk2AwLOo0lLgSOkPdu9GiMPPD0t7ay5vHLmMAyBorL1eO6eFjNOB4ZeSrIHnmKgYsj45ybCKHLLSrB0iQ3kbj4cv46iad+E9kAHwJSPFAkK8kklmamjZxwxSGDKOw61AktU2+r6RUBUlz15RNVTXu42Y/tJqJzJTrabWXYiqQDn4PaQdn0KrOARYkiAOPw95Z89BjafjFzh1fOvjBDP2vq0fjdyKpixahhNgGXsJ3J+tgLx3A8SCSbEdNBLM2FvEthm0ZtETbVJsgvGYXJE1AkmH34dWcRBczkDTx2p0ymAYmDIK/jBT017/IfDG2Xv378jKlx458vLGrWFZBnPPPsloLhJrUuyCqdDW1uitIs0s0Kb7NDGJnrE39zSlR9rUOaQ2lUObe89GwbNvqpFTWuNsYexDlUkgshvyri8g/bwWxFkLNi0PlrOugDBoPBjBAj41BahoCHqMWGJUvnQGDs9t7dkLPItUe8vEKn5AEbjcQZC2vAth4NioRAoFI6TbtWzZMkyZMgVDhgzBvn37jNfXrVuHuXPnYs6cOSguLsbatWuNbYcOHcK8efMwffp0zJs3D4cPH47J4MNF5Fmomjc8LVxaZsR5P7by1GEAJ0De93VYx2po8ggizQK1+8g4gXD6FOSydJKME82omXNH98G4YXlRO14wUpPEiDR7PdHNlIyjZ7JGcZFWlnXPPvjnnhGkPWE0SiXo5DVF9OgROcToO+s/0odITnh+/BCO/94Oz6a3wGb0hG3GbbBf8jDEoVP8ZrV3BnrDoUCVLzXNGxzR+ik0I9WKap+SCQzDwHLGfBBXPaTtH8VuwE2EvCunTp2KK664Ar/97W+N1wgh+NOf/oTXXnsNgwcPxp49e3DppZdi2rRpYFkWS5YswYIFCzBnzhx88MEHuPfee/HKK6/EdCJmsPj0oQ033lyvcpfk49m7iQC+/yjIBzbBcsalYDhznmejU4pIryeKB/K+r2FL85YOCFbm2OGj2XeGsfcnEXQXUuwiGlxyC63ZDL7rJKGIRRatZGj2we/tYO0Jaxs94Dk2ZK6AGVLtAmwWzliUrXfKcHnUNmGXxN0IacenkHZ8CkhOcH2HwzKyOKyn5Y4k2dYUcRPA2DvcMgjayrSZKZY2T4xczkDwA8dC+mkNhILJYQd7hEPIK1pUVOT3dZZl0dDgfZRqaGhATk4OWJZFVVUVdu3ahZdeegkAMGvWLDzwwAOorq5GZmbsJmIG0aemvZkvpC/N9ecF44dCVjQIg8dDOfAdlCPbIJxkLkOuwSmHFYlDNAXyng2QfvwQxFkLiHb05KYG1+x9vEzjx6mDNXtLFBZnO4NUu3cBrtHdNtU/GM39A0L/kCfbvclN0exFqzfUCa3ZN5VM8PNDoydURSP+m2EY5GbYjcgfo+9sUyQOkZyQtn/sNfKyG3z/URBHzgbXo3+7zx1LQnn2viqAL5kpVuz5tbbN/pYxv4Fy+Ad4Nr8D2+TrojtYHyL6+WYYBk899RRuvPFG2O12OBwOPP/88wCAkpIS5ObmguO8hpXjOOTk5KCkpKTTjb3h2UdQ00a/gEk2AQzDQOBZSIrmXUm3p0Pet9G8sXfJyDcRekaIBuXAJni2vAdSXw4udxCEM38Lz7f/xY0pn+H7hl4ATvL7XqdHAQPAZuHBwJtM09EyTrQ0+45CT1IzSiY4wzT2HvOePcsw3q5RsfDsQ3zuRhato+2569rRjtAfeZl2/NLUWtIIu0wTIP28BtKPK0E8jeBPGuM18pmxzaGIFjYLB45lQht7e2sZxwKXR2kTMs2mZEM89VxI2z+Ceto5Mfuxi8jYK4qC5557Ds888wxGjRqFH374AYsWLcLq1aujOrisrMjL2GZnp7R5rUeW90kkKdnqd3tQfqkCAPTvk4GsNBtEgQMvcMjJTUPV8Emo++5DZNhU8MnpIQ/ldCvIzkoKOAZCCJy//ICaL/8LqfwIxJx+yJzxF9hOHgmGYSANHIzGZ+/EGSX/RYalEHxqc9U9/Zgaw8BuE5CulEOqPgGblQfDseHPO0I0AqQkWTrsfO05D9FUNGz7HDUb3oSl1yD0HXIhAIAT+LCOyzbdI316pgesKOlLTqYdDo/arrH7vtdirQYA5OWkIjsj+Pmz0qxwy1qbcze4ZPTLS414TEpdBVhrMliL9/wn9U7Hpt1lSEu3o8ElYbT1MJI//wSe2nLYBpyOzMmXw5Lv32EJRkfdV4FIsYtQiP9xHCjz5k/0yU9vsb1fr3QAAOPnvtKmzcev+zdC++Ft5P72voBPVu2Zd0TGfvfu3SgvL8eoUaMAAKNGjYLNZsOBAwfQq1cvlJWVQVVVcBwHVVVRXl6O/Pz8sM9TVdUYUTeo7OwUVPhZrfe4vBplaXkDksOUGErLvcfzOD2okLxNQeob3KioaIDaezRA3kfZps8gnj496HE0QlDvkMAzxO8Y1ZoT8HzzGtTjO8Gk5sA6ZSH4gWPgYFg4KvUknDS8y8/GPPUDHHtlCWzFfwFrS20xb0dlBS6xfIXjL70IABghTkNNXZbfc8aCRpeEJCvfIecLdL3NoBzfBc+3r0OrPgo2ewCcB7bDfmgHxllG4NfjQ5GXZt7LLWu6Pm6HBxVyaMksycrjaFlDxGNvPe/qGm8Kf0O9E1CCnz/FJqC00tHm3FV1Lgzpkx72mLS6Mri/fR3qr9sAeKtJsml5GKylYJpFw/5PFQzd+Rkm2MuhcX1hO+928L1PRT0QdmRNe653tLBbeVTWOP2O40RpPQBA9kgttntblQO/HKmC1Y/5EQqL4f76VZRu+Qp8/8I2283Mm2WZgE5yRMY+Ly8PpaWlOHjwIE466SQcOHAAVVVV6Nu3L9LT01FQUIBVq1Zhzpw5WLVqFQoKCjpdwgFaavbh0uiSYRE4I9JB4FgjiYXL6AU2ewDk/RtDGnunW4FGSJuKl0RywvPDB5B3fAYIIixnLIAwbIqRft1mPLZ8rJRn4aLGlXB99DfYZ98JIAVElSHv+BTnlb8PjlUgDj8PyvGdOL9yI1a7B4U970iRZLVLa/ZaXRk8m96EcvhHMCk9YJ32f+AHFIHUl6Pxyxdxqfwt6raXQ+u9EGxqtqljOt0KOJbxG/pINA3y7i8g/fQJuKy+EE6ZiIwkK35q9ETcKKU1ZpOqAG+0TUmr7FaPpMLlUcMqgkZkN6StqyD99AnA8RBHzgF4AaSuHFp9GdLq9mOWvR7YuhUSkrEheQbOu/CSsCtNdjWSrXzA1oSNbv+avbEoH6BjlVAwCfKOz+DZ/I5fY99eQhr7Bx98EGvXrkVlZSWuuuoqpKenY/Xq1Vi6dCluvfVW4yZ9+OGHkZ6eDgBYunQp7rrrLjzzzDNITU3FsmXLoj7wSGiOxonM2Our8EDb2vjC4PHwfP0q1KpfwWX1DXic1tmzhGhQ9m6EZ/P/QFwNEE6ZAHH0RWBtwSs72q0CDjhzm6ru/R2uT56CY/wcOD59BaS+DMfZ/vjONgHXj50CvvoY5P/di6L6zwCMDXvukeCRI+/iFUuI7G76UV0LcALE0b+BeNq5Ri0VJi0XSbPuxMtPP48L2a1w/O9uWMZcDGHY1JAGytlU3ri14VbLD8K98RVolYfB5pwEtewXKId/xDlCGjiuP5zVBUjKan+NfiOpysSPbHqyBbuPtMyi1TV8M5o9IcS7nrTpTRBHDfhBZ8Iy5mKwSS0Dz10eBX948nNcMiYDb22px9TB/bu9oQe89XHKavyXnGh0yuA5ps39n5FiAQMEbGLCsDysk66FfHBztIcLwISxX7x4MRYvXtzm9eLiYhQXF/t9z8CBA/H222+3f3RRRve4IvXsk3yTJHw8ewAQBo6D59vXIW3/GNZJ14Jh/Rs6I3vWLkCtPAz3V/+GVnEIbO7JsM24DVx2f1PjsVu9DUz4vmNhneKtulf2zmNg0/JgnXkb3l7jQp7NuwjMZfbGFnEcxkrfQD7wPYSBY8Kef7hEO84+Gii/bod74ysgjVXgB58Ny5iLwNrT2+zHcRx+4k5Fat4onM9thOeb16Ac2QbbubcEjfV2NLWB1CEeBzzf/w/y7i/B2NNgnfp78CeNATQVypGtaNyyFjPl7VDf+QnOvqdDPPUc74J/hF6+rHizllkT709PEeH0KC0W0vXFYj3hKxBq9XF4Nv4bauk+sD36wTr1RvB5/p8abRYetuQkbCnlIWks8jqpO1W0SbbxOHA88AJtclMghy88xyI1SUR1feD2hFzuyW1qckWLxMqg9YmzD5dGl9yiSqUgtDT2jDUZwtCpkHeshbOuFNaJ14DL7O33OAIU5Bz6GM4DX4CxpsA6+XrwJ58R1pfcbuGN6A9h4BiA5ZDEuuHpPRYMx8Ph3tgiKmRP8hj0rf4FPTe+Ai5/SJvOOtGEENKl4uw1Zx083/4XyoFN3kSd4rvBBTBOOql2EWWSDbYL/gB595fwfP0KXB//DbYZfwiY6ej0eKuMEkKg7P8anu/eBPE0Qjj1HFiKLmh+H8dDOGk0GoRB+Nvr67GosAHppZvh+uhxsNkDIBbOAt+vMGwPWFLUkGGXOnrtm7pGD3KaDLCRUJXkX8YhqgJp22pIWz8EI9hgOft3EIZMCFnXJS/Djn1HvRE5XaXvbHtJaqpp70+Ca50960ssSlubJaGMfXs1e9/Sta09ewCwnHEpuNyT4fn6P3C+uwTiyGKII85vobtrZftwR9oq2H6phzDkbFjGzQ/Y4DwYSVYekqwZDUmEAaOQ5rOA4/S0LMhlsQh4X52I38vvwbPxFVjPuSlmtbQVVQMh6HTNnhACZe9XcG96E5A9EEddAHHEeaaS31LsAuqdEhiGgTh0MhhLEtxfPAfn6kdhn/lHMH4qWzrdMnJEN1wfPQ71+E7v09r428H16Of3HOkpFlRrKTiUOxpnTZkHef83kLaugnvtcrAZvb1G/6QxpotkyUrwloQtz623J5QMY68XQfPn2asVh+Be/yK06mPgB46D5cwFIaVGndxMO/YerQXQ/tLGXYUUmwhV819mPJixz0y1tqkE2lEklLHXjU9Emn2rNoIiz7bpA8swjLe2dc9T4PnmNUhb3oNyaAusE68Fm5oDz/f/w5Ddn6MKyRCm3wZrv9MjnosuFzjdihEXrqOoGiRZayEpWEUex6RUiGdcCOn7t6Ac2ATh5HERnz8YnijVsm8PWn053BtegnpiN7i8wbBM+B249J6m35+aJOJIqU8kxcAxYDgBrs9WwLlqGWzn39HC2BFCMNC9E9Olb6C6GFjGXw5h6OSg3nmGT3ITwwkQT5kIYfBZUA5sgrRtFdxfPAtmy3uwnuGt0R4KSQ7ektCXdD8lE2obJQg826K2D1EkSD+8D+mnj8HY0mA799awFw91A2+z8BFXeu1qJPlk0foz9r16+HfgMlIs2H2kOubj80dCGXueY8EAYVe+VDUNTo9iXGDAm6XY2rPXYW2psE39PeSBY+D56hU437sPjDUFxFWPQ2mj8eLxIfh7Oww90Jy443DLbYy9w8jkbB6vt+m4CvH0GVAO/wD31/8B1/MUv5p1e9ELrnWGjEMIgbJvI9zfvAaA8UoNp0wIWxLxljlu+WPO9y+EbcYiuNY8DdfKR7wGPykDmrMW7g0vYTazHZViH/S74GawqTkhzyHwHJJtAmp8yhYwLAdh0JngTx4H5fBWSJvfgeuL55D82ydCFsqSw5Bx0v2UTGidPauW7odr/YsgdaUQTpkAy9h5ET2F6uUR8jJtHdqZKZak+GTRti4mF9yzt8DlUYP2oogV3X9ZPAwYhoEYQeVLh8trPH1LHAg8F9DYG/v0H4Wkix8CP2g8mKR02Ir/gk32SbDY2v8omxSk8qVRKsHnZvL2odWgEsA26VpAkeDe8DIICT+PIRR6hnJHe/bE3Qj3ZyvgXv8iuB79kPSbByAWTIoo+iPVLsDlUdoUzeN7nwrbeX+E5qiGc+VfIe38HI6374Z6fBfec47Gj/2uNGXodQJl0TKMV5qzTr4OkF2Q92wIeSxvs3Fzn3mSlQfPsa08ew/Ski0gmgbPjx/AufJhQFNgO+8OWCdcHZGhB5o9+3jR6wGfypetsmg1QuBwKQH7S2emNNe172gSytgDgIUPv4FJc6kEc569L4w1GbZJ1yDpwvvA5w1Cg0uOyqOs3dIk4/gphuZb8VLHKHMsq2DT82EZfRHUX7fB/cVzIHJ0b7xodqkyi3J8Fxzv3APl8FaIYy6G7fw7wab0CP3GAKT4lExoDZ8/BPbz7wBxN8Lz9X/ApuWBL16CL90FLT5zM4RasOOyB4DLHwJpx1oQLfh9KyuaqbBLwOv4pCeLLdoT1jZK6GnzwLV6GaQt74EfOA5JFz0Avvcwc5MJQHa6DUlWHv3zYtcovqPxrWnvi8vjzaNJDnAfxKIAnlkSSsYB9NaE4ck4/gobiU21ccIl3HorgfCVcVrjr0ZLcx9ab3MN4bTpIKoMacu7cFb9Cus5N4HLMK9pB6NZxom9L0FUGVWf/RuuTR+CTcuDbe49psNXg6Ffo3qHZHxBfeFyBsI+525o5QfBDxrfJMXsC7vAXkaKiCNNGZeBEE+bAdfav0M5uDnoOoukBK5HpFYfh2fTm2BsKeD7jgDf+1TvU4WPjNPLtQ+za76BygHWSddBGDw+rLkEgudY/PWGM2CzdI3orGjgW9Pel0B1cXQyU733UqBY+1iScMbeIoQv4zRXvPSVccx59m2O5Qy8eBMOwWQch0/jEp3W3aoYhoGlcDa4nIFwf/7/4HzvPlgnXg1hYPuTrjpqgVZrqITrsxXQKg5BKJgMyxnzwfDRKeKlG/uGIO0JuYxe4DJ6AQiv4qUv6ckW1DtlI6rK73n6DQeTlgfp5zXgB44NqHvLstYmMxsA5F++g3vDv8BwIggIlH1fAyyHi7je2OnqDbU2F65ta3CFbT0aLD2RUXwL2LTo9g7oqEb3HYW9qcBga8++MUQXuvTkpsSqTpBxEs7YiwIHT5hVLwPJOIqqtWg6bIYGV2S17FsTrFuVfxmnydi3+qHjew2F/aL74fpsBdyf/z+opfu94aBc5LeG1AEyjnJsJ9yf/z8QTUXub/4EZ+bQqB5fv0Zme9HqP7C2sD375qiYHmn+F2AZhoV42rnwbHwFauk+8PlD/O4nKVqLBVqiKvB89zrknZ+DyxsM69Tfg7GlejN4j2xFyu7vcQ6+hvMtb/Odz13DkDtuPnpG2dDHIyzLGLH2vjSrAP6f3nmORWqySD37jsAisJDCLPUbyLMHAEXRTEedeGQVkqxFxcsReA4CzwY39hZfz977f39ljtmkDNhn3wXPprch/7wGasUh2KbfajqOujVGs3Ex+saeEAJp+2pIm98Bm94T9nNuRtKgQXBGuTBWiiHjmOtF6/QTAWWGjKYFu9oGKaCxB7zlOKTN70L+6ZOAxl5WVCP0UmusguuzZ6CVH4Bw2nRYxl5s5Hvw+UPA5w/BVjIOG77ahiXTk1BGMvDhJ/W4PbX9T52JQnBjH/g+yEyxdopmn3ALtF7PPnzNnufYFhq0HvUQjm6vP+KF07gkGHYrD6fHj2bvViDybAsvz1ez9wfD8rCecSms026EVnkY0o8fRjwuw7M3GQZoFiK54P70H5C+/x/4AaNhn3sP2PTYeKE2CweeY4LKOL40r5OEv0ALIGQTE4a3QBg2BcqRbdBqS/3uIykaBIGDcmwnnO8uhVZzHNZp/wfrGZf6LaiXnmxBhZaKhr7jUSb2Nl6jmCMlqLEPfB9kplhoNE5HIEai2Tu9ETS+WqluSMPR7RuaSiynREm/tFt4v31onR65jZygtyb0hCi9K5w0BvxJoyHv+xpEjsz70DX7aMbZq1VH4XzvPihHtsJyxqVeSUKITqNufzAM0xRrb1bGafs0ZYZwojOEoVMBjoP08xq/2yVFxZBGb9kFxpaKpAuWBG2oo2fK1jZ4UNsQvd6ziUJyAGPPMkzQGPqMVAuqGzwxCXsORmLKOBEs0LZeeGs29uaP1RBlzz7JKgRYoFXajDeYjNMaYegUKL98B/mXbyEWTAp7XNHS7ImqQDnyI+TdX0I9vguMLRW2WXcGlDGiTapd9Bt66Q9/uQ1mMOLdTRh71p4GYdCZkPdthFh0AYDmRhZEU1HMf4uhVXvB9x8F6+TrQzbo1mvg1DZKqG30QOTZuIqYiTVJNh5Hytoa+2Rb28qnvmSmWI1y0uFGb7WHhDP2kYZettbgIvHsm2WcKHn2Vh51fppGO91Km5vIGkbTcS53ENiM3pB3rYNwysSwsx49sjeTM5xm3b5o9eWQ96yHvPcrEFc9mOQsiEUXQiiYFPE6QiSkJAmod5iUcdwKbBYu7DkzDIOMFNF0L1rhtBmQ92yAvGsd0PcyAF55y/nZCpxl3Ytfs87E0HOuNZVIpnv2dY0e1Dm87QjjJcO1I0ixiX49++QQzlxz+KUbdj81lmJFwhn7SEMve+e0vCiRaPa6/hsoBjdc7FYeJyodbV53uhWktWpAIfIsGMZc03GGYSAMnQzP1/+BVnEQXM7AsMblkVXTNVp8Ie5GuNe/COXIVoBhwPcdAaFgErjep5kuBhZNUu0iSirNFa1yuBUj0S1cMpLNV0LkMnqC63M65F2fQ5t2iXch9pMnodWcwBuOcegzfAaGmcwYtlt4CDzr9ewbPGE1LaF4PXtZ0VqUiW50ykgO4a3rWbQ1DR70zu44Y594mn1TMpQWhl7mr9ZFZJq9V88L91E/EEkWb0p/axxuuU1UCMMwRn0cMwiDzgQEK6RdX4Q9LknWjDUCs2iOGjhX/hXK0Z8hjpyDpEv/Btv0W8H3Hd4phh7QZRzJlLbq8ihhR+LopKeE13hcPH0GiKse1V+8Cuf7D0BrqII2+WZ86xkcViKbnkVb2+jx1sUJUcee0hJdjvVNrGp0h/bs9XWajl6kTThjrxsh2aSUoxEChztKxt4pI9netqlBpNisvNHm0BeXx7+XaRV508aeEW0QTj4DyoHvQdyNod/gg6+nYwattgTODx6E1lgF23l/hKXoArDJnd/GMiVJgNTkuYXC0dSlKhIyUiyoaTS/YMf1LACb1Rf1m1cDLAf7nLsh9zgFAEwXQtNJS7Y0GXvJqHFPMYe+LuYr5fiTfFuTniKCYYDqAO0JY0XCGXtdfjFb097pVkBI21CqyIx9dBKqdJKsPAgAt493rxHiV7MHEJZnD3gXaqHKkPdtDGtcXhnHnLFXKw7B+eHDgCLBPvsu8D0LwjpXLDFKJphYpHW26lIVDhnJFsiK5jeyyh8Mw8ByxqVIGjoe9rn3gMvsbciJZj93nfRkC0qrnfDIqlHjnmKOlFb1cQghaHS27GjnD45lkR6GdBctEs7Yh9uHtjmhquUF1DVpKYxonNbdrtqLLgf5RuS4PQoI/Cf3WATOqFtjBi6rD9jckyHtWgdCzP+omW02rhzfBeeqZYBggX3O3eB69Dd9jo5Af0xvMLFIq3epioT0CIpj8T0LkHvBbUaJ6uZm4+F9pb0yDg27jISkVpUv3ZIKVSN+S1a0JiPFguoGKuPElHD70DaXSmjl2Tf9aIQv40TPe9I9SV+PUDf8/tL2vZ69Oe9RRxw6BaS+DOrxXabf45G1kNmz8sHNcH38BNjkHrAX3x31WizRIJySCe2RcXwX7CJFdzrMVr3UyfAx8NTYh0frMsf+yqoEIpxF+WiRcMbe8OxNGulAGXECF0HoZZTKG+s0F0NrlhkcQQpyhaPZ6/ADisBYkr2hfiaRQmj28v5v4P78GbDZ/WEv/jPYpIywxtRRNBdDCy7j6J3BIl+g1ePd22PsI5NxfKO2aDROeOjXu7WxN+PZp9gFOFzmcjiiRcIZe6MPrUmjZ1Sxa2Wk9ScEs8Ze0wgc0ZZxdGPvo9kbaft+JAWrJXzPnuFFCKdMgHJkKzRHjan3BNPs5V++g/vLf4LLPwX28++IuCFGR2B49iFkHH+F58JB96jb4+npAQfhLtCmU88+YniOhc3CG0beYaJUgk6yXUSjq21wRSwJeWcsW7YMU6ZMwZAhQ7Bv3z7jdY/HgyVLluDcc8/F7Nmzcc899xjbDh06hHnz5mH69OmYN28eDh8+HJPBR0KzZx+ejNO6GYHu2YdzHILoZc8CvjXtfWUcucU2X6xhavY6QsEkgBDIu780tb8kq35DL+WD38O97nlweYNhm74oauWIY4UocLCKXEgZxxHkMzcDz7HokWbFlr3lEV0foPk+DF+z914DS9NcKeHhWx+nIUQt+9bv04MpOoqQd8bUqVPx2muvoVevXi1ef+yxx2CxWLBmzRqsXLkSt956q7FtyZIlWLBgAdasWYMFCxbg3nvvjf7II6RZszcv43As0yaNPNxoHP1GiKaMY3Sr8rlhoi3jAACbmgOuz6mQ96wH0ULfnB5Za7NAKx/+Ae7PnwOXMxC2GX8ImcrfVTBTMiHY05RZrpg+BCcqHXjxo90R1UzR78NIPXvf3rMU8/hWvjRTBE0nUKerWBLyzigqKkJ+fn6L1xwOB95//33ceuutxg3So4e3BVxVVRV27dqFWbNmAQBmzZqFXbt2obq6czqqtyaSaJwkW9vYeIZhwHPmG5g06tmzUZRxrBYODIMWlS+bJQX/C7SSokHVwm+6IhZMAXHWQjmyLeh+hBBIrWQc5chWuD/zavS2mbfFtIBZtElJEkJWvoy0cYkvp56Uhd9MGogte8rx0XdHwn5/s7EPzzu3WTiIPIs0KuFERIpdMKTeRqcMBuZ+9FMCdLqKJRFp9kePHkV6ejr+8Y9/4MILL8Tll1+OLVu2AABKSkqQm5sLjvPedBzHIScnByUlJdEbdTswNPswjH0gnV0Mo1tVtIugATCycVvIOB5vlq6/R3Kj8qUUvrHn+g4Hk9ID0uZ3QJTA2rKkaCA+51J+/QmuT1eAzeoL+3l/BCMGrtneFUm1iyFr2gf7gQ2HGWP6YkxBDt5dfxA/HagK673GAm2Y0TgMwyA30468zO51XboKSVYfz97tdQzN1EfSPXu9Em5HENHdqaoqjh49iqFDh+LOO+/E9u3bsXDhQnz66adRHVxWVuR1I7KzU/y+npzq/WIKohBwH188iob0VKvffS0iB07gTB0H+ysBAP16pyMrSJOKcElJEqERxhgDYVgk2QTk5LQtGJad5V0MTUqxokd6+GNwzf4/lPz3fjBb30H2eTf43UdvYJ2VYUey4zBKP10OMacv8hcsAWeLXR0QU9cgAnKyknC4tCHo8VnBe2379EpHZmr7nlruuGI07ly+Ef9cuRNPLJqIniFqp+jjEpu8yZ55aWGXln7o9+MhClzIZKCuRKyud7hkZ9mx7ZcKZGenQFYJ0pJFU2Mj+hMYZ9J+6Odrx7wjMvb5+fnged6QaoYPH46MjAwcOnQIPXv2RFlZGVRVBcdxUFUV5eXlbaQgM1RVNULTwtcvs7NTUBGgc5F+vOpaZ8B9fKmtdyMv0+53X45lUN/gMXWcE+XefSSXhIowI2KCYRE4VNe5UFHhNUhVNU7YRM7vmOQmbfl4SR1IiLr2fkkeAHH4TDRs/QhyjyEQBoxqs0tlnQsAwFb/ipJv/gk2NRfiubehupEAjdHtJqUT7Hq3F4EF6hollJXXB2w/WVbhLSfhdrhR4aeZTLgsLB6K+/+9BUv/+S0WX1EUsDa677xral1gANTWOCLS3hWPDGdjxzfUiIRYXu9w4QC4PCpOlNShqtYFm8ibGpsuI5eUN5iei5l5sywT0EmOSMbJzMzE2LFj8fXX3t6Vhw4dQlVVFfr164esrCwUFBRg1apVAIBVq1ahoKAAmZmdX+sE8H4YAm++pn2DK3D6s7fpuPkQTpuFD9hUOlKSWnWrcgQolQCEV+Y4EGLRhWB79Id7w7/8hmJ6ZA3prAMD9/4bjGjzavQdWMY12qTYxZBRE06PAoFnw9bLA9Ej3YbfzxmGsmoXXli1y1R4ntzUf5YusnYsvolVDj8FEwNhEbxrJV1Ks3/wwQcxYcIElJaW4qqrrsL5558PALjvvvvw3HPPYfbs2bjtttvw6KOPIjXVKx0sXboUr776KqZPn45XX30V9913X2xnESYiz5rS7AkhQS+gyHNhReNEM8Zex27hWxgipydwJqfV0Owjf7JgOB62KQsBVYZ73fMgrRZ7ZWcDFiZ/DlaVYJt5W5coaNYejPo4QWLtnW45apVMdQr6Z+KSKSdj6/5KfGxiwVZS1LAjcSjtJ8XH2DeEYewBr27fpTT7xYsXY/HixW1e79OnD/7zn//4fc/AgQPx9ttvt390McIicpBMhF66PN5aF4EuoNBULtkMjVEugqZjtwptyiXoTaxbE063qmCw6XmwnnkZ3Bv+Bemnj2EZ4XUAiCojedM/wXH1qC5ciPTMPu06T1dAv2beiBz/CWCBCs+1l3OKeuPHfRXYtKsM55/RP+i+UhiN7ynRIylCzx5oamvYlTz7eETkOVOefaM7eNyswLOQVfPRONEMu9SxW1t59u7AddWjIePo8EPOBj+gCNLmd6FWHAIhGtxfvgBLzQH813EmSF7HtA6MNWYqXwaTztoDwzDITrP67VnQGl3GoXQs+ne6ut4NSdFM1cXRSbGLRv5NR5CQd4fZblWBSiXoCDxrui5+g0uOatilTpKVb6rN4p2Pt2OS/xtOD4d0h9mpyx8Mw8A64Sow9jS4Pn8Wnm/+C+XAJpQPmIkfpJPa3X+2q5CSZEbGadvzN1pYLTxcntDXS4qwOxilfejGvqzG29EsnO94CvXsY48omNPsQ2XEiSY9e0KI0bgk2uj1WJweBR5ZhaJqJhZooxMNxFiSYJ18PUh9OeSdn0EYOhWluWcDaH+z8a5Cso0HAwRNrAq2TtJebBYOLkkJmVXr9ezj4zPvTui2obTKa+zD+dFPtgnUs481FoEzpbU7jCp2QTR7Ez8abslrhGOi2Vua6+Po4w1UkMsicGAAuE14imbhe54Cy/jfQhg2DZYzfwtP0+caL8aeY715C8FKJjiDPE21F5uFByEIucYkKRr17DsBgWdhETmUVuuefXgLtC6PAsWkFNxeEq7hOOA1RDUmysnqv7qBQy/NReOEU/o0XHzLHOslGQJp9gzDwCKaW68IB3HYNOP/+rHDzeTsyqQmiQGLoTV3BouNjGMTmyubBuvrKytqt0qKiieSrQLKarz5JeFcA92JdLjkDilXET/fyDAQBXMeeaNLBsMEToMXTJZLaAih/bcHm2HsFThcodP2LRE0MAkHvWpjPEWGZKZYUNbkubXG7VFB0L4iaMGwWsxJb17PPn4+8+5Esl0w7EA44dXJJvslRIuENPYWgTNV9bLRJSPJKgTMnBRNhl7qem8sZBxdI3S6FTQ2xez6azauE2nlS7PockKgz6w7MqRvOo5VOFDnZ5FWLykdaeOSUOiefahFWlmmMk5n4bumF1Y0jk2vj0ONfcwQzUbjhIibFXgWiqqFXDwL1Mc2GjTXtJeb26IFMTzhNh0PF4+sxpVXDwBD+3sTw3Yfblu51ShvHCsZx6Ib+1CePU2q6ix0G2Gz8OBY89ego8scJ+TdIQpe3TqkkXZKQaUXszXtY1HxUsdoOu5pXqD1139WxyrE1thLkrlm492JfrkpSLLy2OnH2DuiVPEyEGaNvUxlnE5DN/bhOnPNZY47Jos2vr6VJrEILAgBFDWUR6606VDlix7qFir8ssElgef8lx1uLzzHwiJwTTJOUzROEP04kqbj4RCPnj3LMijol4Fdh2vaOAixl3G8n6XLhGYfbrNxSnTQjX24C+RJVMaJPWZr2jvcwWPjdY00VFicHq0RqyJVehatwyXDInJBi61ZLbHX7OMl7NKXoQMyUdPgMULsdKJVyz4Q+lNaMM2eENLk2Sfk17nTMTz7MNfk9B62dIE2hpjpVmUkQoXQ7IHQnr3LowQsUxsN7Fbeq9k7QxfksoqR9aE1i0eKP88eAIY16fY7D7WUcgwZJ8iieHswEuGCyDiRtiSkRAfDs49g3SbFLlDNPpY096ENbPQkWYOiauaMfYgnBJdHNR7HY0FSU+XLRpcUUk6wxFiz98hqXHr22ek2ZKdbsetwy7LOTo8ChmkOkYw2HMtCFNigMo7RpYpq9p2CbiMiqX3lLZlANfuYYeF1zz6wR66XHu0enr3QtEAbOpPT2pRUZaZGeiR4ZZz4vK2G9c/Enl9rWmQ86uWNYxlqagtRH8fw7OP0c+/qGMY+gtDqjiyZkJB3hyiG1uz1BKVgxl408aMBeBfXYi3jON0yGl1SyBBAvcxxrKQcjxSfnj3gDcF0SyoOlzR3C4pVeWNfbCIfNBpHamqgI0S5MQ7FHOnJIhjGm3wXLslUxokthmcfpMuUnkCTGiRcMjzPPnYG0G7ljdDLUDJONMsc+yMeo3F0TumXAQZoEYLp9MSuVIKOXgwtEHrl1Xj93Ls6ackW3HNlEcYOzQ37vSk2EQ1OOWQYeDRISGNvaPZSYCOtl7RNTTKj2Ycy9qqRCRkL7E2P+fUOKWiMPRD9ypetkeJUswe8T3n981NaGHtHDLpUtcYq8kGL1+nOBl2g7Tz656VG1HI0panUgplmSu0lIe8OIxoniGevF75KTWqfZ68RAneMNXs9CsAtqSEjAqLVrcofGiFNHZPi97Ya2j8TB4/XG7JKsGYx0cL7Yx5ExtGLz1Fj3+1INmLtY79Im5B3h5k4+3qHBFFgDePo9zhGnH3g43gkb6GsWGv2xv9NLNDq44o2+hNOvHr2gNfYa4Rg76+1ADpGs7eGknFoNE63pSNLJiSksdejRYI9OtU7pKB6PWAug1b3yGKt2fv7vz8sMdTsm8sbx6/ROblXGkSeNaQcRwzLG+t4F2iDhAnTOPtui172vCM6ViXk3WHGs69zSEgLIuEA5jR7V5NR7QgZp/X//RFLzV7/POPZsxd4FoP7pGPX4WrIircpTaxlHJuFh9sTuFuVLkfGs3wWryQbDe2psY8JPMeCY5mg8kuDUwqq1wPmNPtmzz62C7TG/0Mu0IbW7H8+WGWqyXVr9M8zWJONeGBo/0yUVDlxrMIBIHa17HVsFh4Ega+Z7mxQz777oZdY6IhYe1N3x7JlyzBlyhQMGTIE+/bta7P9H//4R5tt27ZtQ3FxMaZPn46rr74aVVVV0Rt1FNArXwai3mHe2Af70dDT3GMajROGjBMq9LKyzoUn39qO73aWhj0OvUdAvC8UDhvgLZ2wZU85gNiVN9ZpbmDi/5rRDNrui60pIa+xqyzQTp06Fa+99hp69erVZtvOnTuxbdu2Fts0TcMdd9yBe++9F2vWrEFRUREef/zx6I06Cni7Vfn3yDWNoMElhyxJzDIMeI4J6tk7O1izDyXjWELIOEfLGwFE5mkkgowDAL2yk5BqF7DZMPaxj8YBApc5prVxui8swyDZxncdzb6oqAj5+fltXpckCffffz+WLl3a4vUdO3bAYrGgqKgIADB//nx88skn7R9tFLEEaWDS4JJBCEJq9kBTH9ogmr27AzR7i8CBY73p+qEkBZZhgtbH0aUJKuMEhmUYDO2fico6N4DYG3urGNzYU82+e5NsFztExmnXXfr3v/8dxcXF6N27d4vXS0pK0LNnT+PvzMxMaJqG2tpapKenmz5+VlZyxGPLzk4Jut1uFQCW8btfo1wHAOidnxryOBaRAydwAfdjmx6t+/RKj+njfrJdgMMlo1fPtJCllO1WHgzH+h1zVX1TI3bW//ZgWI7XAwByc1LCfm976ejzjT0tH9/tKgMA9OmZjuzsyO/VUOQ3eh/xRZvYZp7Z2SkQRB4cyyAvNy1mY+hqdPT1jiUZqVa4Zc3UnNoz74iN/datW7Fjxw7cfvvtEZ88FFVVjdC08NOIs7NTUFHREHQfjgUaGj1+9ztyrNb7H0UNeRyeZVDf4P84AFBZ7QADoKHeBUeD28zwI8LaJJ1UVjaG3FfgWdTWu/2O+UDT3KtrXSHn3pqKKu+5nQ1uVHAd14PWzPWONn2y7Mb/3Q4PKhC7dHdPk55bVt6Aikyb8bo+79o6NwSe7fDPoLPojOsdS6w8i5JqZ8g5mZk3yzIBneSIn/s2b96MAwcOYOrUqZgyZQpKS0txzTXXYOPGjcjPz8eJEyeMfaurq8GybFhefawReQ6eAO0Em0slmJFx2JCavdXCxbwBt90qmC6xahU5v/XRFVUzmnNEJuM0LRTGuYwDAJmpVuQ3GfyOKIQGNK//tEZW1LhfFI9nUuwdU+Y44rv0+uuvx/XXX2/8PWXKFDz77LMYPHgwNE2D2+3Gli1bUFRUhDfeeAMzZsyIyoCjhUXgjJ6trdGLoJnT7Nmg9ezdHjVoFm606JFmhWLyKcgq+u9WVVLlhNp0jFBt8PwhJcgCrc7wgT3gdJdGVBMlHPTF/UANTCRFMxL8KN0Pb+VLBRohMXUKTVmhBx98EGvXrkVlZSWuuuoqpKenY/Xq1QH3Z1kWjz76KJYsWQKPx4NevXrhsccei9qgo4EosIE9e6e3Z6yZRdVQnr3LE7rGfDS4csYpyMpKgsvhCbmvVeSMHzRfjlV4ZZi8THvQjM1A6NE4iRIVcsGEAThndJ+Yn8dYoA0SekkXZ7svyTYRGiFweZSIul2ZxZQVWrx4MRYvXhx0ny+++KLF3yNHjsTKlSsjH1mMCRqN0xRjb6ZnrMhzRpyzP1ySErMuRr7YrTyS7aJpY19W03buxysc4FgGA/JTsKep9ks4eMsbszGXrLoKAs8hIyX215ZlGVhELnDopawmzA9sPJJia86ijaWxT9g7RAxi7OucUsgYex2BZ404Z3/EuktVJFhFzm+c/bGKRuRn2ZFsEwPqw8HwyPHZbLwrYAti7CVFowlV3RijGFqMY+0T1thbgmTQ1puoi6MT2tjHtpZ9JFhF3m/Vy+MVjeidnQybxduUPNxIqHiuZd/Z2Cx8QBlHVjTq2XdjmksmxHaRNmHvEFFgoagEqtbWUJupeKnjNfaB9e2u6NlbBK8x9y2s5XQrqKr3oFd2kjHecIulxXOXqs5GL4bmD4lG43Rr9Cg66tnHCKOBSavsV40QNDhlU2GXgLcOTCjNPpalEiLBauFA0LLq5/Gm+Pxe2cmGsQ9XyvHIatw2G+9sgsk4sqJBoD+y3RajzHGMs2gT9pspGsa+pVfudCtQNWLa2AscByWAsVdUb7uxrubZ+6t8ebypTEJvX88+zIgciWr2McMaRMaRZI169t0YUWAh8GzMyxwn7B2ie6Ctdfs6E71nfRGEwJ69URenq2n2ej1/H+NxrKIRVpFDVqrVeBKJxLOnMk5ssAVpTUiTqro3DMMg2SZQzT5W6NELrWUcPXs2zaxmz3kXaP01luiIWvaR4K/M8bEKB3plJ4FhmvMLws2ilaixjxneblWBo3F4auy7Nd4sWurZxwS9MmNrz77BRKNxX/RkFsVPYlVHtCSMhNbdqgghRiQO0PwkEm4WLdXsY4fN4q1UqvlxKmQaetntSbEJVLOPFYGahdeFURcH8Hr2APyGX3ZZz97SUrOvbZTgcCvNxt7w7Klm31XQ11lah8yqmgZVI1TG6eZ0RJnjhL1Dmj37tjIOyzBIMllUTI+C8Kfbd0T/2UjQDbJu7I83lUnonZ0EIHSzjEBQzT526MXWWl8To3EJfaLq1iTbBLpAGyv0x97WMk69Q0KKXTCd8t8tPftWMo7esKRXk2evlzwIx9hrGoGsUM8+VujXrPU1oS0J44MUmwCXR/ErB0eLhDX2lgChl2Z6z/qia/b+PPvm/rNd64vYOvTyWEUj0pJFI7nDu0gbOK7bH4nSkrCzMKS1VjIObTYeH+hZtIEq8UaDhL1DxAChl/XO8Iy97tn7i7V3dlHP3iI2zd3H2Pdu1WkpWKifPwwPk8oJMaE596G1Z9/UkpAa+25NclP0Xyx1+4S9QywBtPZwSiUAzVqp5KdkgltSwbFMl/O6OJaFyLPe6A6N4ESl09DrdbzG3vwCLfXsY4v+dNg696G52Tj93LszHVEyoWtZoQ5E4FkwaBndQAhBnUM2XQQNCK7ZO5vq4pgpldzR6JUvy2qcUFQNvXq08uyDpOf7Q5KosY8lNkvbrGeAPlHFC0aZY+rZRx+GYbxljn08cpdHhaJqYWr2gaNx3B7FWFjralhFHm5ZbS6TkOPPsw9Ds9flBGrsY0KgRDe9SxqVcbo3zWWOY5dFm9B3iCiwLUIvmxOqzDcQ0CUaf5q9y6N2SJeqSLCIHNweFccqGsEwQM+sVsbeyodVLqHZs0/oWypmWEJE41AZp3uTTD372NK6W5WRUBWOZs8H1uxdHsVIYOpq6DLOsQoHcjLsbTxym8V/n9pA6D+a1LOPDSzDwCpybdZRZCP0MqG/yt0enmNhs/BUs48VYqsGJvVhZs8CzfHNgeLsu6pnrzcdP17RiN49ktps12ux+Kv54w+6QBt7vA1M/Efj0KSq7k+sSyYk9B1iEdgWhdDqw6yLA/h69v4yaDum/2wkWEQODU4J5TUu9Mr2Y+wtHFSNBK3V74tEjX3M8dfAhCZVxQ/JdsGQkmNBQht7kW/r2TNoTnAwQyjNvqvF2OtYRQ5V9R4QoE2MPRB+yQT9c6RRIbHDX4QUTaqKH7xljqlnHxMsYkvNvt4hIckmgGPNfyyBPHtCiLclYRerZa/jGyXUO6etsbdGaOypZx87/DUwoUlV8UOXkHGWLVuGKVOmYMiQIdi3bx8AoKamBtdddx2mT5+O2bNn46abbkJ1dbXxnm3btqG4uBjTp0/H1VdfjaqqqtjMoB2IPNvCs68Lo9G4Dssw4DmmjWYvK95qhF2tvLGOXjJB4FnkpNvabA+38qVH1sCAepixxF84bHNSFf3cuzspdrHzF2inTp2K1157Db169TJeYxgG1157LdasWYOVK1eiT58+ePzxxwEAmqbhjjvuwL333os1a9agqKjI2NaV8EbjtNTsw9HrdQSebRON01UrXuronn3PrCSwbNukr3BlHL1xSVdMIIsX/Mk4kqJ5EwTp597tSbYLkBStTQmXaGHK2BcVFSE/P7/Fa+np6Rg7dqzx94gRI3DixAkAwI4dO2CxWFBUVAQAmD9/Pj755JNojTlq+IvGiczYt+1D21UrXuroxr51mQSdcLtVSbRxScyx+ZFxZNp/Nm4wYu1jtEgbFUukaRpef/11TJkyBQBQUlKCnj17GtszMzOhaRpqa2uRnp5u+rhZWW21ZLNkZ6eE3Cc9zQZZ1Yx9G10ycrOSTL3XF6vIgeW5Fu+rdXuNZF52StjHaw9mz5Xd9NkOGZDl9z0a5/0x4EXe3DE5Fjar0KFz9aWzztuR9MiwwyOpyMxKBtf0NMbyLCxmr1EcEY/z7ZWXCgAQLGLA+bVn3lEx9g888ADsdjsuu+yyaBzOoKqqEZpmLs7bl+zsFFRUNITcT5UVeCQVZeX1kBUNLo8KgYWp9/rCsQwaGj0t3neitB4AILmlsI8XKWbnDQCyx6sNptt4v+9xub3byysbTR2zocEDnmM6bK6+hDPv7ozWJBUeO14De9MPa0OjBzzbOZ97ZxGv11uTvQ7i0RO1SLO2XeszM2+WZQI6ye029suWLcORI0fw7LPPgm2KYsnPzzckHQCorq4Gy7JhefUdgR45Iitac0JVGNmzOnrTcV/0hc2uKuMM65+JiycNxCn9Mvxu16OIzJZM8MgqjfWOMVafRXO71fvILykaTaiKE1JiXOa4XXfJE088gR07dmDFihUQxWYjeeqpp8LtdmPLli0AgDfeeAMzZsxo30hjgJ7a75HViLJndQSBhdx6gbaLa/YWkcPMcf3Ac/5vAZZlYBE40yUTaLPx2GMsmvtk0UqKalRepXRvYl3m2JQlevDBB7F27VpUVlbiqquuQnp6Op566ik899xz6N+/P+bPnw8A6N27N1asWAGWZfHoo49iyZIl8Hg86NWrFx577LGYTKA9GF2m2mvsObZNnL3+heyqxt4MNgsXlmefkWyJ8YgSGz0b2+0TDksXaOMHu5UHwwANrk5coF28eDEWL17c5vW9e/cGfM/IkSOxcuXKyEfWAVgMz15DXdMKeLhx9oD3CUH/sdDRPfuuWuLYDOGUOZZkjRZBizH+pDVJ0Yxm5JTuDcswSLYJMfPsE9olEH360OrGOiVKmr3bo0Lk2YAySXfAXy2WQHhlHGrsY4nVaGDSfE1kRaWefRwRy5IJCX2XWFoZe7uFjygTURDaJlXpXaq6MzYLD6fJDFqJGvuY4y/RTW5KqqLEB7HMok3ou6S56bg3GiclAgkHCODZS123lr1ZwpFxPLIKUUzo2ynmWI0GJs0/wJKi0SioOCKW9XES+tvZwrN3ykgLo9qlLyLPtTH2To8Cexeti2MWu4VrUz/dH6qmQVEJLNToxBSLyIGBH8+eRkHFDbEsc5zQd0nr0MtIInEAb20cf5q9tYtWvDSLVTTn2Uu0S1WHwDJMU+XLlqGXVLOPH5JtAhpdCjSTTYPCoXtbo3bSWrNP7e8/wSgU3kJoGgghRkEql0dBWpI9amPtDOwWHpKsQVG1oAvNRnnjbhx51F2wWTgj9JIQAlnWaP/ZOOK0k7JQWedGLMraJbSx1z0ih1uB06O0y7MHAEUlEPgmY9+Fu1SZxWZEf6hItgU29npPAOphxh6bz9OWomogoJ97PDG4TzoG90mPybET+i7RPfvKOjeAyBKqgOYvm28WrSsOonH0H6tQUo7ebJxG48Qe3z60RpN3auwpJkjou4RlGfAci4paFwAgLYIYe6DZs9d1e40QuD1ql+1SZRazNe2pjNNxWC2cEY2jP1EJ9EeWYoKENvaAt+l4ZZ3X2Ecu4zRp/03G3iOpIOjepRIA8zXtaUvCjsNXxqHyGSUcEv4uEQUO1fUeAIg8zr6VZ99cBK17Gz+zrQkl2my8w7D5hMPqP7I0qYpihoS/SywCB7WpZn6kMo4Y0NhTz54SXbwlLFrKODSpimKGhDf2ujdqEbiINec2nn0X7z9rFn38oSpfSnSBtsOwiTw8sgpV04zPnSZVUcyQ8HeJbqBSkyLLngWajb1eHydePHs9A9gdIovWI+kyDjX2scbqEw7roZo9JQwS/i4RDWMfmYQDNC/QtpFxunl0Cs+x4FgmtGev6DJOwt9OMcfmEw5LZRxKOCT8t9Pw7CPU64H41ewZhmkqhhZ8gdYjq2AYdOtyzt0FPZzX7VGbQy+pZ08xQfe2RlFA1+wjaVqi0zYaJz40e8Abax9ygVbSYBE4o1QEJXb4rqPoob5UxqGYoftbo3ZiiYqM01azZxAfSUZmyhxLikr1+g7C5tPARH/goklVFDMkvLHX9c5oGPvmaBxvLXs2Djxdm4UzFXpJ9fqOoVmzVyE3FUaknj3FDAl/l1iaGm60T7Nvu0Db3ROqdMx49vUOCcm2yD8/inn0stm+C7RUs6eYIeHvklh49m6PGhd6PQBTC7SVdW5kp1s7aESJjVGvSFKMhXGO7f5PkJTYk/DGPhqaPcsy4FjGWDBzSUq3L4KmYwvRwEQjBFV1bmSlUWPfEYgCC4bxyjiS7G1JSBfGKWYIaeyXLVuGKVOmYMiQIdi3b5/x+qFDhzBv3jxMnz4d8+bNw+HDh01t62oM6ZuOUYOz0aOdxsq3W1U8lDfWsVm9tVhIgM45dY0SVI2gR5qtg0eWmDAMY/wAS7JKJRyKaULeKVOnTsVrr72GXr16tXh9yZIlWLBgAdasWYMFCxbg3nvvNbWtq9E3NwX/d+Fp7Y4RF3nWqGfv8qhxpdkT0lz/pjV6xdD2/lhSzOPtVuWVcWjxOYpZQt4pRUVFyM/Pb/FaVVUVdu3ahVmzZgEAZs2ahV27dqG6ujrotngmbj37EJUvK2u9jV+ose84vA1M1CbPPj6cCkrsicgilZSUIDc3FxznvdE4jkNOTg5KSkpACAm4LTMzM6zzZGUlRzI8AEB2dkrE740Eq0UAw7HIzk6BW1aRmW7v8DEA0Z93bg/vNbDaRb/HdiknAABDBmZ3aiG0zvisO4uUJAtUQiArGuxWPqHmrpOIcwbaN+8u7X5WVTVC08Lvsp6dnYKKioYYjCgwLAM0OiSUlNZ5C4OpaoePIRbzlpsWZ4+X1sPGtV0IPHKiDmlJIuprnVE9bzh0xvXuTHiWQV2DB0wqAxZIqLkDiXe9dczMm2WZgE5yRMY+Pz8fZWVlUFUVHMdBVVWUl5cjPz8fhJCA2+IZXbN3N1WAtMaJjKOH+rkDRORU1rmphNPB2CwcymtV2OkCLSUMIrpTsrKyUFBQgFWrVgEAVq1ahYKCAmRmZgbdFs/omr0epmiPE2OvNx0PVPmShl12PN4GJt5oHFqmgmKWkMb+wQcfxIQJE1BaWoqrrroK559/PgBg6dKlePXVVzF9+nS8+uqruO+++4z3BNsWrwg8B8nH2FvjJM4+WNNxTSOoqnfTsMsORg+99Mga9ewppglpkRYvXozFixe3eX3gwIF4++23/b4n2LZ4ReBZyKqvZx8fHlewaJzaRk9TjD317DsSm6XZsaB1cShmoXdKlBB5FrKsGS0J40Wzt4gcGPj37CvrmsIuaamEDkW/t2obPDT0kmIaauyjBN/Gs48PY88yDKwBKl82J1RRGacj0UtxKKpGPXuKaeidEiVEnoUkq0bUSrx49oCexBPYs89KtXT0kBIa3+xs2mycYhZ6p0QJXbN3xplmDwSufFlZ60ZaskilhA7GNzub9p+lmIUa+ygh8JxXs/eo4FgmrvqxBqp8WVnnoouznUBLYx8/9xklttA7JUoIPAsCoNElwWbh46rsrM3C+42z9yZUUb2+o7H6tLukoZcUs9A7JUroHla9Q46bipc6epVFX1RNQ02Dh3r2nYDv4j9NqqKYhRr7KKF7WHUOKW4qXur4a01Y2yDRGPtOwnfxn3r2FLPEl1XqRATDs5fizgB6ZZyWC7Q07LLzEHkWLMNAIyRmmr2qKqipqYCiSDE5fnsoL2ehaVpnD6PD8Z03z4vIyMgGx5k34dTYRwnD2Dsl9MmJvDRzV8Rm4aGoGmSlOT3fSKiKsx+27gDDMLBZODjcSswioWpqKmC12pGUlNfl1p94noWiJJ6x1+dNCIHDUY+amgr06GG+wCR9BowSegicrGhxp9n7NrnWqaxzgwGQmUqNfWegS4Wx8uwVRUJSUmqXM/QU7499UlJq2E9d1NhHCV/tNN40ez36w1e3r6xzIT3FQjXjTkIvtBfLz58a+q5LJNeGflOjhBjHxt5f5Uta2rhz0ZP26I9tx/PQQ0vxzjtvdvYwwobeKVHCVzuNN2Pvr/JlRS1tWtKZ6BE5iRh6qSj+eyt093PFmviySp1ICxlHjK8voK2VZ09j7DufWGv2XY2zzirCVVddh2+//RpnnHEm5s+/DMuXP4kDB/ZDkiQUFhbh5pv/gOPHj+Ivf/kTXn31LSiKgvPPn4orr7wGCxZcgc8//xRfffUlli59CK+//io+/3wtVFWBKFpw++13YdCgIW3ONXbsGZg79yI8+OASVFVVIi8vHyzb/Jl/8MG7eOut/0IQRBCi4f77H0G/fv075TMKBTX2USKeZRx9wVk39jX1HmiE0LDLTkR3KDpKxvn65xJs/KkkJsc+6/R8jD8tdFSJxWLBCy+8Ap5n8eCD92HEiJG46657oGka7rtvMVav/hDFxRfA6XSgsrISpaUnMGDAQGzZshkLFlyBH374HkVFowEAM2acj0svvQwAsHnzJjz22F/x/PMvtzkXANx99x0YPrwQV199PY4fP4bf/W4Bxo49AwDwzDN/x2uvvYMePXpAkqQuHRIaX1apE/H90sVTxUugrWdvVLuknn2n0ezZx9dTZDBmzpxl/H/jxg3YvXsn3njjNQCA2+1GTk4uAGDkyCL88MP3KCk5gTlzLsRrr70CWZaxZcv3uOyy3wEA9u7djf/85yXU19eBZVkcPfprwHP9+OMPWLToDgBAr169jR8M77lG46GHlmD8+LNxxhlnoVev3jGZezSIL6vUifga+3ipZa8TyNhnU2PfafTskYQeadYOK3E8/jRz3ncssdnsPn8RPPzw436N66hRo/HDD5tx4sRx3HvvA9i27Ud89tkaEAL07NkLsizjnnvuxD/+8U8MGXIKKisrMHfuzCDnCszDDz+G3bt34ocftuCWWxbi9tv/jDPOGN+eacaMxBD8OoAWnn2cafY8x0LgWWOBtrLORWPsO5nxp+XjpXung03Q8Mjx4yfg1Vf/DVX13pO1tbU4ceI4AK+x37TpWzQ0NCAnJxdFRWPw4ovPGR65JHmgqqrxJPDuu8FbqI4aVYTVqz8EAJw4cRxbtmwG4F28PXHiOIYOPRWXX/47jBkzDvv3743JfKNBfLmgnUg8e/ZAywYmlXVupKdY4qqMM6V7ceutf8QzzzyN3/3uUjAMA0EQccstf0TPnr2Qk5MLu92O008fAcBr/MvKSjFyZBEAICkpGddccwOuu+4KpKamYfLkqSHOdTsefHAJPvtsDfLze6KwcBQAQNM0PPTQUjQ2NoBhWOTm5mLhwptiOu/2wBBCSGcPIhBVVY3QtPCHl52dgoqKhhiMKDjXPboOqkbw9K1nI9kmdPj5YznvPz//HfrlJmPhnFPxyGs/ghCCP182KibnCpfOut6dTSznXVp6BHl5/WJy7PaS6OUSdPxdI5ZlkJXlv1xLu12zdevWYe7cuZgzZw6Ki4uxdu1aAMChQ4cwb948TJ8+HfPmzcPhw4fbe6ouj+7dx1u5BMAb/aHXtK+iTUsolG5Hu/QGQgj+9Kc/4bXXXsPgwYOxZ88eXHrppZg2bRqWLFmCBQsWYM6cOfjggw9w77334pVXXonWuLskAs9CIwQcG3/yhl7mWFE1VDd4kEXDLimUbkW7rRLLsmho8D5KehdEclBTU4Ndu3Zh1ixv+NKsWbOwa9cuVFdXt/d0XRqRZ+Muxl7HbuHh9qioafCAEBqJQ6F0N9plmRiGwVNPPYUbb7wRdrsdDocDzz//PEpKSpCbmwuO88oZHMchJycHJSUlyMzMjMrAuyI8z6HjlfqOwWrxyjiVtXode2rsKZTuRLuMvaIoeO655/DMM89g1KhR+OGHH7Bo0SI8+uijURlcoIUGM2Rnp0RlDOFgt/IQeLZTzq0Tq3NnpdvhlirhaVofGnxSD2RnJcXkXJHQmZ95ZxKreZeXs+C7cCmGrjy2WOI7b5YNz9a0y9jv3r0b5eXlGDXKG5UxatQo2Gw2WCwWlJWVQVVVcBwHVVVRXl6O/PzwkjK6WzSOhWdhFflOiwyJ6bw1DS6PgoPHasAwAJGVLhMBQ6Nxoo+maV024oVG43jRNK3N9Y9ZNE5eXh5KS0tx8OBBAMCBAwdQVVWFfv36oaCgAKtWrQIArFq1CgUFBXEt4QDA1ecV4LJzB3f2MGKCvhZxrNyBDBpjT6F0O9rl2WdnZ2Pp0qW49dZbjWL6Dz/8MNLT07F06VLcddddeOaZZ5Camoply5ZFZcBdmR7p8Ruhohv7o+WN6EEzZymUqPLQQ0txyikFuOiieTE7R7tDR4qLi1FcXNzm9YEDB+Ltt4OnIVO6D7qxr6p3Y0jf9M4dDIUSZRRFAc93TCRdR57Ll/iME6REHd9EMRqJQ+lozjqrCNdd93t89dV61NfX4U9/uhtbtnyPTZu+gaIoeOCBZejffwCqqiqxdOndcDgckCQJZ545HjfeeGvAY/rWrV+w4PK4rpFPjT3FFDax+VahpY0TD3nf15D3bojJsYUhEyAMDl0pMjk5BS+88ArWr/8cf/7zH7F06cNYuPAmvPbav/HKK//Cvfc+gOTkFCxb9iTsdjsURcFtt92E7777BuPGnen3mL516x955IG4rpFPjT3FFL7JYrRpCaUzmDr1XADAkCGnAGAwfvzZTX8XYP36dQC8ESrPPPN3/PzzTwAIqqqqsH//voDGPpFq5FNjTzFFS2NPPftEQxg83pT3HUtEUQQAsCwHUWxOX2RZ1ih1/Oabr6GhoR7PP/8yLBYLli17CJLkCXjMRKqRT+PnKKbQyzazDIPMVEsnj4ZC8U9DQwOysnrAYrGgoqIcGzeuN/3eeK+RTz17iilEgQXLMMhIscRloTdKfHDxxfNxzz134vLLL0F2di5GjRod+k1NxHuNfFrPPo6I9bxvfmoDemcn487fjozZOSKBXu/oQ+vZdz06vZ49JXHIybChb25i1qChULo7VMahmOZPC0aCYxOz5ymF0t2hxp5iGosQfx24KJREgco4FArFL114OS/hieTaUGNPoVDawPMiHI56avC7IIQQOBz14HkxrPdRGYdCobQhIyMbNTUVaGys7eyhtIFl2aiVEOhO+M6b50VkZGSH9X5q7CkUShs4jkePHuE1G+ooaKhtZFAZh0KhUBIAauwpFAolAejSMg7bjpju9ry3O0PnnVjQeScWoeYdbHuXLpdAoVAolOhAZRwKhUJJAKixp1AolASAGnsKhUJJAKixp1AolASAGnsKhUJJAKixp1AolASAGnsKhUJJAKixp1AolASAGnsKhUJJAOLK2B86dAjz5s3D9OnTMW/ePBw+fLizhxQTli1bhilTpmDIkCHYt2+f8Xq8z7+mpgbXXXcdpk+fjtmzZ+Omm25CdXU1AGDbtm0oLi7G9OnTcfXVV6OqqqqTRxtdbrzxRhQXF2Pu3LlYsGABdu/eDSD+r7nOP/7xjxb3e7xf7ylTpmDGjBmYM2cO5syZg6+++gpAO+dN4ojLL7+cvP/++4QQQt5//31y+eWXd/KIYsPmzZvJiRMnyOTJk8nevXuN1+N9/jU1NeS7774z/n7kkUfIn//8Z6KqKpk2bRrZvHkzIYSQFStWkLvuuquzhhkT6uvrjf9/+umnZO7cuYSQ+L/mhBCyY8cOcs011xj3eyJc79bfbUJIu+cdN559VVUVdu3ahVmzZgEAZs2ahV27dhmeXzxRVFSE/PyWtcYTYf7p6ekYO3as8feIESNw4sQJ7NixAxaLBUVFRQCA+fPn45NPPumsYcaElJQU4/+NjY1gGCYhrrkkSbj//vuxdOlS47VEuN7+aO+8u3TVy3AoKSlBbm4uOM7bFJvjOOTk5KCkpASZmZmdPLrYk2jz1zQNr7/+OqZMmYKSkhL07NnT2JaZmQlN01BbW4v09PTOG2SUufvuu/H111+DEIIXXnghIa753//+dxQXF6N3797Ga4lyvW+//XYQQjBq1Cjcdttt7Z533Hj2lMTigQcegN1ux2WXXdbZQ+kwHnroIXz55Zf4wx/+gEcffbSzhxNztm7dih07dmDBggWdPZQO57XXXsOHH36Id955B4QQ3H///e0+ZtwY+/z8fJSVlUFVVQCAqqooLy9vI3fEK4k0/2XLluHIkSN46qmnwLIs8vPzceLECWN7dXU1WJaNKy/Pl7lz52LTpk3Iy8uL62u+efNmHDhwAFOnTsWUKVNQWlqKa665BkeOHIn7661fQ1EUsWDBAvz444/tvs/jxthnZWWhoKAAq1atAgCsWrUKBQUFcfM4G4pEmf8TTzyBHTt2YMWKFRBFEQBw6qmnwu12Y8uWLQCAN954AzNmzOjMYUYVh8OBkpIS4+8vvvgCaWlpcX/Nr7/+emzcuBFffPEFvvjiC+Tl5eHFF1/EtddeG9fX2+l0oqHB22uWEIKPPvoIBQUF7b7P46p5yYEDB3DXXXehvr4eqampWLZsGU466aTOHlbUefDBB7F27VpUVlYiIyMD6enpWL16ddzPf//+/Zg1axb69+8Pq9UKAOjduzdWrFiBH3/8EUuWLIHH40GvXr3w2GOPoUePHp084uhQWVmJG2+8ES6XCyzLIi0tDXfeeSeGDRsW99fclylTpuDZZ5/F4MGD4/p6Hz16FDfffDNUVYWmaRg4cCAWL16MnJycds07row9hUKhUPwTNzIOhUKhUAJDjT2FQqEkANTYUygUSgJAjT2FQqEkANTYUygUSgJAjT2FEoRnn30Wd999d0Tvveuuu/Dkk09GeUQUSmTETW0cCiUWLFy4sLOHQKFEBerZUygUSgJAjT0lrigrK8PNN9+McePGYcqUKXjllVcAAMuXL8ctt9yCRYsWobCwEBdccAH27NljvO/555/H2WefjcLCQkyfPh3ffvut8b7bb7/d2O/zzz/H+eefj6KiIlx++eU4cOCAsW3Xrl244IILUFhYiEWLFsHj8bQY27p16zBnzhwUFRVh/vz5ps5PoUSNqFTap1C6AKqqkgsuuIAsX76ceDwe8uuvv5IpU6aQDRs2kKeffpoMHTqUfPzxx0SSJPLCCy+QyZMnE0mSyIEDB8iECRNIaWkpIYSQo0ePkiNHjhBCCHn66afJH//4R0IIIQcPHiTDhw8nGzduJJIkkeeff55MmzaNeDwe4vF4yKRJk8hLL71EJEkiH3/8MRk6dCh54oknCCGE7Ny5k4wbN45s27aNKIpC3n33XTJ58mTi8XiCnp9CiRbUs6fEDT///DOqq6tx0003QRRF9OnTB5dccgk++ugjAMCwYcMwY8YMCIKAq666CpIkYfv27eA4DpIk4cCBA5BlGb1790bfvn3bHP+jjz7CxIkTMX78eAiCgGuuuQZutxtbt27F9u3bIcsyrrzySgiCgBkzZuC0004z3vvmm29i3rx5GD58ODiOwwUXXABBELBt2zbT56dQ2gNdoKXEDcePH0d5ebnRyQfwlv0tKipCz549kZeXZ7zOsixyc3ON/f/yl79g+fLl+OWXX3DWWWfhrrvuQm5ubovjl5eXt2geoZdXLisrA8dxyM3NBcMwxnbffU+cOIH3338fr776qvGaLMsoLy/HmDFjTJ2fQmkP1LOnxA35+fno3bs3tmzZYvzbunUr/vnPfwIASktLjX01TUNZWRlycnIAALNnz8brr7+OdevWgWEYPP74422On5OT06KeOCHE6BaVnZ2NsrIyEJ+6gr775ufnY+HChS3Gtn37dqOloJnzUyjtgRp7Stxw+umnIykpCc8//zzcbjdUVcW+ffvw008/AQB27tyJtWvXQlEU/Pvf/4Yoihg+fDgOHjyIb7/9FpIkQRRFWCwWsGzbr8bMmTOxfv16fPvtt5BlGf/6178giiIKCwsxYsQI8DyPV155BbIsY+3atfj555+N91588cV44403sH37dhBC4HQ68eWXX6KxsdH0+SmU9kBlHErcwHEcnn32WSxbtgxTp06FJEkYMGAAFi1aBACYOnUqPvroI9x5553o168fli9fDkEQIEkS/va3v+HAgQMQBAGFhYV+28CddNJJeOyxx/DAAw+grKwMBQUFePbZZ40mKsuXL8c999yDp556ChMnTsQ555xjvPe0007DAw88gPvvvx9HjhyB1WrFyJEjUVRUZPr8FEp7oPXsKQnB8uXLceTIESqPUBIW+qxIoVAoCQA19hQKhZIAUBmHQqFQEgDq2VMoFEoCQI09hUKhJADU2FMoFEoCQI09hUKhJADU2FMoFEoCQI09hUKhJAD/HyGIMZHomwikAAAAAElFTkSuQmCC\n" + }, + "metadata": {} + } + ], + "source": [ + "if __name__ == '__main__':\n", + " cfg = PPOConfig()\n", + " # train\n", + " env,agent = env_agent_config(cfg,seed=1)\n", + " rewards, ma_rewards = train(cfg, env, agent)\n", + " make_dir(cfg.result_path, cfg.model_path)\n", + " agent.save(path=cfg.model_path)\n", + " save_results(rewards, ma_rewards, tag='train', path=cfg.result_path)\n", + " plot_rewards(rewards, ma_rewards, tag=\"train\",\n", + " algo=cfg.algo, path=cfg.result_path)\n", + " # eval\n", + " env,agent = env_agent_config(cfg,seed=10)\n", + " agent.load(path=cfg.model_path)\n", + " rewards,ma_rewards = eval(cfg,env,agent)\n", + " save_results(rewards,ma_rewards,tag='eval',path=cfg.result_path)\n", + " plot_rewards(rewards,ma_rewards,tag=\"eval\",env=cfg.env,algo = cfg.algo,path=cfg.result_path)" + ] + } + ] +} \ No newline at end of file diff --git a/codes/PPO/main.py b/codes/PPO/task0_train.py similarity index 54% rename from codes/PPO/main.py rename to codes/PPO/task0_train.py index 85febef..a4600e4 100644 --- a/codes/PPO/main.py +++ b/codes/PPO/task0_train.py @@ -5,7 +5,7 @@ Author: John Email: johnjim0816@gmail.com Date: 2021-03-22 16:18:10 LastEditor: John -LastEditTime: 2021-04-28 10:13:00 +LastEditTime: 2021-05-06 00:43:36 Discription: Environment: ''' @@ -13,8 +13,8 @@ import sys,os curr_path = os.path.dirname(__file__) parent_path=os.path.dirname(curr_path) sys.path.append(parent_path) # add current terminal path to sys.path + import gym -import numpy as np import torch import datetime from PPO.agent import PPO @@ -29,6 +29,8 @@ class PPOConfig: self.algo = 'PPO' self.result_path = curr_path+"/results/" +self.env+'/'+curr_time+'/results/' # path to save results self.model_path = curr_path+"/results/" +self.env+'/'+curr_time+'/models/' # path to save models + self.train_eps = 200 # max training episodes + self.eval_eps = 50 self.batch_size = 5 self.gamma=0.99 self.n_epochs = 4 @@ -38,10 +40,19 @@ class PPOConfig: self.policy_clip=0.2 self.hidden_dim = 256 self.update_fre = 20 # frequency of agent update - self.train_eps = 300 # max training episodes self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # check gpu - + +def env_agent_config(cfg,seed=1): + env = gym.make(cfg.env) + env.seed(seed) + state_dim = env.observation_space.shape[0] + action_dim = env.action_space.n + agent = PPO(state_dim,action_dim,cfg) + return env,agent + def train(cfg,env,agent): + print('Start to train !') + print(f'Env:{cfg.env}, Algorithm:{cfg.algo}, Device:{cfg.device}') rewards= [] ma_rewards = [] # moving average rewards running_steps = 0 @@ -65,17 +76,46 @@ def train(cfg,env,agent): else: ma_rewards.append(ep_reward) print(f"Episode:{i_ep+1}/{cfg.train_eps}, Reward:{ep_reward:.3f}") + print('Complete training！') return rewards,ma_rewards +def eval(cfg,env,agent): + print('Start to eval !') + print(f'Env:{cfg.env}, Algorithm:{cfg.algo}, Device:{cfg.device}') + rewards= [] + ma_rewards = [] # moving average rewards + for i_ep in range(cfg.eval_eps): + state = env.reset() + done = False + ep_reward = 0 + while not done: + action, prob, val = agent.choose_action(state) + state_, reward, done, _ = env.step(action) + ep_reward += reward + state = state_ + rewards.append(ep_reward) + if ma_rewards: + ma_rewards.append( + 0.9*ma_rewards[-1]+0.1*ep_reward) + else: + ma_rewards.append(ep_reward) + print(f"Episode:{i_ep+1}/{cfg.train_eps}, Reward:{ep_reward:.3f}") + print('Complete evaling！') + return rewards,ma_rewards + if __name__ == '__main__': cfg = PPOConfig() - env = gym.make(cfg.env) - env.seed(1) # Set seeds - state_dim=env.observation_space.shape[0] - action_dim=env.action_space.n - agent = PPO(state_dim,action_dim,cfg) - rewards,ma_rewards = train(cfg,env,agent) - make_dir(cfg.result_path,cfg.model_path) + # train + env,agent = env_agent_config(cfg,seed=1) + rewards, ma_rewards = train(cfg, env, agent) + make_dir(cfg.result_path, cfg.model_path) agent.save(path=cfg.model_path) - save_results(rewards,ma_rewards,tag='train',path=cfg.result_path) - plot_rewards(rewards,ma_rewards,tag="train",env=cfg.env,algo = cfg.algo,path=cfg.result_path) \ No newline at end of file + save_results(rewards, ma_rewards, tag='train', path=cfg.result_path) + plot_rewards(rewards, ma_rewards, tag="train", + algo=cfg.algo, path=cfg.result_path) + # eval + env,agent = env_agent_config(cfg,seed=10) + agent.load(path=cfg.model_path) + rewards,ma_rewards = eval(cfg,env,agent) + save_results(rewards,ma_rewards,tag='eval',path=cfg.result_path) + plot_rewards(rewards,ma_rewards,tag="eval",env=cfg.env,algo = cfg.algo,path=cfg.result_path) diff --git a/codes/PPO/task1.py b/codes/PPO/task1.py deleted file mode 100644 index 6972edf..0000000 --- a/codes/PPO/task1.py +++ /dev/null @@ -1,97 +0,0 @@ -#!/usr/bin/env python -# coding=utf-8 -''' -Author: John -Email: johnjim0816@gmail.com -Date: 2021-03-22 16:18:10 -LastEditor: John -LastEditTime: 2021-04-11 01:25:43 -Discription: -Environment: -''' -import sys,os -curr_path = os.path.dirname(__file__) -parent_path=os.path.dirname(curr_path) -sys.path.append(parent_path) # add current terminal path to sys.path -import gym -import numpy as np -import torch -import datetime -from PPO.agent import PPO -from common.plot import plot_rewards -from common.utils import save_results - -SEQUENCE = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") # 获取当前时间 -SAVED_MODEL_PATH = os.path.split(os.path.abspath(__file__))[0]+"/saved_model/"+SEQUENCE+'/' # 生成保存的模型路径 -if not os.path.exists(os.path.split(os.path.abspath(__file__))[0]+"/saved_model/"): # 检测是否存在文件夹 - os.mkdir(os.path.split(os.path.abspath(__file__))[0]+"/saved_model/") -if not os.path.exists(SAVED_MODEL_PATH): # 检测是否存在文件夹 - os.mkdir(SAVED_MODEL_PATH) -RESULT_PATH = os.path.split(os.path.abspath(__file__))[0]+"/results/"+SEQUENCE+'/' # 存储reward的路径 -if not os.path.exists(os.path.split(os.path.abspath(__file__))[0]+"/results/"): # 检测是否存在文件夹 - os.mkdir(os.path.split(os.path.abspath(__file__))[0]+"/results/") -if not os.path.exists(RESULT_PATH): # 检测是否存在文件夹 - os.mkdir(RESULT_PATH) - -class PPOConfig: - def __init__(self) -> None: - self.env = 'LunarLander-v2' - self.algo = 'PPO' - self.batch_size = 128 - self.gamma=0.95 - self.n_epochs = 4 - self.actor_lr = 0.002 - self.critic_lr = 0.005 - self.gae_lambda=0.95 - self.policy_clip=0.2 - self.hidden_dim = 256 - self.update_fre = 20 # frequency of agent update - self.train_eps = 300 # max training episodes - self.train_steps = 1000 - self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # check gpu - -def train(cfg,env,agent): - best_reward = env.reward_range[0] - rewards= [] - ma_rewards = [] # moving average rewards - avg_reward = 0 - running_steps = 0 - for i_episode in range(cfg.train_eps): - state = env.reset() - done = False - ep_reward = 0 - # for i_step in range(cfg.train_steps): - while not done: - action, prob, val = agent.choose_action(state) - state_, reward, done, _ = env.step(action) - running_steps += 1 - ep_reward += reward - agent.memory.push(state, action, prob, val, reward, done) - if running_steps % cfg.update_fre == 0: - agent.update() - state = state_ - # if done: - # break - rewards.append(ep_reward) - if ma_rewards: - ma_rewards.append( - 0.9*ma_rewards[-1]+0.1*ep_reward) - else: - ma_rewards.append(ep_reward) - avg_reward = np.mean(rewards[-100:]) - if avg_reward > best_reward: - best_reward = avg_reward - agent.save(path=SAVED_MODEL_PATH) - print('Episode:{}/{}, Reward:{:.1f}, avg reward:{:.1f}, Loss:{}'.format(i_episode+1,cfg.train_eps,ep_reward,avg_reward,agent.loss)) - return rewards,ma_rewards - -if __name__ == '__main__': - cfg = PPOConfig() - env = gym.make(cfg.env) - env.seed(1) - state_dim=env.observation_space.shape[0] - action_dim=env.action_space.n - agent = PPO(state_dim,action_dim,cfg) - rewards,ma_rewards = train(cfg,env,agent) - save_results(rewards,ma_rewards,tag='train',path=RESULT_PATH) - plot_rewards(rewards,ma_rewards,tag="train",algo = cfg.algo,path=RESULT_PATH) \ No newline at end of file diff --git a/codes/PolicyGradient/agent.py b/codes/PolicyGradient/agent.py index 997f4ae..be67601 100644 --- a/codes/PolicyGradient/agent.py +++ b/codes/PolicyGradient/agent.py @@ -5,7 +5,7 @@ Author: John Email: johnjim0816@gmail.com Date: 2020-11-22 23:27:44 LastEditor: John -LastEditTime: 2021-03-23 16:37:14 +LastEditTime: 2021-05-05 17:33:10 Discription: Environment: ''' @@ -64,7 +64,7 @@ class PolicyGradient: # print(loss) loss.backward() self.optimizer.step() - def save_model(self,path): + def save(self,path): torch.save(self.policy_net.state_dict(), path+'pg_checkpoint.pt') - def load_model(self,path): + def load(self,path): self.policy_net.load_state_dict(torch.load(path+'pg_checkpoint.pt')) \ No newline at end of file diff --git a/codes/PolicyGradient/main.py b/codes/PolicyGradient/main.py deleted file mode 100644 index a35be7a..0000000 --- a/codes/PolicyGradient/main.py +++ /dev/null @@ -1,89 +0,0 @@ -#!/usr/bin/env python -# coding=utf-8 -''' -Author: John -Email: johnjim0816@gmail.com -Date: 2020-11-22 23:21:53 -LastEditor: John -LastEditTime: 2021-03-23 16:38:54 -Discription: -Environment: -''' -import sys,os -sys.path.append(os.getcwd()) # add current terminal path to sys.path -from itertools import count -import datetime -import gym -from PolicyGradient.agent import PolicyGradient -from common.plot import plot_rewards -from common.utils import save_results - -SEQUENCE = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") # obtain current time -SAVED_MODEL_PATH = os.path.split(os.path.abspath(__file__))[0]+"/saved_model/"+SEQUENCE+'/' # path to save model -if not os.path.exists(os.path.split(os.path.abspath(__file__))[0]+"/saved_model/"): - os.mkdir(os.path.split(os.path.abspath(__file__))[0]+"/saved_model/") -if not os.path.exists(SAVED_MODEL_PATH): - os.mkdir(SAVED_MODEL_PATH) -RESULT_PATH = os.path.split(os.path.abspath(__file__))[0]+"/results/"+SEQUENCE+'/' # path to save rewards -if not os.path.exists(os.path.split(os.path.abspath(__file__))[0]+"/results/"): - os.mkdir(os.path.split(os.path.abspath(__file__))[0]+"/results/") -if not os.path.exists(RESULT_PATH): - os.mkdir(RESULT_PATH) - -class PGConfig: - def __init__(self): - self.train_eps = 300 # 训练的episode数目 - self.batch_size = 8 - self.lr = 0.01 # learning rate - self.gamma = 0.99 - self.hidden_dim = 36 # dimmension of hidden layer - -def train(cfg,env,agent): - '''下面带pool都是存放的transition序列用于gradient''' - state_pool = [] # 存放每batch_size个episode的state序列 - action_pool = [] - reward_pool = [] - ''' 存储每个episode的reward用于绘图''' - rewards = [] - ma_rewards = [] - for i_episode in range(cfg.train_eps): - state = env.reset() - ep_reward = 0 - for _ in count(): - action = agent.choose_action(state) # 根据当前环境state选择action - next_state, reward, done, _ = env.step(action) - ep_reward += reward - if done: - reward = 0 - state_pool.append(state) - action_pool.append(float(action)) - reward_pool.append(reward) - state = next_state - if done: - print('Episode:', i_episode, ' Reward:', ep_reward) - break - if i_episode > 0 and i_episode % cfg.batch_size == 0: - agent.update(reward_pool,state_pool,action_pool) - state_pool = [] # 每个episode的state - action_pool = [] - reward_pool = [] - rewards.append(ep_reward) - if ma_rewards: - ma_rewards.append( - 0.9*ma_rewards[-1]+0.1*ep_reward) - else: - ma_rewards.append(ep_reward) - print('complete training！') - return rewards, ma_rewards - -if __name__ == "__main__": - cfg = PGConfig() - env = gym.make('CartPole-v0') # 可google为什么unwrapped gym，此处一般不需要 - env.seed(1) # 设置env随机种子 - state_dim = env.observation_space.shape[0] - action_dim = env.action_space.n - agent = PolicyGradient(state_dim,cfg) - rewards, ma_rewards = train(cfg,env,agent) - agent.save_model(SAVED_MODEL_PATH) - save_results(rewards,ma_rewards,tag='train',path=RESULT_PATH) - plot_rewards(rewards,ma_rewards,tag="train",algo = "Policy Gradient",path=RESULT_PATH) diff --git a/codes/PolicyGradient/outputs/CartPole-v0/20210505-173524/models/pg_checkpoint.pt b/codes/PolicyGradient/outputs/CartPole-v0/20210505-173524/models/pg_checkpoint.pt new file mode 100644 index 0000000..2ea029d Binary files /dev/null and b/codes/PolicyGradient/outputs/CartPole-v0/20210505-173524/models/pg_checkpoint.pt differ diff --git a/codes/PolicyGradient/outputs/CartPole-v0/20210505-173524/results/eval_ma_rewards.npy b/codes/PolicyGradient/outputs/CartPole-v0/20210505-173524/results/eval_ma_rewards.npy new file mode 100644 index 0000000..a8a5243 Binary files /dev/null and b/codes/PolicyGradient/outputs/CartPole-v0/20210505-173524/results/eval_ma_rewards.npy differ diff --git a/codes/PolicyGradient/outputs/CartPole-v0/20210505-173524/results/eval_rewards.npy b/codes/PolicyGradient/outputs/CartPole-v0/20210505-173524/results/eval_rewards.npy new file mode 100644 index 0000000..a8a5243 Binary files /dev/null and b/codes/PolicyGradient/outputs/CartPole-v0/20210505-173524/results/eval_rewards.npy differ diff --git a/codes/PolicyGradient/outputs/CartPole-v0/20210505-173524/results/eval_rewards_curve.png b/codes/PolicyGradient/outputs/CartPole-v0/20210505-173524/results/eval_rewards_curve.png new file mode 100644 index 0000000..2c19fd2 Binary files /dev/null and b/codes/PolicyGradient/outputs/CartPole-v0/20210505-173524/results/eval_rewards_curve.png differ diff --git a/codes/PolicyGradient/outputs/CartPole-v0/20210505-173524/results/train_ma_rewards.npy b/codes/PolicyGradient/outputs/CartPole-v0/20210505-173524/results/train_ma_rewards.npy new file mode 100644 index 0000000..3238411 Binary files /dev/null and b/codes/PolicyGradient/outputs/CartPole-v0/20210505-173524/results/train_ma_rewards.npy differ diff --git a/codes/PolicyGradient/outputs/CartPole-v0/20210505-173524/results/train_rewards.npy b/codes/PolicyGradient/outputs/CartPole-v0/20210505-173524/results/train_rewards.npy new file mode 100644 index 0000000..3450bf8 Binary files /dev/null and b/codes/PolicyGradient/outputs/CartPole-v0/20210505-173524/results/train_rewards.npy differ diff --git a/codes/PolicyGradient/outputs/CartPole-v0/20210505-173524/results/train_rewards_curve.png b/codes/PolicyGradient/outputs/CartPole-v0/20210505-173524/results/train_rewards_curve.png new file mode 100644 index 0000000..5fee65a Binary files /dev/null and b/codes/PolicyGradient/outputs/CartPole-v0/20210505-173524/results/train_rewards_curve.png differ diff --git a/codes/PolicyGradient/results/20210313-114904/ma_rewards_train.npy b/codes/PolicyGradient/results/20210313-114904/ma_rewards_train.npy deleted file mode 100644 index cd5c266..0000000 Binary files a/codes/PolicyGradient/results/20210313-114904/ma_rewards_train.npy and /dev/null differ diff --git a/codes/PolicyGradient/results/20210313-114904/rewards_curve_train.png b/codes/PolicyGradient/results/20210313-114904/rewards_curve_train.png deleted file mode 100644 index 6786b02..0000000 Binary files a/codes/PolicyGradient/results/20210313-114904/rewards_curve_train.png and /dev/null differ diff --git a/codes/PolicyGradient/results/20210313-114904/rewards_train.npy b/codes/PolicyGradient/results/20210313-114904/rewards_train.npy deleted file mode 100644 index 710328a..0000000 Binary files a/codes/PolicyGradient/results/20210313-114904/rewards_train.npy and /dev/null differ diff --git a/codes/PolicyGradient/saved_model/20210313-114904/pg_checkpoint.pth b/codes/PolicyGradient/saved_model/20210313-114904/pg_checkpoint.pth deleted file mode 100644 index 7f5a671..0000000 Binary files a/codes/PolicyGradient/saved_model/20210313-114904/pg_checkpoint.pth and /dev/null differ diff --git a/codes/PolicyGradient/task0_train.py b/codes/PolicyGradient/task0_train.py new file mode 100644 index 0000000..c1f4e5c --- /dev/null +++ b/codes/PolicyGradient/task0_train.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python +# coding=utf-8 +''' +Author: John +Email: johnjim0816@gmail.com +Date: 2020-11-22 23:21:53 +LastEditor: John +LastEditTime: 2021-05-05 17:35:20 +Discription: +Environment: +''' +import sys,os +curr_path = os.path.dirname(__file__) +parent_path = os.path.dirname(curr_path) +sys.path.append(parent_path) # add current terminal path to sys.path + +import gym +import torch +import datetime +from itertools import count + +from PolicyGradient.agent import PolicyGradient +from common.plot import plot_rewards +from common.utils import save_results,make_dir + +curr_time = datetime.datetime.now().strftime( + "%Y%m%d-%H%M%S") # obtain current time + +class PGConfig: + def __init__(self): + self.algo = "PolicyGradient" # name of algo + self.env = 'CartPole-v0' + self.result_path = curr_path+"/outputs/" + self.env + \ + '/'+curr_time+'/results/' # path to save results + self.model_path = curr_path+"/outputs/" + self.env + \ + '/'+curr_time+'/models/' # path to save models + self.train_eps = 300 # 训练的episode数目 + self.eval_eps = 50 + self.batch_size = 8 + self.lr = 0.01 # learning rate + self.gamma = 0.99 + self.hidden_dim = 36 # dimmension of hidden layer + self.device = torch.device( + "cuda" if torch.cuda.is_available() else "cpu") # check gpu + + +def env_agent_config(cfg,seed=1): + env = gym.make(cfg.env) + env.seed(seed) + state_dim = env.observation_space.shape[0] + agent = PolicyGradient(state_dim,cfg) + return env,agent + +def train(cfg,env,agent): + print('Start to eval !') + print(f'Env:{cfg.env}, Algorithm:{cfg.algo}, Device:{cfg.device}') + state_pool = [] # 存放每batch_size个episode的state序列 + action_pool = [] + reward_pool = [] + rewards = [] + ma_rewards = [] + for i_episode in range(cfg.train_eps): + state = env.reset() + ep_reward = 0 + for _ in count(): + action = agent.choose_action(state) # 根据当前环境state选择action + next_state, reward, done, _ = env.step(action) + ep_reward += reward + if done: + reward = 0 + state_pool.append(state) + action_pool.append(float(action)) + reward_pool.append(reward) + state = next_state + if done: + print('Episode:', i_episode, ' Reward:', ep_reward) + break + if i_episode > 0 and i_episode % cfg.batch_size == 0: + agent.update(reward_pool,state_pool,action_pool) + state_pool = [] # 每个episode的state + action_pool = [] + reward_pool = [] + rewards.append(ep_reward) + if ma_rewards: + ma_rewards.append( + 0.9*ma_rewards[-1]+0.1*ep_reward) + else: + ma_rewards.append(ep_reward) + print('complete training！') + return rewards, ma_rewards + + +def eval(cfg,env,agent): + print('Start to eval !') + print(f'Env:{cfg.env}, Algorithm:{cfg.algo}, Device:{cfg.device}') + rewards = [] + ma_rewards = [] + for i_episode in range(cfg.eval_eps): + state = env.reset() + ep_reward = 0 + for _ in count(): + action = agent.choose_action(state) # 根据当前环境state选择action + next_state, reward, done, _ = env.step(action) + ep_reward += reward + if done: + reward = 0 + state = next_state + if done: + print('Episode:', i_episode, ' Reward:', ep_reward) + break + rewards.append(ep_reward) + if ma_rewards: + ma_rewards.append( + 0.9*ma_rewards[-1]+0.1*ep_reward) + else: + ma_rewards.append(ep_reward) + print('complete evaling！') + return rewards, ma_rewards +if __name__ == "__main__": + cfg = PGConfig() + + # train + env,agent = env_agent_config(cfg,seed=1) + rewards, ma_rewards = train(cfg, env, agent) + make_dir(cfg.result_path, cfg.model_path) + agent.save(path=cfg.model_path) + save_results(rewards, ma_rewards, tag='train', path=cfg.result_path) + plot_rewards(rewards, ma_rewards, tag="train", + algo=cfg.algo, path=cfg.result_path) + # eval + env,agent = env_agent_config(cfg,seed=10) + agent.load(path=cfg.model_path) + rewards,ma_rewards = eval(cfg,env,agent) + save_results(rewards,ma_rewards,tag='eval',path=cfg.result_path) + plot_rewards(rewards,ma_rewards,tag="eval",env=cfg.env,algo = cfg.algo,path=cfg.result_path) + diff --git a/codes/QLearning/task0_train.py b/codes/QLearning/task0_train.py index 73fedae..f036162 100644 --- a/codes/QLearning/task0_train.py +++ b/codes/QLearning/task0_train.py @@ -5,7 +5,7 @@ Author: John Email: johnjim0816@gmail.com Date: 2020-09-11 23:03:00 LastEditor: John -LastEditTime: 2021-04-29 17:01:08 +LastEditTime: 2021-05-06 17:04:38 Discription: Environment: ''' @@ -15,6 +15,7 @@ parent_path=os.path.dirname(curr_path) sys.path.append(parent_path) # add current terminal path to sys.path import gym +import torch import datetime from envs.gridworld_env import CliffWalkingWapper @@ -37,6 +38,8 @@ class QlearningConfig: self.epsilon_end = 0.01 # e-greedy策略中的终止epsilon self.epsilon_decay = 200 # e-greedy策略中epsilon的衰减率 self.lr = 0.1 # learning rate + self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # check gpu + def env_agent_config(cfg,seed=1): env = gym.make(cfg.env) @@ -48,6 +51,8 @@ def env_agent_config(cfg,seed=1): return env,agent def train(cfg,env,agent): + print('Start to train !') + print(f'Env:{cfg.env}, Algorithm:{cfg.algo}, Device:{cfg.device}') rewards = [] ma_rewards = [] # moving average reward for i_ep in range(cfg.train_eps): @@ -67,11 +72,14 @@ def train(cfg,env,agent): else: ma_rewards.append(ep_reward) print("Episode:{}/{}: reward:{:.1f}".format(i_ep+1, cfg.train_eps,ep_reward)) + print('Complete training！') return rewards,ma_rewards def eval(cfg,env,agent): # env = gym.make("FrozenLake-v0", is_slippery=False) # 0 left, 1 down, 2 right, 3 up # env = FrozenLakeWapper(env) + print('Start to eval !') + print(f'Env:{cfg.env}, Algorithm:{cfg.algo}, Device:{cfg.device}') rewards = [] # 记录所有episode的reward ma_rewards = [] # 滑动平均的reward for i_ep in range(cfg.eval_eps): @@ -90,6 +98,7 @@ def eval(cfg,env,agent): else: ma_rewards.append(ep_reward) print(f"Episode:{i_ep+1}/{cfg.eval_eps}, reward:{ep_reward:.1f}") + print('Complete evaling！') return rewards,ma_rewards if __name__ == "__main__": diff --git a/codes/README_en.md b/codes/README_en.md index 5e9a30c..5b2f707 100644 --- a/codes/README_en.md +++ b/codes/README_en.md @@ -21,9 +21,6 @@ Note that ```model.py```,```memory.py```,```plot.py``` shall be utilized in diff python 3.7、pytorch 1.6.0-1.7.1、gym 0.17.0-0.18.0 ## Usage -运行带有```train```的py文件或ipynb文件进行训练，如果前面带有```task```如```task0_train.py```，表示对task0任务训练 -类似的带有```eval```即为测试。 - run python scripts or jupyter notebook file with ```train``` to train the agent, if there is a ```task``` like ```task0_train.py```, it means to train with task 0. similar to file with ```eval```, which means to evaluate the agent. @@ -36,7 +33,7 @@ similar to file with ```eval```, which means to evaluate the agent. | [Q-Learning](./QLearning) | [towardsdatascience blog](https://towardsdatascience.com/simple-reinforcement-learning-q-learning-fcddc4b6fe56),[q learning paper](https://ieeexplore.ieee.org/document/8836506) | [CliffWalking-v0](./envs/gym_info.md) | | | [Sarsa](./Sarsa) | [geeksforgeeks blog](https://www.geeksforgeeks.org/sarsa-reinforcement-learning/) | [Racetrack](./envs/racetrack_env.md) | | | [DQN](./DQN) | [DQN Paper](https://www.cs.toronto.edu/~vmnih/docs/dqn.pdf),[Nature DQN Paper](https://www.nature.com/articles/nature14236) | [CartPole-v0](./envs/gym_info.md) | | -| [DQN-cnn](./DQN_cnn) | [DQN Paper](https://www.cs.toronto.edu/~vmnih/docs/dqn.pdf) | [CartPole-v0](./envs/gym_info.md) | 与DQN相比使用了CNN而不是全链接网络 | +| [DQN-cnn](./DQN_cnn) | [DQN Paper](https://www.cs.toronto.edu/~vmnih/docs/dqn.pdf) | [CartPole-v0](./envs/gym_info.md) | | | [DoubleDQN](./DoubleDQN) | [DoubleDQN Paper](https://arxiv.org/abs/1509.06461) | [CartPole-v0](./envs/gym_info.md) | | | [Hierarchical DQN](HierarchicalDQN) | [H-DQN Paper](https://arxiv.org/abs/1604.06057) | [CartPole-v0](./envs/gym_info.md) | | | [PolicyGradient](./PolicyGradient) | [Lil'log](https://lilianweng.github.io/lil-log/2018/04/08/policy-gradient-algorithms.html) | [CartPole-v0](./envs/gym_info.md) | | diff --git a/codes/RandomPolicy/main.py b/codes/RandomPolicy/main.py deleted file mode 100644 index 897cc35..0000000 --- a/codes/RandomPolicy/main.py +++ /dev/null @@ -1,108 +0,0 @@ -#!/usr/bin/env python -# coding=utf-8 -''' -Author: JiangJi -Email: johnjim0816@gmail.com -Date: 2021-04-21 11:07:57 -LastEditor: JiangJi -LastEditTime: 2021-04-21 11:15:00 -Discription: -Environment: -''' -import sys,os -curr_path = os.path.dirname(__file__) -parent_path=os.path.dirname(curr_path) -sys.path.append(parent_path) # add current terminal path to sys.path - -import torch -import gym -import numpy as np -import datetime - -from common.plot import plot_rewards -from common.utils import save_results,make_dir - -curr_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") # obtain current time - - -class TD3Config: - def __init__(self) -> None: - self.algo = 'TD3' - self.env = 'HalfCheetah-v2' - self.seed = 0 - self.result_path = curr_path+"/results/" +self.env+'/'+curr_time+'/results/' # path to save results - self.model_path = curr_path+"/results/" +self.env+'/'+curr_time+'/models/' # path to save models - self.eval_freq = 5e3 # How often (time steps) we evaluate - # self.train_eps = 800 - self.max_timestep = 4000000 # Max time steps to run environment - -# Runs policy for X episodes and returns average reward -# A fixed seed is used for the eval environment -def eval(env_name,seed, eval_episodes=10): - eval_env = gym.make(env_name) - eval_env.seed(seed + 100) - avg_reward = 0. - for _ in range(eval_episodes): - state, done = eval_env.reset(), False - while not done: - # eval_env.render() - action = eval_env.action_space.sample() - state, reward, done, _ = eval_env.step(action) - avg_reward += reward - avg_reward /= eval_episodes - print("---------------------------------------") - print(f"Evaluation over {eval_episodes} episodes: {avg_reward:.3f}") - print("---------------------------------------") - return avg_reward - -def train(cfg,env): - # Evaluate untrained policy - evaluations = [eval(cfg.env, cfg.seed)] - state, done = env.reset(), False - ep_reward = 0 - ep_timesteps = 0 - episode_num = 0 - rewards = [] - ma_rewards = [] # moveing average reward - for t in range(int(cfg.max_timestep)): - ep_timesteps += 1 - # Select action randomly - action = env.action_space.sample() - # Perform action - next_state, reward, done, _ = env.step(action) - state = next_state - ep_reward += reward - if done: - # +1 to account for 0 indexing. +0 on ep_timesteps since it will increment +1 even if done=True - print(f"Episode:{episode_num+1}, Episode T:{ep_timesteps}, Reward:{ep_reward:.3f}") - # Reset environment - state, done = env.reset(), False - rewards.append(ep_reward) - # 计算滑动窗口的reward - if ma_rewards: - ma_rewards.append(0.9*ma_rewards[-1]+0.1*ep_reward) - else: - ma_rewards.append(ep_reward) - ep_reward = 0 - ep_timesteps = 0 - episode_num += 1 - # Evaluate episode - if (t + 1) % cfg.eval_freq == 0: - evaluations.append(eval(cfg.env, cfg.seed)) - return rewards, ma_rewards - -if __name__ == "__main__": - cfg = TD3Config() - env = gym.make(cfg.env) - env.seed(cfg.seed) # Set seeds - torch.manual_seed(cfg.seed) - np.random.seed(cfg.seed) - rewards,ma_rewards = train(cfg,env) - make_dir(cfg.result_path) - save_results(rewards,ma_rewards,tag='train',path=cfg.result_path) - plot_rewards(rewards,ma_rewards,tag="train",env=cfg.env,algo = cfg.algo,path=cfg.result_path) - # cfg.result_path = './TD3/results/HalfCheetah-v2/20210416-130341/' - # agent.load(cfg.result_path) - # eval(cfg.env,agent, cfg.seed) - - diff --git a/codes/RandomPolicy/results/HalfCheetah-v2/20210421-111223/models/ma_rewards_train.npy b/codes/RandomPolicy/results/HalfCheetah-v2/20210421-111223/models/ma_rewards_train.npy deleted file mode 100644 index d542658..0000000 Binary files a/codes/RandomPolicy/results/HalfCheetah-v2/20210421-111223/models/ma_rewards_train.npy and /dev/null differ diff --git a/codes/RandomPolicy/results/HalfCheetah-v2/20210421-111223/models/rewards_curve_train.png b/codes/RandomPolicy/results/HalfCheetah-v2/20210421-111223/models/rewards_curve_train.png deleted file mode 100644 index d54cee0..0000000 Binary files a/codes/RandomPolicy/results/HalfCheetah-v2/20210421-111223/models/rewards_curve_train.png and /dev/null differ diff --git a/codes/RandomPolicy/results/HalfCheetah-v2/20210421-111223/models/rewards_train.npy b/codes/RandomPolicy/results/HalfCheetah-v2/20210421-111223/models/rewards_train.npy deleted file mode 100644 index 1fc3be1..0000000 Binary files a/codes/RandomPolicy/results/HalfCheetah-v2/20210421-111223/models/rewards_train.npy and /dev/null differ diff --git a/codes/SAC/outputs/Pendulum-v0/20210429-135700/models/sac_policy b/codes/SAC/outputs/Pendulum-v0/20210429-135700/models/sac_policy deleted file mode 100644 index ce119d4..0000000 Binary files a/codes/SAC/outputs/Pendulum-v0/20210429-135700/models/sac_policy and /dev/null differ diff --git a/codes/SAC/outputs/Pendulum-v0/20210429-135700/models/sac_policy_optimizer b/codes/SAC/outputs/Pendulum-v0/20210429-135700/models/sac_policy_optimizer deleted file mode 100644 index 348eca7..0000000 Binary files a/codes/SAC/outputs/Pendulum-v0/20210429-135700/models/sac_policy_optimizer and /dev/null differ diff --git a/codes/SAC/outputs/Pendulum-v0/20210429-135700/models/sac_soft_q b/codes/SAC/outputs/Pendulum-v0/20210429-135700/models/sac_soft_q deleted file mode 100644 index 3c4f237..0000000 Binary files a/codes/SAC/outputs/Pendulum-v0/20210429-135700/models/sac_soft_q and /dev/null differ diff --git a/codes/SAC/outputs/Pendulum-v0/20210429-135700/models/sac_soft_q_optimizer b/codes/SAC/outputs/Pendulum-v0/20210429-135700/models/sac_soft_q_optimizer deleted file mode 100644 index 52fac5b..0000000 Binary files a/codes/SAC/outputs/Pendulum-v0/20210429-135700/models/sac_soft_q_optimizer and /dev/null differ diff --git a/codes/SAC/outputs/Pendulum-v0/20210429-135700/models/sac_value b/codes/SAC/outputs/Pendulum-v0/20210429-135700/models/sac_value deleted file mode 100644 index 11989ad..0000000 Binary files a/codes/SAC/outputs/Pendulum-v0/20210429-135700/models/sac_value and /dev/null differ diff --git a/codes/SAC/outputs/Pendulum-v0/20210429-135700/models/sac_value_optimizer b/codes/SAC/outputs/Pendulum-v0/20210429-135700/models/sac_value_optimizer deleted file mode 100644 index 1d9500b..0000000 Binary files a/codes/SAC/outputs/Pendulum-v0/20210429-135700/models/sac_value_optimizer and /dev/null differ diff --git a/codes/SAC/outputs/Pendulum-v0/20210429-135700/results/ma_rewards_train.npy b/codes/SAC/outputs/Pendulum-v0/20210429-135700/results/ma_rewards_train.npy deleted file mode 100644 index b3676ce..0000000 Binary files a/codes/SAC/outputs/Pendulum-v0/20210429-135700/results/ma_rewards_train.npy and /dev/null differ diff --git a/codes/SAC/outputs/Pendulum-v0/20210429-135700/results/rewards_curve_train.png b/codes/SAC/outputs/Pendulum-v0/20210429-135700/results/rewards_curve_train.png deleted file mode 100644 index b870654..0000000 Binary files a/codes/SAC/outputs/Pendulum-v0/20210429-135700/results/rewards_curve_train.png and /dev/null differ diff --git a/codes/SAC/outputs/Pendulum-v0/20210429-135700/results/rewards_train.npy b/codes/SAC/outputs/Pendulum-v0/20210429-135700/results/rewards_train.npy deleted file mode 100644 index 73336b5..0000000 Binary files a/codes/SAC/outputs/Pendulum-v0/20210429-135700/results/rewards_train.npy and /dev/null differ diff --git a/codes/SAC/outputs/Pendulum-v0/20210506-014740/models/sac_policy b/codes/SAC/outputs/Pendulum-v0/20210506-014740/models/sac_policy new file mode 100644 index 0000000..12479e2 Binary files /dev/null and b/codes/SAC/outputs/Pendulum-v0/20210506-014740/models/sac_policy differ diff --git a/codes/SAC/outputs/Pendulum-v0/20210506-014740/models/sac_policy_optimizer b/codes/SAC/outputs/Pendulum-v0/20210506-014740/models/sac_policy_optimizer new file mode 100644 index 0000000..6dea232 Binary files /dev/null and b/codes/SAC/outputs/Pendulum-v0/20210506-014740/models/sac_policy_optimizer differ diff --git a/codes/SAC/outputs/Pendulum-v0/20210506-014740/models/sac_soft_q b/codes/SAC/outputs/Pendulum-v0/20210506-014740/models/sac_soft_q new file mode 100644 index 0000000..d2d5352 Binary files /dev/null and b/codes/SAC/outputs/Pendulum-v0/20210506-014740/models/sac_soft_q differ diff --git a/codes/SAC/outputs/Pendulum-v0/20210506-014740/models/sac_soft_q_optimizer b/codes/SAC/outputs/Pendulum-v0/20210506-014740/models/sac_soft_q_optimizer new file mode 100644 index 0000000..d4c3e48 Binary files /dev/null and b/codes/SAC/outputs/Pendulum-v0/20210506-014740/models/sac_soft_q_optimizer differ diff --git a/codes/SAC/outputs/Pendulum-v0/20210506-014740/models/sac_value b/codes/SAC/outputs/Pendulum-v0/20210506-014740/models/sac_value new file mode 100644 index 0000000..a180f73 Binary files /dev/null and b/codes/SAC/outputs/Pendulum-v0/20210506-014740/models/sac_value differ diff --git a/codes/SAC/outputs/Pendulum-v0/20210506-014740/models/sac_value_optimizer b/codes/SAC/outputs/Pendulum-v0/20210506-014740/models/sac_value_optimizer new file mode 100644 index 0000000..f2ab113 Binary files /dev/null and b/codes/SAC/outputs/Pendulum-v0/20210506-014740/models/sac_value_optimizer differ diff --git a/codes/SAC/outputs/Pendulum-v0/20210506-014740/results/eval_ma_rewards.npy b/codes/SAC/outputs/Pendulum-v0/20210506-014740/results/eval_ma_rewards.npy new file mode 100644 index 0000000..4971d4f Binary files /dev/null and b/codes/SAC/outputs/Pendulum-v0/20210506-014740/results/eval_ma_rewards.npy differ diff --git a/codes/SAC/outputs/Pendulum-v0/20210506-014740/results/eval_rewards.npy b/codes/SAC/outputs/Pendulum-v0/20210506-014740/results/eval_rewards.npy new file mode 100644 index 0000000..46bd706 Binary files /dev/null and b/codes/SAC/outputs/Pendulum-v0/20210506-014740/results/eval_rewards.npy differ diff --git a/codes/SAC/outputs/Pendulum-v0/20210506-014740/results/eval_rewards_curve.png b/codes/SAC/outputs/Pendulum-v0/20210506-014740/results/eval_rewards_curve.png new file mode 100644 index 0000000..3d4dd84 Binary files /dev/null and b/codes/SAC/outputs/Pendulum-v0/20210506-014740/results/eval_rewards_curve.png differ diff --git a/codes/SAC/outputs/Pendulum-v0/20210506-014740/results/train_ma_rewards.npy b/codes/SAC/outputs/Pendulum-v0/20210506-014740/results/train_ma_rewards.npy new file mode 100644 index 0000000..bffae05 Binary files /dev/null and b/codes/SAC/outputs/Pendulum-v0/20210506-014740/results/train_ma_rewards.npy differ diff --git a/codes/SAC/outputs/Pendulum-v0/20210506-014740/results/train_rewards.npy b/codes/SAC/outputs/Pendulum-v0/20210506-014740/results/train_rewards.npy new file mode 100644 index 0000000..37837a6 Binary files /dev/null and b/codes/SAC/outputs/Pendulum-v0/20210506-014740/results/train_rewards.npy differ diff --git a/codes/SAC/outputs/Pendulum-v0/20210506-014740/results/train_rewards_curve.png b/codes/SAC/outputs/Pendulum-v0/20210506-014740/results/train_rewards_curve.png new file mode 100644 index 0000000..399b952 Binary files /dev/null and b/codes/SAC/outputs/Pendulum-v0/20210506-014740/results/train_rewards_curve.png differ diff --git a/codes/SAC/task0_train.ipynb b/codes/SAC/task0_train.ipynb new file mode 100644 index 0000000..9a0c43a --- /dev/null +++ b/codes/SAC/task0_train.ipynb @@ -0,0 +1,197 @@ +{ + "metadata": { + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.10" + }, + "orig_nbformat": 2, + "kernelspec": { + "name": "python3710jvsc74a57bd0fd81e6a9e450d5c245c1a0b5da0b03c89c450f614a13afa2acb1654375922756", + "display_name": "Python 3.7.10 64-bit ('mujoco': conda)" + }, + "metadata": { + "interpreter": { + "hash": "fd81e6a9e450d5c245c1a0b5da0b03c89c450f614a13afa2acb1654375922756" + } + } + }, + "nbformat": 4, + "nbformat_minor": 2, + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "from pathlib import Path\n", + "curr_path = str(Path().absolute())\n", + "parent_path = str(Path().absolute().parent)\n", + "sys.path.append(parent_path) # add current terminal path to sys.path" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "import gym\n", + "import torch\n", + "import datetime\n", + "\n", + "from SAC.env import NormalizedActions\n", + "from SAC.agent import SAC\n", + "from common.utils import save_results, make_dir\n", + "from common.plot import plot_rewards\n", + "\n", + "curr_time = datetime.datetime.now().strftime(\"%Y%m%d-%H%M%S\") # obtain current time" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "class SACConfig:\n", + " def __init__(self) -> None:\n", + " self.algo = 'SAC'\n", + " self.env = 'Pendulum-v0'\n", + " self.result_path = curr_path+\"/outputs/\" +self.env+'/'+curr_time+'/results/' # path to save results\n", + " self.model_path = curr_path+\"/outputs/\" +self.env+'/'+curr_time+'/models/' # path to save models\n", + " self.train_eps = 300\n", + " self.train_steps = 500\n", + " self.eval_eps = 50\n", + " self.eval_steps = 500\n", + " self.gamma = 0.99\n", + " self.mean_lambda=1e-3\n", + " self.std_lambda=1e-3\n", + " self.z_lambda=0.0\n", + " self.soft_tau=1e-2\n", + " self.value_lr = 3e-4\n", + " self.soft_q_lr = 3e-4\n", + " self.policy_lr = 3e-4\n", + " self.capacity = 1000000\n", + " self.hidden_dim = 256\n", + " self.batch_size = 128\n", + " self.device=torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "def env_agent_config(cfg,seed=1):\n", + " env = NormalizedActions(gym.make(\"Pendulum-v0\"))\n", + " env.seed(seed)\n", + " action_dim = env.action_space.shape[0]\n", + " state_dim = env.observation_space.shape[0]\n", + " agent = SAC(state_dim,action_dim,cfg)\n", + " return env,agent" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "def train(cfg,env,agent):\n", + " print('Start to train !')\n", + " print(f'Env: {cfg.env}, Algorithm: {cfg.algo}, Device: {cfg.device}')\n", + " rewards = []\n", + " ma_rewards = [] # moveing average reward\n", + " for i_ep in range(cfg.train_eps):\n", + " state = env.reset()\n", + " ep_reward = 0\n", + " for i_step in range(cfg.train_steps):\n", + " action = agent.policy_net.get_action(state)\n", + " next_state, reward, done, _ = env.step(action)\n", + " agent.memory.push(state, action, reward, next_state, done)\n", + " agent.update()\n", + " state = next_state\n", + " ep_reward += reward\n", + " if done:\n", + " break\n", + " if (i_ep+1)%10==0:\n", + " print(f\"Episode:{i_ep+1}/{cfg.train_eps}, Reward:{ep_reward:.3f}\")\n", + " rewards.append(ep_reward)\n", + " if ma_rewards:\n", + " ma_rewards.append(0.9*ma_rewards[-1]+0.1*ep_reward)\n", + " else:\n", + " ma_rewards.append(ep_reward) \n", + " print('Complete training！')\n", + " return rewards, ma_rewards" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "def eval(cfg,env,agent):\n", + " print('Start to eval !')\n", + " print(f'Env: {cfg.env}, Algorithm: {cfg.algo}, Device: {cfg.device}')\n", + " rewards = []\n", + " ma_rewards = [] # moveing average reward\n", + " for i_ep in range(cfg.eval_eps):\n", + " state = env.reset()\n", + " ep_reward = 0\n", + " for i_step in range(cfg.eval_steps):\n", + " action = agent.policy_net.get_action(state)\n", + " next_state, reward, done, _ = env.step(action)\n", + " state = next_state\n", + " ep_reward += reward\n", + " if done:\n", + " break\n", + " if (i_ep+1)%10==0:\n", + " print(f\"Episode:{i_ep+1}/{cfg.train_eps}, Reward:{ep_reward:.3f}\")\n", + " rewards.append(ep_reward)\n", + " if ma_rewards:\n", + " ma_rewards.append(0.9*ma_rewards[-1]+0.1*ep_reward)\n", + " else:\n", + " ma_rewards.append(ep_reward) \n", + " print('Complete evaling！')\n", + " return rewards, ma_rewards\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "if __name__ == \"__main__\":\n", + " cfg=SACConfig()\n", + " \n", + " # train\n", + " env,agent = env_agent_config(cfg,seed=1)\n", + " rewards, ma_rewards = train(cfg, env, agent)\n", + " make_dir(cfg.result_path, cfg.model_path)\n", + " agent.save(path=cfg.model_path)\n", + " save_results(rewards, ma_rewards, tag='train', path=cfg.result_path)\n", + " plot_rewards(rewards, ma_rewards, tag=\"train\",\n", + " algo=cfg.algo, path=cfg.result_path)\n", + " # eval\n", + " env,agent = env_agent_config(cfg,seed=10)\n", + " agent.load(path=cfg.model_path)\n", + " rewards,ma_rewards = eval(cfg,env,agent)\n", + " save_results(rewards,ma_rewards,tag='eval',path=cfg.result_path)\n", + " plot_rewards(rewards,ma_rewards,tag=\"eval\",env=cfg.env,algo = cfg.algo,path=cfg.result_path)\n" + ] + } + ] +} \ No newline at end of file diff --git a/codes/SAC/task0_train.py b/codes/SAC/task0_train.py index 6956baa..1996b01 100644 --- a/codes/SAC/task0_train.py +++ b/codes/SAC/task0_train.py @@ -5,12 +5,10 @@ Author: JiangJi Email: johnjim0816@gmail.com Date: 2021-04-29 12:59:22 LastEditor: JiangJi -LastEditTime: 2021-04-29 13:56:56 +LastEditTime: 2021-05-06 16:58:01 Discription: Environment: ''' - - import sys,os curr_path = os.path.dirname(__file__) parent_path = os.path.dirname(curr_path) @@ -36,7 +34,8 @@ class SACConfig: self.model_path = curr_path+"/outputs/" +self.env+'/'+curr_time+'/models/' # path to save models self.train_eps = 300 self.train_steps = 500 - + self.eval_eps = 50 + self.eval_steps = 500 self.gamma = 0.99 self.mean_lambda=1e-3 self.std_lambda=1e-3 @@ -49,7 +48,18 @@ class SACConfig: self.hidden_dim = 256 self.batch_size = 128 self.device=torch.device("cuda" if torch.cuda.is_available() else "cpu") + +def env_agent_config(cfg,seed=1): + env = NormalizedActions(gym.make("Pendulum-v0")) + env.seed(seed) + action_dim = env.action_space.shape[0] + state_dim = env.observation_space.shape[0] + agent = SAC(state_dim,action_dim,cfg) + return env,agent + def train(cfg,env,agent): + print('Start to train !') + print(f'Env: {cfg.env}, Algorithm: {cfg.algo}, Device: {cfg.device}') rewards = [] ma_rewards = [] # moveing average reward for i_ep in range(cfg.train_eps): @@ -64,25 +74,58 @@ def train(cfg,env,agent): ep_reward += reward if done: break - print(f"Episode:{i_ep+1}/{cfg.train_eps}, Reward:{ep_reward:.3f}") + if (i_ep+1)%10==0: + print(f"Episode:{i_ep+1}/{cfg.train_eps}, Reward:{ep_reward:.3f}") rewards.append(ep_reward) if ma_rewards: ma_rewards.append(0.9*ma_rewards[-1]+0.1*ep_reward) else: ma_rewards.append(ep_reward) + print('Complete training！') return rewards, ma_rewards + +def eval(cfg,env,agent): + print('Start to eval !') + print(f'Env: {cfg.env}, Algorithm: {cfg.algo}, Device: {cfg.device}') + rewards = [] + ma_rewards = [] # moveing average reward + for i_ep in range(cfg.eval_eps): + state = env.reset() + ep_reward = 0 + for i_step in range(cfg.eval_steps): + action = agent.policy_net.get_action(state) + next_state, reward, done, _ = env.step(action) + state = next_state + ep_reward += reward + if done: + break + if (i_ep+1)%10==0: + print(f"Episode:{i_ep+1}/{cfg.train_eps}, Reward:{ep_reward:.3f}") + rewards.append(ep_reward) + if ma_rewards: + ma_rewards.append(0.9*ma_rewards[-1]+0.1*ep_reward) + else: + ma_rewards.append(ep_reward) + print('Complete evaling！') + return rewards, ma_rewards + if __name__ == "__main__": cfg=SACConfig() - env = NormalizedActions(gym.make("Pendulum-v0")) - action_dim = env.action_space.shape[0] - state_dim = env.observation_space.shape[0] - agent = SAC(state_dim,action_dim,cfg) - rewards,ma_rewards = train(cfg,env,agent) - make_dir(cfg.result_path,cfg.model_path) + + # train + env,agent = env_agent_config(cfg,seed=1) + rewards, ma_rewards = train(cfg, env, agent) + make_dir(cfg.result_path, cfg.model_path) agent.save(path=cfg.model_path) - save_results(rewards,ma_rewards,tag='train',path=cfg.result_path) - plot_rewards(rewards,ma_rewards,tag="train",env=cfg.env,algo = cfg.algo,path=cfg.result_path) - + save_results(rewards, ma_rewards, tag='train', path=cfg.result_path) + plot_rewards(rewards, ma_rewards, tag="train", + algo=cfg.algo, path=cfg.result_path) + # eval + env,agent = env_agent_config(cfg,seed=10) + agent.load(path=cfg.model_path) + rewards,ma_rewards = eval(cfg,env,agent) + save_results(rewards,ma_rewards,tag='eval',path=cfg.result_path) + plot_rewards(rewards,ma_rewards,tag="eval",env=cfg.env,algo = cfg.algo,path=cfg.result_path) diff --git a/codes/Sarsa/main.py b/codes/Sarsa/main.py deleted file mode 100644 index a2363ed..0000000 --- a/codes/Sarsa/main.py +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/env python -# coding=utf-8 -''' -Author: John -Email: johnjim0816@gmail.com -Date: 2021-03-11 17:59:16 -LastEditor: John -LastEditTime: 2021-03-12 17:01:43 -Discription: -Environment: -''' -import sys,os -sys.path.append(os.getcwd()) -import datetime -from envs.racetrack_env import RacetrackEnv -from Sarsa.agent import Sarsa -from common.plot import plot_rewards -from common.utils import save_results - -SEQUENCE = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") # 获取当前时间 -SAVED_MODEL_PATH = os.path.split(os.path.abspath(__file__))[0]+"/saved_model/"+SEQUENCE+'/' # 生成保存的模型路径 -if not os.path.exists(os.path.split(os.path.abspath(__file__))[0]+"/saved_model/"): # 检测是否存在文件夹 - os.mkdir(os.path.split(os.path.abspath(__file__))[0]+"/saved_model/") -if not os.path.exists(SAVED_MODEL_PATH): # 检测是否存在文件夹 - os.mkdir(SAVED_MODEL_PATH) -RESULT_PATH = os.path.split(os.path.abspath(__file__))[0]+"/results/"+SEQUENCE+'/' # 存储reward的路径 -if not os.path.exists(os.path.split(os.path.abspath(__file__))[0]+"/results/"): # 检测是否存在文件夹 - os.mkdir(os.path.split(os.path.abspath(__file__))[0]+"/results/") -if not os.path.exists(RESULT_PATH): # 检测是否存在文件夹 - os.mkdir(RESULT_PATH) - -class SarsaConfig: - ''' parameters for Sarsa - ''' - def __init__(self): - self.epsilon = 0.15 # epsilon: The probability to select a random action . - self.gamma = 0.9 # gamma: Gamma discount factor. - self.lr = 0.2 # learning rate: step size parameter - self.n_episodes = 150 - self.n_steps = 2000 - -def sarsa_train(cfg,env,agent): - rewards = [] - ma_rewards = [] - for i_episode in range(cfg.n_episodes): - # Print out which episode we're on, useful for debugging. - # Generate an episode. - # An episode is an array of (state, action, reward) tuples - state = env.reset() - ep_reward = 0 - while True: - # for t in range(cfg.n_steps): - action = agent.choose_action(state) - next_state, reward, done = env.step(action) - ep_reward+=reward - next_action = agent.choose_action(next_state) - agent.update(state, action, reward, next_state, next_action,done) - state = next_state - if done: - break - if ma_rewards: - ma_rewards.append(ma_rewards[-1]*0.9+ep_reward*0.1) - else: - ma_rewards.append(ep_reward) - rewards.append(ep_reward) - # if (i_episode+1)%10==0: - # print("Episode:{}/{}: Reward:{}".format(i_episode+1, cfg.n_episodes,ep_reward)) - return rewards,ma_rewards - -if __name__ == "__main__": - sarsa_cfg = SarsaConfig() - env = RacetrackEnv() - action_dim=9 - agent = Sarsa(action_dim,sarsa_cfg) - rewards,ma_rewards = sarsa_train(sarsa_cfg,env,agent) - agent.save(path=SAVED_MODEL_PATH) - save_results(rewards,ma_rewards,tag='train',path=RESULT_PATH) - plot_rewards(rewards,ma_rewards,tag="train",algo = "On-Policy First-Visit MC Control",path=RESULT_PATH) - - diff --git a/codes/Sarsa/outputs/CliffWalking-v0/20210506-171245/models/sarsa_model.pkl b/codes/Sarsa/outputs/CliffWalking-v0/20210506-171245/models/sarsa_model.pkl new file mode 100644 index 0000000..ff25fd5 Binary files /dev/null and b/codes/Sarsa/outputs/CliffWalking-v0/20210506-171245/models/sarsa_model.pkl differ diff --git a/codes/Sarsa/outputs/CliffWalking-v0/20210506-171245/results/eval_ma_rewards.npy b/codes/Sarsa/outputs/CliffWalking-v0/20210506-171245/results/eval_ma_rewards.npy new file mode 100644 index 0000000..d7d62e3 Binary files /dev/null and b/codes/Sarsa/outputs/CliffWalking-v0/20210506-171245/results/eval_ma_rewards.npy differ diff --git a/codes/Sarsa/outputs/CliffWalking-v0/20210506-171245/results/eval_rewards.npy b/codes/Sarsa/outputs/CliffWalking-v0/20210506-171245/results/eval_rewards.npy new file mode 100644 index 0000000..de0a816 Binary files /dev/null and b/codes/Sarsa/outputs/CliffWalking-v0/20210506-171245/results/eval_rewards.npy differ diff --git a/codes/Sarsa/outputs/CliffWalking-v0/20210506-171245/results/eval_rewards_curve.png b/codes/Sarsa/outputs/CliffWalking-v0/20210506-171245/results/eval_rewards_curve.png new file mode 100644 index 0000000..3de2db7 Binary files /dev/null and b/codes/Sarsa/outputs/CliffWalking-v0/20210506-171245/results/eval_rewards_curve.png differ diff --git a/codes/Sarsa/outputs/CliffWalking-v0/20210506-171245/results/train_ma_rewards.npy b/codes/Sarsa/outputs/CliffWalking-v0/20210506-171245/results/train_ma_rewards.npy new file mode 100644 index 0000000..3f9bf83 Binary files /dev/null and b/codes/Sarsa/outputs/CliffWalking-v0/20210506-171245/results/train_ma_rewards.npy differ diff --git a/codes/Sarsa/outputs/CliffWalking-v0/20210506-171245/results/train_rewards.npy b/codes/Sarsa/outputs/CliffWalking-v0/20210506-171245/results/train_rewards.npy new file mode 100644 index 0000000..e0fd7e9 Binary files /dev/null and b/codes/Sarsa/outputs/CliffWalking-v0/20210506-171245/results/train_rewards.npy differ diff --git a/codes/Sarsa/outputs/CliffWalking-v0/20210506-171245/results/train_rewards_curve.png b/codes/Sarsa/outputs/CliffWalking-v0/20210506-171245/results/train_rewards_curve.png new file mode 100644 index 0000000..0a8cd37 Binary files /dev/null and b/codes/Sarsa/outputs/CliffWalking-v0/20210506-171245/results/train_rewards_curve.png differ diff --git a/codes/Sarsa/results/20210313-110256/ma_rewards_train.npy b/codes/Sarsa/results/20210313-110256/ma_rewards_train.npy deleted file mode 100644 index 943a6d4..0000000 Binary files a/codes/Sarsa/results/20210313-110256/ma_rewards_train.npy and /dev/null differ diff --git a/codes/Sarsa/results/20210313-110256/rewards_curve_train.png b/codes/Sarsa/results/20210313-110256/rewards_curve_train.png deleted file mode 100644 index ea31886..0000000 Binary files a/codes/Sarsa/results/20210313-110256/rewards_curve_train.png and /dev/null differ diff --git a/codes/Sarsa/results/20210313-110256/rewards_train.npy b/codes/Sarsa/results/20210313-110256/rewards_train.npy deleted file mode 100644 index d0702e8..0000000 Binary files a/codes/Sarsa/results/20210313-110256/rewards_train.npy and /dev/null differ diff --git a/codes/Sarsa/saved_model/20210313-110256/sarsa_model.pkl b/codes/Sarsa/saved_model/20210313-110256/sarsa_model.pkl deleted file mode 100644 index d19971c..0000000 Binary files a/codes/Sarsa/saved_model/20210313-110256/sarsa_model.pkl and /dev/null differ diff --git a/codes/Sarsa/task0_train.py b/codes/Sarsa/task0_train.py new file mode 100644 index 0000000..d21db17 --- /dev/null +++ b/codes/Sarsa/task0_train.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python +# coding=utf-8 +''' +Author: John +Email: johnjim0816@gmail.com +Date: 2021-03-11 17:59:16 +LastEditor: John +LastEditTime: 2021-05-06 17:12:37 +Discription: +Environment: +''' +import sys,os +curr_path = os.path.dirname(__file__) +parent_path = os.path.dirname(curr_path) +sys.path.append(parent_path) # add current terminal path to sys.path + +import datetime +from envs.racetrack_env import RacetrackEnv +from Sarsa.agent import Sarsa +from common.plot import plot_rewards +from common.utils import save_results,make_dir + +curr_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") # obtain current time + +class SarsaConfig: + ''' parameters for Sarsa + ''' + def __init__(self): + self.algo = 'Qlearning' + self.env = 'CliffWalking-v0' # 0 up, 1 right, 2 down, 3 left + self.result_path = curr_path+"/outputs/" +self.env+'/'+curr_time+'/results/' # path to save results + self.model_path = curr_path+"/outputs/" +self.env+'/'+curr_time+'/models/' # path to save models + self.train_eps = 200 + self.eval_eps = 50 + self.epsilon = 0.15 # epsilon: The probability to select a random action . + self.gamma = 0.9 # gamma: Gamma discount factor. + self.lr = 0.2 # learning rate: step size parameter + self.n_steps = 2000 + +def env_agent_config(cfg,seed=1): + env = RacetrackEnv() + action_dim=9 + agent = Sarsa(action_dim,cfg) + return env,agent + +def train(cfg,env,agent): + rewards = [] + ma_rewards = [] + for i_episode in range(cfg.train_eps): + # Print out which episode we're on, useful for debugging. + # Generate an episode. + # An episode is an array of (state, action, reward) tuples + state = env.reset() + ep_reward = 0 + while True: + # for t in range(cfg.n_steps): + action = agent.choose_action(state) + next_state, reward, done = env.step(action) + ep_reward+=reward + next_action = agent.choose_action(next_state) + agent.update(state, action, reward, next_state, next_action,done) + state = next_state + if done: + break + if ma_rewards: + ma_rewards.append(ma_rewards[-1]*0.9+ep_reward*0.1) + else: + ma_rewards.append(ep_reward) + rewards.append(ep_reward) + if (i_episode+1)%10==0: + print("Episode:{}/{}: Reward:{}".format(i_episode+1, cfg.train_eps,ep_reward)) + return rewards,ma_rewards + +def eval(cfg,env,agent): + rewards = [] + ma_rewards = [] + for i_episode in range(cfg.eval_eps): + # Print out which episode we're on, useful for debugging. + # Generate an episode. + # An episode is an array of (state, action, reward) tuples + state = env.reset() + ep_reward = 0 + while True: + # for t in range(cfg.n_steps): + action = agent.choose_action(state) + next_state, reward, done = env.step(action) + ep_reward+=reward + state = next_state + if done: + break + if ma_rewards: + ma_rewards.append(ma_rewards[-1]*0.9+ep_reward*0.1) + else: + ma_rewards.append(ep_reward) + rewards.append(ep_reward) + if (i_episode+1)%10==0: + print("Episode:{}/{}: Reward:{}".format(i_episode+1, cfg.eval_eps,ep_reward)) + print('Complete evaling！') + return rewards,ma_rewards + +if __name__ == "__main__": + cfg = SarsaConfig() + env,agent = env_agent_config(cfg,seed=1) + rewards,ma_rewards = train(cfg,env,agent) + make_dir(cfg.result_path,cfg.model_path) + agent.save(path=cfg.model_path) + save_results(rewards,ma_rewards,tag='train',path=cfg.result_path) + plot_rewards(rewards,ma_rewards,tag="train",env=cfg.env,algo = cfg.algo,path=cfg.result_path) + + env,agent = env_agent_config(cfg,seed=10) + agent.load(path=cfg.model_path) + rewards,ma_rewards = eval(cfg,env,agent) + save_results(rewards,ma_rewards,tag='eval',path=cfg.result_path) + plot_rewards(rewards,ma_rewards,tag="eval",env=cfg.env,algo = cfg.algo,path=cfg.result_path) + + + diff --git a/codes/common/model.py b/codes/common/model.py index 41785fd..257c33b 100644 --- a/codes/common/model.py +++ b/codes/common/model.py @@ -5,7 +5,7 @@ Author: John Email: johnjim0816@gmail.com Date: 2021-03-12 21:14:12 LastEditor: John -LastEditTime: 2021-03-31 13:49:06 +LastEditTime: 2021-05-04 02:45:27 Discription: Environment: ''' @@ -63,7 +63,7 @@ class Actor(nn.Module): def forward(self, x): x = F.relu(self.linear1(x)) x = F.relu(self.linear2(x)) - x = F.tanh(self.linear3(x)) + x = torch.tanh(self.linear3(x)) return x class ActorCritic(nn.Module): diff --git a/codes/common/utils.py b/codes/common/utils.py index 5d51eea..8339db5 100644 --- a/codes/common/utils.py +++ b/codes/common/utils.py @@ -5,7 +5,7 @@ Author: John Email: johnjim0816@gmail.com Date: 2021-03-12 16:02:24 LastEditor: John -LastEditTime: 2021-04-29 15:32:38 +LastEditTime: 2021-05-04 19:58:31 Discription: Environment: ''' @@ -13,10 +13,8 @@ import os import numpy as np from pathlib import Path - - def save_results(rewards,ma_rewards,tag='train',path='./results'): - '''保存reward等结果 + '''save rewards and ma_rewards ''' np.save(path+'{}_rewards.npy'.format(tag), rewards) np.save(path+'{}_ma_rewards.npy'.format(tag), ma_rewards)