diff --git a/codes/DDPG/task0.py b/codes/DDPG/task0.py index 33872f4..550da78 100644 --- a/codes/DDPG/task0.py +++ b/codes/DDPG/task0.py @@ -34,7 +34,7 @@ class DDPGConfig: self.env_name = env_name # 环境名称 self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # 检测GPU self.train_eps = 300 # 训练的回合数 - self.eval_eps = 50 # 测试的回合数 + self.test_eps = 50 # 测试的回合数 self.gamma = 0.99 # 折扣因子 self.critic_lr = 1e-3 # 评论家网络的学习率 self.actor_lr = 1e-4 # 演员网络的学习率 diff --git a/codes/DDPG/train.py b/codes/DDPG/train.py index 8554cd0..4cdfa9d 100644 --- a/codes/DDPG/train.py +++ b/codes/DDPG/train.py @@ -42,7 +42,7 @@ def test(cfg, env, agent): print(f'环境：{cfg.env_name}, 算法：{cfg.algo}, 设备：{cfg.device}') rewards = [] # 记录所有回合的奖励 ma_rewards = [] # 记录所有回合的滑动平均奖励 - for i_ep in range(cfg.eval_eps): + for i_ep in range(cfg.test_eps): state = env.reset() done = False ep_reward = 0 @@ -59,6 +59,6 @@ def test(cfg, env, agent): ma_rewards.append(0.9*ma_rewards[-1]+0.1*ep_reward) else: ma_rewards.append(ep_reward) - print(f"回合：{i_ep+1}/{cfg.eval_eps}，奖励：{ep_reward:.1f}") + print(f"回合：{i_ep+1}/{cfg.test_eps}，奖励：{ep_reward:.1f}") print('完成测试！') return rewards, ma_rewards \ No newline at end of file diff --git a/codes/DQN/task0.py b/codes/DQN/task0.py index e4c326e..7c20144 100644 --- a/codes/DQN/task0.py +++ b/codes/DQN/task0.py @@ -23,7 +23,7 @@ class DQNConfig: self.device = torch.device( "cuda" if torch.cuda.is_available() else "cpu") # 检测GPU self.train_eps = 200 # 训练的回合数 - self.eval_eps = 30 # 测试的回合数 + self.test_eps = 30 # 测试的回合数 # 超参数 self.gamma = 0.95 # 强化学习中的折扣因子 self.epsilon_start = 0.90 # e-greedy策略中初始epsilon diff --git a/codes/DQN/task1.py b/codes/DQN/task1.py index d85a2ef..cf93829 100644 --- a/codes/DQN/task1.py +++ b/codes/DQN/task1.py @@ -26,7 +26,7 @@ class DQNConfig: self.device = torch.device( "cuda" if torch.cuda.is_available() else "cpu") # 检测GPU self.train_eps = 200 # 训练的回合数 - self.eval_eps = 30 # 测试的回合数 + self.test_eps = 30 # 测试的回合数 # 超参数 self.gamma = 0.95 # 强化学习中的折扣因子 self.epsilon_start = 0.90 # e-greedy策略中初始epsilon diff --git a/codes/DQN/train.ipynb b/codes/DQN/train.ipynb index ba4308e..2529826 100644 --- a/codes/DQN/train.ipynb +++ b/codes/DQN/train.ipynb @@ -180,7 +180,7 @@ " self.algo = \"DQN\" # 算法名称\n", " self.env = 'CartPole-v0' # 环境名称\n", " self.train_eps = 200 # 训练的回合数\n", - " self.eval_eps = 20 # 测试的回合数\n", + " self.test_eps = 20 # 测试的回合数\n", " self.gamma = 0.95 # 强化学习中的折扣因子\n", " self.epsilon_start = 0.90 # e-greedy策略中初始epsilon\n", " self.epsilon_end = 0.01 # e-greedy策略中的终止epsilon\n", @@ -365,7 +365,7 @@ " cfg.epsilon_end = 0.0 # e-greedy策略中的终止epsilon\n", " rewards = [] # 记录所有回合的奖励\n", " ma_rewards = [] # 记录所有回合的滑动平均奖励\n", - " for i_ep in range(cfg.eval_eps):\n", + " for i_ep in range(cfg.test_eps):\n", " ep_reward = 0 # 记录一回合内的奖励\n", " state = env.reset() # 重置环境，返回初始状态\n", " while True:\n", @@ -381,7 +381,7 @@ " else:\n", " ma_rewards.append(ep_reward)\n", " if (i_ep+1)%3 == 0: \n", - " print(f\"回合：{i_ep+1}/{cfg.eval_eps}, 奖励：{ep_reward:.1f}\")\n", + " print(f\"回合：{i_ep+1}/{cfg.test_eps}, 奖励：{ep_reward:.1f}\")\n", " print('完成测试！')\n", " return rewards,ma_rewards\n", "\n", diff --git a/codes/DQN/train.py b/codes/DQN/train.py index 4f8510e..54fe1d8 100644 --- a/codes/DQN/train.py +++ b/codes/DQN/train.py @@ -5,7 +5,7 @@ @Email: johnjim0816@gmail.com @Date: 2020-06-12 00:48:57 @LastEditor: John -LastEditTime: 2021-09-15 15:34:13 +LastEditTime: 2021-12-22 11:08:04 @Discription: @Environment: python 3.7.7 ''' @@ -30,13 +30,13 @@ def train(cfg, env, agent): break if (i_ep+1) % cfg.target_update == 0: # 智能体目标网络更新 agent.target_net.load_state_dict(agent.policy_net.state_dict()) - if (i_ep+1)%10 == 0: - print('回合：{}/{}, 奖励：{}'.format(i_ep+1, cfg.train_eps, ep_reward)) rewards.append(ep_reward) if ma_rewards: ma_rewards.append(0.9*ma_rewards[-1]+0.1*ep_reward) else: ma_rewards.append(ep_reward) + if (i_ep+1)%10 == 0: + print('回合：{}/{}, 奖励：{}'.format(i_ep+1, cfg.train_eps, ep_reward)) print('完成训练！') return rewards, ma_rewards @@ -48,7 +48,7 @@ def test(cfg,env,agent): cfg.epsilon_end = 0.0 # e-greedy策略中的终止epsilon rewards = [] # 记录所有回合的奖励 ma_rewards = [] # 记录所有回合的滑动平均奖励 - for i_ep in range(cfg.eval_eps): + for i_ep in range(cfg.test_eps): ep_reward = 0 # 记录一回合内的奖励 state = env.reset() # 重置环境，返回初始状态 while True: @@ -63,7 +63,7 @@ def test(cfg,env,agent): ma_rewards.append(ma_rewards[-1]*0.9+ep_reward*0.1) else: ma_rewards.append(ep_reward) - print(f"回合：{i_ep+1}/{cfg.eval_eps}，奖励：{ep_reward:.1f}") + print(f"回合：{i_ep+1}/{cfg.test_eps}，奖励：{ep_reward:.1f}") print('完成测试！') return rewards,ma_rewards @@ -89,7 +89,7 @@ if __name__ == "__main__": self.env_name = 'CartPole-v0' # 环境名称 self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # 检测GPU self.train_eps = 200 # 训练的回合数 - self.eval_eps = 30 # 测试的回合数 + self.test_eps = 30 # 测试的回合数 # 超参数 self.gamma = 0.95 # 强化学习中的折扣因子 self.epsilon_start = 0.90 # e-greedy策略中初始epsilon diff --git a/codes/PPO/task0.py b/codes/PPO/task0.py index cd55eda..8e0d92a 100644 --- a/codes/PPO/task0.py +++ b/codes/PPO/task0.py @@ -20,7 +20,7 @@ class PPOConfig: self.continuous = False # 环境是否为连续动作 self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # 检测GPU self.train_eps = 200 # 训练的回合数 - self.eval_eps = 20 # 测试的回合数 + self.test_eps = 20 # 测试的回合数 self.batch_size = 5 self.gamma=0.99 self.n_epochs = 4 diff --git a/codes/PPO/task1.py b/codes/PPO/task1.py index 178efba..38d9152 100644 --- a/codes/PPO/task1.py +++ b/codes/PPO/task1.py @@ -20,7 +20,7 @@ class PPOConfig: self.continuous = True # 环境是否为连续动作 self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # 检测GPU self.train_eps = 200 # 训练的回合数 - self.eval_eps = 20 # 测试的回合数 + self.test_eps = 20 # 测试的回合数 self.batch_size = 5 self.gamma=0.99 self.n_epochs = 4 diff --git a/codes/PPO/train.ipynb b/codes/PPO/train.ipynb index 9c74585..b2dc91a 100644 --- a/codes/PPO/train.ipynb +++ b/codes/PPO/train.ipynb @@ -68,7 +68,7 @@ " self.result_path = curr_path+\"/results/\" +self.env+'/'+curr_time+'/results/' # path to save results\n", " self.model_path = curr_path+\"/results/\" +self.env+'/'+curr_time+'/models/' # path to save models\n", " self.train_eps = 200 # max training episodes\n", - " self.eval_eps = 50\n", + " self.test_eps = 50\n", " self.batch_size = 5\n", " self.gamma=0.99\n", " self.n_epochs = 4\n", @@ -144,7 +144,7 @@ " print(f'Env:{cfg.env}, Algorithm:{cfg.algo}, Device:{cfg.device}')\n", " rewards= []\n", " ma_rewards = [] # moving average rewards\n", - " for i_ep in range(cfg.eval_eps):\n", + " for i_ep in range(cfg.test_eps):\n", " state = env.reset()\n", " done = False\n", " ep_reward = 0\n", diff --git a/codes/PPO/train.py b/codes/PPO/train.py index aff54bf..e642df0 100644 --- a/codes/PPO/train.py +++ b/codes/PPO/train.py @@ -32,7 +32,7 @@ def eval(cfg,env,agent): print(f'环境：{cfg.env_name}, 算法：{cfg.algo}, 设备：{cfg.device}') rewards = [] # 记录所有回合的奖励 ma_rewards = [] # 记录所有回合的滑动平均奖励 - for i_ep in range(cfg.eval_eps): + for i_ep in range(cfg.test_eps): state = env.reset() done = False ep_reward = 0 @@ -47,7 +47,7 @@ def eval(cfg,env,agent): 0.9*ma_rewards[-1]+0.1*ep_reward) else: ma_rewards.append(ep_reward) - print('回合：{}/{}, 奖励：{}'.format(i_ep+1, cfg.eval_eps, ep_reward)) + print('回合：{}/{}, 奖励：{}'.format(i_ep+1, cfg.test_eps, ep_reward)) print('完成训练！') return rewards,ma_rewards @@ -74,7 +74,7 @@ if __name__ == '__main__': self.continuous = False # 环境是否为连续动作 self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # 检测GPU self.train_eps = 200 # 训练的回合数 - self.eval_eps = 20 # 测试的回合数 + self.test_eps = 20 # 测试的回合数 self.batch_size = 5 self.gamma=0.99 self.n_epochs = 4 diff --git a/codes/PolicyGradient/model.py b/codes/PolicyGradient/model.py index 7f5b1a8..6d9bc64 100644 --- a/codes/PolicyGradient/model.py +++ b/codes/PolicyGradient/model.py @@ -5,21 +5,22 @@ Author: John Email: johnjim0816@gmail.com Date: 2021-03-23 16:35:58 LastEditor: John -LastEditTime: 2021-03-23 16:36:20 +LastEditTime: 2021-12-21 23:21:26 Discription: Environment: ''' import torch.nn as nn import torch.nn.functional as F class MLP(nn.Module): + ''' 多层感知机 输入：state维度 输出：概率 ''' - def __init__(self,state_dim,hidden_dim = 36): + def __init__(self,input_dim,hidden_dim = 36): super(MLP, self).__init__() - # 24和36为hidden layer的层数，可根据state_dim, action_dim的情况来改变 - self.fc1 = nn.Linear(state_dim, hidden_dim) + # 24和36为hidden layer的层数，可根据input_dim, action_dim的情况来改变 + self.fc1 = nn.Linear(input_dim, hidden_dim) self.fc2 = nn.Linear(hidden_dim,hidden_dim) self.fc3 = nn.Linear(hidden_dim, 1) # Prob of Left diff --git a/codes/PolicyGradient/task0_train.py b/codes/PolicyGradient/task0_train.py index a7fb0d2..b6866f0 100644 --- a/codes/PolicyGradient/task0_train.py +++ b/codes/PolicyGradient/task0_train.py @@ -34,7 +34,7 @@ class PGConfig: self.model_path = curr_path+"/outputs/" + self.env + \ '/'+curr_time+'/models/' # 保存模型的路径 self.train_eps = 300 # 训练的回合数 - self.eval_eps = 30 # 测试的回合数 + self.test_eps = 30 # 测试的回合数 self.batch_size = 8 self.lr = 0.01 # 学习率 self.gamma = 0.99 @@ -94,7 +94,7 @@ def eval(cfg,env,agent): print(f'Env:{cfg.env}, Algorithm:{cfg.algo}, Device:{cfg.device}') rewards = [] ma_rewards = [] - for i_ep in range(cfg.eval_eps): + for i_ep in range(cfg.test_eps): state = env.reset() ep_reward = 0 for _ in count(): diff --git a/codes/QLearning/agent.py b/codes/QLearning/agent.py index 4587c86..be57831 100644 --- a/codes/QLearning/agent.py +++ b/codes/QLearning/agent.py @@ -5,7 +5,7 @@ Author: John Email: johnjim0816@gmail.com Date: 2020-09-11 23:03:00 LastEditor: John -LastEditTime: 2021-09-19 23:05:45 +LastEditTime: 2021-12-22 10:54:57 Discription: use defaultdict to define Q table Environment: ''' @@ -15,17 +15,17 @@ import torch from collections import defaultdict class QLearning(object): - def __init__(self,state_dim, - action_dim,cfg): - self.action_dim = action_dim # dimension of acgtion - self.lr = cfg.lr # learning rate + def __init__(self,n_states, + n_actions,cfg): + self.n_actions = n_actions + self.lr = cfg.lr # 学习率 self.gamma = cfg.gamma self.epsilon = 0 self.sample_count = 0 self.epsilon_start = cfg.epsilon_start self.epsilon_end = cfg.epsilon_end self.epsilon_decay = cfg.epsilon_decay - self.Q_table = defaultdict(lambda: np.zeros(action_dim)) # A nested dictionary that maps state -> (action -> action-value) + self.Q_table = defaultdict(lambda: np.zeros(n_actions)) # 用嵌套字典存放状态->动作->状态-动作值（Q值）的映射，即Q表 def choose_action(self, state): self.sample_count += 1 self.epsilon = self.epsilon_end + (self.epsilon_start - self.epsilon_end) * \ @@ -34,7 +34,7 @@ class QLearning(object): if np.random.uniform(0, 1) > self.epsilon: action = np.argmax(self.Q_table[str(state)]) # 选择Q(s,a)最大对应的动作 else: - action = np.random.choice(self.action_dim) # 随机选择动作 + action = np.random.choice(self.n_actions) # 随机选择动作 return action def predict(self,state): action = np.argmax(self.Q_table[str(state)]) diff --git a/codes/QLearning/outputs/CliffWalking-v0/20210919-211343/models/Qleaning_model.pkl b/codes/QLearning/outputs/CliffWalking-v0/20210919-211343/models/Qleaning_model.pkl deleted file mode 100644 index 45dce51..0000000 Binary files a/codes/QLearning/outputs/CliffWalking-v0/20210919-211343/models/Qleaning_model.pkl and /dev/null differ diff --git a/codes/QLearning/outputs/CliffWalking-v0/20210919-211343/results/train_ma_rewards.npy b/codes/QLearning/outputs/CliffWalking-v0/20210919-211343/results/train_ma_rewards.npy deleted file mode 100644 index 3a8bde0..0000000 Binary files a/codes/QLearning/outputs/CliffWalking-v0/20210919-211343/results/train_ma_rewards.npy and /dev/null differ diff --git a/codes/QLearning/outputs/CliffWalking-v0/20210919-211343/results/train_rewards.npy b/codes/QLearning/outputs/CliffWalking-v0/20210919-211343/results/train_rewards.npy deleted file mode 100644 index 36de6fc..0000000 Binary files a/codes/QLearning/outputs/CliffWalking-v0/20210919-211343/results/train_rewards.npy and /dev/null differ diff --git a/codes/QLearning/outputs/CliffWalking-v0/20210919-211343/results/train_rewards_curve_cn.png b/codes/QLearning/outputs/CliffWalking-v0/20210919-211343/results/train_rewards_curve_cn.png deleted file mode 100644 index 3226b8a..0000000 Binary files a/codes/QLearning/outputs/CliffWalking-v0/20210919-211343/results/train_rewards_curve_cn.png and /dev/null differ diff --git a/codes/QLearning/outputs/CliffWalking-v0/20210919-211442/models/Qleaning_model.pkl b/codes/QLearning/outputs/CliffWalking-v0/20210919-211442/models/Qleaning_model.pkl deleted file mode 100644 index 5c46ec6..0000000 Binary files a/codes/QLearning/outputs/CliffWalking-v0/20210919-211442/models/Qleaning_model.pkl and /dev/null differ diff --git a/codes/QLearning/outputs/CliffWalking-v0/20210919-211442/results/train_ma_rewards.npy b/codes/QLearning/outputs/CliffWalking-v0/20210919-211442/results/train_ma_rewards.npy deleted file mode 100644 index 1d6b889..0000000 Binary files a/codes/QLearning/outputs/CliffWalking-v0/20210919-211442/results/train_ma_rewards.npy and /dev/null differ diff --git a/codes/QLearning/outputs/CliffWalking-v0/20210919-211442/results/train_rewards.npy b/codes/QLearning/outputs/CliffWalking-v0/20210919-211442/results/train_rewards.npy deleted file mode 100644 index 6e6ccf0..0000000 Binary files a/codes/QLearning/outputs/CliffWalking-v0/20210919-211442/results/train_rewards.npy and /dev/null differ diff --git a/codes/QLearning/outputs/CliffWalking-v0/20210919-211442/results/train_rewards_curve_cn.png b/codes/QLearning/outputs/CliffWalking-v0/20210919-211442/results/train_rewards_curve_cn.png deleted file mode 100644 index e1cd04e..0000000 Binary files a/codes/QLearning/outputs/CliffWalking-v0/20210919-211442/results/train_rewards_curve_cn.png and /dev/null differ diff --git a/codes/QLearning/outputs/CliffWalking-v0/20210919-211456/models/Qleaning_model.pkl b/codes/QLearning/outputs/CliffWalking-v0/20210919-211456/models/Qleaning_model.pkl deleted file mode 100644 index 6986805..0000000 Binary files a/codes/QLearning/outputs/CliffWalking-v0/20210919-211456/models/Qleaning_model.pkl and /dev/null differ diff --git a/codes/QLearning/outputs/CliffWalking-v0/20210919-211456/results/train_ma_rewards.npy b/codes/QLearning/outputs/CliffWalking-v0/20210919-211456/results/train_ma_rewards.npy deleted file mode 100644 index e6793df..0000000 Binary files a/codes/QLearning/outputs/CliffWalking-v0/20210919-211456/results/train_ma_rewards.npy and /dev/null differ diff --git a/codes/QLearning/outputs/CliffWalking-v0/20210919-211456/results/train_rewards.npy b/codes/QLearning/outputs/CliffWalking-v0/20210919-211456/results/train_rewards.npy deleted file mode 100644 index e6793df..0000000 Binary files a/codes/QLearning/outputs/CliffWalking-v0/20210919-211456/results/train_rewards.npy and /dev/null differ diff --git a/codes/QLearning/outputs/CliffWalking-v0/20210919-211456/results/train_rewards_curve_cn.png b/codes/QLearning/outputs/CliffWalking-v0/20210919-211456/results/train_rewards_curve_cn.png deleted file mode 100644 index 9c98cc9..0000000 Binary files a/codes/QLearning/outputs/CliffWalking-v0/20210919-211456/results/train_rewards_curve_cn.png and /dev/null differ diff --git a/codes/QLearning/outputs/CliffWalking-v0/20210920-003309/models/Qleaning_model.pkl b/codes/QLearning/outputs/CliffWalking-v0/20210920-003309/models/Qleaning_model.pkl deleted file mode 100644 index 4d6ba95..0000000 Binary files a/codes/QLearning/outputs/CliffWalking-v0/20210920-003309/models/Qleaning_model.pkl and /dev/null differ diff --git a/codes/QLearning/outputs/CliffWalking-v0/20210920-003309/results/eval_rewards_curve_cn.png b/codes/QLearning/outputs/CliffWalking-v0/20210920-003309/results/eval_rewards_curve_cn.png deleted file mode 100644 index 91ca06c..0000000 Binary files a/codes/QLearning/outputs/CliffWalking-v0/20210920-003309/results/eval_rewards_curve_cn.png and /dev/null differ diff --git a/codes/QLearning/outputs/CliffWalking-v0/20210920-003309/results/train_ma_rewards.npy b/codes/QLearning/outputs/CliffWalking-v0/20210920-003309/results/train_ma_rewards.npy deleted file mode 100644 index 7184c7b..0000000 Binary files a/codes/QLearning/outputs/CliffWalking-v0/20210920-003309/results/train_ma_rewards.npy and /dev/null differ diff --git a/codes/QLearning/outputs/CliffWalking-v0/20210920-003309/results/train_rewards.npy b/codes/QLearning/outputs/CliffWalking-v0/20210920-003309/results/train_rewards.npy deleted file mode 100644 index f037a25..0000000 Binary files a/codes/QLearning/outputs/CliffWalking-v0/20210920-003309/results/train_rewards.npy and /dev/null differ diff --git a/codes/QLearning/outputs/CliffWalking-v0/20210920-003309/results/train_rewards_curve_cn.png b/codes/QLearning/outputs/CliffWalking-v0/20210920-003309/results/train_rewards_curve_cn.png deleted file mode 100644 index 9c0943a..0000000 Binary files a/codes/QLearning/outputs/CliffWalking-v0/20210920-003309/results/train_rewards_curve_cn.png and /dev/null differ diff --git a/codes/QLearning/outputs/CliffWalking-v0/20211222-110223/models/Qleaning_model.pkl b/codes/QLearning/outputs/CliffWalking-v0/20211222-110223/models/Qleaning_model.pkl new file mode 100644 index 0000000..dc89386 Binary files /dev/null and b/codes/QLearning/outputs/CliffWalking-v0/20211222-110223/models/Qleaning_model.pkl differ diff --git a/codes/QLearning/outputs/CliffWalking-v0/20210920-003309/results/eval_ma_rewards.npy b/codes/QLearning/outputs/CliffWalking-v0/20211222-110223/results/test_ma_rewards.npy similarity index 100% rename from codes/QLearning/outputs/CliffWalking-v0/20210920-003309/results/eval_ma_rewards.npy rename to codes/QLearning/outputs/CliffWalking-v0/20211222-110223/results/test_ma_rewards.npy diff --git a/codes/QLearning/outputs/CliffWalking-v0/20210920-003309/results/eval_rewards.npy b/codes/QLearning/outputs/CliffWalking-v0/20211222-110223/results/test_rewards.npy similarity index 100% rename from codes/QLearning/outputs/CliffWalking-v0/20210920-003309/results/eval_rewards.npy rename to codes/QLearning/outputs/CliffWalking-v0/20211222-110223/results/test_rewards.npy diff --git a/codes/QLearning/outputs/CliffWalking-v0/20211222-110223/results/test_rewards_curve.png b/codes/QLearning/outputs/CliffWalking-v0/20211222-110223/results/test_rewards_curve.png new file mode 100644 index 0000000..d745634 Binary files /dev/null and b/codes/QLearning/outputs/CliffWalking-v0/20211222-110223/results/test_rewards_curve.png differ diff --git a/codes/QLearning/outputs/CliffWalking-v0/20211222-110223/results/train_ma_rewards.npy b/codes/QLearning/outputs/CliffWalking-v0/20211222-110223/results/train_ma_rewards.npy new file mode 100644 index 0000000..23e7c95 Binary files /dev/null and b/codes/QLearning/outputs/CliffWalking-v0/20211222-110223/results/train_ma_rewards.npy differ diff --git a/codes/QLearning/outputs/CliffWalking-v0/20211222-110223/results/train_rewards.npy b/codes/QLearning/outputs/CliffWalking-v0/20211222-110223/results/train_rewards.npy new file mode 100644 index 0000000..0ceb153 Binary files /dev/null and b/codes/QLearning/outputs/CliffWalking-v0/20211222-110223/results/train_rewards.npy differ diff --git a/codes/QLearning/outputs/CliffWalking-v0/20211222-110223/results/train_rewards_curve.png b/codes/QLearning/outputs/CliffWalking-v0/20211222-110223/results/train_rewards_curve.png new file mode 100644 index 0000000..a15bd2a Binary files /dev/null and b/codes/QLearning/outputs/CliffWalking-v0/20211222-110223/results/train_rewards_curve.png differ diff --git a/codes/QLearning/outputs/CliffWalking-v0/20211222-111747/models/Qleaning_model.pkl b/codes/QLearning/outputs/CliffWalking-v0/20211222-111747/models/Qleaning_model.pkl new file mode 100644 index 0000000..c362dbd Binary files /dev/null and b/codes/QLearning/outputs/CliffWalking-v0/20211222-111747/models/Qleaning_model.pkl differ diff --git a/codes/QLearning/outputs/CliffWalking-v0/20211222-111747/results/test_ma_rewards.npy b/codes/QLearning/outputs/CliffWalking-v0/20211222-111747/results/test_ma_rewards.npy new file mode 100644 index 0000000..9bee5e4 Binary files /dev/null and b/codes/QLearning/outputs/CliffWalking-v0/20211222-111747/results/test_ma_rewards.npy differ diff --git a/codes/QLearning/outputs/CliffWalking-v0/20211222-111747/results/test_rewards.npy b/codes/QLearning/outputs/CliffWalking-v0/20211222-111747/results/test_rewards.npy new file mode 100644 index 0000000..8aeb5dd Binary files /dev/null and b/codes/QLearning/outputs/CliffWalking-v0/20211222-111747/results/test_rewards.npy differ diff --git a/codes/QLearning/outputs/CliffWalking-v0/20211222-111747/results/test_rewards_curve.png b/codes/QLearning/outputs/CliffWalking-v0/20211222-111747/results/test_rewards_curve.png new file mode 100644 index 0000000..5f3ffb5 Binary files /dev/null and b/codes/QLearning/outputs/CliffWalking-v0/20211222-111747/results/test_rewards_curve.png differ diff --git a/codes/QLearning/outputs/CliffWalking-v0/20211222-111747/results/train_ma_rewards.npy b/codes/QLearning/outputs/CliffWalking-v0/20211222-111747/results/train_ma_rewards.npy new file mode 100644 index 0000000..261a3d5 Binary files /dev/null and b/codes/QLearning/outputs/CliffWalking-v0/20211222-111747/results/train_ma_rewards.npy differ diff --git a/codes/QLearning/outputs/CliffWalking-v0/20211222-111747/results/train_rewards.npy b/codes/QLearning/outputs/CliffWalking-v0/20211222-111747/results/train_rewards.npy new file mode 100644 index 0000000..b1a0f23 Binary files /dev/null and b/codes/QLearning/outputs/CliffWalking-v0/20211222-111747/results/train_rewards.npy differ diff --git a/codes/QLearning/outputs/CliffWalking-v0/20211222-111747/results/train_rewards_curve.png b/codes/QLearning/outputs/CliffWalking-v0/20211222-111747/results/train_rewards_curve.png new file mode 100644 index 0000000..9a9d6ad Binary files /dev/null and b/codes/QLearning/outputs/CliffWalking-v0/20211222-111747/results/train_rewards_curve.png differ diff --git a/codes/QLearning/task0.ipynb b/codes/QLearning/task0.ipynb new file mode 100644 index 0000000..a8be93b --- /dev/null +++ b/codes/QLearning/task0.ipynb @@ -0,0 +1,386 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 36, + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "from pathlib import Path\n", + "curr_path = str(Path().absolute())\n", + "parent_path = str(Path().absolute().parent)\n", + "sys.path.append(parent_path) # 添加路径到系统路径\n", + "\n", + "import gym\n", + "import torch\n", + "import math\n", + "import datetime\n", + "import numpy as np\n", + "from collections import defaultdict\n", + "from envs.gridworld_env import CliffWalkingWapper\n", + "from QLearning.agent import QLearning\n", + "from common.utils import plot_rewards\n", + "from common.utils import save_results,make_dir\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## QLearning算法" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "metadata": {}, + "outputs": [], + "source": [ + "class QLearning(object):\n", + " def __init__(self,n_states,\n", + " n_actions,cfg):\n", + " self.n_actions = n_actions \n", + " self.lr = cfg.lr # 学习率\n", + " self.gamma = cfg.gamma \n", + " self.epsilon = 0 \n", + " self.sample_count = 0 \n", + " self.epsilon_start = cfg.epsilon_start\n", + " self.epsilon_end = cfg.epsilon_end\n", + " self.epsilon_decay = cfg.epsilon_decay\n", + " self.Q_table = defaultdict(lambda: np.zeros(n_actions)) # 用嵌套字典存放状态->动作->状态-动作值（Q值）的映射，即Q表\n", + " def choose_action(self, state):\n", + " self.sample_count += 1\n", + " self.epsilon = self.epsilon_end + (self.epsilon_start - self.epsilon_end) * \\\n", + " math.exp(-1. * self.sample_count / self.epsilon_decay) # epsilon是会递减的，这里选择指数递减\n", + " # e-greedy 策略\n", + " if np.random.uniform(0, 1) > self.epsilon:\n", + " action = np.argmax(self.Q_table[str(state)]) # 选择Q(s,a)最大对应的动作\n", + " else:\n", + " action = np.random.choice(self.n_actions) # 随机选择动作\n", + " return action\n", + " def predict(self,state):\n", + " action = np.argmax(self.Q_table[str(state)])\n", + " return action\n", + " def update(self, state, action, reward, next_state, done):\n", + " Q_predict = self.Q_table[str(state)][action] \n", + " if done: # 终止状态\n", + " Q_target = reward \n", + " else:\n", + " Q_target = reward + self.gamma * np.max(self.Q_table[str(next_state)]) \n", + " self.Q_table[str(state)][action] += self.lr * (Q_target - Q_predict)\n", + " def save(self,path):\n", + " import dill\n", + " torch.save(\n", + " obj=self.Q_table,\n", + " f=path+\"Qleaning_model.pkl\",\n", + " pickle_module=dill\n", + " )\n", + " print(\"保存模型成功！\")\n", + " def load(self, path):\n", + " import dill\n", + " self.Q_table =torch.load(f=path+'Qleaning_model.pkl',pickle_module=dill)\n", + " print(\"加载模型成功！\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 训练" + ] + }, + { + "cell_type": "code", + "execution_count": 38, + "metadata": {}, + "outputs": [], + "source": [ + "def train(cfg,env,agent):\n", + " print('开始训练!')\n", + " print(f'环境：{cfg.env_name}, 算法：{cfg.algo_name}, 设备：{cfg.device}')\n", + " rewards = [] # 记录奖励\n", + " ma_rewards = [] # 记录滑动平均奖励\n", + " for i_ep in range(cfg.train_eps):\n", + " ep_reward = 0 # 记录每个episode的reward\n", + " state = env.reset() # 重置环境, 重新开一局（即开始新的一个episode）\n", + " while True:\n", + " action = agent.choose_action(state) # 根据算法选择一个动作\n", + " next_state, reward, done, _ = env.step(action) # 与环境进行一次动作交互\n", + " agent.update(state, action, reward, next_state, done) # Q-learning算法更新\n", + " state = next_state # 存储上一个观察值\n", + " ep_reward += reward\n", + " if done:\n", + " break\n", + " rewards.append(ep_reward)\n", + " if ma_rewards:\n", + " ma_rewards.append(ma_rewards[-1]*0.9+ep_reward*0.1)\n", + " else:\n", + " ma_rewards.append(ep_reward)\n", + " if (i_ep+1)%20 == 0: \n", + " print('回合：{}/{}, 奖励：{}'.format(i_ep+1, cfg.train_eps, ep_reward))\n", + " print('完成训练！')\n", + " return rewards,ma_rewards" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 测试" + ] + }, + { + "cell_type": "code", + "execution_count": 39, + "metadata": {}, + "outputs": [], + "source": [ + "def test(cfg,env,agent):\n", + " # env = gym.make(\"FrozenLake-v0\", is_slippery=False) # 0 left, 1 down, 2 right, 3 up\n", + " # env = FrozenLakeWapper(env)\n", + " print('开始测试!')\n", + " print(f'环境：{cfg.env_name}, 算法：{cfg.algo_name}, 设备：{cfg.device}')\n", + " # 由于测试不需要使用epsilon-greedy策略，所以相应的值设置为0\n", + " cfg.epsilon_start = 0.0 # e-greedy策略中初始epsilon\n", + " cfg.epsilon_end = 0.0 # e-greedy策略中的终止epsilon\n", + " rewards = [] # 记录所有回合的奖励\n", + " ma_rewards = [] # 记录所有回合的滑动平均奖励\n", + " rewards = [] # 记录所有episode的reward\n", + " ma_rewards = [] # 滑动平均的reward\n", + " for i_ep in range(cfg.test_eps):\n", + " ep_reward = 0 # 记录每个episode的reward\n", + " state = env.reset() # 重置环境, 重新开一局（即开始新的一个episode）\n", + " while True:\n", + " action = agent.predict(state) # 根据算法选择一个动作\n", + " next_state, reward, done, _ = env.step(action) # 与环境进行一个交互\n", + " state = next_state # 存储上一个观察值\n", + " ep_reward += reward\n", + " if done:\n", + " break\n", + " rewards.append(ep_reward)\n", + " if ma_rewards:\n", + " ma_rewards.append(ma_rewards[-1]*0.9+ep_reward*0.1)\n", + " else:\n", + " ma_rewards.append(ep_reward)\n", + " print(f\"回合：{i_ep+1}/{cfg.test_eps}，奖励：{ep_reward:.1f}\")\n", + " print('完成测试！')\n", + " return rewards,ma_rewards" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 设置参数" + ] + }, + { + "cell_type": "code", + "execution_count": 40, + "metadata": {}, + "outputs": [], + "source": [ + "curr_time = datetime.datetime.now().strftime(\"%Y%m%d-%H%M%S\") # 获取当前时间\n", + "algo_name = 'Q-learning' # 算法名称\n", + "env_name = 'CliffWalking-v0' # 环境名称\n", + "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\") # 检测GPU\n", + "class QlearningConfig:\n", + " '''训练相关参数'''\n", + " def __init__(self):\n", + " self.algo_name = algo_name # 算法名称\n", + " self.env_name = env_name # 环境名称\n", + " self.device = device # 检测GPU\n", + " self.train_eps = 400 # 训练的回合数\n", + " self.test_eps = 20 # 测试的回合数\n", + " self.gamma = 0.9 # reward的衰减率\n", + " self.epsilon_start = 0.95 # e-greedy策略中初始epsilon\n", + " self.epsilon_end = 0.01 # e-greedy策略中的终止epsilon\n", + " self.epsilon_decay = 300 # e-greedy策略中epsilon的衰减率\n", + " self.lr = 0.1 # 学习率 \n", + "class PlotConfig:\n", + " ''' 绘图相关参数设置\n", + " '''\n", + "\n", + " def __init__(self) -> None:\n", + " self.algo_name = algo_name # 算法名称\n", + " self.env_name = env_name # 环境名称\n", + " self.device = device # 检测GPU\n", + " self.result_path = curr_path + \"/outputs/\" + self.env_name + \\\n", + " '/' + curr_time + '/results/' # 保存结果的路径\n", + " self.model_path = curr_path + \"/outputs/\" + self.env_name + \\\n", + " '/' + curr_time + '/models/' # 保存模型的路径\n", + " self.save = True # 是否保存图片" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 创建环境和智能体" + ] + }, + { + "cell_type": "code", + "execution_count": 41, + "metadata": {}, + "outputs": [], + "source": [ + "def env_agent_config(cfg,seed=1):\n", + " '''创建环境和智能体\n", + " Args:\n", + " cfg ([type]): [description]\n", + " seed (int, optional): 随机种子. Defaults to 1.\n", + " Returns:\n", + " env [type]: 环境\n", + " agent : 智能体\n", + " ''' \n", + " env = gym.make(cfg.env_name) \n", + " env = CliffWalkingWapper(env)\n", + " env.seed(seed) # 设置随机种子\n", + " n_states = env.observation_space.n # 状态维度\n", + " n_actions = env.action_space.n # 动作维度\n", + " agent = QLearning(n_states,n_actions,cfg)\n", + " return env,agent" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 执行训练并输出结果" + ] + }, + { + "cell_type": "code", + "execution_count": 42, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "开始训练!\n", + "环境：CliffWalking-v0, 算法：Q-learning, 设备：cuda\n", + "回合：20/400, 奖励：-82\n", + "回合：40/400, 奖励：-51\n", + "回合：60/400, 奖励：-50\n", + "回合：80/400, 奖励：-53\n", + "回合：100/400, 奖励：-21\n", + "回合：120/400, 奖励：-35\n", + "回合：140/400, 奖励：-44\n", + "回合：160/400, 奖励：-28\n", + "回合：180/400, 奖励：-28\n", + "回合：200/400, 奖励：-17\n", + "回合：220/400, 奖励：-18\n", + "回合：240/400, 奖励：-22\n", + "回合：260/400, 奖励：-19\n", + "回合：280/400, 奖励：-15\n", + "回合：300/400, 奖励：-14\n", + "回合：320/400, 奖励：-13\n", + "回合：340/400, 奖励：-13\n", + "回合：360/400, 奖励：-13\n", + "回合：380/400, 奖励：-13\n", + "回合：400/400, 奖励：-13\n", + "完成训练！\n", + "保存模型成功！\n", + "结果保存完毕！\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYwAAAEcCAYAAADUX4MJAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAABEx0lEQVR4nO3dd3gU1frA8e/MlvROOh2kiUIgFAFRmoB0G1wUG4j+VBAVFRuoXK+ABYWLIopwVa6oV4qASBEQFEQiVRCkE0nvPZvdnd8fIUuybJINaSu8n+fhYbNnyjuzs/vOOWdmjqJpmoYQQghRCbW+AxBCCPH3IAlDCCGEUyRhCCGEcIokDCGEEE6RhCGEEMIpkjCEEEI45YpOGH379mXnzp11vt6YmBgGDhxY5+sVF/3111+0bt0as9lc48vetGkTN910E1FRURw5cqTGlnslHq8pKSncfffdREVFMWvWrFpZh73WrVtz9uxZAKZPn86CBQtsZf/973/p0aMHUVFRpKen89tvv3HLLbcQFRXF5s2baywG++Nv3LhxfP311w6nnTBhAitXrqyxddcmfX0HcCWKjo5mw4YN9R2GqCWzZ8/m5Zdfpn///g7LNU1j8eLFfPXVVyQkJBAYGMjw4cN5/PHHMRqNdRxt5WrzeP3yyy8JCAhg7969KIpSI8tMSkri3XffZfv27eTm5hIaGsqtt97KhAkT8PT0LDPta6+9ZntdVFTErFmz+Oqrr2jTpg0A8+bN4+677+a+++5j7dq1DB48mPXr19vmeeCBB0hISLjkvRtuuIGJEyfWyPZ8/PHHNbKcqvrrr794/vnnOXjwIOHh4UyfPp0ePXpUOM8VXcOoLRaLpb5DqLYrYRvqS1xcHNdcc0255f/85z/56quvmD17Nnv37uWjjz5i586dPPXUU3UY5UX1+VnHxcXRokWLy0oWjmqHGRkZjBkzhsLCQpYvX86+fftYsmQJWVlZnDt3rsLlpaamUlhYSMuWLcvEV/JZdunShVOnTpGWlmZb/9GjRyksLCzz3v79+4mOjq7y9riap59+mnbt2rF7926efPJJJk+ebNvO8lw1CcNqtbJo0SL69+9Pt27deOKJJ8jIyLCVT548mZ49e9K5c2fuvvtujh8/biubNm0aM2bM4KGHHqJjx47s3r2bvn37snjxYoYNG0bnzp2ZMmUKhYWFAOzevZvevXvb5q9oWoCPPvqIXr160atXL77++usyVWp7GRkZPP/88/Tq1YsuXbrw6KOPArBixQr+8Y9/lJm29HLst2Hx4sX07NmzzI/Jpk2bGDZsmFP7y95XX33FgAED6Nq1K4888giJiYll4vjiiy+45ZZbiI6O5tVXX6W8BwxYLBYWLlxI//79iYqK4rbbbiM+Pt5hE1Ppar7FYmH27Nl069aNfv368eOPP5ZZ7jfffMPgwYOJioqiX79+LF++vNxtsVqtvP/++/Tp04cbbriBZ599luzsbEwmE1FRUVgsFkaMGOGwhnHmzBn++9//8tZbbxEVFYVer+eaa65h/vz5bNu2jV9//bXc9drH8Hc/XqdNm8aqVatYvHgxUVFR7Ny5E5PJxOuvv26b//XXX8dkMpWJY9GiRfTs2ZPnn3/+kmUuWbIELy8v3nzzTRo2bAhAeHg4L730kq3WYB/D3LlzOX36NIMGDQKKE8O9995L//79iY2N5ZFHHiEqKoqAgAAaNWrEnj17ADhy5AgtW7akS5cuZd6zWq1cd911bNu2jZEjR9KpUyduuukm5s+f79Rnm5SUxLBhw2w1i9LHccn3ePbs2XTp0oW+ffuWOZZjY2NtTXz3338/r776KlOnTnW4ngkTJvD555+XeW/48OFs3LiR06dPc/jwYSZNmoS7uzsDBw6kVatWldY0r5qE8dlnn7F582Y+//xzduzYgZ+fX5nqau/evdmwYQO7du2iXbt2l3wIa9eu5ZFHHmHv3r107twZgPXr1/Pxxx/zww8/cOzYMVasWFHu+subdvv27SxdupQlS5awadMmdu/eXeF2PPvss+Tn57Nu3Tp27tzJ/fff7/Q+KL0N9913Hx4eHvzyyy+28jVr1tgSRmX7q7Rdu3bx9ttv8+677/LTTz8RGRl5ydn0tm3b+N///se3337L+vXr2bFjh8NlLVmyhHXr1rFo0SL27t3Lv/71L9zd3Svdtq+++oqtW7eyatUqvvnmG77//vsy5UFBQXz44Yfs3buXN954gzfeeIPDhw87XNaKFStYuXIln376KZs3byYvL4/XXnsNo9HIvn37AFi9erXDNu9du3YRFhbG9ddfX+b98PBwOnbsyM8//1zptsCVcbzOmjWLYcOGMX78ePbt20ePHj344IMPOHDgAKtXr+bbb7/l0KFDvP/++7Z5UlJSyMzMZOvWrcycOfOSZe7atYsBAwagqlX76WrWrBlr164FYM+ePbbPNiIigoULF7Jv3z6MRmOZ5LBnzx6io6Pp3Llzmfc6dOiAwWDAw8OD2bNnExMTw4cffsgXX3xRaT9IbGws48aN45577mHChAkOpzl48CDNmjXjl19+YcKECbz44ou2E6ypU6dy/fXXs3v3bh5//HFWr15d7rqGDh1q22aAEydOEBcXx80338yJEydo1KgR3t7etvI2bdpw4sSJCuO/ahLG8uXLefLJJwkLC8NoNPL444+zYcMG2xnrHXfcgbe3N0ajkUmTJnH06FGys7Nt8/fr14/OnTujqipubm5A8ZlBaGgo/v7+9OnThz/++KPc9Zc37fr167ntttu45ppr8PDwYNKkSeUuIykpie3bt/Pqq6/i5+eHwWCga9euTu8D+20YMmSI7YDKyclh+/btDBkyxKn9VdqaNWu4/fbbufbaazEajTz11FPs37+fv/76yzbNQw89hK+vLxEREXTr1o2jR486jPHrr7/miSeeoHnz5iiKQps2bQgICKh029avX899991HeHg4/v7+PPzww2XKb775Zho3boyiKHTt2pWePXsSExPjcFlr1qzh/vvvp1GjRnh5efHUU0/x3XffOdWBnp6eTnBwsMOy4ODgSqv8Ja6E49WRNWvW8NhjjxEUFERgYCCPPfYY3377ra1cVVUmT56M0Wh0eKKQkZFR7v6tCV26dLEdFzExMbaEUfq9ku9ct27daN26Naqq0qZNG4YMGVJhDfLEiRPcd999TJo0idGjR5c7XUREBHfddRc6nY5Ro0aRnJxMSkoKcXFxHDp0yLZ/oqOj6du3b7nL6d+/P0ePHuX8+fNA8b4fMGAARqOR3NxcfHx8ykzv4+NDbm5uhfvnqkkYcXFxPPbYY0RHRxMdHc2tt96KqqqkpqZisVh466236N+/P506dbJ9COnp6bb5w8PDL1lm6QPXw8ODvLy8ctdf3rRJSUmEhYVVuJ4SCQkJ+Pn54efn58QWX8p+2cOGDWPTpk2YTCY2bdpEu3btiIyMBCreX/aSkpJs8wF4eXnh7+9fplnKfvvLOzATEhJo3LhxlbctKSmpzPZFRESUKf/xxx+566676Nq1K9HR0Wzfvr3M51vR9kRGRmI2mx1uu72AgACSk5MdliUnJ9uS34QJE4iKiiIqKqrMD2aJK+F4dSQpKanMZxMREUFSUpLt74CAAFuCc8Tf37/c/VsTunTpwrFjx8jMzOTAgQN07NiRFi1akJycTGZmJnv37rX1Xxw4cIBx48bRvXt3OnfuzPLly8s9pqD4BzskJKTSK9IaNGhge+3h4QFAXl4eSUlJ+Pn52d6Dsvt/+vTptmNq4cKFeHt7c9NNN7Fu3TqguNY5fPhwoPg7mpOTU2a9OTk5eHl5VRjbVZMwwsLC+Oijj4iJibH9O3ToEKGhoaxZs4YffviBJUuW8Ntvv7FlyxaActvZa1JISEiZH9b4+PgKtyEzM5OsrKxLyjw8PCgoKLD97cyXqmXLlkRERLB9+3bWrl3L0KFDy6yrvP3laBtKzmKg+ODOyMhwOG1lwsLCHHZellz9Ut42BgcHl9l3pV+bTCYmT57Mgw8+yM8//0xMTAy9e/cu9/O13564uDj0ej1BQUGVxt+9e3fi4+M5ePBgmffj4+PZv3+/7ez0448/Zt++fezbt8/2JS7tSjhey5s/Li6uzPwhISG2vyvrHL/hhhvYtGkTVqu1Sut1VqNGjQgJCeHLL78kPDzc9gPasWNHvvzyS3Jzc+nYsSNQ3Glc0l/222+/MWbMmAo/g8cff5yAgACefvrpy7oQITg4mMzMTPLz823vld7/r732mu2YeuSRR4DiZql169axb98+CgsL6datG1D83Y+NjS2TNI4ePVrmggBHrpqE8Y9//IN3333X9kOQlpZma2/Mzc3FaDQSEBBAfn4+77zzTp3FNWjQIFasWMHJkyfJz88v055rLyQkhN69e/Pqq6+SmZlJUVGRrW21TZs2HD9+nD/++IPCwkKnO+CGDh3Kf/7zH/bs2WPrFISK95ejZaxYsYI//vgDk8nEO++8w/XXX2/rlKyKO++8k/fee48zZ86gaRpHjx4lPT2dwMBAQkNDWb16NRaLhf/973/Exsba5hs8eDCfffYZCQkJZGZmsmjRIluZyWTCZDIRGBiIXq/nxx9/rLAvoWSfxMbGkpuby9y5cxk8eDB6feVXoTdr1owxY8YwdepU9u/fj8Vi4fjx40yaNImoqKhKL1sscSUcr44MGTKEDz74gLS0NNLS0liwYIGt38wZDzzwALm5uTz33HO2fZOYmMgbb7xRbjNnVUVHR7N06dIyV0J17tyZpUuX0r59e1tTWW5uLn5+fri5uXHw4MEy/QWOGAwG3nvvPfLz83n22WernPQiIyNp37498+fPx2QysW/fPrZu3VrhPDfddBNxcXHMmzfPVkuF4uO0bdu2LFiwgMLCQjZt2sSxY8cqrf1cNQnj3nvvpW/fvjz44INERUVx11132c4CR44cSUREBDfeeCNDhgyxnUHUhZtuuolx48Zx7733MmDAADp06ABQ7vX6c+bMQa/XM3jwYHr06MF//vMfoPgAeOyxx7j//vu55ZZbbB2dlRk6dCh79uyhe/fuBAYG2t6vaH/Z69GjB0888QSTJk2iV69exMbGMnfu3KrsBpsHHniAwYMH8+CDD9KpUydefPFF2xU6M2fOZPHixXTr1o0TJ04QFRVlm++uu+6iV69ejBgxglGjRnHLLbfYyry9vXnppZeYMmUKXbp0Ye3atRW2/d5+++0MHz6ce+65h379+mE0Gnn55Zed3obp06dzxx138Mwzz9ChQweGDh1KREQE77//vtOdtVfK8Wrv0UcfpX379gwfPpzhw4dz7bXX2q70c4a/vz9ffPEFer2eu+66i6ioKO677z58fHxo0qTJZW2TvS5dupCamlrmOxQdHU1qaipdunSxvTdjxgzmzZtHVFQUCxYsYPDgwZUu22g08u9//5vU1FReeOGFKieNt956i/3799OtWzfeffddbr311gr3vdFoZMCAAezcubNMCwLAO++8w++//06XLl146623mDdvXpnfAEcUGUDJtZw8eZKhQ4dy6NAhp85oheubN28emzZtYtmyZfj6+tZ3ODVKjtf6NWXKFJo3b87kyZPrZH1XTQ3DlZV0PGdmZvLmm2/Sp08f+fJdQSZPnszo0aPZv39/fYdSI+R4rT8HDx7k3LlzWK1Wtm/fzg8//FDuEwdqg9QwXMD48ePZv38/Op2OLl26MGPGjDIdgUK4Ejle68+WLVt49dVXycjIICwsjIkTJ3L77bfX2folYQghhHCKNEkJIYRwiiQMIYQQTpGEIYQQwilX9KUN6em5WK1V76IJCvImNTWn8gnrmMRVda4am8RVNRJX1VxuXKqqEBBQ/uNBruiEYbVql5UwSuZ1RRJX1blqbBJX1UhcVVMbcUmTlBBCCKdIwhBCCOEUl04Yp0+fZvTo0QwcOJDRo0dz5syZ+g5JCCGuWi6dMGbMmMHYsWPZsGEDY8eOZfr06fUdkhBCXLVcNmGkpqZy5MgR2xMWhw4dypEjR5wesUwIIUTNctmrpOLj4wkNDUWn0wGg0+kICQkhPj6+0kfwCvF3oWlapYMG1cc6KppH0zQ0QHVQbv+koZJllJ6novkBrJpGSUnJ0kpf8WMttQ7lwjqsFTzhSC2nvCSWyuZXSsVxSaxWrcJ5y1seFSyzoumrOm9Nc9mEUROCgrwrn6gcwcE+lU9UD2ojrj/PpXPkdCojerewfcFPx2USGuiJp7sBAIvFSlJ6PuENvGzl5xKyCfR1x8/f02Fc5f3oHI9NR0GhZSN/2zQFhWay84rY/OtZht3YHG/PS5/xf+xsGmfisxjYvantvZN/ZfDnuXT6dmmMm0HH6bhMNu4+y40dI8nMKUTT68gxWWke6YemaZz4K4MTsRl0ahNKZk4hft5uhAZ6cjouE00Ds8XKsg1HaRHpx7jBbTl9PpO5/43h+pZBtG8eyJGTydx3a1vOnM8gKT2PrJwCtv96nNv6tMLfx4PUzDyycwpISskiKysbPWbM6GnWOJhrmgSz7dfTeBmt+LjByTPJpKZm0qqhD+1bNMDfz5OdhxI4k5DD8N7X4O2pJyunkJ/3/4WbHho18CCv0EJ8SjbtmgdzIi6X1JR0GgbowGImP78QBQ2jXsFdr2EtKkJVNHLNejzd9VzT0JfjsRnk5ptw06tYLBbcDCqR3QfwzoYUIoI88FaLwJSLu2JmdP8W/LwvltT0bPyMVgxaEe1bhbL9t1hM+Tk0CtCjsxbh56UnPTOPXI8ItmVEkptfRKSfSlFmCuFeZryNGnk5uRithYT6qGhF+VBUSJCXimYuRK9YMeoUDHoFk8mMyVSEh1FF06yYiywoaKgXtgvNima1lnoPjHqVgkIzoKHDioqGqljRoWHCwK8BQ/jlbBH+ah7+ai4+SgEeqgl/oxWdJR93nRXNYkaHFT1WdIoFHRo6xYIeKwZVQ9EsxeWKFR1WdFhQFSgetPdiggNQ0Mq8dlR28b2L06rl5PRszZPtHv34KzkXf10BvvpCDJYC3BQzbkoRbooZd6UII2b0igWzYsD64EuEN4xwvMBqcNmHD6ampjJw4EB2796NTqfDYrHQrVs3Nm7c6HQNIzU157KuRQ4O9iE5ObvK8zkjLiUXP28jXhd+iDVNIye/CB8HP5DlxVXyI1tktmLQX9qqeDYhm58PxZOQnkeX1iEY9Crdrw0jt6AIs9mKn7cbSRn5xCXn0r55IDM++ZX41DwmDm9H93ZhpGTm8+wHu2jT2J9nx3YiJ7+IrfvOs3L7KV4c1xk/LyPPLtxlW98dfa+hcQNP3N30NArxJjkjn6+2HOdcbBIN/XWE+Orp1jqI5uFeHDqexPpdp9BhpV9UOHuPJtAkxIvEbDOFman4KvmoaPh5G/BxU/Fy05GZZ0KzWsnOyUd34QusVzT8PFVycgsufIGteBgVMoqMuGmFGBQLesWCm2JBhwWjrvjLarVYURXN9oOjoqHXKVgtljLv2/6v3ZP/WmPRFMyKHk0Dd6UIq1b6Z01BUxSsmoKKFTMq2VYPAnW56Kj60KcWrfhnUa9YOW8OIEDNxVM1lTu9FQWraqBQ02NGj6bqMZk1LBoYDHp0qo6sfDMoCn7e7qCoKDodGTkm8k1WAv08UFUdmqKQnlNEYZGVAB83DAY9mnLhU1VUCs3gm3mcIJ3jG9jMih6z6o5Z0YOqB0WHVdGhlfrfgorJqqA3GMq8b1VUQEFv0GMuspRNA8qFfWxbk1Lm/0KzFVVV0OtUu3L7moNCVp6JJjkH8FfLjr9uVXSYVSMW1YhFdbvwvwGrosfq5kPHux8nu6DqPQ6qqlR4ou2yCQNg3Lhx3HHHHYwYMYLVq1fzv//9j88++8zp+WsrYeQXmjkVn8W1TZ1LXL8dS6JpmC8BPm5MmLOVxqHevPJA8djOOw7EsWT9UV55oAuNQrzZfiCO7QfiGNW7OU3DfNl1OIEz8Vn8ciSRtyb3Jjsrnze/2EdksDd/xmYwuFtjRt7YjOy8Ilb9dBp3o47Dp9NISs/HUmrbn7jjehatOQxFhQzs0pADh0+jL8igdYsIfj+VSiM1Cc+AQBpG9WLx+j8BULES6mWlIL/4bCZMl4G/wYxRK8RPyaWZMRV/LZM8zQ0APRYMqhVVs2BQLBiU6o27bNEUrCgogAUVi6ZgQYemqBRZVSwouLm5odPryTFpmMwanloeeZobit6Il7cnqblW3N3dyC8qbgYJ8vPEy8NISlYhbm4GTsfnYNEg0M+TgiIriqLStlkQ51PySM4sJKfATJe2YRw5m4HZCoVmDUVVaRruR2J6Pka9SlCDQPYdT+aaCB/CGniDolKkqRjdPckxwdm4NM7+lYqPO9zSvTmZBRASGkhWnoXwEH9OJ+aRmJ5HUZGZTi0DWbXjFGfOp9O3c0OSMgoZ0KUJnp5uZBVYOHk+E4tVIzE1l8hAd6LaNUR18yQpy0SeScOiKTQK8cHNrfiE5PCZNP48l0GnVsGkZRVQZLHSte3Fsdb/tWgzvU3bQdHRpUs7dL5BHDqbS2CgH6cS82jdJIiQBr4oRk+Ssi38cTqZjtcE4x/oj2L0wIye3UeSOJ+UhdeRlYSoWTRq0YzA8EhU70DydD4kZ5tp1jgYxeiJYvQAnbHSJrI/YzNwM+hoElZccw0O9iExMYucgiJ8nTi5AkjLKuBfCzcQbTxNcFgwN/Voj+IdiOLhVxyLrvoNLLV5cglw+HQai77aRWN9Kjd0akH36FYo7r4oBrdaietvnTBOnjzJtGnTyMrKwtfXl9mzZ9O8eXOn56+thLFg5SF+O5bM24/1JMCn7Af3y+EEdv6ewFOjO7LjYBy/HE7kj7PpALz+UDde/Gg3AJ9M68sfZ9NZueMUJ/7KBKBhsDd/JV88G2oc6s25xIt/D+jamD9Op9mm0WNBxUrjAAUtO5VANQcfNZ9CzcCNIZkYss6TYfXARy1ARcNdNeOjXBxA3pEktyYczzTS3j8Xr4JE9DgerF7TGdGHtuBUjjupyWnFP1A6AxZNRVP1+Hh70qRpJOcyLDSNDOJsUj4nEnKIDPHjupYhfLrpJAnpBUwYfh2/n0rlz7PJPHRnT1Rvf+LTCmgc5ouiKMSl5FJYZCEs0JPT8Vm0auSPqirEHE1i3/EUxg9paztTO3Aihff+d5CnRnegXZNAVFWp9LO0WK0oilJue7qzKusnKDJb0OtU2zQVxVVktlBYZMXbw1CtmJwxe9lejsVmEBLgwayHb7jsH5rtB+JYur54TO2Z47sSGXz5zcGOXE5cpiILj7z9IwD9Ozdk7IBWNRrT5cZVFbFJOcz45FcAJg5rR/drw2o1rsoShkv3YbRo0YKvv/66vsOg0GRhxfZTjOrdDHejnrMJxR9EZm7hJQlj0ZojAJxLzGbJd2UHpd99JNH2euX2U6zZeaZMeelkUbyMkr81gtQcfv/tAIFqDtFeyTTVJdBYn3axCcFu5M+iIl/O6gIxajmcNwdgRSU8xB/3Ro3Y8XsSbVo1xuIewKafj6IqVrr17sEfO7YwUovBx92Ad2AL1KDr2HmmiLQcC94+XnTu1pETKVYahATSsnEwiqIQkJLLqu+O8tCwtoQGeF6y7/wv/H/dtXBdqfdvv6MFCel5RDYLIrI1DLBqqBcacZuEX9ynEQ0uPtemXakaXde2oWXOlAE6tGzA/Ck32pr7nKFzcoztylR2xmzQ65xelkGvq9L01eHnXXy27uxZe3lKz+/jVb1l1RSjQYebQUdhkQVvz9pPvrXBr9S+9HWB/erSCcNV/LD3LzbFxOLlrmd4r2a2s9H07EKaXkj4FquVzzYcs82z4ddzlyyndIKwTxYARoOKqag4AQSq2bTQJ9FUn0wH33R8TMm26TRFR6ohDK15f77dm0KbaxrSoUMrVO8GPPHhXjwUE08+2J9dO04TcyyZZ/4RRXJGPs3ahuJm1DGiR/FyCk0WVh03MqxnU0IDPVm4uS27ClvSODKIF4ZGA9Cn+4XW2ZIz42ZlY45s4MV7T99c5bOZBv4eNPD3sP2tltfjV0VVSRYC/LyKk7NPNX9QS+ZXFPB2oc/Ax9NAYabFqT5CV+TtYUBRQNPKJo/6IgnDCWZz8Y/41n3ncXfT2/qo0rIK0TSN7PwijpxJY/uBeNs8uw4nXrIcTYMGfu6kZBYAEOTrTmpW8ev+nRsS4WXm1O5t9An4iwamOAAsOjeMwS2xRvQluciTBsGB+DRqja+++OC5vbsZd+PFj7FH52vYFBNLaKAXjUJ92Hc8hSahPrRtEnBJPG5GHS+M63whNg2jXsVkNuDrdfEMv7Yv+RT1y/9CDcPNUL0aTUmtwsfDUGPJvyZ4exhIySzApw6a92qDqir4ehrJzDVJDeNv48Lxn5lrYvkPxwn2dwdg2aY/+f1UKgdOpjqcrXRyaN3In2OxGYzu2xKzRWPtrjP0jYrks41/0qOpgdvcd1J0eDudvayovk3QNx6BvkVXVP9wFKW42aSZg3bJ0skCYHS/ltzZpwWqqnBLdCM6tAjC073yj1lRFIL83IlPzav22ab4+yg5867qvQT2fC8cM67SHFWipCmqLvqDaouvl5HsvCK8XGAbJGE4w+67lJxRYHtdXrIAaBTibUsY44e25dDJVDq1Km7779YulFPHjjPcI4a+WUcoytZhaNcXfYtu6EJbXvaZvaooqLried2MOhqHOn/fxsWE4VpfelF7dBeOleo+CdvNoMOoV6vdF1LTfDwu1Hz+xidBfl5GsvNM1b4woyZIwqhEzNEkdh25tHmpIte3CAKKm5n2HU8BoIGfB306NQSKm3+Kft9I8C9f0c/DAs264tX9TlSf4JoNvoo83YoPh7/zl0tUTeSFiwraOWiyrApFUQjwdb/kIpD6VnIsO7oR9O+iS5sQ2+XF9U0SRiXeX/V7lefx93bj/sFtANDrFErfmGNJjaVg+ydYk0+jbxKFscttqAENXaKvoKQdW+9CbdCidjUO9eGtR3vUyA/947ddZzvpcBURDbzw9TTg7eFacVXFjR1q/o7ty/X33Yv1pFm4L0F+7gzu1pijZ9P5etvJS6ZxN17sQHxv8o2219bMBPK+fR1F74b7TePRt+rlEomiRKBvcd+MK3VaitpX8rlXV2SD8of2rC+9rg/nhmtDa+zy6audJAwHzBYrz36wkztubnFJ2eBujYluEwJAgI8bX287ybAeTctcJls6YXhcOOOypP1F/vdzQdXhOWo6qndQ7W7EZbi1e2OMepWe14XXdyhC1AhVUVDr6J6Wq4GkXQcKCs2kZBbw4erDl5SVvrTN39uNT6b1pdf1ZX9g7a9csqScJW/1P8FqwfPWqS6ZLKD4hrHB3ZuUesaNEEJcJDWMCji6cMRRh7D9e+5uF89oNHMh+ZsXoBg98BzxkssmCyGEqIycSjpQ0RWGji45tb/pqaRJStM0CmNWomUl4d5noiQLIcTfmtQwHHB0D5Ovp4HWjQMc3gRn33Gtv9DBVnRoA0UHv8fQujf6iLa1EqsQQtQVSRgO2D/Ad3C3xozo1QxjBY9PePy26/jv5j9JyyrEqmloBTkU7l2NrtH1uPW+v5YjFkKI2idNUk5oGOJdYbIA6NQqmFaN/IHiGofp4HowFeDW7U7boz2EEOLvTGoYDtg3Sbk7+WC20X2vwd2o5/pQDdPPm9C37I4usFEtRCiEEHVPTn0d0Oy6vY1G5xKGn5eRewe2xvLL56CquHW9ozbCE0KIeiEJw5HLrGEAmM8fwRJ7CLfOI+SqKCHEFUUShgP2F0lVZawA08HvUTz9MbTrV7NBCSFEPZOE4YD9VVJuTjZJWXPTsfz1O4ZWvVD0f9+nYwohhCOSMJzgTA1DKyokb+0sUFQMrXvVQVRCCFG3JGE4YH+VlDM1DPOZ39AyE/EY8BiqX1gtRSaEEPVHEoYD9gnDqK94N2maRtGxHSjeQegad6jFyIQQov7IfRgOlFxW27FlA4wGtdIxK8ynfsUS9wdu3cfITXpCiCuWJAwHSmoYUa0acOP1lY92ZTq0ETUgAkP7W2o5MiGEqD9yOuxAyVVSCpWPPGfNScOadBJ9i+4oMqqXEOIKJr9wFXBm9FTzmd8A0DePruVohBCifknCcMDR483LYz4dgxoQgc7fdQZqF0KI2iAJw4GSTu/KahjW/CwsCX+ibya1CyHElU8ShiMXahiV9WGYz+wFTZOEIYS4KtTqVVLTpk1j586dBAQEADBo0CD+7//+D4CUlBSeffZZzp8/j5ubGzNnzqRDhw6VltUFW4tUJTUM8+kYFN8QVHmEuRDiKlDrl9VOnDiRe+6555L33377baKjo/nkk0+IiYnhmWeeYcOGDSiKUmFZXbh4lVQF05hNWOKOYri2X53FJYQQ9anemqS+//57xowZA0B0dDRGo5FDhw5VWlYXbJ3eFeQBS9JJsJrRR8pY3UKIq0OtJ4wlS5YwbNgwHn30UU6ePAlAeno6mqYRGBhomy48PJyEhIQKy+paRX0YlrijoCjowlrVYURCCFF/qtUkNWrUKOLi4hyW7dy5kyeffJLg4GBUVWXVqlVMmDCBzZs3V2eVVRIU5H1Z851LyALAz9eD4GAfh9PEpRzHLaw5IZGhlx3f5SgvnvrmqnGB68YmcVWNxFU1tRFXtRLGypUrKywPDb34Yzpy5EjeeOMNEhISiIyMBCAtLc1Wk4iPjycsLMzWQe6orKpSU3OwWqtwU8UFJXNkZeeTnJx9abnZRMH5PzFc299heW0JDvap0/U5y1XjAteNTeKqGomrai43LlVVKjzRrtUmqcTERNvrHTt2oKqqLYkMGjSI5cuXAxATE0NBQQHt27evtKxOlFxWW05ntiXpJFjM6MPb1F1MQghRz2r1KqnnnnuO1NRUFEXB29ubDz74AL2+eJVPP/00zzzzDKtWrcLNzY05c+agXngWU0VldaGyPm9LYnFfjC7smjqJRwghXEGtJoylS5eWWxYcHFxueUVldcF+iFZ71tRzKD7BKG5edRSREELUP7nTuwLl3V5hST2HLqhx3QYjhBD1TBKGA1oFt3prRYVomYmokjCEEFcZSRgOlDRJqQ5qGNa0WEBDDZLHgQghri6SMByo6E5vS2osADpJGEKIq4wkDAdsjzd3kDGsqefA6IHi3aCuwxJCiHolCcOBCmsYabHoghrLAweFEFcdSRgVsE8JmmbFmhorHd5CiKuSJAwHbI83t8sYWlYSmAvRyfgXQoirkCQMBy7etlc2Y1hSzwGgNpAahhDi6iMJwxHbs6TKvm1NjQVFRfWPqPuYhBCinknCcECzjeldliX1HKp/OIreWOcxCSFEfZOE4YCG44xhzUxA9Q+v+4CEEMIFSMJw4GIN42LG0KxmtKwUVL+qj8shhBBXAkkYFSlVw9CyU0CzoPpLwhBCXJ0kYThgu6y21HvWzOIxxVXfuh2SVQghXIUkDAcc3ehtzSwePVDxk4QhhLg6ScJwxJYxLqYMa1ZS8TOk3F1zwHchhKhtkjAcuPjwwYusWcmoPiHyDCkhxFVLEoYDmoMb97TsZFTf4PoJSAghXIAkDEfshvTWNCvW7GQUH0kYQoirlyQMB2xNUheqGFpeJljMUsMQQlzVJGE4YLVrkrJmJQGg+obUU0RCCFH/JGE4Ynent5adDIAqTVJCiKuYJAwH7J8lZc1KBkVB8Q6qv6CEEKKeScJwwP5ptdasJBSvQBSdvt5iEkKI+iYJoyIlNYzsZOm/EEJc9SRhOHDxWVIX+jCykqX/Qghx1ZOE4UDpJ4No5kK0/EwUuaRWCHGVk4ThSKkb96xZKYBcISWEENVOGKtXr2bYsGG0a9eOzz//vExZfn4+U6ZMYcCAAQwaNIitW7dWu6wu2JqkFNCy5R4MIYQAqPZlP23btmXu3LksWrTokrLFixfj7e3Npk2bOHPmDHfffTcbN27Ey8vrssvqwsXHmyvFl9SCNEkJIa561a5htGrVipYtW6Kqly5q/fr1jB49GoCmTZvSvn17tm/fXq2yuqCVbpLKTgaDO4qbd52tXwghXFGt9mHExcURGRlp+zs8PJyEhIRqldWNi01S1qwkVN9geay5EOKqV2mT1KhRo4iLi3NYtnPnTnQ6XY0HVVOCgi6vVnA8PhuAwEAvdAXp6APDCA52jYGTXCUOe64aF7hubBJX1UhcVVMbcVWaMFauXHnZC4+IiOD8+fMEBgYCEB8fT7du3apVVhWpqTlYrVrlE9opmSM9PQ//zFS0Bi1JTs6u8nJqWnCwj0vEYc9V4wLXjU3iqhqJq2ouNy5VVSo80a7VJqlBgwbx5ZdfAnDmzBkOHTrEjTfeWK2yOlHyaBBLERTmonj61926hRDCRVU7Yaxdu5bevXvz/fff895779G7d29OnDgBwPjx48nKymLAgAE8/PDDvPbaa3h7e1errC7YxsMozARA9Qqos3ULIYSrqvZltUOHDmXo0KEOyzw9PZk3b16NltWFkqukdAXFCUORhCGEEHKntyMlN+6pJQlDmqSEEEIShiMlNQw1P6P4f6lhCCGEJAxHbHd652eAzghGz/oMRwghXIIkDEdKmqTy01B9guSmPSGEQBKGQ7YaRl6aDMsqhBAXSMJwwNaHkZuG6t2gfoMRQggXIQnDIQ0jRSimHBQfqWEIIQRIwnBI0yBAzQVA9ZEahhBCgCQMhzQNAnTFCUP6MIQQopgkDIc0/JQ8AFRPuQdDCCFAEoZDmga+agEAiqdfPUcjhBCuQRKGAxrgq+ahGTxQ9Mb6DkcIIVyCJAwHNA181ALwkNqFEEKUkIThUHEfhubuW9+BCCGEy5CE4YDUMIQQ4lLVHg/jSnP0px9I3rebJsZ8kBqGEELYSMKwk/fXn/R0+xMAq9QwhBDCRpqk7BTpSj3KXGoYQghhIwnDTpHOw/Zaca+7ccSFEMLVScKwUzph4O5Tf4EIIYSLkYRhx1SqSUpxkxqGEEKUkIRhx6Iv1YfhIX0YQghRQhKGnTJNUnq3+gtECCFcjCQMO+ZSNQxVlbG8hRCihCQMe6rO9lLShRBCXCQJw46ilE4TkjKEEKKE3OltR1VgVV5nsq0ePCD5QgghbKSGYUdRFLYWXEuMqXl9hyKEEC5FEoad0i1SitQwhBDCptoJY/Xq1QwbNox27drx+eeflymbNm0avXv3ZsSIEYwYMYIPPvjAVpaSksKDDz7IwIEDGT58OAcOHHCqrLaV7sNQpA9DCCFsqt2H0bZtW+bOncuiRYsclk+cOJF77rnnkvfffvttoqOj+eSTT4iJieGZZ55hw4YNKIpSYVltU6XPWwghHKp2DaNVq1a0bNkSVa3aor7//nvGjBkDQHR0NEajkUOHDlVaVpckXwghxEW13oexZMkShg0bxqOPPsrJkycBSE9PR9M0AgMDbdOFh4eTkJBQYVldUEs3SUnGEEIIm0qbpEaNGkVcXJzDsp07d6LT6RyWATz55JMEBwejqiqrVq1iwoQJbN68+fKjraKgoKo/PNDb2932OjjYB4O+/O2rD8HBrvkEXVeNC1w3NomraiSuqqmNuCpNGCtXrrzshYeGhtpejxw5kjfeeIOEhAQiIyMBSEtLs9Uk4uPjCQsLIyAgoNyyqkpNzcFq1ao0T15eoe11SkoOep3rXEgWHOxDcnJ2fYdxCVeNC1w3NomraiSuqrncuFRVqfBEu1Z/DRMTE22vd+zYgaqqtiQyaNAgli9fDkBMTAwFBQW0b9++0rLapkiTlBBCOFTtq6TWrl3LnDlzyMrK4ocffmDRokV88skntGzZkueee47U1FQURcHb25sPPvgAvb54lU8//TTPPPMMq1atws3NjTlz5tg6zisqq21l7sOQbm8hhLCpdsIYOnQoQ4cOdVi2dOnScucLDg4ut7yistqmls0YQgghLnCdBnoXIflCCCEck4Rhp2wfhqQMIYQoIQnDjuQIIYRwTBKGHVUyhhBCOCQJw46kCyGEcEwShj3JGEII4ZAkDDvSJCWEEI5JwrAj+UIIIRyThGFHLqUVQgjHJGHYkXwhhBCOScKwI30YQgjhmCQMO9IkJYQQjknCsCPpQgghHJOEYUdqGEII4ZgkDDuq5AshhHBIEoYdqWEIIYRjkjDsSL4QQgjHJGHYkRqGEEI4JgnDjuQLIYRwTBKGHUkYQgjhmCQMO9IkJYQQjknCsCM7RAghHJPfRztSwxBCCMckYdiRfCGEEI5JwrAjNQwhhHBMEoYdeby5EEI4JgnDjuQLIYRwTBKGHWmSEkIIxyRh2JF8IYQQjlU7Ybz66qsMGjSI4cOHM2bMGA4dOmQrS0lJ4cEHH2TgwIEMHz6cAwcOVLustkkfhhBCOFbthNG7d2/WrFnDt99+y8MPP8yTTz5pK3v77beJjo5mw4YNTJ8+nWeeeQZN06pVVtskXwghhGPVThh9+vTBYDAA0LFjRxISErBarQB8//33jBkzBoDo6GiMRqOtBnK5ZUIIIepHjfZhLFu2jJtvvhlVVUlPT0fTNAIDA23l4eHhJCQkXHZZXZAmKSGEcExf2QSjRo0iLi7OYdnOnTvR6XQArFu3jjVr1rBs2bKajbAagoK8qzxPnuVi01dwsE9NhlMjXDEmcN24wHVjk7iqRuKqmtqIq9KEsXLlykoXsmnTJubOncvSpUtp0KABAAEBAQCkpaXZagvx8fGEhYVddllVpabmYLVWre8jIz3P9jo5ObvK66xNwcE+LhcTuG5c4LqxSVxVI3FVzeXGpapKhSfa1W6S2rp1K2+88QaLFy+mYcOGZcoGDRrE8uXLAYiJiaGgoID27dtXq6y2SYuUEEI4VmkNozLPP/88BoOByZMn295bunQpAQEBPP300zzzzDOsWrUKNzc35syZg6oW56jLLatt0ochhBCOVTth/PLLL+WWBQcHs3Tp0hotq22SL4QQwjG509uOPBpECCEck4RhR/KFEEI4JgnDjvRhCCGEY5Iw7EiTlBBCOCYJw47kCyGEcEwShh3JF0II4ZgkDDvSJCWEEI5JwrAj+UIIIRyThGFHahhCCOGYJAw7quQLIYRwSBKGHalhCCGEY5Iw7Ei+EEIIx6r98MErjdQwhKgZFouZ9PRkzGZTtZeVlKTahn52JX/XuPR6IwEBweh0VUsBkjDsSB+GEDUjPT0Zd3dPvLzCqn0iptermM2u98P8d4xL0zRyc7NIT0+mQYPwKi1XmqTsSA1DiJphNpvw8vKV75SLURQFLy/fy6r5ScKwI8e2EDVHkoVrutzPRRKGHTnAhRC17fXXX+Gbb76s7zCqTBKGHUkXQlz5zGbzFbmu2iad3nakhiHElalXr2geeOAhdu36mW7dbmDs2HHMnz+XkyePYzKZiIqKZtKkJzl/PpYXXniWzz//CrPZzJAh/bjvvvGMHXsvP/ywiR07tvHKK6/zxRefs2XLRsxmM0ajG1OnTuOaa1o7XNfIkbfzz3/OIDU1hbCwcFT14rn66tUr+Oqr/2IwGNE0K6+9NosmTZrWyz6qjCQMIUSt+/lQPD8djL/s+RUFNM1xWa/rw+l5nXNX+7i5ufHxx58CMGvWTDp27MS0aS9jtVp59dWXWLfuW4YPH0VeXi4pKSkkJMTRrFkLYmL2MHbsvfz2269ER3cBYNCgIYwbdy9ms5U9e3bz5ptvsGjRUofrevHFZ+jQIYoHH5zI+fN/cf/9Y+nW7QYA3n//PZYt+4YGDRpgMplc8jLdEpIwhBBXjcGDh9pe//TTdv744zDLly8DoKCggJCQUAA6dYrmt99+JT4+jhEjbmPZsk8pKioiJuZX7rnnfgCOHfuDl19eQmZmJqqqEht7rtx17d37G1OmPANAZGRDW9IpXlcXXn99Bj173sgNN/QiMrJhrWx7TZCEIYSodT2vc74W4EhN3e/g4eFZ6i+Nf/3rLYc/0J07d+G33/YQF3ee6dNnsn//XjZv3oCmQUREJEVFRbz88nN88MHHtGzZmpSUZEaOHFzBusr3r3+9yR9/HOa332KYPPkRpk59nhtu6Fmdzaw10ukthLgq9ezZm88//w8WiwWAjIwM4uLOA8UJY/fuXWRnZxMSEkp0dFcWL/7QVjMwmQqxWCyEhoYBsGLF1xWuq3PnaNat+xaAuLjzxMTsAYo7xOPiztOuXXvGjbufrl27c/z4sVrZ3pogNQwhxFXpiSee5v3353H//f9AURQMBiOTJz9NREQkISGheHp6cv31HYHiBJKYmECnTtEAeHl5M378wzzwwD34+vrRp0+/StY1lX/+cwabN28gPDyCqKjOAFitVl5//RVycrJRFJXQ0FAeeeTxWt3u6lA0rbyupL+/1NQcrNaqb96Ds7YA8Mm0vjUdUrUEB/uQnJxd32FcwlXjAteN7WqIKyHhLGFhTWpkWX/HR3DUJ2ficvT5qKpCUJB3ufNIk5QQQginSMIQQgjhFEkYQgghnCIJQwghhFNq5CqpV199lV27dmE0GvH09OTFF1/kuuuuA2DcuHHExcXh7V3ckXLvvfdy++23A3D69GmmTZtGRkYG/v7+zJ49m6ZNm1ZaJoQQou7VSA2jd+/erFmzhm+//ZaHH36YJ598skz5Sy+9xOrVq1m9erUtWQDMmDGDsWPHsmHDBsaOHcv06dOdKhNCCFH3aiRh9OnTB4PBAEDHjh1JSEio9HkoqampHDlyhKFDi2+fHzp0KEeOHCEtLa3CMiGEEPWjxvswli1bxs0331zmaYxz5sxh2LBhTJ06lcTERADi4+MJDQ1Fp9MBoNPpCAkJIT4+vsIyIYQQl6qLMTac6sMYNWoUcXFxDst27txp+2Fft24da9asYdmyZbbyOXPmEB4ejsVi4cMPP2TKlCl88cUXNRB65Sq6AcUZwcE+NRRJzXHFmMB14wLXje1KjyspSUWvr7lz0ppcVk0qictsNqPX183DMxytS1EUVFWxxVPZ/lJVtcqftVNbt3Llykqn2bRpE3PnzmXp0qU0aNDA9n54ePEDx3Q6Hffeey///ve/sVqthIeHk5iYiMViQafTYbFYSEpKIjw8HE3Tyi2risu907uEq92JezXcHVzTXDW2qyEuq9VaY3dB18Qd1b16RfPQQ//Hjh0/kpmZyXPPvUhMzK/s3r0Ts9nMzJmzadq0GampKbzyyovk5uZiMpno0aMnjz76RLnLrOkxNn74YSMWy+WNsWG1apjNVtauXckXXyyrcIwNq9V6yWdd2Z3eNZIOt27dyhtvvMGSJUto2PDikx/NZjMZGRm2BLJu3TpatWqFqqoEBQXRtm1b1q5dy4gRI1i7di1t27YlMDAQoMIyIcTfS9GfP1N0bPtlz68oCuU9xcjQujeGVs493dXb24ePP/6ULVs28/zzT/PKK//ikUceZ9my//Dpp58wffpMvL19mD17Lp6enpjNZp566nF++WUn3bv3cLjMmh5j4x//uAegWmNszJ//HsuW/a/Gx9iokYTx/PPPYzAYmDx5su29pUuX4ubmxsSJEykqKgIgJCSEd955xzbNK6+8wrRp03j//ffx9fVl9uzZTpUJIcTl6NfvFgBat24DKPTseeOFv9vy449bgeIz7/fff49Dhw4CGqmpqRw//me5CaOmx9j47LMlZGVVb4yN6OjaGWOjRhLGL7/8Um7ZihUryi1r0aIFX3/t+LHAFZUJIf5eDK16Ol0LcKSmHvJnNBqB4vZ7o9Fge19VVdtjzr/8chnZ2VksWlR80jt79uuYTIXlLrOmx9j4978/onXrNtUaY2PWrLc4dOhQjY+x4Zq9SEIIUU+ys7MJCmqAm5sbyclJ/PTTj07PWxNjbJTUSKozxsb583/VyhgbMh6GEEKUcuedY3j55ecYN+4ugoND6dy5S+UzXVATY2w89NC91R5jY+bMGWRn1/wYGzIehgMyHkbVuGpc4LqxXQ1xyXgY9UfGw6hjIf4e9R2CEEK4FGmScuCbWUNJTc2p7zCEEMKlSA3DAaNBh14nu0YIIUqTX0UhRK25grtI/9Yu93ORhCGEqBV6vZHc3CxJGi5G0zRyc7PQ641Vnlf6MIQQtSIgIJj09GRycjKqvazi5yS53tVIf9e49HojAQHBVV6uJAwhRK3Q6fQ0aFC1B4aW52q4DLkm1VZc0iQlhBDCKZIwhBBCOOWKbpJSVaVe5q1NElfVuWpsElfVSFxVczlxVTbPFf1oECGEEDVHmqSEEEI4RRKGEEIIp0jCEEII4RRJGEIIIZwiCUMIIYRTJGEIIYRwiiQMIYQQTpGEIYQQwimSMIQQQjhFEkYpp0+fZvTo0QwcOJDRo0dz5syZeoulb9++DBo0iBEjRjBixAh27NgBwP79+xk+fDgDBw7kwQcfJDU1tVbjmD17Nn379qV169b8+eeftvcr2ld1sR/Li6u8/QZ1s+/S09N56KGHGDhwIMOGDePxxx8nLS2t0vXXdmwVxdW6dWuGDRtm22fHjh2zzbdlyxYGDRrEgAEDmDJlCvn5+TUaF8Cjjz7K8OHDGTlyJGPHjuWPP/4A6v8Yqyi2+j7OAP7973+XOf7r5PjShM24ceO0VatWaZqmaatWrdLGjRtXb7H06dNHO3bsWJn3LBaL1r9/f23Pnj2apmnaggULtGnTptVqHHv27NHi4uIuiaeifVUX+7G8uBztN02ru32Xnp6u/fLLL7a/Z82apT3//PMVrr8uYisvLk3TtFatWmk5OTmXzJOTk6P16NFDO336tKZpmvbCCy9o8+fPr9G4NE3TsrKybK83bdqkjRw5UtO0+j/GKoqtvo+z33//XRs/frwtjro6viRhXJCSkqJ17txZM5vNmqZpmtls1jp37qylpqbWSzyODsgDBw5oQ4YMsf2dmpqqdezYsc7jqWhf1fV+dDZh1Ne++/7777X77ruvwvXXR2wlcWla+Qnju+++0yZOnGj7++DBg9qtt95aq3GtXLlSGzVqlEsdY/axaVr9HmeFhYXaXXfdpcXGxtriqKvj64p+Wm1VxMfHExoaik6nA0Cn0xESEkJ8fDyBgYH1EtPUqVPRNI3OnTvz1FNPER8fT0REhK08MDAQq9VKRkYG/v7+dRZXRftK07R634/2+83X17de9p3VauWLL76gb9++Fa6/rmMrHVeJcePGYbFY6N27N5MmTcJoNF4SV0REBPHx8TUeD8CLL77Izz//jKZpfPzxxy51jNnHVqK+jrP33nuP4cOH07BhQ9t7dXV8SR+Gi1q2bBnffvst33zzDZqm8dprr9V3SH8LrrTfZs6ciaenJ/fcc0+9xeCIfVzbtm1jxYoVLFu2jBMnTrBgwYI6j+n1119n27ZtPPnkk8yZM6fO118RR7HV13G2b98+fv/9d8aOHVsn67MnCeOC8PBwEhMTsVgsAFgsFpKSkggPr5khJi8nHgCj0cjYsWPZu3cv4eHhxMXF2aZJS0tDVdU6rV2UxFbevqrv/ehov5W8X5f7bvbs2Zw9e5Z3330XVVUrXH9dxmYfF1zcZ97e3tx5553l7rO4uLha/xxHjhzJ7t27CQsLc7ljrCS29PT0ejvO9uzZw8mTJ+nXrx99+/YlISGB8ePHc/bs2To5viRhXBAUFETbtm1Zu3YtAGvXrqVt27b10hyVl5dHdnbxeLyapvHdd9/Rtm1b2rdvT0FBATExMQAsX76cQYMG1Xl8Fe2r+tyP5e03oE733TvvvMPvv//OggULMBqNla6/rmJzFFdmZiYFBQUAmM1mNmzYYNtnN954I4cOHbJdgbR8+XIGDx5cozHl5uaWaebasmULfn5+LnGMlRebm5tbvR1nEydO5KeffmLLli1s2bKFsLAwFi9ezIQJE+rk+JIBlEo5efIk06ZNIysrC19fX2bPnk3z5s3rPI7Y2FgmTZqExWLBarXSokULXnrpJUJCQti7dy8zZsygsLCQyMhI3nzzTRo0aFBrsfzzn/9k48aNpKSkEBAQgL+/P+vWratwX9XFfnQU18KFC8vdb0Cd7Lvjx48zdOhQmjZtiru7OwANGzZkwYIFFa6/tmMrL64JEyYwffp0FEXBbDYTFRXFCy+8gJeXFwCbN2/mzTffxGq10rZtW2bNmoWnp2eNxZWSksKjjz5Kfn4+qqri5+fHc889x7XXXlvvx1h5sfn6+tb7cVaib9++LFy4kFatWtXJ8SUJQwghhFOkSUoIIYRTJGEIIYRwiiQMIYQQTpGEIYQQwimSMIQQQjhFEoYQtWjhwoW8+OKLlzXvtGnTmDt3bg1HJMTlk2dJCVGLHnnkkfoOQYgaIzUMIYQQTpGEIUQpiYmJTJo0ie7du9O3b18+/fRTAObPn8/kyZOZMmUKUVFRjBo1iqNHj9rmW7RoETfeeCNRUVEMHDiQXbt22eabOnWqbboffviBIUOGEB0dzbhx4zh58qSt7MiRI4waNYqoqCimTJlCYWFhmdi2bt3KiBEjiI6OZsyYMU6tX4gadVkPRRfiCmSxWLRRo0Zp8+fP1woLC7Vz585pffv21bZv367NmzdPa9eunbZ+/XrNZDJpH3/8sdanTx/NZDJpJ0+e1Hr37q0lJCRomqZpsbGx2tmzZzVN07R58+ZpTz/9tKZpmnbq1CmtQ4cO2k8//aSZTCZt0aJFWv/+/bXCwkKtsLBQu/nmm7UlS5ZoJpNJW79+vdauXTvtnXfe0TRN0w4fPqx1795d279/v2Y2m7UVK1Zoffr00QoLCytcvxA1SWoYQlxw6NAh0tLSePzxxzEajTRq1Ii77rqL7777DoBrr72WQYMGYTAYeOCBBzCZTBw4cACdTofJZOLkyZMUFRXRsGFDGjdufMnyv/vuO2666SZ69uyJwWBg/PjxFBQUsG/fPg4cOEBRURH33XcfBoOBQYMGcd1119nm/fLLLxk9ejQdOnRAp9MxatQoDAYD+/fvd3r9QlSXdHoLccH58+dJSkoiOjra9p7FYiE6OpqIiAjCwsJs76uqSmhoqG36F154gfnz53PixAl69erFtGnTCA0NLbP8pKSkMgPZlDz2PDExEZ1OR2hoKIqi2MpLTxsXF8eqVav4/PPPbe8VFRWRlJRE165dnVq/ENUlNQwhLggPD6dhw4bExMTY/u3bt4+PPvoIgISEBNu0VquVxMRE2xNKhw0bxhdffMHWrVtRFIW33nrrkuWHhISUGZdA0zTbyHLBwcEkJiailXoWaOlpw8PDeeSRR8rEduDAAYYOHer0+oWoLkkYQlxw/fXX4+XlxaJFiygoKMBisfDnn39y8OBBAA4fPszGjRsxm8385z//wWg00qFDB06dOsWuXbswmUwYjUbc3NxsgxOVNnjwYH788Ud27dpFUVERn3zyCUajkaioKDp27Iher+fTTz+lqKiIjRs3cujQIdu8d955J8uXL+fAgQNomkZeXh7btm0jJyfH6fULUV3SJCXEBTqdjoULFzJ79mz69euHyWSiWbNmTJkyBYB+/frx3Xff8dxzz9GkSRPmz5+PwWDAZDLx9ttvc/LkSQwGA1FRUQ6H7GzevDlvvvkmM2fOJDExkbZt27Jw4ULbYEbz58/n5Zdf5t133+Wmm25iwIABtnmvu+46Zs6cyWuvvcbZs2dxd3enU6dOREdHO71+IapLxsMQwgnz58/n7Nmz0tQjrmpSbxVCCOEUSRhCCCGcIk1SQgghnCI1DCGEEE6RhCGEEMIpkjCEEEI4RRKGEEIIp0jCEEII4RRJGEIIIZzy/62D+zOTOmZTAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "加载模型成功！\n", + "开始测试!\n", + "环境：CliffWalking-v0, 算法：Q-learning, 设备：cuda\n", + "回合：1/20，奖励：-13.0\n", + "回合：2/20，奖励：-13.0\n", + "回合：3/20，奖励：-13.0\n", + "回合：4/20，奖励：-13.0\n", + "回合：5/20，奖励：-13.0\n", + "回合：6/20，奖励：-13.0\n", + "回合：7/20，奖励：-13.0\n", + "回合：8/20，奖励：-13.0\n", + "回合：9/20，奖励：-13.0\n", + "回合：10/20，奖励：-13.0\n", + "回合：11/20，奖励：-13.0\n", + "回合：12/20，奖励：-13.0\n", + "回合：13/20，奖励：-13.0\n", + "回合：14/20，奖励：-13.0\n", + "回合：15/20，奖励：-13.0\n", + "回合：16/20，奖励：-13.0\n", + "回合：17/20，奖励：-13.0\n", + "回合：18/20，奖励：-13.0\n", + "回合：19/20，奖励：-13.0\n", + "回合：20/20，奖励：-13.0\n", + "完成测试！\n", + "结果保存完毕！\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEcCAYAAADdtCNzAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAzDUlEQVR4nO3de1xM+f8H8NfM1GDVJlQKu4sWtS4NJURuodSo+LLs5s6ua8vqS2QvYpG9hNbl27ov67LfRQpr01qxWffwtfGTXbutUZHQhSYz5/eHr/kana6nKZvX8/HweMzM+ZzzeZ8zn+k155xxjkwQBAFERETPkFd3AURE9HxiQBARkSgGBBERiWJAEBGRKAYEERGJYkAQEZGoGhUQvXv3RlJSUpX3e/r0afTv37/K+6X/+euvv9CqVSs8evSo0pcdHx+PHj16QKVS4ddff6205dbE8Xr79m28/fbbUKlUWLJkiUn6eFarVq3wxx9/AAA+/PBDrFy50jDtm2++QdeuXaFSqZCdnY0zZ86gX79+UKlUOHToUKXV8Oz4GzFiBL799lvRtuPHj8fu3bsrrW9TMqvuAmoCV1dXHDx4sLrLIBOJiIjABx98AC8vL9HpgiBg3bp12LlzJ9LT01G/fn0MHDgQU6dOhVKprOJqS2fK8bpjxw5YW1vj7NmzkMlklbLMzMxMLFu2DImJicjLy4OdnR0GDBiA8ePH46WXXjJqGx4ebnhcWFiIJUuWYOfOnWjdujUAYMWKFXj77bcxatQoxMXFwcfHBwcOHDDMM2bMGKSnpxd5rUuXLnjnnXcqZX3Wrl1bKcspr7/++gtz5szBhQsXYG9vjw8//BBdu3YtcZ4atQdhKjqdrrpLkKwmrEN10Wg0eP3114udvnDhQuzcuRMRERE4e/YsvvrqKyQlJeH999+vwir/pzrfa41GgxYtWlQoHMT2/u7evYthw4ahoKAA27dvx7lz57Bhwwbcv38ff/75Z4nLy8rKQkFBARwdHY3qe/Jeurm54bfffsOdO3cM/V++fBkFBQVGryUnJ8PV1bXc6/O8mTlzJpydnXHixAnMmDEDwcHBhvUsTo0NCL1ej+joaHh5ecHd3R3vvfce7t69a5geHBwMDw8PdOzYEW+//TauXr1qmBYaGoqPPvoIEyZMgIuLC06cOIHevXtj3bp1UKvV6NixI6ZPn46CggIAwIkTJ+Dp6WmYv6S2APDVV1+hW7du6NatG7799lujXeRn3b17F3PmzEG3bt3g5uaGyZMnAwB27dqF4cOHG7V9ejnPrsO6devg4eFh9McjPj4earW6TNvrWTt37kTfvn3RqVMnTJw4ERkZGUZ1bNu2Df369YOrqyvmz5+P4v7Dvk6nw5o1a+Dl5QWVSoVBgwbh5s2booeMnt5t1+l0iIiIgLu7O/r06YMjR44YLfe7776Dj48PVCoV+vTpg+3btxe7Lnq9HqtWrUKvXr3QpUsXzJo1Czk5OdBqtVCpVNDpdPD39xfdg7h+/Tq++eYbfPbZZ1CpVDAzM8Prr7+OqKgo/PTTTzh58mSx/T5bw999vIaGhmLPnj1Yt24dVCoVkpKSoNVq8cknnxjm/+STT6DVao3qiI6OhoeHB+bMmVNkmRs2bEDdunXx6aefokmTJgAAe3t7zJs3z7BX8GwNkZGR+P333+Ht7Q3gcRCMHDkSXl5eSEtLw8SJE6FSqWBtbY2mTZvi1KlTAIBff/0Vjo6OcHNzM3pNr9ejbdu2+OmnnxAQEIAOHTqgR48eiIqKKtN7m5mZCbVabdhzeHocP/kcR0REwM3NDb179zYay2lpaYZDdqNHj8b8+fMREhIi2s/48eOxZcsWo9cGDhyIH374Ab///jsuXbqEadOmoXbt2ujfvz9atmxZ6p5kjQ2Ir7/+GocOHcKWLVtw9OhRWFlZGe1+enp64uDBgzh+/DicnZ2LbPS4uDhMnDgRZ8+eRceOHQEABw4cwNq1a5GQkIArV65g165dxfZfXNvExERs3LgRGzZsQHx8PE6cOFHiesyaNQsPHjzAvn37kJSUhNGjR5d5Gzy9DqNGjUKdOnXwyy+/GKbHxsYaAqK07fW048eP4/PPP8eyZctw7NgxNG7cuMi35Z9++gn//ve/sXfvXhw4cABHjx4VXdaGDRuwb98+REdH4+zZs1i0aBFq165d6rrt3LkThw8fxp49e/Ddd9/h+++/N5reoEED/Otf/8LZs2exePFiLF68GJcuXRJd1q5du7B7925s3rwZhw4dQn5+PsLDw6FUKnHu3DkAQExMjOgx6+PHj6NRo0Zo166d0ev29vZwcXHBzz//XOq6ADVjvC5ZsgRqtRrjxo3DuXPn0LVrV6xevRrnz59HTEwM9u7di4sXL2LVqlWGeW7fvo179+7h8OHDWLBgQZFlHj9+HH379oVcXr4/Vc2aNUNcXBwA4NSpU4b31sHBAWvWrMG5c+egVCqNwuDUqVNwdXVFx44djV5r3749zM3NUadOHUREROD06dP417/+hW3btpV6HiMtLQ0jRoxAUFAQxo8fL9rmwoULaNasGX755ReMHz8eYWFhhi9UISEhaNeuHU6cOIGpU6ciJiam2L78/PwM6wwAqamp0Gg06NmzJ1JTU9G0aVNYWFgYprdu3Rqpqakl1l9jA2L79u2YMWMGGjVqBKVSialTp+LgwYOGb6T/+Mc/YGFhAaVSiWnTpuHy5cvIyckxzN+nTx907NgRcrkctWrVAvA4+e3s7FCvXj306tULKSkpxfZfXNsDBw5g0KBBeP3111GnTh1Mmzat2GVkZmYiMTER8+fPh5WVFczNzdGpU6cyb4Nn18HX19cwgHJzc5GYmAhfX98yba+nxcbGYvDgwXjjjTegVCrx/vvvIzk5GX/99ZehzYQJE/Dyyy/DwcEB7u7uuHz5smiN3377Ld577z00b94cMpkMrVu3hrW1danrduDAAYwaNQr29vaoV68e3n33XaPpPXv2xCuvvAKZTIZOnTrBw8MDp0+fFl1WbGwsRo8ejaZNm6Ju3bp4//33sX///jKd8M7OzoaNjY3oNBsbm1J34Z+oCeNVTGxsLKZMmYIGDRqgfv36mDJlCvbu3WuYLpfLERwcDKVSKfrF4O7du8Vu38rg5uZmGBenT582BMTTrz35zLm7u6NVq1aQy+Vo3bo1fH19S9xDTE1NxahRozBt2jS8+eabxbZzcHDA0KFDoVAoEBgYiFu3buH27dvQaDS4ePGiYfu4urqid+/exS7Hy8sLly9fxo0bNwA83vZ9+/aFUqlEXl4eLC0tjdpbWloiLy+vxO1TYwNCo9FgypQpcHV1haurKwYMGAC5XI6srCzodDp89tln8PLyQocOHQwbPTs72zC/vb19kWU+PVDr1KmD/Pz8Yvsvrm1mZiYaNWpUYj9PpKenw8rKClZWVmVY46KeXbZarUZ8fDy0Wi3i4+Ph7OyMxo0bAyh5ez0rMzPTMB8A1K1bF/Xq1TM6zPTs+hc3ENPT0/HKK6+Ue90yMzON1s/BwcFo+pEjRzB06FB06tQJrq6uSExMNHp/S1qfxo0b49GjR6Lr/ixra2vcunVLdNqtW7cMYTd+/HioVCqoVCqjP5BP1ITxKiYzM9PovXFwcEBmZqbhubW1tSHQxNSrV6/Y7VsZ3NzccOXKFdy7dw/nz5+Hi4sLWrRogVu3buHevXs4e/as4fzD+fPnMWLECHTu3BkdO3bE9u3bix1TwOM/0La2tqX+Yqxhw4aGx3Xq1AEA5OfnIzMzE1ZWVobXAOPt/+GHHxrG1Jo1a2BhYYEePXpg3759AB7vVQ4cOBDA489obm6uUb+5ubmoW7duibXV2IBo1KgRvvrqK5w+fdrw7+LFi7Czs0NsbCwSEhKwYcMGnDlzBj/++CMAFHucvDLZ2toa/SG9efNmietw79493L9/v8i0OnXq4OHDh4bnZfkQOTo6wsHBAYmJiYiLi4Ofn59RX8VtL7F1ePItBXg8mO/evSvatjSNGjUSPdn45Ncpxa2jjY2N0bZ7+rFWq0VwcDDGjh2Ln3/+GadPn4anp2ex7++z66PRaGBmZoYGDRqUWn/nzp1x8+ZNXLhwwej1mzdvIjk52fDtc+3atTh37hzOnTtn+NA+rSaM1+Lm12g0RvPb2toanpd2MrtLly6Ij4+HXq8vV79l1bRpU9ja2mLHjh2wt7c3/MF0cXHBjh07kJeXBxcXFwCPT/I+Od915swZDBs2rMT3YOrUqbC2tsbMmTMr9MMBGxsb3Lt3Dw8ePDC89vT2Dw8PN4ypiRMnAnh8mGnfvn04d+4cCgoK4O7uDuDxZz8tLc0oJC5fvmx0Al9MjQ2I4cOHY9myZYYP/p07dwzHC/Py8qBUKmFtbY0HDx7giy++qLK6vL29sWvXLly7dg0PHjwwOh77LFtbW3h6emL+/Pm4d+8eCgsLDcdGW7dujatXryIlJQUFBQVlPmHm5+eHTZs24dSpU4aTeEDJ20tsGbt27UJKSgq0Wi2++OILtGvXznASsTyGDBmC5cuX4/r16xAEAZcvX0Z2djbq168POzs7xMTEQKfT4d///jfS0tIM8/n4+ODrr79Geno67t27h+joaMM0rVYLrVaL+vXrw8zMDEeOHCnxXMCTbZKWloa8vDxERkbCx8cHZmal/wq8WbNmGDZsGEJCQpCcnAydToerV69i2rRpUKlUpf6M8ImaMF7F+Pr6YvXq1bhz5w7u3LmDlStXGs57lcWYMWOQl5eH2bNnG7ZNRkYGFi9eXOxhy/JydXXFxo0bjX6p1LFjR2zcuBFt2rQxHPrKy8uDlZUVatWqhQsXLhgd7xdjbm6O5cuX48GDB5g1a1a5Q65x48Zo06YNoqKioNVqce7cORw+fLjEeXr06AGNRoMVK1YY9kKBx+PUyckJK1euREFBAeLj43HlypVS925qbECMHDkSvXv3xtixY6FSqTB06FDDt7yAgAA4ODige/fu8PX1NXxDqAo9evTAiBEjMHLkSPTt2xft27cHgGJ/L7906VKYmZnBx8cHXbt2xaZNmwA8fsOnTJmC0aNHo1+/foYTk6Xx8/PDqVOn0LlzZ9SvX9/weknb61ldu3bFe++9h2nTpqFbt25IS0tDZGRkeTaDwZgxY+Dj44OxY8eiQ4cOCAsLM/yCZsGCBVi3bh3c3d2RmpoKlUplmG/o0KHo1q0b/P39ERgYiH79+hmmWVhYYN68eZg+fTrc3NwQFxdX4rHbwYMHY+DAgQgKCkKfPn2gVCrxwQcflHkdPvzwQ/zjH//AP//5T7Rv3x5+fn5wcHDAqlWrynxytaaM12dNnjwZbdq0wcCBAzFw4EC88cYbhl/ilUW9evWwbds2mJmZYejQoVCpVBg1ahQsLS3x6quvVmidnuXm5oasrCyjz5CrqyuysrLg5uZmeO2jjz7CihUroFKpsHLlSvj4+JS6bKVSiS+//BJZWVmYO3duuUPis88+Q3JyMtzd3bFs2TIMGDCgxG2vVCrRt29fJCUlGR0hAIAvvvgC//nPf+Dm5obPPvsMK1asMPobIEbGGwZVr2vXrsHPzw8XL14s0zdWev6tWLEC8fHx2Lp1K15++eXqLqdScbxWr+nTp6N58+YIDg6ukv5q7B7E8+zJieJ79+7h008/Ra9evfhhq0GCg4Px5ptvIjk5ubpLqRQcr9XnwoUL+PPPP6HX65GYmIiEhIRi/0e/KXAPohqMGzcOycnJUCgUcHNzw0cffWR04o7oecLxWn1+/PFHzJ8/H3fv3kWjRo3wzjvvYPDgwVXWPwOCiIhE8RATERGJYkAQEZEoBgQREYmqUT9FyM7Og15fsVMqDRpYICsrt/SG1YT1ScP6pGF90jyv9cnlMlhbF3+5jRoVEHq9UOGAeDL/84z1ScP6pGF90jzv9YnhISYiIhLFgCAiIlE16hATEVUfQRCQnX0LWu1DAFV7OCUzU26yK75WhuquT6Ewg4VFPdSpU/LlvZ/FgCCiSpGbew8ymQx2dk0gk1XtwQkzMzkePXp+A6I66xMEAYWFWty9+/hy+eUJCR5iIqJK8eBBLiwt61V5OFDJZDIZlMpaqFfPBrm5d8s1L99JIqoUer0OCgUPSjyvzM2V0OlKv43u0xgQRFRpSrtDHFWfirw3DAgiIhMLD/8I3323o7rLKDcGBBHVeI8ele/Qyt+lL1PjAUMiqpG6dXPFmDETcPz4z3B374K33hqBqKhIXLt2FVqtFiqVK6ZNm4EbN9Iwd+4sbNmyE48ePYKvbx+MGjUOb701EgkJ8Th69Cd8/PEn2LZtCxISfoBO9whKZS2EhITi9ddbifYVEDAYCxd+hKys22jUyB4Kxf++i8fE7MLOnd/A3FwJQdAjPHwJXn31tWrZRqVhQBBRpfv54k0cu3DTJMvu1s4eHm3ty9S2Vq1aWLt2MwBgyZIFcHHpgNDQD6DX6zF//jzs27cXAwcGIj8/D7dv30Z6ugbNmrXA6dOn8NZbI3HmzEm4uj6+L7W3ty+GDw8CAJw6dQKffroY0dEbRfsKC/sn2rdXYezYd3Djxl8YM+YtdOrUBQCwatVybN36HRo2bAitVvtc//+NSgmImJgYrF27FteuXcPcuXMRFBRkmDZ//nwcP34cSqUSL730EsLCwtC2bdtil3Xnzh34+fnB1dUVK1asqIzyiOgF5ePjZ3h87FgiUlIuYfv2rQCAhw8fwtbWDgDQoYMrzpw5iZs3NfD3H4StWzejsLAQp0+fRFDQaADAlSsp+PrrDbh//x7kcjnS0v4stq+zZ89g+vR/AgAaN24CV9dOhmkdOrjhk08+godHd3Tp0g2NGzcxybpXhkoJCCcnJ0RGRiI6OrrINE9PT8ydOxfm5uY4fPgwZsyYgUOHDhW7rI8//hg9evRAXl5eZZRGRNXAo23Zv+WbUp06Lz31TMCiRZ+J/kHu2NENZ86cgkZzAx9+uADJyWdx6NBBCALg4NAYhYWF+OCD2fjyy6/QqlVr3L59CwEBPiX0VbxFiz5FSsolnDlzGsHBExESMgddunhIWU2TqZST1C1btoSjoyPk8qKL69WrF8zNzQEALi4uSE9PL3aXau/evWjYsCHc3NwqoywiIgMPD09s2bIJOp0OAHD37l1oNDcAPA6IEyeOIycnB7a2dnB17YR16/5lOLyk1RZAp9MZ9jh27fq2xL46dnTFvn17AQAazQ2cPn0SwOMT2BrNDTg7t8GIEaPRqVNnXL16xSTrWxmq9BzE1q1b0bNnT9EgycjIwMaNG/H111/j4MGDFVp+gwYWkuqzsbGUNL+psT5pWJ80pdWXmSmHmVn1/TBSrG8zs//V9P77/8SXXy7HmDFvQSaTwdzcHNOnh+CVV5rCwcEedevWhYuLC8zM5OjUyR3h4R/Aza0TzMzksLJ6GRMmTMSECSNhZWWF3r29ivRp3NcshId/gKCgg3BwaAyVqiPkchnkcmDRoo+Rm5v738uS2GHq1OAq225yubxc40wmCEKpV9UKDAyERqMRnZaUlASFQgEACA0NRZs2bYzOQTyxb98+rFixAlu3bkXDhg2LTH/nnXcwZswYdOnSBbt27cJPP/1U7nMQWVm5Fb7muo2NJW7dyqnQvFWB9UnD+qQpS33p6X+gUaNXq6giY7wWU9k8+x7J5bISv1iXaQ9i9+7dkoqKj49HZGQkNm7cKBoOAJCcnIywsDAAQF5eHgoKCjBhwgR89dVXkvomIqKKMfkhpsOHD2Px4sXYsGEDmjQp/mz9yZMnDY8rugdBRESVp1IOfMXFxcHT0xPff/89li9fDk9PT6SmpgIA5syZg8LCQgQHB8Pf3x/+/v7Izs4GAISFhSEhIaEySiAiokpWpnMQfxc8B1F9WJ80NaE+noMo3vNSX3nPQfBaTEREJIoBQUREohgQREQkigFBRPQ39MknH5v8HhMMCCKiCngR7jHBy30TUaUr/L+fUXgl0STLNm/lCfOWpV/crls3V0yYMAlHjx7BvXv3MHt2GE6fPokTJ5Lw6NEjLFgQgddea4asrNv4+OMw5OXlQavVomtXD0ye/F6xy6zIPSb69/fCqFFjK/UeE09fsshU95hgQBBRjWVhYYm1azfjxx8PYc6cmfj440WYOHEqtm7dhM2b1+PDDxfAwsISERGReOmll/Do0SO8//5U/PJLEjp37iq6zIrcY6J58+aVfo+J0aPfgru7ae8xwYAgokpn3tKjTN/yTa1Pn34AgFatWgOQwcOj+3+fO+HIkcMAAL1ej1WrluPixQsABGRlZeHq1f8rNiAqco+JgIDB+PrrTZV8j4n/XfXaVPeYYEAQUY2lVCoBPL6KqVJpbnhdLpcbLvu9Y8dW5OTcR3T0RtSqVQsREZ9Aqy0odpkVucdEePgnOHv2zN/uHhM8SU1EL7ScnBw0aNAQtWrVwq1bmTh27EiZ563ee0ycAmDae0xwD4KIXmhDhgzDBx/MxogRQ2FjY4eOHct+w7L33puJVatWYPTo4f+9x4QSwcEz4eDQGLa2dnjppZfQrp0LgMeBkZGRjg4dXAEAdetaYNy4dzFhwki8/LIVevXqU0pfIVi48CMcOnQQ9vYOUKk6Anh8iOyTTz5Gbm4OZDI57OzsMHHi1IptjGfwWkz/VROuhVOdWJ80NaE+XoupeM9LfbwWExERVQoGBBERiWJAEFGlqUFHrGucirw3DAgiqhRyuQI6XfVcEoJKV1iohUJRvt8lMSCIqFLUqWOBnJy7EITqPxlL/yMIArTaAty9ewsWFvXKNS9/5kpElcLCwgrZ2beQkfEXgKo91CSXyyvt8hKmUN31KRRmsLS0Rp06dcs1HwOCiCqFTCZD/fq21dJ3TfiZ8POIh5iIiEgUA4KIiERJDoiYmBio1Wo4Oztjy5YtRtPmz58Pb29vDBw4EMOGDcPFixeLXc7x48cxaNAg+Pr6wtfXF5cvX5ZaGhERSSD5HISTkxMiIyMRHR1dZJqnpyfmzp0Lc3NzHD58GDNmzMChQ4eKtMvIyEBYWBjWrl2L5s2b4+HDh9V2ByUiInpMckC0bNkSAIzubvREr169DI9dXFyQnp4OvV5fpO0333wDf39/NG/eHABQu3ZtqWUREZFEVfYrpq1bt6Jnz56iQZKamorGjRtj5MiRuH//Ptzd3TFz5kzDtdyJiKjqlRoQgYGB0Gg0otOSkpKgUChK7WTfvn2IjY3F1q1bRafrdDqcPXsWGzZsQK1atRASEoLo6GhMnVq+S9aWdFXCsrCxsZQ0v6mxPmlYnzSsT5rnvT4xpQbE7t27JXUQHx+PyMhIbNy4EQ0bNhRt4+DggDZt2sDS8vEG9Pb2RkxMTLn74uW+qw/rk4b1ScP6KqZaL/d9+PBhLF68GOvWrUOTJsXfI9XPzw8nTpyAVquFIAg4duwYWrdubcrSiIioFJIDIi4uDp6envj++++xfPlyeHp6IjU1FQAwZ84cFBYWIjg4GP7+/vD390d2djYAICwsDAkJCQCADh06oHv37ggICMDAgQOh0+nw7rvvSi2NiIgk4B3l/ut53QV8gvVJw/qkYX3SPK/18Y5yRERUIQwIIiISxYAgIiJRDAgiIhLFgCAiIlEMCCIiEsWAICIiUQwIIiISxYAgIiJRDAgiIhLFgCAiIlEMCCIiEsWAICIiUQwIIiISxYAgIiJRDAgiIhLFgCAiIlEMCCIiEsWAICIiUQwIIiISxYAgIiJRDAgiIhIlOSBiYmKgVqvh7OyMLVu2GE2bP38+vL29MXDgQAwbNgwXL14UXcaDBw8wc+ZM+Pn5wdfXF9OnT0dubq7U0oiISALJAeHk5ITIyEj4+fkVmebp6YnY2Fjs3bsX7777LmbMmCG6jB07dqCwsBCxsbGIi4uDXq/Htm3bpJZGREQSmEldQMuWLQEAcnnRrOnVq5fhsYuLC9LT06HX64u0lclkePjwIQoLCwEA+fn5aNSokdTSiIhIAskBUVZbt25Fz549RYNk2LBhSE5OhoeHBwCgW7duUKvV5e6jQQMLSTXa2FhKmt/UWJ80rE8a1ifN816fmFIDIjAwEBqNRnRaUlISFApFqZ3s27cPsbGx2Lp1a7HLAYBjx44BAGbOnIl169Zh3LhxpS77aVlZudDrhXLN84SNjSVu3cqp0LxVgfVJw/qkYX3SPK/1yeWyEr9YlxoQu3fvllRAfHw8IiMjsXHjRjRs2FC0zfbt2+Hv749atWoBAAYMGIA9e/aUOyCIiKjymPRnrocPH8bixYuxbt06NGnSpNh2TZo0wbFjxyAIAvR6PY4ePYrXX3/dlKUREVEpJAdEXFwcPD098f3332P58uXw9PREamoqAGDOnDkoLCxEcHAw/P394e/vj+zsbABAWFgYEhISAABTpkzB/fv34efnB7VaDa1Wi0mTJkktjYiIJJAJglCxg/bPIZ6DqD6sTxrWJw3rq5jSzkHwf1ITEZEoBgQREYliQBARkSgGBBERiWJAEBGRKAYEERGJYkAQEZEoBgQREYliQBARkSgGBBERiWJAEBGRKAYEERGJYkAQEZEoBgQREYliQBARkSgGBBERiWJAEBGRKAYEERGJYkAQEZEoBgQREYliQBARkSjJARETEwO1Wg1nZ2ds2bLFaNrq1auhVqsREBAAf39/7N+/v9jl7Ny5E3379oWXlxfCw8Oh1+ullkZERBKYSV2Ak5MTIiMjER0dXWRaUFAQJk2aBADIyMiAj48PPDw8YGVlZdQuLS0NX375Jfbs2YN69ephwoQJ2Lt3LwICAqSWR0REFSR5D6Jly5ZwdHSEXF50UZaWlobH+fn5kMlkonsGBw8ehJeXF+rXrw+5XI4hQ4aUuLdBRESmJ3kPojTbtm3Dpk2bkJ6ejkWLFsHa2rpIm5s3b8LBwcHw3MHBATdv3jR1aQaXftwHs+vHIeiFKuuzvK7IZaxPAtYnDeuTxtT16Zp3xRu9fSt9uaUGRGBgIDQajei0pKQkKBSKEucfPnw4hg8fjitXriAkJARdunQRDYnK0KCBRYXmq1PbHIUAZHJZ5RZUyVifNKxPGtYnjSnrq1PbHDY2lqU3LKdSA2L37t2V0lGrVq1ga2uLkydPon///kbT7O3tjUJIo9HA3t6+3H1kZeVCX4GUbt61H2z8B+PWrZxyz1tVbGwsWZ8ErE8a1idNVdRXkeXL5bISv1ib9GeuqamphsdpaWlISUmBo6NjkXb9+/fHoUOHcOfOHej1enz77bfw8fExZWlERFQKyecg4uLisHTpUty/fx8JCQmIjo7G+vXr4ejoiKioKKSmpsLMzAwKhQLz5s1DixYtAADLly+Hra0thg8fjqZNm2Ly5MkYOnQoAMDDwwMDBw6UWhoREUkgEwTh+T2zU04VPcQEcBdVKtYnDeuThvVVTLUeYiIior8vBgQREYliQBARkSgGBBERiWJAEBGRKAYEERGJYkAQEZEoBgQREYliQBARkSgGBBERiWJAEBGRKAYEERGJYkAQEZEoBgQREYliQBARkSgGBBERiWJAEBGRKAYEERGJYkAQEZEoBgQREYliQBARkSjJARETEwO1Wg1nZ2ds2bLFaNrq1auhVqsREBAAf39/7N+/X3QZhw4dwqBBg+Dn5wdfX1+sX79eallERCSRmdQFODk5ITIyEtHR0UWmBQUFYdKkSQCAjIwM+Pj4wMPDA1ZWVkbtbGxssHr1atjZ2SEnJweDBg1Cu3bt4OrqKrU8IiKqIMkB0bJlSwCAXF50Z8TS0tLwOD8/HzKZDHq9vki79u3bG83TokUL3LhxgwFBRFSNTH4OYtu2bfD29kZgYCAWLFgAa2vrEttfu3YNycnJ6Ny5s6lLIyKiEsgEQRBKahAYGAiNRiM6LSkpCQqFAgAQGhqKNm3aICgoSLTtlStXEBISgs2bNxcbEpmZmRgxYgSmT58OHx+f8qwHERFVslIPMe3evbtSOmrVqhVsbW1x8uRJ9O/fv8j0rKwsjBkzBuPHj69wOGRl5UKvLzHvimVjY4lbt3IqNG9VYH3SsD5pWJ80z2t9crkMDRpYFD/dlJ2npqYaHqelpSElJQWOjo5F2mVnZ2PMmDF4++23MWTIEFOWREREZST5JHVcXByWLl2K+/fvIyEhAdHR0Vi/fj0cHR0RFRWF1NRUmJmZQaFQYN68eWjRogUAYPny5bC1tcXw4cMRHR2N69evY8eOHdixYwcAYOTIkRg8eLDU8oiIqIJKPQfxd8JDTNWH9UnD+qRhfRVTrYeYiIjo74sBQUREohgQREQkigFBRESiGBBERCSKAUFERKIYEEREJIoBQUREohgQREQkigFBRESiGBBERCSKAUFERKIYEEREJIoBQUREohgQREQkigFBRESiGBBERCSKAUFERKIYEEREJIoBQUREohgQREQkigFBRESiJAdETEwM1Go1nJ2dsWXLFqNpq1evhlqtRkBAAPz9/bF///4Sl1VQUABfX18MGjRIallERCSRmdQFODk5ITIyEtHR0UWmBQUFYdKkSQCAjIwM+Pj4wMPDA1ZWVqLLioyMRPv27XH58mWpZRERkUSS9yBatmwJR0dHyOVFF2VpaWl4nJ+fD5lMBr1eL7qc06dP4/r16/D395daEhERVQLJexCl2bZtGzZt2oT09HQsWrQI1tbWRdrk5+dj0aJFWL16Na5fv17hvho0sJBQKWBjY1l6o2rE+qRhfdKwPmme9/rElBoQgYGB0Gg0otOSkpKgUChKnH/48OEYPnw4rly5gpCQEHTp0qVISCxduhRvvfUW7OzsJAVEVlYu9HqhQvPa2Fji1q2cCvdtaqxPGtYnDeuT5nmtTy6XlfjFutSA2L17d6UU0qpVK9ja2uLkyZPo37+/0bQzZ84gMTERq1atQkFBAe7duwe1Wo3Y2NhK6ZuIiMrPpIeYUlNT4ejoCABIS0tDSkqK4fnTng6CEydOICIiArt27TJlaUREVArJAREXF4elS5fi/v37SEhIQHR0NNavXw9HR0dERUUhNTUVZmZmUCgUmDdvHlq0aAEAWL58OWxtbTF8+HDJK0FERJVPJghCxQ7aP4d4DqL6sD5pWJ80rK9iSjsHwf9JTUREohgQREQkigFBRESiGBBERCSKAUFERKIYEEREJIoBQUREohgQREQkigFBRESiGBBERCSKAUFERKIYEEREJIoBQUREohgQREQkigFBRESiGBBERCSKAUFERKIYEEREJIoBQUREohgQREQkigFBRESiJAdETEwM1Go1nJ2dsWXLFqNpq1evhlqtRkBAAPz9/bF///5il5OSkoK3334bAwYMwIABA3DkyBGppRERkQRmUhfg5OSEyMhIREdHF5kWFBSESZMmAQAyMjLg4+MDDw8PWFlZGbXLz8/H1KlT8fnnn8PFxQWPHj1CTk6O1NKIiEgCyQHRsmVLAIBcXnRnxNLS0vA4Pz8fMpkMer2+SLu4uDh07NgRLi4uj4syM4O1tbXU0oiISALJAVGabdu2YdOmTUhPT8eiRYtE//CnpqbCzMwMEyZMQGZmJt544w3Mnj27yJ4GERFVHZkgCEJJDQIDA6HRaESnJSUlQaFQAABCQ0PRpk0bBAUFiba9cuUKQkJCsHnz5iIhsXDhQvz444/Yvn07GjZsiMWLFyM3NxeLFy+uyDoREVElKHUPYvfu3ZXSUatWrWBra4uTJ0+if//+RtPs7e3h7u4OW1tbAIBarcbcuXPL3UdWVi70+hLzrlg2Npa4dev5Pe/B+qRhfdKwPmme1/rkchkaNLAofropO09NTTU8TktLQ0pKChwdHYu08/HxwYULF5CbmwsASExMRKtWrUxZGhERlULyOYi4uDgsXboU9+/fR0JCAqKjo7F+/Xo4OjoiKirKcH5BoVBg3rx5aNGiBQBg+fLlsLW1xfDhw+Hg4IAJEyZg2LBhkMlkaNKkCRYsWCB55YiIqOJKPQfxd8JDTNWH9UnD+qRhfRVTrYeYiIjo74sBQUREohgQREQkigFBRESiGBBERCSKAUFERKIYEEREJIoBQUREohgQREQkigFBRESiGBBERCTK5DcMqkpyuaxa5zc11icN65OG9UnzPNZXWk016mJ9RERUeXiIiYiIRDEgiIhIFAOCiIhEMSCIiEgUA4KIiEQxIIiISBQDgoiIRDEgiIhIFAOCiIhE1ahLbZTm999/R2hoKO7evYt69eohIiICr732mlEbnU6HhQsX4ujRo5DJZHjnnXcwZMgQk9eWnZ2NWbNm4c8//4RSqcSrr76K8PBw1K9f36hdaGgokpKSYG1tDQDw9vbGpEmTTF4fAPTu3RtKpRK1atUCAISEhKB79+5GbR48eIA5c+bg0qVLUCgUmD17Nnr16mXy2v766y9MmTLF8DwnJwe5ubk4efKkUbuoqCh88803sLW1BQB06NABH330kUlqioiIwMGDB3Hjxg3ExsaiZcuWAMo2DgHTj0Wx+so6DgHTj8Xitl9ZxiFg+rEoVl9ZxyFQtWOxwoQXyIgRI4Q9e/YIgiAIe/bsEUaMGFGkze7du4WxY8cKOp1OyMrKErp37y6kpaWZvLbs7Gzhl19+MTxfsmSJMGfOnCLtZs+eLXz99dcmr0dMr169hCtXrpTYJioqSggLCxMEQRB+//13oWvXrkJubm5VlGdk4cKFwvz584u8vmLFCmHJkiVVUsOpU6cEjUZTZLuVZRwKgunHolh9ZR2HgmD6sVjc9ivLOBQE04/F4up7WnHjUBCqdixW1AtziCkrKwu//vor/Pz8AAB+fn749ddfcefOHaN2+/fvx5AhQyCXy1G/fn14eXnh+++/N3l99erVg7u7u+G5i4sLNBqNyfutbAcOHMCbb74JAHjttdfQpk0bJCYmVmkNWq0WsbGxGDx4cJX2+yxXV1fY29sbvVbWcQiYfiyK1fc8jUOx+srD1GOxtPqel3EoxQsTEDdv3oSdnR0UCgUAQKFQwNbWFjdv3izSzsHBwfDc3t4e6enpVVqrXq/Htm3b0Lt3b9HpGzZsgFqtxuTJk3Ht2rUqrS0kJARqtRoff/wx7t+/X2S6RqNB48aNDc+rY/v9+OOPsLOzwxtvvCE6fd++fVCr1Rg7dizOnTtXpbWVdRw+aVudY7G0cQhU31gsbRwC1T8WSxuHQPWOxbJ4YQLi72TBggV46aWXEBQUVGTajBkzEB8fj9jYWPTr1w/jx4+HTqerkrq2bt2KvXv34rvvvoMgCAgPD6+Sfsvru+++K/Zb27Bhw5CQkIDY2FiMGzcOkydPRnZ2dhVX+PdQ0jgEqm8s1oRxCPw9xuILExD29vbIyMgwDGCdTofMzMwiu4j29vZGu9Q3b95Eo0aNqqzOiIgI/PHHH1i2bBnk8qJvj52dneH1gIAA5OfnV9m3oifbSqlU4q233sLZs2eLtHFwcMCNGzcMz6t6+2VkZODUqVNQq9Wi021sbGBubg4A8PDwgL29Pa5evVpl9ZV1HD5pW11jsbRxCFTfWCzLOASqdyyWNg6B6h+LZfHCBESDBg3g5OSEuLg4AEBcXBycnJyK/DrD29sb3377LfR6Pe7cuYNDhw6hf//+VVLjF198gf/85z9YuXIllEqlaJuMjAzD46NHj0Iul8POzs7kteXn5yMnJwcAIAgC9u/fDycnpyLtvL29sWPHDgDA9evXcfHiRdFfmJjK7t270aNHD8Mva5719PZLSUnBjRs30KxZs6oqr8zjEKi+sViWcQhUz1gs6zgEqncsljYOgeofi2XxQt0w6Nq1awgNDcX9+/fx8ssvIyIiAs2bN8eECRMQHByMtm3bQqfTITw8HD///DMAYMKECYYTXaZ09epV+Pn54bXXXkPt2rUBAE2aNMHKlSvh7++P6Oho2NnZYfTo0cjKyoJMJoOFhQVmzZoFFxcXk9eXlpaGadOmQafTQa/Xo0WLFpg3bx5sbW2N6svPz0doaChSUlIgl8vxz3/+E15eXiav74n+/fsjLCwMnp6ehteefn9nz56NS5cuQS6Xw9zcHMHBwejRo4dJalm4cCF++OEH3L59G9bW1qhXrx727dtX7Dh8tlZTj0Wx+pYtW1bsOARQpWNRrL41a9YUOw6frc/UY7G49xcQH4dA9Y3FinqhAoKIiMruhTnERERE5cOAICIiUQwIIiISxYAgIiJRDAgiIhLFgCCqZGvWrEFYWFiF5g0NDUVkZGQlV0RUMS/U5b6JqsLEiROruwSiSsE9CCIiEsWAoBdeRkYGpk2bhs6dO6N3797YvHkzgMc3dAkODsb06dOhUqkQGBiIy5cvG+aLjo5G9+7doVKp0L9/fxw/ftwwX0hIiKFdQkICfH194erqihEjRhhd9fTXX39FYGAgVCoVpk+fjoKCAqPaDh8+DH9/f7i6umLYsGFl6p+o0lTfrSiIqp9OpxMCAwOFqKgooaCgQPjzzz+F3r17C4mJicKKFSsEZ2dn4cCBA4JWqxXWrl0r9OrVS9BqtcK1a9cET09PIT09XRAEQUhLSxP++OMPQRAe3whm5syZgiAIwm+//Sa0b99eOHbsmKDVaoXo6GjBy8tLKCgoEAoKCoSePXsKGzZsELRarXDgwAHB2dlZ+OKLLwRBEIRLly4JnTt3FpKTk4VHjx4Ju3btEnr16iUUFBSU2D9RZeEeBL3QLl68iDt37mDq1KlQKpVo2rQphg4div379wMA3njjDXh7e8Pc3BxjxoyBVqvF+fPnoVAooNVqce3aNRQWFqJJkyZ45ZVXiix///796NGjBzw8PGBubo5x48bh4cOHOHfuHM6fP4/CwkKMGjUK5ubm8Pb2Rtu2bQ3z7tixA2+++Sbat28PhUKBwMBAmJubIzk5ucz9E0nBk9T0Qrtx4wYyMzPh6upqeE2n08HV1RUODg5Gl4d+crXSJ+3nzp2LqKgopKamolu3bggNDS1yNdPMzEyjm/7I5XLDJb8VCgXs7Owgk8kM059uq9FosGfPHmzZssXwWmFhITIzM9GpU6cy9U8kBfcg6IVmb2+PJk2a4PTp04Z/586dw1dffQUARvc30Ov1yMjIMFw5VK1WY9u2bTh8+DBkMhk+++yzIsu3tbU1uqeDIAiGu8rZ2NggIyMDwlPXy3y6rb29PSZOnGhU2/nz5w23Ky1L/0RSMCDohdauXTvUrVsX0dHRePjwIXQ6Hf7v//4PFy5cAABcunQJP/zwAx49eoRNmzZBqVSiffv2+O2333D8+HFotVoolUrUqlVL9MY6Pj4+OHLkCI4fP47CwkKsX78eSqUSKpUKLi4uMDMzw+bNm1FYWIgffvgBFy9eNMw7ZMgQbN++HefPn4cgCMjPz8dPP/2E3NzcMvdPJAUPMdELTaFQYM2aNYiIiECfPn2g1WrRrFkzTJ8+HQDQp08f7N+/H7Nnz8arr76KqKgomJubQ6vV4vPPP8e1a9dgbm4OlUoleuvL5s2b49NPP8WCBQuQkZEBJycnrFmzxnAjnqioKHzwwQdYtmwZevTogb59+xrmbdu2LRYsWIDw8HD88ccfqF27Njp06ABXV9cy908kBe8HQVSMqKgo/PHHHzx0Qy8s7pMSEZEoBgQREYniISYiIhLFPQgiIhLFgCAiIlEMCCIiEsWAICIiUQwIIiISxYAgIiJR/w9MhqhgxjIp/gAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "cfg = QlearningConfig()\n", + "plot_cfg = PlotConfig()\n", + "# 训练\n", + "env, agent = env_agent_config(cfg, seed=1)\n", + "rewards, ma_rewards = train(cfg, env, agent)\n", + "make_dir(plot_cfg.result_path, plot_cfg.model_path) # 创建保存结果和模型路径的文件夹\n", + "agent.save(path=plot_cfg.model_path) # 保存模型\n", + "save_results(rewards, ma_rewards, tag='train',\n", + " path=plot_cfg.result_path) # 保存结果\n", + "plot_rewards(rewards, ma_rewards, plot_cfg, tag=\"train\") # 画出结果\n", + "# 测试\n", + "env, agent = env_agent_config(cfg, seed=10)\n", + "agent.load(path=plot_cfg.model_path) # 导入模型\n", + "rewards, ma_rewards = test(cfg, env, agent)\n", + "save_results(rewards, ma_rewards, tag='test', path=plot_cfg.result_path) # 保存结果\n", + "plot_rewards(rewards, ma_rewards, plot_cfg, tag=\"test\") # 画出结果" + ] + } + ], + "metadata": { + "interpreter": { + "hash": "fbea1422c2cf61ed9c0cfc03f38f71cc9083cc288606edc4170b5309b352ce27" + }, + "kernelspec": { + "display_name": "Python 3.7.11 64-bit ('py37': conda)", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.10" + }, + "orig_nbformat": 2 + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/codes/QLearning/task0.py b/codes/QLearning/task0.py new file mode 100644 index 0000000..3f93d08 --- /dev/null +++ b/codes/QLearning/task0.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python +# coding=utf-8 +''' +Author: John +Email: johnjim0816@gmail.com +Date: 2020-09-11 23:03:00 +LastEditor: John +LastEditTime: 2021-12-22 11:13:23 +Discription: +Environment: +''' +import sys +import os +curr_path = os.path.dirname(os.path.abspath(__file__)) # 当前文件所在绝对路径 +parent_path = os.path.dirname(curr_path) # 父路径 +sys.path.append(parent_path) # 添加路径到系统路径 + +import gym +import torch +import datetime + +from envs.gridworld_env import CliffWalkingWapper +from QLearning.agent import QLearning +from QLearning.train import train,test +from common.utils import plot_rewards,plot_rewards_cn +from common.utils import save_results,make_dir + +curr_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") # 获取当前时间 +algo_name = 'Q-learning' # 算法名称 +env_name = 'CliffWalking-v0' # 环境名称 +device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # 检测GPU +class QlearningConfig: + '''训练相关参数''' + def __init__(self): + self.algo_name = algo_name # 算法名称 + self.env_name = env_name # 环境名称 + self.device = device # 检测GPU + self.train_eps = 400 # 训练的回合数 + self.test_eps = 30 # 测试的回合数 + self.gamma = 0.9 # reward的衰减率 + self.epsilon_start = 0.95 # e-greedy策略中初始epsilon + self.epsilon_end = 0.01 # e-greedy策略中的终止epsilon + self.epsilon_decay = 300 # e-greedy策略中epsilon的衰减率 + self.lr = 0.1 # 学习率 +class PlotConfig: + ''' 绘图相关参数设置 + ''' + + def __init__(self) -> None: + self.algo_name = algo_name # 算法名称 + self.env_name = env_name # 环境名称 + self.device = device # 检测GPU + self.result_path = curr_path + "/outputs/" + self.env_name + \ + '/' + curr_time + '/results/' # 保存结果的路径 + self.model_path = curr_path + "/outputs/" + self.env_name + \ + '/' + curr_time + '/models/' # 保存模型的路径 + self.save = True # 是否保存图片 + +def env_agent_config(cfg,seed=1): + '''创建环境和智能体 + Args: + cfg ([type]): [description] + seed (int, optional): 随机种子. Defaults to 1. + Returns: + env [type]: 环境 + agent : 智能体 + ''' + env = gym.make(cfg.env_name) + env = CliffWalkingWapper(env) + env.seed(seed) # 设置随机种子 + n_states = env.observation_space.n # 状态维度 + n_actions = env.action_space.n # 动作维度 + agent = QLearning(n_states,n_actions,cfg) + return env,agent + +cfg = QlearningConfig() +plot_cfg = PlotConfig() +# 训练 +env, agent = env_agent_config(cfg, seed=1) +rewards, ma_rewards = train(cfg, env, agent) +make_dir(plot_cfg.result_path, plot_cfg.model_path) # 创建保存结果和模型路径的文件夹 +agent.save(path=plot_cfg.model_path) # 保存模型 +save_results(rewards, ma_rewards, tag='train', + path=plot_cfg.result_path) # 保存结果 +plot_rewards(rewards, ma_rewards, plot_cfg, tag="train") # 画出结果 +# 测试 +env, agent = env_agent_config(cfg, seed=10) +agent.load(path=plot_cfg.model_path) # 导入模型 +rewards, ma_rewards = test(cfg, env, agent) +save_results(rewards, ma_rewards, tag='test', path=plot_cfg.result_path) # 保存结果 +plot_rewards(rewards, ma_rewards, plot_cfg, tag="test") # 画出结果 + + diff --git a/codes/QLearning/task0_train.ipynb b/codes/QLearning/task0_train.ipynb deleted file mode 100644 index 5715766..0000000 --- a/codes/QLearning/task0_train.ipynb +++ /dev/null @@ -1,216 +0,0 @@ -{ - "metadata": { - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.11" - }, - "orig_nbformat": 2, - "kernelspec": { - "name": "python3", - "display_name": "Python 3.7.11 64-bit ('py37': conda)" - }, - "interpreter": { - "hash": "fbea1422c2cf61ed9c0cfc03f38f71cc9083cc288606edc4170b5309b352ce27" - } - }, - "nbformat": 4, - "nbformat_minor": 2, - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "source": [ - "import sys\n", - "from pathlib import Path\n", - "curr_path = str(Path().absolute())\n", - "parent_path = str(Path().absolute().parent)\n", - "sys.path.append(parent_path) # add current terminal path to sys.path\n", - "\n", - "import gym\n", - "import datetime\n", - "\n", - "from envs.gridworld_env import CliffWalkingWapper\n", - "from QLearning.agent import QLearning\n", - "from common.plot import plot_rewards\n", - "from common.utils import save_results,make_dir\n", - "curr_time = datetime.datetime.now().strftime(\"%Y%m%d-%H%M%S\") # obtain current time" - ], - "outputs": [], - "metadata": {} - }, - { - "cell_type": "code", - "execution_count": 2, - "source": [ - "class QlearningConfig:\n", - " '''训练相关参数'''\n", - " def __init__(self):\n", - " self.algo = 'Qlearning'\n", - " self.env = 'CliffWalking-v0' # 0 up, 1 right, 2 down, 3 left\n", - " self.result_path = curr_path+\"/outputs/\" +self.env+'/'+curr_time+'/results/' # path to save results\n", - " self.model_path = curr_path+\"/outputs/\" +self.env+'/'+curr_time+'/models/' # path to save models\n", - " self.train_eps = 200 # 训练的episode数目\n", - " self.eval_eps = 30\n", - " self.gamma = 0.9 # reward的衰减率\n", - " self.epsilon_start = 0.95 # e-greedy策略中初始epsilon\n", - " self.epsilon_end = 0.01 # e-greedy策略中的终止epsilon\n", - " self.epsilon_decay = 200 # e-greedy策略中epsilon的衰减率\n", - " self.lr = 0.1 # learning rate" - ], - "outputs": [], - "metadata": {} - }, - { - "cell_type": "code", - "execution_count": 3, - "source": [ - "def env_agent_config(cfg,seed=1):\n", - " env = gym.make(cfg.env) \n", - " env = CliffWalkingWapper(env)\n", - " env.seed(seed)\n", - " state_dim = env.observation_space.n\n", - " action_dim = env.action_space.n\n", - " agent = QLearning(state_dim,action_dim,cfg)\n", - " return env,agent" - ], - "outputs": [], - "metadata": {} - }, - { - "cell_type": "code", - "execution_count": 4, - "source": [ - "def train(cfg,env,agent):\n", - " rewards = [] \n", - " ma_rewards = [] # moving average reward\n", - " for i_ep in range(cfg.train_eps):\n", - " ep_reward = 0 # 记录每个episode的reward\n", - " state = env.reset() # 重置环境, 重新开一局（即开始新的一个episode）\n", - " while True:\n", - " action = agent.choose_action(state) # 根据算法选择一个动作\n", - " next_state, reward, done, _ = env.step(action) # 与环境进行一次动作交互\n", - " agent.update(state, action, reward, next_state, done) # Q-learning算法更新\n", - " state = next_state # 存储上一个观察值\n", - " ep_reward += reward\n", - " if done:\n", - " break\n", - " rewards.append(ep_reward)\n", - " if ma_rewards:\n", - " ma_rewards.append(ma_rewards[-1]*0.9+ep_reward*0.1)\n", - " else:\n", - " ma_rewards.append(ep_reward)\n", - " if (i_ep+1)%10==0:\n", - " print(\"Episode:{}/{}: reward:{:.1f}\".format(i_ep+1, cfg.train_eps,ep_reward))\n", - " return rewards,ma_rewards" - ], - "outputs": [], - "metadata": {} - }, - { - "cell_type": "code", - "execution_count": 5, - "source": [ - "def eval(cfg,env,agent):\n", - " # env = gym.make(\"FrozenLake-v0\", is_slippery=False) # 0 left, 1 down, 2 right, 3 up\n", - " # env = FrozenLakeWapper(env)\n", - " rewards = [] # 记录所有episode的reward\n", - " ma_rewards = [] # 滑动平均的reward\n", - " for i_ep in range(cfg.eval_eps):\n", - " ep_reward = 0 # 记录每个episode的reward\n", - " state = env.reset() # 重置环境, 重新开一局（即开始新的一个episode）\n", - " while True:\n", - " action = agent.predict(state) # 根据算法选择一个动作\n", - " next_state, reward, done, _ = env.step(action) # 与环境进行一个交互\n", - " state = next_state # 存储上一个观察值\n", - " ep_reward += reward\n", - " if done:\n", - " break\n", - " rewards.append(ep_reward)\n", - " if ma_rewards:\n", - " ma_rewards.append(ma_rewards[-1]*0.9+ep_reward*0.1)\n", - " else:\n", - " ma_rewards.append(ep_reward)\n", - " if (i_ep+1)%10==0:\n", - " print(f\"Episode:{i_ep+1}/{cfg.eval_eps}, reward:{ep_reward:.1f}\")\n", - " return rewards,ma_rewards" - ], - "outputs": [], - "metadata": {} - }, - { - "cell_type": "code", - "execution_count": 6, - "source": [ - "cfg = QlearningConfig()\n", - "env,agent = env_agent_config(cfg,seed=1)\n", - "rewards,ma_rewards = train(cfg,env,agent)\n", - "make_dir(cfg.result_path,cfg.model_path)\n", - "agent.save(path=cfg.model_path)\n", - "save_results(rewards,ma_rewards,tag='train',path=cfg.result_path)\n", - "plot_rewards(rewards,ma_rewards,tag=\"train\",env=cfg.env,algo = cfg.algo,path=cfg.result_path)\n", - "\n", - "env,agent = env_agent_config(cfg,seed=10)\n", - "agent.load(path=cfg.model_path)\n", - "rewards,ma_rewards = eval(cfg,env,agent)\n", - "save_results(rewards,ma_rewards,tag='eval',path=cfg.result_path)\n", - "plot_rewards(rewards,ma_rewards,tag=\"eval\",env=cfg.env,algo = cfg.algo,path=cfg.result_path)" - ], - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "Episode:10/200: reward:-287.0\n", - "Episode:20/200: reward:-142.0\n", - "Episode:30/200: reward:-67.0\n", - "Episode:40/200: reward:-61.0\n", - "Episode:50/200: reward:-74.0\n", - "Episode:60/200: reward:-41.0\n", - "Episode:70/200: reward:-55.0\n", - "Episode:80/200: reward:-66.0\n", - "Episode:90/200: reward:-31.0\n", - "Episode:100/200: reward:-31.0\n", - "Episode:110/200: reward:-58.0\n", - "Episode:120/200: reward:-25.0\n", - "Episode:130/200: reward:-18.0\n", - "Episode:140/200: reward:-27.0\n", - "Episode:150/200: reward:-28.0\n", - "Episode:160/200: reward:-25.0\n", - "Episode:170/200: reward:-35.0\n", - "Episode:180/200: reward:-13.0\n", - "Episode:190/200: reward:-22.0\n", - "Episode:200/200: reward:-26.0\n", - "保存模型成功！\n", - "结果保存完毕！\n" - ] - }, - { - "output_type": "display_data", - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEXCAYAAABCjVgAAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAABRX0lEQVR4nO3dZ2AU5drw8f/sbrakF1KoofdOkE6kqAQIHQURQeztIOeVQ8ByFEUFUbGCciyPWACRIkoTFBAivYsghBIIIZvek23zfliyEJIAAUKie/2+wE5mZq+5d3euucvco6iqqiKEEEIAmsoOQAghRNUhSUEIIYSLJAUhhBAukhSEEEK4SFIQQgjhIklBCCGEiySFf6D333+f6dOn35L3evjhhzl+/Pgtea/KNnfuXG6//XamTp1a4m/5+fm89dZb9O/fn+joaKKjo3nnnXfIz893rdOkSRPS0tIqPM4NGzbw6quv3rT9Xem4yyMuLo6nn36a6OhoBg0axH333ceuXbsAOHv2LO3atQPg22+/5ZNPPgFgy5Yt9OrVi+HDh7N371769u3L0KFDad26NevXr3fte8uWLTRp0oSFCxe6lh04cIBu3bpxpVH3vXv35uDBg2zfvp2BAweW+PvBgwf517/+dUPHfa2WLFlCVFQUd955J//973+xWq235H1LUMU/znvvvae+/PLLlR3GP07v3r3VnTt3llhutVrVUaNGqa+88oqan5+vqqqq5uXlqa+88oo6atQo1Wq1qqqqqo0bN1ZTU1Nvacw3Q1nHXR5xcXFqt27d1M2bN7uWxcbGqh06dFD/+usv9cyZM2rbtm1LbBcTE6N++OGHqqqq6vvvv69OmzZNVVVVfe2119RXX33Vtd706dPVRx99VH344Yddyz7++GM1JibminH16tVLPXDggLpt2zZ1wIABN3SMN+Lo0aNqz5491dTUVNVut6uTJk1SP/nkk0qJRVc5qajqczgcvPbaa+zfv5/c3FxUVeXVV1+lcePGREZGsnbtWoKDgwG4++67efLJJ+nSpQuzZ89m586d2O12mjdvzvPPP4+3tze9e/emdevWHD16lH//+9/odDo+/vhjLBYLaWlpDBkyhGeeeQaATz75hCVLluDl5UVERAQbNmzgl19+wWKxlLn/siQlJTF9+nQSExOxWq0MGDCAxx57DIB58+axfv16CgsLyc/PZ8qUKdxxxx28//777Nu3D7PZTJMmTQgPDychIYHk5GQSEhIIDAzknXfeITQ0lN69e/Puu++Sl5fHO++8Q+3atTl27BgWi4UXX3yRzp07k5aWxtSpU4mPj8ff35/g4GAaNWrE008/XSzW3NxcXn31Vfbs2YNWq6Vv375MmjSJqVOn0qhRIx588EEAYmJiXK8vLdenn36auXPnsnLlSgCysrLo06cP69evp6CgoMxyuNT58+d56aWXSEhIQFVVhgwZwkMPPcQzzzxDUlISzz33HBMnTqR///6ubX7++Wfy8/OZNm0aGo2z8m0ymXjuuecYMmQI69atK7Y+wHfffce3336Lw+HA39+fF154gQYNGnDy5EmmT59OXl4eZrOZpk2bMmfOHAwGAy1btqRPnz4cOXKE2bNnc++99/LII4+wdetWzGYz999/P+PHj2fp0qWsXbuWjz/+mLFjx9K2bVv27NlDYmIiHTp0YObMmWg0GpYuXconn3yC0Wikc+fOfPnllxw+fLhYnJcfd/v27Ustn7NnzzJmzBgaNGhAQkICCxYsICQkxLWf+fPnM3z4cHr06OFa1qVLF9566y2MRmOx93z//fdJT0+nRo0abNiwAYPBwPLly8nNzcVut1NQUMCwYcOYNWuWa5tff/2VTz/9lLvvvpu8vDw8PT35/fffueeee0hJSeHFF18kNTWV5ORkatasyZw5cwgKCir1N7Nr1y4mT57MW2+9hdVq5ZVXXuHHH38kJiYGb29vjh49yvnz56lfvz5vv/02Xl5ebNq0idmzZ6PRaGjWrBmxsbF888031KpVq9i+R40axfjx4+nXrx8As2fPRlVV1zkiMDAQgHvuuYdXX32Vhx9+uNQYK5I0H5Vh//79mM1mFi1axKpVqxg6dCjz58/Hx8eHO+64gx9++AFwVomTk5Pp0aMHn3zyCVqtlqVLl/LDDz8QEhLC7NmzXfts1KgRq1evpm/fvnz22We88cYbLF26lEWLFvHJJ5+QlpbGb7/9xtKlS1myZAlLly4lNzfXtf3V9l+ayZMnM3z4cNc+Y2NjWbVqFQkJCcTGxvLVV1+xcuVKJk2axHvvvefaLiEhgWXLlrn2v2vXLt59913WrFmDr68vixYtKvFeBw4cYMKECSxfvpwRI0bwwQcfAPDqq6/SsGFDVq9ezbvvvsuePXtKjfW9996jsLCQVatWsXz5cvbs2cOOHTuu+lkVlWtUVBS5ubkcPHgQgB9//JHIyEj8/PzKLIfLPfvss3Tq1ImVK1fy7bff8sMPP/DTTz8xZ84cV3lffoLfvXs3HTp0cCWEIoqi0LVr1xLHu2PHDpYvX87XX3/N8uXLeeihh1wJcvHixQwZMoRFixaxbt06zp49y8aNGwGwWq306tWLtWvX0qpVKywWCwEBASxcuJD33nuPt956i8LCwhLHFB8fz4IFC/jhhx/Ytm0bO3bs4Pjx48yePZsvvviC5cuX4+3tjd1uL7Ht5cddVvmAM6E+8cQTrF27tlhCADh06BDt27cvsf/IyEhq165dYjnAQw89RO/evRk/fjzr1q1j1KhR9O/fn7feeouOHTsSHx9PRkYGR48exc/Pj3r16tG6dWu2bt2KxWLhwIEDdO/enZ9++om2bduyaNEiNmzYgNFoZMWKFaW+57Zt25g6dSpz584tNd5Dhw7x6aefsmrVKsxmM2vWrCE9PZ3//Oc/vPnmm6xYsYJOnTqRlJRU6v5HjhzJsmXLALDb7fzwww+MHDmSxMREqlev7lovLCyszH1UNKkplKFdu3b4+fmxcOFCzpw5w/bt2/Hy8gKcH+zLL7/Mgw8+yPfff8+wYcPQaDRs3LiR7OxsYmNjAeeP+NKrkYiICMB5spg3bx4bN27kxx9/JC4uDlVVyc/PZ9OmTfTr1w9fX18AxowZw7Zt2wCuuv/L5eXlsXPnTjIzM3n33Xddy44cOUL//v2ZOXMmK1eu5PTp064aUZG2bdui0138etx2222uGknz5s3JzMws8X41atSgWbNmrnWKvvybNm1y/T8kJMR1lXS52NhYpk6dilarRavV8tVXXwG4ti3LpeU6YsQIli1bRqtWrVi6dCmTJ0++ajlcWl579uzhs88+A8DHx4dhw4axefNmBgwYcMUYrsThcBR7vXHjRk6fPs2oUaNcyzIzM8nIyGDy5Mls3bqV+fPnc+rUKcxmM3l5eSWOtUifPn0AaNGiBRaLpdi6RXr16oVGo8Hb25vw8HAyMzM5cuQI3bp1IywsDID77ruP999//4rHcaXyadOmDTqdjrZt25a6raIoJcrhRuj1ejp16sSuXbs4fvw4t99+u+tYt2zZgq+vLy1atMDb25tx48axa9cuPv/8c06dOsWxY8do06ZNiX2eP3+exx57jNGjR9O0adNS37dHjx7o9XoAGjduTGZmJrt27aJBgwaubYYOHVpmn05UVBSzZs0iOTmZw4cPEx4eTt26dUvt97j8IuNWkaRQho0bNzJjxgweeOAB+vTpQ/369V21g4iICGw2GwcOHODHH390dW45HA6mTZtGZGQk4GwOufTKzdPTE3D+uIYOHUrfvn2JiIhg+PDhrF+/HlVV0el0xb4gWq3W9f+r7f9yDocDVVVZuHAhJpMJgLS0NAwGA3/88QdPPPEE48ePp1u3bnTs2JGXX365RKxFLq3iK4pS6pe4rHUuP6ayvuw6nQ5FUVyvExMTMRqNJd7v8g64S2MdPnw4Q4YMYeTIkWRnZ9OpUydycnLKLIfSyuvyZTabrdR4i7Rv356PP/4Yh8OBRqMhJycHRVHw8vJi+/btjB8/vsQ+Bw8ezOTJk12vzWYzfn5+TJo0CbvdTlRUFLfffjuJiYnFYrr8cyk6hqJyu9bPRavVlvk9K8vVykev1xe7kLhU27Zt2bdvH7169Sq2/IMPPqBOnTqlXpVfTY8ePdi5cyf79+9n2rRpgLPmsWjRIgIDA12J4s033+TAgQMMHz6cTp06YbPZSi0nrVbLJ598whNPPEFUVBStW7cusc61lCVc/I4/99xzHDp0CHA2HY0ePZq77rqLH3/8kb179zJy5EgAqlevjtlsdm2flJTkSti3mjQflWHr1q306tWLe++9l1atWrF+/fpi1euRI0fyyiuv0KRJE2rUqAFA9+7d+frrr7FYLDgcDl544QXefvvtEvs+ffo0OTk5PPPMM/Tu3ZsdO3a4tomMjGTdunVkZ2cDzhEJRa51/0W8vb1p27Ytn3/+OeBsYx89ejQbNmxg586dtGzZkgceeIDbbruNDRs2lNp8cDNERka6jiM9PZ3169cXO/kX6dKlC8uWLcPhcGCxWPjXv/7Fzp07CQgIcP2w0tLSXCNWShMaGkqbNm148cUXGTFiBHDlcriUt7c3bdq04euvvwYgOzub5cuX07Vr1yse35133omXlxevvfYaBQUF/PnnnwwbNozHHnsMrVZbormpW7du/PTTT66TwLfffsu4ceMA5yiaJ598kv79+6MoCvv376+Qz6V79+78/vvvriaK77777qrbXG/5ADz44IN89913bNmyxbVs8+bNLFiwoMyr8qvp2bMnW7du5dy5c7Rq1QrA1RS1fv1618XTli1bGDduHEOGDCEoKIjY2NhSyzQ4OJj27dszZcoUJk+eXGzk2JW0b9+eU6dOceTIEQDWrl1LVlYWiqIwY8YMVqxYwYoVKxg9ejTg7INcunQpe/fu5a677gKco6B++eUXUlNTUVWVRYsW0bdv3+sqlxslNYUyjBo1imeffZbo6Gi0Wi0RERGsW7fOdTU4ZMgQ3n777WIn5SeeeIKZM2cydOhQ7HY7zZo1IyYmpsS+mzRpwu23305UVBS+vr7UqVOHhg0bcvr0aXr06MHdd9/NPffcg9FopFGjRq6r22vd/6Vmz57NK6+8QnR0NBaLhYEDBzJo0CBSUlJcHaAeHh506dKFzMxMcnJybm5BAlOnTuX5558nOjoaf39/atSoUaJzEeCpp55ixowZDB48GLvdTv/+/bnzzjtp1aoVzz77LHfddRe1atXitttuu+L7jRw5kokTJzJ37tyrlsPlZs+ezfTp01m6dCkWi4Xo6GiGDRt2xffT6XR8+umnfPTRRwwdOtR11e3p6UlycjK//fZbsSvkHj168PDDDzNhwgQURcHb25sPPvgARVGYNGkSTz75JH5+fphMJlfb+c1Wr149pk6dyoMPPoher6dZs2au79mVlFU+CQkJV9wuPDycefPmMWfOHGbOnInD4SAwMJC5c+fSuHFjzp49W+5jqF27NjabjW7duhW7yOjRowfr1q2jQYMGADz55JPMmjWLjz76CK1WS/v27a9YpkOHDmXt2rW88cYbJRJ6afz9/Xn77beZMmUKGo2Gli1botPpyizPor/fddddrppe06ZNefLJJxk3bhxWq5U2bdpUSiczIENSq5oDBw6o//d//+d6/dlnn6kTJ06svIBugq+++krds2ePqqqqWlhYqA4fPlzduHFjJUd1a5jN5hsezlkR4uPj1ffff1+12+2qqqrq2rVr1REjRlRyVH9P2dnZ6syZM9W8vDxVVVX10KFDardu3VSHw1HJkV0fqSlUMfXq1WP+/PksXrwYRVGoXr06r7zySmWHdUMaNmzIK6+8gsPhwGq10q9fP1fV/p8uODjYNXS5KgkLC8NsNrtqwj4+Prz22muVHdbfkre3Nx4eHowYMQKdTodOp2POnDmlNpH+HSiqKg/ZEUII4SQdzUIIIVwkKQghhHCRpCCEEMJFkoIQQgiXf8Too/T0XByO8veXBwV5k5p688fl3yiJq/yqamwSV/lIXOVX3tg0GoWAAK8y//6PSAoOh3pdSaFo26pI4iq/qhqbxFU+Elf53czYpPlICCGEiyQFIYQQLlUiKaxcuZL+/ftzxx13uCbbEkIIcetVep9CUlIS77zzDkuXLkWv1zNq1Cg6depEw4YNKzs0IYRwO5VeU4iNjaVz5874+/vj6enJXXfdxZo1ayo7LCGEcEuVnhTMZnOxCcNCQkIq7TF0QojiZGq0K/snlk+lNx+VVqjlnV0wKKjsB9dfTXCwz3VvW5H+7nGpqsqhuFSa1w9Cq3F+nlabHZtdxWSomK9dabGZ0/NITMmlTaMbm6lUVVXOmnOoGeyNRlO+7+fVyszuUNmwM55thxJpWb8aw3rd3KZTm93B0dPpBPoaqV7t4vj0y+M6ciqNrDwLtzV3PvErK9dCzIe/MbhnA+7qXNcVa36hDW+Th2u7pLQ8DB5a/H2KP8nuUqqq4lBBq1FIzy4gMSWXxnUC0GlLXpdaUQjwNaKqKt+uO0qXVtVpXi+II6fS0HtoqV/T77rKIa/ASuyBc+Tk2yiw2MjILqRvxzrUr+nH8k3HadmgGo3rBBTb5kRCJruPJJGVe5qxUc3Qe1x8Qt2huBRmf72bCdEt6NmuFqcTs9BqFUIDPfHQFX+S3S+74tm0J4FhtzckM7eQo6fTaVE/iJPnsjh+NoMRvRtx1pzNqthTPDumA7VDfVBVla/WHGHHH+fR6TQM6FqXhrUD2LTnLO2aBNO64cXv9M08X1T6LKnLli1j165dzJgxA4APP/wQVVV56qmnrnkfqak51zVONzjYh+Tk7HJvV9Eujctmd7Dol+Pc2bE2wf5XfwjKrYoLYO2OeKr5mejQpOQJ9+CJVN5ZvJ8BXcIZHul82MmnPx7mRGIW0x+8De1lj+RUVbVcFwOXr39pbLuOmEnPKeSOiNq8s3g/B0+k0qd9Le7p4zzZfrTsEO0aVaNHmxqu7R2qiuaS/aVlFWDUa/E0Ok9+P+86w7frj9EsPIAJ/ZsR5Gd0XdBcHnd+oY3UzAJqhXgTHOyD2ZzFjj/NFFrt1KvuS61gr2Lb/PT7Kb7fdAIPnQaNRmHOU91ZvPE48UnZ9LutDpv2neNMcg51QnwY1achgT5G/m/NETo0CaZDkxDXfs6Yc1i44Rg2u4PGtf0Z1K0uf55OZ/7Kw+QWWDEpNnq1CmJAj0bUDg8jJe3ik8USUnKZ8eUuTAYdbz3ZDYCVW0+y7LeT6LQaXhwXQV6hjW/W/8X5tDym3dcBT6OORb8cZ/fRZDSKQusGQQzuXo8gPyPHEzJxOJwXABarncW/HkdVYXhkfRasPUp2XiF+BoV+EaG0qOXDH3HnqR7kjdVh56dfDxNgcODrYcOSn4d3vZbcM7gbE9/bQk6+lY5NQ2jXuBrJ6fkcOJFKTp6V6kFe9GlfncK8fA7HnScuPpnaQQbCg/Ro7BayMrM5n5yBxl6ISbHgpRTipbGCRkOQnycZ6RlodB60blabP+LMGDQOfI0KyalZ6LDjQENolwE0bH8bR06nU2C18+VPBzFaMjHpbDQO8yQ+MQ0ddkxaB+HBJvy8dOh1GgotNk7EJ+OlsWKgEKNixaRYMSoWjIoVo8aGQwU9NgyKFbvWgF+AH6l5kJ+Ti6+HFb1qQVHtKKgogIJKkldjmtw3tdznMY1GueKFdKUnhaSkJEaPHs2SJUswmUyMGjWKV155pdTno5bln5wU/jydzpvf7qVnm+qMj2p2U9/H4VDZsOcsial5hId6E9m25jXHlZiay3Pzt6MA4/s3pUfrGsXW/X5THD/9fhqAp4e3ok2Dakx87zdyC2w8HN2cLi0uPn921bbT/HYgkf+MbkdugZVjZzPp1e5iLDn5Vox6reuq8vc/zvPl2qN0bh5KdNe6BPoa+WXfORKSsukbUYuXP9+Jza7y4vgIpn+xi+AAE0lpeQzqVpca1byYt+IPFODxIS3p0CSY5b+d5KffT6PTKtzZsRa92oTx6ufbqBbkQ8zY2zh1Pps3FuykWbBKXmYGRh0Mvb0xv+87TV5eAQO71iU924LVFEDrts1ZsjGOVdtO07pBEP261GPrvrNsPXT+wtGo1Ao0MqRnA9o3DSMr18LzH//GbTVUetaFVbEnaNakDgePJuClseBJAf4eFmr6qFhys1B1RkyeRrLTM9AoKgE+zmdYF1gd5OVb8NJa8dZawWYBnQGH3YanxoIRCwrFfyM2rRFTp5HQOJKXPt9JdkYGqqrh3Wedj4Gc/FEsoYGeJKXlkZ1nRQVCvMGbPPRaBY29EL09j471PdFZ8zifaEZvz8eoseKBDb1iQ48dD8WGSetAhxWdakOv2PFQrv0Ro3Y0aBp0ZsfhJEJNVrTWPDyw4oEdo9aBh2JD67CiVcpxDtBoUT08ySuwoqh27DoTqs2CSbFiRYsdLRaHBg+9AS9vE7npqfhqCrDrvSkosKBVHBiVKz+7uyQFu1aPQ2dC7+lFgapHZ/REZzByLjUPrYcBjd5IXHwyvjorWocFH18fatcKAYMniWkFFFhVagZ7k5CSR453HW7r2/eflxTAOST1448/xmq1MmLEiHI/hq4ik4I5PY83v93L5HvbE3KLrtQvjavoak3voeHtJ7vjabxy00t5rrgPn0pj9sJ9aDUKOp2GD57p4bqC33ssmbXb48krtPPc/R2wWO1kFdqpGeAsg6/WHWXz/nM0rOnH0fgMnrs/gnyLjU9/PEzMfR34/Kc/yS2woVEgt8DKo4Na8tpXu9FpFUIDPHn5wdvQKAppWQVM/WQbVpuDWsHepGblUVBo592JPfE2ebD1YCJfrfuL2iHe/L9RbTmVmMVbi/YR6GskLasAPy89o/s2Zu7SfRgVK14eDkK0WWhthRi9vTHkmRnYQk/SufOk5ILe4IGvPQNPjRW7zYpR60DjsGHQqmhUGzounqzsqoJq8CW/0Ion+Vc96dhVBWPrO9kQp5KXkYq3ko/OUYhJsVLX146PkotSkIWiOgBwKFqs6PBwFHKlFilV74nG5EOew0BGeiYeih2NyQe7qpCTb0GrgF6roNd7EBgUgM7Tm6QsG/FnU9B6eNCySS2M3j4oBk8SM+zEHjiDYiukocd5mnicx2YKJD83Dx9NAQWqB0rL/py1+bN7zxEGNLShK8ykMDsDTzUfnS237EAVLYUaIzatEb3RBDo9dkWHDQ98/byxoeNUSiG1qwfi5e2FotNzLsNKRr5KvVpBnE/LwwY0qhuG1uAJBk+W/XaKamd+pZ3hNJkWHZ7+gZj8Aiiwa9HqDRg9TaB1vo85247J04Sfnxc6gwl0BhSdHjyMKDoDeOhRdAYUg5fzb4pCYmouh0+l06t9TVb9fpqVsad4algrmtbxJyEll/BQHxRF4dn3NzIkNJ5QTSZ/ncuhTaNQfPz98awWRoGqx6Jq8PPzQdF5gFaPovUAjQabXcVqd+Dp7e2MQ7lyN66qqsxfeZiUzAJ6d6jJbc1Ci9VgS/OPTAo3qiKTwu+HzjP/x8M8OqgFnZqHXvO+HQ6VrDwL/t5lt7NeS1zvLN7P8YRM8gttjLmjMX061LrQPqui1WhQVZWMHAt5BVaW/XaSuIRMXnmoE4UWOz/vOsOALuEA7PjTTKCPgWr+Jvy89Ph66Vmx5SQ/bDnJ2H5N+HLNUV4YF0G96r78sucsX637i0BfA2lZhQzuXo9DJ1OJS8hi0t1taFDDj//34VY6NAnm3sjavP1/m/D18cRWkEdhdiat2zZl74GT3FEnjxqGPP44mYbO2w+vvETq+1rIyLHgWy0Y/5BQ/krIwpaRTC1vG3l5+fhoCrCpGhwtokg11GbV5iPU9teQm51NsJeKLT+XMJOFNqFgzzKTn5uPHQVfJZ8yfzsGL+wGPzLSs9AqdhSfEPyDAjmfYSUxw4K/nxcNw6uBRsf2I6mcy7DSomEI8fFJmOw5qCi0bFaHarXCUTx9Scu1s3VvPG2b1cTfz5vf9idgsdrxOb+brsZjF9/XwwgeJuwaA3q/IBSvADSeftgVPb//kUhmZg5GjY3atUJp2qYFmoCa/LIvid93H6Nt81oMvL0litELRXPxQuDLNUc4eiaD5++PwGTQYXc4SjTFFTl2NoMAHwPV/IpfzOQWWMm1qkz98DeeaZ2CX94Z/jxXQP0mjUg5dpCW+ovPS1Y8/dH4hqAYfVBMvig+1dB4BZCWY8XLxweTrz+K0RvF6HPhpHdjTxu7/De5Zns8i389zt29GrL41+O88WhnQgI8b+g9rsRitRfrNyjy1uL9FBbaCPQ1EJeQxZtPdK2wGMrrZieFSu9oruqSM53trimZ+aX+fd2OeI4nZPLE0FbFli/+9Tib9p/j7Se7XXfHqkNViUvIpGPTEE4nZbN5/zn6dKjF2h1nWLXtNBNHtGbD7rNsO+wcraXTKtjsKruOmIk357BxbwIH4lKx2hykZhUU2/e0+zpw/GwGNYO9aduwGl9ylKPxGWTmWPh63V+0b+DPo7cH8tP6fcRt30R1TR6tfC2cXXeADJOOYfp0OqanY/82jYl6oBBQAF/gxM/09AbSAKM3bQ1WvGyFpOn98QmtxzlLJo7UdHS5SQQVWlG9/fGtEY6mUMVu8CH+6F80+3MFPsATvoADuNA/qpoAkx9aQtDVbEpmjsqx+FRat26Ef1AgFocGY1B1jputLF1/iPYdWtCvVxsAFq44xL5jKcwe1w1vkwcNgcu7dDu1s/Hn6XRaNapG4o4zfPPrcYb1rE/1rnVd64QAQ5t1cL0eVL/5hVqXnvA+I/m/NX/SsW0jhvVpXuYPtmc7O3EJWdQO9cbLeLHTtlOnYE7nGons2QCNZ8kLivv7NcXhUF2d3WUlBIBGtfxLXe5l9CC8ljd6vY6DHu0wBEew5kQ8s7p25fW9wUzo5svuP84SEFyNscO6lXqiv/bLoxsTGuhMaAfiUtBqFIL8jBX6fqUlBIA6YT5s2n2W7DwL4WFVcxDIzSJJ4SpSMpwn09SswhJ/yyuwsWLrSfIL7aRnFxJwYfSFOSOfDbvPYneoHD6VTliQJ5v2JTCqdyPXj3nB2qPotBru7t0Ac3o+HjpNiSu6xJRc8gptNKrlR7C/ke83nSAn38r+4ynk5Ft5bcFuVODOjrWpWc2LZnUDeGfxfn47kMj5tFwa1vQjISUHo17HlHvbodNAdnISazcdYOeuP8k9f47+NbLQ/7qB5wPN5P4RgNVq47nAHIIzMilYZqcP0OfCb0BFwaGCWqCgehkxhDZDE9IbPANYHRuHl483QcHB7Nn9B4WqjnvHDiEguBo/bz7B6tg4orrWZ1jPBtRIy+Olz3dSmGmnVf0gnhjaEoOHFhPOjvXn927inoYakpOSKVAN3D+oHYrehKL3BA9DsSp4OFDT7qB6mB/JydnoLyxvUl3ldl0w7S4ZdTQ+qikZPSzFRs5czmTQ0b6xc5s7O9ambpgPjev4X/V7EhbovHo9aNaQZPWhWrUrb+Oh09I0PKDEcl9PPQ8NbH7Fbcs7+qk0iqIQ4m/CnJGPUa8lyNeIv7ceg4eWv7K82J/hy9BWNSv9OcNF5XrsbCYhAaYrJsGKVCfUh7xCG3mFNrq2DLv6Bn9jkhSuIjmj7JrCpv0J5Bc626CPnE6ny4Uvy9JNca52+oMnUtn+ZxK7jpjp1CyUBjX9MKfn8eveBMDZrn8uJReTQcczI9vQsNbF4XbHEjIBaFjTj7RsZ1KKS8jk1Pls2jcOJifPQkTTEPpG1HZt07l5KMt+O0GQJof7Glnxr5+KJvs8yrafcGSZCXHYeMITMAOeQAao2nCsxiAMOSlo0eAZEoq+Tlc0QbXR+ASTU2DDK7AaYXXr8NvuMwT5GUv0rwxu1BVFUcgrsDJvuw0/LwMBwdUA6NWuJn+cTKPThaGOoYGePDqoBccTMhnSo16xYYk6rYZq/iaO5nnxZ5rCbc1D0QbWuuJnVNqwRkVR6NqyerFlRr2OsMBr/8prNEqpJ+7SBPgYMHhoORCXCnDL+p9uRIi/iYSUXIx6LcH+zqafYH8T+46nAFA7pPKviIP9TSiKcyhsUYKoDHUuqR3UCa38cqlIkhSuoigZpGYWb37JyrWwbucZmoUHEJ+UzeHTaXRpGUaBxcbOP83ceVttUjIK2HcsmdwC5yiFw6fTaVDTjy0Hz6MoMKBLXTbsPkPfiNociEth1rd7aVjTF19vI+eSczBn5OHr6UFIgAlfLz2KAlsOJFJotdOhcTBdWoahOhzYk45jO3MQ29mDRKbE0z3AgU5xwAFA0aL4haDxr44uvC2KXyhnsnVs3HIAi6rj7nsHERQWStof55m/8jA1qnkxffhtxTq3itKUoig0K+MkWXRF6Wn0oH/n8GJNZgE+Bl4YF1Fs/baNqtG2UbVS91U90JOjZzLIK7RRK7jsed+rEkVRCA00EZ/knNc+JOBvkBQCTOyPS0Gv03Lbhf6ykAATZ5Odx1A75Prv/7lZdFoN1fyMJGcUVGpSqH1JIpDmIzdmsztIyy5EwZkUikb2nDiXxfvfHyCv0MbQHvVZtzOeP0+no6oq5vR8VKBBDT+qB3mx+69kAHy99Px5Ko0BncOJPZRIi3qBDOtZn6E96qEoCv27hPPD1pOcSszGnJ5HNT8jTer407pBEIqiYNRrqRXkyZ5jyYRpM2ics5P8n09jO/cnFOYCCpqQ+uhb9uXomUy8gsKo17Y9Gr/qKNriH3M9h8qH253HEhTmPBk0Dw/AZNAxtEe9q452uJohPerf0PahgRevVmsFV/6J6VqFBXoSn5SDh05zxRu5qorgABM2u4rNbnPVbIL9nW32ngYdgb5V4xhCAzxJziggtBKTgr+3AS+jDq1Wc12DR/5OJClcQVpWAaoKdUK8iTfnkJ1nxddLz0+/n8KhqrxwfwS1Qrw5kxzIrqPJmNPzMac7axYhASZX23XDWn7Ur+7LL3sS2HnETFpWIXdfuGu16Arbz0vP2DubAMVHE6jWAgp3L8f611b+nyOFHD8DPpoCOAB27yB04e3Q1W6FrmYLFKPzBNqm85WPS6NRGNevKTa7w7XMz9vAB8/0qPQ2ZKDYFWHNv0lNAS7GHRJguuHEeitc2sRVlAyKltUK8a4S3wVwNjceOplWqTUFRVFoUicAg0elzwxU4SQpXEHyhU7mpuEBxJtzSM0qwNdLT0aOhTqhPtS6UL1ufqFJ5c/T6eQVOpuKgv1NmAw6Rt7egMa1/cktsLJu5xn+96OziabdVaZdcORlYD97iMLdK1CzU9DWbM55nxacOnkGm19t7hw+GI3XtbV3l6ZNw5JNN1XlJFD04w/wMRQbmVPVuZLC36A/AS5PChdqCheavapC01GR8FAfPHQaalSr3AuEJ4a2rNT3v1UkKVxB0XDUpnUCWLfzDCmZBdSr7ktWbiHVgy5etRTVCk4mZqEozqaiojb1qM7O+wQKLDZX5/OTQ1viobtsmgeHA3vSMWyn9nAm8Q+sKc6x4opfKKboGHTVm+CVmss3h7YzqG3dG0oIVV3RyfXv1HQEEHbhOxFagePob6ZAXyNajYLdobqSQo0gLzSKQv0avpUc3UVdW4bRol7gFUeN3Qp/h9rfzSBJoQyqqpKSUYBWo7hGBBX1K2TmWvDz0rvWVRSFumE+nDqfjadBV2ono1GvY3TfRoQFelI96OIVjyM7GcuBNdjidqAWZINGhym8BUr9ruhqNUcTVMc1BDMs0JPxUU1Lvcr/J/H10hPsb6TpNQwFrUqqB3kRGmCiyd8kbo1GoZq/ifwCq+siJtDXyIxHOlX6PFuX0mgU13BvUfEkKZTirzMZzF64F0+DjiBfI94mD0wGHSmZ+eQV2rDZ1WJJAaBudR9W/R6Pp1FHmwZBpe63d/uLQysdGYkU7v0R2/HfQVHQ1YtAV68DulqtCKkZUuoNT4qi0LNNjRLL/2kUReG1RzpXmeasa2Xw0PL6o10qO4xyqV/dxzU6rsjfpaYjKoYkhQscDpVzqbnUCvZm5xEzqgq5BTYaXJimt5qfkdTMAjJyLAD4el+WFMJ8cagqOfnWKw5HVC15WPatwrJ/NWi1eLS8A32ru9B4B1bcwf0NVdZNSu7mgf7N+PtPdCNuJkkKF8QeOs9nq/7khXERHDqRSot6gYzr19TV9l/Nz8j5tDyycpw3kfl5Fa/O1r1k7PLlc7OoDhu20/uwndqD7eQusFnQNe6OodPdaExVp+1WuJ/SbvwT7k2SwgVH49MBWPzLcZLS8+nToVaxdsywQE8Onkgl3ZUUitcUAnwM+Hnpycy1uOZrAXDkpJK//kMc5hNg8MKjYRc8mvdCW61uxR+UEEKUkySFC45fmFLi6JkMAFrVL94vEBroic2ucvKcs63f77Lmo6LO5v1xqYT4m1DtNqyHfqZw7w+gqhh7P4qu/m0omtIn3BJCiKpAkgKQnWchKT2fLi1C+f2PJKr5GUv0C4ReeP3X2Qx0WgXPUmY+7dgsBLuqYtI5yF/zHvaEP9DWbo2x6xg0frdqXkkhhLh+khSAuIQsACLb1sRDp6VGNa8SI1+Kxs6fNecQ6GsodWRM15bV6dLYn/zVb2M//xfGyAfxaNKj4g9ACCFuEkkKOJuOtBpn80/j2v6lruPrpceo11JgseNXxtwnjtx0V/+BsdejeDS8ynwTQghRxUhSwJkU6oT6lPmADSiaBdOT0+ezS3Qy29POUPDrfByp8aDRYuz7BB71IsrYkxBCVF2SFICk9LwSHculCQ0wlUgKtjMHyP/5QxS9CUOnu9HWaYM2oOYV9iKEEFWXJAVAdajXNF67qF/B90JScGQkkr/+IzR+oZj6TfpHz0ckhHAPcucK4FAp+6Hvlyiaz93P24BqLST/5w9QNDpMd02UhCCE+EeQpIBz8jsNV88KtS/M2hnsZ6Dgt89xpJ/D2OcxNN5Xb3oSQoi/A0kKXKgpXENJ1Arx5pUHb6NR3l5sx7ehjxiKrpZ7zLEuhHAPkhS4UFO4xhk5wxznsWxbiLZOW/TtBlZwZEIIcWu5bUdzetJ5fLych+9Q1av2KTiyk7HGbcd6aD2KVyCmXg+7nnMghBD/FG6bFHKWv8rOWp1pOuA+VPXKj6K0p50lf+UbqIU5aAJrY+z1CIrh7/PsYCGEuFZumxQcKmhzzcCVm48cuenk/zQLtDo8734Nrf8//yE3Qgj35bbtHzmqCQ9rDsCFmkLp61n2/ohamIup/2RJCEKIfzy3TQrZqgm9zZkUnH0KJbOCIy8D69FNeDTuhjZQ7lIWQvzzuW1SyMWE3paLqqqoKmhKqSlY9q0ChwN9WxllJIRwD+6dFOx5OBwOgBJ9CnZzHNY/1uPRuDsa35DKCFEIIW45t00KOaonCipqvvNZCpfmBNVaSP4vn6B4BWDofE8lRSiEELee2yaFXJxPUnPkOR/DeWmfQuHuZahZSRhvf0iGngoh3IrbJoU8xTm53cWk4FxuTzuL9eA6PJr2RFejWWWFJ4QQlcJ9k4KrpuBsPtIoCqrDQeFv/4ei98Rw292VGZ4QQlQKt00KuRdqCmr+xeYjy4HV2JOOYegyGsXoXZnhCSFEpXDbpGBT9NgUnSspeBecx7JrKbp6Eegada3k6IQQonK4bVJQFIVCjZdr9FH4uXXgYcTYY/wV50ESQoh/spueFJYvX0737t0ZPHgwgwcP5p133gHg3LlzjBkzhn79+vH444+Tm5sLQFZWFo888ghRUVGMGTOG5OTkmx1SqTQahQKtF+Rn0Vh3Dv/s4xjaRUuzkRDCrd30pHDw4EFiYmJYsWIFK1asYNKkSQC8/PLL3HvvvaxZs4aWLVvy0UcfATBnzhwiIiJYvXo1I0eOZMaMGTc7pFIpikKBxgs1L4NBnnso1Pvh0bz3LXlvIYSoqiokKSxfvpxBgwbx7LPPkpmZidVqZefOndx1110ADBs2jDVr1gCwceNGoqOjARg4cCCbN2/GarXe7LBKUBTI13iiZCVSW5fG2Vr9UHT6Cn9fIYSoym56UggODubpp59mxYoVVK9enenTp5Oeno63tzc6nc61TlJSEgBms5ng4GAAdDod3t7epKWl3eywStAoCvka5wikw5aaZAS1qvD3FEKIqu66n6ewevVqXn/99WLL6tevzxdffOF6/dBDD9G3b1/+85//lNj+Sp25Gk35clVQUPn7ATw8tKTqwlC8Alic0Ykxvp4EB/uUez8VpSrFcqmqGhdU3dgkrvKRuMrvZsZ23UkhKiqKqKioYsuys7P54osvGD9+POB8eI1OpyMwMJCcnBzsdjtarZbk5GRCQpyTzIWEhJCSkkJYWBg2m42cnBz8/f3LFUtqag4Oh1qubex2B2cMDSm48w7S520jN7eA5OTscu2jogQH+1SZWC5VVeOCqhubxFU+Elf5lTc2jUa54oX0TW0+8vT05H//+x/79+8H4KuvvuKOO+7Aw8ODiIgIVq1aBThHKPXs2ROAyMhIli9fDsCqVauIiIjAw8PjZoZVKo2i4FBVVBTXayGEcHc39XGcWq2WOXPm8NJLL1FQUEDdunWZNWsWAP/973+JiYlh7ty5VK9enbfffhuAiRMnEhMTw4ABA/Dx8WH27Nk3M6QyaRRQHSqq6qxhSE4QQogKeEZzREQEy5YtK7G8Zs2aLFiwoMRyf39/5s2bd7PDuCrlQk2hqNVJblgTQgg3vqNZozifzVxUU5DmIyGEcOOkULKmULnxCCFEVeDGSQEcl/QpSE1BCCHcOCloNAqqimsoq+QEIYRw46RQ1HykSkezEEK4uHFSuNB8hDQfCSFEEbdNChpFQS1WU6jceIQQoipw66TgcIDDdfOaZAUhhHDbpKAozoSgOpyvNZIThBDCfZNCUfORq6YgWUEIIdw3KbhqCkX3KVRyPEIIURW47blQudCnIENShRDiIrdNCs6b1y5pPpKcIIQQ7psULjYfOV9rpE9BCCHcOCmg4LhkllRpPhJCCDdOChoNF5qPnK8lJwghhBsnBWdHs8ySKoQQl3LbpOB8yI50NAshxKXcOCkU9SlcfC2EEO7ObZPC5c1H0tEshBBunRSQWVKFEOIybpsUipqPZJZUIYS4yG2TQlHzkcM1+qiSAxJCiCrAjZNC8eYj6WgWQgg3TgoazeV3NFdyQEIIUQW4bVIoekazw1H0WrKCEEK4bVK4+IxmuaNZCCGKuG1SUIqSgut1pYYjhBBVgtsmBY2Cc0iqQ4akCiFEEbdNCkVJQIakCiHERW6bFIqSgN1+oaYgWUEIIdw3KRTVFOwO6WgWQogibpwUnP/aL4xJlZwghBBunBSKnsns6mhGsoIQQrhtUihKAnaH3NEshBBF3DYpuDqai/oUpKNZCCHcNykUjTa62NFcmdEIIUTV4LZJoWi0kWtIqrQfCSHEjSeFd999l/fff9/1Oisri0ceeYSoqCjGjBlDcnIyABaLhcmTJxMVFcXQoUOJi4sDnLOUzpw5k379+tG/f3927959oyFdExl9JIQQJV13UsjOzmbatGl89tlnxZbPmTOHiIgIVq9ezciRI5kxYwYACxYswGQysXr1aqZNm0ZMTAwAa9euJS4ujlWrVvHhhx8SExODzWa7gUO6NkU1BYfcpyCEEC7XnRQ2bNhA3bp1eeCBB4ot37hxI9HR0QAMHDiQzZs3Y7Va2bhxI4MGDQKgY8eOpKenc+7cOTZt2kT//v3RaDTUq1ePGjVqsHfv3hs4pGtTlANsMveREEK4XHdSGDJkCI888gharbbYcrPZTHBwMAA6nQ5vb2/S0tKKLQcIDg7m/PnzmM1mQkJCSiyvaMolNQXJB0II4aS72gqrV6/m9ddfL7asfv36fPHFF9f8JhpN6blHo9G4nmdwLeuXJSjIu1zrA/j5mgDw8NChURSCg33KvY+KVNXiKVJV44KqG5vEVT4SV/ndzNiumhSioqKIioq65h2GhISQkpJCWFgYNpuNnJwc/P39CQkJITk5mfDwcACSk5MJCQkhNDTU1Rl96fLySE3NcfUNXKucnAIAcvMtKIpCcnJ2ubavSMHBPlUqniJVNS6ourFJXOUjcZVfeWPTaJQrXkjf9CGpkZGRLF++HIBVq1YRERGBh4cHkZGRrFixAoBdu3ZhMBioUaMGPXv2ZOXKldjtdk6fPs2pU6do1arVzQ6rhItDUh1yj4IQQlxw1ZpCeU2cOJGYmBgGDBiAj48Ps2fPBmDs2LG8+OKLDBgwAL1ez6xZswDo168fBw4ccHVCz5gxA6PReLPDKkG5kA6dfQqSFYQQAkBRS2vU/5u5nuajHX8mMW/FH7SsF8jxhEw++ndkBUVXflW1qlpV44KqG5vEVT4SV/lV+eajv4tLn6cgNQUhhHBy26Rw6YR40qcghBBObpsULtYUHFJTEEKIC9w4KTj/dUhNQQghXNw2KWikT0EIIUpw26RwaUezPGBHCCGc3DYpuDqa7TL3kRBCFHHbpFD05DWHQ3U9r1kIIdyd2yaFogN3jj6q1FCEEKLKcNukUNSnYJM+BSGEcHHbpKC5tPlIqgpCCAG4cVJQLuloloqCEEI4uXFSuDAkVZWaghBCFHHbpHDxeQpSUxBCiCJumxRczUcy95EQQri4bVIoqimoKjIkVQghLnDbpHBpIpCaghBCOLltUtBckgikT0EIIZzcNilcWjnQSE1BCCEAN04Kl97FLM1HQgjh5LZJ4dJEIDlBCCGc3DgpXPp/yQpCCAFunBSko1kIIUpy26QgNQUhhCjJbZOC1BSEEKIkt00KxTuaJSsIIQS4cVK4tHYgD9kRQggnt00KxWoKlRiHEEJUJW6bFOTmNSGEKMltk0Lx0UeVF4cQQlQl7psUuHT0kWQFIYQAN04KmkuOXJGOZiGEANw4KShyn4IQQpTgtklBI3c0CyFECW6bFGSWVCGEKMltk0LxaS4kKwghBLhxUpAhqUIIUZIbJwXFlQykT0EIIZx0N7qDd999F41Gw9NPPw3Azp07eeqppwgLCwOgefPmvP7662RlZfHss89y5swZAgMDmTNnDsHBwVgsFp577jkOHTqE0Whk9uzZNGjQ4EbDuiaKoqCqqow+EkKIC667ppCdnc20adP47LPPii0/ePAgEyZMYMWKFaxYsYLXX38dgDlz5hAREcHq1asZOXIkM2bMAGDBggWYTCZWr17NtGnTiImJuYHDKZ+iZCB9CkII4XTdSWHDhg3UrVuXBx54oNjygwcPsnXrVoYMGcJjjz1GYmIiABs3biQ6OhqAgQMHsnnzZqxWKxs3bmTQoEEAdOzYkfT0dM6dO3e9YZVLUTKQ5iMhhHC67qQwZMgQHnnkEbRabbHlPj4+3H///SxfvpzIyEgmTZoEgNlsJjg4GACdToe3tzdpaWnFlgMEBwdz/vz56w2rXIruZJacIIQQTlftU1i9erWrCahI/fr1+eKLL0pdf/r06a7/jx49mrfeeovs7OxS19VoSs9JZS0vS1CQd7nWd73PhWTg6aknONjnuvZRUapaPEWqalxQdWOTuMpH4iq/mxnbVZNCVFQUUVFR17Qzh8PBxx9/XKIGodPpCAkJISUlhbCwMGw2Gzk5Ofj7+xMSEkJycjLh4eEAJCcnExISUq6DSE3NweFQy7UNXGw+KiywkZxceuKqDMHBPlUqniJVNS6ourFJXOUjcZVfeWPTaJQrXkjf1CGpGo2Gn3/+mbVr1wKwfPly2rRpg8lkIjIykuXLlwOwatUqIiIi8PDwIDIykhUrVgCwa9cuDAYDNWrUuJlhlUlRpPlICCEudcNDUi83c+ZMXnjhBT788EMCAwOZNWsWABMnTiQmJoYBAwbg4+PD7NmzARg7diwvvvgiAwYMQK/Xu9a/FYqSgow+EkIIpxtOCkX3JxRp1KgRCxcuLLGev78/8+bNK7HcYDAwc+bMGw3jumilo1kIIYpx2zuaAbmjWQghLuPmSeFC85Fbl4IQQlzk1qdDTVHzEVJTEEIIcPek4Go+qtw4hBCiqnDrpCCjj4QQoji3Tgqu5iOZJlUIIQB3TwpK8X+FEMLduXVSUGSWVCGEKMatk4JGbl4TQohi3DspSEezEEIU49ZJQe5oFkKI4tw6KUjzkRBCFOfWSUHuUxBCiOLcOinIHc1CCFGcmycFqSkIIcSl3DopyJPXhBCiOLdOChc7miUrCCEEuHtSkJqCEEIU49ZJQXHNfSRZQQghwM2TgnQ0CyFEce6dFOTmNSGEKMatk4JMcyGEEMW5eVK40HwkD1QQQggAdJUdQGXSFjUfVXIcQvxT2e020tOTsdks17S+2azB4XBUcFTlV1XjgivHptPpCQgIRqu99lO9WycFaT4SomKlpydjNHri5RV2Tb8znU6DzVb1Tr5VNS4oOzZVVcnNzSI9PZlq1apf8/6k+Qh5HKcQFcVms+Dl5SsXXpVAURS8vHyvuZZWxK2TgtzRLETFk99X5bmesnfvpODqaK7kQIQQ/3gjRkSTmHiussO4Krc+HUqfghBCFOfWHc1y85oQ7mXPnl3MnfsedruD6tWrYzJ5cuJEHA6HgzFj7qd37zsYPLgfixcvx9PTi8cfn0C3bj0ZP34C69evZd++vTz++FO8/vorJCebSUlJpm3bdjz//HT27t3t2nf9+g3417/+zfTpL2A2J1G3bn0sFmfb/vHjx5g1awZ2ux29Xs+0af+ldu06lVwyF7l3UlCkT0GIW2XrwUS2HEi84jqKAqpa/n13b12dbq2ubYTNmTPxLFnyIwsWfE61asE8//zL5Obm8NhjE2jevCUdOkSwd+8e2rXrQGJiIvv27QEmsG1bLH363EFs7BYaNWrMq6/OxGq1ct99Izl69EixfXt7e/P22zNp3Lgps2e/x759e/jll58BWLz4G0aNuo/evfuyYcM6/vjjoCSFqkImxBPC/dSuHY63tze7du2gsLCAn376AYCCggJOnjxBly7d2b17BxqNwp13RrFhwzpsNiv79+9j8uRpGAwGDh8+xOLF33Dq1EkyMzPJz88rtm+AvXt389JLrwHQtm17atSoCUCXLt14++1ZbN8eS9euPbj99j6VUAplc+ukoJEhqULcMt1aXf1q/lbcD2AwGABwOOy88MIrNGnSFIC0tFR8ff3Izs5m4cKv0Wp1dOjQkfj4U/zwwwrq16+PwWBgyZKFbNz4C4MGDWXEiNs4eTIO9UL1pmjf4GyBuPSmMq1WC0CvXn1p2bI1W7f+xnfffcu2bVuZMuX5Cj3m8nDrjmYZkiqE+2rfviPLly8BICUlhXHjRpOUdJ6AgAAMBgNbt26mdeu2tG/fkc8/n0/Xrj0A2LlzO4MGDePOO6MAhWPH/ir1juKIiNtYt241AH/++QcJCWcBePHFqRw+/AdDhgznoYceczU9VRVunRTkcZxCuK8JEx6msLCQsWPvZuLEx3jiiX9Rs2YtwNnE4+3tg6enJx06dCQ5OZmuXbsDcPfd9/L5558wYcIY3n57Ji1bti51qOmDDz5KQsJZ7rvvbr766gtX89HYsQ+wYMHnTJgwhg8/nMPTT0+6dQd9DRRVvZ5unaolNTUHh6P8h7Fk8wlWxZ7iubEdaFDTrwIiuz7BwT4kJ2dXdhglVNW4oOrG5u5xnT9/mrCw8Gtev6pOJ1FV44Krx3b5Z6DRKAQFeZe5vlvXFGT0kRBCFOfWSUGR+xSEEKKY604Ku3fvZvjw4QwePJhx48aRkJAAQFZWFo888ghRUVGMGTOG5ORkACwWC5MnTyYqKoqhQ4cSFxcHOGfymzlzJv369aN///7s3r37JhzWtZHHcQohRHHXnRQmT57MjBkzWLFiBdHR0bz66qsAzJkzh4iICFavXs3IkSOZMWMGAAsWLMBkMrF69WqmTZtGTEwMAGvXriUuLo5Vq1bx4YcfEhMTg81muwmHdnUXp7m4JW8nhBBV3nUlBYvFwsSJE2na1Dm+t0mTJiQmOu9U3LhxI9HR0QAMHDiQzZs3Y7Va2bhxI4MGDQKgY8eOpKenc+7cOTZt2kT//v3RaDTUq1ePGjVqsHfv3ptxbFdV9JAdqSkIIYTTdSUFvV7P4MGDAXA4HHzwwQf07dsXALPZTHBwMAA6nQ5vb2/S0tKKLQcIDg7m/PnzmM1mQkJCSiy/FWRIqhBCFHfVO5pXr17N66+/XmxZ/fr1+eKLL7BYLK7mnkcffbTMfWjKmJtao9FQ2ojYstYvy5WGV11JUTIICvImONjnuvZRUapaPEWqalxQdWNz57jMZg06Xfl+z+Vd/1apqnHBlWPTaDTl+qyvmhSioqKIiooqsTw3N5fHH38cf39/5s6di4eHBwAhISGkpKQQFhaGzWYjJycHf39/QkJCSE5OJjzcOV42OTmZkJAQQkNDXZ3Rly4vj+u9T6Hojub09FyMVejzdvex7dejqsbm7nE5HI5yje+vqvcDVKW4Pv30Y8B5cxxcPTaHw1Hss66w+xQmT55MeHg47777Lnq93rU8MjKS5cuXA7Bq1SoiIiLw8PAgMjKSFStWALBr1y4MBgM1atSgZ8+erFy5ErvdzunTpzl16hStWrW63rDK5eJDdqT9SAgh4DonxDt8+DAbNmygYcOGDBkyBHDWEObPn8/EiROJiYlhwIAB+Pj4MHv2bADGjh3Liy++yIABA9Dr9cyaNQuAfv36ceDAAVcn9IwZMzAajTfh0K5OkZvXhHAbe/bs4ssvP0NV4dy5s9x+ex+8vLz47bdNqKrK7NnvEhgYxPffL2LNmlUUFOSj0Wh4+eXXadiwQbF9jRgRTfPmLTl27CgfffQ/tm2L5bvvvsXhUGnSpCn//vcUPvroXerWrc/QoSP44YdlLFr0NV9/vQSbzcbddw9m8eIVrFjxfYn3qlu3Xon9r179Iz/8sAw/P398fHxo1qwFNpuN119/mZMnT6CqKkOHjmTQoKE3XE7XlRSaN2/O0aNHS/2bv78/8+bNK7HcYDAwc+bMEssVRWHKlClMmTLlekK5IUVdF1Wo5UiIfyzrX1uxHt18xXUURSm1n/FqPJr0xKNxt6uud/jwHyxYsAg/P3+io+/gySef4dNPF/Daay+zfv06BgyIZvPmTXzwwccYDEb+9795LFv2HZMnx5TYV+fOXZk+/XVOnIhj5crlzJ37GQaDgXnzPuDbbxfQpUt3fvxxOUOHjmD37h1kZWWRlpbKqVMnadGiFYWFBaW+16RJ/ym2/yNHDvPTTz/w2WdfoygKjz32AM2ateDgwf1kZWXx5ZffkpqaxgcfzKm8pPBPIdNcCOFe6tdvQGhoGAB+fv5ERNwGQGhoGNnZWXh5efPSS6+yfv06zpyJZ/v2WBo1alLqvpo3bwnA3r27OHv2DI8++gAANpuVxo2bMnr0WNcT1k6fPk2fPneyb99ejhz5g65du1/1vYr2v2fPbjp37oanpyfgnHrbbrdTv34D4uNPM3HiE3Tq1I3HH3/6ppSRWycFRfoUhLhlPBp3u+rVfEV36Op0xU95Rc84KJKUdJ6nn36U4cPvpnPnrgQGBnHsWOmtIkXPTrDbHfTu3ZdnnpkMQF5eHna7HYPBQMOGjVm3bjXh4eG0a9eB3bt3cODAfu69d9xV36to/87aU/HnMtjtdvz8/FmwYDF79uxgy5YtTJhwHwsWLMbH58ZGlbl1y4lG7lMQQlziyJHD1KpVm3vuGUPz5i3Zti0Wh8N+xW3atevA5s0bSU9PQ1VV3nrrdRYv/gaArl278cUX/6Nduw60a9eBLVs2YzIZ8ff3v+b3iojoSGzsFnJycigsLGTz5l8B2LJlE9Onv0C3bj145plnMZlMmM1JN1wGbl1TKOpTkOYjIQRAx46dWbZsCffdNxIPDw+aN2/JiRNxV9ymUaPGPPDAw/zrX4+hqiqNGjXhvvvGA9ClS3dmz36Ddu0i8PX1xd8/gC5dupfrvRo1asLIkaN56KH78fHxITTU+fS6zp278euvGxg9egR6vZ7IyN40aNDwhsvArZ+nEPunmf+tOMScf3XH11N/9Q1uEXcf2349qmps7h6XPE+h4snzFG6iID8jJoMWg4f26isLIYQbcOvmo66talAnyFOSghBCXODWNQWNRsFkcOu8KIQQxbh1UhBCVLx/QLfl39b1lL0kBSFEhdHp9OTmZkliqASqqpKbm4VOV75BNNJ2IoSoMAEBwaSnJ5OTk3FN62s0GhyOqjfKp6rGBVeOTafTExAQXOrfyiJJQQhRYbRaHdWqVb/m9d19CO/1uNmxSfOREEIIF0kKQgghXP4RzUc3MqFdVZ0MT+Iqv6oam8RVPhJX+ZUntqut+4+Y5kIIIcTNIc1HQgghXCQpCCGEcJGkIIQQwkWSghBCCBdJCkIIIVwkKQghhHCRpCCEEMJFkoIQQggXSQpCCCFc/hHTXJTXypUrmTt3LlarlfHjxzNmzJhKi+WDDz5g9erVAERGRvKf//yHqVOnsnv3bkwmEwBPPfUUd9xxxy2P7f777yc1NRWdzvk1mT59OvHx8ZVadt999x1fffWV6/XZs2cZPHgw+fn5lVZmOTk5jBo1innz5lGrVi1iY2N5/fXXKSwsJCoqikmTJgHw559/8vzzz5OTk0NERAQvv/yyq2xvRVyLFi1iwYIFKIpCy5Ytefnll9Hr9XzwwQd8//33+Pr6AnD33XdX6Od6eVxlfd/LKseKdGlscXFxvP32266/JSUl0aZNGz7++ONbWmalnSMq9Dumupnz58+rvXr1UtPT09Xc3Fw1OjpaPXbsWKXEsnXrVvWee+5RCwsLVYvFot5///3qunXr1IEDB6pJSUmVElMRh8OhduvWTbVara5lVansVFVV//rrL/WOO+5QU1NTK63M9u3bpw4cOFBt0aKFeubMGTU/P1+NjIxU4+PjVavVqk6YMEHduHGjqqqqOmDAAHXv3r2qqqrq1KlT1a+//vqWxXXixAn1jjvuULOzs1WHw6H+5z//UT///HNVVVX10UcfVffs2VNhsVwpLlVVS/3srlSOtzK2ImazWe3Tp4968uRJVVVvXZmVdo5YuXJlhX7H3K75KDY2ls6dO+Pv74+npyd33XUXa9asqZRYgoODiYmJQa/X4+HhQYMGDTh37hznzp3jhRdeIDo6mvfee69SHu5x4sQJFEXh4YcfZtCgQXz11VdVquwAXnrpJSZNmoTRaKy0Mlu8eDH//e9/CQkJAeDAgQOEh4dTu3ZtdDod0dHRrFmzhoSEBAoKCmjbti0Aw4YNq9CyuzwuvV7PSy+9hLe3N4qi0LhxY86dOwfAoUOHmD9/PtHR0UyfPp3CwsJbFldeXl6pn11Z5ViRLo/tUrNmzWLUqFHUrVsXuHVlVto54tSpUxX6HXO7pGA2mwkOvvgkopCQEJKSkiollkaNGrk+wFOnTrFq1Sp69OhB586dee2111i8eDG7du1iyZIltzy2rKwsunTpwocffsgXX3zBwoULOXfuXJUpu9jYWAoKCoiKiiI1NbXSymzGjBlERES4Xpf1/bp8eXBwcIWW3eVx1axZk65duwKQlpbG119/TZ8+fcjNzaVZs2ZMmTKFZcuWkZWVxUcffXTL4irrs6uM3+nlsRU5deoUO3bs4P777we4pWVW2jlCUZQK/Y65XVJQS5kUVlEqd0rcY8eOMWHCBKZMmUL9+vX58MMPCQoKwmQyMXbsWDZt2nTLY2rXrh2zZs3C09OTwMBARowYwXvvvVdivcoqu4ULF/LAAw8AULt27SpRZlD296uqfO+SkpIYN24cw4cPp1OnTnh5eTF//nzCw8PR6XRMmDDhlpZdWZ9dVSkvgEWLFnHvvfei1zufdVwZZXbpOaJOnTol/n4zv2NulxRCQ0NJSUlxvTabzaVWF2+V3bt3M378eP7f//t/DB06lKNHj7J27VrX31VVrdDOyLLs2rWL33//vVgcNWvWrBJlZ7FY2LlzJ7179waoMmUGZX+/Ll+enJx8y8suLi6O0aNHM3ToUJ588kkAzp07V6xWdavLrqzPrir9Tjds2ED//v1dr291mV1+jqjo75jbJYWuXbvy+++/k5aWRn5+PuvWraNnz56VEktiYiJPPvkks2fPZsCAAYDzC/baa6+RmZmJ1Wpl0aJFlTLyKDs7m1mzZlFYWEhOTg7Lli3jzTffrBJld/ToUerWrYunpydQdcoMoE2bNpw8eZLTp09jt9v58ccf6dmzJzVr1sRgMLB7924Ali9ffkvLLicnhwcffJCJEycyYcIE13Kj0cibb77JmTNnUFWVr7/++paWXVmfXVnleKulpaVRUFBA7dq1XctuZZmVdo6o6O+Y2w1JDQ0NZdKkSdx///1YrVZGjBhB69atKyWWTz/9lMLCQt544w3XslGjRvHII48wevRobDYbd955JwMHDrzlsfXq1Yv9+/czZMgQHA4H9957Lx06dKgSZXfmzBnCwsJcr5s2bVolygzAYDDwxhtv8PTTT1NYWEhkZCT9+vUDYPbs2Tz//PPk5ubSvHlzVxv1rbBkyRJSUlL47LPP+OyzzwDo3bs3EydOZPr06Tz++ONYrVbat2/vapa7Fa702ZVVjrfS2bNni33XAAIDA29ZmZV1jqjI75g8eU0IIYSL2zUfCSGEKJskBSGEEC6SFIQQQrhIUhBCCOEiSUEIIYSLJAUhboKHH36Y48ePl2ubRx99lKVLl1ZQREJcH7e7T0GIijB//vzKDkGIm0KSgnBrv/zyi+v5EEajkSlTprBlyxaOHTtGSkoKqampNG3alBkzZuDt7c0333zDwoUL8fDwwGAwMH36dBo2bEjv3r159913adWqleu5BRqNhmrVqvHCCy9Qr149kpKSiImJwWw2U6NGDVJTU11xxMXFMWPGDDIyMrDb7YwdO5YRI0aQm5vL1KlTOX36NBqNhhYtWjB9+nQ0Gqnkiwpy/TN9C/H3dvLkSXXgwIFqWlqaqqrO5zN069ZNfeONN9SePXuqycnJqt1uV//973+rb7zxhmqz2dQWLVq45v5ftmyZunDhQlVVVbVXr17qgQMH1NjYWLVv375qamqqqqqq+v3336tRUVGqw+FQn3jiCfWdd95RVVVVT506pbZt21b9/vvvVavVqvbv3189dOiQqqqqmpWVpUZFRal79+5Vly1bpk6YMEFVVVW12Wzqc889p546depWFpNwM1JTEG5r69atmM1mxo8f71qmKArx8fH069ePatWqATBixAhee+01pkyZQr9+/Rg1ahS333473bp1Izo6utg+f/vtN/r3709gYCDgnNN+xowZnD17ltjYWKZMmQJAeHg4nTp1ApxTIsfHxzNt2jTXfgoKCjh8+DA9evTgnXfeYezYsXTt2pVx48YRHh5ekcUi3JwkBeG2HA4HXbp0Yc6cOa5liYmJLFq0CIvFUmy9ouaa2bNn89dffxEbG8v8+fNZsmQJc+fOda2rljJrjKqq2Gy2EtMbF82sabfb8fX1ZcWKFa6/paSk4OPjg8Fg4Oeff2b79u1s27aNBx54gOeff75S5gES7kEaJoXb6ty5M1u3biUuLg6ATZs2MWjQIAoLC9mwYQPZ2dk4HA4WL15Mr169SEtLIzIyEn9/f8aPH88zzzzD0aNHi+2ze/furFq1irS0NAC+//57/P39CQ8Pp0ePHixatAhwTr+8fft2AOrVq4fBYHAlhcTERAYOHMihQ4f45ptvmDp1Kt27d2fy5Ml0796dY8eO3aoiEm5IJsQTbm316tXMmzfPNSf+tGnT+P3339m2bRt2u5309HQ6duzI888/j9FoZOHChXz55ZcYjUa0Wi2TJk2ia9euxTqav/76axYuXIjD4SAwMJAXX3yRRo0akZaWxtSpU4mPjycsLAybzcbQoUMZNmwYR44ccXU022w27r//fkaPHk1eXh7Tpk3j6NGjmEwmatSowYwZM/Dz86vsohP/UJIUhLjM+++/T3p6Oi+++GJlhyLELSfNR0IIIVykpiCEEMJFagpCCCFcJCkIIYRwkaQghBDCRZKCEEIIF0kKQgghXCQpCCGEcPn/2meCVE5jMD0AAAAASUVORK5CYII=", - "text/plain": [ - "
" - ] - }, - "metadata": {} - }, - { - "output_type": "stream", - "name": "stdout", - "text": [ - "加载模型成功！\n" - ] - } - ], - "metadata": {} - } - ] -} \ No newline at end of file diff --git a/codes/QLearning/task0_train.py b/codes/QLearning/task0_train.py deleted file mode 100644 index 2a9e0ea..0000000 --- a/codes/QLearning/task0_train.py +++ /dev/null @@ -1,126 +0,0 @@ -#!/usr/bin/env python -# coding=utf-8 -''' -Author: John -Email: johnjim0816@gmail.com -Date: 2020-09-11 23:03:00 -LastEditor: John -LastEditTime: 2021-09-23 12:22:58 -Discription: -Environment: -''' -import sys,os -curr_path = os.path.dirname(os.path.abspath(__file__)) # 当前路径 -parent_path=os.path.dirname(curr_path) # 父路径，这里就是我们的项目路径 -sys.path.append(parent_path) # 由于需要引用项目路径下的其他模块比如envs，所以需要添加路径到sys.path - -import gym -import torch -import datetime - -from envs.gridworld_env import CliffWalkingWapper -from QLearning.agent import QLearning -from common.plot import plot_rewards,plot_rewards_cn -from common.utils import save_results,make_dir - -curr_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") # 获取当前时间 -class QlearningConfig: - '''训练相关参数''' - def __init__(self): - self.algo = 'Q-learning' # 算法名称 - self.env = 'CliffWalking-v0' # 环境名称 - self.result_path = curr_path+"/outputs/" +self.env+'/'+curr_time+'/results/' # 保存结果的路径 - self.model_path = curr_path+"/outputs/" +self.env+'/'+curr_time+'/models/' # 保存模型的路径 - self.train_eps = 400 # 训练的回合数 - self.eval_eps = 30 # 测试的回合数 - self.gamma = 0.9 # reward的衰减率 - self.epsilon_start = 0.95 # e-greedy策略中初始epsilon - self.epsilon_end = 0.01 # e-greedy策略中的终止epsilon - self.epsilon_decay = 300 # e-greedy策略中epsilon的衰减率 - self.lr = 0.1 # 学习率 - self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # 检测GPU - - -def env_agent_config(cfg,seed=1): - env = gym.make(cfg.env) - env = CliffWalkingWapper(env) - env.seed(seed) # 设置随机种子 - state_dim = env.observation_space.n # 状态维度 - action_dim = env.action_space.n # 动作维度 - agent = QLearning(state_dim,action_dim,cfg) - return env,agent - -def train(cfg,env,agent): - print('开始训练！') - print(f'环境:{cfg.env}, 算法:{cfg.algo}, 设备:{cfg.device}') - rewards = [] # 记录奖励 - ma_rewards = [] # 记录滑动平均奖励 - for i_ep in range(cfg.train_eps): - ep_reward = 0 # 记录每个回合的奖励 - state = env.reset() # 重置环境,即开始新的回合 - while True: - action = agent.choose_action(state) # 根据算法选择一个动作 - next_state, reward, done, _ = env.step(action) # 与环境进行一次动作交互 - print(reward) - agent.update(state, action, reward, next_state, done) # Q学习算法更新 - state = next_state # 更新状态 - ep_reward += reward - if done: - break - rewards.append(ep_reward) - if ma_rewards: - ma_rewards.append(ma_rewards[-1]*0.9+ep_reward*0.1) - else: - ma_rewards.append(ep_reward) - print("回合数：{}/{}，奖励{:.1f}".format(i_ep+1, cfg.train_eps,ep_reward)) - print('完成训练！') - return rewards,ma_rewards - -def eval(cfg,env,agent): - print('开始测试！') - print(f'环境：{cfg.env}, 算法：{cfg.algo}, 设备：{cfg.device}') - for item in agent.Q_table.items(): - print(item) - rewards = [] # 记录所有回合的奖励 - ma_rewards = [] # 滑动平均的奖励 - for i_ep in range(cfg.eval_eps): - ep_reward = 0 # 记录每个episode的reward - state = env.reset() # 重置环境, 重新开一局（即开始新的一个回合） - while True: - action = agent.predict(state) # 根据算法选择一个动作 - next_state, reward, done, _ = env.step(action) # 与环境进行一个交互 - state = next_state # 更新状态 - ep_reward += reward - if done: - break - rewards.append(ep_reward) - if ma_rewards: - ma_rewards.append(ma_rewards[-1]*0.9+ep_reward*0.1) - else: - ma_rewards.append(ep_reward) - print(f"回合数：{i_ep+1}/{cfg.eval_eps}, 奖励：{ep_reward:.1f}") - print('完成测试！') - return rewards,ma_rewards - -if __name__ == "__main__": - cfg = QlearningConfig() - - # 训练 - env,agent = env_agent_config(cfg,seed=0) - rewards,ma_rewards = train(cfg,env,agent) - make_dir(cfg.result_path,cfg.model_path) # 创建文件夹 - agent.save(path=cfg.model_path) # 保存模型 - for item in agent.Q_table.items(): - print(item) - save_results(rewards,ma_rewards,tag='train',path=cfg.result_path) # 保存结果 - plot_rewards_cn(rewards,ma_rewards,tag="train",env=cfg.env,algo = cfg.algo,path=cfg.result_path) - - # # 测试 - env,agent = env_agent_config(cfg,seed=10) - agent.load(path=cfg.model_path) # 加载模型 - rewards,ma_rewards = eval(cfg,env,agent) - - save_results(rewards,ma_rewards,tag='eval',path=cfg.result_path) - plot_rewards_cn(rewards,ma_rewards,tag="eval",env=cfg.env,algo = cfg.algo,path=cfg.result_path) - - diff --git a/codes/QLearning/train.py b/codes/QLearning/train.py new file mode 100644 index 0000000..40a7746 --- /dev/null +++ b/codes/QLearning/train.py @@ -0,0 +1,51 @@ +def train(cfg,env,agent): + print('开始训练！') + print(f'环境:{cfg.env_name}, 算法:{cfg.algo_name}, 设备:{cfg.device}') + rewards = [] # 记录奖励 + ma_rewards = [] # 记录滑动平均奖励 + for i_ep in range(cfg.train_eps): + ep_reward = 0 # 记录每个回合的奖励 + state = env.reset() # 重置环境,即开始新的回合 + while True: + action = agent.choose_action(state) # 根据算法选择一个动作 + next_state, reward, done, _ = env.step(action) # 与环境进行一次动作交互 + agent.update(state, action, reward, next_state, done) # Q学习算法更新 + state = next_state # 更新状态 + ep_reward += reward + if done: + break + rewards.append(ep_reward) + if ma_rewards: + ma_rewards.append(ma_rewards[-1]*0.9+ep_reward*0.1) + else: + ma_rewards.append(ep_reward) + if () + print("回合数：{}/{}，奖励{:.1f}".format(i_ep+1, cfg.train_eps,ep_reward)) + print('完成训练！') + return rewards,ma_rewards + +def test(cfg,env,agent): + print('开始测试！') + print(f'环境：{cfg.env_name}, 算法：{cfg.algo_name}, 设备：{cfg.device}') + for item in agent.Q_table.items(): + print(item) + rewards = [] # 记录所有回合的奖励 + ma_rewards = [] # 滑动平均的奖励 + for i_ep in range(cfg.test_eps): + ep_reward = 0 # 记录每个episode的reward + state = env.reset() # 重置环境, 重新开一局（即开始新的一个回合） + while True: + action = agent.predict(state) # 根据算法选择一个动作 + next_state, reward, done, _ = env.step(action) # 与环境进行一个交互 + state = next_state # 更新状态 + ep_reward += reward + if done: + break + rewards.append(ep_reward) + if ma_rewards: + ma_rewards.append(ma_rewards[-1]*0.9+ep_reward*0.1) + else: + ma_rewards.append(ep_reward) + print(f"回合数：{i_ep+1}/{cfg.test_eps}, 奖励：{ep_reward:.1f}") + print('完成测试！') + return rewards,ma_rewards \ No newline at end of file diff --git a/codes/README.md b/codes/README.md index 49f6ac7..355127c 100644 --- a/codes/README.md +++ b/codes/README.md @@ -13,6 +13,7 @@ 其中```model.py```,```memory.py```,```plot.py``` 由于不同算法都会用到，所以放入```common```文件夹中。 +**注意：新版本中将```model```,```memory```相关内容全部放到了```agent.py```里面，```plot```放到了```common.utils```中。** ## 运行环境 python 3.7、pytorch 1.6.0-1.8.1、gym 0.17.0-0.19.0 diff --git a/codes/SAC/task0_train.ipynb b/codes/SAC/task0_train.ipynb index 8148a4b..14be84e 100644 --- a/codes/SAC/task0_train.ipynb +++ b/codes/SAC/task0_train.ipynb @@ -45,7 +45,7 @@ " self.model_path = curr_path+\"/outputs/\" +self.env+'/'+curr_time+'/models/' # path to save models\n", " self.train_eps = 300\n", " self.train_steps = 500\n", - " self.eval_eps = 50\n", + " self.test_eps = 50\n", " self.eval_steps = 500\n", " self.gamma = 0.99\n", " self.mean_lambda=1e-3\n", @@ -121,7 +121,7 @@ " print(f'Env: {cfg.env}, Algorithm: {cfg.algo}, Device: {cfg.device}')\n", " rewards = []\n", " ma_rewards = [] # moveing average reward\n", - " for i_ep in range(cfg.eval_eps):\n", + " for i_ep in range(cfg.test_eps):\n", " state = env.reset()\n", " ep_reward = 0\n", " for i_step in range(cfg.eval_steps):\n", diff --git a/codes/SAC/task0_train.py b/codes/SAC/task0_train.py index 625f1d7..719b668 100644 --- a/codes/SAC/task0_train.py +++ b/codes/SAC/task0_train.py @@ -33,7 +33,7 @@ class SACConfig: self.model_path = curr_path+"/outputs/" +self.env_name+'/'+curr_time+'/models/' # path to save models self.train_eps = 300 self.train_steps = 500 - self.eval_eps = 50 + self.test_eps = 50 self.eval_steps = 500 self.gamma = 0.99 self.mean_lambda=1e-3 @@ -96,7 +96,7 @@ def eval(cfg,env,agent): print(f'Env: {cfg.env_name}, Algorithm: {cfg.algo}, Device: {cfg.device}') rewards = [] ma_rewards = [] # moveing average reward - for i_ep in range(cfg.eval_eps): + for i_ep in range(cfg.test_eps): state = env.reset() ep_reward = 0 for i_step in range(cfg.eval_steps): diff --git a/codes/Sarsa/task0_train.py b/codes/Sarsa/task0_train.py index d21db17..e477afa 100644 --- a/codes/Sarsa/task0_train.py +++ b/codes/Sarsa/task0_train.py @@ -31,7 +31,7 @@ class SarsaConfig: self.result_path = curr_path+"/outputs/" +self.env+'/'+curr_time+'/results/' # path to save results self.model_path = curr_path+"/outputs/" +self.env+'/'+curr_time+'/models/' # path to save models self.train_eps = 200 - self.eval_eps = 50 + self.test_eps = 50 self.epsilon = 0.15 # epsilon: The probability to select a random action . self.gamma = 0.9 # gamma: Gamma discount factor. self.lr = 0.2 # learning rate: step size parameter @@ -74,7 +74,7 @@ def train(cfg,env,agent): def eval(cfg,env,agent): rewards = [] ma_rewards = [] - for i_episode in range(cfg.eval_eps): + for i_episode in range(cfg.test_eps): # Print out which episode we're on, useful for debugging. # Generate an episode. # An episode is an array of (state, action, reward) tuples @@ -94,7 +94,7 @@ def eval(cfg,env,agent): ma_rewards.append(ep_reward) rewards.append(ep_reward) if (i_episode+1)%10==0: - print("Episode:{}/{}: Reward:{}".format(i_episode+1, cfg.eval_eps,ep_reward)) + print("Episode:{}/{}: Reward:{}".format(i_episode+1, cfg.test_eps,ep_reward)) print('Complete evaling！') return rewards,ma_rewards diff --git a/codes/TD3/README.md b/codes/TD3/README.md new file mode 100644 index 0000000..8001e9c --- /dev/null +++ b/codes/TD3/README.md @@ -0,0 +1 @@ +这是对[Implementation of Twin Delayed Deep Deterministic Policy Gradients (TD3)](https://arxiv.org/abs/1802.09477)的复现 \ No newline at end of file diff --git a/codes/TD3/agent.py b/codes/TD3/agent.py index 3d43700..f77a912 100644 --- a/codes/TD3/agent.py +++ b/codes/TD3/agent.py @@ -1,3 +1,13 @@ +#!/usr/bin/env python +# coding=utf-8 +''' +Author: JiangJi +Email: johnjim0816@gmail.com +Date: 2021-12-22 10:40:05 +LastEditor: JiangJi +LastEditTime: 2021-12-22 10:43:55 +Discription: +''' import copy import numpy as np import torch @@ -5,40 +15,41 @@ import torch.nn as nn import torch.nn.functional as F from TD3.memory import ReplayBuffer - - -# Implementation of Twin Delayed Deep Deterministic Policy Gradients (TD3) -# Paper: https://arxiv.org/abs/1802.09477 - - class Actor(nn.Module): - def __init__(self, state_dim, action_dim, max_action): + + def __init__(self, input_dim, output_dim, max_action): + '''[summary] + + Args: + input_dim (int): 输入维度，这里等于n_states + output_dim (int): 输出维度，这里等于n_actions + max_action (int): action的最大值 + ''' super(Actor, self).__init__() - self.l1 = nn.Linear(state_dim, 256) + self.l1 = nn.Linear(input_dim, 256) self.l2 = nn.Linear(256, 256) - self.l3 = nn.Linear(256, action_dim) - + self.l3 = nn.Linear(256, output_dim) self.max_action = max_action - - + def forward(self, state): + a = F.relu(self.l1(state)) a = F.relu(self.l2(a)) return self.max_action * torch.tanh(self.l3(a)) class Critic(nn.Module): - def __init__(self, state_dim, action_dim): + def __init__(self, input_dim, output_dim): super(Critic, self).__init__() # Q1 architecture - self.l1 = nn.Linear(state_dim + action_dim, 256) + self.l1 = nn.Linear(input_dim + output_dim, 256) self.l2 = nn.Linear(256, 256) self.l3 = nn.Linear(256, 1) # Q2 architecture - self.l4 = nn.Linear(state_dim + action_dim, 256) + self.l4 = nn.Linear(input_dim + output_dim, 256) self.l5 = nn.Linear(256, 256) self.l6 = nn.Linear(256, 1) @@ -68,8 +79,8 @@ class Critic(nn.Module): class TD3(object): def __init__( self, - state_dim, - action_dim, + input_dim, + output_dim, max_action, cfg, ): @@ -83,14 +94,14 @@ class TD3(object): self.device = cfg.device self.total_it = 0 - self.actor = Actor(state_dim, action_dim, max_action).to(self.device) + self.actor = Actor(input_dim, output_dim, max_action).to(self.device) self.actor_target = copy.deepcopy(self.actor) self.actor_optimizer = torch.optim.Adam(self.actor.parameters(), lr=3e-4) - self.critic = Critic(state_dim, action_dim).to(self.device) + self.critic = Critic(input_dim, output_dim).to(self.device) self.critic_target = copy.deepcopy(self.critic) self.critic_optimizer = torch.optim.Adam(self.critic.parameters(), lr=3e-4) - self.memory = ReplayBuffer(state_dim, action_dim) + self.memory = ReplayBuffer(input_dim, output_dim) def choose_action(self, state): state = torch.FloatTensor(state.reshape(1, -1)).to(self.device)