diff --git a/codes/DQN/agent.py b/codes/DQN/agent.py index 514709a..e0cb31c 100644 --- a/codes/DQN/agent.py +++ b/codes/DQN/agent.py @@ -5,7 +5,7 @@ @Email: johnjim0816@gmail.com @Date: 2020-06-12 00:50:49 @LastEditor: John -LastEditTime: 2021-05-07 16:30:05 +LastEditTime: 2021-09-15 02:18:56 @Discription: @Environment: python 3.7.7 ''' @@ -37,18 +37,20 @@ class DQN: self.batch_size = cfg.batch_size self.policy_net = MLP(state_dim, action_dim,hidden_dim=cfg.hidden_dim).to(self.device) self.target_net = MLP(state_dim, action_dim,hidden_dim=cfg.hidden_dim).to(self.device) - for target_param, param in zip(self.target_net.parameters(),self.policy_net.parameters()): # copy params from policy net + for target_param, param in zip(self.target_net.parameters(),self.policy_net.parameters()): # 复制参数到目标网路targe_net target_param.data.copy_(param.data) - self.optimizer = optim.Adam(self.policy_net.parameters(), lr=cfg.lr) + self.optimizer = optim.Adam(self.policy_net.parameters(), lr=cfg.lr) # 优化器 self.memory = ReplayBuffer(cfg.memory_capacity) - def choose_action(self, state): '''选择动作 ''' self.frame_idx += 1 if random.random() > self.epsilon(self.frame_idx): - action = self.predict(state) + with torch.no_grad(): + state = torch.tensor([state], device=self.device, dtype=torch.float32) + q_values = self.policy_net(state) + action = q_values.max(1)[1].item() else: action = random.randrange(self.action_dim) return action diff --git a/codes/DQN/outputs/CartPole-v0/20210429-222344/models/dqn_checkpoint.pth b/codes/DQN/outputs/CartPole-v0/20210429-222344/models/dqn_checkpoint.pth deleted file mode 100644 index 0b192d4..0000000 Binary files a/codes/DQN/outputs/CartPole-v0/20210429-222344/models/dqn_checkpoint.pth and /dev/null differ diff --git a/codes/DQN/outputs/CartPole-v0/20210429-222344/results/eval_ma_rewards.npy b/codes/DQN/outputs/CartPole-v0/20210429-222344/results/eval_ma_rewards.npy deleted file mode 100644 index f91ed3c..0000000 Binary files a/codes/DQN/outputs/CartPole-v0/20210429-222344/results/eval_ma_rewards.npy and /dev/null differ diff --git a/codes/DQN/outputs/CartPole-v0/20210429-222344/results/eval_rewards.npy b/codes/DQN/outputs/CartPole-v0/20210429-222344/results/eval_rewards.npy deleted file mode 100644 index 51e06c2..0000000 Binary files a/codes/DQN/outputs/CartPole-v0/20210429-222344/results/eval_rewards.npy and /dev/null differ diff --git a/codes/DQN/outputs/CartPole-v0/20210429-222344/results/eval_rewards_curve.png b/codes/DQN/outputs/CartPole-v0/20210429-222344/results/eval_rewards_curve.png deleted file mode 100644 index 0327b47..0000000 Binary files a/codes/DQN/outputs/CartPole-v0/20210429-222344/results/eval_rewards_curve.png and /dev/null differ diff --git a/codes/DQN/outputs/CartPole-v0/20210429-222344/results/train_ma_rewards.npy b/codes/DQN/outputs/CartPole-v0/20210429-222344/results/train_ma_rewards.npy deleted file mode 100644 index fadc523..0000000 Binary files a/codes/DQN/outputs/CartPole-v0/20210429-222344/results/train_ma_rewards.npy and /dev/null differ diff --git a/codes/DQN/outputs/CartPole-v0/20210429-222344/results/train_rewards.npy b/codes/DQN/outputs/CartPole-v0/20210429-222344/results/train_rewards.npy deleted file mode 100644 index 61cf9fc..0000000 Binary files a/codes/DQN/outputs/CartPole-v0/20210429-222344/results/train_rewards.npy and /dev/null differ diff --git a/codes/DQN/outputs/CartPole-v0/20210429-222344/results/train_rewards_curve.png b/codes/DQN/outputs/CartPole-v0/20210429-222344/results/train_rewards_curve.png deleted file mode 100644 index b9667f1..0000000 Binary files a/codes/DQN/outputs/CartPole-v0/20210429-222344/results/train_rewards_curve.png and /dev/null differ diff --git a/codes/DQN/outputs/CartPole-v0/20210504-190229/models/dqn_checkpoint.pth b/codes/DQN/outputs/CartPole-v0/20210504-190229/models/dqn_checkpoint.pth deleted file mode 100644 index 76ec7d6..0000000 Binary files a/codes/DQN/outputs/CartPole-v0/20210504-190229/models/dqn_checkpoint.pth and /dev/null differ diff --git a/codes/DQN/outputs/CartPole-v0/20210504-190229/results/eval_ma_rewards.npy b/codes/DQN/outputs/CartPole-v0/20210504-190229/results/eval_ma_rewards.npy deleted file mode 100644 index f51fdef..0000000 Binary files a/codes/DQN/outputs/CartPole-v0/20210504-190229/results/eval_ma_rewards.npy and /dev/null differ diff --git a/codes/DQN/outputs/CartPole-v0/20210504-190229/results/eval_rewards.npy b/codes/DQN/outputs/CartPole-v0/20210504-190229/results/eval_rewards.npy deleted file mode 100644 index 6533c74..0000000 Binary files a/codes/DQN/outputs/CartPole-v0/20210504-190229/results/eval_rewards.npy and /dev/null differ diff --git a/codes/DQN/outputs/CartPole-v0/20210504-190229/results/eval_rewards_curve.png b/codes/DQN/outputs/CartPole-v0/20210504-190229/results/eval_rewards_curve.png deleted file mode 100644 index dd3e728..0000000 Binary files a/codes/DQN/outputs/CartPole-v0/20210504-190229/results/eval_rewards_curve.png and /dev/null differ diff --git a/codes/DQN/outputs/CartPole-v0/20210504-190229/results/train_ma_rewards.npy b/codes/DQN/outputs/CartPole-v0/20210504-190229/results/train_ma_rewards.npy deleted file mode 100644 index 25efa3d..0000000 Binary files a/codes/DQN/outputs/CartPole-v0/20210504-190229/results/train_ma_rewards.npy and /dev/null differ diff --git a/codes/DQN/outputs/CartPole-v0/20210504-190229/results/train_rewards.npy b/codes/DQN/outputs/CartPole-v0/20210504-190229/results/train_rewards.npy deleted file mode 100644 index c45be6e..0000000 Binary files a/codes/DQN/outputs/CartPole-v0/20210504-190229/results/train_rewards.npy and /dev/null differ diff --git a/codes/DQN/outputs/CartPole-v0/20210504-190229/results/train_rewards_curve.png b/codes/DQN/outputs/CartPole-v0/20210504-190229/results/train_rewards_curve.png deleted file mode 100644 index 4a18b49..0000000 Binary files a/codes/DQN/outputs/CartPole-v0/20210504-190229/results/train_rewards_curve.png and /dev/null differ diff --git a/codes/DQN/outputs/CartPole-v0/20210912-013122/models/dqn_checkpoint.pth b/codes/DQN/outputs/CartPole-v0/20210912-013122/models/dqn_checkpoint.pth new file mode 100644 index 0000000..7cb626f Binary files /dev/null and b/codes/DQN/outputs/CartPole-v0/20210912-013122/models/dqn_checkpoint.pth differ diff --git a/codes/DQN/outputs/CartPole-v0/20210912-013122/results/eval_ma_rewards.npy b/codes/DQN/outputs/CartPole-v0/20210912-013122/results/eval_ma_rewards.npy new file mode 100644 index 0000000..516efb9 Binary files /dev/null and b/codes/DQN/outputs/CartPole-v0/20210912-013122/results/eval_ma_rewards.npy differ diff --git a/codes/DQN/outputs/CartPole-v0/20210912-013122/results/eval_rewards.npy b/codes/DQN/outputs/CartPole-v0/20210912-013122/results/eval_rewards.npy new file mode 100644 index 0000000..6498864 Binary files /dev/null and b/codes/DQN/outputs/CartPole-v0/20210912-013122/results/eval_rewards.npy differ diff --git a/codes/DQN/outputs/CartPole-v0/20210912-013122/results/eval_rewards_curve.png b/codes/DQN/outputs/CartPole-v0/20210912-013122/results/eval_rewards_curve.png new file mode 100644 index 0000000..d304a19 Binary files /dev/null and b/codes/DQN/outputs/CartPole-v0/20210912-013122/results/eval_rewards_curve.png differ diff --git a/codes/DQN/outputs/CartPole-v0/20210912-013122/results/train_ma_rewards.npy b/codes/DQN/outputs/CartPole-v0/20210912-013122/results/train_ma_rewards.npy new file mode 100644 index 0000000..1d8f61b Binary files /dev/null and b/codes/DQN/outputs/CartPole-v0/20210912-013122/results/train_ma_rewards.npy differ diff --git a/codes/DQN/outputs/CartPole-v0/20210912-013122/results/train_rewards.npy b/codes/DQN/outputs/CartPole-v0/20210912-013122/results/train_rewards.npy new file mode 100644 index 0000000..32dbedf Binary files /dev/null and b/codes/DQN/outputs/CartPole-v0/20210912-013122/results/train_rewards.npy differ diff --git a/codes/DQN/outputs/CartPole-v0/20210912-013122/results/train_rewards_curve.png b/codes/DQN/outputs/CartPole-v0/20210912-013122/results/train_rewards_curve.png new file mode 100644 index 0000000..147540c Binary files /dev/null and b/codes/DQN/outputs/CartPole-v0/20210912-013122/results/train_rewards_curve.png differ diff --git a/codes/DQN/task0_train.py b/codes/DQN/task0_train.py index e6f5e4a..0d723b4 100644 --- a/codes/DQN/task0_train.py +++ b/codes/DQN/task0_train.py @@ -5,14 +5,14 @@ @Email: johnjim0816@gmail.com @Date: 2020-06-12 00:48:57 @LastEditor: John -LastEditTime: 2021-05-05 16:49:15 +LastEditTime: 2021-09-15 02:19:54 @Discription: @Environment: python 3.7.7 ''' import sys,os -curr_path = os.path.dirname(__file__) -parent_path = os.path.dirname(curr_path) -sys.path.append(parent_path) # add current terminal path to sys.path +curr_path = os.path.dirname(os.path.abspath(__file__)) # 当前文件所在绝对路径 +parent_path = os.path.dirname(curr_path) # 父路径 +sys.path.append(parent_path) # 添加父路径到系统路径sys.path import gym import torch @@ -22,42 +22,41 @@ from common.utils import save_results, make_dir from common.plot import plot_rewards from DQN.agent import DQN -curr_time = datetime.datetime.now().strftime( - "%Y%m%d-%H%M%S") # obtain current time +curr_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") # 获取当前时间 class DQNConfig: def __init__(self): - self.algo = "DQN" # name of algo - self.env = 'CartPole-v0' + self.algo = "DQN" # 算法名称 + self.env = 'CartPole-v0' # 环境名称 self.result_path = curr_path+"/outputs/" + self.env + \ '/'+curr_time+'/results/' # path to save results self.model_path = curr_path+"/outputs/" + self.env + \ '/'+curr_time+'/models/' # path to save models - self.train_eps = 300 # max trainng episodes - self.eval_eps = 50 # number of episodes for evaluating - self.gamma = 0.95 - self.epsilon_start = 0.90 # start epsilon of e-greedy policy - self.epsilon_end = 0.01 - self.epsilon_decay = 500 - self.lr = 0.0001 # learning rate + self.train_eps = 200 # 训练的回合数 + self.eval_eps = 30 # 测试的回合数 + self.gamma = 0.95 + self.epsilon_start = 0.90 # e-greedy策略中初始epsilon + self.epsilon_end = 0.01 # e-greedy策略中的终止epsilon + self.epsilon_decay = 500 # e-greedy策略中epsilon的衰减率 + self.lr = 0.0001 # 学习率 self.memory_capacity = 100000 # capacity of Replay Memory self.batch_size = 64 - self.target_update = 4 # update frequency of target net + self.target_update = 4 # 目标网络的更新频率 self.device = torch.device( - "cuda" if torch.cuda.is_available() else "cpu") # check gpu + "cuda" if torch.cuda.is_available() else "cpu") # jian che self.hidden_dim = 256 # hidden size of net def env_agent_config(cfg,seed=1): env = gym.make(cfg.env) env.seed(seed) - state_dim = env.observation_space.shape[0] - action_dim = env.action_space.n - agent = DQN(state_dim,action_dim,cfg) + n_states = env.observation_space.shape[0] + n_actions = env.action_space.n + agent = DQN(n_states,n_actions,cfg) return env,agent def train(cfg, env, agent): print('Start to train !') - print(f'Env:{cfg.env}, Algorithm:{cfg.algo}, Device:{cfg.device}') + print(f'Env: {cfg.env}, Algorithm: {cfg.algo}, Device: {cfg.device}') rewards = [] ma_rewards = [] # moveing average reward for i_ep in range(cfg.train_eps): @@ -78,7 +77,7 @@ def train(cfg, env, agent): if (i_ep+1)%10 == 0: print('Episode:{}/{}, Reward:{}'.format(i_ep+1, cfg.train_eps, ep_reward)) rewards.append(ep_reward) - # save ma rewards + # save ma_rewards if ma_rewards: ma_rewards.append(0.9*ma_rewards[-1]+0.1*ep_reward) else: @@ -88,7 +87,7 @@ def train(cfg, env, agent): def eval(cfg,env,agent): print('Start to eval !') - print(f'Env:{cfg.env}, Algorithm:{cfg.algo}, Device:{cfg.device}') + print(f'Env: {cfg.env}, Algorithm: {cfg.algo}, Device: {cfg.device}') rewards = [] ma_rewards = [] # moving average rewards for i_ep in range(cfg.eval_eps): @@ -106,8 +105,7 @@ def eval(cfg,env,agent): ma_rewards.append(ma_rewards[-1]*0.9+ep_reward*0.1) else: ma_rewards.append(ep_reward) - if (i_ep+1)%10 == 10: - print(f"Episode:{i_ep+1}/{cfg.eval_eps}, reward:{ep_reward:.1f}") + print(f"Episode:{i_ep+1}/{cfg.eval_eps}, reward:{ep_reward:.1f}") print('Complete evaling！') return rewards,ma_rewards diff --git a/codes/Docs/assets/Qlearning_1.png b/codes/Docs/assets/Qlearning_1.png new file mode 100644 index 0000000..08e3bbc Binary files /dev/null and b/codes/Docs/assets/Qlearning_1.png differ diff --git a/codes/Docs/assets/cliffwalking_1.png b/codes/Docs/assets/cliffwalking_1.png new file mode 100644 index 0000000..ae5b0f8 Binary files /dev/null and b/codes/Docs/assets/cliffwalking_1.png differ diff --git a/codes/Docs/assets/eval_rewards_curve_cn.png b/codes/Docs/assets/eval_rewards_curve_cn.png new file mode 100644 index 0000000..766251e Binary files /dev/null and b/codes/Docs/assets/eval_rewards_curve_cn.png differ diff --git a/codes/Docs/assets/image-20210915020027615.png b/codes/Docs/assets/image-20210915020027615.png new file mode 100644 index 0000000..f27c3e8 Binary files /dev/null and b/codes/Docs/assets/image-20210915020027615.png differ diff --git a/codes/Docs/assets/poster.jpg b/codes/Docs/assets/poster.jpg new file mode 100644 index 0000000..139a48e Binary files /dev/null and b/codes/Docs/assets/poster.jpg differ diff --git a/codes/Docs/assets/train_rewards_curve_cn.png b/codes/Docs/assets/train_rewards_curve_cn.png new file mode 100644 index 0000000..e9f4b83 Binary files /dev/null and b/codes/Docs/assets/train_rewards_curve_cn.png differ diff --git a/codes/Docs/使用DQN解决推车杆问题.md b/codes/Docs/使用DQN解决推车杆问题.md new file mode 100644 index 0000000..a20b05e --- /dev/null +++ b/codes/Docs/使用DQN解决推车杆问题.md @@ -0,0 +1,66 @@ + + +在练习本项目之前，可以先回顾一下之前的项目实战，即使用Q学习解决悬崖寻路问题。本项目将具体实现DQN算法来解决推车杆问题，对应的模拟环境为Open AI Gym中的[CartPole-v0](https://datawhalechina.github.io/easy-rl/#/chapter7/project2?id=cartpole-v0)，我们同样先对该环境做一个简要说明。 + +## CartPole-v0环境简介 + +CartPole-v0是一个经典的入门环境，如下图，它通过向左(动作=0)或向右(动作=1)推动推车来实现竖直杆的平衡，每次实施一个动作后如果能够继续保持平衡就会得到一个+1的奖励，否则杆将无法保持平衡而导致游戏结束。 + +![Gym](assets/poster.jpg) + +我们来看看这个环境的一些参数，执行以下代码： + +```python +import gym +env = gym.make('CartPole-v0') # 建立环境 +env.seed(1) # 随机种子 +n_states = env.observation_space.shape[0] # 状态数 +n_actions = env.action_space.n # 动作数 +state = env.reset() # 初始化环境 +print(f"状态数：{n_states}，动作数：{n_actions}") +print(f"初始状态：{state}") +``` + +可以得到结果： + +```bash +状态数：4，动作数：2 +初始状态：[ 0.03073904 0.00145001 -0.03088818 -0.03131252] +``` + +该环境状态数是四个，分别为车的位置、车的速度、杆的角度以及杆顶部的速度，动作数为两个，并且是离散的向左或者向右。 + +## DQN基本接口 + +介绍完环境之后，我们沿用接口的概念，通过分析伪代码来实现DQN的基本训练模式，以及一些要素比如建立什么网络需要什么模块等等。我们现在常用的DQN伪代码如下： + +![image-20210915020027615](assets/image-20210915020027615.png) + +与传统的Q学习算法相比，DQN使用神经网络来代替之前的Q表格从而存储更多的信息，且由于使用了神经网络所以我们一般需要利用随机梯度下降来优化Q值的预测。此外多了经验回放缓冲区(replay memory)，并且使用两个网络，即目标网络和当前网络。 + +## 经验回放缓冲区 + +从伪代码中可以看出来，经验回放缓冲区的功能有两个，一个是将每一步采集的转移(transition，包括状态，动作，奖励，下一时刻的状态)存储到缓冲区中，并且缓冲区具备一定的容量(capacity)，另一个是在更新策略的时候需要随机采样小批量的转移进行优化。因此我们可以定义一个ReplayBuffer类，包括push和sample两个函数，用于存储和采样。 + +```python +import random +class ReplayBuffer: + def __init__(self, capacity): + self.capacity = capacity # 经验回放的容量 + self.buffer = [] # 缓冲区 + self.position = 0 + + def push(self, state, action, reward, next_state, done): + ''' 缓冲区是一个队列，容量超出时去掉开始存入的转移(transition) + ''' + if len(self.buffer) < self.capacity: + self.buffer.append(None) + self.buffer[self.position] = (state, action, reward, next_state, done) + self.position = (self.position + 1) % self.capacity + + def sample(self, batch_size): + batch = random.sample(self.buffer, batch_size) # 随机采出小批量转移 + state, action, reward, next_state, done = zip(*batch) # 解压成状态，动作等 + return state, action, reward, next_state, done +``` + diff --git a/codes/Docs/使用Q-learning解决悬崖寻路问题.md b/codes/Docs/使用Q-learning解决悬崖寻路问题.md new file mode 100644 index 0000000..244d85b --- /dev/null +++ b/codes/Docs/使用Q-learning解决悬崖寻路问题.md @@ -0,0 +1,165 @@ +# 使用Q学习解决悬崖寻路问题 + +强化学习在运动规划方面也有很大的应用前景，已有很多适用于强化学习的相关仿真环境，小到迷宫，大到贴近真实的自动驾驶环境[CARLA](http://carla.org/)。本次使用[OpenAI Gym](https://gym.openai.com/)开发的CliffWalking-v0环境，带大家入门Q学习算法的代码实战。 + +## CliffWalking-v0环境简介 + +我们首先简单介绍一下这个环境，该环境中文名叫悬崖寻路（CliffWalking），是一个迷宫类问题。如下图，在一个4 x 12的网格中，智能体以网格的左下角位置为起点，以网格的下角位置为终点，目标是移动智能体到达终点位置，智能体每次可以在上、下、左、右这4个方向中移动一步，每移动一步会得到-1单位的奖励。 + +
+cliffwalking_1 +
+起终点之间是一段悬崖，即编号为37～46的网格，智能体移动过程中会有如下的限制： + +* 智能体不能移出网格边界，如果智能体想执行某个动作移出网格，那么这一步智能体不会移动，但是这个操作依然会得到-1单位的奖励 +* 如果智能体“掉入悬崖” ，会立即回到起点位置，并得到-100单位的奖励 +* 当智能体移动到终点时，该回合结束，该回合总奖励为各步奖励之和 + +我们的目标是以最少的步数到达终点，容易看出最少需要13步智能体才能从起点到终点，因此最佳算法收敛的情况下，每回合的总奖励应该是-13，这样人工分析出期望的奖励也便于我们判断算法的收敛情况作出相应调整。 + +现在我们可以在代码中定义环境，如下： + +```python +import gym # 导入gym模块 +from envs.gridworld_env import CliffWalkingWapper # 导入自定义装饰器 + +env = gym.make('CliffWalking-v0') # 定义环境 +env = CliffWalkingWapper(env) # 装饰环境 +``` + +这里我们在程序中使用了一个装饰器重新定义环境，但不影响对环境的理解，感兴趣的同学具体看相关代码。可以由于gym环境封装得比较好，所以我们想要使用这个环境只需要使用gym.make命令输入函数名即可，然后我们可以查看环境的状态和动作数目： + +```python +n_states = env.observation_space.n # 状态数 +n_actions = env.action_space.n # 动作数 +print(f"状态数：{n_states}，动作数：{n_actions}") +``` + +打印出来的结果如下： + +```bash +状态数：48，动作数：4 +``` + +我们的状态数是48个，这里我们设置的是智能体当前所在网格的编号，而动作数是4，这表示有0，1，2，3对应着上下左右四个动作。另外我们也可以初始化环境并打印当前所在的状态： + +```python +state = env.reset() +print(state) +``` + +结果显示为： + +```bash +36 +``` + +也就是说当前智能体的状态即当前所在的网格编号是36，正好对应我们前面讲到的起点。 + +## 强化学习基本接口 + +这里所说的接口就是一般强化学习的训练模式，也是大多数算法伪代码遵循的套路，步骤如下： + +* 初始化环境和智能体 +* 对于每个回合，智能体选择动作 +* 环境接收动作反馈下一个状态和奖励 +* 智能体进行策略更新(学习) +* 多个回合算法收敛之后保存模型以及做后续的分析画图等 + +代码如下： + +```python +env = gym.make('CliffWalking-v0') # 定义环境 +env = CliffWalkingWapper(env) # 装饰环境 +env.seed(1) # 设置随机种子 +n_states = env.observation_space.n # 状态数 +n_actions = env.action_space.n # 动作数 +agent = QLearning(n_states,n_actions,cfg) # cfg存储算法相关参数 +for i_ep in range(cfg.train_eps): # cfg.train_eps表示最大训练的回合数 + ep_reward = 0 # 记录每个回合的奖励 + state = env.reset() # 重置环境 + while True: + action = agent.choose_action(state) # 算法选择一个动作 + next_state, reward, done, _ = env.step(action) # 环境根据动作反馈奖励和下一个状态 + agent.update(state, action, reward, next_state, done) # 算法更新 + state = next_state # 更新状态 + ep_reward += reward + if done: # 终止状态提前停止 + break +``` + +通常我们会记录并分析奖励的变化，所以在接口基础上加一些变量记录每回合的奖励，此外由于强化学习学习过程得到的奖励可能会产生振荡，因此我们也适用一个滑动平均的量来反映奖励变化的趋势，如下： + +```bash +rewards = [] +ma_rewards = [] # 滑动平均奖励 +for i_ep in range(cfg.train_eps): + ep_reward = 0 # 记录每个回合的奖励 + state = env.reset() # 重置环境, 重新开一局（即开始新的一个回合） + while True: + action = agent.choose_action(state) # 根据算法选择一个动作 + next_state, reward, done, _ = env.step(action) # 与环境进行一次动作交互 + agent.update(state, action, reward, next_state, done) # Q-learning算法更新 + state = next_state # 存储上一个观察值 + ep_reward += reward + if done: + break +rewards.append(ep_reward) +if ma_rewards: + ma_rewards.append(ma_rewards[-1]*0.9+ep_reward*0.1) + else: + ma_rewards.append(ep_reward) +``` + +## Q学习算法 + +了解了基本接口之后，现在我们看看Q学习算法具体是怎么实现的，前面讲到智能体其实在整个训练中就做两件事，一个是选择动作，一个是更新策略，所以我们可以定义一个Qlearning类，里面主要包含两个函数choose_action和update。 + +我们先看看choose_action函数是怎么定义的，如下： + +```python +def choose_action(self, state): + self.sample_count += 1 + self.epsilon = self.epsilon_end + (self.epsilon_start - self.epsilon_end) + math.exp(-1. * self.sample_count / self.epsilon_decay) # epsilon是会递减的，这里选择指数递减 + # e-greedy 策略 + if np.random.uniform(0, 1) > self.epsilon: + action = np.argmax(self.Q_table[str(state)]) # 选择Q(s,a)最大对应的动作 + else: + action = np.random.choice(self.action_dim) # 随机选择动作 + return action +``` + + + +一般我们使用$\varepsilon-greedy$策略选择动作，我们的输入就是当前的状态，随机选取一个值，当这个值大于我们设置的$\varepsilon$时，我们选取Q值最大对应的动作，否则随机选择动作，这样就能在训练中让智能体保持一定的探索率，这也是平衡探索与利用的技巧之一。 + +下面是我们要实现的策略更新函数： + +```python +def update(self, state, action, reward, next_state, done): + Q_predict = self.Q_table[str(state)][action] + if done: # 终止状态 + Q_target = reward + else: + Q_target = reward + self.gamma * np.max(self.Q_table[str(next_state)]) + self.Q_table[str(state)][action] += self.lr * (Q_target - Q_predict) +``` + +这里面实现的逻辑就是伪代码中的更新公式： + +image-20210911213241605 + +注意终止状态下，我们是获取不到下一个动作的，我们直接将Q值（Q_target）更新为对应的奖励即可。 + +## 结果分析 + +到现在我们就基本完成了Q学习的代码实现，具体可以查看github上的源码，运行代码结果如下： + +![train_rewards_curve_cn](assets/train_rewards_curve_cn.png) + +由于这个环境比较简单，可以看到算法很快达到收敛，然后我们再测试我们训练好的模型，一般测试模型只需要20到50左右的回合数即可： + +![eval_rewards_curve_cn](assets/eval_rewards_curve_cn.png) + +这里我们测试的回合数为30，可以看到每个回合智能体都达到了最优的奖励，说明我们的算法训练的效果很不错！ diff --git a/codes/DoubleDQN/task0_train.py b/codes/DoubleDQN/task0_train.py index 8d0f842..0148ea2 100644 --- a/codes/DoubleDQN/task0_train.py +++ b/codes/DoubleDQN/task0_train.py @@ -5,7 +5,7 @@ @Email: johnjim0816@gmail.com @Date: 2020-06-12 00:48:57 @LastEditor: John -LastEditTime: 2021-05-04 22:26:59 +LastEditTime: 2021-09-10 15:26:05 @Discription: @Environment: python 3.7.7 ''' @@ -106,7 +106,7 @@ def eval(cfg,env,agent): if __name__ == "__main__": cfg = DoubleDQNConfig() - # train + # 训练 env,agent = env_agent_config(cfg,seed=1) rewards, ma_rewards = train(cfg, env, agent) make_dir(cfg.result_path, cfg.model_path) @@ -115,7 +115,7 @@ if __name__ == "__main__": plot_rewards(rewards, ma_rewards, tag="train", algo=cfg.algo, path=cfg.result_path) - # eval + # 测试 env,agent = env_agent_config(cfg,seed=10) agent.load(path=cfg.model_path) rewards,ma_rewards = eval(cfg,env,agent) diff --git a/codes/MonteCarlo/task0_train.py b/codes/MonteCarlo/task0_train.py index a0a3f32..dae0c95 100644 --- a/codes/MonteCarlo/task0_train.py +++ b/codes/MonteCarlo/task0_train.py @@ -5,7 +5,7 @@ Author: John Email: johnjim0816@gmail.com Date: 2021-03-11 14:26:44 LastEditor: John -LastEditTime: 2021-07-14 17:25:16 +LastEditTime: 2021-05-05 17:27:50 Discription: Environment: ''' @@ -48,7 +48,7 @@ def env_agent_config(cfg,seed=1): return env,agent def train(cfg, env, agent): - print('Start to training !') + print('Start to eval !') print(f'Env:{cfg.env}, Algorithm:{cfg.algo}, Device:{cfg.device}') rewards = [] ma_rewards = [] # moving average rewards @@ -102,7 +102,7 @@ def eval(cfg, env, agent): if __name__ == "__main__": cfg = MCConfig() - ''' train ''' + # train env,agent = env_agent_config(cfg,seed=1) rewards, ma_rewards = train(cfg, env, agent) make_dir(cfg.result_path, cfg.model_path) @@ -110,7 +110,7 @@ if __name__ == "__main__": save_results(rewards, ma_rewards, tag='train', path=cfg.result_path) plot_rewards(rewards, ma_rewards, tag="train", algo=cfg.algo, path=cfg.result_path) - ''' eval ''' + # eval env,agent = env_agent_config(cfg,seed=10) agent.load(path=cfg.model_path) rewards,ma_rewards = eval(cfg,env,agent) diff --git a/codes/PPO/README.md b/codes/PPO/README.md index 90136c6..66825c9 100644 --- a/codes/PPO/README.md +++ b/codes/PPO/README.md @@ -1,5 +1,6 @@ ## 原理简介 -PPO是一种off-policy算法，具有较好的性能，其前身是TRPO算法，也是policy gradient算法的一种，它是现在 OpenAI 默认的强化学习算法，具体原理可参考[PPO算法讲解](https://datawhalechina.github.io/easy-rl/#/chapter5/chapter5)。PPO算法主要有两个变种，一个是结合KL penalty的，一个是用了clip方法，本文实现的是后者即```PPO-clip```。 + +PPO是一种on-policy算法，具有较好的性能，其前身是TRPO算法，也是policy gradient算法的一种，它是现在 OpenAI 默认的强化学习算法，具体原理可参考[PPO算法讲解](https://datawhalechina.github.io/easy-rl/#/chapter5/chapter5)。PPO算法主要有两个变种，一个是结合KL penalty的，一个是用了clip方法，本文实现的是后者即```PPO-clip```。 ## 伪代码 要实现必先了解伪代码，伪代码如下： ![在这里插入图片描述](assets/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L0pvaG5KaW0w,size_16,color_FFFFFF,t_70.png) diff --git a/codes/QLearning/README.md b/codes/QLearning/README.md deleted file mode 100644 index cc8ef5e..0000000 --- a/codes/QLearning/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# Q-learning - -#TODO diff --git a/codes/QLearning/agent.py b/codes/QLearning/agent.py index 3a4eadb..4dff2a9 100644 --- a/codes/QLearning/agent.py +++ b/codes/QLearning/agent.py @@ -5,7 +5,7 @@ Author: John Email: johnjim0816@gmail.com Date: 2020-09-11 23:03:00 LastEditor: John -LastEditTime: 2021-04-29 16:59:41 +LastEditTime: 2021-09-11 21:53:18 Discription: use defaultdict to define Q table Environment: ''' @@ -30,20 +30,20 @@ class QLearning(object): def choose_action(self, state): self.sample_count += 1 self.epsilon = self.epsilon_end + (self.epsilon_start - self.epsilon_end) * \ - math.exp(-1. * self.sample_count / self.epsilon_decay) - # e-greedy policy + math.exp(-1. * self.sample_count / self.epsilon_decay) # epsilon是会递减的，这里选择指数递减 + # e-greedy 策略 if np.random.uniform(0, 1) > self.epsilon: - action = self.predict(state) + action = np.argmax(self.Q_table[str(state)]) # 选择Q(s,a)最大对应的动作 else: - action = np.random.choice(self.action_dim) + action = np.random.choice(self.action_dim) # 随机选择动作 return action def predict(self,state): action = np.argmax(self.Q_table[str(state)]) return action def update(self, state, action, reward, next_state, done): - Q_predict = self.Q_table[str(state)][action] - if done: - Q_target = reward # terminal state + Q_predict = self.Q_table[str(state)][action] + if done: # 终止状态 + Q_target = reward else: Q_target = reward + self.gamma * np.max(self.Q_table[str(next_state)]) self.Q_table[str(state)][action] += self.lr * (Q_target - Q_predict) @@ -54,6 +54,8 @@ class QLearning(object): f=path+"Qleaning_model.pkl", pickle_module=dill ) + print("保存模型成功！") def load(self, path): import dill - self.Q_table =torch.load(f=path+'Qleaning_model.pkl',pickle_module=dill) \ No newline at end of file + self.Q_table =torch.load(f=path+'Qleaning_model.pkl',pickle_module=dill) + print("加载模型成功！") \ No newline at end of file diff --git a/codes/QLearning/agent1.py b/codes/QLearning/agent1.py deleted file mode 100644 index 4f025ac..0000000 --- a/codes/QLearning/agent1.py +++ /dev/null @@ -1,88 +0,0 @@ -#!/usr/bin/env python -# coding=utf-8 -''' -Author: John -Email: johnjim0816@gmail.com -Date: 2020-09-11 23:03:00 -LastEditor: John -LastEditTime: 2021-04-29 17:02:00 -Discription: -Environment: -''' -# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import numpy as np -import math -#!/usr/bin/env python -# coding=utf-8 -''' -Author: John -Email: johnjim0816@gmail.com -Date: 2020-09-11 23:03:00 -LastEditor: John -LastEditTime: 2021-04-29 16:45:33 -Discription: use np array to define Q table -Environment: -''' -import numpy as np -import math - -class QLearning(object): - def __init__(self, - state_dim,action_dim,cfg): - self.action_dim = action_dim # dimension of acgtion - self.lr = cfg.lr # learning rate - self.gamma = cfg.gamma - self.epsilon = 0 - self.sample_count = 0 - self.epsilon_start = cfg.epsilon_start - self.epsilon_end = cfg.epsilon_end - self.epsilon_decay = cfg.epsilon_decay - self.Q_table = np.zeros((state_dim, action_dim)) # Q表 - - def choose_action(self, state): - self.sample_count += 1 - self.epsilon = self.epsilon_end + (self.epsilon_start - self.epsilon_end) * \ - math.exp(-1. * self.sample_count / self.epsilon_decay) - if np.random.uniform(0, 1) > self.epsilon: # 随机选取0-1之间的值，如果大于epsilon就按照贪心策略选取action，否则随机选取 - action = self.predict(state) - else: - action = np.random.choice(self.action_dim) #有一定概率随机探索选取一个动作 - return action - - def predict(self, state): - '''根据输入观测值，采样输出的动作值，带探索，测试模型时使用 - ''' - Q_list = self.Q_table[state, :] - Q_max = np.max(Q_list) - action_list = np.where(Q_list == Q_max)[0] - action = np.random.choice(action_list) # Q_max可能对应多个 action ，可以随机抽取一个 - return action - - def update(self, state, action, reward, next_state, done): - Q_predict = self.Q_table[state, action] - if done: - Q_target = reward # 没有下一个状态了 - else: - Q_target = reward + self.gamma * np.max( - self.Q_table[next_state, :]) # Q_table-learning - self.Q_table[state, action] += self.lr * (Q_target - Q_predict) # 修正q - def save(self,path): - np.save(path+"Q_table.npy", self.Q_table) - def load(self, path): - self.Q_table = np.load(path+"Q_table.npy") - - diff --git a/codes/QLearning/outputs/CliffWalking-v0/20210429-165825/models/Qleaning_model.pkl b/codes/QLearning/outputs/CliffWalking-v0/20210429-165825/models/Qleaning_model.pkl deleted file mode 100644 index cd4f55b..0000000 Binary files a/codes/QLearning/outputs/CliffWalking-v0/20210429-165825/models/Qleaning_model.pkl and /dev/null differ diff --git a/codes/QLearning/outputs/CliffWalking-v0/20210429-165825/results/eval_ma_rewards.npy b/codes/QLearning/outputs/CliffWalking-v0/20210429-165825/results/eval_ma_rewards.npy deleted file mode 100644 index a67d064..0000000 Binary files a/codes/QLearning/outputs/CliffWalking-v0/20210429-165825/results/eval_ma_rewards.npy and /dev/null differ diff --git a/codes/QLearning/outputs/CliffWalking-v0/20210429-165825/results/eval_rewards.npy b/codes/QLearning/outputs/CliffWalking-v0/20210429-165825/results/eval_rewards.npy deleted file mode 100644 index 6de67e1..0000000 Binary files a/codes/QLearning/outputs/CliffWalking-v0/20210429-165825/results/eval_rewards.npy and /dev/null differ diff --git a/codes/QLearning/outputs/CliffWalking-v0/20210429-165825/results/eval_rewards_curve.png b/codes/QLearning/outputs/CliffWalking-v0/20210429-165825/results/eval_rewards_curve.png deleted file mode 100644 index 776685a..0000000 Binary files a/codes/QLearning/outputs/CliffWalking-v0/20210429-165825/results/eval_rewards_curve.png and /dev/null differ diff --git a/codes/QLearning/outputs/CliffWalking-v0/20210429-165825/results/train_ma_rewards.npy b/codes/QLearning/outputs/CliffWalking-v0/20210429-165825/results/train_ma_rewards.npy deleted file mode 100644 index c18ceaf..0000000 Binary files a/codes/QLearning/outputs/CliffWalking-v0/20210429-165825/results/train_ma_rewards.npy and /dev/null differ diff --git a/codes/QLearning/outputs/CliffWalking-v0/20210429-165825/results/train_rewards.npy b/codes/QLearning/outputs/CliffWalking-v0/20210429-165825/results/train_rewards.npy deleted file mode 100644 index fb83837..0000000 Binary files a/codes/QLearning/outputs/CliffWalking-v0/20210429-165825/results/train_rewards.npy and /dev/null differ diff --git a/codes/QLearning/outputs/CliffWalking-v0/20210429-165825/results/train_rewards_curve.png b/codes/QLearning/outputs/CliffWalking-v0/20210429-165825/results/train_rewards_curve.png deleted file mode 100644 index d737d7f..0000000 Binary files a/codes/QLearning/outputs/CliffWalking-v0/20210429-165825/results/train_rewards_curve.png and /dev/null differ diff --git a/codes/QLearning/outputs/CliffWalking-v0/20210429-170453/models/Qleaning_model.pkl b/codes/QLearning/outputs/CliffWalking-v0/20210429-170453/models/Qleaning_model.pkl deleted file mode 100644 index 5d963e7..0000000 Binary files a/codes/QLearning/outputs/CliffWalking-v0/20210429-170453/models/Qleaning_model.pkl and /dev/null differ diff --git a/codes/QLearning/outputs/CliffWalking-v0/20210429-170453/results/eval_ma_rewards.npy b/codes/QLearning/outputs/CliffWalking-v0/20210429-170453/results/eval_ma_rewards.npy deleted file mode 100644 index a67d064..0000000 Binary files a/codes/QLearning/outputs/CliffWalking-v0/20210429-170453/results/eval_ma_rewards.npy and /dev/null differ diff --git a/codes/QLearning/outputs/CliffWalking-v0/20210429-170453/results/eval_rewards.npy b/codes/QLearning/outputs/CliffWalking-v0/20210429-170453/results/eval_rewards.npy deleted file mode 100644 index 6de67e1..0000000 Binary files a/codes/QLearning/outputs/CliffWalking-v0/20210429-170453/results/eval_rewards.npy and /dev/null differ diff --git a/codes/QLearning/outputs/CliffWalking-v0/20210429-170453/results/eval_rewards_curve.png b/codes/QLearning/outputs/CliffWalking-v0/20210429-170453/results/eval_rewards_curve.png deleted file mode 100644 index 2992927..0000000 Binary files a/codes/QLearning/outputs/CliffWalking-v0/20210429-170453/results/eval_rewards_curve.png and /dev/null differ diff --git a/codes/QLearning/outputs/CliffWalking-v0/20210429-170453/results/train_ma_rewards.npy b/codes/QLearning/outputs/CliffWalking-v0/20210429-170453/results/train_ma_rewards.npy deleted file mode 100644 index 13d68d5..0000000 Binary files a/codes/QLearning/outputs/CliffWalking-v0/20210429-170453/results/train_ma_rewards.npy and /dev/null differ diff --git a/codes/QLearning/outputs/CliffWalking-v0/20210429-170453/results/train_rewards.npy b/codes/QLearning/outputs/CliffWalking-v0/20210429-170453/results/train_rewards.npy deleted file mode 100644 index c504109..0000000 Binary files a/codes/QLearning/outputs/CliffWalking-v0/20210429-170453/results/train_rewards.npy and /dev/null differ diff --git a/codes/QLearning/outputs/CliffWalking-v0/20210429-170453/results/train_rewards_curve.png b/codes/QLearning/outputs/CliffWalking-v0/20210429-170453/results/train_rewards_curve.png deleted file mode 100644 index 9f0ed62..0000000 Binary files a/codes/QLearning/outputs/CliffWalking-v0/20210429-170453/results/train_rewards_curve.png and /dev/null differ diff --git a/codes/QLearning/task0_eval.py b/codes/QLearning/task0_eval.py deleted file mode 100644 index 851cfe6..0000000 --- a/codes/QLearning/task0_eval.py +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/env python -# coding=utf-8 -''' -Author: John -Email: johnjim0816@gmail.com -Date: 2020-09-11 23:03:00 -LastEditor: John -LastEditTime: 2021-04-29 17:01:43 -Discription: -Environment: -''' -import sys,os -curr_path = os.path.dirname(__file__) -parent_path=os.path.dirname(curr_path) -sys.path.append(parent_path) # add current terminal path to sys.path - -import gym -import datetime - -from envs.gridworld_env import CliffWalkingWapper -from QLearning.agent import QLearning -from common.plot import plot_rewards -from common.utils import save_results - -curr_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") # obtain current time - -class QlearningConfig: - '''训练相关参数''' - def __init__(self): - self.algo = 'Qlearning' - self.env = 'CliffWalking-v0' # 0 up, 1 right, 2 down, 3 left - self.result_path = curr_path+"/outputs/" +self.env+'/'+curr_time+'/results/' # path to save results - self.model_path = curr_path+"/outputs/" +self.env+'/'+curr_time+'/models/' # path to save models - self.train_eps = 300 # 训练的episode数目 - self.eval_eps = 30 - self.gamma = 0.9 # reward的衰减率 - self.epsilon_start = 0.95 # e-greedy策略中初始epsilon - self.epsilon_end = 0.01 # e-greedy策略中的终止epsilon - self.epsilon_decay = 200 # e-greedy策略中epsilon的衰减率 - self.lr = 0.1 # learning rate - -def env_agent_config(cfg,seed=1): - env = gym.make(cfg.env) - env = CliffWalkingWapper(env) - env.seed(seed) - state_dim = env.observation_space.n - action_dim = env.action_space.n - agent = QLearning(state_dim,action_dim,cfg) - return env,agent - -def eval(cfg,env,agent): - # env = gym.make("FrozenLake-v0", is_slippery=False) # 0 left, 1 down, 2 right, 3 up - # env = FrozenLakeWapper(env) - rewards = [] # 记录所有episode的reward - ma_rewards = [] # 滑动平均的reward - for i_ep in range(cfg.eval_eps): - ep_reward = 0 # 记录每个episode的reward - state = env.reset() # 重置环境, 重新开一局（即开始新的一个episode） - while True: - action = agent.predict(state) # 根据算法选择一个动作 - next_state, reward, done, _ = env.step(action) # 与环境进行一个交互 - state = next_state # 存储上一个观察值 - ep_reward += reward - if done: - break - rewards.append(ep_reward) - if ma_rewards: - ma_rewards.append(ma_rewards[-1]*0.9+ep_reward*0.1) - else: - ma_rewards.append(ep_reward) - print(f"Episode:{i_ep+1}/{cfg.eval_eps}, reward:{ep_reward:.1f}") - return rewards,ma_rewards - -if __name__ == "__main__": - cfg = QlearningConfig() - env,agent = env_agent_config(cfg,seed=15) - cfg.model_path = './'+'QLearning/outputs/CliffWalking-v0/20210429-165825/models'+'/' - cfg.result_path = './'+'QLearning/outputs/CliffWalking-v0/20210429-165825/results'+'/' - agent.load(path=cfg.model_path) - rewards,ma_rewards = eval(cfg,env,agent) - save_results(rewards,ma_rewards,tag='eval',path=cfg.result_path) - plot_rewards(rewards,ma_rewards,tag="eval",env=cfg.env,algo = cfg.algo,path=cfg.result_path) - - diff --git a/codes/QLearning/task0_train.ipynb b/codes/QLearning/task0_train.ipynb index 3862439..5715766 100644 --- a/codes/QLearning/task0_train.ipynb +++ b/codes/QLearning/task0_train.ipynb @@ -10,12 +10,15 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.10" + "version": "3.7.11" }, "orig_nbformat": 2, "kernelspec": { - "name": "python3710jvsc74a57bd0fbea1422c2cf61ed9c0cfc03f38f71cc9083cc288606edc4170b5309b352ce27", - "display_name": "Python 3.7.10 64-bit ('py37': conda)" + "name": "python3", + "display_name": "Python 3.7.11 64-bit ('py37': conda)" + }, + "interpreter": { + "hash": "fbea1422c2cf61ed9c0cfc03f38f71cc9083cc288606edc4170b5309b352ce27" } }, "nbformat": 4, @@ -24,8 +27,6 @@ { "cell_type": "code", "execution_count": 1, - "metadata": {}, - "outputs": [], "source": [ "import sys\n", "from pathlib import Path\n", @@ -41,13 +42,13 @@ "from common.plot import plot_rewards\n", "from common.utils import save_results,make_dir\n", "curr_time = datetime.datetime.now().strftime(\"%Y%m%d-%H%M%S\") # obtain current time" - ] + ], + "outputs": [], + "metadata": {} }, { "cell_type": "code", "execution_count": 2, - "metadata": {}, - "outputs": [], "source": [ "class QlearningConfig:\n", " '''训练相关参数'''\n", @@ -56,20 +57,20 @@ " self.env = 'CliffWalking-v0' # 0 up, 1 right, 2 down, 3 left\n", " self.result_path = curr_path+\"/outputs/\" +self.env+'/'+curr_time+'/results/' # path to save results\n", " self.model_path = curr_path+\"/outputs/\" +self.env+'/'+curr_time+'/models/' # path to save models\n", - " self.train_eps = 300 # 训练的episode数目\n", + " self.train_eps = 200 # 训练的episode数目\n", " self.eval_eps = 30\n", " self.gamma = 0.9 # reward的衰减率\n", " self.epsilon_start = 0.95 # e-greedy策略中初始epsilon\n", " self.epsilon_end = 0.01 # e-greedy策略中的终止epsilon\n", " self.epsilon_decay = 200 # e-greedy策略中epsilon的衰减率\n", " self.lr = 0.1 # learning rate" - ] + ], + "outputs": [], + "metadata": {} }, { "cell_type": "code", "execution_count": 3, - "metadata": {}, - "outputs": [], "source": [ "def env_agent_config(cfg,seed=1):\n", " env = gym.make(cfg.env) \n", @@ -79,13 +80,13 @@ " action_dim = env.action_space.n\n", " agent = QLearning(state_dim,action_dim,cfg)\n", " return env,agent" - ] + ], + "outputs": [], + "metadata": {} }, { "cell_type": "code", "execution_count": 4, - "metadata": {}, - "outputs": [], "source": [ "def train(cfg,env,agent):\n", " rewards = [] \n", @@ -109,13 +110,13 @@ " if (i_ep+1)%10==0:\n", " print(\"Episode:{}/{}: reward:{:.1f}\".format(i_ep+1, cfg.train_eps,ep_reward))\n", " return rewards,ma_rewards" - ] + ], + "outputs": [], + "metadata": {} }, { "cell_type": "code", "execution_count": 5, - "metadata": {}, - "outputs": [], "source": [ "def eval(cfg,env,agent):\n", " # env = gym.make(\"FrozenLake-v0\", is_slippery=False) # 0 left, 1 down, 2 right, 3 up\n", @@ -140,76 +141,13 @@ " if (i_ep+1)%10==0:\n", " print(f\"Episode:{i_ep+1}/{cfg.eval_eps}, reward:{ep_reward:.1f}\")\n", " return rewards,ma_rewards" - ] + ], + "outputs": [], + "metadata": {} }, { "cell_type": "code", "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "Episode:10/300: reward:-158.0\n", - "Episode:20/300: reward:-131.0\n", - "Episode:30/300: reward:-37.0\n", - "Episode:40/300: reward:-93.0\n", - "Episode:50/300: reward:-47.0\n", - "Episode:60/300: reward:-67.0\n", - "Episode:70/300: reward:-56.0\n", - "Episode:80/300: reward:-44.0\n", - "Episode:90/300: reward:-41.0\n", - "Episode:100/300: reward:-61.0\n", - "Episode:110/300: reward:-52.0\n", - "Episode:120/300: reward:-14.0\n", - "Episode:130/300: reward:-44.0\n", - "Episode:140/300: reward:-31.0\n", - "Episode:150/300: reward:-17.0\n", - "Episode:160/300: reward:-35.0\n", - "Episode:170/300: reward:-34.0\n", - "Episode:180/300: reward:-16.0\n", - "Episode:190/300: reward:-20.0\n", - "Episode:200/300: reward:-25.0\n", - "Episode:210/300: reward:-13.0\n", - "Episode:220/300: reward:-16.0\n", - "Episode:230/300: reward:-20.0\n", - "Episode:240/300: reward:-27.0\n", - "Episode:250/300: reward:-17.0\n", - "Episode:260/300: reward:-14.0\n", - "Episode:270/300: reward:-15.0\n", - "Episode:280/300: reward:-20.0\n", - "Episode:290/300: reward:-13.0\n", - "Episode:300/300: reward:-13.0\n", - "results saved!\n" - ] - }, - { - "output_type": "display_data", - "data": { - "text/plain": "
", - "image/svg+xml": "\n\n\n\n \n \n \n \n 2021-04-29T17:04:54.671110\n image/svg+xml\n \n \n Matplotlib v3.3.4, https://matplotlib.org/\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n\n", - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEXCAYAAABCjVgAAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAABK2klEQVR4nO3dd3wU1f7/8dfMtpRNhRRCCb13gpQAAaSFEDoKIorY9Vq4P7kELFcpKlwUEBXLtXzFAooQBGmCF5AqvUgRkRpCOunJtvn9EbIQktBDAvt5Ph482J3MzJ4zk+x7zjlTFE3TNIQQQghALe8CCCGEqDgkFIQQQjhJKAghhHCSUBBCCOEkoSCEEMJJQkEIIYSThMJdaM6cOUyaNOm2fNbjjz/OX3/9dVs+q7zNnTuXrl27MmHChGI/y83N5Z133qFv375ER0cTHR3NzJkzyc3Ndc7ToEEDUlNTy7yca9euZcqUKbdsfVeq9/U4duwYzz33HNHR0fTv358HH3yQHTt2AHDmzBlatWoFwHfffccnn3wCwMaNG+nWrRtDhgxh9+7d9OjRg0GDBtG8eXPWrFnjXPfGjRtp0KAB8+fPd07bt28f4eHhXOms++7du7N//362bdtGv379iv18//79PP/88zdV72u1cOFCIiMj6dWrF//+97+xWq235XOL0cRd57333tPeeOON8i7GXad79+7a9u3bi023Wq3a8OHDtcmTJ2u5ubmapmlaTk6ONnnyZG348OGa1WrVNE3T6tevr6WkpNzWMt8KpdX7ehw7dkwLDw/XNmzY4Jy2efNmrU2bNtqff/6pnT59WmvZsmWx5WJiYrQPPvhA0zRNmzNnjjZx4kRN0zTtzTff1KZMmeKcb9KkSdqTTz6pPf74485pH3/8sRYTE3PFcnXr1k3bt2+ftnXrVi0qKuqm6ngzjhw5onXp0kVLSUnR7Ha7NnbsWO2TTz4pl7LoyyeKKj6Hw8Gbb77J3r17yc7ORtM0pkyZQv369YmIiGDVqlUEBAQAcN999/Hss8/SoUMHZsyYwfbt27Hb7TRu3JhXXnkFs9lM9+7dad68OUeOHOGf//wner2ejz/+GIvFQmpqKgMHDuTFF18E4JNPPmHhwoV4enoSFhbG2rVr+fXXX7FYLKWuvzQJCQlMmjSJ+Ph4rFYrUVFRPPXUUwB89NFHrFmzhvz8fHJzcxk/fjw9e/Zkzpw57Nmzh8TERBo0aEBoaChxcXEkJSURFxeHv78/M2fOJCgoiO7duzN79mxycnKYOXMm1atX5+jRo1gsFl577TXat29PamoqEyZM4NSpU/j6+hIQEEC9evV47rnnipQ1OzubKVOmsGvXLnQ6HT169GDs2LFMmDCBevXq8eijjwIQExPjfH/pdn3uueeYO3cuS5cuBSAjI4N7772XNWvWkJeXV+p2uNS5c+d4/fXXiYuLQ9M0Bg4cyGOPPcaLL75IQkICL7/8Mi+88AJ9+/Z1LvPLL7+Qm5vLxIkTUdWCxre7uzsvv/wyAwcOZPXq1UXmB/jhhx/47rvvcDgc+Pr68uqrr1KnTh2OHz/OpEmTyMnJITExkYYNGzJr1ixMJhNNmzbl3nvv5fDhw8yYMYMHHniAJ554gk2bNpGYmMhDDz3E6NGjWbRoEatWreLjjz9m1KhRtGzZkl27dhEfH0+bNm2YNm0aqqqyaNEiPvnkE9zc3Gjfvj1fffUVBw8eLFLOy+vdunXrErfPmTNnGDlyJHXq1CEuLo558+YRGBjoXM+nn37KkCFD6Ny5s3Nahw4deOedd3BzcyvymXPmzCEtLY2QkBDWrl2LyWQiNjaW7Oxs7HY7eXl5DB48mOnTpzuX+d///sdnn33GfffdR05ODh4eHmzZsoX777+f5ORkXnvtNVJSUkhKSqJq1arMmjWLSpUqlfg3s2PHDsaNG8c777yD1Wpl8uTJLFu2jJiYGMxmM0eOHOHcuXPUrl2bd999F09PT9avX8+MGTNQVZVGjRqxefNmvv32W6pVq1Zk3cOHD2f06NH06dMHgBkzZqBpmvM7wt/fH4D777+fKVOm8Pjjj5dYxrIk3Uel2Lt3L4mJiSxYsIDly5czaNAgPv30U7y8vOjZsyc//fQTUNAkTkpKonPnznzyySfodDoWLVrETz/9RGBgIDNmzHCus169eqxYsYIePXrw+eef8/bbb7No0SIWLFjAJ598QmpqKr/99huLFi1i4cKFLFq0iOzsbOfyV1t/ScaNG8eQIUOc69y8eTPLly8nLi6OzZs38/XXX7N06VLGjh3Le++951wuLi6OxYsXO9e/Y8cOZs+ezcqVK/H29mbBggXFPmvfvn2MGTOG2NhYhg4dyvvvvw/AlClTqFu3LitWrGD27Nns2rWrxLK+99575Ofns3z5cmJjY9m1axe///77VfdV4XaNjIwkOzub/fv3A7Bs2TIiIiLw8fEpdTtc7qWXXqJdu3YsXbqU7777jp9++omff/6ZWbNmObf35V/wO3fupE2bNs5AKKQoCh07dixW399//53Y2Fi++eYbYmNjeeyxx5wB+f333zNw4EAWLFjA6tWrOXPmDOvWrQPAarXSrVs3Vq1aRbNmzbBYLPj5+TF//nzee+893nnnHfLz84vV6dSpU8ybN4+ffvqJrVu38vvvv/PXX38xY8YMvvzyS2JjYzGbzdjt9mLLXl7v0rYPFATqM888w6pVq4oEAsCBAwdo3bp1sfVHRERQvXr1YtMBHnvsMbp3787o0aNZvXo1w4cPp2/fvrzzzju0bduWU6dOcf78eY4cOYKPjw+1atWiefPmbNq0CYvFwr59++jUqRM///wzLVu2ZMGCBaxduxY3NzeWLFlS4mdu3bqVCRMmMHfu3BLLe+DAAT777DOWL19OYmIiK1euJC0tjX/961/85z//YcmSJbRr146EhIQS1z9s2DAWL14MgN1u56effmLYsGHEx8dTpUoV53zBwcGlrqOsSUuhFK1atcLHx4f58+dz+vRptm3bhqenJ1CwY9944w0effRRfvzxRwYPHoyqqqxbt47MzEw2b94MFPwRX3o0EhYWBhR8WXz00UesW7eOZcuWcezYMTRNIzc3l/Xr19OnTx+8vb0BGDlyJFu3bgW46vovl5OTw/bt20lPT2f27NnOaYcPH6Zv375MmzaNpUuXcvLkSWeLqFDLli3R6y/+etxzzz3OFknjxo1JT08v9nkhISE0atTIOU/hL//69eudrwMDA51HSZfbvHkzEyZMQKfTodPp+PrrrwGcy5bm0u06dOhQFi9eTLNmzVi0aBHjxo276na4dHvt2rWLzz//HAAvLy8GDx7Mhg0biIqKumIZrsThcBR5v27dOk6ePMnw4cOd09LT0zl//jzjxo1j06ZNfPrpp5w4cYLExERycnKK1bXQvffeC0CTJk2wWCxF5i3UrVs3VFXFbDYTGhpKeno6hw8fJjw8nODgYAAefPBB5syZc8V6XGn7tGjRAr1eT8uWLUtcVlGUYtvhZhiNRtq1a8eOHTv466+/6Nq1q7OuGzduxNvbmyZNmmA2m3n44YfZsWMHX3zxBSdOnODo0aO0aNGi2DrPnTvHU089xYgRI2jYsGGJn9u5c2eMRiMA9evXJz09nR07dlCnTh3nMoMGDSp1TCcyMpLp06eTlJTEwYMHCQ0NpWbNmiWOe1x+kHG7SCiUYt26dUydOpVHHnmEe++9l9q1aztbB2FhYdhsNvbt28eyZcucg1sOh4OJEycSEREBFHSHXHrk5uHhART8cQ0aNIgePXoQFhbGkCFDWLNmDZqmodfri/yC6HQ65+urrf9yDocDTdOYP38+7u7uAKSmpmIymfjjjz945plnGD16NOHh4bRt25Y33nijWFkLXdrEVxSlxF/i0ua5vE6l/bLr9XoURXG+j4+Px83NrdjnXT4Ad2lZhwwZwsCBAxk2bBiZmZm0a9eOrKysUrdDSdvr8mk2m63E8hZq3bo1H3/8MQ6HA1VVycrKQlEUPD092bZtG6NHjy62zgEDBjBu3Djn+8TERHx8fBg7dix2u53IyEi6du1KfHx8kTJdvl8K61C43a51v+h0ulJ/z0pzte1jNBqLHEhcqmXLluzZs4du3boVmf7+++9To0aNEo/Kr6Zz585s376dvXv3MnHiRKCg5bFgwQL8/f2dQfGf//yHffv2MWTIENq1a4fNZitxO+l0Oj755BOeeeYZIiMjad68ebF5rmVbwsXf8ZdffpkDBw4ABV1HI0aMoHfv3ixbtozdu3czbNgwAKpUqUJiYqJz+YSEBGdg327SfVSKTZs20a1bNx544AGaNWvGmjVrijSvhw0bxuTJk2nQoAEhISEAdOrUiW+++QaLxYLD4eDVV1/l3XffLbbukydPkpWVxYsvvkj37t35/fffnctERESwevVqMjMzgYIzEgpd6/oLmc1mWrZsyRdffAEU9LGPGDGCtWvXsn37dpo2bcojjzzCPffcw9q1a0vsPrgVIiIinPVIS0tjzZo1Rb78C3Xo0IHFixfjcDiwWCw8//zzbN++HT8/P+cfVmpqqvOMlZIEBQXRokULXnvtNYYOHQpceTtcymw206JFC7755hsAMjMziY2NpWPHjlesX69evfD09OTNN98kLy+PQ4cOMXjwYJ566il0Ol2x7qbw8HB+/vln55fAd999x8MPPwwUnEXz7LPP0rdvXxRFYe/evWWyXzp16sSWLVucXRQ//PDDVZe50e0D8Oijj/LDDz+wceNG57QNGzYwb968Uo/Kr6ZLly5s2rSJs2fP0qxZMwBnV9SaNWucB08bN27k4YcfZuDAgVSqVInNmzeXuE0DAgJo3bo148ePZ9y4cUXOHLuS1q1bc+LECQ4fPgzAqlWryMjIQFEUpk6dypIlS1iyZAkjRowACsYgFy1axO7du+nduzdQcBbUr7/+SkpKCpqmsWDBAnr06HFD2+VmSUuhFMOHD+ell14iOjoanU5HWFgYq1evdh4NDhw4kHfffbfIl/IzzzzDtGnTGDRoEHa7nUaNGhETE1Ns3Q0aNKBr165ERkbi7e1NjRo1qFu3LidPnqRz587cd9993H///bi5uVGvXj3n0e21rv9SM2bMYPLkyURHR2OxWOjXrx/9+/cnOTnZOQBqMBjo0KED6enpZGVl3doNCUyYMIFXXnmF6OhofH19CQkJKTa4CPCPf/yDqVOnMmDAAOx2O3379qVXr140a9aMl156id69e1OtWjXuueeeK37esGHDeOGFF5g7d+5Vt8PlZsyYwaRJk1i0aBEWi4Xo6GgGDx58xc/T6/V89tlnfPjhhwwaNMh51O3h4UFSUhK//fZbkSPkzp078/jjjzNmzBgURcFsNvP++++jKApjx47l2WefxcfHB3d3d2ff+a1Wq1YtJkyYwKOPPorRaKRRo0bO37MrKW37xMXFXXG50NBQPvroI2bNmsW0adNwOBz4+/szd+5c6tevz5kzZ667DtWrV8dmsxEeHl7kIKNz586sXr2aOnXqAPDss88yffp0PvzwQ3Q6Ha1bt77iNh00aBCrVq3i7bffLhboJfH19eXdd99l/PjxqKpK06ZN0ev1pW7Pwp/37t3b2dJr2LAhzz77LA8//DBWq5UWLVqUyyAzIKekVjT79u3T/u///s/5/vPPP9deeOGF8ivQLfD1119ru3bt0jRN0/Lz87UhQ4Zo69atK+dS3R6JiYk3fTpnWTh16pQ2Z84czW63a5qmaatWrdKGDh1azqW6M2VmZmrTpk3TcnJyNE3TtAMHDmjh4eGaw+Eo55LdGGkpVDC1atXi008/5fvvv0dRFKpUqcLkyZPLu1g3pW7dukyePBmHw4HVaqVPnz7Opv3dLiAgwHnqckUSHBxMYmKisyXs5eXFm2++Wd7FuiOZzWYMBgNDhw5Fr9ej1+uZNWtWiV2kdwJF0+QhO0IIIQrIQLMQQggnCQUhhBBOEgpCCCGcJBSEEEI43RVnH6WlZeNwXP94eaVKZlJSbv15+eVB6lLx3C31AKlLRXUjdVFVBT8/z1J/fleEgsOh3VAoFC57t5C6VDx3Sz1A6lJR3eq6SPeREEIIJwkFIYQQThUiFJYuXUrfvn3p2bOn82ZbQgghbr9yH1NISEhg5syZLFq0CKPRyPDhw2nXrh1169Yt76IJIYTLKfeWwubNm2nfvj2+vr54eHjQu3dvVq5cWd7FEkIIl1TuoZCYmFjkhmGBgYHl9hg6IcrS3XSbseupy62ud3lsR03TcFzhc6+3TKWtr3B64focl70vbdqtVO7dRyVV7HrvLlipUukPrr+agACvG162ornZupxJzMTh0KgR7H2LSnSR3aFx9FQaDUL9nPs332pnxeYTdA+rjrdnwSMOC0+vu7wu51KyCfB1R6e7eBxjtTnIt9jwdDeU+Dvz/g978DGbGBXZqNjPks/nkmexUTXAjEMDnaqQl28jPiWbaoFmDPqiTyJLzcjj77h02jQMvPBoSQ1VLfqZKem5uJv0WKwOYtf/xeBu9Zz1OJ2QyeTPfueJQc0IaxRU4jbadTiRaoFmvll1GLOHgcY1K7Fm+yk6NqtCz3ahJKXl4uGmJ99qx2jQEbv+L84kZNGlVVUa1fTHYNCxassJsvOsjOjVAINex96jSWzed5b0bAsBvu50aBpEVo6V0BBf5v64l+TULN56ugM7jyTz+8FzjHuoHX+dOc+6nWeoGmhmy754RvVtxPGD58jIstC1TTUOHk/hvQV7GBhRhx/WHmVIt7qcjM9gx8F4Jo5pzzcrD1Ovhh8976nB4ROpfLrkAE3rVKJ7m+ocOZXGqXOZJJ/PoVaAEXtOFrlZ6XRsEkTHFtWJT8lhycbjpKVnE+pvJCXlPDqHhY6tQvHxNfPzb8dwN6lU9jZhtdjIybOg2W3Uq+ZDemYuefkW4hIyqFrZk87Ng0lIzuTQ8WTyLVb8zUbc9KCqkJGZS2VfD3RGE4ePJ1MtwAO9qpGTnY+vWU9yahZGvUJOroUqwf6cij+P6rBTyawjPy+fyn4eaChk5Njw9DCRkJZHlUAvMrLysVhs+HgasNls5OVZsVqt+JqNKJoDVVFwM6qkZuRjsdpwN+qp7OvO+cw8LDY7iuYAuxUPTw8cqoHzGTnoFTtGVUMfWIvEWn35evWfuCsWWjaqysRHw2/5d1i53yV18eLF7Nixg6lTpwLwwQcfoGka//jHP655HSkpWTd0rm5AgBdJSZnXvVxZy7fYMRmv/HjEA8dTqFfNF5OhYL5rqcunS/8gNNibXm1LflD61K92kJlrZcpj7QqOWBxgtTswuxtYtvkEf54+zz/vb+mc32pz8PfZdExGHTUvBMm+Y8ms+v00zw5qioebwTnvxn3xfL78EE8PbErbhgUPdV+/J47/W3mEetV8GH5vPczuBib/3w6iOtXCZrHRvnEwG/fHExrsxQeL9tOzbXU6Ng0myM+DtKx8pn2zi/OZuUR1qMXRM+l4u6tkpJ4nPyudtq3qsnL7ady0PLq0qIKvp5HzaZkYdBphjUP4fNkByMumsrsdd6NKRNvarN3yF5bcLHzcVBrUDiTfrlI9xJ/9x9M4HpeG3WKhUXUvHDYr8UnpVPE14uWmkpSSiZ+HSlp6NnoVTO4e2HIyMLsbyLfY0KsKGEyQm4FOBQcqql5PZT9PfHy88G7Tl/8dh2/XHMWoOvAhi0r6XLzJxF/NxqjaqBPsydnEDHSKA8VhR684cCMfP30+es2CnoJpKg4UNFBUVDczKXkqeocFd9WKEQtuig2HBjb06LCjUy7+3WQ7jNjaj+G7DacxWLNxVyy4q5aC/y/8C/bWkZNvR2/NQkPBTbHiqeTjpeahoPGHXzd+OWmisi4THzUXbyWHYA8bOms2buTjoVgw6yy4kY9K+Xz1ODQFTVFA05z1d2gKDhQcqNg0FUVVsV8ooUGzoik60OnJs6ug6rBYC7adXlXQ7DZ0KgVf6IqCqqpY7YCqFjxwSVHJsdhRFB0aBQdIBr0ON6OOPKudPMuFsDDpQVHRVD252dkYFHvB42j1erLzbIRoCeQoHhgc+RgUO3m+tWj87Izr/g5TVeWKB9LlHgoJCQmMGDGChQsX4u7uzvDhw5k8eXKJz0ctzZ0SCgeOp/Djur+Z8GBrjIaSv/T/OpPOtG938e9H2lItoOQddzw+g8n/t4PwpsGM7FUfk0GHVVFRbHYMepVN++OJT8mhQ9Ngqlb2ZNnmE/iaTXy+/BDuJh1vPdkBT7eCI9r1e84S7O9B09r+PPPuBmx2ByGVPTEZdPh4Gjl4MpUR99Zj8W/Hyci28J+nO5KSkceuP5M4cuo8JxMyMGKjXR0zUW2D+Prnvejz0mlWL5A2jaqy90Q6OTaFpJRMcpLicDeoeLrpad+kCtuPpqBmnMNdy8Go2FDcfbDnZOCmWNErdtxUB4pmx66peKm5qIqGTVMxGI3k2xXcHLl4qzlYLjwWxE2xlri9ypJdU7Cjw6qpqAYjNruGzmEhT3XHageTXsWugc6Rj83ghcWm4WZQcNjt2G1WvJVcjAaVk/k+BBhy8dSyin1ZWtFjdSig6kDVg6oreG/0wC8ggHOZGlZNJT3Hhs5gIMjPk1Pn0rHnZuCGlerVAvDz8yE+08H5PIWEtBwsubmENa5Kep6Dw6fSybPYuMd0jEBdyX8PmsGdfIyk5YKqaChu3mTmWAgMqoRN747ZrxLHDhygoSG+yHIOVY/OwxfczORoJtzM3hg9vVBMns5/DoMHa3fHc/hECr6eegZ3roXZ0x1Nb0QxuJHr0PHf2L2YdA4e7N2QjBwbdhR8vdxxoHLiXBapWRZa1A0k16ZRNdCbxPP57P87FXd3Ex2ahqDqCr7MNRT8/M2kn8/hWNx5Dh5Ppne7mhgNemfXTVxSNtUDzSiKQkaOhXW74ujWuipeHkZnvdKz8vF0N6DXqeTkWTEadGzcH0/Tmv5U9i3+xDWHpqFw8bnOl7ZsT57LxNNNX2S5FdtOYrdr9OtYE4DFG/7m3I61tDYn4HDzoU3LeuiqNia4YZO7LxSg4JTUjz/+GKvVytChQ6/7MXQVORR2HkkiduPf9G0fyumELFb+fop+HWuyaX88kx69B7tdY/2eOPp2CEWnqsxfe5TV208zsmd97m1TjbikLKZ9u5vnhzanblUfABb8epRVv592fkZUh1BWbjvFgE61qFrZkzmL9qMAZg8DY+9rweQvC55pXLiFDIqd5tU9yM3KJCM9E2+jneiIRixduxd/NQsvNQ8TNhRFI9CQjc5hwV2x4qHko3j6k56dj7uWj6eaj1lnQdVu/BnCDkWH4u5Nao4Ds5ZNruKBu7c3uVaFhAwrBqMRm8WC0cuPlEwrJp2G3WbFoNipGVoFm8mX3YfOYDLoaNm0Fm5e3pxItbN/3584UOgY1gCjQU9Wng2d0UR8Wj5Z2Tm0qh9EYFAAipuZb1b/SXxSGg/1a0FgYCVOJ+dxOj6NbQfOkJScwZjI+tStUQl0BrItYNNUfH08ybcrJKTlUTXAk7PJ2dQI8iIxLYdftp9hYJdaaDodHnqF85n5/Lz1JNEda+JrLnj8oqZpHD2Tzsff/caYgN3k52QTWLUqAdWqo3pVRvGqjGquhOLph6IzcDoxi0rebni4XVuPr9Vm5z/z92C3a7zyUJsiX0KZORZSM/IJDb7Y7XDoZBofzt9CW58ksuxGnry/Q8GXttEDDO4EBvmQlJRJ8vlcMnKs1A7xJivXitn9Ymvw2XfW0lx/glybjjH3d8ErIAiMHtfcHfzXmXQCfN3wubCNLpWbb0OnKqUeTF2PitpDcCXrdsfx1aojKAp0bVmVUb0bADdWl6uFQrmPKQBER0cTHR1d3sW4oiUbj2OzOxgSUee6ltt2KIG4pGz+u/QgDWr4ArBy2ylsdgdHTp3n8Kk01uw4Q+0QHzzc9Bw4ngoUtAYAVv1+mqxcK7G//Y3JoGNIRB22H06kSS1/qgV4sudoMj9vOYmXksv+P8+ydU8e4ZVT6d9AY9++o+yP3c1Aj1zcFQt1jMn4KVnosEPh7VJ8Lvz/+0qevPAd4UDBphU0dfU+gZxN18h3GEnHB/fMLHQ6lcAaNTF6eqFz9wKTmTSLjhMpVkKC/KhcJYSN++JISc0iolkAe/+M58DxNAZFdaR2jUqcz8xn6cZjpKZmMHJwOIGVvNh14Zd+aNc6PBzdlITEDM4dSaJeTT/Ssy0E+XlgtRc0sz9ffojW9QMIahiIpmn8mL6fFnUrEdCyKgA1cyzM2m4ipLInD3ZqV2R/1CthH40cGnyhSV8wXhFa3ZvQ6oE0bhjKmaQs6tep7Jz30t5bNz2EBhd8KdYIKvhJoJ8HI3vVBy7+wfp7uzGqV4Min6koCvWr+6J5+PFxRjeycq282i4MU5WSx3OqB17fuJlBryNmZGvsdq3Yl7KXh7HIUS+Ar9lItubGuvPVqRnsha5yzRLXW9nX3XlEe2kgAJg93dl2vhaqomAOqYWiXt/YYN1qPqX+zN1UIb6qyo2vV+HBRMG+KkuuvaWvwaINx3Az6tn7VzLZedYiofDLjtPEp+Tw0IXUjk/J5sjp84Q3DXYOVB4/m17Qd2ix8+fpdABsdgcAe/9KZseRRADmrTpC4vlcABTg6JnzLN96kq0Hz2HUqxw8kQZAQlouuZkZRNdPpLo9jp6mo1h8c/FULWChYI86gEMKzU0eGO3ZWE168jFh8QzCvV44GN3ZfzoHNw8zDeuGsOFgKgcOnSQPAy88HoVm8GD/0TTSs/OJ6lATy/lcMnMs7Nlzlm2HEogZ2Rrfywajgy/8K9SrSi3n60oN7AQfT6VOvcooioK/Bzw8pOhga3izKmTlWoloGQKAqijOsYfCsQmTWrBNn+zfxLmcoig8P7RoV6OXh5FebasT5OdxxX1bSFWVYoPGAP7ebvh7u13TOm5UJW83jp0tOADw9yp+hHwzVEVB1V/bF7OP58XP9rvBcvh4mkg6n4eXp6HE7Slu3KW/G763+PfkchIKV7HlQAK+Xkaycq2kZORhtTmcR5S/7ooj+XwuI+6th0GvsnTzCbb+kcDm/ecY0LkWv2w/TUpGPj3DqvPLjtNFTkFTgN/2FfS/ensYSDyfi5tRh8mgo3FNf7b+Ec+aDfvoWjmHrgHJnDsdR7DuPDYrePvmovtbw+5VGbfqjTgen4dHQFX2HIrD3cOdqMhw9EF1+PtUNnMX7qRKgA8vj76n4MvvwlFjWKuLdexczcHPx7cQGuSFydsfgE7Nqzh/HujrTqCvOyGVPIkOr0lln+J9pldiMuhoVf/Kzyk26FVn/+mtcH/3ktoEFY//hVDQqQpenmV7BHgl7iYdBr2K1ebA3+vGgtDnQvl9Pcv2S8sVXRoEfiV0r91KEgqX+Ozng9QI8qJnWMHZOVabg9SMPPQ6hew8K5oGiedzqVrZk5T0PBJScwA4lZhJnRAf0rMsAPwVl86CtUc5k5QNQNuGgfy27yx5Fjs+ZiPpWRZa1qvM7qPJRLQMwexu4OctJxnS2kwn7zhy9n/PEL8sDIoDbECqJ97BgeSYmnDo7xSqVK9Gg8490FWqAUAbwNfPk/nv/I/IdjUwVC84Zm9c04TR3YOGtSqh15V+SYpBrzLtqQ5crevX3aR3+Wb8rebvXfAH7udlcgZ2eVAUBR9PI8npefh539iXjveFbg2fMu7ecEVe7gb0OgWbXZOWwu20688kEtJynaGQnJ6LBpzPtpBvKRhMXbT+GA1D/XC7ZMDr+NkM6oT4XAgQFZvd4ewKAggNNlM1wJNjcRnc17UuOp1Cg+q+1K3qTfcqGWQdWE3rSsep/GcaFsBUvTkZhsp4VqmGzi8EXXBdFFWPGagdn0FokFex5rlBr/LGmHuKTZv8WDvcr3J6K3DF0BBlp7B7qlIZd1NdC1+zqSAUbrj7qCAMvMuxxXO3UhTlpvfPtZJQuCDPYiM3305cUpbzlLGEtIIv9sJAANh9NJk/jqdSr5oP3p5GVAX+PpuBpmmkZubTpKYfe4+lYLE66NIihKgOoRj0OqpWLgiF0GAvgg1ZWHZ/R+dT+7DsS8Xk7kNQSDX0oX3RVW2Czi+E0nrDa5UyEFkaH/kDrdAKw8D/Bo/Ob6XC35UbHdsoDAP5nSsbvl4m0rMteJRxa11C4YLCrp/cfDspGXlU9nEnMS23xHktNgd/nEgjsl0NEs/nsvVgAonnc7HaHDSo4ceB46nYHRrVA80EXDhTo3FNf06eiMP3yBKyj6wHNPTVmqEPG4S+bnsUnaHEzxJ3t8IwKOsB7WtR2O3jd4NlcY4plHGft6sK8nMnL9923Xd8uF4SChecz8p3vj6TmH0hFHJKnLeStxvns/LpEVYdu8OBoijsOFxwFlGArzsBvu6cS82hSqWC433NZqGV8QSNfVdhP5SKPrQVpg4jUM2Vyr5iokIL9HXHZNQ5T2ktT8H+HrgZdfjd4JiAs9VTxt0brmr4vfWK9FqUFQmFC85faCkAvPfjPnqGVXcOJBd6sn8TqgWaSUzLISvH6uzb69GmmjMU/L1NVKnkcSEUPNEsOeSseBdHwl8o7t549J+ILvD6rnUQdy8PNwMz/xHuvF1JeeraqiptGgQWu+/TtaoeaGbsfS1oUtP/FpdMAHi6GfB0K/seBQmFCwpbCl4eBjJzrKzZWXDFcKNQPw6dLLhGoHaINwG+7lStXPSh17WqeDsHmCt5u1G3mg9nkrLw0eeT8/NMHCmncOv+JPrabVFU2eSiKDdjxfid0OvUmxrEVBSFZrWl9Xunk1NOLkjPsmDQq7z+yD28+UR7jAYd3h5GHuhZ3znP5VdwFjLoVWpX8UKvU/HyMND7nhpM7utFzsJXcKSexr3XcxjqdpBAEEJUePItdcH5rHx8zUbnkdJL97fE3aR3DhTrVAW3K5zaeW9YdWqdTUfLTMKybxXWQ+tQfavgHvn/0FUOvS11EEKIm+XyoWC7cD+d81n5RW7EVafqxfuwuBl1GPXqFUf92zYMpE0VBzlLpqJZctDXbotb54dRjNd39a8QQpQnlw+FJ/6zjhZ1KpGWZSn1pmNmd4Pz1hal0fKzyVk5EzQHHoNfR+dXtQxKK4QQZcvlQwFg77EUdKpC6/qVS/y5l4cRva70VoLlj7Xk71gE1jzc+8VIIAgh7lgSChfYHRqhpZwrPqxrnRLv+qhpGpbdS7HsWISuahNMYYPQBdUt66IKIUSZkVC4RGndRw1D/Uqcbtn7M5Ydi9DXC8ctYgyKWv7nmgshxM2QULjAqFev+f77AI7z8Vh2LC4YUO76KIoiZ/cKIe588k12QbVA8zU/GERz2Mnb8AXojZg6PiiBIIS4a7j8t5nRULAJ6lf3veZlLLt+wn7uT9zCR6F6lP4IQSGEuNO4fChoGnRrVZXBXWpf0/y2s4ex7P4Jfb1wDPU6lnHphBDi9pJQ0MDNpLumh8xodit5Gz5H8QrELfzB21A6IYS4vSQUNA2FaxtLsB74BS0jEbfwB+VKZSHEXcnlQwG46rOJARw56eTv+gldjRboqzcr+0IJIUQ5cPlQ0LRrCwXLzliwWXFrP7zMyySEEOVFQkHT4CrdR5olF+vRTRgahKP6Vrk9BRNCiHIgoQBc7fIE69+/g82CoUGX21ImIYQoLy4dCgWthKvMY7Ng3b8a1bcKqjxGUwhxl3PxUCj4X73CoEL+jkU40uIwtR9+xecpCCHE3cDFQ+FCKpTyXa9Z87Ae/B/6eh3R12hx+womhBDlxLVD4cL/pR3/247vAFs+hkZdb1OJhBCifLl2KBQ2FErpFrL+uQnFOxBdUL3bWCohhCg/Lh4KBalQUiY4MpOxnz2EoV64jCUIIVyGa4fChf9L+tK3Ht0MgKG+3PROCOE6XDsUHBdaCpdP1zSsRzehq9IA1Svg9hdMCCHKyS0PhdjYWDp16sSAAQMYMGAAM2fOBODs2bOMHDmSPn368PTTT5OdnQ1ARkYGTzzxBJGRkYwcOZKkpKRbXaRSldZScCT8hZaegKF+p9tWFiGEqAhueSjs37+fmJgYlixZwpIlSxg7diwAb7zxBg888AArV66kadOmfPjhhwDMmjWLsLAwVqxYwbBhw5g6deqtLlKpSrt4zfrnJtAb0dcKu21lEUKIiqBMQiE2Npb+/fvz0ksvkZ6ejtVqZfv27fTu3RuAwYMHs3LlSgDWrVtHdHQ0AP369WPDhg1YrdZbXawSXbx47ZJpDjvW49vR12wjt8cWQrgc/a1eYUBAAE888QTNmzfn3XffZdKkSYwfPx6z2Yxer3fOk5CQAEBiYiIBAQX99nq9HrPZTGpqKkFBQdf8mZUqmW+orFk5FgDMXm4EBHgBkHvyAFn52fi3CMd8YdqdIuAOK++V3C11uVvqAVKXiupW1+WGQ2HFihW89dZbRabVrl2bL7/80vn+scceo0ePHvzrX/8qtvyVTvNU1etrwKSkZOFwXP0+Rpdz8zQBkJ2VT1JSJgB5ezaBqifHuy65F6bdCQICvJx1uNPdLXW5W+oBUpeK6kbqoqrKFQ+kbzgUIiMjiYyMLDItMzOTL7/8ktGjRwMFffZ6vR5/f3+ysrKw2+3odDqSkpIIDAwEIDAwkOTkZIKDg7HZbGRlZeHr63ujxbouFy9eK3yvYTu5G13VxigGt9tSBiGEqEhu6ZiCh4cH//3vf9m7dy8AX3/9NT179sRgMBAWFsby5cuBgjOUunQpuA11REQEsbGxACxfvpywsDAMBsOtLFapLl68VpAKjrQzaJlJ6ENb3ZbPF0KIiuaWjinodDpmzZrF66+/Tl5eHjVr1mT69OkA/Pvf/yYmJoa5c+dSpUoV3n33XQBeeOEFYmJiiIqKwsvLixkzZtzKIl3R5S0F24ndAOhDW962MgghREVyyweaw8LCWLx4cbHpVatWZd68ecWm+/r68tFHH93qYlwTZ0vhwnvbyT2oAbVRPf3KpTxCCFHeXPaK5rzcHKxWG1DQfaRZ83AkH0dfrUk5l0wIIcrPLW8p3CnOfTWBtCrtgGBQwJ74N2gaumC5I6oQwnW5bEvBoFnQZRZcK6EA9oSjgIJOHrkphHBhLhsKOZoJgyMXKOg+sp87iupXFcXkWc4lE0KI8uOyoZCLCYM9DwDFYcee8Jd0HQkhXJ7rhoJmwugoCAVz9hmw5qGTQWYhhItz3VDAiPFC95FX+lFQFPQhjcq5VEIIUb5cOBRMGLWCloJX+lHUgNoyniCEcHkuGwp5mgmDZsWs5OKedQZ9tablXSQhhCh3LhsKuRTc8K6F8RQKGjoJBSGEcN1QyFOMALQynsCuM6ELrFXOJRJCiPLnsqGQf6GlUM+QQLZPHRTVZS/uFkIIJ5cNhTxMzte5vrXLsSRCCFFxuHAoGJ2vc71Dy7EkQghRcbhuKCjuztcWc0g5lkQIISoOlw2FfOViSwHd7XnSmxBCVHQuGwooOgCOWoOcj+MUQghX57Kn3CjAHPURjmdaeKa8CyOEEBWEy7YUFEUhGzfs6KSlIIQQF7hwKIDdoV14U75lEUKIisJlQ0FVFBwXQkEyQQghCrhsKKBwMRSk+0gIIQAXDgUFBYdWGArlXBghhKggXDYU1CIthXIujBBCVBAuGwqKojgHmhUZVRBCCMCFQwFpKQghRDEuGwqqwsUxhXIuixBCVBQuGwrKJaekSlNBCCEKuHAoXOw+UiUThBACcOVQuOSUVCGEEAVcNxQU0Jy9R9JUEEIIcOlQUC55XY4FEUKICsRlQ+HScQRpKQghRIGbDoXZs2czZ84c5/uMjAyeeOIJIiMjGTlyJElJSQBYLBbGjRtHZGQkgwYN4tixYwBomsa0adPo06cPffv2ZefOnTdbpOsmkSCEEAVuOBQyMzOZOHEin3/+eZHps2bNIiwsjBUrVjBs2DCmTp0KwLx583B3d2fFihVMnDiRmJgYAFatWsWxY8dYvnw5H3zwATExMdhstpuo0rUp2n0ksSCEEHATobB27Vpq1qzJI488UmT6unXriI6OBqBfv35s2LABq9XKunXr6N+/PwBt27YlLS2Ns2fPsn79evr27YuqqtSqVYuQkBB27959E1W6NkW7j8r844QQ4o5ww6EwcOBAnnjiCXQ6XZHpiYmJBAQEAKDX6zGbzaSmphaZDhAQEMC5c+dITEwkMDCw2PSyJq0DIYQo7qrPaF6xYgVvvfVWkWm1a9fmyy+/vOYPUdWSs0dVVbQSrhUobf7SVKpkvq75AYymi1X39/ckIMDrutdR0dwNdSh0t9TlbqkHSF0qqltdl6uGQmRkJJGRkde8wsDAQJKTkwkODsZms5GVlYWvry+BgYEkJSURGhoKQFJSEoGBgQQFBTkHoy+dfj1SUrIu3rLiGtmsdufr82k5JJl0V5i74gsI8CIpKbO8i3FL3C11uVvqAVKXiupG6qKqyhUPpG/5KakRERHExsYCsHz5csLCwjAYDERERLBkyRIAduzYgclkIiQkhC5durB06VLsdjsnT57kxIkTNGvW7FYXq5hLO4+kJ0kIIQpctaVwvV544QViYmKIiorCy8uLGTNmADBq1Chee+01oqKiMBqNTJ8+HYA+ffqwb98+5yD01KlTcXNzu9XFKkbOPhJCiOIUraRO/TvMjXQfzflxH7uPJgMw6dF7qBZw/eMSFYmrN4krorulHiB1qajuiO6jO0WRlkI5lkMIISoSFw6F0t4IIYTrcuFQuBgE8jwFIYQo4LqhUN4FEEKICsh1Q+GSVFCl+0gIIQAXDoUiQSCZIIQQgAuHgiLPUxBCiGJcNhQubR5IJAghRAGXDQW5dbYQQhTnsqFQ9OI1SQUhhACXDoWSXwshhCuTUEAGmoUQopALh4IEgRBCXE5CAbnNhRBCFHLhUCjtjRBCuC7XDYVLX0smCCEE4MqhIM9TEEKIYlw4FC59LbEghBDgwqGgFnlGczkWRAghKhCXDYVLSSYIIUQBlw2Foi0FiQUhhAAXDgXJASGEKM6FQ+HSi9ckIYQQAlw6FC59U27FEEKICkVCAbnNhRBCFHLdUECaCkIIcTnXDQV5noIQQhTjwqEgF68JIcTlXDgULnkt3UdCCAG4dChIS0EIIS7nsqGgyg3xhBCiGJcNhUISB0IIcZHLhoLzKmZJBSGEcHLZUCjsMpJBZiGEuEh/syuYPXs2qqry3HPPAbB9+3b+8Y9/EBwcDEDjxo156623yMjI4KWXXuL06dP4+/sza9YsAgICsFgsvPzyyxw4cAA3NzdmzJhBnTp1brZYV+VsKEgmCCGE0w23FDIzM5k4cSKff/55ken79+9nzJgxLFmyhCVLlvDWW28BMGvWLMLCwlixYgXDhg1j6tSpAMybNw93d3dWrFjBxIkTiYmJuYnqXLvCLJBQEEKIi244FNauXUvNmjV55JFHikzfv38/mzZtYuDAgTz11FPEx8cDsG7dOqKjowHo168fGzZswGq1sm7dOvr37w9A27ZtSUtL4+zZszdarGvm7D6SVBBCCKcb7j4aOHAgAHPmzCky3cvLi6ioKHr06MF3333H2LFjmT9/PomJiQQEBBR8qF6P2WwmNTW1yHSAgIAAzp07R0hIyDWXpVIl83WX38vLDSgIhYAAr+teviK6W+oBd09d7pZ6gNSlorrVdblqKKxYscLZBVSodu3afPnllyXOP2nSJOfrESNG8M4775CZmVnivKpackOltOmlSUnJwuHQrmuZ7Ox8oKAbKSmp5PLdSQICvO6KesDdU5e7pR4gdamobqQuqqpc8UD6qqEQGRlJZGTkNX2Yw+Hg448/5oknnkCn0138EL2ewMBAkpOTCQ4OxmazkZWVha+vL4GBgSQlJREaGgpAUlISgYGB1/R5N8N58Zr0HgkhhNMtPSVVVVV++eUXVq1aBUBsbCwtWrTA3d2diIgIYmNjAVi+fDlhYWEYDAYiIiJYsmQJADt27MBkMl1X19ENuzCWIM9SEEKIi276lNTLTZs2jVdffZUPPvgAf39/pk+fDsALL7xATEwMUVFReHl5MWPGDABGjRrFa6+9RlRUFEaj0Tl/Wbs4viypIIQQhRRN066vM74CupExhQ17z/LlisN4mPS8P7ZLGZXs9nH1ftKK6G6pB0hdKqqyGFNw3SuaC/+XhoIQQji5bCjgvKJZUkEIIQq5bCiozovXyrkgQghRgbhsKMhNUoUQojgXDgW5zYUQQlzOdUOh2AshhBCuGwrO5ykIIYQo5MKhUPi/xIIQQhRy2VCQs4+EEKI4lw2FQvI4TiGEuMhlQ0GRloIQQhTjsqGgOscUyrccQghRkbhsKFw8+0hSQQghCrlsKCAtBSGEKMZlQ0GV26QKIUQxLhsKcvGaEEIU58KhUPR/IYQQrhwKyA3xhBDicq4bCtJSEEKIYlw4FGRMQQghLueyoaDKDfGEEKIYlw2FQhIJQghxkcuGgiLP4xRCiGJcNhRUeRynEEIU47KhIA0FIYQozmVDARloFkKIYlw2FOTJa0IIUZzLhoJcvCaEEMW5biggz1MQQojLuW4oSEtBCCGKceFQkNtcCCHE5Vw4FC5/IYQQwoVDoSAMVMkEIYRwuuFQ2LlzJ0OGDGHAgAE8/PDDxMXFAZCRkcETTzxBZGQkI0eOJCkpCQCLxcK4ceOIjIxk0KBBHDt2DABN05g2bRp9+vShb9++7Ny58xZU6+okC4QQorgbDoVx48YxdepUlixZQnR0NFOmTAFg1qxZhIWFsWLFCoYNG8bUqVMBmDdvHu7u7qxYsYKJEycSExMDwKpVqzh27BjLly/ngw8+ICYmBpvNdguqdmWKXLwmhBDF3FAoWCwWXnjhBRo2bAhAgwYNiI+PB2DdunVER0cD0K9fPzZs2IDVamXdunX0798fgLZt25KWlsbZs2dZv349ffv2RVVVatWqRUhICLt3774VdbsiuXhNCCGK09/IQkajkQEDBgDgcDh4//336dGjBwCJiYkEBAQUrFyvx2w2k5qaWmQ6QEBAAOfOnSMxMZHAwMBi08uatBSEKHt2u420tCRsNkt5F8UpMVHF4XCUdzFuiavVRa834ucXgE537V/1V51zxYoVvPXWW0Wm1a5dmy+//BKLxeLs7nnyySdLXYeqltwgUVUVTdOuef7SVKpkvq75AewXPsNk1BMQ4HXdy1dEd0s94O6py91SD7ixuvz99994enpiNofIAdhtpmkamZnp5OSkUbt27Wte7qqhEBkZSWRkZLHp2dnZPP300/j6+jJ37lwMBgMAgYGBJCcnExwcjM1mIysrC19fXwIDA0lKSiI0NBSApKQkAgMDCQoKcg5GXzr9eqSkZOFwFA+XK0k7nwuA1WonKSnzupatiAICvO6KesDdU5e7pR5w43XJzs4hKKgydrsGXN/faFnR61VstrujpXC1uri7e5GQkFZk36mqcsUD6ZsaaA4NDWX27NkYjUbn9IiICGJjYwFYvnw5YWFhGAwGIiIiWLJkCQA7duzAZDIREhJCly5dWLp0KXa7nZMnT3LixAmaNWt2o8W6ZoqMKQhxW0gLofzcyLa/oTGFgwcPsnbtWurWrcvAgQOBghbCp59+ygsvvEBMTAxRUVF4eXkxY8YMAEaNGsVrr71GVFQURqOR6dOnA9CnTx/27dvnHISeOnUqbm5uN1Ks6yK/p0KI22no0GjmzPmYKlVCyrsoV3RDodC4cWOOHDlS4s98fX356KOPik03mUxMmzat2HRFURg/fjzjx4+/kaLcsIsXr0k6CCFEoRsKhbuBZIEQrmfXrh189NEcbDY7VapUwd3dg7//PobD4WDkyIfo3r0nAwb04fvvY/Hw8OTpp8cQHt6FBx8czZo1q9izZzdPP/0P3nprMklJiSQnJ9GyZSteeWUSu3fvZO7c97DbHdSuXYfnn/8nkya9SmJiAjVr1sZiKTgD66+/jjJ9+lTsdjtGo5GJE/9N9eo1ynnLXOTCoSBjCkLcTpv2x7NxX3yZrLtT8yqEN6tyTfOeOnWShQuXMW/eF1SuHMArr7xBdnYWTz01hsaNm9KmTRi7d++iVas2xMfHs2fPLh58cDRbt27m3nt7snnzRurVq8+UKdOwWq08+OAwjhw5DMDp06dYuHAZZrOZd9+dRv36DZkx4z327NnFr7/+AsD333/L8OEP0r17D9auXc0ff+yXUKgI5DoFIVxTjRo1MZvN7NjxO/n5efz8808A5OXlcfz433To0ImdO39HVRV69Ypk7drV2Gw29u7dw7hxEzGZTBw8eIDvv/+WEyeOk56eTm5uDgDVq4diNhec2bN7905ef/1NAFq2bE1ISFUAOnQI5913p7Nt22Y6duxM1673lsNWKJ3rhkLh/5IJQtwW4c2u/Wi+LJlMJgAcDjuvvjqZBg0K7syQmpqCt7cPmZmZzJ//DTqdnjZt2nLq1AmWLYuldu3amEwmFi6cz7p1v9K//yCGDr2H48ePOa+3Klw3FBxwXnphmU6nA6Bbtx40bdqcTZt+44cfvmPr1k2MH//K7ar+Vbn8XVIlE4RwTa1btyU2diEAycnJPPzwCBISzuHn54fJZGLTpg00b96S1q3b8uWXn9GxY2cAtm/fRv/+g+nVKxJQOHr0zxKvKg4Lu4fVq1cAcOjQH8TFnQHgtdcmcPDgHwwcOITHHnvK2fVUUbhsKKjSVBDCpY0Z8zj5+fmMGnUfL7zwFM888zxVq1YDCrp4zGYvPDw8aNOmLcnJSXTs2AmA++57gC+++IQxY0by7rvTaNq0OfHxZ4ut/9FHnyQu7gwPPngfX3/9pbP7aNSoR5g37wvGjBnJBx/M4rnnxt6+Sl8DRSvpPhN3mBu5ojk338azMzdwT6NAnhrQtIxKdvvI1bMVz91SD7jxupw7d5Lg4NAyKNGNc6UrmqH4PiizK5qFEELcfVw2FFS5eE0IIYpx2VC4+Izmci2GEEJUKC4cCoVnH0kqCCFEIRcOhaL/CyGEkFCQUBBCiEu4cChI95EQQlzOdUOh2AshhKj4PvvsYz777OMyW7/rhoLzlNRyLogQQlQgLntDPCgMBEkFIVzBrl07+OqrzwGIiztD16734unpyW+/rUfTNGbMmI2/fyV+/HEBK1cuJy8vF1VVeeONt6hZs1aRdQ0dGk3jxk05evQIH374X7Zu3cwPP3yHw6HRoEFD/vnP8Xz44Wxq1qzNoEFD+emnxSxY8A3ffLMQm83GffcN4Pvvl7BkyY8lftbl61+xYhk//bQYHx9fvLy8aNSoCTabjSlTJnHs2F8ADBo0jP79B930dnLpUFAURVoKQtwm1j83YT2yoUzWbWjQBUP98KvOd/DgH3z33Q94enoTHd2TZ599kc8+m8ebb77BmjWriYqKZsOG9bz//seYTG78978fsXjxD4wd+69i62rfviOTJr3F338fY+nSWObO/RyTycRHH73Pd9/No0OHTixbFsugQUPZufN3MjIySE1N4cSJ4zRp0oz8/Lwrflbh+g8fPsjPP//E559/g6IoPPXUIzRq1IT9+/eSkZHBF198S3r6ed5/f5aEws1SFEVOPxLChdSuXYegoGBsNgc+Pr6Ehd0DQFBQMJmZGXh6mnn99SmsWbOa06dPsW3bZurVa1Diuho3Lrhn2u7dOzhz5jRPPvkIADablfr1GzJixCjnE9ZOnjzJvff2Ys+e3Rw+/AcdO3a66mcVrn/Xrp20bx+Oh4cHUHDrbbvdTu3adTh16gT//Oc/aN8+nKeffu6WbCMXDwXJBCFuF0P98Gs6mi9Len3Rr7zCZxwUSkg4x3PPPcmQIffRvn1H/P0rcfRoyc+jL3x2gt3uoHv3Hrz44jgAcnJysNvtmEwm6tatz+rVKwgNDaVVqzbs3Pk7+/bt5YEHHr7qZxWuX1EUNK3ocxnsdjs+Pr58++1CtmzZwpYtmxgz5kHmzfseLy+vm9pGLjvQDAUbWzJBCFHo8OGDVKtWnfvvH0njxk3ZunUzDof9isu0atWGDRvWkZaWiqZpvPPOW3z//bcAdOwYzpdf/pdWrdrQqlUbNm7cgLu7G76+vtf8WWFhbdm8eSNZWVnk5+ezYcP/ANi4cT2vv/4KHTt24sUXX8Ld3Z3ExISb3gYu3VJQFblOQQhxUdu27Vm8eCEPPjgMg8FA48ZN+fvvY1dcpl69+jzyyOM8//xTaJpGvXoNePDB0QB06NCJGTPeplWrMLy9vfH19aNDh07X9Vn16jVg2LARPPbYQ3h5eREUVPD0uvbtw1m//ldGjboPo9FIRER36tSpe9PbwGWfpwDw7MwNhDcN5oGe9cugVLeX3Lu/4rlb6gHyPIWKSp6ncIv5e7vh62W6+oxCCOEiXLr7aNY/Izifll3exRBCiArDpVsKbkY9OtWlN4EQQhQh34hCiDJ1Fwxb3rFuZNtLKAghyoxebyQ7O0OCoRxomkZ2dgZ6vfG6lnPpMQUhRNny8wsgLS2JrKzz5V0UJ1VVcTjujrOPrlYXvd6In1/Ada1TQkEIUWZ0Oj2VK1cp72IUIacKX5l0HwkhhHCSUBBCCOF0V3QfqTdx/+ubWbaikbpUPHdLPUDqUlFdb12uNv9dcZsLIYQQt4Z0HwkhhHCSUBBCCOEkoSCEEMJJQkEIIYSThIIQQggnCQUhhBBOEgpCCCGcJBSEEEI4SSgIIYRwcslQWLp0KX379qVnz55888035V2c6/bQQw8RFRXFgAEDGDBgAHv37r3j6pSVlUW/fv04c+YMAJs3byY6OppevXoxc+ZM53yHDh1iyJAh9O7dm5dffhmbzVZeRS7V5XWZMGECvXr1cu6fX375BSi9jhXF+++/T1RUFFFRUUyfPh24M/dLSfW4U/fJ7Nmz6du3L1FRUXzxxRfAbdgnmos5d+6c1q1bNy0tLU3Lzs7WoqOjtaNHj5Z3sa6Zw+HQwsPDNavV6px2p9Vpz549Wr9+/bQmTZpop0+f1nJzc7WIiAjt1KlTmtVq1caMGaOtW7dO0zRNi4qK0nbv3q1pmqZNmDBB++abb8qx5MVdXhdN07R+/fppCQkJRea7Uh0rgk2bNmn333+/lp+fr1ksFu2hhx7Sli5desftl5LqsXr16jtyn2zbtk0bPny4ZrVatdzcXK1bt27aoUOHynyfuFxLYfPmzbRv3x5fX188PDzo3bs3K1euLO9iXbO///4bRVF4/PHH6d+/P19//fUdV6fvv/+ef//73wQGBgKwb98+QkNDqV69Onq9nujoaFauXElcXBx5eXm0bNkSgMGDB1e4el1el5ycHM6ePcurr75KdHQ07733Hg6Ho9Q6VhQBAQHExMRgNBoxGAzUqVOHEydO3HH7paR6nD179o7cJ/fccw9fffUVer2elJQU7HY7GRkZZb5P7oq7pF6PxMREAgIuPokoMDCQffv2lWOJrk9GRgYdOnTg9ddfJy8vj4ceeojIyMg7qk5Tp04t8r6kfZKQkFBsekBAAAkJCbetnNfi8rqkpKTQvn17Jk2ahIeHB08++SQLFy7Ew8OjxDpWFPXq1XO+PnHiBMuXL2fUqFF33H4pqR7ffvstv//++x23TwAMBgPvvfcen3/+OX369Lktfysu11LQSrgprKLcObfRbdWqFdOnT8fDwwN/f3+GDh3Ke++9V2y+O6lOpe2TO3FfVa9enQ8++IBKlSrh7u7OqFGjWL9+/R1Tl6NHjzJmzBjGjx9PjRo1iv38Ttkvl9ajdu3ad/Q+ef7559myZQvx8fGcOHGi2M9v9T5xuVAICgoiOTnZ+T4xMdHZ9L8T7Nixgy1btjjfa5pG1apV7+g6lbZPLp+elJRU4et15MgRVq1a5XyvaRp6vf6O+L3buXMno0eP5v/9v//HoEGD7tj9cnk97tR9cuzYMQ4dOgSAu7s7vXr1Ytu2bWW+T1wuFDp27MiWLVtITU0lNzeX1atX06VLl/Iu1jXLzMxk+vTp5Ofnk5WVxeLFi/nPf/5zR9epRYsWHD9+nJMnT2K321m2bBldunShatWqmEwmdu7cCUBsbGyFr5emabz55pukp6djtVpZsGABPXv2LLWOFUV8fDzPPvssM2bMICoqCrgz90tJ9bhT98mZM2d45ZVXsFgsWCwW1q5dy/Dhw8t8n7jcmEJQUBBjx47loYcewmq1MnToUJo3b17exbpm3bp1Y+/evQwcOBCHw8EDDzxAmzZt7ug6mUwm3n77bZ577jny8/OJiIigT58+AMyYMYNXXnmF7OxsGjduzEMPPVTOpb2yhg0b8sQTTzBixAhsNhu9evWiX79+AKXWsSL47LPPyM/P5+2333ZOGz58+B23X0qrx524TyIiIpx/6zqdjl69ehEVFYW/v3+Z7hN58poQQggnl+s+EkIIUToJBSGEEE4SCkIIIZwkFIQQQjhJKAghhHCSUBDiFnj88cf566+/rmuZJ598kkWLFpVRiYS4MS53nYIQZeHTTz8t7yIIcUtIKAiX9uuvvzJ37lysVitubm6MHz+ejRs3cvToUZKTk0lJSaFhw4ZMnToVs9nMt99+y/z58zEYDJhMJiZNmkTdunXp3r07s2fPplmzZixYsIB58+ahqiqVK1fm1VdfpVatWiQkJBATE0NiYiIhISGkpKQ4y3Hs2DGmTp3K+fPnsdvtjBo1iqFDh5Kdnc2ECRM4efIkqqrSpEkTJk2ahKpKI1+UkRu/27cQd7bjx49r/fr101JTUzVN07Q///xTCw8P195++22tS5cuWlJSkma327V//vOf2ttvv63ZbDatSZMmzvvyL168WJs/f76maZrWrVs3bd++fdrmzZu1Hj16aCkpKZqmadqPP/6oRUZGag6HQ3vmmWe0mTNnapqmaSdOnNBatmyp/fjjj5rVatX69u2rHThwQNM0TcvIyNAiIyO13bt3a4sXL9bGjBmjaZqm2Ww27eWXX9ZOnDhxOzeTcDHSUhAua9OmTSQmJjJ69GjnNEVROHXqFH369KFy5coADB06lDfffJPx48fTp08fhg8fTteuXQkPDyc6OrrIOn/77Tf69u2Lv78/UHBf+6lTp3LmzBk2b97M+PHjAQgNDaVdu3ZAwS2eT506xcSJE53rycvL4+DBg3Tu3JmZM2cyatQoOnbsyMMPP0xoaGhZbhbh4iQUhMtyOBx06NCBWbNmOafFx8ezYMECLBZLkfkKu2tmzJjBn3/+yebNm/n0009ZuHAhc+fOdc6rlXDXGE3TsNlsxW5xrNcX/PnZ7Xa8vb1ZsmSJ82fJycl4eXlhMpn45Zdf2LZtG1u3buWRRx7hlVdeqVD36BF3F+mYFC6rffv2bNq0iWPHjgGwfv16+vfvT35+PmvXriUzMxOHw8H3339Pt27dSE1NJSIiAl9fX0aPHs2LL77IkSNHiqyzU6dOLF++nNTUVAB+/PFHfH19CQ0NpXPnzixYsACAs2fPsm3bNgBq1aqFyWRyhkJ8fDz9+vXjwIEDfPvtt0yYMIFOnToxbtw4OnXqxNGjR2/XJhIuSG6IJ1zaihUr+Oijj5z32J84cSJbtmxh69at2O120tLSaNu2La+88gpubm7Mnz+fr776Cjc3N3Q6HWPHjqVjx45FBpq/+eYb5s+fj8PhwN/fn9dee4169eqRmprKhAkTOHXqFMHBwdhsNgYNGsTgwYM5fPiwc6DZZrPx0EMPMWLECHJycpg4cSJHjhzB3d2dkJAQpk6dio+PT3lvOnGXklAQ4jJz5swhLS2N1157rbyLIsRtJ91HQgghnKSlIIQQwklaCkIIIZwkFIQQQjhJKAghhHCSUBBCCOEkoSCEEMJJQkEIIYTT/wfyEWgyRb2AewAAAABJRU5ErkJggg==\n" - }, - "metadata": {} - }, - { - "output_type": "stream", - "name": "stdout", - "text": [ - "Episode:10/30, reward:-13.0\nEpisode:20/30, reward:-13.0\nEpisode:30/30, reward:-13.0\nresults saved!\n" - ] - }, - { - "output_type": "display_data", - "data": { - "text/plain": "
", - "image/svg+xml": "\n\n\n\n \n \n \n \n 2021-04-29T17:04:55.053953\n image/svg+xml\n \n \n Matplotlib v3.3.4, https://matplotlib.org/\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n\n", - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYQAAAEXCAYAAACtTzM+AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAA0B0lEQVR4nO3deWBN577/8fdOtjkJQii3DtIaWzVUFTEFRUKIIRVtja1W9WjprR9SemoeTqokVa1ekWuooWqIIaXioBFRU6U9WsUpikgiURJk2nv9/hD7isSQQSO7n9df9trredbzXSvy2etZK2ubDMMwEBGRvzyHoh6AiIg8GhQIIiICKBBERCSLAkFERAAFgoiIZFEgiIgIoECwG8HBwUyZMuVP2dbw4cM5efLkn7KtorZw4UI6dOjAhAkTcrx348YNPvroI7y9vfHx8cHHx4ePP/6YGzdu2NapV68eSUlJD32cERERTJs2rdD6u1fdeXHq1ClGjRqFj48PPXv25JVXXuHgwYMAnDt3jqZNmwKwcuVKFi1aBEBkZCSenp707duXI0eO0LlzZ3r37s0zzzzDjh07bH1HRkZSr149Vq1aZVsWExODh4cH97qbvmPHjvz444/s37+fHj165Hj/xx9/5O233y5Q3Q9q7dq1eHl50aVLF/7xj3+QkZHxp2z3rgyxC0FBQcbkyZOLehh2p2PHjsaBAwdyLM/IyDD8/f2NqVOnGjdu3DAMwzCuX79uTJ061fD39zcyMjIMwzCMunXrGomJiX/qmAvD3erOi1OnThkeHh7Gnj17bMuioqKMZ5991vj111+N33//3WjSpEmOduPHjzcWLFhgGIZhBAcHGwEBAYZhGMaMGTOMadOm2dabMmWK8cYbbxjDhw+3Lfv888+N8ePH33Ncnp6eRkxMjBEdHW107969QDUWxPHjx4127doZiYmJhsViMcaMGWMsWrSoyMZjGIZhLto4erRYrVZmzJjB0aNHuXbtGoZhMG3aNOrWrUv79u3Ztm0bbm5uALz44ou89dZbtGrVisDAQA4cOIDFYqFhw4ZMnDgRJycnOnbsyDPPPMPx48d59913MZvNfP7556Snp5OUlISvry+jR48GYNGiRaxdu5Zy5crRvHlzIiIi2LlzJ+np6Xft/27i4uKYMmUKsbGxZGRk0L17d0aMGAHAZ599xo4dO0hLS+PGjRuMGzeOF154geDgYH744Qfi4+OpV68eNWvW5Pz58yQkJHD+/HlcXV35+OOPqVq1Kh07dmT+/Plcv36djz/+mBo1anDixAnS09P54IMPaNmyJUlJSUyYMIGzZ89SoUIF3NzcqFOnDqNGjco21mvXrjFt2jQOHz6Mo6MjnTt3ZsyYMUyYMIE6derw6quvAjB+/Hjb69v366hRo1i4cCGbNm0C4OrVq3Tq1IkdO3aQmpp61/1wu4sXL/Lhhx9y/vx5DMPA19eX1157jdGjRxMXF8f777/PO++8g7e3t63Nt99+y40bNwgICMDB4eaJdpkyZXj//ffx9fVl+/bt2dYH+Oqrr1i5ciVWq5UKFSowadIknnjiCX777TemTJnC9evXiY+Pp379+sybN49SpUrx9NNP06lTJ3755RcCAwN56aWXeP3119m7dy/x8fEMGjSIIUOGsG7dOrZt28bnn3/OwIEDadKkCYcPHyY2NpZnn32W2bNn4+DgwLp161i0aBGlS5emZcuWLF26lGPHjmUb5511N2vWLNf9c+7cOV5++WWeeOIJzp8/z7Jly6hSpYqtny+++IK+ffvStm1b27JWrVrx0UcfUbp06WzbDA4O5vLly1SvXp2IiAhKlSrFhg0buHbtGhaLhdTUVPr06cOcOXNsbf71r3+xePFiXnzxRa5fv07ZsmXZt28f/fv359KlS3zwwQckJiaSkJDAf/3XfzFv3jwqVaqU6/+ZgwcPMnbsWD766CMyMjKYOnUqmzdvZvz48Tg5OXH8+HEuXryIu7s7c+fOpVy5cuzevZvAwEAcHBxo0KABUVFRfPnllzz++OPZ+vb392fIkCF069YNgMDAQAzDsP2OcHV1BaB///5MmzaN4cOH5zrGP4OmjG5z9OhR4uPjWb16NVu3bqV379588cUXODs788ILLxAWFgbcPA1OSEigbdu2LFq0CEdHR9atW0dYWBhVqlQhMDDQ1medOnUIDw+nc+fOhISEMGvWLNatW8fq1atZtGgRSUlJfPfdd6xbt461a9eybt06rl27Zmt/v/5zM3bsWPr27WvrMyoqiq1bt3L+/HmioqJYvnw5mzZtYsyYMQQFBdnanT9/nvXr19v6P3jwIPPnz+ebb77BxcWF1atX59hWTEwMw4YNY8OGDfTr149PPvkEgGnTpvHkk08SHh7O/PnzOXz4cK5jDQoKIi0tja1bt7JhwwYOHz7M999/f99jdWu/enl5ce3aNX788UcANm/eTPv27Slfvvxd98Od3nvvPZ5//nk2bdrEypUrCQsLY8uWLcybN8+2v+/85X7o0CGeffZZWxjcYjKZaN26dY56v//+ezZs2MCKFSvYsGEDr732mi0c16xZg6+vL6tXr2b79u2cO3eOXbt2AZCRkYGnpyfbtm2jUaNGpKenU7FiRVatWkVQUBAfffQRaWlpOWo6e/Ysy5YtIywsjOjoaL7//ntOnjxJYGAgoaGhbNiwAScnJywWS462d9Z9t/0DN8N05MiRbNu2LVsYAPz00080a9YsR//t27enRo0aOZYDvPbaa3Ts2JEhQ4awfft2/P398fb25qOPPuK5557j7Nmz/PHHHxw/fpzy5ctTu3ZtnnnmGfbu3Ut6ejoxMTG0adOGLVu20KRJE1avXk1ERASlS5dm48aNuW4zOjqaCRMmsHDhwlzH+9NPP7F48WK2bt1KfHw833zzDZcvX+b//b//xz//+U82btzI888/T1xcXK79+/n5sX79egAsFgthYWH4+fkRGxtLtWrVbOs99thjd+3jz6IzhNs0bdqU8uXLs2rVKn7//Xf2799PuXLlgJsHdfLkybz66qt8/fXX9OnTBwcHB3bt2kVycjJRUVHAzf/At38Kad68OXDzF8Vnn33Grl272Lx5M6dOncIwDG7cuMHu3bvp1q0bLi4uALz88stER0cD3Lf/O12/fp0DBw5w5coV5s+fb1v2yy+/4O3tzezZs9m0aRNnzpyxnQnd0qRJE8zm//uRaNGihe1MpGHDhly5ciXH9qpXr06DBg1s69z6wd+9e7ft31WqVLF9OrpTVFQUEyZMwNHREUdHR5YvXw5ga3s3t+/Xfv36sX79eho1asS6desYO3bsfffD7fvr8OHDhISEAODs7EyfPn3Ys2cP3bt3v+cY7sVqtWZ7vWvXLs6cOYO/v79t2ZUrV/jjjz8YO3Yse/fu5YsvvuD06dPEx8dz/fr1HLXe0qlTJwCeeuop0tPTs617i6enJw4ODjg5OVGzZk2uXLnCL7/8goeHB4899hgAr7zyCsHBwfes4177p3HjxpjNZpo0aZJrW5PJlGM/FETJkiV5/vnnOXjwICdPnqRDhw62WiMjI3FxceGpp57CycmJwYMHc/DgQZYsWcLp06c5ceIEjRs3ztHnxYsXGTFiBAMGDKB+/fq5brdt27aULFkSgLp163LlyhUOHjzIE088YWvTu3fvu17D8fLyYs6cOSQkJHDs2DFq1qxJrVq1cr3OcecHjD+bAuE2u3btYvr06QwdOpROnTrh7u5uOyto3rw5mZmZxMTEsHnzZtuFLKvVSkBAAO3btwduToHc/omtbNmywM3/WL1796Zz5840b96cvn37smPHDgzDwGw2Z/vhcHR0tP37fv3fyWq1YhgGq1atokyZMgAkJSVRqlQp/v3vfzNy5EiGDBmCh4cHzz33HJMnT84x1ltuP603mUy5/gDfbZ07a7rbD7rZbMZkMtlex8bGUrp06Rzbu/Ni2+1j7du3L76+vvj5+ZGcnMzzzz9PSkrKXfdDbvvrzmWZmZm5jveWZs2a8fnnn2O1WnFwcCAlJQWTyUS5cuXYv38/Q4YMydFnr169GDt2rO11fHw85cuXZ8yYMVgsFry8vOjQoQOxsbHZxnTncblVw6399qDHxdHR8a4/Z3dzv/1TsmTJbB8ibtekSRN++OEHPD09sy3/5JNP+Nvf/pbrp/H7adu2LQcOHODo0aMEBAQAN884Vq9ejaurqy0k/vnPfxITE0Pfvn15/vnnyczMzHU/OTo6smjRIkaOHImXlxfPPPNMjnUeZF/C//2Mv//++/z000/AzemiAQMG0LVrVzZv3syRI0fw8/MDoFq1asTHx9vax8XF2cK6qGjK6DZ79+7F09OTl156iUaNGrFjx45sp9R+fn5MnTqVevXqUb16dQDatGnDihUrSE9Px2q1MmnSJObOnZuj7zNnzpCSksLo0aPp2LEj33//va1N+/bt2b59O8nJycDNOw9uedD+b3FycqJJkyYsWbIEuDmnPmDAACIiIjhw4ABPP/00Q4cOpUWLFkREROQ6ZVAY2rdvb6vj8uXL7NixI9sv/ltatWrF+vXrsVqtpKen8/bbb3PgwAEqVqxo+0+VlJRkuzMlN1WrVqVx48Z88MEH9OvXD7j3fridk5MTjRs3ZsWKFQAkJyezYcMGWrdufc/6unTpQrly5ZgxYwapqan8/PPP9OnThxEjRuDo6JhjisnDw4MtW7bYfgGsXLmSwYMHAzfvlnnrrbfw9vbGZDJx9OjRh3Jc2rRpw759+2zTEl999dV92+R3/wC8+uqrfPXVV0RGRtqW7dmzh2XLlt310/j9tGvXjr1793LhwgUaNWoEYJt+2rFjh+2DU2RkJIMHD8bX15dKlSoRFRWV6z51c3OjWbNmjBs3jrFjx2a7Q+xemjVrxunTp/nll18A2LZtG1evXsVkMjF9+nQ2btzIxo0bGTBgAHDzmuO6des4cuQIXbt2BW7e7bRz504SExMxDIPVq1fTuXPnfO2XwqIzhNv4+/vz3nvv4ePjg6OjI82bN2f79u22T4G+vr7MnTs32y/kkSNHMnv2bHr37o3FYqFBgwaMHz8+R9/16tWjQ4cOeHl54eLiwt/+9jeefPJJzpw5Q9u2bXnxxRfp378/pUuXpk6dOrZPtQ/a/+0CAwOZOnUqPj4+pKen06NHD3r27MmlS5dsFztLlChBq1atuHLlCikpKYW7I4EJEyYwceJEfHx8qFChAtWrV89xIRHg73//O9OnT6dXr15YLBa8vb3p0qULjRo14r333qNr1648/vjjtGjR4p7b8/Pz45133mHhwoX33Q93CgwMZMqUKaxbt4709HR8fHzo06fPPbdnNptZvHgxn376Kb1797Z92i5btiwJCQl899132T4Zt23bluHDhzNs2DBMJhNOTk588sknmEwmxowZw1tvvUX58uUpU6aMba68sNWuXZsJEybw6quvUrJkSRo0aGD7ObuXu+2f8+fP37NdzZo1+eyzz5g3bx6zZ8/GarXi6urKwoULqVu3LufOnctzDTVq1CAzMxMPD49sHzDatm3L9u3beeKJJwB46623mDNnDp9++imOjo40a9bsnvu0d+/ebNu2jVmzZuUI89xUqFCBuXPnMm7cOBwcHHj66acxm8133Z+33u/atavtDK9+/fq89dZbDB48mIyMDBo3blykF5QB3Xb6KIiJiTH+93//1/Y6JCTEeOedd4puQIVg+fLlxuHDhw3DMIy0tDSjb9++xq5du4p4VH+O+Pj4At+y+TCcPXvWCA4ONiwWi2EYhrFt2zajX79+RTyq4ik5OdmYPXu2cf36dcMwDOOnn34yPDw8DKvVWsQjKxidITwCateuzRdffMGaNWswmUxUq1aNqVOnFvWwCuTJJ59k6tSpWK1WMjIy6Natm+103t65ubnZbk9+lDz22GPEx8fbzoCdnZ2ZMWNGUQ+rWHJycqJEiRL069cPs9mM2Wxm3rx5uU6LFicmw9AX5IiIiC4qi4hIFgWCiIgACgQREcmiQBAREaAY/x3C5cvXsFrzfj28UiUnEhML/777omJv9YD91WRv9YD91WRv9UDOmhwcTFSsWO6ebYptIFitRr4C4VZbe2Jv9YD91WRv9YD91WRv9UDea9KUkYiIAAoEERHJUmynjETk0WQYBpcvJ5CengoUj2mY+HiHQn1Ud9ExUbJkaSpWzN9fyisQRKRQpaRcwWQyUbXq45hMxWMSwmx2IDOz+AeCYVj5449LpKRcoUoVlzy3Lx5HS0SKjRs3UnB2rlBswsCemEwOODtX5MaN/N0xpSMmIoXKarXg6KjJh6Li6GjGas3f92koEESk0BX3p34WZwXZ9woEEZGHpF8/H2JjLxT1MB6YAkFERADdZSQidu7w4YMsXBiExWKlWrVqlClTlv/85xRWq5WXXx5Ex44v4OPzAqtXb6Bs2XK8+eYwPDza8corQ9ixYxs//HCEN9/8OzNnTiUhIZ5LlxJo0qQpEydO4ciRQ7a+3d2f4O2332XKlEnEx8dRq5Y76enpAJw8eYI5c6ZjsVgoWbIkAQH/oEaNvxXxnslJgSAiD83eH2OJjIl9KH23eaYaHo2qPdC6v/9+lrVrN7Ns2RIqV3Zj4sTJXLuWwogRw2jY8GmeffY5jhw5TNOmzxIbG8sPPxzmlVeGEB0dRadOLxAVFUmdOnWZNm02GRkZvPKKH8eP/5KtbycnJ+bOnU3duvUJDAzihx8Os3PntwCsWfMl/v6v0LFjZyIitvPvf/9on4Ewf/58HBwcGDVqFACnTp1i0qRJXLt2jdKlS/Phhx/SoEGDXNtmZmby8ssv079///t+qbmISH7VqFETJycnDh78nrS0VLZsCQMgNTWV3377Dx4ebTh06HscHEx06eJFRMR2MjMzOXr0B8aODaBUqVIcO/YTa9Z8yenTv3HlyhVu3LierW+AI0cO8eGHN7+WtEmTZlSv/l8AtGrlwdy5c9i/P4rWrdvSoUOnItgL95fvQEhOTmbmzJls2bKF1157zbZ84sSJvP7663h6erJv3z7GjRtHWFhYrn0sWLCA06dP53cIIvKI82j04J/iH6ZSpUoBN2+JnTRpKvXq1QcgKSkRF5fy3LhxjRUrluPoaObZZ5/j7NnTbN68AXd3d0qVKsXatavYtWsnPXv2pl+/Fvz22yluffvwrb7h5h0+t//Fs6OjIwCenp15+uln2Lv3O776aiXR0XsZN27in1X+A8v3ReWIiAhq1arF0KFDsy338/OjXbt2ANSrV4/Y2NxPFw8dOsTx48fx9PTM7xBERPKkWbPn2LBhLQCXLl1i8OABxMVdpGLFipQqVYq9e/fwzDNNaNbsOUJDF9O6dVsADhzYT8+efejSxQswceLEr7k+6qJ58xZs3x4OwM8//5vz588B8MEHEzh27N/4+vbltddG2KabHjX5PkPw9fUFIDg4ONvy26d+goKC6Ny5c462KSkpzJo1i4ULFxIYGJiv7Veq5JSvdgBubs75bvsosrd6wP5qsrd64O41xcc7YDY/OjcwOjo6YDKZMJsdeP31N5gzZyaDBvXHYrHw97+/Q82aN+fyPTzasHfvd7i4ONGiRQuCgj6ibdt2mM0ODBjwMnPmzGTVquWUK1eWRo0aExcXy+OP17D1DfDGG28yZco/GDjwRWrWrEX16v+Fo6MDQ4e+yowZU/jf/12M2ezI6NH//VD3kYPDzb7z+nNnMm6d99xFeHg4M2fOzLbM3d2d0NBQ4P8C4dY1BLj5cKs5c+YQHR3N0qVLcXbOPqixY8fSrVs3OnXqxPjx42nRokWeryEkJqbk6/nlbm7OJCQk57ndo8re6gH7q8ne6oF713Tx4hkee6zmnzyigrGXZxndcvHiGRo1ejrbMXJwMN33g/R9zxC8vLzw8vJ64IFkZmYybtw44uLicg2DlJQU9u3bx6+//kpQUBCxsbFER0djNpvp2bPnA29HREQKV6Hfdjp79mxSUlIICQmhZMmSOd53cnIiMjLS9vrWGYLCQESkaBVqICQlJbFixQoef/xx/Pz8bMs3btxIREQEO3fuZPr06YW5SRERKST3vYbwqNI1hJvsrR6wv5rsrR7QNYRHXX6vITw6twKIiEiRUiCIiAigQBARkSwKBBGRR9zixZ+zePHnD307CgQREQH0+GsReYgyft1LxvE9D6XvEvXaUaKuxz3XOXz4IEuXhmAYcOHCOTp06ES5cuX47rvdGIZBYOB8XF0r8dVXq9i6dQupqTdwcHBg8uSZ1KpVO1tf/fr50LDh05w4cZxPP/0foqOj+OqrlVitBvXq1efdd8fx6afzqVXLnd69+xEWtp7Vq1ewYsVaMjMzefHFXqxZs5GNG7/mm2+25tjWnf2Hh28mLGw95ctXwNnZmQYNniIzM5OZMyfzn/+cAqB3bz969uxdaPtUZwgiYteOHfs3AQEfsGzZGjZsWEuFChVZvHgZTz5Zhx07tnPtWgq7d+/ik08+Z9myNbRt24H167/Kta+WLVuzcuU6Ll++zKZNG1i4MITQ0C+pWNGVlSuX0arVzcdoAxw69D1Xr14lKSmRmJgfeOqpRqSlpbJnz+67butW/3FxF9myJYyQkBXMm/cpCQnxAPz441GuXr3KkiVfMm/ep/z449FC3Vc6QxCRh6ZEXY/7fop/2Nzdn6Bq1ccAKF++As2btwCgatXHSE6+SrlyTkyZMoMdO7bz++9n2b8/ijp16uXaV8OGTwNw5MhBzp37nTfeuPm058zMDOrWrc+AAQNt34x25swZOnXqwg8/HOGXX/5N69ZtKFfOiQ8/nHbXbd3q//DhQ7Rs6UHZsmWBm4/PtlgsuLs/wdmzZ3j33b/TsqUHb745isKkQBARu2Y2Z/81d+s7Cm6Ji7vI22+PoE8fP1q2bI2rayVOnDiea1+3vvvAYrHSsWNnRo8eC8D169exWCyUKlWKJ5+sy/bt4dSsWZOmTZ/l0KHviYk5yksvDSYu7iKjRr1B374v5rqtW/2bTCYMI/v3KlgsFsqXr8CyZWs4cGA/+/btZdiwV1i2bE2OZ8bll6aMROQv7ZdfjvH444/Tv//LNGz4NNHRUVitlnu2adr0Wfbs2cXly0kYhsFHH81kzZovAWjd2oPQ0P+hadNnadr0WSIj91CmTGkqVKiQta0a991W8+bPERUVSUpKCmlpaezZ8y8AIiN3M2XKJFq3bsPo0e9RpkwZ4uPjCm1f6AxBRP7SnnuuJRs2fM0rr/hRokQJGjZ82nbR9m7q1KnL0KHDefvtERiGQZ069XjllSEAtGrVhsDAWTRt2hwXFxcqVKhIq1ZtbNtav37tfbdVp049/PwG8Nprg3B2dqZq1ZvfOteypQf/+lcEAwe+SMmSJWnfviNPPPFkoe0LPcuomLO3esD+arK3ekDPMnrU6VlGIiJSIAoEEREBFAgi8hAU05lou1CQfa9AEJFC5eDgiMWSWdTD+MuyWDJxcHC8/4q5UCCISKEqU8aJ5OQ/st1HL38Ow7CSnHyZMmXuffH4bnTbqYgUKien8ly+nEBc3DmgeEwdOTg4YLXaQ4CZKFmyNE5O5fPVWoEgIoXKZDLh6lqlqIeRJ/Z4a3B+aMpIREQABYKIiGQpcCDMnz+f4OBg2+tTp07x0ksv0atXL/r378/PP/+co41hGCxYsABfX1+6du3Khg0bCjoMEREpoHwHQnJyMgEBAYSEhGRbPnHiRIYPH87GjRsZPXo048aNy9E2LCyMqKgo1qxZw/Lly5kzZw5Xr17N71BERKQQ5DsQIiIiqFWrFkOHDs223M/Pj3bt2gFQr149YmNjc7QNDw9n2LBhlCxZEjc3N7788ktKly6d36GIiEghyHcg+Pr68vrrr+d4tnifPn1sy4KCgujcuXOOtmfOnOHUqVP079+f3r17c+zYMUqWLJnfoYiISCG4722n4eHhzJw5M9syd3d3QkND79rGMAzmzJnD0aNHWbp0aY73LRYLx48fZ/ny5Vy6dIkBAwbQsGFDatWq9cADv99T++7Fza1wvkziUWFv9YD91WRv9YD91WRv9UDea7pvIHh5eeHl5fXAHWZmZjJu3Dji4uJYunRprt/kU7lyZbp160aJEiWoVq0ajRs35tixY3kKBD3++iZ7qwfsryZ7qwfsryZ7qwdy1lQkj7+ePXs2KSkphISE3PVr3Tw9PQkPD8cwDC5fvkxMTAwNGjQo7KGIiEgeFOpfKiclJbFixQoef/xx/Pz8bMs3btxIREQEO3fuZPr06QwZMoR//vOf9OjRA4vFwsiRI6ldu3ZhDkVERPJI35hWzNlbPWB/NdlbPWB/NdlbPfCITBmJiEjxpEAQERFAgSAiIlkUCCIiAigQREQkiwJBREQABYKIiGRRIIiICKBAEBGRLAoEEREBFAgiIpJFgSAiIoACQUREsigQREQEUCCIiEgWBYKIiAAKBBERyaJAEBERQIEgIiJZFAgiIgIoEEREJIsCQUREgEIIhPnz5xMcHGx7ferUKV566SV69epF//79+fnnn3NtN2PGDLp3706PHj3YvHlzQYchIiIFlO9ASE5OJiAggJCQkGzLJ06cyPDhw9m4cSOjR49m3LhxOdru27ePmJgYwsLCCA0NZfLkydy4cSO/QxERkUKQ70CIiIigVq1aDB06NNtyPz8/2rVrB0C9evWIjY3N0dZisZCWlkZmZiY3btygZMmS+R2GiIgUEpNhGEZBOrg1XTRq1Kgc73344YekpaUxc+bMHO+9/fbb7N+/n+vXr/Pee+8xePDgggxDREQKyHy/FcLDw3P8Qnd3dyc0NPSubQzDYM6cORw9epSlS5fmeH/16tU4OjoSGRnJH3/8waBBg2jcuDFNmjR54IEnJqZgteY9y9zcnElISM5zu0eVvdUD9leTvdUD9leTvdUDOWtycDBRqZLTPdvcNxC8vLzw8vJ64EFkZmYybtw44uLiWLp0Kc7OzjnWiYiIYMCAAZQoUQI3Nzc6dOjAwYMH8xQIIiJSuAr9ttPZs2eTkpJCSEhIrmEAUL9+fXbs2AHA9evXiY6O5umnny7soYiISB4UaiAkJSWxYsUKfvvtN/z8/OjVqxe9evUCbp4VvP/++wCMGDGCzMxMvLy8ePHFF+nVqxctW7YszKGIiEgeFfiiclHRNYSb7K0esL+a7K0esL+a7K0eyN81BP2lsoiIAAoEERHJokAQERFAgSAiIlkUCCIiAigQREQkiwJBREQABYKIiGRRIIiICKBAEBGRLAoEEREBFAgiIpJFgSAiIoACQUREsigQREQEUCCIiEgWBYKIiAAKBBERyaJAEBERQIEgIiJZFAgiIgIUQiDMnz+f4OBg2+uTJ0/i7+9Pz549GThwIOfPn8/RxjAMZs+eTbdu3fD29ubQoUMFHYaIiBRQvgMhOTmZgIAAQkJCsi2fPHkyI0eOJCwsDG9vb+bOnZuj7bZt2zh16hRbt25lwYIFjB8/nszMzPwORURECoE5vw0jIiKoVasWQ4cOzbZ8yZIlmM1mrFYrFy5cwMXFJUfb3bt34+3tjYODA7Vr16Z69eocOXKE5557Lr/DERGRAsp3IPj6+gJkmy4CMJvNXL16FW9vb1JTU1m2bFmOtvHx8VSpUsX22s3NjYsXL+Z3KA/s3zu3YD69D8NqPPRt/VmOO5jsqh6wv5rsrR6wv5qKSz0W99Y81bH7Q+v/voEQHh7OzJkzsy1zd3cnNDT0rm1cXFyIjIxkz549vPnmm0RERODo6Gh73zBy7ngHh7zNXlWq5JSn9QHKlC5BBmByMOW57aPM3uoB+6vJ3uoB+6upONRTpnQJ3NycH3j9vKwLDxAIXl5eeHl5PXCHW7duxcvLC5PJRLt27UhNTeXKlSu4urra1qlatSoJCQm21wkJCdnOGB5EYmIK1jwmunvrLrj16ktCQnKe2j3K3Nyc7aoesL+a7K0esL+ailM9DzrOO2tycDDd94N0od92GhISwrfffgtAdHQ0FStWzBYGAO3atWPTpk1YLBbOnDnD6dOnadSoUWEPRURE8iDf1xDuZtasWUyaNIkFCxbg7OxMUFAQcPMi9M6dO5k+fTrdunUjJiaGnj17AjB9+nRKly5d2EMREZE8MBm5TegXA/mZMoLidWr4IOytHrC/muytHrC/muytHnhEpoxERKR4UiCIiAigQBARkSwKBBERARQIIiKSRYEgIiKAAkFERLIoEEREBFAgiIhIFgWCiIgACgQREcmiQBAREUCBICIiWRQIIiICKBBERCSLAkFERAAFgoiIZFEgiIgIoEAQEZEsCgQREQEUCCIikqXAgTB//nyCg4Ntr0+ePIm/vz89e/Zk4MCBnD9/Pkeba9eu8c477+Dj44OPjw9btmwp6DBERKSA8h0IycnJBAQEEBISkm355MmTGTlyJGFhYXh7ezN37twcbRctWkT16tXZtGkToaGhzJw5k0uXLuV3KCIiUgjM+W0YERFBrVq1GDp0aLblS5YswWw2Y7VauXDhAi4uLjnatmjRgtq1awNQqVIlKlSowKVLl6hcuXJ+hyMiIgWU70Dw9fUFyDZdBGA2m7l69Sre3t6kpqaybNmyHG09PDxs/966dSvp6ek8+eST+R2KiIgUgvsGQnh4ODNnzsy2zN3dndDQ0Lu2cXFxITIykj179vDmm28SERGBo6Njrn3PmDGD//mf/8Fszls2VarklKf1b+fm5pzvto8ie6sH7K8me6sH7K8me6sH8l7TfX8Le3l54eXl9cAdbt26FS8vL0wmE+3atSM1NZUrV67g6uqabb1ly5axePFiFi9eTL169fI0aIDExBSsViPP7dzcnElISM5zu0eVvdUD9leTvdUD9leTvdUDOWtycDDd94N0vqeM7iYkJASz2UyXLl2Ijo6mYsWKOcJgx44dhIaGsnLlSqpVq1bYQxARkXwo9ECYNWsWkyZNYsGCBTg7OxMUFATcvAi9c+dOpk+fTlBQEGlpaYwYMcLWbtq0aTRq1KiwhyMiIg/IZBhG3uddHgGaMrrJ3uoB+6vJ3uoB+6vJ3uqB/E0Z6S+VRUQEUCCIiEgWBYKIiAAKBBERyaJAEBERQIEgIiJZFAgiIgIoEEREJIsCQUREAAWCiIhkUSCIiAigQBARkSwKBBERARQIIiKSRYEgIiKAAkFERLIoEEREBFAgiIhIFgWCiIgACgQREcmiQBAREUCBICIiWQocCPPnzyc4ONj2+uTJk/j7+9OzZ08GDhzI+fPn79o2MzOT/v37s27duoIOQ0RECijfgZCcnExAQAAhISHZlk+ePJmRI0cSFhaGt7c3c+fOvWsfCxYs4PTp0/kdgoiIFCJzfhtGRERQq1Ythg4dmm35kiVLMJvNWK1WLly4gIuLS67tDx06xPHjx/H09MzvEEREpBDlOxB8fX0Bsk0XAZjNZq5evYq3tzepqaksW7YsR9uUlBRmzZrFwoULCQwMzNf2K1Vyylc7ADc353y3fRTZWz1gfzXZWz1gfzXZWz2Q95ruGwjh4eHMnDkz2zJ3d3dCQ0Pv2sbFxYXIyEj27NnDm2++SUREBI6Ojrb3J0+ezIgRI6hcuXKeBnu7xMQUrFYjz+3c3JxJSEjO93YfNfZWD9hfTfZWD9hfTfZWD+SsycHBdN8P0vcNBC8vL7y8vB54EFu3bsXLywuTyUS7du1ITU3lypUruLq6AjfPDvbt28evv/5KUFAQsbGxREdHYzab6dmz5wNvR0RECle+p4zuJiQkBLPZTJcuXYiOjqZixYq2MABwcnIiMjLS9nr8+PG0aNFCYSAiUsQK/e8QZs2axZIlS+jVqxeffPIJQUFBwM2L0O+//35hb05ERAqJyTCMvE/EPwJ0DeEme6sH7K8me6sH7K8me6sH8ncNQX+pLCIigAJBRESyKBBERARQIIiISBYFgoiIAAoEERHJokAQERFAgSAiIlkUCCIiAigQREQkiwJBREQABYKIiGRRIIiICKBAEBGRLAoEEREBFAgiIpJFgSAiIoACQUREsigQREQEUCCIiEgWBYKIiACFEAjz588nODjY9vrkyZP4+/vTs2dPBg4cyPnz53O0MQyDBQsW4OvrS9euXdmwYUNBhyEiIgWU70BITk4mICCAkJCQbMsnT57MyJEjCQsLw9vbm7lz5+ZoGxYWRlRUFGvWrGH58uXMmTOHq1ev5ncoIiJSCMz5bRgREUGtWrUYOnRotuVLlizBbDZjtVq5cOECLi4uOdqGh4czbNgwSpYsiZubG19++SWlS5fO71BERKQQ5DsQfH19AbJNFwGYzWauXr2Kt7c3qampLFu2LEfbM2fOcOrUKRYtWkR6ejrDhw+nVq1a+R2KiIgUgvsGQnh4ODNnzsy2zN3dndDQ0Lu2cXFxITIykj179vDmm28SERGBo6Oj7X2LxcLx48dZvnw5ly5dYsCAATRs2DBPoVCpktMDr3snNzfnfLd9FNlbPWB/NdlbPWB/NdlbPZD3mu4bCF5eXnh5eT1wh1u3bsXLywuTyUS7du1ITU3lypUruLq62tapXLky3bp1o0SJElSrVo3GjRtz7NixPAVCYmIKVqvxwOvf4ubmTEJCcp7bParsrR6wv5rsrR6wv5rsrR7IWZODg+m+H6QL/bbTkJAQvv32WwCio6OpWLFitjAA8PT0JDw8HMMwuHz5MjExMTRo0KCwhyIiInlQ6IEwa9YslixZQq9evfjkk08ICgoCbl6Efv/99wEYMmQIlStXpkePHgwYMICRI0dSu3btwh6KiIjkgckwjLzPuzwCNGV0k73VA/ZXk73VA/ZXk73VA4/IlJGIiBRPCgQREQEUCCIikkWBICIigAJBRESyKBBERARQIIiISBYFgoiIAAoEERHJokAQERFAgSAiIlny/QU5Rc3BwVQkbR9F9lYP2F9N9lYP2F9N9lYPZK/pQeortg+3ExGRwqUpIxERARQIIiKSRYEgIiKAAkFERLIoEEREBFAgiIhIFgWCiIgACgQREcmiQBAREeAvFAibNm3C29ubF154gRUrVhT1cArFoEGD6N69O7169aJXr14cPXq0qIeULykpKfTo0YNz584BEBUVhY+PD126dOHjjz8u4tHl3Z31TJgwgS5dutiO07ffflvEI8ybTz75hO7du9O9e3fmzJkDFP9jlFtNxfk4zZ8/H29vb7p3786SJUuAfB4j4y/g4sWLhqenp3H58mXj2rVrho+Pj3HixImiHlaBWK1Ww8PDw8jIyCjqoRTIDz/8YPTo0cN46qmnjN9//924ceOG0b59e+Ps2bNGRkaGMWzYMGPXrl1FPcwHdmc9hmEYPXr0MOLi4op4ZPmzd+9eo3///kZaWpqRnp5uDBo0yNi0aVOxPka51bR9+/Zie5z2799v+Pv7GxkZGcaNGzcMT09P4+eff87XMfpLnCFERUXRsmVLKlSoQNmyZenatSvffPNNUQ+rQP7zn/9gMpkYPnw4PXv2ZPny5UU9pHxZs2YN//jHP6hSpQoAMTEx1KxZkxo1amA2m/Hx8SlWx+rOeq5fv86FCxeYNGkSPj4+BAUFYbVai3iUD87NzY3x48dTsmRJSpQowRNPPMHp06eL9THKraYLFy4U2+PUokULli5ditlsJjExEYvFwtWrV/N1jP4SgRAfH4+bm5vtdZUqVYiLiyvCERXc1atXadWqFQsWLCA0NJRVq1axd+/eoh5Wnk2fPp3mzZvbXhf3Y3VnPYmJibRs2ZIZM2awZs0aDh48yNq1a4twhHlTp04dmjRpAsDp06fZunUrJpOpWB+j3Gpq27ZtsT5OJUqUICgoiO7du9OqVat8/z/6SwSCkcsDXU2m4v2o26ZNmzJnzhzKli2Lq6sr/fr1Y/fu3UU9rAKzt2NVo0YNFixYQKVKlShTpgwDBw4slsfpxIkTDBs2jHHjxvG3v/0tx/vF8RjdXpO7u3uxP05vv/02+/btIzY2ltOnT+d4/0GO0V8iEKpWrcqlS5dsr+Pj422n9MXVwYMH2bdvn+21YRiYzcX26y1s7O1YHT9+nG3bttleF8fjdOjQIYYMGcJ///d/07t3b7s4RnfWVJyP06lTp/j5558BKFOmDF26dGH//v35OkZ/iUBo3bo1+/btIykpiRs3brB9+3batWtX1MMqkOTkZObMmUNaWhopKSmsX7+eF154oaiHVWCNGzfmt99+48yZM1gsFjZv3lysj5VhGMyYMYMrV66QkZHB6tWri9Vxio2N5a233iIwMJDu3bsDxf8Y5VZTcT5O586dY+LEiaSnp5Oenk5ERAT+/v75OkbFIwILqGrVqowZM4ZBgwaRkZFBv379eOaZZ4p6WAXi6enJ0aNH8fX1xWq18tJLL9G0adOiHlaBlSpVilmzZjFq1CjS0tJo37493bp1K+ph5Vv9+vV5/fXXGTBgAJmZmXTp0oUePXoU9bAe2OLFi0lLS2PWrFm2Zf7+/sX6GN2tpuJ6nNq3b2/7XeDo6EiXLl3o3r07rq6ueT5G+sY0EREB/iJTRiIicn8KBBERARQIIiKSRYEgIiKAAkFERLIoEEQKYPjw4Zw8eTJPbd544w3WrVv3kEYkkn9/ib9DEHlYvvjii6IegkihUSDIX9LOnTtZuHAhGRkZlC5dmnHjxhEZGcmJEye4dOkSiYmJ1K9fn+nTp+Pk5MSXX37JqlWrKFGiBKVKlWLKlCk8+eSTdOzYkfnz59OoUSNWr17NsmXLcHBwoHLlykyaNInatWsTFxfH+PHjiY+Pp3r16iQmJtrGcerUKaZPn84ff/yBxWJh4MCB9OvXj2vXrjFhwgTOnDmDg4MDTz31FFOmTMHBQSf18hA9vKd0izyafvvtN6NHjx5GUlKSYRiG8euvvxoeHh7GrFmzjHbt2hkJCQmGxWIx3n33XWPWrFlGZmam8dRTT9melb9+/Xpj1apVhmEYhqenpxETE2NERUUZnTt3NhITEw3DMIyvv/7a8PLyMqxWqzFy5Ejj448/NgzDME6fPm00adLE+Prrr42MjAzD29vb+OmnnwzDMIyrV68aXl5expEjR4z169cbw4YNMwzDMDIzM43333/fOH369J+5m+QvSGcI8pezd+9e4uPjGTJkiG2ZyWTi7NmzdOvWjcqVKwPQr18/ZsyYwbhx4+jWrRv+/v506NABDw8PfHx8svX53Xff4e3tjaurKwB9+vRh+vTpnDt3jqioKMaNGwdAzZo1ef7554Gbj14+e/YsAQEBtn5SU1M5duwYbdu25eOPP2bgwIG0bt2awYMHU7NmzYe5W0Q0ZSR/PVarlVatWjFv3jzbstjYWFavXk16enq29W5N0QQGBvLrr78SFRXFF198wdq1a1m4cKFtXSOXJ8AYhkFmZiYmkynb+7eeommxWHBxcWHjxo229y5duoSzszOlSpXi22+/Zf/+/URHRzN06FAmTpxYrJ4ZJMWPJiTlL6dly5bs3buXU6dOAbB792569uxJWloaERERJCcnY7VaWbNmDZ6eniQlJdG+fXsqVKjAkCFDGD16NMePH8/WZ5s2bdi6dStJSUkAfP3111SoUIGaNWvStm1bVq9eDcCFCxfYv38/ALVr16ZUqVK2QIiNjaVHjx789NNPfPnll0yYMIE2bdowduxY2rRpw4kTJ/6sXSR/UXq4nfwlhYeH89lnn9meex8QEMC+ffuIjo7GYrFw+fJlnnvuOSZOnEjp0qVZtWoVS5cupXTp0jg6OjJmzBhat26d7aLyihUrWLVqFVarFVdXVz744APq1KlDUlISEyZM4OzZszz22GNkZmbSu3dv+vTpwy+//GK7qJyZmcmgQYMYMGAA169fJyAggOPHj1OmTBmqV6/O9OnTKV++fFHvOrFjCgSRLMHBwVy+fJkPPvigqIciUiQ0ZSQiIoDOEEREJIvOEEREBFAgiIhIFgWCiIgACgQREcmiQBAREUCBICIiWf4/HLka4/aihHoAAAAASUVORK5CYII=\n" - }, - "metadata": {} - } - ], "source": [ "cfg = QlearningConfig()\n", "env,agent = env_agent_config(cfg,seed=1)\n", @@ -224,7 +162,55 @@ "rewards,ma_rewards = eval(cfg,env,agent)\n", "save_results(rewards,ma_rewards,tag='eval',path=cfg.result_path)\n", "plot_rewards(rewards,ma_rewards,tag=\"eval\",env=cfg.env,algo = cfg.algo,path=cfg.result_path)" - ] + ], + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Episode:10/200: reward:-287.0\n", + "Episode:20/200: reward:-142.0\n", + "Episode:30/200: reward:-67.0\n", + "Episode:40/200: reward:-61.0\n", + "Episode:50/200: reward:-74.0\n", + "Episode:60/200: reward:-41.0\n", + "Episode:70/200: reward:-55.0\n", + "Episode:80/200: reward:-66.0\n", + "Episode:90/200: reward:-31.0\n", + "Episode:100/200: reward:-31.0\n", + "Episode:110/200: reward:-58.0\n", + "Episode:120/200: reward:-25.0\n", + "Episode:130/200: reward:-18.0\n", + "Episode:140/200: reward:-27.0\n", + "Episode:150/200: reward:-28.0\n", + "Episode:160/200: reward:-25.0\n", + "Episode:170/200: reward:-35.0\n", + "Episode:180/200: reward:-13.0\n", + "Episode:190/200: reward:-22.0\n", + "Episode:200/200: reward:-26.0\n", + "保存模型成功！\n", + "结果保存完毕！\n" + ] + }, + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEXCAYAAABCjVgAAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAABRX0lEQVR4nO3dZ2AU5drw8f/sbrakF1KoofdOkE6kqAQIHQURQeztIOeVQ8ByFEUFUbGCciyPWACRIkoTFBAivYsghBIIIZvek23zfliyEJIAAUKie/2+wE5mZq+5d3euucvco6iqqiKEEEIAmsoOQAghRNUhSUEIIYSLJAUhhBAukhSEEEK4SFIQQgjhIklBCCGEiySFf6D333+f6dOn35L3evjhhzl+/Pgtea/KNnfuXG6//XamTp1a4m/5+fm89dZb9O/fn+joaKKjo3nnnXfIz893rdOkSRPS0tIqPM4NGzbw6quv3rT9Xem4yyMuLo6nn36a6OhoBg0axH333ceuXbsAOHv2LO3atQPg22+/5ZNPPgFgy5Yt9OrVi+HDh7N371769u3L0KFDad26NevXr3fte8uWLTRp0oSFCxe6lh04cIBu3bpxpVH3vXv35uDBg2zfvp2BAweW+PvBgwf517/+dUPHfa2WLFlCVFQUd955J//973+xWq235H1LUMU/znvvvae+/PLLlR3GP07v3r3VnTt3llhutVrVUaNGqa+88oqan5+vqqqq5uXlqa+88oo6atQo1Wq1qqqqqo0bN1ZTU1Nvacw3Q1nHXR5xcXFqt27d1M2bN7uWxcbGqh06dFD/+usv9cyZM2rbtm1LbBcTE6N++OGHqqqq6vvvv69OmzZNVVVVfe2119RXX33Vtd706dPVRx99VH344Yddyz7++GM1JibminH16tVLPXDggLpt2zZ1wIABN3SMN+Lo0aNqz5491dTUVNVut6uTJk1SP/nkk0qJRVc5qajqczgcvPbaa+zfv5/c3FxUVeXVV1+lcePGREZGsnbtWoKDgwG4++67efLJJ+nSpQuzZ89m586d2O12mjdvzvPPP4+3tze9e/emdevWHD16lH//+9/odDo+/vhjLBYLaWlpDBkyhGeeeQaATz75hCVLluDl5UVERAQbNmzgl19+wWKxlLn/siQlJTF9+nQSExOxWq0MGDCAxx57DIB58+axfv16CgsLyc/PZ8qUKdxxxx28//777Nu3D7PZTJMmTQgPDychIYHk5GQSEhIIDAzknXfeITQ0lN69e/Puu++Sl5fHO++8Q+3atTl27BgWi4UXX3yRzp07k5aWxtSpU4mPj8ff35/g4GAaNWrE008/XSzW3NxcXn31Vfbs2YNWq6Vv375MmjSJqVOn0qhRIx588EEAYmJiXK8vLdenn36auXPnsnLlSgCysrLo06cP69evp6CgoMxyuNT58+d56aWXSEhIQFVVhgwZwkMPPcQzzzxDUlISzz33HBMnTqR///6ubX7++Wfy8/OZNm0aGo2z8m0ymXjuuecYMmQI69atK7Y+wHfffce3336Lw+HA39+fF154gQYNGnDy5EmmT59OXl4eZrOZpk2bMmfOHAwGAy1btqRPnz4cOXKE2bNnc++99/LII4+wdetWzGYz999/P+PHj2fp0qWsXbuWjz/+mLFjx9K2bVv27NlDYmIiHTp0YObMmWg0GpYuXconn3yC0Wikc+fOfPnllxw+fLhYnJcfd/v27Ustn7NnzzJmzBgaNGhAQkICCxYsICQkxLWf+fPnM3z4cHr06OFa1qVLF9566y2MRmOx93z//fdJT0+nRo0abNiwAYPBwPLly8nNzcVut1NQUMCwYcOYNWuWa5tff/2VTz/9lLvvvpu8vDw8PT35/fffueeee0hJSeHFF18kNTWV5ORkatasyZw5cwgKCir1N7Nr1y4mT57MW2+9hdVq5ZVXXuHHH38kJiYGb29vjh49yvnz56lfvz5vv/02Xl5ebNq0idmzZ6PRaGjWrBmxsbF888031KpVq9i+R40axfjx4+nXrx8As2fPRlVV1zkiMDAQgHvuuYdXX32Vhx9+uNQYK5I0H5Vh//79mM1mFi1axKpVqxg6dCjz58/Hx8eHO+64gx9++AFwVomTk5Pp0aMHn3zyCVqtlqVLl/LDDz8QEhLC7NmzXfts1KgRq1evpm/fvnz22We88cYbLF26lEWLFvHJJ5+QlpbGb7/9xtKlS1myZAlLly4lNzfXtf3V9l+ayZMnM3z4cNc+Y2NjWbVqFQkJCcTGxvLVV1+xcuVKJk2axHvvvefaLiEhgWXLlrn2v2vXLt59913WrFmDr68vixYtKvFeBw4cYMKECSxfvpwRI0bwwQcfAPDqq6/SsGFDVq9ezbvvvsuePXtKjfW9996jsLCQVatWsXz5cvbs2cOOHTuu+lkVlWtUVBS5ubkcPHgQgB9//JHIyEj8/PzKLIfLPfvss3Tq1ImVK1fy7bff8sMPP/DTTz8xZ84cV3lffoLfvXs3HTp0cCWEIoqi0LVr1xLHu2PHDpYvX87XX3/N8uXLeeihh1wJcvHixQwZMoRFixaxbt06zp49y8aNGwGwWq306tWLtWvX0qpVKywWCwEBASxcuJD33nuPt956i8LCwhLHFB8fz4IFC/jhhx/Ytm0bO3bs4Pjx48yePZsvvviC5cuX4+3tjd1uL7Ht5cddVvmAM6E+8cQTrF27tlhCADh06BDt27cvsf/IyEhq165dYjnAQw89RO/evRk/fjzr1q1j1KhR9O/fn7feeouOHTsSHx9PRkYGR48exc/Pj3r16tG6dWu2bt2KxWLhwIEDdO/enZ9++om2bduyaNEiNmzYgNFoZMWKFaW+57Zt25g6dSpz584tNd5Dhw7x6aefsmrVKsxmM2vWrCE9PZ3//Oc/vPnmm6xYsYJOnTqRlJRU6v5HjhzJsmXLALDb7fzwww+MHDmSxMREqlev7lovLCyszH1UNKkplKFdu3b4+fmxcOFCzpw5w/bt2/Hy8gKcH+zLL7/Mgw8+yPfff8+wYcPQaDRs3LiR7OxsYmNjAeeP+NKrkYiICMB5spg3bx4bN27kxx9/JC4uDlVVyc/PZ9OmTfTr1w9fX18AxowZw7Zt2wCuuv/L5eXlsXPnTjIzM3n33Xddy44cOUL//v2ZOXMmK1eu5PTp064aUZG2bdui0138etx2222uGknz5s3JzMws8X41atSgWbNmrnWKvvybNm1y/T8kJMR1lXS52NhYpk6dilarRavV8tVXXwG4ti3LpeU6YsQIli1bRqtWrVi6dCmTJ0++ajlcWl579uzhs88+A8DHx4dhw4axefNmBgwYcMUYrsThcBR7vXHjRk6fPs2oUaNcyzIzM8nIyGDy5Mls3bqV+fPnc+rUKcxmM3l5eSWOtUifPn0AaNGiBRaLpdi6RXr16oVGo8Hb25vw8HAyMzM5cuQI3bp1IywsDID77ruP999//4rHcaXyadOmDTqdjrZt25a6raIoJcrhRuj1ejp16sSuXbs4fvw4t99+u+tYt2zZgq+vLy1atMDb25tx48axa9cuPv/8c06dOsWxY8do06ZNiX2eP3+exx57jNGjR9O0adNS37dHjx7o9XoAGjduTGZmJrt27aJBgwaubYYOHVpmn05UVBSzZs0iOTmZw4cPEx4eTt26dUvt97j8IuNWkaRQho0bNzJjxgweeOAB+vTpQ/369V21g4iICGw2GwcOHODHH390dW45HA6mTZtGZGQk4GwOufTKzdPTE3D+uIYOHUrfvn2JiIhg+PDhrF+/HlVV0el0xb4gWq3W9f+r7f9yDocDVVVZuHAhJpMJgLS0NAwGA3/88QdPPPEE48ePp1u3bnTs2JGXX365RKxFLq3iK4pS6pe4rHUuP6ayvuw6nQ5FUVyvExMTMRqNJd7v8g64S2MdPnw4Q4YMYeTIkWRnZ9OpUydycnLKLIfSyuvyZTabrdR4i7Rv356PP/4Yh8OBRqMhJycHRVHw8vJi+/btjB8/vsQ+Bw8ezOTJk12vzWYzfn5+TJo0CbvdTlRUFLfffjuJiYnFYrr8cyk6hqJyu9bPRavVlvk9K8vVykev1xe7kLhU27Zt2bdvH7169Sq2/IMPPqBOnTqlXpVfTY8ePdi5cyf79+9n2rRpgLPmsWjRIgIDA12J4s033+TAgQMMHz6cTp06YbPZSi0nrVbLJ598whNPPEFUVBStW7cusc61lCVc/I4/99xzHDp0CHA2HY0ePZq77rqLH3/8kb179zJy5EgAqlevjtlsdm2flJTkSti3mjQflWHr1q306tWLe++9l1atWrF+/fpi1euRI0fyyiuv0KRJE2rUqAFA9+7d+frrr7FYLDgcDl544QXefvvtEvs+ffo0OTk5PPPMM/Tu3ZsdO3a4tomMjGTdunVkZ2cDzhEJRa51/0W8vb1p27Ytn3/+OeBsYx89ejQbNmxg586dtGzZkgceeIDbbruNDRs2lNp8cDNERka6jiM9PZ3169cXO/kX6dKlC8uWLcPhcGCxWPjXv/7Fzp07CQgIcP2w0tLSXCNWShMaGkqbNm148cUXGTFiBHDlcriUt7c3bdq04euvvwYgOzub5cuX07Vr1yse35133omXlxevvfYaBQUF/PnnnwwbNozHHnsMrVZbormpW7du/PTTT66TwLfffsu4ceMA5yiaJ598kv79+6MoCvv376+Qz6V79+78/vvvriaK77777qrbXG/5ADz44IN89913bNmyxbVs8+bNLFiwoMyr8qvp2bMnW7du5dy5c7Rq1QrA1RS1fv1618XTli1bGDduHEOGDCEoKIjY2NhSyzQ4OJj27dszZcoUJk+eXGzk2JW0b9+eU6dOceTIEQDWrl1LVlYWiqIwY8YMVqxYwYoVKxg9ejTg7INcunQpe/fu5a677gKco6B++eUXUlNTUVWVRYsW0bdv3+sqlxslNYUyjBo1imeffZbo6Gi0Wi0RERGsW7fOdTU4ZMgQ3n777WIn5SeeeIKZM2cydOhQ7HY7zZo1IyYmpsS+mzRpwu23305UVBS+vr7UqVOHhg0bcvr0aXr06MHdd9/NPffcg9FopFGjRq6r22vd/6Vmz57NK6+8QnR0NBaLhYEDBzJo0CBSUlJcHaAeHh506dKFzMxMcnJybm5BAlOnTuX5558nOjoaf39/atSoUaJzEeCpp55ixowZDB48GLvdTv/+/bnzzjtp1aoVzz77LHfddRe1atXitttuu+L7jRw5kokTJzJ37tyrlsPlZs+ezfTp01m6dCkWi4Xo6GiGDRt2xffT6XR8+umnfPTRRwwdOtR11e3p6UlycjK//fZbsSvkHj168PDDDzNhwgQURcHb25sPPvgARVGYNGkSTz75JH5+fphMJlfb+c1Wr149pk6dyoMPPoher6dZs2au79mVlFU+CQkJV9wuPDycefPmMWfOHGbOnInD4SAwMJC5c+fSuHFjzp49W+5jqF27NjabjW7duhW7yOjRowfr1q2jQYMGADz55JPMmjWLjz76CK1WS/v27a9YpkOHDmXt2rW88cYbJRJ6afz9/Xn77beZMmUKGo2Gli1botPpyizPor/fddddrppe06ZNefLJJxk3bhxWq5U2bdpUSiczIENSq5oDBw6o//d//+d6/dlnn6kTJ06svIBugq+++krds2ePqqqqWlhYqA4fPlzduHFjJUd1a5jN5hsezlkR4uPj1ffff1+12+2qqqrq2rVr1REjRlRyVH9P2dnZ6syZM9W8vDxVVVX10KFDardu3VSHw1HJkV0fqSlUMfXq1WP+/PksXrwYRVGoXr06r7zySmWHdUMaNmzIK6+8gsPhwGq10q9fP1fV/p8uODjYNXS5KgkLC8NsNrtqwj4+Prz22muVHdbfkre3Nx4eHowYMQKdTodOp2POnDmlNpH+HSiqKg/ZEUII4SQdzUIIIVwkKQghhHCRpCCEEMJFkoIQQgiXf8Too/T0XByO8veXBwV5k5p688fl3yiJq/yqamwSV/lIXOVX3tg0GoWAAK8y//6PSAoOh3pdSaFo26pI4iq/qhqbxFU+Elf53czYpPlICCGEiyQFIYQQLlUiKaxcuZL+/ftzxx13uCbbEkIIcetVep9CUlIS77zzDkuXLkWv1zNq1Cg6depEw4YNKzs0IYRwO5VeU4iNjaVz5874+/vj6enJXXfdxZo1ayo7LCGEcEuVnhTMZnOxCcNCQkIq7TF0QojiZGq0K/snlk+lNx+VVqjlnV0wKKjsB9dfTXCwz3VvW5H+7nGpqsqhuFSa1w9Cq3F+nlabHZtdxWSomK9dabGZ0/NITMmlTaMbm6lUVVXOmnOoGeyNRlO+7+fVyszuUNmwM55thxJpWb8aw3rd3KZTm93B0dPpBPoaqV7t4vj0y+M6ciqNrDwLtzV3PvErK9dCzIe/MbhnA+7qXNcVa36hDW+Th2u7pLQ8DB5a/H2KP8nuUqqq4lBBq1FIzy4gMSWXxnUC0GlLXpdaUQjwNaKqKt+uO0qXVtVpXi+II6fS0HtoqV/T77rKIa/ASuyBc+Tk2yiw2MjILqRvxzrUr+nH8k3HadmgGo3rBBTb5kRCJruPJJGVe5qxUc3Qe1x8Qt2huBRmf72bCdEt6NmuFqcTs9BqFUIDPfHQFX+S3S+74tm0J4FhtzckM7eQo6fTaVE/iJPnsjh+NoMRvRtx1pzNqthTPDumA7VDfVBVla/WHGHHH+fR6TQM6FqXhrUD2LTnLO2aBNO64cXv9M08X1T6LKnLli1j165dzJgxA4APP/wQVVV56qmnrnkfqak51zVONzjYh+Tk7HJvV9Eujctmd7Dol+Pc2bE2wf5XfwjKrYoLYO2OeKr5mejQpOQJ9+CJVN5ZvJ8BXcIZHul82MmnPx7mRGIW0x+8De1lj+RUVbVcFwOXr39pbLuOmEnPKeSOiNq8s3g/B0+k0qd9Le7p4zzZfrTsEO0aVaNHmxqu7R2qiuaS/aVlFWDUa/E0Ok9+P+86w7frj9EsPIAJ/ZsR5Gd0XdBcHnd+oY3UzAJqhXgTHOyD2ZzFjj/NFFrt1KvuS61gr2Lb/PT7Kb7fdAIPnQaNRmHOU91ZvPE48UnZ9LutDpv2neNMcg51QnwY1achgT5G/m/NETo0CaZDkxDXfs6Yc1i44Rg2u4PGtf0Z1K0uf55OZ/7Kw+QWWDEpNnq1CmJAj0bUDg8jJe3ik8USUnKZ8eUuTAYdbz3ZDYCVW0+y7LeT6LQaXhwXQV6hjW/W/8X5tDym3dcBT6OORb8cZ/fRZDSKQusGQQzuXo8gPyPHEzJxOJwXABarncW/HkdVYXhkfRasPUp2XiF+BoV+EaG0qOXDH3HnqR7kjdVh56dfDxNgcODrYcOSn4d3vZbcM7gbE9/bQk6+lY5NQ2jXuBrJ6fkcOJFKTp6V6kFe9GlfncK8fA7HnScuPpnaQQbCg/Ro7BayMrM5n5yBxl6ISbHgpRTipbGCRkOQnycZ6RlodB60blabP+LMGDQOfI0KyalZ6LDjQENolwE0bH8bR06nU2C18+VPBzFaMjHpbDQO8yQ+MQ0ddkxaB+HBJvy8dOh1GgotNk7EJ+OlsWKgEKNixaRYMSoWjIoVo8aGQwU9NgyKFbvWgF+AH6l5kJ+Ti6+HFb1qQVHtKKgogIJKkldjmtw3tdznMY1GueKFdKUnhaSkJEaPHs2SJUswmUyMGjWKV155pdTno5bln5wU/jydzpvf7qVnm+qMj2p2U9/H4VDZsOcsial5hId6E9m25jXHlZiay3Pzt6MA4/s3pUfrGsXW/X5THD/9fhqAp4e3ok2Dakx87zdyC2w8HN2cLi0uPn921bbT/HYgkf+MbkdugZVjZzPp1e5iLDn5Vox6reuq8vc/zvPl2qN0bh5KdNe6BPoa+WXfORKSsukbUYuXP9+Jza7y4vgIpn+xi+AAE0lpeQzqVpca1byYt+IPFODxIS3p0CSY5b+d5KffT6PTKtzZsRa92oTx6ufbqBbkQ8zY2zh1Pps3FuykWbBKXmYGRh0Mvb0xv+87TV5eAQO71iU924LVFEDrts1ZsjGOVdtO07pBEP261GPrvrNsPXT+wtGo1Ao0MqRnA9o3DSMr18LzH//GbTVUetaFVbEnaNakDgePJuClseBJAf4eFmr6qFhys1B1RkyeRrLTM9AoKgE+zmdYF1gd5OVb8NJa8dZawWYBnQGH3YanxoIRCwrFfyM2rRFTp5HQOJKXPt9JdkYGqqrh3Wedj4Gc/FEsoYGeJKXlkZ1nRQVCvMGbPPRaBY29EL09j471PdFZ8zifaEZvz8eoseKBDb1iQ48dD8WGSetAhxWdakOv2PFQrv0Ro3Y0aBp0ZsfhJEJNVrTWPDyw4oEdo9aBh2JD67CiVcpxDtBoUT08ySuwoqh27DoTqs2CSbFiRYsdLRaHBg+9AS9vE7npqfhqCrDrvSkosKBVHBiVKz+7uyQFu1aPQ2dC7+lFgapHZ/REZzByLjUPrYcBjd5IXHwyvjorWocFH18fatcKAYMniWkFFFhVagZ7k5CSR453HW7r2/eflxTAOST1448/xmq1MmLEiHI/hq4ik4I5PY83v93L5HvbE3KLrtQvjavoak3voeHtJ7vjabxy00t5rrgPn0pj9sJ9aDUKOp2GD57p4bqC33ssmbXb48krtPPc/R2wWO1kFdqpGeAsg6/WHWXz/nM0rOnH0fgMnrs/gnyLjU9/PEzMfR34/Kc/yS2woVEgt8DKo4Na8tpXu9FpFUIDPHn5wdvQKAppWQVM/WQbVpuDWsHepGblUVBo592JPfE2ebD1YCJfrfuL2iHe/L9RbTmVmMVbi/YR6GskLasAPy89o/s2Zu7SfRgVK14eDkK0WWhthRi9vTHkmRnYQk/SufOk5ILe4IGvPQNPjRW7zYpR60DjsGHQqmhUGzounqzsqoJq8CW/0Ion+Vc96dhVBWPrO9kQp5KXkYq3ko/OUYhJsVLX146PkotSkIWiOgBwKFqs6PBwFHKlFilV74nG5EOew0BGeiYeih2NyQe7qpCTb0GrgF6roNd7EBgUgM7Tm6QsG/FnU9B6eNCySS2M3j4oBk8SM+zEHjiDYiukocd5mnicx2YKJD83Dx9NAQWqB0rL/py1+bN7zxEGNLShK8ykMDsDTzUfnS237EAVLYUaIzatEb3RBDo9dkWHDQ98/byxoeNUSiG1qwfi5e2FotNzLsNKRr5KvVpBnE/LwwY0qhuG1uAJBk+W/XaKamd+pZ3hNJkWHZ7+gZj8Aiiwa9HqDRg9TaB1vo85247J04Sfnxc6gwl0BhSdHjyMKDoDeOhRdAYUg5fzb4pCYmouh0+l06t9TVb9fpqVsad4algrmtbxJyEll/BQHxRF4dn3NzIkNJ5QTSZ/ncuhTaNQfPz98awWRoGqx6Jq8PPzQdF5gFaPovUAjQabXcVqd+Dp7e2MQ7lyN66qqsxfeZiUzAJ6d6jJbc1Ci9VgS/OPTAo3qiKTwu+HzjP/x8M8OqgFnZqHXvO+HQ6VrDwL/t5lt7NeS1zvLN7P8YRM8gttjLmjMX061LrQPqui1WhQVZWMHAt5BVaW/XaSuIRMXnmoE4UWOz/vOsOALuEA7PjTTKCPgWr+Jvy89Ph66Vmx5SQ/bDnJ2H5N+HLNUV4YF0G96r78sucsX637i0BfA2lZhQzuXo9DJ1OJS8hi0t1taFDDj//34VY6NAnm3sjavP1/m/D18cRWkEdhdiat2zZl74GT3FEnjxqGPP44mYbO2w+vvETq+1rIyLHgWy0Y/5BQ/krIwpaRTC1vG3l5+fhoCrCpGhwtokg11GbV5iPU9teQm51NsJeKLT+XMJOFNqFgzzKTn5uPHQVfJZ8yfzsGL+wGPzLSs9AqdhSfEPyDAjmfYSUxw4K/nxcNw6uBRsf2I6mcy7DSomEI8fFJmOw5qCi0bFaHarXCUTx9Scu1s3VvPG2b1cTfz5vf9idgsdrxOb+brsZjF9/XwwgeJuwaA3q/IBSvADSeftgVPb//kUhmZg5GjY3atUJp2qYFmoCa/LIvid93H6Nt81oMvL0litELRXPxQuDLNUc4eiaD5++PwGTQYXc4SjTFFTl2NoMAHwPV/IpfzOQWWMm1qkz98DeeaZ2CX94Z/jxXQP0mjUg5dpCW+ovPS1Y8/dH4hqAYfVBMvig+1dB4BZCWY8XLxweTrz+K0RvF6HPhpHdjTxu7/De5Zns8i389zt29GrL41+O88WhnQgI8b+g9rsRitRfrNyjy1uL9FBbaCPQ1EJeQxZtPdK2wGMrrZieFSu9oruqSM53trimZ+aX+fd2OeI4nZPLE0FbFli/+9Tib9p/j7Se7XXfHqkNViUvIpGPTEE4nZbN5/zn6dKjF2h1nWLXtNBNHtGbD7rNsO+wcraXTKtjsKruOmIk357BxbwIH4lKx2hykZhUU2/e0+zpw/GwGNYO9aduwGl9ylKPxGWTmWPh63V+0b+DPo7cH8tP6fcRt30R1TR6tfC2cXXeADJOOYfp0OqanY/82jYl6oBBQAF/gxM/09AbSAKM3bQ1WvGyFpOn98QmtxzlLJo7UdHS5SQQVWlG9/fGtEY6mUMVu8CH+6F80+3MFPsATvoADuNA/qpoAkx9aQtDVbEpmjsqx+FRat26Ef1AgFocGY1B1jputLF1/iPYdWtCvVxsAFq44xL5jKcwe1w1vkwcNgcu7dDu1s/Hn6XRaNapG4o4zfPPrcYb1rE/1rnVd64QAQ5t1cL0eVL/5hVqXnvA+I/m/NX/SsW0jhvVpXuYPtmc7O3EJWdQO9cbLeLHTtlOnYE7nGons2QCNZ8kLivv7NcXhUF2d3WUlBIBGtfxLXe5l9CC8ljd6vY6DHu0wBEew5kQ8s7p25fW9wUzo5svuP84SEFyNscO6lXqiv/bLoxsTGuhMaAfiUtBqFIL8jBX6fqUlBIA6YT5s2n2W7DwL4WFVcxDIzSJJ4SpSMpwn09SswhJ/yyuwsWLrSfIL7aRnFxJwYfSFOSOfDbvPYneoHD6VTliQJ5v2JTCqdyPXj3nB2qPotBru7t0Ac3o+HjpNiSu6xJRc8gptNKrlR7C/ke83nSAn38r+4ynk5Ft5bcFuVODOjrWpWc2LZnUDeGfxfn47kMj5tFwa1vQjISUHo17HlHvbodNAdnISazcdYOeuP8k9f47+NbLQ/7qB5wPN5P4RgNVq47nAHIIzMilYZqcP0OfCb0BFwaGCWqCgehkxhDZDE9IbPANYHRuHl483QcHB7Nn9B4WqjnvHDiEguBo/bz7B6tg4orrWZ1jPBtRIy+Olz3dSmGmnVf0gnhjaEoOHFhPOjvXn927inoYakpOSKVAN3D+oHYrehKL3BA9DsSp4OFDT7qB6mB/JydnoLyxvUl3ldl0w7S4ZdTQ+qikZPSzFRs5czmTQ0b6xc5s7O9ambpgPjev4X/V7EhbovHo9aNaQZPWhWrUrb+Oh09I0PKDEcl9PPQ8NbH7Fbcs7+qk0iqIQ4m/CnJGPUa8lyNeIv7ceg4eWv7K82J/hy9BWNSv9OcNF5XrsbCYhAaYrJsGKVCfUh7xCG3mFNrq2DLv6Bn9jkhSuIjmj7JrCpv0J5Bc626CPnE6ny4Uvy9JNca52+oMnUtn+ZxK7jpjp1CyUBjX9MKfn8eveBMDZrn8uJReTQcczI9vQsNbF4XbHEjIBaFjTj7RsZ1KKS8jk1Pls2jcOJifPQkTTEPpG1HZt07l5KMt+O0GQJof7Glnxr5+KJvs8yrafcGSZCXHYeMITMAOeQAao2nCsxiAMOSlo0eAZEoq+Tlc0QbXR+ASTU2DDK7AaYXXr8NvuMwT5GUv0rwxu1BVFUcgrsDJvuw0/LwMBwdUA6NWuJn+cTKPThaGOoYGePDqoBccTMhnSo16xYYk6rYZq/iaO5nnxZ5rCbc1D0QbWuuJnVNqwRkVR6NqyerFlRr2OsMBr/8prNEqpJ+7SBPgYMHhoORCXCnDL+p9uRIi/iYSUXIx6LcH+zqafYH8T+46nAFA7pPKviIP9TSiKcyhsUYKoDHUuqR3UCa38cqlIkhSuoigZpGYWb37JyrWwbucZmoUHEJ+UzeHTaXRpGUaBxcbOP83ceVttUjIK2HcsmdwC5yiFw6fTaVDTjy0Hz6MoMKBLXTbsPkPfiNociEth1rd7aVjTF19vI+eSczBn5OHr6UFIgAlfLz2KAlsOJFJotdOhcTBdWoahOhzYk45jO3MQ29mDRKbE0z3AgU5xwAFA0aL4haDxr44uvC2KXyhnsnVs3HIAi6rj7nsHERQWStof55m/8jA1qnkxffhtxTq3itKUoig0K+MkWXRF6Wn0oH/n8GJNZgE+Bl4YF1Fs/baNqtG2UbVS91U90JOjZzLIK7RRK7jsed+rEkVRCA00EZ/knNc+JOBvkBQCTOyPS0Gv03Lbhf6ykAATZ5Odx1A75Prv/7lZdFoN1fyMJGcUVGpSqH1JIpDmIzdmsztIyy5EwZkUikb2nDiXxfvfHyCv0MbQHvVZtzOeP0+no6oq5vR8VKBBDT+qB3mx+69kAHy99Px5Ko0BncOJPZRIi3qBDOtZn6E96qEoCv27hPPD1pOcSszGnJ5HNT8jTer407pBEIqiYNRrqRXkyZ5jyYRpM2ics5P8n09jO/cnFOYCCpqQ+uhb9uXomUy8gsKo17Y9Gr/qKNriH3M9h8qH253HEhTmPBk0Dw/AZNAxtEe9q452uJohPerf0PahgRevVmsFV/6J6VqFBXoSn5SDh05zxRu5qorgABM2u4rNbnPVbIL9nW32ngYdgb5V4xhCAzxJziggtBKTgr+3AS+jDq1Wc12DR/5OJClcQVpWAaoKdUK8iTfnkJ1nxddLz0+/n8KhqrxwfwS1Qrw5kxzIrqPJmNPzMac7axYhASZX23XDWn7Ur+7LL3sS2HnETFpWIXdfuGu16Arbz0vP2DubAMVHE6jWAgp3L8f611b+nyOFHD8DPpoCOAB27yB04e3Q1W6FrmYLFKPzBNqm85WPS6NRGNevKTa7w7XMz9vAB8/0qPQ2ZKDYFWHNv0lNAS7GHRJguuHEeitc2sRVlAyKltUK8a4S3wVwNjceOplWqTUFRVFoUicAg0elzwxU4SQpXEHyhU7mpuEBxJtzSM0qwNdLT0aOhTqhPtS6UL1ufqFJ5c/T6eQVOpuKgv1NmAw6Rt7egMa1/cktsLJu5xn+96OziabdVaZdcORlYD97iMLdK1CzU9DWbM55nxacOnkGm19t7hw+GI3XtbV3l6ZNw5JNN1XlJFD04w/wMRQbmVPVuZLC36A/AS5PChdqCheavapC01GR8FAfPHQaalSr3AuEJ4a2rNT3v1UkKVxB0XDUpnUCWLfzDCmZBdSr7ktWbiHVgy5etRTVCk4mZqEozqaiojb1qM7O+wQKLDZX5/OTQ1viobtsmgeHA3vSMWyn9nAm8Q+sKc6x4opfKKboGHTVm+CVmss3h7YzqG3dG0oIVV3RyfXv1HQEEHbhOxFagePob6ZAXyNajYLdobqSQo0gLzSKQv0avpUc3UVdW4bRol7gFUeN3Qp/h9rfzSBJoQyqqpKSUYBWo7hGBBX1K2TmWvDz0rvWVRSFumE+nDqfjadBV2ono1GvY3TfRoQFelI96OIVjyM7GcuBNdjidqAWZINGhym8BUr9ruhqNUcTVMc1BDMs0JPxUU1Lvcr/J/H10hPsb6TpNQwFrUqqB3kRGmCiyd8kbo1GoZq/ifwCq+siJtDXyIxHOlX6PFuX0mgU13BvUfEkKZTirzMZzF64F0+DjiBfI94mD0wGHSmZ+eQV2rDZ1WJJAaBudR9W/R6Pp1FHmwZBpe63d/uLQysdGYkU7v0R2/HfQVHQ1YtAV68DulqtCKkZUuoNT4qi0LNNjRLL/2kUReG1RzpXmeasa2Xw0PL6o10qO4xyqV/dxzU6rsjfpaYjKoYkhQscDpVzqbnUCvZm5xEzqgq5BTYaXJimt5qfkdTMAjJyLAD4el+WFMJ8cagqOfnWKw5HVC15WPatwrJ/NWi1eLS8A32ru9B4B1bcwf0NVdZNSu7mgf7N+PtPdCNuJkkKF8QeOs9nq/7khXERHDqRSot6gYzr19TV9l/Nz8j5tDyycpw3kfl5Fa/O1r1k7PLlc7OoDhu20/uwndqD7eQusFnQNe6OodPdaExVp+1WuJ/SbvwT7k2SwgVH49MBWPzLcZLS8+nToVaxdsywQE8Onkgl3ZUUitcUAnwM+Hnpycy1uOZrAXDkpJK//kMc5hNg8MKjYRc8mvdCW61uxR+UEEKUkySFC45fmFLi6JkMAFrVL94vEBroic2ucvKcs63f77Lmo6LO5v1xqYT4m1DtNqyHfqZw7w+gqhh7P4qu/m0omtIn3BJCiKpAkgKQnWchKT2fLi1C+f2PJKr5GUv0C4ReeP3X2Qx0WgXPUmY+7dgsBLuqYtI5yF/zHvaEP9DWbo2x6xg0frdqXkkhhLh+khSAuIQsACLb1sRDp6VGNa8SI1+Kxs6fNecQ6GsodWRM15bV6dLYn/zVb2M//xfGyAfxaNKj4g9ACCFuEkkKOJuOtBpn80/j2v6lruPrpceo11JgseNXxtwnjtx0V/+BsdejeDS8ynwTQghRxUhSwJkU6oT6lPmADSiaBdOT0+ezS3Qy29POUPDrfByp8aDRYuz7BB71IsrYkxBCVF2SFICk9LwSHculCQ0wlUgKtjMHyP/5QxS9CUOnu9HWaYM2oOYV9iKEEFWXJAVAdajXNF67qF/B90JScGQkkr/+IzR+oZj6TfpHz0ckhHAPcucK4FAp+6Hvlyiaz93P24BqLST/5w9QNDpMd02UhCCE+EeQpIBz8jsNV88KtS/M2hnsZ6Dgt89xpJ/D2OcxNN5Xb3oSQoi/A0kKXKgpXENJ1Arx5pUHb6NR3l5sx7ehjxiKrpZ7zLEuhHAPkhS4UFO4xhk5wxznsWxbiLZOW/TtBlZwZEIIcWu5bUdzetJ5fLych+9Q1av2KTiyk7HGbcd6aD2KVyCmXg+7nnMghBD/FG6bFHKWv8rOWp1pOuA+VPXKj6K0p50lf+UbqIU5aAJrY+z1CIrh7/PsYCGEuFZumxQcKmhzzcCVm48cuenk/zQLtDo8734Nrf8//yE3Qgj35bbtHzmqCQ9rDsCFmkLp61n2/ohamIup/2RJCEKIfzy3TQrZqgm9zZkUnH0KJbOCIy8D69FNeDTuhjZQ7lIWQvzzuW1SyMWE3paLqqqoKmhKqSlY9q0ChwN9WxllJIRwD+6dFOx5OBwOgBJ9CnZzHNY/1uPRuDsa35DKCFEIIW45t00KOaonCipqvvNZCpfmBNVaSP4vn6B4BWDofE8lRSiEELee2yaFXJxPUnPkOR/DeWmfQuHuZahZSRhvf0iGngoh3IrbJoU8xTm53cWk4FxuTzuL9eA6PJr2RFejWWWFJ4QQlcJ9k4KrpuBsPtIoCqrDQeFv/4ei98Rw292VGZ4QQlQKt00KuRdqCmr+xeYjy4HV2JOOYegyGsXoXZnhCSFEpXDbpGBT9NgUnSspeBecx7JrKbp6Eegada3k6IQQonK4bVJQFIVCjZdr9FH4uXXgYcTYY/wV50ESQoh/spueFJYvX0737t0ZPHgwgwcP5p133gHg3LlzjBkzhn79+vH444+Tm5sLQFZWFo888ghRUVGMGTOG5OTkmx1SqTQahQKtF+Rn0Vh3Dv/s4xjaRUuzkRDCrd30pHDw4EFiYmJYsWIFK1asYNKkSQC8/PLL3HvvvaxZs4aWLVvy0UcfATBnzhwiIiJYvXo1I0eOZMaMGTc7pFIpikKBxgs1L4NBnnso1Pvh0bz3LXlvIYSoqiokKSxfvpxBgwbx7LPPkpmZidVqZefOndx1110ADBs2jDVr1gCwceNGoqOjARg4cCCbN2/GarXe7LBKUBTI13iiZCVSW5fG2Vr9UHT6Cn9fIYSoym56UggODubpp59mxYoVVK9enenTp5Oeno63tzc6nc61TlJSEgBms5ng4GAAdDod3t7epKWl3eywStAoCvka5wikw5aaZAS1qvD3FEKIqu66n6ewevVqXn/99WLL6tevzxdffOF6/dBDD9G3b1/+85//lNj+Sp25Gk35clVQUPn7ATw8tKTqwlC8Alic0Ykxvp4EB/uUez8VpSrFcqmqGhdU3dgkrvKRuMrvZsZ23UkhKiqKqKioYsuys7P54osvGD9+POB8eI1OpyMwMJCcnBzsdjtarZbk5GRCQpyTzIWEhJCSkkJYWBg2m42cnBz8/f3LFUtqag4Oh1qubex2B2cMDSm48w7S520jN7eA5OTscu2jogQH+1SZWC5VVeOCqhubxFU+Elf5lTc2jUa54oX0TW0+8vT05H//+x/79+8H4KuvvuKOO+7Aw8ODiIgIVq1aBThHKPXs2ROAyMhIli9fDsCqVauIiIjAw8PjZoZVKo2i4FBVVBTXayGEcHc39XGcWq2WOXPm8NJLL1FQUEDdunWZNWsWAP/973+JiYlh7ty5VK9enbfffhuAiRMnEhMTw4ABA/Dx8WH27Nk3M6QyaRRQHSqq6qxhSE4QQogKeEZzREQEy5YtK7G8Zs2aLFiwoMRyf39/5s2bd7PDuCrlQk2hqNVJblgTQgg3vqNZozifzVxUU5DmIyGEcOOkULKmULnxCCFEVeDGSQEcl/QpSE1BCCHcOCloNAqqimsoq+QEIYRw46RQ1HykSkezEEK4uHFSuNB8hDQfCSFEEbdNChpFQS1WU6jceIQQoipw66TgcIDDdfOaZAUhhHDbpKAozoSgOpyvNZIThBDCfZNCUfORq6YgWUEIIdw3KbhqCkX3KVRyPEIIURW47blQudCnIENShRDiIrdNCs6b1y5pPpKcIIQQ7psULjYfOV9rpE9BCCHcOCmg4LhkllRpPhJCCDdOChoNF5qPnK8lJwghhBsnBWdHs8ySKoQQl3LbpOB8yI50NAshxKXcOCkU9SlcfC2EEO7ObZPC5c1H0tEshBBunRSQWVKFEOIybpsUipqPZJZUIYS4yG2TQlHzkcM1+qiSAxJCiCrAjZNC8eYj6WgWQgg3TgoazeV3NFdyQEIIUQW4bVIoekazw1H0WrKCEEK4bVK4+IxmuaNZCCGKuG1SUIqSgut1pYYjhBBVgtsmBY2Cc0iqQ4akCiFEEbdNCkVJQIakCiHERW6bFIqSgN1+oaYgWUEIIdw3KRTVFOwO6WgWQogibpwUnP/aL4xJlZwghBBunBSKnsns6mhGsoIQQrhtUihKAnaH3NEshBBF3DYpuDqai/oUpKNZCCHcNykUjTa62NFcmdEIIUTV4LZJoWi0kWtIqrQfCSHEjSeFd999l/fff9/1Oisri0ceeYSoqCjGjBlDcnIyABaLhcmTJxMVFcXQoUOJi4sDnLOUzpw5k379+tG/f3927959oyFdExl9JIQQJV13UsjOzmbatGl89tlnxZbPmTOHiIgIVq9ezciRI5kxYwYACxYswGQysXr1aqZNm0ZMTAwAa9euJS4ujlWrVvHhhx8SExODzWa7gUO6NkU1BYfcpyCEEC7XnRQ2bNhA3bp1eeCBB4ot37hxI9HR0QAMHDiQzZs3Y7Va2bhxI4MGDQKgY8eOpKenc+7cOTZt2kT//v3RaDTUq1ePGjVqsHfv3hs4pGtTlANsMveREEK4XHdSGDJkCI888gharbbYcrPZTHBwMAA6nQ5vb2/S0tKKLQcIDg7m/PnzmM1mQkJCSiyvaMolNQXJB0II4aS72gqrV6/m9ddfL7asfv36fPHFF9f8JhpN6blHo9G4nmdwLeuXJSjIu1zrA/j5mgDw8NChURSCg33KvY+KVNXiKVJV44KqG5vEVT4SV/ndzNiumhSioqKIioq65h2GhISQkpJCWFgYNpuNnJwc/P39CQkJITk5mfDwcACSk5MJCQkhNDTU1Rl96fLySE3NcfUNXKucnAIAcvMtKIpCcnJ2ubavSMHBPlUqniJVNS6ourFJXOUjcZVfeWPTaJQrXkjf9CGpkZGRLF++HIBVq1YRERGBh4cHkZGRrFixAoBdu3ZhMBioUaMGPXv2ZOXKldjtdk6fPs2pU6do1arVzQ6rhItDUh1yj4IQQlxw1ZpCeU2cOJGYmBgGDBiAj48Ps2fPBmDs2LG8+OKLDBgwAL1ez6xZswDo168fBw4ccHVCz5gxA6PReLPDKkG5kA6dfQqSFYQQAkBRS2vU/5u5nuajHX8mMW/FH7SsF8jxhEw++ndkBUVXflW1qlpV44KqG5vEVT4SV/lV+eajv4tLn6cgNQUhhHBy26Rw6YR40qcghBBObpsULtYUHFJTEEKIC9w4KTj/dUhNQQghXNw2KWikT0EIIUpw26RwaUezPGBHCCGc3DYpuDqa7TL3kRBCFHHbpFD05DWHQ3U9r1kIIdyd2yaFogN3jj6q1FCEEKLKcNukUNSnYJM+BSGEcHHbpKC5tPlIqgpCCAG4cVJQLuloloqCEEI4uXFSuDAkVZWaghBCFHHbpHDxeQpSUxBCiCJumxRczUcy95EQQri4bVIoqimoKjIkVQghLnDbpHBpIpCaghBCOLltUtBckgikT0EIIZzcNilcWjnQSE1BCCEAN04Kl97FLM1HQgjh5LZJ4dJEIDlBCCGc3DgpXPp/yQpCCAFunBSko1kIIUpy26QgNQUhhCjJbZOC1BSEEKIkt00KxTuaJSsIIQS4cVK4tHYgD9kRQggnt00KxWoKlRiHEEJUJW6bFOTmNSGEKMltk0Lx0UeVF4cQQlQl7psUuHT0kWQFIYQAN04KmkuOXJGOZiGEANw4KShyn4IQQpTgtklBI3c0CyFECW6bFGSWVCGEKMltk0LxaS4kKwghBLhxUpAhqUIIUZIbJwXFlQykT0EIIZx0N7qDd999F41Gw9NPPw3Azp07eeqppwgLCwOgefPmvP7662RlZfHss89y5swZAgMDmTNnDsHBwVgsFp577jkOHTqE0Whk9uzZNGjQ4EbDuiaKoqCqqow+EkKIC667ppCdnc20adP47LPPii0/ePAgEyZMYMWKFaxYsYLXX38dgDlz5hAREcHq1asZOXIkM2bMAGDBggWYTCZWr17NtGnTiImJuYHDKZ+iZCB9CkII4XTdSWHDhg3UrVuXBx54oNjygwcPsnXrVoYMGcJjjz1GYmIiABs3biQ6OhqAgQMHsnnzZqxWKxs3bmTQoEEAdOzYkfT0dM6dO3e9YZVLUTKQ5iMhhHC67qQwZMgQHnnkEbRabbHlPj4+3H///SxfvpzIyEgmTZoEgNlsJjg4GACdToe3tzdpaWnFlgMEBwdz/vz56w2rXIruZJacIIQQTlftU1i9erWrCahI/fr1+eKLL0pdf/r06a7/jx49mrfeeovs7OxS19VoSs9JZS0vS1CQd7nWd73PhWTg6aknONjnuvZRUapaPEWqalxQdWOTuMpH4iq/mxnbVZNCVFQUUVFR17Qzh8PBxx9/XKIGodPpCAkJISUlhbCwMGw2Gzk5Ofj7+xMSEkJycjLh4eEAJCcnExISUq6DSE3NweFQy7UNXGw+KiywkZxceuKqDMHBPlUqniJVNS6ourFJXOUjcZVfeWPTaJQrXkjf1CGpGo2Gn3/+mbVr1wKwfPly2rRpg8lkIjIykuXLlwOwatUqIiIi8PDwIDIykhUrVgCwa9cuDAYDNWrUuJlhlUlRpPlICCEudcNDUi83c+ZMXnjhBT788EMCAwOZNWsWABMnTiQmJoYBAwbg4+PD7NmzARg7diwvvvgiAwYMQK/Xu9a/FYqSgow+EkIIpxtOCkX3JxRp1KgRCxcuLLGev78/8+bNK7HcYDAwc+bMGw3jumilo1kIIYpx2zuaAbmjWQghLuPmSeFC85Fbl4IQQlzk1qdDTVHzEVJTEEIIcPek4Go+qtw4hBCiqnDrpCCjj4QQoji3Tgqu5iOZJlUIIQB3TwpK8X+FEMLduXVSUGSWVCGEKMatk4JGbl4TQohi3DspSEezEEIU49ZJQe5oFkKI4tw6KUjzkRBCFOfWSUHuUxBCiOLcOinIHc1CCFGcmycFqSkIIcSl3DopyJPXhBCiOLdOChc7miUrCCEEuHtSkJqCEEIU49ZJQXHNfSRZQQghwM2TgnQ0CyFEce6dFOTmNSGEKMatk4JMcyGEEMW5eVK40HwkD1QQQggAdJUdQGXSFjUfVXIcQvxT2e020tOTsdks17S+2azB4XBUcFTlV1XjgivHptPpCQgIRqu99lO9WycFaT4SomKlpydjNHri5RV2Tb8znU6DzVb1Tr5VNS4oOzZVVcnNzSI9PZlq1apf8/6k+Qh5HKcQFcVms+Dl5SsXXpVAURS8vHyvuZZWxK2TgtzRLETFk99X5bmesnfvpODqaK7kQIQQ/3gjRkSTmHiussO4Krc+HUqfghBCFOfWHc1y85oQ7mXPnl3MnfsedruD6tWrYzJ5cuJEHA6HgzFj7qd37zsYPLgfixcvx9PTi8cfn0C3bj0ZP34C69evZd++vTz++FO8/vorJCebSUlJpm3bdjz//HT27t3t2nf9+g3417/+zfTpL2A2J1G3bn0sFmfb/vHjx5g1awZ2ux29Xs+0af+ldu06lVwyF7l3UlCkT0GIW2XrwUS2HEi84jqKAqpa/n13b12dbq2ubYTNmTPxLFnyIwsWfE61asE8//zL5Obm8NhjE2jevCUdOkSwd+8e2rXrQGJiIvv27QEmsG1bLH363EFs7BYaNWrMq6/OxGq1ct99Izl69EixfXt7e/P22zNp3Lgps2e/x759e/jll58BWLz4G0aNuo/evfuyYcM6/vjjoCSFqkImxBPC/dSuHY63tze7du2gsLCAn376AYCCggJOnjxBly7d2b17BxqNwp13RrFhwzpsNiv79+9j8uRpGAwGDh8+xOLF33Dq1EkyMzPJz88rtm+AvXt389JLrwHQtm17atSoCUCXLt14++1ZbN8eS9euPbj99j6VUAplc+ukoJEhqULcMt1aXf1q/lbcD2AwGABwOOy88MIrNGnSFIC0tFR8ff3Izs5m4cKv0Wp1dOjQkfj4U/zwwwrq16+PwWBgyZKFbNz4C4MGDWXEiNs4eTIO9UL1pmjf4GyBuPSmMq1WC0CvXn1p2bI1W7f+xnfffcu2bVuZMuX5Cj3m8nDrjmYZkiqE+2rfviPLly8BICUlhXHjRpOUdJ6AgAAMBgNbt26mdeu2tG/fkc8/n0/Xrj0A2LlzO4MGDePOO6MAhWPH/ir1juKIiNtYt241AH/++QcJCWcBePHFqRw+/AdDhgznoYceczU9VRVunRTkcZxCuK8JEx6msLCQsWPvZuLEx3jiiX9Rs2YtwNnE4+3tg6enJx06dCQ5OZmuXbsDcPfd9/L5558wYcIY3n57Ji1bti51qOmDDz5KQsJZ7rvvbr766gtX89HYsQ+wYMHnTJgwhg8/nMPTT0+6dQd9DRRVvZ5unaolNTUHh6P8h7Fk8wlWxZ7iubEdaFDTrwIiuz7BwT4kJ2dXdhglVNW4oOrG5u5xnT9/mrCw8Gtev6pOJ1FV44Krx3b5Z6DRKAQFeZe5vlvXFGT0kRBCFOfWSUGR+xSEEKKY604Ku3fvZvjw4QwePJhx48aRkJAAQFZWFo888ghRUVGMGTOG5ORkACwWC5MnTyYqKoqhQ4cSFxcHOGfymzlzJv369aN///7s3r37JhzWtZHHcQohRHHXnRQmT57MjBkzWLFiBdHR0bz66qsAzJkzh4iICFavXs3IkSOZMWMGAAsWLMBkMrF69WqmTZtGTEwMAGvXriUuLo5Vq1bx4YcfEhMTg81muwmHdnUXp7m4JW8nhBBV3nUlBYvFwsSJE2na1Dm+t0mTJiQmOu9U3LhxI9HR0QAMHDiQzZs3Y7Va2bhxI4MGDQKgY8eOpKenc+7cOTZt2kT//v3RaDTUq1ePGjVqsHfv3ptxbFdV9JAdqSkIIYTTdSUFvV7P4MGDAXA4HHzwwQf07dsXALPZTHBwMAA6nQ5vb2/S0tKKLQcIDg7m/PnzmM1mQkJCSiy/FWRIqhBCFHfVO5pXr17N66+/XmxZ/fr1+eKLL7BYLK7mnkcffbTMfWjKmJtao9FQ2ojYstYvy5WGV11JUTIICvImONjnuvZRUapaPEWqalxQdWNz57jMZg06Xfl+z+Vd/1apqnHBlWPTaDTl+qyvmhSioqKIiooqsTw3N5fHH38cf39/5s6di4eHBwAhISGkpKQQFhaGzWYjJycHf39/QkJCSE5OJjzcOV42OTmZkJAQQkNDXZ3Rly4vj+u9T6Hojub09FyMVejzdvex7dejqsbm7nE5HI5yje+vqvcDVKW4Pv30Y8B5cxxcPTaHw1Hss66w+xQmT55MeHg47777Lnq93rU8MjKS5cuXA7Bq1SoiIiLw8PAgMjKSFStWALBr1y4MBgM1atSgZ8+erFy5ErvdzunTpzl16hStWrW63rDK5eJDdqT9SAgh4DonxDt8+DAbNmygYcOGDBkyBHDWEObPn8/EiROJiYlhwIAB+Pj4MHv2bADGjh3Liy++yIABA9Dr9cyaNQuAfv36ceDAAVcn9IwZMzAajTfh0K5OkZvXhHAbe/bs4ssvP0NV4dy5s9x+ex+8vLz47bdNqKrK7NnvEhgYxPffL2LNmlUUFOSj0Wh4+eXXadiwQbF9jRgRTfPmLTl27CgfffQ/tm2L5bvvvsXhUGnSpCn//vcUPvroXerWrc/QoSP44YdlLFr0NV9/vQSbzcbddw9m8eIVrFjxfYn3qlu3Xon9r179Iz/8sAw/P398fHxo1qwFNpuN119/mZMnT6CqKkOHjmTQoKE3XE7XlRSaN2/O0aNHS/2bv78/8+bNK7HcYDAwc+bMEssVRWHKlClMmTLlekK5IUVdF1Wo5UiIfyzrX1uxHt18xXUURSm1n/FqPJr0xKNxt6uud/jwHyxYsAg/P3+io+/gySef4dNPF/Daay+zfv06BgyIZvPmTXzwwccYDEb+9795LFv2HZMnx5TYV+fOXZk+/XVOnIhj5crlzJ37GQaDgXnzPuDbbxfQpUt3fvxxOUOHjmD37h1kZWWRlpbKqVMnadGiFYWFBaW+16RJ/ym2/yNHDvPTTz/w2WdfoygKjz32AM2ateDgwf1kZWXx5ZffkpqaxgcfzKm8pPBPIdNcCOFe6tdvQGhoGAB+fv5ERNwGQGhoGNnZWXh5efPSS6+yfv06zpyJZ/v2WBo1alLqvpo3bwnA3r27OHv2DI8++gAANpuVxo2bMnr0WNcT1k6fPk2fPneyb99ejhz5g65du1/1vYr2v2fPbjp37oanpyfgnHrbbrdTv34D4uNPM3HiE3Tq1I3HH3/6ppSRWycFRfoUhLhlPBp3u+rVfEV36Op0xU95Rc84KJKUdJ6nn36U4cPvpnPnrgQGBnHsWOmtIkXPTrDbHfTu3ZdnnpkMQF5eHna7HYPBQMOGjVm3bjXh4eG0a9eB3bt3cODAfu69d9xV36to/87aU/HnMtjtdvz8/FmwYDF79uxgy5YtTJhwHwsWLMbH58ZGlbl1y4lG7lMQQlziyJHD1KpVm3vuGUPz5i3Zti0Wh8N+xW3atevA5s0bSU9PQ1VV3nrrdRYv/gaArl278cUX/6Nduw60a9eBLVs2YzIZ8ff3v+b3iojoSGzsFnJycigsLGTz5l8B2LJlE9Onv0C3bj145plnMZlMmM1JN1wGbl1TKOpTkOYjIQRAx46dWbZsCffdNxIPDw+aN2/JiRNxV9ymUaPGPPDAw/zrX4+hqiqNGjXhvvvGA9ClS3dmz36Ddu0i8PX1xd8/gC5dupfrvRo1asLIkaN56KH78fHxITTU+fS6zp278euvGxg9egR6vZ7IyN40aNDwhsvArZ+nEPunmf+tOMScf3XH11N/9Q1uEXcf2349qmps7h6XPE+h4snzFG6iID8jJoMWg4f26isLIYQbcOvmo66talAnyFOSghBCXODWNQWNRsFkcOu8KIQQxbh1UhBCVLx/QLfl39b1lL0kBSFEhdHp9OTmZkliqASqqpKbm4VOV75BNNJ2IoSoMAEBwaSnJ5OTk3FN62s0GhyOqjfKp6rGBVeOTafTExAQXOrfyiJJQQhRYbRaHdWqVb/m9d19CO/1uNmxSfOREEIIF0kKQgghXP4RzUc3MqFdVZ0MT+Iqv6oam8RVPhJX+ZUntqut+4+Y5kIIIcTNIc1HQgghXCQpCCGEcJGkIIQQwkWSghBCCBdJCkIIIVwkKQghhHCRpCCEEMJFkoIQQggXSQpCCCFc/hHTXJTXypUrmTt3LlarlfHjxzNmzJhKi+WDDz5g9erVAERGRvKf//yHqVOnsnv3bkwmEwBPPfUUd9xxxy2P7f777yc1NRWdzvk1mT59OvHx8ZVadt999x1fffWV6/XZs2cZPHgw+fn5lVZmOTk5jBo1innz5lGrVi1iY2N5/fXXKSwsJCoqikmTJgHw559/8vzzz5OTk0NERAQvv/yyq2xvRVyLFi1iwYIFKIpCy5Ytefnll9Hr9XzwwQd8//33+Pr6AnD33XdX6Od6eVxlfd/LKseKdGlscXFxvP32266/JSUl0aZNGz7++ONbWmalnSMq9Dumupnz58+rvXr1UtPT09Xc3Fw1OjpaPXbsWKXEsnXrVvWee+5RCwsLVYvFot5///3qunXr1IEDB6pJSUmVElMRh8OhduvWTbVara5lVansVFVV//rrL/WOO+5QU1NTK63M9u3bpw4cOFBt0aKFeubMGTU/P1+NjIxU4+PjVavVqk6YMEHduHGjqqqqOmDAAHXv3r2qqqrq1KlT1a+//vqWxXXixAn1jjvuULOzs1WHw6H+5z//UT///HNVVVX10UcfVffs2VNhsVwpLlVVS/3srlSOtzK2ImazWe3Tp4968uRJVVVvXZmVdo5YuXJlhX7H3K75KDY2ls6dO+Pv74+npyd33XUXa9asqZRYgoODiYmJQa/X4+HhQYMGDTh37hznzp3jhRdeIDo6mvfee69SHu5x4sQJFEXh4YcfZtCgQXz11VdVquwAXnrpJSZNmoTRaKy0Mlu8eDH//e9/CQkJAeDAgQOEh4dTu3ZtdDod0dHRrFmzhoSEBAoKCmjbti0Aw4YNq9CyuzwuvV7PSy+9hLe3N4qi0LhxY86dOwfAoUOHmD9/PtHR0UyfPp3CwsJbFldeXl6pn11Z5ViRLo/tUrNmzWLUqFHUrVsXuHVlVto54tSpUxX6HXO7pGA2mwkOvvgkopCQEJKSkiollkaNGrk+wFOnTrFq1Sp69OhB586dee2111i8eDG7du1iyZIltzy2rKwsunTpwocffsgXX3zBwoULOXfuXJUpu9jYWAoKCoiKiiI1NbXSymzGjBlERES4Xpf1/bp8eXBwcIWW3eVx1axZk65duwKQlpbG119/TZ8+fcjNzaVZs2ZMmTKFZcuWkZWVxUcffXTL4irrs6uM3+nlsRU5deoUO3bs4P777we4pWVW2jlCUZQK/Y65XVJQS5kUVlEqd0rcY8eOMWHCBKZMmUL9+vX58MMPCQoKwmQyMXbsWDZt2nTLY2rXrh2zZs3C09OTwMBARowYwXvvvVdivcoqu4ULF/LAAw8AULt27SpRZlD296uqfO+SkpIYN24cw4cPp1OnTnh5eTF//nzCw8PR6XRMmDDhlpZdWZ9dVSkvgEWLFnHvvfei1zufdVwZZXbpOaJOnTol/n4zv2NulxRCQ0NJSUlxvTabzaVWF2+V3bt3M378eP7f//t/DB06lKNHj7J27VrX31VVrdDOyLLs2rWL33//vVgcNWvWrBJlZ7FY2LlzJ7179waoMmUGZX+/Ll+enJx8y8suLi6O0aNHM3ToUJ588kkAzp07V6xWdavLrqzPrir9Tjds2ED//v1dr291mV1+jqjo75jbJYWuXbvy+++/k5aWRn5+PuvWraNnz56VEktiYiJPPvkks2fPZsCAAYDzC/baa6+RmZmJ1Wpl0aJFlTLyKDs7m1mzZlFYWEhOTg7Lli3jzTffrBJld/ToUerWrYunpydQdcoMoE2bNpw8eZLTp09jt9v58ccf6dmzJzVr1sRgMLB7924Ali9ffkvLLicnhwcffJCJEycyYcIE13Kj0cibb77JmTNnUFWVr7/++paWXVmfXVnleKulpaVRUFBA7dq1XctuZZmVdo6o6O+Y2w1JDQ0NZdKkSdx///1YrVZGjBhB69atKyWWTz/9lMLCQt544w3XslGjRvHII48wevRobDYbd955JwMHDrzlsfXq1Yv9+/czZMgQHA4H9957Lx06dKgSZXfmzBnCwsJcr5s2bVolygzAYDDwxhtv8PTTT1NYWEhkZCT9+vUDYPbs2Tz//PPk5ubSvHlzVxv1rbBkyRJSUlL47LPP+OyzzwDo3bs3EydOZPr06Tz++ONYrVbat2/vapa7Fa702ZVVjrfS2bNni33XAAIDA29ZmZV1jqjI75g8eU0IIYSL2zUfCSGEKJskBSGEEC6SFIQQQrhIUhBCCOEiSUEIIYSLJAUhboKHH36Y48ePl2ubRx99lKVLl1ZQREJcH7e7T0GIijB//vzKDkGIm0KSgnBrv/zyi+v5EEajkSlTprBlyxaOHTtGSkoKqampNG3alBkzZuDt7c0333zDwoUL8fDwwGAwMH36dBo2bEjv3r159913adWqleu5BRqNhmrVqvHCCy9Qr149kpKSiImJwWw2U6NGDVJTU11xxMXFMWPGDDIyMrDb7YwdO5YRI0aQm5vL1KlTOX36NBqNhhYtWjB9+nQ0Gqnkiwpy/TN9C/H3dvLkSXXgwIFqWlqaqqrO5zN069ZNfeONN9SePXuqycnJqt1uV//973+rb7zxhmqz2dQWLVq45v5ftmyZunDhQlVVVbVXr17qgQMH1NjYWLVv375qamqqqqqq+v3336tRUVGqw+FQn3jiCfWdd95RVVVVT506pbZt21b9/vvvVavVqvbv3189dOiQqqqqmpWVpUZFRal79+5Vly1bpk6YMEFVVVW12Wzqc889p546depWFpNwM1JTEG5r69atmM1mxo8f71qmKArx8fH069ePatWqATBixAhee+01pkyZQr9+/Rg1ahS333473bp1Izo6utg+f/vtN/r3709gYCDgnNN+xowZnD17ltjYWKZMmQJAeHg4nTp1ApxTIsfHxzNt2jTXfgoKCjh8+DA9evTgnXfeYezYsXTt2pVx48YRHh5ekcUi3JwkBeG2HA4HXbp0Yc6cOa5liYmJLFq0CIvFUmy9ouaa2bNn89dffxEbG8v8+fNZsmQJc+fOda2rljJrjKqq2Gy2EtMbF82sabfb8fX1ZcWKFa6/paSk4OPjg8Fg4Oeff2b79u1s27aNBx54gOeff75S5gES7kEaJoXb6ty5M1u3biUuLg6ATZs2MWjQIAoLC9mwYQPZ2dk4HA4WL15Mr169SEtLIzIyEn9/f8aPH88zzzzD0aNHi+2ze/furFq1irS0NAC+//57/P39CQ8Pp0ePHixatAhwTr+8fft2AOrVq4fBYHAlhcTERAYOHMihQ4f45ptvmDp1Kt27d2fy5Ml0796dY8eO3aoiEm5IJsQTbm316tXMmzfPNSf+tGnT+P3339m2bRt2u5309HQ6duzI888/j9FoZOHChXz55ZcYjUa0Wi2TJk2ia9euxTqav/76axYuXIjD4SAwMJAXX3yRRo0akZaWxtSpU4mPjycsLAybzcbQoUMZNmwYR44ccXU022w27r//fkaPHk1eXh7Tpk3j6NGjmEwmatSowYwZM/Dz86vsohP/UJIUhLjM+++/T3p6Oi+++GJlhyLELSfNR0IIIVykpiCEEMJFagpCCCFcJCkIIYRwkaQghBDCRZKCEEIIF0kKQgghXCQpCCGEcPn/2meCVE5jMD0AAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {} + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "加载模型成功！\n" + ] + } + ], + "metadata": {} } ] } \ No newline at end of file diff --git a/codes/QLearning/task0_train.py b/codes/QLearning/task0_train.py index f036162..ebc4fa4 100644 --- a/codes/QLearning/task0_train.py +++ b/codes/QLearning/task0_train.py @@ -5,14 +5,14 @@ Author: John Email: johnjim0816@gmail.com Date: 2020-09-11 23:03:00 LastEditor: John -LastEditTime: 2021-05-06 17:04:38 +LastEditTime: 2021-09-12 01:29:40 Discription: Environment: ''' import sys,os -curr_path = os.path.dirname(__file__) -parent_path=os.path.dirname(curr_path) -sys.path.append(parent_path) # add current terminal path to sys.path +curr_path = os.path.dirname(os.path.abspath(__file__)) # 当前路径 +parent_path=os.path.dirname(curr_path) # 父路径，这里就是我们的项目路径 +sys.path.append(parent_path) # 由于需要引用项目路径下的其他模块比如envs，所以需要添加路径到sys.path import gym import torch @@ -20,49 +20,49 @@ import datetime from envs.gridworld_env import CliffWalkingWapper from QLearning.agent import QLearning -from common.plot import plot_rewards +from common.plot import plot_rewards,plot_rewards_cn from common.utils import save_results,make_dir -curr_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") # obtain current time +curr_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") # 获取当前时间 class QlearningConfig: '''训练相关参数''' def __init__(self): - self.algo = 'Qlearning' - self.env = 'CliffWalking-v0' # 0 up, 1 right, 2 down, 3 left - self.result_path = curr_path+"/outputs/" +self.env+'/'+curr_time+'/results/' # path to save results - self.model_path = curr_path+"/outputs/" +self.env+'/'+curr_time+'/models/' # path to save models - self.train_eps = 300 # 训练的episode数目 - self.eval_eps = 30 + self.algo = 'Q-learning' # 算法名称 + self.env = 'CliffWalking-v0' # 环境名称 + self.result_path = curr_path+"/outputs/" +self.env+'/'+curr_time+'/results/' # 保存结果的路径 + self.model_path = curr_path+"/outputs/" +self.env+'/'+curr_time+'/models/' # 保存模型的路径 + self.train_eps = 200 # 训练的回合数 + self.eval_eps = 30 # 测试的回合数 self.gamma = 0.9 # reward的衰减率 - self.epsilon_start = 0.95 # e-greedy策略中初始epsilon + self.epsilon_start = 0.90 # e-greedy策略中初始epsilon self.epsilon_end = 0.01 # e-greedy策略中的终止epsilon self.epsilon_decay = 200 # e-greedy策略中epsilon的衰减率 - self.lr = 0.1 # learning rate - self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # check gpu + self.lr = 0.05 # 学习率 + self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # 检测GPU def env_agent_config(cfg,seed=1): env = gym.make(cfg.env) env = CliffWalkingWapper(env) - env.seed(seed) - state_dim = env.observation_space.n - action_dim = env.action_space.n - agent = QLearning(state_dim,action_dim,cfg) + env.seed(seed) # 设置随机种子 + n_states = env.observation_space.n # 状态维度 + n_actions = env.action_space.n # 动作维度 + agent = QLearning(n_states,n_actions,cfg) return env,agent def train(cfg,env,agent): - print('Start to train !') - print(f'Env:{cfg.env}, Algorithm:{cfg.algo}, Device:{cfg.device}') + print('开始训练！') + print(f'环境:{cfg.env}, 算法:{cfg.algo}, 设备:{cfg.device}') rewards = [] - ma_rewards = [] # moving average reward + ma_rewards = [] # 滑动平均奖励 for i_ep in range(cfg.train_eps): - ep_reward = 0 # 记录每个episode的reward + ep_reward = 0 # 记录每个回合的奖励 state = env.reset() # 重置环境, 重新开一局（即开始新的一个episode） while True: action = agent.choose_action(state) # 根据算法选择一个动作 next_state, reward, done, _ = env.step(action) # 与环境进行一次动作交互 agent.update(state, action, reward, next_state, done) # Q-learning算法更新 - state = next_state # 存储上一个观察值 + state = next_state # 更新状态 ep_reward += reward if done: break @@ -71,20 +71,18 @@ def train(cfg,env,agent): ma_rewards.append(ma_rewards[-1]*0.9+ep_reward*0.1) else: ma_rewards.append(ep_reward) - print("Episode:{}/{}: reward:{:.1f}".format(i_ep+1, cfg.train_eps,ep_reward)) - print('Complete training！') + print("回合数：{}/{}，奖励{:.1f}".format(i_ep+1, cfg.train_eps,ep_reward)) + print('完成训练！') return rewards,ma_rewards def eval(cfg,env,agent): - # env = gym.make("FrozenLake-v0", is_slippery=False) # 0 left, 1 down, 2 right, 3 up - # env = FrozenLakeWapper(env) - print('Start to eval !') - print(f'Env:{cfg.env}, Algorithm:{cfg.algo}, Device:{cfg.device}') - rewards = [] # 记录所有episode的reward - ma_rewards = [] # 滑动平均的reward + print('开始测试！') + print(f'环境：{cfg.env}, 算法：{cfg.algo}, 设备：{cfg.device}') + rewards = [] # 记录所有回合的奖励 + ma_rewards = [] # 滑动平均的奖励 for i_ep in range(cfg.eval_eps): ep_reward = 0 # 记录每个episode的reward - state = env.reset() # 重置环境, 重新开一局（即开始新的一个episode） + state = env.reset() # 重置环境, 重新开一局（即开始新的一个回合） while True: action = agent.predict(state) # 根据算法选择一个动作 next_state, reward, done, _ = env.step(action) # 与环境进行一个交互 @@ -97,23 +95,26 @@ def eval(cfg,env,agent): ma_rewards.append(ma_rewards[-1]*0.9+ep_reward*0.1) else: ma_rewards.append(ep_reward) - print(f"Episode:{i_ep+1}/{cfg.eval_eps}, reward:{ep_reward:.1f}") - print('Complete evaling！') + print(f"回合数：{i_ep+1}/{cfg.eval_eps}, 奖励：{ep_reward:.1f}") + print('完成测试！') return rewards,ma_rewards if __name__ == "__main__": cfg = QlearningConfig() + + # 训练 env,agent = env_agent_config(cfg,seed=1) rewards,ma_rewards = train(cfg,env,agent) - make_dir(cfg.result_path,cfg.model_path) - agent.save(path=cfg.model_path) - save_results(rewards,ma_rewards,tag='train',path=cfg.result_path) - plot_rewards(rewards,ma_rewards,tag="train",env=cfg.env,algo = cfg.algo,path=cfg.result_path) + make_dir(cfg.result_path,cfg.model_path) # 创建文件夹 + agent.save(path=cfg.model_path) # 保存模型 + save_results(rewards,ma_rewards,tag='train',path=cfg.result_path) # 保存结果 + plot_rewards_cn(rewards,ma_rewards,tag="train",env=cfg.env,algo = cfg.algo,path=cfg.result_path) - env,agent = env_agent_config(cfg,seed=10) - agent.load(path=cfg.model_path) + # # 测试 + # env,agent = env_agent_config(cfg,seed=10) + # agent.load(path=cfg.model_path) # 加载模型 rewards,ma_rewards = eval(cfg,env,agent) save_results(rewards,ma_rewards,tag='eval',path=cfg.result_path) - plot_rewards(rewards,ma_rewards,tag="eval",env=cfg.env,algo = cfg.algo,path=cfg.result_path) + plot_rewards_cn(rewards,ma_rewards,tag="eval",env=cfg.env,algo = cfg.algo,path=cfg.result_path) diff --git a/codes/README.md b/codes/README.md index c392ed3..e7b9e6f 100644 --- a/codes/README.md +++ b/codes/README.md @@ -18,11 +18,11 @@ ## 运行环境 -python 3.7、pytorch 1.6.0-1.7.1、gym 0.17.0-0.18.0 +python 3.7、pytorch 1.6.0-1.7.1、gym 0.17.0-0.19.0 ## 使用说明 -运行带有```train```的py文件或ipynb文件进行训练，如果前面带有```task```如```task0_train.py```，表示对task0任务训练 +运行带有```train```的py文件或ipynb文件进行训练，如果前面带有```task```如```task0_train.py```，表示对task0任务训练， 类似的带有```eval```即为测试。 ## 算法进度 diff --git a/codes/README_en.md b/codes/README_en.md deleted file mode 100644 index 5b2f707..0000000 --- a/codes/README_en.md +++ /dev/null @@ -1,52 +0,0 @@ - - -[Eng](https://github.com/JohnJim0816/reinforcement-learning-tutorials/blob/master/README_en.md)|[中文](https://github.com/JohnJim0816/reinforcement-learning-tutorials/blob/master/README.md) - -## Introduction - -This repo is used to learn basic RL algorithms, we will make it **detailed comment** and **clear structure** as much as possible: - -The code structure mainly contains several scripts as following： - -* ```model.py``` basic network model of RL, like MLP, CNN -* ```memory.py``` Replay Buffer -* ```plot.py``` use seaborn to plot rewards curve，saved in folder ``` result```. -* ```env.py``` to custom or normalize environments -* ```agent.py``` core algorithms, include a python Class with functions(choose action, update) -* ```main.py``` main function - -Note that ```model.py```,```memory.py```,```plot.py``` shall be utilized in different algorithms，thus they are put into ```common``` folder。 - -## Runnig Environment - -python 3.7、pytorch 1.6.0-1.7.1、gym 0.17.0-0.18.0 -## Usage -run python scripts or jupyter notebook file with ```train``` to train the agent, if there is a ```task``` like ```task0_train.py```, it means to train with task 0. - -similar to file with ```eval```, which means to evaluate the agent. - -## Schedule - -| Name | Related materials | Used Envs | Notes | -| :--------------------------------------: | :----------------------------------------------------------: | ----------------------------------------- | :---: | -| [On-Policy First-Visit MC](./MonteCarlo) | [medium blog](https://medium.com/analytics-vidhya/monte-carlo-methods-in-reinforcement-learning-part-1-on-policy-methods-1f004d59686a) | [Racetrack](./envs/racetrack_env.md) | | -| [Q-Learning](./QLearning) | [towardsdatascience blog](https://towardsdatascience.com/simple-reinforcement-learning-q-learning-fcddc4b6fe56),[q learning paper](https://ieeexplore.ieee.org/document/8836506) | [CliffWalking-v0](./envs/gym_info.md) | | -| [Sarsa](./Sarsa) | [geeksforgeeks blog](https://www.geeksforgeeks.org/sarsa-reinforcement-learning/) | [Racetrack](./envs/racetrack_env.md) | | -| [DQN](./DQN) | [DQN Paper](https://www.cs.toronto.edu/~vmnih/docs/dqn.pdf),[Nature DQN Paper](https://www.nature.com/articles/nature14236) | [CartPole-v0](./envs/gym_info.md) | | -| [DQN-cnn](./DQN_cnn) | [DQN Paper](https://www.cs.toronto.edu/~vmnih/docs/dqn.pdf) | [CartPole-v0](./envs/gym_info.md) | | -| [DoubleDQN](./DoubleDQN) | [DoubleDQN Paper](https://arxiv.org/abs/1509.06461) | [CartPole-v0](./envs/gym_info.md) | | -| [Hierarchical DQN](HierarchicalDQN) | [H-DQN Paper](https://arxiv.org/abs/1604.06057) | [CartPole-v0](./envs/gym_info.md) | | -| [PolicyGradient](./PolicyGradient) | [Lil'log](https://lilianweng.github.io/lil-log/2018/04/08/policy-gradient-algorithms.html) | [CartPole-v0](./envs/gym_info.md) | | -| [A2C](./A2C) | [A3C Paper](https://arxiv.org/abs/1602.01783) | [CartPole-v0](./envs/gym_info.md) | | -| [SAC](./SAC) | [SAC Paper](https://arxiv.org/abs/1801.01290) | [Pendulum-v0](./envs/gym_info.md) | | -| [PPO](./PPO) | [PPO paper](https://arxiv.org/abs/1707.06347) | [CartPole-v0](./envs/gym_info.md) | | -| [DDPG](./DDPG) | [DDPG Paper](https://arxiv.org/abs/1509.02971) | [Pendulum-v0](./envs/gym_info.md) | | -| [TD3](./TD3) | [TD3 Paper](https://arxiv.org/abs/1802.09477) | [HalfCheetah-v2]((./envs/mujoco_info.md)) | | - - -## Refs - - -[RL-Adventure-2](https://github.com/higgsfield/RL-Adventure-2) - -[RL-Adventure](https://github.com/higgsfield/RL-Adventure) \ No newline at end of file diff --git a/codes/Sarsa/outputs/CliffWalking-v0/20210506-171245/models/sarsa_model.pkl b/codes/Sarsa/outputs/CliffWalking-v0/20210506-171245/models/sarsa_model.pkl new file mode 100644 index 0000000..ff25fd5 Binary files /dev/null and b/codes/Sarsa/outputs/CliffWalking-v0/20210506-171245/models/sarsa_model.pkl differ diff --git a/codes/Sarsa/outputs/CliffWalking-v0/20210506-171245/results/eval_ma_rewards.npy b/codes/Sarsa/outputs/CliffWalking-v0/20210506-171245/results/eval_ma_rewards.npy new file mode 100644 index 0000000..d7d62e3 Binary files /dev/null and b/codes/Sarsa/outputs/CliffWalking-v0/20210506-171245/results/eval_ma_rewards.npy differ diff --git a/codes/Sarsa/outputs/CliffWalking-v0/20210506-171245/results/eval_rewards.npy b/codes/Sarsa/outputs/CliffWalking-v0/20210506-171245/results/eval_rewards.npy new file mode 100644 index 0000000..de0a816 Binary files /dev/null and b/codes/Sarsa/outputs/CliffWalking-v0/20210506-171245/results/eval_rewards.npy differ diff --git a/codes/Sarsa/outputs/CliffWalking-v0/20210506-171245/results/eval_rewards_curve.png b/codes/Sarsa/outputs/CliffWalking-v0/20210506-171245/results/eval_rewards_curve.png new file mode 100644 index 0000000..3de2db7 Binary files /dev/null and b/codes/Sarsa/outputs/CliffWalking-v0/20210506-171245/results/eval_rewards_curve.png differ diff --git a/codes/Sarsa/outputs/CliffWalking-v0/20210506-171245/results/train_ma_rewards.npy b/codes/Sarsa/outputs/CliffWalking-v0/20210506-171245/results/train_ma_rewards.npy new file mode 100644 index 0000000..3f9bf83 Binary files /dev/null and b/codes/Sarsa/outputs/CliffWalking-v0/20210506-171245/results/train_ma_rewards.npy differ diff --git a/codes/Sarsa/outputs/CliffWalking-v0/20210506-171245/results/train_rewards.npy b/codes/Sarsa/outputs/CliffWalking-v0/20210506-171245/results/train_rewards.npy new file mode 100644 index 0000000..e0fd7e9 Binary files /dev/null and b/codes/Sarsa/outputs/CliffWalking-v0/20210506-171245/results/train_rewards.npy differ diff --git a/codes/Sarsa/outputs/CliffWalking-v0/20210506-171245/results/train_rewards_curve.png b/codes/Sarsa/outputs/CliffWalking-v0/20210506-171245/results/train_rewards_curve.png new file mode 100644 index 0000000..0a8cd37 Binary files /dev/null and b/codes/Sarsa/outputs/CliffWalking-v0/20210506-171245/results/train_rewards_curve.png differ diff --git a/codes/Sarsa/outputs/RacetrackEnv/20210715-085005/models/sarsa_model.pkl b/codes/Sarsa/outputs/RacetrackEnv/20210715-085005/models/sarsa_model.pkl deleted file mode 100644 index a80fa68..0000000 Binary files a/codes/Sarsa/outputs/RacetrackEnv/20210715-085005/models/sarsa_model.pkl and /dev/null differ diff --git a/codes/Sarsa/outputs/RacetrackEnv/20210715-085005/results/eval_ma_rewards.npy b/codes/Sarsa/outputs/RacetrackEnv/20210715-085005/results/eval_ma_rewards.npy deleted file mode 100644 index 91051de..0000000 Binary files a/codes/Sarsa/outputs/RacetrackEnv/20210715-085005/results/eval_ma_rewards.npy and /dev/null differ diff --git a/codes/Sarsa/outputs/RacetrackEnv/20210715-085005/results/eval_rewards.npy b/codes/Sarsa/outputs/RacetrackEnv/20210715-085005/results/eval_rewards.npy deleted file mode 100644 index 176ea2e..0000000 Binary files a/codes/Sarsa/outputs/RacetrackEnv/20210715-085005/results/eval_rewards.npy and /dev/null differ diff --git a/codes/Sarsa/outputs/RacetrackEnv/20210715-085005/results/eval_rewards_curve.png b/codes/Sarsa/outputs/RacetrackEnv/20210715-085005/results/eval_rewards_curve.png deleted file mode 100644 index a72aede..0000000 Binary files a/codes/Sarsa/outputs/RacetrackEnv/20210715-085005/results/eval_rewards_curve.png and /dev/null differ diff --git a/codes/Sarsa/outputs/RacetrackEnv/20210715-085005/results/train_ma_rewards.npy b/codes/Sarsa/outputs/RacetrackEnv/20210715-085005/results/train_ma_rewards.npy deleted file mode 100644 index 47c20c6..0000000 Binary files a/codes/Sarsa/outputs/RacetrackEnv/20210715-085005/results/train_ma_rewards.npy and /dev/null differ diff --git a/codes/Sarsa/outputs/RacetrackEnv/20210715-085005/results/train_rewards.npy b/codes/Sarsa/outputs/RacetrackEnv/20210715-085005/results/train_rewards.npy deleted file mode 100644 index 204a158..0000000 Binary files a/codes/Sarsa/outputs/RacetrackEnv/20210715-085005/results/train_rewards.npy and /dev/null differ diff --git a/codes/Sarsa/outputs/RacetrackEnv/20210715-085005/results/train_rewards_curve.png b/codes/Sarsa/outputs/RacetrackEnv/20210715-085005/results/train_rewards_curve.png deleted file mode 100644 index 22e9755..0000000 Binary files a/codes/Sarsa/outputs/RacetrackEnv/20210715-085005/results/train_rewards_curve.png and /dev/null differ diff --git a/codes/Sarsa/task0_train.py b/codes/Sarsa/task0_train.py index a9a90b3..d21db17 100644 --- a/codes/Sarsa/task0_train.py +++ b/codes/Sarsa/task0_train.py @@ -5,7 +5,7 @@ Author: John Email: johnjim0816@gmail.com Date: 2021-03-11 17:59:16 LastEditor: John -LastEditTime: 2021-07-15 08:52:59 +LastEditTime: 2021-05-06 17:12:37 Discription: Environment: ''' @@ -26,8 +26,8 @@ class SarsaConfig: ''' parameters for Sarsa ''' def __init__(self): - self.algo = 'Sarsa' - self.env = 'RacetrackEnv' # 0 up, 1 right, 2 down, 3 left + self.algo = 'Qlearning' + self.env = 'CliffWalking-v0' # 0 up, 1 right, 2 down, 3 left self.result_path = curr_path+"/outputs/" +self.env+'/'+curr_time+'/results/' # path to save results self.model_path = curr_path+"/outputs/" +self.env+'/'+curr_time+'/models/' # path to save models self.train_eps = 200 @@ -52,15 +52,14 @@ def train(cfg,env,agent): # An episode is an array of (state, action, reward) tuples state = env.reset() ep_reward = 0 - action = agent.choose_action(state) while True: # for t in range(cfg.n_steps): + action = agent.choose_action(state) next_state, reward, done = env.step(action) ep_reward+=reward next_action = agent.choose_action(next_state) agent.update(state, action, reward, next_state, next_action,done) state = next_state - action = next_action if done: break if ma_rewards: diff --git a/codes/common/memory.py b/codes/common/memory.py index a9906e9..d24061d 100644 --- a/codes/common/memory.py +++ b/codes/common/memory.py @@ -5,30 +5,28 @@ @Email: johnjim0816@gmail.com @Date: 2020-06-10 15:27:16 @LastEditor: John -LastEditTime: 2021-03-13 11:37:15 +LastEditTime: 2021-09-15 02:17:59 @Discription: @Environment: python 3.7.7 ''' import random - class ReplayBuffer: - def __init__(self, capacity): - self.capacity = capacity - self.buffer = [] - self.position = 0 + self.capacity = capacity # 经验回放的容量 + self.buffer = [] # 缓冲区 + self.position = 0 def push(self, state, action, reward, next_state, done): + ''' 缓冲区是一个队列，容量超出时去掉开始存入的转移(transition) + ''' if len(self.buffer) < self.capacity: self.buffer.append(None) self.buffer[self.position] = (state, action, reward, next_state, done) - self.position = (self.position + 1) % self.capacity + self.position = (self.position + 1) % self.capacity def sample(self, batch_size): - batch = random.sample(self.buffer, batch_size) - state, action, reward, next_state, done = zip(*batch) + batch = random.sample(self.buffer, batch_size) # 随机采出小批量转移 + state, action, reward, next_state, done = zip(*batch) # 解压成状态，动作等 return state, action, reward, next_state, done - - def __len__(self): - return len(self.buffer) + diff --git a/codes/common/plot.py b/codes/common/plot.py index 92e4d96..5add0bf 100644 --- a/codes/common/plot.py +++ b/codes/common/plot.py @@ -5,12 +5,15 @@ Author: John Email: johnjim0816@gmail.com Date: 2020-10-07 20:57:11 LastEditor: John -LastEditTime: 2021-04-29 15:41:48 +LastEditTime: 2021-09-11 21:35:00 Discription: Environment: ''' import matplotlib.pyplot as plt import seaborn as sns +from matplotlib.font_manager import FontProperties +def chinese_font(): + return FontProperties(fname='/System/Library/Fonts/STHeiti Light.ttc',size=15) # 系统字体路径，此处是mac的 def plot_rewards(rewards,ma_rewards,tag="train",env='CartPole-v0',algo = "DQN",save=True,path='./'): sns.set() plt.title("average learning curve of {} for {}".format(algo,env)) @@ -21,16 +24,20 @@ def plot_rewards(rewards,ma_rewards,tag="train",env='CartPole-v0',algo = "DQN",s if save: plt.savefig(path+"{}_rewards_curve".format(tag)) plt.show() -# def plot_rewards(dic,tag="train",env='CartPole-v0',algo = "DQN",save=True,path='./'): -# sns.set() -# plt.title("average learning curve of {} for {}".format(algo,env)) -# plt.xlabel('epsiodes') -# for key, value in dic.items(): -# plt.plot(value,label=key) -# plt.legend() -# if save: -# plt.savefig(path+algo+"_rewards_curve_{}".format(tag)) -# plt.show() + +def plot_rewards_cn(rewards,ma_rewards,tag="train",env='CartPole-v0',algo = "DQN",save=True,path='./'): + ''' 中文画图 + ''' + sns.set() + plt.title(u"{}环境下Q学习算法的学习曲线".format(env),fontproperties=chinese_font()) + plt.xlabel(u'回合数',fontproperties=chinese_font()) + plt.plot(rewards) + plt.plot(ma_rewards) + plt.legend((u'奖励',u'滑动平均奖励',),loc="best",prop=chinese_font()) + if save: + plt.savefig(path+f"{tag}_rewards_curve_cn") + plt.show() + def plot_losses(losses,algo = "DQN",save=True,path='./'): sns.set() plt.title("loss curve of {}".format(algo)) diff --git a/codes/common/utils.py b/codes/common/utils.py index 8339db5..a3ca7be 100644 --- a/codes/common/utils.py +++ b/codes/common/utils.py @@ -5,7 +5,7 @@ Author: John Email: johnjim0816@gmail.com Date: 2021-03-12 16:02:24 LastEditor: John -LastEditTime: 2021-05-04 19:58:31 +LastEditTime: 2021-09-11 21:48:49 Discription: Environment: ''' @@ -18,7 +18,7 @@ def save_results(rewards,ma_rewards,tag='train',path='./results'): ''' np.save(path+'{}_rewards.npy'.format(tag), rewards) np.save(path+'{}_ma_rewards.npy'.format(tag), ma_rewards) - print('results saved!') + print('结果保存完毕！') def make_dir(*paths): for path in paths: