From 312b57fdff1d956aaad16e39f8e35274d5d85513 Mon Sep 17 00:00:00 2001 From: JohnJim0816 Date: Sun, 4 Apr 2021 16:59:03 +0800 Subject: [PATCH] update --- codes/DQN/main.py | 21 ++--- codes/QLearning/main.ipynb | 152 +++++++++++++++++++++++++++++++++++++ codes/QLearning/main.py | 2 +- codes/README.md | 6 +- codes/README_en.md | 36 +++++---- codes/common/plot.py | 12 +-- codes/common/utils.py | 17 ++++- codes/test.py | 19 +++++ 8 files changed, 221 insertions(+), 44 deletions(-) create mode 100644 codes/QLearning/main.ipynb create mode 100644 codes/test.py diff --git a/codes/DQN/main.py b/codes/DQN/main.py index afc2f5f..99868af 100644 --- a/codes/DQN/main.py +++ b/codes/DQN/main.py @@ -5,13 +5,11 @@ @Email: johnjim0816@gmail.com @Date: 2020-06-12 00:48:57 @LastEditor: John -LastEditTime: 2021-03-30 16:59:19 +LastEditTime: 2021-04-04 00:26:47 @Discription: @Environment: python 3.7.7 ''' import sys,os -from pathlib import Path -import sys,os curr_path = os.path.dirname(__file__) parent_path=os.path.dirname(curr_path) sys.path.append(parent_path) # add current terminal path to sys.path @@ -21,19 +19,13 @@ import torch import datetime from DQN.agent import DQN from common.plot import plot_rewards -from common.utils import save_results +from common.utils import save_results,make_dir,del_empty_dir SEQUENCE = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") # obtain current time SAVED_MODEL_PATH = curr_path+"/saved_model/"+SEQUENCE+'/' # path to save model -if not os.path.exists(curr_path+"/saved_model/"): - os.mkdir(curr_path+"/saved_model/") -if not os.path.exists(SAVED_MODEL_PATH): - os.mkdir(SAVED_MODEL_PATH) RESULT_PATH = curr_path+"/results/"+SEQUENCE+'/' # path to save rewards -if not os.path.exists(curr_path+"/results/"): - os.mkdir(curr_path+"/results/") -if not os.path.exists(RESULT_PATH): - os.mkdir(RESULT_PATH) +make_dir(curr_path+"/saved_model/",curr_path+"/results/") +del_empty_dir(curr_path+"/saved_model/",curr_path+"/results/") class DQNConfig: def __init__(self): @@ -72,8 +64,7 @@ def train(cfg,env,agent): rewards.append(ep_reward) # 计算滑动窗口的reward if ma_rewards: - ma_rewards.append( - 0.9*ma_rewards[-1]+0.1*ep_reward) + ma_rewards.append(0.9*ma_rewards[-1]+0.1*ep_reward) else: ma_rewards.append(ep_reward) print('Complete training!') @@ -87,6 +78,8 @@ if __name__ == "__main__": action_dim = env.action_space.n agent = DQN(state_dim,action_dim,cfg) rewards,ma_rewards = train(cfg,env,agent) + make_dir(SAVED_MODEL_PATH,RESULT_PATH) agent.save(path=SAVED_MODEL_PATH) save_results(rewards,ma_rewards,tag='train',path=RESULT_PATH) plot_rewards(rewards,ma_rewards,tag="train",algo = cfg.algo,path=RESULT_PATH) + del_empty_dir(SAVED_MODEL_PATH,RESULT_PATH) \ No newline at end of file diff --git a/codes/QLearning/main.ipynb b/codes/QLearning/main.ipynb new file mode 100644 index 0000000..91d2a6b --- /dev/null +++ b/codes/QLearning/main.ipynb @@ -0,0 +1,152 @@ +{ + "metadata": { + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.10-final" + }, + "orig_nbformat": 2, + "kernelspec": { + "name": "python3", + "display_name": "Python 3.7.10 64-bit ('py37': conda)", + "metadata": { + "interpreter": { + "hash": "fbea1422c2cf61ed9c0cfc03f38f71cc9083cc288606edc4170b5309b352ce27" + } + } + } + }, + "nbformat": 4, + "nbformat_minor": 2, + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "from pathlib import Path\n", + "curr_path = str(Path().absolute())\n", + "parent_path = str(Path().absolute().parent)\n", + "sys.path.append(parent_path) # add current terminal path to sys.path\n", + "\n", + "import gym\n", + "\n", + "from envs.gridworld_env import CliffWalkingWapper, FrozenLakeWapper\n", + "from QLearning.agent import QLearning\n", + "from common.plot import plot_rewards\n", + "from common.utils import save_results" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "class QlearningConfig:\n", + " '''训练相关参数'''\n", + " def __init__(self):\n", + " self.train_eps = 200 # 训练的episode数目\n", + " self.gamma = 0.9 # reward的衰减率\n", + " self.epsilon_start = 0.99 # e-greedy策略中初始epsilon\n", + " self.epsilon_end = 0.01 # e-greedy策略中的终止epsilon\n", + " self.epsilon_decay = 200 # e-greedy策略中epsilon的衰减率\n", + " self.lr = 0.1 # learning rate" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "def train(cfg,env,agent):\n", + " rewards = [] \n", + " ma_rewards = [] # moving average reward\n", + " steps = [] # 记录所有episode的steps\n", + " for i_episode in range(cfg.train_eps):\n", + " ep_reward = 0 # 记录每个episode的reward\n", + " ep_steps = 0 # 记录每个episode走了多少step\n", + " state = env.reset() # 重置环境, 重新开一局(即开始新的一个episode)\n", + " while True:\n", + " action = agent.choose_action(state) # 根据算法选择一个动作\n", + " next_state, reward, done, _ = env.step(action) # 与环境进行一次动作交互\n", + " agent.update(state, action, reward, next_state, done) # Q-learning算法更新\n", + " state = next_state # 存储上一个观察值\n", + " ep_reward += reward\n", + " ep_steps += 1 # 计算step数\n", + " if done:\n", + " break\n", + " steps.append(ep_steps)\n", + " rewards.append(ep_reward)\n", + " if ma_rewards:\n", + " ma_rewards.append(ma_rewards[-1]*0.9+ep_reward*0.1)\n", + " else:\n", + " ma_rewards.append(ep_reward)\n", + " if (i_episode+1)%10==0:\n", + " print(\"Episode:{}/{}: reward:{:.1f}\".format(i_episode+1, cfg.train_eps,ep_reward))\n", + " return rewards,ma_rewards" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Episode:10/200: reward:-82.0\n", + "Episode:20/200: reward:-59.0\n", + "Episode:30/200: reward:-50.0\n", + "Episode:40/200: reward:-32.0\n", + "Episode:50/200: reward:-102.0\n", + "Episode:60/200: reward:-151.0\n", + "Episode:70/200: reward:-34.0\n", + "Episode:80/200: reward:-71.0\n", + "Episode:90/200: reward:-34.0\n", + "Episode:100/200: reward:-26.0\n", + "Episode:110/200: reward:-32.0\n", + "Episode:120/200: reward:-48.0\n", + "Episode:130/200: reward:-25.0\n", + "Episode:140/200: reward:-31.0\n", + "Episode:150/200: reward:-38.0\n", + "Episode:160/200: reward:-47.0\n", + "Episode:170/200: reward:-29.0\n", + "Episode:180/200: reward:-36.0\n", + "Episode:190/200: reward:-21.0\n", + "Episode:200/200: reward:-34.0\n" + ] + }, + { + "output_type": "display_data", + "data": { + "text/plain": "
", + "image/svg+xml": "\n\n\n\n \n \n \n \n 2021-03-31T18:50:18.442345\n image/svg+xml\n \n \n Matplotlib v3.3.4, https://matplotlib.org/\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n\n", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEXCAYAAABCjVgAAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAABc4klEQVR4nO3dd3wUZf7A8c9sT7JJliSbRgmE0DuGKhipEiB0PBBB4RS7iCcSEDlBEUFUwAInd8rvsAAiRZRYwEOQ3ov0mgRCOqRny8zvj5CVkIQU0mSf9+vlSzI7O/PdZ2fnO0+ZZyRFURQEQRAEAVBVdwCCIAhCzSGSgiAIguAgkoIgCILgIJKCIAiC4CCSgiAIguAgkoIgCILgIJLCX8yHH37I7Nmzq2RfTz75JOfOnauSfVW3JUuW8OCDDzJt2rRCr9lsNpYsWUL//v0ZMGAAAwYMYNasWVy/fr3M+9mzZw+tW7dm8ODBDBkyhMGDBzNs2DB+/fXXEt/bs2dPjh07xrFjx3jxxRfLvO+iNGnShIiICAYPHuz477XXXgNg8ODBpKWllXpb6enpjBs3rsjXPvroI/7+978XWn78+HG6dOnCkSNHSvxMixYtYv369Y7tbd68udA6sbGxNGnShDFjxhR6bdq0aTRp0oSUlBQA7HY7n3/+OcOGDWPw4MH079+fd999F4vFUuT+y7p+adypzO5k7dq1PPXUU+Xe7x0pwl/K4sWLlVmzZlV3GPecnj17Kvv27SvytRdeeEF5+eWXldTUVEVRFMVisSiffvqp0rdvXyU9Pb1M+9m9e7cyYMCAAstOnjyptG3bVklOTr7je3v06KEcPXq0TPsrSePGjUvcb2nFxMQobdu2LfK1+Ph4pWXLlsrVq1cLLH/99deV9957r8z7evTRR5WoqKgiY2jVqpXStWtXJTY21rE8MzNT6dOnT4HPO2PGDOWFF15Q0tLSHOs888wzyiuvvFLkPsu6fmncqczu5Ntvv1UmTpxY7v3eiaZyUk3NJssyb7/9NkeOHCEzMxNFUXjrrbdo3LgxYWFh/PTTT5jNZgAefvhhnnvuObp06cKCBQvYt28fdrud5s2bM2PGDIxGIz179qR169acPn2al19+GY1Gw7/+9S8sFgspKSkMGTKEl156CYBPP/2UNWvW4ObmRmhoKFu2bOHXX3/FYrEUu/3ixMfHM3v2bOLi4rBarQwYMICnn34agKVLl7J582Zyc3PJzs5m6tSp9OnThw8//JDDhw+TkJBAkyZNCAoK4sqVKyQmJnLlyhW8vLz44IMP8PPzo2fPnixatIisrCw++OAD6taty9mzZ7FYLMycOZPOnTuTkpLCtGnTiI6OxmQyYTabadSoES+88EKBWDMzM3nrrbc4ePAgarWa3r17M3nyZKZNm0ajRo0cV5GRkZGOv28t1xdeeIElS5awceNGANLS0ujVqxebN28mJyen2HK41bVr13jjjTe4cuUKiqIwZMgQnnjiCV566SXi4+N57bXXmDRpEv3793e859ChQxw+fJhff/0VjSbv56LVannyySc5ePAgK1eu5IknnqBVq1ZMnDiRHTt2kJCQwLhx43j88cdLdTw2bdoUg8HAlStXcHd355133mHXrl2o1Wpat27NtGnTChwHe/bs4c033+T7778vslyffvppwsLCWL16NQ0aNABg/PjxjBkzht69e5cqJsirRezatYutW7eyZs0asrOzMRqNvP/++0ydOpXU1FQAwsLCeOmll5g2bRo5OTkMHjyYtWvXolarHdvy9fWlZ8+erF27lueeew7IOyaioqLYsGFDgc+0f/9+3nnnHWRZBuCpp57ioYcechwbBoOB48ePM3/+fNRqNX369CkQt1qtJjw8nI0bNzqOg59//plevXrx2WefARATE8PGjRv5/fffHWXr6urKrFmzOHToUKGyKM366enpzJo1i1OnTiFJEt27d3ecD4o7Pm4vszZt2tCrVy9OnTrFggULyM3NZf78+WRnZ6PVannppZd44IEHSv0dlodTNh8dOXKEhIQEVq1axaZNmxg6dCjLli3D3d2dPn368N133wFw/vx5EhMT6d69O59++ilqtZq1a9fy3Xff4evry4IFCxzbbNSoEVFRUfTu3ZvPPvuMd955h7Vr17Jq1So+/fRTUlJS2L59O2vXrmXNmjWsXbuWzMxMx/tL2n5RpkyZwvDhwx3b3LlzJ5s2beLKlSvs3LmTL774go0bNzJ58mQWL17seN+VK1dYt26dY/v79+9n0aJF/Pjjj3h4eLBq1apC+zp69CgTJkxg/fr1jBgxgo8++giAt956i5CQEKKioli0aBEHDx4sMtbFixeTm5vLpk2bWL9+PQcPHmTv3r0lflf55RoeHk5mZibHjh0D4PvvvycsLAxPT89iy+F2r7zyCp06dWLjxo18/fXXfPfdd/zwww8sXLjQUd63JgTISwpt27Z1JIRbde3alQMHDgBgsVioVasWK1euZPHixbz33nvk5uaW+Pkg74SlUqkICQlhyZIlJCQksGHDBjZs2IAsy8yfP7/Y9xZVrseOHWPIkCF88803AERHR3Px4kV69OhR5DYee+yxAs1HycnJhdY5d+4cK1asYMWKFaxevZo6deqwbt06vvzySy5fvkx6ejpz587FYDCwYcOGAgkh35gxY1i7di3KzUkUfvjhBzp27EhgYGCB9T788EPGjx/P2rVrefvtt9m9e3eh7bRs2ZJXX321UELIN2TIEMfvGGD9+vUMHTrU8feJEycICQkpdNFlNpvp27dvoe2VZv233noLk8nExo0b+fbbbzl9+rQjCRV3fNxeZlarlR49evDTTz9Rp04dXnzxRV577TU2btzIvHnzmDJlCjExMUV+5orilDWFdu3a4enpycqVK4mJiWHPnj24ubkBMHLkSGbNmsXf//53vv32W4YNG4ZKpWLr1q2kp6ezc+dOAKxWK97e3o5thoaGAiBJEkuXLmXr1q18//33nD9/HkVRyM7O5rfffqNfv354eHgAeQd3/gFf0vZvl5WVxb59+7hx4waLFi1yLDt16hT9+/dn3rx5bNy4kcuXLztqRPluP8l17NjRcbA3b96cGzduFNpfYGAgzZo1c6yzbt06AH777TfHv319fenXr1+R8e7cuZNp06ahVqtRq9V88cUXAI73FufWch0xYgTr1q2jVatWrF27lilTppRYDreW18GDBx0/Und3d4YNG8a2bdsYMGDAHWOw2WxFLrdYLEiS5Pi7V69eALRo0QKLxUJWVhZ6vb7Q+6Kjoxk8eLBj2/7+/nzyySe4uLiwbds2Jk+ejFarBWDs2LGOK+uiFFeuvr6+PProo0yePJlVq1YxYsSIIk/UAP/3f/+Hl5fXHcugSZMmjmOke/fuTJw4kbi4OLp27co//vEP3N3dizxubtWxY0dcXFzYvXs3Xbp0YdWqVfzjH/8otF54eDizZ8/m119/pWvXrrz88st33G5RWrZsiUql4vjx43h7e5OZmUnjxo0dr6tUKkdNpDRKs/62bdv4+uuvkSQJnU7HqFGj+L//+z8mTpwIFH18FCX/mD969Cj16tWjTZs2QN4FUvv27dm7d2+B466iOWVS2Lp1K3PmzGH8+PH06tWL4OBgx1VFaGgoNpuNo0eP8v3337Ny5Uogr8lp+vTphIWFAXlV31uvBF1dXYG8k8/QoUPp3bs3oaGhDB8+nM2bN6MoChqNxnGVBBT4kZa0/dvJsoyiKKxcuRIXFxcAUlJS0Ov1/PHHHzz77LM8/vjj3H///XTo0IFZs2YVijWfwWBw/FuSpAIxlrTO7Z9JpSq68qnRaAocyHFxcRgMhkL7s1qtBd53a6zDhw9nyJAhjBw5kvT0dDp16kRGRkax5VBUed2+rLgTfr727dvz2WefkZ2d7dh+vj179tChQwfH3/n7zP+ciqLw2muvcfz4cQBGjRpFcHAw9erVY8OGDUXu7/YTjyzLhcrkVsWVa4MGDWjSpAlbtmxh48aNjlpDed36PbRu3ZotW7awa9cudu/ezciRI/n444/x9fV1rLNlyxZH7dTX15dly5YBMHr0aNasWYPJZCIrK4uuXbsW2teoUaPo0aMHO3bsYPv27Xz00UcFrvpLa9CgQXz33Xd4eXk5kvCtn+HChQtkZGQUuPqPj4/n9ddfZ/HixQWO+dKsX9R3d+vxVdTxUZT8si4qCSmKgs1mc1w0VAanbD7asWMHPXr04JFHHqFVq1Zs3rwZu93ueH3kyJG8+eabNGnSxFG17datG19++SUWiwVZlnn99dd5//33C2378uXLZGRk8NJLL9GzZ0/27t3reE9YWBg///wz6enpAKxZs8bxvtJuP5/RaKRt27Z8/vnnQF4b++jRo9myZQv79u2jZcuWjB8/no4dO7Jly5YCn68ihYWFOT5HamoqmzdvLvIqpkuXLqxbtw5ZlrFYLLz44ovs27ePWrVqOU6aKSkp7N+/v9h9+fn50aZNG2bOnMmIESOAO5fDrYxGI23atOHLL78E8tp/169fX+RJ6VZt27alY8eOREZGOq6E7XY7S5Ys4dKlS4waNeqO758zZ46jKWj06NF3XBfyrsJXrlyJ1WpFlmW+/PJL7r///mLXL65cAR555BHmz59PmzZt8PPzK3HfpbVgwQI++eQTevfuzWuvvUZISAiXLl1Co9Fgt9tRFIVevXo5Pnd+QoC8EU179uzhq6++4pFHHily+6NGjeLkyZMMGzaMN998k7S0tEK1ELVaXWJCHzx4MD/++CObNm1i4MCBBV7z8/MjIiKC6dOnk5GRAUBGRgZvvPEGJpOpQEIo7fr5v2FFUbBYLKxevbrE4+vWMrtdmzZtuHjxIkePHgXg7Nmz7Nu3j44dO95xm3fLKZPCqFGj2LdvHxEREfztb3+jbt26xMbGOjLzkCFDOHnyJCNHjnS859lnn6V27doMHTqU/v37oygKkZGRhbbdpEkTHnzwQcLDwxk6dCi//vorISEhXL58mS5duvDwww/zt7/9jWHDhpGenu64+izt9m+1YMECjhw5QkREBCNHjmTgwIEMGjSIgQMHkpqaSv/+/Rk2bBiurq7cuHHDcTBXpGnTpnHhwgUiIiJ48cUXCQwMLPSDAnj++efRarWOoZhhYWH07duXsWPHkpiYyEMPPcSUKVNKPOBHjhzJyZMnC7QPF1cOt1uwYAG7du0iIiKCESNG0LdvX4YNG1biZ3z33Xdp1aoVjz76KBEREfTv35+4uDhWrlyJu7t7KUqp9J555hl8fHwYMmQI4eHh2Gw2xxDRohRXrgA9evQgKyurxMRVVo899hinTp1i4MCBDB8+nDp16jBw4EDMZjPNmzcnPDzc0Ql9O6PRSJ8+fRx9eUV55ZVXWLx4MUOGDGHcuHE8//zz1KlTp8A6PXr0YN68eXdsfvTz86Nhw4bUr18fk8lU6PV//vOfhISEMGrUKAYPHszIkSMJCQnhrbfeKnJ7Ja0/Y8YMUlJSiIiIICIiggYNGhQ54OFWdyozLy8vFi1axJtvvklERAT/+Mc/mDt3rmPwQGWRlOLqMEKFO3bsGIcOHXKMS/788885cuQICxcurN7A7sKXX35J8+bNadeuHRaLhUceeYQXXnjB0QwmVJ+DBw/y+uuv8/3331dqG7Rwb3HKPoXq0qBBA5YtW8bq1auRJImAgADefPPN6g7rroSEhPDmm2862r779esnEkINMHXqVPbu3cu8efNEQhDKRNQUBEEQBAen7FMQBEEQiiaSgiAIguAgkoIgCILgIJKCIAiC4HBPjD5KTc1ElsveX+7tbSQ5ueLH7t8tEVfZ1dTYRFxlI+Iqu7LGplJJ1KrlVuzr90RSkGWlXEkh/701kYir7GpqbCKushFxlV1FxiaajwRBEAQHkRQEQRAEhxqRFDZu3Ej//v3p06ePY8IyQRAEoepVe59CfHw8H3zwAWvXrnXMQd6pUydCQkKqOzRBEASnU+01hZ07d9K5c2dMJhOurq489NBD/Pjjj9UdliAIglOq9qSQkJDgeB4y5D2QIz4+vhojEgRBcF7V3nxU1Hx8ZZ3V0du7+Ifbl8Rsrtj58CtKdcd1NSkDPy831KqC30VFxqUoCqu3nCGsXR38vYsfN11ad4pNUZQSj6uonRcJCvCgeYPiH4Oav60z0amE1K2FWiUhywp7/ogj1yrzYPs6BfanKAoWJLQaFWaTS7ExpGdZOHEhGS9PA43q1nJs43R0KiF1TKhVEjuPxZGQkkVtXyMdm/s73mu1yWw/fIX4lCz+1rsxl6+l8cveaMaFN8Ogz/uJp2VasNtlann8+awLL2+j4/vNyLKQa7Xj5qLFoCt4WkhIzeJaciatQ8zYZYWMLAuexj+fbJeQkoVepy6w7HZ2Wcl7+qBaRVaOlfOxN1CrJRrXq4VGXfDa9GTsDS7FpeFjciG8S30kSSI9y8KilYdw0WsIbebH/pPxtG/qS4/76haI8/j5JPy83GgR7M3VpAzOxVwnwMeNQB8jFqud4xeSCa7tiVol8d32CwQHenJfM182br9AWqaFWu4GwtrXxs2g5fj5ZHYcu0pwoCcjexkdx1diajb/jTqBVq3i2RFtCsSvKAox8ekcPpuILEOu1caFKzc4F3sD2S7z3Mi2hDbzw2aXOXg6AZtNxuiq5fTlVNQqiQAfN2Lj0wjwdqVr60AklRpZVkhIzeLQqTi8PV3p1Kp2ofKtyN9ltScFPz+/Ak/bSkhIKPBYv9JITs4o1zhds9mdxMT0Mr+vIqzffgEvDwMPtAks9Fp548rKsXE29jptQnwKLE/LsrDz2DV6tK+NXlv0c3rz5VhsrNxylm1H4hj2QDADu9a/Y1y5VjuxiRk0DPR0LJNlBQUFdTGP5syXdD2bL6JOcTU+nUf7NiE+NYtdx6+RmWNjdO9G5FrsHL+YQuuG3oXizsi2cikujSPnkjl75TovjW6PyVD04ZxrsTPt01082LY2g7oV/YCSQ2cS+WTtMTxctcyZ2JnYhAzOxFwn22Knvr87LRt443pz+7v/uManG0/wwvBWtGzgxTtfHuJiXBoAWhQOn01i/+kEJo1oTdS+GHYejQOgRf1aPDOkJQa9hr0n4/njYgqjezXi3JU0Pvz2KPabx3D7xmZG9Qxh+5Gr/LL7LO0b+2L21LF7/xncVTnoJBtJ7YNo3zqYbEXDv7/dR256GlrJhmvKGU6fv0ZORjqrYvbTp1MDLidm8/Ou86jlXOp56wkOMHI9I4ezl5MJqKVHIykkpWaiRkarkvFy01DLw4Cbq4HMXDuXr6Yg2W1cr+3N9RtZZGTmEBToiauLgSvJWSSlZqKVZHw8dBgNeY9nzc61kWOxofetT9sBw1kWdY7L19J5bmgrPl57lLTUVNykXAK9DPTpUAcfkyvederj5uHKu18cQJJAUSD1ejatG3rz4bdHSb6eiafGwvkjh/FUZfNbXB1a1jMBsP3oVZZvOoUCSMCD7Wqz63gcki0bNykXNykXF5UFF8mCq2TBVWXFgIVYyUri9zaMkg2zWkYlWzmzx4YeG3rJRrhkh8sKJ/erUVBhkSVsdpm+WFEhc3qeFkmjxe7mi3fHfuy9kMXuQxdwlSy4Sbm4qnIJMdjpbLBCbibXV63jd60Esh1kO2pJJgeZEMmOFjs6yYaflPewr0ub/jw+ZQWaSpCmuBKbOJRsgw831F4EBQeV+XyhUkl3vJCu9qmz4+PjHc9tdXFxYdSoUbz55pu0bt261Nv4qyUFWVF47oNtaFQS7z13P7rbTnhmszvRsam46P88yVmsdnKsdjxcdcVud9nGE+z64xpvjO9APb+8Kwe7LLPg68OcjrnOA20CeTy8KQAX49LIzLHS8rar4v/78RTbDl/F5K5HURTmP9PVcXUXk5KNJcfKgdOJxCSk071NIL8dvsrla+m8/HAbWgZ7k51rY+4XB6jlbuClka2RJIkci42vfjlLaFNfWjf8c3+nLqcy/+tDeHvoef2xDkz91y5yLXmPDf3HqLYcPpvElgOxuLtq6R1al7C2geRY7Hy/8xI7jsahAFqNCr1WjVajolMzX7YfucI/Rrenvr+HYz/nr95gzn8PABDaxExqRi5J13OQJOjc3J/aZje++d85DDoNiTeyqetr5Er8DTxU2ZjUOejJxcNVw5CwJnh6uPH5d4cgJ4PGXR6gYX1/3vy//Qy6vz47jl0jx2IjJycXkzoXjWTHSCY9GhkwKulcio7HoAGNZEexWdEg4+OuITcnB71KJsCoIOdkIFmzkJHQIKORSv9w+buhIKGoNNhRYZUlFFlGhYJKUpAlDbKkRrJbkFGBWoNit6NGRiUpqDQaZNTk2hTsct628mogCiYpA/RuZObYUWPDrqhwkayopMK/11yDN6oGHThw+Cxt/CA3PRWrJW9drWTHXcpB4s/3yYqEtkE7kq0unLwQT4CbTG0PyLqRitqahavKgorizwuKSo1dpcOiaDG4uqDRu2BXabmeDbJKi6vRFXd3N87HZRAbn4ZWBR4uaowuGgL8vLh2w8Kl2BS0ko3G2nhMqszCO5EkJJ0bkosHit6N5HQbOTYFRVLj6eGCVqvFKksYja6g1pFlk3B1cyU2KYvzV27g7qLB002Hq05Feq6CKv4kIdoEAK7gR9OJ8yo8KdSImsLkyZMZN24cVquVESNGlCkhVIXsXFuBE3R5HT6bhNlkQKdVk2uxkwvsORlP99aByIrCiYspNG/gxeVraUxatJ3nhrWi7c2r/v/+dJozMdeZ93QXJEkiNiGDXw/GotOqCanticmoZ9cf1wDYfSLekRTWbrvA6ZjrNK5rYtuRq7Rs4MV9Tcx8+t0fxKdmM6BLEA91rIfRRYusKBw8k0iHZr50aeHPojVHOXgmkY7N/Fi77QK/HrwC5J2IvT0M/PfH0+h1ajxctaz//SLN63uxbOMJYhMziU3M5MSlVFo08OJU9HV+PxbH78fieKBNIOP6NUElSSSn5QCQnJbLV5vPkGuxM2NcKAu/OcKPuy9z7koaLYJMuKuy2b9jLyd35V3xuassPBusobYhBzf7deT0ZCzZWejO2ujrCknbj2DvMQRJ74Zk9OZKYt6PtXVDb06dv0ZTX4nWdexI2TdIPXaYRLIZosmhtZ+GbH0y6tw03Lwshb/A33/CBozVAEbIPHmczMzujHA9Q+f433nAeJ2srGw8XLMp0Eh0s4ssyE2HVVYhS2rUrlpsiprrWXZAjb+PCRd3DySDOznoOHs5FVRq2rSsz+noVDJy7HRoF4La1YRd0rJx+1muXrmGAStNG9ela2gI6bkS/7fxEMF1zQwIa8Zvh2JISknH00VNt3ZBuLi5Iqt1XLyWSe1AEy4aNYqkQlGpUasKXpikZVpIupGDi16Nn5crsqzw095omgbVIjjAgyPnklGrJZrWM6HV/Pleq82OJElo1Cq+23GRY7t283zrGxw+lYLJ00hmdg7BQQHUruOPZDBisUvEp2bz+4HzhGnP4nXyR5ppXTBoAnGp04CT0WnodRrq1/FG7+mF5GpCcjVx4HIucYd/p1f8efTZObTSq3F390JlcMfgGUKqRYPWxxuNizuSwYhkcEPSuYHeDUnviqRzBbW2yOY8j9v+bgm0d9GRk5mL6pbmVBPgfT2bw+eSmLX5NLP6u7Nyy1laNK1H3/ubIundQOeCJP1ZYy6poTu/EajZzf9udTo6lXlf1WNquJlfdp3D4FObpiVsrzyqPSkAjmea1kTR8enMWr6PN8Z3pK5v6fsuLl9LZ++peEaENXQceP/54QSN6pjo3iYAAL1OzZYDsXRrFcDxC8ks/OYoTw9uQVqOHbus8Mu+GNqG+JCWaWHPiXjsskLijRzORF/ns00n0WlVKAr8vC8GCfBw1RLo48aeE/GMeLAhVqvMlgOxdG7hx4T+zXjrv/tZ/b9z+NZyIT41m9o+bvyw6zI/7LrMfU3M9OtYj/QsK21DfGgV7I3ZZGDzgVjaNfJhz4l4OrXwJ6x1ALU9FAw5SVy8kIGXJpuUhASiL8VyYNn33G/L4RG/LGRLDrm/rCenTTdyk13oazhPc38V8ed2cnSliZa9BpB8XcFPdR21JJN0Np6/BWQTcOos/6gVQ0Kynf4uOdTOykSyW8DztgK+oUKye6FyN6Op1xqVxkB6Lpw8HcN9qUfJWnsIAMUzALPdh+me0fil54BHLuSQ9x+ACygqDZKLJyrJhD6wLmmyKzpfXyQ3EyoXTyS9G9czrfx+6DLpmdkE+pq4GJtMf3k7vhc24qHXolUHoanTBFuugtrbF627F5JaiykwgHS7K5LRG0mtLfARZFlh46aT+Hu50uKWZjoD0PmW9dq2LfjRNcCwh0P47veLXLqWTpd+LdFo1NQCnn0qGK1GhSRJ9Ozhz+1UQKOGJsfVZXG9LB5uOjzc/qyVqtQSA7r8GWPbRj5FvIsCCaKur5H1Nj92enTlm6zzvDSgDZ0bFqyZaoFg4PtYb/6V0pK2jbzYfuQai4d2R5Ikintit0lK5dNd2QTcN5ZPvzvB8LDgAvG5FvO+8vI06rFkF75QMJtcaB3szdeo+CnWgz9yAwlr0BSVR9mawEujjq8RkDia4srB6yZGti38/VaEGpEUarJL19JRFLiSmIGPp4F/f3+CZkG1CGsbiFaj5vejcZyJuc6EAQXz+v8OxbLtSBydm/tT19dIjsVG5s02/wYBedcDg+6vzzf/O09MQgbnruS1Se/+I57MXBsScPJyKnHJmRw+m+Robz4fe4NtR69S28eNqWPa46JXc/xCCtuOXKVrywCsdjuffneCM9HXyci2YrHKdG8VgEaton/nIJZu+IMVP51GAl4Z3Y5ryZnsOZnA1kNXSEnLRSVJtKhnRI45zKNBVzl8Oo7/fXOaMCmOnoDu9ysomSnkAAE3P2uApMZNZyBL0eHh5Y6bfzBxaTIJMTF4HomiOQrNXYFcNwJcVajSLpK9fj+dVUZ6mG6ZyCsX7HG18HT3IS0jFZveE12zDqg8/VC5ed284jOCwZhXJb+lzyL/JLfi2kFO2DoxuBls2XGCPoareOWeJ07jS52mjZFcTahcPR1XnCpXz7yrx1uuGIvq8vYGBjds7vj7g9VH+E9GHZoHathyOouPh+Y9gtTltve5mt3JLKZqr1JJPDGweZGvlUQlSQzpHlxo+e1NkdUp/yJq580a7J0uqhoEenD4XBJ/XFJRx2wscVBAoDnvW9p+JK+/JqT27VcNVce3lgvurlp23/ycwYG31zUqhptBi7eHnh3H8/bTIKBy9iOSQhGuJGbw0dpjTB3TnsTr2QCkZuRyIS6NQ2eTOHQ2icPnknhlVDt2Ho/jVPR1hoUFY7pl9MWFq3kn+aPnk6jrayQ1PReAzBwb+04l4ONpoEsLf77533lOXErl4tUbABy7kIxdVuh9Xx3+d+gKq349x5XETELqeBKTkMGxi8lcuJJGeOe8Jh+ANiE+js7lXKsdvU7Nz/ti0GlVGF20NL7ZGde+sRlPo47zV9NoUteEp5sOD51CfbUCl66SkXyB/j4J8M1Ksm251AfquwLpIBsk9NY6KAFNUHvXQ1WrNpKxFpKLJ5LBiMYi46uWHFeKgRYbcxduJ6KpiXPnruDmG8hTw9rjYpeZ/P5mxte/jCo9nkuqujQMDuRkTBrDhvZEY8wbeXPieBxB/h4YfMo2KinAx419JzM4odRne67CVTqRnJtD8/petO1avhNwUbw99FyMS8Pd3RUfz5o7UVp18vYw4KrXcCUxEzeDBpOx+P6w4JsnuJj4DHrdHMF1Jx6uOjxctZy8nIpKkqhfSSfI0pAkiZDanhw6m4TJqMPrlhFeFa2O2ciR88lIQJB/5YxQFEmhCOeu3CA+NZtzsTdISL2ZFNJyMRryTsL3NTFz8Ewi2bk2LsfnXQX+cTGF+1vlXTvnWGxcScprxz5yLpkBXeo7kgJAbGImbUN8MBn1BHi7cuJyChfi0qnv786la3nb69Y6AIvNzrabV0KjejXif4di2XsiAVlRaBVc9LBJvVbtqIFIEnRvHegYBaRGpl8LI3sOnGKo21ky16xDTo0FRWGoBBjBiivaRl3QBHdE5VWHuOsWPvxqN11DG/P3EfcV26Hloi840sig0xBS25Nd53OIv+7K0NZeAGjUKszmWmyxmUm25FA/wJ0hD7ak1W3b69oygPII8HYlM8fG4XNJQF5yVoDa5rsf8norLw8DGdlWriZnUtun/EOi72WSJFHX18jpmOvU9b3z1X9+7Rmgtm/pvqtAHzfSoq9Tz89Y4qi6ytaojolDZ5MKjMKrDHX98pJCoI9bhfRzFkUkhSLkn8CvJGWScEtNwdWgQQK6tvTnwOlE9pyMJzs3b7TMHxdTaN/YzPWMXG5kWFCUvAP9/NUbZGRbSUnL26ZGLWGzK46TVNOgWmw9dAVFyRtGF7X7MnZZoa6vkcf6NeXhHo1QqyT0OjXR8emcuJSKQae+YxX1oQ71OHoumdMx1wltYsYefw7LsZ+xxRylqzWHrh5AkgYpoAm6+hGozcFg9Obk5VQaNW+KwfDnFV1tF4h8qo+jVlIWLRp4sXbbBQDq3dJ0UM/PnQOnE8ix2Lmvibm4t5dL4M37Hc7EXMfH00DSjbzOg4o+cXt75l0NJl7PoU3DotvXBRxJobb5zuXvatDi7+XKtZQs6pawbr7aZiOnoq8TUqf6mo7y5cdQWU1H+er65iXPymo6ApEUAPhlfwynLqei16kZH97MkRRiEzP+rCmk52LQqTG562lc1wTArwfyRuPU9nHjj0spvL/qMJeupXN/q7wOoCHdg/lg9RGOXUgmNT3v5NSygTeHzyVR5+aB36xeLf53c1RPcKAHEwe1wN39z5ucXG8Ze59/4DULKnzDz60kCZ56KIjTBxNpcPIzsq78gaQ3om3YGZVvAyS9G5rAZnmjI27Ryrtukdsz3eGmpDspkBT8/rwSDPIzsu3IVQB8PCu2qh3g/WcXY4/2tdm44xI5Fju1y9gMVRLvW5oIKvoz3Evy+xFKM0ijQYAH11KyCCzld5X/nVZnf0K+4EAPRvdqRJeWldP5my+/yagyE6FICsAPuy6TlWPDZpfpE1qX1Iy8pHA29gbZuTYgLynotWq8PPS4GbQEeLsSm5iBRi3Rp0NdlkedIj3LikYtse1IHL61XGjRwAs3g4bT0XntnkYXLa2CvTh8Lol6fnk/kiY32/v1OjWB3m6oVFKx444bBnriZtDQqblfkZ/DnnIFy9Eo7DFHUWen0RxQXE3oOo5E16IXkrZqT15B/u6OGsat7cm3JgjvCj6h1nLXY9CpybHYaVTbRLOgWpyKvo6XR/kSW3Fu3V5Ff4Z7SfP6XtT1NdI8qFaJ6/brVI+2TX1L3SzSJsSHDpdTC91rUx1UUt55oLL5mlz45+MdqFPKJrbyEEkBsNtlQmp7cCr6Ogmp2Y6aQlpm3hC0OmY3riZloVZJjuphw0BP4pKzqGM20qahNxq1RPc2gZjcdKzbfpHgQA9UkkSQvzuX4zPwdNPh5a6ne5tAArzdCLjZzOHuqqNBgDtuBm2BMdBFcTVoWDype6G2WcWSjeVoFJbDP4Bah6ZeG9S+wahM/qhrN0dSVc/XrJIket1Xh1yrvUDMdXyNjjtWvSu4U06SJAK83YiOT6een5FRvRqRmp5b5qlTSlLLXe/4DD6et485EvJ5exqYNaG4gaUF1fU10r5FQKlvxKrlrueZIS3vJry/pMrqYM4nkgJgsyv4e7vlJYXr2VxPz8XdVUt6lhWAJnVrEZuYSdKNHDo0zRt/HFLHk9+PxVHf3x1Po553nuqCyV2P1SZzOuY6nZrlXc3X83Nn8/4YrDZXzJ4GNGoVTW+7anpxeGukEhJCvltPbooiYz25Fcv+dSg56WhCuqDv+ggqQ82Zz2lwEdNK6LVq/L1ciUvOqpSr7PaNffDzckGnVWM2uWA2VfxJW61SUctdT0parqgpCPcUkRTImwrCVZ83ZC42IYPMHBudW/ix+4+8W1Eb1fVky8FYAMdws0Y32/Qa3mzPzF+u16p5ZVQ7x7aD/Nyx2RWuJmU6+iJud6eJxIojpyeS879l2K+dQR3QFH2nh1H7Fh63XlM1CPAgK8dWaPK1inDrTUyVydvDQGaODbdi5lwShL8ipz+aFUXBZldQqyR8TS6cibkOQNN6tdh3MgEPNx1+tf7svMxv7gjwduOfj3cosQMtv+8A8qq7FcF6YS852z4HBQwPPoGm0f0V3jxS2UY82JA+oZXfBluZGgZ6ort597Ag3CucPinIN+cDVKslfGu5ciY27yYys6eBAG83jC4aTO5FdyqWpm3Pz8sVvVZNrtWO110mBTk9EcvB77Ce3o7KNxiXns+g8qjYIZ1VxWTUl3tUU03xcE/xdEDh3uP0ScFuz0sKGrUKc60/255N7nomRjRHrZZwd9WiVknYZQXvMo5iUUkSdf2MnIu9Ue6agiLL5O7+GuvxzSBJ6Nr0R9dhWLV1IAuCcO9y+rOK7WZSUKukAuPN84Y2/lk8JqOerFwrroay38QV5Ote7qSg2Czk/LoU26WDaJv1QNcuApXRq8zbEQRBKA2nTwp2OW++erVKcoxScdFrCnWA1vLQY8gt36309zUxczkhvcxDF5WcDLJ+Wogcfx591zHoWvYp1/4FQRBKSyQF+c/mI9+bzUdFtf0P7dYAq718E581DarF9KD7yvQeOS2RrKj3UDKSMPR+Fm1wh3LtWxAEoSycPinY7H/WFNwMWowu2gIdy/ma1a+6Jhtr6jWyNryJIttxGfAqGv/GVbZvQRCcm9MnhVtrCgB9O9TFx1R9NyMp1hyurZuHIttxHTQdda3CD+kWBEGoLE6fFBwdzeq8sea3Pqi+qimynZz/fYotKRaX8JdFQhAEocoVP9Wmk7Df0nxUnRRZzksIlw7i3edxNHWcb04XQRCqn0gKcn5NoXqLwnJkE7bze9B1HIlnhwHVGosgCM5LJIX8m9eqsaZgT47GcmAdmuCO6Nr0r7Y4BEEQRFKQq7f5SM5JJ3vzJ0h6I4Zu48Q8OoIgVCunTwq2amw+Uqy5ZEd9gJKRjKHPc0gG8axfQRCql9MnBUdHs7rqr9BzD6xDTryAodcz4l4EQRBqBJEUHH0KVVsU9pQYrMd+Rtv0AbT121fpvgVBEIrj9Enhz+ajqq0p5O78Cknnir7jw1W6X0EQhDtx+qRQHfcp2OJOY796El37QaIfQRCEGkUkhdumuagKlsM/IBnc0TYLq7J9CoIglIbTJwVbFdcU7EmXsMccRduqL5Lmr/3kMUEQ7j0VnhTWr19Pt27dGDx4MIMHD+aDDz4A4OrVq4wZM4Z+/frxzDPPkJmZCUBaWhoTJ04kPDycMWPGkJiYWNEh3VFV3tGsKAq5u75G0hvRNe9Z6fsTBEEoqwo/Ex47dozIyEg2bNjAhg0bmDx5MgCzZs3ikUce4ccff6Rly5Z88sknACxcuJDQ0FCioqIYOXIkc+bMqeiQ7ujPx3FWfk3BdmEv9rjT6DoMR9K7Vfr+BEEQyqpSksL69esZNGgQr7zyCjdu3MBqtbJv3z4eeughAIYNG8aPP/4IwNatW4mIiABg4MCBbNu2DavVWtFhFctWRXc0K7Kd3D2rUXkHoW0q+hIEQaiZKnzqbLPZzMSJE2ndujXvv/8+s2fPZurUqRiNRjQajWOd+Ph4ABISEjCbzXnBaDQYjUZSUlLw8/Mr9T69vcs/gsdg0AHg7+eJVlN5TUiZp3aTkZGMb78ncPPzLHF9s9m90mK5GzU1Lqi5sYm4ykbEVXYVGVu5k0JUVBRz584tsCw4OJjly5c7/n7iiSfo3bs3r776aqH332mOH1UZbyRLTs5Alsv+qEyz2Z0badkApKZkVOq8Q1k7NyIZvck0NSErMb3EuBJLWKc61NS4oObGJuIqGxFX2ZU1NpVKuuOFdLmTQnh4OOHh4QWWpaens3z5ch5//HEgr2NVo9Hg5eVFRkYGdrsdtVpNYmIivr6+APj6+pKUlIS/vz82m42MjAxMJlN5wyozu6ygkqRKTQj2lFjscafQdXwYqYrvnBYEQSiLCj1Dubq68u9//5sjR44A8MUXX9CnTx+0Wi2hoaFs2rQJyBuh9MADDwAQFhbG+vXrAdi0aROhoaFotdqKDOuO7LJSqZ3MiqKQu2cVaA3omj5QafsRBEGoCBXap6BWq1m4cCFvvPEGOTk51K9fn/nz5wPwz3/+k8jISJYsWUJAQADvv/8+AJMmTSIyMpIBAwbg7u7OggULKjKkEtnscqVOcWG7sA97zDH0XR4Rdy8LglDjVXhHc2hoKOvWrSu0vHbt2qxYsaLQcpPJxNKlSys6jFKzywrqSmrSUew2cnd9hcqnPtoWvStlH4IgCBXJ6Ru47Xal0moKtgt7UbKuo+8wXPQlCILwl+D0Zyq7Xa60R3Fa/tiM5OmPuk6LStm+IAhCRRNJQVYqZYoLe8IF5IQL6Fr0QpKcvpgFQfiLcPqzlc0uV8rdzNbT20CjQ9u4W4VvWxAEobI4fVKojI5mRbZhu7AfTVA7JJ1LhW5bEAShMomkUAn3KdivnEDJzUDTsFOFblcQBKGyOX1SqIz7FKzn94DOBU3dVhW6XUEQhMrm9EnBbq/Y5iPFbsV28SCa+vchqavuzmxBEISKIJJCBTcf2WKOgTUbrWg6EgThL8jpk0Le6KOKKwbb+T1IBnfUtZtV2DYFQRCqitMnhYqsKSjWXGyXD6FpEIqkqvAZRARBECqd0yeFirxPwRZ9GGwWMepIEIS/LKdPChV5R7Pt4gEkF0/U/o0rZHuCIAhVTSQFu1Ihcx8psh1b7DE09VqLye8EQfjLcvqzl02umPsU7PHnwJKNum7rCohKEAShejh9Uqio+xTs0UdAUqOp07ICohIEQageIinIFfM8BVvMUdQBjcVcR4Ig/KWJpGCX0dxlTUHOSEZOiUVTTzQdCYLw1yaSQgXUFGzRRwFQ121TESEJgiBUG6dOCoqi3Jw6+26TwhEkdzMqU0AFRSYIglA9nDop2OwKwF3dp6DYLNivnkBTtzWSVDmP9RQEQagqTp0U7HYZ4K6mubDHnc67i7meaDoSBOGvz6mTgu1mUribIam2mKOg1qEObFpRYQmCIFQbJ08KN5uP7qJPwX7tDGq/hkgaXUWFJQiCUG2cOinY5btrPlIs2cjJ0WKuI0EQ7hlOnRSstrtrPrInXABFQe3fqCLDEgRBqDZOnRTscv7oo/LVFOzXzoAkofZtWJFhCYIgVBunTgo2W37zUTlrCvHnUHnVFVNbCIJwz3DupOAYfVT2moIi27HHn0PtJ5qOBEG4d9x1Uli0aBEffvih4++0tDQmTpxIeHg4Y8aMITExEQCLxcKUKVMIDw9n6NChnD9/Hsi7q3jevHn069eP/v37c+DAgbsNqdTym4/K09Esp8SALVf0JwiCcE8pd1JIT09n+vTpfPbZZwWWL1y4kNDQUKKiohg5ciRz5swBYMWKFbi4uBAVFcX06dOJjIwE4KeffuL8+fNs2rSJjz/+mMjISGw22118pNK7m45m+7Wzee8VSUEQhHtIuZPCli1bqF+/PuPHjy+wfOvWrURERAAwcOBAtm3bhtVqZevWrQwaNAiADh06kJqaytWrV/ntt9/o378/KpWKBg0aEBgYyKFDh+7iI5Ve/pDU8jQf2a+dRXLzQmX0ruiwBEEQqo2mvG8cMmQIQIGmI4CEhATMZnPexjUajEYjKSkpBZYDmM1mrl27RkJCAr6+voWWl4W3t7FcnyEmJRsAH28jZrN7qd+nKArRCedwDWpWpveVRWVt927V1Lig5sYm4iobEVfZVWRsJSaFqKgo5s6dW2BZcHAwy5cvL/VOVMU0z6hUKhRFKfX6xUlOzkCWC2+nJPkdzenp2SQmppf6fXJ6IvaMFGymBmV6X2mZze6Vst27VVPjgpobm4irbERcZVfW2FQq6Y4X0iUmhfDwcMLDw0u9Q19fX5KSkvD398dms5GRkYHJZMLX15fExESCgoIASExMxNfXFz8/P0dn9K3Lq4K9nKOPRH+CIAj3qgofkhoWFsb69esB2LRpE6GhoWi1WsLCwtiwYQMA+/fvR6/XExgYyAMPPMDGjRux2+1cvnyZS5cu0apVq4oOq0g2W/mmzrbHnwOtAZVX3coISxAEodqUu0+hOJMmTSIyMpIBAwbg7u7OggULABg7diwzZ85kwIAB6HQ65s+fD0C/fv04evSooxN6zpw5GAyGig6rSLb8uY/KWlNIuoTaJwjpLh/jKQiCUNPcdVJ44YUXCvxtMplYunRpofX0ej3z5s0rtFySJKZOncrUqVPvNpQyczQfleE+BUWRkVNi0TYNq6ywBEEQqo1TX+paHVNnl74YlLREsFlQedWprLAEQRCqjVMnhfLUFOwpMXnvEf0JgiDcg5w7Kchlf8iOnBwDSKi8aldSVIIgCNXHuZPCzeYjlVSGpJASi+Tph6TRV1ZYgiAI1capk4J888Y5VRlqCvaUWNSiP0EQhHuUcyeFMjYfKdYclLQEcX+CIAj3LKdOCvl9CqVtPpJTrwAKKm9RUxAE4d7k1Ekhv6ZQ2i4Fe7IYeSQIwr3NuZOCoqCSJKTS1hRSYkBrQHL3qeTIBEEQqodTJwW7XaYsM1XIKbGovOogSU5dbIIg3MOc+uwmK6UfeaQoCvbkGDHySBCEe5pTJwW7LJe6k1nJTAFLlhh5JAjCPc2pk4IsK6UfeXRzeguVt0gKgiDcu0RSKGXzkT0lFkA0HwmCcE9z6qRgL0NSkJNjkYzeSDrXSo5KEASh+jh1UpBlpdR3M8spMWK6bEEQ7nlOnRTsskJpcoJityJfjxM3rQmCcM9z6qQgK0qpblyTU6+CIotOZkEQ7nnOnRRK2Xwk3+xkFsNRBUG41zl1UihtR7M9JQbUGlSeflUQlSAIQvVx6qRQ2vsU5OQYVLVqI6nUVRCVIAhC9RFJoVTNRzGi6UgQBKfg1EnBXoqagpydhpKdJkYeCYLgFJw6KchKyTWFPzuZa1dFSIIgCNXKuZOCXSlx6mw5LQEAlad/FUQkCIJQvZw7KSglNx8paQmgUiO5eVVRVIIgCNXHqZOCvRT3KchpCUjuZqSyPI1HEAThL8qpz3SyXPIdzXJaPCoP3yqKSBAEoXrddVJYtGgRH374oePvffv20alTJwYPHszgwYOZNm0aAGlpaUycOJHw8HDGjBlDYmIiABaLhSlTphAeHs7QoUM5f/783YZUaiXd0awoCnJaokgKgiA4jXInhfT0dKZPn85nn31WYPmxY8eYMGECGzZsYMOGDcydOxeAhQsXEhoaSlRUFCNHjmTOnDkArFixAhcXF6Kiopg+fTqRkZF38XHKxi7Ldxx9pOSkgzVHJAVBEJxGuZPCli1bqF+/PuPHjy+w/NixY+zYsYMhQ4bw9NNPExcXB8DWrVuJiIgAYODAgWzbtg2r1crWrVsZNGgQAB06dCA1NZWrV6+WN6wykWXu2NGsOEYeiaQgCIJzKHdSGDJkCBMnTkStLjj1g7u7O+PGjWP9+vWEhYUxefJkABISEjCbzQBoNBqMRiMpKSkFlgOYzWauXbtW3rDKpKT7FOQb8QBIoqYgCIKT0JS0QlRUlKMJKF9wcDDLly8vcv3Zs2c7/j169Gjee+890tPTi1xXVcyInuKWF8fb21im9fPZZRkXgxaz2b3I11NO3iAHCb8GwUgabbn2UV7FxVTdampcUHNjE3GVjYir7CoythKTQnh4OOHh4aXamCzL/Otf/ypUg9BoNPj6+pKUlIS/vz82m42MjAxMJhO+vr4kJiYSFBQEQGJiIr6+ZbsyT07OQJaVMr0nL14Fq9VGYmLRSSs7LgbJ6EVSag6QU+btl5fZ7F5sTNWppsYFNTc2EVfZiLjKrqyxqVTSHS+kK3RIqkql4pdffuGnn34CYP369bRp0wYXFxfCwsJYv349AJs2bSI0NBStVktYWBgbNmwAYP/+/ej1egIDAysyrGKVNHW2nJYgOpkFQXAqJdYUymrevHm8/vrrfPzxx3h5eTF//nwAJk2aRGRkJAMGDMDd3Z0FCxYAMHbsWGbOnMmAAQPQ6XSO9atCSVNnK2kJqOu3q7J4BEEQqttdJ4UXXnihwN+NGjVi5cqVhdYzmUwsXbq00HK9Xs+8efPuNoxyudPU2YolGyUnXXQyC4LgVJz6juY7TZ3tmAjPQzxtTRAE5+HUSUFWir+j+c+kIGoKgiA4D6dOCnZ7aWoKIikIguA8nDopyIqCVEwJKGkJSAZ3JJ1L1QYlCIJQjZw7KdxhQjw5LUF0MguC4HScOimU1NEsmo4EQXA2Tp0UihuSqtitKBkpIikIguB0nDopFFdTUNKTAEUkBUEQnI7TJgVZyZsrqaiaghh5JAiCs3LepCDfISlkJAMguftUaUyCIAjVTSSFIvqZlYwUkNRILp5VHJUgCEL1ct6kcKfmo4xkJDcTUhmf6yAIgvBX57RnvfyagrqojubMFFRG76oOSRAEodo5b1K4+UweqbiagtGriiMSBEGofs6bFPJrCrclBUWWUTJTRU1BEASn5LRJwe7oaL4tKWTfANmO5CZqCoIgOB+nTQpKMR3NSmZK3nJRUxAEwQk5bVKQi6kpyBl5SUH0KQiC4IycNinYHTWFgsuVmzeuiZqCIAjOyGmTQnF3NMsZKaDRg861OsISBEGoViIp3N7RnJGMyuiNVMyU2oIgCPcy500KN+9TKNSnkJki+hMEQXBazpsUirtPISMZlUgKgiA4KedNCjc7mm+9o1mxW1Gy05DcRCezIAjOyXmTQhE1BSUzFUDUFARBcFpOmxSKuqPZ8RwFMRxVEAQn5bRJwXFH8y1dCkpG/t3MoqYgCIJzctqkUNR9Co6agpj3SBAEJ+W0ScFexNxHSkYKksEdSaOrrrAEQRCqVbmTwoEDBxg+fDiDBw/mscce48qVKwCkpaUxceJEwsPDGTNmDImJiQBYLBamTJlCeHg4Q4cO5fz580BeM868efPo168f/fv358CBAxXwsUomy3n/L9CnIO5REATByZU7KUyZMoU5c+awYcMGIiIieOuttwBYuHAhoaGhREVFMXLkSObMmQPAihUrcHFxISoqiunTpxMZGQnATz/9xPnz59m0aRMff/wxkZGR2Gy2Cvhod1ZU85GSkYxKNB0JguDEypUULBYLkyZNomnTpgA0adKEuLg4ALZu3UpERAQAAwcOZNu2bVitVrZu3cqgQYMA6NChA6mpqVy9epXffvuN/v37o1KpaNCgAYGBgRw6dKgiPtsdOZ7RXGD0UYoYeSQIglMrV1LQ6XQMHjwYAFmW+eijj+jduzcACQkJmM1mADQaDUajkZSUlALLAcxmM9euXSMhIQFfX99Cyyvb7fcpKJYssGaLkUeCIDg1TUkrREVFMXfu3ALLgoODWb58ORaLxdHc89RTTxW7DdXt81Pfsjx/aGhp1i+Ot7exTOsDGGPTbr7XDbPZHUtCKhmAZ2AdjGb3Mm+voplrQAxFqalxQc2NTcRVNiKusqvI2EpMCuHh4YSHhxdanpmZyTPPPIPJZGLJkiVotVoAfH19SUpKwt/fH5vNRkZGBiaTCV9fXxITEwkKCgIgMTERX19f/Pz8HJ3Rty4vi+TkDMeVf2ldv5GV9//rWbioJWwx0QBk2F3ITkwv07YqmtnsTmI1x1CUmhoX1NzYRFxlI+Iqu7LGplJJd7yQvquO5qCgIBYtWoRO9+cQzrCwMNavXw/Apk2bCA0NRavVEhYWxoYNGwDYv38/er2ewMBAHnjgATZu3Ijdbufy5ctcunSJVq1alTesUrPf1nwk35ziQow+EgTBmZVYUyjKiRMn2LJlCyEhIQwZMgTIqyEsW7aMSZMmERkZyYABA3B3d2fBggUAjB07lpkzZzJgwAB0Oh3z588HoF+/fhw9etTRCT1nzhwMBkMFfLQ7u72jWcm6AYDk4lnp+xYEQaipypUUmjdvzunTp4t8zWQysXTp0kLL9Xo98+bNK7RckiSmTp3K1KlTyxNKueV3ZeQPSVWyriPpjUjqchWJIAjCPcF572i+7T4FJfsGkqupGiMSBEGofk6bFG5/HKecdR3JVTQdCYLg3Jw3Kdw295GSdUMkBUEQnJ7zJoVbagqKoqBk3UAlmo8EQXByzpsUlFuGpOZmgmwTNQVBEJye8yYFR0czyNliOKogCAKIpIAkSX/eoyCajwRBcHLOmxRuuU9Bybqe92/RfCQIgpNz2qRglxUk6WZHs6gpCIIgAE6cFBRF+XPeo6zroNGBtvKn1xAEQajJnDYp2GXlz3mPsm8guXgi3fLAHUEQBGfktElBlpUCN66JexQEQRCcOSnc0nykiCkuBEEQgHLOknovuLWmIGenoQ5sXs0RCfcqu91GamoiNpulukNxSEhQIctydYdRiIir7O4Um0ajo1YtM+oyzP7svElBuTkc1W6D3EwkV4/qDkm4R6WmJmIwuOLm5l9j+q00GhU2W807yYm4yq642BRFITMzjdTURHx8Akq9PedtPpJl1CoJJSfvMXaSQSQFoXLYbBbc3DxqTEIQnIMkSbi5eZS5hurESeHmPQrZaQBILiIpCJVHJAShOpTnuHPepKAoqNQqkRQEoYqMGBFBXNzV6g5DKIHzJgVZQX1LTUElkoIgCIIzdzQrqFSg5IiaguA8Dh7cz5Ili5FlGX//AFxcXLlw4TyyLDNmzDh69uzD4MH9WL16Pa6ubjzzzATuv/8BHn30cTZv/onDhw/xzDPPM3fumyQmJpCUlEjbtu2YMWM2hw4dYMmSxdjtMsHBDXnxxZeZPft1EhLiqV8/GIslr2373LmzzJ8/B7vdjk6nY/r0f1K3br1qLhkhn9MmBbusoFKpULLTQa0RU1wIVWLHsTh+PxpXKdvu1jqA+1uVPMokJiaa9et/4PPPP8PHx8yMGbPIzMzg6acn0Lx5S+67L5RDhw7Srt19xMXFcfjwQR599HF2795Jr1592Lnzdxo1asxbb83DarXy6KMjOX36lGPba9Z8j9Fo5P3359G4cVMWLFjM4cMH+fXXXwBYvforRo16lJ49e7Nly8/88ccxkRRqEKdNCrKcd/OanJ2GZBAjQwTnUbduEEajO/v37yU3N4cffvgOgJycHC5evECXLt04cGAvKpVE377hbNnyMzabjSNHDjNlynT0ej0nThxn9eqvuHTpIjdu3CA7O+uWbRsBOHToAG+88TYAbdu2JzCwNgBdutzP++/PZ8+enXTt2p0HH+xVDaUgFMdpk4KSf59CdppoOhKqzP2tSnc1X5n0ej0Asmzn9dffpEmTpgCkpCTj4eFJeno6K1d+iVqt4b77OhAdfYnvv19PcHAwer2eNWtWsnXrrwwaNJQRIzpy8eJ5lJtPMszfNuSNfLn1piq1Wg1Ajx69admyNTt2bOebb75m9+4dTJ06o6o+vlACp+1ott+8o1nJSRdJQXBK7dt3YP36NQAkJSXx2GOjiY+/Rq1atdDr9ezYsY3WrdvSvn0Hli//D127dgdg3749DBo0jL59wwGJs2fPFHlHbWhoR37+OQqAkyf/4MqVWABmzpzGiRN/MGTIcJ544mlH05NQMzhtUpCVm6OPsm4gubhXdziCUOUmTHiS3Nxcxo59mEmTnubZZ1+kdu06QF4Tj9HojqurK/fd14GkpES6du0GwMMPP8Lnn3/KhAljeP/9ebRs2brIoaZ///tTXLkSy6OPPswXXyx3NB+NHTueFSs+Z8KEMXz88UJeeGFy1X1ooUSSkl/v+wtLTs5wPF6ztN79+hAAT2d9jLZFHwyd/1YZoZWL2exOYmJ6dYdRSE2NC2pubGazO8eOHcffP6i6Qymgpk7bIOIqu5Jiu3btcoHjT6WS8PY2Fru+89YUZAW9ygp2m7hHQRAE4SbnTQqKghvZgLhHQRAEIZ/zJgVZJAVBEITblXtI6oEDB3j77bex2WyYTCbefvttateuzb59+3j++efx9/cHoHnz5sydO5e0tDReeeUVYmJi8PLyYuHChZjNZiwWC6+99hrHjx/HYDCwYMECGjZsWGEfsDiyouAikoIgCEIB5a4pTJkyhTlz5rBhwwYiIiJ46623ADh27BgTJkxgw4YNbNiwgblz5wKwcOFCQkNDiYqKYuTIkcyZMweAFStW4OLiQlRUFNOnTycyMrICPlbJZBnclLwbbiSDGH0kCIIA5UwKFouFSZMm0bRp3k0vTZo0IS4u79b9Y8eOsWPHDoYMGcLTTz/tWL5161YiIiIAGDhwINu2bcNqtbJ161YGDRoEQIcOHUhNTeXq1cqfSdEuK7jZ00FSi0dxCoIg3FSu5iOdTsfgwYOBvIfVfPTRR/Tu3RsAd3d3BgwYQO/evfn666+ZPHkyK1euJCEhAbPZnLdTjQaj0UhKSkqB5QBms5lr164RGBhY6njuNLyqOCq1hLt8A43JjK+fqczvr2xmc82svdTUuKDmxqZSqdBoal73XU2MCURc5XGn2FQqVZl+GyUmhaioKEcTUL7g4GCWL1+OxWIhMjISm83GU089BcDs2bMd640ePZr33nuP9PSix4+rVEV/kOKWF6c89ylYLHaMuusort41bnx7TR5zXxPjgpobm9nsjizLNW6Me0WOu//99984deokTzzx9F1vq6beD1CT4tq0aSOHDh3gtdfeAEqOTZblAr+Nku5TKDEphIeHEx4eXmh5ZmYmzzzzDCaTiSVLlqDVapFlmX/9619MnDjRMc9JXtAafH19SUpKwt/fH5vNRkZGBiaTCV9fXxITEwkKyru5IjExEV9f35LCumuyomC0XUfl3qTS9yUI97Ju3cLo1i2susMQKki5Rx9NmTKFoKAgZs+e7ZhhVKVS8csvvxAUFET//v1Zv349bdq0wcXFhbCwMNavX8/TTz/Npk2bCA0NRavVEhYWxoYNGwgNDWX//v3o9foyNR2Vl0bORS9lI3mYS15ZEO4RBw/u57///QyAK1diefDBXri5ubF9+28oisKCBYvw8vJmx47tLFu2BEWRCQyszZQp0zlx4jjffbeO+fMXAvDtt6uIiYmmceOmjivXESMieOih/uzdu4vs7BxmzJhF06bNuHDhHHPmzMJut9OmTVt2797JqlXrC8R24cI5Fi5cQFZWFqmpKYwa9ShDh45g+PCBfP75l3h5eZOWdoOxY//Gt99+z/79e/nPf5Zis9kICKjN1Kmv4elpYsSICJo3b8nZs6f55JN/s3r11xw4sI+0tDRMJhNz5szH29uHLVt+4T//WYrBYKBx46bY7XZee+0NTp78g8WL3yc3NwdPTxNTpkynXr26BWJ9/vmJeHh4cvHieWbPnktycnKhWDZt+p7U1BSeffZF9u3bzfTprxIV9SsajYZHHx3J4sVLOXz4ECtXfkFubi65ublERs6gbdv2hbZ//vw5/u///oObmxF/f39cXFwB+OijhezfvweVSkW3bmFMmDDxro+RciWFEydOsGXLFkJCQhgyZAgAvr6+LFu2jHnz5vH666/z8ccf4+Xlxfz58wGYNGkSkZGRDBgwAHd3dxYsWADA2LFjmTlzJgMGDECn0znWr2weys0nromkIFQh65kdWE9vq5Rta5s8gLbx/SWud+LEH3z99Te4uXkQEdGH5557if/8ZwVvvz2LzZt/pk+fh3j33bdZsuQ/BAQE8tVX/+X99+fzxhtzePfdvOHlHh4ebN78Ey+88DKXLl0ssH1PT0+WLfsva9asZMWKz5gz513eeusNnnzyabp06caqVV9it9sLxbVx4wbGj/877dp14MqVWB5//BFGjhxFjx69+d//NjN8+N/YuvVXund/kPT0dJYu/YjFi5fi4eHB+vXfsmTJh0RGvg5A585dmT17LrGxMURHX2Lp0s9QqVS8+eZMfv75R/r1G8Dixe/x73//F29vH2bMmIqbmxtWq5V33nmLefM+wN/fnz17djFv3hw+/nhpoXgbNgzh7bffJTU1lTlzZhWKZfToscyenRfP/v37MBgMnDlzCpOpFi4urphMtdiw4Vvmz1+IyWTi++838NVXK2jbtn2B7SclJbJkyWI+//wrPDw8efXVl3BxceXatTh2797JypVryMzMZt68t8jNzS0wU215lCspNG/enNOnTxf5WqNGjVi5cmWh5SaTiaVLCxesXq9n3rx55Qnjrpjyk4J75TdVCUJNEhzcED8/f2w2GU9PE6GhHQHw8/MnPT2NEyf+oFmzFgQE5NXYBw0axooVy9FoNISF9eC3336lQ4dO3Lhxg+bNWxZKCp06db25nxB+++1/pKXd4Nq1OLp0yZtQb8CAwXzzTeFzxPPPv8T+/btZseJzzp0763hGQ79+/Vm06D2GD/8bmzf/xJNPPsOJE8eJj7/Giy/m9WPIsh0Pjz9HETZv3hKAOnXq8vzzk9m4cT3R0Zf5449j1K5dh6NHD9GyZSvM5rzff3j4ALZt20pMzGWuXo0lMvJlx7YyMzOLLMf8fRQXS1BQfTIzM0hLS+Po0UMMH/4whw8fxGBwoWvXbqhUKt5++1127NhOdPRlDh06UKA/NX/7x44doWXL1nh5eQPQt284Bw7sw8fHjF6v58knx9O1azeefPKZu04I4MTPUzAhagpC1dM2vr9UV/OVSaMp+LO/tf8PQFHk2/5WHFf2ffv259//XkJ6ehp9+vQrcvs6na7Ae1UqNaWZd3PmzEg8PDzp2rUbvXr1ZcuWnwFo2rQ56elpnDz5BwkJCbRq1Ybt27fSunUb5s37AIDc3FyysrIc28o/OZ46dZI33niNUaMeoUePXqjVqpsxqYocnGK35zWXLV/+1c2/7aSmphQZ763PpSgulk6durBt2/8Aia5du/Hvfy8FJP7+96fIysriiSfG8dBD/WnTph0NG4bw7berC20/77kUf8aa/31pNBo+/XQ5x44d4vfff+fpp8fz4YefUq/e3U2+WHPHWFWyWlI6VpUBSe9W3aEIQo3SvHlLTpw45pgO+7vv1tK+/X0AtGzZiqSkJH76adPN5ymUzGg0UqdOHXbt2gHAL7/8WOSTDvft28vEic/QvfuDHD58EMCRjPr06ce7775N7959HTH+8ccxoqMvA7B8+b/55JNFhbZ5+PAB2rW7jyFDRlC/fjB79+5BlmVatmzDqVMnSEpKQlEUNm/+GUmSCAqqT1paGkeO5M2i/MMP3/HGG6+VWF7FxdKlSzdWrPic1q3b0qhREy5evEhMzGWaNGlKTEw0KpWKceMmcN99Hdi9e2eRz6Vo3botJ04cIzExAVmWHY81PXPmFM8/P/FmH8RL1K8f7IjhbjhtTaEW6WTralV3GIJQ43h5eTNlymtMn/4KVqsNf39/IiNnOl7v1asPe/bscjx7oTRee20Wc+fOZtmyT2jYsFGRzRwTJjzJU09NwGg0UrduEAEBgcTFXaVOnbo89FB//v3vpY7He3p7+xAZOZOZM6chy3bMZj9mzpxdaJu9evVl+vQpPPbYKNRqDQ0bhhAXd5VatWrx0kuvMHnys+h0egICAtDpPNDpdLz55jssWrQAi8WCq6sbM2bMuuNnu1Ms7drdR3JyEu3a3YckSTRu3BgPDxMAISGNCAlpzCOPjMBgMNC2bXuuXSv8/G4vL29eemkKL730LAaDC/XrNwCgceOmtGzZmjFjHkav19OoURM6d+5a6u+kOE77PIXzSyeBqTYNR71aSVGVX00ec18T44KaG5t4nkKezz9fRkTEUHx8fPjtt1/5+eco5sx5t9riunHjOmvWrGL8+CdRqVQsXPguderUZcSIUUWuX5PuU7hdRT9PwWlrCmoJLF4164cqCPcqPz9/Jk9+Fo1Gg7u7h2OUUHXJfxb1uHF/Q61W07hxUyIihlZrTDWF09YUcrKyCAj0IvV6TiVFVX41+aq3JsYFNTc2UVMoGxFX2Yknr1UQg6srGq22usMQBEGoUZw2KQhCVboHKuTCX1B5jjuRFAShkmk0OjIz00RiEKqUoihkZqah0ehKXvkWTtvRLAhVpVYtM6mpiWRkXK/uUBzybt6qeW3kIq6yu1NsGo2OWrXKdoOuSAqCUMnUag0+PgHVHUYBNbljXsRVNhUdm2g+EgRBEBxEUhAEQRAc7onmI5Wq8DwqVfHeyiTiKruaGpuIq2xEXGVXlthKWveeuHlNEARBqBii+UgQBEFwEElBEARBcBBJQRAEQXAQSUEQBEFwEElBEARBcBBJQRAEQXAQSUEQBEFwEElBEARBcBBJQRAEQXC4J6a5KKuNGzeyZMkSrFYrjz/+OGPGjKm2WD766COioqIACAsL49VXX2XatGkcOHAAFxcXAJ5//nn69OlT5bGNGzeO5ORkNJq8w2T27NlER0dXa9l98803fPHFF46/Y2NjGTx4MNnZ2dVWZhkZGYwaNYqlS5dSp04ddu7cydy5c8nNzSU8PJzJkycDcPLkSWbMmEFGRgahoaHMmjXLUbZVEdeqVatYsWIFkiTRsmVLZs2ahU6n46OPPuLbb7/Fw8MDgIcffrhSv9fb4yrueC+uHCvTrbGdP3+e999/3/FafHw8bdq04V//+leVlllR54hKPcYUJ3Pt2jWlR48eSmpqqpKZmalEREQoZ8+erZZYduzYofztb39TcnNzFYvFoowbN075+eeflYEDByrx8fHVElM+WZaV+++/X7FarY5lNansFEVRzpw5o/Tp00dJTk6utjI7fPiwMnDgQKVFixZKTEyMkp2drYSFhSnR0dGK1WpVJkyYoGzdulVRFEUZMGCAcujQIUVRFGXatGnKl19+WWVxXbhwQenTp4+Snp6uyLKsvPrqq8rnn3+uKIqiPPXUU8rBgwcrLZY7xaUoSpHf3Z3KsSpjy5eQkKD06tVLuXjxoqIoVVdmRZ0jNm7cWKnHmNM1H+3cuZPOnTtjMplwdXXloYce4scff6yWWMxmM5GRkeh0OrRaLQ0bNuTq1atcvXqV119/nYiICBYvXlwtD/e4cOECkiTx5JNPMmjQIL744osaVXYAb7zxBpMnT8ZgMFRbma1evZp//vOf+Pr6AnD06FGCgoKoW7cuGo2GiIgIfvzxR65cuUJOTg5t27YFYNiwYZVadrfHpdPpeOONNzAajUiSROPGjbl69SoAx48fZ9myZURERDB79mxyc3OrLK6srKwiv7viyrEy3R7brebPn8+oUaOoX78+UHVlVtQ54tKlS5V6jDldUkhISMBs/vNJRL6+vsTHx1dLLI0aNXJ8gZcuXWLTpk10796dzp078/bbb7N69Wr279/PmjVrqjy2tLQ0unTpwscff8zy5ctZuXIlV69erTFlt3PnTnJycggPDyc5ObnaymzOnDmEhoY6/i7u+Lp9udlsrtSyuz2u2rVr07VrVwBSUlL48ssv6dWrF5mZmTRr1oypU6eybt060tLS+OSTT6osruK+u+r4nd4eW75Lly6xd+9exo0bB1ClZVbUOUKSpEo9xpwuKShFTAorSdU7Je7Zs2eZMGECU6dOJTg4mI8//hhvb29cXFwYO3Ysv/32W5XH1K5dO+bPn4+rqyteXl6MGDGCxYsXF1qvuspu5cqVjB8/HoC6devWiDKD4o+vmnLcxcfH89hjjzF8+HA6deqEm5sby5YtIygoCI1Gw4QJE6q07Ir77mpKeQGsWrWKRx55BJ0u71nH1VFmt54j6tWrV+j1ijzGnC4p+Pn5kZSU5Pg7ISGhyOpiVTlw4ACPP/44//jHPxg6dCinT5/mp59+cryuKEqldkYWZ//+/ezatatAHLVr164RZWexWNi3bx89e/YEqDFlBsUfX7cvT0xMrPKyO3/+PKNHj2bo0KE899xzAFy9erVAraqqy664764m/U63bNlC//79HX9XdZndfo6o7GPM6ZJC165d2bVrFykpKWRnZ/Pzzz/zwAMPVEsscXFxPPfccyxYsIABAwYAeQfY22+/zY0bN7BaraxatapaRh6lp6czf/58cnNzycjIYN26dbz77rs1ouxOnz5N/fr1cXV1BWpOmQG0adOGixcvcvnyZex2O99//z0PPPAAtWvXRq/Xc+DAAQDWr19fpWWXkZHB3//+dyZNmsSECRMcyw0GA++++y4xMTEoisKXX35ZpWVX3HdXXDlWtZSUFHJycqhbt65jWVWWWVHniMo+xpxuSKqfnx+TJ09m3LhxWK1WRowYQevWrasllv/85z/k5ubyzjvvOJaNGjWKiRMnMnr0aGw2G3379mXgwIFVHluPHj04cuQIQ4YMQZZlHnnkEe67774aUXYxMTH4+/s7/m7atGmNKDMAvV7PO++8wwsvvEBubi5hYWH069cPgAULFjBjxgwyMzNp3ry5o426KqxZs4akpCQ+++wzPvvsMwB69uzJpEmTmD17Ns888wxWq5X27ds7muWqwp2+u+LKsSrFxsYWONYAvLy8qqzMijtHVOYxJp68JgiCIDg4XfORIAiCUDyRFARBEAQHkRQEQRAEB5EUBEEQBAeRFARBEAQHkRQEoQI8+eSTnDt3rkzveeqpp1i7dm0lRSQI5eN09ykIQmVYtmxZdYcgCBVCJAXBqf3666+O50MYDAamTp3K77//ztmzZ0lKSiI5OZmmTZsyZ84cjEYjX331FStXrkSr1aLX65k9ezYhISH07NmTRYsW0apVK8dzC1QqFT4+Prz++us0aNCA+Ph4IiMjSUhIIDAwkOTkZEcc58+fZ86cOVy/fh273c7YsWMZMWIEmZmZTJs2jcuXL6NSqWjRogWzZ89GpRKVfKGSlH+mb0H4a7t48aIycOBAJSUlRVGUvOcz3H///co777yjPPDAA0piYqJit9uVl19+WXnnnXcUm82mtGjRwjH3/7p165SVK1cqiqIoPXr0UI4ePars3LlT6d27t5KcnKwoiqJ8++23Snh4uCLLsvLss88qH3zwgaIoinLp0iWlbdu2yrfffqtYrValf//+yvHjxxVFUZS0tDQlPDxcOXTokLJu3TplwoQJiqIois1mU1577TXl0qVLVVlMgpMRNQXBae3YsYOEhAQef/xxxzJJkoiOjqZfv374+PgAMGLECN5++22mTp1Kv379GDVqFA8++CD3338/ERERBba5fft2+vfvj5eXF5A3p/2cOXOIjY1l586dTJ06FYCgoCA6deoE5E2JHB0dzfTp0x3bycnJ4cSJE3Tv3p0PPviAsWPH0rVrVx577DGCgoIqs1gEJyeSguC0ZFmmS5cuLFy40LEsLi6OVatWYbFYCqyX31yzYMECzpw5w86dO1m2bBlr1qxhyZIljnWVImaNURQFm81WaHrj/Jk17XY7Hh4ebNiwwfFaUlIS7u7u6PV6fvnlF/bs2cPu3bsZP348M2bMqJZ5gATnIBomBafVuXNnduzYwfnz5wH47bffGDRoELm5uWzZsoX09HRkWWb16tX06NGDlJQUwsLCMJlMPP7447z00kucPn26wDa7devGpk2bSElJAeDbb7/FZDIRFBRE9+7dWbVqFZA3/fKePXsAaNCgAXq93pEU4uLiGDhwIMePH+err75i2rRpdOvWjSlTptCtWzfOnj1bVUUkOCExIZ7g1KKioli6dKljTvzp06eza9cudu/ejd1uJzU1lQ4dOjBjxgwMBgMrV67kv//9LwaDAbVazeTJk+natWuBjuYvv/ySlStXIssyXl5ezJw5k0aNGpGSksK0adOIjo7G398fm83G0KFDGTZsGKdOnXJ0NNtsNsaNG8fo0aPJyspi+vTpnD59GhcXFwIDA5kzZw6enp7VXXTCPUokBUG4zYcffkhqaiozZ86s7lAEocqJ5iNBEATBQdQUBEEQBAdRUxAEQRAcRFIQBEEQHERSEARBEBxEUhAEQRAcRFIQBEEQHERSEARBEBz+H0nZIJBEFGUBAAAAAElFTkSuQmCC\n" + }, + "metadata": {} + } + ], + "source": [ + "cfg = QlearningConfig()\n", + "env = gym.make(\"CliffWalking-v0\") # 0 up, 1 right, 2 down, 3 left\n", + "env = CliffWalkingWapper(env)\n", + "action_dim = env.action_space.n\n", + "agent = QLearning(action_dim,cfg)\n", + "rewards,ma_rewards = train(cfg,env,agent)\n", + "plot_rewards(rewards,ma_rewards,tag=\"train\",algo = \"On-Policy First-Visit MC Control\",save=False)" + ] + } + ] +} \ No newline at end of file diff --git a/codes/QLearning/main.py b/codes/QLearning/main.py index a6f7dac..0892bee 100644 --- a/codes/QLearning/main.py +++ b/codes/QLearning/main.py @@ -5,7 +5,7 @@ Author: John Email: johnjim0816@gmail.com Date: 2020-09-11 23:03:00 LastEditor: John -LastEditTime: 2021-03-31 18:21:00 +LastEditTime: 2021-03-31 18:14:59 Discription: Environment: ''' diff --git a/codes/README.md b/codes/README.md index d3dc6ef..38095de 100644 --- a/codes/README.md +++ b/codes/README.md @@ -32,14 +32,14 @@ python 3.7、pytorch 1.6.0-1.7.1、gym 0.17.0-0.18.0 | [Sarsa](./Sarsa) | | [Racetrack](./envs/racetrack_env.md) | | | [DQN](./DQN) | [DQN Paper](https://www.cs.toronto.edu/~vmnih/docs/dqn.pdf) | [CartPole-v0](./envs/gym_info.md) | | | [DQN-cnn](./DQN_cnn) | [DQN Paper](https://www.cs.toronto.edu/~vmnih/docs/dqn.pdf) | [CartPole-v0](./envs/gym_info.md) | 与DQN相比使用了CNN而不是全链接网络 | -| [DoubleDQN](./DoubleDQN) | | [CartPole-v0](./envs/gym_info.md) | 效果不好,待改进 | -| Hierarchical DQN | [H-DQN Paper](https://arxiv.org/abs/1604.06057) | | | +| [DoubleDQN](./DoubleDQN) | | [CartPole-v0](./envs/gym_info.md) | | +| [Hierarchical DQN](HierarchicalDQN) | [H-DQN Paper](https://arxiv.org/abs/1604.06057) | [CartPole-v0](./envs/gym_info.md) | | | [PolicyGradient](./PolicyGradient) | | [CartPole-v0](./envs/gym_info.md) | | | A2C | | [CartPole-v0](./envs/gym_info.md) | | | A3C | | | | | SAC | | | | | [PPO](./PPO) | [PPO paper](https://arxiv.org/abs/1707.06347) | [CartPole-v0](./envs/gym_info.md) | | -| DDPG | [DDPG Paper](https://arxiv.org/abs/1509.02971) | [Pendulum-v0](./envs/gym_info.md) | | +| [DDPG](./DDPG) | [DDPG Paper](https://arxiv.org/abs/1509.02971) | [Pendulum-v0](./envs/gym_info.md) | | | TD3 | [TD3 Paper](https://arxiv.org/abs/1802.09477) | | | | GAIL | | | | diff --git a/codes/README_en.md b/codes/README_en.md index 31c3d1e..2d5bbee 100644 --- a/codes/README_en.md +++ b/codes/README_en.md @@ -15,8 +15,6 @@ The code structure mainly contains several scripts as following: * ```agent.py``` core algorithms, include a python Class with functions(choose action, update) * ```main.py``` main function - - Note that ```model.py```,```memory.py```,```plot.py``` shall be utilized in different algorithms,thus they are put into ```common``` folder。 ## Runnig Environment @@ -28,23 +26,23 @@ run ```main.py``` or ```main.ipynb``` ## Schedule -| Name | Related materials | Used Envs | Notes | -| :----------------------------------------------------------: | :---------------------------------------------------------: | ------------------------------------------------------------ | :----------------------------------------------------------: | -| [On-Policy First-Visit MC](./MonteCarlo) | | [Racetrack](./envs/racetrack_env.md) | | -| [Q-Learning](./QLearning) | | [CliffWalking-v0](./envs/gym_info.md) | | -| [Sarsa](./Sarsa) | | [Racetrack](./envs/racetrack_env.md) | | -| [DQN](./DQN) | [DQN-paper](https://www.cs.toronto.edu/~vmnih/docs/dqn.pdf) | [CartPole-v0](./envs/gym_info.md) | | -| [DQN-cnn](./DQN_cnn) | [DQN-paper](https://www.cs.toronto.edu/~vmnih/docs/dqn.pdf) | [CartPole-v0](./envs/gym_info.md) | | -| [DoubleDQN](./DoubleDQN) | | [CartPole-v0](./envs/gym_info.md) | not well | -| Hierarchical DQN | [Hierarchical DQN](https://arxiv.org/abs/1604.06057) | | | -| [PolicyGradient](./PolicyGradient) | | [CartPole-v0](./envs/gym_info.md) | | -| A2C | | [CartPole-v0](./envs/gym_info.md) | | -| A3C | | | | -| SAC | | | | -| [PPO](./PPO) | [PPO paper](https://arxiv.org/abs/1707.06347) | [CartPole-v0](./envs/gym_info.md) | | -| DDPG | [DDPG Paper](https://arxiv.org/abs/1509.02971) | [Pendulum-v0](./envs/gym_info.md) | | -| TD3 | [Twin Dueling DDPG Paper](https://arxiv.org/abs/1802.09477) | | | -| GAIL | | | | +| Name | Related materials | Used Envs | Notes | +| :--------------------------------------: | :---------------------------------------------------------: | ------------------------------------- | :------: | +| [On-Policy First-Visit MC](./MonteCarlo) | | [Racetrack](./envs/racetrack_env.md) | | +| [Q-Learning](./QLearning) | | [CliffWalking-v0](./envs/gym_info.md) | | +| [Sarsa](./Sarsa) | | [Racetrack](./envs/racetrack_env.md) | | +| [DQN](./DQN) | [DQN-paper](https://www.cs.toronto.edu/~vmnih/docs/dqn.pdf) | [CartPole-v0](./envs/gym_info.md) | | +| [DQN-cnn](./DQN_cnn) | [DQN-paper](https://www.cs.toronto.edu/~vmnih/docs/dqn.pdf) | [CartPole-v0](./envs/gym_info.md) | | +| [DoubleDQN](./DoubleDQN) | | [CartPole-v0](./envs/gym_info.md) | not well | +| [Hierarchical DQN](HierarchicalDQN) | [Hierarchical DQN](https://arxiv.org/abs/1604.06057) | [CartPole-v0](./envs/gym_info.md) | | +| [PolicyGradient](./PolicyGradient) | | [CartPole-v0](./envs/gym_info.md) | | +| A2C | | [CartPole-v0](./envs/gym_info.md) | | +| A3C | | | | +| SAC | | | | +| [PPO](./PPO) | [PPO paper](https://arxiv.org/abs/1707.06347) | [CartPole-v0](./envs/gym_info.md) | | +| [DDPG](./DDPG) | [DDPG Paper](https://arxiv.org/abs/1509.02971) | [Pendulum-v0](./envs/gym_info.md) | | +| TD3 | [Twin Dueling DDPG Paper](https://arxiv.org/abs/1802.09477) | | | +| GAIL | | | | ## Refs diff --git a/codes/common/plot.py b/codes/common/plot.py index b8684d0..ed6934d 100644 --- a/codes/common/plot.py +++ b/codes/common/plot.py @@ -5,28 +5,30 @@ Author: John Email: johnjim0816@gmail.com Date: 2020-10-07 20:57:11 LastEditor: John -LastEditTime: 2021-03-31 14:05:52 +LastEditTime: 2021-03-31 18:47:28 Discription: Environment: ''' import matplotlib.pyplot as plt import seaborn as sns -def plot_rewards(rewards,ma_rewards,tag="train",algo = "DQN",path='./'): +def plot_rewards(rewards,ma_rewards,tag="train",algo = "DQN",save=True,path='./'): sns.set() plt.title("average learning curve of {}".format(algo)) plt.xlabel('epsiodes') plt.plot(rewards,label='rewards') plt.plot(ma_rewards,label='moving average rewards') plt.legend() - plt.savefig(path+"rewards_curve_{}".format(tag)) + if save: + plt.savefig(path+"rewards_curve_{}".format(tag)) plt.show() -def plot_losses(losses,algo = "DQN",path='./'): +def plot_losses(losses,algo = "DQN",save=True,path='./'): sns.set() plt.title("loss curve of {}".format(algo)) plt.xlabel('epsiodes') plt.plot(losses,label='rewards') plt.legend() - plt.savefig(path+"losses_curve") + if save: + plt.savefig(path+"losses_curve") plt.show() diff --git a/codes/common/utils.py b/codes/common/utils.py index 2a44ec5..1b78e16 100644 --- a/codes/common/utils.py +++ b/codes/common/utils.py @@ -5,7 +5,7 @@ Author: John Email: johnjim0816@gmail.com Date: 2021-03-12 16:02:24 LastEditor: John -LastEditTime: 2021-03-12 16:10:28 +LastEditTime: 2021-04-03 21:42:13 Discription: Environment: ''' @@ -18,4 +18,17 @@ def save_results(rewards,ma_rewards,tag='train',path='./results'): ''' np.save(path+'rewards_'+tag+'.npy', rewards) np.save(path+'ma_rewards_'+tag+'.npy', ma_rewards) - print('results saved!') \ No newline at end of file + print('results saved!') + +def make_dir(*paths): + for path in paths: + if not os.path.exists(path): # check if exists + os.mkdir(path) +def del_empty_dir(*paths): + '''del_empty_dir delete empty folders unders "paths" + ''' + for path in paths: + dirs = os.listdir(path) + for dir in dirs: + if not os.listdir(os.path.join(path, dir)): + os.removedirs(os.path.join(path, dir)) \ No newline at end of file diff --git a/codes/test.py b/codes/test.py new file mode 100644 index 0000000..5e534d1 --- /dev/null +++ b/codes/test.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python +# coding=utf-8 +''' +Author: John +Email: johnjim0816@gmail.com +Date: 2021-03-25 23:25:15 +LastEditor: John +LastEditTime: 2021-03-26 16:46:52 +Discription: +Environment: +''' +from collections import defaultdict +import numpy as np +action_dim = 2 +Q_table = defaultdict(lambda: np.zeros(action_dim)) +Q_table[str(0)] = 1 +print(Q_table[str(0)]) +Q_table[str(21)] = 3 +print(Q_table[str(21)]) \ No newline at end of file