diff --git a/codes/A2C/a2c.py b/codes/A2C/a2c.py index bd26785..ba0ed7c 100644 --- a/codes/A2C/a2c.py +++ b/codes/A2C/a2c.py @@ -5,10 +5,11 @@ Author: JiangJi Email: johnjim0816@gmail.com Date: 2021-05-03 22:16:08 LastEditor: JiangJi -LastEditTime: 2021-05-03 22:23:48 +LastEditTime: 2022-07-20 23:54:40 Discription: Environment: ''' +import torch import torch.optim as optim import torch.nn as nn import torch.nn.functional as F @@ -42,7 +43,7 @@ class A2C: ''' def __init__(self,n_states,n_actions,cfg) -> None: self.gamma = cfg.gamma - self.device = cfg.device + self.device = torch.device(cfg.device) self.model = ActorCritic(n_states, n_actions, cfg.hidden_size).to(self.device) self.optimizer = optim.Adam(self.model.parameters()) diff --git a/codes/A2C/outputs/CartPole-v0/20220713-221850/results/params.json b/codes/A2C/outputs/CartPole-v0/20220713-221850/results/params.json new file mode 100644 index 0000000..2773964 --- /dev/null +++ b/codes/A2C/outputs/CartPole-v0/20220713-221850/results/params.json @@ -0,0 +1,14 @@ +{ + "algo_name": "A2C", + "env_name": "CartPole-v0", + "n_envs": 8, + "max_steps": 20000, + "n_steps": 5, + "gamma": 0.99, + "lr": 0.001, + "hidden_dim": 256, + "deivce": "cpu", + "result_path": "C:\\Users\\24438\\Desktop\\rl-tutorials/outputs/CartPole-v0/20220713-221850/results/", + "model_path": "C:\\Users\\24438\\Desktop\\rl-tutorials/outputs/CartPole-v0/20220713-221850/models/", + "save_fig": true +} \ No newline at end of file diff --git a/codes/A2C/outputs/CartPole-v0/20220713-221850/results/params.txt b/codes/A2C/outputs/CartPole-v0/20220713-221850/results/params.txt deleted file mode 100644 index 2daca8c..0000000 --- a/codes/A2C/outputs/CartPole-v0/20220713-221850/results/params.txt +++ /dev/null @@ -1,14 +0,0 @@ ------------------- start ------------------ -algo_name : A2C -env_name : CartPole-v0 -n_envs : 8 -max_steps : 30000 -n_steps : 5 -gamma : 0.99 -lr : 0.001 -hidden_dim : 256 -result_path : c:\Users\24438\Desktop\rl-tutorials\codes\A2C/outputs/CartPole-v0/20220713-221850/results/ -model_path : c:\Users\24438\Desktop\rl-tutorials\codes\A2C/outputs/CartPole-v0/20220713-221850/models/ -save_fig : True -device : cuda -------------------- end ------------------- \ No newline at end of file diff --git a/codes/A2C/task0.ipynb b/codes/A2C/task0.ipynb deleted file mode 100644 index aa9b772..0000000 --- a/codes/A2C/task0.ipynb +++ /dev/null @@ -1,265 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "import sys\n", - "from pathlib import Path\n", - "curr_path = str(Path().absolute()) # 当前路径\n", - "parent_path = str(Path().absolute().parent) # 父路径\n", - "sys.path.append(parent_path) # 添加路径到系统路径\n", - "import math\n", - "import random\n", - "\n", - "import gym\n", - "import numpy as np\n", - "\n", - "import torch\n", - "import torch.nn as nn\n", - "import torch.optim as optim\n", - "import torch.nn.functional as F\n", - "from torch.distributions import Categorical\n", - "\n", - "from IPython.display import clear_output\n", - "import matplotlib.pyplot as plt\n", - "%matplotlib inline" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "use_cuda = torch.cuda.is_available()\n", - "device = torch.device(\"cuda\" if use_cuda else \"cpu\")" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - "from common.multiprocessing_env import SubprocVecEnv\n", - "\n", - "num_envs = 16\n", - "env_name = \"CartPole-v0\"\n", - "\n", - "def make_env():\n", - " def _thunk():\n", - " env = gym.make(env_name)\n", - " return env\n", - "\n", - " return _thunk\n", - "\n", - "envs = [make_env() for i in range(num_envs)]\n", - "envs = SubprocVecEnv(envs)\n", - "\n", - "env = gym.make(env_name)" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [], - "source": [ - "class ActorCritic(nn.Module):\n", - " def __init__(self, num_inputs, num_outputs, hidden_size, std=0.0):\n", - " super(ActorCritic, self).__init__()\n", - " \n", - " self.critic = nn.Sequential(\n", - " nn.Linear(num_inputs, hidden_size),\n", - " nn.ReLU(),\n", - " nn.Linear(hidden_size, 1)\n", - " )\n", - " \n", - " self.actor = nn.Sequential(\n", - " nn.Linear(num_inputs, hidden_size),\n", - " nn.ReLU(),\n", - " nn.Linear(hidden_size, num_outputs),\n", - " nn.Softmax(dim=1),\n", - " )\n", - " \n", - " def forward(self, x):\n", - " value = self.critic(x)\n", - " probs = self.actor(x)\n", - " dist = Categorical(probs)\n", - " return dist, value" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [], - "source": [ - "def plot(frame_idx, rewards):\n", - " clear_output(True)\n", - " plt.figure(figsize=(20,5))\n", - " plt.subplot(131)\n", - " plt.title('frame %s. reward: %s' % (frame_idx, rewards[-1]))\n", - " plt.plot(rewards)\n", - " plt.show()\n", - " \n", - "def test_env(vis=False):\n", - " state = env.reset()\n", - " if vis: env.render()\n", - " done = False\n", - " total_reward = 0\n", - " while not done:\n", - " state = torch.FloatTensor(state).unsqueeze(0).to(device)\n", - " dist, _ = model(state)\n", - " next_state, reward, done, _ = env.step(dist.sample().cpu().numpy()[0])\n", - " state = next_state\n", - " if vis: env.render()\n", - " total_reward += reward\n", - " return total_reward" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [], - "source": [ - "def compute_returns(next_value, rewards, masks, gamma=0.99):\n", - " R = next_value\n", - " returns = []\n", - " for step in reversed(range(len(rewards))):\n", - " R = rewards[step] + gamma * R * masks[step]\n", - " returns.insert(0, R)\n", - " return returns" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [], - "source": [ - "num_inputs = envs.observation_space.shape[0]\n", - "num_outputs = envs.action_space.n\n", - "\n", - "#Hyper params:\n", - "hidden_size = 256\n", - "lr = 3e-4\n", - "num_steps = 5\n", - "\n", - "model = ActorCritic(num_inputs, num_outputs, hidden_size).to(device)\n", - "optimizer = optim.Adam(model.parameters())" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [], - "source": [ - "max_frames = 20000\n", - "frame_idx = 0\n", - "test_rewards = []" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAE/CAYAAABfF5iGAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAA9TklEQVR4nO3dd3hc5ZX48e9RL1aXLEuWZLkK2+CGbYxtwJQQQonpmJBAgCwhZdN2QyDJL8nuJpuQJdmEbEJCAoEEQjeYEGpoprnbkgvutprVLFkzsqRRm/f3x1yZQVaddmek83kePZq59ehq5sw7732LGGNQSikVeaLsDkAppZRvNIErpVSE0gSulFIRShO4UkpFKE3gSikVoTSBK6VUhNIEPkqISImIbBORFhH5mt3xqOARkc+LyLt2x6Hspwl89LgDeNMYk2KMudfuYLyJyAwRWSMiDSLSJCKviEhJn22+KSK1IuIUkQdFJN5rXbGIvCkibSKyW0QuCNS+Y4GI3C8ie0TELSKf72f9FBF5wfrwPyoiP/da91UR2SQiHSLy0BDnERH5sYhUi4hDRN4SkdmB/4tUL03go8ckYOdAK0UkOoSx9JUOPA+UALnABmBN70oR+SRwJ3A+nr9jCvAfXvs/BmwFsoDvAU+LSI6/+46EiMSMdJ9ACNB5S4EvA1v6OX4c8BrwBjABKAAe8drkCPBj4MFhnOca4BbgLCAT+AD4qz+BqyEYY/Qnwn/wvPl6ABdwHJgBPATcB7wItAIXAJfgSWZOoBL4kdcxigED3GytOwbcDiwCyoBm4P/6nPcW4ENr21eAScOMN9M6V5b1/G/Af3utPx+otR7PADqAFK/17wC3+7vvMOI8DHzH+vs7gBhgCfC+dT1KgRXWtucC2732fQ3Y2Oe8l1uP7wQOAC3ALuAKr+0+D7wH/C/QiCd5ZuH5AHTi+fD7L+BdH14n7wKf77PsNuCdYez7Y+ChIbb5DvCk1/PZgMvu98do/tES+ChgjDkPT4L4qjFmnDFmr7XqM8BPgBQ8b95W4EY8JeJLgC+JyOV9DncGMB24DvgVnlLrBXjejNeKyDkAIrIS+C5wJZBjnf+xYYZ8Np4k22g9n40nGfYqBXJFJMtad9AY09Jn/ewA7Dsc1+O5Vul4vj38A08yywT+HXjGKtGvA6aLSLaIxAJzgHwRSRGRRGAhnmsEnuR9FpCG59vCIyKS53XOM4CD1vl+AvwWz4dzHp4PzVu8A7SqP+4cwd/kbQlwWEResqpP3hKR03w81uPAVKvKLBa4CXjZx2OpYdAEPrqtMca8Z4xxG2Ncxpi3jDHbredleBLuOX32+S9r21fxJPzHjDH1xphqPAlovrXd7cBPjTEfGmO6gf8G5onIpMECEpECPAnpW16LxwEOr+e9j1P6Wde7PiUA+w7HvcaYSmNMO/BZ4EVjzIvWNXwN2ARcbK3fiOfD6XQ8HxTvAcvwJMl9vR9YxpinjDFHrGM8AewDFnud84gx5jfWde0ErgJ+YIxpNcbsAB72DtAYc6kx5mcj+Ju8FQCrgHuBfDwfUGusqpWRqsFTUNgDtOOpUvmmj3GpYdAEPrpVej8RkTOsG3oNIuLAk4Sz++xT5/W4vZ/n46zHk4Bfi0iziDQDTYAAEwcKxiqpvgr8zhjjXVo/DqR6Pe993NLPut71vaVqf/YdDu9rOAm4pvdvtv7u5XhKxgBvAyvwJPG3gbfwfECeYz0HQERutFoM9R7jVD7+f/A+Zw6eqhvvZeUjiH8o7XiqY14yxnQC9+Cpspnpw7F+gKfKrRBIwPPt4g0RSQpUsOrjNIGPbn2HmvwbnrrUQmNMGvB7PEnXF5XAF40x6V4/icaY9/vbWEQy8CTv540xP+mzeicw1+v5XKDOKrHuBKaISEqf9TsDsO9weF/DSuCvff7mZK/Sb98E/jZ9Erj1DeWPwFfx3ANIB3bw8f+D9zkbgG48SbFX0QjiH0oZJ79OfDUPeMIYU2WM6TbGPARkALMCdHzVhybwsSUFaDLGuERkMZ46cl/9Hrirt5mYiKSJyDX9bSgiqXhucr5njOmvrvYvwK0iMktE0oHv47kJi1Wfvw34oYgkiMgVeOqXnwnAviP1CHCZiHxSRKKtY66wqoXAc3OzBE91yAZjzE48pfYzgLXWNsl4EmaDdW1uxlMC75cxpgdYDfxIRJJEZBaeuuVhE5E4EUnA8yERa8Xd+95/BFgiIhdYLZW+ARzFc3MaEYmx9o0Gev/mgVrGbMTzDSVXRKJE5HNALLB/JPGqEbD7Lqr+BOYHz9f1L3g9fwj4cZ9trsbz9bsFeAH4P+ARa10xnsQS47V9FVYrC+v5I8D3vZ5/DtjOR61aHhwgtpusY7fiqdbo/Sny2uZbeKprnMCfgXivdcXW39eOp371gj7H92lf4AZg5yDX9HA/5zoDT2m6CU8S/kefv+MDPO3xe58/DXzY5xg/sfY/CvzSOt4XrHWfp08LEzzVKC8wQCsU4CXgu0O8NkyfH+//65V4kqzT2na217of9bPvj6x1Rd7/RzzVJr/FUxfuxNNs8SK73xuj+UesC6+UUirCaBWKUkpFKE3gSikVoTSBK6VUhNIErpRSEUoTuFJKRShbRljrKzs72xQXF9sdhlJKhZ3NmzcfNcb0O4JmWCTw4uJiNm3aZHcYSikVdkRkwKETtApFKaUilCZwpZSKUJrAlVIqQmkCV0qpCKUJXCmlIpQmcKWUilCawJVSKkJpAldKqQilCVwppSKUJnA1pvS4DW/uqcft1olMVOTTBK7GlLf21HPznzfy97IjdoeilN80gasx5UDDcQAeePcQOp2ginSawNWYUt7YBkBZlYPN5cdsjkYp/4TFaITqI8YY2rt6aHF142zvwunqxunqosXVTYurC2e753eLq5vJ2clctaCAtKRYu8OOGBVNbczIHUeds4MH3j3EwuJMu0NSymeawG2yvcrBb9/cj6O9i5aOjyfm7iFusMVGC8nxMTS3dfHzV3Zz2Zx8PnfmJOYUpIcm+Ah2uLGVeYUZnHdKIvevPUBlUxuFmUl2h6WUTzSB2+SZLVX888M65helk5uSwLScGFITY0lJiCElIZbUBM/j3mWpCbGkWs/jY6IQEXYdcfLI+nKe21rNU5urmFOQxmfPmMRlc/NJjIu2+08MO109bo40u1g5N4kblhTxp3cO8vD7h/n+pbPsDk0NwRhDRVMbk7KS7Q4lrGgCt0mtw8WkrCSeun2pz8eYlZ/Kf19xGnd96hSe3VrNI+vKueOZMn78j11cfXohn11SxJSccQGMOrJVH2unx22YlJVEXloiF5+WxxMbK/nGJ2YwLl7fCuHs6c1VfPvpMi6aPYH/XDmb8akJdocUFvQmpk1qnS7y0hIDcqyUhFhuPLOYV75xNk/ctoRzSsbz13WHOe8Xb3PDn9bx8o4aunvcATlXJCtv8tzA7C3F3bJ8Mi0d3Ty5sdLOsNQwPLWpiszkON7cU8/5v3ybxzdUaCsiNIHbps7pIjfApQgR4YwpWfzm+vm8f+f5fPuTJRw+2sbtj2xh2d1v8Kt/7qXO6QroOSNJeWMrAJOyPHXe8wrTOX1SBg+9f5ge7dgTtiqb2thwuIlbl0/m5W+czay8VO5cvZ3P/HE9h4+22h2erTSB26DHbahv6SAvLXhfA3NS4vnKudNYe8e5/OnGhczMS+XXr+9j6c/e4EuPbOb9/UeDdu5wVd7YRkJsFONT4k8su3X5ZCqa2vjnh3U2RqYG83ypp9PVp+fmMzk7mcf+ZQk/vfI0dhxx8MlfreX3bx8Ys98wNYHb4OjxDnrchtwgJvBe0VHCBbNyeejmxbz97+fyhbMms+5gI5/503peGGO9Ecsb25iUmYyInFh24axcJqYn8sC7h2yMTA3EGMPqLVUsnpx5orVQVJRw/eIi/vmtc1hRksPPXtrNyt++x45qh83Rhp4mcBvUOjzVGBNCfCOmKCuJuz41kw/uOp/xKfG8snNslTormlopyvp4k8GY6Cg+v7SYDYeaxmQCCHc7qp0caGjlivkTT1qXm5rAHz63kPtuWEB9Swcrf/seP33pQ1xdPTZEag9N4DaosRJ4MKtQBpMQG82yadl8cODomLkR5HYbqwR+cpvv6xYXkhwXzYNaCg87z26tJi46iotPzRtwm0+dlsc/v3kOVy8o4A9vH+SiX63lgwONIYxyaMGq4hkygYvIgyJSLyI7+iz/VxHZLSI7ReTnXsvvEpH9IrJHRD4ZjKAjXe+NxEDfxByJpVOzOHq8kz11LbbFEEr1LR10dLtP3MD0lpoQyzULC/l72RHqx/BN3nDT3ePm+dIjnHfK+CF7G6clxXL31XP42xfOwG3g+j+u485nynC0d4Uo2oEdOtrKOf/zFhsONQX82MMpgT8EXOS9QETOBVYCc40xs4F7rOWzgFXAbGuf34mI9ijpo8bhIjZayEqOsy2GpdOyAXhvf3iVVILloxYo/XcEuXlZMd1uw18+KA9lWGoQ7+4/ytHjHVyx4OTqk4EsnZbNK984my+ePYUnN1XyiV++zcs7aoMY5eCMMdy1ugynq6vfwoO/hkzgxpi1QN+Pji8BPzPGdFjb1FvLVwKPG2M6jDGHgP3A4gDGOyrUOV2MT0kgKkqG3jhIJqYnUpyVNGZao3zUBrz/N9GkrGQumJnLo+vLx1Qdajh7bms1aYmxrCjJGdF+iXHR3HXxTNZ8ZTnZ4+K5/ZHN3P7XzbZ8u3pyUyXrDjZx16dmBuUbt6914DOAs0RkvYi8LSKLrOUTAe9eEVXWMuWl1uFigk31396WTstm/aGmMdEEq6KxjegoIT994M5Tty6fzLG2Lp7dWh3CyFR/Wju6eWVnHZfMySM+xrcv8acVpLHmq8v4zkWn8Oaeei7/7Xs0t3UGONKB1be4+Mk/PmRxcSarFhUG5Ry+JvAYIBNYAnwbeFK822YNg4jcJiKbRGRTQ0ODj2FEplqnK+QtUPqzbGo2xzu6KRsDrS8ON7YyMT2R2OiBX/JnTM5kdn4qD+pY4bZ7ZWct7V09XNlP65ORiI2O4ksrpvL4bUuob+ngu89uD9n/9j/+vgtXl5ufXnVa0L5t+5rAq4DVxmMD4AaygWrA+6OmwFp2EmPM/caYhcaYhTk5I/uKFMmMMWFTAj9zahbAmKhG8QyENHgdpIhw6/LJ7Ks/ztp9o/+ahLNnt1ZTkJHI6ZMyAnK8+UUZ/NuFJby4vZanNlUF5JiD+eeuOv5RVsO/njeNqUEcj8jXBP4ccC6AiMwA4oCjwPPAKhGJF5HJwHRgQwDiHDWcrm7au3rCogSemRzHrLzUMXEjs7xx6AQOcOmcfHJS4rVJoY3qnS7e23+UK+ZPZIRf7Af1xbOncOaULH74/E4OWjMzBUOLq4v/t2YHJbkpfPGcqUE7DwyvGeFjwAdAiYhUicitwIPAFKtp4ePATVZpfCfwJLALeBn4ijFG7wh5OdGJJwxK4ADLpmWxueLYqL5x19zWiaO9i0mZQw9FGhcTxY1LJvH23gb214+NJpbh5vnSI7gNXO5n9UlfUVHCL6+bS1xMFF9/fBud3cG593PPK3uodbr46VWnERcT3K42w2mFcr0xJs8YE2uMKTDGPGCM6TTGfNYYc6oxZoEx5g2v7X9ijJlqjCkxxrwU1OgjUK0zvBL40qnZdHa72XR49E4v1juNWt9emAO5Yckk4mOieODdw0GMSg3k2a3VzC1IC0rVQ15aIndfdRrbqx388rW9AT/+5vJj/GVdOTedWcyCosBU/wxGe2KGWK2jHQh9N/qBLJ6cSUyU8N6B0VvnO1QTwr4yk+O4csFEVm+poqk1dK0WFOyta2HnEWfAS9/eLjo1j+sXF/KHtQcCev+ns9vNXavLyEtN4N8/WRKw4w5GE3iI1To6ABifGj/ElqGRHB/DvML0UX0js8LqxFM0gqnTbl42mY5uN49tqAhWWKofz26tJjpKuGxuflDP8/8uncXk7GS++eQ2jgXoQ/r3bx9gb91x/uvyU0M2QYgm8BCrdbrISo7zuW1rMCydls32akdYdDsOhvLGNsanxJMUN/w31YzcFM6ans3D7x8OWl2p+ji327BmazVnT88me1xwCzhJcTHcu2o+Ta2d3Lm6zO+mhfvrW/i/N/Zz6Zw8zp+ZG6Aoh6YJPMRqHe22joHSn2VTs3AbWH9wdLZGGW4LlL5uXT6Z+pYO/rF9bA27a5cNh5s44nAFtfrE26kT0/j2J0t4ZWcdj/sxK5Pbbbhr9XYS46L54WWzAxjh0DSBh1itM7gTOfhiflEGibHRvB9mI7gFSnlTK0XDaIHS1zkzcpg2fhwPaMeekHh2SzXJcdFcOGtCyM75heVTWD4tm//4+0721/vWtPCxjRVsPHyM710yk5yU0FaNagIPsTqnKyQTOYxEXEwUiyZn8t4orAd3dfVQ5+yg2IcSuIhw87JidlQ72TiKW+mEA1dXDy9ur+GiU/NIjAtd9WJUlPCLa+eSGBvN1x/fSkf3yJrT1jpc/OzF3SydmsU1pxcEKcqBaQIPIVdXD02tneSFWRUKeKpR9tUfH3XDqVY0jawJYV9Xzi8gPSmWB949GMiwVB9v7K6npaO734kbgi03NYG7r5rDziNOfvHqyJoW/vD5HXT2uPnvK04LaKej4dIEHkL1Tk8LlHArgYOnPTgw6qpReie9HWgY2aEkxkVzwxlFvLqrjgqrPbkKvNVbqslNjT8xvEOoXTh7AjecUcT9aw/yzr7hjc308o4aXtlZxzcumEFxtm+vL39pAg+hE514wrAEPis/lbTE2FFXjdJbAu9vJp7huvHMYqJFeOj9wwGKSnlrau3krT31rJw3kWgbh1j+/iWzmDZ+HP/2ZOmQ7f8d7V38YM1OZual8oWzJocowpNpAg+hGqsTT7jdxATP5MdnTsni/QONo+qGXXljG6kJMaQPMaPLYHJTE7h0Th5PbqqkxTU6m1ra6R/ba+h2Gy6fZ+/I04lx0fx61Tya27q44+nBmxbe/fJujh7v4O6rTht0hMtg0wQeQiemUgvDBA6ecVGqm9tPdD0fDcqb2piUlex3/eSty6dwvKObJ/xobqb699zWakpyU5iZl2J3KMzOT+OOi0r454d1PLq+/05cGw418bf1FdyybDJzCtJDG2AfmsBDqMbhIikumpQQ9dIaqd5p1kZTPXh548kz0fvitII0Fhdn8tD7h+lxj55vKHYrb2xlc/kxrlgQ2JEH/XHLssmcNT2b/3phF/v6zBnr6urhztVlFGQk8q0LZ9gU4Uc0gYdQndMzDni4vFD7mpKdzITUhFEzLkp3j5vqY+1+1X97u2X5ZKqOtfPaLvvmWBxtntt6BBH4dJC7zo9Eb9PCcfExfO3xbR9rWvi7N/dzsKGVn1xx2oh69gaLJvAQqnWEx0w8AxERlk7L4oMDjbhHQSnzSLOLbreh2McWKH19YlYuhZmJPKijFAaEMYbntlWzZHLWoFPd2WF8SgI/v3oOH9Y4+fnLewDYU9vCfW8f4Ir5EzlnRnhMQqMJPITCPYGDZ5q1ptZOdtdG/ljY5U3WIFYBmg08OkpYtaiIDYebqG8ZXe3l7bCtsplDR1ttafs9HOfPzOXGMyfxwLuHeHN3PXeuLmNcfAzfv2Sm3aGdoAk8RNxuQ31LR9iMAz6QpdOsadZGQTXK4caRDSM7HL0zpK/dG/nXZ6QcbV0BHfDsua3VxMdEcdFpoes6P1LfvXgmM3LHcdtfN7G1opkfXDaLrCAPtDUSmsBD5GhrB91uE/YJPC8tkSnZyaOiPXhFYytxMVHkpgTums/KS2V8Sjxv7qkP2DEjwdq9Day4503Ovect3thd5/fxunrc/L2shgtm5ZKa4HsTz2BLiI3m16vmIyKcPSPH9qaOfWkCD5ETU6mFeRUKeErhGw410dUT2cOolje2MSkzKaAzgosI58zI4Z29DXRH+PUZDrfbcO/r+7jpzxsYn5JAbmoCtzy0iR+/sMuvYXbX7m2gqbWTK8IsIfZnZl4qr3/rHO7/3Olh1wBBE3iIhNtcmINZNjWb1s4eSiub7Q7FL8OZid4XK0rG43R1sy3Cr89Qmts6ueXhjfzytb1cPm8iz35lKc9+eSk3nTmJP717iKvue//EUAUj9ezWajKSYjmnJDxuBg6lMDOJhNjwGcO/lybwEKkL4270fZ05NQsRInq2emMM5Y1tPg0jO5Tl07OJjhLe2jO8MTMi0fYqB5f+5l3e23+U/7r8VH557VyS4mJIiI3mP1aeyh8+dzoVTW1ccu87rNlWPaJjt7i6eG1XHZfNzbe1F+NooFcvRGocLmKiJKxugAwkPSmO2fmpEX0js6Glg/aunqCUwNMSY1lQlM5be0dfPbgxhsc3VHDV79/H7TY8+cUz+dySSSdVHXxy9gRe/PpZzMpP5euPb+PbT5XS1tk9rHO8tKOWjm53yCZuGM00gYdIrdPF+JR4WwfrGYllU7PZWtFMe+fIxkcOFyOdyHikVpSMZ0e1c1Q1J3R19XDH02XcuXo7Z0zO5IWvncX8QWZWn5ieyGP/soSvnTeNp7dUcelv3mXXEeeQ53luazXFWUnML0wPYPRjkybwEKl1hN9EDoM5c2oWnT1uNh5usjsUn5SfaEIYnGE+eztyjJbmhOWNrVzxu/d5anMVXzt/Og/dvJjM5Lgh94uJjuJbF5bw6BfO4Lirm8t/9x5/+eDwgANB1Tja+eBgI5fPD5+u85FsyAQuIg+KSL2I7Ohn3b+JiBGRbOu5iMi9IrJfRMpEZEEwgo5EtU5XWI5COJDFkzOJjZaI7VZf3thKlHhKicEwOz+VnJR43hoFzQlf21XHpb95lyPN7fz584v41idmjPib4tKp2bz09bNYNjWLH6zZyRf/upnmtpOHZF2z7QjGEHbN8SLVcErgDwEX9V0oIoXAhYD3kF2fAqZbP7cB9/kfYuQzxnhK4BFwA7NXUlwM8wszeD9Cb2SWN7aRn55IXExwvmSeaE6472jENifs7nHz85d38y9/2URxVjIv/Otyzj1lvM/HyxoXzwM3LeL7l8zkzT31XPzrd076Bvfc1mrmF6XbNgHCaDPkq9sYsxbo73v0/wJ3AN7flVYCfzEe64B0EckLSKQRrKWjm7bOnohogeJt6bQsdhxx9FuSCnflTW0BGwNlICtKcnC0d1Fa1RzU8wTD0eMd3PjgBn731gGuX1zEU7efSWEABv2KihK+cNYUnvnSUmJjorjuDx/wm9f30eM2fFjjZHdtC1fqzcuA8al4IiIrgWpjTGmfVRMB7wGTq6xl/R3jNhHZJCKbGhpGb3MsgLoIagPubdm0bIyBdQcjrxReEaBhZAdz1rQcooSIa064ubyJS+59h83lx/ifq+fw0ytPC3gb5zkF6bzwr8u5bG4+v3htLzf8aR1/eucQMVHCJXPCZ+TBSDfiBC4iScB3gR/4c2JjzP3GmIXGmIU5OZHRmN9X4TyV2mDmFqSTFBcdce3BHe1dHGvrCtgwsgNJS4plQVFGxCRwt9vw5/cOcd0f1pEQG83qLy/lmoWFQTtfSkIsv7puHv9z9RxKKx08s6WKFSU5w7o5qobHlwFtpwKTgVLrLnIBsEVEFgPVgPcrosBaNqbVWCXwvLTwGjJzKHExUSyenBlx7cErgjCI1UBWlORwz6t7aWjpICfFvjb+brfh6PEOahwuahzt1Dhc1DpcHHG4qLWe1zlddPUYLpg5nl9cO4+0xOCPQSIiXLOwkPlFGfz85d188ZypQT/nWDLiBG6M2Q6cuNMhIoeBhcaYoyLyPPBVEXkcOANwGGNqAhVspOqtQhmfGv6dePpaNjWbn+z50DMUboRUAfUOIxusJoTeVpSM555X97J2bwNXnV4Q9POtO9jIriNOap0ujjS3U+twnUjO3X3GcI+LiSIvLYEJqQksKs5kQloCp0xI4bI5+QEdH2Y4po0fx/03LgzpOceCIRO4iDwGrACyRaQK+KEx5oEBNn8RuBjYD7QBNwcozohW43SRkRQblmMpDOXMqR8NL3vlguAnqEDobQNeFOQqFPCMTpg9Lp63QpDA3z9wlM/8cT0A8b3JOS2BMyZ7knNeWgJ5aYknHmcmx2lb61FuyARujLl+iPXFXo8N8BX/wxpd6hwuJkRY9UmvWXmpZCTF8t7+xghK4K1kj4snOQRzj0ZFeZoTvr67jh63CWpP20fWlZORFMur3zyH7HGanJX2xAyJWqeLCRFYfQKeBHXm1CzeP3B0wN514aa8MTijEA5kRUkOzW1dQR2dsN7p4tWddVyzsJCclHhN3grQBB4SkVR/3J+lU7Opcbg45OPQoaEWrGFkB3LW9GyiBN4OYq/MJzdV0u02XL+4KGjnUJFHE3iQdXT30NjayYTUyKxCAU97cID3DoR/c0JXVw81DheTgjCM7EDSk+KYX5TBW3uD05ywx214bEMly6dlM1l7MCovmsCDrN7ZAcCEtMisQgEozkoiPy2B9yNgmrXKII9COJAVM3Ioq3Jw9HhHwI/99t56qpvbueEMLX2rj9MEHmQnOvFE6E1M8LTlXTotmw8ONuJ2h3c9+IkWKKFO4CWelrVrg1AKf3RdBTkp8VwwKzfgx1aRTRN4kEXSXJiDWTYti+a2LnbVDD3es51OjAMegiaE3mbnp5I9Li7gvTKrjrXxxp56Vi0q1Nlr1En0FRFkkTSV2mCWTvXUg4d7r8yKxlZS4mNC3l07Ksoza/nafQ30BPBbyhMbKxFgld68VP3QBB5kNQ4XibHRpCYGv01yMOWmJjA1Jznsx0U53NhGUVaSLc3sVpSMp7ktcKMTdvW4eXxjJeeWjA/auOYqsmkCD7Jap6cJ4Whot7tsWjYbDjXR2R2+41+Hugmht7Ot5oSBqkb55646Glo6uGGJlr5V/zSBB5lnIofIbYHibenUbNq7eoLaYcUfPW5D1bHgzEQ/HOlJccwrTA9Ye/BH11cwMT2Rc2b4PsmCGt00gQdZrcMVcaMQDuTMKVlECbwXps0JjzS309VjKLapBA6eapSyageNfjYnPHS0lXf3H+X6xYURMxG2Cj1N4EHkdhvqWyJrKrXBpCXFcurEtLC9kWlXE0JvK0pyMAbW7vOvGuWxDRXERAnXBnG8bhX5NIEHUWNrJ109JmLHQenP0qnZbK1oprWj2+5QThLKYWQHcmp+mt/NCV1dPTy1qZILZ+cyfpR8+Kvg0AQeRHWjoBNPX8umZdHtNidNVhsOKhrbiIuOsrXJZlSUcPb0HNbu9b054cs7ajnW1sUNZ0wKcHRqtNEEHkS1EToX5mAWTsokLjqK98NwXJTyxjYKMxNtrzM+pySHY21dlPnYnPDR9eVMzk7mzClZgQ1MjTqawIOoxtk7ldroSeCJcdHML0oPyxuZhxtbba0+6XX2dN8nO95T28LGw8f4zOKikM+aoyKPJvAgqnO4iI4SsseNnjpwgOXTstl5xMnK/3uXHz2/kzXbqqlsarN1vHBjDBVNbSGZhWcoGclxzC1M92l0wr+tLycuJiok07OpyBfZ3QPDXK3TRc64eNu/0gfaZ5dMor2rhy0Vx3hiYyUPvX8YgOxx8SwoSmd+UQYLitKZU5BOYlxoppE7eryTts4e2zrx9LVixnh+9fpeGo93kDXMD/C2zm5Wb6nmktPydOZ2NSyawIMo0idyGEhGchx3XHQKAN09bvbUtbClopmt5cfYWtnMq7vqAIiOEmbmpTC/MIMFk9JZUJRBUWZwurlXWC1QisOgCgU8zQn/9597eWffUS6fP3FY+/y99AgtHd06bKwaNk3gQVTrdDEtZ5zdYQRVTHQUs/PTmJ2fxueWeFpNNLV2sq3yGFvKm9lScYzVW6r467pyALKS45hflM4NZ0zi3FMC18MwHNqAezttYhpZyXG8tad+2An80fUVlOSmcPqkjCBHp0YLTeBBVOtwsdyazWYsyUyO47xTcjnvFM/41T1uw966FrZWeBL6u/uO8rXHtrLhexcErIrlcGMbIlCQER5NNntHJ3x7bwNutxnyhmRZVTNlVQ7+c+XsUTFujgoNvYkZJMc7ujne0T0qq1BGylOVkspnzijinmvm8utV82jp6ObF7TUBO0dFYyv5aYnEx4Smzn04VpTk0NTaSVm1Y8ht/7a+gsTY6GGX1pWCYSRwEXlQROpFZIfXsv8Rkd0iUiYiz4pIute6u0Rkv4jsEZFPBinusDdaJnIIhsWTM5mcncwTGysDdsxyG0chHMhZ03MQgbeGGNzK6epizbYjrJyXT2pCbIiiU6PBcErgDwEX9Vn2GnCqMWYOsBe4C0BEZgGrgNnWPr8TkfApEoXQaOzEEygiwnWLCtlwuIkDDccDcsyKxvBL4JnJccwtSB+yPfhzW6tp7+rRnpdqxIZM4MaYtUBTn2WvGmN6B8NYB/Q2Wl0JPG6M6TDGHAL2A4sDGG/EqB0lM/EEy5ULJhITJTwZgFJ4i6uLxtZO24aRHcyKkhxKq5ppau3sd70xhkfXVTCnII3TCtJCHJ2KdIGoA78FeMl6PBHwfkdWWcvGnI/GQdEE3p/xKQmcP3M8z2ypoqvHvwkielughFsJHDzDyxoD7wwwOuHm8mPsqWvRpoPKJ34lcBH5HtANPOrDvreJyCYR2dTQEPiZvO1W42gnPSmWhNgxWYM0LKsWFXH0eCevf+jfBAgVTeGbwOdMTCMzeeDRCR9dX0FKfAyXzc0PcWRqNPA5gYvI54FLgRvMR32oqwHvAYwLrGUnMcbcb4xZaIxZmJOT42sYYavW0aHVJ0M4e0YOE1ITeGJjhV/H+agEHn5VKJ7RCbNZazUn9NbU2sk/ttdw5YKJJMVpi141cj4lcBG5CLgD+LQxps1r1fPAKhGJF5HJwHRgg/9hRp465+iZyCFYoqOEaxYW8PbeBo40t/t8nPLGVrKS4xgXH55JcEXJeBpbO9nepznhM5ur6Ox28xm9eal8NJxmhI8BHwAlIlIlIrcC/wekAK+JyDYR+T2AMWYn8CSwC3gZ+Ioxpido0YexGodrVI1CGCzXLizEbeDpzVU+H6Pcmok+XJ09o7c54UfVKG634W8bKlhUnEHJhBQbo1ORbDitUK43xuQZY2KNMQXGmAeMMdOMMYXGmHnWz+1e2//EGDPVGFNijHlpsGOPVp3dbhpbO7QEPgyFmUksn5bNExsrT6piGK6KprawGQOlP5nJccwpSOetvR/V9X9wsJFDR1u16aDyi/bEDIL6FhfGaAuU4bpuUSHVze2858Ncmx3dPRxxtIfFMLKDWTEjh22VzRyzmhM+ur6cjKRYLjp1gs2RqUimCTwItAnhyFw4O5f0pFge96FNeGVTO8aEZwsUb96THdc7Xby6s45rFhZqKyXll/C86xPhah0dgHbiGa74mGiunF/AX9cdpqm1c0RjYVecmMg4vBP4nIJ0MpJieXtPA5VNbXS7Ddcv1rbfyj9aAg+CGoenRYXexBy+6xYV0tVjWL1lZDczw7kJobdor9EJH9tQyfJp2UzODu+YVfjTBB4EdU4X8TFRpCXqwETDVTIhhflF6TyxsXJEU7OVN7aRHBdNVgTMYLOiJIfG1k6qm9u156UKCE3gQVDr7GBCWoKO6zxCqxYVsq/+OFsqmoe9T3ljK0VZyRFxrc+2RifMSYnnglm5doejRgFN4EFQ62jX+m8fXDonn+S46BH1zCxvamNSmLdA6ZU1Lp6bzizm2xeWEButbz3lP30VBUGtc3TOhRlsydaYIH8vraHF1TXk9j1uQ1VTO5OyIyOBA/zo07O5dlHh0BsqNQyawAPMGEOdjoPis+sWFdLe1cMLZUPP1lPrdNHZ42ZSGA4jq1QoaAIPsKbWTjp73FoC99G8wnRm5I4b1mw95UcjowmhUsGiCTzAdCIH/3hm6yliW2Uzu2udg25bbg0jG+69MJUKFk3gAdY7lVqulsB9dsX8icRFRw1ZCi9vbCM2WshPD4+Z6JUKNU3gAdZbAtdOPL7LTI7jwtm5PLu1GlfXwINZVjS1UpiRRHRU+DchVCoYNIEHWJ3DRZRAzrh4u0OJaKsWFdHc1sWru+oG3Obw0fAeRlapYNMEHmA1Dhc5KfHEaDtfvyydmkVBRuKAbcKNMVREUBtwpYJBs0yA1TpdegMzAKKihOsWFvLe/kYqGttOWt/U2snxjm6KwnwMFKWCSRN4gOlUaoFz9cICogSe3HTyzczeFijFWoWixjBN4AGmU6kFTl5aIitKxvPU5kq6e9wfW1feqG3AldIEHkCtHd20uLq1CWEAXbeokDpnB2/vbfjY8vLGNkSgIEMTuBq7NIEHkHbiCbzzThlP9rj4k2brqWhsY0Jqgs5oo8Y0TeABVOfQqdQCLTY6iqtPL+CN3fXUWx+QYI1CqNUnaozTBB5AWgIPjmsXFtDjNjztNVtPeWOrDmKlxjxN4AFUoyXwoJiSM47FkzN50pqt53hHN0ePd2onHjXmDZnAReRBEakXkR1eyzJF5DUR2Wf9zrCWi4jcKyL7RaRMRBYEM/hwU+d0kZoQQ1KczhUdaKsWFXK4sY31h5pOtAvXKhQ11g2nBP4QcFGfZXcCrxtjpgOvW88BPgVMt35uA+4LTJiRodahEzkEy6dOzSMlIYYnNlaemIm+WDvxqDFuyARujFkLNPVZvBJ42Hr8MHC51/K/GI91QLqI5AUo1rDnmYlHR8YLhsS4aC6fN5EXt9dQVuUA0CoUNeb5Wgeea4zpnTKlFuidoXUi4N3eq8padhIRuU1ENonIpoaGhv42iTi1DhcTUnUQq2C5blEhHd1u/vpBORlJsaQmxNodklK28vsmpjHGAMaH/e43xiw0xizMycnxNwzbdfW4aTiuU6kF06kT0zh1YiotOgaKUoDvCbyut2rE+l1vLa8GvGdsLbCWjXoNLR0Yg1ahBNl1i4oAHQNFKfA9gT8P3GQ9vglY47X8Rqs1yhLA4VXVMqqdaAOeplUowfTpufmkJsQwOz/V7lCUst2Q7d1E5DFgBZAtIlXAD4GfAU+KyK1AOXCttfmLwMXAfqANuDkIMYelE1OpaRVKUKUlxvLOHeeRHK9d6JUaMoEbY64fYNX5/WxrgK/4G1Qk6k3geVqFEnRpSXrzUinQnpgBU+d0ERcTRYYmF6VUiGgCD5Aah2cmHhGdYFcpFRqawANEp1JTSoWaJvAAqXO6dCIHpVRIaQIPAGOMTqWmlAo5TeAB0NzWRWe3W5sQKqVCShN4AJwYB1wTuFIqhDSBB0CdUydyUEqFnibwAKjVBK6UsoEm8ACocbgQgfEpOg6KUip0NIEHQJ3DRfa4eGKj9XIqpUJHM04AaCcepZQdNIEHgM6FqZSygybwANASuFLKDprA/dTe2YOjvUtL4EqpkNME7qcTTQi1BK6UCjFN4H7qnchBS+BKqVDTBO6nWmc7oFOpKaVCTxO4n2odHYCWwJVSoacJ3E91Thcp8TGMix9yelGllAooTeB+qnG0a+lbKWULTeB+qnV2aAJXStnCrwQuIt8UkZ0iskNEHhORBBGZLCLrRWS/iDwhInGBCjYc1TlcegNTKWULnxO4iEwEvgYsNMacCkQDq4C7gf81xkwDjgG3BiLQcNTd46a+RadSU0rZw98qlBggUURigCSgBjgPeNpa/zBwuZ/nCFtHj3fiNtqEUCllD58TuDGmGrgHqMCTuB3AZqDZGNNtbVYFTPQ3yHClvTCVUnbypwolA1gJTAbygWTgohHsf5uIbBKRTQ0NDb6GYatah6cTj97EVErZwZ8qlAuAQ8aYBmNMF7AaWAakW1UqAAVAdX87G2PuN8YsNMYszMnJ8SMM+2g3eqWUnfxJ4BXAEhFJEhEBzgd2AW8CV1vb3ASs8S/E8FXjdBEbLWQmjeqGNkqpMOVPHfh6PDcrtwDbrWPdD3wH+JaI7AeygAcCEGdY6m1CGBUldoeilBqD/Or/bYz5IfDDPosPAov9OW6k0IkclFJ20p6YftCp1JRSdtIE7iNjjJbAlVK20gTuI2d7N64ut5bAlVK20QTuoxqntgFXStlLE7iP9tS2AFCclWxzJEqpsUoTuI/KqhzEx0RRMiHF7lCUUmOUJnAflVU1Mzs/ldhovYRKKXto9vFBd4+b7dUO5hSk2x2KUmoM0wTug331x3F1uZlXmG53KEqpMUwTuA/KqpoBmFOQZm8gSqkxTRO4D0qrHKQkxGgLFKWUrTSB+6C0spm5Bek6iJVSylaawEfI1dXDntoWrT5RStlOE/gI7apx0u022gJFKWU7TeAjVFrZDKAtUJRSttMEPkJlVQ7Gp8TrGChKKdtpAh+h0qpmrT5RSoUFTeAj4Gjv4mBDK/MK9QamUsp+msBHYEe1A0BL4EqpsKAJfARKtQemUiqMaAIfgdLKZoqzkkhPirM7FKWU0gQ+EmVVOgKhUip8aAIfpnqnixqHS6tPlFJhw68ELiLpIvK0iOwWkQ9F5EwRyRSR10Rkn/U7I1DB2qm0ynMDc6524FFKhQl/S+C/Bl42xpwCzAU+BO4EXjfGTAdet55HvLKqZqKjhNn5qXaHopRSgB8JXETSgLOBBwCMMZ3GmGZgJfCwtdnDwOX+hRgeSqscTB8/jqS4GLtDUUopwL8S+GSgAfiziGwVkT+JSDKQa4ypsbapBXL9DdJuxhjKqjxDyCqlVLjwJ4HHAAuA+4wx84FW+lSXGGMMYPrbWURuE5FNIrKpoaHBjzCCr6Kpjea2Lq3/VkqFFX8SeBVQZYxZbz1/Gk9CrxORPADrd31/Oxtj7jfGLDTGLMzJyfEjjODrvYGpLVCUUuHE5wRujKkFKkWkxFp0PrALeB64yVp2E7DGrwjDQFllM/ExUZRMSLE7FKWUOsHfO3L/CjwqInHAQeBmPB8KT4rIrUA5cK2f57BdaVUzs/NTiY3WZvNKqfDhVwI3xmwDFvaz6nx/jhtOunvc7Kh2ct2iQrtDUUqpj9Ei5RD2NxynvauHuTqErFIqzGgCH0LvFGrahFApFW40gQ+htMpBSkIMxVnJdoeilFIfowl8CGVVzcwpSCMqSuwORSmlPkYT+CBcXT3srmnR6hOlVFjSBD6IXTVOut1GxwBXSoUlTeCDKOu9gaktUJRSYUgT+CBKqxzkpMQzITXB7lCUUuokmsAHUWqNQCiiNzCVUuFHE/gAnK4uDja0MlcHsFJKhSlN4APY3jsCoQ4hq5QKU5rAB1Ba1QygJXClVNjSBD6AskoHk7KSSE+KszsUpZTqlybwAXh6YKbbHYZSSg1IE3g/6ltcHHG4tPpEKRXWNIH3o6zScwNT58BUSoUzTeD9KKtqJkpgdn6q3aEopdSANIH3Y1uVgxm5KSTF+TvjnFJKBY8m8D6MMZRZPTCVUiqcaQLvo7Kpnea2LuboAFZKqTCnCbyPbSc68KTbGodSSg1FE3gfZZXNxMdEUTIhxe5QlFJqUH4ncBGJFpGtIvKC9XyyiKwXkf0i8oSIRFRXxrIqB7PyU4mN1s82pVR4C0SW+jrwodfzu4H/NcZMA44BtwbgHCHR3eNme7VDq0+UUhHBrwQuIgXAJcCfrOcCnAc8bW3yMHC5P+cIpf0Nx2nv6tEZeJRSEcHfEvivgDsAt/U8C2g2xnRbz6uAiX6eI2R6e2DqGChKqUjgcwIXkUuBemPMZh/3v01ENonIpoaGBl/DCKhtVc2kxMcwOSvZ7lCUUmpI/pTAlwGfFpHDwON4qk5+DaSLSG8XxgKgur+djTH3G2MWGmMW5uTk+BFG4JRVNTOnMI2oKJ1CTSkV/nxO4MaYu4wxBcaYYmAV8IYx5gbgTeBqa7ObgDV+RxkCrq4edte0aPWJUipiBKOt3HeAb4nIfjx14g8E4RwB92GNk2630SFklVIRIyCjNRlj3gLesh4fBBYH4rihVFrZDOgQskqpyKG9VSxlVQ5yUuKZkJpgdyhKKTUsmsAtpVXNzC1Iw9OUXSmlwp8mcMDp6uJAQ6v2wFRKRRRN4MCOKqsDj9Z/K6UiiCZwoLQ3gU/UFihKqcihCRxPC5RJWUlkJEfUwIlKqTFOEzhWD0yt/1ZKRZgxn8DrW1wccbi0A49SKuKM+QTeOwKhduBRSkUaTeBVzUQJzM5PtTsUpZQakTGfwEurHMzITSEpLiCjCiilVMiM6QRujKG0qpk5Wv+tlIpAYzqBVza109zWpfXfSqmINKYTeGlVM4B2oVdKRaQxncDLqpqJi4miZEKK3aEopdSIjekEXlrpYHZ+KrHRY/oyKKUi1JjNXJsON7Gl4hiLijPtDkUppXwyJhN4fYuLLz+6hYkZiXzl3Gl2h6OUUj4Zcwm8q8fNVx/ditPVxe8/ezppibF2h6SUUj4Zc71XfvbSbjYcbuJX181jZp72vlRKRa4xVQJ/vvQID7x7iM8vLeby+RPtDkcppfwyZhL4ntoWvvN0GQsnZfDdi2faHY5SSvltTCRwp6uL2x/ZzLiEGH53wwLiYsbEn62UGuV8zmQiUigib4rILhHZKSJft5ZnishrIrLP+p0RuHBHzu02/NuTpVQ2tfHbzyxgfGqCneEopVTA+FMU7Qb+zRgzC1gCfEVEZgF3Aq8bY6YDr1vPbXPf2wd4bVcd3714Josna5tvpdTo4XMCN8bUGGO2WI9bgA+BicBK4GFrs4eBy/2M0Wdr9zZwz6t7uGxuPjcvK7YrDKWUCoqAVAaLSDEwH1gP5BpjaqxVtUBuIM4xUlXH2vj641uZMT6Fu686DRGxIwyllAoavxO4iIwDngG+YYxxeq8zxhjADLDfbSKySUQ2NTQ0+BvGx7i6evjSI1vo7jH8/nOn62QNSqlRya8ELiKxeJL3o8aY1dbiOhHJs9bnAfX97WuMud8Ys9AYszAnJ8efME7ywzU72V7t4JfXzWNydnJAj62UUuHCn1YoAjwAfGiM+aXXqueBm6zHNwFrfA9v5B7bUMETmyr56rnT+MQsW2pvlFIqJPypW1gGfA7YLiLbrGXfBX4GPCkitwLlwLV+RTgCpZXN/HDNTs6ans03PzEjVKdVSilb+JzAjTHvAgPdGTzf1+P6qvF4B196ZDM5KfHcu2o+0VF601IpNbqNirt7PW7D1x7fytHWTlZ/aSkZyXF2h6SUUkE3KvqU3/PqHt7b38iPLz+VUyfqDPNKqbEh4hP4yztque+tA1y/uIhrFxbaHY5SSoVMRCfwAw3H+fenSplbmM6PPj3L7nCUUiqkIjqB//TF3cTFRHHfDQuIj4m2OxyllAqpiL6J+Ytr51LR2EZ+eqLdoSilVMhFdAk8LTGW0wr0pqVSamyK6ASulFJjmSZwpZSKUJrAlVIqQmkCV0qpCKUJXCmlIpQmcKWUilCawJVSKkJpAldKqQilCVwppSKUJnCllIpQ4pk43uYgRBrwTL/mi2zgaADDCaRwjg3COz6NzTfhHBuEd3zhGtskY0y/M7+HRQL3h4hsMsYstDuO/oRzbBDe8Wlsvgnn2CC84wvn2AaiVShKKRWhNIErpVSEGg0J/H67AxhEOMcG4R2fxuabcI4Nwju+cI6tXxFfB66UUmPVaCiBK6XUmBQxCVxELhKRPSKyX0Tu7Gd9vIg8Ya1fLyLFIYqrUETeFJFdIrJTRL7ezzYrRMQhItusnx+EIjav8x8Wke3WuTf1s15E5F7r2pWJyIIQxVXidU22iYhTRL7RZ5uQXTsReVBE6kVkh9eyTBF5TUT2Wb8zBtj3JmubfSJyU4hi+x8R2W39z54VkfQB9h30/x/E+H4kItVe/7uLB9h30Pd2kGJ7wiuuwyKybYB9g37t/GKMCfsfIBo4AEwB4oBSYFafbb4M/N56vAp4IkSx5QELrMcpwN5+YlsBvGDj9TsMZA+y/mLgJUCAJcB6m/7HtXjavNpy7YCzgQXADq9lPwfutB7fCdzdz36ZwEHrd4b1OCMEsV0IxFiP7+4vtuH8/4MY34+Afx/G/33Q93YwYuuz/hfAD+y6dv78REoJfDGw3xhz0BjTCTwOrOyzzUrgYevx08D5IiLBDswYU2OM2WI9bgE+BCYG+7wBthL4i/FYB6SLSF6IYzgfOGCM8bVDl9+MMWuBpj6LvV9XDwOX97PrJ4HXjDFNxphjwGvARcGOzRjzqjGm23q6DigI5DlHYoBrNxzDeW8HLTYrR1wLPBbIc4ZKpCTwiUCl1/MqTk6SJ7axXtQOICsk0Vmsapv5wPp+Vp8pIqUi8pKIzA5lXIABXhWRzSJyWz/rh3N9g20VA7+J7Lx2ucaYGutxLZDbzzbhcP1uwfMtqj9D/f+D6atWFc+DA1Q/2X3tzgLqjDH7Blhv57UbUqQk8LAnIuOAZ4BvGGOcfVZvwVM1MBf4DfBciMNbboxZAHwK+IqInB3i8w9KROKATwNP9bPa7mt3gvF8pw67Zlsi8j2gG3h0gE3s+v/fB0wF5gE1eKoqws31DF76Duv3TqQk8Gqg0Ot5gbWs321EJAZIAxpDEZyIxOJJ3o8aY1b3XW+McRpjjluPXwRiRSQ7FLFZ56y2ftcDz+L52uptONc3mD4FbDHG1PVdYfe1A+p6q5Os3/X9bGPb9RORzwOXAjdYHzAnGcb/PyiMMXXGmB5jjBv44wDntfPaxQBXAk8MtI1d1264IiWBbwSmi8hkq7S2Cni+zzbPA713/68G3hjoBR1IVh3aA8CHxphfDrDNhN76eBFZjOe6h+rDJVlEUnof47nxtaPPZs8DN1qtUZYADq9qg1AYsBRk57WzeL+ubgLW9LPNK8CFIpJhVRNcaC0LKhG5CLgD+LQxpm2AbYbz/w9WfN73Ua4Y4LzDeW8HywXAbmNMVX8r7bx2w2b3XdTh/uBpKbEXzx3r71nL/hPPixcgAc9X8P3ABmBKiOJajudrdRmwzfq5GLgduN3a5qvATjx32NcBS0N43aZY5y21Yui9dt7xCfBb69puBxaGML5kPAk5zWuZLdcOz4dIDdCFpy72Vjz3UV4H9gH/BDKtbRcCf/La9xbrtbcfuDlEse3HU3/c+7rrbYWVD7w42P8/RPH91Xo9leFJynl947Oen/TeDnZs1vKHel9nXtuG/Nr586M9MZVSKkJFShWKUkqpPjSBK6VUhNIErpRSEUoTuFJKRShN4EopFaE0gSulVITSBK6UUhFKE7hSSkWo/w8mihhnUD8vKwAAAABJRU5ErkJggg==", - "text/plain": [ - "
" - ] - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" - } - ], - "source": [ - "state = envs.reset()\n", - "\n", - "while frame_idx < max_frames:\n", - "\n", - " log_probs = []\n", - " values = []\n", - " rewards = []\n", - " masks = []\n", - " entropy = 0\n", - "\n", - " for _ in range(num_steps):\n", - " state = torch.FloatTensor(state).to(device)\n", - " dist, value = model(state)\n", - "\n", - " action = dist.sample()\n", - " next_state, reward, done, _ = envs.step(action.cpu().numpy())\n", - "\n", - " log_prob = dist.log_prob(action)\n", - " entropy += dist.entropy().mean()\n", - " \n", - " log_probs.append(log_prob)\n", - " values.append(value)\n", - " rewards.append(torch.FloatTensor(reward).unsqueeze(1).to(device))\n", - " masks.append(torch.FloatTensor(1 - done).unsqueeze(1).to(device))\n", - " \n", - " state = next_state\n", - " frame_idx += 1\n", - " \n", - " if frame_idx % 1000 == 0:\n", - " test_rewards.append(np.mean([test_env() for _ in range(10)]))\n", - " plot(frame_idx, test_rewards)\n", - " \n", - " next_state = torch.FloatTensor(next_state).to(device)\n", - " _, next_value = model(next_state)\n", - " returns = compute_returns(next_value, rewards, masks)\n", - " \n", - " log_probs = torch.cat(log_probs)\n", - " returns = torch.cat(returns).detach()\n", - " values = torch.cat(values)\n", - "\n", - " advantage = returns - values\n", - "\n", - " actor_loss = -(log_probs * advantage.detach()).mean()\n", - " critic_loss = advantage.pow(2).mean()\n", - "\n", - " loss = actor_loss + 0.5 * critic_loss - 0.001 * entropy\n", - "\n", - " optimizer.zero_grad()\n", - " loss.backward()\n", - " optimizer.step()" - ] - } - ], - "metadata": { - "interpreter": { - "hash": "fe38df673a99c62a9fea33a7aceda74c9b65b12ee9d076c5851d98b692a4989a" - }, - "kernelspec": { - "display_name": "Python 3.7.9 64-bit ('py37': conda)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.9" - }, - "orig_nbformat": 4 - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/codes/A2C/task0.py b/codes/A2C/task0.py index bfea4d7..e29266b 100644 --- a/codes/A2C/task0.py +++ b/codes/A2C/task0.py @@ -29,14 +29,13 @@ def get_args(): parser.add_argument('--gamma',default=0.99,type=float,help="discounted factor") parser.add_argument('--lr',default=1e-3,type=float,help="learning rate") parser.add_argument('--hidden_dim',default=256,type=int) + parser.add_argument('--device',default='cpu',type=str,help="cpu or cuda") parser.add_argument('--result_path',default=curr_path + "/outputs/" + parser.parse_args().env_name + \ '/' + curr_time + '/results/' ) parser.add_argument('--model_path',default=curr_path + "/outputs/" + parser.parse_args().env_name + \ '/' + curr_time + '/models/' ) # path to save models parser.add_argument('--save_fig',default=True,type=bool,help="if save figure or not") - args = parser.parse_args() - args.device = torch.device( - "cuda" if torch.cuda.is_available() else "cpu") # check GPU + args = parser.parse_args() return args def make_envs(env_name): @@ -124,14 +123,15 @@ def train(cfg,envs): loss.backward() optimizer.step() print('Finish training!') - return test_rewards, test_ma_rewards + return {'rewards':test_rewards,'ma_rewards':test_ma_rewards} if __name__ == "__main__": cfg = get_args() envs = [make_envs(cfg.env_name) for i in range(cfg.n_envs)] envs = SubprocVecEnv(envs) # training - rewards,ma_rewards = train(cfg,envs) + res_dic = train(cfg,envs) make_dir(cfg.result_path,cfg.model_path) save_args(cfg) - save_results(rewards, ma_rewards, tag='train', path=cfg.result_path) # 保存结果 - plot_rewards(rewards, ma_rewards, cfg, tag="train") # 画出结果 + save_results(res_dic, tag='train', + path=cfg.result_path) + plot_rewards(res_dic['rewards'], res_dic['ma_rewards'], cfg, tag="train") # 画出结果 diff --git a/codes/DDPG/ddpg.py b/codes/DDPG/ddpg.py index 4d2ed42..93894e3 100644 --- a/codes/DDPG/ddpg.py +++ b/codes/DDPG/ddpg.py @@ -73,11 +73,11 @@ class Critic(nn.Module): return x class DDPG: def __init__(self, n_states, n_actions, cfg): - self.device = cfg.device - self.critic = Critic(n_states, n_actions, cfg.hidden_dim).to(cfg.device) - self.actor = Actor(n_states, n_actions, cfg.hidden_dim).to(cfg.device) - self.target_critic = Critic(n_states, n_actions, cfg.hidden_dim).to(cfg.device) - self.target_actor = Actor(n_states, n_actions, cfg.hidden_dim).to(cfg.device) + self.device = torch.device(cfg.device) + self.critic = Critic(n_states, n_actions, cfg.hidden_dim).to(self.device) + self.actor = Actor(n_states, n_actions, cfg.hidden_dim).to(self.device) + self.target_critic = Critic(n_states, n_actions, cfg.hidden_dim).to(self.device) + self.target_actor = Actor(n_states, n_actions, cfg.hidden_dim).to(self.device) # 复制参数到目标网络 for target_param, param in zip(self.target_critic.parameters(), self.critic.parameters()): diff --git a/codes/DDPG/outputs/Pendulum-v1/20220713-225402/results/params.json b/codes/DDPG/outputs/Pendulum-v1/20220713-225402/results/params.json new file mode 100644 index 0000000..7d22454 --- /dev/null +++ b/codes/DDPG/outputs/Pendulum-v1/20220713-225402/results/params.json @@ -0,0 +1,18 @@ +{ + "algo_name": "DDPG", + "env_name": "Pendulum-v1", + "train_eps": 300, + "test_eps": 20, + "gamma": 0.99, + "critic_lr": 0.001, + "actor_lr": 0.0001, + "memory_capacity": 8000, + "batch_size": 128, + "target_update": 2, + "soft_tau": 0.01, + "hidden_dim": 256, + "deivce": "cpu", + "result_path": "C:\\Users\\24438\\Desktop\\rl-tutorials/outputs/DDPG/outputs/Pendulum-v1/20220713-225402/results//", + "model_path": "C:\\Users\\24438\\Desktop\\rl-tutorials/outputs/DDPG/outputs/Pendulum-v1/20220713-225402/models/", + "save_fig": true +} \ No newline at end of file diff --git a/codes/DDPG/outputs/Pendulum-v1/20220713-225402/results/params.txt b/codes/DDPG/outputs/Pendulum-v1/20220713-225402/results/params.txt deleted file mode 100644 index 95a5a55..0000000 --- a/codes/DDPG/outputs/Pendulum-v1/20220713-225402/results/params.txt +++ /dev/null @@ -1,18 +0,0 @@ ------------------- start ------------------ -algo_name : DDPG -env_name : Pendulum-v1 -train_eps : 300 -test_eps : 20 -gamma : 0.99 -critic_lr : 0.001 -actor_lr : 0.0001 -memory_capacity : 8000 -batch_size : 128 -target_update : 2 -soft_tau : 0.01 -hidden_dim : 256 -result_path : c:\Users\24438\Desktop\rl-tutorials\codes\DDPG/outputs/Pendulum-v1/20220713-225402/results/ -model_path : c:\Users\24438\Desktop\rl-tutorials\codes\DDPG/outputs/Pendulum-v1/20220713-225402/models/ -save_fig : True -device : cuda -------------------- end ------------------- \ No newline at end of file diff --git a/codes/DDPG/task0.py b/codes/DDPG/task0.py index 861d7f3..20688d3 100644 --- a/codes/DDPG/task0.py +++ b/codes/DDPG/task0.py @@ -5,7 +5,7 @@ @Email: johnjim0816@gmail.com @Date: 2020-06-11 20:58:21 @LastEditor: John -LastEditTime: 2022-07-13 22:53:11 +LastEditTime: 2022-07-21 21:51:34 @Discription: @Environment: python 3.7.7 ''' @@ -41,14 +41,13 @@ def get_args(): parser.add_argument('--target_update',default=2,type=int) parser.add_argument('--soft_tau',default=1e-2,type=float) parser.add_argument('--hidden_dim',default=256,type=int) + parser.add_argument('--device',default='cpu',type=str,help="cpu or cuda") parser.add_argument('--result_path',default=curr_path + "/outputs/" + parser.parse_args().env_name + \ '/' + curr_time + '/results/' ) parser.add_argument('--model_path',default=curr_path + "/outputs/" + parser.parse_args().env_name + \ '/' + curr_time + '/models/' ) # path to save models - parser.add_argument('--save_fig',default=True,type=bool,help="if save figure or not") - args = parser.parse_args() - args.device = torch.device( - "cuda" if torch.cuda.is_available() else "cpu") # check GPU + parser.add_argument('--save_fig',default=True,type=bool,help="if save figure or not") + args = parser.parse_args() return args def env_agent_config(cfg,seed=1): @@ -87,7 +86,7 @@ def train(cfg, env, agent): else: ma_rewards.append(ep_reward) print('Finish training!') - return rewards, ma_rewards + return {'rewards':rewards,'ma_rewards':ma_rewards} def test(cfg, env, agent): print('Start testing') @@ -112,21 +111,23 @@ def test(cfg, env, agent): ma_rewards.append(ep_reward) print(f"Epside:{i_ep+1}/{cfg.test_eps}, Reward:{ep_reward:.1f}") print('Finish testing!') - return rewards, ma_rewards + return {'rewards':rewards,'ma_rewards':ma_rewards} if __name__ == "__main__": cfg = get_args() # training env,agent = env_agent_config(cfg,seed=1) - rewards, ma_rewards = train(cfg, env, agent) + res_dic = train(cfg, env, agent) make_dir(cfg.result_path, cfg.model_path) save_args(cfg) agent.save(path=cfg.model_path) - save_results(rewards, ma_rewards, tag='train', path=cfg.result_path) - plot_rewards(rewards, ma_rewards, cfg, tag="train") # 画出结果 + save_results(res_dic, tag='train', + path=cfg.result_path) + plot_rewards(res_dic['rewards'], res_dic['ma_rewards'], cfg, tag="train") # testing env,agent = env_agent_config(cfg,seed=10) agent.load(path=cfg.model_path) - rewards,ma_rewards = test(cfg,env,agent) - save_results(rewards,ma_rewards,tag = 'test',path = cfg.result_path) - plot_rewards(rewards, ma_rewards, cfg, tag="test") # 画出结果 + res_dic = test(cfg,env,agent) + save_results(res_dic, tag='test', + path=cfg.result_path) + plot_rewards(res_dic['rewards'], res_dic['ma_rewards'], cfg, tag="test") diff --git a/codes/DQN/dqn.py b/codes/DQN/dqn.py index 0fa0d94..5ce5e1e 100644 --- a/codes/DQN/dqn.py +++ b/codes/DQN/dqn.py @@ -5,7 +5,7 @@ @Email: johnjim0816@gmail.com @Date: 2020-06-12 00:50:49 @LastEditor: John -LastEditTime: 2022-07-13 00:08:18 +LastEditTime: 2022-07-20 23:57:16 @Discription: @Environment: python 3.7.7 ''' @@ -64,8 +64,8 @@ class ReplayBuffer: class DQN: def __init__(self, n_states,n_actions,cfg): - self.n_actions = n_actions # 总的动作个数 - self.device = cfg.device # 设备,cpu或gpu等 + self.n_actions = n_actions + self.device = torch.device(cfg.device) # cpu or cuda self.gamma = cfg.gamma # 奖励的折扣因子 # e-greedy策略相关参数 self.frame_idx = 0 # 用于epsilon的衰减计数 diff --git a/codes/DQN/outputs/CartPole-v0/20220713-211653/results/params.json b/codes/DQN/outputs/CartPole-v0/20220713-211653/results/params.json new file mode 100644 index 0000000..3dfcdd4 --- /dev/null +++ b/codes/DQN/outputs/CartPole-v0/20220713-211653/results/params.json @@ -0,0 +1,19 @@ +{ + "algo_name": "DQN", + "env_name": "CartPole-v0", + "train_eps": 200, + "test_eps": 20, + "gamma": 0.95, + "epsilon_start": 0.95, + "epsilon_end": 0.01, + "epsilon_decay": 500, + "lr": 0.0001, + "memory_capacity": 100000, + "batch_size": 64, + "target_update": 4, + "hidden_dim": 256, + "deivce": "cpu", + "result_path": "C:\\Users\\24438\\Desktop\\rl-tutorials/outputs/CartPole-v0/20220713-211653/results/", + "model_path": "C:\\Users\\24438\\Desktop\\rl-tutorials/outputs/CartPole-v0/20220713-211653/models/", + "save_fig": true +} \ No newline at end of file diff --git a/codes/DQN/outputs/CartPole-v0/20220713-211653/results/params.txt b/codes/DQN/outputs/CartPole-v0/20220713-211653/results/params.txt deleted file mode 100644 index 40eac02..0000000 --- a/codes/DQN/outputs/CartPole-v0/20220713-211653/results/params.txt +++ /dev/null @@ -1,19 +0,0 @@ ------------------- start ------------------ -algo_name : DQN -env_name : CartPole-v0 -train_eps : 200 -test_eps : 20 -gamma : 0.95 -epsilon_start : 0.95 -epsilon_end : 0.01 -epsilon_decay : 500 -lr : 0.0001 -memory_capacity : 100000 -batch_size : 64 -target_update : 4 -hidden_dim : 256 -result_path : C:\Users\24438\Desktop\rl-tutorials\codes\DQN/outputs/CartPole-v0/20220713-211653/results/ -model_path : C:\Users\24438\Desktop\rl-tutorials\codes\DQN/outputs/CartPole-v0/20220713-211653/models/ -save_fig : True -device : cuda -------------------- end ------------------- \ No newline at end of file diff --git a/codes/DQN/task0.py b/codes/DQN/task0.py index 9ccf26f..04344aa 100644 --- a/codes/DQN/task0.py +++ b/codes/DQN/task0.py @@ -1,19 +1,16 @@ -from lib2to3.pytree import type_repr -import sys -import os -from parso import parse +import sys,os +curr_path = os.path.dirname(os.path.abspath(__file__)) # current path +parent_path = os.path.dirname(curr_path) # parent path +sys.path.append(parent_path) # add to system path import torch.nn as nn import torch.nn.functional as F -curr_path = os.path.dirname(os.path.abspath(__file__)) # 当前文件所在绝对路径 -parent_path = os.path.dirname(curr_path) # 父路径 -sys.path.append(parent_path) # 添加路径到系统路径 import gym import torch import datetime import numpy as np import argparse -from common.utils import save_results_1, make_dir +from common.utils import save_results, make_dir from common.utils import plot_rewards,save_args from dqn import DQN @@ -35,14 +32,13 @@ def get_args(): parser.add_argument('--batch_size',default=64,type=int) parser.add_argument('--target_update',default=4,type=int) parser.add_argument('--hidden_dim',default=256,type=int) + parser.add_argument('--device',default='cpu',type=str,help="cpu or cuda") parser.add_argument('--result_path',default=curr_path + "/outputs/" + parser.parse_args().env_name + \ '/' + curr_time + '/results/' ) parser.add_argument('--model_path',default=curr_path + "/outputs/" + parser.parse_args().env_name + \ '/' + curr_time + '/models/' ) # path to save models parser.add_argument('--save_fig',default=True,type=bool,help="if save figure or not") - args = parser.parse_args() - args.device = torch.device( - "cuda" if torch.cuda.is_available() else "cpu") # check GPU + args = parser.parse_args() return args def env_agent_config(cfg,seed=1): @@ -99,8 +95,8 @@ def train(cfg, env, agent): def test(cfg, env, agent): - print('开始测试!') - print(f'环境:{cfg.env_name}, 算法:{cfg.algo_name}, 设备:{cfg.device}') + print('Start testing!') + print(f'Env:{cfg.env_name}, A{cfg.algo_name}, 设备:{cfg.device}') ############# 由于测试不需要使用epsilon-greedy策略,所以相应的值设置为0 ############### cfg.epsilon_start = 0.0 # e-greedy策略中初始epsilon cfg.epsilon_end = 0.0 # e-greedy策略中的终止epsilon @@ -127,7 +123,7 @@ def test(cfg, env, agent): else: ma_rewards.append(ep_reward) print(f'Episode:{i_ep+1}/{cfg.test_eps}, Reward:{ep_reward:.2f}, Step:{ep_step:.2f}') - print('完成测试!') + print('Finish testing') env.close() return {'rewards':rewards,'ma_rewards':ma_rewards,'steps':steps} @@ -137,16 +133,16 @@ if __name__ == "__main__": # 训练 env, agent = env_agent_config(cfg) res_dic = train(cfg, env, agent) - make_dir(cfg.result_path, cfg.model_path) # 创建保存结果和模型路径的文件夹 - save_args(cfg) - agent.save(path=cfg.model_path) # 保存模型 - save_results_1(res_dic, tag='train', - path=cfg.result_path) # 保存结果 - plot_rewards(res_dic['rewards'], res_dic['ma_rewards'], cfg, tag="train") # 画出结果 + make_dir(cfg.result_path, cfg.model_path) + save_args(cfg) # save parameters + agent.save(path=cfg.model_path) # save model + save_results(res_dic, tag='train', + path=cfg.result_path) + plot_rewards(res_dic['rewards'], res_dic['ma_rewards'], cfg, tag="train") # 测试 env, agent = env_agent_config(cfg) agent.load(path=cfg.model_path) # 导入模型 res_dic = test(cfg, env, agent) - save_results_1(res_dic, tag='test', + save_results(res_dic, tag='test', path=cfg.result_path) # 保存结果 plot_rewards(res_dic['rewards'], res_dic['ma_rewards'],cfg, tag="test") # 画出结果 diff --git a/codes/DoubleDQN/double_dqn.py b/codes/DoubleDQN/double_dqn.py index 8dbdc52..78642ea 100644 --- a/codes/DoubleDQN/double_dqn.py +++ b/codes/DoubleDQN/double_dqn.py @@ -5,7 +5,7 @@ @Email: johnjim0816@gmail.com @Date: 2020-06-12 00:50:49 @LastEditor: John -LastEditTime: 2021-11-19 18:07:09 +LastEditTime: 2022-07-21 00:08:26 @Discription: @Environment: python 3.7.7 ''' @@ -65,7 +65,7 @@ class MLP(nn.Module): class DoubleDQN: def __init__(self, n_states, n_actions, cfg): self.n_actions = n_actions # 总的动作个数 - self.device = cfg.device # 设备,cpu或gpu等 + self.device = torch.device(cfg.device) # 设备,cpu或gpu等 self.gamma = cfg.gamma # e-greedy策略相关参数 self.actions_count = 0 @@ -88,8 +88,7 @@ class DoubleDQN: '''选择动作 ''' self.actions_count += 1 - self.epsilon = self.epsilon_end + (self.epsilon_start - self.epsilon_end) * \ - math.exp(-1. * self.actions_count / self.epsilon_decay) + self.epsilon = self.epsilon_end + (self.epsilon_start - self.epsilon_end) * math.exp(-1. * self.actions_count / self.epsilon_decay) if random.random() > self.epsilon: with torch.no_grad(): # 先转为张量便于丢给神经网络,state元素数据原本为float64 diff --git a/codes/DoubleDQN/outputs/CartPole-v0/20211229-145006/models/checkpoint.pth b/codes/DoubleDQN/outputs/CartPole-v0/20211229-145006/models/checkpoint.pth deleted file mode 100644 index 2ec6bfd..0000000 Binary files a/codes/DoubleDQN/outputs/CartPole-v0/20211229-145006/models/checkpoint.pth and /dev/null differ diff --git a/codes/DoubleDQN/outputs/CartPole-v0/20211229-145006/results/test_ma_rewards.npy b/codes/DoubleDQN/outputs/CartPole-v0/20211229-145006/results/test_ma_rewards.npy deleted file mode 100644 index 81e0bba..0000000 Binary files a/codes/DoubleDQN/outputs/CartPole-v0/20211229-145006/results/test_ma_rewards.npy and /dev/null differ diff --git a/codes/DoubleDQN/outputs/CartPole-v0/20211229-145006/results/test_rewards.npy b/codes/DoubleDQN/outputs/CartPole-v0/20211229-145006/results/test_rewards.npy deleted file mode 100644 index e7b6307..0000000 Binary files a/codes/DoubleDQN/outputs/CartPole-v0/20211229-145006/results/test_rewards.npy and /dev/null differ diff --git a/codes/DoubleDQN/outputs/CartPole-v0/20211229-145006/results/test_rewards_curve.png b/codes/DoubleDQN/outputs/CartPole-v0/20211229-145006/results/test_rewards_curve.png deleted file mode 100644 index 4fbd77c..0000000 Binary files a/codes/DoubleDQN/outputs/CartPole-v0/20211229-145006/results/test_rewards_curve.png and /dev/null differ diff --git a/codes/DoubleDQN/outputs/CartPole-v0/20211229-145006/results/train_ma_rewards.npy b/codes/DoubleDQN/outputs/CartPole-v0/20211229-145006/results/train_ma_rewards.npy deleted file mode 100644 index a73bbde..0000000 Binary files a/codes/DoubleDQN/outputs/CartPole-v0/20211229-145006/results/train_ma_rewards.npy and /dev/null differ diff --git a/codes/DoubleDQN/outputs/CartPole-v0/20211229-145006/results/train_rewards_curve.png b/codes/DoubleDQN/outputs/CartPole-v0/20211229-145006/results/train_rewards_curve.png deleted file mode 100644 index cb9dbeb..0000000 Binary files a/codes/DoubleDQN/outputs/CartPole-v0/20211229-145006/results/train_rewards_curve.png and /dev/null differ diff --git a/codes/DoubleDQN/outputs/CartPole-v0/20220721-215416/models/checkpoint.pth b/codes/DoubleDQN/outputs/CartPole-v0/20220721-215416/models/checkpoint.pth new file mode 100644 index 0000000..2818144 Binary files /dev/null and b/codes/DoubleDQN/outputs/CartPole-v0/20220721-215416/models/checkpoint.pth differ diff --git a/codes/DoubleDQN/outputs/CartPole-v0/20220721-215416/results/params.json b/codes/DoubleDQN/outputs/CartPole-v0/20220721-215416/results/params.json new file mode 100644 index 0000000..abc1877 --- /dev/null +++ b/codes/DoubleDQN/outputs/CartPole-v0/20220721-215416/results/params.json @@ -0,0 +1,19 @@ +{ + "algo_name": "DoubleDQN", + "env_name": "CartPole-v0", + "train_eps": 200, + "test_eps": 20, + "gamma": 0.99, + "epsilon_start": 0.95, + "epsilon_end": 0.01, + "epsilon_decay": 500, + "lr": 0.0001, + "memory_capacity": 100000, + "batch_size": 64, + "target_update": 2, + "hidden_dim": 256, + "device": "cuda", + "result_path": "C:\\Users\\24438\\Desktop\\rl-tutorials\\codes\\DoubleDQN/outputs/CartPole-v0/20220721-215416/results/", + "model_path": "C:\\Users\\24438\\Desktop\\rl-tutorials\\codes\\DoubleDQN/outputs/CartPole-v0/20220721-215416/models/", + "save_fig": true +} \ No newline at end of file diff --git a/codes/DoubleDQN/outputs/CartPole-v0/20220721-215416/results/test_ma_rewards.npy b/codes/DoubleDQN/outputs/CartPole-v0/20220721-215416/results/test_ma_rewards.npy new file mode 100644 index 0000000..da15b7f Binary files /dev/null and b/codes/DoubleDQN/outputs/CartPole-v0/20220721-215416/results/test_ma_rewards.npy differ diff --git a/codes/DoubleDQN/outputs/CartPole-v0/20220721-215416/results/test_rewards.npy b/codes/DoubleDQN/outputs/CartPole-v0/20220721-215416/results/test_rewards.npy new file mode 100644 index 0000000..ce7e7be Binary files /dev/null and b/codes/DoubleDQN/outputs/CartPole-v0/20220721-215416/results/test_rewards.npy differ diff --git a/codes/DoubleDQN/outputs/CartPole-v0/20220721-215416/results/test_rewards_curve.png b/codes/DoubleDQN/outputs/CartPole-v0/20220721-215416/results/test_rewards_curve.png new file mode 100644 index 0000000..9123a84 Binary files /dev/null and b/codes/DoubleDQN/outputs/CartPole-v0/20220721-215416/results/test_rewards_curve.png differ diff --git a/codes/DoubleDQN/outputs/CartPole-v0/20220721-215416/results/train_ma_rewards.npy b/codes/DoubleDQN/outputs/CartPole-v0/20220721-215416/results/train_ma_rewards.npy new file mode 100644 index 0000000..b44206b Binary files /dev/null and b/codes/DoubleDQN/outputs/CartPole-v0/20220721-215416/results/train_ma_rewards.npy differ diff --git a/codes/DoubleDQN/outputs/CartPole-v0/20211229-145006/results/train_rewards.npy b/codes/DoubleDQN/outputs/CartPole-v0/20220721-215416/results/train_rewards.npy similarity index 51% rename from codes/DoubleDQN/outputs/CartPole-v0/20211229-145006/results/train_rewards.npy rename to codes/DoubleDQN/outputs/CartPole-v0/20220721-215416/results/train_rewards.npy index 3e707c5..d9b5730 100644 Binary files a/codes/DoubleDQN/outputs/CartPole-v0/20211229-145006/results/train_rewards.npy and b/codes/DoubleDQN/outputs/CartPole-v0/20220721-215416/results/train_rewards.npy differ diff --git a/codes/DoubleDQN/outputs/CartPole-v0/20220721-215416/results/train_rewards_curve.png b/codes/DoubleDQN/outputs/CartPole-v0/20220721-215416/results/train_rewards_curve.png new file mode 100644 index 0000000..d07d996 Binary files /dev/null and b/codes/DoubleDQN/outputs/CartPole-v0/20220721-215416/results/train_rewards_curve.png differ diff --git a/codes/DoubleDQN/task0.py b/codes/DoubleDQN/task0.py index 2f91e1e..66dfcd9 100644 --- a/codes/DoubleDQN/task0.py +++ b/codes/DoubleDQN/task0.py @@ -5,55 +5,49 @@ Author: JiangJi Email: johnjim0816@gmail.com Date: 2021-11-07 18:10:37 LastEditor: JiangJi -LastEditTime: 2021-12-29 15:02:30 +LastEditTime: 2022-07-21 21:52:31 Discription: ''' - import sys,os -curr_path = os.path.dirname(os.path.abspath(__file__)) # 当前文件所在绝对路径 -parent_path = os.path.dirname(curr_path) # 父路径 -sys.path.append(parent_path) # 添加路径到系统路径 +curr_path = os.path.dirname(os.path.abspath(__file__)) # current path +parent_path = os.path.dirname(curr_path) # parent path +sys.path.append(parent_path) # add to system path import gym import torch import datetime +import argparse -from common.utils import save_results, make_dir -from common.utils import plot_rewards +from common.utils import save_results,make_dir +from common.utils import plot_rewards,save_args from DoubleDQN.double_dqn import DoubleDQN -curr_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") # 获取当前时间 - -class Config: - def __init__(self): - ################################## 环境超参数 ################################### - self.algo_name = 'DoubleDQN' # 算法名称 - self.env_name = 'CartPole-v0' # 环境名称 - self.device = torch.device( - "cuda" if torch.cuda.is_available() else "cpu") # 检测GPU - self.train_eps = 200 # 训练的回合数 - self.test_eps = 30 # 测试的回合数 - ################################################################################ - - ################################## 算法超参数 ################################### - self.gamma = 0.95 # 强化学习中的折扣因子 - self.epsilon_start = 0.95 # e-greedy策略中初始epsilon - self.epsilon_end = 0.01 # e-greedy策略中的终止epsilon - self.epsilon_decay = 500 # e-greedy策略中epsilon的衰减率 - self.lr = 0.0001 # 学习率 - self.memory_capacity = 100000 # 经验回放的容量 - self.batch_size = 64 # mini-batch SGD中的批量大小 - self.target_update = 2 # 目标网络的更新频率 - self.hidden_dim = 256 # 网络隐藏层 - ################################################################################ - - ################################# 保存结果相关参数 ############################## - self.result_path = curr_path + "/outputs/" + self.env_name + \ - '/' + curr_time + '/results/' # 保存结果的路径 - self.model_path = curr_path + "/outputs/" + self.env_name + \ - '/' + curr_time + '/models/' # 保存模型的路径 - self.save = True # 是否保存图片 - ################################################################################ +def get_args(): + """ Hyperparameters + """ + curr_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") # Obtain current time + parser = argparse.ArgumentParser(description="hyperparameters") + parser.add_argument('--algo_name',default='DoubleDQN',type=str,help="name of algorithm") + parser.add_argument('--env_name',default='CartPole-v0',type=str,help="name of environment") + parser.add_argument('--train_eps',default=200,type=int,help="episodes of training") + parser.add_argument('--test_eps',default=20,type=int,help="episodes of testing") + parser.add_argument('--gamma',default=0.99,type=float,help="discounted factor") + parser.add_argument('--epsilon_start',default=0.95,type=float,help="initial value of epsilon") + parser.add_argument('--epsilon_end',default=0.01,type=float,help="final value of epsilon") + parser.add_argument('--epsilon_decay',default=500,type=int,help="decay rate of epsilon") + parser.add_argument('--lr',default=0.0001,type=float,help="learning rate") + parser.add_argument('--memory_capacity',default=100000,type=int,help="memory capacity") + parser.add_argument('--batch_size',default=64,type=int) + parser.add_argument('--target_update',default=2,type=int) + parser.add_argument('--hidden_dim',default=256,type=int) + parser.add_argument('--device',default='cpu',type=str,help="cpu or cuda") + parser.add_argument('--result_path',default=curr_path + "/outputs/" + parser.parse_args().env_name + \ + '/' + curr_time + '/results/' ) + parser.add_argument('--model_path',default=curr_path + "/outputs/" + parser.parse_args().env_name + \ + '/' + curr_time + '/models/' ) # path to save models + parser.add_argument('--save_fig',default=True,type=bool,help="if save figure or not") + args = parser.parse_args() + return args def env_agent_config(cfg,seed=1): @@ -65,8 +59,8 @@ def env_agent_config(cfg,seed=1): return env,agent def train(cfg,env,agent): - print('开始训练!') - print(f'环境:{cfg.env_name}, 算法:{cfg.algo_name}, 设备:{cfg.device}') + print('Start training!') + print(f'Env:{cfg.env_name}, Algorithm:{cfg.algo_name}, Device:{cfg.device}') rewards = [] # 记录所有回合的奖励 ma_rewards = [] # 记录所有回合的滑动平均奖励 for i_ep in range(cfg.train_eps): @@ -84,20 +78,19 @@ def train(cfg,env,agent): if i_ep % cfg.target_update == 0: agent.target_net.load_state_dict(agent.policy_net.state_dict()) if (i_ep+1)%10 == 0: - print(f'回合:{i_ep+1}/{cfg.train_eps},奖励:{ep_reward}') + print(f'Env:{i_ep+1}/{cfg.train_eps}, Reward:{ep_reward:.2f}') rewards.append(ep_reward) if ma_rewards: ma_rewards.append( 0.9*ma_rewards[-1]+0.1*ep_reward) else: ma_rewards.append(ep_reward) - print('完成训练!') - env.close() - return rewards,ma_rewards + print('Finish training!') + return {'rewards':rewards,'ma_rewards':ma_rewards} def test(cfg,env,agent): - print('开始测试!') - print(f'环境:{cfg.env_name}, 算法:{cfg.algo_name}, 设备:{cfg.device}') + print('Start testing') + print(f'Env:{cfg.env_name}, Algorithm:{cfg.algo_name}, Device:{cfg.device}') ############# 由于测试不需要使用epsilon-greedy策略,所以相应的值设置为0 ############### cfg.epsilon_start = 0.0 # e-greedy策略中初始epsilon cfg.epsilon_end = 0.0 # e-greedy策略中的终止epsilon @@ -120,25 +113,26 @@ def test(cfg,env,agent): ma_rewards.append(ma_rewards[-1]*0.9+ep_reward*0.1) else: ma_rewards.append(ep_reward) - print(f"回合:{i_ep+1}/{cfg.test_eps},奖励:{ep_reward:.1f}") - print('完成测试!') - env.close() - return rewards,ma_rewards + print(f"Epside:{i_ep+1}/{cfg.test_eps}, Reward:{ep_reward:.1f}") + print('Finish testing!') + return {'rewards':rewards,'ma_rewards':ma_rewards} if __name__ == "__main__": - cfg = Config() - # 训练 - env, agent = env_agent_config(cfg) - rewards, ma_rewards = train(cfg, env, agent) - make_dir(cfg.result_path, cfg.model_path) # 创建保存结果和模型路径的文件夹 - agent.save(path=cfg.model_path) # 保存模型 - save_results(rewards, ma_rewards, tag='train', - path=cfg.result_path) # 保存结果 - plot_rewards(rewards, ma_rewards, cfg, tag="train") # 画出结果 - # 测试 - env, agent = env_agent_config(cfg) - agent.load(path=cfg.model_path) # 导入模型 - rewards, ma_rewards = test(cfg, env, agent) - save_results(rewards, ma_rewards, tag='test', - path=cfg.result_path) # 保存结果 - plot_rewards(rewards, ma_rewards, cfg, tag="test") # 画出结果 + cfg = get_args() + print(cfg.device) + # training + env,agent = env_agent_config(cfg,seed=1) + res_dic = train(cfg, env, agent) + make_dir(cfg.result_path, cfg.model_path) + save_args(cfg) + agent.save(path=cfg.model_path) + save_results(res_dic, tag='train', + path=cfg.result_path) + plot_rewards(res_dic['rewards'], res_dic['ma_rewards'], cfg, tag="train") + # testing + env,agent = env_agent_config(cfg,seed=10) + agent.load(path=cfg.model_path) + res_dic = test(cfg,env,agent) + save_results(res_dic, tag='test', + path=cfg.result_path) + plot_rewards(res_dic['rewards'], res_dic['ma_rewards'], cfg, tag="test") diff --git a/codes/PPO/task0.py b/codes/PPO/task0.py index 2d40944..6a73ff8 100644 --- a/codes/PPO/task0.py +++ b/codes/PPO/task0.py @@ -16,7 +16,7 @@ curr_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") # 获取当前时 class Config: def __init__(self) -> None: ################################## 环境超参数 ################################### - self.algo_name = "DQN" # 算法名称 + self.algo_name = "PPO" # 算法名称 self.env_name = 'CartPole-v0' # 环境名称 self.continuous = False # 环境是否为连续动作 self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # 检测GPU diff --git a/codes/PolicyGradient/task0.py b/codes/PolicyGradient/task0.py index c676fe3..b9e11a0 100644 --- a/codes/PolicyGradient/task0.py +++ b/codes/PolicyGradient/task0.py @@ -5,56 +5,47 @@ Author: John Email: johnjim0816@gmail.com Date: 2020-11-22 23:21:53 LastEditor: John -LastEditTime: 2022-02-10 06:13:21 +LastEditTime: 2022-07-21 21:44:00 Discription: Environment: ''' -import sys -import os -curr_path = os.path.dirname(os.path.abspath(__file__)) # 当前文件所在绝对路径 -parent_path = os.path.dirname(curr_path) # 父路径 -sys.path.append(parent_path) # 添加路径到系统路径 +import sys,os +curr_path = os.path.dirname(os.path.abspath(__file__)) # current path +parent_path = os.path.dirname(curr_path) # parent path +sys.path.append(parent_path) # add to system path import gym import torch import datetime +import argparse from itertools import count from pg import PolicyGradient from common.utils import save_results, make_dir from common.utils import plot_rewards -curr_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") # 获取当前时间 -class Config: - '''超参数 - ''' - - def __init__(self): - ################################## 环境超参数 ################################### - self.algo_name = "PolicyGradient" # 算法名称 - self.env_name = 'CartPole-v0' # 环境名称 - self.device = torch.device( - "cuda" if torch.cuda.is_available() else "cpu") # 检测GPUgjgjlkhfsf风刀霜的撒发十 - self.seed = 10 # 随机种子,置0则不设置随机种子 - self.train_eps = 300 # 训练的回合数 - self.test_eps = 30 # 测试的回合数 - ################################################################################ - - ################################## 算法超参数 ################################### - self.batch_size = 8 # mini-batch SGD中的批量大小 - self.lr = 0.01 # 学习率 - self.gamma = 0.99 # 强化学习中的折扣因子 - self.hidden_dim = 36 # 网络隐藏层 - ################################################################################ - - ################################# 保存结果相关参数 ################################ - self.result_path = curr_path + "/outputs/" + self.env_name + \ - '/' + curr_time + '/results/' # 保存结果的路径 - self.model_path = curr_path + "/outputs/" + self.env_name + \ - '/' + curr_time + '/models/' # 保存模型的路径 - self.save = True # 是否保存图片 - ################################################################################ +def get_args(): + """ Hyperparameters + """ + curr_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") # Obtain current time + parser = argparse.ArgumentParser(description="hyperparameters") + parser.add_argument('--algo_name',default='PolicyGradient',type=str,help="name of algorithm") + parser.add_argument('--env_name',default='CartPole-v0',type=str,help="name of environment") + parser.add_argument('--train_eps',default=300,type=int,help="episodes of training") + parser.add_argument('--test_eps',default=20,type=int,help="episodes of testing") + parser.add_argument('--gamma',default=0.99,type=float,help="discounted factor") + parser.add_argument('--lr',default=0.01,type=float,help="learning rate") + parser.add_argument('--batch_size',default=8,type=int) + parser.add_argument('--hidden_dim',default=36,type=int) + parser.add_argument('--device',default='cpu',type=str,help="cpu or cuda") + parser.add_argument('--result_path',default=curr_path + "/outputs/" + parser.parse_args().env_name + \ + '/' + curr_time + '/results/' ) + parser.add_argument('--model_path',default=curr_path + "/outputs/" + parser.parse_args().env_name + \ + '/' + curr_time + '/models/' ) # path to save models + parser.add_argument('--save_fig',default=True,type=bool,help="if save figure or not") + args = parser.parse_args() + return args def env_agent_config(cfg,seed=1): @@ -65,9 +56,9 @@ def env_agent_config(cfg,seed=1): return env,agent def train(cfg,env,agent): - print('开始训练!') - print(f'环境:{cfg.env_name}, 算法:{cfg.algo_name}, 设备:{cfg.device}') - state_pool = [] # 存放每batch_size个episode的state序列 + print('Start training!') + print(f'Env:{cfg.env_name}, Algorithm:{cfg.algo_name}, Device:{cfg.device}') + state_pool = [] # temp states pool per several episodes action_pool = [] reward_pool = [] rewards = [] @@ -86,11 +77,11 @@ def train(cfg,env,agent): reward_pool.append(reward) state = next_state if done: - print('回合:{}/{}, 奖励:{}'.format(i_ep + 1, cfg.train_eps, ep_reward)) + print(f'Episode:{i_ep+1}/{cfg.train_eps}, Reward:{ep_reward:.2f}') break if i_ep > 0 and i_ep % cfg.batch_size == 0: agent.update(reward_pool,state_pool,action_pool) - state_pool = [] # 每个episode的state + state_pool = [] action_pool = [] reward_pool = [] rewards.append(ep_reward) @@ -99,8 +90,8 @@ def train(cfg,env,agent): 0.9*ma_rewards[-1]+0.1*ep_reward) else: ma_rewards.append(ep_reward) - print('完成训练!') - env.close() + print('Finish training!') + env.close() # close environment return rewards, ma_rewards diff --git a/codes/common/utils.py b/codes/common/utils.py index b47ef72..654b73c 100644 --- a/codes/common/utils.py +++ b/codes/common/utils.py @@ -5,7 +5,7 @@ Author: John Email: johnjim0816@gmail.com Date: 2021-03-12 16:02:24 LastEditor: John -LastEditTime: 2022-07-13 22:15:46 +LastEditTime: 2022-07-21 21:45:33 Discription: Environment: ''' @@ -14,6 +14,7 @@ import numpy as np from pathlib import Path import matplotlib.pyplot as plt import seaborn as sns +import json from matplotlib.font_manager import FontProperties # 导入字体模块 @@ -68,19 +69,19 @@ def plot_losses(losses, algo="DQN", save=True, path='./'): plt.savefig(path+"losses_curve") plt.show() -def save_results_1(dic, tag='train', path='./results'): +def save_results(dic, tag='train', path='./results'): ''' 保存奖励 ''' for key,value in dic.items(): np.save(path+'{}_{}.npy'.format(tag,key),value) print('Results saved!') -def save_results(rewards, ma_rewards, tag='train', path='./results'): - ''' 保存奖励 - ''' - np.save(path+'{}_rewards.npy'.format(tag), rewards) - np.save(path+'{}_ma_rewards.npy'.format(tag), ma_rewards) - print('Result saved!') +# def save_results(rewards, ma_rewards, tag='train', path='./results'): +# ''' 保存奖励 +# ''' +# np.save(path+'{}_rewards.npy'.format(tag), rewards) +# np.save(path+'{}_ma_rewards.npy'.format(tag), ma_rewards) +# print('Result saved!') def make_dir(*paths): @@ -101,11 +102,8 @@ def del_empty_dir(*paths): def save_args(args): # save parameters - argsDict = args.__dict__ - with open(args.result_path+'params.txt', 'w') as f: - f.writelines('------------------ start ------------------' + '\n') - for eachArg, value in argsDict.items(): - f.writelines(eachArg + ' : ' + str(value) + '\n') - f.writelines('------------------- end -------------------') + args_dict = vars(args) + with open(args.result_path+'params.json', 'w') as fp: + json.dump(args_dict, fp) print("Parameters saved!") \ No newline at end of file diff --git a/notebooks/A2C.ipynb b/notebooks/A2C.ipynb new file mode 100644 index 0000000..8966eac --- /dev/null +++ b/notebooks/A2C.ipynb @@ -0,0 +1,370 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import torch\n", + "import torch.optim as optim\n", + "import torch.nn as nn\n", + "import torch.nn.functional as F\n", + "from torch.distributions import Categorical\n", + "import numpy as np\n", + "from multiprocessing import Process, Pipe\n", + "import argparse\n", + "import gym" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 建立Actor和Critic网络" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "class ActorCritic(nn.Module):\n", + " ''' A2C网络模型,包含一个Actor和Critic\n", + " '''\n", + " def __init__(self, input_dim, output_dim, hidden_dim):\n", + " super(ActorCritic, self).__init__()\n", + " self.critic = nn.Sequential(\n", + " nn.Linear(input_dim, hidden_dim),\n", + " nn.ReLU(),\n", + " nn.Linear(hidden_dim, 1)\n", + " )\n", + " \n", + " self.actor = nn.Sequential(\n", + " nn.Linear(input_dim, hidden_dim),\n", + " nn.ReLU(),\n", + " nn.Linear(hidden_dim, output_dim),\n", + " nn.Softmax(dim=1),\n", + " )\n", + " \n", + " def forward(self, x):\n", + " value = self.critic(x)\n", + " probs = self.actor(x)\n", + " dist = Categorical(probs)\n", + " return dist, value" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "class A2C:\n", + " ''' A2C算法\n", + " '''\n", + " def __init__(self,n_states,n_actions,cfg) -> None:\n", + " self.gamma = cfg.gamma\n", + " self.device = cfg.device\n", + " self.model = ActorCritic(n_states, n_actions, cfg.hidden_size).to(self.device)\n", + " self.optimizer = optim.Adam(self.model.parameters())\n", + "\n", + " def compute_returns(self,next_value, rewards, masks):\n", + " R = next_value\n", + " returns = []\n", + " for step in reversed(range(len(rewards))):\n", + " R = rewards[step] + self.gamma * R * masks[step]\n", + " returns.insert(0, R)\n", + " return returns" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "def make_envs(env_name):\n", + " def _thunk():\n", + " env = gym.make(env_name)\n", + " env.seed(2)\n", + " return env\n", + " return _thunk\n", + "def test_env(env,model,vis=False):\n", + " state = env.reset()\n", + " if vis: env.render()\n", + " done = False\n", + " total_reward = 0\n", + " while not done:\n", + " state = torch.FloatTensor(state).unsqueeze(0).to(cfg.device)\n", + " dist, _ = model(state)\n", + " next_state, reward, done, _ = env.step(dist.sample().cpu().numpy()[0])\n", + " state = next_state\n", + " if vis: env.render()\n", + " total_reward += reward\n", + " return total_reward\n", + "\n", + "def compute_returns(next_value, rewards, masks, gamma=0.99):\n", + " R = next_value\n", + " returns = []\n", + " for step in reversed(range(len(rewards))):\n", + " R = rewards[step] + gamma * R * masks[step]\n", + " returns.insert(0, R)\n", + " return returns\n", + "\n", + "\n", + "def train(cfg,envs):\n", + " print('Start training!')\n", + " print(f'Env:{cfg.env_name}, Algorithm:{cfg.algo_name}, Device:{cfg.device}')\n", + " env = gym.make(cfg.env_name) # a single env\n", + " env.seed(10)\n", + " n_states = envs.observation_space.shape[0]\n", + " n_actions = envs.action_space.n\n", + " model = ActorCritic(n_states, n_actions, cfg.hidden_dim).to(cfg.device)\n", + " optimizer = optim.Adam(model.parameters())\n", + " step_idx = 0\n", + " test_rewards = []\n", + " test_ma_rewards = []\n", + " state = envs.reset()\n", + " while step_idx < cfg.max_steps:\n", + " log_probs = []\n", + " values = []\n", + " rewards = []\n", + " masks = []\n", + " entropy = 0\n", + " # rollout trajectory\n", + " for _ in range(cfg.n_steps):\n", + " state = torch.FloatTensor(state).to(cfg.device)\n", + " dist, value = model(state)\n", + " action = dist.sample()\n", + " next_state, reward, done, _ = envs.step(action.cpu().numpy())\n", + " log_prob = dist.log_prob(action)\n", + " entropy += dist.entropy().mean()\n", + " log_probs.append(log_prob)\n", + " values.append(value)\n", + " rewards.append(torch.FloatTensor(reward).unsqueeze(1).to(cfg.device))\n", + " masks.append(torch.FloatTensor(1 - done).unsqueeze(1).to(cfg.device))\n", + " state = next_state\n", + " step_idx += 1\n", + " if step_idx % 200 == 0:\n", + " test_reward = np.mean([test_env(env,model) for _ in range(10)])\n", + " print(f\"step_idx:{step_idx}, test_reward:{test_reward}\")\n", + " test_rewards.append(test_reward)\n", + " if test_ma_rewards:\n", + " test_ma_rewards.append(0.9*test_ma_rewards[-1]+0.1*test_reward)\n", + " else:\n", + " test_ma_rewards.append(test_reward) \n", + " # plot(step_idx, test_rewards) \n", + " next_state = torch.FloatTensor(next_state).to(cfg.device)\n", + " _, next_value = model(next_state)\n", + " returns = compute_returns(next_value, rewards, masks)\n", + " log_probs = torch.cat(log_probs)\n", + " returns = torch.cat(returns).detach()\n", + " values = torch.cat(values)\n", + " advantage = returns - values\n", + " actor_loss = -(log_probs * advantage.detach()).mean()\n", + " critic_loss = advantage.pow(2).mean()\n", + " loss = actor_loss + 0.5 * critic_loss - 0.001 * entropy\n", + " optimizer.zero_grad()\n", + " loss.backward()\n", + " optimizer.step()\n", + " print('Finish training!')\n", + " return test_rewards, test_ma_rewards" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "import seaborn as sns \n", + "def plot_rewards(rewards, ma_rewards, cfg, tag='train'):\n", + " sns.set()\n", + " plt.figure() # 创建一个图形实例,方便同时多画几个图\n", + " plt.title(\"learning curve on {} of {} for {}\".format(\n", + " cfg.device, cfg.algo_name, cfg.env_name))\n", + " plt.xlabel('epsiodes')\n", + " plt.plot(rewards, label='rewards')\n", + " plt.plot(ma_rewards, label='ma rewards')\n", + " plt.legend()\n", + " plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Start training!\n", + "Env:CartPole-v0, Algorithm:A2C, Device:cuda\n", + "step_idx:200, test_reward:18.6\n", + "step_idx:400, test_reward:19.7\n", + "step_idx:600, test_reward:24.2\n", + "step_idx:800, test_reward:19.5\n", + "step_idx:1000, test_reward:33.9\n", + "step_idx:1200, test_reward:36.1\n", + "step_idx:1400, test_reward:32.6\n", + "step_idx:1600, test_reward:36.3\n", + "step_idx:1800, test_reward:38.9\n", + "step_idx:2000, test_reward:60.8\n", + "step_idx:2200, test_reward:41.9\n", + "step_idx:2400, test_reward:42.2\n", + "step_idx:2600, test_reward:71.6\n", + "step_idx:2800, test_reward:123.6\n", + "step_idx:3000, test_reward:57.5\n", + "step_idx:3200, test_reward:155.4\n", + "step_idx:3400, test_reward:111.4\n", + "step_idx:3600, test_reward:133.8\n", + "step_idx:3800, test_reward:133.8\n", + "step_idx:4000, test_reward:114.3\n", + "step_idx:4200, test_reward:165.5\n", + "step_idx:4400, test_reward:119.4\n", + "step_idx:4600, test_reward:173.4\n", + "step_idx:4800, test_reward:115.4\n", + "step_idx:5000, test_reward:159.7\n", + "step_idx:5200, test_reward:178.1\n", + "step_idx:5400, test_reward:137.8\n", + "step_idx:5600, test_reward:146.0\n", + "step_idx:5800, test_reward:187.4\n", + "step_idx:6000, test_reward:200.0\n", + "step_idx:6200, test_reward:169.2\n", + "step_idx:6400, test_reward:167.8\n", + "step_idx:6600, test_reward:184.3\n", + "step_idx:6800, test_reward:162.3\n", + "step_idx:7000, test_reward:125.4\n", + "step_idx:7200, test_reward:150.6\n", + "step_idx:7400, test_reward:152.6\n", + "step_idx:7600, test_reward:122.5\n", + "step_idx:7800, test_reward:136.3\n", + "step_idx:8000, test_reward:131.4\n", + "step_idx:8200, test_reward:174.6\n", + "step_idx:8400, test_reward:91.7\n", + "step_idx:8600, test_reward:170.1\n", + "step_idx:8800, test_reward:166.0\n", + "step_idx:9000, test_reward:150.2\n", + "step_idx:9200, test_reward:104.6\n", + "step_idx:9400, test_reward:147.2\n", + "step_idx:9600, test_reward:111.8\n", + "step_idx:9800, test_reward:118.7\n", + "step_idx:10000, test_reward:102.6\n", + "step_idx:10200, test_reward:99.0\n", + "step_idx:10400, test_reward:64.6\n", + "step_idx:10600, test_reward:133.7\n", + "step_idx:10800, test_reward:119.7\n", + "step_idx:11000, test_reward:112.6\n", + "step_idx:11200, test_reward:116.1\n", + "step_idx:11400, test_reward:116.3\n", + "step_idx:11600, test_reward:116.2\n", + "step_idx:11800, test_reward:115.3\n", + "step_idx:12000, test_reward:109.7\n", + "step_idx:12200, test_reward:110.3\n", + "step_idx:12400, test_reward:131.4\n", + "step_idx:12600, test_reward:128.3\n", + "step_idx:12800, test_reward:128.8\n", + "step_idx:13000, test_reward:119.8\n", + "step_idx:13200, test_reward:108.6\n", + "step_idx:13400, test_reward:128.4\n", + "step_idx:13600, test_reward:138.2\n", + "step_idx:13800, test_reward:119.1\n", + "step_idx:14000, test_reward:140.7\n", + "step_idx:14200, test_reward:145.3\n", + "step_idx:14400, test_reward:154.1\n", + "step_idx:14600, test_reward:165.2\n", + "step_idx:14800, test_reward:138.2\n", + "step_idx:15000, test_reward:143.5\n", + "step_idx:15200, test_reward:125.4\n", + "step_idx:15400, test_reward:137.1\n", + "step_idx:15600, test_reward:150.1\n", + "step_idx:15800, test_reward:132.9\n", + "step_idx:16000, test_reward:140.4\n", + "step_idx:16200, test_reward:141.3\n", + "step_idx:16400, test_reward:135.5\n", + "step_idx:16600, test_reward:135.5\n", + "step_idx:16800, test_reward:125.6\n", + "step_idx:17000, test_reward:126.8\n", + "step_idx:17200, test_reward:124.7\n", + "step_idx:17400, test_reward:129.6\n", + "step_idx:17600, test_reward:114.3\n", + "step_idx:17800, test_reward:57.3\n", + "step_idx:18000, test_reward:164.7\n", + "step_idx:18200, test_reward:165.8\n", + "step_idx:18400, test_reward:196.7\n", + "step_idx:18600, test_reward:198.8\n", + "step_idx:18800, test_reward:200.0\n", + "step_idx:19000, test_reward:199.6\n", + "step_idx:19200, test_reward:189.5\n", + "step_idx:19400, test_reward:177.9\n", + "step_idx:19600, test_reward:159.3\n", + "step_idx:19800, test_reward:127.7\n", + "step_idx:20000, test_reward:143.6\n", + "Finish training!\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXkAAAEXCAYAAABI/TQXAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAB+YElEQVR4nO2dd3hUVfrHP3f6THpCElpAei8KSFMQlCYg2EUF26rrWlb3t67oYt3FxbK6rmV1XdfdVRQQBV3XDqJSBEWq9BJCSEjv0+ee3x+Tmcwkk2TSyeR8nsdHMnPn3nPu3Hnve7/nLYoQQiCRSCSSiETT1gOQSCQSScshjbxEIpFEMNLISyQSSQQjjbxEIpFEMNLISyQSSQQjjbxEIpFEMNLIN5GtW7cyZ86cVjnWCy+8wNq1a1vlWB2FPXv2MHXq1Gbb38aNG5kyZQqXX345dru9xvtvvfUWAwYMYOfOnUGvFxYWctdddzF37lwuvvhinnrqKVRV9b//9ddfs3DhQubPn8/s2bO59957yc7ODjmGDz74gAsuuIBbbrmlSXPJyclh8eLFzJ07l0suuYQrr7ySr776qsH7KSsrY9GiRf6/p06dyowZM5g3b55/Pk8++WTQfEPx4osv8sQTTzT4+LWxa9cuLrvsMmbNmsUNN9xAbm5us+37jEJImsT3338vZs+e3dbDkDSS3bt3iylTpjTb/hYvXixefvnlWt+/+OKLxf/93/+Je++9N+j1//u//xPPPfecEEIIu90urr32WvHee+8JIYT46KOPxKxZs0R6eroQQghVVcWrr74qLrroIuFwOGocY+HChWLt2rVNmkdBQYG44IILxJo1a4SqqkIIIfbv3y/GjRsnNm7c2KB9nTx5UowcOdL/95QpU8Tu3bv9fzscDnHVVVeJt956q879/PWvfxWPP/54g45dGw6HQ0yaNEn8+OOPQgghli9fLn7xi180y77PNHRtfZOJJJxOJ88++yw//PADHo+HwYMHs2TJEqKjo/n666957bXXcDqdFBYWMn/+fO699162bt3K0qVLsVgsWK1W7r//fl5++WXS0tI4fPgwTqeTRx55hHHjxrF48WL69evHLbfcwrBhw7jtttvYtGkTubm5LFq0iBtvvBGPx8PTTz/N+vXriYmJYfjw4Rw9epS33nqrxnhfe+011qxZg06no2fPnixbtowvv/ySzz//nNdeew3weoW+vxcvXkxxcTEnT55k4sSJrF69ms8//5zk5GQArrrqKu68807Gjx9f63kIxOVysWzZMrZs2YJWq2X48OE8+OCDREdHM3XqVC699FK2bNlCdnY2s2bN4ne/+12NORw/fpxHHnmEwsJCNBoNd9xxBxdffDFTp07lhRdeYNiwYQBBf7/zzjv8+9//Jjo6mv79+/v3lZ+fzyOPPEJBQQF5eXl069aNv/zlLyQlJYU17hUrVrBu3TqMRiNlZWU88MADQZ/bunUrJSUl3H///UybNo3s7Gy6dOkCwLRp0zjnnHMAMBqN9OvXj6ysLACef/55/vCHP9CzZ08AFEXhtttuo2vXrjidTgwGg/8YTz75JHv27CEzM5OioiIuv/xyHn/8cQ4cOICiKJx//vn85je/QafTMXToUC688EIOHDjAs88+6z9XAO+88w7nnHMO8+fP9782cOBAXnzxRWJiYgBYvXo1K1euxOVyUVJSwq233sq1117LBx98wOrVq7HZbP7v3G63M2/ePD744IMa36HBYGDUqFEcO3YMgK+++oqXXnoJj8dDdHQ0Dz74IMOHDw/6TE5ODk888QTZ2dm4XC5mz57NL3/5yxr7fu655ygvL+eRRx4B4Ntvv+XFF19k8eLFREdHM2rUKACuuOIKnnzySYqKikhISKixn3ZNW99l2juBnvyLL74oli1b5vd8/vznP4tHH31UqKoqrr/+enH8+HEhhBCnT58WgwYNEgUFBeL7778XAwcOFJmZmf79DRo0SOzbt08IIcQbb7whrrvuOiGEEA888ID4xz/+IYQQon///n7PZ8+ePWLo0KHCbreLd999V1x33XXCbrcLh8Mhbr75ZnH99dfXGPdXX30lpk+fLoqLi4UQQjz55JPilVdeEe+//7647bbb/NsF/v3AAw+IG264wf/e7373O/94jhw5Ii644ALh8XhqPQ/VeeGFF8Rdd90lnE6n8Hg8YvHixeLhhx8WQni9vWXLlvnP17Bhw0RGRkaNfcyfP1+8/fbbQgghsrKyxIUXXijKyspqeIu+v/ft2yfGjx8vcnNzhRBCPPzww35P/l//+pd47bXXhBBeb/kXv/iFeOONNxo07sDvqDq//vWv/XO69dZbxdNPPx1yu59//lmMGjVK7Nu3TxQWFor+/fsLq9UacttQXH/99eLTTz8VQni/oz/84Q9CVVX/9eCbY//+/cWaNWtC7uP222/3n9dQlJeXi6uuukoUFhYKIYTYsWOH31t///33xZgxY0RZWZkQon5P/vTp02LmzJnis88+E0eOHBETJkzwf9ebN28WEydOFGVlZUGe/MKFC8W6deuEEN4nn4ULF4r//e9/NcaZkZEhxo4d63/i+fWvfy1WrVolPv74Y3HzzTcHbXv++eeL/fv31zrn9or05JuRDRs2UFZWxubNmwGvx5eUlISiKLz66qts2LCBjz/+mKNHjyKEwGazAdClSxe6devm30/Xrl0ZNGgQAIMHD2bNmjUhj3fhhRcCMGTIEJxOJ1arlW+++YZ58+ZhNBoBuPrqq0N68Vu2bGHmzJnExcUB8OCDDwKE9LQC8Xk+AFdeeSWPP/44t9xyC++//z6XXXYZGo2m1vNQnW+//Zb77rsPvV4PwMKFC7nzzjtrzC81NZWkpCRKSkpIS0vzv19cXMyBAwe48sorAe95rE8z3rJlCxMnTvQ/fVx99dVs3LgRgBtuuIEff/yRN998k/T0dA4fPsyIESMaPO5Q5OXl8dVXX/H+++8DMH/+fB577DHuvPNOLBaLf7vvvvuO+++/nyVLljBo0CBKSkoA6tWra+Pbb7/l3XffRVEUDAYD11xzDf/+97+57bbbABg9enTIzymKgqij4klUVBSvvvoq33zzDenp6Rw4cACr1ep/f8CAATWe3AL57W9/i8lkQlVV9Ho9V155JTNmzGD58uWMGzfO/z2PHz+exMRE9u7d6/+s1Wrlhx9+oKSkhBdeeMH/2oEDB7j44ouDjpOWlsbAgQNZv34948ePZ8uWLSxdupT169eHHJdWq611zO0VaeSbEVVVeeihh5g8eTIAFRUVOBwOrFYrl156KRdddBGjR4/m8ssv56uvvvL/iAJ/5AAmk8n/77p+bD5DrigKAEIIdLrgr1SjCb22rtVq/Z8DKC0tpbS0tMbxXC5X0OcCxzp69Gjcbje7d+/m448/ZsWKFXWeh+pUN1yqqgYdzzc/3xyrnwffXAPncezYMbp27QoQtL3T6Qy5n8Af9TPPPMPu3bu5/PLLGTt2LG63O+S5r2/coXjvvfcAuOOOO/yfKS8vZ82aNVx33XUAvPnmm/z973/nueeeY8KECQDExcVx1llnsWvXLv9rPn79619zxx13MHDgwFqPG2qsbrfb/3f1a8/HyJEj2blzJ9dff33Q6ytWrMBmszFr1iyuvvpqrrrqKkaNGsXMmTP5+uuv692vj+rykI9Q51sIETRmVVURQrBixQrMZjPgXbg2Go28++67/utw6NChLF26lCuvvJK1a9dSUFDAtGnTiIqKokuXLuTl5fn36XK5KCoqIjU1tc5xt0dkdE0zct5557F8+XKcTieqqvLwww/z3HPPceLECcrLy7n33nuZOnUq27Zt82/T3EyePJmPPvoIp9OJ2+2u9SlgwoQJfPnll5SXlwPeyIV//etfJCYmcvjwYRwOB263O+iHG4orr7ySP/zhDwwYMMBvXGs7D9U5//zzWbFiBS6XC1VVWb58ORMnTgx7rtHR0QwZMsQfcZSdnc2CBQsoKysL8v527tzp/0FPmDCBTZs2cfr0aYCg87Nx40ZuuOEG5s+fT1JSEps3b8bj8TR53B6Ph1WrVvH444+zfv161q9fz4YNG7j99tv5z3/+gxCCN998k+XLl7Nq1aoaxvyuu+5i6dKlnDhxwr+/V155hQMHDtC7d+86z5HvuxBC4HQ6Q+4/FFdffTXbtm3jo48+8hvevXv38te//pX+/fuzd+9eEhMT+dWvfsX555/vv05CnS+dTofH46nzycDHuHHj2LRpEydPngTwr8kEPlFFR0czcuRI3nzzTcDroCxYsIB169axYMECPvzwQz788EOWLl0KeNc7fv75Z1atWsVVV10FwIgRIyguLuann34C4P3332fkyJHExsbWO8b2hvTkm5Ff/epXPPXUU1x66aV4PB4GDRrE4sWLsVgsXHDBBcyaNYvY2Fh69OhB3759OXHiRNCiWXNw2WWXcfz4cebPn4/FYqF79+5+byeQyZMnc+TIERYsWABA3759+cMf/oDJZGLMmDHMmjWL5ORkxo4dy8GDB2s93vz583nuueeCjHht56E6d9xxB0899RTz58/H7XYzfPhwHn744QbN989//jOPP/44b731FoqisHTpUpKTk/ntb3/LY489xsqVKxkyZAhDhgwBvDLC/fffzw033EBUVFTQgt6dd97J008/zSuvvIJWq+Wcc84hIyOjyeP++uuvUVWVuXPnBr1+44038p///Id169bxwgsvEBMTw1133eV/f+bMmdxxxx3MnTsXIQS/+c1vcLvdOBwOhgwZwr///e96r58lS5bwxz/+kblz5+JyuTj//PNDLlBWJz4+nrfeeotnnnmG1157DY1Gg9lsZunSpUycOBGbzcbq1auZOXMmZrOZ4cOHk5iY6L8RBZKcnMzgwYOZNWsW7777bp3H7du3L48++ih33XUXHo8Hk8nEq6++6l/s9fHss8/yhz/8gblz5+J0OpkzZw6XXHJJyH0aDAYuvvhiNm/e7P++9Xo9L730Ek888QQ2m434+Hieeuqpes9Le0QR4dxeJe2GjRs3UlBQwLx58wD44x//iNFo5P7772/jkUkkkrZAGvkIw5fAUlBQgMfjYeDAgTz22GM1PCGJRNIxkEZeIpFIIhi58CqRSCQRjDTyEolEEsFIIy+RSCQRjDTyEolEEsGccXHyRUUVqGrD14KTkqIpKChvgRGd2XTEeXfEOUPHnHdHnDM0bN4ajUJCQlSt759xRl5VRaOMvO+zHZGOOO+OOGfomPPuiHOG5pu3lGskEokkgpFGXiKRSCIYaeQlEokkggnLyL/00kvMnj2b2bNn8/TTTwOwefNm5s6dy/Tp03n++ef92+7fv5/LL7+cGTNm8Pvf/z6oRKhEIpFIWpd6jfzmzZvZuHEja9asYe3atfz88898/PHHPPTQQ7zyyit88skn7N27l2+++QaA+++/n4cffpjPP/8cIQSrVq1q8UlIJBKJJDT1Gvnk5GQWL16MwWBAr9fTp08f0tPT6dmzJ2lpaeh0OubOnctnn33GqVOnsNvtjBw5EvCWvf3ss89aeg4SiUQiqYV6Qyj79evn/3d6ejqffPIJCxcu9LdPA0hJSSEnJ4fc3Nyg15OTk8nJyWnmIUvagoycMl5YvZsli0aTEGOs/wMSSYTi9qj8eDCX9dtPcSKnjE5xJlITLPTrHsfMsT2COpWdCYQdJ3/48GFuv/12HnjgAXQ6HcePHw96v7Y2dQ2dcFJS7X0h6yM5uWOW022NeX+1I4uiMgcVLpX+Z8B5lt91x+FMmbMQgs+2pPPuFwcpKnPQpVMUM8b1pKDETnpWKTuP5DN1bE+6N9N4m2veYRn57du3c8899/DQQw8xe/Zstm3bRn5+vv/93NxcUlJSSE1NDXo9Ly+PlJSUBg2ooKC8UUkAyckx5OWVNfhz7Z3Wmvfuw7kAZGQV0z2xZqep1kR+1x2HM2XOVruLf316gB8P5jEgLZ4bZg5kaO9ENJVObF6xjQde3cK3P55k2pi0evZWPw2Zt0aj1Okc16vJZ2dnc+edd/Lss88ye/ZswNsf8fjx45w4cQKPx8PHH3/MpEmT6NatG0ajke3btwOwdu1aJk2aFNZAJWcuqhAcO1UKQGlF3Q2rJZJI42RuOY+9+QM/Hcrnyil9uP/asxneJ8lv4AGS482kJlrYe7ywDUcamno9+TfeeAOHw8GyZcv8r11zzTUsW7aMu+++G4fDweTJk5k5cybg7b24ZMkSKioqGDx4MIsWLWq50UtahewCK1aHNxS2pMLRxqORSFqXNd8ew+70sPj6c+jbLa7W7Yb2SuS7XVm43B70Om0rjrBu6jXyS5YsYcmSJSHf++ijj2q8NnDgQFavXt30kUnOGI6eKgFAp1UorXC28WgkktalqMxB766xdRp4gGG9E1m3PZNDJ0sY0iuxlUZXPzLjVVIvR0+VEGXS0SM1Rhp5SYej1Ook1mKod7sBaQnotAp7jhW0wqjC54yrQik58zhyqoQ+3eLQahTyim1tPRyJpNVQhaC0wklcdP1G3mjQ0j8tnp/PMF1eevKSOqmwu8gusNKnayxxUQbpyUs6FFa7G48qwvLkAYb2SuJUfgWFpfYWHln4SCMvqZNjWd6omr7d4oiNMlBmdeFR1TYelUTSOpRUOjWxUWEa+d5eLf5MirKRRl5SJ0cyS1AU6FXpyQug3CrDKCUdg9JybzRZXJhGvlunKBJijOw9g3R5aeQldXI0q4TuydGYDDq/N1MiJRtJB6HE2jBPXlEUhvRKZF960RnzxCuNvKRWVFVwLKvUHzoWF+WtWSN1eUlHwZf8F66RBxhyViJWh5uMnDOjN6008pJaycqvwO700KdbLACxUXpAevKSjkNJhQOtRiHKFH4g4oAe8QAczChumUE1EGnkJbVyIsdbO6NXF5+R93oz0pOXdBRKK5zERhkaVGgxPtpIaoKZQyeLW25gDUAaeUmtlFUusPpkGpNBh1GvlZ68pMNQUuEMe9E1kP5p8Rw6WYwaojJvayONvKRWrA4XGkXBbKyqwxEbpZeevKTD4PPkG0r/tHisDjen8ipaYFQNQxp5Sa1U2NxYTLqgR9W4KGOtnrzL7eHpd346Yx5TJZKm0lgjPyAtHuCM+C1IIy+plQq7q8aCU2wdWa+n8is4kFF8RiWCSCSNxVvSwNUouaZTvJmkWCMHM4paYGQNQxp5Sa1Y7W4sJn3Qa3FRhlo9ed+jaX6JrG8jaf9U2FyoQjTKk4cqXT5Ux7zWRBp5Sa3U5smX21y4PTUTPfxGvvjMqdshkTQWnzPTGE8evEa+1OridKG1OYfVYKSRl9RKhd1NlLmmJw9VkTeBZOZ7kz/ypCcviQBKm8HIAxxsY11eGnlJrVTYXFhCePIQOlY+K9/ryZeUO3G5PS0/QImkBSltYHGy6nROtBAbZWjzxVdp5DsoHlWt0xCrQmB1uImqpslX1a8JbgNotbspLHXQrVMUAPklUrKRtG+aKtcoikL/tHgOZrStLh+2kS8vL2fOnDlkZmbyzTffMG/ePP9/48aN4/bbbwfgpZdeYsqUKf73li9f3mKDlzSetd8d54//2V7r+3aHGyGoocnH1VKkzOfFj+jbCZBGXtL+Ka1wotMqmI2N7600IC2eojIHBW34ewhr9Lt27WLJkiWkp6cDMHnyZCZPngxAXl4eCxYs4MEHHwRg7969PPfcc5x99tktM2JJs3D0VEmdC0IVdm/j7to8+epyjU+PH9E3iU++P0G+7CAlaeeUNKKkQXV6psYAkJlfQad4c3MNrUGE5cmvWrWKRx99lJSUlBrvPf3001xzzTWcddZZgNfIv/7668ydO5cnnngCh8NR4zOStie7wIrLreJ0hZZsKuzehdXqnrxRr8VkqFnaICuvAqNBS5+ucei0CnnSk5e0c0obWdIgkNREr2E/XdB2ETZhGfmlS5cyevToGq+np6ezbds2Fi1aBEBFRQWDBg3igQceYM2aNZSWlvLKK68074glTcbmcPuNtM9jr47fk68WXQOhE6JO5VfQrVMUGo1CUqypxeQah9PDidOlLbJviSSQ0orwGnjXRYzFQJRJR05R2xn5JjXyXrlyJddeey0Gg/dEREVF8frrr/vfv/nmm3nooYe47777wt5nUlJ0o8eTnBzT6M+2Zxo670MBWXgGsyHk57WnvIa0e5e4Gu93ijdjc6pBr2cXWBkzOJXk5Bi6JkdTXOFske9jzYYjvP3ZAVYuvRidtuPFDXTEa7yt5lxmczHgrMQmHz8tNYaCMkeD99Nc826SkV+3bh1vvPGG/++srCw2b97MFVdcAYAQAp2uYYcoKChHVRu+Ep2cHENeXlmDP9feCTXvv3/0M53izVw2qXfIzxw4mu//d2ZWMVG6mprj6VzvPh02Z439mw1asvIr/K+XWp0UlztIijaQl1dGnEXP4ZPFLfJ9nMwuxenykJVd0qQFsfZIR7zG22rOqhCUlDsx6pQmHz8p1sjeY4UN2k9D5q3RKHU6x412hQoLC7Hb7aSlpflfM5lMPPPMM5w8eRIhBMuXL2fatGmNPYSkkRw5VUJ6du2SRnZhVWW8clttck1oTR5qyjVZlZmu3ZK9F1qneDPlNhc2R+h9NwWrwzsup/vMaK0miUzKfSUNmijXgDdevqTC2SK/h3BotJHPzMykc+fOQa8lJibyxBNPcMcddzBz5kyEENx0001NHqSkYdidHuy1LKiCdxHIoPd+9T5jXp0KuxudVoNBr63xXlyUgQq7G1eloT1VGT7ZtTJGvlOcCWiZMEpr5VqBq475SSRNpbS8aYlQgXRO9P4u2qq8QYOed9evX+//9/Dhw1m1alWNbWbMmMGMGTOaPjJJo7E7PdgddRj5Qiu9Osdy8GRxrUbeancRZQ59ecT6Sxs4SYw1cSqvnCiTjvho7+ud4rwRBfklNtJSGr/GEgrfgnB1T97p8vDz8UJG9OuEpgkhbxIJVDXwbmp0DUDngAgbX5e11qTjrVxFOG6Pituj4nCFfjRUVUFOkY2enWPQahQqapNrbDWzXX34Lnyfp55ZGVnjiyfuFF/pybdAoTJr5U3JVc3I7zlWwIsf7OHLH042+zElHY/m9ORTEiwoStt58tLIRxh2pyfo/9UpKLXjcqt0SbIQZdbXIdfUrEDpo3tyNDqthudW7mTFusOcyqvw6/EAMWY9Rr22RQqVWSt1zepG3vf6+98c41ReebMfV9KxqCppYGzyvvQ6DZ3iTNLIS5oHe6Wxc9Ri5H0XWudEC1EmHRW22jX52jz55Hgzf7jlXMYMTOGrHzOxOdx+PR68NTs6xZtaxJOvkmuC5+cz+lqtwusf7wtZClkiCZdSqxOdVhPU+rIpdE6MkkZe0jzYKo27062GDEX1Zd51SYoiyqSvNRnKWocnD5CaaOGWOYNZettYLj2/F+OGpAa9nxxnbvbmIW6P6r95VffknS7v39dP609GTjkfbUpv1mNLOhYl5U7iovRNKmkQSOdECzmF1jZp7C2NfDvm6KkSSsqDy0bYne6Af9f05k8XWrEYdcRY9F5Pvha5pjxEV6hQpCZYmDuxVw2vv1OcN+u1OavvBYag1TDylZ792MGpTBzamf9tSW/zZg2RTlsYrJZGCMHRUyWkny4lthmkGh+dkyw43SpFpa1f5kUa+XaKEIJnVuzgw2+PBr1uC4iqcYQIM8wuqKBzkgVFUbyafIiFV5/HXJcnXx+d4kzYnZ5anxQagzVgX6HkGq1GQafVMHvCWQgBhzOLm+3YkmBKrU7u+ct37DqSX//G7YQfDuTyyD+3sfSt7RSU2ms8nTaFzgmVETZtUN6gY6UMRhBOt4rTpVJSHlxDJtiTdwPB3sjpQiuDz0oEqJRranry1jrq1oSLr+JeXrGN6CbsJ5DAG0YouUav8/osKfFmDDoNmbkVSFqGAyeKsDrcnMgp85eXbu+s/e4YTpeHG2YO4NxBqc2aUd05qTJWvsDKkMrfX2shPfl2im+BtbqRDpRoqss1Noeb4nInXZIsAESZddidnhqLlL59Vu8K1RBaIiHKGjDX6nHyLrcHQ6WR12gUuiVHkSmjbFqMAye89Y+KyyKnyqzTpTKgRwKTR3Zr9pIZ8dEGjAZtm0iI0si3U3wLrNWjY+yO2jV5XyW8zomVRr5SR7dWk1RqqyXfEJIrPfnmrCtvrVOTV9HrqiIhuiVHczK3vE078kQy+yuNfFEEGXm3qrZY0TtFUeicYJFGXhI+vkXIGkY+wLBXD6PMLqhm5CszWqs/DVjrqFsTLmajDoNeQ6m1Zi/YxhIo11Svg+90efylGgDSkqMpt7lC9qKVNI2iMgc5Rd6bd1F5BBl5t4q+BSubdk6ytEldeanJt1N8Hru1Trkm2EM/XWBFUbwZeADRlZ569cXXumrJN4QYs55ya+joncbgm6uigMtT05M3BHjy3SvLKZzMKycuuvmiJCRVUk3PzjGR5cl7BFpt85bE8BRn4dr3NQjBRFcFMU4HDusIjJao+j/cTEgj306pkmvc1V4PkGtcNeWaTnEm/wKlz4iXV7tR+J4OmqLJ+/ZfVkuyVWOoKpqmweWqrsmr6AM8+e7J3h9RZm4FQ3slNdsYJLA/owiLUcfw3kl8vDkdt6flZI7WxO2pWrxvKkIIXD+vw7F1pfcFnYE0l5OeFhcVX9gxXPJ/KJrmSbSqD2nk2yk+uaY8hFwTa9FTanXVKFJWbnMF1eLwyTHVJR9/dE0TjXyMWV9rRm1jsNrdRJl0KBqlZoEytwdjQMXMGIuBuGiDXHxtAQ6cKGJAj3gSY40IvIlDSZUL7e0VVQg8qmiWm5VqL8O+/jU8mXvRpg3DNPkWNJZ4jmaVsH7lu1yT+z2OjW9hPP+GZku2qov2f/vtoPhkGbdHxRUQM253uP1JHNXj5K12NxZjlQTj8+Srx7KX212YDFq0mqZdHs3tyVvtLiwmHQa9NmQIZaBcA15dPjNXGvnmJL/ERn6JnYE9EkiI8V5nkaDLeyrlP10T5RrhcWP/4kU82QcwTlyIeeZv0FjiAYizGNji6M/pLpNxHdiAc9f/mjrssJBGvp0SmP0ZGB1jq0xiMug0NRZerXZ3kARjNupQCO3JN9WLB4gxG5rVk6+oHL9Rrwm6sUGlJq8Pvpy7p0STVVCBR5V1bJqLAyeKARjYM4H4yrWOSAijdLm9UVhN9eQdW97Bc/oQpsm3YBhyYZCn7nuKPpg4GV2fsTi3rcb29et4ik416Zj1IY18G+L2qPx8vLBRnw3U3q2O4AQok0GLyaCtsfBqdQQbeY2iYAlR2qDC5mpS+KSPKLOOCru72Yys1eEtmqbXaUPGyVfXU9OSo3F7BKcLm78aZkflQEYR0WY93ZKjiI8gT96t+jz5xptE5/4NuPatxzDiYvR9x9d436D3/i5LrW5MF/wC/bAZuI//gPW932P7/AWEo2WS96SRb0N2HSngzyt3ktOI2NnA8gVBRt7hwWTUYTRogxZehRCVck2whx6qSFlFtZtBY4mpbJ3WXKUNfHKNMUy5ppt/8VVKNs2BEIIDGUUM7BGPRlGIMevRaZWI8OTdlddTYxdePTlHcGx6C23aMAxjrqh1u9goA6VWJ4pWj2n8AqKu/TOGc+bhKc5CLc5u1LHrQxr5NsQnuTQmljww6clmD/bkzQYtJoMuaOHV7vSgClHDeHu97RCefDOUIvCVM2iuMErfTcqg19Qw8i53zciILklRaDWKXHxtJnKLbBSWOhjYMwHwJvjERxsjw5NvgiYvVBX7xv+gmOMwT/0lSh1rWdX7I2tMMRhHX0r01U+hTe3b8IGHgTTybYgv1rsxnq7N4fY/WlqrZbmaDF5PPnDh1XdDqS7DRJlqFilrLk3eb+Tr0OVzCq0UltZf+kD1PYmY9Bj02qACZUKIGslQ4PXKOidZpCffTOw47C1GNqx3VUhqfLQxIjx5l6fxmrz78CbUggyMY69CMdYd/x5nMVDajLkj4RD2jMrLy5kzZw6ZmZkAPPjgg0yfPp158+Yxb948vvzySwA2b97M3LlzmT59Os8//3zLjDpC8D0i2hpj5J0eEmO9mqjPyHtUFadbxWTUYtIHa/K+xdkack217lBCiMquUM3oyYcw8qoq+O+m4/z+9a385/OD9e7L7vAgoHJROViucXsEAmrINeDtYiU9+eZh+6FceqRG+0tWAMTHGCkqb/9ZxVXRNQ0z8sJlx7FtNZqUPuj6jK13+5hqnnxrEJa7tmvXLpYsWUJ6err/tb179/L222+TkpLif81ut/PQQw/x1ltv0aVLF26//Xa++eYbJk+e3OwDjwSqPPmG39ntDjeJMUZyi2z+m4QvrNJk0GEyaCkI8JB9NwJzdbmmWncop1vF7akp6zSG2ox8frGN1z/ex+HMEgx6DQVhFDGzBhRNqy7X+CJtDCH01O7JUWzdl1Op5zdPNcyOSFGZg6OnSrn0/F5BrydEG9lztAAhRKvEfLcUrkYaeefO/yFsJZin3x3W/GMtesptrlZNIAvrKKtWreLRRx/1G3Sr1UpWVhYPP/wwc+fO5a9//SuqqrJ792569uxJWloaOp2OuXPn8tlnn7XoBNozPk8+UG4JF5vTTXy0EY1G8X/ep8GbDFrvwmtACKW/smSIhVer3e1vAOEz+M2iyVtCG/lX1u7lZG45t84dzIShXfz9NOuiwv8k4pNrqoy87996fU1PPq2yvEFmniw73BR2HM4D4JwBKUGvJ8QYcbg8QYEA4bDrSD4/HcprtvE1Fd9vsSGavFpegHP3Z+j6jAtbT4+rDKMsa0XJJix3benSpUF/FxQUMG7cOJ544gksFgu33347q1evxmKxkJyc7N8uJSWFnJycBg0oKSm6/o1qITk5ptGfbQv0lQZXKJoGj93hUkmINxNl0iMUheTkGKyVsb6pydHklzlwugv8+9VV1htJ6xpPckA/1tTkaK8MEm0i2mKgonIfXVJimuV8GvRaPCj+fQkhyMqv4OKJvbjkgn68+/kBynecIj4hqs7IhqzKfrHdOseSkV+B26P69+mpXOhKSrDUGHP/ynuBO2AM7Zm2msOe44V0S45mxMDUII81rWscABqDrkFj++DNH1BVwYyJvevdtjXmnFkZZtspKTrs4+V+/x8UoOusG9HFhfeZ7l2850sbxvlqrnk36pk8LS2Nl19+2f/3woULWbt2LTNnzqyxbUMf4QoKykP2Jq2P5OQY8vLKGvy5tqSkUk4pKLI2eOxWuwtUlSizjsJiG3l5ZWTllALgtLsQHhWb3U1ubimKonC6Upe2Wx3k5VV5waJS6jiRWURKgoXMrGIA3E53s5zPKJOOvIIK/77KbS7vuoFOQ15eGbrKy+PYiQISY2tPjc/2z82JQafB6fL495ntm5vNWWPMDqt3UTAnr6zdXR/VaatrvNzmYs+RAmaN60F+fvD6hk54r6WjGYWYwlQfrHY3mTllKIpC9umSOmWL1ppzQaH3Sa+i3B7W8dSS01Ts2YB+6DSKnCYId4yeyt/bqWJi62gS3pB5azRKnc5xo0ShgwcP8vnnn/v/FkKg0+lITU0lP7+qHVhubm6QZi8Jxhe21VC5xlWpm5sNOqLM+iq5pnKh1ZcMpQpRdYxKuaN69/nqpQ18/49uJv06xqwPkmt8kTSJlYk0vsfX+iQbv1xTGSfv9gi/M+DT5w0h5Bpf84dQ53jXkfw6I38kXnYezkcVglEDkmu850uIakiEzfHTpQi8EVONyRFpCdwN1OQdP30EGh2GERc36Di+rNfWXHxtlJEXQvDkk09SUlKCy+Vi5cqVTJs2jREjRnD8+HFOnDiBx+Ph448/ZtKkSc095ojBZ5yqlwuuD184pNmo82rq1TR5s0HnL9bl0+WtdnfIejRV5Ya9Y2iOrlAAwl6O44f36afPocxWdUEXVjYy9nntsdHhGfmqoml6vzH3nT9fbflQC696nQadVqlh5G0ON39dvZtvdrZsSnkksP1gLkmxRnqm1pQPfKUNGlJy+FhWqf/fWW1QXz0UrgbEyavF2biPbEE/ZCoaS1yDjhNbmSDYnH0W6qNRv+SBAwdy2223sWDBAtxuN9OnT2fOnDkALFu2jLvvvhuHw8HkyZNDSjgSL9W97HDxlTQwG7VEmfUUVHZfsgV58t6v1u70EGMBq8MV0nD7Gof4yg37usk3ti+rEAL30e9xbHkXYStlLrCRUQj1HBSNhsIyOwoqCc4sHD9toFPGXi63KJSV9QFq7xVqdbjQKAomg9ZfUtjp9mA0VIVTBoZQCrcTtTATT0EG86J2UFYRG7S/CpvLW0FRNhWpE5vDzc/phUw5u3tI6dWo12Ix6ihuQELUsVMldIozUVBiJyv/zFgQ9zQgTt7x04eg1TfYiwfvb1Ov07SqJ98gI79+/Xr/v6+77jquu+66GtuMHz+ejz76qOkj6wD4EjAamgwV6LGH8uRNRm8IJVR1hwpV0gCqkqN8CVE/Hcqjd9fYRvW4FG4nti9fwnNyN5rk3pin3c2BdR9xfsV2bP97Cm2XAfQ+sIc/JWSi+8KFE9DGd2WSKYvSvX9H7f9bNFEJIfftK07m2r+B3ns3YuBcv3F3uIJT0p0/r8Px/bvg8c5pkg5O5ZUjxDl+Q2WtpVSzJJh12zNxewRjBtUuuybEGMP25IUQHM0qZWTfThw6WXzGGPlwQyg9RVm4j2zFMGIWGnNsnduGQlEUYi2tGysvM17bkMaGUPrkGpOxdk3eaKgp14Qy8j7vvsLuIruggozccs4dlNqI2YBz58d4Tu7GOG4BlnlL0Hbux+Eel7K8fCKevOM4d/wXjbOcn0VfTFN/SdTCvxJ91ZO8ZZ+KxZ6Ldc3juE/uQXhqng+r3c0IYyaOjf/BUnyUSaYDfiPvj5PXCuwb/+OtIdJ1MKZpdxN1zTNs0J5PN+cxXAe+8e/Pd2NtzVC29kZesY3/bk5nVP9k+narXZaIjzEGefL70wtrNfp5JXbKbS56d42la6cosgrODCMfbgilc8dHoDOgH954haJ6aQMhBN/tzmoxCUca+TbE5z3YHO4GRRQFyTUmHQ6nB4+qYnd60Gk16LQavydvd1W2CXS4QyYD+ba12t1s25+LAowZ2PDFck/hSZw7/oeu3wQMw2f463dEm/Vsc/aBy54h+oZXeNe4gC3RF6HvO87vCZ0y9+fT+AWg0WL79M+U//tXWP/3DM7dn6HavPqtqfwUl2rWoUk+C3vyIC40/YzL6o30cLpVTIoTy6aXce1bj374LMwz7kXfaxSa2GQORZ1DhqY7ji3vopbmes9HpTzVnO0JIwkhBMu/PIRGUVhwUb86t02IrvLkT+WV8+zKnfxvS3rIbY+dKgGgd9dYunTy9jw9E0pBu8OQa9SyPNxHt6EfdEGjvHgfcVEGSiqqrrvsAitvfnLA31axuZFGvg1xByT02Jzhe/NBck2ldm5zeLA5Pf7oGf/CqyPAk69lMTXKpKPc5mLb/hwG9Ij3N4MIF6Gq2L99E8VowTh+QdB7/qxXjw7FYKaozOGPrPERF2XghCOOqCv+iGna3egHTkbYSnB8v4KK5fdh++oVppd/iF0xY57xa+zDLsWkONEf/AIAj9PObdHrUfKPYLrgF5jGXR1UJMps0vORegFoNNi+/juuYz8Qd+hjbon+mlT70QbNtaPw06F8dh8tYN55veoMbQWIjzFQUuFEVQUffHsMIahVhjmWVYpBr6FbchRdk6LwqILcotpLQQvR8HDqxuBbH6srV8O5+zNQFAzDZjTpWLFResoCvHZf2Y3OiZYm7bc2ZPu/NiSwGbW3KFh4i53+BVZjlZG3Otz+WvK+96CqO5TVEVquAa8uf+hkMfkldqaNTmv4PPatQ809hmnq7WhMwREYgVmvQggKyxycUy0ULzbKQPrpMhSDGX2vUeh7jQLAU3gK14ENuA5tQo+LLxKu5VpLPNpOJrY7ezP6+DeoZTPpfWQF8bpcdOffjr5/zTreFqOOww4TpikLsX/9d+w5R0hWtETpdAxSTuHOGoGu66AGzztSsTvdvPPVIbonR3PR6O71bp8QbUQIb1bsjsP5GHQasmsJjTyaVUqvzrFoNRq6ViblZeVb6ZLk/bcQgvTTZfx0KI/tB/Nwq4LHbxrTqDWihuAz8lpNaLlGtZXiOvAdur7j0UQnNulYsVEGyqwuVCHQKN4qqRpF8Z+D5kZ68m2I263iu6QaEmHjD6E0aP0SjM3u9taSr4yqMQWEUKqqwFZHjfgos578EjsaRQkZC10XnsJMHNtWo00bjq7PuBrvB5YbLrN6a3bU9OSNIaNctIndME24jujr/8Jz9qtwRXcBvE8pn9pGgFCxfvAYiWWHeM86DlO/mscH77qD1eFG13c85lm/wTL/Edb3/R1LS+aR74nB9tlf8OQea9C8I5ldRwooKnOw4MK+YUWb+GLl3/7yELFRBqaf24OScmdQ9zLwrp1k5JTRu5tX6uiS5PVcA3X5Dzce5w///pFPv88gxqInv9jGhh0tH+bq8qjotEqtyZuun9eBx9moiJrqxFoMqEL4F/0zcyvonGRptibi1ZFGvg1xeVRiKpMjGhIrb3d60GoU9DqN34j6PHmzz5P3L7y6/Z5/bQW6fGWFB/dK8Df6CAe1NA/bJ8+iGMyYamlKHFikzKfbJsQEP/7HRRtwOD01Oln50erJs+v8Nym9TkOBGkNxl3EIRzmHki5gm3sAmlq8MItRV5lApqJLG442pTflTrAKE6+UTUMYo7F++mc8+SfCnnskk1sZktu7jsXWQHzyXkm5k7kTzuKszt6nudPVvPmMnHI8qqB3ZWq/yaAjKdZEdqW04/aorP/pFEN7J/KXe87jwetHcXb/ZD7fllGjX3Fz43bX3sRbuBw4f/4KXc+z0SZ0bfKxqidEZeaV0z25Zbx4kEa+TXG5VX9yREPCKG0OryyjKEqVXGN3Y3N6/DKNXqdBUbw3hNrKDPvw7WNsA6JqVGsJ1k+eQXhcmC/+LZropJDbxZi98yu3uaqyXWNravJQexag063iUYVfzvIlQ2WfNRPz3Ac5GDMefYgywz58NwdrQBEtX9JXqbBQPO5OFK0e64d/wLn3q1bTgduaU3nlfLTxeI355hXbiIsy+Nd16sOXENUpzsTkkV39Hnp2tciZo5VJUL27Vi1adu0U5dfvdx0poNzm4qJRaX7n4OppAyi1uvh2Z1YjZhg+brX2qpCug9+Co6JZvHgISIiq8D7t5JfY6Z7c+Jpd9SGNfBvi9qjEVWZ8NiSM0uZw+zXKKgPmwu6o0uQVRcFk8EbeWANKAoQiMdaEUa/l7H7hSTXCacP26bMIazGWmfehTaxdtzXovdE+5TYXhWXB2a4+6ittUP0m5TM+To8WXZcBOD0iZLarD9/nAp+WrHY3vgePUiUOy2WPo+06GMfmt7F9/hd/VE8k89nWDNZuPF7j5ppfbAuqGV8fsVEGRg1IZuGMAei0GpLjzWg1CtnVslmPZZWQGGsMWtjv2slCdqEVVRVs2pNNXLSBIb2qciWG9E5iQFo8n249UaN5e3PidqshwyeFquLc8zna1H5oO9cdZRQugZ78qcrqqC1p5OXCaxvicqt+A9cwTd7jN/LRAdE1vq5QPkyVfV59xq22bk/Tx6QxfnBqWKUMhFCxb3gdtfAU5pn31VtiVVEUYix6yir1R63G+3cgvou+JKD5xJFTJeQWWZkwtEuNUgtVGa9VcfJ16Zm+c2UL8uTdJMWayC+xU2Z1obEkYZ55H66fv8KxdSW2T57FcsnvUfQNizRqL6hCsOdYAeCVVeKiq+aZX2Knb/fw0/U1isKdlw7z/+0z9KerGfmjp0ro3TV4v12TonC5VY5llbL7aAEzzk1Do3hLB3jy0yk6VM613UpZl3uag+tyGdCzE2h1KAYL2m6DUXThy4t1UVt9d3fGDkRZPvqxVzfLcSDYyPvyWLqntJxcI418G+L2CKLNejSKgtXREE2+Sns3m3xyjavSyFc9Ypsqa8pbA2rdhMKo12IM03Nz7vgYd/pPGMcvQJc2rP4P4Gsx6MLtVkmIMaKppt37DEygJ//RpuP8fKyQbp2i/XqsT64x1qhdo4YsTuYj8GnHh9XuIjXR4jXylTcgRVEwDJ2GJjYV2+fPY9/wOqaLfoWiRN4D74nTZf42dDlFNgb08HrPbo9KQamd8XGdm7T/LkmWoAib/BIbBaUOZo6NRwgV99GtqEVZ9HHoOduQx8/rsphvzmRi0SbK/3UKXN51ATsQD1weBZwAe+Cyid6MvvcYdP0nok3tV2dv1fpwe0RIR8G150uU6CR0Z53T6H1XJ8qkQ6tRKLW6sDltmI1akuoJU20K0si3Ib7m0xaTrsGevE/m0Wq89Vwq7G4crmAjb9Rrw5JrwsWdsRvnj2vQ9R2Hfuj0sD/n8+QrbK6QMdcxZj2KUmXkhRCkZ5chgHfXHWbmuT2Cxl9VoMxr/J1uNUy5puocV9jdDIwzoVGUoJhlAF2P4RjHXoXj+5U4f/ovxlHzwp5re2HPsQIUvGVqAxdIC8scCAGd4htndIRQUXOP0SvGwe6j3kQnrUbDoZPFAAyKKsa69p+oeccAhRgEN0YDDnCa9Bi0PdH2G4+201loknuR0qcP+XmlHDtZyMcbD4PwoLo9OIpyWNi/iNijW3Ed/BbFFIO2+1B0PUag6zECxRC+3AShPXlPwUk82QcwnHsViia89YlwUBTFn/WaW2SlW3J0i3bVkka+jVBVgSoEeq0GizHYyJdZnfx5xU6/B67Tarh17mB6dfEuWNmcbjobqxInLCadP3Il0Fv3evJu/6JuU/q2quUF2Na/iiapO6ZJNzXooowy68nMLcftUUOmx2s0vnoe3jkUBKS+HzpZ7DfSPiOv02rQKEqQXFOnkTdVRSCB9ybiy0uItuhD1q/RD5uJpyAT5/Y1aBK6ou89ptb97z1WwJFTJcw/v/4GGGcKe44WcFaXWBwuT1C537zKyJrkuPCNpFBV1MKTuI9tw3V4C6KikMlAv+h4SjbmEh0fi2XvYe6Oyybm22yEOQ7TBbei6zsO4bTy9BvfUFThZuZFo5hyTnCehkZvRNGb6NO7K7/u7Y1scXtU7n7hO9YZx7Bw4a24T+zEnbELT+Ze3Ee2gFaHLm04ut5j0CT1RBPTqV5ZxxdCGfTa3i9Aa8AwsPkr6cZaDJRanWTmVXDu4MaVEQkXaeTbCH9BJJ0Gc2Uct4/002Vk5JYzvE8SUSYdW37OYe/xQr+Rtzuq5Brweqq+yJVgucb7utXhXWQ0Ghrvjbj2fQ0uG+aL7kLRNUynjjF7M/zsTg8JsaE/Gxdl8Gvyx097myVce1F//vXpAXYe8fYoCLxJ6XVVfV6dbrXOZBnfTSKwF64qBFEmHTEWfcjSBoqiYDr/Bqwlp7FveB1NXCrapB4h97/tQC6b9mQza2zPJp3j1qLM6uRYVimXnNeLjJwycgIyTvN9Rr4e+U543Di3r8WdfQC1IAPcTlA0aLsPRX/uFeTl5FGxazNdDnyGA0FXjFQY4jAMm4Nh5Gy/p62YYtAndaPIWsK5g8OTiHRaDYN7JrDnaAFM74++7zj0fcchVBVP7lHcx7bhPvYD7vSf/J9RLPHoB0/FMPJiFE3Na8W78FrlKKi2UlxHtqDvfx6KqfkXRX0JgFaHu0XDJ0Ea+TbDZ6D0Wo23mXZA5Ed+ZWPrRTMGkBhrYv+JInIDvK3AUEnweu++H2qgsfP1ebVVFierroWHi1A9uA5tRJs2HE1cw72OaLPe/zSRGBNaBoiNNvjlmvTsUnRahbSUaK69qB9Pv7sDCG54EmTkXR7/AnYoDHoN2oBeuIHyle8GFApFZ8A8/W6sHzyG7fMXsFz6aMiaJU6XByHgZG55gxYsG0tBiZ1os77RN5S9xwsRwLDeSThcHvYcK0BVBRqNQl6xHa1Gqbe0hWPrSlx7v0Sb2g/9wMloO52FtvtQf3312O4ulmyM4ZrzuzJ2UCqL/76DK6f0ode5PWvsa86Es5hY5mjQk+aw3knsOJxPdoHVnzmraDToOvdD17kfYvwC1PwTqCWnUUvz8OQexfnjB7iP/4hp8i1oOwWPw616n6p9uPZvAI8b/dBpYY+pIcRa9P6oppaMrAEZQtlmuAM8+epyTX6JDa1G8WcSpiZY/Ebc7VFxudUanrzvgqmx8OryUFFLLflw8WTuRViL0Q84v1GfD6xNXz3b1Ye3aFOlJ59dSvfkaPQ6DQN7JjB6QDJx0YaghicGvQZngCZfV3SNoiiYjVVPSxX+aCM90eaqyJ9QaCzxmKffg7CVYP/q5ZAVMp2VpY6Pn26dsMulb/3Isnd+qpFRGi57jhUQY9FzVpcYOidacHsEBZVPgvklNpLiTLUmlgG4jnyPa++X6IdOxzLv95gmXIe+/8SgBhrRZj2xFj2ZJYLDud7z2797fMj9DeiRwLghDVvoHdY7yT+XUCiKBm1yL/R9x2M85xIsM+/DNO1uhLUY65oncB3/MWj7QE9eLcnBuesTtGnD0CZ0a9C4wiU2wClpaU9eGvk2whVQ2tRi0gcZ+YISu/eHVul5pyaaySnyevK+kKtATz7QgAeGUBr1Xk3eW2a48Xq86+B3KKYYdD1GNurz0QEhk7UVu4qLMlJaWeTqRE6ZX5oCuHXuYB5eNDpoe71WE1BquO7oGvDeCH1yTZAnbzHUW25Ym9Ib06Sb8WQfxLHtvRrv+6J/Tpxu+V6kFXYXxeVOTpwu46UP9vjPQbioqmDvsUKG9kpCoyikJnhlE58un1dsIzmu9kVXT2Em9m//ibZzf4zjrqrzWJ2TojhdYOXQyWIMeg09OzdfQ+6kOBNdO0XVauSrcyq/AtJGEnXlk2jiUnHu/F/Q++5KTV54XNjWvQIaLabzbmi28VbHZ+STYo21ZqI3F9LItxGBVe8s1TT5/BI7nQJ+aKkJFsqsLqx2d1U4pCFYrvFR3ZN3ulQqbI335FVbKe4TO9D1m4Cibdw+Aj35ujR5jyo4ll2KzeHxp8YD6HXaGjcHvU4bJNfUtfAKBK17BC5Ee6UkV72lnvX9JqAfcL63GFu1RClf+8H0VjDyvoXR0QOS2X+iiDf+tw+XWyUzr5yt+3K8xqwOjmeXUm5zMayPt8iWr/Kh70kxr9heqx4v7OXYvnwRRW/2hpaG0LYD6ZJk4XSh18j36RoXdv/UcBnWO5FDJ4trL4dRyfHsUh75x1Y27MhCMUWjH3QBat5xPAUn/du4KkMoHVtXoeafwDT5FjQxtXcqayo+I9+thaUakEa+zQjU5H21VXwhgfnFtiAjn5Lg+yFasYeIeQ/y5I01vfqicketJQ3qw31kC6ieRks1UGXkdVoNMbW0FfSFhO6qXGQ9q0vd9bq9ck2AJ19HWQMgSBKzBiRXxVj0CBFexrF++CzwuL16bQA+Tz47v6Jeg9NU8oq9ssqcCWdx1ZS+bNufyy//vIFH3tjGax/9zGsf/lzn538+XoiiwNBeXrkjNsqAyaDldKEVm8NNuc1FpxBGXrWVYv34KUR5AaZpd6KxxNc71i6JFsptLjJyy+mfVv/2DWVY7yTcHsGBE8W1biOE4J0vDyGoktN0/caDRovr4Hf+7TwelZ7Oo5Uy1DT0zRgXHwqfkU9LOYOMfHl5OXPmzCEzMxOAlStXMmfOHObOncuDDz6I0+nVU1966SWmTJnCvHnzmDdvHsuXL2+ZkbdzAtuN+TJRrZWx7qVWF50CQthSEysfqYusAU28AzX5KsNZ3ZMHKC5zNsqTF0LgOvgdmuRedZYuqA+fYU+MMdYaeulbON11xFuqtmunumtrG3QaXC4PQoh6NXkg6GmpypPX+aWk2hZfA9EmdEXbfSiufesRapUxd7pULEYdAm8RrpYkLyD6ZebYHtw0ayAzx/bgtrmDmTm2B5l55f4CY6E4lV9BcrzZf+NVFIXURAs5hVb/gn91T161lWL7+GnUktOYZ9yLrnP/sMbaOaB0bksY+X7d4zHqtX7JRhWC3GIbakAtnu/35XA0qxSzUcvJyu9GY4pB1/Ns3Ic3+9dYTJ4yxpR8iqbTWRjH1i1DNQc+Seyszo1vPhIuYRn5Xbt2sWDBAtLT0wE4fvw4b7zxBitWrOCjjz5CVVXeeecdAPbu3ctzzz3Hhx9+yIcffhiyD6ykqmGIvjKEErzeZEHlDy3Ik6/80eUW2rBVavLmoOiaKsMeKOP4oi9UIRpl5NX8dNTCzCZ58VBVAK16YbJAfJ5NZl4FPVJjghZZQ6HTaXBVLkKD17OvC4tR579BWh0uFLxPPb4CauG2ATQMnYawFuM+VrVw53B76FcZVdPSkk1esY1os97//Z8/oitXXtCXcUM6c8HZ3kXCHYfyav18TpGV1ITgG2hqgpnThVb/DSTw2lPLC7B9/BRqaS7mmfeh6z407LH6CpVpNUpQUbLmQq/TMKhnAruPFvDFDyd56O/fs/jVLfx19W7KrE4cTg+rNxylZ+cYpp7TnewCq/9pWT9gEsJRjvvEDoQQXKL9Dq1wY556O4q2ZTVy8D6d//EXYzmnf8tJQj7CMvKrVq3i0UcfJSXF2xbOYDDw2GOPER3tzdTq378/WVneKnF79+7l9ddfZ+7cuTzxxBM4HOF3ce9IBLYb8zfTtrv93lSgJ2/Qa0mMNQbJNYEeu2/hxld+2IdJHxyB01Bc+78GrQF9n7EN/mwgJoMWnbbusLy4qKr3zgpjgc6g0+J0q37Jpj65xhwg1/iagmuUqjo64Rp5bdowlLhUnHu/9L/mdHlITjCTEGMkvYUjbPKKbaQkhNbMU+LNdE+OrtXIC+HtwlT9850TLRSU2v1VI32evPvETirefwS1vADzzHvRdRvcoLEmxZrQaTWc1SUm7IqWDWVYnyQKSu2sWHeY2CgDF4/ryb70Qh578wfe/HQ/RWUOrr2oHz1SY1CFICvfu8Cs7T4UJSoB16GNuA58wwBtJgeSpqKJ79Ii4wxF105RLZrp6iOsX/7SpUuD/u7WrRvdunm9hsLCQpYvX86f/vQnKioqGDRoEA888ADdunVj8eLFvPLKK9x3331hDygpqfEaVXJy863etzSWXO8PKiU5Gk/lop/eqMde2fuxf+8kkgIMffeUGArLnOgqPfXuXeP9i5GdU7zztph0QecgtVK/BUjtFN2g8+OxlZNx5Htihk0iuXvT6pgAzJvUh6F9OtU6BiG8lSSdbpVh/VPqHWtMlJHsAiuxlecoMcFS52eSk6JwuDwkJEahCoWYKAPJyTEoeu/5VHSasM9Pydg5FHzxBjHO0yiKwsW67xmSr9InPo7jOTHEKd0xdGq8vFUXBaUOBvRMqHWs543sxqqvDqI3GfwhuD6KyuzYnR76pAV/vl/PRMSmdI5klWIx6ejZNZqib1Zg+/5DDKm9SL3sN+gTG1dH/fIpfTmra2yTfpt1fXbu5L4IjcLogan0qQzRnD7+LJ5660e27c9l0shuTDg7jVN55cBeim0uRlfur3DEFIq3rEXNPsghVxeKu088o2xIc42lSclQOTk5/OIXv+Dyyy9n7Fivt/f666/737/55pt56KGHGmTkCwrKG9TU2kdycgx5eS0f3dAYjmeXsm1/DldN6eu/cxcUeo18eakdbWU6dXZuKRk55ei0GtwOF3l5VbpvYrSBHw7kklfpbVnL7HgcLpKTY3BVLiQa9dqgc2AP0Jk9bk+Dzo9zz+cItxNP7/Ob5bzOHuvNFq1rX7FRBm9kUbS+zu2Sk2NQPR5sDhfZOV7P2WF31vkZUfmYnpFZRGGJzX+ufI/v2bllYc9TdB0N+uVkvf0IuJ1MMGhwOBIZ6DzMYKGS+do6NKl9MQyagq73mGarlJiQGEVekY0xA5NrHevA7rGoAtZtTWfSiGDDfDizGACLXhP0eUul1PXzsQJGJlaQ8frvUAtPejNEx11DsccAjbwGZlS2D2zsNRTO73pq5Tx928UatSxZOIrvdmUxbmhn8vLK0KkCg17Dz0fzGdHLG1mkpo2FzR8gFIXlFROY5GjYb6QlaYg902iUOp3jRhv5o0ePcuutt3L99ddz8803A5CVlcXmzZu54oorAK93ptPJpNqfDuXx+baTXDapt7+5hT9OXqfx66vWSrkmMEbeR0qCxS/nKEqwBu3T203VMiBNhsbJNUKoOPetR5Pat0ZmYEsSF2WgzOatDlkfvoxXXyJSvdE1AeseVrvLv9it12kxGrRhyzUAisGMcdSluDN2ovYYzZJPncybMoTOCUbeXbOJO8cpJGRvxb7hddj0FtrUvmi7DEDbuT/aTmc1unxxfuWiYl11ZdJSokmKNbHjUF4NI59T6NXcUxPNCCFwp/+E+8ROko0xjDcWkqwpY4pnH8Ieh3n6r9GddXajxnkmYDbqmH5uVRkKjUYhLTnav/gKoIlLxTDmckR8GsUrC1us/V5b0ygLXF5ezi233MJ9993HvHlVFfpMJhPPPPMMY8eOpXv37ixfvpxp01omLbg9Ya+sY+5wqVVG3hMcQgleI19QEhw+6cMXYZN+ugyzQRek5fluEqZqhjxYtw//q/ac2o8oycF4TutWXxzaO4kenWPCKr9gqIyT92W91r/w6qu77y3YlhAQd+8tbRC+kQcwDJ+BYfgMisoc2MQmDAYtPbsmkKPG87OpL9Ovmo8n+wDuo9vwnD6E84f3vR9UFDQJ3dEm90LTqSfaTj3RJKaFZfhPVz7FhdLkhdOG+9Q+hL2M2V2L2X68HFtRF8wJKf5tcoqsaBSFBKUM2+ev4cnYBQYLuOxcE+W9Hk/GjGDQZbehGFs2C7MtSEuJZtv+XIQQ/t+P8ey5lQvy3zZ7HP+ZQqOM/OrVq8nPz+ef//wn//znPwGYOnUqv/71r3niiSe44447cLlcnHPOOdx0003NOuD2iK/HqsPp8YeuBZY10Os0GHQavyd/TmpNLc4XEXEyt5y4qODVf4sxuK+rj1CLs+Hg2rfOm+FaR+XFlmDeeb3C3lZfqd/7PPm62v9BgCdvD/bkwVcKuf4QylD4EqGMOi1xUQYSY42kny5DURR0XQeh6zoI8CYSeXIO48k7jif3GK707XDwW+9OFAVNfFdvqGpKb3Q9z0YTlVDjWL5OS76FUbX4NO6MHbgzduPJPgTCO5ZRwKgocL23DnqPxjB8JprE7uhP72Vh/AEc778LioJx3DX+2iwvvf0tJ0+XMGPUaAZHoIEHr5HfsDOLwlIHSQGOVGA4cyTSICO/fv16AG688UZuvPHGkNvMmDGDGTNmNHlgkYQvdC+wGXFVMpTXozCbdBSWebsUhfLkk+PNKIr35lDdY9frtOh1mqCSBhBcdTJcuUYtL8B9YgeGEbNbJZSssfgerX3ntt6M14CnJV90jY9osyGsOPlQ+L5TX1mFnqkxIcMoFVM0up5no+vplUCEEIiKQjz5J1Dz0/HkpePJ2IX70EYcm95CmzYcff/zvLXRKzX9nIIKEnVWLPv/S3n6dkTJaQA0Cd0xDJ+BtscINDHJeBwVvPDOVs5PzGNw5h7cx34AjZYpqgenYkDXezTGMZcH9eWNSupMfpYguZF15NsDaZXO08nc8iAj7w4oMRKJSMG8FfDVmwk08oFlDcBrhE/mevXCTiE0V71O429XZzbU/Nriow01KjFqNRq/dh1Waz+PC/u3bwIK+kGTw5tcG+Ez6hWVxcXqTYYyVmX/BjYFB68nnxVQDuBYVinx0YZa6+wE4nuSMFbKRf26x7PjcD6nC63+kgGhUBQFJTrJa2grsyuFEIiS07gOb8Z1aCP2r14GvcnfCCPt8BaWxP6Ma7dA220IuqEXoesxskb6vSY6kbizSll+LJW/3HEj7kMbUcsLeHmroMvAkVw7ZVCN8fjkwFDXXqTQPTkKBTiZW8bIflXnzK1WhTNHItLItwI+b9MZwpPXVl5YUSY9R7NKAEJ68oC/XV2o2un/d/XIkJKMUa/1hyfWhVDd2Nf9DU/mXkyTbkYTE15T77ZCX+k5+7JX6y1QVnmTy68MKw325Ksah1jtLp5+9yfO6ZfMbZcMqXccDnewJz9+SCrvf3OUb3dlcdWUuvvfVkdRFJT4LhjHXI5h1KV4svb566K7j26lOzr2G4Yz9rLr0MTW/f0M7JnAlp9zyCrx0H3oNEoqnPy8YSPDa4nCGD+kMy63Suek+he92ysmg47kBDMZucFZyYGJiZFIZM7qDKPKk6+qGOjrRONbZLSYdPiysWs18pULboEZrj5SEixBhcB8mAxaLEZdnUkXQlWxf/0Pb+/WCdejb4FOOM2N35OvDB+t7yZmNGhRFG8pXaCGJ+9weXC6PGzeexqnS/U/VdWHs/K79SX7xEUbGdG3E5v2ZPuf1hqDotGg6z4U06SbiLr+L1jmP8Iy29Uc7TqrXgMPMLCyZ+vBjGIAciurmKbWkkiVGGti/vm9G91zoL2QlhJd47v1fU/1ZVm3V6Qn3wr4Fl4DPXm3WwQ9Hlr8IX2aoFrTgfgWX6tr73VhMmj9TwuhEG4n9g3/wH1sG4Zzr8Qw9KKw992W6P1GPjxPXqMoWIw6f0axJWjh1Xu+y20uvt5xCoDThdaQfT+r4wgR3TN5ZFd+OpTHjsP5jBnojW4pLLXz6dYMSiqcVNhc2J1utFrvgrvZoGNgzwTO6Z8cMitY0WixxaSRb0uvt2OTj+R4M0mxJg5kFHHhqO5V4ZMJkeuph0OPlGh+OpiH3en2/478kW66yLzBReat6wyjKoQyQK6pZkB8mnGnOFOtXrdPNw3lydeGsdKTD4VqLcb63z/hPvYDxrFXYxw5O+z9tjU+I18epiYP3sXXKk8+WK4B+PFgHtkFVoaclYBHFf5olrqo0uSrvpMhZyWSFGvim53eG4bD5eGF1bv5ZmcWp/LKcblVosx69FoNDqeHjNwyln95iP97eRNL3/rR3zsgkLww2/IFMrBHPAcziisLd3nDJ5PqqBXfEUhLiUHgrZHkw+ORmrykCaiq8Bv3oIXXapUTfZ5lXT9CnxcWauG1NmaMCd2X1FOYie3TPyMcVszT72l3iS++5KdwF16Bap58sFwD8OnWE1iMOi6d1Ief038kM6+83lKw1aNrwJt4c/6ILqz97ji5RVY++PYYmbnl3HvVCH9Ho+pk5Vfw06E8Pvn+BO99fZS7LhsW9H6jjHzPBDbtPU1WXgU5hd78i0g1ZOHi+z5P5pT5m8rLEEpJkwisL149ukYf5Ml7DU1d0Q2d4k0M75PEgB7xYR9/9MCUkK87Ni8H1YNl3u9rbVB9JqMP0OR1Wk1YWnKgRBPKky8pd3LR6O70SI1Gq1E4FeDt1YY/Tr5aMtb5w7vy4cbj/OW93ZwutHLFBX1qNfDgLVbVtVMUbo/KR5vSycgpo0dAvkSoCpH14btODmQUhSxM1hFJjDWi0yrkl1bVdaoKoYxMIx+ZszqDsDmqDLvDGRxdowvhydfVek2r0XDvlSMY0KNmokxDUEtO48naj37otHZp4KHKyFvt7hoGtjZ8UUkKwaWafZo8wJSzu6HTauiSZCEzr/7FV4dLRaGmgUiIMTKiTydOF1oZMzCFWWPDO8/Tx6RhNur4cOPxoNfzim3ERxtDRlbVRqc4M53iTBzIKA5ZYrgjoigKJoPOHwwBAYmJERonL418C2NzBjeX8OGq5slHhSHXNBeuA9+Comlynfi2xBCgyYcb+ua7kZqNuiDP31d2eGCPeLpUNrronhxdWbmwbpwuDwaDNuQ6ymWTejPlnG7cfPGgsEvKWkx6po9JY8fh/KCesXnFdlIbEd44sEcCe48VYHd6pCdficmgDXK4fGW/ZQilpFHYAz15V3VPvuqH37VTFNFmfb1t75qK8LhxHfwOXc+RYbVwO1PxxcnbnZ56i5P58Eli1RPDNIrC9TP6s+Ciqo5H3ZKjKCh1BDVYD4XT5cFYi3HonhLNwukDgjKPw2Ha6DQsAd680+Uhp8hK58SGlxsY2DPeX3Pft3Df0TEZtCE9+bqi0NozUpNvYWxhavJdkqL4669b3rN2n/gJYS9DP/CCFj9WSxJ47vRhyjU+4x4VImnsgpHdgv72NVg+lV9Ov8o65aFwuNR6wzcbisWkY8a5aaz57jiP/XMbp/Ir8KiCtNSG91oYGCDtpUi5BvBGnAWulQUWC4xEpJFvYXweg6LUNPJGQ+vXhnHt/wYlOgltA9q4nYkExqXXlwjlw6dnh1PioXuy12vOzKuo08g7XZ4W6Xp00eg0dh0twKjXMnNsD3p3iWXK2J4UhwivrIvEWBMp8WZvnf4OHj7pw2TQ+TusQWAIZWRq8tLItzC+kgaxFkMNuaa1PQe1NBfPqZ8xjLoUpZ1n9wUa9vDlGp8nX/9lnxRrwmzU1rv46nB76i1z3BjMRh1LFo0Oeq2+Spu1cc6AZA6dLI7Y6JGGYjJoKS6rakvqivDoGmnkWxifxxAXbfCnwAO4PKLVF3pcB74BRWnXC64+dE2Qa8Ipu6woCt06RXOqnvIGTmfLePLNyRUX9GnrIZxRmPTBco07wuPkI3NWZxC2SsMeG2XA4a6KrnG7Pa16UQmnDee+r9H1PAdNdGKrHbelUJSqpuUt4cmDV7I5lV+BELW3o3S4m1+Tb240ihLxNWkaggyhlDQrNocbo16L2aALrkLZyp68c986cFoxnD2n1Y7Z0hj8Rr6hnnx4Rr5bcjQVdjfF5bXXmne6PGe8kZcEYzJWj64R6LRK2GGu7Q1p5FsYu9ONyajFqNfWKGvQWp68cDtw7f4cbfehaJPD7750puP35MOUa3yZrYHJT3VRtfhau2RTVwil5MzEqNfiUYVfi3d71IgNnwRp5Fscm8OD2aDzGnlncIGy1lp4de3/BmEvw3DOJa1yvNbCZ+TDXZBMjDVx9+XDGDsoNaztfWGUdRl5h0vF0MA4eEnb4muL6dPlW/O32BaENbPy8nLmzJlDZmYmAJs3b2bu3LlMnz6d559/3r/d/v37ufzyy5kxYwa///3vcbvrTiTpCNicbsxGLQaDxl9PXgjh9eRbwQMUHhfO3Z+i7TIAXef+9X+gHeHT4sOVawDO7pccdnJStFlPfLSBwydLat3G68lLI9+e8JUY9jld3qfqyJRqIAwjv2vXLhYsWEB6ejoAdrudhx56iFdeeYVPPvmEvXv38s033wBw//338/DDD/P5558jhGDVqlUtOvj2gN3hwVTpybs9Kqoq8KgCQVV/15bEdfA7REURhrPntvixWpsqT77lbpbnDe/KziP5bNydXeM9VQicbrVFQiglLUeVJ19p5D0iYiNrIAwjv2rVKh599FFSUrzVDHfv3k3Pnj1JS0tDp9Mxd+5cPvvsM06dOoXdbmfkyJEAXHbZZXz22WctOvj2gNeT1/m9TofLUxWX24LGSa0owrbhHzg2voUmtS/abvW3smtvVGnyLedJzzvvLAb1TOA/nx8MqiUD4ApRS15y5lPTyLfe+lhbUO/Mli5dyujRVUkZubm5JCdXtR9LSUkhJyenxuvJycnk5OQ083DbH3aHB7NB65cIHC5PVRPvFrqwnPvWU7HyAdxHvkc/fAaWmfdFZORAQ6NrGoNWo+H2eUOIseh5ec0ef5MSqNnfVdI+8Mk1dpdXTo50I9/gZKhQMcOKotT6ekNJqqXRcDgkJ8fUv1Er43B5SIgz06myuFRUjMnv1SfEW5plzIH7UJ12TmxdialLH5Ln3oU+oXOT938mkpwcQ3SUt1VeYkJUi373ycCSm8fywEsbWbPxOL+5dhQAaqG3xECnxOb5HsMayxl4jbc0zT3n8sonMIPJQHJyDBqtBrNJd8ad2+YaT4ONfGpqKvn5+f6/c3NzSUlJqfF6Xl6eX+JpCAUF5ahq7ckntZGcHENeXln9G7YiQgisdjdCVXHavbHW2adLqx4Xbc4mj7n6vF1Hvke4HCgj5lHsjoIz7Jw0B745C9X7Y3XYm34e6yPBrGNor0QOZxT5j5WdX1F5fFerXHtn4jXe0rTEnG0V3pIGuXnl5OWVYbW5EEKcUee2IfPWaJQ6neMGP6OMGDGC48ePc+LECTweDx9//DGTJk2iW7duGI1Gtm/fDsDatWuZNGlSQ3cfUTjdKqoQmI06v27rdKm4WrB+tfvoVhRLPNoIi6QJhU/uakm5JpDYKD1l1iq5xhmi9Z/kzMcUIJ1C5IdQNtiTNxqNLFu2jLvvvhuHw8HkyZOZOXMmAM8++yxLliyhoqKCwYMHs2jRomYfcHvCV7fGbND6DYHD5fEb9+bWAYWjAvfJPegHT233BcjCwVdTvrGFuxpKjMVAeaXXpyhKVes/mQzVrqgeJ+92qw3qm9zeCHtm69ev9/97/PjxfPTRRzW2GThwIKtXr26ekUUAvro1pgBP3uHy+Bdhm9uTd6f/BKobfd+xzbrfMxWfBx9u+7+mEmPW41EFVoebKJO+qom3TIZqV+i0GrQapVoIZeQFJviQLkgLYvN78rqg6JqWKm3qOroVJSYZTXLvZt3vmUpDM16biq8cgk+y8bVzlMlQ7Qtvn1etDKGUNB2/XFNZuwa8Om5LhFCqtlI8p/ah7zM2IsMlQ6FvhRDKQGIs3to3ZVbvIrr05Nsvgd2hpJGXNBq/XGPQ+SUFh0vFXenJN6dc4z7+IwgVXZ+OIdVAVVmDcOvJN5WanrzU5NsrgeWGXR4VvS5yHSN5dbYgPk/BbAxeeHU1c/1qIVRchzahie+KJrF7s+yzPeCTwEytFN1S05OvjLeW0TXtDpOhqmCgxyMiugpl5C4pnwHYHFULr77FHmeAJt8cnrwQAseWd1Fzj2I874YOI9UAjB2UgtmoJS7a2CrHqzLywZ68rF3T/jDqtcGefAQb+cid2RmA35Ov9Dh95Yabs91Y8eY1uPZ+iX7YDPSDLmjy/toTFpOecYNbL6NXr/OWp/AZeYfbg06roO0A4aqRhilQk2/F3g5tgfTkWxCbw4NWo/gvIKNB26wFylwHvqXs2+Xo+o7HOO7qDuXFtxUxZj1lNq9c43SqsjhZO8WnyQvhrQobySGU0si3IL4KlD7ja9BpKguUVWa8NsF7EE4r9k3/wdxrBNoLbkFRItcTOZOIsRiCPHmpx7dPfCGUvt9iJHvykTuzMwC7w+3PrgOvXON0qbgqqxc2RZN3n9gJHjcJk69B0ch7dWsRY9H7F15lf9f2S5WRb5mclTOJyJ3ZGYDN4cFsrDLABp9cU+k9aDWNf0R0H/sBJSoRY9e+TR6nJHy8Rr4qGUqGT7ZPTAZvEx/f4mtLNp5payJ3ZmcAdqfbv+gK+Jt5uz0qep2m0Rq6cNpwZ+5B12u0lGlaGZ9cI4TA4fLIRKh2irGyVk2F3XvD1kawJi8tRAtic3gwBXjyPiPvauJqvjtjF3jc6HqPaY5hShpAjEXv9wC9/V3lT6g94pNRKyqbwMgQSkmjsFcuvPow6jX+EMom6fHHfvCWE07t0xzDlDSAGHNl1qvNhcOlSk2+neIz8j7pTWrykkZhc3pqyDW+ZKjGNvEWLjvuk7vR9RolpZo2IDDr1enyyBDKdorPyPvaOUojL2kUdoc7SK4x6LU43GqTCiK5T+4GjwtdLynVtAWB9WtkCGX7xdfntcrIS01e0kDcHhWnW60ZQums9OQbKde4j/2AYo7tEJ2fzkSqe/KypEH7pIYnH8FrK5E7szbGF5oV2HHGaNAi8NaZb4wnr1YU4c7Y5Y2qkan0bYLPyJdbXd4QSunJt0uM1Yy8XHiVNBhfLXmTMdiTByi3uRvsOQhVxf713wEwDJ3WTKOUNBSjXotep6Go3IFHFVKuaadUl2siOYSy0amS7733Hm+//bb/78zMTObNm4fNZmP79u2YzWYA7rrrLqZN63hGyRbCk/c92lfYXX6PMFycO/+LJ2s/psm3oInv0nwDlTQIRVGIsegpKLEDspZ8e6UjhVA22shfeeWVXHnllQAcPnyYO++8k7vuuosbbriBt99+m5SUlGYbZHvE3/qvWpw8eC+shmjy7uyDOLevRdd3PLr+5zXvQCUNJsZs8Bt5mQzVPjHoNCiKNxQWZHRNvTz22GPcd999mEwmsrKyePjhh5k7dy5//etfUVW1OQ7R7vCVMQ2Ua3yP9k53+PWrhcuOff1rKDEpmM5bJCtNngHEWPQUlPo8eWnk2yO+Pq8VcuG1fjZv3ozdbmfWrFkUFBQwbtw4nnzySVatWsWPP/7I6tWrm2Oc7Q5fw5CghdcA/Tbci8p1cCOiohDTBbegGMzNO0hJo4ix6Kmwe2/iUpNvv5gMOv/3qGtCHakznSaXL1yxYgU33XQTAGlpabz88sv+9xYuXMjatWu56qqrwt5fUlJ0o8eSnBzT6M82N3pjAQDdu8aRFOc1zkU2t//96ChDveMVQuXk/q8wdutPl2Gjat3uTJp3a9GWc05JigZyvP/uFN2qY5HfdfNhMekpKnMA0Dk1ttU6jIVLc827SUbe6XTyww8/sGzZMgAOHjxIeno6M2bMALyt6XS6hh2ioKAcVRUNHktycgx5eWUN/lxLkZtfAUBFmR21Urqxltv973vcar3jdZ/YgbvoNLpzLq112zNt3q1BW89Zp1Rdnzaro9XG0tbzbgtacs6BWeclxVaclc1gzgQaMm+NRqnTOW6SXHPw4EHOOussLBYL4DXqTz75JCUlJbhcLlauXNkhI2vAq8krVMXjQjW5JoyQLeeeL1CiEtH1Gt0SQ5Q0El/WKyDj5NsxgYmKkZzx2iRP/uTJk3TuXNVjc+DAgdx2220sWLAAt9vN9OnTmTNnTpMH2R6xOtwYDVo0AQulgZEY9S28egoy8GTtx3DuVSgaaUjOJGLMVeGvMuO1/WIKWC/TRnB0TZOM/MUXX8zFF18c9Np1113Hdddd16RBRQKlFU5iAzw+CPb66guhdO75EnQGDAMntcj4JI1HevKRgc+T12qUIGcs0ojc21cbU1rhJC462MgbdBp8l1JdcbmqrRT3kS3o+5+HYmr8QrSkZQhMZJPRNe0Xn5GP5PBJkEa+xSgud9ZYrVcUxW8U6vLkXQc3gupGP+SiFh2jpHEEGnmjlGvaLb71skgOnwRp5FuMkgoncVGGGq/7jEJtnrwQAtfBb9B27o82oWuLjlHSOMxGHVqNgkJkZ0pGOj5NXnrykgbjdHmwOdzER9c08vV58p7sA4iSHPQDJ7foGCWNR1EUoi16DAatzEBux/jkmkiuWwPNkAwlqUlxhTfeNjaUJ1/PheU68A0YzOh6y7DJM5kYswGhOtp6GJIm4JdrItzIR/bs2ojScq+Rjw+RQeeLxgj1iCjs5biP/4i+7wQU3ZmVfScJJsail4uu7Rz/wmsEx8iD9ORbhOJyr4cXSpM3VBr3UJ6868gW8LjRD5JSzZlOn26xWEzy59Oe8WvyEe7Jy6u0BSiplGtC1cKo8uSDvQchBK7936BJ7oU2qUfLD1LSJC6b1KethyBpIqY6nqojicieXRtRUuFAUYIzI33Upsl7Th9CLcqUC64SSSvhKwMuQyglNSgstVNSXvuiW3G5k9goA5oQF4+hFu/BuX0tijkWfd/xzTtYiUQSEhlCKamVv324l399eqDW90srnMRHhV449ck1gZ68+9Q+b52as+ei6OWCq0TSGoT6LUYiUpNvBNn5VszG2iMrissdISNrIECTr7ywhBA4fngfJSoR/aALmn2sEokkNCYZQikJhdXuwupwU1DqwOnyhNymtmxXqMp49SVDeTJ2oeYexXDOJSjahjX3lkgkjcfYQUIopZFvIHnFVY0/cotsNd5XVRGyOJmPQE9eCBXHjx+gxKagHyAbdEskrYlGUTDqtdKTlwSTX1Jl2E8XWmu8X2ZzIQTE1aLJD+mdxPnDuxAXZcBzcg9qQQbGUfNRNFI5k0hamwE94unZObJbKkrL0kACPflQRr6kjkQogG6dorjp4kEAOE/uAZ0BXe9zW2CkEomkPu69ckRbD6HFkUa+geSX2LAYdRj0GnJCGPniOkoaVMeTtR9t5/4oWvk1SCSSlkHKNQ0kv8ROp3gTnRMtnC4K4clXeD352Fo0eR+qtQS16BTaroNbZJwSiUQC0sg3mLxiG8lxZjonWsgprLnwWuLz5GuRa3x4svYDoOs2qPkHKZFIJJU0SSdYtGgRBQUF6HTe3TzxxBNkZGTwt7/9DZfLxY033hhR/V6FEBSU2BnWO4mEGCPlNhflNhfRAeULSiqcmI26eisUerL2g8GMJqlnSw9bIpF0YBpt5IUQHDt2jA0bNviNfE5ODvfddx8ffPABBoOBa665hrFjx9K3b99mG3BbUlrhxOlWSY43kxRnAryLr327xfm3KSl31LroGog7az+6LgNRNPJhSiKRtByNtjDHjh1DURRuvfVWLrnkEt5++202b97MuHHjiI+Px2KxMGPGDD777LPmHG+bklfijazpFGeiS6IFoMbia0mFM2RHqEDUsnxEaS7arlKqkUgkLUujPfnS0lLGjx/PY489ht1uZ9GiRcyaNYvk5GT/NikpKezevbtB+01Kim7skEhObtl4159PlgDQv1cSXZOj0WoUSu3uoOOW29z0S4uvcyxlWT9QASQPHY2hGcbc0vM+E+mIc4aOOe+OOGdovnk32sifffbZnH322QBYLBauuOIK/vSnP/HLX/4yaLuG9sAsKChHVUWDx5OcHENeXlmDP9cQjp0sAkCjqhQVVtAp3szxzGL/cYUQFJTaGKZPrHMstoM7UEwxFBOP0sQxt8a8zzQ64pyhY867I84ZGjZvjUap0zlutFzz448/smXLFv/fQgi6detGfn6+/7Xc3FxSUlIae4gzjvxiG7FRBn9pgs4J5qCEKLvTg9Ol1qnJCyG88fFdB6EoUo+XSCQtS6OtTFlZGU8//TQOh4Py8nLWrFnDM888w5YtWygsLMRms/HFF18wadKk5hxvm5JfYie5csEVoHOShZwiG6rwPnlUdYSqw8iX5CAqiqQeL5FIWoVGyzVTpkxh165dzJ8/H1VVufbaaxk1ahT33XcfixYtwuVyccUVVzB8+PDmHG+bkldso09AJE1qogWXW6Wo1EFSnCmgpEHNbFehunEf3YZz1yeAjI+XSCStQ5Pi5O+9917uvffeoNfmzp3L3Llzm7LbMxKPqlJY6mDs4ABPPsEbYXO6yOo18rV48p7cY9i+fAlRUYgmviumqb9EE9e59QYvkTQDHo+boqI83G5nqx0zN1eDqqqtdrwzhVDz1ukMJCQko21gGRRZNCVMikodqEKQHG/2v5ZaGUZ5usDKkLMS/dmu1TV5587/gceFeea9aNOGSy1e0i4pKsrDZLIQFdW5wQEVjUWn0+B2dzwjX33eQggqKkopKsqjU6cuDdqXtDZhEhgj7yM+2oDRoPXHyhdXONBqlKAMWOFy4D65B12fc9H1GCkNvKTd4nY7iYqKbTUDL6lCURSiomIb9RQlPfkwyS/21qnpFODJK4pC5wQLm/aeZvuhPErKnSTEGIN+BO7MPeBxous1utXHLJE0N9LAtx2NPffSrQyTvBI7igKJMcGLqpPP7kqvLjEM7pnArHE9uHl28IKq+/iPKKYYtJ37t+ZwJRJJC3LFFXPJzs5q62GEhfTkwyS/xEZijKlGq7ALRnbjgpHdQn5GeFy4T+xC33sMiqbugmUSiUTSEkgjHyb5xXaS4031bxiA59R+cNnQ9RrVQqOSSDouP/30I3/721/xeFS6dOmC2Wzh2LGjqKrKddctYurUacybN5NVq9ZisURxxx03M3HiJK6//ka++upzdu7cwR133MWf/vQH8vJyyc/PY+TIs1my5Al27Nju33fv3n24557f8MQTD5Obm8NZZ/XG6fRq40eOHObpp5fi8XgwGAw89NCjpKX1aOMzE4w08mHw2dYMjpwqYebYhn157uM/gt6MtptsDCKJPDbtyWbj7uwW2fd5w7swcVj9USQnT2awevXHvPXWm3TqlMySJY9TUVHOL395M4MHD2XUqNHs2PETZ589iuzsbHbu/Inrr7+R77/fzIUXTmPz5o3069efP/7xKVwuF9dffyUHDx4I2nd0dDTPPfcU/fsP5Nln/8rOnT+xfv2XAKxa9Q7XXHM9U6dexLp1X/Dzz3ukkW9PCCF4/5tjfPL9CUYPTOHS83uH/1nVg/vEDnQ9R6Bo9fV/QCKRNJi0tJ5ER0fz44/bcDjs/O9/HwFgt9s5fvwY48efx/bt29BoFKZPn8W6dV/gdrvZtWsn99//EEajkX379rJq1Tukpx+npKQEm80atG+AHTu289hjTwIwcuQ5dO3qlWjHj5/Ic889zdatm5kw4XwuuODCNjgLddPhjbwqBHuOFtA/LR6zsep0CCF464tDbNhxigtGduX66QPQaMJf3facPoSwl6E7S0o1kshk4rDwvO2WxGj0BkKoqoeHH/4DAwYMBKCwsIDY2DjKyspYsWI5Wq2OUaPGkJGRzscfr6V3794YjUZWr17Bhg3rueSSS7niinM5fvwoorJMiW/f4I1sCUxO0mq9a2xTplzE0KHD2bTpO957712+/34TDzywpLWmHxYdOrrGanfx0vt7eGH1bl76YA+egC9x/U+n2LDjFDPH9mDhjIYZeCEErr1fgdaALi1yyjpIJGcq55wzhrVrVwOQn5/PDTcsICfnNAkJCRiNRjZt+pbhw0dyzjlj+Ne/3mDChPMB+OGHrVxyyWVMnz4LUDh8+FDIDNvRo8/liy8+BWD//p85dSoTgEceeZB9+35m/vzL+cUvfumXes4kOqyRz8gp44l//cieYwWMH5LK/hNFrFp/FIDj2aWsWHeY4X2SuOKCPg2OT3Uf2og7fTuGUfNQ9DXr2Egkkubl5ptvxeFwsHDhVfz617/kV7+6h27dugNeSSU6OgaLxcKoUWPIz89jwoTzALjqqmt5882/c/PN1/Hcc08xdOjwkKGRt9xyO6dOZXL99Vfx9tv/8ss1CxfexFtvvcnNN1/Hyy//hbvvvq/1Jh0mivA9m5whtEY9+ZJyB4tf+x6zUcuv5g+jb/c43vnyEF9tz2TBhf344oeTgODRm84Nyl4NB7Ukh4r3H0Gb3Avz7N+1eHu/jlhvuyPOGdp+3qdPn6Bz59btSSzLGgQT6juor558h9TkD2eW4HB5+L9rRvr7s141tS+ZeeW8u+4wWo3Cg9eParCBF6ob2/rXQKPFNOVW2b9VIpG0OR3SCp3IKUOrUeiZWnX302k13DF/KP3T4lk0YwC9u8aGvT8hBO7Th7B98RJq3jFMk25EE53UEkOXSCSSBtEhPfkTOWV07RSFXhechRpjMbD4unPC3o9wOXAd2ohr33rUolOgN2MYfRn63uc295AlEomkUXQ4Iy+E4MTpMkb06dTofagVRbj2folz/wZwWtF0OgvjpJvQ9xknF1olEskZRYcz8sXlTsqsLnp2blwndNVajHXN4whbCbpeozEMnY4mta+szieRSM5IOpyRP3HaG53QM7XhRl54XNi+eBHhtGK59FG0nc5q5tFJJBJJ89IkI//SSy/x6afeBIHJkyfzu9/9jgcffJDt27djNnvrrt91111Mmzat6SNtJtJPl6IAaSm1hxyFQgiB/bv/oOYexXTRndLASySSdkGjjfzmzZvZuHEja9asQVEUfvGLX/Dll1+yd+9e3n77bVJSUppznM1GRk45nZMsGA3hl/4VQsW56xPch77DcM4l6HuPacERSiSSjsIbb7wGeJOtWopGG/nk5GQWL16MweDtZ9qnTx+ysrLIysri4YcfJisri2nTpnHXXXehOYPixU/klDGgR3xY2wqh4j6+Hef2D1GLMtGdNQrDqPktOj6JRCJpThpt5Pv16+f/d3p6Op988gnvvPMO27Zt44knnsBisXD77bezevVqrrrqqrD3W1fmVn0kJwfr7E6Xh52H8xg1MBWtRqGozE5RmYMhfZJrbBuI8Lgp37eJku8/xJl7An1SVzrNv4+oQePPyOYfdc0lUumIc4a2nXdurgadrvUdtlDH3L79R/71rzcQQnDqVCZTp15IVFQM3377NUIInnvuRZKSknjvvRV8+ukn2O02FEXhj39cRq9ewdVk58+fzZAhQzl8+BCvvvoG33+/mRUr3kEIlYEDB/Hb3y7mxRf/Qq9evbn88itZu/YD3n33bVau/AC328Vll13CBx98xJo174c8VvX9f/LJf1m79gPi4+OJiYll8OAhgIc//vFxjh3zlla57LIrmT//shrz1mg0Db4GmrzwevjwYW6//XYeeOABevfuzcsvv+x/b+HChaxdu7ZBRr45yxr4ShVce1E/Lhqdxp5jBQB0itaHTA9XbaW4D2/CuedLREUhmoSumKbchq7POGwaDbYCa4PH1dK0dap7W9AR5wxtP29VVYNS7V2HNuE6+G2LHEs/YBL6/hNrTe/3eFR+/nkvb721kri4eObOncadd97LP/7xFk8++Tiff/4Zs2fPZcOGDbz44qsYjSb+8Y9XWb16Fffd97sa+xs7dgKPP/4njh07ytq1H/C3v72B0Wjk1Vdf4q23/s24cRP5+OO1zJt3OT/8sJXS0lJyc/NITz/OkCHDqKiw1Xks3/4PHNjHf//7If/853IUReGXv7yJgQMHs2PHTkpKSvjnP5dTUlLMyy+/wJw582uMU1XVGtdAi5Y12L59O/fccw8PPfQQs2fP5uDBg6SnpzNjxgzAu1ip07VNAM/x7FLWbc9Er9Ow5rtjjBmU6o+sSUupuhP6JBnXoY14Tu4BoaLt3B/DeYvQ9hiOopw5UpNEIqmid+8+pKZ2BiAuLp7Ro71JiKmpnSkrKyUqKprHHvsjX331BSdPZrB162b69RsQcl+DBw8FYMeOH8nMPMntt98EgNvton//gSxYsNDfAerEiRNceOF0du7cwYEDPzNhwnn1Hsu3/59+2s64cROxWCyAt1Sxx+Ohd+8+ZGSc4De/uYtx4yZy5533NNt5arQFzs7O5s477+T5559n/PjxgNeoP/nkk4wbNw6LxcLKlSu59NJLm22w4eJRVf792QFiowzcc8VwnnxrO+99fQSHy0NKghmLyTttd+ZeHFtXohacRIlKwDB8Jrp+E9Emhu7ZKpFIqtD3n4i+/8Q2O351B9JX491HTs5p7r77di6//CrGjZtAYmIShw8fDLkvX+14j0dl6tSLuPfe+wGwWq14PB6MRiN9+/bniy8+pWfPnpx99ii2b9/G7t27uPbaG+o9lm//iqIgRHBdeo/HQ1xcPG+9tYofftjKli2buPHGa/nPf1YRE9N0ea7Rbuobb7yBw+Fg2bJlzJs3j3nz5rFjxw5uu+02FixYwOzZsxk0aBBz5sxp8iAbylc/ZpKRU8610/rTq0ssM8f2YPPe0+w9XkjP1Bg8ucew/u8ZbJ88i3DaME29nagFf8Y49ipp4CWSCOHAgX10757G1Vdfx+DBQ/n++82oqqfOz5x99ii+/XYDRUWFCCH485//xKpV7wAwYcJE/vWvf3D22aM4++xRbNz4LWazifj4+LCPNXr0GDZv3kh5eTkOh4Nvv/0agI0bv+GJJx5mwoTzuPfe32I2W8jNzWmW89BoT37JkiUsWRK6A8p1113X6AE1BVUI9p8oYs13xxjeJ4nRA5IRLgezz0lh58/pKBW5zLRvwbr2AIoxGuP4BegHT5Xt+SSSCGTMmHGsWbOa66+/Er1ez+DBQ/0Lm7XRr19/brrpVu6555cIIejXbwDXX38jAOPHn8ezzy7j7LNHExsbS3x8AuPHn9egY/XrN4Arr1zAL36xiJiYGFJTvZ21xo2byNdfr2PhwqswGAxccMFU+vTp2yznISLqyTsdTvZt/ob9BzOx22wkGV1M6uFBV5KJqCgM2lbVmTCNnIVh6HQUg7k5h94mtPViXFvQEecMbT9vWU++9ZD15Ktx/Ifv6HN4OX00QBSAgsbaGU2X/mgSuqPo9KBosHk0xA4Yi2JqfJimRCKRtCciwsj3nzgV0/gx2Gwu0BlQdEYUnaHGdjVfkUgkksgmIoy8oijEdu6GowM+wkskEkldyCBwiUQSNmfYEl6HorHnXhp5iUQSFjqdgYqKUmno2wAhBBUVpehCyND1ERFyjUQiaXkSEpIpKsqjvLy41Y6p0WhQ1Y4XXRNq3jqdgYSE5AbvSxp5iUQSFlqtjk6durTqMds6bLStaM55S7lGIpFIIhhp5CUSiSSCOePkGo2m8Q2xm/LZ9kxHnHdHnDN0zHl3xDlD+POub7szrqyBRCKRSJoPKddIJBJJBCONvEQikUQw0shLJBJJBCONvEQikUQw0shLJBJJBCONvEQikUQw0shLJBJJBCONvEQikUQw0shLJBJJBBMRRv6///0vF198MdOmTWP58uVtPZwW46WXXmL27NnMnj2bp59+GoDNmzczd+5cpk+fzvPPP9/GI2w5nnrqKRYvXgzA/v37ufzyy5kxYwa///3vcbvdbTy65mf9+vVcdtllzJw5kz/+8Y9Ax/iuP/zwQ/81/tRTTwGR+32Xl5czZ84cMjMzgdq/3ybPX7RzTp8+LaZMmSKKiopERUWFmDt3rjh8+HBbD6vZ2bRpk7j66quFw+EQTqdTLFq0SPz3v/8VkydPFhkZGcLlcombb75ZbNiwoa2H2uxs3rxZjB07VjzwwANCCCFmz54tduzYIYQQ4sEHHxTLly9vw9E1PxkZGeK8884T2dnZwul0igULFogNGzZE/HdttVrFmDFjREFBgXC5XOKKK64QmzZtisjve+fOnWLOnDliyJAh4uTJk8Jms9X6/TZ1/u3ek9+8eTPjxo0jPj4ei8XCjBkz+Oyzz9p6WM1OcnIyixcvxmAwoNfr6dOnD+np6fTs2ZO0tDR0Oh1z586NuLkXFxfz/PPP88tf/hKAU6dOYbfbGTlyJACXXXZZxM35yy+/5OKLL6Zz587o9Xqef/55zGZzxH/XHo8HVVWx2Wy43W7cbjc6nS4iv+9Vq1bx6KOPkpKSAsDu3btDfr/Ncb2fcVUoG0pubi7JyVXdUlJSUti9e3cbjqhl6Nevn//f6enpfPLJJyxcuLDG3HNyctpieC3GI488wn333Ud2djZQ8/tOTk6OuDmfOHECvV7PLbfcQl5eHlOmTKFfv34R/11HR0fz61//mlmzZmEymTj33HPR6/UR+X0vXbo06O9QdiwnJ6dZrvd278mLEEU0FSVyS5MePnyYm2++mQceeIAePXrUeD+S5v7ee+/RpUsXxo8f73+tI3zfHo+HLVu28Mwzz7Bq1Sr27Nnj120DibR5HzhwgPfff5+vv/6ajRs3otFo2LRpU43tIm3eUPt13RzXe7v35FNTU/nxxx/9f+fm5vofgSKN7du3c8899/DQQw8xe/Zstm3bRn5+vv/9SJv7J598Ql5eHvPmzaOkpASr1YqiKEFzzsvLi6g5A3Tq1Inx48eTmJgIwIUXXshnn32GVqv1bxNp3zXAxo0bGT9+PElJSYBXmnjjjTci/vsGrx0L9Vuu/npj5t/uPfkJEyawZcsWCgsLsdlsfPHFF0yaNKmth9XsZGdnc+edd/Lss88ye/ZsAEaMGMHx48c5ceIEHo+Hjz/+OKLm/uabb/Lxxx/z4Ycfcs899zB16lT+9Kc/YTQa2b59OwBr166NqDkDTJkyhY0bN1JaWorH4+G7775j5syZEf1dAwwcOJDNmzdjtVoRQrB+/XrOPffciP++ofbfcrdu3Zo8/4jw5O+77z4WLVqEy+XiiiuuYPjw4W09rGbnjTfewOFwsGzZMv9r11xzDcuWLePuu+/G4XAwefJkZs6c2YajbB2effZZlixZQkVFBYMHD2bRokVtPaRmZcSIEfziF7/g2muvxeVyMXHiRBYsWEDv3r0j+rs+77zz2LdvH5dddhl6vZ5hw4Zx2223MW3atIj+vgGMRmOtv+WmXu+yM5REIpFEMO1erpFIJBJJ7UgjL5FIJBGMNPISiUQSwUgjL5FIJBGMNPISiUQSwUgjL5GE4NZbb+XIkSMN+sztt9/OBx980EIjkkgaR7uPk5dIWoLXX3+9rYcgkTQL0shLIor169fzt7/9DZfLhclk4oEHHmDjxo0cPnyY/Px8CgoKGDhwIEuXLiU6Opp33nmHFStWoNfrMRqNPPHEE/Tt25epU6fywgsvMGzYMFauXMlbb72FRqOhU6dOPPzww/Tq1YucnBwWL15Mbm4uXbt2paCgwD+Oo0ePsnTpUoqLi/F4PCxcuJArrriCiooKHnzwQU6cOIFGo2HIkCE88cQTaDTyoVrSQjRbgWSJpI05fvy4mDNnjigsLBRCCHHo0CExceJEsWzZMjFp0iSRl5cnPB6P+M1vfiOWLVsm3G63GDJkiMjJyRFCCLFmzRqxYsUKIYQQU6ZMEbt37xabN28WF110kSgoKBBCCPH++++LWbNmCVVVxa9+9Svx/PPPCyGESE9PFyNHjhTvv/++cLlc4uKLLxZ79+4VQghRWloqZs2aJXbs2CHWrFkjbr75ZiGEEG63W/z+978X6enprXmaJB0M6clLIoZNmzaRm5vLjTfe6H9NURQyMjKYOXMmnTp1AuCKK67gySef5IEHHmDmzJlcc801XHDBBUycOJG5c+cG7fO7777j4osv9hcLu+yyy1i6dCmZmZls3ryZBx54AICePXsyduxYwFsKOiMjg4ceesi/H7vdzr59+zj//PN5/vnnWbhwIRMmTOCGG26gZ8+eLXlaJB0caeQlEYOqqowfP56//OUv/teys7NZuXIlTqczaDufPPLss89y6NAhNm/ezOuvv87q1av529/+5t9WhKj6IYTA7XbXKAWr03l/Th6Ph9jYWD788EP/e/n5+cTExGA0Gvnyyy/ZunUr33//PTfddBNLliyJuDo0kjMHKQRKIoZx48axadMmjh49CsA333zDJZdcgsPhYN26dZSVlaGqKqtWrWLKlCkUFhYyefJk4uPjufHGG7n33ns5ePBg0D7PO+88PvnkEwoLCwF4//33iY+Pp2fPnpx//vmsXLkSgKysLLZu3QpAr169MBqNfiOfnZ3NnDlz2Lt3L++88w4PPvgg5513Hvfffz/nnXcehw8fbq1TJOmAyAJlkoji008/5dVXX0UIgU6n46GHHmLLli18//33eDweioqKGDNmDEuWLMFkMrFixQr+85//YDKZ0Gq13HfffUyYMCFo4XX58uWsWLECVVVJTEzkkUceoV+/fhQWFvLggw+SkZFB586dcbvdXHrppVx22WUcOHDAv/DqdrtZtGgRCxYswGq18tBDD3Hw4EHMZjNdu3Zl6dKlxMXFtfWpk0Qo0shLIp4XX3yRoqIiHnnkkbYeikTS6ki5RiKRSCIY6clLJBJJBCM9eYlEIolgpJGXSCSSCEYaeYlEIolgpJGXSCSSCEYaeYlEIolgpJGXSCSSCOb/AW9dunIlhGDfAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "import easydict\n", + "from common.multiprocessing_env import SubprocVecEnv\n", + "cfg = easydict.EasyDict({\n", + " \"algo_name\": 'A2C',\n", + " \"env_name\": 'CartPole-v0',\n", + " \"n_envs\": 8,\n", + " \"max_steps\": 20000,\n", + " \"n_steps\":5,\n", + " \"gamma\":0.99,\n", + " \"lr\": 1e-3,\n", + " \"hidden_dim\": 256,\n", + " \"device\":torch.device(\n", + " \"cuda\" if torch.cuda.is_available() else \"cpu\")\n", + "})\n", + "envs = [make_envs(cfg.env_name) for i in range(cfg.n_envs)]\n", + "envs = SubprocVecEnv(envs) \n", + "rewards,ma_rewards = train(cfg,envs)\n", + "plot_rewards(rewards, ma_rewards, cfg, tag=\"train\") # 画出结果" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3.7.12 ('rl_tutorials')", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.12" + }, + "orig_nbformat": 4, + "vscode": { + "interpreter": { + "hash": "4f613f1ab80ec98dc1b91d6e720de51301598a187317378e53e49b773c1123dd" + } + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/notebooks/common/multiprocessing_env.py b/notebooks/common/multiprocessing_env.py new file mode 100644 index 0000000..28c8aba --- /dev/null +++ b/notebooks/common/multiprocessing_env.py @@ -0,0 +1,153 @@ +# 该代码来自 openai baseline,用于多线程环境 +# https://github.com/openai/baselines/tree/master/baselines/common/vec_env + +import numpy as np +from multiprocessing import Process, Pipe + +def worker(remote, parent_remote, env_fn_wrapper): + parent_remote.close() + env = env_fn_wrapper.x() + while True: + cmd, data = remote.recv() + if cmd == 'step': + ob, reward, done, info = env.step(data) + if done: + ob = env.reset() + remote.send((ob, reward, done, info)) + elif cmd == 'reset': + ob = env.reset() + remote.send(ob) + elif cmd == 'reset_task': + ob = env.reset_task() + remote.send(ob) + elif cmd == 'close': + remote.close() + break + elif cmd == 'get_spaces': + remote.send((env.observation_space, env.action_space)) + else: + raise NotImplementedError + +class VecEnv(object): + """ + An abstract asynchronous, vectorized environment. + """ + def __init__(self, num_envs, observation_space, action_space): + self.num_envs = num_envs + self.observation_space = observation_space + self.action_space = action_space + + def reset(self): + """ + Reset all the environments and return an array of + observations, or a tuple of observation arrays. + If step_async is still doing work, that work will + be cancelled and step_wait() should not be called + until step_async() is invoked again. + """ + pass + + def step_async(self, actions): + """ + Tell all the environments to start taking a step + with the given actions. + Call step_wait() to get the results of the step. + You should not call this if a step_async run is + already pending. + """ + pass + + def step_wait(self): + """ + Wait for the step taken with step_async(). + Returns (obs, rews, dones, infos): + - obs: an array of observations, or a tuple of + arrays of observations. + - rews: an array of rewards + - dones: an array of "episode done" booleans + - infos: a sequence of info objects + """ + pass + + def close(self): + """ + Clean up the environments' resources. + """ + pass + + def step(self, actions): + self.step_async(actions) + return self.step_wait() + + +class CloudpickleWrapper(object): + """ + Uses cloudpickle to serialize contents (otherwise multiprocessing tries to use pickle) + """ + def __init__(self, x): + self.x = x + def __getstate__(self): + import cloudpickle + return cloudpickle.dumps(self.x) + def __setstate__(self, ob): + import pickle + self.x = pickle.loads(ob) + + +class SubprocVecEnv(VecEnv): + def __init__(self, env_fns, spaces=None): + """ + envs: list of gym environments to run in subprocesses + """ + self.waiting = False + self.closed = False + nenvs = len(env_fns) + self.nenvs = nenvs + self.remotes, self.work_remotes = zip(*[Pipe() for _ in range(nenvs)]) + self.ps = [Process(target=worker, args=(work_remote, remote, CloudpickleWrapper(env_fn))) + for (work_remote, remote, env_fn) in zip(self.work_remotes, self.remotes, env_fns)] + for p in self.ps: + p.daemon = True # if the main process crashes, we should not cause things to hang + p.start() + for remote in self.work_remotes: + remote.close() + + self.remotes[0].send(('get_spaces', None)) + observation_space, action_space = self.remotes[0].recv() + VecEnv.__init__(self, len(env_fns), observation_space, action_space) + + def step_async(self, actions): + for remote, action in zip(self.remotes, actions): + remote.send(('step', action)) + self.waiting = True + + def step_wait(self): + results = [remote.recv() for remote in self.remotes] + self.waiting = False + obs, rews, dones, infos = zip(*results) + return np.stack(obs), np.stack(rews), np.stack(dones), infos + + def reset(self): + for remote in self.remotes: + remote.send(('reset', None)) + return np.stack([remote.recv() for remote in self.remotes]) + + def reset_task(self): + for remote in self.remotes: + remote.send(('reset_task', None)) + return np.stack([remote.recv() for remote in self.remotes]) + + def close(self): + if self.closed: + return + if self.waiting: + for remote in self.remotes: + remote.recv() + for remote in self.remotes: + remote.send(('close', None)) + for p in self.ps: + p.join() + self.closed = True + + def __len__(self): + return self.nenvs \ No newline at end of file