{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "import math, random\n", "import gym\n", "import numpy as np\n", "import torch\n", "import torch.nn as nn\n", "import torch.optim as optim\n", "import torch.autograd as autograd \n", "import torch.nn.functional as F\n", "from IPython.display import clear_output # 清空单元格输出区域\n", "import matplotlib.pyplot as plt\n", "# %matplotlib inline\n" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "USE_CUDA = torch.cuda.is_available()\n", "Variable = lambda *args, **kwargs: autograd.Variable(*args, **kwargs).cuda() if USE_CUDA else autograd.Variable(*args, **kwargs)" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "from collections import deque\n", "\n", "class ReplayBuffer(object):\n", " def __init__(self, capacity):\n", " self.buffer = deque(maxlen=capacity)\n", " \n", " def push(self, state, action, reward, next_state, done):\n", " state = np.expand_dims(state, 0)\n", " next_state = np.expand_dims(next_state, 0)\n", " \n", " self.buffer.append((state, action, reward, next_state, done))\n", " \n", " def sample(self, batch_size):\n", " state, action, reward, next_state, done = zip(*random.sample(self.buffer, batch_size))\n", " return np.concatenate(state), action, reward, np.concatenate(next_state), done\n", " \n", " def __len__(self):\n", " return len(self.buffer)" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [], "source": [ "env_name = \"CartPole-v0\"\n", "env = gym.make(env_name)" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [], "source": [ "epsilon_start = 1.0\n", "epsilon_final = 0.01\n", "epsilon_decay = 500\n", "\n", "epsilon_by_frame = lambda frame_idx: epsilon_final + (epsilon_start - epsilon_final) * math.exp(-1. * frame_idx / epsilon_decay)" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "[]" ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" }, { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAZ9UlEQVR4nO3dfXQd9X3n8ff3XulK1pP1aNmWbWQb24lJAgaFmIeT0IQQYFvc3W0b3GVDSFL6RFua7u6Bk57Qsv80TdvdZuOS0DakTSgOoWniEojbJpA2FFMLHMAPGGSDbQkbyY+yLcuypO/+MSNzLWTr2rrSaGY+r3Pu0cxvRvd+RyN/PPrNb2bM3RERkfjLRF2AiIgUhwJdRCQhFOgiIgmhQBcRSQgFuohIQpRE9cGNjY3e2toa1ceLiMTS888/v9/dm8ZaFlmgt7a20t7eHtXHi4jEkpntOtsydbmIiCSEAl1EJCEU6CIiCaFAFxFJCAW6iEhCjBvoZvY1M+s2s81nWW5m9iUz6zCzl8zs8uKXKSIi4ynkCP3rwI3nWH4TsCR83Qk8MPGyRETkfI0b6O7+r8DBc6yyCvhbD2wAas1sTrEKHG3jGwf5wg9eQbf9FRE5UzH60FuAPXnznWHbO5jZnWbWbmbtPT09F/RhL+45zANP76D3xOAFfb+ISFJN6UlRd3/Q3dvcva2pacwrV8fVUJUD4MDxk8UsTUQk9ooR6F3A/Lz5eWHbpKivLAPg4PGByfoIEZFYKkagrwM+EY52WQkccfe9RXjfMTVUjhyhK9BFRPKNe3MuM3sEuA5oNLNO4D6gFMDdvwI8AdwMdAB9wB2TVSxAfRjoOkIXETnTuIHu7qvHWe7AbxatonEo0EVExha7K0XLS7NU5rIcOKZAFxHJF7tAB6irzHFQo1xERM4Qy0BvqMzppKiIyCixDPT6ypz60EVERolpoJcp0EVERolloDdUBUfoup+LiMjbYhno9ZU5Tg4O0zcwFHUpIiLTRmwDHTQWXUQkXywDXZf/i4i8UywD/e0jdI1FFxEZEctAbwjvuKirRUVE3hbLQK+vUh+6iMhosQz0ylyWXElGgS4ikieWgW5muvxfRGSUWAY66PJ/EZHRYh3oOkIXEXlbbAO9QbfQFRE5Q3wDvapMwxZFRPLENtAbq8roGxiib2Aw6lJERKaF2AZ6U3VwcdH+ozpKFxGBBAR6z7H+iCsREZkeYhvojeHVoj1HdWJURARiHOinj9AV6CIiQIwDvaGyjIxBj0a6iIgAMQ70bMaoryzTEbqISCi2gQ5BP7oCXUQkEOtAb6ouo+eYAl1EBBIQ6Pt1hC4iAsQ90KuCI3R3j7oUEZHIxTvQq8sYGBymt1+X/4uIxD7QAfarH11EpLBAN7MbzWy7mXWY2T1jLF9gZk+Z2SYze8nMbi5+qe/UVKWLi0RERowb6GaWBdYANwHLgdVmtnzUar8PPOruK4Bbgb8odqFjadTVoiIipxVyhH4l0OHuO919AFgLrBq1jgM14fRM4M3ilXh2OkIXEXlbIYHeAuzJm+8M2/L9AXCbmXUCTwC/NdYbmdmdZtZuZu09PT0XUO6ZZs4opTRr6kMXEaF4J0VXA19393nAzcA3zOwd7+3uD7p7m7u3NTU1TfhDMxmjQZf/i4gAhQV6FzA/b35e2Jbv08CjAO7+LFAONBajwPHoalERkUAhgb4RWGJmC80sR3DSc92odXYDHwEws3cTBPrE+1QK0FRdRnevAl1EZNxAd/dB4C5gPbCNYDTLFjO738xuCVf7PeBXzOxF4BHgkz5Fl28215TRrS4XERFKClnJ3Z8gONmZ3/b5vOmtwDXFLa0wzTXlHDh+klNDw5RmY32dlIjIhMQ+AZtrynFHR+kiknqxD/TZNeUA7Duih0WLSLrFPtCbw0B/q1eBLiLpFvtAnz1TR+giIpCAQK+rKCVXktERuoikXuwD3cxoriljnwJdRFIu9oEOwYlRHaGLSNolItCba8p5S1eLikjKJSLQZ9eUs+9Iv54tKiKplohAb64p58SpIT1bVERSLRmBPlNj0UVEEhHoulpURCRpga4jdBFJsUQE+qya4Nmi3Qp0EUmxRAR6eWmWuopSHaGLSKolItAhGOmiPnQRSbPEBHpL7QzePKxAF5H0Skygz62dQdfhE1GXISISmUQF+pETpzh2UhcXiUg6JSbQW+pmALBXR+kiklLJCfTaYCx6pwJdRFIqMYE+tzY4Qn9TgS4iKZWYQJ9VXU5Jxug6pEAXkXRKTKBnM8bsmeU6QheR1EpMoIPGootIuiUu0DUWXUTSKlGBPrd2Bvt6+xkcGo66FBGRKZeoQG+pm8HQsPPWUT1fVETSJ1GBrqGLIpJmiQr0kYuLNHRRRNIoUYE+coSuE6MikkYFBbqZ3Whm282sw8zuOcs6v2RmW81si5n9XXHLLExFroS6ilI6dYQuIilUMt4KZpYF1gAfBTqBjWa2zt235q2zBLgXuMbdD5nZrMkqeDwL6ivoPNQX1ceLiESmkCP0K4EOd9/p7gPAWmDVqHV+BVjj7ocA3L27uGUWbn59BbsPKtBFJH0KCfQWYE/efGfYlm8psNTMnjGzDWZ241hvZGZ3mlm7mbX39PRcWMXjuKihgq5DJzQWXURSp1gnRUuAJcB1wGrgL82sdvRK7v6gu7e5e1tTU1ORPvpMC+orGBx29ur5oiKSMoUEehcwP29+XtiWrxNY5+6n3P114FWCgJ9yC+orAdh1QN0uIpIuhQT6RmCJmS00sxxwK7Bu1DrfJTg6x8waCbpgdhavzMItaKgAUD+6iKTOuIHu7oPAXcB6YBvwqLtvMbP7zeyWcLX1wAEz2wo8BfxPdz8wWUWfy+yacnLZDLsOHo/i40VEIjPusEUAd38CeGJU2+fzph34bPiKVDZjzKubwR4doYtIyiTqStERCxo0dFFE0ieZgV5fwa4DfQR/OIiIpENiA/1o/yBHTpyKuhQRkSmT2EAHDV0UkXRJZqBr6KKIpFAyA/30EbqGLopIeiQy0CtyJcyuKWfnfgW6iKRHIgMdYFFTJTt7FOgikh6JDfSFjZXs7DmmoYsikhqJDfRFTVX09g9y4PhA1KWIiEyJBAd6cNdFdbuISFokNtAXN1YB8Pr+YxFXIiIyNRIb6C11M8iVZHSELiKpkdhAz2aM1oYKdijQRSQlEhvoAIsaq9ipLhcRSYlkB3pTJbsP9HFKD4wWkRRIeKBXMTjsetiFiKRCwgNdQxdFJD0SHeiLm4Khi691qx9dRJIv0YE+c0Ypc2aW8+pbR6MuRURk0iU60AGWNlezfZ8CXUSSL/GBvmx2NR09xxjUSBcRSbjkB3pzNQODw+zSSBcRSbjkB/rsagB1u4hI4iU+0C+eVYWZAl1Eki/xgV5emqW1oVIjXUQk8RIf6ABLm6vYrkAXkYRLRaAva67mjf3H6T81FHUpIiKTJhWBvnR2NcMOHbpiVEQSLBWB/u45NQBs29sbcSUiIpMnFYG+sKGSylyWLW8q0EUkuQoKdDO70cy2m1mHmd1zjvX+q5m5mbUVr8SJy2SM5XNr2Nx1JOpSREQmzbiBbmZZYA1wE7AcWG1my8dYrxr4HeC5YhdZDJfMncnWvb0MDXvUpYiITIpCjtCvBDrcfae7DwBrgVVjrPe/gS8A/UWsr2je0zKTvoEhXt+ve6OLSDIVEugtwJ68+c6w7TQzuxyY7+7fP9cbmdmdZtZuZu09PT3nXexEvKclODG65U11u4hIMk34pKiZZYA/A35vvHXd/UF3b3P3tqampol+9HlZ3FRFriSjfnQRSaxCAr0LmJ83Py9sG1ENvAd42szeAFYC66bbidHSbIZ3z65mc5dGuohIMhUS6BuBJWa20MxywK3AupGF7n7E3RvdvdXdW4ENwC3u3j4pFU/AJS0z2fzmEdx1YlREkmfcQHf3QeAuYD2wDXjU3beY2f1mdstkF1hM75k7k6P9g+zWvdFFJIFKClnJ3Z8AnhjV9vmzrHvdxMuaHJfNrwXgp3sOc1FDZbTFiIgUWSquFB2xtLmKilyWF3YdiroUEZGiS1Wgl2QzXDqvlhd2H466FBGRoktVoAOsWFDLtr29nBjQrXRFJFlSF+iXL6hjcNh5WePRRSRhUhfoly2oBWDTbvWji0iypC7QG6vKuKihghcU6CKSMKkLdIAV84MTo7rASESSJJWBfsVFdfQcPcmegyeiLkVEpGhSGegrFzUAsGHngYgrEREpnlQG+sWzqmisyvGsAl1EEiSVgW5mfGBRAxt2HlA/uogkRioDHYJul71H+nWjLhFJjNQG+lWL6gH1o4tIcqQ20Bc3Bf3oG3YejLoUEZGiSG2gj/SjP7tD/egikgypDXSAay9uZF9vP691H4u6FBGRCUt1oH9oafCg6h9v74m4EhGRiUt1oM+tncHS5iqefrU76lJERCYs1YEOcN2yWWx8/RDHTw5GXYqIyISkPtA/tLSJgaFhnt2h4YsiEm+pD/S21joqcll1u4hI7KU+0MtKsly9uIGnXunR8EURibXUBzrADctn03X4BFve7I26FBGRC6ZAB65f3kzG4Aeb90VdiojIBVOgA/WVOT6wsIEnN++NuhQRkQumQA/d9N7Z7Og5Tkf30ahLERG5IAr00A3LZwPw5MvqdhGReFKgh2bPLGfFglq+/7K6XUQknhToeX7+shZe2XeUbXs12kVE4keBnufnLp1LScb4h01dUZciInLeFOh56itzXLdsFt/d1MXQsC4yEpF4KSjQzexGM9tuZh1mds8Yyz9rZlvN7CUz+6GZXVT8UqfGf7m8he6jJ3mmY3/UpYiInJdxA93MssAa4CZgObDazJaPWm0T0Obu7wMeA/642IVOlQ+/axY15SU89nxn1KWIiJyXQo7QrwQ63H2nuw8Aa4FV+Su4+1Pu3hfObgDmFbfMqVNemuU/r2jhB5v3ceDYyajLEREpWCGB3gLsyZvvDNvO5tPAk2MtMLM7zazdzNp7eqbvU4JuW3kRA0PDPNquo3QRiY+inhQ1s9uANuCLYy139wfdvc3d25qamor50UW1pLmalYvqefi5XTo5KiKxUUigdwHz8+bnhW1nMLPrgc8Bt7h77Psq/vvKVjoPneDHuk+6iMREIYG+EVhiZgvNLAfcCqzLX8HMVgBfJQjzRCTgDZc0M6u6jIeeeSPqUkRECjJuoLv7IHAXsB7YBjzq7lvM7H4zuyVc7YtAFfBtM/upma07y9vFRmk2w6euXci/vbaflzuPRF2OiMi4LKqn9LS1tXl7e3skn12oo/2nuPqPfsS1FzfywG1XRF2OiAhm9ry7t421TFeKnkN1eSmfuOoifrBlHx3dx6IuR0TknBTo47jjmoXkshn+4umOqEsRETknBfo4GqvKuP3qVv5hUxfb9+nhFyIyfSnQC/Ab1y2mqqyEL65/JepSRETOSoFegNqKHL/2ocX8y7ZuNr5xMOpyRETGpEAv0KeuWUhzTRl/+I9bdPWoiExLCvQCzchl+f3/tJzNXb18c8OuqMsREXkHBfp5+Nn3zeHaixv5k/Xb6e7tj7ocEZEzKNDPg5lx/6pLODk4zH3rthDVRVkiImNRoJ+nRU1V3P3RJTy5eR/feUHPHhWR6UOBfgF+9YOLubK1nvvWbWHPwb7xv0FEZAoo0C9ANmP86S9dCsBvr93EycGhiCsSEVGgX7D59RV88Rfex6bdh7nve+pPF5HoKdAn4Kb3zuE3f2Yxazfu4ZvP7Y66HBFJuZKoC4i7z350Gdv2HuW+721mVnUZH7tkdtQliUhK6Qh9grIZ48u/vIJL59fyW49s4t937I+6JBFJKQV6EVTkSnjok++ntaGCz/xNO890KNRFZOop0IuktiLHNz/zAebXVXDHQxv5py37oi5JRFJGgV5Es6rL+davrmT53Bp+/eEX+NpPXtfoFxGZMgr0IqutyPHwZz7AR941i/sf38r/+PZL9J/SOHURmXwK9ElQWVbCV267gruvX8Lfv9DJqi8/w5Y3j0RdlogknAJ9kmQyxt3XL+WhO97Pwb4Bfn7NM/y/H76mq0pFZNIo0CfZzyybxT/d/UFuuGQ2f/rPr/Kx//Ov/OiVt6IuS0QSSIE+Beoqc6z55cv5+h3vJ5MxPvX1dlY/uIFndxyIujQRSRCLahRGW1ubt7e3R/LZURoYHOabG3bxwI930HP0JFe21nPHNa1cv7yZ0qz+fxWRczOz5929bcxlCvRo9J8aYu1/7OYv/+11ug6fYFZ1GR9//3xWXTaXi2dVR12eiExTCvRpbGjYeeqVbh5+bhdPv9qDOyxrrubm987hw++axSVza8hkLOoyRWSaUKDHxFu9/Tz58l6+//Je2ncdwh3qKkq5enEjVy1u4LL5tSybXa2uGZEUU6DHUPfRfv694wA/6djPT17bz77wodS5kgyXzK3hvS0zuXhWFYubgldzTRlmOpIXSToFesy5O3sOnuDFzsO81HmYF/ccYeveXo6dHDy9TlVZCfPqZjC3dgZza8uZMzP42lxdTl1ljvrKHLUVpZSVZCPcEhGZqHMFuu6HHgNmxoKGChY0VPBzl84FgpDvPnqSHd3H6Og5xo7uY3QeOsGbR/p5YfchDvedGvO9qspKqKsspa4iR2WuhMqyEirLssHX3MjXEirKspSVZCnNGmUlGXIlGXLZYD4XzpeVZCjNBq9sxsiYkc0YWTMyGU63jbRnDP0VITKJCgp0M7sR+HMgC/yVu//RqOVlwN8CVwAHgI+7+xvFLVXymRnNNeU015Rz9cWN71jeNzDI3iP9vNXbz+G+Uxw8PsCh4wMc6jvFob4BDh4f4PjJQboOn6BvYJDjJwc5fnKIE5N835mM8Y7wtzDoR7Lewu0bif6g3U5P57fbmO2W933nXm/a/fcyzQqaZuVMuwOCC63mtz+y5PTBWTGNG+hmlgXWAB8FOoGNZrbO3bfmrfZp4JC7X2xmtwJfAD5e9GqlYBW5ktP96+djaNjpGxikb2CIgcFhTg4Oc2pomIHBYQZGvo5qPzU0zJA7w8PO0LAz5ATTHsy7O0PDvL3OGes67py+K6VDME847zDSKRisktceLnA8b/rM7+eM7/cz3mu63Qdzut2Zc3pVw7QryCdQ0MwZpUWs5G2FHKFfCXS4+04AM1sLrALyA30V8Afh9GPAl83MfLr9hsq4shmjuryU6vLJ+YUTkclTyPi3FmBP3nxn2DbmOu4+CBwBGka/kZndaWbtZtbe09NzYRWLiMiYpnRAs7s/6O5t7t7W1NQ0lR8tIpJ4hQR6FzA/b35e2DbmOmZWAswkODkqIiJTpJBA3wgsMbOFZpYDbgXWjVpnHXB7OP0LwI/Ufy4iMrXGPSnq7oNmdhewnmDY4tfcfYuZ3Q+0u/s64K+Bb5hZB3CQIPRFRGQKFTQO3d2fAJ4Y1fb5vOl+4BeLW5qIiJwP3eVJRCQhFOgiIgkR2c25zKwH2HWB394I7C9iOXGgbU4HbXM6TGSbL3L3Mcd9RxboE2Fm7We721hSaZvTQducDpO1zepyERFJCAW6iEhCxDXQH4y6gAhom9NB25wOk7LNsexDFxGRd4rrEbqIiIyiQBcRSYjYBbqZ3Whm282sw8zuibqeC2Vm883sKTPbamZbzOx3wvZ6M/tnM3st/FoXtpuZfSnc7pfM7PK897o9XP81M7v9bJ85XZhZ1sw2mdnj4fxCM3su3LZvhTeBw8zKwvmOcHlr3nvcG7ZvN7OPRbQpBTGzWjN7zMxeMbNtZnZV0vezmf1u+Hu92cweMbPypO1nM/uamXWb2ea8tqLtVzO7wsxeDr/nS2YFPH/P3WPzIrg52A5gEZADXgSWR13XBW7LHODycLoaeBVYDvwxcE/Yfg/whXD6ZuBJgscYrgSeC9vrgZ3h17pwui7q7Rtn2z8L/B3weDj/KHBrOP0V4NfD6d8AvhJO3wp8K5xeHu77MmBh+DuRjXq7zrG9fwN8JpzOAbVJ3s8ED7x5HZiRt38/mbT9DHwQuBzYnNdWtP0K/Ee4roXfe9O4NUX9QznPH+BVwPq8+XuBe6Ouq0jb9j2C57ZuB+aEbXOA7eH0V4HVeetvD5evBr6a137GetPtRXA//R8CHwYeD39Z9wMlo/cxwR0+rwqnS8L1bPR+z19vur0Ing3wOuEAhNH7L4n7mbefYFYf7rfHgY8lcT8DraMCvSj7NVz2Sl77Geud7RW3LpdCHocXO+GfmCuA54Bmd98bLtoHNIfTZ9v2uP1M/i/wv4DhcL4BOOzBowvhzPrP9mjDOG3zQqAHeCjsZvorM6skwfvZ3buAPwF2A3sJ9tvzJHs/jyjWfm0Jp0e3n1PcAj1xzKwK+HvgbnfvzV/mwX/NiRlXamY/C3S7+/NR1zKFSgj+LH/A3VcAxwn+FD8tgfu5juDB8QuBuUAlcGOkRUUgiv0at0Av5HF4sWFmpQRh/rC7fydsfsvM5oTL5wDdYfvZtj1OP5NrgFvM7A1gLUG3y58DtRY8uhDOrP9sjzaM0zZ3Ap3u/lw4/xhBwCd5P18PvO7uPe5+CvgOwb5P8n4eUaz92hVOj24/p7gFeiGPw4uF8Iz1XwPb3P3P8hblP87vdoK+9ZH2T4Rny1cCR8I/7dYDN5hZXXhkdEPYNu24+73uPs/dWwn23Y/c/b8BTxE8uhDeuc1jPdpwHXBrODpiIbCE4ATStOPu+4A9ZrYsbPoIsJUE72eCrpaVZlYR/p6PbHNi93OeouzXcFmvma0Mf4afyHuvs4v6pMIFnIS4mWBEyA7gc1HXM4HtuJbgz7GXgJ+Gr5sJ+g5/CLwG/AtQH65vwJpwu18G2vLe61NAR/i6I+ptK3D7r+PtUS6LCP6hdgDfBsrC9vJwviNcvijv+z8X/iy2U8DZ/4i39TKgPdzX3yUYzZDo/Qz8IfAKsBn4BsFIlUTtZ+ARgnMEpwj+Evt0Mfcr0Bb+/HYAX2bUifWxXrr0X0QkIeLW5SIiImehQBcRSQgFuohIQijQRUQSQoEuIpIQCnQRkYRQoIuIJMT/B3B4SePGsjO/AAAAAElFTkSuQmCC", "text/plain": [ "
" ] }, "metadata": { "needs_background": "light" }, "output_type": "display_data" } ], "source": [ "plt.plot([epsilon_by_frame(i) for i in range(10000)])" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Dueling DQN 网络\n", "\n", "DQN等算法中使用的是一个简单的三层神经网络：一个输入层，一个隐藏层和一个输出层。如下左图：\n", "\n", "\"image-20211112022028670\"\n", "\n", "而在Dueling DQN中，我们在后面加了两个子网络结构，分别对应上面上到价格函数网络部分和优势函数网络部分。对应上面右图所示。最终Q网络的输出由价格函数网络的输出和优势函数网络的输出线性组合得到。\n", "\n", "我们可以直接使用上一节的价值函数的组合公式得到我们的动作价值，但是这个式子无法辨识最终输出里面$V(S, w, \\alpha)$和$A(S, A, w, \\beta)$各自的作用，为了可以体现这种可辨识性(identifiability),实际使用的组合公式如下：\n", "\n", "$$\n", "Q(S, A, w, \\alpha, \\beta)=V(S, w, \\alpha)+\\left(A(S, A, w, \\beta)-\\frac{1}{\\mathcal{A}} \\sum_{a^{\\prime} \\in \\mathcal{A}} A\\left(S, a^{\\prime}, w, \\beta\\right)\\right)\n", "$$" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [], "source": [ "class DuelingNet(nn.Module):\n", " def __init__(self, n_states, n_actions,hidden_size=128):\n", " super(DuelingNet, self).__init__()\n", " \n", " # 隐藏层\n", " self.hidden = nn.Sequential(\n", " nn.Linear(n_states, hidden_size),\n", " nn.ReLU()\n", " )\n", " \n", " # 优势函数\n", " self.advantage = nn.Sequential(\n", " nn.Linear(hidden_size, hidden_size),\n", " nn.ReLU(),\n", " nn.Linear(hidden_size, n_actions)\n", " )\n", " \n", " # 价值函数\n", " self.value = nn.Sequential(\n", " nn.Linear(hidden_size, hidden_size),\n", " nn.ReLU(),\n", " nn.Linear(hidden_size, 1)\n", " )\n", " \n", " def forward(self, x):\n", " x = self.hidden(x)\n", " advantage = self.advantage(x)\n", " value = self.value(x)\n", " return value + advantage - advantage.mean()\n", " \n", " def act(self, state, epsilon):\n", " if random.random() > epsilon:\n", " with torch.no_grad():\n", " state = Variable(torch.FloatTensor(state).unsqueeze(0))\n", " q_value = self.forward(state)\n", " action = q_value.max(1)[1].item()\n", " else:\n", " action = random.randrange(env.action_space.n)\n", " return action" ] }, { "cell_type": "code", "execution_count": 14, "metadata": {}, "outputs": [ { "ename": "SyntaxError", "evalue": "unexpected EOF while parsing (, line 1)", "output_type": "error", "traceback": [ "\u001b[0;36m File \u001b[0;32m\"\"\u001b[0;36m, line \u001b[0;32m1\u001b[0m\n\u001b[0;31m class DuelingDQN:\u001b[0m\n\u001b[0m ^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m unexpected EOF while parsing\n" ] } ], "source": [ "class DuelingDQN:\n", " def __init__(self,n_states,n_actions,cfg) -> None:\n", " self.batch_size = cfg.batch_size\n", " self.device = cfg.device\n", " self.loss_history = [] # 记录loss的变化\n", " self.frame_idx = 0 # 用于epsilon的衰减计数\n", " self.epsilon = lambda frame_idx: cfg.epsilon_end + \\\n", " (cfg.epsilon_start - cfg.epsilon_end) * \\\n", " math.exp(-1. * frame_idx / cfg.epsilon_decay)\n", " self.policy_net = DuelingNet(n_states, n_actions,hidden_dim=cfg.hidden_dim).to(self.device)\n", " self.target_net = DuelingNet(n_states, n_actions,hidden_dim=cfg.hidden_dim).to(self.device)\n", " for target_param, param in zip(self.target_net.parameters(),self.policy_net.parameters()): # 复制参数到目标网络targe_net\n", " target_param.data.copy_(param.data)\n", " self.optimizer = optim.Adam(self.policy_net.parameters(), lr=cfg.lr) # 优化器\n", " self.memory = ReplayBuffer(cfg.memory_capacity) \n", " def choose_action(self,state):\n", " self.frame_idx += 1\n", " if random.random() > self.epsilon(self.frame_idx):\n", " with torch.no_grad():\n", " state = torch.tensor([state], device=self.device, dtype=torch.float32)\n", " q_values = self.policy_net(state)\n", " action = q_values.max(1)[1].item() # 选择Q值最大的动作\n", " else:\n", " action = random.randrange(self.n_actions)\n", " return action\n", " def update(self):\n", " if len(self.memory) < self.batch_size: # 当memory中不满足一个批量时，不更新策略\n", " return\n", " state, action, reward, next_state, done = self.memory.sample(batch_size)\n", " state = torch.tensor(state, device=self.device, dtype=torch.float)\n", " action = torch.tensor(action, device=self.device).unsqueeze(1) \n", " reward = torch.tensor(reward, device=self.device, dtype=torch.float) \n", " next_state = torch.tensor(next_state, device=self.device, dtype=torch.float)\n", " done = torch.tensor(np.float32(done), device=self.device)\n", " q_values = self.policy_net(state)\n", " next_q_values = self.target_net(next_state)\n", "\n", " q_value = q_values.gather(1, action.unsqueeze(1)).squeeze(1)\n", " next_q_value = next_q_values.max(1)[0]\n", " expected_q_value = reward + gamma * next_q_value * (1 - done)\n", " \n", " loss = (q_value - expected_q_value.detach()).pow(2).mean()\n", " self.loss_history.append(loss)\n", " self.optimizer.zero_grad()\n", " loss.backward()\n", " self.optimizer.step()" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [], "source": [ "current_model = DuelingNet(env.observation_space.shape[0], env.action_space.n)\n", "target_model = DuelingNet(env.observation_space.shape[0], env.action_space.n)\n", "\n", "if USE_CUDA:\n", " current_model = current_model.cuda()\n", " target_model = target_model.cuda()\n", " \n", "optimizer = optim.Adam(current_model.parameters())\n", "\n", "replay_buffer = ReplayBuffer(1000)" ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [], "source": [ "\n", "def update_target(current_model, target_model):\n", " target_model.load_state_dict(current_model.state_dict())" ] }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [], "source": [ "\n", "update_target(current_model, target_model)" ] }, { "cell_type": "code", "execution_count": 11, "metadata": {}, "outputs": [], "source": [ "def compute_td_loss(batch_size):\n", " state, action, reward, next_state, done = replay_buffer.sample(batch_size)\n", "\n", " state = Variable(torch.FloatTensor(np.float32(state)))\n", " next_state = Variable(torch.FloatTensor(np.float32(next_state)))\n", " action = Variable(torch.LongTensor(action))\n", " reward = Variable(torch.FloatTensor(reward))\n", " done = Variable(torch.FloatTensor(done))\n", "\n", " q_values = current_model(state)\n", " next_q_values = target_model(next_state)\n", "\n", " q_value = q_values.gather(1, action.unsqueeze(1)).squeeze(1)\n", " next_q_value = next_q_values.max(1)[0]\n", " expected_q_value = reward + gamma * next_q_value * (1 - done)\n", " \n", " loss = (q_value - expected_q_value.detach()).pow(2).mean()\n", " \n", " optimizer.zero_grad()\n", " loss.backward()\n", " optimizer.step()\n", " \n", " return loss" ] }, { "cell_type": "code", "execution_count": 12, "metadata": {}, "outputs": [], "source": [ "def plot(frame_idx, rewards, losses):\n", " clear_output(True) # 清空单元格输出区域，因为多次打印，每次需要清楚前面打印的图片\n", " plt.figure(figsize=(20,5))\n", " plt.subplot(131)\n", " plt.title('frame %s. reward: %s' % (frame_idx, np.mean(rewards[-10:])))\n", " plt.plot(rewards)\n", " plt.subplot(132)\n", " plt.title('loss')\n", " plt.plot(losses)\n", " plt.show()" ] }, { "cell_type": "code", "execution_count": 13, "metadata": {}, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAvwAAAE/CAYAAAA6zBcIAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAABvqUlEQVR4nO3dd3xkd33v/9dninrf1fZdb/e6FxbbYJtiuklCuQmB5IIhJA43cC+B3CQQcgNplOSSEFLIj9674eKAMcUFMDa2123ttXe93du0klZdmtG07++Pc85oRhpJI2mkkTTv5+Ohx0pn2nfKzvmcz/l8P19zziEiIiIiIstTqNwDEBERERGR+aOAX0RERERkGVPALyIiIiKyjCngFxERERFZxhTwi4iIiIgsYwr4RURERESWMQX8y4SZnW9mj5rZoJn9r3KPR+aPmb3ZzO4p9zhERJYbMztmZi8u9zhESk0B//LxZ8BdzrlG59zHyz2Y8czsk2Z2wMwyZvbmApe/y8w6zGzAzD5rZtU5l202s7vMbMTM9o//Mp7LbSvBVK+9f/CQNrOhnJ8X5Fx+l5l1+a/tY2b2qike54fj7idhZo/P2xMTERGRoijgXz7OA/ZNdqGZhRdwLIU8BvwR8PD4C8zsZcB7gBfhPY+twF/nXOVrwCPACuB9wLfNrH2ut50JM4vM9DalUKLHnfS1993nnGvI+bk757J3Amudc03AzcCXzWxtoTtxzr0i936Ae4FvlWD8IiIiMgcK+JcBM7sTeCHwb35mdaeZfd7MPmFmt5nZMPBCM3ulmT3iZ2tPmNkHcu5js5k5M3uLf1mvmb3NzJ5tZnvNrM/M/m3c4/6emT3lX/dHZnbeZGN0zv27c+4OIF7g4puAzzjn9jnneoG/Bd7sP8ZO4Erg/c65mHPuFuBx4L+V4LbTva7HzOzPzWwvMGxmETO7xszu9V+Px4JsuJm9MDebbWY/MbMHc/7+hZm92v/9PWZ22C+/etLMXpNzvTeb2S/N7J/N7BzwATNbYWa3+u/bA8C2YsYfmOa1n+62e51zqeBPIApsnO52ZrYZuB744kwfU0Sk3Mys2sw+Zman/Z+PBWePzWylmX3f3w/0+N/vIf+yPzezU/73+wEze1F5n4mIRwH/MuCcuwH4BfAOP7v6tH/R7wB/DzQC9wDDwJuAFuCVwP8IgtAcVwM7gN8GPoaXFX8xcBHwOjN7PoBf2vEXwGuBdv/xvzbLp3ARXhY68Biw2sxW+Jcdcc4Njrv8ohLcthhvwHutWoDVwA+AvwPagP8N3OKfMfgVsMPfEUSBS4F1ZtZoZrXAbrzXCOAwXjDcjHc2YnzW/GrgiP94fw/8O16wvhb4Pf8ny9/xvGcGz2m8K8ys28yeNrP/M/6sgn//ceB+4G5gTxH3+SbgF865Y3MYl4hIubwPuAa4HLgMuAr4S/+yPwFO4u37VuPtC52ZnQ+8A3i2c64ReBlwbEFHLTIJBfzL2/ecc790zmWcc3Hn3N3Oucf9v/fiBejPH3ebv/Wv+2O8A4SvOec6nXOn8ALWK/zrvQ34kHPuKT8D/EHg8qmy/FNoAPpz/g5+byxwWXB5YwluW4yPO+dOOOdiwH8HbnPO3ea/hj/BC35v9C9/EHge8Cy8A4tfAtfi7TQOOufOATjnvuWcO+3fxzeAg3g7k8Bp59y/+q9rAu+MxF8554adc08AX8gdoHPu15xzH57Bc8r1c+BiYJX/OG8A/nT8/eO9ZjcCP3bOZYq43zcBn5/lmEREyu13gb/x939deMmZN/qXJfESMOc555LOuV845xyQBqqBC80s6pw75pw7XJbRi4yjgH95O5H7h5ldnTMJsx8vaF857jZnc36PFfi7wf/9POBf/FOafUAPYMD6WYxzCGjK+Tv4fbDAZcHlQdZ+LrctRu5reB7wW8Fz9p/3dXhf/AA/A16AF/T/DC8b/nz/52fBnZjZm8zrqBTcx8Xkvw+5j9kORMZtOz6D8U/JOXfEOXfUP/h4HPgb4DcLXC/pnPsh8FIz+42p7tPMrgPWAN8u1ThFRBbYOvK/a4/72wD+ETgE/NjMjgRnWJ1zh4A/Bj4AdJrZ181sHSKLgAL+5c2N+/urwK3ARudcM/CfeEH6bJwA/tA515LzU+ucu3cW97UP75Rp4DLgrJ8R3wdsNbPGcZfvK8Fti5H7Gp4AvjTuOdfnZNfHB/w/Y1zA758B+RTead8VzrkW4Any34fcx+wCUuTXzW+awfhnyjH1ZyLC9HMIbgK+45wbKtmoREQW1mm8JE9gk78N59ygc+5PnHNbgd8A3h3U6jvnvuqcu86/rQM+srDDFilMAX9laQR6nHNxM7sKr8Z/tv4TeK+ZXQRgZs1m9luTXdnMqsysBi+YjJpZTTDJCW9i51vN7EIza8Grk/w8gD8f4VHg/f5tXoNXH39LCW47U18Gft3MXmZmYf8+X2BmG/zL7wXOxyvPecA5tw/vS/9qvNIZgHq8nUCX/7q8BS/DX5BzLg18B2/ybp2ZXYgXUBdtqtfezF5hZqv933cB/wf4XvC3f3mtmUXN7L8zdjAz2WPVAq9D5TwisrR9DfhLM2s3s5XAX+HtAzCzXzOz7WZmeGWiaSBj3no4N/iTe+N4Z8WLKYEUmXcK+CvLHwF/Y2aDeF9e35ztHTnnvouXufi6mQ3gZalfMcVNfoz35fdc4JP+78/z7+t24B+Au4Bn8E6dvj/ntq/Hm/TaC3wY+E2/pnJOtzWz3zWzorP9zrkTQDBZuQsv4/+n+P+PnHPDeK0v9znnEv7N7gOOO+c6/es8CXzU334WuASv1n8q78ArperAC6Q/l3uhef3v/2KK20/62uO1M91rXien2/AOLj4Y3DX+qWn/+b4T+G3n3MP+415vZuOz+K8G+vDeDxGRperv8OZo7cXr7vawvw28xhY/xSsbvQ/4D+fcXXj1+x8GuvG+r1cB713YYYsUZt48ExERERERWY6U4RcRERERWcYU8IuIiIiILGMK+EVEREREljEF/CIiIiIiy5gCfhERERGRZSxS7gEArFy50m3evLncwxARWZQeeuihbudce7nHUU7aT4iIFFbMPmJRBPybN29mz5495R6GiMiiZGbHyz2GctN+QkSksGL2ESrpERERERFZxhTwi4iIiIgsYwr4RURERESWMQX8IiIiIiLLmAJ+EREREZFlTAG/iIiIiMgypoBfRERERGQZmzbgN7ONZnaXmT1pZvvM7J3+9jYz+4mZHfT/bfW3m5l93MwOmdleM7tyvp+EiIiIiIgUVkyGPwX8iXPuQuAa4O1mdiHwHuAO59wO4A7/b4BXADv8n5uBT5R81CIiIiIiUpRpV9p1zp0Bzvi/D5rZU8B64FXAC/yrfQG4G/hzf/sXnXMO+JWZtZjZWv9+ZImJJ9Pc9vgZRlOZaa8bDhkvu2gNzbXRBRiZyMJ7+uwg9dUR1rfUTrhs78k+NrbW0Vpflbd9JJHiu4+c4uotK9i+qmGhhioiIovMLw52ce22lYRCtuCPPW3An8vMNgNXAPcDq3OC+A5gtf/7euBEzs1O+tvyAn4zuxnvDACbNm2a6bhlgdx9oIt3f/Oxoq8/EEvy+9dvnccRiZTP//raI+xY3ci/vuGKCZe94ZO/4q3Xb+XdL9mZt71vJMn7vvsEH37tJQr4RUQq1E+fPMvvf3EPf3HjLm5+3rYFf/yiA34zawBuAf7YOTdgNnZ04pxzZuZm8sDOuU8CnwTYvXv3jG4rC2cwngTglv/x3IJZzYDD8ZwP3clAPLVQQxNZcF2Do7Q3Vk/YnkhlGE6kGR6d+PlPpb2vt0hYPRJERCrVmYE4AMfPjZTl8YsK+M0sihfsf8U59x1/89mgVMfM1gKd/vZTwMacm2/wt8kSFPdLeTa11RUMdHLVREPEk+kJ24dHU7z7m4/yV79+0ZQHDSKLmXOOvliy4Gc85m9LZybmLpIZ7/9QNLzwp3BFRESguC49BnwGeMo59085F90K3OT/fhPwvZztb/K79VwD9Kt+f+mKJ7xApiY6fXayNhomlpgYDB3uGuJH+87yi6e7Sj4+kYUyNJoinXGMFPiMB5/7ZHriXJdshj+kDL+IiJRHMRn+a4E3Ao+b2aP+tr8APgx808zeChwHXudfdhtwI3AIGAHeUsoBy8IKMpc10fC0162NhrPXzxUESKf6YqUdnMgC6hvxytsKHdSOJLxSniC4zxUcBESU4RcRkTIppkvPPcBke6oXFbi+A94+x3HJIhFPpomEjGgR9cc1VeEpyx1O9Srgl6WrP+YF/IUy/MG2oHwnV8ov81FJj4iIlIvOMcuUYsk0tUVk9wFqIoUD/rgy/LIMBAF/obNYwee+UIY/FWT4VdIjIiJloj2QTCmezFBdZMBfW1W4pCeb4VfAL0vY1CU9fsBfIMOfzHbpUYZfRKTSlastpQJ+mVI8maa2qriPyWSTdoOAv6M/XrCLichS0BdLAJBIZ7JZ+0C2pKdAhj/4zCvDLyJSucqd8tEeSKYUn0lJTzRMLDkxwxnLZj8dZ/0+tCJLTZDhBxgZdyYrlgwm7RbI8Gc0aVdERMpLAb9MKZZMF9WhB7ySnoI1/DnbVNYjS1VQww9j81ICsYQX1KcKnMEK6vqjyvCLiEiZaA8kU4olZhDwR0NTlvSAOvXI0tU3ksj+Pr5Tz1RtOVNqyykiImWmgF+mFE9lZhDwh4mnCk9orPLbeirDL0tVboZ/fMAfm2rSrtpyiohImSnglynFE2lqi1hlF/wa/gIZ/ngyTXNdlLb6KgX8smTl1vAHNfuBoKa/0KRdteUUEZFy0x5IphRPFV/SUxMNM5rKkBlXxxxLeBN/17fUqqRHlqz+WJLWuigwswx/Sm05RUSkzBTwy5SCYL0YtVXe9caX9QSLd61vqVWGX5asvpEka5trgYm9+LMBf4EMf9Clp5jVqkVEZHlzZepOrj2QTCk+ky49/vUmBEPJDDVVYda3ehl+V65Pu8gc9MUSrG2uASautjtW0jNFhj+kDL+ISKWyMu8CFPDLlOLJmU3ahYnBUDyRps7P8MeSaXpzaqFFloJ4Mk08mWFtixfwTyzp8bv0FGrLqYW3RESkzLQHkkmlM45EOlP8wltBSc+4xbdiyTS1foYf1JpTlp4Bv0NPUNIzsS3n5CU9asspIiLlpoBfJhUsmFVTbJeeSCjvdoHcGn6AU30jJRylyPzrywb8Xoa/0GccJpm0m9GkXRERKS8F/DKpIIgJJuNOJ7je+JKeYPGusYA/XsJRisy/oCVne2M1IRtbaCsw5aRdP8OvlXZFRKRctAeSSWUz/JG5TtpNU1sVoqUuSl1VWCU9suQEq+y21lVRVxWZtKRnskm7IYOQJu2KiEiZKOCXSWUD/iIz/DWTTNoNWnuamd+aUyU9srQEJT3NtVFqqyYuMJet4S8waTeZyRBRS04RESkj7YVkUsHk2xn34c8J+J1z2Rp+wGvNqV78ssT0+yU9wVmqCZ2oklNN2nVEld0XEREAytOaXAG/TCo2w0m7QVCfG/CPpryDhuAswTqttitLUF8sQThkNFRHqI2G80p6nHPZmv5kwZV2leEXEal0RnkTP9oLyaSCwL3otpwFaviD3+uCDH9LLb0jyQmTHkUWs/5YkpbaKGY2oaRnNJUh46A6EsI5r51trmTGadEtEREpKwX8MqkgqJn5wltjWc7xnX42qBe/LEF9I0maa6MA1FWF8w5Yg/8nTf7l4yfuptNOLTlFRKSsFPDLpMZKeooL+Kv9Pvy59c3j72OsNacCflk6+mNJmuu8gL42mt+lZ8T/jDfVRICJE3eTmcyyX2XXzDaa2V1m9qSZ7TOzd/rb28zsJ2Z20P+31d9uZvZxMztkZnvN7MryPgMRkeVtee+FZE5G/Ux9sTX8oZBREw3l1fAH2c/cSbuggF+Wlr4Rr6QHvAx/oc94kOFPj5u4m0o7oss/w58C/sQ5dyFwDfB2M7sQeA9wh3NuB3CH/zfAK4Ad/s/NwCcWfsgiIpVj2kjOzD5rZp1m9kTOtm+Y2aP+zzEze9TfvtnMYjmX/ec8jl3mWWyGNfzBdfNq+MeV9KxqrCESMpX0yJLSF0vQUlcFMGHSbvB5b6zxS3rGTdxNVUBbTufcGefcw/7vg8BTwHrgVcAX/Kt9AXi1//urgC86z6+AFjNbu7CjFhGpHJEirvN54N+ALwYbnHO/HfxuZh8F+nOuf9g5d3mJxidlFJ9hSQ94wdBUGf5wyFjTXLPkMvwPP9PLZRtaCGvyZUXKreEfP2k3qOdvDEp6xmX4k+nKmrRrZpuBK4D7gdXOuTP+RR3Aav/39cCJnJud9LedydmGmd2MdwaATZs2zd+gRUQWiCtPV87pM/zOuZ8DPYUuMzMDXgd8rcTjkkVgpjX8wXWnquEHWNdcS0d/vESjnH+HOod47X/cy0+fOlvuoUgZpDOOwXiKlrqcSbvJNM7/1h6r4S88aTeVzhBd5hn+gJk1ALcAf+ycG8i9zHkv2Ix2dc65Tzrndjvndre3t5dwpCIilWWue6HrgbPOuYM527aY2SNm9jMzu36O9y9lFE9mqIqEZpTVrhmX4Q9+r8tZrbehJsLwEmrLebR7GICuwdEyj0TKYcBfZTe3hj+dcST9TH48W8NfeNJuKlMZXXrMLIoX7H/FOfcdf/PZoFTH/7fT334K2Jhz8w3+NhGRZc3KtDuYa8D/BvKz+2eATc65K4B3A181s6ZCNzSzm81sj5nt6erqmuMwZD7Ek2lqIjP7iNSOW4U0W9KTE/DXVYUZGU1PuO1iddovPxqML52DFCmdPj/gz3bpqfIC++CzHdTzBxn+1LgMfzKdIbr8u/QY8BngKefcP+VcdCtwk//7TcD3cra/ye/Wcw3Qn1P6IyIiJTbrvZCZRYDXAt8ItjnnRp1z5/zfHwIOAzsL3V6nahe/eDKdF6gXY9JJuzklPQ3VEYZGl07wHMw3GIgnyzwSmS+3P3Fm0sXg+kYSALTUjk3aBRhJpvx/89tyJsfV8KczrhLmflwLvBG4Iadpw43Ah4GXmNlB4MX+3wC3AUeAQ8CngD8qw5hFRCpGMZN2J/NiYL9z7mSwwczagR7nXNrMtuK1XDsyxzFKmcSS6RnV74NX0nNuOJF3H8H2QF1Vfh/zxS7oKBSUdsjycqJnhLd9+WE+9NpLeMNVEyeGjs/wB+VpwWc45h8oBG05U5nxGX5HTXR5B/zOuXtg0nXjX1Tg+g54+7wOSkREsoppy/k14D7gfDM7aWZv9S96PRMn6z4P2Ou36fw28DbnXMEJv7L4xRLpGbXkBK90Z3RcSY/Z2KJcAPXVYYYTqeykx8XuZDbDv3TOSkjxuoa8uRlnJplI3j+SX8MfnPWKZQN+L8BvnGThrVSmcibtiojI4jRtht8594ZJtr+5wLZb8CZtyTIQT2WonmmGPxKaUMNfGw1jObNU6qsjOOdl/+uq5nKSaWEow7+89fpnpLoGCwf82ZIevw9/kOEPPucjyRRVkRDVEW/7+LacqQpryykiIouP0k4yqXgiTW2Rq+wGJkzaTU48S1DvB0zD8zRx997D3fzyUHdJ7iueTNPtZ4AHVcO/LPX4Af/ZgcJdmIKSnqBGP1vDn83wp6mrCmfr9AtO2lWGX0REWMR9+KVyxVOzKOkpMGl3/DyA+movcJpskuRcfewnB/nI7ftLcl9Bhx4zlfQsV71+Br9zkgx/fyxJY00ku1ruWEmPP2nXP4sV9VtvJiu0LaeIiEyuXO04Awr4ZVKxxOwm7Y6mMmT8oCeeTOf14AeyZTzz1alnJJmic5Js7UwFHXo2r6hXSc8y1TPsva+TfWb6c1bZhbHPb26Gv7YqTMRvvTk+w++V9OirVkREykd7IZnUrDL8fnAfT+UHQ7nqq/NLIkptJJGma2iUdGbu582CDP+uNY1qy7lMBTX83ZN8ZvpiyewquzCxhj/mH9QGWfzxbTm9kh5l+EVEpHwU8MukYomZT9oNDhDiSS/LOVVJz/A8ZfhjiTTpjMvWZs/Fqd4YIYOdqxuJJzMkUpnpbzRHtz1+hu88fHL6K0pJ9PglPRkH54YnZvn7RhLZHvwwsUvPSCJFXTSSrdMf35ZTJT0iIlJuCvhlUvECE26nU+NP8h3LfmYKTNoNAv75yfAHjz1ZTfZMnOyLsbqphhUNXsC3EBN3P3PPUf7zZ4fn/XHE05tzYFiorKcvlsz24IfCk3a9kp5g0u74Lj0ZlfSIiEhZaS8kk4on09kAvlhBNj+WsyjR+IA/KIkYnqdJu0Eg1jk49zr+U70x1rfU0lTjBXwLMXG3czA+aU94Kb2ekQTrmmsA6CrwmekfSWZ78ANEwyGiYct+zsYm7Xr/V5Lja/gzasspIiLlpYBfCkqmM6QyblZdesA7WAC/Lee4Gv6GoEvPPJT0pDMuW3bTVYKJu6f6Yqxvrc0uqjTfE3edc3QNjjIYT81bFyPJ1zeS5Pw1jcDEs0LOOfrH1fCD9zmPT1LDP34eQCrtsh1+REREykF7ISkoCGbGB+vTqR0/oTGRmVDDX1cdZPhLX9KTuwbAXEt60hlHR3/cy/DXBhn+qQP+j9y+n//3yKlZP+bQaCo7/6FDWf55l844+kYS7AwC/nEHicOJNKmMy6vhB69TT3BANr5Lz/i2nMmMJu2KiIhHffhlUQkC59lO2g1KegrNA6gKh4iEbF4m7eZmxeda0nN2IE4q41jfOlbSMzhNSc83HzzB1x98ZtaPmVtS0jGggH++DcSSZBysaaqhpS7K2XEHicEqu83jM/xV4bySnrq8Gv6xkp50xuEcquEXEalw5U77aC8kBY36WeaZT9odK+lxzmXLHXKZGfXVkXkJ+OOJsWBrrr34gx78XoZ/+pKeoPzjydMDuFkewucepJxVwD/vgg49bfVVrGqsnvCZ6Rvx3u/cPvwwtsBcJuOyq0kHJT25k3aDen516RERkXJSwC8FBRn+WU/aTaZJph3pjCtYFlRfFZ6Xkp6R5NhBxPhs7UwFPfg3tNbSWDN9SU8s6ZV/DMRTnOyNzeox8zL8/aVZPEwmF3Toaa2rYnVTzYSzQudyLs9V52f4g/UmaqvG2nImc9pypvzyHpX0iIhIOSngl4KyNfyzXXgrmc45aJh4H3XVkXmZlBqUErXWReec4Q+C9nUttdRXhQkZDMQmH3N/TvZ/3+mBWT1mEPBHQqYM/wII1mpoq6+ivbF6Qpeew51DAGxtr8/bXlsVJpZMZz9vdZO05QzKe1TSIyIi5aS9kBQUBDKFgvWp5NbwT3XQUF8dYWge+vAH4960op6uwdFZl9aAV9LTWhelriqCmdFUG50yw597MPDkmVkG/EOjRMPGeSvqONM/u7MEUrxev6Sntb6KVY01Ez4zBzsHaa2LsqJ+YoY/lkhn6/hrq8KEC9TwB6vuqqRHRETKSQG/FDRVdn4q2YA/mckJhiZ+zOqrwvPSljN4zM0r6kikM3lZ95k61eu15Aw01USnnLSb+1hPnu6f1WN2DozS3lDN2uZaOkrQVlSm1jPsvWdtdV4NfyKdydbtAxw8O8SO1Y2Y5QfstdEwI8lU9v9JXVUYMyMatrwuPUGLTmX4RUSknLQXkoLis5y0Wx0ZW2k3yLYXuo+6qsi8tuU8b4VXgjGXTj2n+rxFtwJNtZEpJ+0Gl21dWc+TBUp6ijnb0DU0SntTDaubajirtpzzrnckQXUkRG1VmFVN1cDYZ8Y5x9NnB9mxqmHC7WqrIvkZfv8zHgmF8vrwa9KuiIjkcpSnL6cCfikoPstJu6GQURMNMTpNDX9DdXheuvTEcjL8MPtOPc45f5Xduuy2xuqpS3qCDP8121Zwuj+enRAajOu6j9zFH33loSnPOnQNehn+Nc3VdA2NTljESUqrZzhBm1+us6rRW203WL+ha3CUgXiKnasbJ9wuKOmJJfLXq4iELW+lXU3aFRERACvzbkABvxQ024W3wAvwY8mxGv66qsiE68zbpN0JGf7ZZcl7R5LEkun8kp7ayJSTdoODgedsXQHk1/HfdaCTU30xbnu8g1//13t44lThkp+uwVHaG6tZ01RDOuPoHlJZz3zqHU5kO/CsavQz/P5B4tNnvQm7hTL8dVVhRpJpYn5XqOAzHgmZJu2KiMiio72QFJTNzkdmHvAHPcqnKumprwozPA+TdoMSi/P8DP/ZWWb4T+f04A801RSZ4fcD/n05dfw/ePwMbfVVfPMPn0MyneG1/3Ev39pzIu/2qXSGc8N+wN/sPa5W251fPSM5GX6/pCdo53qwcxCA7asnBvw10TDOQa8/B6Aum+EPkcpMnLSrDL+ISGX68b4OTvSMlHsYCvilsGwN/ywy/LV+hj+WnGLSbnWEWDJd8pKVmH/WoK2uivqq8Kwz/EFLzg15Gf7pJ+02Vkdob6xmbXNNto4/lkhz1/5OXnbRGq7a0sZt/+t6Lt3QzAdu3Ucm5/n3DCdwzss0r2nyykvOKOCfV73DCVr9gL+uKkJjdSQvw99SF6W9oXrC7YIA/9ywd93goDYasmyQD2SDf2X4RUQq081feoiX/vPPyz0MBfxSWBCsB5NwZ6ImGp62D3+9XwJR6rKeYNXTUMhYVWAhpWKdKpDhb6yJMDSaymu7mGsglqLJX5H1wrVN2ZKenz3dyUgizSsvWQt4LSBfc+V6hhNpOnJ67QdjbW+sZnWzn21WL/551TOcoK1ubBXd9qaxXvyHOr0Ju+M79EBuwO/N06jNzfCrLaeIiOQI4iHw5nZ1lmHfroBfCoon09REQwWDnekEixJN1Ye/rtrbNlLiTj0jiXQ2GGtvrKZrliU9p3pj1EbDtOQEg03+artDk0w27o8lswH/ReuaONw1TDyZ5gePd9BWX8U1W9uy193W7pWJHO4aym7rGhoL+FfWVxMJWd4BgZRWMp1hIJ7KZvjBO7vSORj3O/R4LTkLqfUPWHuGvIC/LnfSbmZiDX+wCq+IiFS27zx8iqs+eAf9I7NvGz4b2gtJQV7AP/NyHvAC/HheH/5CXXq8gGmy4Hm2YjnjXt1UM+uSnoOdg2xtr8874AmC+ckm7g7EkzTVeM/rwnVNpDOOx070ccdTZ3nZRauJ5AR92YC/Myfg9w9O2huqvTMUjdVlbc15pj/GZ+45OqfFyxazoN9+W17A750V6hoapT+WLDhhF6Aump/hD+a6REP5Gf6gZC1YlEtERARgcHSRBfxm9lkz6zSzJ3K2fcDMTpnZo/7PjTmXvdfMDpnZATN72XwNXOZXLJGecQ/+QM24SbuFJv4GXU1GSjxxN5aT4feytbPL8B/oGOT8NfnZ3SCYn2zi7kAsSXM2w98MwCd+dpiRRJob/XKewMqGKppqIhzuGs5uy83wA6xprilrhv/7j53hb7//JHtPzm4RscUuu8pu3bgM/8AoB7MdeibL8PsB/9AoNdEQIT+gj4Tzu/Qk1ZZTREQWgWIy/J8HXl5g+z875y73f24DMLMLgdcDF/m3+Q8zm13UKGUVT2XmEPCHiPslPdWRsWAoV70fMA2XuIZ/JJHOBmOrGqsZSaRnfBahdzhB5+Aou8YH/EGGf4qAP7jOhtZaGmsi3H2gi9a6aLZVZ8DM2NrekF/SMzhKU00ke4ai3AF/EBDfub+zbGOYTz1+dj4vw99UTSyZ5tETfQDsLNChB8YC/u6hRF7bWa9Lj9pyiojI4jLtXsg593Ogp8j7exXwdefcqHPuKHAIuGoO45MyiSXSVM+hpCfo0lM3SZefer+kp9SLbwWTdiGnzeIMg+b9HV47xl1rmvK2NwYZ/klKevpzMvxmxoVrvdu/7KI1eeU8gW0FAv4guw9eSVJHf7xsJTW9fsnLXQeWZ8AfLIyWn+H3uiPdc7CbpppI3vuRK/hc9wwn8g6MIyEr2JZTk3ZFRKSc5pJ2eoeZ7fVLflr9beuB3ObiJ/1tssSMptLUznCV3UAwaXeqsqD66iDDP58lPf7KqTOcuLu/w+uuMyHDXzN5hj+VzjCcSGevA14dP8ArxpXzBLatqufswCiD/v11DsbzAsw1TTWMJNIMzsOKxMXoj3kB8d6T/fPSUeBw1xAfuHXfpF2P5ltvoRp+/yDxoeO97FzdOOmk9bqod/A3/qA2MklbTk3aFRGRcprtXugTwDbgcuAM8NGZ3oGZ3Wxme8xsT1dX1yyHIfMllpjbpN1Ywsvw10yS4R+r4Z+HDH9OSQ/MfLXdAx2DtNZFJ2R3xybtTgz4B/z+/M21Y+Udr758Pa+9Yj3P3bZiwvVhbOLuEb+O38vw12QvX9Ps/Z47cXfvyT5O9i7MAh59I0la/S5Fdx8o/f/RWx89zefvPcaDx3pLft/FCEqWcjsxBQeJiXSGHZOU8wDU5KwtkRvwR8e15Qzq+SOatCsiImU0q4DfOXfWOZd2zmWATzFWtnMK2Jhz1Q3+tkL38Unn3G7n3O729vbZDEPmUTw1t0m7oymvS8/kGf556tKTSFPrZ1+D4K1rhhN39/sTdsdndxurI5hRcPGtYJXd4KAA4LKNLfzTb18+aXZ3fGvOrsHRvEWeVvuLbwV1/LFEmt/51P186Lb9M3o+s9U7kuRZ57WytrmGO/afLfn9B8+7XCVDPcMJ6qvCeQe2QYYfJp+wC+TV7efePhK2vBr+pNpyiojIIjCrvZCZ5dYovAYIOvjcCrzezKrNbAuwA3hgbkOUcphTht/PePaNJCYP+Kvmqw9/Kruyb1NthKpIaEadejIZx9NnByfU7wOEQkZDVaRgSU+Q9W/OCfinc96KOiIh43DXEMOjKYYT6byAc62f4e/wM/w/fOIMQ6MpDpwdLPox5qJ/JEFLXRU37FrFPQe7GU2V9r0Kzmzc8VTpDyaKkbvKbqCxOkKNX8o2VYY/93OdX9ITGlfSoxp+EREpv2Lacn4NuA8438xOmtlbgX8ws8fNbC/wQuBdAM65fcA3gSeB24G3O+dKGyXIgognM7MO+Gv81Xl7hhMFe/CD182kOhIqeZcer6bay76aGaubqmdUf36yN8ZIIj2hfj/QVBstOGm3fxYBfzQcYtOKOo50DWfPQhTK8AeTjr+15yQAx7qHSaTmv+69L5akpTbKDbtWMZxI8+DR0pXeZDKOI91DNPqtSZ85tzBlSrl6RhJ59fvgfWaCM0NTZfjDIcuuQp2b7Y+GbVxJj7r0iIhI+RXTpecNzrm1zrmoc26Dc+4zzrk3Oucucc5d6pz7DefcmZzr/71zbptz7nzn3A/nd/gyX+LJdDZTPlO1OR1MpjpoqK+OlLRLTybjiCfz24kGCykV6yl/wu74HvyBxppJMvzxiSU9xQg69YzvwQ9eqUhLXZSOgTgneka478g5dq5uIJVxHDs3PNldlsRoKs1IIk1rfRXP3baS6kiopGU9p/tjxJMZfvfq8wC4cx5KhqbTO5zI69ATWNVYTWNNhNVNhTv0BILMfu5B7YS2nBnV8IuICBjl3Q8o7SQFxZLpggtmFSMI8gfiqSnnAdRVhUu68FYsOXFl35kuvnXAb8m5c/VUGf6JAf9sMvzgBfzHukc445ftjJ8ovKapho7+Ub790EnM4E9ftguAp+e5rCdY8ru5NkptVZjnblvBnfs7S9YiNFhw7IZdq9jaXs8dZej1XyjDD3D9jnZeecnaSTv0BILPdu5nPDquLWdKbTlFRGQRUMAvEzjn/Az/7Lv0BCbrww/QUB0paUlPEPDXjQv4Z9KH/0DHIJva6rKTisdrqokWnLQblPnktuUsxrb2ehLpDA8f782ON9fqphrO9Me45eGTXLttJdfvWEnIyK4EO1/6/AOYoIPNDbtWcfzcCEe6S3Nm4XCnN/5t7fW8aNcq7j/SU/I1GabTO5wsmOF/54t38OH/dum0tw/+f+TV8E9YaVeTdkVEBH5xqLusj6+9kEyQSGfIOOY8aRemvo+6qjDDpczw+xOA87uu1DAYTxFPFvc4+zsGJq3fB2iapKSnP5akKhzKTvgs1rZV3sTQXx05RzhkEwLQtc017Ds9wMneGL+1ewM10TCb2uo42Dm/Gf7xi1K9cNcqoHTtOQ93DdFSF6WtvooX7lpFIp3hngX8MhxNeSswt9XP7AAtV1C7n/t5D4+ftKu2nCIiAvzXY6fz/v7h4x1c++E7F2wtmsJpTKlo8aT34ZtLH/7s71Nk+OurIyVty1kowx+UyLzjq49kg/HfuWoTz92+csLt48k0R7uHeeUkC2XB5CU9A/EkTbWRactAxtu20gv493cMsrqpmtC4wDCYuNtYE+FlF60BYMfqxgXL8AclShta62hvrOaAP8dhrg53DbGtvQEz49mb22isjnDX/s7sc5xvfX7J0vguPTNRKMMfDY8v6fF+DyvgFxGRHH9/21OA1+p7LvuiYinDLxME2fCZZqsDuQcKU9Xw11dFSlrDH7T4zA3Anr25jQvXNnGke4gnzwzw4yfP8sX7jhe8/aHOITIOzi/QkjPQVBNhcDRFJpNfy94fS854wi5Ac12UlX5nnvH1+zC2+NavX7Yu+7ruWNXA0e7hbI/3+RDU8OcuSrVlRT1HS1XS0zXMtvZ6wCt3ed7O9rw5Aoe7hrj3UPe8ZT56/DMYbQVKeoqVreHP6dITCYXGlfQ4omGb8YGgiIhIKSnDLxMEAf9cFt4KTDlptzpc0gz/iD8fIPfxt6ys57Z3Xp/9+82fe4CTfYVbQO73J+xO1qEHvAy/czCcSNGYU68/EEvOuH4/sLW9nu6h/EW3AuevaSQSMn7nqk3ZbTuCTj3dw+yYZHLxXAWr0OaWGG1ZWV+STj39sSRdg6PZhcfAKxn6weNn+L8/PsC9h8/xyDN9AKxsqOY1V6zjtVduyK5LANBYEy0qa+6cKxhsZ0uW5pBVyXbpieZn+JPj2nKqJaeIiJSbAn6ZIDbHgD+vhn+qkp6qSDZIL4V4tqRn8o/1htZaHj3RV/CyAx0DVEVCbF5RN+ntG2u8+x6ITwz4W2aZLd7W3sADR3uy/d9zXbmplcfe/9K8ScRBf/iDnUPzFvD3xZJEw5Z3tmRLez3dexJe+dIsD24AjnQFE3bHAv4XnN9OyODf7zrMztUNvO/GC9jYVst3HznF5355jE/94mjefVy9pY1v/OFzCt5/z3CC7z16im8/dJJTfTHu+fMbaBg3CbvHP6Ap1KWnWJNO2s1badepQ4+IiJSdAn6ZoNDk15moLbakpzoy60m7P9rXwb5T/bz7pedntwUlPVM95obWOvpGkgzGk3kBO3gZ/p2rG4hM0VElCHQHYknWt9Rmt/fHkpy3on5WzyUobSlU0gNM6Bjk1b57rTlvnGK+wVz0jSRorq3Ky45v9p/fse5hLt3QMuv7Dlpybm0fe71WNlTzpbdeTWNNhEvWN2cf9+UXr+Xc0Ch3PNWZ7ei092Q/333kFAfPDk444PnI7fv59C+OkEw71jTV0DeS5Fj3MBevb8673vhJybNRsA9/KEQ647JnFlKZjCbsiohI2elcs0xQykm7U7XlrK8Kk0hnZrVq7Cd/foTP3Xssb1usQA3/eBtbvez9yd7YhMv2dwxy/urJ6/dhbGGt8RN3B+Ipmmpnd/wcdOqZLOAfr7Yq6NQzfxN3+0aStNblHxAFAfpc6/gPdw0RDRsb2/LPpFy7fSWXbmiZUIKzoqGa1z17I2+5dgtvuXYLf/nKC4iEjG89dHLC/X7i7sO8aNdqbv/j6/nUm3YDhd/rnuGJcxRmKjiTVDeupAfIdupJZ9yUB5AiIiILQXsimWCuk3arI2O3m7qG3wuYgkC9WIPxJI+e6GMwnsqrly608NZ4G1q9rPz4ILBvJEHX4Cjnr2kodLOsbIY/pxe/c47+WHLGi24FLl3fzNrmGi7d0Dz9lX07VjVwaB479fSNJCcEw5va6jArHPB3D40WvSjX4c4hzltRP+ve9Csaqrlh1yq+8/CpvPf/c788SlU4xN+++mJ2rWlivf9en+qbGPB3DsZpqYvOqT9+TbRAht+/v7Rf1pNMO6LK8IuISJkp4JcJ4kUEzlMJhSwb9E91lqCh2rtsaIZ1/Pcf6ckGVMHkUii2pMcLAk/05E/cPezXlW9fNU3A72fxB3N68Y8k0qQzbtZ17SsaqrnvvS/iik2tRd9m+6pGjnQPzVunnl6/pCdXTTTM+pbaCQH/yd4RnvOhO/j+3jNF3bfXknN25U+B39q9ke6hUX7mrwvQN5LglodO8arL12XPlLTWRamNhjlVIMN/ojfGprbJ52oUo65QDb8f3AcLbqXSGWX4RUSk7LQnkgmCTHlNZHYBP4wdLEx10BCURIzMsFNP7gJNvcNjgXesiIC/rb6KuqrwhAz/4U4viM2dSFpIY83Ekp7+cT3rF8LO1Q0k047j5wp3HJqr/tjEkh7wOvWMD/gfONpDMu344RPTB/zJdIZnekamfZ2n84Lz21nZUMW3HjoBwFcfeIZYMs1br9+SvY6Zsb61llMFujKd6BmZUFI0U/XZgD+3LacX8AetOZOZypi0a2afNbNOM3siZ9sHzOyUmT3q/9yYc9l7zeyQmR0ws5eVZ9QiIpVDAb9MENTwzzbDDzk9yqectOtdNjzDkp5fHurOdl0J+qmDd6BSHQlNWLwql5mxobWWk70TM/xV4RAbWqcOAnO79ASClXdn04d/trKdes7Oz4q7hUp6wA/4u4bzyncefqYXgJ8/3T3tfIwTPSMk027OAX80HOLVl6/njqc6OTsQ54v3Hufa7SvYNW4NhfUttZzui+dtS2ccJ3tHsvM5ZuvlF6/lfTdekNcuNMjmB+sHpNIZopXRlvPzwMsLbP9n59zl/s9tAGZ2IfB64CL/Nv9hZrP/shERkWlVxJ5IZiab4Z/lpF0oMuD3M6PDM8jwnx2Ic7BziJdeuBrIL+mJJdJTTtgNbGitm5jh7xpiy8r6aXu7R8Mh6qrC+Rn+kYXP8G9b5ZXEzMfE3XgyTSyZLthmdMvKegZHU5zLOdB65Jk+6qq8NRX2HOuZ8r6DDj3bpimdKsZv7d5IKuN4+1cepmMgzu9ft3XCdda11E6o4e8YiJNMuzmX9LQ3VvMHz9uaN8k4O2nXLzlLVUhbTufcz4Gp3/wxrwK+7pwbdc4dBQ4BV83b4ERERAG/TDTXSbvebb3Au6Zq8vsI2k3OJOC/97BXzvPrl60D8jP8I4n0lD34AxtaazkxLsN/pGs4G0RPp6kmms3qw1i2fy696WeqrirCxrZanp6HDH9QolQow795ZX6nnpFEiv0dg7zhqk1URULcsb9zyvsO5kpsnWMNP3iLkl26oZk9x3vZ2l7P83e2T7jOhtZaeoYTees9POOXQc014C8kWGQryPAn1aXnHWa21y/5CSaprAdO5FznpL9NRETmSUXviaSweDJNyKBqDoFKbYFVSMcLsvEjMyjpuefgOVrrojxn2wpgrJ86QCyZKuogZWNrHYPxVDawTaQyHO8ZYevK4rLOjTURBnNKespRww9eWc+hecjw9/lnLFpqJ2b4twYBv5+p33uyn3TGce32FTxn6wrumi7g7xxiVWN1yQ6OfutZGwD4vWu3FCzlCtZKOJ2T5Q8mbM9LwD+uLadX0rP8M/yT+ASwDbgcOAN8dKZ3YGY3m9keM9vT1dVV4uGJiJRfgcXg54UCfpkglkhTEw1P6Ic+E0HgPVXAH9ThDxWZ4XfO8ctD3Tx3+0pqomEaqyPZFVODcReb4QeydfzP9AyTzrjiM/y10WyQD2UM+Fc3cKRruOjXL/C9R0/x9QeemfTyoEyqUIZ/fUst0bBx9JwX8Af1+1dsbOWGXas40j08ZZ/+Q11Dc67fz/W6Z2/kI//tEl63e2PBy9cXaMP6TM8I4ZCxtmXiysZzFbT5TAVdejJu2jKx5co5d9Y5l3bOZYBPMVa2cwrIfcM2+NsK3ccnnXO7nXO729snnsEREZHiKOCXCeKp9JSBejFqo2GqwqEpyxmCPvwjRbblPNw1TMdAnGu3rQSgtb4qL8M/kihu3MHE3BM9XhB4qMgOPYHz2uo40DFIxq/TDur5G2oWduHql1ywmrTzathn0p7z3+86xL/eeWjSy7MZ/gIBfyQcYlNbXTbD//DxPrasrKe1voobdq0C4M5Jsvydg3H2nuznik0tRY91OtWRML/9bK+cqJAgw59bx/9MzwjrWmrm1IN/MuO79KTSmXl5nKXAzHKXgX4NEHTwuRV4vZlVm9kWYAfwwEKPT0SkklTmnkimFEtk5jRhF7wa/unKa4IVSodHiyvp+aXfjvO67WMBf8/IWKY9nkwX1VloY1t+hn+srry4gP+521dybjjBAb9+vj+WpLEmsuCZ3N2b2/j7V1/Mz57u4n3ffbyoha+GR1Mc6hziVF8sbx5Crv5YkOGfWNIDY605nXM8eqI3G8BvbKtjx6qGSct6vvvwKdIZx2/6ZTgLYXVTDZGQ5fXiP9E7Mi/lPJCb4XfZfyth0q6ZfQ24DzjfzE6a2VuBfzCzx81sL/BC4F0Azrl9wDeBJ4Hbgbc752bWqktERGZkYVOSsiTEU+k5TdgFaK2rmjRgDIRCRl1VuOhJu/cc6mZjWy2bVnjBWltdlO6h/Az/upbpA/7m2igN1ZFsmcfhriHWNNVkS4ymc+12b/7ALw91c8HaJgbiyQWdsJvr9Vdt4nR/nI/fcZC1zbW86yU7p7z+k2cG8GNRDnQM8uzNbROu05ut4S/8nLasrOcXB7s5fm6E7qEEV+YsGHbDrlV89pdHGRpN5b2ezjm+uecEzzqvtegDq1IIh4w1zTUTavhf4nd5mo/Hg5xJu2mXnci7nDnn3lBg82emuP7fA38/fyMSEZFcy39PJDMW92v45+KdL97BZ27aPe316qoiRfXhT6Uz/OrIuWx2H/wM/yxKesb34j88gw49AGuba9nWXp9dAGwgllzw+v1c73rxDn7rWRv4lzsO8l+PnZ7yuo+d6Mv+vr+jcIefvpEkVX770UK2rGxgNJXhNn+hrdwSnRt2rSKZdtxzMH+C5SMn+jjcNZydZLuQcltzDo+m6B5KzHnRrckUnLRbARl+ERFZ3BTwywSlqOFf2VDNjtWN016vvjpcVA3/fUfOMRhP8YLzV2W3tdVV5fXhL7akB8Z68TvnONI584mk121fyf1HekikMgzEUjTVlu9kmZnxwddewiXrm/nwD/czmpr8AOrxU/2saaqhsSbCgY6BgtfpjyVorotOOml780ovWL7loZPUVYU5P+d9ftZ5rTTVRPjJk/llPd/ac5KaaIhXXrqWhbahpTZb0hO0Y53/kp6xSbsV3pZTREQWAe2JZIKBWGrBJqDWV0WKKum57fEz1FeF83qtt9ZXMZJIZ9cNKDbDD/gZ/hidg6MMjqZmHPBfu30lsWSaR57ppb/MGX7wAs33vGIXp/pifOVXk3fgefxkP5duaOb81Y0cmCTD3zucnLScB8i2Lz3cNcylG5rzAtpIOMQrL13Hdx45ye1PdABe96TvP3aaGy9eS2MZSp/Wt9b6i21l5rUHP0yctJus7LacIiIyjcv/5icc6Sp9i+3xFPDLBGcH4qxuLH3LwkLqq8PTTtpNpTP8aN9ZXnTB6rxSo7Z6b45A70gC5xyxZHEr7YIX8A+Npnj4uNdWcqYB/9VbVxAyr46/nDX8ua7dvpJrt6/g3+46VLBV50A8yZHuYS7b2ML5axrZ3zFYcKJvXyxB6xTzL1Y3VWcPrHLr9wN/9WsXctmGFt759Ud46HgPP9rXweBoit/cvfDlPOB16sk46OiP88w89uCHsQx/0DWpUlbaFRGR2fuhnyCbT9MG/P4KiZ1m9kTOtn80s/3+CorfNbMWf/tmM4uZ2aP+z3/O49hlHqTSGbqHRlndVL0gj1dXFZm2pOdXR3roGU5w4yVr8rYHQWnPcIJ40guwaovoww9ka7jvPuDVms+khh+8ib+XbmjhnkPdiyLDH/izl+2iZzjBp39xZMJlT5zqB+CS9c3sWtvEYDzF6f74hOv1jSRpLtCSM2Bm2RV3rygQ8NdWhfnMTbtZ21zDW7+wh0/+/AgbWmu5ZsuK2T6tOQl68Z/qi3GiZ4TGmsi8vV9BcD/WpSejkh4RESm7YvZEnwdePm7bT4CLnXOXAk8D78257LBz7nL/522lGaYslHPDCTIOVjUtTIa/oTqSl43+wr3HuNefDBu47Ykz1FWF8+r3ISfDP5zMHjTUFtldKFh862dPd1FXFWbNLJ7vddtX8tjJfkYSaZoWScB/2cYWXnHxGj79i6OcGxrNu2zvyZyAf41Xd1+ojr9vZOqSHhhbcXeynvorGqr5wu9dRdiMJ88M8N+u3FBwJdyFkO3F3xvjmR6vJedcFpWbStCRJ5vhz7hsmY+IiEi5TBsdOed+DvSM2/Zj51wQpf0Kb6VEWQbODngZ39ULFPDXVYUZ8bv0PPxML++/dR9v+/JD2XGk0hl+9EQHN+xaNaFzUFu9F5T2jCSI+XX8xay0C2OLb3UMxNnW3jCrAPDa7StJ+5ncxZLhB/iTl+5kJJHiP+4+nLf98ZP9bGyrpbW+ip3+RNtCnXr6Ygla66duqfrrl63jDVdtYmXD5GeCzltRz2ff/GxefMEqfufqTbN4JqWxLmfxrSDgny9BR57gc5GqkLacIiKyuJViT/R7wA9z/t5iZo+Y2c/M7PrJbmRmN5vZHjPb09XVNdnVZIGdHfCywgtV0lNf7U3adc7xD7fvp7Uuymgqk11I6oGjPZwbTvDKSyZ2dwlKevpGEsT8g4Ziu/Q010Zp8icmb2ufWTlP4MrzWrLrFSymgH/7qkZ+81kb+NJ9x/NWmN17qo9L17cA3njXt9Sy/0x+wB9PpoknM9M+n5dfvIYPvfaSacdy2cYWPn3TsxfsALKQmmiYlQ3VnOwd4URvbF4D/nChSbuq4RcRkTKbU8BvZu8DUsBX/E1ngE3OuSuAdwNfNbOmQrd1zn3SObfbObe7vb290FWkDBY6w19fHWY4keYXB7v51ZEe3vmiHfzpy87np0918r1HT/ODx89QG51YzgNe0Grm1fAHGf6ZtBMNsvwznbAbqI6EucqvSy9nW85C3vninWDwsZ88DUDvcIITPTEu3dCcvc75ayZ26ukLFt2aooZ/KVrfUsMjz/SRSGXmrQc/5EzazWvLqYBfRETKa9YBv5m9Gfg14Hed3+rDOTfqnDvn//4QcBiYeulPWVQ6B+KEDFZMU9JRKnVVEdIZxwdve4oNrbW84epNvOXaLVy5qYX337qPHz7RwQ0XrCqYuY+EQzTXRukdTmTLgort0gNjdfzbVs1+5dfr/FV3F1OGH7y69Tddcx63PHySg2cHeTyYsDsu4D/cNUQilclu64t56xpM1aVnKVrfWsvBTq/t2Xxm+HPbcjrnSGdU0iMiIuU3qz2Rmb0c+DPgN5xzIznb280s7P++FdgBTGwXIotWx0Cc9sbqBess0lDtZcb3dwzyrhfvpDoSJhwy/vG3LiOWTNMzSTlPoK2uip6RZLakp2YGAX+Q6Z1thh/g1Ves5w1XbeLCtc3TX3mB/dELt1NXFeGjP36avSf7ALh4/dg4d61pJJVxHOke6//bO+xn+BfZAcxcBRN3gXnN8Edy2nIGq+2qpEdERMqtmLacXwPuA843s5Nm9lbg34BG4Cfj2m8+D9hrZo8C3wbe5pzrKXS/sjidHRhd0HrrICO/c3UDr75ifXb7tvYG3nfjBWxsq+UF509e8tVaX0XvcO6k3eID/qu3tLGtvT67cuxsrGqs4UOvvaTouQMLqa2+ij+4fiu37+vglodPsXVlfd56AbvWeNV2uXX8/X6Gf6q2nEtREPCb5Qf/pRbNacsZrLartpwiIsvD8XPDPHair9zDmJVpC4+dc28osPkzk1z3FuCWuQ5KyufsQDxb274Qgi4vf/qyXdkJj4GbnruZm567ecrbt9ZFOd0XHyvpiRZfS//Si9bw0ovWTH/FJeyt12/hi/cd42j3MK+6fF3eZVvb64mGLa9TT1DDv/xKerzP9LrmWqoi8xeAB+U7qZwMv9pyiogsD8//x7sBOPbhV5Z3ILOg1JPk6RxcuEW3AJ63s50f/K/reMmFq2d1+9a6KnpHEsT8Pvw1VfpI52qojvD2F24HvP77uaLhENvaG/J68fcu20m7XlZ/Y9v8ZfdhLMOfTDtSfi9+BfwiIlJui6u1iJTVaMqrmV/Ikp5wyLho3ezr39vqq+jJm7Srj/R4v3vNJkYSKV6TUzIV2LWmkQeOjlXd9cUSVIVDM+p2tBQEq+3O54Rd8FYhDoeMdMZle/GrpEdERMpNeyLJ6hpc2B78pdBaX8VoKkPPsFd7vtwC1VKojoR5xw07WFFgkazz1zRxuj9Ov5/Z7x9J0lIXnbeVaMulqSbCyy9aw4svmN2ZpJmIhIxkJkMyo0m7IiKyOCjgl6xg0a1VZVwkaaba/FrzU30xqiKhCfMAZGpXbGoB4H9+/RHODY3SO5JYduU84GXe//ONz1qQORuRkJHKK+nR16yIiJSX9kSS1RksutW4dAL+1vqxgH8mHXrEc/WWNv7u1RfzqyPneOXH7+HJMwO01C6vCbsLLRIO5U/aVYZfRETKTAG/ZI2tsrt0Snra6r1s9KnemMp5ZsHM+O/XnMd3/sdzqY6GONETW5YZ/oUUDRvJnLacUdXwi4hImWlPJFlnB0eJhm1JtWQMxto1NLooe+EvFRevb+b7//M63vzczbz2yg3lHs6SFgl5Gf6U2nKKiMgioZYmknW2P86qxhpCSyhAafNLepzThN25aqyJ8oHfuKjcw1jyImGvhj+ZVoZfRGQ5iifTnOwdYfuqxnIPpWjaE0nW2cH4kirnAWiqiRIcn6iGXxaDaDjkl/Sohl9EZDl659cf4cX/9HNG/DWAlgIF/JJ1dmB0QXvwl0IoNFaCVKse/LIIeF16xkp61DlKRGR5ue/wOQCSKVfmkRRPAb9knR2IL7mAH8Y69dRG9XGW8ouEQ6Q0aVdERBYR7YkEgJFEisF4ilVLrKQHxnrxa5VdWQzGZ/g1aVdERMpNAb8A0OkvurWUevAHWv3WnDWatCuLQCRspDKatCsiIouH9kQC5PbgX3oBf9CpR5N2ZTGIhkIk0xlN2hURkUVDAb8AXg9+WFqLbgVa6xTwy+Ixvi1nJKSvWRERKS/tiQSATj/Dv2oJZ/hV0iOLQSRoy+nX8EeV4RcRkTJTwC8AdPTHqYmGaKpZehNfleGXxSQaTNr1u/REVMMvIiJlpj2RAF5Jz+qmGsyWXjayLduWUwG/lN9YSY+f4VeXHhGRZalraLTcQyiaAn4Blm4Pfsjpw68MvywCXh/+DOmMFt4SEVmOguToi//pZ9x7qLvMoymOAn4BvBr+pRrwX7C2kZuecx7Xbl9Z7qGIeH34c9pyqqRHRGT5+p1P38+JnpFyD2Na2hMJzjnODoyyunHpdegBqI6E+etXXczKhqU5flleIqEQqbTLtuXUpF0RkeWtbyQ5p9s750o0kskp4BcGR1PEkuklm+EXWUyiYfP68Kstp4hIRVuIQL5Y2hMJZ/uDlpzKkIvM1dhKu8rwi4jI9EZTmXl/jKICfjP7rJl1mtkTOdvazOwnZnbQ/7fV325m9nEzO2Rme83syvkavJTGke5hAM5bUV/mkYgsfZHsSrsZwiFbkp2vRERk4ZwbTsz7YxSb4f888PJx294D3OGc2wHc4f8N8Apgh/9zM/CJuQ9T5tOhziEAtq9qKPNIRJa+qN+WM5V2RNShR0REFoGiAn7n3M+BnnGbXwV8wf/9C8Crc7Z/0Xl+BbSY2doSjFXmydNnB1nfUktD9dJbdEtksQnacibTjqg69IiILDuLqTa/WHPZG612zp3xf+8AVvu/rwdO5FzvpL9NFqmDZ4eU3RcpkWjISKZdtqRHRESk3EqSfnLeoc6MDnfM7GYz22Nme7q6ukoxDJmFdMZxuGuInasV8IuUQtB3P5HKaMKuiMgyVOzcrMV0ImAuAf/ZoFTH/7fT334K2JhzvQ3+tjzOuU8653Y753a3t7fPYRgyFyd6RhhNZdixqrHcQxFZFoKsfjyZVktOERFZFOayN7oVuMn//Sbgeznb3+R367kG6M8p/ZFF5mAwYVcZfpGSCLL6sWSaiDL8IiIyjYXYUxQ1S9PMvga8AFhpZieB9wMfBr5pZm8FjgOv869+G3AjcAgYAd5S4jFLCT19dhCAHarhFymJIKsfS2Y0aVdERKa1EJU/RQX8zrk3THLRiwpc1wFvn8ugZOEc6hxibXMNjTXRcg9FZFkIMvzxRFptOUVEKsD/+MpDbF5Rz5d//+pZ3X4hav2VfqpwT58dZMdq1e+LlEowadcr6amMr1gtzigilexkb4x7DnWXexhTqoy9kRSUzjgOdQ6pnEekhIKsfiyZrqQuPZ9HizOKiORZRE16FPBXslO9Mb9DjwJ+kVIJ6vZjFVTSo8UZRUQWNwX8FSw7YVclPSIlE3TmGU1VfFtOLc4oIrJIVPTeqNJlW3Iqwy9SMpFsH/6M2nL6ZrM4I2iBRhFZnIpcd6toP3myg83v+QHDo6nS3nEOBfwV7ODZQdY01dBcqw49IqUSZPVHEqmKmbQ7iTktzghaoFFEKkP3UAKA4+dG5u0xKnpvVOkOdg6xQwtuiZRUkNXPOIhWSA3/JLQ4o4hUpN7hBJvf8wO+v/d0uYeSpYC/QmX8Dj0q5xEprdzFtiqlpMdfnPE+4HwzO+kvyPhh4CVmdhB4sf83eIszHsFbnPFTwB+VYcgiIvPmcJdXMv25Xx4r70ByFLXwliw/p/pixJJpdmrCrkhJ5XbmqZSSHi3OKCIykdpyStkd7PQ79CjDL1JSuUF+hZf0iIjIIqGAv0I9fdY73bRjlTL8IqWUu9hWpWT4RUQqiVtMqfsiaW9UoY52DbOyoZrmOnXoESml3N77FbTSroiILGIK+CtUXyzBivqqcg9DZNnJDfLDKukREZFFQAF/hRqIpWis0ZxtkVLLDfIrfKVdEZFlqdQLby0E7Y0q1OBokiYtuCVScrltOVXSIyJSwRZRsb8C/gqlDL/I/Iho0q6IyLK2iOL4omlvVKEG40kF/CLzIG/Srmr4RUSWnf5YstxDmDEF/BXIOcdgPEVTjUp6REpNbTlFRGSx0d6oAsWSaVIZR6MCfpGSyw3yI6rhFxGRRUABfwUajKcAaKpVSY9IqUVyynii6tIjIiKLgPZGFWjArz1Thl+k9HK79KgPv4hI5VpMc3sV8FeggSDDr0m7IiWXG+OrLaeISOX47iMnyz2ESSngr0ADcWX4ReaLmWUDfU3aFRGpHO/6xmO4Rdqzc9YpXjM7H/hGzqatwF8BLcAfAF3+9r9wzt0228eR0gtq+JtVwy8yLyKhEMl0Oq+eX0REpFxmHfE55w4AlwOYWRg4BXwXeAvwz865/1uKAUrpDSrDLzKvImGDZH49v4iISLmUam/0IuCwc+54ie5P5tFAzMvwa+EtkfkRBPpqyykiIotBqQL+1wNfy/n7HWa218w+a2atJXoMKZHBeJJIyKiNhss9FJFlKSjliagtp4hIxVpM5fxz3huZWRXwG8C3/E2fALbhlfucAT46ye1uNrM9Zranq6ur0FVkngzEkzTWRDBT9lFkPgQZfnXpERGRxaAU6adXAA87584COOfOOufSzrkM8CngqkI3cs590jm32zm3u729vQTDkGINxlM01ap+X2S+RNSlR0SkIi2mrH6uUuyN3kBOOY+Zrc257DXAEyV4DCmhgVhS9fsi8yicLelRhl9EpFI9fqq/3EPImlPUZ2b1wEuAP8zZ/A9mdjneAmPHxl0mi8BgPEWTOvSIzJuoX7uvgF9ERBaDOQX8zrlhYMW4bW+c04hk3g3GU2xeWVfuYYgsWyrpERGRxUR7owrkTdpVhl9kvkQ0aVdERBYRBfwVSCU9IvMrqracIiKyiGhvVGHSGcfQaEqTdkXmUVDSowy/iEhlWaRNehTwV5qhuLfKrtpyisyfsZV29RUrIiLlp71RhRmIJwGU4ReZRxG15RQRkUVEAX+FCQJ+1fCLzJ+xSbv6ihURkfLT3qjCDAYlPcrwi8ybILMfVoZfREQWAQX8FWYg5mf4VcMvMm/UllNERBYTBfwVJsjwq4ZfZP5k23KqpEdEpKI4tzj79GhvVGHGJu0qwy8yX7Ir7aqkR0REFgEF/BVGGX6R+adJuyIisphob1RhBuNJaqNhBSIi8ygaMsw0aVdERBYHRX0VZiCWoqlW2X2R+RQJh4iG9PUqIiKLgyK/CjM4mlT9vsg8e9Z5rZzqjZV7GCIissAW55RdBfwVZyCWUg9+kXl24yVrufGSteUehoiICKCSnoozGFeGX0RERKSSKOCvMAPxlDr0iIiIiFQQBfwVZjCe1Cq7IiIiIhVEAX+FUYZfREREpLIo4K8g8WSaRCpDk2r4RURERErOLdI2PQr4K0iwyq669IiIiIhUDgX8FWQgngRQDb+IiIhIBVGqt4IEGX7V8IvIQjGzY8AgkAZSzrndZtYGfAPYDBwDXuec6y3XGEVEljtl+CvIQMzL8KsPv4gssBc65y53zu32/34PcIdzbgdwh/+3iIjMkzkH/GZ2zMweN7NHzWyPv63NzH5iZgf9f1vnPlSZq7EafgX8IlJWrwK+4P/+BeDV5RuKiMjyV6oMv7I3S8BgPMjwq6RHRBaMA35sZg+Z2c3+ttXOuTP+7x3A6vIMTUSktByLs03PfEV+rwJe4P/+BeBu4M/n6bGkSJq0KyJlcJ1z7pSZrQJ+Ymb7cy90zjkzK7iH9A8QbgbYtGnT/I9URGSZKkWGf1bZGzO72cz2mNmerq6uEgxDpjMYTxEyqK8Kl3soIlIhnHOn/H87ge8CVwFnzWwtgP9v5yS3/aRzbrdzbnd7e/tCDVlEZNkpRcB/nXPuSuAVwNvN7Hm5FzrnHEw8v6Ev8oU3EEvSWBPFzMo9FBGpAGZWb2aNwe/AS4EngFuBm/yr3QR8rzwjFBGpDHMu6cnN3phZXvbGOXdmquyNLKzBeEr1+yKykFYD3/WTDBHgq865283sQeCbZvZW4DjwujKOUURk2ZtT9OdnbELOucGc7M3fMJa9+TDK3iwaA/GUOvSIyIJxzh0BLiuw/RzwooUfkYhIZZprulfZmyVkIJ5Uhl9ERERknrjF2aRnbgG/sjdLy2A8xfqW2nIPQ0REREQWkFbarSADsSRNtcrwi4iIiFQSBfwVZDCeVA2/iIiISIVRwF8hMhnH4GiKJtXwi4iIiFQUBfwV4txwAuegpa6q3EMRERERkQWkgL9C7D3ZB8DF65vLOxARERERWVAK+CvEw8/0EgkZlyjgFxEREakoCvgrxMPH+7hgbRO1VeFyD0VEREREFpAC/gqQzjgeO9nHlZtayj0UEREREVlgCvgrwIGOQUYSaa7Y1FruoYiIiIjIAlPAXwEeOdELwJUK+EVEREQqjgL+CvDw8T5W1Fexsa223EMRERERWbacK/cIClPAXwEeOdHLFZtaMbNyD0VEREREFpgC/mWubyTBka5hrjyvpdxDEREREZEyUMC/zD1yog+AKzaqfl9ERESkEingX+YeOd5LyOCyjVpwS0RERKQSKeBf4tIZx0PHeya9/JETfexa00RdVWQBRyUiIiIii4UC/iXuR/s6+G+fuI9DnYMTLstkHI8+06f6fRERWZTiyTSn+2LlHoZIyTgWZ5seBfxL3PFzIwAc7hqecNnBziEGR1Oq3xcRkUXjTH+Mv/mvJ0lnHG/53IM898N3lntIIsue6jyWuI5+LzNyomdkwmWPPOMvuHWeAn4RkUqy/S9uo6Uuyp6/fEm5hwJAMp3h+LlhVjfV8M6vP8oDR3t43s6V3HfkXLmHJlIRFPCXQCbjuPWx07zy0rVEwwt70uR0fxyAZwoE/AfODlJfFWbziroFHZOIiJRXKuPoHkqUexhZH7ptP5/95dG8bW/+3INlGo1I5VFJzwz0jSTIZCbWZj14rIc//saj3PHU2QUfU8cUAf/R7mE2r6zXglsiIjJv9p7sY/N7fsCx7omlpYE9UzSXEJH5p4C/SAPxJM/98J3c8vDJCZcF9fNBPf1COjNNwL9lZf1CD0lERJa50VSaD9y6j76RBLc85O0X7z7Qmb3cOceP9nXw/b2nyzVEEclRkQH/P/5oP/9x96EZ3eZw5xAjiTSPneybcNnR7iGgcNA9W1+9/xnec8veKa8zmkrTPTRKOGSc7InlnX1IpDKc6BlhqwJ+ERGZgf/z/57gvsNebX3u/LBUOsObP/cADx3v5ZaHTvH5e4/xodv2F7yPL//qOH/4pYd4x1cfWZAxiywWbnE26Zl9wG9mG83sLjN70sz2mdk7/e0fMLNTZvao/3Nj6YZbGrc8dIov3XccN4N35ah/qvJw58RTlke7vS/EUgb8/++RU3z9wRN0DsQnvU7nwCgAF69vJpHOcHZw7LonekfIONisgF9ERGbgS786zhs+9Sve/Y1Huf4f7sqWq57qi3H3gS7e9Jn7+fe7vKTZNx86UfA+7j7QtWDjFZHpzSXDnwL+xDl3IXAN8HYzu9C/7J+dc5f7P7fNeZSz0DeS4FSB3r4jiRQdA3HO9Mc52Vt879+gNvFw19CEy4IMf6FOObPhnGN/xwAw9ZdmUM5z9ZY2AJ7JKSk66pcZqaRHRESKlc45U/ydR04B8OTpgbzrDCfS2f3rYs1miki+WQf8zrkzzrmH/d8HgaeA9aUaWLGGRlMFt7//1n288TP3T9h+rHssKH7gaPGTiI74AX/n4CgD8WR2eyqd4ZmeEUIGJ3tjeV+Ws9UxEGcg7j2vO/ZPPhH4jN+S86rNfsCfc8ARnJFQwC8iIsV64lT/pJcZxTeAUK8IkcWlJDX8ZrYZuAIIIux3mNleM/usmc1bE/iP/vgAV//9TwsG2Y+e6ONI1zAjifwDgqM5XQRmEvAf7R6mym+5eSRnkavTfXGSacflG1tIZVw2CJ+L/R3eqrk7Vzdwz8FuRlPpgtcLMvy7N7cSMjiRc8biSPcwbfVVtNRVzXk8IiIik1GSX2Txm3PAb2YNwC3AHzvnBoBPANuAy4EzwEcnud3NZrbHzPZ0dc2u1u+8FfUMJ9LZkprAQDyZ7ZhzqDP/smPnvGD92u0reOBYcQG/c45j3cNcs20F4E3gDRzxH/v5O1cBpanj33/GC/jf9vxtDCfSPHi0t+D1OvrjNNZEaKmrYm1zbV5J0bHuYfXfFxGRGSmUmZ8uoP/ifcfnZSwiUjpzCvjNLIoX7H/FOfcdAOfcWedc2jmXAT4FXFXots65Tzrndjvndre3t8/q8S/b0AzA3pP5pyCDgBng4Nn8gP9I1zBrmmp4/s52jnYP0zk4+aTYQNfgKMOJNM/f2U4kZHl1/MEZg+ftXAmUpo7/QMcAa5treMXFa6mOhCYt6zndF2Ntcw0Am9rqJpT0bFnZMOexiIjI0rbvdD/ffLDw5NpijKbSXP3Bn3LnFCWmE6mmRyrTYj3jNZcuPQZ8BnjKOfdPOdvX5lztNcATsx/e1La2N1BXFZ4Q8O873e+PBZ7uHMy77Gj3EFtW1nPVFi9bP1n2PFdQv79zdQObVtTlBfzHuodprI5wyfpmIiErTYa/Y5Dz1zRSWxXmOdtWcOf+zoIdhToG4qxprgXyA/7hUW9i8tZ21e+LiFS6V378Hv5smjbPUzndF+fswCh/+4OnSjgqEVlIc8nwXwu8EbhhXAvOfzCzx81sL/BC4F2lGGgh4ZBx8bpm9o7rjf/k6QFWNlSxc1Ujh8Zl+I92D7OlvZ6L1jVRGw3zwNFz0z5O0KFn84p6trU3ZBfaAu9gYEt7PZFwiPWttTzTM7ca/mQ6w+GuIc5f0wjAi3at4vi5kexBR64z/XHWBRn+FXV0DY4SS6SzZUubVyjgFxFZjjr64wzmNJCYTDw5Ngfsdf/ffXzl/qnLb2YyMVdElo65dOm5xzlnzrlLc1twOufe6Jy7xN/+G865M6Uc8HiXbGhm3+kBUulMdtu+0wNcsLaJHasb8jL8fSMJekeSbFlRTzQc4lnntXJ/ERN3j3YPUxUJsa6llm3tDRw/N5x9vKPdw9nAenxZzWwc6RommXZcsKYJgBfu8uYG3LW/M+96iVSG7qFR1vgB/8Y2r17/RO9IthOROvSIiCxP13zoDl72zz8veFkiNbY/3PV/bs/+/sDRHt733alPuheq4S/2EMBybqwuPSKLy5JfaffSDc2MpjI87WfyE6kMBzsHuWhdMztWNXKyN5bt1DO+VeVVW9o4cHaQ/pGpsyRH/Amw4ZCxrb2eZNpxojfGaMrrRRzc38a2ujnX8Af994MM/4bWOs5f3cid4wL+swNxnCOvhh+8XvzBJObNKzVpV0RkuTrtd2o72TuSl+0/0j1xvZiFMJPFLEVkYUXKPYC5unRDCwCPn+rjwnVNHOocIpl2XLiuiWjIcM5bHfeSDc1jAX/7WMDvHOw53sOLLlgNQOdAnKbaKDXRcPYxjnUPZ4P6bau8ibCHO4dIpTM4R7ZWflNbHT3DCQbjSRprorN6Pgc6BomEjG3tYxNuX7hrFZ/+xREG4kma/Pvt8Ffgza3hB69L0JHuYdY211BXteTfXhERyfHth07SVDP23X6qL8Z1H7kLgOt3rGRDax1vvOa8cg1PpOIt1gPfJZ/hP6+tjsaaSHbibjBh96J1XkkPwEG/rOdY9zAhg42tXnB8+cYWomHjgaM9OOf4zD1HufYjd/J3P3gye//pjOP4uZHsQcI2v/PN4a6hbF19cDAQBN0n5lDHf6BjkK3t9VRFxt6aG3atIpVx/PJgd3Zb0IM/qOFvrYvSUB3hmZ6RvDIjERFZPv73tx7j5i89lP37n3/ydPb3Xxzs5msPPMMHbt1XjqHll/SUZQQiMpklH/CHQsYl65t53F8d8MkzA9RGw2xeUc95K+qJho2Dft/8I93DbGyrywbTNdEwl21o4ecHu3nblx/ib7//JNWRMN/fe4akX6N/ui9GIp1hqx/UN9dFWdlQzeGuobHJvEFJT+tYln229ncMssuv3w9csamF+qowvzycE/D7y5oHNfxmxobWWk72jnhnJNShR0Rk2SsUWPeOJBZ8HJCf2VQNv8jisuQDfvDKep46M8BoKu1P2G0kHDKi4RBbVtZz8KyX4T+aU5oTuGpLG0+dGeCOpzr5y1dewEdfdxl9I0nuO3wuexvI73izrb2ew13DHO0eZmVDVbbMZizDP7uAfyCe5FRfLFu/H4iGQ1yzdQW/PDTWUehMf5yG6khe6dCmtjr2nuyndySZPUAREZGl6XO/PMrm9/yAZDpDIpXh4vf/aMJ1vvXQyZI+5mwW3hKRxW+ZBPzNJNOO/WcGeer0ABeuG8uQ71jVyMHOoexqueNLXV5zxXqu276Sb/zhc/j967fy/J3t1FeFue1xr7nQ+Lp/8Or4D3V6JT25BxDNdVGaaiKzzvA/3eEdmOwaF/ADXLt9JUe7hznZ6913R388O2E3sKmtjs7BUUAtOUVElqpEKsNgPMlHf+yV64yMptn5lz9kaDRV1O1LHaArWS+y9C2LgP+S9d6Ku7c9fobB0RQXrWvOXrZjdQPP9IzwTM8Iw4n0hMWodqxu5Mu/fzXPOq8V8Mp8XnTBan60r4NkOsPR7mEaqiO0N1Rnb7OtvYH+WJInTvVPCKw3rZjYmjOdKe7rd38Q8K9tmnDZdTu8lXzv9bP8Z/pj2XKe3McOqKRHRGTp6I8liSXS7Dvdz86//CGXfODH2QA/WFtlscuv4ddhgshisiwC/g2ttbTWRfm2f2rzwrX5GX7n4KdPeW0ti+lNf+Mla+kdSfKrI+e8lpwr6/K+yLb5wfRIIj0hsN40rjXn7U+c4Yq/+TH35Ey4ncyBjkEaayLZibi5dqxqoL2xmnsOefdzpkCGP+jFHw5Zdj6BiIgsbt995CSX/fWPueCvbueVH79nzvc3my4hqXSGWCI9/RVL/LiFfOT2/dmz7CJLzWItgVsWAb+ZccmGFs4NJwiHLK8GPujU8+N9HUBxpS4vOD8o6+nwW3I25F2e2zJzfK38xrY6TvbGSGcciVSGD962n4F4ird9+SGePD0w5ePu7xjg/NWNeQcXuc/xuu0r+eWhbhKpDF1Do9mWnIFgDsHG1tq8Lj8iIrI4dA7E+c7D+XX37/rGY1PeJjPDQHo2AcdbPv8gF/zV7dNfcR79eF8H8WSaT9x9mD/6ysO8+t9/yU+ePMvPnu4q67hEloNlExVetsEr49nWXp/XQ3/zinoiIePBYz3Z1XKnUxMNc8MFq7n9iTOc7B2ZcFZgfUst1X5AvXnlxAx/Ip3h7ECcbzz4DM/0jPDB11xCQ3WEt3z+AU753XUyGcevjpzj3+86xL/deZB/u/MgT50ZnDBhN9e121dybjjBLw524RwTzgSsb6nFbOKYRERkcfjvn7mfd3/zMfpjUy/4mKvIqtA5+YV/FvpjPz044bInz0ydrAoUSlYB/Pq/3pNtnT2Zx070cfOXHuKv/2uspeijJ/r4gy/u4abPPlDU44vI5JbNykxBHX9u/T5AVSTE5pX1HOocyq6WW4xXXrKG/3rsNABbxq1YGwoZW1bWs79jcGINv59lP9AxyMfvPMRVm9t4w1UbufK8Fn7rE/fx5s8+wCsvXcstD5+c0K8/ZF5QP5lrt68AyJYuja/hr4mGuWbLCq6b4j5ERKQ8vnDvseyq8NFw8TXuMy6VmebqqXSGSNhLWqUzjhs+enf2sp88eXbC9YP5ZcXMR0tnHO+/9QlO9o2VtgZts6fS47cSnW4dmxM9IwyNpugbSfLwM728/YXbOdMfY23z9Mk8kfGcc8SS6ZIuVHrpB37Mf/73K0t2f6WybAL+yze2EA5ZNtOfa+fqBj/gLz7z/YLzV1FXFfbq9MeV9IA3T2A0lck7mwBjAf+HfvgUXYOjfOJ3r8TM2LWmif/vjc/ips89wMd+epBrt6/g3S/ZyUsuXJM9W2CQ/RIuZG1zLVvb6/npU2ezf4/3tZuvKfo5iohI6R3tHqZnOJFtBhH46I8PZH+PhELce6ibZ29pm/b+fvM/75vR408Xlm9/3w951eXr+N6jp2d0v9N5/637eP8sF/16y+ceBMjOUxvv7ECcbz54go/mLDQG3r7/dz99Px/77ct59RXrZ/XYsvR9f+9p3vfdJ9jzly8mOkUcNd7XHjjBX3z3cX72py/gvBJ2NzzoH9gvJssm4F/VVMP3/+d1efX1ge2rGoGOGXWuqYmGuWHXKr6/9wxbCnwI/s+vXViwRdq6llpCBk+fHeJFu1axe/PYl/lzt6/kB//reuqqwmyY5aTa67av5Iv3HQdgbcvEyb0iIkuFmb0c+BcgDHzaOffhMg9pWrFEmrRzPHGqn6v9YD23lMU5xwv/790AfOy3L6e9sZrVTdWMpjJURcKAt9/Y+Zc/XOih5yl1sD/f/vBLD/Hoib4J23/30/cD8MffeJTLN7bw06fO0t5YzeUbWyYEcLFEms/+8ig3P2/rjILCSvPgsR7WNtcwmvIWHZ2sVKtYwRmq6e7nZO8IdVUR2uqrJlx2qi9GPJkuGOMBvOOrjwBw/l/+kIyDb7/tOXnx172Hu3nqzCArG6p4wc5VNNd5axjd7s/vPNI9zP6OQdb5idX66rHweDSV5t/uPMQfvWB70c95/IFpseKpuU2cn8qyCfgBLijQzhK8DD9MnGA7nT9+8U6evbkt+8HI1VpfRWuBD2U07M0TONUX43+/7PwCY5m8Rr8Y1/oBf31VmMbqZfX2iUgFMbMw8O/AS4CTwINmdqtz7slSPs6eYz0c7R7mhl2r6B5KcP6aRk71xejoj/Gs89roHIjzjQdP8Ecv3M63HzrBn9/yOPv/9uV88Lan+OJ9x/nAr1/I1VtX8Ip/+cWMH/uPv/FoKZ9K0frKtNLufCoU7I/3Av9AK3D5xhYuWNtE30iCR0/0caY/DsA//ugAH3/DFVy4ton+WJJnnddKOuPIODerA4FkOkPGOaoj4emvPI2fPnmW3ZtbSWUcVZFQdmHPYsSTaQ51DpHKOD5421P89u6NHO8Z4Q+u30JjTZRTfTGqIyFW5rQZzzUYT7LnWC9v+fyD2W037FrF+3/9QgbjKZpro3z1gWf4xN2HaayJ8FvP2shf/fqFE+7n0784wt/94Cm+/NaruW7HSra/74fZcrC/e/XFtDdW87KL1tA3kqChOsK+0wM88kwvH/ivJ6mKhPj337mScAhu2LWaze/5Qd59/+2rL2aDP1+xvbGabe0NeZUWQdXZR27fz5d//2qioRAO+J1P3Z93P1/9g6t57raVZHLK1P7wSw9lf3/ni3bwrpfsBOBr9z/Dv955iH+981AR78LcfOLuw3zqTbvn5b6tVG205mL37t1uz54983b/nYNx/uCLD/Gvr78ir1f9fPnQD58C4L2vuKDk990fS3LF3/yYLSvrueNPXlDy+xeRxcfMHnLOzc9eoEzM7DnAB5xzL/P/fi+Ac+5Dha4/2/3En397L9995BSJdGYuwxUBYGVDNd1Do/zGZeu4YG0TH7l9/4KPYUV9FeeGl99BnXiLwf7zb18+49sVs4+oiBTxqsYavvf2axfs8eYj0A8010a5essKWgqcdRARWULWAydy/j4JXJ17BTO7GbgZYNOmTbN6kFddsY5v7Dkx/RVFitA95K1mf+tjp7n1sfKURSnYXx6u37Ey2x0r8N4bd83b41VEwL/cfOqm3RTZbEhEZMlyzn0S+CR4Gf7Z3Mdzt63kF3/2QvpjScy8s6SGsba5hlQmQ/dQgvUttTx4rIdLNzQTDoUYHk0RCRvntdVzbniUvpEkG9vq6B9JMpxIsaK+CgckUhma66IMxJKkM46WuiriyTQjiTTtjdX0jSSIJzM01UQYSaQJmbGmuYah0RSxZJqmmghDoynCZjTVRjnRM8JIIs3qphp6hhP0DCfYsbqB4+dGSGccjTXeLjvjHA3VEc4OjDKaSnPeinrq/SYTo6kMPcMJNrbVsr6lluPnRmioidBWV8WZgThV4RCRkDEYT9E1FGdNcy01kRDJtKOlLkoskSaWTNNc6yWV4sk0tVVhUhlHPJFmZUM1wwmvS87qphqiYWMkkSaeTBNPZchknP/aOlL+ejTpjKOuKkw0HPJLX0I453XmaayJ+OU0UBUOEU+l6RnytrfUVXG0e5j66jDnhhJ0DY1y0bomIqEQ54ZGaaqNMhhP0VQT4ZETfWxqqyNkRm00zOBokjN9cUaSada31HLPwW6ef347kZBxuGuIuqoI9dVh/3WN0tEfpypiZDIQjYTo6I+xvqWOptoIITPODScwoLYq7L/XUXqHkyT8BcvCIfPLU/pprImwfZX3vtVXR0ilvY/uqb4Rdq5upDoSJpnOMDyaIpHOkEhlcEBbXRXrWmqJJdOc7B1hXUstNZEw3cOjpNOOrqFRVtRXEQ4Z6YyjOhomkcpwbmiU+uoIDTURcN7nI5HK0FgTxcwr1VnVVENVOMShziFqoiEG4ilqo2F6RxJcsr6ZWDJN2Iwj3cOsbqomkcqwvqWOkUSK0VSGzsFRVjdVMxhPsaG1liNdw6xqquZY9wiRkNFcF8U5r94dvPdyS3s9Q/EUybQjnkxTFQkRDYcYjCeJJdNUhUNUR8MMxpM45y0aOppKMxBLcYFfblUdCZHKZKiNep/9zkGvLKutvopoOEQ4ZKTSjqHRFOetqGNoNEV/LMloMs3QaJpVjdX0DCdY1VhNU200+9rEkxmOdg+xvrWWdMbr6BhLpGiqiTI4mmJTWx1Hu4epjniPEUukWdlYzZGuIX7jsvUc7xlmRX017Y3VE7podQ8l6OiPU18dZsvKeoZGUzTWRHHOzXk+xFxUREmPiMhSppIe7SdERCZTzD5C09RFRKQcHgR2mNkWM6sCXg/cWuYxiYgsSyrpERGRBeecS5nZO4Af4bXl/KxzbnZN3EVEZEoK+EVEpCycc7cBt5V7HCIiy51KekREREREljEF/CIiIiIiy9i8Bfxm9nIzO2Bmh8zsPfP1OCIiIiIiMrl5Cfhzlkx/BXAh8AYzm7j+soiIiIiIzKv5yvBfBRxyzh1xziWArwOvmqfHEhERERGRScxXwF9oyfT18/RYIiIiIiIyibJN2jWzm81sj5nt6erqKtcwRERERESWtfnqw38K2Jjz9wZ/W5Zz7pPAJwHMrMvMjs/ysVYC3bO87XKl1ySfXo+J9JpMtJhfk/PKPYBye+ihh7q1nyianu/ypue7vM3m+U67jzDn3OyGM9WdmkWAp4EX4QX6DwK/Mx+rKJrZHufc7lLf71Km1ySfXo+J9JpMpNdk+aq091bPd3nT813e5uv5zkuGX0umi4iIiIgsDvNV0qMl00VEREREFoHlsNLuJ8s9gEVIr0k+vR4T6TWZSK/J8lVp762e7/Km57u8zcvznZcafhERERERWRyWQ4ZfREREREQmsaQDfjN7uZkdMLNDZvaeco9noZnZRjO7y8yeNLN9ZvZOf3ubmf3EzA76/7aWe6wLzczCZvaImX3f/3uLmd3vf1a+YWZV5R7jQjGzFjP7tpntN7OnzOw5lf4ZMbN3+f9nnjCzr5lZTSV/Rpaz5bCfmOl3vXk+7j/nvWZ2Zc593eRf/6CZ3VSu51SMYr/Hzaza//uQf/nmnPt4r7/9gJm9rExPZVoz+Z5eDu/vTL6Dl+L7a2afNbNOM3siZ1vJ3k8ze5aZPe7f5uNmZtMOyjm3JH/wuv8cBrYCVcBjwIXlHtcCvwZrgSv93xvxWqFeCPwD8B5/+3uAj5R7rGV4bd4NfBX4vv/3N4HX+7//J/A/yj3GBXwtvgD8vv97FdBSyZ8RvFW/jwK1OZ+NN1fyZ2S5/iyX/cRMv+uBG4EfAgZcA9zvb28Djvj/tvq/t5b7+U3xvIv6Hgf+CPhP//fXA9/wf7/Qf8+rgS3+ZyFc7uc1yXMt+nt6qb+/M/0OXorvL/A84ErgiZxtJXs/gQf865p/21dMN6alnOG/CjjknDvinEsAXwdeVeYxLSjn3Bnn3MP+74PAU3j/kV6F9+WB/++ryzLAMjGzDcArgU/7fxtwA/Bt/yoV85qYWTPeF89nAJxzCedcHxX+GcHrUFZr3pohdcAZKvQzsswti/3ELL7rXwV80Xl+BbSY2VrgZcBPnHM9zrle4CfAyxfumRRvht/jua/Dt4EX+dd/FfB159yoc+4ocAjvM7GozOJ7esm/v8zsO3jJvb/OuZ8DPeM2l+T99C9rcs79ynnR/xcpYn+1lAP+9cCJnL9P+tsqkn+K6wrgfmC1c+6Mf1EHsLpc4yqTjwF/BmT8v1cAfc65lP93JX1WtgBdwOf8U+OfNrN6Kvgz4pw7Bfxf4Bm8nUw/8BCV+xlZzpbdfqLI7/rJnvdSej0+RvHf49nn5V/e719/qTzfmX5PL+n3dxbfwUv9/Q2U6v1c7/8+fvuUlnLALz4zawBuAf7YOTeQe5l/9FcxrZjM7NeATufcQ+UeyyIRwTut+Ann3BXAMN6pxKwK/Iy04mVUtgDrgHoWbxZMJKtSvusr8Hu8or6n9R1cnvdzKQf8p4CNOX9v8LdVFDOL4u0AvuKc+46/+ax/ygf/385yja8MrgV+w8yO4Z2+vwH4F7xTZMFCc5X0WTkJnHTO3e///W28HUslf0ZeDBx1znU555LAd/A+N5X6GVnOls1+Yobf9ZM976Xyesz0ezz7vPzLm4FzLJ3nO9Pv6aX+/s70O3ipv7+BUr2fp/zfx2+f0lIO+B8EdvizuqvwJnLcWuYxLSi/hu0zwFPOuX/KuehWIJjNfRPwvYUeW7k4597rnNvgnNuM95m40zn3u8BdwG/6V6uY18Q51wGcMLPz/U0vAp6kgj8jeKeRrzGzOv//UPCaVORnZJlbFvuJWXzX3wq8ye/+cQ3Q75cS/Ah4qZm1+lnWl/rbFpVZfI/nvg6/6V/f+dtf73d52QLswJvsuKjM4nt6Sb+/zPw7eEm/vzlK8n76lw2Y2TX+6/cmitlfzXTm8WL6wZvZ/DTezOz3lXs8ZXj+1+GdEtoLPOr/3IhX23YHcBD4KdBW7rGW6fV5AWPdHbbifREcAr4FVJd7fAv4OlwO7PE/J/8Pb7Z/RX9GgL8G9gNPAF/C6/JQsZ+R5fyzHPYTM/2ux+vc8e/+c34c2J1zX7/nf8YPAW8p93Mr4rlP+z0O1Ph/H/Iv35pz+/f5r8MBiuhkUsbnWfT39HJ4f2fyHbwU31/ga3jzE5J4Z3DeWsr3E9jtv3aHgX/DX0h3qh+ttCsiIiIisowt5ZIeERERERGZhgJ+EREREZFlTAG/iIiIiMgypoBfRERERGQZU8AvIiIiIrKMKeAXEREREVnGFPCLiIiIiCxjCvhFRERERJax/x/69NfX8sNkFgAAAABJRU5ErkJggg==", "text/plain": [ "
" ] }, "metadata": { "needs_background": "light" }, "output_type": "display_data" } ], "source": [ "num_frames = 10000\n", "batch_size = 32\n", "gamma = 0.99\n", "\n", "losses = []\n", "all_rewards = []\n", "ep_reward = 0\n", "\n", "state = env.reset()\n", "for frame_idx in range(1, num_frames + 1):\n", " epsilon = epsilon_by_frame(frame_idx)\n", " action = current_model.act(state, epsilon)\n", " next_state, reward, done, _ = env.step(action)\n", " replay_buffer.push(state, action, reward, next_state, done)\n", " \n", " state = next_state\n", " ep_reward += reward\n", " \n", " if done:\n", " state = env.reset()\n", " all_rewards.append(ep_reward)\n", " ep_reward = 0\n", " \n", " if len(replay_buffer) > batch_size:\n", " loss = compute_td_loss(batch_size)\n", " losses.append(loss.item())\n", " \n", " if frame_idx % 200 == 0:\n", " plot(frame_idx, all_rewards, losses)\n", " \n", " if frame_idx % 100 == 0:\n", " update_target(current_model, target_model)\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## 参考\n", "\n", "[强化学习(十二) Dueling DQN](https://www.cnblogs.com/pinard/p/9923859.html)" ] } ], "metadata": { "interpreter": { "hash": "fe38df673a99c62a9fea33a7aceda74c9b65b12ee9d076c5851d98b692a4989a" }, "kernelspec": { "display_name": "Python 3.7.10 64-bit ('py37': conda)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.7.10" }, "orig_nbformat": 4 }, "nbformat": 4, "nbformat_minor": 2 }