266 lines
28 KiB
Plaintext
266 lines
28 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 4,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import sys\n",
|
|
"from pathlib import Path\n",
|
|
"curr_path = str(Path().absolute()) # 当前路径\n",
|
|
"parent_path = str(Path().absolute().parent) # 父路径\n",
|
|
"sys.path.append(parent_path) # 添加路径到系统路径\n",
|
|
"import math\n",
|
|
"import random\n",
|
|
"\n",
|
|
"import gym\n",
|
|
"import numpy as np\n",
|
|
"\n",
|
|
"import torch\n",
|
|
"import torch.nn as nn\n",
|
|
"import torch.optim as optim\n",
|
|
"import torch.nn.functional as F\n",
|
|
"from torch.distributions import Categorical\n",
|
|
"\n",
|
|
"from IPython.display import clear_output\n",
|
|
"import matplotlib.pyplot as plt\n",
|
|
"%matplotlib inline"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 5,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"use_cuda = torch.cuda.is_available()\n",
|
|
"device = torch.device(\"cuda\" if use_cuda else \"cpu\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 6,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"from common.multiprocessing_env import SubprocVecEnv\n",
|
|
"\n",
|
|
"num_envs = 16\n",
|
|
"env_name = \"CartPole-v0\"\n",
|
|
"\n",
|
|
"def make_env():\n",
|
|
" def _thunk():\n",
|
|
" env = gym.make(env_name)\n",
|
|
" return env\n",
|
|
"\n",
|
|
" return _thunk\n",
|
|
"\n",
|
|
"envs = [make_env() for i in range(num_envs)]\n",
|
|
"envs = SubprocVecEnv(envs)\n",
|
|
"\n",
|
|
"env = gym.make(env_name)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 7,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"class ActorCritic(nn.Module):\n",
|
|
" def __init__(self, num_inputs, num_outputs, hidden_size, std=0.0):\n",
|
|
" super(ActorCritic, self).__init__()\n",
|
|
" \n",
|
|
" self.critic = nn.Sequential(\n",
|
|
" nn.Linear(num_inputs, hidden_size),\n",
|
|
" nn.ReLU(),\n",
|
|
" nn.Linear(hidden_size, 1)\n",
|
|
" )\n",
|
|
" \n",
|
|
" self.actor = nn.Sequential(\n",
|
|
" nn.Linear(num_inputs, hidden_size),\n",
|
|
" nn.ReLU(),\n",
|
|
" nn.Linear(hidden_size, num_outputs),\n",
|
|
" nn.Softmax(dim=1),\n",
|
|
" )\n",
|
|
" \n",
|
|
" def forward(self, x):\n",
|
|
" value = self.critic(x)\n",
|
|
" probs = self.actor(x)\n",
|
|
" dist = Categorical(probs)\n",
|
|
" return dist, value"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 8,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def plot(frame_idx, rewards):\n",
|
|
" clear_output(True)\n",
|
|
" plt.figure(figsize=(20,5))\n",
|
|
" plt.subplot(131)\n",
|
|
" plt.title('frame %s. reward: %s' % (frame_idx, rewards[-1]))\n",
|
|
" plt.plot(rewards)\n",
|
|
" plt.show()\n",
|
|
" \n",
|
|
"def test_env(vis=False):\n",
|
|
" state = env.reset()\n",
|
|
" if vis: env.render()\n",
|
|
" done = False\n",
|
|
" total_reward = 0\n",
|
|
" while not done:\n",
|
|
" state = torch.FloatTensor(state).unsqueeze(0).to(device)\n",
|
|
" dist, _ = model(state)\n",
|
|
" next_state, reward, done, _ = env.step(dist.sample().cpu().numpy()[0])\n",
|
|
" state = next_state\n",
|
|
" if vis: env.render()\n",
|
|
" total_reward += reward\n",
|
|
" return total_reward"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 9,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def compute_returns(next_value, rewards, masks, gamma=0.99):\n",
|
|
" R = next_value\n",
|
|
" returns = []\n",
|
|
" for step in reversed(range(len(rewards))):\n",
|
|
" R = rewards[step] + gamma * R * masks[step]\n",
|
|
" returns.insert(0, R)\n",
|
|
" return returns"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 10,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"num_inputs = envs.observation_space.shape[0]\n",
|
|
"num_outputs = envs.action_space.n\n",
|
|
"\n",
|
|
"#Hyper params:\n",
|
|
"hidden_size = 256\n",
|
|
"lr = 3e-4\n",
|
|
"num_steps = 5\n",
|
|
"\n",
|
|
"model = ActorCritic(num_inputs, num_outputs, hidden_size).to(device)\n",
|
|
"optimizer = optim.Adam(model.parameters())"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 11,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"max_frames = 20000\n",
|
|
"frame_idx = 0\n",
|
|
"test_rewards = []"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 12,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAE/CAYAAABfF5iGAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAA9TklEQVR4nO3dd3hc5ZX48e9RL1aXLEuWZLkK2+CGbYxtwJQQQonpmJBAgCwhZdN2QyDJL8nuJpuQJdmEbEJCAoEEQjeYEGpoprnbkgvutprVLFkzsqRRm/f3x1yZQVaddmek83kePZq59ehq5sw7732LGGNQSikVeaLsDkAppZRvNIErpVSE0gSulFIRShO4UkpFKE3gSikVoTSBK6VUhNIEPkqISImIbBORFhH5mt3xqOARkc+LyLt2x6Hspwl89LgDeNMYk2KMudfuYLyJyAwRWSMiDSLSJCKviEhJn22+KSK1IuIUkQdFJN5rXbGIvCkibSKyW0QuCNS+Y4GI3C8ie0TELSKf72f9FBF5wfrwPyoiP/da91UR2SQiHSLy0BDnERH5sYhUi4hDRN4SkdmB/4tUL03go8ckYOdAK0UkOoSx9JUOPA+UALnABmBN70oR+SRwJ3A+nr9jCvAfXvs/BmwFsoDvAU+LSI6/+46EiMSMdJ9ACNB5S4EvA1v6OX4c8BrwBjABKAAe8drkCPBj4MFhnOca4BbgLCAT+AD4qz+BqyEYY/Qnwn/wvPl6ABdwHJgBPATcB7wItAIXAJfgSWZOoBL4kdcxigED3GytOwbcDiwCyoBm4P/6nPcW4ENr21eAScOMN9M6V5b1/G/Af3utPx+otR7PADqAFK/17wC3+7vvMOI8DHzH+vs7gBhgCfC+dT1KgRXWtucC2732fQ3Y2Oe8l1uP7wQOAC3ALuAKr+0+D7wH/C/QiCd5ZuH5AHTi+fD7L+BdH14n7wKf77PsNuCdYez7Y+ChIbb5DvCk1/PZgMvu98do/tES+ChgjDkPT4L4qjFmnDFmr7XqM8BPgBQ8b95W4EY8JeJLgC+JyOV9DncGMB24DvgVnlLrBXjejNeKyDkAIrIS+C5wJZBjnf+xYYZ8Np4k22g9n40nGfYqBXJFJMtad9AY09Jn/ewA7Dsc1+O5Vul4vj38A08yywT+HXjGKtGvA6aLSLaIxAJzgHwRSRGRRGAhnmsEnuR9FpCG59vCIyKS53XOM4CD1vl+AvwWz4dzHp4PzVu8A7SqP+4cwd/kbQlwWEResqpP3hKR03w81uPAVKvKLBa4CXjZx2OpYdAEPrqtMca8Z4xxG2Ncxpi3jDHbredleBLuOX32+S9r21fxJPzHjDH1xphqPAlovrXd7cBPjTEfGmO6gf8G5onIpMECEpECPAnpW16LxwEOr+e9j1P6Wde7PiUA+w7HvcaYSmNMO/BZ4EVjzIvWNXwN2ARcbK3fiOfD6XQ8HxTvAcvwJMl9vR9YxpinjDFHrGM8AewDFnud84gx5jfWde0ErgJ+YIxpNcbsAB72DtAYc6kx5mcj+Ju8FQCrgHuBfDwfUGusqpWRqsFTUNgDtOOpUvmmj3GpYdAEPrpVej8RkTOsG3oNIuLAk4Sz++xT5/W4vZ/n46zHk4Bfi0iziDQDTYAAEwcKxiqpvgr8zhjjXVo/DqR6Pe993NLPut71vaVqf/YdDu9rOAm4pvdvtv7u5XhKxgBvAyvwJPG3gbfwfECeYz0HQERutFoM9R7jVD7+f/A+Zw6eqhvvZeUjiH8o7XiqY14yxnQC9+Cpspnpw7F+gKfKrRBIwPPt4g0RSQpUsOrjNIGPbn2HmvwbnrrUQmNMGvB7PEnXF5XAF40x6V4/icaY9/vbWEQy8CTv540xP+mzeicw1+v5XKDOKrHuBKaISEqf9TsDsO9weF/DSuCvff7mZK/Sb98E/jZ9Erj1DeWPwFfx3ANIB3bw8f+D9zkbgG48SbFX0QjiH0oZJ79OfDUPeMIYU2WM6TbGPARkALMCdHzVhybwsSUFaDLGuERkMZ46cl/9Hrirt5mYiKSJyDX9bSgiqXhucr5njOmvrvYvwK0iMktE0oHv47kJi1Wfvw34oYgkiMgVeOqXnwnAviP1CHCZiHxSRKKtY66wqoXAc3OzBE91yAZjzE48pfYzgLXWNsl4EmaDdW1uxlMC75cxpgdYDfxIRJJEZBaeuuVhE5E4EUnA8yERa8Xd+95/BFgiIhdYLZW+ARzFc3MaEYmx9o0Gev/mgVrGbMTzDSVXRKJE5HNALLB/JPGqEbD7Lqr+BOYHz9f1L3g9fwj4cZ9trsbz9bsFeAH4P+ARa10xnsQS47V9FVYrC+v5I8D3vZ5/DtjOR61aHhwgtpusY7fiqdbo/Sny2uZbeKprnMCfgXivdcXW39eOp371gj7H92lf4AZg5yDX9HA/5zoDT2m6CU8S/kefv+MDPO3xe58/DXzY5xg/sfY/CvzSOt4XrHWfp08LEzzVKC8wQCsU4CXgu0O8NkyfH+//65V4kqzT2na217of9bPvj6x1Rd7/RzzVJr/FUxfuxNNs8SK73xuj+UesC6+UUirCaBWKUkpFKE3gSikVoTSBK6VUhNIErpRSEUoTuFJKRShbRljrKzs72xQXF9sdhlJKhZ3NmzcfNcb0O4JmWCTw4uJiNm3aZHcYSikVdkRkwKETtApFKaUilCZwpZSKUJrAlVIqQmkCV0qpCKUJXCmlIpQmcKWUilCawJVSKkJpAldKqQilCVwppSKUJnA1pvS4DW/uqcft1olMVOTTBK7GlLf21HPznzfy97IjdoeilN80gasx5UDDcQAeePcQOp2ginSawNWYUt7YBkBZlYPN5cdsjkYp/4TFaITqI8YY2rt6aHF142zvwunqxunqosXVTYurC2e753eLq5vJ2clctaCAtKRYu8OOGBVNbczIHUeds4MH3j3EwuJMu0NSymeawG2yvcrBb9/cj6O9i5aOjyfm7iFusMVGC8nxMTS3dfHzV3Zz2Zx8PnfmJOYUpIcm+Ah2uLGVeYUZnHdKIvevPUBlUxuFmUl2h6WUTzSB2+SZLVX888M65helk5uSwLScGFITY0lJiCElIZbUBM/j3mWpCbGkWs/jY6IQEXYdcfLI+nKe21rNU5urmFOQxmfPmMRlc/NJjIu2+08MO109bo40u1g5N4kblhTxp3cO8vD7h/n+pbPsDk0NwRhDRVMbk7KS7Q4lrGgCt0mtw8WkrCSeun2pz8eYlZ/Kf19xGnd96hSe3VrNI+vKueOZMn78j11cfXohn11SxJSccQGMOrJVH2unx22YlJVEXloiF5+WxxMbK/nGJ2YwLl7fCuHs6c1VfPvpMi6aPYH/XDmb8akJdocUFvQmpk1qnS7y0hIDcqyUhFhuPLOYV75xNk/ctoRzSsbz13WHOe8Xb3PDn9bx8o4aunvcATlXJCtv8tzA7C3F3bJ8Mi0d3Ty5sdLOsNQwPLWpiszkON7cU8/5v3ybxzdUaCsiNIHbps7pIjfApQgR4YwpWfzm+vm8f+f5fPuTJRw+2sbtj2xh2d1v8Kt/7qXO6QroOSNJeWMrAJOyPHXe8wrTOX1SBg+9f5ge7dgTtiqb2thwuIlbl0/m5W+czay8VO5cvZ3P/HE9h4+22h2erTSB26DHbahv6SAvLXhfA3NS4vnKudNYe8e5/OnGhczMS+XXr+9j6c/e4EuPbOb9/UeDdu5wVd7YRkJsFONT4k8su3X5ZCqa2vjnh3U2RqYG83ypp9PVp+fmMzk7mcf+ZQk/vfI0dhxx8MlfreX3bx8Ys98wNYHb4OjxDnrchtwgJvBe0VHCBbNyeejmxbz97+fyhbMms+5gI5/503peGGO9Ecsb25iUmYyInFh24axcJqYn8sC7h2yMTA3EGMPqLVUsnpx5orVQVJRw/eIi/vmtc1hRksPPXtrNyt++x45qh83Rhp4mcBvUOjzVGBNCfCOmKCuJuz41kw/uOp/xKfG8snNslTormlopyvp4k8GY6Cg+v7SYDYeaxmQCCHc7qp0caGjlivkTT1qXm5rAHz63kPtuWEB9Swcrf/seP33pQ1xdPTZEag9N4DaosRJ4MKtQBpMQG82yadl8cODomLkR5HYbqwR+cpvv6xYXkhwXzYNaCg87z26tJi46iotPzRtwm0+dlsc/v3kOVy8o4A9vH+SiX63lgwONIYxyaMGq4hkygYvIgyJSLyI7+iz/VxHZLSI7ReTnXsvvEpH9IrJHRD4ZjKAjXe+NxEDfxByJpVOzOHq8kz11LbbFEEr1LR10dLtP3MD0lpoQyzULC/l72RHqx/BN3nDT3ePm+dIjnHfK+CF7G6clxXL31XP42xfOwG3g+j+u485nynC0d4Uo2oEdOtrKOf/zFhsONQX82MMpgT8EXOS9QETOBVYCc40xs4F7rOWzgFXAbGuf34mI9ijpo8bhIjZayEqOsy2GpdOyAXhvf3iVVILloxYo/XcEuXlZMd1uw18+KA9lWGoQ7+4/ytHjHVyx4OTqk4EsnZbNK984my+ePYUnN1XyiV++zcs7aoMY5eCMMdy1ugynq6vfwoO/hkzgxpi1QN+Pji8BPzPGdFjb1FvLVwKPG2M6jDGHgP3A4gDGOyrUOV2MT0kgKkqG3jhIJqYnUpyVNGZao3zUBrz/N9GkrGQumJnLo+vLx1Qdajh7bms1aYmxrCjJGdF+iXHR3HXxTNZ8ZTnZ4+K5/ZHN3P7XzbZ8u3pyUyXrDjZx16dmBuUbt6914DOAs0RkvYi8LSKLrOUTAe9eEVXWMuWl1uFigk31396WTstm/aGmMdEEq6KxjegoIT994M5Tty6fzLG2Lp7dWh3CyFR/Wju6eWVnHZfMySM+xrcv8acVpLHmq8v4zkWn8Oaeei7/7Xs0t3UGONKB1be4+Mk/PmRxcSarFhUG5Ry+JvAYIBNYAnwbeFK822YNg4jcJiKbRGRTQ0ODj2FEplqnK+QtUPqzbGo2xzu6KRsDrS8ON7YyMT2R2OiBX/JnTM5kdn4qD+pY4bZ7ZWct7V09XNlP65ORiI2O4ksrpvL4bUuob+ngu89uD9n/9j/+vgtXl5ufXnVa0L5t+5rAq4DVxmMD4AaygWrA+6OmwFp2EmPM/caYhcaYhTk5I/uKFMmMMWFTAj9zahbAmKhG8QyENHgdpIhw6/LJ7Ks/ztp9o/+ahLNnt1ZTkJHI6ZMyAnK8+UUZ/NuFJby4vZanNlUF5JiD+eeuOv5RVsO/njeNqUEcj8jXBP4ccC6AiMwA4oCjwPPAKhGJF5HJwHRgQwDiHDWcrm7au3rCogSemRzHrLzUMXEjs7xx6AQOcOmcfHJS4rVJoY3qnS7e23+UK+ZPZIRf7Af1xbOncOaULH74/E4OWjMzBUOLq4v/t2YHJbkpfPGcqUE7DwyvGeFjwAdAiYhUicitwIPAFKtp4ePATVZpfCfwJLALeBn4ijFG7wh5OdGJJwxK4ADLpmWxueLYqL5x19zWiaO9i0mZQw9FGhcTxY1LJvH23gb214+NJpbh5vnSI7gNXO5n9UlfUVHCL6+bS1xMFF9/fBud3cG593PPK3uodbr46VWnERcT3K42w2mFcr0xJs8YE2uMKTDGPGCM6TTGfNYYc6oxZoEx5g2v7X9ijJlqjCkxxrwU1OgjUK0zvBL40qnZdHa72XR49E4v1juNWt9emAO5Yckk4mOieODdw0GMSg3k2a3VzC1IC0rVQ15aIndfdRrbqx388rW9AT/+5vJj/GVdOTedWcyCosBU/wxGe2KGWK2jHQh9N/qBLJ6cSUyU8N6B0VvnO1QTwr4yk+O4csFEVm+poqk1dK0WFOyta2HnEWfAS9/eLjo1j+sXF/KHtQcCev+ns9vNXavLyEtN4N8/WRKw4w5GE3iI1To6ABifGj/ElqGRHB/DvML0UX0js8LqxFM0gqnTbl42mY5uN49tqAhWWKofz26tJjpKuGxuflDP8/8uncXk7GS++eQ2jgXoQ/r3bx9gb91x/uvyU0M2QYgm8BCrdbrISo7zuW1rMCydls32akdYdDsOhvLGNsanxJMUN/w31YzcFM6ans3D7x8OWl2p+ji327BmazVnT88me1xwCzhJcTHcu2o+Ta2d3Lm6zO+mhfvrW/i/N/Zz6Zw8zp+ZG6Aoh6YJPMRqHe22joHSn2VTs3AbWH9wdLZGGW4LlL5uXT6Z+pYO/rF9bA27a5cNh5s44nAFtfrE26kT0/j2J0t4ZWcdj/sxK5Pbbbhr9XYS46L54WWzAxjh0DSBh1itM7gTOfhiflEGibHRvB9mI7gFSnlTK0XDaIHS1zkzcpg2fhwPaMeekHh2SzXJcdFcOGtCyM75heVTWD4tm//4+0721/vWtPCxjRVsPHyM710yk5yU0FaNagIPsTqnKyQTOYxEXEwUiyZn8t4orAd3dfVQ5+yg2IcSuIhw87JidlQ72TiKW+mEA1dXDy9ur+GiU/NIjAtd9WJUlPCLa+eSGBvN1x/fSkf3yJrT1jpc/OzF3SydmsU1pxcEKcqBaQIPIVdXD02tneSFWRUKeKpR9tUfH3XDqVY0jawJYV9Xzi8gPSmWB949GMiwVB9v7K6npaO734kbgi03NYG7r5rDziNOfvHqyJoW/vD5HXT2uPnvK04LaKej4dIEHkL1Tk8LlHArgYOnPTgw6qpReie9HWgY2aEkxkVzwxlFvLqrjgqrPbkKvNVbqslNjT8xvEOoXTh7AjecUcT9aw/yzr7hjc308o4aXtlZxzcumEFxtm+vL39pAg+hE514wrAEPis/lbTE2FFXjdJbAu9vJp7huvHMYqJFeOj9wwGKSnlrau3krT31rJw3kWgbh1j+/iWzmDZ+HP/2ZOmQ7f8d7V38YM1OZual8oWzJocowpNpAg+hGqsTT7jdxATP5MdnTsni/QONo+qGXXljG6kJMaQPMaPLYHJTE7h0Th5PbqqkxTU6m1ra6R/ba+h2Gy6fZ+/I04lx0fx61Tya27q44+nBmxbe/fJujh7v4O6rTht0hMtg0wQeQiemUgvDBA6ecVGqm9tPdD0fDcqb2piUlex3/eSty6dwvKObJ/xobqb699zWakpyU5iZl2J3KMzOT+OOi0r454d1PLq+/05cGw418bf1FdyybDJzCtJDG2AfmsBDqMbhIikumpQQ9dIaqd5p1kZTPXh548kz0fvitII0Fhdn8tD7h+lxj55vKHYrb2xlc/kxrlgQ2JEH/XHLssmcNT2b/3phF/v6zBnr6urhztVlFGQk8q0LZ9gU4Uc0gYdQndMzDni4vFD7mpKdzITUhFEzLkp3j5vqY+1+1X97u2X5ZKqOtfPaLvvmWBxtntt6BBH4dJC7zo9Eb9PCcfExfO3xbR9rWvi7N/dzsKGVn1xx2oh69gaLJvAQqnWEx0w8AxERlk7L4oMDjbhHQSnzSLOLbreh2McWKH19YlYuhZmJPKijFAaEMYbntlWzZHLWoFPd2WF8SgI/v3oOH9Y4+fnLewDYU9vCfW8f4Ir5EzlnRnhMQqMJPITCPYGDZ5q1ptZOdtdG/ljY5U3WIFYBmg08OkpYtaiIDYebqG8ZXe3l7bCtsplDR1ttafs9HOfPzOXGMyfxwLuHeHN3PXeuLmNcfAzfv2Sm3aGdoAk8RNxuQ31LR9iMAz6QpdOsadZGQTXK4caRDSM7HL0zpK/dG/nXZ6QcbV0BHfDsua3VxMdEcdFpoes6P1LfvXgmM3LHcdtfN7G1opkfXDaLrCAPtDUSmsBD5GhrB91uE/YJPC8tkSnZyaOiPXhFYytxMVHkpgTums/KS2V8Sjxv7qkP2DEjwdq9Day4503Ovect3thd5/fxunrc/L2shgtm5ZKa4HsTz2BLiI3m16vmIyKcPSPH9qaOfWkCD5ETU6mFeRUKeErhGw410dUT2cOolje2MSkzKaAzgosI58zI4Z29DXRH+PUZDrfbcO/r+7jpzxsYn5JAbmoCtzy0iR+/sMuvYXbX7m2gqbWTK8IsIfZnZl4qr3/rHO7/3Olh1wBBE3iIhNtcmINZNjWb1s4eSiub7Q7FL8OZid4XK0rG43R1sy3Cr89Qmts6ueXhjfzytb1cPm8iz35lKc9+eSk3nTmJP717iKvue//EUAUj9ezWajKSYjmnJDxuBg6lMDOJhNjwGcO/lybwEKkL4270fZ05NQsRInq2emMM5Y1tPg0jO5Tl07OJjhLe2jO8MTMi0fYqB5f+5l3e23+U/7r8VH557VyS4mJIiI3mP1aeyh8+dzoVTW1ccu87rNlWPaJjt7i6eG1XHZfNzbe1F+NooFcvRGocLmKiJKxugAwkPSmO2fmpEX0js6Glg/aunqCUwNMSY1lQlM5be0dfPbgxhsc3VHDV79/H7TY8+cUz+dySSSdVHXxy9gRe/PpZzMpP5euPb+PbT5XS1tk9rHO8tKOWjm53yCZuGM00gYdIrdPF+JR4WwfrGYllU7PZWtFMe+fIxkcOFyOdyHikVpSMZ0e1c1Q1J3R19XDH02XcuXo7Z0zO5IWvncX8QWZWn5ieyGP/soSvnTeNp7dUcelv3mXXEeeQ53luazXFWUnML0wPYPRjkybwEKl1hN9EDoM5c2oWnT1uNh5usjsUn5SfaEIYnGE+eztyjJbmhOWNrVzxu/d5anMVXzt/Og/dvJjM5Lgh94uJjuJbF5bw6BfO4Lirm8t/9x5/+eDwgANB1Tja+eBgI5fPD5+u85FsyAQuIg+KSL2I7Ohn3b+JiBGRbOu5iMi9IrJfRMpEZEEwgo5EtU5XWI5COJDFkzOJjZaI7VZf3thKlHhKicEwOz+VnJR43hoFzQlf21XHpb95lyPN7fz584v41idmjPib4tKp2bz09bNYNjWLH6zZyRf/upnmtpOHZF2z7QjGEHbN8SLVcErgDwEX9V0oIoXAhYD3kF2fAqZbP7cB9/kfYuQzxnhK4BFwA7NXUlwM8wszeD9Cb2SWN7aRn55IXExwvmSeaE6472jENifs7nHz85d38y9/2URxVjIv/Otyzj1lvM/HyxoXzwM3LeL7l8zkzT31XPzrd076Bvfc1mrmF6XbNgHCaDPkq9sYsxbo73v0/wJ3AN7flVYCfzEe64B0EckLSKQRrKWjm7bOnohogeJt6bQsdhxx9FuSCnflTW0BGwNlICtKcnC0d1Fa1RzU8wTD0eMd3PjgBn731gGuX1zEU7efSWEABv2KihK+cNYUnvnSUmJjorjuDx/wm9f30eM2fFjjZHdtC1fqzcuA8al4IiIrgWpjTGmfVRMB7wGTq6xl/R3jNhHZJCKbGhpGb3MsgLoIagPubdm0bIyBdQcjrxReEaBhZAdz1rQcooSIa064ubyJS+59h83lx/ifq+fw0ytPC3gb5zkF6bzwr8u5bG4+v3htLzf8aR1/eucQMVHCJXPCZ+TBSDfiBC4iScB3gR/4c2JjzP3GmIXGmIU5OZHRmN9X4TyV2mDmFqSTFBcdce3BHe1dHGvrCtgwsgNJS4plQVFGxCRwt9vw5/cOcd0f1pEQG83qLy/lmoWFQTtfSkIsv7puHv9z9RxKKx08s6WKFSU5w7o5qobHlwFtpwKTgVLrLnIBsEVEFgPVgPcrosBaNqbVWCXwvLTwGjJzKHExUSyenBlx7cErgjCI1UBWlORwz6t7aWjpICfFvjb+brfh6PEOahwuahzt1Dhc1DpcHHG4qLWe1zlddPUYLpg5nl9cO4+0xOCPQSIiXLOwkPlFGfz85d188ZypQT/nWDLiBG6M2Q6cuNMhIoeBhcaYoyLyPPBVEXkcOANwGGNqAhVspOqtQhmfGv6dePpaNjWbn+z50DMUboRUAfUOIxusJoTeVpSM555X97J2bwNXnV4Q9POtO9jIriNOap0ujjS3U+twnUjO3X3GcI+LiSIvLYEJqQksKs5kQloCp0xI4bI5+QEdH2Y4po0fx/03LgzpOceCIRO4iDwGrACyRaQK+KEx5oEBNn8RuBjYD7QBNwcozohW43SRkRQblmMpDOXMqR8NL3vlguAnqEDobQNeFOQqFPCMTpg9Lp63QpDA3z9wlM/8cT0A8b3JOS2BMyZ7knNeWgJ5aYknHmcmx2lb61FuyARujLl+iPXFXo8N8BX/wxpd6hwuJkRY9UmvWXmpZCTF8t7+xghK4K1kj4snOQRzj0ZFeZoTvr67jh63CWpP20fWlZORFMur3zyH7HGanJX2xAyJWqeLCRFYfQKeBHXm1CzeP3B0wN514aa8MTijEA5kRUkOzW1dQR2dsN7p4tWddVyzsJCclHhN3grQBB4SkVR/3J+lU7Opcbg45OPQoaEWrGFkB3LW9GyiBN4OYq/MJzdV0u02XL+4KGjnUJFHE3iQdXT30NjayYTUyKxCAU97cID3DoR/c0JXVw81DheTgjCM7EDSk+KYX5TBW3uD05ywx214bEMly6dlM1l7MCovmsCDrN7ZAcCEtMisQgEozkoiPy2B9yNgmrXKII9COJAVM3Ioq3Jw9HhHwI/99t56qpvbueEMLX2rj9MEHmQnOvFE6E1M8LTlXTotmw8ONuJ2h3c9+IkWKKFO4CWelrVrg1AKf3RdBTkp8VwwKzfgx1aRTRN4kEXSXJiDWTYti+a2LnbVDD3es51OjAMegiaE3mbnp5I9Li7gvTKrjrXxxp56Vi0q1Nlr1En0FRFkkTSV2mCWTvXUg4d7r8yKxlZS4mNC3l07Ksoza/nafQ30BPBbyhMbKxFgld68VP3QBB5kNQ4XibHRpCYGv01yMOWmJjA1Jznsx0U53NhGUVaSLc3sVpSMp7ktcKMTdvW4eXxjJeeWjA/auOYqsmkCD7Jap6cJ4Whot7tsWjYbDjXR2R2+41+Hugmht7Ot5oSBqkb55646Glo6uGGJlr5V/zSBB5lnIofIbYHibenUbNq7eoLaYcUfPW5D1bHgzEQ/HOlJccwrTA9Ye/BH11cwMT2Rc2b4PsmCGt00gQdZrcMVcaMQDuTMKVlECbwXps0JjzS309VjKLapBA6eapSyageNfjYnPHS0lXf3H+X6xYURMxG2Cj1N4EHkdhvqWyJrKrXBpCXFcurEtLC9kWlXE0JvK0pyMAbW7vOvGuWxDRXERAnXBnG8bhX5NIEHUWNrJ109JmLHQenP0qnZbK1oprWj2+5QThLKYWQHcmp+mt/NCV1dPTy1qZILZ+cyfpR8+Kvg0AQeRHWjoBNPX8umZdHtNidNVhsOKhrbiIuOsrXJZlSUcPb0HNbu9b054cs7ajnW1sUNZ0wKcHRqtNEEHkS1EToX5mAWTsokLjqK98NwXJTyxjYKMxNtrzM+pySHY21dlPnYnPDR9eVMzk7mzClZgQ1MjTqawIOoxtk7ldroSeCJcdHML0oPyxuZhxtbba0+6XX2dN8nO95T28LGw8f4zOKikM+aoyKPJvAgqnO4iI4SsseNnjpwgOXTstl5xMnK/3uXHz2/kzXbqqlsarN1vHBjDBVNbSGZhWcoGclxzC1M92l0wr+tLycuJiok07OpyBfZ3QPDXK3TRc64eNu/0gfaZ5dMor2rhy0Vx3hiYyUPvX8YgOxx8SwoSmd+UQYLitKZU5BOYlxoppE7eryTts4e2zrx9LVixnh+9fpeGo93kDXMD/C2zm5Wb6nmktPydOZ2NSyawIMo0idyGEhGchx3XHQKAN09bvbUtbClopmt5cfYWtnMq7vqAIiOEmbmpTC/MIMFk9JZUJRBUWZwurlXWC1QisOgCgU8zQn/9597eWffUS6fP3FY+/y99AgtHd06bKwaNk3gQVTrdDEtZ5zdYQRVTHQUs/PTmJ2fxueWeFpNNLV2sq3yGFvKm9lScYzVW6r467pyALKS45hflM4NZ0zi3FMC18MwHNqAezttYhpZyXG8tad+2An80fUVlOSmcPqkjCBHp0YLTeBBVOtwsdyazWYsyUyO47xTcjnvFM/41T1uw966FrZWeBL6u/uO8rXHtrLhexcErIrlcGMbIlCQER5NNntHJ3x7bwNutxnyhmRZVTNlVQ7+c+XsUTFujgoNvYkZJMc7ujne0T0qq1BGylOVkspnzijinmvm8utV82jp6ObF7TUBO0dFYyv5aYnEx4Smzn04VpTk0NTaSVm1Y8ht/7a+gsTY6GGX1pWCYSRwEXlQROpFZIfXsv8Rkd0iUiYiz4pIute6u0Rkv4jsEZFPBinusDdaJnIIhsWTM5mcncwTGysDdsxyG0chHMhZ03MQgbeGGNzK6epizbYjrJyXT2pCbIiiU6PBcErgDwEX9Vn2GnCqMWYOsBe4C0BEZgGrgNnWPr8TkfApEoXQaOzEEygiwnWLCtlwuIkDDccDcsyKxvBL4JnJccwtSB+yPfhzW6tp7+rRnpdqxIZM4MaYtUBTn2WvGmN6B8NYB/Q2Wl0JPG6M6TDGHAL2A4sDGG/EqB0lM/EEy5ULJhITJTwZgFJ4i6uLxtZO24aRHcyKkhxKq5ppau3sd70xhkfXVTCnII3TCtJCHJ2KdIGoA78FeMl6PBHwfkdWWcvGnI/GQdEE3p/xKQmcP3M8z2ypoqvHvwkielughFsJHDzDyxoD7wwwOuHm8mPsqWvRpoPKJ34lcBH5HtANPOrDvreJyCYR2dTQEPiZvO1W42gnPSmWhNgxWYM0LKsWFXH0eCevf+jfBAgVTeGbwOdMTCMzeeDRCR9dX0FKfAyXzc0PcWRqNPA5gYvI54FLgRvMR32oqwHvAYwLrGUnMcbcb4xZaIxZmJOT42sYYavW0aHVJ0M4e0YOE1ITeGJjhV/H+agEHn5VKJ7RCbNZazUn9NbU2sk/ttdw5YKJJMVpi141cj4lcBG5CLgD+LQxps1r1fPAKhGJF5HJwHRgg/9hRp465+iZyCFYoqOEaxYW8PbeBo40t/t8nPLGVrKS4xgXH55JcEXJeBpbO9nepznhM5ur6Ox28xm9eal8NJxmhI8BHwAlIlIlIrcC/wekAK+JyDYR+T2AMWYn8CSwC3gZ+Ioxpido0YexGodrVI1CGCzXLizEbeDpzVU+H6Pcmok+XJ09o7c54UfVKG634W8bKlhUnEHJhBQbo1ORbDitUK43xuQZY2KNMQXGmAeMMdOMMYXGmHnWz+1e2//EGDPVGFNijHlpsGOPVp3dbhpbO7QEPgyFmUksn5bNExsrT6piGK6KprawGQOlP5nJccwpSOetvR/V9X9wsJFDR1u16aDyi/bEDIL6FhfGaAuU4bpuUSHVze2858Ncmx3dPRxxtIfFMLKDWTEjh22VzRyzmhM+ur6cjKRYLjp1gs2RqUimCTwItAnhyFw4O5f0pFge96FNeGVTO8aEZwsUb96THdc7Xby6s45rFhZqKyXll/C86xPhah0dgHbiGa74mGiunF/AX9cdpqm1c0RjYVecmMg4vBP4nIJ0MpJieXtPA5VNbXS7Ddcv1rbfyj9aAg+CGoenRYXexBy+6xYV0tVjWL1lZDczw7kJobdor9EJH9tQyfJp2UzODu+YVfjTBB4EdU4X8TFRpCXqwETDVTIhhflF6TyxsXJEU7OVN7aRHBdNVgTMYLOiJIfG1k6qm9u156UKCE3gQVDr7GBCWoKO6zxCqxYVsq/+OFsqmoe9T3ljK0VZyRFxrc+2RifMSYnnglm5doejRgFN4EFQ62jX+m8fXDonn+S46BH1zCxvamNSmLdA6ZU1Lp6bzizm2xeWEButbz3lP30VBUGtc3TOhRlsydaYIH8vraHF1TXk9j1uQ1VTO5OyIyOBA/zo07O5dlHh0BsqNQyawAPMGEOdjoPis+sWFdLe1cMLZUPP1lPrdNHZ42ZSGA4jq1QoaAIPsKbWTjp73FoC99G8wnRm5I4b1mw95UcjowmhUsGiCTzAdCIH/3hm6yliW2Uzu2udg25bbg0jG+69MJUKFk3gAdY7lVqulsB9dsX8icRFRw1ZCi9vbCM2WshPD4+Z6JUKNU3gAdZbAtdOPL7LTI7jwtm5PLu1GlfXwINZVjS1UpiRRHRU+DchVCoYNIEHWJ3DRZRAzrh4u0OJaKsWFdHc1sWru+oG3Obw0fAeRlapYNMEHmA1Dhc5KfHEaDtfvyydmkVBRuKAbcKNMVREUBtwpYJBs0yA1TpdegMzAKKihOsWFvLe/kYqGttOWt/U2snxjm6KwnwMFKWCSRN4gOlUaoFz9cICogSe3HTyzczeFijFWoWixjBN4AGmU6kFTl5aIitKxvPU5kq6e9wfW1feqG3AldIEHkCtHd20uLq1CWEAXbeokDpnB2/vbfjY8vLGNkSgIEMTuBq7NIEHkHbiCbzzThlP9rj4k2brqWhsY0Jqgs5oo8Y0TeABVOfQqdQCLTY6iqtPL+CN3fXUWx+QYI1CqNUnaozTBB5AWgIPjmsXFtDjNjztNVtPeWOrDmKlxjxN4AFUoyXwoJiSM47FkzN50pqt53hHN0ePd2onHjXmDZnAReRBEakXkR1eyzJF5DUR2Wf9zrCWi4jcKyL7RaRMRBYEM/hwU+d0kZoQQ1KczhUdaKsWFXK4sY31h5pOtAvXKhQ11g2nBP4QcFGfZXcCrxtjpgOvW88BPgVMt35uA+4LTJiRodahEzkEy6dOzSMlIYYnNlaemIm+WDvxqDFuyARujFkLNPVZvBJ42Hr8MHC51/K/GI91QLqI5AUo1rDnmYlHR8YLhsS4aC6fN5EXt9dQVuUA0CoUNeb5Wgeea4zpnTKlFuidoXUi4N3eq8padhIRuU1ENonIpoaGhv42iTi1DhcTUnUQq2C5blEhHd1u/vpBORlJsaQmxNodklK28vsmpjHGAMaH/e43xiw0xizMycnxNwzbdfW4aTiuU6kF06kT0zh1YiotOgaKUoDvCbyut2rE+l1vLa8GvGdsLbCWjXoNLR0Yg1ahBNl1i4oAHQNFKfA9gT8P3GQ9vglY47X8Rqs1yhLA4VXVMqqdaAOeplUowfTpufmkJsQwOz/V7lCUst2Q7d1E5DFgBZAtIlXAD4GfAU+KyK1AOXCttfmLwMXAfqANuDkIMYelE1OpaRVKUKUlxvLOHeeRHK9d6JUaMoEbY64fYNX5/WxrgK/4G1Qk6k3geVqFEnRpSXrzUinQnpgBU+d0ERcTRYYmF6VUiGgCD5Aah2cmHhGdYFcpFRqawANEp1JTSoWaJvAAqXO6dCIHpVRIaQIPAGOMTqWmlAo5TeAB0NzWRWe3W5sQKqVCShN4AJwYB1wTuFIqhDSBB0CdUydyUEqFnibwAKjVBK6UsoEm8ACocbgQgfEpOg6KUip0NIEHQJ3DRfa4eGKj9XIqpUJHM04AaCcepZQdNIEHgM6FqZSygybwANASuFLKDprA/dTe2YOjvUtL4EqpkNME7qcTTQi1BK6UCjFN4H7qnchBS+BKqVDTBO6nWmc7oFOpKaVCTxO4n2odHYCWwJVSoacJ3E91Thcp8TGMix9yelGllAooTeB+qnG0a+lbKWULTeB+qnV2aAJXStnCrwQuIt8UkZ0iskNEHhORBBGZLCLrRWS/iDwhInGBCjYc1TlcegNTKWULnxO4iEwEvgYsNMacCkQDq4C7gf81xkwDjgG3BiLQcNTd46a+RadSU0rZw98qlBggUURigCSgBjgPeNpa/zBwuZ/nCFtHj3fiNtqEUCllD58TuDGmGrgHqMCTuB3AZqDZGNNtbVYFTPQ3yHClvTCVUnbypwolA1gJTAbygWTgohHsf5uIbBKRTQ0NDb6GYatah6cTj97EVErZwZ8qlAuAQ8aYBmNMF7AaWAakW1UqAAVAdX87G2PuN8YsNMYszMnJ8SMM+2g3eqWUnfxJ4BXAEhFJEhEBzgd2AW8CV1vb3ASs8S/E8FXjdBEbLWQmjeqGNkqpMOVPHfh6PDcrtwDbrWPdD3wH+JaI7AeygAcCEGdY6m1CGBUldoeilBqD/Or/bYz5IfDDPosPAov9OW6k0IkclFJ20p6YftCp1JRSdtIE7iNjjJbAlVK20gTuI2d7N64ut5bAlVK20QTuoxqntgFXStlLE7iP9tS2AFCclWxzJEqpsUoTuI/KqhzEx0RRMiHF7lCUUmOUJnAflVU1Mzs/ldhovYRKKXto9vFBd4+b7dUO5hSk2x2KUmoM0wTug331x3F1uZlXmG53KEqpMUwTuA/KqpoBmFOQZm8gSqkxTRO4D0qrHKQkxGgLFKWUrTSB+6C0spm5Bek6iJVSylaawEfI1dXDntoWrT5RStlOE/gI7apx0u022gJFKWU7TeAjVFrZDKAtUJRSttMEPkJlVQ7Gp8TrGChKKdtpAh+h0qpmrT5RSoUFTeAj4Gjv4mBDK/MK9QamUsp+msBHYEe1A0BL4EqpsKAJfARKtQemUiqMaAIfgdLKZoqzkkhPirM7FKWU0gQ+EmVVOgKhUip8aAIfpnqnixqHS6tPlFJhw68ELiLpIvK0iOwWkQ9F5EwRyRSR10Rkn/U7I1DB2qm0ynMDc6524FFKhQl/S+C/Bl42xpwCzAU+BO4EXjfGTAdet55HvLKqZqKjhNn5qXaHopRSgB8JXETSgLOBBwCMMZ3GmGZgJfCwtdnDwOX+hRgeSqscTB8/jqS4GLtDUUopwL8S+GSgAfiziGwVkT+JSDKQa4ypsbapBXL9DdJuxhjKqjxDyCqlVLjwJ4HHAAuA+4wx84FW+lSXGGMMYPrbWURuE5FNIrKpoaHBjzCCr6Kpjea2Lq3/VkqFFX8SeBVQZYxZbz1/Gk9CrxORPADrd31/Oxtj7jfGLDTGLMzJyfEjjODrvYGpLVCUUuHE5wRujKkFKkWkxFp0PrALeB64yVp2E7DGrwjDQFllM/ExUZRMSLE7FKWUOsHfO3L/CjwqInHAQeBmPB8KT4rIrUA5cK2f57BdaVUzs/NTiY3WZvNKqfDhVwI3xmwDFvaz6nx/jhtOunvc7Kh2ct2iQrtDUUqpj9Ei5RD2NxynvauHuTqErFIqzGgCH0LvFGrahFApFW40gQ+htMpBSkIMxVnJdoeilFIfowl8CGVVzcwpSCMqSuwORSmlPkYT+CBcXT3srmnR6hOlVFjSBD6IXTVOut1GxwBXSoUlTeCDKOu9gaktUJRSYUgT+CBKqxzkpMQzITXB7lCUUuokmsAHUWqNQCiiNzCVUuFHE/gAnK4uDja0MlcHsFJKhSlN4APY3jsCoQ4hq5QKU5rAB1Ba1QygJXClVNjSBD6AskoHk7KSSE+KszsUpZTqlybwAXh6YKbbHYZSSg1IE3g/6ltcHHG4tPpEKRXWNIH3o6zScwNT58BUSoUzTeD9KKtqJkpgdn6q3aEopdSANIH3Y1uVgxm5KSTF+TvjnFJKBY8m8D6MMZRZPTCVUiqcaQLvo7Kpnea2LuboAFZKqTCnCbyPbSc68KTbGodSSg1FE3gfZZXNxMdEUTIhxe5QlFJqUH4ncBGJFpGtIvKC9XyyiKwXkf0i8oSIRFRXxrIqB7PyU4mN1s82pVR4C0SW+jrwodfzu4H/NcZMA44BtwbgHCHR3eNme7VDq0+UUhHBrwQuIgXAJcCfrOcCnAc8bW3yMHC5P+cIpf0Nx2nv6tEZeJRSEcHfEvivgDsAt/U8C2g2xnRbz6uAiX6eI2R6e2DqGChKqUjgcwIXkUuBemPMZh/3v01ENonIpoaGBl/DCKhtVc2kxMcwOSvZ7lCUUmpI/pTAlwGfFpHDwON4qk5+DaSLSG8XxgKgur+djTH3G2MWGmMW5uTk+BFG4JRVNTOnMI2oKJ1CTSkV/nxO4MaYu4wxBcaYYmAV8IYx5gbgTeBqa7ObgDV+RxkCrq4edte0aPWJUipiBKOt3HeAb4nIfjx14g8E4RwB92GNk2630SFklVIRIyCjNRlj3gLesh4fBBYH4rihVFrZDOgQskqpyKG9VSxlVQ5yUuKZkJpgdyhKKTUsmsAtpVXNzC1Iw9OUXSmlwp8mcMDp6uJAQ6v2wFRKRRRN4MCOKqsDj9Z/K6UiiCZwoLQ3gU/UFihKqcihCRxPC5RJWUlkJEfUwIlKqTFOEzhWD0yt/1ZKRZgxn8DrW1wccbi0A49SKuKM+QTeOwKhduBRSkUaTeBVzUQJzM5PtTsUpZQakTGfwEurHMzITSEpLiCjCiilVMiM6QRujKG0qpk5Wv+tlIpAYzqBVza109zWpfXfSqmINKYTeGlVM4B2oVdKRaQxncDLqpqJi4miZEKK3aEopdSIjekEXlrpYHZ+KrHRY/oyKKUi1JjNXJsON7Gl4hiLijPtDkUppXwyJhN4fYuLLz+6hYkZiXzl3Gl2h6OUUj4Zcwm8q8fNVx/ditPVxe8/ezppibF2h6SUUj4Zc71XfvbSbjYcbuJX181jZp72vlRKRa4xVQJ/vvQID7x7iM8vLeby+RPtDkcppfwyZhL4ntoWvvN0GQsnZfDdi2faHY5SSvltTCRwp6uL2x/ZzLiEGH53wwLiYsbEn62UGuV8zmQiUigib4rILhHZKSJft5ZnishrIrLP+p0RuHBHzu02/NuTpVQ2tfHbzyxgfGqCneEopVTA+FMU7Qb+zRgzC1gCfEVEZgF3Aq8bY6YDr1vPbXPf2wd4bVcd3714Josna5tvpdTo4XMCN8bUGGO2WI9bgA+BicBK4GFrs4eBy/2M0Wdr9zZwz6t7uGxuPjcvK7YrDKWUCoqAVAaLSDEwH1gP5BpjaqxVtUBuIM4xUlXH2vj641uZMT6Fu686DRGxIwyllAoavxO4iIwDngG+YYxxeq8zxhjADLDfbSKySUQ2NTQ0+BvGx7i6evjSI1vo7jH8/nOn62QNSqlRya8ELiKxeJL3o8aY1dbiOhHJs9bnAfX97WuMud8Ys9AYszAnJ8efME7ywzU72V7t4JfXzWNydnJAj62UUuHCn1YoAjwAfGiM+aXXqueBm6zHNwFrfA9v5B7bUMETmyr56rnT+MQsW2pvlFIqJPypW1gGfA7YLiLbrGXfBX4GPCkitwLlwLV+RTgCpZXN/HDNTs6ans03PzEjVKdVSilb+JzAjTHvAgPdGTzf1+P6qvF4B196ZDM5KfHcu2o+0VF601IpNbqNirt7PW7D1x7fytHWTlZ/aSkZyXF2h6SUUkE3KvqU3/PqHt7b38iPLz+VUyfqDPNKqbEh4hP4yztque+tA1y/uIhrFxbaHY5SSoVMRCfwAw3H+fenSplbmM6PPj3L7nCUUiqkIjqB//TF3cTFRHHfDQuIj4m2OxyllAqpiL6J+Ytr51LR2EZ+eqLdoSilVMhFdAk8LTGW0wr0pqVSamyK6ASulFJjmSZwpZSKUJrAlVIqQmkCV0qpCKUJXCmlIpQmcKWUilCawJVSKkJpAldKqQilCVwppSKUJnCllIpQ4pk43uYgRBrwTL/mi2zgaADDCaRwjg3COz6NzTfhHBuEd3zhGtskY0y/M7+HRQL3h4hsMsYstDuO/oRzbBDe8Wlsvgnn2CC84wvn2AaiVShKKRWhNIErpVSEGg0J/H67AxhEOMcG4R2fxuabcI4Nwju+cI6tXxFfB66UUmPVaCiBK6XUmBQxCVxELhKRPSKyX0Tu7Gd9vIg8Ya1fLyLFIYqrUETeFJFdIrJTRL7ezzYrRMQhItusnx+EIjav8x8Wke3WuTf1s15E5F7r2pWJyIIQxVXidU22iYhTRL7RZ5uQXTsReVBE6kVkh9eyTBF5TUT2Wb8zBtj3JmubfSJyU4hi+x8R2W39z54VkfQB9h30/x/E+H4kItVe/7uLB9h30Pd2kGJ7wiuuwyKybYB9g37t/GKMCfsfIBo4AEwB4oBSYFafbb4M/N56vAp4IkSx5QELrMcpwN5+YlsBvGDj9TsMZA+y/mLgJUCAJcB6m/7HtXjavNpy7YCzgQXADq9lPwfutB7fCdzdz36ZwEHrd4b1OCMEsV0IxFiP7+4vtuH8/4MY34+Afx/G/33Q93YwYuuz/hfAD+y6dv78REoJfDGw3xhz0BjTCTwOrOyzzUrgYevx08D5IiLBDswYU2OM2WI9bgE+BCYG+7wBthL4i/FYB6SLSF6IYzgfOGCM8bVDl9+MMWuBpj6LvV9XDwOX97PrJ4HXjDFNxphjwGvARcGOzRjzqjGm23q6DigI5DlHYoBrNxzDeW8HLTYrR1wLPBbIc4ZKpCTwiUCl1/MqTk6SJ7axXtQOICsk0Vmsapv5wPp+Vp8pIqUi8pKIzA5lXIABXhWRzSJyWz/rh3N9g20VA7+J7Lx2ucaYGutxLZDbzzbhcP1uwfMtqj9D/f+D6atWFc+DA1Q/2X3tzgLqjDH7Blhv57UbUqQk8LAnIuOAZ4BvGGOcfVZvwVM1MBf4DfBciMNbboxZAHwK+IqInB3i8w9KROKATwNP9bPa7mt3gvF8pw67Zlsi8j2gG3h0gE3s+v/fB0wF5gE1eKoqws31DF76Duv3TqQk8Gqg0Ot5gbWs321EJAZIAxpDEZyIxOJJ3o8aY1b3XW+McRpjjluPXwRiRSQ7FLFZ56y2ftcDz+L52uptONc3mD4FbDHG1PVdYfe1A+p6q5Os3/X9bGPb9RORzwOXAjdYHzAnGcb/PyiMMXXGmB5jjBv44wDntfPaxQBXAk8MtI1d1264IiWBbwSmi8hkq7S2Cni+zzbPA713/68G3hjoBR1IVh3aA8CHxphfDrDNhN76eBFZjOe6h+rDJVlEUnof47nxtaPPZs8DN1qtUZYADq9qg1AYsBRk57WzeL+ubgLW9LPNK8CFIpJhVRNcaC0LKhG5CLgD+LQxpm2AbYbz/w9WfN73Ua4Y4LzDeW8HywXAbmNMVX8r7bx2w2b3XdTh/uBpKbEXzx3r71nL/hPPixcgAc9X8P3ABmBKiOJajudrdRmwzfq5GLgduN3a5qvATjx32NcBS0N43aZY5y21Yui9dt7xCfBb69puBxaGML5kPAk5zWuZLdcOz4dIDdCFpy72Vjz3UV4H9gH/BDKtbRcCf/La9xbrtbcfuDlEse3HU3/c+7rrbYWVD7w42P8/RPH91Xo9leFJynl947Oen/TeDnZs1vKHel9nXtuG/Nr586M9MZVSKkJFShWKUkqpPjSBK6VUhNIErpRSEUoTuFJKRShN4EopFaE0gSulVITSBK6UUhFKE7hSSkWo/w8mihhnUD8vKwAAAABJRU5ErkJggg==",
|
|
"text/plain": [
|
|
"<Figure size 1440x360 with 1 Axes>"
|
|
]
|
|
},
|
|
"metadata": {
|
|
"needs_background": "light"
|
|
},
|
|
"output_type": "display_data"
|
|
}
|
|
],
|
|
"source": [
|
|
"state = envs.reset()\n",
|
|
"\n",
|
|
"while frame_idx < max_frames:\n",
|
|
"\n",
|
|
" log_probs = []\n",
|
|
" values = []\n",
|
|
" rewards = []\n",
|
|
" masks = []\n",
|
|
" entropy = 0\n",
|
|
"\n",
|
|
" for _ in range(num_steps):\n",
|
|
" state = torch.FloatTensor(state).to(device)\n",
|
|
" dist, value = model(state)\n",
|
|
"\n",
|
|
" action = dist.sample()\n",
|
|
" next_state, reward, done, _ = envs.step(action.cpu().numpy())\n",
|
|
"\n",
|
|
" log_prob = dist.log_prob(action)\n",
|
|
" entropy += dist.entropy().mean()\n",
|
|
" \n",
|
|
" log_probs.append(log_prob)\n",
|
|
" values.append(value)\n",
|
|
" rewards.append(torch.FloatTensor(reward).unsqueeze(1).to(device))\n",
|
|
" masks.append(torch.FloatTensor(1 - done).unsqueeze(1).to(device))\n",
|
|
" \n",
|
|
" state = next_state\n",
|
|
" frame_idx += 1\n",
|
|
" \n",
|
|
" if frame_idx % 1000 == 0:\n",
|
|
" test_rewards.append(np.mean([test_env() for _ in range(10)]))\n",
|
|
" plot(frame_idx, test_rewards)\n",
|
|
" \n",
|
|
" next_state = torch.FloatTensor(next_state).to(device)\n",
|
|
" _, next_value = model(next_state)\n",
|
|
" returns = compute_returns(next_value, rewards, masks)\n",
|
|
" \n",
|
|
" log_probs = torch.cat(log_probs)\n",
|
|
" returns = torch.cat(returns).detach()\n",
|
|
" values = torch.cat(values)\n",
|
|
"\n",
|
|
" advantage = returns - values\n",
|
|
"\n",
|
|
" actor_loss = -(log_probs * advantage.detach()).mean()\n",
|
|
" critic_loss = advantage.pow(2).mean()\n",
|
|
"\n",
|
|
" loss = actor_loss + 0.5 * critic_loss - 0.001 * entropy\n",
|
|
"\n",
|
|
" optimizer.zero_grad()\n",
|
|
" loss.backward()\n",
|
|
" optimizer.step()"
|
|
]
|
|
}
|
|
],
|
|
"metadata": {
|
|
"interpreter": {
|
|
"hash": "fe38df673a99c62a9fea33a7aceda74c9b65b12ee9d076c5851d98b692a4989a"
|
|
},
|
|
"kernelspec": {
|
|
"display_name": "Python 3.7.9 64-bit ('py37': conda)",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.7.9"
|
|
},
|
|
"orig_nbformat": 4
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 2
|
|
}
|