1.actor-critic.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import math\n",
    "import random\n",
    "\n",
    "import gym\n",
    "import numpy as np\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torch.nn.functional as F\n",
    "from torch.distributions import Categorical"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from IPython.display import clear_output\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Use CUDA</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "use_cuda = torch.cuda.is_available()\n",
    "device   = torch.device(\"cuda\" if use_cuda else \"cpu\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Create Environments</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from common.multiprocessing_env import SubprocVecEnv\n",
    "\n",
    "num_envs = 16\n",
    "env_name = \"CartPole-v0\"\n",
    "\n",
    "def make_env():\n",
    "    def _thunk():\n",
    "        env = gym.make(env_name)\n",
    "        return env\n",
    "\n",
    "    return _thunk\n",
    "\n",
    "envs = [make_env() for i in range(num_envs)]\n",
    "envs = SubprocVecEnv(envs)\n",
    "\n",
    "env = gym.make(env_name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Neural Network</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ActorCritic(nn.Module):\n",
    "    def __init__(self, num_inputs, num_outputs, hidden_size, std=0.0):\n",
    "        super(ActorCritic, self).__init__()\n",
    "        \n",
    "        self.critic = nn.Sequential(\n",
    "            nn.Linear(num_inputs, hidden_size),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(hidden_size, 1)\n",
    "        )\n",
    "        \n",
    "        self.actor = nn.Sequential(\n",
    "            nn.Linear(num_inputs, hidden_size),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(hidden_size, num_outputs),\n",
    "            nn.Softmax(dim=1),\n",
    "        )\n",
    "        \n",
    "    def forward(self, x):\n",
    "        value = self.critic(x)\n",
    "        probs = self.actor(x)\n",
    "        dist  = Categorical(probs)\n",
    "        return dist, value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot(frame_idx, rewards):\n",
    "    clear_output(True)\n",
    "    plt.figure(figsize=(20,5))\n",
    "    plt.subplot(131)\n",
    "    plt.title('frame %s. reward: %s' % (frame_idx, rewards[-1]))\n",
    "    plt.plot(rewards)\n",
    "    plt.show()\n",
    "    \n",
    "def test_env(vis=False):\n",
    "    state = env.reset()\n",
    "    if vis: env.render()\n",
    "    done = False\n",
    "    total_reward = 0\n",
    "    while not done:\n",
    "        state = torch.FloatTensor(state).unsqueeze(0).to(device)\n",
    "        dist, _ = model(state)\n",
    "        next_state, reward, done, _ = env.step(dist.sample().cpu().numpy()[0])\n",
    "        state = next_state\n",
    "        if vis: env.render()\n",
    "        total_reward += reward\n",
    "    return total_reward"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1>A2C: Synchronous Advantage Actor Critic</h1>\n",
    "<h3><a href=\"https://blog.openai.com/baselines-acktr-a2c/#a2canda3c\">OpenAI Blog:</a></h3>\n",
    "<p>The Asynchronous Advantage Actor Critic method (A3C) has been very influential since the paper was published. The algorithm combines a few key ideas:</p>\n",
    "\n",
    "<ul>\n",
    "    <li>An updating scheme that operates on fixed-length segments of experience (say, 20 timesteps) and uses these segments to compute estimators of the returns and advantage function.</li>\n",
    "    <li>Architectures that share layers between the policy and value function.</li>\n",
    "    <li>Asynchronous updates.</li>\n",
    "</ul>\n",
    "\n",
    "<p>After reading the paper, AI researchers wondered whether the asynchrony led to improved performance (e.g. “perhaps the added noise would provide some regularization or exploration?“), or if it was just an implementation detail that allowed for faster training with a CPU-based implementation.</p>\n",
    "\n",
    "<p>As an alternative to the asynchronous implementation, researchers found you can write a synchronous, deterministic implementation that waits for each actor to finish its segment of experience before performing an update, averaging over all of the actors. One advantage of this method is that it can more effectively use of GPUs, which perform best with large batch sizes. This algorithm is naturally called A2C, short for advantage actor critic. (This term has been used in several papers.)</p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_returns(next_value, rewards, masks, gamma=0.99):\n",
    "    R = next_value\n",
    "    returns = []\n",
    "    for step in reversed(range(len(rewards))):\n",
    "        R = rewards[step] + gamma * R * masks[step]\n",
    "        returns.insert(0, R)\n",
    "    return returns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_inputs  = envs.observation_space.shape[0]\n",
    "num_outputs = envs.action_space.n\n",
    "\n",
    "#Hyper params:\n",
    "hidden_size = 256\n",
    "lr          = 3e-4\n",
    "num_steps   = 5\n",
    "\n",
    "model = ActorCritic(num_inputs, num_outputs, hidden_size).to(device)\n",
    "optimizer = optim.Adam(model.parameters())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_frames   = 20000\n",
    "frame_idx    = 0\n",
    "test_rewards = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAE/CAYAAABW/Dj8AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xl8XHW5+PHPk71Zm3TSLUmbpE0L3WnTsrXsAgIKaEVQAQFFLsK9iFcvV/yp93LdrnJV1CuCoKBS2QW9rIJCy9ampU3TPWnTNkubfd8z398fc6ZMQ9JMMss5M3ner9e8MnOWOU/OJE9Ovt/nfL9ijEEppVTki7E7AKWUUsGhCV0ppaKEJnSllIoSmtCVUipKaEJXSqkooQldKaWihCb0KCEi80Vkq4i0i8g/2x2PCi0RqRSRC+yOQzmLJvTo8XXg78aYNGPMfXYH40tE5onIcyJSLyJNIvKyiMwfss1XROSIiLSJyMMikuizLl9E/i4iXSKye2giC2TfiUBEviYiZdYf+wMi8rUh68d9foc51vnWe3RZ7zk7VN+X+jBN6NFjNrBjpJUiEhvGWIaaDDwPzAemARuB57wrReQi4C7gfDzfRyHwHz77rwPeB6YAdwNPiUh2oPuOhYjEjXWfYAjScQW4DsgELgZuE5GrfdYHcn59Y3UBzwD/D8gCSoDHgxC/8pcxRh8R/gBeBwaBHqADmAf8DvgV8ALQCVwAXIrnF7cNOAx8x+c98gED3GCtawZuAVYCpUAL8Ishx70R2GVt+zIw2894s6xjTbFePwZ8z2f9+cAR6/k8oBdI81m/Hrgl0H39iLMS+Dfr++8F4oCZwNNAPXAA+Gdr2ySgG3BZr+8GBoB06/U9wE+t5/58DjcBh4A3reXXAgeBRuu9K4ELxvnzch/w80DP7zDvezPwts/rFOucnGT378hEeegVehQwxpyH55fwNmNMqjFmr7XqM8B3gTRgA57Efh2eK+ZLgX8SkSuGvN2pQBHwaeCneJLHBcBC4CoRORtARC4HvgF8Asi2jr/Oz5DPwpMUGq3XC4FtPuu3AdNEZIq1br8xpn3I+oVB2Ncf1+A5V5MBN/AX6z1y8CS3O0TkImNMD7AJONva72w8CfhMn9dvWM/9+RzOBk4GLhKRBXj+OF+L5w/KFCDXu6GIrBaRFn++GRERYA0f/DcXyPkd6rhtjTGdQAVjO98qAJrQo9tzxpi3jDFuY0yPMeYfxpjt1utSPAn47CH73GNt+wqexLPOGFNnjKnGk7RPsba7Bfi+MWaXMWYA+B6wbLQ2UxHJBX4J3OmzOBVo9XntfZ42zDrv+rQg7OuP+4wxh40x3Xj+W8k2xvynMabPGLMfeBDwNl+8AZxtNZMswXMlfLaIJFn7vgng5+fwHWNMp3XctcBfjTFvGmN68TRpuL0bGmM2GGMm+/n9fAfP7/1vrdeBnN+hgnG+VQA0oUe3w74vRORUq6OqXkRa8SRl15B9jvo87x7mdar1fDbwMxFpsa4Om/C01eaMFIzVLvsK8L/GGN+r+Q4g3ee193n7MOu8671XlIHs6w/fczgbmOn9nq3v+xt4+gXAk9DPAZYD24FX8STq04By738kfn4Ovsed6fvauvJtZIxE5DY8/xlcav1hgMDO71DBON8qAJrQo9vQoTQfw9M5mWeMyQDux5OEx+Mw8CVjzGSfxyRjzNvDbSwimXiS+fPGmO8OWb0DWOrzeilw1EqAO4BCEUkbsn5HEPb1h+85PAwcGPI9pxljLrHWv42n4/dK4A1jzE5gFnAJHzS3gH+fg+9xa4E87wsRScbT7OI3EbkRq3PTGFPlsyqQ8zvUcduKSAowh7GdbxUATegTSxrQZIzpEZFVeNrYx+t+4N9FZCGAiGSIyKeG21BE0vF0mr5ljLlrmE0eBW4SkQUiMhn4Jp5OXaz+gK3At0UkSUSuxNOc8XQQ9h2rjUC7iPybiEwSkVgRWSQiK63jdQGbgS/zQQJ/G88VuG9CH+vn8BRwmdVWngD8J2P43RWRz+JpEvuI1Ux0TCDndxjPAotE5JNWM9O3gFJjzG5/Y1UBsrtXVh/BeQD/AL7g8/p3wH8N2WYtno66duCvwC+AP1jr8vFcFcb5bF8FnOPz+g/AN31eX4unacFbrfHwCLFdb713J55/y72PWT7b3ImneacNT/tuos+6fOv76wb2MKS6Y7z7Ap8FdpzgnFYOc6yZeNq8j+Cp7nl3yHt+3zpWovX6Nut7nzbez8HnHB5imCoXPJ2cHSf4Pg4A/UPO/f1BOr87gM/6vL4A2G291z+AfLt/NybSQ6wPQSmlVITTJhellIoSmtCVUipKaEJXSqkooQldKaWihCZ0pZSKEraMIDeUy+Uy+fn5doehlFKOtHnz5gZjzKijhDoioefn51NSUmJ3GEop5UgictCf7bTJRSmlooQmdKWUihKa0JVSKkpoQldKqSihCV0ppaKEJnSllIoSmtCVUipKjJrQRSTPmi5rp4jsEJF/sZZnicirIrLP+pppLRcRuU9EykWkVESWh/qbUEop5d8V+gDwVWPMAjxzI37ZmoX8LuA1Y0wR8Jr1GuCjeGaNLwJuxjNbuVJKqRAbNaEbY2qNMVus5+3ALjwTAV8OPGJt9ghwhfX8cuBR4/EuMFlEZgQ9cqVUWFW3dFNW3Wp3GBFnYNDNb986wP76jpAfa0xt6CKSD5wCvIdnSq1aa9URPpj5PIfjZyyvYpiZ4EXkZhEpEZGS+vr6MYatlAont9tw86MlXPfwRgbdOsvZWBxu7uY//rKTzQebQ36ssUw0m4pn4tg7jDFtvuuMZx67MX3KxpgHjDHFxpji7OxRx5xRStnoxbIj7Khpo6mzjx01epU+FhV1nivzuVNTQ34svxK6iMTjSeZ/NMY8Yy0+6m1Ksb7WWcurgTyf3XOtZUqpCDQw6ObeV/eQlzUJgPX7GmyOKLJUWE0thdkOSOgiIsBDwC5jzP/4rHoez0zkWF+f81l+nVXtchrQ6tM0o5SKMM++X83++k7uvuRkTpqexgZN6GNSXtdBdloiGZPiQ34sf67QzwSuBc4Tka3W4xLgB8BHRGQfcIH1GuAFYD9QDjwI3Br8sJVS4dA7MMhP/7aPxTkZXLRwOmuKXGw+2Ex336DdoUWMivoO5mSnhOVYo46HbozZAMgIq88fZnsDfDnAuJRSDvD4psNUt3TzvU8sRkRYXZTNg+sP8N6BRs6ZP9Xu8BzPGENFfSeXLQlPoZ/eKaqUGlZ33yA/f72cVQVZnFXkAmBVfhYJsTHa7OKnxs4+Wrv7w9IhCprQlVIjeOSdSurbe/naRfPxdKXBpIRYivMz2VCuCd0f5VaFy5wwdIiCJnSl1DDaevr51T8qOGd+Nivzs45bt7rIxe4j7dS199gUXeTwVrjM0St0pZRdfrP+AK3d/fzrhfM/tG7NXM99I2/pVfqoKuo6mRQfy4z0pLAcTxO6UhGio3eAe/66k22HW0J6nMaOXh5av59LFk9nUU7Gh9YvnJlOZnK81qP7oaK+gzlTU4iJGamuJLg0oSsVAZo7+/jsg+/y0IYD3PTIJmpaukN2rPvfqKC7f5A7PzJv2PUxMcKZc11s2NeAp6hNjaS8riNs7eegCV0pxzvS2sNVv36HXUfa+dZlC+jpd/Ol32+mpz/4teBHWnt45J2DXHlKLnOnpo243ZoiF3Xtvew9GvoBpyJVd98g1S3dmtCVUh4HGztZe//b1LR088gNq7hxdQE//fQyympauevp0qBfIf/89X0YY7jjgqITbre6yNOOvn6fDqw3kv0N4a1wAU3oSjnWrto21t7/Dp29A6y7+TROnzMFgAsWTOPOC+bx5601/Gb9gaAd71BjF49vOszVK2eRl5V8wm1zJk+i0JWi5YsnUFHfCcCcqeG5SxQ0oSvlSJsPNvPpX79DrAhP3nI6S3InH7f+tvPmcsni6Xz/xV28uTc4V8k//dte4mKF28+b69f2q4tcvLe/id4BHQZgOBV1HcQI5E/RhK7UhLV+Xz2f+817ZKUk8OQtpw/bli0i/GjtUuZNS+P2de9T2dAZ0DH3Hm3n2a3VXH96PlP9LLFbPddFd/8gWw6GtuomUpXXd5CXlUxSfGzYjqkJXSkHeXF7LTf+bhOzpyTzxC2nn7DpIyUxjgevK0YEvvhoCR29A+M+7r2v7CElIY5bzp7j9z6nzZlCbIywoVzb0YdTEeYKF9CErpRjPLHpMF9+bAtLcifz+JdOZ2ra6FfKeVnJ/PIzy9nf0Mmdj2/FPY7ZhLYdbuHlHUf54ppCMlMS/N4vPSmeZXmTdVyXYQy6DQcaOsM2yqKXJnSlHOA36/fz9adLWV2Uze9vWjWmsbPPnOvi7ktO5pWdR7nv9X1jPvaPX9lDZnI8N67OH/O+a4pclFa30tLVN+Z9o1lNSze9A269QldqIjHGcO8re/iv/9vFpYtn8JvriklOGHVU6w+54cx8Prk8l5/+bR8vlR3xe7939zeyfl8Dt54zl7SksU/AsKbIhTHwdkXjmPeNZuVhnHbOlyZ0pWzidhu+/fwOfv56OVevzOO+a04hIW58v5IiwnevXMTSvMl89Ymt7D3aPuo+xhh+/PIepqUncu3ps8d13KW5k0lLjNNhAIY4NiiXXqErFf36B93c+cRWHn3nIF86q5Dvf2IxsQGO95EUH8sD164gOTGOLz5aMmozyD/21lNysJnbzysadyVGXGwMp82Zwvp99ToMgI+K+g6yUhLG1CcRDJrQlQqznv5B/ukPm/nz1hq+dtF87vroScfGGw/UtPQk7v/cCmpberh93fsMDLqH3c7t9lyd52VN4qrivGG38deaIhdVzd0cbOwK6H2iSUVd+DtEQRO6UmHV3tPP9Q9v5LXdddxzxSK+fO7coCVzrxWzM7nnioWs39fAD1/aPew2L+04wo6aNr5ywbxxN/N4rZ7rmc1ovd41ekxFfUfY289BE7pSYbPnSDtrf/UOmw8289NPL+Pa08bXbu2PT6+cxfWnz+bB9Qd49v2q49YNuj0dsUVTU7l8WU7AxypwpZAzeRIbdFwXwDMyZmNnX9jbz0ETulIhZ4zhkbcr+dgvNtDY2ctvb1gZlEQ6mm9etoDTCrP4t6e3U1r1wd2cz75fTUV9J1+9cF7A7fbg6ZBdPdfF2xWNIzbxTCR2dYiCJnSlQqqho5cvPFLCt5/fwRlzpvDiv5zFGmukwlCLj43hl59ZTnZqIl/6/Wbq23vpG3Dz07/tZXFOBhctnB60Y60uctHeM0BpdWvQ3jNSaUJXKgq9sbeei3+6nvXlDXznYwv47edXkp2WGNYYpqQm8sB1K2ju6uOf/rCZ3797kKrmbv7VZ+LnYDhzrgsR9K5RPKMsJsTFkJM5KezH1oSuVJD1Dgxyz193cv3DG8lKief5287k82cWBL3z018LZ2bwo7VLKTnYzD1/3cmqgizOKnIF9RhZKQksmpmhCR3PTUWFrpSgNGeN1agJXUQeFpE6ESnzWfa4iGy1HpUistVani8i3T7r7g9l8Eo5TXldO1f88m0e2nCA60+fzfO3reak6el2h8XHls7k1nPmECPwtSBfnXutLnKx5VBzQIOERQPPPKLhb24B8Oce498BvwAe9S4wxnza+1xE7gV8G84qjDHLghWgUpHAGMNjGw9xz193kpwQx0PXF3P+ydPsDus4X7toPjeuLsCVGppmnzVzXfzqHxW8t7/Rcd97uPT0D3K4qSssnd7DGTWhG2PeFJH84daJ58/8VcB5wQ1LqcjR1NnHvz1dyqs7j7KmyMW9n1rq95ji4SQiIUvmACvyM0mKj2H9voYJm9APNnbhNthyUxH4d4V+ImuAo8YY3yHeCkTkfaAN+KYxZn2Ax1DKsd4qb+Arj2+luauPb156MjeeWUCMDW2nTpAYF8uqgikTep5Ruwbl8gq0U/QaYJ3P61pgljHmFOBO4DERGbYBUURuFpESESmpr5+4PwAqMvUNuPn+C7v43EPvkZYUx7O3nskX1hRO2GTutWaui4r6Tmpbu+0OxRbeksVCV4QldBGJAz4BPO5dZozpNcY0Ws83AxXAvOH2N8Y8YIwpNsYUZ2eHpy5XqWDYX9/BJ3/1Nr9+cz/XrJrFX29fw6KcDLvDcoTVVvXMRB19saK+g5zJk5iUEL5p53wFcoV+AbDbGHPsvmIRyRaRWOt5IVAE7A8sRKWcY/PBZi69bwOHm7v49bUr+N6Vi2375XWik6an4UpNnLDli3ZWuIB/ZYvrgHeA+SJSJSI3Wauu5vjmFoCzgFKrjPEp4BZjTFMwA1bKTn9+v5rYGOGlfzkrqHdaRgsRYU2Ri7fKG8Y1HV4kc7uNbaMsevlT5XLNCMs/P8yyp4GnAw9LKWcqrWphcU4G0zOcV8XiFKvnunj2/Wp2HWlj4cyJ0xRV29ZDd/+gbR2ioHeKKuW3vgE3u2rbWZI3cZLUeHjb0Sdas0tFnX1juHhpQldht2FfA/e+ssfuMMZs95E2+gbdLM2dbHcojjYtPYl501LZMMHGR7dzUC4vTegqrIwxfPeFXfz89XLeirBf+G1Vnhuil+TqFfpoVs/NZuOBJnr6B+0OJWwq6jtIT4rDlRreaed8aUJXYVVa1cqu2jZE4Ecv74moeShLD7eQlZJAzuTwj6IXadYUuegdcFNS2Wx3KGFTXuepcLFrEDbQhK7CbN3GQ0yKj+XuS05m6+EWXttVZ3dIfiutamVJboatv7CR4tTCLOJjZULdNVpR38lcG5tbQBO6CqOO3gGe31bDZUtmcP0Z+eRPSebHr+yJiPK2rr4B9tW1s0Tbz/2SnBDH8lmZE+YGo9bufurbe22tQQdN6CqMnt9aQ1ffINecOov42Bi+8pF57D7Szv9tr7U7tFGVVbfhNrBU28/9dta8bHbWttHQ0Wt3KCG33wEdoqAJXYXRnzYd4qTpaZyS57nK/diSmcyflsZPXt3r+LkovXNy6hW6/1bP9ZQvRlrn93hU1HcC9o2y6KUJXYVFWXUrpVWtXL0y71gbdEyMcOeF89jf0Mkz71fbHOGJlVa1MjMjKexTyEWyRTkZZEyKnxD16OV1HcTHCrOykm2NQxO6Cos/bTpEYlwMV56Se9zyCxdMY2luBj/72z56B5xb4lZa1aJX52MUGyOcOXcKG8obIqqaaTwq6jvIn5JCXKy9KVUTugq5rr4B/vx+DZcunkFGcvxx60SEr144n+qWbh7fdNimCE+staufysYuvUN0HFbPzaa2tedYk0S0qqjvsL39HDShqzD467ZaOnoHuObUWcOuX1PkYlVBFj9/vZzuPuddpZdWe9rP9Q7RsVtzbBiA6C1f7B90c6ixizlT7W0/B03oKgzWbTrE3KmpFM/OHHa9iPC1i+ZT397Lo+9UhjU2f5Rad4jqmOdjl5eVzOwpyVE9DMDBxk4G3Eav0FX0232kjfcPtRzXGTqclflZnD0vm1+9UUF7T38YIxzdtsMtFLpSyJgUP/rG6kNWz3XxTkUj/Q6vZBqv8jpPc5Kdoyx6aUJXIfWnjYdJiI3hE8tzR932Xy+cT0tXPw9vqAx9YGPgvUNUjc+aomw6+wZ5/1CL3aGExLFp5/QKXUWznv5BntlSxcWLppOVMvqARYtzM7h44XR+s34/LV19YYhwdHVtPRxp69EKlwCcPmcKMRK97egV9R1MT08iNXHU6SVCThO6CpkXttfS1jPA1avy/N7nzgvn0dE3wP1vOGPmQu8Ii0u1wmXcMibFszRvMuujtB29oq7DER2ioAldhdC6jYfIn5LM6YVT/N5n3rQ0Ll86k9+9fYC69p4QRuef0qoWYmOEBTM0oQdizVwX2w630NrtrP6RQBljHDEol5cmdBUS5XXtbKps5ppVs8Y8OuEdF8yjf9Dwv3+vCFF0/ttW1cq8aWk6EXSAVhdl4zbwTkWj3aEEVV17Lx29A7YPyuWlCV2FxLqNh4mPFT65YvTO0KHyXSlcVZzLY+8dorqlOwTR+ccY47lDVMsVA3bKrMkkJ8Ty7v7oSuhOmHbOlyZ0FXTeztALF0zHlTq+sU9uP68IgPv+ti+YoY3J4aZuWrr69Q7RIIiPjWFOduqxipBo4YRp53xpQldB9/KOIzR39Y+pM3SomZMn8dnTZvHUlioONNhz2/i2Kr1DNJgKs1Ns+yxDpbyug9TEOKalO2PQNk3oKuj+tPEweVmTOHOOK6D3ufWcuSTExvCTV/cGKbKxKa1qISEuhvnT02w5frQpcKVQ3dIdVfOMVtR3Mic7xTGzWGlCV0F1oKGTd/Y3cvXKWcTEBPZDnp2WyA1n5vOX0hp2H2kLUoT+21bVyoIZ6cTbPIJetChwpWAMHGrqsjuUoHHKoFxe+pOqgupPmw4RGyN8ahydocP50llzSE2M495XwnuVPug2lFW36gxFQVTo8iS+/VEy8mJH7wC1rT2OqXABPxK6iDwsInUiUuaz7DsiUi0iW63HJT7r/l1EykVkj4hcFKrAlfP0Dbh5qqSK80+aytT0pKC8Z0ZyPDevKeTVnUfZejh8t45X1HfQ1Teod4gGUb7LM/lDtLSjfzDtnDNuKgL/rtB/B1w8zPKfGGOWWY8XAERkAXA1sNDa539FRAt4J4hXdx6lsbNvxGFyx+uG1QVkpSRw7yt7gvq+J7LN+uOhd4gGT1pSPNlpiRxoiI5KF6dVuIAfCd0Y8ybQ5Of7XQ78yRjTa4w5AJQDqwKIT0WQP206RM7kSZxVlB3U901NjOPWc+awfl9D2OqYt1e3kpoYd6yZQAVHgSt6Kl0q6jqJjRFmT4msK/SR3CYipVaTjHeg6xzAd9qZKmuZinKHGrtYv6+Bq4rziA2wM3Q4nzttNtPSE/nxy3vCMp3ZtqpWFuWkB9yxq45XGE0Jvb6D2VnJJMQ5pytyvJH8CpgDLANqgXvH+gYicrOIlIhISX19dI7CNpE8XnKIGIGrVganM3SopPhYbj+viJKDzfxjb2h/XvoG3OyqadP68xAocKXQ0NEXFWO6lNd1OGLIXF/jSujGmKPGmEFjjBt4kA+aVaoB37tJcq1lw73HA8aYYmNMcXZ2cP9FV+HVP+jmiZIqzp0/lRkZk0J2nKuK88jLmsS9r4T2Kn3PkXb6Bt3aIRoCBS5P80SkX6UPDLqpbOx0zCiLXuNK6CIyw+fllYC3AuZ54GoRSRSRAqAI2BhYiMrpXt9dR317L9esCm5n6FAJcTHccf48yqrbeKnsSMiO471DVCe1CD7vFW2kd4webu6mf9A4ZpRFL3/KFtcB7wDzRaRKRG4C/ltEtotIKXAu8BUAY8wO4AlgJ/AS8GVjTPTcFqaGtW7jIaalJ3LO/ND/p3XFKTkUulL4zYYDITtGaVULWSkJ5GaG7r+NiWpWVjIxAgcivBb92KBcDqpBBxh1ig1jzDXDLH7oBNt/F/huIEGpyFHd0s0be+u5/dy5xIXhjsrYGOGqlXn84MXd7K8PTRumd8o5p9zOHU0S4mLIy0pmf4Q3uRwrWXRYFZRzumdVRHp8k6eo6aqV4x+Ia6w+cUoOsTHCU5urgv7eXX0D7D3aru3nIRQNpYvldR24UhPJSHbWxOGa0NW4DQy6ebLkMGcVZZObmRy2405NT+Lsedk8s6WaQXdwO0d31LThNugY6CHkTejhKD8NFc8YLs7qEAVN6CoAb+ytp7a1h2sCGCZ3vNauyOVIWw8bgjxPpfcOUR0DPXQKXSl09Q1S195rdyjjcmzaOYe1n4MmdBWAdRsP40pN5PyTp4X92OefPJXJyfFBb3YprWplRkYSU9OCMxaN+rCCCB+kq7HTU0fvpFv+vTShq3E50trD67uPclVxri3DyybGxXL50pm8vOMIrV3Bu0mltKpFyxVDrCA7smvRyx1a4QKa0NU4PVFyGLeBT4exM3SoTxXn0Tfg5vnSmqC8X2tXP5WNXdohGmIz0pNIio85NlphpKlw4CiLXprQ1ZgNug2PbzrM6rkuWwcmWjgznZOmpwWt2aW0WqecC4eYGCF/SuRWulTUdTIpPpaZIbwrerw0oasxe29/I9Ut3QHNGRoMIsLaFblsO9zCvqPtAb9faVUrAIu1ySXkInl+0Yr6DgqzUxw5cJsmdDVmb+5rIC5GOHf+VLtD4YpTcogLUk36tsMtFLhSyJjkrNriaFTgSuFQUxf9g267Qxkzp00750sTuhqztysaWD4rk5TEUW80DjlXaiLnnjSVZ96vZiDA5OC9Q1SFXoErlQG3oaq52+5QxqS7b5Dqlm5N6Co6tHT1sb26lTPnuuwO5ZhPrcilvr2XN/eNf1jdurYejrT1aIdomHww6mJkdYzub+jAGBw3yqKXJnQ1Ju/ub8QYOHPuFLtDOebck6YyJSWBJ0vG3+yyzWo/10mhw6PQSuiRVoteYcXrxJuKQBO6GqMN5Q2kJMSyNM85V7LxsTFccUoOf9t1lKbOvnG9R2lVC7ExwsKZmtDDITMlgcnJ8RHXMVpR14EI5Dto2jlfmtDVmLxd3siphVNsuZnoRNauyKV/0PD81mHnUxlVaVUrRVNTmZSgc5qHSyQO0lVe30FeZjJJ8c78OXHWb6VytJqWbvY3dDqq/dzr5BnpLMpJ58lxVLsYYyitatH68zCLxIReUefMQbm8NKErv71lDYTlpPZzX2uX57Kjpo2dNW1j2q+quZvmrn4dkCvM5mSnUtvaQ1ffgN2h+GXQbTjQ0OnYChfQhK7G4K3yBlypCcyflmZ3KMO6fFkO8bFjr0n3TjmnV+jhFWnzi9a0dNM74HZshyhoQld+MsbwVkUjZ8xxOXYmn8yUBC44eRp/3lpN34D/NemlVa0kxMYwz6F/qKJVpCV0Jw/K5aUJXfllX10H9e29rHZg+7mvTxXn0tTZx9/31Pm9z7bDLZw8M52EOP11CCdvpUikzC/6waBcmtBVhNuwz9N+foZD28+9zirKJjst0e+a9EG3oay6VevPbTApIZaZGUkRc4VeUd9BZnI8WSkJdocyIk3oyi9vVzSQPyU5rFPNjUdcbAyfOCWHv++po96PGXH213fQ2Teod4japCA7JWImjK6oc+YsRb40oatRDQy6eXd/E2c4vLnFa+2KXAbdhuf8qEnXO0TtVeDUvRNZAAAgAElEQVRKYX99R0TML+rkQbm8NKGrUW2raqWjd8Dx7edeRdPSWJo3mSdLqkZNFKVVLaQkxFLo8F/UaFXgSqWtZ4DmIM46FQrNnX00dvZpQleR763yBkTg9EJnt5/7+tSKXPYcbaes+sQ16duqWlmUk0GsA8e2nggKI2SQrmMdog4dlMtLE7oa1VvlDSycmU6mgzuDhvrYkpkkxMXw1ObDI27TN+BmV02bo8almWi8pYsVDq90iYQKF/AjoYvIwyJSJyJlPst+JCK7RaRURJ4VkcnW8nwR6RaRrdbj/lAGr0Kvq2+ALYeaOXNOZDS3eGUkx3PRwuk8t62G3oHBYbfZc6SdvkG3joFuo9zMScTHiuMrXSrqO0mIi3F8UYA/V+i/Ay4esuxVYJExZgmwF/h3n3UVxphl1uOW4ISp7LKpspn+QePI8VtGs3ZFLi1d/fxt5/A16XqHqP3iYmOYlZXs+Fr08roOCl0pjm+aGzWhG2PeBJqGLHvFGOMdgOFdIDcEsSkHeKu8gYTYGFbmZ9kdypitnutiRkbSiM0upVUtZCbHk5vpvMl+J5ICV6rjr9D3Hm139B2iXsFoQ78ReNHndYGIvC8ib4jImiC8v7LRW+UNLJ89OSKHlY2NET6xPIc39tZztK3nQ+s9U85NduxQBhNFYXYKBxo7cbudWbrY3NlHVXM3iyJgrPyAErqI3A0MAH+0FtUCs4wxpwB3Ao+JSPoI+94sIiUiUlJfP/6pw1ToNHX2saOmLWLKFYfzyeW5uA08+/7xNeldfQPsPdqu9ecOUOBKoW/ATU2rM+cX3WGN3rkoZ9hU5ijjTugi8nngMuCzxir2Ncb0GmMareebgQpg3nD7G2MeMMYUG2OKs7OzxxuGCqF3KhoBIuaGouEUZqdSPDuTJ0sOH1eTvqOmDbdB7xB1AKcP0lVW47n5LGqv0EXkYuDrwMeNMV0+y7NFJNZ6XggUAfuDEagKvw3lDaQlxrEkx/k/yCeydkUuFfWdbD3ccmzZNuu5VrjYr9DhCX17dSs5kydFRNmuP2WL64B3gPkiUiUiNwG/ANKAV4eUJ54FlIrIVuAp4BZjTNOwb6wc7+2KBk4tnEKcw6abG6tLl8wgKT7muNmMSqtamZ6exNT0JBsjUwDZaYmkJMQ6dsLoHdWtEdHcAhA32gbGmGuGWfzQCNs+DTwdaFDKfoebujjY2MUNZ+TbHUrA0pLi+eiiGfxlWw3fumwBSfGxbK9u1atzhxARCrKdOR1dW08/lY1drF0RGYV8kX3ppULm7QrvdHOR237u61MrcmnvGeDlHUdo7e7nQEOn3iHqIIWuVPY78PZ/73SGCyOk2VETuhrWhvJGpqYlOn64UH+dVjiFnMmTeGpzFdutERb1Ct05ClwpVDV3j3hXr13KqiOnQxQ0oathuN2Gt8sbOHOuc6ebG6uYGOGTK3LZUN7ASztqAViSo1foTlGYnYIxcKixa/SNw6is2tPXkp2WaHcoftGErj5kz9F2Gjv7oqa5xWvt8lyMgXUbD5M/JZmM5Hi7Q1IWb+mi0ya7KKtpi5gOUdCErobxVrm3/Txyhsv1x6wpyZxakMWg22j9ucPkO7B0sbN3gIr6DhZGSHMLaEJXw3irvIHC7BRmZETfGCefKs4DtP3cadKT4nGlJjpqkK5dtW0YA4sjpEMU/ChbVBNL34Cb9w408cnlkVGmNVaXLZnBniNtfHzZTLtDUUMUupxVunisQzSCErpeoavjbKtqoatvMOraz72S4mO5+9IFTE3TG4qcpsDlrAmjt1e34UpNYFp6ZHSIgiZ0NcSGfQ3ERNh0cyo6FGSn0NDRS1uPM+YX3VHjmZ4wkiq9NKFHqNd2HeVpn1vZg+XtigYW52RoBYgKO++YLpUOuErv6R9kX11HxNSfe2kbegTaeKCJL/1+M4PGkJs5iVODdDXd2TvA+4da+OJZhUF5P6XGojDbKl2s77S9CmlXbRuDbhNR7eegV+gRp6alm1v/uJm8rGTyMpP516e20dk7MPqOfth4oIkBt4no8c9V5MrLSiZGnFGLXhZBY6D70oQeQXr6B7n59yX09Lt58LoV/PhTS6lq7ua7L+wKyvtvKG8gIS6GFbMzg/J+So1FYlwsuZnJjqh02VHdyuTkeHImR1bprib0CGGM4a6nS9lR08ZPP72MuVPTWFWQxRfXFPLYe4d4Y2/gsz69Vd7AyvxMkuIjb7o5FR0KXCkccMAgXdurW1kcYR2ioAk9Yjy4fj9/3lrDVz8yjwsWTDu2/M6PzKNoaipff2obrV3jrw5o6Ohl95H2qC1XVJGhwJXCgfrO42aXCrfegUH2Hm2PqDtEvTShR4A39tbzgxd3c8ni6Xz53LnHrUuKj+V/rlpGQ0cf3/nLjnEf421rurkz52hCV/YpzE6hs2+Q+vZe22LYd7SD/kETce3noAnd8SobOrn9sS3Mm5bGj9YuHfZfwMW5Gdx27lyefb+al8pqx3Wct/Y1kJ4UF3G9+iq6OGGQru3WHaKRdMu/lyZ0B+voHeCLj5YQGyM8eF0xKYkjV5nedt5cFuWkc/ezZTR0jO3qxhjDhvIGTp8zhdiYyGozVNHFCRNGl1W3kpYUx6ysZNtiGC9N6A7ldhu+8vhW9jd08svPLCdvlB+u+NgY/ueqZbT3DPCNZ7aPqQ3yUFMX1S3dWq6obDczYxKJcTH2JvSaNhbOTI+4DlHQhO5YP3ttH6/uPMrdl5zMGX4m2nnT0vjqhfN4ZedR/ry12u9jbbCGy/X3OEqFSkyMeMZ0sWnUxf5BN7tq2yKyuQU0oTvSS2W1/Oy1faxdkcsNZ+aPad8vrCmkeHYm33puB7Wt3X7t83Z5IzMyko7deq2UnTyDdNlTulhe10HfgDti+5I0oTvMniPt3PnENpbmTea/rlg05n/7YmOEH39qKQODhq8/VTpq04vbbXi7ooEz5kTPdHMqshW4UjjU2MXAoDvsx/YOmRuJJYugCd1RWrr6+OKjJaQkxvHAtSvGfYNPviuFb1xyEuv3NfDH9w6dcNudtW00d/WzukhHV1TOUOBKYcBtqGr27z/MYCqrbiUlITZi/1vVhO4QA4NubnvsfY609nD/51YwLT2w8bo/d9ps1hS5+N4LuzjYOHJ7pHe6uTO0/lw5hHeQLjs6Rstq2lgwM52YCK320oTuED94cTcbyhv4rysWBWUsFRHhh59cQqwIX3uylEH38E0vb1U0UjQ1NeA/IEoFS4ErFQh/Lfqg27Czpi1im1vAz4QuIg+LSJ2IlPksyxKRV0Vkn/U101ouInKfiJSLSKmILA9V8NHimS1V/GbDAa4/fTZXrcwL2vvOnDyJb398IRsrm3h4w4EPre8dGGTjgUa93V85SmZyPBmT4sM+psv++g66+wcjtsIF/L9C/x1w8ZBldwGvGWOKgNes1wAfBYqsx83ArwIPM3qVVrVw1zPbOa0wi29etiDo7//J5Tl8ZME0fvTKHvYdbT9u3fuHWujpd2tCV44iItYgXeG9Qi+ribw5RIfyK6EbY94EmoYsvhx4xHr+CHCFz/JHjce7wGQRmRGMYKNNXXsPNz+6mezURH75meXExwa/BUxE+N6Vi0lJiOXOJ7bR71M58Fa5Z7q5Uwuzgn5cpQJRaA3SFU5l1W0kxccwJzsyO0QhsDb0acYY78AhRwDvEIA5wGGf7aqsZcpH34CbW/+whZbuPh64bgVTUkM3EW12WiLfvXIx26tb+d+/Vxxb/lZ5A0vzJpOepNPNKWcpzE6hprWH7r7BsB1ze3UrJ89IJy4EF1bhEpTIjafYeUzjXYrIzSJSIiIl9fWBj+UdaZ7aXEXJwWb+e+3SsHTCXLJ4Bpcvm8nPX99HWXUr7T39bKtq1dEVlSN5O0YrT1ChFUxuq0M00uYQHSqQhH7U25Rifa2zllcDvj17uday4xhjHjDGFBtjirOzswMIIzK9u7+RaemJfGxJ+Fqj/uPjC8lKSeDOJ7by5t4GBt1G28+VIx0bdTFMzS4Hm7ro6B2IyCFzfQWS0J8HrreeXw8857P8Oqva5TSg1adpRuEZ3XBTZRPF+VlhvTtzcnICP1y7hL1HO/jGs9tJio9h+Wx7J+NVajj5Ls9gdOGqdPEOmRvJHaLgf9niOuAdYL6IVInITcAPgI+IyD7gAus1wAvAfqAceBC4NehRR7iq5m5qW3tYlR/+zshz50/lmlV5tHb3szI/i8Q4nW5OOU9yQhwzMpLCVou+o7qVhNgYiqamheV4oTLyANs+jDHXjLDq/GG2NcCXAwkq2m2q9BQMrbQhoQPcfekCyus6WLsi15bjK+WPcJYultW0Mn96GglxkdshCn4mdBVcmyqbSUuKY/50e64GUhPjePKWM2w5tlL+KnCl8H/bQ99aa4yhrLqNSxZHfnV1ZP85ilCbKptYMTtTZwdS6gQKXCm0dPXT3NkX0uNUNXfT2t0f8R2ioAk97Jo6+yiv67CtuUWpSOEdpCvU7ejeIXMjvWQRNKGHnbf9fFWBJnSlTqTQqkUPdTv69upW4mLEtibQYNKEHmYllU0kxMWwJDfyrwaUCqXczEnExUjISxfLatoompY27vkHnEQTephtrGxmaW6GlgsqNYq42BhmTUkO6c1Fxhh2VLeyaGbkt5+DJvSw6uobYEd1q7afK+WnwhCXLta29tDY2cfiKPmPWRN6GG091MKA27BS28+V8ou3Ft09wgQtgYr0OUSH0oQeRhsrmxAhKDMSKTURFLhS6R1wU9vWE5L3L6tpI0ZgwQxtclFjtKmyiZOmp+twtUr5yTtIV6jGRi+rbmXu1FQmJURHn5Ym9DDpH3Sz5WALq/L16lwpf30wYXRoKl3Kqlujov7cSxN6mOysaaO7f1Dbz5Uag6lpiaQkxIbk5qK6th7q2nsjfoRFX5rQw8TuAbmUikQiQkF2aCpdomEO0aE0oYfJxgNNzMpKZlp6kt2hKBVRClypoUno1W2IwIIoqUEHTehhYYyh5GCzXp0rNQ4FrhQON3XRN+AefeMxKKtupcCVQmpi9Aw6qwk9DCrqO2nq7GNVgXaIKjVWha4U3AYONQX3Kj3aOkRBE3pYaPu5UuMXivlFGzt6qWntiYohc31pQg+DTQeacKUmHPvBVEr5L99bix7EdvQdNW1AdHWIgib0sNhY2UTx7PBOCK1UtMiYFI8rNSGoCX17lN3y76UJPcRqW7upau7W+nOlAlDgSglqLfqOmlZmZSWTMSm67trWhB5imyqbAVil7edKjVuwJ4wuq25jcZQ1t4Am9JDbdKCJlIRYTp4R+bOhKGWXAlcq9e29tPf0B/xerV39HGrqYmGUdYiCJvSQ21TZxPLZmcTF6qlWary8Y7rstDozA7GjJnrmEB1Ks0wItXb1s+dou5YrKhWg4tmZTElJ4I7Ht3K4qSug94rGW/69NKGH0OZDTRij9edKBWpKaiJ/+MKpdPcPcvUD71Ld0j3u99pe3UbO5ElkpSQEMUJnGHdCF5H5IrLV59EmIneIyHdEpNpn+SXBDDiSbDzQTHyssCxvst2hKBXxTp6Rzh9uOpW2nn6ueeBdjrSOb9KLHdWtLIyi8Vt8jTuhG2P2GGOWGWOWASuALuBZa/VPvOuMMS8EI9BItKmyiUU5GVEzeL5SdluUk8GjN66iqbOPzzz4LnXtY0vq7T397G/ojMoKFwhek8v5QIUx5mCQ3i/i9fQPUlrVouWKSgXZKbMy+e0NKznS1sNnH3yPxo5ev/fdGaV3iHoFK6FfDazzeX2biJSKyMMiMiFHpNp2uIX+QaPt50qFwMr8LH5zfTGHmrr43EMbaenq82u/MiuhR2PJIgQhoYtIAvBx4Elr0a+AOcAyoBa4d4T9bhaREhEpqa+vDzQMx/EOyFWsU84pFRJnzHHx4HXFVNR1cO1DG2ntHr1GfUd1K9PSE5maFp3zEgTjCv2jwBZjzFEAY8xRY8ygMcYNPAisGm4nY8wDxphiY0xxdnZ2EMJwlo2Vzcyblsrk5OjrSVfKKc6al8391y5n95E2Pv/bjXT0Dpxw++1ROGSur2Ak9GvwaW4RkRk+664EyoJwjIgy6DZs0QktlAqL806axs+vWU5pVSs3/HYjXX3DJ/WuvgEq6jtYGKXt5xBgQheRFOAjwDM+i/9bRLaLSClwLvCVQI4RiXbVttHRO8AqHZBLqbC4eNF0fnb1MjYfbOYLj5TQ0z/4oW121bbjNkRthQtAQHMvGWM6gSlDll0bUERRQCe0UCr8Llsyk4FBw1ee2MoXHy3hweuKSYr/oGS4rNp7h2h0doiC3ikaEpsqm8iZPImZkyfZHYpSE8oVp+Tww08sYf2+Bm7945bj5iEtq25lSkoC06N4onZN6EFmjGFTZTMrtbpFKVtctTKP/7piEa/vruP2dVvoH/Qk9bKaNhblZET1RDOa0IPsYGMX9e29OqGFUjb63Gmz+fbHFvDyjqN85fGtdPUNsO9oe1Q3t0CAbejqwzZa7ed6h6hS9rrhzAL6Btx8/8Xd1Lb2MOA2UV2yCJrQg66ksonJyfHMyU61OxSlJrwvnT2HvgE39766F4jeW/69NKEH2abKZopnZxETE73tdEpFktvPL0IE3tzXQG5mdBcqaBt6ENW193CgoZNVBdohqpST3HZeEU986fSo7hAFTehBVWJNCK3150opO2hCD6JNlU0kxcewMMo7XpRSzqQJPYg2VTZxSl4mCXF6WpVS4aeZJ0jae/rZWdOm9edKKdtoQg+SLYdacButP1dK2UcTepCUVDYRGyOcMksnhFZK2UMTepBsPNDEwpnppCRqab9Syh6a0IOgd2CQrYdbtFxRKWUrTehBUFbdSu+AWxO6UspWmtCDYJN1Q5FOCK2UspMm9CDYdKCJwuwUXKmJdoeilJrANKEHyO02lBxs1nJFpZTtNKEHaG9dO63d/dp+rpSynSb0AHnbz1fpHaJKKZtpQg/QpgNNTEtPjPpxlpVSzqcJPQCeCaGbWJmfFfXjLCulnE8TegCqmrupbe3R5hallCNoQg9AyUHPhNDaIaqUcoKABx4RkUqgHRgEBowxxSKSBTwO5AOVwFXGmOZAj+U0Gw80k5YUx7xpaXaHopRSQbtCP9cYs8wYU2y9vgt4zRhTBLxmvY46myqbKJ6dSaxOCK2UcoBQNblcDjxiPX8EuCJEx7FNRX0H5XUdOqGFUsoxgpHQDfCKiGwWkZutZdOMMbXW8yPAtCAcxzHcbsNdT5eSnhTH2hW5doejlFJAENrQgdXGmGoRmQq8KiK7fVcaY4yImKE7Wcn/ZoBZs2YFIYzw+cN7B9lU2cyP1i5halqS3eEopRQQhCt0Y0y19bUOeBZYBRwVkRkA1te6YfZ7wBhTbIwpzs7ODjSMsKlq7uKHL+5mTZFLr86VUo4SUEIXkRQRSfM+By4EyoDngeutza4HngvkOE5hjOEbz5ZhgO9duVhvJlJKOUqgTS7TgGetxBYHPGaMeUlENgFPiMhNwEHgqgCP4whPb6nmzb31/MfHF5KXlWx3OEopdZyAEroxZj+wdJjljcD5gby309S193DPX3dSPDuTa0+bbXc4Sin1IXqnqJ++/dwOuvsH+eHaJcRo3blSyoE0ofvhxe21vFh2hDsuKGJOdqrd4Sil1LA0oY+ipauP//fcDhbOTOeLawrtDkcppUYUjDr0qHbPX3fR3NXHIzeuJD5W//4ppZxLM9QJvLG3nqe3VHHL2YUsnJlhdzhKKXVCmtBH0NE7wDee2c6c7BRuP6/I7nCUUmpU2uQygh+9tJua1m6euuV0kuJj7Q5HKaVGpVfow9hU2cQj7xzk+tPzWTFbR1NUSkUGTehD9PQP8m9PlZKbOYmvXTTf7nCUUspv2uQyxM9e28f+hk5+f9MqUhL19CilIodeofsoq27lgTf3c1VxLmuKImcESKWUAk3ox/QPuvnaU6VkpSRw9yUL7A5HKaXGTNsULL9+o4JdtW38+toVZCTH2x2OUkqNmV6hA+V17dz3WjmXLp7BRQun2x2OUkqNy4RP6INuw9efKiU5MZbvfHyh3eEopdS4TfiE/sjblWw51MK3P7aA7LREu8NRSqlxm9AJ/XBTFz96eQ/nzM/mimU5doejlFIBmbAJ3dvUEiM6P6hSKjpM2IT+s7/t5Z39jXz74wuZOXmS3eEopVTAJmRCf2NvPT//ezlrV+RyVXGe3eEopVRQTLiEXtPSzR1/ep/509K45/JFdoejlFJBM6ESev+gm9se20LfgJtffnY5kxJ0WFylVPSYUHeK/uDF3Ww51MIvPnOKTvaslIo6E+YK/aWyWh7acIDrT5/NZUtm2h2OUkoF3YRI6JUNnXztyVKW5mbwjUtPtjscpZQKiXEndBHJE5G/i8hOEdkhIv9iLf+OiFSLyFbrcUnwwh27nv5Bbv3jFmJihF9+djmJcdpurpSKToG0oQ8AXzXGbBGRNGCziLxqrfuJMebHgYcXuP/4yw521rbx8OeLyc1MtjscpZQKmXEndGNMLVBrPW8XkV2Ao+6ff2ZLFes2HuafzpnDeSdNszscpZQKqaC0oYtIPnAK8J616DYRKRWRh0UkMxjHGKu9R9u5+9kyTi3I4qsfmWdHCEopFVYBJ3QRSQWeBu4wxrQBvwLmAMvwXMHfO8J+N4tIiYiU1NfXBxrGcTp7B/inP2wmJTGOn19zCnGxE6LvVyk1wQWU6UQkHk8y/6Mx5hkAY8xRY8ygMcYNPAisGm5fY8wDxphiY0xxdnbw5u80xvDvz2znQEMn912zjKnpSUF7b6WUcrJAqlwEeAjYZYz5H5/lM3w2uxIoG394Y/eH9w7x/LYa7vzIPM6Y4wrnoZVSylaBVLmcCVwLbBeRrdaybwDXiMgywACVwJcCinAMSqtauOcvOzlnfja3njM3XIdVSilHCKTKZQMw3CDiL4w/nPFr7ern1j9uwZWawE+uWkZMjI5vrpSaWKJiLBdjDF99chtH23p4/Eunk5mSYHdISikVdlFR/vHAm/v5266jfOOSk1k+y5YqSaWUsl3EJ/SNB5r475f3cOniGXz+jHy7w1FKKdtEdEJv6Ojl9nVbmJWVzA8+qfOCKqUmtohO6L94vZyWrn7+97PLSUuKtzscpZSyVUR3iv77JSdx2ZIZnDwj3e5QlFLKdhF9hZ4YF0txfpbdYSillCNEdEJXSin1AU3oSikVJTShK6VUlNCErpRSUUITulJKRQlN6EopFSU0oSulVJTQhK6UUlFCE7pSSkUJTehKKRUlxBhjdwyISD1wcJy7u4CGIIYTbE6OT2MbHyfHBs6OT2Mbn9nGmOzRNnJEQg+EiJQYY4rtjmMkTo5PYxsfJ8cGzo5PYwstbXJRSqkooQldKaWiRDQk9AfsDmAUTo5PYxsfJ8cGzo5PYwuhiG9DV0op5RENV+hKKaWIoIQuIheLyB4RKReRu4ZZnygij1vr3xOR/DDFlScifxeRnSKyQ0T+ZZhtzhGRVhHZaj2+FY7YfI5fKSLbrWOXDLNeROQ+69yVisjyMMU13+ecbBWRNhG5Y8g2YTt3IvKwiNSJSJnPsiwReVVE9llfM0fY93prm30icn2YYvuRiOy2PrNnRWTyCPue8PMPYXzfEZFqn8/ukhH2PeHvdohie9wnrkoR2TrCviE/d0FljHH8A4gFKoBCIAHYBiwYss2twP3W86uBx8MU2wxgufU8Ddg7TGznAH+18fxVAq4TrL8EeBEQ4DTgPZs+4yN46m1tOXfAWcByoMxn2X8Dd1nP7wJ+OMx+WcB+62um9TwzDLFdCMRZz384XGz+fP4hjO87wL/68bmf8Hc7FLENWX8v8C27zl0wH5Fyhb4KKDfG7DfG9AF/Ai4fss3lwCPW86eA80VEQh2YMabWGLPFet4O7AJyQn3cILsceNR4vAtMFpEZYY7hfKDCGDPeG8wCZox5E2gastj35+oR4Iphdr0IeNUY02SMaQZeBS4OdWzGmFeMMQPWy3eB3GAecyxGOHf+8Od3O2SxWTniKmBdMI9pl0hJ6DnAYZ/XVXw4aR7bxvohbwWmhCU6i9XMcwrw3jCrTxeRbSLyoogsDGdcgAFeEZHNInLzMOv9Ob+hdjUj/1LZee6mGWNqredHgGnDbOOE83cjnv+yhjPa5x9Kt1lNQg+P0Fxl97lbAxw1xuwbYb2d527MIiWhO56IpAJPA3cYY9qGrN6CpylhKfBz4M9hDm+1MWY58FHgyyJyVpiPf0IikgB8HHhymNV2n7tjjOd/cMeVhYnI3cAA8McRNrHr8/8VMAdYBtTiadpwmms48dW5o393hoqUhF4N5Pm8zrWWDbuNiMQBGUBjOIITkXg8yfyPxphnhq43xrQZYzqs5y8A8SLiCkds1jGrra91wLN4/s315c/5DaWPAluMMUeHrrD73AFHvc1P1te6Ybax7fyJyOeBy4DPWn9wPsSPzz8kjDFHjTGDxhg38OAIx7Xz3MUBnwAeH2kbu87deEVKQt8EFIlIgXU1dzXw/JBtnge81QVrgddH+gEPJqsN7iFglzHmf0bYZrq3PV9EVuE57+H6Y5MiImne53g60sqGbPY8cJ1V7XIa0OrTzBAOI14l2XnuLL4/V9cDzw2zzcvAhSKSaTUrXGgtCykRuRj4OvBxY0zXCNv48/mHKj7ffpgrRziuP7/boXIBsNsYUzXcSjvP3bjZ3Svr7wNPJcZePD3id1vL/hPPDzNAEp5/2cuBjUBhmOJajeff8FJgq/W4BLgFuMXa5jZgB54e/HeBM8J43gqt426zYvCeO9/4BPildW63A8VhjC8FT4LO8Flmy7nD80elFujH05Z7E55+mNeAfcDfgCxr22LgNz773mj97JUDN4QptnI87c/enztvlddM4IUTff5hiu/31s9TKZ4kPWNofNbrD/1uhzo2a/nvvD9nPtuG/dwF86F3iiqlVJSIlCYXpZRSo5TNNOsAAAAySURBVNCErpRSUUITulJKRQlN6EopFSU0oSulVJTQhK6UUlFCE7pSSkUJTehKKRUl/j/uGOcosdriMAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 1440x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "state = envs.reset()\n",
    "\n",
    "while frame_idx < max_frames:\n",
    "\n",
    "    log_probs = []\n",
    "    values    = []\n",
    "    rewards   = []\n",
    "    masks     = []\n",
    "    entropy = 0\n",
    "\n",
    "    for _ in range(num_steps):\n",
    "        state = torch.FloatTensor(state).to(device)\n",
    "        dist, value = model(state)\n",
    "\n",
    "        action = dist.sample()\n",
    "        next_state, reward, done, _ = envs.step(action.cpu().numpy())\n",
    "\n",
    "        log_prob = dist.log_prob(action)\n",
    "        entropy += dist.entropy().mean()\n",
    "        \n",
    "        log_probs.append(log_prob)\n",
    "        values.append(value)\n",
    "        rewards.append(torch.FloatTensor(reward).unsqueeze(1).to(device))\n",
    "        masks.append(torch.FloatTensor(1 - done).unsqueeze(1).to(device))\n",
    "        \n",
    "        state = next_state\n",
    "        frame_idx += 1\n",
    "        \n",
    "        if frame_idx % 1000 == 0:\n",
    "            test_rewards.append(np.mean([test_env() for _ in range(10)]))\n",
    "            plot(frame_idx, test_rewards)\n",
    "            \n",
    "    next_state = torch.FloatTensor(next_state).to(device)\n",
    "    _, next_value = model(next_state)\n",
    "    returns = compute_returns(next_value, rewards, masks)\n",
    "    \n",
    "    log_probs = torch.cat(log_probs)\n",
    "    returns   = torch.cat(returns).detach()\n",
    "    values    = torch.cat(values)\n",
    "\n",
    "    advantage = returns - values\n",
    "\n",
    "    actor_loss  = -(log_probs * advantage.detach()).mean()\n",
    "    critic_loss = advantage.pow(2).mean()\n",
    "\n",
    "    loss = actor_loss + 0.5 * critic_loss - 0.001 * entropy\n",
    "\n",
    "    optimizer.zero_grad()\n",
    "    loss.backward()\n",
    "    optimizer.step()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "200.0"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_env(True)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:pytorch4]",
   "language": "python",
   "name": "conda-env-pytorch4-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}