friend_foe_gridworld.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-27T16:26:31.229451Z",
     "start_time": "2018-12-27T16:26:31.008474Z"
    }
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from engine import AdvRw2, AdvRwGridworld\n",
    "from agent import ExpSmoother, FPLearningAgent, Level2QAgent, RandomAgent, IndQLearningAgent\n",
    "from agent import Level3QAgent, Level3QAgentMixExp, Level3QAgentMixDir, IndQLearningAgentSoftmax, Level2QAgentSoftmax\n",
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-27T16:26:31.342933Z",
     "start_time": "2018-12-27T16:26:31.329438Z"
    }
   },
   "outputs": [],
   "source": [
    "def moving_average(a, n=3) :\n",
    "    ret = np.cumsum(a, dtype=float)\n",
    "    ret[n:] = ret[n:] - ret[:-n]\n",
    "    return ret[n - 1:] / n\n",
    "\n",
    "def plot(r0ss, r1ss, dir=None):\n",
    "    # We set a fancy theme\n",
    "    plt.style.use('ggplot')\n",
    "    N_EXP = len(r0ss)\n",
    "    #plt.axis([0, max_steps*n_iter, -4.5, 1.5])\n",
    "    for i in range(N_EXP):\n",
    "        plt.plot(moving_average(r0ss[i], 100), 'b', alpha=0.05)\n",
    "        #plt.plot(moving_average(r1ss[i], 100), 'r', alpha=0.05)\n",
    "\n",
    "    plt.plot(moving_average(np.asarray(r0ss).mean(axis=0), 100), 'b', alpha=0.5)\n",
    "    #plt.plot(moving_average(np.asarray(r1ss).mean(axis=0), 100), 'r', alpha=0.5)\n",
    "\n",
    "    plt.xlabel('t');\n",
    "    plt.ylabel('R');\n",
    "\n",
    "    from matplotlib.lines import Line2D\n",
    "    cmap = plt.cm.coolwarm\n",
    "    custom_lines = [Line2D([0], [0], color='b'),\n",
    "                    Line2D([0], [0], color='r')]\n",
    "\n",
    "    #plt.legend(custom_lines,['Agent A']);\n",
    "    if dir is not None:\n",
    "        plt.savefig(dir + '.png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY0AAAENCAYAAADzFzkJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvqOYd8AAAIABJREFUeJztnXuUVNW957+nzqlnV9N0dwEK6lVUNEEBvRiNjqKxr5kYk7i4ifF5R03UCMIYNUuSe5OVtXK9mKyLMFEMZiRqdGYyrlzBMQ8z6SA6hpi0vFSiqIiRiNh0F4+qrnedPX/s+p2zz6lTVaebfpSp32ctFl3n+du//fjt/fvtvY8mhBBgGIZhGB8EJloAhmEY5qMDGw2GYRjGN2w0GIZhGN+w0WAYhmF8w0aDYRiG8Q0bDYZhGMY3bDQYhmEY37DRYBiGYXzDRoNhGIbxDRsNhmEYxjfGRAswFuzdu3dE9yUSCQwMDIyyNKMLyzg6sIyjA8t45DSLfNOnT/d1HY80GIZhGN+w0WAYhmF8w0aDYRiG8Q0bDYZhGMY3bDQYhmEY37DRYBiGYXzDRoNhGIbxzd/kOo3RoFAASiX5T9eBWEweL5UAIeT5QADQNKBYtO+jazMZ+bem2ec0DTBNeT19ZNcwnM/QNCAYtO8rFuU9RCgEpNNAW5t9XzYr36Xr9nOCQSkfpUGVgZ5PsmiafG6hYMvlhp4PyGvUNLsJh4F8Xv5tmvL9waD8X02LYdg6cuvF/V56RqEg7xPCmS4iGLTPuXUJyOOBgJSPZCQ9C9FYD0QgAMTjtv5I98EgUC4DuZwzrV7yAbbu6W/DkOcKBXk8lbKfWS7bzwiF5DvUY0LI37GYlE/VZ7FoP5vSGQrJckrPUPUVDMp7SEfuckqEQsDhw9X5VSzaf2uafI4QUi6SD5Bll55J15smEInI9Gmafa2qe8OQ9xWLtvyqDIYhrwsGZRpJRk0DolH5jnxeyk91TJWd7gekzsplO6/U6w3DvsY0q/NK1Ssdp99UDtrbZZ2m+9X0Fgp2/rqf60UoJHU3lrDR8KBcBgoFDbkcUCrJFicUMiEEkMtplUqtQdcFdF3+raJpJlKpAEIhYTUIagXI5+3rdV1UKrh9LBwWVqEaGnI+O50G0ukAgkET4TAwNATkcoFKgRbWs0MhAU2T8rsLGsmlpq9cFsjlnO9ypkneZ5pU2Gtfm0pJPem6/Y5IpPr5ug5Eo8Jq3Nx6BMgwC5TL9jN0XQCoThcABIPCIZ+aB4DUp2EIFAq2zk0TyGT86cH5LtN6RjAoUCxq0DSBQkFDOu39DMMQlcbdPi+EUBoomdahIQ2hEDA0JMtRqaRZRsgwBEolgUwmYBmFUEhUGhUNgYBZMbCUJiCbdeZBuSxgmgKplLNFJn0JIZBOa5Yedd0uTyrpNJDJ2M9Qy6Fs3GXaVJ0GAiZiMZnnhw/baZCNqUyrELIOSSMjkM3Kxt5+j9Tz0JBWZeDpHrru0CGnjKZpolgMVIyBrRNVdl0X1vsyGVv3XtdTeTNN+btYrJbJnYfhsLDqZiYj6zTlbSQiDZqsP5rn/bUx2WhMBJTh0aioZKpWMRjynBB2ryMcFgiHgbY2WbAPHQpYGRsKCcTjssLSM2UPVaCtTS18sjGORgWGhjTLaJimfA/9BoCODiCZVHuxsnK2tQkEg8LRg5E9F4F43K4A6bSGYFDKrOt2zyUcFjAMKQONKIh8Xjaw1EuNxUSlx1bdHaeeLhlUXddQLgvr+dQoZbOAEBoCAbuBicelrITsSWuVPLGfQYYEEI6GJJPRKselnFSJVaORyUij0d4O5PPyWtO0e3f19EAUCkAyGXA0DPQ3vTcYFGhvr363lE8DIPMrl9MQjUp9kuESQupiyhSgUBCVhlTqVAgN6bTda580yUSpJBu5YlHD0BBgmhpERaB4XBpIMhqGIfPINO0y1NVlVkawsmzYBk3eEw4DgK1XapABIJGQHSq1nFDngt5FcoTDAoODAYeu5DmzMrLRYJqi8r88F4sJhEIC2axthGQdcZ6n31RmqPGn+trRIRvTgYEAAK0y2pL1UI4WZL5Qo9/WJizvghypSt2rZZBGx4AsL/m8TKccqdgy2Xq1O0DSCMhnxOOwyr2uy+dT54+Oq2VgrI1CIzimURe7dyTdGVrVsFP9n1B7TpUjVefoOBVqr/u9cA/V3RXQfY37WW5Z/bzTnU6t1kPqPK9aJ/bvWu+upVev5zuv16y/q9Mvhv3cWjTu9dXD+0XqMzWt2r1p3S2qn+E+7843NQ+8yp37fikLXaRVXePG65z7Pe78Jjm98sv5PPmjVJL1MJerLYf7/c7nOeuc2plzXuf2HjivV3/71auaJ7VkrPcc9VkTCY80XLjdJJRJdKxY1KxYhqxY6rDbeW25rKFUEpVenVbx3QvHc2XB0BwVp1zWkMlQb9q7oMhelHC4tVSXR7msVXrQ1Y1kuawhnxcO2eneeoWS3B+mKRAIeDd6aixG7TGqstF15bLUGZ0LhaqfqVYcuo5GYG4DKs/Ja6R8GsplqSc1fkH6lXEHe3SivqOeHui9+bwGw4Dl6pIjN82Kc3gbSM0aJdB50pFsCAMIBJwjFJkGSyMwzUAl34X1Hjm6ILnkb5KN4ifu/CadqHoslzVks5qrcaeyonnq3E25bN9PsgeDwqG3bFaDYVRbXUqH031HLmOBQECrPEPqyR2zovfm8xR31Kx4DKHG1tzl0h7ZytEZGRmS3b5eli/prrXLEOnYq86pRoNGeySPep1pSleUu1wOp0MzlrDRcKFmJgU41R4bZRxVAiFQ8bHbgTSqNIDduAN2T4PcHhT0cz/DHs1oVY1jIADLFUGFPxIRVjBZPpsMheZ4n7xfVAqmFNAwhGJgvBtLOu6W3QuS110ZqaGn+7x6+25DRAFw0rm7Z18dILUNIemRAvHki6fge7ms+udtYerpQX1vICA7B2S4ZdqkoMWidOt4yWfHCJwdh0JBc5yz3yOsDgfdo+vyGAVNKU9V3VOHgTowNHmAriE/vFq+bP3ZaQqF7HJK19RCndSgNtSaJiydyXgMGYaAFayme+mfWicoeEx6oA6HaUo3j6pnKjPSeGuWASZZQiFZJyndVC5pkgW5Fum9VIdIf3S9YajthLB05dWhUfPQPu4sc/KczEe1/aCy6K4/EwkbDRcUD5C+YGnt43G7cpZKMn7ghWEAU6bYLRsFgQ1D3qP66wGaweI8RnGUbFazAsUqmiZlo5kz4bDA5MnV8sj7vI7LeEK5bAfdarlK1HQZhkBbW+NrNU3O7BoctA2a4VHKaHaOrkvduGMP6nv9Eg5LfdiISsNuH+nsFJ4ztLziM7UIBKSPnPz3NMNF14XVkHvFRNzyueWgSQGRCKzGUMZsnBcmEs68tX3yts9fljt7AoFa9vJ5ezag2uhGozJ2RL579R6vUaAbykuV9nbn7+5ugWJRxjYMQ1TSImlrE5XJH/bEAsDWJ8k5ZYppxX9CIWf5ojKTTsu4RDgsyy25szo6vOsFYBsLgiYzUJrc54NBqhP1cedhqSRHKdL4yPYll5Pns1l5btIkMawyOZ6w0fCAZv64K/1Ih4f1ZhqNFDUgP1KawT86HkPusXhHrZiJ+/xwnlErfjFS2eQMv+oR2nDjEqPNSN5RSzeNntUsLh03zRyzaAQHwutgD+ere6fDZTQLAw1X3S4g/0x8TRrPxkmN+3i/e+T6cMeU/E7XrfWMI9VLrQBto+ubBTWuR5AbbSSQbpspnaqbu7ZcE19Ha8FGw4WX60JlOIVPHTbX8wW7kXGL2i4BddGUl+unEbZvfPj3+oViPo3e4adhGynumAL5yOXftoxut5gfGqXLb3pUmci37kdv9eQi33mtoHWja3Td7eYbXaj8ul1X6nmvThrpheI69fRE6TqSzp5hCF9uueGixvXc8o9H3TxS2D1Vh3JZBs3yeer5yJZAHe63tcnZUXKmRW3CYRloo8Ak+fqFkPPzybesPsfrmeGw3bjQ3G33QrJYTFgLEElGmqqokstplRiEqHqG13F3fIIWxVFMQMZKtMqKcHmfujixrU3OYqJAqDwnz9NqajfutNBzvHROcRFaTU0rv+W8e7kYktITDtu9UFoDQOsPbNngeGcu55xyTTGrri4TmubUb6FgB1dLJc2Rdue75ELASERYfntVD2qZcaPGgdQFaDQSJb8/6YOIRqWvP5fTLB1SLE2d+Uezq7zSTsezWalTNU1uSiVUjcDicXttCEFrEGhhXTYrZaDJC7SuRb1Hrn2RsqgL/draBA4flumTOzTY91AcRD3mha5XLwytZYTomaWSvasA6dJOjz2CorVSMg1SD7Qeyx07IbzKpRt1TddYwUajDuqiL3JVuQtNsVi7Uqvkcs7VnLmchnhcWLOD3A16o2cB9qwSN+piQgCeBoOgwL/XcfeWESQzQQFgakwaucuo0QSq5a7Vs3Knxf0cFZkPwvpfnQVVLGqOabdu1Lzx2iLFa7sKVR51wRXNujFNe6EcGQ/1XfZaCNuAU+PoTJM3hYJmrUh2r+9QcTciXp2Rep2eWmkvFtXpvLVjbF4uO5qO7j5WKNCKamnEaEsPoPZKfUq/c3p27TTJmWqep6rkITeR+/lez6RtPoJB4cg3eo5aP4pFaSRoOjEg62gtgwE07pjSc+s9YzRgo+HCa/EeoU7HU4/54cgWgg2P4SwWBLwrtZ/71PN+0ieE8z00/bLWmotaMtSbWKA2yLXONZaz+lj9joGcZUfTe8kAqo2E+92jGaCt9yz3jL2RUCvtXuVsaMh71p9fSE/uOFQj6pV5L0PmtyzQqKbWKMrvM4eb3zRSkS6yxqMiYjxiNxzTcKEGEqkh8xphECP1PdZaD+GXUEhO6RzOPUeKW2b13fV68fb13kqsV9BVn3+td6u4DQr57aUfvX7N9VodXA+5vkBYrkVyo3n19Lyeqa75IR+9etwvw71+tBoW93NohNZoU70GT3X8ojLuXqDnR7Z6cbVGZcGNn5FJPWqtYK+Vd2SoSyXNGtH7YSxiMG54pOGCXFK0Bw/5CMm3SX5k1WdM1OrVeW1eR4VFrWDUU6nXOyRXg1x0JffkUdcEeL2LnqtuwgZU+8vpOeSDpnNyXx3nCtXq9GgVuZz7OhHptObY3C8ctnddreUPT6c1q7dOvVdKn1tX7nTLFdrSzxwOC2txFN2jyih3c9Wq/MWxmGx01Gfba1vsnnWxKFeB07qbYLDaiKprDtw9UvJtq9BCM1of5N5vKJuV7hB75bKdf/Yiz+qylE4790GLxarjWXSP24dO5YeOU/mw92HybtVpwSmVPYozkRyxmLB2EKjeXNO9sFBzjGKoDKv7vZH7ifbM6uqyd2KgvalI5lprrtQYkYrXuiPaR829NQmt25GuaVtXtP8U6V2tk+m0VjVKc3eERmP0eCQ0ldEwTRPLli1DV1cXli1bhv7+fqxatQqpVAozZ87EkiVLYIxkutAwoB6FXFnrvRWE+r+f2T/q7qLu3V6Hu4bDvW+PG7VR8tpZVoUCteq9tZ7bqJc3HLdCqSSNsd3ANa4EjXqvXo2x33Q0WmtBbjTA30yrej1/L/eZeyRLK8QzmdrPIn04t6Bwzr5ppFe7M2E/T+2punXlZdzU59TCHetwN66a1rgeyJ1nq4/Tc90yHOnaqEhEeLqEvMqPO/8oT2hjQsBuT4JB0TB+WS920QxTh5vKaPzqV7/CjBkzkM1mAQBPPPEEPvvZz+K8887Dj3/8Y2zYsAGXXHLJuMiiuqbquSsOHgxYu6mqPnuazeFGnf1RazaEV++QiEZl4du3zxaKGguvNMgZH3aDV6/xJVlCIVHly5VBSQ2HD2uIxYQVvKeV5XKbdhmfSKXsbw6QEbFnLdmzhgDbheA1Qkqn7ckCZPyoB0g6onOlknyn2hNOp7XKttrUg9Wsle0064fSBsjK6ta9un18PF7dK6eVx87gvPx/6lQTQ0OaQ680a8drVpqallxOq3xPQ4Ouy6AxuSSzWc0hez4vdSJn1snebSQiKqOBap0Sanpoinc2a9/j7oWrjSjd554BRagzhQBA0+Rux21tMg2plJRbbv+uIZORHQmawUWytbXZe6cdOqRZq8PpWxiFAr2fOmXVMbpUSr6DVpzLSR5yZKCWS7W8BoMCsZhAKkUzzGTZCQarG3W7fMHSpbqHFW3HHw7La93GqKurcTmj46rMXm1MrR0YRpOmiWkMDg5iy5YtuPjiiwHI7Qx27NiBc845BwBw4YUXoq+vb8zl8OrZ0bxyr1XitXpZ9Xr4ao/ebZAMQzie6X5+Ou1vFgVgb+neSFY3zmml9iZ1tNU0bRdPlEr2zCQyCvm85jnqoG3QCTmrqbZbQwgpAz3X3VMnPaub3KkNpbtCU8V0f5iKdKre6+4B19KfOjNKxT1SLRQ0z5lZbkPuLoN0vlCwp8hS3CSbVb+zYc+SqxVjUrdlUf/2mm3kJ04F1Noi3/ksShPtl0Z7Q5E+IhEyWE5j6o6TmKa9QaSX3t310z16VuVS4wzu8losalX6KJWqj6myudOujtrUOhKJOPWVzzvLid/4iVf+jGRx6XBpmpHGo48+imuvvdYaZaRSKcRiMeiVUtDV1YVkMjkusngFfL32gKrnhqjXow+FhGMKbr057n5nTfjB79C2XLZ73HL9Qe24jHODxdoTA0bih41GhfXtDjdDQ9WjAkD2cNXdXL3yyH2MvuXhltWPu4u+VwHULhPqs72Mhro5Xa33uFG/0gjYm1YStdwz6l5jNLqpRT0XD6WpvV024o1coQSte1LfQRMJ3PlJPWZaw0DUmyGnbngIOBtuGVdz76bgb3LESHB/uVPGUoRV36lut7fL/bbCYe8RBuBdf8ZieyI/NIXR2Lx5Mzo6OjBz5kzs2LFj2Pf39vait7cXAHDvvfcikUiMSA7DMJBIJJDL2UPGtrba7ilNs/288sM9/t5DX+uKROxPcqobn6nvdPuRAwED5XKX41h7e+13xeN27zkeH75PlFwAkYj9nGjU/iQqzRqiT7y2t0s9dnd3ez6vkbF109YGh29/0iS7hyg/9iT/lsN272fQwipAytveDghRW0bSZyrlzFP1s78qxaL37JjOzmoj4SUn5Qt95pR0aRgGurq6EI16r/xPpaqfITe0rB1/UKHNJY8Eyut6+gekPBQPUq+VHzKTf7s7IAQt1Gxrq75eLm61f8dizrhTKOQsj6WSXRbU8lNL5lzOmS6vutbRYT9TheqxKg/dT3kt32PLR59rVctTrfz3Yjjt0EhpCqOxc+dOvPzyy9i6dSsKhQKy2SweffRRZDIZlMtl6LqOZDKJrq4uz/t7enrQ09Nj/R4YGBiRHIlEAgMDg5WdUc3KitnqLa6JZNIeTnd0mNaW06qf0cvHSIFO2jdf18knTjEPUdnB1n4WBWO7u7uRyQxaC+3In+7Ga0ZMo6l7Xit31SCw7ce2vzEQCAjoup2WXE4gEklgcHDQ8W7A/nIdfX/A7cJQe1MkSyYjKgZKxifUVbm10kOzoUjWQsHpJ96/H5g8udshowrNqnLrkBo16inSDDp6n6YB06aZ1u/9++0ty90Lz6ixisXsfJGuF83qSLS3dyOfH0C5LP35bl1ls5riVrMnFuRyzpiUu/zS7KVaBpxmIKm/a00AaG9PWPXN/QlhIhqVaUylqnssMiZB+rXLhNxhwC53tOpbvd6tD0qbKrv84mPCM6/d36MhSA76X40d0GO8dqwmWQn61K1cv2PfT7EiWb41TJnSZcmXTmuW/qNRgY4O+oyv/S73zC41j/N5OWoZCdOnT/d1XVPENK6++mqsWbMGq1evxu23347TTjsNS5cuxezZs/HSSy8BADZu3Ij58+ePuSzD6QWTyyoctqdEunvy9XoI7sqlrl5WfaDyHd7P8DNy8LvQqlFvhho6Wh0rt46Xx9S0qM+ptzpVdR249zqqNWtNTYvqk1fvV2cNqYbJ/maJ/b/XHkvV+4857/Pazh6Aw6Unr4fn9bWot3Ou6gf32i4+EhGOWUrqNzlqQc9R84i2qFEZzrYUXsal3h5pteR0jwDcMjqvdTfgQvm7erRYLYN6vXfZUcvJSNdGqTMv1Xxy5zet9am1/5e7Po/nWi2gSUYatbjmmmuwatUq/OxnP8MJJ5yAT33qU+P2bj/GwzBkQ+H2uzfy39faQoMy3/6il1bznkaMdC63el86bX8YyDBkryeZDDhkoznvqpzRqPf7ZbooIbIn1ahBpZlBBPm+ab8jANa30clfrc5CUdem0LfJCcOw9/AC7BGKe4oojTBqLf4TQo76KC10nRqzUhnJtEkqayoyffb5fN7ey0x+sdH7Re4pr/Ste/ezRwLF/9SAse02q/bZqwFu98I+e3TsHVQn3IbA/Z0a0p2c3Ses59Eow/29i0KhemeIentCNaJWZyAeF9ZUXHc6aB1TredN5FqNpjMas2fPxuzZswEA06ZNw/Lly8ddhnqL2Eb3PX6Nkz2iAeQ9Y7kLqRe0tkIlFBIN98s5UvxszeBupNX1J+qiOL+NtTtPyAjVWoPi9sNTcNlPoNJtcFUottHoXiofqh7qfX1QNfhe7z1S3PqrtfuB39l8coGt3di7oWB6I4abzuHG3wCni43yp9HaJSovQsg65R4tNhtNZzQmGnWTsnqomwyOpNKp9zS6nwpPMCgDaamU0+c83lCj0+izqGMNVUa55sPpvojHq7+USNTSHVXybFaztvYA/H+budZCz5HqqJFbqF6P04/BGs39r/xC8vqdNi4XxAG1DMZwME3nzLpa1HMTNsLrS4vDQXXvNcNCPi+aIqbRXDg/uFS7t6bcIYBXXw1ixYp2bN1av6abJrBnj+749nW9dw0NaVixoh1/+lNI2atI/k+7ZL7/vo5cTj77ww8Dnj24rVulfCtWtFu9V3WhECCDmDt3Go5tEADZa6ctOdxp6ey0X5bJSFmffdZ+pjpbRR2RuHuGQgC//GUEK1a048UXQ0ilNOzerVvvkZ8wBXp7w46JBsUisGOHgRUr2vH667aAXr206hFE9TX2czVQQ6WO8H75ywjuu68da9ZUTzuqtWpcCKcfmnqTw/mU7fvvB7B3b/3q6p7/r7J7tw7TBPbuDfjeG8o0geeeC1v5kMloeOihNqxY0Y5HHonBNIHXXwfee8+pbLWcUBl94YUQ3njDPkGylkrA6tVxPPdc2Cq3qq4a7ZycTAas8jwwEMCf/hTybZCIffsC1vTXchn4+c+j+Otfa3fz335bd6TFi8OHNau+vfKKrLC5HPCLX0SsmVHDjUXk88Af/xhCb28YAwMBHDig4S9/0XH4sIYnnohhxYp2y308lmhCTER/Y2zZu3fviO5LJBLYuXMQgYD8bm8+71yRqqLOElmzJu64ZunSFDZvDmHevILlHkmlNGzfHsQf/ygj2oUCsGRJGslkAP/7f8cQCACf+lQOgQBw4ol2a3b//XFHcPKYY6L48pf7rfMrVnjPt73ssixOOaXkeZ2uC3zuc1msX2/PH7399hRWrbKv0TSBa67J4I03gnj5ZdkK3nlnCum0vRr5nXd0/OEPEXzmM/JddH8sFkM8nsLgYMCq9F/+cgblMjB9ehm//GUUu3bJSnfHHSkUCsADD1Sno1gErr56CO+8Y+BPfwo7DMEtt6Qt/QCyoS4Wgdtuk8e9euDFomxk2toEOjq6sWnTIUydauKYY8oIBqsXuA0NafjJT9oQDAKXXJJFoaDht7+NIBiUBv6664Yc62vU9TZy3yzgwQfjMAygpyeHv/u7Etrb7e8dHDqkYdOmMM4/P49USrr/yODs2aPjE5+YjF27kvj5z2MOuebNK+DsswvI5TQkErKl3bVLx/r1MYcOAODPfw7i//0/5ywK0wS+9jV5zQMPxBEKATfckMavfhXFhx9KJV999RD+5/9sPB83Foshk8kAAE4/vYAzzywiGBSVb3FLfv/7EF56yZbhtttS1sQOtVxOnmziK1+xp/68/34AwaBzDZM7X+n+a68dwhNP2PLeeac9H/l3v5uKbduyKBSASy/N4cQTS9YIZvv2IDZtksIcdVQZ+/bp1m7FS5ak8fLLQezYEUIup+Gkk4rYu1dHJmM3zDNmlPD++wba20188YtZvPeejjlzili50lmev/a1NNasiVu/P/3pLD72sRLeesvA3LmT8etfp/DGGwZuvHHImnK7cWMEH/tYEc89F8bcuUVs2ODagKwGatqHg9/ZU2w0FBKJBN54IwldNxsajXRas6YQ+qlcbgoF2fA99FDcaoQA2bB99rNZ7NplIJ0OYM8e3XGeKumdd6ZQLsPR0I81xxxTwmWX5axpjU880WbJ9bGPFfH660GHjH645JIc/u//9a4MxaI9F592OiWuu24Ia9dKvRsGrOnOX/7yEHbuDGL79hC+8IUMjjmmjI0bI9ixQ8pWKAALFuTxpz91olTKVOXtsceWcOmlObz/vo6nnpIRdFqrQqgjiVNOKeCTnywgHIZVVg4ckFNh333XwPPPh4f1ZcDzz8/hhBNKWLs2jo4Of3qkacyks09/OoeTTy5h8+aQZfDdLFiQQ3e3iZ/9LDaiLxcStfL6S1/K4J13DMybV8DatXHHufnzCzjnnDzC4epOzz/8Qw6//W0EV1yRwZNPxiCEbHCFAJ5/PoydO4NYsiSFUAh44YUw+vq8hT///Dw+8YkC9u0LYN26KchkMlYeXnhhHjt3GhgcrD2aKBSAT34yjz/+MTzsbTmmTStbxpc4/vgS3n3X+aDubmmkrr46jP/xP/JWPb/zzlTNzmAjLrkkh9NP91hB6gM2GiOAjIZhmIjFZMNYy2hs327gV7+KOrYW+ad/GsJPf+rPgKhrB1Sj4MUppxRx6qklPP101KqkN9yQxt69On7zG/8redSGvRYf/3gRf/5z/WvqFeorrxzC//k/U3wbDZVZs4q47LIcdu/WcfBgAL/5TcQKRrqNBjUApLvu7nLdRsB9XzQaQ7lcLWOhAJx9dgF//KPdGLkb1c9+Notf/tLW+9y5BRw4EMDhwwEsWJDD0097rAAcBv/4jxn8x3/EhmV8VfkB+uJhfTfN3LkFbN/e2GKceWYBW7bI6774xQz+7u/KVv4K3lgGAAAgAElEQVTHYjFoWhpDQ43dIgsXZvDUU7ZuPvOZLH7968bld9GiNN5800Bvb+Oe9qmnFvHGG7L8fuITeWzbFoJhtDmMxpEYyeHwhS9kMGNGGQ8+WN8AuPP5qquG8L/+l3c70tZm4uabh/D66wZMU0M0KteHHXts+Yi/2MdGYwQkEgm8/noSwWBjo/H977dXPsokG62vfS2NtjaBX/86UrfRXbgwg+nTy1i5st0x5fGOO1LYvz+Axx+vLixf+EIWJ50kXU0/+tE0ZDIZR+/yP/2nPM4+2+4KHzyoVfXuFizIYf78InI5YPVqWYh7enI49dSiwzV0553SXXT//fax225LebqPvLjzzlTFzZfEiy+GcNllOezbp+NnP3M2pD09uapGwD2sfuSRGPbtk4YgGATuuiuFjRvD2Lw5ZC14CoWk7otFVKXZC3sFfgwXXZTE88+HHXtf0Xmis9PEaacVLbciyblhQxhbt/prfa68MoPXXgvitdeqy4V7IZ0KNSbHHVfCl74kA0O0ePSZZ6LYu9dpJG+6KY3Vq20dUONIDT3hNvif+1wW+/bpyOeBo48uOzoip59ewCWX1J7CJRfEDuDNNw3s2mXULfu1OhvTppUxdWoZr746stb8uuuGsGePjlmzSohEBH74Q+c7YrEYvvrVD9HbG8arr4aq4l133pmCEMBvfxvGmWcWkU5r+I//sMvrP/3TEDo7TRw+HLBieOWyjP8ZBvCLX3gbvq9/PYVAoLYLWZWvXufgM5/JIhSSnYATTyyN2edc2WiMADIaoZBZ2SqjsdEIhYDLL8/gxBPtSvnOOzqyWQ2zZ8uGPp2WO51Om2YHjX/wg3YrIHnFFfb9mzaF8Ic/yAbq1FOLOO20oqPCh8MJ/Nu/OSvxHXekjmimhRDSpdLV5SwK9A0KTQM++CDg6Ya7+eY0fvxj2VDdeGManZ3CakgaoVamc87J47zznEu8ly+3z59wQglXXikbzsHBAB59tA3z5hVw8cW2LhpVzrPOKqC93cSGDRGcc04E5523H4BsAN5+28CMGWU89FDc6pHedFMaiYQM5P7oR3FMnVrGP/5j1tL1wEAAjz1We2Q5dWoZc+YUMXeu7S4wTWDDhjC2bw9B0wTuuEPGtbJZ4M03g1aPHgBuvjkMYBDRqPfOpeSymTKlbJW1tWtj6O/XMXmyiUwmgCuvHMKMGc6ZET/8YdwxDdTLB/722zqy2UBDV4c7r00TePNNAzt2BB3uGIpjeLmUrr9+CB0dJv7bf3Pm35w5BbzyivPaU04pYudOu9W86aY0Jk1yltv33w/gZz+z8yUWi+HWWz8EUF1Gbrkl7Rn/+stfdPz85zFMmmTippsaL7GmSSV/+EMIr74acjz30Udj1ij4K19JIxwGHnmkzYpTkdG47rohR6fxuuuGMHXq+H3yk43GCPAyGrWmND7xRBR79hiYPbuIz3++xme56rBrl44nn5S9mWXL/Df63d0JfOtbdkN51FFlXHPN8F1BI6FYhKMXd8UVGRx7bPVUHL9GY2hIw5o1cXz840V8+tO5qtkkr79uYP162Yv7r/813fCTmw8/3IZDh+RDaPTx1ltBnHVW9X4jtWRcvz5qzcK6885UQ1dGLUM1EkN++LCG//7f7ZHC8uXhYW+JoxraUMjbIGzeHMTGjXKUN3t2Ef/5Pw+//BL18poM4Zw5xSpdvPBCCH19snNEMgohR8mdnXY+u/V7550pvPeejt7eCKZNK+Ozn/WWPZkM4JFHZAN8++1h6LqU8cknZb0FvA3OWFAqSeM+b14R3d22EaCAe6mUQCwm5aP0UkxmPGGjMQISiQT+/OckIhGzskFfbaPxyCMxRCICV13lsVOZD9Sv5g1ndWcikcCePYN48EHZuLhHOWPN1q1B7NxpYP9+HUuWeO9Q59do+IGm27a3+9PR1q1BHHdc2VE5vagn46FDMpA9bZrZsOFPpTSsXx9Ff7+OxYtTKBQ0RCK192qqhxDAfffJRmPx4hSOOWb4erz3Xqfb08to9PfbblByoYyUI8nrDz8MYPJks+7mih9+GLBmRd16a+OOg0o2K0fJbj0ePizLU7Osg1B16OejbmOFX6PBi/tq0MiU5vOaY1rhcKFtL0aysludxz5z5vgZDAA444wizjhjZLMzRoLclda/jkZDNl33b8jb2wWuu84e6dVbK9EITZNTR0sl50r24XD77SmsXNmO6dPLWLjQu0MzdaqJm25KT3jDqbpr611z220pmObwd2+tdf14jC5GSrMYsnqw0XDhd9yVz1dvqzEcaNfOkbJkScrx3Wtm9BjOFhejjZ+GtB6RCPDNbzaep9/MDacbP9u8M+MHG40RILcQGf/9n1TGa9pgKxKJCM/vwzMMw0bDk0aNRaEg97CZSKPBjB3yOyETLQXDNCdcNVz42XdK/cAPwzBMK8FGQ2E48Qxg5MFKhmGYjypsNDxotI8+TZXlkQbDMK0GGw0PGsU0Dh6kkQYbDYZhWgs2GgruL8E1uo5HGgzDtBpsNBTc34BuFAgfzupUhmGYvwXYaIwAO6YxwYIwDMOMM2w0FPzu+1IoyG9g8Fx+hmFaDW72alAvrpHPczyDYZjWhI2GgjrSoA8ceZHPaxWjwYaDYZjWgo1GDeSX8bzPFQq8hQjDMK0JGw0Fd0yjVsxCjjTGRyaGYZhmgo1GTWq7pwoFjmkwDNOasNFQ8Lv3VC7H7imGYVoTNhoe+JlyyyMNhmFaETYaHtQzGqYJFIsypsEf6WEYptVgo6FQ7Z6qHk3k8/J/HmkwDNOKsNFQ8BPT4G3RGYZpZZric68DAwNYvXo1Dh48CE3T0NPTg0svvRTpdBorV67E/v37MWXKFHz9619HPB4fc3nquZ143ymGYVqZpjAauq7juuuuw8yZM5HNZrFs2TLMmTMHGzduxOmnn47LL78c69evx/r163HttdeOmRzOFeHexoM/9cowTCvTFO6pzs5OzJw5EwAQjUYxY8YMJJNJ9PX1YcGCBQCABQsWoK+vb1zlSqc1JJNOFQ0N2UaDA+EMw7QaTTHSUOnv78fu3btx0kkn4dChQ+js7AQATJ48GYcOHRrTd7tXhK9dG0cwCJx1VgEXXCAj4NksjzQYhmldmspo5HI5rFixAtdffz1isZjjnKZp0Gp07Xt7e9Hb2wsAuPfee5FIJEb0/nzeQFdXFxIJIJsFYjEdwSCwY0cMn/tcGcEg0NamIRYL4LjjwojHgXB4RK8aMYZhjDh94wXLODqwjKNDs8vY7PK5aRqjUSqVsGLFCpx//vk4++yzAQAdHR04cOAAOjs7ceDAAUyaNMnz3p6eHvT09Fi/BwYGRiRDJJJAMplEMGgim9UQj0dx4IAOAHj22Tw++ckCPvgghFwuhMOHh5DLiXE3GolEYsTpGy9YxtGBZRwdml3GZpFv+vTpvq5ripiGEAJr1qzBjBkzcNlll1nH58+fj+effx4A8Pzzz+Oss84aN5nyeWD/ft36vWmTtA779+uIRjmewTBMa9IUI42dO3fihRdewHHHHYdvfOMbAICrrroKl19+OVauXIkNGzZYU27HEnWdxoEDTnsaDgts2xbEu+82hcoYhmEmhKZoAU899VQ8+eSTnue+853vjLM0MhA+OCiNxlVXDeG3v40AAH73u0jVdQzDMK1EU7inmgV1bUa5LP9obxc4+ugyMhkNHR0mAOC//JehiRKRYRhmQmmKkUYzUijI/8NhgUhEIJOR9nXq1DISCRPpNA8zGIZpPXikoaDGNPJ5DYGAQDAIdHbaJyZNMn1/d4NhGOZvDR5pKKjuqXxeQzgsf592WhEA8PLLQfT05CdQQoZhmImFjYaCEPZoQ/3QkqYBp59exOmnF63rGIZhWhF2TyloGhCoaCSfR8NPuvLsKYZhWg02Ggqm6XZPeRsNGmnwiINhmFaDjUYN/HwHnEcaDMO0Gmw0XGiaNBSFgjbu+0oxDMM0O2w0FJyzp4BIhEcaDMMwKmw0FMhoCAEUi43dUwzDMK0GGw0FCmyn0xqEAPbt0+texzAM02qw0VAw5dZSKMrlGJg2rTxxwjAMwzQhvLjPRakECCGDFVOmmBMsDcMwTHPBI40K6mrwUkn+r+scCGcYhlFho+FBqSStQTA4wYIwDMM0GWw0XITDwoppBIMc8WYYhlFho1GBXFOaZo80jBoRH549xTBMq8JGQ4HWadBIg9dpMAzDOGGj4cHQkFQLrwhnGIZxwkajgupy2rlT+qVCoQkShmEYpklho+EiEAD6++VKcB5JMAzDOGGjoSCEvSq80XUMwzCtCBuNCmQIfve7iHWMRxoMwzBO2Gi4eP11uaLvlFNKEywJwzBM88FGwwWNLv7hH3K+r2UYhmkVeMPCCroOtLXZ39Ngg8AwDFMNjzRGAAfCGYZpVXik4SIaNXHyyRzPYBiG8YJHGi6KRQ3hcP1reKTBMEyr8pEYaWzbtg2PPPIITNPExRdfjMsvv3zU3yEEUCjIzQpp+5BacQ06znEPhmFajaYfaZimibVr1+Jb3/oWVq5cid///vf461//OibvSqfl//F4/RV+PNJgGKZVaXqj8fbbb+Ooo47CtGnTYBgGzj33XPT19Y3JuwYG5P+8uy3DMIw3TW80kskkuru7rd/d3d1IJpOj/p5yGXj66QDKZSCTCSCft7dIZxiGYSQfiZhGI3p7e9Hb2wsAuPfee5FIJIb9jHweOPHEAHbujOHcc8M4dAiIxYDJk72vLRSA9vYjlXz4GIYxovSNJyzj6MAyjg7NLmOzy+fmiIzGe++9h5///Oe44447RkueKrq6ujA4OGj9HhwcRFdXl+Oanp4e9PT0WL8HyM80DPJ5YOvWaSiVMkgmUzh0KIBsViCfF9B1GcfI5+W1QgDZrIaDBwUCAVizrYpFea7Wluq5nH1efaZ6zE2xCJRK9vlEIjGi9JVK8lm6Xn/L93xefhs9ELB/m2bj+1R5p05NoL9/AKWSnCwQiTS+j2QslaQ+NU2+OxSSf5umrX9Ns68BbD0Gg946pPOhkEyXEEB7ewIHDw44vs5YKMhn1vo2fLksr3HjVzelykxu9xchScdqWQKceW2a8t2RiNSxaaJqlp96jRcku98t/9XyCcj3lUryPfSOkZbHYlGmNxCQ71DzE5B1JRCwZaV6YBh2/hSLMk/c97ohGSk9ul47jykvwmH57JJr9r277FGZCAad+UrHVXnV8m2aUv5IpFqHVFfd73KTy8nnG4ZdP9T3DZfp06f7uq6h0cjn81i3bh3effddHH300fjSl76EVCqFn/70p3jllVewYMGCkUnokxNPPBEffPAB+vv70dXVhU2bNmHp0qWj/p5CQcOkSQIHDwL5vMwlTZMZH43KDCqXtcq1cmpuKCQLh66LSoGU59WYSLEojwcC9g66xaJ9PVXIbBaIxaQRKhbl84WwK6yUw34uFUjTlNfQfVSZ3Q2lrGAaymUgEBBW2sJh20CUy/K95bKURRZszToXCAjkchp0XTgqidpo5/MaMhlbF1JW0bChEgLI5bTK3wKGId9dLEpZcjkNpmmv2BdCAJDHDEOgVNJQKgHxuK0jqlSy8mtWA0CGrFDQYBjCen+hIN9P34anBobSSToMKE5d07QbF9ME2tqEo5JTHqjpi8dF5TcghGaVi3JZpsVt+EwTyGTkvaWSsHRrmrKsUQdEvSYQsPM3m7Wvl/+LqsZINRDlspxBSHojMhn7+qEh+b98vqwjhYLUoa6LSno0hMMyL6n+0MxESoOuC5TLMm+CQTvt9F6qS4WCVpFNzR+7bEajwsqfSEQ4Oh1qXlA5KBRkWtvaZNkSQn50jcq7mlbKbzWfgkFZB0i3gJSJZCCkDmX5p/wPBoV1Xybj7IiUy/Z1AKBpwtJFICAghAZNE1a5KZVkGijvSfdjSUOjsXbtWuzevRtz587Ftm3b8N5772Hv3r1YsGABbrnlFkyaNGlMBdR1HTfeeCPuuecemKaJiy66CMcee+yovycaFYjFgI6OsuN4uawhnxdWQdB1AU3ToOtAOCwrsJrJgKy87spBvXXZyGhWL1zX7YJBjQ81XgAsYyMbdFnAZI9S8+z10gwwgipjuaxZDS41nuWybOB1XSAatY+bJvVcnBWb0knGR9dlQTYMZ6NMchuGQKGgoViUjTMV7FBIVrhczjYCpqlWNPl8VZ+AvLatTSCd1hwVkyo6AMvQ2g1odUUiQ0yVjtLifh/1BskQlcsy31XjLXuUtryZjGw0aUQjjb9wyJvLyffRsUBA5lE+r1UaM9kgdHXJa9WGW9VxuSwbLdlpceogm9Ucea1SKmlWR4dGVrInbz9DdnRkOtvaRKUHbutFnUFYLmswTWGVWzWt+bxmjRBl2t3dZmc+FouwDDnpk9JEFApwGG41v9V7ZFqd6Qbg6MANDTllpfNEKCQcnaNcTrPynPQDSN3kXNvV2Z0xDcWi2uFz5hUZa+qMAnbbopZtW69OHVIa/I54j5SGRmP79u34wQ9+gI6ODnzmM5/BokWL8N3vfhcf+9jHxl66CmeeeSbOPPPMMX9PsQh0d5uWQaDGljIuEpEFSNNEpbche210PhgUME2tEkx39khpFEG9MUAWKmqA0mln7xyQlScSQSUor1kVUK0U/tJlN9bFogbTtHu3ABVG4epV2n9HIkAu52z45H22ASR0XaC9HcjnqaEVFeOkjjy8DZ7s1QvFWMFhPMgQRyKygaI0BAKy8SOj654STaODaFT2KsloAVAMofxNrjCn/uRow2uqNRmRUglWj1XVI+D9m3QWCMiyAch8VnU8NGTfa/fInY0KpU+FeuUqwaAcXQghGxl1FKnrwtGQAfaIQ3ZqaATrHOk6jYN9r7tnXg/VYFMe2Gm2jzkNq1o2hWPE7kWpZMtHdYo8AG5ZADs/3GiazGu1PgJSLrVx13VYox0ASKc1Rxmg91A5BpzGi9xnQsg6SSNswC6Dui7z01lPx2fWZ0Ojkcvl0NHRAUDOXIpEIuNqMMYL2TuTBSYclqOJUEgeA2ANs93IYb6wfKuA7AkDdqVRXSahkN0w0jAbsF0sBBkoeofXTC5Z2YTVU9R1WIWMXEjUKMpKZ/esvdIvhD0aosIZjYpKZZHvAmjEY1dU+p8Mowq5h/xABrRUkq6ncFhYbiNyw9Az1d4oYLtw1MqrEgwKq9IDUqdqo0R5FY2KKqOs9pZrDf8NQ+Yz5b0XslzZeUKNCxGJCI+eOMklDTH19qnzol4fDMoyRPmhaRQXsHVHjY+qJzW98bhwjG7ceiZkj1aOzgcHbQOilnXThOVWJKNFI5ZQyG4wDYNcPfZzZKcNlj6pIRdCONyIhiFHu/m8LC9qeih/s1koHTtY/wcCwhoxyPzXaqZXxd2Ah8NSx+S6rdXbp/pKIwXqnLqfTcdCocbT/6NRYbkEx2uxcUOjUS6X8dprrzmOuX+fdtppoyvVBCCHiJqjEkv3kezh1QsuDXdIqI4wiEjEdi1Fo9V+7WhUKP5Tu8dk/yZZnM8Nh2VDQsdlpZCjIfJbF4ua1dMxDNnwZ7Oa5T5zEwzKCqs2kBTUdCNdV1KHoZCouIScvSpNcxob1UBR2htBow1q/O1eodPgEHSejA250wIBWO4l8terPepGQUZ3w6+6QqjXTi45d7oMgxoBONxNqqFT9WIYtk+eGl5yCZIsw2lIqBNDI5JAQDRMr65XjwjVtLt77WoavOpALifLNckdDArHKCIUqg7ou+uTarjUOuMuB25Xo6rbehiGPWmCjIzaIXFDnR+7nbCv03U5C5Pm+rS1DW+0UJ2Gsaeh0ejo6MCPfvQj63c8Hnf81jQNDzzwwNhIN45kMgFoWnUhdzdgfiB3hRDDm8kQClGjVX1O12VlUF0/fqAGvtZvCpITsgeLuj0mghosTXMaMBX3KEWOYoQVxG+0z9dwMAynq9DLOLsJBJyNDEDySiIRO+gbCjVuhKkh95oRRNRrGGQjAAD+8trdWPlpRNzuFILyol4D6IU0lCOftUPYZaVaJhUaNfoxiGo6Rsvf71Vm6uFHL9SB+yjQ0GisXr16POSYcKhnHYuJuv5Rol6Bpd7XcBmPIJYb9+iA0uWuvF4Mt3Gh9/l59kiQoxr592jpUnUX+G0UhzPNeCIIh+00efn2h4tXYz+WDLejMdyO1kRAU2c/CnxExBx7ZIzB2QtslQ0J6/nSP0rID2mNfuNwpD3oZkZ2cBq7oRiGYKNR4aijyrj8chOxWOMpH39rGxYahmw4Pio9HWb0GMuRH/O3SdPvPTVeTJokcNppwlqI1WpEo3/bPWqGYUYHNhoetKLRYBiG8QMbDYVWiWEwDMOMFDYaHhzp7CmGYZi/VdhoKLAhYBiGqQ8bjTqwEWEYhnHCRsODxu4ptiYMw7QmbDQU3BuRMQzDME7YaHjgZ0tnhmGYVoSNhgKNNDiWwTAM4w0bjRHDPiyGYVoPNhoe0GcwGYZhGCdsNEYAB8oZhmlV2Gi44BEGwzBMbdhoeMAjCYZhGG/YaDAMwzC+YaPhwq97it1YDMO0Imw0GIZhGN+w0fCAp9wyDMN4w0ZjBHCgnGGYVoWNhgseYTAMw9SGjYYHpsmWg2EYxgs2GgzDMIxv2GhU4S9gwW4shmFaETYaDMMwjG+MiRbg8ccfx+bNm2EYBqZNm4ZFixahra0NALBu3Tps2LABgUAAN9xwA+bNmzcuMjWacsuzpxiGaVUmfKQxZ84crFixAv/+7/+Oo48+GuvWrQMA/PWvf8WmTZtw33334Z//+Z+xdu1amPxJPYZhmAllwo3G3Llzoes6AGDWrFlIJpMAgL6+Ppx77rkIBoOYOnUqjjrqKLz99ttjLo8cYXDAgmEYxosJNxoqGzZssFxQyWQS3d3d1rmuri7LoIw15fK4vIZhGOYjx7jENL73ve/h4MGDVcevvPJKnHXWWQCAp556Crqu4/zzzx/283t7e9Hb2wsAuPfee5FIJEYkp2EYSCQSCIWAeByIxbyvC4flv1BoRK85IkjGZoZlHB1YxtGh2WVsdvncjIvR+Pa3v133/MaNG7F582Z85zvfgVaJQHd1dWFwcNC6JplMoqury/P+np4e9PT0WL8HBgZGJGcikcDg4AAOHw4gnzeRyXhfl05rCIXEhBiNRCIx4vSNFyzj6MAyjg7NLmOzyDd9+nRf1024e2rbtm14+umncffddyMcDlvH58+fj02bNqFYLKK/vx8ffPABTjrppAmUlGEYhpnwKbdr165FqVTC9773PQDAySefjJtvvhnHHnssPvnJT+KOO+5AIBDAV77yFQQC42Pj6k255em2DMO0MhNuNO6///6a5xYuXIiFCxeOozS80pthGKYeE+6eahaEsGdN8WiCYRjGGzYaFcplIJMBaP0gGw6GYZhq2Gi4IGPRyE3FbiyGYVoRNhou2BgwDMPUho1GBdVYsGuKYRjGGzYaLshg1DIcbFAYhmll2GhUoJEGu6cYhmFqw0bDhd9AOMMwTCvCRsNFPi+tRSM3FBsVhmFaETYaDMMwjG/YaFRwxzR4JMEwDFMNG40KbiPBs6cYhmGqYaPBMAzD+IaNhoKuA+O0+zrDMMxHEm4iFTiOwTAMUx82Gi78xizYwDAM04qw0VBgQ8AwDFMfNhouGhkOnj3FMEwrw0ZDQTUIbBwYhmGqYaOhUCpNtAQMwzDNDRsNF7winGEYpjZsNBTCYf/XslFhGKYVYaNRA45pMAzDVMNGQ8HP6IGNCcMwrQwbDQ/Y9cQwDOMNGw0FNhYMwzD1YaNRA/5yH8MwTDVsNDxgg8AwDOMNGw0F1VjU2iKdA+EMw7QybDQU2CAwDMPUh40GwzAM45umMRrPPPMMrrjiChw+fBgAIITAT37yEyxZsgR33XUX3nnnnTGXQdfl/5rWeMjBcQ+GYVqRpjAaAwMDeOWVV5BIJKxjW7duxb59+/DDH/4QN998Mx5++OExl0PTAMMQFeOhoVzmTQwZhmFUmsJoPPbYY7jmmmugKd33l19+GRdccAE0TcOsWbMwNDSEAwcOjLkskQgQjcq/s1kNuZxzSMFxD4ZhWhljogXo6+tDV1cXjj/+eMfxZDLpGHl0d3cjmUyis7Oz6hm9vb3o7e0FANx7772O+4aDYRhIJBLIZqVxKJfl8XAYCIXk3/k8UCgA7e0jesURQzI2Myzj6MAyjg7NLmOzy+dmXIzG9773PRw8eLDq+JVXXol169bhX/7lX47o+T09Pejp6bF+DwwMjOg5iUQCAwMDFaOhwTTtc/G4HGLk80CxqCGfn5ghB8nYzLCMowPLODo0u4zNIt/06dN9XTcuRuPb3/625/H33nsP/f39+MY3vgEAGBwcxN13343ly5ejq6vLocjBwUF0dXWNh7gWgQAchoPgIDjDMK3KhLqnjjvuOEeAe/HixVi+fDkmTZqE+fPn49lnn8V5552Ht956C7FYzNM1NV6YZu0FfwzDMK3ChMc0anHGGWdgy5YtWLp0KUKhEBYtWjRu79Y0Gc+gKbiAbTQ4EM4wTCvTVEZj9erV1t+apuGrX/3qhMghhDQcZChMUxoRo6m0xTAMM/6ww8UDGlGosQseYTAMw7DRqIsQQCAgqgLfHAhnGKZVYaPhARkFGm2wkWAYhpGw0WgAuaVMU3P8ZhiGaUXYaDRACK0qvsEwDNOq8HwgD9QFfYEADS1opMHWg2GY1oVHGh6oowrac4rcUprGLiqGYVoXNhoN8HJL2aMPhmGY1oKNhgc0ujAMaRx4vQbDMIyEYxoeaJq9qy1hz6LileEMw7QuPNIYATzaYBimVWGj4QMKfufz8re6kSHDMEwrwUbDB+SOKhZlcIPXbDAM06qw0fCBrttBcYC/q8EwTOvCzd8IYPcUwzCtChsNn5ChYIPBMEwrw5NHfWIYQLkseLotwzAtDTeBPtE0IBKZaCkYhmEmFnZPMQzDML5ho8EwDMP4ho0GwzAM4xs2GoDzTwgAAAnMSURBVAzDMIxv2GgwDMMwvmGjwTAMw/iGjQbDMAzjGzYaDMMwjG80IfjrEAzDMIw/eKShsGzZsokWoSEs4+jAMo4OLOOR0+zyuWGjwTAMw/iGjQbDMAzjG/273/3udydaiGZi5syZEy1CQ1jG0YFlHB1YxiOn2eVT4UA4wzAM4xt2TzEMwzC+4e9pVNi2bRseeeQRmKaJiy++GJdffvm4vHdgYACrV6/GwYMHoWkaenp6cOmllyKdTmPlypXYv38/pkyZgq9//euIx+MQQuCRRx7B1q1bEQ6HsWjRImtou3HjRjz11FMAgIULF+LCCy8cVVlN08SyZcvQ1dWFZcuWob+/H6tWrUIqlcLMmTOxZMkSGIaBYrGIBx54AO+88w7a29tx++23Y+rUqQCAdevWYcOGDQgEArjhhhswb968UZNvaGgIa9aswZ49e6BpGm699VZMnz69qfT4i1/8Ahs2bICmaTj22GOxaNEiHDx4cEL1+OCDD2LLli3o6OjAihUrAGBUy98777yD1atXo1Ao4IwzzsANN9wATdOOWMbHH38cmzdvhmEYmDZtGhYtWoS2tra6+qlVz2uV5SOVkXjmmWfw+OOP4+GHH8akSZMmTI+jgmBEuVwWt912m9i3b58oFovirrvuEnv27BmXdyeTSbFr1y4hhBCZTEYsXbpU7NmzRzz++ONi3bp1Qggh1q1bJx5//HEhhBCbN28W99xzjzBNU+zcuVN885vfFEIIkUqlxOLFi0UqlXL8PZo888wzYtWqVWL58uVCCCFWrFghXnzxRSGEEA899JD4zW9+I4QQ4tlnnxUPPfSQEEKIF198Udx3331CCCH27Nkj7rrrLlEoFMSHH34obrvtNlEul0dNvvvvv1/09vYKIYQoFosinU43lR4HBwfFokWLRD6fF0JI/T333HMTrscdO3aIXbt2iTvuuMM6Npp6W7Zsmdi5c6cwTVPcc889YsuWLaMi47Zt20SpVLLkJRlr6adePa+VB0cqoxBC7N+/X/zrv/6ruPXWW8WhQ4eEEBOnx9GA3VMA3n77bRx11FGYNm0aDMPAueeei76+vnF5d2dnp9XDiEajmDFjBpLJJPr6+rBgwQIAwIIFCyx5Xn75ZVxwwQXQNA2zZs3C0NAQDhw4gG3btmHOnDmIx+OIx+OYM2cOtm3bNmpyDg4OYsuWLbj44osBAEII7NixA+eccw4A4MILL3TISL2jc845B6+99hqEEOjr68O5556LYDCIqVOn4qijjsLbb789KvJlMhm8/vrr+NSnPgUAMAwDbW1tTadH0zRRKBRQLpdRKBQwefLkCdfjxz/+ccTjccex0dLbgQMHkM1mMWvWLGiahgsuuGBEdctLxrlz50LXdQDArFmzkEwmLdm99FOrntcry0cqIwA89thjuOaaaxyjgonS42jA7ikAyWQS3d3d1u/u7m689dZb4y5Hf38/du/ejZNOOgmHDh1CZ2cnAGDy5Mk4dOiQJWsikXDImkwmq9LQ1dVlVaLR4NFHH8W1116LbDYLAEilUojFYlalVd+nyqLrOmKxGFKpFJLJJE4++eQxkbG/vx+TJk3Cgw8+iL/85S+YOXMmrr/++qbSY1dXFz73uc/h1ltvRSgUwty5czFz5sym0iMxWnrzqlujLSsAbNiwAeeee64lYy39eNXzemX5SOnr60NXVxeOP/54x/Fm1aMfeKTRJORyOaxYsQLXX389YrGY45ymaRPju6ywefNmdHR0NPW0wHK5jN27d+OSSy7BD37wA4TDYaxfv95xzUTrMZ1Oo6+vD6tXr8ZDDz2EXC43qqOYsWKi9daIp556Crqu4/zzz59oURzk83msW7cOX/7ylydalFGFjQakNR8cHLR+Dw4Ooqura9zeXyqVsGLFCpx//vk4++yzAQAdHR04cOAAAODAgQOYNGmSJevAwECVrO40JJPJUUvDzp078fLLL2Px4sVYtWoVXnvtNTz66KPIZDIol8tV71NlKZfLyGQyaG9vH1MZu7u70d3dbfUwzznnHOzevbup9Pjqq69i6tSpmDRpEgzDwNlnn42dO3c2lR6J0dLbWNetjRs3YvPmzVi6dKll2IYrS3t7e808OBI+/PBD9Pf34xvf+AYWL16MwcFB3H333Th48GDT6XE4sNEAcOKJJ+KDDz5Af38/SqUSNm3ahPnz54/Lu4UQWLNmDWbMmIHLLrvMOj5//nw8//zzAIDnn38eZ511lnX8hRdegBACb775JmKxGDo7OzFv3jxs374d6XQa6XQa27dvH7WZSVdffTXWrFmD1atX4/bbb8dpp52GpUuXYvbs2XjppZcAyMpLOvv7v/97bNy4EQDw0ksvYfbs2dA0DfPnz8emTZtQLBbR39+PDz74ACeddNKoyDh58mR0d3dj7969AGQDfcwxxzSVHhOJBN566y3k83kIISwZm0mPxGjprbOzE9FoFG+++SaEEHjhhRdGrW5t27YNTz/9NO6++26Ew2GH7F76qVXPNU2rmQdHwnHHHYeHH34Yq1evxurVq9Hd3Y3vf//7mDx5clPpcbjw4r4KW7ZswWOPPQbTNHHRRRdh4cKF4/LeN954A9/5zndw3HHHWT2lq666CieffDJWrlyJgYGBqimPa9euxfbt2xEKhbBo0SKceOKJAKRfd926dQDkVL2LLrpo1OXdsWMHnnnmGSxbtgwffvghVq1ahXQ6jRNOOAFLlixBMBhEoVDAAw88gN27dyMej+P222/HtGnTAEhXwnPPPYdAIIDrr78eZ5xxxqjJ9u6772LNmjUolUqYOnUqFi1aBCFEU+nxySefxKZNm6DrOo4//nh87WtfQzKZnFA9rlq1Cn/+85+RSqXQ0dGBK664Amedddao6W3Xrl148MEHUSgUMG/ePNx4443Ddnd5ybhu3TqUSiUr+HzyySfj5ptvrqufWvW8Vlk+UhlpYgYALF68GMuXL7em3E6EHkcDNhoMwzCMb9g9xTAMw/iGjQbDMAzjGzYaDMMwjG/YaDAMwzC+YaPBMAzD+IaNBsMwDOMbNhoMM04sXrwYr7zyykSLwTBHBBsNhmEYxje8uI9hxoH7778fL774IgzDQCAQwBe/+EV84QtfmGixGGbYsNFgmHFi8eLFuOWWWzBnzpyJFoVhRgy7pxiGYRjfsNFgGIZhfMNGg2EYhvENGw2GGScmT56M/v7+iRaDYY4IDoQzzDjR19eHn/zkJ8hms1i4cCE+//nPT7RIDDNs2GgwDMMwvmH3FMMwDOMbNhoMwzCMb9hoMAzDML5ho8EwDMP4ho0GwzAM4xs2GgzDMIxv2GgwDMMwvmGjwTAMw/iGjQbDMAzjm/8PTjgIk69TKeUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "N_EXP = 5\n",
    "\n",
    "r0ss = []\n",
    "r1ss = []\n",
    "#\n",
    "for n in range(N_EXP):\n",
    "\n",
    "    gamma = 0.8\n",
    "\n",
    "    env = AdvRwGridworld(max_steps=10, batch_size=1)\n",
    "    n_states = 12\n",
    "    ##\n",
    "    P2 = ExpSmoother(env.available_actions_Adv, env.available_actions_Adv, learning_rate=0.7)\n",
    "    #P1 =  RandomAgent( env.available_actions, p=0.5)\n",
    "    P1 = IndQLearningAgent(env.available_actions_DM, n_states, learning_rate=0.5,\n",
    "                           epsilon=0.1, gamma=gamma, enemy_action_space=None)\n",
    "    P1 = Level2QAgentSoftmax(env.available_actions_DM, env.available_actions_Adv, n_states=12,\n",
    "                       learning_rate=.5, epsilon=0.01, gamma=gamma)\n",
    "\n",
    "    \n",
    "    n_iter = 15000        \n",
    "\n",
    "\n",
    "    r0s = []\n",
    "    r1s = []\n",
    "    \n",
    "\n",
    "    for i in range(n_iter):\n",
    "\n",
    "        # A full episode:\n",
    "        done = False\n",
    "        env.reset()\n",
    "        s = 7\n",
    "\n",
    "        while not done:\n",
    "            \n",
    "            \n",
    "            episode_rewards_DM = 0\n",
    "\n",
    "            # Agents decide\n",
    "            a1 = P1.act(s)\n",
    "            a2 = P2.act()\n",
    "            # This is to test the neutral adversary\n",
    "            #a2 = np.random.choice(2, p=[0.9, 0.1])\n",
    "\n",
    "\n",
    "            # World changes\n",
    "            s_new, rewards, done =  env.step([a1,a2])\n",
    "\n",
    "\n",
    "            # Agents learn\n",
    "\n",
    "            P1.update(s, [a1, a2], [rewards[0], rewards[1]], s_new)\n",
    "            #P1.epsilon *= 0.99\n",
    "            if s_new == 0 or s_new == 8:  # the adversary only updates after choice\n",
    "                if s_new == 0:\n",
    "                    P2.update(0, [a2, 0], [rewards[1], rewards[0]], 0)\n",
    "                else:\n",
    "                    P2.update(0, [a2, 1], [rewards[1], rewards[0]], 0)\n",
    "\n",
    "            s = s_new  \n",
    "            #print(r0, r1)\n",
    "            episode_rewards_DM += rewards[0]\n",
    "            r1s.append(rewards[1])\n",
    "\n",
    "        r0s.append(episode_rewards_DM)\n",
    "        env.reset()\n",
    "        if i%10 == 0:\n",
    "            P1.epsilonA *= 0.995\n",
    "            P1.epsilonB *= 0.9\n",
    "        \n",
    "    print(n)\n",
    "    r0ss.append(r0s)\n",
    "    r1ss.append(r0s)\n",
    "    \n",
    "plot(r0ss, r1ss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-27T16:27:17.407852Z",
     "start_time": "2018-12-27T16:26:35.711004Z"
    }
   },
   "outputs": [],
   "source": [
    "def do_robustness_exp(learning_rate, learning_rate_adv, init_epsilon):\n",
    "\n",
    "    N_EXP = 5\n",
    "\n",
    "    r0ss = []\n",
    "    r1ss = []\n",
    "    #\n",
    "    for n in range(N_EXP):\n",
    "\n",
    "        gamma = 0.8\n",
    "\n",
    "        env = AdvRwGridworld(max_steps=100, batch_size=1)\n",
    "        n_states = 12\n",
    "        ##\n",
    "\n",
    "        P2 = ExpSmoother(env.available_actions_Adv, env.available_actions_Adv, learning_rate=0.75)\n",
    "        #P2 = Level2QAgent(env.available_actions_Adv,env.available_actions_Adv, n_states=12,\n",
    "        #                   learning_rate=0.1, epsilon=0.1, gamma=gamma)\n",
    "        #P1 =  RandomAgent( env.available_actions, p=0.5)\n",
    "        P1 = Level2QAgent(env.available_actions_DM, env.available_actions_Adv, n_states=12,\n",
    "                           learning_rate=learning_rate, epsilon=init_epsilon, gamma=gamma)\n",
    "\n",
    "        P1.enemy.alpha = learning_rate_adv\n",
    "\n",
    "\n",
    "        n_iter = 15000        \n",
    "\n",
    "\n",
    "        r0s = []\n",
    "        r1s = []\n",
    "\n",
    "\n",
    "        for i in range(n_iter):\n",
    "\n",
    "            # A full episode:\n",
    "            done = False\n",
    "            env.reset()\n",
    "            s = 7\n",
    "\n",
    "            while not done:\n",
    "\n",
    "\n",
    "                episode_rewards_DM = 0\n",
    "\n",
    "                # Agents decide\n",
    "                a1 = P1.act(s)\n",
    "                a2 = P2.act(s)\n",
    "                # This is to test the neutral adversary\n",
    "                #a2 = np.random.choice(2, p=[0.9, 0.1])\n",
    "\n",
    "\n",
    "                # World changes\n",
    "                s_new, rewards, done = env.step([a1,a2])\n",
    "\n",
    "\n",
    "                # Agents learn\n",
    "                if s_new == 0 or s_new == 8:  # the adversary only updates after choice\n",
    "\n",
    "                    P1.update(s, [a1, a2], [rewards[0], rewards[1]], s_new)\n",
    "\n",
    "\n",
    "                    if s_new == 0:\n",
    "                        P2.update(0, [a2, 0], [rewards[1], rewards[0]], 0)\n",
    "                    else:\n",
    "                        P2.update(0, [a2, 1], [rewards[1], rewards[0]], 0)\n",
    "\n",
    "\n",
    "                s = s_new  \n",
    "                #print(r0, r1)\n",
    "                episode_rewards_DM += rewards[0]\n",
    "                r1s.append(rewards[1])\n",
    "\n",
    "            r0s.append(episode_rewards_DM)\n",
    "            env.reset()\n",
    "\n",
    "            if i%10 == 0:\n",
    "                P1.epsilonA *= 0.995\n",
    "                P1.epsilonB *= 0.9\n",
    "\n",
    "            #print(P1.epsilonA)\n",
    "\n",
    "\n",
    "        print(n)\n",
    "        #print(P1.prob_type/P1.prob_type.sum())\n",
    "        r0ss.append(r0s[-3000:])\n",
    "        r1ss.append(r0s)\n",
    "    \n",
    "    return np.mean(r0ss), np.std(r0ss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "0.01 0.5 0.5 15.46 47.21301091860167\n",
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "0.01 0.5 0.1 40.77333333333333 27.477056287423196\n",
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "0.01 0.5 0.01 46.32 16.149848296501112\n",
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "0.01 2.0 0.5 15.58 47.171003805303954\n",
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "0.01 2.0 0.1 43.053333333333335 23.64960511768055\n",
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "0.01 2.0 0.01 47.81333333333333 10.828595887227893\n",
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "0.1 0.5 0.5 15.3 47.268488446321186\n",
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "0.1 0.5 0.1 42.82 24.07919433868168\n",
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "0.1 0.5 0.01 48.34 8.097184695929817\n",
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "0.1 2.0 0.5 15.966666666666667 47.03330970942\n",
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "0.1 2.0 0.1 43.04666666666667 23.662019262006265\n",
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "0.1 2.0 0.01 48.513333333333335 6.9591538438392195\n",
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "0.5 0.5 0.5 15.946666666666667 47.0405196494351\n",
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "0.5 0.5 0.1 43.06 23.6371825732256\n",
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "0.5 0.5 0.01 48.406666666666666 7.679927661696357\n",
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "0.5 2.0 0.5 15.193333333333333 47.30513667198897\n",
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "0.5 2.0 0.1 42.98 23.7857015872982\n",
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "0.5 2.0 0.01 48.53333333333333 6.8153421696118\n"
     ]
    }
   ],
   "source": [
    "lrs = np.asarray([0.01, 0.1, 0.5])\n",
    "lrs_enemy = np.asarray([0.5, 2])\n",
    "init_eps = np.asarray([0.5, 0.1, 0.01])\n",
    "\n",
    "rewards = np.zeros([3, 2, 3])\n",
    "stds = np.zeros([3, 2, 3])\n",
    "\n",
    "for i, lr in enumerate(lrs):\n",
    "    for j, lr_enemy in enumerate(lrs_enemy):\n",
    "        for k, init_ep in enumerate(init_eps):\n",
    "            rewards[i,j,k], stds[i,j,k] = do_robustness_exp(lr, lr*lr_enemy, init_ep)\n",
    "            print(lr, lr_enemy, init_ep, rewards[i,j,k], stds[i,j,k])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}