9.Multi-label classification using Neural Networks.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "9.Multi-label classification using Neural Networks.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "V0BqPQCyiZQS",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import matplotlib.pyplot as plt\n",
        "import numpy as np\n",
        "import copy\n",
        "import math\n",
        "import csv\n",
        "from google.colab import files\n",
        "from collections import deque"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HwmL4SD6ieF9",
        "colab_type": "code",
        "outputId": "1c7873fa-97ff-40c4-edba-52d25f87609c",
        "colab": {
          "resources": {
            "http://localhost:8080/nbextensions/google.colab/files.js": {
              "data": "Ly8gQ29weXJpZ2h0IDIwMTcgR29vZ2xlIExMQwovLwovLyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKLy8geW91IG1heSBub3QgdXNlIHRoaXMgZmlsZSBleGNlcHQgaW4gY29tcGxpYW5jZSB3aXRoIHRoZSBMaWNlbnNlLgovLyBZb3UgbWF5IG9idGFpbiBhIGNvcHkgb2YgdGhlIExpY2Vuc2UgYXQKLy8KLy8gICAgICBodHRwOi8vd3d3LmFwYWNoZS5vcmcvbGljZW5zZXMvTElDRU5TRS0yLjAKLy8KLy8gVW5sZXNzIHJlcXVpcmVkIGJ5IGFwcGxpY2FibGUgbGF3IG9yIGFncmVlZCB0byBpbiB3cml0aW5nLCBzb2Z0d2FyZQovLyBkaXN0cmlidXRlZCB1bmRlciB0aGUgTGljZW5zZSBpcyBkaXN0cmlidXRlZCBvbiBhbiAiQVMgSVMiIEJBU0lTLAovLyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KLy8gU2VlIHRoZSBMaWNlbnNlIGZvciB0aGUgc3BlY2lmaWMgbGFuZ3VhZ2UgZ292ZXJuaW5nIHBlcm1pc3Npb25zIGFuZAovLyBsaW1pdGF0aW9ucyB1bmRlciB0aGUgTGljZW5zZS4KCi8qKgogKiBAZmlsZW92ZXJ2aWV3IEhlbHBlcnMgZm9yIGdvb2dsZS5jb2xhYiBQeXRob24gbW9kdWxlLgogKi8KKGZ1bmN0aW9uKHNjb3BlKSB7CmZ1bmN0aW9uIHNwYW4odGV4dCwgc3R5bGVBdHRyaWJ1dGVzID0ge30pIHsKICBjb25zdCBlbGVtZW50ID0gZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgnc3BhbicpOwogIGVsZW1lbnQudGV4dENvbnRlbnQgPSB0ZXh0OwogIGZvciAoY29uc3Qga2V5IG9mIE9iamVjdC5rZXlzKHN0eWxlQXR0cmlidXRlcykpIHsKICAgIGVsZW1lbnQuc3R5bGVba2V5XSA9IHN0eWxlQXR0cmlidXRlc1trZXldOwogIH0KICByZXR1cm4gZWxlbWVudDsKfQoKLy8gTWF4IG51bWJlciBvZiBieXRlcyB3aGljaCB3aWxsIGJlIHVwbG9hZGVkIGF0IGEgdGltZS4KY29uc3QgTUFYX1BBWUxPQURfU0laRSA9IDEwMCAqIDEwMjQ7Ci8vIE1heCBhbW91bnQgb2YgdGltZSB0byBibG9jayB3YWl0aW5nIGZvciB0aGUgdXNlci4KY29uc3QgRklMRV9DSEFOR0VfVElNRU9VVF9NUyA9IDMwICogMTAwMDsKCmZ1bmN0aW9uIF91cGxvYWRGaWxlcyhpbnB1dElkLCBvdXRwdXRJZCkgewogIGNvbnN0IHN0ZXBzID0gdXBsb2FkRmlsZXNTdGVwKGlucHV0SWQsIG91dHB1dElkKTsKICBjb25zdCBvdXRwdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQob3V0cHV0SWQpOwogIC8vIENhY2hlIHN0ZXBzIG9uIHRoZSBvdXRwdXRFbGVtZW50IHRvIG1ha2UgaXQgYXZhaWxhYmxlIGZvciB0aGUgbmV4dCBjYWxsCiAgLy8gdG8gdXBsb2FkRmlsZXNDb250aW51ZSBmcm9tIFB5dGhvbi4KICBvdXRwdXRFbGVtZW50LnN0ZXBzID0gc3RlcHM7CgogIHJldHVybiBfdXBsb2FkRmlsZXNDb250aW51ZShvdXRwdXRJZCk7Cn0KCi8vIFRoaXMgaXMgcm91Z2hseSBhbiBhc3luYyBnZW5lcmF0b3IgKG5vdCBzdXBwb3J0ZWQgaW4gdGhlIGJyb3dzZXIgeWV0KSwKLy8gd2hlcmUgdGhlcmUgYXJlIG11bHRpcGxlIGFzeW5jaHJvbm91cyBzdGVwcyBhbmQgdGhlIFB5dGhvbiBzaWRlIGlzIGdvaW5nCi8vIHRvIHBvbGwgZm9yIGNvbXBsZXRpb24gb2YgZWFjaCBzdGVwLgovLyBUaGlzIHVzZXMgYSBQcm9taXNlIHRvIGJsb2NrIHRoZSBweXRob24gc2lkZSBvbiBjb21wbGV0aW9uIG9mIGVhY2ggc3RlcCwKLy8gdGhlbiBwYXNzZXMgdGhlIHJlc3VsdCBvZiB0aGUgcHJldmlvdXMgc3RlcCBhcyB0aGUgaW5wdXQgdG8gdGhlIG5leHQgc3RlcC4KZnVuY3Rpb24gX3VwbG9hZEZpbGVzQ29udGludWUob3V0cHV0SWQpIHsKICBjb25zdCBvdXRwdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQob3V0cHV0SWQpOwogIGNvbnN0IHN0ZXBzID0gb3V0cHV0RWxlbWVudC5zdGVwczsKCiAgY29uc3QgbmV4dCA9IHN0ZXBzLm5leHQob3V0cHV0RWxlbWVudC5sYXN0UHJvbWlzZVZhbHVlKTsKICByZXR1cm4gUHJvbWlzZS5yZXNvbHZlKG5leHQudmFsdWUucHJvbWlzZSkudGhlbigodmFsdWUpID0+IHsKICAgIC8vIENhY2hlIHRoZSBsYXN0IHByb21pc2UgdmFsdWUgdG8gbWFrZSBpdCBhdmFpbGFibGUgdG8gdGhlIG5leHQKICAgIC8vIHN0ZXAgb2YgdGhlIGdlbmVyYXRvci4KICAgIG91dHB1dEVsZW1lbnQubGFzdFByb21pc2VWYWx1ZSA9IHZhbHVlOwogICAgcmV0dXJuIG5leHQudmFsdWUucmVzcG9uc2U7CiAgfSk7Cn0KCi8qKgogKiBHZW5lcmF0b3IgZnVuY3Rpb24gd2hpY2ggaXMgY2FsbGVkIGJldHdlZW4gZWFjaCBhc3luYyBzdGVwIG9mIHRoZSB1cGxvYWQKICogcHJvY2Vzcy4KICogQHBhcmFtIHtzdHJpbmd9IGlucHV0SWQgRWxlbWVudCBJRCBvZiB0aGUgaW5wdXQgZmlsZSBwaWNrZXIgZWxlbWVudC4KICogQHBhcmFtIHtzdHJpbmd9IG91dHB1dElkIEVsZW1lbnQgSUQgb2YgdGhlIG91dHB1dCBkaXNwbGF5LgogKiBAcmV0dXJuIHshSXRlcmFibGU8IU9iamVjdD59IEl0ZXJhYmxlIG9mIG5leHQgc3RlcHMuCiAqLwpmdW5jdGlvbiogdXBsb2FkRmlsZXNTdGVwKGlucHV0SWQsIG91dHB1dElkKSB7CiAgY29uc3QgaW5wdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQoaW5wdXRJZCk7CiAgaW5wdXRFbGVtZW50LmRpc2FibGVkID0gZmFsc2U7CgogIGNvbnN0IG91dHB1dEVsZW1lbnQgPSBkb2N1bWVudC5nZXRFbGVtZW50QnlJZChvdXRwdXRJZCk7CiAgb3V0cHV0RWxlbWVudC5pbm5lckhUTUwgPSAnJzsKCiAgY29uc3QgcGlja2VkUHJvbWlzZSA9IG5ldyBQcm9taXNlKChyZXNvbHZlKSA9PiB7CiAgICBpbnB1dEVsZW1lbnQuYWRkRXZlbnRMaXN0ZW5lcignY2hhbmdlJywgKGUpID0+IHsKICAgICAgcmVzb2x2ZShlLnRhcmdldC5maWxlcyk7CiAgICB9KTsKICB9KTsKCiAgY29uc3QgY2FuY2VsID0gZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgnYnV0dG9uJyk7CiAgaW5wdXRFbGVtZW50LnBhcmVudEVsZW1lbnQuYXBwZW5kQ2hpbGQoY2FuY2VsKTsKICBjYW5jZWwudGV4dENvbnRlbnQgPSAnQ2FuY2VsIHVwbG9hZCc7CiAgY29uc3QgY2FuY2VsUHJvbWlzZSA9IG5ldyBQcm9taXNlKChyZXNvbHZlKSA9PiB7CiAgICBjYW5jZWwub25jbGljayA9ICgpID0+IHsKICAgICAgcmVzb2x2ZShudWxsKTsKICAgIH07CiAgfSk7CgogIC8vIENhbmNlbCB1cGxvYWQgaWYgdXNlciBoYXNuJ3QgcGlja2VkIGFueXRoaW5nIGluIHRpbWVvdXQuCiAgY29uc3QgdGltZW91dFByb21pc2UgPSBuZXcgUHJvbWlzZSgocmVzb2x2ZSkgPT4gewogICAgc2V0VGltZW91dCgoKSA9PiB7CiAgICAgIHJlc29sdmUobnVsbCk7CiAgICB9LCBGSUxFX0NIQU5HRV9USU1FT1VUX01TKTsKICB9KTsKCiAgLy8gV2FpdCBmb3IgdGhlIHVzZXIgdG8gcGljayB0aGUgZmlsZXMuCiAgY29uc3QgZmlsZXMgPSB5aWVsZCB7CiAgICBwcm9taXNlOiBQcm9taXNlLnJhY2UoW3BpY2tlZFByb21pc2UsIHRpbWVvdXRQcm9taXNlLCBjYW5jZWxQcm9taXNlXSksCiAgICByZXNwb25zZTogewogICAgICBhY3Rpb246ICdzdGFydGluZycsCiAgICB9CiAgfTsKCiAgaWYgKCFmaWxlcykgewogICAgcmV0dXJuIHsKICAgICAgcmVzcG9uc2U6IHsKICAgICAgICBhY3Rpb246ICdjb21wbGV0ZScsCiAgICAgIH0KICAgIH07CiAgfQoKICBjYW5jZWwucmVtb3ZlKCk7CgogIC8vIERpc2FibGUgdGhlIGlucHV0IGVsZW1lbnQgc2luY2UgZnVydGhlciBwaWNrcyBhcmUgbm90IGFsbG93ZWQuCiAgaW5wdXRFbGVtZW50LmRpc2FibGVkID0gdHJ1ZTsKCiAgZm9yIChjb25zdCBmaWxlIG9mIGZpbGVzKSB7CiAgICBjb25zdCBsaSA9IGRvY3VtZW50LmNyZWF0ZUVsZW1lbnQoJ2xpJyk7CiAgICBsaS5hcHBlbmQoc3BhbihmaWxlLm5hbWUsIHtmb250V2VpZ2h0OiAnYm9sZCd9KSk7CiAgICBsaS5hcHBlbmQoc3BhbigKICAgICAgICBgKCR7ZmlsZS50eXBlIHx8ICduL2EnfSkgLSAke2ZpbGUuc2l6ZX0gYnl0ZXMsIGAgKwogICAgICAgIGBsYXN0IG1vZGlmaWVkOiAkewogICAgICAgICAgICBmaWxlLmxhc3RNb2RpZmllZERhdGUgPyBmaWxlLmxhc3RNb2RpZmllZERhdGUudG9Mb2NhbGVEYXRlU3RyaW5nKCkgOgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAnbi9hJ30gLSBgKSk7CiAgICBjb25zdCBwZXJjZW50ID0gc3BhbignMCUgZG9uZScpOwogICAgbGkuYXBwZW5kQ2hpbGQocGVyY2VudCk7CgogICAgb3V0cHV0RWxlbWVudC5hcHBlbmRDaGlsZChsaSk7CgogICAgY29uc3QgZmlsZURhdGFQcm9taXNlID0gbmV3IFByb21pc2UoKHJlc29sdmUpID0+IHsKICAgICAgY29uc3QgcmVhZGVyID0gbmV3IEZpbGVSZWFkZXIoKTsKICAgICAgcmVhZGVyLm9ubG9hZCA9IChlKSA9PiB7CiAgICAgICAgcmVzb2x2ZShlLnRhcmdldC5yZXN1bHQpOwogICAgICB9OwogICAgICByZWFkZXIucmVhZEFzQXJyYXlCdWZmZXIoZmlsZSk7CiAgICB9KTsKICAgIC8vIFdhaXQgZm9yIHRoZSBkYXRhIHRvIGJlIHJlYWR5LgogICAgbGV0IGZpbGVEYXRhID0geWllbGQgewogICAgICBwcm9taXNlOiBmaWxlRGF0YVByb21pc2UsCiAgICAgIHJlc3BvbnNlOiB7CiAgICAgICAgYWN0aW9uOiAnY29udGludWUnLAogICAgICB9CiAgICB9OwoKICAgIC8vIFVzZSBhIGNodW5rZWQgc2VuZGluZyB0byBhdm9pZCBtZXNzYWdlIHNpemUgbGltaXRzLiBTZWUgYi82MjExNTY2MC4KICAgIGxldCBwb3NpdGlvbiA9IDA7CiAgICB3aGlsZSAocG9zaXRpb24gPCBmaWxlRGF0YS5ieXRlTGVuZ3RoKSB7CiAgICAgIGNvbnN0IGxlbmd0aCA9IE1hdGgubWluKGZpbGVEYXRhLmJ5dGVMZW5ndGggLSBwb3NpdGlvbiwgTUFYX1BBWUxPQURfU0laRSk7CiAgICAgIGNvbnN0IGNodW5rID0gbmV3IFVpbnQ4QXJyYXkoZmlsZURhdGEsIHBvc2l0aW9uLCBsZW5ndGgpOwogICAgICBwb3NpdGlvbiArPSBsZW5ndGg7CgogICAgICBjb25zdCBiYXNlNjQgPSBidG9hKFN0cmluZy5mcm9tQ2hhckNvZGUuYXBwbHkobnVsbCwgY2h1bmspKTsKICAgICAgeWllbGQgewogICAgICAgIHJlc3BvbnNlOiB7CiAgICAgICAgICBhY3Rpb246ICdhcHBlbmQnLAogICAgICAgICAgZmlsZTogZmlsZS5uYW1lLAogICAgICAgICAgZGF0YTogYmFzZTY0LAogICAgICAgIH0sCiAgICAgIH07CiAgICAgIHBlcmNlbnQudGV4dENvbnRlbnQgPQogICAgICAgICAgYCR7TWF0aC5yb3VuZCgocG9zaXRpb24gLyBmaWxlRGF0YS5ieXRlTGVuZ3RoKSAqIDEwMCl9JSBkb25lYDsKICAgIH0KICB9CgogIC8vIEFsbCBkb25lLgogIHlpZWxkIHsKICAgIHJlc3BvbnNlOiB7CiAgICAgIGFjdGlvbjogJ2NvbXBsZXRlJywKICAgIH0KICB9Owp9CgpzY29wZS5nb29nbGUgPSBzY29wZS5nb29nbGUgfHwge307CnNjb3BlLmdvb2dsZS5jb2xhYiA9IHNjb3BlLmdvb2dsZS5jb2xhYiB8fCB7fTsKc2NvcGUuZ29vZ2xlLmNvbGFiLl9maWxlcyA9IHsKICBfdXBsb2FkRmlsZXMsCiAgX3VwbG9hZEZpbGVzQ29udGludWUsCn07Cn0pKHNlbGYpOwo=",
              "ok": true,
              "headers": [
                [
                  "content-type",
                  "application/javascript"
                ]
              ],
              "status": 200,
              "status_text": ""
            }
          },
          "base_uri": "https://localhost:8080/",
          "height": 78
        }
      },
      "source": [
        "uploaded = files.upload()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "     <input type=\"file\" id=\"files-8c25f969-85fc-4059-b74e-e854854c8d76\" name=\"files[]\" multiple disabled />\n",
              "     <output id=\"result-8c25f969-85fc-4059-b74e-e854854c8d76\">\n",
              "      Upload widget is only available when the cell has been executed in the\n",
              "      current browser session. Please rerun this cell to enable.\n",
              "      </output>\n",
              "      <script src=\"/nbextensions/google.colab/files.js\"></script> "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Saving mnist.csv to mnist.csv\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JmV8XN4XifFM",
        "colab_type": "code",
        "outputId": "0f4308cc-9550-4a8b-d362-62623cb0dfa8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 144
        }
      },
      "source": [
        "file_data   = \"mnist.csv\"\n",
        "handle_file = open(file_data, \"r\")\n",
        "data        = handle_file.readlines()\n",
        "handle_file.close()\n",
        "\n",
        "size_row    = 28    # height of the image\n",
        "size_col    = 28    # width of the image\n",
        "\n",
        "num_image   = len(data)   #10000개\n",
        "count       = 0     # count for the number of images\n",
        "\n",
        "#\n",
        "# normalize the values of the input data to be [0, 1]\n",
        "#\n",
        "def normalize(data):\n",
        "\n",
        "    data_normalized = (data - min(data)) / (max(data) - min(data))\n",
        "\n",
        "    return(data_normalized)\n",
        "\n",
        "#\n",
        "# example of distance function between two vectors x and y\n",
        "#\n",
        "def distance(x, y):\n",
        "\n",
        "    d = (x - y) ** 2\n",
        "    s = np.sum(d)\n",
        "    # r = np.sqrt(s)\n",
        "\n",
        "    return(s)\n",
        "\n",
        "#\n",
        "# make a matrix each column of which represents an images in a vector form\n",
        "#\n",
        "train_image  = np.empty((size_row * size_col, 6000), dtype=float)\n",
        "train_label  = np.empty(6000, dtype=int)\n",
        "test_image  = np.empty((size_row * size_col, 4000), dtype=float)\n",
        "test_label  = np.empty(4000, dtype=int)\n",
        "\n",
        "for line in data:\n",
        "    if count < 6000 :\n",
        "      line_data   = line.split(',')\n",
        "      label       = line_data[0]\n",
        "      im_vector   = np.asfarray(line_data[1:])\n",
        "      im_vector   = normalize(im_vector)\n",
        "\n",
        "      train_label[count]       = label       # size : 10000\n",
        "      train_image[:, count]    = im_vector   # size : 784 * 10000\n",
        "\n",
        "    else: \n",
        "      line_data   = line.split(',')\n",
        "      label       = line_data[0]\n",
        "      im_vector   = np.asfarray(line_data[1:])\n",
        "      im_vector   = normalize(im_vector)\n",
        "\n",
        "      test_label[count-6000]       = label       # size : 10000\n",
        "      test_image[:, count-6000]    = im_vector   # size : 784 * 10000\n",
        "\n",
        "    count += 1\n",
        "#\n",
        "# plot first 150 images out of 10,000 with their labels\n",
        "#\n",
        "\"\"\"\n",
        "f1 = plt.figure(1)\n",
        "\n",
        "for i in range(150):\n",
        "\n",
        "    label       = list_label[i]\n",
        "    im_vector   = list_image[:, i]\n",
        "    im_matrix   = im_vector.reshape((size_row, size_col))\n",
        "\n",
        "    plt.subplot(10, 15, i+1)\n",
        "    plt.title(label)\n",
        "    plt.imshow(im_matrix, cmap='Greys', interpolation='None')\n",
        "\n",
        "    frame   = plt.gca()\n",
        "    frame.axes.get_xaxis().set_visible(False)\n",
        "    frame.axes.get_yaxis().set_visible(False)\n",
        "\n",
        "\n",
        "#plt.show()\n",
        "\n",
        "#\n",
        "# plot the average image of all the images for each digit\n",
        "#\n",
        "f2 = plt.figure(2)\n",
        "\n",
        "im_average  = np.zeros((size_row * size_col, 10), dtype=float)\n",
        "im_count    = np.zeros(10, dtype=int)\n",
        "\n",
        "for i in range(10):\n",
        "\n",
        "    im_average[:, test_label[i]] += test_image[:, i]\n",
        "    im_count[test_label[i]] += 1\n",
        "\n",
        "for i in range(10):\n",
        "\n",
        "    im_average[:, i] /= im_count[i]\n",
        "\n",
        "    plt.subplot(2, 5, i+1)\n",
        "    plt.title(i)\n",
        "    plt.imshow(im_average[:,i].reshape((size_row, size_col)), cmap='Greys', interpolation='None')\n",
        "\n",
        "    frame   = plt.gca()\n",
        "    frame.axes.get_xaxis().set_visible(False)\n",
        "    frame.axes.get_yaxis().set_visible(False)\n",
        "\n",
        "plt.show()\n",
        "\"\"\"\n",
        "train_costs = deque()\n",
        "test_costs = deque()\n",
        "theta_0s = deque()\n",
        "theta_1s = deque()\n",
        "theta_2s = deque()\n",
        "bias_0s = deque()\n",
        "bias_1s = deque()\n",
        "bias_2s = deque()\n",
        "iters = deque()\n",
        "train_accuracys = deque()\n",
        "test_accuracys = deque()\n",
        "incorrect_predicts = deque()\n",
        "\n",
        "#correct_labels = []\n",
        "correct_labels = deque()\n",
        "#incorrect_labels = []\n",
        "incorrect_labels = deque()\n",
        "\n",
        "print(num_image)  #10000\n",
        "print(len(train_image))  #784\n",
        "print(len(train_image[0]))  #6000\n",
        "print(len(train_label))  #6000\n",
        "print(len(test_image))  #784\n",
        "print(len(test_image[0]))  #4000\n",
        "print(len(test_label))  #4000\n",
        "#list_label 0~5999까지 training data, 6000~9999까지 test data\n",
        "#list_image [:,5999]까지 training data, [:,9999]까지 test data"
      ],
      "execution_count": 125,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "10000\n",
            "784\n",
            "6000\n",
            "6000\n",
            "784\n",
            "4000\n",
            "4000\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "B_UkBCZFii_k",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def forward(theta, value):\n",
        "  #print(theta.shape, value.shape)\n",
        "  #print(\"Theta\", theta.shape)\n",
        "  #print(\"value\", value.shape)\n",
        "  result = np.matmul(theta, value)\n",
        "  return result"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qVq9jp5iikTi",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def sigmoid(value):\n",
        "  \"\"\"\n",
        "  length = len(value)\n",
        "  for i in range(0, length):\n",
        "    value[i] = 1 / (1 + np.exp(-value[i]))\n",
        "  \"\"\"\n",
        "  value = value.astype(float)\n",
        "  value = np.multiply(value,-1)\n",
        "  value = np.exp(value)\n",
        "  value = np.add(1,value)\n",
        "  value = np.divide(1, value)\n",
        "  return value"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PsayRgz3ZkGm",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def addBias(value, bias):\n",
        "  \"\"\"\n",
        "  length = len(bias)\n",
        "  for i in range(0, length):\n",
        "    value[i] += bias[i]\n",
        "  \"\"\"\n",
        "  value = np.add(value, bias)\n",
        "  return value"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Vvg44h6kZWZr",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def neuralNet(x, u, v, w, bias0, bias1, bias2):    #data, theta0, theta1, theta2, bias0, bias1, bias2\n",
        "  y_hat = forward(u, x)\n",
        "  y_hat = addBias(y_hat, bias0)\n",
        "  y = sigmoid(y_hat)\n",
        "  z_hat = forward(v, y)\n",
        "  z_hat = addBias(z_hat, bias1)\n",
        "  z = sigmoid(z_hat)\n",
        "  h_hat = forward(w, z)\n",
        "  h_hat = addBias(h_hat, bias2)\n",
        "  h = sigmoid(h_hat)\n",
        "  return y, z, h"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sjcnYDFMnCRg",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def objectiveFunction(image, label, theta0, theta1, theta2, b0, b1, b2):     #list_image[:, count], list_label -> traing과 test 구분해야겠다.!!\n",
        "  length = len(label)                    #두는 이유는 trainig 6000, test 4000개  갯수가 달라서\n",
        "  cost = 0\n",
        "  for i in range(0, length):\n",
        "    y, z, outputs = neuralNet(image[:,i], theta0, theta1, theta2, b0, b1, b2)  #size = 10x1 good!!!\n",
        "    l = label[i]\n",
        "    #print(\"outputs\",outputs)\n",
        "    temp7 = np.log(outputs)\n",
        "    #print(\"log\",temp7)\n",
        "    temp8 = np.subtract(1, outputs)\n",
        "    temp8 = np.log(temp8)\n",
        "    cost1 = -temp7[l] - np.sum(temp8) + temp8[l]\n",
        "    #print(-temp7[l], temp8[l])\n",
        "    cost += cost1\n",
        "    cost /= length\n",
        "  print(cost)\n",
        "  \n",
        "  return cost"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tzRWAQQG0uxJ",
        "colab_type": "code",
        "outputId": "ed8fb081-3372-4d93-a637-6990a6b0acaf",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 55
        }
      },
      "source": [
        "def gradientDescent(image, label, theta0, theta1, theta2, b0, b1, b2, learning_rate):\n",
        "  lr = learning_rate\n",
        "  length = len(label)\n",
        "  for i in range(0, length):\n",
        "    x = image[:,i]\n",
        "    y, z, h = neuralNet(image[:,i], theta0, theta1, theta2, b0, b1, b2)\n",
        "    l = label[i]\n",
        "\n",
        "    ##first layer\n",
        "    temp1 = np.zeros(10)\n",
        "    temp1 = np.subtract(h, l)\n",
        "    temp = np.multiply(temp1, lr/length)\n",
        "    b2 = np.subtract(b2, temp)\n",
        "\n",
        "    z_transpose = np.reshape(z, (1,49))\n",
        "    temp1 = np.reshape(temp1, (10,1))\n",
        "    temp2 = np.zeros((10,49))\n",
        "    temp2 = np.matmul(temp1, z_transpose)\n",
        "    temp = np.multiply(temp2, lr/length)\n",
        "    theta2 = np.subtract(theta2, temp)\n",
        "    \n",
        "    ##second layer\n",
        "    temp3 = np.zeros(49)\n",
        "    temp1 = np.subtract(h, l)                 #그전의 temp1을 가져와서 썼더니, 차원 size forward matmul에서 차원이 틀려서 안됨!\n",
        "    temp3 = np.matmul(temp1, theta2)\n",
        "    temp3 = np.multiply(temp3, z)\n",
        "    temp3 = np.multiply(temp3, 1-z)\n",
        "    temp = np.multiply(temp3, lr/length)\n",
        "    b1 = np.array(np.subtract(b1, temp), dtype=object)\n",
        "\n",
        "    temp4 = np.zeros((49,196))\n",
        "    temp3 = np.reshape(temp3, (49,1))\n",
        "    y_transpose = np.reshape(y, (1,196))\n",
        "    #print(temp3.shape)\n",
        "    #print(y_transpose.shape)\n",
        "    temp4 = np.matmul(temp3, y_transpose)\n",
        "    temp = np.multiply(temp4, lr/length)        ##***********이거 안넣었다\n",
        "    theta1 = np.array(np.subtract(theta1, temp), dtype=object)\n",
        "\n",
        "    ##third layer\n",
        "    temp5 = np.zeros(196)\n",
        "    temp1 = np.subtract(h, l)\n",
        "    temp5 = np.matmul(temp1, theta2)\n",
        "    temp5 = np.multiply(temp5, z)\n",
        "    temp5 = np.multiply(temp5, 1-z)\n",
        "    theta1 = np.reshape(theta1, (49, 196))\n",
        "    temp5 = np.matmul(temp5, theta1)\n",
        "    temp5 = np.multiply(temp5, y)\n",
        "    temp5 = np.multiply(temp5, 1-y)\n",
        "    temp = np.multiply(temp5, lr/length)\n",
        "    b0 = np.array(np.subtract(b0, temp), dtype=object)\n",
        "\n",
        "    temp6 = np.zeros((196, 784))\n",
        "    temp5 = np.reshape(temp5, (196,1))\n",
        "    x_transpose = np.reshape(x, (1,784))\n",
        "    #print(temp3.shape)\n",
        "    #print(y_transpose.shape)\n",
        "    temp6 = np.matmul(temp5, x_transpose)\n",
        "    temp = np.multiply(temp6, lr/length)        ##***********이거 안넣었다\n",
        "    theta0 = np.array(np.subtract(theta0, temp), dtype=object)\n",
        "\n",
        "  print(\"b2.shape\",b2.shape)\n",
        "  print(\"b1.shape\",b1.shape)\n",
        "  print(\"b0.shape\",b0.shape)\n",
        "  print(\"theta2.shape\",theta2.shape)\n",
        "  print(\"theta1.shape\",theta1.shape)\n",
        "  print(\"theta0.shape\",theta0.shape)\n",
        "  return b2, theta2, b1, theta1, b0, theta0\n",
        "\n",
        "\"\"\"  \n",
        "\n",
        "\n",
        "\n",
        "    ## first layer update\n",
        "    for j in range(0, 10):\n",
        "      estimate1 = (-label[i] + h[j])\n",
        "      b2[j] -= lr * estimate1       #bias2 update!\n",
        "      \n",
        "      for k in range(0, 49):\n",
        "        estimate2 = estimate1 * z[k]\n",
        "        theta2[j][k] -= lr * estimate2   #theta2 update!\n",
        "    \n",
        "    ## second layer update\n",
        "    for l in range(0, 196):\n",
        "      for j in range(0, 49):\n",
        "        estimate3 = 0\n",
        "        for k in range(0, 10):\n",
        "          estimate3 += (-label[i] + h[k]) * theta1[k][j]\n",
        "        b1[j] -= lr * z[j] * (1 - z[j]) * estimate3\n",
        "        theta1[j][l] -= b1[j] * y[l]\n",
        "\"\"\""
      ],
      "execution_count": 131,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'  \\n\\n\\n\\n    ## first layer update\\n    for j in range(0, 10):\\n      estimate1 = (-label[i] + h[j])\\n      b2[j] -= lr * estimate1       #bias2 update!\\n      \\n      for k in range(0, 49):\\n        estimate2 = estimate1 * z[k]\\n        theta2[j][k] -= lr * estimate2   #theta2 update!\\n    \\n    ## second layer update\\n    for l in range(0, 196):\\n      for j in range(0, 49):\\n        estimate3 = 0\\n        for k in range(0, 10):\\n          estimate3 += (-label[i] + h[k]) * theta1[k][j]\\n        b1[j] -= lr * z[j] * (1 - z[j]) * estimate3\\n        theta1[j][l] -= b1[j] * y[l]\\n'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 131
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WN6ghexckeWy",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def checkAccuracy(image, label, theta0, theta1, theta2, b0, b1, b2):\n",
        "  length = len(label)\n",
        "  count = 0\n",
        "  for i in range(0, length):\n",
        "    y, z, h = neuralNet(image[:,i], theta0, theta1, theta2, b0, b1, b2)\n",
        "    if label[i] == np.argmax(h):\n",
        "      count += 1\n",
        "  accuracy = count / length\n",
        "  return accuracy"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TP9QaVGX2KDJ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def checkCorrect(image, label, theta0, theta1, theta2, b0, b1, b2):\n",
        "  length = len(label)\n",
        "  count = 0\n",
        "  labels = []\n",
        "  for i in range(0, length):\n",
        "    if count == 10:\n",
        "      break\n",
        "    else:\n",
        "      y, z, h = neuralNet(image[:,i], theta0, theta1, theta2, b0, b1, b2)\n",
        "      if label[i] == np.argmax(h):\n",
        "        labels.append(i)  #index를 labels 리스트에 추가\n",
        "        count += 1\n",
        "  return labels"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TURh4x9O4ZhA",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def checkIncorrect(image, label, theta0, theta1, theta2, b0, b1, b2):\n",
        "  length = len(label)\n",
        "  count = 0\n",
        "  labels = []\n",
        "  predicts = []\n",
        "  for i in range(0, length):\n",
        "    if count == 10:\n",
        "      break\n",
        "    else:\n",
        "      y, z, h = neuralNet(image[:,i], theta0, theta1, theta2, b0, b1, b2)\n",
        "      if label[i] != np.argmax(h):\n",
        "        labels.append(i)  #index를 labels 리스트에 추가\n",
        "        count += 1\n",
        "        predicts.append(np.argmax(h))\n",
        "  return labels, predicts"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ae53uHmoilyj",
        "colab_type": "code",
        "outputId": "3b970fda-e98f-4c4a-a356-7d992d7dc587",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 3617
        }
      },
      "source": [
        "def main():\n",
        "  mu = 0\n",
        "  sigma = 1\n",
        "\n",
        "  theta_0 = np.random.normal(mu, sigma, size=(196,784))   #size = 196x784\n",
        "  theta_1 = np.random.normal(mu, sigma, size=(49,196))    #size = 49x196\n",
        "  theta_2 = np.random.normal(mu, sigma, size=(10,49))     #size = 10x49\n",
        "  \n",
        "  b_0 = np.random.normal(mu, sigma, 196)                  #size = 196x1\n",
        "  b_1 = np.random.normal(mu, sigma, 49)                   #size = 49x1\n",
        "  b_2 = np.random.normal(mu, sigma, 10)                   #size = 10x1\n",
        "\n",
        "  lr = 0.0003                                                #learning rate\n",
        "\n",
        "  #training step\n",
        "  for iter in range(0, 15):\n",
        "    iters.append(iter)\n",
        "    train_cost = objectiveFunction(train_image, train_label, theta_0, theta_1, theta_2, b_0, b_1, b_2)  #print(cost)\n",
        "    print(\"##########################################111111111111111111111111111111\")\n",
        "    train_costs.append(train_cost)\n",
        "    b_2, theta_2, b_1, theta_1, b_0, theta_0 = gradientDescent(train_image, train_label, theta_0, theta_1, theta_2, b_0, b_1, b_2, lr)\n",
        "    \n",
        "    #############train accuracy 하자!############\n",
        "    train_accuracy = checkAccuracy(train_image, train_label, theta_0, theta_1, theta_2, b_0, b_1, b_2)\n",
        "    train_accuracys.append(train_accuracy)\n",
        "    print(\"##########################################12222222222\")\n",
        "\n",
        "    test_cost = objectiveFunction(test_image, test_label, theta_0, theta_1, theta_2, b_0, b_1, b_2)\n",
        "    print(\"##########################################33333333333\")\n",
        "    test_costs.append(test_cost)\n",
        "    #############test accuracy\n",
        "    test_accuracy = checkAccuracy(test_image, test_label, theta_0, theta_1, theta_2, b_0, b_1, b_2)\n",
        "    test_accuracys.append(test_accuracy)\n",
        "\n",
        "    #train_y, train_z, train_h = neuralNet(train_image, theta_0, theta_1, theta_2, b_0, b_1, b_2) #image 문제\n",
        "    print(\"##########################################4444444444444444444\")\n",
        "\n",
        "    \n",
        "  ######correct label 가져오자!\n",
        "  #correct_labels = checkCorrect(test_image, test_label, theta_0, theta_1, theta_2, b_0, b_1, b_2)  #data의 index만 헐당!\n",
        "  correct_label = checkCorrect(test_image, test_label, theta_0, theta_1, theta_2, b_0, b_1, b_2)\n",
        "  correct_labels.append(correct_label)\n",
        "  #incorrect_labels = checkIncorrect(test_image, test_label, theta_0, theta_1, theta_2, b_0, b_1, b_2)\n",
        "  incorrect_label, predict = checkIncorrect(test_image, test_label, theta_0, theta_1, theta_2, b_0, b_1, b_2)\n",
        "  incorrect_labels.append(incorrect_label)\n",
        "  incorrect_predicts.append(predict)\n",
        "\n",
        "  print(\"iters\",iters)\n",
        "  print(\"train_costs\",train_costs)\n",
        "  print(\"Plotting the loss curve\")\n",
        "  fig = plt.figure()\n",
        "  graph1 = plt.plot(iters, train_costs, 'b')\n",
        "  graph2 = plt.plot(iters, test_costs, 'r')\n",
        "  plt.xlabel('iterations')\n",
        "  plt.ylabel('loss')\n",
        "  plt.show()\n",
        "\n",
        "if __name__ == \"__main__\":\n",
        "  main()"
      ],
      "execution_count": 135,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0.004413280271112331\n",
            "##########################################111111111111111111111111111111\n",
            "b2.shape (10,)\n",
            "b1.shape (49,)\n",
            "b0.shape (196,)\n",
            "theta2.shape (10, 49)\n",
            "theta1.shape (49, 196)\n",
            "theta0.shape (196, 784)\n",
            "##########################################12222222222\n",
            "0.007771740892294841\n",
            "##########################################33333333333\n",
            "##########################################4444444444444444444\n",
            "0.004430508377445608\n",
            "##########################################111111111111111111111111111111\n",
            "b2.shape (10,)\n",
            "b1.shape (49,)\n",
            "b0.shape (196,)\n",
            "theta2.shape (10, 49)\n",
            "theta1.shape (49, 196)\n",
            "theta0.shape (196, 784)\n",
            "##########################################12222222222\n",
            "0.007808811405308316\n",
            "##########################################33333333333\n",
            "##########################################4444444444444444444\n",
            "0.004447782714760732\n",
            "##########################################111111111111111111111111111111\n",
            "b2.shape (10,)\n",
            "b1.shape (49,)\n",
            "b0.shape (196,)\n",
            "theta2.shape (10, 49)\n",
            "theta1.shape (49, 196)\n",
            "theta0.shape (196, 784)\n",
            "##########################################12222222222\n",
            "0.007845968482743746\n",
            "##########################################33333333333\n",
            "##########################################4444444444444444444\n",
            "0.004465103529253866\n",
            "##########################################111111111111111111111111111111\n",
            "b2.shape (10,)\n",
            "b1.shape (49,)\n",
            "b0.shape (196,)\n",
            "theta2.shape (10, 49)\n",
            "theta1.shape (49, 196)\n",
            "theta0.shape (196, 784)\n",
            "##########################################12222222222\n",
            "0.00788320799276366\n",
            "##########################################33333333333\n",
            "##########################################4444444444444444444\n",
            "0.004482471077736326\n",
            "##########################################111111111111111111111111111111\n",
            "b2.shape (10,)\n",
            "b1.shape (49,)\n",
            "b0.shape (196,)\n",
            "theta2.shape (10, 49)\n",
            "theta1.shape (49, 196)\n",
            "theta0.shape (196, 784)\n",
            "##########################################12222222222\n",
            "0.0079205258046187\n",
            "##########################################33333333333\n",
            "##########################################4444444444444444444\n",
            "0.004499885628334252\n",
            "##########################################111111111111111111111111111111\n",
            "b2.shape (10,)\n",
            "b1.shape (49,)\n",
            "b0.shape (196,)\n",
            "theta2.shape (10, 49)\n",
            "theta1.shape (49, 196)\n",
            "theta0.shape (196, 784)\n",
            "##########################################12222222222\n",
            "0.00795791780260037\n",
            "##########################################33333333333\n",
            "##########################################4444444444444444444\n",
            "0.0045173474611988635\n",
            "##########################################111111111111111111111111111111\n",
            "b2.shape (10,)\n",
            "b1.shape (49,)\n",
            "b0.shape (196,)\n",
            "theta2.shape (10, 49)\n",
            "theta1.shape (49, 196)\n",
            "theta0.shape (196, 784)\n",
            "##########################################12222222222\n",
            "0.007995379899694552\n",
            "##########################################33333333333\n",
            "##########################################4444444444444444444\n",
            "0.004534856869226971\n",
            "##########################################111111111111111111111111111111\n",
            "b2.shape (10,)\n",
            "b1.shape (49,)\n",
            "b0.shape (196,)\n",
            "theta2.shape (10, 49)\n",
            "theta1.shape (49, 196)\n",
            "theta0.shape (196, 784)\n",
            "##########################################12222222222\n",
            "0.008032908050812054\n",
            "##########################################33333333333\n",
            "##########################################4444444444444444444\n",
            "0.004552414158788825\n",
            "##########################################111111111111111111111111111111\n",
            "b2.shape (10,)\n",
            "b1.shape (49,)\n",
            "b0.shape (196,)\n",
            "theta2.shape (10, 49)\n",
            "theta1.shape (49, 196)\n",
            "theta0.shape (196, 784)\n",
            "##########################################12222222222\n",
            "0.008070498265502017\n",
            "##########################################33333333333\n",
            "##########################################4444444444444444444\n",
            "0.004570019650466157\n",
            "##########################################111111111111111111111111111111\n",
            "b2.shape (10,)\n",
            "b1.shape (49,)\n",
            "b0.shape (196,)\n",
            "theta2.shape (10, 49)\n",
            "theta1.shape (49, 196)\n",
            "theta0.shape (196, 784)\n",
            "##########################################12222222222\n",
            "0.00810814662008242\n",
            "##########################################33333333333\n",
            "##########################################4444444444444444444\n",
            "0.004587673679797139\n",
            "##########################################111111111111111111111111111111\n",
            "b2.shape (10,)\n",
            "b1.shape (49,)\n",
            "b0.shape (196,)\n",
            "theta2.shape (10, 49)\n",
            "theta1.shape (49, 196)\n",
            "theta0.shape (196, 784)\n",
            "##########################################12222222222\n",
            "0.008145849269089376\n",
            "##########################################33333333333\n",
            "##########################################4444444444444444444\n",
            "0.004605376598028834\n",
            "##########################################111111111111111111111111111111\n",
            "b2.shape (10,)\n",
            "b1.shape (49,)\n",
            "b0.shape (196,)\n",
            "theta2.shape (10, 49)\n",
            "theta1.shape (49, 196)\n",
            "theta0.shape (196, 784)\n",
            "##########################################12222222222\n",
            "0.008183602455994919\n",
            "##########################################33333333333\n",
            "##########################################4444444444444444444\n",
            "0.00462312877287641\n",
            "##########################################111111111111111111111111111111\n",
            "b2.shape (10,)\n",
            "b1.shape (49,)\n",
            "b0.shape (196,)\n",
            "theta2.shape (10, 49)\n",
            "theta1.shape (49, 196)\n",
            "theta0.shape (196, 784)\n",
            "##########################################12222222222\n",
            "0.008221402523136372\n",
            "##########################################33333333333\n",
            "##########################################4444444444444444444\n",
            "0.004640930589287226\n",
            "##########################################111111111111111111111111111111\n",
            "b2.shape (10,)\n",
            "b1.shape (49,)\n",
            "b0.shape (196,)\n",
            "theta2.shape (10, 49)\n",
            "theta1.shape (49, 196)\n",
            "theta0.shape (196, 784)\n",
            "##########################################12222222222\n",
            "0.008259245920806178\n",
            "##########################################33333333333\n",
            "##########################################4444444444444444444\n",
            "0.004658782450211379\n",
            "##########################################111111111111111111111111111111\n",
            "b2.shape (10,)\n",
            "b1.shape (49,)\n",
            "b0.shape (196,)\n",
            "theta2.shape (10, 49)\n",
            "theta1.shape (49, 196)\n",
            "theta0.shape (196, 784)\n",
            "##########################################12222222222\n",
            "0.008297129215462332\n",
            "##########################################33333333333\n",
            "##########################################4444444444444444444\n",
            "iters deque([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14])\n",
            "train_costs deque([0.004413280271112331, 0.004430508377445608, 0.004447782714760732, 0.004465103529253866, 0.004482471077736326, 0.004499885628334252, 0.0045173474611988635, 0.004534856869226971, 0.004552414158788825, 0.004570019650466157, 0.004587673679797139, 0.004605376598028834, 0.00462312877287641, 0.004640930589287226, 0.004658782450211379])\n",
            "Plotting the loss curve\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZQAAAEGCAYAAABCa2PoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3df5BV5Z3n8ffHbkFAgwItImAggqbAGDQdJsaYTUJUdCxJZt2ImWSZhBSZLcmvSSYDM1tTKWtnSxMTJ1UxThElEqMiIXHt2sqKv2Z3pmaNcjHEHxg2HdTQjEIrij9QsOG7f5ynw+Vyb/dtOKdvX/rzquq65zznOed+TxecT5/znHuPIgIzM7MjdUyjCzAzs6ODA8XMzHLhQDEzs1w4UMzMLBcOFDMzy0VrowtopAkTJsS0adMaXYaZWVPZsGHDixHRVtk+rANl2rRplEqlRpdhZtZUJD1Xrd2XvMzMLBcOFDMzy4UDxczMcuFAMTOzXDhQzMwsFw4UMzPLhQPFzMxyMaw/h2JmdlSLgFdege3bs58dOw5Mf+5z8K535fp2DhQzs2aybx+8+GL1kCif7p1/++1DtyHBBz/oQDEzO+rs2VM7GCpD4sUXszOPSiNGwMknw8SJcMopcPbZ2fTEiQfae6cnTICWltx3w4FiZpa3CHj99f4Dond6167q2zn++AMhMGMGnH/+oeHQOz12bHbm0UAOFDOzeuzfDzt31hcQO3bAm29W38748QeC4JxzagfEySfD6NGDu49HyIFiZsPX3r3Q3V1fSHR3Z+MXlVpaDg6CM8+sHRBtbXDssYO/n4PEgWJmR5fXXz/4TKGvkHj55erbGDXqQAicdhq8//21zyROOgmO8ScwwIFiZkPd/v3Zgb/WpaXKtt27q2/nxBMPBMFZZ8HHPlb7TOL44xs+HtGMHChmNvjefvvQS021AmPHDujpOXQbxxyTXULqDYMZM6qfRZx8cvYzcuTg7+cw40Axs3y88UZ9AbF9eza4Xc3IkQcCYfJkOPfc2peaxo0r5NZXO3wOFDOrrvJSU39B8cYb1bczduyBMJg1Cz7ykeqXmSZOhBNO8KWmJuZAMRtOql1qqhYWA7nU9K531Q6ItjY47rjB309rCAeKWbOrdldTrddal5qOO+5AEEyZkl1qqjYW4UtN1odCA0XSfOD7QAtwc0RcW7F8JPAT4H3AS8CVEfFsWrYcWAzsA74cEetS+9eALwABPAF8LiLekjQdWA2MBzYAn42IvUXun1khej9A11849E73dVdTbxDMng0f/WjtkPBdTZaDwgJFUgtwI3Ah0AWsl9QREZvKui0GXo6IGZIWAtcBV0qaBSwEZgOnAg9IOgM4BfgyMCsi3pS0JvW7Na17Q0SslvRPads3FbV/ZgOyZ0/tu5oqX/v6AF1b24EgmDmzejj03tU0YsTg76cNa0WeocwFOiNiC4Ck1cACoDxQFgDfStNrgR9IUmpfHRF7gGckdabt/SHVPErS28Bo4N/TOh8DPp22tSpt14FixYiA116rLyB27Mi+Qrya0aMPhMA735l9gK5WSIwb5w/Q2ZBWZKBMBraWzXcBf1KrT0T0SNpFdslqMvCrinUnR8TDkq4nC5Y3gfsi4j5JE4BXIqKnvH+1oiQtAZYAnHbaaUewe3bU6emBl17qPxx6p/fsqb6d8u9qmjOndkBMnAhjxgzuPpoVqKkG5SWdRHb2Mh14BfiZpM8A99a7jYhYAawAaG9vr/Id0HZU2b27/gHrl16q/rXgxx57cAjMnl07ICZMOKq/q8msL0UGyjZgatn8lNRWrU+XpFZgLNngfK11Pw48ExHdAJJ+AXwQuB04UVJrOkup9l52NKg2YN3XWUStz0a84x0HguDMM+GCCw4efygPiSHwteBmzaDIQFkPzEx3X20jGzz/dEWfDmAR8DBwBfBQRISkDuAOSd8jG5SfCTwK7Ac+IGk02SWveUAprfPPaRur0zbvKXDfLE9vvVV9wLpaUNQasO79bERvEJx++qGD1OXT/myEWe4KC5Q0JrIUWEd22/DKiHhK0jVkIdAB3ALclgbdd5KFDqnfGrIB/B7g6ojYBzwiaS3wWGr/NenyFfA3wGpJ/y2131LUvlk/yp9jXRkM1aZffbX6dioHrOfOrX2pyQPWZg2nqHbNeJhob2+PUqnU6DKaQ+VzI/oKib6eY10+YF3tzMED1mZDnqQNEdFe2d5Ug/KWo4jssaP1hMP27bVve+39Mr+TT4ZTTz34rqbKsJgwAVr9T87saOX/3UeTyu9pqhUOvdN7a3yRwLhxB4Lg7LMPDoXKkPCX+ZlZ4kAZyiKy8YV6wqGvp8+NGHHwpaSzz659ucm3vZrZYXKgDLbes4hawVA5X+vDc+PGHQiCs86CefOqh8TJJ2e3yPoswswK5kA5UpVjEbVCop6ziPIweM97al9mmjDB39NkZkOOA+VwfPe7cMcd9Y1F9IbBe95z4Cyi8gzCZxFmdhRwoByOY4+FU045MBZRbdDaYxFmNsz4cyj+HIqZ2YDU+hyKP1psZma5cKCYmVkuHChmZpYLB4qZmeXCgWJmZrlwoJiZWS4cKGZmlgsHipmZ5cKBYmZmuSg0UCTNl7RZUqekZVWWj5R0V1r+iKRpZcuWp/bNki5ObWdK2lj286qkr6Zl35K0rWzZpUXum5mZHayw7/KS1ALcCFwIdAHrJXVExKaybouBlyNihqSFwHXAlZJmkT1ffjZwKvCApDMiYjMwp2z724C7y7Z3Q0RcX9Q+mZlZbUWeocwFOiNiS0TsBVYDCyr6LABWpem1wDxJSu2rI2JPRDwDdKbtlZsH/D4initsD8zMrG5FBspkYGvZfFdqq9onInqAXcD4OtddCNxZ0bZU0uOSVko6qVpRkpZIKkkqdXd3D2R/zMysD005KC9pBHA58LOy5puA08kuiT0PfLfauhGxIiLaI6K9ra2t8FrNzIaLIgNlGzC1bH5KaqvaR1IrMBZ4qY51LwEei4jtvQ0RsT0i9kXEfuBHHHqJzMzMClRkoKwHZkqans4oFgIdFX06gEVp+grgocge0NIBLEx3gU0HZgKPlq13FRWXuyRNKpv9JPBkbntiZmb9Kuwur4jokbQUWAe0ACsj4ilJ1wCliOgAbgFuk9QJ7CQLHVK/NcAmoAe4OiL2AUgaQ3bn2Bcr3vLbkuYAATxbZbmZmRXIT2z0ExvNzAbET2w0M7NCOVDMzCwXDhQzM8uFA8XMzHLhQDEzs1w4UMzMLBcOFDMzy4UDxczMcuFAMTOzXDhQzMwsFw4UMzPLhQPFzMxy4UAxM7NcOFDMzCwXDhQzM8uFA8XMzHJRaKBImi9ps6ROScuqLB8p6a60/BFJ08qWLU/tmyVdnNrOlLSx7OdVSV9Ny8ZJul/S79LrSUXum5mZHaywQJHUAtwIXALMAq6SNKui22Lg5YiYAdwAXJfWnUX2OODZwHzgh5JaImJzRMyJiDnA+4DdwN1pW8uAByNiJvBgmjczs0FS5BnKXKAzIrZExF5gNbCgos8CYFWaXgvMk6TUvjoi9kTEM0Bn2l65ecDvI+K5KttaBXwi170xM7M+FRkok4GtZfNdqa1qn4joAXYB4+tcdyFwZ9n8xIh4Pk2/AEysVpSkJZJKkkrd3d31742ZmfWpKQflJY0ALgd+Vm15RAQQNZatiIj2iGhva2srsEozs+GlyEDZBkwtm5+S2qr2kdQKjAVeqmPdS4DHImJ7Wdt2SZPStiYBO3LYBzMzq1ORgbIemClpejqjWAh0VPTpABal6SuAh9LZRQewMN0FNh2YCTxatt5VHHy5q3Jbi4B7ctsTMzPrV2tRG46IHklLgXVAC7AyIp6SdA1QiogO4BbgNkmdwE6y0CH1WwNsAnqAqyNiH4CkMcCFwBcr3vJaYI2kxcBzwKeK2jczMzuUshOC4am9vT1KpVKjyzAzayqSNkREe2V7Uw7Km5nZ0ONAMTOzXDhQzMwsFw4UMzPLhQPFzMxy4UAxM7NcOFDMzCwXDhQzM8uFA8XMzHLhQDEzs1w4UMzMLBcOFDMzy4UDxczMcuFAMTOzXDhQzMwsFw4UMzPLhQPFzMxyUWigSJovabOkTknLqiwfKemutPwRSdPKli1P7ZslXVzWfqKktZJ+K+lpSeel9m9J2iZpY/q5tMh9MzOzgxX2THlJLcCNZM9/7wLWS+qIiE1l3RYDL0fEDEkLgeuAKyXNInu+/GzgVOABSWek58p/H7g3Iq6QNAIYXba9GyLi+qL2yczMaivyDGUu0BkRWyJiL7AaWFDRZwGwKk2vBeZJUmpfHRF7IuIZoBOYK2ks8GHgFoCI2BsRrxS4D2ZmVqciA2UysLVsviu1Ve0TET3ALmB8H+tOB7qBH0v6taSbJY0p67dU0uOSVko6qVpRkpZIKkkqdXd3H8HumZlZuboCRdJXJL1DmVskPSbpoqKLq6IVOBe4KSLOAd4AesdmbgJOB+YAzwPfrbaBiFgREe0R0d7W1jYIJZuZDQ/1nqF8PiJeBS4CTgI+C1zbzzrbgKll81NSW9U+klqBscBLfazbBXRFxCOpfS1ZwBAR2yNiX0TsB35EdsnNzMwGSb2BovR6KXBbRDxV1lbLemCmpOlp8Hwh0FHRpwNYlKavAB6KiEjtC9NdYNOBmcCjEfECsFXSmWmdecAmAEmTyrb7SeDJOvfNzMxyUO9dXhsk3Uc2hrFc0gnA/r5WiIgeSUuBdUALsDIinpJ0DVCKiA6ywfXbJHUCO8lCh9RvDVlY9ABXpzu8AL4E3J5CagvwudT+bUlzgACeBb5Y576ZmVkOlJ0Q9NNJOoZsbGJLRLwiaRwwJSIeL7rAIrW3t0epVGp0GWZmTUXShohor2yv95LXecDmFCafAf4r2R1ZZmZmQP2BchOwW9J7ga8Dvwd+UlhVZmbWdOoNlJ40WL4A+EFE3AicUFxZZmbWbOodlH9N0nKy24UvSGMqxxZXlpmZNZt6z1CuBPaQfR7lBbLPhXynsKrMzKzp1BUoKURuB8ZKugx4KyI8hmJmZn9U71evfAp4FPhPwKeARyRdUWRhZmbWXOodQ/k74P0RsQNAUhvwANlXn5iZmdU9hnJMb5gkLw1gXTMzGwbqPUO5V9I64M40fyXwy2JKMjOzZlRXoETEX0v6j8D5qWlFRNxdXFlmZtZs6n4EcET8HPh5gbWYmVkT6zNQJL1G9u29hywCIiLeUUhVZmbWdPoMlIjw16uYmVldfKeWmZnlwoFiZma5KDRQJM2XtFlSp6RlVZaPlHRXWv6IpGlly5an9s2SLi5rP1HSWkm/lfS0pPNS+zhJ90v6XXo9qch9MzOzgxUWKJJagBuBS4BZwFWSZlV0Wwy8HBEzgBuA69K6s8geBzwbmA/8MG0P4PvAvRHxbuC9wNOpfRnwYETMBB5M82ZmNkiKPEOZC3RGxJaI2AusJnueSrkFwKo0vRaYJ0mpfXVE7ImIZ4BOYK6kscCHyZ5FT0TsjYhXqmxrFfCJgvbLzMyqKDJQJgNby+a7UlvVPhHRQ/ZY4fF9rDsd6AZ+LOnXkm6WNCb1mRgRz6fpF4CJ1YqStERSSVKpu7v7sHfOzMwO1myD8q3AucBNEXEO8AZVLm2lp0tW+/wMEbEiItojor2tra3QYs3MhpMiA2UbMLVsfkpqq9pHUiswluyLJ2ut2wV0RcQjqX0tWcAAbJc0KW1rElD+ZZZmZlawIgNlPTBT0nRJI8gG2Tsq+nQAi9L0FcBD6eyiA1iY7gKbDswEHk0P+toq6cy0zjxgU5VtLQLuKWKnzMysurq/y2ugIqJH0lJgHdACrIyIpyRdA5QiooNscP02SZ3ATrLQIfVbQxYWPcDVEbEvbfpLwO0ppLYAn0vt1wJrJC0GniN7EJiZmQ0SZScEw1N7e3uUSqVGl2Fm1lQkbYiI9sr2ZhuUNzOzIcqBYmZmuXCgmJlZLhwoZmaWCweKmZnlwoFiZma5cKCYmVkuHChmZpYLB4qZmeXCgWJmZrlwoJiZWS4cKGZmlgsHipmZ5cKBYmZmuXCgmJlZLhwoZmaWi0IDRdJ8SZsldUpaVmX5SEl3peWPSJpWtmx5at8s6eKy9mclPSFpo6RSWfu3JG1L7RslXVrkvpmZ2cEKewSwpBbgRuBCoAtYL6kjIjaVdVsMvBwRMyQtBK4DrpQ0i+xxwLOBU4EHJJ1R9hjgj0bEi1Xe9oaIuL6ofTIzs9qKPEOZC3RGxJaI2AusBhZU9FkArErTa4F5kpTaV0fEnoh4BuhM2zMzsyGqyECZDGwtm+9KbVX7REQPsAsY38+6AdwnaYOkJRXbWyrpcUkrJZ2Uz26YmVk9mnFQ/kMRcS5wCXC1pA+n9puA04E5wPPAd6utLGmJpJKkUnd396AUbGY2HBQZKNuAqWXzU1Jb1T6SWoGxwEt9rRsRva87gLtJl8IiYntE7IuI/cCPqHGJLCJWRER7RLS3tbUd0Q6amdkBRQbKemCmpOmSRpANsndU9OkAFqXpK4CHIiJS+8J0F9h0YCbwqKQxkk4AkDQGuAh4Ms1PKtvuJ3vbzcxscBR2l1dE9EhaCqwDWoCVEfGUpGuAUkR0ALcAt0nqBHaShQ6p3xpgE9ADXB0R+yRNBO7Oxu1pBe6IiHvTW35b0hyyMZZngS8WtW9mZnYoZScEw1N7e3uUSqX+O5qZ2R9J2hAR7ZXtzTgob2ZmQ5ADxczMcuFAMTOzXDhQzMwsFw4UMzPLhQPFzMxy4UAxM7NcOFDMzCwXDhQzM8uFA8XMzHLhQDEzs1w4UMzMLBcOFDMzy4UDxczMcuFAMTOzXDhQzMwsFw4UMzPLRaGBImm+pM2SOiUtq7J8pKS70vJHJE0rW7Y8tW+WdHFZ+7OSnpC0UVKprH2cpPsl/S69nlTkvpmZ2cEKCxRJLcCNwCXALOAqSbMqui0GXo6IGcANwHVp3Vlkz5efDcwHfpi21+ujETGn4hGUy4AHI2Im8GCaNzOzQVLkGcpcoDMitkTEXmA1sKCizwJgVZpeC8yTpNS+OiL2RMQzQGfaXl/Kt7UK+EQO+2BmZnUqMlAmA1vL5rtSW9U+EdED7ALG97NuAPdJ2iBpSVmfiRHxfJp+AZhYrShJSySVJJW6u7sHvldmZlZVMw7KfygiziW7lHa1pA9XdoiIIAueQ0TEiohoj4j2tra2gks1Mxs+igyUbcDUsvkpqa1qH0mtwFjgpb7WjYje1x3A3Ry4FLZd0qS0rUnAjhz3xczM+lFkoKwHZkqaLmkE2SB7R0WfDmBRmr4CeCidXXQAC9NdYNOBmcCjksZIOgFA0hjgIuDJKttaBNxT0H6ZmVkVrUVtOCJ6JC0F1gEtwMqIeErSNUApIjqAW4DbJHUCO8lCh9RvDbAJ6AGujoh9kiYCd2fj9rQCd0TEvektrwXWSFoMPAd8qqh9MzOzQyk7IRie2tvbo1Qq9d/RzMz+SNKGio9tAM05KG9mZkOQA8XMzHLhQDEzs1w4UMzMLBcOFDMzy4UDxczMcuFAMTOzXDhQzMwsFw4UMzPLhQPFzMxyUdh3eZmZWWNEwJtvwhtvwO7d2Wvl9AUXwCmn5Pu+DhQzs0EWAW+91fcBv9Z0f/O7d2c//X1N4y9/CZdcku9+OVDMzCqUH/BrHdwHGgSV0wP9Xt7jjoPRo2HMmOynd3r8eJg69dD2/qanT8//9+ZAMbOms3//kR/w+1p+OAf8kSMPHLDLD+DjxsGUKQM/4Fe+trQU87vMkwPFzHK3f/+hB+6+DuqHc8AfqLz/wi+fbpYDftEcKGbD0Ntv1z6o1/va17K33hp4Tb0H5sqDdltb/3/B1/qrvnfZqFE+4A+GQgNF0nzg+2RPbLw5Iq6tWD4S+AnwPrJnyV8ZEc+mZcuBxcA+4MsRsa5svRagBGyLiMtS263AfwB2pW5/EREbC9s5s4JEwJ49+RzYa72+/fbAapJqH8AnTarvoN5X23HHwTH+EEPTKyxQ0kH/RuBCoAtYL6kjIjaVdVsMvBwRMyQtBK4DrpQ0i+xxwLOBU4EHJJ0REfvSel8BngbeUfG2fx0Ra4vaJzOAffuyWzKP9C/8vl737x9YTa2ttQ/oEyYM7EBfbdnIkVmomPWlyDOUuUBnRGwBkLQaWED2nPheC4Bvpem1wA+UPTB+AbA6IvYAz6Rnzs8FHpY0BfhT4B+AvyqwfmtCEbB3b30H7cPtcziXc0aNqn7gLh+wLW+vdo2+r4P/scfm/7s0G6giA2UysLVsvgv4k1p9IqJH0i5gfGr/VcW6k9P0PwLfBE6o8p7/IOnvgQeBZSmQbAjp/es+74N8+bJ9+/qvo9wxx9Q+WJ96at8H83r+0h81ypdzbHhoqkF5SZcBOyJig6SPVCxeDrwAjABWAH8DXFNlG0uAJQCnnXZaofU2m95r99UO3AM5oPcVCHsOI+J7786pPFCPHXvoAb+/v+SrvY4Y4cs5ZnkoMlC2AVPL5qektmp9uiS1AmPJBudrrXs5cLmkS4HjgHdI+mlEfCYink9990j6MfCNakVFxAqywKG9vX2Ad5o3Vu+lnHoO9P0d5GtND/Te+5aW2gfqE0+s/Rd9rYN/5V/9vjvHrHkUGSjrgZmSppOFwULg0xV9OoBFwMPAFcBDERGSOoA7JH2PbFB+JvBoRDxMdiZCOkP5RkR8Js1Piojn0xjMJ4AnC9y3Q/T0HDg413uQH2gA9PQMvK5aB+7x4+G00+o7wPfVNmJE/r9LM2tOhQVKGhNZCqwju214ZUQ8JekaoBQRHcAtwG1p0H0nWeiQ+q0hG8DvAa4uu8OrltsltQECNgJ/WciOAddcAz/96cEH/L17B76dyks5o0ZlryeckH1pWz1/2fc1PWqUL+WY2eBRDPQax1Gkvb09SqXSgNe7+WZ48MEjO+D7Uo6ZNStJGyKivbK9qQblh4ovfCH7MTOzA3wzo5mZ5cKBYmZmuXCgmJlZLhwoZmaWCweKmZnlwoFiZma5cKCYmVkuHChmZpaLYf1JeUndwHOHufoE4MUcyylaM9XbTLVCc9XbTLVCc9XbTLXCkdX7zohoq2wc1oFyJCSVqn31wFDVTPU2U63QXPU2U63QXPU2U61QTL2+5GVmZrlwoJiZWS4cKIdvRaMLGKBmqreZaoXmqreZaoXmqreZaoUC6vUYipmZ5cJnKGZmlgsHipmZ5cKBchgkzZe0WVKnpGWNrqcWSVMl/bOkTZKekvSVRtfUH0ktkn4t6X82upb+SDpR0lpJv5X0tKTzGl1TXyR9Lf07eFLSnZKOa3RNvSStlLRD0pNlbeMk3S/pd+n1pEbWWK5Gvd9J/xYel3S3pBMbWWOvarWWLfu6pJA0IY/3cqAMkKQW4EbgEmAWcJWkWY2tqqYe4OsRMQv4AHD1EK6111eApxtdRJ2+D9wbEe8G3ssQrlvSZODLQHtEnAW0AAsbW9VBbgXmV7QtAx6MiJnAg2l+qLiVQ+u9HzgrIs4G/h+wfLCLquFWDq0VSVOBi4A/5PVGDpSBmwt0RsSWiNgLrAYWNLimqiLi+Yh4LE2/RnbAm9zYqmqTNAX4U+DmRtfSH0ljgQ8DtwBExN6IeKWxVfWrFRglqRUYDfx7g+v5o4j4F2BnRfMCYFWaXgV8YlCL6kO1eiPivojoSbO/AqYMemFV1PjdAtwAfBPI7c4sB8rATQa2ls13MYQP0r0kTQPOAR5pbCV9+keyf+D7G11IHaYD3cCP0yW6myWNaXRRtUTENuB6sr9Gnwd2RcR9ja2qXxMj4vk0/QIwsZHFDNDngf/V6CJqkbQA2BYRv8lzuw6UYUDS8cDPga9GxKuNrqcaSZcBOyJiQ6NrqVMrcC5wU0ScA7zB0Lokc5A0/rCALAhPBcZI+kxjq6pfZJ9vaIrPOEj6O7LLzbc3upZqJI0G/hb4+7y37UAZuG3A1LL5KaltSJJ0LFmY3B4Rv2h0PX04H7hc0rNklxE/JumnjS2pT11AV0T0nvGtJQuYoerjwDMR0R0RbwO/AD7Y4Jr6s13SJID0uqPB9fRL0l8AlwF/HkP3Q36nk/1h8Zv0/20K8JikU450ww6UgVsPzJQ0XdIIsoHNjgbXVJUkkV3jfzoivtfoevoSEcsjYkpETCP7nT4UEUP2L+iIeAHYKunM1DQP2NTAkvrzB+ADkkanfxfzGMI3ESQdwKI0vQi4p4G19EvSfLJLtpdHxO5G11NLRDwRESdHxLT0/60LODf9mz4iDpQBSoNuS4F1ZP8h10TEU42tqqbzgc+S/bW/Mf1c2uiijiJfAm6X9DgwB/jvDa6npnQmtRZ4DHiC7P/+kPmqEEl3Ag8DZ0rqkrQYuBa4UNLvyM6wrm1kjeVq1PsD4ATg/vR/7Z8aWmRSo9Zi3mvonpWZmVkz8RmKmZnlwoFiZma5cKCYmVkuHChmZpYLB4qZmeXCgWJ2BCT93/Q6TdKnc97231Z7L7OhyrcNm+VA0keAb0TEZQNYp7XsywSrLX89Io7Poz6zweAzFLMjIOn1NHktcEH6QNvX0nNdviNpfXo+xhdT/49I+ldJHaRP1kv6H5I2pGeVLElt15J9M/BGSbeXv5cy30nPNXlC0pVl2/7fZc9ouT19Kh5J1yp7Ls7jkq4fzN+RDR+tjS7A7CixjLIzlBQMuyLi/ZJGAv8mqffbfc8le27GM2n+8xGxU9IoYL2kn0fEMklLI2JOlff6M7JP5r8XmJDW+Ze07BxgNtlX0/8bcL6kp4FPAu+OiBgqD36yo4/PUMyKcRHwnyVtJHtkwHhgZlr2aFmYAHxZ0m/InqExtaxfLR8C7oyIfRGxHfg/wPvLtt0VEfuBjcA0YBfwFnCLpD8Dhuz3TFlzc6CYFUPAlyJiTvqZXvb8kTf+2Ckbe/k4cF5EvBf4NXAkj+bdUza9D+gdp5lL9l1elwH3HsH2zWpyoJjl4zWyLwbstQ74L+nxAUg6o8YDuMYCL0fEbknvJntUc6+3e9ev8K/AlWmcpo3syZGP1iosPQ9nbET8Evga2aUys9x5DMUsH48D+9Klq1vJnjc/jew5EyJ7umO1R9jeC/xlGufYTHbZq9cK4HFJj0XEn5e13w2cB/yG7KFT34yIF1IgVXMCcI+k48jOnAjCHjcAAAA8SURBVP7q8HbRrG++bdjMzHLhS15mZpYLB4qZmeXCgWJmZrlwoJiZWS4cKGZmlgsHipmZ5cKBYmZmufj//rvD/YdEfDEAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "t3U7TGl2icNm",
        "colab_type": "text"
      },
      "source": [
        "### **1. Plot the loss curve**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_TpBzAaCiVhd",
        "colab_type": "code",
        "outputId": "c0164729-027b-487b-a0c3-01f4fb51fd47",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 279
        }
      },
      "source": [
        "fig = plt.figure()\n",
        "graph1 = plt.plot(iters, train_costs, 'b')\n",
        "graph2 = plt.plot(iters, test_costs, 'r')\n",
        "plt.xlabel('iterations')\n",
        "plt.ylabel('loss')\n",
        "plt.show()"
      ],
      "execution_count": 136,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZQAAAEGCAYAAABCa2PoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3df5BV5Z3n8ffHbkFAgwItImAggqbAGDQdJsaYTUJUdCxJZt2ImWSZhBSZLcmvSSYDM1tTKWtnSxMTJ1UxThElEqMiIXHt2sqKv2Z3pmaNcjHEHxg2HdTQjEIrij9QsOG7f5ynw+Vyb/dtOKdvX/rzquq65zznOed+TxecT5/znHuPIgIzM7MjdUyjCzAzs6ODA8XMzHLhQDEzs1w4UMzMLBcOFDMzy0VrowtopAkTJsS0adMaXYaZWVPZsGHDixHRVtk+rANl2rRplEqlRpdhZtZUJD1Xrd2XvMzMLBcOFDMzy4UDxczMcuFAMTOzXDhQzMwsFw4UMzPLhQPFzMxyMaw/h2JmdlSLgFdege3bs58dOw5Mf+5z8K535fp2DhQzs2aybx+8+GL1kCif7p1/++1DtyHBBz/oQDEzO+rs2VM7GCpD4sUXszOPSiNGwMknw8SJcMopcPbZ2fTEiQfae6cnTICWltx3w4FiZpa3CHj99f4Dond6167q2zn++AMhMGMGnH/+oeHQOz12bHbm0UAOFDOzeuzfDzt31hcQO3bAm29W38748QeC4JxzagfEySfD6NGDu49HyIFiZsPX3r3Q3V1fSHR3Z+MXlVpaDg6CM8+sHRBtbXDssYO/n4PEgWJmR5fXXz/4TKGvkHj55erbGDXqQAicdhq8//21zyROOgmO8ScwwIFiZkPd/v3Zgb/WpaXKtt27q2/nxBMPBMFZZ8HHPlb7TOL44xs+HtGMHChmNvjefvvQS021AmPHDujpOXQbxxyTXULqDYMZM6qfRZx8cvYzcuTg7+cw40Axs3y88UZ9AbF9eza4Xc3IkQcCYfJkOPfc2peaxo0r5NZXO3wOFDOrrvJSU39B8cYb1bczduyBMJg1Cz7ykeqXmSZOhBNO8KWmJuZAMRtOql1qqhYWA7nU9K531Q6ItjY47rjB309rCAeKWbOrdldTrddal5qOO+5AEEyZkl1qqjYW4UtN1odCA0XSfOD7QAtwc0RcW7F8JPAT4H3AS8CVEfFsWrYcWAzsA74cEetS+9eALwABPAF8LiLekjQdWA2MBzYAn42IvUXun1khej9A11849E73dVdTbxDMng0f/WjtkPBdTZaDwgJFUgtwI3Ah0AWsl9QREZvKui0GXo6IGZIWAtcBV0qaBSwEZgOnAg9IOgM4BfgyMCsi3pS0JvW7Na17Q0SslvRPads3FbV/ZgOyZ0/tu5oqX/v6AF1b24EgmDmzejj03tU0YsTg76cNa0WeocwFOiNiC4Ck1cACoDxQFgDfStNrgR9IUmpfHRF7gGckdabt/SHVPErS28Bo4N/TOh8DPp22tSpt14FixYiA116rLyB27Mi+Qrya0aMPhMA735l9gK5WSIwb5w/Q2ZBWZKBMBraWzXcBf1KrT0T0SNpFdslqMvCrinUnR8TDkq4nC5Y3gfsi4j5JE4BXIqKnvH+1oiQtAZYAnHbaaUewe3bU6emBl17qPxx6p/fsqb6d8u9qmjOndkBMnAhjxgzuPpoVqKkG5SWdRHb2Mh14BfiZpM8A99a7jYhYAawAaG9vr/Id0HZU2b27/gHrl16q/rXgxx57cAjMnl07ICZMOKq/q8msL0UGyjZgatn8lNRWrU+XpFZgLNngfK11Pw48ExHdAJJ+AXwQuB04UVJrOkup9l52NKg2YN3XWUStz0a84x0HguDMM+GCCw4efygPiSHwteBmzaDIQFkPzEx3X20jGzz/dEWfDmAR8DBwBfBQRISkDuAOSd8jG5SfCTwK7Ac+IGk02SWveUAprfPPaRur0zbvKXDfLE9vvVV9wLpaUNQasO79bERvEJx++qGD1OXT/myEWe4KC5Q0JrIUWEd22/DKiHhK0jVkIdAB3ALclgbdd5KFDqnfGrIB/B7g6ojYBzwiaS3wWGr/NenyFfA3wGpJ/y2131LUvlk/yp9jXRkM1aZffbX6dioHrOfOrX2pyQPWZg2nqHbNeJhob2+PUqnU6DKaQ+VzI/oKib6eY10+YF3tzMED1mZDnqQNEdFe2d5Ug/KWo4jssaP1hMP27bVve+39Mr+TT4ZTTz34rqbKsJgwAVr9T87saOX/3UeTyu9pqhUOvdN7a3yRwLhxB4Lg7LMPDoXKkPCX+ZlZ4kAZyiKy8YV6wqGvp8+NGHHwpaSzz659ucm3vZrZYXKgDLbes4hawVA5X+vDc+PGHQiCs86CefOqh8TJJ2e3yPoswswK5kA5UpVjEbVCop6ziPIweM97al9mmjDB39NkZkOOA+VwfPe7cMcd9Y1F9IbBe95z4Cyi8gzCZxFmdhRwoByOY4+FU045MBZRbdDaYxFmNsz4cyj+HIqZ2YDU+hyKP1psZma5cKCYmVkuHChmZpYLB4qZmeXCgWJmZrlwoJiZWS4cKGZmlgsHipmZ5cKBYmZmuSg0UCTNl7RZUqekZVWWj5R0V1r+iKRpZcuWp/bNki5ObWdK2lj286qkr6Zl35K0rWzZpUXum5mZHayw7/KS1ALcCFwIdAHrJXVExKaybouBlyNihqSFwHXAlZJmkT1ffjZwKvCApDMiYjMwp2z724C7y7Z3Q0RcX9Q+mZlZbUWeocwFOiNiS0TsBVYDCyr6LABWpem1wDxJSu2rI2JPRDwDdKbtlZsH/D4initsD8zMrG5FBspkYGvZfFdqq9onInqAXcD4OtddCNxZ0bZU0uOSVko6qVpRkpZIKkkqdXd3D2R/zMysD005KC9pBHA58LOy5puA08kuiT0PfLfauhGxIiLaI6K9ra2t8FrNzIaLIgNlGzC1bH5KaqvaR1IrMBZ4qY51LwEei4jtvQ0RsT0i9kXEfuBHHHqJzMzMClRkoKwHZkqans4oFgIdFX06gEVp+grgocge0NIBLEx3gU0HZgKPlq13FRWXuyRNKpv9JPBkbntiZmb9Kuwur4jokbQUWAe0ACsj4ilJ1wCliOgAbgFuk9QJ7CQLHVK/NcAmoAe4OiL2AUgaQ3bn2Bcr3vLbkuYAATxbZbmZmRXIT2z0ExvNzAbET2w0M7NCOVDMzCwXDhQzM8uFA8XMzHLhQDEzs1w4UMzMLBcOFDMzy4UDxczMcuFAMTOzXDhQzMwsFw4UMzPLhQPFzMxy4UAxM7NcOFDMzCwXDhQzM8uFA8XMzHJRaKBImi9ps6ROScuqLB8p6a60/BFJ08qWLU/tmyVdnNrOlLSx7OdVSV9Ny8ZJul/S79LrSUXum5mZHaywQJHUAtwIXALMAq6SNKui22Lg5YiYAdwAXJfWnUX2OODZwHzgh5JaImJzRMyJiDnA+4DdwN1pW8uAByNiJvBgmjczs0FS5BnKXKAzIrZExF5gNbCgos8CYFWaXgvMk6TUvjoi9kTEM0Bn2l65ecDvI+K5KttaBXwi170xM7M+FRkok4GtZfNdqa1qn4joAXYB4+tcdyFwZ9n8xIh4Pk2/AEysVpSkJZJKkkrd3d31742ZmfWpKQflJY0ALgd+Vm15RAQQNZatiIj2iGhva2srsEozs+GlyEDZBkwtm5+S2qr2kdQKjAVeqmPdS4DHImJ7Wdt2SZPStiYBO3LYBzMzq1ORgbIemClpejqjWAh0VPTpABal6SuAh9LZRQewMN0FNh2YCTxatt5VHHy5q3Jbi4B7ctsTMzPrV2tRG46IHklLgXVAC7AyIp6SdA1QiogO4BbgNkmdwE6y0CH1WwNsAnqAqyNiH4CkMcCFwBcr3vJaYI2kxcBzwKeK2jczMzuUshOC4am9vT1KpVKjyzAzayqSNkREe2V7Uw7Km5nZ0ONAMTOzXDhQzMwsFw4UMzPLhQPFzMxy4UAxM7NcOFDMzCwXDhQzM8uFA8XMzHLhQDEzs1w4UMzMLBcOFDMzy4UDxczMcuFAMTOzXDhQzMwsFw4UMzPLhQPFzMxyUWigSJovabOkTknLqiwfKemutPwRSdPKli1P7ZslXVzWfqKktZJ+K+lpSeel9m9J2iZpY/q5tMh9MzOzgxX2THlJLcCNZM9/7wLWS+qIiE1l3RYDL0fEDEkLgeuAKyXNInu+/GzgVOABSWek58p/H7g3Iq6QNAIYXba9GyLi+qL2yczMaivyDGUu0BkRWyJiL7AaWFDRZwGwKk2vBeZJUmpfHRF7IuIZoBOYK2ks8GHgFoCI2BsRrxS4D2ZmVqciA2UysLVsviu1Ve0TET3ALmB8H+tOB7qBH0v6taSbJY0p67dU0uOSVko6qVpRkpZIKkkqdXd3H8HumZlZuboCRdJXJL1DmVskPSbpoqKLq6IVOBe4KSLOAd4AesdmbgJOB+YAzwPfrbaBiFgREe0R0d7W1jYIJZuZDQ/1nqF8PiJeBS4CTgI+C1zbzzrbgKll81NSW9U+klqBscBLfazbBXRFxCOpfS1ZwBAR2yNiX0TsB35EdsnNzMwGSb2BovR6KXBbRDxV1lbLemCmpOlp8Hwh0FHRpwNYlKavAB6KiEjtC9NdYNOBmcCjEfECsFXSmWmdecAmAEmTyrb7SeDJOvfNzMxyUO9dXhsk3Uc2hrFc0gnA/r5WiIgeSUuBdUALsDIinpJ0DVCKiA6ywfXbJHUCO8lCh9RvDVlY9ABXpzu8AL4E3J5CagvwudT+bUlzgACeBb5Y576ZmVkOlJ0Q9NNJOoZsbGJLRLwiaRwwJSIeL7rAIrW3t0epVGp0GWZmTUXShohor2yv95LXecDmFCafAf4r2R1ZZmZmQP2BchOwW9J7ga8Dvwd+UlhVZmbWdOoNlJ40WL4A+EFE3AicUFxZZmbWbOodlH9N0nKy24UvSGMqxxZXlpmZNZt6z1CuBPaQfR7lBbLPhXynsKrMzKzp1BUoKURuB8ZKugx4KyI8hmJmZn9U71evfAp4FPhPwKeARyRdUWRhZmbWXOodQ/k74P0RsQNAUhvwANlXn5iZmdU9hnJMb5gkLw1gXTMzGwbqPUO5V9I64M40fyXwy2JKMjOzZlRXoETEX0v6j8D5qWlFRNxdXFlmZtZs6n4EcET8HPh5gbWYmVkT6zNQJL1G9u29hywCIiLeUUhVZmbWdPoMlIjw16uYmVldfKeWmZnlwoFiZma5KDRQJM2XtFlSp6RlVZaPlHRXWv6IpGlly5an9s2SLi5rP1HSWkm/lfS0pPNS+zhJ90v6XXo9qch9MzOzgxUWKJJagBuBS4BZwFWSZlV0Wwy8HBEzgBuA69K6s8geBzwbmA/8MG0P4PvAvRHxbuC9wNOpfRnwYETMBB5M82ZmNkiKPEOZC3RGxJaI2AusJnueSrkFwKo0vRaYJ0mpfXVE7ImIZ4BOYK6kscCHyZ5FT0TsjYhXqmxrFfCJgvbLzMyqKDJQJgNby+a7UlvVPhHRQ/ZY4fF9rDsd6AZ+LOnXkm6WNCb1mRgRz6fpF4CJ1YqStERSSVKpu7v7sHfOzMwO1myD8q3AucBNEXEO8AZVLm2lp0tW+/wMEbEiItojor2tra3QYs3MhpMiA2UbMLVsfkpqq9pHUiswluyLJ2ut2wV0RcQjqX0tWcAAbJc0KW1rElD+ZZZmZlawIgNlPTBT0nRJI8gG2Tsq+nQAi9L0FcBD6eyiA1iY7gKbDswEHk0P+toq6cy0zjxgU5VtLQLuKWKnzMysurq/y2ugIqJH0lJgHdACrIyIpyRdA5QiooNscP02SZ3ATrLQIfVbQxYWPcDVEbEvbfpLwO0ppLYAn0vt1wJrJC0GniN7EJiZmQ0SZScEw1N7e3uUSqVGl2Fm1lQkbYiI9sr2ZhuUNzOzIcqBYmZmuXCgmJlZLhwoZmaWCweKmZnlwoFiZma5cKCYmVkuHChmZpYLB4qZmeXCgWJmZrlwoJiZWS4cKGZmlgsHipmZ5cKBYmZmuXCgmJlZLhwoZmaWi0IDRdJ8SZsldUpaVmX5SEl3peWPSJpWtmx5at8s6eKy9mclPSFpo6RSWfu3JG1L7RslXVrkvpmZ2cEKewSwpBbgRuBCoAtYL6kjIjaVdVsMvBwRMyQtBK4DrpQ0i+xxwLOBU4EHJJ1R9hjgj0bEi1Xe9oaIuL6ofTIzs9qKPEOZC3RGxJaI2AusBhZU9FkArErTa4F5kpTaV0fEnoh4BuhM2zMzsyGqyECZDGwtm+9KbVX7REQPsAsY38+6AdwnaYOkJRXbWyrpcUkrJZ2Uz26YmVk9mnFQ/kMRcS5wCXC1pA+n9puA04E5wPPAd6utLGmJpJKkUnd396AUbGY2HBQZKNuAqWXzU1Jb1T6SWoGxwEt9rRsRva87gLtJl8IiYntE7IuI/cCPqHGJLCJWRER7RLS3tbUd0Q6amdkBRQbKemCmpOmSRpANsndU9OkAFqXpK4CHIiJS+8J0F9h0YCbwqKQxkk4AkDQGuAh4Ms1PKtvuJ3vbzcxscBR2l1dE9EhaCqwDWoCVEfGUpGuAUkR0ALcAt0nqBHaShQ6p3xpgE9ADXB0R+yRNBO7Oxu1pBe6IiHvTW35b0hyyMZZngS8WtW9mZnYoZScEw1N7e3uUSqX+O5qZ2R9J2hAR7ZXtzTgob2ZmQ5ADxczMcuFAMTOzXDhQzMwsFw4UMzPLhQPFzMxy4UAxM7NcOFDMzCwXDhQzM8uFA8XMzHLhQDEzs1w4UMzMLBcOFDMzy4UDxczMcuFAMTOzXDhQzMwsFw4UMzPLRaGBImm+pM2SOiUtq7J8pKS70vJHJE0rW7Y8tW+WdHFZ+7OSnpC0UVKprH2cpPsl/S69nlTkvpmZ2cEKCxRJLcCNwCXALOAqSbMqui0GXo6IGcANwHVp3Vlkz5efDcwHfpi21+ujETGn4hGUy4AHI2Im8GCaNzOzQVLkGcpcoDMitkTEXmA1sKCizwJgVZpeC8yTpNS+OiL2RMQzQGfaXl/Kt7UK+EQO+2BmZnUqMlAmA1vL5rtSW9U+EdED7ALG97NuAPdJ2iBpSVmfiRHxfJp+AZhYrShJSySVJJW6u7sHvldmZlZVMw7KfygiziW7lHa1pA9XdoiIIAueQ0TEiohoj4j2tra2gks1Mxs+igyUbcDUsvkpqa1qH0mtwFjgpb7WjYje1x3A3Ry4FLZd0qS0rUnAjhz3xczM+lFkoKwHZkqaLmkE2SB7R0WfDmBRmr4CeCidXXQAC9NdYNOBmcCjksZIOgFA0hjgIuDJKttaBNxT0H6ZmVkVrUVtOCJ6JC0F1gEtwMqIeErSNUApIjqAW4DbJHUCO8lCh9RvDbAJ6AGujoh9kiYCd2fj9rQCd0TEvektrwXWSFoMPAd8qqh9MzOzQyk7IRie2tvbo1Qq9d/RzMz+SNKGio9tAM05KG9mZkOQA8XMzHLhQDEzs1w4UMzMLBcOFDMzy4UDxczMcuFAMTOzXDhQzMwsFw4UMzPLhQPFzMxyUdh3eZmZWWNEwJtvwhtvwO7d2Wvl9AUXwCmn5Pu+DhQzs0EWAW+91fcBv9Z0f/O7d2c//X1N4y9/CZdcku9+OVDMzCqUH/BrHdwHGgSV0wP9Xt7jjoPRo2HMmOynd3r8eJg69dD2/qanT8//9+ZAMbOms3//kR/w+1p+OAf8kSMPHLDLD+DjxsGUKQM/4Fe+trQU87vMkwPFzHK3f/+hB+6+DuqHc8AfqLz/wi+fbpYDftEcKGbD0Ntv1z6o1/va17K33hp4Tb0H5sqDdltb/3/B1/qrvnfZqFE+4A+GQgNF0nzg+2RPbLw5Iq6tWD4S+AnwPrJnyV8ZEc+mZcuBxcA+4MsRsa5svRagBGyLiMtS263AfwB2pW5/EREbC9s5s4JEwJ49+RzYa72+/fbAapJqH8AnTarvoN5X23HHwTH+EEPTKyxQ0kH/RuBCoAtYL6kjIjaVdVsMvBwRMyQtBK4DrpQ0i+xxwLOBU4EHJJ0REfvSel8BngbeUfG2fx0Ra4vaJzOAffuyWzKP9C/8vl737x9YTa2ttQ/oEyYM7EBfbdnIkVmomPWlyDOUuUBnRGwBkLQaWED2nPheC4Bvpem1wA+UPTB+AbA6IvYAz6Rnzs8FHpY0BfhT4B+AvyqwfmtCEbB3b30H7cPtcziXc0aNqn7gLh+wLW+vdo2+r4P/scfm/7s0G6giA2UysLVsvgv4k1p9IqJH0i5gfGr/VcW6k9P0PwLfBE6o8p7/IOnvgQeBZSmQbAjp/es+74N8+bJ9+/qvo9wxx9Q+WJ96at8H83r+0h81ypdzbHhoqkF5SZcBOyJig6SPVCxeDrwAjABWAH8DXFNlG0uAJQCnnXZaofU2m95r99UO3AM5oPcVCHsOI+J7786pPFCPHXvoAb+/v+SrvY4Y4cs5ZnkoMlC2AVPL5qektmp9uiS1AmPJBudrrXs5cLmkS4HjgHdI+mlEfCYink9990j6MfCNakVFxAqywKG9vX2Ad5o3Vu+lnHoO9P0d5GtND/Te+5aW2gfqE0+s/Rd9rYN/5V/9vjvHrHkUGSjrgZmSppOFwULg0xV9OoBFwMPAFcBDERGSOoA7JH2PbFB+JvBoRDxMdiZCOkP5RkR8Js1Piojn0xjMJ4AnC9y3Q/T0HDg413uQH2gA9PQMvK5aB+7x4+G00+o7wPfVNmJE/r9LM2tOhQVKGhNZCqwju214ZUQ8JekaoBQRHcAtwG1p0H0nWeiQ+q0hG8DvAa4uu8OrltsltQECNgJ/WciOAddcAz/96cEH/L17B76dyks5o0ZlryeckH1pWz1/2fc1PWqUL+WY2eBRDPQax1Gkvb09SqXSgNe7+WZ48MEjO+D7Uo6ZNStJGyKivbK9qQblh4ovfCH7MTOzA3wzo5mZ5cKBYmZmuXCgmJlZLhwoZmaWCweKmZnlwoFiZma5cKCYmVkuHChmZpaLYf1JeUndwHOHufoE4MUcyylaM9XbTLVCc9XbTLVCc9XbTLXCkdX7zohoq2wc1oFyJCSVqn31wFDVTPU2U63QXPU2U63QXPU2U61QTL2+5GVmZrlwoJiZWS4cKIdvRaMLGKBmqreZaoXmqreZaoXmqreZaoUC6vUYipmZ5cJnKGZmlgsHipmZ5cKBchgkzZe0WVKnpGWNrqcWSVMl/bOkTZKekvSVRtfUH0ktkn4t6X82upb+SDpR0lpJv5X0tKTzGl1TXyR9Lf07eFLSnZKOa3RNvSStlLRD0pNlbeMk3S/pd+n1pEbWWK5Gvd9J/xYel3S3pBMbWWOvarWWLfu6pJA0IY/3cqAMkKQW4EbgEmAWcJWkWY2tqqYe4OsRMQv4AHD1EK6111eApxtdRJ2+D9wbEe8G3ssQrlvSZODLQHtEnAW0AAsbW9VBbgXmV7QtAx6MiJnAg2l+qLiVQ+u9HzgrIs4G/h+wfLCLquFWDq0VSVOBi4A/5PVGDpSBmwt0RsSWiNgLrAYWNLimqiLi+Yh4LE2/RnbAm9zYqmqTNAX4U+DmRtfSH0ljgQ8DtwBExN6IeKWxVfWrFRglqRUYDfx7g+v5o4j4F2BnRfMCYFWaXgV8YlCL6kO1eiPivojoSbO/AqYMemFV1PjdAtwAfBPI7c4sB8rATQa2ls13MYQP0r0kTQPOAR5pbCV9+keyf+D7G11IHaYD3cCP0yW6myWNaXRRtUTENuB6sr9Gnwd2RcR9ja2qXxMj4vk0/QIwsZHFDNDngf/V6CJqkbQA2BYRv8lzuw6UYUDS8cDPga9GxKuNrqcaSZcBOyJiQ6NrqVMrcC5wU0ScA7zB0Lokc5A0/rCALAhPBcZI+kxjq6pfZJ9vaIrPOEj6O7LLzbc3upZqJI0G/hb4+7y37UAZuG3A1LL5KaltSJJ0LFmY3B4Rv2h0PX04H7hc0rNklxE/JumnjS2pT11AV0T0nvGtJQuYoerjwDMR0R0RbwO/AD7Y4Jr6s13SJID0uqPB9fRL0l8AlwF/HkP3Q36nk/1h8Zv0/20K8JikU450ww6UgVsPzJQ0XdIIsoHNjgbXVJUkkV3jfzoivtfoevoSEcsjYkpETCP7nT4UEUP2L+iIeAHYKunM1DQP2NTAkvrzB+ADkkanfxfzGMI3ESQdwKI0vQi4p4G19EvSfLJLtpdHxO5G11NLRDwRESdHxLT0/60LODf9mz4iDpQBSoNuS4F1ZP8h10TEU42tqqbzgc+S/bW/Mf1c2uiijiJfAm6X9DgwB/jvDa6npnQmtRZ4DHiC7P/+kPmqEEl3Ag8DZ0rqkrQYuBa4UNLvyM6wrm1kjeVq1PsD4ATg/vR/7Z8aWmRSo9Zi3mvonpWZmVkz8RmKmZnlwoFiZma5cKCYmVkuHChmZpYLB4qZmeXCgWJ2BCT93/Q6TdKnc97231Z7L7OhyrcNm+VA0keAb0TEZQNYp7XsywSrLX89Io7Poz6zweAzFLMjIOn1NHktcEH6QNvX0nNdviNpfXo+xhdT/49I+ldJHaRP1kv6H5I2pGeVLElt15J9M/BGSbeXv5cy30nPNXlC0pVl2/7fZc9ouT19Kh5J1yp7Ls7jkq4fzN+RDR+tjS7A7CixjLIzlBQMuyLi/ZJGAv8mqffbfc8le27GM2n+8xGxU9IoYL2kn0fEMklLI2JOlff6M7JP5r8XmJDW+Ze07BxgNtlX0/8bcL6kp4FPAu+OiBgqD36yo4/PUMyKcRHwnyVtJHtkwHhgZlr2aFmYAHxZ0m/InqExtaxfLR8C7oyIfRGxHfg/wPvLtt0VEfuBjcA0YBfwFnCLpD8Dhuz3TFlzc6CYFUPAlyJiTvqZXvb8kTf+2Ckbe/k4cF5EvBf4NXAkj+bdUza9D+gdp5lL9l1elwH3HsH2zWpyoJjl4zWyLwbstQ74L+nxAUg6o8YDuMYCL0fEbknvJntUc6+3e9ev8K/AlWmcpo3syZGP1iosPQ9nbET8Evga2aUys9x5DMUsH48D+9Klq1vJnjc/jew5EyJ7umO1R9jeC/xlGufYTHbZq9cK4HFJj0XEn5e13w2cB/yG7KFT34yIF1IgVXMCcI+k48jOnAjCHjcAAAA8SURBVP7q8HbRrG++bdjMzHLhS15mZpYLB4qZmeXCgWJmZrlwoJiZWS4cKGZmlgsHipmZ5cKBYmZmufj//rvD/YdEfDEAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wYAIbkRpjUNZ",
        "colab_type": "text"
      },
      "source": [
        "<br>\n",
        "<br>\n",
        "<br>\n",
        "<br>\n",
        "<br>\n",
        "<br>\n",
        "<br>\n",
        "<br>\n",
        "<br>\n",
        "<br>\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hbvpbieUigAI",
        "colab_type": "text"
      },
      "source": [
        "### **2. Plot the accuracy curve**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_WepyFzxik6n",
        "colab_type": "code",
        "outputId": "ed31fc07-f535-4eee-de1f-4c4d3fe85900",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 279
        }
      },
      "source": [
        "fig = plt.figure()\n",
        "graph3 = plt.plot(iters, train_accuracys, 'b')\n",
        "graph4 = plt.plot(iters, test_accuracys, 'r')\n",
        "plt.xlabel('iterations')\n",
        "plt.ylabel('loss')\n",
        "plt.show()"
      ],
      "execution_count": 150,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAEGCAYAAABiq/5QAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAYc0lEQVR4nO3dfZRddX3v8fcnhKeEPEAyiVxImGhDUm1vQphEwZAaxAjerKb4RK1QBFxRi2i5XF1gXV33dnVpllIttZUSQeVawFJ00FVrSGpFpCIwgQSCCULDBBIImSCEmAdikm//2Ps4Z2bOzJwks2fPmd/ntdZZe599zt77m8k5n/07v/2kiMDMzNIxouwCzMxscDn4zcwS4+A3M0uMg9/MLDEOfjOzxIwsu4B6TJw4MZqbm8suw8ysoaxevXp7RDR1n94Qwd/c3ExbW1vZZZiZNRRJm2pNL7SrR9J4SXdJ2iBpvaSzJL1P0hOSDkpqKXL9ZmbWU9Et/huAFRHxXknHAKOAV4B3AzcVvG4zM6uhsOCXNA5YAHwIICL2AfvIgh9JRa3azMz6UGRXzzSgA/iGpEcl3SxpdL0zS1oqqU1SW0dHR3FVmpklpsjgHwnMAW6MiDOAXcC19c4cEcsjoiUiWpqaeuyUNjOzw1Rk8G8GNkfEg/nzu8g2BGZmVqLCgj8itgLPSZqRT3o78Iui1mdmZvUp+qieq4Db8iN6NgKXSboQ+ArQBPxA0pqIeGfBdVhi9u+HLVugvR2eeQaefx7e9CZYsABOPLHs6qzRRcDWrZ2fr02bYM+eYtZ1ySUwffrALrPQ4I+INUD3Y/Vb84fZYTt4MAvz9vbOL1/18LnnsvDvToIzzoCFC7PHOefA2LGDW7sNfRGwfXvPz1VluGkT7N3bdZ6iDlQ8++yBD341wo1YWlpawmfupiUCXnyx84vW/cv37LOwb1/XeU4+GZqbYdq0nsNJk+DRR+HHP84eDzyQzT9iBJx5ZueGYP58OOGEwf7X2mCLgJdfrh3qlceuXV3nmTCh52erMn7aaTBq1OD+G+ohaXVE9DhR1sFvpai0qLp/4arHu7eomppqh3pzc/bFO+64+te/Z08W/vfem20IHnwQfvMbGDkS5s7t3BCcffbQ/EL3JQI6OnqGWpHdEY2kEvivvtp1+vjxtUO9Mj5mzGBXeuQc/DboemtRVYbdW1QnnZQF+LRptVtWo+s+C+TQ7doFP/tZ5y+Chx+GAwfg6KPhzW/u3BCcddahbWCKUGmt9vZ3bW+H3bu7zjNhQva3bcTwGmhjxtT+jI0fX3ZlA8/BbwNu586+w2fHjq7vHzu275/K48YN8j+gDzt3wv33ZxuBe++F1auz/QrHHpuFf2VDMG9eNm2g7djR+6+hZ57J6qs2blzff1vvx0hTksF/5ZXw1a8WUJDVNHp0733slRZVo16pY8cO+OlPO38RrFmTtbwHywkn1O6CGM6tVTtyvQV/Q1yW+XBdcAFMnFh2FcPX6NFdA2jChMYN9v6MGweLF2cPgF/9Cu67D9auzX4JDLTqoJ82LesGG65/Wxt8w7rFb2aWst5a/L71oplZYhz8ZmaJcfCbmSXGwW9mlhgHv5lZYhz8ZmaJcfCbmSXGwW9mlhgHv5lZYhz8ZmaJcfCbmSWm0OCXNF7SXZI2SFov6SxJJ0laJempfOg7oJqZDaKiW/w3ACsiYiYwC1gPXAv8KCKmAz/Kn5uZ2SApLPgljQMWALcARMS+iHgFWALcmr/tVuCPiqrBzMx6KrLFPw3oAL4h6VFJN0saDUyOiBfy92wFJteaWdJSSW2S2jo6Ogos08wsLUUG/0hgDnBjRJwB7KJbt05kNwOoeUOAiFgeES0R0dLU1FRgmWZmaSky+DcDmyPiwfz5XWQbghclnQyQD7cVWIOZmXVTWPBHxFbgOUkz8klvB34BfB+4NJ92KfC9omowM7Oeir7n7lXAbZKOATYCl5FtbO6UdAWwCXh/wTWYmVmVQoM/ItYAPe73SNb6NzOzEvjMXTOzxDj4zcwS4+A3M0uMg9/MLDEOfjOzxDj4zcwS4+A3M0uMg9/MLDEOfjOzxDj4zcwS4+A3M0uMg9/MLDEOfjOzxDj4zcwS4+A3M0uMg9/MLDEOfjOzxDj4zcwS4+A3M0tMoffcldQO7AQOAPsjokXSLOAfgROAduCDEfFqkXWYmVmnwWjxL4yI2RFRuen6zcC1EfH7QCvwqUGowczMcmV09ZwO3JePrwLeU0INZmbJKjr4A1gpabWkpfm0J4Al+fj7gCm1ZpS0VFKbpLaOjo6CyzQzS0fRwT8/IuYAFwBXSloAXA78maTVwBhgX60ZI2J5RLREREtTU1PBZZqZpaPQ4I+ILflwG1l//ryI2BARiyLiTOAO4L+KrMHMzLoqLPgljZY0pjIOLALWSZqUTxsBfJbsCB8zMxskRbb4JwP3S1oLPAT8ICJWAB+Q9EtgA/A88I0CazAzs24KO44/IjYCs2pMvwG4oaj1mplZ33zmrplZYhz8ZmaJcfCbmSXGwW9mlhgHv5lZYhz8ZmaJcfCbmSXGwW9mlhgHv5lZYhz8ZmaJcfCbmSXGwW9mlhgHv5lZYhz8ZmaJcfCbmSXGwW9mlhgHv5lZYhz8ZmaJKezWiwCS2oGdwAFgf0S0SJpNdoP144D9wJ9FxENF1mFmZp0KDf7cwojYXvX8C8D/i4gfSnpX/vxtg1CHmZlRTldPAGPz8XHA8yXUYGaWrKJb/AGslBTATRGxHPhz4B5J15NteM6uNaOkpcBSgKlTpxZcpplZOopu8c+PiDnABcCVkhYAHwOujogpwNXALbVmjIjlEdESES1NTU0Fl2lmlo5Cgz8ituTDbUArMA+4FPhu/pZ/yaeZmdkgKSz4JY2WNKYyDiwC1pH16f9B/rZzgaeKqsHMzHoqso9/MtAqqbKe2yNihaRfAzdIGgnsJe/HNzOzwVFY8EfERmBWjen3A2cWtV4zM+ubz9w1M0uMg9/MLDEOfjOzxDj4zcwS4+A3M0uMg9/MLDEOfjOzxDj4zcwS4+A3M0uMg9/MLDF1Bb+kT0oaq8wtkh6RtKjo4szMbODV2+K/PCJeJbvC5onAJcCywqoyM7PC1Bv8yofvAr4VEU9UTTMzswZSb/CvlrSSLPjvya+zf7C4sszMrCj1Xpb5CmA2sDEidks6CbisuLLMzKwo9bb4zwKejIhXJF0MfBbYUVxZZmZWlHqD/0Zgt6RZwDXAfwH/v7CqzMysMPUG//6ICGAJ8PcR8Q/AmOLKMjOzotTbx79T0nVkh3GeI2kEcHRxZZmZWVHqDf6LgD8hO55/q6SpwBf7m0lSO7ATOED2q6FF0j8DM/K3jAdeiYjZh1y5mZkdlrqCPw/724C5khYDD0VEvX38CyNie9WyLqqMS/obvJPYzGxQ1XvJhvcDDwHvA94PPCjpvUeyYknKl3XHkSzHzMwOTb1dPX8BzI2IbQCSmoB/B+7qZ74AVkoK4KaIWF712jnAixHxVK0ZJS0FlgJMnTq1zjLNzKw/9R7VM6IS+rmX6px3fkTMAS4ArpS0oOq1D9BHaz8ilkdES0S0NDU11VmmmZn1p94W/wpJ99AZ1BcB/9bfTBGxJR9uk9QKzAPukzQSeDdw5qGXbGZmR6LenbufkvQe4K35pOUR0drXPJJGk/1S2JmPLwL+Kn/5PGBDRGw+zLrNzOww1dviJyK+A3znEJY9GWjN9uEyErg9Ilbkr/0x3qlrZlaKPoNf0k6yHbQ9XgIiIsb2Nm9EbARm9fLahw6hRjMzG0B9Bn9E+LIMZmbDjO+5a2aWGAe/mVliHPxmZolx8JuZJcbBb2aWGAe/mVliHPxmZolx8JuZJcbBb2aWGAe/mVliHPxmZolx8JuZJcbBb2aWGAe/mVliHPxmZolx8JuZJcbBb2aWmEKDX1K7pMclrZHUVjX9KkkbJD0h6QtF1mBmZl3VfbP1I7AwIrZXnkhaCCwBZkXEa5ImDUINZmaWK6Or52PAsoh4DSAitpVQg5lZsooO/gBWSlotaWk+7XTgHEkPSvqJpLm1ZpS0VFKbpLaOjo6CyzQzS0fRXT3zI2JL3p2zStKGfJ0nAW8B5gJ3Snp9RET1jBGxHFgO0NLSEpiZ2YAotMUfEVvy4TagFZgHbAa+G5mHgIPAxCLrMDOzToUFv6TRksZUxoFFwDrgbmBhPv104Bhge2/LMTOzgVVkV89koFVSZT23R8QKSccAX5e0DtgHXNq9m8fMzIpTWPBHxEZgVo3p+4CLi1qvmZn1zWfumpklxsFvZpYYB7+ZWWIc/GZmiXHwm5klxsFvZpYYB7+ZWWIc/GZmiXHwm5klxsFvZpYYB7+ZWWIc/GZmiXHwm5klxsFvZpYYB7+ZWWIc/GZmiXHwm5klxsFvZpYYB7+ZWWIKDX5J7ZIel7RGUls+7f9K2pJPWyPpXUXWYGZmXRV2s/UqCyNie7dpX46I6wdh3WZm1o27eszMElN08AewUtJqSUurpn9c0mOSvi7pxFozSloqqU1SW0dHR8Flmpmlo+jgnx8Rc4ALgCslLQBuBN4AzAZeAP6m1owRsTwiWiKipampqeAyzczSUWjwR8SWfLgNaAXmRcSLEXEgIg4CXwPmFVmDmZl1VVjwSxotaUxlHFgErJN0ctXbLgTWFVWDmZn1VORRPZOBVkmV9dweESskfUvSbLL+/3bgIwXWYGZm3RQW/BGxEZhVY/olRa3TzMz658M5zcwS4+A3M0uMg9/MLDEOfjOzxDj4zcwS4+A3M0uMg9/MLDEOfjOzxDj4zcwS4+A3M0uMg9/MLDEOfjOzxDj4zcwSMxg3Wzez4Wz/fhjZIFFy4ADs3l12FYfm+OMH/O/bIP9bZjbkrF8Py5bB7bfD+PHwu78LM2d2HU6dCiNK6FjYsweefBI2bMjqrAx/+Ut47bXBr+dI/PCHcP75A7pIB7+ZHZqHH4bPfx7uvjtrjX74w1lLev16aG2Fr32t873HHw8zZvTcKJx+Ohx77JHX8tJL2Xqrw33DBmhvh4jsPRK8/vXZet/5Tpg8OZvWKGbMGPBFOvjNrH8R8JOfwOc+B6tWZS38z34WPvEJmDix63u3b+8M4UoQP/AAfPvbnWE8YgRMm5ZtCLpvFE48sevyDh6EZ5/tGe7r12frqjjuuCwk3/xmuPTSrhuZ444r9u/TYBSV/4ghrKWlJdra2souwyw9Bw/CD36QBf7Pf561lq+5Bj7yERg79tCWtXt31tXSfaPQvftl8uQssCdNgqeeyrps9uzpfH3ChNobjNNOK6dbaQiTtDoiWrpPL7TFL6kd2AkcAPZXFyDpGuB6oCkittdegpmVYv9+uPPOrEtn3TpoboavfhUuu+zwW8+jRsHs2dmj2oEDWddMdUt+/Xp49FGYPh0WLuwa8t1/YdghG4yunoXdg13SFGAR8OwgrN/M6rV3L9x6K3zhC7BxI7zxjfCtb8FFF8HRRxezzqOOgje8IXssXlzMOqyLsn4XfRn4NDD0+5nMUrBzJ1x/fbYT9KMfzbpTWlvh8cfh4ouLC30rRdEt/gBWSgrgpohYLmkJsCUi1qqR9qybDUcvvQRf+Qr83d/Byy/DuedmLfxzz22sI1/skBQd/PMjYoukScAqSRuAz5B18/RJ0lJgKcDUqVOLrdIsNVu2wJe+BDfdBLt2wZIlcN112RExNuwV2tUTEVvy4TagFfgDYBqwNt/xeyrwiKTX1Zh3eUS0RERLU1NTkWWapePpp2Hp0qxL54Yb4MILs+6cu+926CeksBa/pNHAiIjYmY8vAv4qIiZVvacdaPFRPTbgOjq6HiXy9NOwb1/ZVZVr7164//6sv/7yy+FTn8o2AJacIrt6JgOteT/+SOD2iFhR4PosNQcOwKZNPU/q2bAh67uuGDUKfud3smHqrrkGrr4aTj657EqsRIUFf0RsBGb1857motZvw8iePV1P/Km+7srevZ3va2rKjvN+73u7ntgzZYpP7DGr4ks22KHbtSsL3W3bBn7ZEbB5c9eQf+aZrtddmTYtC/R3vKMz3GfOzA5BNLN+Ofitd937ySvDTZuKX/exx2bXXZk7Fy65pPMU/enTswt/mdlhc/Cn7uDBLMhrXeGwup/8+OOzVvXZZ8MVV2Tjp5xSzLHekydn11056qiBX7aZOfiTsXdv1j3TPdyffLJnP/nMmfCe93S9Por7yc2GjeEd/H/913DHHWVXUb7du7NWfa1+8vPO67oj1P3kZsPe8A7+170uu8hU6o45pvP65O4nN0ve8A7+D384e5iZ2W+509bMLDEOfjOzxDj4zcwS4+A3M0uMg9/MLDEOfjOzxDj4zcwS4+A3M0uMonIa/xAmqQM43EtCTgQa6Q5fjVRvI9UKjVVvI9UKjVVvI9UKR1bvaRHR4961DRH8R0JSW0S0lF1HvRqp3kaqFRqr3kaqFRqr3kaqFYqp1109ZmaJcfCbmSUmheBfXnYBh6iR6m2kWqGx6m2kWqGx6m2kWqGAeod9H7+ZmXWVQovfzMyqOPjNzBIzrINf0vmSnpT0tKRry66nN5KmSPqxpF9IekLSJ8uuqT+SjpL0qKR/LbuW/kgaL+kuSRskrZd0Vtk19UXS1fnnYJ2kOyQdV3ZNFZK+LmmbpHVV006StErSU/nwxDJrrNZLvV/MPwuPSWqVNL7MGitq1Vr12jWSQtLEgVjXsA1+SUcB/wBcALwR+ICkoXofxv3ANRHxRuAtwJVDuNaKTwLryy6iTjcAKyJiJjCLIVy3pFOATwAtEfF7wFHAH5dbVRffBM7vNu1a4EcRMR34Uf58qPgmPetdBfxeRPxP4JfAdYNdVC++Sc9akTQFWAQ8O1ArGrbBD8wDno6IjRGxD/g2sKTkmmqKiBci4pF8fCdZMJ1SblW9k3Qq8L+Am8uupT+SxgELgFsAImJfRLxSblX9GgkcL2kkMAp4vuR6fisi7gN+1W3yEuDWfPxW4I8Gtag+1Ko3IlZGxP786c+BUwe9sBp6+dsCfBn4NDBgR+IM5+A/BXiu6vlmhnCYVkhqBs4AHiy3kj79LdkH8WDZhdRhGtABfCPvmrpZ0uiyi+pNRGwBridr3b0A7IiIleVW1a/JEfFCPr4VmFxmMYfocuCHZRfRG0lLgC0RsXYglzucg7/hSDoB+A7w5xHxatn11CJpMbAtIlaXXUudRgJzgBsj4gxgF0OrK6KLvH98CdkG638AoyVdXG5V9Yvs+PCGOEZc0l+QdbPeVnYttUgaBXwG+MuBXvZwDv4twJSq56fm04YkSUeThf5tEfHdsuvpw1uBP5TUTtZ9dq6kfyq3pD5tBjZHROUX1F1kG4Kh6jzgmYjoiIjfAN8Fzi65pv68KOlkgHy4reR6+iXpQ8Bi4IMxdE9megNZA2Bt/n07FXhE0uuOdMHDOfgfBqZLmibpGLIdZN8vuaaaJImsD3p9RHyp7Hr6EhHXRcSpEdFM9jf9j4gYsi3SiNgKPCdpRj7p7cAvSiypP88Cb5E0Kv9cvJ0hvDM6933g0nz8UuB7JdbSL0nnk3VV/mFE7C67nt5ExOMRMSkimvPv22ZgTv6ZPiLDNvjznTcfB+4h++LcGRFPlFtVr94KXELWel6TP95VdlHDyFXAbZIeA2YDnyu5nl7lv0zuAh4BHif7jg6ZSwxIugN4AJghabOkK4BlwDskPUX2i2VZmTVW66XevwfGAKvy79o/llpkrpdai1nX0P2VY2ZmRRi2LX4zM6vNwW9mlhgHv5lZYhz8ZmaJcfCbmSXGwW9JkPSzfNgs6U8GeNmfqbUus6HKh3NaUiS9Dfg/EbH4EOYZWXVRr1qv/zoiThiI+swGg1v8lgRJv85HlwHn5CfuXJ3fV+CLkh7Or8/+kfz9b5P0U0nfJz/TV9Ldklbn18pfmk9bRnYlzTWSbqtelzJfzK+r/7iki6qWfW/VPQJuy8/SRdIyZfdleEzS9YP5N7J0jCy7ALNBdi1VLf48wHdExFxJxwL/KalyNcw5ZNdtfyZ/fnlE/ErS8cDDkr4TEddK+nhEzK6xrneTnSk8C5iYz3Nf/toZwJvILrn8n8BbJa0HLgRmRkQMlRuE2PDjFr+lbhHwp5LWkF0KewIwPX/toarQB/iEpLVk13CfUvW+3swH7oiIAxHxIvATYG7VsjdHxEFgDdAM7AD2ArdIejcwZK8jY43NwW+pE3BVRMzOH9Oqrn+/67dvyvYNnAecFRGzgEeBI7kl4mtV4weAyn6EeWTX6lkMrDiC5Zv1ysFvqdlJdoGuinuAj+WXxUbS6b3cqGUc8HJE7JY0k+wWmRW/qczfzU+Bi/L9CE1kdwJ7qLfC8vsxjIuIfwOuJusiMhtw7uO31DwGHMi7bL5Jdj/eZrLrnIvsbl21bh24Avho3g//JFl3T8Vy4DFJj0TEB6umtwJnAWvJbk7y6YjYmm84ahkDfE/ZzdUF/O/D+yea9c2Hc5qZJcZdPWZmiXHwm5klxsFvZpYYB7+ZWWIc/GZmiXHwm5klxsFvZpaY/wYv7Onj791oLAAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CcYJkW0ZvIBq",
        "colab_type": "text"
      },
      "source": [
        "### **3. Plot the accuracy value**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_I-fWRNFvQEv",
        "colab_type": "code",
        "outputId": "64563ec6-aa5e-4ef9-ee61-8b357a4b8e8e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 53
        }
      },
      "source": [
        "final_train_accuracy = train_accuracys[-1]\n",
        "final_test_accuracy = test_accuracys[-1]\n",
        "print(\"final_train_accuracys :\", final_train_accuracy,\"%\")\n",
        "print(\"final_test_accuracys :\", final_test_accuracy,\"%\")"
      ],
      "execution_count": 151,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "final_train_accuracys : 60.79999999999999 %\n",
            "final_test_accuracys : 54.25 %\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "v1k8NptSxNFy",
        "colab_type": "text"
      },
      "source": [
        "### **4. Plot the classification example**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lVBi79mqxQGY",
        "colab_type": "code",
        "outputId": "08cd1c75-6433-4b98-cc88-a31be99afb50",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 509
        }
      },
      "source": [
        "f2 = plt.figure(2)\n",
        "im_correct  = np.zeros((size_row * size_col, 10), dtype=float)\n",
        "print(correct_labels)\n",
        "print(correct_labels[0][1])\n",
        "print(incorrect_labels)\n",
        "label_correct_index = []\n",
        "label_incorrect_index = []\n",
        "\n",
        "for i in range(10):\n",
        "    correct_index = correct_labels[0][i]\n",
        "    im_correct[:, i] += test_image[:, correct_index]\n",
        "    label_correct_index.append(test_label[correct_index])\n",
        "\n",
        "for i in range(10):\n",
        "    plt.subplot(2, 5, i+1)\n",
        "    plt.title(label_correct_index[i])\n",
        "    plt.imshow(im_correct[:,i].reshape((size_row, size_col)), cmap='Greys', interpolation='None')\n",
        "\n",
        "    frame   = plt.gca()\n",
        "    frame.axes.get_xaxis().set_visible(False)\n",
        "    frame.axes.get_yaxis().set_visible(False)\n",
        "plt.show()\n",
        "###################################################\n",
        "print(incorrect_predicts)\n",
        "\n",
        "f2 = plt.figure(2)\n",
        "im_incorrect  = np.zeros((size_row * size_col, 10), dtype=float)\n",
        "\n",
        "for i in range(10):\n",
        "    incorrect_index = incorrect_labels[0][i]\n",
        "    im_incorrect[:, i] += test_image[:, incorrect_index]\n",
        "    label_incorrect_index.append(test_label[incorrect_index])\n",
        "\n",
        "for i in range(10):\n",
        "    plt.subplot(2, 5, i+1)\n",
        "    value = incorrect_predicts[0][i]\n",
        "    plt.title(value)\n",
        "    plt.imshow(im_incorrect[:,i].reshape((size_row, size_col)), cmap='Greys', interpolation='None')\n",
        "\n",
        "    frame   = plt.gca()\n",
        "    frame.axes.get_xaxis().set_visible(False)\n",
        "    frame.axes.get_yaxis().set_visible(False)\n",
        "plt.show()"
      ],
      "execution_count": 139,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "deque([[6, 17, 26, 48, 50, 69, 79, 88, 98, 104]])\n",
            "17\n",
            "deque([[0, 1, 2, 3, 4, 5, 7, 8, 9, 10]])\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWAAAADSCAYAAAB0FBqGAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAWq0lEQVR4nO3de7DUdf3H8edHQOWWIkjgT4eD44VBpXTOqJUWVk6p2ahklqLmjVRAh7FiEtMErcgZMvNSXkCxzLyVyqhpJShqCTilFqKIxwuCgKJyVQa/vz/wfb5nd797zi7u7vuzu6/HDLN7dr/s+fDhe97n/f1835/PJyRJgoiI1N423g0QEWlWCsAiIk4UgEVEnCgAi4g4UQAWEXGiACwi4kQBWETESZQBOISwNu/P5hDCb7zb5Ul9Ukh9kk39UijWPunu3YAsSZL0sechhD7AcuBOvxb5U58UUp9kU78UirVPosyA84wCVgCPezckIuqTQuqTbOqXQtH0ST0E4FOBmYnmTHekPimkPsmmfikUTZ+ECNpQVAhhCLAE2CNJkle82xMD9Ukh9Uk29Uuh2Pok9gz4ZGBuDB0VEfVJIfVJNvVLoaj6JPYAfApwi3cjIqM+KaQ+yaZ+KRRVn0Q7BBFC+DzwCDAoSZI13u2JgfqkkPokm/qlUIx9EnMGfCpwTywdFQn1SSH1STb1S6Ho+iTaDFhEpNHFnAGLiDQ0BWAREScKwCIiThSARUScKACLiDgpazW0AQMGJC0tLVVqShza2tpYtWpVKPX4ZugTgAULFqxKkmTnUo5Vn2Rrhn7Rz0+2YudKWQG4paWF+fPnV65VEWptbS3r+GboE4AQwqulHqs+ydYM/aKfn2zFzhUNQYiIOFEAFhFxogAsIuJEAVhExIkCsIiIEwVgEREnCsAiIk4UgEVEnCgAi4g4UQAWEXFS1lTkrTVu3Lj259dee23Oe8cffzwA++23X87r48ePB2CHHXaocut8HHbYYe3PX3llywatZ599dqd/Z5999gHgS1/6Us7rPXv2BKBHjx6VbKJE6Mtf/jIAs2fPBuDMM88E4KSTTgJg9913B6B///4A9OrVq8YtlHIoAxYRcVKTDDiEkPkc4K677sp5NFOnTgXgggsuAODCCy8EYNttt61aO2vpsccea39ufXLRRRflHGP79eX3Wf77RxxxBADTpk0DYK+99qpsY6vkySefBODQQw8F4KOPPmp/b5tttuQGY8eOBWCXXXYB4Nxzz835DMv67SqgWdg5MX36dABuuummnNctE544cSIAZ5xxRq2bWFWnn356+/MZM2YA8JnPfAZIrxJOPPFEIL0a2HHHHQHYbrvtgDiuDpQBi4g4UQAWEXFSkyEIuzQox7p16wCYMmUKkN6gGjRoUOUa1iAeeuihnK///Oc/A/HflNtpp50A2H777QHYuHFj+3t2KZ1/03bSpEk579ti3nPnzgUa//wYMWIEAAsXLgRgxYoVQDocZRYvXgzAmDFjALjmmmsAePTRR4HGurlt58Kzzz6b83jllVfmHGdDEX369AHgkksuAWD06NEAdO9ek3CYQxmwiIiTmoT8jgPmQ4YMAdLSNMt0LftZvXp15mdYmc3DDz8MQLdu3arT2BpZvnx5+3O7iZDPMlsrUxs5ciQAt956a6fH33DDDUDhDavYDBs2DIC7774bgKOOOqrsz2hrawPg4IMPzvm6UVlWd+mllwLw/vvvA3D99dcD6U05O78sO/zPf/4DwPe+9z0gvUqqV/bvAFizZg0Ar732GgDz5s3L/Dtvv/12zqPFpYsvvhiARYsWAbW9oasMWETEScgfO+pMa2trUq39m+y3mP2mtrKzJ554Iuc4K1066KCDqtKO1tZW5s+fX/KmgtXsk1Kdc845QJoFmc2bN1fse4QQFiRJUtKGX1vbJ5bNdXT//fcDsHTpUiAd67z99tsBeO6553KOt6uF3XbbrezvX65y+gRqe66sX78egF//+tdA4dh5Jc+Njjx/fj788EMA1q5dm/O6xZRnnnkGgCuuuAJIz6WVK1cC6ZWlXW1XUrFzRRmwiIiT2t/2K6Jv374AHHLIIQA8+OCDAOy7775AOr7z3//+F6heBlyPLKspNmGjXnzqU58qeK1YNmITDJ566ikgPW9smvaSJUuq0cS6YZMMfvzjHwPplYGNEf/rX/8CGuvnyCZpWXWNsWn/9miTuyxj3mOPPQD4/e9/D1QnAy5GGbCIiJNoMuB8vXv3BtI7/zNnzgTS31oiAP369QPSWmIbz7OMb+jQoT4Ni4yNs5Zzz6fRWcZs1TcPPPBAzdugDFhExEm0GbAZMGBAztc/+MEPgHQBGqsrbkYffPABAM8//3zO6/nLVTYyqyW2ulgbG37rrbcAZcDGKgHsPoH6Jb2avu+++wDNhBMRaSrRZ8A2W8WWWtywYQMAr7/+OqAMGAprpZtxnG/XXXf1bkKUXnrpJaA5z4mu2PKny5YtA2pTO55PGbCIiJPoM2Apbs6cOUBh/W8zXhV84QtfANJMb/LkyYDPne2YWAacXytuVSIDBw70aViEPK4SlAGLiDipmwzYfjtpLCtlq4jls4qAZmLjd5bhvffee0A6Tm7b0DSr/J+bRpoBVykeM0mVAYuIOKmbDDh/DOuOO+4A0jUAmpGt3pT/m9s2H2xmttbBm2++CajutVHWC2k0yoBFRJzUTQacz+7uNjOrY7Qt3G3fvKxVxZqFzQJ87LHHgHR/sGbPgHXvpGuqghARaSLRZ8CDBw/OebRZK83Mdg+xzFfjeqlRo0YBaY207X129NFHA2mfNRuNAXdNVRAiIk0k+gzY7ujbXP9mzoAXLlwIwDHHHOPcknjZbrmXXXYZkO5yYLXRzThLEDQGHCtlwCIiTqLPgCX17rvvAvDyyy87tyRetpPKscceC6Q7Rf/0pz8FYMaMGS7t8qYx4K7ZuVNLyoBFRJwoAIuIONEQRB2xjSetlGrTpk05X9t2TZIuNmNDEDZtu9mGIGyyTv6kHSk0duzYmn9P/W+IiDhRBlxH9t9/fyBderGtrQ3QjZUsxx13HACXX345kN64bLblKffee29Ak3ay5C/Wr6nIIiJNpG4y4OHDhwMwb948AJYvXw7Axo0bgXR8tBnZtjLNvAhPvr59+wIwcuRIAJYsWQLAySefDKTLmTa6PffcEygcA16/fj0AvXr18mlYBGyDX2MbHIwbN65mbVAGLCLipG4y4Ouuuw5IM5d33nkHSMf0mikDnjhxIgDnnHMOAAcccAAA/fv3d2tTrE477TQApk+fDsCsWbMAeOGFFwAYNmyYT8Nq7KqrrgJgwoQJACxduhRIM+RmZPcJ7KrIFnCyJUxHjBhR9TYoAxYRcVI3GbDdtV67dq1zS/yNGTMm51GKa21tBWD06NEA3HvvvUDzjZePHz8+51Gge/ct4c8qRRYtWgSk4+W1oAxYRMRJ3WTAIlujR48eANx8882+DZHo2Llhy7x6UAYsIuIklDP7I4SwEni1es2JwpAkSXYu9eAm6RMoo1/UJ9mapF/UJ9ky+6WsACwiIpWjIQgREScKwCIiThSARUScKACLiDhRABYRcaIALCLiRAFYRMSJArCIiBMFYBERJwrAIiJOFIBFRJwoAIuIOFEAFhFxogAsIuJEAVhExIkCsIiIEwVgEREnCsAiIk4UgEVEnCgAi4g4UQAWEXGiACwi4kQBWETEiQKwiIgTBWAREScKwCIiThSARUScKACLiDhRABYRcaIALCLiRAFYRMSJArCIiBMFYBERJwrAIiJOFIBFRJwoAIuIOFEAFhFxogAsIuJEAVhExIkCsIiIEwVgEREnCsAiIk4UgEVEnCgAi4g4iTIAhxDW5v3ZHEL4jXe7PKlPigshfCeEsDCEsC6E8HII4VDvNnnSuVJcbOdKd89vXkySJH3seQihD7AcuNOvRf7UJ9lCCIcDU4ETgKeBwb4t8qdzJVuM50qUATjPKGAF8Lh3QyKiPkldCkxOkuSfH3+91LMxEdK5koruXIlyCCLPqcDMJEkS74ZERH0ChBC6Aa3AziGExSGEN0IIV4cQenq3LSI6V4j3XAkx/7+EEIYAS4A9kiR5xbs9MVCfpEIIu7Ali1kAHA1sAu4FZidJMsmzbTHQuZKK9VyJPQM+GZjb7CdPHvVJasPHj79JkmRZkiSrgGnAkY5tionOlVSU50rsAfgU4BbvRkRGffKxJElWA28AHS/j4r2kqz2dKx+L9VyJNgCHED4P/B+6e9tOfZJpBjA+hDAwhNAPmADMcm6TO50rmaI7V2KugjgVuCdJkjXeDYmI+qTQFGAA8CKwEbgDuNy1RXHQuVIounMl6ptwIiKNLNohCBGRRqcALCLiRAFYRMSJArCIiBMFYBERJ2WVoQ0YMCBpaWmpUlPi0NbWxqpVq0KpxzdDnwAsWLBgVZIkO5dyrPokWzP0i35+shU7V8oKwC0tLcyfP79yrYpQa2trWcc3Q58AhBBeLfVY9Um2ZugX/fxkK3auaAhCRMRJzDPhRLbanDlzABg5ciQA3/72twG47bbbAOjWrZtLu0Q6UgYsIuJEGbDUtXfeeQeAmTNnAjB16lQA3n//fQC22WZLjrFs2TIANPW++bz00ksADBs2DIDPfe5zAPz1r38FoHfv3j4NQxmwiIgbZcBS12xM94ILLsh83zLgyy67DIDu3XXKNwu7GpoyZQqQXv08+eSTAJx33nkA3HTTTQ6t20IZsIiIE6UDUpf++c8tG9v+6Ec/6vS4ffbZB4AvfvGLVW+T+Nq4cSMATzzxBABXXHEFAOvXr888fujQobVpWCeUAYuIOFEGLHXpmWeeAeCDDz7IfH+//fYD4B//+EfN2iS+rNLl8MMP7/S4H/7wh0DXV0+1oAxYRMRJVTLgF198EYC99tprqz/D5oe/+uqWKdR2B/PZZ58F4PLLc7dy+uijjwA48MADAfjFL37R/t5hhx221e2IyR/+8AcgzfomTZoEwFtvvZVz3KBBgwBYsGABAIMHD65VE6vG/v9t7Pf888/PPM5muP32t78FYKeddqpB6/wsWrQIgL333jvn9WuvvTbz+L///e8A3HPPPTmvv/DCC5mfUw8+/PBDAK677rpOj5swYQKQVsT06NGjug0rgTJgEREnCsAiIk4qOgRx3333AXDGGWcA5V362iVmCFuWEn3ttdcAWLt2beb79mis4N4uu4866qj29x544AEgXZglNv/+978BWLJkSc7r1p9/+ctfAFizJnuH8fy+sCGJr3zlKwD873//q1xja2zDhg0AXHPNNQBMnDix0+Ot+P7ggw+ubsNqYNSoUUDhcEE12DTdepyqfeGFFwIwbdq0To/75S9/CcS1EJMyYBERJxXJgO2G2SmnnAKkWevq1atL/oz8DPeTsoF5SBdsqaWVK1cC8PzzzwPwk5/8BIA33nij4NhVq1YBabaXz24wWpZfKitMr0d289Uys47/n1muvvpqAM4+++zqNqwG7AZaLTLfRtBVqeHxxx8PVC62VJIyYBERJxXJgC1bscw3BlaKBT7jgY8//jiQ/vb9JGL8zV0NHce4zzrrLKDrzPf0008HYMyYMUDz9FWlHXfccd5NKMvy5cvbn+eXYZoddtgBgD/+8Y9A+VeQtRBfi0REmkRFMmC7W2tT/GwRDA82+WP27Nntrw0cONCpNZVlWYrdxbXx5Keeeirz+Isuuqg2DasQq56BdMJAMXZlMXnyZCCuO9uf1LnnngsUnzSRxc4Nq3zJN3bs2E7//s9+9rNymujGst2OV7U2BTnfVVddBXSd+S5cuBBI79uUon///kC62NPWUgYsIuKkonXAlo3Y9E+rACjFr371q07ft2m3N9xwQ6fHnXbaaYB/1ms10LYwyNy5c4G00qFv377tx9pv8379+gFwySWX5HyWVYjYNFH7jf70008D6RYr+XbfffdP+K+oDeuT5557ruS/Y9OyLfO17MWmaa9btw5I+/KII44ACjPEnj17tj+3rCYWd999d8FrViFhWXKpusqA62UKsm0jZPMEsrS0tADwrW99K/N9u1dlFTOzZs0C0m2sStGnTx8gnW9w/fXXA7k/16VQBiwi4qSiGbAtbmFjwZVkv2G6usvdcRzRk2WlDz30EJDORrPMzO7QwtYvWnTjjTd+kiZGwxbQtkWcstj/u51bdhVgswjtSqNYzfedd96Z+XrHapmHH34YgD333BOAbbfdtrR/QA2Vm/kWW5TH1Fv1QylGjBgB5F7dAJx00kkALF68GIB58+Zt9fewLPpPf/oTAF/96leB8uOPMmAREScNsyD7McccA0CvXr2cW5Jt+PDhFfusTZs2AcWrHxqRXTH8/Oc/z3n92GOPBbZ+tmPHelLLnKy2uKv7DfWgq7HfYlUT9cgqEmbOnAmk905s3Pj222/PeT1/9m3+1zvuuCMA7777bpff2yqOlAGLiNSJ6DNg21K6q/UQbGxw++23r03DHFlFRT2vcvZJ2ezL9957r+KfbavnSX2xtWfsnLBtq4488sjM4y3TtXtXVgVj4+b7778/kI4Vd7yayK8Z3trF3ZUBi4g4iTYDtjUA2tragDTzLVYF0Qjrv1aK3dm3lcQagdX3vv7660BaA10sA7Z6zPPOOy/zfVs9rRoZdAxsq6KulFtVEbM333wTgAMOOABIx/SLsdpnq8Iptn2VzardvHlz0c+aMWNGWW01yoBFRJxEmwHbyli33HKLc0vik38XN9+nP/1pILfGNWbFanQ7stlyra2tQPF7Afa+1fRavebvfvc7AO666y4grcfuyCotGmEd3q7W0qhXlt12vNeTv+7122+/DcCjjz6a+RlWLTFnzhwgzXztqtuqJmyGXDWrYZQBi4g4iTYDLtX3v/997ybUnN2NLTYebuOb9cLWAem4gp3NVspnu4cUYytbnXDCCUBaKbJ06dIu2/HNb34TgIMOOqjLY2PXVf2v7bFXb/bdd18gt7Kh3CsWWyMlf+alXSWVc9Vtdb+HHHJIWW0wyoBFRJxEnwHbOGexOuBGmsnTlb/97W9A8ezQxn532223mrWpEmz2YsdV3WyN16yx2s7Y8Y888khJxx944IHtz+s1K9wa9V790HGta1tvZf369SX93fvvvz/nsVT28wUwevRoIJ2nsN1225X1WUYZsIiIk2gzYKv/tXHO/DpgW0Gs2CyXRmT7zNnVQD6re6y3DNjcfPPN7c9tLdczzzwTKG+3glLYGhIXX3xx+2u9e/eu6Pfw0NXqZ43is5/9bPtz25HH1k8uNRPuimW8ttv7uHHj2t+r1M+YMmARESfRZsBXXnllp+/bmEszrP1gbI+rZvCNb3wDSK+ELGO1+t5Sde++5RSfPn16zuuWNTXT+QONuf6vVS3YPoG2K47VA5fK6sCtssoy3l133bUi7cyiDFhExIkCsIiIk2iHIKR8559/vncTKs6GCB588EHnlsTNFt9ppgXY89mwlS3YdOuttwKFk7Vs0sRZZ52V8/rXvvY1oLYb+ioDFhFxEl0GPG3aNABuu+22nNfzJ2J8/etfr23D6sDQoUO9myBOSl18x46r94kYnbGrJstw8zPdmCgDFhFxEl0GbMX4+QvN5E/EOPHEE2vaLk+2GHmx5SdFSmWTFSQOyoBFRJxElwFLoRUrVgDpdvQi0hiUAYuIOIkuA7ZphLbNvKQLf9x4441AuiiPLSA9fPhwAAYPHuzQOomBVTVYlUP+IuXNtNRmPVEGLCLiJLoMeMKECTmPkvrud7+b89gsSw9K6VTlUF+UAYuIOAnl1JaGEFYCr1avOVEYkiTJzqUe3CR9AmX0i/okW5P0i/okW2a/lBWARUSkcjQEISLiRAFYRMSJArCIiBMFYBERJwrAIiJOFIBFRJwoAIuIOFEAFhFxogAsIuLk/wHoA+/4DE0wzAAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 10 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        },
        {
          "output_type": "stream",
          "text": [
            "deque([[7, 7, 7, 7, 7, 7, 6, 6, 7, 7]])\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWAAAADSCAYAAAB0FBqGAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAZAElEQVR4nO3deZDUxfnH8fdXBESJxtSiICKgICrljQkhioriQQWPoIF4ETXeIUq8RUUg4FXxAMllIgIaI1ck/jRRRFEpQXSNoikD4UwkIKwnIijo9/cHebZ3rmV2d2a6vzOfVxU1u7Mzs73NTM8z3U8/HcVxjIiIlN52vhsgIlKpNACLiHiiAVhExBMNwCIinmgAFhHxRAOwiIgnGoBFRDwJcgCOouiztH9fRVE0zne7fFKfZFKfZKd+yRRqn2zvuwHZxHHc2r6Ooqg1sAaY6q9F/qlPMqlPslO/ZAq1T4KMgNMMANYCL/tuSEDUJ5nUJ9mpXzIF0ydJGIAHA5Ni7ZmuS32SSX2SnfolUzB9EgXQhpyiKOoILAO6xHG83Hd7QqA+yaQ+yU79kim0Pgk9Aj4XmBtCRwVEfZJJfZKd+iVTUH0S+gB8HjDRdyMCoz7JpD7JTv2SKag+CXYKIoqiXsAsoG0cx+t9tycE6pNM6pPs1C+ZQuyTkCPgwcCMUDoqEOqTTOqT7NQvmYLrk2AjYBGRchdyBCwiUtY0AIuIeKIBWETEEw3AIiKeaAAWEfGkQdXQqqqq4k6dOhWpKWFYsWIFNTU1Ub63r4Q+Aaiurq6J47hNPrdVn2RXCf2i1092uZ4rDRqAO3XqxOuvv164VgWoR48eDbp9JfQJQBRFK/O9rfoku0roF71+ssv1XNEUhIiIJxqARUQ8CfJEDJFcrrrqKgDuv/9+AObNmwdAz549vbVJpLEUAYuIeKIIWBIliqKUy+uvvx6AF1980VubRBpLEbCIiCeKgCURNm7cCCjSNdOnTwfgzDPPTLneqhvOnj0bgO233/oS79ChAwCdO3cuVRMlD4qARUQ8UQQcsC1btgDw9ttvA/Dkk08CcMQRRwAwadIkAB5//HHART8XX3wxAPfddx8ArVq1KlGLi+eLL74A4M033/TcEr82b94MwKhRowA3F27s+759+6Zcb7vN+vfvD7jnRjn59NNPAdh///0BWL16NeBeF8cffzzg/vbu3buXuokZFAGLiHiiCDhAFvmecsopADzzzDP13j49M+D3v/89AL169QJg8ODBRWmnlN7DDz8MuE9F+VqxYgUAkydPBmDIkCEA7LPPPgVrmy8ff/wxAN26dQOgpqYGyHxdPP/88wD07t0bgH/+858AtGmTdzmPglMELCLiiSLgAH399dfAtiPfdPZOb3Ne48aNAxQBJ92qVatqv77zzjub9FgWLdp86HPPPQckLxL+6quvar++/PLLARf5bov1gRUOsj7o2rVrIZuYF0XAIiKeaAAWEfHE2xTEHXfcAUB1dTUA//rXvwB46623Um53wQUXAHDwwQcD8KMf/QiAX/7ylwAcffTRAJx88slFbnHp2KaDdG3btgXgyCOPBNzUQnqamaUaffjhh8VqopTQRRddVPv18uXLC/KY//73vwG3IGUbXLp06VKQxy+2KVOm1H5taZgN9d577wFw3XXXAfDYY48BsMMOOzSxdflTBCwi4knJI+DFixcDcOuttwIu5cqkJ5ZPmDAh5XtLQP/ggw8A9661cmWDDicI2p///Oes1y9YsACA9u3b13v/ww47DHDv8JJML730EgAvv/xy0X7HmjVrAPdJ0lKzvvGNbxTtd4bmL3/5CwDXXnst4BavS0ERsIiIJyWPgC26S49882WRr7HUE4v29txzzya0Lgy23dScc845AFRVVaX8vHnz5oBLO5sxYwYAr776asr9ytl2222NIWyr8u233w64T1KWouQz2b6xbA7/888/b/JjnXDCCYBLafvHP/6R8nOLhG0+9Sc/+UmTf6dv9txo0aIFAK1btwbghhtuAODBBx8EYNGiRQA8++yzgNvSvPPOOxe/jUX/DSIikpW3OWBjK/oTJ04E4IknngBc1sMjjzwCwJ/+9CcANm3alHJ/yxiwSLgcIuAxY8akfG99YJe777474CJc65Px48en3O/KK68sajtDYifrjhw5MuX63/zmN4DbhmqFWkK2bNkywGUANYTN/++9996Ai/bs77bXi835pkfCto39hz/8Ye11pYgEC8nWSE477TQARowYAbh5bSvROXToUAAuueQSwP3to0ePBpq+6SUfioBFRDzxlgds8zFTp04FXFRn70qmT58+gMt5PfbYY0vVRG8uvPBCAIYPH5715++//z7gcqHTWVGS9LnkcmSffG677basP7e+sufNnDlzANhvv/2K3rbGsr/lk08+yfs+f/zjHwE47rjjALdekM5yXG2bu0XCS5cuBeC1114D4Kyzzqq9j63b2JpD6ObOnQvAXnvtlXK9zXPb32peeeWVlO8fffRRwEXCFjEXgyJgERFPvEXAn332GQA//vGPAVccpGXLloB797KSjLvuumuJW+jPgAEDgNwR8LbYTrjDDz+8YG0Klc1hps9lplu7di3g5kRtJ5WtkIfE5q0tEqvPd7/7XQBOPfVUIP9dXO3atQPcXK9lj5i//vWvtV9b4ZuQIuD6DhmwbCBbI7E1FZvj3bBhQ72PnV7IvZgUAYuIeFLyCPj0008H3A43m4uyS5u/NHa0Sq5o0PI7991338I31pOklQZMEtv1NHbsWACuueYan83JKj1/9csvv8x52yuuuAJofP0CK8yeHgHXZa/Vyy67rFG/oxgs4gfXB5YFdPXVV6dcNpaNSd///veb9Dj1UQQsIuJJySPg733ve4B717IjVmxexnIgDzjgAAD69esHuGpNlh9sVdPuuusuAHbcccdiNz0xlixZArjC7hZRVZKTTjoJgIceeghwzyM71NMivksvvbT2PpaZ45tFs1bnxNYE6tptt90AOPDAA4venvnz5wNhRcB1a8bcdNNNQGYefFNZNpLtXdhll10K+vigCFhExJuSR8CWzWAVh4YNGwa4iPadd94B3EqksRV9q1dqe/xtt0s5Sc+FNpbbaZ8aLLd14cKFKd/bbkKbu7N38kpi0dq0adMAV+XLfPTRR0Dq0TZJYrUv1q9f36THqW/uNyns04AdN3/LLbcATe8byzH/9a9/Dbi6wVC4T5WKgEVEPPF+KKed8mCXJ554Yr23t7oHFgHfe++9QO6dUEliNY2tLoaxXYLz5s0DoGPHjik///a3vw24nGrLI7WdPJUYAdvOSTuAMYlsfrfuji47ycJ2ydlx85YPnC97rs2cObOpzfTOolHL6Bg4cCDgnv8PPPBAyu3Ta1tY9TNjnxzt04F9Sq/7/1B3p2CT2l6QRxERkQbzHgE3lb1blUMEbHvV08+Es9zV9Mi3kljUYuejWS3XXLYV+Vo2TYg74Yzlg9et4GYRsLEMgO985zsA7LTTTvU+pkV1FvmmP145sDlhO+EiPQK255DN6dp8evr9bQ7YHid9HaEQFAGLiHiS+AjY6h6UI5v71c44N89nOyK3FQHnctBBBwEur7y+mgKhsPlscLuzjEWwXbt2LWmbksDqAtu6kp2paCeybyuTwc7kM/acgcLl2CsCFhHxJPERcHotz3JitWytMr9lOaSzzJC6Fawgs65GObAqXnZmmVW42hbbHWgr2cWs8VpoZ5xxRu3Xlr1w4403+mpOYthuuWbNmgFut609F3LVj7H84VmzZqVcb3WGAd544w0AevTo0aQ2KgIWEfEkOWFADumnJCfZIYccAri9/9OnTwfg7rvvTrnM16BBgwrYujBYVGO5zscccwyQeQK0Rf+2i6lz584p908Si+DA/Z/ayr6dciy52evq6aefBlz9j/Qdp5YZMnv2bCDz/Mm6CnW2oCJgERFPNACLiHiS+CmIcmLHMdnx83ZAYkOT5a2I/dlnn13A1oXF0n9sS2ihtoaGzhYRLTXNpqkWLVpUkMe3gk91tyjbUfdJ9bOf/QyAyZMnAy6drG5aWT7qLnwWqvytImAREU8SHwGvW7cOcOk55bBd17bHvvvuu4BLubIC3eksXeYPf/gDADfffDOQrFQraZhRo0YBcPHFFwNw6KGHAq7MZr7s9WIFoL71rW8B0L1794K0MwR2bJkV57dPiPnq2bMn4A6RgMIt5ioCFhHxJPEhkhWusXf+coiAjR1NY5Gtpb7Yhgs7WNLKFvbp06fUTRTPOnToALjCMZKbpSza68bmhnOxT5KW4mibgApJEbCIiCeJi4AtKd0S7YtRIi40lh1hhaHtUkTyZ5kzNpdbd07XF0XAIiKeJC4CtjJwVqhGRCSpFAGLiHiSuAG4ZcuWtGzZkoEDBzJw4ECiKEpkgRURkcQNwCIi5SKK4zj/G0fROmBl8ZoThI5xHLfJ98YV0ifQgH5Rn2RXIf2iPskua780aAAWEZHC0RSEiIgnGoBFRDzRACwi4okGYBERTzQAi4h4ogFYRMQTDcAiIp5oABYR8UQDsIiIJxqARUQ80QAsIuKJBmAREU80AIuIeKIBWETEEw3AIiKeaAAWEfFEA7CIiCcagEVEPNEALCLiiQZgERFPNACLiHiiAVhExBMNwCIinmgAFhHxRAOwiIgnGoBFRDzRACwi4okGYBERTzQAi4h4ogFYRMQTDcAiIp5oABYR8UQDsIiIJxqARUQ80QAsIuKJBmAREU80AIuIeKIBWETEEw3AIiKeaAAWEfFEA7CIiCcagEVEPNEALCLiiQZgERFPghyAoyj6LO3fV1EUjfPdrhBEUTQoiqJ3oyjaEEXR0iiKjvLdJt/UJ6n0+skUap9s77sB2cRx3Nq+jqKoNbAGmOqvRWGIoqgvcCcwEFgAtPPbIv/UJ5n0+skUap8EOQCnGQCsBV723ZAAjABGxnE8/3/fr/LZmECoT+qn10+mYPokyCmINIOBSXEcx74b4lMURc2AHkCbKIqWRFH0XhRFD0RR1Mp323xRn+RFr59MwfRJFEAbcoqiqCOwDOgSx/Fy3+3xKYqiPdga3VUD/YHNwExgThzHw3y2zRf1Sf30+skUWp+EHgGfC8wNoaMCsPF/l+PiOF4dx3ENcA/Qz2ObfFOf1E+vn0xB9UnoA/B5wETfjQhBHMcfAe8BdT+yhPvxpQTUJ9uk10+moPok2AE4iqJeQHsCWKkMyARgSBRFu0VRtCswFPg/z23yTX2ShV4/mULsk5CzIAYDM+I4Xu+7IQEZBVQBi4FNwBRgtNcW+ac+yU6vn0zB9UnQi3AiIuUs2CkIEZFypwFYRMQTDcAiIp5oABYR8UQDsIiIJw1KQ6uqqoo7depUpKaEYcWKFdTU1ET53r4S+gSgurq6Jo7jNvncVn2SXSX0i14/2eV6rjRoAO7UqROvv/564VoVoB49ejTo9pXQJwBRFK3M97bqk+wqoV/0+sku13NFUxAiIp5oABYR8UQDsIiIJyHXghARyWrLli21Xz/22GMArFqVehjKGWecAUCXLl1K17AGUgQsIuKJtwj4P//5DwD77rsvAM888wwAvXv39tUkSYABAwYA8MQTTwDwwgsvAHreVIqPP/4YgHnz5tVed/7556fcxgqM3X///QC8+OKLgBtrQqIIWETEE28RcIcOHQA45phjAJg2bRqgSEbqF0Vbc/y3225r7HDmmWcC8NxzzwFw4IEH+mlYiQ0fPhyAX/ziF1l//uyzzwJw1FFHAdCiRYvSNKzIFi5cCED//v23edt169YBcPTRRwPuU/ZBBx1UpNY1nCJgERFPgsmCGD9+POAiGnvnFqnPhx9+CEBNTY3nlpTGiBEjABf52ieCdCeeeCIAp59+OgBTpwZzCk+T2LxuQ1gkfMoppwBbt0uHQhGwiIgn3iPgnj17Am7O6sorrwTgjTfe8NamYsoWsVj0by6//PJSNSdxDjjgAABmzpzpuSWls369O8LsV7/6VYPu+9RTTwFu/tMi46T6+uuvAZfpUFd6RoxlWh1xxBEp319wwQUAPPTQQ8VtbB4UAYuIeKIBWETEE+9TEFdccQUAY8eOBdxiyubNmwFo3ry5n4aVkPVBru81ReH8/Oc/B+D2229Pud4WZ4499tiSt6lYli1bBsAJJ5xQe90HH3yQcpuqqioAzj777JTr33zzTQDmzJkDuG25lsbVuXPnwje4BCz9sO5UnqWVdezYMeW2lup69913AzBmzBjATXeuWbMGgLZt2xaxxfVTBCwi4on3CNjewQ877DAAZs+eDUB1dTXgFukqWa4I+Qc/+AHg3tm7detW2oZ5ZIsx5sknn/TUkuKxRaVsaVM77bQTAHPnzgUyC84sXrwYgP333x+AjRs3pjxmUiPgbOzT0J577pn15+eeey7gxpRx48YBcPLJJwPwyiuvANCqVauitjMbRcAiIp54j4CNpYZYBGwpIuUWAVvUOmPGjCY/lj2GXWZLzSk3rVu3BuDqq68G4N577035uUU3l112GQDbbx/MU7zBbHt1tv9Xi+rSI1+LdO2+9knB5k6ffvppAAYPHgxAs2bNCt3sorrxxhsBV5QJ4Pjjjwe2/bfY2ok9R6ywz5dffgkoAhYRqSjBhAeHH3444N7tH3nkEQBuueUWwK1oJt306dOB7An1Fv03Njq2qMB+RzmyiDbXHOZVV10FwEknnQRA165dS9OwIth9992B1BV/W7G/9dZbs96nb9++AMyfPx/IzBqwMp7pc8RJYYd+NvTwT3DlKEePHg3AsGHDALeGMnLkSABatmzZ5HbmSxGwiIgnwUTA9u5k70b27mSr25deeing3tGTLlsu77byey1qzhUp2/d2u0rOFy4HNs9txWQA7rrrLsBFx43Vr18/wOVVDxkypEmPlyS9evUC3KcC+8R43nnnAdC9e/eStaU8RjMRkQQKJgI2Vj7v5ptvBtw78znnnAPAzjvv7KdhAbCI1i5tzrcQGRVJY6UFrSzj2rVrfTanKGzd49FHHy34Y1thmqFDhwKVFQGn75izPOtNmzaVvC2KgEVEPAkuAt5jjz2yXv/f//4XqOwIOJ3Nl1diBNy+fXsAnn/+ecCVqbS815deeglIdhZEY/z0pz8F4J133gFcf1jWxPLlywH47W9/m3I/y7e37IkkswyPXCVtrc6MHV/19ttvl6ZhWSgCFhHxJLgI2HY62U6fyZMnA64A93777eenYQHKVfvBakVUUhZEenaM1QewKmE77LBDydvkw6BBg1Iuc9l7770BN4f+6quvAu4IdzvIMonsE6FlUqVL3x1oVq9eDcAXX3wBlCYfWBGwiIgnwUXA9q5kc1aTJk0C4G9/+xsA119/vZ+GBayQ9SXKxcqVKwF4+OGHAZdHLltZjvGhhx4KuB10FgUm2WuvvQbkPrA0W01hgNNOOw2A2267DYCzzjoLcOsNxYiIFQGLiHgSXARs2rVrB7hq97aq/fe//x1w79wCxx13HFCZEfA+++wDwEUXXQRkru7bqv+WLVuAZFdHK4Y+ffqkfD9q1CgATj311NrrfFQJKwYbS9566616bzd8+HDARcJ2qKf1DcAhhxwCuHrmjaUIWETEk2DDAXvXtUjYcvWuu+46AGbNmuWnYQlSCTUhLKK1EyLS3XPPPQDcdNNNAOyyyy6laVjCWHUxOzXCzqOD0tZGKKZsJ4vkw+aUrcIeuHrK559/PgBHHnlkox5bEbCIiCfBRsDGqjXZSaZLly712RwJlNWRTj8rrtxt2LABcLu7vvnNbzbqcew8NYuAk8xOzbDax+aTTz4B3HPEPmUvWrQIcLU3bJ2pd+/eAHz++ecZv8NO7LEsG8u9t2p1+eadKwIWEfEk+AjY8hP32msvAGpqagBYsmQJkHkmllQmyw+3M+LSdzl9+umnQPnNAVvVQDvrbfz48YCr7WA7S9NZxPzggw8CLoPG+q1uTYikzQEffPDBAPzud78D4JJLLkn5uf2Ntts2/bQdy7BauHAhAO+//z4AY8eOrb3N448/DrhcYltvsblhO+FnWxQBi4h4EnwEbOzdyvau2/dz5swBSnuOU2gsy8HmoSpRmzZtAHe6ts3NGdv5NWXKlJK2q9isJrSd9Ft3pR5cnd/0XV+2lvLUU08BLirccccdgcz84CRp3rw54GqI299iEa3N39533331Po6dO2iXtiMOXP5v+tmOEydOBBQBi4gETwOwiIgniZmCsKOKbApiwYIFgCvUnuuYcqkstgi1atUqoPw37FhZyVxs4ShXYRpjU3i2GFcOryf7m9KnEmyapqHTlnUX62wcsvQ9Y2UB8qUIWETEk8REwJYaYknWd9xxh8/mJEIlFmZv0aIF4BaTTLkeOmmLj3bIppVxnTBhQoMe54YbbgBc2mc5K8SCvR2NZqURGksRsIiIJ4mJgM3o0aNTLkWymTZtmu8mlISlXFnRKkuLsk8/1157LQAvvPAC4A4pvfDCC1Me55prril+YyWDImAREU8SFwFLbjqaSCwitoLh5Z4FknSKgEVEPFEEXEbGjBkDKAIWSQpFwCIinigCLiPdunUDXHFyEQmbImAREU+ihkRLURStA1YWrzlB6BjHcZt8b1whfQIN6Bf1SXYV0i/qk+yy9kuDBmARESkcTUGIiHiiAVhExBMNwCIinmgAFhHxRAOwiIgnGoBFRDzRACwi4okGYBERTzQAi4h48v80OqBhIjbD8gAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 10 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    }
  ]
}