10.Multi-label classification using Neural Networks with a regularization.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "10.Multi-label classification using Neural Networks with a regularization.ipynb의 사본",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8y9YbXjikqbg",
        "colab_type": "text"
      },
      "source": [
        "### **Preparation**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "V0BqPQCyiZQS",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import matplotlib.pyplot as plt\n",
        "import numpy as np\n",
        "import copy\n",
        "import math\n",
        "import csv\n",
        "from google.colab import files\n",
        "from collections import deque"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fO9OGlHqlI6f",
        "colab_type": "text"
      },
      "source": [
        "### **Input data**\n",
        "- load the data file ('mnist.csv')\n",
        "- each row of the data consists of the label `l` and the image pixel values `x` in a vector form\n",
        "- the label is one of the 10 digits from 0 to 9\n",
        "- the image represents its associated label in the grey scale\n",
        "- the number of images is 10,000 and the size of each image is 28x28, $x \\in \\mathbb{R}^{784}$\n",
        "- consider the first 1,000 images for training and the rest 9,000 images for testing\n",
        "- normalise the intensity values of each image so that they ranges from 0 to 1\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HwmL4SD6ieF9",
        "colab_type": "code",
        "outputId": "168cec0b-030d-4b95-c095-d203419b9a11",
        "colab": {
          "resources": {
            "http://localhost:8080/nbextensions/google.colab/files.js": {
              "data": "Ly8gQ29weXJpZ2h0IDIwMTcgR29vZ2xlIExMQwovLwovLyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKLy8geW91IG1heSBub3QgdXNlIHRoaXMgZmlsZSBleGNlcHQgaW4gY29tcGxpYW5jZSB3aXRoIHRoZSBMaWNlbnNlLgovLyBZb3UgbWF5IG9idGFpbiBhIGNvcHkgb2YgdGhlIExpY2Vuc2UgYXQKLy8KLy8gICAgICBodHRwOi8vd3d3LmFwYWNoZS5vcmcvbGljZW5zZXMvTElDRU5TRS0yLjAKLy8KLy8gVW5sZXNzIHJlcXVpcmVkIGJ5IGFwcGxpY2FibGUgbGF3IG9yIGFncmVlZCB0byBpbiB3cml0aW5nLCBzb2Z0d2FyZQovLyBkaXN0cmlidXRlZCB1bmRlciB0aGUgTGljZW5zZSBpcyBkaXN0cmlidXRlZCBvbiBhbiAiQVMgSVMiIEJBU0lTLAovLyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KLy8gU2VlIHRoZSBMaWNlbnNlIGZvciB0aGUgc3BlY2lmaWMgbGFuZ3VhZ2UgZ292ZXJuaW5nIHBlcm1pc3Npb25zIGFuZAovLyBsaW1pdGF0aW9ucyB1bmRlciB0aGUgTGljZW5zZS4KCi8qKgogKiBAZmlsZW92ZXJ2aWV3IEhlbHBlcnMgZm9yIGdvb2dsZS5jb2xhYiBQeXRob24gbW9kdWxlLgogKi8KKGZ1bmN0aW9uKHNjb3BlKSB7CmZ1bmN0aW9uIHNwYW4odGV4dCwgc3R5bGVBdHRyaWJ1dGVzID0ge30pIHsKICBjb25zdCBlbGVtZW50ID0gZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgnc3BhbicpOwogIGVsZW1lbnQudGV4dENvbnRlbnQgPSB0ZXh0OwogIGZvciAoY29uc3Qga2V5IG9mIE9iamVjdC5rZXlzKHN0eWxlQXR0cmlidXRlcykpIHsKICAgIGVsZW1lbnQuc3R5bGVba2V5XSA9IHN0eWxlQXR0cmlidXRlc1trZXldOwogIH0KICByZXR1cm4gZWxlbWVudDsKfQoKLy8gTWF4IG51bWJlciBvZiBieXRlcyB3aGljaCB3aWxsIGJlIHVwbG9hZGVkIGF0IGEgdGltZS4KY29uc3QgTUFYX1BBWUxPQURfU0laRSA9IDEwMCAqIDEwMjQ7Ci8vIE1heCBhbW91bnQgb2YgdGltZSB0byBibG9jayB3YWl0aW5nIGZvciB0aGUgdXNlci4KY29uc3QgRklMRV9DSEFOR0VfVElNRU9VVF9NUyA9IDMwICogMTAwMDsKCmZ1bmN0aW9uIF91cGxvYWRGaWxlcyhpbnB1dElkLCBvdXRwdXRJZCkgewogIGNvbnN0IHN0ZXBzID0gdXBsb2FkRmlsZXNTdGVwKGlucHV0SWQsIG91dHB1dElkKTsKICBjb25zdCBvdXRwdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQob3V0cHV0SWQpOwogIC8vIENhY2hlIHN0ZXBzIG9uIHRoZSBvdXRwdXRFbGVtZW50IHRvIG1ha2UgaXQgYXZhaWxhYmxlIGZvciB0aGUgbmV4dCBjYWxsCiAgLy8gdG8gdXBsb2FkRmlsZXNDb250aW51ZSBmcm9tIFB5dGhvbi4KICBvdXRwdXRFbGVtZW50LnN0ZXBzID0gc3RlcHM7CgogIHJldHVybiBfdXBsb2FkRmlsZXNDb250aW51ZShvdXRwdXRJZCk7Cn0KCi8vIFRoaXMgaXMgcm91Z2hseSBhbiBhc3luYyBnZW5lcmF0b3IgKG5vdCBzdXBwb3J0ZWQgaW4gdGhlIGJyb3dzZXIgeWV0KSwKLy8gd2hlcmUgdGhlcmUgYXJlIG11bHRpcGxlIGFzeW5jaHJvbm91cyBzdGVwcyBhbmQgdGhlIFB5dGhvbiBzaWRlIGlzIGdvaW5nCi8vIHRvIHBvbGwgZm9yIGNvbXBsZXRpb24gb2YgZWFjaCBzdGVwLgovLyBUaGlzIHVzZXMgYSBQcm9taXNlIHRvIGJsb2NrIHRoZSBweXRob24gc2lkZSBvbiBjb21wbGV0aW9uIG9mIGVhY2ggc3RlcCwKLy8gdGhlbiBwYXNzZXMgdGhlIHJlc3VsdCBvZiB0aGUgcHJldmlvdXMgc3RlcCBhcyB0aGUgaW5wdXQgdG8gdGhlIG5leHQgc3RlcC4KZnVuY3Rpb24gX3VwbG9hZEZpbGVzQ29udGludWUob3V0cHV0SWQpIHsKICBjb25zdCBvdXRwdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQob3V0cHV0SWQpOwogIGNvbnN0IHN0ZXBzID0gb3V0cHV0RWxlbWVudC5zdGVwczsKCiAgY29uc3QgbmV4dCA9IHN0ZXBzLm5leHQob3V0cHV0RWxlbWVudC5sYXN0UHJvbWlzZVZhbHVlKTsKICByZXR1cm4gUHJvbWlzZS5yZXNvbHZlKG5leHQudmFsdWUucHJvbWlzZSkudGhlbigodmFsdWUpID0+IHsKICAgIC8vIENhY2hlIHRoZSBsYXN0IHByb21pc2UgdmFsdWUgdG8gbWFrZSBpdCBhdmFpbGFibGUgdG8gdGhlIG5leHQKICAgIC8vIHN0ZXAgb2YgdGhlIGdlbmVyYXRvci4KICAgIG91dHB1dEVsZW1lbnQubGFzdFByb21pc2VWYWx1ZSA9IHZhbHVlOwogICAgcmV0dXJuIG5leHQudmFsdWUucmVzcG9uc2U7CiAgfSk7Cn0KCi8qKgogKiBHZW5lcmF0b3IgZnVuY3Rpb24gd2hpY2ggaXMgY2FsbGVkIGJldHdlZW4gZWFjaCBhc3luYyBzdGVwIG9mIHRoZSB1cGxvYWQKICogcHJvY2Vzcy4KICogQHBhcmFtIHtzdHJpbmd9IGlucHV0SWQgRWxlbWVudCBJRCBvZiB0aGUgaW5wdXQgZmlsZSBwaWNrZXIgZWxlbWVudC4KICogQHBhcmFtIHtzdHJpbmd9IG91dHB1dElkIEVsZW1lbnQgSUQgb2YgdGhlIG91dHB1dCBkaXNwbGF5LgogKiBAcmV0dXJuIHshSXRlcmFibGU8IU9iamVjdD59IEl0ZXJhYmxlIG9mIG5leHQgc3RlcHMuCiAqLwpmdW5jdGlvbiogdXBsb2FkRmlsZXNTdGVwKGlucHV0SWQsIG91dHB1dElkKSB7CiAgY29uc3QgaW5wdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQoaW5wdXRJZCk7CiAgaW5wdXRFbGVtZW50LmRpc2FibGVkID0gZmFsc2U7CgogIGNvbnN0IG91dHB1dEVsZW1lbnQgPSBkb2N1bWVudC5nZXRFbGVtZW50QnlJZChvdXRwdXRJZCk7CiAgb3V0cHV0RWxlbWVudC5pbm5lckhUTUwgPSAnJzsKCiAgY29uc3QgcGlja2VkUHJvbWlzZSA9IG5ldyBQcm9taXNlKChyZXNvbHZlKSA9PiB7CiAgICBpbnB1dEVsZW1lbnQuYWRkRXZlbnRMaXN0ZW5lcignY2hhbmdlJywgKGUpID0+IHsKICAgICAgcmVzb2x2ZShlLnRhcmdldC5maWxlcyk7CiAgICB9KTsKICB9KTsKCiAgY29uc3QgY2FuY2VsID0gZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgnYnV0dG9uJyk7CiAgaW5wdXRFbGVtZW50LnBhcmVudEVsZW1lbnQuYXBwZW5kQ2hpbGQoY2FuY2VsKTsKICBjYW5jZWwudGV4dENvbnRlbnQgPSAnQ2FuY2VsIHVwbG9hZCc7CiAgY29uc3QgY2FuY2VsUHJvbWlzZSA9IG5ldyBQcm9taXNlKChyZXNvbHZlKSA9PiB7CiAgICBjYW5jZWwub25jbGljayA9ICgpID0+IHsKICAgICAgcmVzb2x2ZShudWxsKTsKICAgIH07CiAgfSk7CgogIC8vIENhbmNlbCB1cGxvYWQgaWYgdXNlciBoYXNuJ3QgcGlja2VkIGFueXRoaW5nIGluIHRpbWVvdXQuCiAgY29uc3QgdGltZW91dFByb21pc2UgPSBuZXcgUHJvbWlzZSgocmVzb2x2ZSkgPT4gewogICAgc2V0VGltZW91dCgoKSA9PiB7CiAgICAgIHJlc29sdmUobnVsbCk7CiAgICB9LCBGSUxFX0NIQU5HRV9USU1FT1VUX01TKTsKICB9KTsKCiAgLy8gV2FpdCBmb3IgdGhlIHVzZXIgdG8gcGljayB0aGUgZmlsZXMuCiAgY29uc3QgZmlsZXMgPSB5aWVsZCB7CiAgICBwcm9taXNlOiBQcm9taXNlLnJhY2UoW3BpY2tlZFByb21pc2UsIHRpbWVvdXRQcm9taXNlLCBjYW5jZWxQcm9taXNlXSksCiAgICByZXNwb25zZTogewogICAgICBhY3Rpb246ICdzdGFydGluZycsCiAgICB9CiAgfTsKCiAgaWYgKCFmaWxlcykgewogICAgcmV0dXJuIHsKICAgICAgcmVzcG9uc2U6IHsKICAgICAgICBhY3Rpb246ICdjb21wbGV0ZScsCiAgICAgIH0KICAgIH07CiAgfQoKICBjYW5jZWwucmVtb3ZlKCk7CgogIC8vIERpc2FibGUgdGhlIGlucHV0IGVsZW1lbnQgc2luY2UgZnVydGhlciBwaWNrcyBhcmUgbm90IGFsbG93ZWQuCiAgaW5wdXRFbGVtZW50LmRpc2FibGVkID0gdHJ1ZTsKCiAgZm9yIChjb25zdCBmaWxlIG9mIGZpbGVzKSB7CiAgICBjb25zdCBsaSA9IGRvY3VtZW50LmNyZWF0ZUVsZW1lbnQoJ2xpJyk7CiAgICBsaS5hcHBlbmQoc3BhbihmaWxlLm5hbWUsIHtmb250V2VpZ2h0OiAnYm9sZCd9KSk7CiAgICBsaS5hcHBlbmQoc3BhbigKICAgICAgICBgKCR7ZmlsZS50eXBlIHx8ICduL2EnfSkgLSAke2ZpbGUuc2l6ZX0gYnl0ZXMsIGAgKwogICAgICAgIGBsYXN0IG1vZGlmaWVkOiAkewogICAgICAgICAgICBmaWxlLmxhc3RNb2RpZmllZERhdGUgPyBmaWxlLmxhc3RNb2RpZmllZERhdGUudG9Mb2NhbGVEYXRlU3RyaW5nKCkgOgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAnbi9hJ30gLSBgKSk7CiAgICBjb25zdCBwZXJjZW50ID0gc3BhbignMCUgZG9uZScpOwogICAgbGkuYXBwZW5kQ2hpbGQocGVyY2VudCk7CgogICAgb3V0cHV0RWxlbWVudC5hcHBlbmRDaGlsZChsaSk7CgogICAgY29uc3QgZmlsZURhdGFQcm9taXNlID0gbmV3IFByb21pc2UoKHJlc29sdmUpID0+IHsKICAgICAgY29uc3QgcmVhZGVyID0gbmV3IEZpbGVSZWFkZXIoKTsKICAgICAgcmVhZGVyLm9ubG9hZCA9IChlKSA9PiB7CiAgICAgICAgcmVzb2x2ZShlLnRhcmdldC5yZXN1bHQpOwogICAgICB9OwogICAgICByZWFkZXIucmVhZEFzQXJyYXlCdWZmZXIoZmlsZSk7CiAgICB9KTsKICAgIC8vIFdhaXQgZm9yIHRoZSBkYXRhIHRvIGJlIHJlYWR5LgogICAgbGV0IGZpbGVEYXRhID0geWllbGQgewogICAgICBwcm9taXNlOiBmaWxlRGF0YVByb21pc2UsCiAgICAgIHJlc3BvbnNlOiB7CiAgICAgICAgYWN0aW9uOiAnY29udGludWUnLAogICAgICB9CiAgICB9OwoKICAgIC8vIFVzZSBhIGNodW5rZWQgc2VuZGluZyB0byBhdm9pZCBtZXNzYWdlIHNpemUgbGltaXRzLiBTZWUgYi82MjExNTY2MC4KICAgIGxldCBwb3NpdGlvbiA9IDA7CiAgICB3aGlsZSAocG9zaXRpb24gPCBmaWxlRGF0YS5ieXRlTGVuZ3RoKSB7CiAgICAgIGNvbnN0IGxlbmd0aCA9IE1hdGgubWluKGZpbGVEYXRhLmJ5dGVMZW5ndGggLSBwb3NpdGlvbiwgTUFYX1BBWUxPQURfU0laRSk7CiAgICAgIGNvbnN0IGNodW5rID0gbmV3IFVpbnQ4QXJyYXkoZmlsZURhdGEsIHBvc2l0aW9uLCBsZW5ndGgpOwogICAgICBwb3NpdGlvbiArPSBsZW5ndGg7CgogICAgICBjb25zdCBiYXNlNjQgPSBidG9hKFN0cmluZy5mcm9tQ2hhckNvZGUuYXBwbHkobnVsbCwgY2h1bmspKTsKICAgICAgeWllbGQgewogICAgICAgIHJlc3BvbnNlOiB7CiAgICAgICAgICBhY3Rpb246ICdhcHBlbmQnLAogICAgICAgICAgZmlsZTogZmlsZS5uYW1lLAogICAgICAgICAgZGF0YTogYmFzZTY0LAogICAgICAgIH0sCiAgICAgIH07CiAgICAgIHBlcmNlbnQudGV4dENvbnRlbnQgPQogICAgICAgICAgYCR7TWF0aC5yb3VuZCgocG9zaXRpb24gLyBmaWxlRGF0YS5ieXRlTGVuZ3RoKSAqIDEwMCl9JSBkb25lYDsKICAgIH0KICB9CgogIC8vIEFsbCBkb25lLgogIHlpZWxkIHsKICAgIHJlc3BvbnNlOiB7CiAgICAgIGFjdGlvbjogJ2NvbXBsZXRlJywKICAgIH0KICB9Owp9CgpzY29wZS5nb29nbGUgPSBzY29wZS5nb29nbGUgfHwge307CnNjb3BlLmdvb2dsZS5jb2xhYiA9IHNjb3BlLmdvb2dsZS5jb2xhYiB8fCB7fTsKc2NvcGUuZ29vZ2xlLmNvbGFiLl9maWxlcyA9IHsKICBfdXBsb2FkRmlsZXMsCiAgX3VwbG9hZEZpbGVzQ29udGludWUsCn07Cn0pKHNlbGYpOwo=",
              "ok": true,
              "headers": [
                [
                  "content-type",
                  "application/javascript"
                ]
              ],
              "status": 200,
              "status_text": ""
            }
          },
          "base_uri": "https://localhost:8080/",
          "height": 78
        }
      },
      "source": [
        "uploaded = files.upload()"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "     <input type=\"file\" id=\"files-014da8a9-dc3e-4efc-adf4-cd1b3d56ad4c\" name=\"files[]\" multiple disabled />\n",
              "     <output id=\"result-014da8a9-dc3e-4efc-adf4-cd1b3d56ad4c\">\n",
              "      Upload widget is only available when the cell has been executed in the\n",
              "      current browser session. Please rerun this cell to enable.\n",
              "      </output>\n",
              "      <script src=\"/nbextensions/google.colab/files.js\"></script> "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Saving mnist.csv to mnist.csv\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JmV8XN4XifFM",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "file_data   = \"mnist.csv\"\n",
        "handle_file = open(file_data, \"r\")\n",
        "data        = handle_file.readlines()\n",
        "handle_file.close()\n",
        "\n",
        "size_row    = 28    # height of the image\n",
        "size_col    = 28    # width of the image\n",
        "\n",
        "num_image   = len(data)   #10000개\n",
        "count       = 0     # count for the number of images\n",
        "\n",
        "#\n",
        "# normalize the values of the input data to be [0, 1]\n",
        "#\n",
        "def normalize(data):\n",
        "\n",
        "    data_normalized = (data - min(data)) / (max(data) - min(data))\n",
        "\n",
        "    return(data_normalized)\n",
        "\n",
        "#\n",
        "# example of distance function between two vectors x and y\n",
        "#\n",
        "def distance(x, y):\n",
        "\n",
        "    d = (x - y) ** 2\n",
        "    s = np.sum(d)\n",
        "    # r = np.sqrt(s)\n",
        "\n",
        "    return(s)\n",
        "\n",
        "#\n",
        "# make a matrix each column of which represents an images in a vector form\n",
        "#\n",
        "train_image  = np.empty((size_row * size_col, 1000))\n",
        "train_label  = np.empty(1000, dtype=int)\n",
        "test_image  = np.empty((size_row * size_col, 9000))\n",
        "test_label  = np.empty(9000, dtype=int)\n",
        "\n",
        "for line in data:\n",
        "    if count < 1000 :\n",
        "      line_data   = line.split(',')\n",
        "      label       = line_data[0]\n",
        "      im_vector   = np.asfarray(line_data[1:])\n",
        "      im_vector   = normalize(im_vector)\n",
        "\n",
        "      train_label[count]       = label       # size : 10000\n",
        "      train_image[:, count]    = im_vector   # size : 784 * 10000\n",
        "\n",
        "    else: \n",
        "      line_data   = line.split(',')\n",
        "      label       = line_data[0]\n",
        "      im_vector   = np.asfarray(line_data[1:])\n",
        "      im_vector   = normalize(im_vector)\n",
        "\n",
        "      test_label[count-1000]       = label       # size : 10000\n",
        "      test_image[:, count-1000]    = im_vector   # size : 784 * 10000\n",
        "\n",
        "    count += 1\n",
        "\n",
        "train_costs = deque()\n",
        "test_costs = deque()\n",
        "theta_0s = deque()\n",
        "theta_1s = deque()\n",
        "theta_2s = deque()\n",
        "bias_0s = deque()\n",
        "bias_1s = deque()\n",
        "bias_2s = deque()\n",
        "iters = deque()\n",
        "train_accuracys = deque()\n",
        "test_accuracys = deque()\n",
        "incorrect_predicts = deque()\n",
        "\n",
        "#correct_labels = []\n",
        "correct_labels = deque()\n",
        "#incorrect_labels = []\n",
        "incorrect_labels = deque()\n",
        "\n",
        "#print(num_image)  #10000\n",
        "#print(len(train_image))  #784\n",
        "#print(len(train_image[0]))  #1000\n",
        "#print(len(train_label))  #1000\n",
        "#print(len(test_image))  #784\n",
        "#print(len(test_image[0]))  #9000\n",
        "#print(len(test_label))  #9000\n",
        "#print(train_image.shape)\n",
        "\n",
        "train_length = len(train_label)\n",
        "train_l = np.zeros((10,1000))\n",
        "for i in range(0,train_length):\n",
        "  index = train_label[i]\n",
        "  a = np.zeros(10)\n",
        "  a[index] = 1\n",
        "  train_l[:,i] = a\n",
        "#print(train_l)\n",
        "#print(train_l.shape)\n",
        "\n",
        "test_length = len(test_label)\n",
        "test_l = np.zeros((10,9000))\n",
        "for i in range(0,test_length):\n",
        "  index = test_label[i]\n",
        "  a = np.zeros(10)\n",
        "  a[index] = 1\n",
        "  test_l[:,i] = a\n",
        "#print(test_l)\n",
        "#print(test_l.shape)\n",
        "#print(test_l[:,8999])\n",
        "#list_label 0~5999까지 training data, 6000~9999까지 test data\n",
        "#list_image [:,5999]까지 training data, [:,9999]까지 test data"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KY-UL4ONlv9w",
        "colab_type": "text"
      },
      "source": [
        "### **Neural Network Architecture**\n",
        "- build a neural network for the multi-label classification with 10 labels\n",
        "- construct a neural network that consists of an input layer, an output layer and a sequence of hidden layers\n",
        "- each hidden layer is defined by a logistic unit\n",
        "- a logistic unit consists of a fully connected layer with a bias followed by the sigmoid activation function \n",
        "- the dimensions of the input layer and the output layer are defined by:\n",
        "    - input unit : 784 (+ a bias)\n",
        "    - hidden unit : 3136 (+ a bias)\n",
        "    - output unit : 10"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "B_UkBCZFii_k",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def forward(theta, value):\n",
        "  #print(theta.shape, value.shape)\n",
        "  #print(\"Theta\", theta.shape)\n",
        "  #print(\"value\", value.shape)\n",
        "  result = np.matmul(theta, value)\n",
        "  return result"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qVq9jp5iikTi",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def sigmoid(value):\n",
        "  \"\"\"\n",
        "  length = len(value)\n",
        "  for i in range(0, length):\n",
        "    value[i] = 1 / (1 + np.exp(-value[i]))\n",
        "  \"\"\"\n",
        "  #value = value.astype(float)\n",
        "  value = np.multiply(value,-1)\n",
        "  value = np.exp(value)\n",
        "  value = np.add(1,value)\n",
        "  value = np.divide(1, value)\n",
        "  return value"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PsayRgz3ZkGm",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def addBias(value, bias):\n",
        "  \"\"\"\n",
        "  length = len(bias)\n",
        "  for i in range(0, length):\n",
        "    value[i] += bias[i]\n",
        "  \"\"\"\n",
        "  value = np.add(value, bias)\n",
        "  return value"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Vvg44h6kZWZr",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def neuralNet(x, u, v, bias0, bias1):    #data, theta0, theta1, theta2, bias0, bias1, bias2\n",
        "  y_hat = forward(u, x)\n",
        "  y_hat = addBias(y_hat, bias0)\n",
        "  y = sigmoid(y_hat)\n",
        "  z_hat = forward(v, y)\n",
        "  z_hat = addBias(z_hat, bias1)\n",
        "  z = sigmoid(z_hat)\n",
        "  \"\"\"\n",
        "  h_hat = forward(w, z)\n",
        "  h_hat = addBias(h_hat, bias2)\n",
        "  h = sigmoid(h_hat)\n",
        "  \"\"\"\n",
        "  return y, z"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Gu2qj9p-mE8-",
        "colab_type": "text"
      },
      "source": [
        "### **Objective Function**\n",
        "- $J(\\theta) = \\frac{1}{m} \\sum_{i=1}^m \\sum_{k=0}^{9}( - l^{(i)}_k \\log( h^{(i)}_k ) - (1 - l^{(i)}_k) \\log(1 - h^{(i)}_k) ) + \\frac{\\lambda}{2 n} \\sum_{j=1}^n \\theta_j^2$ \n",
        "- $\\theta_j$ denotes a model parameter where $j = 1, 2, \\cdots, n$\n",
        "- $\\lambda$ is a control parameter for the regularization based on the $L_2^2$-norm (weight decay)\n",
        "- $n$ is the total number of all the model parameters over the entire neural network\n",
        "- $h^{(i)}_k$ denotes the $k$-th element of the output layer for $i$-th sample data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sjcnYDFMnCRg",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def objectiveFunction(image, l, lamda, y, z, theta0, theta1, b0, b1):     #list_image[:, count], list_label -> traing과 test 구분해야겠다.!!\n",
        "  length = len(image[0])                    #두는 이유는 trainig 6000, test 4000개  갯수가 달라서\n",
        "  cost1 = np.log(z)      # **** y size : 10x 1000 바꾸기(main, neural net)\n",
        "  cost1 = np.multiply(cost1, l)\n",
        "  cost1 = np.multiply(cost1, -1)\n",
        "  cost2 = np.subtract(1, z)\n",
        "  cost2 = np.log(cost2)\n",
        "  temp = np.subtract(1, l)\n",
        "  cost2 = np.multiply(cost2, temp)\n",
        "  cost = np.subtract(cost1, cost2)\n",
        "  cost = np.divide(cost,length)\n",
        "  cost = np.sum(cost)\n",
        "  cost_r1 = np.multiply(theta0, theta0)\n",
        "  cost_r1 = np.sum(cost_r1)\n",
        "  cost_r2 = np.multiply(theta1, theta1)\n",
        "  cost_r2 = np.sum(cost_r2)\n",
        "  cost_r = np.add(cost_r1, cost_r2)\n",
        "  cost_r = cost_r * lamda / (2*2489984)          #추후에 theta 추가나 값 size 변경시 이곳 변경!!!!!\n",
        "  cost += cost_r\n",
        "  \n",
        "  return cost"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MXmewEAjmeLj",
        "colab_type": "text"
      },
      "source": [
        "### **Gradient Descent (Back-propagation)**\n",
        "- $\\theta_{k}^{(t+1)} = \\theta_{k}^{(t)} - \\alpha \\frac{\\partial J(\\theta^{(t)})}{\\partial \\theta_{k}}$, for all $k$\n",
        "- you should choose a learning rate $\\alpha$ in such a way that the convergence is achieved\n",
        "- you can use random initial conditions $\\theta_k^{(0)}$ for all $k$ following a normal distribution with mean 0 and stardard deviation some number"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tzRWAQQG0uxJ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def gradientDescent(image, l, lamda, learning_rate, y, z, theta0, theta1, b0, b1):\n",
        "  lr = learning_rate\n",
        "  length = len(image[0])\n",
        "  x_transpose = image.T\n",
        "  y_transpose = y.T\n",
        "\n",
        "  #######theta1\n",
        "  partial_1 = np.subtract(z, l)\n",
        "  partial_theta1 = np.matmul(partial_1, y_transpose)\n",
        "  regular_theta1 = np.sum(theta1)\n",
        "  regular_theta1 = np.multiply(regular_theta1, lamda)\n",
        "  regular_theta1 = np.divide(regular_theta1, 31360) \n",
        "  partial_theta1 = np.add(partial_theta1, regular_theta1)\n",
        "  partial_theta1 = np.multiply(partial_theta1, lr)\n",
        "  theta1 = np.subtract(theta1, partial_theta1)\n",
        "\n",
        "  #######bias1\n",
        "  partial_bias1 = np.ones((1000,1), dtype=float)\n",
        "  partial_bias1 = np.matmul(partial_1, partial_bias1)\n",
        "  partial_bias1 = np.divide(partial_bias1, 1000)      ## 확실치 않음 없애야할수도 있다!\n",
        "  partial_bias1 = np.multiply(partial_bias1, lr)\n",
        "  b1 = np.subtract(b1, partial_bias1)\n",
        "\n",
        "  #######theta0\n",
        "  theta1_transpose = theta1.T\n",
        "  partial_0 = np.matmul(theta1_transpose, partial_1)      #어쩌면 theta1 transose 한 것을 ㄴ허어야할수 있다.\n",
        "  partial_theta0 = np.matmul(partial_0, x_transpose)\n",
        "  regular_theta0 = np.sum(theta0)\n",
        "  regular_theta0 = np.multiply(regular_theta0, lamda)\n",
        "  regular_theta0 = np.divide(regular_theta0, 2458624) \n",
        "  partial_theta0 = np.add(partial_theta0, regular_theta0)\n",
        "  partial_theta0 = np.multiply(partial_theta0, lr)\n",
        "  theta0 = np.subtract(theta0, partial_theta0)\n",
        "\n",
        "  #######bias0\n",
        "  partial_bias0 = np.ones((1000,1), dtype=float)\n",
        "  partial_bias0 = np.matmul(partial_0, partial_bias0)\n",
        "  partial_bias0 = np.divide(partial_bias0, 1000)      ## 확실치 않음 없애야할수도 있다!\n",
        "  partial_bias0 = np.multiply(partial_bias0, lr)\n",
        "  b0 = np.subtract(b0, partial_bias0)\n",
        "\n",
        "  return theta0, b0, theta1, b1"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WN6ghexckeWy",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def checkAccuracy(image, label, z, theta0, b0):\n",
        "  length = len(label)\n",
        "  count = 0\n",
        "  for i in range(0, length):\n",
        "    if label[i] == np.argmax(z[:,i]):\n",
        "      count += 1\n",
        "  accuracy = count / length * 100\n",
        "  return accuracy"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TP9QaVGX2KDJ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def checkCorrect(image, label, z, theta0, b0):\n",
        "  length = len(label)\n",
        "  count = 0\n",
        "  labels = []\n",
        "  for i in range(0, length):\n",
        "    if count == 10:\n",
        "      break\n",
        "    else:\n",
        "      if label[i] == np.argmax(z[:,i]):\n",
        "        labels.append(i)  #index를 labels 리스트에 추가\n",
        "        count += 1\n",
        "  return labels"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TURh4x9O4ZhA",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def checkIncorrect(image, label, z, theta0, b0):\n",
        "  length = len(label)\n",
        "  count = 0\n",
        "  labels = []\n",
        "  predicts = []\n",
        "  for i in range(0, length):\n",
        "    if count == 10:\n",
        "      break\n",
        "    else:\n",
        "      if label[i] != np.argmax(z[:,i]):\n",
        "        labels.append(i)  #index를 labels 리스트에 추가\n",
        "        count += 1\n",
        "        predicts.append(np.argmax(z[:,i]))\n",
        "  return labels, predicts"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ae53uHmoilyj",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def main():\n",
        "  mu = 0\n",
        "  sigma = 0.1\n",
        "  lamda = 0.01\n",
        "\n",
        "  theta_0 = np.random.normal(mu, sigma, size=(3136,784))   #size = 196x784\n",
        "  theta_1 = np.random.normal(mu, sigma, size=(10,3136))    #size = 49x196\n",
        "  #theta_2 = np.random.normal(mu, sigma, size=(10,49))     #size = 10x49\n",
        "  \n",
        "  b_0 = np.random.normal(mu, sigma, size=(3136,1))  \n",
        "  b_1 = np.random.normal(mu, sigma, size=(10,1))                  \n",
        "  #b_2 = np.random.normal(mu, sigma, 10)                   #size = 10x1\n",
        "\n",
        "  lr = 0.00005                                                #learning rate\n",
        "\n",
        "  #training step\n",
        "  for iter in range(0, 580):\n",
        "    iters.append(iter)\n",
        "    y, z = neuralNet(train_image, theta_0, theta_1, b_0, b_1)\n",
        "    #print(\"y\",y)\n",
        "    train_cost = objectiveFunction(train_image, train_l, lamda, y, z, theta_0, theta_1, b_0, b_1)  #print(cost)\n",
        "    #print(\"traincost\",train_cost)\n",
        "    #print(\"##########################################111111111111111111111111111111\",iter)\n",
        "    train_costs.append(train_cost)\n",
        "    theta_0, b_0, theta_1, b_1 = gradientDescent(train_image, train_l, lamda, lr, y, z, theta_0, theta_1, b_0, b_1)\n",
        "    #print(\"b0\", b_0)\n",
        "    #############train accuracy 하자!############\n",
        "    train_accuracy = checkAccuracy(train_image, train_label, z, theta_0, b_0)\n",
        "    #print(\"TRAIN_ACC :\",train_accuracy)\n",
        "    train_accuracys.append(train_accuracy)\n",
        "    #print(\"##########################################12222222222\")\n",
        "    correct_label = checkCorrect(test_image, test_label, z, theta_0, b_0)\n",
        "    correct_labels.append(correct_label)\n",
        "\n",
        "    #########test\n",
        "    y, z = neuralNet(test_image, theta_0, theta_1, b_0, b_1)\n",
        "    test_cost = objectiveFunction(test_image, test_l, lamda, y, z, theta_0, theta_1, b_0, b_1)\n",
        "    #print(\"##########################################33333333333\")\n",
        "    test_costs.append(test_cost)\n",
        "    #############test accuracy\n",
        "    test_accuracy = checkAccuracy(test_image, test_label, z, theta_0, b_0)\n",
        "    #print(\"TEST_ACC :\",test_accuracy)\n",
        "    test_accuracys.append(test_accuracy)\n",
        "    incorrect_label, predict = checkIncorrect(test_image, test_label, z, theta_0, b_0)\n",
        "    incorrect_labels.append(incorrect_label)\n",
        "    incorrect_predicts.append(predict)\n",
        "    #train_y, train_z, train_h = neuralNet(train_image, theta_0, theta_1, theta_2, b_0, b_1, b_2) #image 문제\n",
        "    #print(\"##########################################4444444444444444444\")\n",
        "    theta_0s.append(theta_0)\n",
        "    bias_0s.append(b_0)\n",
        "    \n",
        "  ######correct label 가져오자!\n",
        "  #correct_labels = checkCorrect(test_image, test_label, theta_0, theta_1, theta_2, b_0, b_1, b_2)  #data의 index만 헐당!\n",
        "\n",
        "  #incorrect_labels = checkIncorrect(test_image, test_label, theta_0, theta_1, theta_2, b_0, b_1, b_2)\n",
        "\n",
        "\n",
        "if __name__ == \"__main__\":\n",
        "  main()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pWq0o7J-Z57Z",
        "colab_type": "text"
      },
      "source": [
        "<br>\n",
        "<br>\n",
        "<br>\n",
        "<br>\n",
        "<br>\n",
        "<br>\n",
        "<br>\n",
        "<br>\n",
        "<br>\n",
        "\n",
        "                 \n",
        "                   "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "t3U7TGl2icNm",
        "colab_type": "text"
      },
      "source": [
        "### **1. Plot the loss curve**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_TpBzAaCiVhd",
        "colab_type": "code",
        "outputId": "bfccb0c2-2679-4b92-f5fc-1fbc8fc8cf5e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 281
        }
      },
      "source": [
        "fig = plt.figure()\n",
        "#print(train_costs)\n",
        "graph1 = plt.plot(iters, train_costs, 'b')\n",
        "graph2 = plt.plot(iters, test_costs, 'r')\n",
        "plt.xlabel('iterations')\n",
        "plt.ylabel('loss')\n",
        "plt.show()"
      ],
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYQAAAEICAYAAABfz4NwAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAdgklEQVR4nO3de5ScdZ3n8fenu3OFAAlpEAiYyN0LF7dREHS5DUbEC4MMsiurC+dk1uMgwzqyQdeZnbN7RlZcL6xz0BxhkJGDowGVA8hFhGF0gdAJt0C4hluA0B0QEhIT0t3f/eP3VHel0131dKerquupz+uc5zxVz/NU/X5PaPrT3+f3XBQRmJmZtTW6A2ZmNjk4EMzMDHAgmJlZxoFgZmaAA8HMzDIOBDMzA2oYCJKulNQjaeWw5edLelzSo5K+Vav2zcxsbDpq+N1XAT8Ari4tkHQC8Cng8IjYImmPPF80d+7cmD9/fi36aGZWWMuXL18XEZ15t69ZIETE3ZLmD1v8ReCSiNiSbdOT57vmz59Pd3f3xHbQzKzgJD0/lu3rPYZwEPBhSfdJ+ldJR9W5fTMzG0UtDxmN1t4c4GjgKODnkt4VI9w/Q9IiYBHAfvvtV9dOmpm1onpXCGuA6yNZBgwAc0faMCKWRERXRHR1duY+BGZmZuNU70D4FXACgKSDgKnAujr3wczMRlCzQ0aSrgWOB+ZKWgP8HXAlcGV2KurbwOdHOlxkZmb1V8uzjM4eZdXnatWmmZmNn69UNjMzoBUCYelSWOdhCjOzaoodCGvXwplnwhlnNLonZmaTXrEDYeNGAFbf/WKDO2JmNvkVOxC2bk0zpjS4I2Zmk1+xA6GvL83qfkG2mVnzKXYglFUIvtrBzKyyYgfC228DKRAGBhrcFzOzSa7YgbB5M5AOGfX3N7gvZmaTXLEDYcsWIFUIDgQzs8qKHQiuEMzMcit2ILhCMDPLrdiB4ArBzCw3B4KZmQFFDwQfMjIzy63YgZBVCA4EM7Pqih0IWYXgQ0ZmZtXVLBAkXSmpJ3tc5vB1X5EUkubWqn1gsEIYoM1XKpuZVVHLCuEqYOHwhZL2BU4BXqhh20kWCO30u0IwM6uiZoEQEXcDr4+w6rvARUDtbzeXHTJyIJiZVVfXMQRJnwJeioiH6tLgggWAA8HMLI+6BYKkmcDXgL/Nuf0iSd2Sunt7e8fX6IUXspL3OBDMzHKoZ4WwP7AAeEjSc8A8YIWkd4y0cUQsiYiuiOjq7Owcd6P9tDsQzMxyqNujxCLiEWCP0vssFLoiYl0t23UgmJnlU8vTTq8F7gEOlrRG0nm1aquSPjrooM+BYGZWRc0qhIg4u8r6+bVqu5wrBDOzfIp9pTIOBDOzvBwIZmYGOBDMzCxT6ECIcCCYmeVV6EDYutWBYGaWV6EDoa/PgWBmlpcDwczMgIIHwsDAUCD4eQhmZpW1TCC4QjAzq6zQgeCzjMzM8it0ILhCMDPLr9CBEOGb25mZ5VXoQHCFYGaWX6EDwWMIZmb5FToQXCGYmeVX6EBwhWBmll+hA8EVgplZfoUOBFcIZmb51fKZyldK6pG0smzZpZIel/SwpF9K2q1W7YMrBDOzsahlhXAVsHDYstuB90bEYcCTwMU1bN8VgpnZGNQsECLibuD1Yctui4i+7O29wLxatQ+uEMzMxqKRYwjnAr8ZbaWkRZK6JXX39vaOq4FShdDGgAPBzKyKhgSCpK8DfcA1o20TEUsioisiujo7O8fVzsAADNBGhysEM7OqOurdoKQvAKcBJ0VE1LKtiBQIAP19AaiWzZmZNbW6BoKkhcBFwL+PiE21bq80hgAQ/QOQvTYzs+3V8rTTa4F7gIMlrZF0HvADYBZwu6QHJf2wVu3DthVC9PmYkZlZJTWrECLi7BEWX1Gr9kZSXiEM9PkZmmZmlRT+SuVShTCw1RWCmVklhQ6E7ccQzMxsNIUOBI8hmJnlV+hA8BiCmVl+hQ4EVwhmZvkVOhBcIZiZ5VfoQHCFYGaWX6EDwWcZmZnlV+hAcIVgZpZfoQPBFYKZWX6FDgRXCGZm+RU6EFwhmJnlV+hAcIVgZpZfoQOhvEJgwBWCmVklhQ6E8grBz9A0M6us0IHgMQQzs/wKHQjbjCG4QjAzq6iWj9C8UlKPpJVly+ZIul3SU9l8dq3ah2FjCK4QzMwqqmWFcBWwcNiyxcAdEXEgcEf2vmZ8lpGZWX41C4SIuBt4fdjiTwE/yV7/BPh0rdoHn2VkZjYW9R5D2DMiXslerwX2HG1DSYskdUvq7u3tHVdjPsvIzCy/hg0qR0QAUWH9kojoioiuzs7OcbXhs4zMzPKrdyC8KmkvgGzeU8vGXCGYmeVX70C4Afh89vrzwK9r2djAQPlpp64QzMwqqeVpp9cC9wAHS1oj6TzgEuDPJD0FnJy9r5kIDyqbmeXVUasvjoizR1l1Uq3aHK68QvAhIzOzygp/pbIrBDOzfAodCK4QzMzyK3QguEIwM8uv0IFQXiFowBWCmVklhQ4EVwhmZvkVOhC2GUNwhWBmVlGhA2GbCsEXppmZVVToQPAYgplZfoUOBI8hmJnlV+hAcIVgZpZfoQPBFYKZWX6FDwRXCGZm+RQ6EPwITTOz/AodCNtUCOEKwcyskkIHQnmFoBggRn1gp5mZ5QoESRdI2kXJFZJWSDql1p3bUeUVQjv9PmpkZlZB3grh3IhYD5wCzAbOocZPO5sI5RVCGwO+A7aZWQV5A0HZ/FTgnyPi0bJlYybpQkmPSlop6VpJ08f7XZUMrxAcCGZmo8sbCMsl3UYKhFslzQLGdQBG0j7Al4GuiHgv0A58djzfVY0rBDOz/PI+U/k84AhgdURskjQH+M872O4MSVuBmcDLO/Bdo3KFYGaWX94K4RjgiYh4Q9LngP8OvDmeBiPiJeDbwAvAK8CbEXHb8O0kLZLULam7t7d3PE1tVyF4UNnMbHR5A+FyYJOkw4GvAM8AV4+nQUmzgU8BC4C9gZ2ykNlGRCyJiK6I6Ors7BxPU64QzMzGIG8g9EVEkH6R/yAi/hGYNc42TwaejYjeiNgKXA98aJzfVVGqCMSA2hwIZmZV5A2EDZIuJp1uepOkNmDKONt8ATha0kxJAk4CVo3zuyoqXYgWbe100OdAMDOrIG8gnAVsIV2PsBaYB1w6ngYj4j5gKbACeCTrw5LxfFc1pTGDgbYOVwhmZlXkCoQsBK4BdpV0GrA5IsY1hpB9399FxCER8d6IOCcitoz3uyq3k81dIZiZVZX31hV/ASwDzgT+ArhP0mdq2bGJUKoQoq3dFYKZWRV5r0P4OnBURPQASOoEfks69DNpDVYI7R2uEMzMqsg7htBWCoPMa2P4bMMMXnfgCsHMrKq8FcItkm4Frs3enwXcXJsuTZzyCsGBYGZWWa5AiIivSjoDODZbtCQiflm7bk2MwTGEdg8qm5lVk7dCICKuA66rYV8mXAS0tblCMDPLo2IgSNoAjPScMQEREbvUpFcTZGAAJMCnnZqZVVUxECJivLenmBQGK4QOVwhmZtVM+jOFdoQrBDOz/AodCENjCD7t1MysmkIHwmCFkF2Y5uchmJmNrtCB4ArBzCy/QgfCYIXgQWUzs6oKHQilCgFfmGZmVlWhA8EVgplZfoUOBFcIZmb5FToQShWCPKhsZlZVQwJB0m6Slkp6XNIqScfUop3BCqHDz0MwM6sm983tJtj3gVsi4jOSpgIza9HI6afDu98NXOcKwcysmroHgqRdgY8AXwCIiLeBt2vR1sknp2nDrz2obGZWTSMOGS0AeoF/kvSApB9L2mn4RpIWSeqW1N3b27tDDcqDymZmVTUiEDqA9wOXR8SRwEZg8fCNImJJRHRFRFdnZ+eOtTjFFYKZWTWNCIQ1wJqIuC97v5QUEDWjDlcIZmbV1D0QImIt8KKkg7NFJwGP1bJN+cI0M7OqGnUdwvnANZIeBo4A/qGWjamjnfk8zzE/uwDf8tTMbGQNOe00Ih4EuurVnjraATjsrstg9flwwAH1atrMrGkU+krlQR1DuffAg2pgR8zMJq+WCIRShQDwmTMb2BEzs0msNQJh6lCFMLU218CZmTW91giEsgphGlsa2BMzs8mrJQKhbYorBDOzaloiEFwhmJlV50AwMzOgVQKh7JDRsfwBVq9uYG/MzCanlggEpk0bfPn3/A/Yf//G9cXMbJJqjUDYZZdG98DMbNJr2UB49tkG9MPMbBJr2UB48MEG9MPMbBJr2UCY0hEN6IiZ2eTVsoHwp963GtARM7PJq2UDYcvLrzWgI2Zmk1fLBkL7s083oCNmZpNXywbCrs8sb0BHzMwmr4YFgqR2SQ9IurHmjZVdmAawmgW84/llNW/WzKyZNLJCuABYVa/Gzp39y8HXd3ASBzx3O6tX+b5GZmYlDQkESfOAjwM/rleb3fM+DcANfILlnR9jFzZwxfm+GMHMrKSj+iY18T3gImBWvRq84QZoX9BHIFZ87W64ENav3VSv5s3MJr26VwiSTgN6IqLiqK6kRZK6JXX39vbucLvz50PPuna+f1kbh38gG1N42w/LMTMracQho2OBT0p6DvgZcKKknw7fKCKWRERXRHR1dnZOSMO77w7nnw+aNhUAve0xBDOzkroHQkRcHBHzImI+8FngdxHxubp2IjvrSFtdIZiZlbTGdQjDTU0VQttWVwhmZiWNGlQGICLuAu6qe8NZhdDW5wrBzKykpSsEHzIyMxvSmoGQVQjtfT5kZGZW0pqBUBpD8CEjM7NBrRkIWYXwzb6vwn77wU03wTe+0eBOmZk1liIm/5PDurq6oru7e+K+MALaRsjCJvi3MDPLS9LyiOjKu31rVggSfR3Ttlu88l4/Rc3MWldrBgLQ3z51u2UPXP0I3HZbA3pjZtZ4Db0OoZEGpkyDLRu2WXbO5R+Cy4GtW6GjZf9pzKxFtWyFEKqw61t8OqqZtZ6WDYQpW0e/9fUP/o8DwcxaT8sGQvuW0QPhH/5ucx17YmY2ObRsILTFwKjrpuEKwcxaT8sGQsn6ER7aNh1XCGbWelo+EH7Lydstm8aW9DS1gdGrCDOzomnZQHj5tEUs5QxeZu/t1k1nM3zkI7B4cQN6ZmbWGC0bCL3/60ecyVJ62GO7dXP4IyxbBk891YCemZk1RssGwuGHw733wl8unrPdun/XtiLd12jDhhE+aWZWTHUPBEn7SrpT0mOSHpV0Qb37UPLBD8Ku87YfVO4iu5FeKRDuvRfuvLOOPTMzq79G3J+hD/hKRKyQNAtYLun2iHisAX1hxp4jBELcD8Db69YzFeCYY9IK3w3VzAqs7hVCRLwSESuy1xuAVcA+9e5HSfuendst2ydeAmDr6hd487wL690lM7OGaOgYgqT5wJHAfQ3rxHHH8WW+z1vstN2qndjErld+rwGdMjOrv4YFgqSdgeuAv46I9SOsXySpW1J3b29vLTvC/+XL7MzGbRa/yS4jbsuNN6bXTz4Jb/n5CWZWHA0JBElTSGFwTURcP9I2EbEkIroioquzc/vDOhNtIb/h55w5+P4ejhl5w29/O90e++CD4cwzR97GzKwJNeIsIwFXAKsi4jv1bn80t7KQc/jnwfdPcPDIGw4MwAsvpNe3316HnpmZ1UcjKoRjgXOAEyU9mE2nNqAfg554Am65BZ5+fugpam91zB5x2zUv9MMzz6Q3u+1Wj+6ZmdVF3U87jYjfA6p3u5UcdFCaSt26gxNZfcZXOfdf9uMb/E8W8Nzgtmuf38K8m24C4M322exa/+6amdVEy16pPJq/OedVPsZvOOyYnbj/vedyAndyCx8dXN/FcrjsMgDW9Exl3TrgjjvSgPP998MXvgCvvNKYzpuZ7QAHwjCPvLoHW5nK/vvD734HZy+ez8e4hY9yC9/iq4PbPct89uZl2hZfBCdnd0w9/XT4yU/gO9+BH/4QenoatBdmZmPnQBimLfsXOeAA6OyE445L72/jo/w3vjW43e85jtm8wZwrLh368Evpgjauvx6++EX40pfgxBPh5z+HN97ws5rNbFJzIAxzxRWwZAkcckh639UFJ50En/xken8Uy/j6rMv4PSkpli2+Hlas4M3PfYlNJ2cbrV6d5kuXpnsgnXUWzJ4NRxwBP/0pdHdDX1+d98zMrDJFE9yfp6urK7q7uxvah2XL0s3wStroZ3de45s/3oMDDoDjj0/Lr/jEr3jP8qv54MfmwE03wdq1lb/4lFPg1FPh6KNTWTJnThqPMDPbQZKWR0RX7u0dCPk9/niaTj8dZs2qfHfspUvhrrvgsm9t5k9338/M6QPw3e/CY49Vf87CO98Jhx4KRx4J73oXLFgABx4I++wD7e0Tuk9mVlwOhBrbsAF22QW++U24+OK0bOHCdB3DSA49FFatSuPLDzyQCgIgPaLzmWdSQNx3H/zhD/Dss/Dqq5Uf3TlnDuy9N+y1F+y5Z5rmzk3T7run9TNnptednbDzzq44zFrUWAOhEbe/bmqzZsGf/gTTpqXfwYcdBh/4QPodfvXV6Zd+dlYqkMIAYI/swWxdXWl84oILpvK+9x3KtEMPhTPO2LaRrVth/fqUIk8+CS+/nIKipydNvb3w4oupsddeq35b7hkzUjDMmpXmO+2UppkzYfr0tH7GjPR6+nSYMmXbqaNjfO/b29PU1jb0ur09rR/+uq0tTdLQvHy5TU6ln72IbV/Xe97ItmvdhwUL0v+7deAKoQZuvjmFxaWXwsMPp9/plZx8MnziE3DCCekWSVOnVt5+GxHpDKbXX4c//jFNmzen9z096QZ8mzal0mbDBti4MS3buDEl2+bNaV56vXlzCqTJpjwwhk/Dg2T4HLadV1pWvjzva2j8L6V6t2n185vfpMMQ4+BDRpPQU0+l/49uvBF+8Yv0ALZq9tgDPv5x+PCH08lJ73xnulNGXf5YjoD+/hQMW7emM6JKr8fyvr8/TQMDQ6/7+9P64e8HBlK75fPS58rnpV9+5VP58uGfL+1PaV7pl2n58rG8rhY29Zo3su3J0Iei7v+HPgTveAfj4UBoEps3p3vk3XNPugDuttuqn5AEaYjgoIPSGPP++8P73pdOTtp333Rmq5lZiQOhyQ0MpGB47LE0PfBAml54IR0NqmbPPbcdb95jjzS2XBpz3n33FByzZ6eKY/p0H6I3KyoPKje5trZ0EtHeew/dEaNcf38aMujpSePMTz45NM5cmtatg0cfhbvvTkMDlUydms6aKk2zZg2NPc+aNTT+vPPOaQy6NBZdms+YkcaFd9klhcyUKek7Z81y0Jg1GwdCk2lvH/pL/9BDhy6IG0lEOpTf05PGmEvjzm+8MTStXz80lcadX3wxjTuXxqM3bRr7WKKUgmHKlBQOpeAofz1jRgqVmTOHgmTatKGTnqZNG1pWmpem6dPTvPS50abS+ilTfPatWTUOhAIr/VKeNy9N4zUwkMY8Nm4cCojSSUqlsFi/PoVNX1/a9o030qUWpTNoSycvlb6jry9VMqX3W7em7bdsSd9bixOdSmfDlgKi/HX5stJ2Y5mP5zOlM3NLZ96OdDZutfXVljkEbSwcCFZVW9vQX/L1UjpR6e23h4Jiy5YULKV5ad1oU+nzY5mX2i3NS+E0fHmleaXrCuutdAlI3jDZ0QAqtdeq8+FnMDcbB4JNSqW/umfMaHRPxm5gIAVDtfAoPzO3dCZu+Rm5Y102Ud8z2rItW4aqu5HWj3SWcJ550ZQumRltKr/eMs/0ox+l08/rwYFgNsHa2obGMKy60mUv4w2UyTbv79/2Upjhl9SMdarTRcpAgwJB0kLg+0A78OOIuKQR/TCzxpNSNWiNV/cTAyW1A/8IfAx4N3C2pHfXux9mZratRpwp/gHg6YhYHRFvAz8DPtWAfpiZWZlGBMI+wItl79dky7YhaZGkbkndvb29deucmVmrmrTXkkbEkojoioiuzs7ORnfHzKzwGhEILwH7lr2fly0zM7MGakQg3A8cKGmBpKnAZ4EbGtAPMzMrU/eTvSKiT9JfAbeSTju9MiIerXc/zMxsWw05+zcibgZubkTbZmY2sqZ4HoKkXuD5cX58LrBuArszGRRtn4q2P1C8fSra/kBr7NM7IyL3WTlNEQg7QlL3WB4Q0QyKtk9F2x8o3j4VbX/A+zSSSXvaqZmZ1ZcDwczMgNYIhCWN7kANFG2firY/ULx9Ktr+gPdpO4UfQzAzs3xaoUIwM7McCh0IkhZKekLS05IWN7o/eUi6UlKPpJVly+ZIul3SU9l8drZcki7L9u9hSe9vXM9HJ2lfSXdKekzSo5IuyJY35X5Jmi5pmaSHsv35+2z5Akn3Zf3+l+xKfCRNy94/na2f38j+j0ZSu6QHJN2YvW/2/XlO0iOSHpTUnS1ryp+5Ekm7SVoq6XFJqyQdM5H7VNhAaOLnLlwFLBy2bDFwR0QcCNyRvYe0bwdm0yLg8jr1caz6gK9ExLuBo4EvZf8tmnW/tgAnRsThwBHAQklHA/8b+G5EHAD8ETgv2/484I/Z8u9m201GFwCryt43+/4AnBARR5SditmsP3Ml3wduiYhDgMNJ/70mbp8iopATcAxwa9n7i4GLG92vnH2fD6wse/8EsFf2ei/giez1j4CzR9puMk/Ar4E/K8J+ATOBFcAHSRcEdWTLB3/+SLdpOSZ73ZFtp0b3fdh+zMt+mZwI3Aiomfcn69tzwNxhy5r2Zw7YFXh2+L/1RO5TYSsEcj53oUnsGRGvZK/XAntmr5tuH7PDC0cC99HE+5UdXnkQ6AFuB54B3oiIvmyT8j4P7k+2/k1g9/r2uKrvARcBA9n73Wnu/QEI4DZJyyUtypY17c8csADoBf4pO7T3Y0k7MYH7VORAKKRIUd+Up4ZJ2hm4DvjriFhfvq7Z9isi+iPiCNJf1h8ADmlwl8ZN0mlAT0Qsb3RfJthxEfF+0qGTL0n6SPnKZvuZI1Vj7wcuj4gjgY0MHR4CdnyfihwIRXruwquS9gLI5j3Z8qbZR0lTSGFwTURcny1u+v2KiDeAO0mHVHaTVLphZHmfB/cnW78r8Fqdu1rJscAnJT1HeqTtiaRj1c26PwBExEvZvAf4JSm4m/lnbg2wJiLuy94vJQXEhO1TkQOhSM9duAH4fPb686Rj8KXl/yk7m+Bo4M2y0nHSkCTgCmBVRHynbFVT7pekTkm7Za9nkMZDVpGC4TPZZsP3p7SfnwF+l/0lNylExMURMS8i5pP+P/ldRPxHmnR/ACTtJGlW6TVwCrCSJv2ZA4iItcCLkg7OFp0EPMZE7lOjB0pqPAhzKvAk6fju1xvdn5x9vhZ4BdhK+ovgPNLx2TuAp4DfAnOybUU6k+oZ4BGgq9H9H2WfjiOVsQ8DD2bTqc26X8BhwAPZ/qwE/jZb/i5gGfA08AtgWrZ8evb+6Wz9uxq9DxX27Xjgxmbfn6zvD2XTo6X//5v1Z65sv44AurOfvV8Bsydyn3ylspmZAcU+ZGRmZmPgQDAzM8CBYGZmGQeCmZkBDgQzM8s4EKwlSPp/2Xy+pP8wwd/9tZHaMms2Pu3UWoqk44G/iYjTxvCZjhi6p89I69+KiJ0non9mjeQKwVqCpLeyl5cAH87ukX9hdpO6SyXdn90z/i+z7Y+X9G+SbiBdDYqkX2U3Snu0dLM0SZcAM7Lvu6a8rewK0Uslrczuy39W2XffVXZf+2uyq7mRdInScyMelvTtev4bmXVU38SsUBZTViFkv9jfjIijJE0D/iDptmzb9wPvjYhns/fnRsTr2e0q7pd0XUQslvRXkW50N9yfk64sPRyYm33m7mzdkcB7gJeBPwDHSloFnA4cEhFRuj2GWb24QrBWdwrpfi8Pkm7JvTvpgSIAy8rCAODLkh4C7iXdNOxAKjsOuDbSnVFfBf4VOKrsu9dExADpVh7zSbeR3gxcIenPgU07vHdmY+BAsFYn4PxIT9U6IiIWRESpQtg4uFEaeziZ9GCYw0n3Mpq+A+1uKXvdT3oQTR/pjpxLgdOAW3bg+83GzIFgrWYDMKvs/a3AF7PbcyPpoOzumMPtSnps5CZJh5AeBVqytfT5Yf4NOCsbp+gEPkK6GdyIsudF7BoRNwMXkg41mdWNxxCs1TwM9GeHfq4i3fd/PrAiG9jtBT49wuduAf5Ldpz/CdJho5IlwMOSVkS6bXTJL0nPSXiIdLfXiyJibRYoI5kF/FrSdFLl8l/Ht4tm4+PTTs3MDPAhIzMzyzgQzMwMcCCYmVnGgWBmZoADwczMMg4EMzMDHAhmZpZxIJiZGQD/HwYhYMmIlTzCAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hbvpbieUigAI",
        "colab_type": "text"
      },
      "source": [
        "### **2. Plot the accuracy curve**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_WepyFzxik6n",
        "colab_type": "code",
        "outputId": "30aacfac-82bb-46fe-8fd5-03cc1d97f71a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 279
        }
      },
      "source": [
        "fig = plt.figure()\n",
        "graph3 = plt.plot(iters, train_accuracys, 'b')\n",
        "graph4 = plt.plot(iters, test_accuracys, 'r')\n",
        "plt.xlabel('iterations')\n",
        "plt.ylabel('loss')\n",
        "plt.show()"
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEGCAYAAAB7DNKzAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3deZgcdb3v8fd3tuxMtgECAYYQVtkSE0UwHBZBCIjBgyBwMCo+OXo4ioAXUTzXe3k8V7gcleV6UAQFZVUWWSVAQDEuCQlrIEAStgQSkpA9k2S27/3jV53umenu6STdXd1Tn9fzzNNV1dXd35pM6tO/X1X9ytwdERGRXGriLkBERCqbgkJERPJSUIiISF4KChERyUtBISIiedXFXcCOGDlypDc3N8ddhohIVZk7d+5Kd28qdP2qDorm5mbmzJkTdxkiIlXFzN7ZlvXV9SQiInmVLCjM7FdmttzM5mUsG25mT5jZguhxWLTczOw6M1toZi+Z2fhS1SUiItumlC2KW4CTui27DJjh7vsCM6J5gJOBfaOfacANJaxLRES2QcmCwt2fAVZ1W/xZ4NZo+lZgSsby33jwD2ComY0qVW0iIlK4ch+j2MXdl0bTy4BdoundgcUZ6y2JlvVgZtPMbI6ZzVmxYkXpKhURESDGg9keRiPc5hEJ3f1Gd5/g7hOamgo+u0tERLZTuYPig1SXUvS4PFr+HrBHxnqjo2UiIhKzcl9H8SAwFbgyenwgY/m/m9ldwMeBtRldVCIiFa+tDVpaSvf+y5bB/fenP+Mzn4GJE0v3eZlKFhRmdidwDDDSzJYAPyAExO/M7HzgHeDMaPVHgcnAQqAF+HKp6hKR5Fq6FFavzv7cypWwYEHXZZs3w4svQmdnmF+2DN58M/vrlyyB9euLV2suZuFxt936QFC4+9k5njo+y7oOXFCqWkSkdFpbw7fp3qxaBa+9lp7fvBleegk6OvK/btOmsN7bb8N7O9Ah7Q7r1m376xobYfDgMN2/Pxx2GNTW9lzvqKPgwAPTO/Ji69cPpkwJAVFuVT2Eh4jsuM5OeOutrjvsDz+EN94I0xs2wMsvhx0tQHt7+Jbd2hpe+8YbhQXFjjjwwLCD/NSndmxHvPPOsM8+2d+jvj6EQH191+W77ZY9GJJEQSFS5dra4O9/h1//OnSfQPgWntllkk9rawiDfBobYcCA9PzYsbDnnmH6uONgjz2yvy5TXR0cfjg0NKSXHXwwDBnS+2trNNhQrBQUkkhLl8J998Fll8H48aGvt7YWBg6E/fcPXQzdbdmS/uZcXx+6AjJt3hy+bReTO7z6avYuk44OmD0bXnghrFdbC4ccEnaqZjB5Muy0U2Gfc+CBMHRoej7z23VNTQgC7ayTS0EhVe3NN2H69PQ355qasKOvrw87z4ULQ7/2Aw/A4uiSzs7O9DdvgGeeCT+VqrYWhg3L/lz//nD88TB1KhxzDIweXdbSJCEUFFKxtmwJ39Dffz8cyHzlldBXnjqwuHkz3HZbYackHnoonH56um96r71CX/WIEXDAAaElAeHzXn21a5CkDBuW7mJZsiQcnM3U1ASjSjDwzMiRXb/ti5SbgkLKxj0cNP3Tn0IIQPiWf8cd4dEzrtNvbg7dQ5s3Z3+v1I7z2GND91GqG8i96ymKI0bAfvulg6AQRx/d+zpjxxb+fiLVTkEhRdXRke6nNwtnxFx9NTz0UO7z17MZORJOPDF0raxZE7qLJk6E3XcP4ZCrK0ZEik9BIdvtqadgxoxwDKCjIxxUnTev99d99KPhqlIIferHH9/zrJmkn44oUkkUFNLD5s3heMD69TBrVjjjZvZs+OCDcIxgewwcCNddB1/6kkJApNooKGSr1laYOTN8w98RQ4aEg8e1tXD99WFaRKqXgiKBOjvDQd/33guthMWL4bHH4Je/3Lb3GTw4XIS1777wrW+Fg8Bm4epXEek7FBQJsGULPPccXHFFOK3z7bd7vxJ35EgYMyZcSTt2bAgAs3AweeLEcFaSupBEkkFB0Ud1dobxeqZPh+98J1yLAOFq29NPD2cTLV8eAqNfPxg+HCZNCtcCTJoUgkJEBBQUfUZnZzgL6eabw1DJ8+enL0QbOxZuvz2MybPrrvHWKSLVR0FRpe64I7QGZs4Mp6W+9Va6O2nUqNBFtP/+8LWvwQkndB2ITURkWygoqsTGjXD55fDooz1vrgJw2mlw1lnw6U+Hq5FFRIpFQVElrrkGrr02+3Nf/nI4Y0kHl0WkFDRwcBX48EP41a+yP/eLX4TnFBIiUioKigq3bh2cfXY4BtHcHJaNGQM/+lG4DmLatFjLE5EEUNdTBVq3Dp5+Gn7+83AhHMCPfwwXXxxvXSKSTAqKCtLSAjfeCBdd1HX5D3+okBCR+CgoKsSdd8I55/RcfsUV4WwnEZG4KCgqwMMP9wyJj3wEHnkk3IlNRCROCooYPflkGEbj3HO7Lv/jH8NFcjqTKZJ567v29vCzYUMYB33VKmhrC+OQDB0aBqSqrw/rtraG5zo6wpglZmFZ6n3a2sL454MGhWX9+oX12tvDzbfffz88n3p/CJfAd3SEAbRWr06/R21tus66uvS6bW1hes2a8NnuPX9S25hvWUdH2N5Bg8LwvDXReSip5bW1YbtrasJPts8p9k/q81O/57a2/NuRazrbc5s3h+1K/Xu3t3ddr/tjXMuyPZdZM4R/j9Rgab1N53s+9WMW3vuCC+CUUygHBUWMzjkHVqzouuySS+Ckk+KpZ4d1dIQrA999N70TTQ0o1dER/hPV1oblS5eG0QnfeivcBq/7L0L6ltQOL9d09/mGBthppxB+dXXh7ybX6+NalmubBg9O1wzh7z41ZHOu6czHQp6vr899n+ASUFCU0eLFYciN5cvDBXKZ+8YjjgjHIk44Ib76umhtDTv8xYvT9zBdvTqckztgACxbBn/7W/oOR5Vm0KBQJ4SdTv/+6ZbGgAHhW/mIEbDPPmG+oyP8R9+0KfwHXLs2/AM1NYXXDB8ebsfX0BD+o9bUhBbMli3pIEzt3FKtF7P05zY2hmnoupPJtuPJtrymJuyAWlrC56W+udbWhm2BMN/REerL9n7F/oHw+Q0NYRtTLZrMuqVPUFCU0cSJ4f4PmRoawj558mQ49dR46mLTJnjppTBy4IMPwjvvFP8zUmOU77FHeBw3Dg44IOzQhw2DXXYJO7xBg8LOp7W1644181FEykpBUQK//GX4AnnmmellnZ09QwLC0Bx33hmG4Si5jRvh9dfh2Wdh7tzw7XTJktAySPWlZxoyJOzQx40LV/mNHBm+Na5dG3baTU2w227hm3VnZ9ixt7SEz9l77xAA27tzr9OfpkilMM88UFhlJkyY4HPmzIm7jK02bgwthNRIrZm/2pUr070YEPaD7e1hfz1+fAmKWbkydBM98kg4Oj5zZvb1hg4Nt6YbPx4OOSTcB7WxsQQFiUilMLO57j6h0PX1ta2IBg8Oo7empLq4H3kktBoyPfRQ+PI9blyRPnzLlnAZ9/33wz33hNTKZuzY0Mf18Y+HYocNK1IBItJXKSiKJNV6mD49vay5OfTSZDv2MGYM7LffDnxge3s42Hz//XD11dn7tSAcgD333BAKkyapS0dEtpn2GkWS7Qt8SwssXNh12SmnhBbG6NHb8SGrVsHjj8NVV4W7FeXS1ATf+16456mu2BORHaSgKJJVq9LTw4en5087ret6118Pv/99+szNXq1dGy7d/upX8583fcwxcOGFcOyxOsYgIkWloCiSzKBobQ0thiVLet6NbuTIbQiJxx6Dk0/Ov85tt8GUKemri0VEikxBsYPc4etf79qV1NoaDgksXhx6ijINHtzLG65aBf/93/Af/5F7nauuCvc93XNPXVsgIiWnoNhB69aFu8xlam0N15TddFN6P/722/Daa3n265s2hdbDn/+c/fkxY+DKK+GMMxQOIlJWsdzhzswuMrNXzGyemd1pZv3NbG8zm2VmC83sbjNriKO2bfXhh9mXp8aQS9lrr66nzm7V1hauujvuuOwhMW4c/OEPsGgRfP7zCgkRKbuytyjMbHfgm8BB7r7JzH4HfAGYDPzU3e8ys58D5wM3lLu+bbVyZfblqaB46KEw7l0P774bupeGDoXrruv5/J57wi23hIPTIiIxiqvrqQ4YYGZtwEBgKXAckLorw63A/6KCgmLZMrj00nB70oEDw7GJm24K09mkrs7ucg1Fe3vog7rtNnjmmXC/02wuvTQchxARqQBlDwp3f8/M/gt4F9gEPA7MBda4ezSAO0uA3bO93symAdMA9txzz9IXHLn8cvjtb8NZqF/5Srg+Ytq03Ot373riP/8Tvv/90FJ4993cL1yzRqe3ikhFKfsxCjMbBnwW2BvYDRgEFHwHBne/0d0nuPuEpszBk0osteNPXcqwZUv29epp5QQep19DtzG07rgjPOYKiQEDQktCISEiFSaOrqdPAW+5+woAM7sPOAoYamZ1UatiNPBeDLXllAqKVEBs2ND1+QG0cDgv8FHmcj3fZObzv4MvfT7cr2HBgt4DoKWl+EWLiBRBHEHxLnCEmQ0kdD0dD8wBngbOAO4CpgIPxFBbTqlbI6RaFN2D4recxz9zH1cQrn/Y84UHYf7BcPDB+d/4i1/MfaBDRKQCxHGMYpaZ3QM8B7QDzwM3Ao8Ad5nZD6NlN5e7tny6B0X3sZ1O4AkAjNDltNuzf4CDbuv9jW+9tVglioiURCxnPbn7D4AfdFv8JvCxGMrp1caNYZBWyN2i2IlwO9BDRq+GJVC3qdsKEAbry7z/6dFHl6BaEZHiiuWCu2rzb/8GL74YpnMdo0hp7AiDPq1rPqTnk8OHp6dbWuDJJ4tYpYhIaSgoCpB5wVyuFkVKY3sIitVjMxpHqZsDpQZ6OvTQcJZTfX2RKxURKT4FRQHa29PTmzaFx9QxinO5jdlM3Pr8kCgoNgzPuMYjFRTt7TB7Njz1VCnLFREpKgVFATKDonuL4jbOYyLp+3bvMWgVbdSxy/+5MP2inXYKj62tMHEijBhR4opFRIpHQVGAzKDYsiWExNVXZ1+3/5JF1A/qx8h9GtNnNKVOf91119IWKiJSAhpmvADdWxRz56bmPNvq6X6p1FV6++8fxv3IdvNsEZEKp6AoQEdHenrtWnjzzTD9wC1r4Et5XjhlCnzjG2GU2DIONyIiUkwKigK0tcGFXMO77MmTr3yORYugthZO/qdeht3o1y/7EOIiIlVEQZHHpk1hVPD2driGiwCw9c4zz4RBYOvbN8VboIhIGehgdh5nnw0HHdRzuI6//AUO2WMNHHlkPIWJiJSRgiKPRx8Nj7d9eHKP576y8bquw3EA/PWvZahKRKS8FBR5tLWFxxM6Huvx3MEb/tHzBQceWOKKRETKT0HRixo6ui0Jp8Q2dq7uuXJqiA4RkT5EQdGLnVjXZX4Q4YDFoPXLeq6ssZtEpA9SUPRiGF1bDsNZBTj9Pnw/noJERMpMp8f2ontQjGQlGxlETVtrTBWJiJSXWhS9GMqaLvOjWcIdnNNzRbPwOHs2LFpUhspERMpDLYpedG9RfIG7+DSP91wxFRQTJ/Z8TkSkiqlF0YvuLYomVuRYU0Skb1JQ5HECj7MPXbuRxhBGBPSr/m/XlY86qlxliYiUlbqecmlv53E+3WPx3oT7otrpU+A7l4aFs2fDfvuVszoRkbJRiyKXNWuyLq5J3YNi4MBw96IjjwzHJRoby1iciEj5KChymH5XliuvMw0aBN/+tsZ3EpE+T0GRw4Jns7cotkrd3lREpI9TUOTQryV3i6KNOmhoKGM1IiLxUVBk486/PHRm7ufVmhCRBFFQZLNuHQO2rO2xeA3hgHV946ByVyQiEhsFRTYd3YcWD9ZGQaEWhYgkiYIim96CQsOJi0iCKCiyuO6nvQSFexmrERGJl4Iii6t+lD0o1jA0TAwbVsZqRETipaDIorbH7U+DDUS3Ot1ttzJWIyISLwVFFrmCYutIsqNGlbEaEZF4KSiyyBUUL3NImJg8uYzViIjES0GRRa6g+CMnw3vvKShEJFEUFFnkCgpAxydEJHFiCQozG2pm95jZa2Y238w+YWbDzewJM1sQPcZ2alHeoBARSZi4WhTXAo+5+wHAYcB84DJghrvvC8yI5mNxAk9kXW7o+gkRSZ6yB4WZNQJHAzcDuHuru68BPgvcGq12KzCl3LUB+Kvz+QmXANCJxVGCiEhFiaNFsTewAvi1mT1vZjeZ2SBgF3dfGq2zDNgl24vNbJqZzTGzOStWrCh6cW2L3t06vZn+RX9/EZFqE0dQ1AHjgRvcfRywkW7dTO7ukL2fx91vdPcJ7j6hqamp6MVtXpa+YVGr9evyXP8GdT2JSPLEERRLgCXuPiuav4cQHB+Y2SiA6HF5DLXRsTJ9w6JWQlAsixo3dz5Z/GASEal0ZQ8Kd18GLDaz/aNFxwOvAg8CU6NlU4EHyl0bAGvSLYrNFrqeLuRamDmTxkmHxlKSiEic6mL63G8At5tZA/Am8GVCaP3OzM4H3gHy3GKuhFanWxRtUdfTeobAUUfFUo6ISNxiCQp3fwGYkOWp48tdS3e2JqPrqaYfdOigtogkm67M7q6lZevklqjraRMD4qpGRCR2BQWFmV1oZjtZcLOZPWdmJ5a6uDh4a9vW6baa0PWkFoWIJFmhLYqvuPs64ERgGHAecGXJqopTWzooUqfHKihEJMkKDYrUJcqTgd+6+ysZy/oUzwgKdT2JiBQeFHPN7HFCUEw3syFAZ+nKilFmiwK1KERECj3r6XzgcOBNd28xs+GEU1r7HMvSolBQiEiSFdqi+ATwuruvMbN/Ab4PrC1dWTHKCIpltbuzhkYO/fjAGAsSEYlXoUFxA9BiZocBlwCLgN+UrKo4taeD4u4hX6Xl+Td47KmGGAsSEYlXoUHRHg3U91ng/7n7z4AhpSsrPl26nujHbofvzEA1KEQkwQo9RrHezL5LOC12kpnVAPWlKytGGS2KDmpjLEREpDIU2qI4C9hCuJ5iGTAauLpkVcXIFBQiIl0UFBRRONwONJrZqcBmd+9zxyg6O2H1cgWFiEimQofwOBOYDXyeMKrrLDM7o5SFxeHJJ+ly1lO7KyhERAo9RnE5MNHdlwOYWRPwJOGmQ31GRwfUoxaFiEimQo9R1KRCIvLhNry2atTWQgOtW+cVFCIihbcoHjOz6cCd0fxZwKOlKSk+dXVdWxTqehIRKTAo3P1/mNk/A6nbvN3o7veXrqx41NR0DYpOU1CIiBR8hzt3vxe4t4S1xK77MYo2j+tOsSIilSPvntDM1gOe7SnA3X2nklQVk4ULnOMzWxR97zCMiMg2yxsU7t4nh+nIZuZMuODrHfxrxjLvm7fcEBHZJvrKHJk3D3bnvS7LPFtbSkQkYRQUGd6hOe4SREQqjoIisvNbs3osO++8GAoREakw5lXcvzJhwgSfM2dOcd7Meh6P6OxwahSlItLHmNlcd59Q6PraDeahkBARUVAAsLbbTV3v5AuwYkU8xYiIVBgFBXD9tZ1d5v/MP8HIkTFVIyJSWRQUwMVX7bx1eg4f5RddrqYQEUm2xAfFokUwsOXDrfPvsBdjxuhCOxGRlMQHxZVXdp1vo56FC+OpRUSkEiU+KPb9YGaX+Tbqs50pKyKSWIkPiksfmtRlvo36mCoREalMiQ+K7lppiLsEEZGKoqDopr3wW3SIiCSCgkJERPKKLSjMrNbMnjezh6P5vc1slpktNLO7zUx9QCIiFSDOFsWFwPyM+auAn7r7WGA1cH4cRVnWG/qJiCRXLEFhZqOBU4CbonkDjgPuiVa5FZgSR20iItJVXC2Ka4BLgdQgSyOANe7eHs0vAXaPozC1KEREuip7UJjZqcByd5+7na+fZmZzzGzOihKM8PqlqUV/SxGRqhZHi+Io4DQzexu4i9DldC0w1MxS56aOhm43sI64+43uPsHdJzQ1NRW9uIED1KIQEclU9qBw9++6+2h3bwa+ADzl7ucCTwNnRKtNBR4oZ13vskc5P05EpGpU0nUU3wEuNrOFhGMWN5fzwzV0h4hIdrFehuzufwL+FE2/CXwsrlq2BkUV30NcRKQUKqlFESsFhYhIdgqKiLqeRESyS3ZQZLQelrFrmNh115iKERGpTMkeKrWzc+vk3ZzF5N+cDWedFWNBIiKVJ9lB0dGxddIxOO+8GIsREalMye56ymhRaOgOEZHskh0UGS0KERHJTkERUYtCRCQ7BUVEQSEikp2CIqKgEBHJLtlBkXEwW0REskt2UOhgtohIrxQUkRYGxliIiEjlUlAAL3Iov+PMmIsREalMCgrgJ1yMJ/xXISKSS7L3jlFQdFAbcyEiIpUr2UERnfWkoBARyS3ZQZHRonj44ZhrERGpUAoKYOddaznllJhrERGpUAoKwC3ZvwYRkXySvYdMXUdRq2MUIiK5JDsoooPZrqAQEckp2UGR6nqqUVCIiOSioABQUIiI5KSgQC0KEZF8FBSA1Sb71yAikk+y95A660lEpFfJDorUWU/qehIRySnZQaEWhYhIrxQUqEUhIpJPooNi80a1KEREepPooLjt1hAUs55N9K9BRCSvRO8ht2wKB7PbdT8KEZGcEh0UOw3SHe5ERHqT2KA44gj4+1/aAQWFiEg+iQ2KWbNg6Lp3AFjKqJirERGpXGUPCjPbw8yeNrNXzewVM7swWj7czJ4wswXR47BS13IIL/MWzWxgSKk/SkSkasXRomgHLnH3g4AjgAvM7CDgMmCGu+8LzIjmS+pg5vEyhzBjRqk/SUSkepU9KNx9qbs/F02vB+YDuwOfBW6NVrsVmFKqGqKROxjOKpaxK5MmleqTRESqX6zHKMysGRgHzAJ2cfel0VPLgF1yvGaamc0xszkrVqzYrs9tawuPDbTSRj11ddv1NiIiiRBbUJjZYOBe4Fvuvi7zOXd3wLO9zt1vdPcJ7j6hqalpuz47MyjarQGz7XobEZFEiCUozKyeEBK3u/t90eIPzGxU9PwoYHmpPr9Li6KmoVQfIyLSJ8Rx1pMBNwPz3f0nGU89CEyNpqcCD5Sqhu4tChERyS2OFsVRwHnAcWb2QvQzGbgSOMHMFgCfiuZLou7eu/krR1JLJ5s7FRQiIvmU/TCuu88Ech0VOL4cNbTXNHAkfwdgk4JCRCSvRF6Zvfboz2ydbqM+xkpERCpfIoOizevYQmhJtKIWhYhIPskMirb0QIAKChGR/BIbFJ3RpisoRETyS2xQqEUhIlKYxAeFDmaLiOSX+KBQi0JEJD8FhYJCRCSvxAaFDmaLiBQmsUGhFoWISGESGxRqUYiIFCaxQaGznkRECpP4oFCLQkQkv8QHRepRRESyS2xQpI5R1NAZczUiIpUtsUGxmf6AgkJEpDdlv3FRJWhrg8/zey7gZ7zCR+IuR0SkoiUyKI49Fmp/vA+DB/+EZw6KuxoRkcqWyKCYMCH8iIhI7xJ5jEJERAqnoBARkbwUFCIikpeCQkRE8lJQiIhIXgoKERHJS0EhIiJ5KShERCQvc/e4a9huZrYCeGc7Xz4SWFnEcipBX9umvrY9oG2qBn1te6DnNu3l7k2Fvriqg2JHmNkcd+9T12f3tW3qa9sD2qZq0Ne2B3Z8m9T1JCIieSkoREQkryQHxY1xF1ACfW2b+tr2gLapGvS17YEd3KbEHqMQEZHCJLlFISIiBVBQiIhIXokMCjM7ycxeN7OFZnZZ3PUUysx+ZWbLzWxexrLhZvaEmS2IHodFy83Mrou28SUzGx9f5dmZ2R5m9rSZvWpmr5jZhdHyqtwmM+tvZrPN7MVoe/53tHxvM5sV1X23mTVEy/tF8wuj55vjrD8fM6s1s+fN7OFovqq3yczeNrOXzewFM5sTLavKvzsAMxtqZveY2WtmNt/MPlHM7UlcUJhZLfAz4GTgIOBsM6uWG6LeApzUbdllwAx33xeYEc1D2L59o59pwA1lqnFbtAOXuPtBwBHABdG/RbVu0xbgOHc/DDgcOMnMjgCuAn7q7mOB1cD50frnA6uj5T+N1qtUFwLzM+b7wjYd6+6HZ1xfUK1/dwDXAo+5+wHAYYR/q+Jtj7sn6gf4BDA9Y/67wHfjrmsb6m8G5mXMvw6MiqZHAa9H078Azs62XqX+AA8AJ/SFbQIGAs8BHydcEVsXLd/69wdMBz4RTddF61nctWfZltHRjuY44GHA+sA2vQ2M7LasKv/ugEbgre6/52JuT+JaFMDuwOKM+SXRsmq1i7svjaaXAbtE01W1nVEXxThgFlW8TVEXzQvAcuAJYBGwxt3bo1Uya966PdHza4ER5a24INcAlwKd0fwIqn+bHHjczOaa2bRoWbX+3e0NrAB+HXUP3mRmgyji9iQxKPosD18Pqu58ZzMbDNwLfMvd12U+V23b5O4d7n444Vv4x4ADYi5ph5jZqcByd58bdy1F9kl3H0/ohrnAzI7OfLLK/u7qgPHADe4+DthIupsJ2PHtSWJQvAfskTE/OlpWrT4ws1EA0ePyaHlVbKeZ1RNC4nZ3vy9aXNXbBODua4CnCd0yQ82sLnoqs+at2xM93wh8WOZSe3MUcJqZvQ3cReh+upbq3ibc/b3ocTlwPyHUq/XvbgmwxN1nRfP3EIKjaNuTxKB4Ftg3OmujAfgC8GDMNe2IB4Gp0fRUQj9/avkXozMcjgDWZjRDK4KZGXAzMN/df5LxVFVuk5k1mdnQaHoA4XjLfEJgnBGt1n17Utt5BvBU9M2vYrj7d919tLs3E/6vPOXu51LF22Rmg8xsSGoaOBGYR5X+3bn7MmCxme0fLToeeJVibk/cB2JiOvgzGXiD0H98edz1bEPddwJLgTbCt4jzCf2/M4AFwJPA8GhdI5zdtQh4GZgQd/1ZtueThObwS8AL0c/kat0m4FDg+Wh75gH/M1o+BpgNLAR+D/SLlveP5hdGz4+Jext62b5jgIerfZui2l+Mfl5J7QOq9e8uqvFwYE70t/cHYFgxt0dDeIiISF5J7HoSEZFtoKAQEZG8FBQiIpKXgkJERPJSUIiISF4KCkk0M/tb9NhsZucU+b2/l+2zRKqNTo8VAczsGODb7n7qNrymztPjHWV7foO7Dy5GfSJxUotCEs3MNgWA71cAAAIESURBVESTVwKTovsTXBQN7ne1mT0bjdn/r9H6x5jZX8zsQcLVr5jZH6LB5V5JDTBnZlcCA6L3uz3zs6IrYq82s3nRPRHOynjvP2XcV+D26Op1zOxKC/fteMnM/qucvyORut5XEUmEy8hoUUQ7/LXuPtHM+gF/NbPHo3XHAwe7+1vR/FfcfVU0bMezZnavu19mZv/uYYDA7j5HuJL2MGBk9JpnoufGAR8B3gf+ChxlZvOB04ED3N1Tw4SIlItaFCLZnUgYD+cFwtDnIwg3egGYnRESAN80sxeBfxAGW9uX/D4J3OlhpNkPgD8DEzPee4m7dxKGNGkmDNW9GbjZzD4HtOzw1olsAwWFSHYGfMPDHdAOd/e93T3Voti4daVwbONThJv1HEYY66n/DnzulozpDsLNgdoJo5veA5wKPLYD7y+yzRQUIsF6YEjG/HTg69Ew6JjZftFIo901Em792WJmBxBu6ZrSlnp9N38BzoqOgzQBRxMG0Msqul9Ho7s/ClxE6LISKRsdoxAJXgI6oi6kWwj3XGgGnosOKK8ApmR53WPA16LjCK8Tup9SbgReMrPnPAzNnXI/4T4VLxJGz73U3ZdFQZPNEOABM+tPaOlcvH2bKLJ9dHqsiIjkpa4nERHJS0EhIiJ5KShERCQvBYWIiOSloBARkbwUFCIikpeCQkRE8vr/JQSTpal/Ny4AAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CcYJkW0ZvIBq",
        "colab_type": "text"
      },
      "source": [
        "### **3. Plot the accuracy value**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_I-fWRNFvQEv",
        "colab_type": "code",
        "outputId": "5a0a210a-b829-46d5-f411-552a7009254e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 53
        }
      },
      "source": [
        "final_train_accuracy = train_accuracys[-1]\n",
        "final_test_accuracy = test_accuracys[-1]\n",
        "print(\"final_train_accuracys :\", final_train_accuracy,\"%\")\n",
        "print(\"final_test_accuracys :\", final_test_accuracy,\"%\")"
      ],
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "final_train_accuracys : 98.2 %\n",
            "final_test_accuracys : 86.67777777777778 %\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "v1k8NptSxNFy",
        "colab_type": "text"
      },
      "source": [
        "### **4. Plot the classification example**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lVBi79mqxQGY",
        "colab_type": "code",
        "outputId": "2964a3a5-d20b-4cfa-c39b-60e5acd42e37",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 473
        }
      },
      "source": [
        "f2 = plt.figure(2)\n",
        "im_correct  = np.zeros((size_row * size_col, 10), dtype=float)\n",
        "#print(correct_labels)\n",
        "#print(correct_labels[0][1])\n",
        "#print(incorrect_labels)\n",
        "label_correct_index = []\n",
        "label_incorrect_index = []\n",
        "\n",
        "print(\"correct image 10 collections\")\n",
        "for i in range(10):\n",
        "    correct_index = correct_labels[0][i]\n",
        "    im_correct[:, i] += test_image[:, correct_index]\n",
        "    label_correct_index.append(test_label[correct_index])\n",
        "\n",
        "for i in range(10):\n",
        "    plt.subplot(2, 5, i+1)\n",
        "    plt.title(label_correct_index[i])\n",
        "    plt.imshow(im_correct[:,i].reshape((size_row, size_col)), cmap='Greys', interpolation='None')\n",
        "\n",
        "    frame   = plt.gca()\n",
        "    frame.axes.get_xaxis().set_visible(False)\n",
        "    frame.axes.get_yaxis().set_visible(False)\n",
        "plt.show()\n",
        "###################################################\n",
        "#print(incorrect_predicts)\n",
        "\n",
        "print(\"incorrect image 10 collections\")\n",
        "f2 = plt.figure(2)\n",
        "im_incorrect  = np.zeros((size_row * size_col, 10), dtype=float)\n",
        "\n",
        "for i in range(10):\n",
        "    incorrect_index = incorrect_labels[0][i]\n",
        "    im_incorrect[:, i] += test_image[:, incorrect_index]\n",
        "    label_incorrect_index.append(test_label[incorrect_index])\n",
        "\n",
        "for i in range(10):\n",
        "    plt.subplot(2, 5, i+1)\n",
        "    value = incorrect_predicts[0][i]\n",
        "    plt.title(value)\n",
        "    plt.imshow(im_incorrect[:,i].reshape((size_row, size_col)), cmap='Greys', interpolation='None')\n",
        "\n",
        "    frame   = plt.gca()\n",
        "    frame.axes.get_xaxis().set_visible(False)\n",
        "    frame.axes.get_yaxis().set_visible(False)\n",
        "plt.show()"
      ],
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "correct image 10 collections\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWAAAADSCAYAAAB0FBqGAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAffElEQVR4nO3dd5hU1f3H8fexolgRxRIVUTEq9t5+YsXYxVgewa4YS6Ki2Bt2rFhjLxFLLJgYNIglJtaoxN6wgb0Qsfd4f38snz0zszPsLjsz5w77eT0Pz+7OzM6cvdw5873nfM/3hCzLMDOz+psudQPMzDord8BmZom4AzYzS8QdsJlZIu6AzcwScQdsZpaIO2Azs0Ry2wGHELqFEO4MIXwTQpgQQtgldZtSCyH0DCHcE0KYFEL4KIRwcQhhhtTtSsnnSUs+JuWFEJYOITwYQvgihPBGCGG71G3KbQcMXAL8CPQABgB/DCEsm7ZJyV0KfAIsAKwIrA8ckLRF6fk8acnHpMTkQOWvwCigGzAIGBFC6J20XXlcCRdC6ApMAvpkWTZu8m03AO9nWXZU0sYlFEJ4BTgsy7J7Jv98NjBHlmX7pW1ZGj5PWvIxKS+E0Ad4Apg9m9zphRDGAP/Osuz4VO3KawTcG/hZJ9BkzwGd+lMcGA7sHEKYNYSwEPAbYHTiNqXk86QlH5O2C0CflA3Iawc8G/BlyW1fALMnaEue/IumN9KXwHvA08BfkrYoLZ8nLfmYlPcaTcN3Q0IIM4YQNqVpCG/WlI3Kawf8NTBHyW1zAF8laEsuhBCmoynaHQl0BboDcwPDUrYrMZ8nLfmYlJFl2U/AtsAWwEfAYcCtNAUyyeS1Ax4HzBBCWLLgthWAlxK1Jw+6AYsAF2dZ9kOWZf8FrgU2T9uspHyetORjUkGWZc9nWbZ+lmXzZFnWD+gFPJmyTbnsgLMs+4amSO/kEELXEMI6wDbADWlblk6WZROBt4H9QwgzhBDmAnYHnk/bsnR8nrTkY1JZCGH5EEKXyXMoh9OUTXRdyjblsgOe7ABgFprGbW4G9s+yrLN/ivcHNgM+Bd4AfgIOTdqi9HyetORjUt6uwIc0HZeNgE2yLPshZYNymYZmZtYZ5DkCNjObprkDNjNLxB2wmVki7oDNzBJxB2xmlki7Shl2794969mzZ42akg/jx49n4sSJoa2P7wzHBGDs2LETsyybty2P9TEprzMcF79/yqt0rrSrA+7ZsydPP/109VqVQ6uuumq7Ht8ZjglACGFCWx/rY1JeZzgufv+UV+lc8RCEmVki7oDNzBJxB2xmlog7YDOzRNwBm5kl4g7YzCyR5Fua//jjjwAMHz4cgKOOKr9v4NZbbw3AZZddBsD8889fh9aZNZbvv/8egDFjxgCw3XbFO68ff3zT/pNrrbUWAP369atj66yUI2Azs0SSRcCffvopADvuuCMA//rXvwBYeeWVgfhJPWrUKACuvvpqAL799lsALr30UgCWWGKJOrU4vbFjxwJw0003AfDaa68B8WrgxRdfBOCJJ55I0Lp8ePLJph1m1lxzTQDuu+8+AAYNGgTAW2+9BTQtAIB4zAC6du1ar2ZWzWeffQbAxRdfDMDdd98NxHMlhOJFaTfffDMA+++/f72aaFPgCNjMLJG6R8Cvv/46ANtssw0Qo7jDDjsMgJNPPhmALl26FD1ugw02AGDXXXcFYJ111gHgqaeeAmCRRRapedvr7YcfmnZL2WOPPYA4rjdp0iQAZp21aUft2WabDYhXFaeccgoQryKmZboiuvzyywE4/PDDgRj57bnnngC8//77AEw3XVPM8c477wDxfARYccUV69Di6vjkk08AWHfddQF48803gfh3zzfffACccMIJRb+niF/nzoQJTStkF1100Rq3uPa++eYbAP7whz8AcM011xTdr2Oy2267AfEq6He/+x0A008/fT2aWcQRsJlZInWJgH/66afm74844gggRr6ahT311FMBmGmmmco+x8cff1z088SJEwG46667ADjooIOq2OK0tE+fPsH//Oc/AzG6OfPMMwHYfffdix5/xRVXADEKnJZ9/vnnQMyOefTRRwGYeeaZAXj++abNon/++WcADjnkECCOCYuuvACuvfZaIN9XU8pyUBaDIljRleF1110HxCwj0ZXkBx98AMBLLzXt1fnLL78AsNhii9Wg1bWlK8UDDzwQgD/96U9Ay/FvXSGee+65RbfvtddeAMwyyyw1bWc5joDNzBJxB2xmlkhdhiB0OQRxyGDZZZcF4I477gAqDz2ILi/efvttIF4uHnnkkUXPp0uw1p4vz26//XYg/s1yzjnnAHDooYeW/b3SCZdpkdLMNt54YyBOvPTu3RuA8847D2iZnjhy5EgAevXqBcTL0Yceeqj5MUpRy/MQhCZYNfSwxhprAHD22WcDcWJJP1900UVFv6/hqtLLc03CaSHUlltu2XzfAgssULX218KNN94IxKGHgQMHArDVVlsB8NFHHwGxVvGzzz4LxKGLlH2FI2Azs0TqEgFrEUWhY489Fmj7wLc+pS688EIAdthhBwA22mgjIEZE999/PxAnGxqJPqkPPvjgotuVfrbhhhtO8fcV3eiTXal804KXX34ZgL59+wIto5fRo0cDldOplHalNKwePXoU/T7EKDpv3n333ebvL7jgAiD+XyvS1c+6AlSanRaXzD333EBMxdICjvHjxxd9VUqWng9g2LBhQJywVCpfXijK12T1b3/7W6BllC+awMyDfB1JM7NOpKYRsFKF3nvvvebbBgwYAMD222/foedeb731gJhgv9BCCwFx7Oqxxx5rfuwKK6zQodeqF0UYioSVZqYov9In+iuvvALAiBEjALjtttuAOJ6nNJtGpIUWSiNT5KukeaWbtXUhgSJhmXHGGZu/X3DBBTvW2BopTJ1TGprGZb/66isgRq6KlnW1dP311wMw++yzF33VcdV785577gHiGLPeuwBHH300EI/54MGDq/SXVYei9aWWWgqo/D6pRJHzFltsAcSFTfXgCNjMLJGaRsAat/vwww+bb9MY3gwzVOel5523aafnBx54AIDNN98cKF6MoMyLFInW7aHxa9FYcKVPdO0mu/766wNxwYuiGEXQjUhRjf4f9f+rKExR/pJLLtmu59XCDNHCjTzS4og777yzxX2KZHV1o6wOnTNnnHEGUHmGX1cCGvfWV0WBhWVh//rXvwIx40jPmXLxU+HCLGWvTC0ds6+//hqAvffeu0PP1x6OgM3MEqlJBKzljyeddFKL+5SbV20aE1bxlUsuuaT5vv322w+IeYKNQpGPorz//e9/ADzzzDNAzPzQ7YrwVYwkRXGRatFYpors6G/5y1/+AsQrnfa65ZZbin4eMmTI1Dax5lRw5+9//3uL+wqLCAGcdtppQIxSp5bONZU8hfieuvXWW4G4FF5FbeaYY44OvebUUDYHxNzns846C4AbbrgBaH0sWPnAunrQ1ZYjYDOzTqAmEbBWJ2ncrtBcc81Vi5dsptVihWNEpdFCXunqQCt7NJZ71VVXATHSVWRU6rvvvgPg17/+NRBzRrfddtsatbh2CldiARxzzDHA1Ee+cuWVV3bo9/NGK+H69+9f1ectHBtXHvDf/vY3IGbp6HxLEQEXRreaV1JRemVaKXuodBxc8wuPP/540c+l8wP14AjYzCyRmmZB6JOlcFyqMO+yFvRpt8kmmzTftu+++wKxjsDqq69e0zZMLUW6Kgyu1UmK4JWbWRoBKwdUdTLuvfdeINZFaJQI+Lnnnmv+XgXG9X+lCHhqKV/81VdfLbq98DzJq8JVaYo2N910UyDO4Lc3G6Q9Fl54YQAWX3xxAF544YWavdbU0HmujCiVKFVmh+aiVOND2zXloYStI2Azs0RqGgG3d0VKNRVWw9Lada0R18qxvG7CWOmTeejQoUDcwFR/jzYoVR6wxu8eeeQRIGYO5DUS/uKLL4Di+h1a8aUtqjpasUo51loBpsh3ueWW69Dz1kPh+0jnrFa41bMug/5/CjcyzQOtKdAWXNtttx0QNyDVFaKys3RFoUwKXVHqfq0O/NWvflXztjsCNjNLJNm29LWm1WEQ8/pUla1wi6RGoDGs008/HYiVrZT32Jrll1++Ju2qFo1tKxKGOH632mqrdei5NVOvPFnRMa3Wisx6q2elO2U9qAZ33vXp0weI/+fK7y3NctC2aMqiUL+gSnKOgM3MpmE1+fjXuJTGIjVLCXFMr9bZEIVKK2A1GuU26xNcWR2tbaCo/4e818DQ2HYh5et2NG981KhRQMyqUJWwvF8VpFa4meeJJ54IxFoJO+20E1D7nP5q0RVjKWVNdO/eHaicX19LjoDNzBKpSQQ855xzAvGTsrAGg+qXduvWrRYv3WzSpEnN36tivsaFGzUiXnvttYHKe7+p+pxojXxeIxXVe9DKv0Irr7xyh55beb+DBg0qul377TXqOaC6BcpE0HhntV1xxRXN32vuRFdSWhmX50pyjcIRsJlZIjWdAl5mmWVa3Kac1Frt0qCZdOUEFlIdgUbZMVmRu2bydUVRKX9Z9YFF++bldQz4jTfeAGqzBl97pX355ZdA3GE6r7nf5cw///xArEYGcM011wCxbohWpXV0FwcdJ+24rdeBOJegVXf1yA5ISeeOqr/Vcr7KEbCZWSI1jYC1Jlt7SgEcd9xxQNzDSxWL2ks5e/qqVS+KGgur5GuMURFh3lWq1lRpbztF/UcccQQQ8xr1c16tu+66QJyFnjhxYvN9UxsVa4841dDVDLhm8hsx77fwalG5uBo/V8U81VtpbW5Fv6fVXo8++igA559/PhCrCBY+j46driKmNapxoSwI7QCi/QcdAZuZTYNqGg5ol1LV6YRY50DV9FVdX7s7aK8rZUuIatuKZmY/++yzottVC1cVj6C2laJqQTmYik60A25pZoDGfBWtaBxPM9iV8h/zQpGF/m9VxxXirtmKRiplLShKUb6vVrhp3FyrBeeZZ55qNr2uClcDXnjhhUDc9eTcc88F4s4hypTRVUVpTW5dLanWRin93sMPP9x8W6O9f9pLfVDh6lmIx6iWuyQ7AjYzS8QdsJlZInWZkVA5OIgpYJo82GOPPYoeq8UDKkYumohSaT4NnO+8885AnCDQ8txGThJX2zWZplKKWtCy2WabAXEbmuHDhwPxb+/Vq1fd2loNmqzV8BHAgw8+CMRlylp8omELTSKprKTOFx27xx57DIjF7RtZ4cShFpZoGa2WpWvIbsyYMVN8rtL3kYYXtA293k+N/P5pL50jSvtT8SENf9Vyk05HwGZmidQ9J0cpNYrilFajgXBNOJQu1FCkozKFiqQbdUlpW2jyUgswNOGirdX32WcfIG6trkm7ehbprgb9HypqhXglpEJE+lrJmmuuCcDIkSMB6NGjR7WbmQuKhpVSqaseXR2pOL+K76tov0ovin5fx3lafh+1RhP/Sy+9NBAjYL3/HAGbmU2DkmWlL7jgggAce+yxRV8t0iez0tBGjx4NxM03Ne55yCGHAHEMq1GpiBPAs88+C8Sx3xEjRgCw4447AvFKqF+/fkC8cqpnmdM8WHXVVYu+Kk3NOq7WBcPAEbCZWTKNty6zE9KiE33tDLRUXZtP6qtZrWip/yqrrALALrvsUvPXdARsZpaII2AzM4q3TqsXR8BmZom4AzYzS8QdsJlZIkFrw9v04BA+BSbUrjm5sGiWZfO29cGd5JhAO46Lj0l5neS4+JiUV/a4tKsDNjOz6vEQhJlZIu6AzcwScQdsZpaIO2Azs0TcAZuZJeIO2MwsEXfAZmaJuAM2M0vEHbCZWSLugM3MEnEHbGaWiDtgM7NE3AGbmSXiDtjMLBF3wGZmibgDNjNLxB2wmVki7oDNzBJxB2xmlog7YDOzRNwBm5kl4g7YzCwRd8BmZom4AzYzS8QdsJlZIu6AzcwScQdsZpaIO2Azs0TcAZuZJeIO2MwsEXfAZmaJuAM2M0vEHbCZWSLugM3MEnEHbGaWiDtgM7NE3AGbmSXiDtjMLBF3wGZmibgDNjNLxB2wmVki7oDNzBJxB2xmlog7YDOzRHLbAYcQuoUQ7gwhfBNCmBBC2CV1m1LzMakshLBkCOH7EMKI1G1JzedJeXk8LjOkbsAUXAL8CPQAVgTuDiE8l2XZS2mblZSPSWWXAE+lbkRO+DwpL3fHJWRZluq1KwohdAUmAX2yLBs3+bYbgPezLDsqaeMS8TGpLISwM9AfeBlYIsuygYmblIzPk/LyelzyOgTRG/hZB2qy54BlE7UnD3xMygghzAGcDAxO3Zac8HlSXi6PS1474NmAL0tu+wKYPUFb8sLHpLxTgKuzLHsvdUNywudJebk8LnkdA/4amKPktjmArxK0JS98TEqEEFYENgZWSt2WHPF5Ul4uj0teO+BxwAwhhCWzLHt98m0rAJ15EsHHpKW+QE/gnRACNEU504cQlsmybOWE7UrJ50l5uTwuuZyEAwgh3AJkwD40zVjeA6zdmWdyfUyKhRBmpTiqOZymDnn/LMs+TdKoHPB5Ul4ej0tex4ABDgBmAT4BbqbpTdWpTyB8TIpkWfZtlmUf6R9Nl5nfd+bOdzKfJ+Xl7rjkNgI2M5vW5TkCNjObprkDNjNLxB2wmVki7oDNzBJxB2xmlki7FmJ0794969mzZ42akg/jx49n4sSJoa2P7wzHBGDs2LETsyybty2P9TEprzMcF79/yqt0rrSrA+7ZsydPP/109VqVQ6uuumq7Ht8ZjglACGFCWx/rY1JeZzgufv+UV+lc8RCEmVki7oDNzBJxB2xmlkheq6FZG2y22WYA3HvvvQAss8wyALz0kpf9mzUCR8BmZok4Am4gP/74Y9FXRb6Ta+Hyyy+/APDTTz8BMOOMM9a7iZZT7777LgArrdRUu37SpEllH/fKK68A0Lt37/o0rJNzBGxmlogj4AZy1VVXAXDQQQeVvf+1114DYOzYsQCsueaa9WlYB628ctPmFX379m2+7bzzzkvUmsb23XffAfDQQw8B8Vz54osvAPj8888B2GGHHQCac3DHjx8PwJ133gnAkUceWZf21tMHH3wAxFzljz/+GIBBgwYB8NFHHwFwySWXAPH9tMEGG9SsTY6AzcwSyV0E/MknnwDxk/nGG28EoE+fPgCssMIKQPwUm2+++erdxLp7+OGHATj88MPb9Pgrr7wSiON9M888c20aViVvvfUWAM8++2zzbaeeeioAs846a5I2NZq99toLiGO4Tz31VNH92njhiCOOAGDo0KFAHBteaqmlALj00ksB+P3vf9/8u9PK/4H6DPUxmjvR+0XH6MknnwTi++aZZ54BYM4556x6mxwBm5klkiwC1kz+448/DsRxFn0q6dOotZ/32WcfAI477jgAFlhgAWDayAD49ttvAdhtt90A+P7779v0e9deey0ABx98MADLL798DVrXWN5++20AunTpAsTzpNEpOrv99tuBeM6U2nXXXQE4+eSTgfj+mHfe4vow7733HgBffvll822NGgF/9VXTjvN33HEHEMd81YfI/PPPD0C3bt2A2KeceOKJAGy66aYA/OMf/2j+nWodE0fAZmaJuAM2M0sk2RDEZZddBsDgwYOBeFlQennQ2s/XXHMNAFdffTUQU5ruvvvu5sc06kTdrbfeCsCECe2qeths2LBhQExfm2WWWarTsCpbaKGFgOLLXqVNVetS79xzzwXgiiuuAGDMmDFAcepbI9JkZaWhBznmmGOAtg/N6XkBLr744qlsXVoHHHAAADfffPMUH3fhhRcC0L9//6LbNZGp3y9MjdSQZ0c5AjYzS6TuEfAhhxwCxE+d0kk1ae1nRbVKKdH9//nPf4CYrgYxrWThhRfu+B9QB2+88QZQecGFrL766kBMT7vrrruAmLqnT+6BAwcC8Jvf/Kb6ja0C/R8WauuEY1tpIkVpVqNGjQIaNwI+88wzARg5ciQA001XHEspZeqJJ54AKi8tnmmmmQD4v//7PyBONJW+3xqJFpuMGzeu7P06FgMGDABgueWWK/u44cOHA61H0B3hCNjMLJG6RMCKeiGO2VYa89UYrj7ZRZ9qr776KhAjGo0bal8pPV9hVKUluUrZyfuY8COPPAK0Pq73z3/+E4ipVRtuuCEQI2C56aabgPxFwC+//DIA//3vfwGYe+65m++rVpqYis4cddRRVXm+1JRaddZZZwEx8tV5P2TIEACOPfZYAGabbbYpPp/OnX79+gHxnCp9XzYC9RFbbrklEJfkyxJLLAHAAw88AOQjFdERsJlZIjWNgF9//XUgjvdC/GTV7LaSno8++migcnSq8dvS8RqNdWnppCLsr7/+uvkxKrKhAub//ve/gfwt1lDUv99++03xcUqmV/TSGi1PLYyo85BcXzq2pugN2v63lVKEqOfWOJ6unBqdFhMUZoxAPHZaYqyx3alVeAWqGX8tWMgrRbYa9y6lJf15ugJ2BGxmlkhNImBFWorkCseT9L0ik7333rsqr6nn22WXXYCYAwixyMtzzz0HxBzkwoIjeaAlkyqoXkrLtVVQpZSWUiqLQmNeygy57777mh+7zTbbVKHFHaNCMKJcXYgFUtZff30All56aSAWVOnVq1fR7+pv03OowI9KDa677rpAzIluVJXKdCrPt6ORrxTOoVQ7I6Xe1A/lKfIVR8BmZonUJALWrLZmVAtzCrfeemugepFvKeXGqqgywCabbALEcWFlZeQlAtZWQj///DPQMgdzhhma/pvOOOMMoHKUo6sLfdIvu+yyALz44osAjB49uvmxKSNgjdHdcMMNRbfrvCn8XkV02kqRrp5bEbNypBsxAtbsPsTiVRbp/aLVjfpZ539hX9CR569FbrQjYDOzRGoSAWtbk3K5hKecckotXrKFNdZYo/l7Rdulq+/yQlul3HbbbUDL9h166KFAjO5bo9xPlaHUNvV5+bv19yii2HPPPQE47bTTmh8z/fTTT9VzK5dYVw3Tgs8++6z5++eff77oPh2z2WefvUOvof8LXY0VrqzL+6o4zReoHozO847WsNA2TZXWLFSDI2Azs0RqEiZojLU05xegR48etXjJKdIY8AUXXFD3126Le+65Z4r3b7/99u16Po0lf/PNN0W3K0siFUXiysNWpH788ccD+c8zzYPWqgN29HlLV9ZV8zWqSfneAOeff37RfVoRqvH/9tJmEaeffjoQz0tl1FSTI2Azs0RqEgGXjpmsttpqzfeVboFST7Ucy+kI5bhWsuSSS7br+VRjQTP/c801FxC3KEplnnnmAeLfq9xt1fGop8IIKu9KI7xq0tXS+++/X7PXqAVFqRBX3Iry5Kd2tecJJ5wAxCs1VVOsRR6xI2Azs0RqEgHnYdb0nXfeaf5e9SbUrhTj0FOy2GKLTfF+zeaqwlWlDAHlzG6++eZFt2+xxRZA2qsPiGNpGgtOacSIEQBcfvnliVvSujfffLNmz61Vq6qTLIWV8/L2fiml9/U666wDwNprrz1Vz6MVc8oVVyS94IILdrSJFTkCNjNLpC5jwNrqGuInbrWrcel5lelw0UUXNd+nde2KwDS20yhOOukkINYJnnnmmYE4VqXqaKr58OGHHxb9vmpCdGZdu3Yt+ln1DQrHD9s71j4tePDBB8veXlhvJK97CYr6GdU0bmvfonFkvY8U+SorR1ecteQI2MwskZpEwKX7tak6F8TxrEr7MLVGka6eUzsHK2ev3B5ziny1Qi9ve8NpjE1jt5Xygu+///6in7WvWaWsDj1vYWW4zkp7nmm8XePlpXV186hwRVdplK6dPtZaay0g/p2t0Riv6ieIVtatt956U9fYOilc+bf77rsDcVxfWT8HHnjgFJ9D2SXnnHNO0e06R+pRL9wRsJlZIjWJgPWputJKK7W4T7tSaMffwt2L2/Kcig4VSZeON+trYbW1s88+G4i7Z+TNDz/8ABRnblSDrg5SZz/kgaIZ5aQrAi6MAFdZZZX6N6wNFl100ebvd9ttNyBWe9PKNe2D9sc//hGArbbaCoj5zjq3VIfjhRdeAOL7RRFvYQ2VPCusCKiI9frrrwdi3m6lCFi7ilx33XVFt2+77bYA7LTTTlVt65Q4AjYzS6QmEfDiiy8OxE8S7coLcX82VfDXJ7jGbEvHcFv7WeOcqu2r1V552POsrVQTQeNvujoo3TGirQ477DAgjgtaZYqG8qww71sZMQ899BAQI1vNjWg8VHMsqiWsx+n9o6tB1RlRHnBruyjnUWm93oEDBxbdP27cOACWWmopoLjSG8TIVzvS1JMjYDOzRNwBm5klUpMhCF3+33jjjUDchghg8ODBQByKaK28XunPWlbcp08fIE5K5HWCrT00caKJoltuuQWIQxOFW/YUUvpa7969ARg2bBjQ8lLLpr7Qe15oQu7RRx8F4iT2pEmTih6nSbZSiyyyCBCLmG+00UY1aWc9lU7A77DDDgCceOKJQExRLS21OWDAACDtcnS/Q83MEqnLvi2FaR36XktAS0vJlerbty/QWJNqHaWFIypsr6/Wcbpi0sSwFrNALI/ZCFQgRsvPH3vsMQDeeustIC6n1XZPG2+8MQDbbbcdUL3t6/OgtEykNt8dMmRI0e3apFMLW1SwvUuXLrVuYkWOgM3MEkm2c6GWVHbGAiiWjiLBoUOHAnDqqaembE6HaVl96eKBo48+OkVzkth3330B6N69OxBTUpViqDKTHd2evhYcAZuZJTLt7N1t1gbKglAJQn21xqWshv79+xd9bQSOgM3MEnEHbGaWiDtgM7NE3AGbmSXiDtjMLJHQni3kQwifAhNq15xcWDTLsjZXMO8kxwTacVx8TMrrJMfFx6S8sselXR2wmZlVj4cgzMwScQdsZpaIO2Azs0TcAZuZJeIO2MwsEXfAZmaJuAM2M0vEHbCZWSLugM3MEvl/3E+AmySsRbMAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 10 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        },
        {
          "output_type": "stream",
          "text": [
            "incorrect image 10 collections\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWAAAADSCAYAAAB0FBqGAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAaPElEQVR4nO3deZRUxfnG8W/J6oomcCDKMh5XcE1EIRqDC5pIYkxQ1CgS1ChRIOZEMS5EQQmiWVyCqDEgYlCyYNyOiolrzKIBExcUtwRcfmpERVEQRe/vD87Dne7pZmaY7lu3Z57PP8M0TXdNcaf6vVVvvRWSJMHMzLK3QewGmJm1VR6Azcwi8QBsZhaJB2Azs0g8AJuZReIB2MwsEg/AZmaR5HIADiF0CiFMDyEsCSEsDyH8O4RwSOx2xeQ+ach90pD7pLQQwpgQwvwQwqoQwszY7ZFcDsBAe+BlYBDQBRgP/C6EUBexTbG5TxpynzTkPint/4BJwIzYDakv1MpOuBDCE8DEJEnmxm5LXrhPGnKfNOQ+SYUQJgE9kyQZGbstkN8IuEAIoTuwPbAwdlvywn3SkPukIfdJvuV+AA4hdABmA9cnSbIodnvywH3SkPukIfdJ/uV6AA4hbADcAHwEjIncnFxwnzTkPmnIfVIb2sduQDkhhABMB7oDQ5Ik+Thyk6JznzTkPmnIfVI7cjsAA1cBfYHBSZKsjN2YnHCfNOQ+ach9UiSE0J414107oF0IoTOwOkmS1VHblccsiBBCH2AxsAqo30GjkiSZHaVRkblPGnKfNOQ+KS2EMAE4v+jhiUmSTMi+NalcDsBmZm1BrhfhzMxaMw/AZmaReAA2M4vEA7CZWSQegM3MImlWHnDXrl2Turq6KjUlHxYvXszSpUtDU5/fFvoEYMGCBUuTJOnWlOe6T0prC/3i35/Syl0rzRqA6+rqmD9/fuValUP9+/dv1vPbQp8AhBCWNPW57pPS2kK/+PentHLXiqcgzMwi8QBsZhaJB2Azs0g8AJuZReIB2MwsEg/AZmaR5LkesBX56KOPAPjXv/4FwMCBAwE4/vjjATjhhBMA+NKXvhShdWbWXI6AzcwicQRcQ+68804AjjjiCAA22GDN5+esWbMA+Otf/wrAxIkTATj66KOzbmLVTJs2DYDRo0dX5PWuvPJKAE499dSKvJ7Z+nAEbGYWiSPgHHv11VcBmDJlCgC/+c1v1vn8F198EYAzzzwTaB0R8OGHHw7AzTffXNHXVSR94IEHrn1shx12qOh7xLR06VIA7rnnHiC9hg455BAAvv/97wOw1VZbRWidiSNgM7NIchsB/+9//wPgueeeA2DYsGEAvP766wCsOXkbxo0bB8DFF1+cdROrTlHfVVddVfLvR40aVfBV85p33313Bq3LRqUj32L33nvv2j/XcgT87rvvAnD11VcDcPnllwPp74s89dRTAEydOhVI75p69OiRSTurSX3w6aefFjw+e/aas0jnzJkDwGuvvQbA+++/D8Cee+4JpGsrw4cPB6B9++oPj46Azcwi8QBsZhZJbqcg9t9/fwCeeeaZgsc19SCXXnopAHvvvTcAhx12WAatq64VK1YAMGnSpJJ/f/rppwMNp12GDh0KwG233VbwOhtttFFV2tka1Hoa2nvvvQfA9ttvD8Cbb75Z8PcHH3wwACeffDIAjz32GJAuyp1xxhlA4wu8efPhhx+u/fOFF14IpGPBqlWrSv6bJEmAhmPIXXfdVfB1v/32A9bUKq42R8BmZpHkLgI+7rjjAHj++eeb9PzVq1cD6Tbd1mDs2LEAvPXWWwWPH3rooQD85Cc/Kfnv9In9zjvvAHDMMccAcNNNNwGw4YYbVrytsS1atAgoXEyDym3YyCtFvvo/X7ZsGQC9evUC0mhuxx13BNJNO7pL0iL3vHnzgDRq7NSpU7WbXhEPPPDA2j8rmq9FjoDNzCLJXQSsFCqlgNxyyy1AmioyefJkIE2zaU2WL18OwIMPPljweO/evYE0Ha1ceoyinU033RSAO+64A4A33ngDyGZOq9IUsRWno+lxpY4Vp5AVz+0+++yzQBop19+AUYuUbqbI99vf/jaQbtnu0qVLwfN1p6hrYuHChQC8/PLLQHrt1UoEvO2221bttevPL1ebI2Azs0hyEwFr9VafPkcddRQAQ4YMAdIVzFdeeSVC67KhpPjFixcD6byd5nCbmix/3nnnAfDDH/4QSKO+E088sWJtzYoi1eIIWN8r4itHkXC5SLlWaY63c+fOAFx33XUAdOzYseTzFTFrC7Io22iLLbaoSjurZZNNNln7Z93xffDBBwAMHjy45L/RGHLkkUcCaeaIyrjq90/X1BVXXFHpZjfgCNjMLJLcRMAXXXQRkH6K/fGPfwTgkksuAeD2228HYO7cuSX//QsvvFDtJlaN5udURlI07z1gwIBmvV5xlKcylt/5znfWPpbFNstKUARbLquhsWwH/X1rKT+pbB/dJSlrqFzku3LlSqB85sxJJ50EQLt27SrZzKqrfzeoTBj1jdZMmkrXhPLrR4wYUYkmNokjYDOzSHITBm299dYF3yvPsXv37k3699VcFa025WAqypef//zn6/V6xUcS3XrrrUBhrnStRMCVokhY8+Hl7qTyTpGushzK7fz8+OOPgTSaUyaMaH3hi1/8YlXamaWWFhIq3lH60ksvAdC/f/8WvW5TOAI2M4skN2HQyJEjATj33HOBtFRcW3DttdcC6SqtVnV32223Fr2uXq/c922RsidU6L1WI2FFfaqDoGwh3eUoT77cz3fQQQcB0KdPn6q2M880P64caNHvXxYcAZuZRZKbCFifOsq90yf3o48+CsD1118PpMXHtYNHUaLqJNQizcepSpOyFda3doNqSBRXfSr+vpYoi6FSNR6qXei92s466ywgzZPu2bMnkM79Kq9eecI6ikjZRX379s2usTmlHaeKhGW77bbLrA2OgM3MIslNBCyaC9ZX5QVfdtllQBr5imqA6pPe0voZomin1nI96yuXv6sIsDj3WbUfzjnnHKB8xKu5YKit+WDVrNXuRq0j6C5Hvw/aAafjetQPxVlHFocjYDOzSHIXARd7+umngYY1PxUhDxo0KOsmVZ3mpMpV8G/MQw89VPD9AQccANROpat1aepONkXEimrLHW9f63PB11xzDZCebKFj6HXApLIldDKGrqUsclzz7u233wbSQzxjrJE4AjYziyS3EbCqop1yyilAOhes880uuOACoLZX9kUr1KpeNmPGDAAmTJgAwJZbbtmk19Ge+OIddXmn+dr6Kl21TJFwa7he6tPPo8pe+mqNU8aVspCGDRsGtHxnXXM4AjYziyR3EbAiXdV20B52fUrpxF/lPbYGOsdrm222AdK6pKrp8Pvf/x6APfbYY52vo9MOPvnkk4LHs6zu1ByKfHWSRynK/y2X7dDc92prlBfcmutoN5d2vhXXxxgzZgyQbUaVI2Azs0hyFwHrU6j400m7VvbZZ5/M21Rt+sRV1bKdd94ZSKsyaQV7r732AtIVbvnDH/4AwH333QekpwWoTkCWO3uaoynRbLmdb02NjHW6QWOv01op2tP6gKV3A/r90t11jCwhR8BmZpHkJgLWiv+sWbMA2GmnnYA0quvatWuUdmVps802A9LTi3XqgXYx/elPfyr4Wi5PWK8zduzYqra3UhSdadcaNJ6fW6maELV+QkZjHnnkkZKP77777hm3JL9Uc1wn0GTJEbCZWSQegM3MIok2BaHb5zlz5gDpgpFun7Ww1K1btwiti0MbLh5//HEAxo0bB6QLkE1Npdp1112r0Lrq0QLa5MmTG/xdtbYKDx06tCqvmzf9+vUr+XiHDh0ybomV4gjYzCySaBGwiuwce+yxBY/ff//9QOW3otaSjTfeGEhTpFRgfeDAgUC6OCcq4K7FOi1k1pr6/+faOtxYGllzacGvrVxf+j2z1NSpU2M3YS1HwGZmkWQeAavUYvHR6RMnTgRqb/6ympRephS8F154IWZzolCamL5qHlzHyzcWGVdqK3Ot2nzzzWM3IXdWrFhR8P2yZcsAmD59OpAWuc+CI2Azs0gyj4D//ve/A+l85dlnnw3A+PHjgdZXLtAqSxGsvrb2jRQtNWDAAAD23XdfIN3WXsvHU7XUaaedBqRb/zUWdezYMfO2OAI2M4sk8whY+b2a61UE7MjXrPJUaEa55JZuw1af6IDTLAuxiyNgM7NIMo+AlddpZhbTLrvsAqR59jE4AjYziySoJkOTnhzCm8CS6jUnF/okSdLkAhRtpE+gGf3iPimtjfSL+6S0kv3SrAHYzMwqx1MQZmaReAA2M4vEA7CZWSQegM3MIvEAbGYWiQdgM7NIPACbmUXiAdjMLBIPwGZmkXgANjOLxAOwmVkkHoDNzCLxAGxmFokHYDOzSDwAm5lF4gHYzCwSD8BmZpF4ADYzi8QDsJlZJB6Azcwi8QBsZhaJB2Azs0g8AJuZReIB2MwsEg/AZmaReAA2M4vEA7CZWSQegM3MIvEAbGYWiQdgM7NIPACbmUXiAdjMLBIPwGZmkXgANjOLxAOwmVkkHoDNzCLxAGxmFokHYDOzSDwAm5lF4gHYzCwSD8BmZpF4ADYzi8QDsJlZJB6AzcwiyeUAHEIYE0KYH0JYFUKYGbs9eRBC6BRCmB5CWBJCWB5C+HcI4ZDY7YrN10ohXyel5bVfcjkAA/8HTAJmxG5IjrQHXgYGAV2A8cDvQgh1EduUB75WCvk6KS2X/RKSJIn5/usUQpgE9EySZGTstuRRCOEJYGKSJHNjtyU2Xyvl+TopLQ/9ktcI2BoRQugObA8sjN0Wyy9fJ6XlpV88ANegEEIHYDZwfZIki2K3x/LJ10lpeeoXD8A1JoSwAXAD8BEwJnJzLKd8nZSWt35pH7sB1nQhhABMB7oDQ5Ik+ThykyyHfJ2Ulsd+yeUAHEJoz5q2tQPahRA6A6uTJFkdt2XRXQX0BQYnSbIydmPywNdKSb5OSstdv+QyCyKEMAE4v+jhiUmSTMi+NfkQQugDLAZWAfUHl1FJksyO0qgc8LVSyNdJaXntl1wOwGZmbYEX4czMIvEAbGYWiQdgM7NIPACbmUXiAdjMLJJm5QF37do1qaurq1JT8mHx4sUsXbo0NPX5baFPABYsWLA0SZJuTXmu+6S0ttAv/v0prdy10qwBuK6ujvnz51euVTnUv3//Zj2/LfQJQAhhSVOf6z4prS30i39/Sit3rXgKwswsEg/AZmaReAA2M4vEA7CZWSQegM3MIsllOcq27qOPPgLgsssuA+Css84q+bxvfOMbAFx99dUA9OjRI4PWmVmlOAI2M4skdxHwihUrAPj1r38NwGmnnQbAlVdeCcCpp54ap2EZePPNNwE48sgjAXjooYcA+MIXvgDAj3/8YwDuuOMOAKZPnw6kfTZt2jQAtt1224xabGYt4QjYzCyS3ETADz/8MAAnnXQSAM899xwAG2yw5jPiRz/6EQCHHXYYAFtttVXWTaya559/Hkh/tmeffRaA008/HYALLrgAgM6dOxc8b//99wfguOOOA2CfffYB4J///CcAvXv3rnrbq+Wuu+4C4Gtf+xoAOjjgqaeeAmCnnXZq0esvWrTmMNz//ve/ax/bb7/9ANhwww1b9NqWTx9/vOYION1Fn3jiiQAMHDgwWpscAZuZRRI9An788ccBOOCAAwD45JNPgHTlv1evXgCMHj0agMMPPxyAK664AoC99toru8ZWkD6NAc4880wgjXy/8pWvADBp0iQAOnbsWPI13njjjYLvly5dCsBtt90GwJgx0U/dbpY777xz7Z+HDx8OwJqDbBt+XV/KMJkyZQoAN9xww9q/W7BgAQC77757i94jhldeeQVI1060ZqI7h4MPPhiAfv36ATB+/PismxiN+uDaa68F0rWTm266CYCLL74YSMeYLDkCNjOLxAOwmVkk0acgzj33XCCdetDtwBlnnAHAM888A6S3nsuXLwfSKYuFCxcC0KdPn4xaXBkzZ85c+2dNGWhhae7cuUD5qQfRLZMWkq677jogXbDU62lxrrHXy5r+Ly+99FIgnXKB9HqQL3/5ywD07NmzRe+5ZMmaqoD1px5E01533313i96jmt5++20gXUDSIrWmbzTFIrr9/u1vfwukm3XawhSEpvn23ntvIJ1i2nzzzYF0TNG0zZAhQwDYZJNNAOjWrcmlntebI2Azs0iiRcBPP/00APPmzSt4fMSIEQXf9+3bF4Cbb74ZgE033RSAwYMHA/Duu+9WtZ3VooWA+nQ30NQ0KEW0WpAcNmwYAAceeCCQ9tGf//xnIE1by4sHH3wQgIkTJ5Z9jlLDFOF16tRpvd7r9ddfB9IFzlIGDRq0Xq+dJS1Cq++KFycV4SrV6jOf+QyQbmt///33s2tsJLp7+sEPfgCkke9nP/tZAP72t78B6SLchAkTANhmm22AdMPTuq7LSnEEbGYWSbQIWPMzn376KQBdu3YF0s0GxVR4RpsTatWyZcuANG0I4NhjjwXS6GZ97bvvvgC8+uqrQLpZ5etf/zqQfvID7Lbbbi16r5b48MMPAbjooovKPkeR7tlnn13w/fq69957gXQOuJSRI0e26D2yoI1Kiua22247IL370bb1YroL+sc//lHtJkazevVqIC1fcNVVVwGwxRZbAPDyyy8D6RhT7m7g1ltvBeCcc84BWn7trYsjYDOzSKJnQYg+0du3L92kt956C0hLL2olc8stt8ygdZWjue/XXntt7WOa5yz3szeXVm8V9Wl1V5klkGZexNh2e//99wPrjsY019/S7dS609IaQq075phjCr421fHHHw/A7bffDsCTTz4JwC677FLB1sWlOyoVpdJdte78iu+uNS9eTEWxdHdeTY6AzcwiiRYBd+nSBUg/lbQNVxHuySefDKQ5eYqaNH+oFWt9yuWd8jO14lrfoYceWpX31Jywoh9tTwUYNWoUALNmzarKe69LXV0dkEbqijjq07ZqzWkqctP1ovxWbUUvd/egQke33HJLwePt2rUDCiPj7t27N/MnqR3KhtA8aWvKhlAmlPYQKCNEuc+aJ1dE+7Of/QxI+6LY5z73OSC9RqrJEbCZWSTRImBFQcrv1M62cePGAbB48WIATjnlFADGjh1b8O9vvPHGDFpZOR988AGQzsvWp/nsatEnfv3iPYoMY1But/KUlY9ZysqVKwueW+yb3/wmUH6X31/+8peSj6tPlCHS2lW6qFGeaFejDibQ3Z3GFN0tTZ48GYDzzjtvna+nDKXi3ZjV4AjYzCyS6FkQqlOgouL6NNN8Zf15S4CDDjoISFfJa40+jVWvAaBDhw5VfU9Fh+o7SPNJH330USBOWc9rrrkGSLNAtGMJ4Fe/+lWTXqN4brepYuZBV4IySB577DEgrXeg+c5i9913XzYNi0i/W9rRprlhZUdccsklBc/XnL/yfF966SUgzRrKIkPIEbCZWSTRI2CtXisaeuSRR4D0SKJi2p+dxQplNcSce6t/WKeqaB1xxBFAWnVu4403zqw9eq/vfve7QOFONM3RFtNcuiqXyapVqwCYM2dOpZuZK8ofV8Tb2PWkqLD4edoVFvM4nkpR/Qv9jDrgQF+LKbPm8ssvB+CnP/0pkEbAWXIEbGYWSfQIWDQP88QTTwDla0LoUMYBAwZk07BWpH61L9WTVVW2+kckxVI/l7dcXq+i5hkzZhQ8rrzWxiJgze99/vOfX+92xqQcVdUr0BywojhljTRG//+aF/3FL34BpLnjteRb3/oWkO4h0I5PXRO629PjunvS3cEDDzxQ8Ho777xzdRtcjyNgM7NIchMBiyr+l6McPmVN5O2Uh3L0KaxIXxEHpMfOVzsbor6NNtoos/fKE50astlmm0VuScto96S+nn/++UB6B/mf//wHSCt6LVq0CEijvuIdpLqjqMUIWOtB2j2rNQX9rMXPE+1BeO+99woeVz3tLDgCNjOLJHcRcHFNXH2vPEadbKDaEbVSzUm1L4466iigsAaDzkbTfv1qeeedd9b++Ze//CWQzgvXekRcv75yKTptRLVhW6tdd9214KuyHFQbWqeiqLZGrf+/l6K7zcbMnz+/yi1pnCNgM7NIchMBK79RWQ6KbBUpKi9Yq9c6t2l9d0LF0q9fvwaP6Wc44YQTqvKe2hGkPqtPWQG1MpdeTPv165+oXJ92O33ve98Dajd/vFKU990aI9/mKp773XrrrQFnQZiZtQnRI2DV5FR1es2HKlpTPnBxrdZard2qs+101hnA+PHjAejTpw+w/quwyuXVV1UZ03yvVsYBZs+eDaRnidUqRb7lKqqp3m+tnZxSLV/96ldjNyE3dJq46O4oy7skR8BmZpHkJgIu3vuvPE1FxDNnziz4e+2FrzU77LADAFOnTl372JgxYwAYMWIEAFOmTAHSGriq/Ka+EO1lF+1qK86l3nHHHYHCamPlqmbVCp0uXb8f61Od31rd8VYpL774YsH3tXrnWEnKmCk+801V+bLkCNjMLBIPwGZmkUSfgpg3b17B90qifvjhh4G0RKE2YKhwuNJpapXSoiBNAbvwwguBwrKMkB7fpGOapLjUoApRH3300QCMHj0aSNNrtA26NVDh/nJb13XMU2v6mdfHPffcAzTcltuW6Wii4imIGNOajoDNzCKJHgHX3x4L6aeS0otUfm/48OEATJs2Dci2cHi1aQOGUoRUMEaLcXvuuWfB80RHDG2//fZAGkm35iR7FV7XkTzlFKcYtVWt8RDOllJhojxwBGxmFkn0CFhbjvv37w+kc7wqTKPDI3v27BmhddnSZgEVjtFXS6lQu+a1bd10/I41rlevXpm/pyNgM7NIokfAe+yxB5Aexmm2LsXFt7UmoDnOG2+8EcjmSPFaoDKUltLxTcoSGjp0KJCW6sySI2Azs0iiR8Bm66N3795A40dYtXWKgFUGtdxhp22J7roby6TJgiNgM7NI/HFo1or16NEDgCeffDJyS6wUR8BmZpGE5uwRDyG8CSypXnNyoU+SJN2a+uQ20ifQjH5xn5TWRvrFfVJayX5p1gBsZmaV4ykIM7NIPACbmUXiAdjMLBIPwGZmkXgANjOLxAOwmVkkHoDNzCLxAGxmFokHYDOzSP4fROjw+VJQcXIAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 10 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    }
  ]
}