Recognizing handwritten digits INF283 - Project - 2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of images in X_train (49000, 784)\n",
      "Number of images in X_test (21000, 784)\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from keras.models import Sequential, load_model\n",
    "from keras.layers.core import Dense, Dropout, Activation\n",
    "from keras.utils import np_utils\n",
    "from keras.wrappers.scikit_learn import KerasClassifier\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.model_selection import StratifiedKFold, cross_val_score\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "#DATA-PREPARATION STEP\n",
    "\n",
    "# Considering only a quarter of the input in order to speed up the computation \n",
    "# and try to tune hyperparameters using 3-fold CrossValidation\n",
    "\n",
    "images = np.loadtxt(\"handwritten_digits_images.csv\", delimiter=',')\n",
    "labels = np.loadtxt(\"handwritten_digits_labels.csv\", delimiter=',')\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(images, labels,test_size=0.3, random_state=32)\n",
    "\n",
    "# Making sure that the values are float so that we can get decimal points after division\n",
    "X_train = X_train.astype('float32')\n",
    "X_test = X_test.astype('float32')\n",
    "\n",
    "# Normalizing the RGB codes by dividing it to the max RGB value.\n",
    "X_train /= 255\n",
    "X_test /= 255\n",
    "\n",
    "#Without normalization we will get an accuracy about 40%, whit it about 97%\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best parameters set found on development set:\n",
      "\n",
      "{'batch_size': 50, 'nb_epoch': 15}\n"
     ]
    }
   ],
   "source": [
    "#When you wrap your neural network in a Keras Classifier/Regressor, Keras can takes a multiclass [0,1,2,1] \n",
    "#and transform it into categorical representation [[0,0,0],[0,1,0],[0,0,1],[0,1,0]] just for the NN model fit. \n",
    "#So the hot encoding is not needed.\n",
    "\n",
    "def make_model():\n",
    "    \n",
    "    model = Sequential()\n",
    "    model.add(Dense(512, input_shape=(784,)))\n",
    "    model.add(Activation('relu'))                            \n",
    "    model.add(Dropout(0.2))\n",
    "    model.add(Dense(10))\n",
    "    model.add(Activation('softmax'))\n",
    "    model.compile(loss='categorical_crossentropy', metrics=['accuracy'], optimizer='adam')    \n",
    "    \n",
    "    return model\n",
    "\n",
    "\n",
    "def tuning(X_train,Y_train,X_test,Y_test):\n",
    "\n",
    "    batch_size = [50, 80, 100, 128, 135, 150]\n",
    "    epochs = [15,20,25]\n",
    "    param_grid = dict(batch_size=batch_size, nb_epoch=epochs)\n",
    "\n",
    "    k_model = KerasClassifier(build_fn=make_model, verbose=0)\n",
    "   \n",
    "    clf = GridSearchCV(estimator=k_model, param_grid=param_grid, \n",
    "                                   cv=3,\n",
    "                                   scoring=\"accuracy\", verbose=0 ,n_jobs=-1)\n",
    "    clf.fit(X_train,Y_train)\n",
    "    \n",
    "    print(\"Best parameters set found on development set:\")\n",
    "    print()\n",
    "    print(clf.best_params_)\n",
    "    \n",
    "    model=make_model()\n",
    "    \n",
    "    return clf,model\n",
    "    \n",
    "clf = tuning(X_train,y_train,X_test,y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 49000 samples, validate on 21000 samples\n",
      "Epoch 1/15\n",
      " - 4s - loss: 0.2527 - acc: 0.9254 - val_loss: 0.1411 - val_acc: 0.9584\n",
      "Epoch 2/15\n",
      " - 4s - loss: 0.1125 - acc: 0.9663 - val_loss: 0.1080 - val_acc: 0.9665\n",
      "Epoch 3/15\n",
      " - 4s - loss: 0.0770 - acc: 0.9762 - val_loss: 0.0952 - val_acc: 0.9713\n",
      "Epoch 4/15\n",
      " - 4s - loss: 0.0597 - acc: 0.9809 - val_loss: 0.0828 - val_acc: 0.9743\n",
      "Epoch 5/15\n",
      " - 4s - loss: 0.0475 - acc: 0.9848 - val_loss: 0.0825 - val_acc: 0.9757\n",
      "Epoch 6/15\n",
      " - 4s - loss: 0.0372 - acc: 0.9882 - val_loss: 0.0724 - val_acc: 0.9780\n",
      "Epoch 7/15\n",
      " - 4s - loss: 0.0302 - acc: 0.9902 - val_loss: 0.0743 - val_acc: 0.9780\n",
      "Epoch 8/15\n",
      " - 4s - loss: 0.0252 - acc: 0.9919 - val_loss: 0.0764 - val_acc: 0.9770\n",
      "Epoch 9/15\n",
      " - 4s - loss: 0.0232 - acc: 0.9923 - val_loss: 0.0822 - val_acc: 0.9779\n",
      "Epoch 10/15\n",
      " - 4s - loss: 0.0214 - acc: 0.9927 - val_loss: 0.0718 - val_acc: 0.9809\n",
      "Epoch 11/15\n",
      " - 4s - loss: 0.0188 - acc: 0.9938 - val_loss: 0.0760 - val_acc: 0.9791\n",
      "Epoch 12/15\n",
      " - 4s - loss: 0.0157 - acc: 0.9944 - val_loss: 0.0768 - val_acc: 0.9800\n",
      "Epoch 13/15\n",
      " - 4s - loss: 0.0145 - acc: 0.9952 - val_loss: 0.0758 - val_acc: 0.9819\n",
      "Epoch 14/15\n",
      " - 4s - loss: 0.0140 - acc: 0.9953 - val_loss: 0.0802 - val_acc: 0.9806\n",
      "Epoch 15/15\n",
      " - 4s - loss: 0.0142 - acc: 0.9950 - val_loss: 0.0801 - val_acc: 0.9813\n"
     ]
    }
   ],
   "source": [
    "model = clf[1]\n",
    "\n",
    "#Now we are not wrapping our nn into KerasClassifier so we need hot encoding\n",
    "y_train_enc = np_utils.to_categorical(y_train, 10)\n",
    "y_test_enc = np_utils.to_categorical(y_test, 10)\n",
    "\n",
    "history = model.fit(X_train,y_train_enc, batch_size=50, epochs=15 ,verbose=2,validation_data=(X_test, y_test_enc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEYCAYAAAAJeGK1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzs3Xl8VfWZ+PHPc5fsK0lYAyQsAqIsgoq1uEMFrUsdsaXY2s6o7XSmtVNspa122vl1dKat1dbWii0u1bqMdrEVFbFS9wUQFQRZA4Q9QPbtLs/vj3NuchMSchNucrM879frevZznnsx97nf7/me71dUFWOMMaa38SQ6AGOMMaYtlqCMMcb0SpagjDHG9EqWoIwxxvRKlqCMMcb0SpagjDHG9EqWoIw5QSLyoIj8vxj3LRGRi7o7JmP6A0tQxhhjeiVLUMYYAETEl+gYjIlmCcoMCG7V2s0i8oGI1IjI70RkiIg8JyJVIrJSRHKj9r9MRDaISLmIrBKRSVHbpovIWve4J4CUVte6VETWuce+ISJTYozxEhF5T0QqRWS3iPxnq+2fdM9X7m6/zl2fKiI/E5GdIlIhIq+5684TkdI2PoeL3Pn/FJGnROQREakErhORM0TkTfca+0TkHhFJijp+soi8KCJHROSAiHxXRIaKSK2I5EXtN0NEDomIP5b3bkxbLEGZgeQqYA5wEvBp4Dngu0A+zt/C1wFE5CTgMeAmoABYDvxVRJLcL+s/A78HBgH/554X99jTgGXAjUAecB/wjIgkxxBfDfAFIAe4BPiqiFzhnneUG+8v3ZimAevc434KzAA+4cb0bSAc42dyOfCUe81HgRDwTfczOQu4EPhXN4ZMYCXwPDAcGAe8pKr7gVXAgqjzLgIeV9VAjHEYcwxLUGYg+aWqHlDVPcCrwNuq+p6qNgB/Aqa7+10DPKuqL7pfsD8FUnESwCzAD9ylqgFVfQp4N+oa1wP3qerbqhpS1YeABve441LVVar6oaqGVfUDnCR5rrv588BKVX3Mve5hVV0nIh7gy8A3VHWPe8033PcUizdV9c/uNetUdY2qvqWqQVUtwUmwkRguBfar6s9UtV5Vq1T1bXfbQzhJCRHxAp/DSeLGdJklKDOQHIiar2tjOcOdHw7sjGxQ1TCwGxjhbtujLXtZ3hk1Pxr4lltFVi4i5cBI97jjEpEzReRlt2qsAvgKTkkG9xzb2jgsH6eKsa1tsdjdKoaTRORvIrLfrfb77xhiAPgLcLKIjMEppVao6jtdjMkYwBKUMW3Zi5NoABARwfly3gPsA0a46yJGRc3vBn6sqjlRrzRVfSyG6/4BeAYYqarZwG+AyHV2A2PbOKYMqG9nWw2QFvU+vDjVg9FaD2dwL7AJGK+qWThVoB3FgKrWA0/ilPSuxUpPJg4sQRlzrCeBS0TkQvcm/7dwquneAN4EgsDXRcQnIp8Bzog69n7gK25pSEQk3W38kBnDdTOBI6paLyJnAAujtj0KXCQiC9zr5onINLd0twy4U0SGi4hXRM5y73ltBlLc6/uB7wMd3QvLBCqBahGZCHw1atvfgKEicpOIJItIpoicGbX9YeA64DLgkRjerzHHZQnKmFZU9WOc+ym/xCmhfBr4tKo2qmoj8BmcL+KjOPer/hh17Gqc+1D3uNu3uvvG4l+BH4lIFXAbTqKMnHcXMB8nWR7BaSAx1d28GPgQ517YEeB/AI+qVrjn/C1O6a8GaNGqrw2LcRJjFU6yfSIqhiqc6rtPA/uBLcD5Udtfx2mcsda9f2XMCREbsNAYEy8i8nfgD6r620THYvo+S1DGmLgQkdOBF3HuoVUlOh7T91kVnzHmhInIQzjPSN1kycnEi5WgjDHG9EpWgjLGGNMr9ZvOIfPz87WoqCjRYRhjjOnAmjVrylS19TN5x+g3CaqoqIjVq1cnOgxjjDEdEJGdHe9lVXzGGGN6qX5TgjLGmJ4SDIWpC4SoC4Sob2yer2sMUR81Xxdwl935yHJDIIwCkUZqkaZqkTZr0U3XWu9D0z7a8piog0TA4xG8Ivg80jTv8TjLXo/gEcHrAa/H40yleT+v1526+/m8kf2d9Z8Yl0dhbhrdzRKUMaZPUFUagmFqGoLUul/4jcEwgVCYQEgJhMI0hsIEgq2Wo9Y1LbvHNB8ftT3oHNcQCDcnmFYJJxDqfOtnv1dI8XtJ9XtJ8nnwuN05Rnp1jHR4GOnmMbqzRzrYp/kczkxYlZAq4bAzDYXcadjZFgyFCSuEItvDzitWv1l0miUoY0zfFQ4rtYEQtQ1Bqt2kUt0QpKYhSE1jyJk2BKlpCFHbGDx2W4t9nOXOfIkej88j+L0e/F4hyedx553lyHyq30tWqp8hWcmk+r2kJnmbEswxy0nONCVqPtXvJSXJ07Te7+39d1TCrRJWJMkFw9pi26D0pI5PFgeWoIzppRqDYarqA02lgeiSQiAUJhhWp2QQmYaa54PhMI2h5vkWx0WVJEJhJRBSd+osB8POL2xn6m4Lu9tCSjDcclswHHbXNx8XOU+skn0eMpJ9pCV7SU/ykZHsIzvVz4icFNLc5bQkL+nJzfOpSV6SvB78Po8zjUowzUlHmrf53O0eDx6PdBzUAOTxCB4EvzfRkTgsQRnTzYKhMBV1AcrrApTXNlJeG+BobfR8Y4ttkXW1jaG4x+L1SNOXtM/9Mvd5BF/TVPB6mud9HsHn8ZDs9+F156O3eT3Ol77XLZF43XsckfOkt0oqThLykZHsJS3JR3qyj/QkL74+ULowPc8SlDGdEA4r5XUByqobOFTVQFl1A0drGpsTTp2TfCpqnXVHaxupqg+2ez6PQE5aEjmpfnLS/AzNSmHC0Exy3XWZKT6SfN4WVU++qFKBzyvHVE1F5iP7+bxWcjB9kyUoM+CpKhVu0jlY1UBZdWNT8mk9PVzd2G7VVVaKj5y0JHLT/OSkJVGUn05uWhLZqf6mdTlpfif5uMuZyT5LGsa0wxKU6bdCYWVveR17yusoq26grKqBQ9UNlFU1OtOoxNNWqyyfR8jPSKYgM5nBmclMHp7VtBw9HZSeRFaKz6qpjIkzS1CmT1NVyqob2VFWw46yaraX1bDjUA07ymrYeaSWxmC4xf5ej5CXntSUYMYPznQTTRIFmckURCWe7FS/lW6MSSBLUKZPqKoPuEmo1etQDVUNzfd4/F5hdF46xfnpnD9xMMX56RTmpjYln9y0JEs6xvQRlqBMr9EQDLHrcK1TCooqCW0vq6GsuqFpPxEYkZNKcX46V542guJ8JyGNyc9geE6KVbUZ009YgjI9rj4QYuvBajbuq+Tj/VV8fKCKHWU17Cmva9FdS35GMmPy07lgYgHF+RlOEipIZ9SgNFJ6y4MaxphuYwnKdJtwWCk9Wsem/ZVs2l/Fx/ur2Li/kpKyGiIN4ZJ9HsYNzuC0UblcdVohYwqc0lBRfjpZKf7EvgFjTEJZgjJxUV7byKb9VWzaV8nHB6rYtL+KzfurqIl62HR0XhoThmRy6anDmDgsiwlDMynKS8dr94SMMW3o1gQlIhcDdwNe4Leqeker7aOBZUABcARYpKql7rb/BS7BGRLkReAbauPTJ1xDMMS2gzVs2u9Uz23aX8Wm/ZUcqGy+R5ST5mfi0EyunjmSiUMzmTA0k5OGZJKebL+HzAChCvUVUHsYasqgtixqehgaqyApE1KyIDkLUrKj5qPWJWeBr2f6veuNuu0bQ0S8wK+AOUAp8K6IPKOqH0Xt9lPgYVV9SEQuAG4HrhWRTwBnA1Pc/V4DzgVWdVe85liV9QE+LK3g/dJyNu6r4uP9lWw/VNP0oGqS16meO3tsPhOGZjJxWBYTh2YyODO5qbdlM4A0VEP5TjiyA46WQNU+5ws2bRCk50NaHqTlO/Opg8Dbh36whMNQX35swmkr+USWw4G2z+VPh+QMaKyBxuqOr+1LbZm8IomrdSKLTnIeLwTrIdjQPA3UtVxuMa1vf32gjfVXPwATL4nvZ9zWW+/Gc58BbFXV7QAi8jhwORCdoE4GvunOvwz82Z1XIAVIwulR3g8c6MZYB7z6QIiN+yp5f3c5H5RWsK60nO2Hapq2F+amMnFoJnNOHsLEoU4iKspP7xM9NJs4CYedpHO0xH3tiJovgZpDLff3pUKwrv3zpeS0TFxNicxd17TNnfenNY8r0RFVCDU6SSBQC421zrRpvqbVtC5qvtY5ru5Iy4Sj7fSNmJQJ6e57yC6E4VObE3FkGj3vT20+NhSEhkrnVR+ZVkTNV0JDxbHrKkqdaX3F8T/j4/H4wZcCvuSWU3+KM03OhPSCY7f7UiC3uGvX7KTuTFAjgN1Ry6XAma32eR+4Cqca8EogU0TyVPVNEXkZ2IeToO5R1Y2tLyAiNwA3AIwaNSr+76CfCoWVrQereX93Oe+XOq9N+6qaSkYFmclMLczhM9NHMKUwhymF2eSkDdxqhgGlsaZl0om8juyA8l0Qaq7KRTyQPRJyi2DCfGcaeQ0qhtRcCAWg9ohT8qgtiyqBHG45X74T9qxx5tsrefhS3C95N2n5UlsmlehE1FjTfkJpjzfZSR5J6U4yTM113kvhjFYJJ8/54o4kUn9KZz/lqGv6nMScNqjr5wgFohKZm7TQtpNP0yvZKWX1cjElKBF5Gude0XOqGu5o/8hhbaxrfQ9pMXCPiFwHvALsAYIiMg6YBBS6+70oIueo6istTqa6FFgKMHPmTLs/1QZVpyXd+6XlbkKqYP2eiqaesjNTfEwpzOaGc8YwpTCHqSOzGZqV0jer6MJh51d81V6o3Nc8bawGxP31LVGju0kb64839UTN07zOn+p8oSWlN3+5JWVAUlrLeV9K7CWAExVsbKd00KqUULW/VSnoYMvzJGc5X9JDToaJrZJQ9kjwdtDS0uuHzCHOKxaqTimhpsxNbGUtq9VqjzQnteBB9/NNay5hJaU5VWiRz76tdUnpLf/NIvv1pSrHaF6/mzTzEh1J3MX6L3Iv8CXgFyLyf8CDqrqpg2NKgZFRy4XA3ugdVHUv8BkAEckArlLVCrdk9JaqVrvbngNm4SQxcxxl1Q18UFrOut0VfFDqVNcdqWkEIMnnYfLwLBbMHMnUkdlMKcyhOC+9b/SsEKiDyr1OFVN08olMK/dC9X4It+o5XLxOggDQMKDu2NidmMaLeJq/KJPS3fn0YxNZ07Y08CYdWzII1EXNR1VZBeqa51t/DseLKbvQLQVd7FTdRCeh1NyeS6rgXCsl23nlje2565peKaYEpaorgZUikg18DqdEsxu4H3hEVdsqk78LjBeRYpyS0WeBhdE7iEg+cMQtlS3BKaUB7AKuF5HbcX6rngvc1dk3NxAEQmGWf7iPFRsOsG53OXvKnfpoj8BJQzK5aNJgpo7MYWphDicNySTJ18vuGYWCzs3npuTTeuomobqjxx6blAGZwyBrGBR9ErKGO6/IuszhkDE4PlUZqk6COyaJhaMSRq1TWoskj8grejng7tMYSTA1TrVM5T63dOOWcI65ryBRpbPoEkGq0+CgRekgap/oKqu2Shfp+R2XgoxJkJjLtCKSBywCrgXeAx4FPgl8ETiv9f6qGhSRfwNewGlmvkxVN4jIj4DVqvqMe9ztIqI4paOvuYc/BVwAfIjzE/Z5Vf1rV95gf3WkppHH3tnFw2+WcKCygeHZKZw2OpcvnV3ElMIcThmRRVpSN1dZhEPNN23rK1rd6G3n5m7rm8GB2jZOLE5iyRwGuaNh1Cwn4WSNcJOPm4RSsrr3/bUISZzSWFv8qcAJ3ENoSzjkfDahgHP+nqweNKaXkFgeLRKRPwITgd/jVO/ti9q2WlVndl+IsZk5c6auXr060WF0u80Hqnjg9R38ce0eGoJhZo/P58ufLObc8QXxqapTheoDzU2FI6/aw8e2MIqpiWxKqyax0c99RDWPzRzaXALKGGK/6o3px0RkTSx5I9af2Peo6t/b2tAbklN/Fw4rqzYfZNlrJby2tYwUv4erZhTypU8UMX5IZudPGKiDozvbbip8dGer6iVxkkZ6gZNc8sdDcnaMz2VkOa2FjDGmC2JNUJNEZK2qlgOISC7wOVX9dfeFZmoagjy9tpQHXi9hR1kNQ7NS+PbFE/jc6aPITT9Os+9IKah1M+HIfPX+lvv7051mwXnjYNxF7g1y92Z5zkhLMsaYhIg1QV2vqr+KLKjqURG5HrAE1Q12H6nl4TdLePzd3VTVB5k2ModffG46804Z2vLB2HAIyjZD6Wo4sKFlQjqmFDTCSThNCaio+XmVtDy7v2GM6XViTVAeEZFIX3huN0b25GYcqSqrdx5l2Ws7eGHDfkSE+acO40tnF3HaqFxnp8p9zsOMe1Y7SWnvOqdPL3BaZOUWw6AxMO7CY59XOZGHCY0xJgFiTVAvAE+KyG9wWtV9BXi+26IaQBqDYf72wV4eeL2ED/dUkJ3q58Zzx/KFGfkMq9kEux+GN1c7ialyj3OQxwdDToGp18CImVA4EwaNBU8va0JujDEnINYE9R3gRuCrOM8lrQB+211BDQRl1Q384e1d/P6tnRyuquPCvCN8b+YRZvq249uxFt7+yH2wFMhxm1pHktHQU1v252WMMf1QrA/qhnF6k7i3e8Pp/zbuq+TpVe+w/6PXmaxbeDh9F+MztuKrqYH1OC3gRsxw+jYrnOnMp+cnOmxjjOlxsfbFNx5nKIyTcXoZB0BVx3RTXP1L+S62rnqUw5teZVTdRr4vR8AL6vEjeafAiIVuMprp3EOyqjpjjIm5iu8B4AfAz4Hzcfrls2Zfx1NXDh/9BT54Ana+zjgglSFUDzmDulNmk1p8JjL0VGu8YIwx7Yg1QaWq6ktuS76dwH+KyKs4SctEBBth60r44HH4+HkINaB543koZRHPMZuHF19Nsq/3d3FvjDG9QawJql5EPMAWt3+9PcDg7gurD1F1mnx/8ASsf9oZ5CwtH2ZcB1Ov4am9Bfzn0x9yz8LplpyMMaYTYk1QNwFpwNeB/8Kp5vtidwXVJxzZAR886SSmI9ucPucmzIepn4WxF4DXT30gxM8eWsXUwmwuOXVYoiM2xpg+pcME5T6Uu0BVbwaqce4/xURELsYZLdcL/FZV72i1fTTOEBsFwBFgkaqWuttG4TRlH4nz7NV8VS2J9drdovYIbPiTk5R2vw2IM8zD7P+ASZ92WuBF+d1rO9hfWc/dn53WNwcANMaYBOowQalqSERmRPckEQs3sf0KmIMzeOG7IvKMqn4UtdtPgYdV9SERuQCnpeC17raHgR+r6ovuYIaxjuQbX8EG2PyCk5Q2v+AMR10wES78AUxZ4Az21obD1Q3cu2obF00awplj+t9Il8YY091ireJ7D/iLO5puTWSlqv7xOMecAWxV1e0AIvI4cDkQnaBOBr7pzr8M/Nnd92TAp6ovuteJYVyHOFKFXW85SWnDn5wB9dIHwxk3OL03DJ3SYd91v/z7VuoCIW6ZN6GHgjbGmP4l1gQ1CDiMM4hghALHS1AjgN1Ry6XAma32eR+4Cqca8Eog0x0Y8SSg3B2HqhhYCdyiqqHog92h4W8AGDVqVIxv5TjKtjpJ6YMnoHyn07/dxEudpFR8Hnhj+7hKymp45K2dXHP6SMYN7sJwGMYYY2LuSSLm+05R2ipitK4iXAzcIyLX4YyouwcIunHNBqbjDP/+BHAd8LtWcS0FloIzYGEXYmz26NWwZQWIB4rPhfOWwKRLIbnzCeYnL3xMks/DTReNP6GQjDFmIIu1J4kHODa5oKpfPs5hpTgNHCIKgb2tjt8LfMa9RgZwlapWiEgp8F5U9eCfgVm0SlBxVfRJKJoNp17tDC/eRWt3HeXZD/dx00XjGZxpD+EaY44VCAQoLS2lvr4+0aF0q5SUFAoLC/H7uzZCdqxVfH+LviZOddzedvaNeBcYLyLFOCWjzwILo3cQkXzgiNvX3xKcFn2RY3NFpEBVD+FULXbveO5nf+OET6Gq3L58I/kZyVw/23qBMsa0rbS0lMzMTIqKivptC19V5fDhw5SWllJcXNylc8Raxfd09LKIPIZzX+h4xwTdh3pfwGlmvkxVN4jIj4DVqvoMcB5wu4goThXf19xjQyKyGHhJnH+9NcD9nXpnCbDiowO8W3KUH195CunJseZ+Y8xAU19f36+TE4CIkJeXx6FDh7p8jq5+i44HOmyVoKrLgeWt1t0WNf8U8FQ7x74ITOlifD0uEArzP89tYmxBOtfMHNnxAcaYAa0/J6eIE32Psd6DqqLlPaj9OGNEGdfj7+5me1kN939hJj6v9UZujDEnKqZvUlXNVNWsqNdJrav9BrLqhiB3r9zMGcWDuGiSdVFojOndysvL+fWvf93p4+bPn095eXk3RNS2mBKUiFwpItlRyzkickX3hdW3LH1lO2XVjXx3/qQBUWw3xvRt7SWoUCjUxt7Nli9fTk5OTneFdYxY66J+oKoVkQVVLceG2gDgQGU997+ynUumDGPayJ77hzPGmK665ZZb2LZtG9OmTeP000/n/PPPZ+HChZx66qkAXHHFFcyYMYPJkyezdOnSpuOKioooKyujpKSESZMmcf311zN58mTmzp1LXV1d3OOMtZFEW4nMmqkBd63cTDAc5tufsi6NjDGd98O/buCjvZVxPefJw7P4wacnt7v9jjvuYP369axbt45Vq1ZxySWXsH79+qbm4MuWLWPQoEHU1dVx+umnc9VVV5GX17JP0S1btvDYY49x//33s2DBAp5++mkWLVoU1/cRawlqtYjcKSJjRWSMiPwcp+n3gLb5QBVPvLubRbNGMzovPdHhGGNMl5xxxhktnlX6xS9+wdSpU5k1axa7d+9my5YtxxxTXFzMtGnTAJgxYwYlJSVxjyvWUtC/A7fidDkEsAL4ftyj6WP+57lNpCf7+PoF1qWRMaZrjlfS6Snp6c0/sFetWsXKlSt58803SUtL47zzzmuzx4vk5OSmea/Xm7gqPlWtAW6J+9X7sDe3HealTQf5zsUTyU1PSnQ4xhgTs8zMTKqqqtrcVlFRQW5uLmlpaWzatIm33nqrh6NrFutzUC8CV7uNIxCRXOBxVf1UdwbXW4XDyu3PbWR4dgpfOrso0eEYY0yn5OXlcfbZZ3PKKaeQmprKkCFDmrZdfPHF/OY3v2HKlClMmDCBWbNmJSzOWKv48iPJCUBVj4rIgH3g528f7uOD0gp+dvVUUvzeRIdjjDGd9oc//KHN9cnJyTz33HNtbovcZ8rPz2f9+vVN6xcvXhz3+CD2RhJhdwh2AESkiDZ6Nx8IGoIh/vf5TUwalsUV00ckOhxjjOm3Yi1BfQ94TUT+4S6fgztQ4EDz+zd3Unq0jt//86l4PfZQrjHGdJdYuzp6HpgJfIzTku9bQIdNNkTkYhH5WES2isgxjSxEZLSIvCQiH4jIKhEpbLU9S0T2iMg9Mb2bblZRG+CXf9/K7PH5zB5fkOhwjDGmX4u1kcS/AN/AGXRwHc7ggW/Scgj41sd4gV8Bc3AGL3xXRJ5R1Y+idvsp8LCqPiQiFwC3A9dGbf8v4B/0Er9etZXK+gBL5k1KdCjGGNPvxXoP6hvA6cBOVT0fZyj2jgb5OAPYqqrbVbUReBy4vNU+JwMvufMvR28XkRnAEJxnrhKu9GgtD7xRwmemF3Ly8KxEh2OMMf1erAmqXlXrAUQkWVU3AR317TMC2B21XOqui/Y+cJU7fyWQKSJ5IuIBfgbcfLwLiMgNIrJaRFafyKBYsfjZis0I8K25J3XrdYwxxjhiTVClIpID/Bl4UUT+QsdDvrfVgqB1y7/FwLki8h5wLs7Q8EHgX4Hlqrqb41DVpao6U1VnFhR03z2h9Xsq+NN7e/jyJ4sZnpPabdcxxpie0NXhNgDuuusuamtr4xxR22JtJHGlqpar6n/idHn0O6Cj4TZKgeihZQtpldRUda+qfkZVp+O0FMTtNf0s4N9EpATnPtUXROSOWGKNN1XnodzcND9fPW9sIkIwxpi46isJqtM9kqtqrI0W3gXGi0gxTsnos8DC6B1EJB84oqphYAmwzL3G56P2uQ6YqaoJ6WrpH5sP8frWw9x26clkpfgTEYIxxsRV9HAbc+bMYfDgwTz55JM0NDRw5ZVX8sMf/pCamhoWLFhAaWkpoVCIW2+9lQMHDrB3717OP/988vPzefnll7s1zm4bMkNVgyLyb8ALgBdYpqobRORHwGpVfQY4D7hdRBR4Bfhad8XTFaGwcvvyTYwalMaiWaMTHY4xpj967hbY/2F8zzn0VJjXfqVT9HAbK1as4KmnnuKdd95BVbnssst45ZVXOHToEMOHD+fZZ58FnD76srOzufPOO3n55ZfJz8+Pb8xt6NYxnVR1ObC81brbouafAp7q4BwPAg92Q3gdenptKR8fqOKehdNJ8sV6u84YY/qOFStWsGLFCqZPnw5AdXU1W7ZsYfbs2SxevJjvfOc7XHrppcyePbvHY7NBB9tR1xjizhWbmToyh0tOHZbocIwx/dVxSjo9QVVZsmQJN9544zHb1qxZw/Lly1myZAlz587ltttua+MM3ceKBe1Y9voO9lfW8915ExGxLo2MMf1H9HAbn/rUp1i2bBnV1dUA7Nmzh4MHD7J3717S0tJYtGgRixcvZu3atccc292sBNWGw9UN3LtqGxdNGsKZY/I6PsAYY/qQ6OE25s2bx8KFCznrrLMAyMjI4JFHHmHr1q3cfPPNeDwe/H4/9957LwA33HAD8+bNY9iwYd3eSEJU+0en5DNnztTVq1fH5Vw/+Mt6Hnl7Fy/cdA7jBmfE5ZzGGBOxceNGJk0aGF2mtfVeRWSNqs7s6Fir4mtlR1kNj769i2tOH2nJyRhjEsgSVCs/eWETST4PN100PtGhGGPMgGYJKsqanUdZ/uF+bjhnDIMzUxIdjjGmH+svt1eO50TfoyUol6py+/KNFGQmc/3sMYkOxxjTj6WkpHD48OF+naRUlcOHD5OS0vUf+9aKz/XChgOs3nmU/77yVNKT7WMxxnSfwsJCSktL6e5RGBItJSWFwsLCjndsh30T42T6O1/8mHGDM1gws+sfpjHGxMLv91NcXJzoMHo9S1CAiHDftTMpr23E57VaT2OM6Q0sQbmK89OB9ESHYYwxxmXFBWN7ei5LAAAgAElEQVSMMb1Sv+lJQkQOATtP8DT5QFkcwkkEiz1x+nL8Fnti9OXY4cTjH62qHQ6D3m8SVDyIyOpYut/ojSz2xOnL8VvsidGXY4eei9+q+IwxxvRKlqCMMcb0SpagWlqa6ABOgMWeOH05fos9Mfpy7NBD8ds9KGOMMb2SlaCMMcb0SpagjDHG9EqWoAARuVhEPhaRrSJyS6Lj6QwRGSkiL4vIRhHZICLfSHRMnSUiXhF5T0T+luhYOkNEckTkKRHZ5H7+ZyU6pliJyDfd/1/Wi8hjItKrx5cRkWUiclBE1ketGyQiL4rIFneam8gY29NO7D9x/7/5QET+JCI5iYyxPW3FHrVtsYioiOR31/UHfIISES/wK2AecDLwORE5ObFRdUoQ+JaqTgJmAV/rY/EDfAPYmOgguuBu4HlVnQhMpY+8BxEZAXwdmKmqpwBe4LOJjapDDwIXt1p3C/CSqo4HXnKXe6MHOTb2F4FTVHUKsBlY0tNBxehBjo0dERkJzAF2defFB3yCAs4AtqrqdlVtBB4HLk9wTDFT1X2qutadr8L5khyR2KhiJyKFwCXAbxMdS2eISBZwDvA7AFVtVNXyxEbVKT4gVUR8QBqwN8HxHJeqvgIcabX6cuAhd/4h4IoeDSpGbcWuqitUNeguvgX0ymEU2vncAX4OfBvo1lZ2lqCcL/PdUcul9KEv+GgiUgRMB95ObCSdchfO/+jhRAfSSWOAQ8ADbvXkb0WkT/Q2rKp7gJ/i/PrdB1So6orERtUlQ1R1Hzg/1IDBCY6nq74MPJfoIGIlIpcBe1T1/e6+liUokDbW9bm29yKSATwN3KSqlYmOJxYicilwUFXXJDqWLvABpwH3qup0oIbeW8XUgnuv5nKgGBgOpIvIosRGNTCJyPdwqukfTXQssRCRNOB7wG09cT1LUE6JaWTUciG9vLqjNRHx4ySnR1X1j4mOpxPOBi4TkRKcqtULROSRxIYUs1KgVFUjpdWncBJWX3ARsENVD6lqAPgj8IkEx9QVB0RkGIA7PZjgeDpFRL4IXAp8XvvOA6ljcX7YvO/+3RYCa0VkaHdczBIUvAuMF5FiEUnCuVn8TIJjipmICM59kI2qemei4+kMVV2iqoWqWoTzuf9dVfvEL3lV3Q/sFpEJ7qoLgY8SGFJn7AJmiUia+//PhfSRBh6tPAN80Z3/IvCXBMbSKSJyMfAd4DJVrU10PLFS1Q9VdbCqFrl/t6XAae7fQ9wN+ATl3qj8N+AFnD/SJ1V1Q2Kj6pSzgWtxSh/r3Nf8RAc1QPw78KiIfABMA/47wfHExC31PQWsBT7E+R7o1V3viMhjwJvABBEpFZF/Bu4A5ojIFpwWZXckMsb2tBP7PUAm8KL7N/ubhAbZjnZi77nr952SpTHGmIFkwJegjDHG9E6WoIwxxvRKlqCMMcb0SpagjDHG9EqWoIwxxvRKlqCM6eNE5Ly+1hO8MbGwBGWMMaZXsgRlTA8RkUUi8o77YOZ97jhY1SLyMxFZKyIviUiBu+80EXkrarygXHf9OBFZKSLvu8eMdU+fETU21aNuDxHG9GmWoIzpASIyCbgGOFtVpwEh4PNAOrBWVU8D/gH8wD3kYeA77nhBH0atfxT4lapOxek/b5+7fjpwE86YZmNwehgxpk/zJToAYwaIC4EZwLtu4SYVp3PTMPCEu88jwB9FJBvIUdV/uOsfAv5PRDKBEar6JwBVrQdwz/eOqpa6y+uAIuC17n9bxnQfS1DG9AwBHlLVFiOnisitrfY7Xt9jx6u2a4iaD2F/26YfsCo+Y3rGS8A/ichgABEZJCKjcf4G/8ndZyHwmqpWAEdFZLa7/lrgH+44X6UicoV7jmR3fB5j+iX7lWVMD1DVj0Tk+8AKEfEAAeBrOAMdThaRNUAFzn0qcIaP+I2bgLYDX3LXXwvcJyI/cs9xdQ++DWN6lPVmbkwCiUi1qmYkOg5jeiOr4jPGGNMrWQnKGGNMr2QlKGOMMb2SJShjjDG9kiUoY4wxvZIlKGOMMb2SJShjjDG9kiUoY4wxvZIlKGOMMb2SJShjjDG9kiUoY4wxvZIlKGOMMb2SJShjEkhEHhSR/xfjviUictGJnseYvsISlDHGmF7JEpQxxpheyRKUMR1wq9ZuFpEPRKRGRH4nIkNE5DkRqRKRlSKSG7X/ZSKyQUTKRWSViEyK2jZdRNa6xz0BpLS61qUiss499g0RmdLFmK8Xka0ickREnhGR4e56EZGfi8hBEalw39Mp7rb5IvKRG9seEVncpQ/MmDixBGVMbK4C5gAnAZ8GngO+C+Tj/B19HUBETgIeA24CCoDlwF9FJElEkoA/A78HBgH/554X99jTgGXAjUAecB/wjIgkdyZQEbkAuB1YAAwDdgKPu5vnAue47yMHZwTfw+623wE3qmomcArw985c15h4swRlTGx+qaoHVHUP8Crwtqq+p6oNwJ+A6e5+1wDPquqLqhoAfgqkAp8AZgF+4C5VDajqU8C7Ude4HrhPVd9W1ZCqPgQ0uMd1xueBZaq61o1vCXCWiBThDBOfCUzEGQ9uo6ruc48LACeLSJaqHlXVtZ28rjFxZQnKmNgciJqva2M5Mmz7cJwSCwCqGgZ2AyPcbXu05SihO6PmRwPfcqv3ykWkHBjpHtcZrWOoxikljVDVvwP3AL8CDojIUhHJcne9CpgP7BSRf4jIWZ28rjFxZQnKmPjai5NoAOeeD06S2QPsA0a46yJGRc3vBn6sqjlRrzRVfewEY0jHqTLcA6Cqv1DVGcBknKq+m93176rq5cBgnKrIJzt5XWPiyhKUMfH1JHCJiFwoIn7gWzjVdG8AbwJB4Osi4hORzwBnRB17P/AVETnTbcyQLiKXiEhmJ2P4A/AlEZnm3r/6b5wqyRIROd09vx+oAeqBkHuP7PMiku1WTVYCoRP4HIw5YZagjIkjVf0YWAT8EijDaVDxaVVtVNVG4DPAdcBRnPtVf4w6djXOfah73O1b3X07G8NLwK3A0ziltrHAZ93NWTiJ8ChONeBhnPtkANcCJSJSCXzFfR/GJIy0rA43xhhjegcrQRljjOmVLEEZY4zplSxBGWOM6ZUsQRljjOmVfIkOIF7y8/O1qKgo0WEYY4zpwJo1a8pUtaCj/fpNgioqKmL16tWJDsMYY0wHRGRnx3tZFV+T+kCIfRV1iQ7DGGOMyxIUoKpcde8b/McT72PPhRljTO9gCQoQEa6eUcib2w+zavOhRIdjjDGGfnQP6kQtPHM0D7xRwh3LN3HO+AK8Hun4IGOM6YJAIEBpaSn19fWJDqVbpaSkUFhYiN/v79LxlqBcST4P3/7URL72h7U8vbaUBTNHJjokY0w/VVpaSmZmJkVFRbTs3L7/UFUOHz5MaWkpxcXFXTqHVfFFmX/qUKaOzOHOFZupa7SOnI0x3aO+vp68vLx+m5zAuXWSl5d3QqVES1BRRITvzpvI/sp6lr2+I9HhGGP6sf6cnCJO9D1agmrlzDF5XDRpCPeu2sbh6oZEh2OMMQOWJag23DJvArWNQX75962JDsUYY+KuvLycX//6150+bv78+ZSXl3dDRG2zBNWGcYMzueb0UTzy1k5KymoSHY4xxsRVewkqFDr+vffly5eTk5PTXWEdwxJUO745ZzxJPg8/WfFxokMxxpi4uuWWW9i2bRvTpk3j9NNP5/zzz2fhwoWceuqpAFxxxRXMmDGDyZMns3Tp0qbjioqKKCsro6SkhEmTJnH99dczefJk5s6dS11d/HvisWbm7RicmcL1s8dw90tb+JdPHmX6qNxEh2SM6Yd++NcNfLS3Mq7nPHl4Fj/49OR2t99xxx2sX7+edevWsWrVKi655BLWr1/f1Bx82bJlDBo0iLq6Ok4//XSuuuoq8vLyWpxjy5YtPPbYY9x///0sWLCAp59+mkWLFsX1fVgJ6jiuP2cM+RnJ3L58k3WBZIzpt84444wWzyr94he/YOrUqcyaNYvdu3ezZcuWY44pLi5m2rRpAMyYMYOSkpK4x2UlqOPISPZx00Xj+f6f17Ny40HmnDwk0SEZY/qZ45V0ekp6enrT/KpVq1i5ciVvvvkmaWlpnHfeeW0+y5ScnNw07/V6u6WKz0pQHbjm9JGMKUjnjuc2EgyFEx2OMcacsMzMTKqqqtrcVlFRQW5uLmlpaWzatIm33nqrh6NrlpAEJSIXi8jHIrJVRG5pY/t/iMhHIvKBiLwkIqMTESeA3+vhOxdPZNuhGp5cXZqoMIwxJm7y8vI4++yzOeWUU7j55ptbbLv44osJBoNMmTKFW2+9lVmzZiUoSpCevrciIl5gMzAHKAXeBT6nqh9F7XM+8Laq1orIV4HzVPWa45135syZ2l0DFqoqV//mTXYeqWXV4vNIT7aaUWNM123cuJFJkyYlOowe0dZ7FZE1qjqzo2MTUYI6A9iqqttVtRF4HLg8egdVfVlVa93Ft4DCHo6xBRFhyfxJHKpq4LevWhdIxhjTExKRoEYAu6OWS9117fln4Lm2NojIDSKyWkRWHzrUveM4zRidy7xThnLfK9s4VGVdIBljTHdLRIJqq/fANusZRWQRMBP4SVvbVXWpqs5U1ZkFBQVxDLFtN39qAo3BMHe/tLnbr2WMMQNdIhJUKRA92FIhsLf1TiJyEfA94DJV7RVFljEFGSw8cxSPvbObbYeqEx2OMcb0a4lIUO8C40WkWESSgM8Cz0TvICLTgftwktPBBMTYrq9fOJ4Un4f/fX5TokMxxph+rccTlKoGgX8DXgA2Ak+q6gYR+ZGIXObu9hMgA/g/EVknIs+0c7oel5+RzFfOHcsLGw6wuuRIosMxxph+KyHPQanqclU9SVXHquqP3XW3qeoz7vxFqjpEVae5r8uOf8ae9c+zixmcmcx/L99oXSAZY/qcrg63AXDXXXdRW1vb8Y5xYD1JdEFako//mHMSa3eV88KG/YkOxxhjOqWvJCh74rSL/mlGIb97bQf/8/zHXDhpCH6v5XpjTN8QPdzGnDlzGDx4ME8++SQNDQ1ceeWV/PCHP6SmpoYFCxZQWlpKKBTi1ltv5cCBA+zdu5fzzz+f/Px8Xn755W6N0xJUF/m8HpbMn8iXH1zNY+/s4gtnFSU6JGNMX/TcLbD/w/iec+ipMO+OdjdHD7exYsUKnnrqKd555x1Ulcsuu4xXXnmFQ4cOMXz4cJ599lnA6aMvOzubO++8k5dffpn8/Pz4xtwG+9l/As6fMJhZYwZx98otVNUHEh2OMcZ02ooVK1ixYgXTp0/ntNNOY9OmTWzZsoVTTz2VlStX8p3vfIdXX32V7OzsHo/NSlAnQERYMm8Sl//qdZa+sp1vzZ2Q6JCMMX3NcUo6PUFVWbJkCTfeeOMx29asWcPy5ctZsmQJc+fO5bbbbuvR2KwEdYKmjszh01OHc/+r2zlQeeyYKcYY09tED7fxqU99imXLllFd7XQ+sGfPHg4ePMjevXtJS0tj0aJFLF68mLVr1x5zbHezElQc3Dx3As+v38fPX9zMHVdNSXQ4xhhzXNHDbcybN4+FCxdy1llnAZCRkcEjjzzC1q1bufnmm/F4PPj9fu69914AbrjhBubNm8ewYcO6vZFEjw+30V26c7iNWPzorx/x4Bs7eP6mczhpSGbC4jDG9H423EYPDbchIt8QkSxx/E5E1orI3BM9b1/z7xeMIz3Zx/88Z10gGWNMPMTjHtSXVbUSmAsUAF8CEnvXLwFy05P41/PG8dKmg7y57XCiwzHGmD4vHgkqMnzGfOABVX2ftofU6Pe+dHYRw7JTuP25jYTD/aPq1BjTPfrL7ZXjOdH3GI8EtUZEVuAkqBdEJBMIx+G8PeuF78Hz34UjXR8xN8Xv5VtzJ/BBaQXPfrgvjsEZY/qTlJQUDh8+3K+TlKpy+PBhUlJSunyOeLTi+2dgGrBdVWtFZBBONV/foQr15fD+4/DWr2HCPDjzRig+F6RzhcErp4/gt69u539f2MTcyUNI9nm7KWhjTF9VWFhIaWkp3T0SeKKlpKRQWFjY5eNPuBWfiJwNrFPVGncE3NOAu1V15wmduJPi0oqvch+s/h2sfgBqy6BgkpOoplwDSWkxn+Yfmw/xxWXvcNulJ/PlTxafWEzGGNPP9FgrPuBeoFZEpgLfBnYCD8fhvD0vaxhc8H345ga4/Nfg9cHfboI7J8GLt0H5rphOc874fD45Lp9f/H0LFXXWBZIxxnRFPBJUUJ1i2OU4Jae7gb79IJA/BaZ/Hm58Fb70HIw5F974Jdw9FZ64Fkped6oF2yEi3DJvIuW1Ae5dta0HAzfGmP4jHgmqSkSWANcCz4qIF/DH4byJJwKjPwELHoZvfACf+DqUvAoPzof7ZsN7j0Cg7e6NThmRzZXTR7Ds9R3sKa/r4cCNMabvi0eCugZowHkeaj8wAmfI9v4lZyTM+SF88yP49N0QDsFfvgY/Pxle+i+o3HvMId+aexIAd67Y3NPRGmNMn3fCCcpNSo8C2SJyKVCvqn3zHlQsktJgxnXw1TfgC8/AyFnw6s/grlPhqS/D7neaqv8Kc9P40ieK+ON7pXy0tzKxcRtjTB8Tj66OFgDvAFcDC4C3ReSfTvS8vZ6Ic2/qc3+Ar78HZ34FtqyE382B+y+A95+AYCP/et44slL83PG8dYFkjDGdEY9m5u8Dc1T1oLtcAKxU1alxiC9mie4sFoCGanj/MXj7Pji8BTKGwMwv82jwAr638hCfP3MUN54zllF5sTdZN8aY/qYnm5l7IsnJdbij84rIxSLysYhsFZFb2th+jtvpbLBPlcaSM+CM6+Fr78Cip2HYVFh1OwvfmM+fhj5IeM3D3PSzpdz8yGus31OR6GiNMaZXi0dPEs+LyAvAY+7yNcDy9nZ2W/n9CpgDlALvisgzqvpR1G67gOuAxXGIr+d5PDDuIudVthV5ZynT33+M6b4VzvatsHfLID5IGcOg4imMGD8NGXwyFEyAlKzExm6MMb3ECScoVb1ZRK4CzsbpJHapqv7pOIecAWxV1e0AIvI4zjNUTQlKVUvcbX2vT7/W8sfB/P+Fi2+H8p1wcBP1+zZQvWktSQc2kr/x98imZc37ZxU6iWrwJCiY6L4scRljBp64jKirqk8DT8e4+whgd9RyKXBmPOLo1TxeGDQGBo0hZeJ8TjofGoIh/rJ2F8/84y1Sjm5hZtoB5qQfobi6FM/O1yEY9YxVViEMntictAZPgvyTLHEZY/qtLicoEakC2mphIYCqanvfnG31vtqllhoicgNwA8CoUaO6coqESvZ5WXBGMf80s4iVGw/wm39s444d5eSm+bnurJF8cZKQU70dDm2Eg5ucaclr7Seu/JPcBHYSpOYm7o0ZY0wcdDlBqWpXuzMqBUZGLRcCxz7lGlsMS4Gl4LTi62I8CefxCHMnD2Xu5KG8W3KE+/6xjZ+/tJ17X/FwzcxR/Mvs8xg52235Fw7B0RI4tMl5tZe40gc7VYMFEyB/gpO08idA5tBO99BujDGJcMLNzDt9QREfsBm4ENgDvAssVNUNbez7IPA3VX2qo/P2imbmcbTlQBVLX9nOn9ftIRRWLpkynBvPGcMpI7LbPiAccjqzPfQxlH0Mhza704+hIeoh4eTs5mTVNJ0AOaOcakhjjOlmsTYz7/EEBSAi84G7AC+wTFV/LCI/Alar6jMicjrwJyAXqAf2q+rk452zvyWoiP0V9Sx7fQd/eHsX1Q1BZo/P58ZzxnL2uDwklpKQKlTtb05ahzZB2WYncdVEPR3gS4G88S2TVsEEGDQWfEnd9waNMQNOr05Q3aG/JqiIiroAf3h7F8te38GhqgZOGZHFjeeMZd4pQ/F5u/g4W+2R5mTVNP245bAi4oVBxZCS41YNSttTcWM43j64+7Ve5/VDdiHkFjW/skc6vcqbrlOFxhqoPQx1R5xp7dGo+SOQlO78++YWO9OsEVaSNt3OElQ/VR8I8ef39rD0le1sL6th5KBUrp89hqtnjCQ1KU5fLI01ULalZdJqrHH7GNRWU5yphtvY1tY0HDWPMw02QMXulvfQADKHuwlrdHPiynHnM4Y4z5sNFKpOVW0kybRIOkdazkcvhxrbOaFASrbz7xqOGrPM43eqe6OTVtO0CPypPfFuB7ZAvfNv3VAF9RVR85XHWV/lLEfmo//dj6lpkXa2dWL9lffBxPldfouWoPq5cFh50W35996uclL9XqaNzGHG6FxmFOVy2shcstP60Kgn4bBT5Xi0BI7udKclzrNjR0vc3uKj/l/1JjcnrpyoBJY72lmOZ/N7VeceX6jB+cMPNjrTyCvYEDVtcLdHTxsgFGhjXetj2zhHoK454YSDbccnXqfVZloepA1yptHLqYNabRsEqTlOSSkcgso9cGSH+5nvcOd3wJESaGjV40nG0HaSV7Fz/q42wImU9hqqol6VrZZbrQvUgYac96Ah58dPOHoaal7WkPP/2DH7t1rXtH/YrRnwgsfnvrzOq2ldW8s+p5agxbK3ed/IMkS9n8qWyafdHxVR/GmQnAnJWc7/6y3ms8AbXS0f9XfT4vu+vfW0sz5qftrnYegpHcfZDktQA4Sq8m7JUZ79YC9rd5Xz0b5KQmHn33T84AxmjM7ltNG5zBidy5j89NjuW/VGwQYo3+0mrZLmBBZJZg2teotPHdSctFJzopJKJFk0tkwc7a2LJI6uPQnRNo/PSbBeP/iSnXlfUttTf2rLxHJMEhrkNHzpjtKkKtQdbU5YkaQVSWJVrRrfJme5PxrcpJU90vn8jptw3FdjlVsK74Av1f0yznQ+G/E0J4noeY+nOTG02Ba9j7vc3v6o86MgHHKmGnaX21rX3rKb9FosB5s/r2Q3ubSVZNpLPsmZzv87fZglqAGqtjHI+7srWLvrKGt2Oq/IsPM5aX5mjGpOWFMLc+JXLZhIkS/S8p3HJq6jJc4XoM9NCN6kVsnB7y4ntb3Ol9TymLbWtZtkIudoNe0v93gCdVGfc3TJa4fzbxFdEkjKbE4sLV5ZnVvfx7+YjcMSlAGcqsDtZTWsdZPVml1H2XqwGgCfRzh5eBanjXIS1ozRuQzPsXsMJg7CIecemC8FkjIG1v1C0yFLUKZd5bWNvLervKmEtW53OXWBEADDslOcEpabtE4enoW/q60EjTGmDbEmqLj0xWf6lpy0JM6fOJjzJw4GIBgKs2l/VVPCWrPzKM9+sA+AFL+HKYU5nDwsi3GDMxhbkMG4wRnkZyT13ftZxpg+wUpQpk37K+pb3MfafKCK2sZQ0/bsVD9jC9JbJK2xBRmMHJSG12OJyxjTPqviM3GlquyrqGfboWq2HnReznwNZdUNTfsl+TyMyU9nbEEGYwdntEhiKf5+0jjAGHNCrIrPxJWIMDwnleE5qcweX9BiW0VtgK2HqtnWlLSq2bC3gufW78Nt8Y4IjMhJPabENW5wBoPSrSslY8yxLEGZE5ad5m9qBRitPhCi5HAN2w7WRJW4qnlr+2HqA83PvOSm+RmVl87oQWmMirzynOnQrBQ8VmVozIBkCcp0mxS/l4lDs5g4tGWvDuGwsqe8LqrUVcPuI7Ws213Osx/ua3rQGCDJ66FwUGpz4mqVwNKS7H9hY/or++s2Pc7jEUYOSmPkoDTOnzC4xbZAKMy+8np2Hall55Eadh2pZfeRWnYermVNyVGqGlp295OfkcyoQamMzktnpJu8RrvJqyAj2UpfxvRhlqBMr+L3epzSUV4anyS/xTZVpaIu4CSvw7Utktc7O47wl3V7iCp8kezzMGpQGsNzUsnPSKYgM5n8jCQKMpMpyEgm351mp/otkRnTC1mCMn2GiJCTlkROWhJTCnOO2d4YDLO3vI6dR6KTVw37K+rZcqCKsupGGkPH9vfm8wh5buLKz2hOXtFJbbC7nJ3qt+e/jOkhlqBMv5Hk81CUn05Rfnqb21WVyrogh6obOFTVQFm184rMO9NGNu2roqy6gWD42Ecw/F4hP6Nl8spJSyI71U9Wio+sVL8z706d9X6SfNYbhzGdZQnKDBgiQnaan+w0P+MGZxx330h14qGqhqiE1tgimR2orGfD3grKawM0BI/fE3eq30tWqq9F0ooksqaklhK13Z1mpPhI8npI8nqsGtIMOJagjGlDdHXi+CGZHe5fHwhRWR+gsi5ARV3QnQaorA9QUetOI+vqguyrqGfT/ioq6wNU1bczzlMrPo+Q5PPg93pI8jlJKzL1+8SZtt7Wxv5+r5Dk9TbPR/bxevD7PCR5BX/UuZq3SdN8ZL3f27zOEqiJN0tQxsRBit9Lit/L4MzOD1MfCivV9cHmBFYfSWQBqhuCNATDNAbDBEJR01CYxqC60xCBkNIYdNZX1Qdb7huM7O9MAyFt0ZQ/XnweaUpakQSW4veS7POQ7PeS0sY0st35/Dwk+9xp1Prm7S33TfF5SU1y1tl9wf7JEpQxCeb1NFc99pRQ2ElogXBzIgu4Ca91IgyElEBTcotsU+eYUJiGyPHuvtEJsSEYpiEQoj4Ypj4QoqIuwMFAiAZ3uT5qvqs5U8SpQk1LcpJYWpKXVL+TvJz1vub17e4TtZzkJc3vw+8TvB7B5/G4U2kxtaTY/SxBGTMAeT3ifBnTO/pHVFUCIaUhGKI+EG6RuBqCIRoCYeqjpvWBMHWNIeqDIeoanVdtIER9Y4jaxhB1AWddWXUjtY211AfC1DYGqQuEWvRiciK8rRKWM/U0L3vbWe8RPB7BIyCIM5gv4owwL4KAs82dF3G2RfZ35gXcqbNP9LyTODUyCrQ2jwcd6XtVaR7NXVutbz7GXRfZL2rdjeeO5bRRLXuO6Q6WoIwxCSciJPmcqsEu1JJ2SjisTYmtttEpxUUntTp3ORgKEww71aHO1F0OadvrI8uhdtZH7x9SwgqqYWcUeNWmpKHufFjVXXbmiV7nzqMtj41sb/5cm6dO+opaR3Myk6b/NE2aEmTLY5wEWdvQPBghJ1UAAAY5SURBVLJBd7IEZYwZUDweIS3JR1qSj7xEB2OOyx7OMMYY0ytZgjLGGNMr9ZsBC0X+f3v3GyNXVYdx/PvYqqEtWglUpdtQQIIggbYagzYaYyWpSlpeYPwDTaO+RAGjEQj+SXxhSMR/iUQwoK1hg2ItkZgIrdXUkIggK6VAVQgqLFZbAxTRKIiPL+4hThe2zGw7c+7tPp9kszNn79z7zGTu/O65d/Yc7QX+dJCrORr42yGIU0Oy19Pl/MleR5ezw8HnP872MS+20GFToA4FSb/uZ5bHNkr2erqcP9nr6HJ2GF3+nOKLiIhWSoGKiIhWSoHa37dqBzgIyV5Pl/Mnex1dzg4jyp9rUBER0UrpQUVERCulQEVERCulQAGSVkv6naQHJV1aO88gJC2R9HNJuyTdJ+mi2pkGJWmOpN9I+nHtLIOQtFDSJkm/La//W2pn6pekT5T3y72SbpA05BHwDo6kb0vaI+nenrajJG2V9ED5PfzRS2dgmuxfKu+beyTdJGlhzYzTeaHsPX/7lCRLOnpY25/1BUrSHOAq4N3AqcAHJZ1aN9VA/gN80vYpwJnABR3LD3ARsKt2iBn4OnCL7dcDZ9CR5yBpMXAh8CbbpwFzgA/UTfWiNgCrp7RdCmyzfRKwrdxvow08P/tW4DTbpwO/By4bdag+beD52ZG0BDgLeHiYG5/1BQp4M/Cg7YdsPw18D1hbOVPfbO+2PVFu/53mQ3Jx3VT9kzQGvBe4tnaWQUh6BfB24DoA20/bfqJuqoHMBY6QNBeYB/y5cp4Dsv0L4LEpzWuBjeX2RuCckYbq0wtlt73F9nNTKd8OjI08WB+med0Bvgp8mp4ZOoYhBar5MH+k5/4kHfqA7yVpKbAc+FXdJAP5Gs0b/dBM0jM6JwB7ge+U05PXSppfO1Q/bD8KXElz9Lsb2Gd7S91UM/Jq27uhOVADFlXOM1MfAX5SO0S/JK0BHrW9Y9jbSoH6//QnvTr33XtJC4AfAhfbfrJ2nn5IOhvYY/uu2llmYC6wAvim7eXAP2jvKab9lGs1a4HjgWOB+ZLOr5tqdpJ0Oc1p+vHaWfohaR5wOfC5UWwvBarpMS3puT9Gy093TCXppTTFadz25tp5BrASWCPpjzSnVt8p6fq6kfo2CUzafq63uommYHXBu4A/2N5r+xlgM/DWyplm4q+SXgtQfu+pnGcgktYDZwPnuTv/kHoizYHNjrLfjgETkl4zjI2lQMGdwEmSjpf0MpqLxTdXztQ3NVNiXgfssv2V2nkGYfsy22O2l9K87j+z3Ykjedt/AR6RdHJpWgXcXzHSIB4GzpQ0r7x/VtGRL3hMcTOwvtxeD/yoYpaBSFoNXAKssf3P2nn6ZXun7UW2l5b9dhJYUfaHQ27WF6hyofJjwK00O+mNtu+rm2ogK4F1NL2Pu8vPe2qHmiU+DoxLugdYBnyxcp6+lF7fJmAC2EnzOdDqoXck3QD8EjhZ0qSkjwJXAGdJeoDmG2VX1Mw4nWmyfwM4Etha9tmrq4acxjTZR7f97vQsIyJiNpn1PaiIiGinFKiIiGilFKiIiGilFKiIiGilFKiIiGilFKiIjpP0jq6NBB/RjxSoiIhopRSoiBGRdL6kO8o/Zl5T5sF6StKXJU1I2ibpmLLsMkm398wX9KrS/jpJP5W0ozzmxLL6BT1zU42XESIiOi0FKmIEJJ0CvB9YaXsZ8CxwHjAfmLC9AtgOfL485LvAJWW+oJ097ePAVbbPoBk/b3dpXw5cTDOn2Qk0I4xEdNrc2gEiZolVwBuBO0vn5giawU3/C3y/LHM9sFnSK4GFtreX9o3ADyQdCSy2fROA7X8BlPXdYXuy3L8bWArcNvynFTE8KVARoyFgo+39Zk6V9Nkpyx1o7LEDnbb7d8/tZ8m+HYeBnOKLGI1twLmSFgFIOkrScTT74LllmQ8Bt9neBzwu6W2lfR2wvczzNSnpnLKOl5f5eSIOSznKihgB2/dL+gywRdJLgGeAC2gmOnyDpLuAfTTXqaCZPuLqUoAeAj5c2tcB10j6QlnH+0b4NCJGKqOZR1Qk6SnbC2rniGijnOKLiIhWSg8qIiJaKT2oiIhopRSoiIhopRSoiIhopRSoiIhopRSoiIhopf8Bc7guMey1F08AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Test Loss 0.08005265406304915\n",
      "Test Accuracy 0.9812857142857143\n"
     ]
    }
   ],
   "source": [
    "# plotting the metrics\n",
    "fig = plt.figure()\n",
    "plt.subplot(2,1,1)\n",
    "plt.plot(history.history['acc'])\n",
    "plt.plot(history.history['val_acc'])\n",
    "plt.title('model accuracy')\n",
    "plt.ylabel('accuracy')\n",
    "plt.xlabel('epoch')\n",
    "plt.legend(['train', 'test'], loc='lower right')\n",
    "\n",
    "plt.subplot(2,1,2)\n",
    "plt.plot(history.history['loss'])\n",
    "plt.plot(history.history['val_loss'])\n",
    "plt.title('model loss')\n",
    "plt.ylabel('loss')\n",
    "plt.xlabel('epoch')\n",
    "plt.legend(['train', 'test'], loc='upper right')\n",
    "\n",
    "plt.tight_layout()\n",
    "\n",
    "plt.show()\n",
    "\n",
    "#Evaluating the model\n",
    "\n",
    "loss, accuracy = model.evaluate(X_test, y_test_enc, verbose=2)\n",
    "\n",
    "print()\n",
    "print(\"Test Loss\", loss)\n",
    "print(\"Test Accuracy\", accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "20607  classified correctly\n",
      "393  classified incorrectly\n",
      "\n",
      "Error rate :  1.87 %\n"
     ]
    }
   ],
   "source": [
    "predicted_classes = model.predict_classes(X_test)\n",
    "\n",
    "# see which we predicted correctly and which not ----- Indices of elements that are non-zero.\n",
    "\n",
    "correct_indices = np.nonzero(predicted_classes == y_test)[0]\n",
    "incorrect_indices = np.nonzero(predicted_classes != y_test)[0]\n",
    "\n",
    "print()\n",
    "print(len(correct_indices),\" classified correctly\")\n",
    "print(len(incorrect_indices),\" classified incorrectly\")\n",
    "\n",
    "error_rate = len(incorrect_indices)/(len(correct_indices)+len(incorrect_indices))\n",
    "\n",
    "print()\n",
    "print(\"Error rate : \", round(error_rate*100, 2), \"%\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}