VGG_19_CIFAR10.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# VGG-19 Convolutional Neural Networks: CIFAR10 dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import math\n",
    "import tensorflow_model_optimization as tfmot\n",
    "from tensorflow_model_optimization.sparsity import keras as sparsity\n",
    "# from tensorflow.keras import datasets, layers, models\n",
    "import matplotlib.pyplot as plt\n",
    "from tensorflow.keras.layers import AveragePooling2D, Conv2D, MaxPooling2D, ReLU, BatchNormalization\n",
    "from tensorflow.keras import models, layers, datasets\n",
    "from tensorflow.keras.layers import Dense, Flatten, Reshape, Input, InputLayer, Dropout\n",
    "from tensorflow.keras.models import Sequential, Model\n",
    "from tensorflow.keras.initializers import RandomNormal\n",
    "# import math\n",
    "from sklearn.metrics import accuracy_score, precision_score, recall_score\n",
    "\n",
    "from tensorflow.keras.preprocessing.image import ImageDataGenerator\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'2.2.0'"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "env: CUDA_DEVICE_ORDER=PCI_BUS_ID\n",
      "env: CUDA_VISIBLE_DEVICES=2\n"
     ]
    }
   ],
   "source": [
    "%env CUDA_DEVICE_ORDER=PCI_BUS_ID\n",
    "%env CUDA_VISIBLE_DEVICES=2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 64\n",
    "num_classes = 100\n",
    "num_epochs = 145"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Data preprocessing and cleaning:\n",
    "# input image dimensions\n",
    "img_rows, img_cols = 32, 32\n",
    "\n",
    "# Load CIFAR-10 dataset-\n",
    "# (X_train, y_train), (X_test, y_test) = tf.keras.datasets.cifar10.load_data()\n",
    "\n",
    "# Load CIFAR-100 dataset-\n",
    "(X_train, y_train), (X_test, y_test) = tf.keras.datasets.cifar100.load_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "'input_shape' which will be used = (32, 32, 3)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "if tf.keras.backend.image_data_format() == 'channels_first':\n",
    "    X_train = X_train.reshape(X_train.shape[0], 3, img_rows, img_cols)\n",
    "    X_test = X_test.reshape(X_test.shape[0], 3, img_rows, img_cols)\n",
    "    input_shape = (3, img_rows, img_cols)\n",
    "else:\n",
    "    X_train = X_train.reshape(X_train.shape[0], img_rows, img_cols, 3)\n",
    "    X_test = X_test.reshape(X_test.shape[0], img_rows, img_cols, 3)\n",
    "    input_shape = (img_rows, img_cols, 3)\n",
    "\n",
    "print(\"\\n'input_shape' which will be used = {0}\\n\".format(input_shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Convert datasets to floating point types-\n",
    "X_train = X_train.astype('float32')\n",
    "X_test = X_test.astype('float32')\n",
    "\n",
    "# Normalize the training and testing datasets-\n",
    "X_train /= 255.0\n",
    "X_test /= 255.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# convert class vectors/target to binary class matrices or one-hot encoded values-\n",
    "y_train = tf.keras.utils.to_categorical(y_train, num_classes)\n",
    "y_test = tf.keras.utils.to_categorical(y_test, num_classes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Dimensions of training and testing sets are:\n",
      "X_train.shape = (50000, 32, 32, 3), y_train.shape = (50000, 100)\n",
      "X_test.shape = (10000, 32, 32, 3), y_test.shape = (10000, 100)\n"
     ]
    }
   ],
   "source": [
    "print(\"\\nDimensions of training and testing sets are:\")\n",
    "print(\"X_train.shape = {0}, y_train.shape = {1}\".format(X_train.shape, y_train.shape))\n",
    "print(\"X_test.shape = {0}, y_test.shape = {1}\".format(X_test.shape, y_test.shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Prepare CIFAR10 dataset for _GradientTape_ training:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create training and testing datasets-\n",
    "train_dataset = tf.data.Dataset.from_tensor_slices((X_train, y_train))\n",
    "test_dataset = tf.data.Dataset.from_tensor_slices((X_test, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dataset = train_dataset.shuffle(buffer_size = 20000, reshuffle_each_iteration = True).batch(batch_size = batch_size, drop_remainder = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_dataset = test_dataset.batch(batch_size=batch_size, drop_remainder=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Choose an optimizer and loss function for training-\n",
    "loss_fn = tf.keras.losses.CategoricalCrossentropy()\n",
    "optimizer = tf.keras.optimizers.SGD(learning_rate=0.01, momentum=0.9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Select metrics to measure the error & accuracy of model.\n",
    "# These metrics accumulate the values over epochs and then\n",
    "# print the overall result-\n",
    "train_loss = tf.keras.metrics.Mean(name = 'train_loss')\n",
    "train_accuracy = tf.keras.metrics.CategoricalAccuracy(name = 'train_accuracy')\n",
    "\n",
    "test_loss = tf.keras.metrics.Mean(name = 'test_loss')\n",
    "test_accuracy = tf.keras.metrics.CategoricalAccuracy(name = 'test_accuracy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "def vgg19_dnn():\n",
    "    model = Sequential()\n",
    "    \n",
    "    model.add(\n",
    "        Conv2D(\n",
    "            filters = 64, kernel_size = (3, 3),\n",
    "            activation='relu', kernel_initializer = tf.initializers.he_normal(),\n",
    "            strides = (1, 1), padding = 'same',\n",
    "            input_shape=(32, 32, 3)\n",
    "        )\n",
    "    )\n",
    "    \n",
    "    model.add(BatchNormalization())\n",
    "    \n",
    "    model.add(Dropout(0.3))\n",
    "    \n",
    "    model.add(\n",
    "        Conv2D(\n",
    "            filters = 64, kernel_size = (3, 3),\n",
    "            activation='relu', kernel_initializer = tf.initializers.he_normal(),\n",
    "            strides = (1, 1), padding = 'same'\n",
    "        )\n",
    "    )\n",
    "    \n",
    "    model.add(BatchNormalization())\n",
    "    \n",
    "    model.add(\n",
    "    MaxPooling2D(\n",
    "            pool_size = (2, 2),\n",
    "            strides = (2, 2)\n",
    "        )\n",
    "    )\n",
    "    \n",
    "    model.add(\n",
    "        Conv2D(\n",
    "            filters = 128, kernel_size = (3, 3),\n",
    "            activation='relu', kernel_initializer = tf.initializers.he_normal(),\n",
    "            strides = (1, 1), padding = 'same'\n",
    "        )\n",
    "    )\n",
    "    \n",
    "    model.add(BatchNormalization())\n",
    "    \n",
    "    model.add(Dropout(0.4))\n",
    "    \n",
    "    model.add(\n",
    "        Conv2D(\n",
    "            filters = 128, kernel_size = (3, 3),\n",
    "            activation='relu', kernel_initializer = tf.initializers.he_normal(),\n",
    "            strides = (1, 1), padding = 'same'\n",
    "        )\n",
    "    )\n",
    "    \n",
    "    model.add(BatchNormalization())\n",
    "    \n",
    "    model.add(\n",
    "        MaxPooling2D(\n",
    "            pool_size = (2, 2),\n",
    "            strides = (2, 2)\n",
    "        )\n",
    "    )\n",
    "    \n",
    "    model.add(\n",
    "        Conv2D(\n",
    "            filters = 256, kernel_size = (3, 3),\n",
    "            activation='relu', kernel_initializer = tf.initializers.he_normal(),\n",
    "            strides = (1, 1), padding = 'same'\n",
    "        )\n",
    "    )\n",
    "    \n",
    "    model.add(BatchNormalization())\n",
    "    \n",
    "    model.add(Dropout(0.4))\n",
    "    \n",
    "    model.add(\n",
    "        Conv2D(\n",
    "            filters = 256, kernel_size = (3, 3),\n",
    "            activation='relu', kernel_initializer = tf.initializers.he_normal(),\n",
    "            strides = (1, 1), padding = 'same'\n",
    "        )\n",
    "    )\n",
    "    \n",
    "    model.add(BatchNormalization())\n",
    "    \n",
    "    model.add(Dropout(0.4))\n",
    "    \n",
    "    model.add(\n",
    "        Conv2D(\n",
    "            filters = 256, kernel_size = (3, 3),\n",
    "            activation='relu', kernel_initializer = tf.initializers.he_normal(),\n",
    "            strides = (1, 1), padding = 'same'\n",
    "        )\n",
    "    )\n",
    "    \n",
    "    model.add(BatchNormalization())\n",
    "    \n",
    "    model.add(Dropout(0.4))\n",
    "    \n",
    "    model.add(\n",
    "        Conv2D(\n",
    "            filters = 256, kernel_size = (3, 3),\n",
    "            activation='relu', kernel_initializer = tf.initializers.he_normal(),\n",
    "            strides = (1, 1), padding = 'same'\n",
    "        )\n",
    "    )\n",
    "    \n",
    "    model.add(BatchNormalization())\n",
    "    \n",
    "    model.add(\n",
    "        MaxPooling2D(\n",
    "            pool_size = (2, 2),\n",
    "            strides = (2, 2)\n",
    "        )\n",
    "    )\n",
    "    \n",
    "    model.add(\n",
    "        Conv2D(\n",
    "            filters = 512, kernel_size = (3, 3),\n",
    "            activation='relu', kernel_initializer = tf.initializers.he_normal(),\n",
    "            strides = (1, 1), padding = 'same'\n",
    "        )\n",
    "    )\n",
    "    \n",
    "    model.add(BatchNormalization())\n",
    "    \n",
    "    model.add(Dropout(0.4))\n",
    "    \n",
    "    model.add(\n",
    "        Conv2D(\n",
    "            filters = 512, kernel_size = (3, 3),\n",
    "            activation='relu', kernel_initializer = tf.initializers.he_normal(),\n",
    "            strides = (1, 1), padding = 'same'\n",
    "        )\n",
    "    )\n",
    "    \n",
    "    model.add(BatchNormalization())\n",
    "    \n",
    "    model.add(Dropout(0.4))\n",
    "    \n",
    "    model.add(\n",
    "        Conv2D(\n",
    "            filters = 512, kernel_size = (3, 3),\n",
    "            activation='relu', kernel_initializer = tf.initializers.he_normal(),\n",
    "            strides = (1, 1), padding = 'same'\n",
    "        )\n",
    "    )\n",
    "    \n",
    "    model.add(BatchNormalization())\n",
    "    \n",
    "    model.add(Dropout(0.4))\n",
    "    \n",
    "    model.add(\n",
    "        Conv2D(\n",
    "            filters = 512, kernel_size = (3, 3),\n",
    "            activation='relu', kernel_initializer = tf.initializers.he_normal(),\n",
    "            strides = (1, 1), padding = 'same'\n",
    "        )\n",
    "    )\n",
    "    \n",
    "    model.add(BatchNormalization())\n",
    "    \n",
    "    model.add(\n",
    "        MaxPooling2D(\n",
    "            pool_size = (2, 2),\n",
    "            strides = (2, 2)\n",
    "        )\n",
    "    )\n",
    "    \n",
    "    model.add(\n",
    "        Conv2D(\n",
    "            filters = 512, kernel_size = (3, 3),\n",
    "            activation='relu', kernel_initializer = tf.initializers.he_normal(),\n",
    "            strides = (1, 1), padding = 'same'\n",
    "        )\n",
    "    )\n",
    "    \n",
    "    model.add(BatchNormalization())\n",
    "    \n",
    "    model.add(Dropout(0.4))\n",
    "    \n",
    "    model.add(\n",
    "        Conv2D(\n",
    "            filters = 512, kernel_size = (3, 3),\n",
    "            activation='relu', kernel_initializer = tf.initializers.he_normal(),\n",
    "            strides = (1, 1), padding = 'same'\n",
    "        )\n",
    "    )\n",
    "    \n",
    "    model.add(BatchNormalization())\n",
    "    \n",
    "    model.add(Dropout(0.4))\n",
    "    \n",
    "    model.add(\n",
    "        Conv2D(\n",
    "            filters = 512, kernel_size = (3, 3),\n",
    "            activation='relu', kernel_initializer = tf.initializers.he_normal(),\n",
    "            strides = (1, 1), padding = 'same'\n",
    "        )\n",
    "    )\n",
    "    \n",
    "    model.add(BatchNormalization())\n",
    "    \n",
    "    model.add(Dropout(0.4))\n",
    "    \n",
    "    model.add(\n",
    "        Conv2D(\n",
    "            filters = 512, kernel_size = (3, 3),\n",
    "            activation='relu', kernel_initializer = tf.initializers.he_normal(),\n",
    "            strides = (1, 1), padding = 'same'\n",
    "        )\n",
    "    )\n",
    "    \n",
    "    model.add(BatchNormalization())\n",
    "    \n",
    "    model.add(Dropout(0.4))\n",
    "    \n",
    "    '''\n",
    "    model.add(\n",
    "    AveragePooling2D(\n",
    "        pool_size=(2, 2), strides=(2, 2)\n",
    "    )\n",
    "    )\n",
    "    '''\n",
    "    \n",
    "    model.add(\n",
    "        MaxPooling2D(\n",
    "            pool_size = (2, 2),\n",
    "            strides = (2, 2)\n",
    "        )\n",
    "    )\n",
    "    \n",
    "    model.add(Flatten())\n",
    "    \n",
    "    '''\n",
    "    model.add(\n",
    "        MaxPooling2D(\n",
    "            pool_size = (2, 2),\n",
    "            strides = (2, 2)\n",
    "        )\n",
    "    )\n",
    "    '''\n",
    "    \n",
    "    \n",
    "    model.add(\n",
    "        Dense(\n",
    "            units = 256, activation='relu',\n",
    "            kernel_initializer = tf.initializers.he_normal()\n",
    "        )\n",
    "    )\n",
    "    \n",
    "    model.add(BatchNormalization())\n",
    "    \n",
    "    model.add(Dropout(0.5))\n",
    "    \n",
    "    '''\n",
    "    model.add(\n",
    "    Dense(\n",
    "            units = 256, activation='relu',\n",
    "            kernel_initializer = tf.initializers.he_normal()\n",
    "        )\n",
    "    )\n",
    "    \n",
    "    model.add(BatchNormalization())\n",
    "    '''\n",
    "    \n",
    "    \n",
    "    model.add(\n",
    "        Dense(\n",
    "            units = num_classes, activation='softmax'\n",
    "        )\n",
    "    )\n",
    "    \n",
    "    \n",
    "    # Compile pruned CNN-\n",
    "    model.compile(\n",
    "        loss=tf.keras.losses.categorical_crossentropy,\n",
    "        # optimizer='adam',\n",
    "        optimizer = tf.keras.optimizers.SGD(learning_rate=0.01, momentum=0.9),\n",
    "        metrics=['accuracy']\n",
    "    )\n",
    "    \n",
    "    return model\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "def vgg19_nn():\n",
    "    \"\"\"\n",
    "    Function to define the architecture of a convolutional neural network\n",
    "    model following VGG-19 architecture for CIFAR-10 dataset.\n",
    "    \n",
    "    Vgg-19 architecture-\n",
    "    64, 64, pool                 -- convolutional layers\n",
    "    128, 128, pool               -- convolutional layers\n",
    "    256, 256, 256, 256, max-pool -- convolutional layers\n",
    "    512, 512, 512, 512, max-pool -- convolutional layers\n",
    "    512, 512, 512, 512, avg-pool -- convolutional layers\n",
    "    10                           -- fully connected layers\n",
    "    \n",
    "    Output: Returns designed and compiled convolutional neural network model\n",
    "    \"\"\"\n",
    "\n",
    "    \n",
    "    # l = tf.keras.layers\n",
    "    \n",
    "    model = Sequential()\n",
    "    \n",
    "    model.add(\n",
    "    Conv2D(\n",
    "            filters = 64, kernel_size = (3, 3),\n",
    "            activation='relu', kernel_initializer = tf.initializers.he_normal(),\n",
    "            strides = (1, 1), padding = 'same',\n",
    "            input_shape=(32, 32, 3)\n",
    "        )\n",
    "    )\n",
    "    \n",
    "    model.add(BatchNormalization())\n",
    "    \n",
    "    # model.add(Dropout(0.3))\n",
    "    \n",
    "    model.add(\n",
    "    Conv2D(\n",
    "            filters = 64, kernel_size = (3, 3),\n",
    "            activation='relu', kernel_initializer = tf.initializers.he_normal(),\n",
    "            strides = (1, 1), padding = 'same'\n",
    "        )\n",
    "    )\n",
    "    \n",
    "    model.add(BatchNormalization())\n",
    "    \n",
    "    model.add(\n",
    "    MaxPooling2D(\n",
    "            pool_size = (2, 2),\n",
    "            strides = (2, 2)\n",
    "        )\n",
    "    )\n",
    "    \n",
    "    model.add(\n",
    "    Conv2D(\n",
    "            filters = 128, kernel_size = (3, 3),\n",
    "            activation='relu', kernel_initializer = tf.initializers.he_normal(),\n",
    "            strides = (1, 1), padding = 'same'\n",
    "        )\n",
    "    )\n",
    "    \n",
    "    model.add(BatchNormalization())\n",
    "    \n",
    "    # model.add(Dropout(0.4))\n",
    "    \n",
    "    model.add(\n",
    "    Conv2D(\n",
    "            filters = 128, kernel_size = (3, 3),\n",
    "            activation='relu', kernel_initializer = tf.initializers.he_normal(),\n",
    "            strides = (1, 1), padding = 'same'\n",
    "        )\n",
    "    )\n",
    "    \n",
    "    model.add(BatchNormalization())\n",
    "    \n",
    "    model.add(\n",
    "    MaxPooling2D(\n",
    "            pool_size = (2, 2),\n",
    "            strides = (2, 2)\n",
    "        )\n",
    "    )\n",
    "    \n",
    "    model.add(\n",
    "    Conv2D(\n",
    "            filters = 256, kernel_size = (3, 3),\n",
    "            activation='relu', kernel_initializer = tf.initializers.he_normal(),\n",
    "            strides = (1, 1), padding = 'same'\n",
    "        )\n",
    "    )\n",
    "    \n",
    "    model.add(BatchNormalization())\n",
    "    \n",
    "    # model.add(Dropout(0.4))\n",
    "    \n",
    "    model.add(\n",
    "    Conv2D(\n",
    "            filters = 256, kernel_size = (3, 3),\n",
    "            activation='relu', kernel_initializer = tf.initializers.he_normal(),\n",
    "            strides = (1, 1), padding = 'same'\n",
    "        )\n",
    "    )\n",
    "    \n",
    "    model.add(BatchNormalization())\n",
    "    \n",
    "    # model.add(Dropout(0.4))\n",
    "    \n",
    "    model.add(\n",
    "    Conv2D(\n",
    "            filters = 256, kernel_size = (3, 3),\n",
    "            activation='relu', kernel_initializer = tf.initializers.he_normal(),\n",
    "            strides = (1, 1), padding = 'same'\n",
    "        )\n",
    "    )\n",
    "    \n",
    "    model.add(BatchNormalization())\n",
    "    \n",
    "    # model.add(Dropout(0.4))\n",
    "    \n",
    "    model.add(\n",
    "    Conv2D(\n",
    "            filters = 256, kernel_size = (3, 3),\n",
    "            activation='relu', kernel_initializer = tf.initializers.he_normal(),\n",
    "            strides = (1, 1), padding = 'same'\n",
    "        )\n",
    "    )\n",
    "    \n",
    "    model.add(BatchNormalization())\n",
    "    \n",
    "    model.add(\n",
    "    MaxPooling2D(\n",
    "            pool_size = (2, 2),\n",
    "            strides = (2, 2)\n",
    "        )\n",
    "    )\n",
    "    \n",
    "    model.add(\n",
    "    Conv2D(\n",
    "            filters = 512, kernel_size = (3, 3),\n",
    "            activation='relu', kernel_initializer = tf.initializers.he_normal(),\n",
    "            strides = (1, 1), padding = 'same'\n",
    "        )\n",
    "    )\n",
    "    \n",
    "    model.add(BatchNormalization())\n",
    "    \n",
    "    # model.add(Dropout(0.4))\n",
    "    \n",
    "    model.add(\n",
    "    Conv2D(\n",
    "            filters = 512, kernel_size = (3, 3),\n",
    "            activation='relu', kernel_initializer = tf.initializers.he_normal(),\n",
    "            strides = (1, 1), padding = 'same'\n",
    "        )\n",
    "    )\n",
    "    \n",
    "    model.add(BatchNormalization())\n",
    "    \n",
    "    # model.add(Dropout(0.4))\n",
    "    \n",
    "    model.add(\n",
    "    Conv2D(\n",
    "            filters = 512, kernel_size = (3, 3),\n",
    "            activation='relu', kernel_initializer = tf.initializers.he_normal(),\n",
    "            strides = (1, 1), padding = 'same'\n",
    "        )\n",
    "    )\n",
    "    \n",
    "    model.add(BatchNormalization())\n",
    "    \n",
    "    # model.add(Dropout(0.4))\n",
    "    \n",
    "    model.add(\n",
    "    Conv2D(\n",
    "            filters = 512, kernel_size = (3, 3),\n",
    "            activation='relu', kernel_initializer = tf.initializers.he_normal(),\n",
    "            strides = (1, 1), padding = 'same'\n",
    "        )\n",
    "    )\n",
    "    \n",
    "    model.add(BatchNormalization())\n",
    "    \n",
    "    model.add(\n",
    "    MaxPooling2D(\n",
    "            pool_size = (2, 2),\n",
    "            strides = (2, 2)\n",
    "        )\n",
    "    )\n",
    "    \n",
    "    model.add(\n",
    "    Conv2D(\n",
    "            filters = 512, kernel_size = (3, 3),\n",
    "            activation='relu', kernel_initializer = tf.initializers.he_normal(),\n",
    "            strides = (1, 1), padding = 'same'\n",
    "        )\n",
    "    )\n",
    "    \n",
    "    model.add(BatchNormalization())\n",
    "    \n",
    "    # model.add(Dropout(0.4))\n",
    "    \n",
    "    model.add(\n",
    "    Conv2D(\n",
    "            filters = 512, kernel_size = (3, 3),\n",
    "            activation='relu', kernel_initializer = tf.initializers.he_normal(),\n",
    "            strides = (1, 1), padding = 'same'\n",
    "        )\n",
    "    )\n",
    "    \n",
    "    model.add(BatchNormalization())\n",
    "    \n",
    "    # model.add(Dropout(0.4))\n",
    "    \n",
    "    model.add(\n",
    "    Conv2D(\n",
    "            filters = 512, kernel_size = (3, 3),\n",
    "            activation='relu', kernel_initializer = tf.initializers.he_normal(),\n",
    "            strides = (1, 1), padding = 'same'\n",
    "        )\n",
    "    )\n",
    "    \n",
    "    model.add(BatchNormalization())\n",
    "    \n",
    "    # model.add(Dropout(0.4))\n",
    "    \n",
    "    model.add(\n",
    "    Conv2D(\n",
    "            filters = 512, kernel_size = (3, 3),\n",
    "            activation='relu', kernel_initializer = tf.initializers.he_normal(),\n",
    "            strides = (1, 1), padding = 'same'\n",
    "        )\n",
    "    )\n",
    "    \n",
    "    model.add(BatchNormalization())\n",
    "    \n",
    "    \n",
    "    model.add(\n",
    "    # AveragePooling2D(\n",
    "        MaxPooling2D(\n",
    "            pool_size=(2, 2), strides=(2, 2)\n",
    "        )\n",
    "    )\n",
    "    \n",
    "    model.add(Flatten())\n",
    "    \n",
    "    model.add(\n",
    "    Dense(\n",
    "            units = 256, activation='relu',\n",
    "            kernel_initializer = tf.initializers.he_normal()\n",
    "        )\n",
    "    )\n",
    "    \n",
    "    model.add(BatchNormalization())\n",
    "    \n",
    "    # model.add(Dropout(0.4))\n",
    "    '''\n",
    "    model.add(\n",
    "    Dense(\n",
    "            units = 256, activation='relu',\n",
    "            kernel_initializer = tf.initializers.he_normal()\n",
    "        )\n",
    "    )\n",
    "    \n",
    "    model.add(BatchNormalization())\n",
    "    \n",
    "    model.add(Dropout(0.4))\n",
    "    '''\n",
    "    \n",
    "    model.add(\n",
    "    Dense(\n",
    "            units = num_classes, activation='softmax'\n",
    "        )\n",
    "    )\n",
    "    \n",
    "    \n",
    "    # Compile pruned CNN-\n",
    "    model.compile(\n",
    "        loss=tf.keras.losses.categorical_crossentropy,\n",
    "        # optimizer='adam',\n",
    "        optimizer = tf.keras.optimizers.SGD(learning_rate=0.01, momentum=0.9),\n",
    "        metrics=['accuracy']\n",
    "    )\n",
    "    \n",
    "    return model\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Add a pruning step callback to peg the pruning step to the optimizer's\n",
    "# step. Also add a callback to add pruning summaries to tensorboard\n",
    "'''\n",
    "callback = [\n",
    "             # sparsity.UpdatePruningStep(),\n",
    "             # sparsity.PruningSummaries(log_dir = logdir, profile_batch=0),\n",
    "             tf.keras.callbacks.EarlyStopping(\n",
    "                 monitor='val_loss', patience = 3,\n",
    "                 min_delta=0.001\n",
    "             )\n",
    "]\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Initialize a CNN model-\n",
    "orig_model = vgg19_nn()\n",
    "# orig_model = vgg19_dnn()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save random weights of defined CNN-\n",
    "orig_model.save_weights(\"VGG19_Random_Weights.h5\", overwrite=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get CNN model summary-\n",
    "orig_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Image Augmentation using _ImageDataGenerator_:\n",
    "\n",
    "Refer-\n",
    "https://www.tensorflow.org/api_docs/python/tf/keras/preprocessing/image/ImageDataGenerator#fit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\nfor x, y in datagen.flow(X_train, y_train, batch_size=batch_size, shuffle=True):\\n\\tprint(x.shape, y.shape)\\n'"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Example of using 'tf.keras.preprocessing.image import ImageDataGenerator class's - flow(x, y)':\n",
    "datagen = ImageDataGenerator(\n",
    "    # featurewise_center=True,\n",
    "    # featurewise_std_normalization=True,\n",
    "    rotation_range = 20,\n",
    "    width_shift_range = 0.1, # 0.2\n",
    "    height_shift_range = 0.1, # 0.2\n",
    "    # horizontal_flip = True\n",
    ")\n",
    "\n",
    "\n",
    "# flow():\n",
    "# Takes data & label arrays, generates batches of augmented data.\n",
    "\n",
    "# datagen.flow(X_train, y_train, batch_size=batch_size, shuffle=True)\n",
    "'''\n",
    "for x, y in datagen.flow(X_train, y_train, batch_size=batch_size, shuffle=True):\n",
    "\tprint(x.shape, y.shape)\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "# compute quantities required for featurewise normalization\n",
    "# (std, mean, and principal components if ZCA whitening is applied)\n",
    "datagen.fit(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create mask using winning ticket-\n",
    "\n",
    "# Instantiate a new neural network model for which, the mask is to be created,\n",
    "mask_model = vgg19_nn()\n",
    "    \n",
    "# Load weights of PRUNED model-\n",
    "mask_model.set_weights(orig_model.get_weights())\n",
    "    \n",
    "# For each layer, for each weight which is 0, leave it, as is.\n",
    "# And for weights which survive the pruning,reinitialize it to ONE (1)-\n",
    "for wts in mask_model.trainable_weights:\n",
    "    wts.assign(tf.where(tf.equal(wts, 0.), 0., 1.))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Total # of non-zero masks = 20181311\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Count number of non-zero masks-\n",
    "mask_params = 0\n",
    "\n",
    "for layer in mask_model.trainable_weights:\n",
    "    nonzeroparams = tf.math.count_nonzero(layer, axis = None).numpy()\n",
    "    # print(\"layer: {0} has {1} non-zero masks\".format(layer.shape, nonzeroparams))\n",
    "    mask_params += nonzeroparams\n",
    "\n",
    "print(\"\\nTotal # of non-zero masks = {0}\\n\".format(mask_params))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Python 3 dictionary to hold model training metrics for each of n rounds:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Python3 dict to hold model metrics from each training epoch-\n",
    "history = {}\n",
    "\n",
    "# Initialize all epochs to zeros-\n",
    "history['accuracy'] = np.zeros(shape = num_epochs)\n",
    "history['val_accuracy'] = np.zeros(shape = num_epochs)\n",
    "history['loss'] = np.zeros(shape = num_epochs)\n",
    "history['val_loss'] = np.zeros(shape = num_epochs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['accuracy', 'val_accuracy', 'loss', 'val_loss'])"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Sanity check-\n",
    "history.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train model using _tf.GradientTape_:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "# User input parameters for Early Stopping in manual implementation-\n",
    "minimum_delta = 0.001\n",
    "patience = 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_val_loss = 100\n",
    "loc_patience = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "# best_trained_weights = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Initialize a new LeNet-300-100 model-\n",
    "winning_ticket_model = vgg19_nn()\n",
    "\n",
    "# Load weights of winning ticket-\n",
    "winning_ticket_model.set_weights(orig_model.get_weights())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define 'train_one_step()' and 'test_step()' functions here-\n",
    "@tf.function\n",
    "def train_one_step(model, mask_model, optimizer, x, y):\n",
    "    '''\n",
    "    Function to compute one step of gradient descent optimization\n",
    "    '''\n",
    "    with tf.GradientTape() as tape:\n",
    "        # Make predictions using defined model-\n",
    "        y_pred = model(x)\n",
    "\n",
    "        # Compute loss-\n",
    "        loss = loss_fn(y, y_pred)\n",
    "        \n",
    "    # Compute gradients wrt defined loss and weights and biases-\n",
    "    grads = tape.gradient(loss, model.trainable_variables)\n",
    "    \n",
    "    # type(grads)\n",
    "    # list\n",
    "    \n",
    "    # List to hold element-wise multiplication between-\n",
    "    # computed gradient and masks-\n",
    "    grad_mask_mul = []\n",
    "    \n",
    "    # Perform element-wise multiplication between computed gradients and masks-\n",
    "    for grad_layer, mask in zip(grads, mask_model.trainable_weights):\n",
    "        grad_mask_mul.append(tf.math.multiply(grad_layer, mask))\n",
    "    \n",
    "    # Apply computed gradients to model's weights and biases-\n",
    "    optimizer.apply_gradients(zip(grad_mask_mul, model.trainable_variables))\n",
    "\n",
    "    # Compute accuracy-\n",
    "    train_loss(loss)\n",
    "    train_accuracy(y, y_pred)\n",
    "\n",
    "    return None\n",
    "    \n",
    "    \n",
    "@tf.function\n",
    "def test_step(model, optimizer, data, labels):\n",
    "    \"\"\"\n",
    "    Function to test model performance\n",
    "    on testing dataset\n",
    "    \"\"\"\n",
    "    \n",
    "    predictions = model(data)\n",
    "    t_loss = loss_fn(labels, predictions)\n",
    "\n",
    "    test_loss(t_loss)\n",
    "    test_accuracy(labels, predictions)\n",
    "\n",
    "    return None\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1, Loss: 4.4419, Accuracy: 2.1107, Test Loss: 4.2141, Test Accuracy: 3.520000\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 2, Loss: 4.0936, Accuracy: 5.7058, Test Loss: 3.9415, Test Accuracy: 8.570000\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 3, Loss: 3.8299, Accuracy: 10.0632, Test Loss: 3.6622, Test Accuracy: 12.250000\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 4, Loss: 3.5971, Accuracy: 14.0965, Test Loss: 3.4395, Test Accuracy: 16.539999\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 5, Loss: 3.4106, Accuracy: 17.3015, Test Loss: 3.2038, Test Accuracy: 21.170000\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 6, Loss: 3.2020, Accuracy: 20.9287, Test Loss: 3.0018, Test Accuracy: 24.529999\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 7, Loss: 3.0244, Accuracy: 24.4238, Test Loss: 3.0648, Test Accuracy: 24.799999\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 8, Loss: 2.8608, Accuracy: 27.5608, Test Loss: 2.7702, Test Accuracy: 29.020000\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 9, Loss: 2.7093, Accuracy: 30.7558, Test Loss: 2.7494, Test Accuracy: 31.590000\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 10, Loss: 2.5731, Accuracy: 33.5227, Test Loss: 2.5020, Test Accuracy: 35.040001\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 11, Loss: 2.4561, Accuracy: 35.8575, Test Loss: 2.4972, Test Accuracy: 36.059998\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 12, Loss: 2.3421, Accuracy: 38.4523, Test Loss: 2.3734, Test Accuracy: 38.750000\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 13, Loss: 2.2306, Accuracy: 40.9171, Test Loss: 2.6259, Test Accuracy: 36.320000\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 14, Loss: 2.1452, Accuracy: 42.8437, Test Loss: 2.4824, Test Accuracy: 39.879997\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 15, Loss: 2.0530, Accuracy: 45.1044, Test Loss: 2.2925, Test Accuracy: 42.539997\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 16, Loss: 1.9805, Accuracy: 46.4089, Test Loss: 2.2156, Test Accuracy: 43.970001\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 17, Loss: 1.9023, Accuracy: 48.6776, Test Loss: 2.1550, Test Accuracy: 44.980003\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 18, Loss: 1.8310, Accuracy: 50.1901, Test Loss: 2.1143, Test Accuracy: 46.470001\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 19, Loss: 1.7646, Accuracy: 51.6325, Test Loss: 2.2147, Test Accuracy: 46.009998\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 20, Loss: 1.7032, Accuracy: 52.9750, Test Loss: 2.1449, Test Accuracy: 46.389999\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 21, Loss: 1.6392, Accuracy: 54.5875, Test Loss: 2.1245, Test Accuracy: 47.780003\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 22, Loss: 1.5838, Accuracy: 55.7538, Test Loss: 2.1611, Test Accuracy: 46.759998\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 23, Loss: 1.5341, Accuracy: 57.2823, Test Loss: 2.0777, Test Accuracy: 48.619999\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 24, Loss: 1.4735, Accuracy: 58.7648, Test Loss: 2.1197, Test Accuracy: 49.509998\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 25, Loss: 1.4281, Accuracy: 59.6231, Test Loss: 2.1410, Test Accuracy: 47.930000\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 26, Loss: 1.3798, Accuracy: 61.0175, Test Loss: 2.2456, Test Accuracy: 48.520000\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 27, Loss: 1.3364, Accuracy: 62.0739, Test Loss: 2.0351, Test Accuracy: 50.620003\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 28, Loss: 1.2974, Accuracy: 62.9321, Test Loss: 2.0199, Test Accuracy: 49.389999\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 29, Loss: 1.2615, Accuracy: 64.2306, Test Loss: 2.1909, Test Accuracy: 50.029999\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 30, Loss: 1.2036, Accuracy: 65.6910, Test Loss: 2.1160, Test Accuracy: 52.049999\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 31, Loss: 1.1762, Accuracy: 66.2552, Test Loss: 2.2129, Test Accuracy: 51.560001\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 32, Loss: 1.1339, Accuracy: 67.2475, Test Loss: 2.0854, Test Accuracy: 51.730000\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 33, Loss: 1.1038, Accuracy: 68.0258, Test Loss: 2.2021, Test Accuracy: 51.160004\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 34, Loss: 1.0780, Accuracy: 68.8340, Test Loss: 2.1377, Test Accuracy: 52.189999\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 35, Loss: 1.0369, Accuracy: 69.9304, Test Loss: 2.3432, Test Accuracy: 51.540001\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 36, Loss: 0.9948, Accuracy: 71.1188, Test Loss: 2.2993, Test Accuracy: 52.359997\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 37, Loss: 0.9803, Accuracy: 71.7570, Test Loss: 2.1914, Test Accuracy: 52.020000\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 38, Loss: 0.9509, Accuracy: 72.2411, Test Loss: 2.2915, Test Accuracy: 51.610001\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 39, Loss: 0.9162, Accuracy: 73.2995, Test Loss: 2.2736, Test Accuracy: 52.549999\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 40, Loss: 0.8915, Accuracy: 74.0497, Test Loss: 2.2362, Test Accuracy: 52.759998\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 41, Loss: 0.8558, Accuracy: 75.1040, Test Loss: 2.3210, Test Accuracy: 51.650002\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 42, Loss: 0.8419, Accuracy: 75.4341, Test Loss: 2.3399, Test Accuracy: 52.499996\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 43, Loss: 0.8215, Accuracy: 75.9003, Test Loss: 2.3256, Test Accuracy: 53.410000\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 44, Loss: 0.7804, Accuracy: 77.1767, Test Loss: 2.3953, Test Accuracy: 51.700001\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 45, Loss: 0.7610, Accuracy: 77.6969, Test Loss: 2.3131, Test Accuracy: 54.149998\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 46, Loss: 0.7355, Accuracy: 78.1490, Test Loss: 2.2937, Test Accuracy: 53.460003\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 47, Loss: 0.7222, Accuracy: 78.8512, Test Loss: 2.7024, Test Accuracy: 51.470001\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 48, Loss: 0.6987, Accuracy: 79.4274, Test Loss: 2.2125, Test Accuracy: 54.210003\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 49, Loss: 0.6823, Accuracy: 79.7595, Test Loss: 2.5661, Test Accuracy: 54.000004\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 50, Loss: 0.6573, Accuracy: 80.4758, Test Loss: 2.5624, Test Accuracy: 53.060001\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 51, Loss: 0.6410, Accuracy: 81.0719, Test Loss: 2.3305, Test Accuracy: 54.210003\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 52, Loss: 0.6243, Accuracy: 81.5561, Test Loss: 2.3859, Test Accuracy: 53.289997\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 53, Loss: 0.6081, Accuracy: 82.1303, Test Loss: 2.3815, Test Accuracy: 54.340000\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 54, Loss: 0.5959, Accuracy: 82.2883, Test Loss: 2.5073, Test Accuracy: 52.020000\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 55, Loss: 0.5738, Accuracy: 83.1426, Test Loss: 2.2762, Test Accuracy: 53.480000\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 56, Loss: 0.5632, Accuracy: 83.5407, Test Loss: 2.5802, Test Accuracy: 52.840000\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 57, Loss: 0.5441, Accuracy: 84.0309, Test Loss: 2.6394, Test Accuracy: 54.299999\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 58, Loss: 0.5154, Accuracy: 84.7371, Test Loss: 2.6081, Test Accuracy: 54.070004\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 59, Loss: 0.5074, Accuracy: 85.1612, Test Loss: 2.3270, Test Accuracy: 54.119999\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 60, Loss: 0.4962, Accuracy: 85.3513, Test Loss: 2.5118, Test Accuracy: 55.089996\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 61, Loss: 0.4869, Accuracy: 85.6054, Test Loss: 2.4553, Test Accuracy: 54.989998\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 62, Loss: 0.4694, Accuracy: 86.3116, Test Loss: 2.5925, Test Accuracy: 54.170002\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 63, Loss: 0.4627, Accuracy: 86.3096, Test Loss: 2.4492, Test Accuracy: 55.409996\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 64, Loss: 0.4563, Accuracy: 86.4637, Test Loss: 2.6008, Test Accuracy: 53.579998\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 65, Loss: 0.4416, Accuracy: 87.0118, Test Loss: 2.4354, Test Accuracy: 55.349998\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 66, Loss: 0.4287, Accuracy: 87.2899, Test Loss: 2.5117, Test Accuracy: 55.110001\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 67, Loss: 0.4089, Accuracy: 87.9801, Test Loss: 2.5105, Test Accuracy: 55.250000\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 68, Loss: 0.4038, Accuracy: 88.1382, Test Loss: 2.5724, Test Accuracy: 55.080002\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 69, Loss: 0.4024, Accuracy: 88.1642, Test Loss: 2.8085, Test Accuracy: 53.710003\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 70, Loss: 0.3918, Accuracy: 88.4883, Test Loss: 2.7364, Test Accuracy: 54.799999\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 71, Loss: 0.3683, Accuracy: 89.3686, Test Loss: 2.8380, Test Accuracy: 53.920002\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 72, Loss: 0.3623, Accuracy: 89.3786, Test Loss: 2.4773, Test Accuracy: 54.790001\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 73, Loss: 0.3587, Accuracy: 89.3766, Test Loss: 2.5146, Test Accuracy: 54.459999\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 74, Loss: 0.3412, Accuracy: 89.8928, Test Loss: 2.6856, Test Accuracy: 55.489998\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 75, Loss: 0.3219, Accuracy: 90.5610, Test Loss: 2.5904, Test Accuracy: 55.139999\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 76, Loss: 0.3269, Accuracy: 90.4429, Test Loss: 2.8589, Test Accuracy: 54.570000\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 77, Loss: 0.3216, Accuracy: 90.7330, Test Loss: 2.7352, Test Accuracy: 55.959999\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 78, Loss: 0.3162, Accuracy: 90.8531, Test Loss: 2.5526, Test Accuracy: 55.099998\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 79, Loss: 0.2916, Accuracy: 91.6013, Test Loss: 2.6428, Test Accuracy: 55.570000\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 80, Loss: 0.2966, Accuracy: 91.3072, Test Loss: 2.8359, Test Accuracy: 54.230000\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 81, Loss: 0.2898, Accuracy: 91.4773, Test Loss: 2.5777, Test Accuracy: 54.650002\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 82, Loss: 0.2887, Accuracy: 91.4733, Test Loss: 2.8829, Test Accuracy: 56.089996\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 83, Loss: 0.2783, Accuracy: 91.8314, Test Loss: 2.9520, Test Accuracy: 55.580002\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 84, Loss: 0.2636, Accuracy: 92.2075, Test Loss: 2.9001, Test Accuracy: 55.549995\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 85, Loss: 0.2708, Accuracy: 92.1715, Test Loss: 2.9669, Test Accuracy: 54.919998\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 86, Loss: 0.2600, Accuracy: 92.4616, Test Loss: 3.3621, Test Accuracy: 53.910000\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 87, Loss: 0.2565, Accuracy: 92.5696, Test Loss: 2.8452, Test Accuracy: 56.360001\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 88, Loss: 0.2468, Accuracy: 92.9057, Test Loss: 2.7518, Test Accuracy: 54.939999\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 89, Loss: 0.2346, Accuracy: 93.0718, Test Loss: 2.7809, Test Accuracy: 55.220001\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 90, Loss: 0.2316, Accuracy: 93.3419, Test Loss: 2.9343, Test Accuracy: 55.720001\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 91, Loss: 0.2310, Accuracy: 93.2318, Test Loss: 2.9934, Test Accuracy: 55.669998\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 92, Loss: 0.2294, Accuracy: 93.3739, Test Loss: 2.7957, Test Accuracy: 55.820000\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 93, Loss: 0.2222, Accuracy: 93.6700, Test Loss: 2.5102, Test Accuracy: 54.970001\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 94, Loss: 0.2195, Accuracy: 93.6980, Test Loss: 3.0104, Test Accuracy: 55.459999\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 95, Loss: 0.2107, Accuracy: 93.9621, Test Loss: 3.2167, Test Accuracy: 55.489998\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 96, Loss: 0.2101, Accuracy: 93.9481, Test Loss: 3.4778, Test Accuracy: 53.619999\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 97, Loss: 0.2113, Accuracy: 93.7920, Test Loss: 3.1528, Test Accuracy: 56.129997\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 98, Loss: 0.1977, Accuracy: 94.1741, Test Loss: 3.0094, Test Accuracy: 56.129997\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 99, Loss: 0.1952, Accuracy: 94.4142, Test Loss: 2.9158, Test Accuracy: 55.870003\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 100, Loss: 0.1912, Accuracy: 94.4562, Test Loss: 2.9097, Test Accuracy: 55.849998\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 101, Loss: 0.1896, Accuracy: 94.5362, Test Loss: 2.9554, Test Accuracy: 55.089996\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 102, Loss: 0.1856, Accuracy: 94.6983, Test Loss: 3.0282, Test Accuracy: 54.570000\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 103, Loss: 0.1872, Accuracy: 94.6483, Test Loss: 2.8528, Test Accuracy: 54.879997\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 104, Loss: 0.1778, Accuracy: 94.9104, Test Loss: 2.9751, Test Accuracy: 53.430004\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 105, Loss: 0.1696, Accuracy: 95.1244, Test Loss: 2.8672, Test Accuracy: 55.440002\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 106, Loss: 0.1670, Accuracy: 95.2485, Test Loss: 2.9703, Test Accuracy: 55.369995\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 107, Loss: 0.1657, Accuracy: 95.2125, Test Loss: 3.0520, Test Accuracy: 55.099998\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 108, Loss: 0.1550, Accuracy: 95.4085, Test Loss: 3.0591, Test Accuracy: 54.369999\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 109, Loss: 0.1538, Accuracy: 95.5806, Test Loss: 3.1334, Test Accuracy: 53.890003\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 110, Loss: 0.1599, Accuracy: 95.3985, Test Loss: 3.2987, Test Accuracy: 55.190002\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 111, Loss: 0.1545, Accuracy: 95.5486, Test Loss: 2.6802, Test Accuracy: 56.269997\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 112, Loss: 0.1555, Accuracy: 95.5626, Test Loss: 3.1955, Test Accuracy: 54.889999\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 113, Loss: 0.1501, Accuracy: 95.7546, Test Loss: 2.8347, Test Accuracy: 55.709999\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 114, Loss: 0.1391, Accuracy: 95.9987, Test Loss: 3.0497, Test Accuracy: 56.320000\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 115, Loss: 0.1447, Accuracy: 95.8207, Test Loss: 2.9948, Test Accuracy: 57.099998\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 116, Loss: 0.1408, Accuracy: 95.9587, Test Loss: 3.1318, Test Accuracy: 56.120003\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 117, Loss: 0.1426, Accuracy: 95.9127, Test Loss: 3.0563, Test Accuracy: 54.860001\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 118, Loss: 0.1297, Accuracy: 96.2148, Test Loss: 3.0966, Test Accuracy: 55.199997\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 119, Loss: 0.1385, Accuracy: 96.0267, Test Loss: 3.3038, Test Accuracy: 55.870003\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 120, Loss: 0.1319, Accuracy: 96.2968, Test Loss: 3.0127, Test Accuracy: 55.080002\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 121, Loss: 0.1292, Accuracy: 96.1688, Test Loss: 3.1375, Test Accuracy: 57.050003\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 122, Loss: 0.1336, Accuracy: 96.1768, Test Loss: 3.1730, Test Accuracy: 54.630001\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 123, Loss: 0.1340, Accuracy: 96.2168, Test Loss: 3.1403, Test Accuracy: 56.040001\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 124, Loss: 0.1292, Accuracy: 96.2168, Test Loss: 3.3225, Test Accuracy: 56.419998\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 125, Loss: 0.1237, Accuracy: 96.4989, Test Loss: 3.0029, Test Accuracy: 56.230003\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 126, Loss: 0.1170, Accuracy: 96.6549, Test Loss: 3.0787, Test Accuracy: 55.070000\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 127, Loss: 0.1309, Accuracy: 96.2348, Test Loss: 3.2435, Test Accuracy: 55.580002\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 128, Loss: 0.1160, Accuracy: 96.7410, Test Loss: 3.2707, Test Accuracy: 54.610001\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 129, Loss: 0.1207, Accuracy: 96.4789, Test Loss: 3.3329, Test Accuracy: 54.459999\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 130, Loss: 0.1122, Accuracy: 96.8090, Test Loss: 3.0909, Test Accuracy: 55.409996\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 131, Loss: 0.1112, Accuracy: 96.8610, Test Loss: 3.4455, Test Accuracy: 55.379997\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 132, Loss: 0.1130, Accuracy: 96.6909, Test Loss: 3.0223, Test Accuracy: 55.519997\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 133, Loss: 0.1125, Accuracy: 96.7790, Test Loss: 2.9669, Test Accuracy: 55.299995\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 134, Loss: 0.1067, Accuracy: 96.9970, Test Loss: 3.3158, Test Accuracy: 54.820000\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 135, Loss: 0.1062, Accuracy: 96.9450, Test Loss: 3.0776, Test Accuracy: 56.279999\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 136, Loss: 0.1046, Accuracy: 97.0911, Test Loss: 3.2108, Test Accuracy: 55.779999\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 137, Loss: 0.1070, Accuracy: 96.8950, Test Loss: 3.2271, Test Accuracy: 56.779999\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 138, Loss: 0.0958, Accuracy: 97.2471, Test Loss: 3.0457, Test Accuracy: 55.900002\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 139, Loss: 0.0992, Accuracy: 97.1511, Test Loss: 3.2464, Test Accuracy: 55.419998\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 140, Loss: 0.0997, Accuracy: 97.1431, Test Loss: 3.3294, Test Accuracy: 55.570000\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 141, Loss: 0.0987, Accuracy: 97.2871, Test Loss: 3.2771, Test Accuracy: 55.529999\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 142, Loss: 0.0899, Accuracy: 97.3952, Test Loss: 3.2097, Test Accuracy: 55.849998\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 143, Loss: 0.0892, Accuracy: 97.5492, Test Loss: 3.3503, Test Accuracy: 56.639999\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 144, Loss: 0.1008, Accuracy: 97.1411, Test Loss: 2.9623, Test Accuracy: 56.660004\n",
      "Total number of trainable parameters = 20181311\n",
      "\n",
      "Epoch 145, Loss: 0.0883, Accuracy: 97.4432, Test Loss: 3.1809, Test Accuracy: 56.099998\n",
      "Total number of trainable parameters = 20181311\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Train model using 'GradientTape'-\n",
    "    \n",
    "# Initialize parameters for Early Stopping manual implementation-\n",
    "# best_val_loss = 100\n",
    "# loc_patience = 0\n",
    "    \n",
    "for epoch in range(num_epochs):\n",
    "    \n",
    "    if loc_patience >= patience:\n",
    "        print(\"\\n'EarlyStopping' called!\\n\")\n",
    "        break\n",
    "        \n",
    "    # Reset the metrics at the start of the next epoch\n",
    "    train_loss.reset_states()\n",
    "    train_accuracy.reset_states()\n",
    "    test_loss.reset_states()\n",
    "    test_accuracy.reset_states()\n",
    "    \n",
    "    # variable to keep track of current training step-\n",
    "    curr_step = 0\n",
    "    \n",
    "    # Train using data augmentation (ImageDataGenerator())-\n",
    "    for x, y in datagen.flow(X_train, y_train, batch_size = batch_size, shuffle = True):\n",
    "    # for x, y in zip(augmented_train_batches, y_train):\n",
    "        train_one_step(winning_ticket_model, mask_model, optimizer, x, y)\n",
    "        # print(\"current step = \", curr_step)\n",
    "        curr_step += 1\n",
    "        \n",
    "        # break out of infinite loop to end current training epoch-\n",
    "        if curr_step >= X_train.shape[0] // batch_size:\n",
    "            # print(\"\\nTerminating training (datagen.flow())\")\n",
    "            break\n",
    "\n",
    "    # Testing dataset-\n",
    "    for x_t, y_t in test_dataset:\n",
    "    # for x_t, y_t in zip(validation_batches, y_test):\n",
    "        test_step(winning_ticket_model, optimizer, x_t, y_t)\n",
    "    \n",
    "    '''\n",
    "    for x, y in train_dataset:\n",
    "        train_one_step(winning_ticket_model, mask_model, optimizer, x, y)\n",
    "\n",
    "    for x_t, y_t in test_dataset:\n",
    "        test_step(winning_ticket_model, optimizer, x_t, y_t)\n",
    "\n",
    "    '''\n",
    "\n",
    "    \n",
    "    template = 'Epoch {0}, Loss: {1:.4f}, Accuracy: {2:.4f}, Test Loss: {3:.4f}, Test Accuracy: {4:4f}'\n",
    "    \n",
    "    \n",
    "    # 'epoch' is the index for current training epoch-\n",
    "    history['accuracy'][epoch] = train_accuracy.result() * 100\n",
    "    history['loss'][epoch] = train_loss.result()\n",
    "    history['val_loss'][epoch] = test_loss.result()\n",
    "    history['val_accuracy'][epoch] = test_accuracy.result() * 100\n",
    "    \n",
    "\n",
    "    print(template.format(\n",
    "        epoch + 1, train_loss.result(),\n",
    "        train_accuracy.result() * 100, test_loss.result(),\n",
    "        test_accuracy.result()  *100)\n",
    "         )\n",
    "    \n",
    "    # Count number of non-zero parameters in each layer and in total-\n",
    "    # print(\"layer-wise manner model, number of nonzero parameters in each layer are: \\n\")\n",
    "    model_sum_params = 0\n",
    "    \n",
    "    for layer in winning_ticket_model.trainable_weights:\n",
    "        # print(tf.math.count_nonzero(layer, axis = None).numpy())\n",
    "        model_sum_params += tf.math.count_nonzero(layer, axis = None).numpy()\n",
    "    \n",
    "    print(\"Total number of trainable parameters = {0}\\n\".format(model_sum_params))\n",
    "    \n",
    "    '''\n",
    "    # Manual implementation of code for early stopping-\n",
    "    if test_loss.result() < best_val_loss:\n",
    "        best_trained_weights = copy.deepcopy(winning_ticket_model.get_weights())\n",
    "        print(\"\\ntest_loss.result = {0:.4f}, best_val_loss = {1:.4f}, copied 'best weights'\\n\".format(test_loss.result(), best_val_loss))\n",
    "    '''\n",
    "    \n",
    "    '''\n",
    "    # Code for manual Early Stopping:\n",
    "    if np.abs(test_loss.result() < best_val_loss) >= minimum_delta:\n",
    "        # update 'best_val_loss' variable to lowest loss encountered so far-\n",
    "        best_val_loss = test_loss.result()\n",
    "        \n",
    "        # reset 'loc_patience' variable-\n",
    "        loc_patience = 0\n",
    "        \n",
    "    else:  # there is no improvement in monitored metric 'val_loss'\n",
    "        loc_patience += 1  # number of epochs without any improvement\n",
    "    '''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Visualization of training:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmkAAAHoCAYAAAASdXgyAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAAMTQAADE0B0s6tTgAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOzdd3hUVf7H8fdMeg+BJCSEhJLQIaGIiDQVBJEFZSmiFMWyoNiw7aqrrg1xWdQVXFlUECxrFwEF5CdFQCD0TiAEQoCQkN7bnN8fs8wS6RCYQD6v58kDmXPn3u/cDMwn59xzrsUYYxARERGRasXq7AJERERE5GQKaSIiIiLVkEKaiIiISDWkkCYiIiJSDSmkiYiIiFRDCmkiIiIi1ZBCmoiIiEg1pJAmIleUiIgIZs6cec7bN2jQgA8++ODSFXSR7r77boYPH+7sMqqtmTNnEhER4ewyRJxCIU3kMrj11lsZMmTIKdueeuopWrVq5fg+IyOD8ePHEx0djYeHB+Hh4dx000188cUX2Gy2Ss+dNWsWnTt3xs/PD39/f2JjY3nhhRdIS0s7Yz0PP/wwsbGxuLq6njIglJSU8Oyzz9KgQQO8vLzo3r0727dvv4BXXjM9//zz9OjR45y2feedd5g6deqlLegKNnToUDZu3OjsMkScQiFN5DIYNWoUc+fOJTs7u9LjNpuNzz77jFGjRgGQlpbGNddcw+LFi3nzzTfZunUr8+fP5+677+att94iOTnZ8dxx48bx4IMPMmDAAH755Rc2bNjAxIkTOXz4MB9++OEZ67FarTz44IP07NnzlO3PPfccX3zxBTNmzGDLli3ExsbSq1cv8vLyLvJMyHFlZWUYYwgICCAgIMDZ5VwSJSUlF70PLy8vgoODq6AakSuQEZFLrri42AQGBpp///vflR5fuHChcXFxMYcOHTLGGDN69GgTFRVlioqKTrkfm81mjDFm+fLlBjA//vjjGbc7m1GjRpm77rrrpMfDwsLMzJkzHd9XVFSYkJAQ8/777592X1FRUebNN980t99+u/Hy8jLNmzc3a9euNVu2bDEdO3Y0Pj4+pm/fviYjI8PxnPz8fHPvvfeawMBA4+PjYwYOHGhSU1Md7SUlJeb+++83Pj4+JiIiwsyaNcvUq1fPzJgxw7FNYmKi6devn/Hx8TFhYWHmoYceMgUFBZXqmj59epXWXV5ebp5//nlTr1494+vra7p37242b95sjDFmxowZBqj0lZSUZJYsWWIA89NPP5kWLVoYFxcXk56eftLPID8/3zz00EMmNDTUeHp6mrZt25rVq1eftv7fe/31102zZs2Ml5eXiY6ONu+8806l9rKyMvPXv/7V1K9f33h4eJhmzZqZH374wdG+aNEi07FjR+Ph4WFCQkLM2LFjjTHGJCUlGcDs2bPHse3x11RWVmaMMebFF180119/vZk8ebIJDw83rVq1uuiaZsyYYerVq1dp+3feecc0bNjQeHl5mQ4dOpglS5Y42hITE03v3r2Nn5+f8fPzMx07dqxUs8iVRD1pIpeBh4cHQ4cOZdasWZUenzVrFr169SI8PBybzcY333zDgw8+iKen5yn3Y7FYAPjyyy9p3rw5t9xyyxm3u1AlJSV4eXk5vrdarXh4eLBq1aozPm/SpEkMHDiQTZs20bRpU0aMGMGjjz7Km2++yYoVK0hISOC1115zbP/444+zbNky5syZw/Llyzl06BAjRoxwtE+YMIG5c+fy7bffMm/ePD788EMyMjIc7aWlpfTu3ZuYmBjWr1/PnDlziI+P54knnjiv13u+df/tb3/jxx9/5PPPP2fjxo1cf/319OrVi9zcXIYOHcpjjz3Gddddx5EjRzhy5Aj169ev9Nzp06ezdetW/P39T6rlgQceYPHixcyaNYtt27bx3HPPOYa5ly5disViYf/+/ad9LR4eHkyfPp3t27fz2muv8eyzz/Ljjz862l988UWmT5/O22+/zfbt25k8eTJubm4A7Nixg1tvvZWePXuyceNGfvrpJ5o3b35e53LTpk2sWbOGRYsW8eWXX150Tb/30Ucf8c477/Dee++xbds2Ro4cSd++fR3nZNy4cYSGhhIfH8+6det45JFHsFr1USdXKGenRJGaYuXKlcZisZh9+/YZY4zJy8szPj4+5vPPPzfGGJOammoA89133zmek52dbXx8fBxfn3zyiTHGmD59+pgBAwZU2v8dd9zh2K5Pnz7nVNPpetKGDBliOnToYJKTk01ZWZl56623DGBuvvnm0+4rKirK0etijDG//fabAcxXX33leGzChAmmXbt2xhhjcnNzjaurq5k/f76jfefOnQYw27ZtM8YYExISYv71r3+d1H68J+3jjz827du3r1THypUrjbu7uykvL3fUdbaetPOpu6ioyHh5eZmtW7dW2k9MTIyZPXu2McaY5557znTv3r1S+/Fep6VLl1Z6/MSfQWJiogFMfHz8KWtds2aNadq0qUlJSTnt6/m9P/3pT+aee+4xxhhTWFhoPDw8Kr22E40cOdLceuutp2w71540X19fk5eXV2U1/b4nrWHDhmbu3LmVtunVq5d55ZVXjDHGtGrVysyaNeuMxxe5UujXC5HLpHPnzkRHRzN79mwAvvnmG1xcXBgwYAAAxpiTnuPn58emTZvYtGkTfn5+lJWVnXbbt956i02bNjFq1CiKiooAeP311/H19XV8nat33nmHoKAgoqKi8PT0ZO7cufTp0+esPRKtW7d2/D00NBSAli1bVnosPT0dgH379lFeXk6nTp0c7c2aNSMwMJDdu3eTk5NDWloaHTt2rNTu5+fn+H7r1q1s3ry50mvs1asXpaWlHDp06Jxf7/nUnZiYSFFREZ06dap03MTERPbt23fWY7Vt2/a0bdu3b8fHx4cOHTqcsr1jx47s2rWLevXqnXYf8+fPp0uXLoSGhuLr68tHH33EwYMHAdi7dy8lJSWnndSwbdu2c57wcDoxMTEnvdcupqYT5efnk5SUxNChQyud+yVLljjO/YMPPsh9991H7969mTRpkuM4IlciV2cXIFKTjBw5ko8//pgXXniB2bNnM2TIEMewYkhICP7+/uzevduxvdVqJTo6GgAXFxfH4zExMSxevLjSvuvWrQtArVq1HI+NGTPmtLNKz6Ru3bosXLiQgoIC8vPzCQ0NpVOnTrRr1+6MzztxiOr4kOvvHzs+dHeqoHmi4+1nGrrNz8+nW7duTJs27aS2sLCwM+7/QuvOz88H7EOPgYGBlfYTFBR01mN5e3ufts0Yc1FD1fv27WPgwIE888wzvP322wQEBDBx4kT27t3r2P+ZnKn9eEA/cZvjvzSc6Pev72JrOlFBQQEAn332WaUQDTjC+9ixY+nduzdz585l7ty5vPjiiyxYsICuXbue83FEqgv1pIlcRiNGjCAxMZGvvvqKJUuWOGZ1gv1D8I9//CNTp0519ISdzuDBg9m1axfz5s0743ZBQUFER0c7vs6Xj48PoaGh7N+/n3Xr1vGHP/zhvPdxOo0bN8bV1ZXVq1c7Htu1axfZ2dmOHrWQkBDWrl3raN+9e3elGaaxsbHs2rWLiIiISq8zOjr6tNc0XazmzZvj7u7OkSNHTjrm8ZDm5uZGRUXFee+7VatW5Ofns27duguqbcOGDXh5efHyyy/ToUMHYmJiSEpKcrTHxMTg4eHB0qVLT/n81q1bn7bt+AzL1NRUx2Nbt2695DWdKCQkhLp165KcnHzSuT/eAwrQqFEjHn30URYvXkz37t35/PPPz7pvkepIPWkil1FUVBQ9evTggQceoGHDhnTp0qVS+4QJE/jll1/o2LEjL730Eq1bt6a8vJxly5aRmZnp6E3r1q0bY8aMYejQofz1r3/lpptuok6dOiQkJLBw4cJTXpB+or1795Kfn09mZiYVFRVs2rQJd3d3WrRoAcDKlSvJzMykZcuWJCQk8PDDD9O7d+/TTlS4EH5+fowePZrHHnsMPz8/fHx8ePDBB+nVq5ejjjFjxvC3v/2Nxo0bExwczOOPP15pUsVdd93FxIkTGTp0KM8//zy1atVi586dLFu2jEmTJlVZrSfy9/dn3LhxjB07ltLSUtq1a0dqaipz587lrrvuomXLlkRFRbF792527dpFnTp1zqmHDezh4s4772T48OG8++67NG7cmC1btlC3bl06derE2rVrGTlyJP/3f/93yiHPxo0bk5uby8yZM+nSpQv/+c9/iI+Pd/SAenl58cQTTzgupm/bti179uzBZrPRp08fnnnmGeLi4njuuecYMWIEJSUl/Prrr4wbNw4vLy86dOjAhAkTCAkJISEhgffee++sr+liazqRxWLh2Wef5a9//Su+vr5069aNrKwsFi9eTMeOHbnxxht5/PHHufXWW4mOjubgwYNs2bKF3r17n9P5F6l2nHY1nEgNNXPmTAOYl1566ZTtaWlp5tFHHzUNGzY0bm5uxt/f33Tv3t189NFHpqKiotK2H330kbn22muNj4+PcXd3N02bNjVPPvmkOXLkyBlr6N69+0nLRERFRTnaFy1aZGJiYoy7u7sJCwsz48ePN4WFhWfc5+8v0D/Vhea/vwg8Ly/PjB492gQEBJxyCY7i4mIzevRo4+3tbcLDwx3PP3EJjv3795tBgwaZgIAA4+3tbdq0aWMmTZp02rqqou6Kigrz2muvmQYNGhg3NzcTERFhhg8f7jjveXl55tZbbzW+vr4nLcFx/CL74061BMeYMWNM7dq1jZeXl2nXrp1Zs2aNMeZ/F+onJSWd9vW8/vrrJjg42Pj5+Zl7773XPPnkk5UmMZSVlZlnn33WhIWFGQ8PD9OiRQszb948R/uCBQtMu3btjLu7uwkNDTXjxo1ztG3evNm0b9/eeHl5me7du5uPP/74lEtwVGVNp1qCY9q0aaZZs2bGzc3N1K1b19x+++1m165dxhhjHnroIdOwYUPj4eFh6tWrZ5566inHJBKRK43FmPO4IEBERERELgtdkyYiIiJSDSmkiYiIiFRDCmkiIiIi1ZBCmoiIiEg1pJAmIiIiUg1ddeukeXh4OBZdFBEREanO0tPTKSkpOWXbVRfSgoODSUlJcXYZIiIiImcVERFx2jYNd4qIiIhUQwppIiIiItWQQpqIiIhINXTVXZN2JjabDd0FS05ksViwWvW7ioiIVD+XPKQ98sgj/PDDDxw4cICtW7fSqlUrANLS0hg5ciSJiYl4eHjw/vvv06VLFwAKCwu59957iY+Px2q18sYbbzBw4MALrqG0tJTk5GTKysqq5DXJ1cXNzY3IyEjc3d2dXYqIiIjDJQ9pgwYN4umnn3YEsOP+/Oc/06lTJxYsWEB8fDyDBg0iMTERV1dXJk2ahIeHB3v37iUpKYnrrruOG264gVq1al1QDcnJyfj5+VG7dm0sFktVvCy5ShhjyMjIIDk5mejoaGeXIyIi4nDJQ1q3bt1O+fiXX35JUlISANdccw2hoaGsWLGCHj168MUXXzBz5kwAGjZsSLdu3ZgzZw533333eR/fZrNRVlZG7dq1cXWtUaO7co5q165NZmYmNptNQ58iIlJtOOUTKSMjA5vNVmnR2QYNGpCcnAzYe76ioqJO2fZ7kydPJiIiwvGVn59fqf34NWjqQZPTOf7e0PWKIiJSnTit2+D3oen3H5Antp/pw3P8+PGkpKQ4vnx9fau2UBEREREncEpIq127NmC/FcJxBw4cIDIyEoDIyEj2799/yrarxUsvvURpaekFPXfdunXcdddd57RtXFwcRUVFF3QcERERcR6n9aQNHjyYqVOnAhAfH09qaqpjcsGJbUlJSSxbtoz+/fs7q9RL4m9/+9tpQ1p5efkZn9uhQwc+/fTTczrOpk2b8PLyOu/6qpuznRMREZGrzSW/kv6hhx5izpw5pKam0rNnT3x9fdm7dy8TJ05kxIgRxMTE4O7uzuzZsx0X9j/11FOMHj2a6OhorFYrU6dOJSgoqMpquu/jeA5kFFbZ/k4UVdubD0Zdc8ZtxowZA0Dnzp2xWq0sWrSIp59+Gn9/fxISEjh48CDbt29n+PDh7Nq1i9LSUiIjI/noo48ICQlh6dKlPPnkk6xbt479+/fToUMHHnzwQebPn09OTg7//Oc/6du3L2AfNs7Ly8PX15cGDRpwzz33sHDhQo4cOcK9997L888/D8COHTu45557KCgooE2bNuzbt4/nn3+efv36Vao9NTWVYcOGkZubS3FxMTfddBPvvPMOFouF0tJSnnvuORYsWIDVaiUsLIwFCxYAMHHiRD755BOsViteXl788ssvrF271vE6ALZt20a/fv3Yv3+/43U98sgj/PzzzwwcOJA2bdrw/PPPU1xcTFlZGU888QT33HMPAIcOHeLRRx8lISEBgAEDBjB27Fjatm1LUlIS3t7eAAwbNoxu3boxduzYqvhxi4iIXDKXPKRNnTrV0St2otDQUBYtWnTK5/j4+PDFF19c6tKc5v3332fatGmsWrWq0jV0K1asYPny5Y7H3n77berUqQPAG2+8wcsvv8yUKVNO2l9GRgbt27fn5ZdfZsGCBTz66KOOkPZ72dnZrFq1ivT0dKKjo7nnnnuoV68eI0aM4PHHH2f48OGsX7+ejh07nvL5gYGBzJ07F19fXyoqKhgwYADffPMNgwYNYsKECSQmJrJu3To8PDwcw9kff/wx33//PStXrsTf35+srCw8PDzOep4yMjKIjo7mhRdeACArK4sVK1bg4uJCZmYm7dq1o0+fPoSFhTF8+HD69u3L119/DdiH0oODg+nZsyefffYZ9913H6mpqSxevJjp06ef9dgiIiLOViPXpDhbT5ezDBkypFJo+/TTT5k9ezYlJSUUFRVRt27dUz7Px8eHAQMGAHDdddeRmJh42mMcv5YtODiYRo0akZSUhJ+fH9u2bePOO+8EoH379rRp0+aUz7fZbDzzzDOsWLECYwxpaWnExcUxaNAg5s2bxz/+8Q9HADs+e3fevHmMHTsWf39/gHNe787T05Nhw4Y5vs/IyODee+8lISEBV1dXjh07xvbt2/Hz82PVqlX8/PPPjm2PH/vRRx/lT3/6E/fddx/Tpk3jzjvv1OQSERG5ImhRqGrk971qU6ZM4aeffmLr1q1MnjyZ4uLiUz7P09PT8XcXFxcqKipOe4zfb1teXo4xBovFck7LlEyePJmMjAzWrFnDli1buPPOO09b19m4urpWqvX3+/Hx8alU05gxY+jevTtbt25l06ZNNGnS5KzH7tixI56enixbtozp06fz0EMPXVCtIiIil5tCmpP4+fmRk5Nz2vasrCz8/f0JCgqitLSUadOmXbJaAgICaNGiBZ9//jkAGzduZOvWraetq27dunh6enL06FG++uorR1v//v15++23KSkpAf43e7d///7861//Ijc3F7APuVZUVNCwYUOSkpLIyMgAYPbs2WesMysri6ioKCwWC8uXL2fz5s2APdx26dKFt956y7HtiTOHH330UYYPH07Lli1p0qTJeZ0bERERZ1FIc5InnniCG2+8kbi4ONLS0k5qv+WWW4iOjqZZs2b07t2buLi4S1rPrFmzeOutt2jfvj1Tp04lNjaWgICAk7Z75JFHWLVqFXFxcYwePZqePXs62p555hkaN25M27ZtiYuLY9SoUQCMGDGC2267jeuuu464uDj69u1LSUkJ9erV48knn6RDhw7ccMMNBAYGnrHGN954g6eeeopOnToxc+ZMrr32Wkfb7NmzWb16NS1btiQ2NrbStXuDBg0iPz+fcePGXexpEhGRK1hpuY2dR3L5fuMh3vhpF2/8tIuDmSdPJCwoKWfFnmNOqLAyi7nKllmPiIggJSXF8X1FRQUJCQk0adIEFxcXJ1ZWvRUUFODt7Y3FYmHHjh306NGD3bt3X/D9UquTtWvXOmbKnuq2T3qPiIhcOTLyS1iyOx0fdxfqBnhSN8CTWt7uWC0WrBawGUjOLGDP0XwSjuazP6OAQ1lFpGQVkppbjO13qcfFauHW1mHc26UhR3KKmbv5MP+36yil5TbWPteTOr5nn+h2MX6fW05UIycOyMlWrlzJU0895bi7w/Tp06+KgHbfffexaNEiPvjgA92XU0SkmqiwGXYeyWX74Rw8XF3w83TFz9ON3KIyEtLy2HM0n0NZRTQO8aVdZCDto2qRmlvMZ2uSWbg9lbKK8+tfquXtRr1aXrSJCCQ6xJemdf1oVtePo7klTFueyA+bD/PD5sOO7Ts2COIPceF4uDr3c0M9aVLj6T0iImJ3fCLZ7x3LL2FjcjZtIwMr9SwVlJQzf+sRliekk1NURm5xOXnFZdTydqdVuD8t6wXQONiHnKIyjuaWcDS3mG2HcliTlEle8ZkXKffzcCWv5ORtOjUKYnD7+ri6WDiSU0xqTjE5RWXYjMFm7K+hfpA3MSG+xIT40TDYB1+PM/dJbU3J4av1B6lfy5t+sWGEBVy+ReDVkyYiIiKV5BSVsXR3GltTctiVmseu1FwKSysY1jGSB7o1ItTfk9JyG7N+2887i/eQV1KOxQJx9QO5sWkI+zMK+WnbEQpL7bP0fT1c8fN0xdfDlb1p+aw/kHXK47paLcTWD6RToyDi6tfCZgy5RWXkFZfj5e5Ck1BfokP8CPBy43B2ERuSs1h/IAsPVxcGd4igcXDVL6PUOiKA1hEnX4ftbAppIiIi1VBqTjHH8ktoEuqH+wnDbilZhfy84yiHs4uIrO1Dw9o+NKjjjaebCzabvTeprMJGYWkFBaXlFJVWYDth0Cwlq4iftqWyau8xyv97gZaXmwtN6vpRVFrOhyuSmP3bAQbEhbP+QBb7jhUQUcuLcTdGsyUlh2UJ6WxMzgbsgW1whwj6tQknwMvNcQxjDIeyi9h2KJcDGQXU8nYnxN+DED9PGtTxxtv93OJHeKAX4YFe9GsTXhWn9IqjkCYiInKJFJdVsCYpk+KyClrXCyAswNMxnFhWYeNARiF5xWX4ebri898hucU705i7+TDx+zMxBtxdrbQK96dpXX82Hcxm55Hci67LzcXC9dF16NOqLp0a1SYyyBsXqwVjDL/sSmPKkr18tT4FLzcXnry5Cfd1bYSnm/1ykNJyGxuTs6jt60F0yKl7tSwWCxG1vImo5X3RtdZkCmkiIiJVpLTcxo4juazbn8myhHTWJGVSWm5ztNfx9aBJqC9Hc4s5kFHo6Mn6PQ9XK31bhdEo2IfNKTlsPpjNhuRsanm78cd2EfRqEUqzun4kZxaSdKyAAxmFlFXYcLFasFjAzcWKt7sLPu6ueLq74Gr933Vm/p5udImpU6nn6ziLxcJNzUO5sVkIO4/kUcfPnRA/z0rbuLtaubZR7So6Y3ImCmlO8tJLL/Hss8/i7u5eZft44YUXaNmyJUOHDq2qMkVEahRjDIdzitl8MJvNB7M5mFVIdIgfsREBtImwr+W452geu4/mcSCjkJJyG+UVNspthgMZBWw7nOsIZe4uVjo2DKJbE3sg2nooh60pOWw6mE2ovyc9mobQOMSHWt7uFJaUk1dSTkm5jY4NgujZIrTSxe7GGI7kFBPi54Gry/+GPhvU8aFbk+AqPw8Wi4UW4f5Vvl85P5rd6SQWi4W8vLyLuo9kVeyjOiovL8fV9fL9/lBd3yMiUrXyS8rZf6yA5Ex7r5OXmwve7q6U2WxsS8lhc0o2mw7mcCy/5IL27+/pStvIWrSNDKRdZC06NKh1ztdeSc2l2Z3VzJgxYwDo3LkzVquVRYsW4eXlxfjx49m8eTPFxcV07tyZd999Fzc3N1599VU+/fRTx43L58yZw4QJE07ax9NPP02HDh0YN24cL730EgkJCeTl5ZGYmEjdunX5+uuvHbeZGjduHEuXLiU4OJjY2FjS0tL4+uuvT6r1qaeeYunSpZSVlREQEMAHH3xATEwMAPPnz+ell16itLQUi8XCtGnTuPbaa/ntt994+umnyc3NxRjDK6+8woABA2jQoAHz5s2jVatWAHTo0IFJkybRo0cPevTowfXXX8/q1asBWLhwIbfeeisZGRkUFRURFxfH9OnT8fa2X98wceJEPvnkE6xWK15eXvzyyy8MHjyY4cOHO27KvnDhQl544QXWrFlzCX+aIlIdlVfY2H44lzVJGazZl8nmlLOHL1erheZh/vRuGUps/UDi6gcSGeRNwtE8NqfksDUlGwsWmtb1o2ldPxoF++Dl5oKL1YKr1Yqnm/Wc7oEscq5qZkj77A7ISro0+67VEO78zxk3ef/995k2bRqrVq1y9II98MADdOvWjenTp2OM4f7772fKlCncfffdTJo0iSNHjuDl5UVhYSFWq/WU+/i9NWvWEB8fT1BQEHfccQfTpk3jL3/5C9OmTSM5OZkdO3ZQXl5Ojx49iIiIOOU+nnnmGf7+978D8J///IfHH3+cefPmkZCQwL333svy5ctp0qQJZWVlFBYWkpmZye233863335L586dsdlsZGdnn9Op27RpEwsWLMDNzQ1jDJ999hm1a9fGGMODDz7Ie++9x5NPPsnHH3/M999/z8qVK/H39ycrKwsPDw8ee+wx/va3vzlC2pQpU3QrKJGr1PHhP5sx+Hm64evhSm5RGUsT0li8M43lu9Mda2y5Wu1Dd11j6tCgtg9Rtb3xdLNSWFrhWD6iRbg/LcL8HRfHn6hNROB/hzqjLudLFKmhIa0a+v7771m9ejX/+Mc/ACgqKsLd3R1/f39iYmIYPnw4N998M7feeutpA9Xv3XLLLQQFBQFw3XXXOW6avmTJEkaMGIGrqyuurq4MGzaMX3/99ZT7WLRoEe+++y55eXnYbDbHTdJ//vln+vbt67hhuZubGwEBAcyfP58WLVrQuXNnAKxWq6OGsxkxYgRubvYLWY0xvPXWW8yfP5/y8nJycnLo1q0bAPPmzWPs2LH4+9uvlzh+Z4RevXrx2GOPsXnzZvz9/Vm3bt0pewdFpHorLrOHp4KScgpKy8nIL+VYfgnpeSWkZBWx40guO4/knrQYqsUCxtj/jI0IpGtMHa5tWJt2UYEadpQrUs18156lp8sZjDF8//33NGrU6KS21atXs2rVKpYuXUqnTp34/PPP6dq161n36en5vxk5LqWDzd0AACAASURBVC4ulJeXO451Ll3yycnJPPLII6xdu5ZGjRqxZcsWbrzxxvN4VZW5urpSUVHh+L64uLhS+4k9gp999hnLli1j+fLl+Pn58c9//pPly5ef9RiPPPIIU6dOJSAggNGjRzuGiEWk+sorLmP1vkyWJ6Tz65509mecfMPrE/l5uNI8zJ9mYX64uVjJKy4jv6Qcq8VCt5hgbmgWQrCf/u3Lla9mhrRqwM/Pj5ycHEcw6d+/P2+88Qbvvfcerq6uZGVlkZGRQWhoKHl5eXTt2pWuXbuyfft2Nm7cSNeuXU/ax7m64YYb+OSTTxgyZAjl5eV88cUXhIefvFBgTk4O7u7u1K1bF2MMU6ZMcbT17t2bV1991XHB/fHhzs6dO3PfffexatWqSsOdQUFBNG7cmDVr1hAbG8vatWvZvXv3aWvMysqidu3a+Pn5kZeXx8yZMx0Btn///rz33nvcdttt+Pv7k52djZ+fHy4uLowYMYJXX32VkpIS1q9ff17nRUSqTk5hGSnZhbharbi6WHB3sVI3wBO3E2Ym7jicy4yVSczZfNgxI7JeoBd/iA3H/7/rhnm7uxDk406wrwd1/DwIC/CkXqCXrv2SGkEhzUmeeOIJbrzxRry8vFi0aBFvv/02zzzzDHFxcVitVtzc3Jg4cSKenp4MGjSIgoICLBYLMTExjBo16pT7OFdjxoxh8+bNtGzZkoiICNq1a0dRUdFJ27Vu3ZrBgwfTsmVLIiMj6dWrl6MtOjqaDz/8kGHDhlFWVoaLiwvTpk2jY8eOfPfddzzxxBPk5eVhsVh45ZVX6N+/P6+99hqjRo3iww8/pF27drRs2fK0NY4cOZI5c+bQokUL6tWrR9euXTl06BBgHxY9fPgw1113HW5ubnh7e7N48WK8vb3x9vbmtttu4/Dhw9SvX/+cz4mInD9jDCXlNkrKbJSUV5BVWMbyhHQW7zzKugNZVPxuDTA3FwuNg31pHuZPak4xv+3LAKBdZCD9Y8Pp1iSYhnV8FMBE/ktLcNRQeXl5+Pn5UVJSQv/+/Rk8eDD33Xefs8u6aBUVFbRr144pU6ac05Dw8efoPSJybowxbDuUy7wth5m/9QgpWSf/gufpZqVLdDBtIgKwGUOFzVBUWsG+YwXsTs3jUHYRrlYL/dqEcc/1DYmtH+iEVyJSPWgJDjlJz549KSkpobi4mJ49e3L33Xc7u6SL9sMPP/Dwww/Tt2/fcw5oIvI/BzPt94TckJxFbR/3/97Wx4sym2FvWj570/LYkpLjCGbhAZ4MiAvHy80FD1crXu6uXNOgFtdH1znlLMnjcorKwECA98kr3ovI/yik1VBX49ph/fv3p3///s4uQ+SKUGEzJB0rYPvhHHYczmVZQjq7UvPO+ByLBaKCvLnn+gb0axNO2/qBWK3nPzR5qtsRicjJFNJERGoIm82wOimDL+MPsmjHUccaYUCle0J2ialDfnE5KVmFHMwqxGqxEBNiX7z1TD1kIlK1rvqQdvwC1Kvs0jupQsffG7pYWa4mNpshMT2fg1mFHMoq4kBGIYt2HCU50768RdvIQK5pEESLMH9ahvvTKNgXlxN6xXw9XKkb4EmHBue2zqGIVL2rPqQdnymZkZFB7dq19UEslRhjyMjIwM3NDavVevYniFQjxhiO5pbg5mLB290Vd1crG5KzmL/lCD9tO8LR3Mq3QQr282BM98YM7hBB4+Cr656/Ilejqz6kAURGRpKcnExmZqazS5FqyM3NjcjISGeXIXLO8orL+HbDIWb9tp/E9IJTbhNV25v7uzYkOsSXeoH2CQARtbxwddEvIyJXihoR0tzd3YmOjsZms2nYUyqxWCzqQZMrxp6jecz67QDfbkihoLQCf09Xhnaoj6uLhaLSCorKKmhYx4e+rcNoGe6vkQORK1yNCGnH6cNYRKqDnKIy3lu6F39PN25tHUaDOj6ONpvNcDCrkIKSCiwW+4zKfekFzP7tgGPx1xZh/ozqHEX/2Hp4uetCfpGrVY0KaSIiznYws5B7ZsazNy0fgL8v3E2rev60iQhkz9E8dhzOpeCEWZfHublYGBAXzsjromgXWUu9ZCI1gEKaiMhlsv5AFg/MWkdmYSnP9m1GTKgf87ccYeH2VLYdysXP05XWEQG0DA8gwMsNY8Bg8Pd04w+x4bppuEgNc9XfFkpExBmMsS8Wuzctn+TMQpKOFfDV+hRcLBbeviOO3i3rOrYtKa8gI7+UsABP9ZCJ1DC6LZSIyCVQXFbBNxtScLVaiAzyIbK2N3nFZfy45Qjztx45aeZlvUAv/jW8HW0iKt+r0sPVhfBAr8tZuohcARTSREQuwIGMAsZ+soEdR3JP2V7H14O7ro0krn4gUbV9iKrtTYifh3rKROScKaSJiJynBdtSeerrzeSXlPPITTHE1Q/gQEYhBzIKsVjg5hZ16dgwqNIK/iIi50shTUTkHOUUlvH3Rbv4ZHUyQT7uzBrdka4xwc4uS0SuUgppIiJnUWEzfLnuIH9fuJvMglI6NgjinWFxhAXoOjIRuXQU0kSkxjuUXcQ/Fu4mp6gMg31mps3g+PuRnGL2puUT7OfB5CGx3N62nq4tE5FLTiFNRGq0Q9lF3PHv3ziYWYS7qxUL9lX+LViwWuy3DnNzsfBAt0Y8fGM0fp5uzi5ZRGoIhTQRqbEOZxcx7N+rSckq4s0/tmHINfWdXZKIiINuZikiNdLh7CLu+PdqkjMLeWNgawU0Eal21JMmIjVGhc2wKvEY36xPYcH2VIrLbEwY2Jqh10Q6uzQRkZMopInIVeVwdhHbD+ey60guu1LzSM8voaTcRmm5jfS8Eo7llwDQPqoW93dtSJ9WYU6uWETk1BTSROSqUFpu4/UfdzJz1X7HYxYL1PJ2x9PVirurlVB/D4Z1rM/AdhE0rOPjvGJFRM6BQpqIXPHScot56LMNxO/PomODIAa2q0ezMH+ahPri7a7/5kTkyqT/vUTkimWzGVbsPcaTX20mLa+E+7s25Jk+zXB10ZwoEbnyKaSJyBXFZjOsTDzGgm2p/LzjKGl5JXi5ufDPYW3pHxvu7PJERKqMQpqIXDF2pebyl2+3sjE5G4BGwT78sX0Eg9tH0CjY18nViYhULYU0Ean2ikor+Ocve5i+fB82Yxh1XRQjrosiOsTP2aWJiFwyCmkiUi3E788k6VgBABagqKyChKN57DqSx+7UPPJKymkZ7s+Ega1pExHo3GJFRC4DhTQRcaoKm+HNhbuYtmzfKdsDvNxoHu7PLa3qMqJTlCYFiEiNoZAmIk6TU1jGw//ZyPKEdNpGBjLuhmgsFnubm4uVmBA/Qv09sBx/UESkBlFIE5HLLqeojF/3pDNp4W72ZxQypEMEr9zWCg9XF2eXJiJSbSikicglV1BSzpaUHDYkZ7EsIZ31B7KosBlcrRZeHtCSEZ2i1FsmIvI7CmkicknkFZfxyepk5mw6RMLRPGzG/riXmws3NA3mhmYh3NgshLAAL+cWKiJSTSmkiUiVyiksY8aqJGas3E9OURl1fN3p3bIucfUDaRtZizYRAXi6aVhTRORsFNJEpMosS0jnkc83klNURmSQN8/2bcbtbSNwd9WMTBGR86WQJiIXzRjDtOX7eHPBLnw9XJk0OJbb4sK1XIaIyEVQSBORi1JUWsEz32zhh82HiQnxZfrIDjSo4+PsskRErngKaSJyQWw2w9wth3lzwW4OZRdxc4tQJg+Nw9dD/62IiFQF/W8qIuctfn8mr87fyeaD2fh7uvJCvxbc3bkBVquW0RARqSoKaSJyTmw2wy+70vj38n2s3Z+Jq9XCPdc34JEbY6jl4+7s8kRErjoKaSJyVssS0nl57nYS0wtwc7EwqH0ED/ZoTKNgX2eXJiJy1VJIE5Ez+mnrER7+fCOebi78qXsj7unckLoBns4uS0TkqqeQJiKnNXfzYR77YhO1vN35/P5riQn1c3ZJIiI1hkKaiJykvMLGdxsP8cw3Wwj28+Cz+zvRWEObIiKXlUKaiACwPCGdHzYfZldqLglH8yktt1Ev0IvP7r+WqNpa90xE5HJTSBOp4bILS3l53g6+3XAIgLr+nnRuXJvmYf6M6BRFeKBugC4i4gwKaSI1lDGGhdtTef777RzLL6FH02BeGdCK+kHezi5NRERQSBOpkbYfzmHCj7tYsfcYAV5uTB4Sy+1t62GxaDFaEZHqQiFNpAZJzSlm0qLdfLMhBYA/tovgmT5NCfHXkhoiItWNQppIDWCM4dsNh3hp7nbyisvpEl2Hv/RtRsvwAGeXJiIip6GQJnKVS88r4bnvtrJox1GC/TyYPCSOns1DNLQpIlLNWZ158IULF9K+fXvatm1Lq1at+PjjjwFIS0ujT58+xMTE0KpVK1asWOHMMkWuWCv3HqP328tZtOMo/WPDWfRYN3q1CFVAExG5AjitJ80Yw5133smSJUto06YN+/fvp1mzZgwcOJA///nPdOrUiQULFhAfH8+gQYNITEzE1VUdfyLnwhjDzFX7eXX+TrzdXJhyZ1v6tQl3dlkiInIenJ56srOzAcjNzaV27dp4eHjw5ZdfkpSUBMA111xDaGgoK1asoEePHk6sVOTKUFJewQvfb+eLdQdpVMeHf4/sQHSI7hYgInKlcVpIs1gsfPnllwwcOBAfHx+ysrL49ttvycvLw2azERwc7Ni2QYMGJCcnn3I/kydPZvLkyY7v8/PzL3ntItXR0dxivt1wiC/XHSTpWAHdmgTz7rC2BHi5Obs0ERG5AE4LaeXl5UyYMIE5c+Zw/fXXEx8fz2233caWLVtOul7GGHPa/YwfP57x48c7vo+IiLhkNYtUR7tT83hzwS6W7E7DZqCWtxuP3hTDIzfF4GLVtWciIlcqp4W0TZs2cfjwYa6//nrAPqwZHh7Oli1bAEhPT3f0ph04cIDIyEhnlSpSLZVV2Hh/aSL//GUPFTZDj6YhDOkQwY3NQnF3deqcIBERqQJO+5+8fv36pKSksHv3bgD27t1LYmIiTZo0YfDgwUydOhWA+Ph4UlNT6dKli7NKFal2dhzOZcCUlfzj5wQa1vHh2wev56O7r6FPqzAFNBGRq4TTetJCQ0OZNm0agwYNwmq1Yozhvffeo169ekycOJERI0YQExODu7s7s2fP1sxOkf/6Zn0Kf/luKzab4ZEbo3noxmg8XF2cXZaIiFQxiznTBV9XoIiICFJSUpxdhkiVK6uw8fqPO5mxcj9hAZ68P7w9sfUDnV2WiIhchDPlFnVPiVwBDmYW8vTXW/htXwYdGwQx9a52BPt5OLssERG5hBTSRKqxIzlFTPllL1/EH6TcZhh5XRR/7dcCNxdddyYicrVTSBOphvKKy3j3l73MXLWf0nIb7SIDefLmpnSOruPs0kRE5DJRSBOpRowxfLfxEK//uItj+SW0CPPnyd5NuKGpboguIlLTKKSJVBOpOcU8/PkG4vdnUcvbjTcGtmZIh/pYtSCtiEiNpJAmUg3YbIbHvthI/P4shneK5MmbmxLo7e7sskRExIkU0kSqgRmr9rN6XyZ3XRvJq7e1dnY5IiJSDWiKmIiT7U3L580Fu4iq7c2zfZs7uxwREakmFNJEnKiswsb4LzdRVmHjH4Nj8fFQ57aIiNgppIk4ic1meOvnBLak5PBAt8Z0aBDk7JJERKQa0a/tIpeZMYble47x5oJdbD+cS7O6fjzeK8bZZYmISDWjkCZymZRX2FiWkM4Hvybx274M3F2t3N+1IQ/doBuki4jIyRTSRC6x/ccK+GxtMt9uOMSx/BKsFhjSIYJHezahXqCXs8sTEZFqSiFN5BLafjiHwe//RmFpBZFB3ozv1YQ/to9QOBMRkbNSSBO5RNLzSrj/43WUVxg+GNmBG5uF6O4BIiJyzhTSRC6B4rIK/jR7HYdzipk0OJaeLUKdXZKIiFxhtASHSBUzxvDsd1vZkJzNn7o1YlD7CGeXJCIiVyD1pIlUIZvNMOGnnXy74RA3NQvh6T7NnF2SiIhcoRTSRKpIUWkFj3+xiQXbU2kXGcjbd8ThomvQRETkAimkiVSBtLxi7v94HZtTcujXJoxJg2PxdNPaZyIicuEU0kQu0rH8Egb96zeSMwt56IbGPNGrqWZxiojIRVNIE7kIJeUVjJm9nuTMQl65rRUjOkU5uyQREblKaHanyAUyxvD8d9tYdyCL+7o0VEATEZEqpZAmcoE+XJHEV+tT6N4kmL/0be7sckRE5Cqj4U6R81RhM3y2NpnXf9xJ42Af3r2zrWZxiohIlVNIEzlHxhiWJqQz4cedJBzNJ9jPgw9GXYO/p5uzSxMRkauQQprIOTiSU8TTX2/h1z3H8HSz8vCN0fype2N8PfRPSERELg19woicxc87jvLU15vJLixjYLt6PNW7KWEBXs4uS0RErnIKaSKnUVxWwRs/7WLmqv0Eervx7xHtubllXWeXJSIiNYRCmsgpGGN47D/2Wzx1bBjEO3fEqfdMREQuK4U0kVP4ZPUBFmxPpW/rurw7rJ1mb4qIyGWnddJEfmf74Rxemb+TyCBv3vhjGwU0ERFxCoU0kRMUlJTz8OcbsdkM7w5rq+U1RETEaRTSRE7wwpzt7Esv4Jk+zYitH+jsckREpAZTSBP5r8/WJPPNhhR6NA3m3i4NnV2OiIjUcAppIsD6A1m8+MM2Imp58daQOKy6Dk1ERJxMIU1qvLTcYsZ+sh4Xq4VpI9pTy8fd2SWJiIhoCQ6p2UrLbTz46QbS8kp45444WoYHOLskERERQD1pUoOVVdh48qvNrDuQxX1dGjIgrp6zSxIREXFQT5rUSAUl5Yz9dAPLE9Lp1SKUP9/SzNkliYiIVKKQJjVOZkEp98yMZ/PBbIZ2qM9rt7fC1UWdyiIiUr0opEmNkpJVyMiP1rIvvYBxN0TzxM1NsFg0k1NERKofhTSpMXal5jLqo7Wk5ZXw0h9acPf1WgtNRESqL4U0qRHWJmVy78fxFJdV8M872vKH2HBnlyQiInJGCmly1ftl11HGfrIBV6uFGXd3pEtMHWeXJCIiclYKaXJVW38gk7GfbMDXw5WPR3ekVT2tgyYiIlcGTWmTq9a+9Hzu+3gdVouFGfdco4AmIiJXFPWkyVXpWH4Jd8+IJ7e4nA9GdqBNRKCzSxIRETkvCmly1UnLLeb+WetIzixkwsDW3NAsxNkliYiInDeFNLlqGGP4ct1BXpu/k9zich65MZphHSOdXZaIiMgFUUiTq8LBzEKe+WYLqxIzCPHz4O+DY+ndsq6zyxIREblgCmlyxSursDF6Zjx70vIZ1jGSP9/SjAAvN2eXJSIiclEU0uSKN2NlEnvS8nmsZwyP9Wzi7HJERESqhJbgkCtaak4x7yzeQ2SQN2O6N3Z2OSIiIlVGPWlyRXvtx50UlFbwbv8WeLq5OLscERGRKqOeNLlirUo8xtzNh+nZPIQbm4U6uxwREZEqpZAmV6SyChsvztmOh6uVF//Q0tnliIiIVDmFNLkivbckkT1p+Yzt0Zj6Qd7OLkdERKTKKaTJFWf74Rze/WUPzer6MbaHJguIiMjVSSFNriil5Tae+HIzAJMGx+LhqskCIiJydVJIkyvKu7/sYVdqHg/dEE2regHOLkdEROSSUUiTK8bmg9m8tzSRFmH+PHRDtLPLERERuaQU0uSKkJxRyIOfbsBqgX8MicXdVW9dERG5umkxW6n2ko4VcOf01aTmFjNpUCzNw/ydXZKIiMglp5Am1Vpiej53Tl9Nel4Jbw+NY0BcPWeXJCIiclkopEm1dSCjgDv+vZrMglL+Oawt/dqEO7skERGRy0YhTaql3OIy7v14HcfyS5h6Zzv6tg5zdkkiIiKXla6+lmqnwmZ45PON7E3L59lbmiugiYhIjaSQJtXOGz/tZOnudAa1j+C+rg2dXY6IiIhTKKRJtfLVuoNM/zWJ9lG1eO32VlgsFmeXJCIi4hQKaVJtHMou4sUfthMe4Mn7w9vrlk8iIlKjOTWklZSUMG7cOGJiYmjZsiXDhw8HIC0tjT59+hATE0OrVq1YsWKFM8uUy8AYwwvfb6OwtIIJf2xDsJ+Hs0sSERFxKqfO7vzzn/+M1WolISEBi8XCkSNHHI936tSJBQsWEB8fz6BBg0hMTMTVVZNRr1YLtqXyf7vS6B8bTvcmwc4uR0RExOmclnoKCgqYMWMGKSkpjuuOwsLss/i+/PJLkpKSALjmmmsIDQ1lxYoV9OjRw1nlyiWUW1zGiz9sx9/Tlb/2a+HsckRERKoFpw13JiYmUrt2bV599VU6dOhA165d+b//+z8yMjKw2WwEB/+vN6VBgwYkJyefcj+TJ08mIiLC8ZWfn3+5XoJUkb8v2E1aXgnP9m2uYU4REZH/clpIKysrY9++fbRo0YJ169YxZcoU7rjjDsrLy0+a0WeMOe1+xo8fT0pKiuPL19f3UpcuVWjJ7jQ+WXOAjg2CGNKhvrPLERERqTacFtKioqKwWq3cddddAMTGxtKwYUN27twJQHp6umPbAwcOEBkZ6ZQ65dL5ZddR/jRrPQFebkz4Y2usVi23ISIicpzTQlqdOnW46aabWLhwIWAPYklJSTRt2pTBgwczdepUAOLj40lNTaVLly7OKlUugUXbU/nT7PX4eLjw2X2daBysHlAREZETWcyZxhIvsX379jF69GgyMjJwcXHhxRdf5Pbbb+fo0aOMGDGCpKQk3N3dee+99+jevfs57TMiIoKUlJRLXLlcjAXbUhn32QYCvNz49P5raVbX39kliYiIOMWZcotTQ9qloJBWvaXlFnPDpKV4ubvy+f3XEhPq5+ySREREnOZMuUV3HJDLauKC3RSUVjBhYGsFNBERkTNQSJPLZmNyFt9sSKFrTB16Ng9xdjkiIiLVmkKaXBY2m+GluTtwsVp4oV8L3ThdRETkLBTS5LL4duMhNh/MZuR1URrmFBEROQcKaXLJ5ZeUM3HBLoJ83HmsZxNnlyMiInJFUEiTS27KL3tJzyvhyZubEuDl5uxyREScJ303fPsApG6run3abLDuI8jRygZXG4U0uaT2HyvgoxVJtAjzZ+g1uu2TiNRgyavhw5thyxfwyUDIOlA1+909H+Y9Dj+/WDX7q45St9rPXw2jkCaX1Kvzd1BaYeOl/i1x0W2fRKSm2jkPZg2AijLo8jjkp8Enf4TCzIvf92/2O/Swaz6U5F38/qqbQxvs4XbGLbDlq6rZp80G//cyTG4BbzaGCZHwegTMeahqfiZVRCFNLpllCeks3plGvzZhdGwY5OxyRGq2Ja/bh8SkapSXQOISsFWceTtjYPW/4MsR4OEH98yHni/BLRMhYw98NhRKCy+8jkPrIfk38K4D5UWwc+6pt7NV2IPizH7wRqT9z59ftD9WXnLhx69KRzbDgVWVH8vabz9Hxga+ofDdA7D164s7TnkpfHs//PoPcHGD4KZQry3UbgQbP4Gp18L27+w/OydzdXYBcnUqq7DxyrwdeLpZ+Uvf5s4uR8Q5SgvA3efi9nF4E2z+HK4dA0ENL2wfWfth2USwukL9ayG05cXVdLHiP4S10//3vcUK4W2hSW9ofIM9zFRnxtivK9vxPdz4PHR76tTb5aXCnHGw92cIagTDv7H/CXDtnyD3MKx82x4Yhn4CF7I00W/v2f8cPBNm324fSo27s3Kt6z6Cle9A9gGwukFYLBzeCPt/tW/TsDuM+A6sLud//KqSn2YPjiW5ENMbbn4VfOrAp4OhIB2GzLK/b2f2s58vgNaDzv84JXnwxQjYtwSa9YM/fgBuXvY2Y2Dzf2DhX+Cru6FpX/jDO+DrvHU9FdLkkpj12wH2puXzWM8Y6gV6Obsckcvv8Cb78EyTPjDw3/bf2M9XeQl8PRoyE2HdDOg8DrqMBw/f89vP1v8OEdnKYd54uOcnsJ5mIMUYyEoCiwt4+oOHPxTnQNoOSNsJBceg/d3gH3b+rwegohyWvmEPsAH17I+VFcCmT+xfVjeo29r+weniBq5ecM29ENPrwo53KayYbA9oFiss+zu0uB3qRFfeZscPMPdRKMqEuLugzxv283mim16E7GTY/i2snwEdRp/+mGk74ZdXoe1waHqL/bGcFHsdDbtDw672kLtrvj38+Yf/t47vYf54e09bt6ft59Kvrr1nLX2XPbxt+QKWvQk3/KXqztH5+vlFe0Br0BX2LITE/4OA+vb3Yp83oEV/+3Z3z4OZt9qDWtpOuP7Rk8/rqZQVQ8JP9t6z1K3Q/h649R+Vg6nFAnHDILqnPajtW2r/xcaJdO9OqXJJxwro989fCfR2Z/H47ni5O/G3M5GqVpRt/+A93iNyKhXl8MFNcGST/ftm/WDQDHB1t39fcAw2zLL/ph7S7PT7WfZ3WPLfD+bDm+DoNvALt38Yu3mBqwf4hUG7kf/rDfg9Y2BqRyjMgOb97WGg/xRoN+LkbVO3woK//K+H5XRqNYBRcyEwsvLjeUehKMs+7FZWDIH1ISCi8jaJS2D2bdD9Gbjh2f89nnUA9iyC3T/ZA2FFmf2rNB8wcMub0PH+M9d1JhVlFxaUfy9hoX34LaQF3Pyy/bqyqC728GCx2M/30gn2nkuvIHtPzPGAcSrFOfCv6+0/nzEroHbjk7fZ/B/7xICyQnswvHUydLgHfn7BHrLu/Aqa3GwPhl+OgF6vwPWP2Pc9pSOUF8O4deAbfPK+y4rhw172n/2I7+w9mWB/n6+dDvU7QqPuF3auCjPts1nzUyEszv6+OVVvYfJq+Kg3NL7J3tuYEm9/Hx76//buPK4KKv3j+OeyieCOigurCuYC4o57i5mVlaNZU2Zme7ZvM/2mmcqmZZoap2lfJxtbNS1bLSvXXMDcM5UUBERF0FRU1nt+fxzZFBT0wr3I9/168ULu+nBA7nOf85xzVkL/2+DCf5S/ffZW+Gg87PnVJp9nPwTdx0L2b/Z3JyvJJl9+jcA3wE4rb/gU8vbb8Rv2EAz7y+SmWwAAIABJREFU08krlzl7Kh4zF9MB61JrcguK+MMrS9m86wDv3dCfgZ1aujskqYt++RQWPAPXfHL8i3z2VvtuuG0cdDzXvqg5HDYx+n27rSIEdbLVAlefbHFkH7x1vv2jHxpvXyi7XnZ8grTsFftOfMAddsprwycQfSGMfgUS34KfXoD8g9CiI9z2U8UJ1t5keCUemoXbF2+HF6yaZqspR/aVv23rrjD2bQjuevzjZKyBN4ZB35vgvL/BS32hKB/u+BkCg+xtDu6G+U/axBEgZpyd4sk7ALkH7Itd6y42oczeCt/82f5cJn5uk9V92+H7x2xFqKyGLeCedeWnL+fcbvt+bk+EVlXYNzF7q53y2rvVjuf5f6+8CliZhDfh6wdsVS4gyH7fbWLhrIuhw9mVJ7jH2rPFJt9e3nDTfDv9/OW9djrx0pdsxezb/4MVr9np26s+sr+HJ5OyxE7jhfSBSXPB+2j1piAX5j5kE+vGbe0U4I9/t9PXQ+6HhLegcTBMXmHHpDAPnouyFajbfoKvH4SEN+CSF6D3xMqfP3srvD4MfP3h5oV2evaHv8PhLPDxhwmfQfiAyu+/9mP788fY2/s2tNOXh7PK365JiK34dR1t32gU/79942xb1Zu8vLQi6XRC1hbbL1bR/+OiQljzvu21zNl18jFuEwOxf7S/242DT377WuSSJO2qq67izjvvZODAgS4NztWUpLnXw5+u5/0Vqdw7PJq7h0e5OxzxJM4i+64/Zzc0aW+nulp3tS/+Ze1LgVcH2ySm701w8XPlr//wKtj8denXzcJsEpH9m00+igW2tr03va49cSWjqooK7LYJyYsgciikJdgKhX8z25MUf5t98f49zTYeB7a0Lzo+DeCzybDuIzuV5yywyVloP9trNuhuOP/x8s9ljK3WJH1rK1aRQ8vHkXvAPndhrk2M5j9tp2UueBL63lj+Re3bh2HZS3DDPPucG2bZKdTuY22i+evnsP0n25gdPhhGPg1tY088Fms+sMlWozbQbbTtMSvKg07n2+TE198mmaunw4gn7TQt2CTi2ShoHg63nqRaV9bhvfbnnrbcViVHPV/1CkdRAfynh51eDeljK1Y5mXBgh73eNwCiRthG/hMlVAVHbCKT/RtMmG2TOyhfrep0nh3f8MFw1YdVm4Yr9t1fYemLcM5fIWasTWTXfAAHd9rpzLFv2+/54G54/3LYtc7eb9Tz9s1CsS/uhp+n2aTx8zttD+KJpreL/fIZzJxoE9nCI/b/T/ytsHiq/b2eNLfiNwGbvrI9XgEtbKWsINfev2FzaNnZJuKBrSB9pa3QZm2x9wsbCCP+bhc+fPMnO40//BS2EMk/BCtetxW7Vp3t35SWUfZNTf4h+9Gwmb3OQ7kkSXv99dd55ZVX8PLy4o477mD8+PH4+/u7NFBXUJLmPp+vzeCuD1czqFMQ/7u+v7bckFLGwJf32BePY5V9ES8qtP0macttIncoC+5ZX/rOd+daeH2orUr1mgBbfzy6wq7Q/hFuGW17cbK22Nvu/sU+962Lj08G03+2L6wRg46Padd6Wx2KOt8mWcbAF3fZSlPfG+Gi52w1a90MWP6KreC17wOXvWQrClvm2mmbTsPt4zmLbCUn6XsYci/0nAA44O3hNs4bv4f2vUuff9NX8NHV9l3/2LdOPr7pK23i9ft26HG1rdg5HPZ5p3a1SdNda0qn494bY8cOwLuBTS7ixtvKUlWrj+s/sc3zpgiCY+CCJ0oTF7Av1v+Jtb1td6+1U73F39f5j9vktDoKcmHOZJsENWhqp0r73mgTiPSVtu8uZxeMfrX8Yo0Ns+GTSXDOw3aKq1j2Vpvsb/oaUpfaaeSrPoR2cRU/f3Gye94jtopVVvE0I9gexHHTql6dK1aYB2+cY6fwjNNe1jTUTvEOuKN871TuAZh1A+zdZqusZZ9r+1LbC4nD3ufWJcf/7ldm7l9s5S3+NvvGw7+J/T15/wqbaN04r3xlO2UJTB9jK6U3fFfxVO2x9qfDoufs/yVTBN5+NiG8I+H0F9nUUS6d7ly4cCEvv/wyixYt4tprr+X2228nPDzcJYG6gpI099iefYiL/rOYgAY+fH3XEFo1buDukMSTzH8aFv7DTk+Oet5OAe5Pg0XP2mmOoX+yL7qLnrM9WPGTIXwgfHxN+UrTR+Nh05dwy+KTV3vAJmJvnQchfeH6b0urCRlrbA9MUT784XWIvaL0PsmLbBWr4LCtksWMA78AWwXseB5cPaN0Ogrs9gkLnjq6V5XDvvBUNbna/YutzrSMhpsX2BfVXz61CUHBYbgjsWrTZVD6wp30nX2BPfevpf1fQ/8E5z5cetv96bD0JQiLt4lkdRciFEtebCuj3f5Q8crAJc/D94/CZS/bvrqZk2zl754Ntl+tuoyxW0x8+xf7+9PqLJvc7Esuvc3ge+0WF8XeHmF/3vf+Unn1bd0MuwrT4QVjXrdT2GWlLof/jrSVuOu/Pf57NQbm/c2+yRjx91Pvfdu1AT4eb6fye11rk94Trbh0Oo+vkDmd8EIPuyDh2LE4GWPsG5djE8z1n9jfreaR9mfduqv9nZl1tEdw0le2al0dezaXvqG58j37BqGecmmSlpWVxZtvvsmrr75KXFwca9as4fbbb+fPf/6zS4I9XUrS3OOm/61k3sbdfHCj+tDcoiDXVnxOVgVxOu279LJJRrHfvofsbdD/5uo9977tkLzQTp21jDo+hsS37eqytnG2ubpsf9Lhvbaqk7Ha9qn8+oWtiN00377DfnWATSju3QD7d8Brg6DzxXDVB1WP75uHYMWrtvrV7yZbnXvjbJtcBATZz394A2LHwbaFNkHz9rVbXvz6uU0iAVp1gRu+Bf+mFT9P+s/w+R1HG8B/qvp0XHEC2+VSuzBg7zbwDbQN57Hjqv59gp3amTYKMlbZ6a7UZbZv546V9mdT23L3w7+72/2tbvoRnou2larr557e4+YfhiX/tolzw+Z26jZmrO0P270Rbltqp9l2/Axvngtx18Dol0/8mGmJtsp3KBMG3nm0ktTUPtdrg2yv461L3DOO1bVquk32r3zPvsFwhRVvwHcPl28p8G5gK8aRQ079cfMPuy7GOsolSdqKFSt46aWXmD9/PhMnTmTy5Mm0b9+enJwcunbtSmpqqkuDPlVK0mrfypS9XP7aMi6OacvL43u5O5wzx3d/hbwc2yx8okpH6gpbFWrc1lafwgfY6cDi7Q2KFRXaXpa9W22fU/OI0uvSV8I7F9m+ojtXVW3aAux02htnl/bHNG5r+6caNDnadL7fVnaahdueqIoSl9wDtt9o+xKbmN00H9p0t9et/dhuXnn2XyDzF9g4x1ac2vWsWnxgx/CVeLta7bYltmKSsthWd8IH2qTm4E4YeJftbfH2s6vcQnrbykLGKkiaZ6cojx3TYxljKzu+1WgFKcy3jf2ZG21S0P9W+xFwihtA52TCm+fBwQzbA9f6LDtm7vL9Yzah6nqZ/fkVJ8uuUJhne/GKq01piXYKOXIoXPu5nY5dP8MmV21iTv54v6fZ6u3ONXbRw7A/2x60xDfLT8vXV4X59u9H8QrKjufaPkc5LS5J0mJjY7n77rsr7EV7/fXXueWWW04/UhdQkla7jDGMfXUp69L3M+++YUS2rJ89BS73eyo8f/RFpXVX+464ssTpi3vs6q+gKLvqEOyL/XVflyY7YPch+ul5++9m4baa0aSdrVS9cY5diWWcMOQBuwqwKopXzcVcYSsayQtLK09gX0CDomzl60RbVhQcsUe0hPaz0ynFigrhxV624pZ/0Pb7XP1x1WIrK2meTVAbNre9ZGUXJGRvPZqoZdhep2s/Ld8fVhv2pdgqXrc/VK/ZvDKZm+w0X95+u8dU/G2n/5in6uBu+7tclGf70x7YYhdV1JTP77T9Thc8bbeoCIu3FdyqchbZBR0/Pml/J8BWiSd97d7NXuWMpS04pMbM3bCLW9/7mQnx4fx9dPeT30GqZvFU+GGKnQLb9CX4NbYbonYeWf52ziL411l29dIdiTaZ2TLXrvDyb2YTsaCOsHkufHilbW7vOd5OC7WMhvGf2B6YXevtPl7f/Q0wtlm/7AuSs8j2SJWdqjyUZRMov0b2uYubfg9l2dv7N7HL8U93G4yV/7Xxgp0yO9UEataNtrk8bKDdOqJs31D2VruZZ/yt1avSebKUn+yihktfPPWqnKsUv5HoNNxOj9WkQ9n29zL3d/v1Hz84tX6n/MN2mjxpnl2IcaI3GSKn4UR5S5U3m7nooovIzs4u+TorK4tRo0adfnRSZxUWOfnnt5sI9PPmrvPqQJ+Gp8k/BJ/cYPf8KssY28jcsLlddn/NLNsc/OGVttpSVnqi7aHpcon9OqCFPRJm7Nu2MjZ9tJ0O/fQW+3jjptldzUc+Y1dAvtTXJmhn/wW6j7G7bR/YYStiZc26EZ7tZHe9L35f9/2jdjrzgifLr8oKbGlXY/o2dM0+ZT2utg3LXS45vQrXRc/aBQhXvnd8Y3dQR9swfqYkaGBXrf7xffcnaGAb2Ft2rp2KXmBQ6VYOzcJt9fVU+AXYVZzXz1WCJm5T5SQtIyODoKCgkq9btmxJRkZGjQQldcPHK9PYtucQNw3toNWc1VWQa/uwNnxip1Wyt5Zet3uDXYbfbYzdtqDjubafy+FtG6XLKj5MuThJK9b1UruB5e+p8N8RtqrwhzdKV9TF3wrn/s1OQXUbU7o1QY+r7OfV75c+VtL3dkVeUb7dRmPmdfay1e/Z/Zu6jnbZsFTI1x9uT4Bx757e4zRsbleKBgad/LbiWs3D7RYLxVuS1LReE+22Fcce+yNSx1Q5SXM6nRQWFpZ8nZ+fT15eXo0EJZ5v/+EC/j0viZaNGnDTEL3LLCdnjz1TrjKF+TDjWlut6nie3bJh4TOl16872nNVdluIllE2Edv6g+03gtLtCJqG2pWTx+o1wTY7g+0zix5R/vqhD9jkZ+xbpRWvoI52OnDTl7bRvjAPvnnQbvh56xI7/brxM3h/rG1Kv+g51+/qXxEfP73YStV5edsKryed9ylyCqqcpI0cOZI//vGPLFmyhCVLlnD11Vdz8cX1d1+T+u6JrzaSlZPHn0d2JrCBew+g9QhJ39s+sJf6wnOd7GrC9Z8cf7uiwqN7WX1rp/HGf2IrZetm2H2DnEWwfpbdRT+0f/n7xk+2n1e8aj/vWm83Lz1rVOWJ0sA74IGkyhcCtOp8fPLTc3zpTvZLX7BbQgx9EIK7wRX/s+cG+gbYJK8qx/qIiMgpqXKS9uSTTxITE8N9993HfffdR1xcHE8++WRNxiYeatGWPcz8OZ2h0a24vHfIye9QVyUvsntspa4o7cOqyM/TbGXp52l2GjP2Sntczpw77OaUxYoKYPaNdu+tbmPs7vReXnYndI4eypyyxK4oi7ni+MQrtJ/tyVr7kW2Ormyq81iNWlfv++56mU3Clr8Ki/5lV2cOOLr1gMMBfW+Ah9LsocYiIlJjtLpTquVQXiEj/r2I3w/n8+29Qwlp7iGbEP6eavc4Ch9Yvem376fA5m/sxqFhZSpXG+fYpn5ngf26ZWe7A3jc1eUbsX/51O6i3jwcxs8qPRw4LcHuO9Y0BG6eb5OemZNg81c2CRr7dvnm9fevsNW10Hh7JNLtCRWfNVe88/e5f7PH4+Rk2i0NXD0V+OltsPbohrETPrXVPhERcTmXbcExe/Zs1qxZQ25ubsll//znP08/QhdSklazHp2zgXeXbefxy7px7YAId4djGQOvDYHd6yFiiN38tez5e4VHd8j28St/vxVv2H4rsE355/0NBt5tk5PP77Rn1Y36t62orfvY7q/lGwi9J8KA2+1+YB/80e5af/1caBFZ/vGL9xDrdL5NHJO+s8cFjX7t+B3/M9bYDU3BHq9yy6KKv9eiAng+FgoO2ZWVva61Wyy4WspPMO0i24N25XTXP76IiAAnzluq3Ex0zz33sHXrVn7++WeuuuoqZs6cyfnnqymzPlmZspf/Ld9Ov4gWXNPfc85rJWWxTdCCOsH2n+wO+DGX2y0gdq61x8T4NoTzp0Cv6+wUY9I8mPtnaNHRTjt+ea/dGX3DLNvr1TQMrv3MNtKfdTEMn2Kra0tfsHtPJbxhN2r1C7SVpmMTNLCHP+9YVVqRirsGLn2h4qpXuzjbW7bpSzvVWRlvX7tb+w9T7NddLj3NwatExCC79cexfXEiIlJrqlxJi4mJYe3atfTs2ZO1a9eye/dubrzxRr744ouajrFaVEmrGQVFTka9sITkrEPMvWcIHVqd4oHMNeGj8bDpK7jzZ9vw/u3DsG2+va5RG3sQ9+5f7P5fEUNsFWzWTTZZuulHm4jlH7ZVtdXv2anNaz+zu/Efyxh7xuWSf9vHHP8JhPatPLaCI3aPsaCOcN5jxx+GXNa+7TYJPO+Rys+HBLth7dSuNkn801Z7ZqeIiNRJLqmk+fv74+XlhcPhoKCggODgYHbs2OGyIMWzvbs0hc27D3LXeVGelaDtS7EJWtSI0mOTJnxqV0o2bG43VQV7PuT3j9rd61MW2+0jJnxaeh+/AHuWY+/r7XYXlR3N43DYZf1R59uE7WT9b74N7YaiVdE83O7rdDIBLWDsm/bfStBERM5YVU7SGjduzOHDhxk8eDATJ06kTZs2+Pr6nvyOUuft2p/Lv+dtIbRFQyafXcWDt2tLwpuAsZuzFnM47KHSZfk3sf1l3cfC/KfsrvuRQ45/vJBq7GhfG/uDVeZkKzpFRKTOq/IWHB9++CE+Pj48++yzdOvWDS8vL2bOnFmTsYmH+PtXGzmUX8SUS7vh71uLG4rm7rdTl/srmb7Oy4FV06HVWdDhnKo9ZsRge1ByzOWui1NERKQGVKmSVlRUxAMPPMD06XaV18MPP1yjQYnnWJKUxVfrdjKiazDnnhVcu0+e8AYse8luZzHpm+NXRK79EPL2Q//H3FvVEhERqQFVqqR5e3ur/6weyi908sicDfj7evHIJV1r98mLCu1h3gDpCbDo2fLXO52w4jXwb2Y3jxURETnDVHm6c/jw4dx2220kJCSwcePGkg85c727NIVtWYe445xOtb9p7ZZv7GrMIfdDcAws+qfd+R/sNOis6yH7N7tPmF9g7cYmIiJSC6q8BUdk5PH7QDkcDrZt2+byoE6HtuBwjT0H8zj3uQU0DfDl+/uGubYXrTAPjNOufKzM/y6zRyTd+4s96PuNYfZ4o1HPw5f32BMGulwCo1+FBo1dF5uIiEgtcskWHMnJyS4LSDzfv77bzMG8Qp4dF+vaBG3vNph2CQQGwY0/Ht9nBpCVBNsWQLc/QOM29mPEE3b3/vfGgI+/PeS7z/XqRRMRkTNWlZO01NTUCi8PCwtzWTDiGTbs2M/HK9MY0CGIC7q1cd0D70uxCdqBdPux5j3ofd3xt0t8237ue1PpZX1vhPREyNoCl70CwbXcIyciIlLLqpyk9e7dG4fDgTGG3NxcDh8+TFBQEJmZmTUZn9QyYwxTvvgFB/DIJV1xuKpS9XsavHuJ7TO7eCos+Af8+CR0vxwalNkcN/8QrPkAWnWxh6UXczhgzBuuiUVERKQOqHKStmfPnnJfFx+2LmeWbzbsIjFlH9fEh9GlbSW77lfXwd3w7iibqF32MvQcb5OuL++1xyCd85fS266bYbfV6PeIpjJFRKReq/LqzmONGTOG+fPnuzIW8QDTfkrBz8eLe4dHu+5BF/3TTnVe/C+boAH0vNaekbn0RTiw0162cQ788Dj4Nda2GiIiUu9VuZJ2+PDhkn8XFRWxYsUKdu/eXSNBiXts25NDQspeLotrR1AjF50JmZNpDy1vG2cb/Yt5+8D5j8OHV8J3RzdH3jDLnrc59k2t2BQRkXqvyklao0aNSnrSvL296dSpEy+88EJNxia1bObPdgnwFX1CT+0BDmVBYMvyl614DQpzYch9x09fRl8AEUNscgbQ+WJ7vmbjWj7ZQERExANVOUlzOp01GYe4WWGRk1k/pxPaoiEDOgRV/wEWT4Ufpth9zPpMspflHoCEtyCoE5w16vj7OBxw0bO2N633dXaKU31oIiIiQDV60hITEzl48GDJ1wcPHmTlypU1EpTUvgWb95B5MI9xvUPx8qpmopT1m12tCfDV/fDbD/bfK/9rFwEMuge8KtlrrXUXuH4u9PijEjQREZEyqpyk3XLLLQQElB4N1LBhQ2655ZYaCUpq34yVaTgccHnvkOrd0Rh7AkBRHlz6kt1OY+Z1sGMVLH8FGrfTIgAREZFTUOUkzel04u1dWg3x8fGhsLCwRoKS2rXnYB4/bspkSFQr2jU7wVFNFVnzAaQsttOVvSbAle9DwRH470jI2Q0D7wAfvxqJW0RE5ExW5STNz8+PrVu3lnz922+/4evrWyNBSe2avSqdQqfhyuouGDiUZVdmBraG4Y/ZyyKHwKUv2Mpaw+bQa6KrwxUREakXqrxw4NFHH2Xw4MFcfPHFAHzzzTe8/fbbNRaY1A6n0zBjZRrNA3wZ3rV1+SsL8+CDK8CvEZzzcPmjmA7stA3/R/bB5e/YhKxY3NXgG2BXepY9TUBERESqrMpJ2sUXX8yiRYuYN28eAP/3f/9Hx44daywwqR3Tlqawdc8hbh3WkQY+xzT3L3vZHnQOsOkriL0CYsbZUwF+mQ3OQrtqs9sfjn/gbqNrPHYREZEzmcMYY6pyw9TUVFq3bo2/vz8AR44cISsri9DQU9xTq4aEhISQnp7u7jDqhG17crjohcUEN/Hnm7uHEOBXJmc/kAEv9rF7lv3hdbt6c+sPpddHDIH42yB6ZOUrN0VEROSETpS3VLmSdvnll7No0aLjLluxYsXpRSduUeQ0PPjJOvIKnTx7eY/yCRrAvEeh4BCM/AeE9oMJsyF5MaQsgS6joE2MewIXERGpJ6qcpOXn55dU0cBuwZGXl1cjQUnNe+enZH7evo9JgyLoF9mi/JXbl8H6GRB1gT0VoFjkEPshIiIiNa7KqzsdDgeZmZklX+/evZsqzpSKh9m6J4dnv91MRFAAf7rgrPJXOovgmwfByxdGPu2eAEVERKTqlbS77rqLwYMHc+2112KMYfr06Tz88MM1GZvUkOe/T7LTnON60NDvmH6y5a/ArvX2lIAgLQwRERFxlyonaZMmTaJDhw589dVXOBwO/vvf/zJ48OCajE1qwL5D+Xy7YRf9I1vQN+KYac5NX8N3f7NnbQ59wD0BioiICFCNJC0rK4tPPvmEtWvXcuTIEebPnw9AQkJCjQUnrvfZmh3kFzm5su8xq3J3rIJZN0BACxg/Exo0dk+AIiIiAlSjJ+36668nJCSEXbt28cgjj9C6dWsuuOCCk99RPIYxho8T02jcwIcLu7ctveL3VPjgSjBOuOojaNHBfUGKiIgIUI0kLTU1lT//+c/4+/tzySWXMHv2bJYuXVqTsYmLrd+xn027DnJpXLvSXrSCXJugHdoDY96w222IiIiI21Xr7E6ABg0asHfvXnx8fLRpbB3zcWIaQPmpziX/hsyNcM5foOtlbopMREREjlXlnrTOnTuzd+9errnmGuLj42natCk9e/asydjEhY7kF/H5mgzOatOYmPZN7YXZW22S1qoLDL7XvQGKiIhIOVVO0qZPnw7A3XffTZ8+fdi3bx8XXnhhjQUmrvXNhp0czCvkj31DcTgcYAx8/QAU5cGoqeDt6+4QRUREpIwqJ2llDRo0yNVxSA37ODENPx8vRvdsby/45VPY+iP0uBrCB7o3OBERETlOlXvSpO76LfMgK5L3ckG3NjQL8IPcAzD3/8C/GYz4u7vDExERkQqcUiVN6pY3FyUDMGlQhL1g2cuQswtGPQ+BLd0XmIiIiFRKlbQzXObBXD5dvYM+4c3pFdbc9qKtnwGN2kCva90dnoiIiFRCSdoZ7n9Lt5Nf5OSmoUc3qN39C+zdBl0vBS/vE99ZRERE3EZJ2hnscH4h05dvJ7JlIMO7BNsLN86xn7UnmoiIiEdTknYGm5GYxv4jBdwwOBJvr6Pbbmz8DAJbQdgAd4cnIiIiJ6Ak7QxVWOTk7Z+SaRHox9heIfbCPZsgawt0uURTnSIiIh5OSdoZ6ttfdpO29wgT4sNLz+ksmeoc7b7AREREpEqUpJ2hPkjYjp+3FxMGhJdeuHEOBARBuDYjFhER8XQekaRNmTIFh8PBhg0bAMjMzGTkyJFERUXRvXt3lixZ4uYI65bdB3JZujWbszu3omWjBvbCPVvsQepnjQJvbY8nIiLi6dyepK1atYrly5cTFhZWctlDDz1EfHw8SUlJvPPOO4wfP57CwkI3Rlm3fLE2A2MoPQIKtKpTRESkjnFrkpaXl8ftt9/OK6+8Yg/9PmrGjBncfvvtAPTt25fg4GBV06rhszU7aNzAh3PPal164cY59hioyKHuC0xERESqzK1J2iOPPMI111xDZGRkyWXZ2dk4nU5atWpVcllERASpqakVPsbUqVMJCQkp+cjJyanxuD3Zb5k5bNhxgJHd2+Dve3TBwO9psHs9dL4IvH3dG6CIiIhUiduStGXLlpGYmMjkyZOPu65sVQ3AGFPp49x3332kp6eXfDRq1MjlsdYlc9bsAOCyuDJTnSlHq5Adz3FDRCIiInIq3JakLVy4kE2bNhEZGUlERATp6elccMEFJCQkALBnz56S227fvr1cz5pUzBjDnDUZtG7cgAEdg0qv2P6T/Rw+0D2BiYiISLW5LUl76KGHyMjIICUlhZSUFEJCQvj222+58MILGTduHC+//DIAiYmJ7Nq1i8GDB7sr1DpjddrvpO49zCU92tkTBopt/wmahUPTEPcFJyIiItXikXsxPPPMM0yYMIGoqCj8/PyYPn06Pj4eGarnyD9MwKxrONtrIKPjyiS0B3baA9XjxrsvNhEREak2j8l8UlJSSv4dHBzMd999575g6qCizXM5a/8SHvTfQ9f2fyl6LjC1AAAfcElEQVS9omSqUxvYioiI1CVu3ydNXCM78RMAujk349i7rfSK4kUDEUrSRERE6hIlaWeCgiM0Tf+RAybAfr1+Zul123+CJiG2J01ERETqDCVpZ4CCLT/QwHmEGQ2vgMDWsO5jMAZyMiFri62iHbOtiYiIiHg2j+lJk1OXnTiTNoBfj7HgCIDlr8COn2F/mr2B+tFERETqHCVpdV1hPk1Tv2edM5IhfXtDfmObpBVX0wAitH2JiIhIXaMkrY4r+G0+DZ05rAocw3UtA8HEQcto2DALAlpCozbQooO7wxQREZFqUk9aHZe5YgYAfjF/sBc4HBB7JRzOhqzN6kcTERGpo5Sk1WVFhTRNncevzlAG948vvTxmXOm/1Y8mIiJSJylJq8Pyty2mUdF+VjcaSlhQQOkVzcMh7Og5nepHExERqZPUk1ZXGcO+ha8SDPh2v+z460c+bTeybRld66GJiIjI6VOSVlcteo7g9G+ZV9SbAfEVVMvaxdkPERERqZM03VkXrf8E5j/BRhPB/9r+lZAWge6OSERERFxMSVpds30ZfHYbhxu05vq8+7mkb5S7IxIREZEaoCStLjmyDz66Grx8eSTwb+z3bc1FsW3dHZWIiIjUACVpdcnW+XBkL1kD/sInGUFc2L0NjRqorVBERORMpCStLklZDMDsg10BuLx3iDujERERkRqkJK0uSV6MaRrKOxsN7Zs1JL5DkLsjEhERkRqiJK2uOLATspPIDOrLzv25jO3VHi8vHfckIiJyplKSVlekLAFg3hG7Oe2YXprqFBEROZMpSasrUhYB8FZaCH0jmhPRUnujiYiInMmUpNUVyYs5FBBKSmELRvds7+5oREREpIYpSasL9qfDvmTW+MTgcMCIrm3cHZGIiIjUMCVpdUGy3Xpjzu8d6B3WnFaNG7g5IBEREalpStLqgqP7oy3I78IF3VRFExERqQ+UpNUFyYvJ9Aslk+ZK0kREROoJJWmebl8K7E9lUf5ZnNWmMWFBAe6OSERERGqBkjRPl1w81XmWqmgiIiL1iJI0T5ds90db7uyqJE1ERKQe8XF3AHICBbmYLXPZ7OhIwxZt6NK2sbsjEhERkVqiSpon2/oDjrwDzM7vxwVd2+Bw6KxOERGR+kJJmifbMBuAr4riGaGpThERkXpFSZqnyj8Mm7/hV+/OHAlsT+/w5u6OSERERGqRkjRPlfQdFBxiRm4/hkS1xNtLU50iIiL1iZI0T/XLbAwOviqKZ0hUK3dHIyIiIrVMSZonysuBLd+xNSCWTJozNKqluyMSERGRWqYtODzRlrlQeIRPnH05q01jWjfxd3dEIiIiUstUSfNEG2ZjHF58crg3Q6M11SkiIlIfKUnzNLn74bd57GjahyyaMlT9aCIiIvWSkjRPs30pFOXznemLv68XfSK09YaIiEh9pCTN0+xYBcDnWe2I7xCEv6+3mwMSERERd1CS5mkyVuH08uWXolBtvSEiIlKPKUnzJMbAjlVk+EdRgA/DorX1hoiISH2lJM2T/L4djuxlZUEE7Zr607FVI3dHJCIiIm6iJM2THO1HW3wojKHRrXA4dBSUiIhIfaUkzZNk2CRtremgfjQREZF6TkmaJ9mxmlxHQ1Jox+BO6kcTERGpz5SkeQpnEWbnGtabSGJCW9A0wNfdEYmIiIgbKUnzFFlJOPJzWFUYqVMGREREREmaxyjuR3N2ZKi23hAREan3lKR5iqMrO7f6RdMjpJmbgxERERF383F3AGIVpv/MftOYDp264OOt3FlERKS+UzbgCQrzcexezzpnB4ZEt3Z3NCIiIuIBlKR5gsxf8HYWsM6oH01EREQsJWkewBztR9vduCshzQPcHI2IiIh4AiVp7mQMZKzmyOqZADTvFO/mgERERMRTaOGAOxQVwA9TYP0sOJhBALDG2ZHe3aLdHZmIiIh4CCVp7rBxDix9EZpHwIA7eGpbB/6X3oZVHYLcHZmIiIh4CCVp7rBtgf088UsKGrfn/SnfERvWlAA//ThERETEUk+aOyQvhBYdoFkoG3bs51B+EfGqoomIiEgZStJq274U+D0VIocCsHzbXgDiO7RwY1AiIiLiaZSk1bZtC+3nyGEALN+WjZ+3F73CmrsxKBEREfE0StJqW3JxkjaUgiInK1P20jOsGf6+3u6NS0RERDyKkrTaZAwkL4Lg7hDYUv1oIiIiUiklabUp81c4tKfMVGdxP5qSNBERESlPSVptKp7q7FCmH83Hi55hzdwYlIiIiHgiJWm1KXkROLwhbEBpP1qo+tFERETkeErSaktRIaQsgfa9wb+J+tFERETkhJSk1ZadayDvQMlU57Jt2YD60URERKRiStJqS5mtN8AuGlA/moiIiFRGSVptSV4MPv4Q0k/9aCIiInJSStJqgzGQsRra9gBff9bv2M/h/CL6a6pTREREKuG2JC03N5fRo0cTHR1NXFwcI0eOJCUlBYDMzExGjhxJVFQU3bt3Z8mSJe4K0zX2p0Hu79AmFoCEZJ3XKSIiIifm1krazTffzObNm1mzZg2jRo3i5ptvBuChhx4iPj6epKQk3nnnHcaPH09hYaE7Qz09u9bbz21iAJuk+Xo76Bmq8zpFRESkYm5L0vz9/bnoootwOBwAxMfHs23bNgBmzJjB7bffDkDfvn0JDg6u29W0MklakdOQmLKXmPZNaeinfjQRERGpmMf0pL3wwgtccsklZGdn43Q6adWqVcl1ERERpKamujG607Rznd3EtnVXNu86yMHcQvpFqh9NREREKucRSdpTTz1FUlISTz75JEBJda2YMabS+06dOpWQkJCSj5ycnBqN9ZTsWg+tOoOvPwnJdn+0/pHqRxMREZHKuT1Je+6555g9ezbffPMNAQEBBAXZCtOePXtKbrN9+3bCwsIqvP99991Henp6yUejRo1qJe4qO7wX9qeWLBpITNmHwwG9wtWPJiIiIpVza5I2depUPvzwQ+bNm0ezZqWbuo4bN46XX34ZgMTERHbt2sXgwYPdFebp2b3Bfm4TgzGGFcl76dKmCU0b+ro3LhEREfFoPu564vT0dO6//346dOjAOeecA0CDBg1YsWIFzzzzDBMmTCAqKgo/Pz+mT5+Oj4/bQj09ZRYNJGcdIisnj1Gxbd0bk4iIiHg8t2U+ISEhlfaaBQcH891339VyRDVk5zr7uU0MCRvs/mjqRxMREZGTcXtP2hlv13poGgoBLUhIsUlanwglaSIiInJiStJqUkEuZG0ut4lth1aBtGrcwM2BiYiIiKdTklaT9vwKzkJoE0vG70dI33dEU50iIiJSJUrSalKZRQOJR6c6+2qqU0RERKpASVpNKk7S2say4uih6v1USRMREZEqUJJWk3auA/+m0DSU5Vuzad+sISHNA9wdlYiIiNQBStJqitNpN7JtE0v670fYlnWIwZ1aujsqERERqSOUpNWUfcmQnwNtYlmSlAXAkGglaSIiIlI1StJqys619nObGBYnZeFwwKCOStJERESkapSk1ZSM1QAUtY1jyW9ZxLZvSvNAPzcHJSIiInWFkrSakrEafANZn9ua/UcKGBylKpqIiIhUnZK0muB02unOtj1Y/JvdemNIVCs3ByUiIiJ1iZK0mrB3G+QdgPa9WJyURYCfN73Cmrs7KhEREalDlKTVhIxVABxpFcOq1H0M6BCEn4+GWkRERKpOmUNNOLpoYFVBJIVOwxD1o4mIiEg1KUmrCRmroUFTvtvZEIAh0epHExERkepRkuZqziK7aKCdXTTQvllDOrQMdHdUIiIiUscoSXO1rC1QcJgDLWJLjoJyOBzujkpERETqGCVprrbDLhr4hQ4ADFI/moiIiJwCJWmudnTRwJLDoQD0DtfWGyIiIlJ9Pu4O4IyTsRoatmDBLn9aNfaiXVN/d0ckIiIidZAqaa5UVAC71lPUNo5Nu3PoGdpM/WgiIiJySpSkuVLmRijKY1ejrhQ5DXFhzdwdkYiIiNRRStJc6Wg/2nqnXTTQM1T9aCIiInJqlKS50tEkbUFOe7wcEBvS1M0BiYiISF2lhQOutGMVNApmwQ4fooMDCGyg4RUREZFTo0qaq+QegN0byG3Tm10H8+ipfjQRERE5DUrSXCU9AYyT5IBYQP1oIiIicnqUpLlK6nIAlhd2BlAlTURERE6LkjRX2b4M/BrxbXZrGjfwoWOrRu6OSEREROowJWmuUJgHO1biDOnLmoyD9AhthpeXNrEVERGRU6ckzRUy1kBhLnua9yS3wElcqKY6RURE5PQoSXOF1KUArPPqCqgfTURERE6fkjRX2L4MvHz54WAYgCppIiIictqUpJ0upxPSlkO7OFZm5BLaoiFBjRq4OyoRERGp45Skna49v0LufvLbx7N1Tw6xIaqiiYiIyOlTkna6ttt+tOSAWIyBmPY6r1NEREROn5K005W6DICEok4AxCpJExERERdQknY6jLGLBlp1YWWm3Retm5I0ERERcQElaafj91Q4mAHhA1i/Yz8RQQE0bejr7qhERETkDKAk7XSkJQBwpG1fkrMO0V1VNBEREXERJWmnI/MXALY4OmjRgIiIiLiUkrTTsXsjePmy8mBzAGJClKSJiIiIayhJOx2Zv0KrzqzNOAyg6U4RERFxGSVppyr3AOxPhdZd2HB00UATfy0aEBEREddQknaq9mwCILdFZ7ZlHSJGJw2IiIiICylJO1WZGwFI9goHIKZ9E3dGIyIiImcYJWmnardN0lbntgUgpr0qaSIiIuI6StJOVeZG8GvEsuxAALqpkiYiIiIupCTtVGX+ahcNZBwgsmWgFg2IiIiISylJOxU5mXA4i/ygziRnHdImtiIiIuJyStJOxdFFA+m+kYBOGhARERHXU5J2Ko4uGlib1w6AHqFaNCAiIiKu5ePuAOqko5W077Nb4OdjiNVxUCIiIuJiqqSdisxfMYGtmJ8OPUOb4e/r7e6IRERE5AyjJK26nE7I/JWcJlEczi+if2QLd0ckIiIiZyAladW1PxUKDpHibU8a6BcZ5OaARERE5EykJK26Mn8FYFVuW3y8HPQK16IBERERcT0ladW1+xcA5mW1ICakKQF+WnshIiIirqckrbqOVtJW57aln/rRREREpIYoSauuzI0c9G/HIRpq0YCIiIjUGM3VVYcx0DaOlXlHcDigT4SSNBEREakZqqRVh8OBGf0KDx4aT9e2TXSouoiIiNQYJWnVtHXPIbJy8tWPJiIiIjVKSVo1JSTvBVA/moiIiNQoJWnVtCI5G4C+6kcTERGRGqQkrRqMMazYtpeo1o0IatTA3eGIiIjIGUyrO6vBGPj76O4UFjndHYqIiIic4ZSkVYOXl4Pzuwa7OwwRERGpBzTdKSIiIuKBlKSJiIiIeCAlaSIiIiIeyGOTtKSkJAYOHEh0dDT9+vVj48aN7g5JREREpNZ4bJJ2yy23cPPNN7Nlyxb+9Kc/ccMNN7g7JBEREZFa4zDGGHcHcazMzEyio6PJysrCx8cHYwxt27Zl+fLlREREnPC+ISEhpKen106gIiIiIqfhRHmLR1bS0tLSaNeuHT4+docQh8NBWFgYqampx9126tSphISElHzk5OTUdrgiIiIiLueRSRrYxKysygp+9913H+np6SUfjRo1qo3wRERERGqURyZpoaGhpKenU1hYCNgELS0tjbCwMDdHJiIiIlI7PDJJa926NT179uS9994DYNasWURERJy0H01ERETkTOGRCwcANm/ezHXXXUd2djZNmjTh3XffpVu3bie9nxYOiIiISF1xorzFY8/u7Ny5M8uWLXN3GCIiIiJu4ZHTnSIiIiL1nZI0EREREQ+kJE1ERETEAylJExEREfFAStJEREREPJDHbsFxqho0aECrVq1q9DlycnJ0skEFNC6V09hUTONSOY1NxTQuldPYVMzTx2XPnj3k5eVVeN0Zl6TVBu3FVjGNS+U0NhXTuFROY1MxjUvlNDYVq8vjoulOEREREQ+kJE1ERETEA3k/9thjj7k7iLpowIAB7g7BI2lcKqexqZjGpXIam4ppXCqnsalYXR0X9aSJiIiIeCBNd4qIiIh4ICVpIiIiIh5ISVo1JCUlMXDgQKKjo+nXrx8bN250d0hukZuby+jRo4mOjiYuLo6RI0eSkpICQGZmJiNHjiQqKoru3buzZMkS9wbrJlOmTMHhcLBhwwZA4wKQl5fHHXfcQVRUFN26deOaa64BNDbffvstvXv3pmfPnnTv3p13330XqJ/jctdddxEREVHu/w6ceCwOHz7MVVddRadOnYiOjmb27NnuCL1GVTYu119/PZ07dyYuLo6hQ4eyZs2akuvqw7hA5WNT7N1338XhcPDll1+WXFanxsZIlZ1zzjnmnXfeMcYYM3PmTBMfH+/egNzkyJEj5quvvjJOp9MYY8yLL75ozj//fGOMMZMmTTKPPvqoMcaYhIQEExYWZgoKCtwVqlv8/PPPZuTIkSYsLMysX7/eGKNxMcaYe+65x9x5550lvzcZGRnGmPo9Nk6n07Ro0cKsXbvWGGNMcnKyadCggTlw4EC9HJeFCxeatLQ0Ex4eXvJ/x5gT/45MmTLFTJw40RhjzLZt20xwcLDZu3dvbYdeoyoblzlz5pSMwxdffGGioqJKrqsP42JM5WNjjDFpaWlmwIABJj4+3nzxxRcll9elsVGSVkW7d+82TZs2LfkP4XQ6TXBwsElOTnZvYB4gMTHRdOzY0RhjTGBgoMnMzCy5rm/fvmb+/Pluiqz25ebmmvj4eLNt27ZyfzTq+7jk5OSYpk2bmoMHDx53XX0em+IkbeHChcYYY9auXWvatWtn8vLy6vW4HPuCe6Kx6Nq1q0lISCi5bty4cSVvps80FSUixfbs2WP8/PxMUVGRMaZ+jYsxFY/NhRdeaJYvX26GDRtWLkmrS2Oj6c4qSktLo127dvj4+ADgcDgICwsjNTXVzZG53wsvvMAll1xCdnY2Tqez3LFcERER9WqMHnnkEa655hoiIyNLLtO4wNatWwkKCuKJJ56gT58+DBkyhB9++KHej43D4WDGjBmMGTOG8PBwBg8ezLvvvsvBgwfr9biUdbLfkdTUVMLDwyu8rj75z3/+w0UXXYSXl31Zr+/j8uqrr9KtWzf69+9/3HV1aWx83B1AXeJwOMp9bbR7CU899RRJSUm89tprHDlypF6P0bJly0hMTOQf//jHcdfV53EBKCgoYNu2bXTt2pV//OMfrF27luHDh7Nhw4Z6PTaFhYU8/fTTzJkzh0GDBpGYmMjo0aNZt25dvR6XY51sLMpeXx/H6b333mPGjBksXry43OX1dVySk5N58803+emnnyq9TV0ZG1XSqig0NJT09HQKCwsB+0NNS0sjLCzMzZG5z3PPPcfs2bP55ptvCAgIICgoCLCHxRbbvn17vRmjhQsXsmnTJiIjI4mIiCA9PZ0LLriAhIQEoP6OC0B4eDheXl6MHz8egB49ehAZGcmvv/4K1N+xWbNmDRkZGQwaNAiAvn370q5dO9atWwfU33Ep62R/V8LCwkoWLh17XX3w8ccfM2XKFObNm0fr1q1LLq/P47Js2TIyMjLo0qULERERLF++nBtuuIE333wTqGNj46551rpo2LBh5RYO9O/f370BudG//vUv06tXr+OaLSdOnFiuwTc0NPSMb3auTNkeCY2LMeeff7756quvjDHGpKSkmJYtW5qMjIx6PTa7du0yjRs3Nps2bTLGGJOUlGSaN29u0tPT6/W4HNtfdKKxePTRR8s1gbdu3dpkZ2fXdsi14thx+fjjj02nTp1MSkrKcbetT+NizIn79Y7tSatLY6MkrRo2bdpk4uPjTVRUlOndu7fZsGGDu0Nyi7S0NAOYDh06mB49epgePXqYfv36GWPsi875559vOnXqZLp27WoWLFjg5mjdp+wfDY2LMVu3bjXDhg0z3bt3Nz169DCzZ882xmhsPvjgA9O9e3cTGxtrYmJizIcffmiMqZ/jMnnyZNO+fXvj7e1tgoODSxYknWgscnJyzBVXXGE6duxooqKizMyZM90Vfo2pbFx8fHxMSEhIyd/hHj16mKysLGNM/RgXYyofm7KOTdLq0tjoWCgRERERD6SeNBEREREPpCRNRERExAMpSRMRERHxQErSRERERDyQkjQRERERD6QkTURERMQDKUkTkVrhcDjIycmp1ed89NFH6dKlS4Xn902bNo0tW7ac8mPHxcVx5MiRk97uxhtvPO64nrpiwYIF9OnTx91hiNRbOrtTROqMwsJCfHyq/mfrn//8J6mpqeUO5y42bdo0WrZsSXR0dIX3LSoqwtvbu9LHXrNmTZVieOutt6oWrIjIMVRJExEcDgfPPPMM/fv3JzIyknfeeafkuoiICDZs2FDydZ8+fViwYAEAZ599Ng8++CBDhw4lNDSUZ599lo8++oiBAwcSHh7ORx99VO55nnvuOQYNGkR0dDQffvhhyeWJiYmce+659OnTh169ejFr1iwAUlJSaNmyJY8//jhDhgzhxRdfPC72uXPn0qtXL2JjYxk2bBgbN24EYODAgeTm5nLeeedx1113lbvPW2+9xcqVK7nrrruIi4vj66+/Ztq0aYwcOZJrr72WPn36kJCQwNSpU+nbty89e/akX79+rFixotyYFVcGIyIimDJlCgMHDiQyMpInnnii5HZnn302X375JQDXXXcdkydPZvjw4URHRzNmzBjy8/MB2L9/P2PHjuWss87i3HPPZcKECTzwwAMV/rxONl4PPPAA/fv3p1u3bvz4448l95s+fToxMTHExsZy8cUXs2PHjpLrnnnmGWJiYujRowfx8fEcPnwYsInx5MmT6dGjB926dWPlypWAPUtzxIgRJY83adKkCmMVkdPg7iMPRMT9APP8888bY4zZuHGjadSoUcnZiMeeide7d28zf/58Y4w9buWKK64wRUVFZseOHcbf3988/PDDxhhjVqxYYdq2bVvuOR577DFjjD0iKigoyKSmppp9+/aZnj17moyMDGOMMXv27DFhYWFm586dJjk52QDm/fffrzDu3bt3m6CgILNu3TpjjDHvvfee6datW7nnPHjwYIX3PfaomHfeeccEBgaaLVu2lFyWmZlZ8u9ly5ZV+tjh4eHmnnvuKblPkyZNTHp6+nHPM3HiRDNgwABz+PBhU1hYaAYOHGg++OADY4wx9913n7nhhhuMMcbs3bvXREREmPvvv/+4uKsyXtOmTSuJOTg42OTk5Jj169eb4ODgkrieeOIJc9FFFxljjJk2bZqJj483+/fvL3n+wsJCM3/+fOPj42MSExONMca8+uqrZsSIEcYYY6ZOnWpuuummkrg89exDkbpMlTQRAWD8+PEAdOnSBR8fH3bt2lWl+40bNw4vLy/atWtHy5YtGT16NAC9e/dm586d5Obmltz2xhtvBKBDhw4MHjyYxYsXs3TpUrZt28aFF15IXFwcw4cPxxjD5s2bAfD39+eqq66q8LlXrFhBXFwcMTExJd9Deno6O3fuPKUxGDx4MFFRUSVfr169mmHDhtG9e3duvfVWNm7cWFL5Olbx+LVq1YoOHTqQnJxc4e3GjBlDw4YN8fb2pl+/fmzduhWA+fPnl1SjmjdvXjKOxzrZePn5+TFhwgQA4uPjadOmDWvXrmX+/PmMGjWK9u3bAzB58mR+/PFHjDF8+eWX3HbbbTRp0qTk+Yunejt37lzSlzZgwICSeOPj45k7dy73338/n3/+OYGBgVUZYhGpBvWkiQhgk6Fi3t7eFBYWAuDj40NRUVHJdWWTroruV/x18Yt88eNUxOFwYIwhNjaWRYsWHXd9SkoKgYGBOByOCu9vjKnwuspufzKNGjUq+Xd+fj5jx45lwYIF9O7dmwMHDtC0aVPy8/Px8/M77r6VjV9Vb1fZ93Ksk41XRYrHuezjV3WMKot3wIABrFmzhu+//55Zs2bx17/+ldWrV5+wj09EqkeVNBE5oY4dO5b0YiUkJJRUbE7Ff//7X8AmE0uWLGHw4MEMHDiQpKSkcr1Ta9asqbRiVVZxovDrr78C8NFHHxESEkKbNm1Oet8mTZqwf//+Sq/Pzc2loKCA0NBQgAr74VzpnHPO4d133wXg999/Z86cORXe7mTjlZ+fz/vvvw/Yn9euXbuIjY3lvPPO4+uvvy6pkL722mucd955OBwOLr30Ul599VUOHDhQ8vxlE/OKJCcn06hRI6644gpefPFFtmzZUuurd0XOdKqkicgJPfnkk0ycOJG3336bXr160a1bt1N+rAYNGjBo0CD27NnDiy++WJIAffHFFzz44IPce++9FBQUEBYWxmeffXbSx2vVqhXTp09n/PjxFBUV0axZM2bMmFGlWG6++Wbuv/9+nn32WZ566qnjrm/SpAmPP/44/fr1IywsjEsvvbR632w1PfLII0yaNImuXbsSERHBoEGDaNq06XG3a968+QnHKygoiN9++43+/fuTk5PDBx98QGBgIN26dePpp59mxIgRAISGhvLGG28AMGHCBDIyMhgwYAC+vr4EBATw/fffnzDeBQsWMHXqVLy9vSkqKuLZZ5+tMF4ROXUOY4xxdxAiIvVdQUEBRUVF+Pv7c+DAAQYPHszUqVMZPnx4lR8jJSWFPn36kJWVVYORikhtUSVNRMQD7Nu3jwsvvJCioiKOHDnC+PHjq5WgiciZR5U0EREREQ+khQMiIiIiHkhJmoiIiIgHUpImIiIi4oGUpImIiIh4ICVpIiIiIh5ISZqIiIiIB/p/+fFH4nGrsG0AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 720x560 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig=plt.figure(figsize=(9, 7), dpi= 80, facecolor='w', edgecolor='k')\n",
    "\n",
    "plt.plot(history['accuracy'], label = 'training accuracy')\n",
    "plt.plot(history['val_accuracy'], label = 'testing accuracy')\n",
    "\n",
    "plt.title(\"VGG-19 model metric: accuracies\")\n",
    "plt.xlabel(\"number of training epochs\")\n",
    "plt.ylabel(\"accuracy\")\n",
    "plt.legend(loc = 'best')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlsAAAHoCAYAAAB+cyo3AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAAMTQAADE0B0s6tTgAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOzdd3xUZdYH8N+k94SE9IRQQm+R3gSU4opt174KYoNFdF8Lurou67u4rIW17VoRVgVFXnVZdVERRcFCC8VAIBAghZBGQkjvydz3jzN3ZjKZmcwkM5mU3/fz4XMzM3fuPDOUOZznPOfRKIqigIiIiIicws3VAyAiIiLqyRhsERERETkRgy0iIiIiJ2KwRUREROREDLaIiIiInIjBFhEREZETMdgiIiIiciIGW0TU7cTFxeG9996z+fz+/ftj/fr1zhtQB915551YuHChw67X1d8vUW/DYIuok1x11VW4+eabzT722GOPYdSoUfrbJSUleOSRR5CYmAhvb2/ExMRgzpw5+Oijj6DVals8d+PGjZg2bRoCAwMRFBSEsWPH4qmnnkJRUZHV8fz+97/H2LFj4eHhYfaLvr6+Hk8++ST69+8PX19fzJo1C8ePH2/HO++dVq5cidmzZ9t07j/+8Q+8/vrrzh0QEbkMgy2iTrJ48WJs3boVZWVlLe7XarX48MMPsXjxYgBAUVERJk6ciB07dmDNmjVITU3Fl19+iTvvvBMvv/wycnJy9M994IEHsHz5clx33XX4/vvvcfjwYTz//PPIz8/Hv/71L6vjcXNzw/LlyzF37lyzj//pT3/CRx99hHfffRdHjx7F2LFjMW/ePFRWVnbwkyBVY2MjFEVBcHAwgoODXT0cInIWhYg6RV1dnRISEqK8/fbbLe7fvn274u7uruTl5SmKoih33323kpCQoNTW1pq9jlarVRRFUX788UcFgPLVV19ZPa8tixcvVm6//fZW90dHRyvvvfee/nZzc7MSERGhvPXWWxavlZCQoKxZs0b5zW9+o/j6+irDhw9XkpOTlaNHjyqTJk1S/P39lQULFiglJSX651RVVSn33HOPEhISovj7+yvXX3+9UlhYqH+8vr5eWbJkieLv76/ExcUpGzduVGJjY5V3331Xf05GRoZy9dVXK/7+/kp0dLRy//33K9XV1S3GtW7dOoeOu6mpSVm5cqUSGxurBAQEKLNmzVKOHDmiKIqivPvuuwqAFr+ysrKUnTt3KgCUbdu2KSNGjFDc3d2V4uLiVr8HVVVVyv33369ERkYqPj4+yiWXXKLs27fP4vjNvR/j97t//35lypQpipeXlxIXF6c8//zz+se0Wq3yxBNPKDExMYq3t7cyYMAA/e9xbW2tcu+99yrh4eGKj4+PMnToUOXTTz/VP/fQoUPKrFmzFB8fHyUhIUF56qmnlMbGxjavS9TbMLNF1Em8vb1xyy23YOPGjS3u37hxI+bNm4eYmBhotVps2bIFy5cvh4+Pj9nraDQaAMDHH3+M4cOH48orr7R6XnvV19fD19dXf9vNzQ3e3t7Ys2eP1ee98MILuP7665GSkoKhQ4di0aJFePDBB7FmzRr8/PPPOHXqFP72t7/pz3/44Yfxww8/4PPPP8ePP/6IvLw8LFq0SP/4s88+i61bt+I///kPvvjiC/zrX/9CSUmJ/vGGhgZcccUVGDx4MA4dOoTPP/8cBw4cwIoVK+x6v/aOe9WqVfjqq6+wefNm/PLLL5g+fTrmzZuHiooK3HLLLXjooYcwdepUFBQUoKCgAPHx8S2eu27dOqSmpiIoKKjVWJYuXYodO3Zg48aNOHbsGP70pz/pp4937doFjUaD7Oxsm95XZWUlFixYgJEjRyIlJQVr1qzBqlWr8OGHHwIAPvnkE3z44Yf4+OOPkZ6ejn/961+IjIwEAPzzn//EoUOHsG3bNqSlpeHll1/Wj7ekpATz5s3DggULkJqaivfeew8ffvghXnzxxTavS9TruDraI+pNdu/erWg0GiUzM1NRFEWprKxU/P39lc2bNyuKoiiFhYUKgBbZg7KyMsXf31//64MPPlAURVF+9atfKdddd12L699666368371q1/ZNCZLma2bb75ZmTBhgpKTk6M0NjYqL7/8sgJAmT9/vsVrJSQkKPfdd5/+9t69exUAyieffKK/79lnn1XGjRunKIqiVFRUKB4eHsqXX36pf/zEiRMKAOXYsWOKoihKRESE8uabb7Z6XM1sbdiwQRk/fnyLcezevVvx8vJSmpqa9ONqK7Nlz7hra2sVX19fJTU1tcV1Bg8erLz//vuKoijKn/70J2XWrFktHlczW7t27Wpxv/HvQUZGhgJAOXDggNmx7t+/Xxk6dKiSm5tr9f2o7/fNN99UYmJi9BknRVGUxx9/XJkwYYKiKIrywgsvKHPmzDGbCX3ggQeUu+++2+xrrFq1Srnhhhta3Ldp0yZl0KBBbV6XqLdhZouoE02bNg2JiYl4//33AQBbtmyBu7s7rrvuOgCAoiitnhMYGIiUlBSkpKQgMDAQjY2NFs99+eWXkZKSgsWLF6O2thYA8MwzzyAgIED/y1b/+Mc/EBoaioSEBPj4+GDr1q341a9+BTc36/9sjB49Wv+zmskYOXJki/uKi4sBAJmZmWhqasKUKVP0jw8bNgwhISFIT09HeXk5ioqKMGnSpBaPBwYG6m+npqbiyJEjLd7jvHnz0NDQgLy8PJvfrz3jzsjIQG1tLaZMmdLidTMyMpCZmdnma11yySUWHzt+/Dj8/f0xYcIEs49PmjQJJ0+eRGxsrE3vKz09HePHj4eHh4f+vqlTpyI9PR0AcMMNNyAtLQ3Dhw/XZxlVixYtwr///W+MHz8eTz75JA4dOqR/LDU1Ff/9739bvP977rkH2dnZ0Gq1Vq9L1Nt4tH0KETnSHXfcgQ0bNuCpp57C+++/j5tvvlk/XRcREYGgoCD9FyEg03eJiYkAAHd3d/39gwcPxo4dO1pcOyoqCgDQp08f/X3Lli2zuArSmqioKGzfvh3V1dWoqqpCZGQkpkyZgnHjxll9nqenp/5ndSrT9D51SsxcwGhMfdzalGhVVRVmzpyJtWvXtnosOjra6vXbO+6qqioAMqUXEhLS4jqhoaFtvpafn5/FxxRF6fAUsOn1rOnfvz9Onz6Nbdu2Yfv27bjmmmuwePFivPrqq5g0aRKysrLw5Zdf4uuvv8b06dOxevVqPProo6iqqsKtt96Kp556qtU13dzcrF6XqLdhZouoky1atAgZGRn45JNPsHPnTv0qREC+pG644Qa8/vrr+syUJTfddBNOnjyJL774wup5oaGhSExM1P+yl7+/PyIjI5GdnY2DBw/immuusfsalgwaNAgeHh7Yt2+f/r6TJ0+irKxMn+GKiIhAcnKy/vH09PQWKyLHjh2LkydPIi4ursX7TExMbBEsOdLw4cPh5eWFgoKCVq+pBluenp5obm62+9qjRo1CVVUVDh486JCxDhs2DIcOHUJTU5P+vr1792LYsGH62/7+/rjxxhuxbt06rF+/vsVK1tDQUCxatAibNm3C008/jXfeeQeAfO5paWmt3r/xnzFr1yXqTRhsEXWyhIQEzJ49G0uXLsWAAQMwY8aMFo8/++yzcHNzw6RJk7BlyxacOnUKaWlpePPNN3Hx4kV9dmvmzJlYtmwZbrnlFjz33HM4cOAAsrKysH37dmzfvr1FFsycM2fOICUlBRcvXkRpaSlSUlKQlpamf3z37t3YunUrMjMz8fXXX2PevHm44oorLBbkt0dgYCDuvvtuPPTQQ/jpp59w+PBh3HnnnZg3bx5GjBgBQDJzq1atwo4dO3DkyBHcd999LRYP3H777fDy8sItt9yCAwcO4MyZM9i6dSseffRRh43TVFBQEB544AHcd9992LJlC7KysrB37148+eST+l5kCQkJSE9Px8mTJ3HhwoVW/dEsGThwIG677TYsXLgQ3377LTIzM/HZZ5/pA9Lk5GQMGzbM5inS22+/HfX19bjvvvtw8uRJbN68Ga+++ioeeughAMCGDRvw3nvv4cSJEzh16hQ+++wzDB06FIBMS3/yySc4ffo0UlNT8c033+gfu//++5GRkYElS5bgyJEjSE9Px8cff4zVq1e3eV2iXseVBWNEvdV7772nAFD+8pe/mH28qKhIefDBB5UBAwYonp6eSlBQkDJr1izlnXfeUZqbm1uc+8477yiTJ09W/P39FS8vL2Xo0KHKo48+qhQUFFgdw6xZs1q1J0hISNA//s033yiDBw9WvLy8lOjoaOWRRx5RampqrF7TtBA9KytLAaCcPn1af9+7776rxMbG6m9XVlYqd999txIcHGy29UNdXZ1y9913K35+fkpMTIz++catH7Kzs5Ubb7xRCQ4OVvz8/JQxY8YoL7zwgsVxOWLczc3Nyt/+9jelf//+iqenpxIXF6csXLhQ/7lXVlYqV111lRIQENCq9YNxsbqitF6kUFVVpSxbtkwJCwtTfH19lXHjxin79+9XFMVQZJ+VlWXz+9m/f78yefJkxcvLS4mNjW3R+uHTTz9VJk6cqAQEBCjBwcHKFVdcoZw8eVJRFEVZu3atMnr0aMXX11cJDQ1VbrrpphZ/ro4ePapcccUVir+/vxIYGKhMnDhR2bBhQ5vXJeptNIrSxoQ+EREREbUbpxGJiIiInIjBFhEREZETMdgiIiIiciIGW0REREROxGCLiIiIyIm6ZAd5b29vhIeHu3oYRERERG0qLi5GfX29xce7ZLAVHh6O3NxcVw+DiIiIqE1xcXFWH+c0IhEREZETMdgiIiIiciIGW0RERERO1CVrtoiIiHobrVYL7qDXdWk0Gri5tS9HxWCLiIjIhRoaGpCTk4PGxkZXD4Xa4OnpiX79+sHLy8uu5zHYIiIicqGcnBwEBgYiLCwMGo3G1cMhCxRFQUlJCXJycpCYmGjXcxlsERERuYhWq0VjYyPCwsLg4cGv5K4uLCwMFy9ehFartWtKkQXyRERELqLWaDGj1T2ov0/21tYx2CIiIiJyIgZbREREpPeXv/wFDQ0N7XruwYMHcfvtt9t0blJSEmpra9v1Oubs2rULEyZMcNj1HInBFhEREemtWrXKYrDV1NRk9bkTJkzApk2bbHqdlJQU+Pr62j2+7ojVeERERF3EvRsO4GxJjVOunRDmh/WLJ1o9Z9myZQCAadOmwc3NDd988w3+8Ic/ICgoCKdOncK5c+dw/PhxLFy4ECdPnkRDQwP69euHd955BxEREdi1axceffRRHDx4ENnZ2ZgwYQKWL1+OL7/8EuXl5fjnP/+JBQsWAJD6p8rKSgQEBKB///646667sH37dhQUFOCee+7BypUrAQBpaWm46667UF1djTFjxiAzMxMrV67E1VdfbfW9vP/++1izZg00Gg3i4+Px9ttvIzY2Fvv27cP999+P5uZmNDU14f7778d9992H9evX46WXXoKXlxeam5uxfv16TJ482QGfPDNbREREpPPWW28BAPbs2YOUlBREREQAAH7++Wf8+9//xvHjxwEAr7zyCg4ePIijR49ixowZePrpp81er6SkBOPHj8ehQ4fw2muv4eGHH7b42mVlZdizZw+Sk5Px97//HXl5eQCARYsW4fe//z2OHTuGFStW4MCBA22+j2PHjuGxxx7D119/jaNHj2LatGlYunQpAODZZ5/FihUrkJKSgmPHjuHWW28FAKxYsQI7duxASkoKDh8+jJEjR9r4qbWNmS0iIqIuoq3Mk6vcfPPNCAgI0N/etGkT3n//fdTX16O2thZRUVFmn+fv74/rrrsOADB16lRkZGRYfA211is8PBwDBw5EVlYWAgMDcezYMdx2220AgPHjx2PMmDFtjnfnzp24+uqrERsbCwBYvnw5Vq9eDUVRcNlll2H16tU4c+YMLr/8csyYMQMAcPnll+OOO+7ANddcgyuvvBJDhgyx4ZOxDTNbREREZJVxoPXzzz/jtddew7Zt25CamoqXXnoJdXV1Zp/n4+Oj/9nd3R3Nzc0WX8P03KamJiiKAo1GY3drDPV5KuOfH3roIXzxxReIjo7Gk08+ieXLlwMA/vOf/+C5555DY2MjFixYgP/7v/+z6zWtYbBFREREeoGBgSgvL7f4eGlpKYKCghAaGoqGhgasXbvWaWMJDg7GiBEjsHnzZgDAL7/8gtTU1DafN2fOHHz11VcoLCwEINOjc+bMgUajQXp6OgYOHIglS5bgySefxL59+9DU1ISMjAxMmDABjz76KG688UYkJyc77H30ymnE8xV1+Pv2dMwcEo5rx8a4ejhERERdxooVK3D55ZfD19cX33zzTavHr7zySnzwwQcYNmwY4uLiMG3aNGzfvt1p49m4cSPuuusuvPjii7jkkkswduxYBAcHW33OyJEj8eyzz2L+/PkAoC+QB4BXX30VO3fuhJeXF9zd3fHiiy+iubkZd911F0pLS+Hh4YHw8HC8++67DnsPGqULbjEeFxeH3Nxcp12/qLIOk/72HRZO6YfVvx7ttNchIiKyprm5GadOncKQIUPg7u7u6uF0SdXV1fDz84NGo0FaWhpmz56N9PR09OnTp9PHYun3q624pVdmtvr6e8PTXYOCMvNzzERERNQ17N69G4899ph+i5x169a5JNDqiF4ZbLm5aRAV7IP8cgZbREREXdn8+fP104HdVa8tkI8O9kV+meO2CSAiIiIyp9cGW7EhviivbURNg/WtB4iIiIg6otcGW9HB0s8jn3VbRERE5ES9N9gKkc0vOZVIREREztRrg63YEMlsFZQz2CIiIlL95S9/QUNDg0Ov8dRTT+Gjjz7q6NBa6N+/P44dO+bQazpLrw22ooPVzBanEYmIiFSrVq3qcLBleo2nn34at9xyS0eH1m312mArJpjTiERERMaWLVsGAJg2bRqSkpJQVFSEyspKLFmyBJMmTcKYMWOwbNkyNDY2AgBWr16N4cOHIykpCUlJSTh79qzZa9x555147bXXAEjW67bbbsM111yDESNG4PLLL8fFixcBAA0NDVi6dCmGDBmC6dOnY/ny5bjxxhvbHPeZM2cwd+5cjBkzBklJSfjss88AALW1tbjlllswYsQIjB07Vt9C4vTp05g+fTrGjh2L0aNHY+XKlY79IE30yj5bABDk6wF/L3cUsNcWERF1FR/eCpRmOefafQYAt1nfXPmtt97C2rVrsWfPHv3m00uXLsXMmTOxbt06KIqCJUuW4LXXXsOdd96JF154AQUFBfD19UVNTQ3c3NzMXsPU/v37ceDAAYSGhuLWW2/F2rVr8cc//hFr165FTk4O0tLS0NTUhNmzZyMuLq7Nt3b77bfjnnvuwdKlS3H69GlMmTIF48ePx8GDB1FaWoq0tDQA0Ad1r732Gq666io8+eSTLe53ll4bbGk0GkSH+CKfNVtEREQWffbZZ9i3bx9efPFFAJIt8vLyQlBQEAYPHoyFCxdi/vz5uOqqq2wKjADZXzE0NBQAMHXqVP3m0jt37sSiRYvg4eEBDw8P/Pa3v8VPP/1k9VqVlZVISUnBPffcAwAYPHgwZsyYgZ9//hmTJ0/GyZMnsXz5csyaNQsLFiwAAMycOROPPfYYqqurMWvWLMydO7ddn42tem2wBUj7h4PZpVAUBRqNxtXDISKi3q6NzJMrKIqCzz77DAMHDmz12L59+7Bnzx7s2rULU6ZMwebNm3HppZe2eU0fHx/9z+7u7mhqatK/lr3fx+o2PqbP02g0GDhwINLS0vD9999jx44d+MMf/oCUlBTccMMNmDZtGr799lu89tpreOWVV/DVV1/Z9br26LU1W4A0Nq1tbEZZTaOrh0JERNQlBAYGory8XH/72muvxXPPPacPiEpLS3HmzBlUVlbi/PnzuPTSS/HnP/8ZM2bMwC+//GL2Gra67LLL8MEHH6CpqQl1dXU2rWAMCgpCUlISNmzYAADIyMjA7t27MX36dOTm5kKj0eDaa6/FCy+8AEVRcO7cOZw+fRoRERG44447sGbNGuzbt8/usdqjl2e2dEXy5bXo4+/l4tEQERG53ooVK3D55ZfD19cX33zzDV555RU8/vjjSEpKgpubGzw9PfH888/Dx8cHN954I6qrq6HRaDB48GAsXrzY7DVstWzZMhw5cgQjR45EXFwcxo0bh9ratst9Nm3ahN/97nd45ZVXoNFosH79esTHx2Pbtm144oknoCgKtFotFi1ahDFjxuCZZ57Bpk2b4OXlBUVR8NZbb7X787KFRlHzb11IXFwccnNznf46Hx88hz/8+yjW3zEBc0dEOv31iIiIjDU3N+PUqVMYMmQI3N3dXT2cLqGyshKBgYGor6/Htddei5tuugn33nuvq4cFwPLvV1txS6/ObMWGGDJbRERE5Hpz585FfX096urqMHfuXNx5552uHlKH9epgi/sjEhERdS379+939RAcrlcXyKs1W9yyh4iIiJyld2a2qoqBA+vgGzcJffw82UWeiIhcQm1X0AXLp8kMS20m2tI7g63mBuCH54GJ9yIm5NecRiQiIpdQV/eVlJQgLCyMPR+7MEVRUFJSAk9PT7i52Tcx2DuDrcBowMMXKMlAdLAv0gsr0axV4O7GP+RERNS5+vXrh5ycHKdvGUMd5+npiX79+tn9vN4ZbLm5AaEDgYsZiBnggyatguLKekQF+7T9XCIiIgfy8vJCYmIitFotpxO7MI1GY3dGS9U7gy0ACBsInPwScUHSJyO/vJbBFhERuUx7v8ip6+u9v7OhgwBFi0EexQCAAtZtERERkRP03mArbBAAoJ9SAIDtH4iIiMg5em+wFSrBVkRjHgAgj+0fiIiIyAl6b7Cly2wFVp+Fm4bTiEREROQcvTfYCogEvALgVpqJiEAfTiMSERGRU/TeYEujAUIHACWZiAnxQR4zW0REROQEvTfYAqRuqyIX/YLccKGqHvVNza4eEREREfUwvTvY0tVtjfKVrr3nLta4cjRERETUA/XuYEu3InG4VxEAILO42pWjISIioh6odwdbusxWgkZ6bWVeYLBFREREjtXLg61EAEB4g/TayiyucuVoiIiIqAfq3cGWXxjgHQzv8myE+HlyGpGIiIgcrtOCrVWrVkGj0eDYsWOd9ZJt02hkQ+qLGRjY1x9ZnEYkIiIiB+uUYOvw4cPYt28f+vXr1xkvZ5/QQUBlAYaFuaOkugHlNY2uHhERERH1IE4Pturr63H//ffjjTfegEajcfbL2U9XJD/WX9o/ZFxg3RYRERE5jtODraeeegoLFy7EgAEDLJ7z0ksvIS4uTv+rqqoTAx5d+4chHucBAFms2yIiIiIHcmqwtXfvXhw4cADLly+3et4jjzyC3Nxc/a+AgABnDqslXWYrTikEAGQys0VEREQO5NRg64cffsDJkycxYMAA9O/fH7m5ubjiiiuwbds2Z76sfUIHyqEuB24aNjYlIiIix3JqsPXEE08gPz8f2dnZyM7ORlxcHLZv344rr7zSmS9rH79QwLcP3EuzENfHj8EWEREROVTv7rOlCh0ElGRgQF9/ZJVUQ6tVXD0iIiIi6iE6NdjKzs7GqFGjOvMlbRM2CKguwrBQoKFJi7yyWlePiIiIiHoIZrYA/YrEMb4lALhHIhERETkOgy1AvyIx0V3aP3CPRCIiInIUBluAfkVijDYfALhtDxERETkMgy1An9kKqD4LPy93rkgkIiIih2GwBQA+wYBfX2hKMjGgrz+nEYmIiMhhGGypwgYBF6X9Q355HWoamlw9IiIiIuoBGGypQgcBNSUY3kd6bGVfqHHxgIiIiKgnYLClCpMi+VE+xQC4RyIRERE5BoMtla7X1gCN2v6BRfJERETUcQy2VLoViZFNeQCAM0XMbBEREVHHMdhS6XpteZdnITbEF6fOV7p4QERERNQTMNhSeQcCAZHAxQwMjQpEZnE1Gpu1rh4VERERdXMMtoyFDgJKMjAkMhANzVqcLWHdFhEREXUMgy1jYQOBujKMDpUeW+mFrNsiIiKijmGwZUy3InG4l7R/SGfdFhEREXUQgy1juhWJcUoB3DTAqUIGW0RERNQxDLaM6TJbXmVZ6N/XnysSiYiIqMMYbBnTtX/AxQwMjQxEdkk16hqbXTsmIiIi6tYYbBnz8gMCY4CSDAyODIRWATKKWSRPRERE7cdgy1TYIOBiJoZGBAAApxKJiIioQxhsmQodCNRXYERwPQC2fyAiIqKOYbBlSrciMR4F8HJ3Y2aLiIiIOoTBlindikSP0iwMDPdHOts/EBERUQcw2DKly2ypeyTmldWisq7RtWMiIiKibovBlqnQgYDGHShOx5DIQADA6SLWbREREVH7MNgy5eEN9B0CnD+Gobpgi53kiYiIqL0YbJkTORIozcawUA0A7pFIRERE7cdgy5zIEQCAmPos+Hq64/R5TiMSERFR+zDYMidyFADArfg4hkQGMLNFRERE7cZgy5zIkXI8fxxDIgNRXFmPkqp6146JiIiIuiUGW+YExQI+wcD54xgVGwwAOJpX7uJBERERUXfEYMscjUamEs+nISlOgq2UnDIXD4qIiIi6IwZblkSOBOrLMcy/Al7ubjiSy2CLiIiI7MdgyxJd3Zb3hRMYEROEI+fKoCiKiwdFRERE3Q2DLUsi1CL5Y0iKD0FpTSPOltS4dkxERETU7TDYsiRiuBzPH8cl/UIAACnnOJVIRERE9mGwZYl3ANBnAHD+OMbGMdgiIiKi9mGwZU3kSKDkNBKC3dDHz5PBFhEREdmNwZY1kaMARQtNcTrGxocgLb8C9U3Nrh4VERERdSMMtqwx6iSfFB+ChmYtThRw6x4i6qVqLgKHNwJcmU1kFwZb1hgFW2PjpW7rCKcSiai3Sl4H/Pf3QO5BV4+EqFthsGVNn/6Ap5+0f2CRPBH1dsUn5Via7dJhEHU3DLascXOXFhBFaejj74X+YX4Mtoio9yo5Lceys64dBzmfthn47mmgJMPVI+kRGGy1JXIkUF0MVJ5HUnwIsi5Uo6ymwdWjIiLqXFotcOGM/Fx+zrVjIefL/wX46UXg8AZXj6RHYLDVlogRciw+gSS1biu33IUDIiJygYo8oKlWfi7Lce1YyPnKc+VYVeTacdFlUHEAACAASURBVPQQDLbaEj5MjkUn9UXyKTmcSiSiXkadQgSAMma2eryKPDlWnXftOHoIBlttUTNbRWkYERMET3cNjuYy2CKiXkadQvT0k2lEtn/o2crVYIuZLUdgsNWWgAjAtw9QfBLeHu5IjAjE8fwKV4+KiKhzqZmt/pcCTXVSy0o9FzNbDsVgqy0aDRA+HCg6CSgKRsYEobCiDiVV9a4eGRFR57lwWrJa/SbLbU4l9mxqsFV9AWhucu1YegAGW7aIGA7UlwMV+RgZEwQAzG4RUeeoLQVKu0CrhZIzQNggICRBbrP9Q8+mTiNCAWouuHQoPQGDLVtEDJdj8QmMjAkGwGCLiDrJtseBdZe7tkaqoUbqtMIGG4Ittn/ouZqbgKpCw+2uPpVYeV62kurCGGzZwmhF4vDoQADA8Xy2fyCiTlCcLpmFhirXjeGirrFl38FASLz8zPYPPVdVIaBoAS/5vkNlFw62tFrgX3OBT3/n6pFYxWDLFmpmq+gEAn080T/MD2nMbBFRZ6jUZRhqSlw3hgun5Bg2GPCPANy9WLPVE9RcBD5/QGqSjalTiDFJcnRFZquxFjjznQRT1hQelcC/OL1zxtVODLZs4d8X8A8Hik8AAEbGBCOrpBrV9SwaJCIn0jYD1bql9y4NtnRtH/omAm5uQHA8M1uu9vPL8qsjvlkJ/PI+cPT/Wt5foWtoamuw1VQPVDl4derXTwAfXA/8/KL18858K8fKwi7djoTBlq3Ch0nkrCgYERMERQFOFDC7RUROVF0s0zkAUFPqunGobR/CBssxJJ69ttpLUYC6DpahaJuBH18E9r3Z/mtkfA+kbJKfTbNCamYrdrwc2+q19cPzwKvjgDoHfSeW5wG/6Ma28xkg8wfL5575To7N9bKYpItisGWriOFSM1F+jisSiahzVBYYfnZpZus0EBgDeAfI7ZB+8u9hF/5ys6i5Edj+J6DwmGte/9gWYM3Ajm3wXJwONFRKxqmx1v7nN1QDWx8CPP2BwGig2GQaUW37EJ0EQNN2ZqvgKFBfARSdsH8s5ux5FdA2Alc8I2Pccg9QUdD6vNoy4Fyy4XalmXO6CAZbtjKq2zKsSGSRPBE5UaXRijBXBVuKIm0f+iYa7gvuJ8fuOJV4djew9zXgyGbXvH7GTkDbJBs9t1fuAcPP6h6G9tj5jLTumPNnIH4ycDGrZdBWkQdodNPFfmFtZ7bU4KzYAcFWVTFw6D0gajQwZTlw3auS4f333a37fWXuBJRmIHqsbhwMtrq/cEOwFR7ojYhAb6RxGpGInKkrZLYqCyWLpU4hApLZArpn+4eze+TY0am89io4IseLWe2/Rq5RNsfefmd5h4F9bwCxE4BJS3Wr7RXJXqrK84CAKMDdAwiIbDuzpU47mhbat8e+N2TD80tXSFPxkb8BJi8DcvYAu55pee6ZHXK8ZJEcmdnqASJ07R906daRMUE4VViFxuY2VkoQEbWXcWar1kV9hNR6rb7GwVY3bv+gD7ZcsMdtU70h+1PakWDroOFne34P0j4HNt0oWatrXwXc3IHwofKYcd1WRR4QFCM/B0RYD7bqKqTpN9DxzFZtGXBgvQT2w6813D/vr5K92v0PwzgVReq1+g6R7BzQ8u9LF8Ngy1a+fWRuuygNgKxIbGjW4vR5F/a+IaKerStkti6YFMcDMr0EdL/2D031hik4V2S2ik7IFCLQ/sxWbZn8pz9qtNy2JdiqLQW23At8fIcEKTdtACJHyGPhLRMJaGqQacPgWLkdECmZzXoL33XqFCLQ8cxW8jqp/br0EQkEVR5ewFUvyWe37Q/yHs4fl78fifPkuxkAKvM79vpOxGDLHuHDgOJTgFZrVCTPui0icpLK84C7N+AT4roO2SVGbR9UgdGAm0f3y2zlp8gm2oAELZ1NnUIE2p/Zyj8sx5G/kWNbvweV54E3pgGpnwBDrgSW7wOGX214PGwQoHE3BFuVBQAUIChObgdEyLHaQt2WOoXo5inNUNu7aEKrBfa/JVPUo29q/XjcBCBpIZC5Czix1dDyYfBcqStz82Rmq8eIGC5zyWXZ3LaHiJyvsgAIjJRefy7LbJ0CPHwM2SxAanmCYoDybhZsnd0tR427azJbhUflGD9Ffm8bauy/hjqF2H+mZJ3aCrZOfiEZnzn/C/x2s/x5MubhDYQONARbaqbKOLMFWC6SV3ty9ZsiR2vNRZsbZerP3LVKzshOCcOvBdw9zT9/7v8C3sHA9ieBE1/oNkafJr3fAqNYs9VjGK1IjA/1RaCPBzvJE5HzVBZKFsk31HWZrQungdBBLad1ANkjsbtNI57dI93v4ya4pmar4KiUpPSfIbdLs+2/Ru4BeQ/RYyQL1Fawpdaojb9TCs7NiRgGXMyUaVY1UxWkC7YCo+RoqW5LPT9xrhzNtX8oOwd891fg5ZHSqPTrP7Y+pyBFjjGXWH4vARHAZX+UhRl5B4H+lwKePoZxcjViD2G0IlGj0WBEdBDSCiqg1bKxHxE5WHOjLHkPjJJpkpqSzm8i2lAtX+bGxfGq4HgJWBzVyNLZtM1Azj4gZpxka+oq2t4KxtGvf/4YEDUGCB0g99k7lagoEmxFjZGMVEg/6722FEWCrYgRgF+o5euGD5PmuSVnDJkqNdhSpxEtZbbU1hNqsGXas+voJ8A/xgA/vSBj9g8Hcva2vk6+LtiKTrI8TgCYuETeDwAMnme4PzBKpjpN20N0EQy27KGu2jAqkq+qb0J2SbULB0VEPVJVEQBFMlt+YdLksb6yc8dw/riMIXpM68e6W/uHwlRpBJowDfAJBqBIMbap5HWG7YkcqeQM0Fgjq+r66IIte4vkL2ZKTVTcRLmt/z2w0GurNFumEBOmWb+uWiRfdMKQqWo1jWghs1WRC3gHSQDk6d86s5W8Vqb7bvsE+J8UYOiVMlVZntfyvIIjsvF16EDrY3X3AK57TYK7Eb823B8YIwFjtYO3DXIQBlv28AmSPwi6IsekfiEAgF9yXJCOJqKeTS32DYwyZCU6u25LLeiOGtv6se7W/kGdTkuYDvjKv92t6rbKzgFfPQrsfdXxr1+gq9eKHtv+zJa6kjJughzVYMtSry39e24r2DJq/1CRL4sf1CBLn9myMo0YFCt1U+FDW2a2qoqlxixxDjBkvkxFx01q+V4AyTAWHJHPxs2GsCR2PLBwCxAQbrhPne40XZF4ekfLLvMuwmDLXtFJ8j+UugqM0wVbh3O64ZYVRNS1VemCrQCjYKuze22pwZa5zFZ3a/+Qs0f6S8VP0mW20LpuS82KmGZdHEGtSYoaI7+nHj7WM1uKIp3Uj35suE8fbJlktiwFvGqw1a+NYCtssHw2xSclUxUYbajR8wmRGrFKM8GWokiWSs2ChQ+ToEytLzz9DQBFVkGq1LEbB1sXMyXrGNPGFKI1+vYPJisSv1oBfP5A+6/rIAy27KVuC1CYitgQX0QGeePQWQZbRORg6soqtWYL6Pwi+YIj8iWmZjeMtZVV6UrU2qWo0TJD4WMhs6V+vhVO6NdUeFSm2cIGSfamT3/Lma3mJuCLh4CtDwL/WQJ88bD0v8o9APhHGD77kAQ5Wgy2dstsTFC09bF5+sjUZnG6IVOl0mgsd5GvKZFWGsG6NhEmzb9xapsEcYPnG57Td4isKDRuzFpgY72WNep7NF6RWFchU6lRo9p/XQdhsGUvNfIuOAKNRoNx/frg1PlKVNY1unZcRNSz6KcRo42CrU6cRmxqkPqbKDNZLUD3hazpHtOIF07JZ5cwXW6rwZZpry31861ox36D1iiKTCNGjTJkjPoMkM/OtKC7oQb4aKFktQbOll8H3wE2XiebZ8dNNKwqVIMcc78HFfkSzLU1hagKH2Zov6B2j1cFRJgvkFdrxdSeXEaLyNBUL/tAxk0C/MMMz3FzA+LGy96QTQ1yn7pPpCMyW8YrEtX6sUgGW92P+g+PLhIfn9AHWgU4co7NTYnIgcxmtjox2Co+IUX50WbqtQDp6h0+VLItnb1K0l5qfy018LBUs6VO09aVW+6Y3h7l52TK0jhwDR0gHdGNA7vaMmDDNZIRGnOLFJXfvgWYfJ9MgyrNhnotAPD0tdxry7hGzRbhQ+X6gGFaUBWgW+lnunrTtCeXcWYr+2fpPD/kitavFTcRaK4HzqfK7YIjgFeAtBhpL33NltE0onp9BlvdkF+opG51y1Qv6dcHAOu2iMjBKgsBD1+pL3LFNKK+oNtCZgsABl0uQaG53kpdyalv5Nhvqhwt1WwZB7P2NMhsbgLWzQF2PmP+cXO1b+qqO+O6reR10j9q2u+B36yVgNbdA7jyOdnLMHJUyz0DAcu9tmwtjlepKxIBQ6ZKFRAhgaFpd/hykzYRwfESNBWdAE59LfcNvRKt6IvkDxqK46PG2FYcb4l3kEzTGhfIFx6TI6cRu6mYJElLN1RjVGwQvNzdWLdFRI5VWSj/W9dopKkp0LmZLX2AYCGzBQCD5sgx4zvnj6e9cg9JpmjoAunED1ip2TL6fC21UzD7GgckSPrpJfNF78YrEVV9zKxIPPFfGduc/23dgHTcHcB9u1tumwRY7rV1do8EQWpdV1vUFYmAmcyWhfYP6mekTmdqNIYViae+ltc2DuJUsePkmHtA3n99RcemENXXDowyyWwdl787gW3UrHUCBlvtET0WgAIUpsLbwx2jYoNwOKeUzU2JyHEqCwxTI74hADSdG2wVHpUvfuNtekwlTJO9G8/s6Lxx2UNRgB3/K0Xac54y3K9mtizVbAH2Fcmf1mXOtI3ArmdbP154VPbuU2uaAEP7BzU4K82W84ZeaXm7GnPM9dqqLpFp4IRplrvGm+o7BIDu3CDTYMtC+4eKvNbnhw+XVZ1lOfJezL2+X6isgDyX7JjieFVgtCEjqdVKsBU1yvbPwIkYbLVHtKFIHpC6rcq6JmQUO3COn4h6r6YG+eJXgy03d9nmpbOmEbXN0gQ0eqz1LyovP6D/dODs3vbt8+dsZ74Dsn8Ckm4zbLcGGE0jWliNCNgfbPmFSTH70Y91zWBheI3cg1LP5OFluD84XoJANbN14gs5Dr/G9tcFzK8KVTu02zqFCMjvZR9dFqxVsGVhf8TyPMCvr2HLHKBlhsxcvZYqfpKM+bQuUO9oZguQFYm1pZLlK80CGqu7RL0WwGCrfdRgS1e3NU5Xt8WpRCJyCDWDYDz9oW7Z0xlKMnTdzq3Ua6kGzZFiZ7UIvavQaoEdf5HM22yTvfg8faTPlbmaLXWKscLGXlvlebINT+I8YO5fACjA96vlsboK4P3rZYXf2N+2fJ6Hl0y/qZmtE1ul0/qgy21/j4D5Xlv2FserYsbJtJt/eMv79cGWSQ+r8lzDFKJKDWq9AoGEGZZfSy30P7ZF1xIj0fK5tjIukj+vq9disNWN+YfJ/0p06c9xCRJs/XK2k7s7E1HPZNw9XuUX2nlNTfX1WjZkGxJ1dVtnuljd1rEtshpt8u9aBwSABFXmMlt9B0sTT1uDLXUKcch82UR5xHVA+lfyeXxwg9RyzXgYmLK89XP7DJBgq/I8cG6/7PXn6Wvf+zTttdXcCKR/KRmnvkPsu9bVLwFLd7UuVDe3P6K2WabsWgVbun0LE+e0zOSZUpubNtdLUG+60Xl7GDc2VbOLXaA4HmCw1X7RY6UIsKEGkUE+GB7SjP9JuxX49qm2n0tEZI2+7YOZzFZntFkoVLfpsSGzFT5M9qXrCnVbjXUSKKZ8CHz3tEwXznjY/Lk+wS1rthRFPl+/vtJnytZpxNPfAhp3Q0bqspUyPbjpRiA3GZj+oPmCd0DqthqrpacWlNYrDW2h77Wl6+T/ywdS/zVpqf21Sr59DFOJxswVyFcWSqsI0ynH4Fjg1s3AFRZWZqrUvRQBx9RrAUbBVr6sRNS4my/QdwEGW+0VnSSbXuqi55XeHyNWKURT5k8uHhgRdXuWMlvaJvObJztawRFDt/O2aDSSxSg57doGp7ueA56JAdbOBD67DyjPAS77k2GrI1O+Jpmt+kopcPcLk9YHppktRQG+/iNwarvhvqZ6IHMXED9ZAhUACB8CJN0u3w9THwDmrrIc9KgrEpPflgL6wfPsf9/GvbYaa4Ef1kjAONVMJq29PH0kODUOtkxXIhobtqD1ikZTbu6GVYmOqNcCTDJbqZLZ8/B2zLU7iMFWe+k7yacA2bsxvXwrAEBbYufGokREpixltgDn122Z63beFldPJZaeBX78u2yBM+tx4OaNwO8PyxSiJT7BLWu21M/VL1QyW7WlLYv+L2YC+94APrlLatoAqVNrrG4dJF31InDX18D81dazS+qKxJoLUlyvFu7bS+21dWC9ZHUuXQF4B7bvWpYERLacRlSbsbYVVFkzYJZkAdUpxY5S/3NSnC6fRxeZQgQ6IdiaP38+xowZg6SkJFx66aVISUlx9kt2DrVfyrlkYOuD0Lp7I1k7FF4Npa3rAIiI7KFmttTpG8Ao2HLyQpyyHAlCrPXXMjVwtnxptqff1oXTshWNtU2Z2/Lzy5L1u+oF4LInpW6qraycT4js69dYJ7fVlYh+YYbtaoynEgt13cgbq4F/3y0rRtVmqaar7jy8gYSpbU/jqZktwP5ViMZC+knx+k8vSlZuwt3tv5Ylpvsjqpt1mzZAtcf0/wHu22tbBtUW6n9OMr6XYxcpjgc6Idj6+OOPcfToUaSkpGDFihW4+24n/CFwhYAIqVNI/RgoOQ1l5h+wH7r6htJslw6NiLq5qkKZxjPOTnRWY9NCXQNOW+q1VL59gNjxQOYPUqBtj6+fkKm44/+x73mq8jwgZRMQOwEYeJntzzNt/2Cc2VKnxoynEtVga+hVMqPx/dPA6e0SbKhF4fZSM1saN2m62l7qisTaUmD24y1bMThKUKxc/9wBuW26VU97eHgbtvhxBE8f+bNYrqtf603BVkhIiP7n8vJyuHWkHX9Xo04lRoyA+4wH4d5Xtl9ovJDhwkERUbdn3D1e1VnTiPpVXKPte17iXKkns6cFRMZOQ2H9uWT7Xk+1559AcwMw6w/2FYTr90fUTSXWmstsGQVb549JXdUN62W7mT2vytTi4Hntb5rpHSh1RYPmAAHhbZ9viRpshQ0Gxt7W/utYM/1BaaOx5R5paVGeKwXoAVFtP7czGU+996ZpRAC44447EB8fj5UrV2LDhg2tHn/ppZcQFxen/1VV1U2ag/a/VJYIX/NPwN0TfeOlmVth9kkXD4yIurXKgtZbjHQk2GqsAz5/wDC9Yo06VWS6yqwto26U44H1tp2v1QLf/lm+wPv0l9YH9q60rDwvK/mixwKD59v3XIuZLQvBVmGqrGzz8pOAyztI7rfWuNMW93wL3Nz6e9EucROlb9j8v8peis4QOQK44m/SiPSrRyXYCox23uu1l/r3xq9vy2l4F+uUYGvjxo04d+4cVq9ejccee6zV44888ghyc3P1vwICAjpjWB03eRmwIh2Il+K+QUMkiq7IP+3KURFRd9ZYJ9M1gSYZAzXYak+vrZ1/A355XzY6bosadKir62zVN1Eae5780tCGwJrUTySAmfw7mUKrLZX6LVP1lZavsfdVqbua+Zj92SW1eana/qFFsKVOI+pqtmouSuClZvv6JAA3vgOM+LXUq3WEbwjg5d+xa0SNBp7MN7/psyNNvBcYciVw9COZbu7IFKKzqMFWF9mmR9Wpc3qLFy/Gzp07UVLSQ5p/urm1WFY8cvAgVCk+HSv0JKLercpM2wfA8G+NvZmtnP0y5QUA+b+0fX51iQRa7clYTP6dtDxoK7vVWAd8/1cJeC59RLZuASS7ZexcMvBcPyDtv+bHeeAdqZcaepX9Y7WW2fIL0zU21QVbar2W8bTU4HmSkbK3CamzOKIpaFs0GuC612XqUNGab/vgaurfmy5UrwU4OdiqqKhAfr5hNcenn36KsLAwhIZa6HvSzfl4eeCCZzSC6vLQ1Kx19XCIqCuzVEiu1jD16d/yfp9gKaS2J9hqqJGeU+5esnVKZYFhpaMlNSWGLJq9Bs0BQgcBhzdIzydLktdKEfPMxySwi58i95sGW0c264I3Mxm5Ix/KysAZD7fueG4L05qtmhIAGvmc3dwkQ6KuuFO3frG3jq0n8g8DfvOW/FnsIg1DWwjSZbZ6U7BVXl6OX//61xg9ejTGjh2L119/HV988QU0XSi152iNQQmIxgUcz+0h2Tsicrzs3cDzA4D9b7e8v+ai7KsXGA2MvbXlY+3ZjPq7p4GLGcCcPwOjrpf78ttov1Nzof3BlpsbMGmJTAmm/tv8OeeSgZ3PSFH3pCVyX1A0ENyvZZG8tln2CwSArJ9aTk0qCnD4fcmMtafrOmCU2VKDrVL5fNUMUbBRY1M1s9XFvsBdZtBlwP/8IkXzXc3I64FpvwdGtPPPhZM4NdiKj49HcnIyUlNTceTIEezYsQNJSQ7qFNtF+UYmwkOjxYkTx1w9FCLqipqbgK8eAxoqge1/BHL2GR77bpUEKvNXm29K6Rdme7CVvRvY/6ZkjaYsl337AOtTiVqtXN+vr+3vx1TSbdK2Inlt64L3kgxg862SFblpQ8vu3vGTgAvphveXsxeoLtZt5aJInZAq94CcO+aW9rc5MFezZRxkBsVIfVxjrWz9EhRnuRt9b9Snf5fpzt6CX6j8/eloHZyD9aA+DF1DRD9ZkZifxRWJRGTG4feAouOyes/dC/jkTqCqGMg7DBzaINN9o24w/1x1f0Rb7HtDgppfvyHZmsiR0rqgwEpmq75c9rvrSFDhEwwk/VayQcaBZHWJ7BdYWyrF5epWLar4yXLM1fVxUuu0rn5Jgr8jmw3B2+GNchy3qGPjBFrWbJkGW4D0TSw+ySlE6hAGWw7mFS6dcGvOn0GzthM2jCWi7qO2FPj+bzJNeM0/gKtfkTqqLXdLtkvjBiz4u+VVVL6htm1GrW0Gsn8CYsYZunN7eMvy/fxfLD+/WhfI+XcgswXIJsiAdFr/aCGwYxXw4c3Sl+rKNeZXzfXTBVvn9kuG7cR/pcN6zDhg9E1AyRkg9yBQXwUc/1QydR0JgLyDAGhkGlG/CbVxsKUr/s7YKXsmdqGeTdT9MNhyNF1H4IimAqQXWlmyTES9z67nZWpq7irAOwAYe4tsrZL1I5B3UNrJRFrpRu4XKpmntrYEKzgi5wyc1fL+mEukj5a696Ip4xV5HRE+FLhspQR4J78Efn5J3t+03xvqtExFjJTpx3PJcm5lgWy7o9FIpgyQovjjnwINVcAlHchqAVJf5hMkn1OdmtEzanehZrZOfS1HZraoA7pYN7IeIDgeWo07+mmKsD+rBCNiglw9IiLqCorTgeS3ZVuZ0TcZ7v/Vc9K1vbJAtlqxxrjXlm+I5fOyfpDjgJkt74/W1czmpxiCCWM1F3Sv08HMFgDMekx+NdVLRquxRrJUlrh7AHHjJXt1bIvcpxY5R42RYOzYFhm7hy8w+saOj9EnRGq2zAWZ6uejdsRncTx1ADNbjubuCQTFIUFzHslZ7Wg+SEQ9086/Sfbkyudbtirw8Abu2gYs32+oI7JE30W+jX9bsn6UzuxqHZSqrSJ5R2W2jHl4AxHDZe/Etlaix0+WoOzQe0BwvCE402hkdWZdOZB/WDJebX1WtvAJlmvW6jb3Nn7fag8pbRPgFdBy02giOzHYcgK3sAHo71aE5MwSKPZuP0FEPVPeYZmKipvQ+jE3d9kGpi22NDZtqgfO7pUaKNOGmxEjpCi/M4Mte6j9tprqDFOIqjE3S00b0LHCeGO+IVKzZe59+/WVBQWAbnEBvy6p/finxxn69Icv6oCaCzhT1E32eSQigy8eBn56yXHXa26Unk0dzY7Ysj9i7kGgqbb1FCIAeHhJ4FCQYr5Ivlo3jejvomDLOBA17Z8VGCXTr7ETgITpjnk9n2DZVLm6WG4bB1tuboYGmazXog5isOUMun9QEzTnsSeDzU2JupWmepnG+vll+dkRys9JJ/Q+CR27ji3Blr5ea7b5x2MukeDCeJNllTo96arMlm+IbCod3E82VzZ1/dvAku8ct+edTwgARdo7ALLa05i6IpH1WtRBDLacQbciMcGtCLvPXHDxYIh6kYPvAi8Oa3u1njWl2RIY1VcAmT/Y99xdzwE7nzV/TaD1Fjz26tNfptIyd1k+J/MHwCvQUJ9lylrdVs0FqfXyCujYODvit/8H3PVl50zbqXVfFzPlaBpkqkXyUWOcPxbq0RhsOYPuH9RJIRXYm1nCfRKJOoNWK9moygLgwun2X6fkjOHnE2Y2QLb2+nteA/a/1XqKrvSsHEP6t39cABAQAQy7WvZPvHCm9eP1VdI2of90yxtJG69INKX2mnLllmpBMbKVT2dQV3Tqgy2TzNawq2Q6lj22qIMYbDmDLtga43cRlXVNOJZf4drxEPUklecNmzUby/oBKNMFNeW57b++Gmx5+EiPqOYmG593WrbgMS64VjkqswUAk38nx+S3Wz+Ws1dWzw2Y1foxVcRwyV6Zy2xVX3BdvZYrqFv2lGRKxtDHpJ3GqOuBxVu75rY01K0w2HIGn2DANxTxOA8AnEokcqQfngM+uKH1VJq6hQsAVOS3//pqsHXJIulnlbPHtuflHTb8fOFUy8fKzgLQACHx7R+XKmG69JxK+RCoN2mcbKm/ljF3Tyn4NlckX3PRdfVarqAGV/XlUq/FFYfkJPyT5SyhAxBYmwcfTzcGW0SOVJgqx29WytQdINvMnPwCCB0ot80Vf9uqJEO+hCfcJbfTbJxKzDcOtkymMUuzZXrMERkSjUayWw2VQMrmlo9l/iAtCyKsdKEHpAi9pqTl59RUL9fsVcGWUa8ubjJNTsRgy1n6DICmqhDT+/nh4NlS1DU2u3pERN2fogBFuk3eC1OBox/Jz0f/D2huAGY+Jrc7mtkKS5SAJXQQcGKrIaizJu8woHGXn00zW6VngZAOrkQ0NvomCQiT1xrGduIL+UwGBJjc9wAAIABJREFUzGw7Q9N3iBzVWiXAqNeUA7rHdxfGXfh7U5BJnY7BlrPoViQ+omxAcNNFHMwudfGAiHqA8lzJvoy+GfDtA3z/V6ChBji0QW6PvB7wD29/ZquuQvYODEuUDNLwa4CqQik6t6apQRfoXCqNMI2L7OsqZDqyo20fjHn5AeMXy+ukfwV8uQL46Hb5DKY+0Pbz1Qyg2WCrFwUdLTJbveh9U6djsOUsE+4GYi7ByPwt+MH7YbjvXCV7cBFR+xXrslr9pgCznpCgasu9wIV0YMytgKcPEBTb/szWxQw5hiXKUd2bL+1z688rSgOa64G4SUDYoJaZLbVo3xHF8cYm3itF3R8vAg6sB/pfCty3W/YXbIvuP4Mtgi1XNzR1BeOCeE4jkhMx2HKWoBhgyU403/wB8jQRmJq/EfjyEVePiqh7Kzohx4jh8h+a0IFA+pdy37g75BgUK+0ftO2Yui9Rg61BcowZJ40tT/zXfMd1lVqvFTtOArXSbENDVHUloiOnEQFpjzDi1wA0wOUrgTs+N7+5tKXnatyAi1mG+5jZct04qMdjsOVMGg3cR1yDlwe9g3RtHLQ5+109IqLuTc1shQ+TrWfmrpLbcROBSF1ReFCMtD+oKrL/+ur0n5rZUqcSy3IMhfnmqCsRYy6ReihFawhkSp2U2QKAX78BPHxcatXc3G1/noe3bLTcIthSu8f3opotTx9p8QG07h5P5EAMtjrB1MGRSFfi4VaRK00Hiah9itKAgCjDlM/wa4ArngUW/N1wTnCsHNszlagGW2pNEwAkzpXjWSstIPJ/AQJjZP++voPlPnUqUd9jy8GZLUA2mlb377NX6ECZRlQzdjW6acTeluFRpxJ72/umTsVgqxPMSOyL01rdF0BJBzpbE/VmWi1QnA5EDDPcp9EAU5e33JomSA222lEkX3JGgiZvo+1qYsfJ0VKRfEONTG+q56kr/dS/62VnpYloQJT943Gm0IFAY7UhA9gbpxEBw1Rib3vf1KkYbHWC/mF+uOgnBalKcbqLR0NdRmMt8I+xwH4zncCptfIcoLEGCB9u/Ty1bsneYEtRpGZLrddS+YUCYYOB3APmn1d4FFCaDQGfOgWp9toqzZYaqa7WMLOPrki+VDeVqBbI97ZCcV9mtsj5utjf/p5Jo9EgcuBoAEDZWSt1H9S7lJ2TL+JzrOWzidpfyzizZY6tma2jnwDGdZTVxbL5tBosGYubKL9XVcWtH8szKo4H5MvbP0KCLa1W6r2cUa/VUabtH2pKJMvj7um6MbmCPrPVy4JM6lQMtjrJ6DHj0KS4ofzccVcPhbqKqkI51nCHAZsUpcmxre7ogboaJms1W3XlwKdLgf8sMaxaNC2ONxY3QY7mphLzjYrjVX2HSLBVdR5oqnNOvVZHmQu2elNxvErNaDGzRU7EYKuTTBkcgxxEwauUNVukUyl7Z6K6xPp5JPQrEYdaP8/TR4KGciuZrdwDsmKw7KxhU2urwdZEw/NM5R2WKTnfPob7+ibKfnu5yXK7K2a21DG1CLZ6YcAx42Hg+vUtu8kTORiDrU7i6+WOUr8BiGjMR2V1tauHQ10BM1v2KTohU4TGvZEsCW6jsWnOPsPPyevkaC3YihgBePq1DrZqy6QRqjqFqFKL5E9/K0dH99hyBC8/yQJezJJ6td4abIUPBcbc5OpRUA/HYKsTeUUNg4dGi5SUQ64eCnUFlbpgq/qC9YaZnSXrJ9nIuCvSNksrhfA26rVUQbFAZb7lPQ1z9gGe/sDQBcCZb6UwviRD9jY0N+Xn7iENTvMOt2yWqp9CtBBsnflOjl0xswUY2j/UlUtvst7UPZ6oEzHY6kSxQ5IAAFknfrH9SeV5QEWBk0ZELlWlm0bUNkphtisVnQQ+uB7YeC3w7VNAc5Nrx2OqNFtqnyLaWImoUhubVptpbNrcCOQelDqsycvkvoPvSGarT3/LBeJxE4CGKsN0JgCkbJbjwNktz1WzY5W67FpXrNkCZNueujJD5/zemNki6gQMtjpRaIKsSKzOT4NWa2Mm48ObgU1McfdIamYLMCy7d4XmJuCzZRKcRI4Gdv8D+OA3rh2TKeNtemxhbUViwVGgqRboNxUYMBPoOxT45X3J8JibQlSZ1m1V5APH/yN7EkaNanluSD/prQVI00xbpj5dQS2SV99TbyyQJ+oEDLY6k25qIbbxLI7ll7d9fmOdrMA6nyptAqhnUTNbgKGhpCv8/LJ0QJ/+ILDkO2DCPUDWj8DamY7Pqmq17ZsyLdYFW2312FIFWekin7NXjv0mS1PUSUtkGq25oY1gS7ciMVe3IjF5nQSoU+9vfa6bu6FfV1edQgTMBFvMbBE5A4OtzuTlj3r/WCRq8vH9SRv2bbuYISumACDjO+eOjTpfZRcItgpTgR+elyBm9h9lz7yrXwLm/VWyQulf2X/N7N1AU0Pr+0vPAs/GAqmf2H/NIhtXIqrUxqbmViTm7JVNmNVM1ZhbAC9dx3jThqbGAqOA4H4SbDXUAIfeBUIHAYOvMH++um1PV51CBAyNTdVgy5+ZLSJnYLDVyTyjhmGgWz5+OGFDxkDdWw0wLE+nnqGxVloDqPuyuWLKrqkB+PQ+Ceh/86YEWqphV8lRbQtgq8xdwHsLgOS1rR87tV06wKd9bv9Yi07I1JzxNjrWBFuYRlQUKY6PGg14B8p9PkHA2FvlZ7Ww3ZK4CVKzlfw2UFsKTLnPcmd49VpdOrOlC7bKdJtlM7NF5BQMtjqZW/gw+KARF/IzUFxZb/3kYl2w5RMMZP7Y9YqWqf3Ueq1IXa2PufYP7Z1ys9WJ/8oU9aWPtGzICUhgo3GXtgD2OKnLhJ3a3vqxLN1Kx7N77HtfzU2yz6CtU4iA7G8ItJ5GvJgpn3W/qS3vv/zPwLWvAQnTrV83biIABdj1rPy9HPtby+d2h2DLJ7hlnRa7qBM5BYOtzhYu/wAP1uRh+/FC6+eqma1LFkkWxNJGuNT9qPVakSPlaJrZam4CXp8IrI4AXhoJvH2ZZKHMTc+1l7pP58jftH7M3VOmvy5m2H49RQFO64KsnH1AfaXhseYmaS0BALUXDa9ti4P/knoqddrPFp4+kqUxDbbUeq34yS3v9w0Bxi1qe/9CdQxNdcD4O61n2oZfA8x6Ahh1o+3jdgW1bgtggTyRkzDY6my6PkHD3Auw9YiVposAcCEdCIoDRlwnt8+wbqvH0Ge2dFvPmNZsVeZLK4LAKCAgXFofHPnQ0NfJEUqz5Wip4WboQMlsWepVZarkjFzTO1jaWajBFQAUHJH/METogsuzu227ZlkOsGOVZNqmLrftOaqgGKAit+V9+uL4KfZdSxU9BnD3kqzfpKXWz/X0BS77o0xTdmVqsOXmaZhaJSKHYrDV2XRTCzNCSpCcfRGF5XXmz9NqgQtnJBMWM07S/SyS7znUzFboQMDDp3VmSy3snrQUWLoLuP5tua3uD+gIZWclk2EpOxM6EGiuN/SKasvpb+Q4+3E5Gv95zdzZ8jE16LFGUYAvHgYaq4GrXwG8/G0bhyooTlZTGgeLOfskuFQL6O3l4Q1MfUCCqOC49l2jq1Hrtvz7yupMInI4BludzS8U8A/HCM8C+S45auGLrCJXegH1HSLdqwfOlu7V3EevZ1AzWwFRMt1lWrOlFnarLQzU/lJqvylHKM22vlIuVLcyr8TGqcRT2yVwHH+XrNozzsRm/QB4+AJDfiW1V9m7267bSv1EFoaM/S2QOMe2MRgLipEMm/rZVl+Q7JtpvZa95v4vMPOxjl2jK1EzWyyOJ3IaBluuED4MwdWZCPB2x9ajFlYlqsXxapHtoDkAFEOGgLo3NbMVGKkLtkyCaDXYUrMnQbGAd5Djgq2GGhmDteJt9UvY3IrEi5ktg6X6Sil873+p7LmXOAcozZJArbEWyNkPJEyVzFDCVMmWqSvgVBdOA3mHgPPHpfHotscl83bFM+17j/r2D7qpxI5OIfZUDLaInI7Bliv0HQJNfSUeis9ASe4pnC0qa33OBZNgS/2ffcb3nTNGcq7KQsn0eAfJ9I1pxrLcJLOl0Uh26/xx+1byFRwFNv669fXLcuTYnmArezfwz0uAXc8Z7svcJVmkwfPltvGf15x9Mh05YJbcp674O2s0lZh3GHh9ErDucuDNacDaS6WQfsGa9q+QUwPVinz5z8u2xwFopGs8Gai/z/7hrh0HUQ/m4eoB9ErRYwAA9+Y+iXu9Ae0bj0gX6wVrDOdc0K3WUps4BsfJtiIZ38uXLWsrureq85LV0mgke9NYLRkgT195vCJPirADowzPiRgOnNsPVBXJc22x/y3JhmbuBEYbrYprqzgeMGr/YBps6Qrff1wjQVX8JEO91uB5chwwE3DzkKlENUs3cLYc1Wm8s7uBJF3rhO//Kn+uZ/9RbjfVyX80Rl5v2/s0R81sndoGbP0foLZMat+sNS7tjfxCgd+8Lb3HiMgpGGy5QtLtQEAUmksy8eE3P2Oe5gCiDm8A5jxlKFa+cFqK4o3/t5k4B9j3hhRJqy0DqHuqLDRsDaN27a6+AITEy8/luRJoubkbnhOhW7n4/+3dd3iUVdoG8HtKJpPeew8kAQJJIJQQuoACgmJdXRsuiHVdxbLFXct+lnVVdtVdy6orrICIiqIooCi9twChBkhIQhIS0ttk2vn+OMmkFzCTSTL377pyTZJ533mfOQTycM7zPqfweNeSLZOxsQN8y8L6hiW8jma21BoZT8tkK3df/R15SmD1fcD924CMH2Vy1FBsrfUAQkfLbX/KcwAnLyBQ/icDHiHyuud3yq8zt8n/RCT8Cpj8h87fV1c1zAoeWib3KbxtORA3s/tevz9J/JWtIyDq17iMaAsqByBuBlSpD+F4wh/xjn6m/J/8mR8bjyk6JX95NZ3Bip4sH7O6eNs89U6m+qLthoSpoVamaZF8xYXGZKHB5RbJn98hu5y3dU7DzFZnW8l4D2je/kEIWVcVlCi39CnNkpulV+Y3LiE2GHiVnLG7mC5ruZr2sIoYJ3t4VRbIWS2lunsTLUDObClUciueO79kokVENsNky8auSwzGBlN9o8QT38rHmhL5i9e3xT5wIfUb4XZnryXqeVX1+2K61i8RWma26uuqjHVAdVHjljMNms5sdcXJtfLR0V3WejVVel4mIu6dtC/wjpZ3xVbW38hRck4mcKGj5NL3wGmNheetkq1pjZ9HT27+XESqfNz4glwaHXF38+aa3cHBSc5mLfgJiJrQva9NRHQZmGzZ2Ogobwi3IBxTxkGc3gAYdHIJEWjcyLaBi49cfrlwoMfjpG5UVd/2ob2ZrYau5y1ntlx85bJyV2a2zGbgxFq50fDAqXLZsK6q8fnSLLlEqOqkkqBlkXzDhsUhyXLW9fp3ZPyO7q1bKgQmNnYkj57c/LmGYw+vkO0irNVKIW4m4D/IOq9NRNRFTLZsTKVUYHZCML6uS4ZCXyXv6mpZHN9USLK8U1FX3qNxUjeqrG/70DCz5dykZgto3fahKf/BciPkzrq65x2S7RUGz27s2t6wRY4Q9clWJ0uIQGMxuSXZqt8yKrR+ltUtALh3HXDX17LGqymlUs5+xVzTetbKO7rx/Y++78qbjBIR9QFMtnqBOYlBWG9uWEr8pnXbh6aCR8jHvEM9Exx1v5YzWw3LiA29tlq2fWjKfwigr5JF5x05Wb8kPWhO45ZADcuPNcWylqorGyRbZrbqG5te2C+Tw6aJml8cEJrc9vmT/wDcsar13bMKBRB7NeDkDYx7vPM4iIj6MCZbvUBSmCfgFYlTiiiIk98BF4/Lu73amnkIqf+ldoF1W31Wq5mtlsuI9U04W9ZsAV0vkj+xFnANkLVVLc/panE8IH8GFUo5s2WoBQqOytfsjtYjs94AHj0kl8eJiPoxJlu9gEKhwJyEYKzVJ0OhK5M9kbwHtF1PE5QgC5tZt9V3WWa26pMtraf8M61uObPV1jJiF4rki04BxRnAoGvlUp5npGyg2nCOJdmK7DxWtQbwCJN3JOYfAczG9mexLpdaAzh5ds9rERH1Yky2eok5icFYZx4tvxDm1sXxDTQu8hcuZ7Z6r62vAR9Okzc7tKXyomx14FTfGV2pbL4/YsUFQOnQdkdvv/pi76YzW0IAVUWyrxbQeFfroNmNr+8/qI2ZrciuvR/vaDmzZSmOH9m184iICACTrV5jUKAbFL5xyER9oXBbxfENQkbI4ueKdjaxJtu5lCG3scndBxz6pO1jqgrkEl/TvlMuvo0F8uUXZMG4so2/nlp3OdPUNNna8irw+kDgRT/gjUHAjjcBRw/Z26qB/xB53ZqSJt3jI7v2nryjAUMNcPI7AAr580dERF3GZKuXUCgUmJMUgu+M9YXybRXHN2DdVu+14Rm51KZxk0mPUd/6mMqLMtlqquXMVlvF8Q38h8g7Vk1G2eB2y6uyG/2ga2WSpnEFxtzf/O7ApsuPZedlfF3dc7DhjsTsnfI/AVqPrp1HREQAuF1PrzInMRg3/zgDQwO0mBw3q/0DG5KtvIPy1n7qHc5sBDI2AENvltsp/fQCcOQzYMRdjceYzUB1IRCc1PxcZx/ZzkNXITdgbqs4voH/YHmdvIPA6oWyT9XtK9tfem44B5AzYqVZsji+q0XuTds2cAmRiOiycWarF4nydUFwSDh+e+km6JRO7R/oN0gWPLNIvm0mo+xX1lDDZC1CNL/mhmfkn8u054FRC+QM0PbFzeOoKZYzXy1nthraPxQclY+dzWwBwKp75J2LM1/tONFqek7+YblM2dV6LaB5stVdxfFERHaky8nW+++/j/Jy2Ujz4YcfxsiRI7F161arBWavrksMRmWdEZtPFbV/kEotZ0YuHOq8uaU9OvYV8L/rgW8eaZ4QdQeTAUhbAfxrNPBKGPDVg3IT5X0fymaj4x6Vndm17sCYB2Vh+bHVjee3vBOxQUNj04Ij8rGthqYNGmapKvOAIdcDw+9q/9gGboHyrsczPwHC1LWGpg28IgHUz4JxZouI6LJ1Odn697//DQ8PD+zYsQPp6el46aWX8OSTT1ozNrs0JzEYCgWwJu1CxweGJAN15Y3NJqnRhfou54c/BX58tnteUwhg/3+Bt4YDXz8ob07wjpTbzXxyA7D+94BbMDDud43njLlf1k9tfb0xKbb02GpnZiv/sHzsaGbLN1b2YXMPBea82bXlQIVCzm5V1t9UcTkzW2pHWZTv4Nw4Q0ZERF3W5WRLrZblXT///DPuvvtuXHPNNTAarbxMY4cCPbQYG+2Dn04UorzW0P6BDXeEcSmxtfzD8m680NHAzreAnW//8tc8vgZY+7hs7HnVX4DH04EHtgOP7Acm/R4IHg7M/odszdHA2VsuJ146BWx7Q24w3e7Mlk9j7EDHNVsOWuDOL4F53wJOXl1/DwFNEqXLSbYAYNLTcnm0s70UiYiolS4nW0qlEitXrsRnn32GqVOnAgD0+jbutKJfbO7wEOhNZqxPz2//IN6R2DazWdY9BSUAv/5M1rf98GfgyKorf00hZO2V2gl4aBcw8cnGZpy+McCUPwELNwNxM1qfO/YRwC0I2PQi8GYSkPap/H57M1sN+xe21dC0qaiJrfcb7EzD8iPQte7xTY24S87UERHRZetysvWvf/0LK1euxH333YfIyEicPn0aU6ZMsWZsdmvG0EBo1Ep8daiDpUTPCDkbkrO75wLrC0rOyb0DgxLlzNKdq2Wys+5pObPU0u73gK8eaNxguS3nNskZpxF3A67+lxePq5+c/Zr+f7Iw/vx2+f32araESd5d2NW2DJej6RKgZ3j3vz4REbWpy2sCKSkp+PrrrwEAQggEBQXh7be7YXmGWnHXOmD64AB8dzQfeWW1CPZs485EhQKInQGkLQcytwFRE1ofY4/y0+RjUKJ89AgBxjwAbHwOyPgBGDyn8djaUvl9o07Wd4WlAKmPAHHXNm8oum2x7Pie+siVxeToKgvnR98HHPwE0JXJBLAp5yb7A7qHdM/egy01dJ93DQQcOrjblYiIulWXZ7bmz5+PsrIy6PV6JCUlISAgAO+88441Y7Nr1yfJTvLfHO6gS/zEp2QS8POL3X/XXV/VUPPUkGwBQMKtcjPlhiW8BkdWyURr0u+B4XfKvlWf3Ql8db+86xCQM15Z24Bht/zy2SAHJ2DMQln/1DKZajqT1VG91i/h7C0TrpY9voiIyKq6nGwdOHAAnp6e2LBhA4YPH46CggK8//771ozNrk2O84enswO+7mgp0TtK3vafs1s21CTZOsHBWXZUb+AeDERPlo1AG7bEEQI4sEQW0o97DLj+38Djx4C4WcDRVcBnd8m9Dbf/Qx7f9C5Da1A5yNYMQMd3Iv5S964DbvrQeq9PREStdDnZEvUzJ1u3bsXs2bPh7u4OZVt7t1G30KiVuHZYEE4WVOJEfkX7B058ClA5Aj//X++Z3TKbgfV/ko1Fe5IQcmYrYCigVDV/LvHXsmYq/Uv5de4+uXVN4m2Axll+z9UfuPUTIOE24PQ64OOZwMm1clmxaXG5tTQUyVsz2XL2BhzdrPf6RETUSpezpcDAQDzwwAP4/PPPMW3aNBgMBphMJmvGZvfmDpe/dL/uqOeWRwgwar5MMk5800ORdSJzM7D738CuHl5mLs+RdVhNlxAbDLpW7geYtkJ+fWCJfEy+p/lxKjUw911g9EK5rAgA4x+3WsjNNBTJW2sZkYiIbKLLydby5csxaNAgrFy5Ep6enrhw4QIWLVpkzdjsXnK4F8K8nfDlgVzoDB0ktuMXAQ4uwM8vAeZekAAfWi4fG7ae6SmWeq2E1s9pnIH462UBffZuIH217MMVEN/6WKUSmPl34JqXgYlPA2GjrBt3A8vMVidtH4iIqE/pcrLl6+uL+++/HwqFAnv37kVAQADmzZtnxdBIqVRgXmoULlXp8eXB3PYPdPUDUh6QzTMPfdL5C6/7PfD+JFmT1N1qS4ET38rPK/Maa6SuRM4+YOl1XX+N/Pqtbtqa2QKAxNvl4xfzAWMtkDyv/ddSKICxDwNXPdPlcH+xhjsSObNFRNSvdDnZ2rlzJwYMGIAHHngACxcuxMCBA7Fr1y5rxkYAbhsVBk9nB7y/5RyMpg72QUx9VLYT+OEvciuZ9lQXy21n8tOAPe92f8BHvwBMdbKjOtC419+V2PwykLlFzkK1lHsAeH9iYxNQQM5sKR0Av3bqq8JT5R2FFbmyMD7+hiuPzRqGXAcMmg34dLKpNBER9SldTrYWLVqEzz//HIcOHUJaWho+//xzPP54D9Wy2DEXRzXuHhuJ7JIarEsvaP9AJ09g9j+Bugrg28faL5Y//Clg0sui+q1vNO7V113Slsv9AKf8WX6d30mydWYj8MFUoKJFt/zis3KDZ0AWqbe05z2ZXH37u8Z9B/MPyy1p1Jq2r6VUNs5uJdzaWBjfWwycBty2vP34iYioT+pysqXT6TBu3DjL16mpqdDprLAMRa3MS42E1kGJ97actdwV2qa4GcCwW2WLgyOftX6+od2Bkxcw9x1AXym3kekuF48BeYfkjFHYaPm9zuq2Dq+UG0dvebX59/f/Vz66hwBZ24GaksbnDLXAqe/l59m7ZIJXeVHuOxjYRr1WU6PuA5LuAMY/1vX3RURE9At0OdlydnbGxo2NvZw2b94MZ+deNjPQT3m7aHDbqHAcy6vAtoxO6pdmvgq4+Mu6rJazVud3AsUZsg3C0JuAyAmyo3lHs0+XzgCmLm443lAYP/xOQOsOeEV1voyYXb/d0MH/yWsBMplKWy6X0yb/QW5hk/FD4zkZP8gteSb9Qe4x+ONfGmfB2qvXauDqJxNNDxahExFRz+hysvXWW29h/vz5iI2NRVxcHObNm8ftenrQgglRUCkVeG/L2Y4PdPYGrn1Dbgnz7aPN7048uFQ+Jt8jC8BnvCK/Xv/H1suOQgBb/g78KxnY8KfOAzTqgSMrZTPRsDHye0EJwKUMQF/d9jnlubJdQ2CCTKgaZtmOfSUL7UfNl01GFcrmS4npqwEo5PuY8Yo89vun6q/J7uhERNS7dDnZGjlyJM6cOYPVq1fjiy++QEZGBpKTk60ZGzUR6uWM6xODsfNsMQ7nlHV88JDrgIRfAafXA2selglXTQlw7GsgfCzgFyePCxwmN1c+v13ODtXWv67ZDKz/A7DpJfn1gSVAZQf1YgBwbDVQUyxntRq2ogkcBkAAF4+3fU7DrFbKg0DsTJlk5R0C9n0EqJ1kfZWLr9yz8MxPcsarrgo4vUG+D/dgIP5GYMBUuSSqULbdyoGIiMiGOk22ampqLB8GgwHR0dEYMGAADAYDampqeiJGqrdwUjQA4L87Mjs/+Lp/yQTm8KdyhuvwSnmXYMt2B1f9Re6Xt/Nt4M0EYMtrcm/APe8BEeOBue/J83b9u+3rXMqQW9t8db/s9ZVwW+NzgfVLegWH2z63IdkKTwGm/gWAAli9UNZwDbtJFv0DsiGpoUZ2pD+9XrZtGHqjfE6hAK59HVBr5fvobUXvRERk99SdHeDq6gqFQmEpzFbUz1oIIaBQKNhFvgcNCnTH2GgffHckH3+aNRgB7tr2D1ZrgFuXAivvAA4tky0RtB7AkOubH+fqBzywAzj6ObDlb41LeXHXAjf/F1BpgB1vyoL18Y83bphsrJOzXweWyiXAuGuBqc8C7kGNrx04TD62VySfvVvWXHlFyaQp4VdyKRIARi1o8sZnAT88I5cSa0rkDFbT9+EdDdy9Ru6JSERE1Mt0OrNlNpthMplgNpstnzd8zUSr5907LhJGs8Dy3ec7P1jtCPxqGRA9BTAb5KyTg1Pr41RqIOl24JH9ckbsqr8At/4PcNDKdgkTnpAF6Xv/I483GYEv58sELGw0MP9H4PYVgP+g5q/rFgi4+LVdgK8rBwqPyVmthmXHKX+UyV1IcmOfLkAmU/7xwMnvZKuIqIlyH8OmwlPa7hxPRERkY9xJuo9t2fqZAAAgAElEQVSZOjgAYd5OWL4nu+MtfBo4aIHbVgBz3uy8G7rKARhxFzDxSZmANYi/AfCKBHa/K5OkNQ/LLvHxNwLzvmts89CSQiFntwqPt76jMXcfIMyyHquBVySwYKPcDLqlwbNlIbxJL69LRETURzDZ6mNUSgXuGRuJ4mo9vj3cQaf4pjTOslZL63GFF1XLJURdmWxAemSlrAe78T+AUtXxuYEJgFEnW0401bReq6mgxLa3qxl0rXxUqoHBc67sfRAREdkAk60+6JaRYXDWqPDxjqyOm5x2p8TbAbdgmTRFTQJuWSJnwjrTXt1W9m5ZUN9ZE1LL6yTIj6E3NdaNERER9QFMtvogDycH3JwciuP5FdiXVdozF1U7Ate9BSTfK5clHToozm+qoclofpM7Ek0GIHc/EDqy+XJlRxQK4P6twA3vX17cRERENsZkq4+6JzUSAPDf7V1oA9FdYqYDc/4JOLp2/RzvaDmD1XRmK/+IbN/QcgmxMwpFYzE9ERFRH8Fkq48a4OeKaYP9sf5YAY7kdtLk1JaUKtlotOBIY5f67F3y8XKTLSIioj7IqsmWTqfD3LlzERsbi6SkJMyYMQNZWVnWvKRdeeqaQVAqgJe/P9FztVtXIjhJ3km45Fq5f2LmFtkrK3SUrSMjIiKyOqvPbC1cuBCnTp1CWloaZs+ejYULF1r7knYjLtANtySHYfe5Evx8stDW4bRv4lOyXUPufmDNQ3Ij6cBhgKObrSMjIiKyOqsmW1qtFrNmzbJ0nU9JScG5c+eseUm7s+jqWDg5qPDKupMwmsy2Dqdtrv7ALR8DT54CZr0ORE4ARjPpJiIi+9CjNVtvvfUW5sxhj6TuFOCuxX0TonCmsAqr9ufaOpyOOXkBo+8D5q2VG1YTERHZgR5Ltl5++WVkZGTgpZdeavXc4sWLERoaavmoqqrqqbD6hYWTBsDXVYPFP55GdZ2x8xOIiIiox/RIsvX6669j9erVWLduHZydW28WvGjRIuTm5lo+XF0vo7UAwdVRjcemxeJSVR0+6slWEERERNQpqydbixcvxqeffooff/wRnp6e1r6c3frVqDCEezvjw23nUF5rsHU4REREVM+qyVZubi6eeOIJlJWVYcqUKUhKSsKYMWOseUm75aBS4ndTY1ChM3J2i4iIqBfp4l4pVyY0NLR393/qZ65PCsa/N53Bf7dn4t7USHi5aGwdEhERkd1jB/l+RK1S4nfTYlBVZ8QH29hig4iIqDdgstXPzE4IRoy/K5bszEJxVZ2twyEiIrJ7TLb6GZVSgcenx6JGb8L7Wzm7RUREZGtMtvqhGfGBGBzkjqU7s3ChrNbW4RAREdk1Jlv9kFKpwB9mDkKd0YzXN5yydThERER2jclWPzUp1g+TYv3w1aELOJJbZutwiIiI7BaTrX7smWsHQ6kAXlx7gi04iIiIbITJVj8WG+CGX40Kx96sEmw4dtHW4RAREdklJlv93KLpsXDRqPC3dSegN5ptHQ4REZHdYbLVz/m5OeKhKQORVVyDpTuzbB0OERGR3WGyZQfmj49CmLcT3vwpAxcrdLYOh4iIyK4w2bIDWgcVnpsdj6o6I17+/oStwyEiIrIrTLbsxLQhAZg6yB9r0vKw62yxrcMhIiKyG0y27Mhzc+KhUSvx7Jp0GEwsliciIuoJTLbsSLiPMx6aPAAZhVVYsiPL1uEQERHZBSZbduaBSQMQ7u2Mf2w8jdzSGluHQ0RE1O8x2bIzWgcVXpw7FDV6E/70VTo7yxMREVkZky07NDHWDzcnh2Lr6SKsPnjB1uEQERH1a0y27NSfrx0MX1dH/HXtcRRV1tk6HCIion6LyZad8nTW4MW58SivNeD5b47ZOhwiIqJ+i8mWHZsxNAgzhwbiu6P5WJ9eYOtwiIiI+iUmW3buhevj4a5V4/lvjqGqzmjrcIiIiPodJlt2zt9Niz/MHIyCCh3e+OGUrcMhIiLqd5hsEW4bFYbkCC8s3ZmFI7lltg6HiIioX2GyRVAqFXjphqFQKhT401dHYeRWPkRERN2GyRYBAAYFuuO+idFIv1CBpbvO2zocIiKifoPJFlk8elUMwryd8MYPp5B5qdrW4RAREfULTLbIwkmjwqs3JaDWYMLvVh6C3sjlRCIiol+KyRY1kzrAFw9OGoAjueV440fenUhERPRLMdmiVh6fHovEME+8v+UctmdcsnU4REREfRqTLWrFQaXEW7clwUWjwqJVaSip1ts6JCIioj6LyRa1KcLHBf83dygKK+vwu5WHYDILW4dERETUJzHZonbdMDwEt40Kw7aMS/j7hpO2DoeIiKhPYrJF7VIoFHjh+ngMD5f1W98ezrN1SERERH0Oky3qkKNahffuTIafmyOe/uIIjudV2DokIiKiPoXJFnUqwF2L9+4cAaPZjPuX7UelzmDrkIiIiPoMJlvUJckR3vjztUOQU1KLv61j/RYREVFXMdmiLrsrJQIp0d5YvicbO8+w/xYREVFXMNmiLlMqFfj7TYlwclDh96uPoLrOaOuQiIiIej0mW3RZwn2c8fSMOOSU1OK1DdzOh4iIqDNMtuiy3TM2EiMjvLBkZxZ2nyu2dThERES9GpMtumxKpQJ/vzkBWgclHl5+EOeLq20dEhERUa/FZIuuSLSfK96+fQRKa/SY9/E+7p9IRETUDiZbdMWmDwnA89fFI/NSNRYs3QedwWTrkIiIiHodJlv0i9w9NhL3T4zGwewyPLYyDWZuWE1ERNQMky36xX4/YxBmJwRh/bECvP4D71AkIiJqiskW/WJKpQKv35KIxDBPvLP5LNakXbB1SERERL0Gky3qFloHFT64KxkB7o546osjSMsps3VIREREvQKTLeo2/u5afHD3SCgALPzffhSU62wdEhERkc0x2aJulRDqiddvSURhZR3uXbIP5bUGW4dERERkU0y2qNvNSQzGU9fE4UR+Be5bup8tIYiIyK4x2SKreGjyAMwfH4W9WSV4ZMVBGExmW4dERERkE0y2yCoUCgWemTUYN44IwcYThfj9l0fYg4uIiOyS2tYBUP+lVCrw6k0JqKg1YPXBC1Aq5NcqpcLWoREREfUYzmyRVTmolPjXr0dgSpwfvjiQiydWpcHIJUUiIrIjTLbI6rQOKrx3VzKmDwnA12l5eOyzNNZwERGR3WCyRT3CUa3Cv389AjOHBmLtkXw8uOwgavW8S5GIiPo/JlvUYzRqJd6+fThuGB6CjScu4s6P9qCsRm/rsIiIiKyKyRb1KLVKiTduScR9E6Jw4HwpbnlvF/LKam0dFhERkdUw2aIep1Qq8My1Q/DMrMHIKKzCTe/uxMUKbu1DRET9E5Mtspn7JkbjtZsTkF+uw9NfHIEQ7MNFRET9D5MtsqlbRobhphGh2HK6CCv2Zts6HCIiom7HZIts7rnrhiDE0wkvrj2BrEvVtg6HiIioWzHZIptz1zrgtVsSUGsw4YnPD8PEbX2IiKgfYbJFvULqAF/8Zpy8Q/Hl70+wyzwREfUbTLao13h6RhyGhXjgo+2ZuPm9XThXVGXrkIiIiH4xJlvUa2gdVPj8gbFYODEah3PLMOutbfjfrizepUhERH0aky3qVbQOKvxp1mCsvC8Fvq6OeHbNMby24ZStwyIiIrpiTLaoVxoT7YN1v5uApDBPvLP5LN7ZfMbWIREREV0RJlvUa7lpHbDk3lEYFOiGv68/hU92Zdk6JCIiosvGZIt6NU9nDf43fzSifF3wlzXHsJKNT4mIqI9hskW9nr+bFssWjEGIpxP+sPoo/rj6CHQGk63DIiIi6hImW9QnhHg6Yc0j4zAhxhef7s3BDe/sZGsIIiLqE5hsUZ/h6+qIpfeOxhPTY3GqoAJz3t6O5XvOw8yO80RE1Isx2aI+RalU4LdTY7BswRi4Ozngma/ScfsHu5HJPRWJiKiXYrJFfVLqAF/88PhE3JUSgT2ZJZjxz634ZPd5W4dFRETUCpMt6rPctA74v7lD8dnCFAR6aPGXr9Pxv11Ztg6LiIioGSZb1OeNifbBFw+kItrXBc+uOYZV+3JsHRIREZEFky3qF/zcHLH8vjEI83bC71cfwZq0C7YOiYiICACTLepHgjycsGJBCgLctFi06jDWpxfYOiQiIiLrJ1uPPvooIiMjoVAokJ6ebu3LkZ0L83bGivvGwMtZg99+ehCbTxXaOiQiIrJzVk+2br75Zmzfvh0RERHWvhQRACDazxXLF4yBi6Ma939yADvPXrJ1SEREZMesnmxNnDgRoaGh1r4MUTNxgW745DdjoFEpsWDpfuw8w4SLiIhso1fUbC1evBihoaGWj6oqbsNCv9ywUA8s+c0oAMCvP9yDGf/civ9sPYuLFTobR0ZERPZEIYTokb1OIiMjsXbtWgwdOrTTY0NDQ5Gbm9sDUZE9OFVQiaW7srD2cB4qdEaolAo8PHkAHp0aA7WqV/x/g4iI+rDO8hb+pqF+Ly7QDS/fMAx7n5mGd+8YgUGBbnjr5zP49Qd7kFdWa+vwiIion2OyRXZD66DCzGFBWP1QKu4dF4m9WSWY9dY2bDx+0dahERFRP2b1ZOvhhx+2TK9NmzYNAwcOtPYliTrkqFbhuTnx+ODukQCA+z7Zjw+3nUMPragTEZGd6bGarcvBmi3qKTklNbh3yT6cKazCvNRI/GX2EKiUCluHRUREfQhrtog6EObtjC8fSEVKtDeW7MzC/Z/sR6XOYOuwiIioH2GyRXbPw9kBS38zGjcMD8HGE4WYvngr1h3N57IiERF1CyZbRJB1XItvTcTLNwxDjd6IB5cfxG+W7ENOSY2tQyMioj6OyRZRPYVCgV+PCcdPT0zGDcNDsOlUEWa9tQ27zxXbOjQiIurDmGwRteDn5oh//CoJS+4dBSGAuz/ai++P5ts6LCIi6qOYbBG1Y3KcPz67PwUezg54eMVBLN2ZZeuQiIioD2KyRdSB+GAPrH4wFVG+Lnjum2O44Z0d+HhHJgorub8iERF1DftsEXVBSbUeL353HOvTC1CjN0GpACbG+uHRqTEYEe5l6/CIiMiGOstbmGwRXYYavRE/nSjEmrQ8/HTyIoQApsT5YdH0OAwL9bB1eEREZANMtois5ExhJf6xMQPfHZHF87+bGoPHp8faOCoiIupp7CBPZCUD/d3w71+PwPrHJmBYiAfe/CkD//o5w9ZhERFRL8Nki+gXGhTojmXzx2BIkDte/+E0Pth6ztYhERFRL8Jki6gbeDg74JP5oxEb4IqXvj+BD7edg9nc61boiYjIBphsEXUTH1dHLFswBtG+LnjxuxOY+eY2fHs4DyYmXUREdo3JFlE38nfT4osHU3H/xGjklNbgt58ewvTFW7DhWIGtQyMiIhvh3YhEVlJarcd/d2RiyY4sVNYZMSM+EC9cH48Ad62tQyMiom7E1g9ENlZYqcML3xzHd0fz4aZV44npsbhheCg8nB1sHRoREXUDJltEvcSPxy/iL1+no6BCBweVAuMH+uLahGDMGhYIZ43a1uEREdEVYrJF1ItU1Rnx3ZE8fHe0ADvOXILJLBDorsUfZw3CdYnBUCgUtg6RiIguE5Mtol6qtFqPrw5dwFs/Z6CsxoDRUd54fk48hgS72zo0IiK6DEy2iHq50mo9Xv/hFFbszYYQwPiBvrgzJRzTBgdAreINw0REvR2TLaI+Iv1COd7ZfAY/HLsIo1kgwN0RD04agLvGRkKl5PIiEVFvxWSLqI8prNBh5b4cLN9zHhcr6pAY6oGXbxyG+GAPW4dGRERtYLJF1EfV6I3458YMfLQ9EwAwLzUSCyZEIcjDycaRERFRU0y2iPq4Y3nl+NPqozicWw6VUoFpg/1xZ0oExg3whZLLi0RENsdki6gfMJkFfj5ZiGW7z2NrRhGEAKJ8XXDHmHDcnBwKT2eNrUMkIrJbTLaI+pnzxdVYsScbq/bnoLTGAEe1ErMTgnFnSjiSwjzZq4uIqIcx2SLqp3QGE9al52PZ7mwcOF8KAIgPdsedKRG4LjEYLo7sSk9E1BOYbBHZgeN5FVi25zy+PnQBNXoTNGolxg3wwbQhAZg2OICbXxMRWRGTLSI7UqkzYE1aHtanF2D3uWIYzQJKBfCrUWF4fFos/Jl0ERF1OyZbRHaqQmfAllNF+GT3eezNLIGzRoWFE6OxYEI0XLnESETUbZhsEdk5IQQ2nijEK+tO4FxRNTQqJUZEeGL8QF+Mj/FDYqgHi+qJiH4BJltEBAAwmMz48kAu1h8rwJ5zJag1mAAAiaEeeGjKQEwfHMC+XUREV4DJFhG1ojeacSi7FGuP5OOz/TnQG82IDXDFE1fH4Zr4QFuHR0TUpzDZIqIOFVbq8NG2TCzbfR7VehNuHRmK5+bEs3UEEVEXdZa3KHswFiLqhfzdtPjjrMHY9NRkTIz1w6r9uZj99nYcyS2zdWhERP0CZ7aIyMJsFvh4ZxZeXXcSJiEwOMgNw0I8MCzEE8kRXogNcGUxPRFRC1xGJKLLdiyvHP/cmIG0nDIUVdZZvh/orsWkWD9MjPXDmGhv+Lo62jBKIqLegckWEf0iFyt0OJxTht3nSrDldCHOFlVbnovydcHICC9MjPXDNfGB0KhZmUBE9ofJFhF1q5ySGmw/cwn7skpw4HwpzhfXAAAC3B1x99hI3D46HN4uGhtHSUTUc5hsEZFVXazQ4cuDufjfzvMoqNBBo1ZicJA7YvxdERvgihh/Nwz0d0WIpxP7eBFRv8Rki4h6hMFkxrr0Any+Pwcn8itxqaqu2fNODioM9HfFVYP8ceOIEET4uNgoUiKi7sVki4hsorRajzNFVci4WIWMwkqcKazCsbwKlFTrAQAjI7wwd3gIrokPhJ8bC+2JqO9iskVEvYbJLLDjzCWsPii3DdIZzFAqgJGR3pg5NBA3DA+BpzPrvYiob2GyRUS9UqXOgJ9PFmLd0QJsPl0IncEMF40K96RGYsGEaBbZE1GfwWSLiHq9Gr0R69ML8M7mszhTWAVnjQo3jghBjL8bwrydEOblDHcnB2jVKjg6KOGoVrK5KhH1Gky2iKjPMJkFvj+aj7d+ykBGYVW7x7loVBgT7YNxA30xIcYXMf7sbE9EtsNki4j6HLNZIKe0BjkltcgprUFuaQ2q60zQGUyoM5qRW1qDQ9llMJrlP18jwj3x1+uHYmiIh40jJyJ7xGSLiPql6joj9mQWY0P6RXx+IAcCwG2jwvHUNXGs9yKiHsVki4j6vRP5FXj+m2PYk1kCjVqJKB8XRPu5IMrXBUGeTvBzdYSfmwYhns4I9NDaOlwi6meYbBGRXRBCYO2RfHxxIBeZl6qRW1oDcxv/uk2M9cNvxkViYowfO9oTUbdgskVEdqnOaEJ2cQ0KKnS4VFWH4io9DmaXYn16AcwCGODngvsmROPGEaHcQJuIfhEmW0RETeSW1uCTXeexYm82KnVGBHlo8cCkAfjVqDCYzAIXymqRW1oDB5US4d7OCPZ0goOKyRgRtY/JFhFRG6rqjFi2+zw+3HYOl6r00KiU0JvMrY5TKRUI9tQi3Nu5/sMFE2J8eecjEVkw2SIi6kCt3oSV+7Lx04lC+Lk5ItTLCaFeTtCbBHJKanC+uBrZJbXILq5Gtd4EAFAogF+PDsfT1wyCh7ODjd8BEdkaky0iom4ghEBJtR4ZhVX4x4+nsSezBL6uGiyaHgdnjQq5pTXILa2FWqVAjL8bYgJcMdDfFW6ODnBUK1mMT9SPMdkiIupmQgisPngBL31/AiXV+i6d46BSINBDi7HRPkgd4IvUAT7wd2cbCqL+gMkWEZGVlNXosT69AB5ODgjxckKIpxMMJoHTFytx+mIlMi9Vo7a+632dwYyzRVXIvFRtOX90pDduHBGCWQlBcNc6QGcwIeNiFTKLqxHj74pBgW7choioD2CyRUTUi+SV1WLHmUvYfLoIG49fRJ3RDI1aiVBPJ2QVVzfrDebr6ogJMb4YEe4JPzctfF018HNzRJiXM5cliXoRJltERL1Uhc6AdUfzsfrgBVys0CEu0A2DAt0R6euMYxcqsP3MJZwsqGx1nrtWjVGR3hgT7Y2x0b6ID3Zn8kVkQ0y2iIj6sMJKHU4VVFoas+aX63AouxRHcsstG3EHuDti6uAATBvsj4F+bvB0cYCbo5pLkEQ9hMkWEVE/VKM34lB2GbZmFOHH4xdxrqi62fNqpQJuWjUc1Spo1Eo4qBTQ1H/uqFLCTatGbKAbBgW6YUiQO4I8neCiUTFBI7oCTLaIiOzA2aIqbDlVhIIKHUqr9SitMaBSZ4DeZIbeWP/R5PMKnQEGU/N//tVKBTydHeDr6ohhIR4YHu6F4eGeCPLQQqVUWDrp602y4L/OaIKvqyO0DipbvGWiXoPJFhERtWIwmZF5qRon8itwsqASRZV1KKsxoLxWjwultcgr13XpdVwd1bh6SADmJAZj3EBf7jNJdonJFhERXbbCCh0O5ZThcE4ZSmsMMJnNMJoEzELAUa2Co4MSaqUSB86X4HBuOQDAWaOCj6sG7loHuGnV8HFxhL+7IwLctfBzdYSLoxpOGhWcNSp4ODkgwE0Ld6fmtWUNv5K4nEl9CZMtIiKyqvPF1Vh7JB87zlxCea0BFToDymsMqNAZOz3XUa2Ej4sGBrNArd6EWoMJ7lo1hod7YUS4J4aHeyExzBOujuoeeCdEV4bJFhER2YTeaMalqjpcrNDhUpUeNXojavUm1OhNKKs1oLBCh4IKHYqr9NColXDWqKB1UKGwsg7H88otNWUKBRAX4Ibh4Z6IDXCDh5OD5UProILWQQlHtQreLhq4MCkjG+gsb+FPJRERWYVGrUSwpxOCPZ0u+1ydwYRjeeU4lF2Gg9mlOJRdhk/35nR4jkqpQFKYJ8YP9MW4gb4IcHeUd1+qVajRG3GuqBrniqqQXVILDycHRPg4I8zbGX6ujjCY5Y0DRpOAj6sGge5a9i6jbsOZLSIi6hPyy2uRXVyD8lpD/XKlETqDCXUGE3RGM84VVWH3uRJU1XW+fNkZjVqJMC8nDPR3xbiBvpgY44dIXxcAcsYut7QGpTV6S12as0bOXQghoDeZIQR4l6Yd4cwWERH1C0EeTgjy6HiWzGAy43BOGfZklqBSZ0Sd0QS90QwHlRLRfi6I9nVFhI8zymsNyC6pQXZJDUqq9dColHBQKaFWKVBUWYfzxdU4X1KDjScKseHYRQBAqJcTFArgQmlts22VAHlXpkIB1OpNMJoFVEoFUqK9MWNoEK4ZEgCFQoFD2aU4lFOG7OIahHk7IzbAFTH+bgjy1MLDycHSWsNgMqOwsg4F5bWoqjPBLASEEFBAgQB3LcK8neCmdbDKGJN1cGaLiIioHRU6A3aeKca2jCLsOlsMjVqJSB8XRPg6w8dFg+IqPQor61BYKVtlOGvUcNaoUF4rz9ObzF2+lrNGBScHFUpq9OjsN7OnswPCvJwR5u1U/+iMAX6uiA1whY+rI6rqjPjhWAG+OZyHXWeLMSLcCzcnh2LmsEDLLJzBZEZptR4+ro5Qccn0F2GBPBERkQ1U1Rmx6WQhNp64CAeVEiPqm8RG+bogt7QWGRcrkVFYhYsVOsvSqM5ggp+bIwLdnRDkoYWbVg2lUgGlQgGT2Yy8Mh1ySmuQW1KLnNIaFFToWiVmPi4aVNUZUWc0Q6VUYFiIB47nVUBvMsNZo0JsgBsuVuhwsUIHs5BJ3rAQDySFe2KAnysc1UpoVEqoVUrUGU2oqTOhRm+E3mSGUqGAWqmASqVEoLsWUb4y0XNU2/eSKZMtIiKifqrOaEJemQ7ni6txprAKGRercLqwEk4OKswaFoRZw4Lg7aJBWY0e3x7OwxcHL+BCaY28ccHDCT6uGpwtqsLR3HJU601XFINSAfi6OsLRQSZpGrUKGpUCGrUSGrUSCihQqTOgUmdEhc6ISB9nTIr1w6Q4PwwN9oDOaEJxlR7F1XroDCaYzQImISAE4KBSQqNWQKNSIcTLCd4umg5jOV9cjeN5FRgR4YUAd+0VvZ8rwWSLiIiIOmQyC5wprEJuaQ0MJjP0JgGD0QxHByVc6pdGNWolzELAaBIwmATyymqRWVyNzKJqFFTo5Hn120IZ6h/rjGZAAG5aNdy0DnB2VOF0QaUlsVMrFZYN1bsi0scZw8O9MDTEA84aFdRKBdQqBY5dqMDPpwqb7RGaFOaJa+IDcXV8AAb4uXb7mDXFZIuIiIh6Db3RjAPnS7HldBFOFVTA01kDHxcNvF01cHZQQaVUQKlUQAEFjPUtOeqMZpwtrMLB7FJkFde0+boB7o64apA/hoZ4YPe5Emw6WYiqOiNCPJ2w/fdTrLorAe9GJCIiol5Do1Zi7AAfjB3gc0Xnl1TrcaqgEnqTGSazGQaTQKiXE4YEuVsSqjvGRKDOaMKus8WoqjPafPsnJltERETUZ3i7aLqUqDmqVZgc598DEXWO27MTERERWRGTLSIiIiIrsnqylZGRgdTUVMTGxmL06NE4fvy4tS9JRERE1GtYPdm6//77sXDhQpw+fRpPP/005s+fb+1LEhEREfUaVm39UFhYiNjYWFy6dAlqtRpCCAQFBWH37t2IjIxs9zy2fiAiIqK+orO8xaozWzk5OQgODoZaLW96VCgUCA8PR3Z2drPjFi9ejNDQUMtHVVWVNcMiIiIi6jFWX0Zs2duirYm0RYsWITc31/Lh6mrdTq9EREREPcWqyVZYWBhyc3NhNBoByEQrJycH4eHh1rwsERERUa9h1WTL398fw4cPx7JlywAAX375JSIjIzus1yIiIiLqT6y+N+KpU6cwb948FBcXw93dHUuXLkV8fHyH57BAnoiIiPoKm++NGBcXh127dln7MkRERES9EjvIExEREVkRky0iIiIiK2KyRURERGRFTLaIiIiIrIjJFhEREZEVMdkiIiIisiKr99m6Eo6OjvDz87P6daqqqrg1UBs4Lu3j2LSN49I+jk3bOC5t47i0r0gwcL8AAA4dSURBVDePTVFREerq6tp9vlcmWz2FzVPbxnFpH8embRyX9nFs2sZxaRvHpX19eWy4jEhERERkRUy2iIiIiKxI9fzzzz9v6yBsaezYsbYOoVfiuLSPY9M2jkv7ODZt47i0jePSvr46NnZds0VERERkbVxGJCIiIrIiJltEREREVmSXyVZGRgZSU1MRGxuL0aNH4/jx47YOySZ0Oh3mzp2L2NhYJCUlYcaMGcjKygIAFBYWYsaMGYiJicHQoUOxfft22wZrIy+88AIUCgXS09MBcFwAoK6uDo888ghiYmIQHx+PO++8EwDHZsOGDUhOTsbw4cMxdOhQLF26FID9jcujjz6KyMjIZn9vgI7HoaamBrfffjsGDhyI2NhYrF692hahW117Y/Ob3/wGcXFxSEpKwsSJE5GWlmZ5zh7Gpr1xabB06VIoFAqsXbvW8r0+Ny7CDk2ZMkV8/PHHQgghPv/8c5GSkmLbgGyktrZWfPfdd8JsNgshhHj77bfF9OnThRBC3HvvveK5554TQgixd+9eER4eLgwGg61CtYkDBw6IGTNmiPDwcHH06FEhBMdFCCEee+wx8dvf/tbyc5OXlyeEsO+xMZvNwtvbWxw+fFgIIURmZqZwdHQUFRUVdjcuW7ZsETk5OSIiIsLy90aIjn8+XnjhBXHPPfcIIYQ4d+6cCAgIECUlJT0dutW1NzZr1qyxjMW3334rYmJiLM/Zw9i0Ny5CCJGTkyPGjh0rUlJSxLfffmv5fl8bF7tLti5evCg8PDwsP9hms1kEBASIzMxM2wbWC+zbt08MGDBACCGEi4uLKCwstDw3atQosWnTJhtF1vN0Op1ISUkR586da/YPgL2PS1VVlfDw8BCVlZWtnrPnsWlItrZs2SKEEOLw4cMiODhY1NXV2e24tPzF2dE4DBkyROzdu9fy3C233GL5D3F/1FZS0aCoqEhoNBphMpmEEPY1Nm2Ny8yZM8Xu3bvFpEmTmiVbfW1c7G4ZMScnB8HBwVCr1QAAhUKB8PBwZGdn2zgy23vrrbcwZ84cFBcXw2w2N9syKTIy0q7G6Nlnn8Wdd96JqKgoy/c4LsDZs2fh4+ODF198ESNHjsSECRPw008/2f3YKBQKrFq1CjfeeCMiIiIwfvx4LF26FJWVlXY9Lg06+/nIzs5GREREm8/ZmzfffBOzZs2CUil/Pdvz2Lz77ruIj4/HmDFjWj3X18ZFbesAbEGhUDT7WrD7BV5++WVkZGTgvffeQ21trV2P0a5du7Bv3z787W9/a/WcPY8LABgMBpw7dw5DhgzB3/72Nxw+fBjTpk1Denq6XY+N0WjEK6+8gjVr1mDcuHHYt28f5s6diyNHjtj1uDTV2Tg0fd5ex2jZsmVYtWoVtm3b1uz79jg2mZmZ+OCDD7Bjx452j+lL42J3M1thYWHIzc2F0WgEIP+AcnJyEB4ebuPIbOf111/H6tWrsW7dOjg7O8PHxweA3Fizwfnz5+1mjLZs2YKTJ08iKioKkZGRyM3NxTXXXIO9e/cCsN9xAYCIiAgolUrccccdAIDExERERUXhxIkTAOx3bNLS0pCXl4dx48YBAEaNGoXg4GAcOXIEgP2OS4PO/k0JDw+33JzT8jl78dlnn+GFF17Ajz/+CH9/f8v37XVsdu3ahby8PAwePBiRkZHYvXs35s+fjw8++ABAHxwXW61f2tKkSZOaFciPGTPGtgHZ0BtvvCFGjBjRqrDwnnvuaVbMGhYW1q+LejvStI6A4yLE9OnTxXfffSeEECIrK0v4+vqKvLw8ux6bgoIC4ebmJk6ePCmEECIjI0N4eXmJ3Nxcux2XlvU3HY3Dc88916zY2d/fXxQXF/d0yD2m5dh89tlnYuDAgSIrK6vVsfY0Nh3VsrWs2epr42KXydbJkydFSkqKiImJEcnJySI9Pd3WIdlETk6OACCio6NFYmKiSExMFKNHjxZCyF8e06dPFwMHDhRDhgwRmzdvtnG0ttP0HwCOixBnz54VkyZNEkOHDhWJiYli9erVQgiOzYoVK8TQoUNFQkKCGDZsmPj000+FEPY3Lg899JAICQkRKpVKBAQEWG666WgcqqqqxK233ioGDBggYmJixOeff26r8K2qvbFRq9UiNDTU8u9wYmKiuHTpkhDCPsamvXFpqmWy1dfGhdv1EBEREVmR3dVsEREREfUkJltEREREVsRki4iIiMiKmGwRERERWRGTLSIiIiIrYrJFREREZEVMtojosigUClRVVfXoNZ977jkMHjy4zT3SlixZgtOnT1/xayclJaG2trbT4xYsWNBqG5W+YvPmzRg5cqStwyCyW3a5NyIR2ZbRaLRsBt8Vf//735Gdnd1sI+MGS5Ysga+vL2JjY9s812QyQaVStfvaaWlpXYrhww8/7FqwREQtcGaLqB9RKBR49dVXMWbMGERFReHjjz+2PBcZGYn09HTL1yNHjsTmzZsBAJMnT8ZTTz2FiRMnIiwsDK+99hpWrlyJ1NRUREREYOXKlc2u8/rrr2PcuHGIjY3Fp59+avn+vn37cNVVV2HkyJEYMWIEvvzySwBAVlYWfH198de//hUTJkzA22+/3Sr29evXY8SIEUhISMCkSZNw/PhxAEBqaip0Oh2mTp2KRx99tNk5H374Ifbv349HH30USUlJ+P7777FkyRLMmDEDd999N0aOHIm9e/di8eLFGDVqFIYPH47Ro0djz549zcasYaYuMjISL7zwAlJTUxEVFYUXX3zRctzkyZOxdu1aAMC8efPw0EMPYdq0aYiNjcWNN94IvV4PACgvL8dNN92EQYMG4aqrrsJdd92FJ598ss0/r87G68knn8SYMWMQHx+Pn3/+2XLeJ598gmHDhiEhIQHXXnstLly4YHnu1VdfxbBhw5CYmIiUlBTU1NQAkAnuQw89hMTERMTHx2P//v0A5H6FV199teX17r333jZjJaJfwNYt7Imo+wAQ//znP4UQQhw/fly4urpa9p9rue9YcnKy2LRpkxBCboVx6623CpPJJC5cuCC0Wq145plnhBBC7NmzRwQFBTW7xvPPPy+EkFv3+Pj4iOzsbFFaWiqGDx8u8vLyhBBCFBUVifDwcJGfny8yMzMFALF8+fI247548aLw8fERR44cEUIIsWzZMhEfH9/smpWVlW2e23Ibj48//li4uLiI06dPW75XWFho+XzXrl3tvnZERIR47LHHLOe4u7uL3NzcVte55557xNixY0VNTY0wGo0iNTVVrFixQgghxKJFi8T8+fOFEEKUlJSIyMhI8cQTT7SKuyvjtWTJEkvMAQEBoqqqShw9elQEBARY4nrxxRfFrFmzhBBCLFmyRKSkpIjy8nLL9Y1Go9i0aZNQq9Vi3759Qggh3n33XXH11VcLIYRYvHixuO+++yxx9eb95Yj6Ks5sEfUzd9xxBwBg8ODBUKvVKCgo6NJ5t9xyC5RKJYKDg+Hr64u5c+cCAJKTk5Gfnw+dTmc5dsGCBQCA6OhojB8/Htu2bcPOnTtx7tw5zJw5E0lJSZg2bRqEEDh16hQAQKvV4vbbb2/z2nv27EFSUhKGDRtmeQ+5ubnIz8+/ojEYP348YmJiLF8fOnQIkyZNwtChQ/HAAw/g+PHjlpmolhrGz8/PD9HR0cjMzGzzuBtvvBFOTk5QqVQYPXo0zp49CwDYtGmTZXbIy8vLMo4tdTZeGo0Gd911FwAgJSUFgYGBOHz4MDZt2oTZs2cjJCQEAPDQQw/h559/hhACa9euxYMPPgh3d3fL9RuWUOPi4ix1W2PHjrXEm5KSgvXr1+OJJ57AN998AxcXl64MMRFdBtZsEfUzWq3W8rlKpYLRaAQAqNVqmEwmy3NNk6e2zmv4uuGXdcPrtEWhUEAIgYSEBGzdurXV81lZWXBxcYFCoWjzfCFEm8+1d3xnXF1dLZ/r9XrcdNNN2Lx5M5KTk1FRUQEPDw/o9XpoNJpW57Y3fl09rr330lJn49WWhnFu+vpdHaP24h07dizS0tKwceNGfPnll/jzn/+MQ4cOdVjnRkSXhzNbRHZiwIABllqlvXv3WmZQrsR///tfADIp2L59O8aPH4/U1FRkZGQ0qy1KS0trdwapqYZf+CdOnAAArFy5EqGhoQgMDOz0XHd3d5SXl7f7vE6ng8FgQFhYGAC0WS/WnaZMmYKlS5cCAMrKyrBmzZo2j+tsvPR6PZYvXw5A/nkVFBQgISEBU6dOxffff2+ZsXzvvfcwdepUKBQKXHfddXj33XdRUVFhuX7TBLstmZmZcHV1xa233oq3334bp0+f7vG7TYn6O85sEdmJl156Cffccw8++ugjjBgxAvHx8Vf8Wo6Ojhg3bhyKiorw9ttvWxKZb7/9Fk899RQef/xxGAwGhIeH4+uvv+709fz8/PDJJ5/gjjvugMlkgqenJ1atWtWlWBYuXIgnnngCr732Gl5++eVWz7u7u+Ovf/0rRo8ejfDwcFx33XWX92Yv07PPPot7770XQ4YMQWRkJMaNGwcPD49Wx3l5eXU4Xj4+Pjhz5gzGjBmDqqoqrFixAi4uLoiPj8crr7yCq6++GgAQFhaG//znPwCAu+66C3l5eRg7diwcHBzg7OyMjRs3dhjv5s2bsXjxYqhUKphMJrz22mttxktEV04hhBC2DoKIqL8wGAwwmUzQarWoqKjA+PHjsXjxYkybNq3Lr5GVlYWRI0fi0qVLVoyUiHoKZ7aIiLpRaWkpZs6cCZPJhNraWtxxxx2XlWgRUf/DmS0iIiIiK2KBPBEREZEVMdkiIiIisiImW0RERERWxGSLiIiIyIqYbBERERFZEZMtIiIiIiv6f3MzjbeD+JmPAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 720x560 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig=plt.figure(figsize=(9, 7), dpi= 80, facecolor='w', edgecolor='k')\n",
    "\n",
    "plt.plot(history['loss'], label = 'training loss')\n",
    "plt.plot(history['val_loss'], label = 'testing loss')\n",
    "\n",
    "plt.title(\"VGG-19 model metric: losses\")\n",
    "plt.xlabel(\"number of training epochs\")\n",
    "plt.ylabel(\"loss\")\n",
    "plt.legend(loc = 'best')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Evaluate trained model on validation data-\n",
    "loss, acc = winning_ticket_model.evaluate(X_test, y_test, verbose=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "VGG-19 CNN (145 epochs training without early stopping) metrics on validation data:\n",
      "loss = 2.8174 and accuracy = 59.1800%\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(\"\\nVGG-19 CNN (145 epochs training without early stopping) metrics on validation data:\")\n",
    "print(\"loss = {0:.4f} and accuracy = {1:.4f}%\\n\".format(loss, acc * 100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train model with .fit_generator() method:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Initialize a VGG-19 Deep CNN-\n",
    "model = vgg19_nn()\n",
    "\n",
    "# Load weights using model from above-\n",
    "model.set_weights(orig_model.get_weights())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Train model using data augmentation-\n",
    "'''\n",
    "history = model.fit_generator(\n",
    "    datagen.flow(X_train, y_train, batch_size = batch_size),\n",
    "    validation_data = (X_test, y_test),\n",
    "    steps_per_epoch = X_train.shape[0] // batch_size,\n",
    "    epochs = num_epochs\n",
    ")\n",
    "'''\n",
    "\n",
    "'''\n",
    "# fits the model on batches with real-time data augmentation:\n",
    "history = model.fit_generator(\n",
    "    datagen.flow(X_train, y_train, batch_size = batch_size),\n",
    "    validation_data = (X_test, y_test),\n",
    "    steps_per_epoch = len(X_train) // batch_size,\n",
    "    epochs = num_epochs\n",
    ")\n",
    "'''\n",
    "\n",
    "'''\n",
    "# Might be WRONG, CHECK-\n",
    "\n",
    "# A more \"manual\" example\n",
    "for e in range(num_epochs):\n",
    "    print(\"\\nCurrent epoch = {0}\".format(e))\n",
    "    batches = 0\n",
    "    for x_batch, y_batch in datagen.flow(X_train, y_train, batch_size = batch_size):\n",
    "        model.fit(x_batch, y_batch)\n",
    "        batches += 1\n",
    "        if batches >= len(X_train) / batch_size:\n",
    "            # we need to break the loop by hand because\n",
    "            # the generator loops indefinitely\n",
    "            break\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/145\n",
      "782/782 [==============================] - 79s 101ms/step - loss: 3.7013 - accuracy: 0.1341 - val_loss: 3.3331 - val_accuracy: 0.1959\n",
      "Epoch 2/145\n",
      "782/782 [==============================] - 44s 56ms/step - loss: 2.9167 - accuracy: 0.2651 - val_loss: 2.7362 - val_accuracy: 0.3099\n",
      "Epoch 3/145\n",
      "782/782 [==============================] - 45s 57ms/step - loss: 2.3784 - accuracy: 0.3725 - val_loss: 2.7275 - val_accuracy: 0.3301\n",
      "Epoch 4/145\n",
      "782/782 [==============================] - 45s 57ms/step - loss: 1.9774 - accuracy: 0.4612 - val_loss: 2.1823 - val_accuracy: 0.4310\n",
      "Epoch 5/145\n",
      "782/782 [==============================] - 45s 57ms/step - loss: 1.6434 - accuracy: 0.5371 - val_loss: 2.0752 - val_accuracy: 0.4577\n",
      "Epoch 6/145\n",
      "782/782 [==============================] - 45s 58ms/step - loss: 1.3508 - accuracy: 0.6075 - val_loss: 2.2561 - val_accuracy: 0.4472\n",
      "Epoch 7/145\n",
      "782/782 [==============================] - 45s 58ms/step - loss: 1.0824 - accuracy: 0.6759 - val_loss: 2.1432 - val_accuracy: 0.4694\n",
      "Epoch 8/145\n",
      "782/782 [==============================] - 45s 57ms/step - loss: 0.8313 - accuracy: 0.7457 - val_loss: 2.2816 - val_accuracy: 0.4717\n"
     ]
    }
   ],
   "source": [
    "# Train CNN using early stopping callback-\n",
    "history = orig_model.fit(\n",
    "        x = X_train, y = y_train,\n",
    "        batch_size = batch_size,\n",
    "        epochs = num_epochs,\n",
    "        verbose = 1,\n",
    "        callbacks = callback,\n",
    "        validation_data = (X_test, y_test),\n",
    "        shuffle = True\n",
    "    )\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Training completed of VGG-19 CNN on CIFAR-100 dataset\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(\"\\nTraining completed of VGG-19 CNN on CIFAR-100 dataset\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Evaluate trained model on validation data-\n",
    "loss, acc = orig_model.evaluate(X_test, y_test, verbose=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "VGG-19 CNN metrics on validation data:\n",
      "loss = 2.2816 and accuracy = 47.1700%\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(\"\\nVGG-19 CNN metrics on validation data:\")\n",
    "print(\"loss = {0:.4f} and accuracy = {1:.4f}%\\n\".format(loss, acc * 100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save trained weights-\n",
    "# orig_model.save_weights(\"VGG19_EarlyStopping_Trained_Weights.h5\", overwrite = True)\n",
    "# orig_model.save_weights(\"VGG10_CIFAR100_EarlyStopping_Trained_Weights.h5\", overwrite=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/majumdar/.local/lib/python3.7/site-packages/tensorflow_core/python/ops/resource_variable_ops.py:1781: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "If using Keras pass *_constraint arguments to layers.\n",
      "INFO:tensorflow:Assets written to: vgg19_cnn_saved_model_dense_layers//vgg19_early_stopping_model_dense_layers/assets\n"
     ]
    }
   ],
   "source": [
    "'''\n",
    "# Save the trained CNN model as a 'SavedModel' format-\n",
    "\n",
    "# Create a directory to save files to-\n",
    "!mkdir -p vgg19_cnn_saved_model_dense_layers\n",
    "\n",
    "orig_model.save('vgg19_cnn_saved_model_dense_layers//vgg19_early_stopping_model_dense_layers')\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os, pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/home/majumdar/The_Lottery_Ticket_Hypothesis-Resources'"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.getcwd()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"/home/majumdar/The_Lottery_Ticket_Hypothesis-Resources/VGG_19_history_main_Early_Stopping.pkl\", 'wb') as f:\n",
    "    pickle.dump(history.history, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load saved model (from before)-\n",
    "saved_vgg19 = tf.keras.models.load_model('vgg19_cnn_saved_model/vgg19_early_stopping_model')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get loaded model summary-\n",
    "# saved_vgg19.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Evalaute loaded model on validataion data-\n",
    "loss, acc = saved_vgg19.evaluate(X_test, y_test, verbose=0)\n",
    "\n",
    "print(\"\\nPre-trained VGG-19 CNN model metrics on validation data:\")\n",
    "print(\"loss = {0:.4f} & accuracy = {1:.4f}%\\n\".format(loss, acc * 100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Global Pruning for VGG-19 CNN:\n",
    "\n",
    "_The Lottery Ticket Hypothesis_ states that for VGG-19 CNN instead of using a _layer-wise_ magnitude based pruning strategy, a __global__ magnitude based pruning strategy is used.\n",
    "\n",
    "This means that for all of the approximately 20 million parameters, prune _p%_ of the smallest magnitude weights, irrespective of the conv layer the weight belongs to.\n",
    "\n",
    "Possible approach to _global pruning_:\n",
    "\n",
    "1. Store all prunable parameters to __one-dimensional__ numpy array using (x, y, z, a) reshaped to (x * y * z * a)\n",
    "1. Prune _p%_ of smallest weight based magnitude from this __one-dimensional__ numpy array\n",
    "1. Restore the pruned parameters for each conv layer following reverse of _step 1_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "def prune_vgg19(model, pruning_params_conv):\n",
    "    '''\n",
    "    Inputs:\n",
    "\n",
    "    'model' is the TensorFlow 2.0 defined convolutional neural network\n",
    "    'pruning_params_conv' is the percentage of weights to prune for convolutional layer\n",
    "\n",
    "    Returns:\n",
    "    Python list containing pruned layers\n",
    "    '''\n",
    "\n",
    "    # Python 3 list to hold layer-wise weights-\n",
    "    pruned_weights = []\n",
    "\n",
    "    for layer in saved_vgg19.trainable_weights:\n",
    "        x = layer.numpy()\n",
    "        pruned_weights.append(x)\n",
    "\n",
    "    # Flatten all numpy arrays-\n",
    "    pruned_weights_flattened = [layer.flatten() for layer in pruned_weights]\n",
    "\n",
    "    # Compute the smallest weights to prune based on 'p' parameter-\n",
    "    p = 20  # 20% of smallest global weights to be pruned considering ALL conv layers\n",
    "\n",
    "    threshold = np.percentile(abs(np.concatenate(pruned_weights_flattened)), pruning_params_conv)\n",
    "    # print(\"\\nFor p = {0:.2f}% of weights to be pruned, threshold = {1:.4f}\\n\".format(p, threshold))\n",
    "\n",
    "\n",
    "    # np.concatenate(flattened).shape\n",
    "    # (4032,)\n",
    "\n",
    "    # Prune weights (set values to 0) when smaller than threshhold for CONV layers ONLY-\n",
    "    for layer in pruned_weights:\n",
    "        if len(layer.shape) == 4:\n",
    "            layer[abs(layer) < threshold] = 0\n",
    "\n",
    "\n",
    "    return pruned_weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Initialize a new VGG-19 CNN-\n",
    "pruned_vgg = pruned_nn()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Prune weights based on percentage of weights to prune-\n",
    "pruned_weights = prune_vgg19(pruned_vgg, 20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load Python 3 pruned weights as list-\n",
    "pruned_vgg.set_weights(pruned_weights)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Total number of trainable parameters in pruned VGG-19 CNN = 16024868\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Sanity check-\n",
    "pruned_sum_params = 0\n",
    "    \n",
    "for layer in pruned_vgg.trainable_weights:\n",
    "    # print(tf.math.count_nonzero(layer, axis = None).numpy())\n",
    "    pruned_sum_params += tf.math.count_nonzero(layer, axis = None).numpy()\n",
    "\n",
    "    \n",
    "print(\"\\nTotal number of trainable parameters in pruned VGG-19 CNN = {0}\\n\".format(pruned_sum_params))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1257"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "16024868 - 16023611"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}