NBSVM.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Creating baseline for the project using NBSVM (Naive Bayes - Support Vector Machine)\n",
    "\n",
    "NBSVM was introduced by Sida Wang and Chris Manning in the paper [Baselines and Bigrams: Simple, Good Sentiment and Topic Classiﬁcation](https://nlp.stanford.edu/pubs/sidaw12_simple_sentiment.pdf)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Observations:\n",
    "\n",
    "- Dataset is multiclass **not** only multilabel"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Constants"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-04T23:20:35.009554Z",
     "start_time": "2018-04-04T23:20:35.006977Z"
    }
   },
   "outputs": [],
   "source": [
    "TRAIN_DATA_PATH = \"downloads/train.csv.zip\"\n",
    "TEST_DATA_PATH = \"downloads/test.csv.zip\"\n",
    "label_cols = ['toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate']\n",
    "comment_col = 'comment_text'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-05T00:39:47.225115Z",
     "start_time": "2018-04-05T00:39:47.078402Z"
    }
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.linear_model import SGDClassifier\n",
    "from sklearn.svm import LinearSVC\n",
    "from sklearn.svm import SVC\n",
    "\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.pipeline import FeatureUnion\n",
    "\n",
    "from sklearn.base import BaseEstimator, ClassifierMixin\n",
    "\n",
    "from sklearn.utils.validation import check_X_y\n",
    "from sklearn.utils.validation import check_array\n",
    "from sklearn.utils.validation import check_is_fitted\n",
    "from sklearn.utils.multiclass import unique_labels\n",
    "\n",
    "from sklearn.model_selection import cross_validate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-04T23:20:37.763397Z",
     "start_time": "2018-04-04T23:20:35.760490Z"
    }
   },
   "outputs": [],
   "source": [
    "train = pd.read_csv(TRAIN_DATA_PATH)\n",
    "test = pd.read_csv(TEST_DATA_PATH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-05T09:21:36.153421Z",
     "start_time": "2018-04-05T09:21:35.447264Z"
    }
   },
   "outputs": [],
   "source": [
    "COMMENT = 'comment_text'\n",
    "train[COMMENT].fillna(\"unknown\", inplace=True)\n",
    "test[COMMENT].fillna(\"unknown\", inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-05T09:36:41.747627Z",
     "start_time": "2018-04-05T09:36:41.730568Z"
    }
   },
   "outputs": [],
   "source": [
    "class NbSVMClassifier(BaseEstimator, ClassifierMixin):\n",
    "    \n",
    "    def __init__(self, modelType = 'lg', **kwargs):\n",
    "        self.modelType = modelType\n",
    "        self.modelArgs = kwargs\n",
    "        \n",
    "    def fit(self, X, y):\n",
    "        \n",
    "        # Check that X and y have correct shape\n",
    "        y = y.values\n",
    "        X, y = check_X_y(X, y, accept_sparse=True, multi_output=True)\n",
    "        \n",
    "        self.classes_ = unique_labels(y)\n",
    "        \n",
    "        self._clf = []\n",
    "        self._r = []\n",
    "        \n",
    "        def cr(X, y, y_i):\n",
    "            p = X[y == y_i].sum(axis = 0)\n",
    "            return (1 + p)/ ((y == y_i).sum() + 1)\n",
    "        \n",
    "        for i in self.classes_:\n",
    "            print('Fitting Model for: ', label_cols[i])\n",
    "            y_i = y[:, i]\n",
    "            log_count_ratio = np.log(cr(X, 1,y_i) / cr(X, 0, y_i))\n",
    "            X_enhanced = X.multiply(log_count_ratio)\n",
    "            \n",
    "            if self.modelType == 'lg':\n",
    "                model = LogisticRegression(**self.modelArgs)\n",
    "            elif self.modelType == 'svm':\n",
    "                model = SGDClassifier(**self.modelArgs)\n",
    "                \n",
    "            self._clf.append(model.fit(X_enhanced, y_i))\n",
    "            self._r.append(log_count_ratio)\n",
    "        \n",
    "        return self\n",
    "    \n",
    "    def predict(self, X):\n",
    "        check_is_fitted(self, ['_r', '_clf'])\n",
    "        predict = np.zeros((X.shape[0], len(self.classes_)))\n",
    "        for i in range(predict.shape[1]):\n",
    "            predict[:, i] = self._clf[i].predict(X.multiply(self._r[i]))\n",
    "        return predict\n",
    "    \n",
    "    def predict_proba(self, X):\n",
    "#         if self.modelType == 'svm':\n",
    "#             print('No Probabilistic Interpretation for SVM')\n",
    "#             return None\n",
    "        check_is_fitted(self, ['_r', '_clf'])\n",
    "        predict_proba = np.zeros((X.shape[0], len(self.classes_)))\n",
    "        for i in range(predict_proba.shape[1]):\n",
    "            if self.modelType == 'svm':\n",
    "                predict_proba[:, i] = self._clf[i].decision_function(X.multiply(self._r[i]))\n",
    "            else:\n",
    "                predict_proba[:, i] = self._clf[i].predict_proba(X.multiply(self._r[i]))\n",
    "        return predict_proba"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-05T09:36:45.652138Z",
     "start_time": "2018-04-05T09:36:45.639013Z"
    }
   },
   "outputs": [],
   "source": [
    "pipeline = Pipeline([\n",
    "    ('tfidf', TfidfVectorizer(ngram_range=(1,2), \n",
    "                      min_df = 3, \n",
    "                      max_df = 0.9, \n",
    "                      strip_accents = 'unicode', use_idf=1,\n",
    "                      smooth_idf=1, sublinear_tf=1)),\n",
    "    ('clf', NbSVMClassifier(modelType = 'svm', loss = 'hinge', class_weight = 'balanced', tol = 1e-6))\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-05T00:37:22.936865Z",
     "start_time": "2018-04-04T23:46:51.896278Z"
    }
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import learning_curve\n",
    "train_sizes, train_acc_folds, val_acc_folds = learning_curve(estimator= pipeline, \n",
    "                                                     X = train[comment_col],\n",
    "                                                     train_sizes=np.linspace(0.1, 1, 20),\n",
    "                                                     y =  train[label_cols], cv = 10,\n",
    "                                                     scoring = \"accuracy\",\n",
    "                                                     shuffle = True, n_jobs = 3)\n",
    "\n",
    "train_acc = np.mean(train_acc_folds, axis = 1)\n",
    "train_val = np.mean(val_acc_folds, axis = 1)\n",
    "print([' %.4f' % score for score in train_val])\n",
    "print(\"\\nAccuracy: %0.4f (+/- %0.4f)\" % (np.mean(train_val), np.std(train_val) * 2))\n",
    "fig, ax = plt.subplots(nrows = 1, ncols = 1, figsize = (8, 6))\n",
    "ax.plot(train_sizes, train_acc, lw = 3, color = \"steelblue\", label = \"training\")\n",
    "ax.plot(train_sizes, train_val, lw = 3, color = \"green\", label = \"validation\")\n",
    "\n",
    "ax.grid(alpha=0.25)\n",
    "ax.set_xlabel(\"training set size\", fontsize=16)\n",
    "ax.set_ylabel(\"Accuracy\", fontsize=16)\n",
    "ax.legend(loc=\"upper right\", fontsize=16)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-05T00:42:44.766630Z",
     "start_time": "2018-04-05T00:42:44.486650Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[' 0.9168', ' 0.9180', ' 0.9191', ' 0.9197', ' 0.9194', ' 0.9201', ' 0.9202', ' 0.9203', ' 0.9203', ' 0.9204', ' 0.9201', ' 0.9201', ' 0.9203', ' 0.9202', ' 0.9204', ' 0.9203', ' 0.9204', ' 0.9205', ' 0.9205', ' 0.9204']\n",
      "\n",
      "Accuracy: 0.9199 (+/- 0.0018)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x7f4c9fe77940>"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgAAAAF8CAYAAAC5cAPdAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzs3Xd8leXd+PHPN3sTEpIQQhabMASJLFnOMkTFVVFQrKNabftrtU+1rVZ96nj6dD3uIrWgVZxgtaCiIrLClCWEEUIC2SF7r3P9/jiHQwJBDpicc5Lzfb9eeXHu677uc38v1vme676GGGNQSimllGfxcnUASimllHI+TQCUUkopD6QJgFJKKeWBNAFQSimlPJAmAEoppZQH0gRAKaWU8kCaACillFIeSBMApZRSygNpAqCUUkp5IE0AlFJKKQ/k4+oAOlOvXr1MUlKSq8M4TUtLC97e3q4OwyW07dp2T+OpbffUdoPr2759+/bjxpios9Xr1glAUlIS27Ztc3UYpykvLyc8PNzVYbiEtl3b7mk8te2e2m5wfdtFJNuRevoIQCmllPJAmgAopZRSHkgTAKWUUsoDaQKglFJKeSBNAJRSSikPpAmAUkop5YG69TRApZTqiiorKykqKqKpqcnVoZw3i8VCfn6+q8Nwic5qu6+vL9HR0YSFhXXI+2kCoJRSbqSyspLCwkLi4uIIDAxERFwd0nlpbm7Gx8czP2I6o+3GGOrq6sjNzQXokCRAHwEopZQbKSoqIi4ujqCgoC774a86nogQFBREXFwcRUVFHfKemgAopZQbaWpqIjAw0NVhKDcVGBjYYY+GNAE4Bzkl1ezOLnF1GEqpbk6/+asz6ci/G575gOYc1dQ38frXB/l4WzYRIf784yfT8Pf1zE0ulFJKdQ/aA+AIga++zaPFYiiurOeDTZmujkgppbqMDz/8kL/85S8d/r4LFizgfHd8TUpKYsGCBR0aT1ejCYADgv19uX3aIPvxOxsOU1JV78KIlFKq6+isBODRRx9l+fLl53Xt8uXLefTRRzs4oq5FEwAHTR8dT1JUKAD1TS0sWXPAxREppVT30tDQcE71+/fvz+jRo8/rXqNHj6Z///7ndW13oQmAg7y9vLjnyqH241U7cziUX+HCiJRSyv0tWLCAJUuWkJubi4ggIiQlJbFmzRpEhGXLlnH33XcTFRVFTEwMABkZGcyfP5/k5GQCAwPp168f9913H2VlZae9d+tHAFlZWYgIf//733nssceIjY0lPDyc2bNnk5OT0+baUx8BLF68GBFh06ZN3HrrrYSFhdGnTx9+9rOfUV/ftsc3MzOTmTNnEhQURHR0NA8++CALFy5ERMjKyurQ37/OpIMAz8GYflGMHRjNlkNFGGDh5/v44/zxOmJXKaXO4NFHH6W4uJitW7fy0UcfAeDv709FhfUL1E9/+lNmzJjBG2+8Yf+gzcvLIz4+nr/97W/07NmTzMxMnn76aWbOnElaWtpZ7/nMM88wceJEXnvtNYqKinjwwQeZN28ea9asOeu18+fPZ+7cuSxbtoy0tDQef/xxevbsyRNPPAFAY2MjV1xxBQ0NDbz88stERUWxaNEi3n///fP8HXIdTQDO0d2XD2X74WJaLIbd2aVsPFDIxUN6uzospVQ39oP/XuHqEOw+e3TWOdXv378/UVFR+Pn5MX78eHv5iQ/jsWPHsmjRojbXTJkyhSlTptiPJ06cyIABA5g8eTI7duw4a7d/UlISb731lv24uLiYX/3qV+Tl5dGnT5/vvPaWW26xf9hffvnlbN68maVLl9rLFi9eTGZmJps3b2bs2LEAzJgxg1GjRnH06NGz/G64F30EcI4SeoVw1ZhE+/GrX6TT2NziwoiUUqrrmjNnzmlljY2NPP300wwZMoTAwEB8fX2ZPHkyAAcOnH381cyZM9scjxgxAsChD+hZs9omOCNGjGhz3aZNm0hISLB/+IN1bv71119/1vd2N5oAnId5UwYSEuALQH5ZLR9tzXZxREop1TXFxsaeVvbII4/w+OOPM2/ePFasWMGWLVtYtmwZwGnP49sTERHR5tjf3/97Xdt6cGJ+fj7R0dGnXXdi/EJXoo8AzkNYkB/zpgzklVX7AHhr3SEuHxlHeLC/iyNTSnVH59rt3pW0N4bq7bff5rbbbuN3v/udvay6utqZYZ1RbGws+/btO628sLDQBdF8P07vARCR6SJyQEQyROThds4nisiXIrJbRNaISN9W5/5HRL61/fzQuZG3NTs1kb4RwQDUNDTzr7WHXBmOUkq5LX9/f+rq6hyuX1tbi6+vb5uyf/7znx0d1nkZP348R48eZcuWLfYyYwwffPCBC6M6P05NAETEG3gRmAGkAHNFJOWUan8CXjfGjASeBJ6xXTsLuBAYBYwDHhKRjtkU+Tz4eHtx9xUnpwWu2H6U7OIqV4WjlFJuKyUlhdLSUl5++WW2bt3Knj17vrP+9OnTWbJkCS+99BKrVq3i3nvvZePGjU6K9rstWLCAfv36cd1117FkyRJWrlzJ9ddfb5+i6OXVdZ6sOzvSsUCGMSbTGNMIvA1cc0qdFGC17fVXrc6nAGuNMc3GmBpgNzDdCTGf0biB0YxKjgTAYgwLP093ZThKKeWW7rrrLm6++WZ+85vfMHbsWGbPnv2d9Z9//nmuvvpqfvvb3/LDH/6Qqqoqli5d6qRov5ufnx+rVq1i5MiR3Hvvvdx+++3Ex8dz//33A9CjRw8XR+g4McY472YiNwDTjTF32Y7nA+OMMQ+0qvMWsNkY838ich3wAdALGAP8HrgCCAK2AC8aY/58pvulpqaabdu2dVp7ADILK7n/1XVYbL+Nf5h7ERcNOH2ASGvl5eWEh4d3alzuStuubfc059r29PR0hg4devaKbq65uRkfH88ZZnbVVVeRnp7O4cOHO73tZ/s7IiLbjTGpZ3sfd/zTeQh4QUQWAGuBXKDFGLNKRC4CNgLFQBpw2vw7EbkHuAcgPj6e8vLyTg02wh8uSYnhy73WASCvfPYt/SJG4+115sWBqqo891GBtt0zadsdZ7FYaG5u7qRonKe7tKM9f/3rXwkJCWHgwIFUVVXx/vvvs2LFCl544QWam5s7ve0Wi6VDPtucnQDkAvGtjvvayuyMMXnAdQAiEgJcb4wpt517CnjKdu4t4OCpNzDGLAQWgrUHwBnfOu6+cgRph0qobWwmp7SOjZkVzE5N+s5rPPXbEGjbPZW23TH5+fnd4ptzd+4BCAoK4rnnnuPo0aO0tLQwePBgFi1axJ133gl0ftu9vLw65N+Ts8cAbAUGikiyiPgBNwMfta4gIr1E5ERcjwCv2cq9RSTS9nokMBJY5bTIv0PPEH9unnRyU4nX1xykur7JhREppZTqLPfffz/p6enU1NRQX1/Prl277B/+XYlTEwBjTDPwAPAZkA68a4zZKyJPisjVtmrTgAMichCIwfaNH/AF1onIPqzf8OfZ3s8tzBmXTEx4IACVdU28tU6nBSqllHJfTu+fMcasBFaeUvZYq9fvA6ftqmCMqcc6E8At+fl4c+elQ3h62Q4A/r0li1ljEomzrRWglFJKuZOuM2GxC5iSEsuw+J4ANFsM//hCpwUqpZRyT5oAdCAR4cdXnuyk2HCgkF1ZJS6MSCmllGqfJgAdbHCfcC4bEWc/Xvj5PloszltrQSmllHKEJgCd4I5LB+PvY/2tzSio5IvdOS6OSCmllGpLE4BOEBUWyA0TTk4LXPzVAeoa3WbCglJKKaUJQGe5aWI/IkOt2wOXVjfw7obDLo5IKaW6tqysLESExYsX28sWLFhAUlLSWa9dvHgxIkJWVtY53bO8vJzHH3+cb7755rRz06ZNY9q0aef0fu5EE4BOEuDnwx2XDLEfv78pk6IKx7fDVEopdXaPPvooy5cv77T3Ly8v54knnmg3AXjppZd46aWXOu3enU0TgE502cg4BsZad4ZqbLbwjy/3uzgipZTqXvr378/o0aNdcu+UlBRSUtx2eZqz0gSgE3mdMi1wzd480nPKXBiRUko513vvvYeIsHv37tPOzZw5kwsuuACAF154gQkTJhAREUF4eDjjx49nxYoVZ33/9h4BZGZmMmvWLIKCgoiKiuLnP/85DQ0Np1379ttvc+mllxIVFUVISAijR49myZIl9vNZWVkkJycDcPfddyMibR5BtPcI4MCBA9xwww2Eh4cTGBjI+PHj+fTTT9vUefzxxxERDh06xKxZswgJCSExMZEnn3wSi8Vy1jZ3FE0AOtmIhAgmDeltP/77qn04cwtmpZRypdmzZ9OjRw/+9a9/tSkvLCxk1apV3HbbbYD1w/auu+7ivffe45133iE1NZWrrrrqtA/Ps2lsbOSKK65gx44dvPjiiyxevJgjR47whz/84bS6mZmZ3HDDDbz55pt8+OGHzJ49m7vuuotXXnkFgNjYWJYtWwbAI488QlpaGmlpacyaNavde+fl5TFp0iR2797NCy+8wLvvvkt4eDizZs3ik08+Oa3+nDlzuPTSS/nwww+59tpr+f3vf98mAels3XOrJjdz1+VD2XyoiKYWC+m55Ww8dJxZY3u6OiylVBchT5x5e3FnM78/ty8wAQEB3Hjjjbz11ls8++yzeHlZv3cuXboUgFtuuQWAP/3pT/ZrLBYLl112GQcPHuTll19m+vTpDt9vyZIlZGZmkpaWxvjx4wGYMWMGI0aMOK3ub37zmzb3nDZtGvn5+bz88svce++9+Pv72x8v9OvXz/5+Z/KXv/yFsrIy1q1bx5Ah1jFgM2fOJCUlhd/+9rfMmDGjTf0HH3yQO+64A4DLL7+c1atXs3TpUntZZ9MeACeI7RnEtWOT7MdvbsiioanFdQEppZQT3XbbbeTm5rJ69Wp72RtvvMFll11GbGwsANu3b+eqq64iJiYGHx8ffH19+fzzzzlw4MA53SstLY34+Pg2H9ZeXl7cdNNNp9U9dOgQc+fOJS4uDl9fX3x9fVm0aNE53/OEtWvXMn78eAYMGGAv8/b2Zu7cuezcuZPKyso29U/tSRg+fDhHjx49r3ufD00AnGTupAH0CPIDoKS6kQ82Zbo4IqWUco5JkyaRlJTEG2+8AUB6ejrffPONvfv/2LFjXHbZZZSWlvL888+zceNGtm7dyvTp06mvrz+ne+Xn5xMTE3Na+all1dXVXHHFFezatYtnn32WdevWsXXrVn70ox+1O17AEaWlpfaEprXevXtjjKGsrO0YsIiIiDbH/v7+59ze70MfAThJcIAvt00bxPMrvwXgnQ2H+cGoeCJDA1wcmVLK3Z1rt7u7ERHmzZvH3/72N15++WXeeOMNQkJCmDNnDgCffvopFRUVvPvuu/Tt29d+XW1t7TnfKzY2lr17955WXlhY2OY4LS2N7Oxs1q1bx6RJk+zlzc3nv2hbREQEBQUFp5UXFBQgIvTs6V6PfrUHwIlmjI4nMSoEgPqmFpasOb9uJqWU6mrmz59PdXU1y5Yt48033+S6664jKCgIOPlB7+vra69/8OBBNmzYcM73mTBhAseOHWPTpk32MovFwrvvvtumXnv3LCsr49///nebev7+1gXd6urOvo7L1KlT2bRpU5vFhlpaWnjnnXcYPXo0YWFh59yezqQJgBN5e3m1mRa4amcOGfkVLoxIKaWcY9CgQYwbN46HH36Yo0eP2rv/wToAzsfHh9tuu41Vq1axZMkSrrzyShISEs75Prfffjv9+vXjuuuuY/HixaxcuZJrr732tOfvEydOJCwsjPvvv58VK1bw7rvvMnXqVHr16tWmXkxMDJGRkbz99tt8/fXXbNu2jZKS9nd5/cUvfkF4eDgzZszgrbfe4j//+Q+zZ8/m4MGDPPXUU+fcls6mCYCTjekXxYVJ1m4gA/z9c50WqJTyDPPnzyc3N5e4uDguueQSe/mwYcN48803yc7O5uqrr+aPf/wjzz77LFOmTDnne/j5+fH5558zatQofvKTn3D77beTnJzM7373uzb1oqKiWL58OS0tLdxwww088sgj3HXXXcybN69NPS8vLxYtWkRZWRmXX345F110ER9//HG79+7Tpw/r168nJSWF++67jxtuuIHS0lJWrFhxTjMZnEW684dPamqq2bZtm6vDOM3ezDx+tXSnfZvg3984homt1grozsrLywkPD3d1GC6hbde2OyI9PZ2hQ4d2YkTO0dzcjI+PZw4z6+y2n+3viIhsN8aknu19tAfABeIigrhqTKL9+NUv02lqcd7qT0oppZQmAC4yb8pAQgKsg0/ySmv5cPMRF0eklFLKk2gC4CJhQX7cOmWg/fhfaw/pboFKKaWcRhMAF7o6NZGkqFDAOi3w5c9On7uqlFJKdQZNAFzIx9uLn80abj/eeKCQtAOF33GFUkop1TE0AXCxYfERTB8Vbz9+6bO91Dee/0pUSqmurzvPzlLfT0f+3dAEwA3cedkQwgKtAwKLKur419pDLo5IKeUqvr6+Dq06pzxTXV1dm9ULvw9NANxAWJAfd19xck7nss1HyCqqcmFESilXiY6OJjc3l9raWu0JUHbGGGpra8nNzSU6OrpD3tMzV2lwQ1eM7MuqnTnsOVpKi8Xw3Mo9/On2CXiJ++wDrpTqfCfWi8/Ly6OpqcnF0Zw/i8WCl5dnfsfsrLb7+voSExPTYXsKaALgJkSEn84czn0L19FiMew9Vsbnu3L4QavxAUopzxAWFuZ2G8ecK1390f3b7pnpmZtKjArlhgn97MevfpFORW2jCyNSSinVXWkC4GZumTyQmPBAAKrqmlj0RbqLI1JKKdUdOT0BEJHpInJARDJE5OF2zieKyJcisltE1ohI31bn/igie0UkXUSeE+l+D8gDfL15YPrJtQFW7cphT3b7W08qpZRS58upCYCIeAMvAjOAFGCuiKScUu1PwOvGmJHAk8AztmsnAhcDI4HhwEXAVCeF7lRjB0YzqdXugM+t/FY3C1JKKdWhnN0DMBbIMMZkGmMagbeBa06pkwKstr3+qtV5AwQAfoA/4At022Xz7v1BCoF+3gAcPV7Nsk2ZLo5IKaVUd+LsBCAOONbqOMdW1tou4Drb6zlAqIhEGmPSsCYE+bafz4wx3fYBeVRYILdNHWQ/fnPtIQrKal0YkVJKqe7EHacBPgS8ICILgLVALtAiIgOAocCJMQGfi8hkY8y61heLyD3APQDx8fGUl5c7LXBHVVU5tsjP1EE9+XRHMNnHa2hotvC3j3fy69lD6cpDHxxte3ekbfdMntp2T203dJ22OzsByAVaT2zvayuzM8bkYesBEJEQ4HpjTLmI3A1sMsZU2859AkwA1p1y/UJgIUBqaqpx17mYjsb1y6sv4P+9thED7MguY29BPZOGxnZucJ3MXf9MnEHb7pk8te2e2m7oGm139iOArcBAEUkWET/gZuCj1hVEpJeInIjrEeA12+ujwFQR8RERX6wDALvtI4AThsT1ZOaYBPvxy5/to7ZBNwtSSin1/Tg1ATDGNAMPAJ9h/fB+1xizV0SeFJGrbdWmAQdE5CAQAzxlK38fOAzswTpOYJcx5mNnxu8qd1wyhPBgPwCOV9XzxtcHXRyRUkqprs7pYwCMMSuBlaeUPdbq9ftYP+xPva4F+HGnB+iGQgN9+fEVKfzPhzsB+HDLES4fGUf/3j1cHJlSSqmuSlcC7CIuGd6HUUmRAFiMdW0Ai+4UppRS6jxpAtBFiAgPzBiOr7f1j2x/bjmffHPUxVEppZTqqjQB6ELie4Vw08T+9uPXVu+nrLrBhREppZTqqjQB6GJuntSf2J5BAFTXN/OqbhaklFLqPGgC0MX4+Xjz0xknNwv6ck8uO48cd2FESimluiJNALqgMf2jmJpycjGg51d+S2NziwsjUkop1dVoAtBF/fjKFIL8rbM4c0preG+jbhaklFLKcZoAdFGRoQEsuGSw/Xjp+gxyS2tcGJFSSqmuRBOALuyqMYkMjLUuBtTUYuHFT77F6NoASimlHKAJQBfm7SX8fNYIvGybA27PPM7X+/JdG5RSSqkuQROALm5gbA9mpybZj/++ah819U2uC0gppVSXoAlAN3D7tEFEhPgDUFrdwOI1B1wckVJKKXenCUA3EBzgy71XptiPP96azcG8chdGpJRSyt1pAtBNTEmJZUz/KAAM1s2CWiw6IFAppVT7NAHoJkSEB6YPs28WdCi/gv9sy3JtUEoppdyWJgDdSJ+IYOZOGmA/fm31AXJLdG0ApZRSp9MEoJu5cWI/EnqFAFDf1MKzy3fQ1GJxcVRKKaXcjSYA3Yyfjze/vnYUPrbFAQ7mV/D6moMujkoppZS70QSgGxoQ24MfXTbEfvzexsO6Y6BSSqk2NAHopuaMS2ZMv16AdVbAH/+9k8raRtcGpZRSym1oAtBNeYnw4NUX0CPID4CSqgb++p/duleAUkopQBOAbi0yNIBfzh5pP954oJCV3xx1YURKKaXchSYA3dz4QTHMTk20H/991T6OFle5MCKllFLuQBMAD3D35UNJjLJODWxotvDM8p00Nre4OCqllFKupAmAB/D39eaROaPtqwRmFlbyz9W6YZBSSnkyTQA8RHJMGHdfMdR+vGzzEbZmFLkwIqWUUq6kCYAHuTo1kbEDo+3Hf/5oN+U1DS6MSCmllKtoAuBBRIQHZ4+kZ7A/AGU1Dfz5o106NVAppTyQJgAeJjzYn4euucB+vCWjmI+2ZrkuIKWUUi6hCYAHSu0fxXXjk+3Hr36xn8zCShdGpJRSytmcngCIyHQROSAiGSLycDvnE0XkSxHZLSJrRKSvrfwSEdnZ6qdeRK51dvzdxR2XDKZ/TBgATS0Wnlm2g4YmnRqolFKewqkJgIh4Ay8CM4AUYK6IpJxS7U/A68aYkcCTwDMAxpivjDGjjDGjgEuBWmCV04LvZvx8vHl4zij8fax/BY4er+bVL9JdHJVSSilncXYPwFggwxiTaYxpBN4GrjmlTgqw2vb6q3bOA9wAfGKMqe20SD1AQlQo9/5gmP34423ZbDpY6MKIlFJKOYuzE4A44Fir4xxbWWu7gOtsr+cAoSISeUqdm4GlnRKhh5kxOp6LB8fYj//80S5KqupdGJFSSiln8HF1AO14CHhBRBYAa4FcwP5wWkRigRHAZ+1dLCL3APcAxMfHU15e3tnxnrOqKvdai/+OyYmk55RRWtNIZV0Tz3ywjd9cMwwvkQ6/l7u13Zm07Z7JU9vuqe2GrtN2ZycAuUB8q+O+tjI7Y0weth4AEQkBrjfGtP4UvwlYboxpau8GxpiFwEKA1NRUEx4e3nHRdyB3iis8HH49ZzQP/2szBthzrILV+8u4YUK/Trqf+7Td2bTtnslT2+6p7Yau0XZnPwLYCgwUkWQR8cPalf9R6woi0ktETsT1CPDaKe8xF+3+73Cjkntx08T+9uN/rt7PofwKF0aklFKqMzk1ATDGNAMPYO2+TwfeNcbsFZEnReRqW7VpwAEROQjEAE+duF5EkrD2IHztxLA9xvxpgxgU2wOAZovh2WU7qG9sdnFUSimlOoPT1wEwxqw0xgwyxvQ3xjxlK3vMGPOR7fX7xpiBtjp3GWMaWl2bZYyJM8ZYnB23J/D19uLhOaMJ8PUGIKe0hpdX7XNxVEoppTqDrgSo2oiLDOb+GSenBn664xjr0vNdGJFSSqnOoAmAOs0VI/syNSXWfvy3/+yhqKLOhREppZTqaJoAqNOICD+bNYLoHoEAVNc38b//3kmLRXcNVEqp7kITANWukABf/uvaUXjZlgLYnV3KexsPuzYopZRSHUYTAHVGIxIimDtpoP14yZqD7M8tc2FESimlOoomAOo73TplAEP7Whe0sBjDs8t3UlnX6OKolFJKfV+aAKjv5O3lxcPXjibI37poZH5ZLb9/exv1unWwUkp1aZoAqLPq3TOIX1w10n68L6eMp97fTnOLLseglFJdlSYAyiFTUmK598oU+/GWjGL+8vFuLEZnBiilVFekCYBy2JxxycydNMB+/OWeXBZ+no7RJEAppbocTQDUObl92iBmXphgP16++QjvbNDpgUop1dVoAqDOiYjwwIzhTBrS2172z68OsPKboy6MSiml1LnSBECdM28v4ddzRjEqKdJe9vzKPazXPQOUUqrL0ARAnRc/H28eu2kMA3qHAWAx8OzynezMOu7iyJRSSjlCEwB13oL9fXnqlrHERQQD0NRi4Yl3tnMov8LFkSmllDobTQDU9xIe7M/Tt44lMtQfgNrGZn771hZyS2pcHJlSSqnvogmA+t56hwfx9C3jCAmwrhZYUdvII29tpqSq3sWRKaWUOhNNAFSHSIoO5cmbL8Lfx/pXqrC8jt+8uYWquiYXR6aUUqo9DiUAIiKdHYjq+obFR/C7G8bgbdtDOKu4it+/s1X3DVBKKTfkaA9Atog8KiJ9OjUa1eWNHRjNg7NP7huw91gZT33wje4boJRSbsbRBGA18DCQJSLLROTKToxJdXGXjezLj1vvG3CoSPcNUEopN+NQAmCMWQD0AR4CBgGfishhEfm1iER1Ynyqi7puXDI3X9zffvzlnlxe1X0DlFLKbTg8CNAYU2GMec4YMxyYCmwEHgeOicjbIjKtc0JUXdWCSwYzY3S8/XjZ5iP8e3uuCyNSSil1wvnOAtgALAd2An7AbOBLEdkiIkM7KjjVtYkIP505gotb7RuwNC2bT3bovgFKKeVq55QAiEi8iDwJHAXeBcqBa4BQYDoQCCzp6CBV1+XtJTw8ZxQXtNo34LkVe9iwv8CFUSmllHJ0GuBsEfkPkAn8BFgKDDLGzDDGfGyMsRhjPgd+CYzqvHBVV+Tn483vT9k34JllO9iVVeLiyJRSynM52gPwbyAKuAuIM8b8yhiT2U69w8CbHRWc6j5O7BvQu0cAYN034PF3tpGh+wYopZRLOJoApBpjxhljlhhjGs5UyRiTaYy5o4NiU91MeLA/v712GBEhrfYNWKr7BiillCs4mgAcE5FB7Z0QkUEi0qsDY1LdWHRYAM/cenLfgPIa674BxZV1Lo5MKaU8i6MJwEvAg2c49wvbeaUc0t6+AT9/bQMH88pdHJlSSnkORxOAScBnZzi3CrjY0RuKyHQROSAiGSLycDvnE0V9zk13AAAgAElEQVTkSxHZLSJrRKRvq3MJIrJKRNJFZJ+IJDl6X+VehsVH8NsbLrTvG1BS1cBDS9L4em+eiyNTSinP4GgC0BM402itSiDyDOfaEBFv4EVgBpACzBWRlFOq/Ql43RgzEngSeKbVudeB/zXGDAXGAkUOxq/c0LiBMTx1y1hCAnwBaGi28PSyHbzx9UFdNlgppTqZowlADjDuDOfGAfkOvs9YIMM2WLAReBvrOgKtpWDdewDgqxPnbYmCj226IcaYamNMrYP3VW5qdHIvnvvRxfSNCLaX/WvtIZ7+YIfuIqiUUp3I0QTgfeAREZnVutB2/DDWRYEcEQcca3WcYytrbRdwne31HCBURCKx7kFQbtuMaIeI/K+tR0F1cXGRwfztRxdzYb+TY0nXpefz0JI0jlfWuzAypZTqvnwcrPckMAX4SEQKgFysH9y9gU3AEx0Y00PACyKyAFhru1eLLdbJwGisKxG+AywA/tH6YhG5B7gHID4+nvJy9xtYVlVV5eoQXOa72v7QjEG8vs6XT3dbO5QO5VfwwKK1PDRrKANiQp0VYqfRP3fP5Klt99R2Q9dpu0MJgDGmVkSmAvOBK7A+88/AOgDwX8aYZgfvlwvEtzruaytrfa88bD0AIhICXG+MKReRHGDniQWIRORDYDynJADGmIXAQoDU1FQTHh7uYGjO5a5xOcN3tf0X1/RkUN9sXvx0Ly0WQ1lNE08s+5YHZ1/AtOF9nBhl59A/d8/kqW331HZD12i7oz0AGGOagNdsP+drKzBQRJKxfvDfDNzSuoJtTYFSY4wFeKTV/bYC4SISZYwpBi4Ftn2PWJSbmjUmkbjIYP77vW+orm+isdnCM8t3kH28ivlTB+El4uoQlVKqyzvf3QDPi62n4AGsUwrTgXeNMXtF5EkRudpWbRpwQEQOAjHAU7ZrW7A+HvhSRPYAArzqzPiV84xK6sVzd15MfOTJwYFvrcvgqfe/ob7R0Q4npZRSZ+JwD4CIXAncBwwGAk45bYwx/R15H2PMSmDlKWWPtXr9PtZBh+1d+zkw0tGYVdcWF2EdHPj0sh1sP1wMwPr9BRSU1/L4D1OJCgt0cYRKKdV1Obob4EzgEyAIGALsxzoQLx6wYB2sp1SHCwnw5b9vTuXasUn2soyCSn72jw3szy1zXWBKKdXFOfoI4FGsC/jMtB3/zhgzDRgGeGNNDpTqFN5eXtz3g2H8fNYI+8qBpdUNPLRkE6v35J7laqWUUu1xNAEYAnyM9du+wfbowBhzEHgca4KgVKeaeWECz9w6jtBA68qBTS0W/ufDnSz+6oCuHKiUUufI0QTAAjQbYwxQDCS0OpcHOPT8X6nv64KkSJ770cUk9Aqxly1dn8Ef3ttOnQ4OVEophzmaABwAkmyvtwH/T0RiRSQK6y6BWR0fmlLt6xMRzN/umMhFA6LsZRsOFPLg4jSKKnRbYaWUcoSjCcCbwFDb699jffafAxRgnY//2BmuU6pTBAf48sQPL+K6ccn2ssOF1sGB6Tk6OFAppc7GoQTAGPOiMea/bK+3AyOAHwO/AEbZpu4p5VTeXsKPr0zhF1eNwMc2OLCspoFfvb6JL3fnuDg6pZRyb2dNAETET0R+LiLDT5QZY3KMMYuMMc8ZY/Z1bohKfbfpoxN4dt44wloNDvzjv3ex6It0mlssLo5OKaXc01kTANu2vc8CEZ0fjlLnZ0RiJM/dOanN4MD30jJ56PU0Cst112illDqVo2MA0oF+nRmIUt9XbM8g/vajiYwdGG0vS88p5yevrmN9er4LI1NKKffjaALwGPCoiIzozGCU+r6C/X154oep3HnZEPuiQdX1zfz3+9/wwiff0tjc4uIIlVLKPTi6F8CvgRBgh4hkAflYFwQ6wRhjpnZwbEqdFy8RbprYnxEJETyzfAeF5dapgR9vy+bbo6X85voL2zwqUEopT+RoD0ALsA9YBxwDmm1lJ350pJVyO0P79uSluyczaUhve9mRoioeWLSeVbuOYXT1QKWUB3OoB8C27r9SXU5IgC+/u+FCVnxzlFc+20dTi4WGphb+/NFudh4p4YEZwwnyd3hTTKWU6jYc7QFQqssSEa4ak8hzd15MfGSwvfzLPbk8sGg9GfkVLoxOKaVcw6GvPiIy5Wx1jDG6JbBya/1iwnjhrkm8+OleVu2yLhSUW1rD//vnRu66fAjXXJSEiLg4SqWUcg5H+z7X0HbQX3u8v18oSnW+AD8fHrz6AkYn9+K5lXuoa2yhqcXCy5/tY8eREh6cPZKwID9Xh6mUUp3O0UcAl2Bd87/1z43AEqwbAV3VGcEp1VkuHRHHi3dNZkDvMHvZpoOF3PfqOvYcLXVhZEop5RyO7gXwdTs/y4wxPwI+AmZ3bphKdby4yGD+esdErh2bZC87XlnPf72exlvrDtFi0VkCSqnuqyMGAa4AbuqA91HK6fx8vLnvB8N4/KZUQm17CVgMLFlzkN+8uZmSqnoXR6iUUp2jIxKAweg6AKqLmzA4hpfvmczwhJNbXuzMKuG+hevYmlHkwsiUUqpzODoL4LZ2iv2A4cCdwLKODEopV4gKC+SP88fxr7WHWLouAwNU1Dbyu6VbuXFCPxZcMhgfb505q5TqHhydBbD4DOUNwDvAzzskGqVczNvLi9unDeaCpEj+Z/lOSqsbAOvOgnuOlvLInNH07hnk4iiVUur7c/TrTHI7P7HGmEBjzAJjjK6korqVUUm9ePmeyVw0IMpetj+3nPsWruO9tMM0tehTL6VU1+boLIDsdn4KOzs4pVwpPNifJ2++iLsuP7mzYG1jM4u+2M+PX1nLpoOFup+AUqrLcigBEJGrROSBM5y7X0RmdmxYSrkHLxFunNCfvyyY2GYZ4dzSGn7/zjZ+u3QrR4urXBihUkqdH0cfATwKBJ/hXKDtvFLd1pC4cF758RTuvTKF4FabB20/XMy9C9fxyqp9VNU1uTBCpZQ6N44mAEOAb85wbicwtGPCUcp9+Xh7MWdcMq/dP41ZYxKwPRWgxWJYvvkId760hhXbs3UBIaVUl+BoAuAFhJzhXCjg2zHhKOX+woP9+dnMEbxw12RGJp5cN6CitpHnVn7LA4vWsyurxIURKqXU2TmaAOwCbj3DuVuB3Y7eUESmi8gBEckQkYfbOZ8oIl+KyG4RWSMifVudaxGRnbafjxy9p1KdoX/vMP44fzy/u/5CYnoE2sszCyv5rzc28Yf3t1NQXuvCCJVS6swcXQfgz8AHIvIe8CqQA8QB9wBzsG4MdFYi4g28CFxhe4+tIvKRMWZfq2p/Al43xiwRkUuBZ4D5tnN1xphRDsasVKcTESanxDJ2YDQfbMrk7Q2HaWhqAWBdegGbDxVxw4R+/HBifxdHqpRSbTk6DXA51sV+fgB8AuwBPrMd/8wY4+hKgGOBDGNMpjGmEXgbuOaUOinAatvrr9o5r5Tb8ff15pbJA/nHT6Zy6fA+9vLGZgtvrcvgzpe+Zv2BYp02qJRyGw6va2qMeR7rt/6ZWL+RTwf6GGNePIf7xQHHWh2f6ElobRdwne31HCBURCJtxwEisk1ENonItedwX6WcIioskF/PGc1fFkxgUGwPe/nxqnqeX3WQXy5O42BeuQsjVEopK0cfAQBgjKnC+s2/Mz0EvCAiC4C1QC7QYjuXaIzJFZF+wGoR2WOMOdz6YhG5B+ujCeLj4ykvd7//bKuqPHfeuKe0PS7UiyeuH8bX6UUsTcumotY6RXBfThk/+8cGpg6NZu6ERMKD/VwcqXN4yp97ezy17Z7abug6bXd0M6BfA32NMT9t59xzwDFjzP868Fa5QHyr4762MjtjTB62HgARCQGuN8aU287l2n7NFJE1wGjg8CnXLwQWAqSmpprw8HBHmuh07hqXM3hS2+dM7MmVY/qxdF0GyzYfocViMMCa9CK2HC5l7uQBXDs2CT8fb1eH2uk86c/9VJ7adk9tN3SNtjv6COAOzjzSf6ftvCO2AgNFJFlE/ICbgTaj+UWkl4iciOsR4DVbeU8R8T9RB7gYaD14UCm3FOzvy12XD+XPt45m/KAYe3ltYzP/+HI/P/77WlbvyaXFovsLKKWcx9EEIAE4dIZzmUCiI29ijGkGHsD6GCEdeNcYs1dEnhSRq23VpgEHROQgEAM8ZSsfCmwTkV1YBwc+e8rsAaXcWmx4IE/8MJWnbxlLQq+Ty2rkldbyPx/u5M6XvmblN0dpbG75jndRSqmO4egYgFpOH6x3Ql+s2wI7xBizElh5StljrV6/D7zfznUbgRGO3kcpdzWmfxQv3zOZFduzef3rg1TXNwOQX1bL/63Yw5trD3HDhH7MuDCBAN/u/2hAKeUajvYArAN+daIL/gTb8YO280opB/l4e3HN2GQWP3Ap86cOIjTw5GKax6vqeWXVPm57bjXvbMigpkH3GFBKdTxHewAeBzYCB0XkX1gH7sUB84BIYEFnBKdUdxca6Mu8KQO5blwyK77J5oO0I5TVWDvUKmobeW31Ad7deJhrLkrm2rFJhAV5xqwBpVTncygBMMbsEpFLsK7S92usPQcWYD3WUfq7Oi9Epbq/IH8fbpzQn6tTk/hs5zHeS8ukqKIOgOr6Zt5cd4gPNmVyVWoi149PJiIkwMURK6W6unNZCGiLMWYK1s1/+gKhxphpQLCIvNZJ8SnlUfx9vbn6oiReu38av5w9kriIk7tw1ze18H5aJrc99xUvfPIthbrPgFLqezinhYAAjDF1IhIH3Csi87HOAKgFftTRwSnlqXy9vfjBqHguH9mXdfvyeXtDBkeKrIuLNLVY+HhbNiu/OcplI+L44cX96Rt5ps06lVKqfQ4nACLSA/ghcDsw3la8C3gWWNrxoSmlvL2EacP7MGVYLJsPFvHW+kMczKsAoMViWLUrhy925zB5aCw3TxpAv5gwF0eslOoqvjMBsC3IMx3rh/5sIADIw7qj3/3A/zPGrO3sIJXydF4iTBgcw/hB0Xxz5Dhvr89gd3YpABYDX+/L5+t9+YwfGM3cyQMYEtfTxRErpdzdGRMAEfkzcAsQDdQDy4ElwBdAGNYFfZRSTiQijOkXxZh+Uew9VsrS9RlszSi2n990qIhNh4oYndyLmy/uzwVJkYiICyNWSrmr7+oB+AVgsC7as8AYU3LihIjonqZKudiw+Aj+MHcsh/IrWLo+gw37C+zndhw5zo4jxxnQO4wbJ/RnckpvvL0cHvOrlPIA3/U/wj+AKmAW1qV5XxCRsc4JSynlqIGxPXjsxjEsvHcKl42Iw6vVN/6MgkqeWb6DO15Yw/LNR6hrbHZhpEopd3LGBMAYczfQG7gV2Ab8GEgTkXSsawFoL4BSbiQxKpT/unYU//jJVK4ak4Cfz8l/3oUVdbyyah/z/u9LXlu9n5KqehdGqpRyB9/ZJ2iMqTfGLDXGTMe6IdAjQAvwMCDAsyIyT0R0VRKl3ESfiGB+OnMEb/zsUuZPGUiPVqsHVtc3886Gw9z+/Ff85eNdZBd3jX3LlVId71wWAso3xvzRGDMcGIt1JsBA4HUgv5PiU0qdp/Bgf+ZNHcTrP7uUn84cTp+IIPu5phYLn+3M4Z5X1vLo21vZlVWCMdqpp5QnOeeFgACMMduwbs37S+Aq4LYOjUop1WECfL25akwiM0YnsOlgIe+lHSY9p9x+fsuhIrYcKmJQbA9umNCPSUN1wKBSnuC8EoATjDFNWKcHLu+YcJRSncXbS7h4SG8uHtKbvcdKeT8tk7QDhfbBPAfzK3h62Q56hwdy3bhkrhwVT6Df9/ovQinlxvRft1IeaFh8BMPiI8gpqeaDTUf4YncOjc0WAArK63jps328/vUhZqcmcvVFibr5kFLdkPbzKeXB+kaG8PNZ1gGDt04eSFigr/1cdX0TS9dncNtzX/HX/+zmqA4YVKpb0R4ApRThwf7cNm0QN13cn893HeODTUfIL7PuNtjUYuHTHcf4dMcxLhoQxaXD4xg/KIYgf/3vQ6muTP8FK6XsAny9mZ2axMwLE9l4oID30zLZn3tywODWjGK2ZhTj5+PF2AHRTBvWh4sGRhPg6+3CqJVS50MTAKXUaby9hMlDY5k0pDd7j5Xxflommw6eHDDY2Gxh/f4C1u8vIMDXm/GDYpg2rA9j+vfCz0eTAaW6Ak0AlFJnJCIMT4hgeEIE+WW1rNmbx9d78zhSdHI8QH1TC2v25rFmbx7B/j5MHNybqcNiGZ3cCx9vHWaklLvSBEAp5ZDYnkHMnTSAuZMGcLS4yroF8d48jpXU2OvUNDTz+e4cPt+dQ1igLxcP6c20YX1I6KG9Akq5G00AlFLnLCEqlPlTQ5k3ZSCZhVV8vS+Ptfvy7QMHASrrmvhkxzE+2XGMHkG+TEnpw7RhfUiJ79lmwyKllGtoAqCUOm8iQv/eYfTvHcYdlwzmYH4FX+/N4+t9+RyvPLnhUEVtEx9vy+bjbdn0CgtgytBYpg7rw+A+PRBNBpRyCU0AlFIdQkQY3CecwX3CuevyoaTnlLFmbx7r9hVQVtNgr3e8sp5lm4+wbPMReocHMiWlD5OG9mZQrCYDSjmTJgBKqQ7nJWJfbfDeK4eRti+b7dlVrE/Pp7KuyV6voLyOdzce5t2Nh+kVGsCEwTFMHNybkYkROoBQqU6mCYBSqlN5ewnD+4YzaXgS908fxs6sEtbszWPj/gJqGprt9Y5X1dsfE4QE+DBuYAwTBsdwUf8oAnRPAqU6nP6rUko5jY+3F6n9o0jtH0XjzOFsP3ycDfsL2HSokKpWPQPV9c18uSeXL/fk4ufjxYXJvZg4pDfjBkYTHuzvwhYo1X1oAqCUcgk/H28mDLZ+y2+xWNhztJS0A4Vs2F9AcasBhI3NFjYdKmLToSK8xLqR0cQhvZk4KIbePYNc2AKlujZNAJRSLuft5cWopF6MSurFvVemkFFQycYDBWzcX0hWq02ILAb2HC1lz9FS/r5qH/1iwrh4cAwTBvemX0yoDiJU6hw4PQEQkenA/wHewCJjzLOnnE8EXgOigFJgnjEmp9X5MGAf8KEx5gGnBa6UcgoRYWBsDwbG9uD2aYPJLa1h44EC0g4Usu9YmX05YoDMwkoyCyt5Y+0heocHMnFwbyYOjiElPgJvL00GlPouTk0ARMQbeBG4AsgBtorIR8aYfa2q/Ql43RizREQuBZ4B5rc6/9/AWmfFrJRyrbiIYG6c0J8bJ/SnrLqBtIOFpB0oYMeREppaLPZ6BeV19umFPYL8GNOvFyMSIxmREEHfyGDtHVDqFM7uARgLZBhjMgFE5G3gGqzf6E9IAX5pe/0V8OGJEyIyBogBPgVSnRGwUsp99AzxZ+aFCcy8MIGahia2ZRSz8UAhWzKKqG01o6CitpHV3+ax+ts8ACJC/BmeEMHIxAhGJESSGBWiCYHyeM5OAOKAY62Oc4Bxp9TZBVyH9THBHCBURCKBMuDPwDzg8s4PVSnlzoL9fZk6rA9Th/WhqcXCrqwS+6OC0uqGNnVLqxtYuy+ftfvyAegR5NcmIUiOCdXliZXHccdBgA8BL4jIAqxd/blAC/ATYKUxJue7MncRuQe4ByA+Pp7y8vIz1nWVqqqqs1fqprTtnskZbR8Q6cuAifHMm9CXI0XV7MutJD2vgv15ldQ0tLSpW1HbyIb9BWzYXwBAsL8PQ/uEMTQujKF9wkiKCumwMQSe+ufuqe2GrtN2ZycAuUB8q+O+tjI7Y0we1h4ARCQEuN4YUy4iE4DJIvITIATwE5FqY8zDp1y/EFgIkJqaasLDwzutMd+Hu8blDNp2z+TMtkf07MmYwdbXLRZDVlElu7NL2ZNdwp6jpW1WIwTrLobbjpSy7UgpAEF+PgxL6MmIhEhGJkYwMLbH91qZ0FP/3D213dA12u7sBGArMFBEkrF+8N8M3NK6goj0AkqNMRbgEawzAjDG3NqqzgIg9dQPf6WUOpW3l9C/dw/69+7BnHHJWIzhaHE1e46W2JKC0jZ7FQDUNjazNaOYrRnFAAT4epMS35ORiZGMSopkUJ8eeHvpUsWqa3NqAmCMaRaRB4DPsE4DfM0Ys1dEngS2GWM+AqYBz4iIwfoI4H5nxqiU6t68REiKDiUpOpTZqUkYY8gpqWHP0VJ2Z5ewJ7uU41X1ba6pb2rhm8zjfJN5HLD2EAxPjGBUkjUhSI4J0zEEqssRY8zZa3VRqampZtu2ba4O4zTl5eVdonuoM2jbte3uzhhDQXmdPRnYfbSEwvK677wmLNDX2juQHMmopF5tph12pbZ3JE9tN7i+7SKy3Rhz1ply7jgIUCmlXEZEiO0ZRGzPIH4wyjpkqajCmhDsyiphZ1YJRRVtE4LKuibW7y9gvW1QYWSoP6OSenFBUiT9Inzx0M9B5eY0AVBKqbOI7hHI5SP7cvnIvvYegp1Zx9l5pISdWccpr2lsU7+kqsG+mRFAbM8gLrA9LrggKZKIkABXNEOpNjQBUEqpc3CyhyCBGaMTMMaQXVzNrqzj7MwqYXd2CdX1zW2uyS+rJb+slk93WJdBSYwKse19EMmIxEhCA31d0RTl4TQBUEqp70FaDSq8ZmwyLRZDZmElO45YE4Jvs0toaLa0uSa7uJrs4mr+vTULsPYQ9IsOJTkmjKToUPpFh9G7Z5DuZ6A6lSYASinVgby9Tm5mdNPE/hwvKaWgBnZmlbAr6zjpOeVt9jCAkz0EGw4U2sv8fb1JigolOTqU5JhQkqPDSI4OJSzIz9lNUt2UJgBKKdWJfLy9GJ4QzvCECOZNGUh9Uwt7j5Wy64h1QOGh/Aos7czGamhq4UBeOQfy2q5mGhnqb08Gkm29BvG9QvD9HgsVKc+kCYBSSjlRgK83Y/pFMaZfFACNzS0cO15NZmEVR4oqOVJURVZR1Wn7GZxQUtVASVUx2w4X28u8vYSEXiEkRVt7CvrFhNIvJoyIEH/d9EidkSYASinlQn4+3vaVClsrr2ngSFEVRwqtScGRoiqyi6toPGU8AViXOz5R5yvy7OXhwX7W944Jo3/vMAb0DqNPRLAuWqQATQCUUsothQf7MzrZn9HJvexlLRZDXmmNPTHILKoiq6iSgjMsVFRe08j2w8Vsb9VbEOjnTT9bQmBNDHqQGBWCn493p7dJuRdNAJRSqovw9hLie4UQ3yuEKSmx9vKahiaybD0AWUVVZBZWkllYSV1jy2nvUdfYwt5jZew9VmYv8/ESEqJC7b0E/Xv3oF9MKMH+Oj2xO9MEQCmlurhgf1+GxUcwLD7CXmYxhvzSWjIKKjhcUElGYSWHCypOW7QIoNk2dTGzsJLPd50sj+0ZZE8IrL+G6SJG3YgmAEop1Q15iRAXGUxcZDBTh/UBrPsclFY3WBMCW2JwuLCS/LLadt/jxPTEdekF9rLIUH8G9rZOcxxgm+4YGapJQVekCYBSSnkIESEyNIDI0ADGDoy2l9fUN3G4sJKMgkoybcnB0ePVtFhOn55onYVQxKZDRfayiBB/+9oHJ350RIH70wRAKaU8XHCAdTfDkYmR9rLG5hayi6tPPkIoqCCzsIqGptPHFZRWN7D5UBGbWyUF4UG+DIrrae8tsPYU6LREd6IJgFJKqdP4+XjbP7hPaLEYjh2v5lB+BRkFFbZfK9tNCsprm9hyqIgtrZKCnsH+DIwNsz86GBjbg16hAZoUuIgmAEoppRzi7XVy34MrLugLWJOCnBJrUnDi53BBJfXtJAVlNQ1syShmS8bJaYnhwX4kR1v3QEiKsi5mlNArlCB//XjqbPo7rJRS6rx5ewmJUaEkRoVy+ciTSUF6Vj6F1RYOFVTakoKKdqclltc0suPIcXYcOd6mPCY8kKSoUJKiQkm0JQbxvXS9go6kCYBSSqkO5e0l9I0IYni/cC4baS2zGENOSQ0ZrXoKMs6QFAAUltdRWF7XZlyBl0CfiGBrYhBtSw6iQ4mLCMLbS/dCOFeaACillOp0XmLdryChVwiXjogDTq5VkF1cRVaxdRGjrOIqckpq2p2BYDGQU1JDTkkN6/efnJro6+1F38hge1KQFB1KXEQw0T0C8ffVHoMz0QRAKaWUS7Req2DikN728qYWC7klNfaE4MSvBWW1nJ4WWOuf2AvhVOHBfkT3CCSmR2CrX4Osr8MDCQnw3NUONQFQSinlVny9veyDDVurb2zm6PHqVklBNdlFVRyvqj/je5XXNFJe08jBvIp2zwf5+5xMDsJPJAlB9mQhPNiv285S0ARAKaVUlxDg58OgPuEM6hPepry6vsn6GKFVj0FheR3FlfVYTHt9BifVNjSfsfcAwM/Hi+iwQKJtyUFUWCBRYQG21wFEhXXdxwyaACillOrSQgJO3wsBoMVioaSqgcKKOorKa62/2n5OvG5ve+XWGpst5JTWkFNac8Y6PYL8iAoLoFdYINE9AgjxhcSYWqJ6WBOEyFB/txykqAmAUkqpbsnby4toW/c+CRGnnTfGUFHbaEsQTiYFJxOFWqrrm896n4raRipqG8koqGxVmm1/5SUQERpg7zFo3Xtw4nWPIOc/atAEQCmllEcSEcKD/QkP9mfwKY8VTqhpaLInB8WVdRRX1FNUaX28UFxZx/HK+nZnLLRmMXC8sp7jlfWkU95unVfvnUJCVGi75zqLJgBKKaXUGQT7+5Ic40tyTFi751sshvKaBoor6yiqsCYFOcXlVNQbe8JQVtNw1vtE9Qjs6NDPShMApZRS6jx5e53cYXGIdXkDysvLCQ8/2aPQ2NxCSVUDRSd6ESrrKaqo47jtdW1jM4F+zv841gRAKaWU6kR+Pt7E9gwitmeQq0Npw/2GJSqllFKq02kCoJRSSnkgpycAIjJdRA6ISIaIPNzO+UQR+VJEdovIGhHp26r8GxHZKSJ7ReReZ8eulFJKdRdOTQBExBt4EZgBpABzRSTllGp/Al43xowEngSesZXnAxOMMaOAccDDItLHOZErpZRS3YuzewDGAhnGmExjTCPwNnDNKXVSgNW210rCwMQAABdsSURBVF+dOG+MaTTGnJhL4Y8+vlBKKaXOm7M/ROOAY62Oc2xlre0CrrO9ngOEikgk/P/27j5Kjqs+8/j3mZ6e0WgkWZKFZFkSSGKJsQ9+k2WteQnxYRPsvByTGG/WrBcss6wDLDkbCGfXxics8S4h3oVk48ABHGANxgkkWnAcWxstEDuQZbGEZYz8JluWLKzRCPlFY81oXnqm5+4f93ZPzWjeNdM9mno+OnW66tat6nvntqp+Vfd2NUhaJ+mnaR+3hRAOz3J5zczM5qW5+DXAjwKflbQV+D7QBpQBQgjPAxekW//3SNoWQvh5dmNJNwI3Aqxbt46OjtGfulRPnZ2j/+hEHrju+eS6509e6w2nT91rHQC0Aesyy2tTWlW6qr8aQNIi4J0hhI6ReSQ9BvwisG3EujuAOwA2b94csg9jmEvmarlqwXXPJ9c9f/Jabzg96l7rLoBdwOskbZDUBFwL3JvNIGmFpEq5bga+ktLXSmpJ88uAtwB7a1ZyMzOzeaSmAUAIYQD4ELADeBL46xDC45JulXRVynY5sFfS08Aq4JMp/VzgIUmPAv8IfDqEsKeW5TczM5svaj4GIISwHdg+Iu3jmfltjLitn9K/A1ww6wU0MzPLAX+VzszMLIccAJiZmeWQAwAzM7MccgBgZmaWQw4AzMzMcsgBgJmZWQ45ADAzM8shBwBmZmY55ADAzMwshxwAmJmZ5ZADADMzsxxyAGBmZpZDDgDMzMxyyAGAmZlZDjkAMDMzyyEHAGZmZjnkAMDMzCyHHACYmZnlkAMAMzOzHHIAYGZmlkMOAMzMzHLIAYCZmVkOOQAwMzPLIQcAZmZmOeQAwMzMLIccAJiZmeWQAwAzM7MccgBgZmaWQw4AzMzMcsgBgJmZWQ411voNJV0J/BlQAL4UQvjjEetfA3wFeBXwMvBvQgiHJF0EfB5YApSBT4YQvlnTwpvVSQiBQBj1FRhz3WAYZDAMEhiaHwyDk143cn2xoUhrUyuLmhbRWmxlQeMCJNX5rzP7SuUSXaUuyoNlWoottDS2UGgo1LtYAJQHy/QM9NDd382J0gm6+7vp7u9GEk2FJpoKTRQbivG1UDxpuUEzex0YQqB/sJ/OUicD3QP0DfRRKpfoK/eNOT8wOEB5sMzA4MCwqRyGp00mTyUthIAkKv8a1DC0nHltUMNJaWPlr6Rlt6nMZ9f19vbS2tI65naj7eeGi26gubF5RttiIjUNACQVgM8BvwIcAnZJujeE8EQm26eBr4UQvirpbcCngHcD3cB7QgjPSDobeFjSjhBCRy3rYHNbqVziYMdBDh0/VD2wjHXwmOjgMtr6/sF+BgYH6B/sp7/cP7Q8ifm+/j7KlE9KHxgcGPMEP9c1qIHWYuuwoOCk+eIiGkMjyxctj+kj1i8sLqSgAg1qmLFJEj39PXSVuugqddFZ6hya7xuaH7lu1OW+TvoH+0+qe2NDIy2NLdWAYEHjgup8SzEtN7ZQCAXOWHjGmHmaCk309PdUT9zd/d2c6D8x7nL2ZN9X7julNiyoMGpwMFqaJPoG+sY9mZfKpVMqT15d+4Zr53cAAGwB9oUQ9gNI+gbwDiAbAJwHfCTNPwDcAxBCeLqSIYRwWNJR4l0CBwA5MhgGae9sZ/+x/RzoOMCBYwfia8cB9h/bT9vxttPixDlfDIZBOkuddJY6612UmhsYHJgXdS+HeAehZ6Cn3kXJtZm+EzMZtQ4A1gDPZ5YPAf98RJ5HgauJ3QS/BSyWdGYI4aVKBklbgCbg2dktbr6VyiUOdx6m7XgbbZ1tHO48DEBrMV7BZa/mKld0lfnK1cJUhRA41nts6MR+bOjkfqDjAAc7Dp7yFc/pbLRblcCYtzaztxyzV8fDljPrJ1rXX+6nq9TFif4TnCidyE1bFFRgcfNiGtRAT38PvQO9cyrQXFhceNJUuRVfKpfoL6fXUZZnQ+WOQXNjM82F5nHnK3caGhsaq1NBhWHLY6Y1FEbNU2goIDRql9hod9oqXV8T5c/mG22+kr+7p5vm5uZh22e700Zbbio0zUpbjKfmYwAm4aPAZyVtBb4PtBH7/AGQtBq4C7g+hDA4cmNJNwI3Aqxbt46Ojrl3g6Czs75XDCEEXul7hcNdh2k/0c7hzvTadZj2rnbaT7TT3tXOiz0vTvs9CipUA4KFjQtpbWqltdhKs5o5o+UMWovx1m9rsZX+cj8Hjx+M0ysHOV46Pu33FeLsRWezbsk6WoutNKghHhw0/IBRPVBkDirVvCl/oaEwbH1lvtgw/GCVXS42FGksxO2LDUWKhSIFFSgWivR297J08dJ4sFNj9aBXUGHUE3slgMrOzzUDgwPV29OV167+rmG3rk/0n+DlrpcpN5TpHuimq9RF98DQ+p6BnpPHHjA47riEcdendS2NLbGboZjpcijGLonscjVwLQ51S7QWW1nctLi6vrnQPKwNQgiUyiV6y730DPTQN9BHz0AMDHoGeugrDy0f6zqGiqqu6x3opbfcG18HeimVS7Q0trCwuDB2DaT5yvLCxoXVboPWYuuwPJX56X4+QggMDA5QGizF1xQUVLqoSuVSdX1/uZ9BBmkuZE7eDfEE3lxoplgoVtMb1EBnZyeLFy+eqY/aaWU6de/t6qWX3lkq0ehqHQC0Aesyy2tTWlUI4TDxDgCSFgHvrPTzS1oC3A/cEkL40WhvEEK4A7gDYPPmzWHp0qUzXYcZMZvlOt53nCdfeJJDxw9x6Pgh2jrjFXzlSr7teNus3+4rhzLHS8dP6WQ+ljNbzmTjso1sWLaBDUvjVFl+9RmvrkskPRkdHR2z2u71soIVE+aZr3WfjDzXPa/1htOj7rUOAHYBr5O0gXjivxb419kMklYAL6er+5uJ3whAUhPwbeIAwW01LfUcdqznGI8ceYSHDz/M7iO7efjwwzzz8jMzsu8GNXDWorNYs3gNa5asYc3iNRRUqN4CrgySqsyfKA2lncqtxZbGljFP8BuWbmBxcz6vKszMZlJNA4AQwoCkDwE7iF8D/EoI4XFJtwI/DiHcC1wOfEpSIHYB/Pu0+W8DbwXOTN0DAFtDCD+pZR3q6cXuF9ndvnvYyf5Ax4Fp7au12Fo9qVdfF69h7ZK11eVVi1bR2DC9j0ipXKoGBNkA4cixI1BkWAAhxPql66sn+JWtK+fsLW8zs/mi5mMAQgjbge0j0j6emd8GnHSFH0L4OvD1WS/gHHGk68hJJ/vnjz8/8YbE/vdzX3UuG5ZuqJ7g1y5ZO+xkv6R5yayeZJsKTTS1NLGsZdmw9I6l+b0damY2l8zFQYC5EkKgrbPtpJN9e1f7pLYvNhQ5f9X5XLL6Ejat3sQlqy/h/FXns6BxwSyX3MzMTmcOAGrspe6XeOC5B3jylSfZeXgnO9t2cvTE0Ult21xo5sKzLqye7Det3sQbVr5hzg56MzOzucsBwCzq7u9md/tudrbtZNfhXexs28n+Y/snte3C4kIuOusiNp21iUvOjif8c1ecS7FQnOVSm5lZHjgAmCEDgwM8dvQxdrXFE/3Owzt5/OjjlEN5wm0XNy3m4tUXDzvZn3PmOXPmWeNmZjb/OACYhhACzx57dtjJ/pH2Ryb13fqmQhPnrzify159GVvWbOHSsy/lnBXn1OUxkGZmll8OACYhhMB9T99XPdnvatvFsd5jE24nxOtXvJ4ta7ZUT/YXrLqAnq4ej4Q3M7O6cgAwCZJ4//3vrz4LfyzrlqwbdrK/5OxLWNK85KR8PfhHN8zMrL4cAEzSljVbuOepe6rLy1uWc+nZl1ZP9peuuZSzFp1VxxKamZlNngOASbrm3GvYsHRD9aS/cdlGP63OzMxOWw4AJum6C67juguuq3cxzMzMZoSHnpuZmeWQAwAzM7MccgBgZmaWQw4AzMzMcsgBgJmZWQ45ADAzM8shBwBmZmY55ADAzMwshxwAmJmZ5ZADADMzsxxyAGBmZpZDDgDMzMxySCGEepdh1kh6AThY73KMYgXwYr0LUSeuez657vmT13pD/ev+mhDCqybKNK8DgLlK0o9DCJvrXY56cN1d97zJa93zWm84feruLgAzM7MccgBgZmaWQw4A6uOOehegjlz3fHLd8yev9YbTpO4eA2BmZpZDvgNgZmaWQw4AToGkdZIekPSEpMcl/YeUvlzSdyQ9k16XpXRJul3SPkk/lbQps6/rU/5nJF2fSb9E0p60ze2SVPuajk5SQdIjku5LyxskPZTK+k1JTSm9OS3vS+vXZ/Zxc0rfK+mKTPqVKW2fpJtqXbfxSFoqaZukpyQ9KemNOWrzD6fP+mOS/krSgvna7pK+IumopMcyabPezmO9Ry2NUff/nj7zP5X0bUlLM+um1J7T+czUymh1z6z7fUlB0oq0fHq3ewjB0zQnYDWwKc0vBp4GzgP+G3BTSr8JuC3N/xrwvwEBlwEPpfTlwP70uizNL0vrdqa8Stv+ar3rnan/R4C/BO5Ly38NXJvmvwB8IM1/EPhCmr8W+GaaPw94FGgGNgDPAoU0PQtsBJpSnvPqXd9Mvb8KvC/NNwFL89DmwBrgANCSae+t87XdgbcCm4DHMmmz3s5jvcccqPvbgcY0f1um7lNuz6l+Zupd95S+DthBfLbMivnQ7nX5jzVfJ+BvgV8B9gKrU9pqYG+a/yLwrkz+vWn9u4AvZtK/mNJWA09l0oflq3Nd1wLfA94G3Jc+zC9mDhBvBHak+R3AG9N8Y8on4Gbg5sw+d6Ttqtum9GH56lzvM4gnQY1Iz0ObrwGeTwe1xtTuV8zndgfWM/wkOOvtPNZ71LvuI9b9FnD3aO00UXtO51gxF+oObAMuBJ5jKAA4rdvdXQAzJN2quhh4CFgVQmhPq44Aq9J85QBacSiljZd+aJT0ueB/AP8RGEzLZwIdIYSBtJwta7V+af0rKf9U/x5zwQbgBeB/KnZ/fElSKzlo8xBCG/Bp4GdAO7EdHyYf7V5Ri3Ye6z3mkvcSr15h6nWfzrGiriS9A2gLITw6YtVp3e4OAGaApEXA/wJ+L4RwPLsuxHBuXn3VQtJvAEdDCA/Xuyx10Ei8Pfj5EMLFwAni7bqq+djmAKlP8h3EIOhsoBW4sq6FqqNatPNc/CxJugUYAO6ud1lqQdJC4GPAx2v1nrVqdwcAp0hSkXjyvzuE8K2U/HNJq9P61cDRlN5G7EeqWJvSxktfO0p6vb0ZuErSc8A3iN0AfwYsldSY8mTLWq1fWn8G8BJT/3vMBYeAQyGEh9LyNmJAMN/bHOCXgQMhhBdCCP3At4ifhTy0e0Ut2nms96g7SVuB3wCuSycpmHrdX2Lqn5l6ei0x6H00HfPWArslncVp3u4OAE5BGr35ZeDJEMKfZFbdC1RGfV5PHBtQSX9PGjl6GfBKuuWzA3i7pGXpKuvtxD6xduC4pMvSe70ns6+6CSHcHEJYG0JYTxyo8w8hhOuAB4BrUraR9a78Pa5J+UNKvzaN/N0AvI44QGYX8Lo0Urgpvce9NajahEIIR4DnJZ2Tkv4F8ATzvM2TnwGXSVqYylap+7xv94xatPNY71FXkq4kdvtdFULozqyaUnumz8BUPzN1E0LYE0JYGUJYn455h4iDv49wurd7rQdXzKcJeAvxNs1PgZ+k6deIfVbfA54BvgssT/kFfI44MnYPsDmzr/cC+9J0QyZ9M/BY2uaz1GFAzAR/g8sZ+hbARuJ//H3A3wDNKX1BWt6X1m/MbH9LqtteMqPd09/x6bTulnrXc0SdLwJ+nNr9HuIo31y0OfCHwFOpfHcRR37Py3YH/oo41qGfeND/t7Vo57HeYw7UfR+xX7tyrPvCdNtzOp+ZetZ9xPrnGBoEeFq3u58EaGZmlkPuAjAzM8shBwBmZmY55ADAzMwshxwAmJmZ5ZADADMzsxxyAGA2iyT9pqSPzNK+70wPJpnOts9JunNmS1Q7krZKeu8M7i9I+sRM7c/sdOCvAZrNonSS/eUQwtqJ8k5j368FloQQHpnGthcDx0MIz850uWpB0oPEH5N5ywzt7zLiEx4PTZjZbJ5onDiLmdWCpOYQQt9k85/KyXs6QcN8FkL4Ub3LYFZr7gIwmyXp6v96YE26xRwqt+wlXZ6Wr5b0F5JeAH6e1v0zSXdJOiCpR9J+SZ9PjxQdtv9sF4Ck9WmfvyPpVkntkjok/Z2ktSO2HdYFkG6ph/SI0rslHZd0WNLtkhaM2HajpO2SuiUdlfQZSTem7ddP8De5QtIPJb0iqUvSXkkfH5HnQkn3SjqW6v9/Jf1iZv2DwC8Bb878XR8c5z0XSfpzST+T1JfK/F1Jr8/kqXYBZNpmtCn7N1so6bbUTqX0eoskH1fttOA7AGaz578ArwIuBa5KaSOv8P+c+LOq7yY+BhXiL+09D/wecIz42NSPAduJv50+kZuBHxIfRboS+AzwdeJjmydyF/FRqFen9/pEKsN/BkjPdP8O8RHAHyD+NPL7GHqu+5gkbSQ+73wbcCtQIj43fmMmzybgB8AjwL8DuoH3A9+V9KYQf4Hyg6k+BeB30qbDfoVzhD8l/v0/RnzM6pmkHzEaI/9uTv47vw34JPBkKmcj8Xnv5xHbeQ9wGfAHwHLg98cpj9ncUOvnLHvylKcJuJPYtzwy/XLi70h8exL7aGTodycuHrHv5zLL61OeB0ds/9GUfnYm7Tngzszy1pTnD0dsex/wdGb5xpRvSyZNwKMpff049bgm5VkyTp7vEU+yTZm0Qkq7J5P2IPBPk2yDx4A/mSBPAD4xxrpfAF4mBi6VcVPvTtu8dUTeW4iBzcp6f/Y8eZpo8q0qs/r69sgESU2SPibpKUk9xB8l+UFafc7I/KPYPmJ5T3p99SS2vX+UbbPbXQb8LISws5IQQgjEn8SeyE+IdfmGpGskrcyulNRCvLX/N8CgpMZ0pS3ij6O8dRLvMZpdwNb0N90sqTDZDVO3y98Rf9Dl3amuAFcCB4EfVsqZyvp/gCLx72Q2pzkAMKuv9lHSPkW89f514NeBLcRb8jDUTTCel0csV7odprttc2Z5rN8p//lEOw4h7AOuIB537gKOSPqRpF9KWZYTr/b/gBgoZKcPAcum2b/+u8AXiV0iu4Cjkv5U0sLxNpJUJF71LyD+BG5PZvVK4DWjlLMSGJ05jXKa1ZTHAJjV12jfw70W+FoI4b9WEiQtql2RxtVO7PceadVkNg4hPAA8IKmZ2A9/K3B/GjzYAQwSf171a2NsPzjVAocQuojjIm6W9BpiV8QfE2/V/6dxNv0ccfzGm0P87fesl4ADwG+Pse1zUy2nWa05ADCbXX1AyxS3WUi8msy6YWaKc8p+BNwgaUulG0CSgHdOZSchft3xH1Jg87fAhhDCLkk/AC4Edk9wsu8DFk+18CGEg8BnJF0HvGGsfJI+TLxjcFUIYc8oWf6eWOeuEMJTUy2H2VzgAMBsdj0BLJf0AeDHQO8YJ5Ssvweul7SH2Pd8NfCm2S3mpN1JvGr+lqRbGPoWQOUrimOetCW9n9iPv534LYcVxCvzw8SBegAfAb4P7JD0ZeIdhxXAJqAQQrgp5XsC+KCkfwU8C3SGEPaO8b7/j/jtgz1AF3GcwYXAV8fI/ybg08S7EC8rPiSo4oUQn79wNzEo+56kzxAHQTYBryV+4+A3QwjdY/0tzOYCBwBms+tLxAFhf0T82tlB4mj98fwuceDbJ9PyduBdDPUv100IoSTp7cSvL36BeEL9S+Ah4m31V8bZ/FHgV4ljHFYSxxv8E3BdpX89hLBb0qXErx3eDpxBDDJ2p/eruI04IPJLwCLgHxn7a47fJ96qv4l4zNsPfDiEcPsY+X+BOE5ha5qyvgpsDSH0S7oi7fNGYANwghiM3E/sXjCb0/woYDM7ZZLuA84NIby23mUxs8nxHQAzmxLFHzfqIj5UZzHwL4nfVvhAPctlZlPjAMDMpqoP+DDx+QAFYC/wvhDCl+taKjObEncBmJmZ5ZAfBGRmZpZDDgDMzMxyyAGAmZlZDjkAMDMzyyEHAGZmZjnkAMDMzCyH/j8oS50y9eOeOgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 576x432 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "train_acc = np.mean(train_acc_folds, axis = 1)\n",
    "train_val = np.mean(val_acc_folds, axis = 1)\n",
    "print([' %.4f' % score for score in train_val])\n",
    "print(\"\\nAccuracy: %0.4f (+/- %0.4f)\" % (np.mean(train_val), np.std(train_val) * 2))\n",
    "fig, ax = plt.subplots(nrows = 1, ncols = 1, figsize = (8, 6))\n",
    "ax.plot(train_sizes, train_acc, lw = 3, color = \"steelblue\", label = \"training\")\n",
    "ax.plot(train_sizes, train_val, lw = 3, color = \"green\", label = \"validation\")\n",
    "\n",
    "ax.grid(alpha=0.25)\n",
    "ax.set_xlabel(\"training set size\", fontsize=16)\n",
    "ax.set_ylabel(\"Accuracy\", fontsize=16)\n",
    "ax.legend(loc=\"upper right\", fontsize=16)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-05T09:40:02.805258Z",
     "start_time": "2018-04-05T09:36:48.871447Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Automatic pdb calling has been turned ON\n",
      "Fitting Model for:  toxic\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/hasil/.conda/envs/ml-env/lib/python3.6/site-packages/sklearn/linear_model/stochastic_gradient.py:128: FutureWarning: max_iter and tol parameters have been added in <class 'sklearn.linear_model.stochastic_gradient.SGDClassifier'> in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n",
      "  \"and default tol will be 1e-3.\" % type(self), FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting Model for:  severe_toxic\n",
      "Fitting Model for:  obscene\n",
      "Fitting Model for:  threat\n",
      "Fitting Model for:  insult\n",
      "Fitting Model for:  identity_hate\n",
      "Fitting Model for:  toxic\n",
      "Fitting Model for:  toxic\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/hasil/.conda/envs/ml-env/lib/python3.6/site-packages/sklearn/linear_model/stochastic_gradient.py:128: FutureWarning: max_iter and tol parameters have been added in <class 'sklearn.linear_model.stochastic_gradient.SGDClassifier'> in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n",
      "  \"and default tol will be 1e-3.\" % type(self), FutureWarning)\n",
      "/home/hasil/.conda/envs/ml-env/lib/python3.6/site-packages/sklearn/linear_model/stochastic_gradient.py:128: FutureWarning: max_iter and tol parameters have been added in <class 'sklearn.linear_model.stochastic_gradient.SGDClassifier'> in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n",
      "  \"and default tol will be 1e-3.\" % type(self), FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting Model for:  severe_toxic\n",
      "Fitting Model for:  severe_toxic\n",
      "Fitting Model for:  obscene\n",
      "Fitting Model for:  obscene\n",
      "Fitting Model for:  threat\n",
      "Fitting Model for:  threat\n",
      "Fitting Model for:  insult\n",
      "Fitting Model for:  insult\n",
      "Fitting Model for:  identity_hate\n",
      "Fitting Model for:  identity_hate\n",
      "Fitting Model for:  toxic\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/hasil/.conda/envs/ml-env/lib/python3.6/site-packages/sklearn/linear_model/stochastic_gradient.py:128: FutureWarning: max_iter and tol parameters have been added in <class 'sklearn.linear_model.stochastic_gradient.SGDClassifier'> in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n",
      "  \"and default tol will be 1e-3.\" % type(self), FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting Model for:  severe_toxic\n",
      "Fitting Model for:  obscene\n",
      "Fitting Model for:  threat\n",
      "Fitting Model for:  insult\n",
      "Fitting Model for:  identity_hate\n",
      "Fitting Model for:  toxic\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/hasil/.conda/envs/ml-env/lib/python3.6/site-packages/sklearn/linear_model/stochastic_gradient.py:128: FutureWarning: max_iter and tol parameters have been added in <class 'sklearn.linear_model.stochastic_gradient.SGDClassifier'> in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n",
      "  \"and default tol will be 1e-3.\" % type(self), FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting Model for:  severe_toxic\n",
      "Fitting Model for:  obscene\n",
      "Fitting Model for:  threat\n",
      "Fitting Model for:  toxic\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/hasil/.conda/envs/ml-env/lib/python3.6/site-packages/sklearn/linear_model/stochastic_gradient.py:128: FutureWarning: max_iter and tol parameters have been added in <class 'sklearn.linear_model.stochastic_gradient.SGDClassifier'> in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n",
      "  \"and default tol will be 1e-3.\" % type(self), FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting Model for:  insult\n",
      "Fitting Model for:  severe_toxic\n",
      "Fitting Model for:  identity_hate\n",
      "Fitting Model for:  obscene\n",
      "Fitting Model for:  threat\n",
      "Fitting Model for:  insult\n",
      "Fitting Model for:  identity_hate\n",
      "Fitting Model for:  toxic\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/hasil/.conda/envs/ml-env/lib/python3.6/site-packages/sklearn/linear_model/stochastic_gradient.py:128: FutureWarning: max_iter and tol parameters have been added in <class 'sklearn.linear_model.stochastic_gradient.SGDClassifier'> in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n",
      "  \"and default tol will be 1e-3.\" % type(self), FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting Model for:  severe_toxic\n",
      "Fitting Model for:  obscene\n",
      "Fitting Model for:  threat\n",
      "Fitting Model for:  insult\n",
      "Fitting Model for:  identity_hate\n",
      "Fitting Model for:  toxic\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/hasil/.conda/envs/ml-env/lib/python3.6/site-packages/sklearn/linear_model/stochastic_gradient.py:128: FutureWarning: max_iter and tol parameters have been added in <class 'sklearn.linear_model.stochastic_gradient.SGDClassifier'> in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n",
      "  \"and default tol will be 1e-3.\" % type(self), FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting Model for:  severe_toxic\n",
      "Fitting Model for:  obscene\n",
      "Fitting Model for:  threat\n",
      "Fitting Model for:  insult\n",
      "Fitting Model for:  identity_hate\n",
      "Fitting Model for:  toxic\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/hasil/.conda/envs/ml-env/lib/python3.6/site-packages/sklearn/linear_model/stochastic_gradient.py:128: FutureWarning: max_iter and tol parameters have been added in <class 'sklearn.linear_model.stochastic_gradient.SGDClassifier'> in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n",
      "  \"and default tol will be 1e-3.\" % type(self), FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting Model for:  severe_toxic\n",
      "Fitting Model for:  obscene\n",
      "Fitting Model for:  threat\n",
      "Fitting Model for:  insult\n",
      "Fitting Model for:  identity_hate\n",
      "Fitting Model for:  toxic\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/hasil/.conda/envs/ml-env/lib/python3.6/site-packages/sklearn/linear_model/stochastic_gradient.py:128: FutureWarning: max_iter and tol parameters have been added in <class 'sklearn.linear_model.stochastic_gradient.SGDClassifier'> in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n",
      "  \"and default tol will be 1e-3.\" % type(self), FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting Model for:  severe_toxic\n",
      "Fitting Model for:  obscene\n",
      "Fitting Model for:  threat\n",
      "Fitting Model for:  insult\n",
      "Fitting Model for:  identity_hate\n"
     ]
    }
   ],
   "source": [
    "%pdb on\n",
    "cv_results = cross_validate(pipeline, train[comment_col], train[label_cols],\n",
    "                           n_jobs = 3, return_train_score = False, cv = 10,\n",
    "                           scoring = 'roc_auc')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-05T09:40:06.759919Z",
     "start_time": "2018-04-05T09:40:06.747126Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'fit_time': array([38.66870809, 58.23814154, 59.02181387, 45.50326872, 51.01447153,\n",
       "        56.57273531, 51.96861434, 47.10251284, 53.08789873, 43.98268247]),\n",
       " 'score_time': array([2.80224133, 2.26441073, 3.13931894, 3.61119509, 3.66877818,\n",
       "        2.86443043, 3.56554651, 2.35198331, 2.5002687 , 1.89054728]),\n",
       " 'test_score': array([0.97962782, 0.97771139, 0.97903062, 0.97223756, 0.97473903,\n",
       "        0.97933417, 0.97901685, 0.97617753, 0.97954819, 0.98003277])}"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cv_results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-05T09:41:29.208962Z",
     "start_time": "2018-04-05T09:41:29.190830Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9777455934417734"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.mean(cv_results['test_score'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-03T09:48:37.478563Z",
     "start_time": "2018-04-03T09:48:37.461446Z"
    }
   },
   "source": [
    "CV Result with Linear Regression using NB features\n",
    "``` python\n",
    "{'fit_time': array([79.08852482, 79.20061374, 42.78198028]),\n",
    " 'score_time': array([6.6369772 , 6.62154627, 5.88802028]),\n",
    " 'test_score': array([0.92122728, 0.92169581, 0.92276744])}\n",
    " ```\n",
    " \n",
    " CV Result with Linear SVM using NB features\n",
    " ```python\n",
    "{'fit_time': array([30.04560089, 30.4099865 , 24.19594097]),\n",
    " 'score_time': array([7.33976412, 7.11350369, 6.19125676]),\n",
    " 'test_score': array([0.91893365, 0.92066178, 0.9214326 ])}\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}