fasttext_sentiment_analysis.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.6"
    },
    "colab": {
      "name": "fasttext_sentiment_analysis.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "4d8a7111c88f4946af0aa37656c22e20": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_fcca42ac8786470da5ff030dc7bd093c",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_15088d1cc38c44369d492eb46179e09a",
              "IPY_MODEL_f92b830f020c4c80911f68d982960c21"
            ]
          }
        },
        "fcca42ac8786470da5ff030dc7bd093c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "15088d1cc38c44369d492eb46179e09a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "IntProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_6dc45aa8232744d2b8970e40cfef033d",
            "_dom_classes": [],
            "description": "",
            "_model_name": "IntProgressModel",
            "bar_style": "success",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_60c3bef3e2a14f048f4c599d8deb2182"
          }
        },
        "f92b830f020c4c80911f68d982960c21": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_66d0c37d59174d4fa06011b24c783301",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "2000000it [02:24, 13809.93it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_219cb780eb954638bdc9548ba979cd7e"
          }
        },
        "6dc45aa8232744d2b8970e40cfef033d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "60c3bef3e2a14f048f4c599d8deb2182": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "66d0c37d59174d4fa06011b24c783301": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "219cb780eb954638bdc9548ba979cd7e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "bf46ca3a52004a72b4563e48df280ffb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_212c373c4a944e3f855c4095842b1ae7",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_9adb3afda5f0416786a765abf0a58a1d",
              "IPY_MODEL_7f07ec67876c4f798fd357cb57ac0932"
            ]
          }
        },
        "212c373c4a944e3f855c4095842b1ae7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "9adb3afda5f0416786a765abf0a58a1d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "IntProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_f8b6b893924d4594a43f49d1428e59c7",
            "_dom_classes": [],
            "description": "",
            "_model_name": "IntProgressModel",
            "bar_style": "success",
            "max": 152672,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 152672,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_db1e73cffcbd4042982c831f60765ea6"
          }
        },
        "7f07ec67876c4f798fd357cb57ac0932": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_5c749d61747f4a5abdde0d0b71b2f4aa",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "100% 152672/152672 [00:00&lt;00:00, 193276.47it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_10144f501736432cbda146877e2b48f4"
          }
        },
        "f8b6b893924d4594a43f49d1428e59c7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "db1e73cffcbd4042982c831f60765ea6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "5c749d61747f4a5abdde0d0b71b2f4aa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "10144f501736432cbda146877e2b48f4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/hbaflast/fasttext-sentiment-analysis/blob/master/fasttext_sentiment_analysis.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Vokf5sLsZvUJ",
        "colab_type": "text"
      },
      "source": [
        "# fastText Sentiment Analysis"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kLSSmwhyZvUL",
        "colab_type": "text"
      },
      "source": [
        "## Import"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QD5U1Zg7xRAJ",
        "colab_type": "code",
        "outputId": "39e0c70e-86c3-413e-94e5-d748c47324f9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "# Switch to tf2 (Colab run tensorflow 1.X by default for the moment)\n",
        "%tensorflow_version 2.x"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "TensorFlow 2.x selected.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nv7obIiwZvUM",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import os\n",
        "import io\n",
        "import codecs\n",
        "import joblib\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import tensorflow as tf\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "from datetime import datetime\n",
        "from tqdm import tqdm_notebook\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.metrics import accuracy_score\n",
        "\n",
        "from nltk.tokenize import RegexpTokenizer\n",
        "from tensorflow.keras.preprocessing.text import Tokenizer\n",
        "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
        "\n",
        "from tensorflow.keras.models import Model\n",
        "from tensorflow.keras.layers import Input, Embedding, Dense, Dropout, concatenate\n",
        "from tensorflow.keras.layers import Conv1D, GlobalMaxPooling1D"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VIM-7Zy0ZvUR",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "plt.rcParams[\"figure.figsize\"] = (16, 9)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AJuNO-rPZvUW",
        "colab_type": "text"
      },
      "source": [
        "## Load data"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XBwFugGgx1nb",
        "colab_type": "text"
      },
      "source": [
        "### Kaggle credentials"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xtJobvWlx92Q",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# enter your Kaggle credentionals here\n",
        "os.environ['KAGGLE_USERNAME'] = \"hbaflast\"\n",
        "os.environ['KAGGLE_KEY'] = \"e1fbb7c60a14101f3d3531500537c6d0\""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "q5ZxfVj3x97m",
        "colab_type": "code",
        "outputId": "19bf480a-9e0f-496b-9b30-02bc2f3b42be",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 68
        }
      },
      "source": [
        "!kaggle datasets download hbaflast/french-twitter-sentiment-analysis"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading french-twitter-sentiment-analysis.zip to /content\n",
            " 68% 33.0M/48.3M [00:01<00:01, 15.6MB/s]\n",
            "100% 48.3M/48.3M [00:01<00:00, 41.7MB/s]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SYAzZEDpyCgZ",
        "colab_type": "code",
        "outputId": "a4b0bcdd-89e5-4ca7-8503-22b7b20b0eae",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 68
        }
      },
      "source": [
        "!kaggle datasets download sgadioux19/fasttext-french-2b-300d"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading fasttext-french-2b-300d.zip to /content\n",
            " 99% 1.21G/1.22G [00:20<00:00, 41.8MB/s]\n",
            "100% 1.22G/1.22G [00:20<00:00, 63.1MB/s]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "S2b_sK2VyjGY",
        "colab_type": "code",
        "outputId": "6ea2ff36-c0bc-4d9b-c040-b8d8d0f311f0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "!unzip /content/fasttext-french-2b-300d.zip"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Archive:  /content/fasttext-french-2b-300d.zip\n",
            "  inflating: cc.fr.300.vec           \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4WTqRzljyCuR",
        "colab_type": "text"
      },
      "source": [
        "### Read dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ovGFi5jkZvUX",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df_dataset = pd.read_csv(\"/content/french-twitter-sentiment-analysis.zip\", sep=',')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pIruxUn8ZvUc",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df_dataset = df_dataset.sample(frac=0.1, random_state=42)  # sample to speed-up computation"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iQdDNN-bZvUg",
        "colab_type": "code",
        "outputId": "68991256-dc64-458a-82c9-f7421beb8d63",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 136
        }
      },
      "source": [
        "df_dataset.info()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "Int64Index: 152672 entries, 471040 to 126980\n",
            "Data columns (total 2 columns):\n",
            "label    152672 non-null int64\n",
            "text     152672 non-null object\n",
            "dtypes: int64(1), object(1)\n",
            "memory usage: 3.5+ MB\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_8xBa2soZvUl",
        "colab_type": "code",
        "outputId": "57183578-9727-4e86-b769-8f0f65bdce80",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 534
        }
      },
      "source": [
        "df_dataset['label'].value_counts().plot.bar();"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAA7EAAAIFCAYAAAAED3qOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0\ndHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAdsUlEQVR4nO3dYaye5X3f8d+/uLQ0XQIknkVtMiPF\na0UiNU0sQtVp2sIKJp1qXrQR0TSsCMWTQrZ2mrSSvUFLGimRpmVFSiOh4sVUXSjNWmG1pJ5FUk3T\nRMKhyUgJzTglodgiwY0JrI2alPS/F+eCPjbHnONgfHyZz0d6dO7nf1/3c67nlfXV85zb1d0BAACA\nGfzARm8AAAAA1kvEAgAAMA0RCwAAwDRELAAAANMQsQAAAExDxAIAADCNTRu9ge/X6173ut6+fftG\nbwMAAIDT7IEHHviL7t682rlpI3b79u1ZWlra6G0AAABwmlXVYyc75+vEAAAATGNdEVtV/7aqHqqq\nP6mqT1bVD1fVZVX1uaparqrfrqrzx9ofGs+Xx/ntC6/z/jH/SlVdszDfNWbLVXXz6X6TAAAAnBvW\njNiq2prk3yTZ2d1vSnJekuuTfCTJR7v7DUmeSnLjuOTGJE+N+UfHulTV5eO6NybZleTXq+q8qjov\nyceSXJvk8iTvGmsBAADgOOv9OvGmJBdU1aYkP5LkiSRvT/KpcX5/kuvG8e7xPOP8VVVVY35nd3+n\nu7+aZDnJFeOx3N2Pdvd3k9w51gIAAMBx1ozY7j6S5D8l+fOsxOvTSR5I8q3ufnYsO5xk6zjemuTx\nce2zY/1rF+cnXHOyOQAAABxnPV8nvigrn4xeluTHkrwqK18HPuOqam9VLVXV0tGjRzdiCwAAAGyg\n9Xyd+J8l+Wp3H+3uv0nyu0l+JsmF4+vFSbItyZFxfCTJpUkyzr8myTcX5ydcc7L5C3T3bd29s7t3\nbt686n8ZBAAAwDlsPRH750murKofGX/belWSLyf5bJJfGGv2JLl7HB8YzzPOf6a7e8yvH3cvvizJ\njiSfT3J/kh3jbsfnZ+XmTwde+lsDAADgXLNprQXd/bmq+lSSP07ybJIvJLktyR8kubOqfnXMbh+X\n3J7kN6tqOcmxrERpuvuhqrorKwH8bJKbuvt7SVJV70tyMCt3Pt7X3Q+dvrcIAADAuaJWPiSdz86d\nO3tpaWmjtwEAAMBpVlUPdPfO1c6t97/YAQAAgA0nYgEAAJiGiAUAAGAaIhYAAIBpiFgAAACmIWIB\nAACYhogFAABgGiIWAACAaYhYAAAApiFiAQAAmIaIBQAAYBqbNnoDvPJsv/kPNnoL8Ir0tQ//3EZv\nAQDgJfNJLAAAANMQsQAAAExDxAIAADANEQsAAMA0RCwAAADTELEAAABMQ8QCAAAwDRELAADANEQs\nAAAA0xCxAAAATGPTRm8AAOBct/3mP9joLcAr0tc+/HMbvQVeBj6JBQAAYBoiFgAAgGmIWAAAAKYh\nYgEAAJiGiAUAAGAaIhYAAIBpiFgAAACmIWIBAACYhogFAABgGiIWAACAaYhYAAAApiFiAQAAmIaI\nBQAAYBoiFgAAgGmIWAAAAKYhYgEAAJiGiAUAAGAaIhYAAIBpiFgAAACmIWIBAACYhogFAABgGiIW\nAACAaYhYAAAApiFiAQAAmIaIBQAAYBoiFgAAgGmIWAAAAKYhYgEAAJjGmhFbVT9eVV9ceDxTVb9c\nVRdX1aGqemT8vGisr6q6taqWq+rBqnrLwmvtGesfqao9C/O3VtWXxjW3VlW9PG8XAACAma0Zsd39\nle5+c3e/Oclbk3w7ye8luTnJvd29I8m943mSXJtkx3jsTfLxJKmqi5PckuRtSa5Icstz4TvWvGfh\nul2n5d0BAABwTjnVrxNfleTPuvuxJLuT7B/z/UmuG8e7k9zRK+5LcmFVXZLkmiSHuvtYdz+V5FCS\nXePcq7v7vu7uJHcsvBYAAAA871Qj9voknxzHW7r7iXH89SRbxvHWJI8vXHN4zF5sfniVOQAAABxn\n3RFbVecn+fkkv3PiufEJap/GfZ1sD3uraqmqlo4ePfpy/zoAAADOMqfySey1Sf64u78xnn9jfBU4\n4+eTY34kyaUL120bsxebb1tl/gLdfVt37+zunZs3bz6FrQMAAHAuOJWIfVf+7qvESXIgyXN3GN6T\n5O6F+Q3jLsVXJnl6fO34YJKrq+qicUOnq5McHOeeqaorx12Jb1h4LQAAAHjepvUsqqpXJfnZJP9q\nYfzhJHdV1Y1JHkvyzjG/J8k7kixn5U7G706S7j5WVR9Mcv9Y94HuPjaO35vkE0kuSPLp8QAAAIDj\nrCtiu/uvkrz2hNk3s3K34hPXdpKbTvI6+5LsW2W+lORN69kLAAAAr1ynendiAAAA2DAiFgAAgGmI\nWAAAAKYhYgEAAJiGiAUAAGAaIhYAAIBpiFgAAACmIWIBAACYhogFAABgGiIWAACAaYhYAAAApiFi\nAQAAmIaIBQAAYBoiFgAAgGmIWAAAAKYhYgEAAJiGiAUAAGAaIhYAAIBpiFgAAACmIWIBAACYhogF\nAABgGiIWAACAaYhYAAAApiFiAQAAmIaIBQAAYBoiFgAAgGmIWAAAAKYhYgEAAJiGiAUAAGAaIhYA\nAIBpiFgAAACmIWIBAACYhogFAABgGiIWAACAaYhYAAAApiFiAQAAmIaIBQAAYBoiFgAAgGmIWAAA\nAKYhYgEAAJiGiAUAAGAaIhYAAIBpiFgAAACmIWIBAACYhogFAABgGiIWAACAaYhYAAAApiFiAQAA\nmIaIBQAAYBoiFgAAgGmIWAAAAKaxroitqgur6lNV9adV9XBV/XRVXVxVh6rqkfHzorG2qurWqlqu\nqger6i0Lr7NnrH+kqvYszN9aVV8a19xaVXX63yoAAACzW+8nsb+W5A+7+yeS/GSSh5PcnOTe7t6R\n5N7xPEmuTbJjPPYm+XiSVNXFSW5J8rYkVyS55bnwHWves3Ddrpf2tgAAADgXrRmxVfWaJP84ye1J\n0t3f7e5vJdmdZP9Ytj/JdeN4d5I7esV9SS6sqkuSXJPkUHcf6+6nkhxKsmuce3V339fdneSOhdcC\nAACA563nk9jLkhxN8l+r6gtV9RtV9aokW7r7ibHm60m2jOOtSR5fuP7wmL3Y/PAqcwAAADjOeiJ2\nU5K3JPl4d/9Ukr/K3311OEkyPkHt07+941XV3qpaqqqlo0ePvty/DgAAgLPMeiL2cJLD3f258fxT\nWYnab4yvAmf8fHKcP5Lk0oXrt43Zi823rTJ/ge6+rbt3dvfOzZs3r2PrAAAAnEvWjNju/nqSx6vq\nx8foqiRfTnIgyXN3GN6T5O5xfCDJDeMuxVcmeXp87fhgkqur6qJxQ6erkxwc556pqivHXYlvWHgt\nAAAAeN6mda7710l+q6rOT/JokndnJYDvqqobkzyW5J1j7T1J3pFkOcm3x9p097Gq+mCS+8e6D3T3\nsXH83iSfSHJBkk+PBwAAABxnXRHb3V9MsnOVU1etsraT3HSS19mXZN8q86Ukb1rPXgAAAHjlWu//\nEwsAAAAbTsQCAAAwDRELAADANEQsAAAA0xCxAAAATEPEAgAAMA0RCwAAwDRELAAAANMQsQAAAExD\nxAIAADANEQsAAMA0RCwAAADTELEAAABMQ8QCAAAwDRELAADANEQsAAAA0xCxAAAATEPEAgAAMA0R\nCwAAwDRELAAAANMQsQAAAExDxAIAADANEQsAAMA0RCwAAADTELEAAABMQ8QCAAAwDRELAADANEQs\nAAAA0xCxAAAATEPEAgAAMA0RCwAAwDRELAAAANMQsQAAAExDxAIAADANEQsAAMA0RCwAAADTELEA\nAABMQ8QCAAAwDRELAADANEQsAAAA0xCxAAAATEPEAgAAMA0RCwAAwDRELAAAANMQsQAAAExDxAIA\nADANEQsAAMA0RCwAAADTELEAAABMQ8QCAAAwDRELAADANEQsAAAA01hXxFbV16rqS1X1xapaGrOL\nq+pQVT0yfl405lVVt1bVclU9WFVvWXidPWP9I1W1Z2H+1vH6y+PaOt1vFAAAgPmdyiex/7S739zd\nO8fzm5Pc2907ktw7nifJtUl2jMfeJB9PVqI3yS1J3pbkiiS3PBe+Y817Fq7b9X2/IwAAAM5ZL+Xr\nxLuT7B/H+5NctzC/o1fcl+TCqrokyTVJDnX3se5+KsmhJLvGuVd3933d3UnuWHgtAAAAeN56I7aT\n/I+qeqCq9o7Zlu5+Yhx/PcmWcbw1yeML1x4esxebH15lDgAAAMfZtM51/6i7j1TV309yqKr+dPFk\nd3dV9enf3vFGQO9Nkte//vUv968DAADgLLOuT2K7+8j4+WSS38vK37R+Y3wVOOPnk2P5kSSXLly+\nbcxebL5tlflq+7itu3d2987NmzevZ+sAAACcQ9aM2Kp6VVX9veeOk1yd5E+SHEjy3B2G9yS5exwf\nSHLDuEvxlUmeHl87Ppjk6qq6aNzQ6eokB8e5Z6rqynFX4hsWXgsAAACet56vE29J8nvjf73ZlOS/\ndfcfVtX9Se6qqhuTPJbknWP9PUnekWQ5ybeTvDtJuvtYVX0wyf1j3Qe6+9g4fm+STyS5IMmnxwMA\nAACOs2bEdvejSX5ylfk3k1y1yryT3HSS19qXZN8q86Ukb1rHfgEAAHgFeyn/xQ4AAACcUSIWAACA\naYhYAAAApiFiAQAAmIaIBQAAYBoiFgAAgGmIWAAAAKYhYgEAAJiGiAUAAGAaIhYAAIBpiFgAAACm\nIWIBAACYhogFAABgGiIWAACAaYhYAAAApiFiAQAAmIaIBQAAYBoiFgAAgGmIWAAAAKYhYgEAAJiG\niAUAAGAaIhYAAIBpiFgAAACmIWIBAACYhogFAABgGiIWAACAaYhYAAAApiFiAQAAmIaIBQAAYBoi\nFgAAgGmIWAAAAKYhYgEAAJiGiAUAAGAaIhYAAIBpiFgAAACmIWIBAACYhogFAABgGiIWAACAaYhY\nAAAApiFiAQAAmIaIBQAAYBoiFgAAgGmIWAAAAKYhYgEAAJiGiAUAAGAaIhYAAIBpiFgAAACmIWIB\nAACYhogFAABgGiIWAACAaYhYAAAAprHuiK2q86rqC1X1++P5ZVX1uaparqrfrqrzx/yHxvPlcX77\nwmu8f8y/UlXXLMx3jdlyVd18+t4eAAAA55JT+ST2l5I8vPD8I0k+2t1vSPJUkhvH/MYkT435R8e6\nVNXlSa5P8sYku5L8+gjj85J8LMm1SS5P8q6xFgAAAI6zroitqm1Jfi7Jb4znleTtST41luxPct04\n3j2eZ5y/aqzfneTO7v5Od381yXKSK8Zjubsf7e7vJrlzrAUAAIDjrPeT2P+S5N8n+dvx/LVJvtXd\nz47nh5NsHcdbkzyeJOP802P98/MTrjnZHAAAAI6zZsRW1T9P8mR3P3AG9rPWXvZW1VJVLR09enSj\ntwMAAMAZtp5PYn8myc9X1dey8lXftyf5tSQXVtWmsWZbkiPj+EiSS5NknH9Nkm8uzk+45mTzF+ju\n27p7Z3fv3Lx58zq2DgAAwLlkzYjt7vd397bu3p6VGzN9prv/RZLPJvmFsWxPkrvH8YHxPOP8Z7q7\nx/z6cffiy5LsSPL5JPcn2THudnz++B0HTsu7AwAA4Jyyae0lJ/UrSe6sql9N8oUkt4/57Ul+s6qW\nkxzLSpSmux+qqruSfDnJs0lu6u7vJUlVvS/JwSTnJdnX3Q+9hH0BAABwjjqliO3uP0ryR+P40azc\nWfjENX+d5BdPcv2Hknxolfk9Se45lb0AAADwynMq/08sAAAAbCgRCwAAwDRELAAAANMQsQAAAExD\nxAIAADANEQsAAMA0RCwAAADTELEAAABMQ8QCAAAwDRELAADANEQsAAAA0xCxAAAATEPEAgAAMA0R\nCwAAwDRELAAAANMQsQAAAExDxAIAADANEQsAAMA0RCwAAADTELEAAABMQ8QCAAAwDRELAADANEQs\nAAAA0xCxAAAATEPEAgAAMA0RCwAAwDRELAAAANMQsQAAAExDxAIAADANEQsAAMA0RCwAAADTELEA\nAABMQ8QCAAAwDRELAADANEQsAAAA0xCxAAAATEPEAgAAMA0RCwAAwDRELAAAANMQsQAAAExDxAIA\nADANEQsAAMA0RCwAAADTELEAAABMQ8QCAAAwDRELAADANEQsAAAA0xCxAAAATEPEAgAAMA0RCwAA\nwDRELAAAANNYM2Kr6oer6vNV9X+q6qGq+o9jfllVfa6qlqvqt6vq/DH/ofF8eZzfvvBa7x/zr1TV\nNQvzXWO2XFU3n/63CQAAwLlgPZ/EfifJ27v7J5O8OcmuqroyyUeSfLS735DkqSQ3jvU3JnlqzD86\n1qWqLk9yfZI3JtmV5Ner6ryqOi/Jx5Jcm+TyJO8aawEAAOA4a0Zsr/jL8fQHx6OTvD3Jp8Z8f5Lr\nxvHu8Tzj/FVVVWN+Z3d/p7u/mmQ5yRXjsdzdj3b3d5PcOdYCAADAcdb1N7HjE9MvJnkyyaEkf5bk\nW9397FhyOMnWcbw1yeNJMs4/neS1i/MTrjnZHAAAAI6zrojt7u9195uTbMvKJ6c/8bLu6iSqam9V\nLVXV0tGjRzdiCwAAAGygU7o7cXd/K8lnk/x0kguratM4tS3JkXF8JMmlSTLOvybJNxfnJ1xzsvlq\nv/+27t7Z3Ts3b958KlsHAADgHLCeuxNvrqoLx/EFSX42ycNZidlfGMv2JLl7HB8YzzPOf6a7e8yv\nH3cvvizJjiSfT3J/kh3jbsfnZ+XmTwdOx5sDAADg3LJp7SW5JMn+cRfhH0hyV3f/flV9OcmdVfWr\nSb6Q5Pax/vYkv1lVy0mOZSVK090PVdVdSb6c5NkkN3X395Kkqt6X5GCS85Ls6+6HTts7BAAA4Jyx\nZsR294NJfmqV+aNZ+fvYE+d/neQXT/JaH0ryoVXm9yS5Zx37BQAA4BXslP4mFgAAADaSiAUAAGAa\nIhYAAIBpiFgAAACmIWIBAACYhogFAABgGiIWAACAaYhYAAAApiFiAQAAmIaIBQAAYBoiFgAAgGmI\nWAAAAKYhYgEAAJiGiAUAAGAaIhYAAIBpiFgAAACmIWIBAACYhogFAABgGiIWAACAaYhYAAAApiFi\nAQAAmIaIBQAAYBoiFgAAgGmIWAAAAKYhYgEAAJiGiAUAAGAaIhYAAIBpiFgAAACmIWIBAACYhogF\nAABgGiIWAACAaYhYAAAApiFiAQAAmIaIBQAAYBoiFgAAgGmIWAAAAKYhYgEAAJiGiAUAAGAaIhYA\nAIBpiFgAAACmIWIBAACYhogFAABgGiIWAACAaYhYAAAApiFiAQAAmIaIBQAAYBoiFgAAgGmIWAAA\nAKYhYgEAAJiGiAUAAGAaIhYAAIBpiFgAAACmsWbEVtWlVfXZqvpyVT1UVb805hdX1aGqemT8vGjM\nq6purarlqnqwqt6y8Fp7xvpHqmrPwvytVfWlcc2tVVUvx5sFAABgbuv5JPbZJP+uuy9PcmWSm6rq\n8iQ3J7m3u3ckuXc8T5Jrk+wYj71JPp6sRG+SW5K8LckVSW55LnzHmvcsXLfrpb81AAAAzjVrRmx3\nP9HdfzyO/1+Sh5NsTbI7yf6xbH+S68bx7iR39Ir7klxYVZckuSbJoe4+1t1PJTmUZNc49+ruvq+7\nO8kdC68FAAAAzzulv4mtqu1JfirJ55Js6e4nxqmvJ9kyjrcmeXzhssNj9mLzw6vMAQAA4Djrjtiq\n+tEk/z3JL3f3M4vnxieofZr3ttoe9lbVUlUtHT169OX+dQAAAJxl1hWxVfWDWQnY3+ru3x3jb4yv\nAmf8fHLMjyS5dOHybWP2YvNtq8xfoLtv6+6d3b1z8+bN69k6AAAA55D13J24ktye5OHu/s8Lpw4k\nee4Ow3uS3L0wv2HcpfjKJE+Prx0fTHJ1VV00buh0dZKD49wzVXXl+F03LLwWAAAAPG/TOtb8TJJ/\nmeRLVfXFMfsPST6c5K6qujHJY0neOc7dk+QdSZaTfDvJu5Oku49V1QeT3D/WfaC7j43j9yb5RJIL\nknx6PAAAAOA4a0Zsd/+vJCf7f1uvWmV9J7npJK+1L8m+VeZLSd601l4AAAB4ZTuluxMDAADARhKx\nAAAATEPEAgAAMA0RCwAAwDRELAAAANMQsQAAAExDxAIAADANEQsAAMA0RCwAAADTELEAAABMQ8QC\nAAAwDRELAADANEQsAAAA0xCxAAAATEPEAgAAMA0RCwAAwDRELAAAANMQsQAAAExDxAIAADANEQsA\nAMA0RCwAAADTELEAAABMQ8QCAAAwDRELAADANEQsAAAA0xCxAAAATEPEAgAAMA0RCwAAwDRELAAA\nANMQsQAAAExDxAIAADANEQsAAMA0RCwAAADTELEAAABMQ8QCAAAwDRELAADANEQsAAAA0xCxAAAA\nTEPEAgAAMA0RCwAAwDRELAAAANMQsQAAAExDxAIAADANEQsAAMA0RCwAAADTELEAAABMQ8QCAAAw\nDRELAADANEQsAAAA0xCxAAAATEPEAgAAMI01I7aq9lXVk1X1Jwuzi6vqUFU9Mn5eNOZVVbdW1XJV\nPVhVb1m4Zs9Y/0hV7VmYv7WqvjSuubWq6nS/SQAAAM4N6/kk9hNJdp0wuznJvd29I8m943mSXJtk\nx3jsTfLxZCV6k9yS5G1Jrkhyy3PhO9a8Z+G6E38XAAAAJFlHxHb3/0xy7ITx7iT7x/H+JNctzO/o\nFfclubCqLklyTZJD3X2su59KcijJrnHu1d19X3d3kjsWXgsAAACO8/3+TeyW7n5iHH89yZZxvDXJ\n4wvrDo/Zi80PrzIHAACAF3jJN3Yan6D2adjLmqpqb1UtVdXS0aNHz8SvBAAA4Czy/UbsN8ZXgTN+\nPjnmR5JcurBu25i92HzbKvNVdfdt3b2zu3du3rz5+9w6AAAAs/p+I/ZAkufuMLwnyd0L8xvGXYqv\nTPL0+NrxwSRXV9VF44ZOVyc5OM49U1VXjrsS37DwWgAAAHCcTWstqKpPJvknSV5XVYezcpfhDye5\nq6puTPJYkneO5fckeUeS5STfTvLuJOnuY1X1wST3j3Uf6O7nbhb13qzcAfmCJJ8eDwAAAHiBNSO2\nu991klNXrbK2k9x0ktfZl2TfKvOlJG9aax8AAADwkm/sBAAAAGeKiAUAAGAaIhYAAIBpiFgAAACm\nIWIBAACYhogFAABgGiIWAACAaYhYAAAApiFiAQAAmIaIBQAAYBoiFgAAgGmIWAAAAKYhYgEAAJiG\niAUAAGAaIhYAAIBpiFgAAACmIWIBAACYhogFAABgGiIWAACAaYhYAAAApiFiAQAAmIaIBQAAYBoi\nFgAAgGmIWAAAAKYhYgEAAJiGiAUAAGAaIhYAAIBpiFgAAACmIWIBAACYhogFAABgGiIWAACAaYhY\nAAAApiFiAQAAmIaIBQAAYBoiFgAAgGmIWAAAAKYhYgEAAJiGiAUAAGAaIhYAAIBpiFgAAACmIWIB\nAACYhogFAABgGiIWAACAaYhYAAAApiFiAQAAmIaIBQAAYBoiFgAAgGmIWAAAAKYhYgEAAJiGiAUA\nAGAaIhYAAIBpnDURW1W7quorVbVcVTdv9H4AAAA4+5wVEVtV5yX5WJJrk1ye5F1VdfnG7goAAICz\nzVkRsUmuSLLc3Y9293eT3Jlk9wbvCQAAgLPM2RKxW5M8vvD88JgBAADA8zZt9AZORVXtTbJ3PP3L\nqvrKRu4HXqFel+QvNnoTnLr6yEbvAGBK/t2bmH/7pvYPTnbibInYI0kuXXi+bcyO0923JbntTG0K\neKGqWurunRu9DwA4E/y7B2efs+XrxPcn2VFVl1XV+UmuT3Jgg/cEAADAWeas+CS2u5+tqvclOZjk\nvCT7uvuhDd4WAAAAZ5mzImKTpLvvSXLPRu8DWJOv9APwSuLfPTjLVHdv9B4AAABgXc6Wv4kFAACA\nNYlYAAAApiFiAQAAmMZZc2Mn4OxTVT+RZHeSrWN0JMmB7n5443YFAMArmU9igVVV1a8kuTNJJfn8\neFSST1bVzRu5NwA406rq3Ru9B2CFuxMDq6qq/5vkjd39NyfMz0/yUHfv2JidAcCZV1V/3t2v3+h9\nAL5ODJzc3yb5sSSPnTC/ZJwDgHNKVT14slNJtpzJvQAnJ2KBk/nlJPdW1SNJHh+z1yd5Q5L3bdiu\nAODlsyXJNUmeOmFeSf73md8OsBoRC6yqu/+wqv5hkity/I2d7u/u723czgDgZfP7SX60u7944omq\n+qMzvx1gNf4mFgAAgGm4OzEAAADTELEAAABMQ8QCAAAwDRELAADANEQsAAAA0/j/A5nrw7tgxxkA\nAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 1152x648 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HmeuxIoqZvUp",
        "colab_type": "text"
      },
      "source": [
        "## Text length distribution"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fUOuOW9hZvUq",
        "colab_type": "code",
        "outputId": "97dad967-1cd1-4392-c1d1-8fc78264f0c2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "df_dataset['sent_len'] = df_dataset['text'].apply(lambda x: len(x.split(\" \")))\n",
        "max_seq_len = np.round(df_dataset['sent_len'].mean() + 2 * df_dataset['sent_len'].std()).astype(int)\n",
        "max_seq_len"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "30"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bkG4F3l_ZvUu",
        "colab_type": "code",
        "outputId": "81e7d529-be5f-4c97-b554-ece673701c1f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 537
        }
      },
      "source": [
        "df_dataset['sent_len'].plot.hist()\n",
        "plt.axvline(x=max_seq_len, color='k', linestyle='--', label='max len');"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAA78AAAIICAYAAACmWFnjAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0\ndHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3df7CnZX0f/PdH0MqaKArUOiztmkfE\n0B9ZDVXRxm4EleVHtA8EtFmDeZysI2ae7FSmZRmpNquAM2skPsky0khBaAtbSBQWqIWNW9PBiKjb\nEiWLW4LdJSqwqKigiF7949ykR8KP+/l+97v3Ofd5vWbO7Pe+vvc5531mbzjz3uu6r7taawEAAIAx\ne9rQAQAAAGDWlF8AAABGT/kFAABg9JRfAAAARk/5BQAAYPSUXwAAAEZv/6ED7GsHH3xwW7FixdAx\nAFhkduzYkSQ54ogjBk4CADyRL3zhC/e11g55vPeWXPldsWJFbr311qFjALDIrFq1Kkmybdu2QXMA\nAE+sqr72RO9Z9gwAAMDoKb8AAACM3pJb9gwAk7j++uuHjgAATEH5BYAeli1bNnQEAGAKlj0DQA+b\nNm3Kpk2bho4BAExI+QWAHjZv3pzNmzcPHQMAmJDyCwAAwOgpvwAAAIye8gsAAMDoKb8AAACMnkcd\nAUAP27ZtGzoCADAFM78AAACMnvILAD1s3LgxGzduHDoGADAh5RcAetiyZUu2bNkydAwAYELKLwAA\nAKOn/AIAADB6yi8AAACj51FHANDDAQccMHQEAGAKyi8A9HDDDTcMHQEAmIJlzwAAAIye8gsAPWzY\nsCEbNmwYOgYAMCHLnheYFWddN3SERe+u808YOgIwQlu3bk2SnHPOOQMnAQAmYeYXAACA0VN+AQAA\nGD3lFwAAgNFzzy8A9HDQQQcNHQEAmILyCwA9XH311UNHAACmYNkzAAAAo6f8AkAP69evz/r164eO\nAQBMyLJnAOjhs5/97NARAIApmPkFAABg9JRfAAAARk/5BQAAYPTc8wsAPSxfvnzoCADAFJRfAOjh\n8ssvHzoCADAFy54BAAAYPeUXAHpYt25d1q1bN3QMAGBClj0DQA/bt28fOgIAMAUzvwAAAIye8gsA\nAMDoKb8AAACMnnt+AaCHF7/4xUNHAACmoPwCQA8XXXTR0BEAgClY9gwAAMDoKb8A0MPatWuzdu3a\noWMAABOy7BkAerjjjjuGjgAATMHMLwAAAKOn/AIAADB6yi8AAACj555fAOhh5cqVQ0cAAKag/AJA\nDxdccMHQEQCAKVj2DAAAwOgpvwDQw5o1a7JmzZqhYwAAE7LsmdFZcdZ1Q0dY9O46/4ShI8CCs3v3\n7qEjAABTMPMLAADA6Cm/AAAAjJ7yCwAAwOi55xcAejj66KOHjgAATEH5BYAezjvvvKEjAABTsOwZ\nAACA0VN+AaCHk08+OSeffPLQMQCACVn2DAA97NmzZ+gIAMAUzPwCAAAwesovAAAAozfz8ltV+1XV\nl6pqS3f8wqr6XFXtrKorq+oZ3fjf6o53du+vmPc11nfjO6rqDfPGj+vGdlbVWbP+WQAAAFic9sXM\n728nuX3e8QeTfLi19qIk30ry9m787Um+1Y1/uDsvVXVkkjcn+ftJjkuyqSvU+yX5gySrkxyZ5C3d\nuQCw1x1zzDE55phjho4BAExophteVdXyJCck+UCSf1FVleS1Sf55d8qlSd6X5MIkb+xeJ8lVSX6/\nO/+NSa5orf0wyV9W1c4kL+/O29lau7P7Xld0535llj8TAEvTOeecM3QEAGAKs575vSDJv0zyk+74\noCTfbq090h3vTnJo9/rQJLuSpHv/O935fz3+mM95ovG/oarWVtWtVXXrvffeO+3PBAAAwCIzs/Jb\nVScmuae19oVZfY++WmsXtdaOaq0ddcghhwwdB4BFaPXq1Vm9evXQMQCACc1y2fOrk/xKVR2f5JlJ\nnp3k95IcWFX7d7O7y5Pc3Z1/d5LDkuyuqv2TPCfJnnnjj5r/OU80DgB71UMPPTR0BABgCjOb+W2t\nrW+tLW+trcjchlV/0lr7tSSfTnJKd9rpST7Zvb6mO073/p+01lo3/uZuN+gXJjk8yS1JPp/k8G73\n6Gd03+OaWf08AAAALF4z3fDqCfyrJFdU1fuTfCnJx7rxjyW5rNvQ6v7Mldm01r5cVZszt5HVI0ne\n1Vr7cZJU1W8l+VSS/ZJc3Fr78j79SQAAAFgU9kn5ba1tS7Kte31n/s9uzfPP+UGSX32Cz/9A5naM\nfuz49Umu34tRAQAAGKEhZn4BYNE58cQTh44AAExB+QWAHs4888yhIwAAU5j1c34BAABgcMovAPSw\natWqrFq1augYAMCElF8AAABGT/kFAABg9JRfAAAARk/5BQAAYPQ86ggAejj11FOHjgAATEH5BYAe\nzjjjjKEjAABTsOwZAHp48MEH8+CDDw4dAwCYkJlfAOjh+OOPT5Js27Zt2CAAwETM/AIAADB6yi8A\nAACjp/wCAAAwesovAAAAo2fDKwDo4W1ve9vQEQCAKSi/ANCD8gsAi5tlzwDQw3333Zf77rtv6BgA\nwITM/AJAD6ecckoSz/kFgMXKzC8AAACjp/wCAAAwesovAAAAo6f8AgAAMHo2vAKAHt75zncOHQEA\nmILyCwA9nHbaaUNHAACmYNkzAPSwa9eu7Nq1a+gYAMCEzPwCQA9vfetbk3jOLwAsVmZ+AQAAGD3l\nFwAAgNFTfgEAABg95RcAAIDRs+EVAPTw7ne/e+gIAMAUlF8A6OGkk04aOgIAMAXLngGghx07dmTH\njh1DxwAAJmTmFwB6eMc73pHEc34BYLEy8wsAAMDoKb8AAACMnvILAADA6Cm/AAAAjJ4NrwCgh/e8\n5z1DRwAApqD8AkAPxx577NARAIApWPYMAD1s374927dvHzoGADAhM78A0MO6deuSeM4vACxWZn4B\nAAAYPeUXAACA0VN+AQAAGD3lFwAAgNGz4RUA9HDuuecOHQEAmILyCwA9vOpVrxo6AgAwBcueAaCH\nm2++OTfffPPQMQCACZn5BYAezj777CSe8wsAi5WZXwAAAEZP+QUAAGD0lF8AAABGT/kFAABg9Gx4\nBQA9XHDBBUNHAACmoPwCQA8rV64cOgIAMAXLngGgh5tuuik33XTT0DEAgAmZ+QWAHt7//vcnSY49\n9tiBkwAAkzDzCwAAwOgpvwAAAIye8gsAAMDoKb8AAACMng2vAKCHj370o0NHAACmoPwCQA9HHHHE\n0BEAgClY9gwAPVx77bW59tprh44BAEzIzC8A9PChD30oSXLSSScNnAQAmISZXwAAAEZP+QUAAGD0\nlF8AAABGT/kFAABg9Gx4BQA9XHbZZUNHAACmoPwCQA+HHXbY0BEAgClY9gwAPVx55ZW58sorh44B\nAEzIzC8A9HDhhRcmSU477bSBkwAAkzDzCwAAwOgpvwAAAIye8gsAAMDoKb8AAACMng2vAKCHq666\naugIAMAUlF8A6OHggw8eOgIAMAXLngGgh0suuSSXXHLJ0DEAgAkpvwDQg/ILAIub8gsAAMDoKb8A\nAACMng2vgL9hxVnXDR1hFO46/4ShIwAA0DHzCwAAwOiZ+QWAHq6//vqhIwAAU1B+AaCHZcuWDR0B\nAJiCZc8A0MOmTZuyadOmoWMAABNSfgGgh82bN2fz5s1DxwAAJqT8AgAAMHrKLwAAAKOn/AIAADB6\nyi8AAACjN7PyW1XPrKpbquq/V9WXq+rfdOMvrKrPVdXOqrqyqp7Rjf+t7nhn9/6KeV9rfTe+o6re\nMG/8uG5sZ1WdNaufBQC2bduWbdu2DR0DAJjQLGd+f5jkta21X0iyMslxVfXKJB9M8uHW2ouSfCvJ\n27vz357kW934h7vzUlVHJnlzkr+f5Lgkm6pqv6raL8kfJFmd5Mgkb+nOBQAAgJ8ys/Lb5nyvO3x6\n99GSvDbJVd34pUne1L1+Y3ec7v1jqqq68Staaz9srf1lkp1JXt597Gyt3dlaezjJFd25ALDXbdy4\nMRs3bhw6BgAwoZne89vN0G5Pck+SG5P8zyTfbq090p2yO8mh3etDk+xKku797yQ5aP74Yz7nicYf\nL8faqrq1qm69995798aPBsASs2XLlmzZsmXoGADAhGZafltrP26trUyyPHMztS+Z5fd7khwXtdaO\naq0ddcghhwwRAQAAgAHtk92eW2vfTvLpJEcnObCq9u/eWp7k7u713UkOS5Lu/eck2TN//DGf80Tj\nAAAA8FNmudvzIVV1YPf6gCSvS3J75krwKd1ppyf5ZPf6mu443ft/0lpr3fibu92gX5jk8CS3JPl8\nksO73aOfkblNsa6Z1c8DAADA4rX/U58ysRckubTblflpSTa31rZU1VeSXFFV70/ypSQf687/WJLL\nqmpnkvszV2bTWvtyVW1O8pUkjyR5V2vtx0lSVb+V5FNJ9ktycWvtyzP8eQBYwg444IChIwAAU6i5\nydWl46ijjmq33nrr0DGe0Iqzrhs6ArCX3HX+CUNHAABYUqrqC621ox7vvX1yzy8AAAAMSfkFgB42\nbNiQDRs2DB0DAJiQ8gsAPWzdujVbt24dOgYAMCHlFwAAgNFTfgEAABg95RcAAIDRm+VzfgFgNA46\n6KChIwAAU1B+AaCHq6++eugIAMAULHsGAABg9JRfAOhh/fr1Wb9+/dAxAIAJWfYMAD189rOfHToC\nADAFM78AAACMnvILAADA6Cm/AAAAjJ57fgGgh+XLlw8dAQCYgvILAD1cfvnlQ0cAAKZg2TMAAACj\np/wCQA/r1q3LunXrho4BAEzIsmcA6GH79u1DRwAApmDmFwAAgNFTfgEAABg95RcAAIDRc88vAPTw\n4he/eOgIAMAUlF8A6OGiiy4aOgIAMAXLngEAABg95RcAeli7dm3Wrl07dAwAYEKWPQNAD3fcccfQ\nEQCAKZj5BQAAYPSUXwAAAEZP+QUAAGD03PMLAD2sXLly6AgAwBSUXwDo4YILLhg6AgAwBcueAQAA\nGD3lFwB6WLNmTdasWTN0DABgQpY9A0APu3fvHjoCADAF5RdgRlacdd3QERa9u84/YegIAMBIWPYM\nAADA6Cm/AAAAjJ5lzwDQw9FHHz10BABgCsovAPRw3nnnDR0BAJhCr2XPVfUPZx0EAAAAZqXvPb+b\nquqWqjqjqp4z00QAsACdfPLJOfnkk4eOAQBMqFf5ba39UpJfS3JYki9U1X+oqtfNNBkALCB79uzJ\nnj17ho4BAEyo927PrbWvJnlPkn+V5J8m+UhV/UVV/d+zCgcAAAB7Q997fv9RVX04ye1JXpvkpNba\nz3evPzzDfAAAADC1vrs9/39J/jDJ2a21hx4dbK39VVW9ZybJAAAAYC/pW35PSPJQa+3HSVJVT0vy\nzNbag621y2aWDgAWiGOOOWboCADAFPqW35uSHJvke93xsiT/JcmrZhEKABaac845Z+gIAMAU+m54\n9czW2qPFN93rZbOJBAAAAHtX3/L7/ap62aMHVfWLSR56kvMBYFRWr16d1atXDx0DAJhQ32XP65L8\np6r6qySV5O8kOW1mqQBggXnoIf/mCwCLWa/y21r7fFW9JMkR3dCO1tqPZhcLAAAA9p6+M79J8o+T\nrOg+52VVldbax2eSCgAAAPaiXuW3qi5L8n8l2Z7kx91wS6L8AgAAsOD1nfk9KsmRrbU2yzAAsFCd\neOKJQ0cAAKbQt/z+eeY2ufr6DLMAwIJ15plnDh0BAJhC3/J7cJKvVNUtSX746GBr7VdmkgoAAAD2\nor7l932zDAEAC92qVauSJNu2bRs0BwAwmb6POvqvVfX3khzeWrupqpYl2W+20QAAAGDveFqfk6rq\nN5NcleSj3dChST4xq1AAAACwN/Uqv0neleTVSR5IktbaV5P87VmFAgAAgL2pb/n9YWvt4UcPqmr/\nzD3nFwAAABa8vhte/deqOjvJAVX1uiRnJLl2drEAYGE59dRTh44AAEyhb/k9K8nbk9yW5B1Jrk/y\nh7MKBQALzRlnnDF0BABgCn13e/5Jkn/bfQDAkvPggw8mSZYtWzZwEgBgEr3Kb1X9ZR7nHt/W2s/t\n9UQAsAAdf/zxSTznFwAWq77Lno+a9/qZSX41yfP2fhwAAADY+3rt9txa2zPv4+7W2gVJTphxNgAA\nANgr+i57ftm8w6dlbia476wxAAAADKpvgf3QvNePJLkriWc+AAAAsCj03e35l2cdBAAWsre97W1D\nRwAAptB32fO/eLL3W2u/u3fiAMDCpPwCwOL2/2e353+c5Jru+KQktyT56ixCAcBCc9999yVJDj74\n4IGTAACT6Ft+lyd5WWvtu0lSVe9Lcl1rbc2sggHAQnLKKack8ZxfAFisej3qKMnzkzw87/jhbgwA\nAAAWvL4zvx9PcktV/XF3/KYkl84mEgAAAOxdfXd7/kBV3ZDkl7qh32itfWl2sQAAAGDv6bvsOUmW\nJXmgtfZ7SXZX1QtnlAkAAAD2qr6POnpv5nZ8PiLJv0vy9CSXJ3n17KIBwMLxzne+c+gIAMAU+t7z\n+8+SvDTJF5OktfZXVfWzM0sFAAvMaaedNnQEAGAKfZc9P9xaa0laklTVs2YXCQAWnl27dmXXrl1D\nxwAAJtR35ndzVX00yYFV9ZtJ/p8k/3Z2sQBgYXnrW9+axHN+AWCx6rvb88aqel2SBzJ33++/bq3d\nONNkAAAAsJc8Zfmtqv2S3NRa++UkCi8AAACLzlPe89ta+3GSn1TVc/ZBHgAAANjr+t7z+70kt1XV\njUm+/+hga+3/nUkqAAAA2Iv6lt8/6j4AYEl697vfPXQEAGAKT1p+q+rvttb+V2vt0n0VCAAWopNO\nOmnoCADAFJ7qnt9PPPqiqq6ecRYAWLB27NiRHTt2DB0DAJjQUy17rnmvf26WQQBgIXvHO96RxHN+\nAWCxeqqZ3/YErwEAAGDReKqZ31+oqgcyNwN8QPc63XFrrT17pukAAABgL3jS8tta229fBQEAAIBZ\neaplzwAAALDo9X3OLwAsae95z3uGjgAATEH5BYAejj322KEjAABTmNmy56o6rKo+XVVfqaovV9Vv\nd+PPq6obq+qr3Z/P7carqj5SVTur6n9U1cvmfa3Tu/O/WlWnzxv/xaq6rfucj1RV/c0kADC97du3\nZ/v27UPHAAAmNMt7fh9J8u7W2pFJXpnkXVV1ZJKzkmxtrR2eZGt3nCSrkxzefaxNcmEyV5aTvDfJ\nK5K8PMl7Hy3M3Tm/Oe/zjpvhzwPAErZu3bqsW7du6BgAwIRmVn5ba19vrX2xe/3dJLcnOTTJG5Nc\n2p12aZI3da/fmOTjbc6fJTmwql6Q5A1Jbmyt3d9a+1aSG5Mc17337Nban7XWWpKPz/taAAAA8Nf2\nyW7PVbUiyUuTfC7J81trX+/e+kaS53evD02ya96n7e7Gnmx89+OMP973X1tVt1bVrffee+9UPwsA\nAACLz8zLb1X9TJKrk6xrrT0w/71uxrbNOkNr7aLW2lGttaMOOeSQWX87AAAAFpiZlt+qenrmiu+/\nb639UTf8zW7Jcro/7+nG705y2LxPX96NPdn48scZBwAAgJ8yy92eK8nHktzeWvvdeW9dk+TRHZtP\nT/LJeeO/3u36/Mok3+mWR38qyeur6rndRlevT/Kp7r0HquqV3ff69XlfCwD2qnPPPTfnnnvu0DEA\ngAnN8jm/r07y1iS3VdWjz4Y4O8n5STZX1duTfC3Jqd171yc5PsnOJA8m+Y0kaa3dX1Ubkny+O+93\nWmv3d6/PSHJJkgOS3NB9AMBe96pXvWroCADAFGZWfltr/y3JEz1395jHOb8ledcTfK2Lk1z8OOO3\nJvkHU8QEgF5uvvnmJEowACxWs5z5BYDROPvss5Mk27ZtGzYIADCRffKoIwAAABiS8gsAAMDoKb8A\nAACMnvILAADA6NnwCgB6uOCCC4aOAABMQfkFgB5Wrlw5dAQAYAqWPQNADzfddFNuuummoWMAABMy\n8wsAPbz//e9Pkhx77LEDJwEAJmHmFwAAgNFTfgEAABg95RcAAIDRU34BAAAYPRteAUAPH/3oR4eO\nAABMQfkFgB6OOOKIoSMAAFOw7BkAerj22mtz7bXXDh0DAJiQmV8A6OFDH/pQkuSkk04aOAkAMAkz\nvwAAAIye8gsAAMDoKb8AAACMnvILAADA6NnwCgB6uOyyy4aOAABMQfkFgB4OO+ywoSMAAFOw7BkA\nerjyyitz5ZVXDh0DAJiQmV8A6OHCCy9Mkpx22mkDJwEAJmHmFwAAgNFTfgEAABg95RcAAIDRU34B\nAAAYPRteAUAPV1111dARAIApKL8A0MPBBx88dAQAYAqWPQNAD5dcckkuueSSoWMAABNSfgGgB+UX\nABY35RcAAIDRU34BAAAYPeUXAACA0VN+AQAAGD2POgKAHq6//vqhIwAAU1B+AaCHZcuWDR0BAJiC\nZc8A0MOmTZuyadOmoWMAABNSfgGgh82bN2fz5s1DxwAAJqT8AgAAMHrKLwAAAKOn/AIAADB6yi8A\nAACj51FHANDDtm3bho4AAEzBzC8AAACjp/wCQA8bN27Mxo0bh44BAExI+QWAHrZs2ZItW7YMHQMA\nmJDyCwAAwOgpvwAAAIye8gsAAMDoedQRAPRwwAEHDB0BAJiC8gsAPdxwww1DRwAApmDZMwAAAKOn\n/AJADxs2bMiGDRuGjgEATEj5BYAetm7dmq1btw4dAwCYkPILAADA6Cm/AAAAjJ7yCwAAwOh51BEA\n9HDQQQcNHQEAmILyCwA9XH311UNHAACmYNkzAAAAo6f8AkAP69evz/r164eOAQBMyLJnAOjhs5/9\n7NARAIApmPkFAABg9JRfAAAARk/5BQAAYPTc8wsAPSxfvnzoCADAFJRfAOjh8ssvHzoCADAFy54B\nAAAYPeUXAHpYt25d1q1bN3QMAGBClj0DQA/bt28fOgIAMAUzvwAAAIye8gsAAMDoKb8AAACMnnt+\nAaCHF7/4xUNHAACmoPwCQA8XXXTR0BEAgClY9gwAAMDoKb8A0MPatWuzdu3aoWMAABOy7BkAerjj\njjuGjgAATMHMLwAAAKOn/AIAADB6yi8AAACj555fAOhh5cqVQ0cAAKag/AJADxdccMHQEQCAKVj2\nDAAAwOgpvwDQw5o1a7JmzZqhYwAAE7LsGQB62L1799ARAIApmPkFAABg9JRfAAAARk/5BQAAYPTc\n8wsAPRx99NFDRwAApqD8AkAP55133tARAIApWPYMAADA6Cm/ANDDySefnJNPPnnoGADAhGZWfqvq\n4qq6p6r+fN7Y86rqxqr6avfnc7vxqqqPVNXOqvofVfWyeZ9zenf+V6vq9Hnjv1hVt3Wf85Gqqln9\nLACwZ8+e7NmzZ+gYAMCEZjnze0mS4x4zdlaSra21w5Ns7Y6TZHWSw7uPtUkuTObKcpL3JnlFkpcn\nee+jhbk75zfnfd5jvxcAAAAkmWH5ba19Jsn9jxl+Y5JLu9eXJnnTvPGPtzl/luTAqnpBkjckubG1\ndn9r7VtJbkxyXPfes1trf9Zaa0k+Pu9rAQAAwE/Z1/f8Pr+19vXu9TeSPL97fWiSXfPO292NPdn4\n7scZf1xVtbaqbq2qW++9997pfgIAAAAWncEeddRaa1XV9tH3uijJRUly1FFH7ZPvCcC4HHPMMUNH\nAACmsK/L7zer6gWtta93S5fv6cbvTnLYvPOWd2N3J1n1mPFt3fjyxzkfAGbinHPOGToCADCFfb3s\n+Zokj+7YfHqST84b//Vu1+dXJvlOtzz6U0leX1XP7Ta6en2ST3XvPVBVr+x2ef71eV8LAAAAfsrM\nZn6r6j9mbtb24Kranbldm89Psrmq3p7ka0lO7U6/PsnxSXYmeTDJbyRJa+3+qtqQ5PPdeb/TWnt0\nE60zMrej9AFJbug+AGAmVq9enSS54Qa/bgBgMZpZ+W2tveUJ3vobN011Oza/6wm+zsVJLn6c8VuT\n/INpMgJAXw899NDQEQCAKezrZc8AAACwzym/AAAAjJ7yCwAAwOgN9pxfAFhMTjzxxKEjAABTUH4B\noIczzzxz6AgAwBQsewYAAGD0lF8A6GHVqlVZtWrV0DEAgAkpvwAAAIye8gsAAMDoKb8AAACMnt2e\nAViwVpx13dAR/to37tyTZGFl6uOu808YOgIALAjKLwD08KyX/NLQEQCAKSi/ANDDz77MDCoALGbu\n+QWAHn7yox/kJz/6wdAxAIAJmfkFgB7u+U/vS5L8nX9+/rBBAICJmPkFAABg9JRfAAAARk/5BQAA\nYPSUXwAAAEbPhlcA0MPP/MNjh44AAExB+QWAHpRfAFjcLHsGgB5+/OB38uMHvzN0DABgQsovAPRw\n7yfOy72fOG/oGADAhJRfAAAARk/5BQAAYPSUXwAAAEZP+QUAAGD0POoIAHr42ZceP3QEAGAKyi8A\n9PCsn3/N0BEAgClY9gwAPTzywL155IF7h44BAExI+QWAHu7b8qHct+VDQ8cAACak/AIAADB6yi8A\nAACjp/wCAAAwesovAAAAo+dRRwDQw7Nf/s+GjgAATEH5BYAelr3oFUNHAACmYNkzAPTwoz2786M9\nu4eOAQBMSPkFgB72fOr3s+dTvz90DABgQsovAAAAo6f8AgAAMHrKLwAAAKOn/AIAADB6HnUEAD08\n51VvHjoCADAF5RcAejhgxcqhIwAAU7DsGQB6ePibd+bhb945dAwAYELKLwD0cP/Wi3L/1ouGjgEA\nTEj5BQAAYPSUXwAAAEZP+QUAAGD0lF8AAABGz6OOAKCHA19z+tARAIApKL8A0MMzl//80BEAgClY\n9gwAPfxg9+35we7bh44BAExI+QWAHr79mUvz7c9cOnQMAGBCyi8AAACjp/wCAAAwesovAAAAo6f8\nAgAAMHoedQQAPTzvmLVDRwAApqD8AkAPz3j+zw0dAQCYgmXPANDDQ3dtz0N3bR86BgAwITO/ANDD\nd26+IklywIqVAycBACZh5hcAAIDRU34BAAAYPeUXAACA0VN+AQAAGD0bXgFADwe94beGjgAATEH5\nBYAenn7Q8qEjAABTsOwZAHp4cOfn8uDOzw0dAwCYkJlfAOjhgVv+OEmy7EWvGDgJADAJM78AAACM\nnvILAADA6Cm/AAAAjJ7yC30eJMQAAAZuSURBVAAAwOjZ8AoAejj4xHcPHQEAmILyCwA97P/sQ4aO\nAABMwbJnAOjh+7d/Jt+//TNDxwAAJmTmFwB6+O6Xrk+SPOvnXzNwEgBgEmZ+AQAAGD3lFwAAgNFT\nfgEAABg95RcAAIDRs+EVAPRwyJvWDx0BAJiC8gsAPey37DlDRwAApmDZMwD08L3bbsr3brtp6BgA\nwISUXwDoQfkFgMVN+QUAAGD0lF8AAABGT/kFAABg9Oz2DAAjtuKs64aOsOjddf4JQ0cAYC9QfgGg\nh7/9q+8bOgIAMAXlFwB6eNrTnzl0BABgCu75BYAevvvF6/LdL1pCDACLlfILAD18/y/+NN//iz8d\nOgYAMKFFX36r6riq2lFVO6vqrKHzAAAAsPAs6vJbVfsl+YMkq5McmeQtVXXksKkAAABYaBZ1+U3y\n8iQ7W2t3ttYeTnJFkjcOnAkAAIAFZrHv9nxokl3zjncnecVAWQCAEfKs5Ol5VjKwECz28ttLVa1N\nsrY7/F5V7Zjgyxyc5L69l4pFyDWAa4CDv/bBE10DS5v/D0ygPjh0gr3KNbC0+ftf+P7eE72x2Mvv\n3UkOm3e8vBv7Ka21i5JcNM03qqpbW2tHTfM1WNxcA7gGcA3gGsA1sLT5+1/cFvs9v59PcnhVvbCq\nnpHkzUmuGTgTAAAAC8yinvltrT1SVb+V5FNJ9ktycWvtywPHAgAAYIFZ1OU3SVpr1ye5fh98q6mW\nTTMKrgFcA7gGcA3gGlja/P0vYtVaGzoDAAAAzNRiv+cXAAAAnpLy20NVHVdVO6pqZ1WdNXQeZq+q\nLq6qe6rqz+eNPa+qbqyqr3Z/PnfIjMxOVR1WVZ+uqq9U1Zer6re7cdfAElFVz6yqW6rqv3fXwL/p\nxl9YVZ/rfh9c2W22yIhV1X5V9aWq2tIduwaWkKq6q6puq6rtVXVrN+Z3wRJSVQdW1VVV9RdVdXtV\nHe0aWLyU36dQVfsl+YMkq5McmeQtVXXksKnYBy5Jctxjxs5KsrW1dniSrd0x4/RIkne31o5M8sok\n7+r+u3cNLB0/TPLa1tovJFmZ5LiqemWSDyb5cGvtRUm+leTtA2Zk3/jtJLfPO3YNLD2/3FpbOe/x\nNn4XLC2/l+Q/t9ZekuQXMvf/A9fAIqX8PrWXJ9nZWruztfZwkiuSvHHgTMxYa+0zSe5/zPAbk1za\nvb40yZv2aSj2mdba11trX+xefzdzv+gOjWtgyWhzvtcdPr37aElem+Sqbtw1MHJVtTzJCUn+sDuu\nuAbwu2DJqKrnJHlNko8lSWvt4dbat+MaWLSU36d2aJJd8453d2MsPc9vrX29e/2NJM8fMgz7RlWt\nSPLSJJ+La2BJ6Za7bk9yT5Ibk/zPJN9urT3SneL3wfhdkORfJvlJd3xQXANLTUvyX6rqC1W1thvz\nu2DpeGGSe5P8u+72hz+sqmfFNbBoKb8wgTa3Tbqt0keuqn4mydVJ1rXWHpj/nmtg/FprP26trUyy\nPHOrgF4ycCT2oao6Mck9rbUvDJ2FQf2T1trLMnf727uq6jXz3/S7YPT2T/KyJBe21l6a5Pt5zBJn\n18Diovw+tbuTHDbveHk3xtLzzap6QZJ0f94zcB5mqKqenrni++9ba3/UDbsGlqBuidunkxyd5MCq\n2r97y++DcXt1kl+pqrsyd8vTazN3759rYAlprd3d/XlPkj/O3D+E+V2wdOxOsru19rnu+KrMlWHX\nwCKl/D61zyc5vNvd8RlJ3pzkmoEzMYxrkpzevT49yScHzMIMdff1fSzJ7a213533lmtgiaiqQ6rq\nwO71AUlel7l7vz+d5JTuNNfAiLXW1rfWlrfWVmTud/+ftNZ+La6BJaOqnlVVP/vo6ySvT/Ln8btg\nyWitfSPJrqo6ohs6JslX4hpYtGpupp4nU1XHZ+6+n/2SXNxa+8DAkZixqvqPSVYlOTjJN5O8N8kn\nkmxO8neTfC3Jqa21x26KxQhU1T9J8qdJbsv/udfv7Mzd9+saWAKq6h9lbhOT/TL3D8WbW2u/U1U/\nl7lZwOcl+VKSNa21Hw6XlH2hqlYlObO1dqJrYOno/q7/uDvcP8l/aK19oKoOit8FS0ZVrczcpnfP\nSHJnkt9I93shroFFR/kFAABg9Cx7BgAAYPSUXwAAAEZP+QUAAGD0lF8AAABGT/kFAABg9JRfAAAA\nRk/5BQAAYPSUXwAAAEbvfwPWb9ZoBJhJbwAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 1152x648 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VwwgvvizZvUz",
        "colab_type": "text"
      },
      "source": [
        "## Load fastText word vectors"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PcWej6mDZvU0",
        "colab_type": "code",
        "outputId": "14e41dc3-8d73-4b14-ae72-05037f03be29",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 83,
          "referenced_widgets": [
            "4d8a7111c88f4946af0aa37656c22e20",
            "fcca42ac8786470da5ff030dc7bd093c",
            "15088d1cc38c44369d492eb46179e09a",
            "f92b830f020c4c80911f68d982960c21",
            "6dc45aa8232744d2b8970e40cfef033d",
            "60c3bef3e2a14f048f4c599d8deb2182",
            "66d0c37d59174d4fa06011b24c783301",
            "219cb780eb954638bdc9548ba979cd7e"
          ]
        }
      },
      "source": [
        "word_vectors = {}\n",
        "f = io.open(\"/content/cc.fr.300.vec\", encoding='utf-8')\n",
        "n, d = map(int, f.readline().split())\n",
        "for line in tqdm_notebook(f):\n",
        "    values = line.rstrip().split(' ')\n",
        "    word = values[0]\n",
        "    vector = np.array(values[1:], dtype='float32')\n",
        "    word_vectors[word] = vector\n",
        "f.close()\n",
        "print(f'found {len(word_vectors)} word vectors')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "4d8a7111c88f4946af0aa37656c22e20",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "found 2000000 word vectors\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rIb_lZC9ZvU5",
        "colab_type": "text"
      },
      "source": [
        "## Pre-processing"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Fz8nBPxNZvU6",
        "colab_type": "text"
      },
      "source": [
        "### Clean"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "scrolled": false,
        "id": "H1TtV1dEZvU7",
        "colab_type": "code",
        "outputId": "fa740da1-5b28-40ec-e869-eb3a8849895b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        }
      },
      "source": [
        "df_dataset.head()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>label</th>\n",
              "      <th>text</th>\n",
              "      <th>sent_len</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>471040</th>\n",
              "      <td>0</td>\n",
              "      <td>El som et la société humaine avec cindy! Je ve...</td>\n",
              "      <td>16</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1340507</th>\n",
              "      <td>1</td>\n",
              "      <td>Quel beau jour d'été</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>757669</th>\n",
              "      <td>0</td>\n",
              "      <td>Déteste assis à l'intérieur quand c'est une be...</td>\n",
              "      <td>9</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>119639</th>\n",
              "      <td>0</td>\n",
              "      <td>Lundi matin même pas le soleil</td>\n",
              "      <td>6</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1190949</th>\n",
              "      <td>1</td>\n",
              "      <td>La soeur a juste été bronzée: soph: 'what are ...</td>\n",
              "      <td>23</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "         label                                               text  sent_len\n",
              "471040       0  El som et la société humaine avec cindy! Je ve...        16\n",
              "1340507      1                               Quel beau jour d'été         4\n",
              "757669       0  Déteste assis à l'intérieur quand c'est une be...         9\n",
              "119639       0                     Lundi matin même pas le soleil         6\n",
              "1190949      1  La soeur a juste été bronzée: soph: 'what are ...        23"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8n2fAcHsZvU-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "regex_tokenizer = RegexpTokenizer(r'\\w+')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hcx38CGGZvVB",
        "colab_type": "code",
        "outputId": "2b274f2f-9773-4ab8-ae7f-c7e92beab622",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 66,
          "referenced_widgets": [
            "bf46ca3a52004a72b4563e48df280ffb",
            "212c373c4a944e3f855c4095842b1ae7",
            "9adb3afda5f0416786a765abf0a58a1d",
            "7f07ec67876c4f798fd357cb57ac0932",
            "f8b6b893924d4594a43f49d1428e59c7",
            "db1e73cffcbd4042982c831f60765ea6",
            "5c749d61747f4a5abdde0d0b71b2f4aa",
            "10144f501736432cbda146877e2b48f4"
          ]
        }
      },
      "source": [
        "cleaned_text = []\n",
        "\n",
        "for text in tqdm_notebook(df_dataset['text']):\n",
        "    tokens = regex_tokenizer.tokenize(text)\n",
        "    cleaned_text.append(\" \".join(tokens))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "bf46ca3a52004a72b4563e48df280ffb",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(IntProgress(value=0, max=152672), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rdgqY2MOZvVE",
        "colab_type": "code",
        "outputId": "4cb95a89-928a-49f1-ca7a-c3cacf499b9c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "print(cleaned_text[0])"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "El som et la société humaine avec cindy Je veux vraiment un chiot ou un chaton\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WkGvsotdZvVJ",
        "colab_type": "text"
      },
      "source": [
        "### Tokenize"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "m_99kIzjZvVK",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "max_words = 1e5"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5VyXMIl1ZvVN",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "tokenizer = Tokenizer(num_words=max_words, lower=True, char_level=False)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-1OjVtK1ZvVR",
        "colab_type": "code",
        "outputId": "1ca7d87b-cb26-494e-949a-9ba42ad08ac5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "tokenizer.fit_on_texts(cleaned_text)\n",
        "input_sequences = tokenizer.texts_to_sequences(cleaned_text)\n",
        "\n",
        "word_index = tokenizer.word_index\n",
        "print(\"dictionary size: \", len(word_index))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "dictionary size:  66923\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xnVesnTbZvVU",
        "colab_type": "code",
        "outputId": "27029eea-3245-4d41-d1d9-6ab80d7be653",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "print(input_sequences[0])"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[3530, 15084, 9, 5, 2152, 6488, 33, 8818, 1, 77, 58, 11, 1173, 109, 11, 2811]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8woiteInZvVY",
        "colab_type": "text"
      },
      "source": [
        "### Padding"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RRC2PB1UZvVZ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "input_sequences = pad_sequences(input_sequences, maxlen=max_seq_len)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "J9xLO9QwZvVd",
        "colab_type": "code",
        "outputId": "d3820f85-a118-4233-80f6-38a8a558db9b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 68
        }
      },
      "source": [
        "print(input_sequences[0])"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[    0     0     0     0     0     0     0     0     0     0     0     0\n",
            "     0     0  3530 15084     9     5  2152  6488    33  8818     1    77\n",
            "    58    11  1173   109    11  2811]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZC9t1SJmZvVg",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "labels = df_dataset['label'].values"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PiL9P2-SZvVk",
        "colab_type": "code",
        "outputId": "17d56e8d-54ca-43c4-9545-7e0cbdde0077",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "print(labels[0])"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZzVUxYIfZvVn",
        "colab_type": "text"
      },
      "source": [
        "## Embedding matrix"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fP2wR9ZbZvVo",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "embed_dim = 300"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "w4wBP5MrZvVr",
        "colab_type": "code",
        "outputId": "0a7b0751-a88d-4d38-acc5-600247fdf1f2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "words_not_found = []\n",
        "nb_words = min(max_words, len(word_index))\n",
        "\n",
        "embedding_matrix = np.zeros((nb_words+1, embed_dim))\n",
        "\n",
        "for word, i in word_index.items():\n",
        "    if i > nb_words:\n",
        "        break\n",
        "\n",
        "    vector = word_vectors.get(word)\n",
        "    if vector is not None:\n",
        "        # words not found in embedding index will be all-zeros.\n",
        "        embedding_matrix[i] = vector\n",
        "    else:\n",
        "        words_not_found.append(word)\n",
        "\n",
        "print('number of null word embeddings: {:d}'.format(np.sum(np.sum(embedding_matrix, axis=1) == 0)))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "number of null word embeddings: 24344\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3S5Ve5iiZvVu",
        "colab_type": "code",
        "outputId": "46f81c2c-a754-4466-9b98-55d4d9dc86af",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "print(\"sample words not found: \", np.random.choice(words_not_found, 10))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "sample words not found:  ['avvo' 'shizzzz' 'bessee' 'mleh' 'microsciper' 'chillaxing' 'mallows'\n",
            " 'traci' 'peaple' 'linkety']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "d-8XEa_IZvVy",
        "colab_type": "text"
      },
      "source": [
        "## Train Test Split"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fSo1UHTGZvVz",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "X_train, X_test, y_train, y_test = (\n",
        "    train_test_split(input_sequences, labels, random_state=42, test_size=0.2)\n",
        ")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PbJS6U1vZvV2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "X_train = tf.constant(X_train)\n",
        "X_test = tf.constant(X_test)\n",
        "\n",
        "y_train = tf.constant(y_train)\n",
        "y_test = tf.constant(y_test)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "x96WC67rZvV8",
        "colab_type": "code",
        "outputId": "7dcf107f-fe66-43d3-cedb-6728ba8e835e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "print(f'Train | X shape: {X_train.shape}, y shape: {y_train.shape}')\n",
        "print(f'Test | X shape: {X_test.shape}, y shape: {y_test.shape}')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Train | X shape: (122137, 30), y shape: (122137,)\n",
            "Test | X shape: (30535, 30), y shape: (30535,)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "olAVq30hZvV_",
        "colab_type": "text"
      },
      "source": [
        "## Create model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4Di75fGNZvWA",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def create_model(input_len):\n",
        "    inp = Input(shape=(input_len,))\n",
        "\n",
        "    x = Embedding(nb_words+1, embed_dim,\n",
        "                  weights=[embedding_matrix],\n",
        "                  input_length=input_len, trainable=False)(inp)\n",
        "    \n",
        "    x1 = Conv1D(200, 3, activation='relu', padding='same')(x)\n",
        "    x1 = GlobalMaxPooling1D()(x1)\n",
        "\n",
        "    x2 = Conv1D(200, 4, activation='relu', padding='same')(x)\n",
        "    x2 = GlobalMaxPooling1D()(x2)\n",
        "\n",
        "    x3 = Conv1D(200, 5, activation='relu', padding='same')(x)\n",
        "    x3 = GlobalMaxPooling1D()(x3)\n",
        "\n",
        "    x = concatenate([x1, x2, x3])\n",
        "    x = Dropout(0.5)(x)\n",
        "    x = Dense(30, activation='relu')(x)\n",
        "    x = Dropout(0.5)(x)\n",
        "    x = Dense(1, activation='sigmoid')(x)\n",
        "\n",
        "    model = Model(inputs=inp, outputs=x)\n",
        "\n",
        "    opt = tf.keras.optimizers.Adam(lr=0.001)\n",
        "\n",
        "    model.compile(loss='binary_crossentropy',\n",
        "                  optimizer=opt,\n",
        "                  metrics=['accuracy'])\n",
        "\n",
        "    return model"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "scrolled": true,
        "id": "J4oTEp97ZvWC",
        "colab_type": "code",
        "outputId": "2d1d627b-4d37-4676-9867-0cd04f0dbf45",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 629
        }
      },
      "source": [
        "model = create_model(max_seq_len)\n",
        "model.summary()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"model_1\"\n",
            "__________________________________________________________________________________________________\n",
            "Layer (type)                    Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            "input_2 (InputLayer)            [(None, 30)]         0                                            \n",
            "__________________________________________________________________________________________________\n",
            "embedding_1 (Embedding)         (None, 30, 300)      20077200    input_2[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "conv1d_3 (Conv1D)               (None, 30, 200)      180200      embedding_1[0][0]                \n",
            "__________________________________________________________________________________________________\n",
            "conv1d_4 (Conv1D)               (None, 30, 200)      240200      embedding_1[0][0]                \n",
            "__________________________________________________________________________________________________\n",
            "conv1d_5 (Conv1D)               (None, 30, 200)      300200      embedding_1[0][0]                \n",
            "__________________________________________________________________________________________________\n",
            "global_max_pooling1d_3 (GlobalM (None, 200)          0           conv1d_3[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "global_max_pooling1d_4 (GlobalM (None, 200)          0           conv1d_4[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "global_max_pooling1d_5 (GlobalM (None, 200)          0           conv1d_5[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_1 (Concatenate)     (None, 600)          0           global_max_pooling1d_3[0][0]     \n",
            "                                                                 global_max_pooling1d_4[0][0]     \n",
            "                                                                 global_max_pooling1d_5[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "dropout_2 (Dropout)             (None, 600)          0           concatenate_1[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "dense_2 (Dense)                 (None, 30)           18030       dropout_2[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "dropout_3 (Dropout)             (None, 30)           0           dense_2[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "dense_3 (Dense)                 (None, 1)            31          dropout_3[0][0]                  \n",
            "==================================================================================================\n",
            "Total params: 20,815,861\n",
            "Trainable params: 738,661\n",
            "Non-trainable params: 20,077,200\n",
            "__________________________________________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9pTzhS76ZvWE",
        "colab_type": "text"
      },
      "source": [
        "## Training"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "scrolled": true,
        "id": "JxwSsECoZvWF",
        "colab_type": "code",
        "outputId": "da5b5998-dc7e-40d5-c3c2-cf95967978e5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "loss, metric = model.evaluate(X_test, y_test, batch_size=32, verbose=0)\n",
        "print(f\"Loss before training: {loss:.4f}, Accuracy before training: {metric:.2%}\")"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Loss before training: 0.6909, Accuracy before training: 51.53%\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "scrolled": true,
        "id": "VawyxCyLZvWH",
        "colab_type": "code",
        "outputId": "cf1497cc-f89b-4bef-b4c3-2e74e68b2d5b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        }
      },
      "source": [
        "history = model.fit(X_train, y_train, batch_size=256, epochs=5, validation_data=(X_test, y_test))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Train on 122137 samples, validate on 30535 samples\n",
            "Epoch 1/5\n",
            "122137/122137 [==============================] - 10s 84us/sample - loss: 0.6055 - accuracy: 0.6668 - val_loss: 0.5322 - val_accuracy: 0.7356\n",
            "Epoch 2/5\n",
            "122137/122137 [==============================] - 10s 79us/sample - loss: 0.5276 - accuracy: 0.7415 - val_loss: 0.5027 - val_accuracy: 0.7575\n",
            "Epoch 3/5\n",
            "122137/122137 [==============================] - 10s 78us/sample - loss: 0.5025 - accuracy: 0.7595 - val_loss: 0.4855 - val_accuracy: 0.7689\n",
            "Epoch 4/5\n",
            "122137/122137 [==============================] - 9s 78us/sample - loss: 0.4873 - accuracy: 0.7691 - val_loss: 0.4758 - val_accuracy: 0.7732\n",
            "Epoch 5/5\n",
            "122137/122137 [==============================] - 10s 80us/sample - loss: 0.4753 - accuracy: 0.7765 - val_loss: 0.4774 - val_accuracy: 0.7745\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qSvq3TaMZvWK",
        "colab_type": "text"
      },
      "source": [
        "## Test model on new sentences"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GsK5CuQOZvWK",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def predict(text):\n",
        "  # pre-process text\n",
        "    tokens = regex_tokenizer.tokenize(text)\n",
        "    cleaned_text = [\" \".join(tokens)]\n",
        "    \n",
        "    input_ = tokenizer.texts_to_sequences(cleaned_text)\n",
        "    input_ = pad_sequences(input_, maxlen=max_seq_len)\n",
        "    \n",
        "    pred = model.predict(input_)[0]\n",
        "\n",
        "    return pred"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "U_cGelZzZvWN",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "text = \"Vive la vie lol\""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5yucZ9D12jzy",
        "colab_type": "code",
        "outputId": "47282e02-922a-4e88-bac5-2f53aec5def2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "predict(text)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([0.87678874], dtype=float32)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 158
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "t8UBIg2aZvWS",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "text = \"Je n'aime pas faire la vaisselle\""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ot5CmoT-ZvWU",
        "colab_type": "code",
        "outputId": "46b8fe20-92e2-41dd-d635-c4885f490a51",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "predict(text)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([0.26208484], dtype=float32)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 133
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Itvl6FmK07Pl",
        "colab_type": "text"
      },
      "source": [
        "## Save model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KbXC5L72ZvWY",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "model_save_path = \"model.h5\"\n",
        "model.save(model_save_path)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4Clezf0qZvWb",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "tokenizer_save_path = \"tokenizer.joblib\"\n",
        "joblib.dump(tokenizer, tokenizer_save_path)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OVx_095L1TTp",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GUqA4sjg1TX5",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}