ML_Data_Preparation.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ncw-Yjw_KPmt"
      },
      "source": [
        "## Data Processing for ML models\n",
        "\n",
        "Link to notebook: https://colab.research.google.com/drive/1sVTTF1lUFgPxVLaTiz3MdG-ITvrsJ6Ax?usp=sharing\n",
        "\n",
        "In this notebook, we process train_data.csv and test_data.csv before inputting it into our ML models. We also improve on the limitations of [this paper](https://ieeexplore.ieee.org/document/9084046). The authors of the paper did not share techniques to handle class imbalanced datasets. This could affect model accuracy if some target classes are underrepresented by the dataset. We did this using RandomOverSampler from the imblearn package.\n",
        "\n",
        "Paper reference:\n",
        "Rahman A, Hossen MdS (2019) Sentiment analysis on movie review data using Machine Learning Approach. 2019 International Conference on Bangla Speech and Language Processing (ICBSLP). doi: 10.1109/icbslp47725.2019.201470"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hcNUqhyKKXBT",
        "outputId": "a8def36d-2e8b-438e-c13a-b45adf7bc849"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Collecting contractions\n",
            "  Downloading contractions-0.1.73-py2.py3-none-any.whl (8.7 kB)\n",
            "Collecting textsearch>=0.0.21 (from contractions)\n",
            "  Downloading textsearch-0.0.24-py2.py3-none-any.whl (7.6 kB)\n",
            "Collecting anyascii (from textsearch>=0.0.21->contractions)\n",
            "  Downloading anyascii-0.3.2-py3-none-any.whl (289 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m289.9/289.9 kB\u001b[0m \u001b[31m5.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting pyahocorasick (from textsearch>=0.0.21->contractions)\n",
            "  Downloading pyahocorasick-2.1.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (110 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m110.7/110.7 kB\u001b[0m \u001b[31m9.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hInstalling collected packages: pyahocorasick, anyascii, textsearch, contractions\n",
            "Successfully installed anyascii-0.3.2 contractions-0.1.73 pyahocorasick-2.1.0 textsearch-0.0.24\n"
          ]
        }
      ],
      "source": [
        "!pip install contractions"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BkcGeAd3KPmu",
        "outputId": "e025f06b-40c6-4109-bbd6-66a492d3239c"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Unzipping tokenizers/punkt.zip.\n",
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/stopwords.zip.\n",
            "[nltk_data] Downloading package wordnet to /root/nltk_data...\n"
          ]
        }
      ],
      "source": [
        "import re\n",
        "\n",
        "import contractions\n",
        "import nltk\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "from imblearn.over_sampling import RandomOverSampler\n",
        "from collections import Counter\n",
        "from numpy import *\n",
        "import seaborn as sns\n",
        "from collections import Counter\n",
        "from nltk.corpus import stopwords\n",
        "from nltk.stem import WordNetLemmatizer\n",
        "from nltk.tokenize import TweetTokenizer\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\n",
        "from wordcloud import WordCloud, STOPWORDS\n",
        "\n",
        "\n",
        "# Download necessary NLTK data\n",
        "nltk.download('punkt')\n",
        "nltk.download('stopwords')\n",
        "nltk.download('wordnet')\n",
        "\n",
        "import warnings"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "E1QLnb8lKPmx"
      },
      "source": [
        "## Reading the data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IT-8U27OKPmx",
        "outputId": "44106402-46c3-4793-e012-38c767c59063"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ],
      "source": [
        "# This cell reads files from Google Colab. If not using Colab, change the file directories accordingly\n",
        "from google.colab import drive\n",
        "drive.mount('/content/drive')\n",
        "\n",
        "train_df = pd.read_csv('/content/drive/MyDrive/Datasets/train_data.csv')\n",
        "test_df = pd.read_csv('/content/drive/MyDrive/Datasets/test_data.csv')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pDRx8NJOKPmx"
      },
      "source": [
        "## Additional Processing"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pJjAyt6EKPmy"
      },
      "source": [
        "Remove punctuation marks"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "id": "5NwJAMkiKPmy"
      },
      "outputs": [],
      "source": [
        "def remove_punctuation(text):\n",
        "    punctuation_pattern = re.compile(r'[^\\w\\s]')\n",
        "    clean_text = punctuation_pattern.sub('', text)\n",
        "    return clean_text"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "id": "XxUiXkAKKPmy"
      },
      "outputs": [],
      "source": [
        "train_df['Text'] = train_df['Text'].apply(lambda z: remove_punctuation(z))\n",
        "test_df['Text'] = test_df['Text'].apply(lambda z: remove_punctuation(z))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uZImvuTYKPmy"
      },
      "source": [
        "Tokenization"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "id": "lL5rXDZHKPmy"
      },
      "outputs": [],
      "source": [
        "tokenizer = TweetTokenizer()\n",
        "train_df['Text'] = train_df['Text'].apply(tokenizer.tokenize)\n",
        "test_df['Text'] = test_df['Text'].apply(tokenizer.tokenize)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_oNBT2w1KPmz"
      },
      "source": [
        "Stopword Removal"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Q-XHkAU-KPmz",
        "outputId": "ebf7a939-25c1-4aef-de79-4346828903f3"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{\"she's\", 'here', 'o', 'against', 'too', \"that'll\", 'no', 'most', 'than', 'will', 'not', 'own', 'on', 'from', 'weren', 'i', 'into', 'off', 'being', 'how', 'themselves', 'by', 'needn', 'under', \"you've\", 'my', 'does', 'after', \"doesn't\", \"couldn't\", 'itself', 're', 'other', 'a', \"weren't\", 'wasn', 'myself', 'yourselves', 'at', 'out', 'few', 'until', 'any', 'did', 'while', \"isn't\", 'aren', 'it', 'now', 'during', 'hers', 'was', 'more', 'further', 'should', 'she', \"needn't\", 'and', 'doing', 'who', 'shan', \"wasn't\", \"you'll\", 'each', 'such', 'between', 'why', \"should've\", 'above', 'our', 'of', 'both', 'has', \"won't\", 'he', \"aren't\", 'as', 'doesn', 'them', 'once', 'nor', \"mightn't\", 'don', 'its', \"shan't\", 'won', 'but', 'in', 'some', 'couldn', 'which', 'down', 'your', 'herself', 'having', 'about', 've', 'whom', 'didn', 'very', 'mustn', 'himself', \"it's\", 'ours', 'where', \"hadn't\", 'ourselves', 'that', 'be', 'haven', 'his', 'ma', 'me', 'him', 'hadn', 'is', 'so', 'am', 'again', 'just', 't', 'you', 'have', 'their', 'wouldn', 'yours', 's', \"hasn't\", 'if', 'because', 'are', 'her', 'before', 'this', 'when', 'm', \"haven't\", 'd', 'below', 'to', \"you're\", 'an', 'do', \"don't\", 'with', 'through', 'can', 'what', 'yourself', \"didn't\", 'over', 'mightn', 'or', 'hasn', 'were', 'shouldn', \"shouldn't\", 'we', 'been', 'had', 'for', 'up', 'they', 'those', 'ain', 'the', 'only', 'all', 'y', 'there', 'then', \"wouldn't\", 'll', \"mustn't\", 'isn', \"you'd\", 'these', 'same', 'theirs'}\n"
          ]
        }
      ],
      "source": [
        "# Just to check list of stopwords to ensure that the keep tokens isnt redunat\n",
        "stop_words = set(stopwords.words('english'))\n",
        "print(stop_words)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "id": "AEC4dMX7KPmz"
      },
      "outputs": [],
      "source": [
        "def filter_tokens(tokens):\n",
        "    stop_words = set(stopwords.words('english'))\n",
        "\n",
        "    #Our n-grams analysis has shown that the <br> tag was not removed properly\n",
        "    stop_words.add('br')\n",
        "\n",
        "    #Keep these words in the text as they could correlate to sentiment\n",
        "    keep_in_tokens = [\n",
        "        \"isn't\", \"is\",\n",
        "        \"wasn't\", \"was\",\n",
        "        \"aren't\", \"are\",\n",
        "        \"doesn't\", \"does\",\n",
        "        \"couldn't\", \"could\",\n",
        "        \"won't\", \"will\",\n",
        "        \"shouldn't\", \"should\",\n",
        "        \"didn't\", \"did\",\n",
        "        \"haven't\", \"have\"\n",
        "    ]\n",
        "    for word in keep_in_tokens:\n",
        "        stop_words.discard(word)\n",
        "    filtered_tokens = [token for token in tokens if token not in stop_words]\n",
        "    return filtered_tokens"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "id": "XV9KR7u4KPmz"
      },
      "outputs": [],
      "source": [
        "train_df['Text'] = train_df['Text'].apply(filter_tokens)\n",
        "test_df['Text'] = test_df['Text'].apply(filter_tokens)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mL0JOqPyKPmz"
      },
      "source": [
        "Lemmatization"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "id": "Tr4xt53uKPmz"
      },
      "outputs": [],
      "source": [
        "lemmatizer = WordNetLemmatizer()\n",
        "def lemmatize_tokens(filtered_tokens):\n",
        "  return ' '.join([lemmatizer.lemmatize(token) for token in filtered_tokens])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "id": "8NImZRN7KPmz"
      },
      "outputs": [],
      "source": [
        "train_df['Text'] = train_df['Text'].apply(lemmatize_tokens)\n",
        "test_df['Text'] = test_df['Text'].apply(lemmatize_tokens)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 380
        },
        "id": "c5vflNWtKPm0",
        "outputId": "3937965d-c935-49a4-8aac-019c9fb9a96a"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Deep cleaned train Dataset\n"
          ]
        },
        {
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "summary": "{\n  \"name\": \"train_df\",\n  \"rows\": 39658,\n  \"fields\": [\n    {\n      \"column\": \"Text\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 39653,\n        \"samples\": [\n          \"show accurately described tv bubblegum is chewing good taste last long time like bubblegum leave bad taste mouth many chewsbr show feature simple question take form simply game like hangman guessing simple phrase guessing item short list caller are contestant anybody playbr question though are terribly easy is techno music pumped background is fashioned continuous loop never seems get old never add much show anyway hostess however regardless one is extremely personable pleasant enough watch make amusing facial expression rather enjoy british womanbr is nothing show is simplistic uncompelling although entertaining absolutely nothing else is\",\n          \"unless is huge ironic conspiracy going jaw dropped read positive review film cannot believe film wa even released is badbr admit is kind movie tried watch objectively anyway know bad is funny wa still offended sheer awfulnessbr acting is atrocious cannot have watched rush guessing wa one take per scene really is terrible is worst film have seen many year fact would even call film is tragedy gay black friend noone actually call gay is implied is crazy homophobic is good fact is downright vomit inducing joke die pathetic arses music is bad defies belief person compiled soundtrack essentially chose earmutilatingly bad song ever heard put waste film stock oh good christ cannot believe 80 produced utter garbage like grew cannot find one thing worth note must have dark time cinemagoerbr even contemplate watching film go see psychiatrist will accordingly slap sick sick person\",\n          \"is movie basic element genre make want cry make laugh disgust make angry etc topic story is fortunately disease drug is common trend gay themed movie day focus social interaction character could considered high school elite play direction could little bit sophisticated hand is somehow better really show distress character are experiencing wa intended is remarkable job assuredly achievement specially young director is actually good story give little inside is fat girl acknowledge\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"Sentiment\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 2,\n        \"min\": 1,\n        \"max\": 8,\n        \"num_unique_values\": 8,\n        \"samples\": [\n          6,\n          4,\n          8\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}",
              "type": "dataframe",
              "variable_name": "train_df"
            },
            "text/html": [
              "\n",
              "  <div id=\"df-7ccbad64-8a19-4154-a549-c49d0a29dda6\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Text</th>\n",
              "      <th>Sentiment</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>saw premiered rewatched ifc is great telling m...</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>movie is one alltime favorite think sean penn ...</td>\n",
              "      <td>6</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>describing stalingrad war film may bit inaccur...</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>tale two sister one creepiest film have seen r...</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>well notice imdb offered plot infothat is is p...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>little picture succeeds many big picture fails...</td>\n",
              "      <td>7</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>will love child saddest movie have ever seen d...</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>wow 3d imagery time wa used nicely provide goo...</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>24 is best television show is incredible tv se...</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>wa moved film 1981 went back theater four time...</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-7ccbad64-8a19-4154-a549-c49d0a29dda6')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-7ccbad64-8a19-4154-a549-c49d0a29dda6 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-7ccbad64-8a19-4154-a549-c49d0a29dda6');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-50442ade-dcfe-4466-b167-32bb456cc7f2\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-50442ade-dcfe-4466-b167-32bb456cc7f2')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-50442ade-dcfe-4466-b167-32bb456cc7f2 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "text/plain": [
              "                                                Text  Sentiment\n",
              "0  saw premiered rewatched ifc is great telling m...          8\n",
              "1  movie is one alltime favorite think sean penn ...          6\n",
              "2  describing stalingrad war film may bit inaccur...          8\n",
              "3  tale two sister one creepiest film have seen r...          8\n",
              "4  well notice imdb offered plot infothat is is p...          1\n",
              "5  little picture succeeds many big picture fails...          7\n",
              "6  will love child saddest movie have ever seen d...          8\n",
              "7  wow 3d imagery time wa used nicely provide goo...          2\n",
              "8  24 is best television show is incredible tv se...          8\n",
              "9  wa moved film 1981 went back theater four time...          8"
            ]
          },
          "execution_count": 12,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "print(\"Deep cleaned train Dataset\")\n",
        "train_df.head(10)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 380
        },
        "id": "X_dTTQ8PKPm0",
        "outputId": "7521d5f2-3a60-4556-f3d2-9a2caef1f220"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Deep cleaned test Dataset\n"
          ]
        },
        {
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "summary": "{\n  \"name\": \"test_df\",\n  \"rows\": 9915,\n  \"fields\": [\n    {\n      \"column\": \"Text\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 9915,\n        \"samples\": [\n          \"movie is one awful have ever seen is dialogue awful never end will think is ending is long is 140 160 minute even know know will never watch is like someone took romantic comedy took comedy decided downplay romance leaving u pile crap managed make way screen take word find terrible film is\",\n          \"have start saying long time since have seen have seen 5 time wonderful little romp wa clearly inspired musicalcomedy pairing new fading star musical group prominence kay kysers mystery would good examplebr spike jones unleashed is best part show band play many tune are part action fine job support hugh obrien play face buddy hackett part rumor wa offered lou costello thus abbott costello replacing lead know wa truebr pleasant movie important have much spike jones band film history wish wa released have seen heard two decade hope is lost\",\n          \"excellent film cried cried loved loved wa frustrated film touched heart wa reality check since is reality 19 year old soldier\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"Sentiment\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 2,\n        \"min\": 1,\n        \"max\": 8,\n        \"num_unique_values\": 8,\n        \"samples\": [\n          5,\n          8,\n          6\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}",
              "type": "dataframe",
              "variable_name": "test_df"
            },
            "text/html": [
              "\n",
              "  <div id=\"df-79084b86-acac-47cb-923f-e18775ca6a9d\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Text</th>\n",
              "      <th>Sentiment</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>frank horrigan clint eastwood is harassed mitc...</td>\n",
              "      <td>6</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>carly jones elisha curtberth bad boy brother n...</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>dig would say anyone even like metallica see k...</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>is great premise movie overall plot is origina...</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>underground comedy movie is possibly worst tra...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>plot nutshell duchess voice eva gabor is well ...</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>liked film lot is dark is bulletdodging carcha...</td>\n",
              "      <td>7</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>terrible movie represents perfectly state dege...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>know story group plucky nohopers enter competi...</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>boat builder sleepy town maine is going busine...</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-79084b86-acac-47cb-923f-e18775ca6a9d')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-79084b86-acac-47cb-923f-e18775ca6a9d button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-79084b86-acac-47cb-923f-e18775ca6a9d');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-c3157222-7029-494a-a6c2-25ba3c046887\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-c3157222-7029-494a-a6c2-25ba3c046887')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-c3157222-7029-494a-a6c2-25ba3c046887 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "text/plain": [
              "                                                Text  Sentiment\n",
              "0  frank horrigan clint eastwood is harassed mitc...          6\n",
              "1  carly jones elisha curtberth bad boy brother n...          5\n",
              "2  dig would say anyone even like metallica see k...          5\n",
              "3  is great premise movie overall plot is origina...          4\n",
              "4  underground comedy movie is possibly worst tra...          1\n",
              "5  plot nutshell duchess voice eva gabor is well ...          5\n",
              "6  liked film lot is dark is bulletdodging carcha...          7\n",
              "7  terrible movie represents perfectly state dege...          1\n",
              "8  know story group plucky nohopers enter competi...          5\n",
              "9  boat builder sleepy town maine is going busine...          5"
            ]
          },
          "execution_count": 13,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "print(\"Deep cleaned test Dataset\")\n",
        "test_df.head(10)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "y_6doakxKPm0"
      },
      "source": [
        "## Handling class imbalance"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xrjd2lQEKPm0"
      },
      "source": [
        "In our data preparation notebook, we found that there was an imbalance of classes. We will now perform random oversampling on the train data."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GF5IQLbPKPm1",
        "outputId": "2c10c641-9f76-429d-9291-bb7ec2a57050"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "Sentiment\n",
              "1    7981\n",
              "8    7733\n",
              "6    4672\n",
              "4    4225\n",
              "3    3922\n",
              "5    3829\n",
              "7    3666\n",
              "2    3630\n",
              "Name: count, dtype: int64"
            ]
          },
          "execution_count": 14,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "train_df['Sentiment'].value_counts()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "id": "x304b4nTKPm1"
      },
      "outputs": [],
      "source": [
        "import numpy as np"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "metadata": {
        "id": "39xFj5EVKPm1"
      },
      "outputs": [],
      "source": [
        "ros = RandomOverSampler()\n",
        "train_x, train_y = ros.fit_resample(np.array(train_df['Text']).reshape(-1, 1), np.array(train_df['Sentiment']).reshape(-1, 1))\n",
        "\n",
        "# Get Oversampled training data\n",
        "train_oversampled_df = pd.DataFrame(list(zip([x[0] for x in train_x], train_y)), columns = ['Text', 'Sentiment'])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 17,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "V0l9uy9nKPm1",
        "outputId": "fbf8f51d-3e1a-4571-89e7-2151c3eb4457"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "Sentiment\n",
              "8    7981\n",
              "6    7981\n",
              "1    7981\n",
              "7    7981\n",
              "2    7981\n",
              "4    7981\n",
              "3    7981\n",
              "5    7981\n",
              "Name: count, dtype: int64"
            ]
          },
          "execution_count": 17,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "train_oversampled_df['Sentiment'].value_counts()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xgRRmDGiKPm1"
      },
      "source": [
        "## Export the train and test data as csv"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 18,
      "metadata": {
        "id": "t4tAjbSJKPm1"
      },
      "outputs": [],
      "source": [
        "train_oversampled_df.to_csv(\"ML_train.csv\",index=False)\n",
        "test_df.to_csv(\"ML_test.csv\",index=False)"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.10.13"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}