Sentiment Analysis Classification.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "ygpzJofN55EM"
   },
   "source": [
    "# Airline Tweet Classification and Sentiment Analysis Using Deep Learning (CNN, CNN-LSTM, Word/Character-based Embeddings)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "ioAldGlt55EN"
   },
   "source": [
    "This data originally came from Crowdflower's Data for Everyone library.https://www.figure-eight.com/data-for-everyone/ <BR>\n",
    "\n",
    "As the original source says,<BR>\n",
    "\n",
    "A sentiment analysis job about the problems of each major U.S. airline. Twitter data was scraped from February of 2015 and contributors were asked to first classify **positive**, **negative**, and **neutral** tweets, followed by categorizing negative reasons (such as \"late flight\" or \"rude service\").<BR>\n",
    "\n",
    "In this Notebook, we will go step by step from multi-class Classification perspective of these text sequences (first part). We will perform \n",
    "-  Exploratory Data Analysis\n",
    "-  Data Preprocessing\n",
    "-  Basic CNN Frameworks with word embeddings, character embeddings and ensembles with LSTM\n",
    "-  Hidden layers visualization for CNN and plotting using PCA and TSNE\n",
    "-  False Positive and False Negative Analysis for CNN \n",
    "-  Model Explanation(s) using SHAP for Deep Learning Models\n",
    "-  Model Explanation (s) using LIME\n",
    "\n",
    "Note: We evaluate and understand most CNN based architectures that have been published till 2018.\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#import packages and verify versions\n",
    "\n",
    "!pip install wordcloud\n",
    "# numpy\n",
    "import numpy\n",
    "print('numpy: %s' % numpy.__version__)\n",
    "# scipy\n",
    "import scipy\n",
    "print('scipy: %s' % scipy.__version__)\n",
    "# matplotlib\n",
    "import matplotlib\n",
    "print('matplotlib: %s' % matplotlib.__version__)\n",
    "# pandas\n",
    "import pandas\n",
    "print('pandas: %s' % pandas.__version__)\n",
    "# nltk\n",
    "import nltk\n",
    "print('nltk: %s' % nltk.__version__)\n",
    "# scikit-learn\n",
    "import sklearn\n",
    "print('sklearn: %s' % sklearn.__version__)\n",
    "# wordcloud\n",
    "import wordcloud\n",
    "print('wordcloud: %s' % wordcloud.__version__)\n",
    "# keras\n",
    "import keras\n",
    "print('keras: %s' % keras.__version__)\n",
    "# tensorflow\n",
    "import tensorflow\n",
    "print('tensorflow: %s' % tensorflow.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "w7q5g3xj55EO",
    "outputId": "e2feff7c-c8ff-4f70-bd39-0395780afce5"
   },
   "outputs": [],
   "source": [
    "# Basic packages\n",
    "import pandas as pd \n",
    "import numpy as np\n",
    "import re\n",
    "import collections\n",
    "import time\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "\n",
    "\n",
    "# Packages for data preparation\n",
    "from sklearn.model_selection import train_test_split,StratifiedKFold, cross_val_score\n",
    "from nltk.corpus import stopwords\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.utils.np_utils import to_categorical\n",
    "from keras.utils.vis_utils import plot_model\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.metrics import average_precision_score, precision_recall_curve, confusion_matrix\n",
    "\n",
    "# Packages for modeling\n",
    "from keras import models\n",
    "from keras import layers\n",
    "from keras import regularizers\n",
    "\n",
    "# Word cloud visualization libraries\n",
    "from scipy.misc import imresize\n",
    "from PIL import Image\n",
    "from wordcloud import WordCloud, ImageColorGenerator, STOPWORDS\n",
    "from collections import Counter\n",
    "\n",
    "import itertools\n",
    "\n",
    "from keras.wrappers.scikit_learn import KerasClassifier\n",
    "\n",
    "# Packages for Sequence and CNN based layers\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.models import Sequential, Model, load_model\n",
    "from keras.layers.embeddings import Embedding\n",
    "from keras.layers import Flatten, Dense, Dropout, AlphaDropout, ThresholdedReLU, Convolution1D, ZeroPadding1D, Activation, MaxPooling1D, SpatialDropout1D, Input \n",
    "from keras.layers import GlobalMaxPooling1D, concatenate, LSTM, Bidirectional,BatchNormalization\n",
    "from keras.optimizers import Adam\n",
    "from keras.utils import np_utils\n",
    "from keras.callbacks import ModelCheckpoint, EarlyStopping,TensorBoard"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "X7Pl-nrZ55ET"
   },
   "source": [
    "# Global Parameters and Constants"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "aFtCNcBw55EU"
   },
   "outputs": [],
   "source": [
    "# location of data\n",
    "DATA_FILE = './data/Tweets.csv'\n",
    "\n",
    "# data columns for learning\n",
    "TEXT_COLUMN_NAME = 'text'\n",
    "LABEL_COLUMN_NAME = 'airline_sentiment'\n",
    "\n",
    "# EMBEDDINGS FILE \n",
    "GLOVE_EMBEDDINGS_FILE = 'embeddings/glove.6B.100d.txt'\n",
    "FASTTEXT_EMBEDDINGS_FILE = 'embeddings/wiki-news-300d-1M.vec'\n",
    "\n",
    "# parameters for learning\n",
    "# Parameter indicating the number of words we'll put in the dictionary\n",
    "# (change it per source)\n",
    "NB_WORDS = 10000\n",
    "VAL_SIZE = 1000  # Size of the validation set\n",
    "NB_EPOCHS = 20  # Number of epochs\n",
    "BATCH_SIZE = 512  # Size of the batches used in the mini-batch gradient descent\n",
    "# Maximum number of words in a sequence (see the sentence distribution)\n",
    "MAX_LEN = 24\n",
    "EMBEDDING_DIM = 100  # Number of dimensions of the GloVe word embeddings\n",
    "FASTTEXT_EMBEDDING_DIM = 300  # Number of dimensions of the FastText word embeddings\n",
    "MAX_SENT_LEN = 300  # character based length\n",
    "MAX_DOC_LEN = 5  # Number of sentences in text\n",
    "\n",
    "# labels\n",
    "labels = ['negative', 'neutral', 'positive']\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "_PSg5XvV55EW"
   },
   "source": [
    "# 1. Exploratory Data Analysis"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "AEj1mY9F55EX"
   },
   "source": [
    "## 1.1 Loading the data in Pandas DataFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "4f2RAqHv55EY",
    "outputId": "79b7942e-c9b0-4ed6-9b47-013622068fe6"
   },
   "outputs": [],
   "source": [
    "# read the file\n",
    "tweets = pd.read_csv(DATA_FILE)\n",
    "tweets = tweets.reindex(np.random.permutation(tweets.index))  \n",
    "# only the text and labels\n",
    "tweets = tweets[[TEXT_COLUMN_NAME, LABEL_COLUMN_NAME]]\n",
    "# get a peek of the data\n",
    "tweets.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "76fsGWsp55Eb"
   },
   "source": [
    "## 1.2 Target distribution analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "MXEI3MFI55Ed",
    "outputId": "27fedd7c-73d2-42cb-e75b-311d833a2cae"
   },
   "outputs": [],
   "source": [
    "text = tweets[TEXT_COLUMN_NAME]\n",
    "sentiments = tweets[LABEL_COLUMN_NAME]\n",
    "\n",
    "# unique labels\n",
    "sentiments.unique()\n",
    "# group by\n",
    "grouped = tweets.groupby([LABEL_COLUMN_NAME])[TEXT_COLUMN_NAME].count()\n",
    "plt.bar(\n",
    "    labels,\n",
    "    grouped.values,\n",
    "    align='center',\n",
    "    color=[\n",
    "        'tomato',\n",
    "        'darkolivegreen',\n",
    "        'lightsteelblue'],\n",
    "    alpha=0.5)\n",
    "plt.ylabel('Number of Sentences in Class')\n",
    "plt.xlabel('Class')\n",
    "plt.title('Class Distribution')\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "gGxiOVnh55Ef"
   },
   "source": [
    "## 1.3 Text Length Frequency Per Class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "djRTlaK355Ef",
    "outputId": "c1467ea9-46e4-4831-d73c-32a69559888e"
   },
   "outputs": [],
   "source": [
    "# Set of unique label\n",
    "lbl_set = list(set(sentiments))\n",
    "\n",
    "\n",
    "def length_counter(all_label, all_text, class_name):\n",
    "\n",
    "    class_txt = []\n",
    "\n",
    "    # Seperate all lines for a class\n",
    "    for lbl, txt in zip(all_label, all_text):\n",
    "        if lbl == class_name:\n",
    "            class_txt.append(txt)\n",
    "\n",
    "     # Count words per line of that class\n",
    "    line_word_count = []\n",
    "    for k in range(len(class_txt)):\n",
    "        line_words = class_txt[k].lower().split()\n",
    "        line_word_count.append(len(line_words))\n",
    "\n",
    "    return np.sort(line_word_count)\n",
    "\n",
    "\n",
    "plt.figure(figsize=[8, 6])\n",
    "for lbl in lbl_set:\n",
    "    count = length_counter(sentiments, text, lbl)\n",
    "    plt.plot(np.arange(1, len(count) + 1, 1), count, label=lbl)\n",
    "plt.legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.)\n",
    "plt.title(\"Plot of text length frequency per class\")\n",
    "plt.xlabel(\"Number of Samples (Sorted)\")\n",
    "plt.ylabel(\"Frequency/Counts\")\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "UrCZqY7n55Ei"
   },
   "source": [
    "## 1.4 Number of Words distribution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "9d2NcgfG55Ej",
    "outputId": "70ab3bd6-339c-4c84-fbe1-f98bf0b8a8d2"
   },
   "outputs": [],
   "source": [
    "# ADD WORDS DERIVED COLUMN\n",
    "NUMBER_OF_WORDS_COLUMN = TEXT_COLUMN_NAME + 'NB_WORDS_COLUM'\n",
    "tweets[NUMBER_OF_WORDS_COLUMN] = tweets[TEXT_COLUMN_NAME].str.split().apply(len)\n",
    "data = tweets[NUMBER_OF_WORDS_COLUMN].describe()\n",
    "\n",
    "quartiles = ['Q1', 'Q2', 'Q3']\n",
    "top = [data['25%'], data['50%'], data['75%']]\n",
    "plt.bar(\n",
    "    quartiles,\n",
    "    top,\n",
    "    align='center',\n",
    "    color=[\n",
    "        'red',\n",
    "        'blue',\n",
    "        'green'],\n",
    "    alpha=0.5)\n",
    "plt.ylabel('Number of Words in Text')\n",
    "plt.xlabel('Quartiles')\n",
    "plt.title('Number of Words Distribution')\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "FQrzq-bm55Em"
   },
   "source": [
    "## 1.5 Word Cloud for Positive and Negative Sentiments\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Mr7QxmiK55En",
    "outputId": "76920f70-3a3b-4b8c-d3fe-66f058ea8e7d"
   },
   "outputs": [],
   "source": [
    "negative_df = tweets[tweets[LABEL_COLUMN_NAME] == 'negative']\n",
    "words = ' '.join(negative_df[TEXT_COLUMN_NAME])\n",
    "negative_cleaned_words = \" \".join([word for word in words.split()\n",
    "                                   if 'http' not in word\n",
    "                                   and not word.startswith('@')\n",
    "                                   and word != 'RT'\n",
    "                                   ])\n",
    "wordcloud = WordCloud(stopwords=STOPWORDS,\n",
    "                      background_color='black',\n",
    "                      width=3000,\n",
    "                      height=2500\n",
    "                      ).generate(negative_cleaned_words)\n",
    "plt.figure(1, figsize=(12, 12))\n",
    "plt.imshow(wordcloud)\n",
    "plt.axis('off')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Oka9Y5Dt55Ep",
    "outputId": "e7f0921e-4cd9-4f40-9179-c9635888cdab"
   },
   "outputs": [],
   "source": [
    "positive_df = tweets[tweets[LABEL_COLUMN_NAME] == 'positive']\n",
    "all_positive_words = ' '.join(positive_df[TEXT_COLUMN_NAME])\n",
    "positive_cleaned_words = \" \".join([word for word in all_positive_words.split()\n",
    "                                   if 'http' not in word\n",
    "                                   and not word.startswith('@')\n",
    "                                   and word != 'RT'\n",
    "                                   ])\n",
    "wordcloud = WordCloud(stopwords=STOPWORDS,\n",
    "                      background_color='black',\n",
    "                      width=3000,\n",
    "                      height=2500\n",
    "                      ).generate(positive_cleaned_words)\n",
    "plt.figure(1, figsize=(12, 12))\n",
    "plt.imshow(wordcloud)\n",
    "plt.axis('off')\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "ghJjkGWn55Es"
   },
   "source": [
    "# 2. DATA Preproessing "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "1_-MI76F55Et"
   },
   "source": [
    "## 2.1 Stop words and mentions removal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "WqnUfMem55Eu",
    "outputId": "b2357312-8a9a-4e30-f223-f6f0e8be5d24"
   },
   "outputs": [],
   "source": [
    "# remove stop words with exceptions\n",
    "def remove_stopwords(input_text):\n",
    "    stopwords_list = stopwords.words('english')\n",
    "    # Some words which might indicate a certain sentiment are kept\n",
    "    white_list = [\"n't\", \"not\", \"no\"]\n",
    "    words = input_text.split()\n",
    "    clean_words = [word for word in words if (\n",
    "        word not in stopwords_list or word in white_list) and len(word) > 1]\n",
    "    return \" \".join(clean_words)\n",
    "# remove mentions\n",
    "\n",
    "\n",
    "def remove_mentions(input_text):\n",
    "    return re.sub(r'@\\w+', '', input_text)\n",
    "\n",
    "\n",
    "tweets = tweets[[TEXT_COLUMN_NAME, LABEL_COLUMN_NAME]]\n",
    "tweets[TEXT_COLUMN_NAME] = tweets[TEXT_COLUMN_NAME].apply(\n",
    "    remove_stopwords).apply(remove_mentions)\n",
    "tweets.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "yI_bch8H55Ew"
   },
   "source": [
    "## 2.2 Split dataset in Training and Testing (unseen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "KK6skul455Ey",
    "outputId": "94509f00-2f4c-4bac-956e-cfe2384a58fa"
   },
   "outputs": [],
   "source": [
    "X_train_entire, X_test, y_train_entire, y_test = train_test_split(\n",
    "    tweets[TEXT_COLUMN_NAME], tweets[LABEL_COLUMN_NAME], test_size=0.15, random_state=37)\n",
    "print('# Train data samples:', X_train_entire.shape[0])\n",
    "print('# Test data samples:', X_test.shape[0])\n",
    "assert X_train_entire.shape[0] == y_train_entire.shape[0]\n",
    "assert X_test.shape[0] == y_test.shape[0]\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "A9zmSmzw55E0"
   },
   "source": [
    "## 2.3 Tokenize the dataset on training data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "LG8Htrxy55E2",
    "outputId": "c1074ae8-09cf-4ba6-8cf7-9a26624adfdc"
   },
   "outputs": [],
   "source": [
    "# tokenization with max words defined and filters\n",
    "tk = Tokenizer(num_words=NB_WORDS,\n",
    "               filters='!\"#$%&()*+,-./:;<=>?@[\\\\]^_`{|}~\\t\\n',\n",
    "               lower=True,\n",
    "               split=\" \")\n",
    "tk.fit_on_texts(X_train_entire)\n",
    "\n",
    "print('Fitted tokenizer on {} documents'.format(tk.document_count))\n",
    "print('{} words in dictionary'.format(tk.num_words))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "ItIT7vwb55E6"
   },
   "source": [
    "## 2.4 Sentence Distribution in training data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "kaYeGIG255E7",
    "outputId": "4ca06325-3fb2-4935-eef2-7c7caf801ef4"
   },
   "outputs": [],
   "source": [
    "# understand the sequence distribution for max length \n",
    "seq_lengths = X_train_entire.apply(lambda x: len(x.split(' ')))\n",
    "seq_lengths.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "UEckczj955E-"
   },
   "source": [
    "## 2.5 Convert Train and Test to fixed length sequences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "vqAQIEhH55E-"
   },
   "outputs": [],
   "source": [
    "# convert train and test to sequence using the tokenizer\n",
    "X_train_entire_seq_tok = tk.texts_to_sequences(X_train_entire)\n",
    "X_test_seq_tok = tk.texts_to_sequences(X_test)\n",
    "\n",
    "# pad the sequences \n",
    "X_train_entire_seq = pad_sequences(X_train_entire_seq_tok, maxlen=MAX_LEN)\n",
    "X_test_seq = pad_sequences(X_test_seq_tok, maxlen=MAX_LEN)\n",
    "\n",
    "# perform encoding of \n",
    "le = LabelEncoder()\n",
    "y_train_le = le.fit_transform(y_train_entire)\n",
    "y_test_le = le.transform(y_test)\n",
    "y_train_one_hot = to_categorical(y_train_le)\n",
    "y_test_one_hot = to_categorical(y_test_le)\n",
    "\n",
    "\n",
    "# labels from encoder mapping \n",
    "TARGET_TEXT_LABELS = le.classes_\n",
    "TEXT_LABELS = le.classes_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "OMUVMfo555FA"
   },
   "source": [
    "## 2.6 Validation Dataset from Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "RzkB1Awk55FB",
    "outputId": "e49df399-8558-4193-a9ad-5fe5eaa873f2"
   },
   "outputs": [],
   "source": [
    "X_train_seq, X_valid_seq, y_train, y_valid = train_test_split(\n",
    "    X_train_entire_seq, y_train_one_hot, test_size=0.15, random_state=37)\n",
    "\n",
    "assert X_valid_seq.shape[0] == y_valid.shape[0]\n",
    "assert X_train_seq.shape[0] == y_train.shape[0]\n",
    "\n",
    "print('Shape of training set:', X_train_seq.shape)\n",
    "print('Shape of validation set:', X_valid_seq.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "padfkojX55FE"
   },
   "source": [
    "# 3. Model Building"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "tHeR6aQT55FF"
   },
   "source": [
    "## 3.1 Utilities for training, logging and testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "EfrxWrTQ55FF"
   },
   "outputs": [],
   "source": [
    "def get_predictions_and_confidences(model, X_test, y_test):\n",
    "    \"\"\"\n",
    "    return the predictions and confidences on the test set\n",
    "    :param model: the Keras Classifier Model\n",
    "    :param X_test: test sequences\n",
    "    :param y_test: labels\n",
    "    :return: a list of actual labels\n",
    "    \"\"\"\n",
    "    y_softmax = model.predict(X_test)\n",
    "    y_class_index = []\n",
    "    y_pred_index = []\n",
    "    confidence = []\n",
    "\n",
    "    for i in range(len(y_test)):\n",
    "        probs = y_test[i]\n",
    "        index_arr = np.nonzero(probs)\n",
    "        one_hot_index = index_arr[0].item(0)\n",
    "        y_class_index.append(one_hot_index)\n",
    "\n",
    "    for i in range(0, len(y_softmax)):\n",
    "        probs = y_softmax[i]\n",
    "        predicted_index = np.argmax(probs)\n",
    "        y_pred_index.append(predicted_index)\n",
    "        confidence.append(probs[predicted_index])\n",
    "    return y_class_index, y_pred_index, confidence, y_softmax\n",
    "\n",
    "\n",
    "def train_model(model, X_train, y_train, X_valid, y_valid):\n",
    "    \"\"\"\n",
    "    This function trains the deep model using the training set and valdiation set.\n",
    "\n",
    "    :param model: the Keras Classifier Model\n",
    "    :param X_train: train sequences\n",
    "    :param y_train: trian labels\n",
    "    :return: a list of actual labels\n",
    "\n",
    "    \"\"\"\n",
    "    start = time.time()\n",
    "    checkpoints = []\n",
    "    model.compile(\n",
    "        optimizer=Adam(),\n",
    "        loss='categorical_crossentropy',\n",
    "        metrics=['accuracy'])\n",
    "    checkpoint_file = './Checkpoints/Train-' + model.name + '-best_weights.h5'\n",
    "    checkpoints.append(\n",
    "        ModelCheckpoint(\n",
    "            checkpoint_file,\n",
    "            monitor='val_loss',\n",
    "            verbose=0,\n",
    "            save_best_only=True,\n",
    "            save_weights_only=True,\n",
    "            mode='auto',\n",
    "            period=1))\n",
    "    checkpoints.append(\n",
    "        TensorBoard(\n",
    "            log_dir='./logs',\n",
    "            histogram_freq=0,\n",
    "            write_graph=True,\n",
    "            write_images=False,\n",
    "            embeddings_freq=0,\n",
    "            embeddings_layer_names=None,\n",
    "            embeddings_metadata=None))\n",
    "    checkpoints.append(EarlyStopping(monitor='val_loss', patience=10))\n",
    "\n",
    "    history = model.fit(\n",
    "        X_train,\n",
    "        y_train,\n",
    "        epochs=NB_EPOCHS,\n",
    "        batch_size=BATCH_SIZE,\n",
    "        validation_data=(\n",
    "            X_valid,\n",
    "            y_valid),\n",
    "        verbose=1,\n",
    "        callbacks=checkpoints)\n",
    "    # load the weights from the checkpoint file, the best weights\n",
    "    model.load_weights(checkpoint_file)\n",
    "    end = time.time()\n",
    "    total_time = end - start\n",
    "    print(\"Model Training and Validation time %s secs\" % (total_time))\n",
    "    return history\n",
    "\n",
    "\n",
    "def eval_metric(history, metric_name):\n",
    "\n",
    "    metric = history.history[metric_name]\n",
    "    val_metric = history.history['val_' + metric_name]\n",
    "\n",
    "    e = range(1, NB_EPOCHS + 1)\n",
    "\n",
    "    plt.plot(e, metric, color='navy', label='Train ' + metric_name)\n",
    "    plt.plot(e, val_metric, color='red', label='Validation ' + metric_name)\n",
    "    plt.xlabel('Epochs')\n",
    "    plt.ylabel(metric_name)\n",
    "    plt.legend()\n",
    "    plt.show()\n",
    "\n",
    "# This utility function is from the sklearn docs:\n",
    "# http://scikit-learn.org/stable/auto_examples/model_selection/plot_confusion_matrix.html\n",
    "\n",
    "\n",
    "def plot_confusion_matrix(cm, classes,\n",
    "                          normalize=False,\n",
    "                          title='Confusion matrix',\n",
    "                          cmap=plt.cm.Blues):\n",
    "    \"\"\"\n",
    "    This function prints and plots the confusion matrix.\n",
    "    Normalization can be applied by setting `normalize=True`.\n",
    "    \"\"\"\n",
    "    if normalize:\n",
    "        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n",
    "        print(\"Normalized confusion matrix\")\n",
    "    else:\n",
    "        print('Confusion matrix, without normalization')\n",
    "\n",
    "    print(cm)\n",
    "\n",
    "    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n",
    "    plt.title(title)\n",
    "    plt.colorbar()\n",
    "    tick_marks = np.arange(len(classes))\n",
    "    plt.xticks(tick_marks, classes, rotation=45)\n",
    "    plt.yticks(tick_marks, classes)\n",
    "\n",
    "    fmt = '.2f' if normalize else 'd'\n",
    "    thresh = cm.max() / 2.\n",
    "    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n",
    "        plt.text(j, i, format(cm[i, j], fmt),\n",
    "                 horizontalalignment=\"center\",\n",
    "                 color=\"white\" if cm[i, j] > thresh else \"black\")\n",
    "\n",
    "    plt.tight_layout()\n",
    "    plt.ylabel('True label')\n",
    "    plt.xlabel('Predicted label')\n",
    "\n",
    "\n",
    "def test_model(model, X_test, y_test):\n",
    "    start = time.time()\n",
    "    # checkpoint\n",
    "    checkpoint_file = './Checkpoints/Train-' + model.name + '-best_weights.h5'\n",
    "    model.load_weights(checkpoint_file)\n",
    "    results = model.evaluate(X_test, y_test)\n",
    "    end = time.time()\n",
    "    y_class_index, y_pred_index, confidence, y_softmax = get_predictions_and_confidences(\n",
    "        model, X_test, y_test)\n",
    "    cnf_matrix = confusion_matrix(y_class_index, y_pred_index)\n",
    "    plt.figure(figsize=(6, 5))\n",
    "    plot_confusion_matrix(\n",
    "        cnf_matrix,\n",
    "        classes=TEXT_LABELS,\n",
    "        title=\"Confusion matrix\")\n",
    "    plt.show()\n",
    "    total_time = end - start\n",
    "    print(\"Model Training and Testing time : %s secs\" % (total_time))\n",
    "    print(\"Model Testing Accuracy : %s \" % (results[1]))\n",
    "    average_precision = average_precision_score(\n",
    "        y_test, y_softmax, average='weighted')\n",
    "    print('Average precision score: {0:0.2f}'.format(\n",
    "        average_precision))\n",
    "    return results\n",
    "\n",
    "\n",
    "def test_model_runs(\n",
    "        model,\n",
    "        X_train,\n",
    "        y_train,\n",
    "        X_test,\n",
    "        y_test,\n",
    "        epoch_stop=20,\n",
    "        num_runs=3,\n",
    "        verbose=0):\n",
    "\n",
    "    sum_result = 0\n",
    "    sq_sum_result = 0\n",
    "    for val in range(0, num_runs):\n",
    "        base_results = test_model(\n",
    "            model,\n",
    "            X_train,\n",
    "            y_train,\n",
    "            X_test,\n",
    "            y_test,\n",
    "            epoch_stop,\n",
    "            verbose)\n",
    "        sum_result = sum_result + base_results[1]\n",
    "        sq_sum_result = sq_sum_result + base_results[1] * base_results[1]\n",
    "    mean_accuracy = sum_result / num_runs\n",
    "    variance = sq_sum_result / num_runs - mean_accuracy * mean_accuracy\n",
    "    stddev = variance**(.5)\n",
    "    return mean_accuracy, stddev\n",
    "\n",
    "\n",
    "def compare_loss(history, base_history, model_name, base_history_name):\n",
    "    loss_base_model = base_history.history['val_loss']\n",
    "    loss_model = history.history['val_loss']\n",
    "\n",
    "    e = range(1, NB_EPOCHS + 1)\n",
    "\n",
    "    plt.plot(e, loss_base_model, 'bo', label=base_history_name)\n",
    "    plt.plot(e, loss_model, 'red', label=model_name)\n",
    "    plt.legend()\n",
    "    plt.xlabel('Epochs')\n",
    "    plt.ylabel('loss')\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "def read_embedding(path):\n",
    "\n",
    "    embeddings_index = {}\n",
    "    f = open(path)\n",
    "    for line in f:\n",
    "        values = line.split()\n",
    "        word = values[0]\n",
    "        coefs = np.asarray(values[1:], dtype='float32')\n",
    "        embeddings_index[word] = coefs\n",
    "    f.close()\n",
    "\n",
    "    return embeddings_index\n",
    "\n",
    "\n",
    "def create_embedding_matrix(tokenizer, embedding_dim, embeddings_index):\n",
    "\n",
    "    word_index = tokenizer.word_index\n",
    "    if tokenizer.num_words:  # if num words is set, get rid of words with too high index\n",
    "        word_index = {key: word_index[key] for key in word_index.keys()\n",
    "                      if word_index[key] < (tokenizer.num_words + 1)}\n",
    "    embedding_matrix = np.zeros((len(word_index) + 1, embedding_dim))\n",
    "    for word, i in word_index.items():\n",
    "        embedding_vector = embeddings_index.get(word)\n",
    "        if embedding_vector is not None:\n",
    "            # words not found in embedding index will be all-zeros.\n",
    "            embedding_matrix[i] = embedding_vector\n",
    "\n",
    "    return embedding_matrix\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "VCRazaT255FK"
   },
   "source": [
    "## 3.2 Experiments "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "rSFX91Te55FK"
   },
   "source": [
    "### 3.2.1 Experiment 1- Basic CNN Block Experiment "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "4k3RrOqP55FK",
    "outputId": "29d082de-2dd1-48f2-dbc6-f6a933bad1c7"
   },
   "outputs": [],
   "source": [
    "# base cnn model\n",
    "def base_cnn_model():\n",
    "    model = Sequential(\n",
    "        [\n",
    "            Embedding(\n",
    "                input_dim=NB_WORDS,\n",
    "                output_dim=EMBEDDING_DIM,\n",
    "                input_length=MAX_LEN),\n",
    "            Convolution1D(\n",
    "                filters=64,\n",
    "                kernel_size=3,\n",
    "                padding='same',\n",
    "                activation='relu'),\n",
    "            MaxPooling1D(),\n",
    "            Flatten(),\n",
    "            Dense(\n",
    "                100,\n",
    "                activation='relu'),\n",
    "            Dense(\n",
    "                3,\n",
    "                activation='softmax')])\n",
    "    model.name = 'BaseCNN'\n",
    "    model.summary()\n",
    "    return model\n",
    "\n",
    "\n",
    "# train and validate\n",
    "base_cnn_model = base_cnn_model()\n",
    "base_history = train_model(\n",
    "    base_cnn_model,\n",
    "    X_train_seq,\n",
    "    y_train,\n",
    "    X_valid_seq,\n",
    "    y_valid)\n",
    "# train the entire training data and test on test data on base model\n",
    "test_model(base_cnn_model, X_test_seq, y_test_one_hot)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "mNiu6_k855FQ"
   },
   "source": [
    "### 3.2.2 Experiment 2 - Basic CNN + Dropout/Regularization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "OijpdVP255FR",
    "outputId": "d38dc993-d35e-43dd-aacd-4c504de76cea"
   },
   "outputs": [],
   "source": [
    "# CNN with dropout of 0.2\n",
    "def base_cnn_dropout_model():\n",
    "    model = Sequential(\n",
    "        [\n",
    "            Embedding(\n",
    "                input_dim=NB_WORDS,\n",
    "                output_dim=EMBEDDING_DIM,\n",
    "                input_length=MAX_LEN),\n",
    "            Convolution1D(\n",
    "                filters=64,\n",
    "                kernel_size=3,\n",
    "                padding='same',\n",
    "                activation='relu'),\n",
    "            MaxPooling1D(),\n",
    "            Flatten(),\n",
    "            Dropout(0.2),\n",
    "            Dense(\n",
    "                100,\n",
    "                activation='relu'),\n",
    "            Dropout(0.2),\n",
    "            Dense(\n",
    "                3,\n",
    "                activation='softmax')])\n",
    "    model.name = 'BaseCNNWithDropouts'\n",
    "    model.summary()\n",
    "    return model\n",
    "\n",
    "# train and validate\n",
    "base_cnn_dropout_model = base_cnn_dropout_model()\n",
    "base_cnn_dropout_history = train_model(\n",
    "    base_cnn_dropout_model,\n",
    "    X_train_seq,\n",
    "    y_train,\n",
    "    X_valid_seq,\n",
    "    y_valid)\n",
    "# train the entire training data and test on test data on base model\n",
    "test_model(base_cnn_dropout_model, X_test_seq, y_test_one_hot)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "NIQETyXK55FV"
   },
   "source": [
    "### 3.2.3 Experiment 3 - Base CNN with Regularization (L2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "_fKGPSlk55FV",
    "outputId": "876ed95f-7725-4ba4-ff05-88a85129aea5"
   },
   "outputs": [],
   "source": [
    "# regularization with l2 regularizer\n",
    "def base_cnn_regularization_model():\n",
    "    model = Sequential(\n",
    "        [\n",
    "            Embedding(\n",
    "                input_dim=NB_WORDS,\n",
    "                output_dim=EMBEDDING_DIM,\n",
    "                input_length=MAX_LEN),\n",
    "            Convolution1D(\n",
    "                filters=64,\n",
    "                kernel_size=3,\n",
    "                padding='same',\n",
    "                activation='relu'),\n",
    "            MaxPooling1D(),\n",
    "            Flatten(),\n",
    "            Dense(\n",
    "                100,\n",
    "                kernel_regularizer=regularizers.l2(0.001),\n",
    "                activation='relu'),\n",
    "            Dense(\n",
    "                3,\n",
    "                activation='softmax')])\n",
    "    model.name = 'BaseCNNWithRegularization'\n",
    "    model.summary()\n",
    "    return model\n",
    "\n",
    "\n",
    "# train and validate\n",
    "base_cnn_regularization_model = base_cnn_regularization_model()\n",
    "base_cnn_regularization_history = train_model(\n",
    "    base_cnn_regularization_model,\n",
    "    X_train_seq,\n",
    "    y_train,\n",
    "    X_valid_seq,\n",
    "    y_valid)\n",
    "# test the models\n",
    "test_model(base_cnn_regularization_model, X_test_seq, y_test_one_hot)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "ITn24QMP55FY"
   },
   "source": [
    "### 3.2.4 Experiment 4 Multi-layer and Multi-filter CNN "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "PwQnJK5j55FZ",
    "outputId": "0ed3484f-9552-4b35-feb1-849aa53aec06"
   },
   "outputs": [],
   "source": [
    "def multilayer_multifilter_conv():\n",
    "    graph_in = Input(shape=(NB_WORDS, 100))\n",
    "\n",
    "    convs = []\n",
    "    for filter_size in range(2, 5):\n",
    "        x = Convolution1D(\n",
    "            64,\n",
    "            filter_size,\n",
    "            padding='same',\n",
    "            activation='relu')(graph_in)\n",
    "        convs.append(x)\n",
    "\n",
    "    graph_out = concatenate(convs, axis=1)\n",
    "    graph_out = GlobalMaxPooling1D()(graph_out)\n",
    "    graph = Model(graph_in, graph_out)\n",
    "\n",
    "    model = Sequential(\n",
    "        [\n",
    "            Embedding(\n",
    "                NB_WORDS, output_dim=EMBEDDING_DIM, input_length=MAX_LEN), graph, Dense(\n",
    "                128, activation='relu'), Dense(\n",
    "                    3, activation='softmax')])\n",
    "    model.name = 'MultiLayerMultiFilterCNN'\n",
    "    model.summary()\n",
    "    return model\n",
    "\n",
    "\n",
    "#train and validate\n",
    "multi_filter_conv_model = multilayer_multifilter_conv()\n",
    "multi_filter_conv_history = train_model(\n",
    "    multi_filter_conv_model,\n",
    "    X_train_seq,\n",
    "    y_train,\n",
    "    X_valid_seq,\n",
    "    y_valid)\n",
    "\n",
    "# test the models\n",
    "test_model(multi_filter_conv_model, X_test_seq, y_test_one_hot)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Y8-_rmp855Fb"
   },
   "source": [
    "### 3.2.5 Experiment 5 Multi-layer, Increased Number of Filters and Multi-filter CNN "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "LavfGMap55Fc",
    "outputId": "63cff387-f383-4bee-9081-16b35b38bf1a"
   },
   "outputs": [],
   "source": [
    "def multilayer_multifilter_increased_size_conv():\n",
    "    graph_in = Input(shape=(NB_WORDS, 100))\n",
    "\n",
    "    convs = []\n",
    "    for filter_size in range(2, 5):\n",
    "        x = Convolution1D(\n",
    "            128,\n",
    "            filter_size,\n",
    "            padding='same',\n",
    "            activation='relu')(graph_in)\n",
    "        convs.append(x)\n",
    "\n",
    "    graph_out = concatenate(convs, axis=1)\n",
    "    graph_out = GlobalMaxPooling1D()(graph_out)\n",
    "    graph = Model(graph_in, graph_out)\n",
    "\n",
    "    model = Sequential(\n",
    "        [\n",
    "            Embedding(\n",
    "                NB_WORDS, output_dim=EMBEDDING_DIM, input_length=MAX_LEN), graph, Dense(\n",
    "                128, activation='relu'), Dense(\n",
    "                    3, activation='softmax')])\n",
    "    model.name = 'MultiLayerMultiFilterIncreasedSizeCNN'\n",
    "    model.summary()\n",
    "    return model\n",
    "\n",
    "\n",
    "# train and validate\n",
    "multilayer_multifilter_increased_size_model = multilayer_multifilter_increased_size_conv()\n",
    "multilayer_multifilter_increased_size_history = train_model(\n",
    "    multilayer_multifilter_increased_size_model,\n",
    "    X_train_seq,\n",
    "    y_train,\n",
    "    X_valid_seq,\n",
    "    y_valid)\n",
    "\n",
    "# test the models\n",
    "test_model(\n",
    "    multilayer_multifilter_increased_size_model,\n",
    "    X_test_seq,\n",
    "    y_test_one_hot)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Pa5-OX2R55Ff"
   },
   "source": [
    "## Embedding File load in the memory "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "VrE0pUBl55Fg"
   },
   "outputs": [],
   "source": [
    "embeddings = read_embedding(GLOVE_EMBEDDINGS_FILE)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "collapsed": true,
    "id": "_29Yf_Pl55Fi"
   },
   "source": [
    "### 3.2.6 Experiment 6 Multi-layer, Multi-filter, Static Pretrained Embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "cYArUU0055Fj",
    "outputId": "89d65d9b-04df-4514-e4ac-f140553d0dd1"
   },
   "outputs": [],
   "source": [
    "# create embedding matrix for the experiment\n",
    "emb_matrix = create_embedding_matrix(tk, 100, embeddings)\n",
    "\n",
    "\n",
    "def multilayer_multifilter_glove_cnn():\n",
    "    graph_in = Input(shape=(NB_WORDS + 1, 100))\n",
    "\n",
    "    convs = []\n",
    "    for filter_size in range(2, 5):\n",
    "        x = Convolution1D(\n",
    "            64,\n",
    "            filter_size,\n",
    "            padding='same',\n",
    "            activation='relu')(graph_in)\n",
    "        convs.append(x)\n",
    "\n",
    "    graph_out = concatenate(convs, axis=1)\n",
    "    graph_out = GlobalMaxPooling1D()(graph_out)\n",
    "    graph = Model(graph_in, graph_out)\n",
    "\n",
    "    model = Sequential([Embedding(NB_WORDS + 1,\n",
    "                                  EMBEDDING_DIM,\n",
    "                                  weights=[emb_matrix],\n",
    "                                  trainable=False,\n",
    "                                  input_length=MAX_LEN),\n",
    "                        graph,\n",
    "                        Dense(128,\n",
    "                              activation='relu'),\n",
    "                        Dense(3,\n",
    "                              activation='softmax')])\n",
    "    model.name = 'MultiLayerMultiFilterStaticEmbeddingsCNN'\n",
    "    model.summary()\n",
    "    return model\n",
    "\n",
    "\n",
    "# train and validate\n",
    "multilayer_multifilter_glove_model = multilayer_multifilter_glove_cnn()\n",
    "multilayer_multifilter_glove_history = train_model(\n",
    "    multilayer_multifilter_glove_model, X_train_seq, y_train, X_valid_seq, y_valid)\n",
    "# test the models\n",
    "test_model(multilayer_multifilter_glove_model, X_test_seq, y_test_one_hot)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "pXoPCtK155Fl"
   },
   "source": [
    "### 3.2.7 Experiment  Multi-layer, Multi-filter, Dynamic Pretrained Embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Bdb4yGvQ55Fn",
    "outputId": "8ff96d77-33c2-40ab-cd13-bcdc6bfa26de"
   },
   "outputs": [],
   "source": [
    "# create embedding matrix for the experiment\n",
    "emb_matrix = create_embedding_matrix(tk, 100, embeddings)\n",
    "\n",
    "\n",
    "def multilayer_multifilter_glove_dynamic_cnn():\n",
    "    graph_in = Input(shape=(NB_WORDS + 1, 100))\n",
    "\n",
    "    convs = []\n",
    "    for filter_size in range(2, 5):\n",
    "        x = Convolution1D(\n",
    "            64,\n",
    "            filter_size,\n",
    "            padding='same',\n",
    "            activation='relu')(graph_in)\n",
    "        convs.append(x)\n",
    "\n",
    "    graph_out = concatenate(convs, axis=1)\n",
    "    graph_out = GlobalMaxPooling1D()(graph_out)\n",
    "    graph = Model(graph_in, graph_out)\n",
    "\n",
    "    model = Sequential([Embedding(NB_WORDS + 1,\n",
    "                                  EMBEDDING_DIM,\n",
    "                                  weights=[emb_matrix],\n",
    "                                  trainable=True,\n",
    "                                  input_length=MAX_LEN),\n",
    "                        graph,\n",
    "                        Dense(128,\n",
    "                              activation='relu'),\n",
    "                        Dense(3,\n",
    "                              activation='softmax')])\n",
    "    model.name = 'MultiLayerMultiFilterDynamicEmbeddingsCNN'\n",
    "    model.summary()\n",
    "    return model\n",
    "\n",
    "\n",
    "# train and validate\n",
    "multilayer_multifilter_glove_dynamic_model = multilayer_multifilter_glove_dynamic_cnn()\n",
    "multilayer_multifilter_glove_dynamic_history = train_model(\n",
    "    multilayer_multifilter_glove_dynamic_model,\n",
    "    X_train_seq,\n",
    "    y_train,\n",
    "    X_valid_seq,\n",
    "    y_valid)\n",
    "\n",
    "# test the models\n",
    "test_model(\n",
    "    multilayer_multifilter_glove_dynamic_model,\n",
    "    X_test_seq,\n",
    "    y_test_one_hot)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "rWK1LOLC55Fp"
   },
   "source": [
    "### 3.2.8 Experiment 8 Yoon Kim's Model Single Channel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "IggN6zYY55Fq",
    "outputId": "6a00f51a-c1bb-4c9f-ae60-8327228600cd"
   },
   "outputs": [],
   "source": [
    "# create embedding matrix for the experiment\n",
    "emb_matrix = create_embedding_matrix(tk, 100, embeddings)\n",
    "\n",
    "\n",
    "def single_channel_kim_cnn():\n",
    "    text_seq_input = Input(shape=(MAX_LEN,), dtype='int32')\n",
    "    text_embedding = Embedding(NB_WORDS + 1,\n",
    "                               EMBEDDING_DIM,\n",
    "                               weights=[emb_matrix],\n",
    "                               trainable=True,\n",
    "                               input_length=MAX_LEN)(text_seq_input)\n",
    "\n",
    "    filter_sizes = [3, 4, 5]\n",
    "    convs = []\n",
    "    for filter_size in filter_sizes:\n",
    "        l_conv = Convolution1D(\n",
    "            filters=128,\n",
    "            kernel_size=filter_size,\n",
    "            padding='same',\n",
    "            activation='relu')(text_embedding)\n",
    "        l_pool = MaxPooling1D(filter_size)(l_conv)\n",
    "        convs.append(l_pool)\n",
    "\n",
    "    merge = concatenate(convs, axis=1)\n",
    "    convol = Convolution1D(128, 5, activation='relu')(merge)\n",
    "    pool1 = GlobalMaxPooling1D()(convol)\n",
    "    dense = Dense(128, activation='relu', name='Dense')(pool1)\n",
    "    out = Dense(3, activation='softmax')(dense)\n",
    "    model = Model(\n",
    "        inputs=[text_seq_input],\n",
    "        outputs=out,\n",
    "        name=\"KimSingleChannelCNN\")\n",
    "    model.summary()\n",
    "    return model\n",
    "\n",
    "\n",
    "single_channel_kim_model = single_channel_kim_cnn()\n",
    "single_channel_kim_model_history = train_model(\n",
    "    single_channel_kim_model, X_train_seq, y_train, X_valid_seq, y_valid)\n",
    "# test the models\n",
    "test_model(single_channel_kim_model, X_test_seq, y_test_one_hot)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "YMED0fq-55Fs"
   },
   "source": [
    "## 3.2.9 Experiment 9 Yoon Kim's Model Multiple Channel\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "gjcTA9u455Ft",
    "outputId": "7c9c969e-7c1b-41c3-ee54-cc323c07c32f"
   },
   "outputs": [],
   "source": [
    "# create embedding matrix for the experiment\n",
    "emb_matrix1 = create_embedding_matrix(tk, 100, embeddings)\n",
    "emb_matrix2 = create_embedding_matrix(tk, 100, embeddings)\n",
    "# a static and a dynamic channel\n",
    "\n",
    "\n",
    "def multiple_channel_kim_cnn():\n",
    "    text_seq_input = Input(shape=(MAX_LEN,), dtype='int32')\n",
    "    text_embedding1 = Embedding(\n",
    "        NB_WORDS + 1,\n",
    "        EMBEDDING_DIM,\n",
    "        weights=[emb_matrix1],\n",
    "        input_length=MAX_LEN,\n",
    "        trainable=True)(text_seq_input)\n",
    "    text_embedding2 = Embedding(\n",
    "        NB_WORDS + 1,\n",
    "        EMBEDDING_DIM,\n",
    "        weights=[emb_matrix2],\n",
    "        input_length=MAX_LEN,\n",
    "        trainable=False)(text_seq_input)\n",
    "    filter_sizes = [3, 4, 5]\n",
    "    convs = []\n",
    "    for text_embedding in [text_embedding1, text_embedding2]:\n",
    "        for filter_size in filter_sizes:\n",
    "            l_conv = Convolution1D(\n",
    "                filters=128,\n",
    "                kernel_size=filter_size,\n",
    "                padding='same',\n",
    "                activation='relu')(text_embedding)\n",
    "            l_pool = MaxPooling1D(filter_size)(l_conv)\n",
    "            convs.append(l_pool)\n",
    "    merge = concatenate(convs, axis=1)\n",
    "    convol = Convolution1D(128, 5, activation='relu')(merge)\n",
    "    pool1 = GlobalMaxPooling1D()(convol)\n",
    "    dense = Dense(128, activation='relu', name='last_but_one')(pool1)\n",
    "    out = Dense(3, activation='softmax')(dense)\n",
    "    model = Model(\n",
    "        inputs=[text_seq_input],\n",
    "        outputs=out,\n",
    "        name=\"KimMultipleChannelCNN\")\n",
    "    model.summary()\n",
    "    return model\n",
    "\n",
    "\n",
    "multiple_channel_kim_model = multiple_channel_kim_cnn()\n",
    "multiple_channel_kim_history = train_model(\n",
    "    multiple_channel_kim_model,\n",
    "    X_train_seq,\n",
    "    y_train,\n",
    "    X_valid_seq,\n",
    "    y_valid)\n",
    "# test the models\n",
    "test_model(multiple_channel_kim_model, X_test_seq, y_test_one_hot)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "nQlPGNWe55Fw"
   },
   "source": [
    "## 3.2.10 Experiment 10 Kalchbrenner et al Dynamic CNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "C3TSdw7055Fx"
   },
   "outputs": [],
   "source": [
    "# This code is adapted from arbackus's suggestion in Keras\n",
    "\n",
    "from keras.engine import Layer, InputSpec\n",
    "import tensorflow as tf\n",
    "\n",
    "\n",
    "class KMaxPooling(Layer):\n",
    "    \"\"\"\n",
    "    K-max pooling layer that extracts the k-highest activations from a sequence (2nd dimension).\n",
    "    TensorFlow backend.\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, k=1, axis=1, **kwargs):\n",
    "        super(KMaxPooling, self).__init__(**kwargs)\n",
    "        self.input_spec = InputSpec(ndim=3)\n",
    "        self.k = k\n",
    "\n",
    "        assert axis in [\n",
    "            1, 2], 'expected dimensions (samples, filters, convolved_values),\\\n",
    "                   cannot fold along samples dimension or axis not in list [1,2]'\n",
    "        self.axis = axis\n",
    "\n",
    "        # need to switch the axis with the last elemnet\n",
    "        # to perform transpose for tok k elements since top_k works in last\n",
    "        # axis\n",
    "        self.transpose_perm = [0, 1, 2]  # default\n",
    "        self.transpose_perm[self.axis] = 2\n",
    "        self.transpose_perm[2] = self.axis\n",
    "\n",
    "    def compute_output_shape(self, input_shape):\n",
    "        input_shape_list = list(input_shape)\n",
    "        input_shape_list[self.axis] = self.k\n",
    "        return tuple(input_shape_list)\n",
    "\n",
    "    def call(self, x):\n",
    "        # swap sequence dimension to get top k elements along axis=1\n",
    "        transposed_for_topk = tf.transpose(x, perm=self.transpose_perm)\n",
    "\n",
    "        # extract top_k, returns two tensors [values, indices]\n",
    "        top_k = tf.nn.top_k(\n",
    "            transposed_for_topk,\n",
    "            k=self.k,\n",
    "            sorted=True,\n",
    "            name=None)[0]\n",
    "\n",
    "        # return back to normal dimension but now sequence dimension has only k elements\n",
    "        # performing another transpose will get the tensor back to its original shape\n",
    "        # but will have k as its axis_1 size\n",
    "        transposed_back = tf.transpose(top_k, perm=self.transpose_perm)\n",
    "\n",
    "        return transposed_back\n",
    "\n",
    "\n",
    "class Folding(Layer):\n",
    "\n",
    "    def __init__(self, **kwargs):\n",
    "        super(Folding, self).__init__(**kwargs)\n",
    "        self.input_spec = InputSpec(ndim=3)\n",
    "\n",
    "    def compute_output_shape(self, input_shape):\n",
    "        return (input_shape[0], input_shape[1], int(input_shape[2] / 2))\n",
    "\n",
    "    def call(self, x):\n",
    "        input_shape = x.get_shape().as_list()\n",
    "\n",
    "        # split the tensor along dimension 2 into dimension_axis_size/2\n",
    "        # which will give us 2 tensors\n",
    "        splits = tf.split(\n",
    "            x, num_or_size_splits=int(\n",
    "                input_shape[2] / 2), axis=2)\n",
    "\n",
    "        # reduce sums of the pair of rows we have split onto\n",
    "        reduce_sums = [tf.reduce_sum(split, axis=2) for split in splits]\n",
    "\n",
    "        # stack them up along the same axis we have reduced\n",
    "        row_reduced = tf.stack(reduce_sums, axis=2)\n",
    "        return row_reduced\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "-0gwr0_C55F2",
    "outputId": "7b8f3f29-9d8e-45f8-b25e-658165bad482"
   },
   "outputs": [],
   "source": [
    "trained_embeddings = create_embedding_matrix(tk, 100, embeddings)\n",
    "\n",
    "\n",
    "def dynamic_cnn():\n",
    "    model = Sequential(\n",
    "        [\n",
    "            Embedding(\n",
    "                NB_WORDS + 1,\n",
    "                EMBEDDING_DIM,\n",
    "                weights=[trained_embeddings],\n",
    "                input_length=MAX_LEN,\n",
    "                trainable=True),\n",
    "            ZeroPadding1D(\n",
    "                (49,\n",
    "                 49)),\n",
    "            Convolution1D(\n",
    "                64,\n",
    "                50,\n",
    "                padding=\"same\"),\n",
    "            KMaxPooling(\n",
    "                k=5,\n",
    "                axis=1),\n",
    "            Activation(\"relu\"),\n",
    "            ZeroPadding1D(\n",
    "                (24,\n",
    "                 24)),\n",
    "            Convolution1D(\n",
    "                64,\n",
    "                25,\n",
    "                padding=\"same\"),\n",
    "            Folding(),\n",
    "            KMaxPooling(\n",
    "                k=5,\n",
    "                axis=1),\n",
    "            Activation(\"relu\"),\n",
    "            Flatten(),\n",
    "            Dense(\n",
    "                3,\n",
    "                activation=\"softmax\")])\n",
    "    model.name = 'KalchbrennerDynamicCNN'\n",
    "    model.summary()\n",
    "    return model\n",
    "\n",
    "\n",
    "dynamic_cnn_model = dynamic_cnn()\n",
    "dynamic_cnn_history = train_model(\n",
    "    dynamic_cnn_model,\n",
    "    X_train_seq,\n",
    "    y_train,\n",
    "    X_valid_seq,\n",
    "    y_valid)\n",
    "# test the models\n",
    "test_model(dynamic_cnn_model, X_test_seq, y_test_one_hot)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "d3VJCWoV55F-"
   },
   "source": [
    "## 3.2.11 Experiment 11 Multi Channel Variable size CNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "RXOtfK0Z55F_",
    "outputId": "a55187ec-0030-4191-8e17-efdab95b4e3a"
   },
   "outputs": [],
   "source": [
    "trained_embeddings1 = create_embedding_matrix(tk, 100, embeddings)\n",
    "trained_embeddings2 = create_embedding_matrix(tk, 100, embeddings)\n",
    "\n",
    "# two channels with static and dynamic\n",
    "\n",
    "\n",
    "def multichannel_variable_cnn():\n",
    "    text_seq_input = Input(shape=(MAX_LEN,), dtype='int32')\n",
    "    text_embedding1 = Embedding(\n",
    "        NB_WORDS + 1,\n",
    "        EMBEDDING_DIM,\n",
    "        weights=[trained_embeddings1],\n",
    "        input_length=MAX_LEN,\n",
    "        trainable=True)(text_seq_input)\n",
    "    text_embedding2 = Embedding(\n",
    "        NB_WORDS + 1,\n",
    "        EMBEDDING_DIM,\n",
    "        weights=[trained_embeddings2],\n",
    "        input_length=MAX_LEN,\n",
    "        trainable=False)(text_seq_input)\n",
    "\n",
    "    k_top = 4\n",
    "\n",
    "    layer_1 = []\n",
    "    for text_embedding in [text_embedding1, text_embedding2]:\n",
    "        conv_pools = []\n",
    "        filter_sizes = [3, 5]\n",
    "        for filter_size in filter_sizes:\n",
    "            l_zero = ZeroPadding1D(\n",
    "                (filter_size - 1, filter_size - 1))(text_embedding)\n",
    "            l_conv = Convolution1D(\n",
    "                filters=128,\n",
    "                kernel_size=filter_size,\n",
    "                padding='same',\n",
    "                activation='tanh')(l_zero)\n",
    "            l_pool = KMaxPooling(k=28, axis=1)(l_conv)\n",
    "            conv_pools.append((filter_size, l_pool))\n",
    "            layer_1.append(conv_pools)\n",
    "\n",
    "    last_layer = []\n",
    "    for layer in layer_1:  # no of embeddings used\n",
    "        for (filter_size, input_feature_maps) in layer:\n",
    "            l_zero = ZeroPadding1D(\n",
    "                (filter_size - 1, filter_size - 1))(input_feature_maps)\n",
    "            l_conv = Convolution1D(\n",
    "                filters=128,\n",
    "                kernel_size=filter_size,\n",
    "                padding='same',\n",
    "                activation='tanh')(l_zero)\n",
    "            l_pool = KMaxPooling(k=k_top, axis=1)(l_conv)\n",
    "            last_layer.append(l_pool)\n",
    "\n",
    "    l_merge = concatenate(last_layer, axis=1)\n",
    "    l_flat = Flatten()(l_merge)\n",
    "    l_dense = Dense(128, activation='relu')(l_flat)\n",
    "    l_out = Dense(3, activation='softmax')(l_dense)\n",
    "    model = Model(inputs=[text_seq_input], outputs=l_out)\n",
    "    model.name = 'MultiChannelVariableCNN'\n",
    "    model.summary()\n",
    "    return model\n",
    "\n",
    "\n",
    "multichannel_variable_cnn_model = multichannel_variable_cnn()\n",
    "multichannel_variable_cnn_history = train_model(\n",
    "    multichannel_variable_cnn_model,\n",
    "    X_train_seq,\n",
    "    y_train,\n",
    "    X_valid_seq,\n",
    "    y_valid)\n",
    "# test the models\n",
    "test_model(multichannel_variable_cnn_model, X_test_seq, y_test_one_hot)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "c2ouNp3G55GD"
   },
   "source": [
    "## 3.2.12 Experiment 12 Multi Group  CNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "LeTvfefp55GD"
   },
   "outputs": [],
   "source": [
    "# laod the embedding file\n",
    "fasttext_embeddings = read_embedding(FASTTEXT_EMBEDDINGS_FILE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "E7zZRsng55GF",
    "outputId": "95705b28-5c30-433c-9138-9a1baef414a0"
   },
   "outputs": [],
   "source": [
    "# 100-d GloVe embeddings\n",
    "trained_embeddings1 = create_embedding_matrix(tk, EMBEDDING_DIM, embeddings)\n",
    "# 100-d GloVe embeddings\n",
    "trained_embeddings2 = create_embedding_matrix(tk, EMBEDDING_DIM, embeddings)\n",
    "# 300-d fastText embeddings\n",
    "trained_embeddings3 = create_embedding_matrix(\n",
    "    tk, FASTTEXT_EMBEDDING_DIM, fasttext_embeddings)\n",
    "\n",
    "\n",
    "def multigroup_normconstraint_cnn():\n",
    "    text_seq_input = Input(shape=(MAX_LEN,), dtype='int32')\n",
    "    text_embedding1 = Embedding(\n",
    "        NB_WORDS + 1,\n",
    "        EMBEDDING_DIM,\n",
    "        weights=[trained_embeddings1],\n",
    "        input_length=MAX_LEN,\n",
    "        trainable=True)(text_seq_input)\n",
    "    text_embedding2 = Embedding(\n",
    "        NB_WORDS + 1,\n",
    "        EMBEDDING_DIM,\n",
    "        weights=[trained_embeddings2],\n",
    "        input_length=MAX_LEN,\n",
    "        trainable=False)(text_seq_input)\n",
    "    text_embedding3 = Embedding(\n",
    "        NB_WORDS + 1,\n",
    "        FASTTEXT_EMBEDDING_DIM,\n",
    "        weights=[trained_embeddings3],\n",
    "        input_length=MAX_LEN,\n",
    "        trainable=True)(text_seq_input)\n",
    "\n",
    "    k_top = 4\n",
    "    filter_sizes = [3, 5]\n",
    "\n",
    "    conv_pools = []\n",
    "    for text_embedding in [text_embedding1, text_embedding2, text_embedding3]:\n",
    "        for filter_size in filter_sizes:\n",
    "            l_zero = ZeroPadding1D(\n",
    "                (filter_size - 1, filter_size - 1))(text_embedding)\n",
    "            l_conv = Convolution1D(\n",
    "                filters=16,\n",
    "                kernel_size=filter_size,\n",
    "                padding='same',\n",
    "                activation='tanh')(l_zero)\n",
    "            l_pool = GlobalMaxPooling1D()(l_conv)\n",
    "            conv_pools.append(l_pool)\n",
    "\n",
    "    l_merge = concatenate(conv_pools, axis=1)\n",
    "    l_dense = Dense(\n",
    "        128,\n",
    "        activation='relu',\n",
    "        kernel_regularizer=regularizers.l2(0.001))(l_merge)\n",
    "    l_out = Dense(3, activation='softmax')(l_dense)\n",
    "    model = Model(inputs=[text_seq_input], outputs=l_out)\n",
    "    model.name = 'MultiGroupNormConstraintCNN'\n",
    "    model.summary()\n",
    "    return model\n",
    "\n",
    "\n",
    "multigroup_normconstraint_cnn_model = multigroup_normconstraint_cnn()\n",
    "multigroup_normconstraint_cnn_history = train_model(\n",
    "    multigroup_normconstraint_cnn_model,\n",
    "    X_train_seq,\n",
    "    y_train,\n",
    "    X_valid_seq,\n",
    "    y_valid)\n",
    "# test the models\n",
    "test_model(multigroup_normconstraint_cnn_model, X_test_seq, y_test_one_hot)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "uk7PLbjf55GH"
   },
   "source": [
    "## Character-based tokenziation of Train and Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "VkOTJ9ke55GI",
    "outputId": "6593a73e-ad4b-4d32-fa15-1e5199e440a8"
   },
   "outputs": [],
   "source": [
    "print('Current Sequences are :', X_train_entire.shape)\n",
    "tokenizer = Tokenizer(char_level=True)\n",
    "tokenizer.fit_on_texts(list(X_train_entire))\n",
    "\n",
    "\n",
    "# train data\n",
    "list_tokenized_train = tokenizer.texts_to_sequences(X_train_entire)\n",
    "X_train_entire_char = pad_sequences(list_tokenized_train, maxlen=MAX_SENT_LEN)\n",
    "# test data\n",
    "list_tokenized_test = tokenizer.texts_to_sequences(X_test)\n",
    "X_test_char = pad_sequences(list_tokenized_test, maxlen=MAX_SENT_LEN)\n",
    "\n",
    "X_train_char, X_valid_char, y_train_char, y_valid_char = train_test_split(\n",
    "    X_train_entire_char, y_train_one_hot, test_size=0.15, random_state=37)\n",
    "\n",
    "assert X_valid_char.shape[0] == y_valid_char.shape[0]\n",
    "assert X_train_char.shape[0] == y_train_char.shape[0]\n",
    "\n",
    "print('Shape of Training set:', X_train_char.shape)\n",
    "print('Shape of validation set:', X_valid_char.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "1LPZ8c4X55GK"
   },
   "source": [
    "## 3.13 Experiment 13 Character-based CNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "vqi_S5wI55GL",
    "outputId": "734949e9-d2d9-429a-91d8-8c698aecf0eb"
   },
   "outputs": [],
   "source": [
    "char_embeddings = np.random.randn(1024, 128)\n",
    "char_embeddings.shape\n",
    "\n",
    "\n",
    "def character_multiplefilter_cnn():\n",
    "       # Input layer\n",
    "    inputs = Input(shape=(None, ))\n",
    "    x = Embedding(1024, 128, weights=[char_embeddings],\n",
    "                  input_length=300, trainable=True)(inputs)\n",
    "    filter_sizes = [10, 7, 5, 3]\n",
    "    convs = []\n",
    "    for filter_size in filter_sizes:\n",
    "        l_conv = Convolution1D(\n",
    "            filters=256,\n",
    "            kernel_size=filter_size,\n",
    "            activation='tanh')(x)\n",
    "        l_pool = GlobalMaxPooling1D()(l_conv)\n",
    "        convs.append(l_pool)\n",
    "    x = concatenate(convs, axis=1)\n",
    "    # Fully connected layers\n",
    "    x = Dense(1024, activation='selu', kernel_initializer='lecun_normal')(x)\n",
    "    x = AlphaDropout(0.5)(x)\n",
    "    x = Dense(1024, activation='selu', kernel_initializer='lecun_normal')(x)\n",
    "    x = Dropout(0.5)(x)\n",
    "    # output layer\n",
    "    predictions = Dense(3, activation='softmax')(x)\n",
    "    # Build and compile model\n",
    "    model = Model(inputs=inputs, outputs=predictions)\n",
    "    model.name = 'CharacterBasedMultipleFilterCNN'\n",
    "    model.summary()\n",
    "    return model\n",
    "\n",
    "\n",
    "character_multiplefilter_cnn_model = character_multiplefilter_cnn()\n",
    "character_multiplefilter_cnn_history = train_model(\n",
    "    character_multiplefilter_cnn_model,\n",
    "    X_train_char,\n",
    "    y_train_char,\n",
    "    X_valid_char,\n",
    "    y_valid_char)\n",
    "\n",
    "# test the models\n",
    "test_model(character_multiplefilter_cnn_model, X_test_char, y_test_one_hot)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "1NDxkt1D55GN",
    "outputId": "f06c14a2-b273-4a20-fdbe-c0f5ab953f26"
   },
   "outputs": [],
   "source": [
    "char_embeddings = np.random.randn(1024, 64)\n",
    "\n",
    "\n",
    "def character_zhang_cnn():\n",
    "    # Input layer\n",
    "    inputs = Input(shape=(None, ))\n",
    "    x = Embedding(1024, 64, weights=[char_embeddings],\n",
    "                  input_length=300, trainable=True)(inputs)\n",
    "    # convolution layers\n",
    "    # layer 1\n",
    "    x = Convolution1D(256, 7)(x)\n",
    "    x = ThresholdedReLU(1e-6)(x)\n",
    "    x = MaxPooling1D(3)(x)\n",
    "    # layer 2\n",
    "    x = Convolution1D(256, 7)(x)\n",
    "    x = ThresholdedReLU(1e-6)(x)\n",
    "    x = MaxPooling1D(3)(x)\n",
    "    # layer 3\n",
    "    x = Convolution1D(256, 7)(x)\n",
    "    x = ThresholdedReLU(1e-6)(x)\n",
    "    # layer 4\n",
    "    x = Convolution1D(256, 7)(x)\n",
    "    x = ThresholdedReLU(1e-6)(x)\n",
    "    # layer 5\n",
    "    x = Convolution1D(256, 7)(x)\n",
    "    x = ThresholdedReLU(1e-6)(x)\n",
    "    # layer 6\n",
    "    x = Convolution1D(256, 3)(x)\n",
    "    # layer 7\n",
    "    x = ThresholdedReLU(1e-6)(x)\n",
    "    x = MaxPooling1D(3)(x)\n",
    "    # Flatten\n",
    "    x = Flatten()(x)\n",
    "    # Fully connected layers\n",
    "    x = Dense(1024)(x)\n",
    "    x = ThresholdedReLU(1e-6)(x)\n",
    "    x = Dropout(0.5)(x)\n",
    "    x = Dense(1024)(x)\n",
    "    x = ThresholdedReLU(1e-6)(x)\n",
    "    x = Dropout(0.5)(x)\n",
    "    # Output layer\n",
    "    predictions = Dense(3, activation='softmax')(x)\n",
    "    # Build and compile model\n",
    "    model = Model(inputs=inputs, outputs=predictions)\n",
    "    model.name = 'CharacterBasedZhangCNN'\n",
    "    model.summary()\n",
    "    return model\n",
    "\n",
    "\n",
    "character_zhang_cnn_model = character_zhang_cnn()\n",
    "character_zhang_cnn_history = train_model(\n",
    "    character_zhang_cnn_model,\n",
    "    X_train_char,\n",
    "    y_train_char,\n",
    "    X_valid_char,\n",
    "    y_valid_char)\n",
    "# test the models\n",
    "test_model(character_zhang_cnn_model, X_test_char, y_test_one_hot)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Ja4cjgyD55GR"
   },
   "source": [
    "## 3.2.14  Experiment 14 Very Deep Convolutional Networks for Text Classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "z8nJ9v2555GS",
    "outputId": "30b04331-2f0e-4e52-c417-75b74e3fe079"
   },
   "outputs": [],
   "source": [
    "char_embeddings = np.random.randn(300, 64)\n",
    "char_embeddings.shape\n",
    "\n",
    "\n",
    "def very_deep_character_cnn():\n",
    "    model = Sequential([\n",
    "        Embedding(300, 64, weights=[char_embeddings],\n",
    "                  input_length=300, trainable=True),\n",
    "        Convolution1D(64, 3, padding=\"valid\")\n",
    "    ])\n",
    "\n",
    "    # 4 pairs of convolution blocks followed by pooling\n",
    "    for filter_size in [64, 128, 256]:\n",
    "\n",
    "        # each iteration is a convolution block\n",
    "        for cb_i in [0, 1]:\n",
    "            model.add(Convolution1D(filter_size, 3, padding=\"same\"))\n",
    "            model.add(BatchNormalization())\n",
    "            model.add(Activation(\"relu\"))\n",
    "            model.add(Convolution1D(filter_size, 3, padding=\"same\")),\n",
    "            model.add(BatchNormalization())\n",
    "            model.add(Activation(\"relu\"))\n",
    "\n",
    "        model.add(MaxPooling1D(pool_size=2, strides=3))\n",
    "\n",
    "    # model.add(KMaxPooling(k=2))\n",
    "    model.add(Flatten())\n",
    "    model.add(Dense(4096, activation=\"relu\"))\n",
    "    model.add(Dense(2048, activation=\"relu\"))\n",
    "    model.add(Dense(2048, activation=\"relu\"))\n",
    "    model.add(Dense(3, activation=\"softmax\"))\n",
    "    model.name = \"VeryDeepCharacterBasedCNN\"\n",
    "    model.summary()\n",
    "    return model\n",
    "\n",
    "\n",
    "very_deep_character_cnn_model = very_deep_character_cnn()\n",
    "very_deep_character_cnn_history = train_model(\n",
    "    very_deep_character_cnn_model,\n",
    "    X_train_char,\n",
    "    y_train_char,\n",
    "    X_valid_char,\n",
    "    y_valid_char)\n",
    "# test the models\n",
    "test_model(very_deep_character_cnn_model, X_test_char, y_test_one_hot)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "-FUx1NjZ55GU"
   },
   "source": [
    "## 3.2.15 Experiment 15 CNN with Dilations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "DEFmsgLS55GV",
    "outputId": "dee0d30b-7b7a-449c-a31c-42ee9c25b481"
   },
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "def character_dilated_cnn(conv_layers=2,\n",
    "                          dilation_rates=[0, 2, 4],\n",
    "                          embed_size=256):\n",
    "    inp = Input(shape=(None, ))\n",
    "    x = Embedding(input_dim=len(tokenizer.word_counts) + 1,\n",
    "                  output_dim=embed_size)(inp)\n",
    "    prefilt_x = Dropout(0.25)(x)\n",
    "    out_conv = []\n",
    "    # dilation rate lets us use ngrams and skip grams to process\n",
    "    for dilation_rate in dilation_rates:\n",
    "        x = prefilt_x\n",
    "        for i in range(2):\n",
    "            if dilation_rate > 0:\n",
    "                x = Convolution1D(\n",
    "                    16 * 2**(i),\n",
    "                    kernel_size=3,\n",
    "                    dilation_rate=dilation_rate,\n",
    "                    activation='relu',\n",
    "                    name='ngram_{}_cnn_{}'.format(\n",
    "                        dilation_rate,\n",
    "                        i))(x)\n",
    "            else:\n",
    "                x = Convolution1D(16 * 2**(i),\n",
    "                                  kernel_size=1,\n",
    "                                  activation='relu',\n",
    "                                  name='word_fcl_{}'.format(i))(x)\n",
    "        out_conv += [Dropout(0.5)(GlobalMaxPooling1D()(x))]\n",
    "    x = concatenate(out_conv, axis=-1)\n",
    "    x = Dense(64, activation='relu')(x)\n",
    "    x = Dropout(0.1)(x)\n",
    "    x = Dense(32, activation='relu')(x)\n",
    "    x = Dropout(0.1)(x)\n",
    "    x = Dense(3, activation='sigmoid')(x)\n",
    "    model = Model(inputs=inp, outputs=x)\n",
    "    model.name = 'CharacterDilationCNN'\n",
    "    model.summary()\n",
    "    return model\n",
    "\n",
    "\n",
    "character_dilated_cnn_model = character_dilated_cnn()\n",
    "character_dilated_cnn_history = train_model(\n",
    "    character_dilated_cnn_model,\n",
    "    X_train_char,\n",
    "    y_train_char,\n",
    "    X_valid_char,\n",
    "    y_valid_char)\n",
    "# test the models\n",
    "test_model(character_dilated_cnn_model, X_test_char, y_test_one_hot)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "5XjCbSQ655GX"
   },
   "source": [
    "## 3.2.16 Experiment 16 Ensemble of CNN and LSTM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ABhMPQHM55GX",
    "outputId": "65fd6d45-5344-4f08-9c92-3fc136f0cc9b"
   },
   "outputs": [],
   "source": [
    "trained_embeddings = create_embedding_matrix(tk, EMBEDDING_DIM, embeddings)\n",
    "\n",
    "\n",
    "def ensemble_cnn_lstm():\n",
    "    text_seq_input = Input(shape=(MAX_LEN,), dtype='int32')\n",
    "    text_embedding = Embedding(NB_WORDS + 1,\n",
    "                               EMBEDDING_DIM,\n",
    "                               weights=[trained_embeddings],\n",
    "                               trainable=True,\n",
    "                               input_length=MAX_LEN)(text_seq_input)\n",
    "\n",
    "    filter_sizes = [3, 4, 5, 6, 7]\n",
    "    convs = []\n",
    "    for filter_size in filter_sizes:\n",
    "        l_conv = Convolution1D(\n",
    "            filters=128,\n",
    "            kernel_size=filter_size,\n",
    "            padding='same',\n",
    "            activation='relu')(text_embedding)\n",
    "        l_pool = MaxPooling1D(filter_size)(l_conv)\n",
    "        convs.append(l_pool)\n",
    "\n",
    "    merge = concatenate(convs, axis=1)\n",
    "    flat = Flatten()(merge)\n",
    "    cnn_dense = Dense(128, activation='relu')(flat)\n",
    "    rnn_layer = LSTM(\n",
    "        128,\n",
    "        return_sequences=False,\n",
    "        stateful=False)(\n",
    "        text_embedding,\n",
    "        initial_state=[\n",
    "            cnn_dense,\n",
    "            cnn_dense])\n",
    "    out = Dense(3, activation='softmax')(rnn_layer)\n",
    "    model = Model(inputs=[text_seq_input], outputs=out)\n",
    "    model.name = 'EnsembleCNNLSTM'\n",
    "    model.summary()\n",
    "    return model\n",
    "\n",
    "\n",
    "ensemble_cnn_lstm_model = ensemble_cnn_lstm()\n",
    "ensemble_cnn_lstm_history = train_model(\n",
    "    ensemble_cnn_lstm_model,\n",
    "    X_train_seq,\n",
    "    y_train,\n",
    "    X_valid_seq,\n",
    "    y_valid)\n",
    "# test the models\n",
    "test_model(ensemble_cnn_lstm_model, X_test_seq, y_test_one_hot)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "uZa3H4Xv55Ga"
   },
   "source": [
    "## 3.2.17 Experiment 17 A C-LSTM Neural Network for Text Classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "dCf1lMRQ55Gc",
    "outputId": "7ef87539-f570-4440-d610-df7bc0e21803"
   },
   "outputs": [],
   "source": [
    "trained_embeddings = create_embedding_matrix(tk, 100, embeddings)\n",
    "\n",
    "\n",
    "def c_lstm():\n",
    "    text_seq_input = Input(shape=(MAX_LEN,), dtype='int32')\n",
    "    text_embedding = Embedding(NB_WORDS + 1,\n",
    "                               EMBEDDING_DIM,\n",
    "                               weights=[trained_embeddings],\n",
    "                               trainable=True,\n",
    "                               input_length=MAX_LEN)(text_seq_input)\n",
    "\n",
    "    filter_sizes = [3, 4, 5, 6, 7]\n",
    "    convs = []\n",
    "    for filter_size in filter_sizes:\n",
    "        l_conv = Convolution1D(\n",
    "            filters=128,\n",
    "            kernel_size=filter_size,\n",
    "            padding='same',\n",
    "            activation='relu')(text_embedding)\n",
    "        convs.append(l_conv)\n",
    "\n",
    "    cnn_feature_maps = concatenate(convs, axis=1)\n",
    "    sentence_encoder = LSTM(64, return_sequences=False)(cnn_feature_maps)\n",
    "    fully_connected = Dense(128, activation=\"relu\")(sentence_encoder)\n",
    "    out = Dense(3, activation=\"softmax\")(fully_connected)\n",
    "    model = Model(inputs=[text_seq_input], outputs=out)\n",
    "    model.summary()\n",
    "    return model\n",
    "\n",
    "\n",
    "c_lstm_model = c_lstm()\n",
    "c_lstm_history = train_model(\n",
    "    c_lstm_model,\n",
    "    X_train_seq,\n",
    "    y_train,\n",
    "    X_valid_seq,\n",
    "    y_valid)\n",
    "# test the models\n",
    "test_model(c_lstm_model, X_test_seq, y_test_one_hot)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Wt_4Ki6s55Ge"
   },
   "source": [
    "## 3.2.18 Experiment 18 AC-BLSTM: Asymmetric Convolutional Bidirectional LSTM Networks for Text Classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "-WSwadSy55Gg",
    "outputId": "b239a2c1-d5d4-4e93-9e50-7b9fe9a5fb6b"
   },
   "outputs": [],
   "source": [
    "trained_embeddings = create_embedding_matrix(tk, 100, embeddings)\n",
    "\n",
    "\n",
    "def ac_bilstm():\n",
    "    text_seq_input = Input(shape=(MAX_LEN,), dtype='int32')\n",
    "    text_embedding = Embedding(NB_WORDS + 1,\n",
    "                               EMBEDDING_DIM,\n",
    "                               weights=[trained_embeddings],\n",
    "                               trainable=True,\n",
    "                               input_length=MAX_LEN)(text_seq_input)\n",
    "\n",
    "    filter_sizes = [3, 4, 5, 6, 7]\n",
    "    convs = []\n",
    "    for filter_size in filter_sizes:\n",
    "        l_conv1 = Convolution1D(filters=64, kernel_size=1, strides=1,\n",
    "                                padding=\"same\")(text_embedding)\n",
    "        l_relu1 = Activation(\"relu\")(l_conv1)\n",
    "        l_conv2 = Convolution1D(filters=64, kernel_size=filter_size, strides=1,\n",
    "                                padding=\"same\")(l_relu1)\n",
    "        l_relu2 = Activation(\"relu\")(l_conv2)\n",
    "        convs.append(l_relu2)\n",
    "\n",
    "    l_concat = concatenate(convs, axis=2)\n",
    "    l_blstm = Bidirectional(\n",
    "        LSTM(\n",
    "            32,\n",
    "            activation=\"relu\",\n",
    "            return_sequences=True))(l_concat)\n",
    "    l_dropout = Dropout(0.5)(l_blstm)\n",
    "    l_flatten = Flatten()(l_dropout)\n",
    "    l_fc = Dense(128, activation='sigmoid')(l_flatten)\n",
    "    out = Dense(3, activation=\"softmax\")(l_fc)\n",
    "    model = Model(inputs=[text_seq_input], outputs=out)\n",
    "    model.name = 'AC-BILSTM'\n",
    "    model.summary()\n",
    "    return model\n",
    "\n",
    "\n",
    "ac_bilstm_model = ac_bilstm()\n",
    "ac_bilstm_history = train_model(\n",
    "    ac_bilstm_model,\n",
    "    X_train_seq,\n",
    "    y_train,\n",
    "    X_valid_seq,\n",
    "    y_valid)\n",
    "# test the models\n",
    "test_model(ac_bilstm_model, X_test_seq, y_test_one_hot)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "pebhHyoq55Gh"
   },
   "source": [
    "## 4. Visualization of Layer in CNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "CduBvNqo55Gi",
    "outputId": "d8c67932-d430-47b2-cc35-6d3e7d37a648"
   },
   "outputs": [],
   "source": [
    "# create embedding matrix for the experiment\n",
    "emb_matrix = create_embedding_matrix(tk, 100, embeddings)\n",
    "\n",
    "\n",
    "def single_channel_kim_cnn():\n",
    "    text_seq_input = Input(shape=(MAX_LEN,), dtype='int32')\n",
    "    text_embedding = Embedding(NB_WORDS + 1,\n",
    "                               EMBEDDING_DIM,\n",
    "                               weights=[emb_matrix],\n",
    "                               trainable=True,\n",
    "                               input_length=MAX_LEN)(text_seq_input)\n",
    "\n",
    "    filter_sizes = [3, 4, 5]\n",
    "    convs = []\n",
    "    for filter_size in filter_sizes:\n",
    "        l_conv = Convolution1D(\n",
    "            filters=128,\n",
    "            kernel_size=filter_size,\n",
    "            padding='same',\n",
    "            activation='relu')(text_embedding)\n",
    "        l_pool = MaxPooling1D(filter_size)(l_conv)\n",
    "        convs.append(l_pool)\n",
    "\n",
    "    merge = concatenate(convs, axis=1)\n",
    "    convol = Convolution1D(128, 5, activation='relu')(merge)\n",
    "    pool1 = GlobalMaxPooling1D()(convol)\n",
    "    dense = Dense(128, activation='relu', name='last_but_one')(pool1)\n",
    "    out = Dense(3, activation='softmax')(dense)\n",
    "    model = Model(inputs=[text_seq_input], outputs=out,\n",
    "                  name=\"KimSingleChannelCNN-Visualization\")\n",
    "    model.summary()\n",
    "    return model\n",
    "\n",
    "\n",
    "single_channel_kim_model = single_channel_kim_cnn()\n",
    "# train kim's cnn\n",
    "single_channel_kim_model_history = train_model(\n",
    "    single_channel_kim_model, X_train_seq, y_train, X_valid_seq, y_valid)\n",
    "# test the models to see results are same\n",
    "test_model(single_channel_kim_model, X_test_seq, y_test_one_hot)\n",
    "# make a copy\n",
    "single_channel_kim_model_copy = models.clone_model(\n",
    "    single_channel_kim_model)  # create a copy of Kim's CNN\n",
    "single_channel_kim_model.set_weights(\n",
    "    single_channel_kim_model_copy.get_weights())  # transfer the weights\n",
    "# test the models to see results are same\n",
    "test_model(single_channel_kim_model, X_test_seq, y_test_one_hot)\n",
    "# Get rid of the classification layer\n",
    "single_channel_kim_model_copy.layers.pop()\n",
    "single_channel_kim_model_copy.outputs = [\n",
    "    single_channel_kim_model_copy.layers[-1].output]\n",
    "single_channel_kim_model_copy.layers[-1].outbound_nodes = []\n",
    "single_channel_kim_model_copy.summary()\n",
    "\n",
    "# get the hidden features from a layer back\n",
    "hidden_features = single_channel_kim_model_copy.predict(X_test_seq)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "0-WRmMi755Gk",
    "outputId": "9b271f8f-5d98-42ae-b156-a6a6d4c6b7af"
   },
   "outputs": [],
   "source": [
    "from sklearn.decomposition import PCA\n",
    "from sklearn.manifold import TSNE\n",
    "\n",
    "# perform PCA on the hidden features\n",
    "pca = PCA(n_components=30)\n",
    "pca_result = pca.fit_transform(hidden_features)\n",
    "print('Variance PCA: {}'.format(np.sum(pca.explained_variance_ratio_)))\n",
    "\n",
    "# Run T-SNE on the PCA features.\n",
    "tsne = TSNE(n_components=2, verbose=1)\n",
    "tsne_results = tsne.fit_transform(pca_result)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "IGzccXRy55Gm",
    "outputId": "f6dcdf0d-1632-4fd2-823f-1288e0d51245",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from keras.utils import np_utils\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "color_map = np.argmax(y_test_one_hot, axis=1)\n",
    "plt.figure(figsize=(8,8))\n",
    "for cl in range(3):\n",
    "    indices = np.where(color_map==cl)\n",
    "    indices = indices[0]\n",
    "    plt.scatter(tsne_results[indices,0], tsne_results[indices, 1], label=labels[cl])\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "q8zdpU_t55Go"
   },
   "source": [
    "## 5. False Positive and False Negative Analysis "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "FUylXOc655Gp"
   },
   "outputs": [],
   "source": [
    "y_class_index, y_pred_index, confidence,y_softmax = get_predictions_and_confidences(single_channel_kim_model,X_test_seq, y_test_one_hot)\n",
    "# convert them to data frames\n",
    "class_series = pd.Series(y_class_index)\n",
    "class_frame = pd.DataFrame()\n",
    "class_frame['labels'] = class_series.values\n",
    "predicted_series = pd.Series(y_pred_index)\n",
    "predicted_frame = pd.DataFrame()\n",
    "predicted_frame['predictions'] = predicted_series.values\n",
    "confidence_series = pd.Series(confidence)\n",
    "confidence_frame = pd.DataFrame()\n",
    "confidence_frame['confidence'] = confidence_series.values\n",
    "\n",
    "# Creating a reverse dictionary\n",
    "reverse_word_map = dict(map(reversed, tk.word_index.items()))\n",
    "\n",
    "# Function takes a tokenized sentence and returns the words\n",
    "def sequence_to_text(list_of_indices):\n",
    "    # Looking up words in dictionary\n",
    "    words = [reverse_word_map.get(letter) for letter in list_of_indices]\n",
    "    return(words)\n",
    "\n",
    "# Function to convert sentence word index to words\n",
    "def index_to_sentences(X_test):\n",
    "    sentences =[]\n",
    "    for i in range(len(X_test)):\n",
    "        sequence_string = sequence_to_text(X_test[i])\n",
    "        while None in sequence_string : sequence_string.remove(None)\n",
    "        sentence = \" \".join(sequence_string)\n",
    "        sentences.append(sentence)\n",
    "    return sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "iYe3YBCg55Gq",
    "outputId": "afdc12e4-911f-45da-821a-a3b18f133465"
   },
   "outputs": [],
   "source": [
    "all_texts = index_to_sentences(X_test_seq)\n",
    "se = pd.Series(all_texts)\n",
    "test_data = pd.DataFrame()\n",
    "test_data['text'] = se.values\n",
    "all_frames = [test_data,class_frame,predicted_frame,confidence_frame]\n",
    "predictions_text_labels_frame=pd.concat(all_frames,axis=1)\n",
    "predictions_text_labels_frame.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "3Hwzgq_Z55Gt"
   },
   "source": [
    "## 5.1. False Negative Analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "gNB6Nuy-55Gt",
    "outputId": "dbfcaaff-dbdc-47de-b2f8-1602827c7302"
   },
   "outputs": [],
   "source": [
    "pd.set_option('display.max_colwidth', -1)\n",
    "predictions_text_labels_frame.loc[(predictions_text_labels_frame['labels'] == 2) &\n",
    "                                  predictions_text_labels_frame['predictions'].isin([1, 0])].sort_values(by=['confidence'])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "rhdxt6k255Gv"
   },
   "source": [
    "## 5.2 False Postive Analysis "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "jtL12bHN55Gw",
    "outputId": "dcd1bbd7-b9bb-4f2b-bd6d-d5991015f1cc"
   },
   "outputs": [],
   "source": [
    "pd.set_option('display.max_colwidth', -1)\n",
    "predictions_text_labels_frame.loc[(predictions_text_labels_frame['labels'] == 0) &\n",
    "                                  predictions_text_labels_frame['predictions'].isin([1, 2])].sort_values(by=['confidence'])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Z4A4RRwU55Gx"
   },
   "source": [
    "## 6. Model Explainer using Shap"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "3Cg-QjqH55Gy",
    "outputId": "85fd7643-f4c6-4b72-ac1e-80b611b1a6c9"
   },
   "outputs": [],
   "source": [
    "import shap\n",
    "\n",
    "# we use the first 1000 training examples as our background dataset to\n",
    "# integrate over\n",
    "explainer = shap.KernelExplainer(\n",
    "    single_channel_kim_model.predict, X_train_seq[:1000])\n",
    "# send the test data\n",
    "shap_values = explainer.shap_values(X_test_seq[:100], nsamples=100)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "92_48A7n55G0"
   },
   "source": [
    "## 6.1 Summary Plot of Feature Vectors "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "g-uCS8Lg55G0",
    "outputId": "34f66553-dcdc-4eb0-f11c-de4c64379409"
   },
   "outputs": [],
   "source": [
    "shap.summary_plot(shap_values, X_test_seq[:100])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "SRa_y_8B55G2"
   },
   "source": [
    "## 6.2 Visualization of Output and Features per class \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "KfKhpB2l55G2",
    "outputId": "c280ca5f-9f47-4c18-f151-d23a77ff2b1f"
   },
   "outputs": [],
   "source": [
    "# print the JS visualization code to the notebook\n",
    "shap.initjs()\n",
    "# visualize the negaitve tweets\n",
    "shap.force_plot(explainer.expected_value[0], shap_values[0], X_test_seq[:100])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "jSBrsvFj55G5",
    "outputId": "985c88da-28a0-4f76-d177-f42f4b7df3f8"
   },
   "outputs": [],
   "source": [
    "from lime.lime_text import LimeTextExplainer\n",
    "explainer = LimeTextExplainer(class_names=TEXT_LABELS)\n",
    "\n",
    "\n",
    "def new_predict(texts):\n",
    "    _seq = tk.texts_to_sequences(texts)\n",
    "    _text_data = pad_sequences(_seq, maxlen=MAX_LEN)\n",
    "    return single_channel_kim_model.predict(_text_data)\n",
    "\n",
    "\n",
    "exp = explainer.explain_instance(\n",
    "    'forget reservations thank great company i have cancelled flighted flight once again thank you',\n",
    "    new_predict,\n",
    "    num_features=7,\n",
    "    labels=[\n",
    "        0,\n",
    "        2])\n",
    "\n",
    "print('Explanation for class %s' % TEXT_LABELS[0])\n",
    "print('\\n'.join(map(str, exp.as_list(label=0))))\n",
    "print()\n",
    "print('Explanation for class %s' % TEXT_LABELS[2])\n",
    "print('\\n'.join(map(str, exp.as_list(label=2))))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "azqcVyqC55G7",
    "outputId": "764773c0-7d1e-4672-fbb5-d35f4731b7dc"
   },
   "outputs": [],
   "source": [
    "exp.show_in_notebook(text='forget reservations thank great company i have cancelled flighted flight once again thank you', labels=(0,))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "U7zX7SoR55G9"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "name": "Sentiment Analysis Classification.ipynb",
   "provenance": [],
   "version": "0.3.2"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}