SumLSA (4).ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "Creating LSA-based summization "
      ],
      "metadata": {
        "id": "W6GTPlJzrn8q"
      }
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gnz5Yoc-IRxc",
        "outputId": "d209937c-95c3-4b0b-8057-05ab27f253ba"
      },
      "source": [
        "#import libraries \n",
        "import os.path\n",
        "import nltk\n",
        "from gensim import corpora\n",
        "from gensim.models import LsiModel\n",
        "from nltk.tokenize import RegexpTokenizer\n",
        "from nltk.corpus import stopwords\n",
        "from nltk.stem.porter import PorterStemmer\n",
        "nltk.download('stopwords')\n",
        "nltk.download('punkt')\n",
        "from gensim.models.coherencemodel import CoherenceModel\n",
        "import matplotlib.pyplot as plt\n",
        "import pandas as pd"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Package stopwords is already up-to-date!\n",
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Package punkt is already up-to-date!\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Eag_tXPrtvXv"
      },
      "source": [
        "class Summarization():\n",
        "  def preprocess_data(doc_set):\n",
        "    \"\"\"Input  : document list\n",
        "    Purpose: preprocess text (tokenize, removing stopwords, and stemming)\n",
        "    Output : preprocessed text \"\"\"    \n",
        "    # initialize regex tokenizer\n",
        "    tokenizer = RegexpTokenizer(r'\\w+')\n",
        "    # create English stop words list\n",
        "    en_stop = set(stopwords.words('english'))\n",
        "    # Create p_stemmer of class PorterStemmer\n",
        "    p_stemmer = PorterStemmer()\n",
        "    # list for tokenized documents in loop\n",
        "    texts = []\n",
        "    # loop through document list\n",
        "    for i in doc_set:\n",
        "        # clean and tokenize document string\n",
        "        raw = i.lower()\n",
        "        tokens = tokenizer.tokenize(raw)\n",
        "        # remove stop words from tokens\n",
        "        stopped_tokens = [i for i in tokens if not i in en_stop]\n",
        "        # stem tokens\n",
        "        stemmed_tokens = [p_stemmer.stem(i) for i in stopped_tokens]\n",
        "        # add tokens to list\n",
        "        texts.append(stemmed_tokens)\n",
        "    return texts\n",
        "\n",
        "\n",
        "  def prepare_corpus(doc_clean):\n",
        "    \"\"\" Input  : clean document\n",
        "    Purpose: create term dictionary of our courpus and Converting list of documents (corpus) into Document Term Matrix\n",
        "    Output : term dictionary and Document Term Matrix \"\"\"\n",
        "    dictionary = corpora.Dictionary(doc_clean) \n",
        "    doc_term_matrix = [dictionary.doc2bow(doc) for doc in doc_clean] \n",
        "    return dictionary,doc_term_matrix\n",
        "\n",
        "\n",
        "  def create_gensim_lsa_model(doc_clean,number_of_topics,words):\n",
        "    \"\"\" Input  : clean document, number of topics and number of words associated with each topic\n",
        "    Purpose: create LSA model using gensim\n",
        "    Output : return LSA model \"\"\"\n",
        "    dictionary,doc_term_matrix=Summarization.prepare_corpus(doc_clean)\n",
        "    # generate LSA model\n",
        "    lsamodel = LsiModel(doc_term_matrix, num_topics=number_of_topics, id2word = dictionary)  #train model #lsi module Implements fast truncated SVD.\n",
        "    print(lsamodel.print_topics(num_topics=number_of_topics, num_words=words))\n",
        "    return lsamodel\n",
        "\n",
        "\n",
        "  def compute_coherence_values(dictionary, doc_term_matrix, doc_clean, stop, start=2, step=3):\n",
        "    \"\"\" Input   : dictionary : Gensim dictionary\n",
        "              corpus : Gensim corpus\n",
        "              texts : List of input texts\n",
        "              stop : Max num of topics\n",
        "    purpose : Compute c_v coherence for various number of topics\n",
        "    Output  : model_list : List of LSA topic models coherence_values : Coherence values corresponding to the LDA model with respective number of topics  \"\"\"\n",
        "    \n",
        "    coherence_values = []\n",
        "    model_list = []\n",
        "    for num_topics in range(start, stop, step):\n",
        "      # generate LSA model\n",
        "      model = LsiModel(doc_term_matrix, num_topics=number_of_topics, id2word = dictionary)  # train model\n",
        "      # id2word (dict of (int, object)) – Mapping id -> word.\n",
        "      model_list.append(model)\n",
        "      coherencemodel = CoherenceModel(model=model, texts=doc_clean, dictionary=dictionary, coherence='c_v')\n",
        "      coherence_values.append(coherencemodel.get_coherence())\n",
        "    return model_list, coherence_values\n",
        "\n",
        "\n",
        "  def plot_graph(doc_clean,start, stop, step):\n",
        "    dictionary,doc_term_matrix=Summarization.prepare_corpus(doc_clean)\n",
        "    model_list, coherence_values = Summarization.compute_coherence_values(dictionary, doc_term_matrix,doc_clean, stop, start, step)\n",
        "    # Show graph\n",
        "    x = range(start, stop, step)\n",
        "    plt.plot(x, coherence_values)\n",
        "    plt.xlabel(\"Number of Topics\")\n",
        "    plt.ylabel(\"Coherence score\")\n",
        "    plt.legend((\"coherence_values\"), loc='best')\n",
        "    plt.show()\n",
        "\n",
        " \n",
        "  def takenext(elem):\n",
        "   \"\"\" sort \"\"\"\n",
        "   return elem[1] \n",
        "\n",
        "\n",
        "  def selectTopSent(summSize, numTopics, sortedVec):\n",
        "   topSentences = []\n",
        "   sent_no = []\n",
        "   sentInd = set()\n",
        "   sCount = 0\n",
        "   for i in range(summSize):\n",
        "    for j in range(numTopics):\n",
        "     vecs = sortedVec[j]\n",
        "     si = vecs[i][0]\n",
        "    if si not in sentInd:\n",
        "        sent_no.append(si)\n",
        "        topSentences.append(vecs[i])\n",
        "        sentInd.add(si)\n",
        "        sCount +=1\n",
        "        if sCount == summSize:\n",
        "          return sent_no  "
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}