TweetTopicModeling.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Read in data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "hilary = pd.read_csv('tweetsClintonAll.csv')\n",
    "trump = pd.read_csv('tweetsTrumpAll.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Make columns usable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "hilary.columns = [col.strip() for col in hilary.columns]\n",
    "trump.columns = [col.strip() for col in trump.columns]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Take a look"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ids</th>\n",
       "      <th>screen_name</th>\n",
       "      <th>followers</th>\n",
       "      <th>retweet</th>\n",
       "      <th>inreplyto</th>\n",
       "      <th>favorite</th>\n",
       "      <th>friends</th>\n",
       "      <th>listed</th>\n",
       "      <th>text</th>\n",
       "      <th>location</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>62958859</td>\n",
       "      <td>curish</td>\n",
       "      <td>368</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>1629</td>\n",
       "      <td>5</td>\n",
       "      <td>No privatization of SS; I've paid in, I want m...</td>\n",
       "      <td>Davenport, IA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>104615577</td>\n",
       "      <td>BudgetHawks</td>\n",
       "      <td>7033</td>\n",
       "      <td>2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>3906</td>\n",
       "      <td>343</td>\n",
       "      <td>.@timkaine is right. Trump would increase debt...</td>\n",
       "      <td>Washington, DC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>714985182315450368</td>\n",
       "      <td>Debbierg51</td>\n",
       "      <td>505</td>\n",
       "      <td>260</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>529</td>\n",
       "      <td>115</td>\n",
       "      <td>RT @Ian56789: Hillary Clinton's \"uninformed\" #...</td>\n",
       "      <td>Barrington, RI</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  ids  screen_name  followers  retweet inreplyto  favorite  \\\n",
       "0            62958859       curish        368        0       NaN         0   \n",
       "1           104615577  BudgetHawks       7033        2       NaN         0   \n",
       "2  714985182315450368   Debbierg51        505      260       NaN         0   \n",
       "\n",
       "   friends  listed                                               text  \\\n",
       "0     1629       5  No privatization of SS; I've paid in, I want m...   \n",
       "1     3906     343  .@timkaine is right. Trump would increase debt...   \n",
       "2      529     115  RT @Ian56789: Hillary Clinton's \"uninformed\" #...   \n",
       "\n",
       "         location  \n",
       "0   Davenport, IA  \n",
       "1  Washington, DC  \n",
       "2  Barrington, RI  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hilary.head(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "There are references to Trump in the Hilary tweets and visa-versa. Let's filter the tweets so that they reference just one of the candidates."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def hilary_filter(tweet):\n",
    "    tweet = tweet.lower()\n",
    "    if 'trump' in tweet or 'donald' in tweet:\n",
    "        return None\n",
    "    else:\n",
    "        return tweet\n",
    "    \n",
    "def trump_filter(tweet):\n",
    "    tweet = tweet.lower()\n",
    "    if 'hilary' in tweet or 'clinton' in tweet:\n",
    "        return None\n",
    "    else:\n",
    "        return tweet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "14089"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len([hilary_filter(tweet) for tweet in hilary.text if hilary_filter(tweet)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "17509"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len([trump_filter(tweet) for tweet in trump.text if trump_filter(tweet)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "hilary = hilary[[hilary_filter(tweet) is not None for tweet in hilary.text]].reset_index()\n",
    "trump = trump[[trump_filter(tweet) is not None for tweet in trump.text]].reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(14089, 17509)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(hilary),len(trump)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Lets do some sentiment stuff"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import sentlex\n",
    "import sentlex.sentanalysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "SWN = sentlex.SWN3Lexicon()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "classifier = sentlex.sentanalysis.BasicDocSentiScore()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'RT @Ian56789: Hillary Clinton\\'s \"uninformed\" #BasementDwellers know that Hillary has taken massive bribes from drugs companies to price gou\\xe2\\x80\\xa6'"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweet = hilary.text.values[0]\n",
    "tweet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.42202570820991875, 0.03623642439431913)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "classifier.classify_document(tweet, tagged=False, L=SWN, a=True, v=True, n=False, \n",
    "                             r=False, negation=False, verbose=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'annotated_doc': '',\n",
       " 'doc': 'RT @Ian56789: Hillary Clinton\\'s \"uninformed\" #BasementDwellers know that Hillary has taken massive bribes from drugs companies to price gou\\xe2\\x80\\xa6',\n",
       " 'found_list': Counter({'has/VBZ': 1,\n",
       "          'know/VBP': 1,\n",
       "          'massive/JJ': 1,\n",
       "          'taken/VBN': 1,\n",
       "          'uninformed/JJ': 1}),\n",
       " 'resultneg': 0.03623642439431913,\n",
       " 'resultpos': 0.42202570820991875,\n",
       " 'tokens_found': 5,\n",
       " 'tokens_negated': 0,\n",
       " 'unscored_list': []}"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "classifier.resultdata"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Didn't do so well. Let's try a different one."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from textblob import TextBlob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "blob = TextBlob(tweet.decode('ascii','ignore'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TextBlob(\"RT @Ian56789: Hillary Clinton's \"uninformed\" #BasementDwellers know that Hillary has taken massive bribes from drugs companies to price gou\")"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "blob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Sentiment(polarity=0.0, subjectivity=1.0)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "blob.sentiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from vaderSentiment.vaderSentiment import sentiment as vaderSentiment "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'compound': 0.0, 'neg': 0.0, 'neu': 1.0, 'pos': 0.0}"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vaderSentiment(tweet) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"Hillary Clinton's lawlessness and criminality in its enormity makes Nixon look like a rookie #VPDebate\""
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hilary.text.values[5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'compound': 0.3612, 'neg': 0.0, 'neu': 0.839, 'pos': 0.161}"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vaderSentiment(hilary.text.values[5])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Sentiment analysis is not working that well. Let's pivot to topic modeling."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.decomposition import LatentDirichletAllocation\n",
    "from sklearn.feature_extraction.text import CountVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "cv = CountVectorizer(stop_words='english')\n",
    "X_hil = cv.fit_transform(hilary.text)\n",
    "hil_features = sorted(cv.vocabulary_,key=cv.vocabulary_.get)\n",
    "X_trump = cv.fit_transform(trump.text)\n",
    "trump_features = sorted(cv.vocabulary_,key=cv.vocabulary_.get)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[u'00', u'000', u'00ps', u'0384giscak', u'06']"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hil_features[0:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((14089, 9083), (17509, 13869))"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_hil.shape,X_trump.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lda = LatentDirichletAllocation()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LatentDirichletAllocation(batch_size=128, doc_topic_prior=None,\n",
       "             evaluate_every=-1, learning_decay=0.7, learning_method=None,\n",
       "             learning_offset=10.0, max_doc_update_iter=100, max_iter=10,\n",
       "             mean_change_tol=0.001, n_jobs=1, n_topics=10, perp_tol=0.1,\n",
       "             random_state=None, topic_word_prior=None,\n",
       "             total_samples=1000000.0, verbose=0)"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lda.fit(X_hil)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "hil_topics = pd.DataFrame([np.array(hil_features)[lda.components_[top_ind,:].argsort()[::-1][0:10]] for top_ind in range(10)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>clinton</td>\n",
       "      <td>https</td>\n",
       "      <td>rt</td>\n",
       "      <td>hillary</td>\n",
       "      <td>like</td>\n",
       "      <td>dcexaminer</td>\n",
       "      <td>tonight</td>\n",
       "      <td>ve</td>\n",
       "      <td>look</td>\n",
       "      <td>days</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>https</td>\n",
       "      <td>clinton</td>\n",
       "      <td>rt</td>\n",
       "      <td>state</td>\n",
       "      <td>thing</td>\n",
       "      <td>obamacare</td>\n",
       "      <td>world</td>\n",
       "      <td>campaign</td>\n",
       "      <td>pence</td>\n",
       "      <td>hillary</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>rt</td>\n",
       "      <td>clinton</td>\n",
       "      <td>https</td>\n",
       "      <td>hillary</td>\n",
       "      <td>wikileaks</td>\n",
       "      <td>mike_pence</td>\n",
       "      <td>hillaryclinton</td>\n",
       "      <td>kaine</td>\n",
       "      <td>obamacare</td>\n",
       "      <td>assange</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>hillary</td>\n",
       "      <td>clinton</td>\n",
       "      <td>rt</td>\n",
       "      <td>https</td>\n",
       "      <td>release</td>\n",
       "      <td>judge</td>\n",
       "      <td>indictment</td>\n",
       "      <td>drafts</td>\n",
       "      <td>won</td>\n",
       "      <td>judgement</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>https</td>\n",
       "      <td>clinton</td>\n",
       "      <td>rt</td>\n",
       "      <td>clintons</td>\n",
       "      <td>hillary</td>\n",
       "      <td>live</td>\n",
       "      <td>haiti</td>\n",
       "      <td>refuses</td>\n",
       "      <td>iran</td>\n",
       "      <td>atlantic</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>https</td>\n",
       "      <td>clinton</td>\n",
       "      <td>rt</td>\n",
       "      <td>hillary</td>\n",
       "      <td>white</td>\n",
       "      <td>foundation</td>\n",
       "      <td>dept</td>\n",
       "      <td>leaked</td>\n",
       "      <td>interview</td>\n",
       "      <td>house</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>rt</td>\n",
       "      <td>hillary</td>\n",
       "      <td>clinton</td>\n",
       "      <td>kaine</td>\n",
       "      <td>polls</td>\n",
       "      <td>news</td>\n",
       "      <td>tim</td>\n",
       "      <td>care</td>\n",
       "      <td>dloesch</td>\n",
       "      <td>just</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>clinton</td>\n",
       "      <td>rt</td>\n",
       "      <td>amp</td>\n",
       "      <td>obama</td>\n",
       "      <td>people</td>\n",
       "      <td>hillary</td>\n",
       "      <td>https</td>\n",
       "      <td>pence</td>\n",
       "      <td>right</td>\n",
       "      <td>vpdebate</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>clinton</td>\n",
       "      <td>https</td>\n",
       "      <td>rt</td>\n",
       "      <td>new</td>\n",
       "      <td>foundation</td>\n",
       "      <td>water</td>\n",
       "      <td>hillary</td>\n",
       "      <td>shows</td>\n",
       "      <td>speeches</td>\n",
       "      <td>intersection</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>rt</td>\n",
       "      <td>hillary</td>\n",
       "      <td>clinton</td>\n",
       "      <td>https</td>\n",
       "      <td>did</td>\n",
       "      <td>president</td>\n",
       "      <td>video</td>\n",
       "      <td>really</td>\n",
       "      <td>crazy</td>\n",
       "      <td>benghazi</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         0        1        2         3           4           5  \\\n",
       "0  clinton    https       rt   hillary        like  dcexaminer   \n",
       "1    https  clinton       rt     state       thing   obamacare   \n",
       "2       rt  clinton    https   hillary   wikileaks  mike_pence   \n",
       "3  hillary  clinton       rt     https     release       judge   \n",
       "4    https  clinton       rt  clintons     hillary        live   \n",
       "5    https  clinton       rt   hillary       white  foundation   \n",
       "6       rt  hillary  clinton     kaine       polls        news   \n",
       "7  clinton       rt      amp     obama      people     hillary   \n",
       "8  clinton    https       rt       new  foundation       water   \n",
       "9       rt  hillary  clinton     https         did   president   \n",
       "\n",
       "                6         7          8             9  \n",
       "0         tonight        ve       look          days  \n",
       "1           world  campaign      pence       hillary  \n",
       "2  hillaryclinton     kaine  obamacare       assange  \n",
       "3      indictment    drafts        won     judgement  \n",
       "4           haiti   refuses       iran      atlantic  \n",
       "5            dept    leaked  interview         house  \n",
       "6             tim      care    dloesch          just  \n",
       "7           https     pence      right      vpdebate  \n",
       "8         hillary     shows   speeches  intersection  \n",
       "9           video    really      crazy      benghazi  "
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hil_topics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LatentDirichletAllocation(batch_size=128, doc_topic_prior=None,\n",
       "             evaluate_every=-1, learning_decay=0.7, learning_method=None,\n",
       "             learning_offset=10.0, max_doc_update_iter=100, max_iter=10,\n",
       "             mean_change_tol=0.001, n_jobs=1, n_topics=10, perp_tol=0.1,\n",
       "             random_state=None, topic_word_prior=None,\n",
       "             total_samples=1000000.0, verbose=0)"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lda.fit(X_trump)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "trump_topics = pd.DataFrame([np.array(trump_features)[lda.components_[top_ind,:].argsort()[::-1][0:10]] for top_ind in range(10)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>trump</td>\n",
       "      <td>rt</td>\n",
       "      <td>https</td>\n",
       "      <td>supporters</td>\n",
       "      <td>hillary</td>\n",
       "      <td>2016</td>\n",
       "      <td>mcuban</td>\n",
       "      <td>candidate</td>\n",
       "      <td>daddy</td>\n",
       "      <td>000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>trump</td>\n",
       "      <td>rt</td>\n",
       "      <td>https</td>\n",
       "      <td>pence</td>\n",
       "      <td>said</td>\n",
       "      <td>tax</td>\n",
       "      <td>did</td>\n",
       "      <td>say</td>\n",
       "      <td>realdonaldtrump</td>\n",
       "      <td>kaine</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>trump</td>\n",
       "      <td>rt</td>\n",
       "      <td>https</td>\n",
       "      <td>donald</td>\n",
       "      <td>election</td>\n",
       "      <td>good</td>\n",
       "      <td>american</td>\n",
       "      <td>running</td>\n",
       "      <td>obama</td>\n",
       "      <td>come</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>trump</td>\n",
       "      <td>rt</td>\n",
       "      <td>https</td>\n",
       "      <td>donald</td>\n",
       "      <td>ptsd</td>\n",
       "      <td>working</td>\n",
       "      <td>women</td>\n",
       "      <td>hrc</td>\n",
       "      <td>1st</td>\n",
       "      <td>hard</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>https</td>\n",
       "      <td>rt</td>\n",
       "      <td>trump</td>\n",
       "      <td>amp</td>\n",
       "      <td>state</td>\n",
       "      <td>rally</td>\n",
       "      <td>endorsement</td>\n",
       "      <td>thing</td>\n",
       "      <td>stop</td>\n",
       "      <td>white</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>trump</td>\n",
       "      <td>https</td>\n",
       "      <td>rt</td>\n",
       "      <td>cnn</td>\n",
       "      <td>twitter</td>\n",
       "      <td>nevada</td>\n",
       "      <td>gop</td>\n",
       "      <td>donald</td>\n",
       "      <td>http</td>\n",
       "      <td>racist</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>trump</td>\n",
       "      <td>rt</td>\n",
       "      <td>pence</td>\n",
       "      <td>donald</td>\n",
       "      <td>https</td>\n",
       "      <td>vpdebate</td>\n",
       "      <td>mike</td>\n",
       "      <td>kaine</td>\n",
       "      <td>said</td>\n",
       "      <td>like</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>trump</td>\n",
       "      <td>rt</td>\n",
       "      <td>https</td>\n",
       "      <td>hillary</td>\n",
       "      <td>new</td>\n",
       "      <td>comments</td>\n",
       "      <td>debate</td>\n",
       "      <td>hate</td>\n",
       "      <td>3rd</td>\n",
       "      <td>clowns</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>trump</td>\n",
       "      <td>https</td>\n",
       "      <td>rt</td>\n",
       "      <td>president</td>\n",
       "      <td>america</td>\n",
       "      <td>presidential</td>\n",
       "      <td>campaign</td>\n",
       "      <td>saying</td>\n",
       "      <td>anti</td>\n",
       "      <td>video</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>trump</td>\n",
       "      <td>rt</td>\n",
       "      <td>people</td>\n",
       "      <td>vote</td>\n",
       "      <td>https</td>\n",
       "      <td>donald</td>\n",
       "      <td>debate</td>\n",
       "      <td>won</td>\n",
       "      <td>going</td>\n",
       "      <td>just</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       0      1       2           3         4             5            6  \\\n",
       "0  trump     rt   https  supporters   hillary          2016       mcuban   \n",
       "1  trump     rt   https       pence      said           tax          did   \n",
       "2  trump     rt   https      donald  election          good     american   \n",
       "3  trump     rt   https      donald      ptsd       working        women   \n",
       "4  https     rt   trump         amp     state         rally  endorsement   \n",
       "5  trump  https      rt         cnn   twitter        nevada          gop   \n",
       "6  trump     rt   pence      donald     https      vpdebate         mike   \n",
       "7  trump     rt   https     hillary       new      comments       debate   \n",
       "8  trump  https      rt   president   america  presidential     campaign   \n",
       "9  trump     rt  people        vote     https        donald       debate   \n",
       "\n",
       "           7                8       9  \n",
       "0  candidate            daddy     000  \n",
       "1        say  realdonaldtrump   kaine  \n",
       "2    running            obama    come  \n",
       "3        hrc              1st    hard  \n",
       "4      thing             stop   white  \n",
       "5     donald             http  racist  \n",
       "6      kaine             said    like  \n",
       "7       hate              3rd  clowns  \n",
       "8     saying             anti   video  \n",
       "9        won            going    just  "
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trump_topics"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Lets try adding some relevant stop words."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.stop_words import ENGLISH_STOP_WORDS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "more_stops = ['https','http','rt','trump','hilary','clinton','donald','hillary']\n",
    "all_stops = list(ENGLISH_STOP_WORDS) + more_stops"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "cv = CountVectorizer(stop_words=all_stops)\n",
    "X_hil = cv.fit_transform(hilary.text)\n",
    "hil_features = sorted(cv.vocabulary_,key=cv.vocabulary_.get)\n",
    "X_trump = cv.fit_transform(trump.text)\n",
    "trump_features = sorted(cv.vocabulary_,key=cv.vocabulary_.get)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LatentDirichletAllocation(batch_size=128, doc_topic_prior=None,\n",
       "             evaluate_every=-1, learning_decay=0.7, learning_method=None,\n",
       "             learning_offset=10.0, max_doc_update_iter=100, max_iter=10,\n",
       "             mean_change_tol=0.001, n_jobs=1, n_topics=10, perp_tol=0.1,\n",
       "             random_state=None, topic_word_prior=None,\n",
       "             total_samples=1000000.0, verbose=0)"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lda.fit(X_hil)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "hil_topics = pd.DataFrame([np.array(hil_features)[lda.components_[top_ind,:].argsort()[::-1][0:10]] for top_ind in range(10)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>people</td>\n",
       "      <td>pence</td>\n",
       "      <td>kaine</td>\n",
       "      <td>clintons</td>\n",
       "      <td>debate</td>\n",
       "      <td>just</td>\n",
       "      <td>haiti</td>\n",
       "      <td>mike</td>\n",
       "      <td>drone</td>\n",
       "      <td>vpdebate</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>president</td>\n",
       "      <td>kaine</td>\n",
       "      <td>video</td>\n",
       "      <td>really</td>\n",
       "      <td>campaign</td>\n",
       "      <td>pence</td>\n",
       "      <td>crazy</td>\n",
       "      <td>obamacare</td>\n",
       "      <td>vpdebate</td>\n",
       "      <td>say</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>wikileaks</td>\n",
       "      <td>assange</td>\n",
       "      <td>plan</td>\n",
       "      <td>leaked</td>\n",
       "      <td>refuses</td>\n",
       "      <td>sick</td>\n",
       "      <td>julian</td>\n",
       "      <td>campaign</td>\n",
       "      <td>economy</td>\n",
       "      <td>memo</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>white</td>\n",
       "      <td>emails</td>\n",
       "      <td>dcexaminer</td>\n",
       "      <td>like</td>\n",
       "      <td>days</td>\n",
       "      <td>past</td>\n",
       "      <td>house</td>\n",
       "      <td>paid</td>\n",
       "      <td>early</td>\n",
       "      <td>gotten</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>vpdebate</td>\n",
       "      <td>state</td>\n",
       "      <td>kaine</td>\n",
       "      <td>benghazi</td>\n",
       "      <td>tonight</td>\n",
       "      <td>ve</td>\n",
       "      <td>look</td>\n",
       "      <td>dept</td>\n",
       "      <td>news</td>\n",
       "      <td>makes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>release</td>\n",
       "      <td>judge</td>\n",
       "      <td>foxandfriends</td>\n",
       "      <td>indictment</td>\n",
       "      <td>drafts</td>\n",
       "      <td>fbi</td>\n",
       "      <td>doj</td>\n",
       "      <td>water</td>\n",
       "      <td>foreign</td>\n",
       "      <td>hq6jl2szm9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>did</td>\n",
       "      <td>foundation</td>\n",
       "      <td>new</td>\n",
       "      <td>state</td>\n",
       "      <td>polls</td>\n",
       "      <td>foxnews</td>\n",
       "      <td>strong</td>\n",
       "      <td>watch</td>\n",
       "      <td>natesilver538</td>\n",
       "      <td>shows</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>obamacare</td>\n",
       "      <td>kaine</td>\n",
       "      <td>mike_pence</td>\n",
       "      <td>thing</td>\n",
       "      <td>world</td>\n",
       "      <td>craziest</td>\n",
       "      <td>vpdebate</td>\n",
       "      <td>lying</td>\n",
       "      <td>live</td>\n",
       "      <td>tim</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>hillaryclinton</td>\n",
       "      <td>foundation</td>\n",
       "      <td>truth</td>\n",
       "      <td>email</td>\n",
       "      <td>server</td>\n",
       "      <td>amp</td>\n",
       "      <td>ingrahamangle</td>\n",
       "      <td>won</td>\n",
       "      <td>prisonplanet</td>\n",
       "      <td>june</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>obama</td>\n",
       "      <td>amp</td>\n",
       "      <td>calling</td>\n",
       "      <td>right</td>\n",
       "      <td>pence</td>\n",
       "      <td>failed</td>\n",
       "      <td>tried</td>\n",
       "      <td>admin</td>\n",
       "      <td>bush</td>\n",
       "      <td>iraq</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                0           1              2           3         4         5  \\\n",
       "0          people       pence          kaine    clintons    debate      just   \n",
       "1       president       kaine          video      really  campaign     pence   \n",
       "2       wikileaks     assange           plan      leaked   refuses      sick   \n",
       "3           white      emails     dcexaminer        like      days      past   \n",
       "4        vpdebate       state          kaine    benghazi   tonight        ve   \n",
       "5         release       judge  foxandfriends  indictment    drafts       fbi   \n",
       "6             did  foundation            new       state     polls   foxnews   \n",
       "7       obamacare       kaine     mike_pence       thing     world  craziest   \n",
       "8  hillaryclinton  foundation          truth       email    server       amp   \n",
       "9           obama         amp        calling       right     pence    failed   \n",
       "\n",
       "               6          7              8           9  \n",
       "0          haiti       mike          drone    vpdebate  \n",
       "1          crazy  obamacare       vpdebate         say  \n",
       "2         julian   campaign        economy        memo  \n",
       "3          house       paid          early      gotten  \n",
       "4           look       dept           news       makes  \n",
       "5            doj      water        foreign  hq6jl2szm9  \n",
       "6         strong      watch  natesilver538       shows  \n",
       "7       vpdebate      lying           live         tim  \n",
       "8  ingrahamangle        won   prisonplanet        june  \n",
       "9          tried      admin           bush        iraq  "
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hil_topics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LatentDirichletAllocation(batch_size=128, doc_topic_prior=None,\n",
       "             evaluate_every=-1, learning_decay=0.7, learning_method=None,\n",
       "             learning_offset=10.0, max_doc_update_iter=100, max_iter=10,\n",
       "             mean_change_tol=0.001, n_jobs=1, n_topics=10, perp_tol=0.1,\n",
       "             random_state=None, topic_word_prior=None,\n",
       "             total_samples=1000000.0, verbose=0)"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lda.fit(X_trump)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lda.n_iter_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "trump_topics = pd.DataFrame([np.array(trump_features)[lda.components_[top_ind,:].argsort()[::-1][0:10]] for top_ind in range(10)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>twitter</td>\n",
       "      <td>supporter</td>\n",
       "      <td>daddy</td>\n",
       "      <td>hannity</td>\n",
       "      <td>mcuban</td>\n",
       "      <td>kelly</td>\n",
       "      <td>model</td>\n",
       "      <td>hate</td>\n",
       "      <td>3rd</td>\n",
       "      <td>nevada</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>make</td>\n",
       "      <td>election</td>\n",
       "      <td>women</td>\n",
       "      <td>money</td>\n",
       "      <td>time</td>\n",
       "      <td>policy</td>\n",
       "      <td>female</td>\n",
       "      <td>kylegriffin1</td>\n",
       "      <td>like</td>\n",
       "      <td>need</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>pence</td>\n",
       "      <td>said</td>\n",
       "      <td>did</td>\n",
       "      <td>people</td>\n",
       "      <td>debate</td>\n",
       "      <td>say</td>\n",
       "      <td>mike</td>\n",
       "      <td>new</td>\n",
       "      <td>hrc</td>\n",
       "      <td>gop</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>tax</td>\n",
       "      <td>president</td>\n",
       "      <td>state</td>\n",
       "      <td>presidential</td>\n",
       "      <td>good</td>\n",
       "      <td>rally</td>\n",
       "      <td>amp</td>\n",
       "      <td>thing</td>\n",
       "      <td>stop</td>\n",
       "      <td>white</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>pence</td>\n",
       "      <td>says</td>\n",
       "      <td>vote</td>\n",
       "      <td>vpdebate</td>\n",
       "      <td>won</td>\n",
       "      <td>kaine</td>\n",
       "      <td>cnn</td>\n",
       "      <td>think</td>\n",
       "      <td>right</td>\n",
       "      <td>mike</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>realdonaldtrump</td>\n",
       "      <td>don</td>\n",
       "      <td>media</td>\n",
       "      <td>endorsement</td>\n",
       "      <td>voting</td>\n",
       "      <td>great</td>\n",
       "      <td>today</td>\n",
       "      <td>asked</td>\n",
       "      <td>age</td>\n",
       "      <td>clowns</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>pence</td>\n",
       "      <td>running</td>\n",
       "      <td>campaign</td>\n",
       "      <td>vpdebate</td>\n",
       "      <td>kaine</td>\n",
       "      <td>said</td>\n",
       "      <td>saying</td>\n",
       "      <td>people</td>\n",
       "      <td>like</td>\n",
       "      <td>bad</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>just</td>\n",
       "      <td>know</td>\n",
       "      <td>russia</td>\n",
       "      <td>pence</td>\n",
       "      <td>nonsense</td>\n",
       "      <td>genius</td>\n",
       "      <td>oh</td>\n",
       "      <td>called</td>\n",
       "      <td>got</td>\n",
       "      <td>000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>pence</td>\n",
       "      <td>america</td>\n",
       "      <td>said</td>\n",
       "      <td>year</td>\n",
       "      <td>going</td>\n",
       "      <td>mike_pence</td>\n",
       "      <td>vp</td>\n",
       "      <td>talk</td>\n",
       "      <td>debate</td>\n",
       "      <td>just</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>taxes</td>\n",
       "      <td>amp</td>\n",
       "      <td>ptsd</td>\n",
       "      <td>pay</td>\n",
       "      <td>really</td>\n",
       "      <td>does</td>\n",
       "      <td>2016</td>\n",
       "      <td>candidate</td>\n",
       "      <td>support</td>\n",
       "      <td>years</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 0          1         2             3         4           5  \\\n",
       "0          twitter  supporter     daddy       hannity    mcuban       kelly   \n",
       "1             make   election     women         money      time      policy   \n",
       "2            pence       said       did        people    debate         say   \n",
       "3              tax  president     state  presidential      good       rally   \n",
       "4            pence       says      vote      vpdebate       won       kaine   \n",
       "5  realdonaldtrump        don     media   endorsement    voting       great   \n",
       "6            pence    running  campaign      vpdebate     kaine        said   \n",
       "7             just       know    russia         pence  nonsense      genius   \n",
       "8            pence    america      said          year     going  mike_pence   \n",
       "9            taxes        amp      ptsd           pay    really        does   \n",
       "\n",
       "        6             7        8       9  \n",
       "0   model          hate      3rd  nevada  \n",
       "1  female  kylegriffin1     like    need  \n",
       "2    mike           new      hrc     gop  \n",
       "3     amp         thing     stop   white  \n",
       "4     cnn         think    right    mike  \n",
       "5   today         asked      age  clowns  \n",
       "6  saying        people     like     bad  \n",
       "7      oh        called      got     000  \n",
       "8      vp          talk   debate    just  \n",
       "9    2016     candidate  support   years  "
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trump_topics"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Still not great, let's see try some more topics and more iterations."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lda = LatentDirichletAllocation(n_topics=15,max_iter=50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LatentDirichletAllocation(batch_size=128, doc_topic_prior=None,\n",
       "             evaluate_every=-1, learning_decay=0.7, learning_method=None,\n",
       "             learning_offset=10.0, max_doc_update_iter=100, max_iter=50,\n",
       "             mean_change_tol=0.001, n_jobs=1, n_topics=15, perp_tol=0.1,\n",
       "             random_state=None, topic_word_prior=None,\n",
       "             total_samples=1000000.0, verbose=0)"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lda.fit(X_hil)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "hil_topics = pd.DataFrame([np.array(hil_features)[lda.components_[top_ind,:].argsort()[::-1][0:10]] for top_ind in range(10)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>obamacare</td>\n",
       "      <td>did</td>\n",
       "      <td>really</td>\n",
       "      <td>clintons</td>\n",
       "      <td>crazy</td>\n",
       "      <td>lying</td>\n",
       "      <td>kaine</td>\n",
       "      <td>live</td>\n",
       "      <td>foxandfriends</td>\n",
       "      <td>deal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>obama</td>\n",
       "      <td>amp</td>\n",
       "      <td>right</td>\n",
       "      <td>failed</td>\n",
       "      <td>pence</td>\n",
       "      <td>tried</td>\n",
       "      <td>admin</td>\n",
       "      <td>bush</td>\n",
       "      <td>iraq</td>\n",
       "      <td>extension</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>wikileaks</td>\n",
       "      <td>assange</td>\n",
       "      <td>refuses</td>\n",
       "      <td>doj</td>\n",
       "      <td>julian</td>\n",
       "      <td>prisonplanet</td>\n",
       "      <td>campaign</td>\n",
       "      <td>dealer</td>\n",
       "      <td>arms</td>\n",
       "      <td>breaking</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>release</td>\n",
       "      <td>white</td>\n",
       "      <td>judge</td>\n",
       "      <td>indictment</td>\n",
       "      <td>tonight</td>\n",
       "      <td>drafts</td>\n",
       "      <td>haiti</td>\n",
       "      <td>won</td>\n",
       "      <td>ve</td>\n",
       "      <td>truth</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>obamacare</td>\n",
       "      <td>thing</td>\n",
       "      <td>world</td>\n",
       "      <td>craziest</td>\n",
       "      <td>just</td>\n",
       "      <td>paid</td>\n",
       "      <td>calls</td>\n",
       "      <td>rally</td>\n",
       "      <td>questions</td>\n",
       "      <td>memo</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>past</td>\n",
       "      <td>news</td>\n",
       "      <td>security</td>\n",
       "      <td>moderator</td>\n",
       "      <td>evidence</td>\n",
       "      <td>kaine</td>\n",
       "      <td>families</td>\n",
       "      <td>dems</td>\n",
       "      <td>social</td>\n",
       "      <td>kylegriffin1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>emails</td>\n",
       "      <td>dcexaminer</td>\n",
       "      <td>house</td>\n",
       "      <td>dloesch</td>\n",
       "      <td>care</td>\n",
       "      <td>tax</td>\n",
       "      <td>endorsed</td>\n",
       "      <td>health</td>\n",
       "      <td>hannity</td>\n",
       "      <td>bernie</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>mike_pence</td>\n",
       "      <td>plan</td>\n",
       "      <td>economy</td>\n",
       "      <td>economic</td>\n",
       "      <td>send</td>\n",
       "      <td>bigleaguetruth</td>\n",
       "      <td>women</td>\n",
       "      <td>tailspin</td>\n",
       "      <td>wgvrzdfefz</td>\n",
       "      <td>msnbc</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>kaine</td>\n",
       "      <td>vpdebate</td>\n",
       "      <td>tim</td>\n",
       "      <td>president</td>\n",
       "      <td>debate</td>\n",
       "      <td>benghazi</td>\n",
       "      <td>make</td>\n",
       "      <td>judgement</td>\n",
       "      <td>going</td>\n",
       "      <td>americans</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>like</td>\n",
       "      <td>kaine</td>\n",
       "      <td>vpdebate</td>\n",
       "      <td>water</td>\n",
       "      <td>foxnews</td>\n",
       "      <td>look</td>\n",
       "      <td>abt</td>\n",
       "      <td>hillaryclinton</td>\n",
       "      <td>america</td>\n",
       "      <td>says</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            0           1         2           3         4               5  \\\n",
       "0   obamacare         did    really    clintons     crazy           lying   \n",
       "1       obama         amp     right      failed     pence           tried   \n",
       "2   wikileaks     assange   refuses         doj    julian    prisonplanet   \n",
       "3     release       white     judge  indictment   tonight          drafts   \n",
       "4   obamacare       thing     world    craziest      just            paid   \n",
       "5        past        news  security   moderator  evidence           kaine   \n",
       "6      emails  dcexaminer     house     dloesch      care             tax   \n",
       "7  mike_pence        plan   economy    economic      send  bigleaguetruth   \n",
       "8       kaine    vpdebate       tim   president    debate        benghazi   \n",
       "9        like       kaine  vpdebate       water   foxnews            look   \n",
       "\n",
       "          6               7              8             9  \n",
       "0     kaine            live  foxandfriends          deal  \n",
       "1     admin            bush           iraq     extension  \n",
       "2  campaign          dealer           arms      breaking  \n",
       "3     haiti             won             ve         truth  \n",
       "4     calls           rally      questions          memo  \n",
       "5  families            dems         social  kylegriffin1  \n",
       "6  endorsed          health        hannity        bernie  \n",
       "7     women        tailspin     wgvrzdfefz         msnbc  \n",
       "8      make       judgement          going     americans  \n",
       "9       abt  hillaryclinton        america          says  "
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hil_topics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LatentDirichletAllocation(batch_size=128, doc_topic_prior=None,\n",
       "             evaluate_every=-1, learning_decay=0.7, learning_method=None,\n",
       "             learning_offset=10.0, max_doc_update_iter=100, max_iter=50,\n",
       "             mean_change_tol=0.001, n_jobs=1, n_topics=15, perp_tol=0.1,\n",
       "             random_state=None, topic_word_prior=None,\n",
       "             total_samples=1000000.0, verbose=0)"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lda.fit(X_trump)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "trump_topics = pd.DataFrame([np.array(trump_features)[lda.components_[top_ind,:].argsort()[::-1][0:10]] for top_ind in range(15)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>rally</td>\n",
       "      <td>stop</td>\n",
       "      <td>hannity</td>\n",
       "      <td>today</td>\n",
       "      <td>white</td>\n",
       "      <td>kelly</td>\n",
       "      <td>mcuban</td>\n",
       "      <td>day</td>\n",
       "      <td>clowns</td>\n",
       "      <td>3rd</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>pence</td>\n",
       "      <td>said</td>\n",
       "      <td>like</td>\n",
       "      <td>kaine</td>\n",
       "      <td>mike</td>\n",
       "      <td>vpdebate</td>\n",
       "      <td>did</td>\n",
       "      <td>say</td>\n",
       "      <td>things</td>\n",
       "      <td>supporters</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>think</td>\n",
       "      <td>russia</td>\n",
       "      <td>nonsense</td>\n",
       "      <td>oh</td>\n",
       "      <td>know</td>\n",
       "      <td>time</td>\n",
       "      <td>claims</td>\n",
       "      <td>coming</td>\n",
       "      <td>china</td>\n",
       "      <td>false</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>hate</td>\n",
       "      <td>years</td>\n",
       "      <td>just</td>\n",
       "      <td>polls</td>\n",
       "      <td>pence</td>\n",
       "      <td>denied</td>\n",
       "      <td>ag</td>\n",
       "      <td>called</td>\n",
       "      <td>mike</td>\n",
       "      <td>insults</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>amp</td>\n",
       "      <td>talk</td>\n",
       "      <td>model</td>\n",
       "      <td>good</td>\n",
       "      <td>enforcement</td>\n",
       "      <td>customs</td>\n",
       "      <td>caught</td>\n",
       "      <td>maliagif</td>\n",
       "      <td>best</td>\n",
       "      <td>officers</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>tax</td>\n",
       "      <td>ptsd</td>\n",
       "      <td>just</td>\n",
       "      <td>realdonaldtrump</td>\n",
       "      <td>media</td>\n",
       "      <td>gop</td>\n",
       "      <td>video</td>\n",
       "      <td>way</td>\n",
       "      <td>fox</td>\n",
       "      <td>racist</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>campaign</td>\n",
       "      <td>women</td>\n",
       "      <td>comments</td>\n",
       "      <td>female</td>\n",
       "      <td>insult</td>\n",
       "      <td>nation</td>\n",
       "      <td>endorsement</td>\n",
       "      <td>crowd</td>\n",
       "      <td>conway</td>\n",
       "      <td>role</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>year</td>\n",
       "      <td>really</td>\n",
       "      <td>election</td>\n",
       "      <td>business</td>\n",
       "      <td>running</td>\n",
       "      <td>does</td>\n",
       "      <td>win</td>\n",
       "      <td>old</td>\n",
       "      <td>lying</td>\n",
       "      <td>pence</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>pence</td>\n",
       "      <td>president</td>\n",
       "      <td>putin</td>\n",
       "      <td>great</td>\n",
       "      <td>don</td>\n",
       "      <td>vpdebate</td>\n",
       "      <td>mike_pence</td>\n",
       "      <td>presidential</td>\n",
       "      <td>state</td>\n",
       "      <td>money</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>says</td>\n",
       "      <td>lost</td>\n",
       "      <td>ll</td>\n",
       "      <td>kylegriffin1</td>\n",
       "      <td>got</td>\n",
       "      <td>story</td>\n",
       "      <td>accountant</td>\n",
       "      <td>atlantic</td>\n",
       "      <td>work</td>\n",
       "      <td>remember</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>vote</td>\n",
       "      <td>going</td>\n",
       "      <td>make</td>\n",
       "      <td>realdonaldtrump</td>\n",
       "      <td>tax</td>\n",
       "      <td>bad</td>\n",
       "      <td>supporter</td>\n",
       "      <td>fact</td>\n",
       "      <td>voting</td>\n",
       "      <td>look</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>people</td>\n",
       "      <td>pence</td>\n",
       "      <td>doesn</td>\n",
       "      <td>kaine</td>\n",
       "      <td>vpdebate</td>\n",
       "      <td>right</td>\n",
       "      <td>time</td>\n",
       "      <td>wrong</td>\n",
       "      <td>new</td>\n",
       "      <td>know</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>won</td>\n",
       "      <td>taxes</td>\n",
       "      <td>pay</td>\n",
       "      <td>country</td>\n",
       "      <td>age</td>\n",
       "      <td>foundation</td>\n",
       "      <td>believe</td>\n",
       "      <td>base</td>\n",
       "      <td>love</td>\n",
       "      <td>assange</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>cnn</td>\n",
       "      <td>america</td>\n",
       "      <td>twitter</td>\n",
       "      <td>anti</td>\n",
       "      <td>nuclear</td>\n",
       "      <td>weapons</td>\n",
       "      <td>war</td>\n",
       "      <td>ad</td>\n",
       "      <td>ricky_vaughn99</td>\n",
       "      <td>thedailybeast</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>debate</td>\n",
       "      <td>working</td>\n",
       "      <td>hrc</td>\n",
       "      <td>1st</td>\n",
       "      <td>hard</td>\n",
       "      <td>maga</td>\n",
       "      <td>nevada</td>\n",
       "      <td>vp</td>\n",
       "      <td>come</td>\n",
       "      <td>ananavarro</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           0          1         2                3            4           5  \\\n",
       "0      rally       stop   hannity            today        white       kelly   \n",
       "1      pence       said      like            kaine         mike    vpdebate   \n",
       "2      think     russia  nonsense               oh         know        time   \n",
       "3       hate      years      just            polls        pence      denied   \n",
       "4        amp       talk     model             good  enforcement     customs   \n",
       "5        tax       ptsd      just  realdonaldtrump        media         gop   \n",
       "6   campaign      women  comments           female       insult      nation   \n",
       "7       year     really  election         business      running        does   \n",
       "8      pence  president     putin            great          don    vpdebate   \n",
       "9       says       lost        ll     kylegriffin1          got       story   \n",
       "10      vote      going      make  realdonaldtrump          tax         bad   \n",
       "11    people      pence     doesn            kaine     vpdebate       right   \n",
       "12       won      taxes       pay          country          age  foundation   \n",
       "13       cnn    america   twitter             anti      nuclear     weapons   \n",
       "14    debate    working       hrc              1st         hard        maga   \n",
       "\n",
       "              6             7               8              9  \n",
       "0        mcuban           day          clowns            3rd  \n",
       "1           did           say          things     supporters  \n",
       "2        claims        coming           china          false  \n",
       "3            ag        called            mike        insults  \n",
       "4        caught      maliagif            best       officers  \n",
       "5         video           way             fox         racist  \n",
       "6   endorsement         crowd          conway           role  \n",
       "7           win           old           lying          pence  \n",
       "8    mike_pence  presidential           state          money  \n",
       "9    accountant      atlantic            work       remember  \n",
       "10    supporter          fact          voting           look  \n",
       "11         time         wrong             new           know  \n",
       "12      believe          base            love        assange  \n",
       "13          war            ad  ricky_vaughn99  thedailybeast  \n",
       "14       nevada            vp            come     ananavarro  "
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trump_topics"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}