utils.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "import string"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.corpus import stopwords\n",
    "from nltk.stem import PorterStemmer\n",
    "from nltk.tokenize import TweetTokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from matplotlib.patches import Ellipse\n",
    "import matplotlib.transforms as transforms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def process_tweet(tweet):\n",
    "    '''\n",
    "    Input:\n",
    "        tweet: a string containing a tweet\n",
    "    Output:\n",
    "        tweets_clean: a list of words containing the processed tweet\n",
    "\n",
    "    '''\n",
    "    stemmer = PorterStemmer()\n",
    "    stopwords_english = stopwords.words('english')\n",
    "    # remove stock market tickers like $GE\n",
    "    tweet = re.sub(r'\\$\\w*', '', tweet)\n",
    "    # remove old style retweet text \"RT\"\n",
    "    tweet = re.sub(r'^RT[\\s]+', '', tweet)\n",
    "    # remove hyperlinks\n",
    "    #tweet = re.sub(r'https?:\\/\\/.*[\\r\\n]*', '', tweet)\n",
    "    tweet = re.sub(r'https?://[^\\s\\n\\r]+', '', tweet)\n",
    "    # remove hashtags\n",
    "    # only removing the hash # sign from the word\n",
    "    tweet = re.sub(r'#', '', tweet)\n",
    "    # tokenize tweets\n",
    "    tokenizer = TweetTokenizer(preserve_case=False, strip_handles=True,\n",
    "                               reduce_len=True)\n",
    "    tweet_tokens = tokenizer.tokenize(tweet)\n",
    "\n",
    "    tweets_clean = []\n",
    "    for word in tweet_tokens:\n",
    "        if (word not in stopwords_english and  # remove stopwords\n",
    "                word not in string.punctuation):  # remove punctuation\n",
    "            # tweets_clean.append(word)\n",
    "            stem_word = stemmer.stem(word)  # stemming word\n",
    "            tweets_clean.append(stem_word)\n",
    "\n",
    "    return tweets_clean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test_lookup(func):\n",
    "    freqs = {('sad', 0): 4,\n",
    "             ('happy', 1): 12,\n",
    "             ('oppressed', 0): 7}\n",
    "    word = 'happy'\n",
    "    label = 1\n",
    "    if func(freqs, word, label) == 12:\n",
    "        return 'SUCCESS!!'\n",
    "    return 'Failed Sanity Check!'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def lookup(freqs, word, label):\n",
    "    '''\n",
    "    Input:\n",
    "        freqs: a dictionary with the frequency of each pair (or tuple)\n",
    "        word: the word to look up\n",
    "        label: the label corresponding to the word\n",
    "    Output:\n",
    "        n: the number of times the word with its corresponding label appears.\n",
    "    '''\n",
    "    n = 0  # freqs.get((word, label), 0)\n",
    "\n",
    "    pair = (word, label)\n",
    "    if (pair in freqs):\n",
    "        n = freqs[pair]\n",
    "\n",
    "    return n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "lines_to_next_cell": 2
   },
   "source": [
    "From: https://matplotlib.org/3.1.1/gallery/statistics/confidence_ellipse.html#sphx-glr-gallery-statistics-confidence-ellipse-py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def confidence_ellipse(x, y, ax, n_std=3.0, facecolor='none', **kwargs):\n",
    "    \"\"\"\n",
    "    Create a plot of the covariance confidence ellipse of `x` and `y`\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    x, y : array_like, shape (n, )\n",
    "        Input data.\n",
    "\n",
    "    ax : matplotlib.axes.Axes\n",
    "        The axes object to draw the ellipse into.\n",
    "\n",
    "    n_std : float\n",
    "        The number of standard deviations to determine the ellipse's radiuses.\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    matplotlib.patches.Ellipse\n",
    "\n",
    "    Other parameters\n",
    "    ----------------\n",
    "    kwargs : `~matplotlib.patches.Patch` properties\n",
    "    \"\"\"\n",
    "    if x.size != y.size:\n",
    "        raise ValueError(\"x and y must be the same size\")\n",
    "\n",
    "    cov = np.cov(x, y)\n",
    "    pearson = cov[0, 1] / np.sqrt(cov[0, 0] * cov[1, 1])\n",
    "    # Using a special case to obtain the eigenvalues of this\n",
    "    # two-dimensionl dataset.\n",
    "    ell_radius_x = np.sqrt(1 + pearson)\n",
    "    ell_radius_y = np.sqrt(1 - pearson)\n",
    "    ellipse = Ellipse((0, 0),\n",
    "                      width=ell_radius_x * 2,\n",
    "                      height=ell_radius_y * 2,\n",
    "                      facecolor=facecolor,\n",
    "                      **kwargs)\n",
    "\n",
    "    # Calculating the stdandard deviation of x from\n",
    "    # the squareroot of the variance and multiplying\n",
    "    # with the given number of standard deviations.\n",
    "    scale_x = np.sqrt(cov[0, 0]) * n_std\n",
    "    mean_x = np.mean(x)\n",
    "\n",
    "    # calculating the stdandard deviation of y ...\n",
    "    scale_y = np.sqrt(cov[1, 1]) * n_std\n",
    "    mean_y = np.mean(y)\n",
    "\n",
    "    transf = transforms.Affine2D() \\\n",
    "        .rotate_deg(45) \\\n",
    "        .scale(scale_x, scale_y) \\\n",
    "        .translate(mean_x, mean_y)\n",
    "\n",
    "    ellipse.set_transform(transf + ax.transData)\n",
    "    return ax.add_patch(ellipse)"
   ]
  }
 ],
 "metadata": {
  "jupytext": {
   "cell_metadata_filter": "-all",
   "notebook_metadata_filter": "-all",
   "text_representation": {
    "extension": ".py",
    "format_name": "light"
   }
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}