Final_3_0_UTIL_COSC_74_274_Winter23_ShyamReddy.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Stop words training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "import seaborn as sns\n",
    "import pandas as pd\n",
    "import autograd.numpy as np\n",
    "from autograd import grad\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import data\n",
    "train_data = pd.read_csv('Training.csv')\n",
    "\n",
    "train_data.describe(include='all')\n",
    "\n",
    "# creating y labels less than cutoff to be 0 and 1 otherwise\n",
    "y = train_data['overall'].apply(lambda x: 0 if x <= cutoff else 1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "\n",
    "# Preprocessing data\n",
    "\n",
    "GENERIC_STOP_WORDS = ['ourselves', 'hers', 'between', 'yourself', 'but', 'again', 'there', 'about', 'once', 'during', 'out', 'very', 'having', 'with', 'they', 'own', 'an', 'be', 'some', 'for', 'do', 'its', 'yours', 'such', 'into', 'of', 'most', 'itself', 'other', 'off', 'is', 's', 'am', 'or', 'who', 'as', 'from', 'him', 'each', 'the', 'themselves', 'until', 'below', 'are', 'we', 'these', 'your', 'his', 'through', 'don', 'nor', 'me', 'were', 'her', 'more', 'himself', 'this', 'down', 'should', 'our', 'their', 'while',\n",
    "                          'above', 'both', 'up', 'to', 'ours', 'had', 'she', 'all', 'when', 'at', 'any', 'before', 'them', 'same', 'and', 'been', 'have', 'in', 'will', 'on', 'does', 'yourselves', 'then', 'that', 'because', 'what', 'over', 'why', 'so', 'can', 'did', 'now', 'under', 'he', 'you', 'herself', 'has', 'just', 'where', 'too', 'only', 'myself', 'which', 'those', 'i', 'after', 'few', 'whom', 't', 'being', 'if', 'theirs', 'my', 'against', 'a', 'by', 'doing', 'it', 'how', 'further', 'was', 'here', 'than', 'as']\n",
    "STOP_WORDS_SUMMARY = []\n",
    "STOP_WORDS_REVIEW_TEXT = []\n",
    "\n",
    "\n",
    "\n",
    "def edit_summary(content):\n",
    "\n",
    "    # step 1 - convert the text to only lower case\n",
    "    content = content.lower()\n",
    "\n",
    "    # step 2 - remove unwanted and unicode characters\n",
    "    content = re.sub(\n",
    "        r\"(@\\[A-Za-z0-9]+)|([^0-9A-Za-z \\t])|(\\w+:\\/\\/\\S+)|^rt|http.+?\", \"\", content)\n",
    "\n",
    "    # step 3 - remove stop words\n",
    "    stopwords = GENERIC_STOP_WORDS + STOP_WORDS_SUMMARY\n",
    "\n",
    "    filtered_words = [\n",
    "        word for word in content.split() if word not in stopwords]\n",
    "    text = \" \".join(filtered_words)\n",
    "\n",
    "    return text\n",
    "\n",
    "\n",
    "def edit_review_text(content):\n",
    "\n",
    "    # step 1 - convert the text to only lower case\n",
    "    content = content.lower()\n",
    "\n",
    "    # step 2 - remove unwanted and unicode characters\n",
    "    content = re.sub(\n",
    "        r\"(@\\[A-Za-z0-9]+)|([^0-9A-Za-z \\t])|(\\w+:\\/\\/\\S+)|^rt|http.+?\", \"\", content)\n",
    "\n",
    "    # step 3 - remove stop words\n",
    "    stopwords = GENERIC_STOP_WORDS + STOP_WORDS_REVIEW_TEXT\n",
    "    \n",
    "    filtered_words = [\n",
    "        word for word in content.split() if word not in stopwords]\n",
    "    text = \" \".join(filtered_words)\n",
    "\n",
    "    return text\n",
    "\n",
    "# create categories for vote label\n",
    "\n",
    "\n",
    "def assign_vote_label(i):\n",
    "    if i <= 2.0:\n",
    "        return 'low'\n",
    "    if i > 2.0 and i <= 10.0:\n",
    "        return 'medium'\n",
    "    if i > 10.0 and i <= 50.0:\n",
    "        return 'good'\n",
    "    if i > 50.0:\n",
    "        return 'high'\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Stop words search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['ez', 'ezgo', 'ozzy', 'ozawa', 'zzzzzzzzzzz', 'fzt', 'tzigane', 'aztec', 'zz', 'iz', 'ozzys', 'oz', 'azoom', 'synthetic', 'fyi', 'sylvania', 'type', 'eye', 'syrup', 'hype', 'cylinders', 'typical', 'system', 'eyes', 'ly', 'tyc', 'hybrid', 'sync', 'hyundai', 'hyunda', 'sycho', 'bye', 'syndrome', 'lyrics', 'dynamic', 'cynical', 'byrds', 'dying', 'symphony', 'lynch', 'dylan', 'kyriacos', 'mysterians', 'py', 'hyped', 'dylans', 'byron', 'synths', 'hypegood', 'ryan', 'mystery', 'symphonica', 'nyro', 'pyromania', 'hysteria', 'cycle', 'eyed', 'synthesizer', 'hymns', 'byootay', 'iyer', 'myrath', 'syncopated', 'dyes', 'hydrated', 'tylenol', 'syrups', 'dysrhythmia', 'nyquil', 'ay', 'byullo', 'xylitol', 'rye', 'myth', 'byer', 'typically', 'lysol', 'gym', 'nylon', 'mytouch', 'mycharge', 'symmetry', 'python', 'pyle', 'cylinder', 'eyepiece', '4yo', 'kydex', 'kylin', 'bypass', 'lycrastretchneoprene', 'tying', 'cyclists', 'gymboss', 'cycling', 'cycleops', 'bygone', 'hydration', '4yr', 'eyewear', '2yearold', '2yr', '7yo', '9yo', 'syma', 'gyro', '5yrold', '8yo', 'dynames', 'expected', 'excited', 'expecting', 'explorer', '4x', '2x', 'exact', 'examine', 'expensive', 'xxlarge', 'exposure', 'extremely', 'extra', 'excellent', 'sx', 'exchange', 'expect', 'exactly', 'expedition', 'exhaust', 'expectancy', 'except', 'excursion', 'execution', 'fxdwgi', 'extension', 'expectations', 'rx450', 'exceptions', 'xxl', 'exist', 'expectation', 'extreme', 'rx8', 'extended', 'experience', 'ex', 'explained', 'extend', 'exceeded', 'extraordinarily', 'exposed', 'experimentation', 'existence', 'expressing', 'exploit', 'executed', 'experimental', 'example', 'examples', 'exodus', 'exorcist', 'excesses', 'axis', 'excellentbut', 'exciting', 'excuse', 'excellence', 'xxx', 'excitement', 'experiment', 'excelent', 'exercise', 'extremist', 'expanded', 'exposes', 'exquisite', 'exposition', 'explosives', 'expiration', 'experienced', 'expired', 'extract', 'expires', 'exploded', 'expiring', 'expert', 'oxidized', 'excessive', 'expiraton', 'exceptional', 'extracts', 'exists', 'excelleny', 'expiry', 'external', '3x', 'extremeguard', 'expecatations', 'exctoknife', 'exterior', 'exception', 'extender', 'executive', 'excite', 'extras', 'axe', 'extemely', 'excruciatingly', 'exposing', 'expire', 'expend', 'expectd', 'expander', 'exercises', 'exceeds', 'exfellent', 'excellant', '3x9', '3xwk', '4x4', 'vxl', 'extravaganza', '5x4', 'exta', 'experiments', 'exclusive', 'exstreamly', 'rx', 'exorbitant', 'expansion', 'express', 'explain', 'excess', 'exspansion', 'cx', 'excelente', 'away', 'switch', '2wd', 'vw', 'two', 'awesome', 'owners', 'awful', 'aware', 'twice', 'swollen', 'swivel', 'awd', 'awning', 'swing', 'tw200', '5w40', '4w40', 'tweaking', 'sweet', 'tweet', 'awsome', 'awhile', 'dwarf', 'owes', 'awfuldisappointing', 'wwwsportofdiatractioncom', 'aw', 'gwen', 'awfulthis', 'swine', 'tweedlepop', 'swizz', 'sweat', 'tweed', 'twois', 'swinging', 'owner', 'twist', 'awaited', 'swan', 'owe', 'twofer', 'sweats', 'wwwisrrecordscom', 'twinkiea', 'twang', 'owens', 'twin', 'sweeping', 'twisted', 'ewwwwwwwww', 'aweful', 'sweetners', 'twinings', 'eww', 'twenty', 'sweetener', 'sweetleaf', 'sweeteners', 'sweeter', 'owned', 'sweetness', 'sweetening', 'sweets', 'sweater', 'swiss', 'sweetner', 'awaytrash', 'twoard', 'tweezers', 'sweatproof', 'swings', 'awkward', 'owl', 'switcharoo', 'sweaters', 'swirls', 'sweatpants', 'sweatbands', 'iwb', 'swimming', 'tweaks', 'sweaty', 'dwight', '5wt', 'awfulnot', 'swimsuits', 'swedish', 'swimsuit', 'swim', 'owning', 'swell', 'swimmers', 'sword', 'award', 'sw', 'twelve', 'twins', 'twoyear', 'tweak', 'swords', 'ewok', 'twoplayer', 'owns', 'overpriced', 'ever', 'rv', 'even', 'overspray', 'overlay', 'everyone', 'everybody', 'ive', 'avoid', 'oversized', 'avalons', 'uv', 'avgas', '5v', 'overrated', 'overpromised', 'average', 'overall', 'rvs', 'ave', 'overly', 'everywhere', 'available', 'avalanche', 'everything', 'every', 'overbearing', 'dvd', 'evaporates', 'oversampled', 'overproduced', 'overhype', 'overlycommercialized', 'evil', 'overdone', 'everythings', 'averagenothing', 'eviland', 'overhyped', 'overproductionmix', 'everyday', 'overview', 'overblown', 'overratted', 'tv', 'evans', 'evolve', 'overcomes', 'sv', 'overthetop', 'ova', 'overlook', 'overseas', 'lvb', 'evolving', 'ove', 'overlooked', 'avril', 'overheated', 'overwhelmed', 'event', 'evenly', 'overcome', 'overwhelming', 'overwhelmingly', 'overpoweringly', 'oven', 'eve', 'everyones', 'avocado', 'gv', 'evo', 'overnight', 'eventually', 'overheat', 'avg', 'avid', 'hvh913bt', 'everhave', 'dvds', 'eventuslly', 'evolved', 'avialable', 'overhead', 'pvc', 'overcomplicated', 'overpaid', 'overload', 'overfixed', 'ava', 'avengers', 'busted', 'custom', 'functional', 'purchase', 'buy', 'number', 'pure', 'output', 'much', 'junk', 'sun', 'auto', 'quits', 'hurts', 'junkyour', 'buying', 'sucks', 'cushion', 'hunk', 'autozone', 'pulley', 'buycheap', 'super', 'rusted', 'quality', 'sure', 'put', 'turn', 'customer', 'durable', 'quantity', 'bux', 'dual', 'bulb', 'curved', 'jump', 'buyer', 'rust', 'multiple', 'bulbs', 'cup', 'aura', 'duty', 'turning', 'cute', 'sucky', 'runs', 'bummer', 'buyfalse', 'purchased', 'guage', 'luck', 'rubicon', 'support', 'purpose', 'quiet', 'fullface', 'turned', 'quite', 'lukewarm', 'must', 'rubber', 'supposed', 'superior', 'guess', 'suddenly', 'quit', 'questionable', 'junkfor', 'junkfades', 'turns', 'outdated', 'rubbermaid', 'full', 'buld', 'burned', 'builtin', 'fuel', 'pump', 'dud', 'fuse', 'surge', 'suppressor', 'rusts', 'surfaces', 'quilty', 'outer', 'kubota', 'qualityoverheat', 'burnout', 'qualitydo', 'sunshade', 'suction', 'cups', 'cut', 'juuuuuuuunk', 'bulbsvery', 'tube', 'quarters', 'sub', 'ruined', 'dust', 'autel', 'purchases', 'dupont', 'pushin', 'bulky', 'cutequeen', 'automotive', 'dunno', 'budget', 'automobiles', 'durango', 'fully', 'run', 'bubbled', 'nuts', 'mushroom', 'puig', 'outdoor', 'junkola', 'suited', 'suck', 'bunch', 'buckle', 'turtle', 'pulse', 'suv', 'quialty', 'suitable', 'husband', 'quickly', 'tubes', 'purchasing', 'suspect', 'supply', 'built', 'dumb', 'hubs', 'subaru', 'nut', 'rusting', 'build', 'tubeless', 'outlet', 'fuses', 'outback', 'functionally', 'huge', 'fusion', 'lug', 'guard', 'purge', 'burns', 'buick', 'bug', 'burnt', 'bulds', 'duplicolor', 'bushing', 'audi', 'due', 'suggested', 'bucket', 'funny', 'kurt', 'guy', 'pusher', 'purple', 'pulstar', 'current', 'suppose', 'tundra', 'sufficient', 'substantial', 'quads', 'quick', 'functions', 'rubbery', 'gun', 'bumper', 'sucked', 'tub', 'pulled', 'lumens', 'rugged', 'quad', 'euro', 'tuner', 'cutting', 'tuning', 'bulkhead', 'bumping', 'surprisingly', 'funnily', 'future', 'purring', 'guide', 'curt', 'purolater', 'pureone', 'tune', 'fun', 'fuzz', 'outcome', 'summer', 'rushed', 'quik', 'pully', 'duragloss', 'punishing', 'bushings', 'funky', 'functionality', 'suggest', 'muffler', 'bumperonce', 'supplement', 'superb', 'gutter', 'duramax', 'lubricant', 'guns', 'suzuki', 'quart', 'outside', 'butt', 'superduty', 'supper', 'junior', 'lugs', 'outlast', 'fulton', 'buck', 'outstanding', 'mustang', 'music', 'humanity', 'suggestion', 'guys', 'qualityavoid', 'vu', 'yuck', 'queen', 'que', 'buyers', 'muy', 'rub', 'juice', 'turner', 'bubblegum', 'bubble', 'gum', 'rubbish', 'fuss', 'dull', 'puke', 'public', 'musicians', 'queensryche', 'funnier', 'funniest', 'buble', 'cure', 'puff', 'turd', 'business', 'musical', 'burn', 'burzum', 'audiophile', 'musicanother', 'julian', 'rutherfordthe', 'jumpedtheshark', 'yuk', 'nuclear', 'sugar', 'bust', 'guysmike', 'hunter', 'dullest', 'suckiest', 'huh', 'suffers', 'outrock', 'guitar', 'authoring', 'muddy', 'hustle', 'musicawful', 'autorun', 'audio', 'rush', 'duets', 'cursing', 'publish', 'punishment', 'ruin', 'outshines', 'bull', 'gull', 'punk', 'puppy', 'questionwhy', 'substance', 'fumbles', 'purchaser', 'audience', 'fundamental', 'ruins', 'butthole', 'surfers', 'humpty', 'dumpty', 'surprise', 'muzak', 'surface', 'qualities', 'dursts', 'lukatherwhat', 'numetalcrap', 'lunch', 'mud', 'dualdiscs', 'autotune', 'hurryup', 'tunes', 'mundo', 'buffs', 'puzzling', 'buttons', 'duo', 'guitars', 'murder', 'fullon', 'outtakes', 'musicthis', 'hughes', 'buckley', 'hummm', 'subsabbath', 'outs', 'ludes', 'funk', 'sudden', 'guthrie', 'musicianship', 'outgrown', 'hubbards', 'submerged', 'humble', 'supplier', 'ruining', 'fuin', 'running', 'bullet', 'hurricane', 'juslisen', 'dullish', 'audition', 'murky', 'suits', 'suzis', 'subpar', 'quaint', 'question', 'superfans', 'turkey', 'justin', 'rule', 'sunfighter', 'humor', 'musically', 'putty', 'cuts', 'surround', 'funkdafied', 'hungry', 'musak', 'bumps', 'bumpy', 'justins', 'outing', 'lucky', 'pushed', 'dublin', 'musthave', 'mumbling', 'autoprogrammed', 'punish', 'buffett', 'lune', 'rushs', 'bueno', 'sunday', 'dunham', 'purist', 'multiinstrumentalist', 'futuristic', 'lucy', 'eurethmics', 'guitarist', 'nulcear', 'austria', 'futuresexlovesounds', 'buzz', 'futures', 'puts', 'curates', 'puck', 'butterfly', 'fuzzy', 'duck', 'tull', 'ruby', 'durantes', 'success', 'muted', 'survivor', 'dude', 'supertramp', 'judge', 'duplicate', 'funflowing', 'judys', 'cube', 'hustlin', 'judas', 'succeed', 'cult', 'hundred', 'quirky', 'buddy', 'lush', 'submarine', 'hurrah', 'guttermouth', 'numetal', 'queeen', 'bubbles', 'surprized', 'authentic', 'superdrag', 'subtleties', 'russian', 'guaranteed', 'bucckaw', 'june', 'subtle', 'buena', 'tulls', 'puberty', 'busy', 'supplyin', 'surprised', 'guileless', 'mulligan', 'guessing', 'junkie', 'rules', 'muchjk', 'austin', 'quartet', 'survive', 'musician', 'quench', 'sums', 'quintessential', 'cultclassic', 'punch', 'lust', 'superconductor', 'budokan', 'musci', 'vultures', 'dunhillabc', 'yuletide', 'surprising', 'guadagni', 'luke', 'fuer', 'outkast', 'sung', 'tulsa', 'sur', 'humdingers', 'lunatic', 'surpassed', 'bunny', 'buniatishvili', 'dusty', 'puzzle', 'sum', 'dufays', 'summed', 'rufus', 'suite', 'dutch', 'lutes', 'wu', 'muffin', 'suzy', 'bursting', 'substitute', 'rude', 'substitutiom', 'yucky', 'butter', 'guar', 'mush', 'jug', 'yummy', 'sugary', 'ounce', 'outrageous', 'buttery', 'turds', 'ought', 'currants', 'rum', 'bulged', 'huntsmansuper', 'gummies', 'sulphur', 'sucralose', 'mushy', 'subjective', 'yukky', 'purveyor', 'purposely', 'rubbed', 'cumin', 'pumpkin', 'hurt', 'dunkin', 'sugarfree', 'curry', 'bulk', 'butternut', 'qual', 'yuktoo', 'nutrition', 'murica', 'duplings', 'bugs', 'buckwheatlike', 'yumearths', 'tuna', 'nutritious', 'quinoa', 'jumbo', 'sufferers', 'autoimmune', 'sucanut', 'outweighed', 'nunaturals', 'nustevia', 'purposes', 'automatic', 'gummy', 'kum', 'dump', 'suntan', 'justify', 'august', 'lumpy', 'nuggets', 'subject', 'furniture', 'ouch', 'butters', 'zucchini', 'burning', 'numbness', 'sugarworks', 'substitue', 'tummys', 'pulpy', 'putting', 'bubblemint', 'bublegum', 'quest', 'yum', 'duped', 'burr', 'yummybetter', 'jucky', 'cupcakes', 'mushroomy', 'suckable', 'butnot', 'quickie', 'runaway', 'buttoo', 'punching', 'gumballs', 'july', 'nutty', 'luster', 'buttermilk', 'turkish', 'runny', 'suggests', 'muffins', 'puerh', 'cupcake', 'cuppa', 'punchy', 'burger', 'hugin', 'fullbodied', 'sunrise', 'hummus', 'sugars', 'juicy', 'turmeric', 'sucker', 'quinine', 'hull', 'mustard', 'hunters', 'yummmo', 'bulgar', 'quaker', 'susanb', 'pudding', 'durkees', 'purse', 'yup', 'nuff', 'suppression', 'ful', 'tuity', 'multipurpose', 'bundle', 'supersize', 'yumearth', 'sushi', 'yummers', 'sunflower', 'supposedly', 'dumplings', 'ounces', 'buttteri', 'guilt', 'mummmm', 'lunches', 'bucks', 'junkj', 'button', 'pueden', 'cubed', 'mute', 'lumia', 'publishing', 'buyplease', 'duuuud', 'cumbersome', 'durablegood', 'buggy', 'function', 'turbo', 'putt', 'bulkier', 'auxiliary', 'push', 'durability', 'cu920', 'customizable', 'guardian', 'iu', 'dubious', 'nude', 'gurl', 'cutout', 'lumsing', 'butplease', 'muffles', 'pulls', 'huawei', 'guides', 'outward', 'pull', 'buried', 'human', 'cutouts', 'outlives', 'curtain', 'substantially', 'survived', 'outta', 'fulfilled', 'burner', 'turboboost', 'muche', 'curl', 'automobile', 'multidevice', 'multifunctional', 'hubby', 'outlasting', 'outter', 'zuzo', 'buds', 'numerous', 'supplied', 'sunwee', 'multiport', 'sunny', 'aukey', 'cust', 'suprised', 'hurtbad', 'suffer', 'buttock', 'sunglasses', 'lure', 'burg', 'ruger', 'duds', 'bunched', 'dumbbells', 'suit', 'multidisfunctional', 'dur', 'dummy', 'lululemon', 'pumps', 'sued', 'aua', 'turret', 'qualitygarbage', 'mushrooms', 'supplies', 'kudu', 'yukon', 'cushioned', 'cuz', 'junky', 'buckles', 'hunting', 'surplus', 'sunburn', 'cushioning', 'hung', 'curvy', 'zunni', 'guhrosse', 'bushman', 'currently', 'rustic', 'curve', 'cub', 'cuffs', 'lumbar', 'tuned', 'putter', 'eureka', 'fuelband', 'surely', 'building', 'huggers', 'lube', 'builtjust', 'subfreezing', 'bungee', 'luggage', 'cue', 'customizeable', 'buckeyes', 'supported', 'multi', 'gulp', 'gust', 'pumping', 'cuppot', 'quiets', 'dundees', 'sunglassesvery', 'bulge', 'gunsite', 'dulling', 'gunner', 'luckybum', 'supplemental', 'punctures', 'bugout', 'outdoors', 'humidity', 'tue', 'wubee', 'gunsmith', 'survival', 'muffs', 'fursuit', 'multicolored', 'pullover', 'quicker', 'gunwales', 'buttmassager', 'culling', 'guest', 'hundreds', 'runners', 'outlets', 'outdoorsy', 'bunches', 'bulldog', 'outdoorsman', 'aurora', 'puppet', 'jungle', 'bus', 'funko', 'hubsan', 'numbers', 'dumber', 'vulgar', 'pursuit', 'questions', 'duncan', 'cutie', 'juuce', 'pups', 'outrageously', 'muddled', 'duplicates', 'puzzles', 'builds', 'sunken', 'kudos', 'puppets', 'bummed', 'dustbunny', 'ducks', 'pullback', 'duet', 'suctions', 'tux', 'jury', 'rummyo', 'mumsy', 'jumping', 'cuddly', 'bult', 'cuddle', 'cubes', 'autograph', 'hunt', 'ouchy', 'rubs', 'cutest', 'qualityso', 'outfits', 'nursery', 'duh', 'multitheme', 'pulling', 'numbered', 'rubiks', 'turntable', 'quadcopter', 'tunnel', 'musical2', 'pucker', 'builder', 'duct', 'puppetno', 'supports', 'hulk', 'supersized', 'furby', 'qualityfun', 'quirks', 'supercute', 'mummra', 'puzzel', 'curves', 'superman', 'duaghter', 'duplos', 'puddy', 'outfit', 'tutto', 'gumball', 'duplo', 'autistic', 'munching', 'pumped', 'quaity', 'tucked', 'star', 'stand', 'started', 'item', 'stars', 'stopped', 'store', 'stay', 'sticks', 'stock', 'stuff', 'atv', 'stop', 'states', 'sticker', 'standard', 'attachment', 'still', 'steering', 'stabiltrak', 'stoped', 'strut', 'starting', 'steer', 'stick', 'steaming', 'sturdy', 'statement', 'utter', 'style', 'struts', 'starter', 'stickers', 'stiff', 'stitching', 'stayed', 'streaking', 'stack', 'straight', 'stands', 'step', 'stink', 'strands', 'stretches', 'stays', 'stuck', 'storm', 'gt', 'stretched', 'stripper', 'sturdier', 'strap', 'strip', 'strong', 'staps', 'streaks', 'stated', 'sticky', 'stupid', 'pt', 'straps', 'itll', 'lt', 'sti', 'otherwise', 'storage', 'bt100', 'stabilize', 'stableload', 'xtreme', 'start', 'stiffness', 'stores', 'street', 'stik', 'wth', 'stubborn', 'stylish', 'staining', 'items', 'utilitarian', 'streamline', 'sticking', 'stolen', 'stereo', 'stinky', 'others', 'atomic', 'xterra', 'attaches', 'etc', 'stainless', 'ft', '4th', 'stroke', 'atf', 'stealership', 'stiching', 'starters', 'stopping', 'steel', 'stabil', 'state', '5th', 'attention', 'stone', 'wtf', 'attempt', 'stadium', 'stravinsky', 'uttered', 'atrocity', 'stapp', 'strikingly', 'mtv', 'stones', 'stoopid', 'studio', 'strippedbetter', 'stuffonly', 'atleast', 'stefani', 'stereolab', 'stinkers', 'stinker', 'stinks', 'attn', 'streak', 'strident', 'stage', 'strike', 'studied', 'steps', 'itthen', 'stiffs', 'stan', 'itbut', 'standards', 'stripes', 'stoned', 'itsok', 'otello', 'sty', 'stewart', 'stingy', 'itsnotmattolasfaultbutthemaninthemirror', 'strength', 'story', 'stings', 'stp', 'starz', 'starts', 'stunning', 'strange', 'utterly', 'stripped', 'steve', 'styles', 'strokes', 'storyteller', 'stevie', 'strongeven', 'stages', 'stretch', 'attractive', 'attitude', 'stooges', 'staging', 'strooling', 'stratovarius', 'ethics', 'st', 'stipulate', '6th', 'stretching', 'strongest', 'standing', 'stephen', 'atom', 'steppenwolf', 'stars12', 'stronger', 'staple', 'streets', 'stanzas', 'strangely', 'stellar', 'starand', 'strongly', 'steep', 'stale', 'ethnic', 'stories', 'rt', 'standalone', 'jts', 'structures', 'stardom', 'starship', 'storyworld', '9th', 'steves', 'station', 'italian', 'stills', 'dts', 'steely', 'ethereal', 'ate', 'stevia', 'styrofoam', 'stomach', 'stabilizer', 'starbucks', 'attack', 'starch', 'stems', 'strawberries', 'stevias', 'stash', 'stringytoo', 'strawberry', 'storagedelivery', 'ethiopian', 'storable', 'mt', 'steamer', 'standby', 'ct', 'starws', 'strudel', 'stream', 'stuffed', 'steaks', 'starwest', 'stuf', 'otter', 'stains', 'att', 'othe', 'gti8200', 'stable', 'attached', 'htc', 'stops', 'attach', 'bt', 'stylus', 'otterbox', 'attracts', 'stickiness', 'steal', 'stylo', 'nt', 'stylei', 'etl', 'xt910', 'sturdiness', 'lte', 'study', 'stolid', 'studded', 'stout', 'steppin', 'sturdylightreliable', 'styk', 'studs', 'itemdid', 'stocking', '7th', 'string', 'stickybinding', 'pts', 'strenght', 'stretchy', 'stature', 'stroller', 'steroids', '3t', 'stem', 'strecth', 'iteditsteer', 'stinksliterally', 'straw', 'at44s', 'stinger', 'strengthener', 'stove', 'streamlight', 'itchy', 'stole', 'straped', 'student', 'stanley', 'jt', 'stab', 'ati', 'stakes', 'athletic', 'stip', 'stampers', 'stampede', 'steady', 'sticknot', 'static', 'struggled', 'stain', 'statue', 'aton', 'stained', 'strikeouts', 'storybook', 'stringpain', 'itty', 'styraphome', 'stablemates', 'stickier', 'stunt', 'stepped', 'stella', 'stride', 'strategy', 'strudy', 'steardy', 'attentiongetting', 'stamps', 'starring', 'stinking', 'strung', 'stratus', 'useless', 'isnt', 'use', 'asian', 'ashamed', 'used', 'ask', 'uses', 'usually', 'cs', 'using', 'ss581t', 'psb1', 'issue', 'isuzu', 'us', 'user', 'issues', 'assurance', 'vs', 'useful', 'usable', 'tsx', 'assembly', '3step', 'assumed', 'especially', 'bstock', 'psi', 'isolation', 'usefull', 'escalade', 'escape', 'assess', 'ase', 'hse', 'ms98002t', '1st', 'ssbc', 'vstrom', 'usa', 'rs', 'ass', 'essence', 'nsync', 'nsane', 'estaba', 'island', 'eso', 'es', 'essential', 'astounding', 'asking', 'ashlee', 'dsd', 'lsd', 'bsides', 'psychedelia', 'usual', 'asked', 'esp', 'esperanza', '3stars', 'csny', 'essentials', 'osama', 'aspect', 'aspiration', '3star', 'ismaels', 'rs2', 'esoteric', 'aspiring', 'iss', 'ps', 'assortment', 'bs', 'asskickers', 'isspectacular', 'psychedelic', 'psychedellica', 'astoundingly', 'qs', 'aspirin', '5star', 'assuming', 'aspartame', 'usage', 'astronomical', 'associated', 'msg', 'assamese', 'msgplease', 'users', 'assessment', 'jst', 'ms', 'asthma', 'assort', 'espresso', 'psl', 'asia', '6s', 'usb2', 'usedestroyed', 'usb', 'hs700', '5s', 'useable', '4s', 'asap', '2stars', 'usps', 'assist', 'assembled', 'gs4', 'gs3', 'gshock', 'userthis', 'aside', 'asdescribed', 'gsm', 'sslimagesamazon', 'ss300', 'gs6', 'osdream', 'msr', 'uscg', 'os', 'assemble', 'essentially', 'yse', 'tshirt', 'asics', 'escaping', 'assos', 'usages', 'ishadora', 'usgi', 'jsb', 'mss', 'assault', '3s', '2second', 'oscar', 'ss', 'assembling', 'oriiginal', 'product', 'order', 'great', 'truth', 'crap', 'broken', 'crushes', 'crapamazon', 'bright', 'pretty', 'price', 'around', 'broke', 'wrong', 'crew', 'original', 'arrived', 'frame', 'promised', 'prep', 'properly', 'dried', 'grip', 'break', 'tried', 'cruiser', 'products', 'grand', 'dry', 'proper', 'ordering', 'tries', 'prime', 'greatbut', 'brake', 'front', 'problems', 'pressure', 'fraudulent', 'trouble', 'hrs', 'problem', 'arb', 'ordered', 'drop', 'lrg', 'dremel', 'trying', 'protective', 'bracket', 'try', 'trd', 'troubles', 'practical', 'armor', 'driving', 'drum', 'trust', 'writing', 'arent', 'arctic', 'proof', 'fragile', 'trucks', 'crazy', 'trash', 'brightness', 'driver', 'brands', 'grease', 'crappy', 'arrival', 'crapola', 'friend', 'creating', 'travel', 'trailer', 'programmer', 'cracked', 'wrog', 'brand', 'print', 'brakes', 'drying', 'frustrating', 'promptly', '4runner', 'drips', 'tree', 'probably', 'traveler', 'breeze', 'provides', 'drain', 'friendly', 'drawbacks', 'profusion', 'breaks', 'eraser', 'trickle', 'crimping', 'drops', 'wrench', 'prepared', 'programming', 'bristles', 'wrap', 'crp', 'gray', 'trip', 'wrangler', 'brighter', 'grounding', 'pricey', 'craftsmanship', 'originals', 'triple', 'brightest', 'areas', 'protects', 'oring', 'grey', 'graydient', 'arrive', 'protector', 'wrinkles', 'breathable', 'drive', 'brackets', 'trailblazer', 'pros', 'brake4wd', 'brittle', 'protection', 'cramped', 'trailering', 'trx', 'process', 'ur', 'brown', 'error', 'truck', 'brainercant', 'ergonomic', 'area', 'trim', 'crawler', 'cree', 'priced', 'fried', 'droppings', 'fraying', 'orange', 'frrl', 'project', 'crossing', 'arrow', 'write', 'fresh', 'trac', 'driveway', 'protect', 'pro', 'drilling', 'pricenah', 'true', 'trick', 'uro', 'transparent', 'crate', 'ground', 'crack', 'aroundyou', 'irritating', 'dramatically', 'grips', 'prevent', 'prebled', 'crewcab', 'brushes', 'crossbars', 'crossbar', 'grabbing', 'prescription', 'frontier', 'wrx', 'transmission', 'freight', 'tr', 'primer', 'argue', 'cruiseer', 'brutal', 'prefer', 'drill', 'brigjt', 'broom', 'preventer', 'organizer', 'preferred', 'draw', 'crv', 'trims', 'grime', 'greasert', 'traverse', 'practicality', 'treatment', 'drove', 'oregon', 'transfer', 'drainpan', 'prodect', 'brother', 'crank', 'wrecked', 'lr4', 'provided', 'trackbar', 'traction', 'track', 'brught', 'orginal', 'trailhawk', 'dressing', 'prevents', 'originally', 'fram', 'tranny', 'breaking', 'crosstrek', 'droning', 'orignal', 'drastically', 'drz', 'fricking', 'fraction', 'brought', 'creeper', 'bridgestone', 'prius', 'proformance', 'braking', 'gravity', 'trance', 'presenti', 'present', 'cross', 'arguably', 'groove', 'drunk', 'triphop', 'cry', 'tracks', 'org', 'oracle', 'wreck', 'trainwreck', 'greatest', 'breath', 'written', 'dribble', 'drivel', 'era', 'arghhh', 'growing', 'brings', 'breakfast', 'pretentious', 'artist', 'erasing', 'orbital', 'produced', 'free', 'dragon', 'artists', 'truthful', 'ireland', 'drink', 'dr', 'cripes', 'arse', 'mr', 'praise', 'grave', 'green', 'artspinal', 'proud', 'truly', 'producing', 'brilliant', 'presentation', 'crime', 'previous', 'freaking', 'traces', 'production', 'brat', 'princessshe', 'irreplaceable', 'art', 'franz', 'creed', 'grunge', 'creedites', 'yrs', 'crying', 'drugs', 'crumble', 'dreadful', 'dreamland', 'kracker', 'frontin', 'bruce', 'tripe', 'dreck', 'crashing', 'grew', 'protected', 'fred', 'brutalenough', 'trends', 'brian', 'cranky', 'crucial', 'grow', 'cringeworthy', 'travels', 'drumming', 'trys', 'drowned', 'orchestra', 'greets', 'wrote', 'drakes', 'brisk', 'productionsound', 'prayer', 'producers', 'arrangments', 'arsenio', 'problematic', 'brothers', 'prerock', 'arranger', 'grace', 'brave', 'dreadfull', 'artistic', 'crisis', 'brilliance', 'projects', 'greg', 'dream', 'greatness', 'group', 'broadcast', 'breathe', 'trend', 'artificially', 'trendy', 'tribute', 'iron', 'tracking', 'crowd', 'priest', 'triumph', 'prongs', 'promising', 'growling', 'predictable', 'freddie', 'eras', 'practically', 'uriah', 'drinking', 'artiness', 'urban', 'traffic', 'traks', 'bridge', 'grass', 'dreams', 'preach', 'fraud', 'brandy', 'eric', 'crashed', 'armin', 'tresors', 'frauds', 'crown', 'predicable', 'pressing', 'progressive', 'trumpeter', 'british', 'procol', 'propain', 'brainchildren', 'tranquilizers', 'arthur', 'friends', 'brick', 'brigade', 'grab', 'prog', 'argentina', 'prisoners', 'dropout', 'bread', 'grabbag', 'progress', 'creams', 'sr', 'creativity', 'organized', 'prejudices', 'brit', 'brasil', 'cranes', 'greeting', 'growls', 'pretzel', 'prettyokay', 'grown', 'browne', 'fran', 'cryptic', 'crusader', 'krush', 'kravitz', 'bryan', 'primus', 'grade', 'graz', 'greetings', 'groundbreaking', 'originality', 'critical', 'presenting', 'trylooking', 'krauss', 'growth', 'freeek', 'drawer', 'predecessors', 'greats', 'pretension', 'precursor', 'crackle', 'orchestration', 'traditional', 'orton', 'dressed', 'predictableboring', 'arrangements', 'cruise', 'armstrong', 'trainer', 'grows', 'grower', 'srvs', 'brubeck', 'erykah', 'grant', 'proportions', 'travelling', 'tribal', 'trio', 'crafted', 'crafty', 'kryptonite', 'tres', 'progression', 'francesca', 'bricks', 'trippy', 'craze', 'sr71', 'drummer', 'branigan', 'creeps', 'drake', 'mrvinsky', 'treeso', 'gregorians', 'crowded', 'critic', 'grungy', 'writes', 'projected', 'triofantastic', 'treasure', 'gregg', 'britpop', 'bradley', 'arizona', 'gramps', 'bring', 'tracklisting', 'drama', 'oriented', 'freedom', 'brain', 'wright', 'erykahs', 'previously', 'croft', 'created', 'bruno', 'orleans', 'brazil', 'gran', 'fries', 'irons', 'grail', 'trumpet', 'tradition', 'aretha', 'franklin', 'brasileiras', 'crossroads', 'cracking', 'prophet', 'bravo', 'bravissimo', 'fronted', 'prince', 'britney', 'arsenal', 'dropping', 'treat', 'brillantly', 'programs', 'train', 'writings', 'transcending', '3rd', 'productions', 'frozen', 'trips', 'frustration', 'architectural', 'yr', 'dreamed', 'artistry', 'tranquil', 'groban', 'brad', 'brilliantly', 'progpsych', 'brokendamaged', 'crackers', 'creamers', 'gross', 'fruit', 'arithmetic', 'tropical', 'fruits', 'fructose', 'grrrrosssssss', 'grape', 'protein', 'grocery', 'artisan', 'artificial', 'grapedamnfruit', 'fruitcake', 'grounds', 'irresistible', 'transet', 'preservatives', 'processing', 'organic', 'oreos', 'grainy', 'grams', 'crumbs', 'treats', 'branching', 'bran', 'trace', 'orders', 'drinks', 'trump', 'cranberries', 'drank', 'transfat', 'wreaked', 'trident', 'organics', 'crumbled', 'grains', 'brazilian', 'grossest', 'granola', 'fruity', 'grassy', 'frenchs', 'grind', 'arm', 'crohns', 'granulated', 'translate', 'broccoli', 'grapefruit', 'creamy', 'crock', 'creamiest', 'fridge', 'grove', 'pretzels', 'transit', 'oregano', 'grandkids', 'gritty', 'brownie', 'grandma', 'crumbly', 'grounded', 'praline', 'arsenic', 'fresher', 'pratchetts', 'crushed', 'crab', 'trial', 'dress', 'cream', 'french', 'truffles', 'traded', 'trade', 'crunchy', 'wrapper', 'produces', 'crunch', 'pricing', 'grain', 'groats', 'grandmas', 'freshness', 'orbit', 'creamer', 'frankly', 'crema', 'crispy', 'greens', 'drizzled', 'crescent', 'pronounced', 'breakable', 'grinders', 'drinkers', 'traditionally', 'priceflavor', 'greenishnot', 'freshest', 'crisp', 'ordinary', 'mrs', 'aroma', 'grandson', 'prevention', 'produce', 'breast', 'grinder', 'preventative', 'grilled', 'grill', 'grimes', 'greattasting', 'trail', 'breads', 'premium', 'irish', 'brewing', 'broth', 'frosted', 'creme', 'prepare', 'origin', 'crm', 'erythritol', 'pricy', 'brew', 'property', 'greasy', 'draft', 'frig', 'crash', 'cracker', 'press', 'truffle', 'branded', 'arthritis', 'frapps', 'fruitcakes', 'cravings', 'drinker', 'crust', 'freely', 'aromatic', 'branston', 'brewed', 'crystalized', 'artisana', 'graham', 'wrapped', 'arrives', 'treatsand', 'broths', 'provide', 'prone', 'vrs', 'freeze', 'practices', 'freedompop', 'brace', 'fre', 'creerlo', 'droid', 'privacy', 'frequently', 'provisions', 'crackedafter', 'protectors', 'presses', 'arms', 'greyblue', 'armorsuit', 'pricequality', 'tricky', 'froze', 'greatly', 'professionally', 'tradeoff', 'cracks', 'premature', 'wrist', 'frustrated', 'frequencyfilter', 'errors', 'credit', 'drawing', 'armband', 'dropped', 'crease', 'droops', 'promotion', 'prouduct', 'orientation', 'wristlet', 'arounds', 'protecting', 'drawed', 'craddle', 'drains', 'problemslimitations', 'preference', 'frills', 'tray', 'grippy', 'greasefree', 'crystal', 'griffin', 'trent', 'precisely', 'arcadia', 'prompt', 'traveling', 'protectore', 'urspeedteklive', 'prodcut', 'gripped', 'promisedgreat', 'ergonomics', 'priceit', 'grandfather', 'grat', 'prepaid', 'prints', 'preserving', 'travelers', 'vr', 'tripod', 'tracfone', 'trave', 'praises', 'proofs', 'promises', 'crossbow', 'framed', 'bracelet', 'ar', 'trioxane', 'frayed', 'erratic', 'troy', 'crosscountry', 'breaker', 'training', 'iraq', 'broker', 'greater', 'proportion', 'prc', 'trigger', 'drizzle', 'orifice', 'wraps', 'profile', 'practice', 'grrrr', 'gripper', 'army', 'oranges', 'crinckly', 'mres', 'procuct', 'armour', 'tread', 'drawstring', 'props', 'hr', 'trolling', 'trek', 'trains', 'dries', 'prop', 'trades', 'tropics', 'graphics', 'dragonfly', 'programmable', 'tracker', 'yrds', 'freehubs', 'transportstore', 'brush', 'craftmanship', 'proved', 'brooks', 'greatat', 'organization', 'prior', 'crosman', 'transport', 'propane', 'pressured', 'breeches', 'presta', 'crocodile', 'protectivewith', 'breakin', 'precise', 'trangia', 'precision', 'trike', 'preety', 'grear', 'trunk', 'cranked', 'bronx', 'pride', 'gram', 'dryer', 'arrows', 'archers', 'trails', 'transaction', 'treadmill', 'sram', 'grizzly', 'drives', 'irritation', 'ar15', 'prostaff', 'breasts', 'crystals', 'crooked', 'crayola', 'brite', 'creatures', 'traxxas', 'proprietary', 'frequency', 'erase', 'frog', 'articulation', 'drew', 'breyer', 'drone', 'bristle', 'trivia', 'grimms', 'grapes', 'fromit', 'princess', 'crayons', 'graders', 'transformer', 'grandsons', 'greed', 'artwork', 'groan', 'granddaughter', 'dryerase', 'printer', 'grandaughter', 'preopened', 'dresses', 'presented', 'brainwashing', 'grandchildren', 'qr', 'drained', 'erasers', 'braiding', 'kratt', 'irregularities', 'preschoolers', 'greatgranddaughters', 'crayon', 'dresser', 'breezies', 'bracelets', 'prototypish', 'crib', 'dreaming', 'writeup', 'briefly', 'pretend', 'creative', 'printing', 'creation', 'prodduct', 'priority', 'tracing', 'pricebut', 'preschool', 'promo', 'transformers', 'frequint', 'ir', 'brass', 'craft', 'preschooler', 'printed', 'provoking', 'ordeal', 'triggered', 'bruder', 'create', 'kreo', 'transform', 'vroom', 'bratz', 'bribe', 'frisherprice', 'grandkid', 'private', 'dragons', 'brunette', 'crane', 'wrapping', 'trackmaster', 'trey', 'troopers', 'predator', 'trousers', 'braiiiinnnnnless', 'friggin', 'grandparents', 'triangles', 'brachiosaurus', 'grittiness', 'granddaughters', 'square', 'squished', 'squeaky', 'equipped', 'equipment', 'squeak', 'equal', 'equivalent', 'gq', 'squires', 'squash', 'squares', 'bqq', 'squeeeeeeeeaaaaaak', 'squirrel', 'aqua', 'squirters', 'squeaks', 'equestria', 'squishy', 'equally', 'squirts', 'updated', 'special', 'sponge', 'spray', 'applications', 'spent', 'appeared', 'specially', 'opinion', 'update', 'spouts', 'space', 'operate', 'approved', 'application', 'open', 'spend', 'app', 'spark', 'sprayer', 'apart', 'specified', 'spring', 'updating', 'tpms', 'sportbike', 'span', 'specifically', 'ups', 'rpms', 'sportster', 'spite', 'spectre', 'sport', 'spectacular', 'spec', 'tps', 'speed', 'spout', 'specific', 'appropriately', 'apply', 'opposite', 'hp', 'iphone', 'upon', 'upgrade', 'spin', 'appear', 'spot', '2pack', 'option', 'openers', 'upwards', 'rpm', 'spindle', 'applicators', 'spade', 'apparently', 'optima', 'upper', 'specifications', 'spread', 'spare', 'eps', 'upto', 'mph', 'sparky', 'spending', 'applying', 'uplifting', 'speechless', 'mp3', 'apocalypse', 'mp3s', 'spinal', 'spoiled', 'epic', 'springer', 'opportunity', 'spoken', 'mp3ish', 'spineless', 'lp', 'appropriate', 'sports', 'spirit', 'opps', 'spacy', 'upwhy', 'split', 'spectacularly', 'spectacularlysounding', 'options', 'lpstyle', 'approach', 'appetite', 'ep', 'spilling', 'apolgizes', 'apologize', 'spell', 'spoon', 'appeal', 'opus', 'sparkling', '2pac', 'spins', 'appreciate', 'appreciating', 'vp', 'opeth', 'spans', 'splendid', 'epica', 'springboard', 'ipod', 'dps', 'spectacle', 'appeals', 'spiderfingers', 'opulent', 'eponymous', 'splenda', 'upset', 'spry', 'sprayed', 'apple', 'wprime', 'apricots', 'spores', 'spikes', 'spots', 'apiece', 'spiced', 'spicy', 'spice', 'opening', 'splendidly', 'spike', 'opened', 'appearance', 'upsets', 'spaces', 'sparingly', 'spices', 'opern', 'ppj', 'sprinkles', 'splash', 'appealing', 'apples', 'spaghetti', 'apple2thecore', 'spelt', 'apricot', 'openings', 'spanish', '6p', 'ipad', 'spigen', 'sparkle', 'speck', 'specs', 'springs', 'spoils', 'spared', 'ipad3', 'gps', 'speaker', 'sprint', 'mpow', 'upside', 'iphoneipad', 'apps', 'applies', 'operates', 'splurge', 'speakers', 'splendor', 'splits', 'kp500', 'speeds', 'iphone4', '3pack', 'ipadiphone', 'appears', '6plus', 'speedy', 'speak', 'updates', 'sporty', 'xperia', 'spected', 'opo', 'openedslashed']\n",
      "['ozzy', 'fyi', 'hydrated', '2x', 'exposure', 'extremely', 'execution', 'expert', '3x', '2wd', 'swivel', 'dwarf', 'sweat', 'owner', 'twins', 'rv', 'everybody', 'uv', 'evil', 'overseas', 'overlooked', 'overwhelmed', 'evenly', 'evo', 'quits', 'bux', 'rust', 'quit', 'burnout', 'sunshade', 'suction', 'ruined', 'bubbled', 'suck', 'purchasing', 'suspect', 'functionally', 'kurt', 'sucked', 'funky', 'suggestion', 'vu', 'rub', 'turner', 'bubblegum', 'puke', 'public', 'funniest', 'buble', 'julian', 'hunter', 'outshines', 'submerged', 'bullet', 'punish', 'puts', 'butterfly', 'guaranteed', 'rum', 'mushy', 'purposely', 'rubbed', 'nutrition', 'august', 'bucks', 'cumbersome', 'suffer', 'pumps', 'funko', 'cutie', 'pups', 'stopped', 'atv', 'strut', 'steer', 'wth', 'starters', 'stravinsky', 'atleast', 'stinks', 'stage', 'stripped', 'styrofoam', 'starch', 'att', 'stylus', 'struggled', 'asian', 'isuzu', 'essence', 'es', 'asking', 'ashlee', '5star', 'os', '3s', 'broken', 'broke', 'trd', 'arctic', 'trucks', 'brightness', 'crappy', 'cracked', 'triple', 'transfer', 'prevents', 'fraction', 'trance', 'breath', 'written', 'orbital', 'proud', 'producing', 'freaking', 'dreadful', 'tripe', 'fred', 'grown', 'frustration', 'gross', 'fruits', 'grocery', 'artisan', 'fruitcake', 'preservatives', 'grainy', 'trump', 'crumbled', 'grains', 'pricing', 'mrs', 'draft', 'crash', 'privacy', 'arms', 'cracks', 'ar', 'erase', 'articulation', 'breyer', 'crayons', 'sponge', 'applications', 'appeared', 'tpms', 'spout', 'upon', 'specifications', 'spry', 'apricots', 'spots', 'spiced', '6p', 'springs', 'ipad3']\n",
      "['ez', 'oza', 'azalea', 'ozawa', 'ozzmosis', 'typo', 'mysteriously', 'gy', 'tyler', 'oye', 'ryche', 'bytes', 'myung', 'wyclef', 'lydon', 'wynonna', 'nyquil', 'hypocrisy', 'dysfunctional', 'hydrated', 'xylichew', 'syrupi', 'symbols', '7yr', '9yr', '7yo', 'examine', 'ox', '3xl', 'axel', '5xl', 'expelled', 'execs', 'experiencewhat', 'experiece', 'excitment', 'exploited', 'experimentalism', 'existed', 'axemen', 'exacerbate', 'exiles', 'extremly', 'expendable', 'exhibit', 'exceptionthe', 'exits', 'exercised', 'extol', '7x', 'expenses', 'expectancy', 'experienceupdate', '9x9', 'expanders', 'lxl', 'excavator', 'awg', '5wt', 'pwc', 'tweaked', 'awareness', 'swifts', 'swearing', 'twinkie', 'swooping', 'awfull', 'awash', 'swells', '3way', 'sweetie', 'sweaters', 'awakening', 'awfulthe', 'awfuli', 'twinings', 'swallows', 'swollen', 'twining', 'twopart', 'ownership', 'swivels', 'sweatwicking', 'awayit', 'overtightened', 'avenues', 'evacuation', 'evacuate', 'overrated', 'pvc', 'evidenced', 'dvds', 'oversexed', 'evangelical', 'evaporate', 'overuse', 'overprocessed', 'averages', 'evolutionary', 'overdo', 'everclear', 'dvdif', 'evens', 'overpowered', 'avocados', 'everhard', 'aviator', 'evey', 'sucks', 'sucked', 'buick', 'buyer', 'bux', 'junkyard', 'purolator', 'duralast', 'surge', 'autos', 'turbine', 'runaround', 'purchasesthe', 'buggy', 'fuseholder', 'ruben', 'outupdatethe', 'puddles', 'suck', 'sucksthe', 'bulks', 'purchasethis']\n"
     ]
    }
   ],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "\n",
    "cutoff = 1\n",
    "\n",
    "# summary\n",
    "data = train_data[\"summary\"].astype(str).apply(edit_summary)\n",
    "vectorizer1 = TfidfVectorizer()\n",
    "output1 = vectorizer1.fit_transform(data)\n",
    "dict1 = vectorizer1.vocabulary_\n",
    "\n",
    "# top 3000 words by frequency\n",
    "top_terms1 = sorted(dict1.keys(), key=lambda x: x[1], reverse=True)[:3000]\n",
    "print(top_terms1)\n",
    "feature_names1 = vectorizer1.get_feature_names_out()\n",
    "stop_words1 = []\n",
    "y = train_data['overall'].apply(lambda x: 0 if x <= cutoff else 1)\n",
    "\n",
    "# checking each word if it is relevant to the cutoff\n",
    "for word in top_terms1:\n",
    "    index = np.where(feature_names1 == word)[0]\n",
    "    star_arr = np.where(output1.todense()[:, index] != 0)[0]\n",
    "    num_zeros = np.count_nonzero(np.array(y[star_arr] == 0))\n",
    "    if (np.array(y[star_arr]).size == 0):\n",
    "        continue\n",
    "    ratio = num_zeros/np.array(y[star_arr]).size\n",
    "    # if y=zeros/ total lies in the region of [0.45,0.55] we include that in special stop words\n",
    "    if ratio >= 0.45 and ratio <= 0.55:\n",
    "        stop_words1.append(word)\n",
    "\n",
    "print(stop_words1)\n",
    "\n",
    "\n",
    "# review Text\n",
    "data = train_data[\"reviewText\"].astype(str).apply(edit_review_text)\n",
    "vectorizer2 = TfidfVectorizer()\n",
    "output2 = vectorizer2.fit_transform(data)\n",
    "dict2 = vectorizer2.vocabulary_\n",
    "# top 3000 words by frequency\n",
    "top_terms2 = sorted(dict2.keys(), key=lambda x: x[1], reverse=True)[:3000]\n",
    "feature_names2 = vectorizer2.get_feature_names_out()\n",
    "stop_words2 = []\n",
    "y = train_data['overall'].apply(lambda x: 0 if x <= cutoff else 1)\n",
    "\n",
    "# checking each word if it is relevant to the cutoff\n",
    "for word in top_terms2:\n",
    "    index = np.where(feature_names2 == word)[0]\n",
    "    star_arr = np.where(output2.todense()[:, index] != 0)[0]\n",
    "    num_zeros = np.count_nonzero(np.array(y[star_arr] == 0))\n",
    "    if (np.array(y[star_arr]).size == 0):\n",
    "        continue\n",
    "    ratio = num_zeros/np.array(y[star_arr]).size\n",
    "    # if y=zeros/ total lies in the region of [0.45,0.55] we include that in special stop words\n",
    "    if ratio >= 0.45 and ratio <= 0.55:\n",
    "        stop_words2.append(word)\n",
    "\n",
    "print(stop_words2)\n",
    "\n",
    "# Results\n",
    "\n",
    "# STOP_WORDS_SUMMARY = ['ozzy', 'fyi', 'hydrated', '2x', 'exposure', 'extremely', 'execution', 'expert', '3x', '2wd', 'swivel', 'dwarf', 'sweat', 'owner', 'twins', 'rv', 'everybody', 'uv', 'evil', 'overseas', 'overlooked', 'overwhelmed', 'evenly', 'evo', 'quits', 'bux', 'rust', 'quit', 'burnout', 'sunshade', 'suction', 'ruined', 'bubbled', 'suck', 'purchasing', 'suspect', 'functionally', 'kurt', 'sucked', 'funky', 'suggestion', 'vu', 'rub', 'turner', 'bubblegum', 'puke', 'public', 'funniest', 'buble', 'julian', 'hunter', 'outshines', 'submerged', 'bullet', 'punish', 'puts', 'butterfly', 'guaranteed', 'rum', 'mushy', 'purposely', 'rubbed', 'nutrition', 'august', 'bucks', 'cumbersome', 'suffer', 'pumps', 'funko', 'cutie', 'pups', 'stopped', 'atv', 'strut', 'steer', 'wth', 'starters', 'stravinsky', 'atleast', 'stinks', 'stage', 'stripped', 'styrofoam', 'starch', 'att', 'stylus', 'struggled', 'asian', 'isuzu', 'essence', 'es', 'asking', 'ashlee', '5star', 'os', '3s', 'broken', 'broke', 'trd', 'arctic', 'trucks', 'brightness', 'crappy', 'cracked', 'triple', 'transfer', 'prevents', 'fraction', 'trance', 'breath', 'written', 'orbital', 'proud', 'producing', 'freaking', 'dreadful', 'tripe', 'fred', 'grown', 'frustration', 'gross', 'fruits', 'grocery', 'artisan', 'fruitcake', 'preservatives', 'grainy', 'trump', 'crumbled', 'grains', 'pricing', 'mrs', 'draft', 'crash', 'privacy', 'arms', 'cracks', 'ar', 'erase', 'articulation', 'breyer', 'crayons', 'sponge', 'applications', 'appeared', 'tpms', 'spout', 'upon', 'specifications', 'spry', 'apricots', 'spots', 'spiced', '6p', 'springs', 'ipad3']\n",
    "# STOP_WORDS_REVIEW_TEXT = ['ez', 'oza', 'azalea', 'ozawa', 'ozzmosis', 'typo', 'mysteriously', 'gy', 'tyler', 'oye', 'ryche', 'bytes', 'myung', 'wyclef', 'lydon', 'wynonna', 'nyquil', 'hypocrisy', 'dysfunctional', 'hydrated', 'xylichew', 'syrupi', 'symbols', '7yr', '9yr', '7yo', 'examine', 'ox', '3xl', 'axel', '5xl', 'expelled', 'execs', 'experiencewhat', 'experiece', 'excitment', 'exploited', 'experimentalism', 'existed', 'axemen', 'exacerbate', 'exiles', 'extremly', 'expendable', 'exhibit', 'exceptionthe', 'exits', 'exercised', 'extol', '7x', 'expenses', 'expectancy', 'experienceupdate', '9x9', 'expanders', 'lxl', 'excavator', 'awg', '5wt', 'pwc', 'tweaked', 'awareness', 'swifts', 'swearing', 'twinkie', 'swooping', 'awfull', 'awash', 'swells', '3way', 'sweetie', 'sweaters', 'awakening', 'awfulthe', 'awfuli', 'twinings', 'swallows', 'swollen', 'twining', 'twopart', 'ownership', 'swivels', 'sweatwicking', 'awayit', 'overtightened', 'avenues', 'evacuation', 'evacuate', 'overrated', 'pvc', 'evidenced', 'dvds', 'oversexed', 'evangelical', 'evaporate', 'overuse', 'overprocessed', 'averages', 'evolutionary', 'overdo', 'everclear', 'dvdif', 'evens', 'overpowered', 'avocados', 'everhard', 'aviator', 'evey', 'sucks', 'sucked', 'buick', 'buyer', 'bux', 'junkyard', 'purolator', 'duralast', 'surge', 'autos', 'turbine', 'runaround', 'purchasesthe', 'buggy', 'fuseholder', 'ruben', 'outupdatethe', 'puddles', 'suck', 'sucksthe', 'bulks', 'purchasethis']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['ozzy', 'oz', 'azoom', 'fyi', 'hybrid', 'syndrome', 'hydrated', '4yo', 'expected', 'excited', '2x', 'exposure', 'except', 'extreme', 'example', 'away', '2wd', 'swivel', 'awd', 'twisted', 'sweater', 'tweezers', 'swimming', 'twins', 'ever', 'uv', 'evil', 'overhyped', 'overseas', 'overlooked', 'evenly', 'evo', 'avid', 'overpaid', 'custom', 'much', 'quits', 'hurts', 'cushion', 'quality', 'customer', 'bux', 'multiple', 'bulbs', 'turned', 'supposed', 'guess', 'sunshade', 'cups', 'cut', 'sub', 'purchases', 'dunno', 'bunch', 'suv', 'quickly', 'dumb', 'functionally', 'kurt', 'substantial', 'sucked', 'pulled', 'gutter', 'outside', 'suggestion', 'rub', 'juice', 'turner', 'fuss', 'public', 'funniest', 'business', 'burn', 'hunter', 'suffers', 'outshines', 'puppy', 'qualities', 'autotune', 'guitars', 'outs', 'sudden', 'suits', 'subpar', 'putty', 'punish', 'puts', 'butterfly', 'judge', 'bubbles', 'authentic', 'guaranteed', 'jug', 'rum', 'gummies', 'rubbed', 'sugarfree', 'nutrition', 'nutritious', 'quinoa', 'sufferers', 'purposes', 'justify', 'lumpy', 'putting', 'burr', 'ounces', 'button', 'turbo', 'suffer', 'supplies', 'hunting', 'surplus', 'funko', 'numbers', 'cutie', 'pups', 'duplicates', 'puppets', 'item', 'attachment', 'starting', 'stick', 'stiff', 'step', 'strands', 'stretches', 'storm', 'gt', 'stretched', 'itll', 'street', 'wth', 'items', 'stainless', 'starters', 'atleast', 'stage', 'standards', 'stp', 'strange', 'stripped', 'strawberries', 'att', 'bt', 'steal', 'stroller', 'statue', 'use', 'asian', 'uses', 'usually', 'isuzu', 'issues', 'tsx', 'essence', 'ashlee', '5star', 'os', '3s', 'order', 'original', 'tried', 'cruiser', 'products', 'brake', 'pressure', 'trouble', 'try', 'trd', 'writing', 'arctic', 'driver', 'trailer', 'brand', 'probably', 'wrench', 'bristles', 'wrap', 'grounding', 'drive', 'process', 'error', 'area', 'protect', 'pro', 'transparent', 'prevent', 'transfer', 'traction', 'prevents', 'orignal', 'fraction', 'trance', 'breath', 'written', 'orbital', 'proud', 'previous', 'freaking', 'tripe', 'grow', 'travels', 'problematic', 'brothers', 'grace', 'brilliance', 'projects', 'tribute', 'priest', 'drinking', 'brandy', 'pretzel', 'grown', 'orleans', 'irons', 'dropping', 'programs', 'frustration', 'yr', 'fruits', 'artisan', 'fruitcake', 'trump', 'crumbled', 'grains', 'grapefruit', 'pretzels', 'oregano', 'brownie', 'truffles', 'trade', 'pricing', 'traditionally', 'mrs', 'produce', 'draft', 'crash', 'droid', 'arms', 'armorsuit', 'tricky', 'grippy', 'prints', 'ar', 'army', 'armour', 'drawstring', 'props', 'erase', 'articulation', 'breyer', 'drone', 'grandsons', 'artwork', 'grandaughter', 'dresses', 'equipped', 'squares', 'updated', 'sponge', 'spray', 'applications', 'update', 'operate', 'open', 'spark', 'tpms', 'rpms', 'specifications', 'appropriate', 'sports', 'split', 'options', 'spiced', 'appealing', '6p', 'ipad3', 'speaker', 'operates', 'appears']\n",
      "['ez', 'oza', 'ozzys', 'azalea', 'ozawa', 'ozzmosis', 'czech', 'izumi', 'typo', 'dye', 'fyi', 'eyourlife', 'sync', 'sympathetic', 'tyvek', 'tyc', 'eyeglasses', 'hyper', 'gy', 'synthetics', 'tying', 'synth', 'synths', 'cymbals', 'ryche', 'bytes', 'synthesizers', 'myung', 'wyclef', 'wynonna', 'lynyrd', 'hypocrisy', 'xy', 'lynne', 'mythical', 'cyndi', 'fye', 'cycling', 'dynasty', 'syrupy', 'ryders', 'systemic', 'byron', 'lynn', 'py', 'mystic', 'hynde', 'synthy', 'synchronization', 'dysfunctional', 'sydney', 'xylichew', 'synchronize', 'tylt', 'lysol', 'myselfi', 'symbols', '4yo', 'lycra', '7yr', '9yr', '7yo', 'tykes', 'ex', 'extremely', 'extreme', 'exact', 'explain', 'external', 'excursion', 'explained', 'exterior', 'ox', 'experiencing', 'exist', '2xl', 'existence', 'excessive', 'expert', 'expose', 'extendable', 'expectedwhich', 'expires', 'extending', 'expelled', 'expression', 'execs', 'experiencewhat', 'extinction', 'axe', 'excitment', 'exploited', 'executed', 'axemen', 'excerpt', 'mxpx', 'exiles', 'expendable', 'exhibit', 'exceptionthe', 'xx', 'exemplified', 'experiments', 'excite', 'exploits', 'expend', 'execrable', 'expansive', 'axes', 'exercised', 'explode', 'extol', '7x', 'example2', 'expectancy', 'extruded', 'expectedi', 'expat', 'expiry', 'experienceupdate', '9x9', 'expanders', '1x', '3x9', 'expectthe', 'expansions', 'two', 'away', 'twice', 'awg', 'twisting', 'tweezers', '4wd', 'switching', 'swivel', 'swear', '5w', 'twitchy', 'awkward', 'swirl', 'awning', 'twenty', 'gw', 'sweaty', 'awing', 'awe', 'iwe', 'pwc', 'swizz', 'bw', 'owes', 'tweaked', 'swings', 'swifts', 'swearing', 'twodisc', 'twinkie', 'swooping', 'awash', 'swells', 'awake', '3way', 'swine', 'sweetie', 'dwarfed', 'swoon', 'fw', 'tweedy', 'twothe', 'swords', 'awkwardly', 'twain', 'tweens', 'fword', 'swashbuckling', 'twominute', 'sweeps', 'awaken', 'sweats', 'twister', 'sweetener', 'sweeteners', 'swallows', 'twopart', 'awayit', 'twoyearold', 'even', 'dvd', 'ever', 'overtightened', 'everywhere', 'overheating', 'overkill', 'tv', 'evacuate', 'tvs', 'avoided', 'event', 'pvc', 'evidenced', 'oversexed', 'overdubbed', 'evangelical', 'evaporate', 'overuse', 'overprocessed', 'averages', 'evolutionary', 'overdo', 'everchanging', 'everclear', 'dvdif', 'avantgarde', 'overcompensates', 'avante', 'overbearing', 'oversell', 'everly', 'overproducing', 'oversees', 'evita', 'iverson', 'overated', 'evens', 'overstuffed', 'overcooked', 'overwhelms', 'lv', 'avocados', 'overpay', 'availablei', 'everhard', 'aviator', 'sview', 'evey', 'overfill', 'overpaid', 'guess', 'buy', 'buying', 'purchased', 'bump', 'dust', 'suddenly', 'suction', 'purchase', 'curved', 'auto', 'support', 'supposedly', 'turned', 'output', 'audio', 'pump', 'hurts', 'purchasing', 'nuts', 'bulbs', 'pulled', 'pushes', 'push', 'guys', 'buick', 'turn', 'fuses', 'huge', 'bux', 'rust', 'sufficiently', 'tubes', 'outlet', 'bubbles', 'bubble', 'suggests', 'outta', 'duralast', 'current', 'questions', 'turning', 'automotive', 'bunch', 'autos', 'suppose', 'purchasesthe', 'cure', 'rusty', 'multiple', 'bummer', 'luckily', 'pushed', 'fuseholder', 'rubicon', 'jumped', 'suzuki', 'quad', 'outupdatethe', 'guard', 'puddles', 'june', 'question', 'outright', 'bundled', 'sucksthe', 'bulks', 'purchasethis']\n"
     ]
    }
   ],
   "source": [
    "cutoff = 2\n",
    "\n",
    "# summary\n",
    "data = train_data[\"summary\"].astype(str).apply(edit_summary)\n",
    "vectorizer1 = TfidfVectorizer()\n",
    "output1 = vectorizer1.fit_transform(data)\n",
    "dict1 = vectorizer1.vocabulary_\n",
    "\n",
    "# top 3000 words by frequency\n",
    "top_terms1 = sorted(dict1.keys(), key=lambda x: x[1], reverse=True)[:3000]\n",
    "print(top_terms1)\n",
    "feature_names1 = vectorizer1.get_feature_names_out()\n",
    "stop_words1 = []\n",
    "y = train_data['overall'].apply(lambda x: 0 if x <= cutoff else 1)\n",
    "\n",
    "# checking each word if it is relevant to the cutoff\n",
    "for word in top_terms1:\n",
    "    index = np.where(feature_names1 == word)[0]\n",
    "    star_arr = np.where(output1.todense()[:, index] != 0)[0]\n",
    "    num_zeros = np.count_nonzero(np.array(y[star_arr] == 0))\n",
    "    if (np.array(y[star_arr]).size == 0):\n",
    "        continue\n",
    "    ratio = num_zeros/np.array(y[star_arr]).size\n",
    "    # if y=zeros/ total lies in the region of [0.45,0.55] we include that in special stop words\n",
    "    if ratio >= 0.45 and ratio <= 0.55:\n",
    "        stop_words1.append(word)\n",
    "\n",
    "print(stop_words1)\n",
    "\n",
    "\n",
    "# review Text\n",
    "data = train_data[\"reviewText\"].astype(str).apply(edit_review_text)\n",
    "vectorizer2 = TfidfVectorizer()\n",
    "output2 = vectorizer2.fit_transform(data)\n",
    "dict2 = vectorizer2.vocabulary_\n",
    "# top 3000 words by frequency\n",
    "top_terms2 = sorted(dict2.keys(), key=lambda x: x[1], reverse=True)[:3000]\n",
    "feature_names2 = vectorizer2.get_feature_names_out()\n",
    "stop_words2 = []\n",
    "y = train_data['overall'].apply(lambda x: 0 if x <= cutoff else 1)\n",
    "\n",
    "# checking each word if it is relevant to the cutoff\n",
    "for word in top_terms2:\n",
    "    index = np.where(feature_names2 == word)[0]\n",
    "    star_arr = np.where(output2.todense()[:, index] != 0)[0]\n",
    "    num_zeros = np.count_nonzero(np.array(y[star_arr] == 0))\n",
    "    if (np.array(y[star_arr]).size == 0):\n",
    "        continue\n",
    "    ratio = num_zeros/np.array(y[star_arr]).size\n",
    "    # if y=zeros/ total lies in the region of [0.45,0.55] we include that in special stop words\n",
    "    if ratio >= 0.45 and ratio <= 0.55:\n",
    "        stop_words2.append(word)\n",
    "\n",
    "print(stop_words2)\n",
    "\n",
    "# Results\n",
    "\n",
    "\n",
    "# STOP_WORDS_SUMMARY = ['ozzy', 'oz', 'azoom', 'fyi', 'hybrid', 'syndrome', 'hydrated', '4yo', 'expected', 'excited', '2x', 'exposure', 'except', 'extreme', 'example', 'away', '2wd', 'swivel', 'awd', 'twisted', 'sweater', 'tweezers', 'swimming', 'twins', 'ever', 'uv', 'evil', 'overhyped', 'overseas', 'overlooked', 'evenly', 'evo', 'avid', 'overpaid', 'custom', 'much', 'quits', 'hurts', 'cushion', 'quality', 'customer', 'bux', 'multiple', 'bulbs', 'turned', 'supposed', 'guess', 'sunshade', 'cups', 'cut', 'sub', 'purchases', 'dunno', 'bunch', 'suv', 'quickly', 'dumb', 'functionally', 'kurt', 'substantial', 'sucked', 'pulled', 'gutter', 'outside', 'suggestion', 'rub', 'juice', 'turner', 'fuss', 'public', 'funniest', 'business', 'burn', 'hunter', 'suffers', 'outshines', 'puppy', 'qualities', 'autotune', 'guitars', 'outs', 'sudden', 'suits', 'subpar', 'putty', 'punish', 'puts', 'butterfly', 'judge', 'bubbles', 'authentic', 'guaranteed', 'jug', 'rum', 'gummies', 'rubbed', 'sugarfree', 'nutrition', 'nutritious', 'quinoa', 'sufferers', 'purposes', 'justify', 'lumpy', 'putting', 'burr', 'ounces', 'button', 'turbo', 'suffer', 'supplies', 'hunting', 'surplus', 'funko', 'numbers', 'cutie', 'pups', 'duplicates', 'puppets', 'item', 'attachment', 'starting', 'stick', 'stiff', 'step', 'strands', 'stretches', 'storm', 'gt', 'stretched', 'itll', 'street', 'wth', 'items', 'stainless', 'starters', 'atleast', 'stage', 'standards', 'stp', 'strange', 'stripped', 'strawberries', 'att', 'bt', 'steal', 'stroller', 'statue', 'use', 'asian', 'uses', 'usually', 'isuzu', 'issues', 'tsx', 'essence', 'ashlee', '5star', 'os', '3s', 'order', 'original', 'tried', 'cruiser', 'products', 'brake', 'pressure', 'trouble', 'try', 'trd', 'writing', 'arctic', 'driver', 'trailer', 'brand', 'probably', 'wrench', 'bristles', 'wrap', 'grounding', 'drive', 'process', 'error', 'area', 'protect', 'pro', 'transparent', 'prevent', 'transfer', 'traction', 'prevents', 'orignal', 'fraction', 'trance', 'breath', 'written', 'orbital', 'proud', 'previous', 'freaking', 'tripe', 'grow', 'travels', 'problematic', 'brothers', 'grace', 'brilliance', 'projects', 'tribute', 'priest', 'drinking', 'brandy', 'pretzel', 'grown', 'orleans', 'irons', 'dropping', 'programs', 'frustration', 'yr', 'fruits', 'artisan', 'fruitcake', 'trump', 'crumbled', 'grains', 'grapefruit', 'pretzels', 'oregano', 'brownie', 'truffles', 'trade', 'pricing', 'traditionally', 'mrs', 'produce', 'draft', 'crash', 'droid', 'arms', 'armorsuit', 'tricky', 'grippy', 'prints', 'ar', 'army', 'armour', 'drawstring', 'props', 'erase', 'articulation', 'breyer', 'drone', 'grandsons', 'artwork', 'grandaughter', 'dresses', 'equipped', 'squares', 'updated', 'sponge', 'spray', 'applications', 'update', 'operate', 'open', 'spark', 'tpms', 'rpms', 'specifications', 'appropriate', 'sports', 'split', 'options', 'spiced', 'appealing', '6p', 'ipad3', 'speaker', 'operates', 'appears']\n",
    "# STOP_WORDS_REVIEW_TEXT = ['ez', 'oza', 'ozzys', 'azalea', 'ozawa', 'ozzmosis', 'czech', 'izumi', 'typo', 'dye', 'fyi', 'eyourlife', 'sync', 'sympathetic', 'tyvek', 'tyc', 'eyeglasses', 'hyper', 'gy', 'synthetics', 'tying', 'synth', 'synths', 'cymbals', 'ryche', 'bytes', 'synthesizers', 'myung', 'wyclef', 'wynonna', 'lynyrd', 'hypocrisy', 'xy', 'lynne', 'mythical', 'cyndi', 'fye', 'cycling', 'dynasty', 'syrupy', 'ryders', 'systemic', 'byron', 'lynn', 'py', 'mystic', 'hynde', 'synthy', 'synchronization', 'dysfunctional', 'sydney', 'xylichew', 'synchronize', 'tylt', 'lysol', 'myselfi', 'symbols', '4yo', 'lycra', '7yr', '9yr', '7yo', 'tykes', 'ex', 'extremely', 'extreme', 'exact', 'explain', 'external', 'excursion', 'explained', 'exterior', 'ox', 'experiencing', 'exist', '2xl', 'existence', 'excessive', 'expert', 'expose', 'extendable', 'expectedwhich', 'expires', 'extending', 'expelled', 'expression', 'execs', 'experiencewhat', 'extinction', 'axe', 'excitment', 'exploited', 'executed', 'axemen', 'excerpt', 'mxpx', 'exiles', 'expendable', 'exhibit', 'exceptionthe', 'xx', 'exemplified', 'experiments', 'excite', 'exploits', 'expend', 'execrable', 'expansive', 'axes', 'exercised', 'explode', 'extol', '7x', 'example2', 'expectancy', 'extruded', 'expectedi', 'expat', 'expiry', 'experienceupdate', '9x9', 'expanders', '1x', '3x9', 'expectthe', 'expansions', 'two', 'away', 'twice', 'awg', 'twisting', 'tweezers', '4wd', 'switching', 'swivel', 'swear', '5w', 'twitchy', 'awkward', 'swirl', 'awning', 'twenty', 'gw', 'sweaty', 'awing', 'awe', 'iwe', 'pwc', 'swizz', 'bw', 'owes', 'tweaked', 'swings', 'swifts', 'swearing', 'twodisc', 'twinkie', 'swooping', 'awash', 'swells', 'awake', '3way', 'swine', 'sweetie', 'dwarfed', 'swoon', 'fw', 'tweedy', 'twothe', 'swords', 'awkwardly', 'twain', 'tweens', 'fword', 'swashbuckling', 'twominute', 'sweeps', 'awaken', 'sweats', 'twister', 'sweetener', 'sweeteners', 'swallows', 'twopart', 'awayit', 'twoyearold', 'even', 'dvd', 'ever', 'overtightened', 'everywhere', 'overheating', 'overkill', 'tv', 'evacuate', 'tvs', 'avoided', 'event', 'pvc', 'evidenced', 'oversexed', 'overdubbed', 'evangelical', 'evaporate', 'overuse', 'overprocessed', 'averages', 'evolutionary', 'overdo', 'everchanging', 'everclear', 'dvdif', 'avantgarde', 'overcompensates', 'avante', 'overbearing', 'oversell', 'everly', 'overproducing', 'oversees', 'evita', 'iverson', 'overated', 'evens', 'overstuffed', 'overcooked', 'overwhelms', 'lv', 'avocados', 'overpay', 'availablei', 'everhard', 'aviator', 'sview', 'evey', 'overfill', 'overpaid', 'guess', 'buy', 'buying', 'purchased', 'bump', 'dust', 'suddenly', 'suction', 'purchase', 'curved', 'auto', 'support', 'supposedly', 'turned', 'output', 'audio', 'pump', 'hurts', 'purchasing', 'nuts', 'bulbs', 'pulled', 'pushes', 'push', 'guys', 'buick', 'turn', 'fuses', 'huge', 'bux', 'rust', 'sufficiently', 'tubes', 'outlet', 'bubbles', 'bubble', 'suggests', 'outta', 'duralast', 'current', 'questions', 'turning', 'automotive', 'bunch', 'autos', 'suppose', 'purchasesthe', 'cure', 'rusty', 'multiple', 'bummer', 'luckily', 'pushed', 'fuseholder', 'rubicon', 'jumped', 'suzuki', 'quad', 'outupdatethe', 'guard', 'puddles', 'june', 'question', 'outright', 'bundled', 'sucksthe', 'bulks', 'purchasethis']\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['ozzy', 'oz', 'azoom', 'fyi', 'eyes', 'hybrid', 'lyrics', 'hydrated', '4yo', 'explorer', '2x', 'exposure', 'xxl', 'extreme', 'example', 'excellence', '2wd', 'owners', 'aware', 'awd', 'tweaking', 'twist', 'awaited', 'owe', 'twisted', 'sweater', 'tweezers', 'swimming', 'uv', 'rvs', 'everything', 'every', 'dvd', 'overlooked', 'avid', 'custom', 'cushion', 'super', 'durable', 'bux', 'multiple', 'duty', 'cute', 'pump', 'sunshade', 'purchases', 'suited', 'bunch', 'suv', 'fuses', 'fusion', 'buick', 'audi', 'funny', 'current', 'quads', 'quick', 'bumper', 'guide', 'gutter', 'public', 'business', 'audio', 'punk', 'guitars', 'outs', 'turkey', 'humor', 'buffett', 'bueno', 'sunday', 'buzz', 'judge', 'cube', 'buddy', 'submarine', 'guaranteed', 'jug', 'rum', 'gummies', 'rubbed', 'sugarfree', 'nutritious', 'quinoa', 'sufferers', 'gummy', 'justify', 'quest', 'cupcakes', 'quickie', 'punching', 'supposedly', 'ounces', 'turbo', 'substantially', 'suffer', 'supplies', 'building', 'numbers', 'cutie', 'puppets', 'hunt', 'outfits', 'still', 'strands', 'gt', 'stretched', 'xtreme', 'stores', 'utilitarian', 'stainless', 'starters', 'atleast', 'stage', 'standards', 'strength', 'stp', 'steve', 'styles', 'stages', 'stratovarius', 'steep', 'stevia', 'strawberries', 'strawberry', 'htc', 'bt', 'steal', 'study', 'stroller', 'stove', 'statue', 'isuzu', 'user', 'tsx', 'usa', 'essence', 'ashlee', 'aspect', 'bs', '5star', 'usb', 'aside', 'product', 'around', 'crew', 'grand', 'trd', 'armor', 'arctic', 'driver', 'wrench', 'gray', 'pricey', 'protection', 'trim', 'fresh', 'drilling', 'true', 'ground', 'crossbar', 'wrx', 'practicality', 'transfer', 'traction', 'prevents', 'orignal', 'fraction', 'prius', 'groove', 'breath', 'breakfast', 'orbital', 'proud', 'freaking', 'art', 'travels', 'grace', 'brilliance', 'group', 'iron', 'drinking', 'brigade', 'organized', 'primus', 'groundbreaking', 'critical', 'orchestration', 'dressed', 'travelling', 'kryptonite', 'orleans', 'irons', 'dropping', 'programs', 'frustration', 'yr', 'crackers', 'artisan', 'fruitcake', 'crumbled', 'grains', 'grapefruit', 'creamy', 'pretzels', 'brownie', 'freshness', 'crispy', 'breakable', 'drinkers', 'crisp', 'produce', 'grill', 'pricy', 'draft', 'crash', 'branded', 'wrapped', 'droid', 'protectors', 'armorsuit', 'orientation', 'preference', 'frills', 'tray', 'grippy', 'prints', 'vr', 'practice', 'armour', 'drawstring', 'trek', 'trains', 'dragonfly', 'tracker', 'trike', 'dryer', 'breyer', 'princess', 'grandsons', 'dresses', 'crayon', 'equipped', 'sponge', 'applications', 'operate', 'iphone', 'spot', 'specifications', 'appropriate', 'sports', 'approach', 'appetite', 'spell', 'spins', 'spiced', 'spices', 'spanish', '6p', 'ipad3', 'operates', 'speeds', 'iphone4', '3pack']\n",
      "['oza', 'azalea', 'ozawa', 'czech', 'gza', 'izumi', 'typical', 'types', 'eye', 'nylon', 'eyourlife', 'sympathetic', 'tyvek', 'tyc', 'gy', 'mynbspa', 'synthetics', 'tying', 'cycles', 'lyrical', 'symphonies', 'bytes', 'myth', 'myung', 'gypsys', 'wyclef', 'synthesizer', 'gym', 'lynyrd', 'mysterious', 'xy', 'lynne', 'cyndi', 'fye', 'dynasty', 'lynch', 'systemic', 'byron', 'syncing', 'lynn', 'mystic', 'nyc', 'wynton', 'eyez', 'synthpop', 'kylies', 'ky', 'kylie', 'myrath', 'lyles', 'lyle', 'cy', 'synergy', 'rye', 'synched', 'xylichew', 'cyanide', 'myselfoverall', 'synchronize', 'tylt', 'lysol', '4yo', 'kydex', 'lycra', '3yearold', '7yr', '9yr', '7yearold', 'tykes', '5yr', '5yo', '9yo', 'existing', 'extended', 'exists', 'exactly', '3x', 'extension', 'ox', 'exchange', 'expedition', 'excess', 'extendable', 'oxidation', 'extenders', 'gx470', 'expectedwhich', 'expires', 'exposure', 'extending', 'lx', 'exercises', 'exemplifies', 'experiencewhat', 'extinction', 'experimental', 'exercise', 'excitment', 'exploited', 'expendable', 'exhibit', 'exceptionthe', 'explicit', 'exemplified', 'exaggeration', 'execrable', 'expansive', 'expectationsi', 'examplethe', 'expressiveness', 'excessively', '7x', 'example2', 'expectedi', 'expat', 'expiry', 'experienceupdate', '1x', '3x9', 'axei', 'xxs', 'expander', 'expectthe', 'owning', 'aware', 'tweezers', 'twin', 'ww', '5w', 'awning', 'awing', 'swipe', 'iwe', 'swallow', 'switchi', 'pwc', 'tweaked', 'twinkie', 'swooping', 'awash', 'twang', 'sweeping', '3way', 'swine', 'sweetie', 'dwarfed', 'await', 'tweedy', 'twothe', 'twain', 'fword', 'swashbuckling', 'swamp', 'twominute', 'twists', 'sweeps', 'twentysomething', 'ewan', 'awaya', 'swims', 'swung', 'sweetit', 'sw', 'swallowed', 'swarovski', 'owls', 'awayit', 'swimmer', 'twoyearold', 'twoplayer', 'wwwthroatpunchgamescom', 'overall', 'rv', 'everything', 'overheating', 'oval', 'evacuate', 'tvs', 'everyday', 'eventhough', 'evap', 'wv', 'oversexed', 'everybodys', 'evaporate', 'overtones', 'averages', 'evolutionary', 'overdo', 'everchanging', 'overlooked', 'dvdif', 'overthetop', 'avante', 'oversell', 'everly', 'overproducing', 'evoke', 'evita', 'overjoyed', 'evolving', 'overcoming', 'av', 'evaluating', 'oversimplification', 'ivy', 'overthe', 'lv', 'overpay', 'overs', 'dv', 'ivory', 'avaiable', 'everhard', 'cvs', 'evo', 'aviator', 'sview', 'overfill', 'evaporation', 'everydayproduct', 'durable', 'custom', 'quart', 'pure', 'built', 'aux', 'tundra', 'quick', 'fuel', 'jumps', 'dual', 'super', 'hunting', 'survive', 'bulk', 'bux', 'outta', 'duralast', 'suv', 'suspension', 'autos', 'quarter', 'rundown', 'fun', 'mustang', 'fuseholder', 'rubicon', 'bug', 'cute', 'suzuki', 'bumps', 'suave', 'cushions', 'puddles', 'lunch', 'functionality', 'surrounding', 'bulks', 'purchasethis']\n"
     ]
    }
   ],
   "source": [
    "cutoff = 3\n",
    "\n",
    "# summary\n",
    "data = train_data[\"summary\"].astype(str).apply(edit_summary)\n",
    "vectorizer1 = TfidfVectorizer()\n",
    "output1 = vectorizer1.fit_transform(data)\n",
    "dict1 = vectorizer1.vocabulary_\n",
    "\n",
    "# top 3000 words by frequency\n",
    "top_terms1 = sorted(dict1.keys(), key=lambda x: x[1], reverse=True)[:3000]\n",
    "print(top_terms1)\n",
    "feature_names1 = vectorizer1.get_feature_names_out()\n",
    "stop_words1 = []\n",
    "y = train_data['overall'].apply(lambda x: 0 if x <= cutoff else 1)\n",
    "\n",
    "# checking each word if it is relevant to the cutoff\n",
    "for word in top_terms1:\n",
    "    index = np.where(feature_names1 == word)[0]\n",
    "    star_arr = np.where(output1.todense()[:, index] != 0)[0]\n",
    "    num_zeros = np.count_nonzero(np.array(y[star_arr] == 0))\n",
    "    if (np.array(y[star_arr]).size == 0):\n",
    "        continue\n",
    "    ratio = num_zeros/np.array(y[star_arr]).size\n",
    "    # if y=zeros/ total lies in the region of [0.45,0.55] we include that in special stop words\n",
    "    if ratio >= 0.45 and ratio <= 0.55:\n",
    "        stop_words1.append(word)\n",
    "\n",
    "print(stop_words1)\n",
    "\n",
    "\n",
    "# review Text\n",
    "data = train_data[\"reviewText\"].astype(str).apply(edit_review_text)\n",
    "vectorizer2 = TfidfVectorizer()\n",
    "output2 = vectorizer2.fit_transform(data)\n",
    "dict2 = vectorizer2.vocabulary_\n",
    "# top 3000 words by frequency\n",
    "top_terms2 = sorted(dict2.keys(), key=lambda x: x[1], reverse=True)[:3000]\n",
    "feature_names2 = vectorizer2.get_feature_names_out()\n",
    "stop_words2 = []\n",
    "y = train_data['overall'].apply(lambda x: 0 if x <= cutoff else 1)\n",
    "\n",
    "# checking each word if it is relevant to the cutoff\n",
    "for word in top_terms2:\n",
    "    index = np.where(feature_names2 == word)[0]\n",
    "    star_arr = np.where(output2.todense()[:, index] != 0)[0]\n",
    "    num_zeros = np.count_nonzero(np.array(y[star_arr] == 0))\n",
    "    if (np.array(y[star_arr]).size == 0):\n",
    "        continue\n",
    "    ratio = num_zeros/np.array(y[star_arr]).size\n",
    "    # if y=zeros/ total lies in the region of [0.45,0.55] we include that in special stop words\n",
    "    if ratio >= 0.45 and ratio <= 0.55:\n",
    "        stop_words2.append(word)\n",
    "\n",
    "print(stop_words2)\n",
    "\n",
    "# Results\n",
    "\n",
    "# STOP_WORDS_SUMMARY = ['ozzy', 'oz', 'azoom', 'fyi', 'eyes', 'hybrid', 'lyrics', 'hydrated', '4yo', 'explorer', '2x', 'exposure', 'xxl', 'extreme', 'example', 'excellence', '2wd', 'owners', 'aware', 'awd', 'tweaking', 'twist', 'awaited', 'owe', 'twisted', 'sweater', 'tweezers', 'swimming', 'uv', 'rvs', 'everything', 'every', 'dvd', 'overlooked', 'avid', 'custom', 'cushion', 'super', 'durable', 'bux', 'multiple', 'duty', 'cute', 'pump', 'sunshade', 'purchases', 'suited', 'bunch', 'suv', 'fuses', 'fusion', 'buick', 'audi', 'funny', 'current', 'quads', 'quick', 'bumper', 'guide', 'gutter', 'public', 'business', 'audio', 'punk', 'guitars', 'outs', 'turkey', 'humor', 'buffett', 'bueno', 'sunday', 'buzz', 'judge', 'cube', 'buddy', 'submarine', 'guaranteed', 'jug', 'rum', 'gummies', 'rubbed', 'sugarfree', 'nutritious', 'quinoa', 'sufferers', 'gummy', 'justify', 'quest', 'cupcakes', 'quickie', 'punching', 'supposedly', 'ounces', 'turbo', 'substantially', 'suffer', 'supplies', 'building', 'numbers', 'cutie', 'puppets', 'hunt', 'outfits', 'still', 'strands', 'gt', 'stretched', 'xtreme', 'stores', 'utilitarian', 'stainless', 'starters', 'atleast', 'stage', 'standards', 'strength', 'stp', 'steve', 'styles', 'stages', 'stratovarius', 'steep', 'stevia', 'strawberries', 'strawberry', 'htc', 'bt', 'steal', 'study', 'stroller', 'stove', 'statue', 'isuzu', 'user', 'tsx', 'usa', 'essence', 'ashlee', 'aspect', 'bs', '5star', 'usb', 'aside', 'product', 'around', 'crew', 'grand', 'trd', 'armor', 'arctic', 'driver', 'wrench', 'gray', 'pricey', 'protection', 'trim', 'fresh', 'drilling', 'true', 'ground', 'crossbar', 'wrx', 'practicality', 'transfer', 'traction', 'prevents', 'orignal', 'fraction', 'prius', 'groove', 'breath', 'breakfast', 'orbital', 'proud', 'freaking', 'art', 'travels', 'grace', 'brilliance', 'group', 'iron', 'drinking', 'brigade', 'organized', 'primus', 'groundbreaking', 'critical', 'orchestration', 'dressed', 'travelling', 'kryptonite', 'orleans', 'irons', 'dropping', 'programs', 'frustration', 'yr', 'crackers', 'artisan', 'fruitcake', 'crumbled', 'grains', 'grapefruit', 'creamy', 'pretzels', 'brownie', 'freshness', 'crispy', 'breakable', 'drinkers', 'crisp', 'produce', 'grill', 'pricy', 'draft', 'crash', 'branded', 'wrapped', 'droid', 'protectors', 'armorsuit', 'orientation', 'preference', 'frills', 'tray', 'grippy', 'prints', 'vr', 'practice', 'armour', 'drawstring', 'trek', 'trains', 'dragonfly', 'tracker', 'trike', 'dryer', 'breyer', 'princess', 'grandsons', 'dresses', 'crayon', 'equipped', 'sponge', 'applications', 'operate', 'iphone', 'spot', 'specifications', 'appropriate', 'sports', 'approach', 'appetite', 'spell', 'spins', 'spiced', 'spices', 'spanish', '6p', 'ipad3', 'operates', 'speeds', 'iphone4', '3pack']\n",
    "# STOP_WORDS_REVIEW_TEXT = ['oza', 'azalea', 'ozawa', 'czech', 'gza', 'izumi', 'typical', 'types', 'eye', 'nylon', 'eyourlife', 'sympathetic', 'tyvek', 'tyc', 'gy', 'mynbspa', 'synthetics', 'tying', 'cycles', 'lyrical', 'symphonies', 'bytes', 'myth', 'myung', 'gypsys', 'wyclef', 'synthesizer', 'gym', 'lynyrd', 'mysterious', 'xy', 'lynne', 'cyndi', 'fye', 'dynasty', 'lynch', 'systemic', 'byron', 'syncing', 'lynn', 'mystic', 'nyc', 'wynton', 'eyez', 'synthpop', 'kylies', 'ky', 'kylie', 'myrath', 'lyles', 'lyle', 'cy', 'synergy', 'rye', 'synched', 'xylichew', 'cyanide', 'myselfoverall', 'synchronize', 'tylt', 'lysol', '4yo', 'kydex', 'lycra', '3yearold', '7yr', '9yr', '7yearold', 'tykes', '5yr', '5yo', '9yo', 'existing', 'extended', 'exists', 'exactly', '3x', 'extension', 'ox', 'exchange', 'expedition', 'excess', 'extendable', 'oxidation', 'extenders', 'gx470', 'expectedwhich', 'expires', 'exposure', 'extending', 'lx', 'exercises', 'exemplifies', 'experiencewhat', 'extinction', 'experimental', 'exercise', 'excitment', 'exploited', 'expendable', 'exhibit', 'exceptionthe', 'explicit', 'exemplified', 'exaggeration', 'execrable', 'expansive', 'expectationsi', 'examplethe', 'expressiveness', 'excessively', '7x', 'example2', 'expectedi', 'expat', 'expiry', 'experienceupdate', '1x', '3x9', 'axei', 'xxs', 'expander', 'expectthe', 'owning', 'aware', 'tweezers', 'twin', 'ww', '5w', 'awning', 'awing', 'swipe', 'iwe', 'swallow', 'switchi', 'pwc', 'tweaked', 'twinkie', 'swooping', 'awash', 'twang', 'sweeping', '3way', 'swine', 'sweetie', 'dwarfed', 'await', 'tweedy', 'twothe', 'twain', 'fword', 'swashbuckling', 'swamp', 'twominute', 'twists', 'sweeps', 'twentysomething', 'ewan', 'awaya', 'swims', 'swung', 'sweetit', 'sw', 'swallowed', 'swarovski', 'owls', 'awayit', 'swimmer', 'twoyearold', 'twoplayer', 'wwwthroatpunchgamescom', 'overall', 'rv', 'everything', 'overheating', 'oval', 'evacuate', 'tvs', 'everyday', 'eventhough', 'evap', 'wv', 'oversexed', 'everybodys', 'evaporate', 'overtones', 'averages', 'evolutionary', 'overdo', 'everchanging', 'overlooked', 'dvdif', 'overthetop', 'avante', 'oversell', 'everly', 'overproducing', 'evoke', 'evita', 'overjoyed', 'evolving', 'overcoming', 'av', 'evaluating', 'oversimplification', 'ivy', 'overthe', 'lv', 'overpay', 'overs', 'dv', 'ivory', 'avaiable', 'everhard', 'cvs', 'evo', 'aviator', 'sview', 'overfill', 'evaporation', 'everydayproduct', 'durable', 'custom', 'quart', 'pure', 'built', 'aux', 'tundra', 'quick', 'fuel', 'jumps', 'dual', 'super', 'hunting', 'survive', 'bulk', 'bux', 'outta', 'duralast', 'suv', 'suspension', 'autos', 'quarter', 'rundown', 'fun', 'mustang', 'fuseholder', 'rubicon', 'bug', 'cute', 'suzuki', 'bumps', 'suave', 'cushions', 'puddles', 'lunch', 'functionality', 'surrounding', 'bulks', 'purchasethis']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['hyundai', 'eyed', '4yo', 'cycling', '2x', 'excellence', 'excelent', 'exquisite', '2wd', 'awsome', 'awhile', 'awaited', 'owe', 'twisted', 'swimming', 'rvs', 'overlooked', 'sunshade', 'purchases', 'subaru', 'buick', 'audi', 'quads', 'tuner', 'tune', 'gutter', 'junior', 'musical', 'guitars', 'funk', 'humor', 'bueno', 'sunday', 'buzz', 'dude', 'hurrah', 'guessing', 'junkie', 'jug', 'rum', 'rubbed', 'nutritious', 'muffins', 'turmeric', 'hunters', 'supposedly', 'ounces', 'turbo', 'suffer', 'supplies', 'cutie', 'puppets', 'cuddly', 'gt', 'utilitarian', 'attaches', 'stroke', 'starters', 'standards', '6th', 'stretching', 'stream', 'bt', 'stout', 'strategy', 'isuzu', 'tsx', 'usefull', 'usa', 'ass', 'bs', 'crew', 'trd', 'drilling', 'transmission', 'draw', 'brother', 'traction', 'prevents', 'orignal', 'fraction', 'prius', 'groove', 'proud', 'grace', 'brilliance', 'brigade', 'orchestration', 'treasure', 'previously', 'irons', 'franklin', 'programs', 'frustration', 'protein', 'artisan', 'crumbled', 'grains', 'grapefruit', 'drinkers', 'crisp', 'breads', 'brewing', 'creme', 'branded', 'wrapped', 'armorsuit', 'orientation', 'preference', 'tray', 'prints', 'vr', 'armour', 'dragonfly', 'dryer', 'breyer', 'dresses', 'promo', 'transformers', 'spread', 'appetite', 'ipod', 'spices', 'spanish', 'operates', 'speeds', 'iphone4']\n",
      "['oza', 'rzr', 'azalea', 'czech', 'gza', 'eyourlife', 'tyc', 'ty', 'synthetics', 'dynasty', 'systemic', 'lynn', 'gymnastics', 'wynton', 'eyez', 'kylies', 'ky', 'myrath', 'lyles', 'synergy', 'ayreon', 'nyro', 'hypocritical', 'hybrids', 'sycamore', 'rythm', 'lyon', 'symmetry', 'hydrate', 'myselfoverall', 'synchronize', 'lysol', 'cycled', 'eyesight', '4yo', '5yo', '9yo', '6year', 'ox', 'expedition', 'gx470', 'fx4', 'lx', 'exemplifies', 'exemplified', 'exquisite', 'expectationsi', 'examplethe', 'explodes', 'exhort', '7x', 'example2', 'expectedi', 'expat', 'expiry', 'experiencei', '9x', '3x9', '5w', 'awning', 'sweetie', 'twain', 'fword', 'swashbuckling', 'twentysomething', 'ewan', 'awaya', 'twochannel', 'twitching', 'dwells', 'sweetit', 'swallowed', 'sweetens', 'owls', 'twoyearold', 'twoplayer', 'wwwthroatpunchgamescom', 'overcharging', 'tvs', 'eventhough', 'evap', 'aviation', 'overdrive', 'evaporate', 'everchanging', 'dvdif', 'evoking', 'events', 'overcoming', 'av', 'evergrowing', 'evergreen', 'overcomes', 'dv', 'cvs', 'aviator', 'overtheear', 'averagesized', 'everydayproduct', 'bux', 'duralast', 'lubrication', 'puddles', 'purchasethis']\n"
     ]
    }
   ],
   "source": [
    "cutoff = 4\n",
    "\n",
    "# summary\n",
    "data = train_data[\"summary\"].astype(str).apply(edit_summary)\n",
    "vectorizer1 = TfidfVectorizer()\n",
    "output1 = vectorizer1.fit_transform(data)\n",
    "dict1 = vectorizer1.vocabulary_\n",
    "\n",
    "# top 3000 words by frequency\n",
    "top_terms1 = sorted(dict1.keys(), key=lambda x: x[1], reverse=True)[:3000]\n",
    "print(top_terms1)\n",
    "feature_names1 = vectorizer1.get_feature_names_out()\n",
    "stop_words1 = []\n",
    "y = train_data['overall'].apply(lambda x: 0 if x <= cutoff else 1)\n",
    "\n",
    "# checking each word if it is relevant to the cutoff\n",
    "for word in top_terms1:\n",
    "    index = np.where(feature_names1 == word)[0]\n",
    "    star_arr = np.where(output1.todense()[:, index] != 0)[0]\n",
    "    num_zeros = np.count_nonzero(np.array(y[star_arr] == 0))\n",
    "    if (np.array(y[star_arr]).size == 0):\n",
    "        continue\n",
    "    ratio = num_zeros/np.array(y[star_arr]).size\n",
    "    # if y=zeros/ total lies in the region of [0.45,0.55] we include that in special stop words\n",
    "    if ratio >= 0.45 and ratio <= 0.55:\n",
    "        stop_words1.append(word)\n",
    "\n",
    "print(stop_words1)\n",
    "\n",
    "\n",
    "# review Text\n",
    "data = train_data[\"reviewText\"].astype(str).apply(edit_review_text)\n",
    "vectorizer2 = TfidfVectorizer()\n",
    "output2 = vectorizer2.fit_transform(data)\n",
    "dict2 = vectorizer2.vocabulary_\n",
    "# top 3000 words by frequency\n",
    "top_terms2 = sorted(dict2.keys(), key=lambda x: x[1], reverse=True)[:3000]\n",
    "feature_names2 = vectorizer2.get_feature_names_out()\n",
    "stop_words2 = []\n",
    "y = train_data['overall'].apply(lambda x: 0 if x <= cutoff else 1)\n",
    "\n",
    "# checking each word if it is relevant to the cutoff\n",
    "for word in top_terms2:\n",
    "    index = np.where(feature_names2 == word)[0]\n",
    "    star_arr = np.where(output2.todense()[:, index] != 0)[0]\n",
    "    num_zeros = np.count_nonzero(np.array(y[star_arr] == 0))\n",
    "    if (np.array(y[star_arr]).size == 0):\n",
    "        continue\n",
    "    ratio = num_zeros/np.array(y[star_arr]).size\n",
    "    # if y=zeros/ total lies in the region of [0.45,0.55] we include that in special stop words\n",
    "    if ratio >= 0.45 and ratio <= 0.55:\n",
    "        stop_words2.append(word)\n",
    "\n",
    "print(stop_words2)\n",
    "\n",
    "# Results\n",
    "\n",
    "\n",
    "# STOP_WORDS_SUMMARY = ['hyundai', 'eyed', '4yo', 'cycling', '2x', 'excellence', 'excelent', 'exquisite', '2wd', 'awsome', 'awhile', 'awaited', 'owe', 'twisted', 'swimming', 'rvs', 'overlooked', 'sunshade', 'purchases', 'subaru', 'buick', 'audi', 'quads', 'tuner', 'tune', 'gutter', 'junior', 'musical', 'guitars', 'funk', 'humor', 'bueno', 'sunday', 'buzz', 'dude', 'hurrah', 'guessing', 'junkie', 'jug', 'rum', 'rubbed', 'nutritious', 'muffins', 'turmeric', 'hunters', 'supposedly', 'ounces', 'turbo', 'suffer', 'supplies', 'cutie', 'puppets', 'cuddly', 'gt', 'utilitarian', 'attaches', 'stroke', 'starters', 'standards', '6th', 'stretching', 'stream', 'bt', 'stout', 'strategy', 'isuzu', 'tsx', 'usefull', 'usa', 'ass', 'bs', 'crew', 'trd', 'drilling', 'transmission', 'draw', 'brother', 'traction', 'prevents', 'orignal', 'fraction', 'prius', 'groove', 'proud', 'grace', 'brilliance', 'brigade', 'orchestration', 'treasure', 'previously', 'irons', 'franklin', 'programs', 'frustration', 'protein', 'artisan', 'crumbled', 'grains', 'grapefruit', 'drinkers', 'crisp', 'breads', 'brewing', 'creme', 'branded', 'wrapped', 'armorsuit', 'orientation', 'preference', 'tray', 'prints', 'vr', 'armour', 'dragonfly', 'dryer', 'breyer', 'dresses', 'promo', 'transformers', 'spread', 'appetite', 'ipod', 'spices', 'spanish', 'operates', 'speeds', 'iphone4']\n",
    "# STOP_WORDS_REVIEW_TEXT = ['oza', 'rzr', 'azalea', 'czech', 'gza', 'eyourlife', 'tyc', 'ty', 'synthetics', 'dynasty', 'systemic', 'lynn', 'gymnastics', 'wynton', 'eyez', 'kylies', 'ky', 'myrath', 'lyles', 'synergy', 'ayreon', 'nyro', 'hypocritical', 'hybrids', 'sycamore', 'rythm', 'lyon', 'symmetry', 'hydrate', 'myselfoverall', 'synchronize', 'lysol', 'cycled', 'eyesight', '4yo', '5yo', '9yo', '6year', 'ox', 'expedition', 'gx470', 'fx4', 'lx', 'exemplifies', 'exemplified', 'exquisite', 'expectationsi', 'examplethe', 'explodes', 'exhort', '7x', 'example2', 'expectedi', 'expat', 'expiry', 'experiencei', '9x', '3x9', '5w', 'awning', 'sweetie', 'twain', 'fword', 'swashbuckling', 'twentysomething', 'ewan', 'awaya', 'twochannel', 'twitching', 'dwells', 'sweetit', 'swallowed', 'sweetens', 'owls', 'twoyearold', 'twoplayer', 'wwwthroatpunchgamescom', 'overcharging', 'tvs', 'eventhough', 'evap', 'aviation', 'overdrive', 'evaporate', 'everchanging', 'dvdif', 'evoking', 'events', 'overcoming', 'av', 'evergrowing', 'evergreen', 'overcomes', 'dv', 'cvs', 'aviator', 'overtheear', 'averagesized', 'everydayproduct', 'bux', 'duralast', 'lubrication', 'puddles', 'purchasethis']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['hyundai', 'eyed', '4yo', 'cycling', '2x', 'excellence', 'excelent', 'exquisite', '2wd', 'awsome', 'awhile', 'awaited', 'owe', 'twisted', 'swimming', 'rvs', 'overlooked', 'sunshade', 'purchases', 'subaru', 'buick', 'audi', 'quads', 'tuner', 'tune', 'gutter', 'junior', 'musical', 'guitars', 'funk', 'humor', 'bueno', 'sunday', 'buzz', 'dude', 'hurrah', 'guessing', 'junkie', 'jug', 'rum', 'rubbed', 'nutritious', 'muffins', 'turmeric', 'hunters', 'supposedly', 'ounces', 'turbo', 'suffer', 'supplies', 'cutie', 'puppets', 'cuddly', 'gt', 'utilitarian', 'attaches', 'stroke', 'starters', 'standards', '6th', 'stretching', 'stream', 'bt', 'stout', 'strategy', 'isuzu', 'tsx', 'usefull', 'usa', 'ass', 'bs', 'crew', 'trd', 'drilling', 'transmission', 'draw', 'brother', 'traction', 'prevents', 'orignal', 'fraction', 'prius', 'groove', 'proud', 'grace', 'brilliance', 'brigade', 'orchestration', 'treasure', 'previously', 'irons', 'franklin', 'programs', 'frustration', 'protein', 'artisan', 'crumbled', 'grains', 'grapefruit', 'drinkers', 'crisp', 'breads', 'brewing', 'creme', 'branded', 'wrapped', 'armorsuit', 'orientation', 'preference', 'tray', 'prints', 'vr', 'armour', 'dragonfly', 'dryer', 'breyer', 'dresses', 'promo', 'transformers', 'spread', 'appetite', 'ipod', 'spices', 'spanish', 'operates', 'speeds', 'iphone4']\n",
      "['oza', 'rzr', 'azalea', 'czech', 'gza', 'eyourlife', 'tyc', 'ty', 'synthetics', 'dynasty', 'systemic', 'lynn', 'gymnastics', 'wynton', 'eyez', 'kylies', 'ky', 'myrath', 'lyles', 'synergy', 'ayreon', 'nyro', 'hypocritical', 'hybrids', 'sycamore', 'rythm', 'lyon', 'symmetry', 'hydrate', 'myselfoverall', 'synchronize', 'lysol', 'cycled', 'eyesight', '4yo', '5yo', '9yo', '6year', 'ox', 'expedition', 'gx470', 'fx4', 'lx', 'exemplifies', 'exemplified', 'exquisite', 'expectationsi', 'examplethe', 'explodes', 'exhort', '7x', 'example2', 'expectedi', 'expat', 'expiry', 'experiencei', '9x', '3x9', '5w', 'awning', 'sweetie', 'twain', 'fword', 'swashbuckling', 'twentysomething', 'ewan', 'awaya', 'twochannel', 'twitching', 'dwells', 'sweetit', 'swallowed', 'sweetens', 'owls', 'twoyearold', 'twoplayer', 'wwwthroatpunchgamescom', 'overcharging', 'tvs', 'eventhough', 'evap', 'aviation', 'overdrive', 'evaporate', 'everchanging', 'dvdif', 'evoking', 'events', 'overcoming', 'av', 'evergrowing', 'evergreen', 'overcomes', 'dv', 'cvs', 'aviator', 'overtheear', 'averagesized', 'everydayproduct', 'bux', 'duralast', 'lubrication', 'puddles', 'purchasethis']\n"
     ]
    }
   ],
   "source": [
    "# For multi classification\n",
    "\n",
    "\n",
    "# summary\n",
    "data = train_data[\"summary\"].astype(str).apply(edit_summary)\n",
    "vectorizer1 = TfidfVectorizer()\n",
    "output1 = vectorizer1.fit_transform(data)\n",
    "dict1 = vectorizer1.vocabulary_\n",
    "\n",
    "# top 3000 words by frequency\n",
    "top_terms1 = sorted(dict1.keys(), key=lambda x: x[1], reverse=True)[:3000]\n",
    "print(top_terms1)\n",
    "feature_names1 = vectorizer1.get_feature_names_out()\n",
    "stop_words1 = []\n",
    "y = train_data['overall'].apply(lambda x: 0 if x <= cutoff else 1)\n",
    "\n",
    "# checking each word if it is relevant to the cutoff\n",
    "for word in top_terms1:\n",
    "    index = np.where(feature_names1 == word)[0]\n",
    "    star_arr = np.where(output1.todense()[:, index] != 0)[0]\n",
    "    num_zeros = np.count_nonzero(np.array(y[star_arr] == 0))\n",
    "    if (np.array(y[star_arr]).size == 0):\n",
    "        continue\n",
    "    ratio = num_zeros/np.array(y[star_arr]).size\n",
    "    # if y=zeros/ total lies in the region of [0.45,0.55] we include that in special stop words\n",
    "    if ratio >= 0.45 and ratio <= 0.55:\n",
    "        stop_words1.append(word)\n",
    "\n",
    "print(stop_words1)\n",
    "\n",
    "\n",
    "# review Text\n",
    "data = train_data[\"reviewText\"].astype(str).apply(edit_review_text)\n",
    "vectorizer2 = TfidfVectorizer()\n",
    "output2 = vectorizer2.fit_transform(data)\n",
    "dict2 = vectorizer2.vocabulary_\n",
    "# top 3000 words by frequency\n",
    "top_terms2 = sorted(dict2.keys(), key=lambda x: x[1], reverse=True)[:3000]\n",
    "feature_names2 = vectorizer2.get_feature_names_out()\n",
    "stop_words2 = []\n",
    "y = train_data['overall'].apply(lambda x: 0 if x <= cutoff else 1)\n",
    "\n",
    "# checking each word if it is relevant to the cutoff\n",
    "for word in top_terms2:\n",
    "    index = np.where(feature_names2 == word)[0]\n",
    "    star_arr = np.where(output2.todense()[:, index] != 0)[0]\n",
    "    num_zeros = np.count_nonzero(np.array(y[star_arr] == 0))\n",
    "    if (np.array(y[star_arr]).size == 0):\n",
    "        continue\n",
    "    ratio = num_zeros/np.array(y[star_arr]).size\n",
    "    # if y=zeros/ total lies in the region of [0.45,0.55] we include that in special stop words\n",
    "    if ratio >= 0.45 and ratio <= 0.55:\n",
    "        stop_words2.append(word)\n",
    "\n",
    "print(stop_words2)\n",
    "\n",
    "# Results\n",
    "\n",
    "\n",
    "# STOP_WORDS_SUMMARY = ['hyundai', 'eyed', '4yo', 'cycling', '2x', 'excellence', 'excelent', 'exquisite', '2wd', 'awsome', 'awhile', 'awaited', 'owe', 'twisted', 'swimming', 'rvs', 'overlooked', 'sunshade', 'purchases', 'subaru', 'buick', 'audi', 'quads', 'tuner', 'tune', 'gutter', 'junior', 'musical', 'guitars', 'funk', 'humor', 'bueno', 'sunday', 'buzz', 'dude', 'hurrah', 'guessing', 'junkie', 'jug', 'rum', 'rubbed', 'nutritious', 'muffins', 'turmeric', 'hunters', 'supposedly', 'ounces', 'turbo', 'suffer', 'supplies', 'cutie', 'puppets', 'cuddly', 'gt', 'utilitarian', 'attaches', 'stroke', 'starters', 'standards', '6th', 'stretching', 'stream', 'bt', 'stout', 'strategy', 'isuzu', 'tsx', 'usefull', 'usa', 'ass', 'bs', 'crew', 'trd', 'drilling', 'transmission', 'draw', 'brother', 'traction', 'prevents', 'orignal', 'fraction', 'prius', 'groove', 'proud', 'grace', 'brilliance', 'brigade', 'orchestration', 'treasure', 'previously', 'irons', 'franklin', 'programs', 'frustration', 'protein', 'artisan', 'crumbled', 'grains', 'grapefruit', 'drinkers', 'crisp', 'breads', 'brewing', 'creme', 'branded', 'wrapped', 'armorsuit', 'orientation', 'preference', 'tray', 'prints', 'vr', 'armour', 'dragonfly', 'dryer', 'breyer', 'dresses', 'promo', 'transformers', 'spread', 'appetite', 'ipod', 'spices', 'spanish', 'operates', 'speeds', 'iphone4']\n",
    "# STOP_WORDS_REVIEW_TEXT = ['oza', 'rzr', 'azalea', 'czech', 'gza', 'eyourlife', 'tyc', 'ty', 'synthetics', 'dynasty', 'systemic', 'lynn', 'gymnastics', 'wynton', 'eyez', 'kylies', 'ky', 'myrath', 'lyles', 'synergy', 'ayreon', 'nyro', 'hypocritical', 'hybrids', 'sycamore', 'rythm', 'lyon', 'symmetry', 'hydrate', 'myselfoverall', 'synchronize', 'lysol', 'cycled', 'eyesight', '4yo', '5yo', '9yo', '6year', 'ox', 'expedition', 'gx470', 'fx4', 'lx', 'exemplifies', 'exemplified', 'exquisite', 'expectationsi', 'examplethe', 'explodes', 'exhort', '7x', 'example2', 'expectedi', 'expat', 'expiry', 'experiencei', '9x', '3x9', '5w', 'awning', 'sweetie', 'twain', 'fword', 'swashbuckling', 'twentysomething', 'ewan', 'awaya', 'twochannel', 'twitching', 'dwells', 'sweetit', 'swallowed', 'sweetens', 'owls', 'twoyearold', 'twoplayer', 'wwwthroatpunchgamescom', 'overcharging', 'tvs', 'eventhough', 'evap', 'aviation', 'overdrive', 'evaporate', 'everchanging', 'dvdif', 'evoking', 'events', 'overcoming', 'av', 'evergrowing', 'evergreen', 'overcomes', 'dv', 'cvs', 'aviator', 'overtheear', 'averagesized', 'everydayproduct', 'bux', 'duralast', 'lubrication', 'puddles', 'purchasethis']"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.2"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "vscode": {
   "interpreter": {
    "hash": "7b70de28ce7f16dabb42d6daca88e435fe373ac8e95c52d0bdf357ad057a0126"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}