-
Notifications
You must be signed in to change notification settings - Fork 11
/
preprocessing.py
43 lines (34 loc) · 1.15 KB
/
preprocessing.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
import pandas as pd
import re, nltk
from tqdm import tqdm
from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize
from nltk.stem.lancaster import LancasterStemmer
lancaster_stemmer = LancasterStemmer()
from nltk.stem import WordNetLemmatizer
wordnet_lemmatizer = WordNetLemmatizer()
def take_data_to_shower(tweet):
noises = ['URL', '@USER', '\'ve', 'n\'t', '\'s', '\'m']
for noise in noises:
tweet = tweet.replace(noise, '')
return re.sub(r'[^a-zA-Z]', ' ', tweet)
def tokenize(tweet):
lower_tweet = tweet.lower()
return word_tokenize(lower_tweet)
def remove_stop_words(tokens):
clean_tokens = []
stopWords = set(stopwords.words('english'))
for token in tokens:
if token not in stopWords:
if token.replace(' ', '') != '':
if len(token) > 1:
clean_tokens.append(token)
return clean_tokens
def stem_and_lem(tokens):
clean_tokens = []
for token in tokens:
token = wordnet_lemmatizer.lemmatize(token)
token = lancaster_stemmer.stem(token)
if len(token) > 1:
clean_tokens.append(token)
return clean_tokens