HariniP_ML_descriptive_surveydata.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#importing basic lib\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import warnings\n",
    "warnings.simplefilter('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#importing neccessary libs\n",
    "from sklearn import preprocessing\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.tree import DecisionTreeClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "filename = 'color_survey_answers.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "colours = pd.DataFrame()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "colours = pd.read_csv(filename , delimiter='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>user_id</th>\n",
       "      <th>datestamp</th>\n",
       "      <th>r</th>\n",
       "      <th>g</th>\n",
       "      <th>b</th>\n",
       "      <th>colorname</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1.267419e+09</td>\n",
       "      <td>72</td>\n",
       "      <td>100</td>\n",
       "      <td>175</td>\n",
       "      <td>pastel blue</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1.267419e+09</td>\n",
       "      <td>204</td>\n",
       "      <td>177</td>\n",
       "      <td>246</td>\n",
       "      <td>faint violet</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1.267419e+09</td>\n",
       "      <td>182</td>\n",
       "      <td>226</td>\n",
       "      <td>245</td>\n",
       "      <td>baby blue</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>1.267419e+09</td>\n",
       "      <td>130</td>\n",
       "      <td>64</td>\n",
       "      <td>234</td>\n",
       "      <td>purple</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>1.267419e+09</td>\n",
       "      <td>75</td>\n",
       "      <td>49</td>\n",
       "      <td>234</td>\n",
       "      <td>blue</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  user_id     datestamp    r    g    b     colorname\n",
       "0   1        1  1.267419e+09   72  100  175   pastel blue\n",
       "1   2        1  1.267419e+09  204  177  246  faint violet\n",
       "2   3        1  1.267419e+09  182  226  245     baby blue\n",
       "3   4        1  1.267419e+09  130   64  234        purple\n",
       "4   5        2  1.267419e+09   75   49  234          blue"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "colours.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 3363127 entries, 0 to 3363126\n",
      "Data columns (total 7 columns):\n",
      "id           int64\n",
      "user_id      int64\n",
      "datestamp    float64\n",
      "r            int64\n",
      "g            int64\n",
      "b            int64\n",
      "colorname    object\n",
      "dtypes: float64(1), int64(5), object(1)\n",
      "memory usage: 179.6+ MB\n"
     ]
    }
   ],
   "source": [
    "colours.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#converting date from float to datetime\n",
    "from datetime import datetime\n",
    "colours['datestamp'] = pd.to_datetime(colours['datestamp'],unit='s')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "#extracting new colours into another dataframe from original\n",
    "new_colour = pd.DataFrame()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_colour = colours[colours['colorname'] == 'impatiens juice'].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_colour = colours[colours['colorname'] == 'caribbean surf'].append(new_colour)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_colour = colours[colours['colorname'] == 'medium tan leather'].append(new_colour)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_colour = colours[colours['colorname'] == 'babypoo'].append(new_colour)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "#increaing the occurance of new colours\n",
    "new_colour = new_colour.append([new_colour]*10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 88 entries, 185291 to 3170468\n",
      "Data columns (total 7 columns):\n",
      "id           88 non-null int64\n",
      "user_id      88 non-null int64\n",
      "datestamp    88 non-null datetime64[ns]\n",
      "r            88 non-null int64\n",
      "g            88 non-null int64\n",
      "b            88 non-null int64\n",
      "colorname    88 non-null object\n",
      "dtypes: datetime64[ns](1), int64(5), object(1)\n",
      "memory usage: 5.5+ KB\n"
     ]
    }
   ],
   "source": [
    "new_colour.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "#filtering out unwanted colours and colours which have occurance of 10000 or less\n",
    "colours = colours.groupby('colorname').filter(lambda x: len(x) > 10000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "#combining new colours to the filtered original dataset\n",
    "colours = pd.concat([colours,new_colour],ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "green                 310064\n",
       "blue                  284220\n",
       "purple                245832\n",
       "pink                  129286\n",
       "brown                  74782\n",
       "red                    68985\n",
       "light blue             57381\n",
       "teal                   56336\n",
       "orange                 51449\n",
       "light green            49073\n",
       "yellow                 43494\n",
       "magenta                43139\n",
       "grey                   35664\n",
       "sky blue               34043\n",
       "violet                 32407\n",
       "lime green             30918\n",
       "light purple           29221\n",
       "turquoise              25925\n",
       "lavender               25447\n",
       "cyan                   24421\n",
       "tan                    23710\n",
       "dark green             23624\n",
       "dark blue              23116\n",
       "aqua                   23036\n",
       "forest green           19112\n",
       "mauve                  18899\n",
       "bright green           18578\n",
       "gray                   18003\n",
       "olive                  17339\n",
       "dark purple            16653\n",
       "maroon                 16174\n",
       "lilac                  15539\n",
       "royal blue             14875\n",
       "salmon                 14330\n",
       "black                  14159\n",
       "beige                  14151\n",
       "hot pink               13954\n",
       "navy blue              13431\n",
       "pale green             13297\n",
       "fuschia                12980\n",
       "olive green            12296\n",
       "light brown            12269\n",
       "peach                  11816\n",
       "dark pink              11668\n",
       "sea green              11462\n",
       "lime                   11435\n",
       "periwinkle             11000\n",
       "indigo                 10513\n",
       "babypoo                   55\n",
       "caribbean surf            11\n",
       "medium tan leather        11\n",
       "impatiens juice           11\n",
       "Name: colorname, dtype: int64"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "colours['colorname'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "#shuffling the dataset to have equal distribution of each colour in the dataset\n",
    "colours = colours.sample(frac = 1,random_state= 10).reset_index(drop = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "#custom binary encoding\n",
    "single_colour = ['green','red','blue','yellow','purple','tan','gray','pink','olive','lime']\n",
    "colours_cbe = colours.copy()\n",
    "\n",
    "for color in single_colour:\n",
    "    colours_cbe[color+'_bin'] = np.where(colours['colorname'].str.contains(color), 1, 0)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>user_id</th>\n",
       "      <th>datestamp</th>\n",
       "      <th>r</th>\n",
       "      <th>g</th>\n",
       "      <th>b</th>\n",
       "      <th>colorname</th>\n",
       "      <th>green_bin</th>\n",
       "      <th>red_bin</th>\n",
       "      <th>blue_bin</th>\n",
       "      <th>yellow_bin</th>\n",
       "      <th>purple_bin</th>\n",
       "      <th>tan_bin</th>\n",
       "      <th>gray_bin</th>\n",
       "      <th>pink_bin</th>\n",
       "      <th>olive_bin</th>\n",
       "      <th>lime_bin</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1777700</td>\n",
       "      <td>80744</td>\n",
       "      <td>2010-03-11 23:28:48</td>\n",
       "      <td>78</td>\n",
       "      <td>200</td>\n",
       "      <td>22</td>\n",
       "      <td>bright green</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3103477</td>\n",
       "      <td>138277</td>\n",
       "      <td>2010-03-29 19:38:09</td>\n",
       "      <td>99</td>\n",
       "      <td>183</td>\n",
       "      <td>185</td>\n",
       "      <td>blue</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2921540</td>\n",
       "      <td>130578</td>\n",
       "      <td>2010-03-26 15:00:48</td>\n",
       "      <td>209</td>\n",
       "      <td>255</td>\n",
       "      <td>130</td>\n",
       "      <td>lime</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1694089</td>\n",
       "      <td>77149</td>\n",
       "      <td>2010-03-11 01:55:13</td>\n",
       "      <td>63</td>\n",
       "      <td>37</td>\n",
       "      <td>133</td>\n",
       "      <td>purple</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1291722</td>\n",
       "      <td>59196</td>\n",
       "      <td>2010-03-07 19:02:58</td>\n",
       "      <td>215</td>\n",
       "      <td>82</td>\n",
       "      <td>84</td>\n",
       "      <td>salmon</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        id  user_id           datestamp    r    g    b     colorname  \\\n",
       "0  1777700    80744 2010-03-11 23:28:48   78  200   22  bright green   \n",
       "1  3103477   138277 2010-03-29 19:38:09   99  183  185          blue   \n",
       "2  2921540   130578 2010-03-26 15:00:48  209  255  130          lime   \n",
       "3  1694089    77149 2010-03-11 01:55:13   63   37  133        purple   \n",
       "4  1291722    59196 2010-03-07 19:02:58  215   82   84        salmon   \n",
       "\n",
       "   green_bin  red_bin  blue_bin  yellow_bin  purple_bin  tan_bin  gray_bin  \\\n",
       "0          1        0         0           0           0        0         0   \n",
       "1          0        0         1           0           0        0         0   \n",
       "2          0        0         0           0           0        0         0   \n",
       "3          0        0         0           0           1        0         0   \n",
       "4          0        0         0           0           0        0         0   \n",
       "\n",
       "   pink_bin  olive_bin  lime_bin  \n",
       "0         0          0         0  \n",
       "1         0          0         0  \n",
       "2         0          0         1  \n",
       "3         0          0         0  \n",
       "4         0          0         0  "
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "colours_cbe.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "#taking a subset of dataset\n",
    "#colours_small = colours_cbe.sample(10000,random_state= 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "#x is the input features - taking rgb values\n",
    "train = colours_cbe.iloc[:,3:6].values\n",
    "#y is the target - taking the new encoded color bins, dropping the coloumn colourname\n",
    "target = colours_cbe.iloc[:,7:].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2089594, 10)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#target shape\n",
    "target.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data, test_data,train_target, test_target = train_test_split(train,target,test_size=0.25, random_state=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "#creating pipeline with PCA and classifier\n",
    "#clf =  svm.SVC(gamma='scale', decision_function_shape='ovr')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "#parameter dict\n",
    "param_grid = [ {'max_depth': np.arange(2, 10, 2), 'min_samples_split': np.arange(2, 10, 2), 'max_leaf_nodes' : np.arange(2, 10, 2)}]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 64 candidates, totalling 320 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  42 tasks      | elapsed:  2.2min\n",
      "[Parallel(n_jobs=-1)]: Done 192 tasks      | elapsed:  9.9min\n",
      "[Parallel(n_jobs=-1)]: Done 320 out of 320 | elapsed: 16.8min finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=5, error_score='raise-deprecating',\n",
       "       estimator=DecisionTreeClassifier(class_weight=None, criterion='gini', max_depth=None,\n",
       "            max_features=None, max_leaf_nodes=None,\n",
       "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "            min_samples_leaf=1, min_samples_split=2,\n",
       "            min_weight_fraction_leaf=0.0, presort=False, random_state=None,\n",
       "            splitter='best'),\n",
       "       fit_params=None, iid='warn', n_jobs=-1,\n",
       "       param_grid=[{'max_depth': array([2, 4, 6, 8]), 'min_samples_split': array([2, 4, 6, 8]), 'max_leaf_nodes': array([2, 4, 6, 8])}],\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n",
       "       scoring=None, verbose=1)"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#defining grid\n",
    "grid = GridSearchCV(DecisionTreeClassifier(), param_grid, cv=5,n_jobs=-1, verbose=1)\n",
    "grid.fit(train_data,train_target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "#predicting test data\n",
    "pred = grid.predict(test_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 57.228479\n"
     ]
    }
   ],
   "source": [
    "#test score\n",
    "score = grid.score(test_data,test_target)*100\n",
    "print(\"Accuracy: %f\" % (score))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 51, 169, 132], dtype=int64)"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#test data\n",
    "test_data[190]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The predicted colour is:\n",
      " 2    lime\n",
      "Name: colorname, dtype: object\n"
     ]
    }
   ],
   "source": [
    "#predicting the colourname for the rgb values\n",
    "rgb_value = np.array([51, 169, 132]).reshape(1, -1)\n",
    "predict_encoded = grid.predict(rgb_value)\n",
    "arg_index = np.argmax(predict_encoded)\n",
    "predict_colour = colours_cbe[colours_cbe.index.isin([arg_index])]['colorname']\n",
    "print('The predicted colour is:\\n', predict_colour)\n",
    "#c= b.tolist()\n",
    "#idx = colours_cbe[colours_cbe.index.isin(c)]\n",
    "#dx = c.iloc[c,:]\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}