Dealing with missing values.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "cars_data=pd.read_csv('toyota.csv',index_col=0,na_values=[\"??\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "cars_data2=cars_data.copy()\n",
    "cars_data3=cars_data.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "price        0\n",
       "age          0\n",
       "km           0\n",
       "fudtype      0\n",
       "hp           0\n",
       "metcolor     1\n",
       "automatic    0\n",
       "cc           0\n",
       "doors        0\n",
       "weight       0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cars_data2.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "missing=cars_data2[cars_data2.isnull().any(axis=1)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>price</th>\n",
       "      <th>age</th>\n",
       "      <th>km</th>\n",
       "      <th>fudtype</th>\n",
       "      <th>hp</th>\n",
       "      <th>metcolor</th>\n",
       "      <th>automatic</th>\n",
       "      <th>cc</th>\n",
       "      <th>doors</th>\n",
       "      <th>weight</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>index</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>13950</td>\n",
       "      <td>24</td>\n",
       "      <td>41711</td>\n",
       "      <td>diesel</td>\n",
       "      <td>90</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>2000</td>\n",
       "      <td>3</td>\n",
       "      <td>1165</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       price  age     km fudtype  hp  metcolor  automatic    cc doors  weight\n",
       "index                                                                        \n",
       "2      13950   24  41711  diesel  90       NaN          0  2000     3    1165"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "missing.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>price</th>\n",
       "      <th>age</th>\n",
       "      <th>km</th>\n",
       "      <th>hp</th>\n",
       "      <th>metcolor</th>\n",
       "      <th>automatic</th>\n",
       "      <th>cc</th>\n",
       "      <th>weight</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>4.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>14037.500000</td>\n",
       "      <td>24.000000</td>\n",
       "      <td>52408.500000</td>\n",
       "      <td>90.0</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2000.0</td>\n",
       "      <td>1165.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>635.577166</td>\n",
       "      <td>1.414214</td>\n",
       "      <td>13960.589732</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.577350</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>13500.000000</td>\n",
       "      <td>23.000000</td>\n",
       "      <td>41711.000000</td>\n",
       "      <td>90.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2000.0</td>\n",
       "      <td>1165.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>13687.500000</td>\n",
       "      <td>23.000000</td>\n",
       "      <td>45667.250000</td>\n",
       "      <td>90.0</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2000.0</td>\n",
       "      <td>1165.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>13850.000000</td>\n",
       "      <td>23.500000</td>\n",
       "      <td>47493.000000</td>\n",
       "      <td>90.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2000.0</td>\n",
       "      <td>1165.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>14200.000000</td>\n",
       "      <td>24.500000</td>\n",
       "      <td>54234.250000</td>\n",
       "      <td>90.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2000.0</td>\n",
       "      <td>1165.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>14950.000000</td>\n",
       "      <td>26.000000</td>\n",
       "      <td>72937.000000</td>\n",
       "      <td>90.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2000.0</td>\n",
       "      <td>1165.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              price        age            km    hp  metcolor  automatic  \\\n",
       "count      4.000000   4.000000      4.000000   4.0  3.000000        4.0   \n",
       "mean   14037.500000  24.000000  52408.500000  90.0  0.666667        0.0   \n",
       "std      635.577166   1.414214  13960.589732   0.0  0.577350        0.0   \n",
       "min    13500.000000  23.000000  41711.000000  90.0  0.000000        0.0   \n",
       "25%    13687.500000  23.000000  45667.250000  90.0  0.500000        0.0   \n",
       "50%    13850.000000  23.500000  47493.000000  90.0  1.000000        0.0   \n",
       "75%    14200.000000  24.500000  54234.250000  90.0  1.000000        0.0   \n",
       "max    14950.000000  26.000000  72937.000000  90.0  1.000000        0.0   \n",
       "\n",
       "           cc  weight  \n",
       "count     4.0     4.0  \n",
       "mean   2000.0  1165.0  \n",
       "std       0.0     0.0  \n",
       "min    2000.0  1165.0  \n",
       "25%    2000.0  1165.0  \n",
       "50%    2000.0  1165.0  \n",
       "75%    2000.0  1165.0  \n",
       "max    2000.0  1165.0  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cars_data2.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "cars_data2=cars_data.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>price</th>\n",
       "      <th>age</th>\n",
       "      <th>km</th>\n",
       "      <th>fudtype</th>\n",
       "      <th>hp</th>\n",
       "      <th>metcolor</th>\n",
       "      <th>automatic</th>\n",
       "      <th>cc</th>\n",
       "      <th>doors</th>\n",
       "      <th>weight</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>index</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>13500</td>\n",
       "      <td>23</td>\n",
       "      <td>46986</td>\n",
       "      <td>diesel</td>\n",
       "      <td>90</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>2000</td>\n",
       "      <td>three</td>\n",
       "      <td>1165</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>13750</td>\n",
       "      <td>23</td>\n",
       "      <td>72937</td>\n",
       "      <td>diesel</td>\n",
       "      <td>90</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>2000</td>\n",
       "      <td>3</td>\n",
       "      <td>1165</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>13950</td>\n",
       "      <td>24</td>\n",
       "      <td>41711</td>\n",
       "      <td>diesel</td>\n",
       "      <td>90</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>2000</td>\n",
       "      <td>3</td>\n",
       "      <td>1165</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>14950</td>\n",
       "      <td>26</td>\n",
       "      <td>48000</td>\n",
       "      <td>diesel</td>\n",
       "      <td>90</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>2000</td>\n",
       "      <td>3</td>\n",
       "      <td>1165</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       price  age     km fudtype  hp  metcolor  automatic    cc  doors  weight\n",
       "index                                                                         \n",
       "0      13500   23  46986  diesel  90       1.0          0  2000  three    1165\n",
       "1      13750   23  72937  diesel  90       1.0          0  2000      3    1165\n",
       "2      13950   24  41711  diesel  90       NaN          0  2000      3    1165\n",
       "3      14950   26  48000  diesel  90       0.0          0  2000      3    1165"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cars_data2.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0    2\n",
       "0.0    1\n",
       "Name: metcolor, dtype: int64"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cars_data2['metcolor'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "cars_data2['metcolor'].fillna(cars_data['metcolor'].value_counts().index[0],inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>price</th>\n",
       "      <th>age</th>\n",
       "      <th>km</th>\n",
       "      <th>fudtype</th>\n",
       "      <th>hp</th>\n",
       "      <th>metcolor</th>\n",
       "      <th>automatic</th>\n",
       "      <th>cc</th>\n",
       "      <th>doors</th>\n",
       "      <th>weight</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>index</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>13500</td>\n",
       "      <td>23</td>\n",
       "      <td>46986</td>\n",
       "      <td>diesel</td>\n",
       "      <td>90</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>2000</td>\n",
       "      <td>three</td>\n",
       "      <td>1165</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>13750</td>\n",
       "      <td>23</td>\n",
       "      <td>72937</td>\n",
       "      <td>diesel</td>\n",
       "      <td>90</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>2000</td>\n",
       "      <td>3</td>\n",
       "      <td>1165</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>13950</td>\n",
       "      <td>24</td>\n",
       "      <td>41711</td>\n",
       "      <td>diesel</td>\n",
       "      <td>90</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>2000</td>\n",
       "      <td>3</td>\n",
       "      <td>1165</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>14950</td>\n",
       "      <td>26</td>\n",
       "      <td>48000</td>\n",
       "      <td>diesel</td>\n",
       "      <td>90</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>2000</td>\n",
       "      <td>3</td>\n",
       "      <td>1165</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       price  age     km fudtype  hp  metcolor  automatic    cc  doors  weight\n",
       "index                                                                         \n",
       "0      13500   23  46986  diesel  90       1.0          0  2000  three    1165\n",
       "1      13750   23  72937  diesel  90       1.0          0  2000      3    1165\n",
       "2      13950   24  41711  diesel  90       1.0          0  2000      3    1165\n",
       "3      14950   26  48000  diesel  90       0.0          0  2000      3    1165"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cars_data2.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "price        0\n",
       "age          0\n",
       "km           0\n",
       "fudtype      0\n",
       "hp           0\n",
       "metcolor     0\n",
       "automatic    0\n",
       "cc           0\n",
       "doors        0\n",
       "weight       0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cars_data2.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "cars_data3=cars_data3.apply(lambda x:x.fillna(x.mean()) if x.dtype=='float' else x.fillna(x.value_counts().index[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>price</th>\n",
       "      <th>age</th>\n",
       "      <th>km</th>\n",
       "      <th>fudtype</th>\n",
       "      <th>hp</th>\n",
       "      <th>metcolor</th>\n",
       "      <th>automatic</th>\n",
       "      <th>cc</th>\n",
       "      <th>doors</th>\n",
       "      <th>weight</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>index</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>13500</td>\n",
       "      <td>23</td>\n",
       "      <td>46986</td>\n",
       "      <td>diesel</td>\n",
       "      <td>90</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0</td>\n",
       "      <td>2000</td>\n",
       "      <td>three</td>\n",
       "      <td>1165</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>13750</td>\n",
       "      <td>23</td>\n",
       "      <td>72937</td>\n",
       "      <td>diesel</td>\n",
       "      <td>90</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0</td>\n",
       "      <td>2000</td>\n",
       "      <td>3</td>\n",
       "      <td>1165</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>13950</td>\n",
       "      <td>24</td>\n",
       "      <td>41711</td>\n",
       "      <td>diesel</td>\n",
       "      <td>90</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>0</td>\n",
       "      <td>2000</td>\n",
       "      <td>3</td>\n",
       "      <td>1165</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>14950</td>\n",
       "      <td>26</td>\n",
       "      <td>48000</td>\n",
       "      <td>diesel</td>\n",
       "      <td>90</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0</td>\n",
       "      <td>2000</td>\n",
       "      <td>3</td>\n",
       "      <td>1165</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       price  age     km fudtype  hp  metcolor  automatic    cc  doors  weight\n",
       "index                                                                         \n",
       "0      13500   23  46986  diesel  90  1.000000          0  2000  three    1165\n",
       "1      13750   23  72937  diesel  90  1.000000          0  2000      3    1165\n",
       "2      13950   24  41711  diesel  90  0.666667          0  2000      3    1165\n",
       "3      14950   26  48000  diesel  90  0.000000          0  2000      3    1165"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cars_data3.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}