1_compare_models.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Titanic dataset - Binary classification\n",
    "\n",
    "The Titanic dataset present a binary classification problem, where the goal is to predict whether a passenger survived or not.   \n",
    "In this notebook, I will build a simple Random Forest model to predict the survival of passengers.\n",
    "\n",
    "Will import the data cleaned in the previous step, with the following features:\n",
    "\n",
    "| Variable | Definition                          | Value                        |\n",
    "|----------|-------------------------------------|----------------------------|\n",
    "| survival | Survival                            | 0=No, 1=Yes            |\n",
    "| pclass   | Ticket class                        | 1=1st, 2=2nd, 3=3rd  |\n",
    "| sex      | Sex                                 | 0=Female, 1=Male                           |\n",
    "| Age      | Age in years                        |                            |\n",
    "| sibsp    | # of siblings / spouses aboard the Titanic |                    |\n",
    "| parch    | # of parents / children aboard the Titanic |                    |\n",
    "| fare     | Passenger fare                      |                            |\n",
    "| cabin    | Cabin number                        | 0=NaN/Unidentified, 1=Yes/Valid Cabin nr                           |\n",
    "| embarked | Port of Embarkation                 | 0=Cherbourg, 1=Queenstown, 2=Southampton |"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import libraries\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "pd.set_option('display.float_format', lambda x: '%.2f' % x)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load and inspect data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train dataset shape (rows, columns): (891, 18)\n",
      "Test dataset shape (rows, columns): (418, 17)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PassengerId</th>\n",
       "      <th>Survived</th>\n",
       "      <th>Pclass_2</th>\n",
       "      <th>Pclass_3</th>\n",
       "      <th>Sex_1</th>\n",
       "      <th>SibSp_1</th>\n",
       "      <th>SibSp_2</th>\n",
       "      <th>SibSp_3</th>\n",
       "      <th>SibSp_4</th>\n",
       "      <th>SibSp_5</th>\n",
       "      <th>SibSp_8</th>\n",
       "      <th>Parch_1</th>\n",
       "      <th>Parch_2</th>\n",
       "      <th>Parch_3</th>\n",
       "      <th>Parch_4</th>\n",
       "      <th>Parch_5</th>\n",
       "      <th>Parch_6</th>\n",
       "      <th>Parch_9</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   PassengerId  Survived  Pclass_2  Pclass_3  Sex_1  SibSp_1  SibSp_2  \\\n",
       "0            1         0     False      True   True     True    False   \n",
       "1            2         1     False     False  False     True    False   \n",
       "2            3         1     False      True  False    False    False   \n",
       "\n",
       "   SibSp_3  SibSp_4  SibSp_5  SibSp_8  Parch_1  Parch_2  Parch_3  Parch_4  \\\n",
       "0    False    False    False    False    False    False    False    False   \n",
       "1    False    False    False    False    False    False    False    False   \n",
       "2    False    False    False    False    False    False    False    False   \n",
       "\n",
       "   Parch_5  Parch_6  Parch_9  \n",
       "0    False    False    False  \n",
       "1    False    False    False  \n",
       "2    False    False    False  "
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# load cleaned data\n",
    "train_data = pd.read_csv('data/train_clean_subsetFeatures.csv')\n",
    "test_data = pd.read_csv('data/test_clean_subsetFeatures.csv')\n",
    "\n",
    "# print the shape of the data\n",
    "print('Train dataset shape (rows, columns):', train_data.shape)\n",
    "print('Test dataset shape (rows, columns):', test_data.shape)\n",
    "train_data.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PassengerId</th>\n",
       "      <th>Pclass_2</th>\n",
       "      <th>Pclass_3</th>\n",
       "      <th>Sex_1</th>\n",
       "      <th>SibSp_1</th>\n",
       "      <th>SibSp_2</th>\n",
       "      <th>SibSp_3</th>\n",
       "      <th>SibSp_4</th>\n",
       "      <th>SibSp_5</th>\n",
       "      <th>SibSp_8</th>\n",
       "      <th>Parch_1</th>\n",
       "      <th>Parch_2</th>\n",
       "      <th>Parch_3</th>\n",
       "      <th>Parch_4</th>\n",
       "      <th>Parch_5</th>\n",
       "      <th>Parch_6</th>\n",
       "      <th>Parch_9</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>892</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>893</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>894</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   PassengerId  Pclass_2  Pclass_3  Sex_1  SibSp_1  SibSp_2  SibSp_3  SibSp_4  \\\n",
       "0          892     False      True   True    False    False    False    False   \n",
       "1          893     False      True  False     True    False    False    False   \n",
       "2          894      True     False   True    False    False    False    False   \n",
       "\n",
       "   SibSp_5  SibSp_8  Parch_1  Parch_2  Parch_3  Parch_4  Parch_5  Parch_6  \\\n",
       "0    False    False    False    False    False    False    False    False   \n",
       "1    False    False    False    False    False    False    False    False   \n",
       "2    False    False    False    False    False    False    False    False   \n",
       "\n",
       "   Parch_9  \n",
       "0    False  \n",
       "1    False  \n",
       "2    False  "
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_data.head(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define models\n",
    "\n",
    "1) Logistic Regression\n",
    "2) Random Forest (subset of features)\n",
    "3) Random Forest (all features)\n",
    "4) Support Vector Machine (SVM)\n",
    "5) Xtreme Gradient Boosting (XGBoost)\n",
    "6) Gradient Boosting Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.svm import SVC\n",
    "from xgboost import XGBClassifier\n",
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "from sklearn.model_selection import cross_val_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Age</th>\n",
       "      <th>Fare</th>\n",
       "      <th>SibSp</th>\n",
       "      <th>Pclass_3</th>\n",
       "      <th>Sex_1</th>\n",
       "      <th>Cabin_1</th>\n",
       "      <th>Embarked_2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-0.58</td>\n",
       "      <td>-0.50</td>\n",
       "      <td>0.48</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.66</td>\n",
       "      <td>0.73</td>\n",
       "      <td>0.48</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-0.27</td>\n",
       "      <td>-0.49</td>\n",
       "      <td>-0.48</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    Age  Fare  SibSp  Pclass_3  Sex_1  Cabin_1  Embarked_2\n",
       "0 -0.58 -0.50   0.48      True   True    False        True\n",
       "1  0.66  0.73   0.48     False  False     True       False\n",
       "2 -0.27 -0.49  -0.48      True  False    False        True"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# load cleaned data (all features minus low-importance features)\n",
    "train_data = pd.read_csv('data/train_clean_allFeatures.csv')\n",
    "test_data = pd.read_csv('data/test_clean_allFeatures.csv')\n",
    "\n",
    "y = train_data[\"Survived\"] # target variable\n",
    "X = train_data.drop(['Survived', 'PassengerId', 'Parch', 'Embarked_1', 'Pclass_2'], axis=1) # drop low-importance features\n",
    "X_test = test_data.drop(['PassengerId', 'Parch', 'Embarked_1', 'Pclass_2'], axis=1)\n",
    "\n",
    "# small subset of features\n",
    "train_data_subset = pd.read_csv('data/train_clean_subsetFeatures.csv')\n",
    "test_data_subset = pd.read_csv('data/test_clean_subsetFeatures.csv')\n",
    "\n",
    "y_subset = train_data_subset[\"Survived\"] # target variable\n",
    "X_subset = train_data_subset.drop([\"Survived\", \"PassengerId\"], axis=1)\n",
    "X_test_subset = test_data_subset.drop([\"PassengerId\"], axis=1)\n",
    "X.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cross-Validation Accuracy: [0.82122905 0.78651685 0.78651685 0.79775281 0.84831461] | Mean value: 0.808\n",
      "Cross-Validation F1:       [0.76470588 0.71641791 0.71641791 0.70491803 0.79069767] | Mean value: 0.739\n"
     ]
    }
   ],
   "source": [
    "# Logistic Regression model\n",
    "lr = LogisticRegression(max_iter = 2000, random_state=1)\n",
    "cv_accuracy = cross_val_score(lr, X, y, cv=5, scoring='accuracy')\n",
    "cv_f1 = cross_val_score(lr, X, y, cv=5, scoring='f1')\n",
    "\n",
    "print(f'Cross-Validation Accuracy: {cv_accuracy} | Mean value: {round(cv_accuracy.mean(),3)}')\n",
    "print(f'Cross-Validation F1:       {cv_f1} | Mean value: {round(cv_f1.mean(),3)}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cross-Validation Accuracy: [0.73743017 0.80898876 0.80337079 0.76966292 0.81460674] | Mean value: 0.787\n",
      "Cross-Validation F1:       [0.52525253 0.73846154 0.72440945 0.66666667 0.74015748] | Mean value: 0.679\n"
     ]
    }
   ],
   "source": [
    "# Random Forest model - subset of features\n",
    "rf_subset = RandomForestClassifier(n_estimators=100, max_depth=5, random_state=1)\n",
    "cv_accuracy = cross_val_score(rf_subset, X_subset, y_subset, cv=5, scoring='accuracy')\n",
    "cv_f1 = cross_val_score(rf_subset, X_subset, y_subset, cv=5, scoring='f1')\n",
    "\n",
    "print(f'Cross-Validation Accuracy: {cv_accuracy} | Mean value: {round(cv_accuracy.mean(),3)}')\n",
    "print(f'Cross-Validation F1:       {cv_f1} | Mean value: {round(cv_f1.mean(),3)}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cross-Validation Accuracy: [0.81005587 0.78651685 0.84269663 0.81460674 0.83146067] | Mean value: 0.817\n",
      "Cross-Validation F1:       [0.73846154 0.71212121 0.78787879 0.73170732 0.77941176] | Mean value: 0.75\n"
     ]
    }
   ],
   "source": [
    "# Random Forest model - all features\n",
    "rf = RandomForestClassifier(n_estimators=150, max_depth=10, random_state=1)\n",
    "cv_accuracy = cross_val_score(rf, X, y, cv=5, scoring='accuracy')\n",
    "cv_f1 = cross_val_score(rf, X, y, cv=5, scoring='f1')\n",
    "\n",
    "print(f'Cross-Validation Accuracy: {cv_accuracy} | Mean value: {round(cv_accuracy.mean(),3)}')\n",
    "print(f'Cross-Validation F1:       {cv_f1} | Mean value: {round(cv_f1.mean(),3)}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cross-Validation Accuracy: [0.84357542 0.80337079 0.80337079 0.79775281 0.84831461] | Mean value: 0.819\n",
      "Cross-Validation F1:       [0.77419355 0.72868217 0.73282443 0.69491525 0.79699248] | Mean value: 0.746\n"
     ]
    }
   ],
   "source": [
    "# Support Vector Classifier (SVC) model\n",
    "svc = SVC(probability = True)\n",
    "cv_accuracy = cross_val_score(svc, X, y, cv=5, scoring='accuracy')\n",
    "cv_f1 = cross_val_score(svc, X, y, cv=5, scoring='f1')\n",
    "\n",
    "print(f'Cross-Validation Accuracy: {cv_accuracy} | Mean value: {round(cv_accuracy.mean(),3)}')\n",
    "print(f'Cross-Validation F1:       {cv_f1} | Mean value: {round(cv_f1.mean(),3)}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cross-Validation Accuracy: [0.78212291 0.81460674 0.84269663 0.79213483 0.85393258] | Mean value: 0.817\n",
      "Cross-Validation F1:       [0.70229008 0.7518797  0.79104478 0.70866142 0.80882353] | Mean value: 0.753\n"
     ]
    }
   ],
   "source": [
    "# Xtreme Gradient Boosting (XGBoost) model\n",
    "xgb = XGBClassifier(random_state =1)\n",
    "cv_accuracy = cross_val_score(xgb, X, y, cv=5, scoring='accuracy')\n",
    "cv_f1 = cross_val_score(xgb, X, y, cv=5, scoring='f1')\n",
    "\n",
    "print(f'Cross-Validation Accuracy: {cv_accuracy} | Mean value: {round(cv_accuracy.mean(),3)}')\n",
    "print(f'Cross-Validation F1:       {cv_f1} | Mean value: {round(cv_f1.mean(),3)}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cross-Validation Accuracy: [0.81564246 0.80337079 0.85393258 0.79213483 0.85955056] | Mean value: 0.825\n",
      "Cross-Validation F1:       [0.736      0.73282443 0.79032258 0.69421488 0.81751825] | Mean value: 0.754\n"
     ]
    }
   ],
   "source": [
    "# Gradient Boosting Classifier\n",
    "gbc = GradientBoostingClassifier(random_state =1)\n",
    "cv_accuracy = cross_val_score(gbc, X, y, cv=5, scoring='accuracy')\n",
    "cv_f1 = cross_val_score(gbc, X, y, cv=5, scoring='f1')\n",
    "\n",
    "print(f'Cross-Validation Accuracy: {cv_accuracy} | Mean value: {round(cv_accuracy.mean(),3)}')\n",
    "print(f'Cross-Validation F1:       {cv_f1} | Mean value: {round(cv_f1.mean(),3)}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Baseline Model Performance\n",
    "\n",
    "| Model       | Accuracy | F1-score    |\n",
    "|-------------|----------|-------------|\n",
    "| Logistic Regression  | 80.8     | 73.9        |\n",
    "| Random Forest subset | 78.7     | 67.9        |\n",
    "| Random Forest all    | 81.7     | 75.0        |\n",
    "| SVM                  | 81.9     | 74.6        |\n",
    "| XGBoost              | 81.7     | 75.3        |\n",
    "| Gradient Boosting    | 82.5     | 75.4        |\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Parameter tuning\n",
    "Use GridSearchCV to find the best parameters for the models, while RandomizedSearchCV for Random Forest and XG Boost mopdels to reduce the computational time.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import GridSearchCV \n",
    "from sklearn.model_selection import RandomizedSearchCV "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "# report stats for each model\n",
    "def clf_performance(classifier, model_name):\n",
    "    print(model_name)\n",
    "    print('Best Score: ' + str(classifier.best_score_))\n",
    "    print('Best Parameters: ' + str(classifier.best_params_))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 10 folds for each of 20 candidates, totalling 200 fits\n",
      "Logistic Regression\n",
      "Best Score: 0.8092009987515606\n",
      "Best Parameters: {'C': 0.14384498882876628, 'max_iter': 2000, 'penalty': 'l2', 'solver': 'liblinear'}\n"
     ]
    }
   ],
   "source": [
    "# Logistic Regression model - hyperparameter tuning\n",
    "lr = LogisticRegression(random_state = 1)\n",
    "param_grid = {'max_iter': [2000],\n",
    "                'C': np.logspace(-4, 2, 20),\n",
    "                'penalty': ['l2'],\n",
    "                'solver': ['liblinear']}\n",
    "# Create grid search using 5-fold cross validation\n",
    "clf_lr = GridSearchCV(lr, param_grid=param_grid, cv=10, verbose=True, n_jobs=-1) # n_jobs=-1 means all processors\n",
    "best_clf_lr = clf_lr.fit(X, y)\n",
    "clf_performance(best_clf_lr,'Logistic Regression')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Random Forest model - hyperparameter tuning (subset of features)\n",
    "# Because of the large potential search space, we 1) use RandomizedSearchCV to narrow down the search space\n",
    "# and 2) use GridSearchCV on the smaller search space to find the best parameters\n",
    "\n",
    "# # 1) RandomizedSearchCV - to narrow down search space\n",
    "# rf_subset = RandomForestClassifier(random_state = 1)\n",
    "# param_grid =  {'n_estimators': [100,500,1000], \n",
    "#                 'bootstrap': [True,False],\n",
    "#                 'max_depth': [3,5,10,20,50,75],\n",
    "#                 'max_features': ['auto','sqrt'],\n",
    "#                 'min_samples_leaf': [1,2,4,10],\n",
    "#                 'min_samples_split': [2,5,10]}\n",
    "\n",
    "# clf_rf_subset_rnd = RandomizedSearchCV(rf_subset, param_distributions=param_grid, n_iter=100, cv=5, verbose=True, n_jobs=-1)\n",
    "# best_rf_subset_rnd = clf_rf_subset_rnd.fit(X, y)\n",
    "# clf_performance(best_rf_subset_rnd,'Random Forest')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 10 folds for each of 36 candidates, totalling 360 fits\n",
      "Random Forest\n",
      "Best Score: 0.8002247191011236\n",
      "Best Parameters: {'bootstrap': True, 'max_depth': 4, 'max_features': 'sqrt', 'min_samples_leaf': 2, 'min_samples_split': 2, 'n_estimators': 400}\n"
     ]
    }
   ],
   "source": [
    "# Random Forest model (subset of features)\n",
    "# 2) GridSearchCV on smaller search space\n",
    "param_grid =  {'n_estimators': [400, 500],\n",
    "                'bootstrap': [True],\n",
    "                'max_depth': [3,4,5],\n",
    "                'max_features': ['sqrt'],\n",
    "                'min_samples_leaf': [2,3,4],\n",
    "                'min_samples_split': [2,3]}\n",
    "clf_rf_subset = GridSearchCV(rf, param_grid=param_grid, cv=10, verbose=True, n_jobs=-1)\n",
    "best_clf_rf_subset = clf_rf_subset.fit(X_subset, y_subset)\n",
    "clf_performance(best_clf_rf_subset,'Random Forest')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Random Forest model - hyperparameter tuning (all features)\n",
    "# Because of the large potential search space, we 1) use RandomizedSearchCV to narrow down the search space\n",
    "# and 2) use GridSearchCV on the smaller search space to find the best parameters\n",
    "\n",
    "# # 1) RandomizedSearchCV - to narrow down search space\n",
    "# rf = RandomForestClassifier(random_state = 1)\n",
    "# param_grid =  {'n_estimators': [300, 400, 425],\n",
    "#                 'bootstrap': [True],\n",
    "#                 'max_depth': [4,5,6],\n",
    "#                 'max_features': ['sqrt'],\n",
    "#                 'min_samples_leaf': [2,3],\n",
    "#                 'min_samples_split': [2,3,4]}\n",
    "\n",
    "# clf_rf_rnd = RandomizedSearchCV(rf, param_distributions=param_grid, n_iter=100, cv=5, verbose=True, n_jobs=-1)\n",
    "# best_rf_rnd = clf_rf_rnd.fit(X, y)\n",
    "# clf_performance(best_rf_rnd,'Random Forest')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 10 folds for each of 36 candidates, totalling 360 fits\n",
      "Random Forest\n",
      "Best Score: 0.8238202247191012\n",
      "Best Parameters: {'bootstrap': True, 'max_depth': 5, 'max_features': 'sqrt', 'min_samples_leaf': 2, 'min_samples_split': 2, 'n_estimators': 500}\n"
     ]
    }
   ],
   "source": [
    "# Random Forest model (all features)\n",
    "# 2) GridSearchCV on smaller search space\n",
    "param_grid =  {'n_estimators': [400, 500],\n",
    "                'bootstrap': [True],\n",
    "                'max_depth': [4,5,6],\n",
    "                'max_features': ['sqrt'],\n",
    "                'min_samples_leaf': [2,3,4],\n",
    "                'min_samples_split': [2,3]}\n",
    "clf_rf = GridSearchCV(rf, param_grid=param_grid, cv=10, verbose=True, n_jobs=-1)\n",
    "best_clf_rf = clf_rf.fit(X, y)\n",
    "clf_performance(best_clf_rf,'Random Forest')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 10 folds for each of 2 candidates, totalling 20 fits\n",
      "SVC\n",
      "Best Score: 0.8170411985018726\n",
      "Best Parameters: {'C': 0.7, 'degree': 2, 'kernel': 'rbf'}\n"
     ]
    }
   ],
   "source": [
    "# Support Vector Classifier (SVC) model - hyperparameter tuning\n",
    "svc = SVC(probability = True)\n",
    "param_grid = tuned_parameters = [\n",
    "    {'kernel': ['rbf'], 'degree':[2], 'C': [0.5, 0.7]},\n",
    "    #  {'kernel': ['linear'], 'C': [1]}\n",
    "    # {'kernel': ['poly'], 'degree':[2], 'C': [1e03]}\n",
    "]\n",
    "clf_svc = GridSearchCV(svc, param_grid=param_grid, cv=10, verbose=True, n_jobs=-1)\n",
    "best_clf_svc = clf_svc.fit(X, y)\n",
    "clf_performance(best_clf_svc,'SVC')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # XGBoost model - hyperparameter tuning\n",
    "\n",
    "# # 1) RandomizedSearchCV - to narrow down search space\n",
    "# xgb = XGBClassifier(random_state = 1)\n",
    "# param_grid = {\n",
    "#     'n_estimators': [150],\n",
    "#     'colsample_bytree': [0.3, 0.5],\n",
    "#     'max_depth': [9, 10, 11],\n",
    "#     'reg_alpha': [0.5, 0.9],\n",
    "#     'reg_lambda': [3, 4, 5],\n",
    "#     'subsample': [0.3, 0.4],\n",
    "#     'learning_rate':[0,2, 0.25],\n",
    "#     'gamma': [0.6, 0.8, 1],\n",
    "#     'min_child_weight':[2, 3],\n",
    "#     # 'sampling_method': ['uniform']\n",
    "# }\n",
    "# clf_xgb_rnd = RandomizedSearchCV(xgb, param_distributions = param_grid, n_iter = 1000, cv = 10, verbose = True, n_jobs = -1)\n",
    "# best_clf_xgb_rnd = clf_xgb_rnd.fit(X,y)\n",
    "# clf_performance(best_clf_xgb_rnd,'XGB')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 10 folds for each of 864 candidates, totalling 8640 fits\n",
      "XGB\n",
      "Best Score: 0.8372908863920101\n",
      "Best Parameters: {'colsample_bytree': 0.5, 'gamma': 0.6, 'learning_rate': 0.25, 'max_depth': 9, 'min_child_weight': 2, 'n_estimators': 150, 'reg_alpha': 0.5, 'reg_lambda': 4, 'subsample': 0.4}\n"
     ]
    }
   ],
   "source": [
    "# XGBoost model - hyperparameter tuning\n",
    "\n",
    "# 2) GridSearchCV on smaller search space\n",
    "xgb = XGBClassifier(random_state = 1)\n",
    "param_grid = {\n",
    "    'n_estimators': [150],\n",
    "    'colsample_bytree': [0.3, 0.5],\n",
    "    'max_depth': [9, 10, 11],\n",
    "    'reg_alpha': [0.5, 0.6],\n",
    "    'reg_lambda': [3, 4],\n",
    "    'subsample': [0.3, 0.4],\n",
    "    'learning_rate':[0,2, 0.25],\n",
    "    'gamma': [0.6, 0.8, 1],\n",
    "    'min_child_weight':[2, 3],\n",
    "    # 'sampling_method': ['uniform']\n",
    "}\n",
    "clf_xgb = GridSearchCV(xgb, param_grid=param_grid, cv=10, verbose=True, n_jobs=-1)\n",
    "best_clf_xgb = clf_xgb.fit(X,y)\n",
    "clf_performance(best_clf_xgb,'XGB')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 15 folds for each of 16 candidates, totalling 240 fits\n",
      "Gradient Boosting Classifier\n",
      "Best Score: 0.8340489642184556\n",
      "Best Parameters: {'learning_rate': 0.04, 'loss': 'exponential', 'max_depth': 5, 'max_features': 'sqrt', 'min_samples_leaf': 2, 'min_samples_split': 5, 'n_estimators': 500, 'subsample': 0.5}\n"
     ]
    }
   ],
   "source": [
    "# Gradient Boosting Classifier - hyperparameter tuning\n",
    "gbc = GradientBoostingClassifier(random_state = 1)\n",
    "# param_grid = {                      # Overfitting options\n",
    "#     'n_estimators': [300],          # Increase number of trees\n",
    "#     'learning_rate': [0.06],        # Decrease learning rate\n",
    "#     'max_depth': [4, 5, 6],         # Reduce tree depth\n",
    "#     'min_samples_split': [4, 5],    # Increase min_samples_split\n",
    "#     'min_samples_leaf': [1, 2],     # Increase min_samples_leaf\n",
    "#     'subsample': [0.3],        # Include more aggressive subsampling\n",
    "#     'max_features': ['sqrt'],       # Limit max_features for each split\n",
    "#     'loss': ['exponential']         # Consider different loss functions\n",
    "# }\n",
    "param_grid = {\n",
    "    'n_estimators': [500],                  # Increase number of trees\n",
    "    'learning_rate': [0.03, 0.04],               # Decrease learning rate\n",
    "    'max_depth': [4, 5],                      # Consider slightly lower max depth\n",
    "    'min_samples_split': [5, 6],                # Further increase min_samples_split\n",
    "    'min_samples_leaf': [2, 3],                  # Increase min_samples_leaf\n",
    "    'subsample': [0.5],                     # Increase subsample to balance\n",
    "    'max_features': ['sqrt'],                    # Keep max_features as 'sqrt'\n",
    "    'loss': ['exponential']          # Consider different loss functions\n",
    "}\n",
    "clf_gbc = GridSearchCV(gbc, param_grid=param_grid, cv=15, verbose=True, n_jobs=-1)\n",
    "best_clf_gbc = clf_gbc.fit(X, y)\n",
    "clf_performance(best_clf_gbc,'Gradient Boosting Classifier')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "### Tuned Model Performance\n",
    "\n",
    "| Model                | Accuracy | Accuracy       | Test set  | Test set   |\n",
    "|----------------------|----------|----------------|-----------|------------|\n",
    "|                      | Baseline | Tuned model    | 1st submission | reduced overfit |\n",
    "| Logistic Regression  | 80.5     | 80.9           | 74.880    | 75.598     |\n",
    "| Random Forest subset | 78.7     | 80.0           | 77.751     | 77.272     |\n",
    "| Random Forest all    | 82.2     | 82.4           | 76.315    | 77.990     |\n",
    "| SVM                  | 82.4     | 81.7           | 66.985    | 77.751     |\n",
    "| XGBoost              | 82.0     | 83.7           | 76.076    | 76.555     |\n",
    "| Gradient Boosting    | 82.3     | 83.4           | 76.315    | 77.272     |\n",
    "\n",
    "The Random Forest model performed best with an accuracy of 77.99% on the test set (top 22.3%)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save predictions from the models\n",
    "# Logistic Regression\n",
    "predictions_lr = best_clf_lr.predict(X_test)\n",
    "output = pd.DataFrame({'PassengerId': test_data.PassengerId, 'Survived': predictions_lr})\n",
    "output.to_csv('submissions/logistic_regression.csv', index=False)\n",
    "\n",
    "# Random Forest - subset of features\n",
    "predictions_rf_subset = best_clf_rf_subset.predict(X_test_subset)\n",
    "output = pd.DataFrame({'PassengerId': test_data_subset.PassengerId, 'Survived': predictions_rf_subset})\n",
    "output.to_csv('submissions/random_forest_subset.csv', index=False)\n",
    "\n",
    "# Random Forest - all features\n",
    "predictions_rf = best_clf_rf.predict(X_test)\n",
    "output = pd.DataFrame({'PassengerId': test_data.PassengerId, 'Survived': predictions_rf})\n",
    "output.to_csv('submissions/random_forest.csv', index=False)\n",
    "\n",
    "# SVC\n",
    "predictions_svc = best_clf_svc.predict(X_test)\n",
    "output = pd.DataFrame({'PassengerId': test_data.PassengerId, 'Survived': predictions_svc})\n",
    "output.to_csv('submissions/svc.csv', index=False)\n",
    "\n",
    "# XGBoost\n",
    "predictions_xgb = best_clf_xgb.predict(X_test)\n",
    "output = pd.DataFrame({'PassengerId': test_data.PassengerId, 'Survived': predictions_xgb})\n",
    "output.to_csv('submissions/xgboost.csv', index=False)\n",
    "\n",
    "# Gradient Boosting Classifier\n",
    "predictions_gbc = best_clf_gbc.predict(X_test)\n",
    "output = pd.DataFrame({'PassengerId': test_data.PassengerId, 'Survived': predictions_gbc})\n",
    "output.to_csv('submissions/gradient_boosting.csv', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "kaggle_ML_env",
   "language": "python",
   "name": "kaggle_ml_env"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}