DataPreprocessing2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "886b932c",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import ast\n",
    "import matplotlib.pyplot as plt\n",
    "import warnings\n",
    "warnings.filterwarnings(action = 'ignore')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8856b840",
   "metadata": {},
   "source": [
    "# USER ACTIVITY"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "f8ba1079",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>User_ID</th>\n",
       "      <th>Title</th>\n",
       "      <th>datetime time</th>\n",
       "      <th>datetime Year</th>\n",
       "      <th>datetime Month</th>\n",
       "      <th>datetime Day_of_month</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ID_RT43AK77</td>\n",
       "      <td>comp_ID_IV5D</td>\n",
       "      <td>22:00:38</td>\n",
       "      <td>1</td>\n",
       "      <td>11</td>\n",
       "      <td>22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ID_RT43AK77</td>\n",
       "      <td>comp_ID_IV5D</td>\n",
       "      <td>00:41:02</td>\n",
       "      <td>1</td>\n",
       "      <td>11</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ID_RT43AK77</td>\n",
       "      <td>comp_ID_IV5D</td>\n",
       "      <td>00:53:15</td>\n",
       "      <td>1</td>\n",
       "      <td>11</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ID_RT43AK77</td>\n",
       "      <td>comp_ID_IV5D</td>\n",
       "      <td>01:23:05</td>\n",
       "      <td>1</td>\n",
       "      <td>11</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ID_RT43AK77</td>\n",
       "      <td>comp_ID_IV5D</td>\n",
       "      <td>01:27:38</td>\n",
       "      <td>1</td>\n",
       "      <td>11</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>317287</th>\n",
       "      <td>ID_Z759BCS3</td>\n",
       "      <td>job_ID_5ONI</td>\n",
       "      <td>07:14:47</td>\n",
       "      <td>1</td>\n",
       "      <td>11</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>317288</th>\n",
       "      <td>ID_WNJMADF8</td>\n",
       "      <td>job_ID_5ONI</td>\n",
       "      <td>06:24:46</td>\n",
       "      <td>1</td>\n",
       "      <td>11</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>317289</th>\n",
       "      <td>ID_7NBY9MV2</td>\n",
       "      <td>job_ID_5ONI</td>\n",
       "      <td>11:56:35</td>\n",
       "      <td>1</td>\n",
       "      <td>11</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>317290</th>\n",
       "      <td>ID_LHPJJSOK</td>\n",
       "      <td>job_ID_5ONI</td>\n",
       "      <td>01:27:13</td>\n",
       "      <td>1</td>\n",
       "      <td>11</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>317291</th>\n",
       "      <td>ID_LCFNGARD</td>\n",
       "      <td>job_ID_5ONI</td>\n",
       "      <td>08:40:29</td>\n",
       "      <td>1</td>\n",
       "      <td>11</td>\n",
       "      <td>17</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>317292 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            User_ID         Title datetime time  datetime Year  \\\n",
       "0       ID_RT43AK77  comp_ID_IV5D      22:00:38              1   \n",
       "1       ID_RT43AK77  comp_ID_IV5D      00:41:02              1   \n",
       "2       ID_RT43AK77  comp_ID_IV5D      00:53:15              1   \n",
       "3       ID_RT43AK77  comp_ID_IV5D      01:23:05              1   \n",
       "4       ID_RT43AK77  comp_ID_IV5D      01:27:38              1   \n",
       "...             ...           ...           ...            ...   \n",
       "317287  ID_Z759BCS3   job_ID_5ONI      07:14:47              1   \n",
       "317288  ID_WNJMADF8   job_ID_5ONI      06:24:46              1   \n",
       "317289  ID_7NBY9MV2   job_ID_5ONI      11:56:35              1   \n",
       "317290  ID_LHPJJSOK   job_ID_5ONI      01:27:13              1   \n",
       "317291  ID_LCFNGARD   job_ID_5ONI      08:40:29              1   \n",
       "\n",
       "        datetime Month  datetime Day_of_month  \n",
       "0                   11                     22  \n",
       "1                   11                     23  \n",
       "2                   11                     23  \n",
       "3                   11                     23  \n",
       "4                   11                     23  \n",
       "...                ...                    ...  \n",
       "317287              11                     14  \n",
       "317288              11                     14  \n",
       "317289              11                     20  \n",
       "317290              11                     14  \n",
       "317291              11                     17  \n",
       "\n",
       "[317292 rows x 6 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_userAct = pd.read_csv(\"UserActivity.csv\").drop_duplicates()\n",
    "df_userAct"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "2f5d483e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 317292 entries, 0 to 317291\n",
      "Data columns (total 6 columns):\n",
      " #   Column                 Non-Null Count   Dtype \n",
      "---  ------                 --------------   ----- \n",
      " 0   User_ID                317292 non-null  object\n",
      " 1   Title                  317292 non-null  object\n",
      " 2   datetime time          317292 non-null  object\n",
      " 3   datetime Year          317292 non-null  int64 \n",
      " 4   datetime Month         317292 non-null  int64 \n",
      " 5   datetime Day_of_month  317292 non-null  int64 \n",
      "dtypes: int64(3), object(3)\n",
      "memory usage: 16.9+ MB\n"
     ]
    }
   ],
   "source": [
    "df_userAct.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "a118b129",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10400"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(pd.unique(df_userAct['User_ID']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "b097fae4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(pd.unique(df_userAct['datetime Year']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "c4f72dab",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_userAct.drop('datetime Year', inplace=True, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "e727b3a7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2     61181\n",
       "1     58292\n",
       "5     57258\n",
       "3     56323\n",
       "12    42047\n",
       "4     26962\n",
       "11    15229\n",
       "Name: datetime Month, dtype: int64"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_userAct[\"datetime Month\"].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "996f5e5e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2203"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df_userAct[df_userAct['datetime Month'] == 5].User_ID.unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "53ef3337",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_userAct[\"User_ID\"] = df_userAct[\"User_ID\"].astype(\"string\")\n",
    "df_userAct[\"datetime Month\"] = df_userAct[\"datetime Month\"].astype(\"string\")\n",
    "# df_userAct[\"datetime time\"] = df_userAct[\"datetime time\"].astype(\"string\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "92f96b9f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_userAct['Time user completed action in hour'] = df_userAct['datetime time'].str[0:2]\n",
    "# df_userAct['Time user completed action in minute'] = df_userAct['datetime time'].str[3:5]\n",
    "# df_userAct['Time user completed action in second'] = df_userAct['datetime time'].str[6:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "9b0f475d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_userAct['Time user completed action in hour'] = pd.to_numeric(df_userAct['Time user completed action in hour'])\n",
    "# df_userAct['Time user completed action in minute'] = pd.to_numeric(df_userAct['Time user completed action in minute'])\n",
    "# df_userAct['Time user completed action in second'] = pd.to_numeric(df_userAct['Time user completed action in second'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "1dd236b7",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_userAct.drop(\"datetime time\", inplace=True, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "9a42cbb9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Viewed All Competitions            58176\n",
       "Viewed All Discussions             47172\n",
       "Viewed All Learning Pages          22930\n",
       "Downloaded Competition Datafile    19578\n",
       "Updated Profile                    14748\n",
       "                                   ...  \n",
       "comp_ID_TJK9                           1\n",
       "comp_ID_TBXU                           1\n",
       "blog_ID_NVX3                           1\n",
       "blog_ID_R97T                           1\n",
       "blog_ID_22WI                           1\n",
       "Name: Title, Length: 397, dtype: int64"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_userAct[\"Title\"].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7d7a6ad1",
   "metadata": {},
   "source": [
    "# BLOGS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "3ab6c916",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_blogs = pd.read_csv(\"Blogs.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "26d164c7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Theme</th>\n",
       "      <th>blog_ID</th>\n",
       "      <th>Published At time</th>\n",
       "      <th>Published At Year</th>\n",
       "      <th>Published At Month</th>\n",
       "      <th>Published At Day_of_month</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5.0</td>\n",
       "      <td>ID_Z6MM</td>\n",
       "      <td>10:34:55.071854</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5.0</td>\n",
       "      <td>ID_7IB3</td>\n",
       "      <td>14:51:13.268049</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3.0</td>\n",
       "      <td>ID_JR8K</td>\n",
       "      <td>13:19:08.877076</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2.0</td>\n",
       "      <td>ID_IDAV</td>\n",
       "      <td>17:18:06.151314</td>\n",
       "      <td>5</td>\n",
       "      <td>7</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2.0</td>\n",
       "      <td>ID_7JVL</td>\n",
       "      <td>11:48:36.428428</td>\n",
       "      <td>4</td>\n",
       "      <td>8</td>\n",
       "      <td>25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>112</th>\n",
       "      <td>5.0</td>\n",
       "      <td>ID_BYCA</td>\n",
       "      <td>08:47:22.257135</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>113</th>\n",
       "      <td>2.0</td>\n",
       "      <td>ID_3MFP</td>\n",
       "      <td>09:34:57.035251</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>114</th>\n",
       "      <td>4.0</td>\n",
       "      <td>ID_TTTU</td>\n",
       "      <td>09:54:57.702275</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>115</th>\n",
       "      <td>4.0</td>\n",
       "      <td>ID_ATLA</td>\n",
       "      <td>08:11:05.392827</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>29</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>116</th>\n",
       "      <td>2.0</td>\n",
       "      <td>ID_C4OY</td>\n",
       "      <td>08:12:16.647146</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>27</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>117 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     Theme  blog_ID Published At time  Published At Year  Published At Month  \\\n",
       "0      5.0  ID_Z6MM   10:34:55.071854                  5                   4   \n",
       "1      5.0  ID_7IB3   14:51:13.268049                  5                   4   \n",
       "2      3.0  ID_JR8K   13:19:08.877076                  5                   4   \n",
       "3      2.0  ID_IDAV   17:18:06.151314                  5                   7   \n",
       "4      2.0  ID_7JVL   11:48:36.428428                  4                   8   \n",
       "..     ...      ...               ...                ...                 ...   \n",
       "112    5.0  ID_BYCA   08:47:22.257135                  1                   4   \n",
       "113    2.0  ID_3MFP   09:34:57.035251                  1                   4   \n",
       "114    4.0  ID_TTTU   09:54:57.702275                  1                   4   \n",
       "115    4.0  ID_ATLA   08:11:05.392827                  1                   4   \n",
       "116    2.0  ID_C4OY   08:12:16.647146                  1                   5   \n",
       "\n",
       "     Published At Day_of_month  \n",
       "0                            5  \n",
       "1                            5  \n",
       "2                           17  \n",
       "3                            6  \n",
       "4                           25  \n",
       "..                         ...  \n",
       "112                          1  \n",
       "113                          8  \n",
       "114                         15  \n",
       "115                         29  \n",
       "116                         27  \n",
       "\n",
       "[117 rows x 6 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_blogs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "f87e411b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 117 entries, 0 to 116\n",
      "Data columns (total 6 columns):\n",
      " #   Column                     Non-Null Count  Dtype  \n",
      "---  ------                     --------------  -----  \n",
      " 0   Theme                      114 non-null    float64\n",
      " 1   blog_ID                    117 non-null    object \n",
      " 2   Published At time          117 non-null    object \n",
      " 3   Published At Year          117 non-null    int64  \n",
      " 4   Published At Month         117 non-null    int64  \n",
      " 5   Published At Day_of_month  117 non-null    int64  \n",
      "dtypes: float64(1), int64(3), object(2)\n",
      "memory usage: 5.6+ KB\n"
     ]
    }
   ],
   "source": [
    "df_blogs.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "f4cd0c30",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_blogs.rename(columns={'Published At Year': 'Blog Published Year'},  inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "035ba13b",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_blogs.rename(columns={'Published At Month': 'Blog Published Month'},  inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "1bcfea38",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_blogs[\"Published At time\"] = df_blogs[\"Published At time\"].astype(\"string\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "0c7eee0f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_blogs['Blog Published Hour'] = df_blogs['Published At time'].str[0:2]\n",
    "# df_blogs['Blog Published Minute'] = df_blogs['Published At time'].str[3:5]\n",
    "# df_blogs['Blog Published Second'] = df_blogs['Published At time'].str[6:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "c866cd57",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_blogs.drop(\"Published At time\", inplace=True, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "ed7769f2",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_blogs[\"blog_ID\"] = df_blogs[\"blog_ID\"].astype(\"string\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "0d174d83",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_blogs[\"blog_ID\"] = \"blog_\" + df_blogs[\"blog_ID\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "89ebbad9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<StringArray>\n",
       "['blog_ID_Z6MM', 'blog_ID_7IB3', 'blog_ID_JR8K', 'blog_ID_IDAV',\n",
       " 'blog_ID_7JVL', 'blog_ID_RY9N', 'blog_ID_5SM9', 'blog_ID_NEQG',\n",
       " 'blog_ID_N7KH', 'blog_ID_PTSN',\n",
       " ...\n",
       " 'blog_ID_EUC8', 'blog_ID_SEBQ', 'blog_ID_XNC8', 'blog_ID_QEGV',\n",
       " 'blog_ID_OD7E', 'blog_ID_BYCA', 'blog_ID_3MFP', 'blog_ID_TTTU',\n",
       " 'blog_ID_ATLA', 'blog_ID_C4OY']\n",
       "Length: 117, dtype: string"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_blogs[\"blog_ID\"].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "62e87ff9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 4,  7,  8,  9, 10, 11, 12,  1,  2,  3,  5,  6], dtype=int64)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_blogs[\"Blog Published Month\"].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "d77852a5",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_blogs.rename(columns={'blog_ID': 'Title'},  inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "a0e7ba7f",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_userAct = pd.merge(df_userAct, df_blogs, how=\"left\", on=['Title'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "d83fb247",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>User_ID</th>\n",
       "      <th>Title</th>\n",
       "      <th>datetime Month</th>\n",
       "      <th>datetime Day_of_month</th>\n",
       "      <th>Theme</th>\n",
       "      <th>Blog Published Year</th>\n",
       "      <th>Blog Published Month</th>\n",
       "      <th>Published At Day_of_month</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ID_RT43AK77</td>\n",
       "      <td>comp_ID_IV5D</td>\n",
       "      <td>11</td>\n",
       "      <td>22</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ID_RT43AK77</td>\n",
       "      <td>comp_ID_IV5D</td>\n",
       "      <td>11</td>\n",
       "      <td>23</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ID_RT43AK77</td>\n",
       "      <td>comp_ID_IV5D</td>\n",
       "      <td>11</td>\n",
       "      <td>23</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ID_RT43AK77</td>\n",
       "      <td>comp_ID_IV5D</td>\n",
       "      <td>11</td>\n",
       "      <td>23</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ID_RT43AK77</td>\n",
       "      <td>comp_ID_IV5D</td>\n",
       "      <td>11</td>\n",
       "      <td>23</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>317287</th>\n",
       "      <td>ID_Z759BCS3</td>\n",
       "      <td>job_ID_5ONI</td>\n",
       "      <td>11</td>\n",
       "      <td>14</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>317288</th>\n",
       "      <td>ID_WNJMADF8</td>\n",
       "      <td>job_ID_5ONI</td>\n",
       "      <td>11</td>\n",
       "      <td>14</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>317289</th>\n",
       "      <td>ID_7NBY9MV2</td>\n",
       "      <td>job_ID_5ONI</td>\n",
       "      <td>11</td>\n",
       "      <td>20</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>317290</th>\n",
       "      <td>ID_LHPJJSOK</td>\n",
       "      <td>job_ID_5ONI</td>\n",
       "      <td>11</td>\n",
       "      <td>14</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>317291</th>\n",
       "      <td>ID_LCFNGARD</td>\n",
       "      <td>job_ID_5ONI</td>\n",
       "      <td>11</td>\n",
       "      <td>17</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>317292 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            User_ID         Title datetime Month  datetime Day_of_month  \\\n",
       "0       ID_RT43AK77  comp_ID_IV5D             11                     22   \n",
       "1       ID_RT43AK77  comp_ID_IV5D             11                     23   \n",
       "2       ID_RT43AK77  comp_ID_IV5D             11                     23   \n",
       "3       ID_RT43AK77  comp_ID_IV5D             11                     23   \n",
       "4       ID_RT43AK77  comp_ID_IV5D             11                     23   \n",
       "...             ...           ...            ...                    ...   \n",
       "317287  ID_Z759BCS3   job_ID_5ONI             11                     14   \n",
       "317288  ID_WNJMADF8   job_ID_5ONI             11                     14   \n",
       "317289  ID_7NBY9MV2   job_ID_5ONI             11                     20   \n",
       "317290  ID_LHPJJSOK   job_ID_5ONI             11                     14   \n",
       "317291  ID_LCFNGARD   job_ID_5ONI             11                     17   \n",
       "\n",
       "        Theme  Blog Published Year  Blog Published Month  \\\n",
       "0         NaN                  NaN                   NaN   \n",
       "1         NaN                  NaN                   NaN   \n",
       "2         NaN                  NaN                   NaN   \n",
       "3         NaN                  NaN                   NaN   \n",
       "4         NaN                  NaN                   NaN   \n",
       "...       ...                  ...                   ...   \n",
       "317287    NaN                  NaN                   NaN   \n",
       "317288    NaN                  NaN                   NaN   \n",
       "317289    NaN                  NaN                   NaN   \n",
       "317290    NaN                  NaN                   NaN   \n",
       "317291    NaN                  NaN                   NaN   \n",
       "\n",
       "        Published At Day_of_month  \n",
       "0                             NaN  \n",
       "1                             NaN  \n",
       "2                             NaN  \n",
       "3                             NaN  \n",
       "4                             NaN  \n",
       "...                           ...  \n",
       "317287                        NaN  \n",
       "317288                        NaN  \n",
       "317289                        NaN  \n",
       "317290                        NaN  \n",
       "317291                        NaN  \n",
       "\n",
       "[317292 rows x 8 columns]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_userAct"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2732ed81",
   "metadata": {},
   "source": [
    "# JOBS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "79f73637",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_jobs = pd.read_csv(\"Jobs.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "b3dc7dbe",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Remote</th>\n",
       "      <th>Experience</th>\n",
       "      <th>Industry</th>\n",
       "      <th>Employment Type</th>\n",
       "      <th>Company Size</th>\n",
       "      <th>Data Science Functions</th>\n",
       "      <th>job_ID</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>NaN</td>\n",
       "      <td>2-5</td>\n",
       "      <td>{Government,Health,\"Financial Services\"}</td>\n",
       "      <td>fulltime</td>\n",
       "      <td>50-200</td>\n",
       "      <td>{\"Data modelling\",\"Business/Data Analysis\"}</td>\n",
       "      <td>ID_VJ43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_ZNZV</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>True</td>\n",
       "      <td>&gt;5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>fulltime</td>\n",
       "      <td>&gt;200</td>\n",
       "      <td>{\"Data engineering\",\"Data modelling\"}</td>\n",
       "      <td>ID_PZHM</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>NaN</td>\n",
       "      <td>&gt;5</td>\n",
       "      <td>{Media}</td>\n",
       "      <td>fulltime</td>\n",
       "      <td>1-10</td>\n",
       "      <td>{\"Data engineering\"}</td>\n",
       "      <td>ID_1PBU</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>NaN</td>\n",
       "      <td>1-2</td>\n",
       "      <td>{Media}</td>\n",
       "      <td>fulltime</td>\n",
       "      <td>1-10</td>\n",
       "      <td>{\"Data engineering\",\"Data modelling\",\"ML Appli...</td>\n",
       "      <td>ID_ZTPN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>NaN</td>\n",
       "      <td>1-2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>fulltime</td>\n",
       "      <td>50-200</td>\n",
       "      <td>{\"Data modelling\",\"Business/Data Analysis\",\"Da...</td>\n",
       "      <td>ID_P0AF</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>NaN</td>\n",
       "      <td>2-5</td>\n",
       "      <td>{Insurance}</td>\n",
       "      <td>fulltime</td>\n",
       "      <td>50-200</td>\n",
       "      <td>{\"Data engineering\",\"Data modelling\",\"Business...</td>\n",
       "      <td>ID_7YKR</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>NaN</td>\n",
       "      <td>&gt;5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>fulltime</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_147X</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>NaN</td>\n",
       "      <td>&lt;1</td>\n",
       "      <td>{Location}</td>\n",
       "      <td>internship</td>\n",
       "      <td>1-10</td>\n",
       "      <td>{\"Data engineering\"}</td>\n",
       "      <td>ID_5ONI</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>NaN</td>\n",
       "      <td>&gt;5</td>\n",
       "      <td>{Health}</td>\n",
       "      <td>fulltime</td>\n",
       "      <td>&gt;200</td>\n",
       "      <td>{\"Data engineering\",\"Data modelling\",\"ML Appli...</td>\n",
       "      <td>ID_SOPZ</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>True</td>\n",
       "      <td>1-2</td>\n",
       "      <td>{Security}</td>\n",
       "      <td>internship</td>\n",
       "      <td>1-10</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_PFPU</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>True</td>\n",
       "      <td>1-2</td>\n",
       "      <td>{\"Financial Services\"}</td>\n",
       "      <td>internship</td>\n",
       "      <td>&gt;200</td>\n",
       "      <td>{\"Data engineering\",\"ML Application Deployment...</td>\n",
       "      <td>ID_GYGN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_ZJIX</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>NaN</td>\n",
       "      <td>2-5</td>\n",
       "      <td>{Research}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>10-50</td>\n",
       "      <td>{\"Data engineering\",\"Data modelling\",\"ML Appli...</td>\n",
       "      <td>ID_QZLH</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>NaN</td>\n",
       "      <td>&gt;5</td>\n",
       "      <td>{Research}</td>\n",
       "      <td>fulltime</td>\n",
       "      <td>50-200</td>\n",
       "      <td>{\"Data engineering\",\"ML Application Deployment...</td>\n",
       "      <td>ID_128W</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>False</td>\n",
       "      <td>&lt;1</td>\n",
       "      <td>{Government,Health,\"Financial Services\"}</td>\n",
       "      <td>fulltime</td>\n",
       "      <td>50-200</td>\n",
       "      <td>{\"Business/Data Analysis\",\"Data modelling\"}</td>\n",
       "      <td>ID_4V5P</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>True</td>\n",
       "      <td>2-5</td>\n",
       "      <td>{Zindi}</td>\n",
       "      <td>fulltime</td>\n",
       "      <td>10-50</td>\n",
       "      <td>{\"Data modelling\",\"AI infrastructure\",\"ML Appl...</td>\n",
       "      <td>ID_ZU35</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>NaN</td>\n",
       "      <td>2-5</td>\n",
       "      <td>{Manufacturing,Sales}</td>\n",
       "      <td>fulltime</td>\n",
       "      <td>&gt;200</td>\n",
       "      <td>{\"Data engineering\",\"AI infrastructure\",\"Busin...</td>\n",
       "      <td>ID_93OJ</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>NaN</td>\n",
       "      <td>2-5</td>\n",
       "      <td>{\"Financial Services\",Banking}</td>\n",
       "      <td>fulltime</td>\n",
       "      <td>50-200</td>\n",
       "      <td>{\"Data engineering\",\"Data modelling\",\"Business...</td>\n",
       "      <td>ID_3CPP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>NaN</td>\n",
       "      <td>2-5</td>\n",
       "      <td>{Technology}</td>\n",
       "      <td>fulltime</td>\n",
       "      <td>10-50</td>\n",
       "      <td>{\"Data engineering\",\"Data modelling\"}</td>\n",
       "      <td>ID_QM44</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>NaN</td>\n",
       "      <td>2-5</td>\n",
       "      <td>{Government,Health,\"Financial Services\"}</td>\n",
       "      <td>fulltime</td>\n",
       "      <td>50-200</td>\n",
       "      <td>{\"Business/Data Analysis\"}</td>\n",
       "      <td>ID_J62B</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>NaN</td>\n",
       "      <td>&lt;1</td>\n",
       "      <td>{Research}</td>\n",
       "      <td>internship</td>\n",
       "      <td>10-50</td>\n",
       "      <td>{\"Data modelling\"}</td>\n",
       "      <td>ID_1BD0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>True</td>\n",
       "      <td>2-5</td>\n",
       "      <td>{Zindi}</td>\n",
       "      <td>contract</td>\n",
       "      <td>10-50</td>\n",
       "      <td>{\"Data engineering\",\"Data modelling\",\"AI infra...</td>\n",
       "      <td>ID_BXB1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>NaN</td>\n",
       "      <td>&lt;1</td>\n",
       "      <td>{Ambassador}</td>\n",
       "      <td>parttime</td>\n",
       "      <td>&gt;200</td>\n",
       "      <td>{\"Data modelling\",\"Business/Data Analysis\",\"AI...</td>\n",
       "      <td>ID_JYTV</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>NaN</td>\n",
       "      <td>2-5</td>\n",
       "      <td>{\"Financial Services\"}</td>\n",
       "      <td>fulltime</td>\n",
       "      <td>50-200</td>\n",
       "      <td>{\"Business/Data Analysis\",\"Data modelling\"}</td>\n",
       "      <td>ID_X4AC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>NaN</td>\n",
       "      <td>2-5</td>\n",
       "      <td>{\"Artificial Intelligence\"}</td>\n",
       "      <td>fulltime</td>\n",
       "      <td>50-200</td>\n",
       "      <td>{\"Data modelling\",\"Data engineering\"}</td>\n",
       "      <td>ID_KP8G</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>True</td>\n",
       "      <td>&lt;1</td>\n",
       "      <td>{Logistics}</td>\n",
       "      <td>internship</td>\n",
       "      <td>10-50</td>\n",
       "      <td>{\"Data modelling\",\"Data engineering\"}</td>\n",
       "      <td>ID_FTTN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>True</td>\n",
       "      <td>2-5</td>\n",
       "      <td>{\"Financial Services\"}</td>\n",
       "      <td>fulltime</td>\n",
       "      <td>10-50</td>\n",
       "      <td>{\"Business/Data Analysis\"}</td>\n",
       "      <td>ID_JJTS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>True</td>\n",
       "      <td>&lt;1</td>\n",
       "      <td>{Transportation}</td>\n",
       "      <td>internship</td>\n",
       "      <td>10-50</td>\n",
       "      <td>{\"ML Application Deployment\"}</td>\n",
       "      <td>ID_P6Q8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>NaN</td>\n",
       "      <td>1-2</td>\n",
       "      <td>{Scholarship}</td>\n",
       "      <td>internship</td>\n",
       "      <td>10-50</td>\n",
       "      <td>{\"Business/Data Analysis\",\"Data modelling\",\"Da...</td>\n",
       "      <td>ID_T5KV</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>NaN</td>\n",
       "      <td>2-5</td>\n",
       "      <td>{\"Financial Services\"}</td>\n",
       "      <td>fulltime</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{\"Data engineering\",\"Data modelling\",\"ML Appli...</td>\n",
       "      <td>ID_BJ32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>NaN</td>\n",
       "      <td>&gt;5</td>\n",
       "      <td>{Research}</td>\n",
       "      <td>contract</td>\n",
       "      <td>10-50</td>\n",
       "      <td>{\"Business/Data Analysis\"}</td>\n",
       "      <td>ID_R7BC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>NaN</td>\n",
       "      <td>2-5</td>\n",
       "      <td>{\"Management Consulting\"}</td>\n",
       "      <td>fulltime</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_L3KN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>NaN</td>\n",
       "      <td>&gt;5</td>\n",
       "      <td>{Health}</td>\n",
       "      <td>contract</td>\n",
       "      <td>50-200</td>\n",
       "      <td>{\"ML Application Deployment\",\"Business/Data An...</td>\n",
       "      <td>ID_54A8</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Remote Experience                                  Industry  \\\n",
       "0     NaN        2-5  {Government,Health,\"Financial Services\"}   \n",
       "1     NaN        NaN                                       NaN   \n",
       "2    True         >5                                       NaN   \n",
       "3     NaN         >5                                   {Media}   \n",
       "4     NaN        1-2                                   {Media}   \n",
       "5     NaN        1-2                                       NaN   \n",
       "6     NaN        2-5                               {Insurance}   \n",
       "7     NaN         >5                                       NaN   \n",
       "8     NaN         <1                                {Location}   \n",
       "9     NaN         >5                                  {Health}   \n",
       "10   True        1-2                                {Security}   \n",
       "11   True        1-2                    {\"Financial Services\"}   \n",
       "12    NaN        NaN                                       NaN   \n",
       "13    NaN        2-5                                {Research}   \n",
       "14    NaN         >5                                {Research}   \n",
       "15  False         <1  {Government,Health,\"Financial Services\"}   \n",
       "16   True        2-5                                   {Zindi}   \n",
       "17    NaN        2-5                     {Manufacturing,Sales}   \n",
       "18    NaN        2-5            {\"Financial Services\",Banking}   \n",
       "19    NaN        2-5                              {Technology}   \n",
       "20    NaN        2-5  {Government,Health,\"Financial Services\"}   \n",
       "21    NaN         <1                                {Research}   \n",
       "22   True        2-5                                   {Zindi}   \n",
       "23    NaN         <1                              {Ambassador}   \n",
       "24    NaN        2-5                    {\"Financial Services\"}   \n",
       "25    NaN        2-5               {\"Artificial Intelligence\"}   \n",
       "26   True         <1                               {Logistics}   \n",
       "27   True        2-5                    {\"Financial Services\"}   \n",
       "28   True         <1                          {Transportation}   \n",
       "29    NaN        1-2                             {Scholarship}   \n",
       "30    NaN        2-5                    {\"Financial Services\"}   \n",
       "31    NaN         >5                                {Research}   \n",
       "32    NaN        2-5                 {\"Management Consulting\"}   \n",
       "33    NaN         >5                                  {Health}   \n",
       "\n",
       "   Employment Type Company Size  \\\n",
       "0         fulltime       50-200   \n",
       "1              NaN          NaN   \n",
       "2         fulltime         >200   \n",
       "3         fulltime         1-10   \n",
       "4         fulltime         1-10   \n",
       "5         fulltime       50-200   \n",
       "6         fulltime       50-200   \n",
       "7         fulltime          NaN   \n",
       "8       internship         1-10   \n",
       "9         fulltime         >200   \n",
       "10      internship         1-10   \n",
       "11      internship         >200   \n",
       "12             NaN          NaN   \n",
       "13             NaN        10-50   \n",
       "14        fulltime       50-200   \n",
       "15        fulltime       50-200   \n",
       "16        fulltime        10-50   \n",
       "17        fulltime         >200   \n",
       "18        fulltime       50-200   \n",
       "19        fulltime        10-50   \n",
       "20        fulltime       50-200   \n",
       "21      internship        10-50   \n",
       "22        contract        10-50   \n",
       "23        parttime         >200   \n",
       "24        fulltime       50-200   \n",
       "25        fulltime       50-200   \n",
       "26      internship        10-50   \n",
       "27        fulltime        10-50   \n",
       "28      internship        10-50   \n",
       "29      internship        10-50   \n",
       "30        fulltime          NaN   \n",
       "31        contract        10-50   \n",
       "32        fulltime          NaN   \n",
       "33        contract       50-200   \n",
       "\n",
       "                               Data Science Functions   job_ID  \n",
       "0         {\"Data modelling\",\"Business/Data Analysis\"}  ID_VJ43  \n",
       "1                                                 NaN  ID_ZNZV  \n",
       "2               {\"Data engineering\",\"Data modelling\"}  ID_PZHM  \n",
       "3                                {\"Data engineering\"}  ID_1PBU  \n",
       "4   {\"Data engineering\",\"Data modelling\",\"ML Appli...  ID_ZTPN  \n",
       "5   {\"Data modelling\",\"Business/Data Analysis\",\"Da...  ID_P0AF  \n",
       "6   {\"Data engineering\",\"Data modelling\",\"Business...  ID_7YKR  \n",
       "7                                                 NaN  ID_147X  \n",
       "8                                {\"Data engineering\"}  ID_5ONI  \n",
       "9   {\"Data engineering\",\"Data modelling\",\"ML Appli...  ID_SOPZ  \n",
       "10                                                NaN  ID_PFPU  \n",
       "11  {\"Data engineering\",\"ML Application Deployment...  ID_GYGN  \n",
       "12                                                NaN  ID_ZJIX  \n",
       "13  {\"Data engineering\",\"Data modelling\",\"ML Appli...  ID_QZLH  \n",
       "14  {\"Data engineering\",\"ML Application Deployment...  ID_128W  \n",
       "15        {\"Business/Data Analysis\",\"Data modelling\"}  ID_4V5P  \n",
       "16  {\"Data modelling\",\"AI infrastructure\",\"ML Appl...  ID_ZU35  \n",
       "17  {\"Data engineering\",\"AI infrastructure\",\"Busin...  ID_93OJ  \n",
       "18  {\"Data engineering\",\"Data modelling\",\"Business...  ID_3CPP  \n",
       "19              {\"Data engineering\",\"Data modelling\"}  ID_QM44  \n",
       "20                         {\"Business/Data Analysis\"}  ID_J62B  \n",
       "21                                 {\"Data modelling\"}  ID_1BD0  \n",
       "22  {\"Data engineering\",\"Data modelling\",\"AI infra...  ID_BXB1  \n",
       "23  {\"Data modelling\",\"Business/Data Analysis\",\"AI...  ID_JYTV  \n",
       "24        {\"Business/Data Analysis\",\"Data modelling\"}  ID_X4AC  \n",
       "25              {\"Data modelling\",\"Data engineering\"}  ID_KP8G  \n",
       "26              {\"Data modelling\",\"Data engineering\"}  ID_FTTN  \n",
       "27                         {\"Business/Data Analysis\"}  ID_JJTS  \n",
       "28                      {\"ML Application Deployment\"}  ID_P6Q8  \n",
       "29  {\"Business/Data Analysis\",\"Data modelling\",\"Da...  ID_T5KV  \n",
       "30  {\"Data engineering\",\"Data modelling\",\"ML Appli...  ID_BJ32  \n",
       "31                         {\"Business/Data Analysis\"}  ID_R7BC  \n",
       "32                                                NaN  ID_L3KN  \n",
       "33  {\"ML Application Deployment\",\"Business/Data An...  ID_54A8  "
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_jobs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "37c549e9",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_jobs[\"job_ID\"] = df_jobs[\"job_ID\"].astype(\"string\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "e69bed92",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_jobs[\"job_ID\"] = \"job_\" + df_jobs[\"job_ID\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "ff25a006",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<StringArray>\n",
       "['job_ID_VJ43', 'job_ID_ZNZV', 'job_ID_PZHM', 'job_ID_1PBU', 'job_ID_ZTPN',\n",
       " 'job_ID_P0AF', 'job_ID_7YKR', 'job_ID_147X', 'job_ID_5ONI', 'job_ID_SOPZ',\n",
       " 'job_ID_PFPU', 'job_ID_GYGN', 'job_ID_ZJIX', 'job_ID_QZLH', 'job_ID_128W',\n",
       " 'job_ID_4V5P', 'job_ID_ZU35', 'job_ID_93OJ', 'job_ID_3CPP', 'job_ID_QM44',\n",
       " 'job_ID_J62B', 'job_ID_1BD0', 'job_ID_BXB1', 'job_ID_JYTV', 'job_ID_X4AC',\n",
       " 'job_ID_KP8G', 'job_ID_FTTN', 'job_ID_JJTS', 'job_ID_P6Q8', 'job_ID_T5KV',\n",
       " 'job_ID_BJ32', 'job_ID_R7BC', 'job_ID_L3KN', 'job_ID_54A8']\n",
       "Length: 34, dtype: string"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_jobs[\"job_ID\"].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "532695da",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_jobs.rename(columns={'job_ID': 'Title'},  inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "9010548d",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_userAct = pd.merge(df_userAct, df_jobs, how=\"left\", on=['Title'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "822646b0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>User_ID</th>\n",
       "      <th>Title</th>\n",
       "      <th>datetime Month</th>\n",
       "      <th>datetime Day_of_month</th>\n",
       "      <th>Theme</th>\n",
       "      <th>Blog Published Year</th>\n",
       "      <th>Blog Published Month</th>\n",
       "      <th>Published At Day_of_month</th>\n",
       "      <th>Remote</th>\n",
       "      <th>Experience</th>\n",
       "      <th>Industry</th>\n",
       "      <th>Employment Type</th>\n",
       "      <th>Company Size</th>\n",
       "      <th>Data Science Functions</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ID_RT43AK77</td>\n",
       "      <td>comp_ID_IV5D</td>\n",
       "      <td>11</td>\n",
       "      <td>22</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ID_RT43AK77</td>\n",
       "      <td>comp_ID_IV5D</td>\n",
       "      <td>11</td>\n",
       "      <td>23</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ID_RT43AK77</td>\n",
       "      <td>comp_ID_IV5D</td>\n",
       "      <td>11</td>\n",
       "      <td>23</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ID_RT43AK77</td>\n",
       "      <td>comp_ID_IV5D</td>\n",
       "      <td>11</td>\n",
       "      <td>23</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ID_RT43AK77</td>\n",
       "      <td>comp_ID_IV5D</td>\n",
       "      <td>11</td>\n",
       "      <td>23</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>317287</th>\n",
       "      <td>ID_Z759BCS3</td>\n",
       "      <td>job_ID_5ONI</td>\n",
       "      <td>11</td>\n",
       "      <td>14</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>&lt;1</td>\n",
       "      <td>{Location}</td>\n",
       "      <td>internship</td>\n",
       "      <td>1-10</td>\n",
       "      <td>{\"Data engineering\"}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>317288</th>\n",
       "      <td>ID_WNJMADF8</td>\n",
       "      <td>job_ID_5ONI</td>\n",
       "      <td>11</td>\n",
       "      <td>14</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>&lt;1</td>\n",
       "      <td>{Location}</td>\n",
       "      <td>internship</td>\n",
       "      <td>1-10</td>\n",
       "      <td>{\"Data engineering\"}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>317289</th>\n",
       "      <td>ID_7NBY9MV2</td>\n",
       "      <td>job_ID_5ONI</td>\n",
       "      <td>11</td>\n",
       "      <td>20</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>&lt;1</td>\n",
       "      <td>{Location}</td>\n",
       "      <td>internship</td>\n",
       "      <td>1-10</td>\n",
       "      <td>{\"Data engineering\"}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>317290</th>\n",
       "      <td>ID_LHPJJSOK</td>\n",
       "      <td>job_ID_5ONI</td>\n",
       "      <td>11</td>\n",
       "      <td>14</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>&lt;1</td>\n",
       "      <td>{Location}</td>\n",
       "      <td>internship</td>\n",
       "      <td>1-10</td>\n",
       "      <td>{\"Data engineering\"}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>317291</th>\n",
       "      <td>ID_LCFNGARD</td>\n",
       "      <td>job_ID_5ONI</td>\n",
       "      <td>11</td>\n",
       "      <td>17</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>&lt;1</td>\n",
       "      <td>{Location}</td>\n",
       "      <td>internship</td>\n",
       "      <td>1-10</td>\n",
       "      <td>{\"Data engineering\"}</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>317292 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            User_ID         Title datetime Month  datetime Day_of_month  \\\n",
       "0       ID_RT43AK77  comp_ID_IV5D             11                     22   \n",
       "1       ID_RT43AK77  comp_ID_IV5D             11                     23   \n",
       "2       ID_RT43AK77  comp_ID_IV5D             11                     23   \n",
       "3       ID_RT43AK77  comp_ID_IV5D             11                     23   \n",
       "4       ID_RT43AK77  comp_ID_IV5D             11                     23   \n",
       "...             ...           ...            ...                    ...   \n",
       "317287  ID_Z759BCS3   job_ID_5ONI             11                     14   \n",
       "317288  ID_WNJMADF8   job_ID_5ONI             11                     14   \n",
       "317289  ID_7NBY9MV2   job_ID_5ONI             11                     20   \n",
       "317290  ID_LHPJJSOK   job_ID_5ONI             11                     14   \n",
       "317291  ID_LCFNGARD   job_ID_5ONI             11                     17   \n",
       "\n",
       "        Theme  Blog Published Year  Blog Published Month  \\\n",
       "0         NaN                  NaN                   NaN   \n",
       "1         NaN                  NaN                   NaN   \n",
       "2         NaN                  NaN                   NaN   \n",
       "3         NaN                  NaN                   NaN   \n",
       "4         NaN                  NaN                   NaN   \n",
       "...       ...                  ...                   ...   \n",
       "317287    NaN                  NaN                   NaN   \n",
       "317288    NaN                  NaN                   NaN   \n",
       "317289    NaN                  NaN                   NaN   \n",
       "317290    NaN                  NaN                   NaN   \n",
       "317291    NaN                  NaN                   NaN   \n",
       "\n",
       "        Published At Day_of_month Remote Experience    Industry  \\\n",
       "0                             NaN    NaN        NaN         NaN   \n",
       "1                             NaN    NaN        NaN         NaN   \n",
       "2                             NaN    NaN        NaN         NaN   \n",
       "3                             NaN    NaN        NaN         NaN   \n",
       "4                             NaN    NaN        NaN         NaN   \n",
       "...                           ...    ...        ...         ...   \n",
       "317287                        NaN    NaN         <1  {Location}   \n",
       "317288                        NaN    NaN         <1  {Location}   \n",
       "317289                        NaN    NaN         <1  {Location}   \n",
       "317290                        NaN    NaN         <1  {Location}   \n",
       "317291                        NaN    NaN         <1  {Location}   \n",
       "\n",
       "       Employment Type Company Size Data Science Functions  \n",
       "0                  NaN          NaN                    NaN  \n",
       "1                  NaN          NaN                    NaN  \n",
       "2                  NaN          NaN                    NaN  \n",
       "3                  NaN          NaN                    NaN  \n",
       "4                  NaN          NaN                    NaN  \n",
       "...                ...          ...                    ...  \n",
       "317287      internship         1-10   {\"Data engineering\"}  \n",
       "317288      internship         1-10   {\"Data engineering\"}  \n",
       "317289      internship         1-10   {\"Data engineering\"}  \n",
       "317290      internship         1-10   {\"Data engineering\"}  \n",
       "317291      internship         1-10   {\"Data engineering\"}  \n",
       "\n",
       "[317292 rows x 14 columns]"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_userAct"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "06425e79",
   "metadata": {},
   "source": [
    "# USER ACTIVITY(Contd.)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "89850e46",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 317292 entries, 0 to 317291\n",
      "Data columns (total 14 columns):\n",
      " #   Column                     Non-Null Count   Dtype  \n",
      "---  ------                     --------------   -----  \n",
      " 0   User_ID                    317292 non-null  string \n",
      " 1   Title                      317292 non-null  object \n",
      " 2   datetime Month             317292 non-null  string \n",
      " 3   datetime Day_of_month      317292 non-null  int64  \n",
      " 4   Theme                      2709 non-null    float64\n",
      " 5   Blog Published Year        2712 non-null    float64\n",
      " 6   Blog Published Month       2712 non-null    float64\n",
      " 7   Published At Day_of_month  2712 non-null    float64\n",
      " 8   Remote                     493 non-null     object \n",
      " 9   Experience                 1492 non-null    object \n",
      " 10  Industry                   1417 non-null    object \n",
      " 11  Employment Type            1488 non-null    object \n",
      " 12  Company Size               1463 non-null    object \n",
      " 13  Data Science Functions     1468 non-null    object \n",
      "dtypes: float64(4), int64(1), object(7), string(2)\n",
      "memory usage: 36.3+ MB\n"
     ]
    }
   ],
   "source": [
    "df_userAct.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "09ee6f73",
   "metadata": {},
   "outputs": [],
   "source": [
    "new = df_userAct[\"datetime Month\"].copy()\n",
    "df_userAct[\"User_ID\"] = df_userAct[\"User_ID\"].str.cat(new, sep =\"_Month_\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "6cf63d8c",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_userAct.drop('datetime Month', inplace=True, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "93a2c424",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_userAct[\"Title\"] = df_userAct[\"Title\"].astype(\"string\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "ff742265",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_userAct[\"Title\"] = df_userAct[\"Title\"].str[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "0a363f6c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<StringArray>\n",
       "[                     'comp_ID_IV5D',                      'comp_ID_M3ST',\n",
       "                      'comp_ID_ZN92',                      'comp_ID_DD5S',\n",
       "                      'comp_ID_1261',                      'comp_ID_X0PD',\n",
       "                      'comp_ID_L3EB',                      'comp_ID_JPR9',\n",
       "                      'comp_ID_TDAE',                      'comp_ID_Q8XJ',\n",
       " ...\n",
       "                   'Votes (Up/Down)',       'Transferred Team Leadership',\n",
       "                        'badge_PLDS',                  'Changed Password',\n",
       "                      'Deleted Team', 'Accepted Team Leadership Transfer',\n",
       "  'Revoked Team Leadership Transfer',                  'Report Something',\n",
       "           'Kicked Member From Team',                         'Left Team']\n",
       "Length: 397, dtype: string"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_userAct[\"Title\"].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "46fb9c27",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "1c9862de",
   "metadata": {},
   "outputs": [],
   "source": [
    "le = preprocessing.LabelEncoder()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "9b1b7a16",
   "metadata": {},
   "outputs": [],
   "source": [
    "# le.fit(df_userAct[\"Title\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "ef986fc5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_userAct[\"Title\"] = le.transform(df_userAct[\"Title\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "bd5e1a33",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_userAct[\"Title\"].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "c90ab218",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Viewed_All_Competitions                 58176\n",
       "comp_ID                                 55540\n",
       "Viewed_All_Discussions                  47172\n",
       "Viewed_All_Learning_Pages               22930\n",
       "Downloaded_Competition_Datafile         19578\n",
       "Updated_Profile                         14748\n",
       "Created_Submission                      13649\n",
       "identify                                12339\n",
       "Signed_Up                               10862\n",
       "create_alias                             9346\n",
       "badge_OCZE                               8864\n",
       "Confirmed_Email                          8818\n",
       "Updated_Discussion                       6498\n",
       "Joined_Competition                       6086\n",
       "Viewed_All_Jobs                          4139\n",
       "Signed_In                                4100\n",
       "Updated_Submission                       2929\n",
       "blog_ID                                  2712\n",
       "badge_HYIO                               1760\n",
       "Signed_Out                               1660\n",
       "job_ID                                   1492\n",
       "Viewed_Discussion                        1207\n",
       "Updated_Comment                           656\n",
       "badge_MLPD                                457\n",
       "Joined_Team                               384\n",
       "Invited_Member_To_Team                    344\n",
       "Created_Team                              310\n",
       "Votes_(Up/Down)                           149\n",
       "Applied_To_Job                            126\n",
       "Viewed_FAQ                                104\n",
       "Updated_Team                               42\n",
       "badge_PLDS                                 40\n",
       "Deleted_Team                               28\n",
       "Transferred_Team_Leadership                14\n",
       "Updated_Discussion_Team_Participants       10\n",
       "Accepted_Team_Leadership_Transfer           9\n",
       "Changed_Password                            5\n",
       "Left_Team                                   4\n",
       "Kicked_Member_From_Team                     3\n",
       "Revoked_Team_Leadership_Transfer            1\n",
       "Report_Something                            1\n",
       "Name: Title, dtype: int64"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def remove_id(x):\n",
    "    x = x.replace('$', '')\n",
    "    if 'ID' in x:\n",
    "        return \"_\".join(x.split('_')[:-1])\n",
    "    \n",
    "    return x.replace(' ', '_')\n",
    "\n",
    "df_userAct.Title.apply(remove_id).value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "b27061ec",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['comp_ID', 'job_ID', 'blog_ID', 'create_alias', 'identify',\n",
       "       'Signed_Up', 'Viewed_All_Competitions', 'Viewed_All_Discussions',\n",
       "       'Viewed_All_Learning_Pages', 'Confirmed_Email',\n",
       "       'Joined_Competition', 'Downloaded_Competition_Datafile',\n",
       "       'Signed_In', 'Updated_Discussion', 'Joined_Team',\n",
       "       'Created_Submission', 'Viewed_Discussion', 'Updated_Submission',\n",
       "       'Signed_Out', 'badge_OCZE', 'Updated_Profile', 'Viewed_FAQ',\n",
       "       'Viewed_All_Jobs', 'Created_Team', 'Invited_Member_To_Team',\n",
       "       'Updated_Team', 'Updated_Comment',\n",
       "       'Updated_Discussion_Team_Participants', 'badge_HYIO', 'badge_MLPD',\n",
       "       'Applied_To_Job', 'Votes_(Up/Down)', 'Transferred_Team_Leadership',\n",
       "       'badge_PLDS', 'Changed_Password', 'Deleted_Team',\n",
       "       'Accepted_Team_Leadership_Transfer',\n",
       "       'Revoked_Team_Leadership_Transfer', 'Report_Something',\n",
       "       'Kicked_Member_From_Team', 'Left_Team'], dtype=object)"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_userAct['main_activities'] = df_userAct.Title.apply(remove_id)\n",
    "df_userAct.main_activities.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "c9504bf5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Viewed_All_Competitions', 'comp_ID', 'Viewed_All_Discussions',\n",
       "       'Viewed_All_Learning_Pages', 'Downloaded_Competition_Datafile',\n",
       "       'Updated_Profile', 'Created_Submission', 'identify', 'Signed_Up',\n",
       "       'create_alias', 'badge_OCZE', 'Confirmed_Email', 'Updated_Discussion',\n",
       "       'Joined_Competition', 'Viewed_All_Jobs', 'Signed_In',\n",
       "       'Updated_Submission', 'blog_ID', 'badge_HYIO', 'Signed_Out', 'job_ID',\n",
       "       'Viewed_Discussion'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "act_vc = df_userAct.main_activities.value_counts()\n",
    "selected_activities = act_vc[act_vc > 1000].keys()\n",
    "selected_activities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "9fa2c617",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>User_ID</th>\n",
       "      <th>Title</th>\n",
       "      <th>datetime Day_of_month</th>\n",
       "      <th>Theme</th>\n",
       "      <th>Blog Published Year</th>\n",
       "      <th>Blog Published Month</th>\n",
       "      <th>Published At Day_of_month</th>\n",
       "      <th>Remote</th>\n",
       "      <th>Experience</th>\n",
       "      <th>Industry</th>\n",
       "      <th>Employment Type</th>\n",
       "      <th>Company Size</th>\n",
       "      <th>Data Science Functions</th>\n",
       "      <th>main_activities</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ID_RT43AK77_Month_11</td>\n",
       "      <td>comp_ID_IV5D</td>\n",
       "      <td>22</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>comp_ID</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ID_RT43AK77_Month_11</td>\n",
       "      <td>comp_ID_IV5D</td>\n",
       "      <td>23</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>comp_ID</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ID_RT43AK77_Month_11</td>\n",
       "      <td>comp_ID_IV5D</td>\n",
       "      <td>23</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>comp_ID</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ID_RT43AK77_Month_11</td>\n",
       "      <td>comp_ID_IV5D</td>\n",
       "      <td>23</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>comp_ID</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ID_RT43AK77_Month_11</td>\n",
       "      <td>comp_ID_IV5D</td>\n",
       "      <td>23</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>comp_ID</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>314600</th>\n",
       "      <td>ID_Z759BCS3_Month_11</td>\n",
       "      <td>job_ID_5ONI</td>\n",
       "      <td>14</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>&lt;1</td>\n",
       "      <td>{Location}</td>\n",
       "      <td>internship</td>\n",
       "      <td>1-10</td>\n",
       "      <td>{\"Data engineering\"}</td>\n",
       "      <td>job_ID</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>314601</th>\n",
       "      <td>ID_WNJMADF8_Month_11</td>\n",
       "      <td>job_ID_5ONI</td>\n",
       "      <td>14</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>&lt;1</td>\n",
       "      <td>{Location}</td>\n",
       "      <td>internship</td>\n",
       "      <td>1-10</td>\n",
       "      <td>{\"Data engineering\"}</td>\n",
       "      <td>job_ID</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>314602</th>\n",
       "      <td>ID_7NBY9MV2_Month_11</td>\n",
       "      <td>job_ID_5ONI</td>\n",
       "      <td>20</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>&lt;1</td>\n",
       "      <td>{Location}</td>\n",
       "      <td>internship</td>\n",
       "      <td>1-10</td>\n",
       "      <td>{\"Data engineering\"}</td>\n",
       "      <td>job_ID</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>314603</th>\n",
       "      <td>ID_LHPJJSOK_Month_11</td>\n",
       "      <td>job_ID_5ONI</td>\n",
       "      <td>14</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>&lt;1</td>\n",
       "      <td>{Location}</td>\n",
       "      <td>internship</td>\n",
       "      <td>1-10</td>\n",
       "      <td>{\"Data engineering\"}</td>\n",
       "      <td>job_ID</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>314604</th>\n",
       "      <td>ID_LCFNGARD_Month_11</td>\n",
       "      <td>job_ID_5ONI</td>\n",
       "      <td>17</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>&lt;1</td>\n",
       "      <td>{Location}</td>\n",
       "      <td>internship</td>\n",
       "      <td>1-10</td>\n",
       "      <td>{\"Data engineering\"}</td>\n",
       "      <td>job_ID</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>314605 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                     User_ID         Title  datetime Day_of_month  Theme  \\\n",
       "0       ID_RT43AK77_Month_11  comp_ID_IV5D                     22    NaN   \n",
       "1       ID_RT43AK77_Month_11  comp_ID_IV5D                     23    NaN   \n",
       "2       ID_RT43AK77_Month_11  comp_ID_IV5D                     23    NaN   \n",
       "3       ID_RT43AK77_Month_11  comp_ID_IV5D                     23    NaN   \n",
       "4       ID_RT43AK77_Month_11  comp_ID_IV5D                     23    NaN   \n",
       "...                      ...           ...                    ...    ...   \n",
       "314600  ID_Z759BCS3_Month_11   job_ID_5ONI                     14    NaN   \n",
       "314601  ID_WNJMADF8_Month_11   job_ID_5ONI                     14    NaN   \n",
       "314602  ID_7NBY9MV2_Month_11   job_ID_5ONI                     20    NaN   \n",
       "314603  ID_LHPJJSOK_Month_11   job_ID_5ONI                     14    NaN   \n",
       "314604  ID_LCFNGARD_Month_11   job_ID_5ONI                     17    NaN   \n",
       "\n",
       "        Blog Published Year  Blog Published Month  Published At Day_of_month  \\\n",
       "0                       NaN                   NaN                        NaN   \n",
       "1                       NaN                   NaN                        NaN   \n",
       "2                       NaN                   NaN                        NaN   \n",
       "3                       NaN                   NaN                        NaN   \n",
       "4                       NaN                   NaN                        NaN   \n",
       "...                     ...                   ...                        ...   \n",
       "314600                  NaN                   NaN                        NaN   \n",
       "314601                  NaN                   NaN                        NaN   \n",
       "314602                  NaN                   NaN                        NaN   \n",
       "314603                  NaN                   NaN                        NaN   \n",
       "314604                  NaN                   NaN                        NaN   \n",
       "\n",
       "       Remote Experience    Industry Employment Type Company Size  \\\n",
       "0         NaN        NaN         NaN             NaN          NaN   \n",
       "1         NaN        NaN         NaN             NaN          NaN   \n",
       "2         NaN        NaN         NaN             NaN          NaN   \n",
       "3         NaN        NaN         NaN             NaN          NaN   \n",
       "4         NaN        NaN         NaN             NaN          NaN   \n",
       "...       ...        ...         ...             ...          ...   \n",
       "314600    NaN         <1  {Location}      internship         1-10   \n",
       "314601    NaN         <1  {Location}      internship         1-10   \n",
       "314602    NaN         <1  {Location}      internship         1-10   \n",
       "314603    NaN         <1  {Location}      internship         1-10   \n",
       "314604    NaN         <1  {Location}      internship         1-10   \n",
       "\n",
       "       Data Science Functions main_activities  \n",
       "0                         NaN         comp_ID  \n",
       "1                         NaN         comp_ID  \n",
       "2                         NaN         comp_ID  \n",
       "3                         NaN         comp_ID  \n",
       "4                         NaN         comp_ID  \n",
       "...                       ...             ...  \n",
       "314600   {\"Data engineering\"}          job_ID  \n",
       "314601   {\"Data engineering\"}          job_ID  \n",
       "314602   {\"Data engineering\"}          job_ID  \n",
       "314603   {\"Data engineering\"}          job_ID  \n",
       "314604   {\"Data engineering\"}          job_ID  \n",
       "\n",
       "[314605 rows x 14 columns]"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_userAct = df_userAct[df_userAct.main_activities.isin(selected_activities)].reset_index(drop=True)\n",
    "df_userAct"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "4b0c0512",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>User_ID</th>\n",
       "      <th>main_activities</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>111314</th>\n",
       "      <td>ID_ZZVUJ45W_Month_3</td>\n",
       "      <td>Downloaded_Competition_Datafile</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>111318</th>\n",
       "      <td>ID_ZZVUJ45W_Month_3</td>\n",
       "      <td>Viewed_All_Discussions</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>111313</th>\n",
       "      <td>ID_ZZVUJ45W_Month_3</td>\n",
       "      <td>Confirmed_Email</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>111315</th>\n",
       "      <td>ID_ZZVUJ45W_Month_3</td>\n",
       "      <td>Joined_Competition</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>111316</th>\n",
       "      <td>ID_ZZVUJ45W_Month_3</td>\n",
       "      <td>Signed_Up</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ID_000H9XOP_Month_5</td>\n",
       "      <td>Viewed_All_Competitions</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ID_000H9XOP_Month_5</td>\n",
       "      <td>Viewed_All_Discussions</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ID_000H9XOP_Month_5</td>\n",
       "      <td>comp_ID</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ID_000H9XOP_Month_5</td>\n",
       "      <td>create_alias</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>ID_000H9XOP_Month_5</td>\n",
       "      <td>identify</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>111323 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                    User_ID                  main_activities  count\n",
       "111314  ID_ZZVUJ45W_Month_3  Downloaded_Competition_Datafile      3\n",
       "111318  ID_ZZVUJ45W_Month_3           Viewed_All_Discussions      2\n",
       "111313  ID_ZZVUJ45W_Month_3                  Confirmed_Email      1\n",
       "111315  ID_ZZVUJ45W_Month_3               Joined_Competition      1\n",
       "111316  ID_ZZVUJ45W_Month_3                        Signed_Up      1\n",
       "...                     ...                              ...    ...\n",
       "1       ID_000H9XOP_Month_5          Viewed_All_Competitions      1\n",
       "2       ID_000H9XOP_Month_5           Viewed_All_Discussions      1\n",
       "3       ID_000H9XOP_Month_5                          comp_ID      1\n",
       "4       ID_000H9XOP_Month_5                     create_alias      1\n",
       "5       ID_000H9XOP_Month_5                         identify      1\n",
       "\n",
       "[111323 rows x 3 columns]"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset = df_userAct.groupby(['User_ID', 'main_activities']).size().reset_index().\\\n",
    "    rename(columns={0: 'count'}).\\\n",
    "        sort_values(['User_ID', 'count'], ascending=False)\n",
    "\n",
    "dataset "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "6db17e9f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "22"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.main_activities.nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "9d248410",
   "metadata": {},
   "outputs": [],
   "source": [
    "all_activities = dataset.main_activities.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "6968f049",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>User_ID</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ID_ZZQK0LU6_Month_3</td>\n",
       "      <td>35</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ID_ZZJKFBR5_Month_2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ID_ZZJKFBR5_Month_12</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ID_ZZFZI3DK_Month_5</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ID_ZZ852GLE_Month_12</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7411</th>\n",
       "      <td>ID_01BEG5HQ_Month_3</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7412</th>\n",
       "      <td>ID_015M2NE1_Month_12</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7413</th>\n",
       "      <td>ID_011YM9VH_Month_3</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7414</th>\n",
       "      <td>ID_011YM9VH_Month_2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7415</th>\n",
       "      <td>ID_000H9XOP_Month_5</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>7416 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                   User_ID  count\n",
       "0      ID_ZZQK0LU6_Month_3     35\n",
       "1      ID_ZZJKFBR5_Month_2      2\n",
       "2     ID_ZZJKFBR5_Month_12     24\n",
       "3      ID_ZZFZI3DK_Month_5      2\n",
       "4     ID_ZZ852GLE_Month_12      7\n",
       "...                    ...    ...\n",
       "7411   ID_01BEG5HQ_Month_3      8\n",
       "7412  ID_015M2NE1_Month_12      1\n",
       "7413   ID_011YM9VH_Month_3      4\n",
       "7414   ID_011YM9VH_Month_2      1\n",
       "7415   ID_000H9XOP_Month_5      1\n",
       "\n",
       "[7416 rows x 2 columns]"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset[dataset.main_activities == \"comp_ID\"].drop(columns='main_activities').reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "dc0e775d",
   "metadata": {},
   "outputs": [],
   "source": [
    "for activity in all_activities:\n",
    "    dataset[activity] = np.zeros((dataset.shape[0], 1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "aeb34fa6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>User_ID</th>\n",
       "      <th>main_activities</th>\n",
       "      <th>count</th>\n",
       "      <th>Downloaded_Competition_Datafile</th>\n",
       "      <th>Viewed_All_Discussions</th>\n",
       "      <th>Confirmed_Email</th>\n",
       "      <th>Joined_Competition</th>\n",
       "      <th>Signed_Up</th>\n",
       "      <th>Viewed_All_Competitions</th>\n",
       "      <th>Viewed_All_Learning_Pages</th>\n",
       "      <th>...</th>\n",
       "      <th>job_ID</th>\n",
       "      <th>Updated_Discussion</th>\n",
       "      <th>Viewed_All_Jobs</th>\n",
       "      <th>Created_Submission</th>\n",
       "      <th>Signed_Out</th>\n",
       "      <th>Updated_Profile</th>\n",
       "      <th>blog_ID</th>\n",
       "      <th>Viewed_Discussion</th>\n",
       "      <th>Updated_Submission</th>\n",
       "      <th>badge_HYIO</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>111314</th>\n",
       "      <td>ID_ZZVUJ45W_Month_3</td>\n",
       "      <td>Downloaded_Competition_Datafile</td>\n",
       "      <td>3</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>111318</th>\n",
       "      <td>ID_ZZVUJ45W_Month_3</td>\n",
       "      <td>Viewed_All_Discussions</td>\n",
       "      <td>2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>111313</th>\n",
       "      <td>ID_ZZVUJ45W_Month_3</td>\n",
       "      <td>Confirmed_Email</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>111315</th>\n",
       "      <td>ID_ZZVUJ45W_Month_3</td>\n",
       "      <td>Joined_Competition</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>111316</th>\n",
       "      <td>ID_ZZVUJ45W_Month_3</td>\n",
       "      <td>Signed_Up</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ID_000H9XOP_Month_5</td>\n",
       "      <td>Viewed_All_Competitions</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ID_000H9XOP_Month_5</td>\n",
       "      <td>Viewed_All_Discussions</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ID_000H9XOP_Month_5</td>\n",
       "      <td>comp_ID</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ID_000H9XOP_Month_5</td>\n",
       "      <td>create_alias</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>ID_000H9XOP_Month_5</td>\n",
       "      <td>identify</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>111323 rows × 25 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                    User_ID                  main_activities  count  \\\n",
       "111314  ID_ZZVUJ45W_Month_3  Downloaded_Competition_Datafile      3   \n",
       "111318  ID_ZZVUJ45W_Month_3           Viewed_All_Discussions      2   \n",
       "111313  ID_ZZVUJ45W_Month_3                  Confirmed_Email      1   \n",
       "111315  ID_ZZVUJ45W_Month_3               Joined_Competition      1   \n",
       "111316  ID_ZZVUJ45W_Month_3                        Signed_Up      1   \n",
       "...                     ...                              ...    ...   \n",
       "1       ID_000H9XOP_Month_5          Viewed_All_Competitions      1   \n",
       "2       ID_000H9XOP_Month_5           Viewed_All_Discussions      1   \n",
       "3       ID_000H9XOP_Month_5                          comp_ID      1   \n",
       "4       ID_000H9XOP_Month_5                     create_alias      1   \n",
       "5       ID_000H9XOP_Month_5                         identify      1   \n",
       "\n",
       "        Downloaded_Competition_Datafile  Viewed_All_Discussions  \\\n",
       "111314                              0.0                     0.0   \n",
       "111318                              0.0                     0.0   \n",
       "111313                              0.0                     0.0   \n",
       "111315                              0.0                     0.0   \n",
       "111316                              0.0                     0.0   \n",
       "...                                 ...                     ...   \n",
       "1                                   0.0                     0.0   \n",
       "2                                   0.0                     0.0   \n",
       "3                                   0.0                     0.0   \n",
       "4                                   0.0                     0.0   \n",
       "5                                   0.0                     0.0   \n",
       "\n",
       "        Confirmed_Email  Joined_Competition  Signed_Up  \\\n",
       "111314              0.0                 0.0        0.0   \n",
       "111318              0.0                 0.0        0.0   \n",
       "111313              0.0                 0.0        0.0   \n",
       "111315              0.0                 0.0        0.0   \n",
       "111316              0.0                 0.0        0.0   \n",
       "...                 ...                 ...        ...   \n",
       "1                   0.0                 0.0        0.0   \n",
       "2                   0.0                 0.0        0.0   \n",
       "3                   0.0                 0.0        0.0   \n",
       "4                   0.0                 0.0        0.0   \n",
       "5                   0.0                 0.0        0.0   \n",
       "\n",
       "        Viewed_All_Competitions  Viewed_All_Learning_Pages  ...  job_ID  \\\n",
       "111314                      0.0                        0.0  ...     0.0   \n",
       "111318                      0.0                        0.0  ...     0.0   \n",
       "111313                      0.0                        0.0  ...     0.0   \n",
       "111315                      0.0                        0.0  ...     0.0   \n",
       "111316                      0.0                        0.0  ...     0.0   \n",
       "...                         ...                        ...  ...     ...   \n",
       "1                           0.0                        0.0  ...     0.0   \n",
       "2                           0.0                        0.0  ...     0.0   \n",
       "3                           0.0                        0.0  ...     0.0   \n",
       "4                           0.0                        0.0  ...     0.0   \n",
       "5                           0.0                        0.0  ...     0.0   \n",
       "\n",
       "        Updated_Discussion  Viewed_All_Jobs  Created_Submission  Signed_Out  \\\n",
       "111314                 0.0              0.0                 0.0         0.0   \n",
       "111318                 0.0              0.0                 0.0         0.0   \n",
       "111313                 0.0              0.0                 0.0         0.0   \n",
       "111315                 0.0              0.0                 0.0         0.0   \n",
       "111316                 0.0              0.0                 0.0         0.0   \n",
       "...                    ...              ...                 ...         ...   \n",
       "1                      0.0              0.0                 0.0         0.0   \n",
       "2                      0.0              0.0                 0.0         0.0   \n",
       "3                      0.0              0.0                 0.0         0.0   \n",
       "4                      0.0              0.0                 0.0         0.0   \n",
       "5                      0.0              0.0                 0.0         0.0   \n",
       "\n",
       "        Updated_Profile  blog_ID  Viewed_Discussion  Updated_Submission  \\\n",
       "111314              0.0      0.0                0.0                 0.0   \n",
       "111318              0.0      0.0                0.0                 0.0   \n",
       "111313              0.0      0.0                0.0                 0.0   \n",
       "111315              0.0      0.0                0.0                 0.0   \n",
       "111316              0.0      0.0                0.0                 0.0   \n",
       "...                 ...      ...                ...                 ...   \n",
       "1                   0.0      0.0                0.0                 0.0   \n",
       "2                   0.0      0.0                0.0                 0.0   \n",
       "3                   0.0      0.0                0.0                 0.0   \n",
       "4                   0.0      0.0                0.0                 0.0   \n",
       "5                   0.0      0.0                0.0                 0.0   \n",
       "\n",
       "        badge_HYIO  \n",
       "111314         0.0  \n",
       "111318         0.0  \n",
       "111313         0.0  \n",
       "111315         0.0  \n",
       "111316         0.0  \n",
       "...            ...  \n",
       "1              0.0  \n",
       "2              0.0  \n",
       "3              0.0  \n",
       "4              0.0  \n",
       "5              0.0  \n",
       "\n",
       "[111323 rows x 25 columns]"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "a62c733e",
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(len(dataset)):\n",
    "    for activity in all_activities:\n",
    "        if dataset.loc[i, \"main_activities\"] == activity:\n",
    "            dataset.loc[i, activity] = dataset.loc[i, \"count\"] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "3cb59242",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = dataset.groupby(\"User_ID\").sum().reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "298b7bbf",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset.drop(\"count\", inplace=True, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "c3f9523f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 15139 entries, 0 to 15138\n",
      "Data columns (total 23 columns):\n",
      " #   Column                           Non-Null Count  Dtype  \n",
      "---  ------                           --------------  -----  \n",
      " 0   User_ID                          15139 non-null  object \n",
      " 1   Downloaded_Competition_Datafile  15139 non-null  float64\n",
      " 2   Viewed_All_Discussions           15139 non-null  float64\n",
      " 3   Confirmed_Email                  15139 non-null  float64\n",
      " 4   Joined_Competition               15139 non-null  float64\n",
      " 5   Signed_Up                        15139 non-null  float64\n",
      " 6   Viewed_All_Competitions          15139 non-null  float64\n",
      " 7   Viewed_All_Learning_Pages        15139 non-null  float64\n",
      " 8   badge_OCZE                       15139 non-null  float64\n",
      " 9   create_alias                     15139 non-null  float64\n",
      " 10  identify                         15139 non-null  float64\n",
      " 11  comp_ID                          15139 non-null  float64\n",
      " 12  Signed_In                        15139 non-null  float64\n",
      " 13  job_ID                           15139 non-null  float64\n",
      " 14  Updated_Discussion               15139 non-null  float64\n",
      " 15  Viewed_All_Jobs                  15139 non-null  float64\n",
      " 16  Created_Submission               15139 non-null  float64\n",
      " 17  Signed_Out                       15139 non-null  float64\n",
      " 18  Updated_Profile                  15139 non-null  float64\n",
      " 19  blog_ID                          15139 non-null  float64\n",
      " 20  Viewed_Discussion                15139 non-null  float64\n",
      " 21  Updated_Submission               15139 non-null  float64\n",
      " 22  badge_HYIO                       15139 non-null  float64\n",
      "dtypes: float64(22), object(1)\n",
      "memory usage: 2.7+ MB\n"
     ]
    }
   ],
   "source": [
    "dataset.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "d566f32d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>User_ID</th>\n",
       "      <th>Downloaded_Competition_Datafile</th>\n",
       "      <th>Viewed_All_Discussions</th>\n",
       "      <th>Confirmed_Email</th>\n",
       "      <th>Joined_Competition</th>\n",
       "      <th>Signed_Up</th>\n",
       "      <th>Viewed_All_Competitions</th>\n",
       "      <th>Viewed_All_Learning_Pages</th>\n",
       "      <th>badge_OCZE</th>\n",
       "      <th>create_alias</th>\n",
       "      <th>...</th>\n",
       "      <th>job_ID</th>\n",
       "      <th>Updated_Discussion</th>\n",
       "      <th>Viewed_All_Jobs</th>\n",
       "      <th>Created_Submission</th>\n",
       "      <th>Signed_Out</th>\n",
       "      <th>Updated_Profile</th>\n",
       "      <th>blog_ID</th>\n",
       "      <th>Viewed_Discussion</th>\n",
       "      <th>Updated_Submission</th>\n",
       "      <th>badge_HYIO</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ID_000H9XOP_Month_5</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ID_003YJHQA_Month_4</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ID_004UKSJV_Month_1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ID_004UKSJV_Month_12</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ID_008D2J6C_Month_1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15134</th>\n",
       "      <td>ID_ZZJKFBR5_Month_2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15135</th>\n",
       "      <td>ID_ZZMHNLNT_Month_4</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15136</th>\n",
       "      <td>ID_ZZQK0LU6_Month_3</td>\n",
       "      <td>7.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15137</th>\n",
       "      <td>ID_ZZUNZJ2U_Month_2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15138</th>\n",
       "      <td>ID_ZZVUJ45W_Month_3</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>15139 rows × 23 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                    User_ID  Downloaded_Competition_Datafile  \\\n",
       "0       ID_000H9XOP_Month_5                              0.0   \n",
       "1       ID_003YJHQA_Month_4                              0.0   \n",
       "2       ID_004UKSJV_Month_1                              0.0   \n",
       "3      ID_004UKSJV_Month_12                              0.0   \n",
       "4       ID_008D2J6C_Month_1                              0.0   \n",
       "...                     ...                              ...   \n",
       "15134   ID_ZZJKFBR5_Month_2                              0.0   \n",
       "15135   ID_ZZMHNLNT_Month_4                              0.0   \n",
       "15136   ID_ZZQK0LU6_Month_3                              7.0   \n",
       "15137   ID_ZZUNZJ2U_Month_2                              0.0   \n",
       "15138   ID_ZZVUJ45W_Month_3                              3.0   \n",
       "\n",
       "       Viewed_All_Discussions  Confirmed_Email  Joined_Competition  Signed_Up  \\\n",
       "0                         1.0              0.0                 0.0        1.0   \n",
       "1                         2.0              1.0                 0.0        1.0   \n",
       "2                         0.0              0.0                 0.0        0.0   \n",
       "3                         1.0              1.0                 0.0        1.0   \n",
       "4                         1.0              1.0                 0.0        1.0   \n",
       "...                       ...              ...                 ...        ...   \n",
       "15134                     0.0              0.0                 0.0        0.0   \n",
       "15135                     1.0              1.0                 0.0        1.0   \n",
       "15136                     4.0              1.0                 2.0        1.0   \n",
       "15137                     1.0              1.0                 0.0        1.0   \n",
       "15138                     2.0              1.0                 1.0        1.0   \n",
       "\n",
       "       Viewed_All_Competitions  Viewed_All_Learning_Pages  badge_OCZE  \\\n",
       "0                          1.0                        0.0         0.0   \n",
       "1                          1.0                        1.0         1.0   \n",
       "2                          0.0                        0.0         1.0   \n",
       "3                          1.0                        1.0         0.0   \n",
       "4                          1.0                        1.0         1.0   \n",
       "...                        ...                        ...         ...   \n",
       "15134                      0.0                        0.0         0.0   \n",
       "15135                      1.0                        1.0         1.0   \n",
       "15136                     14.0                        1.0         1.0   \n",
       "15137                      3.0                        3.0         1.0   \n",
       "15138                      1.0                        1.0         1.0   \n",
       "\n",
       "       create_alias  ...  job_ID  Updated_Discussion  Viewed_All_Jobs  \\\n",
       "0               1.0  ...     0.0                 0.0              0.0   \n",
       "1               1.0  ...     0.0                 0.0              0.0   \n",
       "2               0.0  ...     0.0                 0.0              0.0   \n",
       "3               1.0  ...     0.0                 0.0              0.0   \n",
       "4               1.0  ...     0.0                 0.0              0.0   \n",
       "...             ...  ...     ...                 ...              ...   \n",
       "15134           0.0  ...     0.0                 0.0              0.0   \n",
       "15135           1.0  ...     0.0                 0.0              0.0   \n",
       "15136           0.0  ...     0.0                 0.0              0.0   \n",
       "15137           1.0  ...     0.0                 0.0              0.0   \n",
       "15138           1.0  ...     0.0                 0.0              0.0   \n",
       "\n",
       "       Created_Submission  Signed_Out  Updated_Profile  blog_ID  \\\n",
       "0                     0.0         0.0              0.0      0.0   \n",
       "1                     0.0         0.0              2.0      0.0   \n",
       "2                     0.0         0.0              0.0      0.0   \n",
       "3                     0.0         0.0              4.0      0.0   \n",
       "4                     0.0         0.0              2.0      0.0   \n",
       "...                   ...         ...              ...      ...   \n",
       "15134                 0.0         0.0              0.0      0.0   \n",
       "15135                 0.0         0.0              0.0      0.0   \n",
       "15136                 0.0         0.0              0.0      0.0   \n",
       "15137                 0.0         0.0              0.0      0.0   \n",
       "15138                 0.0         0.0              0.0      0.0   \n",
       "\n",
       "       Viewed_Discussion  Updated_Submission  badge_HYIO  \n",
       "0                    0.0                 0.0         0.0  \n",
       "1                    0.0                 0.0         0.0  \n",
       "2                    0.0                 0.0         0.0  \n",
       "3                    0.0                 0.0         0.0  \n",
       "4                    0.0                 0.0         0.0  \n",
       "...                  ...                 ...         ...  \n",
       "15134                0.0                 0.0         0.0  \n",
       "15135                0.0                 0.0         0.0  \n",
       "15136                0.0                 0.0         0.0  \n",
       "15137                0.0                 0.0         0.0  \n",
       "15138                0.0                 0.0         0.0  \n",
       "\n",
       "[15139 rows x 23 columns]"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "08ce9dc8",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_userAct = pd.merge(df_userAct, dataset, how=\"left\", on=['User_ID'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "04b89deb",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_userAct.drop(\"Title\", inplace=True, axis=1)\n",
    "df_userAct.drop(\"main_activities\", inplace=True, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "dfed0338",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_userAct.rename(columns={'Theme': 'Blog Theme'},  inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "f265f9d2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([nan, True, False], dtype=object)"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_userAct[\"Remote\"].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "4a29388b",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_userAct[\"Remote\"] = df_userAct[\"Remote\"].replace(to_replace=True, value=\"1\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "06fb0ca7",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_userAct[\"Remote\"] = df_userAct[\"Remote\"].replace(to_replace=False, value=\"0\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "1a210f3f",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_userAct[\"Remote\"].fillna('-1', inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "f8792593",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0         -1\n",
       "1         -1\n",
       "2         -1\n",
       "3         -1\n",
       "4         -1\n",
       "          ..\n",
       "314600    -1\n",
       "314601    -1\n",
       "314602    -1\n",
       "314603    -1\n",
       "314604    -1\n",
       "Name: Remote, Length: 314605, dtype: object"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_userAct[\"Remote\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "035d6afd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['-1', '1', '0'], dtype=object)"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_userAct[\"Remote\"].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "0d640c34",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([nan, '2-5', '<1', '>5', '1-2'], dtype=object)"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_userAct[\"Experience\"].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "fe01c14a",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_userAct[\"Experience\"] = df_userAct[\"Experience\"].replace(to_replace=\"<1\", value=\"1\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "id": "47dcf06b",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_userAct[\"Experience\"] = df_userAct[\"Experience\"].replace(to_replace=\"1-2\", value=\"2\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "0a8aac0c",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_userAct[\"Experience\"] = df_userAct[\"Experience\"].replace(to_replace=\"2-5\", value=\"3\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "id": "b43046fb",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_userAct[\"Experience\"] = df_userAct[\"Experience\"].replace(to_replace=\">5\", value=\"4\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "id": "8c47748d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([nan, '3', '1', '4', '2'], dtype=object)"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_userAct[\"Experience\"].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "id": "dc32cb7b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([nan, '{\"Financial Services\",Banking}', '{Location}',\n",
       "       '{Government,Health,\"Financial Services\"}', '{Research}',\n",
       "       '{Ambassador}', '{Transportation}', '{Scholarship}', '{Logistics}',\n",
       "       '{Media}', '{Manufacturing,Sales}', '{\"Financial Services\"}',\n",
       "       '{Insurance}', '{\"Management Consulting\"}',\n",
       "       '{\"Artificial Intelligence\"}', '{Health}', '{Zindi}',\n",
       "       '{Technology}'], dtype=object)"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_userAct[\"Industry\"].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "id": "16aafa10",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_userAct[\"Industry_FS\"] = np.zeros((df_userAct.shape[0], 1))\n",
    "df_userAct[\"Industry_Bank\"] = np.zeros((df_userAct.shape[0], 1))\n",
    "df_userAct[\"Industry_Loc\"] = np.zeros((df_userAct.shape[0], 1))\n",
    "df_userAct[\"Industry_Gov\"] = np.zeros((df_userAct.shape[0], 1))\n",
    "df_userAct[\"Industry_Health\"] = np.zeros((df_userAct.shape[0], 1))\n",
    "df_userAct[\"Industry_Res\"] = np.zeros((df_userAct.shape[0], 1))\n",
    "df_userAct[\"Industry_Amb\"] = np.zeros((df_userAct.shape[0], 1))\n",
    "df_userAct[\"Industry_Transp\"] = np.zeros((df_userAct.shape[0], 1))\n",
    "df_userAct[\"Industry_Scholar\"] = np.zeros((df_userAct.shape[0], 1))\n",
    "df_userAct[\"Industry_Log\"] = np.zeros((df_userAct.shape[0], 1))\n",
    "df_userAct[\"Industry_Media\"] = np.zeros((df_userAct.shape[0], 1))\n",
    "df_userAct[\"Industry_Manuf\"] = np.zeros((df_userAct.shape[0], 1))\n",
    "df_userAct[\"Industry_Sales\"] = np.zeros((df_userAct.shape[0], 1))\n",
    "df_userAct[\"Industry_Insur\"] = np.zeros((df_userAct.shape[0], 1))\n",
    "df_userAct[\"Industry_Manage\"] = np.zeros((df_userAct.shape[0], 1))\n",
    "df_userAct[\"Industry_AI\"] = np.zeros((df_userAct.shape[0], 1))\n",
    "df_userAct[\"Industry_Zindi\"] = np.zeros((df_userAct.shape[0], 1))\n",
    "df_userAct[\"Industry_Tech\"] = np.zeros((df_userAct.shape[0], 1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "id": "768414b7",
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(len(df_userAct)):\n",
    "    if df_userAct.loc[i, \"Industry\"] == '{\"Financial Services\",Banking}':\n",
    "        df_userAct.loc[i, \"Industry_FS\"] = 1\n",
    "        df_userAct.loc[i,\"Industry_Bank\"] = 1\n",
    "        \n",
    "    elif df_userAct.loc[i,\"Industry\"] == '{Location}':\n",
    "        df_userAct.loc[i,\"Industry_Loc\"]= 1\n",
    "        \n",
    "    elif df_userAct.loc[i,\"Industry\"] == '{Government,Health,\"Financial Services\"}':\n",
    "        df_userAct.loc[i,\"Industry_Gov\"] = 1\n",
    "        df_userAct.loc[i,\"Industry_Health\"] = 1\n",
    "        df_userAct.loc[i,\"Industry_FS\"] = 1\n",
    "        \n",
    "    elif df_userAct.loc[i,\"Industry\"] == '{Research}':\n",
    "        df_userAct.loc[i,\"Industry_Res\"] = 1\n",
    "        \n",
    "    elif df_userAct.loc[i,\"Industry\"] == '{Ambassador}':\n",
    "        df_userAct.loc[i,\"Industry_Amb\"] = 1\n",
    "        \n",
    "    elif df_userAct.loc[i,\"Industry\"] == '{Transportation}':\n",
    "        df_userAct.loc[i,\"Industry_Transp\"] = 1\n",
    "        \n",
    "    elif df_userAct.loc[i,\"Industry\"] == '{Scholarship}':\n",
    "        df_userAct.loc[i,\"Industry_Scholar\"] = 1\n",
    "        \n",
    "    elif df_userAct.loc[i,\"Industry\"] == '{Logistics}': \n",
    "        df_userAct.loc[i,\"Industry_Log\"] = 1\n",
    "         \n",
    "    elif df_userAct.loc[i,\"Industry\"] == '{Media}':\n",
    "        df_userAct.loc[i,\"Industry_Media\"] = 1\n",
    "        \n",
    "    elif df_userAct.loc[i,\"Industry\"] == '{Manufacturing,Sales}':\n",
    "        df_userAct.loc[i,\"Industry_Manuf\"] = 1\n",
    "        df_userAct.loc[i,\"Industry_Sales\"] = 1\n",
    "        \n",
    "    elif df_userAct.loc[i,\"Industry\"] == '{\"Financial Services\"}':\n",
    "        df_userAct.loc[i,\"Industry_FS\"] = 1\n",
    "        \n",
    "    elif df_userAct.loc[i,\"Industry\"] == '{Insurance}':\n",
    "        df_userAct.loc[i,\"Industry_Insur\"] = 1\n",
    "        \n",
    "    elif df_userAct.loc[i,\"Industry\"] == '{\"Management Consulting\"}':\n",
    "        df_userAct.loc[i,\"Industry_Manage\"] = 1\n",
    "        \n",
    "    elif df_userAct.loc[i,\"Industry\"] == '{\"Artificial Intelligence\"}':\n",
    "        df_userAct.loc[i,\"Industry_AI\"] = 1\n",
    "        \n",
    "    elif df_userAct.loc[i,\"Industry\"] == '{Health}':\n",
    "        df_userAct.loc[i,\"Industry_Health\"] = 1\n",
    "        \n",
    "    elif df_userAct.loc[i,\"Industry\"] == '{Zindi}':\n",
    "        df_userAct.loc[i,\"Industry_Zindi\"] = 1\n",
    "        \n",
    "    elif df_userAct.loc[i,\"Industry\"] == '{Technology}':\n",
    "        df_userAct.loc[i,\"Industry_Tech\"] = 1\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "id": "2cc4c035",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_userAct.drop(\"Industry\", inplace =True, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "id": "24da3527",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([nan, 'fulltime', 'internship', 'contract', 'parttime'],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_userAct[\"Employment Type\"].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "id": "0b39abaf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-1 {color: black;background-color: white;}#sk-container-id-1 pre{padding: 0;}#sk-container-id-1 div.sk-toggleable {background-color: white;}#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-1 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-1 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-1 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-1 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-1 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-1 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-1 div.sk-item {position: relative;z-index: 1;}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-1 div.sk-item::before, #sk-container-id-1 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-1 div.sk-label-container {text-align: center;}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-1 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>LabelEncoder()</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" checked><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">LabelEncoder</label><div class=\"sk-toggleable__content\"><pre>LabelEncoder()</pre></div></div></div></div></div>"
      ],
      "text/plain": [
       "LabelEncoder()"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "le.fit(df_userAct[\"Employment Type\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "id": "12fde1d0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['contract', 'fulltime', 'internship', 'parttime', nan],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "le.classes_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "id": "8369c4e6",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_userAct[\"Employment Type\"] = le.transform(df_userAct[\"Employment Type\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "id": "9288bb87",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([4, 1, 2, 0, 3])"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_userAct[\"Employment Type\"].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "id": "5f59768c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([nan, '50-200', '1-10', '10-50', '>200'], dtype=object)"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_userAct[\"Company Size\"].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "id": "1ba1de1d",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_userAct[\"Company Size\"] = df_userAct[\"Company Size\"].replace(to_replace=\"1-10\", value=\"1\")\n",
    "df_userAct[\"Company Size\"] = df_userAct[\"Company Size\"].replace(to_replace=\"10-50\", value=\"2\")\n",
    "df_userAct[\"Company Size\"] = df_userAct[\"Company Size\"].replace(to_replace=\"50-200\", value=\"3\")\n",
    "df_userAct[\"Company Size\"] = df_userAct[\"Company Size\"].replace(to_replace=\">200\", value=\"4\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "id": "d56a532f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([nan, '3', '1', '2', '4'], dtype=object)"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_userAct[\"Company Size\"].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "id": "3e1c14f7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([nan,\n",
       "       '{\"Data engineering\",\"Data modelling\",\"Business/Data Analysis\",\"AI infrastructure\"}',\n",
       "       '{\"Data engineering\"}', '{\"Business/Data Analysis\"}',\n",
       "       '{\"Data modelling\",\"Business/Data Analysis\",\"AI infrastructure\"}',\n",
       "       '{\"ML Application Deployment\"}',\n",
       "       '{\"Business/Data Analysis\",\"Data modelling\",\"Data engineering\"}',\n",
       "       '{\"Data modelling\",\"Data engineering\"}',\n",
       "       '{\"Business/Data Analysis\",\"Data modelling\"}',\n",
       "       '{\"Data engineering\",\"Data modelling\",\"ML Application Deployment\",\"Business/Data Analysis\"}',\n",
       "       '{\"Data engineering\",\"AI infrastructure\",\"Business/Data Analysis\"}',\n",
       "       '{\"Data modelling\",\"Business/Data Analysis\"}',\n",
       "       '{\"Data modelling\"}',\n",
       "       '{\"Data engineering\",\"Data modelling\",\"Business/Data Analysis\"}',\n",
       "       '{\"Data engineering\",\"ML Application Deployment\",\"Business/Data Analysis\"}',\n",
       "       '{\"Data engineering\",\"Data modelling\"}',\n",
       "       '{\"Data modelling\",\"Business/Data Analysis\",\"Data engineering\",\"ML Application Deployment\"}',\n",
       "       '{\"Data engineering\",\"Data modelling\",\"ML Application Deployment\",\"AI infrastructure\"}',\n",
       "       '{\"Data engineering\",\"Data modelling\",\"AI infrastructure\",\"Business/Data Analysis\"}',\n",
       "       '{\"ML Application Deployment\",\"Business/Data Analysis\",\"Data modelling\"}',\n",
       "       '{\"Data engineering\",\"ML Application Deployment\",\"Data modelling\"}'],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_userAct[\"Data Science Functions\"].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "id": "7d1f481c",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_userAct[\"Function_DE\"] = np.zeros((df_userAct.shape[0], 1))\n",
    "df_userAct[\"Function_DM\"] = np.zeros((df_userAct.shape[0], 1))\n",
    "df_userAct[\"Function_Bus\"] = np.zeros((df_userAct.shape[0], 1))\n",
    "df_userAct[\"Function_AI\"] = np.zeros((df_userAct.shape[0], 1))\n",
    "df_userAct[\"Function_ML\"] = np.zeros((df_userAct.shape[0], 1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "id": "04459bcb",
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(len(df_userAct)):\n",
    "    if df_userAct.loc[i, \"Data Science Functions\"] == '{\"Data engineering\",\"Data modelling\",\"Business/Data Analysis\",\"AI infrastructure\"}':\n",
    "        df_userAct.loc[i, \"Function_DE\"] = 1\n",
    "        df_userAct.loc[i, \"Function_DM\" ] = 1\n",
    "        df_userAct.loc[i, \"Function_Bus\" ] = 1\n",
    "        df_userAct.loc[i, \"Function_AI\" ] = 1\n",
    "    elif df_userAct.loc[i, \"Data Science Functions\"] == '{\"Data engineering\"}':\n",
    "        df_userAct.loc[i, \"Function_DE\"] = 1\n",
    "    elif df_userAct.loc[i, \"Data Science Functions\"] == '{\"Business/Data Analysis\"}':\n",
    "        df_userAct.loc[i,\"Function_Bus\" ] = 1\n",
    "    elif df_userAct.loc[i, \"Data Science Functions\"] == '{\"Data modelling\",\"Business/Data Analysis\",\"AI infrastructure\"}':\n",
    "        df_userAct.loc[i,\"Function_AI\" ] = 1\n",
    "        df_userAct.loc[i,\"Function_Bus\" ] = 1\n",
    "        df_userAct.loc[i,\"Function_DM\" ] = 1\n",
    "    elif df_userAct.loc[i, \"Data Science Functions\"] == '{\"ML Application Deployment\"}':\n",
    "        df_userAct.loc[i,\"Function_ML\" ] = 1\n",
    "    elif df_userAct.loc[i, \"Data Science Functions\"] == '{\"Business/Data Analysis\",\"Data modelling\",\"Data engineering\"}':\n",
    "        df_userAct.loc[i,\"Function_Bus\" ] = 1\n",
    "        df_userAct.loc[i,\"Function_DM\" ] = 1\n",
    "        df_userAct.loc[i, \"Function_DE\"] = 1\n",
    "    elif df_userAct.loc[i, \"Data Science Functions\"] == '{\"Data modelling\",\"Data engineering\"}':\n",
    "        df_userAct.loc[i,\"Function_DM\" ] = 1\n",
    "        df_userAct.loc[i, \"Function_DE\"] = 1\n",
    "    elif df_userAct.loc[i, \"Data Science Functions\"] == '{\"Business/Data Analysis\",\"Data modelling\"}':\n",
    "        df_userAct.loc[i,\"Function_Bus\" ] = 1\n",
    "        df_userAct.loc[i,\"Function_DM\" ] = 1\n",
    "    elif df_userAct.loc[i, \"Data Science Functions\"] == '{\"Data engineering\",\"Data modelling\",\"ML Application Deployment\",\"Business/Data Analysis\"}':\n",
    "        df_userAct.loc[i,\"Function_ML\" ] = 1\n",
    "        df_userAct.loc[i,\"Function_Bus\" ] = 1\n",
    "        df_userAct.loc[i,\"Function_DM\" ] = 1\n",
    "        df_userAct.loc[i, \"Function_DE\"] = 1\n",
    "    elif df_userAct.loc[i, \"Data Science Functions\"] == '{\"Data engineering\",\"AI infrastructure\",\"Business/Data Analysis\"}':\n",
    "        df_userAct.loc[i,\"Function_AI\" ] = 1\n",
    "        df_userAct.loc[i,\"Function_Bus\" ] = 1\n",
    "        df_userAct.loc[i, \"Function_DE\"] = 1\n",
    "    elif df_userAct.loc[i, \"Data Science Functions\"] == '{\"Data modelling\",\"Business/Data Analysis\"}':\n",
    "        df_userAct.loc[i,\"Function_Bus\" ] = 1\n",
    "        df_userAct.loc[i,\"Function_DM\" ] = 1\n",
    "    elif df_userAct.loc[i, \"Data Science Functions\"] == '{\"Data modelling\"}':\n",
    "        df_userAct.loc[i,\"Function_DM\" ] = 1\n",
    "    elif df_userAct.loc[i, \"Data Science Functions\"] == '{\"Data engineering\",\"Data modelling\",\"Business/Data Analysis\"}':\n",
    "        df_userAct.loc[i,\"Function_Bus\" ] = 1\n",
    "        df_userAct.loc[i,\"Function_DM\" ] = 1\n",
    "        df_userAct.loc[i, \"Function_DE\"] = 1\n",
    "    elif df_userAct.loc[i, \"Data Science Functions\"] == '{\"Data engineering\",\"ML Application Deployment\",\"Business/Data Analysis\"}':\n",
    "        df_userAct.loc[i,\"Function_ML\" ] = 1\n",
    "        df_userAct.loc[i,\"Function_Bus\" ] = 1\n",
    "        df_userAct.loc[i, \"Function_DE\"] = 1\n",
    "    elif df_userAct.loc[i, \"Data Science Functions\"] ==  '{\"Data engineering\",\"Data modelling\",\"ML Application Deployment\",\"AI infrastructure\"}':\n",
    "        df_userAct.loc[i,\"Function_ML\" ] = 1\n",
    "        df_userAct.loc[i,\"Function_AI\" ] = 1\n",
    "        df_userAct.loc[i,\"Function_DM\" ] = 1\n",
    "        df_userAct.loc[i, \"Function_DE\"] = 1\n",
    "    elif df_userAct.loc[i, \"Data Science Functions\"] == '{\"Data engineering\",\"Data modelling\",\"AI infrastructure\",\"Business/Data Analysis\"}':\n",
    "        df_userAct.loc[i,\"Function_AI\" ] = 1\n",
    "        df_userAct.loc[i,\"Function_Bus\" ] = 1\n",
    "        df_userAct.loc[i,\"Function_DM\" ] = 1\n",
    "        df_userAct.loc[i, \"Function_DE\"] = 1\n",
    "    elif df_userAct.loc[i, \"Data Science Functions\"] == '{\"ML Application Deployment\",\"Business/Data Analysis\",\"Data modelling\"}':\n",
    "        df_userAct.loc[i,\"Function_ML\" ] = 1\n",
    "        df_userAct.loc[i,\"Function_Bus\" ] = 1\n",
    "        df_userAct.loc[i,\"Function_DM\" ] = 1\n",
    "    elif df_userAct.loc[i, \"Data Science Functions\"] == '{\"Data engineering\",\"ML Application Deployment\",\"Data modelling\"}':\n",
    "        df_userAct.loc[i,\"Function_ML\" ] = 1\n",
    "        df_userAct.loc[i,\"Function_DM\" ] = 1\n",
    "        df_userAct.loc[i, \"Function_DE\"] = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "id": "f5212329",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0., 1.])"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_userAct[\"Function_DE\"].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "id": "b13aaf50",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_userAct.drop(\"Data Science Functions\", inplace=True, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "id": "80c4a231",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_userAct[\"Blog Published Hour\"] = pd.to_numeric(df_userAct[\"Blog Published Hour\"])\n",
    "# df_userAct[\"Blog Published Minute\"] = pd.to_numeric(df_userAct[\"Blog Published Minute\"])\n",
    "# df_userAct[\"Blog Published Second\"] = pd.to_numeric(df_userAct[\"Blog Published Second\"])\n",
    "df_userAct[\"Remote\"] = pd.to_numeric(df_userAct[\"Remote\"])\n",
    "df_userAct[\"Experience\"] = pd.to_numeric(df_userAct[\"Experience\"])\n",
    "df_userAct[\"Company Size\"] = pd.to_numeric(df_userAct[\"Company Size\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "id": "8f933f20",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 314605 entries, 0 to 314604\n",
      "Data columns (total 55 columns):\n",
      " #   Column                           Non-Null Count   Dtype  \n",
      "---  ------                           --------------   -----  \n",
      " 0   User_ID                          314605 non-null  object \n",
      " 1   datetime Day_of_month            314605 non-null  int64  \n",
      " 2   Blog Theme                       2709 non-null    float64\n",
      " 3   Blog Published Year              2712 non-null    float64\n",
      " 4   Blog Published Month             2712 non-null    float64\n",
      " 5   Published At Day_of_month        2712 non-null    float64\n",
      " 6   Remote                           314605 non-null  int64  \n",
      " 7   Experience                       1492 non-null    float64\n",
      " 8   Employment Type                  314605 non-null  int32  \n",
      " 9   Company Size                     1463 non-null    float64\n",
      " 10  Downloaded_Competition_Datafile  314605 non-null  float64\n",
      " 11  Viewed_All_Discussions           314605 non-null  float64\n",
      " 12  Confirmed_Email                  314605 non-null  float64\n",
      " 13  Joined_Competition               314605 non-null  float64\n",
      " 14  Signed_Up                        314605 non-null  float64\n",
      " 15  Viewed_All_Competitions          314605 non-null  float64\n",
      " 16  Viewed_All_Learning_Pages        314605 non-null  float64\n",
      " 17  badge_OCZE                       314605 non-null  float64\n",
      " 18  create_alias                     314605 non-null  float64\n",
      " 19  identify                         314605 non-null  float64\n",
      " 20  comp_ID                          314605 non-null  float64\n",
      " 21  Signed_In                        314605 non-null  float64\n",
      " 22  job_ID                           314605 non-null  float64\n",
      " 23  Updated_Discussion               314605 non-null  float64\n",
      " 24  Viewed_All_Jobs                  314605 non-null  float64\n",
      " 25  Created_Submission               314605 non-null  float64\n",
      " 26  Signed_Out                       314605 non-null  float64\n",
      " 27  Updated_Profile                  314605 non-null  float64\n",
      " 28  blog_ID                          314605 non-null  float64\n",
      " 29  Viewed_Discussion                314605 non-null  float64\n",
      " 30  Updated_Submission               314605 non-null  float64\n",
      " 31  badge_HYIO                       314605 non-null  float64\n",
      " 32  Industry_FS                      314605 non-null  float64\n",
      " 33  Industry_Bank                    314605 non-null  float64\n",
      " 34  Industry_Loc                     314605 non-null  float64\n",
      " 35  Industry_Gov                     314605 non-null  float64\n",
      " 36  Industry_Health                  314605 non-null  float64\n",
      " 37  Industry_Res                     314605 non-null  float64\n",
      " 38  Industry_Amb                     314605 non-null  float64\n",
      " 39  Industry_Transp                  314605 non-null  float64\n",
      " 40  Industry_Scholar                 314605 non-null  float64\n",
      " 41  Industry_Log                     314605 non-null  float64\n",
      " 42  Industry_Media                   314605 non-null  float64\n",
      " 43  Industry_Manuf                   314605 non-null  float64\n",
      " 44  Industry_Sales                   314605 non-null  float64\n",
      " 45  Industry_Insur                   314605 non-null  float64\n",
      " 46  Industry_Manage                  314605 non-null  float64\n",
      " 47  Industry_AI                      314605 non-null  float64\n",
      " 48  Industry_Zindi                   314605 non-null  float64\n",
      " 49  Industry_Tech                    314605 non-null  float64\n",
      " 50  Function_DE                      314605 non-null  float64\n",
      " 51  Function_DM                      314605 non-null  float64\n",
      " 52  Function_Bus                     314605 non-null  float64\n",
      " 53  Function_AI                      314605 non-null  float64\n",
      " 54  Function_ML                      314605 non-null  float64\n",
      "dtypes: float64(51), int32(1), int64(2), object(1)\n",
      "memory usage: 141.3+ MB\n"
     ]
    }
   ],
   "source": [
    "df_userAct.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "id": "ee896afa",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_temp1 = df_userAct.groupby('User_ID').max() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "id": "f89f49fd",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_temp2 = df_userAct.groupby('User_ID').mean() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "id": "78b0c80e",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_temp3 = df_userAct.groupby('User_ID').count() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "id": "3889682c",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_temp4 = df_userAct.groupby('User_ID').min() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "id": "b7596df2",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_temp = df_temp2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "id": "458de006",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Last day active\n",
    "df_temp[\"datetime Day_of_month\"] = df_temp1[\"datetime Day_of_month\"]\n",
    "df_temp[\"Published At Day_of_month\"] = df_temp1[\"Published At Day_of_month\"]\n",
    "df_temp[\"Blog Published Year\"] = df_temp1[\"Blog Published Year\"]\n",
    "df_temp[\"Blog Published Month\"] = df_temp1[\"Blog Published Month\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "id": "2f036e73",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_temp[\"count_UA\"] = df_temp3[\"datetime Day_of_month\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "id": "ed925960",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_temp[\"Day duration\"] = df_temp1[\"datetime Day_of_month\"] - df_temp4[\"datetime Day_of_month\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6db2e425",
   "metadata": {},
   "source": [
    "Will add 1 to \"day duration\" after merging."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "id": "3e1a9f4b",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_userAct = df_temp "
   ]
  },
  {
   "cell_type": "raw",
   "id": "de727517",
   "metadata": {},
   "source": [
    "df_userAct.sort_values('User_ID')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "id": "b14e9f1b",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_userAct = df_userAct.reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "id": "3d647748",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>User_ID</th>\n",
       "      <th>datetime Day_of_month</th>\n",
       "      <th>Blog Theme</th>\n",
       "      <th>Blog Published Year</th>\n",
       "      <th>Blog Published Month</th>\n",
       "      <th>Published At Day_of_month</th>\n",
       "      <th>Remote</th>\n",
       "      <th>Experience</th>\n",
       "      <th>Employment Type</th>\n",
       "      <th>Company Size</th>\n",
       "      <th>...</th>\n",
       "      <th>Industry_AI</th>\n",
       "      <th>Industry_Zindi</th>\n",
       "      <th>Industry_Tech</th>\n",
       "      <th>Function_DE</th>\n",
       "      <th>Function_DM</th>\n",
       "      <th>Function_Bus</th>\n",
       "      <th>Function_AI</th>\n",
       "      <th>Function_ML</th>\n",
       "      <th>count_UA</th>\n",
       "      <th>Day duration</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ID_000H9XOP_Month_5</td>\n",
       "      <td>26</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ID_003YJHQA_Month_4</td>\n",
       "      <td>12</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ID_004UKSJV_Month_1</td>\n",
       "      <td>15</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ID_004UKSJV_Month_12</td>\n",
       "      <td>29</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ID_008D2J6C_Month_1</td>\n",
       "      <td>15</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15134</th>\n",
       "      <td>ID_ZZJKFBR5_Month_2</td>\n",
       "      <td>20</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15135</th>\n",
       "      <td>ID_ZZMHNLNT_Month_4</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15136</th>\n",
       "      <td>ID_ZZQK0LU6_Month_3</td>\n",
       "      <td>30</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>67</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15137</th>\n",
       "      <td>ID_ZZUNZJ2U_Month_2</td>\n",
       "      <td>30</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>12</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15138</th>\n",
       "      <td>ID_ZZVUJ45W_Month_3</td>\n",
       "      <td>5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>13</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>15139 rows × 57 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                    User_ID  datetime Day_of_month  Blog Theme  \\\n",
       "0       ID_000H9XOP_Month_5                     26         NaN   \n",
       "1       ID_003YJHQA_Month_4                     12         NaN   \n",
       "2       ID_004UKSJV_Month_1                     15         NaN   \n",
       "3      ID_004UKSJV_Month_12                     29         NaN   \n",
       "4       ID_008D2J6C_Month_1                     15         NaN   \n",
       "...                     ...                    ...         ...   \n",
       "15134   ID_ZZJKFBR5_Month_2                     20         NaN   \n",
       "15135   ID_ZZMHNLNT_Month_4                      1         NaN   \n",
       "15136   ID_ZZQK0LU6_Month_3                     30         NaN   \n",
       "15137   ID_ZZUNZJ2U_Month_2                     30         NaN   \n",
       "15138   ID_ZZVUJ45W_Month_3                      5         NaN   \n",
       "\n",
       "       Blog Published Year  Blog Published Month  Published At Day_of_month  \\\n",
       "0                      NaN                   NaN                        NaN   \n",
       "1                      NaN                   NaN                        NaN   \n",
       "2                      NaN                   NaN                        NaN   \n",
       "3                      NaN                   NaN                        NaN   \n",
       "4                      NaN                   NaN                        NaN   \n",
       "...                    ...                   ...                        ...   \n",
       "15134                  NaN                   NaN                        NaN   \n",
       "15135                  NaN                   NaN                        NaN   \n",
       "15136                  NaN                   NaN                        NaN   \n",
       "15137                  NaN                   NaN                        NaN   \n",
       "15138                  NaN                   NaN                        NaN   \n",
       "\n",
       "       Remote  Experience  Employment Type  Company Size  ...  Industry_AI  \\\n",
       "0        -1.0         NaN              4.0           NaN  ...          0.0   \n",
       "1        -1.0         NaN              4.0           NaN  ...          0.0   \n",
       "2        -1.0         NaN              4.0           NaN  ...          0.0   \n",
       "3        -1.0         NaN              4.0           NaN  ...          0.0   \n",
       "4        -1.0         NaN              4.0           NaN  ...          0.0   \n",
       "...       ...         ...              ...           ...  ...          ...   \n",
       "15134    -1.0         NaN              4.0           NaN  ...          0.0   \n",
       "15135    -1.0         NaN              4.0           NaN  ...          0.0   \n",
       "15136    -1.0         NaN              4.0           NaN  ...          0.0   \n",
       "15137    -1.0         NaN              4.0           NaN  ...          0.0   \n",
       "15138    -1.0         NaN              4.0           NaN  ...          0.0   \n",
       "\n",
       "       Industry_Zindi  Industry_Tech  Function_DE  Function_DM  Function_Bus  \\\n",
       "0                 0.0            0.0          0.0          0.0           0.0   \n",
       "1                 0.0            0.0          0.0          0.0           0.0   \n",
       "2                 0.0            0.0          0.0          0.0           0.0   \n",
       "3                 0.0            0.0          0.0          0.0           0.0   \n",
       "4                 0.0            0.0          0.0          0.0           0.0   \n",
       "...               ...            ...          ...          ...           ...   \n",
       "15134             0.0            0.0          0.0          0.0           0.0   \n",
       "15135             0.0            0.0          0.0          0.0           0.0   \n",
       "15136             0.0            0.0          0.0          0.0           0.0   \n",
       "15137             0.0            0.0          0.0          0.0           0.0   \n",
       "15138             0.0            0.0          0.0          0.0           0.0   \n",
       "\n",
       "       Function_AI  Function_ML  count_UA  Day duration  \n",
       "0              0.0          0.0         6             4  \n",
       "1              0.0          0.0        11             0  \n",
       "2              0.0          0.0         1             0  \n",
       "3              0.0          0.0        11             1  \n",
       "4              0.0          0.0        10             8  \n",
       "...            ...          ...       ...           ...  \n",
       "15134          0.0          0.0         2             0  \n",
       "15135          0.0          0.0         8             0  \n",
       "15136          0.0          0.0        67             1  \n",
       "15137          0.0          0.0        12             0  \n",
       "15138          0.0          0.0        13             0  \n",
       "\n",
       "[15139 rows x 57 columns]"
      ]
     },
     "execution_count": 107,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_userAct"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0eb20903",
   "metadata": {},
   "source": [
    "# DISCUSSIONS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "id": "f291591c",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_disc = pd.read_csv(\"Discussion.csv\").drop_duplicates()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "id": "1a3bebda",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 1439 entries, 0 to 1438\n",
      "Data columns (total 9 columns):\n",
      " #   Column                   Non-Null Count  Dtype  \n",
      "---  ------                   --------------  -----  \n",
      " 0   Competition ID           1439 non-null   object \n",
      " 1   Personal                 1439 non-null   int64  \n",
      " 2   Theme                    159 non-null    float64\n",
      " 3   Disc_ID                  1439 non-null   object \n",
      " 4   User_ID                  1439 non-null   object \n",
      " 5   Created At time          1439 non-null   object \n",
      " 6   Created At Year          1439 non-null   int64  \n",
      " 7   Created At Month         1439 non-null   int64  \n",
      " 8   Created At Day_of_month  1439 non-null   int64  \n",
      "dtypes: float64(1), int64(4), object(4)\n",
      "memory usage: 112.4+ KB\n"
     ]
    }
   ],
   "source": [
    "df_disc.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "id": "1bd34152",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Competition ID</th>\n",
       "      <th>Personal</th>\n",
       "      <th>Theme</th>\n",
       "      <th>Disc_ID</th>\n",
       "      <th>User_ID</th>\n",
       "      <th>Created At time</th>\n",
       "      <th>Created At Year</th>\n",
       "      <th>Created At Month</th>\n",
       "      <th>Created At Day_of_month</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>GeneralDiscussion</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_0BK7TH</td>\n",
       "      <td>ID_AAGFVLHH</td>\n",
       "      <td>19:48:26.862175</td>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "      <td>27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>GeneralDiscussion</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_ZFJ9J7</td>\n",
       "      <td>ID_0AJGQILU</td>\n",
       "      <td>09:37:52.039526</td>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>GeneralDiscussion</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_ME45XX</td>\n",
       "      <td>ID_U90KK6KL</td>\n",
       "      <td>17:50:33.894800</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ID_45W9</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_2BG9IX</td>\n",
       "      <td>ID_U90KK6KL</td>\n",
       "      <td>07:57:55.178845</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>GeneralDiscussion</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_251XVV</td>\n",
       "      <td>ID_F7HIGFHQ</td>\n",
       "      <td>19:56:13.041893</td>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "      <td>27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1434</th>\n",
       "      <td>GeneralDiscussion</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_09W0TZ</td>\n",
       "      <td>ID_AAJKI0P3</td>\n",
       "      <td>14:15:31.081484</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1435</th>\n",
       "      <td>GeneralDiscussion</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_613AUJ</td>\n",
       "      <td>ID_8A3X5IWU</td>\n",
       "      <td>11:06:11.201934</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1436</th>\n",
       "      <td>GeneralDiscussion</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_TCN8TB</td>\n",
       "      <td>ID_3NL0QLE6</td>\n",
       "      <td>04:57:31.578153</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1437</th>\n",
       "      <td>GeneralDiscussion</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_FTJ37Z</td>\n",
       "      <td>ID_3PL1LQQX</td>\n",
       "      <td>05:11:59.313759</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1438</th>\n",
       "      <td>GeneralDiscussion</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>ID_ZOC5GY</td>\n",
       "      <td>ID_S4XIIL0Z</td>\n",
       "      <td>07:13:04.296241</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1439 rows × 9 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         Competition ID  Personal  Theme    Disc_ID      User_ID  \\\n",
       "0     GeneralDiscussion         1    NaN  ID_0BK7TH  ID_AAGFVLHH   \n",
       "1     GeneralDiscussion         1    NaN  ID_ZFJ9J7  ID_0AJGQILU   \n",
       "2     GeneralDiscussion         1    NaN  ID_ME45XX  ID_U90KK6KL   \n",
       "3               ID_45W9         1    NaN  ID_2BG9IX  ID_U90KK6KL   \n",
       "4     GeneralDiscussion         1    NaN  ID_251XVV  ID_F7HIGFHQ   \n",
       "...                 ...       ...    ...        ...          ...   \n",
       "1434  GeneralDiscussion         1    NaN  ID_09W0TZ  ID_AAJKI0P3   \n",
       "1435  GeneralDiscussion         1    NaN  ID_613AUJ  ID_8A3X5IWU   \n",
       "1436  GeneralDiscussion         1    NaN  ID_TCN8TB  ID_3NL0QLE6   \n",
       "1437  GeneralDiscussion         1    NaN  ID_FTJ37Z  ID_3PL1LQQX   \n",
       "1438  GeneralDiscussion         0    1.0  ID_ZOC5GY  ID_S4XIIL0Z   \n",
       "\n",
       "      Created At time  Created At Year  Created At Month  \\\n",
       "0     19:48:26.862175                1                12   \n",
       "1     09:37:52.039526                1                12   \n",
       "2     17:50:33.894800                1                 2   \n",
       "3     07:57:55.178845                1                 2   \n",
       "4     19:56:13.041893                1                12   \n",
       "...               ...              ...               ...   \n",
       "1434  14:15:31.081484                1                 5   \n",
       "1435  11:06:11.201934                1                 5   \n",
       "1436  04:57:31.578153                1                 5   \n",
       "1437  05:11:59.313759                1                 5   \n",
       "1438  07:13:04.296241                1                 3   \n",
       "\n",
       "      Created At Day_of_month  \n",
       "0                          27  \n",
       "1                          20  \n",
       "2                           9  \n",
       "3                          10  \n",
       "4                          27  \n",
       "...                       ...  \n",
       "1434                       23  \n",
       "1435                       14  \n",
       "1436                       12  \n",
       "1437                        9  \n",
       "1438                       19  \n",
       "\n",
       "[1439 rows x 9 columns]"
      ]
     },
     "execution_count": 110,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_disc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "id": "db27b848",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1017"
      ]
     },
     "execution_count": 111,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(pd.unique(df_disc['User_ID']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "id": "dfd7452c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 112,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(pd.unique(df_disc['Created At Year']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "id": "9c1b99f1",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_disc.drop(\"Created At Year\", inplace=True, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "id": "301938a4",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_disc[\"User_ID\"] = df_disc[\"User_ID\"].astype(\"string\")\n",
    "df_disc[\"Created At Month\"] = df_disc[\"Created At Month\"].astype(\"string\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "id": "82a2059d",
   "metadata": {},
   "outputs": [],
   "source": [
    "new = df_disc[\"Created At Month\"].copy()\n",
    "df_disc[\"User_ID\"] = df_disc[\"User_ID\"].str.cat(new, sep =\"_Month_\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "id": "c9c8e0e0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([nan,  3.,  1.,  4.,  5.,  2.])"
      ]
     },
     "execution_count": 116,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.unique(df_disc['Theme'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "id": "8096303d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_disc[\"Created At time\"]= df_disc[\"Created At time\"].astype(\"string\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "id": "86f1b6fb",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_disc['Discussion Hour'] = df_disc['Created At time'].str[0:2]\n",
    "# df_disc['Discussion Minute'] = df_disc['Created At time'].str[3:5]\n",
    "# df_disc['Discussion Second'] = df_disc['Created At time'].str[6:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "id": "4de51266",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_disc['Discussion Hour'] = pd.to_numeric(df_disc['Discussion Hour'])\n",
    "# df_disc['Discussion Minute'] = pd.to_numeric(df_disc['Discussion Minute'])\n",
    "# df_disc['Discussion Second'] = pd.to_numeric(df_disc['Discussion Second'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "id": "1e419882",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_disc.drop('Created At time',  inplace = True, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "id": "c689da5d",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_disc.rename(columns={'Created At Month': 'Discussion_month'},  inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "id": "1be8674a",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_disc.rename(columns={'Created At Day_of_month': 'Discussion_day'},  inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "id": "fb9c1994",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Competition ID</th>\n",
       "      <th>Personal</th>\n",
       "      <th>Theme</th>\n",
       "      <th>Disc_ID</th>\n",
       "      <th>User_ID</th>\n",
       "      <th>Discussion_month</th>\n",
       "      <th>Discussion_day</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>GeneralDiscussion</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_0BK7TH</td>\n",
       "      <td>ID_AAGFVLHH_Month_12</td>\n",
       "      <td>12</td>\n",
       "      <td>27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>GeneralDiscussion</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_ZFJ9J7</td>\n",
       "      <td>ID_0AJGQILU_Month_12</td>\n",
       "      <td>12</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>GeneralDiscussion</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_ME45XX</td>\n",
       "      <td>ID_U90KK6KL_Month_2</td>\n",
       "      <td>2</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ID_45W9</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_2BG9IX</td>\n",
       "      <td>ID_U90KK6KL_Month_2</td>\n",
       "      <td>2</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>GeneralDiscussion</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_251XVV</td>\n",
       "      <td>ID_F7HIGFHQ_Month_12</td>\n",
       "      <td>12</td>\n",
       "      <td>27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1434</th>\n",
       "      <td>GeneralDiscussion</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_09W0TZ</td>\n",
       "      <td>ID_AAJKI0P3_Month_5</td>\n",
       "      <td>5</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1435</th>\n",
       "      <td>GeneralDiscussion</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_613AUJ</td>\n",
       "      <td>ID_8A3X5IWU_Month_5</td>\n",
       "      <td>5</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1436</th>\n",
       "      <td>GeneralDiscussion</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_TCN8TB</td>\n",
       "      <td>ID_3NL0QLE6_Month_5</td>\n",
       "      <td>5</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1437</th>\n",
       "      <td>GeneralDiscussion</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_FTJ37Z</td>\n",
       "      <td>ID_3PL1LQQX_Month_5</td>\n",
       "      <td>5</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1438</th>\n",
       "      <td>GeneralDiscussion</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>ID_ZOC5GY</td>\n",
       "      <td>ID_S4XIIL0Z_Month_3</td>\n",
       "      <td>3</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1439 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         Competition ID  Personal  Theme    Disc_ID               User_ID  \\\n",
       "0     GeneralDiscussion         1    NaN  ID_0BK7TH  ID_AAGFVLHH_Month_12   \n",
       "1     GeneralDiscussion         1    NaN  ID_ZFJ9J7  ID_0AJGQILU_Month_12   \n",
       "2     GeneralDiscussion         1    NaN  ID_ME45XX   ID_U90KK6KL_Month_2   \n",
       "3               ID_45W9         1    NaN  ID_2BG9IX   ID_U90KK6KL_Month_2   \n",
       "4     GeneralDiscussion         1    NaN  ID_251XVV  ID_F7HIGFHQ_Month_12   \n",
       "...                 ...       ...    ...        ...                   ...   \n",
       "1434  GeneralDiscussion         1    NaN  ID_09W0TZ   ID_AAJKI0P3_Month_5   \n",
       "1435  GeneralDiscussion         1    NaN  ID_613AUJ   ID_8A3X5IWU_Month_5   \n",
       "1436  GeneralDiscussion         1    NaN  ID_TCN8TB   ID_3NL0QLE6_Month_5   \n",
       "1437  GeneralDiscussion         1    NaN  ID_FTJ37Z   ID_3PL1LQQX_Month_5   \n",
       "1438  GeneralDiscussion         0    1.0  ID_ZOC5GY   ID_S4XIIL0Z_Month_3   \n",
       "\n",
       "     Discussion_month  Discussion_day  \n",
       "0                  12              27  \n",
       "1                  12              20  \n",
       "2                   2               9  \n",
       "3                   2              10  \n",
       "4                  12              27  \n",
       "...               ...             ...  \n",
       "1434                5              23  \n",
       "1435                5              14  \n",
       "1436                5              12  \n",
       "1437                5               9  \n",
       "1438                3              19  \n",
       "\n",
       "[1439 rows x 7 columns]"
      ]
     },
     "execution_count": 123,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_disc"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4df67006",
   "metadata": {},
   "source": [
    "# COMMENTS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "id": "3760bc57",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_com = pd.read_csv(\"Comments.csv\").drop_duplicates()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "id": "930c46a1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 467 entries, 0 to 466\n",
      "Data columns (total 6 columns):\n",
      " #   Column                   Non-Null Count  Dtype \n",
      "---  ------                   --------------  ----- \n",
      " 0   Disc_ID                  467 non-null    object\n",
      " 1   User_ID                  467 non-null    object\n",
      " 2   Created At time          467 non-null    object\n",
      " 3   Created At Year          467 non-null    int64 \n",
      " 4   Created At Month         467 non-null    int64 \n",
      " 5   Created At Day_of_month  467 non-null    int64 \n",
      "dtypes: int64(3), object(3)\n",
      "memory usage: 25.5+ KB\n"
     ]
    }
   ],
   "source": [
    "df_com.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "id": "af159fb1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Disc_ID</th>\n",
       "      <th>User_ID</th>\n",
       "      <th>Created At time</th>\n",
       "      <th>Created At Year</th>\n",
       "      <th>Created At Month</th>\n",
       "      <th>Created At Day_of_month</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ID_ONQEAR</td>\n",
       "      <td>ID_CEDNKSVV</td>\n",
       "      <td>16:18:44.841697</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ID_E0NHSS</td>\n",
       "      <td>ID_CEDNKSVV</td>\n",
       "      <td>17:30:40.566763</td>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ID_3XKIOF</td>\n",
       "      <td>ID_CEDNKSVV</td>\n",
       "      <td>13:28:47.950841</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ID_XCC6W4</td>\n",
       "      <td>ID_CEDNKSVV</td>\n",
       "      <td>18:42:17.362333</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ID_OIGXGK</td>\n",
       "      <td>ID_CEDNKSVV</td>\n",
       "      <td>18:57:54.540062</td>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>462</th>\n",
       "      <td>ID_DK2VV3</td>\n",
       "      <td>ID_GPXBD1BG</td>\n",
       "      <td>06:11:25.314720</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>463</th>\n",
       "      <td>ID_5MJRT0</td>\n",
       "      <td>ID_2F218EUV</td>\n",
       "      <td>21:53:36.148890</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>464</th>\n",
       "      <td>ID_RMQMJA</td>\n",
       "      <td>ID_9W2RXCMI</td>\n",
       "      <td>14:06:35.178439</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>465</th>\n",
       "      <td>ID_V5431U</td>\n",
       "      <td>ID_RQEQ74SK</td>\n",
       "      <td>13:03:58.602412</td>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "      <td>29</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>466</th>\n",
       "      <td>ID_V5431U</td>\n",
       "      <td>ID_7ICXZH0P</td>\n",
       "      <td>13:04:38.747046</td>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "      <td>29</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>467 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       Disc_ID      User_ID  Created At time  Created At Year  \\\n",
       "0    ID_ONQEAR  ID_CEDNKSVV  16:18:44.841697                1   \n",
       "1    ID_E0NHSS  ID_CEDNKSVV  17:30:40.566763                1   \n",
       "2    ID_3XKIOF  ID_CEDNKSVV  13:28:47.950841                1   \n",
       "3    ID_XCC6W4  ID_CEDNKSVV  18:42:17.362333                1   \n",
       "4    ID_OIGXGK  ID_CEDNKSVV  18:57:54.540062                1   \n",
       "..         ...          ...              ...              ...   \n",
       "462  ID_DK2VV3  ID_GPXBD1BG  06:11:25.314720                1   \n",
       "463  ID_5MJRT0  ID_2F218EUV  21:53:36.148890                1   \n",
       "464  ID_RMQMJA  ID_9W2RXCMI  14:06:35.178439                1   \n",
       "465  ID_V5431U  ID_RQEQ74SK  13:03:58.602412                1   \n",
       "466  ID_V5431U  ID_7ICXZH0P  13:04:38.747046                1   \n",
       "\n",
       "     Created At Month  Created At Day_of_month  \n",
       "0                   5                       11  \n",
       "1                  12                       23  \n",
       "2                   1                        3  \n",
       "3                   1                        5  \n",
       "4                  12                       19  \n",
       "..                ...                      ...  \n",
       "462                 1                       14  \n",
       "463                 1                       15  \n",
       "464                 1                       23  \n",
       "465                12                       29  \n",
       "466                12                       29  \n",
       "\n",
       "[467 rows x 6 columns]"
      ]
     },
     "execution_count": 126,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_com"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "id": "fc4d3538",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "192"
      ]
     },
     "execution_count": 127,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(pd.unique(df_com['User_ID']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "id": "83317ab3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 128,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(pd.unique(df_com['Created At Year']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "id": "d29bf202",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_com.drop('Created At Year', inplace=True, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "id": "9d065f8f",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_com[\"User_ID\"] = df_com[\"User_ID\"].astype(\"string\")\n",
    "df_com[\"Created At Month\"] = df_com[\"Created At Month\"].astype(\"string\")\n",
    "# df_com[\"Created At time\"] = df_com[\"Created At time\"].astype(\"string\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "id": "32c775b5",
   "metadata": {},
   "outputs": [],
   "source": [
    "new = df_com[\"Created At Month\"].copy()\n",
    "df_com[\"User_ID\"] = df_com[\"User_ID\"].str.cat(new, sep =\"_Month_\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "id": "485d1c24",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_com.drop('Created At Month', inplace=True, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "id": "214d4de2",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_com['Comment Hour'] = df_com['Created At time'].str[0:2]\n",
    "# df_com['Comment Minute'] = df_com['Created At time'].str[3:5]\n",
    "# df_com['Comment Second'] = df_com['Created At time'].str[6:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "id": "1b1c5000",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_com.drop(\"Created At time\", inplace=True, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "id": "0ebb3294",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_com.rename(columns={'Created At Month': 'comment_month'},  inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "id": "197302f4",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_com.rename(columns={'Created At Day_of_month': 'comment_day'},  inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "id": "dfb7af8f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Disc_ID</th>\n",
       "      <th>User_ID</th>\n",
       "      <th>comment_day</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ID_ONQEAR</td>\n",
       "      <td>ID_CEDNKSVV_Month_5</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ID_E0NHSS</td>\n",
       "      <td>ID_CEDNKSVV_Month_12</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ID_3XKIOF</td>\n",
       "      <td>ID_CEDNKSVV_Month_1</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ID_XCC6W4</td>\n",
       "      <td>ID_CEDNKSVV_Month_1</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ID_OIGXGK</td>\n",
       "      <td>ID_CEDNKSVV_Month_12</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>462</th>\n",
       "      <td>ID_DK2VV3</td>\n",
       "      <td>ID_GPXBD1BG_Month_1</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>463</th>\n",
       "      <td>ID_5MJRT0</td>\n",
       "      <td>ID_2F218EUV_Month_1</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>464</th>\n",
       "      <td>ID_RMQMJA</td>\n",
       "      <td>ID_9W2RXCMI_Month_1</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>465</th>\n",
       "      <td>ID_V5431U</td>\n",
       "      <td>ID_RQEQ74SK_Month_12</td>\n",
       "      <td>29</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>466</th>\n",
       "      <td>ID_V5431U</td>\n",
       "      <td>ID_7ICXZH0P_Month_12</td>\n",
       "      <td>29</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>467 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       Disc_ID               User_ID  comment_day\n",
       "0    ID_ONQEAR   ID_CEDNKSVV_Month_5           11\n",
       "1    ID_E0NHSS  ID_CEDNKSVV_Month_12           23\n",
       "2    ID_3XKIOF   ID_CEDNKSVV_Month_1            3\n",
       "3    ID_XCC6W4   ID_CEDNKSVV_Month_1            5\n",
       "4    ID_OIGXGK  ID_CEDNKSVV_Month_12           19\n",
       "..         ...                   ...          ...\n",
       "462  ID_DK2VV3   ID_GPXBD1BG_Month_1           14\n",
       "463  ID_5MJRT0   ID_2F218EUV_Month_1           15\n",
       "464  ID_RMQMJA   ID_9W2RXCMI_Month_1           23\n",
       "465  ID_V5431U  ID_RQEQ74SK_Month_12           29\n",
       "466  ID_V5431U  ID_7ICXZH0P_Month_12           29\n",
       "\n",
       "[467 rows x 3 columns]"
      ]
     },
     "execution_count": 137,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_com"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "aa2ee7ac",
   "metadata": {},
   "source": [
    "# MERGE 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "id": "10f6ce45",
   "metadata": {},
   "outputs": [],
   "source": [
    "result1 = pd.merge(df_disc, df_com, how=\"outer\", on=['User_ID', 'Disc_ID'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "id": "e46c312d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Competition ID</th>\n",
       "      <th>Personal</th>\n",
       "      <th>Theme</th>\n",
       "      <th>Disc_ID</th>\n",
       "      <th>User_ID</th>\n",
       "      <th>Discussion_month</th>\n",
       "      <th>Discussion_day</th>\n",
       "      <th>comment_day</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>GeneralDiscussion</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_0BK7TH</td>\n",
       "      <td>ID_AAGFVLHH_Month_12</td>\n",
       "      <td>12</td>\n",
       "      <td>27.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>GeneralDiscussion</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_ZFJ9J7</td>\n",
       "      <td>ID_0AJGQILU_Month_12</td>\n",
       "      <td>12</td>\n",
       "      <td>20.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>GeneralDiscussion</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_ME45XX</td>\n",
       "      <td>ID_U90KK6KL_Month_2</td>\n",
       "      <td>2</td>\n",
       "      <td>9.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ID_45W9</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_2BG9IX</td>\n",
       "      <td>ID_U90KK6KL_Month_2</td>\n",
       "      <td>2</td>\n",
       "      <td>10.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>GeneralDiscussion</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_251XVV</td>\n",
       "      <td>ID_F7HIGFHQ_Month_12</td>\n",
       "      <td>12</td>\n",
       "      <td>27.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1848</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_4R4V7V</td>\n",
       "      <td>ID_JKCAR4CP_Month_1</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>NaN</td>\n",
       "      <td>25.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1849</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_ZGEWPJ</td>\n",
       "      <td>ID_LNXTXN3P_Month_1</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1850</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_DK2VV3</td>\n",
       "      <td>ID_GPXBD1BG_Month_1</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>NaN</td>\n",
       "      <td>14.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1851</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_RMQMJA</td>\n",
       "      <td>ID_9W2RXCMI_Month_1</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>NaN</td>\n",
       "      <td>23.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1852</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_V5431U</td>\n",
       "      <td>ID_7ICXZH0P_Month_12</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>NaN</td>\n",
       "      <td>29.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1853 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         Competition ID  Personal  Theme    Disc_ID               User_ID  \\\n",
       "0     GeneralDiscussion       1.0    NaN  ID_0BK7TH  ID_AAGFVLHH_Month_12   \n",
       "1     GeneralDiscussion       1.0    NaN  ID_ZFJ9J7  ID_0AJGQILU_Month_12   \n",
       "2     GeneralDiscussion       1.0    NaN  ID_ME45XX   ID_U90KK6KL_Month_2   \n",
       "3               ID_45W9       1.0    NaN  ID_2BG9IX   ID_U90KK6KL_Month_2   \n",
       "4     GeneralDiscussion       1.0    NaN  ID_251XVV  ID_F7HIGFHQ_Month_12   \n",
       "...                 ...       ...    ...        ...                   ...   \n",
       "1848                NaN       NaN    NaN  ID_4R4V7V   ID_JKCAR4CP_Month_1   \n",
       "1849                NaN       NaN    NaN  ID_ZGEWPJ   ID_LNXTXN3P_Month_1   \n",
       "1850                NaN       NaN    NaN  ID_DK2VV3   ID_GPXBD1BG_Month_1   \n",
       "1851                NaN       NaN    NaN  ID_RMQMJA   ID_9W2RXCMI_Month_1   \n",
       "1852                NaN       NaN    NaN  ID_V5431U  ID_7ICXZH0P_Month_12   \n",
       "\n",
       "     Discussion_month  Discussion_day  comment_day  \n",
       "0                  12            27.0          NaN  \n",
       "1                  12            20.0          NaN  \n",
       "2                   2             9.0          NaN  \n",
       "3                   2            10.0          NaN  \n",
       "4                  12            27.0          NaN  \n",
       "...               ...             ...          ...  \n",
       "1848             <NA>             NaN         25.0  \n",
       "1849             <NA>             NaN          5.0  \n",
       "1850             <NA>             NaN         14.0  \n",
       "1851             <NA>             NaN         23.0  \n",
       "1852             <NA>             NaN         29.0  \n",
       "\n",
       "[1853 rows x 8 columns]"
      ]
     },
     "execution_count": 139,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "id": "252fe48c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 1853 entries, 0 to 1852\n",
      "Data columns (total 8 columns):\n",
      " #   Column            Non-Null Count  Dtype  \n",
      "---  ------            --------------  -----  \n",
      " 0   Competition ID    1462 non-null   object \n",
      " 1   Personal          1462 non-null   float64\n",
      " 2   Theme             178 non-null    float64\n",
      " 3   Disc_ID           1853 non-null   object \n",
      " 4   User_ID           1853 non-null   string \n",
      " 5   Discussion_month  1462 non-null   string \n",
      " 6   Discussion_day    1462 non-null   float64\n",
      " 7   comment_day       467 non-null    float64\n",
      "dtypes: float64(4), object(2), string(2)\n",
      "memory usage: 130.3+ KB\n"
     ]
    }
   ],
   "source": [
    "result1.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b1583267",
   "metadata": {},
   "source": [
    "# COMPETITIONS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "id": "5a8227ce",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_comp = pd.read_csv(\"Competition.csv\").drop_duplicates()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "id": "d1a71db1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 247 entries, 0 to 246\n",
      "Data columns (total 20 columns):\n",
      " #   Column                   Non-Null Count  Dtype  \n",
      "---  ------                   --------------  -----  \n",
      " 0   FeatureA                 247 non-null    object \n",
      " 1   FeatureB                 247 non-null    object \n",
      " 2   FeatureC                 196 non-null    float64\n",
      " 3   FeatureD                 247 non-null    int64  \n",
      " 4   FeatureE                 247 non-null    int64  \n",
      " 5   SecretCode               247 non-null    int64  \n",
      " 6   FeatureF                 247 non-null    object \n",
      " 7   FeatureG                 247 non-null    object \n",
      " 8   FeatureH                 247 non-null    object \n",
      " 9   FeatureI                 186 non-null    float64\n",
      " 10  Country_ID               121 non-null    object \n",
      " 11  Comp_ID                  247 non-null    object \n",
      " 12  End Time time            224 non-null    object \n",
      " 13  End Time Year            224 non-null    float64\n",
      " 14  End Time Month           224 non-null    float64\n",
      " 15  End Time Day_of_month    224 non-null    float64\n",
      " 16  Start Time time          247 non-null    object \n",
      " 17  Start Time Year          247 non-null    int64  \n",
      " 18  Start Time Month         247 non-null    int64  \n",
      " 19  Start Time Day_of_month  247 non-null    int64  \n",
      "dtypes: float64(5), int64(6), object(9)\n",
      "memory usage: 40.5+ KB\n"
     ]
    }
   ],
   "source": [
    "df_comp.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "id": "bc6e19e5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>FeatureA</th>\n",
       "      <th>FeatureB</th>\n",
       "      <th>FeatureC</th>\n",
       "      <th>FeatureD</th>\n",
       "      <th>FeatureE</th>\n",
       "      <th>SecretCode</th>\n",
       "      <th>FeatureF</th>\n",
       "      <th>FeatureG</th>\n",
       "      <th>FeatureH</th>\n",
       "      <th>FeatureI</th>\n",
       "      <th>Country_ID</th>\n",
       "      <th>Comp_ID</th>\n",
       "      <th>End Time time</th>\n",
       "      <th>End Time Year</th>\n",
       "      <th>End Time Month</th>\n",
       "      <th>End Time Day_of_month</th>\n",
       "      <th>Start Time time</th>\n",
       "      <th>Start Time Year</th>\n",
       "      <th>Start Time Month</th>\n",
       "      <th>Start Time Day_of_month</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>[2, 4]</td>\n",
       "      <td>[14]</td>\n",
       "      <td>15.0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>[3]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>G3F0</td>\n",
       "      <td>ID_AU0T</td>\n",
       "      <td>21:05:00</td>\n",
       "      <td>2.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>07:00:00</td>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>[1]</td>\n",
       "      <td>[]</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>[1]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[1]</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0Q35</td>\n",
       "      <td>ID_O21T</td>\n",
       "      <td>15:30:00</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>12:00:00</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>[1]</td>\n",
       "      <td>[9]</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>[1, 5]</td>\n",
       "      <td>[5]</td>\n",
       "      <td>[1]</td>\n",
       "      <td>2.0</td>\n",
       "      <td>EGI9</td>\n",
       "      <td>ID_W6XE</td>\n",
       "      <td>21:59:00</td>\n",
       "      <td>2.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>08:00:00</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>[1]</td>\n",
       "      <td>[4]</td>\n",
       "      <td>[]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_A2FN</td>\n",
       "      <td>12:00:00</td>\n",
       "      <td>2.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>22.0</td>\n",
       "      <td>12:00:00</td>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>[]</td>\n",
       "      <td>[6]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>[]</td>\n",
       "      <td>[4]</td>\n",
       "      <td>[2]</td>\n",
       "      <td>9.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_X0PD</td>\n",
       "      <td>23:59:00</td>\n",
       "      <td>2.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>31.0</td>\n",
       "      <td>09:00:00</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>242</th>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>[2]</td>\n",
       "      <td>[5]</td>\n",
       "      <td>[2]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_GR92</td>\n",
       "      <td>15:00:00</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>05:00:00</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>243</th>\n",
       "      <td>[1]</td>\n",
       "      <td>[]</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>[1]</td>\n",
       "      <td>[5]</td>\n",
       "      <td>[1]</td>\n",
       "      <td>2.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_8BH9</td>\n",
       "      <td>23:59:00</td>\n",
       "      <td>1.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>27.0</td>\n",
       "      <td>15:00:00</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>244</th>\n",
       "      <td>[1]</td>\n",
       "      <td>[]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>[1]</td>\n",
       "      <td>[3]</td>\n",
       "      <td>[]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>H03V</td>\n",
       "      <td>ID_IHRZ</td>\n",
       "      <td>17:00:00</td>\n",
       "      <td>1.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>08:57:00</td>\n",
       "      <td>1</td>\n",
       "      <td>10</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>245</th>\n",
       "      <td>[1]</td>\n",
       "      <td>[]</td>\n",
       "      <td>25.0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>[1]</td>\n",
       "      <td>[5]</td>\n",
       "      <td>[2]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>EGI9</td>\n",
       "      <td>ID_CMWO</td>\n",
       "      <td>12:00:00</td>\n",
       "      <td>1.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>31.0</td>\n",
       "      <td>12:00:00</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>246</th>\n",
       "      <td>[1]</td>\n",
       "      <td>[]</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>[3]</td>\n",
       "      <td>[3]</td>\n",
       "      <td>[2]</td>\n",
       "      <td>2.0</td>\n",
       "      <td>ZEWU</td>\n",
       "      <td>ID_OAR6</td>\n",
       "      <td>23:59:00</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>11:00:00</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>247 rows × 20 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    FeatureA FeatureB  FeatureC  FeatureD  FeatureE  SecretCode FeatureF  \\\n",
       "0     [2, 4]     [14]      15.0         0         2           1      [3]   \n",
       "1        [1]       []       3.0         0         3           1      [1]   \n",
       "2        [1]      [9]       2.0         0         2           1   [1, 5]   \n",
       "3         []       []       1.0         0         2           1      [1]   \n",
       "4         []      [6]       NaN         0         2           0       []   \n",
       "..       ...      ...       ...       ...       ...         ...      ...   \n",
       "242       []       []       NaN         1         2           0      [2]   \n",
       "243      [1]       []       2.0         1         2           0      [1]   \n",
       "244      [1]       []       NaN         0         2           1      [1]   \n",
       "245      [1]       []      25.0         0         2           1      [1]   \n",
       "246      [1]       []       2.0         0         2           0      [3]   \n",
       "\n",
       "    FeatureG FeatureH  FeatureI Country_ID  Comp_ID End Time time  \\\n",
       "0         []       []       NaN       G3F0  ID_AU0T      21:05:00   \n",
       "1         []      [1]       3.0       0Q35  ID_O21T      15:30:00   \n",
       "2        [5]      [1]       2.0       EGI9  ID_W6XE      21:59:00   \n",
       "3        [4]       []       NaN        NaN  ID_A2FN      12:00:00   \n",
       "4        [4]      [2]       9.0        NaN  ID_X0PD      23:59:00   \n",
       "..       ...      ...       ...        ...      ...           ...   \n",
       "242      [5]      [2]       NaN        NaN  ID_GR92      15:00:00   \n",
       "243      [5]      [1]       2.0        NaN  ID_8BH9      23:59:00   \n",
       "244      [3]       []       NaN       H03V  ID_IHRZ      17:00:00   \n",
       "245      [5]      [2]       NaN       EGI9  ID_CMWO      12:00:00   \n",
       "246      [3]      [2]       2.0       ZEWU  ID_OAR6      23:59:00   \n",
       "\n",
       "     End Time Year  End Time Month  End Time Day_of_month Start Time time  \\\n",
       "0              2.0             6.0                   19.0        07:00:00   \n",
       "1              2.0             2.0                   15.0        12:00:00   \n",
       "2              2.0             5.0                    9.0        08:00:00   \n",
       "3              2.0             6.0                   22.0        12:00:00   \n",
       "4              2.0             5.0                   31.0        09:00:00   \n",
       "..             ...             ...                    ...             ...   \n",
       "242            1.0             2.0                    1.0        05:00:00   \n",
       "243            1.0             6.0                   27.0        15:00:00   \n",
       "244            1.0            10.0                   20.0        08:57:00   \n",
       "245            1.0             5.0                   31.0        12:00:00   \n",
       "246            1.0             4.0                    4.0        11:00:00   \n",
       "\n",
       "     Start Time Year  Start Time Month  Start Time Day_of_month  \n",
       "0                  2                 6                       19  \n",
       "1                  2                 2                        1  \n",
       "2                  2                 3                       28  \n",
       "3                  2                 6                       12  \n",
       "4                  2                 5                       27  \n",
       "..               ...               ...                      ...  \n",
       "242                1                 1                        1  \n",
       "243                1                 6                        1  \n",
       "244                1                10                       19  \n",
       "245                1                 5                        1  \n",
       "246                1                 4                        2  \n",
       "\n",
       "[247 rows x 20 columns]"
      ]
     },
     "execution_count": 143,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_comp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "id": "de6978c9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "247"
      ]
     },
     "execution_count": 144,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(pd.unique(df_comp['Comp_ID']))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cd89891d",
   "metadata": {},
   "source": [
    "# COMPETITION PARTICIPATION"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "id": "901743eb",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_compP = pd.read_csv(\"CompetitionPartipation.csv\").drop_duplicates()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "id": "1d6b3db1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 8385 entries, 0 to 8384\n",
      "Data columns (total 8 columns):\n",
      " #   Column                       Non-Null Count  Dtype \n",
      "---  ------                       --------------  ----- \n",
      " 0   Competition ID               8385 non-null   object\n",
      " 1   Participant Type             8385 non-null   int64 \n",
      " 2   Successful Submission Count  1449 non-null   object\n",
      " 3   User_ID                      8385 non-null   object\n",
      " 4   Created At time              8385 non-null   object\n",
      " 5   Created At Year              8385 non-null   int64 \n",
      " 6   Created At Month             8385 non-null   int64 \n",
      " 7   Created At Day_of_month      8385 non-null   int64 \n",
      "dtypes: int64(4), object(4)\n",
      "memory usage: 589.6+ KB\n"
     ]
    }
   ],
   "source": [
    "df_compP.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "id": "86de401c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Competition ID</th>\n",
       "      <th>Participant Type</th>\n",
       "      <th>Successful Submission Count</th>\n",
       "      <th>User_ID</th>\n",
       "      <th>Created At time</th>\n",
       "      <th>Created At Year</th>\n",
       "      <th>Created At Month</th>\n",
       "      <th>Created At Day_of_month</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ID_WPP3</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_40G01FGV</td>\n",
       "      <td>19:38:04.689865</td>\n",
       "      <td>1</td>\n",
       "      <td>11</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ID_ZN92</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_40G01FGV</td>\n",
       "      <td>19:38:04.670583</td>\n",
       "      <td>1</td>\n",
       "      <td>11</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ID_I9BN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_40G01FGV</td>\n",
       "      <td>12:06:55.884320</td>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "      <td>17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ID_PDKH</td>\n",
       "      <td>1</td>\n",
       "      <td>count 10</td>\n",
       "      <td>ID_40G01FGV</td>\n",
       "      <td>11:24:04.239058</td>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ID_CFN7</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_P33ABQX5</td>\n",
       "      <td>05:16:09.052129</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8380</th>\n",
       "      <td>ID_F7QD</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_QR2HEY6L</td>\n",
       "      <td>22:10:28.532519</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8381</th>\n",
       "      <td>ID_F7QD</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_K1MDGNKE</td>\n",
       "      <td>20:54:47.902303</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8382</th>\n",
       "      <td>ID_F7QD</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_M8FQBYIH</td>\n",
       "      <td>18:57:10.479540</td>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "      <td>27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8383</th>\n",
       "      <td>ID_F7QD</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_9IP2OICN</td>\n",
       "      <td>21:03:14.798218</td>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8384</th>\n",
       "      <td>ID_F7QD</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_JUE09Z86</td>\n",
       "      <td>11:02:22.739630</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8385 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     Competition ID  Participant Type Successful Submission Count  \\\n",
       "0           ID_WPP3                 1                         NaN   \n",
       "1           ID_ZN92                 1                         NaN   \n",
       "2           ID_I9BN                 1                         NaN   \n",
       "3           ID_PDKH                 1                    count 10   \n",
       "4           ID_CFN7                 1                         NaN   \n",
       "...             ...               ...                         ...   \n",
       "8380        ID_F7QD                 1                         NaN   \n",
       "8381        ID_F7QD                 1                         NaN   \n",
       "8382        ID_F7QD                 1                         NaN   \n",
       "8383        ID_F7QD                 1                         NaN   \n",
       "8384        ID_F7QD                 1                         NaN   \n",
       "\n",
       "          User_ID  Created At time  Created At Year  Created At Month  \\\n",
       "0     ID_40G01FGV  19:38:04.689865                1                11   \n",
       "1     ID_40G01FGV  19:38:04.670583                1                11   \n",
       "2     ID_40G01FGV  12:06:55.884320                1                12   \n",
       "3     ID_40G01FGV  11:24:04.239058                1                12   \n",
       "4     ID_P33ABQX5  05:16:09.052129                1                 5   \n",
       "...           ...              ...              ...               ...   \n",
       "8380  ID_QR2HEY6L  22:10:28.532519                1                 1   \n",
       "8381  ID_K1MDGNKE  20:54:47.902303                1                 3   \n",
       "8382  ID_M8FQBYIH  18:57:10.479540                1                12   \n",
       "8383  ID_9IP2OICN  21:03:14.798218                1                12   \n",
       "8384  ID_JUE09Z86  11:02:22.739630                1                 5   \n",
       "\n",
       "      Created At Day_of_month  \n",
       "0                          20  \n",
       "1                          20  \n",
       "2                          17  \n",
       "3                           6  \n",
       "4                          26  \n",
       "...                       ...  \n",
       "8380                       15  \n",
       "8381                        1  \n",
       "8382                       27  \n",
       "8383                       13  \n",
       "8384                        5  \n",
       "\n",
       "[8385 rows x 8 columns]"
      ]
     },
     "execution_count": 147,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_compP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "id": "81bf4f33",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5245"
      ]
     },
     "execution_count": 148,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(pd.unique(df_compP['User_ID']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "id": "26c84ab7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 149,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(pd.unique(df_compP['Created At Year']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "id": "6084626a",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_compP.drop('Created At Year', inplace=True, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "id": "35437760",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_compP.rename(columns={'Competition ID': 'Comp_ID'},  inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "id": "6df35171",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_compP[\"User_ID\"] = df_compP[\"User_ID\"].astype(\"string\")\n",
    "df_compP[\"Created At Month\"] = df_compP[\"Created At Month\"].astype(\"string\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "id": "0591ca7e",
   "metadata": {},
   "outputs": [],
   "source": [
    "new = df_compP[\"Created At Month\"].copy()\n",
    "df_compP[\"User_ID\"] = df_compP[\"User_ID\"].str.cat(new, sep =\"_Month_\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "id": "4be9aab2",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_compP.drop('Created At Month', inplace=True, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "id": "be3d24ef",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_compP.rename(columns={'Created At time': 'Comp_participation_time'},  inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "id": "75488f11",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_compP.rename(columns={'Created At Day_of_month': 'Comp_participation_day'},  inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "id": "f43dfb68",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Comp_ID</th>\n",
       "      <th>Participant Type</th>\n",
       "      <th>Successful Submission Count</th>\n",
       "      <th>User_ID</th>\n",
       "      <th>Created At time</th>\n",
       "      <th>Comp_participation_day</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ID_WPP3</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_40G01FGV_Month_11</td>\n",
       "      <td>19:38:04.689865</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ID_ZN92</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_40G01FGV_Month_11</td>\n",
       "      <td>19:38:04.670583</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ID_I9BN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_40G01FGV_Month_12</td>\n",
       "      <td>12:06:55.884320</td>\n",
       "      <td>17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ID_PDKH</td>\n",
       "      <td>1</td>\n",
       "      <td>count 10</td>\n",
       "      <td>ID_40G01FGV_Month_12</td>\n",
       "      <td>11:24:04.239058</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ID_CFN7</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_P33ABQX5_Month_5</td>\n",
       "      <td>05:16:09.052129</td>\n",
       "      <td>26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8380</th>\n",
       "      <td>ID_F7QD</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_QR2HEY6L_Month_1</td>\n",
       "      <td>22:10:28.532519</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8381</th>\n",
       "      <td>ID_F7QD</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_K1MDGNKE_Month_3</td>\n",
       "      <td>20:54:47.902303</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8382</th>\n",
       "      <td>ID_F7QD</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_M8FQBYIH_Month_12</td>\n",
       "      <td>18:57:10.479540</td>\n",
       "      <td>27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8383</th>\n",
       "      <td>ID_F7QD</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_9IP2OICN_Month_12</td>\n",
       "      <td>21:03:14.798218</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8384</th>\n",
       "      <td>ID_F7QD</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_JUE09Z86_Month_5</td>\n",
       "      <td>11:02:22.739630</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8385 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      Comp_ID  Participant Type Successful Submission Count  \\\n",
       "0     ID_WPP3                 1                         NaN   \n",
       "1     ID_ZN92                 1                         NaN   \n",
       "2     ID_I9BN                 1                         NaN   \n",
       "3     ID_PDKH                 1                    count 10   \n",
       "4     ID_CFN7                 1                         NaN   \n",
       "...       ...               ...                         ...   \n",
       "8380  ID_F7QD                 1                         NaN   \n",
       "8381  ID_F7QD                 1                         NaN   \n",
       "8382  ID_F7QD                 1                         NaN   \n",
       "8383  ID_F7QD                 1                         NaN   \n",
       "8384  ID_F7QD                 1                         NaN   \n",
       "\n",
       "                   User_ID  Created At time  Comp_participation_day  \n",
       "0     ID_40G01FGV_Month_11  19:38:04.689865                      20  \n",
       "1     ID_40G01FGV_Month_11  19:38:04.670583                      20  \n",
       "2     ID_40G01FGV_Month_12  12:06:55.884320                      17  \n",
       "3     ID_40G01FGV_Month_12  11:24:04.239058                       6  \n",
       "4      ID_P33ABQX5_Month_5  05:16:09.052129                      26  \n",
       "...                    ...              ...                     ...  \n",
       "8380   ID_QR2HEY6L_Month_1  22:10:28.532519                      15  \n",
       "8381   ID_K1MDGNKE_Month_3  20:54:47.902303                       1  \n",
       "8382  ID_M8FQBYIH_Month_12  18:57:10.479540                      27  \n",
       "8383  ID_9IP2OICN_Month_12  21:03:14.798218                      13  \n",
       "8384   ID_JUE09Z86_Month_5  11:02:22.739630                       5  \n",
       "\n",
       "[8385 rows x 6 columns]"
      ]
     },
     "execution_count": 157,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_compP"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "19b75814",
   "metadata": {},
   "source": [
    "# MERGE 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "id": "f46842f9",
   "metadata": {},
   "outputs": [],
   "source": [
    "result2 = pd.merge(df_compP, df_comp, how=\"outer\", on= 'Comp_ID')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "id": "421d0c7d",
   "metadata": {},
   "outputs": [],
   "source": [
    "result2 = result2[result2['User_ID'].notna()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "id": "5b4488e5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 8385 entries, 0 to 8384\n",
      "Data columns (total 25 columns):\n",
      " #   Column                       Non-Null Count  Dtype  \n",
      "---  ------                       --------------  -----  \n",
      " 0   Comp_ID                      8385 non-null   object \n",
      " 1   Participant Type             8385 non-null   float64\n",
      " 2   Successful Submission Count  1449 non-null   object \n",
      " 3   User_ID                      8385 non-null   string \n",
      " 4   Created At time              8385 non-null   object \n",
      " 5   Comp_participation_day       8385 non-null   float64\n",
      " 6   FeatureA                     8385 non-null   object \n",
      " 7   FeatureB                     8385 non-null   object \n",
      " 8   FeatureC                     5655 non-null   float64\n",
      " 9   FeatureD                     8385 non-null   int64  \n",
      " 10  FeatureE                     8385 non-null   int64  \n",
      " 11  SecretCode                   8385 non-null   int64  \n",
      " 12  FeatureF                     8385 non-null   object \n",
      " 13  FeatureG                     8385 non-null   object \n",
      " 14  FeatureH                     8385 non-null   object \n",
      " 15  FeatureI                     7206 non-null   float64\n",
      " 16  Country_ID                   4073 non-null   object \n",
      " 17  End Time time                6340 non-null   object \n",
      " 18  End Time Year                6340 non-null   float64\n",
      " 19  End Time Month               6340 non-null   float64\n",
      " 20  End Time Day_of_month        6340 non-null   float64\n",
      " 21  Start Time time              8385 non-null   object \n",
      " 22  Start Time Year              8385 non-null   int64  \n",
      " 23  Start Time Month             8385 non-null   int64  \n",
      " 24  Start Time Day_of_month      8385 non-null   int64  \n",
      "dtypes: float64(7), int64(6), object(11), string(1)\n",
      "memory usage: 1.7+ MB\n"
     ]
    }
   ],
   "source": [
    "result2.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "id": "cdd809f8",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Comp_ID</th>\n",
       "      <th>Participant Type</th>\n",
       "      <th>Successful Submission Count</th>\n",
       "      <th>User_ID</th>\n",
       "      <th>Created At time</th>\n",
       "      <th>Comp_participation_day</th>\n",
       "      <th>FeatureA</th>\n",
       "      <th>FeatureB</th>\n",
       "      <th>FeatureC</th>\n",
       "      <th>FeatureD</th>\n",
       "      <th>...</th>\n",
       "      <th>FeatureI</th>\n",
       "      <th>Country_ID</th>\n",
       "      <th>End Time time</th>\n",
       "      <th>End Time Year</th>\n",
       "      <th>End Time Month</th>\n",
       "      <th>End Time Day_of_month</th>\n",
       "      <th>Start Time time</th>\n",
       "      <th>Start Time Year</th>\n",
       "      <th>Start Time Month</th>\n",
       "      <th>Start Time Day_of_month</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ID_WPP3</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_40G01FGV_Month_11</td>\n",
       "      <td>19:38:04.689865</td>\n",
       "      <td>20.0</td>\n",
       "      <td>[1]</td>\n",
       "      <td>[12, 16]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>00:00:00</td>\n",
       "      <td>3.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>24.0</td>\n",
       "      <td>00:00:00</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ID_WPP3</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_D5I1M68U_Month_4</td>\n",
       "      <td>21:15:54.617702</td>\n",
       "      <td>7.0</td>\n",
       "      <td>[1]</td>\n",
       "      <td>[12, 16]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>00:00:00</td>\n",
       "      <td>3.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>24.0</td>\n",
       "      <td>00:00:00</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ID_WPP3</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_G23UREIF_Month_11</td>\n",
       "      <td>16:43:35.871761</td>\n",
       "      <td>30.0</td>\n",
       "      <td>[1]</td>\n",
       "      <td>[12, 16]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>00:00:00</td>\n",
       "      <td>3.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>24.0</td>\n",
       "      <td>00:00:00</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ID_WPP3</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_H4DF1YA9_Month_11</td>\n",
       "      <td>17:55:40.888019</td>\n",
       "      <td>23.0</td>\n",
       "      <td>[1]</td>\n",
       "      <td>[12, 16]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>00:00:00</td>\n",
       "      <td>3.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>24.0</td>\n",
       "      <td>00:00:00</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ID_WPP3</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_F840O4ZH_Month_5</td>\n",
       "      <td>09:42:13.812496</td>\n",
       "      <td>16.0</td>\n",
       "      <td>[1]</td>\n",
       "      <td>[12, 16]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>00:00:00</td>\n",
       "      <td>3.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>24.0</td>\n",
       "      <td>00:00:00</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8380</th>\n",
       "      <td>ID_CNBR</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_6RPFZR7I_Month_11</td>\n",
       "      <td>06:38:13.766016</td>\n",
       "      <td>17.0</td>\n",
       "      <td>[2, 7]</td>\n",
       "      <td>[]</td>\n",
       "      <td>9.0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>9.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>00:00:00</td>\n",
       "      <td>3.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>00:00:00</td>\n",
       "      <td>3</td>\n",
       "      <td>8</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8381</th>\n",
       "      <td>ID_208Z</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_YEZY3ABB_Month_1</td>\n",
       "      <td>14:26:36.511271</td>\n",
       "      <td>6.0</td>\n",
       "      <td>[1, 5, 4]</td>\n",
       "      <td>[5, 7]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>N6EN</td>\n",
       "      <td>23:59:00</td>\n",
       "      <td>3.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>17.0</td>\n",
       "      <td>00:00:00</td>\n",
       "      <td>4</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8382</th>\n",
       "      <td>ID_H3QQ</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_GS03UEEX_Month_4</td>\n",
       "      <td>10:40:22.978880</td>\n",
       "      <td>22.0</td>\n",
       "      <td>[2]</td>\n",
       "      <td>[]</td>\n",
       "      <td>17.0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>17.0</td>\n",
       "      <td>4PRY</td>\n",
       "      <td>00:00:00</td>\n",
       "      <td>3.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>17.0</td>\n",
       "      <td>00:00:00</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8383</th>\n",
       "      <td>ID_7079</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_1CY5JAGU_Month_4</td>\n",
       "      <td>04:23:17.682554</td>\n",
       "      <td>11.0</td>\n",
       "      <td>[7, 3, 2]</td>\n",
       "      <td>[15]</td>\n",
       "      <td>10.0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>10.0</td>\n",
       "      <td>G3F0</td>\n",
       "      <td>23:59:00</td>\n",
       "      <td>3.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>06:00:00</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8384</th>\n",
       "      <td>ID_DFKK</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_2YXKSD1Q_Month_4</td>\n",
       "      <td>15:34:15.045823</td>\n",
       "      <td>7.0</td>\n",
       "      <td>[5]</td>\n",
       "      <td>[15]</td>\n",
       "      <td>10.0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>10.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>23:59:00</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>06:00:00</td>\n",
       "      <td>2</td>\n",
       "      <td>10</td>\n",
       "      <td>26</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8385 rows × 25 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      Comp_ID  Participant Type Successful Submission Count  \\\n",
       "0     ID_WPP3               1.0                         NaN   \n",
       "1     ID_WPP3               1.0                         NaN   \n",
       "2     ID_WPP3               1.0                         NaN   \n",
       "3     ID_WPP3               1.0                         NaN   \n",
       "4     ID_WPP3               1.0                         NaN   \n",
       "...       ...               ...                         ...   \n",
       "8380  ID_CNBR               1.0                         NaN   \n",
       "8381  ID_208Z               1.0                         NaN   \n",
       "8382  ID_H3QQ               1.0                         NaN   \n",
       "8383  ID_7079               1.0                         NaN   \n",
       "8384  ID_DFKK               1.0                         NaN   \n",
       "\n",
       "                   User_ID  Created At time  Comp_participation_day  \\\n",
       "0     ID_40G01FGV_Month_11  19:38:04.689865                    20.0   \n",
       "1      ID_D5I1M68U_Month_4  21:15:54.617702                     7.0   \n",
       "2     ID_G23UREIF_Month_11  16:43:35.871761                    30.0   \n",
       "3     ID_H4DF1YA9_Month_11  17:55:40.888019                    23.0   \n",
       "4      ID_F840O4ZH_Month_5  09:42:13.812496                    16.0   \n",
       "...                    ...              ...                     ...   \n",
       "8380  ID_6RPFZR7I_Month_11  06:38:13.766016                    17.0   \n",
       "8381   ID_YEZY3ABB_Month_1  14:26:36.511271                     6.0   \n",
       "8382   ID_GS03UEEX_Month_4  10:40:22.978880                    22.0   \n",
       "8383   ID_1CY5JAGU_Month_4  04:23:17.682554                    11.0   \n",
       "8384   ID_2YXKSD1Q_Month_4  15:34:15.045823                     7.0   \n",
       "\n",
       "       FeatureA  FeatureB  FeatureC  FeatureD  ...  FeatureI  Country_ID  \\\n",
       "0           [1]  [12, 16]       NaN         1  ...       NaN         NaN   \n",
       "1           [1]  [12, 16]       NaN         1  ...       NaN         NaN   \n",
       "2           [1]  [12, 16]       NaN         1  ...       NaN         NaN   \n",
       "3           [1]  [12, 16]       NaN         1  ...       NaN         NaN   \n",
       "4           [1]  [12, 16]       NaN         1  ...       NaN         NaN   \n",
       "...         ...       ...       ...       ...  ...       ...         ...   \n",
       "8380     [2, 7]        []       9.0         1  ...       9.0         NaN   \n",
       "8381  [1, 5, 4]    [5, 7]       NaN         1  ...       NaN        N6EN   \n",
       "8382        [2]        []      17.0         1  ...      17.0        4PRY   \n",
       "8383  [7, 3, 2]      [15]      10.0         1  ...      10.0        G3F0   \n",
       "8384        [5]      [15]      10.0         1  ...      10.0         NaN   \n",
       "\n",
       "     End Time time End Time Year End Time Month  End Time Day_of_month  \\\n",
       "0         00:00:00           3.0            9.0                   24.0   \n",
       "1         00:00:00           3.0            9.0                   24.0   \n",
       "2         00:00:00           3.0            9.0                   24.0   \n",
       "3         00:00:00           3.0            9.0                   24.0   \n",
       "4         00:00:00           3.0            9.0                   24.0   \n",
       "...            ...           ...            ...                    ...   \n",
       "8380      00:00:00           3.0           11.0                   20.0   \n",
       "8381      23:59:00           3.0           12.0                   17.0   \n",
       "8382      00:00:00           3.0            9.0                   17.0   \n",
       "8383      23:59:00           3.0            6.0                   29.0   \n",
       "8384      23:59:00           2.0            2.0                    4.0   \n",
       "\n",
       "     Start Time time Start Time Year  Start Time Month  \\\n",
       "0           00:00:00               4                 6   \n",
       "1           00:00:00               4                 6   \n",
       "2           00:00:00               4                 6   \n",
       "3           00:00:00               4                 6   \n",
       "4           00:00:00               4                 6   \n",
       "...              ...             ...               ...   \n",
       "8380        00:00:00               3                 8   \n",
       "8381        00:00:00               4                 7   \n",
       "8382        00:00:00               4                 5   \n",
       "8383        06:00:00               3                 4   \n",
       "8384        06:00:00               2                10   \n",
       "\n",
       "      Start Time Day_of_month  \n",
       "0                          25  \n",
       "1                          25  \n",
       "2                          25  \n",
       "3                          25  \n",
       "4                          25  \n",
       "...                       ...  \n",
       "8380                       16  \n",
       "8381                        2  \n",
       "8382                       18  \n",
       "8383                       11  \n",
       "8384                       26  \n",
       "\n",
       "[8385 rows x 25 columns]"
      ]
     },
     "execution_count": 161,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result2"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9bb36659",
   "metadata": {},
   "source": [
    "# USERS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "id": "c7178f32",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_user = pd.read_csv('Users.csv').drop_duplicates()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "id": "3b2e8c30",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 12413 entries, 0 to 12412\n",
      "Data columns (total 8 columns):\n",
      " #   Column                   Non-Null Count  Dtype \n",
      "---  ------                   --------------  ----- \n",
      " 0   FeatureX                 12413 non-null  int64 \n",
      " 1   FeatureY                 12413 non-null  int64 \n",
      " 2   User_ID                  12413 non-null  object\n",
      " 3   Countries_ID             6531 non-null   object\n",
      " 4   Created At time          12413 non-null  object\n",
      " 5   Created At Year          12413 non-null  int64 \n",
      " 6   Created At Month         12413 non-null  int64 \n",
      " 7   Created At Day_of_month  12413 non-null  int64 \n",
      "dtypes: int64(5), object(3)\n",
      "memory usage: 872.8+ KB\n"
     ]
    }
   ],
   "source": [
    "df_user.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "id": "81843e49",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>FeatureX</th>\n",
       "      <th>FeatureY</th>\n",
       "      <th>User_ID</th>\n",
       "      <th>Countries_ID</th>\n",
       "      <th>Created At time</th>\n",
       "      <th>Created At Year</th>\n",
       "      <th>Created At Month</th>\n",
       "      <th>Created At Day_of_month</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>ID_DC6S4E9O</td>\n",
       "      <td>X9GR</td>\n",
       "      <td>19:33:13.663391</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>ID_E8S97OUT</td>\n",
       "      <td>X9GR</td>\n",
       "      <td>22:18:18.228921</td>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "      <td>27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>ID_QZ1HASL3</td>\n",
       "      <td>X9GR</td>\n",
       "      <td>23:13:03.266635</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>ID_SDV9Q3E9</td>\n",
       "      <td>X9GR</td>\n",
       "      <td>09:50:09.697573</td>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "      <td>27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>ID_W942LN7F</td>\n",
       "      <td>X9GR</td>\n",
       "      <td>13:33:06.531662</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12408</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>ID_C5393PD0</td>\n",
       "      <td>7DKX</td>\n",
       "      <td>05:02:39.352427</td>\n",
       "      <td>1</td>\n",
       "      <td>11</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12409</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>ID_QX7XRT0C</td>\n",
       "      <td>K4DG</td>\n",
       "      <td>02:50:19.027320</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12410</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>ID_RGN2RK66</td>\n",
       "      <td>ONON</td>\n",
       "      <td>22:06:33.713692</td>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12411</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>ID_XZS1QE0W</td>\n",
       "      <td>3YQR</td>\n",
       "      <td>06:56:02.570173</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12412</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>ID_8K4FGJYO</td>\n",
       "      <td>G4PG</td>\n",
       "      <td>10:22:54.369939</td>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>12413 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       FeatureX  FeatureY      User_ID Countries_ID  Created At time  \\\n",
       "0             0         0  ID_DC6S4E9O         X9GR  19:33:13.663391   \n",
       "1             1         0  ID_E8S97OUT         X9GR  22:18:18.228921   \n",
       "2             0         0  ID_QZ1HASL3         X9GR  23:13:03.266635   \n",
       "3             1         0  ID_SDV9Q3E9         X9GR  09:50:09.697573   \n",
       "4             1         0  ID_W942LN7F         X9GR  13:33:06.531662   \n",
       "...         ...       ...          ...          ...              ...   \n",
       "12408         0         1  ID_C5393PD0         7DKX  05:02:39.352427   \n",
       "12409         0         1  ID_QX7XRT0C         K4DG  02:50:19.027320   \n",
       "12410         0         1  ID_RGN2RK66         ONON  22:06:33.713692   \n",
       "12411         0         0  ID_XZS1QE0W         3YQR  06:56:02.570173   \n",
       "12412         0         1  ID_8K4FGJYO         G4PG  10:22:54.369939   \n",
       "\n",
       "       Created At Year  Created At Month  Created At Day_of_month  \n",
       "0                    1                 1                       16  \n",
       "1                    1                12                       27  \n",
       "2                    1                 1                        1  \n",
       "3                    1                12                       27  \n",
       "4                    1                 1                       16  \n",
       "...                ...               ...                      ...  \n",
       "12408                1                11                        6  \n",
       "12409                1                 3                       25  \n",
       "12410                1                12                        7  \n",
       "12411                1                 5                       27  \n",
       "12412                1                12                        5  \n",
       "\n",
       "[12413 rows x 8 columns]"
      ]
     },
     "execution_count": 164,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_user"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "id": "135d278c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "12413"
      ]
     },
     "execution_count": 165,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(pd.unique(df_user['User_ID']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "id": "a6536600",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 166,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(pd.unique(df_user['Created At Year']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "id": "3f8fabc6",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_user.drop('Created At Year', inplace=True, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "id": "9651893b",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_user[\"User_ID\"] = df_user[\"User_ID\"].astype(\"string\")\n",
    "df_user[\"Created At Month\"] = df_user[\"Created At Month\"].astype(\"string\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "id": "60f3d1ea",
   "metadata": {},
   "outputs": [],
   "source": [
    "new = df_user[\"Created At Month\"].copy()\n",
    "df_user[\"User_ID\"] = df_user[\"User_ID\"].str.cat(new, sep =\"_Month_\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "id": "4b0ad8a8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "12413"
      ]
     },
     "execution_count": 170,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(pd.unique(df_user['User_ID']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "id": "b5662e4a",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_user.rename(columns={'Created At Month': 'User sign-in month'},  inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "id": "07c053b3",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_user.rename(columns={'Created At time': 'User sign-in time'},  inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "id": "e8b05b00",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_user.rename(columns={'Created At Day_of_month': 'User sign-in day'},  inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "id": "97d057bd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>FeatureX</th>\n",
       "      <th>FeatureY</th>\n",
       "      <th>User_ID</th>\n",
       "      <th>Countries_ID</th>\n",
       "      <th>User sign-in time</th>\n",
       "      <th>User sign-in month</th>\n",
       "      <th>User sign-in day</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>ID_DC6S4E9O_Month_1</td>\n",
       "      <td>X9GR</td>\n",
       "      <td>19:33:13.663391</td>\n",
       "      <td>1</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>ID_E8S97OUT_Month_12</td>\n",
       "      <td>X9GR</td>\n",
       "      <td>22:18:18.228921</td>\n",
       "      <td>12</td>\n",
       "      <td>27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>ID_QZ1HASL3_Month_1</td>\n",
       "      <td>X9GR</td>\n",
       "      <td>23:13:03.266635</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>ID_SDV9Q3E9_Month_12</td>\n",
       "      <td>X9GR</td>\n",
       "      <td>09:50:09.697573</td>\n",
       "      <td>12</td>\n",
       "      <td>27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>ID_W942LN7F_Month_1</td>\n",
       "      <td>X9GR</td>\n",
       "      <td>13:33:06.531662</td>\n",
       "      <td>1</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12408</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>ID_C5393PD0_Month_11</td>\n",
       "      <td>7DKX</td>\n",
       "      <td>05:02:39.352427</td>\n",
       "      <td>11</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12409</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>ID_QX7XRT0C_Month_3</td>\n",
       "      <td>K4DG</td>\n",
       "      <td>02:50:19.027320</td>\n",
       "      <td>3</td>\n",
       "      <td>25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12410</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>ID_RGN2RK66_Month_12</td>\n",
       "      <td>ONON</td>\n",
       "      <td>22:06:33.713692</td>\n",
       "      <td>12</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12411</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>ID_XZS1QE0W_Month_5</td>\n",
       "      <td>3YQR</td>\n",
       "      <td>06:56:02.570173</td>\n",
       "      <td>5</td>\n",
       "      <td>27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12412</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>ID_8K4FGJYO_Month_12</td>\n",
       "      <td>G4PG</td>\n",
       "      <td>10:22:54.369939</td>\n",
       "      <td>12</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>12413 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       FeatureX  FeatureY               User_ID Countries_ID  \\\n",
       "0             0         0   ID_DC6S4E9O_Month_1         X9GR   \n",
       "1             1         0  ID_E8S97OUT_Month_12         X9GR   \n",
       "2             0         0   ID_QZ1HASL3_Month_1         X9GR   \n",
       "3             1         0  ID_SDV9Q3E9_Month_12         X9GR   \n",
       "4             1         0   ID_W942LN7F_Month_1         X9GR   \n",
       "...         ...       ...                   ...          ...   \n",
       "12408         0         1  ID_C5393PD0_Month_11         7DKX   \n",
       "12409         0         1   ID_QX7XRT0C_Month_3         K4DG   \n",
       "12410         0         1  ID_RGN2RK66_Month_12         ONON   \n",
       "12411         0         0   ID_XZS1QE0W_Month_5         3YQR   \n",
       "12412         0         1  ID_8K4FGJYO_Month_12         G4PG   \n",
       "\n",
       "      User sign-in time User sign-in month  User sign-in day  \n",
       "0       19:33:13.663391                  1                16  \n",
       "1       22:18:18.228921                 12                27  \n",
       "2       23:13:03.266635                  1                 1  \n",
       "3       09:50:09.697573                 12                27  \n",
       "4       13:33:06.531662                  1                16  \n",
       "...                 ...                ...               ...  \n",
       "12408   05:02:39.352427                 11                 6  \n",
       "12409   02:50:19.027320                  3                25  \n",
       "12410   22:06:33.713692                 12                 7  \n",
       "12411   06:56:02.570173                  5                27  \n",
       "12412   10:22:54.369939                 12                 5  \n",
       "\n",
       "[12413 rows x 7 columns]"
      ]
     },
     "execution_count": 174,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_user"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a171fc36",
   "metadata": {},
   "source": [
    "# MERGE 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "id": "f4391e48",
   "metadata": {},
   "outputs": [],
   "source": [
    "result3 = pd.merge(df_userAct, df_user, how=\"outer\", on= 'User_ID')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "id": "19c94b43",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>User_ID</th>\n",
       "      <th>datetime Day_of_month</th>\n",
       "      <th>Blog Theme</th>\n",
       "      <th>Blog Published Year</th>\n",
       "      <th>Blog Published Month</th>\n",
       "      <th>Published At Day_of_month</th>\n",
       "      <th>Remote</th>\n",
       "      <th>Experience</th>\n",
       "      <th>Employment Type</th>\n",
       "      <th>Company Size</th>\n",
       "      <th>...</th>\n",
       "      <th>Function_AI</th>\n",
       "      <th>Function_ML</th>\n",
       "      <th>count_UA</th>\n",
       "      <th>Day duration</th>\n",
       "      <th>FeatureX</th>\n",
       "      <th>FeatureY</th>\n",
       "      <th>Countries_ID</th>\n",
       "      <th>User sign-in time</th>\n",
       "      <th>User sign-in month</th>\n",
       "      <th>User sign-in day</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ID_000H9XOP_Month_5</td>\n",
       "      <td>26.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>15:18:11.439200</td>\n",
       "      <td>5</td>\n",
       "      <td>22.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ID_003YJHQA_Month_4</td>\n",
       "      <td>12.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>GYYX</td>\n",
       "      <td>10:30:32.261855</td>\n",
       "      <td>4</td>\n",
       "      <td>12.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ID_004UKSJV_Month_1</td>\n",
       "      <td>15.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ID_004UKSJV_Month_12</td>\n",
       "      <td>29.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>IFO5</td>\n",
       "      <td>08:32:14.997443</td>\n",
       "      <td>12</td>\n",
       "      <td>28.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ID_008D2J6C_Month_1</td>\n",
       "      <td>15.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>GYYX</td>\n",
       "      <td>09:42:39.695313</td>\n",
       "      <td>1</td>\n",
       "      <td>7.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17182</th>\n",
       "      <td>ID_Y02FKZWI_Month_3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7JT1</td>\n",
       "      <td>04:04:35.100180</td>\n",
       "      <td>3</td>\n",
       "      <td>26.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17183</th>\n",
       "      <td>ID_FDOGWALS_Month_11</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>UKC2</td>\n",
       "      <td>13:30:49.814791</td>\n",
       "      <td>11</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17184</th>\n",
       "      <td>ID_YJCY8FF5_Month_5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>H801</td>\n",
       "      <td>15:54:12.676400</td>\n",
       "      <td>5</td>\n",
       "      <td>11.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17185</th>\n",
       "      <td>ID_QDL8A9FQ_Month_4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NRDP</td>\n",
       "      <td>09:49:35.535847</td>\n",
       "      <td>4</td>\n",
       "      <td>28.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17186</th>\n",
       "      <td>ID_C5393PD0_Month_11</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>7DKX</td>\n",
       "      <td>05:02:39.352427</td>\n",
       "      <td>11</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>17187 rows × 63 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                    User_ID  datetime Day_of_month  Blog Theme  \\\n",
       "0       ID_000H9XOP_Month_5                   26.0         NaN   \n",
       "1       ID_003YJHQA_Month_4                   12.0         NaN   \n",
       "2       ID_004UKSJV_Month_1                   15.0         NaN   \n",
       "3      ID_004UKSJV_Month_12                   29.0         NaN   \n",
       "4       ID_008D2J6C_Month_1                   15.0         NaN   \n",
       "...                     ...                    ...         ...   \n",
       "17182   ID_Y02FKZWI_Month_3                    NaN         NaN   \n",
       "17183  ID_FDOGWALS_Month_11                    NaN         NaN   \n",
       "17184   ID_YJCY8FF5_Month_5                    NaN         NaN   \n",
       "17185   ID_QDL8A9FQ_Month_4                    NaN         NaN   \n",
       "17186  ID_C5393PD0_Month_11                    NaN         NaN   \n",
       "\n",
       "       Blog Published Year  Blog Published Month  Published At Day_of_month  \\\n",
       "0                      NaN                   NaN                        NaN   \n",
       "1                      NaN                   NaN                        NaN   \n",
       "2                      NaN                   NaN                        NaN   \n",
       "3                      NaN                   NaN                        NaN   \n",
       "4                      NaN                   NaN                        NaN   \n",
       "...                    ...                   ...                        ...   \n",
       "17182                  NaN                   NaN                        NaN   \n",
       "17183                  NaN                   NaN                        NaN   \n",
       "17184                  NaN                   NaN                        NaN   \n",
       "17185                  NaN                   NaN                        NaN   \n",
       "17186                  NaN                   NaN                        NaN   \n",
       "\n",
       "       Remote  Experience  Employment Type  Company Size  ...  Function_AI  \\\n",
       "0        -1.0         NaN              4.0           NaN  ...          0.0   \n",
       "1        -1.0         NaN              4.0           NaN  ...          0.0   \n",
       "2        -1.0         NaN              4.0           NaN  ...          0.0   \n",
       "3        -1.0         NaN              4.0           NaN  ...          0.0   \n",
       "4        -1.0         NaN              4.0           NaN  ...          0.0   \n",
       "...       ...         ...              ...           ...  ...          ...   \n",
       "17182     NaN         NaN              NaN           NaN  ...          NaN   \n",
       "17183     NaN         NaN              NaN           NaN  ...          NaN   \n",
       "17184     NaN         NaN              NaN           NaN  ...          NaN   \n",
       "17185     NaN         NaN              NaN           NaN  ...          NaN   \n",
       "17186     NaN         NaN              NaN           NaN  ...          NaN   \n",
       "\n",
       "       Function_ML  count_UA  Day duration  FeatureX  FeatureY  Countries_ID  \\\n",
       "0              0.0       6.0           4.0       0.0       1.0           NaN   \n",
       "1              0.0      11.0           0.0       0.0       1.0          GYYX   \n",
       "2              0.0       1.0           0.0       NaN       NaN           NaN   \n",
       "3              0.0      11.0           1.0       0.0       0.0          IFO5   \n",
       "4              0.0      10.0           8.0       0.0       0.0          GYYX   \n",
       "...            ...       ...           ...       ...       ...           ...   \n",
       "17182          NaN       NaN           NaN       0.0       0.0          7JT1   \n",
       "17183          NaN       NaN           NaN       0.0       0.0          UKC2   \n",
       "17184          NaN       NaN           NaN       0.0       0.0          H801   \n",
       "17185          NaN       NaN           NaN       0.0       0.0          NRDP   \n",
       "17186          NaN       NaN           NaN       0.0       1.0          7DKX   \n",
       "\n",
       "       User sign-in time  User sign-in month  User sign-in day  \n",
       "0        15:18:11.439200                   5              22.0  \n",
       "1        10:30:32.261855                   4              12.0  \n",
       "2                    NaN                <NA>               NaN  \n",
       "3        08:32:14.997443                  12              28.0  \n",
       "4        09:42:39.695313                   1               7.0  \n",
       "...                  ...                 ...               ...  \n",
       "17182    04:04:35.100180                   3              26.0  \n",
       "17183    13:30:49.814791                  11               5.0  \n",
       "17184    15:54:12.676400                   5              11.0  \n",
       "17185    09:49:35.535847                   4              28.0  \n",
       "17186    05:02:39.352427                  11               6.0  \n",
       "\n",
       "[17187 rows x 63 columns]"
      ]
     },
     "execution_count": 176,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result3"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "40dcd973",
   "metadata": {},
   "source": [
    "# MERGE 4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "id": "23ad4c6f",
   "metadata": {},
   "outputs": [],
   "source": [
    "result4 = pd.merge(result1, result2, how=\"outer\", on= 'User_ID')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "id": "7bcccc18",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 9927 entries, 0 to 9926\n",
      "Data columns (total 32 columns):\n",
      " #   Column                       Non-Null Count  Dtype  \n",
      "---  ------                       --------------  -----  \n",
      " 0   Competition ID               2305 non-null   object \n",
      " 1   Personal                     2305 non-null   float64\n",
      " 2   Theme                        272 non-null    float64\n",
      " 3   Disc_ID                      2891 non-null   object \n",
      " 4   User_ID                      9927 non-null   string \n",
      " 5   Discussion_month             2305 non-null   string \n",
      " 6   Discussion_day               2305 non-null   float64\n",
      " 7   comment_day                  708 non-null    float64\n",
      " 8   Comp_ID                      9300 non-null   object \n",
      " 9   Participant Type             9300 non-null   float64\n",
      " 10  Successful Submission Count  1813 non-null   object \n",
      " 11  Created At time              9300 non-null   object \n",
      " 12  Comp_participation_day       9300 non-null   float64\n",
      " 13  FeatureA                     9300 non-null   object \n",
      " 14  FeatureB                     9300 non-null   object \n",
      " 15  FeatureC                     6155 non-null   float64\n",
      " 16  FeatureD                     9300 non-null   float64\n",
      " 17  FeatureE                     9300 non-null   float64\n",
      " 18  SecretCode                   9300 non-null   float64\n",
      " 19  FeatureF                     9300 non-null   object \n",
      " 20  FeatureG                     9300 non-null   object \n",
      " 21  FeatureH                     9300 non-null   object \n",
      " 22  FeatureI                     7964 non-null   float64\n",
      " 23  Country_ID                   4565 non-null   object \n",
      " 24  End Time time                7086 non-null   object \n",
      " 25  End Time Year                7086 non-null   float64\n",
      " 26  End Time Month               7086 non-null   float64\n",
      " 27  End Time Day_of_month        7086 non-null   float64\n",
      " 28  Start Time time              9300 non-null   object \n",
      " 29  Start Time Year              9300 non-null   float64\n",
      " 30  Start Time Month             9300 non-null   float64\n",
      " 31  Start Time Day_of_month      9300 non-null   float64\n",
      "dtypes: float64(17), object(13), string(2)\n",
      "memory usage: 2.5+ MB\n"
     ]
    }
   ],
   "source": [
    "result4.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "id": "cb30280a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Competition ID</th>\n",
       "      <th>Personal</th>\n",
       "      <th>Theme</th>\n",
       "      <th>Disc_ID</th>\n",
       "      <th>User_ID</th>\n",
       "      <th>Discussion_month</th>\n",
       "      <th>Discussion_day</th>\n",
       "      <th>comment_day</th>\n",
       "      <th>Comp_ID</th>\n",
       "      <th>Participant Type</th>\n",
       "      <th>...</th>\n",
       "      <th>FeatureI</th>\n",
       "      <th>Country_ID</th>\n",
       "      <th>End Time time</th>\n",
       "      <th>End Time Year</th>\n",
       "      <th>End Time Month</th>\n",
       "      <th>End Time Day_of_month</th>\n",
       "      <th>Start Time time</th>\n",
       "      <th>Start Time Year</th>\n",
       "      <th>Start Time Month</th>\n",
       "      <th>Start Time Day_of_month</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>GeneralDiscussion</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_0BK7TH</td>\n",
       "      <td>ID_AAGFVLHH_Month_12</td>\n",
       "      <td>12</td>\n",
       "      <td>27.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_528W</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>31.0</td>\n",
       "      <td>X9GR</td>\n",
       "      <td>22:59:00</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>22:59:00</td>\n",
       "      <td>1.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>28.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>GeneralDiscussion</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_ZFJ9J7</td>\n",
       "      <td>ID_0AJGQILU_Month_12</td>\n",
       "      <td>12</td>\n",
       "      <td>20.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>GeneralDiscussion</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_ME45XX</td>\n",
       "      <td>ID_U90KK6KL_Month_2</td>\n",
       "      <td>2</td>\n",
       "      <td>9.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_JRC2</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>29.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>23:59:00</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>28.0</td>\n",
       "      <td>06:00:00</td>\n",
       "      <td>1.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>24.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>GeneralDiscussion</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_ME45XX</td>\n",
       "      <td>ID_U90KK6KL_Month_2</td>\n",
       "      <td>2</td>\n",
       "      <td>9.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_7CF2</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>28.0</td>\n",
       "      <td>YSLJ</td>\n",
       "      <td>23:59:00</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>08:00:00</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>7.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>GeneralDiscussion</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_ME45XX</td>\n",
       "      <td>ID_U90KK6KL_Month_2</td>\n",
       "      <td>2</td>\n",
       "      <td>9.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_45W9</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>28.0</td>\n",
       "      <td>YSLJ</td>\n",
       "      <td>12:00:00</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>09:10:00</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9922</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_B3CQS9BQ_Month_3</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_UCC7</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>15.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>23:59:00</td>\n",
       "      <td>2.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>06:00:00</td>\n",
       "      <td>2.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>28.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9923</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_6RPFZR7I_Month_11</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_CNBR</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>9.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>00:00:00</td>\n",
       "      <td>3.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>00:00:00</td>\n",
       "      <td>3.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>16.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9924</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_YEZY3ABB_Month_1</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_208Z</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>N6EN</td>\n",
       "      <td>23:59:00</td>\n",
       "      <td>3.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>17.0</td>\n",
       "      <td>00:00:00</td>\n",
       "      <td>4.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9925</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_GS03UEEX_Month_4</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_H3QQ</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>17.0</td>\n",
       "      <td>4PRY</td>\n",
       "      <td>00:00:00</td>\n",
       "      <td>3.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>17.0</td>\n",
       "      <td>00:00:00</td>\n",
       "      <td>4.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>18.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9926</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_1CY5JAGU_Month_4</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_7079</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>10.0</td>\n",
       "      <td>G3F0</td>\n",
       "      <td>23:59:00</td>\n",
       "      <td>3.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>06:00:00</td>\n",
       "      <td>3.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>11.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>9927 rows × 32 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         Competition ID  Personal  Theme    Disc_ID               User_ID  \\\n",
       "0     GeneralDiscussion       1.0    NaN  ID_0BK7TH  ID_AAGFVLHH_Month_12   \n",
       "1     GeneralDiscussion       1.0    NaN  ID_ZFJ9J7  ID_0AJGQILU_Month_12   \n",
       "2     GeneralDiscussion       1.0    NaN  ID_ME45XX   ID_U90KK6KL_Month_2   \n",
       "3     GeneralDiscussion       1.0    NaN  ID_ME45XX   ID_U90KK6KL_Month_2   \n",
       "4     GeneralDiscussion       1.0    NaN  ID_ME45XX   ID_U90KK6KL_Month_2   \n",
       "...                 ...       ...    ...        ...                   ...   \n",
       "9922                NaN       NaN    NaN        NaN   ID_B3CQS9BQ_Month_3   \n",
       "9923                NaN       NaN    NaN        NaN  ID_6RPFZR7I_Month_11   \n",
       "9924                NaN       NaN    NaN        NaN   ID_YEZY3ABB_Month_1   \n",
       "9925                NaN       NaN    NaN        NaN   ID_GS03UEEX_Month_4   \n",
       "9926                NaN       NaN    NaN        NaN   ID_1CY5JAGU_Month_4   \n",
       "\n",
       "     Discussion_month  Discussion_day  comment_day  Comp_ID  Participant Type  \\\n",
       "0                  12            27.0          NaN  ID_528W               1.0   \n",
       "1                  12            20.0          NaN      NaN               NaN   \n",
       "2                   2             9.0          NaN  ID_JRC2               1.0   \n",
       "3                   2             9.0          NaN  ID_7CF2               1.0   \n",
       "4                   2             9.0          NaN  ID_45W9               1.0   \n",
       "...               ...             ...          ...      ...               ...   \n",
       "9922             <NA>             NaN          NaN  ID_UCC7               1.0   \n",
       "9923             <NA>             NaN          NaN  ID_CNBR               1.0   \n",
       "9924             <NA>             NaN          NaN  ID_208Z               1.0   \n",
       "9925             <NA>             NaN          NaN  ID_H3QQ               1.0   \n",
       "9926             <NA>             NaN          NaN  ID_7079               1.0   \n",
       "\n",
       "      ... FeatureI Country_ID  End Time time End Time Year End Time Month  \\\n",
       "0     ...     31.0       X9GR       22:59:00           1.0            2.0   \n",
       "1     ...      NaN        NaN            NaN           NaN            NaN   \n",
       "2     ...     29.0        NaN       23:59:00           1.0            3.0   \n",
       "3     ...     28.0       YSLJ       23:59:00           1.0            3.0   \n",
       "4     ...     28.0       YSLJ       12:00:00           1.0            2.0   \n",
       "...   ...      ...        ...            ...           ...            ...   \n",
       "9922  ...     15.0        NaN       23:59:00           2.0            3.0   \n",
       "9923  ...      9.0        NaN       00:00:00           3.0           11.0   \n",
       "9924  ...      NaN       N6EN       23:59:00           3.0           12.0   \n",
       "9925  ...     17.0       4PRY       00:00:00           3.0            9.0   \n",
       "9926  ...     10.0       G3F0       23:59:00           3.0            6.0   \n",
       "\n",
       "      End Time Day_of_month  Start Time time  Start Time Year  \\\n",
       "0                       3.0         22:59:00              1.0   \n",
       "1                       NaN              NaN              NaN   \n",
       "2                      28.0         06:00:00              1.0   \n",
       "3                       7.0         08:00:00              1.0   \n",
       "4                      10.0         09:10:00              1.0   \n",
       "...                     ...              ...              ...   \n",
       "9922                   29.0         06:00:00              2.0   \n",
       "9923                   20.0         00:00:00              3.0   \n",
       "9924                   17.0         00:00:00              4.0   \n",
       "9925                   17.0         00:00:00              4.0   \n",
       "9926                   29.0         06:00:00              3.0   \n",
       "\n",
       "      Start Time Month Start Time Day_of_month  \n",
       "0                 12.0                    28.0  \n",
       "1                  NaN                     NaN  \n",
       "2                 12.0                    24.0  \n",
       "3                  2.0                     7.0  \n",
       "4                  2.0                     9.0  \n",
       "...                ...                     ...  \n",
       "9922              12.0                    28.0  \n",
       "9923               8.0                    16.0  \n",
       "9924               7.0                     2.0  \n",
       "9925               5.0                    18.0  \n",
       "9926               4.0                    11.0  \n",
       "\n",
       "[9927 rows x 32 columns]"
      ]
     },
     "execution_count": 179,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "id": "6d97d41b",
   "metadata": {},
   "outputs": [],
   "source": [
    "result4.rename(columns={'End Time time': 'competition_ended_time'},  inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "id": "144c2644",
   "metadata": {},
   "outputs": [],
   "source": [
    "result4.rename(columns={'End Time Year': 'competition_ended_year'},  inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "id": "a4f6df9d",
   "metadata": {},
   "outputs": [],
   "source": [
    "result4.rename(columns={'End Time Month': 'competition_ended_month'},  inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "id": "8613dfa6",
   "metadata": {},
   "outputs": [],
   "source": [
    "result4.rename(columns={'End Time Day_of_month': 'competition_ended_day'},  inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "id": "5bb8beb7",
   "metadata": {},
   "outputs": [],
   "source": [
    "result4.rename(columns={'Start Time time': 'competition_started_time'},  inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "id": "f1a5cce5",
   "metadata": {},
   "outputs": [],
   "source": [
    "result4.rename(columns={'Start Time Year': 'competition_started_year'},  inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "id": "42e88248",
   "metadata": {},
   "outputs": [],
   "source": [
    "result4.rename(columns={'Start Time Month': 'competition_started_month'},  inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "id": "39d441ea",
   "metadata": {},
   "outputs": [],
   "source": [
    "result4.rename(columns={'Start Time Day_of_month': 'competition_started_day'},  inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "id": "1a03d171",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 9927 entries, 0 to 9926\n",
      "Data columns (total 32 columns):\n",
      " #   Column                       Non-Null Count  Dtype  \n",
      "---  ------                       --------------  -----  \n",
      " 0   Competition ID               2305 non-null   object \n",
      " 1   Personal                     2305 non-null   float64\n",
      " 2   Theme                        272 non-null    float64\n",
      " 3   Disc_ID                      2891 non-null   object \n",
      " 4   User_ID                      9927 non-null   string \n",
      " 5   Discussion_month             2305 non-null   string \n",
      " 6   Discussion_day               2305 non-null   float64\n",
      " 7   comment_day                  708 non-null    float64\n",
      " 8   Comp_ID                      9300 non-null   object \n",
      " 9   Participant Type             9300 non-null   float64\n",
      " 10  Successful Submission Count  1813 non-null   object \n",
      " 11  Created At time              9300 non-null   object \n",
      " 12  Comp_participation_day       9300 non-null   float64\n",
      " 13  FeatureA                     9300 non-null   object \n",
      " 14  FeatureB                     9300 non-null   object \n",
      " 15  FeatureC                     6155 non-null   float64\n",
      " 16  FeatureD                     9300 non-null   float64\n",
      " 17  FeatureE                     9300 non-null   float64\n",
      " 18  SecretCode                   9300 non-null   float64\n",
      " 19  FeatureF                     9300 non-null   object \n",
      " 20  FeatureG                     9300 non-null   object \n",
      " 21  FeatureH                     9300 non-null   object \n",
      " 22  FeatureI                     7964 non-null   float64\n",
      " 23  Country_ID                   4565 non-null   object \n",
      " 24  competition_ended_time       7086 non-null   object \n",
      " 25  competition_ended_year       7086 non-null   float64\n",
      " 26  competition_ended_month      7086 non-null   float64\n",
      " 27  competition_ended_day        7086 non-null   float64\n",
      " 28  competition_started_time     9300 non-null   object \n",
      " 29  competition_started_year     9300 non-null   float64\n",
      " 30  competition_started_month    9300 non-null   float64\n",
      " 31  competition_started_day      9300 non-null   float64\n",
      "dtypes: float64(17), object(13), string(2)\n",
      "memory usage: 2.5+ MB\n"
     ]
    }
   ],
   "source": [
    "result4.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "id": "4935f6c1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Competition ID</th>\n",
       "      <th>Personal</th>\n",
       "      <th>Theme</th>\n",
       "      <th>Disc_ID</th>\n",
       "      <th>User_ID</th>\n",
       "      <th>Discussion_month</th>\n",
       "      <th>Discussion_day</th>\n",
       "      <th>comment_day</th>\n",
       "      <th>Comp_ID</th>\n",
       "      <th>Participant Type</th>\n",
       "      <th>...</th>\n",
       "      <th>FeatureI</th>\n",
       "      <th>Country_ID</th>\n",
       "      <th>competition_ended_time</th>\n",
       "      <th>competition_ended_year</th>\n",
       "      <th>competition_ended_month</th>\n",
       "      <th>competition_ended_day</th>\n",
       "      <th>competition_started_time</th>\n",
       "      <th>competition_started_year</th>\n",
       "      <th>competition_started_month</th>\n",
       "      <th>competition_started_day</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>GeneralDiscussion</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_0BK7TH</td>\n",
       "      <td>ID_AAGFVLHH_Month_12</td>\n",
       "      <td>12</td>\n",
       "      <td>27.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_528W</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>31.0</td>\n",
       "      <td>X9GR</td>\n",
       "      <td>22:59:00</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>22:59:00</td>\n",
       "      <td>1.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>28.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>GeneralDiscussion</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_ZFJ9J7</td>\n",
       "      <td>ID_0AJGQILU_Month_12</td>\n",
       "      <td>12</td>\n",
       "      <td>20.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>GeneralDiscussion</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_ME45XX</td>\n",
       "      <td>ID_U90KK6KL_Month_2</td>\n",
       "      <td>2</td>\n",
       "      <td>9.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_JRC2</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>29.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>23:59:00</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>28.0</td>\n",
       "      <td>06:00:00</td>\n",
       "      <td>1.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>24.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>GeneralDiscussion</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_ME45XX</td>\n",
       "      <td>ID_U90KK6KL_Month_2</td>\n",
       "      <td>2</td>\n",
       "      <td>9.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_7CF2</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>28.0</td>\n",
       "      <td>YSLJ</td>\n",
       "      <td>23:59:00</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>08:00:00</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>7.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>GeneralDiscussion</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_ME45XX</td>\n",
       "      <td>ID_U90KK6KL_Month_2</td>\n",
       "      <td>2</td>\n",
       "      <td>9.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_45W9</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>28.0</td>\n",
       "      <td>YSLJ</td>\n",
       "      <td>12:00:00</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>09:10:00</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9922</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_B3CQS9BQ_Month_3</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_UCC7</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>15.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>23:59:00</td>\n",
       "      <td>2.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>06:00:00</td>\n",
       "      <td>2.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>28.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9923</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_6RPFZR7I_Month_11</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_CNBR</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>9.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>00:00:00</td>\n",
       "      <td>3.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>00:00:00</td>\n",
       "      <td>3.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>16.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9924</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_YEZY3ABB_Month_1</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_208Z</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>N6EN</td>\n",
       "      <td>23:59:00</td>\n",
       "      <td>3.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>17.0</td>\n",
       "      <td>00:00:00</td>\n",
       "      <td>4.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9925</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_GS03UEEX_Month_4</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_H3QQ</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>17.0</td>\n",
       "      <td>4PRY</td>\n",
       "      <td>00:00:00</td>\n",
       "      <td>3.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>17.0</td>\n",
       "      <td>00:00:00</td>\n",
       "      <td>4.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>18.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9926</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_1CY5JAGU_Month_4</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ID_7079</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>10.0</td>\n",
       "      <td>G3F0</td>\n",
       "      <td>23:59:00</td>\n",
       "      <td>3.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>06:00:00</td>\n",
       "      <td>3.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>11.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>9927 rows × 32 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         Competition ID  Personal  Theme    Disc_ID               User_ID  \\\n",
       "0     GeneralDiscussion       1.0    NaN  ID_0BK7TH  ID_AAGFVLHH_Month_12   \n",
       "1     GeneralDiscussion       1.0    NaN  ID_ZFJ9J7  ID_0AJGQILU_Month_12   \n",
       "2     GeneralDiscussion       1.0    NaN  ID_ME45XX   ID_U90KK6KL_Month_2   \n",
       "3     GeneralDiscussion       1.0    NaN  ID_ME45XX   ID_U90KK6KL_Month_2   \n",
       "4     GeneralDiscussion       1.0    NaN  ID_ME45XX   ID_U90KK6KL_Month_2   \n",
       "...                 ...       ...    ...        ...                   ...   \n",
       "9922                NaN       NaN    NaN        NaN   ID_B3CQS9BQ_Month_3   \n",
       "9923                NaN       NaN    NaN        NaN  ID_6RPFZR7I_Month_11   \n",
       "9924                NaN       NaN    NaN        NaN   ID_YEZY3ABB_Month_1   \n",
       "9925                NaN       NaN    NaN        NaN   ID_GS03UEEX_Month_4   \n",
       "9926                NaN       NaN    NaN        NaN   ID_1CY5JAGU_Month_4   \n",
       "\n",
       "     Discussion_month  Discussion_day  comment_day  Comp_ID  Participant Type  \\\n",
       "0                  12            27.0          NaN  ID_528W               1.0   \n",
       "1                  12            20.0          NaN      NaN               NaN   \n",
       "2                   2             9.0          NaN  ID_JRC2               1.0   \n",
       "3                   2             9.0          NaN  ID_7CF2               1.0   \n",
       "4                   2             9.0          NaN  ID_45W9               1.0   \n",
       "...               ...             ...          ...      ...               ...   \n",
       "9922             <NA>             NaN          NaN  ID_UCC7               1.0   \n",
       "9923             <NA>             NaN          NaN  ID_CNBR               1.0   \n",
       "9924             <NA>             NaN          NaN  ID_208Z               1.0   \n",
       "9925             <NA>             NaN          NaN  ID_H3QQ               1.0   \n",
       "9926             <NA>             NaN          NaN  ID_7079               1.0   \n",
       "\n",
       "      ... FeatureI Country_ID  competition_ended_time competition_ended_year  \\\n",
       "0     ...     31.0       X9GR                22:59:00                    1.0   \n",
       "1     ...      NaN        NaN                     NaN                    NaN   \n",
       "2     ...     29.0        NaN                23:59:00                    1.0   \n",
       "3     ...     28.0       YSLJ                23:59:00                    1.0   \n",
       "4     ...     28.0       YSLJ                12:00:00                    1.0   \n",
       "...   ...      ...        ...                     ...                    ...   \n",
       "9922  ...     15.0        NaN                23:59:00                    2.0   \n",
       "9923  ...      9.0        NaN                00:00:00                    3.0   \n",
       "9924  ...      NaN       N6EN                23:59:00                    3.0   \n",
       "9925  ...     17.0       4PRY                00:00:00                    3.0   \n",
       "9926  ...     10.0       G3F0                23:59:00                    3.0   \n",
       "\n",
       "     competition_ended_month  competition_ended_day  competition_started_time  \\\n",
       "0                        2.0                    3.0                  22:59:00   \n",
       "1                        NaN                    NaN                       NaN   \n",
       "2                        3.0                   28.0                  06:00:00   \n",
       "3                        3.0                    7.0                  08:00:00   \n",
       "4                        2.0                   10.0                  09:10:00   \n",
       "...                      ...                    ...                       ...   \n",
       "9922                     3.0                   29.0                  06:00:00   \n",
       "9923                    11.0                   20.0                  00:00:00   \n",
       "9924                    12.0                   17.0                  00:00:00   \n",
       "9925                     9.0                   17.0                  00:00:00   \n",
       "9926                     6.0                   29.0                  06:00:00   \n",
       "\n",
       "      competition_started_year  competition_started_month  \\\n",
       "0                          1.0                       12.0   \n",
       "1                          NaN                        NaN   \n",
       "2                          1.0                       12.0   \n",
       "3                          1.0                        2.0   \n",
       "4                          1.0                        2.0   \n",
       "...                        ...                        ...   \n",
       "9922                       2.0                       12.0   \n",
       "9923                       3.0                        8.0   \n",
       "9924                       4.0                        7.0   \n",
       "9925                       4.0                        5.0   \n",
       "9926                       3.0                        4.0   \n",
       "\n",
       "     competition_started_day  \n",
       "0                       28.0  \n",
       "1                        NaN  \n",
       "2                       24.0  \n",
       "3                        7.0  \n",
       "4                        9.0  \n",
       "...                      ...  \n",
       "9922                    28.0  \n",
       "9923                    16.0  \n",
       "9924                     2.0  \n",
       "9925                    18.0  \n",
       "9926                    11.0  \n",
       "\n",
       "[9927 rows x 32 columns]"
      ]
     },
     "execution_count": 189,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result4"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9fce6837",
   "metadata": {},
   "source": [
    "# MERGE 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "id": "96c1ecbc",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_final = pd.merge(result4, result3, how=\"outer\", on= 'User_ID')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "id": "00f20228",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_final.drop('Competition ID', inplace=True, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 192,
   "id": "9c523d52",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_final.drop('Disc_ID', inplace=True, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "id": "73eeae88",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_final.drop('Comp_ID', inplace=True, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 194,
   "id": "dc2bab79",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['X9GR', nan, 'YSLJ', 'G3F0', 'GYYX', '03GB', 'ZEWU', 'DYYF',\n",
       "       'KYT4', 'ISIQ', 'N6EN', 'EGI9', 'H03V', 'SE4K', '4PRY'],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 194,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_final[\"Country_ID\"].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 195,
   "id": "a9fba33d",
   "metadata": {},
   "outputs": [],
   "source": [
    "first_column = df_final.pop('User_ID')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 196,
   "id": "5bbeaa82",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_final.insert(0, 'User_ID', first_column)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 197,
   "id": "06cbe88c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 20998 entries, 0 to 20997\n",
      "Data columns (total 91 columns):\n",
      " #   Column                           Non-Null Count  Dtype  \n",
      "---  ------                           --------------  -----  \n",
      " 0   User_ID                          20998 non-null  object \n",
      " 1   Personal                         2305 non-null   float64\n",
      " 2   Theme                            272 non-null    float64\n",
      " 3   Discussion_month                 2305 non-null   string \n",
      " 4   Discussion_day                   2305 non-null   float64\n",
      " 5   comment_day                      708 non-null    float64\n",
      " 6   Participant Type                 9300 non-null   float64\n",
      " 7   Successful Submission Count      1813 non-null   object \n",
      " 8   Created At time                  9300 non-null   object \n",
      " 9   Comp_participation_day           9300 non-null   float64\n",
      " 10  FeatureA                         9300 non-null   object \n",
      " 11  FeatureB                         9300 non-null   object \n",
      " 12  FeatureC                         6155 non-null   float64\n",
      " 13  FeatureD                         9300 non-null   float64\n",
      " 14  FeatureE                         9300 non-null   float64\n",
      " 15  SecretCode                       9300 non-null   float64\n",
      " 16  FeatureF                         9300 non-null   object \n",
      " 17  FeatureG                         9300 non-null   object \n",
      " 18  FeatureH                         9300 non-null   object \n",
      " 19  FeatureI                         7964 non-null   float64\n",
      " 20  Country_ID                       4565 non-null   object \n",
      " 21  competition_ended_time           7086 non-null   object \n",
      " 22  competition_ended_year           7086 non-null   float64\n",
      " 23  competition_ended_month          7086 non-null   float64\n",
      " 24  competition_ended_day            7086 non-null   float64\n",
      " 25  competition_started_time         9300 non-null   object \n",
      " 26  competition_started_year         9300 non-null   float64\n",
      " 27  competition_started_month        9300 non-null   float64\n",
      " 28  competition_started_day          9300 non-null   float64\n",
      " 29  datetime Day_of_month            18014 non-null  float64\n",
      " 30  Blog Theme                       1847 non-null   float64\n",
      " 31  Blog Published Year              1847 non-null   float64\n",
      " 32  Blog Published Month             1847 non-null   float64\n",
      " 33  Published At Day_of_month        1847 non-null   float64\n",
      " 34  Remote                           18014 non-null  float64\n",
      " 35  Experience                       1049 non-null   float64\n",
      " 36  Employment Type                  18014 non-null  float64\n",
      " 37  Company Size                     1037 non-null   float64\n",
      " 38  Downloaded_Competition_Datafile  18014 non-null  float64\n",
      " 39  Viewed_All_Discussions           18014 non-null  float64\n",
      " 40  Confirmed_Email                  18014 non-null  float64\n",
      " 41  Joined_Competition               18014 non-null  float64\n",
      " 42  Signed_Up                        18014 non-null  float64\n",
      " 43  Viewed_All_Competitions          18014 non-null  float64\n",
      " 44  Viewed_All_Learning_Pages        18014 non-null  float64\n",
      " 45  badge_OCZE                       18014 non-null  float64\n",
      " 46  create_alias                     18014 non-null  float64\n",
      " 47  identify                         18014 non-null  float64\n",
      " 48  comp_ID                          18014 non-null  float64\n",
      " 49  Signed_In                        18014 non-null  float64\n",
      " 50  job_ID                           18014 non-null  float64\n",
      " 51  Updated_Discussion               18014 non-null  float64\n",
      " 52  Viewed_All_Jobs                  18014 non-null  float64\n",
      " 53  Created_Submission               18014 non-null  float64\n",
      " 54  Signed_Out                       18014 non-null  float64\n",
      " 55  Updated_Profile                  18014 non-null  float64\n",
      " 56  blog_ID                          18014 non-null  float64\n",
      " 57  Viewed_Discussion                18014 non-null  float64\n",
      " 58  Updated_Submission               18014 non-null  float64\n",
      " 59  badge_HYIO                       18014 non-null  float64\n",
      " 60  Industry_FS                      18014 non-null  float64\n",
      " 61  Industry_Bank                    18014 non-null  float64\n",
      " 62  Industry_Loc                     18014 non-null  float64\n",
      " 63  Industry_Gov                     18014 non-null  float64\n",
      " 64  Industry_Health                  18014 non-null  float64\n",
      " 65  Industry_Res                     18014 non-null  float64\n",
      " 66  Industry_Amb                     18014 non-null  float64\n",
      " 67  Industry_Transp                  18014 non-null  float64\n",
      " 68  Industry_Scholar                 18014 non-null  float64\n",
      " 69  Industry_Log                     18014 non-null  float64\n",
      " 70  Industry_Media                   18014 non-null  float64\n",
      " 71  Industry_Manuf                   18014 non-null  float64\n",
      " 72  Industry_Sales                   18014 non-null  float64\n",
      " 73  Industry_Insur                   18014 non-null  float64\n",
      " 74  Industry_Manage                  18014 non-null  float64\n",
      " 75  Industry_AI                      18014 non-null  float64\n",
      " 76  Industry_Zindi                   18014 non-null  float64\n",
      " 77  Industry_Tech                    18014 non-null  float64\n",
      " 78  Function_DE                      18014 non-null  float64\n",
      " 79  Function_DM                      18014 non-null  float64\n",
      " 80  Function_Bus                     18014 non-null  float64\n",
      " 81  Function_AI                      18014 non-null  float64\n",
      " 82  Function_ML                      18014 non-null  float64\n",
      " 83  count_UA                         18014 non-null  float64\n",
      " 84  Day duration                     18014 non-null  float64\n",
      " 85  FeatureX                         14729 non-null  float64\n",
      " 86  FeatureY                         14729 non-null  float64\n",
      " 87  Countries_ID                     7807 non-null   object \n",
      " 88  User sign-in time                14729 non-null  object \n",
      " 89  User sign-in month               14729 non-null  string \n",
      " 90  User sign-in day                 14729 non-null  float64\n",
      "dtypes: float64(76), object(13), string(2)\n",
      "memory usage: 14.7+ MB\n"
     ]
    }
   ],
   "source": [
    "df_final.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 198,
   "id": "217f2886",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>User_ID</th>\n",
       "      <th>Personal</th>\n",
       "      <th>Theme</th>\n",
       "      <th>Discussion_month</th>\n",
       "      <th>Discussion_day</th>\n",
       "      <th>comment_day</th>\n",
       "      <th>Participant Type</th>\n",
       "      <th>Successful Submission Count</th>\n",
       "      <th>Created At time</th>\n",
       "      <th>Comp_participation_day</th>\n",
       "      <th>...</th>\n",
       "      <th>Function_AI</th>\n",
       "      <th>Function_ML</th>\n",
       "      <th>count_UA</th>\n",
       "      <th>Day duration</th>\n",
       "      <th>FeatureX</th>\n",
       "      <th>FeatureY</th>\n",
       "      <th>Countries_ID</th>\n",
       "      <th>User sign-in time</th>\n",
       "      <th>User sign-in month</th>\n",
       "      <th>User sign-in day</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ID_AAGFVLHH_Month_12</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>12</td>\n",
       "      <td>27.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>10:53:52.558030</td>\n",
       "      <td>28.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>35.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>X9GR</td>\n",
       "      <td>08:26:28.847530</td>\n",
       "      <td>12</td>\n",
       "      <td>20.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ID_0AJGQILU_Month_12</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>12</td>\n",
       "      <td>20.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>09:28:59.699371</td>\n",
       "      <td>12</td>\n",
       "      <td>20.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ID_U90KK6KL_Month_2</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2</td>\n",
       "      <td>9.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>18:57:59.998938</td>\n",
       "      <td>12.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>95.0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>YSLJ</td>\n",
       "      <td>15:29:09.389920</td>\n",
       "      <td>2</td>\n",
       "      <td>8.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ID_U90KK6KL_Month_2</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2</td>\n",
       "      <td>9.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>18:49:47.531900</td>\n",
       "      <td>12.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>95.0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>YSLJ</td>\n",
       "      <td>15:29:09.389920</td>\n",
       "      <td>2</td>\n",
       "      <td>8.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ID_U90KK6KL_Month_2</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2</td>\n",
       "      <td>9.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>15:34:35.872471</td>\n",
       "      <td>8.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>95.0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>YSLJ</td>\n",
       "      <td>15:29:09.389920</td>\n",
       "      <td>2</td>\n",
       "      <td>8.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20993</th>\n",
       "      <td>ID_Y02FKZWI_Month_3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7JT1</td>\n",
       "      <td>04:04:35.100180</td>\n",
       "      <td>3</td>\n",
       "      <td>26.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20994</th>\n",
       "      <td>ID_FDOGWALS_Month_11</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>UKC2</td>\n",
       "      <td>13:30:49.814791</td>\n",
       "      <td>11</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20995</th>\n",
       "      <td>ID_YJCY8FF5_Month_5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>H801</td>\n",
       "      <td>15:54:12.676400</td>\n",
       "      <td>5</td>\n",
       "      <td>11.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20996</th>\n",
       "      <td>ID_QDL8A9FQ_Month_4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NRDP</td>\n",
       "      <td>09:49:35.535847</td>\n",
       "      <td>4</td>\n",
       "      <td>28.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20997</th>\n",
       "      <td>ID_C5393PD0_Month_11</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>7DKX</td>\n",
       "      <td>05:02:39.352427</td>\n",
       "      <td>11</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>20998 rows × 91 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                    User_ID  Personal  Theme Discussion_month  Discussion_day  \\\n",
       "0      ID_AAGFVLHH_Month_12       1.0    NaN               12            27.0   \n",
       "1      ID_0AJGQILU_Month_12       1.0    NaN               12            20.0   \n",
       "2       ID_U90KK6KL_Month_2       1.0    NaN                2             9.0   \n",
       "3       ID_U90KK6KL_Month_2       1.0    NaN                2             9.0   \n",
       "4       ID_U90KK6KL_Month_2       1.0    NaN                2             9.0   \n",
       "...                     ...       ...    ...              ...             ...   \n",
       "20993   ID_Y02FKZWI_Month_3       NaN    NaN             <NA>             NaN   \n",
       "20994  ID_FDOGWALS_Month_11       NaN    NaN             <NA>             NaN   \n",
       "20995   ID_YJCY8FF5_Month_5       NaN    NaN             <NA>             NaN   \n",
       "20996   ID_QDL8A9FQ_Month_4       NaN    NaN             <NA>             NaN   \n",
       "20997  ID_C5393PD0_Month_11       NaN    NaN             <NA>             NaN   \n",
       "\n",
       "       comment_day  Participant Type Successful Submission Count  \\\n",
       "0              NaN               1.0                         NaN   \n",
       "1              NaN               NaN                         NaN   \n",
       "2              NaN               1.0                         NaN   \n",
       "3              NaN               1.0                         NaN   \n",
       "4              NaN               1.0                         NaN   \n",
       "...            ...               ...                         ...   \n",
       "20993          NaN               NaN                         NaN   \n",
       "20994          NaN               NaN                         NaN   \n",
       "20995          NaN               NaN                         NaN   \n",
       "20996          NaN               NaN                         NaN   \n",
       "20997          NaN               NaN                         NaN   \n",
       "\n",
       "       Created At time  Comp_participation_day  ... Function_AI Function_ML  \\\n",
       "0      10:53:52.558030                    28.0  ...         0.0         0.0   \n",
       "1                  NaN                     NaN  ...         0.0         0.0   \n",
       "2      18:57:59.998938                    12.0  ...         0.0         0.0   \n",
       "3      18:49:47.531900                    12.0  ...         0.0         0.0   \n",
       "4      15:34:35.872471                     8.0  ...         0.0         0.0   \n",
       "...                ...                     ...  ...         ...         ...   \n",
       "20993              NaN                     NaN  ...         NaN         NaN   \n",
       "20994              NaN                     NaN  ...         NaN         NaN   \n",
       "20995              NaN                     NaN  ...         NaN         NaN   \n",
       "20996              NaN                     NaN  ...         NaN         NaN   \n",
       "20997              NaN                     NaN  ...         NaN         NaN   \n",
       "\n",
       "       count_UA  Day duration  FeatureX  FeatureY Countries_ID  \\\n",
       "0          35.0           8.0       0.0       0.0         X9GR   \n",
       "1          14.0           0.0       0.0       0.0          NaN   \n",
       "2          95.0          16.0       1.0       1.0         YSLJ   \n",
       "3          95.0          16.0       1.0       1.0         YSLJ   \n",
       "4          95.0          16.0       1.0       1.0         YSLJ   \n",
       "...         ...           ...       ...       ...          ...   \n",
       "20993       NaN           NaN       0.0       0.0         7JT1   \n",
       "20994       NaN           NaN       0.0       0.0         UKC2   \n",
       "20995       NaN           NaN       0.0       0.0         H801   \n",
       "20996       NaN           NaN       0.0       0.0         NRDP   \n",
       "20997       NaN           NaN       0.0       1.0         7DKX   \n",
       "\n",
       "      User sign-in time User sign-in month  User sign-in day  \n",
       "0       08:26:28.847530                 12              20.0  \n",
       "1       09:28:59.699371                 12              20.0  \n",
       "2       15:29:09.389920                  2               8.0  \n",
       "3       15:29:09.389920                  2               8.0  \n",
       "4       15:29:09.389920                  2               8.0  \n",
       "...                 ...                ...               ...  \n",
       "20993   04:04:35.100180                  3              26.0  \n",
       "20994   13:30:49.814791                 11               5.0  \n",
       "20995   15:54:12.676400                  5              11.0  \n",
       "20996   09:49:35.535847                  4              28.0  \n",
       "20997   05:02:39.352427                 11               6.0  \n",
       "\n",
       "[20998 rows x 91 columns]"
      ]
     },
     "execution_count": 198,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_final"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d7dd7b76",
   "metadata": {},
   "source": [
    "# DATA CLEANING AND TRANSFORMATION"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 199,
   "id": "edcc2b9b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(20998, 91)"
      ]
     },
     "execution_count": 199,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_final.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 200,
   "id": "d336d32d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 1.,  0., nan])"
      ]
     },
     "execution_count": 200,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_final[\"Personal\"].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 201,
   "id": "243dfe8e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 1.,  0., -1.])"
      ]
     },
     "execution_count": 201,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_final['Personal'].fillna(-1, inplace = True)\n",
    "df_final[\"Personal\"].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 202,
   "id": "709fae56",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([nan,  3.,  1.,  4.,  5.,  2.])"
      ]
     },
     "execution_count": 202,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_final[\"Theme\"].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 203,
   "id": "a2fc4bd7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0., 3., 1., 4., 5., 2.])"
      ]
     },
     "execution_count": 203,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_final['Theme'].fillna(0, inplace = True)\n",
    "df_final[\"Theme\"].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 204,
   "id": "e237b32e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_final['Discussion Hour'].isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 205,
   "id": "257a7da1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_final['Discussion Hour'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 206,
   "id": "75b18b7e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_final['Discussion Hour'].fillna('-1', inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 207,
   "id": "bef171b5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_final['Discussion Minute'].isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 208,
   "id": "5528f0eb",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_final['Discussion Minute'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 209,
   "id": "3cc49b72",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_final['Discussion Minute'].fillna('-1', inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 210,
   "id": "3a6d67af",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_final['Discussion Second'].isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 211,
   "id": "49956586",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_final['Discussion Second'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 212,
   "id": "0339be7e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_final['Discussion Second'].fillna('-1', inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 213,
   "id": "14034432",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_final['Discussion_day'].fillna('0', inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 214,
   "id": "872da0d3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_final['Comment Hour'].isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 215,
   "id": "7a7ed74a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_final['Comment Hour'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 216,
   "id": "90a2684d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_final['Comment Hour'].fillna('-1', inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 217,
   "id": "fd2d288b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_final['Comment Minute'].isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 218,
   "id": "2eaefc3a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_final['Comment Minute'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 219,
   "id": "db33e421",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_final['Comment Minute'].fillna('-1', inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 220,
   "id": "e971265a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_final['Comment Second'].isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 221,
   "id": "050d7b26",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_final['Comment Second'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 222,
   "id": "98dca0f5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_final['Comment Second'].fillna('-1', inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 223,
   "id": "bf422c98",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([nan, 11.,  3.,  5., 23., 19., 20., 14., 29.,  1., 16.,  9., 13.,\n",
       "       28.,  7., 24., 21., 15., 12., 17., 26., 25., 22., 18., 31.,  6.,\n",
       "        8.])"
      ]
     },
     "execution_count": 223,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_final['comment_day'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 224,
   "id": "548840b5",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_final['comment_day'].fillna(0, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 225,
   "id": "8e00f64b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "11698"
      ]
     },
     "execution_count": 225,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_final['Participant Type'].isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 226,
   "id": "19c7bc29",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 1., nan])"
      ]
     },
     "execution_count": 226,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_final[\"Participant Type\"].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 227,
   "id": "33cf336e",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_final['Participant Type'].fillna(0, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 228,
   "id": "999010a6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([nan, 'count 10', 'count 8', 'count 7', 'count 9', 'count 6',\n",
       "       'count 3', 'count 5', 'count 4'], dtype=object)"
      ]
     },
     "execution_count": 228,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_final[\"Successful Submission Count\"].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 229,
   "id": "fc5214b6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "19185"
      ]
     },
     "execution_count": 229,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_final['Successful Submission Count'].isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 230,
   "id": "53b6a4b4",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_final['Successful Submission Count'] = df_final['Successful Submission Count'].str[6:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 231,
   "id": "044e0032",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([nan, '10', '8', '7', '9', '6', '3', '5', '4'], dtype=object)"
      ]
     },
     "execution_count": 231,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_final['Successful Submission Count'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 232,
   "id": "5b34c507",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_final['Successful Submission Count'].fillna(0, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 233,
   "id": "60a0d8cb",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_final['competition participation hour'] = df_final['Comp_participation_time'].str[0:2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 234,
   "id": "8f9c9af7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_final['competition participation minute'] = df_final['Comp_participation_time'].str[3:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 235,
   "id": "cb2a9f24",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_final['competition participation second'] = df_final['Comp_participation_time'].str[6:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 236,
   "id": "0cc19cda",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_final['competition participation hour'].isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 237,
   "id": "2d3b3bd8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_final['competition participation hour'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 238,
   "id": "eca361a8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_final['competition participation hour'].fillna(-1, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 239,
   "id": "dcbd0e8b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_final['competition participation minute'].isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 240,
   "id": "d5950cbe",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_final['competition participation minute'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 241,
   "id": "344d38be",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_final['competition participation minute'].fillna(-1, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 242,
   "id": "53c75bce",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_final['competition participation second'].isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 243,
   "id": "31838a58",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_final['competition participation second'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 244,
   "id": "a3807f2b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_final['competition participation second'].fillna(-1, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 245,
   "id": "389ec48b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_final.drop('Comp_participation_time', axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 246,
   "id": "cfb08778",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([28., nan, 12.,  8., 17., 19.,  9., 29., 11.,  7.,  1.,  2., 30.,\n",
       "       13., 21., 31.,  3., 10., 20., 26., 27., 16., 14., 25., 23.,  4.,\n",
       "        5., 15., 18., 24., 22.,  6.])"
      ]
     },
     "execution_count": 246,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_final['Comp_participation_day'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 247,
   "id": "2c878b97",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_final['Comp_participation_day'].fillna(0, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 248,
   "id": "9d9d90d8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "11698"
      ]
     },
     "execution_count": 248,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_final['FeatureA'].isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 249,
   "id": "fa424d3e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['[]', nan, '[1]', '[2]', '[1, 7]', '[5]', '[4]', '[1, 6]',\n",
       "       '[2, 4]', '[7]', '[3]', '[3, 2]', '[4, 2]', '[3, 7]', '[2, 3]',\n",
       "       '[4, 7, 2]', '[1, 9]', '[2, 4, 5]', '[5, 4]', '[5, 2]',\n",
       "       '[1, 4, 5]', '[8]', '[2, 7]', '[1, 5, 4]', '[7, 3, 2]'],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 249,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_final['FeatureA'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 250,
   "id": "173a1668",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_final['FeatureA'].fillna('[-1]', inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 251,
   "id": "0f61033d",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_final['FeatureA'] = df_final.FeatureA.map(ast.literal_eval)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 252,
   "id": "e9969826",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 252,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_final['FeatureA'].isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 253,
   "id": "2aa56679",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['[]', nan, '[5]', '[15]', '[9]', '[16]', '[14]', '[7, 5]', '[7]',\n",
       "       '[2]', '[1]', '[5, 6]', '[6]', '[12, 16]', '[10]', '[12]',\n",
       "       '[5, 7]', '[14, 7]', '[14, 8]'], dtype=object)"
      ]
     },
     "execution_count": 253,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_final['FeatureB'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 254,
   "id": "0d2e61c4",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_final['FeatureB'].fillna('[-1]', inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 255,
   "id": "1a28ef18",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_final['FeatureB'] = df_final.FeatureB.map(ast.literal_eval)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 256,
   "id": "8c6dd234",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 256,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_final['FeatureB'].isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 257,
   "id": "be9e405c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 257,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_final['FeatureB'].isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 258,
   "id": "7f1959ba",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([nan, 28.,  4.,  2., 10.,  5., 15., 29.,  1.,  6., 11., 13.,  7.,\n",
       "       24., 14., 35.,  8.,  9., 31.,  3., 16., 25., 27., 20., 26., 37.,\n",
       "       12., 18., 17.])"
      ]
     },
     "execution_count": 258,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_final['FeatureC'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 259,
   "id": "95be403d",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_final['FeatureC'].fillna(-1, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 260,
   "id": "5bf0c2a7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0., nan,  1.])"
      ]
     },
     "execution_count": 260,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_final['FeatureD'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 261,
   "id": "c8165638",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_final['FeatureD'].fillna(-1, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 262,
   "id": "bea84a69",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 3., nan,  2.,  1.])"
      ]
     },
     "execution_count": 262,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_final['FeatureE'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 263,
   "id": "14e47be1",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_final['FeatureE'].fillna(-1, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 264,
   "id": "e86ce0a6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['[]', nan, '[7]', '[10]', '[1]', '[2]', '[2, 3]', '[3]', '[2, 8]',\n",
       "       '[15]', '[6]', '[3, 2]', '[1, 5]', '[9]', '[2, 11]', '[1, 12]',\n",
       "       '[1, 3]', '[2, 13]', '[3, 1]', '[2, 1]', '[8]', '[1, 11]'],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 264,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_final['FeatureF'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 265,
   "id": "7ff2c5fe",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_final['FeatureF'].fillna('[-1]', inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 266,
   "id": "0ee10e97",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_final['FeatureF'] = df_final.FeatureF.map(ast.literal_eval)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 267,
   "id": "ac1bc586",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 267,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_final['FeatureF'].isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 268,
   "id": "5854f711",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 268,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_final['FeatureF'].isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 269,
   "id": "06c6d03b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['[5]', nan, '[4]', '[3]', '[]', '[5, 3, 4]'], dtype=object)"
      ]
     },
     "execution_count": 269,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_final['FeatureG'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 270,
   "id": "d0abe41a",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_final['FeatureG'].fillna('[-1]', inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 271,
   "id": "854a4de3",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_final['FeatureG'] = df_final.FeatureG.map(ast.literal_eval)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 272,
   "id": "7f0db25a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 272,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_final['FeatureG'].isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 273,
   "id": "2c29cfbc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 273,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_final['FeatureG'].isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 274,
   "id": "78cc5f1e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['[1]', nan, '[]', '[10]', '[2]', '[3, 2]', '[2, 3]', '[3]', '[6]',\n",
       "       '[7]', '[8, 10]', '[9]', '[1, 3]', '[2, 8]', '[3, 1]', '[2, 1]',\n",
       "       '[8]'], dtype=object)"
      ]
     },
     "execution_count": 274,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_final['FeatureH'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 275,
   "id": "a9f33892",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_final['FeatureH'].fillna('[-1]', inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 276,
   "id": "eacb0bd5",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_final['FeatureH'] = df_final.FeatureH.map(ast.literal_eval)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 277,
   "id": "997ceac0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 277,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_final['FeatureH'].isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 278,
   "id": "c866bdb3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 278,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_final['FeatureH'].isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 279,
   "id": "c8099462",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([31., nan, 29., 28.,  4.,  2., 10.,  5., 15.,  1.,  6., 33., 11.,\n",
       "       13.,  7., 24., 14., 35.,  8.,  9.,  3., 16., 25., 27., 20., 26.,\n",
       "       37., 12., 18., 17.])"
      ]
     },
     "execution_count": 279,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_final['FeatureI'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 280,
   "id": "452eaaa3",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_final['FeatureI'].fillna(-1, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 281,
   "id": "1626e422",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.,  1., nan])"
      ]
     },
     "execution_count": 281,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_final['FeatureX'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 282,
   "id": "b1eb2904",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_final['FeatureX'].fillna(-1, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 283,
   "id": "78bcc98f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.,  1., nan,  3.])"
      ]
     },
     "execution_count": 283,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_final['FeatureY'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 284,
   "id": "f2c85e50",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_final['FeatureY'].fillna(-1, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 285,
   "id": "8d810851",
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(len(df_final)):\n",
    "    arr = df_final.loc[i, \"FeatureA\"]\n",
    "    mean = np.mean(arr)\n",
    "    df_final.loc[i, \"FeatureA\"] = mean\n",
    "    arr = df_final.loc[i, \"FeatureB\"]\n",
    "    mean = np.mean(arr)\n",
    "    df_final.loc[i, \"FeatureB\"] = mean\n",
    "    arr = df_final.loc[i, \"FeatureF\"]\n",
    "    mean = np.mean(arr)\n",
    "    df_final.loc[i, \"FeatureF\"] = mean\n",
    "    arr = df_final.loc[i, \"FeatureG\"]\n",
    "    mean = np.mean(arr)\n",
    "    df_final.loc[i, \"FeatureG\"] = mean\n",
    "    arr = df_final.loc[i, \"FeatureH\"]\n",
    "    mean = np.mean(arr)\n",
    "    df_final.loc[i, \"FeatureH\"] = mean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 286,
   "id": "5838bd20",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_final['FeatureA'].fillna(0, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 287,
   "id": "d32362c9",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_final['FeatureB'].fillna(0, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 288,
   "id": "842fb6be",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_final['FeatureF'].fillna(0, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 289,
   "id": "5c7d1b31",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_final['FeatureG'].fillna(0, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 290,
   "id": "4e3da499",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_final['FeatureH'].fillna(0, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 291,
   "id": "4c6cec9c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "11698"
      ]
     },
     "execution_count": 291,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_final['SecretCode'].isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 292,
   "id": "c1e26e24",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 1., nan,  0.])"
      ]
     },
     "execution_count": 292,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_final['SecretCode'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 293,
   "id": "ba7de0e4",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_final['SecretCode'].fillna(-1, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 294,
   "id": "28d4770a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['22:59:00', nan, '23:59:00', '12:00:00', '11:59:00', '15:00:00',\n",
       "       '16:00:00', '06:00:00', '23:09:00', '00:00:00', '07:00:00',\n",
       "       '05:59:00', '14:00:00', '23:30:00', '11:00:00', '02:00:00',\n",
       "       '18:00:00', '09:30:00'], dtype=object)"
      ]
     },
     "execution_count": 294,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_final['competition_ended_time'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 295,
   "id": "a5c5f5ea",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_final['competition ended hour'] = df_final['competition_ended_time'].str[0:2]\n",
    "# df_final['competition ended minute'] = df_final['competition_ended_time'].str[3:5]\n",
    "# df_final['competition ended second'] = df_final['competition_ended_time'].str[6:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 296,
   "id": "2573f1dd",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_final['competition ended hour'].isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 297,
   "id": "d61e3dfe",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_final['competition ended hour'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 298,
   "id": "03f5bc99",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_final['competition ended hour'].fillna(-1, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 299,
   "id": "27abb6b4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_final['competition ended minute'].isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 300,
   "id": "b0dc6339",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_final['competition ended minute'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 301,
   "id": "9dc923f1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_final['competition ended minute'].fillna(-1, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 302,
   "id": "5149e6b7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_final['competition ended second'].isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 303,
   "id": "1daf976d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_final['competition ended second'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 304,
   "id": "8d6e8419",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_final['competition ended second'].fillna(-1, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 305,
   "id": "cd7dae07",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_final.drop('competition_ended_time', axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 306,
   "id": "8dc88c60",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1.000e+00,       nan, 2.023e+03, 4.000e+00, 3.000e+00, 2.000e+00,\n",
       "       5.000e+00])"
      ]
     },
     "execution_count": 306,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_final['competition_ended_year'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 307,
   "id": "c55ac0fc",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_final['competition_ended_year'].fillna(0, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 308,
   "id": "b2811b01",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 2., nan,  3.,  5.,  7.,  1., 10.,  4.,  9.,  6.,  8., 12., 11.])"
      ]
     },
     "execution_count": 308,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_final['competition_ended_month'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 309,
   "id": "fa2b466f",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_final['competition_ended_month'].fillna(0, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 310,
   "id": "2ceca791",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 3., nan, 28.,  7., 10.,  2.,  8., 26., 31.,  4.,  1., 24.,  5.,\n",
       "       14., 22., 23., 18., 30., 25., 12., 13., 19., 29., 21.,  6., 27.,\n",
       "       17., 15.,  9., 16., 20.])"
      ]
     },
     "execution_count": 310,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_final['competition_ended_day'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 311,
   "id": "09832490",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_final['competition_ended_day'].fillna(0, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 312,
   "id": "22c426e7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['22:59:00', nan, '06:00:00', '08:00:00', '09:10:00', '02:00:00',\n",
       "       '07:00:00', '11:00:00', '05:00:00', '00:00:00', '05:13:00',\n",
       "       '23:00:00', '10:00:00', '23:59:00', '09:00:00', '00:01:00',\n",
       "       '10:30:00', '14:00:00', '12:00:00', '13:30:00', '15:00:00',\n",
       "       '04:00:00', '20:00:00', '09:30:00', '13:00:00', '01:59:00',\n",
       "       '17:00:00', '05:59:00'], dtype=object)"
      ]
     },
     "execution_count": 312,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_final['competition_started_time'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 313,
   "id": "c28d4ad0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_final['competition started hour'] = df_final['competition_started_time'].str[0:2]\n",
    "# df_final['competition started minute'] = df_final['competition_started_time'].str[3:5]\n",
    "# df_final['competition started second'] = df_final['competition_started_time'].str[6:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 314,
   "id": "636b3d4b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_final['competition started hour'].isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 315,
   "id": "f9ade4d3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_final['competition started hour'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 316,
   "id": "21483759",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_final['competition started hour'].fillna(-1, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 317,
   "id": "d7ac4df1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_final['competition started minute'].isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 318,
   "id": "e7304644",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_final['competition started minute'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 319,
   "id": "092d27da",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_final['competition started minute'].fillna(-1, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 320,
   "id": "67a4edcd",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_final['competition started second'].isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 321,
   "id": "441819e6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_final['competition started second'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 322,
   "id": "f5680373",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_final['competition started second'].fillna(-1, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 323,
   "id": "53654de0",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_final.drop('competition_started_time', axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 324,
   "id": "627fcf41",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 1., nan,  4.,  3.,  5.,  2.])"
      ]
     },
     "execution_count": 324,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_final['competition_started_year'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 325,
   "id": "3c359fe2",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_final['competition_started_year'].fillna(0, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 326,
   "id": "982671cc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([12., nan,  2.,  4.,  5.,  1., 11.,  9.,  3.,  6., 10.,  8.,  7.])"
      ]
     },
     "execution_count": 326,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_final['competition_started_month'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 327,
   "id": "7b1880e8",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_final['competition_started_month'].fillna(0, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 328,
   "id": "84674d98",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([28., nan, 24.,  7.,  9.,  4., 15.,  5., 23.,  1., 22.,  3., 13.,\n",
       "       29., 14., 17., 21., 11., 19.,  6., 16.,  2., 20., 30., 26., 27.,\n",
       "       25., 18., 12.])"
      ]
     },
     "execution_count": 328,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_final['competition_started_day'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 329,
   "id": "409b1c91",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_final['competition_started_day'].fillna(0, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 330,
   "id": "21f1373d",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_final.rename(columns={'datetime Day_of_month': 'Day last active'},  inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 331,
   "id": "60d9a9d4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_final['User sign-in hour'] = df_final['User sign-in time'].str[0:2]\n",
    "# df_final['User sign-in minute'] = df_final['User sign-in time'].str[3:5]\n",
    "# df_final['User sign-in second'] = df_final['User sign-in time'].str[6:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 332,
   "id": "e1df43fc",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_final['User sign-in hour'].isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 333,
   "id": "f7ef9b60",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_final['User sign-in hour'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 334,
   "id": "e291fba5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_final['User sign-in hour'].fillna(-1, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 335,
   "id": "ec0c8c35",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_final['User sign-in minute'].isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 336,
   "id": "5dd8ff21",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_final['User sign-in minute'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 337,
   "id": "2765b0f7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_final['User sign-in minute'].fillna(-1, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 338,
   "id": "03dca499",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_final['User sign-in second'].isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 339,
   "id": "15901947",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_final['User sign-in second'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 340,
   "id": "cf0aeff2",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_final['User sign-in second'].fillna(-1, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 341,
   "id": "dda92750",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_final.drop('User sign-in time', axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 342,
   "id": "ab3f0138",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([28., 20., 24., 27., 29., 31., 26., 30., 15., 21.,  5.,  6., 25.,\n",
       "       nan,  2., 10., 23.,  9.,  4., 19., 14.,  3.,  7., 16., 11., 18.,\n",
       "       22., 17.,  8., 12., 13.,  1.])"
      ]
     },
     "execution_count": 342,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_final['Day last active'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 343,
   "id": "6d107de2",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_final['competition_started_day'].fillna(0, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 344,
   "id": "18c03d40",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2984"
      ]
     },
     "execution_count": 344,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_final['count_UA'].isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 345,
   "id": "0f3b08b1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "268"
      ]
     },
     "execution_count": 345,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df_final['count_UA'].unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 346,
   "id": "fce902e2",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_final['count_UA'].fillna(0, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 347,
   "id": "9b1fbb53",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_final['Comment Hour'].fillna(-1, inplace = True)\n",
    "# df_final['Comment Minute'].fillna(-1, inplace = True)\n",
    "# df_final['Comment Second'].fillna(-1, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 348,
   "id": "3a0d93a8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6269"
      ]
     },
     "execution_count": 348,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_final['User sign-in day'].isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 349,
   "id": "1d991b12",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([20.,  8., 27., nan, 13., 21., 31.,  3., 29., 17., 14.,  6.,  5.,\n",
       "       23., 26.,  4., 16., 28., 10., 24., 18., 22., 30.,  7.,  1., 15.,\n",
       "       25., 12.,  2.,  9., 19., 11.])"
      ]
     },
     "execution_count": 349,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_final['User sign-in day'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 350,
   "id": "f924906b",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_final['User sign-in day'].fillna(0, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 351,
   "id": "7caa4a4d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2984"
      ]
     },
     "execution_count": 351,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_final['Day duration'].isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 352,
   "id": "b414eec9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "32"
      ]
     },
     "execution_count": 352,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df_final['Day duration'].unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 353,
   "id": "5a8318c7",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_final['Day duration'].fillna(-1, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 354,
   "id": "3dba2740",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_final['Day duration'] += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 355,
   "id": "db2cc5c8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_final['Discussion Hour'] = pd.to_numeric(df_final['Discussion Hour'])\n",
    "# df_final['Discussion Minute'] = pd.to_numeric(df_final['Discussion Minute'])\n",
    "# df_final['Discussion Second'] = pd.to_numeric(df_final['Discussion Second'])\n",
    "df_final['Discussion_day'] = pd.to_numeric(df_final['Discussion_day'])\n",
    "df_final['Discussion_month'] = pd.to_numeric(df_final['Discussion_month'])\n",
    "# df_final['Comment Hour'] = pd.to_numeric(df_final['Comment Hour'])\n",
    "# df_final['Comment Minute'] = pd.to_numeric(df_final['Comment Minute'])\n",
    "# df_final['Comment Second'] = pd.to_numeric(df_final['Comment Second'])\n",
    "df_final['Successful Submission Count'] = pd.to_numeric(df_final['Successful Submission Count'])\n",
    "df_final['User sign-in month'] = pd.to_numeric(df_final['User sign-in month'])\n",
    "# df_final['competition participation hour'] = pd.to_numeric(df_final['competition participation hour'])\n",
    "# df_final['competition participation minute'] = pd.to_numeric(df_final['competition participation minute'])\n",
    "# df_final['competition participation second'] = pd.to_numeric(df_final['competition participation second'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 356,
   "id": "b48aa25e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_final['competition ended hour'] = pd.to_numeric(df_final['competition ended hour'])\n",
    "# df_final['competition ended minute'] = pd.to_numeric(df_final['competition ended minute'])\n",
    "# df_final['competition ended second'] = pd.to_numeric(df_final['competition ended second'])\n",
    "# df_final['competition started hour'] = pd.to_numeric(df_final['competition started hour'])\n",
    "# df_final['competition started minute'] = pd.to_numeric(df_final['competition started minute'])\n",
    "# df_final['competition started second'] = pd.to_numeric(df_final['competition started second'])\n",
    "# df_final['User sign-in hour'] = pd.to_numeric(df_final['User sign-in hour'])\n",
    "# df_final['User sign-in minute'] = pd.to_numeric(df_final['User sign-in minute'])\n",
    "# df_final['User sign-in second'] = pd.to_numeric(df_final['User sign-in second'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 357,
   "id": "bba0fee5",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_final[\"User sign-in month\"].fillna(0, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 358,
   "id": "9b968c92",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_final[\"Day last active\"].fillna(0, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 359,
   "id": "36d7e48e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_final[\"Time user completed action in hour\"].fillna(-1, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 360,
   "id": "4d56d0d5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_final[\"Time user completed action in minute\"].fillna(-1, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 361,
   "id": "8cdda5ef",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_final[\"Time user completed action in second\"].fillna(-1, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 362,
   "id": "7ecbb5a4",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_final[\"Discussion_month\"].fillna(0, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 363,
   "id": "e6daf47f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_final[\"competition participation second\"].fillna(-1, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 364,
   "id": "d913a6e5",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_final.drop(\"Countries_ID\", inplace=True, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 365,
   "id": "6e0e4f63",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_final.drop(\"Discussion_month\", inplace=True, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 366,
   "id": "3f795382",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_final.drop(\"User sign-in month\", inplace=True, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 367,
   "id": "0588ff7b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1.        ,        nan, 2.        , 4.5       , 3.75      ,\n",
       "       3.        , 4.        , 4.66666667, 3.5       , 5.        ,\n",
       "       2.5       , 2.8       , 4.33333333, 3.6875    , 3.13333333,\n",
       "       3.4       , 3.18181818, 3.60869565, 3.8       , 1.66666667,\n",
       "       3.05263158, 3.45454545, 2.16666667, 2.66666667, 3.33333333,\n",
       "       3.25      , 1.5       , 1.8       , 1.25      , 1.38888889,\n",
       "       2.75      , 4.25      , 3.66666667, 2.1       , 3.83333333,\n",
       "       2.2       , 3.6       , 4.14285714, 3.27272727, 3.42857143,\n",
       "       2.33333333, 2.78571429, 2.6       , 1.6       , 3.28571429,\n",
       "       3.71428571, 4.71428571, 1.75      , 4.75      , 3.85714286,\n",
       "       2.34782609, 2.3       , 1.4       , 2.83333333, 1.82352941])"
      ]
     },
     "execution_count": 367,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_final[\"Blog Theme\"].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 368,
   "id": "3bd081d0",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_final[\"Blog Theme\"].fillna(0, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 369,
   "id": "0797034f",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_final.rename(columns={'Title': 'Activity Type'},  inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 370,
   "id": "1a7eb726",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_final[\"Activity Type\"].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 371,
   "id": "e018dbe7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_final[\"Activity Type\"].fillna(0, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 372,
   "id": "46b11db1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 20998 entries, 0 to 20997\n",
      "Data columns (total 85 columns):\n",
      " #   Column                           Non-Null Count  Dtype  \n",
      "---  ------                           --------------  -----  \n",
      " 0   User_ID                          20998 non-null  object \n",
      " 1   Personal                         20998 non-null  float64\n",
      " 2   Theme                            20998 non-null  float64\n",
      " 3   Discussion_day                   20998 non-null  float64\n",
      " 4   comment_day                      20998 non-null  float64\n",
      " 5   Participant Type                 20998 non-null  float64\n",
      " 6   Successful Submission Count      20998 non-null  int64  \n",
      " 7   Created At time                  9300 non-null   object \n",
      " 8   Comp_participation_day           20998 non-null  float64\n",
      " 9   FeatureA                         20998 non-null  float64\n",
      " 10  FeatureB                         20998 non-null  float64\n",
      " 11  FeatureC                         20998 non-null  float64\n",
      " 12  FeatureD                         20998 non-null  float64\n",
      " 13  FeatureE                         20998 non-null  float64\n",
      " 14  SecretCode                       20998 non-null  float64\n",
      " 15  FeatureF                         20998 non-null  float64\n",
      " 16  FeatureG                         20998 non-null  float64\n",
      " 17  FeatureH                         20998 non-null  float64\n",
      " 18  FeatureI                         20998 non-null  float64\n",
      " 19  Country_ID                       4565 non-null   object \n",
      " 20  competition_ended_year           20998 non-null  float64\n",
      " 21  competition_ended_month          20998 non-null  float64\n",
      " 22  competition_ended_day            20998 non-null  float64\n",
      " 23  competition_started_year         20998 non-null  float64\n",
      " 24  competition_started_month        20998 non-null  float64\n",
      " 25  competition_started_day          20998 non-null  float64\n",
      " 26  Day last active                  20998 non-null  float64\n",
      " 27  Blog Theme                       20998 non-null  float64\n",
      " 28  Blog Published Year              1847 non-null   float64\n",
      " 29  Blog Published Month             1847 non-null   float64\n",
      " 30  Published At Day_of_month        1847 non-null   float64\n",
      " 31  Remote                           18014 non-null  float64\n",
      " 32  Experience                       1049 non-null   float64\n",
      " 33  Employment Type                  18014 non-null  float64\n",
      " 34  Company Size                     1037 non-null   float64\n",
      " 35  Downloaded_Competition_Datafile  18014 non-null  float64\n",
      " 36  Viewed_All_Discussions           18014 non-null  float64\n",
      " 37  Confirmed_Email                  18014 non-null  float64\n",
      " 38  Joined_Competition               18014 non-null  float64\n",
      " 39  Signed_Up                        18014 non-null  float64\n",
      " 40  Viewed_All_Competitions          18014 non-null  float64\n",
      " 41  Viewed_All_Learning_Pages        18014 non-null  float64\n",
      " 42  badge_OCZE                       18014 non-null  float64\n",
      " 43  create_alias                     18014 non-null  float64\n",
      " 44  identify                         18014 non-null  float64\n",
      " 45  comp_ID                          18014 non-null  float64\n",
      " 46  Signed_In                        18014 non-null  float64\n",
      " 47  job_ID                           18014 non-null  float64\n",
      " 48  Updated_Discussion               18014 non-null  float64\n",
      " 49  Viewed_All_Jobs                  18014 non-null  float64\n",
      " 50  Created_Submission               18014 non-null  float64\n",
      " 51  Signed_Out                       18014 non-null  float64\n",
      " 52  Updated_Profile                  18014 non-null  float64\n",
      " 53  blog_ID                          18014 non-null  float64\n",
      " 54  Viewed_Discussion                18014 non-null  float64\n",
      " 55  Updated_Submission               18014 non-null  float64\n",
      " 56  badge_HYIO                       18014 non-null  float64\n",
      " 57  Industry_FS                      18014 non-null  float64\n",
      " 58  Industry_Bank                    18014 non-null  float64\n",
      " 59  Industry_Loc                     18014 non-null  float64\n",
      " 60  Industry_Gov                     18014 non-null  float64\n",
      " 61  Industry_Health                  18014 non-null  float64\n",
      " 62  Industry_Res                     18014 non-null  float64\n",
      " 63  Industry_Amb                     18014 non-null  float64\n",
      " 64  Industry_Transp                  18014 non-null  float64\n",
      " 65  Industry_Scholar                 18014 non-null  float64\n",
      " 66  Industry_Log                     18014 non-null  float64\n",
      " 67  Industry_Media                   18014 non-null  float64\n",
      " 68  Industry_Manuf                   18014 non-null  float64\n",
      " 69  Industry_Sales                   18014 non-null  float64\n",
      " 70  Industry_Insur                   18014 non-null  float64\n",
      " 71  Industry_Manage                  18014 non-null  float64\n",
      " 72  Industry_AI                      18014 non-null  float64\n",
      " 73  Industry_Zindi                   18014 non-null  float64\n",
      " 74  Industry_Tech                    18014 non-null  float64\n",
      " 75  Function_DE                      18014 non-null  float64\n",
      " 76  Function_DM                      18014 non-null  float64\n",
      " 77  Function_Bus                     18014 non-null  float64\n",
      " 78  Function_AI                      18014 non-null  float64\n",
      " 79  Function_ML                      18014 non-null  float64\n",
      " 80  count_UA                         20998 non-null  float64\n",
      " 81  Day duration                     20998 non-null  float64\n",
      " 82  FeatureX                         20998 non-null  float64\n",
      " 83  FeatureY                         20998 non-null  float64\n",
      " 84  User sign-in day                 20998 non-null  float64\n",
      "dtypes: float64(81), int64(1), object(3)\n",
      "memory usage: 14.3+ MB\n"
     ]
    }
   ],
   "source": [
    "df_final.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 373,
   "id": "e5782be3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_final.drop(\"Created At time\", inplace=True, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 374,
   "id": "f323e9e8",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_final[\"Blog Published Year\"].fillna(-1, inplace = True)\n",
    "df_final[\"Blog Published Month\"].fillna(-1, inplace = True)\n",
    "df_final[\"Published At Day_of_month\"].fillna(-1, inplace = True)\n",
    "# df_final[\"Blog Published Hour\"].fillna(-1, inplace = True)\n",
    "# df_final[\"Blog Published Minute\"].fillna(-1, inplace = True)\n",
    "# df_final[\"Blog Published Second\"].fillna(-1, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 375,
   "id": "293a12b8",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_final[\"Remote\"].fillna(0, inplace = True)\n",
    "df_final[\"Experience\"].fillna(0, inplace = True)\n",
    "df_final[\"Company Size\"].fillna(0, inplace = True)\n",
    "df_final[\"Employment Type\"].fillna(0, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 376,
   "id": "c974327b",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_final[\"Industry_FS\"].fillna(0, inplace = True)\n",
    "df_final[\"Industry_Bank\"].fillna(0, inplace = True)\n",
    "df_final[\"Industry_Loc\"].fillna(0, inplace = True)\n",
    "df_final[\"Industry_Gov\"].fillna(0, inplace = True)\n",
    "df_final[\"Industry_Health\"].fillna(0, inplace = True)\n",
    "df_final[\"Industry_Res\"].fillna(0, inplace = True)\n",
    "df_final[\"Industry_Amb\"].fillna(0, inplace = True)\n",
    "df_final[\"Industry_Transp\"].fillna(0, inplace = True)\n",
    "df_final[\"Industry_Scholar\"].fillna(0, inplace = True)\n",
    "df_final[\"Industry_Log\"].fillna(0, inplace = True)\n",
    "df_final[\"Industry_Media\"].fillna(0, inplace = True)\n",
    "df_final[\"Industry_Manuf\"].fillna(0, inplace = True)\n",
    "df_final[\"Industry_Sales\"].fillna(0, inplace = True)\n",
    "df_final[\"Industry_Insur\"].fillna(0, inplace = True)\n",
    "df_final[\"Industry_Manage\"].fillna(0, inplace = True)\n",
    "df_final[\"Industry_AI\"].fillna(0, inplace = True)\n",
    "df_final[\"Industry_Zindi\"].fillna(0, inplace = True)\n",
    "df_final[\"Industry_Tech\"].fillna(0, inplace = True)\n",
    "df_final[\"Function_DE\"].fillna(0, inplace = True)\n",
    "df_final[\"Function_DM\"].fillna(0, inplace = True)\n",
    "df_final[\"Function_Bus\"].fillna(0, inplace = True)\n",
    "df_final[\"Function_AI\"].fillna(0, inplace = True)\n",
    "df_final[\"Function_ML\"].fillna(0, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 377,
   "id": "704052ba",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 20998 entries, 0 to 20997\n",
      "Data columns (total 85 columns):\n",
      " #   Column                           Non-Null Count  Dtype  \n",
      "---  ------                           --------------  -----  \n",
      " 0   User_ID                          20998 non-null  object \n",
      " 1   Personal                         20998 non-null  float64\n",
      " 2   Theme                            20998 non-null  float64\n",
      " 3   Discussion_day                   20998 non-null  float64\n",
      " 4   comment_day                      20998 non-null  float64\n",
      " 5   Participant Type                 20998 non-null  float64\n",
      " 6   Successful Submission Count      20998 non-null  int64  \n",
      " 7   Created At time                  9300 non-null   object \n",
      " 8   Comp_participation_day           20998 non-null  float64\n",
      " 9   FeatureA                         20998 non-null  float64\n",
      " 10  FeatureB                         20998 non-null  float64\n",
      " 11  FeatureC                         20998 non-null  float64\n",
      " 12  FeatureD                         20998 non-null  float64\n",
      " 13  FeatureE                         20998 non-null  float64\n",
      " 14  SecretCode                       20998 non-null  float64\n",
      " 15  FeatureF                         20998 non-null  float64\n",
      " 16  FeatureG                         20998 non-null  float64\n",
      " 17  FeatureH                         20998 non-null  float64\n",
      " 18  FeatureI                         20998 non-null  float64\n",
      " 19  Country_ID                       4565 non-null   object \n",
      " 20  competition_ended_year           20998 non-null  float64\n",
      " 21  competition_ended_month          20998 non-null  float64\n",
      " 22  competition_ended_day            20998 non-null  float64\n",
      " 23  competition_started_year         20998 non-null  float64\n",
      " 24  competition_started_month        20998 non-null  float64\n",
      " 25  competition_started_day          20998 non-null  float64\n",
      " 26  Day last active                  20998 non-null  float64\n",
      " 27  Blog Theme                       20998 non-null  float64\n",
      " 28  Blog Published Year              20998 non-null  float64\n",
      " 29  Blog Published Month             20998 non-null  float64\n",
      " 30  Published At Day_of_month        20998 non-null  float64\n",
      " 31  Remote                           20998 non-null  float64\n",
      " 32  Experience                       20998 non-null  float64\n",
      " 33  Employment Type                  20998 non-null  float64\n",
      " 34  Company Size                     20998 non-null  float64\n",
      " 35  Downloaded_Competition_Datafile  18014 non-null  float64\n",
      " 36  Viewed_All_Discussions           18014 non-null  float64\n",
      " 37  Confirmed_Email                  18014 non-null  float64\n",
      " 38  Joined_Competition               18014 non-null  float64\n",
      " 39  Signed_Up                        18014 non-null  float64\n",
      " 40  Viewed_All_Competitions          18014 non-null  float64\n",
      " 41  Viewed_All_Learning_Pages        18014 non-null  float64\n",
      " 42  badge_OCZE                       18014 non-null  float64\n",
      " 43  create_alias                     18014 non-null  float64\n",
      " 44  identify                         18014 non-null  float64\n",
      " 45  comp_ID                          18014 non-null  float64\n",
      " 46  Signed_In                        18014 non-null  float64\n",
      " 47  job_ID                           18014 non-null  float64\n",
      " 48  Updated_Discussion               18014 non-null  float64\n",
      " 49  Viewed_All_Jobs                  18014 non-null  float64\n",
      " 50  Created_Submission               18014 non-null  float64\n",
      " 51  Signed_Out                       18014 non-null  float64\n",
      " 52  Updated_Profile                  18014 non-null  float64\n",
      " 53  blog_ID                          18014 non-null  float64\n",
      " 54  Viewed_Discussion                18014 non-null  float64\n",
      " 55  Updated_Submission               18014 non-null  float64\n",
      " 56  badge_HYIO                       18014 non-null  float64\n",
      " 57  Industry_FS                      20998 non-null  float64\n",
      " 58  Industry_Bank                    20998 non-null  float64\n",
      " 59  Industry_Loc                     20998 non-null  float64\n",
      " 60  Industry_Gov                     20998 non-null  float64\n",
      " 61  Industry_Health                  20998 non-null  float64\n",
      " 62  Industry_Res                     20998 non-null  float64\n",
      " 63  Industry_Amb                     20998 non-null  float64\n",
      " 64  Industry_Transp                  20998 non-null  float64\n",
      " 65  Industry_Scholar                 20998 non-null  float64\n",
      " 66  Industry_Log                     20998 non-null  float64\n",
      " 67  Industry_Media                   20998 non-null  float64\n",
      " 68  Industry_Manuf                   20998 non-null  float64\n",
      " 69  Industry_Sales                   20998 non-null  float64\n",
      " 70  Industry_Insur                   20998 non-null  float64\n",
      " 71  Industry_Manage                  20998 non-null  float64\n",
      " 72  Industry_AI                      20998 non-null  float64\n",
      " 73  Industry_Zindi                   20998 non-null  float64\n",
      " 74  Industry_Tech                    20998 non-null  float64\n",
      " 75  Function_DE                      20998 non-null  float64\n",
      " 76  Function_DM                      20998 non-null  float64\n",
      " 77  Function_Bus                     20998 non-null  float64\n",
      " 78  Function_AI                      20998 non-null  float64\n",
      " 79  Function_ML                      20998 non-null  float64\n",
      " 80  count_UA                         20998 non-null  float64\n",
      " 81  Day duration                     20998 non-null  float64\n",
      " 82  FeatureX                         20998 non-null  float64\n",
      " 83  FeatureY                         20998 non-null  float64\n",
      " 84  User sign-in day                 20998 non-null  float64\n",
      "dtypes: float64(81), int64(1), object(3)\n",
      "memory usage: 14.3+ MB\n"
     ]
    }
   ],
   "source": [
    "df_final.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "24e1619e",
   "metadata": {},
   "source": [
    "### Label Encoder for Country_ID"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 378,
   "id": "cc13ba1f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['X9GR', nan, 'YSLJ', 'G3F0', 'GYYX', '03GB', 'ZEWU', 'DYYF',\n",
       "       'KYT4', 'ISIQ', 'N6EN', 'EGI9', 'H03V', 'SE4K', '4PRY'],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 378,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_final[\"Country_ID\"].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 379,
   "id": "082ea9de",
   "metadata": {},
   "outputs": [],
   "source": [
    "le = preprocessing.LabelEncoder()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 380,
   "id": "6111974b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-2 {color: black;background-color: white;}#sk-container-id-2 pre{padding: 0;}#sk-container-id-2 div.sk-toggleable {background-color: white;}#sk-container-id-2 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-2 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-2 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-2 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-2 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-2 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-2 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-2 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-2 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-2 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-2 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-2 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-2 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-2 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-2 div.sk-item {position: relative;z-index: 1;}#sk-container-id-2 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-2 div.sk-item::before, #sk-container-id-2 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-2 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-2 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-2 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-2 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-2 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-2 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-2 div.sk-label-container {text-align: center;}#sk-container-id-2 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-2 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-2\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>LabelEncoder()</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-2\" type=\"checkbox\" checked><label for=\"sk-estimator-id-2\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">LabelEncoder</label><div class=\"sk-toggleable__content\"><pre>LabelEncoder()</pre></div></div></div></div></div>"
      ],
      "text/plain": [
       "LabelEncoder()"
      ]
     },
     "execution_count": 380,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "le.fit(df_final[\"Country_ID\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 381,
   "id": "e1c0a3eb",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_final[\"Country_ID\"] = le.transform(df_final[\"Country_ID\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 382,
   "id": "077cf945",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([11, 14, 12,  4,  5,  0, 13,  2,  8,  7,  9,  3,  6, 10,  1])"
      ]
     },
     "execution_count": 382,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_final[\"Country_ID\"].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 383,
   "id": "58ada006",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 20998 entries, 0 to 20997\n",
      "Data columns (total 85 columns):\n",
      " #   Column                           Non-Null Count  Dtype  \n",
      "---  ------                           --------------  -----  \n",
      " 0   User_ID                          20998 non-null  object \n",
      " 1   Personal                         20998 non-null  float64\n",
      " 2   Theme                            20998 non-null  float64\n",
      " 3   Discussion_day                   20998 non-null  float64\n",
      " 4   comment_day                      20998 non-null  float64\n",
      " 5   Participant Type                 20998 non-null  float64\n",
      " 6   Successful Submission Count      20998 non-null  int64  \n",
      " 7   Created At time                  9300 non-null   object \n",
      " 8   Comp_participation_day           20998 non-null  float64\n",
      " 9   FeatureA                         20998 non-null  float64\n",
      " 10  FeatureB                         20998 non-null  float64\n",
      " 11  FeatureC                         20998 non-null  float64\n",
      " 12  FeatureD                         20998 non-null  float64\n",
      " 13  FeatureE                         20998 non-null  float64\n",
      " 14  SecretCode                       20998 non-null  float64\n",
      " 15  FeatureF                         20998 non-null  float64\n",
      " 16  FeatureG                         20998 non-null  float64\n",
      " 17  FeatureH                         20998 non-null  float64\n",
      " 18  FeatureI                         20998 non-null  float64\n",
      " 19  Country_ID                       20998 non-null  int32  \n",
      " 20  competition_ended_year           20998 non-null  float64\n",
      " 21  competition_ended_month          20998 non-null  float64\n",
      " 22  competition_ended_day            20998 non-null  float64\n",
      " 23  competition_started_year         20998 non-null  float64\n",
      " 24  competition_started_month        20998 non-null  float64\n",
      " 25  competition_started_day          20998 non-null  float64\n",
      " 26  Day last active                  20998 non-null  float64\n",
      " 27  Blog Theme                       20998 non-null  float64\n",
      " 28  Blog Published Year              20998 non-null  float64\n",
      " 29  Blog Published Month             20998 non-null  float64\n",
      " 30  Published At Day_of_month        20998 non-null  float64\n",
      " 31  Remote                           20998 non-null  float64\n",
      " 32  Experience                       20998 non-null  float64\n",
      " 33  Employment Type                  20998 non-null  float64\n",
      " 34  Company Size                     20998 non-null  float64\n",
      " 35  Downloaded_Competition_Datafile  18014 non-null  float64\n",
      " 36  Viewed_All_Discussions           18014 non-null  float64\n",
      " 37  Confirmed_Email                  18014 non-null  float64\n",
      " 38  Joined_Competition               18014 non-null  float64\n",
      " 39  Signed_Up                        18014 non-null  float64\n",
      " 40  Viewed_All_Competitions          18014 non-null  float64\n",
      " 41  Viewed_All_Learning_Pages        18014 non-null  float64\n",
      " 42  badge_OCZE                       18014 non-null  float64\n",
      " 43  create_alias                     18014 non-null  float64\n",
      " 44  identify                         18014 non-null  float64\n",
      " 45  comp_ID                          18014 non-null  float64\n",
      " 46  Signed_In                        18014 non-null  float64\n",
      " 47  job_ID                           18014 non-null  float64\n",
      " 48  Updated_Discussion               18014 non-null  float64\n",
      " 49  Viewed_All_Jobs                  18014 non-null  float64\n",
      " 50  Created_Submission               18014 non-null  float64\n",
      " 51  Signed_Out                       18014 non-null  float64\n",
      " 52  Updated_Profile                  18014 non-null  float64\n",
      " 53  blog_ID                          18014 non-null  float64\n",
      " 54  Viewed_Discussion                18014 non-null  float64\n",
      " 55  Updated_Submission               18014 non-null  float64\n",
      " 56  badge_HYIO                       18014 non-null  float64\n",
      " 57  Industry_FS                      20998 non-null  float64\n",
      " 58  Industry_Bank                    20998 non-null  float64\n",
      " 59  Industry_Loc                     20998 non-null  float64\n",
      " 60  Industry_Gov                     20998 non-null  float64\n",
      " 61  Industry_Health                  20998 non-null  float64\n",
      " 62  Industry_Res                     20998 non-null  float64\n",
      " 63  Industry_Amb                     20998 non-null  float64\n",
      " 64  Industry_Transp                  20998 non-null  float64\n",
      " 65  Industry_Scholar                 20998 non-null  float64\n",
      " 66  Industry_Log                     20998 non-null  float64\n",
      " 67  Industry_Media                   20998 non-null  float64\n",
      " 68  Industry_Manuf                   20998 non-null  float64\n",
      " 69  Industry_Sales                   20998 non-null  float64\n",
      " 70  Industry_Insur                   20998 non-null  float64\n",
      " 71  Industry_Manage                  20998 non-null  float64\n",
      " 72  Industry_AI                      20998 non-null  float64\n",
      " 73  Industry_Zindi                   20998 non-null  float64\n",
      " 74  Industry_Tech                    20998 non-null  float64\n",
      " 75  Function_DE                      20998 non-null  float64\n",
      " 76  Function_DM                      20998 non-null  float64\n",
      " 77  Function_Bus                     20998 non-null  float64\n",
      " 78  Function_AI                      20998 non-null  float64\n",
      " 79  Function_ML                      20998 non-null  float64\n",
      " 80  count_UA                         20998 non-null  float64\n",
      " 81  Day duration                     20998 non-null  float64\n",
      " 82  FeatureX                         20998 non-null  float64\n",
      " 83  FeatureY                         20998 non-null  float64\n",
      " 84  User sign-in day                 20998 non-null  float64\n",
      "dtypes: float64(81), int32(1), int64(1), object(2)\n",
      "memory usage: 14.2+ MB\n"
     ]
    }
   ],
   "source": [
    "df_final.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 384,
   "id": "b64fb5bc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "17387"
      ]
     },
     "execution_count": 384,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df_final[\"User_ID\"].unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 385,
   "id": "f874d1d8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>User_ID</th>\n",
       "      <th>Personal</th>\n",
       "      <th>Theme</th>\n",
       "      <th>Discussion_day</th>\n",
       "      <th>comment_day</th>\n",
       "      <th>Participant Type</th>\n",
       "      <th>Successful Submission Count</th>\n",
       "      <th>Created At time</th>\n",
       "      <th>Comp_participation_day</th>\n",
       "      <th>FeatureA</th>\n",
       "      <th>...</th>\n",
       "      <th>Function_DE</th>\n",
       "      <th>Function_DM</th>\n",
       "      <th>Function_Bus</th>\n",
       "      <th>Function_AI</th>\n",
       "      <th>Function_ML</th>\n",
       "      <th>count_UA</th>\n",
       "      <th>Day duration</th>\n",
       "      <th>FeatureX</th>\n",
       "      <th>FeatureY</th>\n",
       "      <th>User sign-in day</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ID_AAGFVLHH_Month_12</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>27.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>10:53:52.558030</td>\n",
       "      <td>28.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>35.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>20.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ID_0AJGQILU_Month_12</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>20.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ID_U90KK6KL_Month_2</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>18:57:59.998938</td>\n",
       "      <td>12.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>95.0</td>\n",
       "      <td>17.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>8.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ID_U90KK6KL_Month_2</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>18:49:47.531900</td>\n",
       "      <td>12.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>95.0</td>\n",
       "      <td>17.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>8.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ID_U90KK6KL_Month_2</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>15:34:35.872471</td>\n",
       "      <td>8.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>95.0</td>\n",
       "      <td>17.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>8.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20993</th>\n",
       "      <td>ID_Y02FKZWI_Month_3</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>26.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20994</th>\n",
       "      <td>ID_FDOGWALS_Month_11</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20995</th>\n",
       "      <td>ID_YJCY8FF5_Month_5</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20996</th>\n",
       "      <td>ID_QDL8A9FQ_Month_4</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>28.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20997</th>\n",
       "      <td>ID_C5393PD0_Month_11</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>20998 rows × 85 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                    User_ID  Personal  Theme  Discussion_day  comment_day  \\\n",
       "0      ID_AAGFVLHH_Month_12       1.0    0.0            27.0          0.0   \n",
       "1      ID_0AJGQILU_Month_12       1.0    0.0            20.0          0.0   \n",
       "2       ID_U90KK6KL_Month_2       1.0    0.0             9.0          0.0   \n",
       "3       ID_U90KK6KL_Month_2       1.0    0.0             9.0          0.0   \n",
       "4       ID_U90KK6KL_Month_2       1.0    0.0             9.0          0.0   \n",
       "...                     ...       ...    ...             ...          ...   \n",
       "20993   ID_Y02FKZWI_Month_3      -1.0    0.0             0.0          0.0   \n",
       "20994  ID_FDOGWALS_Month_11      -1.0    0.0             0.0          0.0   \n",
       "20995   ID_YJCY8FF5_Month_5      -1.0    0.0             0.0          0.0   \n",
       "20996   ID_QDL8A9FQ_Month_4      -1.0    0.0             0.0          0.0   \n",
       "20997  ID_C5393PD0_Month_11      -1.0    0.0             0.0          0.0   \n",
       "\n",
       "       Participant Type  Successful Submission Count  Created At time  \\\n",
       "0                   1.0                            0  10:53:52.558030   \n",
       "1                   0.0                            0              NaN   \n",
       "2                   1.0                            0  18:57:59.998938   \n",
       "3                   1.0                            0  18:49:47.531900   \n",
       "4                   1.0                            0  15:34:35.872471   \n",
       "...                 ...                          ...              ...   \n",
       "20993               0.0                            0              NaN   \n",
       "20994               0.0                            0              NaN   \n",
       "20995               0.0                            0              NaN   \n",
       "20996               0.0                            0              NaN   \n",
       "20997               0.0                            0              NaN   \n",
       "\n",
       "       Comp_participation_day  FeatureA  ...  Function_DE  Function_DM  \\\n",
       "0                        28.0       0.0  ...          0.0          0.0   \n",
       "1                         0.0      -1.0  ...          0.0          0.0   \n",
       "2                        12.0       1.0  ...          0.0          0.0   \n",
       "3                        12.0       2.0  ...          0.0          0.0   \n",
       "4                         8.0       0.0  ...          0.0          0.0   \n",
       "...                       ...       ...  ...          ...          ...   \n",
       "20993                     0.0      -1.0  ...          0.0          0.0   \n",
       "20994                     0.0      -1.0  ...          0.0          0.0   \n",
       "20995                     0.0      -1.0  ...          0.0          0.0   \n",
       "20996                     0.0      -1.0  ...          0.0          0.0   \n",
       "20997                     0.0      -1.0  ...          0.0          0.0   \n",
       "\n",
       "       Function_Bus  Function_AI  Function_ML  count_UA  Day duration  \\\n",
       "0               0.0          0.0          0.0      35.0           9.0   \n",
       "1               0.0          0.0          0.0      14.0           1.0   \n",
       "2               0.0          0.0          0.0      95.0          17.0   \n",
       "3               0.0          0.0          0.0      95.0          17.0   \n",
       "4               0.0          0.0          0.0      95.0          17.0   \n",
       "...             ...          ...          ...       ...           ...   \n",
       "20993           0.0          0.0          0.0       0.0           0.0   \n",
       "20994           0.0          0.0          0.0       0.0           0.0   \n",
       "20995           0.0          0.0          0.0       0.0           0.0   \n",
       "20996           0.0          0.0          0.0       0.0           0.0   \n",
       "20997           0.0          0.0          0.0       0.0           0.0   \n",
       "\n",
       "       FeatureX  FeatureY  User sign-in day  \n",
       "0           0.0       0.0              20.0  \n",
       "1           0.0       0.0              20.0  \n",
       "2           1.0       1.0               8.0  \n",
       "3           1.0       1.0               8.0  \n",
       "4           1.0       1.0               8.0  \n",
       "...         ...       ...               ...  \n",
       "20993       0.0       0.0              26.0  \n",
       "20994       0.0       0.0               5.0  \n",
       "20995       0.0       0.0              11.0  \n",
       "20996       0.0       0.0              28.0  \n",
       "20997       0.0       1.0               6.0  \n",
       "\n",
       "[20998 rows x 85 columns]"
      ]
     },
     "execution_count": 385,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_final"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 386,
   "id": "5aea350f",
   "metadata": {},
   "outputs": [],
   "source": [
    "temp = df_final.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 387,
   "id": "920d1381",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "17387"
      ]
     },
     "execution_count": 387,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(temp[\"User_ID\"].unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 388,
   "id": "8dcd13eb",
   "metadata": {},
   "outputs": [],
   "source": [
    "temp = temp.groupby(\"User_ID\").mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 389,
   "id": "788815f4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Personal</th>\n",
       "      <th>Theme</th>\n",
       "      <th>Discussion_day</th>\n",
       "      <th>comment_day</th>\n",
       "      <th>Participant Type</th>\n",
       "      <th>Successful Submission Count</th>\n",
       "      <th>Comp_participation_day</th>\n",
       "      <th>FeatureA</th>\n",
       "      <th>FeatureB</th>\n",
       "      <th>FeatureC</th>\n",
       "      <th>...</th>\n",
       "      <th>Function_DE</th>\n",
       "      <th>Function_DM</th>\n",
       "      <th>Function_Bus</th>\n",
       "      <th>Function_AI</th>\n",
       "      <th>Function_ML</th>\n",
       "      <th>count_UA</th>\n",
       "      <th>Day duration</th>\n",
       "      <th>FeatureX</th>\n",
       "      <th>FeatureY</th>\n",
       "      <th>User sign-in day</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>User_ID</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>ID_000H9XOP_Month_5</th>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>22.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ID_003YJHQA_Month_4</th>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>12.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ID_004UKSJV_Month_1</th>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ID_004UKSJV_Month_12</th>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>28.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ID_008D2J6C_Month_1</th>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ID_ZZMHNLNT_Month_4</th>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ID_ZZQK0LU6_Month_3</th>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>1.5</td>\n",
       "      <td>0.0</td>\n",
       "      <td>13.5</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>67.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>29.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ID_ZZUNZJ2U_Month_2</th>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>30.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ID_ZZVM2KO7_Month_4</th>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>22.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ID_ZZVUJ45W_Month_3</th>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>17387 rows × 83 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                      Personal  Theme  Discussion_day  comment_day  \\\n",
       "User_ID                                                              \n",
       "ID_000H9XOP_Month_5       -1.0    0.0             0.0          0.0   \n",
       "ID_003YJHQA_Month_4       -1.0    0.0             0.0          0.0   \n",
       "ID_004UKSJV_Month_1       -1.0    0.0             0.0          0.0   \n",
       "ID_004UKSJV_Month_12      -1.0    0.0             0.0          0.0   \n",
       "ID_008D2J6C_Month_1       -1.0    0.0             0.0          0.0   \n",
       "...                        ...    ...             ...          ...   \n",
       "ID_ZZMHNLNT_Month_4       -1.0    0.0             0.0          0.0   \n",
       "ID_ZZQK0LU6_Month_3       -1.0    0.0             0.0          0.0   \n",
       "ID_ZZUNZJ2U_Month_2       -1.0    0.0             0.0          0.0   \n",
       "ID_ZZVM2KO7_Month_4       -1.0    0.0             0.0          0.0   \n",
       "ID_ZZVUJ45W_Month_3       -1.0    0.0             0.0          0.0   \n",
       "\n",
       "                      Participant Type  Successful Submission Count  \\\n",
       "User_ID                                                               \n",
       "ID_000H9XOP_Month_5                0.0                          0.0   \n",
       "ID_003YJHQA_Month_4                0.0                          0.0   \n",
       "ID_004UKSJV_Month_1                0.0                          0.0   \n",
       "ID_004UKSJV_Month_12               0.0                          0.0   \n",
       "ID_008D2J6C_Month_1                0.0                          0.0   \n",
       "...                                ...                          ...   \n",
       "ID_ZZMHNLNT_Month_4                0.0                          0.0   \n",
       "ID_ZZQK0LU6_Month_3                1.0                          0.0   \n",
       "ID_ZZUNZJ2U_Month_2                0.0                          0.0   \n",
       "ID_ZZVM2KO7_Month_4                0.0                          0.0   \n",
       "ID_ZZVUJ45W_Month_3                1.0                          0.0   \n",
       "\n",
       "                      Comp_participation_day  FeatureA  FeatureB  FeatureC  \\\n",
       "User_ID                                                                      \n",
       "ID_000H9XOP_Month_5                      0.0      -1.0      -1.0      -1.0   \n",
       "ID_003YJHQA_Month_4                      0.0      -1.0      -1.0      -1.0   \n",
       "ID_004UKSJV_Month_1                      0.0      -1.0      -1.0      -1.0   \n",
       "ID_004UKSJV_Month_12                     0.0      -1.0      -1.0      -1.0   \n",
       "ID_008D2J6C_Month_1                      0.0      -1.0      -1.0      -1.0   \n",
       "...                                      ...       ...       ...       ...   \n",
       "ID_ZZMHNLNT_Month_4                      0.0      -1.0      -1.0      -1.0   \n",
       "ID_ZZQK0LU6_Month_3                     29.0       1.5       0.0      13.5   \n",
       "ID_ZZUNZJ2U_Month_2                      0.0      -1.0      -1.0      -1.0   \n",
       "ID_ZZVM2KO7_Month_4                      0.0      -1.0      -1.0      -1.0   \n",
       "ID_ZZVUJ45W_Month_3                      5.0       1.0       7.0       4.0   \n",
       "\n",
       "                      ...  Function_DE  Function_DM  Function_Bus  \\\n",
       "User_ID               ...                                           \n",
       "ID_000H9XOP_Month_5   ...          0.0          0.0           0.0   \n",
       "ID_003YJHQA_Month_4   ...          0.0          0.0           0.0   \n",
       "ID_004UKSJV_Month_1   ...          0.0          0.0           0.0   \n",
       "ID_004UKSJV_Month_12  ...          0.0          0.0           0.0   \n",
       "ID_008D2J6C_Month_1   ...          0.0          0.0           0.0   \n",
       "...                   ...          ...          ...           ...   \n",
       "ID_ZZMHNLNT_Month_4   ...          0.0          0.0           0.0   \n",
       "ID_ZZQK0LU6_Month_3   ...          0.0          0.0           0.0   \n",
       "ID_ZZUNZJ2U_Month_2   ...          0.0          0.0           0.0   \n",
       "ID_ZZVM2KO7_Month_4   ...          0.0          0.0           0.0   \n",
       "ID_ZZVUJ45W_Month_3   ...          0.0          0.0           0.0   \n",
       "\n",
       "                      Function_AI  Function_ML  count_UA  Day duration  \\\n",
       "User_ID                                                                  \n",
       "ID_000H9XOP_Month_5           0.0          0.0       6.0           5.0   \n",
       "ID_003YJHQA_Month_4           0.0          0.0      11.0           1.0   \n",
       "ID_004UKSJV_Month_1           0.0          0.0       1.0           1.0   \n",
       "ID_004UKSJV_Month_12          0.0          0.0      11.0           2.0   \n",
       "ID_008D2J6C_Month_1           0.0          0.0      10.0           9.0   \n",
       "...                           ...          ...       ...           ...   \n",
       "ID_ZZMHNLNT_Month_4           0.0          0.0       8.0           1.0   \n",
       "ID_ZZQK0LU6_Month_3           0.0          0.0      67.0           2.0   \n",
       "ID_ZZUNZJ2U_Month_2           0.0          0.0      12.0           1.0   \n",
       "ID_ZZVM2KO7_Month_4           0.0          0.0       0.0           0.0   \n",
       "ID_ZZVUJ45W_Month_3           0.0          0.0      13.0           1.0   \n",
       "\n",
       "                      FeatureX  FeatureY  User sign-in day  \n",
       "User_ID                                                     \n",
       "ID_000H9XOP_Month_5        0.0       1.0              22.0  \n",
       "ID_003YJHQA_Month_4        0.0       1.0              12.0  \n",
       "ID_004UKSJV_Month_1       -1.0      -1.0               0.0  \n",
       "ID_004UKSJV_Month_12       0.0       0.0              28.0  \n",
       "ID_008D2J6C_Month_1        0.0       0.0               7.0  \n",
       "...                        ...       ...               ...  \n",
       "ID_ZZMHNLNT_Month_4        0.0       0.0               1.0  \n",
       "ID_ZZQK0LU6_Month_3        0.0       3.0              29.0  \n",
       "ID_ZZUNZJ2U_Month_2        0.0       0.0              30.0  \n",
       "ID_ZZVM2KO7_Month_4        0.0       0.0              22.0  \n",
       "ID_ZZVUJ45W_Month_3        0.0       0.0               5.0  \n",
       "\n",
       "[17387 rows x 83 columns]"
      ]
     },
     "execution_count": 389,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "temp.sort_values(\"User_ID\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 390,
   "id": "b444df2c",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_max = df_final.groupby('User_ID').max()\n",
    "df_sum = df_final.groupby('User_ID').sum()\n",
    "df_mean = df_final.groupby('User_ID').mean()\n",
    "df_min = df_final.groupby('User_ID').min()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 391,
   "id": "dc1aca50",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_final = df_mean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 392,
   "id": "918881e5",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_final[\"count_UA\"] = df_sum[\"count_UA\"]\n",
    "df_final[\"Discussion_day\"] = df_max[\"Discussion_day\"]\n",
    "df_final[\"Comp_participation_day\"] = df_max[\"Comp_participation_day\"]\n",
    "df_final[\"competition_ended_day\"] = df_max[\"competition_ended_day\"]\n",
    "df_final[\"competition_started_day\"] = df_min[\"competition_started_day\"]\n",
    "df_final[\"Day last active\"] = df_max[\"Day last active\"]\n",
    "df_final[\"User sign-in day\"] = df_min[\"User sign-in day\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 393,
   "id": "fae3b241",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_final.reset_index(level=0, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 394,
   "id": "5d0a7885",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_final.columns = df_final.columns.str.replace(' ', '_')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 395,
   "id": "83caa20f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>User_ID</th>\n",
       "      <th>Personal</th>\n",
       "      <th>Theme</th>\n",
       "      <th>Discussion_day</th>\n",
       "      <th>comment_day</th>\n",
       "      <th>Participant_Type</th>\n",
       "      <th>Successful_Submission_Count</th>\n",
       "      <th>Comp_participation_day</th>\n",
       "      <th>FeatureA</th>\n",
       "      <th>FeatureB</th>\n",
       "      <th>...</th>\n",
       "      <th>Function_DE</th>\n",
       "      <th>Function_DM</th>\n",
       "      <th>Function_Bus</th>\n",
       "      <th>Function_AI</th>\n",
       "      <th>Function_ML</th>\n",
       "      <th>count_UA</th>\n",
       "      <th>Day_duration</th>\n",
       "      <th>FeatureX</th>\n",
       "      <th>FeatureY</th>\n",
       "      <th>User_sign-in_day</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ID_000H9XOP_Month_5</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>22.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ID_003YJHQA_Month_4</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>12.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ID_004UKSJV_Month_1</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ID_004UKSJV_Month_12</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>28.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ID_008D2J6C_Month_1</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17382</th>\n",
       "      <td>ID_ZZMHNLNT_Month_4</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17383</th>\n",
       "      <td>ID_ZZQK0LU6_Month_3</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>1.5</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>134.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>29.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17384</th>\n",
       "      <td>ID_ZZUNZJ2U_Month_2</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>30.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17385</th>\n",
       "      <td>ID_ZZVM2KO7_Month_4</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>22.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17386</th>\n",
       "      <td>ID_ZZVUJ45W_Month_3</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>52.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>17387 rows × 84 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                    User_ID  Personal  Theme  Discussion_day  comment_day  \\\n",
       "0       ID_000H9XOP_Month_5      -1.0    0.0             0.0          0.0   \n",
       "1       ID_003YJHQA_Month_4      -1.0    0.0             0.0          0.0   \n",
       "2       ID_004UKSJV_Month_1      -1.0    0.0             0.0          0.0   \n",
       "3      ID_004UKSJV_Month_12      -1.0    0.0             0.0          0.0   \n",
       "4       ID_008D2J6C_Month_1      -1.0    0.0             0.0          0.0   \n",
       "...                     ...       ...    ...             ...          ...   \n",
       "17382   ID_ZZMHNLNT_Month_4      -1.0    0.0             0.0          0.0   \n",
       "17383   ID_ZZQK0LU6_Month_3      -1.0    0.0             0.0          0.0   \n",
       "17384   ID_ZZUNZJ2U_Month_2      -1.0    0.0             0.0          0.0   \n",
       "17385   ID_ZZVM2KO7_Month_4      -1.0    0.0             0.0          0.0   \n",
       "17386   ID_ZZVUJ45W_Month_3      -1.0    0.0             0.0          0.0   \n",
       "\n",
       "       Participant_Type  Successful_Submission_Count  Comp_participation_day  \\\n",
       "0                   0.0                          0.0                     0.0   \n",
       "1                   0.0                          0.0                     0.0   \n",
       "2                   0.0                          0.0                     0.0   \n",
       "3                   0.0                          0.0                     0.0   \n",
       "4                   0.0                          0.0                     0.0   \n",
       "...                 ...                          ...                     ...   \n",
       "17382               0.0                          0.0                     0.0   \n",
       "17383               1.0                          0.0                    29.0   \n",
       "17384               0.0                          0.0                     0.0   \n",
       "17385               0.0                          0.0                     0.0   \n",
       "17386               1.0                          0.0                     5.0   \n",
       "\n",
       "       FeatureA  FeatureB  ...  Function_DE  Function_DM  Function_Bus  \\\n",
       "0          -1.0      -1.0  ...          0.0          0.0           0.0   \n",
       "1          -1.0      -1.0  ...          0.0          0.0           0.0   \n",
       "2          -1.0      -1.0  ...          0.0          0.0           0.0   \n",
       "3          -1.0      -1.0  ...          0.0          0.0           0.0   \n",
       "4          -1.0      -1.0  ...          0.0          0.0           0.0   \n",
       "...         ...       ...  ...          ...          ...           ...   \n",
       "17382      -1.0      -1.0  ...          0.0          0.0           0.0   \n",
       "17383       1.5       0.0  ...          0.0          0.0           0.0   \n",
       "17384      -1.0      -1.0  ...          0.0          0.0           0.0   \n",
       "17385      -1.0      -1.0  ...          0.0          0.0           0.0   \n",
       "17386       1.0       7.0  ...          0.0          0.0           0.0   \n",
       "\n",
       "       Function_AI  Function_ML  count_UA  Day_duration  FeatureX  FeatureY  \\\n",
       "0              0.0          0.0       6.0           5.0       0.0       1.0   \n",
       "1              0.0          0.0      11.0           1.0       0.0       1.0   \n",
       "2              0.0          0.0       1.0           1.0      -1.0      -1.0   \n",
       "3              0.0          0.0      11.0           2.0       0.0       0.0   \n",
       "4              0.0          0.0      10.0           9.0       0.0       0.0   \n",
       "...            ...          ...       ...           ...       ...       ...   \n",
       "17382          0.0          0.0       8.0           1.0       0.0       0.0   \n",
       "17383          0.0          0.0     134.0           2.0       0.0       3.0   \n",
       "17384          0.0          0.0      12.0           1.0       0.0       0.0   \n",
       "17385          0.0          0.0       0.0           0.0       0.0       0.0   \n",
       "17386          0.0          0.0      52.0           1.0       0.0       0.0   \n",
       "\n",
       "       User_sign-in_day  \n",
       "0                  22.0  \n",
       "1                  12.0  \n",
       "2                   0.0  \n",
       "3                  28.0  \n",
       "4                   7.0  \n",
       "...                 ...  \n",
       "17382               1.0  \n",
       "17383              29.0  \n",
       "17384              30.0  \n",
       "17385              22.0  \n",
       "17386               5.0  \n",
       "\n",
       "[17387 rows x 84 columns]"
      ]
     },
     "execution_count": 395,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_final"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 396,
   "id": "ffaa0422",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_final.to_csv('DataForLSTM.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "245d91e5",
   "metadata": {},
   "source": [
    "## Groupby User_ID"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 397,
   "id": "6f817477",
   "metadata": {},
   "outputs": [],
   "source": [
    "temp = df_final.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 398,
   "id": "d28ed748",
   "metadata": {},
   "outputs": [],
   "source": [
    "temp[\"User_ID\"] = temp[\"User_ID\"].str[0:11]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 399,
   "id": "1d4a1bc8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "12413"
      ]
     },
     "execution_count": 399,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(temp[\"User_ID\"].unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 400,
   "id": "322545c7",
   "metadata": {},
   "outputs": [],
   "source": [
    "temp = temp.groupby(\"User_ID\").mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 401,
   "id": "e9c12b96",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Personal</th>\n",
       "      <th>Theme</th>\n",
       "      <th>Discussion_day</th>\n",
       "      <th>comment_day</th>\n",
       "      <th>Participant_Type</th>\n",
       "      <th>Successful_Submission_Count</th>\n",
       "      <th>Comp_participation_day</th>\n",
       "      <th>FeatureA</th>\n",
       "      <th>FeatureB</th>\n",
       "      <th>FeatureC</th>\n",
       "      <th>...</th>\n",
       "      <th>Function_DE</th>\n",
       "      <th>Function_DM</th>\n",
       "      <th>Function_Bus</th>\n",
       "      <th>Function_AI</th>\n",
       "      <th>Function_ML</th>\n",
       "      <th>count_UA</th>\n",
       "      <th>Day_duration</th>\n",
       "      <th>FeatureX</th>\n",
       "      <th>FeatureY</th>\n",
       "      <th>User_sign-in_day</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>User_ID</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>ID_000H9XOP</th>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>22.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ID_003YJHQA</th>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>12.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ID_004UKSJV</th>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>1.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>14.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ID_008D2J6C</th>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ID_00ETFE7J</th>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ID_ZZMHNLNT</th>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ID_ZZQK0LU6</th>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>1.5</td>\n",
       "      <td>0.0</td>\n",
       "      <td>13.5</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>134.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>29.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ID_ZZUNZJ2U</th>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>30.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ID_ZZVM2KO7</th>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>22.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ID_ZZVUJ45W</th>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>52.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>12413 rows × 83 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             Personal  Theme  Discussion_day  comment_day  Participant_Type  \\\n",
       "User_ID                                                                       \n",
       "ID_000H9XOP      -1.0    0.0             0.0          0.0               0.0   \n",
       "ID_003YJHQA      -1.0    0.0             0.0          0.0               0.0   \n",
       "ID_004UKSJV      -1.0    0.0             0.0          0.0               0.0   \n",
       "ID_008D2J6C      -1.0    0.0             0.0          0.0               0.0   \n",
       "ID_00ETFE7J      -1.0    0.0             0.0          0.0               0.0   \n",
       "...               ...    ...             ...          ...               ...   \n",
       "ID_ZZMHNLNT      -1.0    0.0             0.0          0.0               0.0   \n",
       "ID_ZZQK0LU6      -1.0    0.0             0.0          0.0               1.0   \n",
       "ID_ZZUNZJ2U      -1.0    0.0             0.0          0.0               0.0   \n",
       "ID_ZZVM2KO7      -1.0    0.0             0.0          0.0               0.0   \n",
       "ID_ZZVUJ45W      -1.0    0.0             0.0          0.0               1.0   \n",
       "\n",
       "             Successful_Submission_Count  Comp_participation_day  FeatureA  \\\n",
       "User_ID                                                                      \n",
       "ID_000H9XOP                          0.0                     0.0      -1.0   \n",
       "ID_003YJHQA                          0.0                     0.0      -1.0   \n",
       "ID_004UKSJV                          0.0                     0.0      -1.0   \n",
       "ID_008D2J6C                          0.0                     0.0      -1.0   \n",
       "ID_00ETFE7J                          0.0                     0.0      -1.0   \n",
       "...                                  ...                     ...       ...   \n",
       "ID_ZZMHNLNT                          0.0                     0.0      -1.0   \n",
       "ID_ZZQK0LU6                          0.0                    29.0       1.5   \n",
       "ID_ZZUNZJ2U                          0.0                     0.0      -1.0   \n",
       "ID_ZZVM2KO7                          0.0                     0.0      -1.0   \n",
       "ID_ZZVUJ45W                          0.0                     5.0       1.0   \n",
       "\n",
       "             FeatureB  FeatureC  ...  Function_DE  Function_DM  Function_Bus  \\\n",
       "User_ID                          ...                                           \n",
       "ID_000H9XOP      -1.0      -1.0  ...          0.0          0.0           0.0   \n",
       "ID_003YJHQA      -1.0      -1.0  ...          0.0          0.0           0.0   \n",
       "ID_004UKSJV      -1.0      -1.0  ...          0.0          0.0           0.0   \n",
       "ID_008D2J6C      -1.0      -1.0  ...          0.0          0.0           0.0   \n",
       "ID_00ETFE7J      -1.0      -1.0  ...          0.0          0.0           0.0   \n",
       "...               ...       ...  ...          ...          ...           ...   \n",
       "ID_ZZMHNLNT      -1.0      -1.0  ...          0.0          0.0           0.0   \n",
       "ID_ZZQK0LU6       0.0      13.5  ...          0.0          0.0           0.0   \n",
       "ID_ZZUNZJ2U      -1.0      -1.0  ...          0.0          0.0           0.0   \n",
       "ID_ZZVM2KO7      -1.0      -1.0  ...          0.0          0.0           0.0   \n",
       "ID_ZZVUJ45W       7.0       4.0  ...          0.0          0.0           0.0   \n",
       "\n",
       "             Function_AI  Function_ML  count_UA  Day_duration  FeatureX  \\\n",
       "User_ID                                                                   \n",
       "ID_000H9XOP          0.0          0.0       6.0           5.0       0.0   \n",
       "ID_003YJHQA          0.0          0.0      11.0           1.0       0.0   \n",
       "ID_004UKSJV          0.0          0.0       6.0           1.5      -0.5   \n",
       "ID_008D2J6C          0.0          0.0      10.0           9.0       0.0   \n",
       "ID_00ETFE7J          0.0          0.0      10.0          10.0       0.0   \n",
       "...                  ...          ...       ...           ...       ...   \n",
       "ID_ZZMHNLNT          0.0          0.0       8.0           1.0       0.0   \n",
       "ID_ZZQK0LU6          0.0          0.0     134.0           2.0       0.0   \n",
       "ID_ZZUNZJ2U          0.0          0.0      12.0           1.0       0.0   \n",
       "ID_ZZVM2KO7          0.0          0.0       0.0           0.0       0.0   \n",
       "ID_ZZVUJ45W          0.0          0.0      52.0           1.0       0.0   \n",
       "\n",
       "             FeatureY  User_sign-in_day  \n",
       "User_ID                                  \n",
       "ID_000H9XOP       1.0              22.0  \n",
       "ID_003YJHQA       1.0              12.0  \n",
       "ID_004UKSJV      -0.5              14.0  \n",
       "ID_008D2J6C       0.0               7.0  \n",
       "ID_00ETFE7J       0.0               6.0  \n",
       "...               ...               ...  \n",
       "ID_ZZMHNLNT       0.0               1.0  \n",
       "ID_ZZQK0LU6       3.0              29.0  \n",
       "ID_ZZUNZJ2U       0.0              30.0  \n",
       "ID_ZZVM2KO7       0.0              22.0  \n",
       "ID_ZZVUJ45W       0.0               5.0  \n",
       "\n",
       "[12413 rows x 83 columns]"
      ]
     },
     "execution_count": 401,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "temp.sort_values(\"User_ID\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 402,
   "id": "af8312a2",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_max = temp.groupby('User_ID').max()\n",
    "df_sum = temp.groupby('User_ID').sum()\n",
    "df_mean = temp.groupby('User_ID').mean()\n",
    "df_min = temp.groupby('User_ID').min()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 403,
   "id": "f58c7d44",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_final = df_mean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 404,
   "id": "d54db73d",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_final[\"count_UA\"] = df_sum[\"count_UA\"]\n",
    "df_final[\"Discussion_day\"] = df_max[\"Discussion_day\"]\n",
    "df_final[\"Comp_participation_day\"] = df_max[\"Comp_participation_day\"]\n",
    "df_final[\"competition_ended_day\"] = df_max[\"competition_ended_day\"]\n",
    "df_final[\"competition_started_day\"] = df_min[\"competition_started_day\"]\n",
    "df_final[\"Day_last_active\"] = df_max[\"Day_last_active\"]\n",
    "df_final[\"User_sign-in_day\"] = df_min[\"User_sign-in_day\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 405,
   "id": "57e00fed",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Personal</th>\n",
       "      <th>Theme</th>\n",
       "      <th>Discussion_day</th>\n",
       "      <th>comment_day</th>\n",
       "      <th>Participant_Type</th>\n",
       "      <th>Successful_Submission_Count</th>\n",
       "      <th>Comp_participation_day</th>\n",
       "      <th>FeatureA</th>\n",
       "      <th>FeatureB</th>\n",
       "      <th>FeatureC</th>\n",
       "      <th>...</th>\n",
       "      <th>Function_DE</th>\n",
       "      <th>Function_DM</th>\n",
       "      <th>Function_Bus</th>\n",
       "      <th>Function_AI</th>\n",
       "      <th>Function_ML</th>\n",
       "      <th>count_UA</th>\n",
       "      <th>Day_duration</th>\n",
       "      <th>FeatureX</th>\n",
       "      <th>FeatureY</th>\n",
       "      <th>User_sign-in_day</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>User_ID</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>ID_000H9XOP</th>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>22.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ID_003YJHQA</th>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>12.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ID_004UKSJV</th>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>1.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>14.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ID_008D2J6C</th>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ID_00ETFE7J</th>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ID_ZZMHNLNT</th>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ID_ZZQK0LU6</th>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>1.5</td>\n",
       "      <td>0.0</td>\n",
       "      <td>13.5</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>134.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>29.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ID_ZZUNZJ2U</th>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>30.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ID_ZZVM2KO7</th>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>22.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ID_ZZVUJ45W</th>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>52.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>12413 rows × 83 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             Personal  Theme  Discussion_day  comment_day  Participant_Type  \\\n",
       "User_ID                                                                       \n",
       "ID_000H9XOP      -1.0    0.0             0.0          0.0               0.0   \n",
       "ID_003YJHQA      -1.0    0.0             0.0          0.0               0.0   \n",
       "ID_004UKSJV      -1.0    0.0             0.0          0.0               0.0   \n",
       "ID_008D2J6C      -1.0    0.0             0.0          0.0               0.0   \n",
       "ID_00ETFE7J      -1.0    0.0             0.0          0.0               0.0   \n",
       "...               ...    ...             ...          ...               ...   \n",
       "ID_ZZMHNLNT      -1.0    0.0             0.0          0.0               0.0   \n",
       "ID_ZZQK0LU6      -1.0    0.0             0.0          0.0               1.0   \n",
       "ID_ZZUNZJ2U      -1.0    0.0             0.0          0.0               0.0   \n",
       "ID_ZZVM2KO7      -1.0    0.0             0.0          0.0               0.0   \n",
       "ID_ZZVUJ45W      -1.0    0.0             0.0          0.0               1.0   \n",
       "\n",
       "             Successful_Submission_Count  Comp_participation_day  FeatureA  \\\n",
       "User_ID                                                                      \n",
       "ID_000H9XOP                          0.0                     0.0      -1.0   \n",
       "ID_003YJHQA                          0.0                     0.0      -1.0   \n",
       "ID_004UKSJV                          0.0                     0.0      -1.0   \n",
       "ID_008D2J6C                          0.0                     0.0      -1.0   \n",
       "ID_00ETFE7J                          0.0                     0.0      -1.0   \n",
       "...                                  ...                     ...       ...   \n",
       "ID_ZZMHNLNT                          0.0                     0.0      -1.0   \n",
       "ID_ZZQK0LU6                          0.0                    29.0       1.5   \n",
       "ID_ZZUNZJ2U                          0.0                     0.0      -1.0   \n",
       "ID_ZZVM2KO7                          0.0                     0.0      -1.0   \n",
       "ID_ZZVUJ45W                          0.0                     5.0       1.0   \n",
       "\n",
       "             FeatureB  FeatureC  ...  Function_DE  Function_DM  Function_Bus  \\\n",
       "User_ID                          ...                                           \n",
       "ID_000H9XOP      -1.0      -1.0  ...          0.0          0.0           0.0   \n",
       "ID_003YJHQA      -1.0      -1.0  ...          0.0          0.0           0.0   \n",
       "ID_004UKSJV      -1.0      -1.0  ...          0.0          0.0           0.0   \n",
       "ID_008D2J6C      -1.0      -1.0  ...          0.0          0.0           0.0   \n",
       "ID_00ETFE7J      -1.0      -1.0  ...          0.0          0.0           0.0   \n",
       "...               ...       ...  ...          ...          ...           ...   \n",
       "ID_ZZMHNLNT      -1.0      -1.0  ...          0.0          0.0           0.0   \n",
       "ID_ZZQK0LU6       0.0      13.5  ...          0.0          0.0           0.0   \n",
       "ID_ZZUNZJ2U      -1.0      -1.0  ...          0.0          0.0           0.0   \n",
       "ID_ZZVM2KO7      -1.0      -1.0  ...          0.0          0.0           0.0   \n",
       "ID_ZZVUJ45W       7.0       4.0  ...          0.0          0.0           0.0   \n",
       "\n",
       "             Function_AI  Function_ML  count_UA  Day_duration  FeatureX  \\\n",
       "User_ID                                                                   \n",
       "ID_000H9XOP          0.0          0.0       6.0           5.0       0.0   \n",
       "ID_003YJHQA          0.0          0.0      11.0           1.0       0.0   \n",
       "ID_004UKSJV          0.0          0.0       6.0           1.5      -0.5   \n",
       "ID_008D2J6C          0.0          0.0      10.0           9.0       0.0   \n",
       "ID_00ETFE7J          0.0          0.0      10.0          10.0       0.0   \n",
       "...                  ...          ...       ...           ...       ...   \n",
       "ID_ZZMHNLNT          0.0          0.0       8.0           1.0       0.0   \n",
       "ID_ZZQK0LU6          0.0          0.0     134.0           2.0       0.0   \n",
       "ID_ZZUNZJ2U          0.0          0.0      12.0           1.0       0.0   \n",
       "ID_ZZVM2KO7          0.0          0.0       0.0           0.0       0.0   \n",
       "ID_ZZVUJ45W          0.0          0.0      52.0           1.0       0.0   \n",
       "\n",
       "             FeatureY  User_sign-in_day  \n",
       "User_ID                                  \n",
       "ID_000H9XOP       1.0              22.0  \n",
       "ID_003YJHQA       1.0              12.0  \n",
       "ID_004UKSJV      -0.5              14.0  \n",
       "ID_008D2J6C       0.0               7.0  \n",
       "ID_00ETFE7J       0.0               6.0  \n",
       "...               ...               ...  \n",
       "ID_ZZMHNLNT       0.0               1.0  \n",
       "ID_ZZQK0LU6       3.0              29.0  \n",
       "ID_ZZUNZJ2U       0.0              30.0  \n",
       "ID_ZZVM2KO7       0.0              22.0  \n",
       "ID_ZZVUJ45W       0.0               5.0  \n",
       "\n",
       "[12413 rows x 83 columns]"
      ]
     },
     "execution_count": 405,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_mean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 406,
   "id": "a3709d89",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_final.reset_index(level=0, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 407,
   "id": "d0698bdd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 12413 entries, 0 to 12412\n",
      "Data columns (total 84 columns):\n",
      " #   Column                           Non-Null Count  Dtype  \n",
      "---  ------                           --------------  -----  \n",
      " 0   User_ID                          12413 non-null  object \n",
      " 1   Personal                         12413 non-null  float64\n",
      " 2   Theme                            12413 non-null  float64\n",
      " 3   Discussion_day                   12413 non-null  float64\n",
      " 4   comment_day                      12413 non-null  float64\n",
      " 5   Participant_Type                 12413 non-null  float64\n",
      " 6   Successful_Submission_Count      12413 non-null  float64\n",
      " 7   Comp_participation_day           12413 non-null  float64\n",
      " 8   FeatureA                         12413 non-null  float64\n",
      " 9   FeatureB                         12413 non-null  float64\n",
      " 10  FeatureC                         12413 non-null  float64\n",
      " 11  FeatureD                         12413 non-null  float64\n",
      " 12  FeatureE                         12413 non-null  float64\n",
      " 13  SecretCode                       12413 non-null  float64\n",
      " 14  FeatureF                         12413 non-null  float64\n",
      " 15  FeatureG                         12413 non-null  float64\n",
      " 16  FeatureH                         12413 non-null  float64\n",
      " 17  FeatureI                         12413 non-null  float64\n",
      " 18  Country_ID                       12413 non-null  float64\n",
      " 19  competition_ended_year           12413 non-null  float64\n",
      " 20  competition_ended_month          12413 non-null  float64\n",
      " 21  competition_ended_day            12413 non-null  float64\n",
      " 22  competition_started_year         12413 non-null  float64\n",
      " 23  competition_started_month        12413 non-null  float64\n",
      " 24  competition_started_day          12413 non-null  float64\n",
      " 25  Day_last_active                  12413 non-null  float64\n",
      " 26  Blog_Theme                       12413 non-null  float64\n",
      " 27  Blog_Published_Year              12413 non-null  float64\n",
      " 28  Blog_Published_Month             12413 non-null  float64\n",
      " 29  Published_At_Day_of_month        12413 non-null  float64\n",
      " 30  Remote                           12413 non-null  float64\n",
      " 31  Experience                       12413 non-null  float64\n",
      " 32  Employment_Type                  12413 non-null  float64\n",
      " 33  Company_Size                     12413 non-null  float64\n",
      " 34  Downloaded_Competition_Datafile  10400 non-null  float64\n",
      " 35  Viewed_All_Discussions           10400 non-null  float64\n",
      " 36  Confirmed_Email                  10400 non-null  float64\n",
      " 37  Joined_Competition               10400 non-null  float64\n",
      " 38  Signed_Up                        10400 non-null  float64\n",
      " 39  Viewed_All_Competitions          10400 non-null  float64\n",
      " 40  Viewed_All_Learning_Pages        10400 non-null  float64\n",
      " 41  badge_OCZE                       10400 non-null  float64\n",
      " 42  create_alias                     10400 non-null  float64\n",
      " 43  identify                         10400 non-null  float64\n",
      " 44  comp_ID                          10400 non-null  float64\n",
      " 45  Signed_In                        10400 non-null  float64\n",
      " 46  job_ID                           10400 non-null  float64\n",
      " 47  Updated_Discussion               10400 non-null  float64\n",
      " 48  Viewed_All_Jobs                  10400 non-null  float64\n",
      " 49  Created_Submission               10400 non-null  float64\n",
      " 50  Signed_Out                       10400 non-null  float64\n",
      " 51  Updated_Profile                  10400 non-null  float64\n",
      " 52  blog_ID                          10400 non-null  float64\n",
      " 53  Viewed_Discussion                10400 non-null  float64\n",
      " 54  Updated_Submission               10400 non-null  float64\n",
      " 55  badge_HYIO                       10400 non-null  float64\n",
      " 56  Industry_FS                      12413 non-null  float64\n",
      " 57  Industry_Bank                    12413 non-null  float64\n",
      " 58  Industry_Loc                     12413 non-null  float64\n",
      " 59  Industry_Gov                     12413 non-null  float64\n",
      " 60  Industry_Health                  12413 non-null  float64\n",
      " 61  Industry_Res                     12413 non-null  float64\n",
      " 62  Industry_Amb                     12413 non-null  float64\n",
      " 63  Industry_Transp                  12413 non-null  float64\n",
      " 64  Industry_Scholar                 12413 non-null  float64\n",
      " 65  Industry_Log                     12413 non-null  float64\n",
      " 66  Industry_Media                   12413 non-null  float64\n",
      " 67  Industry_Manuf                   12413 non-null  float64\n",
      " 68  Industry_Sales                   12413 non-null  float64\n",
      " 69  Industry_Insur                   12413 non-null  float64\n",
      " 70  Industry_Manage                  12413 non-null  float64\n",
      " 71  Industry_AI                      12413 non-null  float64\n",
      " 72  Industry_Zindi                   12413 non-null  float64\n",
      " 73  Industry_Tech                    12413 non-null  float64\n",
      " 74  Function_DE                      12413 non-null  float64\n",
      " 75  Function_DM                      12413 non-null  float64\n",
      " 76  Function_Bus                     12413 non-null  float64\n",
      " 77  Function_AI                      12413 non-null  float64\n",
      " 78  Function_ML                      12413 non-null  float64\n",
      " 79  count_UA                         12413 non-null  float64\n",
      " 80  Day_duration                     12413 non-null  float64\n",
      " 81  FeatureX                         12413 non-null  float64\n",
      " 82  FeatureY                         12413 non-null  float64\n",
      " 83  User_sign-in_day                 12413 non-null  float64\n",
      "dtypes: float64(83), object(1)\n",
      "memory usage: 8.0+ MB\n"
     ]
    }
   ],
   "source": [
    "df_final.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 408,
   "id": "a754a1ca",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>User_ID</th>\n",
       "      <th>Personal</th>\n",
       "      <th>Theme</th>\n",
       "      <th>Discussion_day</th>\n",
       "      <th>comment_day</th>\n",
       "      <th>Participant_Type</th>\n",
       "      <th>Successful_Submission_Count</th>\n",
       "      <th>Comp_participation_day</th>\n",
       "      <th>FeatureA</th>\n",
       "      <th>FeatureB</th>\n",
       "      <th>...</th>\n",
       "      <th>Function_DE</th>\n",
       "      <th>Function_DM</th>\n",
       "      <th>Function_Bus</th>\n",
       "      <th>Function_AI</th>\n",
       "      <th>Function_ML</th>\n",
       "      <th>count_UA</th>\n",
       "      <th>Day_duration</th>\n",
       "      <th>FeatureX</th>\n",
       "      <th>FeatureY</th>\n",
       "      <th>User_sign-in_day</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ID_000H9XOP</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>22.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ID_003YJHQA</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>12.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ID_004UKSJV</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>1.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>14.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ID_008D2J6C</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ID_00ETFE7J</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12408</th>\n",
       "      <td>ID_ZZMHNLNT</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12409</th>\n",
       "      <td>ID_ZZQK0LU6</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>1.5</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>134.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>29.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12410</th>\n",
       "      <td>ID_ZZUNZJ2U</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>30.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12411</th>\n",
       "      <td>ID_ZZVM2KO7</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>22.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12412</th>\n",
       "      <td>ID_ZZVUJ45W</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>52.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>12413 rows × 84 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           User_ID  Personal  Theme  Discussion_day  comment_day  \\\n",
       "0      ID_000H9XOP      -1.0    0.0             0.0          0.0   \n",
       "1      ID_003YJHQA      -1.0    0.0             0.0          0.0   \n",
       "2      ID_004UKSJV      -1.0    0.0             0.0          0.0   \n",
       "3      ID_008D2J6C      -1.0    0.0             0.0          0.0   \n",
       "4      ID_00ETFE7J      -1.0    0.0             0.0          0.0   \n",
       "...            ...       ...    ...             ...          ...   \n",
       "12408  ID_ZZMHNLNT      -1.0    0.0             0.0          0.0   \n",
       "12409  ID_ZZQK0LU6      -1.0    0.0             0.0          0.0   \n",
       "12410  ID_ZZUNZJ2U      -1.0    0.0             0.0          0.0   \n",
       "12411  ID_ZZVM2KO7      -1.0    0.0             0.0          0.0   \n",
       "12412  ID_ZZVUJ45W      -1.0    0.0             0.0          0.0   \n",
       "\n",
       "       Participant_Type  Successful_Submission_Count  Comp_participation_day  \\\n",
       "0                   0.0                          0.0                     0.0   \n",
       "1                   0.0                          0.0                     0.0   \n",
       "2                   0.0                          0.0                     0.0   \n",
       "3                   0.0                          0.0                     0.0   \n",
       "4                   0.0                          0.0                     0.0   \n",
       "...                 ...                          ...                     ...   \n",
       "12408               0.0                          0.0                     0.0   \n",
       "12409               1.0                          0.0                    29.0   \n",
       "12410               0.0                          0.0                     0.0   \n",
       "12411               0.0                          0.0                     0.0   \n",
       "12412               1.0                          0.0                     5.0   \n",
       "\n",
       "       FeatureA  FeatureB  ...  Function_DE  Function_DM  Function_Bus  \\\n",
       "0          -1.0      -1.0  ...          0.0          0.0           0.0   \n",
       "1          -1.0      -1.0  ...          0.0          0.0           0.0   \n",
       "2          -1.0      -1.0  ...          0.0          0.0           0.0   \n",
       "3          -1.0      -1.0  ...          0.0          0.0           0.0   \n",
       "4          -1.0      -1.0  ...          0.0          0.0           0.0   \n",
       "...         ...       ...  ...          ...          ...           ...   \n",
       "12408      -1.0      -1.0  ...          0.0          0.0           0.0   \n",
       "12409       1.5       0.0  ...          0.0          0.0           0.0   \n",
       "12410      -1.0      -1.0  ...          0.0          0.0           0.0   \n",
       "12411      -1.0      -1.0  ...          0.0          0.0           0.0   \n",
       "12412       1.0       7.0  ...          0.0          0.0           0.0   \n",
       "\n",
       "       Function_AI  Function_ML  count_UA  Day_duration  FeatureX  FeatureY  \\\n",
       "0              0.0          0.0       6.0           5.0       0.0       1.0   \n",
       "1              0.0          0.0      11.0           1.0       0.0       1.0   \n",
       "2              0.0          0.0       6.0           1.5      -0.5      -0.5   \n",
       "3              0.0          0.0      10.0           9.0       0.0       0.0   \n",
       "4              0.0          0.0      10.0          10.0       0.0       0.0   \n",
       "...            ...          ...       ...           ...       ...       ...   \n",
       "12408          0.0          0.0       8.0           1.0       0.0       0.0   \n",
       "12409          0.0          0.0     134.0           2.0       0.0       3.0   \n",
       "12410          0.0          0.0      12.0           1.0       0.0       0.0   \n",
       "12411          0.0          0.0       0.0           0.0       0.0       0.0   \n",
       "12412          0.0          0.0      52.0           1.0       0.0       0.0   \n",
       "\n",
       "       User_sign-in_day  \n",
       "0                  22.0  \n",
       "1                  12.0  \n",
       "2                  14.0  \n",
       "3                   7.0  \n",
       "4                   6.0  \n",
       "...                 ...  \n",
       "12408               1.0  \n",
       "12409              29.0  \n",
       "12410              30.0  \n",
       "12411              22.0  \n",
       "12412               5.0  \n",
       "\n",
       "[12413 rows x 84 columns]"
      ]
     },
     "execution_count": 408,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_final"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 409,
   "id": "cf20f44d",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_final.to_csv('Dataset1.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c53e445c",
   "metadata": {},
   "source": [
    "## Normalization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 410,
   "id": "70bcefaf",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_temp = df_final.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 411,
   "id": "ad38357e",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_temp.drop(\"User_ID\", inplace=True, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 412,
   "id": "d11695d9",
   "metadata": {},
   "outputs": [],
   "source": [
    "for column in df_temp.columns:\n",
    "    df_temp[column] = df_temp[column]  / df_temp[column].abs().max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 413,
   "id": "2aef3570",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Personal</th>\n",
       "      <th>Theme</th>\n",
       "      <th>Discussion_day</th>\n",
       "      <th>comment_day</th>\n",
       "      <th>Participant_Type</th>\n",
       "      <th>Successful_Submission_Count</th>\n",
       "      <th>Comp_participation_day</th>\n",
       "      <th>FeatureA</th>\n",
       "      <th>FeatureB</th>\n",
       "      <th>FeatureC</th>\n",
       "      <th>...</th>\n",
       "      <th>Function_DE</th>\n",
       "      <th>Function_DM</th>\n",
       "      <th>Function_Bus</th>\n",
       "      <th>Function_AI</th>\n",
       "      <th>Function_ML</th>\n",
       "      <th>count_UA</th>\n",
       "      <th>Day_duration</th>\n",
       "      <th>FeatureX</th>\n",
       "      <th>FeatureY</th>\n",
       "      <th>User_sign-in_day</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.1250</td>\n",
       "      <td>-0.0625</td>\n",
       "      <td>-0.028571</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000527</td>\n",
       "      <td>0.161290</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.709677</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.1250</td>\n",
       "      <td>-0.0625</td>\n",
       "      <td>-0.028571</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000967</td>\n",
       "      <td>0.032258</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.387097</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.1250</td>\n",
       "      <td>-0.0625</td>\n",
       "      <td>-0.028571</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000527</td>\n",
       "      <td>0.048387</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.166667</td>\n",
       "      <td>0.451613</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.1250</td>\n",
       "      <td>-0.0625</td>\n",
       "      <td>-0.028571</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000879</td>\n",
       "      <td>0.290323</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.225806</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.1250</td>\n",
       "      <td>-0.0625</td>\n",
       "      <td>-0.028571</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000879</td>\n",
       "      <td>0.322581</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.193548</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12408</th>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.1250</td>\n",
       "      <td>-0.0625</td>\n",
       "      <td>-0.028571</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000703</td>\n",
       "      <td>0.032258</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.032258</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12409</th>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.935484</td>\n",
       "      <td>0.1875</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.385714</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.011776</td>\n",
       "      <td>0.064516</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.935484</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12410</th>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.1250</td>\n",
       "      <td>-0.0625</td>\n",
       "      <td>-0.028571</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001055</td>\n",
       "      <td>0.032258</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.967742</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12411</th>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.1250</td>\n",
       "      <td>-0.0625</td>\n",
       "      <td>-0.028571</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.709677</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12412</th>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.161290</td>\n",
       "      <td>0.1250</td>\n",
       "      <td>0.4375</td>\n",
       "      <td>0.114286</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.004570</td>\n",
       "      <td>0.032258</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.161290</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>12413 rows × 83 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       Personal  Theme  Discussion_day  comment_day  Participant_Type  \\\n",
       "0          -1.0    0.0             0.0          0.0               0.0   \n",
       "1          -1.0    0.0             0.0          0.0               0.0   \n",
       "2          -1.0    0.0             0.0          0.0               0.0   \n",
       "3          -1.0    0.0             0.0          0.0               0.0   \n",
       "4          -1.0    0.0             0.0          0.0               0.0   \n",
       "...         ...    ...             ...          ...               ...   \n",
       "12408      -1.0    0.0             0.0          0.0               0.0   \n",
       "12409      -1.0    0.0             0.0          0.0               1.0   \n",
       "12410      -1.0    0.0             0.0          0.0               0.0   \n",
       "12411      -1.0    0.0             0.0          0.0               0.0   \n",
       "12412      -1.0    0.0             0.0          0.0               1.0   \n",
       "\n",
       "       Successful_Submission_Count  Comp_participation_day  FeatureA  \\\n",
       "0                              0.0                0.000000   -0.1250   \n",
       "1                              0.0                0.000000   -0.1250   \n",
       "2                              0.0                0.000000   -0.1250   \n",
       "3                              0.0                0.000000   -0.1250   \n",
       "4                              0.0                0.000000   -0.1250   \n",
       "...                            ...                     ...       ...   \n",
       "12408                          0.0                0.000000   -0.1250   \n",
       "12409                          0.0                0.935484    0.1875   \n",
       "12410                          0.0                0.000000   -0.1250   \n",
       "12411                          0.0                0.000000   -0.1250   \n",
       "12412                          0.0                0.161290    0.1250   \n",
       "\n",
       "       FeatureB  FeatureC  ...  Function_DE  Function_DM  Function_Bus  \\\n",
       "0       -0.0625 -0.028571  ...          0.0          0.0           0.0   \n",
       "1       -0.0625 -0.028571  ...          0.0          0.0           0.0   \n",
       "2       -0.0625 -0.028571  ...          0.0          0.0           0.0   \n",
       "3       -0.0625 -0.028571  ...          0.0          0.0           0.0   \n",
       "4       -0.0625 -0.028571  ...          0.0          0.0           0.0   \n",
       "...         ...       ...  ...          ...          ...           ...   \n",
       "12408   -0.0625 -0.028571  ...          0.0          0.0           0.0   \n",
       "12409    0.0000  0.385714  ...          0.0          0.0           0.0   \n",
       "12410   -0.0625 -0.028571  ...          0.0          0.0           0.0   \n",
       "12411   -0.0625 -0.028571  ...          0.0          0.0           0.0   \n",
       "12412    0.4375  0.114286  ...          0.0          0.0           0.0   \n",
       "\n",
       "       Function_AI  Function_ML  count_UA  Day_duration  FeatureX  FeatureY  \\\n",
       "0              0.0          0.0  0.000527      0.161290       0.0  0.333333   \n",
       "1              0.0          0.0  0.000967      0.032258       0.0  0.333333   \n",
       "2              0.0          0.0  0.000527      0.048387      -0.5 -0.166667   \n",
       "3              0.0          0.0  0.000879      0.290323       0.0  0.000000   \n",
       "4              0.0          0.0  0.000879      0.322581       0.0  0.000000   \n",
       "...            ...          ...       ...           ...       ...       ...   \n",
       "12408          0.0          0.0  0.000703      0.032258       0.0  0.000000   \n",
       "12409          0.0          0.0  0.011776      0.064516       0.0  1.000000   \n",
       "12410          0.0          0.0  0.001055      0.032258       0.0  0.000000   \n",
       "12411          0.0          0.0  0.000000      0.000000       0.0  0.000000   \n",
       "12412          0.0          0.0  0.004570      0.032258       0.0  0.000000   \n",
       "\n",
       "       User_sign-in_day  \n",
       "0              0.709677  \n",
       "1              0.387097  \n",
       "2              0.451613  \n",
       "3              0.225806  \n",
       "4              0.193548  \n",
       "...                 ...  \n",
       "12408          0.032258  \n",
       "12409          0.935484  \n",
       "12410          0.967742  \n",
       "12411          0.709677  \n",
       "12412          0.161290  \n",
       "\n",
       "[12413 rows x 83 columns]"
      ]
     },
     "execution_count": 413,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_temp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 414,
   "id": "79ed0d71",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_temp.insert(0, 'User_ID', np.array(df_final[\"User_ID\"]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 415,
   "id": "0f14f09a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>User_ID</th>\n",
       "      <th>Personal</th>\n",
       "      <th>Theme</th>\n",
       "      <th>Discussion_day</th>\n",
       "      <th>comment_day</th>\n",
       "      <th>Participant_Type</th>\n",
       "      <th>Successful_Submission_Count</th>\n",
       "      <th>Comp_participation_day</th>\n",
       "      <th>FeatureA</th>\n",
       "      <th>FeatureB</th>\n",
       "      <th>...</th>\n",
       "      <th>Function_DE</th>\n",
       "      <th>Function_DM</th>\n",
       "      <th>Function_Bus</th>\n",
       "      <th>Function_AI</th>\n",
       "      <th>Function_ML</th>\n",
       "      <th>count_UA</th>\n",
       "      <th>Day_duration</th>\n",
       "      <th>FeatureX</th>\n",
       "      <th>FeatureY</th>\n",
       "      <th>User_sign-in_day</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ID_000H9XOP</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.1250</td>\n",
       "      <td>-0.0625</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000527</td>\n",
       "      <td>0.161290</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.709677</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ID_003YJHQA</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.1250</td>\n",
       "      <td>-0.0625</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000967</td>\n",
       "      <td>0.032258</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.387097</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ID_004UKSJV</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.1250</td>\n",
       "      <td>-0.0625</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000527</td>\n",
       "      <td>0.048387</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.166667</td>\n",
       "      <td>0.451613</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ID_008D2J6C</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.1250</td>\n",
       "      <td>-0.0625</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000879</td>\n",
       "      <td>0.290323</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.225806</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ID_00ETFE7J</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.1250</td>\n",
       "      <td>-0.0625</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000879</td>\n",
       "      <td>0.322581</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.193548</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12408</th>\n",
       "      <td>ID_ZZMHNLNT</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.1250</td>\n",
       "      <td>-0.0625</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000703</td>\n",
       "      <td>0.032258</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.032258</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12409</th>\n",
       "      <td>ID_ZZQK0LU6</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.935484</td>\n",
       "      <td>0.1875</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.011776</td>\n",
       "      <td>0.064516</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.935484</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12410</th>\n",
       "      <td>ID_ZZUNZJ2U</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.1250</td>\n",
       "      <td>-0.0625</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001055</td>\n",
       "      <td>0.032258</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.967742</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12411</th>\n",
       "      <td>ID_ZZVM2KO7</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.1250</td>\n",
       "      <td>-0.0625</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.709677</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12412</th>\n",
       "      <td>ID_ZZVUJ45W</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.161290</td>\n",
       "      <td>0.1250</td>\n",
       "      <td>0.4375</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.004570</td>\n",
       "      <td>0.032258</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.161290</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>12413 rows × 84 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           User_ID  Personal  Theme  Discussion_day  comment_day  \\\n",
       "0      ID_000H9XOP      -1.0    0.0             0.0          0.0   \n",
       "1      ID_003YJHQA      -1.0    0.0             0.0          0.0   \n",
       "2      ID_004UKSJV      -1.0    0.0             0.0          0.0   \n",
       "3      ID_008D2J6C      -1.0    0.0             0.0          0.0   \n",
       "4      ID_00ETFE7J      -1.0    0.0             0.0          0.0   \n",
       "...            ...       ...    ...             ...          ...   \n",
       "12408  ID_ZZMHNLNT      -1.0    0.0             0.0          0.0   \n",
       "12409  ID_ZZQK0LU6      -1.0    0.0             0.0          0.0   \n",
       "12410  ID_ZZUNZJ2U      -1.0    0.0             0.0          0.0   \n",
       "12411  ID_ZZVM2KO7      -1.0    0.0             0.0          0.0   \n",
       "12412  ID_ZZVUJ45W      -1.0    0.0             0.0          0.0   \n",
       "\n",
       "       Participant_Type  Successful_Submission_Count  Comp_participation_day  \\\n",
       "0                   0.0                          0.0                0.000000   \n",
       "1                   0.0                          0.0                0.000000   \n",
       "2                   0.0                          0.0                0.000000   \n",
       "3                   0.0                          0.0                0.000000   \n",
       "4                   0.0                          0.0                0.000000   \n",
       "...                 ...                          ...                     ...   \n",
       "12408               0.0                          0.0                0.000000   \n",
       "12409               1.0                          0.0                0.935484   \n",
       "12410               0.0                          0.0                0.000000   \n",
       "12411               0.0                          0.0                0.000000   \n",
       "12412               1.0                          0.0                0.161290   \n",
       "\n",
       "       FeatureA  FeatureB  ...  Function_DE  Function_DM  Function_Bus  \\\n",
       "0       -0.1250   -0.0625  ...          0.0          0.0           0.0   \n",
       "1       -0.1250   -0.0625  ...          0.0          0.0           0.0   \n",
       "2       -0.1250   -0.0625  ...          0.0          0.0           0.0   \n",
       "3       -0.1250   -0.0625  ...          0.0          0.0           0.0   \n",
       "4       -0.1250   -0.0625  ...          0.0          0.0           0.0   \n",
       "...         ...       ...  ...          ...          ...           ...   \n",
       "12408   -0.1250   -0.0625  ...          0.0          0.0           0.0   \n",
       "12409    0.1875    0.0000  ...          0.0          0.0           0.0   \n",
       "12410   -0.1250   -0.0625  ...          0.0          0.0           0.0   \n",
       "12411   -0.1250   -0.0625  ...          0.0          0.0           0.0   \n",
       "12412    0.1250    0.4375  ...          0.0          0.0           0.0   \n",
       "\n",
       "       Function_AI  Function_ML  count_UA  Day_duration  FeatureX  FeatureY  \\\n",
       "0              0.0          0.0  0.000527      0.161290       0.0  0.333333   \n",
       "1              0.0          0.0  0.000967      0.032258       0.0  0.333333   \n",
       "2              0.0          0.0  0.000527      0.048387      -0.5 -0.166667   \n",
       "3              0.0          0.0  0.000879      0.290323       0.0  0.000000   \n",
       "4              0.0          0.0  0.000879      0.322581       0.0  0.000000   \n",
       "...            ...          ...       ...           ...       ...       ...   \n",
       "12408          0.0          0.0  0.000703      0.032258       0.0  0.000000   \n",
       "12409          0.0          0.0  0.011776      0.064516       0.0  1.000000   \n",
       "12410          0.0          0.0  0.001055      0.032258       0.0  0.000000   \n",
       "12411          0.0          0.0  0.000000      0.000000       0.0  0.000000   \n",
       "12412          0.0          0.0  0.004570      0.032258       0.0  0.000000   \n",
       "\n",
       "       User_sign-in_day  \n",
       "0              0.709677  \n",
       "1              0.387097  \n",
       "2              0.451613  \n",
       "3              0.225806  \n",
       "4              0.193548  \n",
       "...                 ...  \n",
       "12408          0.032258  \n",
       "12409          0.935484  \n",
       "12410          0.967742  \n",
       "12411          0.709677  \n",
       "12412          0.161290  \n",
       "\n",
       "[12413 rows x 84 columns]"
      ]
     },
     "execution_count": 415,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_temp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 416,
   "id": "2294dd07",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_final = df_temp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 417,
   "id": "f9c65771",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 12413 entries, 0 to 12412\n",
      "Data columns (total 84 columns):\n",
      " #   Column                           Non-Null Count  Dtype  \n",
      "---  ------                           --------------  -----  \n",
      " 0   User_ID                          12413 non-null  object \n",
      " 1   Personal                         12413 non-null  float64\n",
      " 2   Theme                            12413 non-null  float64\n",
      " 3   Discussion_day                   12413 non-null  float64\n",
      " 4   comment_day                      12413 non-null  float64\n",
      " 5   Participant_Type                 12413 non-null  float64\n",
      " 6   Successful_Submission_Count      12413 non-null  float64\n",
      " 7   Comp_participation_day           12413 non-null  float64\n",
      " 8   FeatureA                         12413 non-null  float64\n",
      " 9   FeatureB                         12413 non-null  float64\n",
      " 10  FeatureC                         12413 non-null  float64\n",
      " 11  FeatureD                         12413 non-null  float64\n",
      " 12  FeatureE                         12413 non-null  float64\n",
      " 13  SecretCode                       12413 non-null  float64\n",
      " 14  FeatureF                         12413 non-null  float64\n",
      " 15  FeatureG                         12413 non-null  float64\n",
      " 16  FeatureH                         12413 non-null  float64\n",
      " 17  FeatureI                         12413 non-null  float64\n",
      " 18  Country_ID                       12413 non-null  float64\n",
      " 19  competition_ended_year           12413 non-null  float64\n",
      " 20  competition_ended_month          12413 non-null  float64\n",
      " 21  competition_ended_day            12413 non-null  float64\n",
      " 22  competition_started_year         12413 non-null  float64\n",
      " 23  competition_started_month        12413 non-null  float64\n",
      " 24  competition_started_day          12413 non-null  float64\n",
      " 25  Day_last_active                  12413 non-null  float64\n",
      " 26  Blog_Theme                       12413 non-null  float64\n",
      " 27  Blog_Published_Year              12413 non-null  float64\n",
      " 28  Blog_Published_Month             12413 non-null  float64\n",
      " 29  Published_At_Day_of_month        12413 non-null  float64\n",
      " 30  Remote                           12413 non-null  float64\n",
      " 31  Experience                       12413 non-null  float64\n",
      " 32  Employment_Type                  12413 non-null  float64\n",
      " 33  Company_Size                     12413 non-null  float64\n",
      " 34  Downloaded_Competition_Datafile  10400 non-null  float64\n",
      " 35  Viewed_All_Discussions           10400 non-null  float64\n",
      " 36  Confirmed_Email                  10400 non-null  float64\n",
      " 37  Joined_Competition               10400 non-null  float64\n",
      " 38  Signed_Up                        10400 non-null  float64\n",
      " 39  Viewed_All_Competitions          10400 non-null  float64\n",
      " 40  Viewed_All_Learning_Pages        10400 non-null  float64\n",
      " 41  badge_OCZE                       10400 non-null  float64\n",
      " 42  create_alias                     10400 non-null  float64\n",
      " 43  identify                         10400 non-null  float64\n",
      " 44  comp_ID                          10400 non-null  float64\n",
      " 45  Signed_In                        10400 non-null  float64\n",
      " 46  job_ID                           10400 non-null  float64\n",
      " 47  Updated_Discussion               10400 non-null  float64\n",
      " 48  Viewed_All_Jobs                  10400 non-null  float64\n",
      " 49  Created_Submission               10400 non-null  float64\n",
      " 50  Signed_Out                       10400 non-null  float64\n",
      " 51  Updated_Profile                  10400 non-null  float64\n",
      " 52  blog_ID                          10400 non-null  float64\n",
      " 53  Viewed_Discussion                10400 non-null  float64\n",
      " 54  Updated_Submission               10400 non-null  float64\n",
      " 55  badge_HYIO                       10400 non-null  float64\n",
      " 56  Industry_FS                      12413 non-null  float64\n",
      " 57  Industry_Bank                    12413 non-null  float64\n",
      " 58  Industry_Loc                     12413 non-null  float64\n",
      " 59  Industry_Gov                     12413 non-null  float64\n",
      " 60  Industry_Health                  12413 non-null  float64\n",
      " 61  Industry_Res                     12413 non-null  float64\n",
      " 62  Industry_Amb                     12413 non-null  float64\n",
      " 63  Industry_Transp                  12413 non-null  float64\n",
      " 64  Industry_Scholar                 12413 non-null  float64\n",
      " 65  Industry_Log                     12413 non-null  float64\n",
      " 66  Industry_Media                   12413 non-null  float64\n",
      " 67  Industry_Manuf                   12413 non-null  float64\n",
      " 68  Industry_Sales                   12413 non-null  float64\n",
      " 69  Industry_Insur                   12413 non-null  float64\n",
      " 70  Industry_Manage                  12413 non-null  float64\n",
      " 71  Industry_AI                      12413 non-null  float64\n",
      " 72  Industry_Zindi                   12413 non-null  float64\n",
      " 73  Industry_Tech                    12413 non-null  float64\n",
      " 74  Function_DE                      12413 non-null  float64\n",
      " 75  Function_DM                      12413 non-null  float64\n",
      " 76  Function_Bus                     12413 non-null  float64\n",
      " 77  Function_AI                      12413 non-null  float64\n",
      " 78  Function_ML                      12413 non-null  float64\n",
      " 79  count_UA                         12413 non-null  float64\n",
      " 80  Day_duration                     12413 non-null  float64\n",
      " 81  FeatureX                         12413 non-null  float64\n",
      " 82  FeatureY                         12413 non-null  float64\n",
      " 83  User_sign-in_day                 12413 non-null  float64\n",
      "dtypes: float64(83), object(1)\n",
      "memory usage: 8.0+ MB\n"
     ]
    }
   ],
   "source": [
    "df_final.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 418,
   "id": "7adef9b7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>User_ID</th>\n",
       "      <th>Personal</th>\n",
       "      <th>Theme</th>\n",
       "      <th>Discussion_day</th>\n",
       "      <th>comment_day</th>\n",
       "      <th>Participant_Type</th>\n",
       "      <th>Successful_Submission_Count</th>\n",
       "      <th>Comp_participation_day</th>\n",
       "      <th>FeatureA</th>\n",
       "      <th>FeatureB</th>\n",
       "      <th>...</th>\n",
       "      <th>Function_DE</th>\n",
       "      <th>Function_DM</th>\n",
       "      <th>Function_Bus</th>\n",
       "      <th>Function_AI</th>\n",
       "      <th>Function_ML</th>\n",
       "      <th>count_UA</th>\n",
       "      <th>Day_duration</th>\n",
       "      <th>FeatureX</th>\n",
       "      <th>FeatureY</th>\n",
       "      <th>User_sign-in_day</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ID_000H9XOP</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.1250</td>\n",
       "      <td>-0.0625</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000527</td>\n",
       "      <td>0.161290</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.709677</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ID_003YJHQA</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.1250</td>\n",
       "      <td>-0.0625</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000967</td>\n",
       "      <td>0.032258</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.387097</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ID_004UKSJV</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.1250</td>\n",
       "      <td>-0.0625</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000527</td>\n",
       "      <td>0.048387</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.166667</td>\n",
       "      <td>0.451613</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ID_008D2J6C</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.1250</td>\n",
       "      <td>-0.0625</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000879</td>\n",
       "      <td>0.290323</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.225806</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ID_00ETFE7J</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.1250</td>\n",
       "      <td>-0.0625</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000879</td>\n",
       "      <td>0.322581</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.193548</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12408</th>\n",
       "      <td>ID_ZZMHNLNT</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.1250</td>\n",
       "      <td>-0.0625</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000703</td>\n",
       "      <td>0.032258</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.032258</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12409</th>\n",
       "      <td>ID_ZZQK0LU6</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.935484</td>\n",
       "      <td>0.1875</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.011776</td>\n",
       "      <td>0.064516</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.935484</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12410</th>\n",
       "      <td>ID_ZZUNZJ2U</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.1250</td>\n",
       "      <td>-0.0625</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001055</td>\n",
       "      <td>0.032258</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.967742</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12411</th>\n",
       "      <td>ID_ZZVM2KO7</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.1250</td>\n",
       "      <td>-0.0625</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.709677</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12412</th>\n",
       "      <td>ID_ZZVUJ45W</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.161290</td>\n",
       "      <td>0.1250</td>\n",
       "      <td>0.4375</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.004570</td>\n",
       "      <td>0.032258</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.161290</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>12413 rows × 84 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           User_ID  Personal  Theme  Discussion_day  comment_day  \\\n",
       "0      ID_000H9XOP      -1.0    0.0             0.0          0.0   \n",
       "1      ID_003YJHQA      -1.0    0.0             0.0          0.0   \n",
       "2      ID_004UKSJV      -1.0    0.0             0.0          0.0   \n",
       "3      ID_008D2J6C      -1.0    0.0             0.0          0.0   \n",
       "4      ID_00ETFE7J      -1.0    0.0             0.0          0.0   \n",
       "...            ...       ...    ...             ...          ...   \n",
       "12408  ID_ZZMHNLNT      -1.0    0.0             0.0          0.0   \n",
       "12409  ID_ZZQK0LU6      -1.0    0.0             0.0          0.0   \n",
       "12410  ID_ZZUNZJ2U      -1.0    0.0             0.0          0.0   \n",
       "12411  ID_ZZVM2KO7      -1.0    0.0             0.0          0.0   \n",
       "12412  ID_ZZVUJ45W      -1.0    0.0             0.0          0.0   \n",
       "\n",
       "       Participant_Type  Successful_Submission_Count  Comp_participation_day  \\\n",
       "0                   0.0                          0.0                0.000000   \n",
       "1                   0.0                          0.0                0.000000   \n",
       "2                   0.0                          0.0                0.000000   \n",
       "3                   0.0                          0.0                0.000000   \n",
       "4                   0.0                          0.0                0.000000   \n",
       "...                 ...                          ...                     ...   \n",
       "12408               0.0                          0.0                0.000000   \n",
       "12409               1.0                          0.0                0.935484   \n",
       "12410               0.0                          0.0                0.000000   \n",
       "12411               0.0                          0.0                0.000000   \n",
       "12412               1.0                          0.0                0.161290   \n",
       "\n",
       "       FeatureA  FeatureB  ...  Function_DE  Function_DM  Function_Bus  \\\n",
       "0       -0.1250   -0.0625  ...          0.0          0.0           0.0   \n",
       "1       -0.1250   -0.0625  ...          0.0          0.0           0.0   \n",
       "2       -0.1250   -0.0625  ...          0.0          0.0           0.0   \n",
       "3       -0.1250   -0.0625  ...          0.0          0.0           0.0   \n",
       "4       -0.1250   -0.0625  ...          0.0          0.0           0.0   \n",
       "...         ...       ...  ...          ...          ...           ...   \n",
       "12408   -0.1250   -0.0625  ...          0.0          0.0           0.0   \n",
       "12409    0.1875    0.0000  ...          0.0          0.0           0.0   \n",
       "12410   -0.1250   -0.0625  ...          0.0          0.0           0.0   \n",
       "12411   -0.1250   -0.0625  ...          0.0          0.0           0.0   \n",
       "12412    0.1250    0.4375  ...          0.0          0.0           0.0   \n",
       "\n",
       "       Function_AI  Function_ML  count_UA  Day_duration  FeatureX  FeatureY  \\\n",
       "0              0.0          0.0  0.000527      0.161290       0.0  0.333333   \n",
       "1              0.0          0.0  0.000967      0.032258       0.0  0.333333   \n",
       "2              0.0          0.0  0.000527      0.048387      -0.5 -0.166667   \n",
       "3              0.0          0.0  0.000879      0.290323       0.0  0.000000   \n",
       "4              0.0          0.0  0.000879      0.322581       0.0  0.000000   \n",
       "...            ...          ...       ...           ...       ...       ...   \n",
       "12408          0.0          0.0  0.000703      0.032258       0.0  0.000000   \n",
       "12409          0.0          0.0  0.011776      0.064516       0.0  1.000000   \n",
       "12410          0.0          0.0  0.001055      0.032258       0.0  0.000000   \n",
       "12411          0.0          0.0  0.000000      0.000000       0.0  0.000000   \n",
       "12412          0.0          0.0  0.004570      0.032258       0.0  0.000000   \n",
       "\n",
       "       User_sign-in_day  \n",
       "0              0.709677  \n",
       "1              0.387097  \n",
       "2              0.451613  \n",
       "3              0.225806  \n",
       "4              0.193548  \n",
       "...                 ...  \n",
       "12408          0.032258  \n",
       "12409          0.935484  \n",
       "12410          0.967742  \n",
       "12411          0.709677  \n",
       "12412          0.161290  \n",
       "\n",
       "[12413 rows x 84 columns]"
      ]
     },
     "execution_count": 418,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_final"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 419,
   "id": "1bb3155b",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_final.to_csv('DatasetNorm.csv')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}