data_prep.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "data_prep.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "O38kMwEQITYf",
        "JLSuglopxawI",
        "C751SdHhN07n",
        "ptNp4gASm2H5",
        "IrqfjdME18qj",
        "M1Fqi2bHnKam",
        "caoopHQe9wpa",
        "JvSYv8y0lGAd",
        "0pQxo4jEocYc",
        "hXJIR0kMi57t",
        "me3R8Xh1itLU",
        "sANslYgNISo6",
        "vETkLlxOvF8o",
        "stEKkungvjrD",
        "lCiQqT3fvrdG",
        "LRmustQU1pyA",
        "15wg8hcFF8mW",
        "Usmr-BMRv8hk",
        "90QTFk8gwKEJ",
        "xOFXkfCo1qkd",
        "HBsvTuew7NVS",
        "__a_oOhN7NVS",
        "3osiJlZJ7NVX",
        "dDj-Kuqlusx6"
      ]
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "341036acf87d4b56b0bbaf7f8c66f747": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_485c5140b6bf48658810c8779c74e229",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_ebc933df127b4529bc418b3cd22af2ea",
              "IPY_MODEL_933b59cf285142bb888d454e9cb5b595"
            ]
          }
        },
        "485c5140b6bf48658810c8779c74e229": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ebc933df127b4529bc418b3cd22af2ea": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_0fe810b31c724f93b2855f4ff9f1b6c1",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 31,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 31,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_8932585d31a34e5f987b49af1857e831"
          }
        },
        "933b59cf285142bb888d454e9cb5b595": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_d02cb85978634c9b827b997fad499f62",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 31/31 [1:48:40&lt;00:00, 210.33s/it]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_f5a787830e6b4ece844ee931640804f9"
          }
        },
        "0fe810b31c724f93b2855f4ff9f1b6c1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "8932585d31a34e5f987b49af1857e831": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d02cb85978634c9b827b997fad499f62": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "f5a787830e6b4ece844ee931640804f9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "O38kMwEQITYf"
      },
      "source": [
        "---\n",
        "---\n",
        "\n",
        "# 1. Intro & Lit Review\n",
        "\n",
        "---\n",
        "---"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PKfShQX3Igu3"
      },
      "source": [
        "Our sleep quality is closely tied to our mental and physical wellbeing. Poor sleep quality can lead to numerous health problems, such as obesity [1], epilepsy [2], depression and suicidal ideation [3]. Thus, it is important for us to understand what the causes of poor sleep are, and how we can address them. Predicting sleep is an important step in this direction. If certain signals are found to correlate with the quality of sleep, then we can work to see what is influencing these signals and to change them to improve sleep.\n",
        "There are two types of approaches typically seen for the topic of sleep prediction, those being: prediction of the quality of a person’s sleep based on data collected on them throughout the course of the day, and prediction of the future sleep state using data collected during sleep.\n",
        "\n",
        "The first of these approaches we will look at is predicting the quality of a person’s sleep using the data collected on them throughout the course of the day. In [4], they use sleep and sleep-related factors measured during the morning, afternoon and evening periods of the day to predict quality of sleep. In this research, they showed there is a significant correlation between sleep quality and both physical activity and exposure to light. Similarly, in [5], they used wearable sensor data collected while awake to predict sleep efficiency. These kinds of long term predictive models could be used by employers to ensure employees in key, high-risk positions are not fatigued, as studied in [6], and also could be used by medical practitioners to give recommendations to patients about how they can change their lifestyle in order to improve their sleep quality and hence, improve their general health and wellbeing.\n",
        "\n",
        "The second approach, and perhaps the more interesting one, is the prediction of a person’s sleep state based on data collected on them during that sleep session. The key to this forecasting is first being able to classify the person’s previous and current sleep states. We can then use the data on these classified sleep states to predict the future sleep states. In [7], to classify the sleep stages, they apply a convolutional neural network to multi-channel EEG time series data. They report an average of 92% accuracy across the five stages of sleep, showing that there are ways to accurately classify a person’s sleep state. Using only this kind of classification model, a patient's sleep could be monitored in a retrospective way to help diagnose various disorders such as insomnia, but for our purpose, we would use this model in conjunction with another model to forecast the person’s future sleep pattern in a more proactive fashion.\n",
        "\n",
        "Being able to forecast sleep a few minutes or hours into the future could prove very useful in certain scenarios. Three of these scenarios are in the case of hospital patients with an emphasis on dementia patients, very young children, and criminals post surgery.\n",
        "As pointed out in [8] by the UK Alzheimer's Society, people with dementia can have restless sleep and oftentimes, they can wake up in the middle of the night being quite disorientated. This disorientation can result in them doing things they shouldn’t do, like leaving the house in the middle of the night. In cases like this, some sort of warning on the person’s sleep activity would be a great help as it could give a nurse, a spouse or a carer the opportunity to intercept the person and to reassure them. Even in instances outside of the dementia case study, models like this can provide vital information to healthcare staff in hospitals, nursing homes, and elsewhere to greatly improve efficiency of resources and to improve the quality of care for patients.\n",
        "Prediction of sleep models might also be handy for parents of very young children. We think it would be beneficial to be able to forecast the sleep state of your child in order to be able to plan your day and behaviours better around this. Sometimes, parents limit their activities as they may be unsure whether their child will wake up soon. If we could predict the sleep state and even be able to project a time when the child would wake up, this would offer much more flexibility to parents and give them an oppertunity to plan things more in line with their child's sleep.\n",
        "There are also cases where the prediction of sleep quality is perhaps too granular and it is the binary sleep vs awake that is important. A situation we identified where this might be the case would be when criminals have a medical procedure. If a criminal gets surgery and is sleeping in their recovery, we would want to be able to inform police when that person is due to wake up so that they can be more alert to a potential escape.\n",
        "\n",
        "While these three examples are not an exhaustive list of practical applications of a sleep forecasting model, they help to illustrate the wide range of use cases such a technology could have. While there has not been too much research into this area to date, we hope to fill this research void with our work, building on from both previous works on sleep state classification and previous works on prediction of sleep quality using data collected throughout the day.\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JLSuglopxawI"
      },
      "source": [
        "### References"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UEgEhobDVpBY"
      },
      "source": [
        "**1.**   G Beccuti, S Pannain, Sleep and Obesity, 2011. DOI: 10.1097/MCO.0b013e3283479109 \n",
        "\n",
        "**2.**   B Mallow, Sleep Deprivation and Epilepsy, 2004. DOI: 10.1111/j.1535-7597.2004.04509.X \n",
        "\n",
        "**3.**   D Littlewood, S Kyle, L Carter, S Peters, D Pratt, P Gooding, Short sleep duration and poor sleep quality predict next-day suicidal ideation: an ecological momentary assessment study, 2019. DOI: 10.1017/S0033291718001009\n",
        "\n",
        "**4.**   K Park, S Lee, S Wang, S Kim, S Lee, S Cho, S Park, E Lee, Sleep Prediction Algorithm Based On Deep Learning Technology, 2019. DOI: 10.1093/sleep/zsz067.425 \n",
        "\n",
        "**5.**   A Sathyanarayana, S Joty, L Fernandez-Luque, F Ofli, J Srivastava, A Elmagarmid, T Arora, S Taheri, Sleep Quality Prediction From Wearable Data Using Deep Learning, 2016. DOI: 10.2196/mhealth.6562 \n",
        "\n",
        "**6.**   Sahand Hajifar, Hongyue Sun, Fadel M. Megahed, L. Allison Jones-Farmer, Ehsan Rashedi, Lora A. Cavuoto, A forecasting framework for predicting perceived fatigue: Using time series methods to forecast ratings of perceived exertion with features from wearable sensors, 2021. DOI: 10.1016/j.apergo.2020.103262 \n",
        "\n",
        "**7.**   M Santos, Z Cui, X Zheng, X Shao, L Cui, Automatic Sleep Stage Classification Based on Convolutional Neural Network and Fine-Grained Segments, 2018. DOI: 10.1155/2018/9248410 **\n",
        "\n",
        "**8.**  Alzheimers Society. https://www.alzheimers.org.uk/about-dementia/symptoms-and-diagnosis/sleep-and-night-time-disturbance"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "C751SdHhN07n"
      },
      "source": [
        "---\n",
        "---\n",
        "\n",
        "# 2. Setup\n",
        "\n",
        "---\n",
        "---"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ptNp4gASm2H5"
      },
      "source": [
        "### Import the relevant packages"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ivoRQvW_vi8d"
      },
      "source": [
        "import os\n",
        "from google.colab import drive\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import seaborn as sns\n",
        "import matplotlib.pyplot as plt\n",
        "from tqdm.auto import tqdm"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IrqfjdME18qj"
      },
      "source": [
        "### Mount the drive"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QZIfGeuLv_KM",
        "outputId": "d4384737-165c-4d94-93ac-d1017a3e3fb2"
      },
      "source": [
        "drive.mount('/content/drive')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "M1Fqi2bHnKam"
      },
      "source": [
        "### Set the directory for reading in and outputting the data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NXkgP3r5DMnt"
      },
      "source": [
        "#os.chdir('/content/drive/My Drive/College - 4th Year/CA4015_AdvancedML/Assignment_3/new/notebooks/data') # Liam\n",
        "os.chdir('/content/drive/My Drive/Assignment_3/new/notebooks/data')                                       # Nathan & Adam"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qJQhpUbPwNHL",
        "outputId": "e22747f6-45db-4bf8-9c62-8528aba50528"
      },
      "source": [
        "!pwd"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content/drive/.shortcut-targets-by-id/1CNknqtSZpMl1LnBDxjTucYpgXSD_ekVG/Assignment_3/new/notebooks/data\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "caoopHQe9wpa"
      },
      "source": [
        "### Set the maximum number of rows and columns seen in the output"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "m6QZAwlX920M"
      },
      "source": [
        "pd.set_option('max_rows', 100)\n",
        "pd.set_option('max_columns', 25)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JvSYv8y0lGAd"
      },
      "source": [
        "### Disable Warning"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jdFc5l_5lGgj"
      },
      "source": [
        "pd.options.mode.chained_assignment = None"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0pQxo4jEocYc"
      },
      "source": [
        "---\n",
        "---\n",
        "\n",
        "# 3. Load, Join & Clean the data\n",
        "\n",
        "---\n",
        "---"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hXJIR0kMi57t"
      },
      "source": [
        "\n",
        "### Create a function to read in, clean and join the different data sources"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "m8feuLuKEv09"
      },
      "source": [
        "def load_subject_ids(data_folder):\n",
        "    \"\"\"\n",
        "    Create a list containing the subject_id's of all the participants in the study\n",
        "    \"\"\"\n",
        "\n",
        "    subject_ids = []\n",
        "    for filename in os.listdir(data_folder):\n",
        "        subject_id = int(filename.split('_')[0])\n",
        "\n",
        "        # add this to a list of all subject_ids\n",
        "        subject_ids.append(subject_id)\n",
        "        \n",
        "    # remove duplicates and then sort the list\n",
        "    sorted_subject_ids = sorted(list(set(subject_ids)))\n",
        "\n",
        "    return sorted_subject_ids"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pqaUmbwTwVrq"
      },
      "source": [
        "def load_subjects_cropped_data(data_folder, subject_id):\n",
        "    \"\"\"\n",
        "    For a particular subject, read in the data recorded by their apple watch\n",
        "    \"\"\"\n",
        "\n",
        "    # read in heart rate data\n",
        "    heart_df = pd.read_csv(os.path.join(data_folder, '{}_cleaned_hr.out'.format(subject_id)), sep=\" \", names=[\"time\", \"heart_rate\"])\n",
        "\n",
        "    # read in motion data\n",
        "    motion_df = pd.read_csv(os.path.join(data_folder, '{}_cleaned_motion.out'.format(subject_id)), sep=\" \", names=[\"time\", \"x_motion\", \"y_motion\", \"z_motion\"])\n",
        "\n",
        "    # read in the step count data\n",
        "    steps_df = pd.read_csv(os.path.join(data_folder, '{}_cleaned_counts.out'.format(subject_id)), names=[\"time\", \"step_count\"])\n",
        "\n",
        "    # read in psg sleep label data\n",
        "    labels_df = pd.read_csv(os.path.join(data_folder, '{}_cleaned_psg.out'.format(subject_id)), sep=\" \", names=[\"time\", \"sleep_label\"])\n",
        "\n",
        "    # clean the psg data to remove '-1' values - turn values with '-1' to 'nan'\n",
        "    labels_df = labels_df[labels_df[\"sleep_label\"] != -1]\n",
        "    \n",
        "    return heart_df, motion_df, steps_df, labels_df"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JKaA-A-WQp6n"
      },
      "source": [
        "def resample(df_to_resample, hz):\n",
        "    \"\"\"\n",
        "    Take in a dataframe of one sensor - eg. |Time|Heart_Rate| - And make the time step interval uniform across all rows\n",
        "    \"\"\"\n",
        "    \n",
        "    df = df_to_resample.copy()\n",
        "\n",
        "    df[\"time\"] = pd.to_datetime(df[\"time\"], unit=\"s\")\n",
        "    df.set_index(\"time\", inplace=True)\n",
        "\n",
        "    new_index = pd.date_range(0, df.index.max(), freq=f\"{1000/hz:.0f}ms\")\n",
        "    df_resampled = df.reindex(df.index.union(new_index)).interpolate().reindex(new_index)\n",
        "\n",
        "    return df_resampled"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tjwYmUvl8_LK"
      },
      "source": [
        "def load_author_data(data_folder, subject_id):\n",
        "    \"\"\"\n",
        "    For a particular subject, read in the features that the author of the study created\n",
        "    \"\"\"\n",
        "    \n",
        "    # Read in the 5 feature tables defined for each subject_id\n",
        "    heart_rate_feature_df = pd.read_csv(os.path.join(data_folder, f\"{subject_id}_hr_feature.out\"), header=None, names=[\"heart_rate_feature\"])\n",
        "    count_feature_df = pd.read_csv(os.path.join(data_folder, f\"{subject_id}_count_feature.out\"), header=None, names=[\"count_feature\"])\n",
        "    time_feature_df = pd.read_csv(os.path.join(data_folder, f\"{subject_id}_time_feature.out\"), header=None, names=[\"time_feature\"])\n",
        "    cosine_feature_df = pd.read_csv(os.path.join(data_folder, f\"{subject_id}_cosine_feature.out\"), header=None, names=[\"cosine_feature\"])\n",
        "    psg_label_df = pd.read_csv(os.path.join(data_folder, f\"{subject_id}_psg_labels.out\"), header=None, names=[\"psg_label\"])\n",
        "\n",
        "    # Convert the time from hours into a timestamp in seconds\n",
        "    time_feature_df[\"time\"] = pd.to_datetime((time_feature_df[\"time_feature\"]*60*60).apply(lambda t: int(round(t))), unit=\"s\")\n",
        "    \n",
        "    subject_features = pd.concat([time_feature_df, heart_rate_feature_df, count_feature_df, cosine_feature_df, psg_label_df], axis=1)\n",
        "\n",
        "    return subject_features"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "T4k8UvA2SRk3"
      },
      "source": [
        "def load_and_preprocess_data(cropped_data_folder, author_data_folder, subject_id, hz=1/30):\n",
        "    \"\"\"\n",
        "    For a particular subject:\n",
        "        1. Read in the cropped sensor data\n",
        "        2. Fix the time interval for each sensor dataframe to be uniform\n",
        "        3. Merge all that subjects sensor data into one table\n",
        "\n",
        "        4. Read in the authors features\n",
        "        5. Fix the time interval for these features to be uniform\n",
        "        6. Join these authors features to the merged sensor dataframe created in step 1, 2 & 3\n",
        "\n",
        "        7. Drop the rows in this dataframe that have an Nan value in one of the specified columns\n",
        "    \"\"\"\n",
        "\n",
        "    # ----------------------\n",
        "    # ==== CROPPED DATA ====\n",
        "    # ----------------------\n",
        "\n",
        "    # Load in the sensor data for this given subject\n",
        "    heart_df, motion_df, steps_df, labels_df = load_subjects_cropped_data(cropped_data_folder, subject_id)\n",
        "\n",
        "    # Resample the time column of these sensor dataframes to get a uniform timestamp step\n",
        "    resampled_dfs = []\n",
        "    for df in [heart_df, motion_df, steps_df, labels_df]:\n",
        "        resampled_df = pd.DataFrame(columns=df.columns) if len(df) == 0 else resample(df, hz)\n",
        "        resampled_dfs.append(resampled_df)\n",
        "\n",
        "    # Merge these sensor dataframes into one\n",
        "    all_sensor_df = resampled_dfs[0]\n",
        "    for _df in resampled_dfs[1:]:\n",
        "        all_sensor_df = all_sensor_df.merge(_df, how=\"outer\", left_index=True, right_index=True)\n",
        "\n",
        "    # drop the rows in this dataframe where there is no corresponding sleep state value\n",
        "    fixed_all_sensor_df = all_sensor_df.dropna(subset=[\"sleep_label\"])\n",
        "\n",
        "    # Then reset the timestamp index back to 0\n",
        "    fixed_all_sensor_df.index = (fixed_all_sensor_df.index - fixed_all_sensor_df.index[0]) + pd.Timestamp('1970-01-01 00:00:00')\n",
        "\n",
        "    # The 'sleep_label' will now contain some float values from the interpolation used in the resampling.\n",
        "    # As there are now no NAN's in this column, this can be fixed by rounding these floats to integers\n",
        "    fixed_all_sensor_df[\"sleep_label\"] = fixed_all_sensor_df[\"sleep_label\"].round().astype(int)\n",
        "\n",
        "\n",
        "    # -----------------------\n",
        "    # ===== AUTHOR DATA =====\n",
        "    # -----------------------\n",
        "\n",
        "    # Load in the features created by th author of the study\n",
        "    author_feature_df = load_author_data(author_data_folder, subject_id)\n",
        "\n",
        "    # Resample these features\n",
        "    resampled_author_df = resample(author_feature_df, hz)\n",
        "\n",
        "    # The 'psg_label' will now contain some float values from the interpolation used in the resampling.\n",
        "    # As there are no NAN's in this column, this can be fixed by rounding these floats to integers\n",
        "    resampled_author_df[\"psg_label\"] = resampled_author_df[\"psg_label\"].round().astype(int)\n",
        "\n",
        "    # Merge these author features with the rest of the sensor data\n",
        "    sensor_nd_author_feature_df = fixed_all_sensor_df.merge(resampled_author_df, left_index=True, right_index=True)\n",
        "\n",
        "    # Take the time out of the index and make it a column\n",
        "    df = sensor_nd_author_feature_df.reset_index().rename(columns={\"index\": \"time\"})\n",
        "\n",
        "    # Drop the rows that contain NAN values in their heart_rate or one of the motion columns\n",
        "    no_nan_df = df.dropna(subset=[\"heart_rate\", \"x_motion\", \"y_motion\", \"z_motion\"]).reset_index(drop=True)\n",
        "\n",
        "    return no_nan_df"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "J7UsEEurH7Fe"
      },
      "source": [
        "def load_and_clean_all_data(cropped_data_folder, author_data_folder, hz=1/30):\n",
        "    \"\"\"\n",
        "    Create a list of all subjects, then for each subject, create a dataframe of all the sensor data for that subject.\n",
        "    Then join this dataframe created for each subject into one big dataframe\n",
        "    \"\"\"\n",
        "\n",
        "    # create a list of all the subject_ids\n",
        "    subject_ids = load_subject_ids(cropped_data_folder)\n",
        "    \n",
        "    all_data_df = pd.DataFrame()\n",
        "\n",
        "    # iterate through the subject_ids and load and clean its data\n",
        "    for subject_id in tqdm(subject_ids):\n",
        "\n",
        "        final_df = load_and_preprocess_data(cropped_data_folder, author_data_folder, subject_id, hz=hz)\n",
        "\n",
        "        # Drop the 'sleep_label' column as we will proceed with the 'psg_label' column used by the author\n",
        "        #final_df = df.drop(columns=[\"sleep_label\"])\n",
        "\n",
        "        # Add a subject_id column to this dataframe\n",
        "        final_df[\"subject_id\"] = subject_id\n",
        "\n",
        "        all_data_df = pd.concat([all_data_df, final_df], axis=0)\n",
        "\n",
        "        # output this dataframe to a csv in the above created output directory\n",
        "        # df.to_csv(os.path.join(output_folder, f\"{subject}.csv\"), index=False)\n",
        "\n",
        "    return all_data_df.reset_index(drop=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "me3R8Xh1itLU"
      },
      "source": [
        "### Load, clean and join the data for each subject"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Uj3UEnd-oAo5"
      },
      "source": [
        "We chose to have one row for each 30 seconds of data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 66,
          "referenced_widgets": [
            "341036acf87d4b56b0bbaf7f8c66f747",
            "485c5140b6bf48658810c8779c74e229",
            "ebc933df127b4529bc418b3cd22af2ea",
            "933b59cf285142bb888d454e9cb5b595",
            "0fe810b31c724f93b2855f4ff9f1b6c1",
            "8932585d31a34e5f987b49af1857e831",
            "d02cb85978634c9b827b997fad499f62",
            "f5a787830e6b4ece844ee931640804f9"
          ]
        },
        "id": "E5G9D8ZqFpRx",
        "outputId": "cb5f716b-18f7-4f8d-b2d5-0317a3bc19b7"
      },
      "source": [
        "all_subjects_data_df = load_and_clean_all_data('./cropped', './author_features', hz=1/30)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "341036acf87d4b56b0bbaf7f8c66f747",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=31.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "J4eVxDCljsWi",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c44f48ce-0557-455a-ad68-8015192a4078"
      },
      "source": [
        "all_subjects_data_df.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(25837, 13)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 197
        },
        "id": "mTPMjibDwc_s",
        "outputId": "a01cbb50-8ee5-4594-9dde-8d38c13a04cd"
      },
      "source": [
        "all_subjects_data_df.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>time</th>\n",
              "      <th>heart_rate</th>\n",
              "      <th>x_motion</th>\n",
              "      <th>y_motion</th>\n",
              "      <th>z_motion</th>\n",
              "      <th>step_count</th>\n",
              "      <th>sleep_label</th>\n",
              "      <th>time_feature</th>\n",
              "      <th>heart_rate_feature</th>\n",
              "      <th>count_feature</th>\n",
              "      <th>cosine_feature</th>\n",
              "      <th>psg_label</th>\n",
              "      <th>subject_id</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1970-01-01 00:00:00</td>\n",
              "      <td>90.0</td>\n",
              "      <td>-0.559524</td>\n",
              "      <td>0.690460</td>\n",
              "      <td>-0.220322</td>\n",
              "      <td>102.845</td>\n",
              "      <td>0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.524519</td>\n",
              "      <td>38.634412</td>\n",
              "      <td>-0.258819</td>\n",
              "      <td>0</td>\n",
              "      <td>46343</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1970-01-01 00:00:30</td>\n",
              "      <td>101.0</td>\n",
              "      <td>-0.631027</td>\n",
              "      <td>0.672783</td>\n",
              "      <td>-0.314140</td>\n",
              "      <td>58.330</td>\n",
              "      <td>0</td>\n",
              "      <td>0.008333</td>\n",
              "      <td>0.514774</td>\n",
              "      <td>31.640606</td>\n",
              "      <td>-0.260926</td>\n",
              "      <td>0</td>\n",
              "      <td>46343</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1970-01-01 00:01:00</td>\n",
              "      <td>92.0</td>\n",
              "      <td>-0.115539</td>\n",
              "      <td>0.853889</td>\n",
              "      <td>0.774826</td>\n",
              "      <td>12.280</td>\n",
              "      <td>0</td>\n",
              "      <td>0.016667</td>\n",
              "      <td>0.497230</td>\n",
              "      <td>22.570999</td>\n",
              "      <td>-0.263031</td>\n",
              "      <td>0</td>\n",
              "      <td>46343</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1970-01-01 00:01:30</td>\n",
              "      <td>82.0</td>\n",
              "      <td>-0.345749</td>\n",
              "      <td>-0.284614</td>\n",
              "      <td>0.888367</td>\n",
              "      <td>0.000</td>\n",
              "      <td>0</td>\n",
              "      <td>0.025000</td>\n",
              "      <td>0.489079</td>\n",
              "      <td>19.341522</td>\n",
              "      <td>-0.265135</td>\n",
              "      <td>0</td>\n",
              "      <td>46343</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>1970-01-01 00:02:00</td>\n",
              "      <td>83.0</td>\n",
              "      <td>-0.315613</td>\n",
              "      <td>-0.276604</td>\n",
              "      <td>0.906998</td>\n",
              "      <td>0.000</td>\n",
              "      <td>0</td>\n",
              "      <td>0.033333</td>\n",
              "      <td>0.482398</td>\n",
              "      <td>21.960207</td>\n",
              "      <td>-0.267238</td>\n",
              "      <td>0</td>\n",
              "      <td>46343</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                 time  heart_rate  x_motion  y_motion  z_motion  step_count  \\\n",
              "0 1970-01-01 00:00:00        90.0 -0.559524  0.690460 -0.220322     102.845   \n",
              "1 1970-01-01 00:00:30       101.0 -0.631027  0.672783 -0.314140      58.330   \n",
              "2 1970-01-01 00:01:00        92.0 -0.115539  0.853889  0.774826      12.280   \n",
              "3 1970-01-01 00:01:30        82.0 -0.345749 -0.284614  0.888367       0.000   \n",
              "4 1970-01-01 00:02:00        83.0 -0.315613 -0.276604  0.906998       0.000   \n",
              "\n",
              "   sleep_label  time_feature  heart_rate_feature  count_feature  \\\n",
              "0            0      0.000000            0.524519      38.634412   \n",
              "1            0      0.008333            0.514774      31.640606   \n",
              "2            0      0.016667            0.497230      22.570999   \n",
              "3            0      0.025000            0.489079      19.341522   \n",
              "4            0      0.033333            0.482398      21.960207   \n",
              "\n",
              "   cosine_feature  psg_label  subject_id  \n",
              "0       -0.258819          0       46343  \n",
              "1       -0.260926          0       46343  \n",
              "2       -0.263031          0       46343  \n",
              "3       -0.265135          0       46343  \n",
              "4       -0.267238          0       46343  "
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sANslYgNISo6"
      },
      "source": [
        "### Fix the time column - change it to an integer"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6ChXUdNBIYAT"
      },
      "source": [
        "all_subjects_data_df['time'] = all_subjects_data_df['time'].astype(int)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 197
        },
        "id": "IqKf1a5CJ92Q",
        "outputId": "972620af-0475-48fb-ce36-12607cd725f1"
      },
      "source": [
        "all_subjects_data_df.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>time</th>\n",
              "      <th>heart_rate</th>\n",
              "      <th>x_motion</th>\n",
              "      <th>y_motion</th>\n",
              "      <th>z_motion</th>\n",
              "      <th>step_count</th>\n",
              "      <th>sleep_label</th>\n",
              "      <th>time_feature</th>\n",
              "      <th>heart_rate_feature</th>\n",
              "      <th>count_feature</th>\n",
              "      <th>cosine_feature</th>\n",
              "      <th>psg_label</th>\n",
              "      <th>subject_id</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>90.0</td>\n",
              "      <td>-0.559524</td>\n",
              "      <td>0.690460</td>\n",
              "      <td>-0.220322</td>\n",
              "      <td>102.845</td>\n",
              "      <td>0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.524519</td>\n",
              "      <td>38.634412</td>\n",
              "      <td>-0.258819</td>\n",
              "      <td>0</td>\n",
              "      <td>46343</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>30000000000</td>\n",
              "      <td>101.0</td>\n",
              "      <td>-0.631027</td>\n",
              "      <td>0.672783</td>\n",
              "      <td>-0.314140</td>\n",
              "      <td>58.330</td>\n",
              "      <td>0</td>\n",
              "      <td>0.008333</td>\n",
              "      <td>0.514774</td>\n",
              "      <td>31.640606</td>\n",
              "      <td>-0.260926</td>\n",
              "      <td>0</td>\n",
              "      <td>46343</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>60000000000</td>\n",
              "      <td>92.0</td>\n",
              "      <td>-0.115539</td>\n",
              "      <td>0.853889</td>\n",
              "      <td>0.774826</td>\n",
              "      <td>12.280</td>\n",
              "      <td>0</td>\n",
              "      <td>0.016667</td>\n",
              "      <td>0.497230</td>\n",
              "      <td>22.570999</td>\n",
              "      <td>-0.263031</td>\n",
              "      <td>0</td>\n",
              "      <td>46343</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>90000000000</td>\n",
              "      <td>82.0</td>\n",
              "      <td>-0.345749</td>\n",
              "      <td>-0.284614</td>\n",
              "      <td>0.888367</td>\n",
              "      <td>0.000</td>\n",
              "      <td>0</td>\n",
              "      <td>0.025000</td>\n",
              "      <td>0.489079</td>\n",
              "      <td>19.341522</td>\n",
              "      <td>-0.265135</td>\n",
              "      <td>0</td>\n",
              "      <td>46343</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>120000000000</td>\n",
              "      <td>83.0</td>\n",
              "      <td>-0.315613</td>\n",
              "      <td>-0.276604</td>\n",
              "      <td>0.906998</td>\n",
              "      <td>0.000</td>\n",
              "      <td>0</td>\n",
              "      <td>0.033333</td>\n",
              "      <td>0.482398</td>\n",
              "      <td>21.960207</td>\n",
              "      <td>-0.267238</td>\n",
              "      <td>0</td>\n",
              "      <td>46343</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "           time  heart_rate  x_motion  y_motion  z_motion  step_count  \\\n",
              "0             0        90.0 -0.559524  0.690460 -0.220322     102.845   \n",
              "1   30000000000       101.0 -0.631027  0.672783 -0.314140      58.330   \n",
              "2   60000000000        92.0 -0.115539  0.853889  0.774826      12.280   \n",
              "3   90000000000        82.0 -0.345749 -0.284614  0.888367       0.000   \n",
              "4  120000000000        83.0 -0.315613 -0.276604  0.906998       0.000   \n",
              "\n",
              "   sleep_label  time_feature  heart_rate_feature  count_feature  \\\n",
              "0            0      0.000000            0.524519      38.634412   \n",
              "1            0      0.008333            0.514774      31.640606   \n",
              "2            0      0.016667            0.497230      22.570999   \n",
              "3            0      0.025000            0.489079      19.341522   \n",
              "4            0      0.033333            0.482398      21.960207   \n",
              "\n",
              "   cosine_feature  psg_label  subject_id  \n",
              "0       -0.258819          0       46343  \n",
              "1       -0.260926          0       46343  \n",
              "2       -0.263031          0       46343  \n",
              "3       -0.265135          0       46343  \n",
              "4       -0.267238          0       46343  "
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 44
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vETkLlxOvF8o"
      },
      "source": [
        "---\n",
        "---\n",
        "\n",
        "# 4. Split the data into a Train, Test and Validation set\n",
        "\n",
        "---\n",
        "---"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "stEKkungvjrD"
      },
      "source": [
        "### Define the function to split the data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SHTiiedjBx2x"
      },
      "source": [
        "def split_data(df, train=0.7, val=0.2):\n",
        "    \"\"\"\n",
        "    Split the data into a train, test and validation set\n",
        "    \"\"\"\n",
        "\n",
        "    num_rows = len(df)\n",
        "    val_start = int(num_rows * train)\n",
        "    test_start = int(num_rows * (train + val))\n",
        "\n",
        "    train_df = df[:val_start]\n",
        "    val_df = df[val_start:test_start]\n",
        "    test_df = df[test_start:]\n",
        "\n",
        "    return train_df, val_df, test_df"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lCiQqT3fvrdG"
      },
      "source": [
        "### Split the data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eFWV9knW6w9v"
      },
      "source": [
        "train_df, val_df, test_df = split_data(all_subjects_data_df, 0.7, 0.2)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CFxzvzTqP59b",
        "outputId": "799f2438-4123-4d8f-864d-7656df078c98"
      },
      "source": [
        "print(train_df.shape)\n",
        "print(val_df.shape)\n",
        "print(test_df.shape)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(18085, 13)\n",
            "(5168, 13)\n",
            "(2584, 13)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LRmustQU1pyA"
      },
      "source": [
        "---\n",
        "---\n",
        "\n",
        "# 6. Standardise all the data\n",
        "\n",
        "---\n",
        "---"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "15wg8hcFF8mW"
      },
      "source": [
        "### Standardise the train, validation, test and full dataframes"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ldDWaEmmCsUI"
      },
      "source": [
        "def standardise(df_to_be_standardised, columns, mean=None, std=None):\n",
        "\n",
        "    # make a copy of the dataframe so we dont overwrite it\n",
        "    df = df_to_be_standardised.copy()\n",
        "\n",
        "    # standardise columns in-place\n",
        "    if mean is None or std is None:\n",
        "        mean = df[columns].mean()\n",
        "        std = df[columns].std()\n",
        "\n",
        "    df[columns] = (df[columns] - mean) / std\n",
        "\n",
        "    return df, mean, std"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BVtwb7FUpZ9Z",
        "outputId": "26f21e40-488a-475e-f6bb-3fa1ec933f0b"
      },
      "source": [
        "# Only standardise feature columns\n",
        "input_cols = train_df.drop(columns=['sleep_label', 'psg_label', 'subject_id']).columns\n",
        "\n",
        "std_train_df, train_mean, train_std = standardise(train_df, input_cols)\n",
        "std_val_df, _, _ = standardise(val_df, input_cols, train_mean, train_std)\n",
        "std_test_df, _, _ = standardise(test_df, input_cols, train_mean, train_std)\n",
        "std_full_df, _, _ = standardise(all_subjects_data_df, input_cols, train_mean, train_std)\n",
        "\n",
        "print(\"Mean:\")\n",
        "print(train_mean)\n",
        "print(\"Standard Deviation:\")\n",
        "print(train_std)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mean:\n",
            "time                  1.296487e+13\n",
            "heart_rate            6.413870e+01\n",
            "x_motion             -1.133552e-01\n",
            "y_motion             -1.228230e-01\n",
            "z_motion             -3.895808e-01\n",
            "step_count            1.813057e+00\n",
            "time_feature          3.601353e+00\n",
            "heart_rate_feature    3.646799e-01\n",
            "count_feature         1.718910e+00\n",
            "cosine_feature       -7.780936e-01\n",
            "dtype: float64\n",
            "Standard Deviation:\n",
            "time                  8.094516e+12\n",
            "heart_rate            1.103747e+01\n",
            "x_motion              3.421223e-01\n",
            "y_motion              4.934719e-01\n",
            "z_motion              6.740018e-01\n",
            "step_count            1.173729e+01\n",
            "time_feature          2.248477e+00\n",
            "heart_rate_feature    3.325092e-01\n",
            "count_feature         6.674443e+00\n",
            "cosine_feature        2.117324e-01\n",
            "dtype: float64\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Usmr-BMRv8hk"
      },
      "source": [
        "### Look at these standardised dataframes"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 197
        },
        "id": "eloiq4H2I8g1",
        "outputId": "1d5c6730-6b0c-4c83-f1f4-e7f3e1feee64"
      },
      "source": [
        "std_train_df.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>time</th>\n",
              "      <th>heart_rate</th>\n",
              "      <th>x_motion</th>\n",
              "      <th>y_motion</th>\n",
              "      <th>z_motion</th>\n",
              "      <th>step_count</th>\n",
              "      <th>sleep_label</th>\n",
              "      <th>time_feature</th>\n",
              "      <th>heart_rate_feature</th>\n",
              "      <th>count_feature</th>\n",
              "      <th>cosine_feature</th>\n",
              "      <th>psg_label</th>\n",
              "      <th>subject_id</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>-1.601686</td>\n",
              "      <td>2.343046</td>\n",
              "      <td>-1.304122</td>\n",
              "      <td>1.648084</td>\n",
              "      <td>0.251126</td>\n",
              "      <td>8.607776</td>\n",
              "      <td>0</td>\n",
              "      <td>-1.601686</td>\n",
              "      <td>0.480706</td>\n",
              "      <td>5.530874</td>\n",
              "      <td>2.452504</td>\n",
              "      <td>0</td>\n",
              "      <td>46343</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>-1.597980</td>\n",
              "      <td>3.339652</td>\n",
              "      <td>-1.513119</td>\n",
              "      <td>1.612262</td>\n",
              "      <td>0.111929</td>\n",
              "      <td>4.815162</td>\n",
              "      <td>0</td>\n",
              "      <td>-1.597980</td>\n",
              "      <td>0.451398</td>\n",
              "      <td>4.483025</td>\n",
              "      <td>2.442553</td>\n",
              "      <td>0</td>\n",
              "      <td>46343</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>-1.594274</td>\n",
              "      <td>2.524247</td>\n",
              "      <td>-0.006384</td>\n",
              "      <td>1.979267</td>\n",
              "      <td>1.727603</td>\n",
              "      <td>0.891768</td>\n",
              "      <td>0</td>\n",
              "      <td>-1.594273</td>\n",
              "      <td>0.398636</td>\n",
              "      <td>3.124169</td>\n",
              "      <td>2.432611</td>\n",
              "      <td>0</td>\n",
              "      <td>46343</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>-1.590567</td>\n",
              "      <td>1.618243</td>\n",
              "      <td>-0.679271</td>\n",
              "      <td>-0.327864</td>\n",
              "      <td>1.896059</td>\n",
              "      <td>-0.154470</td>\n",
              "      <td>0</td>\n",
              "      <td>-1.590567</td>\n",
              "      <td>0.374122</td>\n",
              "      <td>2.640312</td>\n",
              "      <td>2.422674</td>\n",
              "      <td>0</td>\n",
              "      <td>46343</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>-1.586861</td>\n",
              "      <td>1.708843</td>\n",
              "      <td>-0.591184</td>\n",
              "      <td>-0.311630</td>\n",
              "      <td>1.923702</td>\n",
              "      <td>-0.154470</td>\n",
              "      <td>0</td>\n",
              "      <td>-1.586861</td>\n",
              "      <td>0.354029</td>\n",
              "      <td>3.032657</td>\n",
              "      <td>2.412742</td>\n",
              "      <td>0</td>\n",
              "      <td>46343</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "       time  heart_rate  x_motion  y_motion  z_motion  step_count  \\\n",
              "0 -1.601686    2.343046 -1.304122  1.648084  0.251126    8.607776   \n",
              "1 -1.597980    3.339652 -1.513119  1.612262  0.111929    4.815162   \n",
              "2 -1.594274    2.524247 -0.006384  1.979267  1.727603    0.891768   \n",
              "3 -1.590567    1.618243 -0.679271 -0.327864  1.896059   -0.154470   \n",
              "4 -1.586861    1.708843 -0.591184 -0.311630  1.923702   -0.154470   \n",
              "\n",
              "   sleep_label  time_feature  heart_rate_feature  count_feature  \\\n",
              "0            0     -1.601686            0.480706       5.530874   \n",
              "1            0     -1.597980            0.451398       4.483025   \n",
              "2            0     -1.594273            0.398636       3.124169   \n",
              "3            0     -1.590567            0.374122       2.640312   \n",
              "4            0     -1.586861            0.354029       3.032657   \n",
              "\n",
              "   cosine_feature  psg_label  subject_id  \n",
              "0        2.452504          0       46343  \n",
              "1        2.442553          0       46343  \n",
              "2        2.432611          0       46343  \n",
              "3        2.422674          0       46343  \n",
              "4        2.412742          0       46343  "
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 49
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 197
        },
        "id": "QvXlGBrPv0Y-",
        "outputId": "b728a95d-4b7e-413b-bdf5-8f2e7d156960"
      },
      "source": [
        "std_val_df.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>time</th>\n",
              "      <th>heart_rate</th>\n",
              "      <th>x_motion</th>\n",
              "      <th>y_motion</th>\n",
              "      <th>z_motion</th>\n",
              "      <th>step_count</th>\n",
              "      <th>sleep_label</th>\n",
              "      <th>time_feature</th>\n",
              "      <th>heart_rate_feature</th>\n",
              "      <th>count_feature</th>\n",
              "      <th>cosine_feature</th>\n",
              "      <th>psg_label</th>\n",
              "      <th>subject_id</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>18085</th>\n",
              "      <td>0.577567</td>\n",
              "      <td>0.213935</td>\n",
              "      <td>-1.495124</td>\n",
              "      <td>0.498384</td>\n",
              "      <td>-0.562286</td>\n",
              "      <td>-0.15447</td>\n",
              "      <td>5</td>\n",
              "      <td>0.577567</td>\n",
              "      <td>0.192422</td>\n",
              "      <td>-0.257536</td>\n",
              "      <td>-1.046431</td>\n",
              "      <td>5</td>\n",
              "      <td>8000685</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>18086</th>\n",
              "      <td>0.581274</td>\n",
              "      <td>-0.012566</td>\n",
              "      <td>-1.500876</td>\n",
              "      <td>0.489463</td>\n",
              "      <td>-0.558278</td>\n",
              "      <td>-0.15447</td>\n",
              "      <td>5</td>\n",
              "      <td>0.581273</td>\n",
              "      <td>0.557916</td>\n",
              "      <td>-0.257536</td>\n",
              "      <td>-1.046691</td>\n",
              "      <td>5</td>\n",
              "      <td>8000685</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>18087</th>\n",
              "      <td>0.584980</td>\n",
              "      <td>0.757538</td>\n",
              "      <td>-1.497220</td>\n",
              "      <td>0.510753</td>\n",
              "      <td>-0.569202</td>\n",
              "      <td>-0.15447</td>\n",
              "      <td>5</td>\n",
              "      <td>0.584980</td>\n",
              "      <td>0.686721</td>\n",
              "      <td>-0.257536</td>\n",
              "      <td>-1.046927</td>\n",
              "      <td>5</td>\n",
              "      <td>8000685</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>18088</th>\n",
              "      <td>0.588686</td>\n",
              "      <td>0.576337</td>\n",
              "      <td>-1.516553</td>\n",
              "      <td>0.462067</td>\n",
              "      <td>-0.561459</td>\n",
              "      <td>-0.15447</td>\n",
              "      <td>5</td>\n",
              "      <td>0.588686</td>\n",
              "      <td>0.703659</td>\n",
              "      <td>-0.257536</td>\n",
              "      <td>-1.047140</td>\n",
              "      <td>5</td>\n",
              "      <td>8000685</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>18089</th>\n",
              "      <td>0.592392</td>\n",
              "      <td>-0.057866</td>\n",
              "      <td>-1.508682</td>\n",
              "      <td>0.465067</td>\n",
              "      <td>-0.560373</td>\n",
              "      <td>-0.15447</td>\n",
              "      <td>5</td>\n",
              "      <td>0.592392</td>\n",
              "      <td>0.704723</td>\n",
              "      <td>-0.257536</td>\n",
              "      <td>-1.047334</td>\n",
              "      <td>5</td>\n",
              "      <td>8000685</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "           time  heart_rate  x_motion  y_motion  z_motion  step_count  \\\n",
              "18085  0.577567    0.213935 -1.495124  0.498384 -0.562286    -0.15447   \n",
              "18086  0.581274   -0.012566 -1.500876  0.489463 -0.558278    -0.15447   \n",
              "18087  0.584980    0.757538 -1.497220  0.510753 -0.569202    -0.15447   \n",
              "18088  0.588686    0.576337 -1.516553  0.462067 -0.561459    -0.15447   \n",
              "18089  0.592392   -0.057866 -1.508682  0.465067 -0.560373    -0.15447   \n",
              "\n",
              "       sleep_label  time_feature  heart_rate_feature  count_feature  \\\n",
              "18085            5      0.577567            0.192422      -0.257536   \n",
              "18086            5      0.581273            0.557916      -0.257536   \n",
              "18087            5      0.584980            0.686721      -0.257536   \n",
              "18088            5      0.588686            0.703659      -0.257536   \n",
              "18089            5      0.592392            0.704723      -0.257536   \n",
              "\n",
              "       cosine_feature  psg_label  subject_id  \n",
              "18085       -1.046431          5     8000685  \n",
              "18086       -1.046691          5     8000685  \n",
              "18087       -1.046927          5     8000685  \n",
              "18088       -1.047140          5     8000685  \n",
              "18089       -1.047334          5     8000685  "
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 50
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 197
        },
        "id": "FmNDyUSLv4zk",
        "outputId": "dd4f966e-81f0-4043-eb4c-a8a69132d60b"
      },
      "source": [
        "std_test_df.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>time</th>\n",
              "      <th>heart_rate</th>\n",
              "      <th>x_motion</th>\n",
              "      <th>y_motion</th>\n",
              "      <th>z_motion</th>\n",
              "      <th>step_count</th>\n",
              "      <th>sleep_label</th>\n",
              "      <th>time_feature</th>\n",
              "      <th>heart_rate_feature</th>\n",
              "      <th>count_feature</th>\n",
              "      <th>cosine_feature</th>\n",
              "      <th>psg_label</th>\n",
              "      <th>subject_id</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>23253</th>\n",
              "      <td>-1.471969</td>\n",
              "      <td>-0.284367</td>\n",
              "      <td>-1.286817</td>\n",
              "      <td>0.891131</td>\n",
              "      <td>-0.572212</td>\n",
              "      <td>-0.15447</td>\n",
              "      <td>2</td>\n",
              "      <td>-1.471968</td>\n",
              "      <td>-0.637137</td>\n",
              "      <td>-0.257536</td>\n",
              "      <td>2.108055</td>\n",
              "      <td>2</td>\n",
              "      <td>9106476</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>23254</th>\n",
              "      <td>-1.468262</td>\n",
              "      <td>0.032734</td>\n",
              "      <td>-1.286729</td>\n",
              "      <td>0.883695</td>\n",
              "      <td>-0.574001</td>\n",
              "      <td>-0.15447</td>\n",
              "      <td>2</td>\n",
              "      <td>-1.468262</td>\n",
              "      <td>-0.649877</td>\n",
              "      <td>-0.257536</td>\n",
              "      <td>2.098340</td>\n",
              "      <td>2</td>\n",
              "      <td>9106476</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>23255</th>\n",
              "      <td>-1.464556</td>\n",
              "      <td>-0.103166</td>\n",
              "      <td>-1.291255</td>\n",
              "      <td>0.895584</td>\n",
              "      <td>-0.569009</td>\n",
              "      <td>-0.15447</td>\n",
              "      <td>3</td>\n",
              "      <td>-1.464556</td>\n",
              "      <td>-0.685912</td>\n",
              "      <td>-0.257536</td>\n",
              "      <td>2.088630</td>\n",
              "      <td>3</td>\n",
              "      <td>9106476</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>23256</th>\n",
              "      <td>-1.460850</td>\n",
              "      <td>-0.012566</td>\n",
              "      <td>-1.279770</td>\n",
              "      <td>0.894115</td>\n",
              "      <td>-0.570333</td>\n",
              "      <td>-0.15447</td>\n",
              "      <td>3</td>\n",
              "      <td>-1.460850</td>\n",
              "      <td>-0.700504</td>\n",
              "      <td>-0.257536</td>\n",
              "      <td>2.078929</td>\n",
              "      <td>3</td>\n",
              "      <td>9106476</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>23257</th>\n",
              "      <td>-1.457144</td>\n",
              "      <td>0.078035</td>\n",
              "      <td>-1.286973</td>\n",
              "      <td>0.885210</td>\n",
              "      <td>-0.568240</td>\n",
              "      <td>-0.15447</td>\n",
              "      <td>2</td>\n",
              "      <td>-1.457144</td>\n",
              "      <td>-0.709255</td>\n",
              "      <td>-0.257536</td>\n",
              "      <td>2.069237</td>\n",
              "      <td>2</td>\n",
              "      <td>9106476</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "           time  heart_rate  x_motion  y_motion  z_motion  step_count  \\\n",
              "23253 -1.471969   -0.284367 -1.286817  0.891131 -0.572212    -0.15447   \n",
              "23254 -1.468262    0.032734 -1.286729  0.883695 -0.574001    -0.15447   \n",
              "23255 -1.464556   -0.103166 -1.291255  0.895584 -0.569009    -0.15447   \n",
              "23256 -1.460850   -0.012566 -1.279770  0.894115 -0.570333    -0.15447   \n",
              "23257 -1.457144    0.078035 -1.286973  0.885210 -0.568240    -0.15447   \n",
              "\n",
              "       sleep_label  time_feature  heart_rate_feature  count_feature  \\\n",
              "23253            2     -1.471968           -0.637137      -0.257536   \n",
              "23254            2     -1.468262           -0.649877      -0.257536   \n",
              "23255            3     -1.464556           -0.685912      -0.257536   \n",
              "23256            3     -1.460850           -0.700504      -0.257536   \n",
              "23257            2     -1.457144           -0.709255      -0.257536   \n",
              "\n",
              "       cosine_feature  psg_label  subject_id  \n",
              "23253        2.108055          2     9106476  \n",
              "23254        2.098340          2     9106476  \n",
              "23255        2.088630          3     9106476  \n",
              "23256        2.078929          3     9106476  \n",
              "23257        2.069237          2     9106476  "
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 51
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 197
        },
        "id": "3fufMb-_v42J",
        "outputId": "460b71fa-8dd4-4b4c-d478-9adb074bc1f0"
      },
      "source": [
        "std_full_df.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>time</th>\n",
              "      <th>heart_rate</th>\n",
              "      <th>x_motion</th>\n",
              "      <th>y_motion</th>\n",
              "      <th>z_motion</th>\n",
              "      <th>step_count</th>\n",
              "      <th>sleep_label</th>\n",
              "      <th>time_feature</th>\n",
              "      <th>heart_rate_feature</th>\n",
              "      <th>count_feature</th>\n",
              "      <th>cosine_feature</th>\n",
              "      <th>psg_label</th>\n",
              "      <th>subject_id</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>-1.601686</td>\n",
              "      <td>2.343046</td>\n",
              "      <td>-1.304122</td>\n",
              "      <td>1.648084</td>\n",
              "      <td>0.251126</td>\n",
              "      <td>8.607776</td>\n",
              "      <td>0</td>\n",
              "      <td>-1.601686</td>\n",
              "      <td>0.480706</td>\n",
              "      <td>5.530874</td>\n",
              "      <td>2.452504</td>\n",
              "      <td>0</td>\n",
              "      <td>46343</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>-1.597980</td>\n",
              "      <td>3.339652</td>\n",
              "      <td>-1.513119</td>\n",
              "      <td>1.612262</td>\n",
              "      <td>0.111929</td>\n",
              "      <td>4.815162</td>\n",
              "      <td>0</td>\n",
              "      <td>-1.597980</td>\n",
              "      <td>0.451398</td>\n",
              "      <td>4.483025</td>\n",
              "      <td>2.442553</td>\n",
              "      <td>0</td>\n",
              "      <td>46343</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>-1.594274</td>\n",
              "      <td>2.524247</td>\n",
              "      <td>-0.006384</td>\n",
              "      <td>1.979267</td>\n",
              "      <td>1.727603</td>\n",
              "      <td>0.891768</td>\n",
              "      <td>0</td>\n",
              "      <td>-1.594273</td>\n",
              "      <td>0.398636</td>\n",
              "      <td>3.124169</td>\n",
              "      <td>2.432611</td>\n",
              "      <td>0</td>\n",
              "      <td>46343</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>-1.590567</td>\n",
              "      <td>1.618243</td>\n",
              "      <td>-0.679271</td>\n",
              "      <td>-0.327864</td>\n",
              "      <td>1.896059</td>\n",
              "      <td>-0.154470</td>\n",
              "      <td>0</td>\n",
              "      <td>-1.590567</td>\n",
              "      <td>0.374122</td>\n",
              "      <td>2.640312</td>\n",
              "      <td>2.422674</td>\n",
              "      <td>0</td>\n",
              "      <td>46343</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>-1.586861</td>\n",
              "      <td>1.708843</td>\n",
              "      <td>-0.591184</td>\n",
              "      <td>-0.311630</td>\n",
              "      <td>1.923702</td>\n",
              "      <td>-0.154470</td>\n",
              "      <td>0</td>\n",
              "      <td>-1.586861</td>\n",
              "      <td>0.354029</td>\n",
              "      <td>3.032657</td>\n",
              "      <td>2.412742</td>\n",
              "      <td>0</td>\n",
              "      <td>46343</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "       time  heart_rate  x_motion  y_motion  z_motion  step_count  \\\n",
              "0 -1.601686    2.343046 -1.304122  1.648084  0.251126    8.607776   \n",
              "1 -1.597980    3.339652 -1.513119  1.612262  0.111929    4.815162   \n",
              "2 -1.594274    2.524247 -0.006384  1.979267  1.727603    0.891768   \n",
              "3 -1.590567    1.618243 -0.679271 -0.327864  1.896059   -0.154470   \n",
              "4 -1.586861    1.708843 -0.591184 -0.311630  1.923702   -0.154470   \n",
              "\n",
              "   sleep_label  time_feature  heart_rate_feature  count_feature  \\\n",
              "0            0     -1.601686            0.480706       5.530874   \n",
              "1            0     -1.597980            0.451398       4.483025   \n",
              "2            0     -1.594273            0.398636       3.124169   \n",
              "3            0     -1.590567            0.374122       2.640312   \n",
              "4            0     -1.586861            0.354029       3.032657   \n",
              "\n",
              "   cosine_feature  psg_label  subject_id  \n",
              "0        2.452504          0       46343  \n",
              "1        2.442553          0       46343  \n",
              "2        2.432611          0       46343  \n",
              "3        2.422674          0       46343  \n",
              "4        2.412742          0       46343  "
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 52
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "90QTFk8gwKEJ"
      },
      "source": [
        "### Look at the spread of the variables in the data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 388
        },
        "id": "gSU-uW-oG4n8",
        "outputId": "441b0cdc-7035-4d93-f5b0-1ca16483fdba"
      },
      "source": [
        "df_std = (all_subjects_data_df[input_cols] - train_mean) / train_std\n",
        "\n",
        "df_std = df_std.melt(var_name='Column', value_name='Normalized')\n",
        "\n",
        "plt.figure(figsize=(12, 6))\n",
        "\n",
        "ax = sns.violinplot(x='Column', y='Normalized', data=df_std)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAtAAAAF0CAYAAAAKMg75AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdeZhcZZ328e+vqnpLurN0ujv7RhYSQAgQ9sWgA4pEcRTHF0deRUYUF5DFccSNUYfXGWV0EGcUREFABUUUgmYgYQt7QhbIvpGQpdPprN2dXqvqef84p7qr966kq09V1/25rr66utZfnT7LfZ7znOeYcw4REREREembUNAFiIiIiIhkEwVoEREREZEUKECLiIiIiKRAAVpEREREJAUK0CIiIiIiKVCAFhERERFJQSToAvqirKzMTZkyJegyRERERGSQe+ONN/Y558p7ek5WBOgpU6awbNmyoMsQERERkUHOzLb39hx14RARERERSYECtIiIiIhIChSgRURERERSoAAtIiIiIpICBWgRERERkRQoQIuIiIiIpEABWkREREQkBQrQIiIiIiIpUIAWEREREUmBArSIiIiISAoUoEVEREREUqAALSIiIiKSAgVoERFJm1/84hds3rw56DJERPqVArSIiKRFY2MjDz30EN///veDLkVEpF8pQIuISFrt2rUr6BJERPqVArSIiIiISAoUoEVEREREUqAALSIiIiKSAgVoEREREZEUKECLiIiIiKRAAVpEREREJAUK0CIiIiIiKVCAFhERERFJgQK0iIiIiEgKFKBFRERERFKgAC0iIiIikgIFaBERERGRFChAi4iIiIikQAFaRERERCQFCtAiIiIiIilQgBYRERERSYECtIiIiIhIChSgRURERERSoAAtIiIiIpICBWgRERERkRSkLUCb2UQze9bM1prZGjO7wb//NjPbZWYr/Z8PpKsGEREREZH+Fknje0eBm51zy82sBHjDzJ72H/uxc+5HafxsEREREZG0SFuAds5VApX+7VozWweMT9fniYiIiIgMhAHpA21mU4BTgdf8u75kZm+a2a/MbORA1CAiIiIi0h/SHqDNrBh4FPiKc64G+B9gGjAHr4X6jm5ed62ZLTOzZdXV1ekuU0RERESkT9IaoM0sDy88P+Sc+xOAc67KORdzzsWBe4Azu3qtc+5u59xc59zc8vLydJYpIiIiItJn6RyFw4B7gXXOuf9Mun9s0tP+HlidrhpERERERPpbOkfhOA+4CnjLzFb6990KXGlmcwAHbAM+l8YaRERERET6VTpH4XgRsC4e+mu6PlNEREREJN10JUIRERERkRQoQIuIiIiIpEABWkREREQkBQrQIiIiIiIpUIAWEREREUmBArSIiIiISAoUoEVEREREUqAALSIiIhKAw4cPc+WVV7Jq1aqgS5EUKUCLiIiIBGD9+vXs2rWLBx98MOhSJEUK0CIiIiIBcs4FXYKkSAFaRERERCQFCtAiIiIiIilQgBYREREJkLpwZB8FaBEREZEAmVnQJUiKFKBFRERERFKgAC0iIiIikgIFaBERERGRFChAi4iIiIikQAFaRERERCQFCtAiIiIiIilQgBYRERERSYECtIiIiIhIChSgRURERERSoAAtIiIiIpICBWgRERERkRQoQIuIiIiIpEABWkREREQkBQrQIiIiIiIpUIAWEREREUmBArSIiKSFcy7oEkRE0kIBWkRE0srMgi5BRKRfKUCLiIiIiKRAAVpEREREJAUK0CIiIiIiKVCAFhGRtNLJhCIy2ChAi4hIWujkQREZrBSgRURERERSoAAtIiIiIpICBWgRERERkRSkLUCb2UQze9bM1prZGjO7wb+/1MyeNrNN/u+R6apBRERERKS/pbMFOgrc7Jw7ATgb+KKZnQD8C7DYOTcDWOz/LSIiIiKSFdIWoJ1zlc655f7tWmAdMB64HLjff9r9wIfTVYOIiIiISH8bkD7QZjYFOBV4DRjtnKv0H9oDjB6IGkRERERE+kPaA7SZFQOPAl9xztUkP+a80fW7HGHfzK41s2Vmtqy6ujrdZYqIiIiI9ElaA7SZ5eGF54ecc3/y764ys7H+42OBvV291jl3t3NurnNubnl5eTrLFBERERHps3SOwmHAvcA659x/Jj30OPAp//angL+kqwYRERERkf4WSeN7nwdcBbxlZiv9+24FfgA8YmbXANuBf0hjDSIiIiIi/SptAdo59yJg3Tz83nR9roiIiIhIOulKhCIiIiIiKVCAFhERERFJgQK0iIiIiEgKFKBFRERERFKgAC0iIiIikgIFaBERERGRFChAi4iIiIikQAFaRERERCQFCtAiIpIWzrmgSxARSQsFaBERSSuz7i5KKyKSnRSgRURERERSoAAtIiIiIpICBWgRERERkRQoQIuIiIiIpEABWkREREQkBQrQIiIiIiIpUIAWEZG00njQIjLYKECLiEhaaPxnERmsFKBFRERERFKgAC0iImmhrhsiMlgpQIuISFqpK4eIDDYK0CIiIiIB0tGa7KMALSIiIhIgHaXJPgrQIiIiIgFQy3P2UoAWERERCZBaoLOPArSIiKSVWtlEeqZlJPsoQIuISFqpdU2ka1o2spcCtIiIiIhIChSgRURERERSoAAtIiIiIpICBWgREUkrnSAlIoONArSIiKSFTpASkcFKAVpEREREJAUK0CIiIiIiKVCAFhERERFJgQK0iIiISIB0om32UYAWERERCZBOuM0+CtAiIiIiIilQgBYRERERSYECtIiIiIhICtIWoM3sV2a218xWJ913m5ntMrOV/s8H0vX5IiIiIiLpkM4W6PuA93dx/4+dc3P8n7+m8fNFRCRAGllARAartAVo59wLwIF0vb+IiGQHjTAg0jPtbGafIPpAf8nM3vS7eIzs7klmdq2ZLTOzZdXV1QNZn4iIiMiA0U5m9on09KCZPQF0u1vknPtQip/3P8D3/Pf8HnAH8Jlu3vtu4G6AuXPnatdMRCRLqXVNpGtaNrJXjwEa+JH/+yPAGOBB/+8rgapUP8w51/oaM7sHWJDqe4iISHZR65pIz7SMZJ8eA7Rz7nkAM7vDOTc36aEnzGxZqh9mZmOdc5X+n38PrO7p+SIiIiIimaa3FuiEoWZ2nHNuK4CZTQWG9vQCM/sdMA8oM7OdwHeAeWY2B68Lxzbgc0dZt4iIZAkdphbpmZaR7NPXAH0j8JyZbQUMmEwv4dc5d2UXd9+bWnkiIpKtdFhapGdaRrJXnwK0c26hmc0AZvl3rXfONaWvLBERERGRzNSnYezMbAjwVeBLzrlVwCQzm5/WykREREREMlBfx4H+NdAMnOP/vQv4floqEhEREckh6gOdffoaoKc55/4DaAFwztXj9YUWERHpkkKBSN+oL3T26WuAbjazIvyLqpjZNEB9oEVEpFcKByIy2PR1FI7bgIXARDN7CDgP+HSaahIRERERyVh9HYXjKTN7Azgbr+vGDc65fWmtTEREREQkA/V1FI7FwFnOuSedcwucc/vM7O401yYiIiIy6Ol8gezT1z7QU4Gvmdl3ku6b292TRURERKRnOj8ge/U1QB8C3guMNrMnzGx4GmsSEZFBRK1rIjLY9DVAm3Mu6pz7AvAo8CJQkb6yREQk26l1TUQGq76OwvHzxA3n3H1m9hbwxfSUJCIiIiKSuXoM0GY2zDlXA/zBzEqTHnobuCWtlYmIiIgMYurelL16a4H+LTAfeAPvIirJx+MccFya6hIRERHJCerulH16DNDOufn+76kDU46IiAwWal0TkcGqty4cp/X0uHNuef+WIyIig41a10RksOmtC8cdPTzmgPf0Yy0iIiIiIhmvty4cFw1UISIiMjipK4eIDDZ9HcYOMzsJOAEoTNznnPtNOooSEZHBQ104RGSw6VOA9i/hPQ8vQP8VuBTvYioK0CIiIiLHQEdpsk9fr0R4Bd6lvPc4564GTgF0OW8REemVwoFIz3SUJvv0NUA3OOfiQNTMhgF7gYnpK0tERLKdQoGIDFZ97QO9zMxGAPfgXVSlDnglbVWJiEjWU8uziAxWfQrQzrkv+Dd/bmYLgWHOuTfTV5aIiAwWaokWkcEmlVE4TgamJF5jZtOdc39KU10iIiIiIhmpr6Nw/Ao4GVgDxP27HaAALSIiIiI5pa8t0Gc7505IayUiIiIiOUjnC2Sfvo7C8YqZKUCLiIiI9DOdJ5B9+toC/Ru8EL0HaAIMcM65k9NWmYiIDApqXRPpmZaR7NPXAH0vcBXwFm19oEVERLqlVjWRvtGykn36GqCrnXOPp7USEREREZEs0NcAvcLMfgs8gdeFAwANYyciIt3RYWmRvtGykn36GqCL8ILzJUn3aRg7ERHplQ5Pi/RMy0j26TVAm1kY2O+cu2UA6hERERERyWi9DmPnnIsB5w1ALSIiIiIiGa+vXThWmtnjwB+AI4k71QdaRERE5NioD3T26WuALgT2A+9Juk99oEVEpFcKByI9Ux/o7NOnAO2cuzrdhYiIyOCiUCAig1WfLuVtZhPM7DEz2+v/PGpmE9JdnIiIiMhgpaMz2atPARr4NfA4MM7/ecK/r1tm9is/bK9Ouq/UzJ42s03+75FHW7iIiIjIYKCjNdmnrwG63Dn3a+dc1P+5Dyjv5TX3Ae/vcN+/AIudczOAxf7fIiIiIjlLLdHZp68Ber+ZfdLMwv7PJ/FOKuyWc+4F4ECHuy8H7vdv3w98OKVqRURERAYJtTxnr74G6M8A/wDsASqBK4CjObFwtHOu0r+9Bxjd3RPN7FozW2Zmy6qrq4/io0REREQyl1qes1dfR+HYDnyoPz/YOefMrNs5xzl3N3A3wNy5czWHiYiIDBL19fUMGTIk6DIyhlqis0+PAdrMvt3Dw845970UP6/KzMY65yrNbCywN8XXi4hIllDrmnTl4MGDXH755fzzP/8z8+fPD7ockaPSWxeOI138AFwDfO0oPu9x4FP+7U8BfzmK9xARkSyi1jVJVlVVBcCf//zngCvJHNrZzD49tkA75+5I3DazEuAGvL7Pvwfu6O51/vN/B8wDysxsJ/Ad4AfAI2Z2DbAdr1+1iIiISM7RzmX26rUPtJmVAjcB/4g3csZpzrmDvb3OOXdlNw+9N6UKRURERAYhtTxnr976QP8Q+AjeyXzvcs7VDUhVIiIiMiip1bUzTZPs01sf6Jvxrjz4TWC3mdX4P7VmVpP+8kREREREMktvfaD7Ok60iIhIl3SYWqRnWkayjwKyiIikhQ5Li8hgpQAtIiJpoVY16YrmizaJnUxNk+yjAC0iImmllmiRrmnZyF4K0CIikhZqVZOuKDS20TKSvRSgRURERAKknYrsowAtIiJppXAgIoONArSIiKSVDlOLdE3LRvZSgBYRkbTQCAPSFc0PMhgoQIuISFqpC4d0RfOFdjKzmQK0iIiIDDiFxjbamcg+CtAiIpIWCkgiPdMykr0UoEVERGTAqdW1jaZF9lGAFhGRtFD/ThEZrBSgRUQkLdSqJj3RjlUbTYvsowAtIiIiEgDtZGYvBWgREUkLtapJTxQe22hZyT4K0CIiklYKStIVhUYtG9lMAVpEREQGjEJjG+1EZC8FaBERERkwCo2daaci+yhAi4iIiIikQAFaREREJEBqlc8+CtAiIpJWCgeSTPNDZ+rCkX0UoEVEJK0UDiSZ5ofOtFORfRSgRUQkrRQOJJnmh860U5F9FKBFRCStFA4kWWJ+0HyhnYlspgAtIiIiEiDtTGQfBWgREREZcGp9lWymAC0p27JlC9XV1UGXISIiWUytrm20M5F9FKAlZVdffTVf//rXgy5DRESymEJjG+1MZB8FaDkqGzduDLoEEckSCkrSFYXGNlpGso8CtIiIpIUCkvREobGNlpXsowAtIiIiIpICBWgREREZcGp1baPW+OyjAC0iIiIDTqFROxHZTAFaRETSQgFJRAYrBWgREUkrtbKJyGATCeJDzWwbUAvEgKhzbm4QdYiIiMjAShyZ0I5VGx2tyT6BBGjfRc65fQF+voiIDACFA5GeaWci+6gLh4iIpJXCgSRLzA/asZJsFlSAdsBTZvaGmV0bUA0iIiIigdPORPYJqgvH+c65XWZWATxtZuudcy8kP8EP1tcCTJo0KYgaRUSkHygcSDL1ge5M0yL7BNIC7Zzb5f/eCzwGnNnFc+52zs11zs0tLy8f6BJFROQYJYJSKKTegtJGYbGNdi6z14Cv1cxsqJmVJG4DlwCrB7oOERFJL4UD6Ynmjzbaqcg+QXThGA085s8sEeC3zrmFAdQhIiIDQEFJpGdaRrLPgAdo59xW4JSB/lwRERGRTKQW6OyjjmkiIpIWGq5MuqKTCGUwUIAWEZG0UlAS6Zl2MrOPArSIiIgMGB2Z6Ew7mdlHAVpERNIiEZAUlER6pmUk+yhAi4hIWqivq0jPtGxkLwVoERERGXAKj2p5zmYK0CIikhbqwiE90XzRRjsT2UcBWkRE0krhQKRn2pnIPgrQIsdg4cKFbNiwIegyRDKSQoFIzzQiSfYK4lLeIoPG7bffzvDhw3niiSeCLkVERLKMTrTNXmqBFjlGhw8fDroEkYykcCAig5UCtIiIpJUOT0sy7VjJYKAALSL94vXXX6eqqiroMiQDKShJMvX7baNpkb0UoEWkX9xyyy185zvfCboMyUAKByIy2ChAi0i/Wbt2bdAlSAZR65pIz9SdJXspQIuISFokQoHCgSSLxWJBlyByzBSgJSXRaDToEjLGT37yk6BLEBHJOi0tLUGXkHG0kwlNTU3cdddd7Nu3L+hS+kQBWlLS0NAQdAkZY/369UGXkDHuvPPOoEuQDKYuHJJMLdCdaRmBl19+mUceeYTf/va3QZfSJwrQkpL6+vqgS8gY2gi02bx5c9AlSAaKx+OAWtekvcS6U/NFG00LePTRRwHYs2dPwJX0jQK0pOTee+8NuoSMoQDdRq0n7f3sZz/jzTffDLqMwCXmC80fkiyx7tR8IckSDTFr1q3Liu2rAnQfvfjii7z11ltBlxG4bdu2BV1CxkhewLNhYU8nde1p09TUxMMPP6wh/ZKodc3zpz/9SWOlo3NpupLrOxMHDx6kvr4eF8nj4P79WdFFUgG6j2699Va+/OUvB11G4BKHZAWam5tbb1dXVwdYSfAOHjzYejvXd7K+8Y1vALB///6c37HI9VCQrLq6mp/85Cc6+Zi2kwg1f0hCovU5XjKs3d+ZTAG6DxIrPIXH9i2tud4fura2tvX2smXLAqwkeIcOHSIfMOD5558PupzAPPfccyx9fWnr37fcfAs1NTUBVhQsdeFoc9dddwGwevXqgCsJXqLxQUcmNFZ6wuHDhwFwefmAt03JdArQfbB0adsGccWKFQFWErzk4YeyZaiZdDh06BA1NTW4IoeVGH9b+LegSwpMS0sLLS0tDAFGhEK88847QZcUiFdffZXvfOc7hEN5jCoex9nHfZC1a9dy41duzNlD1jqJsM3KlSsBLygkwkKuSmxHNF+0yfVp0XpE1wzM2h3hzVQK0L04ePAgu3btxoUiUFjCD3/0o6z4x6ZLchDIhj3EdElsDCmA2IQYq99aTWNjY7BFBSQcDgMQB5qAwsLCQOsJypNPPkleuIARQ8oJWZhJo2Zx8oR5bNq8iY0bNwZdXiByvVUtYcWKFRw8eJCheUWEQyF+9rOfBV1SoJqamgDNH6CjNAmtR/jNIBTKivOKFKB78Zvf/IZ4PEa8aASNk89l544dPPHEE0GXFZjkbiy5PBj+/v37vRthYKi38kvuB5xLQqEQhYWFNAL18ThTp04NuqRAnHTSSTRHG2mKev2eo/EWtu1fTUlxCZMmTQq4umAk1he5HA6cc9z1058SCUUYO7Sciyefy8KFC9m0aVPQpQXmmWeeAXJ7vkjQkH6e5MBsCtDZzznH008vwkUKIRQhNmIirricp59+OujSMkIur/xCoaRFx3VxX44Jh8MkVnfFxcWB1hKUj370o5w651RqGw8Sj8dYX/kaB49U8Y1vfiNnp4m6cMA777zDps2bKS0cRsiM9009j0gozKJFi4IuLTB79+4FyNmjdsmyISgOhHbTwUJZcc5Z7m7x++Dw4cPU1BzGhfNa74sOLeftHB5lIDk053ILdLuF288GubxD0dzURAHepNi9e3fQ5QQiEolw4003Ao6maAPvHFjHOeecw7nnnht0aYHJ5WUiYfv27QAURgoAGJJXxJjistb7c1GiK2Cu9wUHjUiS0BqgzSBkCtDZrnWUCWubTC6cT2MOD02V3Ac6eRSKXNN2wgNeNw7I2b7x0WiUlmiUfGB4KERlZWXQJQVm3LhxAMRdjKZoQ+vfuUpdOODIkSMAhJK2I0XhwpwexSgRGmtqanJ2vZmgEUk87QKzunBkvy7/gRbCOZcVe0fp0NjYSEHIETLYunVr0OUEJnnecOaFg1wdaSFxQpAB+eT2RVUSre/hUIQh+SU52xrfUS6Hg9Y+rkn3hSxELEfXF6Ax9JMlpkUu72RCx6O6aoHOem0hKWnV528IsuGf29/27t3LkSNHKM6LM2tklGefWZyT0wE6dF8JdXFfDkkMZxgGhsTj7M/h4Q137twJeAG6pKCUHTt2BFxRsDTCQPK2om07EjIjlqPrzng8TnNzc+voPbt27Qq4omAlGhxyeScTktYRZoBlxTpDAboH7Q7T+1wo3P6xHLJgwQIARhQ43jOukco9Ve3GyM4lrSe/GBDxbiZaYnPNqlWrACgAxgMbNm7M2cPTiaMQhhGyMNFo5h+GHAi5HA4S24pQ0jTIC0Vobsq9bQhAVVUVzjny8rxzi3J9J1ND+nk6tkBnw/RQgO5BIgRYUx2h+v0Url1A+KC3sCf6teWSl196kaKIIy/kOL2ihYKI8dJLLwVdVrAyfxlPu0VPP00EyANm4YXIJUuWBFxVMEaOHAl4faAbW44wqrQ04IokaIltxe66veyo3cMdr/+aqiP7c3IbAm1HaSKRCJFIpPXvXJVogW5pyc0dqoR2XWbVBzr7tQboeBSLtRCu3UOopaHdY7mipaWFrVvfpijsJca8EEwtbmFTjl4gYtSoUd4NB1Zv7e/LIStWrGDlqlWU+H9PAsotxG/uvz8nj9KUl5cDEIvHaIodoay8LOCKgqUuHN62woCGaBMN0SY2HtxOY6yJ+vrcDNCJ8wJCoRBFRUU5f55AIkDnWqboKHkUDhcKZ0WXSAXoHnR5SN56eGwQ2759O9FYjMJw24ZwUkmULVu25GQ/6PHjx3s3YsAR7xD16NGjA61poNXX1/P/br+dAqAeqAR+DYxycXbs3Mmvf/3rYAsMQGInKu5i1DfVtgbqXKVxoL0uHGbtN7WG0dyc+QEhHRLD9yUuwPT2228HXFGwEkci6upyd1Qr6NAtNhTKipPyFaB70PUhhNw8iTBx1awDTSG214b5/rJidh8J09jUlJOH4FqvRlkLtj07+mv1t4ceeog9VVWMxLuEdxOwDWgETgN+97vf5dxYtwUFBUTCEaKxZlpizYwZMybokgKVa+vJrnS1HTEsKw5Rp8Obb77ZegLhsGHD2LNnT9uVXXNQYizsmpqagCsJVuJ8qnB1Fa72MBs2bAi4ot4pQPegp1CUaxuGLVu2YEBjzGiIhVh/KI+6Fmt9LNds3rwZAItZaxeOdevWBVnSgGppaeGxRx/lRKCwi8cvAcLO8dhjjw1wZcELR8LEnBeOSkpKenn24KZxoL3v3qn93cC53NqGANTV1bFp0yYiEe/M68Q5AytXrgyyrEAdOODtPNTV1WdFt4V0OXDgAADW3IS1tGTFdSYUoHvQU0jOpQAdi8V45eWXKIy03wgWhB1FEePFF18MqLLgdHUFrVyaDpWVldTV1zOrm8eHYkxwjg3r1w9oXUG78847aW5uJh73Dj8uXLgw4IqC1ToGcg534fCmQefvH8/BnYrVq1fjnGsN0MXFxUQikdaRfHLRnj2VRPxta1VVVcDVBKfjERl14chyXe0NJtoScmlP8YknnmDHzl2MLGi/02DAu8c1sHjRItbnUFCqra3t3Ac+AqvXrA6moAAUFnrtzj2dJthsRmFR0cAUlCE2b96Mcw7nD8+yZ8+egCsKVqJfYy41OHTU0tJCx/2HRBeOXOvGsWLFCsAbBrS2tpaVK1cSDodZvnx5wJUFo7a2ltraI0yc6C0fuTwmduvQsD6vISKz1xuBBGgze7+ZbTCzzWb2L0HU0BddnhXrrwlz5Wprzc3N3PvLezihNMqwvM4z899PbaQk33HP3XcHUF0w1q5d2+k+F3Fs2LAhZ0aeKC8vpyA/n556Lu43Y9KkSQNWUybK5ZbX+vp67vrpTwFvPfLyyy8HXFEwGhoa2l3GG9ou650r2xHwurI8//zzRCKR1p2HQ4cOEQqFeOedd3LufAlo67oye3aMUIicbYk/dOhQpwAdj8fZmOGjfA14gDazMPAz4FLgBOBKMzthoOvoi7q6us53+iu+bOif0x+2bdvG4Zpa3jOuscvHh+Y5zq5oZOWq3OnD1uXY1/nQ3NScMy0pZkZFeTmHunm8CUdjPE5FRcWA1pVpcrXvb0tLC9/85jdZv3497xpVwvCCPL71rW/x5ptvBl3agKupqekUoMM5th0B7ySx3bt3k5+f3+7+vLw8zCznzpeor6/nqaeeoqDAmDwlztixcV544bnWK7vmkueee67L+xcvXjywhaQoEsBnnglsds5tBTCz3wOXA52b9QJ24MABLzAnn+zhr/gSHd4Hu8QhlPxw98/JD7uMP9TSn1a9uQqX57CWpNbFfMDgrbfe4uyzzw6stoE0acoUNlVWEorHKSoq4rLLLuPJJ5+ksaGB6sRzcqwFumNgHsyH6Jubm9m3bx979+5t91NVVcX27dvZvXs3J5aWMG5oEWWFBSytPszNN9/MrOOPp2L0aCoqKhjt/078lJSUDLpW+4MHDhKx9ivQsH9F24MHDzJ27NggyhpQzjnuueceioqKWq9AmBAKhRgzZgyPP/44V1555aAeDvTAgQO89NJLLFnyAsuWLSMajXHSSVHCYZg1O8riRTv5yEc+wgknzObCC9/NBRdcwMSJE4MuO+0WLV6Mi+Rh0bausSbmMIgAACAASURBVPHCIhYtXswXvvCFjF0nBBGgxwPJ1+7cCZwVQB298kKytQsHdfEWLBzJmQCdWNm1xKEh2n5aNPid/KNxIy+S19PbDCqNjY2dj90YWNhyanzwSCRCHK8f9GWXXcb1118PwKI//pFY0nNySccuPB0PS2aLWCzG/v37O4XjvXv3UrVnD1VVVRzq4kTa/EiYwnCIgpBxUmkJQ/MibD18hNLCfE4rG8bmw0d4Z8M6Nq1fR0NLtNMOR0F+PuXl5YweM6ZTuE78DBkyZKAmQ784cOAADtqtO2PxWOtjueD5559nw4YNzJo1q8vzAqZOnUpVVRX33nsvt956awAVpodzjt27d7NkyRKWLHmB1avX4Jxj+HDjpJNaOG5ajDFjvGVg9uw4FRVNvL01xNat6/j5z9fx85//nMmTJ7WG6ZkzZxIKDa5T1+rr672TS4uGYnVtAdoVFrF/3z62b9/OlClTgiuwBxm7dTOza4FrIbhWrEOHDuFoHw4eeXwh5BV1OQrDYJQYhqu2xaiPGpfNb5sWzy94uPWxkuKhgdU40IoKizpfwtuBizqKcuikuU0bNlARj3MAePLJJ8H/PRxIjH68efNmzjnnnKBKHHAdL8985MgRbxizDGtBaWpqYtu2bV22Hlft2cP+Awc6HVWKhEMUhcPkh4yh4RCjhg+lKByiMBGaw2EiobbveaiphdW1jXzgsvn89ckFnFRSyLtGDWt93DlHUyxOYyxOYyxGY9T/fWAfm/ftZc2bjsaWzmfiDx0yhIqKCsaMHdspXE+YMCGjLl4Ti8WoraulKFLAZZd9sHXdufiJp4CuR/MZbJxz/OY3v2Ho0KGMHTu2ywBdWFjI+PHjeeqpp/jMZz6TleOnR6NRduzYwaZNm9i8eTObNm1i06aN1NR43XTKyuGMM1o47rg4o8pc64mlC/8Woa7OOO/8KGPHOkaNijH3jBi1tbB1a5i3t27joYfe4YEHHqCwsIBp06YzY8YMZsyYwfTp0znuuOMoKCgI8Jsfm40bNxKPxXAFBZDUa9b532nNmjUK0El2AcnHJCb497XjnLsbuBtg7ty5gXQkrDtSD6FQu3DgIkU4XM5cdrO8vJzioUPYXtvEkIhrNy0q/KF3ttflMXX29CDLHDDRaJTKPZVA+xal+mg9oSEhduzY0cs7DB7FJSUcrKqi0Dn2NDTwxz/+EYDR0Hpy4dChubNjBZ0DUSwWY926dZxwQmad5vFv//Zv3fY7DBkMz89jREEhI/LzKIqEKYyEiJiltCNwoLGZD1w2ny9/+cuAY+VTf2NEQduRKjPzwnckDHR9BCvuh+yGaIy6ligHm1o41NTI29u28fa2bZ1rD4V48MEHmTBhQp/rTKfEEYj8UH67dWdpXgkHm2pyYjuydu1aNm/ezPHHH4+ZEY1G2607E8OVTZw4kZ07d/LEE0/w2c9+NuCqe3bkyBG2bNnCpk2b/N8b2bp1Ky3+Dl84DKNGwfgJUeaUOSZPiTN8eOcYU1lp7N5dwmWXXcbChQu45H2HGTvWe15JCZxySoxTTonR0ADbt4fYWxVl3763+Otf19Lc7D3PQsbECROYOfN4pk+fzvTpXsBOjLGd6RInCrq89n3jieRh+QVs2LCByy67LIDKehdEgF4KzDCzqXjB+f8Anwigjl61tLSAhWlICgeUDMe5aM4MY2dmTJ8xk51vv0FRJE5DXdu0KBrhiDvYVRfi3TNnBlzpwHj66adpqG+AgvZHJv7wtz8QHRvlhSUvUF1dnVGtYOlyxcc+xu23305X3/QpjGHFQ7n44osHvK6gLFmypFOANoz/+q//4o477qC4uDigyjq75pprOPHEE70W56oq9lRWUlVVRU1tLXEHB5taONjUggFFeXnkh2jX2lwUCVMYDlMUCRHp5pByaWE+f31yAeD465NPclJJV5fc8SSCcmM0RoP/u9EPzk3O0RiN0RLr0CIeiVBRXs7YsWMZPWYMFRUVTJ48mXHjxvXjlDo2ie1EfjiPww21SevO8naPD2Z/+ctfiEQirX2bo9Eo8+fPb113LliwAPBaoUtLS1mwYAGf/vSnO/WVzgT3338/f/3bk1TubmtFN4MxY+KceFKc8rI4ZeWOESMc4R7OG0rYtSvEZZddxpe/fD0Y7NjxO8aO7XzeRFERzJoVZ9asxIWJoKbG2Fdt7NsXYu/ebTz33A4WLVrU+prS0hGcdtpcbr311ozuSrdu3TooGQYddqxqm5poqRjDmgy+QNmAT1XnXNTMvgT8LxAGfuWcWzPQdfRF9yfGWU6dNFdQUECNg3AXjU/OQdzR6czqwai2tpb//p//hlIg1r7bAgXgjnfEtsW48847+d73vhdssQPgkksu4Ve//CU1e/e2uz8ObMNx9cc+NuivxOecY/Xq1Tz66KM888wzREJ5RONtoaiksJT16zdw1Sev4spPXMnFF1+cES1DkydPZvLkyZ3ub2xsbA3Ve/fuZY/f3zkRst/Zt6/TiZH5Ya+FusBvUS6KhMjzQ3VF2Hh+weNURCLUtUSpa4kSd87rtuGH5KY4NLS0dOoVNaykhNETxjNm7FhGjx7d+lNRUcGYMWMYMWJExvcHTWwnumu4H+zbkf3797No0SJGjx7dGuIikUi7dWdyuJswYQKrVq3imWee4X3ve18gNfekvr6eeCxGKBRKusomVFaGqK2NsKcyRsmwOMNKHCXDHMOGOUpKvJ+uMuz48XEWLlwABgv/toBL3tf1/BCPQ12dUVvrBefaGqOm1qitCVFXF6am1tHVhS2PHOliJLEMEo/HeWP5cprHTiC8Z1e7RqmHn3qalqnT2fLGK9TW1mbktiSQ3RLn3F+Bvwbx2XJ0WpzRVURu8bd6mdbHMx3uv/9+Dh8+TOy9MUKrQjQcTjoyUQwMhdjsGM8//zzLly/ntNNOC7TedAuFQowsLeVQhwCdCEKjRo0a+KIGwL59+1ixYgXLly/n1VdeZf+B/eSF85k99myqa3ewr66tR1pB3hDOmHopb+58jrvuuouf/ey/OfHEEzjrrLM49dRTmTVrVkbtfBYWFnYbrsHrknLw4MHWUJ38U1lZyd6qKnYcOtLFK9ufXBkOhykvK2NyUjge47ciJ/5OXKxHstdf/vIXotFou5EkIpEIdXV1revOESNGtD5WWlpKcXExjzzyCJdccknGbVeuu+46rrvuOqLRKPv376eyspI9e/a0/lRWVlJZuYutW/YR63DEZOjQECUlMUpK4l64LnHk5TlmzaplxfLfM2uWo+awcfBAiNo6LyTX1hq1tWFq69oHZDOjtHQkY8aOZfbssYwdO5YxY8a0/lRUVGRFv+g1a9Zw6OBBonMvIHRgf/vuskOHEZ0ynfjSl3jppZd4//vfH3C1nWVuu75kjFNOOYXXXnuN44Z13sV9vcrb+J988skDXdaAe/GlF3FjHPTQgOhmOmyt8fLLLw/6AA3eyWgdN3GW9Nhg0NjYyKuvvsrSpUt544032L17NwD5kUIqSiZx5tSzGD9yBnnhfJ5ac3+7w5AtsSZGFY/lollXcri+mh0HN7D77a3cu/pXgCMvL5+TTjqRuXPncu655zJt2rRgv2wvwuEwZWVllJWVceKJJ3b5nPr6+h7HNw6Hw4wcOZJwX45xD1aZlQvTIh6Pe/29S0v7PHKKmTFu3Dg2btzIli1bmD49M8+tSXRJ6W7IvVgsxr59+zqFa+/3LrZure4UsJOZGaNGjWTMmHGccIIXjpNDckVFRUbteB+t3//+91h+AS3TZpK/diUN+5IapUrLiY0ZDyNL+f0jj3DxxRdn3DpDAVp6NX/+fO6995ccbuq81l9SWcCkiROYM2dOAJUNrLJRZVS+XUm8q2NlCUfAxd2gbX3taNjw4Z3OAE4c4M/EQ26pevLJJ/nxf/6Y5pZm8iMFlBVP5JSJ8ygvmciIIRWdLpDREm3isg+1HYb82+NPtz42fEg5w4eUc9L482mKNrCvdid7a3ewbeMOVqxYwT333MOMGTP44Q9/SGlp6YB+z/40ZMiQrBtqTvrfgQMHqK6uZsaMGSm9LjHvr1mzJmMDdG/C4XBrwD7llFM6PZ4YJrKrK9cmdlIzsQ94f3HO8cgjj7BkyRIaz5kH+d20lptRf9a72brwMe68806uv/76jArRCtDSqxEjRjBt2jR2bd3Q6bFtdXm8f94ZGXeoLR0+9rGP8ea33sS2d/9dQ2tCFBYVcumllw5gZcGZNm0aq1a2vwplU9Jj2e7hhx+muaWZ4oIRzB53DqOGjqOkcGS383tepKDdYciCSNc7EQWRIsaPnEHFsMkcPFJF5eEtbNizlE2bNvH6669n5OFKSV2uXokSvO3GkCFD2L17N6WlpX0akScWi7F161aAQX0BkXA4nLNXaa2treXHP/4xixYtomXa8TSd3vMwpy0zZtNUtZvHHnuMbdu3c+vXv54xF9tRgO5BLq/8OqqoGM3utzdSFPZaXyeXxJgwNMb6Qy5nVgQXXnghEyZOYOf2nbhOpzwBLWC7jA9//MPt+vUNZmeeeSaPPvooeXjXlhkL1AAjhw8fFAH6u9/9Lvfddx+vvvIqS9/+GwB5kXyGFZYxvKiMEUMqWn8ioTzywgUcqt3behiyuKQM8NYl9c21HKqv4lD9Xg7VV1PTtI/ahoOtnzVlylTe975LcmrkksEul7chkUiEH/zgB3zjG99g6dKljBkzhkmTJlFcXNzaxaekpITi4mKi0Si7du1i586dNDc387nPfY5TTz014G8g/e3IkSN89tpr2V1ZSePZF9I09zzo7URgMxrPfy+x0jJWvvA0n7nmn7jn7l9kxGg7CtA96G7l53p4bLAaPXo0LS7E1BLvkNM359axoy7Eol2FWTno/dEwM84+62we/fOjxCfH4ZD/wAhwIxxUAw7OOisjL6yZFqeffjoF+fnkNzczCrga+H9mXHzhhRk/QkJfTJkyhdtuu41oNMr27dtZv349mzdvZsuWLWzetJmt1W8CELIQI4eOxjlHJJSHWYjhReWEQ3m8umUB+47spL7JCw1eP8/xzD11DscddxwzZ85k1qxZGTE6hwyMxPGLwT4Kx5w5c7jvvvt48MEHWbBgAZWVlYwbN46hQ4cSCoWYM2cOu3bt4rXXXqOpqYm5c+dy9dVX8653vSvo0iUNtm3bxu5du4iNKqdlyozuh6fpyIzY+Ek0j5tA7fatrF69WgE608ViUeKRQkLO69UZHzKK+NBRhI/sax38PVeMHj2apqgj7oyQeTsPBxr9oapypAUa4IILLvBaF4cDfiNzfJ63EQwtCVFcUpxTK//8/HzGjB5NtX8BmWag0blBd/g1Eokwbdq0dq3qzjn27t3Lxo0bWbt2LatWrWLNGu9SvfnhQppiR9hXt5Phw4ZzzvlnctJJJzF79mymTp2aU1eszGWJIf9KC4dT2+yNTjKxZAzjS8aw6eA7nYYEHIzKy8u58cYbueqqq3jggQd4/PHHicfjjBo1iuXLl1NTU8OcOXP4/Oc/n3EXHJL+NXv2bK688kr+8Mc/Ev79vTB8JE1TptNy3ExiZRWEDnlH5OIjRhIrG014zy4iWzZSsG0Ttr8asxAXX3IJ7373u4P9Ij4F6B60tESJl4yGRu/iCI0nzAegcN2TOTEAfrJEa2Jyu3vidiZ16k+3OXPmcOJJJ7J23VriQ+NtTUn7wPYY//i5fxwUZ0enora2lkRbcz5eV466uswef7Q/mFnriUIXXHAB4I17e80113DgwAGaGxq5/fbbOfvsszP6QgaSPomTxM4ZP6f15OObz7wagOd3LM2phpiysjJuvPFGrrjiCm666SaqqqoAuO2227joooty4jyaXBcKhbjuuuv4xCc+wZIlS3hhyRLeeOMNWlYtheISGuacQfMpZ5C3YTVFy1+lYNVSQuEw73rXu7jgyo8zb968jGqw01q9R66bQwyWc104EmfVx5IuqFLT7MWmXGpNMzM+c/VnuPnmm72lxx+q1jYbQ4uH8tGPfjTQ+oLQ0NjYeiHmEEZ+yDhypKuxgAe/UaNGMWHCBA4cOADA+eefH3BFkomccxi51xUQvJMDb7nlFr761a9y7rnn8p73vCfokmSADR8+nPnz5zN//nwaGhp47bXXePyJJ1j20jMUvfQs4Jh5/Cw++vlrOf/88zN2RCcF6B6MGjWKA9XtL82Lc0Saaygry41LVyfMnTuXUCjEwaYQo4tiOAeLdxUyZnQFkyZNCrq8AXX66aczqmwU+2r2eQG6BcKVYd77gffm5MUfigoLaW5sBCCOoznucnoYs8HQ91v6x8iR3ogte+sPtLt/f+MhYi6e1cMVHovTTz+dK6+8kvnz5wddigSsqKiIefPmMW/ePF544QVeeeUVZs+ezQc/+MGMPyqhNX0P3nPRRYRrKrFoY+t9eZVv4RrrMqYPzkAZM2YMl156KQebQrTEjaV789hyOMz//dSnc6oLB3gB6dL3X4o1G8TAthsu6nJm6LqOiouLSZwK1YJ3Ke/i4uIAKxLJDMXFxZwxdy7P7nid5pjXXSPu4jyyfiGRcITzzjsv4AqDEYlEuO666wbduRJybC688EK+9rWv8aEPfSjjwzMoQPfo4x//OLNmzSbUcBgXKSRSvZH8Ha/z7ne/OycPO1199dUYsL8xxJ+3DWHihPE5Gxo//OEPezcaILwlzPHHH5+zJ8AMSQrQiV3NXG6BFkl24003kVeYT+WRasqGlPLQmgWs2ruBL3zxC4wdOzbo8kTkKClA96CgoIAf/eiHTJ48mcjBbRRsfYHTTj+db33rW1mxd9TfKioqKB01ikPNId6pDfGPn7wq51qfEyoqKhg+fDjWYLgax0c+8pGcnCcAjj/+eJrwrkC4xr9v5szc6uIk0p3x48dzx3/eQTg/zMu7VvDiruVcddVVXHHFFUGXJiLHQAG6F8OGDeN73/tu69//etttOTfKQrLkPnu52AqfbPjw4a23c/VQLHhHagCqgGfMmHv66cyaNSvYokQyyMyZM/nCF78IQF5eHv/0T/8UcEUicqwUoPtgypQpnHzyKXzgAx9oF5pyUfKh+Vw8YS5ZQUFB6+1hw4YFWEmwJkyYwOgxY2gGmpzjlq9+NWdb4wGmT58edAmSgebNm8fIESO56aabcnr5EBksNApHH91110+DLiEjzJgxg1WrVgVdRkbQ2L5tysrK2LNnD0BGXCEqSNdff33rpbxFEoYNG8ZfHv9L0GWISD9RC7Sk5Prrrw+6hIyhVqQ22pnoLFfPDxARyQXa6okcJY33K905+eSTufzyy4MuQ0RE0kQBWlJ2ww03aPxOFKCle3fddVfQJYiISBopQEvKcvFy1V1RgG4zffp0Vq5cGXQZIiIiA0IJQESOmfrGi4hILlGAFhERERFJgQK0yFHSeL/tTZs2jS/6F4sQEREZzMw5F3QNvZo7d65btmxZ0GWIdHLhhRcC8MILLwRcSfCccxraT0REsp6ZveGcm9vTc9QCLSL9QuFZRERyhQK0yDE644wzgi5BREREBpCGsRM5Bg888AAjR44MugwREREZQArQIsdg8uTJQZcgIiIiA0xdOEREREREUqAALSIiIiKSAgVoEREREZEUKECLiIiIiKRAAVpEREREJAUK0CIiIiIiKVCAFhERERFJgQK0iIiIiEgKFKBFRERERFKgAC0iIiIikgIFaBERERGRFChAi4iIiIikwJxzQdfQKzOrBrYHXEYZsC/gGjKFpkUbTYs2mhZtNC3aaFq00bRoo2nRRtOiTaZMi8nOufKenpAVAToTmNky59zcoOvIBJoWbTQt2mhatNG0aKNp0UbToo2mRRtNizbZNC3UhUNEREREJAUK0CIiIiIiKVCA7ru7gy4gg2hatNG0aKNp0UbToo2mRRtNizaaFm00LdpkzbRQH2gRERERkRSoBVpEREREJAU5HaDNbISZfcG/Pc7M/hh0TSIiQTGzr5jZkKDrSIWZzTGzDwRdh4jklpwO0MAI4AsAzrndzrkrAq7nmJnZFDNbnYb37beNlF/jJ/rjvTKNmX3YzE5I+vu7ZvZ3QdYUFE0LT8dlx8w+ZGb/EmRNPfgKkFUBGpgD9HndNJANJ2ZWbmavmdkKM7vgKF7/aTMb14fnDfr1vpn90MzWmNkPj+K1gexk9WWH1Mw+ZmbrzOzZo3j/1nk5KGY218zu7Of3zIppkusB+gfANDNbaWZ/SKyA/JXWn83saTPbZmZfMrOb/JXgq2ZW6j9vmpktNLM3zGyJmc0K9NukiZlFSH0jFenh4SnAoAzQwIeB1tDonPu2c25RgPUESdPC027Zcc497pz7QYD1AGBmQ83sSTNbZWarzew7wDjg2cSGy8wuMbNXzGy5v44s9u/fZmb/YWZvmdnrZja9h88ZbWaP+Z+zyszO9e+/yf/c1Wb2Ff++dkHQzG4xs9v828+Z2b/7n7fRzC4ws3zgu8DH/fX4x/vw1Qey4eS9wFvOuVOdc0uO4vWfxvuf9Fkv695U3yeT1vvXAic75756FK9N6XsAmOdYM1JfdkivAT7rnLvoKN6/dV5OhZmFj+KzuuScW+acu76/3s+XHdPEOZezP3gL9Ooubn8a2AyUAOXAYeDz/mM/Br7i314MzPBvnwU8kyHfaR1wD7AGeAooAqYBC4E3gCXALP/5HwReA1YAi4DR/v23AQ8ALwG/A94BqoGVwMe7+eyOr5nif9Zy/+dc/3mv+tN0JXAjEAZ+CCwF3gQ+l4bpcob/3oXAUH/anNTF8+YBzwN/Abbi7WT9I/A68BYwLWk6P+O/52JgEnAucAB42/9u04D7gCv817zXn85vAb8CCvz7twH/6k+jtxL/mzTOI99NzMP+3/8G3JCj0+Lzfn0r/Vqf7eZ5df48ugZvOTkTeM6fLh/yn1MI/NqvewVwEZBPh2UHb/1yV3fTzr//PuBO4GX/M65Iw3f/KHBP0t/D/elf5v9dBrwADPX//hrw7aT/0zf82/8XWNDD5zxM2zoz7H/O6f50GgoU+9P1VJLWw/7zbwFu828/B9zh3/4AsMi/3To9+/i9fw80+P+PP9B+vf9n4Gn/+30JuMn/X74KlPrP63Jd2sXnzOnwvy8CLgFewZu//wAU+8/9Nt76bzXeKAQGXIE3321Ien3y/2cu8Jx/+yd469Q9/u9ngcf8+eowsJ4sXu8DjwMx2pahcuBR/7VLgfP8553pT98VeMvO8XS9DN4G3JI0/zYCa4E/AVuASv++l/DWj0vx1mePJK8TktaTzwF/9KfzQ/7/73qgGW8+72698u2k//EPu5smeMvIYtrWi5d3MS//0K9lQdL73wV8OmmZ/Xf/Pf4P7efF1/HmvVX+/3IKXa+XPpb0vBeSvv+CpHnhV7StG69PquWT/uesBH4BhDN8mrQun93Ol/29Us6mH3oO0MkblneA8f7tz+CtrIqT/kmJn3UZ8p2iwBz/70f8GbfLsA+MpG00ln+ibQN1G94GoihpmvS4keriNUOAQv/2DGCZf7vjDH0t8E3/dgGwDJiahmnzfeBHwM+Ar3fznHnAIWCsX8su4F/9x24AfuLffgL4VNI88Wf/9n0khZ3E33jhagcw07//N7SFim3Al/3bXwB+OQDzyHL/dghvgzEqF6dFUm15eBv9D3bzuAMu9W8/hrdjmgecAqz0778Z+JV/exbeeqOQDssO7QN0T9PuD/7/5wRgcxq+80zaNiAXJE3/RECbj3dJ3cT6bS1wb9Lzjkuadvt7+Jxq/B2kpPtuAL6b9Pf38ALHFHoO0ImgNDoxTTpO3z7O/wPScNLhf93TDklp0mseSMyH/neem/RY8v+nY4B2wFn+39vxdpQXAxfiNapk9XofP7D6t38LnO/fnoS/7QWGARH/9t8Bj3b1PfyabwFOBDb602cK3vLs8OdN4A5/mhveMrkMuDC5Hv97HQYm4C2vryTV1vr/6uF7tf6Pu5smQAQYljQfbfZrmkL75aXjNO4YFv+547zoT4Nq4AeJeZHu10tv0ZaFRnT8TH+6vuzXXgbsx1s/zPbfM89/3n8D/zdTp0nH5bO7n3451DNINSXdjif9Hcf7x4WAQ865OQNdWB+87Zxb6d9+A2+GOhf4g5klnlPg/54APGxmY/H21N9Oep/HnXMNKX528mvygLvMbA5e68HMbl5zCXCymSUOpQ7HW/G+3c3zj9Z38fZiG/E21t1Z6pyrBDCzLXhhCbyVR+KQ0jnAR/zbDwD/0ctnH4/3f9no/30/8EW8DR94LR/g/b8+Qho557aZ2X4zOxUviKxwzu3v5umDelok+S+8cPFEN48347U6gvfdm5xzLWb2Ft7yBXA+8FMA59x6M9tO9/N8Qk/T7s/OuTiw1sxGp/Jl+sI5t9HMTsNrzf2+mS3u8BQDnnbOXdndW3Rz+1hEad+1sLDD44n1cAzSsv161jlXC9Sa2WG8jT54//OT/S4s3a1Le3M23s7QS/5r8/HCFsBFZvbPeOGzFK9Fvrt5sTsHnHOv+bdHAZ/FW77PxWvFuwdvnQzZv97/O+CEpP/BMP9/Mxy438xm4M2Ted28PuE9eDuql/t/H8bbxn/P/9uAiXgt2pPw5r8ZeEEr2evOuZ0AZrYSb53wYh++R0fdTZOdwO1mdqFf33i8/22qHvZ/t86LeMERvJ1GnHMHzKy79dJLwH1m9ght6+mOnnTONQFNZrbXr/O9eEedlvr/syJgbx9rHvBp0sXy2aVcD9C1eK0NKXPO1ZjZ22b2MefcH8yb4ic751b1b4lHJTn8x/Bmqu7C/k+B/3TOPW5m8/D2IBOOHMVnJ7/mRqAKb68+hBdcu2J4rY7/exSfl4pReEcO8vA2zN19v952ntIh3cGgo1/itcqMwTvk1p1BPy3M7NPAZLxD9t1pcX6zBEnTwTkX768+p11InvbW7bOOknknpx1wzj1oZofwWiIT68R9eIfcf2Zm051zyefEWgAAB0lJREFUm81sKF7rU2LH5+N43Xo+Ts8bmsXAdcBP/H6GxXit/feZ2Q/87/b3wFV464sKMxuFdxh3Pm07Lt056vV4F9LZcNLlDomZFeK1yM11zu3w+3x33HFISN7B6Pic5g6f9QjwD865sV28T7av90PA2c65du9tZnfh7QT9vZlNwWvJ7EpPO2rJO4MGNDjn5pjZL4GnnHOP+H2j85Oe13Gbe7TrhC6nib+OKgdO93fct9H1PNLbDmji/9Q6L5rZl4Exzrlv9Facc+7zZnYWcBnwhpmd3sXTupoWBtzvnPt6b5/RhQGfJn0tLKdPIvRb3V7yT1pJ+cxevP6g15jZKrwWg8t7eX5QaoC3zexj0HpyxCn+Y8PxDs0DfKqH9ziajdRwoNJvRbsKrxWkq/f6X+A6M8vz65vpb6z72y+Ab+H1Ufv3Y3yvl/H6TIE3HyROEOpuOm0ApiSdbHUVXv/ioDwGvB+vb/ix7rhk7bTwNwC3AJ/059NjsQTv+2NmM/FarDbQ87LT3bQbCO8CXvdbzL6D18XpbmChmT3rnKvG28n6nZm9iReSk0+UHunffwNeaOrODXgtrG/hHVU4wTm3HO+Q+Ot4fXF/6Zxb4ZxrwTtS9DpeX+T1ffgez+K1Rvb1JMJjajih+3Vpb14FzkvM9+adxDmTtg36Pr8VNfmkxo61bsNryQOvD3t3NuAdNn/bvBEN5gyy9f5TwJcTf/it3YnPTnyvTyc9v6vpeBpeP99P4nUJSLy+nrZlMgo0+/+XbcC7zawC+BC9t2539bm96W6aDAf2+kHxIrwd/q7efzveslBgZiPwWn67kjwvPgP8g5md6X9mKd2sl8xsmnPuNefct/G6fUzs4/daDFzhTzvs/7d3PyFS1nEcxz+fEhLqFAR1cjv3x4XCLtEfqWtgUAaRuBBdKutiBnkoKgIFqQ4mWLiXciPKQ2YRtWZkUJixruZByryUZn8Mgly39dvh+xt9nJ3JfXZzdsZ5v06zj/Ob+c3jPL/5Pr9/X/tK24vOU6ZhPs5J9fpsq997oBUR01YFR8SwsnFv/D3Q6t8i4rAyCOkFD0p6zfZa5YU/olwI8KxyOPIP5YV0bZvyOyU9XX5sX4qIt9s8r2qjpHdtr1D2IjXu9PZJmio3HsPK4fMBSXtLT/5x5Q4O/5tSh8mIeKv0gn1pe2lEjM7yJR+XtMX2amV9h8rxEUmbba9S5YcwIk7aHlKe6wXKqSSbZvt55ioiTjl3WjgREVNzfLlePhePKYfMd5Zhuz0R8fAsX2uj8hobV/7wroyIiXKez1w7TWXanbsLrvToNN887VGZhlKeM6q8yWplfUSsmcH7HFOLzoWI2CBpQ4vjryoXUDYfv6Py+FeVqTMR8ft/1LFVfX6z3eg4OTjTchXt2tLzve/x0mu21XZj2sfaMpVms3Jx1lHl9dAwLGmT7b+V032ek/SG7efVvndVysWQ9yuD1y3KXuBfdPG0+6uUoyP7lHHM58oFweuUUzjWSvqg3edQLkBcoeylP6Icpd2hnOf/s6ShyjX5ivLmcYFy6stS5aLGmfTUN25If4qZ7SjxulqfkzclvV/alj0qN5ZN3+UPI2K1c3rFfuVUmG9bvUn1u6icgrRQ+X04Ucq0a5fWO6fHWBkUj0m6/XwfKiK+K/8nHzt77yeV0/aOdOM5qV6fyjnyLZHKG+hDpRHbK+m+iDg03/VBb3EOl95cAlkA6Dt93wMN9BtncpPtkrYRPGM2qqNyDbafUW5zVfVORLzYkUoBQAfRA43ayvD7E02Hd0fEo/NRn9mwfYNydXHVRETcMh/1mU+ci7Nsf6Xpuyo8FBHj81Ef9IZ+uHnoRLt/MbdFtC3T9fo5IYAGAAAAaujrXTgAAACAugigAQAAgBoIoAGgy9i+2vaI7e9tf2N7R7s9SW0PlC2bAAAdwi4cANBFyl6n25SZux4oxxYr96ptuycpAKBz6IEGgO5ypzLpz5nkMhExJukL2+tt77c97hZZ92yvdKYzbvy93ZmqWbb/KuUP2P7E9hLbn9n+wfY9lfLv2f7I9iHb6y74pwWAHkQADQDd5Xplyutm90oalLRY0l3KrGDX1HjdyyWNRsR1ynS3L0i6W9IyZfrshkFJy5Wpvpfbnmm6XgDoGwTQANAbbpW0NSKmSnrsXaqRwlrSKWVqZUkal7QrIibL44HK8z6NiD8j4qQytfGiOdccAC4yBNAA0F0OSLpplmX/0bnt+sLK48k4u/H/aUkTkhQRp3XuepiJyuMpsVYGAKYhgAaA7jIq6TLbjzQO2L5R0gnllIpLbV8l6TZJXzeV/VHSoO1LytSLJR2qMwD0FXoWAKCLRETYXibpZdtrJJ1UBsZPSrpC0pikkPRURBy1PVApvlvSYeXUi4OS9nau5gDQP0jlDQAAANTAFA4AAACgBgJoAAAAoAYCaAAAAKAGAmgAAACgBgJoAAAAoAYCaAAAAKAGAmgAAACgBgJoAAAAoIZ/Ae4HKCZxLF20AAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 864x432 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xOFXkfCo1qkd"
      },
      "source": [
        "---\n",
        "---\n",
        "\n",
        "# 7. Output the standardised data\n",
        "\n",
        "---\n",
        "---"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4FdIv2ZJ0Q6i"
      },
      "source": [
        "Writing the train, validation and test set data to csv files"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rm3SofabGDXp"
      },
      "source": [
        "std_train_df.to_csv('std_1_train.csv', index=False)\n",
        "std_val_df.to_csv('std_1_val.csv', index=False)\n",
        "std_test_df.to_csv('std_1_test.csv', index=False)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "S_M1YExX0Y2E"
      },
      "source": [
        "Write the full data to a csv file"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yP2Cayia0ZVf"
      },
      "source": [
        "std_full_df.to_csv('std_1_full.csv', index=False)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HBsvTuew7NVS"
      },
      "source": [
        "---\n",
        "---\n",
        "\n",
        "# 8. Standardise all the data except the time_feature\n",
        "\n",
        "---\n",
        "---"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "__a_oOhN7NVS"
      },
      "source": [
        "### Standardise the train, validation, test and full dataframes"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6P3bhjkw7NVW",
        "outputId": "f4fcc4c7-8e50-4dfa-e955-9428251c0a53"
      },
      "source": [
        "# Only standardise feature columns\n",
        "input_cols_2 = train_df.drop(columns=['time', 'time_feature', 'sleep_label', 'psg_label', 'subject_id']).columns\n",
        "\n",
        "std_2_train_df, train_mean_2, train_std_2 = standardise(train_df, input_cols_2)\n",
        "std_2_val_df, _, _ = standardise(val_df, input_cols_2, train_mean_2, train_std_2)\n",
        "std_2_test_df, _, _ = standardise(test_df, input_cols_2, train_mean_2, train_std_2)\n",
        "std_2_full_df, _, _ = standardise(all_subjects_data_df, input_cols_2, train_mean_2, train_std_2)\n",
        "\n",
        "print(\"Mean:\")\n",
        "print(train_mean_2)\n",
        "print(\"Standard Deviation:\")\n",
        "print(train_std_2)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mean:\n",
            "heart_rate            64.138695\n",
            "x_motion              -0.113355\n",
            "y_motion              -0.122823\n",
            "z_motion              -0.389581\n",
            "step_count             1.813057\n",
            "heart_rate_feature     0.364680\n",
            "count_feature          1.718910\n",
            "cosine_feature        -0.778094\n",
            "dtype: float64\n",
            "Standard Deviation:\n",
            "heart_rate            11.037470\n",
            "x_motion               0.342122\n",
            "y_motion               0.493472\n",
            "z_motion               0.674002\n",
            "step_count            11.737288\n",
            "heart_rate_feature     0.332509\n",
            "count_feature          6.674443\n",
            "cosine_feature         0.211732\n",
            "dtype: float64\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3osiJlZJ7NVX"
      },
      "source": [
        "### Look at these standardised dataframes"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 197
        },
        "id": "A0qY0FVw7NVX",
        "outputId": "b7546895-507b-4745-d651-4d0775936388"
      },
      "source": [
        "std_2_train_df.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>time</th>\n",
              "      <th>heart_rate</th>\n",
              "      <th>x_motion</th>\n",
              "      <th>y_motion</th>\n",
              "      <th>z_motion</th>\n",
              "      <th>step_count</th>\n",
              "      <th>sleep_label</th>\n",
              "      <th>time_feature</th>\n",
              "      <th>heart_rate_feature</th>\n",
              "      <th>count_feature</th>\n",
              "      <th>cosine_feature</th>\n",
              "      <th>psg_label</th>\n",
              "      <th>subject_id</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1970-01-01 00:00:00</td>\n",
              "      <td>2.343046</td>\n",
              "      <td>-1.304122</td>\n",
              "      <td>1.648084</td>\n",
              "      <td>0.251126</td>\n",
              "      <td>8.607776</td>\n",
              "      <td>0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.480706</td>\n",
              "      <td>5.530874</td>\n",
              "      <td>2.452504</td>\n",
              "      <td>0</td>\n",
              "      <td>46343</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1970-01-01 00:00:30</td>\n",
              "      <td>3.339652</td>\n",
              "      <td>-1.513119</td>\n",
              "      <td>1.612262</td>\n",
              "      <td>0.111929</td>\n",
              "      <td>4.815162</td>\n",
              "      <td>0</td>\n",
              "      <td>0.008333</td>\n",
              "      <td>0.451398</td>\n",
              "      <td>4.483025</td>\n",
              "      <td>2.442553</td>\n",
              "      <td>0</td>\n",
              "      <td>46343</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1970-01-01 00:01:00</td>\n",
              "      <td>2.524247</td>\n",
              "      <td>-0.006384</td>\n",
              "      <td>1.979267</td>\n",
              "      <td>1.727603</td>\n",
              "      <td>0.891768</td>\n",
              "      <td>0</td>\n",
              "      <td>0.016667</td>\n",
              "      <td>0.398636</td>\n",
              "      <td>3.124169</td>\n",
              "      <td>2.432611</td>\n",
              "      <td>0</td>\n",
              "      <td>46343</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1970-01-01 00:01:30</td>\n",
              "      <td>1.618243</td>\n",
              "      <td>-0.679271</td>\n",
              "      <td>-0.327864</td>\n",
              "      <td>1.896059</td>\n",
              "      <td>-0.154470</td>\n",
              "      <td>0</td>\n",
              "      <td>0.025000</td>\n",
              "      <td>0.374122</td>\n",
              "      <td>2.640312</td>\n",
              "      <td>2.422674</td>\n",
              "      <td>0</td>\n",
              "      <td>46343</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>1970-01-01 00:02:00</td>\n",
              "      <td>1.708843</td>\n",
              "      <td>-0.591184</td>\n",
              "      <td>-0.311630</td>\n",
              "      <td>1.923702</td>\n",
              "      <td>-0.154470</td>\n",
              "      <td>0</td>\n",
              "      <td>0.033333</td>\n",
              "      <td>0.354029</td>\n",
              "      <td>3.032657</td>\n",
              "      <td>2.412742</td>\n",
              "      <td>0</td>\n",
              "      <td>46343</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                 time  heart_rate  x_motion  y_motion  z_motion  step_count  \\\n",
              "0 1970-01-01 00:00:00    2.343046 -1.304122  1.648084  0.251126    8.607776   \n",
              "1 1970-01-01 00:00:30    3.339652 -1.513119  1.612262  0.111929    4.815162   \n",
              "2 1970-01-01 00:01:00    2.524247 -0.006384  1.979267  1.727603    0.891768   \n",
              "3 1970-01-01 00:01:30    1.618243 -0.679271 -0.327864  1.896059   -0.154470   \n",
              "4 1970-01-01 00:02:00    1.708843 -0.591184 -0.311630  1.923702   -0.154470   \n",
              "\n",
              "   sleep_label  time_feature  heart_rate_feature  count_feature  \\\n",
              "0            0      0.000000            0.480706       5.530874   \n",
              "1            0      0.008333            0.451398       4.483025   \n",
              "2            0      0.016667            0.398636       3.124169   \n",
              "3            0      0.025000            0.374122       2.640312   \n",
              "4            0      0.033333            0.354029       3.032657   \n",
              "\n",
              "   cosine_feature  psg_label  subject_id  \n",
              "0        2.452504          0       46343  \n",
              "1        2.442553          0       46343  \n",
              "2        2.432611          0       46343  \n",
              "3        2.422674          0       46343  \n",
              "4        2.412742          0       46343  "
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 32
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 197
        },
        "id": "7WYMKDw67NVX",
        "outputId": "bb96ba54-0fe8-4175-ffdc-4aa96b58eaa4"
      },
      "source": [
        "std_2_val_df.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>time</th>\n",
              "      <th>heart_rate</th>\n",
              "      <th>x_motion</th>\n",
              "      <th>y_motion</th>\n",
              "      <th>z_motion</th>\n",
              "      <th>step_count</th>\n",
              "      <th>sleep_label</th>\n",
              "      <th>time_feature</th>\n",
              "      <th>heart_rate_feature</th>\n",
              "      <th>count_feature</th>\n",
              "      <th>cosine_feature</th>\n",
              "      <th>psg_label</th>\n",
              "      <th>subject_id</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>18085</th>\n",
              "      <td>1970-01-01 04:54:00</td>\n",
              "      <td>0.213935</td>\n",
              "      <td>-1.495124</td>\n",
              "      <td>0.498384</td>\n",
              "      <td>-0.562286</td>\n",
              "      <td>-0.15447</td>\n",
              "      <td>5</td>\n",
              "      <td>4.900000</td>\n",
              "      <td>0.192422</td>\n",
              "      <td>-0.257536</td>\n",
              "      <td>-1.046431</td>\n",
              "      <td>5</td>\n",
              "      <td>8000685</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>18086</th>\n",
              "      <td>1970-01-01 04:54:30</td>\n",
              "      <td>-0.012566</td>\n",
              "      <td>-1.500876</td>\n",
              "      <td>0.489463</td>\n",
              "      <td>-0.558278</td>\n",
              "      <td>-0.15447</td>\n",
              "      <td>5</td>\n",
              "      <td>4.908333</td>\n",
              "      <td>0.557916</td>\n",
              "      <td>-0.257536</td>\n",
              "      <td>-1.046691</td>\n",
              "      <td>5</td>\n",
              "      <td>8000685</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>18087</th>\n",
              "      <td>1970-01-01 04:55:00</td>\n",
              "      <td>0.757538</td>\n",
              "      <td>-1.497220</td>\n",
              "      <td>0.510753</td>\n",
              "      <td>-0.569202</td>\n",
              "      <td>-0.15447</td>\n",
              "      <td>5</td>\n",
              "      <td>4.916667</td>\n",
              "      <td>0.686721</td>\n",
              "      <td>-0.257536</td>\n",
              "      <td>-1.046927</td>\n",
              "      <td>5</td>\n",
              "      <td>8000685</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>18088</th>\n",
              "      <td>1970-01-01 04:55:30</td>\n",
              "      <td>0.576337</td>\n",
              "      <td>-1.516553</td>\n",
              "      <td>0.462067</td>\n",
              "      <td>-0.561459</td>\n",
              "      <td>-0.15447</td>\n",
              "      <td>5</td>\n",
              "      <td>4.925000</td>\n",
              "      <td>0.703659</td>\n",
              "      <td>-0.257536</td>\n",
              "      <td>-1.047140</td>\n",
              "      <td>5</td>\n",
              "      <td>8000685</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>18089</th>\n",
              "      <td>1970-01-01 04:56:00</td>\n",
              "      <td>-0.057866</td>\n",
              "      <td>-1.508682</td>\n",
              "      <td>0.465067</td>\n",
              "      <td>-0.560373</td>\n",
              "      <td>-0.15447</td>\n",
              "      <td>5</td>\n",
              "      <td>4.933333</td>\n",
              "      <td>0.704723</td>\n",
              "      <td>-0.257536</td>\n",
              "      <td>-1.047334</td>\n",
              "      <td>5</td>\n",
              "      <td>8000685</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                     time  heart_rate  x_motion  y_motion  z_motion  \\\n",
              "18085 1970-01-01 04:54:00    0.213935 -1.495124  0.498384 -0.562286   \n",
              "18086 1970-01-01 04:54:30   -0.012566 -1.500876  0.489463 -0.558278   \n",
              "18087 1970-01-01 04:55:00    0.757538 -1.497220  0.510753 -0.569202   \n",
              "18088 1970-01-01 04:55:30    0.576337 -1.516553  0.462067 -0.561459   \n",
              "18089 1970-01-01 04:56:00   -0.057866 -1.508682  0.465067 -0.560373   \n",
              "\n",
              "       step_count  sleep_label  time_feature  heart_rate_feature  \\\n",
              "18085    -0.15447            5      4.900000            0.192422   \n",
              "18086    -0.15447            5      4.908333            0.557916   \n",
              "18087    -0.15447            5      4.916667            0.686721   \n",
              "18088    -0.15447            5      4.925000            0.703659   \n",
              "18089    -0.15447            5      4.933333            0.704723   \n",
              "\n",
              "       count_feature  cosine_feature  psg_label  subject_id  \n",
              "18085      -0.257536       -1.046431          5     8000685  \n",
              "18086      -0.257536       -1.046691          5     8000685  \n",
              "18087      -0.257536       -1.046927          5     8000685  \n",
              "18088      -0.257536       -1.047140          5     8000685  \n",
              "18089      -0.257536       -1.047334          5     8000685  "
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 33
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 197
        },
        "id": "SCa_PW8z7NVY",
        "outputId": "b5cd4237-6d00-4848-f1e8-624e5d8815fa"
      },
      "source": [
        "std_2_test_df.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>time</th>\n",
              "      <th>heart_rate</th>\n",
              "      <th>x_motion</th>\n",
              "      <th>y_motion</th>\n",
              "      <th>z_motion</th>\n",
              "      <th>step_count</th>\n",
              "      <th>sleep_label</th>\n",
              "      <th>time_feature</th>\n",
              "      <th>heart_rate_feature</th>\n",
              "      <th>count_feature</th>\n",
              "      <th>cosine_feature</th>\n",
              "      <th>psg_label</th>\n",
              "      <th>subject_id</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>23253</th>\n",
              "      <td>1970-01-01 00:17:30</td>\n",
              "      <td>-0.284367</td>\n",
              "      <td>-1.286817</td>\n",
              "      <td>0.891131</td>\n",
              "      <td>-0.572212</td>\n",
              "      <td>-0.15447</td>\n",
              "      <td>2</td>\n",
              "      <td>0.291667</td>\n",
              "      <td>-0.637137</td>\n",
              "      <td>-0.257536</td>\n",
              "      <td>2.108055</td>\n",
              "      <td>2</td>\n",
              "      <td>9106476</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>23254</th>\n",
              "      <td>1970-01-01 00:18:00</td>\n",
              "      <td>0.032734</td>\n",
              "      <td>-1.286729</td>\n",
              "      <td>0.883695</td>\n",
              "      <td>-0.574001</td>\n",
              "      <td>-0.15447</td>\n",
              "      <td>2</td>\n",
              "      <td>0.300000</td>\n",
              "      <td>-0.649877</td>\n",
              "      <td>-0.257536</td>\n",
              "      <td>2.098340</td>\n",
              "      <td>2</td>\n",
              "      <td>9106476</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>23255</th>\n",
              "      <td>1970-01-01 00:18:30</td>\n",
              "      <td>-0.103166</td>\n",
              "      <td>-1.291255</td>\n",
              "      <td>0.895584</td>\n",
              "      <td>-0.569009</td>\n",
              "      <td>-0.15447</td>\n",
              "      <td>3</td>\n",
              "      <td>0.308333</td>\n",
              "      <td>-0.685912</td>\n",
              "      <td>-0.257536</td>\n",
              "      <td>2.088630</td>\n",
              "      <td>3</td>\n",
              "      <td>9106476</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>23256</th>\n",
              "      <td>1970-01-01 00:19:00</td>\n",
              "      <td>-0.012566</td>\n",
              "      <td>-1.279770</td>\n",
              "      <td>0.894115</td>\n",
              "      <td>-0.570333</td>\n",
              "      <td>-0.15447</td>\n",
              "      <td>3</td>\n",
              "      <td>0.316667</td>\n",
              "      <td>-0.700504</td>\n",
              "      <td>-0.257536</td>\n",
              "      <td>2.078929</td>\n",
              "      <td>3</td>\n",
              "      <td>9106476</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>23257</th>\n",
              "      <td>1970-01-01 00:19:30</td>\n",
              "      <td>0.078035</td>\n",
              "      <td>-1.286973</td>\n",
              "      <td>0.885210</td>\n",
              "      <td>-0.568240</td>\n",
              "      <td>-0.15447</td>\n",
              "      <td>2</td>\n",
              "      <td>0.325000</td>\n",
              "      <td>-0.709255</td>\n",
              "      <td>-0.257536</td>\n",
              "      <td>2.069237</td>\n",
              "      <td>2</td>\n",
              "      <td>9106476</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                     time  heart_rate  x_motion  y_motion  z_motion  \\\n",
              "23253 1970-01-01 00:17:30   -0.284367 -1.286817  0.891131 -0.572212   \n",
              "23254 1970-01-01 00:18:00    0.032734 -1.286729  0.883695 -0.574001   \n",
              "23255 1970-01-01 00:18:30   -0.103166 -1.291255  0.895584 -0.569009   \n",
              "23256 1970-01-01 00:19:00   -0.012566 -1.279770  0.894115 -0.570333   \n",
              "23257 1970-01-01 00:19:30    0.078035 -1.286973  0.885210 -0.568240   \n",
              "\n",
              "       step_count  sleep_label  time_feature  heart_rate_feature  \\\n",
              "23253    -0.15447            2      0.291667           -0.637137   \n",
              "23254    -0.15447            2      0.300000           -0.649877   \n",
              "23255    -0.15447            3      0.308333           -0.685912   \n",
              "23256    -0.15447            3      0.316667           -0.700504   \n",
              "23257    -0.15447            2      0.325000           -0.709255   \n",
              "\n",
              "       count_feature  cosine_feature  psg_label  subject_id  \n",
              "23253      -0.257536        2.108055          2     9106476  \n",
              "23254      -0.257536        2.098340          2     9106476  \n",
              "23255      -0.257536        2.088630          3     9106476  \n",
              "23256      -0.257536        2.078929          3     9106476  \n",
              "23257      -0.257536        2.069237          2     9106476  "
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 34
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 197
        },
        "id": "unpNHID67NVY",
        "outputId": "e854a9d6-8a3f-4966-8d06-d42d8de67465"
      },
      "source": [
        "std_2_full_df.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>time</th>\n",
              "      <th>heart_rate</th>\n",
              "      <th>x_motion</th>\n",
              "      <th>y_motion</th>\n",
              "      <th>z_motion</th>\n",
              "      <th>step_count</th>\n",
              "      <th>sleep_label</th>\n",
              "      <th>time_feature</th>\n",
              "      <th>heart_rate_feature</th>\n",
              "      <th>count_feature</th>\n",
              "      <th>cosine_feature</th>\n",
              "      <th>psg_label</th>\n",
              "      <th>subject_id</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1970-01-01 00:00:00</td>\n",
              "      <td>2.343046</td>\n",
              "      <td>-1.304122</td>\n",
              "      <td>1.648084</td>\n",
              "      <td>0.251126</td>\n",
              "      <td>8.607776</td>\n",
              "      <td>0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.480706</td>\n",
              "      <td>5.530874</td>\n",
              "      <td>2.452504</td>\n",
              "      <td>0</td>\n",
              "      <td>46343</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1970-01-01 00:00:30</td>\n",
              "      <td>3.339652</td>\n",
              "      <td>-1.513119</td>\n",
              "      <td>1.612262</td>\n",
              "      <td>0.111929</td>\n",
              "      <td>4.815162</td>\n",
              "      <td>0</td>\n",
              "      <td>0.008333</td>\n",
              "      <td>0.451398</td>\n",
              "      <td>4.483025</td>\n",
              "      <td>2.442553</td>\n",
              "      <td>0</td>\n",
              "      <td>46343</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1970-01-01 00:01:00</td>\n",
              "      <td>2.524247</td>\n",
              "      <td>-0.006384</td>\n",
              "      <td>1.979267</td>\n",
              "      <td>1.727603</td>\n",
              "      <td>0.891768</td>\n",
              "      <td>0</td>\n",
              "      <td>0.016667</td>\n",
              "      <td>0.398636</td>\n",
              "      <td>3.124169</td>\n",
              "      <td>2.432611</td>\n",
              "      <td>0</td>\n",
              "      <td>46343</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1970-01-01 00:01:30</td>\n",
              "      <td>1.618243</td>\n",
              "      <td>-0.679271</td>\n",
              "      <td>-0.327864</td>\n",
              "      <td>1.896059</td>\n",
              "      <td>-0.154470</td>\n",
              "      <td>0</td>\n",
              "      <td>0.025000</td>\n",
              "      <td>0.374122</td>\n",
              "      <td>2.640312</td>\n",
              "      <td>2.422674</td>\n",
              "      <td>0</td>\n",
              "      <td>46343</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>1970-01-01 00:02:00</td>\n",
              "      <td>1.708843</td>\n",
              "      <td>-0.591184</td>\n",
              "      <td>-0.311630</td>\n",
              "      <td>1.923702</td>\n",
              "      <td>-0.154470</td>\n",
              "      <td>0</td>\n",
              "      <td>0.033333</td>\n",
              "      <td>0.354029</td>\n",
              "      <td>3.032657</td>\n",
              "      <td>2.412742</td>\n",
              "      <td>0</td>\n",
              "      <td>46343</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                 time  heart_rate  x_motion  y_motion  z_motion  step_count  \\\n",
              "0 1970-01-01 00:00:00    2.343046 -1.304122  1.648084  0.251126    8.607776   \n",
              "1 1970-01-01 00:00:30    3.339652 -1.513119  1.612262  0.111929    4.815162   \n",
              "2 1970-01-01 00:01:00    2.524247 -0.006384  1.979267  1.727603    0.891768   \n",
              "3 1970-01-01 00:01:30    1.618243 -0.679271 -0.327864  1.896059   -0.154470   \n",
              "4 1970-01-01 00:02:00    1.708843 -0.591184 -0.311630  1.923702   -0.154470   \n",
              "\n",
              "   sleep_label  time_feature  heart_rate_feature  count_feature  \\\n",
              "0            0      0.000000            0.480706       5.530874   \n",
              "1            0      0.008333            0.451398       4.483025   \n",
              "2            0      0.016667            0.398636       3.124169   \n",
              "3            0      0.025000            0.374122       2.640312   \n",
              "4            0      0.033333            0.354029       3.032657   \n",
              "\n",
              "   cosine_feature  psg_label  subject_id  \n",
              "0        2.452504          0       46343  \n",
              "1        2.442553          0       46343  \n",
              "2        2.432611          0       46343  \n",
              "3        2.422674          0       46343  \n",
              "4        2.412742          0       46343  "
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 35
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dDj-Kuqlusx6"
      },
      "source": [
        "---\n",
        "---\n",
        "\n",
        "# 9. Output version 2 of the standardised data\n",
        "\n",
        "---\n",
        "---"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "owqh7p3uusx6"
      },
      "source": [
        "Writing the train, validation and test set data to csv files"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Q92iLgthusx7"
      },
      "source": [
        "std_2_train_df.to_csv('std_2_train.csv', index=False)\n",
        "std_2_val_df.to_csv('std_2_val.csv', index=False)\n",
        "std_2_test_df.to_csv('std_2_test.csv', index=False)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yul5kL6cusx7"
      },
      "source": [
        "Write the full data to a csv file"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sraDcKAEusx7"
      },
      "source": [
        "std_2_full_df.to_csv('std_2_full.csv', index=False)"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}