customer_segmentation.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "customer_segmentation.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "20qc1UkavH9Q"
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "\"\"\" Customer Segmentation using Similarity Measures\n",
        "    CSE444 Ravan SADIGLI 20160807005 \n",
        "    \"\"\"\n",
        "class customer_segmentation(object):\n",
        "\n",
        "  def __init__(self,var1):\n",
        "    self.var1 = var1\n",
        "\n",
        "  \"\"\"\n",
        "  First, we need to clear the data. So we need to use the pandas library to read the dataset\n",
        "  (Note that I only use 3rd libraries for data preprocessing, pandas to read the dataset,\n",
        "  NumPy to convert to the array).  The below method provides us with ready-to-use data\n",
        "  for calculating both Mahalanobis and Minkowski.\n",
        "  \"\"\"  \n",
        "  def similarity_measure(self):\n",
        "    # Firstly, we read .xlsx file using pandas library\n",
        "    data_xls = pd.read_excel('CustomerGroups.xlsx', dtype=str, index_col=None)\n",
        "    # Then converting .csv and reading .csv to use pandas dataFrame\n",
        "    data_xls.to_csv('CustomerGroups.csv', encoding='utf-8', index=False)\n",
        "    df = pd.read_csv(\"CustomerGroups.csv\")\n",
        "    # In the CustomerGroups.csv file, we have missing data, so we need to clear these data. \n",
        "    # To do this, we fill this missing data with the value nan and then drop these rows.\n",
        "    df=df.replace(r'^\\s*$', np.nan, regex=True).dropna()\n",
        "\n",
        "    # Since, our dataset contains string values, we need to take care of that. \n",
        "    # We will convert these string values to integer or float numbers step by step.\n",
        "    # By using the map() function, we convert string values to 0 or 1, respectively, for Gender column\n",
        "    df[\"Gender\"] = df[\"Gender\"].map({'Male': 0, 'Female':1})\n",
        "    # Converting Graduated column 1 or 0, respectively\n",
        "    df[\"Graduated\"] = df[\"Graduated\"].map({'Yes': 1, 'No':0})\n",
        "    # In Spending_score column, we have three values low, average and high. We convert these string to integers, low for 0, average 1, high 2 \n",
        "    df[\"Spending_Score\"] = df[\"Spending_Score\"].map({'Low': 0, 'Average':1,'High':2})\n",
        "    # We do the same thing for ever_married column\n",
        "    df[\"Ever_Married\"] = df[\"Ever_Married\"].map({'Yes': 1, 'No':0})\n",
        "\n",
        "\n",
        "    \"\"\"\n",
        "    In the profession, the column describes the job of customers and has a great\n",
        "    impact on customer segmentation. We cannot do to transform what we did above.\n",
        "    Because it is not something that is desirable or recommended. Since mapping \n",
        "    is not recommended for such a column, we need to apply one-hot encoding for \n",
        "    this column. So we create a column for each job, for example, if the client's\n",
        "    job is engineer we put 1, otherwise we put 0. After applying one hot encoding \n",
        "    we have 16 columns (we didn't use the ID column for calculation because \n",
        "    it doesn't make sense for distance measurement).\n",
        "    \"\"\"\n",
        "    for col in [\"Profession\"]:\n",
        "      df[col] = df[col].astype('category')\n",
        "    df = pd.get_dummies(data=df,columns=['Profession'])\n",
        "    \n",
        "    # Using get_customer_by_ID, we get data about the customers by using ID column\n",
        "    customer1 = self.get_customer_by_ID(df,\"ID\",459264)\n",
        "    customer2 = self.get_customer_by_ID(df,\"ID\",464608)\n",
        "\n",
        "    # Converting dataFrame to integer type array \n",
        "    customer1np = (customer1.to_numpy()).astype(int)\n",
        "    customer2np = (customer2.to_numpy()).astype(int)\n",
        "\n",
        "    print(\"Minkowski distance: \",self.calculateMinkowski(customer1np[0],customer2np[0],df,4))\n",
        "    print(\"Mahalonobis distance: \",self.calculateMahalonobis(customer1np[0],customer2np[0],df))\n",
        "\n",
        "\n",
        "  def calculateMinkowski(self,customer1, customer2, df, R):\n",
        "    # Formula: (sum of (customer1 - customer2)^R)^R\n",
        "    # R - order of the norm\n",
        "    sum = 0\n",
        "    n = len(df.columns) # dimension\n",
        "\n",
        "    for i in range (1,n):\n",
        "      sum += abs(customer1[i] - customer2[i])**R\n",
        "\n",
        "    # by using the above formula, it gives us the desired result\n",
        "    return sum**(1/R)\n",
        "\n",
        "\n",
        "  \"\"\"\n",
        "  To calculate mahalonobis distance, we need covariance matrix. We can achieve \n",
        "  this using our dataset.\n",
        "  \"\"\"\n",
        "  def calculateMahalonobis(self,customer1, customer2, df):\n",
        "    #Formula: (customer1 - customer2) * S ^-1 * (customer1 -customer2)^T\n",
        "    # Using the formula, we need to calculate 3 things: customer points difference, and its transpose matrix,\n",
        "    # and covariance matrix. We will do this step by step.\n",
        "\n",
        "    if len(customer1) == len(customer2):\n",
        "      d = len(customer1) - 2 # dimension\n",
        "    else:\n",
        "      print(\"ERROR OCCURED\")\n",
        "      return\n",
        "\n",
        "    # initialization matrix for (customer1 - customer2).\n",
        "    # it will create matrix 1x15 dimensions\n",
        "    x_y = [[0 for i in range(d)] for j in range(1)]\n",
        "\n",
        "    # (customer1 - customer2)\n",
        "    for i in range (1,d):\n",
        "      x_y[0][i] = customer1[i] - customer2[i]\n",
        "\n",
        "    # transpose of customer1 - customer2 matrix with the 15x1 dimensions\n",
        "    x_yTranspose = [[0 for i in range(1)] for j in range(d)]\n",
        "\n",
        "    # (customer1 -customer2)^T\n",
        "    for i in range(len(x_y)):\n",
        "      for j in range(len(x_y[0])):\n",
        "          x_yTranspose[j][i] = x_y[i][j]\n",
        "\n",
        "\n",
        "    # Using the findDatasetCovariance, we can access covariance matrix with 15x15 matrix\n",
        "    S = self.findDatasetCovariance(df)\n",
        "\n",
        "    # Using the inverse method, we get inverse of the covariance matrix \n",
        "    S_inverse = self.inverse(S)\n",
        "\n",
        "    # converting (customer1 - customer2), inverse of covariance matrix, and tranpose(customer1 - customer2)\n",
        "    x_y = self.listToArray(x_y)\n",
        "    x_yTranspose = self.listToArray(x_yTranspose)\n",
        "    S_inverse =  self.listToArray(S_inverse)\n",
        "\n",
        "    # variable for the holding multiplication of covariance matrix and\n",
        "    # transpose of customers' point differences. it will have 1x15 dimensions\n",
        "    tempMatrix = [[0 for i in range(1)] for j in range(d)]\n",
        "\n",
        "    # nested loop for the  multiplication of covariance matrix and\n",
        "    # transpose of customers' point differences. 15x15 dimension * 15x1 dimensions respectively\n",
        "    for i in range(len(S_inverse[0])):\n",
        "      for j in range(len(x_yTranspose[0][0])):\n",
        "          for k in range(len(S_inverse[0])):\n",
        "              tempMatrix[i][j] += S_inverse[0][i][k] * x_yTranspose[0][k][j]\n",
        "\n",
        "    # multiplication of (customer1 - customer2) * tempMatrix. 1x15 dimension * 15x1 dimension\n",
        "    # and, result of the multiplication will give scalar value. \n",
        "    finalResult = 0\n",
        "    for i in range(0,d):\n",
        "      finalResult += x_y[0][0][i] * tempMatrix[i][0]\n",
        "    \n",
        "    # (1x15) * (15x15) * (15x1) = scalar value = mahalonobis distance\n",
        "    return finalResult**(1.0/2)\n",
        "\n",
        "  \"\"\" This method provides us to generate covariance matrix like desribed below with 15x15 dimensions:\n",
        "\n",
        "                      Gender      Age       Graduated     . . . . . .\n",
        "  Gender              var(G,G)    cov(G,A)  cov(G,G)      . . . . . .\n",
        "  Age                 cov(E,G)    var(E,A)  cov(E,G)      . . . . . .  \n",
        "  Graduated           cov(G,G)    cov(G,A)  var(G,G)      . . . . . . \n",
        "  .                    .            .         .                 \n",
        "  .                    .            .         .   \n",
        "  .                    .            .         .\n",
        "  Profession_Lawyer   cov(P,G)    cov(P,A)  cov(P,G)      . . . . . . \n",
        "  \"\"\"\n",
        "  def findDatasetCovariance(self, df):\n",
        "    # the array that holds column names to access data  \n",
        "    dataSetNames = [\"Gender\", \"Ever_Married\", \"Age\", \"Graduated\", \"Work_Experience\",\n",
        "                    \"Spending_Score\",\"Family_Size\",\"Profession_Artist\", \"Profession_Doctor\",\n",
        "                   \"Profession_Engineer\", \"Profession_Entertainment\", \"Profession_Executive\",\n",
        "                    \"Profession_Homemaker\", \"Profession_Lawyer\", \"Profession_Marketing\"]\n",
        "\n",
        "    #initialization of the covariance matrix\n",
        "    cov_of_dataset = [[0 for i in range(len(dataSetNames))] for j in range(len(dataSetNames))]\n",
        "\n",
        "    # Using the nested for loop, we create covariance matrix.\n",
        "    for i in range(len(dataSetNames)):\n",
        "      for j in range(len(dataSetNames)):\n",
        "       cov_of_dataset[i][j] = self.cov((df[dataSetNames[i]].to_numpy()).astype(int)\n",
        "       ,(df[dataSetNames[j]].to_numpy()).astype(int)) \n",
        "\n",
        "    return cov_of_dataset\n",
        "\n",
        "  # This method calculate covariance for each attribute using covariance matrix formula\n",
        "  def cov(self, a, b):\n",
        "    # Formula: (sum of((column1 - mean of column1)* (column2 -mean of column2)))/n\n",
        "      if len(a) != len(b):\n",
        "          return\n",
        "\n",
        "      # calculating mean for each column using method\n",
        "      b_mean = self.mean(b)\n",
        "      a_mean = self.mean(a)\n",
        "\n",
        "      sum = 0\n",
        "\n",
        "      # Implementation of the formula\n",
        "      for i in range(0, len(a)):\n",
        "          sum += ((a[i] - a_mean) * (b[i] - b_mean))\n",
        "\n",
        "      return sum/(len(a))\n",
        "\n",
        "  # finding mean of the dataset\n",
        "  def mean(self, a):\n",
        "    sum = 0 \n",
        "\n",
        "    for i in range(0, len(a)):\n",
        "      sum += a[i] \n",
        "\n",
        "    return sum/(len(a))\n",
        "\n",
        "  # This methods provides us to calculate inverse of 15x15 \n",
        "  # covariance matrix using Gauss Jordan method\n",
        "  def inverse(self, covariance):\n",
        "\n",
        "    n = len(covariance)\n",
        "    # initializing to zeros for storing augmented matrix with 15x30 dimensions\n",
        "    inverse = [[0 for i in range(2*len(covariance))] for j in range(len(covariance))]\n",
        "\n",
        "    for i in range(n):\n",
        "      for j in range(n):\n",
        "        inverse[i][j] = covariance[i][j]\n",
        "\n",
        "    # creating augmented matrix\n",
        "    for i in range(n):\n",
        "      for j in range(n):\n",
        "        if i == j:\n",
        "          inverse[i][j+n] = 1\n",
        "\n",
        "    # Applying Gauss Jordan method\n",
        "    for i in range(n):\n",
        "      if inverse[i][i] == 0:\n",
        "        print(\"ERROR OCCURED\")\n",
        "      for j in range(n):\n",
        "        if i != j:\n",
        "          ratio = inverse[j][i]/inverse[i][i]\n",
        "          for k in range(2*n):\n",
        "            inverse[j][k] = inverse[j][k] - ratio * inverse[i][k]\n",
        "\n",
        "    # Row operation to make diagonal element to 1\n",
        "    for i in range(n):\n",
        "      divisor = inverse[i][i]\n",
        "      for j in range(n,2*n):\n",
        "        inverse[i][j] = inverse[i][j]/divisor\n",
        "    inverseList = []\n",
        "\n",
        "    # converting to list\n",
        "    for i in range(n):\n",
        "        for j in range(n, 2*n):\n",
        "          inverseList.append(inverse[i][j])\n",
        "\n",
        "    # converting list to array using listToMatrix method\n",
        "    inverseMatrix = self.listToArray(inverseList)\n",
        "    return inverseMatrix\n",
        "\n",
        "  def listToArray(self, a):\n",
        "    m = []\n",
        "    while a != []:\n",
        "        m.append(a[:15])\n",
        "        a = a[15:]\n",
        "    return m\n",
        "\n",
        "  # gettting customer's data from the csv file \n",
        "  def get_customer_by_ID(self, df, col, val):\n",
        "    return df[df[col]==val]"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "hw = customer_segmentation(0)\n",
        "hw.similarity_measure()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "u04Gi8ydxXnL",
        "outputId": "fe670927-044a-46e9-8f33-9b7bbd3a73ff"
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Minkowski distance:  7.0021855651307225\n",
            "Mahalonobis distance:  16.177404391705053\n"
          ]
        }
      ]
    }
  ]
}