recsys.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "view-in-github"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Vlasdislav/recsys/blob/main/recsys.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "T1qZogC6sBYc",
        "outputId": "4550a0db-01ce-45df-f11b-a642047cd786"
      },
      "outputs": [],
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7Hu24KH_dok1"
      },
      "source": [
        "# Установка и импорт библиотек"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "Knco5t4ctDKB",
        "outputId": "3e28031e-9ae2-4161-e534-9ff8489aa53f"
      },
      "outputs": [],
      "source": [
        "!pip install pymorphy2\n",
        "!pip install -U pip setuptools wheel\n",
        "!pip install -U spacy\n",
        "!python -m spacy download en_core_web_sm\n",
        "!python -m spacy download ru_core_news_sm\n",
        "!pip install jupyter-dash\n",
        "!pip install dash-bootstrap-components"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 260
        },
        "id": "FxHiqcndrkvL",
        "outputId": "99f6823a-5efa-43d6-e069-100ca6174e5a"
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "from pymorphy2.analyzer import MorphAnalyzer\n",
        "from nltk.corpus import stopwords\n",
        "import nltk\n",
        "\n",
        "nltk.download('stopwords')\n",
        "nltk.download('punkt')\n",
        "nltk.download('wordnet')\n",
        "nltk.download('omw-1.4')\n",
        "\n",
        "from sklearn.feature_extraction.text import TfidfVectorizer\n",
        "from sklearn.cluster import KMeans\n",
        "from sklearn.cluster import AgglomerativeClustering\n",
        "\n",
        "from ast import literal_eval\n",
        "from random import sample\n",
        "import re\n",
        "\n",
        "from sklearn import metrics\n",
        "from sklearn.metrics.pairwise import cosine_similarity\n",
        "import spacy\n",
        "\n",
        "from jupyter_dash import JupyterDash\n",
        "import dash_core_components as dcc\n",
        "import dash_html_components as html\n",
        "import dash_bootstrap_components as dbc\n",
        "from dash.dependencies import Input, State, Output"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "l-nr7gZ7dx3x"
      },
      "source": [
        "# Подготовка данных"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "id": "tD1VZUhztQy4"
      },
      "outputs": [],
      "source": [
        "NUM_DATA = 1000\n",
        "NUM_CLUSTERS = 23"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "id": "H3acnZMdr-BB"
      },
      "outputs": [],
      "source": [
        "data = pd.read_csv('/content/drive/MyDrive/Colab Notebooks/data/IT_vacancies_full.csv')[:NUM_DATA]  # Получаем первые NUM_DATA вакансий\n",
        "data_small = data.loc[:, ['Ids', 'Employer', 'Name', 'Description']]                                # Получение нового DataFrame для дальнейшей работы"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 902
        },
        "id": "C6HYpr1z77Lo",
        "outputId": "41077e8a-5f38-4187-d0d2-06ca30c49e45"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-2b3841e7-ceff-4338-9abf-86ee45c45056\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Ids</th>\n",
              "      <th>Employer</th>\n",
              "      <th>Name</th>\n",
              "      <th>Salary</th>\n",
              "      <th>From</th>\n",
              "      <th>To</th>\n",
              "      <th>Experience</th>\n",
              "      <th>Schedule</th>\n",
              "      <th>Keys</th>\n",
              "      <th>Description</th>\n",
              "      <th>Area</th>\n",
              "      <th>Professional roles</th>\n",
              "      <th>Specializations</th>\n",
              "      <th>Profarea names</th>\n",
              "      <th>Published at</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>49313809</td>\n",
              "      <td>Space307</td>\n",
              "      <td>Golang Developer (Кипр)</td>\n",
              "      <td>True</td>\n",
              "      <td>251322.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>От 3 до 6 лет</td>\n",
              "      <td>Полный день</td>\n",
              "      <td>['Docker', 'Golang', 'Redis', 'Английский язык...</td>\n",
              "      <td>Мы в Space307 разрабатываем международную торг...</td>\n",
              "      <td>Санкт-Петербург</td>\n",
              "      <td>['Программист, разработчик']</td>\n",
              "      <td>['Программирование, Разработка']</td>\n",
              "      <td>['Информационные технологии, интернет, телеком']</td>\n",
              "      <td>2021-12-02 12:15:37+03:00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>48813842</td>\n",
              "      <td>Монополия</td>\n",
              "      <td>Е-mail маркетолог</td>\n",
              "      <td>True</td>\n",
              "      <td>60900.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>От 1 года до 3 лет</td>\n",
              "      <td>Полный день</td>\n",
              "      <td>['Грамотность', 'Написание текстов', 'Грамотна...</td>\n",
              "      <td>С 2015 года наш IT блок меняет рынок автотранс...</td>\n",
              "      <td>Санкт-Петербург</td>\n",
              "      <td>['Менеджер по маркетингу и рекламе']</td>\n",
              "      <td>['Маркетинг']</td>\n",
              "      <td>['Информационные технологии, интернет, телеком']</td>\n",
              "      <td>2021-12-02 10:33:15+03:00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>49413720</td>\n",
              "      <td>Eden Springs</td>\n",
              "      <td>Оператор call-центра (удаленно)</td>\n",
              "      <td>False</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>От 1 года до 3 лет</td>\n",
              "      <td>Удаленная работа</td>\n",
              "      <td>['Клиентоориентированность', 'Ориентация на ре...</td>\n",
              "      <td>Что нужно будет делать:  Принимать входящие зв...</td>\n",
              "      <td>Санкт-Петербург</td>\n",
              "      <td>['Оператор call-центра, специалист контактного...</td>\n",
              "      <td>['Маркетинг', 'Продажи по телефону, Телемаркет...</td>\n",
              "      <td>['Продажи', 'Информационные технологии, интерн...</td>\n",
              "      <td>2021-12-02 10:29:37+03:00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>46460892</td>\n",
              "      <td>Импорт Хоум</td>\n",
              "      <td>Ведущий SMM специалист</td>\n",
              "      <td>True</td>\n",
              "      <td>60000.0</td>\n",
              "      <td>80000.0</td>\n",
              "      <td>От 1 года до 3 лет</td>\n",
              "      <td>Полный день</td>\n",
              "      <td>['Продвижение бренда', 'Креативность', 'Adobe ...</td>\n",
              "      <td>В данный момент мы ищем в нашу команду самого ...</td>\n",
              "      <td>Санкт-Петербург</td>\n",
              "      <td>['SMM-менеджер, контент-менеджер']</td>\n",
              "      <td>['Управление маркетингом', 'PR, Маркетинговые ...</td>\n",
              "      <td>['Информационные технологии, интернет, телеком...</td>\n",
              "      <td>2021-12-01 16:57:02+03:00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>49555567</td>\n",
              "      <td>Pride Games Studio</td>\n",
              "      <td>UX/UI Designer</td>\n",
              "      <td>False</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>От 1 года до 3 лет</td>\n",
              "      <td>Полный день</td>\n",
              "      <td>['UI', 'UX', 'gamedev', 'game design', 'проект...</td>\n",
              "      <td>Pride Games Studio — это команда единомышленни...</td>\n",
              "      <td>Санкт-Петербург</td>\n",
              "      <td>['Дизайнер, художник']</td>\n",
              "      <td>['Игровое ПО', 'Программирование, Разработка',...</td>\n",
              "      <td>['Маркетинг, реклама, PR', 'Информационные тех...</td>\n",
              "      <td>2021-12-01 16:48:24+03:00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>995</th>\n",
              "      <td>49554167</td>\n",
              "      <td>LCG Recruiting</td>\n",
              "      <td>Менеджер по продажам</td>\n",
              "      <td>True</td>\n",
              "      <td>60900.0</td>\n",
              "      <td>174000.0</td>\n",
              "      <td>От 1 года до 3 лет</td>\n",
              "      <td>Полный день</td>\n",
              "      <td>['Пользователь ПК', 'Работа в команде', 'Грамо...</td>\n",
              "      <td>Компания WelcomePro открывает конкурс на должн...</td>\n",
              "      <td>Санкт-Петербург</td>\n",
              "      <td>['Менеджер по продажам, менеджер по работе с к...</td>\n",
              "      <td>['Продажи', 'Прямые продажи', 'Продажи по теле...</td>\n",
              "      <td>['Информационные технологии, интернет, телеком...</td>\n",
              "      <td>2021-12-01 11:23:37+03:00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>996</th>\n",
              "      <td>50096504</td>\n",
              "      <td>Пакт</td>\n",
              "      <td>Тестировщик веб-приложений</td>\n",
              "      <td>True</td>\n",
              "      <td>60000.0</td>\n",
              "      <td>80000.0</td>\n",
              "      <td>От 1 года до 3 лет</td>\n",
              "      <td>Удаленная работа</td>\n",
              "      <td>['CRM', 'Деловая коммуникация', 'Test case', '...</td>\n",
              "      <td>Внимание, конкурс на вакансию открыт до 25 дек...</td>\n",
              "      <td>Санкт-Петербург</td>\n",
              "      <td>['Тестировщик']</td>\n",
              "      <td>['Системная интеграция', 'Тестирование', 'CRM ...</td>\n",
              "      <td>['Информационные технологии, интернет, телеком']</td>\n",
              "      <td>2021-12-01 11:23:30+03:00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>997</th>\n",
              "      <td>45555166</td>\n",
              "      <td>Колл-Тулз</td>\n",
              "      <td>Middle Python developer (релокация в Сочи)</td>\n",
              "      <td>True</td>\n",
              "      <td>150000.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>От 1 года до 3 лет</td>\n",
              "      <td>Полный день</td>\n",
              "      <td>['Python', 'jQuery', 'PostgreSQL', 'Django Fra...</td>\n",
              "      <td>Middle Python developer (релокация в Сочи) При...</td>\n",
              "      <td>Санкт-Петербург</td>\n",
              "      <td>['Программист, разработчик']</td>\n",
              "      <td>['Программирование, Разработка']</td>\n",
              "      <td>['Информационные технологии, интернет, телеком']</td>\n",
              "      <td>2021-12-01 11:23:28+03:00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>998</th>\n",
              "      <td>49495639</td>\n",
              "      <td>КОННЕКТИВ ПЛМ</td>\n",
              "      <td>Программист</td>\n",
              "      <td>False</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Более 6 лет</td>\n",
              "      <td>Полный день</td>\n",
              "      <td>['Английский язык', 'Информационные технологии...</td>\n",
              "      <td>Требования  Базовое понимание принципов разраб...</td>\n",
              "      <td>Санкт-Петербург</td>\n",
              "      <td>['Программист, разработчик']</td>\n",
              "      <td>['Программирование, Разработка', 'Системная ин...</td>\n",
              "      <td>['Информационные технологии, интернет, телеком...</td>\n",
              "      <td>2021-12-01 11:23:14+03:00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>999</th>\n",
              "      <td>40759435</td>\n",
              "      <td>SkyNet</td>\n",
              "      <td>Специалист по работе с клиентами</td>\n",
              "      <td>True</td>\n",
              "      <td>39150.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Нет опыта</td>\n",
              "      <td>Сменный график</td>\n",
              "      <td>['Телефонные переговоры', 'Деловое общение', '...</td>\n",
              "      <td>SKYNET - ОПЕРАТОР СВЯЗИ. НАШ ПРИНЦИП-ДЕЛАТЬ ХО...</td>\n",
              "      <td>Санкт-Петербург</td>\n",
              "      <td>['Менеджер по продажам, менеджер по работе с к...</td>\n",
              "      <td>['Информационные технологии', 'Начальный урове...</td>\n",
              "      <td>['Административный персонал', 'Начало карьеры,...</td>\n",
              "      <td>2021-12-01 11:23:14+03:00</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>1000 rows × 15 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-2b3841e7-ceff-4338-9abf-86ee45c45056')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-2b3841e7-ceff-4338-9abf-86ee45c45056 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-2b3841e7-ceff-4338-9abf-86ee45c45056');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "          Ids            Employer                                        Name  \\\n",
              "0    49313809            Space307                     Golang Developer (Кипр)   \n",
              "1    48813842           Монополия                           Е-mail маркетолог   \n",
              "2    49413720        Eden Springs             Оператор call-центра (удаленно)   \n",
              "3    46460892         Импорт Хоум                      Ведущий SMM специалист   \n",
              "4    49555567  Pride Games Studio                              UX/UI Designer   \n",
              "..        ...                 ...                                         ...   \n",
              "995  49554167      LCG Recruiting                        Менеджер по продажам   \n",
              "996  50096504                Пакт                  Тестировщик веб-приложений   \n",
              "997  45555166           Колл-Тулз  Middle Python developer (релокация в Сочи)   \n",
              "998  49495639       КОННЕКТИВ ПЛМ                                 Программист   \n",
              "999  40759435              SkyNet            Специалист по работе с клиентами   \n",
              "\n",
              "     Salary      From        To          Experience          Schedule  \\\n",
              "0      True  251322.0       NaN       От 3 до 6 лет       Полный день   \n",
              "1      True   60900.0       NaN  От 1 года до 3 лет       Полный день   \n",
              "2     False       NaN       NaN  От 1 года до 3 лет  Удаленная работа   \n",
              "3      True   60000.0   80000.0  От 1 года до 3 лет       Полный день   \n",
              "4     False       NaN       NaN  От 1 года до 3 лет       Полный день   \n",
              "..      ...       ...       ...                 ...               ...   \n",
              "995    True   60900.0  174000.0  От 1 года до 3 лет       Полный день   \n",
              "996    True   60000.0   80000.0  От 1 года до 3 лет  Удаленная работа   \n",
              "997    True  150000.0       NaN  От 1 года до 3 лет       Полный день   \n",
              "998   False       NaN       NaN         Более 6 лет       Полный день   \n",
              "999    True   39150.0       NaN           Нет опыта    Сменный график   \n",
              "\n",
              "                                                  Keys  \\\n",
              "0    ['Docker', 'Golang', 'Redis', 'Английский язык...   \n",
              "1    ['Грамотность', 'Написание текстов', 'Грамотна...   \n",
              "2    ['Клиентоориентированность', 'Ориентация на ре...   \n",
              "3    ['Продвижение бренда', 'Креативность', 'Adobe ...   \n",
              "4    ['UI', 'UX', 'gamedev', 'game design', 'проект...   \n",
              "..                                                 ...   \n",
              "995  ['Пользователь ПК', 'Работа в команде', 'Грамо...   \n",
              "996  ['CRM', 'Деловая коммуникация', 'Test case', '...   \n",
              "997  ['Python', 'jQuery', 'PostgreSQL', 'Django Fra...   \n",
              "998  ['Английский язык', 'Информационные технологии...   \n",
              "999  ['Телефонные переговоры', 'Деловое общение', '...   \n",
              "\n",
              "                                           Description             Area  \\\n",
              "0    Мы в Space307 разрабатываем международную торг...  Санкт-Петербург   \n",
              "1    С 2015 года наш IT блок меняет рынок автотранс...  Санкт-Петербург   \n",
              "2    Что нужно будет делать:  Принимать входящие зв...  Санкт-Петербург   \n",
              "3    В данный момент мы ищем в нашу команду самого ...  Санкт-Петербург   \n",
              "4    Pride Games Studio — это команда единомышленни...  Санкт-Петербург   \n",
              "..                                                 ...              ...   \n",
              "995  Компания WelcomePro открывает конкурс на должн...  Санкт-Петербург   \n",
              "996  Внимание, конкурс на вакансию открыт до 25 дек...  Санкт-Петербург   \n",
              "997  Middle Python developer (релокация в Сочи) При...  Санкт-Петербург   \n",
              "998  Требования  Базовое понимание принципов разраб...  Санкт-Петербург   \n",
              "999  SKYNET - ОПЕРАТОР СВЯЗИ. НАШ ПРИНЦИП-ДЕЛАТЬ ХО...  Санкт-Петербург   \n",
              "\n",
              "                                    Professional roles  \\\n",
              "0                         ['Программист, разработчик']   \n",
              "1                 ['Менеджер по маркетингу и рекламе']   \n",
              "2    ['Оператор call-центра, специалист контактного...   \n",
              "3                   ['SMM-менеджер, контент-менеджер']   \n",
              "4                               ['Дизайнер, художник']   \n",
              "..                                                 ...   \n",
              "995  ['Менеджер по продажам, менеджер по работе с к...   \n",
              "996                                    ['Тестировщик']   \n",
              "997                       ['Программист, разработчик']   \n",
              "998                       ['Программист, разработчик']   \n",
              "999  ['Менеджер по продажам, менеджер по работе с к...   \n",
              "\n",
              "                                       Specializations  \\\n",
              "0                     ['Программирование, Разработка']   \n",
              "1                                        ['Маркетинг']   \n",
              "2    ['Маркетинг', 'Продажи по телефону, Телемаркет...   \n",
              "3    ['Управление маркетингом', 'PR, Маркетинговые ...   \n",
              "4    ['Игровое ПО', 'Программирование, Разработка',...   \n",
              "..                                                 ...   \n",
              "995  ['Продажи', 'Прямые продажи', 'Продажи по теле...   \n",
              "996  ['Системная интеграция', 'Тестирование', 'CRM ...   \n",
              "997                   ['Программирование, Разработка']   \n",
              "998  ['Программирование, Разработка', 'Системная ин...   \n",
              "999  ['Информационные технологии', 'Начальный урове...   \n",
              "\n",
              "                                        Profarea names  \\\n",
              "0     ['Информационные технологии, интернет, телеком']   \n",
              "1     ['Информационные технологии, интернет, телеком']   \n",
              "2    ['Продажи', 'Информационные технологии, интерн...   \n",
              "3    ['Информационные технологии, интернет, телеком...   \n",
              "4    ['Маркетинг, реклама, PR', 'Информационные тех...   \n",
              "..                                                 ...   \n",
              "995  ['Информационные технологии, интернет, телеком...   \n",
              "996   ['Информационные технологии, интернет, телеком']   \n",
              "997   ['Информационные технологии, интернет, телеком']   \n",
              "998  ['Информационные технологии, интернет, телеком...   \n",
              "999  ['Административный персонал', 'Начало карьеры,...   \n",
              "\n",
              "                  Published at  \n",
              "0    2021-12-02 12:15:37+03:00  \n",
              "1    2021-12-02 10:33:15+03:00  \n",
              "2    2021-12-02 10:29:37+03:00  \n",
              "3    2021-12-01 16:57:02+03:00  \n",
              "4    2021-12-01 16:48:24+03:00  \n",
              "..                         ...  \n",
              "995  2021-12-01 11:23:37+03:00  \n",
              "996  2021-12-01 11:23:30+03:00  \n",
              "997  2021-12-01 11:23:28+03:00  \n",
              "998  2021-12-01 11:23:14+03:00  \n",
              "999  2021-12-01 11:23:14+03:00  \n",
              "\n",
              "[1000 rows x 15 columns]"
            ]
          },
          "execution_count": 6,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "id": "mdLRDheiMG1B"
      },
      "outputs": [],
      "source": [
        "def to_text(arr):\n",
        "  \"\"\"Конвертация списка слов в текст\"\"\"\n",
        "  return arr.apply(lambda x: ' '.join(literal_eval(x))\n",
        "    if isinstance(x, str) else '')\n",
        "\n",
        "def create_combined_data(x):\n",
        "  \"\"\"Объединение значимых признаков в один текст\"\"\"\n",
        "  return x['Name'] + ' ' + to_text(x['Professional roles']) + ' ' + to_text(x['Keys']) + ' ' + to_text(x['Specializations']) + ' ' + to_text(x['Profarea names'])\n",
        "\n",
        "data_small['Description'] = create_combined_data(data)  # Перезаписываем признак Description новым описанием"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0IOU1q5enMmv"
      },
      "source": [
        "# Очистка данных и векторизация"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "id": "FyEjKhhDhpHo"
      },
      "outputs": [],
      "source": [
        "STOPWORDS = set(stopwords.words('russian')) | set(stopwords.words('english'))   # Множество русских и английских стоп слов\n",
        "PATTERN_ALL_SIGNS = '[«»—.,!@#$%^&*()_+=-?:;№\"\\|/<>{}-]'                        # Шаблон знаков\n",
        "\n",
        "def del_all_signs(text):\n",
        "  \"\"\"Удаление всех знаков из текста\"\"\"\n",
        "  return re.sub(PATTERN_ALL_SIGNS, ' ', text)\n",
        "\n",
        "def del_stop_words(text):\n",
        "  \"\"\"Удаление стоп слова из текста\"\"\"\n",
        "  text_split = text.split(' ')\n",
        "  for word in text_split:\n",
        "    if word in STOPWORDS:\n",
        "      text_split.remove(word)\n",
        "  return ' '.join(text_split)\n",
        "\n",
        "morph = MorphAnalyzer()\n",
        "\n",
        "def text2normal(text):\n",
        "  \"\"\"Лемматизация всех слов текста\"\"\"\n",
        "  text = text.split()\n",
        "  res = []\n",
        "  for word in text:\n",
        "    res.append(morph.parse(word)[0].normal_form)\n",
        "  return ' '.join(res)\n",
        "\n",
        "def text_clean(text):\n",
        "  \"\"\"Основная функция очистки текста,\n",
        "    исполняет ранее описанные функции\"\"\"\n",
        "  text = text.lower()\n",
        "  text = del_all_signs(text)\n",
        "  text = text2normal(text)\n",
        "  text = del_stop_words(text)\n",
        "  return text\n",
        "\n",
        "data_small['description_normal'] = data_small['Description'].apply(text_clean)  # Получаем текст пригодный для векторизации"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "id": "yz7OsFp3uWBf"
      },
      "outputs": [],
      "source": [
        "documents = data_small['description_normal'].values.astype('U') # Получаем данные из ранее подготовленных текстов\n",
        "vectorizer = TfidfVectorizer(stop_words=['english', 'russian']) # Инициализируем объект для векторизации текста\n",
        "features = vectorizer.fit_transform(documents)                  # Применение TfidfVectorizer, чтобы преобразовать текст в векторы признаков"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hFhje5tidSj8"
      },
      "source": [
        "# Определение количества кластеров (Метод силуэтов)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2AUiB-CNUOeb",
        "outputId": "a96da99c-773e-4dc8-a183-acbe965dbb37"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Оптимальное количество кластеров = 22\n"
          ]
        }
      ],
      "source": [
        "scores = []\n",
        "\n",
        "for i in range(2, 41):\n",
        "  model = KMeans(n_clusters=i, init='k-means++', max_iter=100, n_init=100).fit(features)\n",
        "  score = metrics.silhouette_score(features, model.labels_, metric='euclidean', sample_size=features.shape[0])\n",
        "  scores.append(score)\n",
        "\n",
        "num_clusters = np.argmax(scores)\n",
        "print('\\nОптимальное количество кластеров =', num_clusters)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BNxHsiUzdZLp"
      },
      "source": [
        "# Кластеризация методом k-means"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "id": "_UKMxJxu96Z9"
      },
      "outputs": [],
      "source": [
        "model = KMeans(n_clusters=NUM_CLUSTERS, init='k-means++', max_iter=100, n_init=100).fit(features)\n",
        "data_small['cluster'] = model.labels_"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rcVg0BvrwaES",
        "outputId": "fc534f1a-deb4-491f-9c9c-7c07bb8e8771"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Cluster centroids: \n",
            "\n",
            "Cluster 0:\n",
            " 1с\n",
            " программист\n",
            " предприятие\n",
            " управление\n",
            " программирование\n",
            " erp\n",
            " система\n",
            " web\n",
            " разработчик\n",
            " инженер\n",
            "---------------\n",
            "Cluster 1:\n",
            " центр\n",
            " call\n",
            " оператор\n",
            " начальный\n",
            " мало\n",
            " уровень\n",
            " опыт\n",
            " административный\n",
            " сотрудник\n",
            " работа\n",
            "---------------\n",
            "Cluster 2:\n",
            " проект\n",
            " управление\n",
            " руководитель\n",
            " project\n",
            " группа\n",
            " разработка\n",
            " технология\n",
            " информационный\n",
            " интернет\n",
            " менеджмент\n",
            "---------------\n",
            "Cluster 3:\n",
            " поддержка\n",
            " технический\n",
            " специалист\n",
            " helpdesk\n",
            " мало\n",
            " начальный\n",
            " опыт\n",
            " уровень\n",
            " системный\n",
            " интернет\n",
            "---------------\n",
            "Cluster 4:\n",
            " администратор\n",
            " системный\n",
            " сетевой\n",
            " технология\n",
            " администрирование\n",
            " интернет\n",
            " linux\n",
            " инженер\n",
            " поддержка\n",
            " windows\n",
            "---------------\n",
            "Cluster 5:\n",
            " дизайнер\n",
            " дизайн\n",
            " художник\n",
            " adobe\n",
            " графика\n",
            " игровой\n",
            " живопись\n",
            " развлечение\n",
            " масса\n",
            " медиа\n",
            "---------------\n",
            "Cluster 6:\n",
            " игровой\n",
            " game\n",
            " unreal\n",
            " engine\n",
            " unity\n",
            " developer\n",
            " программист\n",
            " разработчик\n",
            " software\n",
            " programming\n",
            "---------------\n",
            "Cluster 7:\n",
            " web\n",
            " инженер\n",
            " мастер\n",
            " разработчик\n",
            " erp\n",
            " предприятие\n",
            " система\n",
            " программист\n",
            " управление\n",
            " программирование\n",
            "---------------\n",
            "Cluster 8:\n",
            " qa\n",
            " тестирование\n",
            " тестировщик\n",
            " engineer\n",
            " automation\n",
            " интернет\n",
            " технология\n",
            " информационный\n",
            " телек\n",
            " java\n",
            "---------------\n",
            "Cluster 9:\n",
            " маркетинг\n",
            " реклама\n",
            " pr\n",
            " менеджер\n",
            " контент\n",
            " интернет\n",
            " маркетинговый\n",
            " копирайтер\n",
            " seo\n",
            " google\n",
            "---------------\n",
            "Cluster 10:\n",
            " сервисный\n",
            " инсталляция\n",
            " инженер\n",
            " монтажник\n",
            " сервис\n",
            " механик\n",
            " оборудование\n",
            " настройка\n",
            " телекоммуникация\n",
            " рабочий\n",
            "---------------\n",
            "Cluster 11:\n",
            " аналитик\n",
            " управление\n",
            " анализ\n",
            " системный\n",
            " проект\n",
            " интеграция\n",
            " бизнес\n",
            " система\n",
            " предприятие\n",
            " erp\n",
            "---------------\n",
            "Cluster 12:\n",
            " js\n",
            " node\n",
            " javascript\n",
            " разработчик\n",
            " программист\n",
            " developer\n",
            " vue\n",
            " typescript\n",
            " разработка\n",
            " react\n",
            "---------------\n",
            "Cluster 13:\n",
            " менеджер\n",
            " продажа\n",
            " клиент\n",
            " работа\n",
            " деловой\n",
            " общение\n",
            " компьютерный\n",
            " маркетинг\n",
            " интернет\n",
            " проект\n",
            "---------------\n",
            "Cluster 14:\n",
            " php\n",
            " web\n",
            " mysql\n",
            " разработчик\n",
            " программист\n",
            " git\n",
            " программирование\n",
            " javascript\n",
            " инженер\n",
            " разработка\n",
            "---------------\n",
            "Cluster 15:\n",
            " программист\n",
            " разработчик\n",
            " инженер\n",
            " разработка\n",
            " программирование\n",
            " технология\n",
            " информационный\n",
            " интернет\n",
            " linux\n",
            " телек\n",
            "---------------\n",
            "Cluster 16:\n",
            " тестирование\n",
            " тестировщик\n",
            " qa\n",
            " инженер\n",
            " функциональный\n",
            " пользовательский\n",
            " регресcионный\n",
            " интерфейс\n",
            " интернет\n",
            " нагрузочный\n",
            "---------------\n",
            "Cluster 17:\n",
            " станок\n",
            " оператор\n",
            " чпу\n",
            " наладчик\n",
            " токарь\n",
            " машиностроение\n",
            " фрезеровщик\n",
            " сельский\n",
            " хозяйство\n",
            " производство\n",
            "---------------\n",
            "Cluster 18:\n",
            " react\n",
            " frontend\n",
            " разработчик\n",
            " javascript\n",
            " программист\n",
            " программирование\n",
            " git\n",
            " typescript\n",
            " разработка\n",
            " redux\n",
            "---------------\n",
            "Cluster 19:\n",
            " net\n",
            " sql\n",
            " ms\n",
            " asp\n",
            " программист\n",
            " разработчик\n",
            " server\n",
            " framework\n",
            " web\n",
            " инженер\n",
            "---------------\n",
            "Cluster 20:\n",
            " java\n",
            " spring\n",
            " разработчик\n",
            " framework\n",
            " программист\n",
            " kotlin\n",
            " developer\n",
            " программирование\n",
            " разработка\n",
            " hibernate\n",
            "---------------\n",
            "Cluster 21:\n",
            " продажа\n",
            " менеджер\n",
            " клиент\n",
            " работа\n",
            " переговоры\n",
            " активный\n",
            " телемаркетинг\n",
            " телефон\n",
            " b2b\n",
            " навык\n",
            "---------------\n",
            "Cluster 22:\n",
            " безопасность\n",
            " компьютерный\n",
            " информационный\n",
            " специалист\n",
            " программист\n",
            " технология\n",
            " инженер\n",
            " телек\n",
            " интернет\n",
            " разработчик\n",
            "---------------\n"
          ]
        }
      ],
      "source": [
        "# Вывод кластеров с их содержимым\n",
        "print('Cluster centroids: \\n')\n",
        "older_centroids = model.cluster_centers_.argsort()[:, ::-1]\n",
        "terms = vectorizer.get_feature_names_out()\n",
        "\n",
        "content_clusters = []\n",
        "\n",
        "for i in range(NUM_CLUSTERS):\n",
        "  print('Cluster %d:' % i)\n",
        "  content_clusters.append([terms[j] for j in older_centroids[i, :10]])\n",
        "  for j in older_centroids[i, :10]:\n",
        "    print(' %s' % terms[j])\n",
        "  print('---------------')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wW1-Hap2dffl"
      },
      "source": [
        "# Интерфейс для работы с ядром рекомендательной системы (k-means)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "id": "3DJunmZc04sg"
      },
      "outputs": [],
      "source": [
        "def predict_top(id, data_small, top):\n",
        "  \"\"\"Функция возвращает для конкретной вакансии по id топ подходящих вакансий в количестве top (Кластеризация k-means)\"\"\"\n",
        "  tops = [x for x in range(len(data_small['cluster'])) if x != id and data_small['cluster'][x] == data_small['cluster'][id]]\n",
        "  # return data.iloc[sample(tops, len(tops))[:top]] # Возвращает случайные вакансии из кластера\n",
        "  return data.iloc[tops[:top]]                      # Возвращает топ ближайших вакансий"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 706
        },
        "id": "WnlYZs87T9q4",
        "outputId": "4fd4f580-c996-4b16-8da3-2ca8680a1254"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Dash app running on:\n"
          ]
        },
        {
          "data": {
            "application/javascript": "(async (port, path, text, element) => {\n    if (!google.colab.kernel.accessAllowed) {\n      return;\n    }\n    element.appendChild(document.createTextNode(''));\n    const url = await google.colab.kernel.proxyPort(port);\n    const anchor = document.createElement('a');\n    anchor.href = new URL(path, url).toString();\n    anchor.target = '_blank';\n    anchor.setAttribute('data-href', url + path);\n    anchor.textContent = text;\n    element.appendChild(anchor);\n  })(8050, \"/\", \"http://127.0.0.1:8050/\", window.element)",
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/javascript": "(async (port, path, width, height, cache, element) => {\n    if (!google.colab.kernel.accessAllowed && !cache) {\n      return;\n    }\n    element.appendChild(document.createTextNode(''));\n    const url = await google.colab.kernel.proxyPort(port, {cache});\n    const iframe = document.createElement('iframe');\n    iframe.src = new URL(path, url).toString();\n    iframe.height = height;\n    iframe.width = width;\n    iframe.style.border = 0;\n    iframe.allow = [\n        'accelerometer',\n        'autoplay',\n        'camera',\n        'clipboard-read',\n        'clipboard-write',\n        'gyroscope',\n        'magnetometer',\n        'microphone',\n        'serial',\n        'usb',\n        'xr-spatial-tracking',\n    ].join('; ');\n    element.appendChild(iframe);\n  })(8050, \"/\", \"100%\", 650, false, window.element)",
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "app = JupyterDash(__name__, external_stylesheets=[dbc.themes.BOOTSTRAP])\n",
        "\n",
        "search_bar = dbc.Row(\n",
        "    [\n",
        "        dbc.Col(dbc.Input(id='input-on-submit', placeholder=\"Введите индекс вакансии\", type=\"search\")),\n",
        "        dbc.Col(\n",
        "            dbc.Button(\n",
        "                \"Поиск\", id='submit-val', color=\"primary\", className=\"ms-2\", n_clicks=0\n",
        "            ),\n",
        "            width=\"auto\",\n",
        "        ),\n",
        "    ],\n",
        "    className=\"g-0 ms-auto flex-nowrap mt-3 mt-md-0\",\n",
        "    align=\"center\",\n",
        ")\n",
        "\n",
        "app.layout = html.Div([\n",
        "    html.Br(),\n",
        "    dbc.Row([\n",
        "        dbc.Col(html.Div(), md=3),\n",
        "        dbc.Col(\n",
        "            search_bar,\n",
        "            md=6\n",
        "        ),\n",
        "        dbc.Col(html.Div(), md=3),\n",
        "    ]),\n",
        "    html.Br(),\n",
        "    html.Div(id='container-button-basic'),\n",
        "])\n",
        "\n",
        "@app.callback(\n",
        "    Output('container-button-basic', 'children'),\n",
        "    Input('submit-val', 'n_clicks'),\n",
        "    State('input-on-submit', 'value')\n",
        ")\n",
        "\n",
        "def update_output(n_clicks, value):\n",
        "    top = 3\n",
        "\n",
        "    prediction = predict_top(int(value), data_small, top)\n",
        "\n",
        "    output = []\n",
        "\n",
        "    output.append(\n",
        "        html.Div(\n",
        "          dbc.Container(\n",
        "              [\n",
        "                  html.H1(\"#\" + str(value) + \" \" + data['Name'][int(value)], className=\"display-6\"),\n",
        "                  html.P(\n",
        "                     data['Employer'][int(value)],\n",
        "                      className=\"lead\",\n",
        "                  ),\n",
        "                  html.Hr(className=\"my-2\"),\n",
        "                  html.P(\n",
        "                      data['Description'][int(value)],\n",
        "                      style={ \"overflow\": \"hidden\",  \"-webkit-line-clamp\": \"5\", \"-webkit-box-orient\": \"vertical\", \"display\": \"-webkit-box\" }\n",
        "                  ),\n",
        "              ],\n",
        "              fluid=True,\n",
        "              className=\"py-3\",\n",
        "          ),\n",
        "          className=\"p-3 bg-light rounded-3\",\n",
        "        style={ \"width\": \"1200px\", \"margin\": \"auto\" }\n",
        "        )\n",
        "      )\n",
        "    output.append(html.Br())\n",
        "\n",
        "    for i in range(top):\n",
        "      output.append(\n",
        "        html.Div(\n",
        "          dbc.Container(\n",
        "              [\n",
        "                  html.H1(\"#\" + str(prediction.index[i]) + \" \" + prediction.iloc[i]['Name'], className=\"display-6\"),\n",
        "                  html.P(\n",
        "                      prediction.iloc[i]['Employer'],\n",
        "                      className=\"lead\",\n",
        "                  ),\n",
        "                  html.Hr(className=\"my-2\"),\n",
        "                  html.P(\n",
        "                      prediction.iloc[i]['Description'],\n",
        "                      style={ \"overflow\": \"hidden\",  \"-webkit-line-clamp\": \"5\", \"-webkit-box-orient\": \"vertical\", \"display\": \"-webkit-box\" }\n",
        "                  ),\n",
        "              ],\n",
        "              fluid=True,\n",
        "              className=\"py-3\",\n",
        "          ),\n",
        "          className=\"p-3 bg-light rounded-3\",\n",
        "          style={ \"width\": \"1200px\", \"margin\": \"auto\" }\n",
        "        )\n",
        "      )\n",
        "      output.append(html.Br())\n",
        "\n",
        "    return html.P(output)\n",
        "\n",
        "app.run_server(host='127.0.0.1')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "P5HkRaOnnh9y"
      },
      "source": [
        "# Использование косинусной близости для кластеризации"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "id": "dCpaY6X9ngi6"
      },
      "outputs": [],
      "source": [
        "def extract_best_indices(m, topk, mask=None):\n",
        "    if len(m.shape) > 1:\n",
        "        cos_sim = np.cosine_similarity(m, axis=0)\n",
        "    else:\n",
        "        cos_sim = m\n",
        "\n",
        "    index = np.argsort(cos_sim)[::-1]\n",
        "    if mask is not None:\n",
        "        assert mask.shape == m.shape\n",
        "        mask = mask[index]\n",
        "    else:\n",
        "        mask = np.ones(len(cos_sim))\n",
        "    mask = np.logical_or(cos_sim[index] != 0, mask)\n",
        "    best_index = index[mask][:topk]\n",
        "    return best_index"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "metadata": {
        "id": "rfEPfKmgn2oS"
      },
      "outputs": [],
      "source": [
        "def predict_spacy(model, query_sentence, embed_mat, topk):\n",
        "    topk += 1\n",
        "    query_embed = model(query_sentence)\n",
        "    mat = np.array([query_embed.similarity(line) for line in embed_mat])\n",
        "\n",
        "    mat_mask = np.array(\n",
        "        [True if line.vector_norm else False for line in embed_mat])\n",
        "    best_index = extract_best_indices(mat, topk=topk, mask=mat_mask)\n",
        "    return best_index[1:topk]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 17,
      "metadata": {
        "id": "cr6ywZz2oA64"
      },
      "outputs": [],
      "source": [
        "nlp = spacy.load(\"ru_core_news_sm\")\n",
        "data_small['spacy_description'] = data_small['Description'].apply(nlp)\n",
        "embed_mat = data_small['spacy_description'].values"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 18,
      "metadata": {
        "id": "ZWQDR0xjy87Y"
      },
      "outputs": [],
      "source": [
        "def predict_top(id, data_small, top):\n",
        "    \"\"\"Функция возвращает для конкретной вакансии по id\n",
        "    топ подходящих вакансий в количестве top (Косинусная схожесть)\"\"\"\n",
        "    test_sentence = data_small.iloc[id]['Description']\n",
        "    embed_mat = data_small['spacy_description'].values\n",
        "    best_index = predict_spacy(nlp, test_sentence, embed_mat, top)\n",
        "    return data.iloc[best_index]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KfLswRvNXn-2"
      },
      "source": [
        "# Интерфейс для работы с ядром рекомендательной системы (cosine similarity)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 19,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 86
        },
        "id": "FVrzHjIDzPKo",
        "outputId": "5efa037f-ca45-473d-ecad-686340f14ff9"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Dash app running on:\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Address already in use\n",
            "Port 8050 is in use by another program. Either identify and stop that program, or start the server with a different port.\n"
          ]
        },
        {
          "data": {
            "application/javascript": "(async (port, path, text, element) => {\n    if (!google.colab.kernel.accessAllowed) {\n      return;\n    }\n    element.appendChild(document.createTextNode(''));\n    const url = await google.colab.kernel.proxyPort(port);\n    const anchor = document.createElement('a');\n    anchor.href = new URL(path, url).toString();\n    anchor.target = '_blank';\n    anchor.setAttribute('data-href', url + path);\n    anchor.textContent = text;\n    element.appendChild(anchor);\n  })(8050, \"/\", \"http://0.0.0.0:8050/\", window.element)",
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "app = JupyterDash(__name__, external_stylesheets=[dbc.themes.BOOTSTRAP])\n",
        "\n",
        "search_bar = dbc.Row(\n",
        "    [\n",
        "        dbc.Col(dbc.Input(id='input-on-submit', placeholder=\"Введите индекс вакансии\", type=\"search\")),\n",
        "        dbc.Col(\n",
        "            dbc.Button(\n",
        "                \"Поиск\", id='submit-val', color=\"primary\", className=\"ms-2\", n_clicks=0\n",
        "            ),\n",
        "            width=\"auto\",\n",
        "        ),\n",
        "    ],\n",
        "    className=\"g-0 ms-auto flex-nowrap mt-3 mt-md-0\",\n",
        "    align=\"center\",\n",
        ")\n",
        "\n",
        "app.layout = html.Div([\n",
        "    html.Br(),\n",
        "    dbc.Row([\n",
        "        dbc.Col(html.Div(), md=3),\n",
        "        dbc.Col(\n",
        "            search_bar,\n",
        "            md=6\n",
        "        ),\n",
        "        dbc.Col(html.Div(), md=3),\n",
        "    ]),\n",
        "    html.Br(),\n",
        "    html.Div(id='container-button-basic'),\n",
        "])\n",
        "\n",
        "@app.callback(\n",
        "    Output('container-button-basic', 'children'),\n",
        "    Input('submit-val', 'n_clicks'),\n",
        "    State('input-on-submit', 'value')\n",
        ")\n",
        "\n",
        "def update_output(n_clicks, value):\n",
        "    top = 3\n",
        "\n",
        "    prediction = predict_top(int(value), data_small, top)\n",
        "\n",
        "    output = []\n",
        "\n",
        "    output.append(\n",
        "        html.Div(\n",
        "          dbc.Container(\n",
        "              [\n",
        "                  html.H1(\"#\" + str(value) + \" \" + data['Name'][int(value)], className=\"display-6\"),\n",
        "                  html.P(\n",
        "                     data['Employer'][int(value)],\n",
        "                      className=\"lead\",\n",
        "                  ),\n",
        "                  html.Hr(className=\"my-2\"),\n",
        "                  html.P(\n",
        "                      data['Description'][int(value)],\n",
        "                      style={ \"overflow\": \"hidden\",  \"-webkit-line-clamp\": \"5\", \"-webkit-box-orient\": \"vertical\", \"display\": \"-webkit-box\" }\n",
        "                  ),\n",
        "              ],\n",
        "              fluid=True,\n",
        "              className=\"py-3\",\n",
        "          ),\n",
        "          className=\"p-3 bg-light rounded-3\",\n",
        "        style={ \"width\": \"1200px\", \"margin\": \"auto\" }\n",
        "        )\n",
        "      )\n",
        "    output.append(html.Br())\n",
        "\n",
        "    for i in range(top):\n",
        "      output.append(\n",
        "        html.Div(\n",
        "          dbc.Container(\n",
        "              [\n",
        "                  html.H1(\"#\" + str(prediction.index[i]) + \" \" + prediction.iloc[i]['Name'], className=\"display-6\"),\n",
        "                  html.P(\n",
        "                      prediction.iloc[i]['Employer'],\n",
        "                      className=\"lead\",\n",
        "                  ),\n",
        "                  html.Hr(className=\"my-2\"),\n",
        "                  html.P(\n",
        "                      prediction.iloc[i]['Description'],\n",
        "                      style={ \"overflow\": \"hidden\",  \"-webkit-line-clamp\": \"5\", \"-webkit-box-orient\": \"vertical\", \"display\": \"-webkit-box\" }\n",
        "                  ),\n",
        "              ],\n",
        "              fluid=True,\n",
        "              className=\"py-3\",\n",
        "          ),\n",
        "          className=\"p-3 bg-light rounded-3\",\n",
        "          style={ \"width\": \"1200px\", \"margin\": \"auto\" }\n",
        "        )\n",
        "      )\n",
        "      output.append(html.Br())\n",
        "\n",
        "    return html.P(output)\n",
        "\n",
        "app.run_server(host='0.0.0.0')"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "authorship_tag": "ABX9TyPhJazxeNrti/8ZeBmgKfi+",
      "include_colab_link": true,
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}