Langchain_Pinecone_openai_Indexing_.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/SathishNaqel/AIModels/blob/main/Langchain_Pinecone_openai_Indexing_.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!import os"
      ],
      "metadata": {
        "id": "4EIGR9DMGi-l",
        "outputId": "b7f61854-8fa5-429f-ed99-b4c30833604a",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/bin/bash: line 1: import: command not found\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "aYpmmfQ-ZbIh"
      },
      "outputs": [],
      "source": [
        "!pip install --upgrade langchain openai  -q"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install sentence_transformers -q"
      ],
      "metadata": {
        "id": "cYfy2IhtvGVv"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install unstructured -q\n",
        "!pip install unstructured[local-inference] -q\n",
        "!pip install detectron2@git+https://github.com/facebookresearch/detectron2.git@v0.6#egg=detectron2 -q"
      ],
      "metadata": {
        "id": "XbL5zdf9aG8a",
        "outputId": "8ab94524-2e4d-46df-bb5b-9703c788552e",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for detectron2 (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for fvcore (setup.py) ... \u001b[?25l\u001b[?25hdone\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!apt-get install poppler-utils"
      ],
      "metadata": {
        "id": "HwalYTVZoRlH",
        "outputId": "bc5b8a29-d3f7-4e80-89df-13086cb67874",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Reading package lists... Done\n",
            "Building dependency tree... Done\n",
            "Reading state information... Done\n",
            "The following NEW packages will be installed:\n",
            "  poppler-utils\n",
            "0 upgraded, 1 newly installed, 0 to remove and 16 not upgraded.\n",
            "Need to get 186 kB of archives.\n",
            "After this operation, 696 kB of additional disk space will be used.\n",
            "Get:1 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 poppler-utils amd64 22.02.0-2ubuntu0.2 [186 kB]\n",
            "Fetched 186 kB in 1s (362 kB/s)\n",
            "Selecting previously unselected package poppler-utils.\n",
            "(Reading database ... 120901 files and directories currently installed.)\n",
            "Preparing to unpack .../poppler-utils_22.02.0-2ubuntu0.2_amd64.deb ...\n",
            "Unpacking poppler-utils (22.02.0-2ubuntu0.2) ...\n",
            "Setting up poppler-utils (22.02.0-2ubuntu0.2) ...\n",
            "Processing triggers for man-db (2.10.2-1) ...\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "https://python.langchain.com/en/latest/modules/indexes/document_loaders/examples/directory_loader.html"
      ],
      "metadata": {
        "id": "2AssnWUVlxtH"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from langchain.document_loaders import DirectoryLoader\n",
        "\n",
        "directory = '/content/data'\n",
        "\n",
        "def load_docs(directory):\n",
        "  loader = DirectoryLoader(directory)\n",
        "  documents = loader.load()\n",
        "  return documents\n",
        "\n",
        "documents = load_docs(directory)\n",
        "len(documents)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fulYnj9nZr3n",
        "outputId": "38787473-db55-4558-a73f-b4e534911e5d"
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Unzipping tokenizers/punkt.zip.\n",
            "[nltk_data] Downloading package averaged_perceptron_tagger to\n",
            "[nltk_data]     /root/nltk_data...\n",
            "[nltk_data]   Unzipping taggers/averaged_perceptron_tagger.zip.\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1"
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "https://python.langchain.com/en/latest/modules/indexes/text_splitters/getting_started.html"
      ],
      "metadata": {
        "id": "VwaCosqsogzw"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
        "\n",
        "def split_docs(documents,chunk_size=500,chunk_overlap=20):\n",
        "  text_splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap)\n",
        "  docs = text_splitter.split_documents(documents)\n",
        "  return docs\n",
        "\n",
        "docs = split_docs(documents)\n",
        "print(len(docs))"
      ],
      "metadata": {
        "id": "5lF8jA6xZ0Hm",
        "outputId": "fe3d014a-5aa9-494b-a280-8088341346e1",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "529\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(docs[8].page_content)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tVcFjgHJZ8S9",
        "outputId": "dcd46f3f-8c70-409f-af96-865247a0b498"
      },
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "conditions that do not detract from the rights of employees acquired under the Labor Law, its implementing regulations and the decisions issued in implementation thereof; provided that these additions or amendments shall not be effective until approved by the Ministry of Human Resources and Social Development.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#requires for open ai embedding\n",
        "!pip install tiktoken -q"
      ],
      "metadata": {
        "id": "6PiPwt-FaYwl",
        "outputId": "8748f131-b879-4389-bb06-62cfa2fa0ecb",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[?25l     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.0/1.7 MB\u001b[0m \u001b[31m?\u001b[0m eta \u001b[36m-:--:--\u001b[0m\r\u001b[2K     \u001b[91m━━\u001b[0m\u001b[90m╺\u001b[0m\u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.1/1.7 MB\u001b[0m \u001b[31m2.6 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\r\u001b[2K     \u001b[91m━━━━━━━\u001b[0m\u001b[90m╺\u001b[0m\u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.3/1.7 MB\u001b[0m \u001b[31m4.4 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\r\u001b[2K     \u001b[91m━━━━━━━━━━━━━━━\u001b[0m\u001b[90m╺\u001b[0m\u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.7/1.7 MB\u001b[0m \u001b[31m6.3 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\r\u001b[2K     \u001b[91m━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[90m╺\u001b[0m\u001b[90m━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.1/1.7 MB\u001b[0m \u001b[31m7.7 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\r\u001b[2K     \u001b[91m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[91m╸\u001b[0m\u001b[90m━━\u001b[0m \u001b[32m1.6/1.7 MB\u001b[0m \u001b[31m9.2 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\r\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.7/1.7 MB\u001b[0m \u001b[31m8.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!import os"
      ],
      "metadata": {
        "id": "GOov50M-GoI4",
        "outputId": "64a48e07-6990-4996-8e42-466d1db1893c",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/bin/bash: line 1: import: command not found\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "openai.api_key= \"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\""
      ],
      "metadata": {
        "id": "eyqOCaMKFzTR"
      },
      "execution_count": 23,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install langchain==0.0.189\n",
        "!pip install pinecone-client\n",
        "!pip install openai\n",
        "!pip install tiktoken\n",
        "!pip install nest_asyncio"
      ],
      "metadata": {
        "id": "CUA85uxVHBrV",
        "outputId": "038302d6-4a8e-4a6d-87af-08853a3532e1",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: nest_asyncio in /usr/local/lib/python3.10/dist-packages (1.5.7)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "os.environ[\"OPENAI_API_KEY\"] = \"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\""
      ],
      "metadata": {
        "id": "ay_PUfNaHGNq"
      },
      "execution_count": 28,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import openai\n",
        "from langchain.embeddings.openai import OpenAIEmbeddings\n",
        "embeddings = OpenAIEmbeddings(model_name=\"ada\")\n",
        "# from langchain.embeddings import SentenceTransformerEmbeddings\n",
        "# embeddings = SentenceTransformerEmbeddings(model_name=\"all-MiniLM-L6-v2\")"
      ],
      "metadata": {
        "id": "F5GY9voPa0av"
      },
      "execution_count": 29,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "query_result = embeddings.embed_query(\"Hello world\")\n",
        "len(query_result)"
      ],
      "metadata": {
        "id": "-s2p3rUwvOvW",
        "outputId": "339ff015-1d23-49be-f6b4-cde840237cf4",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "384"
            ]
          },
          "metadata": {},
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install pinecone-client -q"
      ],
      "metadata": {
        "id": "LXhIY5SrrRec",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "6bcfea21-b524-454d-b9b5-b34ea3b7bd0c"
      },
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[?25l     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.0/179.1 kB\u001b[0m \u001b[31m?\u001b[0m eta \u001b[36m-:--:--\u001b[0m\r\u001b[2K     \u001b[91m━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[90m╺\u001b[0m\u001b[90m━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m81.9/179.1 kB\u001b[0m \u001b[31m2.5 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\r\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m179.1/179.1 kB\u001b[0m \u001b[31m3.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m61.4/61.4 kB\u001b[0m \u001b[31m5.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m300.4/300.4 kB\u001b[0m \u001b[31m9.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "https://python.langchain.com/en/latest/modules/indexes/vectorstores/examples/pinecone.html"
      ],
      "metadata": {
        "id": "vySq5oI5sU5V"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import pinecone\n",
        "from langchain.vectorstores import Pinecone\n",
        "# initialize pinecone\n",
        "pinecone.init(\n",
        "    api_key=\"ec93067f-6dfe-4267-b89a-f8a327474d30\",  # find at app.pinecone.io\n",
        "    environment=\"gcp-starter\"  # next to api key in console\n",
        ")\n",
        "\n",
        "index_name = \"naqelweb\"\n",
        "\n",
        "index = Pinecone.from_documents(docs, embeddings, index_name=index_name)"
      ],
      "metadata": {
        "id": "hfIpYLV-acks"
      },
      "execution_count": 30,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def get_similiar_docs(query,k=1,score=False):\n",
        "  if score:\n",
        "    similar_docs = index.similarity_search_with_score(query,k=k)\n",
        "  else:\n",
        "    similar_docs = index.similarity_search(query,k=k)\n",
        "  return similar_docs\n",
        "\n",
        "query = \"How many days paid leave i can take\"\n",
        "similar_docs = get_similiar_docs(query)\n",
        "similar_docs\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "o5r7YLpbchAD",
        "outputId": "eab25430-09ee-4ede-9da4-7dcf1dcd094c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[Document(page_content='Employee shall be entitled to a full-pay annual leave of not less than twenty-one (21) days per each year of service. It is extendable to a minimum of thirty (30) days if the worker spends five consecutive years in service. Subject to the consent of the leave based establishment, on the period spent in service during the year. In addition, there may be agreed in the Employment Contract to increase such annual leave.\\n\\nthe worker may\\n\\ntake a part of his annual', metadata={'source': '/content/data/HRPolicyEnglish_1.pdf'})]"
            ]
          },
          "metadata": {},
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "How is India's economy?\n",
        "\n",
        "and its culture?\n",
        "\n",
        "its relationship with USA?"
      ],
      "metadata": {
        "id": "YVOoc9GXHjOG"
      }
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "wlkrKhY1PMXm"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}