Document_Based_QA_With_PineCone_Langchain.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "aa7d5073",
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: langchain in /home/mohammed_shaneeb/anaconda3/lib/python3.9/site-packages (0.0.168)\n",
      "Requirement already satisfied: openai in /home/mohammed_shaneeb/anaconda3/lib/python3.9/site-packages (0.27.6)\n",
      "Requirement already satisfied: PyYAML>=5.4.1 in /home/mohammed_shaneeb/anaconda3/lib/python3.9/site-packages (from langchain) (6.0)\n",
      "Requirement already satisfied: SQLAlchemy<3,>=1.4 in /home/mohammed_shaneeb/anaconda3/lib/python3.9/site-packages (from langchain) (1.4.39)\n",
      "Requirement already satisfied: aiohttp<4.0.0,>=3.8.3 in /home/mohammed_shaneeb/anaconda3/lib/python3.9/site-packages (from langchain) (3.8.3)\n",
      "Requirement already satisfied: async-timeout<5.0.0,>=4.0.0 in /home/mohammed_shaneeb/anaconda3/lib/python3.9/site-packages (from langchain) (4.0.2)\n",
      "Requirement already satisfied: dataclasses-json<0.6.0,>=0.5.7 in /home/mohammed_shaneeb/anaconda3/lib/python3.9/site-packages (from langchain) (0.5.7)\n",
      "Requirement already satisfied: numexpr<3.0.0,>=2.8.4 in /home/mohammed_shaneeb/anaconda3/lib/python3.9/site-packages (from langchain) (2.8.4)\n",
      "Requirement already satisfied: numpy<2,>=1 in /home/mohammed_shaneeb/anaconda3/lib/python3.9/site-packages (from langchain) (1.23.5)\n",
      "Requirement already satisfied: openapi-schema-pydantic<2.0,>=1.2 in /home/mohammed_shaneeb/anaconda3/lib/python3.9/site-packages (from langchain) (1.2.4)\n",
      "Requirement already satisfied: pydantic<2,>=1 in /home/mohammed_shaneeb/anaconda3/lib/python3.9/site-packages (from langchain) (1.10.12)\n",
      "Requirement already satisfied: requests<3,>=2 in /home/mohammed_shaneeb/anaconda3/lib/python3.9/site-packages (from langchain) (2.31.0)\n",
      "Requirement already satisfied: tenacity<9.0.0,>=8.1.0 in /home/mohammed_shaneeb/anaconda3/lib/python3.9/site-packages (from langchain) (8.2.2)\n",
      "Requirement already satisfied: tqdm in /home/mohammed_shaneeb/anaconda3/lib/python3.9/site-packages (from openai) (4.65.0)\n",
      "Requirement already satisfied: attrs>=17.3.0 in /home/mohammed_shaneeb/anaconda3/lib/python3.9/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (22.1.0)\n",
      "Requirement already satisfied: charset-normalizer<3.0,>=2.0 in /home/mohammed_shaneeb/anaconda3/lib/python3.9/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (2.0.4)\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in /home/mohammed_shaneeb/anaconda3/lib/python3.9/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (6.0.2)\n",
      "Requirement already satisfied: yarl<2.0,>=1.0 in /home/mohammed_shaneeb/anaconda3/lib/python3.9/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (1.8.1)\n",
      "Requirement already satisfied: frozenlist>=1.1.1 in /home/mohammed_shaneeb/anaconda3/lib/python3.9/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (1.3.3)\n",
      "Requirement already satisfied: aiosignal>=1.1.2 in /home/mohammed_shaneeb/anaconda3/lib/python3.9/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (1.2.0)\n",
      "Requirement already satisfied: marshmallow<4.0.0,>=3.3.0 in /home/mohammed_shaneeb/anaconda3/lib/python3.9/site-packages (from dataclasses-json<0.6.0,>=0.5.7->langchain) (3.19.0)\n",
      "Requirement already satisfied: marshmallow-enum<2.0.0,>=1.5.1 in /home/mohammed_shaneeb/anaconda3/lib/python3.9/site-packages (from dataclasses-json<0.6.0,>=0.5.7->langchain) (1.5.1)\n",
      "Requirement already satisfied: typing-inspect>=0.4.0 in /home/mohammed_shaneeb/anaconda3/lib/python3.9/site-packages (from dataclasses-json<0.6.0,>=0.5.7->langchain) (0.8.0)\n",
      "Requirement already satisfied: typing-extensions>=4.2.0 in /home/mohammed_shaneeb/anaconda3/lib/python3.9/site-packages (from pydantic<2,>=1->langchain) (4.7.1)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /home/mohammed_shaneeb/anaconda3/lib/python3.9/site-packages (from requests<3,>=2->langchain) (3.4)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /home/mohammed_shaneeb/anaconda3/lib/python3.9/site-packages (from requests<3,>=2->langchain) (1.26.16)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /home/mohammed_shaneeb/anaconda3/lib/python3.9/site-packages (from requests<3,>=2->langchain) (2023.7.22)\n",
      "Requirement already satisfied: greenlet!=0.4.17 in /home/mohammed_shaneeb/anaconda3/lib/python3.9/site-packages (from SQLAlchemy<3,>=1.4->langchain) (2.0.1)\n",
      "Requirement already satisfied: packaging>=17.0 in /home/mohammed_shaneeb/.local/lib/python3.9/site-packages (from marshmallow<4.0.0,>=3.3.0->dataclasses-json<0.6.0,>=0.5.7->langchain) (23.0)\n",
      "Requirement already satisfied: mypy-extensions>=0.3.0 in /home/mohammed_shaneeb/anaconda3/lib/python3.9/site-packages (from typing-inspect>=0.4.0->dataclasses-json<0.6.0,>=0.5.7->langchain) (0.4.3)\n"
     ]
    }
   ],
   "source": [
    "!pip install langchain openai"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "5d22db25",
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting unstructured[all-docs]\n",
      "  Obtaining dependency information for unstructured[all-docs] from https://files.pythonhosted.org/packages/e3/97/478e5f01e8922acc140ee35adde4c5e6861f7a693fe57daafe53947e6602/unstructured-0.10.8-py3-none-any.whl.metadata\n",
      "  Downloading unstructured-0.10.8-py3-none-any.whl.metadata (22 kB)\n",
      "Requirement already satisfied: chardet in /home/mohammed_shaneeb/anaconda3/lib/python3.9/site-packages (from unstructured[all-docs]) (4.0.0)\n",
      "Requirement already satisfied: filetype in /home/mohammed_shaneeb/anaconda3/lib/python3.9/site-packages (from unstructured[all-docs]) (1.2.0)\n",
      "Collecting python-magic (from unstructured[all-docs])\n",
      "  Downloading python_magic-0.4.27-py2.py3-none-any.whl (13 kB)\n",
      "Requirement already satisfied: lxml in /home/mohammed_shaneeb/anaconda3/lib/python3.9/site-packages (from unstructured[all-docs]) (4.9.1)\n",
      "Requirement already satisfied: nltk in /home/mohammed_shaneeb/anaconda3/lib/python3.9/site-packages (from unstructured[all-docs]) (3.5)\n",
      "Requirement already satisfied: tabulate in /home/mohammed_shaneeb/anaconda3/lib/python3.9/site-packages (from unstructured[all-docs]) (0.8.10)\n",
      "Requirement already satisfied: requests in /home/mohammed_shaneeb/anaconda3/lib/python3.9/site-packages (from unstructured[all-docs]) (2.31.0)\n",
      "Requirement already satisfied: beautifulsoup4 in /home/mohammed_shaneeb/anaconda3/lib/python3.9/site-packages (from unstructured[all-docs]) (4.12.2)\n",
      "Requirement already satisfied: emoji in /home/mohammed_shaneeb/anaconda3/lib/python3.9/site-packages (from unstructured[all-docs]) (2.4.0)\n",
      "Requirement already satisfied: pandas in /home/mohammed_shaneeb/anaconda3/lib/python3.9/site-packages (from unstructured[all-docs]) (1.5.3)\n",
      "Collecting python-pptx (from unstructured[all-docs])\n",
      "  Downloading python-pptx-0.6.21.tar.gz (10.1 MB)\n",
      "\u001b[2K     \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m10.1/10.1 MB\u001b[0m \u001b[31m542.5 kB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0mm eta \u001b[36m0:00:01\u001b[0m[36m0:00:01\u001b[0m\n",
      "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25hCollecting pdfminer.six (from unstructured[all-docs])\n",
      "  Using cached pdfminer.six-20221105-py3-none-any.whl (5.6 MB)\n",
      "Requirement already satisfied: markdown in /home/mohammed_shaneeb/anaconda3/lib/python3.9/site-packages (from unstructured[all-docs]) (3.4.1)\n",
      "Requirement already satisfied: Pillow<10 in /home/mohammed_shaneeb/anaconda3/lib/python3.9/site-packages (from unstructured[all-docs]) (7.2.0)\n",
      "Collecting xlrd (from unstructured[all-docs])\n",
      "  Downloading xlrd-2.0.1-py2.py3-none-any.whl (96 kB)\n",
      "\u001b[2K     \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m96.5/96.5 kB\u001b[0m \u001b[31m529.3 kB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m kB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m:01\u001b[0m\n",
      "\u001b[?25hCollecting ebooklib (from unstructured[all-docs])\n",
      "  Downloading EbookLib-0.18.tar.gz (115 kB)\n",
      "\u001b[2K     \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m115.5/115.5 kB\u001b[0m \u001b[31m339.1 kB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m kB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m:01\u001b[0m\n",
      "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25hCollecting python-docx (from unstructured[all-docs])\n",
      "  Downloading python-docx-0.8.11.tar.gz (5.6 MB)\n",
      "\u001b[2K     \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.6/5.6 MB\u001b[0m \u001b[31m255.6 kB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0mm eta \u001b[36m0:00:01\u001b[0m[36m0:00:01\u001b[0m\n",
      "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25hCollecting msg-parser (from unstructured[all-docs])\n",
      "  Downloading msg_parser-1.2.0-py2.py3-none-any.whl (101 kB)\n",
      "\u001b[2K     \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m101.8/101.8 kB\u001b[0m \u001b[31m711.2 kB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m kB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m:01\u001b[0m\n",
      "\u001b[?25hCollecting pypandoc (from unstructured[all-docs])\n",
      "  Downloading pypandoc-1.11-py3-none-any.whl (20 kB)\n",
      "Requirement already satisfied: openpyxl in /home/mohammed_shaneeb/anaconda3/lib/python3.9/site-packages (from unstructured[all-docs]) (3.0.10)\n",
      "Collecting pdf2image (from unstructured[all-docs])\n",
      "  Downloading pdf2image-1.16.3-py3-none-any.whl (11 kB)\n",
      "Collecting unstructured-inference (from unstructured[all-docs])\n",
      "  Obtaining dependency information for unstructured-inference from https://files.pythonhosted.org/packages/f8/18/6f4bb4901004b3e36ecc7b2e1e031a3de3779fea7aa15459e30d784bef1e/unstructured_inference-0.5.17-py3-none-any.whl.metadata\n",
      "  Downloading unstructured_inference-0.5.17-py3-none-any.whl.metadata (6.5 kB)\n",
      "Requirement already satisfied: soupsieve>1.2 in /home/mohammed_shaneeb/anaconda3/lib/python3.9/site-packages (from beautifulsoup4->unstructured[all-docs]) (2.4)\n",
      "Requirement already satisfied: six in /home/mohammed_shaneeb/.local/lib/python3.9/site-packages (from ebooklib->unstructured[all-docs]) (1.16.0)\n",
      "Requirement already satisfied: importlib-metadata>=4.4 in /home/mohammed_shaneeb/.local/lib/python3.9/site-packages (from markdown->unstructured[all-docs]) (6.0.0)\n",
      "Collecting olefile>=0.46 (from msg-parser->unstructured[all-docs])\n",
      "  Downloading olefile-0.46.zip (112 kB)\n",
      "\u001b[2K     \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m112.2/112.2 kB\u001b[0m \u001b[31m1.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m[31m1.0 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\n",
      "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25hRequirement already satisfied: click in /home/mohammed_shaneeb/anaconda3/lib/python3.9/site-packages (from nltk->unstructured[all-docs]) (8.1.6)\n",
      "Requirement already satisfied: joblib in /home/mohammed_shaneeb/anaconda3/lib/python3.9/site-packages (from nltk->unstructured[all-docs]) (1.2.0)\n",
      "Requirement already satisfied: regex in /home/mohammed_shaneeb/anaconda3/lib/python3.9/site-packages (from nltk->unstructured[all-docs]) (2022.7.9)\n",
      "Requirement already satisfied: tqdm in /home/mohammed_shaneeb/anaconda3/lib/python3.9/site-packages (from nltk->unstructured[all-docs]) (4.65.0)\n",
      "Requirement already satisfied: et_xmlfile in /home/mohammed_shaneeb/anaconda3/lib/python3.9/site-packages (from openpyxl->unstructured[all-docs]) (1.1.0)\n",
      "Requirement already satisfied: python-dateutil>=2.8.1 in /home/mohammed_shaneeb/anaconda3/lib/python3.9/site-packages (from pandas->unstructured[all-docs]) (2.8.1)\n",
      "Requirement already satisfied: pytz>=2020.1 in /home/mohammed_shaneeb/anaconda3/lib/python3.9/site-packages (from pandas->unstructured[all-docs]) (2022.7)\n",
      "Requirement already satisfied: numpy>=1.20.3 in /home/mohammed_shaneeb/anaconda3/lib/python3.9/site-packages (from pandas->unstructured[all-docs]) (1.23.5)\n",
      "Requirement already satisfied: charset-normalizer>=2.0.0 in /home/mohammed_shaneeb/anaconda3/lib/python3.9/site-packages (from pdfminer.six->unstructured[all-docs]) (2.0.4)\n",
      "Requirement already satisfied: cryptography>=36.0.0 in /home/mohammed_shaneeb/anaconda3/lib/python3.9/site-packages (from pdfminer.six->unstructured[all-docs]) (41.0.2)\n",
      "Collecting XlsxWriter>=0.5.7 (from python-pptx->unstructured[all-docs])\n",
      "  Obtaining dependency information for XlsxWriter>=0.5.7 from https://files.pythonhosted.org/packages/37/94/25d3ec8587974de7ebd790232aa3155abfe44ed23df7ccaa4645977a1cbe/XlsxWriter-3.1.2-py3-none-any.whl.metadata\n",
      "  Downloading XlsxWriter-3.1.2-py3-none-any.whl.metadata (2.5 kB)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /home/mohammed_shaneeb/anaconda3/lib/python3.9/site-packages (from requests->unstructured[all-docs]) (3.4)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /home/mohammed_shaneeb/anaconda3/lib/python3.9/site-packages (from requests->unstructured[all-docs]) (1.26.16)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /home/mohammed_shaneeb/anaconda3/lib/python3.9/site-packages (from requests->unstructured[all-docs]) (2023.7.22)\n",
      "Collecting layoutparser[layoutmodels,tesseract] (from unstructured-inference->unstructured[all-docs])\n",
      "  Downloading layoutparser-0.3.4-py3-none-any.whl (19.2 MB)\n",
      "\u001b[2K     \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m19.2/19.2 MB\u001b[0m \u001b[31m1.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0mm eta \u001b[36m0:00:01\u001b[0m[36m0:00:01\u001b[0mm\n",
      "\u001b[?25hRequirement already satisfied: python-multipart in /home/mohammed_shaneeb/anaconda3/lib/python3.9/site-packages (from unstructured-inference->unstructured[all-docs]) (0.0.6)\n",
      "Requirement already satisfied: huggingface-hub in /home/mohammed_shaneeb/anaconda3/lib/python3.9/site-packages (from unstructured-inference->unstructured[all-docs]) (0.15.1)\n",
      "Requirement already satisfied: opencv-python!=4.7.0.68 in /home/mohammed_shaneeb/anaconda3/lib/python3.9/site-packages (from unstructured-inference->unstructured[all-docs]) (4.7.0.72)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting onnxruntime (from unstructured-inference->unstructured[all-docs])\n",
      "  Obtaining dependency information for onnxruntime from https://files.pythonhosted.org/packages/50/d5/f156d808c9cc59e7a8b87843a5313719d01eec7d3b17ca3f6b6f0fbee0f0/onnxruntime-1.15.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata\n",
      "  Downloading onnxruntime-1.15.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (4.0 kB)\n",
      "Requirement already satisfied: transformers>=4.25.1 in /home/mohammed_shaneeb/anaconda3/lib/python3.9/site-packages (from unstructured-inference->unstructured[all-docs]) (4.29.2)\n",
      "Requirement already satisfied: cffi>=1.12 in /home/mohammed_shaneeb/anaconda3/lib/python3.9/site-packages (from cryptography>=36.0.0->pdfminer.six->unstructured[all-docs]) (1.15.1)\n",
      "Requirement already satisfied: zipp>=0.5 in /home/mohammed_shaneeb/.local/lib/python3.9/site-packages (from importlib-metadata>=4.4->markdown->unstructured[all-docs]) (3.15.0)\n",
      "Requirement already satisfied: filelock in /home/mohammed_shaneeb/anaconda3/lib/python3.9/site-packages (from transformers>=4.25.1->unstructured-inference->unstructured[all-docs]) (3.9.0)\n",
      "Requirement already satisfied: packaging>=20.0 in /home/mohammed_shaneeb/.local/lib/python3.9/site-packages (from transformers>=4.25.1->unstructured-inference->unstructured[all-docs]) (23.0)\n",
      "Requirement already satisfied: pyyaml>=5.1 in /home/mohammed_shaneeb/anaconda3/lib/python3.9/site-packages (from transformers>=4.25.1->unstructured-inference->unstructured[all-docs]) (6.0)\n",
      "Requirement already satisfied: tokenizers!=0.11.3,<0.14,>=0.11.1 in /home/mohammed_shaneeb/anaconda3/lib/python3.9/site-packages (from transformers>=4.25.1->unstructured-inference->unstructured[all-docs]) (0.13.2)\n",
      "Requirement already satisfied: fsspec in /home/mohammed_shaneeb/anaconda3/lib/python3.9/site-packages (from huggingface-hub->unstructured-inference->unstructured[all-docs]) (2023.4.0)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in /home/mohammed_shaneeb/anaconda3/lib/python3.9/site-packages (from huggingface-hub->unstructured-inference->unstructured[all-docs]) (4.7.1)\n",
      "Requirement already satisfied: scipy in /home/mohammed_shaneeb/anaconda3/lib/python3.9/site-packages (from layoutparser[layoutmodels,tesseract]->unstructured-inference->unstructured[all-docs]) (1.10.1)\n",
      "Collecting iopath (from layoutparser[layoutmodels,tesseract]->unstructured-inference->unstructured[all-docs])\n",
      "  Downloading iopath-0.1.10.tar.gz (42 kB)\n",
      "\u001b[2K     \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m42.2/42.2 kB\u001b[0m \u001b[31m1.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25hCollecting pdfplumber (from layoutparser[layoutmodels,tesseract]->unstructured-inference->unstructured[all-docs])\n",
      "  Obtaining dependency information for pdfplumber from https://files.pythonhosted.org/packages/c1/f9/1a41afffe5e7a98ab9b6a6dd3dab9d99b677fae2536f676397c4506f6554/pdfplumber-0.10.2-py3-none-any.whl.metadata\n",
      "  Downloading pdfplumber-0.10.2-py3-none-any.whl.metadata (35 kB)\n",
      "Collecting pytesseract (from layoutparser[layoutmodels,tesseract]->unstructured-inference->unstructured[all-docs])\n",
      "  Using cached pytesseract-0.3.10-py3-none-any.whl (14 kB)\n",
      "Requirement already satisfied: torch in /home/mohammed_shaneeb/anaconda3/lib/python3.9/site-packages (from layoutparser[layoutmodels,tesseract]->unstructured-inference->unstructured[all-docs]) (2.0.1)\n",
      "Requirement already satisfied: torchvision in /home/mohammed_shaneeb/anaconda3/lib/python3.9/site-packages (from layoutparser[layoutmodels,tesseract]->unstructured-inference->unstructured[all-docs]) (0.15.2a0)\n",
      "Collecting effdet (from layoutparser[layoutmodels,tesseract]->unstructured-inference->unstructured[all-docs])\n",
      "  Obtaining dependency information for effdet from https://files.pythonhosted.org/packages/9c/13/563119fe0af82aca5a3b89399c435953072c39515c2e818eb82793955c3b/effdet-0.4.1-py3-none-any.whl.metadata\n",
      "  Downloading effdet-0.4.1-py3-none-any.whl.metadata (33 kB)\n",
      "Collecting coloredlogs (from onnxruntime->unstructured-inference->unstructured[all-docs])\n",
      "  Downloading coloredlogs-15.0.1-py2.py3-none-any.whl (46 kB)\n",
      "\u001b[2K     \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m46.0/46.0 kB\u001b[0m \u001b[31m2.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: flatbuffers in /home/mohammed_shaneeb/anaconda3/lib/python3.9/site-packages (from onnxruntime->unstructured-inference->unstructured[all-docs]) (23.5.9)\n",
      "Requirement already satisfied: protobuf in /home/mohammed_shaneeb/anaconda3/lib/python3.9/site-packages (from onnxruntime->unstructured-inference->unstructured[all-docs]) (4.21.12)\n",
      "Requirement already satisfied: sympy in /home/mohammed_shaneeb/anaconda3/lib/python3.9/site-packages (from onnxruntime->unstructured-inference->unstructured[all-docs]) (1.11.1)\n",
      "Requirement already satisfied: pycparser in /home/mohammed_shaneeb/anaconda3/lib/python3.9/site-packages (from cffi>=1.12->cryptography>=36.0.0->pdfminer.six->unstructured[all-docs]) (2.21)\n",
      "Collecting humanfriendly>=9.1 (from coloredlogs->onnxruntime->unstructured-inference->unstructured[all-docs])\n",
      "  Downloading humanfriendly-10.0-py2.py3-none-any.whl (86 kB)\n",
      "\u001b[2K     \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m86.8/86.8 kB\u001b[0m \u001b[31m2.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m[31m2.6 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\n",
      "\u001b[?25hCollecting timm>=0.9.2 (from effdet->layoutparser[layoutmodels,tesseract]->unstructured-inference->unstructured[all-docs])\n",
      "  Obtaining dependency information for timm>=0.9.2 from https://files.pythonhosted.org/packages/14/38/05b37b7692e521bbada22593ac3b6d7ba3f378d56b5d1ccb322a541bbb6e/timm-0.9.5-py3-none-any.whl.metadata\n",
      "  Downloading timm-0.9.5-py3-none-any.whl.metadata (69 kB)\n",
      "\u001b[2K     \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m69.4/69.4 kB\u001b[0m \u001b[31m1.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\n",
      "\u001b[?25hCollecting pycocotools>=2.0.2 (from effdet->layoutparser[layoutmodels,tesseract]->unstructured-inference->unstructured[all-docs])\n",
      "  Obtaining dependency information for pycocotools>=2.0.2 from https://files.pythonhosted.org/packages/fe/83/ae272705f1b2e4efe0cd2e7e2de0233386ed578a2356c6409e3db268e2db/pycocotools-2.0.7-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata\n",
      "  Downloading pycocotools-2.0.7-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (1.1 kB)\n",
      "Collecting omegaconf>=2.0 (from effdet->layoutparser[layoutmodels,tesseract]->unstructured-inference->unstructured[all-docs])\n",
      "  Downloading omegaconf-2.3.0-py3-none-any.whl (79 kB)\n",
      "\u001b[2K     \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m79.5/79.5 kB\u001b[0m \u001b[31m2.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m[31m2.2 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: networkx in /home/mohammed_shaneeb/anaconda3/lib/python3.9/site-packages (from torch->layoutparser[layoutmodels,tesseract]->unstructured-inference->unstructured[all-docs]) (3.1)\n",
      "Requirement already satisfied: jinja2 in /home/mohammed_shaneeb/anaconda3/lib/python3.9/site-packages (from torch->layoutparser[layoutmodels,tesseract]->unstructured-inference->unstructured[all-docs]) (3.1.2)\n",
      "Collecting portalocker (from iopath->layoutparser[layoutmodels,tesseract]->unstructured-inference->unstructured[all-docs])\n",
      "  Using cached portalocker-2.7.0-py2.py3-none-any.whl (15 kB)\n",
      "Collecting Pillow<10 (from unstructured[all-docs])\n",
      "  Using cached Pillow-9.5.0-cp39-cp39-manylinux_2_28_x86_64.whl (3.4 MB)\n",
      "Collecting pypdfium2>=4.18.0 (from pdfplumber->layoutparser[layoutmodels,tesseract]->unstructured-inference->unstructured[all-docs])\n",
      "  Obtaining dependency information for pypdfium2>=4.18.0 from https://files.pythonhosted.org/packages/cc/bd/2052075f453728535cc782bcf0fd3794149c516caf2de240060f52308c4b/pypdfium2-4.19.0-py3-none-manylinux_2_17_x86_64.whl.metadata\n",
      "  Downloading pypdfium2-4.19.0-py3-none-manylinux_2_17_x86_64.whl.metadata (41 kB)\n",
      "\u001b[2K     \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m41.3/41.3 kB\u001b[0m \u001b[31m2.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: mpmath>=0.19 in /home/mohammed_shaneeb/anaconda3/lib/python3.9/site-packages (from sympy->onnxruntime->unstructured-inference->unstructured[all-docs]) (1.3.0)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting antlr4-python3-runtime==4.9.* (from omegaconf>=2.0->effdet->layoutparser[layoutmodels,tesseract]->unstructured-inference->unstructured[all-docs])\n",
      "  Downloading antlr4-python3-runtime-4.9.3.tar.gz (117 kB)\n",
      "\u001b[2K     \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m117.0/117.0 kB\u001b[0m \u001b[31m1.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\n",
      "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25hRequirement already satisfied: matplotlib>=2.1.0 in /home/mohammed_shaneeb/anaconda3/lib/python3.9/site-packages (from pycocotools>=2.0.2->effdet->layoutparser[layoutmodels,tesseract]->unstructured-inference->unstructured[all-docs]) (3.7.1)\n",
      "Requirement already satisfied: safetensors in /home/mohammed_shaneeb/anaconda3/lib/python3.9/site-packages (from timm>=0.9.2->effdet->layoutparser[layoutmodels,tesseract]->unstructured-inference->unstructured[all-docs]) (0.3.1)\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in /home/mohammed_shaneeb/anaconda3/lib/python3.9/site-packages (from jinja2->torch->layoutparser[layoutmodels,tesseract]->unstructured-inference->unstructured[all-docs]) (2.1.1)\n",
      "Requirement already satisfied: contourpy>=1.0.1 in /home/mohammed_shaneeb/anaconda3/lib/python3.9/site-packages (from matplotlib>=2.1.0->pycocotools>=2.0.2->effdet->layoutparser[layoutmodels,tesseract]->unstructured-inference->unstructured[all-docs]) (1.0.5)\n",
      "Requirement already satisfied: cycler>=0.10 in /home/mohammed_shaneeb/anaconda3/lib/python3.9/site-packages (from matplotlib>=2.1.0->pycocotools>=2.0.2->effdet->layoutparser[layoutmodels,tesseract]->unstructured-inference->unstructured[all-docs]) (0.11.0)\n",
      "Requirement already satisfied: fonttools>=4.22.0 in /home/mohammed_shaneeb/anaconda3/lib/python3.9/site-packages (from matplotlib>=2.1.0->pycocotools>=2.0.2->effdet->layoutparser[layoutmodels,tesseract]->unstructured-inference->unstructured[all-docs]) (4.25.0)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in /home/mohammed_shaneeb/anaconda3/lib/python3.9/site-packages (from matplotlib>=2.1.0->pycocotools>=2.0.2->effdet->layoutparser[layoutmodels,tesseract]->unstructured-inference->unstructured[all-docs]) (1.4.4)\n",
      "Requirement already satisfied: pyparsing>=2.3.1 in /home/mohammed_shaneeb/anaconda3/lib/python3.9/site-packages (from matplotlib>=2.1.0->pycocotools>=2.0.2->effdet->layoutparser[layoutmodels,tesseract]->unstructured-inference->unstructured[all-docs]) (3.0.9)\n",
      "Requirement already satisfied: importlib-resources>=3.2.0 in /home/mohammed_shaneeb/anaconda3/lib/python3.9/site-packages (from matplotlib>=2.1.0->pycocotools>=2.0.2->effdet->layoutparser[layoutmodels,tesseract]->unstructured-inference->unstructured[all-docs]) (5.2.0)\n",
      "Downloading unstructured-0.10.8-py3-none-any.whl (1.5 MB)\n",
      "\u001b[2K   \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.5/1.5 MB\u001b[0m \u001b[31m2.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0mm eta \u001b[36m0:00:01\u001b[0m0:01\u001b[0m:01\u001b[0m\n",
      "\u001b[?25hDownloading unstructured_inference-0.5.17-py3-none-any.whl (46 kB)\n",
      "\u001b[2K   \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m46.3/46.3 kB\u001b[0m \u001b[31m954.5 kB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0mB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\n",
      "\u001b[?25hDownloading XlsxWriter-3.1.2-py3-none-any.whl (153 kB)\n",
      "\u001b[2K   \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m153.0/153.0 kB\u001b[0m \u001b[31m1.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\n",
      "\u001b[?25hDownloading onnxruntime-1.15.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (5.9 MB)\n",
      "\u001b[2K   \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.9/5.9 MB\u001b[0m \u001b[31m2.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0mm eta \u001b[36m0:00:01\u001b[0m[36m0:00:01\u001b[0m\n",
      "\u001b[?25hDownloading effdet-0.4.1-py3-none-any.whl (112 kB)\n",
      "\u001b[2K   \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m112.5/112.5 kB\u001b[0m \u001b[31m2.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m31m2.7 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\n",
      "\u001b[?25hDownloading pdfplumber-0.10.2-py3-none-any.whl (47 kB)\n",
      "\u001b[2K   \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m47.5/47.5 kB\u001b[0m \u001b[31m2.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading pycocotools-2.0.7-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (435 kB)\n",
      "\u001b[2K   \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m435.1/435.1 kB\u001b[0m \u001b[31m2.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m[31m2.4 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\n",
      "\u001b[?25hDownloading pypdfium2-4.19.0-py3-none-manylinux_2_17_x86_64.whl (3.0 MB)\n",
      "\u001b[2K   \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.0/3.0 MB\u001b[0m \u001b[31m538.2 kB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0mm eta \u001b[36m0:00:01\u001b[0m[36m0:00:01\u001b[0m\n",
      "\u001b[?25hDownloading timm-0.9.5-py3-none-any.whl (2.2 MB)\n",
      "\u001b[2K   \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.2/2.2 MB\u001b[0m \u001b[31m1.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0mm eta \u001b[36m0:00:01\u001b[0m[36m0:00:01\u001b[0m0m\n",
      "\u001b[?25hBuilding wheels for collected packages: ebooklib, python-docx, python-pptx, olefile, iopath, antlr4-python3-runtime\n",
      "  Building wheel for ebooklib (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for ebooklib: filename=EbookLib-0.18-py3-none-any.whl size=38777 sha256=94cbb36df75d2b8926fd8efa379de4c56becf1214af79f8dba8a8d9d5f72c0d6\n",
      "  Stored in directory: /home/mohammed_shaneeb/.cache/pip/wheels/47/3a/ec/289c2f96d54695a17d260684be304d20a8d0bf50b08b75862e\n",
      "  Building wheel for python-docx (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for python-docx: filename=python_docx-0.8.11-py3-none-any.whl size=184490 sha256=15bd82d2118310ea7d36074f952545cd0b617deb11e258ed6186ef9dfb92a98e\n",
      "  Stored in directory: /home/mohammed_shaneeb/.cache/pip/wheels/83/8b/7c/09ae60c42c7ba4ed2dddaf2b8b9186cb105255856d6ed3dba5\n",
      "  Building wheel for python-pptx (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for python-pptx: filename=python_pptx-0.6.21-py3-none-any.whl size=470935 sha256=9013aeb4be477f72a7a27d19ccf95e91a89adbf35b1c2336692aa9c3c9bf0894\n",
      "  Stored in directory: /home/mohammed_shaneeb/.cache/pip/wheels/0e/4a/ed/9653bc799915f52dce3f04d14946fbd85cce9c3cdedc9cfa71\n",
      "  Building wheel for olefile (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for olefile: filename=olefile-0.46-py2.py3-none-any.whl size=35417 sha256=80b426c55e9ea907613677666df0d1aaf16813b511c52bfaced0bebcac0f7b77\n",
      "  Stored in directory: /home/mohammed_shaneeb/.cache/pip/wheels/64/b8/ba/ebba30390fbd997074f35e42a842ce3fd933213cac8753414e\n",
      "  Building wheel for iopath (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for iopath: filename=iopath-0.1.10-py3-none-any.whl size=31532 sha256=cf33978cd93e3e3867f0dbb9918a18220c7d6dd36c4a8286292584dc88586027\n",
      "  Stored in directory: /home/mohammed_shaneeb/.cache/pip/wheels/c1/13/6d/441d8f2af76ee6d2a3e67eebb1d0c556fefcee0a8b32266a8e\n",
      "  Building wheel for antlr4-python3-runtime (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for antlr4-python3-runtime: filename=antlr4_python3_runtime-4.9.3-py3-none-any.whl size=144554 sha256=24e5f6e795ed185dc4ae806abc3b924ab34454b29c60ec5434147cb1421e0049\n",
      "  Stored in directory: /home/mohammed_shaneeb/.cache/pip/wheels/23/cf/80/f3efa822e6ab23277902ee9165fe772eeb1dfb8014f359020a\n",
      "Successfully built ebooklib python-docx python-pptx olefile iopath antlr4-python3-runtime\n",
      "Installing collected packages: antlr4-python3-runtime, XlsxWriter, xlrd, python-magic, python-docx, pypdfium2, pypandoc, portalocker, Pillow, omegaconf, olefile, humanfriendly, ebooklib, unstructured, python-pptx, pytesseract, pdf2image, msg-parser, iopath, coloredlogs, pycocotools, pdfminer.six, onnxruntime, timm, pdfplumber, layoutparser, effdet, unstructured-inference\n",
      "  Attempting uninstall: Pillow\n",
      "    Found existing installation: Pillow 7.2.0\n",
      "    Uninstalling Pillow-7.2.0:\n",
      "      Successfully uninstalled Pillow-7.2.0\n",
      "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
      "label-studio-converter 0.0.30 requires Pillow<8,>=7, but you have pillow 9.5.0 which is incompatible.\n",
      "pandasai 0.8.2 requires ipython<9.0.0,>=8.13.1, but you have ipython 8.12.0 which is incompatible.\u001b[0m\u001b[31m\n",
      "\u001b[0mSuccessfully installed Pillow-9.4.0 XlsxWriter-3.1.2 antlr4-python3-runtime-4.9.3 coloredlogs-15.0.1 ebooklib-0.18 effdet-0.4.1 humanfriendly-10.0 iopath-0.1.10 layoutparser-0.3.4 msg-parser-1.2.0 olefile-0.46 omegaconf-2.3.0 onnxruntime-1.15.1 pdf2image-1.16.3 pdfminer.six-20221105 pdfplumber-0.10.2 portalocker-2.7.0 pycocotools-2.0.7 pypandoc-1.11 pypdfium2-4.19.0 pytesseract-0.3.10 python-docx-0.8.11 python-magic-0.4.27 python-pptx-0.6.21 timm-0.9.5 unstructured-0.10.8 unstructured-inference-0.5.17 xlrd-2.0.1\n"
     ]
    }
   ],
   "source": [
    "!pip install \"unstructured[all-docs]\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "c6a1fd1f",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[sudo] password for mohammed_shaneeb: \n"
     ]
    }
   ],
   "source": [
    "!apt-get install poppler-utils"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8810d11b",
   "metadata": {},
   "source": [
    "## OPENAI configuration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "42d2e05a",
   "metadata": {},
   "outputs": [],
   "source": [
    "from dotenv import load_dotenv\n",
    "import os\n",
    "\n",
    "load_dotenv()\n",
    "\n",
    "OPENAI_API_KEY = os.environ.get('OPENAI_API_KEY')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5aa62dec",
   "metadata": {},
   "source": [
    "## Loading Documents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "d1bc0c28",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from langchain.document_loaders import DirectoryLoader\n",
    "\n",
    "directory ='data'\n",
    "\n",
    "def load_docs(directory):\n",
    "    loader = DirectoryLoader(directory)\n",
    "    documents = loader.load()\n",
    "    return  documents\n",
    "\n",
    "documents = load_docs(directory)\n",
    "len(documents)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ec8dc620",
   "metadata": {},
   "source": [
    "## Splitting Texts into Chunks\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "378f74e4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "64"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
    "\n",
    "def split_doc(documents,chunk_size=1000,chunk_overlap=20):\n",
    "    text_splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap)\n",
    "    docs = text_splitter.split_documents(documents)\n",
    "    return docs\n",
    "\n",
    "docs = split_doc(documents)\n",
    "len(docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "3829d8d4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "page_content='1\\n\\nAdda247 Publications For More Study Material Visit: adda247.com\\n\\nChandrayaan-3\\n\\nChandrayaan 3: All you need to know about India’s lunar mission\\n\\nThe Indian Space Research Organisation (ISRO) has made significant progress in its ambitious Chandrayaan-3 mission, which aims to demonstrate the capability of a soft landing on the Moon. Following the setback of the Chandrayaan-2 mission, ISRO has diligently worked towards rectifying the issues and is now preparing for the launch of Chandrayaan-3 on July 14.\\n\\nObjectives: Demonstrating Soft Landing and Scientific Exploration The primary objective of the Chandrayaan-3 mission is to successfully achieve a soft landing on the Moon’s surface. By accomplishing this feat, India aims to join the exclusive group of nations that have successfully landed on the Moon. The mission also intends to conduct chemical analyses, gather valuable scientific data, and contribute to advancements in lunar research.' metadata={'source': 'data/Chandrayaan-3-E-book.pdf'}\n"
     ]
    }
   ],
   "source": [
    "print(docs[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "959371ad",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GOVERNMENT OF INDIA\n",
      "\n",
      "BUDGET 2023-2024\n",
      "\n",
      "SPEECH\n",
      "\n",
      "OF NIRMALA SITHARAMAN MINISTER OF FINANCE\n",
      "\n",
      "February 1, 2023\n",
      "\n",
      "CONTENTS\n",
      "\n",
      "PART-A\n",
      "\n",
      " Achievements since 2014: Leaving no one behind  Vision for Amrit Kaal – an empowered and inclusive economy \n",
      "\n",
      "Priorities of this Budget i. Inclusive Development ii. Reaching the Last Mile iii. iv. Unleashing the Potential v. Green Growth vi. Youth Power vii. Financial Sector\n",
      "\n",
      "Introduction\n",
      "\n",
      "Infrastructure and Investment\n",
      "\n",
      "Page No. 1 2 3 5\n",
      "\n",
      "\n",
      "\n",
      "Fiscal Management\n",
      "\n",
      "24\n",
      "\n",
      "PART B\n",
      "\n",
      "Indirect Taxes\n",
      "\n",
      "27\n",
      "\n",
      "Green Mobility  Electronics  Electrical  Chemicals and Petrochemicals  Marine products    Metals  Compounded Rubber  Cigarettes\n",
      "\n",
      "Lab Grown Diamonds Precious Metals\n",
      "\n",
      "Direct Taxes\n",
      "\n",
      "30\n",
      "\n",
      "MSMEs and Professionals  Cooperation  Start-Ups  Appeals  Better targeting of tax concessions  Rationalisation  Others \n",
      "\n",
      "Personal Income Tax\n",
      "\n",
      "Annexures\n",
      "\n",
      "35\n",
      "\n",
      "Annexure to Part B of the Budget Speech 2023-24\n"
     ]
    }
   ],
   "source": [
    "print(docs[0].page_content)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "25350f76",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Chandrayaan-3, building upon Chandrayaan-2, aims to demonstrate advanced lunar exploration capabilities. Consisting of a Lander and Rover configuration, this mission endeavors to uncover the Moon’s secrets through scientific payloads and cutting-edge technology.\n",
      "\n",
      "Lander Module:\n",
      "\n",
      "The indigenous Lander module is the centerpiece of Chandrayaan-3. • Designed for soft landing, it carries scientific payloads to study lunar phenomena. • Propulsion Module (PM) carries the Lander and Rover configuration to a 100 km lunar orbit. • PM features the Spectro-polarimetry of Habitable Planet Earth (SHAPE) payload, analyzing Earth’s spectral and\n",
      "\n",
      "polarimetric measurements from lunar orbit.\n",
      "\n",
      "Lander Payloads:\n",
      "\n",
      "Chandra’s Surface Thermophysical Experiment (ChaSTE): Measures thermal conductivity and temperature. •\n",
      "\n",
      "Instrument for Lunar Seismic Activity (ILSA): Gauges seismicity around the landing site, probing lunar crust and mantle.\n"
     ]
    }
   ],
   "source": [
    "print(docs[52].page_content)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "570ce328",
   "metadata": {},
   "source": [
    "## Embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "1d0a4451",
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install tiktoken -q"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "5463746d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1536\n",
      "[-0.012789793312549591, -0.01759009249508381, -0.00378015311434865, -0.009547775611281395, -0.02362513169646263]\n"
     ]
    }
   ],
   "source": [
    "import openai\n",
    "from langchain.embeddings.openai import OpenAIEmbeddings\n",
    "\n",
    "embeddings = OpenAIEmbeddings()\n",
    "text = \"Mohammed Shaneeb\"\n",
    "query_result = embeddings.embed_query(text)\n",
    "print(len(query_result))\n",
    "print(query_result[:5])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "63717c63",
   "metadata": {},
   "source": [
    "## Vecotor DB PineCone"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "77961dea",
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install pinecone-client -q"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "e0c7f284",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: tqdm in /home/mohammed_shaneeb/anaconda3/lib/python3.9/site-packages (4.65.0)\r\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_28989/2009859399.py:2: TqdmExperimentalWarning: Using `tqdm.autonotebook.tqdm` in notebook mode. Use `tqdm.tqdm` instead to force console mode (e.g. in jupyter console)\n",
      "  from tqdm.autonotebook import tqdm\n"
     ]
    }
   ],
   "source": [
    "!pip install tqdm\n",
    "from tqdm.autonotebook import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "3aba02d4",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/mohammed_shaneeb/anaconda3/lib/python3.9/site-packages/pinecone/index.py:4: TqdmExperimentalWarning: Using `tqdm.autonotebook.tqdm` in notebook mode. Use `tqdm.tqdm` instead to force console mode (e.g. in jupyter console)\n",
      "  from tqdm.autonotebook import tqdm\n"
     ]
    }
   ],
   "source": [
    "import pinecone\n",
    "from langchain.vectorstores import Pinecone\n",
    "\n",
    "PineCone_API_KEY = os.environ.get('PineCone_API_KEY')\n",
    "pinecone.init(\n",
    "    api_key = PineCone_API_KEY,\n",
    "    environment = \"gcp-starter\"\n",
    ")\n",
    "index_name = 'qa-langchain'\n",
    "\n",
    "index = Pinecone.from_documents(docs,embeddings,index_name=index_name)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "eb075401",
   "metadata": {},
   "source": [
    "## Get Similar Docs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "7d2168c9",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_similar_docs(query,k=2,score=False):\n",
    "    if score:\n",
    "        similar_docs = index.similarity_search_with_score(query,k=k)\n",
    "    else:\n",
    "        similar_docs = index.similarity_search(query,k)\n",
    "    return similar_docs\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "7b29e35d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(Document(page_content='genesis and structure. Insights into overarching mechanisms influencing Solar System formation and development can be gleaned.\\n\\n17\\n\\nAdda247 Publications For More Study Material Visit: adda247.com\\n\\nFuture Lunar Bases Potential:\\n\\nProximity of water ice near the south pole makes it an attractive site for establishing lunar bases. • Water extraction and refinement offer crucial supplies for extended missions and sustaining human presence. • Permanent shadow areas mitigate temperature fluctuations, making them potential sites for habitat construction.\\n\\nAstronomical Observations:\\n\\nMoon’s south pole provides a pristine, elevated position for unobstructed astronomical observations. • Earth’s atmospheric interference and light pollution are minimized, enabling in-depth space exploration. • Astronomers can study remote galaxies, cosmic background radiation, and transient celestial events effectively.\\n\\nExploring the Moon’s South Pole: Temperature, Range, and Area', metadata={'source': 'data/Chandrayaan-3-E-book.pdf'}),\n",
       "  0.885922551),\n",
       " (Document(page_content='Accumulated over billions of years due to extreme cold and limited sunlight, these ice deposits are valuable resources\\n\\nfor future human missions.\\n\\nWater ice offers applications like drinking water, fuel production, and life support systems, making this region vital for\\n\\npotential lunar habitats.\\n\\nExploring Lunar History: Limited sunlight in the south pole maintains the surface’s pristine state, preserving the Moon’s historical record.\\n\\nGeological features, rock structures, and impact craters here provide insights into the Moon’s origin, development, and\\n\\n\\n\\ngeological transformations. Scientists can extract valuable information about the Moon’s evolution spanning billions of years.\\n\\nSolar System Evolution:\\n\\nPolar craters have captured substances from the Solar System’s initial phases, including comets and asteroids. • The Moon’s south pole becomes a treasure trove for studying these cosmic entities, enhancing understanding of their', metadata={'source': 'data/Chandrayaan-3-E-book.pdf'}),\n",
       "  0.837518632),\n",
       " (Document(page_content='Chandrayaan-3 to Join Growing List of Lunar Missions The Chandrayaan-3 landing will be the latest in a series of lunar missions in recent years. In 2019, China landed the Chang’e 4 probe on the far side of the Moon. In 2021, the United States landed the Perseverance rover on Mars, which is currently exploring the Jezero crater. The Chandrayaan-3 landing will also coincide with the arrival of Russia’s Luna-25 mission, which is scheduled to land on the Moon’s south pole in August 2023. The growing number of lunar missions is a sign of the growing interest in lunar exploration. The Moon is a potentially valuable resource for future human exploration, and it is also a key stepping stone to Mars.\\n\\n4\\n\\nAdda247 Publications For More Study Material Visit: adda247.com', metadata={'source': 'data/Chandrayaan-3-E-book.pdf'}),\n",
       "  0.83739996),\n",
       " (Document(page_content='The area around the Moon’s south pole is characterized by both illuminated and shadowed regions. • The continuous shadow areas are of particular interest due to their potential for water ice accumulation. • The presence of craters and rugged terrain poses challenges for landing and exploration.\\n\\nScientific Significance:\\n\\n\\n\\nInvestigating the temperature variations and ice presence contributes to understanding lunar geology and surface processes.\\n\\n18\\n\\nAdda247 Publications For More Study Material Visit: adda247.com\\n\\nThe unique lighting conditions in the polar regions enable studying ancient impact craters and preserving historical\\n\\nrecords.\\n\\nThese studies aid in unraveling the Moon’s geological history, Solar System evolution, and potential resource utilization.\\n\\nChandrayaan-3 Mission Components: Unlocking Lunar Mysteries', metadata={'source': 'data/Chandrayaan-3-E-book.pdf'}),\n",
       "  0.824481905),\n",
       " (Document(page_content='Chandrayaan-3, building upon Chandrayaan-2, aims to demonstrate advanced lunar exploration capabilities. Consisting of a Lander and Rover configuration, this mission endeavors to uncover the Moon’s secrets through scientific payloads and cutting-edge technology.\\n\\nLander Module:\\n\\nThe indigenous Lander module is the centerpiece of Chandrayaan-3. • Designed for soft landing, it carries scientific payloads to study lunar phenomena. • Propulsion Module (PM) carries the Lander and Rover configuration to a 100 km lunar orbit. • PM features the Spectro-polarimetry of Habitable Planet Earth (SHAPE) payload, analyzing Earth’s spectral and\\n\\npolarimetric measurements from lunar orbit.\\n\\nLander Payloads:\\n\\nChandra’s Surface Thermophysical Experiment (ChaSTE): Measures thermal conductivity and temperature. •\\n\\nInstrument for Lunar Seismic Activity (ILSA): Gauges seismicity around the landing site, probing lunar crust and mantle.', metadata={'source': 'data/Chandrayaan-3-E-book.pdf'}),\n",
       "  0.821183622)]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "query = 'Future Lunar Bases Potential'\n",
    "similar_docs = get_similar_docs(query,score=True,k=5)\n",
    "similar_docs"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1b6c6209",
   "metadata": {},
   "source": [
    "## LLM model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "c6302e87",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/mohammed_shaneeb/anaconda3/lib/python3.9/site-packages/langchain/llms/openai.py:169: UserWarning: You are trying to use a chat model. This way of initializing it is no longer supported. Instead, please use: `from langchain.chat_models import ChatOpenAI`\n",
      "  warnings.warn(\n",
      "/home/mohammed_shaneeb/anaconda3/lib/python3.9/site-packages/langchain/llms/openai.py:696: UserWarning: You are trying to use a chat model. This way of initializing it is no longer supported. Instead, please use: `from langchain.chat_models import ChatOpenAI`\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "from langchain.llms import OpenAI\n",
    "model_name = \"gpt-3.5-turbo\"\n",
    "llm = OpenAI(model_name=model_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "e2293ed3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "- The area around the Moon's south pole is characterized by both illuminated and shadowed regions.\n",
      "- It spans a substantial range of latitude, approximately 70 degrees south.\n",
      "- The continuous shadow areas in the south pole are of particular interest due to their potential for water ice accumulation.\n",
      "- The presence of craters and rugged terrain poses challenges for landing and exploration.\n"
     ]
    }
   ],
   "source": [
    "from langchain.chains.question_answering import load_qa_chain\n",
    "chain = load_qa_chain(llm, chain_type='stuff')\n",
    "\n",
    "def get_answer(query):\n",
    "    similar_docs = get_similar_docs(query)\n",
    "    answer = chain.run(input_documents=similar_docs,question=query)\n",
    "    return answer\n",
    "\n",
    "query = 'what are the Characteristics of the South Pole Area. give me answer in points'\n",
    "print(get_answer(query))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "e59046e3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'S Unnikrishnan Nair is the director of the Vikram Sarabhai Space Centre (VSSC) located in Thumba, Thiruvananthapuram district, Kerala. He is responsible for leading the key functions of the centre and overseeing crucial missions. The VSSC is known for developing the Geosynchronous Satellite Launch Vehicle (GSLV) Mark-III, which was later renamed as Launch Vehicle Mark-III.'"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "query = \"who is S Unnikrishnan Nair. give me some information about him\"\n",
    "get_answer(query)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.17"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}