store_embeddings_in_pinecone.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Loading Text, Chunking, Embedding and Upserting into Pinecone Index\n",
    "\n",
    "Got most of these from James Briggs' notebook: https://www.pinecone.io/learn/langchain-retrieval-augmentation/"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. Load Text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "doc_path = (r\"hr_policy.txt\")\n",
    "\n",
    "# Open the file\n",
    "with open(doc_path, 'r') as f:\n",
    "    # Read the file\n",
    "    contents = f.read()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "28"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# set up tokenizer\n",
    "import tiktoken\n",
    "tokenizer = tiktoken.get_encoding('p50k_base')\n",
    "\n",
    "\n",
    "# create the length function\n",
    "def tiktoken_len(text):\n",
    "    tokens = tokenizer.encode(\n",
    "        text,\n",
    "        disallowed_special=()\n",
    "    )\n",
    "    return len(tokens)\n",
    "\n",
    "# sample\n",
    "tiktoken_len(\"hello I am a chunk of text and using the tiktoken_len function \"\n",
    "             \"we can find the length of this chunk of text in tokens\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. Create chunking function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'HR POLICY MANUAL - LEAVE POLICY\\nI. PURPOSE\\nThis policy is designed to provide a clear and consistent understanding of the leave benefits provided by our company to its employees. It covers the rules and regulations regarding Vacation Leave, Sick Leave, and Service Incentive Leave.\\nII. SCOPE\\nThis policy applies to all regular full-time employees of the company, regardless of their position or department.\\nIII. LEAVE POLICY\\nA. Vacation Leave\\n1.\\tEligibility and Accrual: All regular full-time employees are eligible for Vacation Leave. Employees will earn 1.25 days of Vacation Leave per month of service, accruing to 15 days per year.\\n2.\\tApplication: Leave applications must be submitted through the Employee Self Service portal at least one day before the intended leave date. Approval from the immediate supervisor is required.\\n3.\\tUnused Leave: Unused Vacation Leave can be carried over to the next year. However, the total accumulated leave should not exceed 30 days. Any excess leave will be forfeited.\\n4.\\tEncashment: Unused Vacation Leave can be encashed at the end of the year at the basic salary rate. Encashment rate is basic salary divided by 30 days multiplied by unused leaves to be encashed.\\n5.\\tDuring Probation: Employees on probation are not eligible for Vacation Leave.\\nB. Sick Leave\\n1.\\tEligibility and Accrual: All regular full-time employees are eligible for Sick Leave. Employees will earn 1.25 days of Sick Leave per month of service, accruing to 15 days per year.'"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
    "\n",
    "text_splitter = RecursiveCharacterTextSplitter(\n",
    "    chunk_size=400,\n",
    "    chunk_overlap=20,\n",
    "    length_function=tiktoken_len,\n",
    "    separators=[\"\\n\\n\", \"\\n\", \" \", \"\"]\n",
    ")\n",
    "\n",
    "chunks = text_splitter.split_text(contents)\n",
    "chunks[0]"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3. Create Embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# initialize embedding function\n",
    "from langchain.embeddings.openai import OpenAIEmbeddings\n",
    "import os\n",
    "\n",
    "OPENAI_API_KEY = \"<your openai api key from platform.openai.com>\" # there is a free tier. still trying to figure out how to use the azure deployment instead\n",
    "\n",
    "model_name = 'text-embedding-ada-002'\n",
    "\n",
    "# set embeddings function\n",
    "embed = OpenAIEmbeddings(\n",
    "    model = model_name,\n",
    "    openai_api_key=OPENAI_API_KEY\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create data format from chunked text for upserting into Pinecone index. Format: id, embeddings, metadata\n",
    "from uuid import uuid4\n",
    "\n",
    "vectors = [(str(uuid4()), embed.embed_documents([text])[0], {\"text\": text}) for text in chunks]\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### How the 'vectors' or embeddings look when printed. \n",
    "There are 1536 elements to the vector representing each chunk of data."
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![vectors](assets/vectors.png)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4. Prep Pinecone Index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/stepanogil/ChatBotDev/.venv/lib/python3.10/site-packages/pinecone/index.py:4: TqdmExperimentalWarning: Using `tqdm.autonotebook.tqdm` in notebook mode. Use `tqdm.tqdm` instead to force console mode (e.g. in jupyter console)\n",
      "  from tqdm.autonotebook import tqdm\n"
     ]
    }
   ],
   "source": [
    "import pinecone\n",
    "\n",
    "index_name = 'tk-policy'\n",
    "dimension=1536\n",
    "\n",
    "pinecone.init(\n",
    "        api_key=\"<your pineceone api key>\",  # get yours from pinecone.io. there is a free tier.\n",
    "        environment=\"<your pinecone environment>\"  \n",
    ")\n",
    "\n",
    "# delete index if it exists\n",
    "if index_name in pinecone.list_indexes():\n",
    "    pinecone.delete_index(index_name)\n",
    "\n",
    "# create index\n",
    "pinecone.create_index(\n",
    "        name=index_name,\n",
    "        metric='cosine',\n",
    "        dimension=dimension       \n",
    ")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 5. Upsert vectors to index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'dimension': 1536,\n",
       " 'index_fullness': 0.0,\n",
       " 'namespaces': {'': {'vector_count': 16}},\n",
       " 'total_vector_count': 16}"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# connect to index\n",
    "index = pinecone.Index(index_name)\n",
    "\n",
    "# upsert vectors to pinecone\n",
    "index.upsert(\n",
    "    vectors=vectors,\n",
    "    #namespace=index_name, \n",
    "    values=True, \n",
    "    include_metadata=True\n",
    "    )\n",
    "\n",
    "index.describe_index_stats()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}