03-retrievalqa-with-llama2-70b.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/1zuu/anaconda3/envs/llm/lib/python3.10/site-packages/InstructorEmbedding/instructor.py:7: TqdmExperimentalWarning: Using `tqdm.autonotebook.tqdm` in notebook mode. Use `tqdm.tqdm` instead to force console mode (e.g. in jupyter console)\n",
      "  from tqdm.autonotebook import trange\n"
     ]
    }
   ],
   "source": [
    "import logging\n",
    "import together, os, yaml\n",
    "from langchain.llms.base import LLM\n",
    "from pydantic import Extra, Field, root_validator\n",
    "from typing import Any, Dict, List, Mapping, Optional\n",
    "from langchain.callbacks.manager import CallbackManagerForLLMRun\n",
    "from langchain.llms.utils import enforce_stop_tokens\n",
    "from langchain.utils import get_from_dict_or_env\n",
    "\n",
    "from langchain.chains import RetrievalQA\n",
    "from langchain.vectorstores import Chroma\n",
    "from InstructorEmbedding import INSTRUCTOR\n",
    "from langchain.document_loaders import TextLoader\n",
    "from langchain.document_loaders import PyPDFLoader\n",
    "from langchain.document_loaders import DirectoryLoader\n",
    "from langchain.embeddings import HuggingFaceInstructEmbeddings\n",
    "from langchain.text_splitter import RecursiveCharacterTextSplitter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('cadentials.yaml') as f:\n",
    "    credentials = yaml.load(f, Loader=yaml.FullLoader)\n",
    "\n",
    "os.environ['HUGGINGFACEHUB_API_TOKEN'] = credentials['HUGGINGFACEHUB_API_TOKEN']\n",
    "os.environ['TOGETHER_AI_API'] = credentials['TOGETHER_AI_API']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Austism/chronos-hermes-13b\n",
      "EleutherAI/llemma_7b\n",
      "EleutherAI/pythia-12b-v0\n",
      "EleutherAI/pythia-1b-v0\n",
      "EleutherAI/pythia-2.8b-v0\n",
      "EleutherAI/pythia-6.9b\n",
      "Gryphe/MythoMax-L2-13b\n",
      "HuggingFaceH4/starchat-alpha\n",
      "NousResearch/Nous-Hermes-13b\n",
      "NousResearch/Nous-Hermes-Llama2-13b\n",
      "NousResearch/Nous-Hermes-Llama2-70b\n",
      "NousResearch/Nous-Hermes-llama-2-7b\n",
      "NumbersStation/nsql-llama-2-7B\n",
      "Open-Orca/Mistral-7B-OpenOrca\n",
      "OpenAssistant/llama2-70b-oasst-sft-v10\n",
      "OpenAssistant/oasst-sft-4-pythia-12b-epoch-3.5\n",
      "OpenAssistant/stablelm-7b-sft-v7-epoch-3\n",
      "Phind/Phind-CodeLlama-34B-Python-v1\n",
      "Phind/Phind-CodeLlama-34B-v2\n",
      "SG161222/Realistic_Vision_V3.0_VAE\n",
      "WizardLM/WizardCoder-15B-V1.0\n",
      "WizardLM/WizardCoder-Python-34B-V1.0\n",
      "WizardLM/WizardLM-70B-V1.0\n",
      "bigcode/starcoder\n",
      "databricks/dolly-v2-3b\n",
      "databricks/dolly-v2-7b\n",
      "defog/sqlcoder\n",
      "garage-bAInd/Platypus2-70B-instruct\n",
      "huggyllama/llama-13b\n",
      "huggyllama/llama-30b\n",
      "huggyllama/llama-65b\n",
      "huggyllama/llama-7b\n",
      "lmsys/fastchat-t5-3b-v1.0\n",
      "lmsys/vicuna-13b-v1.5-16k\n",
      "lmsys/vicuna-13b-v1.5\n",
      "lmsys/vicuna-7b-v1.5\n",
      "mistralai/Mistral-7B-Instruct-v0.1\n",
      "mistralai/Mistral-7B-v0.1\n",
      "prompthero/openjourney\n",
      "runwayml/stable-diffusion-v1-5\n",
      "sentence-transformers/msmarco-bert-base-dot-v5\n",
      "stabilityai/stable-diffusion-2-1\n",
      "stabilityai/stable-diffusion-xl-base-1.0\n",
      "teknium/OpenHermes-2-Mistral-7B\n",
      "togethercomputer/CodeLlama-13b-Instruct\n",
      "togethercomputer/CodeLlama-13b-Python\n",
      "togethercomputer/CodeLlama-13b\n",
      "togethercomputer/CodeLlama-34b-Instruct\n",
      "togethercomputer/CodeLlama-34b-Python\n",
      "togethercomputer/CodeLlama-34b\n",
      "togethercomputer/CodeLlama-7b-Instruct\n",
      "togethercomputer/CodeLlama-7b-Python\n",
      "togethercomputer/CodeLlama-7b\n",
      "togethercomputer/GPT-JT-6B-v1\n",
      "togethercomputer/GPT-JT-Moderation-6B\n",
      "togethercomputer/GPT-NeoXT-Chat-Base-20B\n",
      "togethercomputer/Koala-13B\n",
      "togethercomputer/LLaMA-2-7B-32K\n",
      "togethercomputer/Llama-2-7B-32K-Instruct\n",
      "togethercomputer/Pythia-Chat-Base-7B-v0.16\n",
      "togethercomputer/Qwen-7B-Chat\n",
      "togethercomputer/Qwen-7B\n",
      "togethercomputer/RedPajama-INCITE-7B-Base\n",
      "togethercomputer/RedPajama-INCITE-7B-Chat\n",
      "togethercomputer/RedPajama-INCITE-7B-Instruct\n",
      "togethercomputer/RedPajama-INCITE-Base-3B-v1\n",
      "togethercomputer/RedPajama-INCITE-Chat-3B-v1\n",
      "togethercomputer/RedPajama-INCITE-Instruct-3B-v1\n",
      "togethercomputer/alpaca-7b\n",
      "togethercomputer/bert-base-uncased\n",
      "togethercomputer/codegen2-16B\n",
      "togethercomputer/codegen2-7B\n",
      "togethercomputer/falcon-40b-instruct\n",
      "togethercomputer/falcon-40b\n",
      "togethercomputer/falcon-7b-instruct\n",
      "togethercomputer/falcon-7b\n",
      "togethercomputer/guanaco-13b\n",
      "togethercomputer/guanaco-65b\n",
      "togethercomputer/guanaco-7b\n",
      "togethercomputer/llama-2-13b-chat\n",
      "togethercomputer/llama-2-13b\n",
      "togethercomputer/llama-2-70b-chat\n",
      "togethercomputer/llama-2-70b\n",
      "togethercomputer/llama-2-7b-chat\n",
      "togethercomputer/llama-2-7b\n",
      "togethercomputer/m2-bert-80M-32k\n",
      "togethercomputer/m2-bert-80M-8k\n",
      "togethercomputer/mpt-30b-instruct\n",
      "togethercomputer/mpt-30b\n",
      "togethercomputer/mpt-7b-chat\n",
      "upstage/SOLAR-0-70b-16bit\n",
      "wavymulder/Analog-Diffusion\n",
      "lmsys/vicuna-13b-v1.3\n",
      "togethercomputer/replit-code-v1-3b\n",
      "togethercomputer/mpt-7b\n",
      "togethercomputer/mpt-30b-chat\n",
      "google/flan-t5-xxl\n",
      "google/flan-t5-xl\n",
      "togethercomputer/mpt-7b-instruct\n",
      "NumbersStation/nsql-6B\n",
      "togethercomputer/Koala-7B\n",
      "databricks/dolly-v2-12b\n",
      "EleutherAI/gpt-neox-20b\n",
      "OpenAssistant/oasst-sft-6-llama-30b-xor\n",
      "Salesforce/instructcodet5p-16b\n",
      "EleutherAI/gpt-j-6b\n",
      "lmsys/vicuna-7b-v1.3\n",
      "stabilityai/stablelm-base-alpha-3b\n",
      "stabilityai/stablelm-base-alpha-7b\n",
      "togethercomputer/guanaco-33b\n"
     ]
    }
   ],
   "source": [
    "together.api_key = os.environ[\"TOGETHER_AI_API\"]\n",
    "\n",
    "# list available models and descriptons\n",
    "models = together.Models.list()\n",
    "for m in models:\n",
    "    print(m['name'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'success': True,\n",
       " 'value': 'c610a1e207672bd2d5958ddddbfd72463f3aedbacddb359280bb815b41865fa0-a7126062027b3f68eef944eb24f2aaa38be87bec659c81dc7cb776c13d010208'}"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "together.Models.start(\"togethercomputer/llama-2-13b-chat\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "class TogetherLLM(LLM):\n",
    "    \"\"\"Together large language models.\"\"\"\n",
    "\n",
    "    model: str = \"togethercomputer/llama-2-70b-chat\" # model endpoint to use\n",
    "    together_ai_api: str = os.environ[\"TOGETHER_AI_API\"] # Together API key\n",
    "    temperature: float = 0.7 # What sampling temperature to use.\n",
    "    max_tokens: int = 512 # The maximum number of tokens to generate in the completion.\n",
    "    class Config:\n",
    "        extra = Extra.forbid\n",
    "\n",
    "    @root_validator()\n",
    "    def validate_environment(cls, values: Dict) -> Dict:\n",
    "        \"\"\"Validate that the API key is set.\"\"\"\n",
    "        api_key = get_from_dict_or_env(\n",
    "            values, \"together_ai_api\", \"TOGETHER_AI_API\"\n",
    "        )\n",
    "        values[\"together_ai_api\"] = api_key\n",
    "        return values\n",
    "\n",
    "    @property\n",
    "    def _llm_type(self) -> str:\n",
    "        \"\"\"Return type of LLM.\"\"\"\n",
    "        return \"together\"\n",
    "\n",
    "    def _call(\n",
    "        self,\n",
    "        prompt: str,\n",
    "        **kwargs: Any,\n",
    "    ) -> str:\n",
    "        \"\"\"Call to Together endpoint.\"\"\"\n",
    "        together.api_key = self.together_ai_api\n",
    "        output = together.Complete.create(prompt,\n",
    "                                          model=self.model,\n",
    "                                          max_tokens=self.max_tokens,\n",
    "                                          temperature=self.temperature,\n",
    "                                          )\n",
    "        text = output['output']['choices'][0]['text']\n",
    "        return text"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Loader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "142"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "loader = DirectoryLoader(\n",
    "                        'data/new_papers/', \n",
    "                        glob=\"./*.pdf\", \n",
    "                        loader_cls=PyPDFLoader\n",
    "                        )\n",
    "documents = loader.load()\n",
    "len(documents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "659"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)\n",
    "texts = text_splitter.split_documents(documents)\n",
    "len(texts)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "load INSTRUCTOR_Transformer\n",
      "'NoneType' object has no attribute 'cadam32bit_grad_fp32'\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/1zuu/anaconda3/envs/llm/lib/python3.10/site-packages/bitsandbytes/cextension.py:34: UserWarning: The installed version of bitsandbytes was compiled without GPU support. 8-bit optimizers, 8-bit multiplication, and GPU quantization are unavailable.\n",
      "  warn(\"The installed version of bitsandbytes was compiled without GPU support. \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "max_seq_length  512\n"
     ]
    }
   ],
   "source": [
    "instructor_embeddings = HuggingFaceInstructEmbeddings(\n",
    "                                                    model_name=\"hkunlp/instructor-xl\", \n",
    "                                                    model_kwargs={\"device\": \"mps\"}\n",
    "                                                    )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Chroma DB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "persist_directory = 'db/03'\n",
    "\n",
    "## Here is the nmew embeddings being used\n",
    "embedding = instructor_embeddings\n",
    "\n",
    "vectordb = Chroma.from_documents(\n",
    "                                documents=texts,\n",
    "                                embedding=embedding,\n",
    "                                persist_directory=persist_directory\n",
    "                                )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "retriever = vectordb.as_retriever(search_kwargs={\"k\": 5})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "llm = TogetherLLM(\n",
    "                model= \"togethercomputer/llama-2-70b-chat\",\n",
    "                temperature = 0.1,\n",
    "                max_tokens = 1024\n",
    "                )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create the chain to answer questions\n",
    "qa_chain = RetrievalQA.from_chain_type(\n",
    "                                    llm=llm,\n",
    "                                    chain_type=\"stuff\",\n",
    "                                    retriever=retriever,\n",
    "                                    return_source_documents=True\n",
    "                                    )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "import textwrap\n",
    "\n",
    "def wrap_text_preserve_newlines(text, width=110):\n",
    "    lines = text.split('\\n')\n",
    "    wrapped_lines = [textwrap.fill(line, width=width) for line in lines]\n",
    "    wrapped_text = '\\n'.join(wrapped_lines)\n",
    "\n",
    "    return wrapped_text\n",
    "\n",
    "def process_llm_response(llm_response):\n",
    "    print(wrap_text_preserve_newlines(llm_response['result']))\n",
    "    print('\\n\\nSources:')\n",
    "    for source in llm_response[\"source_documents\"]:\n",
    "        print(source.metadata['source'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Flash attention is a new attention algorithm that computes exact attention with far fewer memory accesses. It\n",
      "is designed to avoid reading and writing the attention matrix to and from HBM, which reduces the memory\n",
      "accesses and improves the performance. It uses tiling to split the input into blocks and make several passes\n",
      "over input blocks, thus incrementally performing the softmax reduction. It also stores the softmax\n",
      "normalization factor from the forward pass to quickly recompute attention on-chip in the backward pass, which\n",
      "is faster than the standard approach of reading the intermediate attention matrix from HBM.\n",
      "\n",
      "\n",
      "Sources:\n",
      "data/new_papers/Flash-attention.pdf\n",
      "data/new_papers/Flash-attention.pdf\n",
      "data/new_papers/Flash-attention.pdf\n",
      "data/new_papers/Flash-attention.pdf\n",
      "data/new_papers/Flash-attention.pdf\n"
     ]
    }
   ],
   "source": [
    "# full example\n",
    "query = \"What is Flash attention?\"\n",
    "llm_response = qa_chain(query)\n",
    "process_llm_response(llm_response)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}