Merge pull request #260 from parea-ai/sdk-write-parent_trace_id

add langchain tracer

parea/cookbook/langchain/trace_langchain_RAG_evals.py

@@ -0,0 +1,182 @@
+import os
+from datetime import datetime
+from operator import itemgetter
+
+from dotenv import load_dotenv
+
+# LangChain libs
+from langchain.chat_models import ChatOpenAI
+from langchain.document_loaders import RecursiveUrlLoader
+from langchain.document_transformers import Html2TextTransformer
+from langchain.embeddings import OpenAIEmbeddings
+from langchain.prompts import ChatPromptTemplate
+from langchain.schema.output_parser import StrOutputParser
+from langchain.text_splitter import TokenTextSplitter
+from langchain.vectorstores import Chroma
+
+# Parea libs
+from parea import Parea
+from parea.evals.general import answer_matches_target_llm_grader_factory
+from parea.evals.rag import (
+    answer_context_faithfulness_binary_factory,
+    answer_context_faithfulness_precision_factory,
+    answer_context_faithfulness_statement_level_factory,
+    context_query_relevancy_factory,
+    percent_target_supported_by_context_factory,
+)
+from parea.evals.utils import EvalFuncTuple, run_evals_in_thread_and_log
+from parea.schemas.log import LLMInputs, Log
+from parea.utils.trace_integrations.langchain import PareaAILangchainTracer
+
+load_dotenv()
+
+# Need to instantiate Parea for tracing and evals
+p = Parea(api_key=os.getenv("PAREA_API_KEY"))
+
+
+class DocumentRetriever:
+    def __init__(self, url: str):
+        # Load in langsmith documentation as test
+        api_loader = RecursiveUrlLoader(url)
+        raw_documents = api_loader.load()
+
+        # Transformer
+        doc_transformer = Html2TextTransformer()
+        transformed = doc_transformer.transform_documents(raw_documents)
+
+        # Splitter
+        text_splitter = TokenTextSplitter(
+            model_name="gpt-3.5-turbo",
+            chunk_size=2000,
+            chunk_overlap=200,
+        )
+        documents = text_splitter.split_documents(transformed)
+
+        # Define vector store based
+        embeddings = OpenAIEmbeddings()
+        vectorstore = Chroma.from_documents(documents, embeddings)
+        self.retriever = vectorstore.as_retriever(search_kwargs={"k": 4})
+
+    def get_retriever(self):
+        return self.retriever
+
+
+class DocumentationChain:
+    def __init__(self, retriever, model: str):
+        prompt = ChatPromptTemplate.from_messages(
+            [
+                (
+                    "system",
+                    "You are a helpful documentation Q&A assistant, trained to answer"
+                    " questions from LangSmith's documentation."
+                    " LangChain is a framework for building applications using large language models."
+                    "\nThe current time is {time}.\n\nRelevant documents will be retrieved in the following messages.",
+                ),
+                ("system", "{context}"),
+                ("human", "{question}"),
+            ]
+        ).partial(time=str(datetime.now()))
+
+        model = ChatOpenAI(model=model, temperature=0)
+
+        response_generator = prompt | model | StrOutputParser()
+
+        self.chain = (
+            # The runnable map here routes the original inputs to a context and a question dictionary to pass to the response generator
+            {"context": itemgetter("question") | retriever | self._format_docs, "question": itemgetter("question")}
+            | response_generator
+        )
+
+    def get_context(self) -> str:
+        """Helper to get the context from a retrieval chain, so we can use it for evaluation metrics."""
+        return self.context
+
+    def _format_docs(self, docs) -> str:
+        context = "\n\n".join(doc.page_content for doc in docs)
+        # set context as an attribute, so we can access it later
+        self.context = context
+        return context
+
+    def get_chain(self):
+        return self.chain
+
+
+# EXAMPLE EVALUATION TEST CASES
+eval_questions = [
+    "What is the population of New York City as of 2020?",
+    "Which borough of New York City has the highest population? Only respond with the name of the borough.",
+    "What is the economic significance of New York City?",
+    "How did New York City get its name?",
+    "What is the significance of the Statue of Liberty in New York City?",
+]
+
+eval_answers = [
+    "8,804,190",
+    "Brooklyn",
+    """New York City's economic significance is vast, as it serves as the global financial capital, housing Wall
+    Street and major financial institutions. Its diverse economy spans technology, media, healthcare, education,
+    and more, making it resilient to economic fluctuations. NYC is a hub for international business, attracting
+    global companies, and boasts a large, skilled labor force. Its real estate market, tourism, cultural industries,
+    and educational institutions further fuel its economic prowess. The city's transportation network and global
+    influence amplify its impact on the world stage, solidifying its status as a vital economic player and cultural
+    epicenter.""",
+    """New York City got its name when it came under British control in 1664. King Charles II of England granted the
+    lands to his brother, the Duke of York, who named the city New York in his own honor.""",
+    """The Statue of Liberty in New York City holds great significance as a symbol of the United States and its
+    ideals of liberty and peace. It greeted millions of immigrants who arrived in the U.S. by ship in the late 19th
+    and early 20th centuries, representing hope and freedom for those seeking a better life. It has since become an
+    iconic landmark and a global symbol of cultural diversity and freedom.""",
+]
+
+# set up evaluation functions we want to test, provide the name of the relevant fields needed for each eval function
+EVALS = [
+    # question field only
+    EvalFuncTuple(name="matches_target", func=answer_matches_target_llm_grader_factory(question_field="question")),
+    # context field only
+    EvalFuncTuple(name="faithfulness_precision", func=answer_context_faithfulness_precision_factory(context_field="context")),
+    # questions field and single context field
+    EvalFuncTuple(name="faithfulness_binary", func=answer_context_faithfulness_binary_factory(question_field="question", context_field="context")),
+    # questions field and accepts multiple context fields
+    EvalFuncTuple(name="faithfulness_statement", func=answer_context_faithfulness_statement_level_factory(question_field="question", context_fields=["context"])),
+    EvalFuncTuple(name="relevancy", func=context_query_relevancy_factory(question_field="question", context_fields=["context"])),
+    EvalFuncTuple(name="supported_by_context", func=percent_target_supported_by_context_factory(question_field="question", context_fields=["context"])),
+]
+
+
+def create_log(model, question, context, output, target, provider="openai") -> Log:
+    """Creates a log object for evaluation metric functions."""
+    inputs = {"question": question, "context": context}  # fields named question and context to align w/ eval functions
+    log_config = LLMInputs(model=model, provider=provider)  # provider and model needed for precision eval metric
+    return Log(configuration=log_config, inputs=inputs, output=output, target=target)
+
+
+def main():
+    model = "gpt-3.5-turbo-16k"
+    # instantiate tracer integration
+    handler = PareaAILangchainTracer()
+    # set up retriever
+    retriever = DocumentRetriever("https://en.wikipedia.org/wiki/New_York_City").get_retriever()
+    # set up chain
+    dc = DocumentationChain(retriever, model)
+
+    # iterate through questions and answers
+    for question, answer in zip(eval_questions, eval_answers):
+        # call chain and attached parea tracer as a callback for logs
+        output = dc.get_chain().invoke({"question": question}, config={"callbacks": [handler]})
+        print(f"Question: {question}\nAnswer: {output} \n")
+
+        # get parent trace id from the tracer
+        parent_trace_id = handler.get_parent_trace_id()
+        # after chain is called, get the context for evaluation metric functions
+        context = dc.get_context()
+        # build log component needed for evaluation metric functions
+        log = create_log(model, question, context, output, answer)
+
+        # helper function to run evaluation metrics in a thread to avoid blocking return of chain
+        run_evals_in_thread_and_log(trace_id=str(parent_trace_id), log=log, eval_funcs=EVALS, verbose=True)
+
+
+if __name__ == "__main__":
+    print("Running evals...")
+    main()
+    print("Done!")

parea/cookbook/langchain/trace_langchain_anthropic_function_calling.py

@@ -0,0 +1,58 @@
+import os
+
+from dotenv import load_dotenv
+from langchain.chains import create_extraction_chain
+from langchain.schema import HumanMessage
+from langchain_experimental.llms.anthropic_functions import AnthropicFunctions
+
+from parea import Parea
+from parea.utils.trace_integrations.langchain import PareaAILangchainTracer
+
+load_dotenv()
+
+p = Parea(api_key=os.getenv("PAREA_API_KEY"))
+
+model = AnthropicFunctions(model="claude-2")
+
+functions = [
+    {
+        "name": "get_current_weather",
+        "description": "Get the current weather in a given location",
+        "parameters": {
+            "type": "object",
+            "properties": {
+                "location": {
+                    "type": "string",
+                    "description": "The city and state, e.g. San Francisco, CA",
+                },
+                "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]},
+            },
+            "required": ["location"],
+        },
+    }
+]
+
+
+schema = {
+    "properties": {
+        "name": {"type": "string"},
+        "height": {"type": "integer"},
+        "hair_color": {"type": "string"},
+    },
+    "required": ["name", "height"],
+}
+inp = """Alex is 5 feet tall. Claudia is 1 feet taller Alex and jumps higher than him. Claudia is a brunette and Alex
+is blonde."""
+
+chain = create_extraction_chain(schema, model)
+
+
+def main():
+    response = model.predict_messages([HumanMessage(content="whats the weater in boston?")], functions=functions, callbacks=[PareaAILangchainTracer()])
+    print(response)
+    result = chain.run(inp, callbacks=[PareaAILangchainTracer()])
+    print(result)
+
+
+if __name__ == "__main__":
+    main()

parea/cookbook/langchain/trace_langchain_rag_agents.py

@@ -0,0 +1,46 @@
+import os
+
+from dotenv import load_dotenv
+from langchain.agents.agent_toolkits import create_conversational_retrieval_agent, create_retriever_tool
+from langchain.chat_models import ChatOpenAI
+from langchain.document_loaders import TextLoader
+from langchain.embeddings import OpenAIEmbeddings
+from langchain.text_splitter import CharacterTextSplitter
+from langchain.vectorstores import FAISS
+
+from parea import Parea
+from parea.utils.trace_integrations.langchain import PareaAILangchainTracer
+
+load_dotenv()
+
+p = Parea(api_key=os.getenv("PAREA_API_KEY"))
+
+loader = TextLoader("../data/state_of_the_union.txt")
+
+
+documents = loader.load()
+text_splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=0)
+texts = text_splitter.split_documents(documents)
+embeddings = OpenAIEmbeddings()
+db = FAISS.from_documents(texts, embeddings)
+retriever = db.as_retriever()
+tool = create_retriever_tool(
+    retriever,
+    "search_state_of_union",
+    "Searches and returns documents regarding the state-of-the-union.",
+)
+tools = [tool]
+
+
+llm = ChatOpenAI(temperature=0)
+
+agent_executor = create_conversational_retrieval_agent(llm, tools)
+
+
+def main():
+    result = agent_executor({"input": "what did the president say about kentaji brown jackson in the most recent state of the union?"}, callbacks=[PareaAILangchainTracer()])
+    print(result)
+
+
+if __name__ == "__main__":
+    main()

parea/cookbook/langchain/trace_langchain_rag_question_answering.py

@@ -0,0 +1,49 @@
+import os
+
+import bs4
+from dotenv import load_dotenv
+from langchain import hub
+from langchain.chat_models import ChatOpenAI
+from langchain.document_loaders import WebBaseLoader
+from langchain.embeddings import OpenAIEmbeddings
+from langchain.schema import StrOutputParser
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain.vectorstores import Chroma
+from langchain_core.runnables import RunnablePassthrough
+
+from parea import Parea
+from parea.utils.trace_integrations.langchain import PareaAILangchainTracer
+
+load_dotenv()
+
+p = Parea(api_key=os.getenv("PAREA_API_KEY"))
+
+loader = WebBaseLoader(
+    web_paths=("https://lilianweng.github.io/posts/2023-06-23-agent/",),
+    bs_kwargs=dict(parse_only=bs4.SoupStrainer(class_=("post-content", "post-title", "post-header"))),
+)
+docs = loader.load()
+
+text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)
+splits = text_splitter.split_documents(docs)
+
+vectorstore = Chroma.from_documents(documents=splits, embedding=OpenAIEmbeddings())
+retriever = vectorstore.as_retriever()
+
+prompt = hub.pull("rlm/rag-prompt")
+llm = ChatOpenAI(temperature=0)
+
+
+def format_docs(docs):
+    return "\n\n".join(doc.page_content for doc in docs)
+
+
+rag_chain = {"context": retriever | format_docs, "question": RunnablePassthrough()} | prompt | llm | StrOutputParser()
+
+
+def main():
+    rag_chain.invoke("What is Task Decomposition?", config={"callbacks": [PareaAILangchainTracer()]})
+
+
+if __name__ == "__main__":
+    main()

parea/cookbook/tracing_with_agent.py

@@ -12,7 +12,7 @@
 
 load_dotenv()
 
-p = Parea(api_key=os.getenv("DEV_API_KEY"))
+p = Parea(api_key=os.getenv("PAREA_API_KEY"))
 
 # Parea SDK makes it easy to use different LLMs with the same apis structure and standardized request/response schemas.
 LLM_OPTIONS = [("gpt-3.5-turbo", "openai"), ("gpt-4", "openai"), ("claude-instant-1", "anthropic"), ("claude-2", "anthropic")]

parea/cookbook/tracing_with_deployed_prompt.py

@@ -1,5 +1,3 @@
-from typing import Tuple
-
 import json
 import os
 from datetime import datetime
@@ -13,7 +11,7 @@
 
 load_dotenv()
 
-p = Parea(api_key=os.getenv("DEV_API_KEY"))
+p = Parea(api_key=os.getenv("PAREA_API_KEY"))
 
 
 def deployed_argument_generator(query: str, additional_description: str = "") -> str:

parea/cookbook/tracing_with_open_ai_endpoint_directly.py

@@ -12,7 +12,7 @@
 
 openai.api_key = os.getenv("OPENAI_API_KEY")
 
-p = Parea(api_key=os.getenv("DEV_API_KEY"))
+p = Parea(api_key=os.getenv("PAREA_API_KEY"))
 
 
 def call_llm(data: list[dict], model: str = "gpt-3.5-turbo", temperature: float = 0.0) -> str:

parea/cookbook/tracing_with_openai_with_functions.py

@@ -15,7 +15,7 @@
 PLACES_URL = "https://maps.googleapis.com/maps/api/place/nearbysearch/json"
 openai.api_key = os.getenv("OPENAI_API_KEY")
 
-p = Parea(api_key=os.getenv("DEV_API_KEY"))
+p = Parea(api_key=os.getenv("PAREA_API_KEY"))
 
 functions = [
     {

parea/cookbook/tracing_without_deployed_prompt.py

@@ -10,7 +10,7 @@
 
 load_dotenv()
 
-p = Parea(api_key=os.getenv("DEV_API_KEY"))
+p = Parea(api_key=os.getenv("PAREA_API_KEY"))
 
 
 @trace  # <--- If you want to log the inputs to the LLM call you can optionally add a trace decorator here