From 77ab1198549f60979e074ad117ef21afdce6779b Mon Sep 17 00:00:00 2001 From: Joschka Braun Date: Sun, 20 Aug 2023 15:22:02 -0700 Subject: [PATCH 1/4] feat: add log endpoint --- parea/client.py | 18 +++++++++++++++++- parea/schemas/models.py | 5 +++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/parea/client.py b/parea/client.py index 4f26263e..722b3013 100644 --- a/parea/client.py +++ b/parea/client.py @@ -3,11 +3,13 @@ from attrs import asdict, define, field from parea.api_client import HTTPClient -from parea.schemas.models import Completion, CompletionResponse, FeedbackRequest, UseDeployedPrompt, UseDeployedPromptResponse +from parea.schemas.models import Completion, CompletionResponse, FeedbackRequest, UseDeployedPrompt, \ + UseDeployedPromptResponse, LogRequest COMPLETION_ENDPOINT = "/completion" DEPLOYED_PROMPT_ENDPOINT = "/deployed-prompt" RECORD_FEEDBACK_ENDPOINT = "/feedback" +LOG_ENDPOINT = "/log" @define @@ -64,6 +66,20 @@ async def arecord_feedback(self, data: FeedbackRequest) -> None: data=asdict(data), ) + def log(self, data: LogRequest) -> None: + self._client.request( + "POST", + LOG_ENDPOINT, + data=asdict(data), + ) + + async def alog(self, data: LogRequest) -> None: + await self._client.request_async( + "POST", + LOG_ENDPOINT, + data=asdict(data), + ) + def gen_trace_id() -> str: """Generate a unique trace id for each chain of requests""" diff --git a/parea/schemas/models.py b/parea/schemas/models.py index 33ad8625..1310037f 100644 --- a/parea/schemas/models.py +++ b/parea/schemas/models.py @@ -108,3 +108,8 @@ class FeedbackRequest: inference_id: Optional[str] = None name: Optional[str] = None target: Optional[str] = None + + +@define +class LogRequest: + pass From c3acd74bfa47c95ed4388690e6815c955fb64d2d Mon Sep 17 00:00:00 2001 From: Joschka Braun Date: Sun, 20 Aug 2023 17:30:27 -0700 Subject: [PATCH 2/4] feat: add example usage --- README.md | 70 ++++++++++++++++++++++++++++++++++++++++- parea/schemas/models.py | 31 ++++++++++++++++-- 2 files changed, 98 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 832e58ed..82fb8d82 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ from parea.schemas.models import Completion, UseDeployedPrompt, CompletionRespon load_dotenv() -p = Parea(api_key=os.getenv("API_KEY")) +p = Parea(api_key=os.getenv("PAREA_API_KEY")) # You will find this deployment_id in the Parea dashboard deployment_id = '' @@ -78,6 +78,74 @@ async def main_async(): print(deployed_prompt) ``` + +### Logging results from LLM providers + +```python +import os + +from dotenv import load_dotenv + +import openai +from parea import Parea +from parea.schemas.models import LogRequest + +load_dotenv() + +openai.api_key = os.getenv("OPENAI_API_KEY") +p = Parea(api_key=os.getenv("PAREA_API_KEY")) + + +# define your OpenAI call as you would normally +x = "Golang" +y = "Fiber" +inputs = {"x": x, "y": y} +messages = [ + {"role": "user", "content": f"Write a hello world program using {x} and the {y} framework."}, +] +model = "gpt-3.5-turbo" +model_params = { + "temperature": 0.7, + "top_p": 1.0, +} +completion = openai.ChatCompletion.create( + model=model, + messages=messages, + **model_params +) +output = completion.choices[0].message['content'] + +# the LogRequest schema +log_request: LogRequest = LogRequest( + status="success", + name='Test Log', + llm_inputs={ + "x": x, + "y": y, + }, + llm_configuration={ + 'model': model, + 'messages': messages, + 'model_params': model_params, + }, + output=output, + input_tokens=completion.usage['prompt_tokens'], + output_tokens=completion.usage['completion_tokens'], + total_tokens=completion.usage['total_tokens'], +) + + +def main(): + p.log(data=log_request) + + +async def main_async(): + await p.alog(data=log_request) + + +``` + + ### Open source community features Ready-to-use [Pull Requests templates](https://github.com/parea-ai/parea-sdk/blob/master/.github/PULL_REQUEST_TEMPLATE.md) diff --git a/parea/schemas/models.py b/parea/schemas/models.py index 1310037f..f3d893b9 100644 --- a/parea/schemas/models.py +++ b/parea/schemas/models.py @@ -21,7 +21,6 @@ class Message: @define class ModelParams: - model: str temp: float = 0.5 top_p: float = 1.0 frequency_penalty: float = 0.0 @@ -112,4 +111,32 @@ class FeedbackRequest: @define class LogRequest: - pass + inference_id: Optional[str] = None + trace_id: Optional[str] = None + trace_name: Optional[str] = None + end_user_identifier: Optional[str] = None + error: Optional[str] = None + status: Optional[str] = None + + start_timestamp: str = "" + end_timestamp: str = "" + duration: float = 0.0 + + deployment_id: Optional[str] = None + name: Optional[str] = None + evaluation_metrics_ids: Optional[list[int]] = None + metadata: Optional[dict] = None + cache_hit: bool = False + target: Optional[str] = None + tags: Optional[list[str]] = None + + llm_inputs: Optional[dict[str, Any]] = None + llm_configuration: LLMInputs = LLMInputs() + + output: Optional[str] = "" + latency: Optional[float] = 0.0 + input_tokens: Optional[int] = 0 + output_tokens: Optional[int] = 0 + total_tokens: Optional[int] = 0 + cost: Optional[float] = None + feedback_score: Optional[float] = None From b6332bcac95e9161db192711bea87f1d836b4c4b Mon Sep 17 00:00:00 2001 From: Joschka Braun Date: Sun, 20 Aug 2023 17:34:19 -0700 Subject: [PATCH 3/4] style --- parea/client.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/parea/client.py b/parea/client.py index 722b3013..748bedc3 100644 --- a/parea/client.py +++ b/parea/client.py @@ -3,8 +3,7 @@ from attrs import asdict, define, field from parea.api_client import HTTPClient -from parea.schemas.models import Completion, CompletionResponse, FeedbackRequest, UseDeployedPrompt, \ - UseDeployedPromptResponse, LogRequest +from parea.schemas.models import Completion, CompletionResponse, FeedbackRequest, LogRequest, UseDeployedPrompt, UseDeployedPromptResponse COMPLETION_ENDPOINT = "/completion" DEPLOYED_PROMPT_ENDPOINT = "/deployed-prompt" From 4644ffa8e66d59b0cb7c18cb1968204fa5087a2c Mon Sep 17 00:00:00 2001 From: Joschka Braun Date: Sun, 20 Aug 2023 18:00:46 -0700 Subject: [PATCH 4/4] style --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index 82fb8d82..990af423 100644 --- a/README.md +++ b/README.md @@ -141,8 +141,6 @@ def main(): async def main_async(): await p.alog(data=log_request) - - ```