From fa1f412497cd819edd54d09506db52a1b8da7d0f Mon Sep 17 00:00:00 2001 From: Joschka Braun Date: Thu, 20 Jun 2024 10:02:47 -0400 Subject: [PATCH 1/4] fix: instructor key error --- parea/cookbook/assets/__init__.py | 0 .../evals_and_experiments/__init__.py | 0 parea/cookbook/langchain/__init__.py | 0 parea/cookbook/marvin/__init__.py | 0 parea/cookbook/openai/__init__.py | 0 parea/utils/trace_integrations/instructor.py | 20 +++++++++++-------- 6 files changed, 12 insertions(+), 8 deletions(-) delete mode 100644 parea/cookbook/assets/__init__.py delete mode 100644 parea/cookbook/evals_and_experiments/__init__.py delete mode 100644 parea/cookbook/langchain/__init__.py delete mode 100644 parea/cookbook/marvin/__init__.py delete mode 100644 parea/cookbook/openai/__init__.py diff --git a/parea/cookbook/assets/__init__.py b/parea/cookbook/assets/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/parea/cookbook/evals_and_experiments/__init__.py b/parea/cookbook/evals_and_experiments/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/parea/cookbook/langchain/__init__.py b/parea/cookbook/langchain/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/parea/cookbook/marvin/__init__.py b/parea/cookbook/marvin/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/parea/cookbook/openai/__init__.py b/parea/cookbook/openai/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/parea/utils/trace_integrations/instructor.py b/parea/utils/trace_integrations/instructor.py index c2e57e53..af80e9f6 100644 --- a/parea/utils/trace_integrations/instructor.py +++ b/parea/utils/trace_integrations/instructor.py @@ -1,8 +1,10 @@ +from json import JSONDecodeError from typing import Any, Callable, Mapping, Tuple import contextvars from instructor.retry import InstructorRetryException +from pydantic import ValidationError from wrapt import wrap_object from parea import trace @@ -33,6 +35,12 @@ def instrument_instructor_validation_errors() -> None: ) +def get_reasons(exception: Exception) -> list[str]: + if isinstance(exception, InstructorRetryException): + return [str(arg) for arg in exception.args] + return [str(exception)] + + def report_instructor_validation_errors() -> None: reason = "\n\n\n".join(instructor_val_errs.get()) if reason: @@ -82,11 +90,9 @@ def __call__( )( wrapped )(*args, **kwargs) - except InstructorRetryException as e: + except (InstructorRetryException, ValidationError, JSONDecodeError) as e: instructor_val_err_count.set(instructor_val_err_count.get() + 1) - reasons = [] - for arg in e.args: - reasons.append(str(arg)) + reasons = get_reasons(e) instructor_val_errs.set(instructor_val_errs.get() + reasons) report_instructor_validation_errors() @@ -105,11 +111,9 @@ def __call__( kwargs: Mapping[str, Any], ) -> Any: if instructor_trace_id.get() is not None: - if len(args) > 1 and args[1] is not None and isinstance(args[1], InstructorRetryException): + if len(args) > 1 and args[1] is not None and isinstance(args[1], (InstructorRetryException, ValidationError, JSONDecodeError)): instructor_val_err_count.set(instructor_val_err_count.get() + 1) - reasons = [] - for arg in args[1].args: - reasons.append(str(arg)) + reasons = get_reasons(args[1]) instructor_val_errs.set(instructor_val_errs.get() + reasons) else: report_instructor_validation_errors() From 62c1cdd4c95ffb6261ca01aacaed9afd34ef420e Mon Sep 17 00:00:00 2001 From: Joschka Braun Date: Thu, 20 Jun 2024 10:02:56 -0400 Subject: [PATCH 2/4] fix: instructor key error --- parea/cookbook/anthropic/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 parea/cookbook/anthropic/__init__.py diff --git a/parea/cookbook/anthropic/__init__.py b/parea/cookbook/anthropic/__init__.py deleted file mode 100644 index e69de29b..00000000 From 1e29689fe977ad56171fd3d2bb35f521ba660d15 Mon Sep 17 00:00:00 2001 From: Joschka Braun Date: Thu, 20 Jun 2024 10:51:49 -0400 Subject: [PATCH 3/4] fix: nested instructor outputs and update output --- .../instructor/instructor_streaming.py | 65 +++++++++++++++++++ parea/wrapper/wrapper.py | 20 +++++- 2 files changed, 83 insertions(+), 2 deletions(-) create mode 100644 parea/cookbook/instructor/instructor_streaming.py diff --git a/parea/cookbook/instructor/instructor_streaming.py b/parea/cookbook/instructor/instructor_streaming.py new file mode 100644 index 00000000..76f92948 --- /dev/null +++ b/parea/cookbook/instructor/instructor_streaming.py @@ -0,0 +1,65 @@ +import os + +import instructor +from dotenv import load_dotenv +from openai import AsyncOpenAI + + +from parea import Parea + +load_dotenv() + +client = AsyncOpenAI() + +p = Parea(api_key=os.getenv("PAREA_API_KEY")) +p.wrap_openai_client(client, "instructor") + +client = instructor.from_openai(client) + + +from pydantic import BaseModel + + + +class UserDetail(BaseModel): + name: str + age: int + + +async def main(): + user = client.completions.create_partial( + model="gpt-3.5-turbo", + max_tokens=1024, + max_retries=3, + messages=[ + { + "role": "user", + "content": "Please crea a user", + } + ], + response_model=UserDetail, + ) + # print(user) + async for u in user: + print(u) + + user2 = client.completions.create_partial( + model="gpt-3.5-turbo", + max_tokens=1024, + max_retries=3, + messages=[ + { + "role": "user", + "content": "Please crea a user", + } + ], + response_model=UserDetail, + ) + async for u in user2: + print(u) + + +if __name__ == "__main__": + import asyncio + + asyncio.run(main()) diff --git a/parea/wrapper/wrapper.py b/parea/wrapper/wrapper.py index f6994798..b4782b13 100644 --- a/parea/wrapper/wrapper.py +++ b/parea/wrapper/wrapper.py @@ -12,8 +12,9 @@ from parea.constants import PAREA_OS_ENV_EXPERIMENT_UUID from parea.evals.utils import _make_evaluations from parea.helpers import is_logging_disabled, timezone_aware_now -from parea.schemas.models import TraceLog, UpdateTraceScenario -from parea.utils.trace_utils import call_eval_funcs_then_log, execution_order_counters, fill_trace_data, trace_context, trace_data +from parea.schemas.models import TraceLog, UpdateTraceScenario, UpdateLog +from parea.utils.trace_utils import call_eval_funcs_then_log, execution_order_counters, fill_trace_data, trace_context, \ + trace_data, logger_update_record from parea.wrapper.utils import safe_format_template_to_prompt, skip_decorator_if_func_in_stack logger = logging.getLogger() @@ -213,7 +214,22 @@ def final_log(): self.log(trace_id) try: + trace_context_before_reset = trace_context.get() trace_context.reset(context_token) + trace_context_after_reset = trace_context.get() + if len(trace_context_after_reset) > len(trace_context_before_reset): + # this can happen if this is a streaming call and the LLM client got modified (e.g. instructor) + # so we need to manually reset the trace context to the previous state + trace_context.set(trace_context_before_reset) + # if the parent trace didn't have any output, we can also update the output + if (parent_trace_id := trace_data.get()[trace_id].parent_trace_id) is not None and not trace_data.get()[parent_trace_id].output: + logger_update_record( + UpdateLog( + trace_id=parent_trace_id, + field_name_to_value_map={"output": trace_data.get()[trace_id].output}, + root_trace_id=trace_data.get()[parent_trace_id].root_trace_id, + ) + ) except IndexError: pass From ad6dbb5fdfec5a228bbadcff2d8c0ac2c05a9d25 Mon Sep 17 00:00:00 2001 From: Joschka Braun Date: Thu, 20 Jun 2024 10:52:36 -0400 Subject: [PATCH 4/4] chore: bump version --- parea/cookbook/instructor/instructor_streaming.py | 2 -- parea/schemas/models.py | 3 ++- parea/utils/trace_integrations/instructor.py | 2 +- parea/wrapper/wrapper.py | 5 ++--- pyproject.toml | 2 +- 5 files changed, 6 insertions(+), 8 deletions(-) diff --git a/parea/cookbook/instructor/instructor_streaming.py b/parea/cookbook/instructor/instructor_streaming.py index 76f92948..46208439 100644 --- a/parea/cookbook/instructor/instructor_streaming.py +++ b/parea/cookbook/instructor/instructor_streaming.py @@ -4,7 +4,6 @@ from dotenv import load_dotenv from openai import AsyncOpenAI - from parea import Parea load_dotenv() @@ -20,7 +19,6 @@ from pydantic import BaseModel - class UserDetail(BaseModel): name: str age: int diff --git a/parea/schemas/models.py b/parea/schemas/models.py index 4b47d91f..19786964 100644 --- a/parea/schemas/models.py +++ b/parea/schemas/models.py @@ -1,6 +1,7 @@ +from typing import Any, Dict, Iterable, List, Optional, Tuple + import json from enum import Enum -from typing import Any, Dict, Iterable, List, Optional, Tuple from attrs import define, field, validators diff --git a/parea/utils/trace_integrations/instructor.py b/parea/utils/trace_integrations/instructor.py index af80e9f6..00f02b93 100644 --- a/parea/utils/trace_integrations/instructor.py +++ b/parea/utils/trace_integrations/instructor.py @@ -1,7 +1,7 @@ -from json import JSONDecodeError from typing import Any, Callable, Mapping, Tuple import contextvars +from json import JSONDecodeError from instructor.retry import InstructorRetryException from pydantic import ValidationError diff --git a/parea/wrapper/wrapper.py b/parea/wrapper/wrapper.py index b4782b13..0c35cd77 100644 --- a/parea/wrapper/wrapper.py +++ b/parea/wrapper/wrapper.py @@ -12,9 +12,8 @@ from parea.constants import PAREA_OS_ENV_EXPERIMENT_UUID from parea.evals.utils import _make_evaluations from parea.helpers import is_logging_disabled, timezone_aware_now -from parea.schemas.models import TraceLog, UpdateTraceScenario, UpdateLog -from parea.utils.trace_utils import call_eval_funcs_then_log, execution_order_counters, fill_trace_data, trace_context, \ - trace_data, logger_update_record +from parea.schemas.models import TraceLog, UpdateLog, UpdateTraceScenario +from parea.utils.trace_utils import call_eval_funcs_then_log, execution_order_counters, fill_trace_data, logger_update_record, trace_context, trace_data from parea.wrapper.utils import safe_format_template_to_prompt, skip_decorator_if_func_in_stack logger = logging.getLogger() diff --git a/pyproject.toml b/pyproject.toml index 21f6c464..a09d6cd5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "parea-ai" packages = [{ include = "parea" }] -version = "0.2.178" +version = "0.2.179" description = "Parea python sdk" readme = "README.md" authors = ["joel-parea-ai "]