From 0ec2c0df933d8f533dd5129d8f7771346ed52fcd Mon Sep 17 00:00:00 2001 From: Joschka Braun Date: Mon, 15 Jan 2024 19:23:23 -0500 Subject: [PATCH] fix: ignore traces from evaluation metrics --- parea/evals/utils.py | 5 +++-- parea/utils/trace_integrations/langchain.py | 2 +- parea/wrapper/utils.py | 12 +++++++++--- parea/wrapper/wrapper.py | 5 +++-- pyproject.toml | 2 +- 5 files changed, 17 insertions(+), 9 deletions(-) diff --git a/parea/evals/utils.py b/parea/evals/utils.py index a6a3b90c..814f9c09 100644 --- a/parea/evals/utils.py +++ b/parea/evals/utils.py @@ -99,7 +99,8 @@ def ndcg(y_true, ranking): return dcg(y_true, ranking) / best -def make_evaluations(trace_id: str, log: Log, eval_funcs: list[EvalFuncTuple], verbose: bool = False): +# note name is extra odd to make sure that skip_decorator_if_func_in_stack works in 99.9% of cases +def _make_evaluations_(trace_id: str, log: Log, eval_funcs: list[EvalFuncTuple], verbose: bool = False): scores = [NamedEvaluationScore(name=eval.name, score=eval.func(log)) for eval in eval_funcs] if verbose: print(f"###Eval Results###") @@ -113,7 +114,7 @@ def run_evals_in_thread_and_log(trace_id: str, log: Log, eval_funcs: list[EvalFu import threading logging_thread = threading.Thread( - target=make_evaluations, + target=_make_evaluations_, kwargs={"trace_id": trace_id, "log": log, "eval_funcs": eval_funcs, "verbose": verbose}, ) logging_thread.start() diff --git a/parea/utils/trace_integrations/langchain.py b/parea/utils/trace_integrations/langchain.py index 3a1b34e9..e3f89e90 100644 --- a/parea/utils/trace_integrations/langchain.py +++ b/parea/utils/trace_integrations/langchain.py @@ -20,7 +20,7 @@ def _persist_run(self, run: Union[Run, LLMRun, ChainRun, ToolRun]) -> None: data = run.dict() if experiment_uuid := os.getenv(PAREA_OS_ENV_EXPERIMENT_UUID, None): data["experiment_uuid"] = experiment_uuid - parea_logger.record_vendor_log(run.dict(), TraceIntegrations.LANGCHAIN) + parea_logger.record_vendor_log(data, TraceIntegrations.LANGCHAIN) def get_parent_trace_id(self) -> UUID: return self.parent_trace_id diff --git a/parea/wrapper/utils.py b/parea/wrapper/utils.py index 50cc1ecf..ac5f46b9 100644 --- a/parea/wrapper/utils.py +++ b/parea/wrapper/utils.py @@ -1,15 +1,21 @@ from typing import Callable -import inspect +import sys from functools import wraps -def skip_decorator_if_func_in_stack(func_to_check: Callable) -> Callable: +# https://gist.github.com/JettJones/c236494013f22723c1822126df944b12 +def skip_decorator_if_func_in_stack(*funcs_to_check: Callable) -> Callable: def decorator_wrapper(decorator: Callable) -> Callable: def new_decorator(self, func: Callable) -> Callable: # Include self @wraps(func) def wrapper(*args, **kwargs): - if any(func_to_check.__name__ in frame.function for frame in inspect.stack()): + frame = sys._getframe().f_back + caller_names = "" + while frame: + caller_names += frame.f_code.co_name + "|" + frame = frame.f_back + if any(func_to_check.__name__ in caller_names for func_to_check in funcs_to_check): return func(*args, **kwargs) return decorator(self, func)(*args, **kwargs) # Include self diff --git a/parea/wrapper/wrapper.py b/parea/wrapper/wrapper.py index 029a943b..12bdd200 100644 --- a/parea/wrapper/wrapper.py +++ b/parea/wrapper/wrapper.py @@ -9,6 +9,7 @@ from parea.cache.cache import Cache from parea.constants import PAREA_OS_ENV_EXPERIMENT_UUID +from parea.evals.utils import _make_evaluations_ from parea.helpers import date_and_time_string_to_timestamp from parea.schemas.models import TraceLog from parea.utils.trace_utils import call_eval_funcs_then_log, to_date_and_time_string, trace_context, trace_data @@ -96,7 +97,7 @@ def _init_trace(self) -> tuple[str, float]: return trace_id, start_time - @skip_decorator_if_func_in_stack(call_eval_funcs_then_log) + @skip_decorator_if_func_in_stack(call_eval_funcs_then_log, _make_evaluations_) def async_decorator(self, orig_func: Callable) -> Callable: async def wrapper(*args, **kwargs): trace_id, start_time = self._init_trace() @@ -127,7 +128,7 @@ async def wrapper(*args, **kwargs): return wrapper - @skip_decorator_if_func_in_stack(call_eval_funcs_then_log) + @skip_decorator_if_func_in_stack(call_eval_funcs_then_log, _make_evaluations_) def sync_decorator(self, orig_func: Callable) -> Callable: def wrapper(*args, **kwargs): trace_id, start_time = self._init_trace() diff --git a/pyproject.toml b/pyproject.toml index bba6c61e..1995bc3b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "parea-ai" packages = [{ include = "parea" }] -version = "0.2.32" +version = "0.2.33" description = "Parea python sdk" readme = "README.md" authors = ["joel-parea-ai "]