Merge pull request #327 from parea-ai/fix-langchain-experiment-logging

fix: ignore traces from evaluation metrics

parea/evals/utils.py

@@ -99,7 +99,8 @@ def ndcg(y_true, ranking):
     return dcg(y_true, ranking) / best
 
 
-def make_evaluations(trace_id: str, log: Log, eval_funcs: list[EvalFuncTuple], verbose: bool = False):
+# note name is extra odd to make sure that skip_decorator_if_func_in_stack works in 99.9% of cases
+def _make_evaluations_(trace_id: str, log: Log, eval_funcs: list[EvalFuncTuple], verbose: bool = False):
     scores = [NamedEvaluationScore(name=eval.name, score=eval.func(log)) for eval in eval_funcs]
     if verbose:
         print(f"###Eval Results###")
@@ -113,7 +114,7 @@ def run_evals_in_thread_and_log(trace_id: str, log: Log, eval_funcs: list[EvalFu
     import threading
 
     logging_thread = threading.Thread(
-        target=make_evaluations,
+        target=_make_evaluations_,
         kwargs={"trace_id": trace_id, "log": log, "eval_funcs": eval_funcs, "verbose": verbose},
     )
     logging_thread.start()

parea/utils/trace_integrations/langchain.py

@@ -20,7 +20,7 @@ def _persist_run(self, run: Union[Run, LLMRun, ChainRun, ToolRun]) -> None:
         data = run.dict()
         if experiment_uuid := os.getenv(PAREA_OS_ENV_EXPERIMENT_UUID, None):
             data["experiment_uuid"] = experiment_uuid
-        parea_logger.record_vendor_log(run.dict(), TraceIntegrations.LANGCHAIN)
+        parea_logger.record_vendor_log(data, TraceIntegrations.LANGCHAIN)
 
     def get_parent_trace_id(self) -> UUID:
         return self.parent_trace_id

parea/wrapper/utils.py

@@ -1,15 +1,21 @@
 from typing import Callable
 
-import inspect
+import sys
 from functools import wraps
 
 
-def skip_decorator_if_func_in_stack(func_to_check: Callable) -> Callable:
+# https://gist.github.com/JettJones/c236494013f22723c1822126df944b12
+def skip_decorator_if_func_in_stack(*funcs_to_check: Callable) -> Callable:
     def decorator_wrapper(decorator: Callable) -> Callable:
         def new_decorator(self, func: Callable) -> Callable:  # Include self
             @wraps(func)
             def wrapper(*args, **kwargs):
-                if any(func_to_check.__name__ in frame.function for frame in inspect.stack()):
+                frame = sys._getframe().f_back
+                caller_names = ""
+                while frame:
+                    caller_names += frame.f_code.co_name + "|"
+                    frame = frame.f_back
+                if any(func_to_check.__name__ in caller_names for func_to_check in funcs_to_check):
                     return func(*args, **kwargs)
                 return decorator(self, func)(*args, **kwargs)  # Include self
 

parea/wrapper/wrapper.py

@@ -9,6 +9,7 @@
 
 from parea.cache.cache import Cache
 from parea.constants import PAREA_OS_ENV_EXPERIMENT_UUID
+from parea.evals.utils import _make_evaluations_
 from parea.helpers import date_and_time_string_to_timestamp
 from parea.schemas.models import TraceLog
 from parea.utils.trace_utils import call_eval_funcs_then_log, to_date_and_time_string, trace_context, trace_data
@@ -96,7 +97,7 @@ def _init_trace(self) -> tuple[str, float]:
 
         return trace_id, start_time
 
-    @skip_decorator_if_func_in_stack(call_eval_funcs_then_log)
+    @skip_decorator_if_func_in_stack(call_eval_funcs_then_log, _make_evaluations_)
     def async_decorator(self, orig_func: Callable) -> Callable:
         async def wrapper(*args, **kwargs):
             trace_id, start_time = self._init_trace()
@@ -127,7 +128,7 @@ async def wrapper(*args, **kwargs):
 
         return wrapper
 
-    @skip_decorator_if_func_in_stack(call_eval_funcs_then_log)
+    @skip_decorator_if_func_in_stack(call_eval_funcs_then_log, _make_evaluations_)
     def sync_decorator(self, orig_func: Callable) -> Callable:
         def wrapper(*args, **kwargs):
             trace_id, start_time = self._init_trace()

pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "poetry.core.masonry.api"
 [tool.poetry]
 name = "parea-ai"
 packages = [{ include = "parea" }]
-version = "0.2.32"
+version = "0.2.33"
 description = "Parea python sdk"
 readme = "README.md"
 authors = ["joel-parea-ai <[email protected]>"]