diff --git a/parea/evals/utils.py b/parea/evals/utils.py index 4db9f3f9..646afd5c 100644 --- a/parea/evals/utils.py +++ b/parea/evals/utils.py @@ -169,10 +169,11 @@ def run_evals_synchronous(trace_id: str, log: Log, eval_funcs: List[EvalFuncTupl def get_tokens(model: str, text: str) -> List[int]: if not text: return [] + fallback_model = "cl100k_base" try: - encoding = tiktoken.encoding_for_model(model) + encoding = tiktoken.encoding_for_model(model or fallback_model) except KeyError: - encoding = tiktoken.get_encoding("cl100k_base") + encoding = tiktoken.get_encoding(fallback_model) try: return encoding.encode(text) except Exception as e: diff --git a/pyproject.toml b/pyproject.toml index d60146b8..b9e23084 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "parea-ai" packages = [{ include = "parea" }] -version = "0.2.129" +version = "0.2.130" description = "Parea python sdk" readme = "README.md" authors = ["joel-parea-ai "]