Merge pull request #346 from parea-ai/refactor-update-default-tokens

fix: provide default for non openai models

parea/evals/utils.py

@@ -128,12 +128,7 @@ def get_tokens(model: str, text: str) -> Union[str, list[int]]:
         return []
     try:
         encoding = tiktoken.encoding_for_model(model)
-        tokens = encoding.encode(text)
     except KeyError:
-        regex = re.compile(r"\b(a|an|the)\b", re.UNICODE)
-        text = text.lower()
-        text = "".join(char for char in text if char not in set(string.punctuation))
-        text = re.sub(regex, " ", text)
-        text = " ".join(text.split())
-        tokens = text.split()
+        encoding = tiktoken.get_encoding("cl100k_base")
+    tokens = encoding.encode(text)
     return tokens

pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "poetry.core.masonry.api"
 [tool.poetry]
 name = "parea-ai"
 packages = [{ include = "parea" }]
-version = "0.2.36"
+version = "0.2.37"
 description = "Parea python sdk"
 readme = "README.md"
 authors = ["joel-parea-ai <[email protected]>"]