Skip to content

Commit

Permalink
fix: truncated error messages in plain text tokenizer (#172)
Browse files Browse the repository at this point in the history
  • Loading branch information
adubovik authored Nov 14, 2024
1 parent 1e65165 commit 837a7a0
Showing 1 changed file with 4 additions and 3 deletions.
7 changes: 4 additions & 3 deletions aidial_adapter_openai/utils/tokenizer.py
Original file line number Diff line number Diff line change
Expand Up @@ -96,15 +96,15 @@ def _process_raw_message(
pass
else:
raise InternalServerError(
f"Unexpected type of content in message: {value!r}"
f"Unexpected type of content in message: {type(value)}"
)

elif key == "role":
if isinstance(value, str):
tokens += calculate_text_tokens(value)
else:
raise InternalServerError(
f"Unexpected type of 'role' field in message: {value!r}"
f"Unexpected type of 'role' field in message: {type(value)}"
)
return tokens

Expand All @@ -116,8 +116,9 @@ class PlainTextTokenizer(BaseTokenizer[dict]):
"""

def _handle_custom_content_part(self, content_part: Any):
short_content_part = str(content_part)[:100]
raise InternalServerError(
f"Unexpected type of content in message: {content_part!r}"
f"Unexpected type of content in message: {short_content_part!r}"
f"Use MultiModalTokenizer for messages with images"
)

Expand Down

0 comments on commit 837a7a0

Please sign in to comment.