Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add method to print TLM model name #284

Merged
merged 5 commits into from
Aug 2, 2024
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions cleanlab_studio/internal/constants.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
"claude-3-sonnet",
"claude-3.5-sonnet",
]
_TLM_DEFAULT_MODEL: str = "gpt-4o-mini"
_TLM_MAX_RETRIES: int = 3 # TODO: finalize this number
TLM_MAX_TOKEN_RANGE: Tuple[int, int] = (64, 512) # (min, max)
TLM_NUM_CANDIDATE_RESPONSES_RANGE: Tuple[int, int] = (1, 20) # (min, max)
Expand Down
21 changes: 15 additions & 6 deletions cleanlab_studio/studio/trustworthy_language_model.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
from cleanlab_studio.internal.constants import (
_TLM_MAX_RETRIES,
_VALID_TLM_QUALITY_PRESETS,
_TLM_DEFAULT_MODEL,
)
from cleanlab_studio.internal.tlm.concurrency import TlmRateHandler
from cleanlab_studio.internal.tlm.validation import (
Expand Down Expand Up @@ -97,10 +98,16 @@ def __init__(
)

self._return_log = False
if options is not None:
validate_tlm_options(options)
if "log" in options.keys():
self._return_log = True

options_dict = options or {}
validate_tlm_options(options_dict)
if "log" in options_dict.keys():
self._return_log = True

# explicitly specify the default model
self._options = {**{"model": _TLM_DEFAULT_MODEL}, **options_dict}

self._quality_preset = quality_preset

if timeout is not None and not (isinstance(timeout, int) or isinstance(timeout, float)):
raise ValidationError("timeout must be a integer or float value")
Expand All @@ -110,8 +117,6 @@ def __init__(

is_notebook_flag = is_notebook()

self._quality_preset = quality_preset
self._options = options
self._timeout = timeout if timeout is not None and timeout > 0 else None
self._verbose = verbose if verbose is not None else is_notebook_flag

Expand Down Expand Up @@ -616,6 +621,10 @@ async def _get_trustworthiness_score_async(
return None
raise e

def get_model_name(self) -> str:
"""Returns the underlying LLM used to obtain responses and scoring trustworthiness."""
huiwengoh marked this conversation as resolved.
Show resolved Hide resolved
return cast(str, self._options["model"])


class TLMResponse(TypedDict):
"""A typed dict containing the response, trustworthiness score and additional logs from the Trustworthy Language Model.
Expand Down
9 changes: 8 additions & 1 deletion cleanlab_studio/utils/tlm_hybrid.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
**This module is not meant to be imported and used directly.** Instead, use [`Studio.TLMHybrid()`](/reference/python/studio/#method-tlmhybrid) to instantiate a [TLMHybrid](#class-tlmhybrid) object, and then you can use the methods like [`prompt()`](#method-prompt) documented on this page.
"""

from typing import List, Optional, Union, cast, Sequence
from typing import List, Dict, Optional, Union, cast, Sequence
import numpy as np

from cleanlab_studio.errors import ValidationError
Expand Down Expand Up @@ -253,3 +253,10 @@ def _try_batch_score(
]
else:
raise ValueError(f"score_response has invalid type")

def get_model_names(self) -> Dict[str, str]:
"""Returns the underlying LLMs used to obtain responses and scoring trustworthiness."""
huiwengoh marked this conversation as resolved.
Show resolved Hide resolved
return {
"response_model": self._tlm_response.get_model_name(),
"score_model": self._tlm_score.get_model_name(),
}
Loading