Merge pull request #263 from bespokelabsai/ryanm/null-response

Small fixes
bespokelabsai · Dec 16, 2024 · 739c153 · 739c153
2 parents 0470c6e + 6e77a81
commit 739c153
Show file tree

Hide file tree

Showing 3 changed files with 10 additions and 7 deletions.
diff --git a/src/bespokelabs/curator/llm/llm.py b/src/bespokelabs/curator/llm/llm.py
@@ -58,7 +58,7 @@ def __init__(
         presence_penalty: Optional[float] = None,
         frequency_penalty: Optional[float] = None,
         max_retries: Optional[int] = None,
-        require_all_responses: Optional[bool] = None,
+        require_all_responses: Optional[bool] = True,
     ):
         """Initialize a LLM.
 

diff --git a/src/bespokelabs/curator/request_processor/base_online_request_processor.py b/src/bespokelabs/curator/request_processor/base_online_request_processor.py
@@ -481,9 +481,6 @@ async def handle_single_request_with_retries(
             status_tracker.num_tasks_succeeded += 1
             status_tracker.pbar.update(1)
 
-            if generic_response.response_message is None:
-                raise ValueError(f"Request {request.task_id} returned no response message")
-
         except Exception as e:
             logger.warning(
                 f"Request {request.task_id} failed with Exception {e}, attempts left {request.attempts_left}"

diff --git a/src/bespokelabs/curator/request_processor/litellm_online_request_processor.py b/src/bespokelabs/curator/request_processor/litellm_online_request_processor.py
@@ -18,6 +18,7 @@
 logger = logging.getLogger(__name__)
 
 litellm.suppress_debug_info = True
+REQUEST_TIMEOUT = 10 * 60.0  # same as openai python sdk
 
 
 class LiteLLMOnlineRequestProcessor(BaseOnlineRequestProcessor):
@@ -50,7 +51,7 @@ def __init__(
         frequency_penalty: Optional[float] = None,
         max_requests_per_minute: Optional[int] = None,
         max_tokens_per_minute: Optional[int] = None,
-        require_all_responses: bool = False,
+        require_all_responses: Optional[bool] = None,
         max_retries: Optional[int] = None,
     ):
         super().__init__(
@@ -268,15 +269,15 @@ async def call_single_request(
                     await self.client.chat.completions.create_with_completion(
                         **request.api_specific_request,
                         response_model=request.prompt_formatter.response_format,
-                        timeout=60.0,
+                        timeout=REQUEST_TIMEOUT,
                     )
                 )
                 response_message = (
                     response.model_dump() if hasattr(response, "model_dump") else response
                 )
             else:
                 completion_obj = await litellm.acompletion(
-                    **request.api_specific_request, timeout=60.0
+                    **request.api_specific_request, timeout=REQUEST_TIMEOUT
                 )
                 response_message = completion_obj["choices"][0]["message"]["content"]
         except litellm.RateLimitError as e:
@@ -300,6 +301,11 @@ async def call_single_request(
         except litellm.NotFoundError as e:
             cost = 0
 
+        if response_message is None:
+            raise ValueError(
+                f"Request {request.task_id} returned no response message with raw response {completion_obj.model_dump()}"
+            )
+
         # Create and return response
         return GenericResponse(
             response_message=response_message,