Skip to content

Commit 6ed76ff

Browse files
uc4w6ckrrishdholakia
uc4w6c
and
krrishdholakia
authored
feat: change guardrail_information to list type (#16127)
* feat: change guardrail_information to list type to support displaying multiple guardrails * fix: add missing commit and revert auto-format changes in utils.py --------- Co-authored-by: Krish Dholakia <[email protected]>
1 parent 74ae7ae commit 6ed76ff

File tree

14 files changed

+649
-444
lines changed

14 files changed

+649
-444
lines changed

‎docs/my-website/docs/proxy/logging_spec.md‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -91,7 +91,7 @@ Inherits from `StandardLoggingUserAPIKeyMetadata` and adds:
9191
| `applied_guardrails` | `Optional[List[str]]` | List of applied guardrail names |
9292
| `usage_object` | `Optional[dict]` | Raw usage object from the LLM provider |
9393
| `cold_storage_object_key` | `Optional[str]` | S3/GCS object key for cold storage retrieval |
94-
| `guardrail_information` | `Optional[StandardLoggingGuardrailInformation]` | Guardrail information |
94+
| `guardrail_information` | `Optional[list[StandardLoggingGuardrailInformation]]` | Guardrail information |
9595

9696

9797
## StandardLoggingVectorStoreRequest
@@ -170,7 +170,7 @@ A literal type with two possible values:
170170
| `guardrail_mode` | `Optional[Union[GuardrailEventHooks, List[GuardrailEventHooks]]]` | Guardrail mode |
171171
| `guardrail_request` | `Optional[dict]` | Guardrail request |
172172
| `guardrail_response` | `Optional[Union[dict, str, List[dict]]]` | Guardrail response |
173-
| `guardrail_status` | `Literal["success", "failure", "blocked"]` | Guardrail execution status: `success` = no violations detected, `blocked` = content blocked/modified due to policy violations, `failure` = technical error or API failure |
173+
| `guardrail_status` | `Literal["success", "guardrail_intervened", "guardrail_failed_to_respond"]` | Guardrail execution status: `success` = no violations detected, `blocked` = content blocked/modified due to policy violations, `failure` = technical error or API failure |
174174
| `start_time` | `Optional[float]` | Start time of the guardrail |
175175
| `end_time` | `Optional[float]` | End time of the guardrail |
176176
| `duration` | `Optional[float]` | Duration of the guardrail in seconds |

‎litellm/integrations/custom_guardrail.py‎

Lines changed: 34 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,6 @@ def __init__(
5959
self.mask_response_content: bool = mask_response_content
6060

6161
if supported_event_hooks:
62-
6362
## validate event_hook is in supported_event_hooks
6463
self._validate_event_hook(event_hook, supported_event_hooks)
6564
super().__init__(**kwargs)
@@ -80,7 +79,6 @@ def _validate_event_hook(
8079
],
8180
supported_event_hooks: List[GuardrailEventHooks],
8281
) -> None:
83-
8482
def _validate_event_hook_list_is_in_supported_event_hooks(
8583
event_hook: Union[List[GuardrailEventHooks], List[str]],
8684
supported_event_hooks: List[GuardrailEventHooks],
@@ -130,23 +128,19 @@ def _guardrail_is_in_requested_guardrails(
130128
self,
131129
requested_guardrails: Union[List[str], List[Dict[str, DynamicGuardrailParams]]],
132130
) -> bool:
133-
134131
for _guardrail in requested_guardrails:
135132
if isinstance(_guardrail, dict):
136133
if self.guardrail_name in _guardrail:
137-
138134
return True
139135
elif isinstance(_guardrail, str):
140136
if self.guardrail_name == _guardrail:
141-
142137
return True
143138

144139
return False
145140

146141
async def async_pre_call_deployment_hook(
147142
self, kwargs: Dict[str, Any], call_type: Optional[CallTypes]
148143
) -> Optional[dict]:
149-
150144
from litellm.proxy._types import UserAPIKeyAuth
151145

152146
# should run guardrail
@@ -385,14 +379,24 @@ def add_standard_logging_guardrail_information_to_request_data(
385379
duration=duration,
386380
masked_entity_count=masked_entity_count,
387381
)
382+
383+
def _append_guardrail_info(container: dict) -> None:
384+
key = "standard_logging_guardrail_information"
385+
existing = container.get(key)
386+
if existing is None:
387+
container[key] = [slg]
388+
elif isinstance(existing, list):
389+
existing.append(slg)
390+
else:
391+
# should not happen
392+
container[key] = [existing, slg]
393+
388394
if "metadata" in request_data:
389395
if request_data["metadata"] is None:
390396
request_data["metadata"] = {}
391-
request_data["metadata"]["standard_logging_guardrail_information"] = slg
397+
_append_guardrail_info(request_data["metadata"])
392398
elif "litellm_metadata" in request_data:
393-
request_data["litellm_metadata"][
394-
"standard_logging_guardrail_information"
395-
] = slg
399+
_append_guardrail_info(request_data["litellm_metadata"])
396400
else:
397401
verbose_logger.warning(
398402
"unable to log guardrail information. No metadata found in request_data"
@@ -497,37 +501,46 @@ def update_in_memory_litellm_params(self, litellm_params: LitellmParams) -> None
497501
"""
498502
for key, value in vars(litellm_params).items():
499503
setattr(self, key, value)
500-
501-
def get_guardrails_messages_for_call_type(self, call_type: CallTypes, data: Optional[dict] = None) -> Optional[List[AllMessageValues]]:
504+
505+
def get_guardrails_messages_for_call_type(
506+
self, call_type: CallTypes, data: Optional[dict] = None
507+
) -> Optional[List[AllMessageValues]]:
502508
"""
503509
Returns the messages for the given call type and data
504510
"""
505511
if call_type is None or data is None:
506512
return None
507-
513+
508514
#########################################################
509-
# /chat/completions
510-
# /messages
515+
# /chat/completions
516+
# /messages
511517
# Both endpoints store the messages in the "messages" key
512518
#########################################################
513-
if call_type == CallTypes.completion.value or call_type == CallTypes.acompletion.value or call_type == CallTypes.anthropic_messages.value:
519+
if (
520+
call_type == CallTypes.completion.value
521+
or call_type == CallTypes.acompletion.value
522+
or call_type == CallTypes.anthropic_messages.value
523+
):
514524
return data.get("messages")
515-
525+
516526
#########################################################
517-
# /responses
527+
# /responses
518528
# User/System messages are stored in the "input" key, use litellm transformation to get the messages
519529
#########################################################
520-
if call_type == CallTypes.responses.value or call_type == CallTypes.aresponses.value:
530+
if (
531+
call_type == CallTypes.responses.value
532+
or call_type == CallTypes.aresponses.value
533+
):
521534
from typing import cast
522535

523536
from litellm.responses.litellm_completion_transformation.transformation import (
524537
LiteLLMCompletionResponsesConfig,
525538
)
526-
539+
527540
input_data = data.get("input")
528541
if input_data is None:
529542
return None
530-
543+
531544
messages = LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages(
532545
input=input_data,
533546
responses_api_request=data,

‎litellm/integrations/datadog/datadog_llm_obs.py‎

Lines changed: 15 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -498,7 +498,9 @@ def _get_dd_llm_obs_payload_metadata(
498498
"guardrail_information": standard_logging_payload.get(
499499
"guardrail_information", None
500500
),
501-
"is_streamed_request": self._get_stream_value_from_payload(standard_logging_payload),
501+
"is_streamed_request": self._get_stream_value_from_payload(
502+
standard_logging_payload
503+
),
502504
}
503505

504506
#########################################################
@@ -548,21 +550,24 @@ def _get_latency_metrics(
548550

549551
# Guardrail overhead latency
550552
guardrail_info: Optional[
551-
StandardLoggingGuardrailInformation
553+
list[StandardLoggingGuardrailInformation]
552554
] = standard_logging_payload.get("guardrail_information")
553555
if guardrail_info is not None:
554-
_guardrail_duration_seconds: Optional[float] = guardrail_info.get(
555-
"duration"
556-
)
557-
if _guardrail_duration_seconds is not None:
556+
total_duration = 0.0
557+
for info in guardrail_info:
558+
_guardrail_duration_seconds: Optional[float] = info.get("duration")
559+
if _guardrail_duration_seconds is not None:
560+
total_duration += float(_guardrail_duration_seconds)
561+
562+
if total_duration > 0:
558563
# Convert from seconds to milliseconds for consistency
559-
latency_metrics["guardrail_overhead_time_ms"] = (
560-
_guardrail_duration_seconds * 1000
561-
)
564+
latency_metrics["guardrail_overhead_time_ms"] = total_duration * 1000
562565

563566
return latency_metrics
564567

565-
def _get_stream_value_from_payload(self, standard_logging_payload: StandardLoggingPayload) -> bool:
568+
def _get_stream_value_from_payload(
569+
self, standard_logging_payload: StandardLoggingPayload
570+
) -> bool:
566571
"""
567572
Extract the stream value from standard logging payload.
568573

‎litellm/integrations/langfuse/langfuse.py‎

Lines changed: 45 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -688,11 +688,17 @@ def _log_langfuse_v2( # noqa: PLR0915
688688
"completion_tokens": _usage_obj.completion_tokens,
689689
"total_cost": cost if self._supports_costs() else None,
690690
}
691-
usage_details = LangfuseUsageDetails(input=_usage_obj.prompt_tokens,
692-
output=_usage_obj.completion_tokens,
693-
total=_usage_obj.total_tokens,
694-
cache_creation_input_tokens=_usage_obj.get('cache_creation_input_tokens', 0),
695-
cache_read_input_tokens=_usage_obj.get('cache_read_input_tokens', 0))
691+
usage_details = LangfuseUsageDetails(
692+
input=_usage_obj.prompt_tokens,
693+
output=_usage_obj.completion_tokens,
694+
total=_usage_obj.total_tokens,
695+
cache_creation_input_tokens=_usage_obj.get(
696+
"cache_creation_input_tokens", 0
697+
),
698+
cache_read_input_tokens=_usage_obj.get(
699+
"cache_read_input_tokens", 0
700+
),
701+
)
696702

697703
generation_name = clean_metadata.pop("generation_name", None)
698704
if generation_name is None:
@@ -790,7 +796,7 @@ def _get_responses_api_content_for_langfuse(
790796
"""
791797
Get the responses API content for Langfuse logging
792798
"""
793-
if hasattr(response_obj, 'output') and response_obj.output:
799+
if hasattr(response_obj, "output") and response_obj.output:
794800
# ResponsesAPIResponse.output is a list of strings
795801
return response_obj.output
796802
else:
@@ -880,29 +886,44 @@ def _log_guardrail_information_as_span(
880886
guardrail_information = standard_logging_object.get(
881887
"guardrail_information", None
882888
)
883-
if guardrail_information is None:
889+
if not guardrail_information:
884890
verbose_logger.debug(
885-
"Not logging guardrail information as span because guardrail_information is None"
891+
"Not logging guardrail information as span because guardrail_information is empty"
886892
)
887893
return
888894

889-
span = trace.span(
890-
name="guardrail",
891-
input=guardrail_information.get("guardrail_request", None),
892-
output=guardrail_information.get("guardrail_response", None),
893-
metadata={
894-
"guardrail_name": guardrail_information.get("guardrail_name", None),
895-
"guardrail_mode": guardrail_information.get("guardrail_mode", None),
896-
"guardrail_masked_entity_count": guardrail_information.get(
897-
"masked_entity_count", None
898-
),
899-
},
900-
start_time=guardrail_information.get("start_time", None), # type: ignore
901-
end_time=guardrail_information.get("end_time", None), # type: ignore
902-
)
895+
if not isinstance(guardrail_information, list):
896+
verbose_logger.debug(
897+
"Not logging guardrail information as span because guardrail_information is not a list: %s",
898+
type(guardrail_information),
899+
)
900+
return
901+
902+
for guardrail_entry in guardrail_information:
903+
if not isinstance(guardrail_entry, dict):
904+
verbose_logger.debug(
905+
"Skipping guardrail entry with unexpected type: %s",
906+
type(guardrail_entry),
907+
)
908+
continue
909+
910+
span = trace.span(
911+
name="guardrail",
912+
input=guardrail_entry.get("guardrail_request", None),
913+
output=guardrail_entry.get("guardrail_response", None),
914+
metadata={
915+
"guardrail_name": guardrail_entry.get("guardrail_name", None),
916+
"guardrail_mode": guardrail_entry.get("guardrail_mode", None),
917+
"guardrail_masked_entity_count": guardrail_entry.get(
918+
"masked_entity_count", None
919+
),
920+
},
921+
start_time=guardrail_entry.get("start_time", None), # type: ignore
922+
end_time=guardrail_entry.get("end_time", None), # type: ignore
923+
)
903924

904-
verbose_logger.debug(f"Logged guardrail information as span: {span}")
905-
span.end()
925+
verbose_logger.debug(f"Logged guardrail information as span: {span}")
926+
span.end()
906927

907928

908929
def _add_prompt_to_generation_params(

0 commit comments

Comments
 (0)