fix: resolve SessionInput state serialization to cloud evaluation service

PiperOrigin-RevId: 924891381

Author: vertex-sdk-bot

agentplatform/_genai/evals.py

@@ -43,6 +43,56 @@
 logger = logging.getLogger("agentplatform_genai.evals")
 
 
+def _AgentRunConfig_from_vertex(
+    from_object: Union[dict[str, Any], object],
+    parent_object: Optional[dict[str, Any]] = None,
+) -> dict[str, Any]:
+    to_object: dict[str, Any] = {}
+    if getv(from_object, ["sessionInput"]) is not None:
+        setv(
+            to_object,
+            ["session_input"],
+            _SessionInput_from_vertex(getv(from_object, ["sessionInput"]), to_object),
+        )
+
+    if getv(from_object, ["agentEngine"]) is not None:
+        setv(to_object, ["agent_engine"], getv(from_object, ["agentEngine"]))
+
+    if getv(from_object, ["userSimulatorConfig"]) is not None:
+        setv(
+            to_object,
+            ["user_simulator_config"],
+            getv(from_object, ["userSimulatorConfig"]),
+        )
+
+    return to_object
+
+
+def _AgentRunConfig_to_vertex(
+    from_object: Union[dict[str, Any], object],
+    parent_object: Optional[dict[str, Any]] = None,
+) -> dict[str, Any]:
+    to_object: dict[str, Any] = {}
+    if getv(from_object, ["session_input"]) is not None:
+        setv(
+            to_object,
+            ["sessionInput"],
+            _SessionInput_to_vertex(getv(from_object, ["session_input"]), to_object),
+        )
+
+    if getv(from_object, ["agent_engine"]) is not None:
+        setv(to_object, ["agentEngine"], getv(from_object, ["agent_engine"]))
+
+    if getv(from_object, ["user_simulator_config"]) is not None:
+        setv(
+            to_object,
+            ["userSimulatorConfig"],
+            getv(from_object, ["user_simulator_config"]),
+        )
+
+    return to_object
+
+
 def _CreateEvaluationItemParameters_to_vertex(
     from_object: Union[dict[str, Any], object],
     parent_object: Optional[dict[str, Any]] = None,
@@ -137,6 +187,9 @@ def _CreateEvaluationRunParameters_to_vertex(
             [item for item in getv(from_object, ["analysis_configs"])],
         )
 
+    if getv(from_object, ["dummy_session_input"]) is not None:
+        _SessionInput_to_vertex(getv(from_object, ["dummy_session_input"]), to_object)
+
     return to_object
 
 
@@ -464,7 +517,13 @@ def _EvaluationRunInferenceConfig_from_vertex(
         setv(to_object, ["prompt_template"], getv(from_object, ["promptTemplate"]))
 
     if getv(from_object, ["agentRunConfig"]) is not None:
-        setv(to_object, ["agent_run_config"], getv(from_object, ["agentRunConfig"]))
+        setv(
+            to_object,
+            ["agent_run_config"],
+            _AgentRunConfig_from_vertex(
+                getv(from_object, ["agentRunConfig"]), to_object
+            ),
+        )
 
     if getv(from_object, ["agents"]) is not None:
         setv(to_object, ["agent_configs"], getv(from_object, ["agents"]))
@@ -487,7 +546,13 @@ def _EvaluationRunInferenceConfig_to_vertex(
         setv(to_object, ["promptTemplate"], getv(from_object, ["prompt_template"]))
 
     if getv(from_object, ["agent_run_config"]) is not None:
-        setv(to_object, ["agentRunConfig"], getv(from_object, ["agent_run_config"]))
+        setv(
+            to_object,
+            ["agentRunConfig"],
+            _AgentRunConfig_to_vertex(
+                getv(from_object, ["agent_run_config"]), to_object
+            ),
+        )
 
     if getv(from_object, ["agent_configs"]) is not None:
         setv(to_object, ["agents"], getv(from_object, ["agent_configs"]))
@@ -902,6 +967,40 @@ def _RubricBasedMetricSpec_to_vertex(
     return to_object
 
 
+def _SessionInput_from_vertex(
+    from_object: Union[dict[str, Any], object],
+    parent_object: Optional[dict[str, Any]] = None,
+) -> dict[str, Any]:
+    to_object: dict[str, Any] = {}
+    if getv(from_object, ["userId"]) is not None:
+        setv(to_object, ["user_id"], getv(from_object, ["userId"]))
+
+    if getv(from_object, ["sessionState"]) is not None:
+        setv(to_object, ["state"], getv(from_object, ["sessionState"]))
+
+    if getv(from_object, ["parameters", "app_name"]) is not None:
+        setv(to_object, ["app_name"], getv(from_object, ["parameters", "app_name"]))
+
+    return to_object
+
+
+def _SessionInput_to_vertex(
+    from_object: Union[dict[str, Any], object],
+    parent_object: Optional[dict[str, Any]] = None,
+) -> dict[str, Any]:
+    to_object: dict[str, Any] = {}
+    if getv(from_object, ["user_id"]) is not None:
+        setv(to_object, ["userId"], getv(from_object, ["user_id"]))
+
+    if getv(from_object, ["state"]) is not None:
+        setv(to_object, ["sessionState"], getv(from_object, ["state"]))
+
+    if getv(from_object, ["app_name"]) is not None:
+        setv(to_object, ["parameters", "app_name"], getv(from_object, ["app_name"]))
+
+    return to_object
+
+
 def _UnifiedMetric_from_vertex(
     from_object: Union[dict[str, Any], object],
     parent_object: Optional[dict[str, Any]] = None,
@@ -1174,6 +1273,10 @@ def _create_evaluation_run(
         ] = None,
         config: Optional[types.CreateEvaluationRunConfigOrDict] = None,
         analysis_configs: Optional[list[types.AnalysisConfigOrDict]] = None,
+        dummy_session_input: Optional[types.evals.SessionInputOrDict] = None,
+        dummy_user_simulator_config: Optional[
+            types.evals.UserSimulatorConfigOrDict
+        ] = None,
     ) -> types.EvaluationRun:
         """
         Creates an EvaluationRun.
@@ -1188,6 +1291,8 @@ def _create_evaluation_run(
             inference_configs=inference_configs,
             config=config,
             analysis_configs=analysis_configs,
+            dummy_session_input=dummy_session_input,
+            dummy_user_simulator_config=dummy_user_simulator_config,
         )
 
         request_url_dict: Optional[dict[str, str]]
@@ -3321,6 +3426,10 @@ async def _create_evaluation_run(
         ] = None,
         config: Optional[types.CreateEvaluationRunConfigOrDict] = None,
         analysis_configs: Optional[list[types.AnalysisConfigOrDict]] = None,
+        dummy_session_input: Optional[types.evals.SessionInputOrDict] = None,
+        dummy_user_simulator_config: Optional[
+            types.evals.UserSimulatorConfigOrDict
+        ] = None,
     ) -> types.EvaluationRun:
         """
         Creates an EvaluationRun.
@@ -3335,6 +3444,8 @@ async def _create_evaluation_run(
             inference_configs=inference_configs,
             config=config,
             analysis_configs=analysis_configs,
+            dummy_session_input=dummy_session_input,
+            dummy_user_simulator_config=dummy_user_simulator_config,
         )
 
         request_url_dict: Optional[dict[str, str]]

agentplatform/_genai/types/common.py

@@ -2560,13 +2560,13 @@ class AgentRunConfig(_common.BaseModel):
 class AgentRunConfigDict(TypedDict, total=False):
     """Configuration for an Agent Run."""
 
-    session_input: Optional[evals_types.SessionInput]
+    session_input: Optional[evals_types.SessionInputDict]
     """The session input to get agent running results."""
 
     agent_engine: Optional[str]
     """The resource name of the Agent Engine."""
 
-    user_simulator_config: Optional[evals_types.UserSimulatorConfig]
+    user_simulator_config: Optional[evals_types.UserSimulatorConfigDict]
     """Used for multi-turn agent run.
         Contains configuration for a user simulator that
         uses an LLM to generate messages on behalf of the user."""
@@ -2765,6 +2765,12 @@ class _CreateEvaluationRunParameters(_common.BaseModel):
     analysis_configs: Optional[list[AnalysisConfig]] = Field(
         default=None, description=""""""
     )
+    dummy_session_input: Optional[evals_types.SessionInput] = Field(
+        default=None, description=""""""
+    )
+    dummy_user_simulator_config: Optional[evals_types.UserSimulatorConfig] = Field(
+        default=None, description=""""""
+    )
 
 
 class _CreateEvaluationRunParametersDict(TypedDict, total=False):
@@ -2794,6 +2800,12 @@ class _CreateEvaluationRunParametersDict(TypedDict, total=False):
     analysis_configs: Optional[list[AnalysisConfigDict]]
     """"""
 
+    dummy_session_input: Optional[evals_types.SessionInputDict]
+    """"""
+
+    dummy_user_simulator_config: Optional[evals_types.UserSimulatorConfigDict]
+    """"""
+
 
 _CreateEvaluationRunParametersOrDict = Union[
     _CreateEvaluationRunParameters, _CreateEvaluationRunParametersDict

agentplatform/_genai/types/evals.py

@@ -36,6 +36,85 @@ class Importance(_common.CaseInSensitiveEnum):
     """Low importance."""
 
 
+class SessionInput(_common.BaseModel):
+    """This field is experimental and may change in future versions.
+
+    Input to initialize a session and run an agent, used for agent evaluation.
+    """
+
+    user_id: Optional[str] = Field(default=None, description="""The user id.""")
+    state: Optional[dict[str, str]] = Field(
+        default=None, description="""The state of the session."""
+    )
+    app_name: Optional[str] = Field(
+        default=None,
+        description="""The name of the app, used for local ADK agent run Runner and Session.""",
+    )
+
+
+class SessionInputDict(TypedDict, total=False):
+    """This field is experimental and may change in future versions.
+
+    Input to initialize a session and run an agent, used for agent evaluation.
+    """
+
+    user_id: Optional[str]
+    """The user id."""
+
+    state: Optional[dict[str, str]]
+    """The state of the session."""
+
+    app_name: Optional[str]
+    """The name of the app, used for local ADK agent run Runner and Session."""
+
+
+SessionInputOrDict = Union[SessionInput, SessionInputDict]
+
+
+class UserSimulatorConfig(_common.BaseModel):
+    """Configuration for a user simulator.
+
+    Uses an LLM to generate multi-turn messages that simulate a user.
+    """
+
+    model_name: Optional[str] = Field(
+        default=None,
+        description="""The model name to get next user message for multi-turn agent run.""",
+    )
+    model_configuration: Optional[genai_types.GenerateContentConfig] = Field(
+        default=None, description="""The configuration for the model."""
+    )
+    max_turn: Optional[int] = Field(
+        default=None,
+        description="""Maximum number of invocations allowed by the multi-turn agent
+      running. This property allows us to stop a run-off conversation
+      where the agent and the user simulator get into a never ending loop.
+      The initial fixed prompt is also counted as an invocation.""",
+    )
+
+
+class UserSimulatorConfigDict(TypedDict, total=False):
+    """Configuration for a user simulator.
+
+    Uses an LLM to generate multi-turn messages that simulate a user.
+    """
+
+    model_name: Optional[str]
+    """The model name to get next user message for multi-turn agent run."""
+
+    model_configuration: Optional[genai_types.GenerateContentConfigDict]
+    """The configuration for the model."""
+
+    max_turn: Optional[int]
+    """Maximum number of invocations allowed by the multi-turn agent
+      running. This property allows us to stop a run-off conversation
+      where the agent and the user simulator get into a never ending loop.
+      The initial fixed prompt is also counted as an invocation."""
+
+
+UserSimulatorConfigOrDict = Union[UserSimulatorConfig, UserSimulatorConfigDict]
+
+
 class AgentConfig(_common.BaseModel):
     """Represents configuration for an Agent."""
 
@@ -456,41 +535,6 @@ class AgentInfoDict(TypedDict, total=False):
 AgentInfoOrDict = Union[AgentInfo, AgentInfoDict]
 
 
-class SessionInput(_common.BaseModel):
-    """This field is experimental and may change in future versions.
-
-    Input to initialize a session and run an agent, used for agent evaluation.
-    """
-
-    user_id: Optional[str] = Field(default=None, description="""The user id.""")
-    state: Optional[dict[str, str]] = Field(
-        default=None, description="""The state of the session."""
-    )
-    app_name: Optional[str] = Field(
-        default=None,
-        description="""The name of the app, used for local ADK agent run Runner and Session.""",
-    )
-
-
-class SessionInputDict(TypedDict, total=False):
-    """This field is experimental and may change in future versions.
-
-    Input to initialize a session and run an agent, used for agent evaluation.
-    """
-
-    user_id: Optional[str]
-    """The user id."""
-
-    state: Optional[dict[str, str]]
-    """The state of the session."""
-
-    app_name: Optional[str]
-    """The name of the app, used for local ADK agent run Runner and Session."""
-
-
-SessionInputOrDict = Union[SessionInput, SessionInputDict]
-
-
 class UserScenario(_common.BaseModel):
     """User scenario to help simulate multi-turn agent run results."""
 
@@ -586,50 +630,6 @@ class UserScenarioGenerationConfigDict(TypedDict, total=False):
 ]
 
 
-class UserSimulatorConfig(_common.BaseModel):
-    """Configuration for a user simulator.
-
-    Uses an LLM to generate multi-turn messages that simulate a user.
-    """
-
-    model_name: Optional[str] = Field(
-        default=None,
-        description="""The model name to get next user message for multi-turn agent run.""",
-    )
-    model_configuration: Optional[genai_types.GenerateContentConfig] = Field(
-        default=None, description="""The configuration for the model."""
-    )
-    max_turn: Optional[int] = Field(
-        default=None,
-        description="""Maximum number of invocations allowed by the multi-turn agent
-      running. This property allows us to stop a run-off conversation
-      where the agent and the user simulator get into a never ending loop.
-      The initial fixed prompt is also counted as an invocation.""",
-    )
-
-
-class UserSimulatorConfigDict(TypedDict, total=False):
-    """Configuration for a user simulator.
-
-    Uses an LLM to generate multi-turn messages that simulate a user.
-    """
-
-    model_name: Optional[str]
-    """The model name to get next user message for multi-turn agent run."""
-
-    model_configuration: Optional[genai_types.GenerateContentConfigDict]
-    """The configuration for the model."""
-
-    max_turn: Optional[int]
-    """Maximum number of invocations allowed by the multi-turn agent
-      running. This property allows us to stop a run-off conversation
-      where the agent and the user simulator get into a never ending loop.
-      The initial fixed prompt is also counted as an invocation."""
-
-
-UserSimulatorConfigOrDict = Union[UserSimulatorConfig, UserSimulatorConfigDict]
-
-
 class Event(_common.BaseModel):
     """Represents an event in a conversation between agents and users.