Skip to content

feat: structured output #944

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
linkliti wants to merge 1 commit into
base: development
Choose a base branch
from
Open

feat: structured output #944

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 18 additions & 1 deletion agent.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,12 +1,13 @@
import asyncio, random, string
import nest_asyncio
from pydantic import BaseModel

nest_asyncio.apply()

from collections import OrderedDict
from dataclasses import dataclass, field
from datetime import datetime, timezone
from typing import Any, Awaitable, Coroutine, Dict, Literal
from typing import Any, Awaitable, Coroutine, Dict, List, Literal
from enum import Enum
import models

Expand Down Expand Up @@ -323,6 +324,13 @@ def __init__(self, **kwargs):
for key, value in kwargs.items():
setattr(self, key, value)

# output structure for agent chat
class AgentResponseSchema(BaseModel):
thoughts: List[str]
headline: str
tool_name: str
tool_args: Dict[str, Any]


# intervention exception class - skips rest of message loop iteration
class InterventionException(Exception):
Expand Down Expand Up @@ -435,6 +443,7 @@ async def stream_callback(chunk: str, full: str):
messages=prompt,
response_callback=stream_callback,
reasoning_callback=reasoning_callback,
allow_structured_output=True,
)

# Notify extensions to finalize their stream filters
Expand Down Expand Up @@ -776,12 +785,19 @@ async def call_chat_model(
response_callback: Callable[[str, str], Awaitable[None]] | None = None,
reasoning_callback: Callable[[str, str], Awaitable[None]] | None = None,
background: bool = False,
allow_structured_output: bool = False,
):
response = ""

# model class
model = self.get_chat_model()

kwargs = {}
# use structured output if agent message and is enabled in config
if allow_structured_output and model.a0_model_conf and model.a0_model_conf.structured_output:
kwargs["response_format"] = AgentResponseSchema
# kwargs["response_format"] = {"type": "json_object"} # alternative

# call model
response, reasoning = await model.unified_call(
messages=messages,
Expand All @@ -790,6 +806,7 @@ async def call_chat_model(
rate_limiter_callback=(
self.rate_limiter_callback if not background else None
),
**kwargs,
)

return response, reasoning
Expand Down
1 change: 1 addition & 0 deletions initialize.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ def _normalize_model_kwargs(kwargs: dict) -> dict:
api_base=current_settings["chat_model_api_base"],
ctx_length=current_settings["chat_model_ctx_length"],
vision=current_settings["chat_model_vision"],
structured_output=current_settings["chat_model_structured_output"],
limit_requests=current_settings["chat_model_rl_requests"],
limit_input=current_settings["chat_model_rl_input"],
limit_output=current_settings["chat_model_rl_output"],
Expand Down
1 change: 1 addition & 0 deletions models.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,7 @@ class ModelConfig:
limit_input: int = 0
limit_output: int = 0
vision: bool = False
structured_output: bool = False
kwargs: dict = field(default_factory=dict)

def build_kwargs(self):
Expand Down
2 changes: 2 additions & 0 deletions python/helpers/settings.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ class Settings(TypedDict):
chat_model_ctx_length: int
chat_model_ctx_history: float
chat_model_vision: bool
chat_model_structured_output: bool
chat_model_rl_requests: int
chat_model_rl_input: int
chat_model_rl_output: int
Expand Down Expand Up @@ -459,6 +460,7 @@ def get_default_settings() -> Settings:
chat_model_ctx_length=get_default_value("chat_model_ctx_length", 100000),
chat_model_ctx_history=get_default_value("chat_model_ctx_history", 0.7),
chat_model_vision=get_default_value("chat_model_vision", True),
chat_model_structured_output=get_default_value("chat_model_structured_output", True),
chat_model_rl_requests=get_default_value("chat_model_rl_requests", 0),
chat_model_rl_input=get_default_value("chat_model_rl_input", 0),
chat_model_rl_output=get_default_value("chat_model_rl_output", 0),
Expand Down
15 changes: 15 additions & 0 deletions webui/components/settings/agent/chat_model.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,21 @@
</div>
</div>

<div class="field">
<div class="field-label">
<div class="field-title">Structured output</div>
<div class="field-description">
Enables strict JSON output for responses to reduce agent message misformatting.
</div>
</div>
<div class="field-control">
<label class="toggle">
<input type="checkbox" x-model="$store.settingsStore.settings.chat_model_structured_output" />
<span class="toggler"></span>
</label>
</div>
</div>

<div class="field">
<div class="field-label">
<div class="field-title">Requests per minute limit</div>
Expand Down