Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .env.example
Original file line number Diff line number Diff line change
Expand Up @@ -223,6 +223,9 @@ EVA_MODEL_LIST='[
#i Seconds of user silence after the assistant stops speaking before nudging it to reprompt
#i the caller (Pipecat cascade + audio-LLM pipelines). Cancelled the moment the user starts
#i speaking, so it only fires on genuine silence when turn detection drops a user turn.
#i Also applies to Cartesia (turn.inactivity_check_in_secs) and Phonic (no_input_poke_sec,
#i "Respond after"); provision those agents with the same value via
#i scripts/provision_cartesia_agents.py / scripts/provision_phonic_agents.py.
#i Leave unset to keep the old behavior of ending the conversation once the provider's
#i inactivity timeout elapses.
#d int
Expand Down
1 change: 1 addition & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ dependencies = [
"azure-cognitiveservices-speech>=1.31.0",
"cartesia>=3.0.0",
"assemblyai>=0.17.0",
"phonic>=0.33.9",
"setuptools>=65.0.0",
"fastapi>=0.100.0",
"uvicorn>=0.23.0",
Expand Down
29 changes: 18 additions & 11 deletions scripts/provision_cartesia_agents.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
set here from EVA's own options, and the assistant server refuses to run against an agent whose
settings do not match the run config:

--language -> language.primary (EVA_LANGUAGE, default en; fr-CA uses fr)
--language -> language.primary (EVA_LANGUAGE, default en; no region tags)
--pre-tool-speech -> each tool's pre_tool_speech (EVA_MODEL__PRE_TOOL_SPEECH, default auto)
--turn-end-fallback-time -> turn.inactivity_check_in_secs (EVA_TURN_END_FALLBACK_TIME, default unset)

Expand All @@ -19,7 +19,9 @@
[--pre-tool-speech auto|off] [--turn-end-fallback-time SECONDS] [--domains airline itsm medical_hr]

Reads CARTESIA_API_KEY from the environment or .env. Tool and agent IDs are kept in
output/cartesia_agents.json so a rerun updates the same agents. Prints each domain's
output/cartesia_agents.json. Agents are keyed by domain and language, so a rerun updates the same agent
for that domain and language while other languages keep their own agents. Tools are keyed by domain and
tool name and shared by every language's agent for that domain. Prints each domain's
EVA_MODEL__S2S_PARAMS (without the api_key) to use for the run.
"""

Expand All @@ -33,7 +35,7 @@
from dotenv import dotenv_values

from eva.models.agents import AgentConfig
from eva.models.config import LANGUAGE_DISPLAY_NAMES, ModelConfig, base_language
from eva.models.config import LANGUAGE_DISPLAY_NAMES, ModelConfig
from eva.utils import cartesia_agents

DOMAINS = ("airline", "itsm", "medical_hr")
Expand Down Expand Up @@ -63,7 +65,7 @@ def agent_config(
"initial_message": cartesia_agents.INITIAL_MESSAGE_VARIABLE,
"speculative_generation": True,
"model": {"id": model, "fallback_strategy": None},
"language": {"primary": base_language(language)},
"language": {"primary": language},
"turn": {"inactivity_check_in_secs": turn_end_fallback_time},
"audio": {
"input": {"noise_suppression": "off", "asr_model": "auto"},
Expand Down Expand Up @@ -91,7 +93,7 @@ def main() -> None:
"--language",
choices=[lang.value for lang in LANGUAGE_DISPLAY_NAMES],
default="en",
help="Must match EVA_LANGUAGE for the run",
help="Base language code without a region tag (e.g. fr, not fr-CA); must match EVA_LANGUAGE for the run",
)
parser.add_argument("--voice", default=DEFAULT_VOICE_ID, help="Cartesia voice ID for the agent (default: Katie)")
parser.add_argument(
Expand All @@ -108,6 +110,10 @@ def main() -> None:
)
parser.add_argument("--domains", nargs="+", default=list(DOMAINS), choices=DOMAINS)
args = parser.parse_args()
if "-" in args.language:
raise SystemExit(
f"--language takes a base language code without a region tag; use {args.language.split('-')[0]}"
)
if args.turn_end_fallback_time is not None and not 5 <= args.turn_end_fallback_time <= 60:
raise SystemExit("Cartesia supports --turn-end-fallback-time between 5 and 60 seconds")

Expand All @@ -134,19 +140,20 @@ def main() -> None:
STATE.write_text(json.dumps(state, indent=2))
tool_ids.append(state["tools"][key])

name = f"eva-{domain.replace('_', '-')}-{args.model.replace('.', '').replace('-', '')}"
agent_key = f"{domain}/{args.language}"
name = f"eva-{domain.replace('_', '-')}-{args.language.lower()}-{args.model.replace('.', '').replace('-', '')}"
payload = {
"name": name,
"description": "EVA-Bench agent",
"config": agent_config(args.model, tool_ids, args.turn_end_fallback_time, args.language, args.voice),
}
if domain in state["agents"]:
request("PATCH", f"/v1/agents/{state['agents'][domain]}", body=payload)
if agent_key in state["agents"]:
request("PATCH", f"/v1/agents/{state['agents'][agent_key]}", body=payload)
else:
state["agents"][domain] = request("POST", "/v1/agents", body=payload)["id"]
state["agents"][agent_key] = request("POST", "/v1/agents", body=payload)["id"]
STATE.write_text(json.dumps(state, indent=2))
print(f"{domain}: {state['agents'][domain]} with {len(tool_ids)} tools", file=sys.stderr)
print(f"{domain}: " + json.dumps({"model": args.model, "assistant_agent_id": state["agents"][domain]}))
print(f"{agent_key}: {state['agents'][agent_key]} with {len(tool_ids)} tools", file=sys.stderr)
print(f"{agent_key}: " + json.dumps({"model": args.model, "assistant_agent_id": state["agents"][agent_key]}))


if __name__ == "__main__":
Expand Down
163 changes: 163 additions & 0 deletions scripts/provision_phonic_agents.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,163 @@
"""Create or update one Phonic agent per EVA domain, with EVA's tools as WebSocket tools.

The agent passes EVA's system prompt and greeting through untouched: they are sent with each session
as the template variables instructions and initial_message. Each EVA tool is registered as a
``custom_websocket`` tool (executed by the assistant server over the STS socket) and attached to the
agent.

Usage:
python scripts/provision_phonic_agents.py [--model phonic-1] [--voice sabrina] [--language en]
[--domains airline itsm medical_hr] [--backchannel]

Deploys all domains by default, with assistant backchannel cues off unless --backchannel is passed.
Timing settings mirror the run config: --turn-end-fallback-time (EVA_TURN_END_FALLBACK_TIME) sets
"Respond after" (no_input_poke_sec), --conversation-time-limit-seconds
(EVA_CONVERSATION_TIME_LIMIT_SECONDS, default 600) sets the max conversation duration, and the
silence limit (no_input_end_conversation_sec) is the user simulator's INACTIVITY_TIMEOUT_SECONDS.
The assistant server sends the same values with every session, so the run matches the saved agent.

Reads PHONIC_API_KEY from the environment or .env. Agent IDs are kept in output/phonic_agents.json.
Agents are keyed by domain and language, so a rerun updates the same agent for that domain and language
while other languages keep their own agents. Tool names are unique per Phonic organization, so tools are
looked up by name (updated if present, created otherwise) and attached to the domain's agents. Prints each domain's
EVA_MODEL__S2S_PARAMS (without the api_key) to use for the run.
"""

import argparse
import asyncio
import json
import os
import sys
from pathlib import Path

from dotenv import dotenv_values
from phonic import AsyncPhonic
from phonic.core.api_error import ApiError

from eva.models.agents import AgentConfig
from eva.models.config import LANGUAGE_DISPLAY_NAMES, RunConfig
from eva.user_simulator.cascade.simulator import INACTIVITY_TIMEOUT_SECONDS

DOMAINS = ("airline", "itsm", "medical_hr")
STATE = Path("output/phonic_agents.json")
TOOL_CALL_OUTPUT_TIMEOUT_MS = 30000
# Phonic reserves system_* names for its own system variables (e.g. {{system_time}}).
SYSTEM_PROMPT_VARIABLE = "{{instructions}}"
INITIAL_MESSAGE_VARIABLE = "{{initial_message}}"


def tool_body(fn: dict) -> dict:
return {
"name": fn["name"],
"description": fn["description"],
"execution_mode": "sync",
"parameters": fn["parameters"],
"tool_call_output_timeout_ms": TOOL_CALL_OUTPUT_TIMEOUT_MS,
}


async def upsert_tool(client: AsyncPhonic, fn: dict) -> None:
body = tool_body(fn)
try:
await client.tools.get(fn["name"])
except ApiError as e:
if e.status_code != 404:
raise
await client.tools.create(type="custom_websocket", **body)
else:
await client.tools.update(fn["name"], **body) # a tool's type cannot change


def agent_config(
model: str,
tool_names: list[str],
language: str,
voice_id: str,
backchannel: bool,
turn_end_fallback_time: int | None,
conversation_time_limit_seconds: int,
) -> dict:
return {
"no_input_poke_sec": turn_end_fallback_time,
"no_input_end_conversation_sec": int(INACTIVITY_TIMEOUT_SECONDS),
"max_conversation_duration_sec": conversation_time_limit_seconds,
"enable_assistant_backchannel": backchannel,
"model": model,
"voice_id": voice_id,
"default_language": language,
"system_prompt": SYSTEM_PROMPT_VARIABLE,
"welcome_message": INITIAL_MESSAGE_VARIABLE,
"generate_welcome_message": False,
"template_variables": {"instructions": {"default_value": ""}, "initial_message": {"default_value": ""}},
"tools": tool_names,
}


async def main() -> None:
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
parser.add_argument("--model", default="phonic-1", help="Phonic model id (phonic-1 or phonic-0.5)")
parser.add_argument("--voice", default="sabrina", help="Phonic voice_id for the agent")
parser.add_argument(
"--language",
choices=[lang.value for lang in LANGUAGE_DISPLAY_NAMES],
default="en",
help="Base language code without a region tag (e.g. fr, not fr-CA); must match EVA_LANGUAGE for the run",
)
parser.add_argument("--domains", nargs="+", default=list(DOMAINS), choices=DOMAINS)
parser.add_argument(
"--turn-end-fallback-time",
type=int,
default=RunConfig.model_fields["turn_end_fallback_time"].default,
help="Must match EVA_TURN_END_FALLBACK_TIME for the run -> no_input_poke_sec (unset disables the poke)",
)
parser.add_argument(
"--conversation-time-limit-seconds",
type=int,
default=RunConfig.model_fields["conversation_time_limit_seconds"].default,
help="Must match EVA_CONVERSATION_TIME_LIMIT_SECONDS for the run -> max_conversation_duration_sec",
)
parser.add_argument("--backchannel", action="store_true", help="Enable assistant backchannel cues (off by default)")
args = parser.parse_args()
if "-" in args.language:
raise SystemExit(
f"--language takes a base language code without a region tag; use {args.language.split('-')[0]}"
)

key = os.environ.get("PHONIC_API_KEY") or dotenv_values(".env").get("PHONIC_API_KEY")
if not key:
raise SystemExit("Set PHONIC_API_KEY in the environment or .env")
client = AsyncPhonic(api_key=key)

state = json.loads(STATE.read_text()) if STATE.exists() else {"agents": {}}
STATE.parent.mkdir(parents=True, exist_ok=True)
for domain in args.domains:
# Tools keep EVA's order, the order the model sees them in.
tool_names = []
for tool in AgentConfig.from_yaml(f"configs/agents/{domain}_agent.yaml").build_tools_for_agent():
fn = tool["function"]
await upsert_tool(client, fn)
tool_names.append(fn["name"])

agent_key = f"{domain}/{args.language}"
name = f"eva-{domain.replace('_', '-')}-{args.language.lower()}-{args.model.replace('.', '').replace('-', '')}"
config = agent_config(
args.model,
tool_names,
args.language,
args.voice,
args.backchannel,
args.turn_end_fallback_time,
args.conversation_time_limit_seconds,
)
if agent_key in state["agents"]:
await client.agents.update(state["agents"][agent_key]["id"], name=name, **config)
else:
state["agents"][agent_key] = {"id": (await client.agents.create(name=name, **config)).id, "name": name}
state["agents"][agent_key]["name"] = name
STATE.write_text(json.dumps(state, indent=2))
print(f"{agent_key}: {state['agents'][agent_key]['id']} with {len(tool_names)} tools", file=sys.stderr)
print(f"{agent_key}: " + json.dumps({"model": args.model, "agent": name, "voice": args.voice}))


if __name__ == "__main__":
asyncio.run(main())
6 changes: 4 additions & 2 deletions src/eva/assistant/elevenlabs_server.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,8 @@
from eva.models.agents import AgentConfig
from eva.models.config import ModelConfig
from eva.utils.audio_utils import (
MulawStreamUpsampler,
create_twilio_media_message,
mulaw_8k_to_pcm16_16k,
parse_twilio_media_message,
)
from eva.utils.logging import get_logger
Expand Down Expand Up @@ -359,6 +359,8 @@ async def _on_end_session() -> None:

# ----- Concurrent tasks -----

_user_recording_upsampler = MulawStreamUpsampler(self._audio_sample_rate)

async def _forward_user_audio() -> None:
"""Read Twilio WS messages, convert audio, send to ElevenLabs."""
nonlocal stream_sid, twilio_connected
Expand Down Expand Up @@ -404,7 +406,7 @@ async def _forward_user_audio() -> None:

# Record user audio as 16 kHz PCM for the WAV file,
# anchored to real time so gaps are preserved.
pcm_16k = mulaw_8k_to_pcm16_16k(mulaw_bytes)
pcm_16k = _user_recording_upsampler.process(mulaw_bytes)
now = time.monotonic()
if _record_t0 is None:
_record_t0 = now
Expand Down
5 changes: 3 additions & 2 deletions src/eva/assistant/gemini_live_server.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,9 +31,9 @@
from eva.models.agents import AgentConfig
from eva.models.config import ModelConfig
from eva.utils.audio_utils import (
MulawStreamUpsampler,
create_twilio_media_message,
mulaw_8k_to_pcm16_16k,
mulaw_8k_to_pcm16_24k,
parse_twilio_media_message,
pcm16_24k_to_mulaw_8k,
sync_buffer_to_position,
Expand Down Expand Up @@ -453,6 +453,7 @@ async def _handle_session(self, websocket: WebSocket) -> None:
_assistant_turn_text: list[str] = []
_user_turn_text: list[str] = []

_user_recording_upsampler = MulawStreamUpsampler(_RECORDING_SAMPLE_RATE)
_in_model_turn = False
_user_speaking = False
_user_speech_start_ts: str | None = None # Timestamp from audio_interface (speech start)
Expand Down Expand Up @@ -519,7 +520,7 @@ async def _forward_user_audio() -> None:
# Convert 8 kHz mulaw -> 16 kHz PCM for Gemini
pcm_16k = mulaw_8k_to_pcm16_16k(mulaw_bytes)

pcm_24k = mulaw_8k_to_pcm16_24k(mulaw_bytes)
pcm_24k = _user_recording_upsampler.process(mulaw_bytes)
if not _in_model_turn:
sync_buffer_to_position(self.assistant_audio_buffer, len(self.user_audio_buffer))
self.user_audio_buffer.extend(pcm_24k)
Expand Down
6 changes: 5 additions & 1 deletion src/eva/assistant/openai_realtime_server.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
from eva.assistant.base_server import AbstractAssistantServer
from eva.assistant.pipeline.observers import FrameworkLogWriter, MetricsLogWriter
from eva.utils.audio_utils import (
MulawStreamUpsampler,
create_twilio_media_message,
mulaw_8k_to_pcm16_24k,
parse_twilio_media_message,
Expand Down Expand Up @@ -104,6 +105,9 @@ def __init__(self, **kwargs: Any) -> None:
self._user_speaking: bool = False
self._bot_speaking: bool = False
self._user_frame_count: int = 0
# Recording-only: the model still receives the ratecv audio below, so the
# benchmark input is unchanged; only the saved audio_user.wav is cleaner.
self._user_recording_upsampler = MulawStreamUpsampler(OPENAI_SAMPLE_RATE)
self._delta_count: int = 0

# User speech start timestamp from audio_interface (source of truth)
Expand Down Expand Up @@ -401,7 +405,7 @@ async def _forward_user_audio(self, websocket: WebSocket, conn: Any) -> None:
sync_target = len(self.user_audio_buffer)
sync_buffer_to_position(self.assistant_audio_buffer, sync_target)
synced = len(self.assistant_audio_buffer) - asst_before
self.user_audio_buffer.extend(pcm16_24k)
self.user_audio_buffer.extend(self._user_recording_upsampler.process(mulaw_bytes))
self._last_user_audio_mono = time.monotonic()
self._user_frame_count += 1
if self._user_frame_count % 50 == 0:
Expand Down
Loading
Loading