feat(voice): add AssemblyAI streaming STT provider

.changeset/salty-balloons-melt.md

@@ -0,0 +1,6 @@
+---
+"@cloudflare/voice": patch
+"@cloudflare/voice-assemblyai": patch
+---
+
+Add an AssemblyAI streaming STT provider for the voice pipeline.

README.md

@@ -146,7 +146,7 @@ The [`examples/`](examples) directory has 30+ self-contained demos. A non-exhaus
 - **Chat & assistants** — [`assistant/`](examples/assistant), [`agents-as-tools/`](examples/agents-as-tools), [`agent-skills/`](examples/agent-skills), [`workspace-chat/`](examples/workspace-chat), [`resumable-stream-chat/`](examples/resumable-stream-chat), [`structured-input/`](examples/structured-input), [`dynamic-tools/`](examples/dynamic-tools), [`multi-ai-chat/`](examples/multi-ai-chat), [`context-overflow-recovery/`](examples/context-overflow-recovery)
 - **MCP** — [`mcp/`](examples/mcp), [`mcp-client/`](examples/mcp-client), [`mcp-server/`](examples/mcp-server), [`mcp-worker/`](examples/mcp-worker), [`mcp-worker-authenticated/`](examples/mcp-worker-authenticated), [`mcp-elicitation/`](examples/mcp-elicitation), [`mcp-rpc-transport/`](examples/mcp-rpc-transport), [`webmcp/`](examples/webmcp)
 - **Code Mode & sandboxes** — [`codemode/`](examples/codemode), [`codemode-mcp/`](examples/codemode-mcp), [`codemode-mcp-openapi/`](examples/codemode-mcp-openapi), [`dynamic-workers/`](examples/dynamic-workers), [`dynamic-workers-playground/`](examples/dynamic-workers-playground), [`worker-bundler-playground/`](examples/worker-bundler-playground)
-- **Voice** — [`voice-agent/`](examples/voice-agent), [`voice-input/`](examples/voice-input), [`elevenlabs-starter/`](examples/elevenlabs-starter)
+- **Voice** — [`voice-agent/`](examples/voice-agent) is the unified voice pipeline example with Workers AI, AssemblyAI, Telnyx, and ElevenLabs STT; [`voice-input/`](examples/voice-input) covers dictation; [`telnyx-voice-agent/`](examples/telnyx-voice-agent) covers phone transport; [`elevenlabs-starter/`](examples/elevenlabs-starter) covers broader ElevenLabs media APIs
 - **Workflows & approvals** — [`workflows/`](examples/workflows), [`a2a/`](examples/a2a)
 - **Auth, payments, comms** — [`auth-agent/`](examples/auth-agent), [`cross-domain/`](examples/cross-domain), [`x402/`](examples/x402), [`x402-mcp/`](examples/x402-mcp), [`email-agent/`](examples/email-agent), [`github-webhook/`](examples/github-webhook), [`push-notifications/`](examples/push-notifications)
 - **Game & misc** — [`tictactoe/`](examples/tictactoe), [`ai-chat/`](examples/ai-chat)

docs/voice/index.md

@@ -508,11 +508,42 @@ tts = new WorkersAITTS(this.env.AI, {
 
 ### Third-Party Providers
 
-| Package                        | Class           | Description             |
-| ------------------------------ | --------------- | ----------------------- |
-| `@cloudflare/voice-deepgram`   | `DeepgramSTT`   | Continuous STT          |
-| `@cloudflare/voice-elevenlabs` | `ElevenLabsTTS` | High-quality TTS        |
-| `@cloudflare/voice-twilio`     | Twilio adapter  | Telephony (phone calls) |
+| Package                        | Class                            | Description                                 |
+| ------------------------------ | -------------------------------- | ------------------------------------------- |
+| `@cloudflare/voice-assemblyai` | `AssemblyAISTT`                  | Continuous STT (Universal 3.5 Pro Realtime) |
+| `@cloudflare/voice-deepgram`   | `DeepgramSTT`                    | Continuous STT                              |
+| `@cloudflare/voice-elevenlabs` | `ElevenLabsSTT`, `ElevenLabsTTS` | Continuous STT and high-quality TTS         |
+| `@cloudflare/voice-telnyx`     | `TelnyxSTT`, `TelnyxTTS`         | Continuous STT, TTS, and phone transport    |
+| `@cloudflare/voice-twilio`     | Twilio adapter                   | Telephony (phone calls)                     |
+
+**AssemblyAI STT:**
+
+```typescript
+import { AssemblyAISTT } from "@cloudflare/voice-assemblyai";
+
+export class MyAgent extends VoiceAgent<Env> {
+  transcriber = new AssemblyAISTT({
+    apiKey: this.env.ASSEMBLYAI_API_KEY
+  });
+  tts = new WorkersAITTS(this.env.AI);
+}
+```
+
+After each agent reply, the pipeline automatically feeds the spoken text back to AssemblyAI as conversational context (`agent_context`), improving recognition of short answers like "yes" or "7pm". See the [package README](https://github.com/cloudflare/agents/tree/main/voice-providers/assemblyai) for the full options table.
+
+**ElevenLabs STT:**
+
+```typescript
+import { ElevenLabsSTT } from "@cloudflare/voice-elevenlabs";
+
+export class MyAgent extends VoiceAgent<Env> {
+  transcriber = new ElevenLabsSTT({
+    apiKey: this.env.ELEVENLABS_API_KEY,
+    keyterms: ["Cloudflare", "Workers AI"]
+  });
+  tts = new WorkersAITTS(this.env.AI);
+}
+```
 
 **ElevenLabs TTS:**
 
@@ -688,7 +719,7 @@ History survives Durable Object restarts and client reconnections. Voice agents
 
 ## Examples
 
-- [`examples/voice-agent`](https://github.com/cloudflare/agents/tree/main/examples/voice-agent) — full voice agent with Workers AI
+- [`examples/voice-agent`](https://github.com/cloudflare/agents/tree/main/examples/voice-agent) — full voice agent with Workers AI, AssemblyAI, Telnyx, and ElevenLabs STT provider options
 - [`examples/voice-input`](https://github.com/cloudflare/agents/tree/main/examples/voice-input) — voice input (dictation) example
 
 ## Related

examples/voice-agent/.env.example

@@ -1,6 +1,8 @@
-# Optional: ElevenLabs TTS (higher quality voices)
-# Get a key at https://elevenlabs.io/app/settings/api-keys
-# Without this key, the agent uses Workers AI TTS (Deepgram Aura) by default.
+# Optional external STT providers. Workers AI STT is the default and needs no key.
+ASSEMBLYAI_API_KEY=
+TELNYX_API_KEY=
 ELEVENLABS_API_KEY=
+
+# Optional WebRTC/SFU transport mode.
 CLOUDFLARE_REALTIME_SFU_APP_ID=
-CLOUDFLARE_REALTIME_SFU_API_TOKEN=
+CLOUDFLARE_REALTIME_SFU_API_TOKEN=

examples/voice-agent/README.md

@@ -2,9 +2,11 @@
 
 A real-time voice agent running entirely inside a Durable Object. Talk to an AI assistant that can answer questions, set spoken reminders, and check the weather — with streaming responses, interruption support, and conversation memory across sessions.
 
-Uses Workers AI for all models — zero external API keys required:
+Uses Workers AI by default, so it runs with zero external API keys. The STT
+provider selector can also route the same voice pipeline through AssemblyAI,
+Telnyx, or ElevenLabs when their API keys are configured:
 
-- **STT**: Deepgram Flux (`@cf/deepgram/flux`) by default, with a Nova 3 (`@cf/deepgram/nova-3`) option in the UI
+- **STT**: Workers AI Flux (`@cf/deepgram/flux`) by default, Workers AI Nova 3 (`@cf/deepgram/nova-3`), AssemblyAI Universal 3.5 Pro Realtime, Telnyx STT, or ElevenLabs Scribe v2 Realtime
 - **TTS**: Deepgram Aura (`@cf/deepgram/aura-1`)
 - **Turn detection**: Flux `StartOfTurn` / `EndOfTurn` events
 - **LLM**: Kimi K2.7 Code (`@cf/moonshotai/kimi-k2.7-code`), GPT OSS 20B, or GLM 4.7 Flash
@@ -16,7 +18,14 @@ npm install
 npm run start
 ```
 
-No API keys needed — all AI models run via the Workers AI binding.
+No API keys are needed for the default Workers AI STT/TTS/LLM path. To try
+external STT providers, copy `.env.example` to `.env` and set the relevant key:
+
+```bash
+ASSEMBLYAI_API_KEY=...
+TELNYX_API_KEY=...
+ELEVENLABS_API_KEY=...
+```
 
 ## How it works
 
@@ -37,7 +46,7 @@ Browser                          Durable Object (VoiceAgent)
 
 1. Browser captures mic audio via AudioWorklet, downsamples to 16kHz mono PCM
 2. PCM streams to the Agent over the existing WebSocket connection (binary frames)
-3. Flux detects speech start and turn completion server-side
+3. The selected STT provider detects speech and turn completion server-side
 4. Agent runs the voice pipeline: STT → LLM (with tools) → streaming TTS
 5. TTS audio streams back per-sentence as MP3 while the LLM is still generating
 6. Browser decodes and plays audio through the selected speaker when supported; user can interrupt at any time
@@ -46,7 +55,9 @@ Browser                          Durable Object (VoiceAgent)
 
 - **Streaming TTS** — LLM output is split into sentences and synthesized concurrently, so the user hears the first sentence while the rest is still being generated.
 - **Interruption handling** — speak over the agent to cut it off mid-sentence. Flux speech-start events abort the server pipeline and stop queued browser playback; client audio-level detection remains as a fallback.
-- **Server-side turn detection** — Flux handles speech boundaries, so the example does not need client-side end-of-speech signaling to run the voice pipeline.
+- **Provider selector** — choose Workers AI, AssemblyAI, Telnyx, or ElevenLabs STT without changing the agent code.
+- **Server-side turn detection** — the selected STT provider handles speech boundaries, so the example does not need client-side end-of-speech signaling to run the voice pipeline.
+- **Provider tuning** — AssemblyAI exposes latency/accuracy mode, Voice Focus, context prompt, and keyterms; Telnyx exposes model/language; ElevenLabs exposes Scribe cleanup, background filtering, and keyterms.
 - **Speaker selection** — choose an audio output device for assistant playback. Unsupported browsers keep using the system default output.
 - **Conversation persistence** — all messages are stored in SQLite and survive restarts. The agent remembers previous conversations.
 - **Agent tools** — the LLM can call `get_current_time`, `set_reminder`, and `get_weather` during conversation.

examples/voice-agent/package.json

@@ -11,6 +11,9 @@
   "dependencies": {
     "@cloudflare/kumo": "^2.6.0",
     "@cloudflare/voice": "*",
+    "@cloudflare/voice-assemblyai": "*",
+    "@cloudflare/voice-elevenlabs": "*",
+    "@cloudflare/voice-telnyx": "*",
     "@phosphor-icons/react": "^2.1.10",
     "agents": "*",
     "ai": "^6.0.202",

examples/voice-agent/src/client.tsx

@@ -27,6 +27,12 @@ import {
 } from "@cloudflare/kumo";
 import { useEffect, useRef, useState, useCallback } from "react";
 import { useSFUVoice } from "./use-sfu-voice";
+import {
+  DEFAULT_STT_SETTINGS,
+  ProviderSettings,
+  getSttQuery,
+  type SttSettings
+} from "./stt-settings";
 import { createRoot } from "react-dom/client";
 import "./styles.css";
 
@@ -118,10 +124,14 @@ function getAudioOutputLabel(device: MediaDeviceInfo, index: number) {
 
 function WebRTCApp({
   llmModel,
-  onLlmModelChange
+  onLlmModelChange,
+  sttSettings,
+  onSttSettingsChange
 }: {
   llmModel: LlmModel;
   onLlmModelChange: (m: LlmModel) => void;
+  sttSettings: SttSettings;
+  onSttSettingsChange: (settings: SttSettings) => void;
 }) {
   const sessionId = useRef(getSessionId()).current;
 
@@ -141,7 +151,7 @@ function WebRTCApp({
   } = useSFUVoice({
     agent: "my-voice-agent",
     name: sessionId,
-    query: { llm: llmModel }
+    query: { llm: llmModel, ...getSttQuery(sttSettings) }
   });
 
   const transcriptEndRef = useRef<HTMLDivElement>(null);
@@ -202,6 +212,12 @@ function WebRTCApp({
         </Button>
       </div>
 
+      <ProviderSettings
+        settings={sttSettings}
+        disabled={isInCall}
+        onChange={onSttSettingsChange}
+      />
+
       {/* Error banner */}
       {error && (
         <div className="mb-4 px-4 py-2.5 rounded-lg bg-red-500/10 border border-red-500/20 text-sm text-red-600 dark:text-red-400">
@@ -380,7 +396,8 @@ function App() {
   const [transport, setTransport] = useState<"websocket" | "webrtc">(
     "websocket"
   );
-  const [sttModel, setSttModel] = useState<"flux" | "nova-3">("flux");
+  const [sttSettings, setSttSettings] =
+    useState<SttSettings>(DEFAULT_STT_SETTINGS);
   const [llmModel, setLlmModel] = useState<LlmModel>("glm");
   const [outputDeviceId, setOutputDeviceId] = useState("default");
 
@@ -401,7 +418,7 @@ function App() {
   } = useVoiceAgent({
     agent: "my-voice-agent",
     name: sessionId,
-    query: { model: sttModel, llm: llmModel },
+    query: { llm: llmModel, ...getSttQuery(sttSettings) },
     outputDeviceId,
     onReconnect: () => {
       setToast("Reconnected to agent.");
@@ -595,7 +612,12 @@ function App() {
             </Button>
           </div>
 
-          <WebRTCApp llmModel={llmModel} onLlmModelChange={setLlmModel} />
+          <WebRTCApp
+            llmModel={llmModel}
+            onLlmModelChange={setLlmModel}
+            sttSettings={sttSettings}
+            onSttSettingsChange={setSttSettings}
+          />
 
           {/* Footer */}
           <div className="mt-4 flex justify-center">
@@ -652,24 +674,11 @@ function App() {
           </Button>
         </div>
 
-        {/* STT model selector */}
-        <div className="mb-4 flex items-center justify-center gap-2">
-          <span className="text-xs text-kumo-secondary">STT Model:</span>
-          <Button
-            variant={sttModel === "flux" ? "primary" : "ghost"}
-            size="sm"
-            onClick={() => setSttModel("flux")}
-          >
-            Flux
-          </Button>
-          <Button
-            variant={sttModel === "nova-3" ? "primary" : "ghost"}
-            size="sm"
-            onClick={() => setSttModel("nova-3")}
-          >
-            Nova 3
-          </Button>
-        </div>
+        <ProviderSettings
+          settings={sttSettings}
+          disabled={isInCall}
+          onChange={setSttSettings}
+        />
 
         {/* LLM model selector */}
         <div className="mb-4 flex items-center justify-center gap-2">

examples/voice-agent/src/server.ts

@@ -6,15 +6,17 @@ import {
 } from "agents";
 import {
   withVoice,
-  WorkersAIFluxSTT,
-  WorkersAINova3STT,
   WorkersAITTS,
   type VoiceTurnContext,
   type Transcriber
 } from "@cloudflare/voice";
 import { streamText, tool, stepCountIs } from "ai";
 import { createWorkersAI } from "workers-ai-provider";
 import { z } from "zod";
+import {
+  createVoiceTranscriber,
+  getMissingSttProviderKey
+} from "./stt-providers";
 
 const VoiceAgent = withVoice(Agent);
 
@@ -31,12 +33,7 @@ export class MyVoiceAgent extends VoiceAgent<Env> {
   tts = new WorkersAITTS(this.env.AI);
 
   createTranscriber(connection: Connection): Transcriber {
-    const url = new URL(connection.uri ?? "http://localhost");
-    const model = url.searchParams.get("model");
-    if (model === "nova-3") {
-      return new WorkersAINova3STT(this.env.AI);
-    }
-    return new WorkersAIFluxSTT(this.env.AI);
+    return createVoiceTranscriber(connection, this.env);
   }
 
   // --- Single-speaker enforcement ---
@@ -48,6 +45,12 @@ export class MyVoiceAgent extends VoiceAgent<Env> {
   #activeSpeakerId: string | null = null;
 
   beforeCallStart(connection: Connection): boolean {
+    const missingKey = getMissingSttProviderKey(connection, this.env);
+    if (missingKey) {
+      connection.send(JSON.stringify({ type: "error", message: missingKey }));
+      return false;
+    }
+
     if (this.#activeSpeakerId && this.#activeSpeakerId !== connection.id) {
       connection.send(
         JSON.stringify({

examples/voice-agent/src/stt-providers.ts

@@ -0,0 +1,72 @@
+import type { Transcriber } from "@cloudflare/voice";
+import type { Connection } from "agents";
+import { createAssemblyAITranscriber } from "./stt-providers/assemblyai";
+import { createElevenLabsTranscriber } from "./stt-providers/elevenlabs";
+import { createTelnyxTranscriber } from "./stt-providers/telnyx";
+import type { SttProvider } from "./stt-providers/types";
+import { getEnvString } from "./stt-providers/utils";
+import { createWorkersAITranscriber } from "./stt-providers/workers-ai";
+
+export function createVoiceTranscriber(
+  connection: Connection,
+  env: Env
+): Transcriber {
+  const url = getConnectionUrl(connection);
+  const provider = getSttProvider(connection);
+
+  if (provider === "assemblyai") {
+    return createAssemblyAITranscriber(env, url);
+  }
+
+  if (provider === "telnyx") {
+    return createTelnyxTranscriber(env, url);
+  }
+
+  if (provider === "elevenlabs") {
+    return createElevenLabsTranscriber(env, url);
+  }
+
+  return createWorkersAITranscriber(env, provider);
+}
+
+export function getMissingSttProviderKey(
+  connection: Connection,
+  env: Env
+): string | null {
+  const provider = getSttProvider(connection);
+  if (provider === "assemblyai" && !getEnvString(env, "ASSEMBLYAI_API_KEY")) {
+    return "AssemblyAI STT requires ASSEMBLYAI_API_KEY in your .env file or Worker secrets.";
+  }
+  if (provider === "telnyx" && !getEnvString(env, "TELNYX_API_KEY")) {
+    return "Telnyx STT requires TELNYX_API_KEY in your .env file or Worker secrets.";
+  }
+  if (provider === "elevenlabs" && !getEnvString(env, "ELEVENLABS_API_KEY")) {
+    return "ElevenLabs STT requires ELEVENLABS_API_KEY in your .env file or Worker secrets.";
+  }
+  return null;
+}
+
+function getConnectionUrl(connection: Connection): URL {
+  return new URL(connection.uri ?? "http://localhost");
+}
+
+function getSttProvider(connection: Connection): SttProvider {
+  const url = getConnectionUrl(connection);
+  const provider = url.searchParams.get("stt");
+
+  if (
+    provider === "workers-ai-nova-3" ||
+    provider === "assemblyai" ||
+    provider === "telnyx" ||
+    provider === "elevenlabs"
+  ) {
+    return provider;
+  }
+
+  // Preserve the old query parameter so existing links keep working.
+  if (url.searchParams.get("model") === "nova-3") {
+    return "workers-ai-nova-3";
+  }
+
+  return "workers-ai-flux";
+}