From cd82addab6bf0511145150055f896f125f2f7197 Mon Sep 17 00:00:00 2001 From: Justin Beckwith Date: Tue, 11 Aug 2026 15:44:08 -0700 Subject: [PATCH 1/8] feat: add Realtime WebSocket and WebRTC support --- Gemfile | 1 + Gemfile.lock | 40 +- README.md | 69 ++ examples/realtime/README.md | 202 ++++++ examples/realtime/mcp_approval.rb | 168 +++++ examples/realtime/realtime_conversation.rb | 454 ++++++++++++ examples/realtime/sideband.rb | 49 ++ examples/realtime/sip.rb | 57 ++ examples/realtime/translation.rb | 49 ++ examples/realtime/webrtc_call.rb | 22 + examples/realtime/webrtc_conversation.html | 223 ++++++ examples/realtime/webrtc_conversation.rb | 149 ++++ examples/realtime/websocket_audio.rb | 34 + examples/realtime/websocket_text.rb | 60 ++ examples/realtime/websocket_transcription.rb | 83 +++ lib/openai.rb | 14 + lib/openai/client.rb | 87 ++- lib/openai/errors.rb | 44 +- lib/openai/internal/transport/base_client.rb | 13 + .../models/realtime/call_create_params.rb | 30 + .../models/realtime/call_create_response.rb | 33 + lib/openai/realtime/base_connection.rb | 143 ++++ lib/openai/realtime/connection.rb | 34 + lib/openai/realtime/connection_manager.rb | 66 ++ lib/openai/realtime/connection_resources.rb | 196 ++++++ lib/openai/realtime/sideband_connection.rb | 17 + .../realtime/transcription_connection.rb | 26 + lib/openai/realtime/translation_connection.rb | 26 + .../realtime/transports/async_websocket.rb | 78 +++ lib/openai/realtime/unknown_server_event.rb | 42 ++ lib/openai/resources/realtime.rb | 61 ++ lib/openai/resources/realtime/calls.rb | 46 ++ lib/openai/resources/realtime/translations.rb | 46 ++ .../resources/realtime/translations/calls.rb | 39 ++ .../realtime/translations/client_secrets.rb | 54 ++ openai.gemspec | 2 +- rbi/openai/client.rbi | 16 + rbi/openai/errors.rbi | 40 +- rbi/openai/internal/transport/base_client.rbi | 9 + .../models/realtime/call_create_params.rbi | 49 ++ .../models/realtime/call_create_response.rbi | 48 ++ rbi/openai/realtime/base_connection.rbi | 40 ++ rbi/openai/realtime/connection.rbi | 70 ++ rbi/openai/realtime/connection_manager.rbi | 44 ++ rbi/openai/realtime/connection_resources.rbi | 328 +++++++++ rbi/openai/realtime/sideband_connection.rbi | 24 + .../realtime/transcription_connection.rbi | 71 ++ .../realtime/translation_connection.rbi | 73 ++ .../realtime/transports/async_websocket.rbi | 50 ++ rbi/openai/realtime/unknown_server_event.rbi | 25 + rbi/openai/resources/realtime.rbi | 47 ++ rbi/openai/resources/realtime/calls.rbi | 11 + .../resources/realtime/translations.rbi | 44 ++ .../resources/realtime/translations/calls.rbi | 25 + .../realtime/translations/client_secrets.rbi | 32 + realtime.md | 405 +++++++++++ sig/openai/client.rbs | 9 + sig/openai/errors.rbs | 24 +- sig/openai/internal/transport/base_client.rbs | 4 + .../models/realtime/call_create_params.rbs | 32 + .../models/realtime/call_create_response.rbs | 26 + sig/openai/realtime/base_connection.rbs | 20 + sig/openai/realtime/connection.rbs | 36 + sig/openai/realtime/connection_manager.rbs | 21 + sig/openai/realtime/connection_resources.rbs | 136 ++++ sig/openai/realtime/sideband_connection.rbs | 11 + .../realtime/transcription_connection.rbs | 34 + .../realtime/translation_connection.rbs | 36 + .../realtime/transports/async_websocket.rbs | 26 + sig/openai/realtime/unknown_server_event.rbs | 13 + sig/openai/resources/realtime.rbs | 40 ++ sig/openai/resources/realtime/calls.rbs | 7 + .../resources/realtime/translations.rbs | 21 + .../resources/realtime/translations/calls.rbs | 16 + .../realtime/translations/client_secrets.rbs | 19 + .../async_websocket_transport_test.rb | 554 +++++++++++++++ test/openai/realtime/connection_test.rb | 577 +++++++++++++++ test/openai/realtime/examples_test.rb | 656 ++++++++++++++++++ test/openai/resources/realtime/calls_test.rb | 89 +++ .../resources/realtime/translations_test.rb | 181 +++++ 80 files changed, 6790 insertions(+), 6 deletions(-) create mode 100644 examples/realtime/README.md create mode 100755 examples/realtime/mcp_approval.rb create mode 100755 examples/realtime/realtime_conversation.rb create mode 100755 examples/realtime/sideband.rb create mode 100755 examples/realtime/sip.rb create mode 100755 examples/realtime/translation.rb create mode 100755 examples/realtime/webrtc_call.rb create mode 100644 examples/realtime/webrtc_conversation.html create mode 100755 examples/realtime/webrtc_conversation.rb create mode 100755 examples/realtime/websocket_audio.rb create mode 100755 examples/realtime/websocket_text.rb create mode 100755 examples/realtime/websocket_transcription.rb create mode 100644 lib/openai/models/realtime/call_create_params.rb create mode 100644 lib/openai/models/realtime/call_create_response.rb create mode 100644 lib/openai/realtime/base_connection.rb create mode 100644 lib/openai/realtime/connection.rb create mode 100644 lib/openai/realtime/connection_manager.rb create mode 100644 lib/openai/realtime/connection_resources.rb create mode 100644 lib/openai/realtime/sideband_connection.rb create mode 100644 lib/openai/realtime/transcription_connection.rb create mode 100644 lib/openai/realtime/translation_connection.rb create mode 100644 lib/openai/realtime/transports/async_websocket.rb create mode 100644 lib/openai/realtime/unknown_server_event.rb create mode 100644 lib/openai/resources/realtime/translations.rb create mode 100644 lib/openai/resources/realtime/translations/calls.rb create mode 100644 lib/openai/resources/realtime/translations/client_secrets.rb create mode 100644 rbi/openai/models/realtime/call_create_params.rbi create mode 100644 rbi/openai/models/realtime/call_create_response.rbi create mode 100644 rbi/openai/realtime/base_connection.rbi create mode 100644 rbi/openai/realtime/connection.rbi create mode 100644 rbi/openai/realtime/connection_manager.rbi create mode 100644 rbi/openai/realtime/connection_resources.rbi create mode 100644 rbi/openai/realtime/sideband_connection.rbi create mode 100644 rbi/openai/realtime/transcription_connection.rbi create mode 100644 rbi/openai/realtime/translation_connection.rbi create mode 100644 rbi/openai/realtime/transports/async_websocket.rbi create mode 100644 rbi/openai/realtime/unknown_server_event.rbi create mode 100644 rbi/openai/resources/realtime/translations.rbi create mode 100644 rbi/openai/resources/realtime/translations/calls.rbi create mode 100644 rbi/openai/resources/realtime/translations/client_secrets.rbi create mode 100644 realtime.md create mode 100644 sig/openai/models/realtime/call_create_params.rbs create mode 100644 sig/openai/models/realtime/call_create_response.rbs create mode 100644 sig/openai/realtime/base_connection.rbs create mode 100644 sig/openai/realtime/connection.rbs create mode 100644 sig/openai/realtime/connection_manager.rbs create mode 100644 sig/openai/realtime/connection_resources.rbs create mode 100644 sig/openai/realtime/sideband_connection.rbs create mode 100644 sig/openai/realtime/transcription_connection.rbs create mode 100644 sig/openai/realtime/translation_connection.rbs create mode 100644 sig/openai/realtime/transports/async_websocket.rbs create mode 100644 sig/openai/realtime/unknown_server_event.rbs create mode 100644 sig/openai/resources/realtime/translations.rbs create mode 100644 sig/openai/resources/realtime/translations/calls.rbs create mode 100644 sig/openai/resources/realtime/translations/client_secrets.rbs create mode 100644 test/openai/realtime/async_websocket_transport_test.rb create mode 100644 test/openai/realtime/connection_test.rb create mode 100644 test/openai/realtime/examples_test.rb create mode 100644 test/openai/resources/realtime/translations_test.rb diff --git a/Gemfile b/Gemfile index cfb2366a1..318265cde 100644 --- a/Gemfile +++ b/Gemfile @@ -17,6 +17,7 @@ end group :development, :test do gem "async" + gem "async-websocket" gem "aws-sdk-core", "~> 3" gem "minitest", "~> 5.27" gem "minitest-focus" diff --git a/Gemfile.lock b/Gemfile.lock index d206ab023..497722853 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -42,6 +42,24 @@ GEM io-event (~> 1.11) metrics (~> 0.12) traces (~> 0.18) + async-http (0.95.1) + async (>= 2.10.2) + async-pool (~> 0.11) + io-endpoint (~> 0.14) + io-stream (~> 0.6) + metrics (~> 0.12) + protocol-http (~> 0.62) + protocol-http1 (~> 0.39) + protocol-http2 (~> 0.26) + protocol-url (~> 0.2) + traces (~> 0.10) + async-pool (0.11.2) + async (>= 2.0) + async-websocket (0.30.1) + async-http (~> 0.76) + protocol-http (~> 0.34) + protocol-rack (~> 0.7) + protocol-websocket (~> 0.17) aws-eventstream (1.4.0) aws-partitions (1.1261.0) aws-sdk-core (3.252.0) @@ -85,7 +103,10 @@ GEM hashdiff (1.2.1) i18n (1.15.2) concurrent-ruby (~> 1.0) + io-endpoint (0.17.2) io-event (1.11.2) + io-stream (0.14.0) + openssl (>= 3.3) jmespath (1.6.2) json (2.21.2) language_server-protocol (3.17.0.5) @@ -106,14 +127,30 @@ GEM minitest (~> 5.0) mutex_m (0.3.0) netrc (0.11.0) + openssl (4.0.2) parallel (1.27.0) parser (3.3.10.0) ast (~> 2.4.1) racc prettier_print (1.2.1) prism (1.6.0) + protocol-hpack (1.5.1) + protocol-http (0.70.0) + protocol-http1 (0.40.2) + protocol-http (~> 0.68) + protocol-http2 (0.26.2) + protocol-hpack (~> 1.4) + protocol-http (~> 0.62) + protocol-rack (0.22.1) + io-stream (>= 0.10) + protocol-http (~> 0.58) + rack (>= 1.0) + protocol-url (0.11.0) + protocol-websocket (0.21.1) + protocol-http (~> 0.2) public_suffix (7.0.5) racc (1.8.1) + rack (3.2.6) rainbow (3.1.1) rake (13.3.1) rb-fsevent (0.11.2) @@ -203,7 +240,7 @@ GEM addressable (>= 2.8.0) crack (>= 0.3.2) hashdiff (>= 0.4.0, < 2.0.0) - webrick (1.9.1) + webrick (1.9.2) yard (0.9.44) yard-sorbet (0.9.0) sorbet-runtime @@ -221,6 +258,7 @@ PLATFORMS DEPENDENCIES async + async-websocket aws-sdk-core (~> 3) minitest (~> 5.27) minitest-focus diff --git a/README.md b/README.md index 21bf0691b..1e01cda80 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,75 @@ stream.each do |event| end ``` +### Realtime + +The SDK supports server-side Realtime WebSockets, WebRTC session negotiation, +sideband connections for WebRTC and SIP calls, and continuous translation. Add +the WebSocket adapter alongside the SDK: + +```ruby +gem "openai" +gem "async-websocket" +``` + +Then open a block-scoped connection. Events are decoded into the existing typed +Realtime models, and the socket is closed when the block exits: + +```ruby +client.realtime.connect(model: "gpt-realtime-2.1") do |connection| + connection.session.update( + type: :realtime, + output_modalities: [:text] + ) + connection.conversation.items.create( + type: :message, + role: :user, + content: [{type: :input_text, text: "Hello from Ruby"}] + ) + connection.response.create + + connection.each do |event| + case event + when OpenAI::Realtime::ResponseTextDeltaEvent + print(event.delta) + when OpenAI::Realtime::ResponseDoneEvent + raise "Response ended with #{event.response.status}" unless event.response.status == :completed + break + when OpenAI::Realtime::RealtimeErrorEvent + raise event.error.message + end + end +end +``` + +Run the live text smoke test with `OPENAI_API_KEY` set: + +```console +$ bundle exec ruby examples/realtime/websocket_text.rb +``` + +For live speech-to-text, provide raw mono 24 kHz PCM16 input. The example opens +the dedicated `intent: :transcription` connection: + +```console +$ REALTIME_INPUT_PCM=input.pcm bundle exec ruby examples/realtime/websocket_transcription.rb +``` + +For a natural hands-free voice conversation, run the Ruby WebRTC negotiation +endpoint and open the printed localhost URL in a browser: + +```console +$ bundle exec ruby examples/realtime/webrtc_conversation.rb +``` + +The API key and session configuration stay in Ruby. The browser owns the +microphone, remote audio, acoustic echo cancellation, and WebRTC peer. + +See [realtime.md](realtime.md) for WebRTC, audio, transcription, translation, +SIP, sideband, MCP, lifecycle, and custom transport guidance. The +[Realtime example runbook](examples/realtime/README.md) lists every executable +example, its prerequisites, and its observable success criteria. + ### Pagination List methods in the OpenAI API are paginated. diff --git a/examples/realtime/README.md b/examples/realtime/README.md new file mode 100644 index 000000000..edd013131 --- /dev/null +++ b/examples/realtime/README.md @@ -0,0 +1,202 @@ +# Realtime examples + +These examples run against the live OpenAI service with `OPENAI_API_KEY` set. +They default to `gpt-realtime-2.1`; override `OPENAI_REALTIME_MODEL` when needed. + +## Browser voice conversation (recommended) + +For natural hands-free conversation, use the WebRTC example. Ruby keeps the +standard API key private, accepts the browser's SDP offer, configures the +Realtime call, and returns the SDP answer. The browser owns the media peer and +requests acoustic echo cancellation, noise suppression, and automatic gain +control. + +```sh +bundle exec ruby examples/realtime/webrtc_conversation.rb +``` + +Open `http://127.0.0.1:4567`, click **Start conversation**, and allow microphone +access. Talk normally and speak over the model to interrupt it. Click **Stop** +to close the peer and ask Ruby to hang up the call. No API key is sent to the +browser. Override the port with `REALTIME_DEMO_PORT`. + +## WebSocket microphone loop (advanced) + +`realtime_conversation.rb` keeps one full-duplex WebSocket session open. It +streams 100 ms microphone chunks continuously, uses server VAD to detect turn +boundaries and create responses automatically, and plays output audio as deltas +arrive. There is no push-to-talk control. + +This low-level path has no acoustic echo cancellation. It is useful for server +audio pipelines and protocol debugging, but it is not a laptop speakerphone. +Install FFmpeg (which also supplies `ffplay`) and **use headphones** so the +microphone does not capture the model's voice: + +```sh +brew install ffmpeg +bundle exec ruby examples/realtime/realtime_conversation.rb +``` + +Start speaking after `Connected` appears. Speak again whenever you want to +interrupt: the service cancels the response and the example immediately clears +local playback, drops already-queued audio deltas, and truncates the unheard +audio from conversation history. Press Control-C to end the session. + +On macOS, the default microphone is AVFoundation audio device `0`. List devices +and select a different index if necessary: + +```sh +ffmpeg -f avfoundation -list_devices true -i "" + +REALTIME_MIC_DEVICE=:1 \ +bundle exec ruby examples/realtime/realtime_conversation.rb +``` + +macOS may ask for microphone permission for your terminal on the first run. +Customize the voice or spoken behavior with `OPENAI_REALTIME_VOICE` and +`OPENAI_REALTIME_INSTRUCTIONS`. The example defaults to the recommended `marin` +voice. Linux defaults to PulseAudio's `default` device; override +`REALTIME_AUDIO_INPUT_FORMAT` and `REALTIME_MIC_DEVICE` for another FFmpeg input. + +For a deterministic live smoke test, provide paced PCM input and capture output +instead of opening audio devices: + +```sh +REALTIME_INPUT_PCM=input.pcm \ +REALTIME_OUTPUT_PCM=response.pcm \ +OPENAI_REALTIME_TIMEOUT=30 \ +bundle exec ruby examples/realtime/realtime_conversation.rb +``` + +## WebSocket text + +```sh +bundle exec ruby examples/realtime/websocket_text.rb +``` + +A successful run prints `session.created`, `session.updated`, streamed assistant +text, and `response.done status=completed`. + +## WebSocket audio + +The input must be raw, headerless, mono 24 kHz PCM16 little-endian audio. For +example, convert a WAV file with: + +```sh +ffmpeg -i input.wav -f s16le -acodec pcm_s16le -ac 1 -ar 24000 input.pcm +``` + +Then run the example and play its output: + +```sh +REALTIME_INPUT_PCM=input.pcm \ +REALTIME_OUTPUT_PCM=response.pcm \ +bundle exec ruby examples/realtime/websocket_audio.rb + +ffplay -f s16le -ar 24000 -ch_layout mono response.pcm +``` + +A successful run exits with status 0 and writes a non-empty output file. + +## Realtime transcription + +Use the same raw mono 24 kHz PCM16 input format as the audio example: + +```sh +REALTIME_INPUT_PCM=input.pcm \ +OPENAI_REALTIME_TRANSCRIPTION_MODEL=gpt-live-transcribe \ +bundle exec ruby examples/realtime/websocket_transcription.rb +``` + +A successful run streams transcript deltas, then prints the completed +transcript with its `item_id`. Completion events for different input turns may +arrive out of order, so applications should correlate them by `item_id`. +The example opens a dedicated `intent: :transcription` connection. Configure +its transcription model with `OPENAI_REALTIME_TRANSCRIPTION_MODEL`. + +## Realtime translation + +Translation uses the same raw PCM input format and reads translated transcript +and audio while input is still being uploaded: + +```sh +REALTIME_INPUT_PCM=input.pcm \ +REALTIME_OUTPUT_PCM=translation.pcm \ +TARGET_LANGUAGE=es \ +bundle exec ruby examples/realtime/translation.rb +``` + +A successful run prints the translated transcript, exits after +`session.closed`, and writes non-empty translated PCM audio. + +## MCP approval + +Point the example at a Streamable HTTP MCP server. This invocation uses the +public OpenAI developer-documentation server: + +```sh +MCP_SERVER_URL=https://developers.openai.com/mcp \ +OPENAI_REALTIME_PROMPT='Search the OpenAI docs for the Realtime WebSocket URL.' \ +bundle exec ruby examples/realtime/mcp_approval.rb +``` + +The example waits for tool import, selects an advertised tool, waits for the +tool-choice update to be acknowledged, approves the request, waits for the tool +to finish, and asks the model for the final answer. A successful run prints all +five checkpoints. Set `OPENAI_REALTIME_DEBUG=1` to print every event type. + +## WebRTC call creation + +`webrtc_call.rb` is the server endpoint core: send a browser-generated SDP offer +on stdin, return stdout as `application/sdp`, and retain the call ID printed on +stderr. + +```sh +bundle exec ruby examples/realtime/webrtc_call.rb < offer.sdp > answer.sdp +``` + +The recommended `webrtc_conversation.rb` example above exercises this exchange +end to end with a real `RTCPeerConnection`. `webrtc_call.rb` remains a minimal +stdin/stdout building block for integrating the same SDK call into another web +framework. + +## Sideband control + +Use the call ID from WebRTC call creation or a verified incoming SIP webhook: + +```sh +OPENAI_REALTIME_CALL_ID=rtc_... \ +bundle exec ruby examples/realtime/sideband.rb +``` + +For a bounded smoke test, set `OPENAI_REALTIME_STOP_AFTER=session.updated` and +`OPENAI_REALTIME_TIMEOUT=30`. A real paired test keeps the browser or SIP peer +connected while this process attaches to the same call. + +## SIP + +The SIP example requires an OpenAI project with SIP configured and a real +incoming call. Obtain `OPENAI_REALTIME_CALL_ID` from a verified +`realtime.call.incoming` webhook, then run: + +```sh +OPENAI_REALTIME_CALL_ID=rtc_... \ +bundle exec ruby examples/realtime/sip.rb +``` + +The script accepts the call, attaches a sideband WebSocket, and prints the audio +transcript. `OPENAI_REALTIME_STOP_AFTER` and `OPENAI_REALTIME_TIMEOUT` provide +bounded smoke-test controls. Without a carrier-originated incoming call, the +same accept-then-attach orchestration is covered by the local example and HTTP +resource tests, but that is not a substitute for the final telephony smoke test. + +## Local protocol coverage + +The test suite exercises the examples' orchestration and the documented local +WebSocket lifecycle, including text, transcription, full-duplex translation, +MCP approval, fragmented frames, normal and abnormal closure, sideband control, +and SIP accept-then-attach behavior: + +```sh +./scripts/test +``` diff --git a/examples/realtime/mcp_approval.rb b/examples/realtime/mcp_approval.rb new file mode 100755 index 000000000..98dc39f36 --- /dev/null +++ b/examples/realtime/mcp_approval.rb @@ -0,0 +1,168 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require_relative "../../lib/openai" +require "timeout" + +module OpenAI + module Examples + module Realtime + module MCPApproval + module_function + + def configure(connection, server_url:) + connection.session.update( + type: :realtime, + output_modalities: [:text], + tools: [ + { + type: :mcp, + server_label: "remote", + server_url: server_url, + require_approval: :always + } + ] + ) + end + + def select_tool(connection, server_label:, tool_name:) + connection.session.update( + type: :realtime, + tools: [{type: :mcp, server_label: server_label}], + tool_choice: {type: :mcp, server_label: server_label, name: tool_name} + ) + end + + def send_prompt(connection, prompt:) + connection.conversation.items.create( + type: :message, + role: :user, + content: [{type: :input_text, text: prompt}] + ) + connection.response.create + end + + def approve_request(connection, item) + connection.conversation.items.respond_to_mcp_approval( + approval_request_id: item.id, + approve: true, + reason: "Approved by the application policy" + ) + end + + def run_session(connection, prompt:, output: $stdout, debug: false) + state = { + completed_item_ids: {}, + tool_lists: {}, + waiting_for_tool_update: false, + selected_tool: false, + mcp_call_pending: false + } + + connection.each do |event| + output.puts("[mcp event] #{event.type}") if debug + break if handle_event(connection, event, prompt: prompt, output: output, state: state) == :done + + select_discovered_tool(connection, output: output, state: state) + end + end + + def handle_event(connection, event, prompt:, output:, state:) + case event + when OpenAI::Realtime::McpListToolsCompleted + state[:completed_item_ids][event.item_id] = true + when OpenAI::Realtime::SessionUpdatedEvent + handle_session_updated(connection, prompt: prompt, state: state) + when OpenAI::Realtime::ConversationItemDone + handle_conversation_item(connection, event.item, output: output, state: state) + when OpenAI::Realtime::McpListToolsFailed + raise "MCP tool discovery failed for #{event.item_id}" + when OpenAI::Realtime::ResponseMcpCallArgumentsDone + state[:mcp_call_pending] = true + output.puts("[mcp] tool call requested") + when OpenAI::Realtime::ResponseMcpCallCompleted + handle_tool_completed(connection, output: output, state: state) + when OpenAI::Realtime::ResponseMcpCallFailed + raise "MCP tool call failed for #{event.item_id}" + when OpenAI::Realtime::RealtimeErrorEvent + raise event.error.message + when OpenAI::Realtime::ResponseTextDeltaEvent + output.print(event.delta) + output.flush + when OpenAI::Realtime::ResponseDoneEvent + handle_response_done(event, output: output, state: state) + end + end + + def handle_session_updated(connection, prompt:, state:) + return unless state[:waiting_for_tool_update] + + send_prompt(connection, prompt: prompt) + state[:waiting_for_tool_update] = false + end + + def handle_conversation_item(connection, item, output:, state:) + if item.is_a?(OpenAI::Realtime::RealtimeMcpListTools) + state[:tool_lists][item.id] = item + elsif item.is_a?(OpenAI::Realtime::RealtimeMcpApprovalRequest) + output.puts("[mcp] approving #{item.id}") + approve_request(connection, item) + end + end + + def handle_tool_completed(connection, output:, state:) + state[:mcp_call_pending] = false + output.puts("[mcp] tool call completed") + connection.response.create(tool_choice: :none) + end + + def handle_response_done(event, output:, state:) + raise "Response ended with #{event.response.status}" unless event.response.status == :completed + + if state[:mcp_call_pending] + output.puts("[mcp] waiting for approval and tool execution") + else + output.puts("\n[mcp] response.done status=completed") + :done + end + end + + def select_discovered_tool(connection, output:, state:) + return if state[:selected_tool] + + tool_list = state[:tool_lists].values.find { |item| state[:completed_item_ids][item.id] } + return unless tool_list + raise "The MCP server imported no tools" if tool_list.tools.empty? + + tool_names = tool_list.tools.map(&:name) + output.puts("[mcp] tools discovered: #{tool_names.join(', ')}") + select_tool(connection, server_label: tool_list.server_label, tool_name: tool_names.fetch(0)) + state[:selected_tool] = true + state[:waiting_for_tool_update] = true + end + + def run(client:, model:, server_url:, prompt:, output: $stdout, debug: false) + client.realtime.connect(model: model) do |connection| + configure(connection, server_url: server_url) + run_session(connection, prompt: prompt, output: output, debug: debug) + end + end + end + end + end +end + +if $PROGRAM_NAME == __FILE__ + client = OpenAI::Client.new + timeout = Integer(ENV.fetch("OPENAI_REALTIME_TIMEOUT", "60")) + + Timeout.timeout(timeout) do + OpenAI::Examples::Realtime::MCPApproval.run( + client: client, + model: ENV.fetch("OPENAI_REALTIME_MODEL", "gpt-realtime-2.1"), + server_url: ENV.fetch("MCP_SERVER_URL"), + prompt: ENV.fetch("OPENAI_REALTIME_PROMPT", "Use the configured MCP server."), + debug: ENV["OPENAI_REALTIME_DEBUG"] == "1" + ) + end +end diff --git a/examples/realtime/realtime_conversation.rb b/examples/realtime/realtime_conversation.rb new file mode 100755 index 000000000..4d2e782f9 --- /dev/null +++ b/examples/realtime/realtime_conversation.rb @@ -0,0 +1,454 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require "async" +require "timeout" +require_relative "../../lib/openai" + +module OpenAI + module Examples + module Realtime + module Conversation + AUDIO_BYTES_PER_SECOND = 48_000 + CHUNK_BYTES = 4_800 + + class FFmpegMicrophone + def initialize(input_format:, device:) + @input_format = input_format + @device = device + @pid = nil + @stopping = false + end + + def each_chunk + return enum_for(__method__) unless block_given? + + reader, writer = IO.pipe + @pid = Process.spawn(*command, out: writer, err: $stderr) + writer.close + reader.binmode + while (chunk = reader.read(CHUNK_BYTES)) + yield(chunk) + end + rescue Errno::ENOENT + raise "ffmpeg was not found; install it with `brew install ffmpeg`" + ensure + writer&.close unless writer&.closed? + reader&.close unless reader&.closed? + wait_for_capture + end + + def stop + return unless @pid + + @stopping = true + Process.kill("INT", @pid) + rescue Errno::ESRCH + nil + end + + private def command + [ + "ffmpeg", + "-nostdin", + "-loglevel", + "error", + "-thread_queue_size", + "1024", + "-f", + @input_format, + "-i", + @device, + "-ac", + "1", + "-ar", + "24000", + "-acodec", + "pcm_s16le", + "-f", + "s16le", + "pipe:1" + ] + end + + private def wait_for_capture + return unless @pid + + _, status = Process.wait2(@pid) + return if @stopping || status.success? + + raise "ffmpeg microphone capture exited with status #{status.exitstatus}" + rescue Errno::ECHILD + nil + ensure + @pid = nil + end + end + + class PCMFileMicrophone + def initialize(path) + @path = path + @stopping = false + end + + def each_chunk + return enum_for(__method__) unless block_given? + + File.open(@path, "rb") do |input| + while !@stopping && (chunk = input.read(CHUNK_BYTES)) + yield(chunk) + sleep(chunk.bytesize.fdiv(AUDIO_BYTES_PER_SECOND)) + end + end + 10.times do + break if @stopping + + yield("\0".b * CHUNK_BYTES) + sleep(CHUNK_BYTES.fdiv(AUDIO_BYTES_PER_SECOND)) + end + end + + def stop = @stopping = true + end + + class FFplaySpeaker + def initialize + @input = nil + @pid = nil + end + + def write(bytes) + start unless @input + @input.write(bytes) + @input.flush + rescue Errno::EPIPE + interrupt + raise "ffplay stopped while playing Realtime audio" + end + + def interrupt + return unless @pid + + Process.kill("TERM", @pid) + @input&.close unless @input&.closed? + Process.wait(@pid) + rescue Errno::ESRCH, Errno::ECHILD + nil + ensure + @input = nil + @pid = nil + end + + alias_method :close, :interrupt + + private def start + reader, writer = IO.pipe + @pid = Process.spawn(*command, in: reader, out: File::NULL, err: $stderr) + reader.close + writer.binmode + @input = writer + rescue Errno::ENOENT + raise "ffplay was not found; install it with `brew install ffmpeg`" + ensure + reader&.close unless reader&.closed? + end + + private def command + [ + "ffplay", + "-nodisp", + "-nostats", + "-loglevel", + "error", + "-fflags", + "nobuffer", + "-flags", + "low_delay", + "-probesize", + "32", + "-analyzeduration", + "0", + "-f", + "s16le", + "-ar", + "24000", + "-ch_layout", + "mono", + "-i", + "pipe:0" + ] + end + end + + class PCMFileSpeaker + def initialize(path) + @output = File.open(path, "wb") + end + + def write(bytes) = @output.write(bytes) + + def interrupt = @output.flush + + def close + @output.close unless @output.closed? + end + end + + class AudioPlayback + def initialize(speaker, clock: nil) + @speaker = speaker + @clock = clock || -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) } + @interrupted_response_id = nil + reset_current + end + + def write(event) + return if interrupted?(event.response_id) + + start_response(event) unless current?(event) + bytes = Base64.strict_decode64(event.delta) + @speaker.write(bytes) + @received_bytes += bytes.bytesize + end + + def interrupt(connection) + unless @item_id + @speaker.interrupt + return + end + + audio_end_ms = played_ms + response_id = @response_id + item_id = @item_id + content_index = @content_index + + @speaker.interrupt + connection.conversation.items.truncate( + item_id: item_id, + content_index: content_index, + audio_end_ms: audio_end_ms + ) + @interrupted_response_id = response_id + reset_current + response_id + end + + def interrupted?(response_id) = response_id == @interrupted_response_id + + def finish(response_id) + @interrupted_response_id = nil if interrupted?(response_id) + end + + def close = @speaker.close + + private def current?(event) + @response_id == event.response_id && + @item_id == event.item_id && + @content_index == event.content_index + end + + private def start_response(event) + @response_id = event.response_id + @item_id = event.item_id + @content_index = event.content_index + @received_bytes = 0 + @started_at = @clock.call + end + + private def played_ms + elapsed_ms = ((@clock.call - @started_at) * 1_000).floor + received_ms = (@received_bytes * 1_000) / AUDIO_BYTES_PER_SECOND + [elapsed_ms, received_ms].min.clamp(0, received_ms) + end + + private def reset_current + @response_id = nil + @item_id = nil + @content_index = nil + @received_bytes = 0 + @started_at = nil + end + end + + module_function + + def configure(connection, voice:, instructions:) + connection.session.update( + type: :realtime, + output_modalities: [:audio], + instructions: instructions, + audio: { + input: { + format: {type: :"audio/pcm", rate: 24_000}, + noise_reduction: {type: :near_field}, + turn_detection: { + type: :server_vad, + threshold: 0.5, + prefix_padding_ms: 300, + silence_duration_ms: 500, + create_response: true, + interrupt_response: true + } + }, + output: { + format: {type: :"audio/pcm", rate: 24_000}, + voice: voice + } + } + ) + end + + def forward_microphone(connection, microphone) + microphone.each_chunk do |chunk| + connection.input_audio_buffer.append_bytes(chunk) + end + end + + def handle_event(event, connection:, playback:, output:) + case event + when OpenAI::Realtime::ResponseAudioDeltaEvent + playback.write(event) + when OpenAI::Realtime::ResponseAudioTranscriptDeltaEvent + return if playback.interrupted?(event.response_id) + + output.print(event.delta) + output.flush + when OpenAI::Realtime::ResponseAudioTranscriptDoneEvent + return if playback.interrupted?(event.response_id) + + output.puts + when OpenAI::Realtime::InputAudioBufferSpeechStartedEvent + output.puts if playback.interrupt(connection) + when OpenAI::Realtime::RealtimeErrorEvent + raise event.error.message + when OpenAI::Realtime::ResponseDoneEvent + playback.finish(event.response.id) + return if [:cancelled, :completed].include?(event.response.status) + + raise "Realtime response ended with #{event.response.status}" + end + end + + def wait_until_ready(connection) + while (event = connection.receive) + return if event.is_a?(OpenAI::Realtime::SessionUpdatedEvent) + raise event.error.message if event.is_a?(OpenAI::Realtime::RealtimeErrorEvent) + end + raise "Realtime connection closed before session.updated" + end + + def stream_events(connection, microphone:, playback:, output:, stop_after: nil) + connection.each do |event| + handle_event(event, connection: connection, playback: playback, output: output) + break if stop_after == event.type.to_s + end + ensure + microphone.stop + end + + def run_session( + connection, + microphone:, + speaker:, + voice:, + instructions:, + output: $stdout, + stop_after: nil + ) + configure(connection, voice: voice, instructions: instructions) + wait_until_ready(connection) + output.puts( + "Connected. Talk naturally; speak over the model to interrupt. " \ + "Press Ctrl-C to exit. Use headphones to prevent echo." + ) + playback = AudioPlayback.new(speaker) + + receiver = Async do + stream_events( + connection, + microphone: microphone, + playback: playback, + output: output, + stop_after: stop_after + ) + end + sender = Async { forward_microphone(connection, microphone) } + sender.wait + receiver.wait + ensure + microphone.stop + playback&.close || speaker.close + sender&.stop + receiver&.stop + end + + def run( + client:, + model:, + microphone:, + speaker:, + voice:, + instructions:, + output: $stdout, + stop_after: nil + ) + client.realtime.connect(model: model) do |connection| + run_session( + connection, + microphone: microphone, + speaker: speaker, + voice: voice, + instructions: instructions, + output: output, + stop_after: stop_after + ) + end + end + end + end + end +end + +if $PROGRAM_NAME == __FILE__ + begin + input_path = ENV["REALTIME_INPUT_PCM"] + microphone = + if input_path + OpenAI::Examples::Realtime::Conversation::PCMFileMicrophone.new(input_path) + else + default_format = RUBY_PLATFORM.include?("darwin") ? "avfoundation" : "pulse" + default_device = RUBY_PLATFORM.include?("darwin") ? ":0" : "default" + OpenAI::Examples::Realtime::Conversation::FFmpegMicrophone.new( + input_format: ENV.fetch("REALTIME_AUDIO_INPUT_FORMAT", default_format), + device: ENV.fetch("REALTIME_MIC_DEVICE", default_device) + ) + end + stop_after = input_path ? "response.done" : nil + output_path = ENV["REALTIME_OUTPUT_PCM"] + speaker = + if output_path + OpenAI::Examples::Realtime::Conversation::PCMFileSpeaker.new(output_path) + else + OpenAI::Examples::Realtime::Conversation::FFplaySpeaker.new + end + run = lambda do + OpenAI::Examples::Realtime::Conversation.run( + client: OpenAI::Client.new, + model: ENV.fetch("OPENAI_REALTIME_MODEL", "gpt-realtime-2.1"), + microphone: microphone, + speaker: speaker, + voice: ENV.fetch("OPENAI_REALTIME_VOICE", "marin"), + instructions: ENV.fetch( + "OPENAI_REALTIME_INSTRUCTIONS", + "Have a natural spoken conversation. Keep each response concise." + ), + stop_after: stop_after + ) + end + + timeout = ENV["OPENAI_REALTIME_TIMEOUT"] + timeout ? Timeout.timeout(Integer(timeout)) { run.call } : run.call + rescue Interrupt + warn("\nConversation ended.") + end +end diff --git a/examples/realtime/sideband.rb b/examples/realtime/sideband.rb new file mode 100755 index 000000000..730f673d3 --- /dev/null +++ b/examples/realtime/sideband.rb @@ -0,0 +1,49 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require_relative "../../lib/openai" +require "timeout" + +module OpenAI + module Examples + module Realtime + module Sideband + module_function + + def stream(connection, output: $stdout, stop_after: nil) + connection.each do |event| + output.puts("#{event.type}: #{event.to_h}") + output.flush + break if stop_after == event.type.to_s + end + end + + def run(client:, call_id:, stop_after: nil, output: $stdout) + client.realtime.connect(call_id: call_id) do |connection| + connection.session.update( + type: :realtime, + instructions: "Use server-side business rules and keep answers short." + ) + stream(connection, output: output, stop_after: stop_after) + end + end + end + end + end +end + +if $PROGRAM_NAME == __FILE__ + # CALL_ID may come from calls.create.call_id for WebRTC or from a verified + # realtime.call.incoming webhook for SIP. + client = OpenAI::Client.new + run = lambda do + OpenAI::Examples::Realtime::Sideband.run( + client: client, + call_id: ENV.fetch("OPENAI_REALTIME_CALL_ID"), + stop_after: ENV["OPENAI_REALTIME_STOP_AFTER"] + ) + end + + timeout = ENV["OPENAI_REALTIME_TIMEOUT"] + timeout ? Timeout.timeout(Integer(timeout)) { run.call } : run.call +end diff --git a/examples/realtime/sip.rb b/examples/realtime/sip.rb new file mode 100755 index 000000000..30a99cc57 --- /dev/null +++ b/examples/realtime/sip.rb @@ -0,0 +1,57 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require_relative "../../lib/openai" +require "timeout" + +module OpenAI + module Examples + module Realtime + module SIP + module_function + + def stream(connection, output: $stdout, stop_after: nil) + connection.each do |event| + case event + when OpenAI::Realtime::ResponseAudioTranscriptDeltaEvent + output.print(event.delta) + output.flush + when OpenAI::Realtime::RealtimeErrorEvent + raise event.error.message + end + break if stop_after == event.type.to_s + end + end + + def run(client:, call_id:, model:, output: $stdout, stop_after: nil) + client.realtime.calls.accept( + call_id, + type: :realtime, + model: model, + instructions: "You are answering a phone call. Be warm and concise." + ) + + client.realtime.connect(call_id: call_id) do |connection| + stream(connection, output: output, stop_after: stop_after) + end + end + end + end + end +end + +if $PROGRAM_NAME == __FILE__ + # Obtain the call ID from a verified realtime.call.incoming webhook via + # client.webhooks.unwrap(payload, headers). + run = lambda do + OpenAI::Examples::Realtime::SIP.run( + client: OpenAI::Client.new, + call_id: ENV.fetch("OPENAI_REALTIME_CALL_ID"), + model: ENV.fetch("OPENAI_REALTIME_MODEL", "gpt-realtime-2.1"), + stop_after: ENV["OPENAI_REALTIME_STOP_AFTER"] + ) + end + + timeout = ENV["OPENAI_REALTIME_TIMEOUT"] + timeout ? Timeout.timeout(Integer(timeout)) { run.call } : run.call +end diff --git a/examples/realtime/translation.rb b/examples/realtime/translation.rb new file mode 100755 index 000000000..d295625c7 --- /dev/null +++ b/examples/realtime/translation.rb @@ -0,0 +1,49 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require_relative "../../lib/openai" +require "async" + +def read_translation(connection, output) + Async do + connection.each do |event| + case event + when OpenAI::Realtime::RealtimeTranslationOutputTranscriptDeltaEvent + print(event.delta) + when OpenAI::Realtime::RealtimeTranslationOutputAudioDeltaEvent + output.write(Base64.strict_decode64(event.delta)) + when OpenAI::Realtime::RealtimeTranslationSessionClosedEvent + puts + break + when OpenAI::Realtime::RealtimeErrorEvent + raise event.error.message + end + end + end +end + +def write_translation_input(connection, input_path) + File.open(input_path, "rb") do |input| + while (chunk = input.read(9_600)) + connection.input_audio_buffer.append_bytes(chunk) + end + end +ensure + connection.session.close +end + +input_path = ENV.fetch("REALTIME_INPUT_PCM") +output_path = ENV.fetch("REALTIME_OUTPUT_PCM", "translation-output.pcm") +model = ENV.fetch("OPENAI_REALTIME_TRANSLATION_MODEL", "gpt-realtime-translate") +client = OpenAI::Client.new + +File.open(output_path, "wb") do |output| + client.realtime.translations.connect(model: model) do |connection| + connection.session.update( + audio: {output: {language: ENV.fetch("TARGET_LANGUAGE", "es")}} + ) + reader = read_translation(connection, output) + write_translation_input(connection, input_path) + reader.wait + end +end diff --git a/examples/realtime/webrtc_call.rb b/examples/realtime/webrtc_call.rb new file mode 100755 index 000000000..4b5835d68 --- /dev/null +++ b/examples/realtime/webrtc_call.rb @@ -0,0 +1,22 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require_relative "../../lib/openai" + +# Pipe a browser's SDP offer to this process. The SDP answer is written to stdout, +# which makes this suitable as the core of a Rails, Sinatra, or Rack endpoint. +offer = $stdin.read +raise "Expected an SDP offer on stdin" if offer.empty? + +client = OpenAI::Client.new +call = client.realtime.calls.create( + sdp: offer, + session: { + type: :realtime, + model: ENV.fetch("OPENAI_REALTIME_MODEL", "gpt-realtime-2.1"), + audio: {output: {voice: :marin}} + } +) + +warn("Created Realtime call #{call.call_id || 'without a Location header'}") +$stdout.write(call.sdp) diff --git a/examples/realtime/webrtc_conversation.html b/examples/realtime/webrtc_conversation.html new file mode 100644 index 000000000..ac68d4c3f --- /dev/null +++ b/examples/realtime/webrtc_conversation.html @@ -0,0 +1,223 @@ + + + + + + Ruby Realtime conversation + + + +
+

Realtime conversation

+

+ Talk naturally and speak over the model whenever you want to interrupt. + The browser handles the microphone, playback, and acoustic echo cancellation. +

+
+ + +
+
Ready
+
+
+ +
+ + + + diff --git a/examples/realtime/webrtc_conversation.rb b/examples/realtime/webrtc_conversation.rb new file mode 100755 index 000000000..3f1d10bea --- /dev/null +++ b/examples/realtime/webrtc_conversation.rb @@ -0,0 +1,149 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require_relative "../../lib/openai" + +module OpenAI + module Examples + module Realtime + module WebRTCConversation + MAX_SDP_BYTES = 1_048_576 + + class App + def initialize(client:, html: File.binread(File.join(__dir__, "webrtc_conversation.html"))) + @client = client + @html = html + @call_ids = {} + @call_ids_lock = Mutex.new + end + + def handle(request, response) + case [request.request_method, request.path] + when ["GET", "/"] + render(response, status: 200, content_type: "text/html; charset=utf-8", body: @html) + when ["POST", "/session"] + create_session(request, response) + when ["POST", "/hangup"] + hangup(request, response) + else + render(response, status: 404, content_type: "text/plain; charset=utf-8", body: "Not found\n") + end + rescue ArgumentError => e + render(response, status: 400, content_type: "text/plain; charset=utf-8", body: "#{e.message}\n") + rescue StandardError => e + warn("Realtime WebRTC request failed: #{e.class}: #{e.message}") + render( + response, + status: 502, + content_type: "text/plain; charset=utf-8", + body: "Realtime request failed\n" + ) + end + + private def create_session(request, response) + content_type = request["content-type"].to_s.split(";", 2).first + unless content_type == "application/sdp" + raise ArgumentError, "Expected Content-Type: application/sdp" + end + + offer = request.body.to_s + raise ArgumentError, "Expected an SDP offer" if offer.empty? + raise ArgumentError, "SDP offer is too large" if offer.bytesize > MAX_SDP_BYTES + raise ArgumentError, "Invalid SDP offer" unless offer.lstrip.start_with?("v=") + + call = @client.realtime.calls.create(sdp: offer, session: session_config) + remember(call.call_id) + response["X-OpenAI-Call-ID"] = call.call_id if call.call_id + render(response, status: 201, content_type: "application/sdp", body: call.sdp) + end + + private def hangup(request, response) + call_id = request.body.to_s.strip + raise ArgumentError, "Expected a call ID" if call_id.empty? + + known = @call_ids_lock.synchronize { @call_ids.include?(call_id) } + unless known + return render( + response, + status: 404, + content_type: "text/plain; charset=utf-8", + body: "Unknown call\n" + ) + end + + begin + @client.realtime.calls.hangup(call_id) + rescue OpenAI::Errors::NotFoundError + # Closing the browser peer can end the call before this cleanup request arrives. + end + @call_ids_lock.synchronize { @call_ids.delete(call_id) } + render(response, status: 204, content_type: "text/plain; charset=utf-8", body: "") + end + + private def session_config + { + type: :realtime, + model: ENV.fetch("OPENAI_REALTIME_MODEL", "gpt-realtime-2.1"), + output_modalities: [:audio], + instructions: ENV.fetch( + "OPENAI_REALTIME_INSTRUCTIONS", + "Have a natural spoken conversation. Keep each response concise." + ), + audio: { + input: { + turn_detection: { + type: :server_vad, + threshold: 0.5, + prefix_padding_ms: 300, + silence_duration_ms: 700, + create_response: true, + interrupt_response: true + } + }, + output: {voice: ENV.fetch("OPENAI_REALTIME_VOICE", "marin")} + } + } + end + + private def remember(call_id) + @call_ids_lock.synchronize { @call_ids[call_id] = true } if call_id + end + + private def render(response, status:, content_type:, body:) + response.status = status + response["Content-Type"] = content_type + response["Cache-Control"] = "no-store" + response["X-Content-Type-Options"] = "nosniff" + response.body = body + end + end + + module_function + + def run + require("webrick") + + host = ENV.fetch("REALTIME_DEMO_HOST", "127.0.0.1") + port = Integer(ENV.fetch("REALTIME_DEMO_PORT", "4567")) + app = App.new(client: OpenAI::Client.new) + server = WEBrick::HTTPServer.new( + BindAddress: host, + Port: port, + Logger: WEBrick::Log.new($stderr, WEBrick::BasicLog::WARN), + AccessLog: [] + ) + server.mount_proc("/") { |request, response| app.handle(request, response) } + shutdown = proc { server.shutdown } + Signal.trap("INT", &shutdown) + Signal.trap("TERM", &shutdown) + puts("Open http://#{host}:#{port} and click Start conversation. Press Ctrl-C here to stop.") + server.start + rescue LoadError + raise "webrick is required; run `bundle install` before starting this example" + end + end + end + end +end + +OpenAI::Examples::Realtime::WebRTCConversation.run if $PROGRAM_NAME == __FILE__ diff --git a/examples/realtime/websocket_audio.rb b/examples/realtime/websocket_audio.rb new file mode 100755 index 000000000..2ae255940 --- /dev/null +++ b/examples/realtime/websocket_audio.rb @@ -0,0 +1,34 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require_relative "../../lib/openai" + +input_path = ENV.fetch("REALTIME_INPUT_PCM") +output_path = ENV.fetch("REALTIME_OUTPUT_PCM", "realtime-output.pcm") +client = OpenAI::Client.new + +File.open(output_path, "wb") do |output| + client.realtime.connect(model: ENV.fetch("OPENAI_REALTIME_MODEL", "gpt-realtime-2.1")) do |connection| + File.open(input_path, "rb") do |input| + while (chunk = input.read(4_800)) + connection.input_audio_buffer.append_bytes(chunk) + end + end + connection.input_audio_buffer.commit + connection.response.create + + connection.each do |event| + case event + when OpenAI::Realtime::ResponseAudioDeltaEvent + output.write(Base64.strict_decode64(event.delta)) + when OpenAI::Realtime::ResponseDoneEvent + raise "Response ended with #{event.response.status}" unless event.response.status == :completed + break + when OpenAI::Realtime::RealtimeErrorEvent + raise event.error.message + end + end + end +end + +warn("Wrote raw PCM16 audio to #{output_path}") diff --git a/examples/realtime/websocket_text.rb b/examples/realtime/websocket_text.rb new file mode 100755 index 000000000..3e8549a84 --- /dev/null +++ b/examples/realtime/websocket_text.rb @@ -0,0 +1,60 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require "timeout" +require_relative "../../lib/openai" + +def stream_response(connection) + started_response = false + connection.each do |event| + case event + when OpenAI::Realtime::SessionCreatedEvent + puts("[realtime] session.created") + when OpenAI::Realtime::SessionUpdatedEvent + puts("[realtime] session.updated") + when OpenAI::Realtime::ResponseTextDeltaEvent + print("[assistant] ") unless started_response + started_response = true + print(event.delta) + $stdout.flush + when OpenAI::Realtime::ResponseDoneEvent + puts if started_response + status = event.response.status + raise "Realtime response ended with status #{status.inspect}" unless status == :completed + + puts("[realtime] response.done status=completed") + break + when OpenAI::Realtime::RealtimeErrorEvent + raise "Realtime API error: #{event.error.message}" + end + end +end + +client = OpenAI::Client.new +model = ENV.fetch("OPENAI_REALTIME_MODEL", "gpt-realtime-2.1") +prompt = ENV.fetch("OPENAI_REALTIME_PROMPT", "Say hello from Ruby.") +timeout = Integer(ENV.fetch("OPENAI_REALTIME_TIMEOUT", "30")) +session = { + type: :realtime, + output_modalities: [:text], + instructions: "Be concise and friendly." +} +item = { + type: :message, + role: :user, + content: [{type: :input_text, text: prompt}] +} + +puts("[realtime] connecting with #{model}") + +Timeout.timeout(timeout) do + client.realtime.connect(model: model) do |connection| + puts("[realtime] connected; sending prompt: #{prompt.inspect}") + connection.session.update(**session) + connection.conversation.items.create(**item) + connection.response.create + stream_response(connection) + end +end + +puts("[realtime] smoke test passed") diff --git a/examples/realtime/websocket_transcription.rb b/examples/realtime/websocket_transcription.rb new file mode 100755 index 000000000..54784da6f --- /dev/null +++ b/examples/realtime/websocket_transcription.rb @@ -0,0 +1,83 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require "timeout" +require_relative "../../lib/openai" + +module OpenAI + module Examples + module Realtime + module WebSocketTranscription + module_function + + def configure(connection, transcription_model:) + connection.session.update( + audio: { + input: { + format: {type: :"audio/pcm", rate: 24_000}, + transcription: {model: transcription_model}, + turn_detection: nil + } + } + ) + end + + def wait_until_ready(connection) + loop do + event = connection.receive + raise "Realtime connection closed before session.updated" if event.nil? + raise event.error.message if event.is_a?(OpenAI::Realtime::RealtimeErrorEvent) + + return if event.is_a?(OpenAI::Realtime::SessionUpdatedEvent) + end + end + + def upload(connection, input_path:) + File.open(input_path, "rb") do |input| + while (chunk = input.read(4_800)) + connection.input_audio_buffer.append_bytes(chunk) + end + end + connection.input_audio_buffer.commit + end + + def print_transcript(connection, output: $stdout) + connection.each do |event| + case event + when OpenAI::Realtime::ConversationItemInputAudioTranscriptionDeltaEvent + output.print(event.delta) + output.flush + when OpenAI::Realtime::ConversationItemInputAudioTranscriptionCompletedEvent + output.puts("\n[#{event.item_id}] #{event.transcript}") + break + when OpenAI::Realtime::RealtimeErrorEvent + raise event.error.message + end + end + end + + def run(client:, input_path:, transcription_model:, output: $stdout) + client.realtime.connect(intent: :transcription) do |connection| + configure(connection, transcription_model: transcription_model) + wait_until_ready(connection) + upload(connection, input_path: input_path) + print_transcript(connection, output: output) + end + end + end + end + end +end + +if $PROGRAM_NAME == __FILE__ + Timeout.timeout(Integer(ENV.fetch("OPENAI_REALTIME_TIMEOUT", "30"))) do + OpenAI::Examples::Realtime::WebSocketTranscription.run( + client: OpenAI::Client.new, + input_path: ENV.fetch("REALTIME_INPUT_PCM"), + transcription_model: ENV.fetch( + "OPENAI_REALTIME_TRANSCRIPTION_MODEL", + "gpt-live-transcribe" + ) + ) + end +end diff --git a/lib/openai.rb b/lib/openai.rb index 2cdf3b4fd..81999905b 100644 --- a/lib/openai.rb +++ b/lib/openai.rb @@ -800,6 +800,8 @@ require_relative "openai/models/other_file_chunking_strategy_object" require_relative "openai/models/realtime/audio_transcription" require_relative "openai/models/realtime/call_accept_params" +require_relative "openai/models/realtime/call_create_params" +require_relative "openai/models/realtime/call_create_response" require_relative "openai/models/realtime/call_hangup_params" require_relative "openai/models/realtime/call_refer_params" require_relative "openai/models/realtime/call_reject_params" @@ -1165,6 +1167,15 @@ require_relative "openai/models/webhooks/unwrap_webhook_event" require_relative "openai/models/webhooks/webhook_unwrap_params" require_relative "openai/models" +require_relative "openai/realtime/unknown_server_event" +require_relative "openai/realtime/base_connection" +require_relative "openai/realtime/connection_resources" +require_relative "openai/realtime/connection" +require_relative "openai/realtime/sideband_connection" +require_relative "openai/realtime/transcription_connection" +require_relative "openai/realtime/translation_connection" +require_relative "openai/realtime/connection_manager" +require_relative "openai/realtime/transports/async_websocket" require_relative "openai/resources/admin" require_relative "openai/resources/admin/organization" require_relative "openai/resources/admin/organization/admin_api_keys" @@ -1245,6 +1256,9 @@ require_relative "openai/resources/realtime" require_relative "openai/resources/realtime/calls" require_relative "openai/resources/realtime/client_secrets" +require_relative "openai/resources/realtime/translations" +require_relative "openai/resources/realtime/translations/calls" +require_relative "openai/resources/realtime/translations/client_secrets" require_relative "openai/resources/responses" require_relative "openai/resources/responses/input_items" require_relative "openai/resources/responses/input_tokens" diff --git a/lib/openai/client.rb b/lib/openai/client.rb index 6400ca209..e060cca18 100644 --- a/lib/openai/client.rb +++ b/lib/openai/client.rb @@ -32,6 +32,11 @@ class Client < OpenAI::Internal::Transport::BaseClient # @return [String, nil] attr_reader :webhook_secret + # Optional base URL used for WebSocket connections. + # + # @return [URI::Generic, nil] + attr_reader :websocket_base_url + # @return [OpenAI::Auth::WorkloadIdentityAuth, nil] # @api private attr_reader :workload_identity_auth @@ -157,6 +162,80 @@ class Client < OpenAI::Internal::Transport::BaseClient {"authorization" => "Bearer #{@admin_api_key}"} end + # Build a fully authenticated Realtime WebSocket handshake request. Realtime + # transports use this boundary so provider authentication and request options stay + # consistent with ordinary SDK requests. + # + # @api private + # + # @param path [String] + # @param query [Hash{String=>String}] + # @param options [OpenAI::RequestOptions, Hash{Symbol=>Object}, nil] + # + # @return [Hash{Symbol=>Object}] + def realtime_connection_request(path:, query:, options: nil) + if @provider_runtime && @provider_runtime.name != "azure" + raise OpenAI::Errors::Error, + "Realtime WebSocket connections are not supported by the #{@provider_runtime.name} provider." + end + + opts = options.to_h + OpenAI::RequestOptions.validate!(opts) + request = build_request( + { + method: :get, + path: path, + query: query, + security: {bearer_auth: true} + }, + opts + ) + + workload_identity_header = "Bearer #{WORKLOAD_IDENTITY_API_KEY_PLACEHOLDER}" + if @workload_identity_auth && request.fetch(:headers)["authorization"] == workload_identity_header + token = @workload_identity_auth.get_token + request = request.merge( + headers: request.fetch(:headers).merge("authorization" => "Bearer #{token}") + ) + end + + request = prepare_request(request, redirect_count: 0, retry_count: 0) + if @websocket_base_url + url = OpenAI::Internal::Util.join_parsed_uri( + OpenAI::Internal::Util.parse_uri(@websocket_base_url.to_s), + {path: OpenAI::Internal::Util.interpolate_path(path)} + ) + url.query = request.fetch(:url).query + request = request.merge(url: url) + end + + url = request.fetch(:url).dup + url.scheme = {"http" => "ws", "https" => "wss"}.fetch(url.scheme, url.scheme) + headers = request.fetch(:headers).except("accept", "content-type") + request.merge(url: url, headers: headers) + end + + # @api private + # + # @param value [String, nil] + # @return [URI::Generic, nil] + private def parse_websocket_base_url(value) + return if value.nil? + + uri = URI(value) + valid_scheme = %w[http https ws wss].include?(uri.scheme) + ambiguous_component = uri.userinfo || uri.query || uri.fragment + unless uri.absolute? && uri.host && valid_scheme && !ambiguous_component + message = + "`websocket_base_url` must be an absolute HTTP or WebSocket URL " \ + "without credentials, query, or fragment" + raise ArgumentError, message + end + uri + rescue URI::Error => e + raise ArgumentError, "`websocket_base_url` is not a valid URL", cause: e + end + # Creates and returns a new client for interacting with the API. # # @param api_key [String, nil] Defaults to `ENV["OPENAI_API_KEY"]`. @@ -232,6 +311,9 @@ class Client < OpenAI::Internal::Transport::BaseClient # @param default_headers [Hash{String=>String, nil}, nil] Extra headers to send # with every request. Explicit values override `ENV["OPENAI_CUSTOM_HEADERS"]`. # + # @param websocket_base_url [String, nil] Override the base URL for WebSocket + # connections. The HTTP base URL is used when omitted. + # # @param max_retries [Integer] Max number of retries to attempt after a failed retryable request. # # @param timeout [Float, nil] @@ -259,6 +341,7 @@ def initialize( provider: nil, base_url: OpenAI::Internal::OMIT, default_headers: nil, + websocket_base_url: nil, max_retries: self.class::DEFAULT_MAX_RETRIES, timeout: self.class::DEFAULT_TIMEOUT_IN_SECONDS, initial_retry_delay: self.class::DEFAULT_INITIAL_RETRY_DELAY, @@ -275,7 +358,8 @@ def initialize( api_key: api_key, admin_api_key: admin_api_key, workload_identity: workload_identity, - base_url: base_url + base_url: base_url, + websocket_base_url: websocket_base_url }.filter_map do |name, value| name unless value.equal?(OpenAI::Internal::OMIT) || value.nil? end @@ -352,6 +436,7 @@ def initialize( @admin_api_key = admin_api_key&.to_s @webhook_secret = webhook_secret&.to_s @provider_runtime = provider_runtime + @websocket_base_url = parse_websocket_base_url(websocket_base_url) super( base_url: base_url, diff --git a/lib/openai/errors.rb b/lib/openai/errors.rb index d810efa2b..76456f161 100644 --- a/lib/openai/errors.rb +++ b/lib/openai/errors.rb @@ -5,12 +5,54 @@ module Errors class Error < StandardError # @!attribute cause # - # @return [StandardError, nil] + # @return [Exception, nil] end class InvalidWebhookSignatureError < OpenAI::Errors::Error end + # Raised when a Realtime WebSocket cannot be opened or used. + class RealtimeConnectionError < OpenAI::Errors::Error + # @return [URI::Generic] + attr_reader :url + + # @return [Exception, nil] + def cause = @cause.nil? ? super : @cause + + # @api private + # + # @param url [URI::Generic] + # @param message [String, nil] + # @param cause [Exception, nil] + def initialize(url:, message: nil, cause: nil) + @url = url + @cause = cause + detail = cause && !cause.message.empty? ? ": #{cause.message}" : "." + super(message || "Realtime WebSocket connection error#{detail}") + end + end + + # Raised when a Realtime WebSocket message cannot be parsed as a typed event. + class RealtimeProtocolError < OpenAI::Errors::Error + # @return [String] + attr_reader :data + + # @return [StandardError, nil] + def cause = @cause.nil? ? super : @cause + + # @api private + # + # @param data [String] + # @param message [String, nil] + # @param cause [StandardError, nil] + def initialize(data:, message: nil, cause: nil) + @data = data + @cause = cause + detail = cause && !cause.message.empty? ? ": #{cause.message}" : "." + super(message || "Invalid Realtime WebSocket event#{detail}") + end + end + class ConversionError < OpenAI::Errors::Error # @return [StandardError, nil] def cause = @cause.nil? ? super : @cause diff --git a/lib/openai/internal/transport/base_client.rb b/lib/openai/internal/transport/base_client.rb index d037f5ff3..dbd2d8740 100644 --- a/lib/openai/internal/transport/base_client.rb +++ b/lib/openai/internal/transport/base_client.rb @@ -611,6 +611,19 @@ def send_request(request, redirect_count:, retry_count:, send_retry_header:, &co end end + # Execute the request specified by `req` without decoding its response. This is + # used by resources whose successful response is not modeled JSON. + # + # @api private + # + # @param req [Hash{Symbol=>Object}] + # + # @return [OpenAI::HTTPClient::Response] + def request_raw(req) + _, response, log_context = perform_request(req) + finish_request(log_context, response) { response } + end + # Execute the request specified by `req`. This is the method that all resource # methods call into. # diff --git a/lib/openai/models/realtime/call_create_params.rb b/lib/openai/models/realtime/call_create_params.rb new file mode 100644 index 000000000..beec64067 --- /dev/null +++ b/lib/openai/models/realtime/call_create_params.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module OpenAI + module Models + module Realtime + # @see OpenAI::Resources::Realtime::Calls#create + class CallCreateParams < OpenAI::Internal::Type::BaseModel + extend OpenAI::Internal::Type::RequestParameters::Converter + include OpenAI::Internal::Type::RequestParameters + + # @!attribute sdp + # The WebRTC Session Description Protocol offer. + # + # @return [String] + required :sdp, String + + # @!attribute session + # Optional session configuration sent with the SDP offer. + # + # @return [OpenAI::Models::Realtime::RealtimeSessionCreateRequest, nil] + optional :session, -> { OpenAI::Realtime::RealtimeSessionCreateRequest } + + # @!method initialize(sdp:, session: nil, request_options: {}) + # @param sdp [String] The WebRTC Session Description Protocol offer. + # @param session [OpenAI::Models::Realtime::RealtimeSessionCreateRequest] Optional session configuration. + # @param request_options [OpenAI::RequestOptions, Hash{Symbol=>Object}] + end + end + end +end diff --git a/lib/openai/models/realtime/call_create_response.rb b/lib/openai/models/realtime/call_create_response.rb new file mode 100644 index 000000000..581bd7fae --- /dev/null +++ b/lib/openai/models/realtime/call_create_response.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module OpenAI + module Models + module Realtime + # The SDP answer and response metadata returned when a WebRTC call is created. + class CallCreateResponse < OpenAI::Internal::Type::BaseModel + # @!attribute sdp + # The WebRTC Session Description Protocol answer. + # + # @return [String] + required :sdp, String + + # @!attribute call_id + # The call ID parsed from the response Location header, when present. + # + # @return [String, nil] + optional :call_id, String + + # @!attribute headers + # The response headers, normalized to lowercase names. + # + # @return [Hash{String=>String}] + required :headers, -> { OpenAI::Internal::Type::HashOf[String] } + + # @!method initialize(sdp:, headers:, call_id: nil) + # @param sdp [String] The WebRTC Session Description Protocol answer. + # @param headers [Hash{Symbol,String=>String}] The normalized response headers. + # @param call_id [String, nil] The call ID parsed from the Location header. + end + end + end +end diff --git a/lib/openai/realtime/base_connection.rb b/lib/openai/realtime/base_connection.rb new file mode 100644 index 000000000..2172acead --- /dev/null +++ b/lib/openai/realtime/base_connection.rb @@ -0,0 +1,143 @@ +# frozen_string_literal: true + +module OpenAI + module Realtime + # Shared protocol machinery for live Realtime WebSocket connections. + # + # @api private + class BaseConnection + include Enumerable + + # @return [URI::Generic] + attr_reader :url + + # @api private + def initialize(socket:, url:, server_event_type:, client_event_type:) + @socket = socket + @url = url + @server_event_type = server_event_type + @client_event_type = client_event_type + @server_event_names = discriminator_values(@server_event_type) + @client_event_names = discriminator_values(@client_event_type) + end + + # Yield server events until the remote peer closes the connection. + def each + return enum_for(__method__) unless block_given? + + while (event = receive) + yield(event) + end + self + end + + # Receive and parse the next server event, or return nil after a clean close. + def receive + data = receive_raw + return nil if data.nil? + + parse_event(data) + end + + # Receive the next raw WebSocket message. + def receive_raw + message = @socket.read + message&.to_str + end + + # Parse raw JSON as a typed server event. Valid events that are newer than this + # SDK remain observable as {UnknownServerEvent} values. + def parse_event(data) + parsed = JSON.parse(data, symbolize_names: true) + type = event_type(parsed) + unless @server_event_names.key?(type.to_s) + return OpenAI::Realtime::UnknownServerEvent.new(data: parsed) + end + + state = OpenAI::Internal::Type::Converter.new_coerce_state + event = OpenAI::Internal::Type::Converter.coerce(@server_event_type, parsed, state: state) + if (cause = coercion_error(state)) + raise OpenAI::Errors::RealtimeProtocolError.new(data: data, cause: cause) + end + event + rescue OpenAI::Errors::RealtimeProtocolError + raise + rescue StandardError => e + raise OpenAI::Errors::RealtimeProtocolError.new(data: data, cause: e) + end + + # Validate, encode, and send a typed client event. + def send_event(event) + validate_discriminator!(event, @client_event_names, kind: "client") if event.is_a?(Hash) + state = OpenAI::Internal::Type::Converter.new_coerce_state + coerced = OpenAI::Internal::Type::Converter.coerce(@client_event_type, event, state: state) + if (cause = coercion_error(state)) + raise ArgumentError.new("Invalid Realtime client event: #{cause.message}"), cause: cause + end + payload = OpenAI::Internal::Type::Converter.dump(@client_event_type, coerced) + validate_discriminator!(payload, @client_event_names, kind: "client") + send_raw(JSON.generate(payload)) + end + + # Send an already encoded text message. + def send_raw(data) + if closed? + raise OpenAI::Errors::RealtimeConnectionError.new( + url: @url, + message: "Cannot send on a closed Realtime WebSocket." + ) + end + @socket.write(data) + nil + end + + # Close the connection. + def close(code: 1000, reason: "") + return if closed? + + @socket.close(code: code, reason: reason) + nil + end + + # @return [Boolean] + def closed? = @socket.closed? + + private def discriminator_values(union) + union.variants.to_h do |variant| + value = variant.fields.fetch(:type).fetch(:const) + [value.to_s, true] + end + end + + private def event_type(event) + unless event.is_a?(Hash) + raise ArgumentError, "Realtime server event must be a JSON object" + end + + type = event[:type] + return type if type.is_a?(String) || type.is_a?(Symbol) + + raise ArgumentError, "Realtime server event type must be a string or symbol" + end + + private def validate_discriminator!(event, allowed, kind:) + type = + if event.key?(:type) + event.fetch(:type) + elsif event.key?("type") + event.fetch("type") + end + return if type && allowed.key?(type.to_s) + + raise ArgumentError, "Unknown Realtime #{kind} event type: #{type.inspect}" + end + + private def coercion_error(state) + return state[:error] if state[:error] + return if state.fetch(:exactness).fetch(:no).zero? + + ArgumentError.new("Realtime event is missing required fields or contains invalid values") + end + end + end +end diff --git a/lib/openai/realtime/connection.rb b/lib/openai/realtime/connection.rb new file mode 100644 index 000000000..c9dd1b3d3 --- /dev/null +++ b/lib/openai/realtime/connection.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module OpenAI + module Realtime + # A live, typed Realtime WebSocket connection. + class Connection < OpenAI::Realtime::BaseConnection + # @return [OpenAI::Realtime::ConnectionResources::Session] + attr_reader :session + + # @return [OpenAI::Realtime::ConnectionResources::Response] + attr_reader :response + + # @return [OpenAI::Realtime::ConnectionResources::InputAudioBuffer] + attr_reader :input_audio_buffer + + # @return [OpenAI::Realtime::ConnectionResources::Conversation] + attr_reader :conversation + + # @api private + def initialize(socket:, url:) + super( + socket: socket, + url: url, + server_event_type: OpenAI::Realtime::RealtimeServerEvent, + client_event_type: OpenAI::Realtime::RealtimeClientEvent + ) + @session = OpenAI::Realtime::ConnectionResources::Session.new(self) + @response = OpenAI::Realtime::ConnectionResources::Response.new(self) + @input_audio_buffer = OpenAI::Realtime::ConnectionResources::InputAudioBuffer.new(self) + @conversation = OpenAI::Realtime::ConnectionResources::Conversation.new(self) + end + end + end +end diff --git a/lib/openai/realtime/connection_manager.rb b/lib/openai/realtime/connection_manager.rb new file mode 100644 index 000000000..ebb247af4 --- /dev/null +++ b/lib/openai/realtime/connection_manager.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +module OpenAI + module Realtime + # Internal block-scoped lifecycle manager for Realtime WebSocket connections. + # + # @api private + class ConnectionManager + # @api private + def initialize( + client:, + path:, + query:, + connection_class:, + transport:, + request_options:, + transport_options: + ) + @client = client + @path = path + @query = query + @connection_class = connection_class + @transport = transport + @request_options = request_options + @transport_options = transport_options + end + + # Open the WebSocket and yield a typed connection for the lifetime of the block. + # + # @yieldparam connection [OpenAI::Realtime::Connection] + # @return [Object] + def open + raise ArgumentError, "A block is required to open a Realtime WebSocket." unless block_given? + + request = @client.realtime_connection_request( + path: @path, + query: @query, + options: @request_options + ) + transport = @transport || OpenAI::Realtime::Transports::AsyncWebSocket.new + unless transport.respond_to?(:open) + raise ArgumentError, "`transport` must respond to `open`" + end + + transport.open( + url: request.fetch(:url), + headers: request.fetch(:headers), + timeout: request.fetch(:timeout), + **@transport_options + ) do |socket| + connection = @connection_class.new(socket: socket, url: request.fetch(:url)) + begin + yield(connection) + ensure + pending_error = $ERROR_INFO + begin + connection.close unless connection.closed? + rescue StandardError + raise if pending_error.nil? + end + end + end + end + end + end +end diff --git a/lib/openai/realtime/connection_resources.rb b/lib/openai/realtime/connection_resources.rb new file mode 100644 index 000000000..c070cfedb --- /dev/null +++ b/lib/openai/realtime/connection_resources.rb @@ -0,0 +1,196 @@ +# frozen_string_literal: true + +module OpenAI + module Realtime + module ConnectionResources + class Base + # @api private + def initialize(connection) + @connection = connection + end + + private def compact_event(event) = event.compact + + private def event_payload(params, *metadata_keys) + payload = params.to_h.dup + metadata = metadata_keys.to_h do |key| + value = payload.key?(key) ? payload.delete(key) : payload.delete(key.to_s) + [key, value] + end + [payload, metadata] + end + end + + class Session < Base + # Update the session using Ruby-style resource keywords. The helper adds the + # protocol's nested `session` envelope. + def update(params) + session, metadata = event_payload(params, :event_id) + @connection.send_event( + compact_event(type: :"session.update", session: session, **metadata) + ) + end + end + + class TranscriptionSession < Base + # A transcription connection already selects its session capability. Add the + # required wire discriminator so callers only pass transcription fields. + def update(params) + session, metadata = event_payload(params, :event_id) + session.delete(:type) + session.delete("type") + session[:type] = :transcription + @connection.send_event( + compact_event(type: :"session.update", session: session, **metadata) + ) + end + end + + class TranslationSession < Session + # Gracefully flush and close a translation session. + def close(event_id: nil) + @connection.send_event(compact_event(type: :"session.close", event_id: event_id)) + end + end + + class Response < Base + # Create a response from resource fields, omitting the optional wire envelope + # when no response-specific fields are supplied. + def create(params = {}) + response, metadata = event_payload(params, :event_id) + @connection.send_event( + compact_event( + type: :"response.create", + response: response.empty? ? nil : response, + **metadata + ) + ) + end + + def cancel(response_id: nil, event_id: nil) + @connection.send_event( + compact_event(type: :"response.cancel", response_id: response_id, event_id: event_id) + ) + end + end + + class InputAudioBuffer < Base + def append(audio:, event_id: nil) + @connection.send_event( + compact_event(type: :"input_audio_buffer.append", audio: audio, event_id: event_id) + ) + end + + def append_bytes(bytes, event_id: nil) + append(audio: Base64.strict_encode64(bytes), event_id: event_id) + end + + def commit(event_id: nil) + @connection.send_event(compact_event(type: :"input_audio_buffer.commit", event_id: event_id)) + end + + def clear(event_id: nil) + @connection.send_event(compact_event(type: :"input_audio_buffer.clear", event_id: event_id)) + end + end + + class TranslationInputAudioBuffer < Base + def append(audio:, event_id: nil) + @connection.send_event( + compact_event(type: :"session.input_audio_buffer.append", audio: audio, event_id: event_id) + ) + end + + def append_bytes(bytes, event_id: nil) + append(audio: Base64.strict_encode64(bytes), event_id: event_id) + end + end + + class Conversation < Base + # @return [OpenAI::Realtime::ConnectionResources::ConversationItems] + attr_reader :items + + # @api private + def initialize(connection) + super + @items = ConversationItems.new(connection) + end + end + + class ConversationItems < Base + # Create an item from resource fields. Event metadata stays at the outer wire + # level while the remaining fields become the nested item. + def create(params) + item, metadata = event_payload(params, :event_id, :previous_item_id) + @connection.send_event( + compact_event( + type: :"conversation.item.create", + item: item, + **metadata + ) + ) + end + + def delete(item_id:, event_id: nil) + @connection.send_event( + compact_event(type: :"conversation.item.delete", item_id: item_id, event_id: event_id) + ) + end + + def retrieve(item_id:, event_id: nil) + @connection.send_event( + compact_event(type: :"conversation.item.retrieve", item_id: item_id, event_id: event_id) + ) + end + + def truncate(item_id:, content_index:, audio_end_ms:, event_id: nil) + @connection.send_event( + compact_event( + type: :"conversation.item.truncate", + item_id: item_id, + content_index: content_index, + audio_end_ms: audio_end_ms, + event_id: event_id + ) + ) + end + + # Send the output of a locally executed function call back to the model. + def create_function_call_output(call_id:, output:, id: nil, status: nil, event_id: nil) + item = compact_event( + type: :function_call_output, + call_id: call_id, + output: output, + id: id, + status: status + ) + create(**item, event_id: event_id) + end + + # Approve or reject a pending remote MCP tool call. + def respond_to_mcp_approval( + approval_request_id:, + approve:, + reason: nil, + id: nil, + event_id: nil + ) + item = compact_event( + type: :mcp_approval_response, + id: id || "mcpa_#{SecureRandom.hex(12)}", + approval_request_id: approval_request_id, + approve: approve, + reason: reason + ) + create(**item, event_id: event_id) + end + end + + class OutputAudioBuffer < Base + def clear(event_id: nil) + @connection.send_event(compact_event(type: :"output_audio_buffer.clear", event_id: event_id)) + end + end + end + end +end diff --git a/lib/openai/realtime/sideband_connection.rb b/lib/openai/realtime/sideband_connection.rb new file mode 100644 index 000000000..177e4f7db --- /dev/null +++ b/lib/openai/realtime/sideband_connection.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module OpenAI + module Realtime + # A server-side control connection attached to a WebRTC or SIP call. + class SidebandConnection < OpenAI::Realtime::Connection + # @return [OpenAI::Realtime::ConnectionResources::OutputAudioBuffer] + attr_reader :output_audio_buffer + + # @api private + def initialize(socket:, url:) + super + @output_audio_buffer = OpenAI::Realtime::ConnectionResources::OutputAudioBuffer.new(self) + end + end + end +end diff --git a/lib/openai/realtime/transcription_connection.rb b/lib/openai/realtime/transcription_connection.rb new file mode 100644 index 000000000..0d0a906cb --- /dev/null +++ b/lib/openai/realtime/transcription_connection.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module OpenAI + module Realtime + # A live connection for streaming transcription sessions. + class TranscriptionConnection < OpenAI::Realtime::BaseConnection + # @return [OpenAI::Realtime::ConnectionResources::TranscriptionSession] + attr_reader :session + + # @return [OpenAI::Realtime::ConnectionResources::InputAudioBuffer] + attr_reader :input_audio_buffer + + # @api private + def initialize(socket:, url:) + super( + socket: socket, + url: url, + server_event_type: OpenAI::Realtime::RealtimeServerEvent, + client_event_type: OpenAI::Realtime::RealtimeClientEvent + ) + @session = OpenAI::Realtime::ConnectionResources::TranscriptionSession.new(self) + @input_audio_buffer = OpenAI::Realtime::ConnectionResources::InputAudioBuffer.new(self) + end + end + end +end diff --git a/lib/openai/realtime/translation_connection.rb b/lib/openai/realtime/translation_connection.rb new file mode 100644 index 000000000..4f089a852 --- /dev/null +++ b/lib/openai/realtime/translation_connection.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module OpenAI + module Realtime + # A live connection for the dedicated continuous translation protocol. + class TranslationConnection < OpenAI::Realtime::BaseConnection + # @return [OpenAI::Realtime::ConnectionResources::TranslationSession] + attr_reader :session + + # @return [OpenAI::Realtime::ConnectionResources::TranslationInputAudioBuffer] + attr_reader :input_audio_buffer + + # @api private + def initialize(socket:, url:) + super( + socket: socket, + url: url, + server_event_type: OpenAI::Realtime::RealtimeTranslationServerEvent, + client_event_type: OpenAI::Realtime::RealtimeTranslationClientEvent + ) + @session = OpenAI::Realtime::ConnectionResources::TranslationSession.new(self) + @input_audio_buffer = OpenAI::Realtime::ConnectionResources::TranslationInputAudioBuffer.new(self) + end + end + end +end diff --git a/lib/openai/realtime/transports/async_websocket.rb b/lib/openai/realtime/transports/async_websocket.rb new file mode 100644 index 000000000..29dbea4f5 --- /dev/null +++ b/lib/openai/realtime/transports/async_websocket.rb @@ -0,0 +1,78 @@ +# frozen_string_literal: true + +module OpenAI + module Realtime + module Transports + # Default Realtime transport backed by the optional async-websocket gem. + class AsyncWebSocket + class Socket + # @api private + def initialize(connection, url:) + @connection = connection + @url = url + end + + def read + @connection.read + rescue StandardError => e + raise OpenAI::Errors::RealtimeConnectionError.new(url: @url, cause: e) + end + + def write(message) + @connection.write(message) + @connection.flush + rescue StandardError => e + raise OpenAI::Errors::RealtimeConnectionError.new(url: @url, cause: e) + end + + def close(code: 1000, reason: "") + @connection.close(code, reason) + rescue StandardError => e + raise OpenAI::Errors::RealtimeConnectionError.new(url: @url, cause: e) + end + + def closed? = @connection.closed? + end + + def open(url:, headers:, timeout:, **endpoint_options) + load_dependencies(url) + + # Classic WebSocket negotiation uses HTTP/1.1. Pinning ALPN also avoids an + # HTTP/2 selection on servers that advertise both protocols. + options = { + alpn_protocols: ::Async::HTTP::Protocol::HTTP11.names, + **endpoint_options + } + endpoint = ::Async::HTTP::Endpoint.parse( + url.to_s, + timeout: timeout, + **options + ) + block_error = nil + ::Async::WebSocket::Client.connect(endpoint, headers: headers) do |connection| + yield(Socket.new(connection, url: url)) + rescue StandardError => e + block_error = e + raise + end + rescue OpenAI::Errors::RealtimeConnectionError + raise + rescue StandardError => e + raise if e.equal?(block_error) + + raise OpenAI::Errors::RealtimeConnectionError.new(url: url, cause: e) + end + + private def load_dependencies(url) + require("async/websocket/client") + require("async/http/endpoint") + rescue LoadError => e + message = + "Realtime WebSockets require the `async-websocket` gem. " \ + "Add `gem \"async-websocket\"` to your Gemfile." + raise OpenAI::Errors::RealtimeConnectionError.new(url: url, message: message, cause: e) + end + end + end + end +end diff --git a/lib/openai/realtime/unknown_server_event.rb b/lib/openai/realtime/unknown_server_event.rb new file mode 100644 index 000000000..99a1818d4 --- /dev/null +++ b/lib/openai/realtime/unknown_server_event.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +module OpenAI + module Realtime + # A valid JSON event whose discriminator is newer than this SDK version. + class UnknownServerEvent + # @return [Symbol] + attr_reader :type + + # @return [Hash{Symbol=>Object}] + attr_reader :data + + # @api private + def initialize(data:) + value = data.fetch(:type) + unless value.is_a?(String) || value.is_a?(Symbol) + raise ArgumentError, "Realtime server event type must be a string or symbol" + end + + @type = value.to_sym + @data = freeze_json(data) + freeze + end + + # @return [Hash{Symbol=>Object}] + def to_h = @data + + private def freeze_json(value) + case value + when Hash + value.each do |key, item| + freeze_json(key) + freeze_json(item) + end + when Array + value.each { |item| freeze_json(item) } + end + value.freeze + end + end + end +end diff --git a/lib/openai/resources/realtime.rb b/lib/openai/resources/realtime.rb index f361d152a..7625acf49 100644 --- a/lib/openai/resources/realtime.rb +++ b/lib/openai/resources/realtime.rb @@ -9,6 +9,66 @@ class Realtime # @return [OpenAI::Resources::Realtime::Calls] attr_reader :calls + # @return [OpenAI::Resources::Realtime::Translations] + attr_reader :translations + + # Open a server-side Realtime WebSocket. Pass exactly one of `model` to start a + # conversation session, `intent: :transcription` to start transcription, or + # `call_id` to attach a sideband connection to a WebRTC or SIP call. A block is + # required and the socket is closed on every exit path. + # + # @param model [String, nil] + # @param call_id [String, nil] + # @param intent [Symbol, nil] + # @param request_options [OpenAI::RequestOptions, Hash{Symbol=>Object}, nil] + # @param transport [#open, nil] An alternate WebSocket transport. + # @param transport_options [Hash{Symbol=>Object}] + # @yieldparam connection [OpenAI::Realtime::Connection] + # @return [Object] + def connect( + model: nil, + call_id: nil, + intent: nil, + request_options: nil, + transport: nil, + transport_options: {}, + &block + ) + raise ArgumentError, "A block is required to open a Realtime WebSocket." unless block + + targets = [model, call_id, intent].compact + unless targets.one? + message = "Pass exactly one of `model`, `call_id`, or `intent` when opening a Realtime WebSocket." + raise ArgumentError, message + end + unless intent.nil? || intent == :transcription + raise ArgumentError, "The only supported Realtime connection intent is `transcription`." + end + + query, connection_class = connection_target(model: model, call_id: call_id, intent: intent) + manager = OpenAI::Realtime::ConnectionManager.new( + client: @client, + path: "realtime", + query: query, + connection_class: connection_class, + transport: transport, + request_options: request_options, + transport_options: transport_options + ) + manager.open(&block) + end + + # @api private + def connection_target(model:, call_id:, intent:) + if model + [{"model" => model}, OpenAI::Realtime::Connection] + elsif call_id + [{"call_id" => call_id}, OpenAI::Realtime::SidebandConnection] + else + [{"intent" => intent.to_s}, OpenAI::Realtime::TranscriptionConnection] + end + end + # @api private # # @param client [OpenAI::Client] @@ -16,6 +76,7 @@ def initialize(client:) @client = client @client_secrets = OpenAI::Resources::Realtime::ClientSecrets.new(client: client) @calls = OpenAI::Resources::Realtime::Calls.new(client: client) + @translations = OpenAI::Resources::Realtime::Translations.new(client: client) end end end diff --git a/lib/openai/resources/realtime/calls.rb b/lib/openai/resources/realtime/calls.rb index a0cfb6f4d..81c1a1b4f 100644 --- a/lib/openai/resources/realtime/calls.rb +++ b/lib/openai/resources/realtime/calls.rb @@ -4,6 +4,52 @@ module OpenAI module Resources class Realtime class Calls + # Create a WebRTC call from an SDP offer. When `session` is supplied, the SDK + # sends the offer and session configuration as typed multipart form parts. + # + # @overload create(sdp:, session: nil, request_options: {}) + # + # @param sdp [String] The WebRTC Session Description Protocol offer. + # @param session [OpenAI::Models::Realtime::RealtimeSessionCreateRequest] Optional session configuration. + # @param request_options [OpenAI::RequestOptions, Hash{Symbol=>Object}, nil] + # + # @return [OpenAI::Models::Realtime::CallCreateResponse] + # + # @see OpenAI::Models::Realtime::CallCreateParams + def create(params) + parsed, options = OpenAI::Realtime::CallCreateParams.dump_request(params) + sdp = parsed.fetch(:sdp) + session = parsed[:session] + headers, body = + if session.nil? + [{"accept" => "application/sdp", "content-type" => "application/sdp"}, sdp] + else + [ + {"accept" => "application/sdp", "content-type" => "multipart/form-data"}, + { + sdp: OpenAI::FilePart.new(sdp, content_type: "application/sdp"), + session: OpenAI::FilePart.new(JSON.generate(session), content_type: "application/json") + } + ] + end + + response = @client.request_raw( + method: :post, + path: "realtime/calls", + headers: headers, + body: body, + security: {bearer_auth: true}, + options: options + ) + location = response.headers["location"] + call_id = location&.then { URI.parse(_1).path.split("/").last } + OpenAI::Realtime::CallCreateResponse.new( + sdp: response.body.to_a.join, + call_id: call_id, + headers: response.headers + ) + end + # Some parameter documentations has been truncated, see # {OpenAI::Models::Realtime::CallAcceptParams} for more details. # diff --git a/lib/openai/resources/realtime/translations.rb b/lib/openai/resources/realtime/translations.rb new file mode 100644 index 000000000..3f8e81ffc --- /dev/null +++ b/lib/openai/resources/realtime/translations.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module OpenAI + module Resources + class Realtime + # Dedicated Realtime translation endpoints and WebSocket connections. + class Translations + # @return [OpenAI::Resources::Realtime::Translations::ClientSecrets] + attr_reader :client_secrets + + # @return [OpenAI::Resources::Realtime::Translations::Calls] + attr_reader :calls + + # Open a typed translation WebSocket connection. + # + # @param model [String] + # @param request_options [OpenAI::RequestOptions, Hash{Symbol=>Object}, nil] + # @param transport [#open, nil] + # @param transport_options [Hash{Symbol=>Object}] + # @yieldparam connection [OpenAI::Realtime::TranslationConnection] + # @return [Object] + def connect(model:, request_options: nil, transport: nil, transport_options: {}, &block) + raise ArgumentError, "A block is required to open a Realtime WebSocket." unless block + + manager = OpenAI::Realtime::ConnectionManager.new( + client: @client, + path: "realtime/translations", + query: {"model" => model}, + connection_class: OpenAI::Realtime::TranslationConnection, + transport: transport, + request_options: request_options, + transport_options: transport_options + ) + manager.open(&block) + end + + # @api private + def initialize(client:) + @client = client + @client_secrets = ClientSecrets.new(client: client) + @calls = Calls.new(client: client) + end + end + end + end +end diff --git a/lib/openai/resources/realtime/translations/calls.rb b/lib/openai/resources/realtime/translations/calls.rb new file mode 100644 index 000000000..2dd04a824 --- /dev/null +++ b/lib/openai/resources/realtime/translations/calls.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +module OpenAI + module Resources + class Realtime + class Translations + class Calls + # Create a translation WebRTC call from an SDP offer. + # + # @param sdp [String] + # @param request_options [OpenAI::RequestOptions, Hash{Symbol=>Object}, nil] + # @return [OpenAI::Models::Realtime::CallCreateResponse] + def create(sdp:, request_options: nil) + response = @client.request_raw( + method: :post, + path: "realtime/translations/calls", + headers: {"accept" => "application/sdp", "content-type" => "application/sdp"}, + body: String(sdp), + security: {bearer_auth: true}, + options: request_options + ) + location = response.headers["location"] + call_id = location&.then { URI.parse(_1).path.split("/").last } + OpenAI::Realtime::CallCreateResponse.new( + sdp: response.body.to_a.join, + call_id: call_id, + headers: response.headers + ) + end + + # @api private + def initialize(client:) + @client = client + end + end + end + end + end +end diff --git a/lib/openai/resources/realtime/translations/client_secrets.rb b/lib/openai/resources/realtime/translations/client_secrets.rb new file mode 100644 index 000000000..d3c59d3eb --- /dev/null +++ b/lib/openai/resources/realtime/translations/client_secrets.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +module OpenAI + module Resources + class Realtime + class Translations + class ClientSecrets + # Create a short-lived client secret for a Realtime translation session. + # + # @param session [OpenAI::Models::Realtime::RealtimeTranslationSessionCreateRequest, Hash] + # @param expires_after [OpenAI::Models::Realtime::RealtimeTranslationClientSecretCreateRequest::ExpiresAfter, Hash, nil] + # @param request_options [OpenAI::RequestOptions, Hash{Symbol=>Object}, nil] + # @return [OpenAI::Models::Realtime::RealtimeTranslationClientSecretCreateResponse] + def create(session:, expires_after: nil, request_options: nil) + input = {session: session} + input[:expires_after] = expires_after unless expires_after.nil? + state = OpenAI::Internal::Type::Converter.new_coerce_state + request = OpenAI::Internal::Type::Converter.coerce( + OpenAI::Realtime::RealtimeTranslationClientSecretCreateRequest, + input, + state: state + ) + cause = state[:error] + if cause.nil? && !state.fetch(:exactness).fetch(:no).zero? + cause = ArgumentError.new("request is missing required fields or contains invalid values") + end + if cause + message = "Invalid translation client secret request: #{cause.message}" + raise ArgumentError.new(message), cause: cause + end + body = OpenAI::Internal::Type::Converter.dump( + OpenAI::Realtime::RealtimeTranslationClientSecretCreateRequest, + request + ) + + @client.request( + method: :post, + path: "realtime/translations/client_secrets", + body: body, + model: OpenAI::Realtime::RealtimeTranslationClientSecretCreateResponse, + security: {bearer_auth: true}, + options: request_options + ) + end + + # @api private + def initialize(client:) + @client = client + end + end + end + end + end +end diff --git a/openai.gemspec b/openai.gemspec index 27122f711..517393174 100644 --- a/openai.gemspec +++ b/openai.gemspec @@ -24,7 +24,7 @@ Gem::Specification.new do |s| "CHANGELOG.md", ".ignore" ] - s.extra_rdoc_files = ["README.md", "VERSIONING.md"] + s.extra_rdoc_files = ["README.md", "VERSIONING.md", "realtime.md"] s.add_dependency "base64" s.add_dependency "cgi" s.add_dependency "connection_pool", ">= 2.2.3" diff --git a/rbi/openai/client.rbi b/rbi/openai/client.rbi index 95e4ec364..71b893cf1 100644 --- a/rbi/openai/client.rbi +++ b/rbi/openai/client.rbi @@ -25,6 +25,9 @@ module OpenAI sig { returns(T.nilable(String)) } attr_reader :webhook_secret + sig { returns(T.nilable(URI::Generic)) } + attr_reader :websocket_base_url + # Given a prompt, the model will return one or more predicted completions, and can # also return the probabilities of alternative tokens at each position. sig { returns(OpenAI::Resources::Completions) } @@ -135,6 +138,17 @@ module OpenAI private def admin_api_key_auth end + # @api private + sig do + params( + path: String, + query: T::Hash[String, String], + options: T.nilable(OpenAI::RequestOptions::OrHash) + ).returns(OpenAI::Internal::Transport::BaseClient::RequestInput) + end + def realtime_connection_request(path:, query:, options: nil) + end + # @api private sig do override @@ -160,6 +174,7 @@ module OpenAI provider: T.nilable(OpenAI::Provider), base_url: T.nilable(String), default_headers: T.nilable(T::Hash[String, T.nilable(String)]), + websocket_base_url: T.nilable(String), max_retries: Integer, timeout: T.nilable(Float), initial_retry_delay: Float, @@ -189,6 +204,7 @@ module OpenAI # Extra headers to send with every request. Explicit values override # `ENV["OPENAI_CUSTOM_HEADERS"]`. default_headers: nil, + websocket_base_url: nil, # Max number of retries to attempt after a failed retryable request. max_retries: OpenAI::Client::DEFAULT_MAX_RETRIES, timeout: OpenAI::Client::DEFAULT_TIMEOUT_IN_SECONDS, diff --git a/rbi/openai/errors.rbi b/rbi/openai/errors.rbi index 5307238e2..d955c8164 100644 --- a/rbi/openai/errors.rbi +++ b/rbi/openai/errors.rbi @@ -3,7 +3,7 @@ module OpenAI module Errors class Error < StandardError - sig { returns(T.nilable(StandardError)) } + sig { returns(T.nilable(Exception)) } attr_accessor :cause end @@ -26,6 +26,44 @@ module OpenAI end end + class RealtimeConnectionError < OpenAI::Errors::Error + sig { returns(URI::Generic) } + attr_reader :url + + sig { returns(T.nilable(Exception)) } + def cause + end + + sig do + params( + url: URI::Generic, + message: T.nilable(String), + cause: T.nilable(Exception) + ).returns(T.attached_class) + end + def self.new(url:, message: nil, cause: nil) + end + end + + class RealtimeProtocolError < OpenAI::Errors::Error + sig { returns(String) } + attr_reader :data + + sig { returns(T.nilable(StandardError)) } + def cause + end + + sig do + params( + data: String, + message: T.nilable(String), + cause: T.nilable(StandardError) + ).returns(T.attached_class) + end + def self.new(data:, message: nil, cause: nil) + end + end + class APIError < OpenAI::Errors::Error sig { returns(URI::Generic) } attr_accessor :url diff --git a/rbi/openai/internal/transport/base_client.rbi b/rbi/openai/internal/transport/base_client.rbi index 627dc659f..7465ea40c 100644 --- a/rbi/openai/internal/transport/base_client.rbi +++ b/rbi/openai/internal/transport/base_client.rbi @@ -302,6 +302,15 @@ module OpenAI ) end + # @api private + sig do + params( + req: OpenAI::Internal::Transport::BaseClient::RequestComponents + ).returns(OpenAI::HTTPClient::Response) + end + def request_raw(req) + end + # Execute the request specified by `req`. This is the method that all resource # methods call into. # diff --git a/rbi/openai/models/realtime/call_create_params.rbi b/rbi/openai/models/realtime/call_create_params.rbi new file mode 100644 index 000000000..ab5ff8a41 --- /dev/null +++ b/rbi/openai/models/realtime/call_create_params.rbi @@ -0,0 +1,49 @@ +# typed: strong + +module OpenAI + module Models + module Realtime + class CallCreateParams < OpenAI::Internal::Type::BaseModel + extend OpenAI::Internal::Type::RequestParameters::Converter + include OpenAI::Internal::Type::RequestParameters + + OrHash = + T.type_alias do + T.any(OpenAI::Realtime::CallCreateParams, OpenAI::Internal::AnyHash) + end + + sig { returns(String) } + attr_accessor :sdp + + sig do + returns(T.nilable(OpenAI::Realtime::RealtimeSessionCreateRequest)) + end + attr_accessor :session + + sig do + params( + sdp: String, + session: + T.nilable(OpenAI::Realtime::RealtimeSessionCreateRequest::OrHash), + request_options: OpenAI::RequestOptions::OrHash + ).returns(T.attached_class) + end + def self.new(sdp:, session: nil, request_options: {}) + end + + sig do + override.returns( + { + sdp: String, + session: + T.nilable(OpenAI::Realtime::RealtimeSessionCreateRequest), + request_options: OpenAI::RequestOptions + } + ) + end + def to_hash + end + end + end + end +end diff --git a/rbi/openai/models/realtime/call_create_response.rbi b/rbi/openai/models/realtime/call_create_response.rbi new file mode 100644 index 000000000..5e1fa9645 --- /dev/null +++ b/rbi/openai/models/realtime/call_create_response.rbi @@ -0,0 +1,48 @@ +# typed: strong + +module OpenAI + module Models + module Realtime + class CallCreateResponse < OpenAI::Internal::Type::BaseModel + OrHash = + T.type_alias do + T.any( + OpenAI::Realtime::CallCreateResponse, + OpenAI::Internal::AnyHash + ) + end + + sig { returns(String) } + attr_accessor :sdp + + sig { returns(T.nilable(String)) } + attr_accessor :call_id + + sig { returns(T::Hash[String, String]) } + attr_accessor :headers + + sig do + params( + sdp: String, + headers: T::Hash[String, String], + call_id: T.nilable(String) + ).returns(T.attached_class) + end + def self.new(sdp:, headers:, call_id: nil) + end + + sig do + override.returns( + { + sdp: String, + call_id: T.nilable(String), + headers: T::Hash[String, String] + } + ) + end + def to_hash + end + end + end + end +end diff --git a/rbi/openai/realtime/base_connection.rbi b/rbi/openai/realtime/base_connection.rbi new file mode 100644 index 000000000..1b1b515f5 --- /dev/null +++ b/rbi/openai/realtime/base_connection.rbi @@ -0,0 +1,40 @@ +# typed: strong + +module OpenAI + module Models + module Realtime + class BaseConnection + sig { returns(URI::Generic) } + attr_reader :url + + # @api private + sig do + params( + socket: T.untyped, + url: URI::Generic, + server_event_type: T.untyped, + client_event_type: T.untyped + ).returns(T.attached_class) + end + def self.new(socket:, url:, server_event_type:, client_event_type:) + end + + sig { returns(T.nilable(String)) } + def receive_raw + end + + sig { params(data: String).void } + def send_raw(data) + end + + sig { params(code: Integer, reason: String).void } + def close(code: 1000, reason: "") + end + + sig { returns(T::Boolean) } + def closed? + end + end + end + end +end diff --git a/rbi/openai/realtime/connection.rbi b/rbi/openai/realtime/connection.rbi new file mode 100644 index 000000000..0a9e05d5e --- /dev/null +++ b/rbi/openai/realtime/connection.rbi @@ -0,0 +1,70 @@ +# typed: strong + +module OpenAI + module Models + module Realtime + class Connection < OpenAI::Realtime::BaseConnection + include Enumerable + + ServerEvent = + T.type_alias do + T.any( + OpenAI::Realtime::RealtimeServerEvent::Variants, + OpenAI::Realtime::UnknownServerEvent + ) + end + + ClientEvent = + T.type_alias do + T.any( + OpenAI::Realtime::RealtimeClientEvent::Variants, + OpenAI::Internal::AnyHash + ) + end + + Elem = type_member { { fixed: ServerEvent } } + + sig { returns(OpenAI::Realtime::ConnectionResources::Session) } + attr_reader :session + + sig { returns(OpenAI::Realtime::ConnectionResources::Response) } + attr_reader :response + + sig { returns(OpenAI::Realtime::ConnectionResources::InputAudioBuffer) } + attr_reader :input_audio_buffer + + sig { returns(OpenAI::Realtime::ConnectionResources::Conversation) } + attr_reader :conversation + + # @api private + sig do + params(socket: T.untyped, url: URI::Generic).returns(T.attached_class) + end + def self.new(socket:, url:) + end + + sig do + params( + block: T.nilable(T.proc.params(event: ServerEvent).void) + ).returns( + T.any(OpenAI::Realtime::Connection, T::Enumerator[ServerEvent]) + ) + end + def each(&block) + end + + sig { returns(T.nilable(ServerEvent)) } + def receive + end + + sig { params(data: String).returns(ServerEvent) } + def parse_event(data) + end + + sig { params(event: ClientEvent).void } + def send_event(event) + end + end + end + end +end diff --git a/rbi/openai/realtime/connection_manager.rbi b/rbi/openai/realtime/connection_manager.rbi new file mode 100644 index 000000000..5af34758a --- /dev/null +++ b/rbi/openai/realtime/connection_manager.rbi @@ -0,0 +1,44 @@ +# typed: strong + +module OpenAI + module Models + module Realtime + class ConnectionManager + # @api private + sig do + params( + client: OpenAI::Client, + path: String, + query: T::Hash[String, String], + connection_class: T.class_of(OpenAI::Realtime::BaseConnection), + transport: T.untyped, + request_options: T.nilable(OpenAI::RequestOptions::OrHash), + transport_options: T::Hash[Symbol, T.untyped] + ).returns(T.attached_class) + end + def self.new( + client:, + path:, + query:, + connection_class:, + transport:, + request_options:, + transport_options: + ) + end + + sig do + params( + block: + T + .proc + .params(connection: OpenAI::Realtime::BaseConnection) + .returns(T.untyped) + ).returns(T.untyped) + end + def open(&block) + end + end + end + end +end diff --git a/rbi/openai/realtime/connection_resources.rbi b/rbi/openai/realtime/connection_resources.rbi new file mode 100644 index 000000000..1d74ea96b --- /dev/null +++ b/rbi/openai/realtime/connection_resources.rbi @@ -0,0 +1,328 @@ +# typed: strong + +module OpenAI + module Models + module Realtime + module ConnectionResources + class Base + # @api private + sig do + params(connection: OpenAI::Realtime::BaseConnection).returns( + T.attached_class + ) + end + def self.new(connection) + end + end + + class Session < Base + sig do + params( + type: Symbol, + audio: + T.any( + OpenAI::Realtime::RealtimeAudioConfig::OrHash, + OpenAI::Realtime::RealtimeTranscriptionSessionAudio::OrHash + ), + include: T::Array[Symbol], + instructions: String, + max_output_tokens: T.any(Integer, Symbol), + model: T.any(String, Symbol), + output_modalities: T::Array[Symbol], + parallel_tool_calls: T::Boolean, + prompt: T.nilable(OpenAI::Responses::ResponsePrompt::OrHash), + reasoning: OpenAI::Realtime::RealtimeReasoning::OrHash, + tool_choice: + T.any( + OpenAI::Responses::ToolChoiceOptions::OrSymbol, + OpenAI::Responses::ToolChoiceFunction::OrHash, + OpenAI::Responses::ToolChoiceMcp::OrHash + ), + tools: + T::Array[ + T.any( + OpenAI::Realtime::RealtimeFunctionTool::OrHash, + OpenAI::Realtime::RealtimeToolsConfigUnion::Mcp::OrHash + ) + ], + tracing: + T.nilable( + T.any( + Symbol, + OpenAI::Realtime::RealtimeTracingConfig::TracingConfiguration::OrHash + ) + ), + truncation: + T.any( + OpenAI::Realtime::RealtimeTruncation::RealtimeTruncationStrategy::OrSymbol, + OpenAI::Realtime::RealtimeTruncationRetentionRatio::OrHash + ), + event_id: T.nilable(String) + ).void + end + def update( + type:, + audio: nil, + include: nil, + instructions: nil, + max_output_tokens: nil, + model: nil, + output_modalities: nil, + parallel_tool_calls: nil, + prompt: nil, + reasoning: nil, + tool_choice: nil, + tools: nil, + tracing: nil, + truncation: nil, + event_id: nil + ) + end + end + + class TranscriptionSession < Base + sig do + params( + audio: + OpenAI::Realtime::RealtimeTranscriptionSessionAudio::OrHash, + include: + T::Array[ + OpenAI::Realtime::RealtimeTranscriptionSessionCreateRequest::Include::OrSymbol + ], + event_id: T.nilable(String) + ).void + end + def update(audio: nil, include: nil, event_id: nil) + end + end + + class TranslationSession < Session + sig do + params( + audio: + OpenAI::Realtime::RealtimeTranslationSessionUpdateRequest::Audio::OrHash, + event_id: T.nilable(String) + ).void + end + def update(audio: nil, event_id: nil) + end + + sig { params(event_id: T.nilable(String)).void } + def close(event_id: nil) + end + end + + class Response < Base + sig do + params( + audio: OpenAI::Realtime::RealtimeResponseCreateAudioOutput::OrHash, + conversation: T.any(String, Symbol), + input: + T::Array[ + T.any( + OpenAI::Realtime::RealtimeConversationItemSystemMessage::OrHash, + OpenAI::Realtime::RealtimeConversationItemUserMessage::OrHash, + OpenAI::Realtime::RealtimeConversationItemAssistantMessage::OrHash, + OpenAI::Realtime::RealtimeConversationItemFunctionCall::OrHash, + OpenAI::Realtime::RealtimeConversationItemFunctionCallOutput::OrHash, + OpenAI::Realtime::RealtimeMcpApprovalResponse::OrHash, + OpenAI::Realtime::RealtimeMcpListTools::OrHash, + OpenAI::Realtime::RealtimeMcpToolCall::OrHash, + OpenAI::Realtime::RealtimeMcpApprovalRequest::OrHash + ) + ], + instructions: String, + max_output_tokens: T.any(Integer, Symbol), + metadata: T.nilable(T::Hash[Symbol, String]), + output_modalities: T::Array[Symbol], + parallel_tool_calls: T::Boolean, + prompt: T.nilable(OpenAI::Responses::ResponsePrompt::OrHash), + reasoning: OpenAI::Realtime::RealtimeReasoning::OrHash, + tool_choice: + T.any( + OpenAI::Responses::ToolChoiceOptions::OrSymbol, + OpenAI::Responses::ToolChoiceFunction::OrHash, + OpenAI::Responses::ToolChoiceMcp::OrHash + ), + tools: + T::Array[ + T.any( + OpenAI::Realtime::RealtimeFunctionTool::OrHash, + OpenAI::Realtime::RealtimeResponseCreateMcpTool::OrHash + ) + ], + event_id: T.nilable(String) + ).void + end + def create( + audio: nil, + conversation: nil, + input: nil, + instructions: nil, + max_output_tokens: nil, + metadata: nil, + output_modalities: nil, + parallel_tool_calls: nil, + prompt: nil, + reasoning: nil, + tool_choice: nil, + tools: nil, + event_id: nil + ) + end + + sig do + params( + response_id: T.nilable(String), + event_id: T.nilable(String) + ).void + end + def cancel(response_id: nil, event_id: nil) + end + end + + class InputAudioBuffer < Base + sig { params(audio: String, event_id: T.nilable(String)).void } + def append(audio:, event_id: nil) + end + + sig { params(bytes: String, event_id: T.nilable(String)).void } + def append_bytes(bytes, event_id: nil) + end + + sig { params(event_id: T.nilable(String)).void } + def commit(event_id: nil) + end + + sig { params(event_id: T.nilable(String)).void } + def clear(event_id: nil) + end + end + + class TranslationInputAudioBuffer < Base + sig { params(audio: String, event_id: T.nilable(String)).void } + def append(audio:, event_id: nil) + end + + sig { params(bytes: String, event_id: T.nilable(String)).void } + def append_bytes(bytes, event_id: nil) + end + end + + class Conversation < Base + sig do + returns(OpenAI::Realtime::ConnectionResources::ConversationItems) + end + attr_reader :items + end + + class ConversationItems < Base + sig do + params( + type: Symbol, + arguments: String, + approval_request_id: T.nilable(String), + approve: T::Boolean, + call_id: String, + content: T::Array[OpenAI::Internal::AnyHash], + error: T.untyped, + id: String, + name: String, + object: Symbol, + output: T.nilable(String), + reason: T.nilable(String), + role: Symbol, + server_label: String, + status: Symbol, + tools: T::Array[OpenAI::Realtime::RealtimeMcpListTools::Tool::OrHash], + event_id: T.nilable(String), + previous_item_id: T.nilable(String) + ).void + end + def create( + type:, + arguments: nil, + approval_request_id: nil, + approve: nil, + call_id: nil, + content: nil, + error: nil, + id: nil, + name: nil, + object: nil, + output: nil, + reason: nil, + role: nil, + server_label: nil, + status: nil, + tools: nil, + event_id: nil, + previous_item_id: nil + ) + end + + sig { params(item_id: String, event_id: T.nilable(String)).void } + def delete(item_id:, event_id: nil) + end + + sig { params(item_id: String, event_id: T.nilable(String)).void } + def retrieve(item_id:, event_id: nil) + end + + sig do + params( + item_id: String, + content_index: Integer, + audio_end_ms: Integer, + event_id: T.nilable(String) + ).void + end + def truncate(item_id:, content_index:, audio_end_ms:, event_id: nil) + end + + sig do + params( + call_id: String, + output: String, + id: T.nilable(String), + status: T.nilable(T.any(String, Symbol)), + event_id: T.nilable(String) + ).void + end + def create_function_call_output( + call_id:, + output:, + id: nil, + status: nil, + event_id: nil + ) + end + + sig do + params( + approval_request_id: String, + approve: T::Boolean, + reason: T.nilable(String), + id: T.nilable(String), + event_id: T.nilable(String) + ).void + end + def respond_to_mcp_approval( + approval_request_id:, + approve:, + reason: nil, + id: nil, + event_id: nil + ) + end + end + + class OutputAudioBuffer < Base + sig { params(event_id: T.nilable(String)).void } + def clear(event_id: nil) + end + end + end + end + end +end diff --git a/rbi/openai/realtime/sideband_connection.rbi b/rbi/openai/realtime/sideband_connection.rbi new file mode 100644 index 000000000..2e0e0c304 --- /dev/null +++ b/rbi/openai/realtime/sideband_connection.rbi @@ -0,0 +1,24 @@ +# typed: strong + +module OpenAI + module Models + module Realtime + class SidebandConnection < OpenAI::Realtime::Connection + Elem = + type_member { { fixed: OpenAI::Realtime::Connection::ServerEvent } } + + sig do + returns(OpenAI::Realtime::ConnectionResources::OutputAudioBuffer) + end + attr_reader :output_audio_buffer + + # @api private + sig do + params(socket: T.untyped, url: URI::Generic).returns(T.attached_class) + end + def self.new(socket:, url:) + end + end + end + end +end diff --git a/rbi/openai/realtime/transcription_connection.rbi b/rbi/openai/realtime/transcription_connection.rbi new file mode 100644 index 000000000..ed6feb5a2 --- /dev/null +++ b/rbi/openai/realtime/transcription_connection.rbi @@ -0,0 +1,71 @@ +# typed: strong + +module OpenAI + module Models + module Realtime + class TranscriptionConnection < OpenAI::Realtime::BaseConnection + include Enumerable + + ServerEvent = + T.type_alias do + T.any( + OpenAI::Realtime::RealtimeServerEvent::Variants, + OpenAI::Realtime::UnknownServerEvent + ) + end + + ClientEvent = + T.type_alias do + T.any( + OpenAI::Realtime::RealtimeClientEvent::Variants, + OpenAI::Internal::AnyHash + ) + end + + Elem = type_member { { fixed: ServerEvent } } + + sig do + returns(OpenAI::Realtime::ConnectionResources::TranscriptionSession) + end + attr_reader :session + + sig do + returns(OpenAI::Realtime::ConnectionResources::InputAudioBuffer) + end + attr_reader :input_audio_buffer + + # @api private + sig do + params(socket: T.untyped, url: URI::Generic).returns(T.attached_class) + end + def self.new(socket:, url:) + end + + sig do + params( + block: T.nilable(T.proc.params(event: ServerEvent).void) + ).returns( + T.any( + OpenAI::Realtime::TranscriptionConnection, + T::Enumerator[ServerEvent] + ) + ) + end + def each(&block) + end + + sig { returns(T.nilable(ServerEvent)) } + def receive + end + + sig { params(data: String).returns(ServerEvent) } + def parse_event(data) + end + + sig { params(event: ClientEvent).void } + def send_event(event) + end + end + end + end +end diff --git a/rbi/openai/realtime/translation_connection.rbi b/rbi/openai/realtime/translation_connection.rbi new file mode 100644 index 000000000..77ef0e0c2 --- /dev/null +++ b/rbi/openai/realtime/translation_connection.rbi @@ -0,0 +1,73 @@ +# typed: strong + +module OpenAI + module Models + module Realtime + class TranslationConnection < OpenAI::Realtime::BaseConnection + include Enumerable + + ServerEvent = + T.type_alias do + T.any( + OpenAI::Realtime::RealtimeTranslationServerEvent::Variants, + OpenAI::Realtime::UnknownServerEvent + ) + end + + ClientEvent = + T.type_alias do + T.any( + OpenAI::Realtime::RealtimeTranslationClientEvent::Variants, + OpenAI::Internal::AnyHash + ) + end + + Elem = type_member { { fixed: ServerEvent } } + + sig do + returns(OpenAI::Realtime::ConnectionResources::TranslationSession) + end + attr_reader :session + + sig do + returns( + OpenAI::Realtime::ConnectionResources::TranslationInputAudioBuffer + ) + end + attr_reader :input_audio_buffer + + # @api private + sig do + params(socket: T.untyped, url: URI::Generic).returns(T.attached_class) + end + def self.new(socket:, url:) + end + + sig do + params( + block: T.nilable(T.proc.params(event: ServerEvent).void) + ).returns( + T.any( + OpenAI::Realtime::TranslationConnection, + T::Enumerator[ServerEvent] + ) + ) + end + def each(&block) + end + + sig { returns(T.nilable(ServerEvent)) } + def receive + end + + sig { params(data: String).returns(ServerEvent) } + def parse_event(data) + end + + sig { params(event: ClientEvent).void } + def send_event(event) + end + end + end + end +end diff --git a/rbi/openai/realtime/transports/async_websocket.rbi b/rbi/openai/realtime/transports/async_websocket.rbi new file mode 100644 index 000000000..059bbc273 --- /dev/null +++ b/rbi/openai/realtime/transports/async_websocket.rbi @@ -0,0 +1,50 @@ +# typed: strong + +module OpenAI + module Models + module Realtime + module Transports + class AsyncWebSocket + class Socket + # @api private + sig do + params(connection: T.untyped, url: URI::Generic).returns( + T.attached_class + ) + end + def self.new(connection, url:) + end + + sig { returns(T.untyped) } + def read + end + + sig { params(message: String).void } + def write(message) + end + + sig { params(code: Integer, reason: String).void } + def close(code: 1000, reason: "") + end + + sig { returns(T::Boolean) } + def closed? + end + end + + sig do + params( + url: URI::Generic, + headers: T::Hash[String, String], + timeout: Float, + endpoint_options: T.untyped, + block: T.proc.params(socket: Socket).returns(T.untyped) + ).returns(T.untyped) + end + def open(url:, headers:, timeout:, **endpoint_options, &block) + end + end + end + end + end +end diff --git a/rbi/openai/realtime/unknown_server_event.rbi b/rbi/openai/realtime/unknown_server_event.rbi new file mode 100644 index 000000000..3147d4ae9 --- /dev/null +++ b/rbi/openai/realtime/unknown_server_event.rbi @@ -0,0 +1,25 @@ +# typed: strong + +module OpenAI + module Models + module Realtime + class UnknownServerEvent + sig { returns(Symbol) } + attr_reader :type + + sig { returns(T::Hash[Symbol, T.untyped]) } + attr_reader :data + + sig do + params(data: T::Hash[Symbol, T.untyped]).returns(T.attached_class) + end + def self.new(data:) + end + + sig { returns(T::Hash[Symbol, T.untyped]) } + def to_h + end + end + end + end +end diff --git a/rbi/openai/resources/realtime.rbi b/rbi/openai/resources/realtime.rbi index 93c144442..2c66113f6 100644 --- a/rbi/openai/resources/realtime.rbi +++ b/rbi/openai/resources/realtime.rbi @@ -9,6 +9,53 @@ module OpenAI sig { returns(OpenAI::Resources::Realtime::Calls) } attr_reader :calls + sig { returns(OpenAI::Resources::Realtime::Translations) } + attr_reader :translations + + sig do + params( + model: T.nilable(String), + call_id: T.nilable(String), + intent: T.nilable(Symbol), + request_options: T.nilable(OpenAI::RequestOptions::OrHash), + transport: T.untyped, + transport_options: T::Hash[Symbol, T.untyped], + block: + T + .proc + .params( + connection: + T.any( + OpenAI::Realtime::Connection, + OpenAI::Realtime::SidebandConnection, + OpenAI::Realtime::TranscriptionConnection + ) + ) + .returns(T.untyped) + ).returns(T.untyped) + end + def connect( + model: nil, + call_id: nil, + intent: nil, + request_options: nil, + transport: nil, + transport_options: {}, + &block + ) + end + + # @api private + sig do + params( + model: T.nilable(String), + call_id: T.nilable(String), + intent: T.nilable(Symbol) + ).returns(T::Array[T.untyped]) + end + def connection_target(model:, call_id:, intent:) + end + # @api private sig { params(client: OpenAI::Client).returns(T.attached_class) } def self.new(client:) diff --git a/rbi/openai/resources/realtime/calls.rbi b/rbi/openai/resources/realtime/calls.rbi index 8133bcf03..0c2539d9e 100644 --- a/rbi/openai/resources/realtime/calls.rbi +++ b/rbi/openai/resources/realtime/calls.rbi @@ -4,6 +4,17 @@ module OpenAI module Resources class Realtime class Calls + sig do + params( + sdp: String, + session: + T.nilable(OpenAI::Realtime::RealtimeSessionCreateRequest::OrHash), + request_options: OpenAI::RequestOptions::OrHash + ).returns(OpenAI::Realtime::CallCreateResponse) + end + def create(sdp:, session: nil, request_options: {}) + end + # Accept an incoming SIP call and configure the realtime session that will handle # it. sig do diff --git a/rbi/openai/resources/realtime/translations.rbi b/rbi/openai/resources/realtime/translations.rbi new file mode 100644 index 000000000..016650be4 --- /dev/null +++ b/rbi/openai/resources/realtime/translations.rbi @@ -0,0 +1,44 @@ +# typed: strong + +module OpenAI + module Resources + class Realtime + class Translations + sig do + returns(OpenAI::Resources::Realtime::Translations::ClientSecrets) + end + attr_reader :client_secrets + + sig { returns(OpenAI::Resources::Realtime::Translations::Calls) } + attr_reader :calls + + sig do + params( + model: String, + request_options: T.nilable(OpenAI::RequestOptions::OrHash), + transport: T.untyped, + transport_options: T::Hash[Symbol, T.untyped], + block: + T + .proc + .params(connection: OpenAI::Realtime::TranslationConnection) + .returns(T.untyped) + ).returns(T.untyped) + end + def connect( + model:, + request_options: nil, + transport: nil, + transport_options: {}, + &block + ) + end + + # @api private + sig { params(client: OpenAI::Client).returns(T.attached_class) } + def self.new(client:) + end + end + end + end +end diff --git a/rbi/openai/resources/realtime/translations/calls.rbi b/rbi/openai/resources/realtime/translations/calls.rbi new file mode 100644 index 000000000..c0fb9d2c3 --- /dev/null +++ b/rbi/openai/resources/realtime/translations/calls.rbi @@ -0,0 +1,25 @@ +# typed: strong + +module OpenAI + module Resources + class Realtime + class Translations + class Calls + sig do + params( + sdp: String, + request_options: T.nilable(OpenAI::RequestOptions::OrHash) + ).returns(OpenAI::Realtime::CallCreateResponse) + end + def create(sdp:, request_options: nil) + end + + # @api private + sig { params(client: OpenAI::Client).returns(T.attached_class) } + def self.new(client:) + end + end + end + end + end +end diff --git a/rbi/openai/resources/realtime/translations/client_secrets.rbi b/rbi/openai/resources/realtime/translations/client_secrets.rbi new file mode 100644 index 000000000..3b3d0d2e3 --- /dev/null +++ b/rbi/openai/resources/realtime/translations/client_secrets.rbi @@ -0,0 +1,32 @@ +# typed: strong + +module OpenAI + module Resources + class Realtime + class Translations + class ClientSecrets + sig do + params( + session: + OpenAI::Realtime::RealtimeTranslationSessionCreateRequest::OrHash, + expires_after: + T.nilable( + OpenAI::Realtime::RealtimeTranslationClientSecretCreateRequest::ExpiresAfter::OrHash + ), + request_options: T.nilable(OpenAI::RequestOptions::OrHash) + ).returns( + OpenAI::Realtime::RealtimeTranslationClientSecretCreateResponse + ) + end + def create(session:, expires_after: nil, request_options: nil) + end + + # @api private + sig { params(client: OpenAI::Client).returns(T.attached_class) } + def self.new(client:) + end + end + end + end + end +end diff --git a/realtime.md b/realtime.md new file mode 100644 index 000000000..1d92e130e --- /dev/null +++ b/realtime.md @@ -0,0 +1,405 @@ +# Realtime API + +The Ruby SDK supports the server-side parts of the Realtime API: + +- typed WebSocket sessions for text, audio, transcription, function calls, and remote MCP; +- WebRTC SDP negotiation from a trusted Ruby backend; +- sideband WebSocket control for active WebRTC and SIP calls; +- SIP accept, reject, refer, and hangup controls; and +- dedicated WebSocket, client-secret, and WebRTC translation endpoints. + +Ruby does not provide a broadly deployed, standard WebRTC media stack. The SDK +therefore handles the backend SDP exchange and call controls while browsers or a +specialized media service own `RTCPeerConnection`, microphone capture, codecs, +jitter buffering, echo cancellation, and playback. + +## Installation + +HTTP and WebRTC session-negotiation methods use the base SDK. WebSockets use an +optional adapter so applications that only use HTTP do not acquire an event-loop +dependency: + +```ruby +gem "openai" +gem "async-websocket" +``` + +The adapter is fiber-scheduler-aware and works both inside and outside an +existing Async reactor. You can instead inject an object implementing the small +transport contract described below. + +## Start a WebSocket session + +`connect` is block-scoped. This is the safest lifecycle in Rails jobs, +Rack servers, CLI programs, and long-running workers because normal returns and +exceptions both close the socket. + +```ruby +client = OpenAI::Client.new + +client.realtime.connect(model: "gpt-realtime-2.1") do |connection| + connection.session.update( + type: :realtime, + output_modalities: [:text], + instructions: "Be concise." + ) + + connection.conversation.items.create( + type: :message, + role: :user, + content: [{type: :input_text, text: "Hello"}] + ) + connection.response.create + + connection.each do |event| + case event + when OpenAI::Realtime::ResponseTextDeltaEvent + print(event.delta) + when OpenAI::Realtime::ResponseDoneEvent + raise "Response ended with #{event.response.status}" unless event.response.status == :completed + break + when OpenAI::Realtime::RealtimeErrorEvent + raise event.error.message + end + end +end +``` + +To exercise that flow against the live service with visible lifecycle output, +set `OPENAI_API_KEY` and run: + +```console +$ bundle exec ruby examples/realtime/websocket_text.rb +``` + +Set `OPENAI_REALTIME_MODEL`, `OPENAI_REALTIME_PROMPT`, or +`OPENAI_REALTIME_TIMEOUT` to override the example defaults. + +`connect` requires a block. The yielded connection is valid only for the lifetime +of that block and is always closed when the block exits, including exceptional exits. + +The connection exposes `receive`, `receive_raw`, `parse_event`, `send_event`, +`send_raw`, `each`, `close`, and `closed?`. The resource +helpers construct and validate the common client events: + +- `session.update`; +- `response.create` and `response.cancel`; +- `input_audio_buffer.append`, `append_bytes`, `commit`, and `clear`; +- sideband-only `output_audio_buffer.clear`; and +- conversation item create, retrieve, truncate, delete, function-call output, + and MCP approval response. + +`append` accepts API-ready Base64. `append_bytes` accepts binary audio and uses +strict Base64 encoding. Sending is synchronous, and iteration reads one message +at a time, so the caller naturally applies backpressure instead of filling an +unbounded SDK queue. A connection supports one reader fiber and one writer +fiber at the same time, which is useful for continuous audio. Do not run +multiple readers or multiple writers against the same connection. + +The yielded class reflects the protocol's capabilities: + +- `connect(model:)` yields `OpenAI::Realtime::Connection` for ordinary sessions; +- `connect(intent: :transcription)` yields + `OpenAI::Realtime::TranscriptionConnection`, exposing only session and input + audio buffer operations; +- `connect(call_id:)` yields `OpenAI::Realtime::SidebandConnection`, adding + `output_audio_buffer`; and +- `translations.connect` yields `OpenAI::Realtime::TranslationConnection`, + which intentionally has no conversation or response lifecycle. + +The resource helpers accept the resource fields directly and add the wire +envelope internally: use `session.update(type: ...)`, +`response.create(instructions: ...)`, and `conversation.items.create(type: ...)`. +Use `send_event` only when an application deliberately needs an exact protocol +event hash. + +## Stream transcription + +Realtime transcription is a distinct session mode. Configure a transcription +model, append raw mono 24 kHz PCM16 input, and commit the buffer: + +```ruby +client.realtime.connect(intent: :transcription) do |connection| + connection.session.update( + audio: { + input: { + format: {type: :"audio/pcm", rate: 24_000}, + transcription: {model: "gpt-live-transcribe"}, + turn_detection: nil + } + } + ) + + # Wait for SessionUpdatedEvent before streaming input. + connection.input_audio_buffer.append_bytes(pcm_chunk) + connection.input_audio_buffer.commit + + connection.each do |event| + case event + when OpenAI::Realtime::ConversationItemInputAudioTranscriptionDeltaEvent + print(event.delta) + when OpenAI::Realtime::ConversationItemInputAudioTranscriptionCompletedEvent + puts event.transcript + end + end +end +``` + +Completion events for different turns are not guaranteed to arrive in input +order, so correlate deltas and completions with `item_id`. The dedicated +connection uses the service's `intent=transcription` handshake and adds the +session's `type: :transcription` discriminator internally; the transcription +model belongs under `audio.input.transcription.model`. Run the complete live sample with +`REALTIME_INPUT_PCM=input.pcm bundle exec ruby +examples/realtime/websocket_transcription.rb`. + +## Create a WebRTC call + +WebRTC media is intentionally not a Ruby SDK responsibility. In a browser or +mobile voice application, the client owns `RTCPeerConnection`, microphone +permissions, audio playback, jitter buffering, and acoustic echo cancellation. +Ruby owns the trusted-server control plane: API credentials, session +configuration, SDP negotiation, sideband tools and policy, and call lifecycle. +The SDK therefore supports WebRTC session negotiation and server controls; it +does not attempt to expose a Ruby WebRTC peer API. + +Use `calls.create` in the trusted application server that receives a browser's +SDP offer: + +```ruby +call = client.realtime.calls.create( + sdp: browser_offer, + session: { + type: :realtime, + model: "gpt-realtime-2.1", + audio: {output: {voice: :marin}} + }, + request_options: { + extra_headers: {"OpenAI-Safety-Identifier" => hashed_user_id} + } +) + +render plain: call.sdp, content_type: "application/sdp" +``` + +The result preserves `sdp`, normalized response `headers`, and `call_id` parsed +from the `Location` header. Keep the call ID if the application will open a +server-side sideband connection: + +```ruby +client.realtime.connect(call_id: call.call_id) do |connection| + connection.session.update( + type: :realtime, + instructions: "Apply private server policy." + ) + connection.each { |event| audit(event) } +end +``` + +The SDK sends raw `application/sdp` when only an offer is provided; use that +form with a client configured with a previously minted ephemeral session token. +With a standard server API key, provide `session:` and the SDK sends multipart +form data with correctly typed `sdp` and `session` parts. It never exposes a +standard API key to browser code. + +## SIP calls + +Verify incoming webhooks before using their call ID, accept or reject the call, +then use the same sideband connection API: + +```ruby +event = client.webhooks.unwrap(raw_body, request_headers) +return unless event.is_a?(OpenAI::Webhooks::RealtimeCallIncomingWebhookEvent) + +call_id = event.data.call_id +client.realtime.calls.accept( + call_id, + type: :realtime, + model: "gpt-realtime-2.1", + instructions: "You are answering a phone call." +) + +client.realtime.connect(call_id: call_id) do |connection| + connection.each { |realtime_event| handle(realtime_event) } +end +``` + +`calls.reject`, `calls.refer`, and `calls.hangup` provide the remaining SIP and +call-lifecycle controls. + +## Translation + +Translation has dedicated endpoints and distinct typed event unions: + +```ruby +require "async" + +client.realtime.translations.connect(model: "gpt-realtime-translate") do |connection| + connection.session.update(audio: {output: {language: "es"}}) + + reader = Async do + connection.each do |event| + case event + when OpenAI::Realtime::RealtimeTranslationOutputTranscriptDeltaEvent + print(event.delta) + when OpenAI::Realtime::RealtimeTranslationOutputAudioDeltaEvent + play(Base64.strict_decode64(event.delta)) + when OpenAI::Realtime::RealtimeTranslationSessionClosedEvent + break + end + end + end + + while (chunk = pcm16_source.read(9_600)) + connection.input_audio_buffer.append_bytes(chunk) + end + connection.session.close + reader.wait +end +``` + +Call `session.close` after the last audio chunk and continue reading until the +server sends `session.closed`; closing the socket immediately can discard output +that is still draining. For browser WebRTC translation, mint an ephemeral secret +with `translations.client_secrets.create`, then post the browser offer with +`translations.calls.create`. + +## Function calls and MCP approvals + +Function call output and MCP approval responses are ordinary conversation items. +The helpers supply the required item shape and generate an MCP response ID when +one is not supplied: + +```ruby +connection.conversation.items.create_function_call_output( + call_id: function_call.call_id, + output: JSON.generate(result) +) + +connection.conversation.items.respond_to_mcp_approval( + approval_request_id: approval_item.id, + approve: policy.allows?(approval_item), + reason: "Evaluated by application policy" +) +``` + +Remote MCP servers are configured through the typed `tools` field in +`session.update` or `response.create`; all MCP progress, approval, and completion +events decode through `RealtimeServerEvent`. Before a dependent prompt, wait for +both `McpListToolsCompleted` and the `ConversationItemDone` event whose +`RealtimeMcpListTools` item contains the imported tool names. If you then update +`tool_choice`, wait for `SessionUpdatedEvent` before creating the response. + +The first completed `ResponseDoneEvent` can finish the MCP argument-generation +phase while approval and tool execution are still pending. Keep reading, answer +the `RealtimeMcpApprovalRequest`, wait for `ResponseMcpCallCompleted`, and create +a follow-up response (usually with `tool_choice: :none`) to turn the tool output +into a final assistant answer. The approval helper generates item IDs within the +service's 32-character limit. + +## Lifecycle and failures + +The SDK does not automatically reconnect or replay events. Realtime sessions are +stateful, and replaying audio, item creation, tool output, or approval decisions +can duplicate side effects. Treat an unexpected close as an application-level +decision: terminate, or reconnect and deliberately rebuild the state that is +safe for that use case. + +- Invalid JSON or a payload that cannot match the selected event union raises + `OpenAI::Errors::RealtimeProtocolError`, with `data` and `cause`. +- Handshake and socket I/O failures from the default adapter raise + `OpenAI::Errors::RealtimeConnectionError`, with `url` and `cause`. +- API `error` events remain typed `RealtimeErrorEvent` values in the normal event + stream so applications can follow the API's recoverability guidance. +- Valid events introduced after the installed SDK version remain observable as + immutable `UnknownServerEvent` values instead of terminating the session. +- Exceptions raised by the application block propagate unchanged; cleanup does + not replace them with a close error. + +Use `request_options` for handshake headers, query parameters, and timeout: + +```ruby +client.realtime.connect( + model: "gpt-realtime-2.1", + request_options: { + timeout: 30, + extra_headers: {"OpenAI-Safety-Identifier" => hashed_user_id}, + extra_query: {trace: "enabled"} + } +) { |connection| run_session(connection) } +``` + +`websocket_base_url:` configures a separate gateway for WebSockets. Azure +provider clients derive the WebSocket URL from the configured Azure v1 endpoint +and resolve provider-owned API-key or bearer authentication immediately before +the handshake. Providers without a Realtime WebSocket surface fail before a +credential is sent. + +## Custom transport + +Pass `transport:` to `connect` to replace the adapter. The object implements: + +```ruby +transport.open(url:, headers:, timeout:, **transport_options) do |socket| + # socket.read # String-like message or nil + # socket.write(json_string) # synchronous write/backpressure + # socket.close(code:, reason:) + # socket.closed? +end +``` + +The SDK owns URL construction, authentication, typed JSON encoding/decoding, and +block cleanup. The transport owns the WebSocket handshake, frame I/O, TLS, +proxies, and transport-specific failures. Internal calls use direct methods; the +contract does not require subclassing or reflection. + +## Example and developer-site coverage + +Repository examples live under `examples/realtime/`; their +[runbook](examples/realtime/README.md) includes prerequisites, commands, bounded +smoke-test controls, and observable pass criteria. They cover a hands-free +full-duplex voice conversation, text, raw PCM audio, streaming transcription, +WebRTC SDP exchange, sideband, SIP, MCP approvals, and translation. The +developer website should expose the same progression so users encounter the +simplest successful path before protocol details: + +| Guide surface | Ruby sample required | +| --- | --- | +| Realtime overview | Minimal text WebSocket with typed deltas and block cleanup | +| WebSocket | Text, PCM16 input/output, manual `receive`, Enumerable iteration, error handling | +| Transcription | `intent: :transcription`, PCM16 streaming, delta/completed events, and `item_id` correlation | +| WebRTC | Complete browser peer plus Ruby endpoint; Rails and Sinatra variants using `calls.create` | +| Server controls | Preserve `call.call_id`, open sideband, update session, hang up | +| SIP | Verify webhook, accept/reject, sideband, refer, hang up, idempotent webhook handling | +| Conversations | Text item, audio commit, cancel/interruption, truncate played audio | +| Function calling | Detect function call, execute locally, send typed output item | +| Remote MCP | Configure server, inspect approval request, approve/reject, observe completion | +| Translation | Client secret, browser WebRTC call, server WebSocket PCM loop, graceful close | +| Authentication | Standard key on server, ephemeral browser secret, safety identifier | +| Azure | Deployment/model targeting and provider-owned handshake authentication | +| Operations | Timeout, backpressure, logging event IDs, abnormal close, deliberate reconnect policy | + +The WebRTC browser half remains JavaScript because Ruby runs on the trusted +server; Ruby snippets should own secrets, session configuration, SDP proxying, +webhook verification, tools, and business logic. + +The local voice example is also exercised against the installed FFmpeg tools, +not only Ruby fakes. In particular, raw PCM playback declares mono input with +ffplay's `-ch_layout mono`; `-ac 1` is an ffmpeg transcoding option and current +ffplay releases reject it. Playback also disables ffplay's terminal statistics: +without `-nostats`, ffplay emits an ANSI clear-line sequence that can erase the +first transcript delta from the terminal. Playback-command compatibility and a +headless PCM process smoke test guard both boundaries. + +WebSocket barge-in requires client playback state in addition to server VAD. +On `input_audio_buffer.speech_started`, the voice example stops ffplay, records +the elapsed audio offset, sends `conversation.item.truncate`, and ignores queued +audio and transcript deltas for the cancelled response. A cancelled +`response.done` is an expected interruption outcome rather than an exception. + +The FFmpeg WebSocket loop cannot provide acoustic echo cancellation and must +not be positioned as the default laptop conversation sample. With speakers, +model output can re-enter the microphone and repeatedly trigger VAD, producing +one-word responses. The primary interactive voice example uses browser WebRTC; +the Ruby WebSocket loop remains useful for headless server audio, telephony +bridges, deterministic PCM tests, and protocol debugging. diff --git a/sig/openai/client.rbs b/sig/openai/client.rbs index a65e15abc..87903612e 100644 --- a/sig/openai/client.rbs +++ b/sig/openai/client.rbs @@ -20,6 +20,8 @@ module OpenAI attr_reader webhook_secret: String? + attr_reader websocket_base_url: URI::Generic? + attr_reader completions: OpenAI::Resources::Completions attr_reader chat: OpenAI::Resources::Chat @@ -76,6 +78,12 @@ module OpenAI private def admin_api_key_auth: -> ::Hash[String, String] + def realtime_connection_request: ( + path: String, + query: ::Hash[String, String], + ?options: OpenAI::request_opts? + ) -> OpenAI::Internal::Transport::BaseClient::request_input + private def prepare_request: ( OpenAI::Internal::Transport::BaseClient::request_input request, redirect_count: Integer, @@ -92,6 +100,7 @@ module OpenAI ?provider: OpenAI::Provider?, ?base_url: String?, ?default_headers: ::Hash[String, String?]?, + ?websocket_base_url: String?, ?max_retries: Integer, ?timeout: Float?, ?initial_retry_delay: Float, diff --git a/sig/openai/errors.rbs b/sig/openai/errors.rbs index aa1428662..e38e6fe62 100644 --- a/sig/openai/errors.rbs +++ b/sig/openai/errors.rbs @@ -1,7 +1,7 @@ module OpenAI module Errors class Error < StandardError - attr_accessor cause: StandardError? + attr_accessor cause: Exception? end class ConversionError < OpenAI::Errors::Error @@ -16,6 +16,28 @@ module OpenAI ) -> void end + class RealtimeConnectionError < OpenAI::Errors::Error + attr_reader url: URI::Generic + def cause: -> Exception? + + def initialize: ( + url: URI::Generic, + ?message: String?, + ?cause: Exception? + ) -> void + end + + class RealtimeProtocolError < OpenAI::Errors::Error + attr_reader data: String + def cause: -> StandardError? + + def initialize: ( + data: String, + ?message: String?, + ?cause: StandardError? + ) -> void + end + class APIError < OpenAI::Errors::Error attr_accessor url: URI::Generic diff --git a/sig/openai/internal/transport/base_client.rbs b/sig/openai/internal/transport/base_client.rbs index 1b5517583..b717419ee 100644 --- a/sig/openai/internal/transport/base_client.rbs +++ b/sig/openai/internal/transport/base_client.rbs @@ -143,6 +143,10 @@ module OpenAI -> OpenAI::Internal::Logging::Context } -> OpenAI::HTTPClient::Response + def request_raw: ( + OpenAI::Internal::Transport::BaseClient::request_components req + ) -> OpenAI::HTTPClient::Response + def request: ( Symbol method, String | ::Array[String] path, diff --git a/sig/openai/models/realtime/call_create_params.rbs b/sig/openai/models/realtime/call_create_params.rbs new file mode 100644 index 000000000..0f9d91aaf --- /dev/null +++ b/sig/openai/models/realtime/call_create_params.rbs @@ -0,0 +1,32 @@ +module OpenAI + module Models + module Realtime + type call_create_params = + { + sdp: String, + session: OpenAI::Realtime::RealtimeSessionCreateRequest? + } + & OpenAI::Internal::Type::request_parameters + + class CallCreateParams < OpenAI::Internal::Type::BaseModel + extend OpenAI::Internal::Type::RequestParameters::Converter + include OpenAI::Internal::Type::RequestParameters + + attr_accessor sdp: String + attr_accessor session: OpenAI::Realtime::RealtimeSessionCreateRequest? + + def initialize: ( + sdp: String, + ?session: OpenAI::Realtime::RealtimeSessionCreateRequest, + ?request_options: OpenAI::request_opts + ) -> void + + def to_hash: -> { + sdp: String, + session: OpenAI::Realtime::RealtimeSessionCreateRequest?, + request_options: OpenAI::RequestOptions + } + end + end + end +end diff --git a/sig/openai/models/realtime/call_create_response.rbs b/sig/openai/models/realtime/call_create_response.rbs new file mode 100644 index 000000000..ea05d00be --- /dev/null +++ b/sig/openai/models/realtime/call_create_response.rbs @@ -0,0 +1,26 @@ +module OpenAI + module Models + module Realtime + type call_create_response = + { sdp: String, call_id: String?, headers: ::Hash[String, String] } + + class CallCreateResponse < OpenAI::Internal::Type::BaseModel + attr_accessor sdp: String + attr_accessor call_id: String? + attr_accessor headers: ::Hash[String, String] + + def initialize: ( + sdp: String, + headers: ::Hash[String, String], + ?call_id: String? + ) -> void + + def to_hash: -> { + sdp: String, + call_id: String?, + headers: ::Hash[String, String] + } + end + end + end +end diff --git a/sig/openai/realtime/base_connection.rbs b/sig/openai/realtime/base_connection.rbs new file mode 100644 index 000000000..2e6d1660b --- /dev/null +++ b/sig/openai/realtime/base_connection.rbs @@ -0,0 +1,20 @@ +module OpenAI + module Models + module Realtime + class BaseConnection + attr_reader url: URI::Generic + + def initialize: ( + socket: untyped, + url: URI::Generic, + server_event_type: untyped, + client_event_type: untyped + ) -> void + def receive_raw: -> String? + def send_raw: (String data) -> nil + def close: (?code: Integer, ?reason: String) -> nil + def closed?: -> bool + end + end + end +end diff --git a/sig/openai/realtime/connection.rbs b/sig/openai/realtime/connection.rbs new file mode 100644 index 000000000..52caffcfa --- /dev/null +++ b/sig/openai/realtime/connection.rbs @@ -0,0 +1,36 @@ +module OpenAI + module Models + module Realtime + type connection_server_event = + OpenAI::Models::Realtime::realtime_server_event + | OpenAI::Realtime::UnknownServerEvent + + type connection_client_event = + OpenAI::Models::Realtime::realtime_client_event + | ::Hash[Symbol | String, untyped] + + class Connection < OpenAI::Realtime::BaseConnection + include Enumerable[OpenAI::Models::Realtime::connection_server_event] + + attr_reader session: OpenAI::Realtime::ConnectionResources::Session + attr_reader response: OpenAI::Realtime::ConnectionResources::Response + attr_reader input_audio_buffer: OpenAI::Realtime::ConnectionResources::InputAudioBuffer + attr_reader conversation: OpenAI::Realtime::ConnectionResources::Conversation + + def initialize: (socket: untyped, url: URI::Generic) -> void + def each: + -> Enumerator[OpenAI::Models::Realtime::connection_server_event, self] + | { + (OpenAI::Models::Realtime::connection_server_event event) -> void + } -> self + def receive: -> OpenAI::Models::Realtime::connection_server_event? + def parse_event: ( + String data + ) -> OpenAI::Models::Realtime::connection_server_event + def send_event: ( + OpenAI::Models::Realtime::connection_client_event event + ) -> nil + end + end + end +end diff --git a/sig/openai/realtime/connection_manager.rbs b/sig/openai/realtime/connection_manager.rbs new file mode 100644 index 000000000..1813a520e --- /dev/null +++ b/sig/openai/realtime/connection_manager.rbs @@ -0,0 +1,21 @@ +module OpenAI + module Models + module Realtime + class ConnectionManager + def initialize: ( + client: OpenAI::Client, + path: String, + query: ::Hash[String, String], + connection_class: Class, + transport: untyped, + request_options: OpenAI::request_opts?, + transport_options: ::Hash[Symbol, untyped] + ) -> void + + def open: { + (OpenAI::Realtime::BaseConnection connection) -> top + } -> top + end + end + end +end diff --git a/sig/openai/realtime/connection_resources.rbs b/sig/openai/realtime/connection_resources.rbs new file mode 100644 index 000000000..ab570b8d3 --- /dev/null +++ b/sig/openai/realtime/connection_resources.rbs @@ -0,0 +1,136 @@ +module OpenAI + module Models + module Realtime + module ConnectionResources + class Base + def initialize: (OpenAI::Realtime::BaseConnection connection) -> void + end + + class Session < Base + def update: ( + type: Symbol, + ?audio: OpenAI::Models::Realtime::realtime_audio_config + | OpenAI::Models::Realtime::realtime_transcription_session_audio, + ?include: ::Array[Symbol], + ?instructions: String, + ?max_output_tokens: Integer | Symbol, + ?model: String | Symbol, + ?output_modalities: ::Array[Symbol], + ?parallel_tool_calls: bool, + ?prompt: OpenAI::Models::Responses::response_prompt?, + ?reasoning: OpenAI::Models::Realtime::realtime_reasoning, + ?tool_choice: OpenAI::Models::Realtime::realtime_tool_choice_config, + ?tools: OpenAI::Models::Realtime::realtime_tools_config, + ?tracing: OpenAI::Models::Realtime::realtime_tracing_config?, + ?truncation: OpenAI::Models::Realtime::realtime_truncation, + ?event_id: String? + ) -> nil + end + + class TranscriptionSession < Base + def update: ( + ?audio: OpenAI::Models::Realtime::realtime_transcription_session_audio, + ?include: ::Array[OpenAI::Models::Realtime::RealtimeTranscriptionSessionCreateRequest::include_], + ?event_id: String? + ) -> nil + end + + class TranslationSession < Session + def update: ( + ?audio: OpenAI::Realtime::RealtimeTranslationSessionUpdateRequest::Audio + | ::Hash[Symbol | String, untyped], + ?event_id: String? + ) -> nil + def close: (?event_id: String?) -> nil + end + + class Response < Base + def create: ( + ?audio: OpenAI::Models::Realtime::realtime_response_create_audio_output, + ?conversation: String | Symbol, + ?input: ::Array[OpenAI::Models::Realtime::conversation_item], + ?instructions: String, + ?max_output_tokens: Integer | Symbol, + ?metadata: ::Hash[Symbol, String]?, + ?output_modalities: ::Array[Symbol], + ?parallel_tool_calls: bool, + ?prompt: OpenAI::Models::Responses::response_prompt?, + ?reasoning: OpenAI::Models::Realtime::realtime_reasoning, + ?tool_choice: OpenAI::Models::Realtime::RealtimeResponseCreateParams::tool_choice, + ?tools: ::Array[ + OpenAI::Models::Realtime::realtime_function_tool + | OpenAI::Models::Realtime::realtime_response_create_mcp_tool + ], + ?event_id: String? + ) -> nil + def cancel: (?response_id: String?, ?event_id: String?) -> nil + end + + class InputAudioBuffer < Base + def append: (audio: String, ?event_id: String?) -> nil + def append_bytes: (String bytes, ?event_id: String?) -> nil + def commit: (?event_id: String?) -> nil + def clear: (?event_id: String?) -> nil + end + + class TranslationInputAudioBuffer < Base + def append: (audio: String, ?event_id: String?) -> nil + def append_bytes: (String bytes, ?event_id: String?) -> nil + end + + class Conversation < Base + attr_reader items: OpenAI::Realtime::ConnectionResources::ConversationItems + end + + class ConversationItems < Base + def create: ( + type: Symbol, + ?arguments: String, + ?approval_request_id: String?, + ?approve: bool, + ?call_id: String, + ?content: ::Array[::Hash[Symbol | String, untyped]], + ?error: untyped, + ?id: String, + ?name: String, + ?object: Symbol, + ?output: String?, + ?reason: String?, + ?role: Symbol, + ?server_label: String, + ?status: Symbol, + ?tools: ::Array[OpenAI::Realtime::RealtimeMcpListTools::Tool | ::Hash[Symbol | String, untyped]], + ?event_id: String?, + ?previous_item_id: String? + ) -> nil + def delete: (item_id: String, ?event_id: String?) -> nil + def retrieve: (item_id: String, ?event_id: String?) -> nil + def truncate: ( + item_id: String, + content_index: Integer, + audio_end_ms: Integer, + ?event_id: String? + ) -> nil + def create_function_call_output: ( + call_id: String, + output: String, + ?id: String?, + ?status: (String | Symbol)?, + ?event_id: String? + ) -> nil + def respond_to_mcp_approval: ( + approval_request_id: String, + approve: bool, + ?reason: String?, + ?id: String?, + ?event_id: String? + ) -> nil + end + + class OutputAudioBuffer < Base + def clear: (?event_id: String?) -> nil + end + end + end + end +end diff --git a/sig/openai/realtime/sideband_connection.rbs b/sig/openai/realtime/sideband_connection.rbs new file mode 100644 index 000000000..cfdf03744 --- /dev/null +++ b/sig/openai/realtime/sideband_connection.rbs @@ -0,0 +1,11 @@ +module OpenAI + module Models + module Realtime + class SidebandConnection < OpenAI::Realtime::Connection + attr_reader output_audio_buffer: OpenAI::Realtime::ConnectionResources::OutputAudioBuffer + + def initialize: (socket: untyped, url: URI::Generic) -> void + end + end + end +end diff --git a/sig/openai/realtime/transcription_connection.rbs b/sig/openai/realtime/transcription_connection.rbs new file mode 100644 index 000000000..ea8032c2f --- /dev/null +++ b/sig/openai/realtime/transcription_connection.rbs @@ -0,0 +1,34 @@ +module OpenAI + module Models + module Realtime + type transcription_connection_server_event = + OpenAI::Models::Realtime::realtime_server_event + | OpenAI::Realtime::UnknownServerEvent + + type transcription_connection_client_event = + OpenAI::Models::Realtime::realtime_client_event + | ::Hash[Symbol | String, untyped] + + class TranscriptionConnection < OpenAI::Realtime::BaseConnection + include Enumerable[OpenAI::Models::Realtime::transcription_connection_server_event] + + attr_reader session: OpenAI::Realtime::ConnectionResources::TranscriptionSession + attr_reader input_audio_buffer: OpenAI::Realtime::ConnectionResources::InputAudioBuffer + + def initialize: (socket: untyped, url: URI::Generic) -> void + def each: + -> Enumerator[OpenAI::Models::Realtime::transcription_connection_server_event, self] + | { + (OpenAI::Models::Realtime::transcription_connection_server_event event) -> void + } -> self + def receive: -> OpenAI::Models::Realtime::transcription_connection_server_event? + def parse_event: ( + String data + ) -> OpenAI::Models::Realtime::transcription_connection_server_event + def send_event: ( + OpenAI::Models::Realtime::transcription_connection_client_event event + ) -> nil + end + end + end +end diff --git a/sig/openai/realtime/translation_connection.rbs b/sig/openai/realtime/translation_connection.rbs new file mode 100644 index 000000000..d8042c341 --- /dev/null +++ b/sig/openai/realtime/translation_connection.rbs @@ -0,0 +1,36 @@ +module OpenAI + module Models + module Realtime + type translation_connection_server_event = + OpenAI::Models::Realtime::realtime_translation_server_event + | OpenAI::Realtime::UnknownServerEvent + + type translation_connection_client_event = + OpenAI::Models::Realtime::realtime_translation_client_event + | ::Hash[Symbol | String, untyped] + + class TranslationConnection < OpenAI::Realtime::BaseConnection + include Enumerable[OpenAI::Models::Realtime::translation_connection_server_event] + + attr_reader session: OpenAI::Realtime::ConnectionResources::TranslationSession + attr_reader input_audio_buffer: OpenAI::Realtime::ConnectionResources::TranslationInputAudioBuffer + + def initialize: (socket: untyped, url: URI::Generic) -> void + def each: + -> Enumerator[OpenAI::Models::Realtime::translation_connection_server_event, self] + | { + ( + OpenAI::Models::Realtime::translation_connection_server_event event + ) -> void + } -> self + def receive: -> OpenAI::Models::Realtime::translation_connection_server_event? + def parse_event: ( + String data + ) -> OpenAI::Models::Realtime::translation_connection_server_event + def send_event: ( + OpenAI::Models::Realtime::translation_connection_client_event event + ) -> nil + end + end + end +end diff --git a/sig/openai/realtime/transports/async_websocket.rbs b/sig/openai/realtime/transports/async_websocket.rbs new file mode 100644 index 000000000..b03980ac2 --- /dev/null +++ b/sig/openai/realtime/transports/async_websocket.rbs @@ -0,0 +1,26 @@ +module OpenAI + module Models + module Realtime + module Transports + class AsyncWebSocket + class Socket + def initialize: (untyped connection, url: URI::Generic) -> void + def read: -> untyped + def write: (String message) -> void + def close: (?code: Integer, ?reason: String) -> void + def closed?: -> bool + end + + def open: ( + url: URI::Generic, + headers: ::Hash[String, String], + timeout: Float, + **untyped endpoint_options + ) { + (Socket socket) -> top + } -> top + end + end + end + end +end diff --git a/sig/openai/realtime/unknown_server_event.rbs b/sig/openai/realtime/unknown_server_event.rbs new file mode 100644 index 000000000..8fcb1d07d --- /dev/null +++ b/sig/openai/realtime/unknown_server_event.rbs @@ -0,0 +1,13 @@ +module OpenAI + module Models + module Realtime + class UnknownServerEvent + attr_reader type: Symbol + attr_reader data: ::Hash[Symbol, untyped] + + def initialize: (data: ::Hash[Symbol, untyped]) -> void + def to_h: -> ::Hash[Symbol, untyped] + end + end + end +end diff --git a/sig/openai/resources/realtime.rbs b/sig/openai/resources/realtime.rbs index 999debec0..b7bf1007f 100644 --- a/sig/openai/resources/realtime.rbs +++ b/sig/openai/resources/realtime.rbs @@ -5,6 +5,46 @@ module OpenAI attr_reader calls: OpenAI::Resources::Realtime::Calls + attr_reader translations: OpenAI::Resources::Realtime::Translations + + def connect: + ( + model: String, + ?call_id: nil, + ?intent: nil, + ?request_options: OpenAI::request_opts?, + ?transport: untyped?, + ?transport_options: ::Hash[Symbol, untyped] + ) { + (OpenAI::Realtime::Connection connection) -> top + } -> top + | ( + call_id: String, + ?model: nil, + ?intent: nil, + ?request_options: OpenAI::request_opts?, + ?transport: untyped?, + ?transport_options: ::Hash[Symbol, untyped] + ) { + (OpenAI::Realtime::SidebandConnection connection) -> top + } -> top + | ( + intent: :transcription, + ?model: nil, + ?call_id: nil, + ?request_options: OpenAI::request_opts?, + ?transport: untyped?, + ?transport_options: ::Hash[Symbol, untyped] + ) { + (OpenAI::Realtime::TranscriptionConnection connection) -> top + } -> top + + def connection_target: ( + model: String?, + call_id: String?, + intent: (:transcription)? + ) -> [::Hash[String, String], untyped] + def initialize: (client: OpenAI::Client) -> void end end diff --git a/sig/openai/resources/realtime/calls.rbs b/sig/openai/resources/realtime/calls.rbs index 7f89f2417..6cad50fd9 100644 --- a/sig/openai/resources/realtime/calls.rbs +++ b/sig/openai/resources/realtime/calls.rbs @@ -2,6 +2,13 @@ module OpenAI module Resources class Realtime class Calls + def create: ( + sdp: String, + ?session: OpenAI::Realtime::RealtimeSessionCreateRequest + | ::Hash[Symbol | String, untyped], + ?request_options: OpenAI::request_opts + ) -> OpenAI::Realtime::CallCreateResponse + def accept: ( String call_id, ?audio: OpenAI::Realtime::RealtimeAudioConfig, diff --git a/sig/openai/resources/realtime/translations.rbs b/sig/openai/resources/realtime/translations.rbs new file mode 100644 index 000000000..5dd578f21 --- /dev/null +++ b/sig/openai/resources/realtime/translations.rbs @@ -0,0 +1,21 @@ +module OpenAI + module Resources + class Realtime + class Translations + attr_reader client_secrets: OpenAI::Resources::Realtime::Translations::ClientSecrets + attr_reader calls: OpenAI::Resources::Realtime::Translations::Calls + + def connect: ( + model: String, + ?request_options: OpenAI::request_opts?, + ?transport: untyped?, + ?transport_options: ::Hash[Symbol, untyped] + ) { + (OpenAI::Realtime::TranslationConnection connection) -> top + } -> top + + def initialize: (client: OpenAI::Client) -> void + end + end + end +end diff --git a/sig/openai/resources/realtime/translations/calls.rbs b/sig/openai/resources/realtime/translations/calls.rbs new file mode 100644 index 000000000..db1ebfdde --- /dev/null +++ b/sig/openai/resources/realtime/translations/calls.rbs @@ -0,0 +1,16 @@ +module OpenAI + module Resources + class Realtime + class Translations + class Calls + def create: ( + sdp: String, + ?request_options: OpenAI::request_opts? + ) -> OpenAI::Realtime::CallCreateResponse + + def initialize: (client: OpenAI::Client) -> void + end + end + end + end +end diff --git a/sig/openai/resources/realtime/translations/client_secrets.rbs b/sig/openai/resources/realtime/translations/client_secrets.rbs new file mode 100644 index 000000000..e609efe1a --- /dev/null +++ b/sig/openai/resources/realtime/translations/client_secrets.rbs @@ -0,0 +1,19 @@ +module OpenAI + module Resources + class Realtime + class Translations + class ClientSecrets + def create: ( + session: OpenAI::Realtime::RealtimeTranslationSessionCreateRequest + | ::Hash[Symbol | String, untyped], + ?expires_after: (OpenAI::Realtime::RealtimeTranslationClientSecretCreateRequest::ExpiresAfter + | ::Hash[(Symbol | String), untyped])?, + ?request_options: OpenAI::request_opts? + ) -> OpenAI::Realtime::RealtimeTranslationClientSecretCreateResponse + + def initialize: (client: OpenAI::Client) -> void + end + end + end + end +end diff --git a/test/openai/realtime/async_websocket_transport_test.rb b/test/openai/realtime/async_websocket_transport_test.rb new file mode 100644 index 000000000..8264f9bc6 --- /dev/null +++ b/test/openai/realtime/async_websocket_transport_test.rb @@ -0,0 +1,554 @@ +# frozen_string_literal: true + +require "async/http/server" +require "async/queue" +require "async/websocket/adapters/http" +require "async/websocket/server" +require "socket" + +require_relative "../test_helper" + +class OpenAI::Test::AsyncWebSocketTransportTest < Minitest::Test + extend Minitest::Serial + + def test_default_transport_exchanges_events_with_a_local_websocket_server + port = available_port + endpoint = Async::HTTP::Endpoint.parse("http://127.0.0.1:#{port}") + fallback = ->(_request) { Protocol::HTTP::Response[404, {}, []] } + websocket = Async::WebSocket::Server.new(fallback) do |connection| + message = connection.read + next if message.nil? + + event = JSON.parse(message.to_str) + raise "unexpected client event" unless event.fetch("type") == "session.update" + + connection.write( + Protocol::WebSocket::TextMessage.generate( + type: "response.output_text.delta", + event_id: "event_1", + response_id: "response_1", + item_id: "item_1", + output_index: 0, + content_index: 0, + delta: "hello over websocket" + ) + ) + connection.flush + end + server = Async::HTTP::Server.new(websocket, endpoint) + + Sync do |task| + server_task = task.async { server.run.wait } + client = OpenAI::Client.new( + api_key: "test-key", + base_url: "http://127.0.0.1:#{port}/v1", + timeout: 5 + ) + + event = client.realtime.connect(model: "gpt-realtime") do |connection| + connection.session.update(type: :realtime) + connection.receive + end + + assert_instance_of(OpenAI::Realtime::ResponseTextDeltaEvent, event) + assert_equal("hello over websocket", event.delta) + assert_application_errors_propagate(client) + ensure + server_task&.stop + end + end + + def test_documented_text_lifecycle_over_a_local_websocket + with_websocket_server(method(:serve_text_lifecycle)) do |_task, client| + client.realtime.connect(model: "gpt-realtime-2.1") do |connection| + assert_instance_of(OpenAI::Realtime::SessionCreatedEvent, connection.receive) + connection.session.update(type: :realtime, output_modalities: [:text]) + assert_instance_of(OpenAI::Realtime::SessionUpdatedEvent, connection.receive) + + connection.conversation.items.create( + type: :message, + role: :user, + content: [{type: :input_text, text: "Say hello."}] + ) + connection.response.create + + delta = connection.receive + done = connection.receive + assert_instance_of(OpenAI::Realtime::ResponseTextDeltaEvent, delta) + assert_equal("hello from the local service", delta.delta) + assert_instance_of(OpenAI::Realtime::ResponseDoneEvent, done) + assert_equal(:completed, done.response.status) + assert_nil(connection.receive) + end + end + end + + def test_function_call_round_trip_over_a_local_websocket + handler = lambda do |connection| + unless read_event(connection).fetch("type") == "conversation.item.create" + raise "expected conversation item" + end + raise "expected response.create" unless read_event(connection).fetch("type") == "response.create" + + write_event( + connection, + type: "response.function_call_arguments.done", + event_id: "event_call", + response_id: "response_1", + item_id: "item_call", + output_index: 0, + call_id: "call_1", + name: "get_weather", + arguments: JSON.generate(location: "Paris") + ) + + output = read_event(connection) + raise "expected function output" unless output.dig("item", "type") == "function_call_output" + raise "unexpected call id" unless output.dig("item", "call_id") == "call_1" + raise "expected follow-up response" unless read_event(connection).fetch("type") == "response.create" + + write_event( + connection, + type: "response.done", + event_id: "event_done", + response: {id: "response_2", status: "completed", output: []} + ) + end + + with_websocket_server(handler) do |_task, client| + client.realtime.connect(model: "gpt-realtime-2.1") do |connection| + connection.conversation.items.create( + type: :message, + role: :user, + content: [{type: :input_text, text: "What is the weather?"}] + ) + connection.response.create + + call = connection.receive + assert_instance_of(OpenAI::Realtime::ResponseFunctionCallArgumentsDoneEvent, call) + connection.conversation.items.create_function_call_output( + call_id: call.call_id, + output: JSON.generate(temperature_c: 18) + ) + connection.response.create + + done = connection.receive + assert_instance_of(OpenAI::Realtime::ResponseDoneEvent, done) + assert_equal(:completed, done.response.status) + end + end + end + + def test_mcp_approval_lifecycle_over_a_local_websocket + with_websocket_server(method(:serve_mcp_approval)) do |_task, client| + client.realtime.connect(model: "gpt-realtime-2.1") do |connection| + exercise_mcp_approval(connection) + end + end + end + + def test_translation_reads_output_while_input_is_still_streaming + first_audio = "\x00\x01".b * 4_800 + second_audio = "\x02\x03".b * 4_800 + translated_audio = "\x04\x05".b * 4_800 + handler = lambda do |connection| + serve_translation(connection, first_audio:, second_audio:, translated_audio:) + end + + with_websocket_server(handler) do |task, client| + client.realtime.translations.connect(model: "gpt-realtime-translate") do |connection| + connection.session.update(audio: {output: {language: "es"}}) + events = Async::Queue.new + reader = task.async do + connection.each do |event| + events.enqueue(event) + break if event.is_a?(OpenAI::Realtime::RealtimeTranslationSessionClosedEvent) + end + end + + connection.input_audio_buffer.append_bytes(first_audio) + transcript = events.dequeue + audio = events.dequeue + assert_instance_of(OpenAI::Realtime::RealtimeTranslationOutputTranscriptDeltaEvent, transcript) + assert_equal("hola", transcript.delta) + assert_instance_of(OpenAI::Realtime::RealtimeTranslationOutputAudioDeltaEvent, audio) + assert_equal(translated_audio, Base64.strict_decode64(audio.delta)) + + connection.input_audio_buffer.append_bytes(second_audio) + connection.session.close + assert_instance_of(OpenAI::Realtime::RealtimeTranslationSessionClosedEvent, events.dequeue) + reader.wait + end + end + end + + def test_documented_transcription_lifecycle_over_a_local_websocket + audio = "\x00\x01".b * 2_400 + handler = ->(connection) { serve_transcription(connection, audio: audio) } + + with_websocket_server(handler) do |_task, client| + exercise_transcription(client, audio: audio) + end + end + + def test_default_transport_reassembles_fragmented_text_messages + handler = lambda do |connection| + payload = JSON.generate( + type: "response.output_text.delta", + event_id: "event_fragmented", + response_id: "response_1", + item_id: "item_1", + output_index: 0, + content_index: 0, + delta: "fragmented" + ) + midpoint = payload.bytesize / 2 + first = Protocol::WebSocket::TextFrame.new(false).pack(payload.byteslice(0, midpoint)) + second = Protocol::WebSocket::ContinuationFrame.new(true).pack(payload.byteslice(midpoint..)) + connection.write_frame(first) + connection.write_frame(second) + connection.flush + end + + with_websocket_server(handler) do |_task, client| + event = client.realtime.connect(model: "gpt-realtime-2.1", &:receive) + + assert_instance_of(OpenAI::Realtime::ResponseTextDeltaEvent, event) + assert_equal("fragmented", event.delta) + end + end + + def test_default_transport_wraps_an_abnormal_remote_close + handler = ->(connection) { connection.close(1011, "service failed") } + + with_websocket_server(handler) do |_task, client| + error = assert_raises(OpenAI::Errors::RealtimeConnectionError) do + client.realtime.connect(model: "gpt-realtime-2.1", &:receive) + end + + assert_instance_of(Protocol::WebSocket::ClosedError, error.cause) + assert_includes(error.message, "service failed") + end + end + + def test_default_transport_wraps_handshake_failures + port = available_port + client = OpenAI::Client.new( + api_key: "test-key", + base_url: "http://127.0.0.1:#{port}/v1", + timeout: 0.5 + ) + + error = assert_raises(OpenAI::Errors::RealtimeConnectionError) do + client.realtime.connect(model: "gpt-realtime") { |_connection| nil } + end + + assert_equal("ws://127.0.0.1:#{port}/v1/realtime?model=gpt-realtime", error.url.to_s) + refute_nil(error.cause) + assert_includes(error.message, error.cause.message) + end + + def test_missing_optional_dependency_preserves_load_error_as_the_cause + transport_class = Class.new(OpenAI::Realtime::Transports::AsyncWebSocket) do + private def require(path) + raise LoadError, "cannot load #{path}" if path == "async/websocket/client" + + super + end + end + url = URI("wss://example.com/v1/realtime?model=gpt-realtime-2.1") + + error = assert_raises(OpenAI::Errors::RealtimeConnectionError) do + transport_class.new.open(url:, headers: {}, timeout: 1) { |_socket| nil } + end + + assert_instance_of(LoadError, error.cause) + assert_includes(error.message, "Add `gem \"async-websocket\"` to your Gemfile") + end + + private def assert_application_errors_propagate(client) + error = assert_raises(RuntimeError) do + client.realtime.connect(model: "gpt-realtime") do |_connection| + raise "application failed" + end + end + assert_equal("application failed", error.message) + + load_error = assert_raises(LoadError) do + client.realtime.connect(model: "gpt-realtime") do |_connection| + raise LoadError, "application dependency failed" + end + end + assert_equal("application dependency failed", load_error.message) + end + + private def serve_text_lifecycle(connection) + write_event( + connection, + type: "session.created", + event_id: "event_created", + session: {type: "realtime"} + ) + + update = read_event(connection) + raise "expected session.update" unless update.fetch("type") == "session.update" + write_event( + connection, + type: "session.updated", + event_id: "event_updated", + session: update.fetch("session") + ) + + unless read_event(connection).fetch("type") == "conversation.item.create" + raise "expected conversation item" + end + raise "expected response.create" unless read_event(connection).fetch("type") == "response.create" + write_event( + connection, + type: "response.output_text.delta", + event_id: "event_delta", + response_id: "response_1", + item_id: "item_1", + output_index: 0, + content_index: 0, + delta: "hello from the local service" + ) + write_event( + connection, + type: "response.done", + event_id: "event_done", + response: {id: "response_1", status: "completed", output: []} + ) + end + + private def serve_mcp_approval(connection) + update = read_event(connection) + raise "expected MCP session update" unless update.dig("session", "tools", 0, "type") == "mcp" + write_event( + connection, + type: "mcp_list_tools.completed", + event_id: "event_tools", + item_id: "item_tools" + ) + + raise "expected prompt item" unless read_event(connection).fetch("type") == "conversation.item.create" + raise "expected response.create" unless read_event(connection).fetch("type") == "response.create" + write_event( + connection, + type: "conversation.item.done", + event_id: "event_approval", + previous_item_id: nil, + item: { + type: "mcp_approval_request", + id: "approval_1", + arguments: JSON.generate(date: "tomorrow"), + name: "check_calendar", + server_label: "calendar" + } + ) + + approval = read_event(connection) + raise "expected approval response" unless approval.dig("item", "type") == "mcp_approval_response" + raise "unexpected approval id" unless approval.dig("item", "approval_request_id") == "approval_1" + raise "expected approval" unless approval.dig("item", "approve") + write_event( + connection, + type: "response.done", + event_id: "event_done", + response: {id: "response_1", status: "completed", output: []} + ) + end + + private def exercise_mcp_approval(connection) + connection.session.update( + type: :realtime, + tools: [ + { + type: :mcp, + server_label: "calendar", + server_url: "https://example.com/mcp", + require_approval: :always + } + ] + ) + + assert_instance_of(OpenAI::Realtime::McpListToolsCompleted, connection.receive) + connection.conversation.items.create( + type: :message, + role: :user, + content: [{type: :input_text, text: "Check tomorrow's calendar."}] + ) + connection.response.create + + approval = connection.receive + assert_instance_of(OpenAI::Realtime::ConversationItemDone, approval) + assert_instance_of(OpenAI::Realtime::RealtimeMcpApprovalRequest, approval.item) + connection.conversation.items.respond_to_mcp_approval( + approval_request_id: approval.item.id, + approve: true, + reason: "Allowed by test policy" + ) + + done = connection.receive + assert_instance_of(OpenAI::Realtime::ResponseDoneEvent, done) + assert_equal(:completed, done.response.status) + end + + private def serve_translation(connection, first_audio:, second_audio:, translated_audio:) + raise "expected translation update" unless read_event(connection).fetch("type") == "session.update" + first_append = read_event(connection) + unless first_append.fetch("type") == "session.input_audio_buffer.append" + raise "expected first audio frame" + end + raise "unexpected first audio" unless Base64.strict_decode64(first_append.fetch("audio")) == first_audio + + write_event( + connection, + type: "session.output_transcript.delta", + event_id: "event_transcript", + delta: "hola", + elapsed_ms: 200 + ) + write_event( + connection, + type: "session.output_audio.delta", + event_id: "event_audio", + delta: Base64.strict_encode64(translated_audio), + elapsed_ms: 200, + format: "pcm16", + sample_rate: 24_000, + channels: 1 + ) + + second_append = read_event(connection) + unless second_append.fetch("type") == "session.input_audio_buffer.append" + raise "expected second audio frame" + end + unless Base64.strict_decode64(second_append.fetch("audio")) == second_audio + raise "unexpected second audio" + end + raise "expected session.close" unless read_event(connection).fetch("type") == "session.close" + + write_event(connection, type: "session.closed", event_id: "event_closed") + end + + private def serve_transcription(connection, audio:) + write_event( + connection, + type: "session.created", + event_id: "event_created", + session: {type: "transcription"} + ) + + update = read_event(connection) + raise "expected transcription session update" unless update.fetch("type") == "session.update" + + write_event( + connection, + type: "session.updated", + event_id: "event_updated", + session: update.fetch("session") + ) + + append = read_event(connection) + raise "expected audio append" unless append.fetch("type") == "input_audio_buffer.append" + unless Base64.strict_decode64(append.fetch("audio")) == audio + raise "unexpected transcription audio" + end + raise "expected audio commit" unless read_event(connection).fetch("type") == "input_audio_buffer.commit" + + write_event( + connection, + type: "conversation.item.input_audio_transcription.delta", + event_id: "event_delta", + item_id: "item_1", + content_index: 0, + delta: "hello" + ) + write_event( + connection, + type: "conversation.item.input_audio_transcription.completed", + event_id: "event_completed", + item_id: "item_1", + content_index: 0, + transcript: "hello from ruby", + usage: {type: "duration", seconds: 0.1} + ) + end + + private def exercise_transcription(client, audio:) + client.realtime.connect(intent: :transcription) do |connection| + assert_instance_of(OpenAI::Realtime::SessionCreatedEvent, connection.receive) + configure_transcription(connection) + assert_instance_of(OpenAI::Realtime::SessionUpdatedEvent, connection.receive) + + connection.input_audio_buffer.append_bytes(audio) + connection.input_audio_buffer.commit + assert_transcription_events(connection) + end + end + + private def configure_transcription(connection) + connection.session.update( + audio: { + input: { + format: {type: :"audio/pcm", rate: 24_000}, + transcription: {model: "gpt-live-transcribe"}, + turn_detection: nil + } + } + ) + end + + private def assert_transcription_events(connection) + delta = connection.receive + completed = connection.receive + assert_instance_of(OpenAI::Realtime::ConversationItemInputAudioTranscriptionDeltaEvent, delta) + assert_equal("hello", delta.delta) + assert_instance_of( + OpenAI::Realtime::ConversationItemInputAudioTranscriptionCompletedEvent, + completed + ) + assert_equal(delta.item_id, completed.item_id) + assert_equal("hello from ruby", completed.transcript) + end + + private def with_websocket_server(handler) + port = available_port + endpoint = Async::HTTP::Endpoint.parse("http://127.0.0.1:#{port}") + fallback = ->(_request) { Protocol::HTTP::Response[404, {}, []] } + websocket = Async::WebSocket::Server.new(fallback, &handler) + server = Async::HTTP::Server.new(websocket, endpoint) + + Sync do |task| + server_task = task.async { server.run.wait } + client = OpenAI::Client.new( + api_key: "test-key", + base_url: "http://127.0.0.1:#{port}/v1", + timeout: 5 + ) + yield(task, client) + ensure + server_task&.stop + end + end + + private def read_event(connection) + message = connection.read + raise "client closed before sending the expected event" if message.nil? + + JSON.parse(message.to_str) + end + + private def write_event(connection, **event) + connection.write(Protocol::WebSocket::TextMessage.generate(event)) + connection.flush + end + + private def available_port + server = TCPServer.new("127.0.0.1", 0) + server.local_address.ip_port + ensure + server&.close + end +end diff --git a/test/openai/realtime/connection_test.rb b/test/openai/realtime/connection_test.rb new file mode 100644 index 000000000..9a774c782 --- /dev/null +++ b/test/openai/realtime/connection_test.rb @@ -0,0 +1,577 @@ +# frozen_string_literal: true + +require_relative "../test_helper" + +class OpenAI::Test::RealtimeConnectionTest < Minitest::Test + class FakeSocket + attr_reader :writes, :close_args + + def initialize(*reads) + @reads = reads + @writes = [] + @closed = false + end + + def read = @reads.shift + def write(message) = @writes << message + def closed? = @closed + + def close(code: 1000, reason: "") + @closed = true + @close_args = {code: code, reason: reason} + end + end + + class FakeTransport + attr_reader :open_args + + def initialize(socket) + @socket = socket + end + + def open(url:, headers:, timeout:, **options) + @open_args = {url: url, headers: headers, timeout: timeout, options: options} + yield(@socket) + end + end + + class FailingCloseSocket < FakeSocket + def close(code: 1000, reason: "") + super + raise IOError, "close failed" + end + end + + def client(**options) + OpenAI::Client.new( + api_key: "test-key", + base_url: "https://example.com/v1", + **options + ) + end + + def test_connect_opens_a_typed_block_scoped_connection + socket = FakeSocket.new( + JSON.generate( + type: "response.output_text.delta", + event_id: "event_1", + response_id: "response_1", + item_id: "item_1", + output_index: 0, + content_index: 0, + delta: "hello" + ) + ) + transport = FakeTransport.new(socket) + event = nil + + result = client.realtime.connect( + model: "gpt-realtime", + request_options: { + extra_query: {trace: "yes"}, + extra_headers: {"X-Trace-ID" => "trace_1"}, + timeout: 12 + }, + transport: transport, + transport_options: {max_frame_size: 1_024} + ) do |connection| + assert_instance_of(OpenAI::Realtime::Connection, connection) + event = connection.receive + :block_result + end + + assert_equal(:block_result, result) + assert_instance_of(OpenAI::Realtime::ResponseTextDeltaEvent, event) + assert_equal("hello", event.delta) + assert_equal("wss://example.com/v1/realtime?model=gpt-realtime&trace=yes", transport.open_args[:url].to_s) + assert_equal("Bearer test-key", transport.open_args[:headers].fetch("authorization")) + assert_equal("trace_1", transport.open_args[:headers].fetch("x-trace-id")) + assert_equal(12.0, transport.open_args[:timeout]) + assert_equal({max_frame_size: 1_024}, transport.open_args[:options]) + assert(socket.closed?) + end + + def test_connect_requires_exactly_one_connection_target + error = assert_raises(ArgumentError) do + client.realtime.connect(transport: FakeTransport.new(FakeSocket.new)) { |_connection| nil } + end + assert_includes(error.message, "exactly one") + + error = assert_raises(ArgumentError) do + client.realtime.connect( + model: "gpt-realtime", + call_id: "rtc_123", + transport: FakeTransport.new(FakeSocket.new) + ) { |_connection| nil } + end + assert_includes(error.message, "exactly one") + + error = assert_raises(ArgumentError) do + client.realtime.connect( + model: "gpt-realtime", + intent: :transcription, + transport: FakeTransport.new(FakeSocket.new) + ) { |_connection| nil } + end + assert_includes(error.message, "exactly one") + end + + def test_connect_to_a_dedicated_transcription_session + socket = FakeSocket.new + transport = FakeTransport.new(socket) + + client.realtime.connect(intent: :transcription, transport: transport) do |connection| + assert_instance_of(OpenAI::Realtime::TranscriptionConnection, connection) + assert_instance_of(OpenAI::Realtime::ConnectionResources::TranscriptionSession, connection.session) + assert_respond_to(connection.input_audio_buffer, :commit) + assert_respond_to(connection.input_audio_buffer, :clear) + refute_respond_to(connection, :response) + refute_respond_to(connection, :conversation) + refute_respond_to(connection, :output_audio_buffer) + connection.session.update(audio: {input: {turn_detection: nil}}, event_id: "update_1") + end + + assert_equal( + "wss://example.com/v1/realtime?intent=transcription", + transport.open_args[:url].to_s + ) + update = JSON.parse(socket.writes.fetch(0), symbolize_names: true) + assert_equal(:transcription, update.dig(:session, :type).to_sym) + assert_equal("update_1", update.fetch(:event_id)) + refute(update.fetch(:session).key?(:event_id)) + end + + def test_connect_rejects_an_unknown_intent + error = assert_raises(ArgumentError) do + client.realtime.connect( + intent: :future, + transport: FakeTransport.new(FakeSocket.new) + ) { |_connection| nil } + end + + assert_includes(error.message, "transcription") + end + + def test_connect_to_an_existing_webrtc_or_sip_call + transport = FakeTransport.new(FakeSocket.new) + + client.realtime.connect(call_id: "rtc_123", transport: transport) do |connection| + assert_instance_of(OpenAI::Realtime::SidebandConnection, connection) + assert_respond_to(connection, :output_audio_buffer) + end + + assert_equal("wss://example.com/v1/realtime?call_id=rtc_123", transport.open_args[:url].to_s) + end + + def test_connect_uses_an_explicit_websocket_base_url + transport = FakeTransport.new(FakeSocket.new) + custom_client = client(websocket_base_url: "wss://socket.example.test/custom/v2") + + custom_client.realtime.connect(model: "gpt-realtime", transport: transport) { |_connection| nil } + + assert_equal( + "wss://socket.example.test/custom/v2/realtime?model=gpt-realtime", + transport.open_args[:url].to_s + ) + end + + def test_websocket_base_url_rejects_ambiguous_or_unsafe_urls + invalid_urls = [ + "/socket", + "ftp://socket.example.test/v1", + "wss://user:secret@socket.example.test/v1", + "wss://socket.example.test/v1?tenant=one", + "wss://socket.example.test/v1#fragment" + ] + + invalid_urls.each do |url| + error = assert_raises(ArgumentError) { client(websocket_base_url: url) } + assert_includes(error.message, "`websocket_base_url`") + end + end + + def test_connect_uses_azure_provider_authentication + transport = FakeTransport.new(FakeSocket.new) + azure_client = OpenAI::Client.new( + provider: OpenAI::Providers.azure( + endpoint: "https://resource.openai.azure.com", + api_key: "azure-key" + ) + ) + + azure_client.realtime.connect(model: "deployment", transport: transport) { |_connection| nil } + + assert_equal( + "wss://resource.openai.azure.com/openai/v1/realtime?model=deployment", + transport.open_args[:url].to_s + ) + assert_equal("azure-key", transport.open_args[:headers].fetch("api-key")) + refute(transport.open_args[:headers].key?("authorization")) + end + + def test_connect_resolves_workload_identity_before_the_handshake + subject_token_provider = + OpenAI::Auth::SubjectTokenProviders::K8sServiceAccountTokenProvider.new( + token_path: "/not-read-by-this-test" + ) + config = OpenAI::Auth::WorkloadIdentity.new( + identity_provider_id: "idp_123", + service_account_id: "sa_123", + provider: subject_token_provider + ) + workload_client = OpenAI::Client.new( + api_key: nil, + workload_identity: config, + organization: "org_123", + base_url: "https://example.com/v1" + ) + transport = FakeTransport.new(FakeSocket.new) + + workload_client.workload_identity_auth.stub(:get_token, "exchanged-token") do + workload_client.realtime.connect(model: "gpt-realtime", transport: transport) do |_connection| + nil + end + end + + assert_equal("Bearer exchanged-token", transport.open_args[:headers].fetch("authorization")) + end + + def test_unsupported_provider_fails_before_resolving_or_sending_credentials + credential_requested = false + provider = OpenAI::Providers.bedrock( + region: "us-east-1", + token_provider: lambda do + credential_requested = true + "bedrock-token" + end + ) + provider_client = OpenAI::Client.new(provider: provider) + transport = FakeTransport.new(FakeSocket.new) + + error = assert_raises(OpenAI::Errors::Error) do + provider_client.realtime.connect(model: "gpt-realtime", transport: transport) do |_connection| + nil + end + end + + assert_includes(error.message, "not supported") + refute(credential_requested) + assert_nil(transport.open_args) + end + + def test_connection_sends_typed_events_and_audio_bytes + socket = FakeSocket.new + transport = FakeTransport.new(socket) + + client.realtime.connect(model: "gpt-realtime", transport: transport) do |connection| + send_typed_resource_events(connection) + end + + events = socket.writes.map { JSON.parse(_1, symbolize_names: true) } + assert_equal( + [ + :"session.update", + :"response.create", + :"input_audio_buffer.append", + :"input_audio_buffer.commit", + :"conversation.item.create", + :"conversation.item.create", + :"conversation.item.create", + :"conversation.item.create" + ], + events.map { _1.fetch(:type).to_sym } + ) + assert_equal("session_update_1", events[0].fetch(:event_id)) + assert_equal("Be concise", events[0].dig(:session, :instructions)) + refute(events[0].fetch(:session).key?(:event_id)) + assert_equal("response_create_1", events[1].fetch(:event_id)) + assert_equal("Say hello", events[1].dig(:response, :instructions)) + refute(events[1].fetch(:response).key?(:event_id)) + assert_equal(Base64.strict_encode64("\x00\x01".b), events[2].fetch(:audio)) + assert_equal("item_create_1", events[4].fetch(:event_id)) + assert_equal("root", events[4].fetch(:previous_item_id)) + assert_equal("Hello", events[4].dig(:item, :content, 0, :text)) + refute(events[4].fetch(:item).key?(:previous_item_id)) + assert_equal(:function_call_output, events[5].fetch(:item).fetch(:type).to_sym) + assert_equal(:mcp_approval_response, events[6].fetch(:item).fetch(:type).to_sym) + assert(events[6].fetch(:item).fetch(:approve)) + generated_id = events[7].fetch(:item).fetch(:id) + assert_operator(generated_id.length, :<=, 32) + assert_match(/\Amcpa_[0-9a-f]+\z/, generated_id) + end + + def test_standard_connection_only_exposes_standard_websocket_capabilities + transport = FakeTransport.new(FakeSocket.new) + + client.realtime.connect(model: "gpt-realtime", transport: transport) do |connection| + refute_respond_to(connection, :output_audio_buffer) + refute_respond_to(connection.session, :close) + assert_respond_to(connection.input_audio_buffer, :commit) + assert_respond_to(connection.input_audio_buffer, :clear) + assert_respond_to(connection.conversation, :items) + refute_respond_to(connection.conversation, :item) + end + end + + def test_sideband_connection_exposes_output_audio_buffer_controls + socket = FakeSocket.new + transport = FakeTransport.new(socket) + + client.realtime.connect(call_id: "rtc_123", transport: transport) do |connection| + connection.output_audio_buffer.clear(event_id: "clear_1") + end + + assert_equal( + {"type" => "output_audio_buffer.clear", "event_id" => "clear_1"}, + JSON.parse(socket.writes.fetch(0)) + ) + end + + def test_sideband_interruption_helpers_preserve_documented_order + socket = FakeSocket.new + transport = FakeTransport.new(socket) + + client.realtime.connect(call_id: "rtc_123", transport: transport) do |connection| + connection.response.cancel + connection.conversation.items.truncate(item_id: "item_1", content_index: 0, audio_end_ms: 640) + connection.output_audio_buffer.clear + end + + assert_equal( + [ + "response.cancel", + "conversation.item.truncate", + "output_audio_buffer.clear" + ], + socket.writes.map { JSON.parse(_1).fetch("type") } + ) + end + + def test_all_standard_resource_helpers_emit_typed_events + socket = FakeSocket.new + transport = FakeTransport.new(socket) + + client.realtime.connect(model: "gpt-realtime", transport: transport) do |connection| + connection.response.cancel(response_id: "response_1", event_id: "cancel_1") + connection.input_audio_buffer.clear(event_id: "clear_1") + connection.conversation.items.retrieve(item_id: "item_1", event_id: "retrieve_1") + connection.conversation.items.truncate( + item_id: "item_1", + content_index: 0, + audio_end_ms: 250, + event_id: "truncate_1" + ) + connection.conversation.items.delete(item_id: "item_1", event_id: "delete_1") + end + + assert_equal( + [ + "response.cancel", + "input_audio_buffer.clear", + "conversation.item.retrieve", + "conversation.item.truncate", + "conversation.item.delete" + ], + socket.writes.map { JSON.parse(_1).fetch("type") } + ) + end + + def test_connection_does_not_override_ruby_send + transport = FakeTransport.new(FakeSocket.new) + + client.realtime.connect(model: "gpt-realtime", transport: transport) do |connection| + assert_equal(false, connection.send(:closed?)) + end + end + + def test_connection_rejects_an_unknown_client_event + transport = FakeTransport.new(FakeSocket.new) + + error = assert_raises(ArgumentError) do + client.realtime.connect(model: "gpt-realtime", transport: transport) do |connection| + connection.send_event(type: "future.unknown.event") + end + end + + assert_includes(error.message, "future.unknown.event") + end + + def test_connection_rejects_a_client_event_missing_required_fields + transport = FakeTransport.new(FakeSocket.new) + + error = assert_raises(ArgumentError) do + client.realtime.connect(model: "gpt-realtime", transport: transport) do |connection| + connection.send_event(type: "conversation.item.create") + end + end + + assert_includes(error.message, "required fields") + end + + def test_connect_requires_a_block + error = assert_raises(ArgumentError) do + client.realtime.connect(model: "gpt-realtime", transport: FakeTransport.new(FakeSocket.new)) + end + + assert_includes(error.message, "block is required") + end + + def test_connection_each_returns_a_typed_enumerator_without_a_block + event_data = JSON.generate( + type: "response.output_text.delta", + event_id: "event_1", + response_id: "response_1", + item_id: "item_1", + output_index: 0, + content_index: 0, + delta: "hello" + ) + transport = FakeTransport.new(FakeSocket.new(event_data, nil)) + events = nil + + client.realtime.connect(model: "gpt-realtime", transport: transport) do |connection| + events = connection.each.to_a + end + + assert_equal(1, events.length) + assert_instance_of(OpenAI::Realtime::ResponseTextDeltaEvent, events.first) + end + + def test_connection_each_returns_the_connection_with_a_block + data = JSON.generate(type: "future.event") + transport = FakeTransport.new(FakeSocket.new(data, nil)) + + client.realtime.connect(model: "gpt-realtime", transport: transport) do |connection| + count = 0 + result = connection.each { |_event| count += 1 } + + assert_equal(1, count) + assert_same(connection, result) + end + end + + def test_connection_closes_when_the_block_raises + socket = FakeSocket.new + transport = FakeTransport.new(socket) + + assert_raises(RuntimeError) do + client.realtime.connect(model: "gpt-realtime", transport: transport) do |_connection| + raise "boom" + end + end + + assert(socket.closed?) + end + + def test_application_error_wins_when_cleanup_also_fails + transport = FakeTransport.new(FailingCloseSocket.new) + + error = assert_raises(RuntimeError) do + client.realtime.connect(model: "gpt-realtime", transport: transport) do |_connection| + raise "application failed" + end + end + + assert_equal("application failed", error.message) + end + + def test_cleanup_error_propagates_after_a_successful_block + transport = FakeTransport.new(FailingCloseSocket.new) + + error = assert_raises(IOError) do + client.realtime.connect(model: "gpt-realtime", transport: transport) { |_connection| :done } + end + + assert_equal("close failed", error.message) + end + + def test_malformed_server_event_raises_a_realtime_protocol_error + transport = FakeTransport.new(FakeSocket.new("not-json")) + + error = assert_raises(OpenAI::Errors::RealtimeProtocolError) do + client.realtime.connect(model: "gpt-realtime", transport: transport, &:receive) + end + + assert_equal("not-json", error.data) + assert_instance_of(JSON::ParserError, error.cause) + end + + def test_api_error_remains_a_typed_event_in_the_stream + transport = FakeTransport.new( + FakeSocket.new( + JSON.generate( + type: "error", + event_id: "event_1", + error: {type: "invalid_request_error", message: "Try another value"} + ) + ) + ) + + event = client.realtime.connect(model: "gpt-realtime", transport: transport, &:receive) + + assert_instance_of(OpenAI::Realtime::RealtimeErrorEvent, event) + assert_equal("Try another value", event.error.message) + end + + def test_unknown_server_event_remains_observable + data = JSON.generate(type: "future.unknown.event", nested: {values: ["future value"]}) + transport = FakeTransport.new(FakeSocket.new(data)) + + event = client.realtime.connect(model: "gpt-realtime", transport: transport, &:receive) + + assert_instance_of(OpenAI::Realtime::UnknownServerEvent, event) + assert_equal(:"future.unknown.event", event.type) + assert_equal( + {type: "future.unknown.event", nested: {values: ["future value"]}}, + event.data + ) + assert(event.frozen?) + assert(event.data.frozen?) + assert(event.data.fetch(:nested).frozen?) + assert(event.data.dig(:nested, :values).frozen?) + assert(event.data.dig(:nested, :values, 0).frozen?) + end + + def test_server_event_missing_required_fields_raises_a_realtime_protocol_error + data = JSON.generate(type: "response.output_text.delta") + transport = FakeTransport.new(FakeSocket.new(data)) + + error = assert_raises(OpenAI::Errors::RealtimeProtocolError) do + client.realtime.connect(model: "gpt-realtime", transport: transport, &:receive) + end + + assert_equal(data, error.data) + assert_includes(error.cause.message, "required fields") + end + + private def send_typed_resource_events(connection) + connection.session.update( + type: :realtime, + instructions: "Be concise", + event_id: "session_update_1" + ) + connection.response.create(instructions: "Say hello", event_id: "response_create_1") + connection.input_audio_buffer.append_bytes("\x00\x01".b) + connection.input_audio_buffer.commit + connection.conversation.items.create( + type: :message, + role: :user, + content: [{type: :input_text, text: "Hello"}], + previous_item_id: "root", + event_id: "item_create_1" + ) + connection.conversation.items.create_function_call_output( + call_id: "call_1", + output: JSON.generate(temperature: 72) + ) + connection.conversation.items.respond_to_mcp_approval( + approval_request_id: "approval_1", + approve: true, + id: "approval_response_1" + ) + connection.conversation.items.respond_to_mcp_approval( + approval_request_id: "approval_2", + approve: false + ) + end +end diff --git a/test/openai/realtime/examples_test.rb b/test/openai/realtime/examples_test.rb new file mode 100644 index 000000000..efa58cecc --- /dev/null +++ b/test/openai/realtime/examples_test.rb @@ -0,0 +1,656 @@ +# frozen_string_literal: true + +require "open3" +require_relative "../test_helper" +require_relative "../../../examples/realtime/mcp_approval" +require_relative "../../../examples/realtime/realtime_conversation" +require_relative "../../../examples/realtime/sideband" +require_relative "../../../examples/realtime/sip" +require_relative "../../../examples/realtime/webrtc_conversation" + +class OpenAI::Test::RealtimeExamplesTest < Minitest::Test + Event = Data.define(:type, :data) do + def to_h = data + end + + class RecordingSession + attr_reader :updates + + def initialize + @updates = [] + end + + def update(**params) + @updates << params + end + end + + class RecordingResource + attr_reader :calls, :truncations + + def initialize + @calls = [] + @truncations = [] + end + + def create(**params) + @calls << params + end + + def truncate(**params) + @truncations << params + end + end + + class RecordingConversation + attr_reader :items + + def initialize + @items = RecordingResource.new + end + end + + class RecordingAudioBuffer + attr_reader :chunks + + def initialize + @chunks = [] + end + + def append_bytes(bytes) + @chunks << bytes + end + end + + class RecordingConnection + attr_reader :conversation, :input_audio_buffer, :response, :session + + def initialize(events = []) + @events = events + @session = RecordingSession.new + @conversation = RecordingConversation.new + @input_audio_buffer = RecordingAudioBuffer.new + @response = RecordingResource.new + end + + def each(&block) + @events.each(&block) + end + end + + class RecordingCalls + attr_reader :accepts + + def initialize + @accepts = [] + end + + def accept(call_id, **params) + @accepts << [call_id, params] + end + end + + class RecordingRealtime + attr_reader :calls, :connections + + def initialize(connection) + @calls = RecordingCalls.new + @connection = connection + @connections = [] + end + + def connect(call_id:) + @connections << call_id + yield(@connection) + end + end + + RecordingClient = Data.define(:realtime) + + HTTPRequest = Data.define(:request_method, :path, :body, :headers) do + def [](name) = headers[name.downcase] + end + + class HTTPResponse + attr_accessor :body, :status + attr_reader :headers + + def initialize + @headers = {} + end + + def []=(name, value) + @headers[name] = value + end + end + + class RecordingWebRTCCalls + attr_accessor :hangup_error + attr_reader :creates, :hangups + + def initialize + @creates = [] + @hangups = [] + end + + def create(**params) + @creates << params + OpenAI::Realtime::CallCreateResponse.new( + sdp: "answer-sdp", + call_id: "rtc_example", + headers: {} + ) + end + + def hangup(call_id) + @hangups << call_id + raise @hangup_error if @hangup_error + end + end + + class RecordingMicrophone + def initialize(chunks) + @chunks = chunks + end + + def each_chunk(&block) + @chunks.each(&block) + end + end + + class RecordingSpeaker + attr_reader :audio, :interruptions + + def initialize + @audio = +"".b + @interruptions = 0 + end + + def write(bytes) + @audio << bytes + end + + def interrupt + @interruptions += 1 + end + + def close = nil + end + + def test_realtime_conversation_configures_continuous_server_vad + connection = RecordingConnection.new + + OpenAI::Examples::Realtime::Conversation.configure( + connection, + voice: :marin, + instructions: "Speak naturally." + ) + + assert_equal( + { + type: :realtime, + output_modalities: [:audio], + instructions: "Speak naturally.", + audio: { + input: { + format: {type: :"audio/pcm", rate: 24_000}, + noise_reduction: {type: :near_field}, + turn_detection: { + type: :server_vad, + threshold: 0.5, + prefix_padding_ms: 300, + silence_duration_ms: 500, + create_response: true, + interrupt_response: true + } + }, + output: { + format: {type: :"audio/pcm", rate: 24_000}, + voice: :marin + } + } + }, + connection.session.updates.fetch(0) + ) + end + + def test_realtime_conversation_streams_microphone_chunks_without_committing + connection = RecordingConnection.new + microphone = RecordingMicrophone.new(["first".b, "second".b]) + + OpenAI::Examples::Realtime::Conversation.forward_microphone(connection, microphone) + + assert_equal(["first".b, "second".b], connection.input_audio_buffer.chunks) + assert_empty(connection.response.calls) + end + + def test_realtime_conversation_streams_audio_and_interrupts_local_playback + speaker = RecordingSpeaker.new + connection = RecordingConnection.new + times = [0.0, 0.1].each + playback = OpenAI::Examples::Realtime::Conversation::AudioPlayback.new( + speaker, + clock: -> { times.next } + ) + output = StringIO.new + audio = "\x00\x01".b * 2_400 + event_fields = { + event_id: "event_1", + response_id: "response_1", + item_id: "item_1", + output_index: 0, + content_index: 0 + } + + OpenAI::Examples::Realtime::Conversation.handle_event( + OpenAI::Realtime::ResponseAudioDeltaEvent.new(**event_fields, delta: Base64.strict_encode64(audio)), + connection: connection, + playback: playback, + output: output + ) + OpenAI::Examples::Realtime::Conversation.handle_event( + OpenAI::Realtime::ResponseAudioTranscriptDeltaEvent.new(**event_fields, delta: "Hello"), + connection: connection, + playback: playback, + output: output + ) + OpenAI::Examples::Realtime::Conversation.handle_event( + OpenAI::Realtime::InputAudioBufferSpeechStartedEvent.new( + event_id: "event_2", + item_id: "item_2", + audio_start_ms: 100 + ), + connection: connection, + playback: playback, + output: output + ) + + assert_equal(audio, speaker.audio) + assert_equal("Hello\n", output.string) + assert_equal(1, speaker.interruptions) + assert_equal( + [{item_id: "item_1", content_index: 0, audio_end_ms: 100}], + connection.conversation.items.truncations + ) + end + + def test_realtime_conversation_drops_queued_audio_after_interruption + speaker = RecordingSpeaker.new + connection = RecordingConnection.new + times = [0.0, 0.05].each + playback = OpenAI::Examples::Realtime::Conversation::AudioPlayback.new( + speaker, + clock: -> { times.next } + ) + output = StringIO.new + event_fields = { + event_id: "event_1", + response_id: "response_1", + item_id: "item_1", + output_index: 0, + content_index: 0 + } + first_audio = "\x00\x01".b * 2_400 + queued_audio = "\x02\x03".b * 2_400 + + playback.write( + OpenAI::Realtime::ResponseAudioDeltaEvent.new( + **event_fields, + delta: Base64.strict_encode64(first_audio) + ) + ) + OpenAI::Examples::Realtime::Conversation.handle_event( + OpenAI::Realtime::InputAudioBufferSpeechStartedEvent.new( + event_id: "event_2", + item_id: "item_2", + audio_start_ms: 50 + ), + connection: connection, + playback: playback, + output: output + ) + playback.write( + OpenAI::Realtime::ResponseAudioDeltaEvent.new( + **event_fields, + event_id: "event_3", + delta: Base64.strict_encode64(queued_audio) + ) + ) + OpenAI::Examples::Realtime::Conversation.handle_event( + OpenAI::Realtime::ResponseDoneEvent.new( + event_id: "event_4", + response: OpenAI::Realtime::RealtimeResponse.new(id: "response_1", status: :cancelled) + ), + connection: connection, + playback: playback, + output: output + ) + + assert_equal(first_audio, speaker.audio) + assert_equal("\n", output.string) + assert_equal(1, speaker.interruptions) + end + + def test_realtime_conversation_can_capture_output_audio_for_a_live_smoke_test + Tempfile.create(["realtime-conversation", ".pcm"]) do |file| + speaker = OpenAI::Examples::Realtime::Conversation::PCMFileSpeaker.new(file.path) + + speaker.write("\x00\x01".b) + speaker.interrupt + speaker.write("\x02\x03".b) + speaker.close + + assert_equal("\x00\x01\x02\x03".b, File.binread(file.path)) + end + end + + def test_realtime_conversation_uses_ffplays_input_channel_layout_option + speaker_class = Class.new(OpenAI::Examples::Realtime::Conversation::FFplaySpeaker) do + def playback_command = command + end + command = speaker_class.new.playback_command + + assert_equal(["-ch_layout", "mono"], command.slice(command.index("-ch_layout"), 2)) + refute_includes(command, "-ac") + end + + def test_realtime_conversation_ffplay_command_accepts_raw_pcm + speaker_class = Class.new(OpenAI::Examples::Realtime::Conversation::FFplaySpeaker) do + def playback_command = command + end + command = speaker_class.new.playback_command + command.insert(command.index("-i"), "-autoexit") + + _, stderr, status = Open3.capture3( + {"SDL_AUDIODRIVER" => "dummy"}, + *command, + stdin_data: "\0".b * 4_800 + ) + + assert_predicate(status, :success?, stderr) + assert_empty(stderr, "ffplay must not emit terminal control sequences") + rescue Errno::ENOENT + skip("ffplay is not installed") + end + + def test_webrtc_conversation_serves_a_browser_peer_with_audio_processing + calls = RecordingWebRTCCalls.new + app = OpenAI::Examples::Realtime::WebRTCConversation::App.new( + client: RecordingClient.new(Data.define(:calls).new(calls)) + ) + response = HTTPResponse.new + + app.handle( + HTTPRequest.new( + request_method: "GET", + path: "/", + body: "", + headers: {} + ), + response + ) + + assert_equal(200, response.status) + assert_equal("text/html; charset=utf-8", response.headers.fetch("Content-Type")) + assert_includes(response.body, "new RTCPeerConnection()") + assert_includes(response.body, "echoCancellation: true") + assert_includes(response.body, "noiseSuppression: true") + assert_includes(response.body, "autoGainControl: true") + refute_includes(response.body, "OPENAI_API_KEY") + end + + def test_webrtc_conversation_negotiates_and_hangs_up_through_ruby + calls = RecordingWebRTCCalls.new + app = OpenAI::Examples::Realtime::WebRTCConversation::App.new( + client: RecordingClient.new(Data.define(:calls).new(calls)), + html: "test" + ) + response = HTTPResponse.new + + app.handle( + HTTPRequest.new( + request_method: "POST", + path: "/session", + body: "v=0\r\nt=0 0\r\n", + headers: {"content-type" => "application/sdp"} + ), + response + ) + + assert_equal(201, response.status) + assert_equal("answer-sdp", response.body) + assert_equal("rtc_example", response.headers.fetch("X-OpenAI-Call-ID")) + assert_equal("v=0\r\nt=0 0\r\n", calls.creates.fetch(0).fetch(:sdp)) + session = calls.creates.fetch(0).fetch(:session) + assert_equal("gpt-realtime-2.1", session.fetch(:model)) + assert_equal(true, session.dig(:audio, :input, :turn_detection, :interrupt_response)) + + hangup_response = HTTPResponse.new + app.handle( + HTTPRequest.new( + request_method: "POST", + path: "/hangup", + body: "rtc_example", + headers: {} + ), + hangup_response + ) + + assert_equal(204, hangup_response.status) + assert_equal(["rtc_example"], calls.hangups) + end + + def test_webrtc_conversation_rejects_non_sdp_requests + calls = RecordingWebRTCCalls.new + app = OpenAI::Examples::Realtime::WebRTCConversation::App.new( + client: RecordingClient.new(Data.define(:calls).new(calls)), + html: "test" + ) + response = HTTPResponse.new + + app.handle( + HTTPRequest.new( + request_method: "POST", + path: "/session", + body: "not sdp", + headers: {"content-type" => "text/plain"} + ), + response + ) + + assert_equal(400, response.status) + assert_empty(calls.creates) + end + + def test_webrtc_conversation_treats_an_already_closed_call_as_hung_up + calls = RecordingWebRTCCalls.new + calls.hangup_error = OpenAI::Errors::NotFoundError.new( + url: URI("https://api.openai.com/v1/realtime/calls/rtc_example/hangup"), + status: 404, + headers: {}, + body: {}, + request: nil, + response: nil + ) + app = OpenAI::Examples::Realtime::WebRTCConversation::App.new( + client: RecordingClient.new(Data.define(:calls).new(calls)), + html: "test" + ) + app.handle( + HTTPRequest.new( + request_method: "POST", + path: "/session", + body: "v=0\r\nt=0 0\r\n", + headers: {"content-type" => "application/sdp"} + ), + HTTPResponse.new + ) + response = HTTPResponse.new + + app.handle( + HTTPRequest.new( + request_method: "POST", + path: "/hangup", + body: "rtc_example", + headers: {} + ), + response + ) + + assert_equal(204, response.status) + assert_equal(["rtc_example"], calls.hangups) + end + + def test_mcp_console_example_requests_text_output + connection = RecordingConnection.new + + OpenAI::Examples::Realtime::MCPApproval.configure( + connection, + server_url: "https://developers.openai.com/mcp" + ) + + session = connection.session.updates.fetch(0) + assert_equal([:text], session.fetch(:output_modalities)) + refute(session.key?(:tool_choice)) + end + + def test_mcp_console_example_requires_the_discovered_tool + tools_item = OpenAI::Realtime::RealtimeMcpListTools.new( + id: "mcp_tools_1", + server_label: "remote", + tools: [ + OpenAI::Realtime::RealtimeMcpListTools::Tool.new( + name: "search_openai_docs", + description: "Search the OpenAI developer documentation", + input_schema: {type: "object"} + ) + ] + ) + connection = RecordingConnection.new( + [ + OpenAI::Realtime::ConversationItemDone.new( + event_id: "event_1", + item: tools_item + ), + OpenAI::Realtime::McpListToolsCompleted.new( + event_id: "event_2", + item_id: "mcp_tools_1" + ), + OpenAI::Realtime::SessionUpdatedEvent.new( + event_id: "event_3", + session: {} + ) + ] + ) + + OpenAI::Examples::Realtime::MCPApproval.run_session( + connection, + prompt: "Search the docs", + output: StringIO.new + ) + + assert_equal( + { + type: :realtime, + tools: [{type: :mcp, server_label: "remote"}], + tool_choice: {type: :mcp, server_label: "remote", name: "search_openai_docs"} + }, + connection.session.updates.fetch(0) + ) + assert_equal({}, connection.response.calls.fetch(0)) + end + + def test_mcp_console_example_waits_for_the_tool_choice_update + tools_item = OpenAI::Realtime::RealtimeMcpListTools.new( + id: "mcp_tools_1", + server_label: "remote", + tools: [ + OpenAI::Realtime::RealtimeMcpListTools::Tool.new( + name: "search_openai_docs", + input_schema: {type: "object"} + ) + ] + ) + connection = RecordingConnection.new( + [ + OpenAI::Realtime::ConversationItemDone.new(event_id: "event_1", item: tools_item), + OpenAI::Realtime::McpListToolsCompleted.new(event_id: "event_2", item_id: "mcp_tools_1") + ] + ) + + OpenAI::Examples::Realtime::MCPApproval.run_session( + connection, + prompt: "Search the docs", + output: StringIO.new + ) + + assert_empty(connection.response.calls) + assert_empty(connection.conversation.items.calls) + end + + def test_mcp_console_example_requests_a_final_answer_after_the_tool_completes + connection = RecordingConnection.new( + [ + OpenAI::Realtime::ResponseMcpCallCompleted.new( + event_id: "event_1", + item_id: "mcp_call_1", + output_index: 0 + ) + ] + ) + + OpenAI::Examples::Realtime::MCPApproval.run_session( + connection, + prompt: "Search the docs", + output: StringIO.new + ) + + assert_equal({tool_choice: :none}, connection.response.calls.fetch(0)) + end + + def test_sideband_smoke_mode_stops_after_the_selected_event + events = [ + Event.new(type: :"session.updated", data: {type: :"session.updated"}), + Event.new(type: :"response.done", data: {type: :"response.done"}) + ] + connection = RecordingConnection.new(events) + output = StringIO.new + + OpenAI::Examples::Realtime::Sideband.stream( + connection, + output: output, + stop_after: "session.updated" + ) + + assert_includes(output.string, "session.updated") + refute_includes(output.string, "response.done") + end + + def test_sip_example_accepts_then_attaches_to_the_call + connection = RecordingConnection.new( + [Event.new(type: :"session.created", data: {type: :"session.created"})] + ) + realtime = RecordingRealtime.new(connection) + + OpenAI::Examples::Realtime::SIP.run( + client: RecordingClient.new(realtime: realtime), + call_id: "rtc_123", + model: "gpt-realtime-2.1", + output: StringIO.new, + stop_after: "session.created" + ) + + assert_equal(["rtc_123"], realtime.connections) + assert_equal( + [ + "rtc_123", + { + type: :realtime, + model: "gpt-realtime-2.1", + instructions: "You are answering a phone call. Be warm and concise." + } + ], + realtime.calls.accepts.fetch(0) + ) + end +end diff --git a/test/openai/resources/realtime/calls_test.rb b/test/openai/resources/realtime/calls_test.rb index 938be40ed..a8d2e136e 100644 --- a/test/openai/resources/realtime/calls_test.rb +++ b/test/openai/resources/realtime/calls_test.rb @@ -3,6 +3,95 @@ require_relative "../../test_helper" class OpenAI::Test::Resources::Realtime::CallsTest < OpenAI::Test::ResourceTest + class CapturingLogger + attr_reader :events + + def initialize = @events = [] + + def debug(message) = @events << [:debug, message] + def info(message) = @events << [:info, message] + def warn(message) = @events << [:warn, message] + def error(message) = @events << [:error, message] + end + + class StubHTTPClient < OpenAI::HTTPClient + attr_reader :request + + def execute(request) + @request = request + OpenAI::HTTPClient::Response.new( + status: 201, + headers: { + "Content-Type" => "application/sdp", + "Location" => "/v1/realtime/calls/rtc_123", + "X-Request-ID" => "req_realtime_call" + }, + body: "answer-sdp" + ) + end + end + + def test_create_with_an_sdp_offer + http_client = StubHTTPClient.new + client = OpenAI::Client.new( + api_key: "test-key", + base_url: "https://example.com/v1", + http_client: http_client + ) + + response = client.realtime.calls.create(sdp: "offer-sdp") + + assert_equal("answer-sdp", response.sdp) + assert_equal("rtc_123", response.call_id) + assert_equal("/v1/realtime/calls/rtc_123", response.headers.fetch("location")) + assert_equal(:post, http_client.request.method) + assert_equal("https://example.com/v1/realtime/calls", http_client.request.url.to_s) + assert_equal("application/sdp", http_client.request.headers.fetch("content-type")) + assert_equal("offer-sdp", http_client.request.body) + end + + def test_create_with_session_configuration_uses_typed_multipart_parts + http_client = StubHTTPClient.new + client = OpenAI::Client.new( + api_key: "test-key", + base_url: "https://example.com/v1", + http_client: http_client + ) + + client.realtime.calls.create( + sdp: "offer-sdp", + session: {type: :realtime, model: "gpt-realtime"} + ) + + content_type = http_client.request.headers.fetch("content-type") + body = http_client.request.body.to_a.join + + assert_match(%r{\Amultipart/form-data; boundary=}, content_type) + assert_includes(body, "name=\"sdp\"\r\nContent-Type: application/sdp\r\n\r\noffer-sdp") + assert_includes(body, "name=\"session\"\r\nContent-Type: application/json") + assert_includes(body, JSON.generate(type: :realtime, model: "gpt-realtime")) + refute_includes(body, "filename=") + end + + def test_create_preserves_request_observability_for_raw_sdp_responses + logger = CapturingLogger.new + client = OpenAI::Client.new( + api_key: "test-key", + base_url: "https://example.com/v1", + http_client: StubHTTPClient.new, + logger: logger, + log_level: :debug + ) + + response = client.realtime.calls.create(sdp: "offer-sdp") + + assert_equal("answer-sdp", response.sdp) + log = logger.events.map(&:last).join("\n") + assert_includes(log, "request complete") + assert_includes(log, "request_id=req_realtime_call") + assert_includes(log, "response body") + end + def test_accept_required_params response = @openai.realtime.calls.accept("call_id", type: :realtime) diff --git a/test/openai/resources/realtime/translations_test.rb b/test/openai/resources/realtime/translations_test.rb new file mode 100644 index 000000000..86055ec85 --- /dev/null +++ b/test/openai/resources/realtime/translations_test.rb @@ -0,0 +1,181 @@ +# frozen_string_literal: true + +require_relative "../../test_helper" + +class OpenAI::Test::Resources::Realtime::TranslationsTest < Minitest::Test + class StubHTTPClient < OpenAI::HTTPClient + attr_reader :requests + + def initialize(*responses) + super() + @responses = responses + @requests = [] + end + + def execute(request) + @requests << request + @responses.shift + end + end + + class FakeSocket + attr_reader :writes + + def initialize(*reads) + @reads = reads + @writes = [] + @closed = false + end + + def read = @reads.shift + def write(message) = @writes << message + def closed? = @closed + def close(code: 1000, reason: "") = (@closed = [code, reason].any?) + end + + class FakeTransport + attr_reader :url + + def initialize(socket) + @socket = socket + end + + def open(url:, **_options) + @url = url + yield(@socket) + end + end + + def test_create_client_secret + http_client = StubHTTPClient.new( + OpenAI::HTTPClient::Response.new( + status: 200, + headers: {"content-type" => "application/json"}, + body: JSON.generate( + value: "ek_test", + expires_at: 123, + session: { + id: "sess_123", + type: "translation", + model: "gpt-realtime-translate", + expires_at: 456, + audio: {} + } + ) + ) + ) + client = OpenAI::Client.new(api_key: "test-key", http_client: http_client) + + response = client.realtime.translations.client_secrets.create( + session: { + model: "gpt-realtime-translate", + audio: {output: {language: "es"}} + } + ) + + assert_instance_of(OpenAI::Realtime::RealtimeTranslationClientSecretCreateResponse, response) + assert_equal("ek_test", response.value) + request = http_client.requests.fetch(0) + assert_equal("/v1/realtime/translations/client_secrets", request.url.path) + assert_equal( + { + session: { + model: "gpt-realtime-translate", + audio: {output: {language: "es"}} + } + }, + JSON.parse(request.body, symbolize_names: true) + ) + end + + def test_create_client_secret_rejects_an_invalid_session_before_http + http_client = StubHTTPClient.new + client = OpenAI::Client.new(api_key: "test-key", http_client: http_client) + + error = assert_raises(ArgumentError) do + client.realtime.translations.client_secrets.create(session: {audio: {}}) + end + + assert_includes(error.message, "missing required fields") + assert_empty(http_client.requests) + end + + def test_create_translation_call + http_client = StubHTTPClient.new( + OpenAI::HTTPClient::Response.new( + status: 201, + headers: { + "content-type" => "application/sdp", + "location" => "/v1/realtime/translations/calls/rtc_translation" + }, + body: "answer-sdp" + ) + ) + client = OpenAI::Client.new(api_key: "ek_test", http_client: http_client) + + response = client.realtime.translations.calls.create(sdp: "offer-sdp") + + assert_equal("answer-sdp", response.sdp) + assert_equal("rtc_translation", response.call_id) + request = http_client.requests.fetch(0) + assert_equal("/v1/realtime/translations/calls", request.url.path) + assert_equal("application/sdp", request.headers.fetch("content-type")) + assert_equal("offer-sdp", request.body) + end + + def test_connect_translation_session_with_typed_events_and_helpers + socket = FakeSocket.new( + JSON.generate( + type: "session.output_transcript.delta", + event_id: "event_1", + delta: "hola", + item_id: "item_1" + ) + ) + transport = FakeTransport.new(socket) + client = OpenAI::Client.new(api_key: "test-key") + event = nil + + client.realtime.translations.connect( + model: "gpt-realtime-translate", + transport: transport + ) do |connection| + assert_instance_of(OpenAI::Realtime::TranslationConnection, connection) + refute_respond_to(connection, :response) + refute_respond_to(connection, :conversation) + refute_respond_to(connection, :output_audio_buffer) + assert_respond_to(connection.session, :close) + refute_respond_to(connection.input_audio_buffer, :commit) + refute_respond_to(connection.input_audio_buffer, :clear) + connection.session.update(audio: {output: {language: "es"}}) + connection.input_audio_buffer.append_bytes("\x00\x01".b) + connection.session.close + event = connection.receive + end + + assert_equal( + "wss://api.openai.com/v1/realtime/translations?model=gpt-realtime-translate", + transport.url.to_s + ) + assert_instance_of(OpenAI::Realtime::RealtimeTranslationOutputTranscriptDeltaEvent, event) + assert_equal("hola", event.delta) + assert_equal( + [ + "session.update", + "session.input_audio_buffer.append", + "session.close" + ], + socket.writes.map { JSON.parse(_1).fetch("type") } + ) + end + + def test_connect_requires_a_block + client = OpenAI::Client.new(api_key: "test-key") + + error = assert_raises(ArgumentError) do + client.realtime.translations.connect(model: "gpt-realtime-translate") + end + + assert_includes(error.message, "block is required") + end +end From 072c1824fb23475782f372b13f0619c0c8e6678d Mon Sep 17 00:00:00 2001 From: Justin Beckwith Date: Tue, 11 Aug 2026 16:38:01 -0700 Subject: [PATCH 2/8] fix: address Realtime review feedback --- README.md | 63 +----- examples/realtime/README.md | 11 +- examples/realtime/realtime_conversation.rb | 79 ++++++- examples/realtime/sip.rb | 12 + examples/realtime/websocket_text.rb | 115 ++++++---- .../realtime/transports/async_websocket.rb | 2 + lib/openai/resources/realtime/calls.rb | 2 +- .../resources/realtime/translations/calls.rb | 17 +- realtime.md | 6 +- .../async_websocket_transport_test.rb | 54 +++++ test/openai/realtime/examples_test.rb | 205 ++++++++++++++++-- test/openai/resources/realtime/calls_test.rb | 2 + .../resources/realtime/translations_test.rb | 7 +- 13 files changed, 419 insertions(+), 156 deletions(-) diff --git a/README.md b/README.md index 1e01cda80..233b1771d 100644 --- a/README.md +++ b/README.md @@ -53,71 +53,18 @@ end ### Realtime The SDK supports server-side Realtime WebSockets, WebRTC session negotiation, -sideband connections for WebRTC and SIP calls, and continuous translation. Add -the WebSocket adapter alongside the SDK: +sideband control, SIP, transcription, and translation. WebSocket connections use +the optional `async-websocket` adapter. -```ruby -gem "openai" -gem "async-websocket" -``` - -Then open a block-scoped connection. Events are decoded into the existing typed -Realtime models, and the socket is closed when the block exits: - -```ruby -client.realtime.connect(model: "gpt-realtime-2.1") do |connection| - connection.session.update( - type: :realtime, - output_modalities: [:text] - ) - connection.conversation.items.create( - type: :message, - role: :user, - content: [{type: :input_text, text: "Hello from Ruby"}] - ) - connection.response.create - - connection.each do |event| - case event - when OpenAI::Realtime::ResponseTextDeltaEvent - print(event.delta) - when OpenAI::Realtime::ResponseDoneEvent - raise "Response ended with #{event.response.status}" unless event.response.status == :completed - break - when OpenAI::Realtime::RealtimeErrorEvent - raise event.error.message - end - end -end -``` - -Run the live text smoke test with `OPENAI_API_KEY` set: +Run a typed text smoke test, or start the natural hands-free browser voice demo: ```console $ bundle exec ruby examples/realtime/websocket_text.rb -``` - -For live speech-to-text, provide raw mono 24 kHz PCM16 input. The example opens -the dedicated `intent: :transcription` connection: - -```console -$ REALTIME_INPUT_PCM=input.pcm bundle exec ruby examples/realtime/websocket_transcription.rb -``` - -For a natural hands-free voice conversation, run the Ruby WebRTC negotiation -endpoint and open the printed localhost URL in a browser: - -```console $ bundle exec ruby examples/realtime/webrtc_conversation.rb ``` -The API key and session configuration stay in Ruby. The browser owns the -microphone, remote audio, acoustic echo cancellation, and WebRTC peer. - -See [realtime.md](realtime.md) for WebRTC, audio, transcription, translation, -SIP, sideband, MCP, lifecycle, and custom transport guidance. The -[Realtime example runbook](examples/realtime/README.md) lists every executable -example, its prerequisites, and its observable success criteria. +See [the Realtime guide](realtime.md) for the typed API and architecture, and +[the example runbook](examples/realtime/README.md) for every runnable workflow. ### Pagination diff --git a/examples/realtime/README.md b/examples/realtime/README.md index edd013131..39bc7938d 100644 --- a/examples/realtime/README.md +++ b/examples/realtime/README.md @@ -40,7 +40,9 @@ bundle exec ruby examples/realtime/realtime_conversation.rb Start speaking after `Connected` appears. Speak again whenever you want to interrupt: the service cancels the response and the example immediately clears local playback, drops already-queued audio deltas, and truncates the unheard -audio from conversation history. Press Control-C to end the session. +audio from conversation history. A bounded single-writer queue serializes the +microphone and truncation events sent over the socket. Press Control-C to end +the session. On macOS, the default microphone is AVFoundation audio device `0`. List devices and select a different index if necessary: @@ -75,7 +77,8 @@ bundle exec ruby examples/realtime/websocket_text.rb ``` A successful run prints `session.created`, `session.updated`, streamed assistant -text, and `response.done status=completed`. +text, and `response.done status=completed`. An EOF before the completed +`response.done` is reported as a failed smoke test. ## WebSocket audio @@ -186,7 +189,9 @@ bundle exec ruby examples/realtime/sip.rb The script accepts the call, attaches a sideband WebSocket, and prints the audio transcript. `OPENAI_REALTIME_STOP_AFTER` and `OPENAI_REALTIME_TIMEOUT` provide -bounded smoke-test controls. Without a carrier-originated incoming call, the +bounded smoke-test controls. Once accepted, the call is hung up during cleanup, +including after a timeout or sideband failure; an already-ended call is treated +as successfully cleaned up. Without a carrier-originated incoming call, the same accept-then-attach orchestration is covered by the local example and HTTP resource tests, but that is not a substitute for the final telephony smoke test. diff --git a/examples/realtime/realtime_conversation.rb b/examples/realtime/realtime_conversation.rb index 4d2e782f9..4eee825b8 100755 --- a/examples/realtime/realtime_conversation.rb +++ b/examples/realtime/realtime_conversation.rb @@ -194,6 +194,40 @@ def close end end + # Serializes every client event through one writer fiber. The one-item queue + # preserves microphone backpressure while allowing the reader to enqueue an + # interruption without writing to the socket itself. + class OutboundWriter + def initialize(connection) + @connection = connection + @queue = Thread::SizedQueue.new(1) + end + + def append_audio(bytes) + @queue.push([:append_audio, bytes]) + end + + def truncate(**params) + @queue.push([:truncate, params]) + end + + def run + while (message = @queue.pop) + operation, payload = message + case operation + when :append_audio + @connection.input_audio_buffer.append_bytes(payload) + when :truncate + @connection.conversation.items.truncate(**payload) + end + end + end + + def close + @queue.close unless @queue.closed? + end + end + class AudioPlayback def initialize(speaker, clock: nil) @speaker = speaker @@ -211,7 +245,8 @@ def write(event) @received_bytes += bytes.bytesize end - def interrupt(connection) + def interrupt(outbound) + expire_finished_playback unless @item_id @speaker.interrupt return @@ -223,7 +258,7 @@ def interrupt(connection) content_index = @content_index @speaker.interrupt - connection.conversation.items.truncate( + outbound.truncate( item_id: item_id, content_index: content_index, audio_end_ms: audio_end_ms @@ -236,7 +271,12 @@ def interrupt(connection) def interrupted?(response_id) = response_id == @interrupted_response_id def finish(response_id) - @interrupted_response_id = nil if interrupted?(response_id) + if interrupted?(response_id) + @interrupted_response_id = nil + elsif response_id == @response_id + @finished_response_id = response_id + expire_finished_playback + end end def close = @speaker.close @@ -261,12 +301,24 @@ def close = @speaker.close [elapsed_ms, received_ms].min.clamp(0, received_ms) end + private def expire_finished_playback + return unless @finished_response_id == @response_id + return unless elapsed_audio_bytes >= @received_bytes + + reset_current + end + + private def elapsed_audio_bytes + ((@clock.call - @started_at) * AUDIO_BYTES_PER_SECOND).floor + end + private def reset_current @response_id = nil @item_id = nil @content_index = nil @received_bytes = 0 @started_at = nil + @finished_response_id = nil end end @@ -298,13 +350,13 @@ def configure(connection, voice:, instructions:) ) end - def forward_microphone(connection, microphone) + def forward_microphone(outbound, microphone) microphone.each_chunk do |chunk| - connection.input_audio_buffer.append_bytes(chunk) + outbound.append_audio(chunk) end end - def handle_event(event, connection:, playback:, output:) + def handle_event(event, outbound:, playback:, output:) case event when OpenAI::Realtime::ResponseAudioDeltaEvent playback.write(event) @@ -318,7 +370,7 @@ def handle_event(event, connection:, playback:, output:) output.puts when OpenAI::Realtime::InputAudioBufferSpeechStartedEvent - output.puts if playback.interrupt(connection) + output.puts if playback.interrupt(outbound) when OpenAI::Realtime::RealtimeErrorEvent raise event.error.message when OpenAI::Realtime::ResponseDoneEvent @@ -337,9 +389,9 @@ def wait_until_ready(connection) raise "Realtime connection closed before session.updated" end - def stream_events(connection, microphone:, playback:, output:, stop_after: nil) + def stream_events(connection, microphone:, outbound:, playback:, output:, stop_after: nil) connection.each do |event| - handle_event(event, connection: connection, playback: playback, output: output) + handle_event(event, outbound: outbound, playback: playback, output: output) break if stop_after == event.type.to_s end ensure @@ -362,24 +414,31 @@ def run_session( "Press Ctrl-C to exit. Use headphones to prevent echo." ) playback = AudioPlayback.new(speaker) + outbound = OutboundWriter.new(connection) + writer = Async { outbound.run } receiver = Async do stream_events( connection, microphone: microphone, + outbound: outbound, playback: playback, output: output, stop_after: stop_after ) end - sender = Async { forward_microphone(connection, microphone) } + sender = Async { forward_microphone(outbound, microphone) } sender.wait + outbound.close + writer.wait receiver.wait ensure microphone.stop + outbound&.close playback&.close || speaker.close sender&.stop receiver&.stop + writer&.stop end def run( diff --git a/examples/realtime/sip.rb b/examples/realtime/sip.rb index 30a99cc57..0a3bb6913 100755 --- a/examples/realtime/sip.rb +++ b/examples/realtime/sip.rb @@ -24,16 +24,28 @@ def stream(connection, output: $stdout, stop_after: nil) end def run(client:, call_id:, model:, output: $stdout, stop_after: nil) + accepted = false client.realtime.calls.accept( call_id, type: :realtime, model: model, instructions: "You are answering a phone call. Be warm and concise." ) + accepted = true client.realtime.connect(call_id: call_id) do |connection| stream(connection, output: output, stop_after: stop_after) end + ensure + hangup(client, call_id, active_error: $ERROR_INFO) if accepted + end + + def hangup(client, call_id, active_error:) + client.realtime.calls.hangup(call_id) + rescue OpenAI::Errors::NotFoundError + nil + rescue StandardError + raise unless active_error end end end diff --git a/examples/realtime/websocket_text.rb b/examples/realtime/websocket_text.rb index 3e8549a84..5317dbd08 100755 --- a/examples/realtime/websocket_text.rb +++ b/examples/realtime/websocket_text.rb @@ -4,57 +4,76 @@ require "timeout" require_relative "../../lib/openai" -def stream_response(connection) - started_response = false - connection.each do |event| - case event - when OpenAI::Realtime::SessionCreatedEvent - puts("[realtime] session.created") - when OpenAI::Realtime::SessionUpdatedEvent - puts("[realtime] session.updated") - when OpenAI::Realtime::ResponseTextDeltaEvent - print("[assistant] ") unless started_response - started_response = true - print(event.delta) - $stdout.flush - when OpenAI::Realtime::ResponseDoneEvent - puts if started_response - status = event.response.status - raise "Realtime response ended with status #{status.inspect}" unless status == :completed - - puts("[realtime] response.done status=completed") - break - when OpenAI::Realtime::RealtimeErrorEvent - raise "Realtime API error: #{event.error.message}" +module OpenAI + module Examples + module Realtime + module WebSocketText + module_function + + def stream_response(connection, output: $stdout) + started_response = false + completed_response = false + connection.each do |event| + case event + when OpenAI::Realtime::SessionCreatedEvent + output.puts("[realtime] session.created") + when OpenAI::Realtime::SessionUpdatedEvent + output.puts("[realtime] session.updated") + when OpenAI::Realtime::ResponseTextDeltaEvent + output.print("[assistant] ") unless started_response + started_response = true + output.print(event.delta) + output.flush + when OpenAI::Realtime::ResponseDoneEvent + output.puts if started_response + status = event.response.status + raise "Realtime response ended with status #{status.inspect}" unless status == :completed + + output.puts("[realtime] response.done status=completed") + completed_response = true + break + when OpenAI::Realtime::RealtimeErrorEvent + raise "Realtime API error: #{event.error.message}" + end + end + return if completed_response + + raise "Realtime connection closed before response.done" + end + + def run(client:, model:, prompt:, output: $stdout) + session = { + type: :realtime, + output_modalities: [:text], + instructions: "Be concise and friendly." + } + item = { + type: :message, + role: :user, + content: [{type: :input_text, text: prompt}] + } + + output.puts("[realtime] connecting with #{model}") + client.realtime.connect(model: model) do |connection| + output.puts("[realtime] connected; sending prompt: #{prompt.inspect}") + connection.session.update(**session) + connection.conversation.items.create(**item) + connection.response.create + stream_response(connection, output: output) + end + output.puts("[realtime] smoke test passed") + end + end end end end -client = OpenAI::Client.new -model = ENV.fetch("OPENAI_REALTIME_MODEL", "gpt-realtime-2.1") -prompt = ENV.fetch("OPENAI_REALTIME_PROMPT", "Say hello from Ruby.") -timeout = Integer(ENV.fetch("OPENAI_REALTIME_TIMEOUT", "30")) -session = { - type: :realtime, - output_modalities: [:text], - instructions: "Be concise and friendly." -} -item = { - type: :message, - role: :user, - content: [{type: :input_text, text: prompt}] -} - -puts("[realtime] connecting with #{model}") - -Timeout.timeout(timeout) do - client.realtime.connect(model: model) do |connection| - puts("[realtime] connected; sending prompt: #{prompt.inspect}") - connection.session.update(**session) - connection.conversation.items.create(**item) - connection.response.create - stream_response(connection) +if $PROGRAM_NAME == __FILE__ + Timeout.timeout(Integer(ENV.fetch("OPENAI_REALTIME_TIMEOUT", "30"))) do + OpenAI::Examples::Realtime::WebSocketText.run( + client: OpenAI::Client.new, + model: ENV.fetch("OPENAI_REALTIME_MODEL", "gpt-realtime-2.1"), + prompt: ENV.fetch("OPENAI_REALTIME_PROMPT", "Say hello from Ruby.") + ) end end - -puts("[realtime] smoke test passed") diff --git a/lib/openai/realtime/transports/async_websocket.rb b/lib/openai/realtime/transports/async_websocket.rb index 29dbea4f5..15c5d3fcd 100644 --- a/lib/openai/realtime/transports/async_websocket.rb +++ b/lib/openai/realtime/transports/async_websocket.rb @@ -49,6 +49,8 @@ def open(url:, headers:, timeout:, **endpoint_options) **options ) block_error = nil + # Client.connect owns the Sync boundary: it starts a reactor for ordinary + # synchronous callers and reuses the current task when one already exists. ::Async::WebSocket::Client.connect(endpoint, headers: headers) do |connection| yield(Socket.new(connection, url: url)) rescue StandardError => e diff --git a/lib/openai/resources/realtime/calls.rb b/lib/openai/resources/realtime/calls.rb index 81c1a1b4f..4d02f9790 100644 --- a/lib/openai/resources/realtime/calls.rb +++ b/lib/openai/resources/realtime/calls.rb @@ -47,7 +47,7 @@ def create(params) sdp: response.body.to_a.join, call_id: call_id, headers: response.headers - ) + )._set_last_response(response.metadata) end # Some parameter documentations has been truncated, see diff --git a/lib/openai/resources/realtime/translations/calls.rb b/lib/openai/resources/realtime/translations/calls.rb index 2dd04a824..132db5d0f 100644 --- a/lib/openai/resources/realtime/translations/calls.rb +++ b/lib/openai/resources/realtime/translations/calls.rb @@ -11,20 +11,9 @@ class Calls # @param request_options [OpenAI::RequestOptions, Hash{Symbol=>Object}, nil] # @return [OpenAI::Models::Realtime::CallCreateResponse] def create(sdp:, request_options: nil) - response = @client.request_raw( - method: :post, - path: "realtime/translations/calls", - headers: {"accept" => "application/sdp", "content-type" => "application/sdp"}, - body: String(sdp), - security: {bearer_auth: true}, - options: request_options - ) - location = response.headers["location"] - call_id = location&.then { URI.parse(_1).path.split("/").last } - OpenAI::Realtime::CallCreateResponse.new( - sdp: response.body.to_a.join, - call_id: call_id, - headers: response.headers + @client.realtime.calls.create( + sdp: sdp, + request_options: request_options ) end diff --git a/realtime.md b/realtime.md index 1d92e130e..98079461e 100644 --- a/realtime.md +++ b/realtime.md @@ -95,6 +95,8 @@ at a time, so the caller naturally applies backpressure instead of filling an unbounded SDK queue. A connection supports one reader fiber and one writer fiber at the same time, which is useful for continuous audio. Do not run multiple readers or multiple writers against the same connection. +The hands-free WebSocket example uses a bounded outbound queue so microphone +audio and interruption-driven truncation events share exactly one writer fiber. The yielded class reflects the protocol's capabilities: @@ -262,7 +264,9 @@ Call `session.close` after the last audio chunk and continue reading until the server sends `session.closed`; closing the socket immediately can discard output that is still draining. For browser WebRTC translation, mint an ephemeral secret with `translations.client_secrets.create`, then post the browser offer with -`translations.calls.create`. +`translations.calls.create`. That convenience method delegates to the shared +`/v1/realtime/calls` endpoint; translation is selected by the ephemeral secret, +not by a separate calls route. ## Function calls and MCP approvals diff --git a/test/openai/realtime/async_websocket_transport_test.rb b/test/openai/realtime/async_websocket_transport_test.rb index 8264f9bc6..15bb58ce2 100644 --- a/test/openai/realtime/async_websocket_transport_test.rb +++ b/test/openai/realtime/async_websocket_transport_test.rb @@ -3,6 +3,7 @@ require "async/http/server" require "async/queue" require "async/websocket/adapters/http" +require "async/websocket/client" require "async/websocket/server" require "socket" @@ -11,6 +12,59 @@ class OpenAI::Test::AsyncWebSocketTransportTest < Minitest::Test extend Minitest::Serial + class SynchronousConnection + attr_reader :scheduler + + def initialize + @closed = false + @scheduler = nil + end + + def connect! + @scheduler = Fiber.scheduler + self + end + + def read = "message" + def write(_message) = nil + def flush = nil + def closed? = @closed + def close(*) = (@closed = true) + end + + class SynchronousClient + attr_reader :closed + + def initialize(connection) + @connection = connection + @closed = false + end + + def connect(*) = @connection.connect! + def close = (@closed = true) + end + + def test_default_transport_enters_a_reactor_for_synchronous_callers + assert_nil(Fiber.scheduler) + connection = SynchronousConnection.new + client = SynchronousClient.new(connection) + transport = OpenAI::Realtime::Transports::AsyncWebSocket.new + url = URI("wss://example.com/v1/realtime?model=gpt-realtime-2.1") + + result = Async::WebSocket::Client.stub(:open, client) do + transport.open(url: url, headers: {}, timeout: 1) do |socket| + [socket.read, Fiber.scheduler] + end + end + + assert_equal("message", result.fetch(0)) + assert_same(connection.scheduler, result.fetch(1)) + refute_nil(connection.scheduler) + assert_predicate(connection, :closed?) + assert_predicate(client, :closed) + assert_nil(Fiber.scheduler) + end + def test_default_transport_exchanges_events_with_a_local_websocket_server port = available_port endpoint = Async::HTTP::Endpoint.parse("http://127.0.0.1:#{port}") diff --git a/test/openai/realtime/examples_test.rb b/test/openai/realtime/examples_test.rb index efa58cecc..8932ff8ba 100644 --- a/test/openai/realtime/examples_test.rb +++ b/test/openai/realtime/examples_test.rb @@ -7,6 +7,7 @@ require_relative "../../../examples/realtime/sideband" require_relative "../../../examples/realtime/sip" require_relative "../../../examples/realtime/webrtc_conversation" +require_relative "../../../examples/realtime/websocket_text" class OpenAI::Test::RealtimeExamplesTest < Minitest::Test Event = Data.define(:type, :data) do @@ -28,9 +29,10 @@ def update(**params) class RecordingResource attr_reader :calls, :truncations - def initialize + def initialize(writer_fibers = nil) @calls = [] @truncations = [] + @writer_fibers = writer_fibers end def create(**params) @@ -38,6 +40,7 @@ def create(**params) end def truncate(**params) + @writer_fibers&.push(Fiber.current) @truncations << params end end @@ -45,31 +48,46 @@ def truncate(**params) class RecordingConversation attr_reader :items - def initialize - @items = RecordingResource.new + def initialize(writer_fibers = nil) + @items = RecordingResource.new(writer_fibers) end end class RecordingAudioBuffer attr_reader :chunks - def initialize + def initialize(writer_fibers = nil) @chunks = [] + @writer_fibers = writer_fibers end def append_bytes(bytes) + @writer_fibers&.push(Fiber.current) @chunks << bytes end end + class RecordingOutbound + attr_reader :chunks, :truncations + + def initialize + @chunks = [] + @truncations = [] + end + + def append_audio(bytes) = @chunks << bytes + def truncate(**params) = @truncations << params + end + class RecordingConnection - attr_reader :conversation, :input_audio_buffer, :response, :session + attr_reader :conversation, :input_audio_buffer, :response, :session, :writer_fibers def initialize(events = []) @events = events + @writer_fibers = [] @session = RecordingSession.new - @conversation = RecordingConversation.new - @input_audio_buffer = RecordingAudioBuffer.new + @conversation = RecordingConversation.new(@writer_fibers) + @input_audio_buffer = RecordingAudioBuffer.new(@writer_fibers) @response = RecordingResource.new end @@ -78,16 +96,29 @@ def each(&block) end end + class ExplodingConnection < RecordingConnection + def each + raise "sideband failed" + end + end + class RecordingCalls - attr_reader :accepts + attr_accessor :hangup_error + attr_reader :accepts, :hangups def initialize @accepts = [] + @hangups = [] end def accept(call_id, **params) @accepts << [call_id, params] end + + def hangup(call_id) + @hangups << call_id + raise @hangup_error if @hangup_error + end end class RecordingRealtime @@ -215,24 +246,44 @@ def test_realtime_conversation_configures_continuous_server_vad end def test_realtime_conversation_streams_microphone_chunks_without_committing - connection = RecordingConnection.new + outbound = RecordingOutbound.new microphone = RecordingMicrophone.new(["first".b, "second".b]) - OpenAI::Examples::Realtime::Conversation.forward_microphone(connection, microphone) + OpenAI::Examples::Realtime::Conversation.forward_microphone(outbound, microphone) + + assert_equal(["first".b, "second".b], outbound.chunks) + end + + def test_realtime_conversation_serializes_audio_and_interruptions_through_one_writer + connection = RecordingConnection.new + outbound = OpenAI::Examples::Realtime::Conversation::OutboundWriter.new(connection) + + Sync do |task| + writer = task.async { outbound.run } + outbound.append_audio("first".b) + outbound.truncate(item_id: "item_1", content_index: 0, audio_end_ms: 100) + outbound.append_audio("second".b) + outbound.close + writer.wait + end assert_equal(["first".b, "second".b], connection.input_audio_buffer.chunks) - assert_empty(connection.response.calls) + assert_equal( + [{item_id: "item_1", content_index: 0, audio_end_ms: 100}], + connection.conversation.items.truncations + ) + assert_equal(1, connection.writer_fibers.uniq.size) end def test_realtime_conversation_streams_audio_and_interrupts_local_playback speaker = RecordingSpeaker.new - connection = RecordingConnection.new times = [0.0, 0.1].each playback = OpenAI::Examples::Realtime::Conversation::AudioPlayback.new( speaker, clock: -> { times.next } ) output = StringIO.new + outbound = RecordingOutbound.new audio = "\x00\x01".b * 2_400 event_fields = { event_id: "event_1", @@ -244,13 +295,13 @@ def test_realtime_conversation_streams_audio_and_interrupts_local_playback OpenAI::Examples::Realtime::Conversation.handle_event( OpenAI::Realtime::ResponseAudioDeltaEvent.new(**event_fields, delta: Base64.strict_encode64(audio)), - connection: connection, + outbound: outbound, playback: playback, output: output ) OpenAI::Examples::Realtime::Conversation.handle_event( OpenAI::Realtime::ResponseAudioTranscriptDeltaEvent.new(**event_fields, delta: "Hello"), - connection: connection, + outbound: outbound, playback: playback, output: output ) @@ -260,7 +311,7 @@ def test_realtime_conversation_streams_audio_and_interrupts_local_playback item_id: "item_2", audio_start_ms: 100 ), - connection: connection, + outbound: outbound, playback: playback, output: output ) @@ -270,19 +321,19 @@ def test_realtime_conversation_streams_audio_and_interrupts_local_playback assert_equal(1, speaker.interruptions) assert_equal( [{item_id: "item_1", content_index: 0, audio_end_ms: 100}], - connection.conversation.items.truncations + outbound.truncations ) end def test_realtime_conversation_drops_queued_audio_after_interruption speaker = RecordingSpeaker.new - connection = RecordingConnection.new times = [0.0, 0.05].each playback = OpenAI::Examples::Realtime::Conversation::AudioPlayback.new( speaker, clock: -> { times.next } ) output = StringIO.new + outbound = RecordingOutbound.new event_fields = { event_id: "event_1", response_id: "response_1", @@ -305,7 +356,7 @@ def test_realtime_conversation_drops_queued_audio_after_interruption item_id: "item_2", audio_start_ms: 50 ), - connection: connection, + outbound: outbound, playback: playback, output: output ) @@ -321,7 +372,7 @@ def test_realtime_conversation_drops_queued_audio_after_interruption event_id: "event_4", response: OpenAI::Realtime::RealtimeResponse.new(id: "response_1", status: :cancelled) ), - connection: connection, + outbound: outbound, playback: playback, output: output ) @@ -331,6 +382,36 @@ def test_realtime_conversation_drops_queued_audio_after_interruption assert_equal(1, speaker.interruptions) end + def test_realtime_conversation_does_not_truncate_fully_played_completed_audio + speaker = RecordingSpeaker.new + outbound = RecordingOutbound.new + times = [0.0, 0.05, 0.2].each + playback = OpenAI::Examples::Realtime::Conversation::AudioPlayback.new( + speaker, + clock: -> { times.next } + ) + fields = { + event_id: "event_1", + response_id: "response_1", + item_id: "item_1", + output_index: 0, + content_index: 0 + } + audio = "\x00\x01".b * 2_400 + + playback.write( + OpenAI::Realtime::ResponseAudioDeltaEvent.new( + **fields, + delta: Base64.strict_encode64(audio) + ) + ) + playback.finish("response_1") + playback.interrupt(outbound) + + assert_empty(outbound.truncations) + assert_equal(1, speaker.interruptions) + end + def test_realtime_conversation_can_capture_output_audio_for_a_live_smoke_test Tempfile.create(["realtime-conversation", ".pcm"]) do |file| speaker = OpenAI::Examples::Realtime::Conversation::PCMFileSpeaker.new(file.path) @@ -652,5 +733,91 @@ def test_sip_example_accepts_then_attaches_to_the_call ], realtime.calls.accepts.fetch(0) ) + assert_equal(["rtc_123"], realtime.calls.hangups) + end + + def test_sip_example_hangs_up_without_masking_a_sideband_error + realtime = RecordingRealtime.new(ExplodingConnection.new) + realtime.calls.hangup_error = RuntimeError.new("hangup failed") + + error = assert_raises(RuntimeError) do + OpenAI::Examples::Realtime::SIP.run( + client: RecordingClient.new(realtime: realtime), + call_id: "rtc_123", + model: "gpt-realtime-2.1", + output: StringIO.new + ) + end + + assert_equal("sideband failed", error.message) + assert_equal(["rtc_123"], realtime.calls.hangups) + end + + def test_sip_example_tolerates_an_already_ended_call_during_cleanup + realtime = RecordingRealtime.new(RecordingConnection.new) + realtime.calls.hangup_error = OpenAI::Errors::NotFoundError.new( + url: URI("https://api.openai.com/v1/realtime/calls/rtc_123/hangup"), + status: 404, + headers: {}, + body: {}, + request: nil, + response: nil + ) + + OpenAI::Examples::Realtime::SIP.run( + client: RecordingClient.new(realtime: realtime), + call_id: "rtc_123", + model: "gpt-realtime-2.1", + output: StringIO.new + ) + + assert_equal(["rtc_123"], realtime.calls.hangups) + end + + def test_sip_example_reports_a_cleanup_failure_after_a_clean_session + realtime = RecordingRealtime.new(RecordingConnection.new) + realtime.calls.hangup_error = RuntimeError.new("hangup failed") + + error = assert_raises(RuntimeError) do + OpenAI::Examples::Realtime::SIP.run( + client: RecordingClient.new(realtime: realtime), + call_id: "rtc_123", + model: "gpt-realtime-2.1", + output: StringIO.new + ) + end + + assert_equal("hangup failed", error.message) + assert_equal(["rtc_123"], realtime.calls.hangups) + end + + def test_websocket_text_rejects_a_connection_that_closes_before_response_done + error = assert_raises(RuntimeError) do + OpenAI::Examples::Realtime::WebSocketText.stream_response( + RecordingConnection.new, + output: StringIO.new + ) + end + + assert_equal("Realtime connection closed before response.done", error.message) + end + + def test_websocket_text_accepts_only_a_completed_response + connection = RecordingConnection.new( + [ + OpenAI::Realtime::ResponseDoneEvent.new( + event_id: "event_1", + response: OpenAI::Realtime::RealtimeResponse.new( + id: "response_1", + status: :completed + ) + ) + ] + ) + output = StringIO.new + + OpenAI::Examples::Realtime::WebSocketText.stream_response(connection, output: output) + + assert_includes(output.string, "response.done status=completed") end end diff --git a/test/openai/resources/realtime/calls_test.rb b/test/openai/resources/realtime/calls_test.rb index a8d2e136e..724cb3d43 100644 --- a/test/openai/resources/realtime/calls_test.rb +++ b/test/openai/resources/realtime/calls_test.rb @@ -43,6 +43,8 @@ def test_create_with_an_sdp_offer assert_equal("answer-sdp", response.sdp) assert_equal("rtc_123", response.call_id) + assert_equal("req_realtime_call", response._request_id) + assert_equal(201, response.last_response.status) assert_equal("/v1/realtime/calls/rtc_123", response.headers.fetch("location")) assert_equal(:post, http_client.request.method) assert_equal("https://example.com/v1/realtime/calls", http_client.request.url.to_s) diff --git a/test/openai/resources/realtime/translations_test.rb b/test/openai/resources/realtime/translations_test.rb index 86055ec85..56399536d 100644 --- a/test/openai/resources/realtime/translations_test.rb +++ b/test/openai/resources/realtime/translations_test.rb @@ -106,7 +106,8 @@ def test_create_translation_call status: 201, headers: { "content-type" => "application/sdp", - "location" => "/v1/realtime/translations/calls/rtc_translation" + "location" => "/v1/realtime/calls/rtc_translation", + "x-request-id" => "req_translation_call" }, body: "answer-sdp" ) @@ -117,8 +118,10 @@ def test_create_translation_call assert_equal("answer-sdp", response.sdp) assert_equal("rtc_translation", response.call_id) + assert_equal("req_translation_call", response._request_id) + assert_equal(201, response.last_response.status) request = http_client.requests.fetch(0) - assert_equal("/v1/realtime/translations/calls", request.url.path) + assert_equal("/v1/realtime/calls", request.url.path) assert_equal("application/sdp", request.headers.fetch("content-type")) assert_equal("offer-sdp", request.body) end From 987ede3b598813fc6e6ac07d5293006c2f395a9c Mon Sep 17 00:00:00 2001 From: Justin Beckwith Date: Tue, 11 Aug 2026 17:31:30 -0700 Subject: [PATCH 3/8] fix: harden Realtime example lifecycles --- examples/realtime/README.md | 9 +- examples/realtime/realtime_conversation.rb | 32 +++++- examples/realtime/translation.rb | 101 +++++++++++------- examples/realtime/websocket_audio.rb | 75 ++++++++----- .../realtime/example_stream_lifecycle_test.rb | 84 +++++++++++++++ test/openai/realtime/examples_test.rb | 75 +++++++++++++ 6 files changed, 307 insertions(+), 69 deletions(-) create mode 100644 test/openai/realtime/example_stream_lifecycle_test.rb diff --git a/examples/realtime/README.md b/examples/realtime/README.md index 39bc7938d..9fad6e315 100644 --- a/examples/realtime/README.md +++ b/examples/realtime/README.md @@ -42,7 +42,8 @@ interrupt: the service cancels the response and the example immediately clears local playback, drops already-queued audio deltas, and truncates the unheard audio from conversation history. A bounded single-writer queue serializes the microphone and truncation events sent over the socket. Press Control-C to end -the session. +the session. If that writer fails, the example stops microphone capture, +unblocks any queued producer, and reports the original connection error. On macOS, the default microphone is AVFoundation audio device `0`. List devices and select a different index if necessary: @@ -99,7 +100,8 @@ bundle exec ruby examples/realtime/websocket_audio.rb ffplay -f s16le -ar 24000 -ch_layout mono response.pcm ``` -A successful run exits with status 0 and writes a non-empty output file. +A successful run exits with status 0 and writes a non-empty output file. An EOF +before a completed `response.done` is reported as a failed smoke test. ## Realtime transcription @@ -130,7 +132,8 @@ bundle exec ruby examples/realtime/translation.rb ``` A successful run prints the translated transcript, exits after -`session.closed`, and writes non-empty translated PCM audio. +`session.closed`, and writes non-empty translated PCM audio. An EOF before +`session.closed` is reported as incomplete instead of silently succeeding. ## MCP approval diff --git a/examples/realtime/realtime_conversation.rb b/examples/realtime/realtime_conversation.rb index 4eee825b8..99e77a832 100755 --- a/examples/realtime/realtime_conversation.rb +++ b/examples/realtime/realtime_conversation.rb @@ -201,14 +201,15 @@ class OutboundWriter def initialize(connection) @connection = connection @queue = Thread::SizedQueue.new(1) + @failure = nil end def append_audio(bytes) - @queue.push([:append_audio, bytes]) + enqueue([:append_audio, bytes]) end def truncate(**params) - @queue.push([:truncate, params]) + enqueue([:truncate, params]) end def run @@ -221,11 +222,24 @@ def run @connection.conversation.items.truncate(**payload) end end + rescue StandardError => e + @failure = e + raise + ensure + close end def close @queue.close unless @queue.closed? end + + private def enqueue(message) + @queue.push(message) + rescue ClosedQueueError + raise @failure if @failure + + raise + end end class AudioPlayback @@ -302,6 +316,7 @@ def close = @speaker.close end private def expire_finished_playback + return unless @finished_response_id return unless @finished_response_id == @response_id return unless elapsed_audio_bytes >= @received_bytes @@ -416,7 +431,14 @@ def run_session( playback = AudioPlayback.new(speaker) outbound = OutboundWriter.new(connection) - writer = Async { outbound.run } + writer = Async do + outbound.run + nil + rescue StandardError => e + e + ensure + microphone.stop + end receiver = Async do stream_events( connection, @@ -430,7 +452,9 @@ def run_session( sender = Async { forward_microphone(outbound, microphone) } sender.wait outbound.close - writer.wait + writer_error = writer.wait + raise writer_error if writer_error + receiver.wait ensure microphone.stop diff --git a/examples/realtime/translation.rb b/examples/realtime/translation.rb index d295625c7..717fac8a0 100755 --- a/examples/realtime/translation.rb +++ b/examples/realtime/translation.rb @@ -1,49 +1,74 @@ #!/usr/bin/env ruby # frozen_string_literal: true -require_relative "../../lib/openai" require "async" +require_relative "../../lib/openai" -def read_translation(connection, output) - Async do - connection.each do |event| - case event - when OpenAI::Realtime::RealtimeTranslationOutputTranscriptDeltaEvent - print(event.delta) - when OpenAI::Realtime::RealtimeTranslationOutputAudioDeltaEvent - output.write(Base64.strict_decode64(event.delta)) - when OpenAI::Realtime::RealtimeTranslationSessionClosedEvent - puts - break - when OpenAI::Realtime::RealtimeErrorEvent - raise event.error.message - end - end - end -end +module OpenAI + module Examples + module Realtime + module Translation + module_function -def write_translation_input(connection, input_path) - File.open(input_path, "rb") do |input| - while (chunk = input.read(9_600)) - connection.input_audio_buffer.append_bytes(chunk) + def stream(connection, audio_output:, transcript_output: $stdout) + session_closed = false + connection.each do |event| + case event + when OpenAI::Realtime::RealtimeTranslationOutputTranscriptDeltaEvent + transcript_output.print(event.delta) + transcript_output.flush + when OpenAI::Realtime::RealtimeTranslationOutputAudioDeltaEvent + audio_output.write(Base64.strict_decode64(event.delta)) + when OpenAI::Realtime::RealtimeTranslationSessionClosedEvent + transcript_output.puts + session_closed = true + break + when OpenAI::Realtime::RealtimeErrorEvent + raise event.error.message + end + end + return if session_closed + + raise "Realtime translation connection closed before session.closed" + end + + def write_input(connection, input_path) + File.open(input_path, "rb") do |input| + while (chunk = input.read(9_600)) + connection.input_audio_buffer.append_bytes(chunk) + end + end + ensure + connection.session.close + end + + def run(client:, model:, input_path:, output_path:, target_language:, transcript_output: $stdout) + File.open(output_path, "wb") do |audio_output| + client.realtime.translations.connect(model: model) do |connection| + connection.session.update(audio: {output: {language: target_language}}) + reader = Async do + stream( + connection, + audio_output: audio_output, + transcript_output: transcript_output + ) + end + write_input(connection, input_path) + reader.wait + end + end + end + end end end -ensure - connection.session.close end -input_path = ENV.fetch("REALTIME_INPUT_PCM") -output_path = ENV.fetch("REALTIME_OUTPUT_PCM", "translation-output.pcm") -model = ENV.fetch("OPENAI_REALTIME_TRANSLATION_MODEL", "gpt-realtime-translate") -client = OpenAI::Client.new - -File.open(output_path, "wb") do |output| - client.realtime.translations.connect(model: model) do |connection| - connection.session.update( - audio: {output: {language: ENV.fetch("TARGET_LANGUAGE", "es")}} - ) - reader = read_translation(connection, output) - write_translation_input(connection, input_path) - reader.wait - end +if $PROGRAM_NAME == __FILE__ + OpenAI::Examples::Realtime::Translation.run( + client: OpenAI::Client.new, + model: ENV.fetch("OPENAI_REALTIME_TRANSLATION_MODEL", "gpt-realtime-translate"), + input_path: ENV.fetch("REALTIME_INPUT_PCM"), + output_path: ENV.fetch("REALTIME_OUTPUT_PCM", "translation-output.pcm"), + target_language: ENV.fetch("TARGET_LANGUAGE", "es") + ) end diff --git a/examples/realtime/websocket_audio.rb b/examples/realtime/websocket_audio.rb index 2ae255940..38a067364 100755 --- a/examples/realtime/websocket_audio.rb +++ b/examples/realtime/websocket_audio.rb @@ -3,32 +3,59 @@ require_relative "../../lib/openai" -input_path = ENV.fetch("REALTIME_INPUT_PCM") -output_path = ENV.fetch("REALTIME_OUTPUT_PCM", "realtime-output.pcm") -client = OpenAI::Client.new - -File.open(output_path, "wb") do |output| - client.realtime.connect(model: ENV.fetch("OPENAI_REALTIME_MODEL", "gpt-realtime-2.1")) do |connection| - File.open(input_path, "rb") do |input| - while (chunk = input.read(4_800)) - connection.input_audio_buffer.append_bytes(chunk) - end - end - connection.input_audio_buffer.commit - connection.response.create - - connection.each do |event| - case event - when OpenAI::Realtime::ResponseAudioDeltaEvent - output.write(Base64.strict_decode64(event.delta)) - when OpenAI::Realtime::ResponseDoneEvent - raise "Response ended with #{event.response.status}" unless event.response.status == :completed - break - when OpenAI::Realtime::RealtimeErrorEvent - raise event.error.message +module OpenAI + module Examples + module Realtime + module WebSocketAudio + module_function + + def stream_response(connection, output:) + completed_response = false + connection.each do |event| + case event + when OpenAI::Realtime::ResponseAudioDeltaEvent + output.write(Base64.strict_decode64(event.delta)) + when OpenAI::Realtime::ResponseDoneEvent + status = event.response.status + raise "Response ended with #{status}" unless status == :completed + + completed_response = true + break + when OpenAI::Realtime::RealtimeErrorEvent + raise event.error.message + end + end + return if completed_response + + raise "Realtime connection closed before response.done" + end + + def run(client:, model:, input_path:, output_path:) + File.open(output_path, "wb") do |output| + client.realtime.connect(model: model) do |connection| + File.open(input_path, "rb") do |input| + while (chunk = input.read(4_800)) + connection.input_audio_buffer.append_bytes(chunk) + end + end + connection.input_audio_buffer.commit + connection.response.create + stream_response(connection, output: output) + end + end + + warn("Wrote raw PCM16 audio to #{output_path}") + end end end end end -warn("Wrote raw PCM16 audio to #{output_path}") +if $PROGRAM_NAME == __FILE__ + OpenAI::Examples::Realtime::WebSocketAudio.run( + client: OpenAI::Client.new, + model: ENV.fetch("OPENAI_REALTIME_MODEL", "gpt-realtime-2.1"), + input_path: ENV.fetch("REALTIME_INPUT_PCM"), + output_path: ENV.fetch("REALTIME_OUTPUT_PCM", "realtime-output.pcm") + ) +end diff --git a/test/openai/realtime/example_stream_lifecycle_test.rb b/test/openai/realtime/example_stream_lifecycle_test.rb new file mode 100644 index 000000000..76b7351e1 --- /dev/null +++ b/test/openai/realtime/example_stream_lifecycle_test.rb @@ -0,0 +1,84 @@ +# frozen_string_literal: true + +require_relative "../test_helper" +require_relative "../../../examples/realtime/translation" +require_relative "../../../examples/realtime/websocket_audio" + +class OpenAI::Test::RealtimeExampleStreamLifecycleTest < Minitest::Test + class RecordingConnection + def initialize(events = []) + @events = events + end + + def each(&block) + @events.each(&block) + end + end + + def test_websocket_audio_rejects_a_connection_that_closes_before_response_done + error = assert_raises(RuntimeError) do + OpenAI::Examples::Realtime::WebSocketAudio.stream_response( + RecordingConnection.new, + output: StringIO.new + ) + end + + assert_equal("Realtime connection closed before response.done", error.message) + end + + def test_websocket_audio_accepts_only_a_completed_response + connection = RecordingConnection.new( + [ + OpenAI::Realtime::ResponseDoneEvent.new( + event_id: "event_1", + response: OpenAI::Realtime::RealtimeResponse.new( + id: "response_1", + status: :completed + ) + ) + ] + ) + + result = OpenAI::Examples::Realtime::WebSocketAudio.stream_response( + connection, + output: StringIO.new + ) + + assert_nil(result) + end + + def test_translation_rejects_a_connection_that_closes_before_session_closed + error = assert_raises(RuntimeError) do + OpenAI::Examples::Realtime::Translation.stream( + RecordingConnection.new, + audio_output: StringIO.new, + transcript_output: StringIO.new + ) + end + + assert_equal( + "Realtime translation connection closed before session.closed", + error.message + ) + end + + def test_translation_accepts_only_a_session_closed_event + connection = RecordingConnection.new( + [ + OpenAI::Realtime::RealtimeTranslationSessionClosedEvent.new( + event_id: "event_1" + ) + ] + ) + + transcript_output = StringIO.new + result = OpenAI::Examples::Realtime::Translation.stream( + connection, + audio_output: StringIO.new, + transcript_output: transcript_output + ) + + assert_nil(result) + assert_equal("\n", transcript_output.string) + end +end diff --git a/test/openai/realtime/examples_test.rb b/test/openai/realtime/examples_test.rb index 8932ff8ba..2b3084a20 100644 --- a/test/openai/realtime/examples_test.rb +++ b/test/openai/realtime/examples_test.rb @@ -54,6 +54,7 @@ def initialize(writer_fibers = nil) end class RecordingAudioBuffer + attr_accessor :append_error attr_reader :chunks def initialize(writer_fibers = nil) @@ -63,6 +64,8 @@ def initialize(writer_fibers = nil) def append_bytes(bytes) @writer_fibers&.push(Fiber.current) + raise @append_error if @append_error + @chunks << bytes end end @@ -94,6 +97,10 @@ def initialize(events = []) def each(&block) @events.each(&block) end + + def receive + @events.shift + end end class ExplodingConnection < RecordingConnection @@ -180,13 +187,18 @@ def hangup(call_id) end class RecordingMicrophone + attr_reader :stopped + def initialize(chunks) @chunks = chunks + @stopped = false end def each_chunk(&block) @chunks.each(&block) end + + def stop = @stopped = true end class RecordingSpeaker @@ -275,6 +287,69 @@ def test_realtime_conversation_serializes_audio_and_interruptions_through_one_wr assert_equal(1, connection.writer_fibers.uniq.size) end + def test_realtime_conversation_unblocks_a_producer_when_the_writer_fails + connection = RecordingConnection.new + connection.input_audio_buffer.append_error = RuntimeError.new("write failed") + outbound = OpenAI::Examples::Realtime::Conversation::OutboundWriter.new(connection) + + error = Sync do |task| + writer = task.async do + outbound.run + nil + rescue StandardError => e + e + end + outbound.append_audio("first".b) + producer_error = assert_raises(RuntimeError) do + outbound.append_audio("second".b) + end + writer_error = writer.wait + + assert_same(writer_error, producer_error) + producer_error + end + + assert_equal("write failed", error.message) + end + + def test_realtime_conversation_handles_initial_speech_before_assistant_audio + speaker = RecordingSpeaker.new + outbound = RecordingOutbound.new + playback = OpenAI::Examples::Realtime::Conversation::AudioPlayback.new( + speaker, + clock: -> { raise "clock must not be read without playback" } + ) + + playback.interrupt(outbound) + + assert_empty(outbound.truncations) + assert_equal(1, speaker.interruptions) + end + + def test_realtime_conversation_propagates_writer_failure_without_deadlock + connection = RecordingConnection.new( + [OpenAI::Realtime::SessionUpdatedEvent.new(event_id: "event_1", session: {})] + ) + connection.input_audio_buffer.append_error = RuntimeError.new("write failed") + microphone = RecordingMicrophone.new(["first".b, "second".b]) + + error = Sync do + assert_raises(RuntimeError) do + OpenAI::Examples::Realtime::Conversation.run_session( + connection, + microphone: microphone, + speaker: RecordingSpeaker.new, + voice: :marin, + instructions: "Speak naturally.", + output: StringIO.new + ) + end + end + + assert_equal("write failed", error.message) + assert_predicate(microphone, :stopped) + end + def test_realtime_conversation_streams_audio_and_interrupts_local_playback speaker = RecordingSpeaker.new times = [0.0, 0.1].each From b49fe4517aa8894cecaacc9c6862acdcd6ee20f2 Mon Sep 17 00:00:00 2001 From: Justin Beckwith Date: Tue, 11 Aug 2026 17:42:06 -0700 Subject: [PATCH 4/8] fix: close remaining Realtime review gaps --- README.md | 16 +--- examples/realtime/README.md | 11 ++- examples/realtime/mcp_approval.rb | 9 +- examples/realtime/translation.rb | 20 ++++- examples/realtime/websocket_transcription.rb | 5 ++ lib/openai/internal/logging.rb | 47 ++++++++++- lib/openai/internal/transport/base_client.rb | 2 +- rbi/openai/internal/logging.rbi | 37 ++++++++ realtime.md | 18 +++- sig/openai/internal/logging.rbs | 18 ++++ test/openai/logging_test.rb | 84 +++++++++++++++++++ .../realtime/example_stream_lifecycle_test.rb | 77 +++++++++++++++++ test/openai/realtime/examples_test.rb | 29 +++++-- 13 files changed, 338 insertions(+), 35 deletions(-) diff --git a/README.md b/README.md index 233b1771d..5e1ea3f82 100644 --- a/README.md +++ b/README.md @@ -52,19 +52,9 @@ end ### Realtime -The SDK supports server-side Realtime WebSockets, WebRTC session negotiation, -sideband control, SIP, transcription, and translation. WebSocket connections use -the optional `async-websocket` adapter. - -Run a typed text smoke test, or start the natural hands-free browser voice demo: - -```console -$ bundle exec ruby examples/realtime/websocket_text.rb -$ bundle exec ruby examples/realtime/webrtc_conversation.rb -``` - -See [the Realtime guide](realtime.md) for the typed API and architecture, and -[the example runbook](examples/realtime/README.md) for every runnable workflow. +Server-side WebSockets, WebRTC negotiation, sideband, SIP, transcription, and +translation are covered in the [Realtime guide](realtime.md) and runnable +[example runbook](examples/realtime/README.md). ### Pagination diff --git a/examples/realtime/README.md b/examples/realtime/README.md index 9fad6e315..b194442d4 100644 --- a/examples/realtime/README.md +++ b/examples/realtime/README.md @@ -117,7 +117,8 @@ A successful run streams transcript deltas, then prints the completed transcript with its `item_id`. Completion events for different input turns may arrive out of order, so applications should correlate them by `item_id`. The example opens a dedicated `intent: :transcription` connection. Configure -its transcription model with `OPENAI_REALTIME_TRANSCRIPTION_MODEL`. +its transcription model with `OPENAI_REALTIME_TRANSCRIPTION_MODEL`. An EOF +before the completed transcription event is reported as a failed smoke test. ## Realtime translation @@ -133,7 +134,9 @@ bundle exec ruby examples/realtime/translation.rb A successful run prints the translated transcript, exits after `session.closed`, and writes non-empty translated PCM audio. An EOF before -`session.closed` is reported as incomplete instead of silently succeeding. +`session.closed` is reported as incomplete instead of silently succeeding. If +upload and graceful close both fail, the upload error remains the primary +failure so the interrupted input operation is not hidden by cleanup. ## MCP approval @@ -149,7 +152,9 @@ bundle exec ruby examples/realtime/mcp_approval.rb The example waits for tool import, selects an advertised tool, waits for the tool-choice update to be acknowledged, approves the request, waits for the tool to finish, and asks the model for the final answer. A successful run prints all -five checkpoints. Set `OPENAI_REALTIME_DEBUG=1` to print every event type. +five checkpoints and reaches the final completed `response.done`; EOF before +that event is a failed smoke test. Set `OPENAI_REALTIME_DEBUG=1` to print every +event type. ## WebRTC call creation diff --git a/examples/realtime/mcp_approval.rb b/examples/realtime/mcp_approval.rb index 98dc39f36..3ee0cb4bd 100755 --- a/examples/realtime/mcp_approval.rb +++ b/examples/realtime/mcp_approval.rb @@ -51,6 +51,7 @@ def approve_request(connection, item) end def run_session(connection, prompt:, output: $stdout, debug: false) + completed = false state = { completed_item_ids: {}, tool_lists: {}, @@ -61,10 +62,16 @@ def run_session(connection, prompt:, output: $stdout, debug: false) connection.each do |event| output.puts("[mcp event] #{event.type}") if debug - break if handle_event(connection, event, prompt: prompt, output: output, state: state) == :done + if handle_event(connection, event, prompt: prompt, output: output, state: state) == :done + completed = true + break + end select_discovered_tool(connection, output: output, state: state) end + return if completed + + raise "Realtime connection closed before the final response.done" end def handle_event(connection, event, prompt:, output:, state:) diff --git a/examples/realtime/translation.rb b/examples/realtime/translation.rb index 717fac8a0..8f110a2eb 100755 --- a/examples/realtime/translation.rb +++ b/examples/realtime/translation.rb @@ -33,13 +33,25 @@ def stream(connection, audio_output:, transcript_output: $stdout) end def write_input(connection, input_path) - File.open(input_path, "rb") do |input| - while (chunk = input.read(9_600)) - connection.input_audio_buffer.append_bytes(chunk) + upload_error = nil + begin + File.open(input_path, "rb") do |input| + while (chunk = input.read(9_600)) + connection.input_audio_buffer.append_bytes(chunk) + end end + rescue StandardError => e + upload_error = e + raise + ensure + close_session(connection, preserve_error: !upload_error.nil?) end - ensure + end + + def close_session(connection, preserve_error:) connection.session.close + rescue StandardError + raise unless preserve_error end def run(client:, model:, input_path:, output_path:, target_language:, transcript_output: $stdout) diff --git a/examples/realtime/websocket_transcription.rb b/examples/realtime/websocket_transcription.rb index 54784da6f..0bc520412 100755 --- a/examples/realtime/websocket_transcription.rb +++ b/examples/realtime/websocket_transcription.rb @@ -42,6 +42,7 @@ def upload(connection, input_path:) end def print_transcript(connection, output: $stdout) + completed = false connection.each do |event| case event when OpenAI::Realtime::ConversationItemInputAudioTranscriptionDeltaEvent @@ -49,11 +50,15 @@ def print_transcript(connection, output: $stdout) output.flush when OpenAI::Realtime::ConversationItemInputAudioTranscriptionCompletedEvent output.puts("\n[#{event.item_id}] #{event.transcript}") + completed = true break when OpenAI::Realtime::RealtimeErrorEvent raise event.error.message end end + return if completed + + raise "Realtime connection closed before transcription completed" end def run(client:, input_path:, transcription_model:, output: $stdout) diff --git a/lib/openai/internal/logging.rb b/lib/openai/internal/logging.rb index e633353ec..7537d85ce 100644 --- a/lib/openai/internal/logging.rb +++ b/lib/openai/internal/logging.rb @@ -112,6 +112,21 @@ def observe_stream(stream, response:) ObservedStream.new(stream: stream, context: self, response: response) end + def observe_raw_response(response) + return response unless enabled?(:error) + + OpenAI::HTTPClient::Response.new( + status: response.status, + headers: response.headers, + body: ObservedEnumerable.new( + enumerable: response.body, + context: self, + response: response, + close: -> { OpenAI::Internal::Util.close_fused!(response.body) } + ) + ) + end + def request_failed(error) status = error.is_a?(OpenAI::Errors::APIError) ? error.status : nil request_id = error.is_a?(OpenAI::Errors::APIError) ? error.request_id : nil @@ -236,10 +251,34 @@ def initialize(stream:, context:, response:) end private def iterator - source = @stream.to_enum + ObservedEnumerable.new( + enumerable: @stream, + context: @context, + response: @response, + close: -> { @stream.close } + ) + end + end + + class ObservedEnumerable + include Enumerable + + def initialize(enumerable:, context:, response:, close:) + @enumerable = enumerable + @context = context + @response = response + @close = close + @iterator = iterator + end + + def each(&block) = @iterator.each(&block) + def close = OpenAI::Internal::Util.close_fused!(@iterator) + + private def iterator + source = @enumerable.to_enum observed = Enumerator.new do |yielder| loop do - event = + chunk = begin source.next rescue StopIteration @@ -249,10 +288,10 @@ def initialize(stream:, context:, response:) @context.request_failed(e) raise end - yielder << event + yielder << chunk end end - OpenAI::Internal::Util.fused_enum(observed) { @stream.close } + OpenAI::Internal::Util.fused_enum(observed, &@close) end end diff --git a/lib/openai/internal/transport/base_client.rb b/lib/openai/internal/transport/base_client.rb index dbd2d8740..32ccc50ae 100644 --- a/lib/openai/internal/transport/base_client.rb +++ b/lib/openai/internal/transport/base_client.rb @@ -621,7 +621,7 @@ def send_request(request, redirect_count:, retry_count:, send_retry_header:, &co # @return [OpenAI::HTTPClient::Response] def request_raw(req) _, response, log_context = perform_request(req) - finish_request(log_context, response) { response } + log_context.observe_raw_response(response) end # Execute the request specified by `req`. This is the method that all resource diff --git a/rbi/openai/internal/logging.rbi b/rbi/openai/internal/logging.rbi index 9a0b7df71..e0902b4ee 100644 --- a/rbi/openai/internal/logging.rbi +++ b/rbi/openai/internal/logging.rbi @@ -77,6 +77,14 @@ module OpenAI def observe_stream(stream, response:) end + sig do + params(response: OpenAI::HTTPClient::Response).returns( + OpenAI::HTTPClient::Response + ) + end + def observe_raw_response(response) + end + sig { params(error: StandardError).void } def request_failed(error) end @@ -140,6 +148,35 @@ module OpenAI end end + class ObservedEnumerable + include Enumerable + + Elem = type_member(:out) + + sig do + params( + enumerable: T::Enumerable[Elem], + context: OpenAI::Internal::Logging::Context, + response: OpenAI::HTTPClient::Response, + close: T.proc.void + ).void + end + def initialize(enumerable:, context:, response:, close:) + end + + sig { params(block: T.untyped).returns(T.untyped) } + def each(&block) + end + + sig { void } + def close + end + + sig { returns(T::Enumerable[Elem]) } + private def iterator + end + end + class << self sig { params(value: T.any(Symbol, String)).returns(Symbol) } def normalize_level(value) diff --git a/realtime.md b/realtime.md index 98079461e..b2f1ac62b 100644 --- a/realtime.md +++ b/realtime.md @@ -298,8 +298,10 @@ The first completed `ResponseDoneEvent` can finish the MCP argument-generation phase while approval and tool execution are still pending. Keep reading, answer the `RealtimeMcpApprovalRequest`, wait for `ResponseMcpCallCompleted`, and create a follow-up response (usually with `tool_choice: :none`) to turn the tool output -into a final assistant answer. The approval helper generates item IDs within the -service's 32-character limit. +into a final assistant answer. Treat only that follow-up response's completed +`ResponseDoneEvent` as success; clean EOF during discovery, approval, or tool +execution is still an incomplete workflow. The approval helper generates item +IDs within the service's 32-character limit. ## Lifecycle and failures @@ -309,6 +311,18 @@ can duplicate side effects. Treat an unexpected close as an application-level decision: terminate, or reconnect and deliberately rebuild the state that is safe for that use case. +Runnable smoke tests require their protocol-specific terminal event rather than +treating a clean WebSocket EOF as success: completed `response.done` for text, +audio, and MCP; transcription completion for transcription; and +`session.closed` for translation. Cleanup must also preserve an active upload or +processing error when a graceful close fails, while surfacing the close failure +after an otherwise successful operation. + +Raw WebRTC SDP responses remain lazily consumed. Request observability follows +that body lifecycle: completion is recorded only after the SDP body is fully +drained, and a body read failure records `request failed` without a contradictory +completion event. + - Invalid JSON or a payload that cannot match the selected event union raises `OpenAI::Errors::RealtimeProtocolError`, with `data` and `cause`. - Handshake and socket I/O failures from the default adapter raise diff --git a/sig/openai/internal/logging.rbs b/sig/openai/internal/logging.rbs index aa0b8755f..20f1ce0c2 100644 --- a/sig/openai/internal/logging.rbs +++ b/sig/openai/internal/logging.rbs @@ -38,6 +38,9 @@ module OpenAI OpenAI::Internal::Type::BaseStream[top, Elem] stream, response: OpenAI::HTTPClient::Response ) -> OpenAI::Internal::Type::BaseStream[top, Elem] + def observe_raw_response: ( + OpenAI::HTTPClient::Response response + ) -> OpenAI::HTTPClient::Response def request_failed: (StandardError error) -> void def response_body: ( String body, @@ -74,6 +77,21 @@ module OpenAI private def iterator: -> Enumerable[Elem] end + class ObservedEnumerable[Elem] + include Enumerable[Elem] + + def initialize: ( + enumerable: Enumerable[Elem], + context: OpenAI::Internal::Logging::Context, + response: OpenAI::HTTPClient::Response, + close: ^() -> void + ) -> void + + def each: { (Elem) -> void } -> void + def close: -> void + private def iterator: -> Enumerable[Elem] + end + def self.normalize_level: (Symbol | String value) -> Symbol def self.validate_logger!: (OpenAI::_Logger? logger) -> void def self.default_logger: -> OpenAI::_Logger diff --git a/test/openai/logging_test.rb b/test/openai/logging_test.rb index b98c3df7d..82864c6db 100644 --- a/test/openai/logging_test.rb +++ b/test/openai/logging_test.rb @@ -57,6 +57,24 @@ class UnbufferableChunk < String def byteslice(*) = raise("stream chunk was buffered") end + class FailingBody + include Enumerable + + attr_reader :close_count + + def initialize(error) + @error = error + @close_count = 0 + end + + def each + yield("partial") + raise @error + end + + def close = (@close_count += 1) + end + def setup super @openai_log = ENV.delete("OPENAI_LOG") @@ -614,6 +632,72 @@ def test_stream_failures_log_an_error_without_a_success_event refute_includes(logger.events.fetch(0).fetch(1), "request complete") end + def test_raw_response_completion_is_logged_only_after_body_consumption + logger = CapturingLogger.new + source = CloseableBody.new("answer-sdp") + http_client = StubHTTPClient.new do |_request| + OpenAI::HTTPClient::Response.new( + status: 200, + headers: {"content-type" => "application/sdp", "x-request-id" => "req_raw"}, + body: source + ) + end + client = diagnostic_client(http_client: http_client, logger: logger, log_level: :info) + + response = client.request_raw(method: :post, path: "realtime/calls") + + assert_empty(logger.events) + assert_equal(["answer-sdp"], response.body.to_a) + assert_equal(1, source.close_count) + assert_equal([:info], logger.events.map(&:first)) + assert_includes(logger.events.fetch(0).fetch(1), "request complete") + end + + def test_raw_response_body_failure_logs_an_error_without_a_success_event + logger = CapturingLogger.new + failure = OpenAI::Errors::APIConnectionError.new( + url: URI("https://example.com/v1/realtime/calls") + ) + source = FailingBody.new(failure) + http_client = StubHTTPClient.new do |_request| + OpenAI::HTTPClient::Response.new( + status: 200, + headers: {"content-type" => "application/sdp"}, + body: source + ) + end + client = diagnostic_client(http_client: http_client, logger: logger, log_level: :info) + response = client.request_raw(method: :post, path: "realtime/calls") + + assert_empty(logger.events) + error = assert_raises(OpenAI::Errors::APIConnectionError) { response.body.to_a } + + assert_same(failure, error) + assert_equal(1, source.close_count) + assert_equal([:error], logger.events.map(&:first)) + assert_includes(logger.events.fetch(0).fetch(1), "request failed") + refute_includes(logger.events.fetch(0).fetch(1), "request complete") + end + + def test_closing_a_partially_consumed_raw_response_closes_its_source + logger = CapturingLogger.new + source = CloseableBody.new("first", "second") + http_client = StubHTTPClient.new do |_request| + OpenAI::HTTPClient::Response.new( + status: 200, + headers: {"content-type" => "application/sdp"}, + body: source + ) + end + client = diagnostic_client(http_client: http_client, logger: logger, log_level: :info) + response = client.request_raw(method: :post, path: "realtime/calls") + + assert_equal("first", response.body.first) + + assert_equal(1, source.close_count) + assert_empty(logger.events) + end + def test_consumer_exceptions_are_not_logged_as_request_failures logger = CapturingLogger.new http_client = StubHTTPClient.new do |_request| diff --git a/test/openai/realtime/example_stream_lifecycle_test.rb b/test/openai/realtime/example_stream_lifecycle_test.rb index 76b7351e1..a646562ac 100644 --- a/test/openai/realtime/example_stream_lifecycle_test.rb +++ b/test/openai/realtime/example_stream_lifecycle_test.rb @@ -1,8 +1,10 @@ # frozen_string_literal: true require_relative "../test_helper" +require_relative "../../../examples/realtime/mcp_approval" require_relative "../../../examples/realtime/translation" require_relative "../../../examples/realtime/websocket_audio" +require_relative "../../../examples/realtime/websocket_transcription" class OpenAI::Test::RealtimeExampleStreamLifecycleTest < Minitest::Test class RecordingConnection @@ -15,6 +17,27 @@ def each(&block) end end + class RaisingEndpoint + attr_reader :calls + + def initialize(error) + @error = error + @calls = [] + end + + def append_bytes(bytes) + @calls << [:append_bytes, bytes] + raise @error + end + + def close + @calls << [:close] + raise @error + end + end + + TranslationConnection = Data.define(:input_audio_buffer, :session) + def test_websocket_audio_rejects_a_connection_that_closes_before_response_done error = assert_raises(RuntimeError) do OpenAI::Examples::Realtime::WebSocketAudio.stream_response( @@ -81,4 +104,58 @@ def test_translation_accepts_only_a_session_closed_event assert_nil(result) assert_equal("\n", transcript_output.string) end + + def test_transcription_rejects_a_connection_that_closes_before_completion + error = assert_raises(RuntimeError) do + OpenAI::Examples::Realtime::WebSocketTranscription.print_transcript( + RecordingConnection.new, + output: StringIO.new + ) + end + + assert_equal("Realtime connection closed before transcription completed", error.message) + end + + def test_transcription_accepts_only_a_completed_event + connection = RecordingConnection.new( + [ + OpenAI::Realtime::ConversationItemInputAudioTranscriptionCompletedEvent.new( + content_index: 0, + event_id: "event_1", + item_id: "item_1", + transcript: "Hello", + usage: {type: :duration, seconds: 0.25} + ) + ] + ) + output = StringIO.new + + result = OpenAI::Examples::Realtime::WebSocketTranscription.print_transcript( + connection, + output: output + ) + + assert_nil(result) + assert_equal("\n[item_1] Hello\n", output.string) + end + + def test_translation_preserves_an_upload_error_when_session_close_also_fails + upload_error = RuntimeError.new("upload failed") + input_audio_buffer = RaisingEndpoint.new(upload_error) + session = RaisingEndpoint.new(RuntimeError.new("close failed")) + connection = TranslationConnection.new(input_audio_buffer, session) + + Tempfile.create("translation-input") do |input| + input.write("audio") + input.flush + + error = assert_raises(RuntimeError) do + OpenAI::Examples::Realtime::Translation.write_input(connection, input.path) + end + + assert_same(upload_error, error) + end + assert_equal([[:append_bytes, "audio"]], input_audio_buffer.calls) + assert_equal([[:close]], session.calls) + end end diff --git a/test/openai/realtime/examples_test.rb b/test/openai/realtime/examples_test.rb index 2b3084a20..4b641e223 100644 --- a/test/openai/realtime/examples_test.rb +++ b/test/openai/realtime/examples_test.rb @@ -695,7 +695,8 @@ def test_mcp_console_example_requires_the_discovered_tool OpenAI::Realtime::SessionUpdatedEvent.new( event_id: "event_3", session: {} - ) + ), + completed_response_event ] ) @@ -734,12 +735,15 @@ def test_mcp_console_example_waits_for_the_tool_choice_update ] ) - OpenAI::Examples::Realtime::MCPApproval.run_session( - connection, - prompt: "Search the docs", - output: StringIO.new - ) + error = assert_raises(RuntimeError) do + OpenAI::Examples::Realtime::MCPApproval.run_session( + connection, + prompt: "Search the docs", + output: StringIO.new + ) + end + assert_equal("Realtime connection closed before the final response.done", error.message) assert_empty(connection.response.calls) assert_empty(connection.conversation.items.calls) end @@ -751,7 +755,8 @@ def test_mcp_console_example_requests_a_final_answer_after_the_tool_completes event_id: "event_1", item_id: "mcp_call_1", output_index: 0 - ) + ), + completed_response_event ] ) @@ -895,4 +900,14 @@ def test_websocket_text_accepts_only_a_completed_response assert_includes(output.string, "response.done status=completed") end + + private def completed_response_event + OpenAI::Realtime::ResponseDoneEvent.new( + event_id: "event_done", + response: OpenAI::Realtime::RealtimeResponse.new( + id: "response_done", + status: :completed + ) + ) + end end From 1068f8a7b446e7a7465a471adaac57ac4dbd5413 Mon Sep 17 00:00:00 2001 From: Justin Beckwith Date: Tue, 11 Aug 2026 17:44:38 -0700 Subject: [PATCH 5/8] fix: enforce Realtime smoke completion --- examples/realtime/README.md | 8 +++- examples/realtime/realtime_conversation.rb | 9 ++++- examples/realtime/websocket_audio.rb | 6 ++- .../realtime/transports/async_websocket.rbi | 2 +- realtime.md | 8 ++++ .../realtime/transports/async_websocket.rbs | 2 +- .../async_websocket_transport_test.rb | 2 +- .../realtime/example_stream_lifecycle_test.rb | 40 +++++++++++++++++-- test/openai/realtime/examples_test.rb | 23 +++++++++++ 9 files changed, 90 insertions(+), 10 deletions(-) diff --git a/examples/realtime/README.md b/examples/realtime/README.md index b194442d4..2684ef064 100644 --- a/examples/realtime/README.md +++ b/examples/realtime/README.md @@ -71,6 +71,9 @@ OPENAI_REALTIME_TIMEOUT=30 \ bundle exec ruby examples/realtime/realtime_conversation.rb ``` +This bounded mode exits successfully only after the requested `response.done`; +a clean EOF before that event fails the smoke test. + ## WebSocket text ```sh @@ -100,8 +103,9 @@ bundle exec ruby examples/realtime/websocket_audio.rb ffplay -f s16le -ar 24000 -ch_layout mono response.pcm ``` -A successful run exits with status 0 and writes a non-empty output file. An EOF -before a completed `response.done` is reported as a failed smoke test. +A successful run exits with status 0 only after writing audio bytes and seeing a +completed `response.done`. EOF before completion, or completion without an +audio delta, is reported as a failed smoke test. ## Realtime transcription diff --git a/examples/realtime/realtime_conversation.rb b/examples/realtime/realtime_conversation.rb index 99e77a832..eac84fb13 100755 --- a/examples/realtime/realtime_conversation.rb +++ b/examples/realtime/realtime_conversation.rb @@ -405,10 +405,17 @@ def wait_until_ready(connection) end def stream_events(connection, microphone:, outbound:, playback:, output:, stop_after: nil) + stop_event_seen = stop_after.nil? connection.each do |event| handle_event(event, outbound: outbound, playback: playback, output: output) - break if stop_after == event.type.to_s + if stop_after == event.type.to_s + stop_event_seen = true + break + end end + return if stop_event_seen + + raise "Realtime connection closed before #{stop_after}" ensure microphone.stop end diff --git a/examples/realtime/websocket_audio.rb b/examples/realtime/websocket_audio.rb index 38a067364..4cc32772b 100755 --- a/examples/realtime/websocket_audio.rb +++ b/examples/realtime/websocket_audio.rb @@ -11,13 +11,17 @@ module WebSocketAudio def stream_response(connection, output:) completed_response = false + audio_bytes = 0 connection.each do |event| case event when OpenAI::Realtime::ResponseAudioDeltaEvent - output.write(Base64.strict_decode64(event.delta)) + audio = Base64.strict_decode64(event.delta) + output.write(audio) + audio_bytes += audio.bytesize when OpenAI::Realtime::ResponseDoneEvent status = event.response.status raise "Response ended with #{status}" unless status == :completed + raise "Response completed without audio output" if audio_bytes.zero? completed_response = true break diff --git a/rbi/openai/realtime/transports/async_websocket.rbi b/rbi/openai/realtime/transports/async_websocket.rbi index 059bbc273..6869175c2 100644 --- a/rbi/openai/realtime/transports/async_websocket.rbi +++ b/rbi/openai/realtime/transports/async_websocket.rbi @@ -36,7 +36,7 @@ module OpenAI params( url: URI::Generic, headers: T::Hash[String, String], - timeout: Float, + timeout: T.nilable(Float), endpoint_options: T.untyped, block: T.proc.params(socket: Socket).returns(T.untyped) ).returns(T.untyped) diff --git a/realtime.md b/realtime.md index b2f1ac62b..2abd69f7f 100644 --- a/realtime.md +++ b/realtime.md @@ -318,6 +318,11 @@ audio, and MCP; transcription completion for transcription; and processing error when a graceful close fails, while surfacing the close failure after an otherwise successful operation. +Terminal status alone is insufficient when a workflow promises an artifact: +the raw-audio smoke also requires at least one decoded audio byte before its +completed `response.done`. Bounded conversation playback similarly requires the +explicit event requested by `stop_after`; EOF before it is a failed run. + Raw WebRTC SDP responses remain lazily consumed. Request observability follows that body lifecycle: completion is recorded only after the SDP body is fully drained, and a body read failure records `request failed` without a contradictory @@ -366,6 +371,9 @@ transport.open(url:, headers:, timeout:, **transport_options) do |socket| end ``` +`timeout` is `Float?` at the transport boundary because `Client.new(timeout: +nil)` and per-request `timeout: nil` deliberately disable the handshake timeout. + The SDK owns URL construction, authentication, typed JSON encoding/decoding, and block cleanup. The transport owns the WebSocket handshake, frame I/O, TLS, proxies, and transport-specific failures. Internal calls use direct methods; the diff --git a/sig/openai/realtime/transports/async_websocket.rbs b/sig/openai/realtime/transports/async_websocket.rbs index b03980ac2..d30db5995 100644 --- a/sig/openai/realtime/transports/async_websocket.rbs +++ b/sig/openai/realtime/transports/async_websocket.rbs @@ -14,7 +14,7 @@ module OpenAI def open: ( url: URI::Generic, headers: ::Hash[String, String], - timeout: Float, + timeout: Float?, **untyped endpoint_options ) { (Socket socket) -> top diff --git a/test/openai/realtime/async_websocket_transport_test.rb b/test/openai/realtime/async_websocket_transport_test.rb index 15bb58ce2..744b5109d 100644 --- a/test/openai/realtime/async_websocket_transport_test.rb +++ b/test/openai/realtime/async_websocket_transport_test.rb @@ -52,7 +52,7 @@ def test_default_transport_enters_a_reactor_for_synchronous_callers url = URI("wss://example.com/v1/realtime?model=gpt-realtime-2.1") result = Async::WebSocket::Client.stub(:open, client) do - transport.open(url: url, headers: {}, timeout: 1) do |socket| + transport.open(url: url, headers: {}, timeout: nil) do |socket| [socket.read, Fiber.scheduler] end end diff --git a/test/openai/realtime/example_stream_lifecycle_test.rb b/test/openai/realtime/example_stream_lifecycle_test.rb index a646562ac..26e258128 100644 --- a/test/openai/realtime/example_stream_lifecycle_test.rb +++ b/test/openai/realtime/example_stream_lifecycle_test.rb @@ -49,11 +49,21 @@ def test_websocket_audio_rejects_a_connection_that_closes_before_response_done assert_equal("Realtime connection closed before response.done", error.message) end - def test_websocket_audio_accepts_only_a_completed_response + def test_websocket_audio_accepts_audio_followed_by_a_completed_response + audio = "pcm".b + output = StringIO.new connection = RecordingConnection.new( [ - OpenAI::Realtime::ResponseDoneEvent.new( + OpenAI::Realtime::ResponseAudioDeltaEvent.new( event_id: "event_1", + response_id: "response_1", + item_id: "item_1", + output_index: 0, + content_index: 0, + delta: Base64.strict_encode64(audio) + ), + OpenAI::Realtime::ResponseDoneEvent.new( + event_id: "event_2", response: OpenAI::Realtime::RealtimeResponse.new( id: "response_1", status: :completed @@ -64,10 +74,34 @@ def test_websocket_audio_accepts_only_a_completed_response result = OpenAI::Examples::Realtime::WebSocketAudio.stream_response( connection, - output: StringIO.new + output: output ) assert_nil(result) + assert_equal(audio, output.string) + end + + def test_websocket_audio_rejects_a_completed_response_without_audio + connection = RecordingConnection.new( + [ + OpenAI::Realtime::ResponseDoneEvent.new( + event_id: "event_1", + response: OpenAI::Realtime::RealtimeResponse.new( + id: "response_1", + status: :completed + ) + ) + ] + ) + + error = assert_raises(RuntimeError) do + OpenAI::Examples::Realtime::WebSocketAudio.stream_response( + connection, + output: StringIO.new + ) + end + + assert_equal("Response completed without audio output", error.message) end def test_translation_rejects_a_connection_that_closes_before_session_closed diff --git a/test/openai/realtime/examples_test.rb b/test/openai/realtime/examples_test.rb index 4b641e223..50fbefbcb 100644 --- a/test/openai/realtime/examples_test.rb +++ b/test/openai/realtime/examples_test.rb @@ -350,6 +350,29 @@ def test_realtime_conversation_propagates_writer_failure_without_deadlock assert_predicate(microphone, :stopped) end + def test_realtime_conversation_rejects_eof_before_the_requested_stop_event + microphone = RecordingMicrophone.new([]) + playback = OpenAI::Examples::Realtime::Conversation::AudioPlayback.new( + RecordingSpeaker.new + ) + + error = assert_raises(RuntimeError) do + OpenAI::Examples::Realtime::Conversation.stream_events( + RecordingConnection.new, + microphone: microphone, + outbound: RecordingOutbound.new, + playback: playback, + output: StringIO.new, + stop_after: "response.done" + ) + end + + assert_equal("Realtime connection closed before response.done", error.message) + assert_predicate(microphone, :stopped) + ensure + playback&.close + end + def test_realtime_conversation_streams_audio_and_interrupts_local_playback speaker = RecordingSpeaker.new times = [0.0, 0.1].each From 74ca19c7192668c27c8b7081f2c94a2c30d9a10e Mon Sep 17 00:00:00 2001 From: Justin Beckwith Date: Tue, 11 Aug 2026 19:27:46 -0700 Subject: [PATCH 6/8] fix: address Realtime lifecycle feedback --- examples/realtime/README.md | 24 +++-- examples/realtime/event_stream.rb | 23 ++++ examples/realtime/realtime_conversation.rb | 14 +-- examples/realtime/sideband.rb | 6 +- examples/realtime/sip.rb | 6 +- examples/realtime/sorbet_connection_types.rb | 35 ++++++ examples/realtime/translation.rb | 20 ++-- examples/realtime/websocket_audio.rb | 10 +- examples/realtime/websocket_text.rb | 9 +- examples/realtime/websocket_transcription.rb | 15 ++- lib/openai/resources/realtime.rb | 101 ++++++++++++------ rbi/openai/resources/realtime.rbi | 51 ++++----- realtime.md | 34 +++--- sig/openai/resources/realtime.rbs | 54 ++++------ .../async_websocket_transport_test.rb | 2 +- test/openai/realtime/connection_test.rb | 44 +------- .../realtime/example_stream_lifecycle_test.rb | 61 ++++++++++- test/openai/realtime/examples_test.rb | 28 ++++- 18 files changed, 326 insertions(+), 211 deletions(-) create mode 100644 examples/realtime/event_stream.rb create mode 100644 examples/realtime/sorbet_connection_types.rb diff --git a/examples/realtime/README.md b/examples/realtime/README.md index 2684ef064..6c8dad131 100644 --- a/examples/realtime/README.md +++ b/examples/realtime/README.md @@ -120,9 +120,10 @@ bundle exec ruby examples/realtime/websocket_transcription.rb A successful run streams transcript deltas, then prints the completed transcript with its `item_id`. Completion events for different input turns may arrive out of order, so applications should correlate them by `item_id`. -The example opens a dedicated `intent: :transcription` connection. Configure -its transcription model with `OPENAI_REALTIME_TRANSCRIPTION_MODEL`. An EOF -before the completed transcription event is reported as a failed smoke test. +The example opens a dedicated connection with `connect_transcription`. +Configure its transcription model with `OPENAI_REALTIME_TRANSCRIPTION_MODEL`. +An EOF before the completed transcription event is reported as a failed smoke +test. ## Realtime translation @@ -138,9 +139,10 @@ bundle exec ruby examples/realtime/translation.rb A successful run prints the translated transcript, exits after `session.closed`, and writes non-empty translated PCM audio. An EOF before -`session.closed` is reported as incomplete instead of silently succeeding. If -upload and graceful close both fail, the upload error remains the primary -failure so the interrupted input operation is not hidden by cleanup. +`session.closed`, or a closed session with no translated audio, is reported as +incomplete instead of silently succeeding. If upload and graceful close both +fail, the upload error remains the primary failure so the interrupted input +operation is not hidden by cleanup. ## MCP approval @@ -186,7 +188,8 @@ bundle exec ruby examples/realtime/sideband.rb For a bounded smoke test, set `OPENAI_REALTIME_STOP_AFTER=session.updated` and `OPENAI_REALTIME_TIMEOUT=30`. A real paired test keeps the browser or SIP peer -connected while this process attaches to the same call. +connected while this process attaches to the same call. EOF before the selected +event fails the bounded smoke test. ## SIP @@ -203,9 +206,10 @@ The script accepts the call, attaches a sideband WebSocket, and prints the audio transcript. `OPENAI_REALTIME_STOP_AFTER` and `OPENAI_REALTIME_TIMEOUT` provide bounded smoke-test controls. Once accepted, the call is hung up during cleanup, including after a timeout or sideband failure; an already-ended call is treated -as successfully cleaned up. Without a carrier-originated incoming call, the -same accept-then-attach orchestration is covered by the local example and HTTP -resource tests, but that is not a substitute for the final telephony smoke test. +as successfully cleaned up. EOF before the selected event fails a bounded run. +Without a carrier-originated incoming call, the same accept-then-attach +orchestration is covered by the local example and HTTP resource tests, but that +is not a substitute for the final telephony smoke test. ## Local protocol coverage diff --git a/examples/realtime/event_stream.rb b/examples/realtime/event_stream.rb new file mode 100644 index 000000000..15caa9945 --- /dev/null +++ b/examples/realtime/event_stream.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module OpenAI + module Examples + module Realtime + module EventStream + module_function + + def each_until(connection, stop_after: nil, closed_message: nil) + reached_target = false + connection.each do |event| + yield(event) + reached_target = stop_after == event.type.to_s + break if reached_target + end + return if stop_after.nil? || reached_target + + raise(closed_message || "Realtime connection closed before #{stop_after}") + end + end + end + end +end diff --git a/examples/realtime/realtime_conversation.rb b/examples/realtime/realtime_conversation.rb index eac84fb13..a80a570d7 100755 --- a/examples/realtime/realtime_conversation.rb +++ b/examples/realtime/realtime_conversation.rb @@ -4,6 +4,7 @@ require "async" require "timeout" require_relative "../../lib/openai" +require_relative "event_stream" module OpenAI module Examples @@ -22,6 +23,7 @@ def initialize(input_format:, device:) def each_chunk return enum_for(__method__) unless block_given? + return if @stopping reader, writer = IO.pipe @pid = Process.spawn(*command, out: writer, err: $stderr) @@ -39,9 +41,9 @@ def each_chunk end def stop + @stopping = true return unless @pid - @stopping = true Process.kill("INT", @pid) rescue Errno::ESRCH nil @@ -405,17 +407,9 @@ def wait_until_ready(connection) end def stream_events(connection, microphone:, outbound:, playback:, output:, stop_after: nil) - stop_event_seen = stop_after.nil? - connection.each do |event| + EventStream.each_until(connection, stop_after: stop_after) do |event| handle_event(event, outbound: outbound, playback: playback, output: output) - if stop_after == event.type.to_s - stop_event_seen = true - break - end end - return if stop_event_seen - - raise "Realtime connection closed before #{stop_after}" ensure microphone.stop end diff --git a/examples/realtime/sideband.rb b/examples/realtime/sideband.rb index 730f673d3..780260820 100755 --- a/examples/realtime/sideband.rb +++ b/examples/realtime/sideband.rb @@ -2,6 +2,7 @@ # frozen_string_literal: true require_relative "../../lib/openai" +require_relative "event_stream" require "timeout" module OpenAI @@ -11,15 +12,14 @@ module Sideband module_function def stream(connection, output: $stdout, stop_after: nil) - connection.each do |event| + EventStream.each_until(connection, stop_after: stop_after) do |event| output.puts("#{event.type}: #{event.to_h}") output.flush - break if stop_after == event.type.to_s end end def run(client:, call_id:, stop_after: nil, output: $stdout) - client.realtime.connect(call_id: call_id) do |connection| + client.realtime.connect_to_call(call_id: call_id) do |connection| connection.session.update( type: :realtime, instructions: "Use server-side business rules and keep answers short." diff --git a/examples/realtime/sip.rb b/examples/realtime/sip.rb index 0a3bb6913..869adeded 100755 --- a/examples/realtime/sip.rb +++ b/examples/realtime/sip.rb @@ -2,6 +2,7 @@ # frozen_string_literal: true require_relative "../../lib/openai" +require_relative "event_stream" require "timeout" module OpenAI @@ -11,7 +12,7 @@ module SIP module_function def stream(connection, output: $stdout, stop_after: nil) - connection.each do |event| + EventStream.each_until(connection, stop_after: stop_after) do |event| case event when OpenAI::Realtime::ResponseAudioTranscriptDeltaEvent output.print(event.delta) @@ -19,7 +20,6 @@ def stream(connection, output: $stdout, stop_after: nil) when OpenAI::Realtime::RealtimeErrorEvent raise event.error.message end - break if stop_after == event.type.to_s end end @@ -33,7 +33,7 @@ def run(client:, call_id:, model:, output: $stdout, stop_after: nil) ) accepted = true - client.realtime.connect(call_id: call_id) do |connection| + client.realtime.connect_to_call(call_id: call_id) do |connection| stream(connection, output: output, stop_after: stop_after) end ensure diff --git a/examples/realtime/sorbet_connection_types.rb b/examples/realtime/sorbet_connection_types.rb new file mode 100644 index 000000000..a231aa0eb --- /dev/null +++ b/examples/realtime/sorbet_connection_types.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true +# typed: true + +require_relative "../../lib/openai" + +# Static contract fixture for the three capability-specific connection methods. +# Defining, but never invoking, this method keeps the fixture inert at runtime. +module OpenAI + module Examples + module Realtime + module SorbetConnectionTypes + module_function + + def verify + client = OpenAI::Client.new + + client.realtime.connect(model: "gpt-realtime-2.1") do |connection| + T.assert_type!(connection, OpenAI::Realtime::Connection) + connection.response.create + end + + client.realtime.connect_to_call(call_id: "rtc_123") do |connection| + T.assert_type!(connection, OpenAI::Realtime::SidebandConnection) + connection.output_audio_buffer.clear + end + + client.realtime.connect_transcription do |connection| + T.assert_type!(connection, OpenAI::Realtime::TranscriptionConnection) + connection.input_audio_buffer.commit + end + end + end + end + end +end diff --git a/examples/realtime/translation.rb b/examples/realtime/translation.rb index 8f110a2eb..fb737bf43 100755 --- a/examples/realtime/translation.rb +++ b/examples/realtime/translation.rb @@ -3,6 +3,7 @@ require "async" require_relative "../../lib/openai" +require_relative "event_stream" module OpenAI module Examples @@ -11,25 +12,28 @@ module Translation module_function def stream(connection, audio_output:, transcript_output: $stdout) - session_closed = false - connection.each do |event| + audio_bytes = 0 + EventStream.each_until( + connection, + stop_after: "session.closed", + closed_message: "Realtime translation connection closed before session.closed" + ) do |event| case event when OpenAI::Realtime::RealtimeTranslationOutputTranscriptDeltaEvent transcript_output.print(event.delta) transcript_output.flush when OpenAI::Realtime::RealtimeTranslationOutputAudioDeltaEvent - audio_output.write(Base64.strict_decode64(event.delta)) + audio = Base64.strict_decode64(event.delta) + audio_output.write(audio) + audio_bytes += audio.bytesize when OpenAI::Realtime::RealtimeTranslationSessionClosedEvent + raise "Translation session closed without audio output" if audio_bytes.zero? + transcript_output.puts - session_closed = true - break when OpenAI::Realtime::RealtimeErrorEvent raise event.error.message end end - return if session_closed - - raise "Realtime translation connection closed before session.closed" end def write_input(connection, input_path) diff --git a/examples/realtime/websocket_audio.rb b/examples/realtime/websocket_audio.rb index 4cc32772b..c7b870d1b 100755 --- a/examples/realtime/websocket_audio.rb +++ b/examples/realtime/websocket_audio.rb @@ -2,6 +2,7 @@ # frozen_string_literal: true require_relative "../../lib/openai" +require_relative "event_stream" module OpenAI module Examples @@ -10,9 +11,8 @@ module WebSocketAudio module_function def stream_response(connection, output:) - completed_response = false audio_bytes = 0 - connection.each do |event| + EventStream.each_until(connection, stop_after: "response.done") do |event| case event when OpenAI::Realtime::ResponseAudioDeltaEvent audio = Base64.strict_decode64(event.delta) @@ -22,16 +22,10 @@ def stream_response(connection, output:) status = event.response.status raise "Response ended with #{status}" unless status == :completed raise "Response completed without audio output" if audio_bytes.zero? - - completed_response = true - break when OpenAI::Realtime::RealtimeErrorEvent raise event.error.message end end - return if completed_response - - raise "Realtime connection closed before response.done" end def run(client:, model:, input_path:, output_path:) diff --git a/examples/realtime/websocket_text.rb b/examples/realtime/websocket_text.rb index 5317dbd08..72b33a4d2 100755 --- a/examples/realtime/websocket_text.rb +++ b/examples/realtime/websocket_text.rb @@ -3,6 +3,7 @@ require "timeout" require_relative "../../lib/openai" +require_relative "event_stream" module OpenAI module Examples @@ -12,8 +13,7 @@ module WebSocketText def stream_response(connection, output: $stdout) started_response = false - completed_response = false - connection.each do |event| + EventStream.each_until(connection, stop_after: "response.done") do |event| case event when OpenAI::Realtime::SessionCreatedEvent output.puts("[realtime] session.created") @@ -30,15 +30,10 @@ def stream_response(connection, output: $stdout) raise "Realtime response ended with status #{status.inspect}" unless status == :completed output.puts("[realtime] response.done status=completed") - completed_response = true - break when OpenAI::Realtime::RealtimeErrorEvent raise "Realtime API error: #{event.error.message}" end end - return if completed_response - - raise "Realtime connection closed before response.done" end def run(client:, model:, prompt:, output: $stdout) diff --git a/examples/realtime/websocket_transcription.rb b/examples/realtime/websocket_transcription.rb index 0bc520412..9c55de1c7 100755 --- a/examples/realtime/websocket_transcription.rb +++ b/examples/realtime/websocket_transcription.rb @@ -3,6 +3,7 @@ require "timeout" require_relative "../../lib/openai" +require_relative "event_stream" module OpenAI module Examples @@ -42,27 +43,25 @@ def upload(connection, input_path:) end def print_transcript(connection, output: $stdout) - completed = false - connection.each do |event| + EventStream.each_until( + connection, + stop_after: "conversation.item.input_audio_transcription.completed", + closed_message: "Realtime connection closed before transcription completed" + ) do |event| case event when OpenAI::Realtime::ConversationItemInputAudioTranscriptionDeltaEvent output.print(event.delta) output.flush when OpenAI::Realtime::ConversationItemInputAudioTranscriptionCompletedEvent output.puts("\n[#{event.item_id}] #{event.transcript}") - completed = true - break when OpenAI::Realtime::RealtimeErrorEvent raise event.error.message end end - return if completed - - raise "Realtime connection closed before transcription completed" end def run(client:, input_path:, transcription_model:, output: $stdout) - client.realtime.connect(intent: :transcription) do |connection| + client.realtime.connect_transcription do |connection| configure(connection, transcription_model: transcription_model) wait_until_ready(connection) upload(connection, input_path: input_path) diff --git a/lib/openai/resources/realtime.rb b/lib/openai/resources/realtime.rb index 7625acf49..a5f606525 100644 --- a/lib/openai/resources/realtime.rb +++ b/lib/openai/resources/realtime.rb @@ -12,40 +12,90 @@ class Realtime # @return [OpenAI::Resources::Realtime::Translations] attr_reader :translations - # Open a server-side Realtime WebSocket. Pass exactly one of `model` to start a - # conversation session, `intent: :transcription` to start transcription, or - # `call_id` to attach a sideband connection to a WebRTC or SIP call. A block is - # required and the socket is closed on every exit path. + # Open a server-side Realtime conversation WebSocket. A block is required and + # the socket is closed on every exit path. # - # @param model [String, nil] - # @param call_id [String, nil] - # @param intent [Symbol, nil] + # @param model [String] # @param request_options [OpenAI::RequestOptions, Hash{Symbol=>Object}, nil] # @param transport [#open, nil] An alternate WebSocket transport. # @param transport_options [Hash{Symbol=>Object}] # @yieldparam connection [OpenAI::Realtime::Connection] # @return [Object] def connect( - model: nil, - call_id: nil, - intent: nil, + model:, request_options: nil, transport: nil, transport_options: {}, &block ) - raise ArgumentError, "A block is required to open a Realtime WebSocket." unless block + open_connection( + query: {"model" => model}, + connection_class: OpenAI::Realtime::Connection, + request_options: request_options, + transport: transport, + transport_options: transport_options, + &block + ) + end - targets = [model, call_id, intent].compact - unless targets.one? - message = "Pass exactly one of `model`, `call_id`, or `intent` when opening a Realtime WebSocket." - raise ArgumentError, message - end - unless intent.nil? || intent == :transcription - raise ArgumentError, "The only supported Realtime connection intent is `transcription`." - end + # Attach a sideband WebSocket to an existing WebRTC or SIP call. + # + # @param call_id [String] + # @param request_options [OpenAI::RequestOptions, Hash{Symbol=>Object}, nil] + # @param transport [#open, nil] An alternate WebSocket transport. + # @param transport_options [Hash{Symbol=>Object}] + # @yieldparam connection [OpenAI::Realtime::SidebandConnection] + # @return [Object] + def connect_to_call( + call_id:, + request_options: nil, + transport: nil, + transport_options: {}, + &block + ) + open_connection( + query: {"call_id" => call_id}, + connection_class: OpenAI::Realtime::SidebandConnection, + request_options: request_options, + transport: transport, + transport_options: transport_options, + &block + ) + end + + # Open a dedicated Realtime transcription WebSocket. + # + # @param request_options [OpenAI::RequestOptions, Hash{Symbol=>Object}, nil] + # @param transport [#open, nil] An alternate WebSocket transport. + # @param transport_options [Hash{Symbol=>Object}] + # @yieldparam connection [OpenAI::Realtime::TranscriptionConnection] + # @return [Object] + def connect_transcription( + request_options: nil, + transport: nil, + transport_options: {}, + &block + ) + open_connection( + query: {"intent" => "transcription"}, + connection_class: OpenAI::Realtime::TranscriptionConnection, + request_options: request_options, + transport: transport, + transport_options: transport_options, + &block + ) + end + + private def open_connection( + query:, + connection_class:, + request_options:, + transport:, + transport_options:, + &block + ) + raise ArgumentError, "A block is required to open a Realtime WebSocket." unless block - query, connection_class = connection_target(model: model, call_id: call_id, intent: intent) manager = OpenAI::Realtime::ConnectionManager.new( client: @client, path: "realtime", @@ -58,17 +108,6 @@ def connect( manager.open(&block) end - # @api private - def connection_target(model:, call_id:, intent:) - if model - [{"model" => model}, OpenAI::Realtime::Connection] - elsif call_id - [{"call_id" => call_id}, OpenAI::Realtime::SidebandConnection] - else - [{"intent" => intent.to_s}, OpenAI::Realtime::TranscriptionConnection] - end - end - # @api private # # @param client [OpenAI::Client] diff --git a/rbi/openai/resources/realtime.rbi b/rbi/openai/resources/realtime.rbi index 2c66113f6..9927621b4 100644 --- a/rbi/openai/resources/realtime.rbi +++ b/rbi/openai/resources/realtime.rbi @@ -14,46 +14,37 @@ module OpenAI sig do params( - model: T.nilable(String), - call_id: T.nilable(String), - intent: T.nilable(Symbol), + model: String, request_options: T.nilable(OpenAI::RequestOptions::OrHash), transport: T.untyped, transport_options: T::Hash[Symbol, T.untyped], - block: - T - .proc - .params( - connection: - T.any( - OpenAI::Realtime::Connection, - OpenAI::Realtime::SidebandConnection, - OpenAI::Realtime::TranscriptionConnection - ) - ) - .returns(T.untyped) + block: T.proc.params(connection: OpenAI::Realtime::Connection).returns(T.untyped) ).returns(T.untyped) end - def connect( - model: nil, - call_id: nil, - intent: nil, - request_options: nil, - transport: nil, - transport_options: {}, - &block - ) + def connect(model:, request_options: nil, transport: nil, transport_options: {}, &block) + end + + sig do + params( + call_id: String, + request_options: T.nilable(OpenAI::RequestOptions::OrHash), + transport: T.untyped, + transport_options: T::Hash[Symbol, T.untyped], + block: T.proc.params(connection: OpenAI::Realtime::SidebandConnection).returns(T.untyped) + ).returns(T.untyped) + end + def connect_to_call(call_id:, request_options: nil, transport: nil, transport_options: {}, &block) end - # @api private sig do params( - model: T.nilable(String), - call_id: T.nilable(String), - intent: T.nilable(Symbol) - ).returns(T::Array[T.untyped]) + request_options: T.nilable(OpenAI::RequestOptions::OrHash), + transport: T.untyped, + transport_options: T::Hash[Symbol, T.untyped], + block: T.proc.params(connection: OpenAI::Realtime::TranscriptionConnection).returns(T.untyped) + ).returns(T.untyped) end - def connection_target(model:, call_id:, intent:) + def connect_transcription(request_options: nil, transport: nil, transport_options: {}, &block) end # @api private diff --git a/realtime.md b/realtime.md index 2abd69f7f..798d2ad13 100644 --- a/realtime.md +++ b/realtime.md @@ -101,10 +101,10 @@ audio and interruption-driven truncation events share exactly one writer fiber. The yielded class reflects the protocol's capabilities: - `connect(model:)` yields `OpenAI::Realtime::Connection` for ordinary sessions; -- `connect(intent: :transcription)` yields +- `connect_transcription` yields `OpenAI::Realtime::TranscriptionConnection`, exposing only session and input audio buffer operations; -- `connect(call_id:)` yields `OpenAI::Realtime::SidebandConnection`, adding +- `connect_to_call(call_id:)` yields `OpenAI::Realtime::SidebandConnection`, adding `output_audio_buffer`; and - `translations.connect` yields `OpenAI::Realtime::TranslationConnection`, which intentionally has no conversation or response lifecycle. @@ -121,7 +121,7 @@ Realtime transcription is a distinct session mode. Configure a transcription model, append raw mono 24 kHz PCM16 input, and commit the buffer: ```ruby -client.realtime.connect(intent: :transcription) do |connection| +client.realtime.connect_transcription do |connection| connection.session.update( audio: { input: { @@ -189,7 +189,7 @@ from the `Location` header. Keep the call ID if the application will open a server-side sideband connection: ```ruby -client.realtime.connect(call_id: call.call_id) do |connection| +client.realtime.connect_to_call(call_id: call.call_id) do |connection| connection.session.update( type: :realtime, instructions: "Apply private server policy." @@ -221,7 +221,7 @@ client.realtime.calls.accept( instructions: "You are answering a phone call." ) -client.realtime.connect(call_id: call_id) do |connection| +client.realtime.connect_to_call(call_id: call_id) do |connection| connection.each { |realtime_event| handle(realtime_event) } end ``` @@ -313,15 +313,23 @@ safe for that use case. Runnable smoke tests require their protocol-specific terminal event rather than treating a clean WebSocket EOF as success: completed `response.done` for text, -audio, and MCP; transcription completion for transcription; and -`session.closed` for translation. Cleanup must also preserve an active upload or -processing error when a graceful close fails, while surfacing the close failure -after an otherwise successful operation. +audio, and MCP; transcription completion for transcription; `session.closed` +for translation; and the requested `OPENAI_REALTIME_STOP_AFTER` checkpoint for +bounded sideband, SIP, and conversation runs. Cleanup must also preserve an +active upload or processing error when a graceful close fails, while surfacing +the close failure after an otherwise successful operation. Terminal status alone is insufficient when a workflow promises an artifact: -the raw-audio smoke also requires at least one decoded audio byte before its -completed `response.done`. Bounded conversation playback similarly requires the -explicit event requested by `stop_after`; EOF before it is a failed run. +the raw-audio and translation smokes also require at least one decoded audio +byte before their terminal events. Microphone shutdown is latched before capture +starts, so an immediately closed connection cannot start an orphaned ffmpeg +process after cleanup has begun. + +The three standard Realtime connection modes use distinct method names rather +than a keyword-discriminated overload: `connect(model:)`, `connect_to_call`, and +`connect_transcription`. This keeps block parameter types precise in both RBS +and RBI; Sorbet does not support overloads that discriminate on keyword +arguments. Raw WebRTC SDP responses remain lazily consumed. Request observability follows that body lifecycle: completion is recorded only after the SDP body is fully @@ -393,7 +401,7 @@ simplest successful path before protocol details: | --- | --- | | Realtime overview | Minimal text WebSocket with typed deltas and block cleanup | | WebSocket | Text, PCM16 input/output, manual `receive`, Enumerable iteration, error handling | -| Transcription | `intent: :transcription`, PCM16 streaming, delta/completed events, and `item_id` correlation | +| Transcription | `connect_transcription`, PCM16 streaming, delta/completed events, and `item_id` correlation | | WebRTC | Complete browser peer plus Ruby endpoint; Rails and Sinatra variants using `calls.create` | | Server controls | Preserve `call.call_id`, open sideband, update session, hang up | | SIP | Verify webhook, accept/reject, sideband, refer, hang up, idempotent webhook handling | diff --git a/sig/openai/resources/realtime.rbs b/sig/openai/resources/realtime.rbs index b7bf1007f..c7d74d7cc 100644 --- a/sig/openai/resources/realtime.rbs +++ b/sig/openai/resources/realtime.rbs @@ -7,43 +7,25 @@ module OpenAI attr_reader translations: OpenAI::Resources::Realtime::Translations - def connect: - ( - model: String, - ?call_id: nil, - ?intent: nil, - ?request_options: OpenAI::request_opts?, - ?transport: untyped?, - ?transport_options: ::Hash[Symbol, untyped] - ) { - (OpenAI::Realtime::Connection connection) -> top - } -> top - | ( - call_id: String, - ?model: nil, - ?intent: nil, - ?request_options: OpenAI::request_opts?, - ?transport: untyped?, - ?transport_options: ::Hash[Symbol, untyped] - ) { - (OpenAI::Realtime::SidebandConnection connection) -> top - } -> top - | ( - intent: :transcription, - ?model: nil, - ?call_id: nil, - ?request_options: OpenAI::request_opts?, - ?transport: untyped?, - ?transport_options: ::Hash[Symbol, untyped] - ) { - (OpenAI::Realtime::TranscriptionConnection connection) -> top - } -> top + def connect: ( + model: String, + ?request_options: OpenAI::request_opts?, + ?transport: untyped?, + ?transport_options: ::Hash[Symbol, untyped] + ) { (OpenAI::Realtime::Connection connection) -> top } -> top - def connection_target: ( - model: String?, - call_id: String?, - intent: (:transcription)? - ) -> [::Hash[String, String], untyped] + def connect_to_call: ( + call_id: String, + ?request_options: OpenAI::request_opts?, + ?transport: untyped?, + ?transport_options: ::Hash[Symbol, untyped] + ) { (OpenAI::Realtime::SidebandConnection connection) -> top } -> top + + def connect_transcription: ( + ?request_options: OpenAI::request_opts?, + ?transport: untyped?, + ?transport_options: ::Hash[Symbol, untyped] + ) { (OpenAI::Realtime::TranscriptionConnection connection) -> top } -> top def initialize: (client: OpenAI::Client) -> void end diff --git a/test/openai/realtime/async_websocket_transport_test.rb b/test/openai/realtime/async_websocket_transport_test.rb index 744b5109d..c2e101471 100644 --- a/test/openai/realtime/async_websocket_transport_test.rb +++ b/test/openai/realtime/async_websocket_transport_test.rb @@ -531,7 +531,7 @@ def test_missing_optional_dependency_preserves_load_error_as_the_cause end private def exercise_transcription(client, audio:) - client.realtime.connect(intent: :transcription) do |connection| + client.realtime.connect_transcription do |connection| assert_instance_of(OpenAI::Realtime::SessionCreatedEvent, connection.receive) configure_transcription(connection) assert_instance_of(OpenAI::Realtime::SessionUpdatedEvent, connection.receive) diff --git a/test/openai/realtime/connection_test.rb b/test/openai/realtime/connection_test.rb index 9a774c782..e75010054 100644 --- a/test/openai/realtime/connection_test.rb +++ b/test/openai/realtime/connection_test.rb @@ -91,36 +91,11 @@ def test_connect_opens_a_typed_block_scoped_connection assert(socket.closed?) end - def test_connect_requires_exactly_one_connection_target - error = assert_raises(ArgumentError) do - client.realtime.connect(transport: FakeTransport.new(FakeSocket.new)) { |_connection| nil } - end - assert_includes(error.message, "exactly one") - - error = assert_raises(ArgumentError) do - client.realtime.connect( - model: "gpt-realtime", - call_id: "rtc_123", - transport: FakeTransport.new(FakeSocket.new) - ) { |_connection| nil } - end - assert_includes(error.message, "exactly one") - - error = assert_raises(ArgumentError) do - client.realtime.connect( - model: "gpt-realtime", - intent: :transcription, - transport: FakeTransport.new(FakeSocket.new) - ) { |_connection| nil } - end - assert_includes(error.message, "exactly one") - end - def test_connect_to_a_dedicated_transcription_session socket = FakeSocket.new transport = FakeTransport.new(socket) - client.realtime.connect(intent: :transcription, transport: transport) do |connection| + client.realtime.connect_transcription(transport: transport) do |connection| assert_instance_of(OpenAI::Realtime::TranscriptionConnection, connection) assert_instance_of(OpenAI::Realtime::ConnectionResources::TranscriptionSession, connection.session) assert_respond_to(connection.input_audio_buffer, :commit) @@ -141,21 +116,10 @@ def test_connect_to_a_dedicated_transcription_session refute(update.fetch(:session).key?(:event_id)) end - def test_connect_rejects_an_unknown_intent - error = assert_raises(ArgumentError) do - client.realtime.connect( - intent: :future, - transport: FakeTransport.new(FakeSocket.new) - ) { |_connection| nil } - end - - assert_includes(error.message, "transcription") - end - def test_connect_to_an_existing_webrtc_or_sip_call transport = FakeTransport.new(FakeSocket.new) - client.realtime.connect(call_id: "rtc_123", transport: transport) do |connection| + client.realtime.connect_to_call(call_id: "rtc_123", transport: transport) do |connection| assert_instance_of(OpenAI::Realtime::SidebandConnection, connection) assert_respond_to(connection, :output_audio_buffer) end @@ -317,7 +281,7 @@ def test_sideband_connection_exposes_output_audio_buffer_controls socket = FakeSocket.new transport = FakeTransport.new(socket) - client.realtime.connect(call_id: "rtc_123", transport: transport) do |connection| + client.realtime.connect_to_call(call_id: "rtc_123", transport: transport) do |connection| connection.output_audio_buffer.clear(event_id: "clear_1") end @@ -331,7 +295,7 @@ def test_sideband_interruption_helpers_preserve_documented_order socket = FakeSocket.new transport = FakeTransport.new(socket) - client.realtime.connect(call_id: "rtc_123", transport: transport) do |connection| + client.realtime.connect_to_call(call_id: "rtc_123", transport: transport) do |connection| connection.response.cancel connection.conversation.items.truncate(item_id: "item_1", content_index: 0, audio_end_ms: 640) connection.output_audio_buffer.clear diff --git a/test/openai/realtime/example_stream_lifecycle_test.rb b/test/openai/realtime/example_stream_lifecycle_test.rb index 26e258128..4c8744186 100644 --- a/test/openai/realtime/example_stream_lifecycle_test.rb +++ b/test/openai/realtime/example_stream_lifecycle_test.rb @@ -2,6 +2,8 @@ require_relative "../test_helper" require_relative "../../../examples/realtime/mcp_approval" +require_relative "../../../examples/realtime/sideband" +require_relative "../../../examples/realtime/sip" require_relative "../../../examples/realtime/translation" require_relative "../../../examples/realtime/websocket_audio" require_relative "../../../examples/realtime/websocket_transcription" @@ -38,6 +40,34 @@ def close TranslationConnection = Data.define(:input_audio_buffer, :session) + Event = Data.define(:type, :data) do + def to_h = data + end + + def test_sideband_rejects_eof_before_the_requested_event + error = assert_raises(RuntimeError) do + OpenAI::Examples::Realtime::Sideband.stream( + RecordingConnection.new, + output: StringIO.new, + stop_after: "session.updated" + ) + end + + assert_equal("Realtime connection closed before session.updated", error.message) + end + + def test_sip_rejects_eof_before_the_requested_event + error = assert_raises(RuntimeError) do + OpenAI::Examples::Realtime::SIP.stream( + RecordingConnection.new, + output: StringIO.new, + stop_after: "response.done" + ) + end + + assert_equal("Realtime connection closed before response.done", error.message) + end + def test_websocket_audio_rejects_a_connection_that_closes_before_response_done error = assert_raises(RuntimeError) do OpenAI::Examples::Realtime::WebSocketAudio.stream_response( @@ -119,7 +149,7 @@ def test_translation_rejects_a_connection_that_closes_before_session_closed ) end - def test_translation_accepts_only_a_session_closed_event + def test_translation_rejects_session_closed_without_audio connection = RecordingConnection.new( [ OpenAI::Realtime::RealtimeTranslationSessionClosedEvent.new( @@ -128,14 +158,41 @@ def test_translation_accepts_only_a_session_closed_event ] ) + error = assert_raises(RuntimeError) do + OpenAI::Examples::Realtime::Translation.stream( + connection, + audio_output: StringIO.new, + transcript_output: StringIO.new + ) + end + + assert_equal("Translation session closed without audio output", error.message) + end + + def test_translation_accepts_audio_followed_by_session_closed + audio = "translated pcm".b + connection = RecordingConnection.new( + [ + OpenAI::Realtime::RealtimeTranslationOutputAudioDeltaEvent.new( + event_id: "event_1", + delta: Base64.strict_encode64(audio) + ), + OpenAI::Realtime::RealtimeTranslationSessionClosedEvent.new( + event_id: "event_2" + ) + ] + ) + audio_output = StringIO.new transcript_output = StringIO.new + result = OpenAI::Examples::Realtime::Translation.stream( connection, - audio_output: StringIO.new, + audio_output: audio_output, transcript_output: transcript_output ) assert_nil(result) + assert_equal(audio, audio_output.string) assert_equal("\n", transcript_output.string) end diff --git a/test/openai/realtime/examples_test.rb b/test/openai/realtime/examples_test.rb index 50fbefbcb..84b682472 100644 --- a/test/openai/realtime/examples_test.rb +++ b/test/openai/realtime/examples_test.rb @@ -137,7 +137,7 @@ def initialize(connection) @connections = [] end - def connect(call_id:) + def connect_to_call(call_id:) @connections << call_id yield(@connection) end @@ -350,6 +350,32 @@ def test_realtime_conversation_propagates_writer_failure_without_deadlock assert_predicate(microphone, :stopped) end + def test_realtime_conversation_latches_stop_before_ffmpeg_capture_starts + connection = RecordingConnection.new( + [OpenAI::Realtime::SessionUpdatedEvent.new(event_id: "event_1", session: {})] + ) + microphone = OpenAI::Examples::Realtime::Conversation::FFmpegMicrophone.new( + input_format: "test", + device: "test" + ) + spawn = ->(*) { raise "microphone spawned after stop" } + + result = Process.stub(:spawn, spawn) do + Sync do + OpenAI::Examples::Realtime::Conversation.run_session( + connection, + microphone: microphone, + speaker: RecordingSpeaker.new, + voice: :marin, + instructions: "Speak naturally.", + output: StringIO.new + ) + end + end + + assert_nil(result) + end + def test_realtime_conversation_rejects_eof_before_the_requested_stop_event microphone = RecordingMicrophone.new([]) playback = OpenAI::Examples::Realtime::Conversation::AudioPlayback.new( From 0279a5a6bbe7d214fa4b40b9f42d1a294d34394e Mon Sep 17 00:00:00 2001 From: Justin Beckwith Date: Tue, 11 Aug 2026 22:50:15 -0700 Subject: [PATCH 7/8] Fix Realtime lifecycle review findings --- examples/realtime/README.md | 9 +- examples/realtime/event_stream.rb | 9 + examples/realtime/realtime_conversation.rb | 14 +- examples/realtime/translation.rb | 54 +++--- examples/realtime/websocket_audio.rb | 14 ++ examples/realtime/websocket_text.rb | 3 + examples/realtime/websocket_transcription.rb | 16 +- lib/openai/resources/realtime/calls.rb | 11 +- realtime.md | 16 +- .../realtime/example_stream_lifecycle_test.rb | 154 ++++++++++++++++++ test/openai/realtime/examples_test.rb | 80 ++++++--- test/openai/resources/realtime/calls_test.rb | 28 ++++ 12 files changed, 333 insertions(+), 75 deletions(-) diff --git a/examples/realtime/README.md b/examples/realtime/README.md index 6c8dad131..7c9a57ad8 100644 --- a/examples/realtime/README.md +++ b/examples/realtime/README.md @@ -82,7 +82,8 @@ bundle exec ruby examples/realtime/websocket_text.rb A successful run prints `session.created`, `session.updated`, streamed assistant text, and `response.done status=completed`. An EOF before the completed -`response.done` is reported as a failed smoke test. +`response.done`, or a completed response without a non-empty text delta, is +reported as a failed smoke test. ## WebSocket audio @@ -105,7 +106,9 @@ ffplay -f s16le -ar 24000 -ch_layout mono response.pcm A successful run exits with status 0 only after writing audio bytes and seeing a completed `response.done`. EOF before completion, or completion without an -audio delta, is reported as a failed smoke test. +audio delta, is reported as a failed smoke test. The example disables automatic +turn detection and waits for `session.updated` before uploading, so its explicit +buffer commit and `response.create` cannot race server VAD. ## Realtime transcription @@ -143,6 +146,8 @@ A successful run prints the translated transcript, exits after incomplete instead of silently succeeding. If upload and graceful close both fail, the upload error remains the primary failure so the interrupted input operation is not hidden by cleanup. +If the reader fails while input is still being uploaded, the example cancels +the uploader immediately instead of draining the rest of the file. ## MCP approval diff --git a/examples/realtime/event_stream.rb b/examples/realtime/event_stream.rb index 15caa9945..4f461fd84 100644 --- a/examples/realtime/event_stream.rb +++ b/examples/realtime/event_stream.rb @@ -17,6 +17,15 @@ def each_until(connection, stop_after: nil, closed_message: nil) raise(closed_message || "Realtime connection closed before #{stop_after}") end + + def wait_for(connection, event_class, closed_message:) + while (event = connection.receive) + raise event.error.message if event.is_a?(OpenAI::Realtime::RealtimeErrorEvent) + return event if event.is_a?(event_class) + end + + raise closed_message + end end end end diff --git a/examples/realtime/realtime_conversation.rb b/examples/realtime/realtime_conversation.rb index a80a570d7..1a49b6a08 100755 --- a/examples/realtime/realtime_conversation.rb +++ b/examples/realtime/realtime_conversation.rb @@ -398,14 +398,6 @@ def handle_event(event, outbound:, playback:, output:) end end - def wait_until_ready(connection) - while (event = connection.receive) - return if event.is_a?(OpenAI::Realtime::SessionUpdatedEvent) - raise event.error.message if event.is_a?(OpenAI::Realtime::RealtimeErrorEvent) - end - raise "Realtime connection closed before session.updated" - end - def stream_events(connection, microphone:, outbound:, playback:, output:, stop_after: nil) EventStream.each_until(connection, stop_after: stop_after) do |event| handle_event(event, outbound: outbound, playback: playback, output: output) @@ -424,7 +416,11 @@ def run_session( stop_after: nil ) configure(connection, voice: voice, instructions: instructions) - wait_until_ready(connection) + EventStream.wait_for( + connection, + OpenAI::Realtime::SessionUpdatedEvent, + closed_message: "Realtime connection closed before session.updated" + ) output.puts( "Connected. Talk naturally; speak over the model to interrupt. " \ "Press Ctrl-C to exit. Use headphones to prevent echo." diff --git a/examples/realtime/translation.rb b/examples/realtime/translation.rb index fb737bf43..c07566380 100755 --- a/examples/realtime/translation.rb +++ b/examples/realtime/translation.rb @@ -37,19 +37,13 @@ def stream(connection, audio_output:, transcript_output: $stdout) end def write_input(connection, input_path) - upload_error = nil - begin - File.open(input_path, "rb") do |input| - while (chunk = input.read(9_600)) - connection.input_audio_buffer.append_bytes(chunk) - end + File.open(input_path, "rb") do |input| + while (chunk = input.read(9_600)) + connection.input_audio_buffer.append_bytes(chunk) end - rescue StandardError => e - upload_error = e - raise - ensure - close_session(connection, preserve_error: !upload_error.nil?) end + ensure + close_session(connection, preserve_error: !$ERROR_INFO.nil?) end def close_session(connection, preserve_error:) @@ -58,19 +52,39 @@ def close_session(connection, preserve_error:) raise unless preserve_error end + def exchange(connection, input_path:, audio_output:, transcript_output:) + uploader = nil + reader = Async do + stream( + connection, + audio_output: audio_output, + transcript_output: transcript_output + ) + nil + rescue StandardError => e + uploader&.stop + e + end + uploader = Async { write_input(connection, input_path) } + uploader.stop if reader.finished? + uploader.wait + reader_error = reader.wait + raise reader_error if reader_error + ensure + uploader&.stop + reader&.stop + end + def run(client:, model:, input_path:, output_path:, target_language:, transcript_output: $stdout) File.open(output_path, "wb") do |audio_output| client.realtime.translations.connect(model: model) do |connection| connection.session.update(audio: {output: {language: target_language}}) - reader = Async do - stream( - connection, - audio_output: audio_output, - transcript_output: transcript_output - ) - end - write_input(connection, input_path) - reader.wait + exchange( + connection, + input_path: input_path, + audio_output: audio_output, + transcript_output: transcript_output + ) end end end diff --git a/examples/realtime/websocket_audio.rb b/examples/realtime/websocket_audio.rb index c7b870d1b..c8c08aeaa 100755 --- a/examples/realtime/websocket_audio.rb +++ b/examples/realtime/websocket_audio.rb @@ -10,6 +10,14 @@ module Realtime module WebSocketAudio module_function + def configure(connection) + connection.session.update( + type: :realtime, + output_modalities: [:audio], + audio: {input: {turn_detection: nil}} + ) + end + def stream_response(connection, output:) audio_bytes = 0 EventStream.each_until(connection, stop_after: "response.done") do |event| @@ -31,6 +39,12 @@ def stream_response(connection, output:) def run(client:, model:, input_path:, output_path:) File.open(output_path, "wb") do |output| client.realtime.connect(model: model) do |connection| + configure(connection) + EventStream.wait_for( + connection, + OpenAI::Realtime::SessionUpdatedEvent, + closed_message: "Realtime connection closed before session.updated" + ) File.open(input_path, "rb") do |input| while (chunk = input.read(4_800)) connection.input_audio_buffer.append_bytes(chunk) diff --git a/examples/realtime/websocket_text.rb b/examples/realtime/websocket_text.rb index 72b33a4d2..1f771e5fc 100755 --- a/examples/realtime/websocket_text.rb +++ b/examples/realtime/websocket_text.rb @@ -13,6 +13,7 @@ module WebSocketText def stream_response(connection, output: $stdout) started_response = false + received_text = false EventStream.each_until(connection, stop_after: "response.done") do |event| case event when OpenAI::Realtime::SessionCreatedEvent @@ -22,12 +23,14 @@ def stream_response(connection, output: $stdout) when OpenAI::Realtime::ResponseTextDeltaEvent output.print("[assistant] ") unless started_response started_response = true + received_text = true unless event.delta.empty? output.print(event.delta) output.flush when OpenAI::Realtime::ResponseDoneEvent output.puts if started_response status = event.response.status raise "Realtime response ended with status #{status.inspect}" unless status == :completed + raise "Realtime response completed without text output" unless received_text output.puts("[realtime] response.done status=completed") when OpenAI::Realtime::RealtimeErrorEvent diff --git a/examples/realtime/websocket_transcription.rb b/examples/realtime/websocket_transcription.rb index 9c55de1c7..c9c70889d 100755 --- a/examples/realtime/websocket_transcription.rb +++ b/examples/realtime/websocket_transcription.rb @@ -23,16 +23,6 @@ def configure(connection, transcription_model:) ) end - def wait_until_ready(connection) - loop do - event = connection.receive - raise "Realtime connection closed before session.updated" if event.nil? - raise event.error.message if event.is_a?(OpenAI::Realtime::RealtimeErrorEvent) - - return if event.is_a?(OpenAI::Realtime::SessionUpdatedEvent) - end - end - def upload(connection, input_path:) File.open(input_path, "rb") do |input| while (chunk = input.read(4_800)) @@ -63,7 +53,11 @@ def print_transcript(connection, output: $stdout) def run(client:, input_path:, transcription_model:, output: $stdout) client.realtime.connect_transcription do |connection| configure(connection, transcription_model: transcription_model) - wait_until_ready(connection) + EventStream.wait_for( + connection, + OpenAI::Realtime::SessionUpdatedEvent, + closed_message: "Realtime connection closed before session.updated" + ) upload(connection, input_path: input_path) print_transcript(connection, output: output) end diff --git a/lib/openai/resources/realtime/calls.rb b/lib/openai/resources/realtime/calls.rb index 4d02f9790..ef06734da 100644 --- a/lib/openai/resources/realtime/calls.rb +++ b/lib/openai/resources/realtime/calls.rb @@ -42,7 +42,7 @@ def create(params) options: options ) location = response.headers["location"] - call_id = location&.then { URI.parse(_1).path.split("/").last } + call_id = call_id_from_location(location) OpenAI::Realtime::CallCreateResponse.new( sdp: response.body.to_a.join, call_id: call_id, @@ -50,6 +50,15 @@ def create(params) )._set_last_response(response.metadata) end + # A Location header is helpful metadata, but the SDP answer remains usable + # when an intermediary supplies a malformed value. + private def call_id_from_location(location) + path = location&.then { URI.parse(_1).path } + path&.split("/")&.last + rescue URI::InvalidURIError + nil + end + # Some parameter documentations has been truncated, see # {OpenAI::Models::Realtime::CallAcceptParams} for more details. # diff --git a/realtime.md b/realtime.md index 798d2ad13..1a62164af 100644 --- a/realtime.md +++ b/realtime.md @@ -185,8 +185,9 @@ render plain: call.sdp, content_type: "application/sdp" ``` The result preserves `sdp`, normalized response `headers`, and `call_id` parsed -from the `Location` header. Keep the call ID if the application will open a -server-side sideband connection: +from the optional `Location` header. A missing or malformed `Location` leaves +`call_id` as `nil` without discarding an otherwise valid SDP answer. Keep the +call ID if the application will open a server-side sideband connection: ```ruby client.realtime.connect_to_call(call_id: call.call_id) do |connection| @@ -320,10 +321,13 @@ active upload or processing error when a graceful close fails, while surfacing the close failure after an otherwise successful operation. Terminal status alone is insufficient when a workflow promises an artifact: -the raw-audio and translation smokes also require at least one decoded audio -byte before their terminal events. Microphone shutdown is latched before capture -starts, so an immediately closed connection cannot start an orphaned ffmpeg -process after cleanup has begun. +the text smoke requires a non-empty text delta, while the raw-audio and +translation smokes require at least one decoded audio byte before their terminal +events. The file-audio smoke disables VAD and waits for the acknowledged session +update before its manual commit. Translation reader failures cancel an in-flight +upload rather than waiting for the input file to drain. Microphone shutdown is +latched before capture starts, so an immediately closed connection cannot start +an orphaned ffmpeg process after cleanup has begun. The three standard Realtime connection modes use distinct method names rather than a keyword-discriminated overload: `connect(model:)`, `connect_to_call`, and diff --git a/test/openai/realtime/example_stream_lifecycle_test.rb b/test/openai/realtime/example_stream_lifecycle_test.rb index 4c8744186..0ab7e418a 100644 --- a/test/openai/realtime/example_stream_lifecycle_test.rb +++ b/test/openai/realtime/example_stream_lifecycle_test.rb @@ -1,11 +1,13 @@ # frozen_string_literal: true require_relative "../test_helper" +require "async/notification" require_relative "../../../examples/realtime/mcp_approval" require_relative "../../../examples/realtime/sideband" require_relative "../../../examples/realtime/sip" require_relative "../../../examples/realtime/translation" require_relative "../../../examples/realtime/websocket_audio" +require_relative "../../../examples/realtime/websocket_text" require_relative "../../../examples/realtime/websocket_transcription" class OpenAI::Test::RealtimeExampleStreamLifecycleTest < Minitest::Test @@ -17,6 +19,8 @@ def initialize(events = []) def each(&block) @events.each(&block) end + + def receive = @events.shift end class RaisingEndpoint @@ -38,12 +42,74 @@ def close end end + class ClosingSession + attr_reader :closed + + def close = @closed = true + end + + class BlockingAudioBuffer + attr_reader :chunks + + def initialize(upload_started) + @upload_started = upload_started + @chunks = [] + end + + def append_bytes(bytes) + @chunks << bytes + @upload_started.signal + Kernel.sleep(3_600) + end + end + + class ReaderFailureConnection + attr_reader :input_audio_buffer, :session + + def initialize(event) + upload_started = Async::Notification.new + @event = event + @input_audio_buffer = BlockingAudioBuffer.new(upload_started) + @session = ClosingSession.new + @upload_started = upload_started + end + + def each + @upload_started.wait + yield(@event) + end + end + TranslationConnection = Data.define(:input_audio_buffer, :session) Event = Data.define(:type, :data) do def to_h = data end + def test_wait_for_rejects_eof_before_the_target_event + error = assert_raises(RuntimeError) do + OpenAI::Examples::Realtime::EventStream.wait_for( + RecordingConnection.new, + OpenAI::Realtime::SessionUpdatedEvent, + closed_message: "Realtime connection closed before session.updated" + ) + end + + assert_equal("Realtime connection closed before session.updated", error.message) + end + + def test_wait_for_surfaces_an_api_error_before_the_target_event + error = assert_raises(RuntimeError) do + OpenAI::Examples::Realtime::EventStream.wait_for( + RecordingConnection.new([realtime_error_event("session update failed")]), + OpenAI::Realtime::SessionUpdatedEvent, + closed_message: "Realtime connection closed before session.updated" + ) + end + + assert_equal("session update failed", error.message) + end + def test_sideband_rejects_eof_before_the_requested_event error = assert_raises(RuntimeError) do OpenAI::Examples::Realtime::Sideband.stream( @@ -134,6 +200,51 @@ def test_websocket_audio_rejects_a_completed_response_without_audio assert_equal("Response completed without audio output", error.message) end + def test_websocket_text_rejects_a_connection_that_closes_before_response_done + error = assert_raises(RuntimeError) do + OpenAI::Examples::Realtime::WebSocketText.stream_response( + RecordingConnection.new, + output: StringIO.new + ) + end + + assert_equal("Realtime connection closed before response.done", error.message) + end + + def test_websocket_text_rejects_a_completed_response_without_text + connection = RecordingConnection.new([completed_response_event]) + output = StringIO.new + + error = assert_raises(RuntimeError) do + OpenAI::Examples::Realtime::WebSocketText.stream_response(connection, output: output) + end + + assert_equal("Realtime response completed without text output", error.message) + refute_includes(output.string, "response.done status=completed") + end + + def test_websocket_text_accepts_non_empty_text_and_a_completed_response + connection = RecordingConnection.new( + [ + OpenAI::Realtime::ResponseTextDeltaEvent.new( + content_index: 0, + delta: "Hello from Ruby.", + event_id: "event_1", + item_id: "item_1", + output_index: 0, + response_id: "response_1" + ), + completed_response_event + ] + ) + output = StringIO.new + + OpenAI::Examples::Realtime::WebSocketText.stream_response(connection, output: output) + + assert_includes(output.string, "Hello from Ruby.") + assert_includes(output.string, "response.done status=completed") + end + def test_translation_rejects_a_connection_that_closes_before_session_closed error = assert_raises(RuntimeError) do OpenAI::Examples::Realtime::Translation.stream( @@ -249,4 +360,47 @@ def test_translation_preserves_an_upload_error_when_session_close_also_fails assert_equal([[:append_bytes, "audio"]], input_audio_buffer.calls) assert_equal([[:close]], session.calls) end + + def test_translation_stops_uploading_when_the_reader_fails + connection = ReaderFailureConnection.new(realtime_error_event("translation failed")) + + Tempfile.create("translation-input") do |input| + input.write("audio") + input.flush + + error = Timeout.timeout(1) do + Sync do + assert_raises(RuntimeError) do + OpenAI::Examples::Realtime::Translation.exchange( + connection, + input_path: input.path, + audio_output: StringIO.new, + transcript_output: StringIO.new + ) + end + end + end + + assert_equal("translation failed", error.message) + end + assert_equal(["audio"], connection.input_audio_buffer.chunks) + assert(connection.session.closed) + end + + private def completed_response_event + OpenAI::Realtime::ResponseDoneEvent.new( + event_id: "event_done", + response: OpenAI::Realtime::RealtimeResponse.new( + id: "response_done", + status: :completed + ) + ) + end + + private def realtime_error_event(message) + OpenAI::Realtime::RealtimeErrorEvent.new( + event_id: "event_error", + error: OpenAI::Realtime::RealtimeError.new(type: "server_error", message: message) + ) + end end diff --git a/test/openai/realtime/examples_test.rb b/test/openai/realtime/examples_test.rb index 84b682472..11666713a 100644 --- a/test/openai/realtime/examples_test.rb +++ b/test/openai/realtime/examples_test.rb @@ -7,7 +7,7 @@ require_relative "../../../examples/realtime/sideband" require_relative "../../../examples/realtime/sip" require_relative "../../../examples/realtime/webrtc_conversation" -require_relative "../../../examples/realtime/websocket_text" +require_relative "../../../examples/realtime/websocket_audio" class OpenAI::Test::RealtimeExamplesTest < Minitest::Test Event = Data.define(:type, :data) do @@ -55,10 +55,11 @@ def initialize(writer_fibers = nil) class RecordingAudioBuffer attr_accessor :append_error - attr_reader :chunks + attr_reader :chunks, :commits def initialize(writer_fibers = nil) @chunks = [] + @commits = 0 @writer_fibers = writer_fibers end @@ -68,6 +69,8 @@ def append_bytes(bytes) @chunks << bytes end + + def commit = @commits += 1 end class RecordingOutbound @@ -83,10 +86,11 @@ def truncate(**params) = @truncations << params end class RecordingConnection - attr_reader :conversation, :input_audio_buffer, :response, :session, :writer_fibers + attr_reader :conversation, :input_audio_buffer, :received, :response, :session, :writer_fibers def initialize(events = []) @events = events + @received = [] @writer_fibers = [] @session = RecordingSession.new @conversation = RecordingConversation.new(@writer_fibers) @@ -99,7 +103,7 @@ def each(&block) end def receive - @events.shift + @events.shift.tap { |event| @received << event&.type } end end @@ -141,6 +145,11 @@ def connect_to_call(call_id:) @connections << call_id yield(@connection) end + + def connect(model:) + @connections << model + yield(@connection) + end end RecordingClient = Data.define(:realtime) @@ -920,34 +929,53 @@ def test_sip_example_reports_a_cleanup_failure_after_a_clean_session assert_equal(["rtc_123"], realtime.calls.hangups) end - def test_websocket_text_rejects_a_connection_that_closes_before_response_done - error = assert_raises(RuntimeError) do - OpenAI::Examples::Realtime::WebSocketText.stream_response( - RecordingConnection.new, - output: StringIO.new - ) - end - - assert_equal("Realtime connection closed before response.done", error.message) - end - - def test_websocket_text_accepts_only_a_completed_response + def test_websocket_audio_disables_vad_and_waits_for_session_updated + audio = "assistant pcm".b connection = RecordingConnection.new( [ - OpenAI::Realtime::ResponseDoneEvent.new( - event_id: "event_1", - response: OpenAI::Realtime::RealtimeResponse.new( - id: "response_1", - status: :completed - ) - ) + OpenAI::Realtime::SessionUpdatedEvent.new(event_id: "event_1", session: {}), + OpenAI::Realtime::ResponseAudioDeltaEvent.new( + event_id: "event_2", + response_id: "response_1", + item_id: "item_1", + output_index: 0, + content_index: 0, + delta: Base64.strict_encode64(audio) + ), + completed_response_event ] ) - output = StringIO.new + realtime = RecordingRealtime.new(connection) + + Tempfile.create("realtime-input") do |input| + input.write("input pcm") + input.flush + Tempfile.create("realtime-output") do |output| + _stdout, _stderr = capture_io do + OpenAI::Examples::Realtime::WebSocketAudio.run( + client: RecordingClient.new(realtime: realtime), + model: "gpt-realtime-2.1", + input_path: input.path, + output_path: output.path + ) + end - OpenAI::Examples::Realtime::WebSocketText.stream_response(connection, output: output) + assert_equal(audio, File.binread(output.path)) + end + end - assert_includes(output.string, "response.done status=completed") + assert_equal( + { + type: :realtime, + output_modalities: [:audio], + audio: {input: {turn_detection: nil}} + }, + connection.session.updates.fetch(0) + ) + assert_equal(["input pcm"], connection.input_audio_buffer.chunks) + assert_equal(1, connection.input_audio_buffer.commits) + assert_equal([{}], connection.response.calls) + assert_equal([:"session.updated"], connection.received) end private def completed_response_event diff --git a/test/openai/resources/realtime/calls_test.rb b/test/openai/resources/realtime/calls_test.rb index 724cb3d43..01b40ec12 100644 --- a/test/openai/resources/realtime/calls_test.rb +++ b/test/openai/resources/realtime/calls_test.rb @@ -31,6 +31,20 @@ def execute(request) end end + class MalformedLocationHTTPClient < OpenAI::HTTPClient + def execute(_request) + OpenAI::HTTPClient::Response.new( + status: 201, + headers: { + "Content-Type" => "application/sdp", + "Location" => "http://[malformed", + "X-Request-ID" => "req_malformed_location" + }, + body: "usable-answer-sdp" + ) + end + end + def test_create_with_an_sdp_offer http_client = StubHTTPClient.new client = OpenAI::Client.new( @@ -94,6 +108,20 @@ def test_create_preserves_request_observability_for_raw_sdp_responses assert_includes(log, "response body") end + def test_create_ignores_a_malformed_optional_location_header + client = OpenAI::Client.new( + api_key: "test-key", + base_url: "https://example.com/v1", + http_client: MalformedLocationHTTPClient.new + ) + + response = client.realtime.calls.create(sdp: "offer-sdp") + + assert_equal("usable-answer-sdp", response.sdp) + assert_nil(response.call_id) + assert_equal("req_malformed_location", response._request_id) + end + def test_accept_required_params response = @openai.realtime.calls.accept("call_id", type: :realtime) From d0174de591857fb78576997faa87b507d6048661 Mon Sep 17 00:00:00 2001 From: Justin Beckwith Date: Wed, 12 Aug 2026 09:35:58 -0700 Subject: [PATCH 8/8] Prevent eager translation upload after reader failure --- examples/realtime/README.md | 3 +- examples/realtime/translation.rb | 8 +++- realtime.md | 7 +-- .../realtime/example_stream_lifecycle_test.rb | 46 +++++++++++++++++++ 4 files changed, 59 insertions(+), 5 deletions(-) diff --git a/examples/realtime/README.md b/examples/realtime/README.md index 7c9a57ad8..40f6ea6a6 100644 --- a/examples/realtime/README.md +++ b/examples/realtime/README.md @@ -147,7 +147,8 @@ incomplete instead of silently succeeding. If upload and graceful close both fail, the upload error remains the primary failure so the interrupted input operation is not hidden by cleanup. If the reader fails while input is still being uploaded, the example cancels -the uploader immediately instead of draining the rest of the file. +the uploader immediately instead of draining the rest of the file. If the +reader has already failed on buffered input or EOF, upload never starts. ## MCP approval diff --git a/examples/realtime/translation.rb b/examples/realtime/translation.rb index c07566380..9292635a7 100755 --- a/examples/realtime/translation.rb +++ b/examples/realtime/translation.rb @@ -65,8 +65,14 @@ def exchange(connection, input_path:, audio_output:, transcript_output:) uploader&.stop e end + if reader.finished? + reader_error = reader.wait + raise reader_error if reader_error + + return + end + uploader = Async { write_input(connection, input_path) } - uploader.stop if reader.finished? uploader.wait reader_error = reader.wait raise reader_error if reader_error diff --git a/realtime.md b/realtime.md index 1a62164af..124c90a76 100644 --- a/realtime.md +++ b/realtime.md @@ -325,9 +325,10 @@ the text smoke requires a non-empty text delta, while the raw-audio and translation smokes require at least one decoded audio byte before their terminal events. The file-audio smoke disables VAD and waits for the acknowledged session update before its manual commit. Translation reader failures cancel an in-flight -upload rather than waiting for the input file to drain. Microphone shutdown is -latched before capture starts, so an immediately closed connection cannot start -an orphaned ffmpeg process after cleanup has begun. +upload rather than waiting for the input file to drain; an already-buffered +reader failure prevents the uploader from starting at all. Microphone shutdown +is latched before capture starts, so an immediately closed connection cannot +start an orphaned ffmpeg process after cleanup has begun. The three standard Realtime connection modes use distinct method names rather than a keyword-discriminated overload: `connect(model:)`, `connect_to_call`, and diff --git a/test/openai/realtime/example_stream_lifecycle_test.rb b/test/openai/realtime/example_stream_lifecycle_test.rb index 0ab7e418a..f60f5a034 100644 --- a/test/openai/realtime/example_stream_lifecycle_test.rb +++ b/test/openai/realtime/example_stream_lifecycle_test.rb @@ -48,6 +48,14 @@ class ClosingSession def close = @closed = true end + class RecordingEndpoint + attr_reader :calls + + def initialize = @calls = [] + def append_bytes(bytes) = @calls << [:append_bytes, bytes] + def close = @calls << [:close] + end + class BlockingAudioBuffer attr_reader :chunks @@ -80,6 +88,18 @@ def each end end + class BufferedReaderFailureConnection + attr_reader :input_audio_buffer, :session + + def initialize(event) + @event = event + @input_audio_buffer = RecordingEndpoint.new + @session = RecordingEndpoint.new + end + + def each = yield(@event) + end + TranslationConnection = Data.define(:input_audio_buffer, :session) Event = Data.define(:type, :data) do @@ -387,6 +407,32 @@ def test_translation_stops_uploading_when_the_reader_fails assert(connection.session.closed) end + def test_translation_does_not_start_upload_after_a_buffered_reader_failure + connection = BufferedReaderFailureConnection.new( + realtime_error_event("translation failed before upload") + ) + + Tempfile.create("translation-input") do |input| + input.write("audio") + input.flush + + error = Sync do + assert_raises(RuntimeError) do + OpenAI::Examples::Realtime::Translation.exchange( + connection, + input_path: input.path, + audio_output: StringIO.new, + transcript_output: StringIO.new + ) + end + end + + assert_equal("translation failed before upload", error.message) + end + assert_empty(connection.input_audio_buffer.calls) + assert_empty(connection.session.calls) + end + private def completed_response_event OpenAI::Realtime::ResponseDoneEvent.new( event_id: "event_done",