diff --git a/Gemfile b/Gemfile
index cfb2366a1..318265cde 100644
--- a/Gemfile
+++ b/Gemfile
@@ -17,6 +17,7 @@ end
group :development, :test do
gem "async"
+ gem "async-websocket"
gem "aws-sdk-core", "~> 3"
gem "minitest", "~> 5.27"
gem "minitest-focus"
diff --git a/Gemfile.lock b/Gemfile.lock
index d206ab023..497722853 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -42,6 +42,24 @@ GEM
io-event (~> 1.11)
metrics (~> 0.12)
traces (~> 0.18)
+ async-http (0.95.1)
+ async (>= 2.10.2)
+ async-pool (~> 0.11)
+ io-endpoint (~> 0.14)
+ io-stream (~> 0.6)
+ metrics (~> 0.12)
+ protocol-http (~> 0.62)
+ protocol-http1 (~> 0.39)
+ protocol-http2 (~> 0.26)
+ protocol-url (~> 0.2)
+ traces (~> 0.10)
+ async-pool (0.11.2)
+ async (>= 2.0)
+ async-websocket (0.30.1)
+ async-http (~> 0.76)
+ protocol-http (~> 0.34)
+ protocol-rack (~> 0.7)
+ protocol-websocket (~> 0.17)
aws-eventstream (1.4.0)
aws-partitions (1.1261.0)
aws-sdk-core (3.252.0)
@@ -85,7 +103,10 @@ GEM
hashdiff (1.2.1)
i18n (1.15.2)
concurrent-ruby (~> 1.0)
+ io-endpoint (0.17.2)
io-event (1.11.2)
+ io-stream (0.14.0)
+ openssl (>= 3.3)
jmespath (1.6.2)
json (2.21.2)
language_server-protocol (3.17.0.5)
@@ -106,14 +127,30 @@ GEM
minitest (~> 5.0)
mutex_m (0.3.0)
netrc (0.11.0)
+ openssl (4.0.2)
parallel (1.27.0)
parser (3.3.10.0)
ast (~> 2.4.1)
racc
prettier_print (1.2.1)
prism (1.6.0)
+ protocol-hpack (1.5.1)
+ protocol-http (0.70.0)
+ protocol-http1 (0.40.2)
+ protocol-http (~> 0.68)
+ protocol-http2 (0.26.2)
+ protocol-hpack (~> 1.4)
+ protocol-http (~> 0.62)
+ protocol-rack (0.22.1)
+ io-stream (>= 0.10)
+ protocol-http (~> 0.58)
+ rack (>= 1.0)
+ protocol-url (0.11.0)
+ protocol-websocket (0.21.1)
+ protocol-http (~> 0.2)
public_suffix (7.0.5)
racc (1.8.1)
+ rack (3.2.6)
rainbow (3.1.1)
rake (13.3.1)
rb-fsevent (0.11.2)
@@ -203,7 +240,7 @@ GEM
addressable (>= 2.8.0)
crack (>= 0.3.2)
hashdiff (>= 0.4.0, < 2.0.0)
- webrick (1.9.1)
+ webrick (1.9.2)
yard (0.9.44)
yard-sorbet (0.9.0)
sorbet-runtime
@@ -221,6 +258,7 @@ PLATFORMS
DEPENDENCIES
async
+ async-websocket
aws-sdk-core (~> 3)
minitest (~> 5.27)
minitest-focus
diff --git a/README.md b/README.md
index 21bf0691b..5e1ea3f82 100644
--- a/README.md
+++ b/README.md
@@ -50,6 +50,12 @@ stream.each do |event|
end
```
+### Realtime
+
+Server-side WebSockets, WebRTC negotiation, sideband, SIP, transcription, and
+translation are covered in the [Realtime guide](realtime.md) and runnable
+[example runbook](examples/realtime/README.md).
+
### Pagination
List methods in the OpenAI API are paginated.
diff --git a/examples/realtime/README.md b/examples/realtime/README.md
new file mode 100644
index 000000000..40f6ea6a6
--- /dev/null
+++ b/examples/realtime/README.md
@@ -0,0 +1,229 @@
+# Realtime examples
+
+These examples run against the live OpenAI service with `OPENAI_API_KEY` set.
+They default to `gpt-realtime-2.1`; override `OPENAI_REALTIME_MODEL` when needed.
+
+## Browser voice conversation (recommended)
+
+For natural hands-free conversation, use the WebRTC example. Ruby keeps the
+standard API key private, accepts the browser's SDP offer, configures the
+Realtime call, and returns the SDP answer. The browser owns the media peer and
+requests acoustic echo cancellation, noise suppression, and automatic gain
+control.
+
+```sh
+bundle exec ruby examples/realtime/webrtc_conversation.rb
+```
+
+Open `http://127.0.0.1:4567`, click **Start conversation**, and allow microphone
+access. Talk normally and speak over the model to interrupt it. Click **Stop**
+to close the peer and ask Ruby to hang up the call. No API key is sent to the
+browser. Override the port with `REALTIME_DEMO_PORT`.
+
+## WebSocket microphone loop (advanced)
+
+`realtime_conversation.rb` keeps one full-duplex WebSocket session open. It
+streams 100 ms microphone chunks continuously, uses server VAD to detect turn
+boundaries and create responses automatically, and plays output audio as deltas
+arrive. There is no push-to-talk control.
+
+This low-level path has no acoustic echo cancellation. It is useful for server
+audio pipelines and protocol debugging, but it is not a laptop speakerphone.
+Install FFmpeg (which also supplies `ffplay`) and **use headphones** so the
+microphone does not capture the model's voice:
+
+```sh
+brew install ffmpeg
+bundle exec ruby examples/realtime/realtime_conversation.rb
+```
+
+Start speaking after `Connected` appears. Speak again whenever you want to
+interrupt: the service cancels the response and the example immediately clears
+local playback, drops already-queued audio deltas, and truncates the unheard
+audio from conversation history. A bounded single-writer queue serializes the
+microphone and truncation events sent over the socket. Press Control-C to end
+the session. If that writer fails, the example stops microphone capture,
+unblocks any queued producer, and reports the original connection error.
+
+On macOS, the default microphone is AVFoundation audio device `0`. List devices
+and select a different index if necessary:
+
+```sh
+ffmpeg -f avfoundation -list_devices true -i ""
+
+REALTIME_MIC_DEVICE=:1 \
+bundle exec ruby examples/realtime/realtime_conversation.rb
+```
+
+macOS may ask for microphone permission for your terminal on the first run.
+Customize the voice or spoken behavior with `OPENAI_REALTIME_VOICE` and
+`OPENAI_REALTIME_INSTRUCTIONS`. The example defaults to the recommended `marin`
+voice. Linux defaults to PulseAudio's `default` device; override
+`REALTIME_AUDIO_INPUT_FORMAT` and `REALTIME_MIC_DEVICE` for another FFmpeg input.
+
+For a deterministic live smoke test, provide paced PCM input and capture output
+instead of opening audio devices:
+
+```sh
+REALTIME_INPUT_PCM=input.pcm \
+REALTIME_OUTPUT_PCM=response.pcm \
+OPENAI_REALTIME_TIMEOUT=30 \
+bundle exec ruby examples/realtime/realtime_conversation.rb
+```
+
+This bounded mode exits successfully only after the requested `response.done`;
+a clean EOF before that event fails the smoke test.
+
+## WebSocket text
+
+```sh
+bundle exec ruby examples/realtime/websocket_text.rb
+```
+
+A successful run prints `session.created`, `session.updated`, streamed assistant
+text, and `response.done status=completed`. An EOF before the completed
+`response.done`, or a completed response without a non-empty text delta, is
+reported as a failed smoke test.
+
+## WebSocket audio
+
+The input must be raw, headerless, mono 24 kHz PCM16 little-endian audio. For
+example, convert a WAV file with:
+
+```sh
+ffmpeg -i input.wav -f s16le -acodec pcm_s16le -ac 1 -ar 24000 input.pcm
+```
+
+Then run the example and play its output:
+
+```sh
+REALTIME_INPUT_PCM=input.pcm \
+REALTIME_OUTPUT_PCM=response.pcm \
+bundle exec ruby examples/realtime/websocket_audio.rb
+
+ffplay -f s16le -ar 24000 -ch_layout mono response.pcm
+```
+
+A successful run exits with status 0 only after writing audio bytes and seeing a
+completed `response.done`. EOF before completion, or completion without an
+audio delta, is reported as a failed smoke test. The example disables automatic
+turn detection and waits for `session.updated` before uploading, so its explicit
+buffer commit and `response.create` cannot race server VAD.
+
+## Realtime transcription
+
+Use the same raw mono 24 kHz PCM16 input format as the audio example:
+
+```sh
+REALTIME_INPUT_PCM=input.pcm \
+OPENAI_REALTIME_TRANSCRIPTION_MODEL=gpt-live-transcribe \
+bundle exec ruby examples/realtime/websocket_transcription.rb
+```
+
+A successful run streams transcript deltas, then prints the completed
+transcript with its `item_id`. Completion events for different input turns may
+arrive out of order, so applications should correlate them by `item_id`.
+The example opens a dedicated connection with `connect_transcription`.
+Configure its transcription model with `OPENAI_REALTIME_TRANSCRIPTION_MODEL`.
+An EOF before the completed transcription event is reported as a failed smoke
+test.
+
+## Realtime translation
+
+Translation uses the same raw PCM input format and reads translated transcript
+and audio while input is still being uploaded:
+
+```sh
+REALTIME_INPUT_PCM=input.pcm \
+REALTIME_OUTPUT_PCM=translation.pcm \
+TARGET_LANGUAGE=es \
+bundle exec ruby examples/realtime/translation.rb
+```
+
+A successful run prints the translated transcript, exits after
+`session.closed`, and writes non-empty translated PCM audio. An EOF before
+`session.closed`, or a closed session with no translated audio, is reported as
+incomplete instead of silently succeeding. If upload and graceful close both
+fail, the upload error remains the primary failure so the interrupted input
+operation is not hidden by cleanup.
+If the reader fails while input is still being uploaded, the example cancels
+the uploader immediately instead of draining the rest of the file. If the
+reader has already failed on buffered input or EOF, upload never starts.
+
+## MCP approval
+
+Point the example at a Streamable HTTP MCP server. This invocation uses the
+public OpenAI developer-documentation server:
+
+```sh
+MCP_SERVER_URL=https://developers.openai.com/mcp \
+OPENAI_REALTIME_PROMPT='Search the OpenAI docs for the Realtime WebSocket URL.' \
+bundle exec ruby examples/realtime/mcp_approval.rb
+```
+
+The example waits for tool import, selects an advertised tool, waits for the
+tool-choice update to be acknowledged, approves the request, waits for the tool
+to finish, and asks the model for the final answer. A successful run prints all
+five checkpoints and reaches the final completed `response.done`; EOF before
+that event is a failed smoke test. Set `OPENAI_REALTIME_DEBUG=1` to print every
+event type.
+
+## WebRTC call creation
+
+`webrtc_call.rb` is the server endpoint core: send a browser-generated SDP offer
+on stdin, return stdout as `application/sdp`, and retain the call ID printed on
+stderr.
+
+```sh
+bundle exec ruby examples/realtime/webrtc_call.rb < offer.sdp > answer.sdp
+```
+
+The recommended `webrtc_conversation.rb` example above exercises this exchange
+end to end with a real `RTCPeerConnection`. `webrtc_call.rb` remains a minimal
+stdin/stdout building block for integrating the same SDK call into another web
+framework.
+
+## Sideband control
+
+Use the call ID from WebRTC call creation or a verified incoming SIP webhook:
+
+```sh
+OPENAI_REALTIME_CALL_ID=rtc_... \
+bundle exec ruby examples/realtime/sideband.rb
+```
+
+For a bounded smoke test, set `OPENAI_REALTIME_STOP_AFTER=session.updated` and
+`OPENAI_REALTIME_TIMEOUT=30`. A real paired test keeps the browser or SIP peer
+connected while this process attaches to the same call. EOF before the selected
+event fails the bounded smoke test.
+
+## SIP
+
+The SIP example requires an OpenAI project with SIP configured and a real
+incoming call. Obtain `OPENAI_REALTIME_CALL_ID` from a verified
+`realtime.call.incoming` webhook, then run:
+
+```sh
+OPENAI_REALTIME_CALL_ID=rtc_... \
+bundle exec ruby examples/realtime/sip.rb
+```
+
+The script accepts the call, attaches a sideband WebSocket, and prints the audio
+transcript. `OPENAI_REALTIME_STOP_AFTER` and `OPENAI_REALTIME_TIMEOUT` provide
+bounded smoke-test controls. Once accepted, the call is hung up during cleanup,
+including after a timeout or sideband failure; an already-ended call is treated
+as successfully cleaned up. EOF before the selected event fails a bounded run.
+Without a carrier-originated incoming call, the same accept-then-attach
+orchestration is covered by the local example and HTTP resource tests, but that
+is not a substitute for the final telephony smoke test.
+
+## Local protocol coverage
+
+The test suite exercises the examples' orchestration and the documented local
+WebSocket lifecycle, including text, transcription, full-duplex translation,
+MCP approval, fragmented frames, normal and abnormal closure, sideband control,
+and SIP accept-then-attach behavior:
+
+```sh
+./scripts/test
+```
diff --git a/examples/realtime/event_stream.rb b/examples/realtime/event_stream.rb
new file mode 100644
index 000000000..4f461fd84
--- /dev/null
+++ b/examples/realtime/event_stream.rb
@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module OpenAI
+ module Examples
+ module Realtime
+ module EventStream
+ module_function
+
+ def each_until(connection, stop_after: nil, closed_message: nil)
+ reached_target = false
+ connection.each do |event|
+ yield(event)
+ reached_target = stop_after == event.type.to_s
+ break if reached_target
+ end
+ return if stop_after.nil? || reached_target
+
+ raise(closed_message || "Realtime connection closed before #{stop_after}")
+ end
+
+ def wait_for(connection, event_class, closed_message:)
+ while (event = connection.receive)
+ raise event.error.message if event.is_a?(OpenAI::Realtime::RealtimeErrorEvent)
+ return event if event.is_a?(event_class)
+ end
+
+ raise closed_message
+ end
+ end
+ end
+ end
+end
diff --git a/examples/realtime/mcp_approval.rb b/examples/realtime/mcp_approval.rb
new file mode 100755
index 000000000..3ee0cb4bd
--- /dev/null
+++ b/examples/realtime/mcp_approval.rb
@@ -0,0 +1,175 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require_relative "../../lib/openai"
+require "timeout"
+
+module OpenAI
+ module Examples
+ module Realtime
+ module MCPApproval
+ module_function
+
+ def configure(connection, server_url:)
+ connection.session.update(
+ type: :realtime,
+ output_modalities: [:text],
+ tools: [
+ {
+ type: :mcp,
+ server_label: "remote",
+ server_url: server_url,
+ require_approval: :always
+ }
+ ]
+ )
+ end
+
+ def select_tool(connection, server_label:, tool_name:)
+ connection.session.update(
+ type: :realtime,
+ tools: [{type: :mcp, server_label: server_label}],
+ tool_choice: {type: :mcp, server_label: server_label, name: tool_name}
+ )
+ end
+
+ def send_prompt(connection, prompt:)
+ connection.conversation.items.create(
+ type: :message,
+ role: :user,
+ content: [{type: :input_text, text: prompt}]
+ )
+ connection.response.create
+ end
+
+ def approve_request(connection, item)
+ connection.conversation.items.respond_to_mcp_approval(
+ approval_request_id: item.id,
+ approve: true,
+ reason: "Approved by the application policy"
+ )
+ end
+
+ def run_session(connection, prompt:, output: $stdout, debug: false)
+ completed = false
+ state = {
+ completed_item_ids: {},
+ tool_lists: {},
+ waiting_for_tool_update: false,
+ selected_tool: false,
+ mcp_call_pending: false
+ }
+
+ connection.each do |event|
+ output.puts("[mcp event] #{event.type}") if debug
+ if handle_event(connection, event, prompt: prompt, output: output, state: state) == :done
+ completed = true
+ break
+ end
+
+ select_discovered_tool(connection, output: output, state: state)
+ end
+ return if completed
+
+ raise "Realtime connection closed before the final response.done"
+ end
+
+ def handle_event(connection, event, prompt:, output:, state:)
+ case event
+ when OpenAI::Realtime::McpListToolsCompleted
+ state[:completed_item_ids][event.item_id] = true
+ when OpenAI::Realtime::SessionUpdatedEvent
+ handle_session_updated(connection, prompt: prompt, state: state)
+ when OpenAI::Realtime::ConversationItemDone
+ handle_conversation_item(connection, event.item, output: output, state: state)
+ when OpenAI::Realtime::McpListToolsFailed
+ raise "MCP tool discovery failed for #{event.item_id}"
+ when OpenAI::Realtime::ResponseMcpCallArgumentsDone
+ state[:mcp_call_pending] = true
+ output.puts("[mcp] tool call requested")
+ when OpenAI::Realtime::ResponseMcpCallCompleted
+ handle_tool_completed(connection, output: output, state: state)
+ when OpenAI::Realtime::ResponseMcpCallFailed
+ raise "MCP tool call failed for #{event.item_id}"
+ when OpenAI::Realtime::RealtimeErrorEvent
+ raise event.error.message
+ when OpenAI::Realtime::ResponseTextDeltaEvent
+ output.print(event.delta)
+ output.flush
+ when OpenAI::Realtime::ResponseDoneEvent
+ handle_response_done(event, output: output, state: state)
+ end
+ end
+
+ def handle_session_updated(connection, prompt:, state:)
+ return unless state[:waiting_for_tool_update]
+
+ send_prompt(connection, prompt: prompt)
+ state[:waiting_for_tool_update] = false
+ end
+
+ def handle_conversation_item(connection, item, output:, state:)
+ if item.is_a?(OpenAI::Realtime::RealtimeMcpListTools)
+ state[:tool_lists][item.id] = item
+ elsif item.is_a?(OpenAI::Realtime::RealtimeMcpApprovalRequest)
+ output.puts("[mcp] approving #{item.id}")
+ approve_request(connection, item)
+ end
+ end
+
+ def handle_tool_completed(connection, output:, state:)
+ state[:mcp_call_pending] = false
+ output.puts("[mcp] tool call completed")
+ connection.response.create(tool_choice: :none)
+ end
+
+ def handle_response_done(event, output:, state:)
+ raise "Response ended with #{event.response.status}" unless event.response.status == :completed
+
+ if state[:mcp_call_pending]
+ output.puts("[mcp] waiting for approval and tool execution")
+ else
+ output.puts("\n[mcp] response.done status=completed")
+ :done
+ end
+ end
+
+ def select_discovered_tool(connection, output:, state:)
+ return if state[:selected_tool]
+
+ tool_list = state[:tool_lists].values.find { |item| state[:completed_item_ids][item.id] }
+ return unless tool_list
+ raise "The MCP server imported no tools" if tool_list.tools.empty?
+
+ tool_names = tool_list.tools.map(&:name)
+ output.puts("[mcp] tools discovered: #{tool_names.join(', ')}")
+ select_tool(connection, server_label: tool_list.server_label, tool_name: tool_names.fetch(0))
+ state[:selected_tool] = true
+ state[:waiting_for_tool_update] = true
+ end
+
+ def run(client:, model:, server_url:, prompt:, output: $stdout, debug: false)
+ client.realtime.connect(model: model) do |connection|
+ configure(connection, server_url: server_url)
+ run_session(connection, prompt: prompt, output: output, debug: debug)
+ end
+ end
+ end
+ end
+ end
+end
+
+if $PROGRAM_NAME == __FILE__
+ client = OpenAI::Client.new
+ timeout = Integer(ENV.fetch("OPENAI_REALTIME_TIMEOUT", "60"))
+
+ Timeout.timeout(timeout) do
+ OpenAI::Examples::Realtime::MCPApproval.run(
+ client: client,
+ model: ENV.fetch("OPENAI_REALTIME_MODEL", "gpt-realtime-2.1"),
+ server_url: ENV.fetch("MCP_SERVER_URL"),
+ prompt: ENV.fetch("OPENAI_REALTIME_PROMPT", "Use the configured MCP server."),
+ debug: ENV["OPENAI_REALTIME_DEBUG"] == "1"
+ )
+ end
+end
diff --git a/examples/realtime/realtime_conversation.rb b/examples/realtime/realtime_conversation.rb
new file mode 100755
index 000000000..1a49b6a08
--- /dev/null
+++ b/examples/realtime/realtime_conversation.rb
@@ -0,0 +1,534 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "async"
+require "timeout"
+require_relative "../../lib/openai"
+require_relative "event_stream"
+
+module OpenAI
+ module Examples
+ module Realtime
+ module Conversation
+ AUDIO_BYTES_PER_SECOND = 48_000
+ CHUNK_BYTES = 4_800
+
+ class FFmpegMicrophone
+ def initialize(input_format:, device:)
+ @input_format = input_format
+ @device = device
+ @pid = nil
+ @stopping = false
+ end
+
+ def each_chunk
+ return enum_for(__method__) unless block_given?
+ return if @stopping
+
+ reader, writer = IO.pipe
+ @pid = Process.spawn(*command, out: writer, err: $stderr)
+ writer.close
+ reader.binmode
+ while (chunk = reader.read(CHUNK_BYTES))
+ yield(chunk)
+ end
+ rescue Errno::ENOENT
+ raise "ffmpeg was not found; install it with `brew install ffmpeg`"
+ ensure
+ writer&.close unless writer&.closed?
+ reader&.close unless reader&.closed?
+ wait_for_capture
+ end
+
+ def stop
+ @stopping = true
+ return unless @pid
+
+ Process.kill("INT", @pid)
+ rescue Errno::ESRCH
+ nil
+ end
+
+ private def command
+ [
+ "ffmpeg",
+ "-nostdin",
+ "-loglevel",
+ "error",
+ "-thread_queue_size",
+ "1024",
+ "-f",
+ @input_format,
+ "-i",
+ @device,
+ "-ac",
+ "1",
+ "-ar",
+ "24000",
+ "-acodec",
+ "pcm_s16le",
+ "-f",
+ "s16le",
+ "pipe:1"
+ ]
+ end
+
+ private def wait_for_capture
+ return unless @pid
+
+ _, status = Process.wait2(@pid)
+ return if @stopping || status.success?
+
+ raise "ffmpeg microphone capture exited with status #{status.exitstatus}"
+ rescue Errno::ECHILD
+ nil
+ ensure
+ @pid = nil
+ end
+ end
+
+ class PCMFileMicrophone
+ def initialize(path)
+ @path = path
+ @stopping = false
+ end
+
+ def each_chunk
+ return enum_for(__method__) unless block_given?
+
+ File.open(@path, "rb") do |input|
+ while !@stopping && (chunk = input.read(CHUNK_BYTES))
+ yield(chunk)
+ sleep(chunk.bytesize.fdiv(AUDIO_BYTES_PER_SECOND))
+ end
+ end
+ 10.times do
+ break if @stopping
+
+ yield("\0".b * CHUNK_BYTES)
+ sleep(CHUNK_BYTES.fdiv(AUDIO_BYTES_PER_SECOND))
+ end
+ end
+
+ def stop = @stopping = true
+ end
+
+ class FFplaySpeaker
+ def initialize
+ @input = nil
+ @pid = nil
+ end
+
+ def write(bytes)
+ start unless @input
+ @input.write(bytes)
+ @input.flush
+ rescue Errno::EPIPE
+ interrupt
+ raise "ffplay stopped while playing Realtime audio"
+ end
+
+ def interrupt
+ return unless @pid
+
+ Process.kill("TERM", @pid)
+ @input&.close unless @input&.closed?
+ Process.wait(@pid)
+ rescue Errno::ESRCH, Errno::ECHILD
+ nil
+ ensure
+ @input = nil
+ @pid = nil
+ end
+
+ alias_method :close, :interrupt
+
+ private def start
+ reader, writer = IO.pipe
+ @pid = Process.spawn(*command, in: reader, out: File::NULL, err: $stderr)
+ reader.close
+ writer.binmode
+ @input = writer
+ rescue Errno::ENOENT
+ raise "ffplay was not found; install it with `brew install ffmpeg`"
+ ensure
+ reader&.close unless reader&.closed?
+ end
+
+ private def command
+ [
+ "ffplay",
+ "-nodisp",
+ "-nostats",
+ "-loglevel",
+ "error",
+ "-fflags",
+ "nobuffer",
+ "-flags",
+ "low_delay",
+ "-probesize",
+ "32",
+ "-analyzeduration",
+ "0",
+ "-f",
+ "s16le",
+ "-ar",
+ "24000",
+ "-ch_layout",
+ "mono",
+ "-i",
+ "pipe:0"
+ ]
+ end
+ end
+
+ class PCMFileSpeaker
+ def initialize(path)
+ @output = File.open(path, "wb")
+ end
+
+ def write(bytes) = @output.write(bytes)
+
+ def interrupt = @output.flush
+
+ def close
+ @output.close unless @output.closed?
+ end
+ end
+
+ # Serializes every client event through one writer fiber. The one-item queue
+ # preserves microphone backpressure while allowing the reader to enqueue an
+ # interruption without writing to the socket itself.
+ class OutboundWriter
+ def initialize(connection)
+ @connection = connection
+ @queue = Thread::SizedQueue.new(1)
+ @failure = nil
+ end
+
+ def append_audio(bytes)
+ enqueue([:append_audio, bytes])
+ end
+
+ def truncate(**params)
+ enqueue([:truncate, params])
+ end
+
+ def run
+ while (message = @queue.pop)
+ operation, payload = message
+ case operation
+ when :append_audio
+ @connection.input_audio_buffer.append_bytes(payload)
+ when :truncate
+ @connection.conversation.items.truncate(**payload)
+ end
+ end
+ rescue StandardError => e
+ @failure = e
+ raise
+ ensure
+ close
+ end
+
+ def close
+ @queue.close unless @queue.closed?
+ end
+
+ private def enqueue(message)
+ @queue.push(message)
+ rescue ClosedQueueError
+ raise @failure if @failure
+
+ raise
+ end
+ end
+
+ class AudioPlayback
+ def initialize(speaker, clock: nil)
+ @speaker = speaker
+ @clock = clock || -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) }
+ @interrupted_response_id = nil
+ reset_current
+ end
+
+ def write(event)
+ return if interrupted?(event.response_id)
+
+ start_response(event) unless current?(event)
+ bytes = Base64.strict_decode64(event.delta)
+ @speaker.write(bytes)
+ @received_bytes += bytes.bytesize
+ end
+
+ def interrupt(outbound)
+ expire_finished_playback
+ unless @item_id
+ @speaker.interrupt
+ return
+ end
+
+ audio_end_ms = played_ms
+ response_id = @response_id
+ item_id = @item_id
+ content_index = @content_index
+
+ @speaker.interrupt
+ outbound.truncate(
+ item_id: item_id,
+ content_index: content_index,
+ audio_end_ms: audio_end_ms
+ )
+ @interrupted_response_id = response_id
+ reset_current
+ response_id
+ end
+
+ def interrupted?(response_id) = response_id == @interrupted_response_id
+
+ def finish(response_id)
+ if interrupted?(response_id)
+ @interrupted_response_id = nil
+ elsif response_id == @response_id
+ @finished_response_id = response_id
+ expire_finished_playback
+ end
+ end
+
+ def close = @speaker.close
+
+ private def current?(event)
+ @response_id == event.response_id &&
+ @item_id == event.item_id &&
+ @content_index == event.content_index
+ end
+
+ private def start_response(event)
+ @response_id = event.response_id
+ @item_id = event.item_id
+ @content_index = event.content_index
+ @received_bytes = 0
+ @started_at = @clock.call
+ end
+
+ private def played_ms
+ elapsed_ms = ((@clock.call - @started_at) * 1_000).floor
+ received_ms = (@received_bytes * 1_000) / AUDIO_BYTES_PER_SECOND
+ [elapsed_ms, received_ms].min.clamp(0, received_ms)
+ end
+
+ private def expire_finished_playback
+ return unless @finished_response_id
+ return unless @finished_response_id == @response_id
+ return unless elapsed_audio_bytes >= @received_bytes
+
+ reset_current
+ end
+
+ private def elapsed_audio_bytes
+ ((@clock.call - @started_at) * AUDIO_BYTES_PER_SECOND).floor
+ end
+
+ private def reset_current
+ @response_id = nil
+ @item_id = nil
+ @content_index = nil
+ @received_bytes = 0
+ @started_at = nil
+ @finished_response_id = nil
+ end
+ end
+
+ module_function
+
+ def configure(connection, voice:, instructions:)
+ connection.session.update(
+ type: :realtime,
+ output_modalities: [:audio],
+ instructions: instructions,
+ audio: {
+ input: {
+ format: {type: :"audio/pcm", rate: 24_000},
+ noise_reduction: {type: :near_field},
+ turn_detection: {
+ type: :server_vad,
+ threshold: 0.5,
+ prefix_padding_ms: 300,
+ silence_duration_ms: 500,
+ create_response: true,
+ interrupt_response: true
+ }
+ },
+ output: {
+ format: {type: :"audio/pcm", rate: 24_000},
+ voice: voice
+ }
+ }
+ )
+ end
+
+ def forward_microphone(outbound, microphone)
+ microphone.each_chunk do |chunk|
+ outbound.append_audio(chunk)
+ end
+ end
+
+ def handle_event(event, outbound:, playback:, output:)
+ case event
+ when OpenAI::Realtime::ResponseAudioDeltaEvent
+ playback.write(event)
+ when OpenAI::Realtime::ResponseAudioTranscriptDeltaEvent
+ return if playback.interrupted?(event.response_id)
+
+ output.print(event.delta)
+ output.flush
+ when OpenAI::Realtime::ResponseAudioTranscriptDoneEvent
+ return if playback.interrupted?(event.response_id)
+
+ output.puts
+ when OpenAI::Realtime::InputAudioBufferSpeechStartedEvent
+ output.puts if playback.interrupt(outbound)
+ when OpenAI::Realtime::RealtimeErrorEvent
+ raise event.error.message
+ when OpenAI::Realtime::ResponseDoneEvent
+ playback.finish(event.response.id)
+ return if [:cancelled, :completed].include?(event.response.status)
+
+ raise "Realtime response ended with #{event.response.status}"
+ end
+ end
+
+ def stream_events(connection, microphone:, outbound:, playback:, output:, stop_after: nil)
+ EventStream.each_until(connection, stop_after: stop_after) do |event|
+ handle_event(event, outbound: outbound, playback: playback, output: output)
+ end
+ ensure
+ microphone.stop
+ end
+
+ def run_session(
+ connection,
+ microphone:,
+ speaker:,
+ voice:,
+ instructions:,
+ output: $stdout,
+ stop_after: nil
+ )
+ configure(connection, voice: voice, instructions: instructions)
+ EventStream.wait_for(
+ connection,
+ OpenAI::Realtime::SessionUpdatedEvent,
+ closed_message: "Realtime connection closed before session.updated"
+ )
+ output.puts(
+ "Connected. Talk naturally; speak over the model to interrupt. " \
+ "Press Ctrl-C to exit. Use headphones to prevent echo."
+ )
+ playback = AudioPlayback.new(speaker)
+ outbound = OutboundWriter.new(connection)
+
+ writer = Async do
+ outbound.run
+ nil
+ rescue StandardError => e
+ e
+ ensure
+ microphone.stop
+ end
+ receiver = Async do
+ stream_events(
+ connection,
+ microphone: microphone,
+ outbound: outbound,
+ playback: playback,
+ output: output,
+ stop_after: stop_after
+ )
+ end
+ sender = Async { forward_microphone(outbound, microphone) }
+ sender.wait
+ outbound.close
+ writer_error = writer.wait
+ raise writer_error if writer_error
+
+ receiver.wait
+ ensure
+ microphone.stop
+ outbound&.close
+ playback&.close || speaker.close
+ sender&.stop
+ receiver&.stop
+ writer&.stop
+ end
+
+ def run(
+ client:,
+ model:,
+ microphone:,
+ speaker:,
+ voice:,
+ instructions:,
+ output: $stdout,
+ stop_after: nil
+ )
+ client.realtime.connect(model: model) do |connection|
+ run_session(
+ connection,
+ microphone: microphone,
+ speaker: speaker,
+ voice: voice,
+ instructions: instructions,
+ output: output,
+ stop_after: stop_after
+ )
+ end
+ end
+ end
+ end
+ end
+end
+
+if $PROGRAM_NAME == __FILE__
+ begin
+ input_path = ENV["REALTIME_INPUT_PCM"]
+ microphone =
+ if input_path
+ OpenAI::Examples::Realtime::Conversation::PCMFileMicrophone.new(input_path)
+ else
+ default_format = RUBY_PLATFORM.include?("darwin") ? "avfoundation" : "pulse"
+ default_device = RUBY_PLATFORM.include?("darwin") ? ":0" : "default"
+ OpenAI::Examples::Realtime::Conversation::FFmpegMicrophone.new(
+ input_format: ENV.fetch("REALTIME_AUDIO_INPUT_FORMAT", default_format),
+ device: ENV.fetch("REALTIME_MIC_DEVICE", default_device)
+ )
+ end
+ stop_after = input_path ? "response.done" : nil
+ output_path = ENV["REALTIME_OUTPUT_PCM"]
+ speaker =
+ if output_path
+ OpenAI::Examples::Realtime::Conversation::PCMFileSpeaker.new(output_path)
+ else
+ OpenAI::Examples::Realtime::Conversation::FFplaySpeaker.new
+ end
+ run = lambda do
+ OpenAI::Examples::Realtime::Conversation.run(
+ client: OpenAI::Client.new,
+ model: ENV.fetch("OPENAI_REALTIME_MODEL", "gpt-realtime-2.1"),
+ microphone: microphone,
+ speaker: speaker,
+ voice: ENV.fetch("OPENAI_REALTIME_VOICE", "marin"),
+ instructions: ENV.fetch(
+ "OPENAI_REALTIME_INSTRUCTIONS",
+ "Have a natural spoken conversation. Keep each response concise."
+ ),
+ stop_after: stop_after
+ )
+ end
+
+ timeout = ENV["OPENAI_REALTIME_TIMEOUT"]
+ timeout ? Timeout.timeout(Integer(timeout)) { run.call } : run.call
+ rescue Interrupt
+ warn("\nConversation ended.")
+ end
+end
diff --git a/examples/realtime/sideband.rb b/examples/realtime/sideband.rb
new file mode 100755
index 000000000..780260820
--- /dev/null
+++ b/examples/realtime/sideband.rb
@@ -0,0 +1,49 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require_relative "../../lib/openai"
+require_relative "event_stream"
+require "timeout"
+
+module OpenAI
+ module Examples
+ module Realtime
+ module Sideband
+ module_function
+
+ def stream(connection, output: $stdout, stop_after: nil)
+ EventStream.each_until(connection, stop_after: stop_after) do |event|
+ output.puts("#{event.type}: #{event.to_h}")
+ output.flush
+ end
+ end
+
+ def run(client:, call_id:, stop_after: nil, output: $stdout)
+ client.realtime.connect_to_call(call_id: call_id) do |connection|
+ connection.session.update(
+ type: :realtime,
+ instructions: "Use server-side business rules and keep answers short."
+ )
+ stream(connection, output: output, stop_after: stop_after)
+ end
+ end
+ end
+ end
+ end
+end
+
+if $PROGRAM_NAME == __FILE__
+ # CALL_ID may come from calls.create.call_id for WebRTC or from a verified
+ # realtime.call.incoming webhook for SIP.
+ client = OpenAI::Client.new
+ run = lambda do
+ OpenAI::Examples::Realtime::Sideband.run(
+ client: client,
+ call_id: ENV.fetch("OPENAI_REALTIME_CALL_ID"),
+ stop_after: ENV["OPENAI_REALTIME_STOP_AFTER"]
+ )
+ end
+
+ timeout = ENV["OPENAI_REALTIME_TIMEOUT"]
+ timeout ? Timeout.timeout(Integer(timeout)) { run.call } : run.call
+end
diff --git a/examples/realtime/sip.rb b/examples/realtime/sip.rb
new file mode 100755
index 000000000..869adeded
--- /dev/null
+++ b/examples/realtime/sip.rb
@@ -0,0 +1,69 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require_relative "../../lib/openai"
+require_relative "event_stream"
+require "timeout"
+
+module OpenAI
+ module Examples
+ module Realtime
+ module SIP
+ module_function
+
+ def stream(connection, output: $stdout, stop_after: nil)
+ EventStream.each_until(connection, stop_after: stop_after) do |event|
+ case event
+ when OpenAI::Realtime::ResponseAudioTranscriptDeltaEvent
+ output.print(event.delta)
+ output.flush
+ when OpenAI::Realtime::RealtimeErrorEvent
+ raise event.error.message
+ end
+ end
+ end
+
+ def run(client:, call_id:, model:, output: $stdout, stop_after: nil)
+ accepted = false
+ client.realtime.calls.accept(
+ call_id,
+ type: :realtime,
+ model: model,
+ instructions: "You are answering a phone call. Be warm and concise."
+ )
+ accepted = true
+
+ client.realtime.connect_to_call(call_id: call_id) do |connection|
+ stream(connection, output: output, stop_after: stop_after)
+ end
+ ensure
+ hangup(client, call_id, active_error: $ERROR_INFO) if accepted
+ end
+
+ def hangup(client, call_id, active_error:)
+ client.realtime.calls.hangup(call_id)
+ rescue OpenAI::Errors::NotFoundError
+ nil
+ rescue StandardError
+ raise unless active_error
+ end
+ end
+ end
+ end
+end
+
+if $PROGRAM_NAME == __FILE__
+ # Obtain the call ID from a verified realtime.call.incoming webhook via
+ # client.webhooks.unwrap(payload, headers).
+ run = lambda do
+ OpenAI::Examples::Realtime::SIP.run(
+ client: OpenAI::Client.new,
+ call_id: ENV.fetch("OPENAI_REALTIME_CALL_ID"),
+ model: ENV.fetch("OPENAI_REALTIME_MODEL", "gpt-realtime-2.1"),
+ stop_after: ENV["OPENAI_REALTIME_STOP_AFTER"]
+ )
+ end
+
+ timeout = ENV["OPENAI_REALTIME_TIMEOUT"]
+ timeout ? Timeout.timeout(Integer(timeout)) { run.call } : run.call
+end
diff --git a/examples/realtime/sorbet_connection_types.rb b/examples/realtime/sorbet_connection_types.rb
new file mode 100644
index 000000000..a231aa0eb
--- /dev/null
+++ b/examples/realtime/sorbet_connection_types.rb
@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+# typed: true
+
+require_relative "../../lib/openai"
+
+# Static contract fixture for the three capability-specific connection methods.
+# Defining, but never invoking, this method keeps the fixture inert at runtime.
+module OpenAI
+ module Examples
+ module Realtime
+ module SorbetConnectionTypes
+ module_function
+
+ def verify
+ client = OpenAI::Client.new
+
+ client.realtime.connect(model: "gpt-realtime-2.1") do |connection|
+ T.assert_type!(connection, OpenAI::Realtime::Connection)
+ connection.response.create
+ end
+
+ client.realtime.connect_to_call(call_id: "rtc_123") do |connection|
+ T.assert_type!(connection, OpenAI::Realtime::SidebandConnection)
+ connection.output_audio_buffer.clear
+ end
+
+ client.realtime.connect_transcription do |connection|
+ T.assert_type!(connection, OpenAI::Realtime::TranscriptionConnection)
+ connection.input_audio_buffer.commit
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/examples/realtime/translation.rb b/examples/realtime/translation.rb
new file mode 100755
index 000000000..9292635a7
--- /dev/null
+++ b/examples/realtime/translation.rb
@@ -0,0 +1,110 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "async"
+require_relative "../../lib/openai"
+require_relative "event_stream"
+
+module OpenAI
+ module Examples
+ module Realtime
+ module Translation
+ module_function
+
+ def stream(connection, audio_output:, transcript_output: $stdout)
+ audio_bytes = 0
+ EventStream.each_until(
+ connection,
+ stop_after: "session.closed",
+ closed_message: "Realtime translation connection closed before session.closed"
+ ) do |event|
+ case event
+ when OpenAI::Realtime::RealtimeTranslationOutputTranscriptDeltaEvent
+ transcript_output.print(event.delta)
+ transcript_output.flush
+ when OpenAI::Realtime::RealtimeTranslationOutputAudioDeltaEvent
+ audio = Base64.strict_decode64(event.delta)
+ audio_output.write(audio)
+ audio_bytes += audio.bytesize
+ when OpenAI::Realtime::RealtimeTranslationSessionClosedEvent
+ raise "Translation session closed without audio output" if audio_bytes.zero?
+
+ transcript_output.puts
+ when OpenAI::Realtime::RealtimeErrorEvent
+ raise event.error.message
+ end
+ end
+ end
+
+ def write_input(connection, input_path)
+ File.open(input_path, "rb") do |input|
+ while (chunk = input.read(9_600))
+ connection.input_audio_buffer.append_bytes(chunk)
+ end
+ end
+ ensure
+ close_session(connection, preserve_error: !$ERROR_INFO.nil?)
+ end
+
+ def close_session(connection, preserve_error:)
+ connection.session.close
+ rescue StandardError
+ raise unless preserve_error
+ end
+
+ def exchange(connection, input_path:, audio_output:, transcript_output:)
+ uploader = nil
+ reader = Async do
+ stream(
+ connection,
+ audio_output: audio_output,
+ transcript_output: transcript_output
+ )
+ nil
+ rescue StandardError => e
+ uploader&.stop
+ e
+ end
+ if reader.finished?
+ reader_error = reader.wait
+ raise reader_error if reader_error
+
+ return
+ end
+
+ uploader = Async { write_input(connection, input_path) }
+ uploader.wait
+ reader_error = reader.wait
+ raise reader_error if reader_error
+ ensure
+ uploader&.stop
+ reader&.stop
+ end
+
+ def run(client:, model:, input_path:, output_path:, target_language:, transcript_output: $stdout)
+ File.open(output_path, "wb") do |audio_output|
+ client.realtime.translations.connect(model: model) do |connection|
+ connection.session.update(audio: {output: {language: target_language}})
+ exchange(
+ connection,
+ input_path: input_path,
+ audio_output: audio_output,
+ transcript_output: transcript_output
+ )
+ end
+ end
+ end
+ end
+ end
+ end
+end
+
+if $PROGRAM_NAME == __FILE__
+ OpenAI::Examples::Realtime::Translation.run(
+ client: OpenAI::Client.new,
+ model: ENV.fetch("OPENAI_REALTIME_TRANSLATION_MODEL", "gpt-realtime-translate"),
+ input_path: ENV.fetch("REALTIME_INPUT_PCM"),
+ output_path: ENV.fetch("REALTIME_OUTPUT_PCM", "translation-output.pcm"),
+ target_language: ENV.fetch("TARGET_LANGUAGE", "es")
+ )
+end
diff --git a/examples/realtime/webrtc_call.rb b/examples/realtime/webrtc_call.rb
new file mode 100755
index 000000000..4b5835d68
--- /dev/null
+++ b/examples/realtime/webrtc_call.rb
@@ -0,0 +1,22 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require_relative "../../lib/openai"
+
+# Pipe a browser's SDP offer to this process. The SDP answer is written to stdout,
+# which makes this suitable as the core of a Rails, Sinatra, or Rack endpoint.
+offer = $stdin.read
+raise "Expected an SDP offer on stdin" if offer.empty?
+
+client = OpenAI::Client.new
+call = client.realtime.calls.create(
+ sdp: offer,
+ session: {
+ type: :realtime,
+ model: ENV.fetch("OPENAI_REALTIME_MODEL", "gpt-realtime-2.1"),
+ audio: {output: {voice: :marin}}
+ }
+)
+
+warn("Created Realtime call #{call.call_id || 'without a Location header'}")
+$stdout.write(call.sdp)
diff --git a/examples/realtime/webrtc_conversation.html b/examples/realtime/webrtc_conversation.html
new file mode 100644
index 000000000..ac68d4c3f
--- /dev/null
+++ b/examples/realtime/webrtc_conversation.html
@@ -0,0 +1,223 @@
+
+
+
+
+
+ Ruby Realtime conversation
+
+
+
+
+
Realtime conversation
+
+ Talk naturally and speak over the model whenever you want to interrupt.
+ The browser handles the microphone, playback, and acoustic echo cancellation.
+
+
+
+
+
+
Ready
+
+
+
+
+
+
+
+
diff --git a/examples/realtime/webrtc_conversation.rb b/examples/realtime/webrtc_conversation.rb
new file mode 100755
index 000000000..3f1d10bea
--- /dev/null
+++ b/examples/realtime/webrtc_conversation.rb
@@ -0,0 +1,149 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require_relative "../../lib/openai"
+
+module OpenAI
+ module Examples
+ module Realtime
+ module WebRTCConversation
+ MAX_SDP_BYTES = 1_048_576
+
+ class App
+ def initialize(client:, html: File.binread(File.join(__dir__, "webrtc_conversation.html")))
+ @client = client
+ @html = html
+ @call_ids = {}
+ @call_ids_lock = Mutex.new
+ end
+
+ def handle(request, response)
+ case [request.request_method, request.path]
+ when ["GET", "/"]
+ render(response, status: 200, content_type: "text/html; charset=utf-8", body: @html)
+ when ["POST", "/session"]
+ create_session(request, response)
+ when ["POST", "/hangup"]
+ hangup(request, response)
+ else
+ render(response, status: 404, content_type: "text/plain; charset=utf-8", body: "Not found\n")
+ end
+ rescue ArgumentError => e
+ render(response, status: 400, content_type: "text/plain; charset=utf-8", body: "#{e.message}\n")
+ rescue StandardError => e
+ warn("Realtime WebRTC request failed: #{e.class}: #{e.message}")
+ render(
+ response,
+ status: 502,
+ content_type: "text/plain; charset=utf-8",
+ body: "Realtime request failed\n"
+ )
+ end
+
+ private def create_session(request, response)
+ content_type = request["content-type"].to_s.split(";", 2).first
+ unless content_type == "application/sdp"
+ raise ArgumentError, "Expected Content-Type: application/sdp"
+ end
+
+ offer = request.body.to_s
+ raise ArgumentError, "Expected an SDP offer" if offer.empty?
+ raise ArgumentError, "SDP offer is too large" if offer.bytesize > MAX_SDP_BYTES
+ raise ArgumentError, "Invalid SDP offer" unless offer.lstrip.start_with?("v=")
+
+ call = @client.realtime.calls.create(sdp: offer, session: session_config)
+ remember(call.call_id)
+ response["X-OpenAI-Call-ID"] = call.call_id if call.call_id
+ render(response, status: 201, content_type: "application/sdp", body: call.sdp)
+ end
+
+ private def hangup(request, response)
+ call_id = request.body.to_s.strip
+ raise ArgumentError, "Expected a call ID" if call_id.empty?
+
+ known = @call_ids_lock.synchronize { @call_ids.include?(call_id) }
+ unless known
+ return render(
+ response,
+ status: 404,
+ content_type: "text/plain; charset=utf-8",
+ body: "Unknown call\n"
+ )
+ end
+
+ begin
+ @client.realtime.calls.hangup(call_id)
+ rescue OpenAI::Errors::NotFoundError
+ # Closing the browser peer can end the call before this cleanup request arrives.
+ end
+ @call_ids_lock.synchronize { @call_ids.delete(call_id) }
+ render(response, status: 204, content_type: "text/plain; charset=utf-8", body: "")
+ end
+
+ private def session_config
+ {
+ type: :realtime,
+ model: ENV.fetch("OPENAI_REALTIME_MODEL", "gpt-realtime-2.1"),
+ output_modalities: [:audio],
+ instructions: ENV.fetch(
+ "OPENAI_REALTIME_INSTRUCTIONS",
+ "Have a natural spoken conversation. Keep each response concise."
+ ),
+ audio: {
+ input: {
+ turn_detection: {
+ type: :server_vad,
+ threshold: 0.5,
+ prefix_padding_ms: 300,
+ silence_duration_ms: 700,
+ create_response: true,
+ interrupt_response: true
+ }
+ },
+ output: {voice: ENV.fetch("OPENAI_REALTIME_VOICE", "marin")}
+ }
+ }
+ end
+
+ private def remember(call_id)
+ @call_ids_lock.synchronize { @call_ids[call_id] = true } if call_id
+ end
+
+ private def render(response, status:, content_type:, body:)
+ response.status = status
+ response["Content-Type"] = content_type
+ response["Cache-Control"] = "no-store"
+ response["X-Content-Type-Options"] = "nosniff"
+ response.body = body
+ end
+ end
+
+ module_function
+
+ def run
+ require("webrick")
+
+ host = ENV.fetch("REALTIME_DEMO_HOST", "127.0.0.1")
+ port = Integer(ENV.fetch("REALTIME_DEMO_PORT", "4567"))
+ app = App.new(client: OpenAI::Client.new)
+ server = WEBrick::HTTPServer.new(
+ BindAddress: host,
+ Port: port,
+ Logger: WEBrick::Log.new($stderr, WEBrick::BasicLog::WARN),
+ AccessLog: []
+ )
+ server.mount_proc("/") { |request, response| app.handle(request, response) }
+ shutdown = proc { server.shutdown }
+ Signal.trap("INT", &shutdown)
+ Signal.trap("TERM", &shutdown)
+ puts("Open http://#{host}:#{port} and click Start conversation. Press Ctrl-C here to stop.")
+ server.start
+ rescue LoadError
+ raise "webrick is required; run `bundle install` before starting this example"
+ end
+ end
+ end
+ end
+end
+
+OpenAI::Examples::Realtime::WebRTCConversation.run if $PROGRAM_NAME == __FILE__
diff --git a/examples/realtime/websocket_audio.rb b/examples/realtime/websocket_audio.rb
new file mode 100755
index 000000000..c8c08aeaa
--- /dev/null
+++ b/examples/realtime/websocket_audio.rb
@@ -0,0 +1,73 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require_relative "../../lib/openai"
+require_relative "event_stream"
+
+module OpenAI
+ module Examples
+ module Realtime
+ module WebSocketAudio
+ module_function
+
+ def configure(connection)
+ connection.session.update(
+ type: :realtime,
+ output_modalities: [:audio],
+ audio: {input: {turn_detection: nil}}
+ )
+ end
+
+ def stream_response(connection, output:)
+ audio_bytes = 0
+ EventStream.each_until(connection, stop_after: "response.done") do |event|
+ case event
+ when OpenAI::Realtime::ResponseAudioDeltaEvent
+ audio = Base64.strict_decode64(event.delta)
+ output.write(audio)
+ audio_bytes += audio.bytesize
+ when OpenAI::Realtime::ResponseDoneEvent
+ status = event.response.status
+ raise "Response ended with #{status}" unless status == :completed
+ raise "Response completed without audio output" if audio_bytes.zero?
+ when OpenAI::Realtime::RealtimeErrorEvent
+ raise event.error.message
+ end
+ end
+ end
+
+ def run(client:, model:, input_path:, output_path:)
+ File.open(output_path, "wb") do |output|
+ client.realtime.connect(model: model) do |connection|
+ configure(connection)
+ EventStream.wait_for(
+ connection,
+ OpenAI::Realtime::SessionUpdatedEvent,
+ closed_message: "Realtime connection closed before session.updated"
+ )
+ File.open(input_path, "rb") do |input|
+ while (chunk = input.read(4_800))
+ connection.input_audio_buffer.append_bytes(chunk)
+ end
+ end
+ connection.input_audio_buffer.commit
+ connection.response.create
+ stream_response(connection, output: output)
+ end
+ end
+
+ warn("Wrote raw PCM16 audio to #{output_path}")
+ end
+ end
+ end
+ end
+end
+
+if $PROGRAM_NAME == __FILE__
+ OpenAI::Examples::Realtime::WebSocketAudio.run(
+ client: OpenAI::Client.new,
+ model: ENV.fetch("OPENAI_REALTIME_MODEL", "gpt-realtime-2.1"),
+ input_path: ENV.fetch("REALTIME_INPUT_PCM"),
+ output_path: ENV.fetch("REALTIME_OUTPUT_PCM", "realtime-output.pcm")
+ )
+end
diff --git a/examples/realtime/websocket_text.rb b/examples/realtime/websocket_text.rb
new file mode 100755
index 000000000..1f771e5fc
--- /dev/null
+++ b/examples/realtime/websocket_text.rb
@@ -0,0 +1,77 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "timeout"
+require_relative "../../lib/openai"
+require_relative "event_stream"
+
+module OpenAI
+ module Examples
+ module Realtime
+ module WebSocketText
+ module_function
+
+ def stream_response(connection, output: $stdout)
+ started_response = false
+ received_text = false
+ EventStream.each_until(connection, stop_after: "response.done") do |event|
+ case event
+ when OpenAI::Realtime::SessionCreatedEvent
+ output.puts("[realtime] session.created")
+ when OpenAI::Realtime::SessionUpdatedEvent
+ output.puts("[realtime] session.updated")
+ when OpenAI::Realtime::ResponseTextDeltaEvent
+ output.print("[assistant] ") unless started_response
+ started_response = true
+ received_text = true unless event.delta.empty?
+ output.print(event.delta)
+ output.flush
+ when OpenAI::Realtime::ResponseDoneEvent
+ output.puts if started_response
+ status = event.response.status
+ raise "Realtime response ended with status #{status.inspect}" unless status == :completed
+ raise "Realtime response completed without text output" unless received_text
+
+ output.puts("[realtime] response.done status=completed")
+ when OpenAI::Realtime::RealtimeErrorEvent
+ raise "Realtime API error: #{event.error.message}"
+ end
+ end
+ end
+
+ def run(client:, model:, prompt:, output: $stdout)
+ session = {
+ type: :realtime,
+ output_modalities: [:text],
+ instructions: "Be concise and friendly."
+ }
+ item = {
+ type: :message,
+ role: :user,
+ content: [{type: :input_text, text: prompt}]
+ }
+
+ output.puts("[realtime] connecting with #{model}")
+ client.realtime.connect(model: model) do |connection|
+ output.puts("[realtime] connected; sending prompt: #{prompt.inspect}")
+ connection.session.update(**session)
+ connection.conversation.items.create(**item)
+ connection.response.create
+ stream_response(connection, output: output)
+ end
+ output.puts("[realtime] smoke test passed")
+ end
+ end
+ end
+ end
+end
+
+if $PROGRAM_NAME == __FILE__
+ Timeout.timeout(Integer(ENV.fetch("OPENAI_REALTIME_TIMEOUT", "30"))) do
+ OpenAI::Examples::Realtime::WebSocketText.run(
+ client: OpenAI::Client.new,
+ model: ENV.fetch("OPENAI_REALTIME_MODEL", "gpt-realtime-2.1"),
+ prompt: ENV.fetch("OPENAI_REALTIME_PROMPT", "Say hello from Ruby.")
+ )
+ end
+end
diff --git a/examples/realtime/websocket_transcription.rb b/examples/realtime/websocket_transcription.rb
new file mode 100755
index 000000000..c9c70889d
--- /dev/null
+++ b/examples/realtime/websocket_transcription.rb
@@ -0,0 +1,81 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "timeout"
+require_relative "../../lib/openai"
+require_relative "event_stream"
+
+module OpenAI
+ module Examples
+ module Realtime
+ module WebSocketTranscription
+ module_function
+
+ def configure(connection, transcription_model:)
+ connection.session.update(
+ audio: {
+ input: {
+ format: {type: :"audio/pcm", rate: 24_000},
+ transcription: {model: transcription_model},
+ turn_detection: nil
+ }
+ }
+ )
+ end
+
+ def upload(connection, input_path:)
+ File.open(input_path, "rb") do |input|
+ while (chunk = input.read(4_800))
+ connection.input_audio_buffer.append_bytes(chunk)
+ end
+ end
+ connection.input_audio_buffer.commit
+ end
+
+ def print_transcript(connection, output: $stdout)
+ EventStream.each_until(
+ connection,
+ stop_after: "conversation.item.input_audio_transcription.completed",
+ closed_message: "Realtime connection closed before transcription completed"
+ ) do |event|
+ case event
+ when OpenAI::Realtime::ConversationItemInputAudioTranscriptionDeltaEvent
+ output.print(event.delta)
+ output.flush
+ when OpenAI::Realtime::ConversationItemInputAudioTranscriptionCompletedEvent
+ output.puts("\n[#{event.item_id}] #{event.transcript}")
+ when OpenAI::Realtime::RealtimeErrorEvent
+ raise event.error.message
+ end
+ end
+ end
+
+ def run(client:, input_path:, transcription_model:, output: $stdout)
+ client.realtime.connect_transcription do |connection|
+ configure(connection, transcription_model: transcription_model)
+ EventStream.wait_for(
+ connection,
+ OpenAI::Realtime::SessionUpdatedEvent,
+ closed_message: "Realtime connection closed before session.updated"
+ )
+ upload(connection, input_path: input_path)
+ print_transcript(connection, output: output)
+ end
+ end
+ end
+ end
+ end
+end
+
+if $PROGRAM_NAME == __FILE__
+ Timeout.timeout(Integer(ENV.fetch("OPENAI_REALTIME_TIMEOUT", "30"))) do
+ OpenAI::Examples::Realtime::WebSocketTranscription.run(
+ client: OpenAI::Client.new,
+ input_path: ENV.fetch("REALTIME_INPUT_PCM"),
+ transcription_model: ENV.fetch(
+ "OPENAI_REALTIME_TRANSCRIPTION_MODEL",
+ "gpt-live-transcribe"
+ )
+ )
+ end
+end
diff --git a/lib/openai.rb b/lib/openai.rb
index 2cdf3b4fd..81999905b 100644
--- a/lib/openai.rb
+++ b/lib/openai.rb
@@ -800,6 +800,8 @@
require_relative "openai/models/other_file_chunking_strategy_object"
require_relative "openai/models/realtime/audio_transcription"
require_relative "openai/models/realtime/call_accept_params"
+require_relative "openai/models/realtime/call_create_params"
+require_relative "openai/models/realtime/call_create_response"
require_relative "openai/models/realtime/call_hangup_params"
require_relative "openai/models/realtime/call_refer_params"
require_relative "openai/models/realtime/call_reject_params"
@@ -1165,6 +1167,15 @@
require_relative "openai/models/webhooks/unwrap_webhook_event"
require_relative "openai/models/webhooks/webhook_unwrap_params"
require_relative "openai/models"
+require_relative "openai/realtime/unknown_server_event"
+require_relative "openai/realtime/base_connection"
+require_relative "openai/realtime/connection_resources"
+require_relative "openai/realtime/connection"
+require_relative "openai/realtime/sideband_connection"
+require_relative "openai/realtime/transcription_connection"
+require_relative "openai/realtime/translation_connection"
+require_relative "openai/realtime/connection_manager"
+require_relative "openai/realtime/transports/async_websocket"
require_relative "openai/resources/admin"
require_relative "openai/resources/admin/organization"
require_relative "openai/resources/admin/organization/admin_api_keys"
@@ -1245,6 +1256,9 @@
require_relative "openai/resources/realtime"
require_relative "openai/resources/realtime/calls"
require_relative "openai/resources/realtime/client_secrets"
+require_relative "openai/resources/realtime/translations"
+require_relative "openai/resources/realtime/translations/calls"
+require_relative "openai/resources/realtime/translations/client_secrets"
require_relative "openai/resources/responses"
require_relative "openai/resources/responses/input_items"
require_relative "openai/resources/responses/input_tokens"
diff --git a/lib/openai/client.rb b/lib/openai/client.rb
index 6400ca209..e060cca18 100644
--- a/lib/openai/client.rb
+++ b/lib/openai/client.rb
@@ -32,6 +32,11 @@ class Client < OpenAI::Internal::Transport::BaseClient
# @return [String, nil]
attr_reader :webhook_secret
+ # Optional base URL used for WebSocket connections.
+ #
+ # @return [URI::Generic, nil]
+ attr_reader :websocket_base_url
+
# @return [OpenAI::Auth::WorkloadIdentityAuth, nil]
# @api private
attr_reader :workload_identity_auth
@@ -157,6 +162,80 @@ class Client < OpenAI::Internal::Transport::BaseClient
{"authorization" => "Bearer #{@admin_api_key}"}
end
+ # Build a fully authenticated Realtime WebSocket handshake request. Realtime
+ # transports use this boundary so provider authentication and request options stay
+ # consistent with ordinary SDK requests.
+ #
+ # @api private
+ #
+ # @param path [String]
+ # @param query [Hash{String=>String}]
+ # @param options [OpenAI::RequestOptions, Hash{Symbol=>Object}, nil]
+ #
+ # @return [Hash{Symbol=>Object}]
+ def realtime_connection_request(path:, query:, options: nil)
+ if @provider_runtime && @provider_runtime.name != "azure"
+ raise OpenAI::Errors::Error,
+ "Realtime WebSocket connections are not supported by the #{@provider_runtime.name} provider."
+ end
+
+ opts = options.to_h
+ OpenAI::RequestOptions.validate!(opts)
+ request = build_request(
+ {
+ method: :get,
+ path: path,
+ query: query,
+ security: {bearer_auth: true}
+ },
+ opts
+ )
+
+ workload_identity_header = "Bearer #{WORKLOAD_IDENTITY_API_KEY_PLACEHOLDER}"
+ if @workload_identity_auth && request.fetch(:headers)["authorization"] == workload_identity_header
+ token = @workload_identity_auth.get_token
+ request = request.merge(
+ headers: request.fetch(:headers).merge("authorization" => "Bearer #{token}")
+ )
+ end
+
+ request = prepare_request(request, redirect_count: 0, retry_count: 0)
+ if @websocket_base_url
+ url = OpenAI::Internal::Util.join_parsed_uri(
+ OpenAI::Internal::Util.parse_uri(@websocket_base_url.to_s),
+ {path: OpenAI::Internal::Util.interpolate_path(path)}
+ )
+ url.query = request.fetch(:url).query
+ request = request.merge(url: url)
+ end
+
+ url = request.fetch(:url).dup
+ url.scheme = {"http" => "ws", "https" => "wss"}.fetch(url.scheme, url.scheme)
+ headers = request.fetch(:headers).except("accept", "content-type")
+ request.merge(url: url, headers: headers)
+ end
+
+ # @api private
+ #
+ # @param value [String, nil]
+ # @return [URI::Generic, nil]
+ private def parse_websocket_base_url(value)
+ return if value.nil?
+
+ uri = URI(value)
+ valid_scheme = %w[http https ws wss].include?(uri.scheme)
+ ambiguous_component = uri.userinfo || uri.query || uri.fragment
+ unless uri.absolute? && uri.host && valid_scheme && !ambiguous_component
+ message =
+ "`websocket_base_url` must be an absolute HTTP or WebSocket URL " \
+ "without credentials, query, or fragment"
+ raise ArgumentError, message
+ end
+ uri
+ rescue URI::Error => e
+ raise ArgumentError, "`websocket_base_url` is not a valid URL", cause: e
+ end
+
# Creates and returns a new client for interacting with the API.
#
# @param api_key [String, nil] Defaults to `ENV["OPENAI_API_KEY"]`.
@@ -232,6 +311,9 @@ class Client < OpenAI::Internal::Transport::BaseClient
# @param default_headers [Hash{String=>String, nil}, nil] Extra headers to send
# with every request. Explicit values override `ENV["OPENAI_CUSTOM_HEADERS"]`.
#
+ # @param websocket_base_url [String, nil] Override the base URL for WebSocket
+ # connections. The HTTP base URL is used when omitted.
+ #
# @param max_retries [Integer] Max number of retries to attempt after a failed retryable request.
#
# @param timeout [Float, nil]
@@ -259,6 +341,7 @@ def initialize(
provider: nil,
base_url: OpenAI::Internal::OMIT,
default_headers: nil,
+ websocket_base_url: nil,
max_retries: self.class::DEFAULT_MAX_RETRIES,
timeout: self.class::DEFAULT_TIMEOUT_IN_SECONDS,
initial_retry_delay: self.class::DEFAULT_INITIAL_RETRY_DELAY,
@@ -275,7 +358,8 @@ def initialize(
api_key: api_key,
admin_api_key: admin_api_key,
workload_identity: workload_identity,
- base_url: base_url
+ base_url: base_url,
+ websocket_base_url: websocket_base_url
}.filter_map do |name, value|
name unless value.equal?(OpenAI::Internal::OMIT) || value.nil?
end
@@ -352,6 +436,7 @@ def initialize(
@admin_api_key = admin_api_key&.to_s
@webhook_secret = webhook_secret&.to_s
@provider_runtime = provider_runtime
+ @websocket_base_url = parse_websocket_base_url(websocket_base_url)
super(
base_url: base_url,
diff --git a/lib/openai/errors.rb b/lib/openai/errors.rb
index d810efa2b..76456f161 100644
--- a/lib/openai/errors.rb
+++ b/lib/openai/errors.rb
@@ -5,12 +5,54 @@ module Errors
class Error < StandardError
# @!attribute cause
#
- # @return [StandardError, nil]
+ # @return [Exception, nil]
end
class InvalidWebhookSignatureError < OpenAI::Errors::Error
end
+ # Raised when a Realtime WebSocket cannot be opened or used.
+ class RealtimeConnectionError < OpenAI::Errors::Error
+ # @return [URI::Generic]
+ attr_reader :url
+
+ # @return [Exception, nil]
+ def cause = @cause.nil? ? super : @cause
+
+ # @api private
+ #
+ # @param url [URI::Generic]
+ # @param message [String, nil]
+ # @param cause [Exception, nil]
+ def initialize(url:, message: nil, cause: nil)
+ @url = url
+ @cause = cause
+ detail = cause && !cause.message.empty? ? ": #{cause.message}" : "."
+ super(message || "Realtime WebSocket connection error#{detail}")
+ end
+ end
+
+ # Raised when a Realtime WebSocket message cannot be parsed as a typed event.
+ class RealtimeProtocolError < OpenAI::Errors::Error
+ # @return [String]
+ attr_reader :data
+
+ # @return [StandardError, nil]
+ def cause = @cause.nil? ? super : @cause
+
+ # @api private
+ #
+ # @param data [String]
+ # @param message [String, nil]
+ # @param cause [StandardError, nil]
+ def initialize(data:, message: nil, cause: nil)
+ @data = data
+ @cause = cause
+ detail = cause && !cause.message.empty? ? ": #{cause.message}" : "."
+ super(message || "Invalid Realtime WebSocket event#{detail}")
+ end
+ end
+
class ConversionError < OpenAI::Errors::Error
# @return [StandardError, nil]
def cause = @cause.nil? ? super : @cause
diff --git a/lib/openai/internal/logging.rb b/lib/openai/internal/logging.rb
index e633353ec..7537d85ce 100644
--- a/lib/openai/internal/logging.rb
+++ b/lib/openai/internal/logging.rb
@@ -112,6 +112,21 @@ def observe_stream(stream, response:)
ObservedStream.new(stream: stream, context: self, response: response)
end
+ def observe_raw_response(response)
+ return response unless enabled?(:error)
+
+ OpenAI::HTTPClient::Response.new(
+ status: response.status,
+ headers: response.headers,
+ body: ObservedEnumerable.new(
+ enumerable: response.body,
+ context: self,
+ response: response,
+ close: -> { OpenAI::Internal::Util.close_fused!(response.body) }
+ )
+ )
+ end
+
def request_failed(error)
status = error.is_a?(OpenAI::Errors::APIError) ? error.status : nil
request_id = error.is_a?(OpenAI::Errors::APIError) ? error.request_id : nil
@@ -236,10 +251,34 @@ def initialize(stream:, context:, response:)
end
private def iterator
- source = @stream.to_enum
+ ObservedEnumerable.new(
+ enumerable: @stream,
+ context: @context,
+ response: @response,
+ close: -> { @stream.close }
+ )
+ end
+ end
+
+ class ObservedEnumerable
+ include Enumerable
+
+ def initialize(enumerable:, context:, response:, close:)
+ @enumerable = enumerable
+ @context = context
+ @response = response
+ @close = close
+ @iterator = iterator
+ end
+
+ def each(&block) = @iterator.each(&block)
+ def close = OpenAI::Internal::Util.close_fused!(@iterator)
+
+ private def iterator
+ source = @enumerable.to_enum
observed = Enumerator.new do |yielder|
loop do
- event =
+ chunk =
begin
source.next
rescue StopIteration
@@ -249,10 +288,10 @@ def initialize(stream:, context:, response:)
@context.request_failed(e)
raise
end
- yielder << event
+ yielder << chunk
end
end
- OpenAI::Internal::Util.fused_enum(observed) { @stream.close }
+ OpenAI::Internal::Util.fused_enum(observed, &@close)
end
end
diff --git a/lib/openai/internal/transport/base_client.rb b/lib/openai/internal/transport/base_client.rb
index d037f5ff3..32ccc50ae 100644
--- a/lib/openai/internal/transport/base_client.rb
+++ b/lib/openai/internal/transport/base_client.rb
@@ -611,6 +611,19 @@ def send_request(request, redirect_count:, retry_count:, send_retry_header:, &co
end
end
+ # Execute the request specified by `req` without decoding its response. This is
+ # used by resources whose successful response is not modeled JSON.
+ #
+ # @api private
+ #
+ # @param req [Hash{Symbol=>Object}]
+ #
+ # @return [OpenAI::HTTPClient::Response]
+ def request_raw(req)
+ _, response, log_context = perform_request(req)
+ log_context.observe_raw_response(response)
+ end
+
# Execute the request specified by `req`. This is the method that all resource
# methods call into.
#
diff --git a/lib/openai/models/realtime/call_create_params.rb b/lib/openai/models/realtime/call_create_params.rb
new file mode 100644
index 000000000..beec64067
--- /dev/null
+++ b/lib/openai/models/realtime/call_create_params.rb
@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module OpenAI
+ module Models
+ module Realtime
+ # @see OpenAI::Resources::Realtime::Calls#create
+ class CallCreateParams < OpenAI::Internal::Type::BaseModel
+ extend OpenAI::Internal::Type::RequestParameters::Converter
+ include OpenAI::Internal::Type::RequestParameters
+
+ # @!attribute sdp
+ # The WebRTC Session Description Protocol offer.
+ #
+ # @return [String]
+ required :sdp, String
+
+ # @!attribute session
+ # Optional session configuration sent with the SDP offer.
+ #
+ # @return [OpenAI::Models::Realtime::RealtimeSessionCreateRequest, nil]
+ optional :session, -> { OpenAI::Realtime::RealtimeSessionCreateRequest }
+
+ # @!method initialize(sdp:, session: nil, request_options: {})
+ # @param sdp [String] The WebRTC Session Description Protocol offer.
+ # @param session [OpenAI::Models::Realtime::RealtimeSessionCreateRequest] Optional session configuration.
+ # @param request_options [OpenAI::RequestOptions, Hash{Symbol=>Object}]
+ end
+ end
+ end
+end
diff --git a/lib/openai/models/realtime/call_create_response.rb b/lib/openai/models/realtime/call_create_response.rb
new file mode 100644
index 000000000..581bd7fae
--- /dev/null
+++ b/lib/openai/models/realtime/call_create_response.rb
@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module OpenAI
+ module Models
+ module Realtime
+ # The SDP answer and response metadata returned when a WebRTC call is created.
+ class CallCreateResponse < OpenAI::Internal::Type::BaseModel
+ # @!attribute sdp
+ # The WebRTC Session Description Protocol answer.
+ #
+ # @return [String]
+ required :sdp, String
+
+ # @!attribute call_id
+ # The call ID parsed from the response Location header, when present.
+ #
+ # @return [String, nil]
+ optional :call_id, String
+
+ # @!attribute headers
+ # The response headers, normalized to lowercase names.
+ #
+ # @return [Hash{String=>String}]
+ required :headers, -> { OpenAI::Internal::Type::HashOf[String] }
+
+ # @!method initialize(sdp:, headers:, call_id: nil)
+ # @param sdp [String] The WebRTC Session Description Protocol answer.
+ # @param headers [Hash{Symbol,String=>String}] The normalized response headers.
+ # @param call_id [String, nil] The call ID parsed from the Location header.
+ end
+ end
+ end
+end
diff --git a/lib/openai/realtime/base_connection.rb b/lib/openai/realtime/base_connection.rb
new file mode 100644
index 000000000..2172acead
--- /dev/null
+++ b/lib/openai/realtime/base_connection.rb
@@ -0,0 +1,143 @@
+# frozen_string_literal: true
+
+module OpenAI
+ module Realtime
+ # Shared protocol machinery for live Realtime WebSocket connections.
+ #
+ # @api private
+ class BaseConnection
+ include Enumerable
+
+ # @return [URI::Generic]
+ attr_reader :url
+
+ # @api private
+ def initialize(socket:, url:, server_event_type:, client_event_type:)
+ @socket = socket
+ @url = url
+ @server_event_type = server_event_type
+ @client_event_type = client_event_type
+ @server_event_names = discriminator_values(@server_event_type)
+ @client_event_names = discriminator_values(@client_event_type)
+ end
+
+ # Yield server events until the remote peer closes the connection.
+ def each
+ return enum_for(__method__) unless block_given?
+
+ while (event = receive)
+ yield(event)
+ end
+ self
+ end
+
+ # Receive and parse the next server event, or return nil after a clean close.
+ def receive
+ data = receive_raw
+ return nil if data.nil?
+
+ parse_event(data)
+ end
+
+ # Receive the next raw WebSocket message.
+ def receive_raw
+ message = @socket.read
+ message&.to_str
+ end
+
+ # Parse raw JSON as a typed server event. Valid events that are newer than this
+ # SDK remain observable as {UnknownServerEvent} values.
+ def parse_event(data)
+ parsed = JSON.parse(data, symbolize_names: true)
+ type = event_type(parsed)
+ unless @server_event_names.key?(type.to_s)
+ return OpenAI::Realtime::UnknownServerEvent.new(data: parsed)
+ end
+
+ state = OpenAI::Internal::Type::Converter.new_coerce_state
+ event = OpenAI::Internal::Type::Converter.coerce(@server_event_type, parsed, state: state)
+ if (cause = coercion_error(state))
+ raise OpenAI::Errors::RealtimeProtocolError.new(data: data, cause: cause)
+ end
+ event
+ rescue OpenAI::Errors::RealtimeProtocolError
+ raise
+ rescue StandardError => e
+ raise OpenAI::Errors::RealtimeProtocolError.new(data: data, cause: e)
+ end
+
+ # Validate, encode, and send a typed client event.
+ def send_event(event)
+ validate_discriminator!(event, @client_event_names, kind: "client") if event.is_a?(Hash)
+ state = OpenAI::Internal::Type::Converter.new_coerce_state
+ coerced = OpenAI::Internal::Type::Converter.coerce(@client_event_type, event, state: state)
+ if (cause = coercion_error(state))
+ raise ArgumentError.new("Invalid Realtime client event: #{cause.message}"), cause: cause
+ end
+ payload = OpenAI::Internal::Type::Converter.dump(@client_event_type, coerced)
+ validate_discriminator!(payload, @client_event_names, kind: "client")
+ send_raw(JSON.generate(payload))
+ end
+
+ # Send an already encoded text message.
+ def send_raw(data)
+ if closed?
+ raise OpenAI::Errors::RealtimeConnectionError.new(
+ url: @url,
+ message: "Cannot send on a closed Realtime WebSocket."
+ )
+ end
+ @socket.write(data)
+ nil
+ end
+
+ # Close the connection.
+ def close(code: 1000, reason: "")
+ return if closed?
+
+ @socket.close(code: code, reason: reason)
+ nil
+ end
+
+ # @return [Boolean]
+ def closed? = @socket.closed?
+
+ private def discriminator_values(union)
+ union.variants.to_h do |variant|
+ value = variant.fields.fetch(:type).fetch(:const)
+ [value.to_s, true]
+ end
+ end
+
+ private def event_type(event)
+ unless event.is_a?(Hash)
+ raise ArgumentError, "Realtime server event must be a JSON object"
+ end
+
+ type = event[:type]
+ return type if type.is_a?(String) || type.is_a?(Symbol)
+
+ raise ArgumentError, "Realtime server event type must be a string or symbol"
+ end
+
+ private def validate_discriminator!(event, allowed, kind:)
+ type =
+ if event.key?(:type)
+ event.fetch(:type)
+ elsif event.key?("type")
+ event.fetch("type")
+ end
+ return if type && allowed.key?(type.to_s)
+
+ raise ArgumentError, "Unknown Realtime #{kind} event type: #{type.inspect}"
+ end
+
+ private def coercion_error(state)
+ return state[:error] if state[:error]
+ return if state.fetch(:exactness).fetch(:no).zero?
+
+ ArgumentError.new("Realtime event is missing required fields or contains invalid values")
+ end
+ end
+ end
+end
diff --git a/lib/openai/realtime/connection.rb b/lib/openai/realtime/connection.rb
new file mode 100644
index 000000000..c9dd1b3d3
--- /dev/null
+++ b/lib/openai/realtime/connection.rb
@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module OpenAI
+ module Realtime
+ # A live, typed Realtime WebSocket connection.
+ class Connection < OpenAI::Realtime::BaseConnection
+ # @return [OpenAI::Realtime::ConnectionResources::Session]
+ attr_reader :session
+
+ # @return [OpenAI::Realtime::ConnectionResources::Response]
+ attr_reader :response
+
+ # @return [OpenAI::Realtime::ConnectionResources::InputAudioBuffer]
+ attr_reader :input_audio_buffer
+
+ # @return [OpenAI::Realtime::ConnectionResources::Conversation]
+ attr_reader :conversation
+
+ # @api private
+ def initialize(socket:, url:)
+ super(
+ socket: socket,
+ url: url,
+ server_event_type: OpenAI::Realtime::RealtimeServerEvent,
+ client_event_type: OpenAI::Realtime::RealtimeClientEvent
+ )
+ @session = OpenAI::Realtime::ConnectionResources::Session.new(self)
+ @response = OpenAI::Realtime::ConnectionResources::Response.new(self)
+ @input_audio_buffer = OpenAI::Realtime::ConnectionResources::InputAudioBuffer.new(self)
+ @conversation = OpenAI::Realtime::ConnectionResources::Conversation.new(self)
+ end
+ end
+ end
+end
diff --git a/lib/openai/realtime/connection_manager.rb b/lib/openai/realtime/connection_manager.rb
new file mode 100644
index 000000000..ebb247af4
--- /dev/null
+++ b/lib/openai/realtime/connection_manager.rb
@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+module OpenAI
+ module Realtime
+ # Internal block-scoped lifecycle manager for Realtime WebSocket connections.
+ #
+ # @api private
+ class ConnectionManager
+ # @api private
+ def initialize(
+ client:,
+ path:,
+ query:,
+ connection_class:,
+ transport:,
+ request_options:,
+ transport_options:
+ )
+ @client = client
+ @path = path
+ @query = query
+ @connection_class = connection_class
+ @transport = transport
+ @request_options = request_options
+ @transport_options = transport_options
+ end
+
+ # Open the WebSocket and yield a typed connection for the lifetime of the block.
+ #
+ # @yieldparam connection [OpenAI::Realtime::Connection]
+ # @return [Object]
+ def open
+ raise ArgumentError, "A block is required to open a Realtime WebSocket." unless block_given?
+
+ request = @client.realtime_connection_request(
+ path: @path,
+ query: @query,
+ options: @request_options
+ )
+ transport = @transport || OpenAI::Realtime::Transports::AsyncWebSocket.new
+ unless transport.respond_to?(:open)
+ raise ArgumentError, "`transport` must respond to `open`"
+ end
+
+ transport.open(
+ url: request.fetch(:url),
+ headers: request.fetch(:headers),
+ timeout: request.fetch(:timeout),
+ **@transport_options
+ ) do |socket|
+ connection = @connection_class.new(socket: socket, url: request.fetch(:url))
+ begin
+ yield(connection)
+ ensure
+ pending_error = $ERROR_INFO
+ begin
+ connection.close unless connection.closed?
+ rescue StandardError
+ raise if pending_error.nil?
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/openai/realtime/connection_resources.rb b/lib/openai/realtime/connection_resources.rb
new file mode 100644
index 000000000..c070cfedb
--- /dev/null
+++ b/lib/openai/realtime/connection_resources.rb
@@ -0,0 +1,196 @@
+# frozen_string_literal: true
+
+module OpenAI
+ module Realtime
+ module ConnectionResources
+ class Base
+ # @api private
+ def initialize(connection)
+ @connection = connection
+ end
+
+ private def compact_event(event) = event.compact
+
+ private def event_payload(params, *metadata_keys)
+ payload = params.to_h.dup
+ metadata = metadata_keys.to_h do |key|
+ value = payload.key?(key) ? payload.delete(key) : payload.delete(key.to_s)
+ [key, value]
+ end
+ [payload, metadata]
+ end
+ end
+
+ class Session < Base
+ # Update the session using Ruby-style resource keywords. The helper adds the
+ # protocol's nested `session` envelope.
+ def update(params)
+ session, metadata = event_payload(params, :event_id)
+ @connection.send_event(
+ compact_event(type: :"session.update", session: session, **metadata)
+ )
+ end
+ end
+
+ class TranscriptionSession < Base
+ # A transcription connection already selects its session capability. Add the
+ # required wire discriminator so callers only pass transcription fields.
+ def update(params)
+ session, metadata = event_payload(params, :event_id)
+ session.delete(:type)
+ session.delete("type")
+ session[:type] = :transcription
+ @connection.send_event(
+ compact_event(type: :"session.update", session: session, **metadata)
+ )
+ end
+ end
+
+ class TranslationSession < Session
+ # Gracefully flush and close a translation session.
+ def close(event_id: nil)
+ @connection.send_event(compact_event(type: :"session.close", event_id: event_id))
+ end
+ end
+
+ class Response < Base
+ # Create a response from resource fields, omitting the optional wire envelope
+ # when no response-specific fields are supplied.
+ def create(params = {})
+ response, metadata = event_payload(params, :event_id)
+ @connection.send_event(
+ compact_event(
+ type: :"response.create",
+ response: response.empty? ? nil : response,
+ **metadata
+ )
+ )
+ end
+
+ def cancel(response_id: nil, event_id: nil)
+ @connection.send_event(
+ compact_event(type: :"response.cancel", response_id: response_id, event_id: event_id)
+ )
+ end
+ end
+
+ class InputAudioBuffer < Base
+ def append(audio:, event_id: nil)
+ @connection.send_event(
+ compact_event(type: :"input_audio_buffer.append", audio: audio, event_id: event_id)
+ )
+ end
+
+ def append_bytes(bytes, event_id: nil)
+ append(audio: Base64.strict_encode64(bytes), event_id: event_id)
+ end
+
+ def commit(event_id: nil)
+ @connection.send_event(compact_event(type: :"input_audio_buffer.commit", event_id: event_id))
+ end
+
+ def clear(event_id: nil)
+ @connection.send_event(compact_event(type: :"input_audio_buffer.clear", event_id: event_id))
+ end
+ end
+
+ class TranslationInputAudioBuffer < Base
+ def append(audio:, event_id: nil)
+ @connection.send_event(
+ compact_event(type: :"session.input_audio_buffer.append", audio: audio, event_id: event_id)
+ )
+ end
+
+ def append_bytes(bytes, event_id: nil)
+ append(audio: Base64.strict_encode64(bytes), event_id: event_id)
+ end
+ end
+
+ class Conversation < Base
+ # @return [OpenAI::Realtime::ConnectionResources::ConversationItems]
+ attr_reader :items
+
+ # @api private
+ def initialize(connection)
+ super
+ @items = ConversationItems.new(connection)
+ end
+ end
+
+ class ConversationItems < Base
+ # Create an item from resource fields. Event metadata stays at the outer wire
+ # level while the remaining fields become the nested item.
+ def create(params)
+ item, metadata = event_payload(params, :event_id, :previous_item_id)
+ @connection.send_event(
+ compact_event(
+ type: :"conversation.item.create",
+ item: item,
+ **metadata
+ )
+ )
+ end
+
+ def delete(item_id:, event_id: nil)
+ @connection.send_event(
+ compact_event(type: :"conversation.item.delete", item_id: item_id, event_id: event_id)
+ )
+ end
+
+ def retrieve(item_id:, event_id: nil)
+ @connection.send_event(
+ compact_event(type: :"conversation.item.retrieve", item_id: item_id, event_id: event_id)
+ )
+ end
+
+ def truncate(item_id:, content_index:, audio_end_ms:, event_id: nil)
+ @connection.send_event(
+ compact_event(
+ type: :"conversation.item.truncate",
+ item_id: item_id,
+ content_index: content_index,
+ audio_end_ms: audio_end_ms,
+ event_id: event_id
+ )
+ )
+ end
+
+ # Send the output of a locally executed function call back to the model.
+ def create_function_call_output(call_id:, output:, id: nil, status: nil, event_id: nil)
+ item = compact_event(
+ type: :function_call_output,
+ call_id: call_id,
+ output: output,
+ id: id,
+ status: status
+ )
+ create(**item, event_id: event_id)
+ end
+
+ # Approve or reject a pending remote MCP tool call.
+ def respond_to_mcp_approval(
+ approval_request_id:,
+ approve:,
+ reason: nil,
+ id: nil,
+ event_id: nil
+ )
+ item = compact_event(
+ type: :mcp_approval_response,
+ id: id || "mcpa_#{SecureRandom.hex(12)}",
+ approval_request_id: approval_request_id,
+ approve: approve,
+ reason: reason
+ )
+ create(**item, event_id: event_id)
+ end
+ end
+
+ class OutputAudioBuffer < Base
+ def clear(event_id: nil)
+ @connection.send_event(compact_event(type: :"output_audio_buffer.clear", event_id: event_id))
+ end
+ end
+ end
+ end
+end
diff --git a/lib/openai/realtime/sideband_connection.rb b/lib/openai/realtime/sideband_connection.rb
new file mode 100644
index 000000000..177e4f7db
--- /dev/null
+++ b/lib/openai/realtime/sideband_connection.rb
@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module OpenAI
+ module Realtime
+ # A server-side control connection attached to a WebRTC or SIP call.
+ class SidebandConnection < OpenAI::Realtime::Connection
+ # @return [OpenAI::Realtime::ConnectionResources::OutputAudioBuffer]
+ attr_reader :output_audio_buffer
+
+ # @api private
+ def initialize(socket:, url:)
+ super
+ @output_audio_buffer = OpenAI::Realtime::ConnectionResources::OutputAudioBuffer.new(self)
+ end
+ end
+ end
+end
diff --git a/lib/openai/realtime/transcription_connection.rb b/lib/openai/realtime/transcription_connection.rb
new file mode 100644
index 000000000..0d0a906cb
--- /dev/null
+++ b/lib/openai/realtime/transcription_connection.rb
@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module OpenAI
+ module Realtime
+ # A live connection for streaming transcription sessions.
+ class TranscriptionConnection < OpenAI::Realtime::BaseConnection
+ # @return [OpenAI::Realtime::ConnectionResources::TranscriptionSession]
+ attr_reader :session
+
+ # @return [OpenAI::Realtime::ConnectionResources::InputAudioBuffer]
+ attr_reader :input_audio_buffer
+
+ # @api private
+ def initialize(socket:, url:)
+ super(
+ socket: socket,
+ url: url,
+ server_event_type: OpenAI::Realtime::RealtimeServerEvent,
+ client_event_type: OpenAI::Realtime::RealtimeClientEvent
+ )
+ @session = OpenAI::Realtime::ConnectionResources::TranscriptionSession.new(self)
+ @input_audio_buffer = OpenAI::Realtime::ConnectionResources::InputAudioBuffer.new(self)
+ end
+ end
+ end
+end
diff --git a/lib/openai/realtime/translation_connection.rb b/lib/openai/realtime/translation_connection.rb
new file mode 100644
index 000000000..4f089a852
--- /dev/null
+++ b/lib/openai/realtime/translation_connection.rb
@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module OpenAI
+ module Realtime
+ # A live connection for the dedicated continuous translation protocol.
+ class TranslationConnection < OpenAI::Realtime::BaseConnection
+ # @return [OpenAI::Realtime::ConnectionResources::TranslationSession]
+ attr_reader :session
+
+ # @return [OpenAI::Realtime::ConnectionResources::TranslationInputAudioBuffer]
+ attr_reader :input_audio_buffer
+
+ # @api private
+ def initialize(socket:, url:)
+ super(
+ socket: socket,
+ url: url,
+ server_event_type: OpenAI::Realtime::RealtimeTranslationServerEvent,
+ client_event_type: OpenAI::Realtime::RealtimeTranslationClientEvent
+ )
+ @session = OpenAI::Realtime::ConnectionResources::TranslationSession.new(self)
+ @input_audio_buffer = OpenAI::Realtime::ConnectionResources::TranslationInputAudioBuffer.new(self)
+ end
+ end
+ end
+end
diff --git a/lib/openai/realtime/transports/async_websocket.rb b/lib/openai/realtime/transports/async_websocket.rb
new file mode 100644
index 000000000..15c5d3fcd
--- /dev/null
+++ b/lib/openai/realtime/transports/async_websocket.rb
@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+module OpenAI
+ module Realtime
+ module Transports
+ # Default Realtime transport backed by the optional async-websocket gem.
+ class AsyncWebSocket
+ class Socket
+ # @api private
+ def initialize(connection, url:)
+ @connection = connection
+ @url = url
+ end
+
+ def read
+ @connection.read
+ rescue StandardError => e
+ raise OpenAI::Errors::RealtimeConnectionError.new(url: @url, cause: e)
+ end
+
+ def write(message)
+ @connection.write(message)
+ @connection.flush
+ rescue StandardError => e
+ raise OpenAI::Errors::RealtimeConnectionError.new(url: @url, cause: e)
+ end
+
+ def close(code: 1000, reason: "")
+ @connection.close(code, reason)
+ rescue StandardError => e
+ raise OpenAI::Errors::RealtimeConnectionError.new(url: @url, cause: e)
+ end
+
+ def closed? = @connection.closed?
+ end
+
+ def open(url:, headers:, timeout:, **endpoint_options)
+ load_dependencies(url)
+
+ # Classic WebSocket negotiation uses HTTP/1.1. Pinning ALPN also avoids an
+ # HTTP/2 selection on servers that advertise both protocols.
+ options = {
+ alpn_protocols: ::Async::HTTP::Protocol::HTTP11.names,
+ **endpoint_options
+ }
+ endpoint = ::Async::HTTP::Endpoint.parse(
+ url.to_s,
+ timeout: timeout,
+ **options
+ )
+ block_error = nil
+ # Client.connect owns the Sync boundary: it starts a reactor for ordinary
+ # synchronous callers and reuses the current task when one already exists.
+ ::Async::WebSocket::Client.connect(endpoint, headers: headers) do |connection|
+ yield(Socket.new(connection, url: url))
+ rescue StandardError => e
+ block_error = e
+ raise
+ end
+ rescue OpenAI::Errors::RealtimeConnectionError
+ raise
+ rescue StandardError => e
+ raise if e.equal?(block_error)
+
+ raise OpenAI::Errors::RealtimeConnectionError.new(url: url, cause: e)
+ end
+
+ private def load_dependencies(url)
+ require("async/websocket/client")
+ require("async/http/endpoint")
+ rescue LoadError => e
+ message =
+ "Realtime WebSockets require the `async-websocket` gem. " \
+ "Add `gem \"async-websocket\"` to your Gemfile."
+ raise OpenAI::Errors::RealtimeConnectionError.new(url: url, message: message, cause: e)
+ end
+ end
+ end
+ end
+end
diff --git a/lib/openai/realtime/unknown_server_event.rb b/lib/openai/realtime/unknown_server_event.rb
new file mode 100644
index 000000000..99a1818d4
--- /dev/null
+++ b/lib/openai/realtime/unknown_server_event.rb
@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module OpenAI
+ module Realtime
+ # A valid JSON event whose discriminator is newer than this SDK version.
+ class UnknownServerEvent
+ # @return [Symbol]
+ attr_reader :type
+
+ # @return [Hash{Symbol=>Object}]
+ attr_reader :data
+
+ # @api private
+ def initialize(data:)
+ value = data.fetch(:type)
+ unless value.is_a?(String) || value.is_a?(Symbol)
+ raise ArgumentError, "Realtime server event type must be a string or symbol"
+ end
+
+ @type = value.to_sym
+ @data = freeze_json(data)
+ freeze
+ end
+
+ # @return [Hash{Symbol=>Object}]
+ def to_h = @data
+
+ private def freeze_json(value)
+ case value
+ when Hash
+ value.each do |key, item|
+ freeze_json(key)
+ freeze_json(item)
+ end
+ when Array
+ value.each { |item| freeze_json(item) }
+ end
+ value.freeze
+ end
+ end
+ end
+end
diff --git a/lib/openai/resources/realtime.rb b/lib/openai/resources/realtime.rb
index f361d152a..a5f606525 100644
--- a/lib/openai/resources/realtime.rb
+++ b/lib/openai/resources/realtime.rb
@@ -9,6 +9,105 @@ class Realtime
# @return [OpenAI::Resources::Realtime::Calls]
attr_reader :calls
+ # @return [OpenAI::Resources::Realtime::Translations]
+ attr_reader :translations
+
+ # Open a server-side Realtime conversation WebSocket. A block is required and
+ # the socket is closed on every exit path.
+ #
+ # @param model [String]
+ # @param request_options [OpenAI::RequestOptions, Hash{Symbol=>Object}, nil]
+ # @param transport [#open, nil] An alternate WebSocket transport.
+ # @param transport_options [Hash{Symbol=>Object}]
+ # @yieldparam connection [OpenAI::Realtime::Connection]
+ # @return [Object]
+ def connect(
+ model:,
+ request_options: nil,
+ transport: nil,
+ transport_options: {},
+ &block
+ )
+ open_connection(
+ query: {"model" => model},
+ connection_class: OpenAI::Realtime::Connection,
+ request_options: request_options,
+ transport: transport,
+ transport_options: transport_options,
+ &block
+ )
+ end
+
+ # Attach a sideband WebSocket to an existing WebRTC or SIP call.
+ #
+ # @param call_id [String]
+ # @param request_options [OpenAI::RequestOptions, Hash{Symbol=>Object}, nil]
+ # @param transport [#open, nil] An alternate WebSocket transport.
+ # @param transport_options [Hash{Symbol=>Object}]
+ # @yieldparam connection [OpenAI::Realtime::SidebandConnection]
+ # @return [Object]
+ def connect_to_call(
+ call_id:,
+ request_options: nil,
+ transport: nil,
+ transport_options: {},
+ &block
+ )
+ open_connection(
+ query: {"call_id" => call_id},
+ connection_class: OpenAI::Realtime::SidebandConnection,
+ request_options: request_options,
+ transport: transport,
+ transport_options: transport_options,
+ &block
+ )
+ end
+
+ # Open a dedicated Realtime transcription WebSocket.
+ #
+ # @param request_options [OpenAI::RequestOptions, Hash{Symbol=>Object}, nil]
+ # @param transport [#open, nil] An alternate WebSocket transport.
+ # @param transport_options [Hash{Symbol=>Object}]
+ # @yieldparam connection [OpenAI::Realtime::TranscriptionConnection]
+ # @return [Object]
+ def connect_transcription(
+ request_options: nil,
+ transport: nil,
+ transport_options: {},
+ &block
+ )
+ open_connection(
+ query: {"intent" => "transcription"},
+ connection_class: OpenAI::Realtime::TranscriptionConnection,
+ request_options: request_options,
+ transport: transport,
+ transport_options: transport_options,
+ &block
+ )
+ end
+
+ private def open_connection(
+ query:,
+ connection_class:,
+ request_options:,
+ transport:,
+ transport_options:,
+ &block
+ )
+ raise ArgumentError, "A block is required to open a Realtime WebSocket." unless block
+
+ manager = OpenAI::Realtime::ConnectionManager.new(
+ client: @client,
+ path: "realtime",
+ query: query,
+ connection_class: connection_class,
+ transport: transport,
+ request_options: request_options,
+ transport_options: transport_options
+ )
+ manager.open(&block)
+ end
+
# @api private
#
# @param client [OpenAI::Client]
@@ -16,6 +115,7 @@ def initialize(client:)
@client = client
@client_secrets = OpenAI::Resources::Realtime::ClientSecrets.new(client: client)
@calls = OpenAI::Resources::Realtime::Calls.new(client: client)
+ @translations = OpenAI::Resources::Realtime::Translations.new(client: client)
end
end
end
diff --git a/lib/openai/resources/realtime/calls.rb b/lib/openai/resources/realtime/calls.rb
index a0cfb6f4d..ef06734da 100644
--- a/lib/openai/resources/realtime/calls.rb
+++ b/lib/openai/resources/realtime/calls.rb
@@ -4,6 +4,61 @@ module OpenAI
module Resources
class Realtime
class Calls
+ # Create a WebRTC call from an SDP offer. When `session` is supplied, the SDK
+ # sends the offer and session configuration as typed multipart form parts.
+ #
+ # @overload create(sdp:, session: nil, request_options: {})
+ #
+ # @param sdp [String] The WebRTC Session Description Protocol offer.
+ # @param session [OpenAI::Models::Realtime::RealtimeSessionCreateRequest] Optional session configuration.
+ # @param request_options [OpenAI::RequestOptions, Hash{Symbol=>Object}, nil]
+ #
+ # @return [OpenAI::Models::Realtime::CallCreateResponse]
+ #
+ # @see OpenAI::Models::Realtime::CallCreateParams
+ def create(params)
+ parsed, options = OpenAI::Realtime::CallCreateParams.dump_request(params)
+ sdp = parsed.fetch(:sdp)
+ session = parsed[:session]
+ headers, body =
+ if session.nil?
+ [{"accept" => "application/sdp", "content-type" => "application/sdp"}, sdp]
+ else
+ [
+ {"accept" => "application/sdp", "content-type" => "multipart/form-data"},
+ {
+ sdp: OpenAI::FilePart.new(sdp, content_type: "application/sdp"),
+ session: OpenAI::FilePart.new(JSON.generate(session), content_type: "application/json")
+ }
+ ]
+ end
+
+ response = @client.request_raw(
+ method: :post,
+ path: "realtime/calls",
+ headers: headers,
+ body: body,
+ security: {bearer_auth: true},
+ options: options
+ )
+ location = response.headers["location"]
+ call_id = call_id_from_location(location)
+ OpenAI::Realtime::CallCreateResponse.new(
+ sdp: response.body.to_a.join,
+ call_id: call_id,
+ headers: response.headers
+ )._set_last_response(response.metadata)
+ end
+
+ # A Location header is helpful metadata, but the SDP answer remains usable
+ # when an intermediary supplies a malformed value.
+ private def call_id_from_location(location)
+ path = location&.then { URI.parse(_1).path }
+ path&.split("/")&.last
+ rescue URI::InvalidURIError
+ nil
+ end
+
# Some parameter documentations has been truncated, see
# {OpenAI::Models::Realtime::CallAcceptParams} for more details.
#
diff --git a/lib/openai/resources/realtime/translations.rb b/lib/openai/resources/realtime/translations.rb
new file mode 100644
index 000000000..3f8e81ffc
--- /dev/null
+++ b/lib/openai/resources/realtime/translations.rb
@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module OpenAI
+ module Resources
+ class Realtime
+ # Dedicated Realtime translation endpoints and WebSocket connections.
+ class Translations
+ # @return [OpenAI::Resources::Realtime::Translations::ClientSecrets]
+ attr_reader :client_secrets
+
+ # @return [OpenAI::Resources::Realtime::Translations::Calls]
+ attr_reader :calls
+
+ # Open a typed translation WebSocket connection.
+ #
+ # @param model [String]
+ # @param request_options [OpenAI::RequestOptions, Hash{Symbol=>Object}, nil]
+ # @param transport [#open, nil]
+ # @param transport_options [Hash{Symbol=>Object}]
+ # @yieldparam connection [OpenAI::Realtime::TranslationConnection]
+ # @return [Object]
+ def connect(model:, request_options: nil, transport: nil, transport_options: {}, &block)
+ raise ArgumentError, "A block is required to open a Realtime WebSocket." unless block
+
+ manager = OpenAI::Realtime::ConnectionManager.new(
+ client: @client,
+ path: "realtime/translations",
+ query: {"model" => model},
+ connection_class: OpenAI::Realtime::TranslationConnection,
+ transport: transport,
+ request_options: request_options,
+ transport_options: transport_options
+ )
+ manager.open(&block)
+ end
+
+ # @api private
+ def initialize(client:)
+ @client = client
+ @client_secrets = ClientSecrets.new(client: client)
+ @calls = Calls.new(client: client)
+ end
+ end
+ end
+ end
+end
diff --git a/lib/openai/resources/realtime/translations/calls.rb b/lib/openai/resources/realtime/translations/calls.rb
new file mode 100644
index 000000000..132db5d0f
--- /dev/null
+++ b/lib/openai/resources/realtime/translations/calls.rb
@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module OpenAI
+ module Resources
+ class Realtime
+ class Translations
+ class Calls
+ # Create a translation WebRTC call from an SDP offer.
+ #
+ # @param sdp [String]
+ # @param request_options [OpenAI::RequestOptions, Hash{Symbol=>Object}, nil]
+ # @return [OpenAI::Models::Realtime::CallCreateResponse]
+ def create(sdp:, request_options: nil)
+ @client.realtime.calls.create(
+ sdp: sdp,
+ request_options: request_options
+ )
+ end
+
+ # @api private
+ def initialize(client:)
+ @client = client
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/openai/resources/realtime/translations/client_secrets.rb b/lib/openai/resources/realtime/translations/client_secrets.rb
new file mode 100644
index 000000000..d3c59d3eb
--- /dev/null
+++ b/lib/openai/resources/realtime/translations/client_secrets.rb
@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module OpenAI
+ module Resources
+ class Realtime
+ class Translations
+ class ClientSecrets
+ # Create a short-lived client secret for a Realtime translation session.
+ #
+ # @param session [OpenAI::Models::Realtime::RealtimeTranslationSessionCreateRequest, Hash]
+ # @param expires_after [OpenAI::Models::Realtime::RealtimeTranslationClientSecretCreateRequest::ExpiresAfter, Hash, nil]
+ # @param request_options [OpenAI::RequestOptions, Hash{Symbol=>Object}, nil]
+ # @return [OpenAI::Models::Realtime::RealtimeTranslationClientSecretCreateResponse]
+ def create(session:, expires_after: nil, request_options: nil)
+ input = {session: session}
+ input[:expires_after] = expires_after unless expires_after.nil?
+ state = OpenAI::Internal::Type::Converter.new_coerce_state
+ request = OpenAI::Internal::Type::Converter.coerce(
+ OpenAI::Realtime::RealtimeTranslationClientSecretCreateRequest,
+ input,
+ state: state
+ )
+ cause = state[:error]
+ if cause.nil? && !state.fetch(:exactness).fetch(:no).zero?
+ cause = ArgumentError.new("request is missing required fields or contains invalid values")
+ end
+ if cause
+ message = "Invalid translation client secret request: #{cause.message}"
+ raise ArgumentError.new(message), cause: cause
+ end
+ body = OpenAI::Internal::Type::Converter.dump(
+ OpenAI::Realtime::RealtimeTranslationClientSecretCreateRequest,
+ request
+ )
+
+ @client.request(
+ method: :post,
+ path: "realtime/translations/client_secrets",
+ body: body,
+ model: OpenAI::Realtime::RealtimeTranslationClientSecretCreateResponse,
+ security: {bearer_auth: true},
+ options: request_options
+ )
+ end
+
+ # @api private
+ def initialize(client:)
+ @client = client
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/openai.gemspec b/openai.gemspec
index 27122f711..517393174 100644
--- a/openai.gemspec
+++ b/openai.gemspec
@@ -24,7 +24,7 @@ Gem::Specification.new do |s|
"CHANGELOG.md",
".ignore"
]
- s.extra_rdoc_files = ["README.md", "VERSIONING.md"]
+ s.extra_rdoc_files = ["README.md", "VERSIONING.md", "realtime.md"]
s.add_dependency "base64"
s.add_dependency "cgi"
s.add_dependency "connection_pool", ">= 2.2.3"
diff --git a/rbi/openai/client.rbi b/rbi/openai/client.rbi
index 95e4ec364..71b893cf1 100644
--- a/rbi/openai/client.rbi
+++ b/rbi/openai/client.rbi
@@ -25,6 +25,9 @@ module OpenAI
sig { returns(T.nilable(String)) }
attr_reader :webhook_secret
+ sig { returns(T.nilable(URI::Generic)) }
+ attr_reader :websocket_base_url
+
# Given a prompt, the model will return one or more predicted completions, and can
# also return the probabilities of alternative tokens at each position.
sig { returns(OpenAI::Resources::Completions) }
@@ -135,6 +138,17 @@ module OpenAI
private def admin_api_key_auth
end
+ # @api private
+ sig do
+ params(
+ path: String,
+ query: T::Hash[String, String],
+ options: T.nilable(OpenAI::RequestOptions::OrHash)
+ ).returns(OpenAI::Internal::Transport::BaseClient::RequestInput)
+ end
+ def realtime_connection_request(path:, query:, options: nil)
+ end
+
# @api private
sig do
override
@@ -160,6 +174,7 @@ module OpenAI
provider: T.nilable(OpenAI::Provider),
base_url: T.nilable(String),
default_headers: T.nilable(T::Hash[String, T.nilable(String)]),
+ websocket_base_url: T.nilable(String),
max_retries: Integer,
timeout: T.nilable(Float),
initial_retry_delay: Float,
@@ -189,6 +204,7 @@ module OpenAI
# Extra headers to send with every request. Explicit values override
# `ENV["OPENAI_CUSTOM_HEADERS"]`.
default_headers: nil,
+ websocket_base_url: nil,
# Max number of retries to attempt after a failed retryable request.
max_retries: OpenAI::Client::DEFAULT_MAX_RETRIES,
timeout: OpenAI::Client::DEFAULT_TIMEOUT_IN_SECONDS,
diff --git a/rbi/openai/errors.rbi b/rbi/openai/errors.rbi
index 5307238e2..d955c8164 100644
--- a/rbi/openai/errors.rbi
+++ b/rbi/openai/errors.rbi
@@ -3,7 +3,7 @@
module OpenAI
module Errors
class Error < StandardError
- sig { returns(T.nilable(StandardError)) }
+ sig { returns(T.nilable(Exception)) }
attr_accessor :cause
end
@@ -26,6 +26,44 @@ module OpenAI
end
end
+ class RealtimeConnectionError < OpenAI::Errors::Error
+ sig { returns(URI::Generic) }
+ attr_reader :url
+
+ sig { returns(T.nilable(Exception)) }
+ def cause
+ end
+
+ sig do
+ params(
+ url: URI::Generic,
+ message: T.nilable(String),
+ cause: T.nilable(Exception)
+ ).returns(T.attached_class)
+ end
+ def self.new(url:, message: nil, cause: nil)
+ end
+ end
+
+ class RealtimeProtocolError < OpenAI::Errors::Error
+ sig { returns(String) }
+ attr_reader :data
+
+ sig { returns(T.nilable(StandardError)) }
+ def cause
+ end
+
+ sig do
+ params(
+ data: String,
+ message: T.nilable(String),
+ cause: T.nilable(StandardError)
+ ).returns(T.attached_class)
+ end
+ def self.new(data:, message: nil, cause: nil)
+ end
+ end
+
class APIError < OpenAI::Errors::Error
sig { returns(URI::Generic) }
attr_accessor :url
diff --git a/rbi/openai/internal/logging.rbi b/rbi/openai/internal/logging.rbi
index 9a0b7df71..e0902b4ee 100644
--- a/rbi/openai/internal/logging.rbi
+++ b/rbi/openai/internal/logging.rbi
@@ -77,6 +77,14 @@ module OpenAI
def observe_stream(stream, response:)
end
+ sig do
+ params(response: OpenAI::HTTPClient::Response).returns(
+ OpenAI::HTTPClient::Response
+ )
+ end
+ def observe_raw_response(response)
+ end
+
sig { params(error: StandardError).void }
def request_failed(error)
end
@@ -140,6 +148,35 @@ module OpenAI
end
end
+ class ObservedEnumerable
+ include Enumerable
+
+ Elem = type_member(:out)
+
+ sig do
+ params(
+ enumerable: T::Enumerable[Elem],
+ context: OpenAI::Internal::Logging::Context,
+ response: OpenAI::HTTPClient::Response,
+ close: T.proc.void
+ ).void
+ end
+ def initialize(enumerable:, context:, response:, close:)
+ end
+
+ sig { params(block: T.untyped).returns(T.untyped) }
+ def each(&block)
+ end
+
+ sig { void }
+ def close
+ end
+
+ sig { returns(T::Enumerable[Elem]) }
+ private def iterator
+ end
+ end
+
class << self
sig { params(value: T.any(Symbol, String)).returns(Symbol) }
def normalize_level(value)
diff --git a/rbi/openai/internal/transport/base_client.rbi b/rbi/openai/internal/transport/base_client.rbi
index 627dc659f..7465ea40c 100644
--- a/rbi/openai/internal/transport/base_client.rbi
+++ b/rbi/openai/internal/transport/base_client.rbi
@@ -302,6 +302,15 @@ module OpenAI
)
end
+ # @api private
+ sig do
+ params(
+ req: OpenAI::Internal::Transport::BaseClient::RequestComponents
+ ).returns(OpenAI::HTTPClient::Response)
+ end
+ def request_raw(req)
+ end
+
# Execute the request specified by `req`. This is the method that all resource
# methods call into.
#
diff --git a/rbi/openai/models/realtime/call_create_params.rbi b/rbi/openai/models/realtime/call_create_params.rbi
new file mode 100644
index 000000000..ab5ff8a41
--- /dev/null
+++ b/rbi/openai/models/realtime/call_create_params.rbi
@@ -0,0 +1,49 @@
+# typed: strong
+
+module OpenAI
+ module Models
+ module Realtime
+ class CallCreateParams < OpenAI::Internal::Type::BaseModel
+ extend OpenAI::Internal::Type::RequestParameters::Converter
+ include OpenAI::Internal::Type::RequestParameters
+
+ OrHash =
+ T.type_alias do
+ T.any(OpenAI::Realtime::CallCreateParams, OpenAI::Internal::AnyHash)
+ end
+
+ sig { returns(String) }
+ attr_accessor :sdp
+
+ sig do
+ returns(T.nilable(OpenAI::Realtime::RealtimeSessionCreateRequest))
+ end
+ attr_accessor :session
+
+ sig do
+ params(
+ sdp: String,
+ session:
+ T.nilable(OpenAI::Realtime::RealtimeSessionCreateRequest::OrHash),
+ request_options: OpenAI::RequestOptions::OrHash
+ ).returns(T.attached_class)
+ end
+ def self.new(sdp:, session: nil, request_options: {})
+ end
+
+ sig do
+ override.returns(
+ {
+ sdp: String,
+ session:
+ T.nilable(OpenAI::Realtime::RealtimeSessionCreateRequest),
+ request_options: OpenAI::RequestOptions
+ }
+ )
+ end
+ def to_hash
+ end
+ end
+ end
+ end
+end
diff --git a/rbi/openai/models/realtime/call_create_response.rbi b/rbi/openai/models/realtime/call_create_response.rbi
new file mode 100644
index 000000000..5e1fa9645
--- /dev/null
+++ b/rbi/openai/models/realtime/call_create_response.rbi
@@ -0,0 +1,48 @@
+# typed: strong
+
+module OpenAI
+ module Models
+ module Realtime
+ class CallCreateResponse < OpenAI::Internal::Type::BaseModel
+ OrHash =
+ T.type_alias do
+ T.any(
+ OpenAI::Realtime::CallCreateResponse,
+ OpenAI::Internal::AnyHash
+ )
+ end
+
+ sig { returns(String) }
+ attr_accessor :sdp
+
+ sig { returns(T.nilable(String)) }
+ attr_accessor :call_id
+
+ sig { returns(T::Hash[String, String]) }
+ attr_accessor :headers
+
+ sig do
+ params(
+ sdp: String,
+ headers: T::Hash[String, String],
+ call_id: T.nilable(String)
+ ).returns(T.attached_class)
+ end
+ def self.new(sdp:, headers:, call_id: nil)
+ end
+
+ sig do
+ override.returns(
+ {
+ sdp: String,
+ call_id: T.nilable(String),
+ headers: T::Hash[String, String]
+ }
+ )
+ end
+ def to_hash
+ end
+ end
+ end
+ end
+end
diff --git a/rbi/openai/realtime/base_connection.rbi b/rbi/openai/realtime/base_connection.rbi
new file mode 100644
index 000000000..1b1b515f5
--- /dev/null
+++ b/rbi/openai/realtime/base_connection.rbi
@@ -0,0 +1,40 @@
+# typed: strong
+
+module OpenAI
+ module Models
+ module Realtime
+ class BaseConnection
+ sig { returns(URI::Generic) }
+ attr_reader :url
+
+ # @api private
+ sig do
+ params(
+ socket: T.untyped,
+ url: URI::Generic,
+ server_event_type: T.untyped,
+ client_event_type: T.untyped
+ ).returns(T.attached_class)
+ end
+ def self.new(socket:, url:, server_event_type:, client_event_type:)
+ end
+
+ sig { returns(T.nilable(String)) }
+ def receive_raw
+ end
+
+ sig { params(data: String).void }
+ def send_raw(data)
+ end
+
+ sig { params(code: Integer, reason: String).void }
+ def close(code: 1000, reason: "")
+ end
+
+ sig { returns(T::Boolean) }
+ def closed?
+ end
+ end
+ end
+ end
+end
diff --git a/rbi/openai/realtime/connection.rbi b/rbi/openai/realtime/connection.rbi
new file mode 100644
index 000000000..0a9e05d5e
--- /dev/null
+++ b/rbi/openai/realtime/connection.rbi
@@ -0,0 +1,70 @@
+# typed: strong
+
+module OpenAI
+ module Models
+ module Realtime
+ class Connection < OpenAI::Realtime::BaseConnection
+ include Enumerable
+
+ ServerEvent =
+ T.type_alias do
+ T.any(
+ OpenAI::Realtime::RealtimeServerEvent::Variants,
+ OpenAI::Realtime::UnknownServerEvent
+ )
+ end
+
+ ClientEvent =
+ T.type_alias do
+ T.any(
+ OpenAI::Realtime::RealtimeClientEvent::Variants,
+ OpenAI::Internal::AnyHash
+ )
+ end
+
+ Elem = type_member { { fixed: ServerEvent } }
+
+ sig { returns(OpenAI::Realtime::ConnectionResources::Session) }
+ attr_reader :session
+
+ sig { returns(OpenAI::Realtime::ConnectionResources::Response) }
+ attr_reader :response
+
+ sig { returns(OpenAI::Realtime::ConnectionResources::InputAudioBuffer) }
+ attr_reader :input_audio_buffer
+
+ sig { returns(OpenAI::Realtime::ConnectionResources::Conversation) }
+ attr_reader :conversation
+
+ # @api private
+ sig do
+ params(socket: T.untyped, url: URI::Generic).returns(T.attached_class)
+ end
+ def self.new(socket:, url:)
+ end
+
+ sig do
+ params(
+ block: T.nilable(T.proc.params(event: ServerEvent).void)
+ ).returns(
+ T.any(OpenAI::Realtime::Connection, T::Enumerator[ServerEvent])
+ )
+ end
+ def each(&block)
+ end
+
+ sig { returns(T.nilable(ServerEvent)) }
+ def receive
+ end
+
+ sig { params(data: String).returns(ServerEvent) }
+ def parse_event(data)
+ end
+
+ sig { params(event: ClientEvent).void }
+ def send_event(event)
+ end
+ end
+ end
+ end
+end
diff --git a/rbi/openai/realtime/connection_manager.rbi b/rbi/openai/realtime/connection_manager.rbi
new file mode 100644
index 000000000..5af34758a
--- /dev/null
+++ b/rbi/openai/realtime/connection_manager.rbi
@@ -0,0 +1,44 @@
+# typed: strong
+
+module OpenAI
+ module Models
+ module Realtime
+ class ConnectionManager
+ # @api private
+ sig do
+ params(
+ client: OpenAI::Client,
+ path: String,
+ query: T::Hash[String, String],
+ connection_class: T.class_of(OpenAI::Realtime::BaseConnection),
+ transport: T.untyped,
+ request_options: T.nilable(OpenAI::RequestOptions::OrHash),
+ transport_options: T::Hash[Symbol, T.untyped]
+ ).returns(T.attached_class)
+ end
+ def self.new(
+ client:,
+ path:,
+ query:,
+ connection_class:,
+ transport:,
+ request_options:,
+ transport_options:
+ )
+ end
+
+ sig do
+ params(
+ block:
+ T
+ .proc
+ .params(connection: OpenAI::Realtime::BaseConnection)
+ .returns(T.untyped)
+ ).returns(T.untyped)
+ end
+ def open(&block)
+ end
+ end
+ end
+ end
+end
diff --git a/rbi/openai/realtime/connection_resources.rbi b/rbi/openai/realtime/connection_resources.rbi
new file mode 100644
index 000000000..1d74ea96b
--- /dev/null
+++ b/rbi/openai/realtime/connection_resources.rbi
@@ -0,0 +1,328 @@
+# typed: strong
+
+module OpenAI
+ module Models
+ module Realtime
+ module ConnectionResources
+ class Base
+ # @api private
+ sig do
+ params(connection: OpenAI::Realtime::BaseConnection).returns(
+ T.attached_class
+ )
+ end
+ def self.new(connection)
+ end
+ end
+
+ class Session < Base
+ sig do
+ params(
+ type: Symbol,
+ audio:
+ T.any(
+ OpenAI::Realtime::RealtimeAudioConfig::OrHash,
+ OpenAI::Realtime::RealtimeTranscriptionSessionAudio::OrHash
+ ),
+ include: T::Array[Symbol],
+ instructions: String,
+ max_output_tokens: T.any(Integer, Symbol),
+ model: T.any(String, Symbol),
+ output_modalities: T::Array[Symbol],
+ parallel_tool_calls: T::Boolean,
+ prompt: T.nilable(OpenAI::Responses::ResponsePrompt::OrHash),
+ reasoning: OpenAI::Realtime::RealtimeReasoning::OrHash,
+ tool_choice:
+ T.any(
+ OpenAI::Responses::ToolChoiceOptions::OrSymbol,
+ OpenAI::Responses::ToolChoiceFunction::OrHash,
+ OpenAI::Responses::ToolChoiceMcp::OrHash
+ ),
+ tools:
+ T::Array[
+ T.any(
+ OpenAI::Realtime::RealtimeFunctionTool::OrHash,
+ OpenAI::Realtime::RealtimeToolsConfigUnion::Mcp::OrHash
+ )
+ ],
+ tracing:
+ T.nilable(
+ T.any(
+ Symbol,
+ OpenAI::Realtime::RealtimeTracingConfig::TracingConfiguration::OrHash
+ )
+ ),
+ truncation:
+ T.any(
+ OpenAI::Realtime::RealtimeTruncation::RealtimeTruncationStrategy::OrSymbol,
+ OpenAI::Realtime::RealtimeTruncationRetentionRatio::OrHash
+ ),
+ event_id: T.nilable(String)
+ ).void
+ end
+ def update(
+ type:,
+ audio: nil,
+ include: nil,
+ instructions: nil,
+ max_output_tokens: nil,
+ model: nil,
+ output_modalities: nil,
+ parallel_tool_calls: nil,
+ prompt: nil,
+ reasoning: nil,
+ tool_choice: nil,
+ tools: nil,
+ tracing: nil,
+ truncation: nil,
+ event_id: nil
+ )
+ end
+ end
+
+ class TranscriptionSession < Base
+ sig do
+ params(
+ audio:
+ OpenAI::Realtime::RealtimeTranscriptionSessionAudio::OrHash,
+ include:
+ T::Array[
+ OpenAI::Realtime::RealtimeTranscriptionSessionCreateRequest::Include::OrSymbol
+ ],
+ event_id: T.nilable(String)
+ ).void
+ end
+ def update(audio: nil, include: nil, event_id: nil)
+ end
+ end
+
+ class TranslationSession < Session
+ sig do
+ params(
+ audio:
+ OpenAI::Realtime::RealtimeTranslationSessionUpdateRequest::Audio::OrHash,
+ event_id: T.nilable(String)
+ ).void
+ end
+ def update(audio: nil, event_id: nil)
+ end
+
+ sig { params(event_id: T.nilable(String)).void }
+ def close(event_id: nil)
+ end
+ end
+
+ class Response < Base
+ sig do
+ params(
+ audio: OpenAI::Realtime::RealtimeResponseCreateAudioOutput::OrHash,
+ conversation: T.any(String, Symbol),
+ input:
+ T::Array[
+ T.any(
+ OpenAI::Realtime::RealtimeConversationItemSystemMessage::OrHash,
+ OpenAI::Realtime::RealtimeConversationItemUserMessage::OrHash,
+ OpenAI::Realtime::RealtimeConversationItemAssistantMessage::OrHash,
+ OpenAI::Realtime::RealtimeConversationItemFunctionCall::OrHash,
+ OpenAI::Realtime::RealtimeConversationItemFunctionCallOutput::OrHash,
+ OpenAI::Realtime::RealtimeMcpApprovalResponse::OrHash,
+ OpenAI::Realtime::RealtimeMcpListTools::OrHash,
+ OpenAI::Realtime::RealtimeMcpToolCall::OrHash,
+ OpenAI::Realtime::RealtimeMcpApprovalRequest::OrHash
+ )
+ ],
+ instructions: String,
+ max_output_tokens: T.any(Integer, Symbol),
+ metadata: T.nilable(T::Hash[Symbol, String]),
+ output_modalities: T::Array[Symbol],
+ parallel_tool_calls: T::Boolean,
+ prompt: T.nilable(OpenAI::Responses::ResponsePrompt::OrHash),
+ reasoning: OpenAI::Realtime::RealtimeReasoning::OrHash,
+ tool_choice:
+ T.any(
+ OpenAI::Responses::ToolChoiceOptions::OrSymbol,
+ OpenAI::Responses::ToolChoiceFunction::OrHash,
+ OpenAI::Responses::ToolChoiceMcp::OrHash
+ ),
+ tools:
+ T::Array[
+ T.any(
+ OpenAI::Realtime::RealtimeFunctionTool::OrHash,
+ OpenAI::Realtime::RealtimeResponseCreateMcpTool::OrHash
+ )
+ ],
+ event_id: T.nilable(String)
+ ).void
+ end
+ def create(
+ audio: nil,
+ conversation: nil,
+ input: nil,
+ instructions: nil,
+ max_output_tokens: nil,
+ metadata: nil,
+ output_modalities: nil,
+ parallel_tool_calls: nil,
+ prompt: nil,
+ reasoning: nil,
+ tool_choice: nil,
+ tools: nil,
+ event_id: nil
+ )
+ end
+
+ sig do
+ params(
+ response_id: T.nilable(String),
+ event_id: T.nilable(String)
+ ).void
+ end
+ def cancel(response_id: nil, event_id: nil)
+ end
+ end
+
+ class InputAudioBuffer < Base
+ sig { params(audio: String, event_id: T.nilable(String)).void }
+ def append(audio:, event_id: nil)
+ end
+
+ sig { params(bytes: String, event_id: T.nilable(String)).void }
+ def append_bytes(bytes, event_id: nil)
+ end
+
+ sig { params(event_id: T.nilable(String)).void }
+ def commit(event_id: nil)
+ end
+
+ sig { params(event_id: T.nilable(String)).void }
+ def clear(event_id: nil)
+ end
+ end
+
+ class TranslationInputAudioBuffer < Base
+ sig { params(audio: String, event_id: T.nilable(String)).void }
+ def append(audio:, event_id: nil)
+ end
+
+ sig { params(bytes: String, event_id: T.nilable(String)).void }
+ def append_bytes(bytes, event_id: nil)
+ end
+ end
+
+ class Conversation < Base
+ sig do
+ returns(OpenAI::Realtime::ConnectionResources::ConversationItems)
+ end
+ attr_reader :items
+ end
+
+ class ConversationItems < Base
+ sig do
+ params(
+ type: Symbol,
+ arguments: String,
+ approval_request_id: T.nilable(String),
+ approve: T::Boolean,
+ call_id: String,
+ content: T::Array[OpenAI::Internal::AnyHash],
+ error: T.untyped,
+ id: String,
+ name: String,
+ object: Symbol,
+ output: T.nilable(String),
+ reason: T.nilable(String),
+ role: Symbol,
+ server_label: String,
+ status: Symbol,
+ tools: T::Array[OpenAI::Realtime::RealtimeMcpListTools::Tool::OrHash],
+ event_id: T.nilable(String),
+ previous_item_id: T.nilable(String)
+ ).void
+ end
+ def create(
+ type:,
+ arguments: nil,
+ approval_request_id: nil,
+ approve: nil,
+ call_id: nil,
+ content: nil,
+ error: nil,
+ id: nil,
+ name: nil,
+ object: nil,
+ output: nil,
+ reason: nil,
+ role: nil,
+ server_label: nil,
+ status: nil,
+ tools: nil,
+ event_id: nil,
+ previous_item_id: nil
+ )
+ end
+
+ sig { params(item_id: String, event_id: T.nilable(String)).void }
+ def delete(item_id:, event_id: nil)
+ end
+
+ sig { params(item_id: String, event_id: T.nilable(String)).void }
+ def retrieve(item_id:, event_id: nil)
+ end
+
+ sig do
+ params(
+ item_id: String,
+ content_index: Integer,
+ audio_end_ms: Integer,
+ event_id: T.nilable(String)
+ ).void
+ end
+ def truncate(item_id:, content_index:, audio_end_ms:, event_id: nil)
+ end
+
+ sig do
+ params(
+ call_id: String,
+ output: String,
+ id: T.nilable(String),
+ status: T.nilable(T.any(String, Symbol)),
+ event_id: T.nilable(String)
+ ).void
+ end
+ def create_function_call_output(
+ call_id:,
+ output:,
+ id: nil,
+ status: nil,
+ event_id: nil
+ )
+ end
+
+ sig do
+ params(
+ approval_request_id: String,
+ approve: T::Boolean,
+ reason: T.nilable(String),
+ id: T.nilable(String),
+ event_id: T.nilable(String)
+ ).void
+ end
+ def respond_to_mcp_approval(
+ approval_request_id:,
+ approve:,
+ reason: nil,
+ id: nil,
+ event_id: nil
+ )
+ end
+ end
+
+ class OutputAudioBuffer < Base
+ sig { params(event_id: T.nilable(String)).void }
+ def clear(event_id: nil)
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/rbi/openai/realtime/sideband_connection.rbi b/rbi/openai/realtime/sideband_connection.rbi
new file mode 100644
index 000000000..2e0e0c304
--- /dev/null
+++ b/rbi/openai/realtime/sideband_connection.rbi
@@ -0,0 +1,24 @@
+# typed: strong
+
+module OpenAI
+ module Models
+ module Realtime
+ class SidebandConnection < OpenAI::Realtime::Connection
+ Elem =
+ type_member { { fixed: OpenAI::Realtime::Connection::ServerEvent } }
+
+ sig do
+ returns(OpenAI::Realtime::ConnectionResources::OutputAudioBuffer)
+ end
+ attr_reader :output_audio_buffer
+
+ # @api private
+ sig do
+ params(socket: T.untyped, url: URI::Generic).returns(T.attached_class)
+ end
+ def self.new(socket:, url:)
+ end
+ end
+ end
+ end
+end
diff --git a/rbi/openai/realtime/transcription_connection.rbi b/rbi/openai/realtime/transcription_connection.rbi
new file mode 100644
index 000000000..ed6feb5a2
--- /dev/null
+++ b/rbi/openai/realtime/transcription_connection.rbi
@@ -0,0 +1,71 @@
+# typed: strong
+
+module OpenAI
+ module Models
+ module Realtime
+ class TranscriptionConnection < OpenAI::Realtime::BaseConnection
+ include Enumerable
+
+ ServerEvent =
+ T.type_alias do
+ T.any(
+ OpenAI::Realtime::RealtimeServerEvent::Variants,
+ OpenAI::Realtime::UnknownServerEvent
+ )
+ end
+
+ ClientEvent =
+ T.type_alias do
+ T.any(
+ OpenAI::Realtime::RealtimeClientEvent::Variants,
+ OpenAI::Internal::AnyHash
+ )
+ end
+
+ Elem = type_member { { fixed: ServerEvent } }
+
+ sig do
+ returns(OpenAI::Realtime::ConnectionResources::TranscriptionSession)
+ end
+ attr_reader :session
+
+ sig do
+ returns(OpenAI::Realtime::ConnectionResources::InputAudioBuffer)
+ end
+ attr_reader :input_audio_buffer
+
+ # @api private
+ sig do
+ params(socket: T.untyped, url: URI::Generic).returns(T.attached_class)
+ end
+ def self.new(socket:, url:)
+ end
+
+ sig do
+ params(
+ block: T.nilable(T.proc.params(event: ServerEvent).void)
+ ).returns(
+ T.any(
+ OpenAI::Realtime::TranscriptionConnection,
+ T::Enumerator[ServerEvent]
+ )
+ )
+ end
+ def each(&block)
+ end
+
+ sig { returns(T.nilable(ServerEvent)) }
+ def receive
+ end
+
+ sig { params(data: String).returns(ServerEvent) }
+ def parse_event(data)
+ end
+
+ sig { params(event: ClientEvent).void }
+ def send_event(event)
+ end
+ end
+ end
+ end
+end
diff --git a/rbi/openai/realtime/translation_connection.rbi b/rbi/openai/realtime/translation_connection.rbi
new file mode 100644
index 000000000..77ef0e0c2
--- /dev/null
+++ b/rbi/openai/realtime/translation_connection.rbi
@@ -0,0 +1,73 @@
+# typed: strong
+
+module OpenAI
+ module Models
+ module Realtime
+ class TranslationConnection < OpenAI::Realtime::BaseConnection
+ include Enumerable
+
+ ServerEvent =
+ T.type_alias do
+ T.any(
+ OpenAI::Realtime::RealtimeTranslationServerEvent::Variants,
+ OpenAI::Realtime::UnknownServerEvent
+ )
+ end
+
+ ClientEvent =
+ T.type_alias do
+ T.any(
+ OpenAI::Realtime::RealtimeTranslationClientEvent::Variants,
+ OpenAI::Internal::AnyHash
+ )
+ end
+
+ Elem = type_member { { fixed: ServerEvent } }
+
+ sig do
+ returns(OpenAI::Realtime::ConnectionResources::TranslationSession)
+ end
+ attr_reader :session
+
+ sig do
+ returns(
+ OpenAI::Realtime::ConnectionResources::TranslationInputAudioBuffer
+ )
+ end
+ attr_reader :input_audio_buffer
+
+ # @api private
+ sig do
+ params(socket: T.untyped, url: URI::Generic).returns(T.attached_class)
+ end
+ def self.new(socket:, url:)
+ end
+
+ sig do
+ params(
+ block: T.nilable(T.proc.params(event: ServerEvent).void)
+ ).returns(
+ T.any(
+ OpenAI::Realtime::TranslationConnection,
+ T::Enumerator[ServerEvent]
+ )
+ )
+ end
+ def each(&block)
+ end
+
+ sig { returns(T.nilable(ServerEvent)) }
+ def receive
+ end
+
+ sig { params(data: String).returns(ServerEvent) }
+ def parse_event(data)
+ end
+
+ sig { params(event: ClientEvent).void }
+ def send_event(event)
+ end
+ end
+ end
+ end
+end
diff --git a/rbi/openai/realtime/transports/async_websocket.rbi b/rbi/openai/realtime/transports/async_websocket.rbi
new file mode 100644
index 000000000..6869175c2
--- /dev/null
+++ b/rbi/openai/realtime/transports/async_websocket.rbi
@@ -0,0 +1,50 @@
+# typed: strong
+
+module OpenAI
+ module Models
+ module Realtime
+ module Transports
+ class AsyncWebSocket
+ class Socket
+ # @api private
+ sig do
+ params(connection: T.untyped, url: URI::Generic).returns(
+ T.attached_class
+ )
+ end
+ def self.new(connection, url:)
+ end
+
+ sig { returns(T.untyped) }
+ def read
+ end
+
+ sig { params(message: String).void }
+ def write(message)
+ end
+
+ sig { params(code: Integer, reason: String).void }
+ def close(code: 1000, reason: "")
+ end
+
+ sig { returns(T::Boolean) }
+ def closed?
+ end
+ end
+
+ sig do
+ params(
+ url: URI::Generic,
+ headers: T::Hash[String, String],
+ timeout: T.nilable(Float),
+ endpoint_options: T.untyped,
+ block: T.proc.params(socket: Socket).returns(T.untyped)
+ ).returns(T.untyped)
+ end
+ def open(url:, headers:, timeout:, **endpoint_options, &block)
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/rbi/openai/realtime/unknown_server_event.rbi b/rbi/openai/realtime/unknown_server_event.rbi
new file mode 100644
index 000000000..3147d4ae9
--- /dev/null
+++ b/rbi/openai/realtime/unknown_server_event.rbi
@@ -0,0 +1,25 @@
+# typed: strong
+
+module OpenAI
+ module Models
+ module Realtime
+ class UnknownServerEvent
+ sig { returns(Symbol) }
+ attr_reader :type
+
+ sig { returns(T::Hash[Symbol, T.untyped]) }
+ attr_reader :data
+
+ sig do
+ params(data: T::Hash[Symbol, T.untyped]).returns(T.attached_class)
+ end
+ def self.new(data:)
+ end
+
+ sig { returns(T::Hash[Symbol, T.untyped]) }
+ def to_h
+ end
+ end
+ end
+ end
+end
diff --git a/rbi/openai/resources/realtime.rbi b/rbi/openai/resources/realtime.rbi
index 93c144442..9927621b4 100644
--- a/rbi/openai/resources/realtime.rbi
+++ b/rbi/openai/resources/realtime.rbi
@@ -9,6 +9,44 @@ module OpenAI
sig { returns(OpenAI::Resources::Realtime::Calls) }
attr_reader :calls
+ sig { returns(OpenAI::Resources::Realtime::Translations) }
+ attr_reader :translations
+
+ sig do
+ params(
+ model: String,
+ request_options: T.nilable(OpenAI::RequestOptions::OrHash),
+ transport: T.untyped,
+ transport_options: T::Hash[Symbol, T.untyped],
+ block: T.proc.params(connection: OpenAI::Realtime::Connection).returns(T.untyped)
+ ).returns(T.untyped)
+ end
+ def connect(model:, request_options: nil, transport: nil, transport_options: {}, &block)
+ end
+
+ sig do
+ params(
+ call_id: String,
+ request_options: T.nilable(OpenAI::RequestOptions::OrHash),
+ transport: T.untyped,
+ transport_options: T::Hash[Symbol, T.untyped],
+ block: T.proc.params(connection: OpenAI::Realtime::SidebandConnection).returns(T.untyped)
+ ).returns(T.untyped)
+ end
+ def connect_to_call(call_id:, request_options: nil, transport: nil, transport_options: {}, &block)
+ end
+
+ sig do
+ params(
+ request_options: T.nilable(OpenAI::RequestOptions::OrHash),
+ transport: T.untyped,
+ transport_options: T::Hash[Symbol, T.untyped],
+ block: T.proc.params(connection: OpenAI::Realtime::TranscriptionConnection).returns(T.untyped)
+ ).returns(T.untyped)
+ end
+ def connect_transcription(request_options: nil, transport: nil, transport_options: {}, &block)
+ end
+
# @api private
sig { params(client: OpenAI::Client).returns(T.attached_class) }
def self.new(client:)
diff --git a/rbi/openai/resources/realtime/calls.rbi b/rbi/openai/resources/realtime/calls.rbi
index 8133bcf03..0c2539d9e 100644
--- a/rbi/openai/resources/realtime/calls.rbi
+++ b/rbi/openai/resources/realtime/calls.rbi
@@ -4,6 +4,17 @@ module OpenAI
module Resources
class Realtime
class Calls
+ sig do
+ params(
+ sdp: String,
+ session:
+ T.nilable(OpenAI::Realtime::RealtimeSessionCreateRequest::OrHash),
+ request_options: OpenAI::RequestOptions::OrHash
+ ).returns(OpenAI::Realtime::CallCreateResponse)
+ end
+ def create(sdp:, session: nil, request_options: {})
+ end
+
# Accept an incoming SIP call and configure the realtime session that will handle
# it.
sig do
diff --git a/rbi/openai/resources/realtime/translations.rbi b/rbi/openai/resources/realtime/translations.rbi
new file mode 100644
index 000000000..016650be4
--- /dev/null
+++ b/rbi/openai/resources/realtime/translations.rbi
@@ -0,0 +1,44 @@
+# typed: strong
+
+module OpenAI
+ module Resources
+ class Realtime
+ class Translations
+ sig do
+ returns(OpenAI::Resources::Realtime::Translations::ClientSecrets)
+ end
+ attr_reader :client_secrets
+
+ sig { returns(OpenAI::Resources::Realtime::Translations::Calls) }
+ attr_reader :calls
+
+ sig do
+ params(
+ model: String,
+ request_options: T.nilable(OpenAI::RequestOptions::OrHash),
+ transport: T.untyped,
+ transport_options: T::Hash[Symbol, T.untyped],
+ block:
+ T
+ .proc
+ .params(connection: OpenAI::Realtime::TranslationConnection)
+ .returns(T.untyped)
+ ).returns(T.untyped)
+ end
+ def connect(
+ model:,
+ request_options: nil,
+ transport: nil,
+ transport_options: {},
+ &block
+ )
+ end
+
+ # @api private
+ sig { params(client: OpenAI::Client).returns(T.attached_class) }
+ def self.new(client:)
+ end
+ end
+ end
+ end
+end
diff --git a/rbi/openai/resources/realtime/translations/calls.rbi b/rbi/openai/resources/realtime/translations/calls.rbi
new file mode 100644
index 000000000..c0fb9d2c3
--- /dev/null
+++ b/rbi/openai/resources/realtime/translations/calls.rbi
@@ -0,0 +1,25 @@
+# typed: strong
+
+module OpenAI
+ module Resources
+ class Realtime
+ class Translations
+ class Calls
+ sig do
+ params(
+ sdp: String,
+ request_options: T.nilable(OpenAI::RequestOptions::OrHash)
+ ).returns(OpenAI::Realtime::CallCreateResponse)
+ end
+ def create(sdp:, request_options: nil)
+ end
+
+ # @api private
+ sig { params(client: OpenAI::Client).returns(T.attached_class) }
+ def self.new(client:)
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/rbi/openai/resources/realtime/translations/client_secrets.rbi b/rbi/openai/resources/realtime/translations/client_secrets.rbi
new file mode 100644
index 000000000..3b3d0d2e3
--- /dev/null
+++ b/rbi/openai/resources/realtime/translations/client_secrets.rbi
@@ -0,0 +1,32 @@
+# typed: strong
+
+module OpenAI
+ module Resources
+ class Realtime
+ class Translations
+ class ClientSecrets
+ sig do
+ params(
+ session:
+ OpenAI::Realtime::RealtimeTranslationSessionCreateRequest::OrHash,
+ expires_after:
+ T.nilable(
+ OpenAI::Realtime::RealtimeTranslationClientSecretCreateRequest::ExpiresAfter::OrHash
+ ),
+ request_options: T.nilable(OpenAI::RequestOptions::OrHash)
+ ).returns(
+ OpenAI::Realtime::RealtimeTranslationClientSecretCreateResponse
+ )
+ end
+ def create(session:, expires_after: nil, request_options: nil)
+ end
+
+ # @api private
+ sig { params(client: OpenAI::Client).returns(T.attached_class) }
+ def self.new(client:)
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/realtime.md b/realtime.md
new file mode 100644
index 000000000..124c90a76
--- /dev/null
+++ b/realtime.md
@@ -0,0 +1,444 @@
+# Realtime API
+
+The Ruby SDK supports the server-side parts of the Realtime API:
+
+- typed WebSocket sessions for text, audio, transcription, function calls, and remote MCP;
+- WebRTC SDP negotiation from a trusted Ruby backend;
+- sideband WebSocket control for active WebRTC and SIP calls;
+- SIP accept, reject, refer, and hangup controls; and
+- dedicated WebSocket, client-secret, and WebRTC translation endpoints.
+
+Ruby does not provide a broadly deployed, standard WebRTC media stack. The SDK
+therefore handles the backend SDP exchange and call controls while browsers or a
+specialized media service own `RTCPeerConnection`, microphone capture, codecs,
+jitter buffering, echo cancellation, and playback.
+
+## Installation
+
+HTTP and WebRTC session-negotiation methods use the base SDK. WebSockets use an
+optional adapter so applications that only use HTTP do not acquire an event-loop
+dependency:
+
+```ruby
+gem "openai"
+gem "async-websocket"
+```
+
+The adapter is fiber-scheduler-aware and works both inside and outside an
+existing Async reactor. You can instead inject an object implementing the small
+transport contract described below.
+
+## Start a WebSocket session
+
+`connect` is block-scoped. This is the safest lifecycle in Rails jobs,
+Rack servers, CLI programs, and long-running workers because normal returns and
+exceptions both close the socket.
+
+```ruby
+client = OpenAI::Client.new
+
+client.realtime.connect(model: "gpt-realtime-2.1") do |connection|
+ connection.session.update(
+ type: :realtime,
+ output_modalities: [:text],
+ instructions: "Be concise."
+ )
+
+ connection.conversation.items.create(
+ type: :message,
+ role: :user,
+ content: [{type: :input_text, text: "Hello"}]
+ )
+ connection.response.create
+
+ connection.each do |event|
+ case event
+ when OpenAI::Realtime::ResponseTextDeltaEvent
+ print(event.delta)
+ when OpenAI::Realtime::ResponseDoneEvent
+ raise "Response ended with #{event.response.status}" unless event.response.status == :completed
+ break
+ when OpenAI::Realtime::RealtimeErrorEvent
+ raise event.error.message
+ end
+ end
+end
+```
+
+To exercise that flow against the live service with visible lifecycle output,
+set `OPENAI_API_KEY` and run:
+
+```console
+$ bundle exec ruby examples/realtime/websocket_text.rb
+```
+
+Set `OPENAI_REALTIME_MODEL`, `OPENAI_REALTIME_PROMPT`, or
+`OPENAI_REALTIME_TIMEOUT` to override the example defaults.
+
+`connect` requires a block. The yielded connection is valid only for the lifetime
+of that block and is always closed when the block exits, including exceptional exits.
+
+The connection exposes `receive`, `receive_raw`, `parse_event`, `send_event`,
+`send_raw`, `each`, `close`, and `closed?`. The resource
+helpers construct and validate the common client events:
+
+- `session.update`;
+- `response.create` and `response.cancel`;
+- `input_audio_buffer.append`, `append_bytes`, `commit`, and `clear`;
+- sideband-only `output_audio_buffer.clear`; and
+- conversation item create, retrieve, truncate, delete, function-call output,
+ and MCP approval response.
+
+`append` accepts API-ready Base64. `append_bytes` accepts binary audio and uses
+strict Base64 encoding. Sending is synchronous, and iteration reads one message
+at a time, so the caller naturally applies backpressure instead of filling an
+unbounded SDK queue. A connection supports one reader fiber and one writer
+fiber at the same time, which is useful for continuous audio. Do not run
+multiple readers or multiple writers against the same connection.
+The hands-free WebSocket example uses a bounded outbound queue so microphone
+audio and interruption-driven truncation events share exactly one writer fiber.
+
+The yielded class reflects the protocol's capabilities:
+
+- `connect(model:)` yields `OpenAI::Realtime::Connection` for ordinary sessions;
+- `connect_transcription` yields
+ `OpenAI::Realtime::TranscriptionConnection`, exposing only session and input
+ audio buffer operations;
+- `connect_to_call(call_id:)` yields `OpenAI::Realtime::SidebandConnection`, adding
+ `output_audio_buffer`; and
+- `translations.connect` yields `OpenAI::Realtime::TranslationConnection`,
+ which intentionally has no conversation or response lifecycle.
+
+The resource helpers accept the resource fields directly and add the wire
+envelope internally: use `session.update(type: ...)`,
+`response.create(instructions: ...)`, and `conversation.items.create(type: ...)`.
+Use `send_event` only when an application deliberately needs an exact protocol
+event hash.
+
+## Stream transcription
+
+Realtime transcription is a distinct session mode. Configure a transcription
+model, append raw mono 24 kHz PCM16 input, and commit the buffer:
+
+```ruby
+client.realtime.connect_transcription do |connection|
+ connection.session.update(
+ audio: {
+ input: {
+ format: {type: :"audio/pcm", rate: 24_000},
+ transcription: {model: "gpt-live-transcribe"},
+ turn_detection: nil
+ }
+ }
+ )
+
+ # Wait for SessionUpdatedEvent before streaming input.
+ connection.input_audio_buffer.append_bytes(pcm_chunk)
+ connection.input_audio_buffer.commit
+
+ connection.each do |event|
+ case event
+ when OpenAI::Realtime::ConversationItemInputAudioTranscriptionDeltaEvent
+ print(event.delta)
+ when OpenAI::Realtime::ConversationItemInputAudioTranscriptionCompletedEvent
+ puts event.transcript
+ end
+ end
+end
+```
+
+Completion events for different turns are not guaranteed to arrive in input
+order, so correlate deltas and completions with `item_id`. The dedicated
+connection uses the service's `intent=transcription` handshake and adds the
+session's `type: :transcription` discriminator internally; the transcription
+model belongs under `audio.input.transcription.model`. Run the complete live sample with
+`REALTIME_INPUT_PCM=input.pcm bundle exec ruby
+examples/realtime/websocket_transcription.rb`.
+
+## Create a WebRTC call
+
+WebRTC media is intentionally not a Ruby SDK responsibility. In a browser or
+mobile voice application, the client owns `RTCPeerConnection`, microphone
+permissions, audio playback, jitter buffering, and acoustic echo cancellation.
+Ruby owns the trusted-server control plane: API credentials, session
+configuration, SDP negotiation, sideband tools and policy, and call lifecycle.
+The SDK therefore supports WebRTC session negotiation and server controls; it
+does not attempt to expose a Ruby WebRTC peer API.
+
+Use `calls.create` in the trusted application server that receives a browser's
+SDP offer:
+
+```ruby
+call = client.realtime.calls.create(
+ sdp: browser_offer,
+ session: {
+ type: :realtime,
+ model: "gpt-realtime-2.1",
+ audio: {output: {voice: :marin}}
+ },
+ request_options: {
+ extra_headers: {"OpenAI-Safety-Identifier" => hashed_user_id}
+ }
+)
+
+render plain: call.sdp, content_type: "application/sdp"
+```
+
+The result preserves `sdp`, normalized response `headers`, and `call_id` parsed
+from the optional `Location` header. A missing or malformed `Location` leaves
+`call_id` as `nil` without discarding an otherwise valid SDP answer. Keep the
+call ID if the application will open a server-side sideband connection:
+
+```ruby
+client.realtime.connect_to_call(call_id: call.call_id) do |connection|
+ connection.session.update(
+ type: :realtime,
+ instructions: "Apply private server policy."
+ )
+ connection.each { |event| audit(event) }
+end
+```
+
+The SDK sends raw `application/sdp` when only an offer is provided; use that
+form with a client configured with a previously minted ephemeral session token.
+With a standard server API key, provide `session:` and the SDK sends multipart
+form data with correctly typed `sdp` and `session` parts. It never exposes a
+standard API key to browser code.
+
+## SIP calls
+
+Verify incoming webhooks before using their call ID, accept or reject the call,
+then use the same sideband connection API:
+
+```ruby
+event = client.webhooks.unwrap(raw_body, request_headers)
+return unless event.is_a?(OpenAI::Webhooks::RealtimeCallIncomingWebhookEvent)
+
+call_id = event.data.call_id
+client.realtime.calls.accept(
+ call_id,
+ type: :realtime,
+ model: "gpt-realtime-2.1",
+ instructions: "You are answering a phone call."
+)
+
+client.realtime.connect_to_call(call_id: call_id) do |connection|
+ connection.each { |realtime_event| handle(realtime_event) }
+end
+```
+
+`calls.reject`, `calls.refer`, and `calls.hangup` provide the remaining SIP and
+call-lifecycle controls.
+
+## Translation
+
+Translation has dedicated endpoints and distinct typed event unions:
+
+```ruby
+require "async"
+
+client.realtime.translations.connect(model: "gpt-realtime-translate") do |connection|
+ connection.session.update(audio: {output: {language: "es"}})
+
+ reader = Async do
+ connection.each do |event|
+ case event
+ when OpenAI::Realtime::RealtimeTranslationOutputTranscriptDeltaEvent
+ print(event.delta)
+ when OpenAI::Realtime::RealtimeTranslationOutputAudioDeltaEvent
+ play(Base64.strict_decode64(event.delta))
+ when OpenAI::Realtime::RealtimeTranslationSessionClosedEvent
+ break
+ end
+ end
+ end
+
+ while (chunk = pcm16_source.read(9_600))
+ connection.input_audio_buffer.append_bytes(chunk)
+ end
+ connection.session.close
+ reader.wait
+end
+```
+
+Call `session.close` after the last audio chunk and continue reading until the
+server sends `session.closed`; closing the socket immediately can discard output
+that is still draining. For browser WebRTC translation, mint an ephemeral secret
+with `translations.client_secrets.create`, then post the browser offer with
+`translations.calls.create`. That convenience method delegates to the shared
+`/v1/realtime/calls` endpoint; translation is selected by the ephemeral secret,
+not by a separate calls route.
+
+## Function calls and MCP approvals
+
+Function call output and MCP approval responses are ordinary conversation items.
+The helpers supply the required item shape and generate an MCP response ID when
+one is not supplied:
+
+```ruby
+connection.conversation.items.create_function_call_output(
+ call_id: function_call.call_id,
+ output: JSON.generate(result)
+)
+
+connection.conversation.items.respond_to_mcp_approval(
+ approval_request_id: approval_item.id,
+ approve: policy.allows?(approval_item),
+ reason: "Evaluated by application policy"
+)
+```
+
+Remote MCP servers are configured through the typed `tools` field in
+`session.update` or `response.create`; all MCP progress, approval, and completion
+events decode through `RealtimeServerEvent`. Before a dependent prompt, wait for
+both `McpListToolsCompleted` and the `ConversationItemDone` event whose
+`RealtimeMcpListTools` item contains the imported tool names. If you then update
+`tool_choice`, wait for `SessionUpdatedEvent` before creating the response.
+
+The first completed `ResponseDoneEvent` can finish the MCP argument-generation
+phase while approval and tool execution are still pending. Keep reading, answer
+the `RealtimeMcpApprovalRequest`, wait for `ResponseMcpCallCompleted`, and create
+a follow-up response (usually with `tool_choice: :none`) to turn the tool output
+into a final assistant answer. Treat only that follow-up response's completed
+`ResponseDoneEvent` as success; clean EOF during discovery, approval, or tool
+execution is still an incomplete workflow. The approval helper generates item
+IDs within the service's 32-character limit.
+
+## Lifecycle and failures
+
+The SDK does not automatically reconnect or replay events. Realtime sessions are
+stateful, and replaying audio, item creation, tool output, or approval decisions
+can duplicate side effects. Treat an unexpected close as an application-level
+decision: terminate, or reconnect and deliberately rebuild the state that is
+safe for that use case.
+
+Runnable smoke tests require their protocol-specific terminal event rather than
+treating a clean WebSocket EOF as success: completed `response.done` for text,
+audio, and MCP; transcription completion for transcription; `session.closed`
+for translation; and the requested `OPENAI_REALTIME_STOP_AFTER` checkpoint for
+bounded sideband, SIP, and conversation runs. Cleanup must also preserve an
+active upload or processing error when a graceful close fails, while surfacing
+the close failure after an otherwise successful operation.
+
+Terminal status alone is insufficient when a workflow promises an artifact:
+the text smoke requires a non-empty text delta, while the raw-audio and
+translation smokes require at least one decoded audio byte before their terminal
+events. The file-audio smoke disables VAD and waits for the acknowledged session
+update before its manual commit. Translation reader failures cancel an in-flight
+upload rather than waiting for the input file to drain; an already-buffered
+reader failure prevents the uploader from starting at all. Microphone shutdown
+is latched before capture starts, so an immediately closed connection cannot
+start an orphaned ffmpeg process after cleanup has begun.
+
+The three standard Realtime connection modes use distinct method names rather
+than a keyword-discriminated overload: `connect(model:)`, `connect_to_call`, and
+`connect_transcription`. This keeps block parameter types precise in both RBS
+and RBI; Sorbet does not support overloads that discriminate on keyword
+arguments.
+
+Raw WebRTC SDP responses remain lazily consumed. Request observability follows
+that body lifecycle: completion is recorded only after the SDP body is fully
+drained, and a body read failure records `request failed` without a contradictory
+completion event.
+
+- Invalid JSON or a payload that cannot match the selected event union raises
+ `OpenAI::Errors::RealtimeProtocolError`, with `data` and `cause`.
+- Handshake and socket I/O failures from the default adapter raise
+ `OpenAI::Errors::RealtimeConnectionError`, with `url` and `cause`.
+- API `error` events remain typed `RealtimeErrorEvent` values in the normal event
+ stream so applications can follow the API's recoverability guidance.
+- Valid events introduced after the installed SDK version remain observable as
+ immutable `UnknownServerEvent` values instead of terminating the session.
+- Exceptions raised by the application block propagate unchanged; cleanup does
+ not replace them with a close error.
+
+Use `request_options` for handshake headers, query parameters, and timeout:
+
+```ruby
+client.realtime.connect(
+ model: "gpt-realtime-2.1",
+ request_options: {
+ timeout: 30,
+ extra_headers: {"OpenAI-Safety-Identifier" => hashed_user_id},
+ extra_query: {trace: "enabled"}
+ }
+) { |connection| run_session(connection) }
+```
+
+`websocket_base_url:` configures a separate gateway for WebSockets. Azure
+provider clients derive the WebSocket URL from the configured Azure v1 endpoint
+and resolve provider-owned API-key or bearer authentication immediately before
+the handshake. Providers without a Realtime WebSocket surface fail before a
+credential is sent.
+
+## Custom transport
+
+Pass `transport:` to `connect` to replace the adapter. The object implements:
+
+```ruby
+transport.open(url:, headers:, timeout:, **transport_options) do |socket|
+ # socket.read # String-like message or nil
+ # socket.write(json_string) # synchronous write/backpressure
+ # socket.close(code:, reason:)
+ # socket.closed?
+end
+```
+
+`timeout` is `Float?` at the transport boundary because `Client.new(timeout:
+nil)` and per-request `timeout: nil` deliberately disable the handshake timeout.
+
+The SDK owns URL construction, authentication, typed JSON encoding/decoding, and
+block cleanup. The transport owns the WebSocket handshake, frame I/O, TLS,
+proxies, and transport-specific failures. Internal calls use direct methods; the
+contract does not require subclassing or reflection.
+
+## Example and developer-site coverage
+
+Repository examples live under `examples/realtime/`; their
+[runbook](examples/realtime/README.md) includes prerequisites, commands, bounded
+smoke-test controls, and observable pass criteria. They cover a hands-free
+full-duplex voice conversation, text, raw PCM audio, streaming transcription,
+WebRTC SDP exchange, sideband, SIP, MCP approvals, and translation. The
+developer website should expose the same progression so users encounter the
+simplest successful path before protocol details:
+
+| Guide surface | Ruby sample required |
+| --- | --- |
+| Realtime overview | Minimal text WebSocket with typed deltas and block cleanup |
+| WebSocket | Text, PCM16 input/output, manual `receive`, Enumerable iteration, error handling |
+| Transcription | `connect_transcription`, PCM16 streaming, delta/completed events, and `item_id` correlation |
+| WebRTC | Complete browser peer plus Ruby endpoint; Rails and Sinatra variants using `calls.create` |
+| Server controls | Preserve `call.call_id`, open sideband, update session, hang up |
+| SIP | Verify webhook, accept/reject, sideband, refer, hang up, idempotent webhook handling |
+| Conversations | Text item, audio commit, cancel/interruption, truncate played audio |
+| Function calling | Detect function call, execute locally, send typed output item |
+| Remote MCP | Configure server, inspect approval request, approve/reject, observe completion |
+| Translation | Client secret, browser WebRTC call, server WebSocket PCM loop, graceful close |
+| Authentication | Standard key on server, ephemeral browser secret, safety identifier |
+| Azure | Deployment/model targeting and provider-owned handshake authentication |
+| Operations | Timeout, backpressure, logging event IDs, abnormal close, deliberate reconnect policy |
+
+The WebRTC browser half remains JavaScript because Ruby runs on the trusted
+server; Ruby snippets should own secrets, session configuration, SDP proxying,
+webhook verification, tools, and business logic.
+
+The local voice example is also exercised against the installed FFmpeg tools,
+not only Ruby fakes. In particular, raw PCM playback declares mono input with
+ffplay's `-ch_layout mono`; `-ac 1` is an ffmpeg transcoding option and current
+ffplay releases reject it. Playback also disables ffplay's terminal statistics:
+without `-nostats`, ffplay emits an ANSI clear-line sequence that can erase the
+first transcript delta from the terminal. Playback-command compatibility and a
+headless PCM process smoke test guard both boundaries.
+
+WebSocket barge-in requires client playback state in addition to server VAD.
+On `input_audio_buffer.speech_started`, the voice example stops ffplay, records
+the elapsed audio offset, sends `conversation.item.truncate`, and ignores queued
+audio and transcript deltas for the cancelled response. A cancelled
+`response.done` is an expected interruption outcome rather than an exception.
+
+The FFmpeg WebSocket loop cannot provide acoustic echo cancellation and must
+not be positioned as the default laptop conversation sample. With speakers,
+model output can re-enter the microphone and repeatedly trigger VAD, producing
+one-word responses. The primary interactive voice example uses browser WebRTC;
+the Ruby WebSocket loop remains useful for headless server audio, telephony
+bridges, deterministic PCM tests, and protocol debugging.
diff --git a/sig/openai/client.rbs b/sig/openai/client.rbs
index a65e15abc..87903612e 100644
--- a/sig/openai/client.rbs
+++ b/sig/openai/client.rbs
@@ -20,6 +20,8 @@ module OpenAI
attr_reader webhook_secret: String?
+ attr_reader websocket_base_url: URI::Generic?
+
attr_reader completions: OpenAI::Resources::Completions
attr_reader chat: OpenAI::Resources::Chat
@@ -76,6 +78,12 @@ module OpenAI
private def admin_api_key_auth: -> ::Hash[String, String]
+ def realtime_connection_request: (
+ path: String,
+ query: ::Hash[String, String],
+ ?options: OpenAI::request_opts?
+ ) -> OpenAI::Internal::Transport::BaseClient::request_input
+
private def prepare_request: (
OpenAI::Internal::Transport::BaseClient::request_input request,
redirect_count: Integer,
@@ -92,6 +100,7 @@ module OpenAI
?provider: OpenAI::Provider?,
?base_url: String?,
?default_headers: ::Hash[String, String?]?,
+ ?websocket_base_url: String?,
?max_retries: Integer,
?timeout: Float?,
?initial_retry_delay: Float,
diff --git a/sig/openai/errors.rbs b/sig/openai/errors.rbs
index aa1428662..e38e6fe62 100644
--- a/sig/openai/errors.rbs
+++ b/sig/openai/errors.rbs
@@ -1,7 +1,7 @@
module OpenAI
module Errors
class Error < StandardError
- attr_accessor cause: StandardError?
+ attr_accessor cause: Exception?
end
class ConversionError < OpenAI::Errors::Error
@@ -16,6 +16,28 @@ module OpenAI
) -> void
end
+ class RealtimeConnectionError < OpenAI::Errors::Error
+ attr_reader url: URI::Generic
+ def cause: -> Exception?
+
+ def initialize: (
+ url: URI::Generic,
+ ?message: String?,
+ ?cause: Exception?
+ ) -> void
+ end
+
+ class RealtimeProtocolError < OpenAI::Errors::Error
+ attr_reader data: String
+ def cause: -> StandardError?
+
+ def initialize: (
+ data: String,
+ ?message: String?,
+ ?cause: StandardError?
+ ) -> void
+ end
+
class APIError < OpenAI::Errors::Error
attr_accessor url: URI::Generic
diff --git a/sig/openai/internal/logging.rbs b/sig/openai/internal/logging.rbs
index aa0b8755f..20f1ce0c2 100644
--- a/sig/openai/internal/logging.rbs
+++ b/sig/openai/internal/logging.rbs
@@ -38,6 +38,9 @@ module OpenAI
OpenAI::Internal::Type::BaseStream[top, Elem] stream,
response: OpenAI::HTTPClient::Response
) -> OpenAI::Internal::Type::BaseStream[top, Elem]
+ def observe_raw_response: (
+ OpenAI::HTTPClient::Response response
+ ) -> OpenAI::HTTPClient::Response
def request_failed: (StandardError error) -> void
def response_body: (
String body,
@@ -74,6 +77,21 @@ module OpenAI
private def iterator: -> Enumerable[Elem]
end
+ class ObservedEnumerable[Elem]
+ include Enumerable[Elem]
+
+ def initialize: (
+ enumerable: Enumerable[Elem],
+ context: OpenAI::Internal::Logging::Context,
+ response: OpenAI::HTTPClient::Response,
+ close: ^() -> void
+ ) -> void
+
+ def each: { (Elem) -> void } -> void
+ def close: -> void
+ private def iterator: -> Enumerable[Elem]
+ end
+
def self.normalize_level: (Symbol | String value) -> Symbol
def self.validate_logger!: (OpenAI::_Logger? logger) -> void
def self.default_logger: -> OpenAI::_Logger
diff --git a/sig/openai/internal/transport/base_client.rbs b/sig/openai/internal/transport/base_client.rbs
index 1b5517583..b717419ee 100644
--- a/sig/openai/internal/transport/base_client.rbs
+++ b/sig/openai/internal/transport/base_client.rbs
@@ -143,6 +143,10 @@ module OpenAI
-> OpenAI::Internal::Logging::Context
} -> OpenAI::HTTPClient::Response
+ def request_raw: (
+ OpenAI::Internal::Transport::BaseClient::request_components req
+ ) -> OpenAI::HTTPClient::Response
+
def request: (
Symbol method,
String | ::Array[String] path,
diff --git a/sig/openai/models/realtime/call_create_params.rbs b/sig/openai/models/realtime/call_create_params.rbs
new file mode 100644
index 000000000..0f9d91aaf
--- /dev/null
+++ b/sig/openai/models/realtime/call_create_params.rbs
@@ -0,0 +1,32 @@
+module OpenAI
+ module Models
+ module Realtime
+ type call_create_params =
+ {
+ sdp: String,
+ session: OpenAI::Realtime::RealtimeSessionCreateRequest?
+ }
+ & OpenAI::Internal::Type::request_parameters
+
+ class CallCreateParams < OpenAI::Internal::Type::BaseModel
+ extend OpenAI::Internal::Type::RequestParameters::Converter
+ include OpenAI::Internal::Type::RequestParameters
+
+ attr_accessor sdp: String
+ attr_accessor session: OpenAI::Realtime::RealtimeSessionCreateRequest?
+
+ def initialize: (
+ sdp: String,
+ ?session: OpenAI::Realtime::RealtimeSessionCreateRequest,
+ ?request_options: OpenAI::request_opts
+ ) -> void
+
+ def to_hash: -> {
+ sdp: String,
+ session: OpenAI::Realtime::RealtimeSessionCreateRequest?,
+ request_options: OpenAI::RequestOptions
+ }
+ end
+ end
+ end
+end
diff --git a/sig/openai/models/realtime/call_create_response.rbs b/sig/openai/models/realtime/call_create_response.rbs
new file mode 100644
index 000000000..ea05d00be
--- /dev/null
+++ b/sig/openai/models/realtime/call_create_response.rbs
@@ -0,0 +1,26 @@
+module OpenAI
+ module Models
+ module Realtime
+ type call_create_response =
+ { sdp: String, call_id: String?, headers: ::Hash[String, String] }
+
+ class CallCreateResponse < OpenAI::Internal::Type::BaseModel
+ attr_accessor sdp: String
+ attr_accessor call_id: String?
+ attr_accessor headers: ::Hash[String, String]
+
+ def initialize: (
+ sdp: String,
+ headers: ::Hash[String, String],
+ ?call_id: String?
+ ) -> void
+
+ def to_hash: -> {
+ sdp: String,
+ call_id: String?,
+ headers: ::Hash[String, String]
+ }
+ end
+ end
+ end
+end
diff --git a/sig/openai/realtime/base_connection.rbs b/sig/openai/realtime/base_connection.rbs
new file mode 100644
index 000000000..2e6d1660b
--- /dev/null
+++ b/sig/openai/realtime/base_connection.rbs
@@ -0,0 +1,20 @@
+module OpenAI
+ module Models
+ module Realtime
+ class BaseConnection
+ attr_reader url: URI::Generic
+
+ def initialize: (
+ socket: untyped,
+ url: URI::Generic,
+ server_event_type: untyped,
+ client_event_type: untyped
+ ) -> void
+ def receive_raw: -> String?
+ def send_raw: (String data) -> nil
+ def close: (?code: Integer, ?reason: String) -> nil
+ def closed?: -> bool
+ end
+ end
+ end
+end
diff --git a/sig/openai/realtime/connection.rbs b/sig/openai/realtime/connection.rbs
new file mode 100644
index 000000000..52caffcfa
--- /dev/null
+++ b/sig/openai/realtime/connection.rbs
@@ -0,0 +1,36 @@
+module OpenAI
+ module Models
+ module Realtime
+ type connection_server_event =
+ OpenAI::Models::Realtime::realtime_server_event
+ | OpenAI::Realtime::UnknownServerEvent
+
+ type connection_client_event =
+ OpenAI::Models::Realtime::realtime_client_event
+ | ::Hash[Symbol | String, untyped]
+
+ class Connection < OpenAI::Realtime::BaseConnection
+ include Enumerable[OpenAI::Models::Realtime::connection_server_event]
+
+ attr_reader session: OpenAI::Realtime::ConnectionResources::Session
+ attr_reader response: OpenAI::Realtime::ConnectionResources::Response
+ attr_reader input_audio_buffer: OpenAI::Realtime::ConnectionResources::InputAudioBuffer
+ attr_reader conversation: OpenAI::Realtime::ConnectionResources::Conversation
+
+ def initialize: (socket: untyped, url: URI::Generic) -> void
+ def each:
+ -> Enumerator[OpenAI::Models::Realtime::connection_server_event, self]
+ | {
+ (OpenAI::Models::Realtime::connection_server_event event) -> void
+ } -> self
+ def receive: -> OpenAI::Models::Realtime::connection_server_event?
+ def parse_event: (
+ String data
+ ) -> OpenAI::Models::Realtime::connection_server_event
+ def send_event: (
+ OpenAI::Models::Realtime::connection_client_event event
+ ) -> nil
+ end
+ end
+ end
+end
diff --git a/sig/openai/realtime/connection_manager.rbs b/sig/openai/realtime/connection_manager.rbs
new file mode 100644
index 000000000..1813a520e
--- /dev/null
+++ b/sig/openai/realtime/connection_manager.rbs
@@ -0,0 +1,21 @@
+module OpenAI
+ module Models
+ module Realtime
+ class ConnectionManager
+ def initialize: (
+ client: OpenAI::Client,
+ path: String,
+ query: ::Hash[String, String],
+ connection_class: Class,
+ transport: untyped,
+ request_options: OpenAI::request_opts?,
+ transport_options: ::Hash[Symbol, untyped]
+ ) -> void
+
+ def open: {
+ (OpenAI::Realtime::BaseConnection connection) -> top
+ } -> top
+ end
+ end
+ end
+end
diff --git a/sig/openai/realtime/connection_resources.rbs b/sig/openai/realtime/connection_resources.rbs
new file mode 100644
index 000000000..ab570b8d3
--- /dev/null
+++ b/sig/openai/realtime/connection_resources.rbs
@@ -0,0 +1,136 @@
+module OpenAI
+ module Models
+ module Realtime
+ module ConnectionResources
+ class Base
+ def initialize: (OpenAI::Realtime::BaseConnection connection) -> void
+ end
+
+ class Session < Base
+ def update: (
+ type: Symbol,
+ ?audio: OpenAI::Models::Realtime::realtime_audio_config
+ | OpenAI::Models::Realtime::realtime_transcription_session_audio,
+ ?include: ::Array[Symbol],
+ ?instructions: String,
+ ?max_output_tokens: Integer | Symbol,
+ ?model: String | Symbol,
+ ?output_modalities: ::Array[Symbol],
+ ?parallel_tool_calls: bool,
+ ?prompt: OpenAI::Models::Responses::response_prompt?,
+ ?reasoning: OpenAI::Models::Realtime::realtime_reasoning,
+ ?tool_choice: OpenAI::Models::Realtime::realtime_tool_choice_config,
+ ?tools: OpenAI::Models::Realtime::realtime_tools_config,
+ ?tracing: OpenAI::Models::Realtime::realtime_tracing_config?,
+ ?truncation: OpenAI::Models::Realtime::realtime_truncation,
+ ?event_id: String?
+ ) -> nil
+ end
+
+ class TranscriptionSession < Base
+ def update: (
+ ?audio: OpenAI::Models::Realtime::realtime_transcription_session_audio,
+ ?include: ::Array[OpenAI::Models::Realtime::RealtimeTranscriptionSessionCreateRequest::include_],
+ ?event_id: String?
+ ) -> nil
+ end
+
+ class TranslationSession < Session
+ def update: (
+ ?audio: OpenAI::Realtime::RealtimeTranslationSessionUpdateRequest::Audio
+ | ::Hash[Symbol | String, untyped],
+ ?event_id: String?
+ ) -> nil
+ def close: (?event_id: String?) -> nil
+ end
+
+ class Response < Base
+ def create: (
+ ?audio: OpenAI::Models::Realtime::realtime_response_create_audio_output,
+ ?conversation: String | Symbol,
+ ?input: ::Array[OpenAI::Models::Realtime::conversation_item],
+ ?instructions: String,
+ ?max_output_tokens: Integer | Symbol,
+ ?metadata: ::Hash[Symbol, String]?,
+ ?output_modalities: ::Array[Symbol],
+ ?parallel_tool_calls: bool,
+ ?prompt: OpenAI::Models::Responses::response_prompt?,
+ ?reasoning: OpenAI::Models::Realtime::realtime_reasoning,
+ ?tool_choice: OpenAI::Models::Realtime::RealtimeResponseCreateParams::tool_choice,
+ ?tools: ::Array[
+ OpenAI::Models::Realtime::realtime_function_tool
+ | OpenAI::Models::Realtime::realtime_response_create_mcp_tool
+ ],
+ ?event_id: String?
+ ) -> nil
+ def cancel: (?response_id: String?, ?event_id: String?) -> nil
+ end
+
+ class InputAudioBuffer < Base
+ def append: (audio: String, ?event_id: String?) -> nil
+ def append_bytes: (String bytes, ?event_id: String?) -> nil
+ def commit: (?event_id: String?) -> nil
+ def clear: (?event_id: String?) -> nil
+ end
+
+ class TranslationInputAudioBuffer < Base
+ def append: (audio: String, ?event_id: String?) -> nil
+ def append_bytes: (String bytes, ?event_id: String?) -> nil
+ end
+
+ class Conversation < Base
+ attr_reader items: OpenAI::Realtime::ConnectionResources::ConversationItems
+ end
+
+ class ConversationItems < Base
+ def create: (
+ type: Symbol,
+ ?arguments: String,
+ ?approval_request_id: String?,
+ ?approve: bool,
+ ?call_id: String,
+ ?content: ::Array[::Hash[Symbol | String, untyped]],
+ ?error: untyped,
+ ?id: String,
+ ?name: String,
+ ?object: Symbol,
+ ?output: String?,
+ ?reason: String?,
+ ?role: Symbol,
+ ?server_label: String,
+ ?status: Symbol,
+ ?tools: ::Array[OpenAI::Realtime::RealtimeMcpListTools::Tool | ::Hash[Symbol | String, untyped]],
+ ?event_id: String?,
+ ?previous_item_id: String?
+ ) -> nil
+ def delete: (item_id: String, ?event_id: String?) -> nil
+ def retrieve: (item_id: String, ?event_id: String?) -> nil
+ def truncate: (
+ item_id: String,
+ content_index: Integer,
+ audio_end_ms: Integer,
+ ?event_id: String?
+ ) -> nil
+ def create_function_call_output: (
+ call_id: String,
+ output: String,
+ ?id: String?,
+ ?status: (String | Symbol)?,
+ ?event_id: String?
+ ) -> nil
+ def respond_to_mcp_approval: (
+ approval_request_id: String,
+ approve: bool,
+ ?reason: String?,
+ ?id: String?,
+ ?event_id: String?
+ ) -> nil
+ end
+
+ class OutputAudioBuffer < Base
+ def clear: (?event_id: String?) -> nil
+ end
+ end
+ end
+ end
+end
diff --git a/sig/openai/realtime/sideband_connection.rbs b/sig/openai/realtime/sideband_connection.rbs
new file mode 100644
index 000000000..cfdf03744
--- /dev/null
+++ b/sig/openai/realtime/sideband_connection.rbs
@@ -0,0 +1,11 @@
+module OpenAI
+ module Models
+ module Realtime
+ class SidebandConnection < OpenAI::Realtime::Connection
+ attr_reader output_audio_buffer: OpenAI::Realtime::ConnectionResources::OutputAudioBuffer
+
+ def initialize: (socket: untyped, url: URI::Generic) -> void
+ end
+ end
+ end
+end
diff --git a/sig/openai/realtime/transcription_connection.rbs b/sig/openai/realtime/transcription_connection.rbs
new file mode 100644
index 000000000..ea8032c2f
--- /dev/null
+++ b/sig/openai/realtime/transcription_connection.rbs
@@ -0,0 +1,34 @@
+module OpenAI
+ module Models
+ module Realtime
+ type transcription_connection_server_event =
+ OpenAI::Models::Realtime::realtime_server_event
+ | OpenAI::Realtime::UnknownServerEvent
+
+ type transcription_connection_client_event =
+ OpenAI::Models::Realtime::realtime_client_event
+ | ::Hash[Symbol | String, untyped]
+
+ class TranscriptionConnection < OpenAI::Realtime::BaseConnection
+ include Enumerable[OpenAI::Models::Realtime::transcription_connection_server_event]
+
+ attr_reader session: OpenAI::Realtime::ConnectionResources::TranscriptionSession
+ attr_reader input_audio_buffer: OpenAI::Realtime::ConnectionResources::InputAudioBuffer
+
+ def initialize: (socket: untyped, url: URI::Generic) -> void
+ def each:
+ -> Enumerator[OpenAI::Models::Realtime::transcription_connection_server_event, self]
+ | {
+ (OpenAI::Models::Realtime::transcription_connection_server_event event) -> void
+ } -> self
+ def receive: -> OpenAI::Models::Realtime::transcription_connection_server_event?
+ def parse_event: (
+ String data
+ ) -> OpenAI::Models::Realtime::transcription_connection_server_event
+ def send_event: (
+ OpenAI::Models::Realtime::transcription_connection_client_event event
+ ) -> nil
+ end
+ end
+ end
+end
diff --git a/sig/openai/realtime/translation_connection.rbs b/sig/openai/realtime/translation_connection.rbs
new file mode 100644
index 000000000..d8042c341
--- /dev/null
+++ b/sig/openai/realtime/translation_connection.rbs
@@ -0,0 +1,36 @@
+module OpenAI
+ module Models
+ module Realtime
+ type translation_connection_server_event =
+ OpenAI::Models::Realtime::realtime_translation_server_event
+ | OpenAI::Realtime::UnknownServerEvent
+
+ type translation_connection_client_event =
+ OpenAI::Models::Realtime::realtime_translation_client_event
+ | ::Hash[Symbol | String, untyped]
+
+ class TranslationConnection < OpenAI::Realtime::BaseConnection
+ include Enumerable[OpenAI::Models::Realtime::translation_connection_server_event]
+
+ attr_reader session: OpenAI::Realtime::ConnectionResources::TranslationSession
+ attr_reader input_audio_buffer: OpenAI::Realtime::ConnectionResources::TranslationInputAudioBuffer
+
+ def initialize: (socket: untyped, url: URI::Generic) -> void
+ def each:
+ -> Enumerator[OpenAI::Models::Realtime::translation_connection_server_event, self]
+ | {
+ (
+ OpenAI::Models::Realtime::translation_connection_server_event event
+ ) -> void
+ } -> self
+ def receive: -> OpenAI::Models::Realtime::translation_connection_server_event?
+ def parse_event: (
+ String data
+ ) -> OpenAI::Models::Realtime::translation_connection_server_event
+ def send_event: (
+ OpenAI::Models::Realtime::translation_connection_client_event event
+ ) -> nil
+ end
+ end
+ end
+end
diff --git a/sig/openai/realtime/transports/async_websocket.rbs b/sig/openai/realtime/transports/async_websocket.rbs
new file mode 100644
index 000000000..d30db5995
--- /dev/null
+++ b/sig/openai/realtime/transports/async_websocket.rbs
@@ -0,0 +1,26 @@
+module OpenAI
+ module Models
+ module Realtime
+ module Transports
+ class AsyncWebSocket
+ class Socket
+ def initialize: (untyped connection, url: URI::Generic) -> void
+ def read: -> untyped
+ def write: (String message) -> void
+ def close: (?code: Integer, ?reason: String) -> void
+ def closed?: -> bool
+ end
+
+ def open: (
+ url: URI::Generic,
+ headers: ::Hash[String, String],
+ timeout: Float?,
+ **untyped endpoint_options
+ ) {
+ (Socket socket) -> top
+ } -> top
+ end
+ end
+ end
+ end
+end
diff --git a/sig/openai/realtime/unknown_server_event.rbs b/sig/openai/realtime/unknown_server_event.rbs
new file mode 100644
index 000000000..8fcb1d07d
--- /dev/null
+++ b/sig/openai/realtime/unknown_server_event.rbs
@@ -0,0 +1,13 @@
+module OpenAI
+ module Models
+ module Realtime
+ class UnknownServerEvent
+ attr_reader type: Symbol
+ attr_reader data: ::Hash[Symbol, untyped]
+
+ def initialize: (data: ::Hash[Symbol, untyped]) -> void
+ def to_h: -> ::Hash[Symbol, untyped]
+ end
+ end
+ end
+end
diff --git a/sig/openai/resources/realtime.rbs b/sig/openai/resources/realtime.rbs
index 999debec0..c7d74d7cc 100644
--- a/sig/openai/resources/realtime.rbs
+++ b/sig/openai/resources/realtime.rbs
@@ -5,6 +5,28 @@ module OpenAI
attr_reader calls: OpenAI::Resources::Realtime::Calls
+ attr_reader translations: OpenAI::Resources::Realtime::Translations
+
+ def connect: (
+ model: String,
+ ?request_options: OpenAI::request_opts?,
+ ?transport: untyped?,
+ ?transport_options: ::Hash[Symbol, untyped]
+ ) { (OpenAI::Realtime::Connection connection) -> top } -> top
+
+ def connect_to_call: (
+ call_id: String,
+ ?request_options: OpenAI::request_opts?,
+ ?transport: untyped?,
+ ?transport_options: ::Hash[Symbol, untyped]
+ ) { (OpenAI::Realtime::SidebandConnection connection) -> top } -> top
+
+ def connect_transcription: (
+ ?request_options: OpenAI::request_opts?,
+ ?transport: untyped?,
+ ?transport_options: ::Hash[Symbol, untyped]
+ ) { (OpenAI::Realtime::TranscriptionConnection connection) -> top } -> top
+
def initialize: (client: OpenAI::Client) -> void
end
end
diff --git a/sig/openai/resources/realtime/calls.rbs b/sig/openai/resources/realtime/calls.rbs
index 7f89f2417..6cad50fd9 100644
--- a/sig/openai/resources/realtime/calls.rbs
+++ b/sig/openai/resources/realtime/calls.rbs
@@ -2,6 +2,13 @@ module OpenAI
module Resources
class Realtime
class Calls
+ def create: (
+ sdp: String,
+ ?session: OpenAI::Realtime::RealtimeSessionCreateRequest
+ | ::Hash[Symbol | String, untyped],
+ ?request_options: OpenAI::request_opts
+ ) -> OpenAI::Realtime::CallCreateResponse
+
def accept: (
String call_id,
?audio: OpenAI::Realtime::RealtimeAudioConfig,
diff --git a/sig/openai/resources/realtime/translations.rbs b/sig/openai/resources/realtime/translations.rbs
new file mode 100644
index 000000000..5dd578f21
--- /dev/null
+++ b/sig/openai/resources/realtime/translations.rbs
@@ -0,0 +1,21 @@
+module OpenAI
+ module Resources
+ class Realtime
+ class Translations
+ attr_reader client_secrets: OpenAI::Resources::Realtime::Translations::ClientSecrets
+ attr_reader calls: OpenAI::Resources::Realtime::Translations::Calls
+
+ def connect: (
+ model: String,
+ ?request_options: OpenAI::request_opts?,
+ ?transport: untyped?,
+ ?transport_options: ::Hash[Symbol, untyped]
+ ) {
+ (OpenAI::Realtime::TranslationConnection connection) -> top
+ } -> top
+
+ def initialize: (client: OpenAI::Client) -> void
+ end
+ end
+ end
+end
diff --git a/sig/openai/resources/realtime/translations/calls.rbs b/sig/openai/resources/realtime/translations/calls.rbs
new file mode 100644
index 000000000..db1ebfdde
--- /dev/null
+++ b/sig/openai/resources/realtime/translations/calls.rbs
@@ -0,0 +1,16 @@
+module OpenAI
+ module Resources
+ class Realtime
+ class Translations
+ class Calls
+ def create: (
+ sdp: String,
+ ?request_options: OpenAI::request_opts?
+ ) -> OpenAI::Realtime::CallCreateResponse
+
+ def initialize: (client: OpenAI::Client) -> void
+ end
+ end
+ end
+ end
+end
diff --git a/sig/openai/resources/realtime/translations/client_secrets.rbs b/sig/openai/resources/realtime/translations/client_secrets.rbs
new file mode 100644
index 000000000..e609efe1a
--- /dev/null
+++ b/sig/openai/resources/realtime/translations/client_secrets.rbs
@@ -0,0 +1,19 @@
+module OpenAI
+ module Resources
+ class Realtime
+ class Translations
+ class ClientSecrets
+ def create: (
+ session: OpenAI::Realtime::RealtimeTranslationSessionCreateRequest
+ | ::Hash[Symbol | String, untyped],
+ ?expires_after: (OpenAI::Realtime::RealtimeTranslationClientSecretCreateRequest::ExpiresAfter
+ | ::Hash[(Symbol | String), untyped])?,
+ ?request_options: OpenAI::request_opts?
+ ) -> OpenAI::Realtime::RealtimeTranslationClientSecretCreateResponse
+
+ def initialize: (client: OpenAI::Client) -> void
+ end
+ end
+ end
+ end
+end
diff --git a/test/openai/logging_test.rb b/test/openai/logging_test.rb
index b98c3df7d..82864c6db 100644
--- a/test/openai/logging_test.rb
+++ b/test/openai/logging_test.rb
@@ -57,6 +57,24 @@ class UnbufferableChunk < String
def byteslice(*) = raise("stream chunk was buffered")
end
+ class FailingBody
+ include Enumerable
+
+ attr_reader :close_count
+
+ def initialize(error)
+ @error = error
+ @close_count = 0
+ end
+
+ def each
+ yield("partial")
+ raise @error
+ end
+
+ def close = (@close_count += 1)
+ end
+
def setup
super
@openai_log = ENV.delete("OPENAI_LOG")
@@ -614,6 +632,72 @@ def test_stream_failures_log_an_error_without_a_success_event
refute_includes(logger.events.fetch(0).fetch(1), "request complete")
end
+ def test_raw_response_completion_is_logged_only_after_body_consumption
+ logger = CapturingLogger.new
+ source = CloseableBody.new("answer-sdp")
+ http_client = StubHTTPClient.new do |_request|
+ OpenAI::HTTPClient::Response.new(
+ status: 200,
+ headers: {"content-type" => "application/sdp", "x-request-id" => "req_raw"},
+ body: source
+ )
+ end
+ client = diagnostic_client(http_client: http_client, logger: logger, log_level: :info)
+
+ response = client.request_raw(method: :post, path: "realtime/calls")
+
+ assert_empty(logger.events)
+ assert_equal(["answer-sdp"], response.body.to_a)
+ assert_equal(1, source.close_count)
+ assert_equal([:info], logger.events.map(&:first))
+ assert_includes(logger.events.fetch(0).fetch(1), "request complete")
+ end
+
+ def test_raw_response_body_failure_logs_an_error_without_a_success_event
+ logger = CapturingLogger.new
+ failure = OpenAI::Errors::APIConnectionError.new(
+ url: URI("https://example.com/v1/realtime/calls")
+ )
+ source = FailingBody.new(failure)
+ http_client = StubHTTPClient.new do |_request|
+ OpenAI::HTTPClient::Response.new(
+ status: 200,
+ headers: {"content-type" => "application/sdp"},
+ body: source
+ )
+ end
+ client = diagnostic_client(http_client: http_client, logger: logger, log_level: :info)
+ response = client.request_raw(method: :post, path: "realtime/calls")
+
+ assert_empty(logger.events)
+ error = assert_raises(OpenAI::Errors::APIConnectionError) { response.body.to_a }
+
+ assert_same(failure, error)
+ assert_equal(1, source.close_count)
+ assert_equal([:error], logger.events.map(&:first))
+ assert_includes(logger.events.fetch(0).fetch(1), "request failed")
+ refute_includes(logger.events.fetch(0).fetch(1), "request complete")
+ end
+
+ def test_closing_a_partially_consumed_raw_response_closes_its_source
+ logger = CapturingLogger.new
+ source = CloseableBody.new("first", "second")
+ http_client = StubHTTPClient.new do |_request|
+ OpenAI::HTTPClient::Response.new(
+ status: 200,
+ headers: {"content-type" => "application/sdp"},
+ body: source
+ )
+ end
+ client = diagnostic_client(http_client: http_client, logger: logger, log_level: :info)
+ response = client.request_raw(method: :post, path: "realtime/calls")
+
+ assert_equal("first", response.body.first)
+
+ assert_equal(1, source.close_count)
+ assert_empty(logger.events)
+ end
+
def test_consumer_exceptions_are_not_logged_as_request_failures
logger = CapturingLogger.new
http_client = StubHTTPClient.new do |_request|
diff --git a/test/openai/realtime/async_websocket_transport_test.rb b/test/openai/realtime/async_websocket_transport_test.rb
new file mode 100644
index 000000000..c2e101471
--- /dev/null
+++ b/test/openai/realtime/async_websocket_transport_test.rb
@@ -0,0 +1,608 @@
+# frozen_string_literal: true
+
+require "async/http/server"
+require "async/queue"
+require "async/websocket/adapters/http"
+require "async/websocket/client"
+require "async/websocket/server"
+require "socket"
+
+require_relative "../test_helper"
+
+class OpenAI::Test::AsyncWebSocketTransportTest < Minitest::Test
+ extend Minitest::Serial
+
+ class SynchronousConnection
+ attr_reader :scheduler
+
+ def initialize
+ @closed = false
+ @scheduler = nil
+ end
+
+ def connect!
+ @scheduler = Fiber.scheduler
+ self
+ end
+
+ def read = "message"
+ def write(_message) = nil
+ def flush = nil
+ def closed? = @closed
+ def close(*) = (@closed = true)
+ end
+
+ class SynchronousClient
+ attr_reader :closed
+
+ def initialize(connection)
+ @connection = connection
+ @closed = false
+ end
+
+ def connect(*) = @connection.connect!
+ def close = (@closed = true)
+ end
+
+ def test_default_transport_enters_a_reactor_for_synchronous_callers
+ assert_nil(Fiber.scheduler)
+ connection = SynchronousConnection.new
+ client = SynchronousClient.new(connection)
+ transport = OpenAI::Realtime::Transports::AsyncWebSocket.new
+ url = URI("wss://example.com/v1/realtime?model=gpt-realtime-2.1")
+
+ result = Async::WebSocket::Client.stub(:open, client) do
+ transport.open(url: url, headers: {}, timeout: nil) do |socket|
+ [socket.read, Fiber.scheduler]
+ end
+ end
+
+ assert_equal("message", result.fetch(0))
+ assert_same(connection.scheduler, result.fetch(1))
+ refute_nil(connection.scheduler)
+ assert_predicate(connection, :closed?)
+ assert_predicate(client, :closed)
+ assert_nil(Fiber.scheduler)
+ end
+
+ def test_default_transport_exchanges_events_with_a_local_websocket_server
+ port = available_port
+ endpoint = Async::HTTP::Endpoint.parse("http://127.0.0.1:#{port}")
+ fallback = ->(_request) { Protocol::HTTP::Response[404, {}, []] }
+ websocket = Async::WebSocket::Server.new(fallback) do |connection|
+ message = connection.read
+ next if message.nil?
+
+ event = JSON.parse(message.to_str)
+ raise "unexpected client event" unless event.fetch("type") == "session.update"
+
+ connection.write(
+ Protocol::WebSocket::TextMessage.generate(
+ type: "response.output_text.delta",
+ event_id: "event_1",
+ response_id: "response_1",
+ item_id: "item_1",
+ output_index: 0,
+ content_index: 0,
+ delta: "hello over websocket"
+ )
+ )
+ connection.flush
+ end
+ server = Async::HTTP::Server.new(websocket, endpoint)
+
+ Sync do |task|
+ server_task = task.async { server.run.wait }
+ client = OpenAI::Client.new(
+ api_key: "test-key",
+ base_url: "http://127.0.0.1:#{port}/v1",
+ timeout: 5
+ )
+
+ event = client.realtime.connect(model: "gpt-realtime") do |connection|
+ connection.session.update(type: :realtime)
+ connection.receive
+ end
+
+ assert_instance_of(OpenAI::Realtime::ResponseTextDeltaEvent, event)
+ assert_equal("hello over websocket", event.delta)
+ assert_application_errors_propagate(client)
+ ensure
+ server_task&.stop
+ end
+ end
+
+ def test_documented_text_lifecycle_over_a_local_websocket
+ with_websocket_server(method(:serve_text_lifecycle)) do |_task, client|
+ client.realtime.connect(model: "gpt-realtime-2.1") do |connection|
+ assert_instance_of(OpenAI::Realtime::SessionCreatedEvent, connection.receive)
+ connection.session.update(type: :realtime, output_modalities: [:text])
+ assert_instance_of(OpenAI::Realtime::SessionUpdatedEvent, connection.receive)
+
+ connection.conversation.items.create(
+ type: :message,
+ role: :user,
+ content: [{type: :input_text, text: "Say hello."}]
+ )
+ connection.response.create
+
+ delta = connection.receive
+ done = connection.receive
+ assert_instance_of(OpenAI::Realtime::ResponseTextDeltaEvent, delta)
+ assert_equal("hello from the local service", delta.delta)
+ assert_instance_of(OpenAI::Realtime::ResponseDoneEvent, done)
+ assert_equal(:completed, done.response.status)
+ assert_nil(connection.receive)
+ end
+ end
+ end
+
+ def test_function_call_round_trip_over_a_local_websocket
+ handler = lambda do |connection|
+ unless read_event(connection).fetch("type") == "conversation.item.create"
+ raise "expected conversation item"
+ end
+ raise "expected response.create" unless read_event(connection).fetch("type") == "response.create"
+
+ write_event(
+ connection,
+ type: "response.function_call_arguments.done",
+ event_id: "event_call",
+ response_id: "response_1",
+ item_id: "item_call",
+ output_index: 0,
+ call_id: "call_1",
+ name: "get_weather",
+ arguments: JSON.generate(location: "Paris")
+ )
+
+ output = read_event(connection)
+ raise "expected function output" unless output.dig("item", "type") == "function_call_output"
+ raise "unexpected call id" unless output.dig("item", "call_id") == "call_1"
+ raise "expected follow-up response" unless read_event(connection).fetch("type") == "response.create"
+
+ write_event(
+ connection,
+ type: "response.done",
+ event_id: "event_done",
+ response: {id: "response_2", status: "completed", output: []}
+ )
+ end
+
+ with_websocket_server(handler) do |_task, client|
+ client.realtime.connect(model: "gpt-realtime-2.1") do |connection|
+ connection.conversation.items.create(
+ type: :message,
+ role: :user,
+ content: [{type: :input_text, text: "What is the weather?"}]
+ )
+ connection.response.create
+
+ call = connection.receive
+ assert_instance_of(OpenAI::Realtime::ResponseFunctionCallArgumentsDoneEvent, call)
+ connection.conversation.items.create_function_call_output(
+ call_id: call.call_id,
+ output: JSON.generate(temperature_c: 18)
+ )
+ connection.response.create
+
+ done = connection.receive
+ assert_instance_of(OpenAI::Realtime::ResponseDoneEvent, done)
+ assert_equal(:completed, done.response.status)
+ end
+ end
+ end
+
+ def test_mcp_approval_lifecycle_over_a_local_websocket
+ with_websocket_server(method(:serve_mcp_approval)) do |_task, client|
+ client.realtime.connect(model: "gpt-realtime-2.1") do |connection|
+ exercise_mcp_approval(connection)
+ end
+ end
+ end
+
+ def test_translation_reads_output_while_input_is_still_streaming
+ first_audio = "\x00\x01".b * 4_800
+ second_audio = "\x02\x03".b * 4_800
+ translated_audio = "\x04\x05".b * 4_800
+ handler = lambda do |connection|
+ serve_translation(connection, first_audio:, second_audio:, translated_audio:)
+ end
+
+ with_websocket_server(handler) do |task, client|
+ client.realtime.translations.connect(model: "gpt-realtime-translate") do |connection|
+ connection.session.update(audio: {output: {language: "es"}})
+ events = Async::Queue.new
+ reader = task.async do
+ connection.each do |event|
+ events.enqueue(event)
+ break if event.is_a?(OpenAI::Realtime::RealtimeTranslationSessionClosedEvent)
+ end
+ end
+
+ connection.input_audio_buffer.append_bytes(first_audio)
+ transcript = events.dequeue
+ audio = events.dequeue
+ assert_instance_of(OpenAI::Realtime::RealtimeTranslationOutputTranscriptDeltaEvent, transcript)
+ assert_equal("hola", transcript.delta)
+ assert_instance_of(OpenAI::Realtime::RealtimeTranslationOutputAudioDeltaEvent, audio)
+ assert_equal(translated_audio, Base64.strict_decode64(audio.delta))
+
+ connection.input_audio_buffer.append_bytes(second_audio)
+ connection.session.close
+ assert_instance_of(OpenAI::Realtime::RealtimeTranslationSessionClosedEvent, events.dequeue)
+ reader.wait
+ end
+ end
+ end
+
+ def test_documented_transcription_lifecycle_over_a_local_websocket
+ audio = "\x00\x01".b * 2_400
+ handler = ->(connection) { serve_transcription(connection, audio: audio) }
+
+ with_websocket_server(handler) do |_task, client|
+ exercise_transcription(client, audio: audio)
+ end
+ end
+
+ def test_default_transport_reassembles_fragmented_text_messages
+ handler = lambda do |connection|
+ payload = JSON.generate(
+ type: "response.output_text.delta",
+ event_id: "event_fragmented",
+ response_id: "response_1",
+ item_id: "item_1",
+ output_index: 0,
+ content_index: 0,
+ delta: "fragmented"
+ )
+ midpoint = payload.bytesize / 2
+ first = Protocol::WebSocket::TextFrame.new(false).pack(payload.byteslice(0, midpoint))
+ second = Protocol::WebSocket::ContinuationFrame.new(true).pack(payload.byteslice(midpoint..))
+ connection.write_frame(first)
+ connection.write_frame(second)
+ connection.flush
+ end
+
+ with_websocket_server(handler) do |_task, client|
+ event = client.realtime.connect(model: "gpt-realtime-2.1", &:receive)
+
+ assert_instance_of(OpenAI::Realtime::ResponseTextDeltaEvent, event)
+ assert_equal("fragmented", event.delta)
+ end
+ end
+
+ def test_default_transport_wraps_an_abnormal_remote_close
+ handler = ->(connection) { connection.close(1011, "service failed") }
+
+ with_websocket_server(handler) do |_task, client|
+ error = assert_raises(OpenAI::Errors::RealtimeConnectionError) do
+ client.realtime.connect(model: "gpt-realtime-2.1", &:receive)
+ end
+
+ assert_instance_of(Protocol::WebSocket::ClosedError, error.cause)
+ assert_includes(error.message, "service failed")
+ end
+ end
+
+ def test_default_transport_wraps_handshake_failures
+ port = available_port
+ client = OpenAI::Client.new(
+ api_key: "test-key",
+ base_url: "http://127.0.0.1:#{port}/v1",
+ timeout: 0.5
+ )
+
+ error = assert_raises(OpenAI::Errors::RealtimeConnectionError) do
+ client.realtime.connect(model: "gpt-realtime") { |_connection| nil }
+ end
+
+ assert_equal("ws://127.0.0.1:#{port}/v1/realtime?model=gpt-realtime", error.url.to_s)
+ refute_nil(error.cause)
+ assert_includes(error.message, error.cause.message)
+ end
+
+ def test_missing_optional_dependency_preserves_load_error_as_the_cause
+ transport_class = Class.new(OpenAI::Realtime::Transports::AsyncWebSocket) do
+ private def require(path)
+ raise LoadError, "cannot load #{path}" if path == "async/websocket/client"
+
+ super
+ end
+ end
+ url = URI("wss://example.com/v1/realtime?model=gpt-realtime-2.1")
+
+ error = assert_raises(OpenAI::Errors::RealtimeConnectionError) do
+ transport_class.new.open(url:, headers: {}, timeout: 1) { |_socket| nil }
+ end
+
+ assert_instance_of(LoadError, error.cause)
+ assert_includes(error.message, "Add `gem \"async-websocket\"` to your Gemfile")
+ end
+
+ private def assert_application_errors_propagate(client)
+ error = assert_raises(RuntimeError) do
+ client.realtime.connect(model: "gpt-realtime") do |_connection|
+ raise "application failed"
+ end
+ end
+ assert_equal("application failed", error.message)
+
+ load_error = assert_raises(LoadError) do
+ client.realtime.connect(model: "gpt-realtime") do |_connection|
+ raise LoadError, "application dependency failed"
+ end
+ end
+ assert_equal("application dependency failed", load_error.message)
+ end
+
+ private def serve_text_lifecycle(connection)
+ write_event(
+ connection,
+ type: "session.created",
+ event_id: "event_created",
+ session: {type: "realtime"}
+ )
+
+ update = read_event(connection)
+ raise "expected session.update" unless update.fetch("type") == "session.update"
+ write_event(
+ connection,
+ type: "session.updated",
+ event_id: "event_updated",
+ session: update.fetch("session")
+ )
+
+ unless read_event(connection).fetch("type") == "conversation.item.create"
+ raise "expected conversation item"
+ end
+ raise "expected response.create" unless read_event(connection).fetch("type") == "response.create"
+ write_event(
+ connection,
+ type: "response.output_text.delta",
+ event_id: "event_delta",
+ response_id: "response_1",
+ item_id: "item_1",
+ output_index: 0,
+ content_index: 0,
+ delta: "hello from the local service"
+ )
+ write_event(
+ connection,
+ type: "response.done",
+ event_id: "event_done",
+ response: {id: "response_1", status: "completed", output: []}
+ )
+ end
+
+ private def serve_mcp_approval(connection)
+ update = read_event(connection)
+ raise "expected MCP session update" unless update.dig("session", "tools", 0, "type") == "mcp"
+ write_event(
+ connection,
+ type: "mcp_list_tools.completed",
+ event_id: "event_tools",
+ item_id: "item_tools"
+ )
+
+ raise "expected prompt item" unless read_event(connection).fetch("type") == "conversation.item.create"
+ raise "expected response.create" unless read_event(connection).fetch("type") == "response.create"
+ write_event(
+ connection,
+ type: "conversation.item.done",
+ event_id: "event_approval",
+ previous_item_id: nil,
+ item: {
+ type: "mcp_approval_request",
+ id: "approval_1",
+ arguments: JSON.generate(date: "tomorrow"),
+ name: "check_calendar",
+ server_label: "calendar"
+ }
+ )
+
+ approval = read_event(connection)
+ raise "expected approval response" unless approval.dig("item", "type") == "mcp_approval_response"
+ raise "unexpected approval id" unless approval.dig("item", "approval_request_id") == "approval_1"
+ raise "expected approval" unless approval.dig("item", "approve")
+ write_event(
+ connection,
+ type: "response.done",
+ event_id: "event_done",
+ response: {id: "response_1", status: "completed", output: []}
+ )
+ end
+
+ private def exercise_mcp_approval(connection)
+ connection.session.update(
+ type: :realtime,
+ tools: [
+ {
+ type: :mcp,
+ server_label: "calendar",
+ server_url: "https://example.com/mcp",
+ require_approval: :always
+ }
+ ]
+ )
+
+ assert_instance_of(OpenAI::Realtime::McpListToolsCompleted, connection.receive)
+ connection.conversation.items.create(
+ type: :message,
+ role: :user,
+ content: [{type: :input_text, text: "Check tomorrow's calendar."}]
+ )
+ connection.response.create
+
+ approval = connection.receive
+ assert_instance_of(OpenAI::Realtime::ConversationItemDone, approval)
+ assert_instance_of(OpenAI::Realtime::RealtimeMcpApprovalRequest, approval.item)
+ connection.conversation.items.respond_to_mcp_approval(
+ approval_request_id: approval.item.id,
+ approve: true,
+ reason: "Allowed by test policy"
+ )
+
+ done = connection.receive
+ assert_instance_of(OpenAI::Realtime::ResponseDoneEvent, done)
+ assert_equal(:completed, done.response.status)
+ end
+
+ private def serve_translation(connection, first_audio:, second_audio:, translated_audio:)
+ raise "expected translation update" unless read_event(connection).fetch("type") == "session.update"
+ first_append = read_event(connection)
+ unless first_append.fetch("type") == "session.input_audio_buffer.append"
+ raise "expected first audio frame"
+ end
+ raise "unexpected first audio" unless Base64.strict_decode64(first_append.fetch("audio")) == first_audio
+
+ write_event(
+ connection,
+ type: "session.output_transcript.delta",
+ event_id: "event_transcript",
+ delta: "hola",
+ elapsed_ms: 200
+ )
+ write_event(
+ connection,
+ type: "session.output_audio.delta",
+ event_id: "event_audio",
+ delta: Base64.strict_encode64(translated_audio),
+ elapsed_ms: 200,
+ format: "pcm16",
+ sample_rate: 24_000,
+ channels: 1
+ )
+
+ second_append = read_event(connection)
+ unless second_append.fetch("type") == "session.input_audio_buffer.append"
+ raise "expected second audio frame"
+ end
+ unless Base64.strict_decode64(second_append.fetch("audio")) == second_audio
+ raise "unexpected second audio"
+ end
+ raise "expected session.close" unless read_event(connection).fetch("type") == "session.close"
+
+ write_event(connection, type: "session.closed", event_id: "event_closed")
+ end
+
+ private def serve_transcription(connection, audio:)
+ write_event(
+ connection,
+ type: "session.created",
+ event_id: "event_created",
+ session: {type: "transcription"}
+ )
+
+ update = read_event(connection)
+ raise "expected transcription session update" unless update.fetch("type") == "session.update"
+
+ write_event(
+ connection,
+ type: "session.updated",
+ event_id: "event_updated",
+ session: update.fetch("session")
+ )
+
+ append = read_event(connection)
+ raise "expected audio append" unless append.fetch("type") == "input_audio_buffer.append"
+ unless Base64.strict_decode64(append.fetch("audio")) == audio
+ raise "unexpected transcription audio"
+ end
+ raise "expected audio commit" unless read_event(connection).fetch("type") == "input_audio_buffer.commit"
+
+ write_event(
+ connection,
+ type: "conversation.item.input_audio_transcription.delta",
+ event_id: "event_delta",
+ item_id: "item_1",
+ content_index: 0,
+ delta: "hello"
+ )
+ write_event(
+ connection,
+ type: "conversation.item.input_audio_transcription.completed",
+ event_id: "event_completed",
+ item_id: "item_1",
+ content_index: 0,
+ transcript: "hello from ruby",
+ usage: {type: "duration", seconds: 0.1}
+ )
+ end
+
+ private def exercise_transcription(client, audio:)
+ client.realtime.connect_transcription do |connection|
+ assert_instance_of(OpenAI::Realtime::SessionCreatedEvent, connection.receive)
+ configure_transcription(connection)
+ assert_instance_of(OpenAI::Realtime::SessionUpdatedEvent, connection.receive)
+
+ connection.input_audio_buffer.append_bytes(audio)
+ connection.input_audio_buffer.commit
+ assert_transcription_events(connection)
+ end
+ end
+
+ private def configure_transcription(connection)
+ connection.session.update(
+ audio: {
+ input: {
+ format: {type: :"audio/pcm", rate: 24_000},
+ transcription: {model: "gpt-live-transcribe"},
+ turn_detection: nil
+ }
+ }
+ )
+ end
+
+ private def assert_transcription_events(connection)
+ delta = connection.receive
+ completed = connection.receive
+ assert_instance_of(OpenAI::Realtime::ConversationItemInputAudioTranscriptionDeltaEvent, delta)
+ assert_equal("hello", delta.delta)
+ assert_instance_of(
+ OpenAI::Realtime::ConversationItemInputAudioTranscriptionCompletedEvent,
+ completed
+ )
+ assert_equal(delta.item_id, completed.item_id)
+ assert_equal("hello from ruby", completed.transcript)
+ end
+
+ private def with_websocket_server(handler)
+ port = available_port
+ endpoint = Async::HTTP::Endpoint.parse("http://127.0.0.1:#{port}")
+ fallback = ->(_request) { Protocol::HTTP::Response[404, {}, []] }
+ websocket = Async::WebSocket::Server.new(fallback, &handler)
+ server = Async::HTTP::Server.new(websocket, endpoint)
+
+ Sync do |task|
+ server_task = task.async { server.run.wait }
+ client = OpenAI::Client.new(
+ api_key: "test-key",
+ base_url: "http://127.0.0.1:#{port}/v1",
+ timeout: 5
+ )
+ yield(task, client)
+ ensure
+ server_task&.stop
+ end
+ end
+
+ private def read_event(connection)
+ message = connection.read
+ raise "client closed before sending the expected event" if message.nil?
+
+ JSON.parse(message.to_str)
+ end
+
+ private def write_event(connection, **event)
+ connection.write(Protocol::WebSocket::TextMessage.generate(event))
+ connection.flush
+ end
+
+ private def available_port
+ server = TCPServer.new("127.0.0.1", 0)
+ server.local_address.ip_port
+ ensure
+ server&.close
+ end
+end
diff --git a/test/openai/realtime/connection_test.rb b/test/openai/realtime/connection_test.rb
new file mode 100644
index 000000000..e75010054
--- /dev/null
+++ b/test/openai/realtime/connection_test.rb
@@ -0,0 +1,541 @@
+# frozen_string_literal: true
+
+require_relative "../test_helper"
+
+class OpenAI::Test::RealtimeConnectionTest < Minitest::Test
+ class FakeSocket
+ attr_reader :writes, :close_args
+
+ def initialize(*reads)
+ @reads = reads
+ @writes = []
+ @closed = false
+ end
+
+ def read = @reads.shift
+ def write(message) = @writes << message
+ def closed? = @closed
+
+ def close(code: 1000, reason: "")
+ @closed = true
+ @close_args = {code: code, reason: reason}
+ end
+ end
+
+ class FakeTransport
+ attr_reader :open_args
+
+ def initialize(socket)
+ @socket = socket
+ end
+
+ def open(url:, headers:, timeout:, **options)
+ @open_args = {url: url, headers: headers, timeout: timeout, options: options}
+ yield(@socket)
+ end
+ end
+
+ class FailingCloseSocket < FakeSocket
+ def close(code: 1000, reason: "")
+ super
+ raise IOError, "close failed"
+ end
+ end
+
+ def client(**options)
+ OpenAI::Client.new(
+ api_key: "test-key",
+ base_url: "https://example.com/v1",
+ **options
+ )
+ end
+
+ def test_connect_opens_a_typed_block_scoped_connection
+ socket = FakeSocket.new(
+ JSON.generate(
+ type: "response.output_text.delta",
+ event_id: "event_1",
+ response_id: "response_1",
+ item_id: "item_1",
+ output_index: 0,
+ content_index: 0,
+ delta: "hello"
+ )
+ )
+ transport = FakeTransport.new(socket)
+ event = nil
+
+ result = client.realtime.connect(
+ model: "gpt-realtime",
+ request_options: {
+ extra_query: {trace: "yes"},
+ extra_headers: {"X-Trace-ID" => "trace_1"},
+ timeout: 12
+ },
+ transport: transport,
+ transport_options: {max_frame_size: 1_024}
+ ) do |connection|
+ assert_instance_of(OpenAI::Realtime::Connection, connection)
+ event = connection.receive
+ :block_result
+ end
+
+ assert_equal(:block_result, result)
+ assert_instance_of(OpenAI::Realtime::ResponseTextDeltaEvent, event)
+ assert_equal("hello", event.delta)
+ assert_equal("wss://example.com/v1/realtime?model=gpt-realtime&trace=yes", transport.open_args[:url].to_s)
+ assert_equal("Bearer test-key", transport.open_args[:headers].fetch("authorization"))
+ assert_equal("trace_1", transport.open_args[:headers].fetch("x-trace-id"))
+ assert_equal(12.0, transport.open_args[:timeout])
+ assert_equal({max_frame_size: 1_024}, transport.open_args[:options])
+ assert(socket.closed?)
+ end
+
+ def test_connect_to_a_dedicated_transcription_session
+ socket = FakeSocket.new
+ transport = FakeTransport.new(socket)
+
+ client.realtime.connect_transcription(transport: transport) do |connection|
+ assert_instance_of(OpenAI::Realtime::TranscriptionConnection, connection)
+ assert_instance_of(OpenAI::Realtime::ConnectionResources::TranscriptionSession, connection.session)
+ assert_respond_to(connection.input_audio_buffer, :commit)
+ assert_respond_to(connection.input_audio_buffer, :clear)
+ refute_respond_to(connection, :response)
+ refute_respond_to(connection, :conversation)
+ refute_respond_to(connection, :output_audio_buffer)
+ connection.session.update(audio: {input: {turn_detection: nil}}, event_id: "update_1")
+ end
+
+ assert_equal(
+ "wss://example.com/v1/realtime?intent=transcription",
+ transport.open_args[:url].to_s
+ )
+ update = JSON.parse(socket.writes.fetch(0), symbolize_names: true)
+ assert_equal(:transcription, update.dig(:session, :type).to_sym)
+ assert_equal("update_1", update.fetch(:event_id))
+ refute(update.fetch(:session).key?(:event_id))
+ end
+
+ def test_connect_to_an_existing_webrtc_or_sip_call
+ transport = FakeTransport.new(FakeSocket.new)
+
+ client.realtime.connect_to_call(call_id: "rtc_123", transport: transport) do |connection|
+ assert_instance_of(OpenAI::Realtime::SidebandConnection, connection)
+ assert_respond_to(connection, :output_audio_buffer)
+ end
+
+ assert_equal("wss://example.com/v1/realtime?call_id=rtc_123", transport.open_args[:url].to_s)
+ end
+
+ def test_connect_uses_an_explicit_websocket_base_url
+ transport = FakeTransport.new(FakeSocket.new)
+ custom_client = client(websocket_base_url: "wss://socket.example.test/custom/v2")
+
+ custom_client.realtime.connect(model: "gpt-realtime", transport: transport) { |_connection| nil }
+
+ assert_equal(
+ "wss://socket.example.test/custom/v2/realtime?model=gpt-realtime",
+ transport.open_args[:url].to_s
+ )
+ end
+
+ def test_websocket_base_url_rejects_ambiguous_or_unsafe_urls
+ invalid_urls = [
+ "/socket",
+ "ftp://socket.example.test/v1",
+ "wss://user:secret@socket.example.test/v1",
+ "wss://socket.example.test/v1?tenant=one",
+ "wss://socket.example.test/v1#fragment"
+ ]
+
+ invalid_urls.each do |url|
+ error = assert_raises(ArgumentError) { client(websocket_base_url: url) }
+ assert_includes(error.message, "`websocket_base_url`")
+ end
+ end
+
+ def test_connect_uses_azure_provider_authentication
+ transport = FakeTransport.new(FakeSocket.new)
+ azure_client = OpenAI::Client.new(
+ provider: OpenAI::Providers.azure(
+ endpoint: "https://resource.openai.azure.com",
+ api_key: "azure-key"
+ )
+ )
+
+ azure_client.realtime.connect(model: "deployment", transport: transport) { |_connection| nil }
+
+ assert_equal(
+ "wss://resource.openai.azure.com/openai/v1/realtime?model=deployment",
+ transport.open_args[:url].to_s
+ )
+ assert_equal("azure-key", transport.open_args[:headers].fetch("api-key"))
+ refute(transport.open_args[:headers].key?("authorization"))
+ end
+
+ def test_connect_resolves_workload_identity_before_the_handshake
+ subject_token_provider =
+ OpenAI::Auth::SubjectTokenProviders::K8sServiceAccountTokenProvider.new(
+ token_path: "/not-read-by-this-test"
+ )
+ config = OpenAI::Auth::WorkloadIdentity.new(
+ identity_provider_id: "idp_123",
+ service_account_id: "sa_123",
+ provider: subject_token_provider
+ )
+ workload_client = OpenAI::Client.new(
+ api_key: nil,
+ workload_identity: config,
+ organization: "org_123",
+ base_url: "https://example.com/v1"
+ )
+ transport = FakeTransport.new(FakeSocket.new)
+
+ workload_client.workload_identity_auth.stub(:get_token, "exchanged-token") do
+ workload_client.realtime.connect(model: "gpt-realtime", transport: transport) do |_connection|
+ nil
+ end
+ end
+
+ assert_equal("Bearer exchanged-token", transport.open_args[:headers].fetch("authorization"))
+ end
+
+ def test_unsupported_provider_fails_before_resolving_or_sending_credentials
+ credential_requested = false
+ provider = OpenAI::Providers.bedrock(
+ region: "us-east-1",
+ token_provider: lambda do
+ credential_requested = true
+ "bedrock-token"
+ end
+ )
+ provider_client = OpenAI::Client.new(provider: provider)
+ transport = FakeTransport.new(FakeSocket.new)
+
+ error = assert_raises(OpenAI::Errors::Error) do
+ provider_client.realtime.connect(model: "gpt-realtime", transport: transport) do |_connection|
+ nil
+ end
+ end
+
+ assert_includes(error.message, "not supported")
+ refute(credential_requested)
+ assert_nil(transport.open_args)
+ end
+
+ def test_connection_sends_typed_events_and_audio_bytes
+ socket = FakeSocket.new
+ transport = FakeTransport.new(socket)
+
+ client.realtime.connect(model: "gpt-realtime", transport: transport) do |connection|
+ send_typed_resource_events(connection)
+ end
+
+ events = socket.writes.map { JSON.parse(_1, symbolize_names: true) }
+ assert_equal(
+ [
+ :"session.update",
+ :"response.create",
+ :"input_audio_buffer.append",
+ :"input_audio_buffer.commit",
+ :"conversation.item.create",
+ :"conversation.item.create",
+ :"conversation.item.create",
+ :"conversation.item.create"
+ ],
+ events.map { _1.fetch(:type).to_sym }
+ )
+ assert_equal("session_update_1", events[0].fetch(:event_id))
+ assert_equal("Be concise", events[0].dig(:session, :instructions))
+ refute(events[0].fetch(:session).key?(:event_id))
+ assert_equal("response_create_1", events[1].fetch(:event_id))
+ assert_equal("Say hello", events[1].dig(:response, :instructions))
+ refute(events[1].fetch(:response).key?(:event_id))
+ assert_equal(Base64.strict_encode64("\x00\x01".b), events[2].fetch(:audio))
+ assert_equal("item_create_1", events[4].fetch(:event_id))
+ assert_equal("root", events[4].fetch(:previous_item_id))
+ assert_equal("Hello", events[4].dig(:item, :content, 0, :text))
+ refute(events[4].fetch(:item).key?(:previous_item_id))
+ assert_equal(:function_call_output, events[5].fetch(:item).fetch(:type).to_sym)
+ assert_equal(:mcp_approval_response, events[6].fetch(:item).fetch(:type).to_sym)
+ assert(events[6].fetch(:item).fetch(:approve))
+ generated_id = events[7].fetch(:item).fetch(:id)
+ assert_operator(generated_id.length, :<=, 32)
+ assert_match(/\Amcpa_[0-9a-f]+\z/, generated_id)
+ end
+
+ def test_standard_connection_only_exposes_standard_websocket_capabilities
+ transport = FakeTransport.new(FakeSocket.new)
+
+ client.realtime.connect(model: "gpt-realtime", transport: transport) do |connection|
+ refute_respond_to(connection, :output_audio_buffer)
+ refute_respond_to(connection.session, :close)
+ assert_respond_to(connection.input_audio_buffer, :commit)
+ assert_respond_to(connection.input_audio_buffer, :clear)
+ assert_respond_to(connection.conversation, :items)
+ refute_respond_to(connection.conversation, :item)
+ end
+ end
+
+ def test_sideband_connection_exposes_output_audio_buffer_controls
+ socket = FakeSocket.new
+ transport = FakeTransport.new(socket)
+
+ client.realtime.connect_to_call(call_id: "rtc_123", transport: transport) do |connection|
+ connection.output_audio_buffer.clear(event_id: "clear_1")
+ end
+
+ assert_equal(
+ {"type" => "output_audio_buffer.clear", "event_id" => "clear_1"},
+ JSON.parse(socket.writes.fetch(0))
+ )
+ end
+
+ def test_sideband_interruption_helpers_preserve_documented_order
+ socket = FakeSocket.new
+ transport = FakeTransport.new(socket)
+
+ client.realtime.connect_to_call(call_id: "rtc_123", transport: transport) do |connection|
+ connection.response.cancel
+ connection.conversation.items.truncate(item_id: "item_1", content_index: 0, audio_end_ms: 640)
+ connection.output_audio_buffer.clear
+ end
+
+ assert_equal(
+ [
+ "response.cancel",
+ "conversation.item.truncate",
+ "output_audio_buffer.clear"
+ ],
+ socket.writes.map { JSON.parse(_1).fetch("type") }
+ )
+ end
+
+ def test_all_standard_resource_helpers_emit_typed_events
+ socket = FakeSocket.new
+ transport = FakeTransport.new(socket)
+
+ client.realtime.connect(model: "gpt-realtime", transport: transport) do |connection|
+ connection.response.cancel(response_id: "response_1", event_id: "cancel_1")
+ connection.input_audio_buffer.clear(event_id: "clear_1")
+ connection.conversation.items.retrieve(item_id: "item_1", event_id: "retrieve_1")
+ connection.conversation.items.truncate(
+ item_id: "item_1",
+ content_index: 0,
+ audio_end_ms: 250,
+ event_id: "truncate_1"
+ )
+ connection.conversation.items.delete(item_id: "item_1", event_id: "delete_1")
+ end
+
+ assert_equal(
+ [
+ "response.cancel",
+ "input_audio_buffer.clear",
+ "conversation.item.retrieve",
+ "conversation.item.truncate",
+ "conversation.item.delete"
+ ],
+ socket.writes.map { JSON.parse(_1).fetch("type") }
+ )
+ end
+
+ def test_connection_does_not_override_ruby_send
+ transport = FakeTransport.new(FakeSocket.new)
+
+ client.realtime.connect(model: "gpt-realtime", transport: transport) do |connection|
+ assert_equal(false, connection.send(:closed?))
+ end
+ end
+
+ def test_connection_rejects_an_unknown_client_event
+ transport = FakeTransport.new(FakeSocket.new)
+
+ error = assert_raises(ArgumentError) do
+ client.realtime.connect(model: "gpt-realtime", transport: transport) do |connection|
+ connection.send_event(type: "future.unknown.event")
+ end
+ end
+
+ assert_includes(error.message, "future.unknown.event")
+ end
+
+ def test_connection_rejects_a_client_event_missing_required_fields
+ transport = FakeTransport.new(FakeSocket.new)
+
+ error = assert_raises(ArgumentError) do
+ client.realtime.connect(model: "gpt-realtime", transport: transport) do |connection|
+ connection.send_event(type: "conversation.item.create")
+ end
+ end
+
+ assert_includes(error.message, "required fields")
+ end
+
+ def test_connect_requires_a_block
+ error = assert_raises(ArgumentError) do
+ client.realtime.connect(model: "gpt-realtime", transport: FakeTransport.new(FakeSocket.new))
+ end
+
+ assert_includes(error.message, "block is required")
+ end
+
+ def test_connection_each_returns_a_typed_enumerator_without_a_block
+ event_data = JSON.generate(
+ type: "response.output_text.delta",
+ event_id: "event_1",
+ response_id: "response_1",
+ item_id: "item_1",
+ output_index: 0,
+ content_index: 0,
+ delta: "hello"
+ )
+ transport = FakeTransport.new(FakeSocket.new(event_data, nil))
+ events = nil
+
+ client.realtime.connect(model: "gpt-realtime", transport: transport) do |connection|
+ events = connection.each.to_a
+ end
+
+ assert_equal(1, events.length)
+ assert_instance_of(OpenAI::Realtime::ResponseTextDeltaEvent, events.first)
+ end
+
+ def test_connection_each_returns_the_connection_with_a_block
+ data = JSON.generate(type: "future.event")
+ transport = FakeTransport.new(FakeSocket.new(data, nil))
+
+ client.realtime.connect(model: "gpt-realtime", transport: transport) do |connection|
+ count = 0
+ result = connection.each { |_event| count += 1 }
+
+ assert_equal(1, count)
+ assert_same(connection, result)
+ end
+ end
+
+ def test_connection_closes_when_the_block_raises
+ socket = FakeSocket.new
+ transport = FakeTransport.new(socket)
+
+ assert_raises(RuntimeError) do
+ client.realtime.connect(model: "gpt-realtime", transport: transport) do |_connection|
+ raise "boom"
+ end
+ end
+
+ assert(socket.closed?)
+ end
+
+ def test_application_error_wins_when_cleanup_also_fails
+ transport = FakeTransport.new(FailingCloseSocket.new)
+
+ error = assert_raises(RuntimeError) do
+ client.realtime.connect(model: "gpt-realtime", transport: transport) do |_connection|
+ raise "application failed"
+ end
+ end
+
+ assert_equal("application failed", error.message)
+ end
+
+ def test_cleanup_error_propagates_after_a_successful_block
+ transport = FakeTransport.new(FailingCloseSocket.new)
+
+ error = assert_raises(IOError) do
+ client.realtime.connect(model: "gpt-realtime", transport: transport) { |_connection| :done }
+ end
+
+ assert_equal("close failed", error.message)
+ end
+
+ def test_malformed_server_event_raises_a_realtime_protocol_error
+ transport = FakeTransport.new(FakeSocket.new("not-json"))
+
+ error = assert_raises(OpenAI::Errors::RealtimeProtocolError) do
+ client.realtime.connect(model: "gpt-realtime", transport: transport, &:receive)
+ end
+
+ assert_equal("not-json", error.data)
+ assert_instance_of(JSON::ParserError, error.cause)
+ end
+
+ def test_api_error_remains_a_typed_event_in_the_stream
+ transport = FakeTransport.new(
+ FakeSocket.new(
+ JSON.generate(
+ type: "error",
+ event_id: "event_1",
+ error: {type: "invalid_request_error", message: "Try another value"}
+ )
+ )
+ )
+
+ event = client.realtime.connect(model: "gpt-realtime", transport: transport, &:receive)
+
+ assert_instance_of(OpenAI::Realtime::RealtimeErrorEvent, event)
+ assert_equal("Try another value", event.error.message)
+ end
+
+ def test_unknown_server_event_remains_observable
+ data = JSON.generate(type: "future.unknown.event", nested: {values: ["future value"]})
+ transport = FakeTransport.new(FakeSocket.new(data))
+
+ event = client.realtime.connect(model: "gpt-realtime", transport: transport, &:receive)
+
+ assert_instance_of(OpenAI::Realtime::UnknownServerEvent, event)
+ assert_equal(:"future.unknown.event", event.type)
+ assert_equal(
+ {type: "future.unknown.event", nested: {values: ["future value"]}},
+ event.data
+ )
+ assert(event.frozen?)
+ assert(event.data.frozen?)
+ assert(event.data.fetch(:nested).frozen?)
+ assert(event.data.dig(:nested, :values).frozen?)
+ assert(event.data.dig(:nested, :values, 0).frozen?)
+ end
+
+ def test_server_event_missing_required_fields_raises_a_realtime_protocol_error
+ data = JSON.generate(type: "response.output_text.delta")
+ transport = FakeTransport.new(FakeSocket.new(data))
+
+ error = assert_raises(OpenAI::Errors::RealtimeProtocolError) do
+ client.realtime.connect(model: "gpt-realtime", transport: transport, &:receive)
+ end
+
+ assert_equal(data, error.data)
+ assert_includes(error.cause.message, "required fields")
+ end
+
+ private def send_typed_resource_events(connection)
+ connection.session.update(
+ type: :realtime,
+ instructions: "Be concise",
+ event_id: "session_update_1"
+ )
+ connection.response.create(instructions: "Say hello", event_id: "response_create_1")
+ connection.input_audio_buffer.append_bytes("\x00\x01".b)
+ connection.input_audio_buffer.commit
+ connection.conversation.items.create(
+ type: :message,
+ role: :user,
+ content: [{type: :input_text, text: "Hello"}],
+ previous_item_id: "root",
+ event_id: "item_create_1"
+ )
+ connection.conversation.items.create_function_call_output(
+ call_id: "call_1",
+ output: JSON.generate(temperature: 72)
+ )
+ connection.conversation.items.respond_to_mcp_approval(
+ approval_request_id: "approval_1",
+ approve: true,
+ id: "approval_response_1"
+ )
+ connection.conversation.items.respond_to_mcp_approval(
+ approval_request_id: "approval_2",
+ approve: false
+ )
+ end
+end
diff --git a/test/openai/realtime/example_stream_lifecycle_test.rb b/test/openai/realtime/example_stream_lifecycle_test.rb
new file mode 100644
index 000000000..f60f5a034
--- /dev/null
+++ b/test/openai/realtime/example_stream_lifecycle_test.rb
@@ -0,0 +1,452 @@
+# frozen_string_literal: true
+
+require_relative "../test_helper"
+require "async/notification"
+require_relative "../../../examples/realtime/mcp_approval"
+require_relative "../../../examples/realtime/sideband"
+require_relative "../../../examples/realtime/sip"
+require_relative "../../../examples/realtime/translation"
+require_relative "../../../examples/realtime/websocket_audio"
+require_relative "../../../examples/realtime/websocket_text"
+require_relative "../../../examples/realtime/websocket_transcription"
+
+class OpenAI::Test::RealtimeExampleStreamLifecycleTest < Minitest::Test
+ class RecordingConnection
+ def initialize(events = [])
+ @events = events
+ end
+
+ def each(&block)
+ @events.each(&block)
+ end
+
+ def receive = @events.shift
+ end
+
+ class RaisingEndpoint
+ attr_reader :calls
+
+ def initialize(error)
+ @error = error
+ @calls = []
+ end
+
+ def append_bytes(bytes)
+ @calls << [:append_bytes, bytes]
+ raise @error
+ end
+
+ def close
+ @calls << [:close]
+ raise @error
+ end
+ end
+
+ class ClosingSession
+ attr_reader :closed
+
+ def close = @closed = true
+ end
+
+ class RecordingEndpoint
+ attr_reader :calls
+
+ def initialize = @calls = []
+ def append_bytes(bytes) = @calls << [:append_bytes, bytes]
+ def close = @calls << [:close]
+ end
+
+ class BlockingAudioBuffer
+ attr_reader :chunks
+
+ def initialize(upload_started)
+ @upload_started = upload_started
+ @chunks = []
+ end
+
+ def append_bytes(bytes)
+ @chunks << bytes
+ @upload_started.signal
+ Kernel.sleep(3_600)
+ end
+ end
+
+ class ReaderFailureConnection
+ attr_reader :input_audio_buffer, :session
+
+ def initialize(event)
+ upload_started = Async::Notification.new
+ @event = event
+ @input_audio_buffer = BlockingAudioBuffer.new(upload_started)
+ @session = ClosingSession.new
+ @upload_started = upload_started
+ end
+
+ def each
+ @upload_started.wait
+ yield(@event)
+ end
+ end
+
+ class BufferedReaderFailureConnection
+ attr_reader :input_audio_buffer, :session
+
+ def initialize(event)
+ @event = event
+ @input_audio_buffer = RecordingEndpoint.new
+ @session = RecordingEndpoint.new
+ end
+
+ def each = yield(@event)
+ end
+
+ TranslationConnection = Data.define(:input_audio_buffer, :session)
+
+ Event = Data.define(:type, :data) do
+ def to_h = data
+ end
+
+ def test_wait_for_rejects_eof_before_the_target_event
+ error = assert_raises(RuntimeError) do
+ OpenAI::Examples::Realtime::EventStream.wait_for(
+ RecordingConnection.new,
+ OpenAI::Realtime::SessionUpdatedEvent,
+ closed_message: "Realtime connection closed before session.updated"
+ )
+ end
+
+ assert_equal("Realtime connection closed before session.updated", error.message)
+ end
+
+ def test_wait_for_surfaces_an_api_error_before_the_target_event
+ error = assert_raises(RuntimeError) do
+ OpenAI::Examples::Realtime::EventStream.wait_for(
+ RecordingConnection.new([realtime_error_event("session update failed")]),
+ OpenAI::Realtime::SessionUpdatedEvent,
+ closed_message: "Realtime connection closed before session.updated"
+ )
+ end
+
+ assert_equal("session update failed", error.message)
+ end
+
+ def test_sideband_rejects_eof_before_the_requested_event
+ error = assert_raises(RuntimeError) do
+ OpenAI::Examples::Realtime::Sideband.stream(
+ RecordingConnection.new,
+ output: StringIO.new,
+ stop_after: "session.updated"
+ )
+ end
+
+ assert_equal("Realtime connection closed before session.updated", error.message)
+ end
+
+ def test_sip_rejects_eof_before_the_requested_event
+ error = assert_raises(RuntimeError) do
+ OpenAI::Examples::Realtime::SIP.stream(
+ RecordingConnection.new,
+ output: StringIO.new,
+ stop_after: "response.done"
+ )
+ end
+
+ assert_equal("Realtime connection closed before response.done", error.message)
+ end
+
+ def test_websocket_audio_rejects_a_connection_that_closes_before_response_done
+ error = assert_raises(RuntimeError) do
+ OpenAI::Examples::Realtime::WebSocketAudio.stream_response(
+ RecordingConnection.new,
+ output: StringIO.new
+ )
+ end
+
+ assert_equal("Realtime connection closed before response.done", error.message)
+ end
+
+ def test_websocket_audio_accepts_audio_followed_by_a_completed_response
+ audio = "pcm".b
+ output = StringIO.new
+ connection = RecordingConnection.new(
+ [
+ OpenAI::Realtime::ResponseAudioDeltaEvent.new(
+ event_id: "event_1",
+ response_id: "response_1",
+ item_id: "item_1",
+ output_index: 0,
+ content_index: 0,
+ delta: Base64.strict_encode64(audio)
+ ),
+ OpenAI::Realtime::ResponseDoneEvent.new(
+ event_id: "event_2",
+ response: OpenAI::Realtime::RealtimeResponse.new(
+ id: "response_1",
+ status: :completed
+ )
+ )
+ ]
+ )
+
+ result = OpenAI::Examples::Realtime::WebSocketAudio.stream_response(
+ connection,
+ output: output
+ )
+
+ assert_nil(result)
+ assert_equal(audio, output.string)
+ end
+
+ def test_websocket_audio_rejects_a_completed_response_without_audio
+ connection = RecordingConnection.new(
+ [
+ OpenAI::Realtime::ResponseDoneEvent.new(
+ event_id: "event_1",
+ response: OpenAI::Realtime::RealtimeResponse.new(
+ id: "response_1",
+ status: :completed
+ )
+ )
+ ]
+ )
+
+ error = assert_raises(RuntimeError) do
+ OpenAI::Examples::Realtime::WebSocketAudio.stream_response(
+ connection,
+ output: StringIO.new
+ )
+ end
+
+ assert_equal("Response completed without audio output", error.message)
+ end
+
+ def test_websocket_text_rejects_a_connection_that_closes_before_response_done
+ error = assert_raises(RuntimeError) do
+ OpenAI::Examples::Realtime::WebSocketText.stream_response(
+ RecordingConnection.new,
+ output: StringIO.new
+ )
+ end
+
+ assert_equal("Realtime connection closed before response.done", error.message)
+ end
+
+ def test_websocket_text_rejects_a_completed_response_without_text
+ connection = RecordingConnection.new([completed_response_event])
+ output = StringIO.new
+
+ error = assert_raises(RuntimeError) do
+ OpenAI::Examples::Realtime::WebSocketText.stream_response(connection, output: output)
+ end
+
+ assert_equal("Realtime response completed without text output", error.message)
+ refute_includes(output.string, "response.done status=completed")
+ end
+
+ def test_websocket_text_accepts_non_empty_text_and_a_completed_response
+ connection = RecordingConnection.new(
+ [
+ OpenAI::Realtime::ResponseTextDeltaEvent.new(
+ content_index: 0,
+ delta: "Hello from Ruby.",
+ event_id: "event_1",
+ item_id: "item_1",
+ output_index: 0,
+ response_id: "response_1"
+ ),
+ completed_response_event
+ ]
+ )
+ output = StringIO.new
+
+ OpenAI::Examples::Realtime::WebSocketText.stream_response(connection, output: output)
+
+ assert_includes(output.string, "Hello from Ruby.")
+ assert_includes(output.string, "response.done status=completed")
+ end
+
+ def test_translation_rejects_a_connection_that_closes_before_session_closed
+ error = assert_raises(RuntimeError) do
+ OpenAI::Examples::Realtime::Translation.stream(
+ RecordingConnection.new,
+ audio_output: StringIO.new,
+ transcript_output: StringIO.new
+ )
+ end
+
+ assert_equal(
+ "Realtime translation connection closed before session.closed",
+ error.message
+ )
+ end
+
+ def test_translation_rejects_session_closed_without_audio
+ connection = RecordingConnection.new(
+ [
+ OpenAI::Realtime::RealtimeTranslationSessionClosedEvent.new(
+ event_id: "event_1"
+ )
+ ]
+ )
+
+ error = assert_raises(RuntimeError) do
+ OpenAI::Examples::Realtime::Translation.stream(
+ connection,
+ audio_output: StringIO.new,
+ transcript_output: StringIO.new
+ )
+ end
+
+ assert_equal("Translation session closed without audio output", error.message)
+ end
+
+ def test_translation_accepts_audio_followed_by_session_closed
+ audio = "translated pcm".b
+ connection = RecordingConnection.new(
+ [
+ OpenAI::Realtime::RealtimeTranslationOutputAudioDeltaEvent.new(
+ event_id: "event_1",
+ delta: Base64.strict_encode64(audio)
+ ),
+ OpenAI::Realtime::RealtimeTranslationSessionClosedEvent.new(
+ event_id: "event_2"
+ )
+ ]
+ )
+ audio_output = StringIO.new
+ transcript_output = StringIO.new
+
+ result = OpenAI::Examples::Realtime::Translation.stream(
+ connection,
+ audio_output: audio_output,
+ transcript_output: transcript_output
+ )
+
+ assert_nil(result)
+ assert_equal(audio, audio_output.string)
+ assert_equal("\n", transcript_output.string)
+ end
+
+ def test_transcription_rejects_a_connection_that_closes_before_completion
+ error = assert_raises(RuntimeError) do
+ OpenAI::Examples::Realtime::WebSocketTranscription.print_transcript(
+ RecordingConnection.new,
+ output: StringIO.new
+ )
+ end
+
+ assert_equal("Realtime connection closed before transcription completed", error.message)
+ end
+
+ def test_transcription_accepts_only_a_completed_event
+ connection = RecordingConnection.new(
+ [
+ OpenAI::Realtime::ConversationItemInputAudioTranscriptionCompletedEvent.new(
+ content_index: 0,
+ event_id: "event_1",
+ item_id: "item_1",
+ transcript: "Hello",
+ usage: {type: :duration, seconds: 0.25}
+ )
+ ]
+ )
+ output = StringIO.new
+
+ result = OpenAI::Examples::Realtime::WebSocketTranscription.print_transcript(
+ connection,
+ output: output
+ )
+
+ assert_nil(result)
+ assert_equal("\n[item_1] Hello\n", output.string)
+ end
+
+ def test_translation_preserves_an_upload_error_when_session_close_also_fails
+ upload_error = RuntimeError.new("upload failed")
+ input_audio_buffer = RaisingEndpoint.new(upload_error)
+ session = RaisingEndpoint.new(RuntimeError.new("close failed"))
+ connection = TranslationConnection.new(input_audio_buffer, session)
+
+ Tempfile.create("translation-input") do |input|
+ input.write("audio")
+ input.flush
+
+ error = assert_raises(RuntimeError) do
+ OpenAI::Examples::Realtime::Translation.write_input(connection, input.path)
+ end
+
+ assert_same(upload_error, error)
+ end
+ assert_equal([[:append_bytes, "audio"]], input_audio_buffer.calls)
+ assert_equal([[:close]], session.calls)
+ end
+
+ def test_translation_stops_uploading_when_the_reader_fails
+ connection = ReaderFailureConnection.new(realtime_error_event("translation failed"))
+
+ Tempfile.create("translation-input") do |input|
+ input.write("audio")
+ input.flush
+
+ error = Timeout.timeout(1) do
+ Sync do
+ assert_raises(RuntimeError) do
+ OpenAI::Examples::Realtime::Translation.exchange(
+ connection,
+ input_path: input.path,
+ audio_output: StringIO.new,
+ transcript_output: StringIO.new
+ )
+ end
+ end
+ end
+
+ assert_equal("translation failed", error.message)
+ end
+ assert_equal(["audio"], connection.input_audio_buffer.chunks)
+ assert(connection.session.closed)
+ end
+
+ def test_translation_does_not_start_upload_after_a_buffered_reader_failure
+ connection = BufferedReaderFailureConnection.new(
+ realtime_error_event("translation failed before upload")
+ )
+
+ Tempfile.create("translation-input") do |input|
+ input.write("audio")
+ input.flush
+
+ error = Sync do
+ assert_raises(RuntimeError) do
+ OpenAI::Examples::Realtime::Translation.exchange(
+ connection,
+ input_path: input.path,
+ audio_output: StringIO.new,
+ transcript_output: StringIO.new
+ )
+ end
+ end
+
+ assert_equal("translation failed before upload", error.message)
+ end
+ assert_empty(connection.input_audio_buffer.calls)
+ assert_empty(connection.session.calls)
+ end
+
+ private def completed_response_event
+ OpenAI::Realtime::ResponseDoneEvent.new(
+ event_id: "event_done",
+ response: OpenAI::Realtime::RealtimeResponse.new(
+ id: "response_done",
+ status: :completed
+ )
+ )
+ end
+
+ private def realtime_error_event(message)
+ OpenAI::Realtime::RealtimeErrorEvent.new(
+ event_id: "event_error",
+ error: OpenAI::Realtime::RealtimeError.new(type: "server_error", message: message)
+ )
+ end
+end
diff --git a/test/openai/realtime/examples_test.rb b/test/openai/realtime/examples_test.rb
new file mode 100644
index 000000000..11666713a
--- /dev/null
+++ b/test/openai/realtime/examples_test.rb
@@ -0,0 +1,990 @@
+# frozen_string_literal: true
+
+require "open3"
+require_relative "../test_helper"
+require_relative "../../../examples/realtime/mcp_approval"
+require_relative "../../../examples/realtime/realtime_conversation"
+require_relative "../../../examples/realtime/sideband"
+require_relative "../../../examples/realtime/sip"
+require_relative "../../../examples/realtime/webrtc_conversation"
+require_relative "../../../examples/realtime/websocket_audio"
+
+class OpenAI::Test::RealtimeExamplesTest < Minitest::Test
+ Event = Data.define(:type, :data) do
+ def to_h = data
+ end
+
+ class RecordingSession
+ attr_reader :updates
+
+ def initialize
+ @updates = []
+ end
+
+ def update(**params)
+ @updates << params
+ end
+ end
+
+ class RecordingResource
+ attr_reader :calls, :truncations
+
+ def initialize(writer_fibers = nil)
+ @calls = []
+ @truncations = []
+ @writer_fibers = writer_fibers
+ end
+
+ def create(**params)
+ @calls << params
+ end
+
+ def truncate(**params)
+ @writer_fibers&.push(Fiber.current)
+ @truncations << params
+ end
+ end
+
+ class RecordingConversation
+ attr_reader :items
+
+ def initialize(writer_fibers = nil)
+ @items = RecordingResource.new(writer_fibers)
+ end
+ end
+
+ class RecordingAudioBuffer
+ attr_accessor :append_error
+ attr_reader :chunks, :commits
+
+ def initialize(writer_fibers = nil)
+ @chunks = []
+ @commits = 0
+ @writer_fibers = writer_fibers
+ end
+
+ def append_bytes(bytes)
+ @writer_fibers&.push(Fiber.current)
+ raise @append_error if @append_error
+
+ @chunks << bytes
+ end
+
+ def commit = @commits += 1
+ end
+
+ class RecordingOutbound
+ attr_reader :chunks, :truncations
+
+ def initialize
+ @chunks = []
+ @truncations = []
+ end
+
+ def append_audio(bytes) = @chunks << bytes
+ def truncate(**params) = @truncations << params
+ end
+
+ class RecordingConnection
+ attr_reader :conversation, :input_audio_buffer, :received, :response, :session, :writer_fibers
+
+ def initialize(events = [])
+ @events = events
+ @received = []
+ @writer_fibers = []
+ @session = RecordingSession.new
+ @conversation = RecordingConversation.new(@writer_fibers)
+ @input_audio_buffer = RecordingAudioBuffer.new(@writer_fibers)
+ @response = RecordingResource.new
+ end
+
+ def each(&block)
+ @events.each(&block)
+ end
+
+ def receive
+ @events.shift.tap { |event| @received << event&.type }
+ end
+ end
+
+ class ExplodingConnection < RecordingConnection
+ def each
+ raise "sideband failed"
+ end
+ end
+
+ class RecordingCalls
+ attr_accessor :hangup_error
+ attr_reader :accepts, :hangups
+
+ def initialize
+ @accepts = []
+ @hangups = []
+ end
+
+ def accept(call_id, **params)
+ @accepts << [call_id, params]
+ end
+
+ def hangup(call_id)
+ @hangups << call_id
+ raise @hangup_error if @hangup_error
+ end
+ end
+
+ class RecordingRealtime
+ attr_reader :calls, :connections
+
+ def initialize(connection)
+ @calls = RecordingCalls.new
+ @connection = connection
+ @connections = []
+ end
+
+ def connect_to_call(call_id:)
+ @connections << call_id
+ yield(@connection)
+ end
+
+ def connect(model:)
+ @connections << model
+ yield(@connection)
+ end
+ end
+
+ RecordingClient = Data.define(:realtime)
+
+ HTTPRequest = Data.define(:request_method, :path, :body, :headers) do
+ def [](name) = headers[name.downcase]
+ end
+
+ class HTTPResponse
+ attr_accessor :body, :status
+ attr_reader :headers
+
+ def initialize
+ @headers = {}
+ end
+
+ def []=(name, value)
+ @headers[name] = value
+ end
+ end
+
+ class RecordingWebRTCCalls
+ attr_accessor :hangup_error
+ attr_reader :creates, :hangups
+
+ def initialize
+ @creates = []
+ @hangups = []
+ end
+
+ def create(**params)
+ @creates << params
+ OpenAI::Realtime::CallCreateResponse.new(
+ sdp: "answer-sdp",
+ call_id: "rtc_example",
+ headers: {}
+ )
+ end
+
+ def hangup(call_id)
+ @hangups << call_id
+ raise @hangup_error if @hangup_error
+ end
+ end
+
+ class RecordingMicrophone
+ attr_reader :stopped
+
+ def initialize(chunks)
+ @chunks = chunks
+ @stopped = false
+ end
+
+ def each_chunk(&block)
+ @chunks.each(&block)
+ end
+
+ def stop = @stopped = true
+ end
+
+ class RecordingSpeaker
+ attr_reader :audio, :interruptions
+
+ def initialize
+ @audio = +"".b
+ @interruptions = 0
+ end
+
+ def write(bytes)
+ @audio << bytes
+ end
+
+ def interrupt
+ @interruptions += 1
+ end
+
+ def close = nil
+ end
+
+ def test_realtime_conversation_configures_continuous_server_vad
+ connection = RecordingConnection.new
+
+ OpenAI::Examples::Realtime::Conversation.configure(
+ connection,
+ voice: :marin,
+ instructions: "Speak naturally."
+ )
+
+ assert_equal(
+ {
+ type: :realtime,
+ output_modalities: [:audio],
+ instructions: "Speak naturally.",
+ audio: {
+ input: {
+ format: {type: :"audio/pcm", rate: 24_000},
+ noise_reduction: {type: :near_field},
+ turn_detection: {
+ type: :server_vad,
+ threshold: 0.5,
+ prefix_padding_ms: 300,
+ silence_duration_ms: 500,
+ create_response: true,
+ interrupt_response: true
+ }
+ },
+ output: {
+ format: {type: :"audio/pcm", rate: 24_000},
+ voice: :marin
+ }
+ }
+ },
+ connection.session.updates.fetch(0)
+ )
+ end
+
+ def test_realtime_conversation_streams_microphone_chunks_without_committing
+ outbound = RecordingOutbound.new
+ microphone = RecordingMicrophone.new(["first".b, "second".b])
+
+ OpenAI::Examples::Realtime::Conversation.forward_microphone(outbound, microphone)
+
+ assert_equal(["first".b, "second".b], outbound.chunks)
+ end
+
+ def test_realtime_conversation_serializes_audio_and_interruptions_through_one_writer
+ connection = RecordingConnection.new
+ outbound = OpenAI::Examples::Realtime::Conversation::OutboundWriter.new(connection)
+
+ Sync do |task|
+ writer = task.async { outbound.run }
+ outbound.append_audio("first".b)
+ outbound.truncate(item_id: "item_1", content_index: 0, audio_end_ms: 100)
+ outbound.append_audio("second".b)
+ outbound.close
+ writer.wait
+ end
+
+ assert_equal(["first".b, "second".b], connection.input_audio_buffer.chunks)
+ assert_equal(
+ [{item_id: "item_1", content_index: 0, audio_end_ms: 100}],
+ connection.conversation.items.truncations
+ )
+ assert_equal(1, connection.writer_fibers.uniq.size)
+ end
+
+ def test_realtime_conversation_unblocks_a_producer_when_the_writer_fails
+ connection = RecordingConnection.new
+ connection.input_audio_buffer.append_error = RuntimeError.new("write failed")
+ outbound = OpenAI::Examples::Realtime::Conversation::OutboundWriter.new(connection)
+
+ error = Sync do |task|
+ writer = task.async do
+ outbound.run
+ nil
+ rescue StandardError => e
+ e
+ end
+ outbound.append_audio("first".b)
+ producer_error = assert_raises(RuntimeError) do
+ outbound.append_audio("second".b)
+ end
+ writer_error = writer.wait
+
+ assert_same(writer_error, producer_error)
+ producer_error
+ end
+
+ assert_equal("write failed", error.message)
+ end
+
+ def test_realtime_conversation_handles_initial_speech_before_assistant_audio
+ speaker = RecordingSpeaker.new
+ outbound = RecordingOutbound.new
+ playback = OpenAI::Examples::Realtime::Conversation::AudioPlayback.new(
+ speaker,
+ clock: -> { raise "clock must not be read without playback" }
+ )
+
+ playback.interrupt(outbound)
+
+ assert_empty(outbound.truncations)
+ assert_equal(1, speaker.interruptions)
+ end
+
+ def test_realtime_conversation_propagates_writer_failure_without_deadlock
+ connection = RecordingConnection.new(
+ [OpenAI::Realtime::SessionUpdatedEvent.new(event_id: "event_1", session: {})]
+ )
+ connection.input_audio_buffer.append_error = RuntimeError.new("write failed")
+ microphone = RecordingMicrophone.new(["first".b, "second".b])
+
+ error = Sync do
+ assert_raises(RuntimeError) do
+ OpenAI::Examples::Realtime::Conversation.run_session(
+ connection,
+ microphone: microphone,
+ speaker: RecordingSpeaker.new,
+ voice: :marin,
+ instructions: "Speak naturally.",
+ output: StringIO.new
+ )
+ end
+ end
+
+ assert_equal("write failed", error.message)
+ assert_predicate(microphone, :stopped)
+ end
+
+ def test_realtime_conversation_latches_stop_before_ffmpeg_capture_starts
+ connection = RecordingConnection.new(
+ [OpenAI::Realtime::SessionUpdatedEvent.new(event_id: "event_1", session: {})]
+ )
+ microphone = OpenAI::Examples::Realtime::Conversation::FFmpegMicrophone.new(
+ input_format: "test",
+ device: "test"
+ )
+ spawn = ->(*) { raise "microphone spawned after stop" }
+
+ result = Process.stub(:spawn, spawn) do
+ Sync do
+ OpenAI::Examples::Realtime::Conversation.run_session(
+ connection,
+ microphone: microphone,
+ speaker: RecordingSpeaker.new,
+ voice: :marin,
+ instructions: "Speak naturally.",
+ output: StringIO.new
+ )
+ end
+ end
+
+ assert_nil(result)
+ end
+
+ def test_realtime_conversation_rejects_eof_before_the_requested_stop_event
+ microphone = RecordingMicrophone.new([])
+ playback = OpenAI::Examples::Realtime::Conversation::AudioPlayback.new(
+ RecordingSpeaker.new
+ )
+
+ error = assert_raises(RuntimeError) do
+ OpenAI::Examples::Realtime::Conversation.stream_events(
+ RecordingConnection.new,
+ microphone: microphone,
+ outbound: RecordingOutbound.new,
+ playback: playback,
+ output: StringIO.new,
+ stop_after: "response.done"
+ )
+ end
+
+ assert_equal("Realtime connection closed before response.done", error.message)
+ assert_predicate(microphone, :stopped)
+ ensure
+ playback&.close
+ end
+
+ def test_realtime_conversation_streams_audio_and_interrupts_local_playback
+ speaker = RecordingSpeaker.new
+ times = [0.0, 0.1].each
+ playback = OpenAI::Examples::Realtime::Conversation::AudioPlayback.new(
+ speaker,
+ clock: -> { times.next }
+ )
+ output = StringIO.new
+ outbound = RecordingOutbound.new
+ audio = "\x00\x01".b * 2_400
+ event_fields = {
+ event_id: "event_1",
+ response_id: "response_1",
+ item_id: "item_1",
+ output_index: 0,
+ content_index: 0
+ }
+
+ OpenAI::Examples::Realtime::Conversation.handle_event(
+ OpenAI::Realtime::ResponseAudioDeltaEvent.new(**event_fields, delta: Base64.strict_encode64(audio)),
+ outbound: outbound,
+ playback: playback,
+ output: output
+ )
+ OpenAI::Examples::Realtime::Conversation.handle_event(
+ OpenAI::Realtime::ResponseAudioTranscriptDeltaEvent.new(**event_fields, delta: "Hello"),
+ outbound: outbound,
+ playback: playback,
+ output: output
+ )
+ OpenAI::Examples::Realtime::Conversation.handle_event(
+ OpenAI::Realtime::InputAudioBufferSpeechStartedEvent.new(
+ event_id: "event_2",
+ item_id: "item_2",
+ audio_start_ms: 100
+ ),
+ outbound: outbound,
+ playback: playback,
+ output: output
+ )
+
+ assert_equal(audio, speaker.audio)
+ assert_equal("Hello\n", output.string)
+ assert_equal(1, speaker.interruptions)
+ assert_equal(
+ [{item_id: "item_1", content_index: 0, audio_end_ms: 100}],
+ outbound.truncations
+ )
+ end
+
+ def test_realtime_conversation_drops_queued_audio_after_interruption
+ speaker = RecordingSpeaker.new
+ times = [0.0, 0.05].each
+ playback = OpenAI::Examples::Realtime::Conversation::AudioPlayback.new(
+ speaker,
+ clock: -> { times.next }
+ )
+ output = StringIO.new
+ outbound = RecordingOutbound.new
+ event_fields = {
+ event_id: "event_1",
+ response_id: "response_1",
+ item_id: "item_1",
+ output_index: 0,
+ content_index: 0
+ }
+ first_audio = "\x00\x01".b * 2_400
+ queued_audio = "\x02\x03".b * 2_400
+
+ playback.write(
+ OpenAI::Realtime::ResponseAudioDeltaEvent.new(
+ **event_fields,
+ delta: Base64.strict_encode64(first_audio)
+ )
+ )
+ OpenAI::Examples::Realtime::Conversation.handle_event(
+ OpenAI::Realtime::InputAudioBufferSpeechStartedEvent.new(
+ event_id: "event_2",
+ item_id: "item_2",
+ audio_start_ms: 50
+ ),
+ outbound: outbound,
+ playback: playback,
+ output: output
+ )
+ playback.write(
+ OpenAI::Realtime::ResponseAudioDeltaEvent.new(
+ **event_fields,
+ event_id: "event_3",
+ delta: Base64.strict_encode64(queued_audio)
+ )
+ )
+ OpenAI::Examples::Realtime::Conversation.handle_event(
+ OpenAI::Realtime::ResponseDoneEvent.new(
+ event_id: "event_4",
+ response: OpenAI::Realtime::RealtimeResponse.new(id: "response_1", status: :cancelled)
+ ),
+ outbound: outbound,
+ playback: playback,
+ output: output
+ )
+
+ assert_equal(first_audio, speaker.audio)
+ assert_equal("\n", output.string)
+ assert_equal(1, speaker.interruptions)
+ end
+
+ def test_realtime_conversation_does_not_truncate_fully_played_completed_audio
+ speaker = RecordingSpeaker.new
+ outbound = RecordingOutbound.new
+ times = [0.0, 0.05, 0.2].each
+ playback = OpenAI::Examples::Realtime::Conversation::AudioPlayback.new(
+ speaker,
+ clock: -> { times.next }
+ )
+ fields = {
+ event_id: "event_1",
+ response_id: "response_1",
+ item_id: "item_1",
+ output_index: 0,
+ content_index: 0
+ }
+ audio = "\x00\x01".b * 2_400
+
+ playback.write(
+ OpenAI::Realtime::ResponseAudioDeltaEvent.new(
+ **fields,
+ delta: Base64.strict_encode64(audio)
+ )
+ )
+ playback.finish("response_1")
+ playback.interrupt(outbound)
+
+ assert_empty(outbound.truncations)
+ assert_equal(1, speaker.interruptions)
+ end
+
+ def test_realtime_conversation_can_capture_output_audio_for_a_live_smoke_test
+ Tempfile.create(["realtime-conversation", ".pcm"]) do |file|
+ speaker = OpenAI::Examples::Realtime::Conversation::PCMFileSpeaker.new(file.path)
+
+ speaker.write("\x00\x01".b)
+ speaker.interrupt
+ speaker.write("\x02\x03".b)
+ speaker.close
+
+ assert_equal("\x00\x01\x02\x03".b, File.binread(file.path))
+ end
+ end
+
+ def test_realtime_conversation_uses_ffplays_input_channel_layout_option
+ speaker_class = Class.new(OpenAI::Examples::Realtime::Conversation::FFplaySpeaker) do
+ def playback_command = command
+ end
+ command = speaker_class.new.playback_command
+
+ assert_equal(["-ch_layout", "mono"], command.slice(command.index("-ch_layout"), 2))
+ refute_includes(command, "-ac")
+ end
+
+ def test_realtime_conversation_ffplay_command_accepts_raw_pcm
+ speaker_class = Class.new(OpenAI::Examples::Realtime::Conversation::FFplaySpeaker) do
+ def playback_command = command
+ end
+ command = speaker_class.new.playback_command
+ command.insert(command.index("-i"), "-autoexit")
+
+ _, stderr, status = Open3.capture3(
+ {"SDL_AUDIODRIVER" => "dummy"},
+ *command,
+ stdin_data: "\0".b * 4_800
+ )
+
+ assert_predicate(status, :success?, stderr)
+ assert_empty(stderr, "ffplay must not emit terminal control sequences")
+ rescue Errno::ENOENT
+ skip("ffplay is not installed")
+ end
+
+ def test_webrtc_conversation_serves_a_browser_peer_with_audio_processing
+ calls = RecordingWebRTCCalls.new
+ app = OpenAI::Examples::Realtime::WebRTCConversation::App.new(
+ client: RecordingClient.new(Data.define(:calls).new(calls))
+ )
+ response = HTTPResponse.new
+
+ app.handle(
+ HTTPRequest.new(
+ request_method: "GET",
+ path: "/",
+ body: "",
+ headers: {}
+ ),
+ response
+ )
+
+ assert_equal(200, response.status)
+ assert_equal("text/html; charset=utf-8", response.headers.fetch("Content-Type"))
+ assert_includes(response.body, "new RTCPeerConnection()")
+ assert_includes(response.body, "echoCancellation: true")
+ assert_includes(response.body, "noiseSuppression: true")
+ assert_includes(response.body, "autoGainControl: true")
+ refute_includes(response.body, "OPENAI_API_KEY")
+ end
+
+ def test_webrtc_conversation_negotiates_and_hangs_up_through_ruby
+ calls = RecordingWebRTCCalls.new
+ app = OpenAI::Examples::Realtime::WebRTCConversation::App.new(
+ client: RecordingClient.new(Data.define(:calls).new(calls)),
+ html: "test"
+ )
+ response = HTTPResponse.new
+
+ app.handle(
+ HTTPRequest.new(
+ request_method: "POST",
+ path: "/session",
+ body: "v=0\r\nt=0 0\r\n",
+ headers: {"content-type" => "application/sdp"}
+ ),
+ response
+ )
+
+ assert_equal(201, response.status)
+ assert_equal("answer-sdp", response.body)
+ assert_equal("rtc_example", response.headers.fetch("X-OpenAI-Call-ID"))
+ assert_equal("v=0\r\nt=0 0\r\n", calls.creates.fetch(0).fetch(:sdp))
+ session = calls.creates.fetch(0).fetch(:session)
+ assert_equal("gpt-realtime-2.1", session.fetch(:model))
+ assert_equal(true, session.dig(:audio, :input, :turn_detection, :interrupt_response))
+
+ hangup_response = HTTPResponse.new
+ app.handle(
+ HTTPRequest.new(
+ request_method: "POST",
+ path: "/hangup",
+ body: "rtc_example",
+ headers: {}
+ ),
+ hangup_response
+ )
+
+ assert_equal(204, hangup_response.status)
+ assert_equal(["rtc_example"], calls.hangups)
+ end
+
+ def test_webrtc_conversation_rejects_non_sdp_requests
+ calls = RecordingWebRTCCalls.new
+ app = OpenAI::Examples::Realtime::WebRTCConversation::App.new(
+ client: RecordingClient.new(Data.define(:calls).new(calls)),
+ html: "test"
+ )
+ response = HTTPResponse.new
+
+ app.handle(
+ HTTPRequest.new(
+ request_method: "POST",
+ path: "/session",
+ body: "not sdp",
+ headers: {"content-type" => "text/plain"}
+ ),
+ response
+ )
+
+ assert_equal(400, response.status)
+ assert_empty(calls.creates)
+ end
+
+ def test_webrtc_conversation_treats_an_already_closed_call_as_hung_up
+ calls = RecordingWebRTCCalls.new
+ calls.hangup_error = OpenAI::Errors::NotFoundError.new(
+ url: URI("https://api.openai.com/v1/realtime/calls/rtc_example/hangup"),
+ status: 404,
+ headers: {},
+ body: {},
+ request: nil,
+ response: nil
+ )
+ app = OpenAI::Examples::Realtime::WebRTCConversation::App.new(
+ client: RecordingClient.new(Data.define(:calls).new(calls)),
+ html: "test"
+ )
+ app.handle(
+ HTTPRequest.new(
+ request_method: "POST",
+ path: "/session",
+ body: "v=0\r\nt=0 0\r\n",
+ headers: {"content-type" => "application/sdp"}
+ ),
+ HTTPResponse.new
+ )
+ response = HTTPResponse.new
+
+ app.handle(
+ HTTPRequest.new(
+ request_method: "POST",
+ path: "/hangup",
+ body: "rtc_example",
+ headers: {}
+ ),
+ response
+ )
+
+ assert_equal(204, response.status)
+ assert_equal(["rtc_example"], calls.hangups)
+ end
+
+ def test_mcp_console_example_requests_text_output
+ connection = RecordingConnection.new
+
+ OpenAI::Examples::Realtime::MCPApproval.configure(
+ connection,
+ server_url: "https://developers.openai.com/mcp"
+ )
+
+ session = connection.session.updates.fetch(0)
+ assert_equal([:text], session.fetch(:output_modalities))
+ refute(session.key?(:tool_choice))
+ end
+
+ def test_mcp_console_example_requires_the_discovered_tool
+ tools_item = OpenAI::Realtime::RealtimeMcpListTools.new(
+ id: "mcp_tools_1",
+ server_label: "remote",
+ tools: [
+ OpenAI::Realtime::RealtimeMcpListTools::Tool.new(
+ name: "search_openai_docs",
+ description: "Search the OpenAI developer documentation",
+ input_schema: {type: "object"}
+ )
+ ]
+ )
+ connection = RecordingConnection.new(
+ [
+ OpenAI::Realtime::ConversationItemDone.new(
+ event_id: "event_1",
+ item: tools_item
+ ),
+ OpenAI::Realtime::McpListToolsCompleted.new(
+ event_id: "event_2",
+ item_id: "mcp_tools_1"
+ ),
+ OpenAI::Realtime::SessionUpdatedEvent.new(
+ event_id: "event_3",
+ session: {}
+ ),
+ completed_response_event
+ ]
+ )
+
+ OpenAI::Examples::Realtime::MCPApproval.run_session(
+ connection,
+ prompt: "Search the docs",
+ output: StringIO.new
+ )
+
+ assert_equal(
+ {
+ type: :realtime,
+ tools: [{type: :mcp, server_label: "remote"}],
+ tool_choice: {type: :mcp, server_label: "remote", name: "search_openai_docs"}
+ },
+ connection.session.updates.fetch(0)
+ )
+ assert_equal({}, connection.response.calls.fetch(0))
+ end
+
+ def test_mcp_console_example_waits_for_the_tool_choice_update
+ tools_item = OpenAI::Realtime::RealtimeMcpListTools.new(
+ id: "mcp_tools_1",
+ server_label: "remote",
+ tools: [
+ OpenAI::Realtime::RealtimeMcpListTools::Tool.new(
+ name: "search_openai_docs",
+ input_schema: {type: "object"}
+ )
+ ]
+ )
+ connection = RecordingConnection.new(
+ [
+ OpenAI::Realtime::ConversationItemDone.new(event_id: "event_1", item: tools_item),
+ OpenAI::Realtime::McpListToolsCompleted.new(event_id: "event_2", item_id: "mcp_tools_1")
+ ]
+ )
+
+ error = assert_raises(RuntimeError) do
+ OpenAI::Examples::Realtime::MCPApproval.run_session(
+ connection,
+ prompt: "Search the docs",
+ output: StringIO.new
+ )
+ end
+
+ assert_equal("Realtime connection closed before the final response.done", error.message)
+ assert_empty(connection.response.calls)
+ assert_empty(connection.conversation.items.calls)
+ end
+
+ def test_mcp_console_example_requests_a_final_answer_after_the_tool_completes
+ connection = RecordingConnection.new(
+ [
+ OpenAI::Realtime::ResponseMcpCallCompleted.new(
+ event_id: "event_1",
+ item_id: "mcp_call_1",
+ output_index: 0
+ ),
+ completed_response_event
+ ]
+ )
+
+ OpenAI::Examples::Realtime::MCPApproval.run_session(
+ connection,
+ prompt: "Search the docs",
+ output: StringIO.new
+ )
+
+ assert_equal({tool_choice: :none}, connection.response.calls.fetch(0))
+ end
+
+ def test_sideband_smoke_mode_stops_after_the_selected_event
+ events = [
+ Event.new(type: :"session.updated", data: {type: :"session.updated"}),
+ Event.new(type: :"response.done", data: {type: :"response.done"})
+ ]
+ connection = RecordingConnection.new(events)
+ output = StringIO.new
+
+ OpenAI::Examples::Realtime::Sideband.stream(
+ connection,
+ output: output,
+ stop_after: "session.updated"
+ )
+
+ assert_includes(output.string, "session.updated")
+ refute_includes(output.string, "response.done")
+ end
+
+ def test_sip_example_accepts_then_attaches_to_the_call
+ connection = RecordingConnection.new(
+ [Event.new(type: :"session.created", data: {type: :"session.created"})]
+ )
+ realtime = RecordingRealtime.new(connection)
+
+ OpenAI::Examples::Realtime::SIP.run(
+ client: RecordingClient.new(realtime: realtime),
+ call_id: "rtc_123",
+ model: "gpt-realtime-2.1",
+ output: StringIO.new,
+ stop_after: "session.created"
+ )
+
+ assert_equal(["rtc_123"], realtime.connections)
+ assert_equal(
+ [
+ "rtc_123",
+ {
+ type: :realtime,
+ model: "gpt-realtime-2.1",
+ instructions: "You are answering a phone call. Be warm and concise."
+ }
+ ],
+ realtime.calls.accepts.fetch(0)
+ )
+ assert_equal(["rtc_123"], realtime.calls.hangups)
+ end
+
+ def test_sip_example_hangs_up_without_masking_a_sideband_error
+ realtime = RecordingRealtime.new(ExplodingConnection.new)
+ realtime.calls.hangup_error = RuntimeError.new("hangup failed")
+
+ error = assert_raises(RuntimeError) do
+ OpenAI::Examples::Realtime::SIP.run(
+ client: RecordingClient.new(realtime: realtime),
+ call_id: "rtc_123",
+ model: "gpt-realtime-2.1",
+ output: StringIO.new
+ )
+ end
+
+ assert_equal("sideband failed", error.message)
+ assert_equal(["rtc_123"], realtime.calls.hangups)
+ end
+
+ def test_sip_example_tolerates_an_already_ended_call_during_cleanup
+ realtime = RecordingRealtime.new(RecordingConnection.new)
+ realtime.calls.hangup_error = OpenAI::Errors::NotFoundError.new(
+ url: URI("https://api.openai.com/v1/realtime/calls/rtc_123/hangup"),
+ status: 404,
+ headers: {},
+ body: {},
+ request: nil,
+ response: nil
+ )
+
+ OpenAI::Examples::Realtime::SIP.run(
+ client: RecordingClient.new(realtime: realtime),
+ call_id: "rtc_123",
+ model: "gpt-realtime-2.1",
+ output: StringIO.new
+ )
+
+ assert_equal(["rtc_123"], realtime.calls.hangups)
+ end
+
+ def test_sip_example_reports_a_cleanup_failure_after_a_clean_session
+ realtime = RecordingRealtime.new(RecordingConnection.new)
+ realtime.calls.hangup_error = RuntimeError.new("hangup failed")
+
+ error = assert_raises(RuntimeError) do
+ OpenAI::Examples::Realtime::SIP.run(
+ client: RecordingClient.new(realtime: realtime),
+ call_id: "rtc_123",
+ model: "gpt-realtime-2.1",
+ output: StringIO.new
+ )
+ end
+
+ assert_equal("hangup failed", error.message)
+ assert_equal(["rtc_123"], realtime.calls.hangups)
+ end
+
+ def test_websocket_audio_disables_vad_and_waits_for_session_updated
+ audio = "assistant pcm".b
+ connection = RecordingConnection.new(
+ [
+ OpenAI::Realtime::SessionUpdatedEvent.new(event_id: "event_1", session: {}),
+ OpenAI::Realtime::ResponseAudioDeltaEvent.new(
+ event_id: "event_2",
+ response_id: "response_1",
+ item_id: "item_1",
+ output_index: 0,
+ content_index: 0,
+ delta: Base64.strict_encode64(audio)
+ ),
+ completed_response_event
+ ]
+ )
+ realtime = RecordingRealtime.new(connection)
+
+ Tempfile.create("realtime-input") do |input|
+ input.write("input pcm")
+ input.flush
+ Tempfile.create("realtime-output") do |output|
+ _stdout, _stderr = capture_io do
+ OpenAI::Examples::Realtime::WebSocketAudio.run(
+ client: RecordingClient.new(realtime: realtime),
+ model: "gpt-realtime-2.1",
+ input_path: input.path,
+ output_path: output.path
+ )
+ end
+
+ assert_equal(audio, File.binread(output.path))
+ end
+ end
+
+ assert_equal(
+ {
+ type: :realtime,
+ output_modalities: [:audio],
+ audio: {input: {turn_detection: nil}}
+ },
+ connection.session.updates.fetch(0)
+ )
+ assert_equal(["input pcm"], connection.input_audio_buffer.chunks)
+ assert_equal(1, connection.input_audio_buffer.commits)
+ assert_equal([{}], connection.response.calls)
+ assert_equal([:"session.updated"], connection.received)
+ end
+
+ private def completed_response_event
+ OpenAI::Realtime::ResponseDoneEvent.new(
+ event_id: "event_done",
+ response: OpenAI::Realtime::RealtimeResponse.new(
+ id: "response_done",
+ status: :completed
+ )
+ )
+ end
+end
diff --git a/test/openai/resources/realtime/calls_test.rb b/test/openai/resources/realtime/calls_test.rb
index 938be40ed..01b40ec12 100644
--- a/test/openai/resources/realtime/calls_test.rb
+++ b/test/openai/resources/realtime/calls_test.rb
@@ -3,6 +3,125 @@
require_relative "../../test_helper"
class OpenAI::Test::Resources::Realtime::CallsTest < OpenAI::Test::ResourceTest
+ class CapturingLogger
+ attr_reader :events
+
+ def initialize = @events = []
+
+ def debug(message) = @events << [:debug, message]
+ def info(message) = @events << [:info, message]
+ def warn(message) = @events << [:warn, message]
+ def error(message) = @events << [:error, message]
+ end
+
+ class StubHTTPClient < OpenAI::HTTPClient
+ attr_reader :request
+
+ def execute(request)
+ @request = request
+ OpenAI::HTTPClient::Response.new(
+ status: 201,
+ headers: {
+ "Content-Type" => "application/sdp",
+ "Location" => "/v1/realtime/calls/rtc_123",
+ "X-Request-ID" => "req_realtime_call"
+ },
+ body: "answer-sdp"
+ )
+ end
+ end
+
+ class MalformedLocationHTTPClient < OpenAI::HTTPClient
+ def execute(_request)
+ OpenAI::HTTPClient::Response.new(
+ status: 201,
+ headers: {
+ "Content-Type" => "application/sdp",
+ "Location" => "http://[malformed",
+ "X-Request-ID" => "req_malformed_location"
+ },
+ body: "usable-answer-sdp"
+ )
+ end
+ end
+
+ def test_create_with_an_sdp_offer
+ http_client = StubHTTPClient.new
+ client = OpenAI::Client.new(
+ api_key: "test-key",
+ base_url: "https://example.com/v1",
+ http_client: http_client
+ )
+
+ response = client.realtime.calls.create(sdp: "offer-sdp")
+
+ assert_equal("answer-sdp", response.sdp)
+ assert_equal("rtc_123", response.call_id)
+ assert_equal("req_realtime_call", response._request_id)
+ assert_equal(201, response.last_response.status)
+ assert_equal("/v1/realtime/calls/rtc_123", response.headers.fetch("location"))
+ assert_equal(:post, http_client.request.method)
+ assert_equal("https://example.com/v1/realtime/calls", http_client.request.url.to_s)
+ assert_equal("application/sdp", http_client.request.headers.fetch("content-type"))
+ assert_equal("offer-sdp", http_client.request.body)
+ end
+
+ def test_create_with_session_configuration_uses_typed_multipart_parts
+ http_client = StubHTTPClient.new
+ client = OpenAI::Client.new(
+ api_key: "test-key",
+ base_url: "https://example.com/v1",
+ http_client: http_client
+ )
+
+ client.realtime.calls.create(
+ sdp: "offer-sdp",
+ session: {type: :realtime, model: "gpt-realtime"}
+ )
+
+ content_type = http_client.request.headers.fetch("content-type")
+ body = http_client.request.body.to_a.join
+
+ assert_match(%r{\Amultipart/form-data; boundary=}, content_type)
+ assert_includes(body, "name=\"sdp\"\r\nContent-Type: application/sdp\r\n\r\noffer-sdp")
+ assert_includes(body, "name=\"session\"\r\nContent-Type: application/json")
+ assert_includes(body, JSON.generate(type: :realtime, model: "gpt-realtime"))
+ refute_includes(body, "filename=")
+ end
+
+ def test_create_preserves_request_observability_for_raw_sdp_responses
+ logger = CapturingLogger.new
+ client = OpenAI::Client.new(
+ api_key: "test-key",
+ base_url: "https://example.com/v1",
+ http_client: StubHTTPClient.new,
+ logger: logger,
+ log_level: :debug
+ )
+
+ response = client.realtime.calls.create(sdp: "offer-sdp")
+
+ assert_equal("answer-sdp", response.sdp)
+ log = logger.events.map(&:last).join("\n")
+ assert_includes(log, "request complete")
+ assert_includes(log, "request_id=req_realtime_call")
+ assert_includes(log, "response body")
+ end
+
+ def test_create_ignores_a_malformed_optional_location_header
+ client = OpenAI::Client.new(
+ api_key: "test-key",
+ base_url: "https://example.com/v1",
+ http_client: MalformedLocationHTTPClient.new
+ )
+
+ response = client.realtime.calls.create(sdp: "offer-sdp")
+
+ assert_equal("usable-answer-sdp", response.sdp)
+ assert_nil(response.call_id)
+ assert_equal("req_malformed_location", response._request_id)
+ end
+
def test_accept_required_params
response = @openai.realtime.calls.accept("call_id", type: :realtime)
diff --git a/test/openai/resources/realtime/translations_test.rb b/test/openai/resources/realtime/translations_test.rb
new file mode 100644
index 000000000..56399536d
--- /dev/null
+++ b/test/openai/resources/realtime/translations_test.rb
@@ -0,0 +1,184 @@
+# frozen_string_literal: true
+
+require_relative "../../test_helper"
+
+class OpenAI::Test::Resources::Realtime::TranslationsTest < Minitest::Test
+ class StubHTTPClient < OpenAI::HTTPClient
+ attr_reader :requests
+
+ def initialize(*responses)
+ super()
+ @responses = responses
+ @requests = []
+ end
+
+ def execute(request)
+ @requests << request
+ @responses.shift
+ end
+ end
+
+ class FakeSocket
+ attr_reader :writes
+
+ def initialize(*reads)
+ @reads = reads
+ @writes = []
+ @closed = false
+ end
+
+ def read = @reads.shift
+ def write(message) = @writes << message
+ def closed? = @closed
+ def close(code: 1000, reason: "") = (@closed = [code, reason].any?)
+ end
+
+ class FakeTransport
+ attr_reader :url
+
+ def initialize(socket)
+ @socket = socket
+ end
+
+ def open(url:, **_options)
+ @url = url
+ yield(@socket)
+ end
+ end
+
+ def test_create_client_secret
+ http_client = StubHTTPClient.new(
+ OpenAI::HTTPClient::Response.new(
+ status: 200,
+ headers: {"content-type" => "application/json"},
+ body: JSON.generate(
+ value: "ek_test",
+ expires_at: 123,
+ session: {
+ id: "sess_123",
+ type: "translation",
+ model: "gpt-realtime-translate",
+ expires_at: 456,
+ audio: {}
+ }
+ )
+ )
+ )
+ client = OpenAI::Client.new(api_key: "test-key", http_client: http_client)
+
+ response = client.realtime.translations.client_secrets.create(
+ session: {
+ model: "gpt-realtime-translate",
+ audio: {output: {language: "es"}}
+ }
+ )
+
+ assert_instance_of(OpenAI::Realtime::RealtimeTranslationClientSecretCreateResponse, response)
+ assert_equal("ek_test", response.value)
+ request = http_client.requests.fetch(0)
+ assert_equal("/v1/realtime/translations/client_secrets", request.url.path)
+ assert_equal(
+ {
+ session: {
+ model: "gpt-realtime-translate",
+ audio: {output: {language: "es"}}
+ }
+ },
+ JSON.parse(request.body, symbolize_names: true)
+ )
+ end
+
+ def test_create_client_secret_rejects_an_invalid_session_before_http
+ http_client = StubHTTPClient.new
+ client = OpenAI::Client.new(api_key: "test-key", http_client: http_client)
+
+ error = assert_raises(ArgumentError) do
+ client.realtime.translations.client_secrets.create(session: {audio: {}})
+ end
+
+ assert_includes(error.message, "missing required fields")
+ assert_empty(http_client.requests)
+ end
+
+ def test_create_translation_call
+ http_client = StubHTTPClient.new(
+ OpenAI::HTTPClient::Response.new(
+ status: 201,
+ headers: {
+ "content-type" => "application/sdp",
+ "location" => "/v1/realtime/calls/rtc_translation",
+ "x-request-id" => "req_translation_call"
+ },
+ body: "answer-sdp"
+ )
+ )
+ client = OpenAI::Client.new(api_key: "ek_test", http_client: http_client)
+
+ response = client.realtime.translations.calls.create(sdp: "offer-sdp")
+
+ assert_equal("answer-sdp", response.sdp)
+ assert_equal("rtc_translation", response.call_id)
+ assert_equal("req_translation_call", response._request_id)
+ assert_equal(201, response.last_response.status)
+ request = http_client.requests.fetch(0)
+ assert_equal("/v1/realtime/calls", request.url.path)
+ assert_equal("application/sdp", request.headers.fetch("content-type"))
+ assert_equal("offer-sdp", request.body)
+ end
+
+ def test_connect_translation_session_with_typed_events_and_helpers
+ socket = FakeSocket.new(
+ JSON.generate(
+ type: "session.output_transcript.delta",
+ event_id: "event_1",
+ delta: "hola",
+ item_id: "item_1"
+ )
+ )
+ transport = FakeTransport.new(socket)
+ client = OpenAI::Client.new(api_key: "test-key")
+ event = nil
+
+ client.realtime.translations.connect(
+ model: "gpt-realtime-translate",
+ transport: transport
+ ) do |connection|
+ assert_instance_of(OpenAI::Realtime::TranslationConnection, connection)
+ refute_respond_to(connection, :response)
+ refute_respond_to(connection, :conversation)
+ refute_respond_to(connection, :output_audio_buffer)
+ assert_respond_to(connection.session, :close)
+ refute_respond_to(connection.input_audio_buffer, :commit)
+ refute_respond_to(connection.input_audio_buffer, :clear)
+ connection.session.update(audio: {output: {language: "es"}})
+ connection.input_audio_buffer.append_bytes("\x00\x01".b)
+ connection.session.close
+ event = connection.receive
+ end
+
+ assert_equal(
+ "wss://api.openai.com/v1/realtime/translations?model=gpt-realtime-translate",
+ transport.url.to_s
+ )
+ assert_instance_of(OpenAI::Realtime::RealtimeTranslationOutputTranscriptDeltaEvent, event)
+ assert_equal("hola", event.delta)
+ assert_equal(
+ [
+ "session.update",
+ "session.input_audio_buffer.append",
+ "session.close"
+ ],
+ socket.writes.map { JSON.parse(_1).fetch("type") }
+ )
+ end
+
+ def test_connect_requires_a_block
+ client = OpenAI::Client.new(api_key: "test-key")
+
+ error = assert_raises(ArgumentError) do
+ client.realtime.translations.connect(model: "gpt-realtime-translate")
+ end
+
+ assert_includes(error.message, "block is required")
+ end
+end