diff --git a/Gemfile b/Gemfile index cfb2366a1..318265cde 100644 --- a/Gemfile +++ b/Gemfile @@ -17,6 +17,7 @@ end group :development, :test do gem "async" + gem "async-websocket" gem "aws-sdk-core", "~> 3" gem "minitest", "~> 5.27" gem "minitest-focus" diff --git a/Gemfile.lock b/Gemfile.lock index d206ab023..497722853 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -42,6 +42,24 @@ GEM io-event (~> 1.11) metrics (~> 0.12) traces (~> 0.18) + async-http (0.95.1) + async (>= 2.10.2) + async-pool (~> 0.11) + io-endpoint (~> 0.14) + io-stream (~> 0.6) + metrics (~> 0.12) + protocol-http (~> 0.62) + protocol-http1 (~> 0.39) + protocol-http2 (~> 0.26) + protocol-url (~> 0.2) + traces (~> 0.10) + async-pool (0.11.2) + async (>= 2.0) + async-websocket (0.30.1) + async-http (~> 0.76) + protocol-http (~> 0.34) + protocol-rack (~> 0.7) + protocol-websocket (~> 0.17) aws-eventstream (1.4.0) aws-partitions (1.1261.0) aws-sdk-core (3.252.0) @@ -85,7 +103,10 @@ GEM hashdiff (1.2.1) i18n (1.15.2) concurrent-ruby (~> 1.0) + io-endpoint (0.17.2) io-event (1.11.2) + io-stream (0.14.0) + openssl (>= 3.3) jmespath (1.6.2) json (2.21.2) language_server-protocol (3.17.0.5) @@ -106,14 +127,30 @@ GEM minitest (~> 5.0) mutex_m (0.3.0) netrc (0.11.0) + openssl (4.0.2) parallel (1.27.0) parser (3.3.10.0) ast (~> 2.4.1) racc prettier_print (1.2.1) prism (1.6.0) + protocol-hpack (1.5.1) + protocol-http (0.70.0) + protocol-http1 (0.40.2) + protocol-http (~> 0.68) + protocol-http2 (0.26.2) + protocol-hpack (~> 1.4) + protocol-http (~> 0.62) + protocol-rack (0.22.1) + io-stream (>= 0.10) + protocol-http (~> 0.58) + rack (>= 1.0) + protocol-url (0.11.0) + protocol-websocket (0.21.1) + protocol-http (~> 0.2) public_suffix (7.0.5) racc (1.8.1) + rack (3.2.6) rainbow (3.1.1) rake (13.3.1) rb-fsevent (0.11.2) @@ -203,7 +240,7 @@ GEM addressable (>= 2.8.0) crack (>= 0.3.2) hashdiff (>= 0.4.0, < 2.0.0) - webrick (1.9.1) + webrick (1.9.2) yard (0.9.44) yard-sorbet (0.9.0) sorbet-runtime @@ -221,6 +258,7 @@ PLATFORMS DEPENDENCIES async + async-websocket aws-sdk-core (~> 3) minitest (~> 5.27) minitest-focus diff --git a/README.md b/README.md index 21bf0691b..5e1ea3f82 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,12 @@ stream.each do |event| end ``` +### Realtime + +Server-side WebSockets, WebRTC negotiation, sideband, SIP, transcription, and +translation are covered in the [Realtime guide](realtime.md) and runnable +[example runbook](examples/realtime/README.md). + ### Pagination List methods in the OpenAI API are paginated. diff --git a/examples/realtime/README.md b/examples/realtime/README.md new file mode 100644 index 000000000..40f6ea6a6 --- /dev/null +++ b/examples/realtime/README.md @@ -0,0 +1,229 @@ +# Realtime examples + +These examples run against the live OpenAI service with `OPENAI_API_KEY` set. +They default to `gpt-realtime-2.1`; override `OPENAI_REALTIME_MODEL` when needed. + +## Browser voice conversation (recommended) + +For natural hands-free conversation, use the WebRTC example. Ruby keeps the +standard API key private, accepts the browser's SDP offer, configures the +Realtime call, and returns the SDP answer. The browser owns the media peer and +requests acoustic echo cancellation, noise suppression, and automatic gain +control. + +```sh +bundle exec ruby examples/realtime/webrtc_conversation.rb +``` + +Open `http://127.0.0.1:4567`, click **Start conversation**, and allow microphone +access. Talk normally and speak over the model to interrupt it. Click **Stop** +to close the peer and ask Ruby to hang up the call. No API key is sent to the +browser. Override the port with `REALTIME_DEMO_PORT`. + +## WebSocket microphone loop (advanced) + +`realtime_conversation.rb` keeps one full-duplex WebSocket session open. It +streams 100 ms microphone chunks continuously, uses server VAD to detect turn +boundaries and create responses automatically, and plays output audio as deltas +arrive. There is no push-to-talk control. + +This low-level path has no acoustic echo cancellation. It is useful for server +audio pipelines and protocol debugging, but it is not a laptop speakerphone. +Install FFmpeg (which also supplies `ffplay`) and **use headphones** so the +microphone does not capture the model's voice: + +```sh +brew install ffmpeg +bundle exec ruby examples/realtime/realtime_conversation.rb +``` + +Start speaking after `Connected` appears. Speak again whenever you want to +interrupt: the service cancels the response and the example immediately clears +local playback, drops already-queued audio deltas, and truncates the unheard +audio from conversation history. A bounded single-writer queue serializes the +microphone and truncation events sent over the socket. Press Control-C to end +the session. If that writer fails, the example stops microphone capture, +unblocks any queued producer, and reports the original connection error. + +On macOS, the default microphone is AVFoundation audio device `0`. List devices +and select a different index if necessary: + +```sh +ffmpeg -f avfoundation -list_devices true -i "" + +REALTIME_MIC_DEVICE=:1 \ +bundle exec ruby examples/realtime/realtime_conversation.rb +``` + +macOS may ask for microphone permission for your terminal on the first run. +Customize the voice or spoken behavior with `OPENAI_REALTIME_VOICE` and +`OPENAI_REALTIME_INSTRUCTIONS`. The example defaults to the recommended `marin` +voice. Linux defaults to PulseAudio's `default` device; override +`REALTIME_AUDIO_INPUT_FORMAT` and `REALTIME_MIC_DEVICE` for another FFmpeg input. + +For a deterministic live smoke test, provide paced PCM input and capture output +instead of opening audio devices: + +```sh +REALTIME_INPUT_PCM=input.pcm \ +REALTIME_OUTPUT_PCM=response.pcm \ +OPENAI_REALTIME_TIMEOUT=30 \ +bundle exec ruby examples/realtime/realtime_conversation.rb +``` + +This bounded mode exits successfully only after the requested `response.done`; +a clean EOF before that event fails the smoke test. + +## WebSocket text + +```sh +bundle exec ruby examples/realtime/websocket_text.rb +``` + +A successful run prints `session.created`, `session.updated`, streamed assistant +text, and `response.done status=completed`. An EOF before the completed +`response.done`, or a completed response without a non-empty text delta, is +reported as a failed smoke test. + +## WebSocket audio + +The input must be raw, headerless, mono 24 kHz PCM16 little-endian audio. For +example, convert a WAV file with: + +```sh +ffmpeg -i input.wav -f s16le -acodec pcm_s16le -ac 1 -ar 24000 input.pcm +``` + +Then run the example and play its output: + +```sh +REALTIME_INPUT_PCM=input.pcm \ +REALTIME_OUTPUT_PCM=response.pcm \ +bundle exec ruby examples/realtime/websocket_audio.rb + +ffplay -f s16le -ar 24000 -ch_layout mono response.pcm +``` + +A successful run exits with status 0 only after writing audio bytes and seeing a +completed `response.done`. EOF before completion, or completion without an +audio delta, is reported as a failed smoke test. The example disables automatic +turn detection and waits for `session.updated` before uploading, so its explicit +buffer commit and `response.create` cannot race server VAD. + +## Realtime transcription + +Use the same raw mono 24 kHz PCM16 input format as the audio example: + +```sh +REALTIME_INPUT_PCM=input.pcm \ +OPENAI_REALTIME_TRANSCRIPTION_MODEL=gpt-live-transcribe \ +bundle exec ruby examples/realtime/websocket_transcription.rb +``` + +A successful run streams transcript deltas, then prints the completed +transcript with its `item_id`. Completion events for different input turns may +arrive out of order, so applications should correlate them by `item_id`. +The example opens a dedicated connection with `connect_transcription`. +Configure its transcription model with `OPENAI_REALTIME_TRANSCRIPTION_MODEL`. +An EOF before the completed transcription event is reported as a failed smoke +test. + +## Realtime translation + +Translation uses the same raw PCM input format and reads translated transcript +and audio while input is still being uploaded: + +```sh +REALTIME_INPUT_PCM=input.pcm \ +REALTIME_OUTPUT_PCM=translation.pcm \ +TARGET_LANGUAGE=es \ +bundle exec ruby examples/realtime/translation.rb +``` + +A successful run prints the translated transcript, exits after +`session.closed`, and writes non-empty translated PCM audio. An EOF before +`session.closed`, or a closed session with no translated audio, is reported as +incomplete instead of silently succeeding. If upload and graceful close both +fail, the upload error remains the primary failure so the interrupted input +operation is not hidden by cleanup. +If the reader fails while input is still being uploaded, the example cancels +the uploader immediately instead of draining the rest of the file. If the +reader has already failed on buffered input or EOF, upload never starts. + +## MCP approval + +Point the example at a Streamable HTTP MCP server. This invocation uses the +public OpenAI developer-documentation server: + +```sh +MCP_SERVER_URL=https://developers.openai.com/mcp \ +OPENAI_REALTIME_PROMPT='Search the OpenAI docs for the Realtime WebSocket URL.' \ +bundle exec ruby examples/realtime/mcp_approval.rb +``` + +The example waits for tool import, selects an advertised tool, waits for the +tool-choice update to be acknowledged, approves the request, waits for the tool +to finish, and asks the model for the final answer. A successful run prints all +five checkpoints and reaches the final completed `response.done`; EOF before +that event is a failed smoke test. Set `OPENAI_REALTIME_DEBUG=1` to print every +event type. + +## WebRTC call creation + +`webrtc_call.rb` is the server endpoint core: send a browser-generated SDP offer +on stdin, return stdout as `application/sdp`, and retain the call ID printed on +stderr. + +```sh +bundle exec ruby examples/realtime/webrtc_call.rb < offer.sdp > answer.sdp +``` + +The recommended `webrtc_conversation.rb` example above exercises this exchange +end to end with a real `RTCPeerConnection`. `webrtc_call.rb` remains a minimal +stdin/stdout building block for integrating the same SDK call into another web +framework. + +## Sideband control + +Use the call ID from WebRTC call creation or a verified incoming SIP webhook: + +```sh +OPENAI_REALTIME_CALL_ID=rtc_... \ +bundle exec ruby examples/realtime/sideband.rb +``` + +For a bounded smoke test, set `OPENAI_REALTIME_STOP_AFTER=session.updated` and +`OPENAI_REALTIME_TIMEOUT=30`. A real paired test keeps the browser or SIP peer +connected while this process attaches to the same call. EOF before the selected +event fails the bounded smoke test. + +## SIP + +The SIP example requires an OpenAI project with SIP configured and a real +incoming call. Obtain `OPENAI_REALTIME_CALL_ID` from a verified +`realtime.call.incoming` webhook, then run: + +```sh +OPENAI_REALTIME_CALL_ID=rtc_... \ +bundle exec ruby examples/realtime/sip.rb +``` + +The script accepts the call, attaches a sideband WebSocket, and prints the audio +transcript. `OPENAI_REALTIME_STOP_AFTER` and `OPENAI_REALTIME_TIMEOUT` provide +bounded smoke-test controls. Once accepted, the call is hung up during cleanup, +including after a timeout or sideband failure; an already-ended call is treated +as successfully cleaned up. EOF before the selected event fails a bounded run. +Without a carrier-originated incoming call, the same accept-then-attach +orchestration is covered by the local example and HTTP resource tests, but that +is not a substitute for the final telephony smoke test. + +## Local protocol coverage + +The test suite exercises the examples' orchestration and the documented local +WebSocket lifecycle, including text, transcription, full-duplex translation, +MCP approval, fragmented frames, normal and abnormal closure, sideband control, +and SIP accept-then-attach behavior: + +```sh +./scripts/test +``` diff --git a/examples/realtime/event_stream.rb b/examples/realtime/event_stream.rb new file mode 100644 index 000000000..4f461fd84 --- /dev/null +++ b/examples/realtime/event_stream.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module OpenAI + module Examples + module Realtime + module EventStream + module_function + + def each_until(connection, stop_after: nil, closed_message: nil) + reached_target = false + connection.each do |event| + yield(event) + reached_target = stop_after == event.type.to_s + break if reached_target + end + return if stop_after.nil? || reached_target + + raise(closed_message || "Realtime connection closed before #{stop_after}") + end + + def wait_for(connection, event_class, closed_message:) + while (event = connection.receive) + raise event.error.message if event.is_a?(OpenAI::Realtime::RealtimeErrorEvent) + return event if event.is_a?(event_class) + end + + raise closed_message + end + end + end + end +end diff --git a/examples/realtime/mcp_approval.rb b/examples/realtime/mcp_approval.rb new file mode 100755 index 000000000..3ee0cb4bd --- /dev/null +++ b/examples/realtime/mcp_approval.rb @@ -0,0 +1,175 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require_relative "../../lib/openai" +require "timeout" + +module OpenAI + module Examples + module Realtime + module MCPApproval + module_function + + def configure(connection, server_url:) + connection.session.update( + type: :realtime, + output_modalities: [:text], + tools: [ + { + type: :mcp, + server_label: "remote", + server_url: server_url, + require_approval: :always + } + ] + ) + end + + def select_tool(connection, server_label:, tool_name:) + connection.session.update( + type: :realtime, + tools: [{type: :mcp, server_label: server_label}], + tool_choice: {type: :mcp, server_label: server_label, name: tool_name} + ) + end + + def send_prompt(connection, prompt:) + connection.conversation.items.create( + type: :message, + role: :user, + content: [{type: :input_text, text: prompt}] + ) + connection.response.create + end + + def approve_request(connection, item) + connection.conversation.items.respond_to_mcp_approval( + approval_request_id: item.id, + approve: true, + reason: "Approved by the application policy" + ) + end + + def run_session(connection, prompt:, output: $stdout, debug: false) + completed = false + state = { + completed_item_ids: {}, + tool_lists: {}, + waiting_for_tool_update: false, + selected_tool: false, + mcp_call_pending: false + } + + connection.each do |event| + output.puts("[mcp event] #{event.type}") if debug + if handle_event(connection, event, prompt: prompt, output: output, state: state) == :done + completed = true + break + end + + select_discovered_tool(connection, output: output, state: state) + end + return if completed + + raise "Realtime connection closed before the final response.done" + end + + def handle_event(connection, event, prompt:, output:, state:) + case event + when OpenAI::Realtime::McpListToolsCompleted + state[:completed_item_ids][event.item_id] = true + when OpenAI::Realtime::SessionUpdatedEvent + handle_session_updated(connection, prompt: prompt, state: state) + when OpenAI::Realtime::ConversationItemDone + handle_conversation_item(connection, event.item, output: output, state: state) + when OpenAI::Realtime::McpListToolsFailed + raise "MCP tool discovery failed for #{event.item_id}" + when OpenAI::Realtime::ResponseMcpCallArgumentsDone + state[:mcp_call_pending] = true + output.puts("[mcp] tool call requested") + when OpenAI::Realtime::ResponseMcpCallCompleted + handle_tool_completed(connection, output: output, state: state) + when OpenAI::Realtime::ResponseMcpCallFailed + raise "MCP tool call failed for #{event.item_id}" + when OpenAI::Realtime::RealtimeErrorEvent + raise event.error.message + when OpenAI::Realtime::ResponseTextDeltaEvent + output.print(event.delta) + output.flush + when OpenAI::Realtime::ResponseDoneEvent + handle_response_done(event, output: output, state: state) + end + end + + def handle_session_updated(connection, prompt:, state:) + return unless state[:waiting_for_tool_update] + + send_prompt(connection, prompt: prompt) + state[:waiting_for_tool_update] = false + end + + def handle_conversation_item(connection, item, output:, state:) + if item.is_a?(OpenAI::Realtime::RealtimeMcpListTools) + state[:tool_lists][item.id] = item + elsif item.is_a?(OpenAI::Realtime::RealtimeMcpApprovalRequest) + output.puts("[mcp] approving #{item.id}") + approve_request(connection, item) + end + end + + def handle_tool_completed(connection, output:, state:) + state[:mcp_call_pending] = false + output.puts("[mcp] tool call completed") + connection.response.create(tool_choice: :none) + end + + def handle_response_done(event, output:, state:) + raise "Response ended with #{event.response.status}" unless event.response.status == :completed + + if state[:mcp_call_pending] + output.puts("[mcp] waiting for approval and tool execution") + else + output.puts("\n[mcp] response.done status=completed") + :done + end + end + + def select_discovered_tool(connection, output:, state:) + return if state[:selected_tool] + + tool_list = state[:tool_lists].values.find { |item| state[:completed_item_ids][item.id] } + return unless tool_list + raise "The MCP server imported no tools" if tool_list.tools.empty? + + tool_names = tool_list.tools.map(&:name) + output.puts("[mcp] tools discovered: #{tool_names.join(', ')}") + select_tool(connection, server_label: tool_list.server_label, tool_name: tool_names.fetch(0)) + state[:selected_tool] = true + state[:waiting_for_tool_update] = true + end + + def run(client:, model:, server_url:, prompt:, output: $stdout, debug: false) + client.realtime.connect(model: model) do |connection| + configure(connection, server_url: server_url) + run_session(connection, prompt: prompt, output: output, debug: debug) + end + end + end + end + end +end + +if $PROGRAM_NAME == __FILE__ + client = OpenAI::Client.new + timeout = Integer(ENV.fetch("OPENAI_REALTIME_TIMEOUT", "60")) + + Timeout.timeout(timeout) do + OpenAI::Examples::Realtime::MCPApproval.run( + client: client, + model: ENV.fetch("OPENAI_REALTIME_MODEL", "gpt-realtime-2.1"), + server_url: ENV.fetch("MCP_SERVER_URL"), + prompt: ENV.fetch("OPENAI_REALTIME_PROMPT", "Use the configured MCP server."), + debug: ENV["OPENAI_REALTIME_DEBUG"] == "1" + ) + end +end diff --git a/examples/realtime/realtime_conversation.rb b/examples/realtime/realtime_conversation.rb new file mode 100755 index 000000000..1a49b6a08 --- /dev/null +++ b/examples/realtime/realtime_conversation.rb @@ -0,0 +1,534 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require "async" +require "timeout" +require_relative "../../lib/openai" +require_relative "event_stream" + +module OpenAI + module Examples + module Realtime + module Conversation + AUDIO_BYTES_PER_SECOND = 48_000 + CHUNK_BYTES = 4_800 + + class FFmpegMicrophone + def initialize(input_format:, device:) + @input_format = input_format + @device = device + @pid = nil + @stopping = false + end + + def each_chunk + return enum_for(__method__) unless block_given? + return if @stopping + + reader, writer = IO.pipe + @pid = Process.spawn(*command, out: writer, err: $stderr) + writer.close + reader.binmode + while (chunk = reader.read(CHUNK_BYTES)) + yield(chunk) + end + rescue Errno::ENOENT + raise "ffmpeg was not found; install it with `brew install ffmpeg`" + ensure + writer&.close unless writer&.closed? + reader&.close unless reader&.closed? + wait_for_capture + end + + def stop + @stopping = true + return unless @pid + + Process.kill("INT", @pid) + rescue Errno::ESRCH + nil + end + + private def command + [ + "ffmpeg", + "-nostdin", + "-loglevel", + "error", + "-thread_queue_size", + "1024", + "-f", + @input_format, + "-i", + @device, + "-ac", + "1", + "-ar", + "24000", + "-acodec", + "pcm_s16le", + "-f", + "s16le", + "pipe:1" + ] + end + + private def wait_for_capture + return unless @pid + + _, status = Process.wait2(@pid) + return if @stopping || status.success? + + raise "ffmpeg microphone capture exited with status #{status.exitstatus}" + rescue Errno::ECHILD + nil + ensure + @pid = nil + end + end + + class PCMFileMicrophone + def initialize(path) + @path = path + @stopping = false + end + + def each_chunk + return enum_for(__method__) unless block_given? + + File.open(@path, "rb") do |input| + while !@stopping && (chunk = input.read(CHUNK_BYTES)) + yield(chunk) + sleep(chunk.bytesize.fdiv(AUDIO_BYTES_PER_SECOND)) + end + end + 10.times do + break if @stopping + + yield("\0".b * CHUNK_BYTES) + sleep(CHUNK_BYTES.fdiv(AUDIO_BYTES_PER_SECOND)) + end + end + + def stop = @stopping = true + end + + class FFplaySpeaker + def initialize + @input = nil + @pid = nil + end + + def write(bytes) + start unless @input + @input.write(bytes) + @input.flush + rescue Errno::EPIPE + interrupt + raise "ffplay stopped while playing Realtime audio" + end + + def interrupt + return unless @pid + + Process.kill("TERM", @pid) + @input&.close unless @input&.closed? + Process.wait(@pid) + rescue Errno::ESRCH, Errno::ECHILD + nil + ensure + @input = nil + @pid = nil + end + + alias_method :close, :interrupt + + private def start + reader, writer = IO.pipe + @pid = Process.spawn(*command, in: reader, out: File::NULL, err: $stderr) + reader.close + writer.binmode + @input = writer + rescue Errno::ENOENT + raise "ffplay was not found; install it with `brew install ffmpeg`" + ensure + reader&.close unless reader&.closed? + end + + private def command + [ + "ffplay", + "-nodisp", + "-nostats", + "-loglevel", + "error", + "-fflags", + "nobuffer", + "-flags", + "low_delay", + "-probesize", + "32", + "-analyzeduration", + "0", + "-f", + "s16le", + "-ar", + "24000", + "-ch_layout", + "mono", + "-i", + "pipe:0" + ] + end + end + + class PCMFileSpeaker + def initialize(path) + @output = File.open(path, "wb") + end + + def write(bytes) = @output.write(bytes) + + def interrupt = @output.flush + + def close + @output.close unless @output.closed? + end + end + + # Serializes every client event through one writer fiber. The one-item queue + # preserves microphone backpressure while allowing the reader to enqueue an + # interruption without writing to the socket itself. + class OutboundWriter + def initialize(connection) + @connection = connection + @queue = Thread::SizedQueue.new(1) + @failure = nil + end + + def append_audio(bytes) + enqueue([:append_audio, bytes]) + end + + def truncate(**params) + enqueue([:truncate, params]) + end + + def run + while (message = @queue.pop) + operation, payload = message + case operation + when :append_audio + @connection.input_audio_buffer.append_bytes(payload) + when :truncate + @connection.conversation.items.truncate(**payload) + end + end + rescue StandardError => e + @failure = e + raise + ensure + close + end + + def close + @queue.close unless @queue.closed? + end + + private def enqueue(message) + @queue.push(message) + rescue ClosedQueueError + raise @failure if @failure + + raise + end + end + + class AudioPlayback + def initialize(speaker, clock: nil) + @speaker = speaker + @clock = clock || -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) } + @interrupted_response_id = nil + reset_current + end + + def write(event) + return if interrupted?(event.response_id) + + start_response(event) unless current?(event) + bytes = Base64.strict_decode64(event.delta) + @speaker.write(bytes) + @received_bytes += bytes.bytesize + end + + def interrupt(outbound) + expire_finished_playback + unless @item_id + @speaker.interrupt + return + end + + audio_end_ms = played_ms + response_id = @response_id + item_id = @item_id + content_index = @content_index + + @speaker.interrupt + outbound.truncate( + item_id: item_id, + content_index: content_index, + audio_end_ms: audio_end_ms + ) + @interrupted_response_id = response_id + reset_current + response_id + end + + def interrupted?(response_id) = response_id == @interrupted_response_id + + def finish(response_id) + if interrupted?(response_id) + @interrupted_response_id = nil + elsif response_id == @response_id + @finished_response_id = response_id + expire_finished_playback + end + end + + def close = @speaker.close + + private def current?(event) + @response_id == event.response_id && + @item_id == event.item_id && + @content_index == event.content_index + end + + private def start_response(event) + @response_id = event.response_id + @item_id = event.item_id + @content_index = event.content_index + @received_bytes = 0 + @started_at = @clock.call + end + + private def played_ms + elapsed_ms = ((@clock.call - @started_at) * 1_000).floor + received_ms = (@received_bytes * 1_000) / AUDIO_BYTES_PER_SECOND + [elapsed_ms, received_ms].min.clamp(0, received_ms) + end + + private def expire_finished_playback + return unless @finished_response_id + return unless @finished_response_id == @response_id + return unless elapsed_audio_bytes >= @received_bytes + + reset_current + end + + private def elapsed_audio_bytes + ((@clock.call - @started_at) * AUDIO_BYTES_PER_SECOND).floor + end + + private def reset_current + @response_id = nil + @item_id = nil + @content_index = nil + @received_bytes = 0 + @started_at = nil + @finished_response_id = nil + end + end + + module_function + + def configure(connection, voice:, instructions:) + connection.session.update( + type: :realtime, + output_modalities: [:audio], + instructions: instructions, + audio: { + input: { + format: {type: :"audio/pcm", rate: 24_000}, + noise_reduction: {type: :near_field}, + turn_detection: { + type: :server_vad, + threshold: 0.5, + prefix_padding_ms: 300, + silence_duration_ms: 500, + create_response: true, + interrupt_response: true + } + }, + output: { + format: {type: :"audio/pcm", rate: 24_000}, + voice: voice + } + } + ) + end + + def forward_microphone(outbound, microphone) + microphone.each_chunk do |chunk| + outbound.append_audio(chunk) + end + end + + def handle_event(event, outbound:, playback:, output:) + case event + when OpenAI::Realtime::ResponseAudioDeltaEvent + playback.write(event) + when OpenAI::Realtime::ResponseAudioTranscriptDeltaEvent + return if playback.interrupted?(event.response_id) + + output.print(event.delta) + output.flush + when OpenAI::Realtime::ResponseAudioTranscriptDoneEvent + return if playback.interrupted?(event.response_id) + + output.puts + when OpenAI::Realtime::InputAudioBufferSpeechStartedEvent + output.puts if playback.interrupt(outbound) + when OpenAI::Realtime::RealtimeErrorEvent + raise event.error.message + when OpenAI::Realtime::ResponseDoneEvent + playback.finish(event.response.id) + return if [:cancelled, :completed].include?(event.response.status) + + raise "Realtime response ended with #{event.response.status}" + end + end + + def stream_events(connection, microphone:, outbound:, playback:, output:, stop_after: nil) + EventStream.each_until(connection, stop_after: stop_after) do |event| + handle_event(event, outbound: outbound, playback: playback, output: output) + end + ensure + microphone.stop + end + + def run_session( + connection, + microphone:, + speaker:, + voice:, + instructions:, + output: $stdout, + stop_after: nil + ) + configure(connection, voice: voice, instructions: instructions) + EventStream.wait_for( + connection, + OpenAI::Realtime::SessionUpdatedEvent, + closed_message: "Realtime connection closed before session.updated" + ) + output.puts( + "Connected. Talk naturally; speak over the model to interrupt. " \ + "Press Ctrl-C to exit. Use headphones to prevent echo." + ) + playback = AudioPlayback.new(speaker) + outbound = OutboundWriter.new(connection) + + writer = Async do + outbound.run + nil + rescue StandardError => e + e + ensure + microphone.stop + end + receiver = Async do + stream_events( + connection, + microphone: microphone, + outbound: outbound, + playback: playback, + output: output, + stop_after: stop_after + ) + end + sender = Async { forward_microphone(outbound, microphone) } + sender.wait + outbound.close + writer_error = writer.wait + raise writer_error if writer_error + + receiver.wait + ensure + microphone.stop + outbound&.close + playback&.close || speaker.close + sender&.stop + receiver&.stop + writer&.stop + end + + def run( + client:, + model:, + microphone:, + speaker:, + voice:, + instructions:, + output: $stdout, + stop_after: nil + ) + client.realtime.connect(model: model) do |connection| + run_session( + connection, + microphone: microphone, + speaker: speaker, + voice: voice, + instructions: instructions, + output: output, + stop_after: stop_after + ) + end + end + end + end + end +end + +if $PROGRAM_NAME == __FILE__ + begin + input_path = ENV["REALTIME_INPUT_PCM"] + microphone = + if input_path + OpenAI::Examples::Realtime::Conversation::PCMFileMicrophone.new(input_path) + else + default_format = RUBY_PLATFORM.include?("darwin") ? "avfoundation" : "pulse" + default_device = RUBY_PLATFORM.include?("darwin") ? ":0" : "default" + OpenAI::Examples::Realtime::Conversation::FFmpegMicrophone.new( + input_format: ENV.fetch("REALTIME_AUDIO_INPUT_FORMAT", default_format), + device: ENV.fetch("REALTIME_MIC_DEVICE", default_device) + ) + end + stop_after = input_path ? "response.done" : nil + output_path = ENV["REALTIME_OUTPUT_PCM"] + speaker = + if output_path + OpenAI::Examples::Realtime::Conversation::PCMFileSpeaker.new(output_path) + else + OpenAI::Examples::Realtime::Conversation::FFplaySpeaker.new + end + run = lambda do + OpenAI::Examples::Realtime::Conversation.run( + client: OpenAI::Client.new, + model: ENV.fetch("OPENAI_REALTIME_MODEL", "gpt-realtime-2.1"), + microphone: microphone, + speaker: speaker, + voice: ENV.fetch("OPENAI_REALTIME_VOICE", "marin"), + instructions: ENV.fetch( + "OPENAI_REALTIME_INSTRUCTIONS", + "Have a natural spoken conversation. Keep each response concise." + ), + stop_after: stop_after + ) + end + + timeout = ENV["OPENAI_REALTIME_TIMEOUT"] + timeout ? Timeout.timeout(Integer(timeout)) { run.call } : run.call + rescue Interrupt + warn("\nConversation ended.") + end +end diff --git a/examples/realtime/sideband.rb b/examples/realtime/sideband.rb new file mode 100755 index 000000000..780260820 --- /dev/null +++ b/examples/realtime/sideband.rb @@ -0,0 +1,49 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require_relative "../../lib/openai" +require_relative "event_stream" +require "timeout" + +module OpenAI + module Examples + module Realtime + module Sideband + module_function + + def stream(connection, output: $stdout, stop_after: nil) + EventStream.each_until(connection, stop_after: stop_after) do |event| + output.puts("#{event.type}: #{event.to_h}") + output.flush + end + end + + def run(client:, call_id:, stop_after: nil, output: $stdout) + client.realtime.connect_to_call(call_id: call_id) do |connection| + connection.session.update( + type: :realtime, + instructions: "Use server-side business rules and keep answers short." + ) + stream(connection, output: output, stop_after: stop_after) + end + end + end + end + end +end + +if $PROGRAM_NAME == __FILE__ + # CALL_ID may come from calls.create.call_id for WebRTC or from a verified + # realtime.call.incoming webhook for SIP. + client = OpenAI::Client.new + run = lambda do + OpenAI::Examples::Realtime::Sideband.run( + client: client, + call_id: ENV.fetch("OPENAI_REALTIME_CALL_ID"), + stop_after: ENV["OPENAI_REALTIME_STOP_AFTER"] + ) + end + + timeout = ENV["OPENAI_REALTIME_TIMEOUT"] + timeout ? Timeout.timeout(Integer(timeout)) { run.call } : run.call +end diff --git a/examples/realtime/sip.rb b/examples/realtime/sip.rb new file mode 100755 index 000000000..869adeded --- /dev/null +++ b/examples/realtime/sip.rb @@ -0,0 +1,69 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require_relative "../../lib/openai" +require_relative "event_stream" +require "timeout" + +module OpenAI + module Examples + module Realtime + module SIP + module_function + + def stream(connection, output: $stdout, stop_after: nil) + EventStream.each_until(connection, stop_after: stop_after) do |event| + case event + when OpenAI::Realtime::ResponseAudioTranscriptDeltaEvent + output.print(event.delta) + output.flush + when OpenAI::Realtime::RealtimeErrorEvent + raise event.error.message + end + end + end + + def run(client:, call_id:, model:, output: $stdout, stop_after: nil) + accepted = false + client.realtime.calls.accept( + call_id, + type: :realtime, + model: model, + instructions: "You are answering a phone call. Be warm and concise." + ) + accepted = true + + client.realtime.connect_to_call(call_id: call_id) do |connection| + stream(connection, output: output, stop_after: stop_after) + end + ensure + hangup(client, call_id, active_error: $ERROR_INFO) if accepted + end + + def hangup(client, call_id, active_error:) + client.realtime.calls.hangup(call_id) + rescue OpenAI::Errors::NotFoundError + nil + rescue StandardError + raise unless active_error + end + end + end + end +end + +if $PROGRAM_NAME == __FILE__ + # Obtain the call ID from a verified realtime.call.incoming webhook via + # client.webhooks.unwrap(payload, headers). + run = lambda do + OpenAI::Examples::Realtime::SIP.run( + client: OpenAI::Client.new, + call_id: ENV.fetch("OPENAI_REALTIME_CALL_ID"), + model: ENV.fetch("OPENAI_REALTIME_MODEL", "gpt-realtime-2.1"), + stop_after: ENV["OPENAI_REALTIME_STOP_AFTER"] + ) + end + + timeout = ENV["OPENAI_REALTIME_TIMEOUT"] + timeout ? Timeout.timeout(Integer(timeout)) { run.call } : run.call +end diff --git a/examples/realtime/sorbet_connection_types.rb b/examples/realtime/sorbet_connection_types.rb new file mode 100644 index 000000000..a231aa0eb --- /dev/null +++ b/examples/realtime/sorbet_connection_types.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true +# typed: true + +require_relative "../../lib/openai" + +# Static contract fixture for the three capability-specific connection methods. +# Defining, but never invoking, this method keeps the fixture inert at runtime. +module OpenAI + module Examples + module Realtime + module SorbetConnectionTypes + module_function + + def verify + client = OpenAI::Client.new + + client.realtime.connect(model: "gpt-realtime-2.1") do |connection| + T.assert_type!(connection, OpenAI::Realtime::Connection) + connection.response.create + end + + client.realtime.connect_to_call(call_id: "rtc_123") do |connection| + T.assert_type!(connection, OpenAI::Realtime::SidebandConnection) + connection.output_audio_buffer.clear + end + + client.realtime.connect_transcription do |connection| + T.assert_type!(connection, OpenAI::Realtime::TranscriptionConnection) + connection.input_audio_buffer.commit + end + end + end + end + end +end diff --git a/examples/realtime/translation.rb b/examples/realtime/translation.rb new file mode 100755 index 000000000..9292635a7 --- /dev/null +++ b/examples/realtime/translation.rb @@ -0,0 +1,110 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require "async" +require_relative "../../lib/openai" +require_relative "event_stream" + +module OpenAI + module Examples + module Realtime + module Translation + module_function + + def stream(connection, audio_output:, transcript_output: $stdout) + audio_bytes = 0 + EventStream.each_until( + connection, + stop_after: "session.closed", + closed_message: "Realtime translation connection closed before session.closed" + ) do |event| + case event + when OpenAI::Realtime::RealtimeTranslationOutputTranscriptDeltaEvent + transcript_output.print(event.delta) + transcript_output.flush + when OpenAI::Realtime::RealtimeTranslationOutputAudioDeltaEvent + audio = Base64.strict_decode64(event.delta) + audio_output.write(audio) + audio_bytes += audio.bytesize + when OpenAI::Realtime::RealtimeTranslationSessionClosedEvent + raise "Translation session closed without audio output" if audio_bytes.zero? + + transcript_output.puts + when OpenAI::Realtime::RealtimeErrorEvent + raise event.error.message + end + end + end + + def write_input(connection, input_path) + File.open(input_path, "rb") do |input| + while (chunk = input.read(9_600)) + connection.input_audio_buffer.append_bytes(chunk) + end + end + ensure + close_session(connection, preserve_error: !$ERROR_INFO.nil?) + end + + def close_session(connection, preserve_error:) + connection.session.close + rescue StandardError + raise unless preserve_error + end + + def exchange(connection, input_path:, audio_output:, transcript_output:) + uploader = nil + reader = Async do + stream( + connection, + audio_output: audio_output, + transcript_output: transcript_output + ) + nil + rescue StandardError => e + uploader&.stop + e + end + if reader.finished? + reader_error = reader.wait + raise reader_error if reader_error + + return + end + + uploader = Async { write_input(connection, input_path) } + uploader.wait + reader_error = reader.wait + raise reader_error if reader_error + ensure + uploader&.stop + reader&.stop + end + + def run(client:, model:, input_path:, output_path:, target_language:, transcript_output: $stdout) + File.open(output_path, "wb") do |audio_output| + client.realtime.translations.connect(model: model) do |connection| + connection.session.update(audio: {output: {language: target_language}}) + exchange( + connection, + input_path: input_path, + audio_output: audio_output, + transcript_output: transcript_output + ) + end + end + end + end + end + end +end + +if $PROGRAM_NAME == __FILE__ + OpenAI::Examples::Realtime::Translation.run( + client: OpenAI::Client.new, + model: ENV.fetch("OPENAI_REALTIME_TRANSLATION_MODEL", "gpt-realtime-translate"), + input_path: ENV.fetch("REALTIME_INPUT_PCM"), + output_path: ENV.fetch("REALTIME_OUTPUT_PCM", "translation-output.pcm"), + target_language: ENV.fetch("TARGET_LANGUAGE", "es") + ) +end diff --git a/examples/realtime/webrtc_call.rb b/examples/realtime/webrtc_call.rb new file mode 100755 index 000000000..4b5835d68 --- /dev/null +++ b/examples/realtime/webrtc_call.rb @@ -0,0 +1,22 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require_relative "../../lib/openai" + +# Pipe a browser's SDP offer to this process. The SDP answer is written to stdout, +# which makes this suitable as the core of a Rails, Sinatra, or Rack endpoint. +offer = $stdin.read +raise "Expected an SDP offer on stdin" if offer.empty? + +client = OpenAI::Client.new +call = client.realtime.calls.create( + sdp: offer, + session: { + type: :realtime, + model: ENV.fetch("OPENAI_REALTIME_MODEL", "gpt-realtime-2.1"), + audio: {output: {voice: :marin}} + } +) + +warn("Created Realtime call #{call.call_id || 'without a Location header'}") +$stdout.write(call.sdp) diff --git a/examples/realtime/webrtc_conversation.html b/examples/realtime/webrtc_conversation.html new file mode 100644 index 000000000..ac68d4c3f --- /dev/null +++ b/examples/realtime/webrtc_conversation.html @@ -0,0 +1,223 @@ + + + + + + Ruby Realtime conversation + + + +
+

Realtime conversation

+

+ Talk naturally and speak over the model whenever you want to interrupt. + The browser handles the microphone, playback, and acoustic echo cancellation. +

+
+ + +
+
Ready
+
+
+ +
+ + + + diff --git a/examples/realtime/webrtc_conversation.rb b/examples/realtime/webrtc_conversation.rb new file mode 100755 index 000000000..3f1d10bea --- /dev/null +++ b/examples/realtime/webrtc_conversation.rb @@ -0,0 +1,149 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require_relative "../../lib/openai" + +module OpenAI + module Examples + module Realtime + module WebRTCConversation + MAX_SDP_BYTES = 1_048_576 + + class App + def initialize(client:, html: File.binread(File.join(__dir__, "webrtc_conversation.html"))) + @client = client + @html = html + @call_ids = {} + @call_ids_lock = Mutex.new + end + + def handle(request, response) + case [request.request_method, request.path] + when ["GET", "/"] + render(response, status: 200, content_type: "text/html; charset=utf-8", body: @html) + when ["POST", "/session"] + create_session(request, response) + when ["POST", "/hangup"] + hangup(request, response) + else + render(response, status: 404, content_type: "text/plain; charset=utf-8", body: "Not found\n") + end + rescue ArgumentError => e + render(response, status: 400, content_type: "text/plain; charset=utf-8", body: "#{e.message}\n") + rescue StandardError => e + warn("Realtime WebRTC request failed: #{e.class}: #{e.message}") + render( + response, + status: 502, + content_type: "text/plain; charset=utf-8", + body: "Realtime request failed\n" + ) + end + + private def create_session(request, response) + content_type = request["content-type"].to_s.split(";", 2).first + unless content_type == "application/sdp" + raise ArgumentError, "Expected Content-Type: application/sdp" + end + + offer = request.body.to_s + raise ArgumentError, "Expected an SDP offer" if offer.empty? + raise ArgumentError, "SDP offer is too large" if offer.bytesize > MAX_SDP_BYTES + raise ArgumentError, "Invalid SDP offer" unless offer.lstrip.start_with?("v=") + + call = @client.realtime.calls.create(sdp: offer, session: session_config) + remember(call.call_id) + response["X-OpenAI-Call-ID"] = call.call_id if call.call_id + render(response, status: 201, content_type: "application/sdp", body: call.sdp) + end + + private def hangup(request, response) + call_id = request.body.to_s.strip + raise ArgumentError, "Expected a call ID" if call_id.empty? + + known = @call_ids_lock.synchronize { @call_ids.include?(call_id) } + unless known + return render( + response, + status: 404, + content_type: "text/plain; charset=utf-8", + body: "Unknown call\n" + ) + end + + begin + @client.realtime.calls.hangup(call_id) + rescue OpenAI::Errors::NotFoundError + # Closing the browser peer can end the call before this cleanup request arrives. + end + @call_ids_lock.synchronize { @call_ids.delete(call_id) } + render(response, status: 204, content_type: "text/plain; charset=utf-8", body: "") + end + + private def session_config + { + type: :realtime, + model: ENV.fetch("OPENAI_REALTIME_MODEL", "gpt-realtime-2.1"), + output_modalities: [:audio], + instructions: ENV.fetch( + "OPENAI_REALTIME_INSTRUCTIONS", + "Have a natural spoken conversation. Keep each response concise." + ), + audio: { + input: { + turn_detection: { + type: :server_vad, + threshold: 0.5, + prefix_padding_ms: 300, + silence_duration_ms: 700, + create_response: true, + interrupt_response: true + } + }, + output: {voice: ENV.fetch("OPENAI_REALTIME_VOICE", "marin")} + } + } + end + + private def remember(call_id) + @call_ids_lock.synchronize { @call_ids[call_id] = true } if call_id + end + + private def render(response, status:, content_type:, body:) + response.status = status + response["Content-Type"] = content_type + response["Cache-Control"] = "no-store" + response["X-Content-Type-Options"] = "nosniff" + response.body = body + end + end + + module_function + + def run + require("webrick") + + host = ENV.fetch("REALTIME_DEMO_HOST", "127.0.0.1") + port = Integer(ENV.fetch("REALTIME_DEMO_PORT", "4567")) + app = App.new(client: OpenAI::Client.new) + server = WEBrick::HTTPServer.new( + BindAddress: host, + Port: port, + Logger: WEBrick::Log.new($stderr, WEBrick::BasicLog::WARN), + AccessLog: [] + ) + server.mount_proc("/") { |request, response| app.handle(request, response) } + shutdown = proc { server.shutdown } + Signal.trap("INT", &shutdown) + Signal.trap("TERM", &shutdown) + puts("Open http://#{host}:#{port} and click Start conversation. Press Ctrl-C here to stop.") + server.start + rescue LoadError + raise "webrick is required; run `bundle install` before starting this example" + end + end + end + end +end + +OpenAI::Examples::Realtime::WebRTCConversation.run if $PROGRAM_NAME == __FILE__ diff --git a/examples/realtime/websocket_audio.rb b/examples/realtime/websocket_audio.rb new file mode 100755 index 000000000..c8c08aeaa --- /dev/null +++ b/examples/realtime/websocket_audio.rb @@ -0,0 +1,73 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require_relative "../../lib/openai" +require_relative "event_stream" + +module OpenAI + module Examples + module Realtime + module WebSocketAudio + module_function + + def configure(connection) + connection.session.update( + type: :realtime, + output_modalities: [:audio], + audio: {input: {turn_detection: nil}} + ) + end + + def stream_response(connection, output:) + audio_bytes = 0 + EventStream.each_until(connection, stop_after: "response.done") do |event| + case event + when OpenAI::Realtime::ResponseAudioDeltaEvent + audio = Base64.strict_decode64(event.delta) + output.write(audio) + audio_bytes += audio.bytesize + when OpenAI::Realtime::ResponseDoneEvent + status = event.response.status + raise "Response ended with #{status}" unless status == :completed + raise "Response completed without audio output" if audio_bytes.zero? + when OpenAI::Realtime::RealtimeErrorEvent + raise event.error.message + end + end + end + + def run(client:, model:, input_path:, output_path:) + File.open(output_path, "wb") do |output| + client.realtime.connect(model: model) do |connection| + configure(connection) + EventStream.wait_for( + connection, + OpenAI::Realtime::SessionUpdatedEvent, + closed_message: "Realtime connection closed before session.updated" + ) + File.open(input_path, "rb") do |input| + while (chunk = input.read(4_800)) + connection.input_audio_buffer.append_bytes(chunk) + end + end + connection.input_audio_buffer.commit + connection.response.create + stream_response(connection, output: output) + end + end + + warn("Wrote raw PCM16 audio to #{output_path}") + end + end + end + end +end + +if $PROGRAM_NAME == __FILE__ + OpenAI::Examples::Realtime::WebSocketAudio.run( + client: OpenAI::Client.new, + model: ENV.fetch("OPENAI_REALTIME_MODEL", "gpt-realtime-2.1"), + input_path: ENV.fetch("REALTIME_INPUT_PCM"), + output_path: ENV.fetch("REALTIME_OUTPUT_PCM", "realtime-output.pcm") + ) +end diff --git a/examples/realtime/websocket_text.rb b/examples/realtime/websocket_text.rb new file mode 100755 index 000000000..1f771e5fc --- /dev/null +++ b/examples/realtime/websocket_text.rb @@ -0,0 +1,77 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require "timeout" +require_relative "../../lib/openai" +require_relative "event_stream" + +module OpenAI + module Examples + module Realtime + module WebSocketText + module_function + + def stream_response(connection, output: $stdout) + started_response = false + received_text = false + EventStream.each_until(connection, stop_after: "response.done") do |event| + case event + when OpenAI::Realtime::SessionCreatedEvent + output.puts("[realtime] session.created") + when OpenAI::Realtime::SessionUpdatedEvent + output.puts("[realtime] session.updated") + when OpenAI::Realtime::ResponseTextDeltaEvent + output.print("[assistant] ") unless started_response + started_response = true + received_text = true unless event.delta.empty? + output.print(event.delta) + output.flush + when OpenAI::Realtime::ResponseDoneEvent + output.puts if started_response + status = event.response.status + raise "Realtime response ended with status #{status.inspect}" unless status == :completed + raise "Realtime response completed without text output" unless received_text + + output.puts("[realtime] response.done status=completed") + when OpenAI::Realtime::RealtimeErrorEvent + raise "Realtime API error: #{event.error.message}" + end + end + end + + def run(client:, model:, prompt:, output: $stdout) + session = { + type: :realtime, + output_modalities: [:text], + instructions: "Be concise and friendly." + } + item = { + type: :message, + role: :user, + content: [{type: :input_text, text: prompt}] + } + + output.puts("[realtime] connecting with #{model}") + client.realtime.connect(model: model) do |connection| + output.puts("[realtime] connected; sending prompt: #{prompt.inspect}") + connection.session.update(**session) + connection.conversation.items.create(**item) + connection.response.create + stream_response(connection, output: output) + end + output.puts("[realtime] smoke test passed") + end + end + end + end +end + +if $PROGRAM_NAME == __FILE__ + Timeout.timeout(Integer(ENV.fetch("OPENAI_REALTIME_TIMEOUT", "30"))) do + OpenAI::Examples::Realtime::WebSocketText.run( + client: OpenAI::Client.new, + model: ENV.fetch("OPENAI_REALTIME_MODEL", "gpt-realtime-2.1"), + prompt: ENV.fetch("OPENAI_REALTIME_PROMPT", "Say hello from Ruby.") + ) + end +end diff --git a/examples/realtime/websocket_transcription.rb b/examples/realtime/websocket_transcription.rb new file mode 100755 index 000000000..c9c70889d --- /dev/null +++ b/examples/realtime/websocket_transcription.rb @@ -0,0 +1,81 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require "timeout" +require_relative "../../lib/openai" +require_relative "event_stream" + +module OpenAI + module Examples + module Realtime + module WebSocketTranscription + module_function + + def configure(connection, transcription_model:) + connection.session.update( + audio: { + input: { + format: {type: :"audio/pcm", rate: 24_000}, + transcription: {model: transcription_model}, + turn_detection: nil + } + } + ) + end + + def upload(connection, input_path:) + File.open(input_path, "rb") do |input| + while (chunk = input.read(4_800)) + connection.input_audio_buffer.append_bytes(chunk) + end + end + connection.input_audio_buffer.commit + end + + def print_transcript(connection, output: $stdout) + EventStream.each_until( + connection, + stop_after: "conversation.item.input_audio_transcription.completed", + closed_message: "Realtime connection closed before transcription completed" + ) do |event| + case event + when OpenAI::Realtime::ConversationItemInputAudioTranscriptionDeltaEvent + output.print(event.delta) + output.flush + when OpenAI::Realtime::ConversationItemInputAudioTranscriptionCompletedEvent + output.puts("\n[#{event.item_id}] #{event.transcript}") + when OpenAI::Realtime::RealtimeErrorEvent + raise event.error.message + end + end + end + + def run(client:, input_path:, transcription_model:, output: $stdout) + client.realtime.connect_transcription do |connection| + configure(connection, transcription_model: transcription_model) + EventStream.wait_for( + connection, + OpenAI::Realtime::SessionUpdatedEvent, + closed_message: "Realtime connection closed before session.updated" + ) + upload(connection, input_path: input_path) + print_transcript(connection, output: output) + end + end + end + end + end +end + +if $PROGRAM_NAME == __FILE__ + Timeout.timeout(Integer(ENV.fetch("OPENAI_REALTIME_TIMEOUT", "30"))) do + OpenAI::Examples::Realtime::WebSocketTranscription.run( + client: OpenAI::Client.new, + input_path: ENV.fetch("REALTIME_INPUT_PCM"), + transcription_model: ENV.fetch( + "OPENAI_REALTIME_TRANSCRIPTION_MODEL", + "gpt-live-transcribe" + ) + ) + end +end diff --git a/lib/openai.rb b/lib/openai.rb index 2cdf3b4fd..81999905b 100644 --- a/lib/openai.rb +++ b/lib/openai.rb @@ -800,6 +800,8 @@ require_relative "openai/models/other_file_chunking_strategy_object" require_relative "openai/models/realtime/audio_transcription" require_relative "openai/models/realtime/call_accept_params" +require_relative "openai/models/realtime/call_create_params" +require_relative "openai/models/realtime/call_create_response" require_relative "openai/models/realtime/call_hangup_params" require_relative "openai/models/realtime/call_refer_params" require_relative "openai/models/realtime/call_reject_params" @@ -1165,6 +1167,15 @@ require_relative "openai/models/webhooks/unwrap_webhook_event" require_relative "openai/models/webhooks/webhook_unwrap_params" require_relative "openai/models" +require_relative "openai/realtime/unknown_server_event" +require_relative "openai/realtime/base_connection" +require_relative "openai/realtime/connection_resources" +require_relative "openai/realtime/connection" +require_relative "openai/realtime/sideband_connection" +require_relative "openai/realtime/transcription_connection" +require_relative "openai/realtime/translation_connection" +require_relative "openai/realtime/connection_manager" +require_relative "openai/realtime/transports/async_websocket" require_relative "openai/resources/admin" require_relative "openai/resources/admin/organization" require_relative "openai/resources/admin/organization/admin_api_keys" @@ -1245,6 +1256,9 @@ require_relative "openai/resources/realtime" require_relative "openai/resources/realtime/calls" require_relative "openai/resources/realtime/client_secrets" +require_relative "openai/resources/realtime/translations" +require_relative "openai/resources/realtime/translations/calls" +require_relative "openai/resources/realtime/translations/client_secrets" require_relative "openai/resources/responses" require_relative "openai/resources/responses/input_items" require_relative "openai/resources/responses/input_tokens" diff --git a/lib/openai/client.rb b/lib/openai/client.rb index 6400ca209..e060cca18 100644 --- a/lib/openai/client.rb +++ b/lib/openai/client.rb @@ -32,6 +32,11 @@ class Client < OpenAI::Internal::Transport::BaseClient # @return [String, nil] attr_reader :webhook_secret + # Optional base URL used for WebSocket connections. + # + # @return [URI::Generic, nil] + attr_reader :websocket_base_url + # @return [OpenAI::Auth::WorkloadIdentityAuth, nil] # @api private attr_reader :workload_identity_auth @@ -157,6 +162,80 @@ class Client < OpenAI::Internal::Transport::BaseClient {"authorization" => "Bearer #{@admin_api_key}"} end + # Build a fully authenticated Realtime WebSocket handshake request. Realtime + # transports use this boundary so provider authentication and request options stay + # consistent with ordinary SDK requests. + # + # @api private + # + # @param path [String] + # @param query [Hash{String=>String}] + # @param options [OpenAI::RequestOptions, Hash{Symbol=>Object}, nil] + # + # @return [Hash{Symbol=>Object}] + def realtime_connection_request(path:, query:, options: nil) + if @provider_runtime && @provider_runtime.name != "azure" + raise OpenAI::Errors::Error, + "Realtime WebSocket connections are not supported by the #{@provider_runtime.name} provider." + end + + opts = options.to_h + OpenAI::RequestOptions.validate!(opts) + request = build_request( + { + method: :get, + path: path, + query: query, + security: {bearer_auth: true} + }, + opts + ) + + workload_identity_header = "Bearer #{WORKLOAD_IDENTITY_API_KEY_PLACEHOLDER}" + if @workload_identity_auth && request.fetch(:headers)["authorization"] == workload_identity_header + token = @workload_identity_auth.get_token + request = request.merge( + headers: request.fetch(:headers).merge("authorization" => "Bearer #{token}") + ) + end + + request = prepare_request(request, redirect_count: 0, retry_count: 0) + if @websocket_base_url + url = OpenAI::Internal::Util.join_parsed_uri( + OpenAI::Internal::Util.parse_uri(@websocket_base_url.to_s), + {path: OpenAI::Internal::Util.interpolate_path(path)} + ) + url.query = request.fetch(:url).query + request = request.merge(url: url) + end + + url = request.fetch(:url).dup + url.scheme = {"http" => "ws", "https" => "wss"}.fetch(url.scheme, url.scheme) + headers = request.fetch(:headers).except("accept", "content-type") + request.merge(url: url, headers: headers) + end + + # @api private + # + # @param value [String, nil] + # @return [URI::Generic, nil] + private def parse_websocket_base_url(value) + return if value.nil? + + uri = URI(value) + valid_scheme = %w[http https ws wss].include?(uri.scheme) + ambiguous_component = uri.userinfo || uri.query || uri.fragment + unless uri.absolute? && uri.host && valid_scheme && !ambiguous_component + message = + "`websocket_base_url` must be an absolute HTTP or WebSocket URL " \ + "without credentials, query, or fragment" + raise ArgumentError, message + end + uri + rescue URI::Error => e + raise ArgumentError, "`websocket_base_url` is not a valid URL", cause: e + end + # Creates and returns a new client for interacting with the API. # # @param api_key [String, nil] Defaults to `ENV["OPENAI_API_KEY"]`. @@ -232,6 +311,9 @@ class Client < OpenAI::Internal::Transport::BaseClient # @param default_headers [Hash{String=>String, nil}, nil] Extra headers to send # with every request. Explicit values override `ENV["OPENAI_CUSTOM_HEADERS"]`. # + # @param websocket_base_url [String, nil] Override the base URL for WebSocket + # connections. The HTTP base URL is used when omitted. + # # @param max_retries [Integer] Max number of retries to attempt after a failed retryable request. # # @param timeout [Float, nil] @@ -259,6 +341,7 @@ def initialize( provider: nil, base_url: OpenAI::Internal::OMIT, default_headers: nil, + websocket_base_url: nil, max_retries: self.class::DEFAULT_MAX_RETRIES, timeout: self.class::DEFAULT_TIMEOUT_IN_SECONDS, initial_retry_delay: self.class::DEFAULT_INITIAL_RETRY_DELAY, @@ -275,7 +358,8 @@ def initialize( api_key: api_key, admin_api_key: admin_api_key, workload_identity: workload_identity, - base_url: base_url + base_url: base_url, + websocket_base_url: websocket_base_url }.filter_map do |name, value| name unless value.equal?(OpenAI::Internal::OMIT) || value.nil? end @@ -352,6 +436,7 @@ def initialize( @admin_api_key = admin_api_key&.to_s @webhook_secret = webhook_secret&.to_s @provider_runtime = provider_runtime + @websocket_base_url = parse_websocket_base_url(websocket_base_url) super( base_url: base_url, diff --git a/lib/openai/errors.rb b/lib/openai/errors.rb index d810efa2b..76456f161 100644 --- a/lib/openai/errors.rb +++ b/lib/openai/errors.rb @@ -5,12 +5,54 @@ module Errors class Error < StandardError # @!attribute cause # - # @return [StandardError, nil] + # @return [Exception, nil] end class InvalidWebhookSignatureError < OpenAI::Errors::Error end + # Raised when a Realtime WebSocket cannot be opened or used. + class RealtimeConnectionError < OpenAI::Errors::Error + # @return [URI::Generic] + attr_reader :url + + # @return [Exception, nil] + def cause = @cause.nil? ? super : @cause + + # @api private + # + # @param url [URI::Generic] + # @param message [String, nil] + # @param cause [Exception, nil] + def initialize(url:, message: nil, cause: nil) + @url = url + @cause = cause + detail = cause && !cause.message.empty? ? ": #{cause.message}" : "." + super(message || "Realtime WebSocket connection error#{detail}") + end + end + + # Raised when a Realtime WebSocket message cannot be parsed as a typed event. + class RealtimeProtocolError < OpenAI::Errors::Error + # @return [String] + attr_reader :data + + # @return [StandardError, nil] + def cause = @cause.nil? ? super : @cause + + # @api private + # + # @param data [String] + # @param message [String, nil] + # @param cause [StandardError, nil] + def initialize(data:, message: nil, cause: nil) + @data = data + @cause = cause + detail = cause && !cause.message.empty? ? ": #{cause.message}" : "." + super(message || "Invalid Realtime WebSocket event#{detail}") + end + end + class ConversionError < OpenAI::Errors::Error # @return [StandardError, nil] def cause = @cause.nil? ? super : @cause diff --git a/lib/openai/internal/logging.rb b/lib/openai/internal/logging.rb index e633353ec..7537d85ce 100644 --- a/lib/openai/internal/logging.rb +++ b/lib/openai/internal/logging.rb @@ -112,6 +112,21 @@ def observe_stream(stream, response:) ObservedStream.new(stream: stream, context: self, response: response) end + def observe_raw_response(response) + return response unless enabled?(:error) + + OpenAI::HTTPClient::Response.new( + status: response.status, + headers: response.headers, + body: ObservedEnumerable.new( + enumerable: response.body, + context: self, + response: response, + close: -> { OpenAI::Internal::Util.close_fused!(response.body) } + ) + ) + end + def request_failed(error) status = error.is_a?(OpenAI::Errors::APIError) ? error.status : nil request_id = error.is_a?(OpenAI::Errors::APIError) ? error.request_id : nil @@ -236,10 +251,34 @@ def initialize(stream:, context:, response:) end private def iterator - source = @stream.to_enum + ObservedEnumerable.new( + enumerable: @stream, + context: @context, + response: @response, + close: -> { @stream.close } + ) + end + end + + class ObservedEnumerable + include Enumerable + + def initialize(enumerable:, context:, response:, close:) + @enumerable = enumerable + @context = context + @response = response + @close = close + @iterator = iterator + end + + def each(&block) = @iterator.each(&block) + def close = OpenAI::Internal::Util.close_fused!(@iterator) + + private def iterator + source = @enumerable.to_enum observed = Enumerator.new do |yielder| loop do - event = + chunk = begin source.next rescue StopIteration @@ -249,10 +288,10 @@ def initialize(stream:, context:, response:) @context.request_failed(e) raise end - yielder << event + yielder << chunk end end - OpenAI::Internal::Util.fused_enum(observed) { @stream.close } + OpenAI::Internal::Util.fused_enum(observed, &@close) end end diff --git a/lib/openai/internal/transport/base_client.rb b/lib/openai/internal/transport/base_client.rb index d037f5ff3..32ccc50ae 100644 --- a/lib/openai/internal/transport/base_client.rb +++ b/lib/openai/internal/transport/base_client.rb @@ -611,6 +611,19 @@ def send_request(request, redirect_count:, retry_count:, send_retry_header:, &co end end + # Execute the request specified by `req` without decoding its response. This is + # used by resources whose successful response is not modeled JSON. + # + # @api private + # + # @param req [Hash{Symbol=>Object}] + # + # @return [OpenAI::HTTPClient::Response] + def request_raw(req) + _, response, log_context = perform_request(req) + log_context.observe_raw_response(response) + end + # Execute the request specified by `req`. This is the method that all resource # methods call into. # diff --git a/lib/openai/models/realtime/call_create_params.rb b/lib/openai/models/realtime/call_create_params.rb new file mode 100644 index 000000000..beec64067 --- /dev/null +++ b/lib/openai/models/realtime/call_create_params.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module OpenAI + module Models + module Realtime + # @see OpenAI::Resources::Realtime::Calls#create + class CallCreateParams < OpenAI::Internal::Type::BaseModel + extend OpenAI::Internal::Type::RequestParameters::Converter + include OpenAI::Internal::Type::RequestParameters + + # @!attribute sdp + # The WebRTC Session Description Protocol offer. + # + # @return [String] + required :sdp, String + + # @!attribute session + # Optional session configuration sent with the SDP offer. + # + # @return [OpenAI::Models::Realtime::RealtimeSessionCreateRequest, nil] + optional :session, -> { OpenAI::Realtime::RealtimeSessionCreateRequest } + + # @!method initialize(sdp:, session: nil, request_options: {}) + # @param sdp [String] The WebRTC Session Description Protocol offer. + # @param session [OpenAI::Models::Realtime::RealtimeSessionCreateRequest] Optional session configuration. + # @param request_options [OpenAI::RequestOptions, Hash{Symbol=>Object}] + end + end + end +end diff --git a/lib/openai/models/realtime/call_create_response.rb b/lib/openai/models/realtime/call_create_response.rb new file mode 100644 index 000000000..581bd7fae --- /dev/null +++ b/lib/openai/models/realtime/call_create_response.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module OpenAI + module Models + module Realtime + # The SDP answer and response metadata returned when a WebRTC call is created. + class CallCreateResponse < OpenAI::Internal::Type::BaseModel + # @!attribute sdp + # The WebRTC Session Description Protocol answer. + # + # @return [String] + required :sdp, String + + # @!attribute call_id + # The call ID parsed from the response Location header, when present. + # + # @return [String, nil] + optional :call_id, String + + # @!attribute headers + # The response headers, normalized to lowercase names. + # + # @return [Hash{String=>String}] + required :headers, -> { OpenAI::Internal::Type::HashOf[String] } + + # @!method initialize(sdp:, headers:, call_id: nil) + # @param sdp [String] The WebRTC Session Description Protocol answer. + # @param headers [Hash{Symbol,String=>String}] The normalized response headers. + # @param call_id [String, nil] The call ID parsed from the Location header. + end + end + end +end diff --git a/lib/openai/realtime/base_connection.rb b/lib/openai/realtime/base_connection.rb new file mode 100644 index 000000000..2172acead --- /dev/null +++ b/lib/openai/realtime/base_connection.rb @@ -0,0 +1,143 @@ +# frozen_string_literal: true + +module OpenAI + module Realtime + # Shared protocol machinery for live Realtime WebSocket connections. + # + # @api private + class BaseConnection + include Enumerable + + # @return [URI::Generic] + attr_reader :url + + # @api private + def initialize(socket:, url:, server_event_type:, client_event_type:) + @socket = socket + @url = url + @server_event_type = server_event_type + @client_event_type = client_event_type + @server_event_names = discriminator_values(@server_event_type) + @client_event_names = discriminator_values(@client_event_type) + end + + # Yield server events until the remote peer closes the connection. + def each + return enum_for(__method__) unless block_given? + + while (event = receive) + yield(event) + end + self + end + + # Receive and parse the next server event, or return nil after a clean close. + def receive + data = receive_raw + return nil if data.nil? + + parse_event(data) + end + + # Receive the next raw WebSocket message. + def receive_raw + message = @socket.read + message&.to_str + end + + # Parse raw JSON as a typed server event. Valid events that are newer than this + # SDK remain observable as {UnknownServerEvent} values. + def parse_event(data) + parsed = JSON.parse(data, symbolize_names: true) + type = event_type(parsed) + unless @server_event_names.key?(type.to_s) + return OpenAI::Realtime::UnknownServerEvent.new(data: parsed) + end + + state = OpenAI::Internal::Type::Converter.new_coerce_state + event = OpenAI::Internal::Type::Converter.coerce(@server_event_type, parsed, state: state) + if (cause = coercion_error(state)) + raise OpenAI::Errors::RealtimeProtocolError.new(data: data, cause: cause) + end + event + rescue OpenAI::Errors::RealtimeProtocolError + raise + rescue StandardError => e + raise OpenAI::Errors::RealtimeProtocolError.new(data: data, cause: e) + end + + # Validate, encode, and send a typed client event. + def send_event(event) + validate_discriminator!(event, @client_event_names, kind: "client") if event.is_a?(Hash) + state = OpenAI::Internal::Type::Converter.new_coerce_state + coerced = OpenAI::Internal::Type::Converter.coerce(@client_event_type, event, state: state) + if (cause = coercion_error(state)) + raise ArgumentError.new("Invalid Realtime client event: #{cause.message}"), cause: cause + end + payload = OpenAI::Internal::Type::Converter.dump(@client_event_type, coerced) + validate_discriminator!(payload, @client_event_names, kind: "client") + send_raw(JSON.generate(payload)) + end + + # Send an already encoded text message. + def send_raw(data) + if closed? + raise OpenAI::Errors::RealtimeConnectionError.new( + url: @url, + message: "Cannot send on a closed Realtime WebSocket." + ) + end + @socket.write(data) + nil + end + + # Close the connection. + def close(code: 1000, reason: "") + return if closed? + + @socket.close(code: code, reason: reason) + nil + end + + # @return [Boolean] + def closed? = @socket.closed? + + private def discriminator_values(union) + union.variants.to_h do |variant| + value = variant.fields.fetch(:type).fetch(:const) + [value.to_s, true] + end + end + + private def event_type(event) + unless event.is_a?(Hash) + raise ArgumentError, "Realtime server event must be a JSON object" + end + + type = event[:type] + return type if type.is_a?(String) || type.is_a?(Symbol) + + raise ArgumentError, "Realtime server event type must be a string or symbol" + end + + private def validate_discriminator!(event, allowed, kind:) + type = + if event.key?(:type) + event.fetch(:type) + elsif event.key?("type") + event.fetch("type") + end + return if type && allowed.key?(type.to_s) + + raise ArgumentError, "Unknown Realtime #{kind} event type: #{type.inspect}" + end + + private def coercion_error(state) + return state[:error] if state[:error] + return if state.fetch(:exactness).fetch(:no).zero? + + ArgumentError.new("Realtime event is missing required fields or contains invalid values") + end + end + end +end diff --git a/lib/openai/realtime/connection.rb b/lib/openai/realtime/connection.rb new file mode 100644 index 000000000..c9dd1b3d3 --- /dev/null +++ b/lib/openai/realtime/connection.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module OpenAI + module Realtime + # A live, typed Realtime WebSocket connection. + class Connection < OpenAI::Realtime::BaseConnection + # @return [OpenAI::Realtime::ConnectionResources::Session] + attr_reader :session + + # @return [OpenAI::Realtime::ConnectionResources::Response] + attr_reader :response + + # @return [OpenAI::Realtime::ConnectionResources::InputAudioBuffer] + attr_reader :input_audio_buffer + + # @return [OpenAI::Realtime::ConnectionResources::Conversation] + attr_reader :conversation + + # @api private + def initialize(socket:, url:) + super( + socket: socket, + url: url, + server_event_type: OpenAI::Realtime::RealtimeServerEvent, + client_event_type: OpenAI::Realtime::RealtimeClientEvent + ) + @session = OpenAI::Realtime::ConnectionResources::Session.new(self) + @response = OpenAI::Realtime::ConnectionResources::Response.new(self) + @input_audio_buffer = OpenAI::Realtime::ConnectionResources::InputAudioBuffer.new(self) + @conversation = OpenAI::Realtime::ConnectionResources::Conversation.new(self) + end + end + end +end diff --git a/lib/openai/realtime/connection_manager.rb b/lib/openai/realtime/connection_manager.rb new file mode 100644 index 000000000..ebb247af4 --- /dev/null +++ b/lib/openai/realtime/connection_manager.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +module OpenAI + module Realtime + # Internal block-scoped lifecycle manager for Realtime WebSocket connections. + # + # @api private + class ConnectionManager + # @api private + def initialize( + client:, + path:, + query:, + connection_class:, + transport:, + request_options:, + transport_options: + ) + @client = client + @path = path + @query = query + @connection_class = connection_class + @transport = transport + @request_options = request_options + @transport_options = transport_options + end + + # Open the WebSocket and yield a typed connection for the lifetime of the block. + # + # @yieldparam connection [OpenAI::Realtime::Connection] + # @return [Object] + def open + raise ArgumentError, "A block is required to open a Realtime WebSocket." unless block_given? + + request = @client.realtime_connection_request( + path: @path, + query: @query, + options: @request_options + ) + transport = @transport || OpenAI::Realtime::Transports::AsyncWebSocket.new + unless transport.respond_to?(:open) + raise ArgumentError, "`transport` must respond to `open`" + end + + transport.open( + url: request.fetch(:url), + headers: request.fetch(:headers), + timeout: request.fetch(:timeout), + **@transport_options + ) do |socket| + connection = @connection_class.new(socket: socket, url: request.fetch(:url)) + begin + yield(connection) + ensure + pending_error = $ERROR_INFO + begin + connection.close unless connection.closed? + rescue StandardError + raise if pending_error.nil? + end + end + end + end + end + end +end diff --git a/lib/openai/realtime/connection_resources.rb b/lib/openai/realtime/connection_resources.rb new file mode 100644 index 000000000..c070cfedb --- /dev/null +++ b/lib/openai/realtime/connection_resources.rb @@ -0,0 +1,196 @@ +# frozen_string_literal: true + +module OpenAI + module Realtime + module ConnectionResources + class Base + # @api private + def initialize(connection) + @connection = connection + end + + private def compact_event(event) = event.compact + + private def event_payload(params, *metadata_keys) + payload = params.to_h.dup + metadata = metadata_keys.to_h do |key| + value = payload.key?(key) ? payload.delete(key) : payload.delete(key.to_s) + [key, value] + end + [payload, metadata] + end + end + + class Session < Base + # Update the session using Ruby-style resource keywords. The helper adds the + # protocol's nested `session` envelope. + def update(params) + session, metadata = event_payload(params, :event_id) + @connection.send_event( + compact_event(type: :"session.update", session: session, **metadata) + ) + end + end + + class TranscriptionSession < Base + # A transcription connection already selects its session capability. Add the + # required wire discriminator so callers only pass transcription fields. + def update(params) + session, metadata = event_payload(params, :event_id) + session.delete(:type) + session.delete("type") + session[:type] = :transcription + @connection.send_event( + compact_event(type: :"session.update", session: session, **metadata) + ) + end + end + + class TranslationSession < Session + # Gracefully flush and close a translation session. + def close(event_id: nil) + @connection.send_event(compact_event(type: :"session.close", event_id: event_id)) + end + end + + class Response < Base + # Create a response from resource fields, omitting the optional wire envelope + # when no response-specific fields are supplied. + def create(params = {}) + response, metadata = event_payload(params, :event_id) + @connection.send_event( + compact_event( + type: :"response.create", + response: response.empty? ? nil : response, + **metadata + ) + ) + end + + def cancel(response_id: nil, event_id: nil) + @connection.send_event( + compact_event(type: :"response.cancel", response_id: response_id, event_id: event_id) + ) + end + end + + class InputAudioBuffer < Base + def append(audio:, event_id: nil) + @connection.send_event( + compact_event(type: :"input_audio_buffer.append", audio: audio, event_id: event_id) + ) + end + + def append_bytes(bytes, event_id: nil) + append(audio: Base64.strict_encode64(bytes), event_id: event_id) + end + + def commit(event_id: nil) + @connection.send_event(compact_event(type: :"input_audio_buffer.commit", event_id: event_id)) + end + + def clear(event_id: nil) + @connection.send_event(compact_event(type: :"input_audio_buffer.clear", event_id: event_id)) + end + end + + class TranslationInputAudioBuffer < Base + def append(audio:, event_id: nil) + @connection.send_event( + compact_event(type: :"session.input_audio_buffer.append", audio: audio, event_id: event_id) + ) + end + + def append_bytes(bytes, event_id: nil) + append(audio: Base64.strict_encode64(bytes), event_id: event_id) + end + end + + class Conversation < Base + # @return [OpenAI::Realtime::ConnectionResources::ConversationItems] + attr_reader :items + + # @api private + def initialize(connection) + super + @items = ConversationItems.new(connection) + end + end + + class ConversationItems < Base + # Create an item from resource fields. Event metadata stays at the outer wire + # level while the remaining fields become the nested item. + def create(params) + item, metadata = event_payload(params, :event_id, :previous_item_id) + @connection.send_event( + compact_event( + type: :"conversation.item.create", + item: item, + **metadata + ) + ) + end + + def delete(item_id:, event_id: nil) + @connection.send_event( + compact_event(type: :"conversation.item.delete", item_id: item_id, event_id: event_id) + ) + end + + def retrieve(item_id:, event_id: nil) + @connection.send_event( + compact_event(type: :"conversation.item.retrieve", item_id: item_id, event_id: event_id) + ) + end + + def truncate(item_id:, content_index:, audio_end_ms:, event_id: nil) + @connection.send_event( + compact_event( + type: :"conversation.item.truncate", + item_id: item_id, + content_index: content_index, + audio_end_ms: audio_end_ms, + event_id: event_id + ) + ) + end + + # Send the output of a locally executed function call back to the model. + def create_function_call_output(call_id:, output:, id: nil, status: nil, event_id: nil) + item = compact_event( + type: :function_call_output, + call_id: call_id, + output: output, + id: id, + status: status + ) + create(**item, event_id: event_id) + end + + # Approve or reject a pending remote MCP tool call. + def respond_to_mcp_approval( + approval_request_id:, + approve:, + reason: nil, + id: nil, + event_id: nil + ) + item = compact_event( + type: :mcp_approval_response, + id: id || "mcpa_#{SecureRandom.hex(12)}", + approval_request_id: approval_request_id, + approve: approve, + reason: reason + ) + create(**item, event_id: event_id) + end + end + + class OutputAudioBuffer < Base + def clear(event_id: nil) + @connection.send_event(compact_event(type: :"output_audio_buffer.clear", event_id: event_id)) + end + end + end + end +end diff --git a/lib/openai/realtime/sideband_connection.rb b/lib/openai/realtime/sideband_connection.rb new file mode 100644 index 000000000..177e4f7db --- /dev/null +++ b/lib/openai/realtime/sideband_connection.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module OpenAI + module Realtime + # A server-side control connection attached to a WebRTC or SIP call. + class SidebandConnection < OpenAI::Realtime::Connection + # @return [OpenAI::Realtime::ConnectionResources::OutputAudioBuffer] + attr_reader :output_audio_buffer + + # @api private + def initialize(socket:, url:) + super + @output_audio_buffer = OpenAI::Realtime::ConnectionResources::OutputAudioBuffer.new(self) + end + end + end +end diff --git a/lib/openai/realtime/transcription_connection.rb b/lib/openai/realtime/transcription_connection.rb new file mode 100644 index 000000000..0d0a906cb --- /dev/null +++ b/lib/openai/realtime/transcription_connection.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module OpenAI + module Realtime + # A live connection for streaming transcription sessions. + class TranscriptionConnection < OpenAI::Realtime::BaseConnection + # @return [OpenAI::Realtime::ConnectionResources::TranscriptionSession] + attr_reader :session + + # @return [OpenAI::Realtime::ConnectionResources::InputAudioBuffer] + attr_reader :input_audio_buffer + + # @api private + def initialize(socket:, url:) + super( + socket: socket, + url: url, + server_event_type: OpenAI::Realtime::RealtimeServerEvent, + client_event_type: OpenAI::Realtime::RealtimeClientEvent + ) + @session = OpenAI::Realtime::ConnectionResources::TranscriptionSession.new(self) + @input_audio_buffer = OpenAI::Realtime::ConnectionResources::InputAudioBuffer.new(self) + end + end + end +end diff --git a/lib/openai/realtime/translation_connection.rb b/lib/openai/realtime/translation_connection.rb new file mode 100644 index 000000000..4f089a852 --- /dev/null +++ b/lib/openai/realtime/translation_connection.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module OpenAI + module Realtime + # A live connection for the dedicated continuous translation protocol. + class TranslationConnection < OpenAI::Realtime::BaseConnection + # @return [OpenAI::Realtime::ConnectionResources::TranslationSession] + attr_reader :session + + # @return [OpenAI::Realtime::ConnectionResources::TranslationInputAudioBuffer] + attr_reader :input_audio_buffer + + # @api private + def initialize(socket:, url:) + super( + socket: socket, + url: url, + server_event_type: OpenAI::Realtime::RealtimeTranslationServerEvent, + client_event_type: OpenAI::Realtime::RealtimeTranslationClientEvent + ) + @session = OpenAI::Realtime::ConnectionResources::TranslationSession.new(self) + @input_audio_buffer = OpenAI::Realtime::ConnectionResources::TranslationInputAudioBuffer.new(self) + end + end + end +end diff --git a/lib/openai/realtime/transports/async_websocket.rb b/lib/openai/realtime/transports/async_websocket.rb new file mode 100644 index 000000000..15c5d3fcd --- /dev/null +++ b/lib/openai/realtime/transports/async_websocket.rb @@ -0,0 +1,80 @@ +# frozen_string_literal: true + +module OpenAI + module Realtime + module Transports + # Default Realtime transport backed by the optional async-websocket gem. + class AsyncWebSocket + class Socket + # @api private + def initialize(connection, url:) + @connection = connection + @url = url + end + + def read + @connection.read + rescue StandardError => e + raise OpenAI::Errors::RealtimeConnectionError.new(url: @url, cause: e) + end + + def write(message) + @connection.write(message) + @connection.flush + rescue StandardError => e + raise OpenAI::Errors::RealtimeConnectionError.new(url: @url, cause: e) + end + + def close(code: 1000, reason: "") + @connection.close(code, reason) + rescue StandardError => e + raise OpenAI::Errors::RealtimeConnectionError.new(url: @url, cause: e) + end + + def closed? = @connection.closed? + end + + def open(url:, headers:, timeout:, **endpoint_options) + load_dependencies(url) + + # Classic WebSocket negotiation uses HTTP/1.1. Pinning ALPN also avoids an + # HTTP/2 selection on servers that advertise both protocols. + options = { + alpn_protocols: ::Async::HTTP::Protocol::HTTP11.names, + **endpoint_options + } + endpoint = ::Async::HTTP::Endpoint.parse( + url.to_s, + timeout: timeout, + **options + ) + block_error = nil + # Client.connect owns the Sync boundary: it starts a reactor for ordinary + # synchronous callers and reuses the current task when one already exists. + ::Async::WebSocket::Client.connect(endpoint, headers: headers) do |connection| + yield(Socket.new(connection, url: url)) + rescue StandardError => e + block_error = e + raise + end + rescue OpenAI::Errors::RealtimeConnectionError + raise + rescue StandardError => e + raise if e.equal?(block_error) + + raise OpenAI::Errors::RealtimeConnectionError.new(url: url, cause: e) + end + + private def load_dependencies(url) + require("async/websocket/client") + require("async/http/endpoint") + rescue LoadError => e + message = + "Realtime WebSockets require the `async-websocket` gem. " \ + "Add `gem \"async-websocket\"` to your Gemfile." + raise OpenAI::Errors::RealtimeConnectionError.new(url: url, message: message, cause: e) + end + end + end + end +end diff --git a/lib/openai/realtime/unknown_server_event.rb b/lib/openai/realtime/unknown_server_event.rb new file mode 100644 index 000000000..99a1818d4 --- /dev/null +++ b/lib/openai/realtime/unknown_server_event.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +module OpenAI + module Realtime + # A valid JSON event whose discriminator is newer than this SDK version. + class UnknownServerEvent + # @return [Symbol] + attr_reader :type + + # @return [Hash{Symbol=>Object}] + attr_reader :data + + # @api private + def initialize(data:) + value = data.fetch(:type) + unless value.is_a?(String) || value.is_a?(Symbol) + raise ArgumentError, "Realtime server event type must be a string or symbol" + end + + @type = value.to_sym + @data = freeze_json(data) + freeze + end + + # @return [Hash{Symbol=>Object}] + def to_h = @data + + private def freeze_json(value) + case value + when Hash + value.each do |key, item| + freeze_json(key) + freeze_json(item) + end + when Array + value.each { |item| freeze_json(item) } + end + value.freeze + end + end + end +end diff --git a/lib/openai/resources/realtime.rb b/lib/openai/resources/realtime.rb index f361d152a..a5f606525 100644 --- a/lib/openai/resources/realtime.rb +++ b/lib/openai/resources/realtime.rb @@ -9,6 +9,105 @@ class Realtime # @return [OpenAI::Resources::Realtime::Calls] attr_reader :calls + # @return [OpenAI::Resources::Realtime::Translations] + attr_reader :translations + + # Open a server-side Realtime conversation WebSocket. A block is required and + # the socket is closed on every exit path. + # + # @param model [String] + # @param request_options [OpenAI::RequestOptions, Hash{Symbol=>Object}, nil] + # @param transport [#open, nil] An alternate WebSocket transport. + # @param transport_options [Hash{Symbol=>Object}] + # @yieldparam connection [OpenAI::Realtime::Connection] + # @return [Object] + def connect( + model:, + request_options: nil, + transport: nil, + transport_options: {}, + &block + ) + open_connection( + query: {"model" => model}, + connection_class: OpenAI::Realtime::Connection, + request_options: request_options, + transport: transport, + transport_options: transport_options, + &block + ) + end + + # Attach a sideband WebSocket to an existing WebRTC or SIP call. + # + # @param call_id [String] + # @param request_options [OpenAI::RequestOptions, Hash{Symbol=>Object}, nil] + # @param transport [#open, nil] An alternate WebSocket transport. + # @param transport_options [Hash{Symbol=>Object}] + # @yieldparam connection [OpenAI::Realtime::SidebandConnection] + # @return [Object] + def connect_to_call( + call_id:, + request_options: nil, + transport: nil, + transport_options: {}, + &block + ) + open_connection( + query: {"call_id" => call_id}, + connection_class: OpenAI::Realtime::SidebandConnection, + request_options: request_options, + transport: transport, + transport_options: transport_options, + &block + ) + end + + # Open a dedicated Realtime transcription WebSocket. + # + # @param request_options [OpenAI::RequestOptions, Hash{Symbol=>Object}, nil] + # @param transport [#open, nil] An alternate WebSocket transport. + # @param transport_options [Hash{Symbol=>Object}] + # @yieldparam connection [OpenAI::Realtime::TranscriptionConnection] + # @return [Object] + def connect_transcription( + request_options: nil, + transport: nil, + transport_options: {}, + &block + ) + open_connection( + query: {"intent" => "transcription"}, + connection_class: OpenAI::Realtime::TranscriptionConnection, + request_options: request_options, + transport: transport, + transport_options: transport_options, + &block + ) + end + + private def open_connection( + query:, + connection_class:, + request_options:, + transport:, + transport_options:, + &block + ) + raise ArgumentError, "A block is required to open a Realtime WebSocket." unless block + + manager = OpenAI::Realtime::ConnectionManager.new( + client: @client, + path: "realtime", + query: query, + connection_class: connection_class, + transport: transport, + request_options: request_options, + transport_options: transport_options + ) + manager.open(&block) + end + # @api private # # @param client [OpenAI::Client] @@ -16,6 +115,7 @@ def initialize(client:) @client = client @client_secrets = OpenAI::Resources::Realtime::ClientSecrets.new(client: client) @calls = OpenAI::Resources::Realtime::Calls.new(client: client) + @translations = OpenAI::Resources::Realtime::Translations.new(client: client) end end end diff --git a/lib/openai/resources/realtime/calls.rb b/lib/openai/resources/realtime/calls.rb index a0cfb6f4d..ef06734da 100644 --- a/lib/openai/resources/realtime/calls.rb +++ b/lib/openai/resources/realtime/calls.rb @@ -4,6 +4,61 @@ module OpenAI module Resources class Realtime class Calls + # Create a WebRTC call from an SDP offer. When `session` is supplied, the SDK + # sends the offer and session configuration as typed multipart form parts. + # + # @overload create(sdp:, session: nil, request_options: {}) + # + # @param sdp [String] The WebRTC Session Description Protocol offer. + # @param session [OpenAI::Models::Realtime::RealtimeSessionCreateRequest] Optional session configuration. + # @param request_options [OpenAI::RequestOptions, Hash{Symbol=>Object}, nil] + # + # @return [OpenAI::Models::Realtime::CallCreateResponse] + # + # @see OpenAI::Models::Realtime::CallCreateParams + def create(params) + parsed, options = OpenAI::Realtime::CallCreateParams.dump_request(params) + sdp = parsed.fetch(:sdp) + session = parsed[:session] + headers, body = + if session.nil? + [{"accept" => "application/sdp", "content-type" => "application/sdp"}, sdp] + else + [ + {"accept" => "application/sdp", "content-type" => "multipart/form-data"}, + { + sdp: OpenAI::FilePart.new(sdp, content_type: "application/sdp"), + session: OpenAI::FilePart.new(JSON.generate(session), content_type: "application/json") + } + ] + end + + response = @client.request_raw( + method: :post, + path: "realtime/calls", + headers: headers, + body: body, + security: {bearer_auth: true}, + options: options + ) + location = response.headers["location"] + call_id = call_id_from_location(location) + OpenAI::Realtime::CallCreateResponse.new( + sdp: response.body.to_a.join, + call_id: call_id, + headers: response.headers + )._set_last_response(response.metadata) + end + + # A Location header is helpful metadata, but the SDP answer remains usable + # when an intermediary supplies a malformed value. + private def call_id_from_location(location) + path = location&.then { URI.parse(_1).path } + path&.split("/")&.last + rescue URI::InvalidURIError + nil + end + # Some parameter documentations has been truncated, see # {OpenAI::Models::Realtime::CallAcceptParams} for more details. # diff --git a/lib/openai/resources/realtime/translations.rb b/lib/openai/resources/realtime/translations.rb new file mode 100644 index 000000000..3f8e81ffc --- /dev/null +++ b/lib/openai/resources/realtime/translations.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module OpenAI + module Resources + class Realtime + # Dedicated Realtime translation endpoints and WebSocket connections. + class Translations + # @return [OpenAI::Resources::Realtime::Translations::ClientSecrets] + attr_reader :client_secrets + + # @return [OpenAI::Resources::Realtime::Translations::Calls] + attr_reader :calls + + # Open a typed translation WebSocket connection. + # + # @param model [String] + # @param request_options [OpenAI::RequestOptions, Hash{Symbol=>Object}, nil] + # @param transport [#open, nil] + # @param transport_options [Hash{Symbol=>Object}] + # @yieldparam connection [OpenAI::Realtime::TranslationConnection] + # @return [Object] + def connect(model:, request_options: nil, transport: nil, transport_options: {}, &block) + raise ArgumentError, "A block is required to open a Realtime WebSocket." unless block + + manager = OpenAI::Realtime::ConnectionManager.new( + client: @client, + path: "realtime/translations", + query: {"model" => model}, + connection_class: OpenAI::Realtime::TranslationConnection, + transport: transport, + request_options: request_options, + transport_options: transport_options + ) + manager.open(&block) + end + + # @api private + def initialize(client:) + @client = client + @client_secrets = ClientSecrets.new(client: client) + @calls = Calls.new(client: client) + end + end + end + end +end diff --git a/lib/openai/resources/realtime/translations/calls.rb b/lib/openai/resources/realtime/translations/calls.rb new file mode 100644 index 000000000..132db5d0f --- /dev/null +++ b/lib/openai/resources/realtime/translations/calls.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module OpenAI + module Resources + class Realtime + class Translations + class Calls + # Create a translation WebRTC call from an SDP offer. + # + # @param sdp [String] + # @param request_options [OpenAI::RequestOptions, Hash{Symbol=>Object}, nil] + # @return [OpenAI::Models::Realtime::CallCreateResponse] + def create(sdp:, request_options: nil) + @client.realtime.calls.create( + sdp: sdp, + request_options: request_options + ) + end + + # @api private + def initialize(client:) + @client = client + end + end + end + end + end +end diff --git a/lib/openai/resources/realtime/translations/client_secrets.rb b/lib/openai/resources/realtime/translations/client_secrets.rb new file mode 100644 index 000000000..d3c59d3eb --- /dev/null +++ b/lib/openai/resources/realtime/translations/client_secrets.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +module OpenAI + module Resources + class Realtime + class Translations + class ClientSecrets + # Create a short-lived client secret for a Realtime translation session. + # + # @param session [OpenAI::Models::Realtime::RealtimeTranslationSessionCreateRequest, Hash] + # @param expires_after [OpenAI::Models::Realtime::RealtimeTranslationClientSecretCreateRequest::ExpiresAfter, Hash, nil] + # @param request_options [OpenAI::RequestOptions, Hash{Symbol=>Object}, nil] + # @return [OpenAI::Models::Realtime::RealtimeTranslationClientSecretCreateResponse] + def create(session:, expires_after: nil, request_options: nil) + input = {session: session} + input[:expires_after] = expires_after unless expires_after.nil? + state = OpenAI::Internal::Type::Converter.new_coerce_state + request = OpenAI::Internal::Type::Converter.coerce( + OpenAI::Realtime::RealtimeTranslationClientSecretCreateRequest, + input, + state: state + ) + cause = state[:error] + if cause.nil? && !state.fetch(:exactness).fetch(:no).zero? + cause = ArgumentError.new("request is missing required fields or contains invalid values") + end + if cause + message = "Invalid translation client secret request: #{cause.message}" + raise ArgumentError.new(message), cause: cause + end + body = OpenAI::Internal::Type::Converter.dump( + OpenAI::Realtime::RealtimeTranslationClientSecretCreateRequest, + request + ) + + @client.request( + method: :post, + path: "realtime/translations/client_secrets", + body: body, + model: OpenAI::Realtime::RealtimeTranslationClientSecretCreateResponse, + security: {bearer_auth: true}, + options: request_options + ) + end + + # @api private + def initialize(client:) + @client = client + end + end + end + end + end +end diff --git a/openai.gemspec b/openai.gemspec index 27122f711..517393174 100644 --- a/openai.gemspec +++ b/openai.gemspec @@ -24,7 +24,7 @@ Gem::Specification.new do |s| "CHANGELOG.md", ".ignore" ] - s.extra_rdoc_files = ["README.md", "VERSIONING.md"] + s.extra_rdoc_files = ["README.md", "VERSIONING.md", "realtime.md"] s.add_dependency "base64" s.add_dependency "cgi" s.add_dependency "connection_pool", ">= 2.2.3" diff --git a/rbi/openai/client.rbi b/rbi/openai/client.rbi index 95e4ec364..71b893cf1 100644 --- a/rbi/openai/client.rbi +++ b/rbi/openai/client.rbi @@ -25,6 +25,9 @@ module OpenAI sig { returns(T.nilable(String)) } attr_reader :webhook_secret + sig { returns(T.nilable(URI::Generic)) } + attr_reader :websocket_base_url + # Given a prompt, the model will return one or more predicted completions, and can # also return the probabilities of alternative tokens at each position. sig { returns(OpenAI::Resources::Completions) } @@ -135,6 +138,17 @@ module OpenAI private def admin_api_key_auth end + # @api private + sig do + params( + path: String, + query: T::Hash[String, String], + options: T.nilable(OpenAI::RequestOptions::OrHash) + ).returns(OpenAI::Internal::Transport::BaseClient::RequestInput) + end + def realtime_connection_request(path:, query:, options: nil) + end + # @api private sig do override @@ -160,6 +174,7 @@ module OpenAI provider: T.nilable(OpenAI::Provider), base_url: T.nilable(String), default_headers: T.nilable(T::Hash[String, T.nilable(String)]), + websocket_base_url: T.nilable(String), max_retries: Integer, timeout: T.nilable(Float), initial_retry_delay: Float, @@ -189,6 +204,7 @@ module OpenAI # Extra headers to send with every request. Explicit values override # `ENV["OPENAI_CUSTOM_HEADERS"]`. default_headers: nil, + websocket_base_url: nil, # Max number of retries to attempt after a failed retryable request. max_retries: OpenAI::Client::DEFAULT_MAX_RETRIES, timeout: OpenAI::Client::DEFAULT_TIMEOUT_IN_SECONDS, diff --git a/rbi/openai/errors.rbi b/rbi/openai/errors.rbi index 5307238e2..d955c8164 100644 --- a/rbi/openai/errors.rbi +++ b/rbi/openai/errors.rbi @@ -3,7 +3,7 @@ module OpenAI module Errors class Error < StandardError - sig { returns(T.nilable(StandardError)) } + sig { returns(T.nilable(Exception)) } attr_accessor :cause end @@ -26,6 +26,44 @@ module OpenAI end end + class RealtimeConnectionError < OpenAI::Errors::Error + sig { returns(URI::Generic) } + attr_reader :url + + sig { returns(T.nilable(Exception)) } + def cause + end + + sig do + params( + url: URI::Generic, + message: T.nilable(String), + cause: T.nilable(Exception) + ).returns(T.attached_class) + end + def self.new(url:, message: nil, cause: nil) + end + end + + class RealtimeProtocolError < OpenAI::Errors::Error + sig { returns(String) } + attr_reader :data + + sig { returns(T.nilable(StandardError)) } + def cause + end + + sig do + params( + data: String, + message: T.nilable(String), + cause: T.nilable(StandardError) + ).returns(T.attached_class) + end + def self.new(data:, message: nil, cause: nil) + end + end + class APIError < OpenAI::Errors::Error sig { returns(URI::Generic) } attr_accessor :url diff --git a/rbi/openai/internal/logging.rbi b/rbi/openai/internal/logging.rbi index 9a0b7df71..e0902b4ee 100644 --- a/rbi/openai/internal/logging.rbi +++ b/rbi/openai/internal/logging.rbi @@ -77,6 +77,14 @@ module OpenAI def observe_stream(stream, response:) end + sig do + params(response: OpenAI::HTTPClient::Response).returns( + OpenAI::HTTPClient::Response + ) + end + def observe_raw_response(response) + end + sig { params(error: StandardError).void } def request_failed(error) end @@ -140,6 +148,35 @@ module OpenAI end end + class ObservedEnumerable + include Enumerable + + Elem = type_member(:out) + + sig do + params( + enumerable: T::Enumerable[Elem], + context: OpenAI::Internal::Logging::Context, + response: OpenAI::HTTPClient::Response, + close: T.proc.void + ).void + end + def initialize(enumerable:, context:, response:, close:) + end + + sig { params(block: T.untyped).returns(T.untyped) } + def each(&block) + end + + sig { void } + def close + end + + sig { returns(T::Enumerable[Elem]) } + private def iterator + end + end + class << self sig { params(value: T.any(Symbol, String)).returns(Symbol) } def normalize_level(value) diff --git a/rbi/openai/internal/transport/base_client.rbi b/rbi/openai/internal/transport/base_client.rbi index 627dc659f..7465ea40c 100644 --- a/rbi/openai/internal/transport/base_client.rbi +++ b/rbi/openai/internal/transport/base_client.rbi @@ -302,6 +302,15 @@ module OpenAI ) end + # @api private + sig do + params( + req: OpenAI::Internal::Transport::BaseClient::RequestComponents + ).returns(OpenAI::HTTPClient::Response) + end + def request_raw(req) + end + # Execute the request specified by `req`. This is the method that all resource # methods call into. # diff --git a/rbi/openai/models/realtime/call_create_params.rbi b/rbi/openai/models/realtime/call_create_params.rbi new file mode 100644 index 000000000..ab5ff8a41 --- /dev/null +++ b/rbi/openai/models/realtime/call_create_params.rbi @@ -0,0 +1,49 @@ +# typed: strong + +module OpenAI + module Models + module Realtime + class CallCreateParams < OpenAI::Internal::Type::BaseModel + extend OpenAI::Internal::Type::RequestParameters::Converter + include OpenAI::Internal::Type::RequestParameters + + OrHash = + T.type_alias do + T.any(OpenAI::Realtime::CallCreateParams, OpenAI::Internal::AnyHash) + end + + sig { returns(String) } + attr_accessor :sdp + + sig do + returns(T.nilable(OpenAI::Realtime::RealtimeSessionCreateRequest)) + end + attr_accessor :session + + sig do + params( + sdp: String, + session: + T.nilable(OpenAI::Realtime::RealtimeSessionCreateRequest::OrHash), + request_options: OpenAI::RequestOptions::OrHash + ).returns(T.attached_class) + end + def self.new(sdp:, session: nil, request_options: {}) + end + + sig do + override.returns( + { + sdp: String, + session: + T.nilable(OpenAI::Realtime::RealtimeSessionCreateRequest), + request_options: OpenAI::RequestOptions + } + ) + end + def to_hash + end + end + end + end +end diff --git a/rbi/openai/models/realtime/call_create_response.rbi b/rbi/openai/models/realtime/call_create_response.rbi new file mode 100644 index 000000000..5e1fa9645 --- /dev/null +++ b/rbi/openai/models/realtime/call_create_response.rbi @@ -0,0 +1,48 @@ +# typed: strong + +module OpenAI + module Models + module Realtime + class CallCreateResponse < OpenAI::Internal::Type::BaseModel + OrHash = + T.type_alias do + T.any( + OpenAI::Realtime::CallCreateResponse, + OpenAI::Internal::AnyHash + ) + end + + sig { returns(String) } + attr_accessor :sdp + + sig { returns(T.nilable(String)) } + attr_accessor :call_id + + sig { returns(T::Hash[String, String]) } + attr_accessor :headers + + sig do + params( + sdp: String, + headers: T::Hash[String, String], + call_id: T.nilable(String) + ).returns(T.attached_class) + end + def self.new(sdp:, headers:, call_id: nil) + end + + sig do + override.returns( + { + sdp: String, + call_id: T.nilable(String), + headers: T::Hash[String, String] + } + ) + end + def to_hash + end + end + end + end +end diff --git a/rbi/openai/realtime/base_connection.rbi b/rbi/openai/realtime/base_connection.rbi new file mode 100644 index 000000000..1b1b515f5 --- /dev/null +++ b/rbi/openai/realtime/base_connection.rbi @@ -0,0 +1,40 @@ +# typed: strong + +module OpenAI + module Models + module Realtime + class BaseConnection + sig { returns(URI::Generic) } + attr_reader :url + + # @api private + sig do + params( + socket: T.untyped, + url: URI::Generic, + server_event_type: T.untyped, + client_event_type: T.untyped + ).returns(T.attached_class) + end + def self.new(socket:, url:, server_event_type:, client_event_type:) + end + + sig { returns(T.nilable(String)) } + def receive_raw + end + + sig { params(data: String).void } + def send_raw(data) + end + + sig { params(code: Integer, reason: String).void } + def close(code: 1000, reason: "") + end + + sig { returns(T::Boolean) } + def closed? + end + end + end + end +end diff --git a/rbi/openai/realtime/connection.rbi b/rbi/openai/realtime/connection.rbi new file mode 100644 index 000000000..0a9e05d5e --- /dev/null +++ b/rbi/openai/realtime/connection.rbi @@ -0,0 +1,70 @@ +# typed: strong + +module OpenAI + module Models + module Realtime + class Connection < OpenAI::Realtime::BaseConnection + include Enumerable + + ServerEvent = + T.type_alias do + T.any( + OpenAI::Realtime::RealtimeServerEvent::Variants, + OpenAI::Realtime::UnknownServerEvent + ) + end + + ClientEvent = + T.type_alias do + T.any( + OpenAI::Realtime::RealtimeClientEvent::Variants, + OpenAI::Internal::AnyHash + ) + end + + Elem = type_member { { fixed: ServerEvent } } + + sig { returns(OpenAI::Realtime::ConnectionResources::Session) } + attr_reader :session + + sig { returns(OpenAI::Realtime::ConnectionResources::Response) } + attr_reader :response + + sig { returns(OpenAI::Realtime::ConnectionResources::InputAudioBuffer) } + attr_reader :input_audio_buffer + + sig { returns(OpenAI::Realtime::ConnectionResources::Conversation) } + attr_reader :conversation + + # @api private + sig do + params(socket: T.untyped, url: URI::Generic).returns(T.attached_class) + end + def self.new(socket:, url:) + end + + sig do + params( + block: T.nilable(T.proc.params(event: ServerEvent).void) + ).returns( + T.any(OpenAI::Realtime::Connection, T::Enumerator[ServerEvent]) + ) + end + def each(&block) + end + + sig { returns(T.nilable(ServerEvent)) } + def receive + end + + sig { params(data: String).returns(ServerEvent) } + def parse_event(data) + end + + sig { params(event: ClientEvent).void } + def send_event(event) + end + end + end + end +end diff --git a/rbi/openai/realtime/connection_manager.rbi b/rbi/openai/realtime/connection_manager.rbi new file mode 100644 index 000000000..5af34758a --- /dev/null +++ b/rbi/openai/realtime/connection_manager.rbi @@ -0,0 +1,44 @@ +# typed: strong + +module OpenAI + module Models + module Realtime + class ConnectionManager + # @api private + sig do + params( + client: OpenAI::Client, + path: String, + query: T::Hash[String, String], + connection_class: T.class_of(OpenAI::Realtime::BaseConnection), + transport: T.untyped, + request_options: T.nilable(OpenAI::RequestOptions::OrHash), + transport_options: T::Hash[Symbol, T.untyped] + ).returns(T.attached_class) + end + def self.new( + client:, + path:, + query:, + connection_class:, + transport:, + request_options:, + transport_options: + ) + end + + sig do + params( + block: + T + .proc + .params(connection: OpenAI::Realtime::BaseConnection) + .returns(T.untyped) + ).returns(T.untyped) + end + def open(&block) + end + end + end + end +end diff --git a/rbi/openai/realtime/connection_resources.rbi b/rbi/openai/realtime/connection_resources.rbi new file mode 100644 index 000000000..1d74ea96b --- /dev/null +++ b/rbi/openai/realtime/connection_resources.rbi @@ -0,0 +1,328 @@ +# typed: strong + +module OpenAI + module Models + module Realtime + module ConnectionResources + class Base + # @api private + sig do + params(connection: OpenAI::Realtime::BaseConnection).returns( + T.attached_class + ) + end + def self.new(connection) + end + end + + class Session < Base + sig do + params( + type: Symbol, + audio: + T.any( + OpenAI::Realtime::RealtimeAudioConfig::OrHash, + OpenAI::Realtime::RealtimeTranscriptionSessionAudio::OrHash + ), + include: T::Array[Symbol], + instructions: String, + max_output_tokens: T.any(Integer, Symbol), + model: T.any(String, Symbol), + output_modalities: T::Array[Symbol], + parallel_tool_calls: T::Boolean, + prompt: T.nilable(OpenAI::Responses::ResponsePrompt::OrHash), + reasoning: OpenAI::Realtime::RealtimeReasoning::OrHash, + tool_choice: + T.any( + OpenAI::Responses::ToolChoiceOptions::OrSymbol, + OpenAI::Responses::ToolChoiceFunction::OrHash, + OpenAI::Responses::ToolChoiceMcp::OrHash + ), + tools: + T::Array[ + T.any( + OpenAI::Realtime::RealtimeFunctionTool::OrHash, + OpenAI::Realtime::RealtimeToolsConfigUnion::Mcp::OrHash + ) + ], + tracing: + T.nilable( + T.any( + Symbol, + OpenAI::Realtime::RealtimeTracingConfig::TracingConfiguration::OrHash + ) + ), + truncation: + T.any( + OpenAI::Realtime::RealtimeTruncation::RealtimeTruncationStrategy::OrSymbol, + OpenAI::Realtime::RealtimeTruncationRetentionRatio::OrHash + ), + event_id: T.nilable(String) + ).void + end + def update( + type:, + audio: nil, + include: nil, + instructions: nil, + max_output_tokens: nil, + model: nil, + output_modalities: nil, + parallel_tool_calls: nil, + prompt: nil, + reasoning: nil, + tool_choice: nil, + tools: nil, + tracing: nil, + truncation: nil, + event_id: nil + ) + end + end + + class TranscriptionSession < Base + sig do + params( + audio: + OpenAI::Realtime::RealtimeTranscriptionSessionAudio::OrHash, + include: + T::Array[ + OpenAI::Realtime::RealtimeTranscriptionSessionCreateRequest::Include::OrSymbol + ], + event_id: T.nilable(String) + ).void + end + def update(audio: nil, include: nil, event_id: nil) + end + end + + class TranslationSession < Session + sig do + params( + audio: + OpenAI::Realtime::RealtimeTranslationSessionUpdateRequest::Audio::OrHash, + event_id: T.nilable(String) + ).void + end + def update(audio: nil, event_id: nil) + end + + sig { params(event_id: T.nilable(String)).void } + def close(event_id: nil) + end + end + + class Response < Base + sig do + params( + audio: OpenAI::Realtime::RealtimeResponseCreateAudioOutput::OrHash, + conversation: T.any(String, Symbol), + input: + T::Array[ + T.any( + OpenAI::Realtime::RealtimeConversationItemSystemMessage::OrHash, + OpenAI::Realtime::RealtimeConversationItemUserMessage::OrHash, + OpenAI::Realtime::RealtimeConversationItemAssistantMessage::OrHash, + OpenAI::Realtime::RealtimeConversationItemFunctionCall::OrHash, + OpenAI::Realtime::RealtimeConversationItemFunctionCallOutput::OrHash, + OpenAI::Realtime::RealtimeMcpApprovalResponse::OrHash, + OpenAI::Realtime::RealtimeMcpListTools::OrHash, + OpenAI::Realtime::RealtimeMcpToolCall::OrHash, + OpenAI::Realtime::RealtimeMcpApprovalRequest::OrHash + ) + ], + instructions: String, + max_output_tokens: T.any(Integer, Symbol), + metadata: T.nilable(T::Hash[Symbol, String]), + output_modalities: T::Array[Symbol], + parallel_tool_calls: T::Boolean, + prompt: T.nilable(OpenAI::Responses::ResponsePrompt::OrHash), + reasoning: OpenAI::Realtime::RealtimeReasoning::OrHash, + tool_choice: + T.any( + OpenAI::Responses::ToolChoiceOptions::OrSymbol, + OpenAI::Responses::ToolChoiceFunction::OrHash, + OpenAI::Responses::ToolChoiceMcp::OrHash + ), + tools: + T::Array[ + T.any( + OpenAI::Realtime::RealtimeFunctionTool::OrHash, + OpenAI::Realtime::RealtimeResponseCreateMcpTool::OrHash + ) + ], + event_id: T.nilable(String) + ).void + end + def create( + audio: nil, + conversation: nil, + input: nil, + instructions: nil, + max_output_tokens: nil, + metadata: nil, + output_modalities: nil, + parallel_tool_calls: nil, + prompt: nil, + reasoning: nil, + tool_choice: nil, + tools: nil, + event_id: nil + ) + end + + sig do + params( + response_id: T.nilable(String), + event_id: T.nilable(String) + ).void + end + def cancel(response_id: nil, event_id: nil) + end + end + + class InputAudioBuffer < Base + sig { params(audio: String, event_id: T.nilable(String)).void } + def append(audio:, event_id: nil) + end + + sig { params(bytes: String, event_id: T.nilable(String)).void } + def append_bytes(bytes, event_id: nil) + end + + sig { params(event_id: T.nilable(String)).void } + def commit(event_id: nil) + end + + sig { params(event_id: T.nilable(String)).void } + def clear(event_id: nil) + end + end + + class TranslationInputAudioBuffer < Base + sig { params(audio: String, event_id: T.nilable(String)).void } + def append(audio:, event_id: nil) + end + + sig { params(bytes: String, event_id: T.nilable(String)).void } + def append_bytes(bytes, event_id: nil) + end + end + + class Conversation < Base + sig do + returns(OpenAI::Realtime::ConnectionResources::ConversationItems) + end + attr_reader :items + end + + class ConversationItems < Base + sig do + params( + type: Symbol, + arguments: String, + approval_request_id: T.nilable(String), + approve: T::Boolean, + call_id: String, + content: T::Array[OpenAI::Internal::AnyHash], + error: T.untyped, + id: String, + name: String, + object: Symbol, + output: T.nilable(String), + reason: T.nilable(String), + role: Symbol, + server_label: String, + status: Symbol, + tools: T::Array[OpenAI::Realtime::RealtimeMcpListTools::Tool::OrHash], + event_id: T.nilable(String), + previous_item_id: T.nilable(String) + ).void + end + def create( + type:, + arguments: nil, + approval_request_id: nil, + approve: nil, + call_id: nil, + content: nil, + error: nil, + id: nil, + name: nil, + object: nil, + output: nil, + reason: nil, + role: nil, + server_label: nil, + status: nil, + tools: nil, + event_id: nil, + previous_item_id: nil + ) + end + + sig { params(item_id: String, event_id: T.nilable(String)).void } + def delete(item_id:, event_id: nil) + end + + sig { params(item_id: String, event_id: T.nilable(String)).void } + def retrieve(item_id:, event_id: nil) + end + + sig do + params( + item_id: String, + content_index: Integer, + audio_end_ms: Integer, + event_id: T.nilable(String) + ).void + end + def truncate(item_id:, content_index:, audio_end_ms:, event_id: nil) + end + + sig do + params( + call_id: String, + output: String, + id: T.nilable(String), + status: T.nilable(T.any(String, Symbol)), + event_id: T.nilable(String) + ).void + end + def create_function_call_output( + call_id:, + output:, + id: nil, + status: nil, + event_id: nil + ) + end + + sig do + params( + approval_request_id: String, + approve: T::Boolean, + reason: T.nilable(String), + id: T.nilable(String), + event_id: T.nilable(String) + ).void + end + def respond_to_mcp_approval( + approval_request_id:, + approve:, + reason: nil, + id: nil, + event_id: nil + ) + end + end + + class OutputAudioBuffer < Base + sig { params(event_id: T.nilable(String)).void } + def clear(event_id: nil) + end + end + end + end + end +end diff --git a/rbi/openai/realtime/sideband_connection.rbi b/rbi/openai/realtime/sideband_connection.rbi new file mode 100644 index 000000000..2e0e0c304 --- /dev/null +++ b/rbi/openai/realtime/sideband_connection.rbi @@ -0,0 +1,24 @@ +# typed: strong + +module OpenAI + module Models + module Realtime + class SidebandConnection < OpenAI::Realtime::Connection + Elem = + type_member { { fixed: OpenAI::Realtime::Connection::ServerEvent } } + + sig do + returns(OpenAI::Realtime::ConnectionResources::OutputAudioBuffer) + end + attr_reader :output_audio_buffer + + # @api private + sig do + params(socket: T.untyped, url: URI::Generic).returns(T.attached_class) + end + def self.new(socket:, url:) + end + end + end + end +end diff --git a/rbi/openai/realtime/transcription_connection.rbi b/rbi/openai/realtime/transcription_connection.rbi new file mode 100644 index 000000000..ed6feb5a2 --- /dev/null +++ b/rbi/openai/realtime/transcription_connection.rbi @@ -0,0 +1,71 @@ +# typed: strong + +module OpenAI + module Models + module Realtime + class TranscriptionConnection < OpenAI::Realtime::BaseConnection + include Enumerable + + ServerEvent = + T.type_alias do + T.any( + OpenAI::Realtime::RealtimeServerEvent::Variants, + OpenAI::Realtime::UnknownServerEvent + ) + end + + ClientEvent = + T.type_alias do + T.any( + OpenAI::Realtime::RealtimeClientEvent::Variants, + OpenAI::Internal::AnyHash + ) + end + + Elem = type_member { { fixed: ServerEvent } } + + sig do + returns(OpenAI::Realtime::ConnectionResources::TranscriptionSession) + end + attr_reader :session + + sig do + returns(OpenAI::Realtime::ConnectionResources::InputAudioBuffer) + end + attr_reader :input_audio_buffer + + # @api private + sig do + params(socket: T.untyped, url: URI::Generic).returns(T.attached_class) + end + def self.new(socket:, url:) + end + + sig do + params( + block: T.nilable(T.proc.params(event: ServerEvent).void) + ).returns( + T.any( + OpenAI::Realtime::TranscriptionConnection, + T::Enumerator[ServerEvent] + ) + ) + end + def each(&block) + end + + sig { returns(T.nilable(ServerEvent)) } + def receive + end + + sig { params(data: String).returns(ServerEvent) } + def parse_event(data) + end + + sig { params(event: ClientEvent).void } + def send_event(event) + end + end + end + end +end diff --git a/rbi/openai/realtime/translation_connection.rbi b/rbi/openai/realtime/translation_connection.rbi new file mode 100644 index 000000000..77ef0e0c2 --- /dev/null +++ b/rbi/openai/realtime/translation_connection.rbi @@ -0,0 +1,73 @@ +# typed: strong + +module OpenAI + module Models + module Realtime + class TranslationConnection < OpenAI::Realtime::BaseConnection + include Enumerable + + ServerEvent = + T.type_alias do + T.any( + OpenAI::Realtime::RealtimeTranslationServerEvent::Variants, + OpenAI::Realtime::UnknownServerEvent + ) + end + + ClientEvent = + T.type_alias do + T.any( + OpenAI::Realtime::RealtimeTranslationClientEvent::Variants, + OpenAI::Internal::AnyHash + ) + end + + Elem = type_member { { fixed: ServerEvent } } + + sig do + returns(OpenAI::Realtime::ConnectionResources::TranslationSession) + end + attr_reader :session + + sig do + returns( + OpenAI::Realtime::ConnectionResources::TranslationInputAudioBuffer + ) + end + attr_reader :input_audio_buffer + + # @api private + sig do + params(socket: T.untyped, url: URI::Generic).returns(T.attached_class) + end + def self.new(socket:, url:) + end + + sig do + params( + block: T.nilable(T.proc.params(event: ServerEvent).void) + ).returns( + T.any( + OpenAI::Realtime::TranslationConnection, + T::Enumerator[ServerEvent] + ) + ) + end + def each(&block) + end + + sig { returns(T.nilable(ServerEvent)) } + def receive + end + + sig { params(data: String).returns(ServerEvent) } + def parse_event(data) + end + + sig { params(event: ClientEvent).void } + def send_event(event) + end + end + end + end +end diff --git a/rbi/openai/realtime/transports/async_websocket.rbi b/rbi/openai/realtime/transports/async_websocket.rbi new file mode 100644 index 000000000..6869175c2 --- /dev/null +++ b/rbi/openai/realtime/transports/async_websocket.rbi @@ -0,0 +1,50 @@ +# typed: strong + +module OpenAI + module Models + module Realtime + module Transports + class AsyncWebSocket + class Socket + # @api private + sig do + params(connection: T.untyped, url: URI::Generic).returns( + T.attached_class + ) + end + def self.new(connection, url:) + end + + sig { returns(T.untyped) } + def read + end + + sig { params(message: String).void } + def write(message) + end + + sig { params(code: Integer, reason: String).void } + def close(code: 1000, reason: "") + end + + sig { returns(T::Boolean) } + def closed? + end + end + + sig do + params( + url: URI::Generic, + headers: T::Hash[String, String], + timeout: T.nilable(Float), + endpoint_options: T.untyped, + block: T.proc.params(socket: Socket).returns(T.untyped) + ).returns(T.untyped) + end + def open(url:, headers:, timeout:, **endpoint_options, &block) + end + end + end + end + end +end diff --git a/rbi/openai/realtime/unknown_server_event.rbi b/rbi/openai/realtime/unknown_server_event.rbi new file mode 100644 index 000000000..3147d4ae9 --- /dev/null +++ b/rbi/openai/realtime/unknown_server_event.rbi @@ -0,0 +1,25 @@ +# typed: strong + +module OpenAI + module Models + module Realtime + class UnknownServerEvent + sig { returns(Symbol) } + attr_reader :type + + sig { returns(T::Hash[Symbol, T.untyped]) } + attr_reader :data + + sig do + params(data: T::Hash[Symbol, T.untyped]).returns(T.attached_class) + end + def self.new(data:) + end + + sig { returns(T::Hash[Symbol, T.untyped]) } + def to_h + end + end + end + end +end diff --git a/rbi/openai/resources/realtime.rbi b/rbi/openai/resources/realtime.rbi index 93c144442..9927621b4 100644 --- a/rbi/openai/resources/realtime.rbi +++ b/rbi/openai/resources/realtime.rbi @@ -9,6 +9,44 @@ module OpenAI sig { returns(OpenAI::Resources::Realtime::Calls) } attr_reader :calls + sig { returns(OpenAI::Resources::Realtime::Translations) } + attr_reader :translations + + sig do + params( + model: String, + request_options: T.nilable(OpenAI::RequestOptions::OrHash), + transport: T.untyped, + transport_options: T::Hash[Symbol, T.untyped], + block: T.proc.params(connection: OpenAI::Realtime::Connection).returns(T.untyped) + ).returns(T.untyped) + end + def connect(model:, request_options: nil, transport: nil, transport_options: {}, &block) + end + + sig do + params( + call_id: String, + request_options: T.nilable(OpenAI::RequestOptions::OrHash), + transport: T.untyped, + transport_options: T::Hash[Symbol, T.untyped], + block: T.proc.params(connection: OpenAI::Realtime::SidebandConnection).returns(T.untyped) + ).returns(T.untyped) + end + def connect_to_call(call_id:, request_options: nil, transport: nil, transport_options: {}, &block) + end + + sig do + params( + request_options: T.nilable(OpenAI::RequestOptions::OrHash), + transport: T.untyped, + transport_options: T::Hash[Symbol, T.untyped], + block: T.proc.params(connection: OpenAI::Realtime::TranscriptionConnection).returns(T.untyped) + ).returns(T.untyped) + end + def connect_transcription(request_options: nil, transport: nil, transport_options: {}, &block) + end + # @api private sig { params(client: OpenAI::Client).returns(T.attached_class) } def self.new(client:) diff --git a/rbi/openai/resources/realtime/calls.rbi b/rbi/openai/resources/realtime/calls.rbi index 8133bcf03..0c2539d9e 100644 --- a/rbi/openai/resources/realtime/calls.rbi +++ b/rbi/openai/resources/realtime/calls.rbi @@ -4,6 +4,17 @@ module OpenAI module Resources class Realtime class Calls + sig do + params( + sdp: String, + session: + T.nilable(OpenAI::Realtime::RealtimeSessionCreateRequest::OrHash), + request_options: OpenAI::RequestOptions::OrHash + ).returns(OpenAI::Realtime::CallCreateResponse) + end + def create(sdp:, session: nil, request_options: {}) + end + # Accept an incoming SIP call and configure the realtime session that will handle # it. sig do diff --git a/rbi/openai/resources/realtime/translations.rbi b/rbi/openai/resources/realtime/translations.rbi new file mode 100644 index 000000000..016650be4 --- /dev/null +++ b/rbi/openai/resources/realtime/translations.rbi @@ -0,0 +1,44 @@ +# typed: strong + +module OpenAI + module Resources + class Realtime + class Translations + sig do + returns(OpenAI::Resources::Realtime::Translations::ClientSecrets) + end + attr_reader :client_secrets + + sig { returns(OpenAI::Resources::Realtime::Translations::Calls) } + attr_reader :calls + + sig do + params( + model: String, + request_options: T.nilable(OpenAI::RequestOptions::OrHash), + transport: T.untyped, + transport_options: T::Hash[Symbol, T.untyped], + block: + T + .proc + .params(connection: OpenAI::Realtime::TranslationConnection) + .returns(T.untyped) + ).returns(T.untyped) + end + def connect( + model:, + request_options: nil, + transport: nil, + transport_options: {}, + &block + ) + end + + # @api private + sig { params(client: OpenAI::Client).returns(T.attached_class) } + def self.new(client:) + end + end + end + end +end diff --git a/rbi/openai/resources/realtime/translations/calls.rbi b/rbi/openai/resources/realtime/translations/calls.rbi new file mode 100644 index 000000000..c0fb9d2c3 --- /dev/null +++ b/rbi/openai/resources/realtime/translations/calls.rbi @@ -0,0 +1,25 @@ +# typed: strong + +module OpenAI + module Resources + class Realtime + class Translations + class Calls + sig do + params( + sdp: String, + request_options: T.nilable(OpenAI::RequestOptions::OrHash) + ).returns(OpenAI::Realtime::CallCreateResponse) + end + def create(sdp:, request_options: nil) + end + + # @api private + sig { params(client: OpenAI::Client).returns(T.attached_class) } + def self.new(client:) + end + end + end + end + end +end diff --git a/rbi/openai/resources/realtime/translations/client_secrets.rbi b/rbi/openai/resources/realtime/translations/client_secrets.rbi new file mode 100644 index 000000000..3b3d0d2e3 --- /dev/null +++ b/rbi/openai/resources/realtime/translations/client_secrets.rbi @@ -0,0 +1,32 @@ +# typed: strong + +module OpenAI + module Resources + class Realtime + class Translations + class ClientSecrets + sig do + params( + session: + OpenAI::Realtime::RealtimeTranslationSessionCreateRequest::OrHash, + expires_after: + T.nilable( + OpenAI::Realtime::RealtimeTranslationClientSecretCreateRequest::ExpiresAfter::OrHash + ), + request_options: T.nilable(OpenAI::RequestOptions::OrHash) + ).returns( + OpenAI::Realtime::RealtimeTranslationClientSecretCreateResponse + ) + end + def create(session:, expires_after: nil, request_options: nil) + end + + # @api private + sig { params(client: OpenAI::Client).returns(T.attached_class) } + def self.new(client:) + end + end + end + end + end +end diff --git a/realtime.md b/realtime.md new file mode 100644 index 000000000..124c90a76 --- /dev/null +++ b/realtime.md @@ -0,0 +1,444 @@ +# Realtime API + +The Ruby SDK supports the server-side parts of the Realtime API: + +- typed WebSocket sessions for text, audio, transcription, function calls, and remote MCP; +- WebRTC SDP negotiation from a trusted Ruby backend; +- sideband WebSocket control for active WebRTC and SIP calls; +- SIP accept, reject, refer, and hangup controls; and +- dedicated WebSocket, client-secret, and WebRTC translation endpoints. + +Ruby does not provide a broadly deployed, standard WebRTC media stack. The SDK +therefore handles the backend SDP exchange and call controls while browsers or a +specialized media service own `RTCPeerConnection`, microphone capture, codecs, +jitter buffering, echo cancellation, and playback. + +## Installation + +HTTP and WebRTC session-negotiation methods use the base SDK. WebSockets use an +optional adapter so applications that only use HTTP do not acquire an event-loop +dependency: + +```ruby +gem "openai" +gem "async-websocket" +``` + +The adapter is fiber-scheduler-aware and works both inside and outside an +existing Async reactor. You can instead inject an object implementing the small +transport contract described below. + +## Start a WebSocket session + +`connect` is block-scoped. This is the safest lifecycle in Rails jobs, +Rack servers, CLI programs, and long-running workers because normal returns and +exceptions both close the socket. + +```ruby +client = OpenAI::Client.new + +client.realtime.connect(model: "gpt-realtime-2.1") do |connection| + connection.session.update( + type: :realtime, + output_modalities: [:text], + instructions: "Be concise." + ) + + connection.conversation.items.create( + type: :message, + role: :user, + content: [{type: :input_text, text: "Hello"}] + ) + connection.response.create + + connection.each do |event| + case event + when OpenAI::Realtime::ResponseTextDeltaEvent + print(event.delta) + when OpenAI::Realtime::ResponseDoneEvent + raise "Response ended with #{event.response.status}" unless event.response.status == :completed + break + when OpenAI::Realtime::RealtimeErrorEvent + raise event.error.message + end + end +end +``` + +To exercise that flow against the live service with visible lifecycle output, +set `OPENAI_API_KEY` and run: + +```console +$ bundle exec ruby examples/realtime/websocket_text.rb +``` + +Set `OPENAI_REALTIME_MODEL`, `OPENAI_REALTIME_PROMPT`, or +`OPENAI_REALTIME_TIMEOUT` to override the example defaults. + +`connect` requires a block. The yielded connection is valid only for the lifetime +of that block and is always closed when the block exits, including exceptional exits. + +The connection exposes `receive`, `receive_raw`, `parse_event`, `send_event`, +`send_raw`, `each`, `close`, and `closed?`. The resource +helpers construct and validate the common client events: + +- `session.update`; +- `response.create` and `response.cancel`; +- `input_audio_buffer.append`, `append_bytes`, `commit`, and `clear`; +- sideband-only `output_audio_buffer.clear`; and +- conversation item create, retrieve, truncate, delete, function-call output, + and MCP approval response. + +`append` accepts API-ready Base64. `append_bytes` accepts binary audio and uses +strict Base64 encoding. Sending is synchronous, and iteration reads one message +at a time, so the caller naturally applies backpressure instead of filling an +unbounded SDK queue. A connection supports one reader fiber and one writer +fiber at the same time, which is useful for continuous audio. Do not run +multiple readers or multiple writers against the same connection. +The hands-free WebSocket example uses a bounded outbound queue so microphone +audio and interruption-driven truncation events share exactly one writer fiber. + +The yielded class reflects the protocol's capabilities: + +- `connect(model:)` yields `OpenAI::Realtime::Connection` for ordinary sessions; +- `connect_transcription` yields + `OpenAI::Realtime::TranscriptionConnection`, exposing only session and input + audio buffer operations; +- `connect_to_call(call_id:)` yields `OpenAI::Realtime::SidebandConnection`, adding + `output_audio_buffer`; and +- `translations.connect` yields `OpenAI::Realtime::TranslationConnection`, + which intentionally has no conversation or response lifecycle. + +The resource helpers accept the resource fields directly and add the wire +envelope internally: use `session.update(type: ...)`, +`response.create(instructions: ...)`, and `conversation.items.create(type: ...)`. +Use `send_event` only when an application deliberately needs an exact protocol +event hash. + +## Stream transcription + +Realtime transcription is a distinct session mode. Configure a transcription +model, append raw mono 24 kHz PCM16 input, and commit the buffer: + +```ruby +client.realtime.connect_transcription do |connection| + connection.session.update( + audio: { + input: { + format: {type: :"audio/pcm", rate: 24_000}, + transcription: {model: "gpt-live-transcribe"}, + turn_detection: nil + } + } + ) + + # Wait for SessionUpdatedEvent before streaming input. + connection.input_audio_buffer.append_bytes(pcm_chunk) + connection.input_audio_buffer.commit + + connection.each do |event| + case event + when OpenAI::Realtime::ConversationItemInputAudioTranscriptionDeltaEvent + print(event.delta) + when OpenAI::Realtime::ConversationItemInputAudioTranscriptionCompletedEvent + puts event.transcript + end + end +end +``` + +Completion events for different turns are not guaranteed to arrive in input +order, so correlate deltas and completions with `item_id`. The dedicated +connection uses the service's `intent=transcription` handshake and adds the +session's `type: :transcription` discriminator internally; the transcription +model belongs under `audio.input.transcription.model`. Run the complete live sample with +`REALTIME_INPUT_PCM=input.pcm bundle exec ruby +examples/realtime/websocket_transcription.rb`. + +## Create a WebRTC call + +WebRTC media is intentionally not a Ruby SDK responsibility. In a browser or +mobile voice application, the client owns `RTCPeerConnection`, microphone +permissions, audio playback, jitter buffering, and acoustic echo cancellation. +Ruby owns the trusted-server control plane: API credentials, session +configuration, SDP negotiation, sideband tools and policy, and call lifecycle. +The SDK therefore supports WebRTC session negotiation and server controls; it +does not attempt to expose a Ruby WebRTC peer API. + +Use `calls.create` in the trusted application server that receives a browser's +SDP offer: + +```ruby +call = client.realtime.calls.create( + sdp: browser_offer, + session: { + type: :realtime, + model: "gpt-realtime-2.1", + audio: {output: {voice: :marin}} + }, + request_options: { + extra_headers: {"OpenAI-Safety-Identifier" => hashed_user_id} + } +) + +render plain: call.sdp, content_type: "application/sdp" +``` + +The result preserves `sdp`, normalized response `headers`, and `call_id` parsed +from the optional `Location` header. A missing or malformed `Location` leaves +`call_id` as `nil` without discarding an otherwise valid SDP answer. Keep the +call ID if the application will open a server-side sideband connection: + +```ruby +client.realtime.connect_to_call(call_id: call.call_id) do |connection| + connection.session.update( + type: :realtime, + instructions: "Apply private server policy." + ) + connection.each { |event| audit(event) } +end +``` + +The SDK sends raw `application/sdp` when only an offer is provided; use that +form with a client configured with a previously minted ephemeral session token. +With a standard server API key, provide `session:` and the SDK sends multipart +form data with correctly typed `sdp` and `session` parts. It never exposes a +standard API key to browser code. + +## SIP calls + +Verify incoming webhooks before using their call ID, accept or reject the call, +then use the same sideband connection API: + +```ruby +event = client.webhooks.unwrap(raw_body, request_headers) +return unless event.is_a?(OpenAI::Webhooks::RealtimeCallIncomingWebhookEvent) + +call_id = event.data.call_id +client.realtime.calls.accept( + call_id, + type: :realtime, + model: "gpt-realtime-2.1", + instructions: "You are answering a phone call." +) + +client.realtime.connect_to_call(call_id: call_id) do |connection| + connection.each { |realtime_event| handle(realtime_event) } +end +``` + +`calls.reject`, `calls.refer`, and `calls.hangup` provide the remaining SIP and +call-lifecycle controls. + +## Translation + +Translation has dedicated endpoints and distinct typed event unions: + +```ruby +require "async" + +client.realtime.translations.connect(model: "gpt-realtime-translate") do |connection| + connection.session.update(audio: {output: {language: "es"}}) + + reader = Async do + connection.each do |event| + case event + when OpenAI::Realtime::RealtimeTranslationOutputTranscriptDeltaEvent + print(event.delta) + when OpenAI::Realtime::RealtimeTranslationOutputAudioDeltaEvent + play(Base64.strict_decode64(event.delta)) + when OpenAI::Realtime::RealtimeTranslationSessionClosedEvent + break + end + end + end + + while (chunk = pcm16_source.read(9_600)) + connection.input_audio_buffer.append_bytes(chunk) + end + connection.session.close + reader.wait +end +``` + +Call `session.close` after the last audio chunk and continue reading until the +server sends `session.closed`; closing the socket immediately can discard output +that is still draining. For browser WebRTC translation, mint an ephemeral secret +with `translations.client_secrets.create`, then post the browser offer with +`translations.calls.create`. That convenience method delegates to the shared +`/v1/realtime/calls` endpoint; translation is selected by the ephemeral secret, +not by a separate calls route. + +## Function calls and MCP approvals + +Function call output and MCP approval responses are ordinary conversation items. +The helpers supply the required item shape and generate an MCP response ID when +one is not supplied: + +```ruby +connection.conversation.items.create_function_call_output( + call_id: function_call.call_id, + output: JSON.generate(result) +) + +connection.conversation.items.respond_to_mcp_approval( + approval_request_id: approval_item.id, + approve: policy.allows?(approval_item), + reason: "Evaluated by application policy" +) +``` + +Remote MCP servers are configured through the typed `tools` field in +`session.update` or `response.create`; all MCP progress, approval, and completion +events decode through `RealtimeServerEvent`. Before a dependent prompt, wait for +both `McpListToolsCompleted` and the `ConversationItemDone` event whose +`RealtimeMcpListTools` item contains the imported tool names. If you then update +`tool_choice`, wait for `SessionUpdatedEvent` before creating the response. + +The first completed `ResponseDoneEvent` can finish the MCP argument-generation +phase while approval and tool execution are still pending. Keep reading, answer +the `RealtimeMcpApprovalRequest`, wait for `ResponseMcpCallCompleted`, and create +a follow-up response (usually with `tool_choice: :none`) to turn the tool output +into a final assistant answer. Treat only that follow-up response's completed +`ResponseDoneEvent` as success; clean EOF during discovery, approval, or tool +execution is still an incomplete workflow. The approval helper generates item +IDs within the service's 32-character limit. + +## Lifecycle and failures + +The SDK does not automatically reconnect or replay events. Realtime sessions are +stateful, and replaying audio, item creation, tool output, or approval decisions +can duplicate side effects. Treat an unexpected close as an application-level +decision: terminate, or reconnect and deliberately rebuild the state that is +safe for that use case. + +Runnable smoke tests require their protocol-specific terminal event rather than +treating a clean WebSocket EOF as success: completed `response.done` for text, +audio, and MCP; transcription completion for transcription; `session.closed` +for translation; and the requested `OPENAI_REALTIME_STOP_AFTER` checkpoint for +bounded sideband, SIP, and conversation runs. Cleanup must also preserve an +active upload or processing error when a graceful close fails, while surfacing +the close failure after an otherwise successful operation. + +Terminal status alone is insufficient when a workflow promises an artifact: +the text smoke requires a non-empty text delta, while the raw-audio and +translation smokes require at least one decoded audio byte before their terminal +events. The file-audio smoke disables VAD and waits for the acknowledged session +update before its manual commit. Translation reader failures cancel an in-flight +upload rather than waiting for the input file to drain; an already-buffered +reader failure prevents the uploader from starting at all. Microphone shutdown +is latched before capture starts, so an immediately closed connection cannot +start an orphaned ffmpeg process after cleanup has begun. + +The three standard Realtime connection modes use distinct method names rather +than a keyword-discriminated overload: `connect(model:)`, `connect_to_call`, and +`connect_transcription`. This keeps block parameter types precise in both RBS +and RBI; Sorbet does not support overloads that discriminate on keyword +arguments. + +Raw WebRTC SDP responses remain lazily consumed. Request observability follows +that body lifecycle: completion is recorded only after the SDP body is fully +drained, and a body read failure records `request failed` without a contradictory +completion event. + +- Invalid JSON or a payload that cannot match the selected event union raises + `OpenAI::Errors::RealtimeProtocolError`, with `data` and `cause`. +- Handshake and socket I/O failures from the default adapter raise + `OpenAI::Errors::RealtimeConnectionError`, with `url` and `cause`. +- API `error` events remain typed `RealtimeErrorEvent` values in the normal event + stream so applications can follow the API's recoverability guidance. +- Valid events introduced after the installed SDK version remain observable as + immutable `UnknownServerEvent` values instead of terminating the session. +- Exceptions raised by the application block propagate unchanged; cleanup does + not replace them with a close error. + +Use `request_options` for handshake headers, query parameters, and timeout: + +```ruby +client.realtime.connect( + model: "gpt-realtime-2.1", + request_options: { + timeout: 30, + extra_headers: {"OpenAI-Safety-Identifier" => hashed_user_id}, + extra_query: {trace: "enabled"} + } +) { |connection| run_session(connection) } +``` + +`websocket_base_url:` configures a separate gateway for WebSockets. Azure +provider clients derive the WebSocket URL from the configured Azure v1 endpoint +and resolve provider-owned API-key or bearer authentication immediately before +the handshake. Providers without a Realtime WebSocket surface fail before a +credential is sent. + +## Custom transport + +Pass `transport:` to `connect` to replace the adapter. The object implements: + +```ruby +transport.open(url:, headers:, timeout:, **transport_options) do |socket| + # socket.read # String-like message or nil + # socket.write(json_string) # synchronous write/backpressure + # socket.close(code:, reason:) + # socket.closed? +end +``` + +`timeout` is `Float?` at the transport boundary because `Client.new(timeout: +nil)` and per-request `timeout: nil` deliberately disable the handshake timeout. + +The SDK owns URL construction, authentication, typed JSON encoding/decoding, and +block cleanup. The transport owns the WebSocket handshake, frame I/O, TLS, +proxies, and transport-specific failures. Internal calls use direct methods; the +contract does not require subclassing or reflection. + +## Example and developer-site coverage + +Repository examples live under `examples/realtime/`; their +[runbook](examples/realtime/README.md) includes prerequisites, commands, bounded +smoke-test controls, and observable pass criteria. They cover a hands-free +full-duplex voice conversation, text, raw PCM audio, streaming transcription, +WebRTC SDP exchange, sideband, SIP, MCP approvals, and translation. The +developer website should expose the same progression so users encounter the +simplest successful path before protocol details: + +| Guide surface | Ruby sample required | +| --- | --- | +| Realtime overview | Minimal text WebSocket with typed deltas and block cleanup | +| WebSocket | Text, PCM16 input/output, manual `receive`, Enumerable iteration, error handling | +| Transcription | `connect_transcription`, PCM16 streaming, delta/completed events, and `item_id` correlation | +| WebRTC | Complete browser peer plus Ruby endpoint; Rails and Sinatra variants using `calls.create` | +| Server controls | Preserve `call.call_id`, open sideband, update session, hang up | +| SIP | Verify webhook, accept/reject, sideband, refer, hang up, idempotent webhook handling | +| Conversations | Text item, audio commit, cancel/interruption, truncate played audio | +| Function calling | Detect function call, execute locally, send typed output item | +| Remote MCP | Configure server, inspect approval request, approve/reject, observe completion | +| Translation | Client secret, browser WebRTC call, server WebSocket PCM loop, graceful close | +| Authentication | Standard key on server, ephemeral browser secret, safety identifier | +| Azure | Deployment/model targeting and provider-owned handshake authentication | +| Operations | Timeout, backpressure, logging event IDs, abnormal close, deliberate reconnect policy | + +The WebRTC browser half remains JavaScript because Ruby runs on the trusted +server; Ruby snippets should own secrets, session configuration, SDP proxying, +webhook verification, tools, and business logic. + +The local voice example is also exercised against the installed FFmpeg tools, +not only Ruby fakes. In particular, raw PCM playback declares mono input with +ffplay's `-ch_layout mono`; `-ac 1` is an ffmpeg transcoding option and current +ffplay releases reject it. Playback also disables ffplay's terminal statistics: +without `-nostats`, ffplay emits an ANSI clear-line sequence that can erase the +first transcript delta from the terminal. Playback-command compatibility and a +headless PCM process smoke test guard both boundaries. + +WebSocket barge-in requires client playback state in addition to server VAD. +On `input_audio_buffer.speech_started`, the voice example stops ffplay, records +the elapsed audio offset, sends `conversation.item.truncate`, and ignores queued +audio and transcript deltas for the cancelled response. A cancelled +`response.done` is an expected interruption outcome rather than an exception. + +The FFmpeg WebSocket loop cannot provide acoustic echo cancellation and must +not be positioned as the default laptop conversation sample. With speakers, +model output can re-enter the microphone and repeatedly trigger VAD, producing +one-word responses. The primary interactive voice example uses browser WebRTC; +the Ruby WebSocket loop remains useful for headless server audio, telephony +bridges, deterministic PCM tests, and protocol debugging. diff --git a/sig/openai/client.rbs b/sig/openai/client.rbs index a65e15abc..87903612e 100644 --- a/sig/openai/client.rbs +++ b/sig/openai/client.rbs @@ -20,6 +20,8 @@ module OpenAI attr_reader webhook_secret: String? + attr_reader websocket_base_url: URI::Generic? + attr_reader completions: OpenAI::Resources::Completions attr_reader chat: OpenAI::Resources::Chat @@ -76,6 +78,12 @@ module OpenAI private def admin_api_key_auth: -> ::Hash[String, String] + def realtime_connection_request: ( + path: String, + query: ::Hash[String, String], + ?options: OpenAI::request_opts? + ) -> OpenAI::Internal::Transport::BaseClient::request_input + private def prepare_request: ( OpenAI::Internal::Transport::BaseClient::request_input request, redirect_count: Integer, @@ -92,6 +100,7 @@ module OpenAI ?provider: OpenAI::Provider?, ?base_url: String?, ?default_headers: ::Hash[String, String?]?, + ?websocket_base_url: String?, ?max_retries: Integer, ?timeout: Float?, ?initial_retry_delay: Float, diff --git a/sig/openai/errors.rbs b/sig/openai/errors.rbs index aa1428662..e38e6fe62 100644 --- a/sig/openai/errors.rbs +++ b/sig/openai/errors.rbs @@ -1,7 +1,7 @@ module OpenAI module Errors class Error < StandardError - attr_accessor cause: StandardError? + attr_accessor cause: Exception? end class ConversionError < OpenAI::Errors::Error @@ -16,6 +16,28 @@ module OpenAI ) -> void end + class RealtimeConnectionError < OpenAI::Errors::Error + attr_reader url: URI::Generic + def cause: -> Exception? + + def initialize: ( + url: URI::Generic, + ?message: String?, + ?cause: Exception? + ) -> void + end + + class RealtimeProtocolError < OpenAI::Errors::Error + attr_reader data: String + def cause: -> StandardError? + + def initialize: ( + data: String, + ?message: String?, + ?cause: StandardError? + ) -> void + end + class APIError < OpenAI::Errors::Error attr_accessor url: URI::Generic diff --git a/sig/openai/internal/logging.rbs b/sig/openai/internal/logging.rbs index aa0b8755f..20f1ce0c2 100644 --- a/sig/openai/internal/logging.rbs +++ b/sig/openai/internal/logging.rbs @@ -38,6 +38,9 @@ module OpenAI OpenAI::Internal::Type::BaseStream[top, Elem] stream, response: OpenAI::HTTPClient::Response ) -> OpenAI::Internal::Type::BaseStream[top, Elem] + def observe_raw_response: ( + OpenAI::HTTPClient::Response response + ) -> OpenAI::HTTPClient::Response def request_failed: (StandardError error) -> void def response_body: ( String body, @@ -74,6 +77,21 @@ module OpenAI private def iterator: -> Enumerable[Elem] end + class ObservedEnumerable[Elem] + include Enumerable[Elem] + + def initialize: ( + enumerable: Enumerable[Elem], + context: OpenAI::Internal::Logging::Context, + response: OpenAI::HTTPClient::Response, + close: ^() -> void + ) -> void + + def each: { (Elem) -> void } -> void + def close: -> void + private def iterator: -> Enumerable[Elem] + end + def self.normalize_level: (Symbol | String value) -> Symbol def self.validate_logger!: (OpenAI::_Logger? logger) -> void def self.default_logger: -> OpenAI::_Logger diff --git a/sig/openai/internal/transport/base_client.rbs b/sig/openai/internal/transport/base_client.rbs index 1b5517583..b717419ee 100644 --- a/sig/openai/internal/transport/base_client.rbs +++ b/sig/openai/internal/transport/base_client.rbs @@ -143,6 +143,10 @@ module OpenAI -> OpenAI::Internal::Logging::Context } -> OpenAI::HTTPClient::Response + def request_raw: ( + OpenAI::Internal::Transport::BaseClient::request_components req + ) -> OpenAI::HTTPClient::Response + def request: ( Symbol method, String | ::Array[String] path, diff --git a/sig/openai/models/realtime/call_create_params.rbs b/sig/openai/models/realtime/call_create_params.rbs new file mode 100644 index 000000000..0f9d91aaf --- /dev/null +++ b/sig/openai/models/realtime/call_create_params.rbs @@ -0,0 +1,32 @@ +module OpenAI + module Models + module Realtime + type call_create_params = + { + sdp: String, + session: OpenAI::Realtime::RealtimeSessionCreateRequest? + } + & OpenAI::Internal::Type::request_parameters + + class CallCreateParams < OpenAI::Internal::Type::BaseModel + extend OpenAI::Internal::Type::RequestParameters::Converter + include OpenAI::Internal::Type::RequestParameters + + attr_accessor sdp: String + attr_accessor session: OpenAI::Realtime::RealtimeSessionCreateRequest? + + def initialize: ( + sdp: String, + ?session: OpenAI::Realtime::RealtimeSessionCreateRequest, + ?request_options: OpenAI::request_opts + ) -> void + + def to_hash: -> { + sdp: String, + session: OpenAI::Realtime::RealtimeSessionCreateRequest?, + request_options: OpenAI::RequestOptions + } + end + end + end +end diff --git a/sig/openai/models/realtime/call_create_response.rbs b/sig/openai/models/realtime/call_create_response.rbs new file mode 100644 index 000000000..ea05d00be --- /dev/null +++ b/sig/openai/models/realtime/call_create_response.rbs @@ -0,0 +1,26 @@ +module OpenAI + module Models + module Realtime + type call_create_response = + { sdp: String, call_id: String?, headers: ::Hash[String, String] } + + class CallCreateResponse < OpenAI::Internal::Type::BaseModel + attr_accessor sdp: String + attr_accessor call_id: String? + attr_accessor headers: ::Hash[String, String] + + def initialize: ( + sdp: String, + headers: ::Hash[String, String], + ?call_id: String? + ) -> void + + def to_hash: -> { + sdp: String, + call_id: String?, + headers: ::Hash[String, String] + } + end + end + end +end diff --git a/sig/openai/realtime/base_connection.rbs b/sig/openai/realtime/base_connection.rbs new file mode 100644 index 000000000..2e6d1660b --- /dev/null +++ b/sig/openai/realtime/base_connection.rbs @@ -0,0 +1,20 @@ +module OpenAI + module Models + module Realtime + class BaseConnection + attr_reader url: URI::Generic + + def initialize: ( + socket: untyped, + url: URI::Generic, + server_event_type: untyped, + client_event_type: untyped + ) -> void + def receive_raw: -> String? + def send_raw: (String data) -> nil + def close: (?code: Integer, ?reason: String) -> nil + def closed?: -> bool + end + end + end +end diff --git a/sig/openai/realtime/connection.rbs b/sig/openai/realtime/connection.rbs new file mode 100644 index 000000000..52caffcfa --- /dev/null +++ b/sig/openai/realtime/connection.rbs @@ -0,0 +1,36 @@ +module OpenAI + module Models + module Realtime + type connection_server_event = + OpenAI::Models::Realtime::realtime_server_event + | OpenAI::Realtime::UnknownServerEvent + + type connection_client_event = + OpenAI::Models::Realtime::realtime_client_event + | ::Hash[Symbol | String, untyped] + + class Connection < OpenAI::Realtime::BaseConnection + include Enumerable[OpenAI::Models::Realtime::connection_server_event] + + attr_reader session: OpenAI::Realtime::ConnectionResources::Session + attr_reader response: OpenAI::Realtime::ConnectionResources::Response + attr_reader input_audio_buffer: OpenAI::Realtime::ConnectionResources::InputAudioBuffer + attr_reader conversation: OpenAI::Realtime::ConnectionResources::Conversation + + def initialize: (socket: untyped, url: URI::Generic) -> void + def each: + -> Enumerator[OpenAI::Models::Realtime::connection_server_event, self] + | { + (OpenAI::Models::Realtime::connection_server_event event) -> void + } -> self + def receive: -> OpenAI::Models::Realtime::connection_server_event? + def parse_event: ( + String data + ) -> OpenAI::Models::Realtime::connection_server_event + def send_event: ( + OpenAI::Models::Realtime::connection_client_event event + ) -> nil + end + end + end +end diff --git a/sig/openai/realtime/connection_manager.rbs b/sig/openai/realtime/connection_manager.rbs new file mode 100644 index 000000000..1813a520e --- /dev/null +++ b/sig/openai/realtime/connection_manager.rbs @@ -0,0 +1,21 @@ +module OpenAI + module Models + module Realtime + class ConnectionManager + def initialize: ( + client: OpenAI::Client, + path: String, + query: ::Hash[String, String], + connection_class: Class, + transport: untyped, + request_options: OpenAI::request_opts?, + transport_options: ::Hash[Symbol, untyped] + ) -> void + + def open: { + (OpenAI::Realtime::BaseConnection connection) -> top + } -> top + end + end + end +end diff --git a/sig/openai/realtime/connection_resources.rbs b/sig/openai/realtime/connection_resources.rbs new file mode 100644 index 000000000..ab570b8d3 --- /dev/null +++ b/sig/openai/realtime/connection_resources.rbs @@ -0,0 +1,136 @@ +module OpenAI + module Models + module Realtime + module ConnectionResources + class Base + def initialize: (OpenAI::Realtime::BaseConnection connection) -> void + end + + class Session < Base + def update: ( + type: Symbol, + ?audio: OpenAI::Models::Realtime::realtime_audio_config + | OpenAI::Models::Realtime::realtime_transcription_session_audio, + ?include: ::Array[Symbol], + ?instructions: String, + ?max_output_tokens: Integer | Symbol, + ?model: String | Symbol, + ?output_modalities: ::Array[Symbol], + ?parallel_tool_calls: bool, + ?prompt: OpenAI::Models::Responses::response_prompt?, + ?reasoning: OpenAI::Models::Realtime::realtime_reasoning, + ?tool_choice: OpenAI::Models::Realtime::realtime_tool_choice_config, + ?tools: OpenAI::Models::Realtime::realtime_tools_config, + ?tracing: OpenAI::Models::Realtime::realtime_tracing_config?, + ?truncation: OpenAI::Models::Realtime::realtime_truncation, + ?event_id: String? + ) -> nil + end + + class TranscriptionSession < Base + def update: ( + ?audio: OpenAI::Models::Realtime::realtime_transcription_session_audio, + ?include: ::Array[OpenAI::Models::Realtime::RealtimeTranscriptionSessionCreateRequest::include_], + ?event_id: String? + ) -> nil + end + + class TranslationSession < Session + def update: ( + ?audio: OpenAI::Realtime::RealtimeTranslationSessionUpdateRequest::Audio + | ::Hash[Symbol | String, untyped], + ?event_id: String? + ) -> nil + def close: (?event_id: String?) -> nil + end + + class Response < Base + def create: ( + ?audio: OpenAI::Models::Realtime::realtime_response_create_audio_output, + ?conversation: String | Symbol, + ?input: ::Array[OpenAI::Models::Realtime::conversation_item], + ?instructions: String, + ?max_output_tokens: Integer | Symbol, + ?metadata: ::Hash[Symbol, String]?, + ?output_modalities: ::Array[Symbol], + ?parallel_tool_calls: bool, + ?prompt: OpenAI::Models::Responses::response_prompt?, + ?reasoning: OpenAI::Models::Realtime::realtime_reasoning, + ?tool_choice: OpenAI::Models::Realtime::RealtimeResponseCreateParams::tool_choice, + ?tools: ::Array[ + OpenAI::Models::Realtime::realtime_function_tool + | OpenAI::Models::Realtime::realtime_response_create_mcp_tool + ], + ?event_id: String? + ) -> nil + def cancel: (?response_id: String?, ?event_id: String?) -> nil + end + + class InputAudioBuffer < Base + def append: (audio: String, ?event_id: String?) -> nil + def append_bytes: (String bytes, ?event_id: String?) -> nil + def commit: (?event_id: String?) -> nil + def clear: (?event_id: String?) -> nil + end + + class TranslationInputAudioBuffer < Base + def append: (audio: String, ?event_id: String?) -> nil + def append_bytes: (String bytes, ?event_id: String?) -> nil + end + + class Conversation < Base + attr_reader items: OpenAI::Realtime::ConnectionResources::ConversationItems + end + + class ConversationItems < Base + def create: ( + type: Symbol, + ?arguments: String, + ?approval_request_id: String?, + ?approve: bool, + ?call_id: String, + ?content: ::Array[::Hash[Symbol | String, untyped]], + ?error: untyped, + ?id: String, + ?name: String, + ?object: Symbol, + ?output: String?, + ?reason: String?, + ?role: Symbol, + ?server_label: String, + ?status: Symbol, + ?tools: ::Array[OpenAI::Realtime::RealtimeMcpListTools::Tool | ::Hash[Symbol | String, untyped]], + ?event_id: String?, + ?previous_item_id: String? + ) -> nil + def delete: (item_id: String, ?event_id: String?) -> nil + def retrieve: (item_id: String, ?event_id: String?) -> nil + def truncate: ( + item_id: String, + content_index: Integer, + audio_end_ms: Integer, + ?event_id: String? + ) -> nil + def create_function_call_output: ( + call_id: String, + output: String, + ?id: String?, + ?status: (String | Symbol)?, + ?event_id: String? + ) -> nil + def respond_to_mcp_approval: ( + approval_request_id: String, + approve: bool, + ?reason: String?, + ?id: String?, + ?event_id: String? + ) -> nil + end + + class OutputAudioBuffer < Base + def clear: (?event_id: String?) -> nil + end + end + end + end +end diff --git a/sig/openai/realtime/sideband_connection.rbs b/sig/openai/realtime/sideband_connection.rbs new file mode 100644 index 000000000..cfdf03744 --- /dev/null +++ b/sig/openai/realtime/sideband_connection.rbs @@ -0,0 +1,11 @@ +module OpenAI + module Models + module Realtime + class SidebandConnection < OpenAI::Realtime::Connection + attr_reader output_audio_buffer: OpenAI::Realtime::ConnectionResources::OutputAudioBuffer + + def initialize: (socket: untyped, url: URI::Generic) -> void + end + end + end +end diff --git a/sig/openai/realtime/transcription_connection.rbs b/sig/openai/realtime/transcription_connection.rbs new file mode 100644 index 000000000..ea8032c2f --- /dev/null +++ b/sig/openai/realtime/transcription_connection.rbs @@ -0,0 +1,34 @@ +module OpenAI + module Models + module Realtime + type transcription_connection_server_event = + OpenAI::Models::Realtime::realtime_server_event + | OpenAI::Realtime::UnknownServerEvent + + type transcription_connection_client_event = + OpenAI::Models::Realtime::realtime_client_event + | ::Hash[Symbol | String, untyped] + + class TranscriptionConnection < OpenAI::Realtime::BaseConnection + include Enumerable[OpenAI::Models::Realtime::transcription_connection_server_event] + + attr_reader session: OpenAI::Realtime::ConnectionResources::TranscriptionSession + attr_reader input_audio_buffer: OpenAI::Realtime::ConnectionResources::InputAudioBuffer + + def initialize: (socket: untyped, url: URI::Generic) -> void + def each: + -> Enumerator[OpenAI::Models::Realtime::transcription_connection_server_event, self] + | { + (OpenAI::Models::Realtime::transcription_connection_server_event event) -> void + } -> self + def receive: -> OpenAI::Models::Realtime::transcription_connection_server_event? + def parse_event: ( + String data + ) -> OpenAI::Models::Realtime::transcription_connection_server_event + def send_event: ( + OpenAI::Models::Realtime::transcription_connection_client_event event + ) -> nil + end + end + end +end diff --git a/sig/openai/realtime/translation_connection.rbs b/sig/openai/realtime/translation_connection.rbs new file mode 100644 index 000000000..d8042c341 --- /dev/null +++ b/sig/openai/realtime/translation_connection.rbs @@ -0,0 +1,36 @@ +module OpenAI + module Models + module Realtime + type translation_connection_server_event = + OpenAI::Models::Realtime::realtime_translation_server_event + | OpenAI::Realtime::UnknownServerEvent + + type translation_connection_client_event = + OpenAI::Models::Realtime::realtime_translation_client_event + | ::Hash[Symbol | String, untyped] + + class TranslationConnection < OpenAI::Realtime::BaseConnection + include Enumerable[OpenAI::Models::Realtime::translation_connection_server_event] + + attr_reader session: OpenAI::Realtime::ConnectionResources::TranslationSession + attr_reader input_audio_buffer: OpenAI::Realtime::ConnectionResources::TranslationInputAudioBuffer + + def initialize: (socket: untyped, url: URI::Generic) -> void + def each: + -> Enumerator[OpenAI::Models::Realtime::translation_connection_server_event, self] + | { + ( + OpenAI::Models::Realtime::translation_connection_server_event event + ) -> void + } -> self + def receive: -> OpenAI::Models::Realtime::translation_connection_server_event? + def parse_event: ( + String data + ) -> OpenAI::Models::Realtime::translation_connection_server_event + def send_event: ( + OpenAI::Models::Realtime::translation_connection_client_event event + ) -> nil + end + end + end +end diff --git a/sig/openai/realtime/transports/async_websocket.rbs b/sig/openai/realtime/transports/async_websocket.rbs new file mode 100644 index 000000000..d30db5995 --- /dev/null +++ b/sig/openai/realtime/transports/async_websocket.rbs @@ -0,0 +1,26 @@ +module OpenAI + module Models + module Realtime + module Transports + class AsyncWebSocket + class Socket + def initialize: (untyped connection, url: URI::Generic) -> void + def read: -> untyped + def write: (String message) -> void + def close: (?code: Integer, ?reason: String) -> void + def closed?: -> bool + end + + def open: ( + url: URI::Generic, + headers: ::Hash[String, String], + timeout: Float?, + **untyped endpoint_options + ) { + (Socket socket) -> top + } -> top + end + end + end + end +end diff --git a/sig/openai/realtime/unknown_server_event.rbs b/sig/openai/realtime/unknown_server_event.rbs new file mode 100644 index 000000000..8fcb1d07d --- /dev/null +++ b/sig/openai/realtime/unknown_server_event.rbs @@ -0,0 +1,13 @@ +module OpenAI + module Models + module Realtime + class UnknownServerEvent + attr_reader type: Symbol + attr_reader data: ::Hash[Symbol, untyped] + + def initialize: (data: ::Hash[Symbol, untyped]) -> void + def to_h: -> ::Hash[Symbol, untyped] + end + end + end +end diff --git a/sig/openai/resources/realtime.rbs b/sig/openai/resources/realtime.rbs index 999debec0..c7d74d7cc 100644 --- a/sig/openai/resources/realtime.rbs +++ b/sig/openai/resources/realtime.rbs @@ -5,6 +5,28 @@ module OpenAI attr_reader calls: OpenAI::Resources::Realtime::Calls + attr_reader translations: OpenAI::Resources::Realtime::Translations + + def connect: ( + model: String, + ?request_options: OpenAI::request_opts?, + ?transport: untyped?, + ?transport_options: ::Hash[Symbol, untyped] + ) { (OpenAI::Realtime::Connection connection) -> top } -> top + + def connect_to_call: ( + call_id: String, + ?request_options: OpenAI::request_opts?, + ?transport: untyped?, + ?transport_options: ::Hash[Symbol, untyped] + ) { (OpenAI::Realtime::SidebandConnection connection) -> top } -> top + + def connect_transcription: ( + ?request_options: OpenAI::request_opts?, + ?transport: untyped?, + ?transport_options: ::Hash[Symbol, untyped] + ) { (OpenAI::Realtime::TranscriptionConnection connection) -> top } -> top + def initialize: (client: OpenAI::Client) -> void end end diff --git a/sig/openai/resources/realtime/calls.rbs b/sig/openai/resources/realtime/calls.rbs index 7f89f2417..6cad50fd9 100644 --- a/sig/openai/resources/realtime/calls.rbs +++ b/sig/openai/resources/realtime/calls.rbs @@ -2,6 +2,13 @@ module OpenAI module Resources class Realtime class Calls + def create: ( + sdp: String, + ?session: OpenAI::Realtime::RealtimeSessionCreateRequest + | ::Hash[Symbol | String, untyped], + ?request_options: OpenAI::request_opts + ) -> OpenAI::Realtime::CallCreateResponse + def accept: ( String call_id, ?audio: OpenAI::Realtime::RealtimeAudioConfig, diff --git a/sig/openai/resources/realtime/translations.rbs b/sig/openai/resources/realtime/translations.rbs new file mode 100644 index 000000000..5dd578f21 --- /dev/null +++ b/sig/openai/resources/realtime/translations.rbs @@ -0,0 +1,21 @@ +module OpenAI + module Resources + class Realtime + class Translations + attr_reader client_secrets: OpenAI::Resources::Realtime::Translations::ClientSecrets + attr_reader calls: OpenAI::Resources::Realtime::Translations::Calls + + def connect: ( + model: String, + ?request_options: OpenAI::request_opts?, + ?transport: untyped?, + ?transport_options: ::Hash[Symbol, untyped] + ) { + (OpenAI::Realtime::TranslationConnection connection) -> top + } -> top + + def initialize: (client: OpenAI::Client) -> void + end + end + end +end diff --git a/sig/openai/resources/realtime/translations/calls.rbs b/sig/openai/resources/realtime/translations/calls.rbs new file mode 100644 index 000000000..db1ebfdde --- /dev/null +++ b/sig/openai/resources/realtime/translations/calls.rbs @@ -0,0 +1,16 @@ +module OpenAI + module Resources + class Realtime + class Translations + class Calls + def create: ( + sdp: String, + ?request_options: OpenAI::request_opts? + ) -> OpenAI::Realtime::CallCreateResponse + + def initialize: (client: OpenAI::Client) -> void + end + end + end + end +end diff --git a/sig/openai/resources/realtime/translations/client_secrets.rbs b/sig/openai/resources/realtime/translations/client_secrets.rbs new file mode 100644 index 000000000..e609efe1a --- /dev/null +++ b/sig/openai/resources/realtime/translations/client_secrets.rbs @@ -0,0 +1,19 @@ +module OpenAI + module Resources + class Realtime + class Translations + class ClientSecrets + def create: ( + session: OpenAI::Realtime::RealtimeTranslationSessionCreateRequest + | ::Hash[Symbol | String, untyped], + ?expires_after: (OpenAI::Realtime::RealtimeTranslationClientSecretCreateRequest::ExpiresAfter + | ::Hash[(Symbol | String), untyped])?, + ?request_options: OpenAI::request_opts? + ) -> OpenAI::Realtime::RealtimeTranslationClientSecretCreateResponse + + def initialize: (client: OpenAI::Client) -> void + end + end + end + end +end diff --git a/test/openai/logging_test.rb b/test/openai/logging_test.rb index b98c3df7d..82864c6db 100644 --- a/test/openai/logging_test.rb +++ b/test/openai/logging_test.rb @@ -57,6 +57,24 @@ class UnbufferableChunk < String def byteslice(*) = raise("stream chunk was buffered") end + class FailingBody + include Enumerable + + attr_reader :close_count + + def initialize(error) + @error = error + @close_count = 0 + end + + def each + yield("partial") + raise @error + end + + def close = (@close_count += 1) + end + def setup super @openai_log = ENV.delete("OPENAI_LOG") @@ -614,6 +632,72 @@ def test_stream_failures_log_an_error_without_a_success_event refute_includes(logger.events.fetch(0).fetch(1), "request complete") end + def test_raw_response_completion_is_logged_only_after_body_consumption + logger = CapturingLogger.new + source = CloseableBody.new("answer-sdp") + http_client = StubHTTPClient.new do |_request| + OpenAI::HTTPClient::Response.new( + status: 200, + headers: {"content-type" => "application/sdp", "x-request-id" => "req_raw"}, + body: source + ) + end + client = diagnostic_client(http_client: http_client, logger: logger, log_level: :info) + + response = client.request_raw(method: :post, path: "realtime/calls") + + assert_empty(logger.events) + assert_equal(["answer-sdp"], response.body.to_a) + assert_equal(1, source.close_count) + assert_equal([:info], logger.events.map(&:first)) + assert_includes(logger.events.fetch(0).fetch(1), "request complete") + end + + def test_raw_response_body_failure_logs_an_error_without_a_success_event + logger = CapturingLogger.new + failure = OpenAI::Errors::APIConnectionError.new( + url: URI("https://example.com/v1/realtime/calls") + ) + source = FailingBody.new(failure) + http_client = StubHTTPClient.new do |_request| + OpenAI::HTTPClient::Response.new( + status: 200, + headers: {"content-type" => "application/sdp"}, + body: source + ) + end + client = diagnostic_client(http_client: http_client, logger: logger, log_level: :info) + response = client.request_raw(method: :post, path: "realtime/calls") + + assert_empty(logger.events) + error = assert_raises(OpenAI::Errors::APIConnectionError) { response.body.to_a } + + assert_same(failure, error) + assert_equal(1, source.close_count) + assert_equal([:error], logger.events.map(&:first)) + assert_includes(logger.events.fetch(0).fetch(1), "request failed") + refute_includes(logger.events.fetch(0).fetch(1), "request complete") + end + + def test_closing_a_partially_consumed_raw_response_closes_its_source + logger = CapturingLogger.new + source = CloseableBody.new("first", "second") + http_client = StubHTTPClient.new do |_request| + OpenAI::HTTPClient::Response.new( + status: 200, + headers: {"content-type" => "application/sdp"}, + body: source + ) + end + client = diagnostic_client(http_client: http_client, logger: logger, log_level: :info) + response = client.request_raw(method: :post, path: "realtime/calls") + + assert_equal("first", response.body.first) + + assert_equal(1, source.close_count) + assert_empty(logger.events) + end + def test_consumer_exceptions_are_not_logged_as_request_failures logger = CapturingLogger.new http_client = StubHTTPClient.new do |_request| diff --git a/test/openai/realtime/async_websocket_transport_test.rb b/test/openai/realtime/async_websocket_transport_test.rb new file mode 100644 index 000000000..c2e101471 --- /dev/null +++ b/test/openai/realtime/async_websocket_transport_test.rb @@ -0,0 +1,608 @@ +# frozen_string_literal: true + +require "async/http/server" +require "async/queue" +require "async/websocket/adapters/http" +require "async/websocket/client" +require "async/websocket/server" +require "socket" + +require_relative "../test_helper" + +class OpenAI::Test::AsyncWebSocketTransportTest < Minitest::Test + extend Minitest::Serial + + class SynchronousConnection + attr_reader :scheduler + + def initialize + @closed = false + @scheduler = nil + end + + def connect! + @scheduler = Fiber.scheduler + self + end + + def read = "message" + def write(_message) = nil + def flush = nil + def closed? = @closed + def close(*) = (@closed = true) + end + + class SynchronousClient + attr_reader :closed + + def initialize(connection) + @connection = connection + @closed = false + end + + def connect(*) = @connection.connect! + def close = (@closed = true) + end + + def test_default_transport_enters_a_reactor_for_synchronous_callers + assert_nil(Fiber.scheduler) + connection = SynchronousConnection.new + client = SynchronousClient.new(connection) + transport = OpenAI::Realtime::Transports::AsyncWebSocket.new + url = URI("wss://example.com/v1/realtime?model=gpt-realtime-2.1") + + result = Async::WebSocket::Client.stub(:open, client) do + transport.open(url: url, headers: {}, timeout: nil) do |socket| + [socket.read, Fiber.scheduler] + end + end + + assert_equal("message", result.fetch(0)) + assert_same(connection.scheduler, result.fetch(1)) + refute_nil(connection.scheduler) + assert_predicate(connection, :closed?) + assert_predicate(client, :closed) + assert_nil(Fiber.scheduler) + end + + def test_default_transport_exchanges_events_with_a_local_websocket_server + port = available_port + endpoint = Async::HTTP::Endpoint.parse("http://127.0.0.1:#{port}") + fallback = ->(_request) { Protocol::HTTP::Response[404, {}, []] } + websocket = Async::WebSocket::Server.new(fallback) do |connection| + message = connection.read + next if message.nil? + + event = JSON.parse(message.to_str) + raise "unexpected client event" unless event.fetch("type") == "session.update" + + connection.write( + Protocol::WebSocket::TextMessage.generate( + type: "response.output_text.delta", + event_id: "event_1", + response_id: "response_1", + item_id: "item_1", + output_index: 0, + content_index: 0, + delta: "hello over websocket" + ) + ) + connection.flush + end + server = Async::HTTP::Server.new(websocket, endpoint) + + Sync do |task| + server_task = task.async { server.run.wait } + client = OpenAI::Client.new( + api_key: "test-key", + base_url: "http://127.0.0.1:#{port}/v1", + timeout: 5 + ) + + event = client.realtime.connect(model: "gpt-realtime") do |connection| + connection.session.update(type: :realtime) + connection.receive + end + + assert_instance_of(OpenAI::Realtime::ResponseTextDeltaEvent, event) + assert_equal("hello over websocket", event.delta) + assert_application_errors_propagate(client) + ensure + server_task&.stop + end + end + + def test_documented_text_lifecycle_over_a_local_websocket + with_websocket_server(method(:serve_text_lifecycle)) do |_task, client| + client.realtime.connect(model: "gpt-realtime-2.1") do |connection| + assert_instance_of(OpenAI::Realtime::SessionCreatedEvent, connection.receive) + connection.session.update(type: :realtime, output_modalities: [:text]) + assert_instance_of(OpenAI::Realtime::SessionUpdatedEvent, connection.receive) + + connection.conversation.items.create( + type: :message, + role: :user, + content: [{type: :input_text, text: "Say hello."}] + ) + connection.response.create + + delta = connection.receive + done = connection.receive + assert_instance_of(OpenAI::Realtime::ResponseTextDeltaEvent, delta) + assert_equal("hello from the local service", delta.delta) + assert_instance_of(OpenAI::Realtime::ResponseDoneEvent, done) + assert_equal(:completed, done.response.status) + assert_nil(connection.receive) + end + end + end + + def test_function_call_round_trip_over_a_local_websocket + handler = lambda do |connection| + unless read_event(connection).fetch("type") == "conversation.item.create" + raise "expected conversation item" + end + raise "expected response.create" unless read_event(connection).fetch("type") == "response.create" + + write_event( + connection, + type: "response.function_call_arguments.done", + event_id: "event_call", + response_id: "response_1", + item_id: "item_call", + output_index: 0, + call_id: "call_1", + name: "get_weather", + arguments: JSON.generate(location: "Paris") + ) + + output = read_event(connection) + raise "expected function output" unless output.dig("item", "type") == "function_call_output" + raise "unexpected call id" unless output.dig("item", "call_id") == "call_1" + raise "expected follow-up response" unless read_event(connection).fetch("type") == "response.create" + + write_event( + connection, + type: "response.done", + event_id: "event_done", + response: {id: "response_2", status: "completed", output: []} + ) + end + + with_websocket_server(handler) do |_task, client| + client.realtime.connect(model: "gpt-realtime-2.1") do |connection| + connection.conversation.items.create( + type: :message, + role: :user, + content: [{type: :input_text, text: "What is the weather?"}] + ) + connection.response.create + + call = connection.receive + assert_instance_of(OpenAI::Realtime::ResponseFunctionCallArgumentsDoneEvent, call) + connection.conversation.items.create_function_call_output( + call_id: call.call_id, + output: JSON.generate(temperature_c: 18) + ) + connection.response.create + + done = connection.receive + assert_instance_of(OpenAI::Realtime::ResponseDoneEvent, done) + assert_equal(:completed, done.response.status) + end + end + end + + def test_mcp_approval_lifecycle_over_a_local_websocket + with_websocket_server(method(:serve_mcp_approval)) do |_task, client| + client.realtime.connect(model: "gpt-realtime-2.1") do |connection| + exercise_mcp_approval(connection) + end + end + end + + def test_translation_reads_output_while_input_is_still_streaming + first_audio = "\x00\x01".b * 4_800 + second_audio = "\x02\x03".b * 4_800 + translated_audio = "\x04\x05".b * 4_800 + handler = lambda do |connection| + serve_translation(connection, first_audio:, second_audio:, translated_audio:) + end + + with_websocket_server(handler) do |task, client| + client.realtime.translations.connect(model: "gpt-realtime-translate") do |connection| + connection.session.update(audio: {output: {language: "es"}}) + events = Async::Queue.new + reader = task.async do + connection.each do |event| + events.enqueue(event) + break if event.is_a?(OpenAI::Realtime::RealtimeTranslationSessionClosedEvent) + end + end + + connection.input_audio_buffer.append_bytes(first_audio) + transcript = events.dequeue + audio = events.dequeue + assert_instance_of(OpenAI::Realtime::RealtimeTranslationOutputTranscriptDeltaEvent, transcript) + assert_equal("hola", transcript.delta) + assert_instance_of(OpenAI::Realtime::RealtimeTranslationOutputAudioDeltaEvent, audio) + assert_equal(translated_audio, Base64.strict_decode64(audio.delta)) + + connection.input_audio_buffer.append_bytes(second_audio) + connection.session.close + assert_instance_of(OpenAI::Realtime::RealtimeTranslationSessionClosedEvent, events.dequeue) + reader.wait + end + end + end + + def test_documented_transcription_lifecycle_over_a_local_websocket + audio = "\x00\x01".b * 2_400 + handler = ->(connection) { serve_transcription(connection, audio: audio) } + + with_websocket_server(handler) do |_task, client| + exercise_transcription(client, audio: audio) + end + end + + def test_default_transport_reassembles_fragmented_text_messages + handler = lambda do |connection| + payload = JSON.generate( + type: "response.output_text.delta", + event_id: "event_fragmented", + response_id: "response_1", + item_id: "item_1", + output_index: 0, + content_index: 0, + delta: "fragmented" + ) + midpoint = payload.bytesize / 2 + first = Protocol::WebSocket::TextFrame.new(false).pack(payload.byteslice(0, midpoint)) + second = Protocol::WebSocket::ContinuationFrame.new(true).pack(payload.byteslice(midpoint..)) + connection.write_frame(first) + connection.write_frame(second) + connection.flush + end + + with_websocket_server(handler) do |_task, client| + event = client.realtime.connect(model: "gpt-realtime-2.1", &:receive) + + assert_instance_of(OpenAI::Realtime::ResponseTextDeltaEvent, event) + assert_equal("fragmented", event.delta) + end + end + + def test_default_transport_wraps_an_abnormal_remote_close + handler = ->(connection) { connection.close(1011, "service failed") } + + with_websocket_server(handler) do |_task, client| + error = assert_raises(OpenAI::Errors::RealtimeConnectionError) do + client.realtime.connect(model: "gpt-realtime-2.1", &:receive) + end + + assert_instance_of(Protocol::WebSocket::ClosedError, error.cause) + assert_includes(error.message, "service failed") + end + end + + def test_default_transport_wraps_handshake_failures + port = available_port + client = OpenAI::Client.new( + api_key: "test-key", + base_url: "http://127.0.0.1:#{port}/v1", + timeout: 0.5 + ) + + error = assert_raises(OpenAI::Errors::RealtimeConnectionError) do + client.realtime.connect(model: "gpt-realtime") { |_connection| nil } + end + + assert_equal("ws://127.0.0.1:#{port}/v1/realtime?model=gpt-realtime", error.url.to_s) + refute_nil(error.cause) + assert_includes(error.message, error.cause.message) + end + + def test_missing_optional_dependency_preserves_load_error_as_the_cause + transport_class = Class.new(OpenAI::Realtime::Transports::AsyncWebSocket) do + private def require(path) + raise LoadError, "cannot load #{path}" if path == "async/websocket/client" + + super + end + end + url = URI("wss://example.com/v1/realtime?model=gpt-realtime-2.1") + + error = assert_raises(OpenAI::Errors::RealtimeConnectionError) do + transport_class.new.open(url:, headers: {}, timeout: 1) { |_socket| nil } + end + + assert_instance_of(LoadError, error.cause) + assert_includes(error.message, "Add `gem \"async-websocket\"` to your Gemfile") + end + + private def assert_application_errors_propagate(client) + error = assert_raises(RuntimeError) do + client.realtime.connect(model: "gpt-realtime") do |_connection| + raise "application failed" + end + end + assert_equal("application failed", error.message) + + load_error = assert_raises(LoadError) do + client.realtime.connect(model: "gpt-realtime") do |_connection| + raise LoadError, "application dependency failed" + end + end + assert_equal("application dependency failed", load_error.message) + end + + private def serve_text_lifecycle(connection) + write_event( + connection, + type: "session.created", + event_id: "event_created", + session: {type: "realtime"} + ) + + update = read_event(connection) + raise "expected session.update" unless update.fetch("type") == "session.update" + write_event( + connection, + type: "session.updated", + event_id: "event_updated", + session: update.fetch("session") + ) + + unless read_event(connection).fetch("type") == "conversation.item.create" + raise "expected conversation item" + end + raise "expected response.create" unless read_event(connection).fetch("type") == "response.create" + write_event( + connection, + type: "response.output_text.delta", + event_id: "event_delta", + response_id: "response_1", + item_id: "item_1", + output_index: 0, + content_index: 0, + delta: "hello from the local service" + ) + write_event( + connection, + type: "response.done", + event_id: "event_done", + response: {id: "response_1", status: "completed", output: []} + ) + end + + private def serve_mcp_approval(connection) + update = read_event(connection) + raise "expected MCP session update" unless update.dig("session", "tools", 0, "type") == "mcp" + write_event( + connection, + type: "mcp_list_tools.completed", + event_id: "event_tools", + item_id: "item_tools" + ) + + raise "expected prompt item" unless read_event(connection).fetch("type") == "conversation.item.create" + raise "expected response.create" unless read_event(connection).fetch("type") == "response.create" + write_event( + connection, + type: "conversation.item.done", + event_id: "event_approval", + previous_item_id: nil, + item: { + type: "mcp_approval_request", + id: "approval_1", + arguments: JSON.generate(date: "tomorrow"), + name: "check_calendar", + server_label: "calendar" + } + ) + + approval = read_event(connection) + raise "expected approval response" unless approval.dig("item", "type") == "mcp_approval_response" + raise "unexpected approval id" unless approval.dig("item", "approval_request_id") == "approval_1" + raise "expected approval" unless approval.dig("item", "approve") + write_event( + connection, + type: "response.done", + event_id: "event_done", + response: {id: "response_1", status: "completed", output: []} + ) + end + + private def exercise_mcp_approval(connection) + connection.session.update( + type: :realtime, + tools: [ + { + type: :mcp, + server_label: "calendar", + server_url: "https://example.com/mcp", + require_approval: :always + } + ] + ) + + assert_instance_of(OpenAI::Realtime::McpListToolsCompleted, connection.receive) + connection.conversation.items.create( + type: :message, + role: :user, + content: [{type: :input_text, text: "Check tomorrow's calendar."}] + ) + connection.response.create + + approval = connection.receive + assert_instance_of(OpenAI::Realtime::ConversationItemDone, approval) + assert_instance_of(OpenAI::Realtime::RealtimeMcpApprovalRequest, approval.item) + connection.conversation.items.respond_to_mcp_approval( + approval_request_id: approval.item.id, + approve: true, + reason: "Allowed by test policy" + ) + + done = connection.receive + assert_instance_of(OpenAI::Realtime::ResponseDoneEvent, done) + assert_equal(:completed, done.response.status) + end + + private def serve_translation(connection, first_audio:, second_audio:, translated_audio:) + raise "expected translation update" unless read_event(connection).fetch("type") == "session.update" + first_append = read_event(connection) + unless first_append.fetch("type") == "session.input_audio_buffer.append" + raise "expected first audio frame" + end + raise "unexpected first audio" unless Base64.strict_decode64(first_append.fetch("audio")) == first_audio + + write_event( + connection, + type: "session.output_transcript.delta", + event_id: "event_transcript", + delta: "hola", + elapsed_ms: 200 + ) + write_event( + connection, + type: "session.output_audio.delta", + event_id: "event_audio", + delta: Base64.strict_encode64(translated_audio), + elapsed_ms: 200, + format: "pcm16", + sample_rate: 24_000, + channels: 1 + ) + + second_append = read_event(connection) + unless second_append.fetch("type") == "session.input_audio_buffer.append" + raise "expected second audio frame" + end + unless Base64.strict_decode64(second_append.fetch("audio")) == second_audio + raise "unexpected second audio" + end + raise "expected session.close" unless read_event(connection).fetch("type") == "session.close" + + write_event(connection, type: "session.closed", event_id: "event_closed") + end + + private def serve_transcription(connection, audio:) + write_event( + connection, + type: "session.created", + event_id: "event_created", + session: {type: "transcription"} + ) + + update = read_event(connection) + raise "expected transcription session update" unless update.fetch("type") == "session.update" + + write_event( + connection, + type: "session.updated", + event_id: "event_updated", + session: update.fetch("session") + ) + + append = read_event(connection) + raise "expected audio append" unless append.fetch("type") == "input_audio_buffer.append" + unless Base64.strict_decode64(append.fetch("audio")) == audio + raise "unexpected transcription audio" + end + raise "expected audio commit" unless read_event(connection).fetch("type") == "input_audio_buffer.commit" + + write_event( + connection, + type: "conversation.item.input_audio_transcription.delta", + event_id: "event_delta", + item_id: "item_1", + content_index: 0, + delta: "hello" + ) + write_event( + connection, + type: "conversation.item.input_audio_transcription.completed", + event_id: "event_completed", + item_id: "item_1", + content_index: 0, + transcript: "hello from ruby", + usage: {type: "duration", seconds: 0.1} + ) + end + + private def exercise_transcription(client, audio:) + client.realtime.connect_transcription do |connection| + assert_instance_of(OpenAI::Realtime::SessionCreatedEvent, connection.receive) + configure_transcription(connection) + assert_instance_of(OpenAI::Realtime::SessionUpdatedEvent, connection.receive) + + connection.input_audio_buffer.append_bytes(audio) + connection.input_audio_buffer.commit + assert_transcription_events(connection) + end + end + + private def configure_transcription(connection) + connection.session.update( + audio: { + input: { + format: {type: :"audio/pcm", rate: 24_000}, + transcription: {model: "gpt-live-transcribe"}, + turn_detection: nil + } + } + ) + end + + private def assert_transcription_events(connection) + delta = connection.receive + completed = connection.receive + assert_instance_of(OpenAI::Realtime::ConversationItemInputAudioTranscriptionDeltaEvent, delta) + assert_equal("hello", delta.delta) + assert_instance_of( + OpenAI::Realtime::ConversationItemInputAudioTranscriptionCompletedEvent, + completed + ) + assert_equal(delta.item_id, completed.item_id) + assert_equal("hello from ruby", completed.transcript) + end + + private def with_websocket_server(handler) + port = available_port + endpoint = Async::HTTP::Endpoint.parse("http://127.0.0.1:#{port}") + fallback = ->(_request) { Protocol::HTTP::Response[404, {}, []] } + websocket = Async::WebSocket::Server.new(fallback, &handler) + server = Async::HTTP::Server.new(websocket, endpoint) + + Sync do |task| + server_task = task.async { server.run.wait } + client = OpenAI::Client.new( + api_key: "test-key", + base_url: "http://127.0.0.1:#{port}/v1", + timeout: 5 + ) + yield(task, client) + ensure + server_task&.stop + end + end + + private def read_event(connection) + message = connection.read + raise "client closed before sending the expected event" if message.nil? + + JSON.parse(message.to_str) + end + + private def write_event(connection, **event) + connection.write(Protocol::WebSocket::TextMessage.generate(event)) + connection.flush + end + + private def available_port + server = TCPServer.new("127.0.0.1", 0) + server.local_address.ip_port + ensure + server&.close + end +end diff --git a/test/openai/realtime/connection_test.rb b/test/openai/realtime/connection_test.rb new file mode 100644 index 000000000..e75010054 --- /dev/null +++ b/test/openai/realtime/connection_test.rb @@ -0,0 +1,541 @@ +# frozen_string_literal: true + +require_relative "../test_helper" + +class OpenAI::Test::RealtimeConnectionTest < Minitest::Test + class FakeSocket + attr_reader :writes, :close_args + + def initialize(*reads) + @reads = reads + @writes = [] + @closed = false + end + + def read = @reads.shift + def write(message) = @writes << message + def closed? = @closed + + def close(code: 1000, reason: "") + @closed = true + @close_args = {code: code, reason: reason} + end + end + + class FakeTransport + attr_reader :open_args + + def initialize(socket) + @socket = socket + end + + def open(url:, headers:, timeout:, **options) + @open_args = {url: url, headers: headers, timeout: timeout, options: options} + yield(@socket) + end + end + + class FailingCloseSocket < FakeSocket + def close(code: 1000, reason: "") + super + raise IOError, "close failed" + end + end + + def client(**options) + OpenAI::Client.new( + api_key: "test-key", + base_url: "https://example.com/v1", + **options + ) + end + + def test_connect_opens_a_typed_block_scoped_connection + socket = FakeSocket.new( + JSON.generate( + type: "response.output_text.delta", + event_id: "event_1", + response_id: "response_1", + item_id: "item_1", + output_index: 0, + content_index: 0, + delta: "hello" + ) + ) + transport = FakeTransport.new(socket) + event = nil + + result = client.realtime.connect( + model: "gpt-realtime", + request_options: { + extra_query: {trace: "yes"}, + extra_headers: {"X-Trace-ID" => "trace_1"}, + timeout: 12 + }, + transport: transport, + transport_options: {max_frame_size: 1_024} + ) do |connection| + assert_instance_of(OpenAI::Realtime::Connection, connection) + event = connection.receive + :block_result + end + + assert_equal(:block_result, result) + assert_instance_of(OpenAI::Realtime::ResponseTextDeltaEvent, event) + assert_equal("hello", event.delta) + assert_equal("wss://example.com/v1/realtime?model=gpt-realtime&trace=yes", transport.open_args[:url].to_s) + assert_equal("Bearer test-key", transport.open_args[:headers].fetch("authorization")) + assert_equal("trace_1", transport.open_args[:headers].fetch("x-trace-id")) + assert_equal(12.0, transport.open_args[:timeout]) + assert_equal({max_frame_size: 1_024}, transport.open_args[:options]) + assert(socket.closed?) + end + + def test_connect_to_a_dedicated_transcription_session + socket = FakeSocket.new + transport = FakeTransport.new(socket) + + client.realtime.connect_transcription(transport: transport) do |connection| + assert_instance_of(OpenAI::Realtime::TranscriptionConnection, connection) + assert_instance_of(OpenAI::Realtime::ConnectionResources::TranscriptionSession, connection.session) + assert_respond_to(connection.input_audio_buffer, :commit) + assert_respond_to(connection.input_audio_buffer, :clear) + refute_respond_to(connection, :response) + refute_respond_to(connection, :conversation) + refute_respond_to(connection, :output_audio_buffer) + connection.session.update(audio: {input: {turn_detection: nil}}, event_id: "update_1") + end + + assert_equal( + "wss://example.com/v1/realtime?intent=transcription", + transport.open_args[:url].to_s + ) + update = JSON.parse(socket.writes.fetch(0), symbolize_names: true) + assert_equal(:transcription, update.dig(:session, :type).to_sym) + assert_equal("update_1", update.fetch(:event_id)) + refute(update.fetch(:session).key?(:event_id)) + end + + def test_connect_to_an_existing_webrtc_or_sip_call + transport = FakeTransport.new(FakeSocket.new) + + client.realtime.connect_to_call(call_id: "rtc_123", transport: transport) do |connection| + assert_instance_of(OpenAI::Realtime::SidebandConnection, connection) + assert_respond_to(connection, :output_audio_buffer) + end + + assert_equal("wss://example.com/v1/realtime?call_id=rtc_123", transport.open_args[:url].to_s) + end + + def test_connect_uses_an_explicit_websocket_base_url + transport = FakeTransport.new(FakeSocket.new) + custom_client = client(websocket_base_url: "wss://socket.example.test/custom/v2") + + custom_client.realtime.connect(model: "gpt-realtime", transport: transport) { |_connection| nil } + + assert_equal( + "wss://socket.example.test/custom/v2/realtime?model=gpt-realtime", + transport.open_args[:url].to_s + ) + end + + def test_websocket_base_url_rejects_ambiguous_or_unsafe_urls + invalid_urls = [ + "/socket", + "ftp://socket.example.test/v1", + "wss://user:secret@socket.example.test/v1", + "wss://socket.example.test/v1?tenant=one", + "wss://socket.example.test/v1#fragment" + ] + + invalid_urls.each do |url| + error = assert_raises(ArgumentError) { client(websocket_base_url: url) } + assert_includes(error.message, "`websocket_base_url`") + end + end + + def test_connect_uses_azure_provider_authentication + transport = FakeTransport.new(FakeSocket.new) + azure_client = OpenAI::Client.new( + provider: OpenAI::Providers.azure( + endpoint: "https://resource.openai.azure.com", + api_key: "azure-key" + ) + ) + + azure_client.realtime.connect(model: "deployment", transport: transport) { |_connection| nil } + + assert_equal( + "wss://resource.openai.azure.com/openai/v1/realtime?model=deployment", + transport.open_args[:url].to_s + ) + assert_equal("azure-key", transport.open_args[:headers].fetch("api-key")) + refute(transport.open_args[:headers].key?("authorization")) + end + + def test_connect_resolves_workload_identity_before_the_handshake + subject_token_provider = + OpenAI::Auth::SubjectTokenProviders::K8sServiceAccountTokenProvider.new( + token_path: "/not-read-by-this-test" + ) + config = OpenAI::Auth::WorkloadIdentity.new( + identity_provider_id: "idp_123", + service_account_id: "sa_123", + provider: subject_token_provider + ) + workload_client = OpenAI::Client.new( + api_key: nil, + workload_identity: config, + organization: "org_123", + base_url: "https://example.com/v1" + ) + transport = FakeTransport.new(FakeSocket.new) + + workload_client.workload_identity_auth.stub(:get_token, "exchanged-token") do + workload_client.realtime.connect(model: "gpt-realtime", transport: transport) do |_connection| + nil + end + end + + assert_equal("Bearer exchanged-token", transport.open_args[:headers].fetch("authorization")) + end + + def test_unsupported_provider_fails_before_resolving_or_sending_credentials + credential_requested = false + provider = OpenAI::Providers.bedrock( + region: "us-east-1", + token_provider: lambda do + credential_requested = true + "bedrock-token" + end + ) + provider_client = OpenAI::Client.new(provider: provider) + transport = FakeTransport.new(FakeSocket.new) + + error = assert_raises(OpenAI::Errors::Error) do + provider_client.realtime.connect(model: "gpt-realtime", transport: transport) do |_connection| + nil + end + end + + assert_includes(error.message, "not supported") + refute(credential_requested) + assert_nil(transport.open_args) + end + + def test_connection_sends_typed_events_and_audio_bytes + socket = FakeSocket.new + transport = FakeTransport.new(socket) + + client.realtime.connect(model: "gpt-realtime", transport: transport) do |connection| + send_typed_resource_events(connection) + end + + events = socket.writes.map { JSON.parse(_1, symbolize_names: true) } + assert_equal( + [ + :"session.update", + :"response.create", + :"input_audio_buffer.append", + :"input_audio_buffer.commit", + :"conversation.item.create", + :"conversation.item.create", + :"conversation.item.create", + :"conversation.item.create" + ], + events.map { _1.fetch(:type).to_sym } + ) + assert_equal("session_update_1", events[0].fetch(:event_id)) + assert_equal("Be concise", events[0].dig(:session, :instructions)) + refute(events[0].fetch(:session).key?(:event_id)) + assert_equal("response_create_1", events[1].fetch(:event_id)) + assert_equal("Say hello", events[1].dig(:response, :instructions)) + refute(events[1].fetch(:response).key?(:event_id)) + assert_equal(Base64.strict_encode64("\x00\x01".b), events[2].fetch(:audio)) + assert_equal("item_create_1", events[4].fetch(:event_id)) + assert_equal("root", events[4].fetch(:previous_item_id)) + assert_equal("Hello", events[4].dig(:item, :content, 0, :text)) + refute(events[4].fetch(:item).key?(:previous_item_id)) + assert_equal(:function_call_output, events[5].fetch(:item).fetch(:type).to_sym) + assert_equal(:mcp_approval_response, events[6].fetch(:item).fetch(:type).to_sym) + assert(events[6].fetch(:item).fetch(:approve)) + generated_id = events[7].fetch(:item).fetch(:id) + assert_operator(generated_id.length, :<=, 32) + assert_match(/\Amcpa_[0-9a-f]+\z/, generated_id) + end + + def test_standard_connection_only_exposes_standard_websocket_capabilities + transport = FakeTransport.new(FakeSocket.new) + + client.realtime.connect(model: "gpt-realtime", transport: transport) do |connection| + refute_respond_to(connection, :output_audio_buffer) + refute_respond_to(connection.session, :close) + assert_respond_to(connection.input_audio_buffer, :commit) + assert_respond_to(connection.input_audio_buffer, :clear) + assert_respond_to(connection.conversation, :items) + refute_respond_to(connection.conversation, :item) + end + end + + def test_sideband_connection_exposes_output_audio_buffer_controls + socket = FakeSocket.new + transport = FakeTransport.new(socket) + + client.realtime.connect_to_call(call_id: "rtc_123", transport: transport) do |connection| + connection.output_audio_buffer.clear(event_id: "clear_1") + end + + assert_equal( + {"type" => "output_audio_buffer.clear", "event_id" => "clear_1"}, + JSON.parse(socket.writes.fetch(0)) + ) + end + + def test_sideband_interruption_helpers_preserve_documented_order + socket = FakeSocket.new + transport = FakeTransport.new(socket) + + client.realtime.connect_to_call(call_id: "rtc_123", transport: transport) do |connection| + connection.response.cancel + connection.conversation.items.truncate(item_id: "item_1", content_index: 0, audio_end_ms: 640) + connection.output_audio_buffer.clear + end + + assert_equal( + [ + "response.cancel", + "conversation.item.truncate", + "output_audio_buffer.clear" + ], + socket.writes.map { JSON.parse(_1).fetch("type") } + ) + end + + def test_all_standard_resource_helpers_emit_typed_events + socket = FakeSocket.new + transport = FakeTransport.new(socket) + + client.realtime.connect(model: "gpt-realtime", transport: transport) do |connection| + connection.response.cancel(response_id: "response_1", event_id: "cancel_1") + connection.input_audio_buffer.clear(event_id: "clear_1") + connection.conversation.items.retrieve(item_id: "item_1", event_id: "retrieve_1") + connection.conversation.items.truncate( + item_id: "item_1", + content_index: 0, + audio_end_ms: 250, + event_id: "truncate_1" + ) + connection.conversation.items.delete(item_id: "item_1", event_id: "delete_1") + end + + assert_equal( + [ + "response.cancel", + "input_audio_buffer.clear", + "conversation.item.retrieve", + "conversation.item.truncate", + "conversation.item.delete" + ], + socket.writes.map { JSON.parse(_1).fetch("type") } + ) + end + + def test_connection_does_not_override_ruby_send + transport = FakeTransport.new(FakeSocket.new) + + client.realtime.connect(model: "gpt-realtime", transport: transport) do |connection| + assert_equal(false, connection.send(:closed?)) + end + end + + def test_connection_rejects_an_unknown_client_event + transport = FakeTransport.new(FakeSocket.new) + + error = assert_raises(ArgumentError) do + client.realtime.connect(model: "gpt-realtime", transport: transport) do |connection| + connection.send_event(type: "future.unknown.event") + end + end + + assert_includes(error.message, "future.unknown.event") + end + + def test_connection_rejects_a_client_event_missing_required_fields + transport = FakeTransport.new(FakeSocket.new) + + error = assert_raises(ArgumentError) do + client.realtime.connect(model: "gpt-realtime", transport: transport) do |connection| + connection.send_event(type: "conversation.item.create") + end + end + + assert_includes(error.message, "required fields") + end + + def test_connect_requires_a_block + error = assert_raises(ArgumentError) do + client.realtime.connect(model: "gpt-realtime", transport: FakeTransport.new(FakeSocket.new)) + end + + assert_includes(error.message, "block is required") + end + + def test_connection_each_returns_a_typed_enumerator_without_a_block + event_data = JSON.generate( + type: "response.output_text.delta", + event_id: "event_1", + response_id: "response_1", + item_id: "item_1", + output_index: 0, + content_index: 0, + delta: "hello" + ) + transport = FakeTransport.new(FakeSocket.new(event_data, nil)) + events = nil + + client.realtime.connect(model: "gpt-realtime", transport: transport) do |connection| + events = connection.each.to_a + end + + assert_equal(1, events.length) + assert_instance_of(OpenAI::Realtime::ResponseTextDeltaEvent, events.first) + end + + def test_connection_each_returns_the_connection_with_a_block + data = JSON.generate(type: "future.event") + transport = FakeTransport.new(FakeSocket.new(data, nil)) + + client.realtime.connect(model: "gpt-realtime", transport: transport) do |connection| + count = 0 + result = connection.each { |_event| count += 1 } + + assert_equal(1, count) + assert_same(connection, result) + end + end + + def test_connection_closes_when_the_block_raises + socket = FakeSocket.new + transport = FakeTransport.new(socket) + + assert_raises(RuntimeError) do + client.realtime.connect(model: "gpt-realtime", transport: transport) do |_connection| + raise "boom" + end + end + + assert(socket.closed?) + end + + def test_application_error_wins_when_cleanup_also_fails + transport = FakeTransport.new(FailingCloseSocket.new) + + error = assert_raises(RuntimeError) do + client.realtime.connect(model: "gpt-realtime", transport: transport) do |_connection| + raise "application failed" + end + end + + assert_equal("application failed", error.message) + end + + def test_cleanup_error_propagates_after_a_successful_block + transport = FakeTransport.new(FailingCloseSocket.new) + + error = assert_raises(IOError) do + client.realtime.connect(model: "gpt-realtime", transport: transport) { |_connection| :done } + end + + assert_equal("close failed", error.message) + end + + def test_malformed_server_event_raises_a_realtime_protocol_error + transport = FakeTransport.new(FakeSocket.new("not-json")) + + error = assert_raises(OpenAI::Errors::RealtimeProtocolError) do + client.realtime.connect(model: "gpt-realtime", transport: transport, &:receive) + end + + assert_equal("not-json", error.data) + assert_instance_of(JSON::ParserError, error.cause) + end + + def test_api_error_remains_a_typed_event_in_the_stream + transport = FakeTransport.new( + FakeSocket.new( + JSON.generate( + type: "error", + event_id: "event_1", + error: {type: "invalid_request_error", message: "Try another value"} + ) + ) + ) + + event = client.realtime.connect(model: "gpt-realtime", transport: transport, &:receive) + + assert_instance_of(OpenAI::Realtime::RealtimeErrorEvent, event) + assert_equal("Try another value", event.error.message) + end + + def test_unknown_server_event_remains_observable + data = JSON.generate(type: "future.unknown.event", nested: {values: ["future value"]}) + transport = FakeTransport.new(FakeSocket.new(data)) + + event = client.realtime.connect(model: "gpt-realtime", transport: transport, &:receive) + + assert_instance_of(OpenAI::Realtime::UnknownServerEvent, event) + assert_equal(:"future.unknown.event", event.type) + assert_equal( + {type: "future.unknown.event", nested: {values: ["future value"]}}, + event.data + ) + assert(event.frozen?) + assert(event.data.frozen?) + assert(event.data.fetch(:nested).frozen?) + assert(event.data.dig(:nested, :values).frozen?) + assert(event.data.dig(:nested, :values, 0).frozen?) + end + + def test_server_event_missing_required_fields_raises_a_realtime_protocol_error + data = JSON.generate(type: "response.output_text.delta") + transport = FakeTransport.new(FakeSocket.new(data)) + + error = assert_raises(OpenAI::Errors::RealtimeProtocolError) do + client.realtime.connect(model: "gpt-realtime", transport: transport, &:receive) + end + + assert_equal(data, error.data) + assert_includes(error.cause.message, "required fields") + end + + private def send_typed_resource_events(connection) + connection.session.update( + type: :realtime, + instructions: "Be concise", + event_id: "session_update_1" + ) + connection.response.create(instructions: "Say hello", event_id: "response_create_1") + connection.input_audio_buffer.append_bytes("\x00\x01".b) + connection.input_audio_buffer.commit + connection.conversation.items.create( + type: :message, + role: :user, + content: [{type: :input_text, text: "Hello"}], + previous_item_id: "root", + event_id: "item_create_1" + ) + connection.conversation.items.create_function_call_output( + call_id: "call_1", + output: JSON.generate(temperature: 72) + ) + connection.conversation.items.respond_to_mcp_approval( + approval_request_id: "approval_1", + approve: true, + id: "approval_response_1" + ) + connection.conversation.items.respond_to_mcp_approval( + approval_request_id: "approval_2", + approve: false + ) + end +end diff --git a/test/openai/realtime/example_stream_lifecycle_test.rb b/test/openai/realtime/example_stream_lifecycle_test.rb new file mode 100644 index 000000000..f60f5a034 --- /dev/null +++ b/test/openai/realtime/example_stream_lifecycle_test.rb @@ -0,0 +1,452 @@ +# frozen_string_literal: true + +require_relative "../test_helper" +require "async/notification" +require_relative "../../../examples/realtime/mcp_approval" +require_relative "../../../examples/realtime/sideband" +require_relative "../../../examples/realtime/sip" +require_relative "../../../examples/realtime/translation" +require_relative "../../../examples/realtime/websocket_audio" +require_relative "../../../examples/realtime/websocket_text" +require_relative "../../../examples/realtime/websocket_transcription" + +class OpenAI::Test::RealtimeExampleStreamLifecycleTest < Minitest::Test + class RecordingConnection + def initialize(events = []) + @events = events + end + + def each(&block) + @events.each(&block) + end + + def receive = @events.shift + end + + class RaisingEndpoint + attr_reader :calls + + def initialize(error) + @error = error + @calls = [] + end + + def append_bytes(bytes) + @calls << [:append_bytes, bytes] + raise @error + end + + def close + @calls << [:close] + raise @error + end + end + + class ClosingSession + attr_reader :closed + + def close = @closed = true + end + + class RecordingEndpoint + attr_reader :calls + + def initialize = @calls = [] + def append_bytes(bytes) = @calls << [:append_bytes, bytes] + def close = @calls << [:close] + end + + class BlockingAudioBuffer + attr_reader :chunks + + def initialize(upload_started) + @upload_started = upload_started + @chunks = [] + end + + def append_bytes(bytes) + @chunks << bytes + @upload_started.signal + Kernel.sleep(3_600) + end + end + + class ReaderFailureConnection + attr_reader :input_audio_buffer, :session + + def initialize(event) + upload_started = Async::Notification.new + @event = event + @input_audio_buffer = BlockingAudioBuffer.new(upload_started) + @session = ClosingSession.new + @upload_started = upload_started + end + + def each + @upload_started.wait + yield(@event) + end + end + + class BufferedReaderFailureConnection + attr_reader :input_audio_buffer, :session + + def initialize(event) + @event = event + @input_audio_buffer = RecordingEndpoint.new + @session = RecordingEndpoint.new + end + + def each = yield(@event) + end + + TranslationConnection = Data.define(:input_audio_buffer, :session) + + Event = Data.define(:type, :data) do + def to_h = data + end + + def test_wait_for_rejects_eof_before_the_target_event + error = assert_raises(RuntimeError) do + OpenAI::Examples::Realtime::EventStream.wait_for( + RecordingConnection.new, + OpenAI::Realtime::SessionUpdatedEvent, + closed_message: "Realtime connection closed before session.updated" + ) + end + + assert_equal("Realtime connection closed before session.updated", error.message) + end + + def test_wait_for_surfaces_an_api_error_before_the_target_event + error = assert_raises(RuntimeError) do + OpenAI::Examples::Realtime::EventStream.wait_for( + RecordingConnection.new([realtime_error_event("session update failed")]), + OpenAI::Realtime::SessionUpdatedEvent, + closed_message: "Realtime connection closed before session.updated" + ) + end + + assert_equal("session update failed", error.message) + end + + def test_sideband_rejects_eof_before_the_requested_event + error = assert_raises(RuntimeError) do + OpenAI::Examples::Realtime::Sideband.stream( + RecordingConnection.new, + output: StringIO.new, + stop_after: "session.updated" + ) + end + + assert_equal("Realtime connection closed before session.updated", error.message) + end + + def test_sip_rejects_eof_before_the_requested_event + error = assert_raises(RuntimeError) do + OpenAI::Examples::Realtime::SIP.stream( + RecordingConnection.new, + output: StringIO.new, + stop_after: "response.done" + ) + end + + assert_equal("Realtime connection closed before response.done", error.message) + end + + def test_websocket_audio_rejects_a_connection_that_closes_before_response_done + error = assert_raises(RuntimeError) do + OpenAI::Examples::Realtime::WebSocketAudio.stream_response( + RecordingConnection.new, + output: StringIO.new + ) + end + + assert_equal("Realtime connection closed before response.done", error.message) + end + + def test_websocket_audio_accepts_audio_followed_by_a_completed_response + audio = "pcm".b + output = StringIO.new + connection = RecordingConnection.new( + [ + OpenAI::Realtime::ResponseAudioDeltaEvent.new( + event_id: "event_1", + response_id: "response_1", + item_id: "item_1", + output_index: 0, + content_index: 0, + delta: Base64.strict_encode64(audio) + ), + OpenAI::Realtime::ResponseDoneEvent.new( + event_id: "event_2", + response: OpenAI::Realtime::RealtimeResponse.new( + id: "response_1", + status: :completed + ) + ) + ] + ) + + result = OpenAI::Examples::Realtime::WebSocketAudio.stream_response( + connection, + output: output + ) + + assert_nil(result) + assert_equal(audio, output.string) + end + + def test_websocket_audio_rejects_a_completed_response_without_audio + connection = RecordingConnection.new( + [ + OpenAI::Realtime::ResponseDoneEvent.new( + event_id: "event_1", + response: OpenAI::Realtime::RealtimeResponse.new( + id: "response_1", + status: :completed + ) + ) + ] + ) + + error = assert_raises(RuntimeError) do + OpenAI::Examples::Realtime::WebSocketAudio.stream_response( + connection, + output: StringIO.new + ) + end + + assert_equal("Response completed without audio output", error.message) + end + + def test_websocket_text_rejects_a_connection_that_closes_before_response_done + error = assert_raises(RuntimeError) do + OpenAI::Examples::Realtime::WebSocketText.stream_response( + RecordingConnection.new, + output: StringIO.new + ) + end + + assert_equal("Realtime connection closed before response.done", error.message) + end + + def test_websocket_text_rejects_a_completed_response_without_text + connection = RecordingConnection.new([completed_response_event]) + output = StringIO.new + + error = assert_raises(RuntimeError) do + OpenAI::Examples::Realtime::WebSocketText.stream_response(connection, output: output) + end + + assert_equal("Realtime response completed without text output", error.message) + refute_includes(output.string, "response.done status=completed") + end + + def test_websocket_text_accepts_non_empty_text_and_a_completed_response + connection = RecordingConnection.new( + [ + OpenAI::Realtime::ResponseTextDeltaEvent.new( + content_index: 0, + delta: "Hello from Ruby.", + event_id: "event_1", + item_id: "item_1", + output_index: 0, + response_id: "response_1" + ), + completed_response_event + ] + ) + output = StringIO.new + + OpenAI::Examples::Realtime::WebSocketText.stream_response(connection, output: output) + + assert_includes(output.string, "Hello from Ruby.") + assert_includes(output.string, "response.done status=completed") + end + + def test_translation_rejects_a_connection_that_closes_before_session_closed + error = assert_raises(RuntimeError) do + OpenAI::Examples::Realtime::Translation.stream( + RecordingConnection.new, + audio_output: StringIO.new, + transcript_output: StringIO.new + ) + end + + assert_equal( + "Realtime translation connection closed before session.closed", + error.message + ) + end + + def test_translation_rejects_session_closed_without_audio + connection = RecordingConnection.new( + [ + OpenAI::Realtime::RealtimeTranslationSessionClosedEvent.new( + event_id: "event_1" + ) + ] + ) + + error = assert_raises(RuntimeError) do + OpenAI::Examples::Realtime::Translation.stream( + connection, + audio_output: StringIO.new, + transcript_output: StringIO.new + ) + end + + assert_equal("Translation session closed without audio output", error.message) + end + + def test_translation_accepts_audio_followed_by_session_closed + audio = "translated pcm".b + connection = RecordingConnection.new( + [ + OpenAI::Realtime::RealtimeTranslationOutputAudioDeltaEvent.new( + event_id: "event_1", + delta: Base64.strict_encode64(audio) + ), + OpenAI::Realtime::RealtimeTranslationSessionClosedEvent.new( + event_id: "event_2" + ) + ] + ) + audio_output = StringIO.new + transcript_output = StringIO.new + + result = OpenAI::Examples::Realtime::Translation.stream( + connection, + audio_output: audio_output, + transcript_output: transcript_output + ) + + assert_nil(result) + assert_equal(audio, audio_output.string) + assert_equal("\n", transcript_output.string) + end + + def test_transcription_rejects_a_connection_that_closes_before_completion + error = assert_raises(RuntimeError) do + OpenAI::Examples::Realtime::WebSocketTranscription.print_transcript( + RecordingConnection.new, + output: StringIO.new + ) + end + + assert_equal("Realtime connection closed before transcription completed", error.message) + end + + def test_transcription_accepts_only_a_completed_event + connection = RecordingConnection.new( + [ + OpenAI::Realtime::ConversationItemInputAudioTranscriptionCompletedEvent.new( + content_index: 0, + event_id: "event_1", + item_id: "item_1", + transcript: "Hello", + usage: {type: :duration, seconds: 0.25} + ) + ] + ) + output = StringIO.new + + result = OpenAI::Examples::Realtime::WebSocketTranscription.print_transcript( + connection, + output: output + ) + + assert_nil(result) + assert_equal("\n[item_1] Hello\n", output.string) + end + + def test_translation_preserves_an_upload_error_when_session_close_also_fails + upload_error = RuntimeError.new("upload failed") + input_audio_buffer = RaisingEndpoint.new(upload_error) + session = RaisingEndpoint.new(RuntimeError.new("close failed")) + connection = TranslationConnection.new(input_audio_buffer, session) + + Tempfile.create("translation-input") do |input| + input.write("audio") + input.flush + + error = assert_raises(RuntimeError) do + OpenAI::Examples::Realtime::Translation.write_input(connection, input.path) + end + + assert_same(upload_error, error) + end + assert_equal([[:append_bytes, "audio"]], input_audio_buffer.calls) + assert_equal([[:close]], session.calls) + end + + def test_translation_stops_uploading_when_the_reader_fails + connection = ReaderFailureConnection.new(realtime_error_event("translation failed")) + + Tempfile.create("translation-input") do |input| + input.write("audio") + input.flush + + error = Timeout.timeout(1) do + Sync do + assert_raises(RuntimeError) do + OpenAI::Examples::Realtime::Translation.exchange( + connection, + input_path: input.path, + audio_output: StringIO.new, + transcript_output: StringIO.new + ) + end + end + end + + assert_equal("translation failed", error.message) + end + assert_equal(["audio"], connection.input_audio_buffer.chunks) + assert(connection.session.closed) + end + + def test_translation_does_not_start_upload_after_a_buffered_reader_failure + connection = BufferedReaderFailureConnection.new( + realtime_error_event("translation failed before upload") + ) + + Tempfile.create("translation-input") do |input| + input.write("audio") + input.flush + + error = Sync do + assert_raises(RuntimeError) do + OpenAI::Examples::Realtime::Translation.exchange( + connection, + input_path: input.path, + audio_output: StringIO.new, + transcript_output: StringIO.new + ) + end + end + + assert_equal("translation failed before upload", error.message) + end + assert_empty(connection.input_audio_buffer.calls) + assert_empty(connection.session.calls) + end + + private def completed_response_event + OpenAI::Realtime::ResponseDoneEvent.new( + event_id: "event_done", + response: OpenAI::Realtime::RealtimeResponse.new( + id: "response_done", + status: :completed + ) + ) + end + + private def realtime_error_event(message) + OpenAI::Realtime::RealtimeErrorEvent.new( + event_id: "event_error", + error: OpenAI::Realtime::RealtimeError.new(type: "server_error", message: message) + ) + end +end diff --git a/test/openai/realtime/examples_test.rb b/test/openai/realtime/examples_test.rb new file mode 100644 index 000000000..11666713a --- /dev/null +++ b/test/openai/realtime/examples_test.rb @@ -0,0 +1,990 @@ +# frozen_string_literal: true + +require "open3" +require_relative "../test_helper" +require_relative "../../../examples/realtime/mcp_approval" +require_relative "../../../examples/realtime/realtime_conversation" +require_relative "../../../examples/realtime/sideband" +require_relative "../../../examples/realtime/sip" +require_relative "../../../examples/realtime/webrtc_conversation" +require_relative "../../../examples/realtime/websocket_audio" + +class OpenAI::Test::RealtimeExamplesTest < Minitest::Test + Event = Data.define(:type, :data) do + def to_h = data + end + + class RecordingSession + attr_reader :updates + + def initialize + @updates = [] + end + + def update(**params) + @updates << params + end + end + + class RecordingResource + attr_reader :calls, :truncations + + def initialize(writer_fibers = nil) + @calls = [] + @truncations = [] + @writer_fibers = writer_fibers + end + + def create(**params) + @calls << params + end + + def truncate(**params) + @writer_fibers&.push(Fiber.current) + @truncations << params + end + end + + class RecordingConversation + attr_reader :items + + def initialize(writer_fibers = nil) + @items = RecordingResource.new(writer_fibers) + end + end + + class RecordingAudioBuffer + attr_accessor :append_error + attr_reader :chunks, :commits + + def initialize(writer_fibers = nil) + @chunks = [] + @commits = 0 + @writer_fibers = writer_fibers + end + + def append_bytes(bytes) + @writer_fibers&.push(Fiber.current) + raise @append_error if @append_error + + @chunks << bytes + end + + def commit = @commits += 1 + end + + class RecordingOutbound + attr_reader :chunks, :truncations + + def initialize + @chunks = [] + @truncations = [] + end + + def append_audio(bytes) = @chunks << bytes + def truncate(**params) = @truncations << params + end + + class RecordingConnection + attr_reader :conversation, :input_audio_buffer, :received, :response, :session, :writer_fibers + + def initialize(events = []) + @events = events + @received = [] + @writer_fibers = [] + @session = RecordingSession.new + @conversation = RecordingConversation.new(@writer_fibers) + @input_audio_buffer = RecordingAudioBuffer.new(@writer_fibers) + @response = RecordingResource.new + end + + def each(&block) + @events.each(&block) + end + + def receive + @events.shift.tap { |event| @received << event&.type } + end + end + + class ExplodingConnection < RecordingConnection + def each + raise "sideband failed" + end + end + + class RecordingCalls + attr_accessor :hangup_error + attr_reader :accepts, :hangups + + def initialize + @accepts = [] + @hangups = [] + end + + def accept(call_id, **params) + @accepts << [call_id, params] + end + + def hangup(call_id) + @hangups << call_id + raise @hangup_error if @hangup_error + end + end + + class RecordingRealtime + attr_reader :calls, :connections + + def initialize(connection) + @calls = RecordingCalls.new + @connection = connection + @connections = [] + end + + def connect_to_call(call_id:) + @connections << call_id + yield(@connection) + end + + def connect(model:) + @connections << model + yield(@connection) + end + end + + RecordingClient = Data.define(:realtime) + + HTTPRequest = Data.define(:request_method, :path, :body, :headers) do + def [](name) = headers[name.downcase] + end + + class HTTPResponse + attr_accessor :body, :status + attr_reader :headers + + def initialize + @headers = {} + end + + def []=(name, value) + @headers[name] = value + end + end + + class RecordingWebRTCCalls + attr_accessor :hangup_error + attr_reader :creates, :hangups + + def initialize + @creates = [] + @hangups = [] + end + + def create(**params) + @creates << params + OpenAI::Realtime::CallCreateResponse.new( + sdp: "answer-sdp", + call_id: "rtc_example", + headers: {} + ) + end + + def hangup(call_id) + @hangups << call_id + raise @hangup_error if @hangup_error + end + end + + class RecordingMicrophone + attr_reader :stopped + + def initialize(chunks) + @chunks = chunks + @stopped = false + end + + def each_chunk(&block) + @chunks.each(&block) + end + + def stop = @stopped = true + end + + class RecordingSpeaker + attr_reader :audio, :interruptions + + def initialize + @audio = +"".b + @interruptions = 0 + end + + def write(bytes) + @audio << bytes + end + + def interrupt + @interruptions += 1 + end + + def close = nil + end + + def test_realtime_conversation_configures_continuous_server_vad + connection = RecordingConnection.new + + OpenAI::Examples::Realtime::Conversation.configure( + connection, + voice: :marin, + instructions: "Speak naturally." + ) + + assert_equal( + { + type: :realtime, + output_modalities: [:audio], + instructions: "Speak naturally.", + audio: { + input: { + format: {type: :"audio/pcm", rate: 24_000}, + noise_reduction: {type: :near_field}, + turn_detection: { + type: :server_vad, + threshold: 0.5, + prefix_padding_ms: 300, + silence_duration_ms: 500, + create_response: true, + interrupt_response: true + } + }, + output: { + format: {type: :"audio/pcm", rate: 24_000}, + voice: :marin + } + } + }, + connection.session.updates.fetch(0) + ) + end + + def test_realtime_conversation_streams_microphone_chunks_without_committing + outbound = RecordingOutbound.new + microphone = RecordingMicrophone.new(["first".b, "second".b]) + + OpenAI::Examples::Realtime::Conversation.forward_microphone(outbound, microphone) + + assert_equal(["first".b, "second".b], outbound.chunks) + end + + def test_realtime_conversation_serializes_audio_and_interruptions_through_one_writer + connection = RecordingConnection.new + outbound = OpenAI::Examples::Realtime::Conversation::OutboundWriter.new(connection) + + Sync do |task| + writer = task.async { outbound.run } + outbound.append_audio("first".b) + outbound.truncate(item_id: "item_1", content_index: 0, audio_end_ms: 100) + outbound.append_audio("second".b) + outbound.close + writer.wait + end + + assert_equal(["first".b, "second".b], connection.input_audio_buffer.chunks) + assert_equal( + [{item_id: "item_1", content_index: 0, audio_end_ms: 100}], + connection.conversation.items.truncations + ) + assert_equal(1, connection.writer_fibers.uniq.size) + end + + def test_realtime_conversation_unblocks_a_producer_when_the_writer_fails + connection = RecordingConnection.new + connection.input_audio_buffer.append_error = RuntimeError.new("write failed") + outbound = OpenAI::Examples::Realtime::Conversation::OutboundWriter.new(connection) + + error = Sync do |task| + writer = task.async do + outbound.run + nil + rescue StandardError => e + e + end + outbound.append_audio("first".b) + producer_error = assert_raises(RuntimeError) do + outbound.append_audio("second".b) + end + writer_error = writer.wait + + assert_same(writer_error, producer_error) + producer_error + end + + assert_equal("write failed", error.message) + end + + def test_realtime_conversation_handles_initial_speech_before_assistant_audio + speaker = RecordingSpeaker.new + outbound = RecordingOutbound.new + playback = OpenAI::Examples::Realtime::Conversation::AudioPlayback.new( + speaker, + clock: -> { raise "clock must not be read without playback" } + ) + + playback.interrupt(outbound) + + assert_empty(outbound.truncations) + assert_equal(1, speaker.interruptions) + end + + def test_realtime_conversation_propagates_writer_failure_without_deadlock + connection = RecordingConnection.new( + [OpenAI::Realtime::SessionUpdatedEvent.new(event_id: "event_1", session: {})] + ) + connection.input_audio_buffer.append_error = RuntimeError.new("write failed") + microphone = RecordingMicrophone.new(["first".b, "second".b]) + + error = Sync do + assert_raises(RuntimeError) do + OpenAI::Examples::Realtime::Conversation.run_session( + connection, + microphone: microphone, + speaker: RecordingSpeaker.new, + voice: :marin, + instructions: "Speak naturally.", + output: StringIO.new + ) + end + end + + assert_equal("write failed", error.message) + assert_predicate(microphone, :stopped) + end + + def test_realtime_conversation_latches_stop_before_ffmpeg_capture_starts + connection = RecordingConnection.new( + [OpenAI::Realtime::SessionUpdatedEvent.new(event_id: "event_1", session: {})] + ) + microphone = OpenAI::Examples::Realtime::Conversation::FFmpegMicrophone.new( + input_format: "test", + device: "test" + ) + spawn = ->(*) { raise "microphone spawned after stop" } + + result = Process.stub(:spawn, spawn) do + Sync do + OpenAI::Examples::Realtime::Conversation.run_session( + connection, + microphone: microphone, + speaker: RecordingSpeaker.new, + voice: :marin, + instructions: "Speak naturally.", + output: StringIO.new + ) + end + end + + assert_nil(result) + end + + def test_realtime_conversation_rejects_eof_before_the_requested_stop_event + microphone = RecordingMicrophone.new([]) + playback = OpenAI::Examples::Realtime::Conversation::AudioPlayback.new( + RecordingSpeaker.new + ) + + error = assert_raises(RuntimeError) do + OpenAI::Examples::Realtime::Conversation.stream_events( + RecordingConnection.new, + microphone: microphone, + outbound: RecordingOutbound.new, + playback: playback, + output: StringIO.new, + stop_after: "response.done" + ) + end + + assert_equal("Realtime connection closed before response.done", error.message) + assert_predicate(microphone, :stopped) + ensure + playback&.close + end + + def test_realtime_conversation_streams_audio_and_interrupts_local_playback + speaker = RecordingSpeaker.new + times = [0.0, 0.1].each + playback = OpenAI::Examples::Realtime::Conversation::AudioPlayback.new( + speaker, + clock: -> { times.next } + ) + output = StringIO.new + outbound = RecordingOutbound.new + audio = "\x00\x01".b * 2_400 + event_fields = { + event_id: "event_1", + response_id: "response_1", + item_id: "item_1", + output_index: 0, + content_index: 0 + } + + OpenAI::Examples::Realtime::Conversation.handle_event( + OpenAI::Realtime::ResponseAudioDeltaEvent.new(**event_fields, delta: Base64.strict_encode64(audio)), + outbound: outbound, + playback: playback, + output: output + ) + OpenAI::Examples::Realtime::Conversation.handle_event( + OpenAI::Realtime::ResponseAudioTranscriptDeltaEvent.new(**event_fields, delta: "Hello"), + outbound: outbound, + playback: playback, + output: output + ) + OpenAI::Examples::Realtime::Conversation.handle_event( + OpenAI::Realtime::InputAudioBufferSpeechStartedEvent.new( + event_id: "event_2", + item_id: "item_2", + audio_start_ms: 100 + ), + outbound: outbound, + playback: playback, + output: output + ) + + assert_equal(audio, speaker.audio) + assert_equal("Hello\n", output.string) + assert_equal(1, speaker.interruptions) + assert_equal( + [{item_id: "item_1", content_index: 0, audio_end_ms: 100}], + outbound.truncations + ) + end + + def test_realtime_conversation_drops_queued_audio_after_interruption + speaker = RecordingSpeaker.new + times = [0.0, 0.05].each + playback = OpenAI::Examples::Realtime::Conversation::AudioPlayback.new( + speaker, + clock: -> { times.next } + ) + output = StringIO.new + outbound = RecordingOutbound.new + event_fields = { + event_id: "event_1", + response_id: "response_1", + item_id: "item_1", + output_index: 0, + content_index: 0 + } + first_audio = "\x00\x01".b * 2_400 + queued_audio = "\x02\x03".b * 2_400 + + playback.write( + OpenAI::Realtime::ResponseAudioDeltaEvent.new( + **event_fields, + delta: Base64.strict_encode64(first_audio) + ) + ) + OpenAI::Examples::Realtime::Conversation.handle_event( + OpenAI::Realtime::InputAudioBufferSpeechStartedEvent.new( + event_id: "event_2", + item_id: "item_2", + audio_start_ms: 50 + ), + outbound: outbound, + playback: playback, + output: output + ) + playback.write( + OpenAI::Realtime::ResponseAudioDeltaEvent.new( + **event_fields, + event_id: "event_3", + delta: Base64.strict_encode64(queued_audio) + ) + ) + OpenAI::Examples::Realtime::Conversation.handle_event( + OpenAI::Realtime::ResponseDoneEvent.new( + event_id: "event_4", + response: OpenAI::Realtime::RealtimeResponse.new(id: "response_1", status: :cancelled) + ), + outbound: outbound, + playback: playback, + output: output + ) + + assert_equal(first_audio, speaker.audio) + assert_equal("\n", output.string) + assert_equal(1, speaker.interruptions) + end + + def test_realtime_conversation_does_not_truncate_fully_played_completed_audio + speaker = RecordingSpeaker.new + outbound = RecordingOutbound.new + times = [0.0, 0.05, 0.2].each + playback = OpenAI::Examples::Realtime::Conversation::AudioPlayback.new( + speaker, + clock: -> { times.next } + ) + fields = { + event_id: "event_1", + response_id: "response_1", + item_id: "item_1", + output_index: 0, + content_index: 0 + } + audio = "\x00\x01".b * 2_400 + + playback.write( + OpenAI::Realtime::ResponseAudioDeltaEvent.new( + **fields, + delta: Base64.strict_encode64(audio) + ) + ) + playback.finish("response_1") + playback.interrupt(outbound) + + assert_empty(outbound.truncations) + assert_equal(1, speaker.interruptions) + end + + def test_realtime_conversation_can_capture_output_audio_for_a_live_smoke_test + Tempfile.create(["realtime-conversation", ".pcm"]) do |file| + speaker = OpenAI::Examples::Realtime::Conversation::PCMFileSpeaker.new(file.path) + + speaker.write("\x00\x01".b) + speaker.interrupt + speaker.write("\x02\x03".b) + speaker.close + + assert_equal("\x00\x01\x02\x03".b, File.binread(file.path)) + end + end + + def test_realtime_conversation_uses_ffplays_input_channel_layout_option + speaker_class = Class.new(OpenAI::Examples::Realtime::Conversation::FFplaySpeaker) do + def playback_command = command + end + command = speaker_class.new.playback_command + + assert_equal(["-ch_layout", "mono"], command.slice(command.index("-ch_layout"), 2)) + refute_includes(command, "-ac") + end + + def test_realtime_conversation_ffplay_command_accepts_raw_pcm + speaker_class = Class.new(OpenAI::Examples::Realtime::Conversation::FFplaySpeaker) do + def playback_command = command + end + command = speaker_class.new.playback_command + command.insert(command.index("-i"), "-autoexit") + + _, stderr, status = Open3.capture3( + {"SDL_AUDIODRIVER" => "dummy"}, + *command, + stdin_data: "\0".b * 4_800 + ) + + assert_predicate(status, :success?, stderr) + assert_empty(stderr, "ffplay must not emit terminal control sequences") + rescue Errno::ENOENT + skip("ffplay is not installed") + end + + def test_webrtc_conversation_serves_a_browser_peer_with_audio_processing + calls = RecordingWebRTCCalls.new + app = OpenAI::Examples::Realtime::WebRTCConversation::App.new( + client: RecordingClient.new(Data.define(:calls).new(calls)) + ) + response = HTTPResponse.new + + app.handle( + HTTPRequest.new( + request_method: "GET", + path: "/", + body: "", + headers: {} + ), + response + ) + + assert_equal(200, response.status) + assert_equal("text/html; charset=utf-8", response.headers.fetch("Content-Type")) + assert_includes(response.body, "new RTCPeerConnection()") + assert_includes(response.body, "echoCancellation: true") + assert_includes(response.body, "noiseSuppression: true") + assert_includes(response.body, "autoGainControl: true") + refute_includes(response.body, "OPENAI_API_KEY") + end + + def test_webrtc_conversation_negotiates_and_hangs_up_through_ruby + calls = RecordingWebRTCCalls.new + app = OpenAI::Examples::Realtime::WebRTCConversation::App.new( + client: RecordingClient.new(Data.define(:calls).new(calls)), + html: "test" + ) + response = HTTPResponse.new + + app.handle( + HTTPRequest.new( + request_method: "POST", + path: "/session", + body: "v=0\r\nt=0 0\r\n", + headers: {"content-type" => "application/sdp"} + ), + response + ) + + assert_equal(201, response.status) + assert_equal("answer-sdp", response.body) + assert_equal("rtc_example", response.headers.fetch("X-OpenAI-Call-ID")) + assert_equal("v=0\r\nt=0 0\r\n", calls.creates.fetch(0).fetch(:sdp)) + session = calls.creates.fetch(0).fetch(:session) + assert_equal("gpt-realtime-2.1", session.fetch(:model)) + assert_equal(true, session.dig(:audio, :input, :turn_detection, :interrupt_response)) + + hangup_response = HTTPResponse.new + app.handle( + HTTPRequest.new( + request_method: "POST", + path: "/hangup", + body: "rtc_example", + headers: {} + ), + hangup_response + ) + + assert_equal(204, hangup_response.status) + assert_equal(["rtc_example"], calls.hangups) + end + + def test_webrtc_conversation_rejects_non_sdp_requests + calls = RecordingWebRTCCalls.new + app = OpenAI::Examples::Realtime::WebRTCConversation::App.new( + client: RecordingClient.new(Data.define(:calls).new(calls)), + html: "test" + ) + response = HTTPResponse.new + + app.handle( + HTTPRequest.new( + request_method: "POST", + path: "/session", + body: "not sdp", + headers: {"content-type" => "text/plain"} + ), + response + ) + + assert_equal(400, response.status) + assert_empty(calls.creates) + end + + def test_webrtc_conversation_treats_an_already_closed_call_as_hung_up + calls = RecordingWebRTCCalls.new + calls.hangup_error = OpenAI::Errors::NotFoundError.new( + url: URI("https://api.openai.com/v1/realtime/calls/rtc_example/hangup"), + status: 404, + headers: {}, + body: {}, + request: nil, + response: nil + ) + app = OpenAI::Examples::Realtime::WebRTCConversation::App.new( + client: RecordingClient.new(Data.define(:calls).new(calls)), + html: "test" + ) + app.handle( + HTTPRequest.new( + request_method: "POST", + path: "/session", + body: "v=0\r\nt=0 0\r\n", + headers: {"content-type" => "application/sdp"} + ), + HTTPResponse.new + ) + response = HTTPResponse.new + + app.handle( + HTTPRequest.new( + request_method: "POST", + path: "/hangup", + body: "rtc_example", + headers: {} + ), + response + ) + + assert_equal(204, response.status) + assert_equal(["rtc_example"], calls.hangups) + end + + def test_mcp_console_example_requests_text_output + connection = RecordingConnection.new + + OpenAI::Examples::Realtime::MCPApproval.configure( + connection, + server_url: "https://developers.openai.com/mcp" + ) + + session = connection.session.updates.fetch(0) + assert_equal([:text], session.fetch(:output_modalities)) + refute(session.key?(:tool_choice)) + end + + def test_mcp_console_example_requires_the_discovered_tool + tools_item = OpenAI::Realtime::RealtimeMcpListTools.new( + id: "mcp_tools_1", + server_label: "remote", + tools: [ + OpenAI::Realtime::RealtimeMcpListTools::Tool.new( + name: "search_openai_docs", + description: "Search the OpenAI developer documentation", + input_schema: {type: "object"} + ) + ] + ) + connection = RecordingConnection.new( + [ + OpenAI::Realtime::ConversationItemDone.new( + event_id: "event_1", + item: tools_item + ), + OpenAI::Realtime::McpListToolsCompleted.new( + event_id: "event_2", + item_id: "mcp_tools_1" + ), + OpenAI::Realtime::SessionUpdatedEvent.new( + event_id: "event_3", + session: {} + ), + completed_response_event + ] + ) + + OpenAI::Examples::Realtime::MCPApproval.run_session( + connection, + prompt: "Search the docs", + output: StringIO.new + ) + + assert_equal( + { + type: :realtime, + tools: [{type: :mcp, server_label: "remote"}], + tool_choice: {type: :mcp, server_label: "remote", name: "search_openai_docs"} + }, + connection.session.updates.fetch(0) + ) + assert_equal({}, connection.response.calls.fetch(0)) + end + + def test_mcp_console_example_waits_for_the_tool_choice_update + tools_item = OpenAI::Realtime::RealtimeMcpListTools.new( + id: "mcp_tools_1", + server_label: "remote", + tools: [ + OpenAI::Realtime::RealtimeMcpListTools::Tool.new( + name: "search_openai_docs", + input_schema: {type: "object"} + ) + ] + ) + connection = RecordingConnection.new( + [ + OpenAI::Realtime::ConversationItemDone.new(event_id: "event_1", item: tools_item), + OpenAI::Realtime::McpListToolsCompleted.new(event_id: "event_2", item_id: "mcp_tools_1") + ] + ) + + error = assert_raises(RuntimeError) do + OpenAI::Examples::Realtime::MCPApproval.run_session( + connection, + prompt: "Search the docs", + output: StringIO.new + ) + end + + assert_equal("Realtime connection closed before the final response.done", error.message) + assert_empty(connection.response.calls) + assert_empty(connection.conversation.items.calls) + end + + def test_mcp_console_example_requests_a_final_answer_after_the_tool_completes + connection = RecordingConnection.new( + [ + OpenAI::Realtime::ResponseMcpCallCompleted.new( + event_id: "event_1", + item_id: "mcp_call_1", + output_index: 0 + ), + completed_response_event + ] + ) + + OpenAI::Examples::Realtime::MCPApproval.run_session( + connection, + prompt: "Search the docs", + output: StringIO.new + ) + + assert_equal({tool_choice: :none}, connection.response.calls.fetch(0)) + end + + def test_sideband_smoke_mode_stops_after_the_selected_event + events = [ + Event.new(type: :"session.updated", data: {type: :"session.updated"}), + Event.new(type: :"response.done", data: {type: :"response.done"}) + ] + connection = RecordingConnection.new(events) + output = StringIO.new + + OpenAI::Examples::Realtime::Sideband.stream( + connection, + output: output, + stop_after: "session.updated" + ) + + assert_includes(output.string, "session.updated") + refute_includes(output.string, "response.done") + end + + def test_sip_example_accepts_then_attaches_to_the_call + connection = RecordingConnection.new( + [Event.new(type: :"session.created", data: {type: :"session.created"})] + ) + realtime = RecordingRealtime.new(connection) + + OpenAI::Examples::Realtime::SIP.run( + client: RecordingClient.new(realtime: realtime), + call_id: "rtc_123", + model: "gpt-realtime-2.1", + output: StringIO.new, + stop_after: "session.created" + ) + + assert_equal(["rtc_123"], realtime.connections) + assert_equal( + [ + "rtc_123", + { + type: :realtime, + model: "gpt-realtime-2.1", + instructions: "You are answering a phone call. Be warm and concise." + } + ], + realtime.calls.accepts.fetch(0) + ) + assert_equal(["rtc_123"], realtime.calls.hangups) + end + + def test_sip_example_hangs_up_without_masking_a_sideband_error + realtime = RecordingRealtime.new(ExplodingConnection.new) + realtime.calls.hangup_error = RuntimeError.new("hangup failed") + + error = assert_raises(RuntimeError) do + OpenAI::Examples::Realtime::SIP.run( + client: RecordingClient.new(realtime: realtime), + call_id: "rtc_123", + model: "gpt-realtime-2.1", + output: StringIO.new + ) + end + + assert_equal("sideband failed", error.message) + assert_equal(["rtc_123"], realtime.calls.hangups) + end + + def test_sip_example_tolerates_an_already_ended_call_during_cleanup + realtime = RecordingRealtime.new(RecordingConnection.new) + realtime.calls.hangup_error = OpenAI::Errors::NotFoundError.new( + url: URI("https://api.openai.com/v1/realtime/calls/rtc_123/hangup"), + status: 404, + headers: {}, + body: {}, + request: nil, + response: nil + ) + + OpenAI::Examples::Realtime::SIP.run( + client: RecordingClient.new(realtime: realtime), + call_id: "rtc_123", + model: "gpt-realtime-2.1", + output: StringIO.new + ) + + assert_equal(["rtc_123"], realtime.calls.hangups) + end + + def test_sip_example_reports_a_cleanup_failure_after_a_clean_session + realtime = RecordingRealtime.new(RecordingConnection.new) + realtime.calls.hangup_error = RuntimeError.new("hangup failed") + + error = assert_raises(RuntimeError) do + OpenAI::Examples::Realtime::SIP.run( + client: RecordingClient.new(realtime: realtime), + call_id: "rtc_123", + model: "gpt-realtime-2.1", + output: StringIO.new + ) + end + + assert_equal("hangup failed", error.message) + assert_equal(["rtc_123"], realtime.calls.hangups) + end + + def test_websocket_audio_disables_vad_and_waits_for_session_updated + audio = "assistant pcm".b + connection = RecordingConnection.new( + [ + OpenAI::Realtime::SessionUpdatedEvent.new(event_id: "event_1", session: {}), + OpenAI::Realtime::ResponseAudioDeltaEvent.new( + event_id: "event_2", + response_id: "response_1", + item_id: "item_1", + output_index: 0, + content_index: 0, + delta: Base64.strict_encode64(audio) + ), + completed_response_event + ] + ) + realtime = RecordingRealtime.new(connection) + + Tempfile.create("realtime-input") do |input| + input.write("input pcm") + input.flush + Tempfile.create("realtime-output") do |output| + _stdout, _stderr = capture_io do + OpenAI::Examples::Realtime::WebSocketAudio.run( + client: RecordingClient.new(realtime: realtime), + model: "gpt-realtime-2.1", + input_path: input.path, + output_path: output.path + ) + end + + assert_equal(audio, File.binread(output.path)) + end + end + + assert_equal( + { + type: :realtime, + output_modalities: [:audio], + audio: {input: {turn_detection: nil}} + }, + connection.session.updates.fetch(0) + ) + assert_equal(["input pcm"], connection.input_audio_buffer.chunks) + assert_equal(1, connection.input_audio_buffer.commits) + assert_equal([{}], connection.response.calls) + assert_equal([:"session.updated"], connection.received) + end + + private def completed_response_event + OpenAI::Realtime::ResponseDoneEvent.new( + event_id: "event_done", + response: OpenAI::Realtime::RealtimeResponse.new( + id: "response_done", + status: :completed + ) + ) + end +end diff --git a/test/openai/resources/realtime/calls_test.rb b/test/openai/resources/realtime/calls_test.rb index 938be40ed..01b40ec12 100644 --- a/test/openai/resources/realtime/calls_test.rb +++ b/test/openai/resources/realtime/calls_test.rb @@ -3,6 +3,125 @@ require_relative "../../test_helper" class OpenAI::Test::Resources::Realtime::CallsTest < OpenAI::Test::ResourceTest + class CapturingLogger + attr_reader :events + + def initialize = @events = [] + + def debug(message) = @events << [:debug, message] + def info(message) = @events << [:info, message] + def warn(message) = @events << [:warn, message] + def error(message) = @events << [:error, message] + end + + class StubHTTPClient < OpenAI::HTTPClient + attr_reader :request + + def execute(request) + @request = request + OpenAI::HTTPClient::Response.new( + status: 201, + headers: { + "Content-Type" => "application/sdp", + "Location" => "/v1/realtime/calls/rtc_123", + "X-Request-ID" => "req_realtime_call" + }, + body: "answer-sdp" + ) + end + end + + class MalformedLocationHTTPClient < OpenAI::HTTPClient + def execute(_request) + OpenAI::HTTPClient::Response.new( + status: 201, + headers: { + "Content-Type" => "application/sdp", + "Location" => "http://[malformed", + "X-Request-ID" => "req_malformed_location" + }, + body: "usable-answer-sdp" + ) + end + end + + def test_create_with_an_sdp_offer + http_client = StubHTTPClient.new + client = OpenAI::Client.new( + api_key: "test-key", + base_url: "https://example.com/v1", + http_client: http_client + ) + + response = client.realtime.calls.create(sdp: "offer-sdp") + + assert_equal("answer-sdp", response.sdp) + assert_equal("rtc_123", response.call_id) + assert_equal("req_realtime_call", response._request_id) + assert_equal(201, response.last_response.status) + assert_equal("/v1/realtime/calls/rtc_123", response.headers.fetch("location")) + assert_equal(:post, http_client.request.method) + assert_equal("https://example.com/v1/realtime/calls", http_client.request.url.to_s) + assert_equal("application/sdp", http_client.request.headers.fetch("content-type")) + assert_equal("offer-sdp", http_client.request.body) + end + + def test_create_with_session_configuration_uses_typed_multipart_parts + http_client = StubHTTPClient.new + client = OpenAI::Client.new( + api_key: "test-key", + base_url: "https://example.com/v1", + http_client: http_client + ) + + client.realtime.calls.create( + sdp: "offer-sdp", + session: {type: :realtime, model: "gpt-realtime"} + ) + + content_type = http_client.request.headers.fetch("content-type") + body = http_client.request.body.to_a.join + + assert_match(%r{\Amultipart/form-data; boundary=}, content_type) + assert_includes(body, "name=\"sdp\"\r\nContent-Type: application/sdp\r\n\r\noffer-sdp") + assert_includes(body, "name=\"session\"\r\nContent-Type: application/json") + assert_includes(body, JSON.generate(type: :realtime, model: "gpt-realtime")) + refute_includes(body, "filename=") + end + + def test_create_preserves_request_observability_for_raw_sdp_responses + logger = CapturingLogger.new + client = OpenAI::Client.new( + api_key: "test-key", + base_url: "https://example.com/v1", + http_client: StubHTTPClient.new, + logger: logger, + log_level: :debug + ) + + response = client.realtime.calls.create(sdp: "offer-sdp") + + assert_equal("answer-sdp", response.sdp) + log = logger.events.map(&:last).join("\n") + assert_includes(log, "request complete") + assert_includes(log, "request_id=req_realtime_call") + assert_includes(log, "response body") + end + + def test_create_ignores_a_malformed_optional_location_header + client = OpenAI::Client.new( + api_key: "test-key", + base_url: "https://example.com/v1", + http_client: MalformedLocationHTTPClient.new + ) + + response = client.realtime.calls.create(sdp: "offer-sdp") + + assert_equal("usable-answer-sdp", response.sdp) + assert_nil(response.call_id) + assert_equal("req_malformed_location", response._request_id) + end + def test_accept_required_params response = @openai.realtime.calls.accept("call_id", type: :realtime) diff --git a/test/openai/resources/realtime/translations_test.rb b/test/openai/resources/realtime/translations_test.rb new file mode 100644 index 000000000..56399536d --- /dev/null +++ b/test/openai/resources/realtime/translations_test.rb @@ -0,0 +1,184 @@ +# frozen_string_literal: true + +require_relative "../../test_helper" + +class OpenAI::Test::Resources::Realtime::TranslationsTest < Minitest::Test + class StubHTTPClient < OpenAI::HTTPClient + attr_reader :requests + + def initialize(*responses) + super() + @responses = responses + @requests = [] + end + + def execute(request) + @requests << request + @responses.shift + end + end + + class FakeSocket + attr_reader :writes + + def initialize(*reads) + @reads = reads + @writes = [] + @closed = false + end + + def read = @reads.shift + def write(message) = @writes << message + def closed? = @closed + def close(code: 1000, reason: "") = (@closed = [code, reason].any?) + end + + class FakeTransport + attr_reader :url + + def initialize(socket) + @socket = socket + end + + def open(url:, **_options) + @url = url + yield(@socket) + end + end + + def test_create_client_secret + http_client = StubHTTPClient.new( + OpenAI::HTTPClient::Response.new( + status: 200, + headers: {"content-type" => "application/json"}, + body: JSON.generate( + value: "ek_test", + expires_at: 123, + session: { + id: "sess_123", + type: "translation", + model: "gpt-realtime-translate", + expires_at: 456, + audio: {} + } + ) + ) + ) + client = OpenAI::Client.new(api_key: "test-key", http_client: http_client) + + response = client.realtime.translations.client_secrets.create( + session: { + model: "gpt-realtime-translate", + audio: {output: {language: "es"}} + } + ) + + assert_instance_of(OpenAI::Realtime::RealtimeTranslationClientSecretCreateResponse, response) + assert_equal("ek_test", response.value) + request = http_client.requests.fetch(0) + assert_equal("/v1/realtime/translations/client_secrets", request.url.path) + assert_equal( + { + session: { + model: "gpt-realtime-translate", + audio: {output: {language: "es"}} + } + }, + JSON.parse(request.body, symbolize_names: true) + ) + end + + def test_create_client_secret_rejects_an_invalid_session_before_http + http_client = StubHTTPClient.new + client = OpenAI::Client.new(api_key: "test-key", http_client: http_client) + + error = assert_raises(ArgumentError) do + client.realtime.translations.client_secrets.create(session: {audio: {}}) + end + + assert_includes(error.message, "missing required fields") + assert_empty(http_client.requests) + end + + def test_create_translation_call + http_client = StubHTTPClient.new( + OpenAI::HTTPClient::Response.new( + status: 201, + headers: { + "content-type" => "application/sdp", + "location" => "/v1/realtime/calls/rtc_translation", + "x-request-id" => "req_translation_call" + }, + body: "answer-sdp" + ) + ) + client = OpenAI::Client.new(api_key: "ek_test", http_client: http_client) + + response = client.realtime.translations.calls.create(sdp: "offer-sdp") + + assert_equal("answer-sdp", response.sdp) + assert_equal("rtc_translation", response.call_id) + assert_equal("req_translation_call", response._request_id) + assert_equal(201, response.last_response.status) + request = http_client.requests.fetch(0) + assert_equal("/v1/realtime/calls", request.url.path) + assert_equal("application/sdp", request.headers.fetch("content-type")) + assert_equal("offer-sdp", request.body) + end + + def test_connect_translation_session_with_typed_events_and_helpers + socket = FakeSocket.new( + JSON.generate( + type: "session.output_transcript.delta", + event_id: "event_1", + delta: "hola", + item_id: "item_1" + ) + ) + transport = FakeTransport.new(socket) + client = OpenAI::Client.new(api_key: "test-key") + event = nil + + client.realtime.translations.connect( + model: "gpt-realtime-translate", + transport: transport + ) do |connection| + assert_instance_of(OpenAI::Realtime::TranslationConnection, connection) + refute_respond_to(connection, :response) + refute_respond_to(connection, :conversation) + refute_respond_to(connection, :output_audio_buffer) + assert_respond_to(connection.session, :close) + refute_respond_to(connection.input_audio_buffer, :commit) + refute_respond_to(connection.input_audio_buffer, :clear) + connection.session.update(audio: {output: {language: "es"}}) + connection.input_audio_buffer.append_bytes("\x00\x01".b) + connection.session.close + event = connection.receive + end + + assert_equal( + "wss://api.openai.com/v1/realtime/translations?model=gpt-realtime-translate", + transport.url.to_s + ) + assert_instance_of(OpenAI::Realtime::RealtimeTranslationOutputTranscriptDeltaEvent, event) + assert_equal("hola", event.delta) + assert_equal( + [ + "session.update", + "session.input_audio_buffer.append", + "session.close" + ], + socket.writes.map { JSON.parse(_1).fetch("type") } + ) + end + + def test_connect_requires_a_block + client = OpenAI::Client.new(api_key: "test-key") + + error = assert_raises(ArgumentError) do + client.realtime.translations.connect(model: "gpt-realtime-translate") + end + + assert_includes(error.message, "block is required") + end +end