// @ts-check /** * Minimal WebRTC client for a speech-to-speech realtime endpoint. * * Sibling of `ws/s2s-ws-client.js` with the SAME public surface (constructor * options subset, methods, dispatched events), so `main.js` can pick either * class from the transport setting and wire it identically. Direct mode only: * there is no load-balancer/queue path over WebRTC yet. * * Handshake (OpenAI Realtime GA "calls" endpoint, via the same-origin proxy): * * 1. getUserMedia -> add the mic track + create the `oai-events` data * channel on an RTCPeerConnection, `createOffer`, wait for ICE gathering. * 2. POST the offer SDP (Content-Type: application/sdp) to `callsUrl` * (usually the same-origin `api/calls` proxy, which forwards it to the * s2s server's /v1/realtime/calls) -> 201 + answer SDP. * 3. `setRemoteDescription(answer)`; once the data channel opens the server * pushes `session.created` and we reply with `session.update`. * * After that the JSON protocol on the data channel is the one the WebSocket * transport speaks, with the audio moved out of it: * * - Mic audio rides the RTP media track (Opus), NOT * `input_audio_buffer.append` — the server rejects `append` over WebRTC. * - Assistant audio arrives as a remote media track, NOT as * `response.output_audio.delta` events. There is no client playback * buffer: barge-in flushing happens server-side, so `speech_started` * only needs to flip the UI state here. * * Because no audio events exist, "the assistant is audibly speaking" is * detected from the output analyser's RMS (with a short hang time), and * `response.done` / `speech_started` remain the authoritative exits — the * same status contract the WS client exposes. * * @typedef {"idle" | "connecting" | "connected" | "user-speaking" | * "processing" | "ai-speaking" | "closed" | "error" * } RtcStatus * * @typedef {Object} RtcClientOptions * @property {string} callsUrl URL to POST the SDP offer to (same-origin proxy * like `api/calls`, or a direct `/v1/realtime/calls` URL when CORS allows). * @property {RTCIceServer[]} [iceServers] STUN/TURN servers for the peer * connection (from /api/config). Empty/absent -> browser defaults (host * candidates only — fine locally, may not traverse NATs). * @property {string} voice * @property {string} instructions * @property {string} [startupGreeting] Hidden user prompt that asks the model * to greet once after the initial session configuration. * @property {MediaStream} [micStream] Live mic stream. Provide this OR `acquireMic`. * @property {() => Promise} [acquireMic] Lazily obtain the mic * stream once connect() actually runs (same contract as the WS client). * @property {AudioContext} [audioContext] Pre-created (and resumed) context — * created inside the tap gesture so iOS lets it start. * @property {import("../ws/s2s-ws-client.js").ToolDef[]} [tools] Function tools * declared in the initial `session.update`. * @property {string} [audioOutputId] MediaDeviceInfo.deviceId for speakers * (AudioContext.setSinkId when supported). */ import { extractResponseTranscript } from "../ws/codec.js"; import { OrbVisualiser, VIS_FFT_SIZE } from "../ws/orb-visualizer.js"; // The server's data channel label (aiortc side ignores any other label). const DATA_CHANNEL_LABEL = "oai-events"; // Give up on the handshake if the data channel hasn't opened this long after // the SDP answer was applied (ICE failed silently, e.g. NAT without STUN). // The server holds its slot behind a 30 s watchdog; stay under it. const DC_OPEN_TIMEOUT_MS = 20_000; // Don't wait forever for ICE gathering before POSTing the offer — host // candidates land near-instantly; STUN answers within a couple of seconds. const ICE_GATHERING_TIMEOUT_MS = 3_000; // Output-RMS gate for "the assistant is audibly speaking": open above the // threshold, and hang on briefly so inter-word gaps don't flap the status. const SPEAKING_OPEN_DB = -50; const SPEAKING_HANG_MS = 250; const LEVEL_POLL_MS = 50; /** Build an Error carrying a `code` so callers can branch on the failure kind. * @param {string} message @param {string} code */ function _codedError(message, code) { const err = /** @type {Error & { code?: string }} */ (new Error(message)); err.code = code; return err; } export class S2sRtcRealtimeClient extends EventTarget { /** @param {RtcClientOptions} options */ constructor(options) { super(); /** @type {RtcClientOptions} */ this.options = options; /** @type {import("../ws/s2s-ws-client.js").ToolDef[]} */ this._tools = options.tools ?? []; /** @type {(() => Promise) | null} */ this._acquireMic = options.acquireMic ?? null; /** @type {boolean} Set by close() so late async steps unwind quietly. */ this._closed = false; /** @type {RTCPeerConnection | null} */ this._pc = null; /** @type {RTCDataChannel | null} */ this._dc = null; /** @type {AudioContext | null} */ this._ctx = null; /** @type {MediaStreamAudioSourceNode | null} */ this._micSrc = null; /** @type {MediaStreamAudioSourceNode | null} */ this._remoteSrc = null; /** @type {HTMLAudioElement | null} Muted sink for the Chrome quirk: a * remote WebRTC track stays silent in WebAudio unless the stream is ALSO * attached to an