Add user audio replay

main.js

@@ -378,7 +378,13 @@ function pushToolsToSession() {
 // ── Chat view ───────────────────────────────────────────────────────────────
 // Owns the history panel, the ephemeral bubbles, and all transcript/tool
 // streaming state. The client's events are forwarded to its on* methods.
-const chat = new ChatView();
+let userAudioReplaying = false;
+const chat = new ChatView({
+  onUserAudioPlaybackChange(playing) {
+    userAudioReplaying = playing;
+    syncMicMuteState();
+  },
+});
 
 // ── Account / limiter ─────────────────────────────────────────────────────
 // Login chip + daily-limit modal (inert unless the deploy is in LB mode). The
@@ -400,6 +406,15 @@ let client = null;
 let micStream = null;
 let micMuted = false;
 
+/** Apply both the user's mute choice and the temporary replay guard. */
+function syncMicMuteState() {
+  const muted = micMuted || userAudioReplaying;
+  for (const track of micStream?.getAudioTracks() ?? []) {
+    track.enabled = !muted;
+  }
+  client?.setMuted(muted);
+}
+
 /** @param {AppState} next */
 function setState(next) {
   currentState = next;
@@ -1209,10 +1224,7 @@ async function handleStartError(err) {
 micBtn.addEventListener("click", () => {
   if (!micStream || !client) return;
   micMuted = !micMuted;
-  for (const track of micStream.getAudioTracks()) {
-    track.enabled = !micMuted;
-  }
-  client.setMuted(micMuted);
+  syncMicMuteState();
   micBtn.classList.toggle("muted", micMuted);
   micBtn.setAttribute("aria-label", micMuted ? "Unmute" : "Mute");
   micBtn.title = micMuted ? "Unmute" : "Mute";
@@ -1457,6 +1469,7 @@ async function doStart(audioContext = null) {
         ...common,
       });
   client = c;
+  c.setMuted(micMuted || userAudioReplaying);
 
   c.addEventListener("queue", (e) => {
     const { position, queueId } = /** @type {CustomEvent<{ position: number; queueId: string }>} */ (e).detail;
@@ -1485,6 +1498,10 @@ async function doStart(audioContext = null) {
     const d = /** @type {CustomEvent<{ role: "user" | "assistant"; text: string; partial: boolean; itemId?: string; responseId?: string }>} */ (e).detail;
     chat.onTranscript(d);
   });
+  c.addEventListener("user-audio", (e) => {
+    const detail = /** @type {CustomEvent<{ itemId?: string; audio: Blob; durationMs?: number; truncated?: boolean }>} */ (e).detail;
+    chat.onUserAudio(detail);
+  });
 
   c.addEventListener("response-finished", (e) => {
     const detail = /** @type {CustomEvent<{ responseId: string; status: string; audible?: boolean; transcript?: string }>} */ (e).detail;

style.css

@@ -1887,6 +1887,29 @@ body.cam-on .footer {
  * distinguishing, so the panel reads as one quiet column. */
 .hist-msg.user .hist-body.partial { opacity: 0.65; }
 
+.hist-audio {
+  display: flex;
+  flex-direction: column;
+  gap: 5px;
+  padding: 8px 10px;
+  border: 1px solid color-mix(in srgb, var(--voice-user) 22%, var(--border));
+  border-radius: var(--radius-md);
+  background: color-mix(in srgb, var(--voice-user) 6%, var(--bg-elev-2));
+}
+.hist-audio-label {
+  font-family: var(--font-mono);
+  font-size: 9px;
+  letter-spacing: 0.06em;
+  text-transform: uppercase;
+  color: color-mix(in srgb, var(--voice-user) 68%, var(--text-faint));
+}
+.hist-audio audio {
+  display: block;
+  width: min(260px, 100%);
+  height: 34px;
+  color-scheme: light dark;
+}
+
 /* ─── Tool call history item ─────────────────────────────────────────── */
 
 .hist-tool-header {

ui/chat.js

@@ -1,9 +1,9 @@
 // @ts-check
 /**
  * ChatView — owns the whole conversation surface: the slide-in history panel,
- * the ephemeral on-orb bubbles, and all the transcript/tool/streaming
- * bookkeeping. main.js wires the realtime client's events straight to the
- * `on*` methods here and otherwise doesn't touch chat state.
+ * the ephemeral on-orb bubbles, and all the transcript/tool/streaming and
+ * user-audio bookkeeping. main.js wires the realtime client's events straight
+ * to the `on*` methods here and otherwise doesn't touch chat state.
  *
  * Two parallel surfaces share one shape (see `_buildMessageEl`):
  *   - ephemeral bubbles  (`.bubble` / `.bubble-*`)  fade on a timer
@@ -24,7 +24,10 @@ const CHAT_BUBBLE_SVG = `<svg width="28" height="28" viewBox="0 0 24 24" fill="n
 const EMPTY_STATE_HTML = `<div id="chat-empty" class="chat-empty">${CHAT_BUBBLE_SVG}<span class="chat-empty-title">No messages yet</span><span class="chat-empty-hint">Tap the orb and start talking</span></div>`;
 
 export class ChatView {
-  constructor() {
+  /**
+   * @param {{ onUserAudioPlaybackChange?: (playing: boolean) => void }} [options]
+   */
+  constructor(options = {}) {
     /** @type {HTMLButtonElement} */
     this._chatBtn = $("#chat-btn");
     /** @type {HTMLSpanElement} */
@@ -46,6 +49,13 @@ export class ChatView {
     // ── User transcript state (keyed by item_id) ───────────────────────────
     /** @type {Map<string, HTMLElement>} */
     this._userHistByItem = new Map();
+    /** @type {Map<string, { audio: HTMLAudioElement, url: string }>} */
+    this._userAudioByItem = new Map();
+    /** @type {Set<string>} */
+    this._audioUrls = new Set();
+    /** @type {HTMLAudioElement | null} */
+    this._activeUserAudio = null;
+    this._onUserAudioPlaybackChange = options.onUserAudioPlaybackChange ?? (() => {});
     /** @type {HTMLElement | null} */
     this._activeUserBubble = null;
     this._activeUserItemId = "";
@@ -121,7 +131,7 @@ export class ChatView {
     const el = document.createElement("div");
     el.className = `${container} ${role}`;
     const label = role === "user" ? "You" : "Assistant";
-    el.innerHTML = `<div class="${prefix}-role">${label}</div><div class="${prefix}-body${partial ? " partial" : ""}">${escHtml(text)}</div>`;
+    el.innerHTML = `<div class="${prefix}-role">${label}</div><div class="${prefix}-body${partial ? " partial" : ""}"${text ? "" : " hidden"}>${escHtml(text)}</div>`;
     return el;
   }
 
@@ -216,6 +226,11 @@ export class ChatView {
 
   /** Reset the panel to the empty state and clear the unread badge. */
   clear() {
+    this._stopUserAudioPlayback();
+    for (const url of this._audioUrls) URL.revokeObjectURL(url);
+    this._audioUrls.clear();
+    this._userAudioByItem.clear();
+    this._userHistByItem.clear();
     this.renderEmptyState();
     this._chatBadge.classList.remove("visible");
   }
@@ -236,10 +251,28 @@ export class ChatView {
     const body = /** @type {HTMLElement | null} */ (el.querySelector(".hist-body"));
     if (!body) return;
     body.textContent = text;
+    body.hidden = !text;
     body.classList.toggle("partial", partial);
     this._scrollToBottom();
   }
 
+  /** @param {string} itemId */
+  _ensureUserHist(itemId) {
+    let hist = this._userHistByItem.get(itemId);
+    if (!hist) {
+      hist = this._appendHistMsg("user", "", false);
+      this._userHistByItem.set(itemId, hist);
+    }
+    return hist;
+  }
+
+  _stopUserAudioPlayback() {
+    const active = this._activeUserAudio;
+    this._activeUserAudio = null;
+    if (active && !active.paused) active.pause();
+    this._onUserAudioPlaybackChange(false);
+  }
+
   /**
    * Append a tool-call row to the conversation. We only add it once the tool
    * has run, so the expandable toggle carries BOTH the call input and its result.
@@ -335,13 +368,8 @@ export class ChatView {
       const id = d.itemId || this._activeUserItemId || `_u${++this._anonSeq}`;
       const text = d.text;
 
-      let hist = this._userHistByItem.get(id);
-      if (!hist) {
-        hist = this._appendHistMsg("user", text, d.partial);
-        this._userHistByItem.set(id, hist);
-      } else {
-        this._updateHistMsg(hist, text, d.partial);
-      }
+      const hist = this._ensureUserHist(id);
+      this._updateHistMsg(hist, text, d.partial);
 
       // One ephemeral bubble per active item. Purely timer-based: the timer is
       // refreshed on every delta, so it stays while the user keeps talking and
@@ -374,6 +402,71 @@ export class ChatView {
     }
   }
 
+  /**
+   * Attach the browser-local recording to its user turn. This also creates an
+   * audio-only row when STT is disabled and no transcript events arrive.
+   * Reopened VAD segments reuse item_id; the client sends a replacement WAV
+   * containing the accumulated utterance, so the row keeps one player.
+   * @param {{ itemId?: string, audio: Blob, durationMs?: number, truncated?: boolean }} detail
+   */
+  onUserAudio(detail) {
+    const id = detail.itemId || `_u${++this._anonSeq}`;
+    const hist = this._ensureUserHist(id);
+    let container = /** @type {HTMLElement | null} */ (hist.querySelector(".hist-audio"));
+    const isNewPlayer = !container;
+    if (!container) {
+      container = document.createElement("div");
+      container.className = "hist-audio";
+      container.innerHTML = `
+        <div class="hist-audio-label">Audio sent to the model</div>
+        <audio controls preload="metadata" aria-label="Replay your audio"></audio>
+      `;
+      hist.appendChild(container);
+    }
+
+    const audio = /** @type {HTMLAudioElement} */ (container.querySelector("audio"));
+    const previous = this._userAudioByItem.get(id);
+    if (previous) {
+      if (this._activeUserAudio === previous.audio) this._stopUserAudioPlayback();
+      URL.revokeObjectURL(previous.url);
+      this._audioUrls.delete(previous.url);
+    }
+
+    const url = URL.createObjectURL(detail.audio);
+    this._audioUrls.add(url);
+    this._userAudioByItem.set(id, { audio, url });
+    audio.src = url;
+    audio.title = detail.truncated
+      ? "Replay your audio (the beginning was no longer buffered)"
+      : "Replay the audio sent to the model";
+    const label = container.querySelector(".hist-audio-label");
+    if (label) {
+      label.textContent = detail.truncated
+        ? "Audio sent to the model · beginning unavailable"
+        : "Audio sent to the model";
+    }
+
+    if (isNewPlayer) {
+      audio.addEventListener("play", () => {
+        if (this._activeUserAudio && this._activeUserAudio !== audio) {
+          this._activeUserAudio.pause();
+        }
+        this._activeUserAudio = audio;
+        this._onUserAudioPlaybackChange(true);
+      });
+      const stopped = () => {
+        if (this._activeUserAudio !== audio) return;
+        this._activeUserAudio = null;
+        this._onUserAudioPlaybackChange(false);
+      };
+      audio.addEventListener("pause", stopped);
+      audio.addEventListener("ended", stopped);
+      audio.addEventListener("error", stopped);
+    }
+    this._scrollToBottom();
+    this._markUnread();
+  }
+
   /**
    * A response closed (completed or cancelled).
    * @param {{ responseId: string; status: string; audible?: boolean; transcript?: string }} detail

ws/s2s-ws-client.js

@@ -93,6 +93,7 @@ import {
   trimTrailingSlash,
 } from "./codec.js";
 import { OrbVisualiser, VIS_FFT_SIZE } from "./orb-visualizer.js";
+import { SentAudioRecorder } from "./user-audio-recorder.js";
 
 /** Build an Error carrying a `code` (and optional extra fields) so callers can
  *  branch on the failure kind: "limit" | "queue-full" | "queue-expired" | "aborted".
@@ -204,6 +205,9 @@ export class S2sWsRealtimeClient extends EventTarget {
     this._sessionConfigured = false;
     this._startupGreeting = options.startupGreeting?.trim() ?? "";
     this._startupGreetingSent = false;
+    // Bounded, browser-local copy of the exact PCM frames sent over this
+    // socket. Backend VAD timestamps turn it into replayable user utterances.
+    this._userAudioRecorder = new SentAudioRecorder();
     this._debug = (() => { try { return localStorage.getItem("s2s.debug") === "1"; } catch { return false; } })();
   }
 
@@ -619,6 +623,9 @@ export class S2sWsRealtimeClient extends EventTarget {
     if (this._muted) return;
     const b64 = base64FromArrayBuffer(pcm16Buffer);
     this._send({ type: "input_audio_buffer.append", audio: b64 });
+    // Record only after the append is accepted for sending. This intentionally
+    // excludes pre-configuration and muted audio, just like the backend input.
+    this._userAudioRecorder.append(pcm16Buffer);
   }
 
   /**
@@ -683,10 +690,23 @@ export class S2sWsRealtimeClient extends EventTarget {
         // otherwise keep playing over the user's barge-in.
         this._playbackNode?.port.postMessage({ kind: "clear" });
         this._aiSpeaking = false;
+        this._userAudioRecorder.speechStarted({
+          itemId: typeof event.item_id === "string" ? event.item_id : "",
+          audioStartMs: Number(event.audio_start_ms),
+        });
         this._setStatus("user-speaking");
         break;
 
       case "input_audio_buffer.speech_stopped":
+        {
+          const recording = this._userAudioRecorder.speechStopped({
+            itemId: typeof event.item_id === "string" ? event.item_id : "",
+            audioEndMs: Number(event.audio_end_ms),
+          });
+          if (recording) {
+            this.dispatchEvent(new CustomEvent("user-audio", { detail: recording }));
+          }
+        }
         if (this._status === "user-speaking") this._setStatus("processing");
         break;
 
@@ -1098,6 +1118,7 @@ export class S2sWsRealtimeClient extends EventTarget {
     // Abort a queue wait in progress: flag it and wake the poll sleep so
     // `_pollQueue` throws "aborted" and connect() unwinds cleanly.
     this._closed = true;
+    this._userAudioRecorder.reset();
     if (this._queueWake) {
       clearTimeout(this._queueTimer);
       const wake = this._queueWake;

ws/user-audio-recorder.js

@@ -0,0 +1,198 @@
+// @ts-check
+/**
+ * Keep a bounded copy of the PCM16 frames sent over the realtime WebSocket and
+ * turn the backend's VAD boundaries into browser-playable WAV blobs.
+ *
+ * This records the post-resampling, post-noise-gate signal—not a second raw-mic
+ * capture—so replay is as close as the browser can get to the audio delivered
+ * to the backend. Nothing leaves the page beyond the existing realtime stream.
+ */
+
+export const USER_AUDIO_SAMPLE_RATE = 16000;
+const BYTES_PER_SAMPLE = 2;
+const DEFAULT_PREROLL_MS = 5000;
+const DEFAULT_MAX_BUFFER_MS = 120000;
+
+/** @param {DataView} view @param {number} offset @param {string} value */
+function _writeAscii(view, offset, value) {
+  for (let i = 0; i < value.length; i++) {
+    view.setUint8(offset + i, value.charCodeAt(i));
+  }
+}
+
+/**
+ * Wrap little-endian mono PCM16 in a standard WAV container.
+ * @param {Uint8Array} pcm
+ * @param {number} [sampleRate]
+ * @returns {Blob}
+ */
+export function pcm16ToWavBlob(pcm, sampleRate = USER_AUDIO_SAMPLE_RATE) {
+  const dataLength = pcm.byteLength - (pcm.byteLength % BYTES_PER_SAMPLE);
+  const wav = new ArrayBuffer(44 + dataLength);
+  const view = new DataView(wav);
+  _writeAscii(view, 0, "RIFF");
+  view.setUint32(4, 36 + dataLength, true);
+  _writeAscii(view, 8, "WAVE");
+  _writeAscii(view, 12, "fmt ");
+  view.setUint32(16, 16, true); // PCM fmt chunk length
+  view.setUint16(20, 1, true); // linear PCM
+  view.setUint16(22, 1, true); // mono
+  view.setUint32(24, sampleRate, true);
+  view.setUint32(28, sampleRate * BYTES_PER_SAMPLE, true);
+  view.setUint16(32, BYTES_PER_SAMPLE, true);
+  view.setUint16(34, 16, true);
+  _writeAscii(view, 36, "data");
+  view.setUint32(40, dataLength, true);
+  new Uint8Array(wav, 44).set(pcm.subarray(0, dataLength));
+  return new Blob([wav], { type: "audio/wav" });
+}
+
+/** @param {Uint8Array} first @param {Uint8Array} second */
+function _concat(first, second) {
+  const result = new Uint8Array(first.byteLength + second.byteLength);
+  result.set(first, 0);
+  result.set(second, first.byteLength);
+  return result;
+}
+
+export class SentAudioRecorder {
+  /**
+   * @param {{ sampleRate?: number, preRollMs?: number, maxBufferMs?: number }} [options]
+   */
+  constructor(options = {}) {
+    this.sampleRate = options.sampleRate ?? USER_AUDIO_SAMPLE_RATE;
+    this._preRollSamples = Math.round(
+      (this.sampleRate * (options.preRollMs ?? DEFAULT_PREROLL_MS)) / 1000,
+    );
+    this._maxBufferSamples = Math.round(
+      (this.sampleRate * (options.maxBufferMs ?? DEFAULT_MAX_BUFFER_MS)) / 1000,
+    );
+    /** @type {{ startSample: number, endSample: number, bytes: Uint8Array }[]} */
+    this._chunks = [];
+    this._sentSamples = 0;
+    /** @type {{ itemId: string, requestedStartSample: number } | null} */
+    this._active = null;
+    this._lastItemId = "";
+    this._lastItemPcm = new Uint8Array(0);
+  }
+
+  /** Store one PCM16 frame that was actually sent to the backend.
+   * @param {ArrayBuffer} buffer */
+  append(buffer) {
+    const evenLength = buffer.byteLength - (buffer.byteLength % BYTES_PER_SAMPLE);
+    if (evenLength <= 0) return;
+    const bytes = new Uint8Array(buffer.slice(0, evenLength));
+    const startSample = this._sentSamples;
+    const endSample = startSample + evenLength / BYTES_PER_SAMPLE;
+    this._chunks.push({ startSample, endSample, bytes });
+    this._sentSamples = endSample;
+    this._prune();
+  }
+
+  /**
+   * Remember where the backend says this speech item began. The event normally
+   * arrives after confirmation, so the bounded pre-roll retains its onset.
+   * @param {{ itemId?: string, audioStartMs?: number }} boundary
+   */
+  speechStarted(boundary) {
+    const itemId = boundary.itemId || `audio_${this._sentSamples}`;
+    const requestedStartSample = this._sampleAtMs(boundary.audioStartMs, this._sentSamples);
+    this._active = { itemId, requestedStartSample };
+    if (itemId !== this._lastItemId) {
+      this._lastItemId = itemId;
+      this._lastItemPcm = new Uint8Array(0);
+    }
+    this._prune();
+  }
+
+  /**
+   * Finalize the active VAD segment. Reopened segments carrying the same
+   * item_id replace the prior recording with their concatenation, matching the
+   * chat view's one-row-per-item behavior.
+   * @param {{ itemId?: string, audioEndMs?: number }} boundary
+   * @returns {{ itemId: string, audio: Blob, durationMs: number, truncated: boolean } | null}
+   */
+  speechStopped(boundary) {
+    const active = this._active;
+    if (!active) return null;
+    this._active = null;
+
+    const itemId = boundary.itemId || active.itemId;
+    const availableStart = this._chunks[0]?.startSample ?? this._sentSamples;
+    const startSample = Math.max(active.requestedStartSample, availableStart);
+    let endSample = this._sampleAtMs(boundary.audioEndMs, this._sentSamples);
+    if (endSample <= startSample) endSample = this._sentSamples;
+    endSample = Math.min(endSample, this._sentSamples);
+
+    const segment = this._slice(startSample, endSample);
+    if (segment.byteLength === 0) {
+      this._prune();
+      return null;
+    }
+
+    if (itemId !== this._lastItemId) {
+      this._lastItemId = itemId;
+      this._lastItemPcm = new Uint8Array(0);
+    }
+    this._lastItemPcm = _concat(this._lastItemPcm, segment);
+    const durationMs =
+      (this._lastItemPcm.byteLength / BYTES_PER_SAMPLE / this.sampleRate) * 1000;
+    const result = {
+      itemId,
+      audio: pcm16ToWavBlob(this._lastItemPcm, this.sampleRate),
+      durationMs,
+      truncated: active.requestedStartSample < availableStart,
+    };
+    this._prune();
+    return result;
+  }
+
+  reset() {
+    this._chunks = [];
+    this._sentSamples = 0;
+    this._active = null;
+    this._lastItemId = "";
+    this._lastItemPcm = new Uint8Array(0);
+  }
+
+  /** @param {number | undefined} ms @param {number} fallback */
+  _sampleAtMs(ms, fallback) {
+    if (!Number.isFinite(ms) || Number(ms) < 0) return fallback;
+    return Math.max(0, Math.min(Math.round((Number(ms) * this.sampleRate) / 1000), this._sentSamples));
+  }
+
+  /** @param {number} startSample @param {number} endSample */
+  _slice(startSample, endSample) {
+    /** @type {Uint8Array[]} */
+    const parts = [];
+    let length = 0;
+    for (const chunk of this._chunks) {
+      const overlapStart = Math.max(startSample, chunk.startSample);
+      const overlapEnd = Math.min(endSample, chunk.endSample);
+      if (overlapEnd <= overlapStart) continue;
+      const from = (overlapStart - chunk.startSample) * BYTES_PER_SAMPLE;
+      const to = (overlapEnd - chunk.startSample) * BYTES_PER_SAMPLE;
+      const part = chunk.bytes.slice(from, to);
+      parts.push(part);
+      length += part.byteLength;
+    }
+    const result = new Uint8Array(length);
+    let offset = 0;
+    for (const part of parts) {
+      result.set(part, offset);
+      offset += part.byteLength;
+    }
+    return result;
+  }
+
+  _prune() {
+    const hardFloor = Math.max(0, this._sentSamples - this._maxBufferSamples);
+    const softFloor = this._active
+      ? this._active.requestedStartSample
+      : Math.max(0, this._sentSamples - this._preRollSamples);
+    const floor = Math.max(hardFloor, softFloor);
+    while (this._chunks.length && this._chunks[0].endSample <= floor) {
+      this._chunks.shift();
+    }
+  }
+}