ws/user-audio-recorder.js

// @ts-check
/**
 * Keep a bounded copy of the PCM16 frames sent over the realtime WebSocket and
 * turn the backend's VAD boundaries into browser-playable WAV blobs.
 *
 * This records the post-resampling, post-noise-gate signal—not a second raw-mic
 * capture—so replay is as close as the browser can get to the audio delivered
 * to the backend. Nothing leaves the page beyond the existing realtime stream.
 */

export const USER_AUDIO_SAMPLE_RATE = 16000;
const BYTES_PER_SAMPLE = 2;
const DEFAULT_PREROLL_MS = 5000;
const DEFAULT_MAX_BUFFER_MS = 120000;

/** @param {DataView} view @param {number} offset @param {string} value */
function _writeAscii(view, offset, value) {
  for (let i = 0; i < value.length; i++) {
    view.setUint8(offset + i, value.charCodeAt(i));
  }
}

/**
 * Wrap little-endian mono PCM16 in a standard WAV container.
 * @param {Uint8Array} pcm
 * @param {number} [sampleRate]
 * @returns {Blob}
 */
export function pcm16ToWavBlob(pcm, sampleRate = USER_AUDIO_SAMPLE_RATE) {
  const dataLength = pcm.byteLength - (pcm.byteLength % BYTES_PER_SAMPLE);
  const wav = new ArrayBuffer(44 + dataLength);
  const view = new DataView(wav);
  _writeAscii(view, 0, "RIFF");
  view.setUint32(4, 36 + dataLength, true);
  _writeAscii(view, 8, "WAVE");
  _writeAscii(view, 12, "fmt ");
  view.setUint32(16, 16, true); // PCM fmt chunk length
  view.setUint16(20, 1, true); // linear PCM
  view.setUint16(22, 1, true); // mono
  view.setUint32(24, sampleRate, true);
  view.setUint32(28, sampleRate * BYTES_PER_SAMPLE, true);
  view.setUint16(32, BYTES_PER_SAMPLE, true);
  view.setUint16(34, 16, true);
  _writeAscii(view, 36, "data");
  view.setUint32(40, dataLength, true);
  new Uint8Array(wav, 44).set(pcm.subarray(0, dataLength));
  return new Blob([wav], { type: "audio/wav" });
}

/** @param {Uint8Array} first @param {Uint8Array} second */
function _concat(first, second) {
  const result = new Uint8Array(first.byteLength + second.byteLength);
  result.set(first, 0);
  result.set(second, first.byteLength);
  return result;
}

export class SentAudioRecorder {
  /**
   * @param {{ sampleRate?: number, preRollMs?: number, maxBufferMs?: number }} [options]
   */
  constructor(options = {}) {
    this.sampleRate = options.sampleRate ?? USER_AUDIO_SAMPLE_RATE;
    this._preRollSamples = Math.round(
      (this.sampleRate * (options.preRollMs ?? DEFAULT_PREROLL_MS)) / 1000,
    );
    this._maxBufferSamples = Math.round(
      (this.sampleRate * (options.maxBufferMs ?? DEFAULT_MAX_BUFFER_MS)) / 1000,
    );
    /** @type {{ startSample: number, endSample: number, bytes: Uint8Array }[]} */
    this._chunks = [];
    this._sentSamples = 0;
    /** @type {{ itemId: string, requestedStartSample: number } | null} */
    this._active = null;
    this._lastItemId = "";
    this._lastItemPcm = new Uint8Array(0);
  }

  /** Store one PCM16 frame that was actually sent to the backend.
   * @param {ArrayBuffer} buffer */
  append(buffer) {
    const evenLength = buffer.byteLength - (buffer.byteLength % BYTES_PER_SAMPLE);
    if (evenLength <= 0) return;
    const bytes = new Uint8Array(buffer.slice(0, evenLength));
    const startSample = this._sentSamples;
    const endSample = startSample + evenLength / BYTES_PER_SAMPLE;
    this._chunks.push({ startSample, endSample, bytes });
    this._sentSamples = endSample;
    this._prune();
  }

  /**
   * Remember where the backend says this speech item began. The event normally
   * arrives after confirmation, so the bounded pre-roll retains its onset.
   * @param {{ itemId?: string, audioStartMs?: number }} boundary
   */
  speechStarted(boundary) {
    const itemId = boundary.itemId || `audio_${this._sentSamples}`;
    const requestedStartSample = this._sampleAtMs(boundary.audioStartMs, this._sentSamples);
    this._active = { itemId, requestedStartSample };
    if (itemId !== this._lastItemId) {
      this._lastItemId = itemId;
      this._lastItemPcm = new Uint8Array(0);
    }
    this._prune();
  }

  /**
   * Finalize the active VAD segment. Reopened segments carrying the same
   * item_id replace the prior recording with their concatenation, matching the
   * chat view's one-row-per-item behavior.
   * @param {{ itemId?: string, audioEndMs?: number }} boundary
   * @returns {{ itemId: string, audio: Blob, durationMs: number, truncated: boolean } | null}
   */
  speechStopped(boundary) {
    const active = this._active;
    if (!active) return null;
    this._active = null;

    const itemId = boundary.itemId || active.itemId;
    const availableStart = this._chunks[0]?.startSample ?? this._sentSamples;
    const startSample = Math.max(active.requestedStartSample, availableStart);
    let endSample = this._sampleAtMs(boundary.audioEndMs, this._sentSamples);
    if (endSample <= startSample) endSample = this._sentSamples;
    endSample = Math.min(endSample, this._sentSamples);

    const segment = this._slice(startSample, endSample);
    if (segment.byteLength === 0) {
      this._prune();
      return null;
    }

    if (itemId !== this._lastItemId) {
      this._lastItemId = itemId;
      this._lastItemPcm = new Uint8Array(0);
    }
    this._lastItemPcm = _concat(this._lastItemPcm, segment);
    const durationMs =
      (this._lastItemPcm.byteLength / BYTES_PER_SAMPLE / this.sampleRate) * 1000;
    const result = {
      itemId,
      audio: pcm16ToWavBlob(this._lastItemPcm, this.sampleRate),
      durationMs,
      truncated: active.requestedStartSample < availableStart,
    };
    this._prune();
    return result;
  }

  reset() {
    this._chunks = [];
    this._sentSamples = 0;
    this._active = null;
    this._lastItemId = "";
    this._lastItemPcm = new Uint8Array(0);
  }

  /** @param {number | undefined} ms @param {number} fallback */
  _sampleAtMs(ms, fallback) {
    if (!Number.isFinite(ms) || Number(ms) < 0) return fallback;
    return Math.max(0, Math.min(Math.round((Number(ms) * this.sampleRate) / 1000), this._sentSamples));
  }

  /** @param {number} startSample @param {number} endSample */
  _slice(startSample, endSample) {
    /** @type {Uint8Array[]} */
    const parts = [];
    let length = 0;
    for (const chunk of this._chunks) {
      const overlapStart = Math.max(startSample, chunk.startSample);
      const overlapEnd = Math.min(endSample, chunk.endSample);
      if (overlapEnd <= overlapStart) continue;
      const from = (overlapStart - chunk.startSample) * BYTES_PER_SAMPLE;
      const to = (overlapEnd - chunk.startSample) * BYTES_PER_SAMPLE;
      const part = chunk.bytes.slice(from, to);
      parts.push(part);
      length += part.byteLength;
    }
    const result = new Uint8Array(length);
    let offset = 0;
    for (const part of parts) {
      result.set(part, offset);
      offset += part.byteLength;
    }
    return result;
  }

  _prune() {
    const hardFloor = Math.max(0, this._sentSamples - this._maxBufferSamples);
    const softFloor = this._active
      ? this._active.requestedStartSample
      : Math.max(0, this._sentSamples - this._preRollSamples);
    const floor = Math.max(hardFloor, softFloor);
    while (this._chunks.length && this._chunks[0].endSample <= floor) {
      this._chunks.shift();
    }
  }
}