""" web_app.py ────────── FastAPI + WebSocket server for the Aara voice concierge. CPU-only · HuggingFace Spaces Fixes vs v1: • asyncio.Semaphore(3) caps concurrent inference turns • Per-IP connection limit (max 5) + per-session audio rate limit (max 12/min) • Audio payload size cap: 512 KB • Session stored in LRU cache (max 200, 30-min TTL) — no memory leak • SILENCE_STOP_MS fixed to 900ms in all HTML (was 8000ms in minimal UI) • ffmpeg subprocess timeout=30s • request_id threads through all log lines • Hourly cleanup task for output_audio/ • Silero VAD results visible in status messages """ from __future__ import annotations import asyncio import json import logging import os import subprocess import threading import time import uuid from collections import OrderedDict from contextlib import asynccontextmanager from dataclasses import dataclass, field from pathlib import Path from typing import Any, Optional, cast import numpy as np from fastapi import FastAPI, WebSocket, WebSocketDisconnect from fastapi.responses import HTMLResponse, JSONResponse from fastapi.staticfiles import StaticFiles from starlette.websockets import WebSocketState from create_hotel_database import initialize_database from voice_agent_standalone import AaraConfig, AaraVoiceAgent, SessionState # ───────────────────────────────────────────── # Logging # ───────────────────────────────────────────── logging.basicConfig( level=logging.INFO, format="[%(asctime)s] %(levelname)s %(name)s: %(message)s", datefmt="%H:%M:%S", ) logger = logging.getLogger("AARA.Web") # ───────────────────────────────────────────── # Constants # ───────────────────────────────────────────── OUTPUT_AUDIO_DIR = Path("output_audio") OUTPUT_AUDIO_DIR.mkdir(parents=True, exist_ok=True) REFERENCE_AUDIO_DIR = Path("reference_audio") REFERENCE_AUDIO_DIR.mkdir(parents=True, exist_ok=True) MAX_CONCURRENT_TURNS = 3 # asyncio.Semaphore value MAX_CONNECTIONS_PER_IP = 5 # WebSocket connections per IP address MAX_AUDIO_PER_MINUTE = 12 # Audio messages per session per minute MAX_AUDIO_BYTES = 512 * 1024 # 512 KB — ~30s of WebM/Opus at typical bitrates AUDIO_CLEANUP_INTERVAL = 3600 # Seconds between audio file cleanup runs AUDIO_MAX_AGE = 3600 # Delete output_audio files older than 1 hour # ───────────────────────────────────────────── # Voice options for UI (English only) # ───────────────────────────────────────────── VOICE_OPTIONS_HTML = ( '' '' ) def _audio_url_from_path(path: Optional[str]) -> Optional[str]: if not path: return None try: file_path = Path(path) if file_path.exists(): return f"/audio/{file_path.name}" except Exception: return None return None def _reference_url_from_path(path: Optional[str]) -> Optional[str]: if not path: return None try: file_path = Path(path) if file_path.exists() and file_path.parent.resolve() == REFERENCE_AUDIO_DIR.resolve(): return f"/reference-audio/{file_path.name}" except Exception: return None return None def _websocket_is_open(websocket: WebSocket) -> bool: return ( getattr(websocket, "client_state", None) != WebSocketState.DISCONNECTED and getattr(websocket, "application_state", None) != WebSocketState.DISCONNECTED ) async def _safe_send_json(websocket: WebSocket, payload: dict[str, Any]) -> None: if not _websocket_is_open(websocket): raise WebSocketDisconnect(code=1001) try: await websocket.send_json(payload) except WebSocketDisconnect: raise except RuntimeError as exc: if "websocket.send" in str(exc).lower(): raise WebSocketDisconnect(code=1001) from exc raise async def _safe_close(websocket: WebSocket, code: int = 1000) -> None: if not _websocket_is_open(websocket): return try: await websocket.close(code=code) except RuntimeError: return # ═══════════════════════════════════════════════════════════════════ # Session LRU cache (replaces unbounded dict — no memory leak) # ═══════════════════════════════════════════════════════════════════ @dataclass class BrowserSessionState: session: SessionState = field(default_factory=SessionState) greeted: bool = False class _SessionLRU: """Thread-safe LRU cache for BrowserSessionState with TTL eviction.""" def __init__(self, maxsize: int = 200, ttl_seconds: int = 1800) -> None: self._cache: OrderedDict[str, tuple[float, BrowserSessionState]] = OrderedDict() self._maxsize = maxsize self._ttl = ttl_seconds self._lock = threading.Lock() def get(self, key: str) -> Optional[BrowserSessionState]: with self._lock: entry = self._cache.get(key) if entry is None: return None ts, value = entry if time.time() - ts > self._ttl: del self._cache[key] return None self._cache.move_to_end(key) self._cache[key] = (time.time(), value) return value def set(self, key: str, value: BrowserSessionState) -> None: with self._lock: if key in self._cache: self._cache.move_to_end(key) self._cache[key] = (time.time(), value) while len(self._cache) > self._maxsize: self._cache.popitem(last=False) def touch(self, key: str) -> None: with self._lock: entry = self._cache.get(key) if entry is not None: self._cache.move_to_end(key) self._cache[key] = (time.time(), entry[1]) def __len__(self) -> int: return len(self._cache) # ═══════════════════════════════════════════════════════════════════ # Rate limiting # ═══════════════════════════════════════════════════════════════════ _ip_connections: dict[str, int] = {} _ip_conn_lock = threading.Lock() _session_audio_times: dict[str, list[float]] = {} _audio_times_lock = threading.Lock() def _get_client_ip(websocket: WebSocket) -> str: client = getattr(websocket, "client", None) return client.host if client else "unknown" def _ip_can_connect(ip: str) -> bool: with _ip_conn_lock: return _ip_connections.get(ip, 0) < MAX_CONNECTIONS_PER_IP def _ip_connect(ip: str) -> None: with _ip_conn_lock: _ip_connections[ip] = _ip_connections.get(ip, 0) + 1 def _ip_disconnect(ip: str) -> None: with _ip_conn_lock: _ip_connections[ip] = max(0, _ip_connections.get(ip, 1) - 1) def _session_is_rate_limited(session_id: str) -> bool: """Return True if this session has exceeded MAX_AUDIO_PER_MINUTE audio messages.""" now = time.time() cutoff = now - 60.0 with _audio_times_lock: times = _session_audio_times.get(session_id, []) times = [t for t in times if t > cutoff] _session_audio_times[session_id] = times if len(times) >= MAX_AUDIO_PER_MINUTE: return True times.append(now) _session_audio_times[session_id] = times return False # ═══════════════════════════════════════════════════════════════════ # Global state # ═══════════════════════════════════════════════════════════════════ _agent: Optional[AaraVoiceAgent] = None _agent_lock = threading.Lock() _browser_sessions = _SessionLRU(maxsize=200, ttl_seconds=1800) _turn_semaphore: Optional[asyncio.Semaphore] = None # created in lifespan def get_agent() -> AaraVoiceAgent: global _agent if _agent is None: with _agent_lock: if _agent is None: logger.info("Starting shared Sahara Star agent ...") cfg = AaraConfig() cfg.unload_models_between_turns = False cfg.enable_multi_pass_asr = False cfg.enable_server_tts = True # Prefer consistent server TTS, fallback stays in browser # NOTE: asr_confidence_threshold, llm_max_tokens etc. inherit from # AaraConfig defaults — do NOT override them here. cfg.enable_silero_vad = True # Use package API; no torch.hub prompt needed if not Path(cfg.db_path).exists(): initialize_database(cfg.db_path, verbose=False) _agent = AaraVoiceAgent(cfg) return _agent # ═══════════════════════════════════════════════════════════════════ # Audio cleanup task # ═══════════════════════════════════════════════════════════════════ async def _cleanup_audio_loop() -> None: """Periodically delete old output_audio files to prevent disk fill.""" while True: await asyncio.sleep(AUDIO_CLEANUP_INTERVAL) cutoff = time.time() - AUDIO_MAX_AGE deleted = 0 for f in OUTPUT_AUDIO_DIR.glob("*"): try: if f.is_file() and f.stat().st_mtime < cutoff: f.unlink() deleted += 1 except Exception: pass if deleted: logger.info("Audio cleanup: deleted %d old files", deleted) # Also evict old response cache entries try: agent = get_agent() evicted = agent.llm.cache.evict_old_audio(AUDIO_MAX_AGE) if evicted: logger.info("Cache eviction: removed %d old audio entries", evicted) except Exception: pass # ═══════════════════════════════════════════════════════════════════ # Session helpers # ═══════════════════════════════════════════════════════════════════ def _normalise_language(language: Optional[str]) -> str: """Always return English - multilingual support removed.""" return "en" def _restore_session(client_id: str, snapshot: Optional[dict[str, Any]]) -> SessionState: session = SessionState() session.session_id = client_id if not snapshot: return session session.language_preference = str(snapshot.get("language") or "en") session.voice_preference = str(snapshot.get("voice_preference") or "female") session.last_intent = str(snapshot.get("last_intent") or "") try: restored: Any = json.loads(snapshot.get("state_json") or "{}") if isinstance(restored, dict): session.task_state.update(cast(dict[str, Any], restored)) except Exception: pass return session def get_browser_session(client_id: str) -> BrowserSessionState: existing = _browser_sessions.get(client_id) if existing is not None: return existing agent = get_agent() db = cast(Any, agent.db) snapshot = db.get_session_snapshot(client_id) session = _restore_session(client_id, snapshot) greeted = bool(snapshot and int(snapshot.get("turn_count") or 0) > 0) browser_session = BrowserSessionState(session=session, greeted=greeted) _browser_sessions.set(client_id, browser_session) return browser_session def update_preferences(browser_session: BrowserSessionState, language: Optional[str], voice_preference: Optional[str]) -> None: agent = get_agent() session = browser_session.session session.language_preference = "en" if voice_preference: agent.set_voice_preference_for_session(voice_preference, session) # ═══════════════════════════════════════════════════════════════════ # Audio decode # ═══════════════════════════════════════════════════════════════════ def _decode_browser_audio(waveform_bytes: bytes) -> np.ndarray: """ Decode WebM/Opus bytes to float32 PCM at 16kHz using ffmpeg. FIXED: added timeout=30 to prevent hung subprocesses. """ ffmpeg_cmd = [ "ffmpeg", "-y", "-i", "pipe:0", "-f", "f32le", "-acodec", "pcm_f32le", "-ar", "16000", "-ac", "1", "pipe:1", ] try: completed = subprocess.run( ffmpeg_cmd, input=waveform_bytes, check=True, capture_output=True, timeout=30, # FIXED: was no timeout — hung on malformed audio ) except FileNotFoundError as exc: raise RuntimeError("ffmpeg is required to decode browser audio") from exc except subprocess.TimeoutExpired as exc: raise RuntimeError("ffmpeg timed out decoding audio (malformed or too large)") from exc except subprocess.CalledProcessError as exc: stderr = exc.stderr.decode(errors="ignore") if exc.stderr else str(exc) raise RuntimeError(f"ffmpeg failed: {stderr}") from exc audio = np.frombuffer(completed.stdout, dtype=np.float32).copy() if audio.size == 0: raise RuntimeError("ffmpeg produced no audio samples") return audio # ═══════════════════════════════════════════════════════════════════ # Turn processing # ═══════════════════════════════════════════════════════════════════ def process_browser_audio( browser_session: BrowserSessionState, audio_bytes: bytes, request_id: str = "", ) -> dict[str, Any]: agent = get_agent() logger.info( "[%s] Audio recv: session=%s lang=%s voice=%s bytes=%d", request_id, browser_session.session.session_id, browser_session.session.language_preference, browser_session.session.voice_preference, len(audio_bytes), ) try: audio = _decode_browser_audio(audio_bytes) except RuntimeError as exc: logger.warning("[%s] Audio decode failed: %s", request_id, exc) return {"error": str(exc)} language_hint = browser_session.session.language_preference or None result = agent.process_turn( audio=audio, language_hint=language_hint, play_audio=False, session=browser_session.session, voice_preference=browser_session.session.voice_preference, unload_models_between_turns=agent.cfg.unload_models_between_turns, request_id=request_id, ) if result.get("error"): return {"error": result["error"], **result} return { "type": "result", "transcription": "", "response": "", "spoken_text": result.get("response", ""), "language": result.get("language", browser_session.session.language_preference), "rtt_seconds": result.get("rtt_seconds", 0.0), "audio_url": _audio_url_from_path(cast(Optional[str], result.get("audio_path"))), "voice_profile": result.get("voice_profile", browser_session.session.voice_preference), "voice_reference_url": _reference_url_from_path(cast(Optional[str], result.get("voice_reference_path"))), } # ═══════════════════════════════════════════════════════════════════ # WebSocket message handlers # ═══════════════════════════════════════════════════════════════════ async def send_greeting(websocket: WebSocket, browser_session: BrowserSessionState) -> None: agent = get_agent() session = browser_session.session greeting_text = agent.llm.generate_greeting( language=session.language_preference, voice_preference=session.voice_preference, session=session, ) voice_profile, audio_path = await asyncio.to_thread( agent.build_voice_output, greeting_text, preference=session.voice_preference, session=session, request_id=session.session_id[-8:], ) session.task_state["opening_greeting_sent"] = True logger.info("Greeting: session=%s text=%r", session.session_id, greeting_text[:80]) await _safe_send_json(websocket, { "type": "greeting", "message": greeting_text, "spoken_text": greeting_text, "language": session.language_preference, "audio_url": _audio_url_from_path(audio_path), "voice_profile": voice_profile.key, "voice_reference_url": _reference_url_from_path(voice_profile.reference_audio_path), }) async def send_ready(websocket: WebSocket, client_id: str, browser_session: BrowserSessionState) -> None: await _safe_send_json(websocket, { "type": "ready", "message": "Connected to Sahara Star concierge.", "client_id": client_id, "session_id": browser_session.session.session_id, }) async def handle_settings(websocket: WebSocket, browser_session: BrowserSessionState, data: dict[str, Any]) -> None: update_preferences(browser_session, data.get("language"), data.get("voice_preference")) await _safe_send_json(websocket, {"type": "status", "message": "Settings updated.", "level": "ok"}) async def handle_reset(websocket: WebSocket, browser_session: BrowserSessionState) -> None: browser_session.session.clear() browser_session.greeted = False await _safe_send_json(websocket, {"type": "status", "message": "Conversation reset.", "level": "ok"}) async def handle_text_message( websocket: WebSocket, browser_session: BrowserSessionState, text_payload: str ) -> None: try: data = json.loads(text_payload) except json.JSONDecodeError: return message_type = data.get("type") if message_type == "start": # Client clicked the Start button — send greeting once, then client begins listening loop if not browser_session.greeted: await send_greeting(websocket, browser_session) browser_session.greeted = True else: await _safe_send_json(websocket, {"type": "status", "message": "Ready to listen.", "level": "ok"}) elif message_type == "settings": await handle_settings(websocket, browser_session, data) elif message_type == "reset": await handle_reset(websocket, browser_session) async def handle_audio_message( websocket: WebSocket, browser_session: BrowserSessionState, audio_bytes: bytes, request_id: str = "", ) -> None: assert _turn_semaphore is not None # Warn immediately if semaphore is full so the user isn't silently waiting if _turn_semaphore.locked() if hasattr(_turn_semaphore, "locked") else False: await _safe_send_json(websocket, { "type": "status", "message": "One moment, I'm helping another guest...", "level": "warn", }) async with _turn_semaphore: await _safe_send_json(websocket, { "type": "status", "message": "Processing your request...", "level": "busy" }) try: result = await asyncio.to_thread( process_browser_audio, browser_session, bytes(audio_bytes), request_id ) except Exception as exc: logger.error("[%s] Audio processing error: %s", request_id, exc, exc_info=True) await _safe_send_json(websocket, { "type": "error", "message": "I couldn't process that audio. Please try again." }) return if result.get("error"): await _safe_send_json(websocket, {"type": "error", "message": result["error"]}) return await _safe_send_json(websocket, result) async def bootstrap_session(websocket: WebSocket) -> tuple[str, BrowserSessionState]: try: hello_raw = await asyncio.wait_for(websocket.receive_text(), timeout=5.0) hello = json.loads(hello_raw) except asyncio.TimeoutError: hello = {"type": "hello", "client_id": str(uuid.uuid4()), "language": "en", "voice_preference": "female"} except Exception: hello = {"type": "hello", "client_id": str(uuid.uuid4()), "language": "en", "voice_preference": "female"} if hello.get("type") != "hello": await _safe_send_json(websocket, {"type": "error", "message": "Expected hello handshake."}) await _safe_close(websocket, code=1002) raise WebSocketDisconnect(code=1002) client_id = str(hello.get("client_id") or uuid.uuid4()) browser_session = get_browser_session(client_id) update_preferences(browser_session, hello.get("language"), hello.get("voice_preference")) await send_ready(websocket, client_id, browser_session) return client_id, browser_session # ═══════════════════════════════════════════════════════════════════ # FastAPI lifespan # ═══════════════════════════════════════════════════════════════════ @asynccontextmanager async def lifespan(_: FastAPI): global _turn_semaphore _turn_semaphore = asyncio.Semaphore(MAX_CONCURRENT_TURNS) # Warm up models in background thread agent = await asyncio.to_thread(get_agent) warmup_jobs: list[tuple[str, Any]] = [ ("ASR", asyncio.to_thread(agent.asr.warmup)), ("LLM", asyncio.to_thread(agent.llm.warmup)), ] if agent.tts is not None: warmup_jobs.append(("TTS", asyncio.to_thread(agent.tts.warmup))) warmup_results = await asyncio.gather(*(job for _, job in warmup_jobs), return_exceptions=True) for (label, _), outcome in zip(warmup_jobs, warmup_results): if isinstance(outcome, Exception): logger.warning("%s warmup issue (will retry on first request): %s", label, outcome) else: logger.info("%s warmup done", label) # Start background cleanup task cleanup_task = asyncio.create_task(_cleanup_audio_loop()) yield cleanup_task.cancel() try: await cleanup_task except asyncio.CancelledError: pass # ═══════════════════════════════════════════════════════════════════ # App # ═══════════════════════════════════════════════════════════════════ app = FastAPI(title="Sahara Star Voice Concierge", lifespan=lifespan) app.mount("/audio", StaticFiles(directory=str(OUTPUT_AUDIO_DIR)), name="audio") app.mount("/reference-audio", StaticFiles(directory=str(REFERENCE_AUDIO_DIR)), name="reference-audio") @app.get("/api/health") async def api_health() -> JSONResponse: agent = get_agent() ok, checks = agent.health_check() checks["active_sessions"] = len(_browser_sessions) checks["semaphore_available"] = _turn_semaphore._value if _turn_semaphore else 0 return JSONResponse({"ok": bool(ok), "checks": {str(k): bool(v) if isinstance(v, bool) else v for k, v in checks.items()}}) # ═══════════════════════════════════════════════════════════════════ # Routes # ═══════════════════════════════════════════════════════════════════ @app.get("/", response_class=HTMLResponse) async def index() -> HTMLResponse: """Single-button voice concierge UI. Each page refresh = fresh session.""" return HTMLResponse(""" Sahara Star · Voice Concierge
Sahara Star Hotels · Mumbai

Aara
Concierge

Your voice-first hotel assistant.
Speak naturally — I understand you.

Connecting...
""") # ═══════════════════════════════════════════════════════════════════ # WebSocket endpoint # ═══════════════════════════════════════════════════════════════════ @app.websocket("/ws") async def websocket_endpoint(websocket: WebSocket) -> None: await websocket.accept() client_ip = _get_client_ip(websocket) client_id: Optional[str] = None browser_session: Optional[BrowserSessionState] = None # Per-IP connection limit if not _ip_can_connect(client_ip): logger.warning("Connection rejected (IP limit): %s", client_ip) await _safe_send_json(websocket, {"type": "error", "message": "Too many connections from your address. Please try again shortly."}) await _safe_close(websocket, code=1013) return _ip_connect(client_ip) try: logger.info("WS accepted from %s", client_ip) client_id, browser_session = await bootstrap_session(websocket) # Greeting is sent lazily when client sends {type:"start"} (button click) # so a page refresh always starts fresh without an auto-spoken greeting. while True: message = await websocket.receive() if message.get("type") == "websocket.disconnect": break text_payload: Optional[str] = message.get("text") bytes_payload: Optional[bytes] = message.get("bytes") if text_payload and browser_session: await handle_text_message(websocket, browser_session, text_payload) continue if bytes_payload and browser_session: # Audio payload size guard if len(bytes_payload) > MAX_AUDIO_BYTES: logger.warning("[%s] Audio rejected: %d bytes > %d limit", client_id, len(bytes_payload), MAX_AUDIO_BYTES) await _safe_send_json(websocket, {"type": "error", "message": f"Audio too large ({len(bytes_payload)//1024}KB). Please keep requests under 30 seconds."}) continue # Per-session rate limit if _session_is_rate_limited(browser_session.session.session_id): logger.warning("[%s] Rate limited", client_id) await _safe_send_json(websocket, {"type": "error", "message": "Too many requests. Please wait a moment before speaking again."}) continue request_id = uuid.uuid4().hex[:8] await handle_audio_message(websocket, browser_session, bytes(bytes_payload), request_id=request_id) _browser_sessions.touch(client_id) # Refresh TTL on activity except WebSocketDisconnect: logger.info("WS disconnected: %s (%s)", client_id, client_ip) except Exception as exc: logger.error("WS failure: %s", exc, exc_info=True) try: await _safe_send_json(websocket, {"type": "error", "message": "An internal error occurred."}) except Exception: pass finally: _ip_disconnect(client_ip) if client_id and browser_session is not None: _browser_sessions.set(client_id, browser_session) if __name__ == "__main__": import uvicorn port = int(os.environ.get("PORT", "7860")) host = "0.0.0.0" if os.environ.get("SPACE_ID") else "127.0.0.1" uvicorn.run("web_app:app", host=host, port=port, reload=False)