fix: fail fast on slow backend requests

README.md

@@ -62,5 +62,9 @@ Recommended environment variables:
 - `REQUEST_LOG_DIR`: optional persistent request log directory. Defaults to `/data/logs` when `/data` exists
 - `GRADIO_QUEUE_MAX_SIZE`: defaults to `30`
 - `GRADIO_DEFAULT_CONCURRENCY_LIMIT`: defaults to `6` (uses async server pool bridge for thread-safe concurrency)
+- `NANOVLLM_API_CONNECT_TIMEOUT`: defaults to `5`
+- `NANOVLLM_API_ASR_TIMEOUT`: defaults to `30`
+- `NANOVLLM_API_DENOISE_TIMEOUT`: defaults to `60`
+- `NANOVLLM_API_GENERATE_TIMEOUT`: defaults to `120`
 - `DENOISE_MAX_CONCURRENT`: defaults to `1` (limits concurrent ZipEnhancer denoise requests to avoid GPU OOM)
 - `GRADIO_SSR_MODE`: defaults to `false`

app.py

@@ -5,6 +5,7 @@ import os
 import re
 import sys
 import tempfile
+import time
 from datetime import datetime, timezone
 from pathlib import Path
 from threading import Lock
@@ -62,6 +63,21 @@ def _get_bool_env(name: str, default: bool) -> bool:
     raise ValueError(f"Invalid boolean env: {name}={value!r}")
 
 
+def _api_timeout(read_timeout_env: str, default_read_timeout: float) -> tuple[float, float]:
+    return (
+        _get_float_env("NANOVLLM_API_CONNECT_TIMEOUT", 5.0),
+        _get_float_env(read_timeout_env, default_read_timeout),
+    )
+
+
+class BackendBusyError(RuntimeError):
+    pass
+
+
+class BackendTimeoutError(TimeoutError):
+    pass
+
+
 # ---------- Request Logging ----------
 
 
@@ -95,16 +111,18 @@ def _append_request_log(payload: dict) -> None:
         fp.write(json.dumps(record, ensure_ascii=False) + "\n")
 
 
-def _begin_generation_request() -> None:
+def _begin_generation_request() -> int:
     global _active_generation_requests
     with _active_generation_lock:
         _active_generation_requests += 1
+        return _active_generation_requests
 
 
-def _end_generation_request() -> None:
+def _end_generation_request() -> int:
     global _active_generation_requests
     with _active_generation_lock:
         _active_generation_requests = max(0, _active_generation_requests - 1)
+        return _active_generation_requests
 
 
 def _get_active_generation_requests() -> int:
@@ -123,11 +141,16 @@ def _api_asr(audio_path: str) -> str:
     wav_b64 = base64.b64encode(path.read_bytes()).decode("utf-8")
     wav_fmt = path.suffix.lstrip(".").lower() or "wav"
 
-    resp = requests.post(
-        f"{NANOVLLM_API_BASE}/asr",
-        json={"wav_base64": wav_b64, "wav_format": wav_fmt},
-        timeout=60,
-    )
+    try:
+        resp = requests.post(
+            f"{NANOVLLM_API_BASE}/asr",
+            json={"wav_base64": wav_b64, "wav_format": wav_fmt},
+            timeout=_api_timeout("NANOVLLM_API_ASR_TIMEOUT", 30.0),
+        )
+    except requests.Timeout as exc:
+        raise BackendTimeoutError("ASR request timed out") from exc
+    if resp.status_code == 429:
+        raise BackendBusyError("ASR backend is busy")
     resp.raise_for_status()
     return resp.json().get("text", "")
 
@@ -140,11 +163,16 @@ def _api_denoise(audio_path: str) -> str:
     wav_b64 = base64.b64encode(path.read_bytes()).decode("utf-8")
     wav_fmt = path.suffix.lstrip(".").lower() or "wav"
 
-    resp = requests.post(
-        f"{NANOVLLM_API_BASE}/denoise",
-        json={"wav_base64": wav_b64, "wav_format": wav_fmt},
-        timeout=120,
-    )
+    try:
+        resp = requests.post(
+            f"{NANOVLLM_API_BASE}/denoise",
+            json={"wav_base64": wav_b64, "wav_format": wav_fmt},
+            timeout=_api_timeout("NANOVLLM_API_DENOISE_TIMEOUT", 60.0),
+        )
+    except requests.Timeout as exc:
+        raise BackendTimeoutError("Denoise request timed out") from exc
+    if resp.status_code == 429:
+        raise BackendBusyError("Denoise backend is busy")
     resp.raise_for_status()
 
     denoised_b64 = resp.json()["wav_base64"]
@@ -333,19 +361,29 @@ def _api_generate(payload: dict) -> str:
 
     url = f"{NANOVLLM_API_BASE}/generate"
     logger.info(f"Calling {url} ...")
-
-    resp = requests.post(url, json=payload, stream=True, timeout=300)
-    resp.raise_for_status()
-
-    tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".mp3")
+    tmp = None
     try:
+        resp = requests.post(
+            url,
+            json=payload,
+            stream=True,
+            timeout=_api_timeout("NANOVLLM_API_GENERATE_TIMEOUT", 120.0),
+        )
+        if resp.status_code == 429:
+            raise BackendBusyError("Generate backend is busy")
+        resp.raise_for_status()
+
+        tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".mp3")
         for chunk in resp.iter_content(chunk_size=64 * 1024):
             tmp.write(chunk)
         tmp.close()
         return tmp.name
+    except requests.Timeout as exc:
+        raise BackendTimeoutError("Generate request timed out") from exc
     except Exception:
-        tmp.close()
-        if os.path.exists(tmp.name):
+        if tmp is not None:
+            tmp.close()
+        if tmp is not None and os.path.exists(tmp.name):
             os.unlink(tmp.name)
         raise
 
@@ -372,7 +410,8 @@ def generate_tts_audio(
     denoise: bool = True,
     request: Optional[gr.Request] = None,
 ) -> str:
-    _begin_generation_request()
+    started_at = time.monotonic()
+    active_at_start = _begin_generation_request()
     request_payload = {
         "event": "tts_request",
         "ui_language": _resolve_ui_language(request),
@@ -384,7 +423,14 @@ def generate_tts_audio(
         "do_normalize": bool(do_normalize),
         "denoise": bool(denoise),
         "has_reference_audio": bool(reference_wav_path_input and reference_wav_path_input.strip()),
+        "active_generation_requests_at_start": active_at_start,
     }
+    logger.info(
+        "TTS request started: active=%s, has_ref=%s, use_prompt_text=%s",
+        active_at_start,
+        request_payload["has_reference_audio"],
+        request_payload["use_prompt_text"],
+    )
     if request_payload["has_reference_audio"]:
         try:
             request_payload["reference_audio_duration_seconds"] = round(
@@ -471,29 +517,63 @@ def generate_tts_audio(
                     pass
 
         try:
-            _append_request_log({**request_payload, "status": "success"})
+            duration_seconds = round(time.monotonic() - started_at, 3)
+            _append_request_log(
+                {**request_payload, "status": "success", "duration_seconds": duration_seconds}
+            )
         except Exception as exc:
             logger.warning(f"Failed to append request log: {exc}")
 
         return mp3_path
 
+    except BackendBusyError as exc:
+        duration_seconds = round(time.monotonic() - started_at, 3)
+        logger.warning("TTS backend busy after %.3fs: %s", duration_seconds, exc)
+        try:
+            _append_request_log(
+                {**request_payload, "status": "backend_busy", "error": str(exc), "duration_seconds": duration_seconds}
+            )
+        except Exception:
+            pass
+        raise gr.Error(_get_i18n_text("backend_busy_error", request)) from exc
+    except BackendTimeoutError as exc:
+        duration_seconds = round(time.monotonic() - started_at, 3)
+        logger.warning("TTS backend timeout after %.3fs: %s", duration_seconds, exc)
+        try:
+            _append_request_log(
+                {**request_payload, "status": "backend_timeout", "error": str(exc), "duration_seconds": duration_seconds}
+            )
+        except Exception:
+            pass
+        raise gr.Error(_get_i18n_text("backend_timeout_error", request)) from exc
     except (ValueError, gr.Error) as exc:
+        duration_seconds = round(time.monotonic() - started_at, 3)
         try:
-            _append_request_log({**request_payload, "status": "rejected", "error": str(exc)})
+            _append_request_log(
+                {**request_payload, "status": "rejected", "error": str(exc), "duration_seconds": duration_seconds}
+            )
         except Exception:
             pass
         if isinstance(exc, gr.Error):
             raise
         raise gr.Error(str(exc)) from exc
     except Exception as exc:
+        duration_seconds = round(time.monotonic() - started_at, 3)
         logger.exception("Generation failed")
         try:
-            _append_request_log({**request_payload, "status": "error", "error": str(exc)})
+            _append_request_log(
+                {**request_payload, "status": "error", "error": str(exc), "duration_seconds": duration_seconds}
+            )
         except Exception:
             pass
         raise gr.Error(_get_i18n_text("backend_retry_error", request)) from exc
     finally:
-        _end_generation_request()
+        remaining = _end_generation_request()
+        logger.info(
+            "TTS request finished: duration=%.3fs, remaining_active=%s",
+            time.monotonic() - started_at,
+            remaining,
+        )
 
 
 # ---------- Inline i18n (en + zh-CN) ----------
@@ -596,6 +676,8 @@ _I18N_TRANSLATIONS = {
         "denoise_busy_error": "Too many reference-audio enhancement requests are running. Please try again in a moment.",
         "denoise_failed_error": "Reference audio enhancement failed. Please try disabling denoise or use a cleaner clip.",
         "backend_retry_error": "The backend is temporarily unstable. Please try again in a moment.",
+        "backend_busy_error": "The backend is busy. Please try again in a moment.",
+        "backend_timeout_error": "The backend request timed out. Please try shorter text or reference audio.",
         "asr_failed_error": "ASR failed. Please fill the transcript manually or try another reference audio.",
         "usage_instructions": _USAGE_INSTRUCTIONS_EN,
         "examples_footer": _EXAMPLES_FOOTER_EN,
@@ -622,6 +704,8 @@ _I18N_TRANSLATIONS = {
         "denoise_busy_error": "当前参考音频降噪请求过多，请稍后再试。",
         "denoise_failed_error": "参考音频降噪失败，请尝试关闭降噪或更换更干净的音频。",
         "backend_retry_error": "后端暂时不稳定，请稍后再试。",
+        "backend_busy_error": "后端当前繁忙，请稍后再试。",
+        "backend_timeout_error": "后端请求超时，请尝试缩短文本或参考音频后重试。",
         "asr_failed_error": "ASR 识别失败，请手动填写参考音频文本，或更换一段参考音频后重试。",
         "usage_instructions": _USAGE_INSTRUCTIONS_ZH,
         "examples_footer": _EXAMPLES_FOOTER_ZH,