"""RW-Voice EQ ASR — public Gradio app.
Reads pre-computed parquet artefacts from a private HF dataset and renders
the leaderboard, per-condition breakdowns, and a small set of curated
audio samples.
"""
import base64
import html
import os
from pathlib import Path
import gradio as gr
import pandas as pd
from huggingface_hub import hf_hub_download
from huggingface_hub.constants import HF_HUB_CACHE
HF_TOKEN = os.environ.get("HF_TOKEN")
HF_DATA_REPO = os.environ.get("HF_DATA_REPO", "HumeAI/hume-asr-leaderboard-data")
def _cache_path(filename: str) -> Path:
"""Cache files live in the private HF Dataset — downloaded on startup."""
return Path(hf_hub_download(
repo_id=HF_DATA_REPO,
filename=filename,
repo_type="dataset",
token=HF_TOKEN,
))
def _logo_data_uri(filename: str) -> str:
"""Return a data: URI for a logo file next to app.py, or empty if missing."""
p = Path(__file__).parent / filename
if not p.exists():
return ""
mimes = {".svg": "image/svg+xml", ".png": "image/png",
".avif": "image/avif", ".webp": "image/webp"}
mime = mimes.get(p.suffix.lower(), "application/octet-stream")
b64 = base64.b64encode(p.read_bytes()).decode("ascii")
return f"data:{mime};base64,{b64}"
# Black logo shows in light mode; white logo shows in dark mode. Gradio
# toggles a `.dark` class on the container — the CSS in build_app() uses
# that to swap which ![]()
is visible.
_LOGO_LIGHT_URI = _logo_data_uri("Hume_Logo_Black.avif")
_LOGO_DARK_URI = _logo_data_uri("Hume_Logo_White.avif")
CACHE = _cache_path("leaderboard.json")
GOLDEN_CACHE = _cache_path("golden_samples.json")
EMO_CACHE = _cache_path("emotions_breakdown.json")
ACCENTS_CACHE = _cache_path("accents_breakdown.json")
BG_CACHE = _cache_path("noise_music_breakdown.json")
DESCRIPTION = """\
The **RW-Voice EQ ASR** evaluates open-source and proprietary English speech recognition models against human-curated references across four real-world conditions: accents, emotions, background audio (music & noise), and conversational dialogue.
Models are ranked by **Average WER** — the arithmetic mean across the four datasets (lower is better). We also report **RTFx**, the inverse real-time factor (higher is faster). Scoring follows HuggingFace's [open_asr_leaderboard](https://huggingface.co/spaces/hf-audio/open_asr_leaderboard) methodology — their text normalizer plus corpus-level WER — so numbers here are directly comparable to HF's.
"""
DISPLAY_COLS = [
"rank", "model", "license", "size_b", "avg_wer", "rtfx",
"accents_wer", "emo_wer", "noise_music_wer", "daikon_wer",
]
HEADERS = {
"rank": "#",
"model": "Model",
"license": "License",
"size_b": "Size (B)",
"avg_wer": "Avg WER",
"rtfx": "RTFx",
"accents_wer": "Accents",
"emo_wer": "Emotions",
"noise_music_wer": "Background Audio",
"daikon_wer": "Conversational",
}
def _fmt(df: pd.DataFrame) -> pd.DataFrame:
df = df.copy().reset_index(drop=True)
df.insert(0, "rank", df.index + 1)
wer_cols = ["avg_wer", "accents_wer", "emo_wer", "noise_music_wer", "daikon_wer"]
for c in wer_cols:
df[c] = df[c].apply(lambda v: f"{v*100:.2f}%" if pd.notna(v) else "—")
# Proprietary models are served behind APIs — wall-clock includes network
# latency, server-side queuing, and batching we can't see. RTFx isn't a
# property of the model in that regime, so we don't report it (matches
# HF's open_asr_leaderboard convention of "NA" for closed APIs).
def _rtfx(row):
if row.get("license") == "proprietary":
return "NA"
v = row["rtfx"]
return f"{v:.2f}" if pd.notna(v) else "—"
df["rtfx"] = df.apply(_rtfx, axis=1)
df["size_b"] = df["size_b"].apply(lambda v: f"{v:.1f}" if pd.notna(v) else "—")
# Model column → clickable HF link when hf_id is set
if "hf_id" in df.columns:
def _link(row):
hid = row.get("hf_id")
label = hid if (hid and isinstance(hid, str)) else row["model"]
if hid and isinstance(hid, str):
return f"[{label}](https://huggingface.co/{hid})"
return label
df["model"] = df.apply(_link, axis=1)
return df[DISPLAY_COLS].rename(columns=HEADERS)
def load_cache() -> pd.DataFrame:
return pd.read_json(CACHE, orient="records")
def load_golden_cache() -> pd.DataFrame:
if not GOLDEN_CACHE.exists():
return pd.DataFrame()
return pd.read_json(GOLDEN_CACHE, orient="records")
def _attach_overall(breakdown: pd.DataFrame, main: pd.DataFrame, dataset: str) -> pd.DataFrame:
if breakdown.empty or main.empty:
return breakdown
src = main[["model", f"{dataset}_wer"]].rename(columns={f"{dataset}_wer": "overall_wer"})
return breakdown.merge(src, on="model", how="left")
def load_emo_cache(main_df: pd.DataFrame | None = None) -> pd.DataFrame:
df = pd.read_json(EMO_CACHE, orient="records")
if main_df is not None:
df = _attach_overall(df, main_df, "emo")
return df
def load_accents_cache(main_df: pd.DataFrame | None = None) -> pd.DataFrame:
df = pd.read_json(ACCENTS_CACHE, orient="records")
if main_df is not None:
df = _attach_overall(df, main_df, "accents")
return df
def load_bg_cache(main_df: pd.DataFrame | None = None) -> pd.DataFrame:
df = pd.read_json(BG_CACHE, orient="records")
if main_df is not None:
df = _attach_overall(df, main_df, "noise_music")
return df
# ─── Tab-specific helpers ───────────────────────────────────────────────
EMO_BUCKETS = [
("positive", "Positive"),
("negative_high_arousal", "Negative high-arousal"),
("negative_low_arousal", "Negative low-arousal"),
]
EMO_LEGEND = (
"**Legend.** "
"**Positive** — joy, amusement, excitement, love, surprise. "
"**Negative high-arousal** — anger, contempt, disgust, fear. "
"**Negative low-arousal** — sadness, grief, pain, boredom, anxiety."
)
BG_BUCKETS = [
("music", "Music"),
("noise", "Noise"),
]
BG_LEGEND = (
"**Legend.** Background type is derived from per-segment audio probabilities. "
"**Noise** — strong background noise with little or no music. "
"**Music** — background music with little background noise. "
"Speech is in the foreground in both buckets."
)
ACCENT_SCHEMES = {
"Native / 2nd language / Foreign": [
("kachru_inner_circle", "Native"),
("kachru_outer_circle", "2nd language"),
("kachru_expanding_circle", "Foreign language"),
],
"Native vs Non-native": [("native_native", "Native"),
("native_non_native", "Non-native")],
"Standard American vs Non": [("us_us", "Standard American"),
("us_non_us", "Non-Standard-American")],
}
ACCENT_LEGENDS = {
"Native / 2nd language / Foreign": (
"**Legend.** "
"**Native** — General American, Southern American, AAVE, Canadian, "
"British, Scottish, Irish, Australian, New Zealand, South African. "
"**2nd language** (English as an institutional 2nd language) "
"— Indian, South Asian, Caribbean, Nigerian, West African, East African, "
"Filipino, Singaporean / Malaysian. "
"**Foreign language** (English as a foreign language) — European-accented, "
"Spanish-accented, Middle Eastern, East / Southeast Asian, Other / Non-native."
),
"Native vs Non-native": (
"**Legend.** "
"**Native** — General American, Southern American, AAVE, Canadian, British, "
"Scottish, Irish, Australian, New Zealand, South African. "
"**Non-native** — Caribbean, Nigerian, West African, East African, Indian, "
"South Asian, Filipino, Singaporean / Malaysian, European-accented, "
"Spanish-accented, Middle Eastern, East / Southeast Asian, Other / Non-native."
),
"Standard American vs Non": (
"**Legend.** "
"**Standard American** — General American only. "
"**Non-Standard-American** — every other accent in the dataset, including "
"Southern American and AAVE."
),
}
def _fmt_breakdown(df: pd.DataFrame, columns: list[tuple[str, str]],
sort_col_wer: str) -> pd.DataFrame:
out = df.copy()
out = out.sort_values(sort_col_wer, na_position="last").reset_index(drop=True)
out.insert(0, "rank", out.index + 1)
wer_cols = [f"{p}_wer" for p, _ in columns]
if "overall_wer" in out.columns:
wer_cols.insert(0, "overall_wer")
for c in wer_cols:
out[c] = out[c].apply(lambda v: f"{v*100:.2f}%" if pd.notna(v) else "—")
out["size_b"] = out["size_b"].apply(lambda v: f"{v:.1f}" if pd.notna(v) else "—")
def _link(row):
hid = row.get("hf_id")
label = hid if (hid and isinstance(hid, str)) else row["model"]
if hid and isinstance(hid, str):
return f"[{label}](https://huggingface.co/{hid})"
return label
out["model"] = out.apply(_link, axis=1)
display_cols = ["rank", "model", "license", "size_b"]
if "overall_wer" in out.columns:
display_cols.append("overall_wer")
display_cols += [f"{p}_wer" for p, _ in columns]
headers = {
"rank": "#",
"model": "Model",
"license": "License",
"size_b": "Size (B)",
"overall_wer": "Overall",
}
for p, h in columns:
headers[f"{p}_wer"] = h
return out[display_cols].rename(columns=headers)
def filter_breakdown(df: pd.DataFrame, license_filter: str) -> pd.DataFrame:
return df if license_filter == "all" else df[df["license"] == license_filter]
def _audio_url(audio_path: str) -> str:
fname = Path(audio_path).name
try:
local = hf_hub_download(
repo_id=HF_DATA_REPO,
filename=f"audio/{fname}",
repo_type="dataset",
token=HF_TOKEN,
)
return f"/gradio_api/file={local}"
except Exception:
return ""
def render_golden_html(df_golden: pd.DataFrame, dataset_filter: str = "All") -> str:
if df_golden.empty:
return "No golden samples available.
"
df = df_golden if dataset_filter == "All" else df_golden[df_golden["dataset_label"] == dataset_filter]
if df.empty:
return f"No samples for {html.escape(dataset_filter)}.
"
parts: list[str] = [""]
for ds_label, ds_group in df.groupby("dataset_label", sort=False):
parts.append(
f"
"
f""
f"{html.escape(ds_label)} "
f"({len(ds_group)} samples)
"
f""
)
for tag, tag_group in ds_group.groupby("tag", sort=False):
tag_disp = (tag or "—").replace("tag_", "")
parts.append(
f"
"
f"{html.escape(tag_disp)} "
f"({len(tag_group)})
"
f""
)
for _, row in tag_group.iterrows():
t = html.escape(row["transcript"] or "")
parts.append(
"
"
)
parts.append("
")
parts.append("
")
parts.append("
"
f"
"
"
RW-Voice EQ ASR
"
"