"""SLM Judge Leaderboard — Gradio Space (a meta-leaderboard: how well each SLM judge tracks human ratings on Hume's TTS evals). Single-board sibling of the Real World VoiceEQ Space (same rendering machinery, same self-describing data schema, same shared dataset — this Space just lists one modality). The board renders as a heatmap-style ranking table from a self-describing JSON file that holds a *list* of boards (an Overall board + one per factor). The boards are pivoted into one wide table: each factor's `score` becomes a heatmapped column. Rows are ranked by the first factor column by default; clicking any factor column re-sorts and re-ranks, and rows with no value in the active column sink to the bottom, unranked (rank/provider/License are not sortable). The Overall board's composite score and the `coverage` count are not shown as columns. Heatmap modes: * Default (no board-level "heatmap" key): absolute 1–5 rater scale, higher = greener (all rater-scored tabs). * "heatmap": {"mode": "absolute", "stops": [...], "unit", "legend"}: fixed value anchors shared by every column, so a color means the same thing across columns; each column's "direction" is honoured ("asc" => lower is better) — the ASR WER tab (and the standalone SLM Judge Space's Pearson-r board). * "heatmap": {"mode": "normalized"}: each factor column is scaled by its own min/max across providers (direction honoured too), best value greenest. Theming: light and dark render from the same markup. All card colors live in `--lb-*` custom properties on `.ttslb` (light values = the canonical look) with a `.dark .ttslb` override set; each heatmap pill carries both themes' colors as inline custom properties. Gradio toggles the `dark` class on from `?__theme=` / system preference, so the tables restyle live with the page theme. Data source resolution order (per modality file): 1. $LEADERBOARD_DATASET (HF dataset holding all three JSONs, set on the Space). Private datasets need an $HF_TOKEN Space secret with read access. 2. local ./data/ (local development). Run locally: pip install -r requirements.txt && python app.py """ import base64 import html import json import os from pathlib import Path import gradio as gr HERE = Path(__file__).resolve().parent DATA_DIR = HERE / "data" DATASET_REPO = os.environ.get("LEADERBOARD_DATASET", "").strip() # Tab label / manifest key / board JSON. One file per modality; every tab renders # the same pivoted table from its own file. The key is what samples.json records # use in their "modality" field. MODALITIES = [ ("SLM Judge", "slm_judge", "slm_judge_leaderboard.json"), ] # Board ids kept in the data files but not rendered as columns (the SLM Judge # sibling hides its "Overall" mean column this way; nothing is hidden here). HIDDEN_BOARDS: frozenset = frozenset({"overall_avg_r"}) # Curated audio samples (optional): a samples.json manifest next to the board # JSONs, each record {modality, factor, label?, group?, model?, text?, # transcript?, audio} where "audio" is a repo-relative path (e.g. # "samples/foo.mp3") resolved against the dataset (or ./data locally). A # record's modality routes it to that tab, which appends a samples section # (per-factor category tabs) below the board table. "text" (generation prompt) # is kept but never shown; "transcript" (reference transcript, ASR golden # samples) is rendered next to the player. Consecutive records sharing a # "group" (e.g. the Accents tab's Native / 2nd language / Foreign buckets) # render under one shared subheading instead of per-row labels; a group whose # records carry a "scenario" renders a bold "Scenario:" blurb as its lead-in # instead of the heading. # A record with "turns" is a conversation (STS samples) rendered as a card: # header = model (or "label" for stimulus cards) + optional "score" # ({value, max, label}) badge + "verdict"/"verdict_label" chip + "caption"; # each turn is {role: user|agent, speaker?, transcript, audio?, tag?, note?}. SAMPLES_NAME = "samples.json" # Each board in the data carries a single primary `score` column plus a `coverage` # count. Coverage is kept in the merged metrics but not shown as a column (were a # board to emit one, it would render as a plain count, not heatmapped). COVERAGE_FIELD = "coverage" # Heatmap palette, red (worst) -> green (best), one row per theme. The dark row # keeps the light row's hues but deepened and luminance-matched to the dark card, # so the light ink stays >= 4.5:1 along the whole interpolated ramp. _PALETTE = [(192, 86, 74), (217, 140, 95), (232, 192, 106), (230, 224, 138), (197, 217, 107), (156, 204, 101), (139, 195, 74)] _PALETTE_DARK = [(130, 62, 54), (131, 74, 41), (121, 91, 28), (108, 103, 31), (91, 104, 34), (74, 103, 41), (73, 100, 40)] # Absolute 1–5 anchors: (score, palette colour) for the default rater-scale mode. _STOPS = (2.20, 2.50, 2.80, 3.00, 3.20, 3.50, 3.90) _ANCHORS = list(zip(_STOPS, _PALETTE)) _ANCHORS_DARK = list(zip(_STOPS, _PALETTE_DARK)) _DARK_FG = "#e9edf2" def _num(x): try: return float(x) except (TypeError, ValueError): return None def _hex(rgb): return "#%02x%02x%02x" % tuple(rgb) def _lerp(a, b, t): return [round(x + (y - x) * t) for x, y in zip(a, b)] def _ramp(anchors, v): """Interpolate v along (value, rgb) anchor stops -> hex.""" lo, hi = anchors[0][0], anchors[-1][0] v = max(lo, min(hi, v)) rgb = anchors[-1][1] for (av, argb), (bv, brgb) in zip(anchors, anchors[1:]): if v <= bv: t = 0.0 if bv == av else (v - av) / (bv - av) rgb = _lerp(argb, brgb, t) break return _hex(rgb) def _heat_abs(v): """Absolute 1–5 rater scale: higher is greener. Returns the two themes' (bg, fg) pairs: ((light bg, light fg), (dark bg, dark fg)).""" light = (_ramp(_ANCHORS, v), "#ffffff" if v <= 2.30 else "#33352f") return light, (_ramp(_ANCHORS_DARK, v), _DARK_FG) def _heat_t(t): """Normalised position t in [0,1] (0 = worst/red, 1 = best/green). Same ((bg, fg), (bg, fg)) light/dark shape as _heat_abs.""" t = max(0.0, min(1.0, t)) x = t * (len(_PALETTE) - 1) i = int(x) def pick(pal): return _hex(pal[-1] if i >= len(pal) - 1 else _lerp(pal[i], pal[i + 1], x - i)) light = (pick(_PALETTE), "#ffffff" if t <= 0.12 else "#33352f") return light, (pick(_PALETTE_DARK), _DARK_FG) def _make_color_fn(board: dict, factor_cols: list): """Return color_fn(col, value) -> (bg, fg), per the board's heatmap mode.""" hm = board.get("heatmap") if not hm: return lambda col, v: _heat_abs(v) if hm.get("mode") == "absolute" and hm.get("stops"): # Fixed value anchors shared by every column (board-supplied stops, e.g. # WER percentages), so a color means the same thing in every column — # the absolute-scale analogue of the rater tabs' 1–5 anchors. A column's # "asc" direction flips the ramp (low WER = green). stops = [float(s) for s in hm["stops"]] def fn_abs(col, v): x = max(stops[0], min(stops[-1], float(v))) t = 1.0 for i, (a, b) in enumerate(zip(stops, stops[1:])): if x <= b: f = 0.0 if b == a else (x - a) / (b - a) t = (i + f) / (len(stops) - 1) break if col.get("direction") == "asc": t = 1.0 - t return _heat_t(t) return fn_abs parts = board.get("participants", []) stats = {} for c in factor_cols: vals = [_num((p.get("metrics") or {}).get(c["field"])) for p in parts] vals = [x for x in vals if x is not None] stats[c["field"]] = (min(vals), max(vals)) if vals else (0.0, 1.0) def fn(col, v): lo, hi = stats[col["field"]] t = 0.5 if hi == lo else (v - lo) / (hi - lo) if col.get("direction") == "asc": # lower is better (e.g. WER) t = 1.0 - t return _heat_t(t) return fn def _data_path(data_name: str) -> Path: if DATASET_REPO: from huggingface_hub import hf_hub_download return Path(hf_hub_download( DATASET_REPO, data_name, repo_type="dataset", token=os.environ.get("HF_TOKEN"), )) local = DATA_DIR / data_name if local.exists(): return local raise FileNotFoundError(f"No data found. Set $LEADERBOARD_DATASET or add {local}.") def load_boards(data_name: str) -> list: """The full list of boards in a modality's data file, minus any HIDDEN_BOARDS. A single-object file (legacy shape) is wrapped so callers always get a list.""" data = json.loads(_data_path(data_name).read_text()) boards = data if isinstance(data, list) else [data] return [b for b in boards if b.get("id") not in HIDDEN_BOARDS] def load_samples() -> list: """samples.json from the dataset (or ./data), [] when absent.""" try: return json.loads(_data_path(SAMPLES_NAME).read_text()) except Exception: return [] def _audio_src(rel_path: str) -> str: """Servable URL for a repo-relative audio path. Files are pulled into the HF cache (dataset mode) or read from ./data, then exposed through gradio's static file route — both roots are passed to launch(allowed_paths=...).""" if DATASET_REPO: from huggingface_hub import hf_hub_download try: local = hf_hub_download( DATASET_REPO, rel_path, repo_type="dataset", token=os.environ.get("HF_TOKEN"), ) return f"/gradio_api/file={local}" except Exception: return "" local = DATA_DIR / rel_path return f"/gradio_api/file={local}" if local.exists() else "" def _factor_label(board: dict) -> str: """Column label for a factor board: its title without the modality prefix ("TTS — Acting / Role-fit" → "Acting / Role-fit").""" t = board.get("title", "") return t.split(" — ", 1)[1] if " — " in t else t def merge_boards(boards: list) -> dict: """Pivot the per-factor boards into one wide board for a single table. The Overall board (id "overall") supplies per-model license/size and any provider-level `note` (rendered as a marker on the model name); its rank, composite score and `coverage` count are kept on the participant but not displayed — ranking is dynamic, by whichever column the table is sorted on. Every other board contributes one factor column keyed by its id, whose value is that board's `score` for the model. The participant roster is shared, so a model's row gathers its score from each board. A file with no Overall board (a single-factor file) treats every board as a factor board; license/size then come from the factor boards' participants.""" if not boards: return {"title": "", "description": "", "keyMetric": {}, "metricColumns": [], "participants": []} overall = next((b for b in boards if b.get("id") == "overall"), None) factor_boards = [b for b in boards if b is not overall] metric_columns = [] for b in factor_boards: # The factor board's description doubles as the column-header tooltip; # its keyMetric supplies the column's unit and sort direction (WER-style # boards declare "asc" = lower is better). km = b.get("keyMetric") or {} metric_columns.append({"field": b["id"], "label": _factor_label(b), "unit": km.get("unit", ""), "direction": km.get("direction", "desc"), "description": (b.get("description") or "").strip(), # a note on the board itself footnotes the header "note": (b.get("note") or "").strip()}) by_model: dict = {} order: list = [] def slot(p): model = p["model"] e = by_model.get(model) if e is None: e = {"rank": None, "model": model, "license": p.get("license"), "sizeB": p.get("sizeB"), "note": None, "notes": {}, "metrics": {"composite": None, "coverage": None}} by_model[model] = e order.append(model) return e for p in (overall.get("participants", []) if overall else []): e = slot(p) e["rank"] = p.get("rank") e["license"] = p.get("license") e["sizeB"] = p.get("sizeB") e["note"] = p.get("note") # provider-level caveat -> marker on the name e["metrics"]["composite"] = (p.get("metrics") or {}).get("score") e["metrics"]["coverage"] = (p.get("metrics") or {}).get("coverage") for b in factor_boards: for p in b.get("participants", []): e = slot(p) e["metrics"][b["id"]] = (p.get("metrics") or {}).get("score") if p.get("note"): # why this factor's score is missing -> footnote e["notes"][b["id"]] = p["note"] if e["metrics"].get("coverage") is None: e["metrics"]["coverage"] = (p.get("metrics") or {}).get("coverage") meta = overall or boards[0] return { "id": "merged", "title": meta.get("title", ""), "description": meta.get("description", ""), "keyMetric": {}, # any board declaring a heatmap mode (normalized WER-style tabs) sets it # for the whole merged table "heatmap": next((b.get("heatmap") for b in boards if b.get("heatmap")), None), "metricColumns": metric_columns, "participants": [by_model[m] for m in order], } # ── About tab ─────────────────────────────────────────────────────────────── # Content lives in about.json next to the board JSONs ({intro: [...], # panels: [{tab, title, paragraphs}], outro: [...]}). Each panel renders as # one clickable card that jumps to that leaderboard tab (wired in APP_JS); # "tab" must match the MODALITIES label exactly. ABOUT_NAME = "slm_judge_about.json" def load_about() -> dict: """about.json from the dataset (or ./data), {} when absent — the About tab is simply skipped without it.""" try: return json.loads(_data_path(ABOUT_NAME).read_text()) except Exception: return {} def render_about(about: dict) -> str: """The About tab: intro paragraphs, one clickable card per leaderboard panel (title + description, click = jump to that tab), outro. Panels whose tab isn't currently in MODALITIES (e.g. Voice Controllability while its tab is retired) are kept in the data but not rendered.""" tabs = {label for label, _key, _fname in MODALITIES} parts = ["

"] for p in about.get("intro", []): parts.append(f"

{html.escape(p)}

") cards = [] for panel in about.get("panels", []): if panel.get("tab") not in tabs: continue body = "".join(f"

{html.escape(q)}

" for q in panel.get("paragraphs", [])) cards.append( f"

" f"

{html.escape(panel.get('title') or panel['tab'])}" f"→

{body}

") parts.append("

" + "".join(cards) + "

") for p in about.get("outro", []): parts.append(f"

{html.escape(p)}

") parts.append("

") return "".join(parts) def _logo_data_uri(suffix: str = "") -> str: mimes = {".avif": "image/avif", ".svg": "image/svg+xml", ".png": "image/png", ".webp": "image/webp"} for stem in ("hume-logo", "hume_logo"): for ext, mime in mimes.items(): p = HERE / f"{stem}{suffix}{ext}" if p.exists(): b64 = base64.b64encode(p.read_bytes()).decode("ascii") return f"data:{mime};base64,{b64}" return "" _LOGO_URI = _logo_data_uri() # Dark-surface wordmark (light ink, same pastel dots); when the asset is missing # the light logo is used alone, un-swapped. _LOGO_DARK_URI = _logo_data_uri("-dark") def license_choices(board: dict) -> list: seen = [] for p in board.get("participants", []): lic = (p.get("license") or "").strip() if lic and lic not in seen: seen.append(lic) return ["all"] + sorted(seen) # The card's colors all live in `--lb-*` custom properties on `.ttslb`. The light # values ARE the original look, verbatim; `.dark .ttslb` (gradio toggles `dark` on # from ?__theme= / system preference) swaps in a dark set whose grays are # contrast-matched to the light hierarchy against the dark card. `color-scheme` # follows, so native audio controls and scrollbars flip too. Heatmap pills read # `--hm-*` properties that each cell sets inline for both themes. Injected once # into alongside APP_JS via launch(head=...) and shared by all tabs, so # the card is a class (`.ttslb`), not an id. TABLE_CSS = """ """ def _ramp_css(palette) -> str: """linear-gradient for the legend's 1–5 bar; colors clamp outside the anchor span exactly like the pill ramp does.""" stops = ", ".join(f"{_hex(rgb)} {(v - 1) / 4 * 100:.1f}%" for v, rgb in zip(_STOPS, palette)) return f"linear-gradient(90deg, {_hex(palette[0])} 0%, {stops}, {_hex(palette[-1])} 100%)" def _csv_icon_css(stroke: str) -> str: """Fully percent-encoded data-URI url(...) of the download glyph (arrow into tray), stroked in the given color.""" from urllib.parse import quote svg = ("") return f"url(\"data:image/svg+xml,{quote(svg, safe='')}\")" TABLE_CSS = (TABLE_CSS.replace("@RAMP_LIGHT@", _ramp_css(_PALETTE)) .replace("@RAMP_DARK@", _ramp_css(_PALETTE_DARK)) # idle/hover strokes mirror --lb-ink-2/--lb-ink per theme .replace("@CSV_ICON_L@", _csv_icon_css("#6b7280")) .replace("@CSV_ICON_LH@", _csv_icon_css("#1f2328")) .replace("@CSV_ICON_D@", _csv_icon_css("#b0b5be")) .replace("@CSV_ICON_DH@", _csv_icon_css("#e7eaef"))) # Behaviour, injected into (a """ def _row_html(p, heat_cols, primary_col, cov_col, rank_label, color_fn, marker_of=None, active_col=None) -> str: m = p.get("metrics", {}) or {} model_raw = str(p.get("model", "")) lic_raw = (p.get("license") or "").strip() or "—" # a provider-level note (a caveat about the model itself, e.g. "this name is # a cascade alias") marks the model name rather than any one factor cell pnote = p.get("note") pmark = (marker_of or {}).get(pnote) if pnote else None if pmark: ptip = html.escape(pnote, quote=True) prov_cell = (f"{html.escape(model_raw)}" f"{pmark}") else: prov_cell = f"{html.escape(model_raw)}" cells = [ f"{rank_label}", prov_cell, f"{html.escape(lic_raw)}", ] # primary score + any extra factor columns: heatmapped pills. A participant # "note" renders as a footnote marker — alone in the cell when the value is # missing (it explains the gap), after the pill when a value is present (it # caveats the number). Marked cells also carry the note as a hover tooltip. for c in heat_cols: v = _num(m.get(c["field"])) note = (p.get("notes") or {}).get(c["field"]) mark = (marker_of or {}).get(note) tip = html.escape(note, quote=True) if mark else "" marker = f"{mark}" if mark else "" if v is None: if mark: cells.append(f"{marker}") else: cells.append("·") else: (bg, fg), (bgd, fgd) = color_fn(c, v) cls = "pill primary" if c is primary_col else "pill" # dim = outside the initially-sorted column (focus mode); the sort # JS re-marks these as the active column changes td_classes = ((["notecell"] if mark else []) + (["dim"] if active_col is not None and c is not active_col else [])) td_cls = f" class='{' '.join(td_classes)}'" if td_classes else "" # unit suffix ("%", "x") rides along in the pill; data-v stays the # bare number so sorting and CSV export keep working txt = f"{v:.2f}{html.escape(c.get('unit') or '')}" cells.append( f"" f"{txt}{marker}" ) # coverage: plain count (e.g. "9/9"), not heatmapped. if cov_col is not None: cov = _num(m.get(COVERAGE_FIELD)) if cov is None: cells.append("·") else: cov_txt = f"{int(cov)}{cov_col.get('unit', '')}" cells.append(f"{html.escape(cov_txt)}") return "".join(cells) def render_board_html(board: dict, lic: str = "all", fname: str = "board") -> str: cols = board.get("metricColumns", []) by_field = {c["field"]: c for c in cols} key_field = (board.get("keyMetric") or {}).get("field") primary_col = (next((c for c in cols if c.get("primary")), None) or by_field.get(key_field)) cov_col = by_field.get(COVERAGE_FIELD) # Heatmapped columns: the primary score first, then any extra factor columns # (coverage is excluded — it's a count, rendered plain and shown last). factor_cols = [c for c in cols if c is not primary_col and c["field"] != COVERAGE_FIELD] heat_cols = ([primary_col] if primary_col else []) + factor_cols color_fn = _make_color_fn(board, heat_cols) # Display order after rank/provider/License: primary, extra factors, coverage. value_cols = heat_cols + ([cov_col] if cov_col else []) parts = board.get("participants", []) if lic != "all": parts = [p for p in parts if (p.get("license") or "") == lic] # Default ranking: the first value column, best first. Every row is always # shown; rows with no value in the active sort column sink to the bottom, # unranked (rank "—"). first_col = value_cols[0] if value_cols else None def _cell(p, c): return _num((p.get("metrics") or {}).get(c["field"])) if first_col is not None: ranked = [p for p in parts if _cell(p, first_col) is not None] unranked = [p for p in parts if _cell(p, first_col) is None] ranked.sort(key=lambda p: _cell(p, first_col), reverse=first_col.get("direction") != "asc") else: ranked, unranked = list(parts), [] # Footnote markers: one symbol per distinct note among the visible cells — a # column-level note marks the header (a caveat about the whole metric, shown # without a hover since the header already carries the ⓘ description tip), a # provider-level note marks the model name (a caveat about the model itself), # a note on an empty cell explains the gap, and a note on a valued cell flags # a caveat (e.g. r reported as 0 where zero variance left it undefined). # Collection order tracks visual position: headers, then rows, then cells. _MARKS = ("*", "††", "‡‡", "§§", "‖‖", "¶¶") marker_of = {} for c in value_cols: note = c.get("note") if note and note not in marker_of: marker_of[note] = _MARKS[len(marker_of) % len(_MARKS)] for p in parts: note = p.get("note") if note and note not in marker_of: marker_of[note] = _MARKS[len(marker_of) % len(_MARKS)] for c in value_cols: for p in parts: note = (p.get("notes") or {}).get(c["field"]) if note and note not in marker_of: marker_of[note] = _MARKS[len(marker_of) % len(_MARKS)] # Sortable headers (factor columns only): data-ci = column index, data-num = # numeric, data-desc = first click sorts descending. (The sort behaviour lives # in SORT_JS.) `#` is display position, renumbered on every sort; the rank, # provider, and License columns are not sortable. ths = [ "#", "provider", "License", ] for j, c in enumerate(value_cols): d = "asc" if c.get("direction") == "asc" else "desc" init = f" data-sort='{d}'" if c is first_col else "" arrow = (" ▲" if d == "asc" else " ▼") if c is first_col else " ↕" tip = (c.get("description") or "").strip() info = (f"ⓘ" if tip else "") hmark = marker_of.get(c.get("note")) hmarker = f"{hmark}" if hmark else "" ths.append( f"" f"{html.escape(c['label'])}{hmarker}{arrow}{info}" ) header = "".join(ths) body = [] for i, p in enumerate(ranked + unranked): alt = "alt" if (i + 1) % 2 == 0 else "" label = i + 1 if i < len(ranked) else "—" body.append(f"{_row_html(p, heat_cols, primary_col, cov_col, label, color_fn, marker_of, first_col)}") if not body: return "

No providers match this filter.

" footnotes = "" if marker_of: footnotes = ("

" + "".join( f"

{mark} {html.escape(note)}

" for note, mark in marker_of.items()) + "

") # Toolbar: the score-scale legend (left) and a vertical stack of labelled # controls (right): the "Toggle colors" switch over "Download CSV", both # handled in APP_JS. All sit inside .tbl-box, so they align with the # table's edges. The legend follows the heatmap mode: # 1–5 rater scale by default, the board's own value anchors for "absolute" # boards (ends flipped for asc columns so red stays on the worst end), and a # generic worst→best for "normalized" boards. hm = board.get("heatmap") or {} if hm.get("mode") == "absolute" and hm.get("stops"): unit, cap = hm.get("unit", ""), (hm.get("legend") or "").strip() ends = [f"{float(s):g}{unit}" for s in (hm["stops"][0], hm["stops"][-1])] if value_cols and value_cols[0].get("direction") == "asc": ends.reverse() legend = (f"

{html.escape(ends[0])}" f"{html.escape(ends[1])}" + (f"{html.escape(cap)}" if cap else "") + "

") elif hm: # normalized boards default to explaining the per-column scaling; a # board-supplied "legend" caption overrides it cap = (hm.get("legend") or "scaled per column").strip() legend = ("

worst" "best" f"{html.escape(cap)}

") else: legend = ("

1" "5" "mean human rating

") toolbar = (f"") return ( "

" + toolbar + "

" + header + "" + "".join(body) + "

" + footnotes + "

" ) def _convo_html(s: dict) -> str: """A conversation sample ({"turns": [...]}) as a bordered card: header row (model or label, score badge, verdict chip), optional caption, then the turns — role gutter, per-turn player when audio exists, transcript, and optional tag pill / muted note (curator commentary).""" title = s.get("model") or s.get("label") or "" title_cls = "smp-cv-model" if s.get("model") else "smp-cv-label" # an anonymized outcome title ("Poorly rated conversation") wears its # verdict as a colored dot; chips only render when verdict_label is set tone = s.get("verdict") if s.get("verdict") in ("good", "poor") else "" if tone and not s.get("model"): title_cls += f" {tone}" head = [f"{html.escape(title)}"] sc = s.get("score") if sc: head.append(f"{sc['value']:g} / {sc['max']:g}" f" {html.escape(sc.get('label', ''))}") if s.get("verdict_label"): head.append(f"" f"{html.escape(s['verdict_label'])}") parts = [f"

{''.join(head)}

"] if s.get("caption"): parts.append(f"

{html.escape(s['caption'])}

") turns = [] for t in s.get("turns", []): role = "agent" if t.get("role") == "agent" else "user" speaker = t.get("speaker") or ("Agent" if role == "agent" else "User") src = _audio_src(t["audio"]) if t.get("audio") else "" player = f"

" if src else "" txt = html.escape(t.get("transcript") or "").replace("\n", "
") tag = (f" {html.escape(t['tag'])}" if t.get("tag") else "") note = (f"

{html.escape(t['note'])}

" if t.get("note") else "") turns.append(f"

" f"{html.escape(speaker)}" f"

{player}" f"

{txt}{tag}

{note}

") parts.append(f"

{''.join(turns)}

") return f"

{''.join(parts)}

" def render_samples_section(samples: list, factor_labels: dict, heading: str = "Sample Generations") -> str: """A modality tab's samples card ("Sample Generations"; "Golden Samples" for ASR's human-curated references; "Sample Conversations" for STS's curated multi-turn contrasts), rendered below the board table: one category tab per factor (first-seen manifest order, switching in APP_JS), each a list of label + audio player rows. A single-category modality gets no tab row — just the rows. A sample's "text" (generation prompt) and "model" attribution are kept in the manifest but not displayed; a "transcript" (the human reference for ASR golden samples) is rendered next to the player. Rows carrying a "group" cluster under one shared subheading (with a sample count) instead of repeating a per-row label — the manifest is expected to keep a group's rows adjacent.""" if not samples: return "" factors = [] for s in samples: # first-seen factor order if s.get("factor") not in factors: factors.append(s.get("factor")) tabs, panels = [], [] for j, f in enumerate(factors): rows = [s for s in samples if s.get("factor") == f] f_label = factor_labels.get(f) or (f or "—").replace("_", " ").title() tabs.append(f"") rows_html = [] cur_group = None for s in rows: group = (s.get("group") or "").strip() # each run of same-group records opens with a lead-in: a bold # "Scenario:" blurb when the record carries one (STS conversation # groups), else the group name + count subheading (ASR tag # buckets, whose grouped flat rows drop the head column and # indent under it) if group and group != cur_group: if s.get("scenario"): rows_html.append(f"

Scenario:" f" {html.escape(s['scenario'])}

") elif group != f_label: # a bucket named exactly like its tab would just repeat the # tab label — skip the subheading, keep the grouped rows n = sum(1 for r in rows if (r.get("group") or "").strip() == group) rows_html.append(f"

{html.escape(group)} " f"({n})

") cur_group = group or None if s.get("turns"): rows_html.append(_convo_html(s)) continue src = _audio_src(s.get("audio", "")) player = (f"

" if src else "audio pending") transcript = (s.get("transcript") or "").strip() if group: body = (f"{player}

{html.escape(transcript)}

" if transcript else player) rows_html.append(f"

{body}

") continue head = (f"{html.escape(s['label'])}" if s.get("label") else "") # a reference transcript (ASR golden samples) shares the row with the # player; the row's .txt class narrows the head and pins the player if transcript: rows_html.append( f"

{head}

{player}" f"

{html.escape(transcript)}

") else: rows_html.append(f"

{head}

{player}

") panels.append(f"

" + "".join(rows_html) + "

") tab_row = ("

" + "".join(tabs) + "

") if len(factors) > 1 else "" return (f"

{html.escape(heading)}

" + tab_row + "".join(panels) + "

") def build_demo() -> gr.Blocks: title = "SLM Judge Leaderboard" # NB: the CSS/JS goes to launch(head=...) — gradio 6 moved `head` off Blocks. with gr.Blocks(title=title) as demo: if _LOGO_URI: # 26px: the current wordmark asset is cropped to its ink (the original # had padding filling ~61% of a 48px box), sized down a touch further if _LOGO_DARK_URI: logo = (f" Hume AI

" f" Hume AI

") else: logo = f" Hume AI

" gr.HTML( "

" + logo + f"

{html.escape(title)}

" "

" ) else: gr.Markdown(f"# {title}") samples = load_samples() about = load_about() with gr.Tabs(): if about: with gr.Tab("About"): gr.HTML(render_about(about)) for label, key, data_name in MODALITIES: board = merge_boards(load_boards(data_name)) factor_labels = {c["field"]: c["label"] for c in board["metricColumns"]} # CSV filenames follow the visible tab label, e.g. # rw-voice-eq-speech-understanding.csv slug = label.lower().replace(" ", "-") with gr.Tab(label): gr.Markdown(board.get("description", "")) with gr.Row(): license_filter = gr.Radio( choices=license_choices(board), value="all", label="License", ) table = gr.HTML(render_board_html(board, "all", slug)) license_filter.change( lambda lic, b=board, s=slug: render_board_html(b, lic, s), inputs=license_filter, outputs=table, ) # this modality's curated samples, below the board mod_samples = [s for s in samples if s.get("modality") == key] if mod_samples: heading = {"asr": "Golden Samples", "sts": "Sample Conversations"}.get( key, "Sample Generations") gr.HTML(render_samples_section(mod_samples, factor_labels, heading)) return demo demo = build_demo() if __name__ == "__main__": from huggingface_hub.constants import HF_HUB_CACHE # SSR mode (gradio 6 default) needs to reach the app over localhost, which # HF Spaces blocks — disable it so the Space serves normally. allowed_paths # lets the /gradio_api/file= route serve sample audio from the HF cache # (dataset mode) or ./data (local dev). head= injects the table CSS + JS # (gradio 6 moved it here from the Blocks constructor). demo.launch(ssr_mode=False, head=TABLE_CSS + APP_JS, allowed_paths=[HF_HUB_CACHE, str(DATA_DIR)])