import { describe, expect, it } from "vitest" // Executable spec for `notes/transformations/08-benchmark-display-names.md`. // // Replicates BENCHMARK_NAMES + normalizeBenchmarkKeyForLookup + humanizeToken + // getBenchmarkDisplayName from lib/model-data.ts:90-148 verbatim. // // Also replicates the duplicate getBenchmarkDisplayName from // lib/eval-processing.ts:861-885 (Group D — functionally dead, tested for // completeness so a pipeline implementer porting the rules sees the // disagreement explicitly). // --------------------------------------------------------------------------- // Active implementation — lib/model-data.ts // --------------------------------------------------------------------------- function humanizeToken(token: string): string { return token .split(/[_-]+/g) .filter(Boolean) .map((part) => part.charAt(0).toUpperCase() + part.slice(1)) .join(" ") } const BENCHMARK_NAMES: Record = { hfopenllm_v2: "HF Open LLM v2", helm_lite: "HELM Lite", helm_capabilities: "HELM Capabilities", helm_classic: "HELM Classic", helm_instruct: "HELM Instruct", helm_mmlu: "HELM MMLU", reward_bench: "RewardBench", reward_bench_2: "RewardBench 2", bfcl: "BFCL", global_mmlu_lite: "Global MMLU Lite", swe_bench: "SWE-bench", arc_agi: "ARC-AGI", tau_bench_2: "TAU-Bench 2", ace: "ACE", apex_agents: "APEX Agents", apex_v1: "APEX v1", appworld: "AppWorld", browsecompplus: "BrowseComp+", livecodebenchpro: "LiveCodeBench Pro", sciarena: "SciArena", terminal_bench_2_0: "Terminal Bench 2.0", la_leaderboard: "LA Leaderboard", theory_of_mind: "Theory of Mind", fibble_arena: "Fibble Arena", fibble1_arena: "Fibble Arena v1", fibble2_arena: "Fibble Arena v2", fibble3_arena: "Fibble Arena v3", fibble4_arena: "Fibble Arena v4", fibble5_arena: "Fibble Arena v5", wordle_arena: "Wordle Arena", } function normalizeBenchmarkKeyForLookup(key: string): string { return key.toLowerCase().replace(/[-.\s]+/g, "_").replace(/^_+|_+$/g, "") } function getBenchmarkDisplayName(benchmark: string): string { return BENCHMARK_NAMES[normalizeBenchmarkKeyForLookup(benchmark)] ?? humanizeToken(benchmark) } // --------------------------------------------------------------------------- // Group A — Map hits (normalized-key lookup) // --------------------------------------------------------------------------- describe("Group A — BENCHMARK_NAMES map hits", () => { const cases = [ // Exact normalized matches { input: "hfopenllm_v2", expected: "HF Open LLM v2" }, { input: "helm_lite", expected: "HELM Lite" }, { input: "helm_capabilities", expected: "HELM Capabilities" }, { input: "helm_classic", expected: "HELM Classic" }, { input: "helm_instruct", expected: "HELM Instruct" }, { input: "helm_mmlu", expected: "HELM MMLU" }, { input: "reward_bench", expected: "RewardBench" }, { input: "reward_bench_2", expected: "RewardBench 2" }, { input: "bfcl", expected: "BFCL" }, { input: "global_mmlu_lite", expected: "Global MMLU Lite" }, { input: "swe_bench", expected: "SWE-bench" }, { input: "arc_agi", expected: "ARC-AGI" }, { input: "tau_bench_2", expected: "TAU-Bench 2" }, { input: "ace", expected: "ACE" }, { input: "apex_agents", expected: "APEX Agents" }, { input: "apex_v1", expected: "APEX v1" }, { input: "appworld", expected: "AppWorld" }, { input: "browsecompplus", expected: "BrowseComp+" }, { input: "livecodebenchpro", expected: "LiveCodeBench Pro" }, { input: "sciarena", expected: "SciArena" }, { input: "terminal_bench_2_0", expected: "Terminal Bench 2.0" }, { input: "la_leaderboard", expected: "LA Leaderboard" }, { input: "theory_of_mind", expected: "Theory of Mind" }, { input: "fibble_arena", expected: "Fibble Arena" }, { input: "fibble1_arena", expected: "Fibble Arena v1" }, { input: "fibble2_arena", expected: "Fibble Arena v2" }, { input: "fibble3_arena", expected: "Fibble Arena v3" }, { input: "fibble4_arena", expected: "Fibble Arena v4" }, { input: "fibble5_arena", expected: "Fibble Arena v5" }, { input: "wordle_arena", expected: "Wordle Arena" }, // Case- and separator-insensitive lookups (normalize: lower; -/./space -> _; trim _) { input: "HELM Lite", expected: "HELM Lite", why: "space -> _ during normalize" }, { input: "helm-lite", expected: "HELM Lite", why: "dash -> _" }, { input: "helm.lite", expected: "HELM Lite", why: "dot -> _" }, { input: "HELM-LITE", expected: "HELM Lite", why: "lower + dash -> _" }, { input: " helm lite ", expected: "HELM Lite", why: "whitespace runs collapse, edges trim" }, { input: "ARC.AGI", expected: "ARC-AGI", why: "dot -> _" }, { input: "Reward-Bench-2", expected: "RewardBench 2" }, ] it.each(cases)("'$input' -> '$expected'", ({ input, expected }) => { expect(getBenchmarkDisplayName(input)).toBe(expected) }) }) // --------------------------------------------------------------------------- // Group B — Tokenize fallback (humanizeToken on the *original* input) // --------------------------------------------------------------------------- describe("Group B — humanizeToken fallback for non-map inputs", () => { const cases = [ // Single-token acronyms — only first char gets uppercased (NOT a real acronym map) { input: "bbh", expected: "Bbh", why: "humanizeToken only uppercases first char of each token; map doesn't have 'bbh'" }, { input: "gpqa", expected: "Gpqa", why: "same — visibly wrong but TS-as-spec" }, { input: "mmlu", expected: "Mmlu", why: "same — visibly wrong; the suite-name companion table in benchmark-detail.tsx fixes this, but the active getBenchmarkDisplayName does NOT" }, { input: "gsm8k", expected: "Gsm8k", why: "digits inside don't capitalize differently" }, { input: "humaneval", expected: "Humaneval" }, { input: "truthfulqa", expected: "Truthfulqa" }, // Already-uppercase passthrough (charAt(0).toUpperCase() is a no-op on already-upper char) { input: "MATH", expected: "MATH", why: "M is already upper; ATH preserved by slice(1)" }, { input: "MMLU", expected: "MMLU", why: "M upper; MLU preserved" }, { input: "BBQ", expected: "BBQ" }, { input: "MMLU-PRO", expected: "MMLU PRO", why: "split on - -> ['MMLU','PRO'] -> first-char-upper (no-op) -> join with space" }, // Multi-token snake/dash inputs that miss the map { input: "swe-bench-verified", expected: "Swe Bench Verified", why: "split on -, each first-cap" }, { input: "swe_bench_verified_mini", expected: "Swe Bench Verified Mini" }, { input: "multi_swe_bench", expected: "Multi Swe Bench" }, { input: "helm_air_bench", expected: "Helm Air Bench", why: "not in map (only the ~30 listed suite keys are)" }, { input: "helm_safety", expected: "Helm Safety" }, { input: "swe_bench_verified", expected: "Swe Bench Verified", why: "swe_bench is in map but swe_bench_verified is not" }, { input: "cocoabench", expected: "Cocoabench" }, { input: "llm_stats", expected: "Llm Stats" }, { input: "artificial_analysis_llms", expected: "Artificial Analysis Llms" }, // The fallback uses the *original* (unnormalized) input — spaces survive! { input: "Helm air bench", expected: "Helm air bench", why: "fallback splits on [_-]+ ONLY; the spaces don't trigger split; first char of the lone token already upper" }, { input: "helm air bench", expected: "Helm air bench", why: "single token (spaces don't split); lowercase 'h' becomes 'H', rest unchanged" }, ] it.each(cases)("'$input' -> '$expected' ($why)", ({ input, expected }) => { expect(getBenchmarkDisplayName(input)).toBe(expected) }) }) // --------------------------------------------------------------------------- // Group C — Edge cases // --------------------------------------------------------------------------- describe("Group C — edge cases", () => { it("empty string -> empty string", () => { // normalize -> "" (no map hit). humanizeToken: "".split(/[_-]+/) -> [""] -> filter(Boolean) -> [] -> [].join(" ") -> "" expect(getBenchmarkDisplayName("")).toBe("") }) it("single underscore -> empty string", () => { // normalize: "_" -> "" (edges stripped). no map hit. humanizeToken: "_".split(/[_-]+/) -> ["",""] -> filter -> [] -> "" expect(getBenchmarkDisplayName("_")).toBe("") }) it("triple-underscore-padded map key collapses to map hit during normalize", () => { // normalize: "___helm___lite___" -> lower (no-op) -> internal runs of _ stay (but [-.\s]+ doesn't include _!) -> wait // Let's check carefully: normalizeBenchmarkKeyForLookup uses /[-.\s]+/g (NOT _). // So "___helm___lite___".replace(/[-.\s]+/g, "_") is unchanged. // Then .replace(/^_+|_+$/g, "") strips edge _ runs. Internal "___" stays as-is. // Result: "helm___lite" — NOT "helm_lite". So this misses the map! expect(getBenchmarkDisplayName("___helm___lite___")).toBe("Helm Lite") // humanizeToken splits on [_-]+ which collapses the runs: "helm___lite".split(/[_-]+/) -> ["helm","lite"] -> ["Helm","Lite"] }) it("single char -> uppercased single char via fallback", () => { expect(getBenchmarkDisplayName("a")).toBe("A") }) it("a-b -> 'A B' via fallback", () => { expect(getBenchmarkDisplayName("a-b")).toBe("A B") }) it("lookup is case-insensitive even for substantive transforms", () => { expect(getBenchmarkDisplayName("APEX_AGENTS")).toBe("APEX Agents") expect(getBenchmarkDisplayName("Browsecompplus")).toBe("BrowseComp+") expect(getBenchmarkDisplayName("LIVECODEBENCHPRO")).toBe("LiveCodeBench Pro") }) it("two-space whitespace collapses for normalize (map lookup)", () => { expect(getBenchmarkDisplayName("helm lite")).toBe("HELM Lite") }) }) // --------------------------------------------------------------------------- // Group D — Duplicate getBenchmarkDisplayName in lib/eval-processing.ts // (functionally dead — only called by groupEvaluationsByBenchmark which has // no importers. Tested for completeness so the divergence in semantics is // explicit.) // --------------------------------------------------------------------------- function getBenchmarkDisplayNameDuplicate(name: string | undefined | null): string { if (!name) return "Unknown Benchmark" const mapping: Record = { MMLU: "Massive Multitask Language Understanding", "MMLU-Pro": "MMLU Professional", GSM8K: "Grade School Math 8K", HumanEval: "Human Eval (Code)", MBPP: "Mostly Basic Python Problems", HellaSwag: "HellaSwag (Commonsense)", ARC: "AI2 Reasoning Challenge", TruthfulQA: "TruthfulQA", BBH: "Big-Bench Hard", MATH: "MATH Dataset", } for (const [key, value] of Object.entries(mapping)) { if (name.toUpperCase().includes(key.toUpperCase())) { return value } } return name } describe("Group D — duplicate getBenchmarkDisplayName (eval-processing.ts) — substring-include rule", () => { const cases: Array<{ input: string | null | undefined; expected: string; why?: string }> = [ { input: null, expected: "Unknown Benchmark", why: "guard: !name" }, { input: undefined, expected: "Unknown Benchmark", why: "guard" }, { input: "", expected: "Unknown Benchmark", why: "guard (empty string is falsy)" }, { input: "MMLU", expected: "Massive Multitask Language Understanding", why: "substring match on MMLU" }, { input: "mmlu", expected: "Massive Multitask Language Understanding", why: "case-insensitive (toUpperCase)" }, { input: "MMLU-Pro", expected: "Massive Multitask Language Understanding", why: "iteration order: MMLU is matched first (insertion order); MMLU-Pro entry never reached. KNOWN SOFT-BUG, document don't fix.", }, { input: "GSM8K", expected: "Grade School Math 8K" }, { input: "HumanEval", expected: "Human Eval (Code)" }, { input: "MBPP", expected: "Mostly Basic Python Problems" }, { input: "HellaSwag", expected: "HellaSwag (Commonsense)" }, { input: "ARC", expected: "AI2 Reasoning Challenge" }, { input: "TruthfulQA", expected: "TruthfulQA", why: "key === value" }, { input: "BBH", expected: "Big-Bench Hard" }, { input: "MATH", expected: "MATH Dataset" }, { input: "helm_lite", expected: "helm_lite", why: "no substring match -> passthrough" }, { input: "MMLU Lite something", expected: "Massive Multitask Language Understanding", why: "substring match still fires when the key appears anywhere in the input", }, { input: "ARC-AGI", expected: "AI2 Reasoning Challenge", why: "substring 'ARC' matches; this overwrites the more specific intent of 'ARC-AGI' — soft-bug", }, ] it.each(cases)("'$input' -> '$expected' ($why)", ({ input, expected }) => { expect(getBenchmarkDisplayNameDuplicate(input)).toBe(expected) }) })