"use client" // Force recompile import Link from "next/link" import { usePathname, useSearchParams } from "next/navigation" import { useAudienceMode } from "@/components/audience-mode-provider" import { formatDateISO, humanizeEvaluationId } from "@/lib/utils" import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card" import { Badge } from "@/components/ui/badge" import { Button } from "@/components/ui/button" import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select" import { Separator } from "@/components/ui/separator" import { Tabs, TabsList, TabsTrigger } from "@/components/ui/tabs" import { Progress } from "@/components/ui/progress" import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "@/components/ui/collapsible" import { Dialog, DialogContent, DialogDescription, DialogHeader, DialogTitle } from "@/components/ui/dialog" import { Input } from "@/components/ui/input" import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table" import { getRelationshipBadgeTone, getRelationshipDisplayName, getRelationshipShortLabel, } from "@/components/signals/provenance-badge" import { SignalsRowBadges } from "@/components/signals/signals-row-badges" import { SignalTooltip } from "@/components/signals/signal-tooltip" import { DropdownMenu, DropdownMenuContent, DropdownMenuItem, DropdownMenuLabel, DropdownMenuSeparator, DropdownMenuTrigger, } from "@/components/ui/dropdown-menu" import { ExternalLink, TrendingUp, Info, Database, Settings, FileCode, Building, Calendar, User, Server, ChevronDown, ChevronUp, BarChart3, Award, AlertTriangle, ArrowUpRight, Cpu, Tag, Globe, Network, Activity, MessageSquare, Clock, Hash, Layers, Search, FlaskConical, Scale, BookOpenText, Plus, X, List, LayoutGrid } from "lucide-react" import type { BenchmarkCard, BenchmarkEvaluation, CategoryType, EvaluationResult } from "@/lib/benchmark-schema" import { getCategoryColor as getCategoryTone, inferCategoryFromBenchmark } from "@/lib/benchmark-schema" import { formatTagLabel } from "@/lib/benchmark-tags" import type { BenchmarkEvalSummary } from "@/lib/eval-processing" import type { ModelSummaryCore } from "@/lib/benchmark-schema" import { lookupBenchmarkCard } from "@/lib/benchmark-metadata-utils" import type { BenchmarkEvaluationCardData } from "@/components/benchmark-evaluation-card" import type { ComparisonEvalEntry, ComparisonIndex, ComparisonMetricEntry, ComparisonScoreEntry, EvalHierarchy, PeerRanksMap, SubmissionAxis, } from "@/lib/backend-artifacts" import { fetchPeerRanks } from "@/lib/dashboard-data-client" import { buildHierarchyEvalIndex, type HierarchyEvalLocation, } from "@/lib/hierarchy-lookup" import { type CSSProperties, Fragment, useState, useEffect, useMemo } from "react" interface BenchmarkDetailProps { summary: ModelSummaryCore benchmarkCards?: Record modelCards?: BenchmarkEvaluationCardData[] evalHierarchy?: EvalHierarchy | null comparisonIndex?: ComparisonIndex | null } interface BenchmarkVariant { evaluation: BenchmarkEvaluation result: EvaluationResult label: string variantType: "setup" | "slice" | "setup+slice" | "default" metricLabel: string setupLabel: string | null sliceLabel: string | null displayScore: string normalizedScore: number rankPosition: number | null rankTotal: number | null rankRatio: number | null /** Companion sampling-error metric (e.g. `prompt_strict_stderr` next to * `prompt_strict_acc`). Filtered out of primary listings — the value is * surfaced only inside the deep-dive row's score cell. */ auxStderr?: number auxStderrUnit?: string } interface BenchmarkGroup { key: string title: string canonicalTitle: string evalDetailHref: string category: CategoryType description: string scoreType: EvaluationResult["metric_config"]["score_type"] | "mixed" avgRawScore: number avgNormalizedScore: number avgDisplayScore: string bestRankPosition: number | null bestRankTotal: number | null bestRankRatio: number | null domains: string[] benchmarkCard?: BenchmarkCard variants: BenchmarkVariant[] } interface CompositeGroup { compositeKey: string compositeName: string benchmarks: BenchmarkGroup[] avgRawScore: number avgNormalizedScore: number avgDisplayScore: string bestRank: { position: number; total: number } | null } const INSTANCE_PREVIEW_LIMIT = 5 // SUITE_DISPLAY_NAMES (38-entry hardcoded slug→display map) was deleted // in Step 4c of the hierarchy-alignment work // (notes/hierarchy-alignment.md §6 / §7 Step 4). The producer now ships // curated display names for every family / composite / benchmark via // prettify_display + the registry's display_overrides.yaml. This // component reads the shipped name; the DISPLAY_TOKEN_OVERRIDES / // DISPLAY_NAME_OVERRIDES below remain as a per-token polish layer // (mostly for raw model identifier rendering, where the producer's // metadata doesn't carry a curated display). const DISPLAY_TOKEN_OVERRIDES: Record = { ace: "ACE", apex: "APEX", api: "API", ai: "AI", ai2: "AI2", bbh: "BBH", diy: "DIY", gpt: "GPT", gpqa: "GPQA", helm: "HELM", hf: "HF", ibm: "IBM", ifeval: "IFEval", la: "LA", llm: "LLM", math: "MATH", md: "MD", mmlu: "MMLU", musr: "MUSR", oecd: "OECD", nist: "NIST", openai: "OpenAI", swe: "SWE", tau: "TAU", ui: "UI", ux: "UX", xai: "xAI", } const DISPLAY_NAME_OVERRIDES: Record = { apex: "APEX", apex_agents: "APEX Agents", apex_v1: "APEX v1", openai: "OpenAI", xai: "xAI", nvidia: "NVIDIA", ibm: "IBM", } const AMBIGUOUS_GROUP_LABELS = new Set(["overall", "score", "accuracy"]) // Treat these as "no real name" when deciding whether to fall back to // metric_summary_id derivation. Some sources (e.g. ifeval%2Fifeval, // hfopenllm-v2 raw metrics) ship metric_name="" or a placeholder like // "score" / "metric" while the meaningful name lives in the local // segment of the metric_summary_id (`ifeval%3Aprompt_strict_acc`). const GENERIC_METRIC_LABELS = new Set([ "", "metric", "score", "accuracy", "value", "result", ]) // Sampling-error companion metrics travel alongside score metrics in some // snapshots (`ifeval%3Aprompt_strict_stderr` next to `…_acc`). Surfacing // them as their own rows / tabs makes IFEval-style benchmarks look like // they ship 10 indistinguishable splits, so the renderer hides them from // primary lists and folds the value into the score cell of the matching // row in the deep dive (see BenchmarkVariant.auxStderr). const STDERR_SUFFIX_PATTERN = /_(stderr|std_err|standard_error)$/i const SCORE_SUFFIX_PATTERN = /_(acc|accuracy|score|value|result)$/i function isStderrMetricId(id: string | null | undefined): boolean { if (!id) return false const local = id.split("%3A").pop() ?? id return STDERR_SUFFIX_PATTERN.test(local) } /** * Strip the trailing score / stderr suffix so a metric and its companion * stderr collapse onto the same key. Both `ifeval%3Aprompt_strict_acc` * and `ifeval%3Aprompt_strict_stderr` map to `ifeval%3Aprompt_strict`. */ function metricPairKey(id: string | null | undefined): string | null { if (!id) return null const trimmed = id.replace(STDERR_SUFFIX_PATTERN, "").replace(SCORE_SUFFIX_PATTERN, "") return trimmed.length > 0 ? trimmed : id } /** * Pick the most informative metric label available given the upstream * `metric_name` (which may be empty or generic) and `metric_summary_id` * (whose local part — e.g. `prompt_strict_acc` from * `ifeval%3Aprompt_strict_acc` — is the only carrier of identity for * sources that don't populate metric_name). */ function deriveMetricTabLabel( metricName: string | null | undefined, metricSummaryId: string | null | undefined, ): string { const trimmed = metricName?.trim() ?? "" if (trimmed && !GENERIC_METRIC_LABELS.has(trimmed.toLowerCase())) { return trimmed } const id = metricSummaryId ?? "" if (id) { const local = id.split("%3A").pop() ?? id if (local && !GENERIC_METRIC_LABELS.has(local.toLowerCase())) { return normalizeDisplayLabel(local) } } return trimmed || "Score" } function normalizeDisplayKey(value: string) { return value.toLowerCase().replace(/[^a-z0-9]+/g, "_").replace(/^_+|_+$/g, "") } function normalizeDisplayToken(token: string) { const prefixMatch = token.match(/^[^a-z0-9]*/i) const suffixMatch = token.match(/[^a-z0-9]*$/i) const prefix = prefixMatch?.[0] ?? "" const suffix = suffixMatch?.[0] ?? "" const core = token.slice(prefix.length, token.length - suffix.length) if (!core) { return token } const override = DISPLAY_TOKEN_OVERRIDES[normalizeDisplayKey(core)] if (override) { return `${prefix}${override}${suffix}` } if (/[A-Z]/.test(core.slice(1))) { return `${prefix}${core}${suffix}` } if (/^\d/.test(core)) { return `${prefix}${core}${suffix}` } return `${prefix}${core.charAt(0).toUpperCase()}${core.slice(1).toLowerCase()}${suffix}` } function normalizeDisplayLabel(value: string | null | undefined): string { if (!value) { return "" } const normalizedKey = normalizeDisplayKey(value) const override = DISPLAY_NAME_OVERRIDES[normalizedKey] if (override) { return override } return value .split("/") .map((segment) => { const cleaned = segment.replace(/[_-]+/g, " ").replace(/\s+/g, " ").trim() if (!cleaned) { return "" } const cleanedOverride = DISPLAY_NAME_OVERRIDES[normalizeDisplayKey(cleaned)] if (cleanedOverride) { return cleanedOverride } return cleaned.split(" ").map(normalizeDisplayToken).join(" ") }) .filter(Boolean) .join(" / ") } function formatRawScoreValue(score: number, unit?: string) { if (!Number.isFinite(score)) { return "N/A" } const precision = Math.abs(score) >= 100 ? 1 : Math.abs(score) >= 10 ? 2 : 3 const value = score.toFixed(precision).replace(/0+$/g, "").replace(/\.$/, "") const normalizedUnit = normalizeDisplayLabel(unit) if (!normalizedUnit || normalizedUnit === "Accuracy" || normalizedUnit === "Pass@1" || normalizedUnit === "Score") { return value } return `${value} ${normalizedUnit}` } function getModelDisplayName(value: string | null | undefined) { return normalizeDisplayLabel(value) || "Unknown Model" } function getOrganizationDisplayName(value: string | null | undefined) { return normalizeDisplayLabel(value) || "Unknown Organization" } function getSourceTypeDisplayName(value: string | null | undefined) { return normalizeDisplayLabel(value?.replace(/_/g, " ")) || "Unknown" } function formatEvalLibrary(library: { name: string; version?: string }) { const version = library.version?.trim() return version && version.toLowerCase() !== "unknown" ? `${library.name} ${version}` : library.name } function normalizeCompositeKey(key: string): string { const k = key.toLowerCase().replace(/[-.\s]+/g, "_").replace(/^_+|_+$/g, "") if (/^fibble\d*_arena$/.test(k)) return "fibble_arena" if (/^arc_agi_v\d+/.test(k)) return "arc_agi" return k } function doesLabelMatchSuiteKey(label: string | null | undefined, compositeKey: string) { if (!label) { return false } return normalizeCompositeKey(normalizeDisplayKey(label)) === normalizeCompositeKey(compositeKey) } function getHierarchyLocation( group: BenchmarkGroup, hierarchyIndex: Map | null, ): HierarchyEvalLocation | undefined { if (!hierarchyIndex) { return undefined } for (const variant of group.variants) { const evalSummaryId = variant.evaluation.eval_summary_id if (evalSummaryId) { const location = hierarchyIndex.get(evalSummaryId) if (location) { return location } } } return undefined } function getCompositeKey( group: BenchmarkGroup, hierarchyIndex: Map | null, ): string { // Prefer the curated grouping from hierarchy.json. The eval row's own // family_id is null for ~7% of evals (e.g. CySE2 composites) and points // at the leaf for singleton families, so the hierarchy is the only // source that captures family→composite groupings authoritatively. const location = getHierarchyLocation(group, hierarchyIndex) if (location) { return normalizeCompositeKey(location.familyKey) } const evaluation = group.variants[0]?.evaluation const backendSuiteKey = evaluation?.family_id return normalizeCompositeKey(backendSuiteKey ?? group.key) } function getCompositeDisplayName(key: string): string { // Display names come from the shipped hierarchy.json. This helper // is used in fallback paths where only a raw key is in scope; it // applies the same per-token polish (DISPLAY_TOKEN_OVERRIDES) the // rest of the renderer uses. return normalizeDisplayLabel(key) } function getCompositeName( group: BenchmarkGroup, compositeKey: string, hierarchyIndex: Map | null, ): string { const location = getHierarchyLocation(group, hierarchyIndex) if (location?.familyDisplayName) { return location.familyDisplayName } const evaluation = group.variants[0]?.evaluation const benchmarkCardName = group.benchmarkCard?.benchmark_details?.name const backendParentName = evaluation?.benchmark_parent_name const backendFamilyName = evaluation?.benchmark_family_name if (doesLabelMatchSuiteKey(backendParentName, compositeKey)) { return normalizeDisplayLabel(backendParentName) } if (doesLabelMatchSuiteKey(backendFamilyName, compositeKey)) { return normalizeDisplayLabel(backendFamilyName) } if (doesLabelMatchSuiteKey(benchmarkCardName, compositeKey)) { return normalizeDisplayLabel(benchmarkCardName) } return getCompositeDisplayName(compositeKey) } function groupByComposite( groups: BenchmarkGroup[], modelIds: string[], peerRanks: PeerRanksMap, hierarchyIndex: Map | null ): CompositeGroup[] { const composites = new Map() for (const group of groups) { const key = getCompositeKey(group, hierarchyIndex) const existing = composites.get(key) ?? [] existing.push(group) composites.set(key, existing) } return Array.from(composites.entries()).map(([compositeKey, benchmarks]) => { const scores = benchmarks.map(b => b.avgNormalizedScore).filter(Number.isFinite) const avgScore = scores.length > 0 ? scores.reduce((a, b) => a + b, 0) / scores.length : 0 const rawScores = benchmarks.map((benchmark) => benchmark.avgRawScore).filter(Number.isFinite) const avgRawScore = rawScores.length > 0 ? rawScores.reduce((a, b) => a + b, 0) / rawScores.length : 0 // Find best rank across all benchmarks in the composite let bestRank: { position: number; total: number } | null = null for (const b of benchmarks) { const rank = getGroupPeerRank(b, modelIds, peerRanks) if (!rank) continue if (!bestRank || (rank.position / rank.total) < (bestRank.position / bestRank.total)) { bestRank = rank } } return { compositeKey, compositeName: benchmarks[0] ? getCompositeName(benchmarks[0], compositeKey, hierarchyIndex) : getCompositeDisplayName(compositeKey), benchmarks, avgRawScore, avgNormalizedScore: avgScore, avgDisplayScore: formatRawScoreValue(avgRawScore), bestRank, } }).sort((a, b) => { // Sort by best peer rank ratio (lower = better); unranked composites go to the bottom const aRatio = a.bestRank ? a.bestRank.position / (a.bestRank.total || a.bestRank.position) : Infinity const bRatio = b.bestRank ? b.bestRank.position / (b.bestRank.total || b.bestRank.position) : Infinity if (aRatio !== bRatio) return aRatio - bRatio return b.avgNormalizedScore - a.avgNormalizedScore }) } interface VariantRowData { rowKey: string variant: BenchmarkVariant configMap: Record configEntries: Array<[string, string]> sampleCount: number | null } interface DeepDiveVariantRow { rowKey: string variant: BenchmarkVariant evalSummaryId: string configMap: Record configEntries: Array<[string, string]> } const GENERIC_RESULT_NAMES = new Set([ "score", "accuracy", "mean win rate", "exact match", "f1", "pass@1", ]) function getResultBenchmarkName( evaluation: BenchmarkEvaluation, result: EvaluationResult ) { if (evaluation.display_name) { return evaluation.display_name } if (evaluation.slice_name) { return evaluation.slice_name } if (evaluation.benchmark_leaf_name) { return evaluation.benchmark_leaf_name } if (evaluation.benchmark_parent_name) { return evaluation.benchmark_parent_name } if (evaluation.benchmark) { return evaluation.benchmark } if (result.display_name) { return result.display_name } return result.evaluation_name } function getResultDisplayName( evaluation: BenchmarkEvaluation, result: EvaluationResult ) { if (result.canonical_display_name) { return result.canonical_display_name } if (evaluation.canonical_display_name) { return evaluation.canonical_display_name } const benchmarkName = evaluation.benchmark_parent_name || evaluation.benchmark || getResultBenchmarkName(evaluation, result) const metricName = result.display_name || result.evaluation_name if (GENERIC_RESULT_NAMES.has(metricName.toLowerCase())) { return `${benchmarkName} - ${metricName}` } return metricName } function getMetricDisplayLabel(result: EvaluationResult) { const candidates = [ result.display_name, result.canonical_display_name, result.evaluation_name, ] let firstNonEmpty = "" for (const candidate of candidates) { const value = candidate?.trim() if (!value) continue const segments = value .split("/") .map((segment) => segment.trim()) .filter(Boolean) const leaf = segments[segments.length - 1] ?? value const normalised = normalizeDisplayLabel(leaf) if (!firstNonEmpty) firstNonEmpty = normalised if ( normalised && !GENERIC_METRIC_LABELS.has(normalised.toLowerCase()) ) { return normalised } } // Last-ditch: derive from metric_summary_id local part. Some upstream // sources (e.g. ifeval%2Fifeval's 10 prompt/inst/strict/loose metrics) // ship empty or generic display fields, so the only meaningful // identity is the local segment of the summary id. const summaryId = result.metric_summary_id ?? "" if (summaryId) { const local = summaryId.split("%3A").pop() ?? summaryId if (local && !GENERIC_METRIC_LABELS.has(local.toLowerCase())) { return normalizeDisplayLabel(local) } } return firstNonEmpty || "Metric" } function getVariantDescriptor( evaluation: BenchmarkEvaluation, result: EvaluationResult ): Pick { const evaluationVariantRaw = getEvaluationVariantLabel(evaluation) const evaluationVariant = evaluationVariantRaw ? formatSetupDisplayLabel(evaluationVariantRaw) : null const metricLabel = getMetricDisplayLabel(result) const metricKey = normalizeDisplayKey(metricLabel) const metricIsAmbiguous = AMBIGUOUS_GROUP_LABELS.has(metricKey) const sliceLabel = evaluation.slice_name ? normalizeDisplayLabel(evaluation.slice_name) : null const setupLabel = evaluationVariant ? formatSetupDisplayLabel(evaluationVariant) : null const baseLabel = sliceLabel ? (metricIsAmbiguous ? sliceLabel : `${sliceLabel} · ${metricLabel}`) : metricLabel if (setupLabel && sliceLabel) { return { label: `${setupLabel} · ${baseLabel}`, variantType: "setup+slice", metricLabel, setupLabel, sliceLabel, } } if (setupLabel) { return { label: metricIsAmbiguous ? `Setup: ${setupLabel}` : `${setupLabel} · ${metricLabel}`, variantType: "setup", metricLabel, setupLabel, sliceLabel: null, } } if (sliceLabel || !metricIsAmbiguous) { return { label: baseLabel, variantType: sliceLabel ? "slice" : "default", metricLabel, setupLabel: null, sliceLabel: sliceLabel ?? null, } } return { label: metricLabel, variantType: "default", metricLabel, setupLabel: null, sliceLabel: null, } } function formatMetadataValue(value: unknown) { if (value == null) { return null } if (typeof value === "string") { return value } if ( typeof value === "number" || typeof value === "boolean" || typeof value === "bigint" ) { return String(value) } try { return JSON.stringify(value, null, 2) } catch { return String(value) } } function isPlainObject(value: unknown): value is Record { return typeof value === "object" && value !== null && !Array.isArray(value) } function collectConfigEntries( source: Record, prefix = "", depth = 0 ): Array<[string, string]> { const entries: Array<[string, string]> = [] for (const [key, value] of Object.entries(source)) { const nextKey = prefix ? `${prefix}.${key}` : key if (isPlainObject(value) && depth < 1) { entries.push(...collectConfigEntries(value, nextKey, depth + 1)) continue } const formattedValue = formatMetadataValue(value) if (formattedValue) { entries.push([nextKey, formattedValue]) } } return entries } function unquoteJsonString(value: string): string { // Values like "\"true\"" or "\"2048\"" are JSON-encoded strings — unwrap them const trimmed = value.trim() if (trimmed.startsWith('"') && trimmed.endsWith('"')) { try { const parsed = JSON.parse(trimmed) if (typeof parsed === "string") return parsed } catch { /* fall through */ } } return value } function getConfigDisplayValue(value: string) { const unquoted = unquoteJsonString(value) return unquoted.length > 40 ? `${unquoted.slice(0, 37)}…` : unquoted } /** Parse a HELM-style nested detail entry like '{"tab":"Efficiency","score":"106.9"}' */ function parseHelmDetailEntry(value: unknown): { tab?: string; score?: string; description?: string } | null { if (typeof value !== "string") return null try { const parsed = JSON.parse(value) if (parsed && typeof parsed === "object" && ("score" in parsed || "tab" in parsed)) { return parsed as { tab?: string; score?: string; description?: string } } } catch { /* not JSON */ } return null } function getTableConfigLabel(row: VariantRowData) { if (row.variant.setupLabel) { return row.variant.setupLabel } if (row.variant.variantType === "slice") { return "Default setup" } return "Default config" } function getComparisonScoreEntryForVariant( row: DeepDiveVariantRow, comparisonIndex?: ComparisonIndex | null ) { if (!comparisonIndex || !row.evalSummaryId) { return null } const metricSummaryId = row.variant.result.metric_summary_id if (!metricSummaryId) { return null } const metricEntry = comparisonIndex.evals[row.evalSummaryId]?.metrics.find( (metric) => metric.metric_summary_id === metricSummaryId ) if (!metricEntry) { return null } const modelId = row.variant.evaluation.model_info.id return ( metricEntry.scores.find( (score) => score.model_route_id === modelId || score.model_family_id === modelId ) ?? null ) } function getVariantRunLabels( row: DeepDiveVariantRow, comparisonIndex?: ComparisonIndex | null ) { const comparisonScoreEntry = getComparisonScoreEntryForVariant(row, comparisonIndex) const submissions = comparisonScoreEntry?.submissions ?? [] const targetScore = row.variant.result.score_details.score const matchingSubmissions = submissions.filter( (submission) => Math.abs(submission.score - targetScore) <= 1e-6 ) const candidateSubmissions = matchingSubmissions.length > 0 ? matchingSubmissions : submissions.length === 1 ? submissions : [] const runLabels = Array.from( new Set( candidateSubmissions .map((submission) => normalizeDisplayLabel(submission.run_label)) .filter(Boolean) ) ) if (runLabels.length > 0) { return runLabels } const headlineRunLabel = normalizeDisplayLabel(comparisonScoreEntry?.headline_run_label) return headlineRunLabel ? [headlineRunLabel] : [] } function getVariantConfigDisambiguation( row: DeepDiveVariantRow, similarRows: DeepDiveVariantRow[] ) { const differingKeys = Array.from( new Set( similarRows.flatMap((candidate) => Object.keys(candidate.configMap).filter((key) => key.toLowerCase() !== "setup") ) ) ) .filter((key) => { const values = new Set(similarRows.map((candidate) => candidate.configMap[key]).filter(Boolean)) return values.size > 1 }) .sort((a, b) => a.localeCompare(b)) return differingKeys .map((key) => [key, row.configMap[key]] as const) .filter((entry): entry is readonly [string, string] => Boolean(entry[1])) .slice(0, 2) .map(([key, value]) => `${formatConfigLabel(key)}=${getConfigDisplayValue(value)}`) } // Compact date formatter — alias of the shared YYYY-MM-DD formatter so // every "Updated" / "Released" / per-row date in this component reads // consistently. Kept as a separate name for the existing call sites. const formatCompactDate = formatDateISO function formatParamsBillions(value: unknown) { const numericValue = typeof value === "number" ? value : typeof value === "string" ? Number.parseFloat(value) : Number.NaN if (!Number.isFinite(numericValue)) { return null } if (numericValue >= 100) { return `${Math.round(numericValue)}B` } if (numericValue >= 10) { return `${numericValue.toFixed(1)}B` } return `${numericValue.toFixed(1)}B` } function getModelScaleDescription(value: unknown) { const numericValue = typeof value === "number" ? value : typeof value === "string" ? Number.parseFloat(value) : Number.NaN if (!Number.isFinite(numericValue)) { return null } const rounded = numericValue >= 100 ? Math.round(numericValue) : Number.parseFloat(numericValue.toFixed(1)) const scaleLabel = numericValue < 10 ? "Small model" : numericValue < 70 ? "Mid-size model" : "Large model" return `${scaleLabel} (${rounded} billion parameters)` } function getPolicyBenchmarkNarrative(name: string) { const value = name.toLowerCase() if (value.includes("ifeval")) { return { label: "Following instructions", description: "Can the model follow detailed formatting and content rules?", } } if (value.includes("bbh")) { return { label: "Reasoning and logic", description: "Multi-step reasoning across diverse tasks.", } } if (value.includes("math")) { return { label: "Advanced math", description: "Hard competition-level mathematics.", } } if (value.includes("gpqa")) { return { label: "Expert knowledge", description: "Graduate-level science questions across biology, physics, and chemistry.", } } if (value.includes("musr")) { return { label: "Complex narrative reasoning", description: "Reasoning over stories and real-world scenarios.", } } if (value.includes("mmlu")) { return { label: "Broad knowledge", description: "Professional and academic knowledge across many subject areas.", } } if (value.includes("tau-bench")) { return { label: "Agentic task completion", description: "Multi-step task execution in realistic workflow settings.", } } if (value.includes("swe-bench")) { return { label: "Software engineering", description: "Issue resolution and code-change performance on real repositories.", } } if (value.includes("rewardbench")) { return { label: "Preference alignment", description: "How well the model matches preference-style judgments.", } } return { label: name, description: "Reported benchmark evidence for this model.", } } function getPolicySignalLevel(score: number) { if (score >= 0.7) { return { label: "Good", tone: "bg-emerald-100 text-emerald-800 dark:bg-emerald-950/50 dark:text-emerald-300", } } if (score >= 0.4) { return { label: "Moderate", tone: "bg-amber-100 text-amber-800 dark:bg-amber-950/50 dark:text-amber-300", } } return { label: "Low", tone: "bg-rose-100 text-rose-800 dark:bg-rose-950/50 dark:text-rose-300", } } function getBenchmarkSpread(group: BenchmarkGroup) { if (group.variants.length <= 1) { return 0 } return group.variants[0].normalizedScore - group.variants[group.variants.length - 1].normalizedScore } function getBenchmarkSourceCount(group: BenchmarkGroup) { return new Set(group.variants.map((variant) => getOrganizationDisplayName(variant.evaluation.source_metadata.source_organization_name))).size } function getVariantTypeTone(variantType: BenchmarkVariant["variantType"]) { switch (variantType) { case "setup": return "bg-sky-100 text-sky-800 dark:bg-sky-950/50 dark:text-sky-300" case "slice": return "bg-violet-100 text-violet-800 dark:bg-violet-950/50 dark:text-violet-300" case "setup+slice": return "bg-amber-100 text-amber-800 dark:bg-amber-950/50 dark:text-amber-300" default: return "bg-muted text-muted-foreground" } } function getVariantTypeLabel(variantType: BenchmarkVariant["variantType"]) { switch (variantType) { case "setup": return "Setup change" case "slice": return "Benchmark slice" case "setup+slice": return "Setup + slice" default: return "Single run" } } function formatSetupDisplayLabel(setupLabel: string | null) { if (!setupLabel) { return "Default setup" } const normalized = setupLabel.trim() if (!normalized || normalized.toLowerCase() === "default" || normalized.endsWith("__default")) { return "Default setup" } const cleaned = normalized .replace(/^setup[:=]\s*/i, "") .replace(/[_-]+/g, " ") .replace(/\s+/g, " ") .trim() if (!cleaned) { return "Default setup" } return normalizeDisplayLabel(cleaned) } function getVariantPrimaryLabel(variant: BenchmarkVariant, groupTitle: string) { if (variant.sliceLabel) { return variant.sliceLabel } if (variant.metricLabel) { const normalizedMetricKey = normalizeDisplayKey(variant.metricLabel) if (!AMBIGUOUS_GROUP_LABELS.has(normalizedMetricKey)) { return variant.metricLabel } } if (variant.variantType === "default" || variant.variantType === "setup") { return groupTitle } return variant.label } function getGroupSliceLabels(group: BenchmarkGroup) { return Array.from( new Set( group.variants .map((variant) => variant.sliceLabel?.trim()) .filter((label): label is string => Boolean(label)) ) ) } function getGroupSliceCount(group: BenchmarkGroup) { return getGroupSliceLabels(group).length } function getBenchmarkGroupHeading(group: BenchmarkGroup) { return group.canonicalTitle } function getCompositeBadgeMeta(composite: CompositeGroup) { if (composite.benchmarks.length > 1) { return { count: composite.benchmarks.length, label: `sub-benchmark${composite.benchmarks.length === 1 ? "" : "s"}`, className: "border-sky-200/80 bg-sky-50/70 text-sky-700 dark:border-sky-900/60 dark:bg-sky-950/30 dark:text-sky-300", } } const singleGroup = composite.benchmarks[0] if (!singleGroup) { return null } const compositeMatchesBenchmark = normalizeCompositeKey(composite.compositeName) === normalizeCompositeKey(singleGroup.title) if (!compositeMatchesBenchmark) { return { count: 1, label: "sub-benchmark", className: "border-sky-200/80 bg-sky-50/70 text-sky-700 dark:border-sky-900/60 dark:bg-sky-950/30 dark:text-sky-300", } } const sliceCount = getGroupSliceCount(singleGroup) if (sliceCount > 0) { return { count: sliceCount, label: `slice${sliceCount === 1 ? "" : "s"}`, className: "border-emerald-200/70 bg-emerald-50 text-emerald-700 dark:border-emerald-900/60 dark:bg-emerald-950/30 dark:text-emerald-300", } } return null } interface ScoreRange { min: number max: number } const DEFAULT_SCORE_RANGE: ScoreRange = { min: 0, max: 1 } function getScoreRange(values: number[]): ScoreRange { const finiteValues = values.filter((value) => Number.isFinite(value)) if (finiteValues.length === 0) { return DEFAULT_SCORE_RANGE } return { min: Math.min(...finiteValues), max: Math.max(...finiteValues), } } function normalizeWithinRange(value: number, range: ScoreRange): number { if (!Number.isFinite(value)) { return 0.5 } const span = range.max - range.min if (span <= 0) { return 0.5 } return Math.max(0, Math.min(1, (value - range.min) / span)) } function formatNormalizedPercent(value: number) { if (!Number.isFinite(value)) { return "N/A" } return `${(value * 100).toFixed(1)}%` } function isRangeEdge(value: number, range: ScoreRange, edge: "min" | "max") { const span = range.max - range.min if (!Number.isFinite(value) || span <= 0) { return false } const target = edge === "max" ? range.max : range.min const tolerance = Math.max(1e-4, span * 0.005) return Math.abs(value - target) <= tolerance } function getRangeLabels( items: T[], getValue: (item: T) => number, getLabel: (item: T) => string ) { const finiteItems = items.filter((item) => Number.isFinite(getValue(item))) if (finiteItems.length === 0) { return { minLabel: "N/A", maxLabel: "N/A" } } const minItem = [...finiteItems].sort((a, b) => getValue(a) - getValue(b))[0] const maxItem = [...finiteItems].sort((a, b) => getValue(b) - getValue(a))[0] return { minLabel: getLabel(minItem), maxLabel: getLabel(maxItem), } } function ScoreRail({ meanValue, meanLabel, range, minLabel, maxLabel, }: { meanValue: number meanLabel: string range: ScoreRange minLabel: string maxLabel: string }) { const meanPercent = normalizeWithinRange(meanValue, range) * 100 const globalMinTitle = `Min: ${minLabel}` const globalMaxTitle = `Max: ${maxLabel}` return (
) } function parseNumericRank(value: unknown) { if (typeof value === "number") { return Number.isFinite(value) ? value : null } if (typeof value === "string") { const parsed = Number.parseFloat(value.replace(/[^0-9.]/g, "")) return Number.isFinite(parsed) ? parsed : null } return null } function parseRankFraction(value: unknown) { if (typeof value !== "string") { return null } const match = value.match(/(\d+)\s*\/\s*(\d+)/) if (!match) { return null } const position = Number.parseInt(match[1], 10) const total = Number.parseInt(match[2], 10) if (!Number.isFinite(position) || !Number.isFinite(total) || total <= 0) { return null } return { position, total } } function findRankFromObject(value: unknown, depth = 0): { position: number; total: number | null } | null { if (depth > 4 || value == null) { return null } const fraction = parseRankFraction(value) if (fraction) { return fraction } if (typeof value !== "object" || Array.isArray(value)) { return null } const record = value as Record const keys = Object.keys(record) const lowered = Object.fromEntries(keys.map((key) => [key.toLowerCase(), record[key]])) const positionCandidates = ["rank", "position", "place", "standing"] const totalCandidates = ["total", "out_of", "num_models", "model_count", "total_models", "population"] let position: number | null = null let total: number | null = null for (const key of positionCandidates) { if (key in lowered) { position = parseNumericRank(lowered[key]) if (position != null) { break } } } for (const key of totalCandidates) { if (key in lowered) { total = parseNumericRank(lowered[key]) if (total != null) { break } } } if (position != null) { return { position, total } } for (const nestedValue of Object.values(record)) { const nested = findRankFromObject(nestedValue, depth + 1) if (nested) { return nested } } return null } function getVariantPeerRank(result: EvaluationResult) { const fromDetails = findRankFromObject(result.score_details.details) if (fromDetails?.position != null) { return fromDetails } const fromSource = findRankFromObject(result.source_data) if (fromSource?.position != null) { return fromSource } if (result.evaluation_name.toLowerCase().includes("rank")) { const scoreRank = parseNumericRank(result.score_details.score) if (scoreRank != null) { return { position: scoreRank, total: null } } } return null } function buildVariantStructuredSections(variant: BenchmarkVariant) { const detailEntries = variant.result.score_details.details ? Object.entries(variant.result.score_details.details) : [] const numericBreakdown: Array<[string, unknown]> = [] const helmMetrics: Array<{ label: string; tab: string; score: string }> = [] const structuredBreakdown: Array<[string, unknown]> = [] for (const [key, value] of detailEntries) { if (typeof value === "number") { numericBreakdown.push([key, value]) continue } const parsed = parseHelmDetailEntry(value) if (parsed?.score != null && parsed.score !== "") { helmMetrics.push({ label: key, tab: parsed.tab ?? "", score: parsed.score }) continue } // Skip internal HELM meta-fields that add no user value if (key === "description" || key === "tab") continue structuredBreakdown.push([key, value]) } return { numericBreakdown, helmMetrics, structuredBreakdown } } function formatConfigLabel(key: string) { return key .split(".") .slice(-2) .join(" ") .replace(/_/g, " ") .replace(/\b\w/g, (letter) => letter.toUpperCase()) } function getVariantConfigMap(variant: BenchmarkVariant) { const configMap: Record = {} const setup = getEvaluationVariantLabel(variant.evaluation) if (setup) { configMap.setup = setup } // Prefer result-level generation config, fall back to eval-level const genConfig = variant.result.generation_config ?? variant.evaluation.generation_config if (genConfig?.generation_args) { for (const [key, value] of collectConfigEntries(genConfig.generation_args)) { configMap[key] = value } } if (genConfig?.additional_details) { const ad = genConfig.additional_details if (typeof ad === "string") { configMap.additional_details = ad } else if (typeof ad === "object") { for (const [key, value] of collectConfigEntries(ad)) { configMap[key] = value } } } if (genConfig?.prompt_template) { configMap.prompt_template = genConfig.prompt_template } return configMap } function normalizeScoreForDisplay(result: EvaluationResult) { const minScore = result.metric_config.min_score ?? 0 const maxScore = result.metric_config.max_score ?? 1 const range = maxScore - minScore if (range <= 0) { return 0 } const rawNormalized = (result.score_details.score - minScore) / range const normalized = result.metric_config.lower_is_better ? 1 - rawNormalized : rawNormalized return Math.max(0, Math.min(1, normalized)) } function slugifyEvalSummaryId(value: string) { return value.toLowerCase().replace(/[^a-z0-9]+/g, "_").replace(/^_|_$/g, "") } function getEvalDetailHref( evaluation: BenchmarkEvaluation, result: EvaluationResult, returnTo?: string ) { const baseHref = evaluation.eval_summary_id ? `/evals/${evaluation.eval_summary_id}` : `/evals/${slugifyEvalSummaryId(`${evaluation.benchmark || getResultBenchmarkName(evaluation, result)}__${result.evaluation_name}`)}` if (!returnTo) { return baseHref } const params = new URLSearchParams({ from: returnTo }) return `${baseHref}?${params.toString()}` } function getEvalSummaryIdFromHref(href: string) { const [, id = ""] = href.split("/evals/") return id.split("?")[0]?.split("#")[0] ?? "" } function getGroupPeerRank( group: BenchmarkGroup, modelIds: string[], peerRanks: PeerRanksMap ): { position: number; total: number } | null { let best: { position: number; total: number } | null = null for (const variant of group.variants) { const evalSummaryId = variant.evaluation.eval_summary_id ?? getEvalSummaryIdFromHref(getEvalDetailHref(variant.evaluation, variant.result)) const evalRanks = peerRanks[evalSummaryId] if (!evalRanks) continue // Try all known model IDs for this model family for (const mid of modelIds) { const rank = evalRanks[mid] if (rank == null) continue if (best == null) { best = rank } else { const rankRatio = rank.total > 0 ? rank.position / rank.total : rank.position const bestRatio = best.total > 0 ? best.position / best.total : best.position if (rankRatio < bestRatio) { best = rank } } } } return best ?? (group.bestRankPosition != null ? { position: group.bestRankPosition, total: group.bestRankTotal ?? 0 } : null) } // peer-ranks.json now ships as a sidecar inside the pinned `SNAPSHOT_URL` // snapshot (Stage J emits it alongside hierarchy.json / comparison-index.json // — see eval_cards_backend_pipeline commit ffbfe71). Routing through the // same `/api/peer-ranks` endpoint as the other sidecars keeps peer ranks // pinned to the snapshot the rest of the page is reading from, instead of // drifting to the unversioned `main`-branch copy at the dataset root. let peerRanksPromise: Promise | null = null function loadPeerRanks(): Promise { if (!peerRanksPromise) { peerRanksPromise = fetchPeerRanks().catch(() => ({} as PeerRanksMap)) } return peerRanksPromise } async function fetchPeerRankForModel(evalSummaryId: string, modelId: string) { const ranks = await loadPeerRanks() return ranks[evalSummaryId]?.[modelId] ?? null } function formatResultDisplayScore(result: EvaluationResult) { return formatRawScoreValue(result.score_details.score, result.metric_config.unit) } function toComparableTimestamp(timestamp: string) { const numericTimestamp = Number.parseFloat(timestamp) if (Number.isFinite(numericTimestamp)) { return numericTimestamp } const parsedTimestamp = new Date(timestamp).getTime() return Number.isFinite(parsedTimestamp) ? parsedTimestamp : Number.NEGATIVE_INFINITY } function getVariantDedupKey(variant: BenchmarkVariant) { const configEntries = Object.entries(getVariantConfigMap(variant)).sort(([a], [b]) => a.localeCompare(b)) const sourceDataName = !Array.isArray(variant.result.source_data) && variant.result.source_data?.dataset_name ? variant.result.source_data.dataset_name : !Array.isArray(variant.evaluation.source_data) && variant.evaluation.source_data?.dataset_name ? variant.evaluation.source_data.dataset_name : "" return JSON.stringify({ label: variant.label, metricSummaryId: variant.result.metric_summary_id, metricKey: variant.result.metric_key, metricLabel: variant.metricLabel, variantType: variant.variantType, setupLabel: variant.setupLabel, sliceLabel: variant.sliceLabel, displayScore: variant.displayScore, sourceOrganization: getOrganizationDisplayName(variant.evaluation.source_metadata.source_organization_name), sourceName: normalizeDisplayLabel(variant.evaluation.source_metadata.source_name ?? ""), sourceType: variant.evaluation.source_metadata.source_type, sourceDataName, configEntries, }) } function buildBenchmarkGroups( entries: Array<{ evaluation: BenchmarkEvaluation; result: EvaluationResult; category: CategoryType }>, benchmarkCards: Record | undefined, returnTo?: string ): BenchmarkGroup[] { const groups = new Map() // Pre-pass: bucket stderr companion values keyed by (groupKey, pairKey) // so the second pass can attach each stderr's value to the matching // score variant via `auxStderr`. Stderr entries themselves are dropped // from the variant list to avoid showing them as standalone rows. const stderrByPair = new Map() for (const entry of entries) { const summaryId = entry.result.metric_summary_id ?? "" if (!isStderrMetricId(summaryId)) continue const groupKey = entry.evaluation.eval_summary_id ?? entry.evaluation.parent_benchmark_id ?? entry.evaluation.family_id ?? "benchmark" const pairKey = metricPairKey(summaryId) if (!pairKey) continue const score = entry.result.score_details.score if (!Number.isFinite(score)) continue stderrByPair.set(`${groupKey}::${pairKey}`, { score, unit: entry.result.metric_config.unit, }) } for (const entry of entries) { if (isStderrMetricId(entry.result.metric_summary_id)) continue const rawBenchmarkName = entry.evaluation.benchmark || entry.evaluation.benchmark_parent_name || getResultBenchmarkName(entry.evaluation, entry.result) // Slice evals (e.g. AIR-Bench's ~30 per-category cells, all // `is_slice=true` with `parent_benchmark_id="air-bench-2024"`) // collapse into ONE BenchmarkGroup keyed on the parent's // eval_summary_id (`%2Fair-bench-2024`). The slices then // populate the plotbox view dropdown instead of fanning out into // ~30 look-alike plotboxes. Title prefers the parent name so the // grouped card reads "AIR-Bench 2024" rather than "Confidentiality". const isFoldableSlice = Boolean( entry.evaluation.is_slice && entry.evaluation.parent_benchmark_id, ) const sliceParentEvalSummaryId = (() => { if (!isFoldableSlice) return null const evalId = entry.evaluation.eval_summary_id ?? "" const sourcePrefix = evalId.includes("%2F") ? evalId.split("%2F")[0] : null if (!sourcePrefix) return null return `${sourcePrefix}%2F${entry.evaluation.parent_benchmark_id}` })() const title = isFoldableSlice ? (entry.evaluation.benchmark_parent_name || entry.evaluation.parent_benchmark_id || entry.evaluation.display_name || entry.evaluation.benchmark || getResultBenchmarkName(entry.evaluation, entry.result)) : (entry.evaluation.display_name || entry.evaluation.slice_name || entry.evaluation.benchmark_leaf_name || entry.evaluation.benchmark_parent_name || entry.evaluation.benchmark || getResultBenchmarkName(entry.evaluation, entry.result)) const canonicalTitle = isFoldableSlice ? title : (entry.evaluation.canonical_display_name || (entry.evaluation.slice_name && (entry.evaluation.benchmark_parent_name || entry.evaluation.benchmark) ? `${entry.evaluation.benchmark_parent_name || entry.evaluation.benchmark} / ${entry.evaluation.slice_name}` : title)) // eval_summary_id is producer-shipped on every v3 entry; the // remaining ?? tiers are for legacy snapshots without that field. const groupKey = sliceParentEvalSummaryId ?? entry.evaluation.eval_summary_id ?? entry.evaluation.parent_benchmark_id ?? entry.evaluation.family_id ?? "benchmark" const card = benchmarkCards ? lookupBenchmarkCard(benchmarkCards, rawBenchmarkName) : undefined const normalizedScore = normalizeScoreForDisplay(entry.result) const displayScore = formatResultDisplayScore(entry.result) const rawScore = entry.result.score_details.score const rankInfo = getVariantPeerRank(entry.result) const rankPosition = rankInfo?.position ?? null const rankTotal = rankInfo?.total ?? null const rankRatio = rankPosition != null && rankTotal != null && rankTotal > 0 ? rankPosition / rankTotal : rankPosition != null ? rankPosition : null const descriptor = getVariantDescriptor(entry.evaluation, entry.result) const pairKey = metricPairKey(entry.result.metric_summary_id ?? "") const auxStderrEntry = pairKey ? stderrByPair.get(`${groupKey}::${pairKey}`) : undefined const variant: BenchmarkVariant = { evaluation: entry.evaluation, result: entry.result, label: descriptor.label, variantType: descriptor.variantType, metricLabel: descriptor.metricLabel, setupLabel: descriptor.setupLabel, sliceLabel: descriptor.sliceLabel, displayScore, normalizedScore, rankPosition, rankTotal, rankRatio, ...(auxStderrEntry ? { auxStderr: auxStderrEntry.score, auxStderrUnit: auxStderrEntry.unit } : {}), } const existing = groups.get(groupKey) if (!existing) { groups.set(groupKey, { key: groupKey, title, canonicalTitle, evalDetailHref: getEvalDetailHref(entry.evaluation, entry.result, returnTo), category: entry.category, description: entry.result.metric_config.evaluation_description ?? "", scoreType: entry.result.metric_config.score_type ?? "continuous", avgRawScore: rawScore, avgNormalizedScore: normalizedScore, avgDisplayScore: formatRawScoreValue(rawScore, entry.result.metric_config.unit), bestRankPosition: rankPosition, bestRankTotal: rankTotal, bestRankRatio: rankRatio, domains: card?.benchmark_details?.domains ?? [], benchmarkCard: card, variants: [variant], }) continue } existing.variants.push(variant) const newDesc = entry.result.metric_config.evaluation_description ?? "" if ((existing.description ?? "").length < newDesc.length) { existing.description = newDesc } if (existing.scoreType !== entry.result.metric_config.score_type) { existing.scoreType = "mixed" } } return Array.from(groups.values()) .map((group) => { const dedupedVariants = new Map() // --- BEGIN PATCH: Auto-detect and renormalize mixed scales --- // Group by metric_summary_id (or metric_key) for scale detection const metricGroups = new Map() for (const variant of group.variants) { const key = variant.result.metric_summary_id || variant.result.metric_key || "default" if (!metricGroups.has(key)) metricGroups.set(key, []) metricGroups.get(key)!.push(variant) } let scaleWarning = false for (const [metricKey, variants] of metricGroups.entries()) { // Collect all min/max for this metric const mins = variants.map(v => v.result.metric_config.min_score ?? null).filter(x => x !== null) const maxs = variants.map(v => v.result.metric_config.max_score ?? null).filter(x => x !== null) // If any variant is missing min/max, skip normalization for this group if (mins.length !== variants.length || maxs.length !== variants.length) continue const uniqueMins = Array.from(new Set(mins)) const uniqueMaxs = Array.from(new Set(maxs)) // If there are multiple scales, renormalize all to the most common (or largest span) if (uniqueMins.length > 1 || uniqueMaxs.length > 1) { scaleWarning = true // Pick the most common (min, max) pair, or the one with the largest range const rangeCounts = new Map() for (const v of variants) { const k = `${v.result.metric_config.min_score}|${v.result.metric_config.max_score}` rangeCounts.set(k, (rangeCounts.get(k) || 0) + 1) } let canonical = Array.from(rangeCounts.entries()).sort((a, b) => b[1] - a[1])[0]?.[0] if (!canonical) canonical = `${Math.min(...mins)}|${Math.max(...maxs)}` const [canonicalMin, canonicalMax] = canonical.split("|").map(Number) for (const v of variants) { const min = v.result.metric_config.min_score ?? 0 const max = v.result.metric_config.max_score ?? 1 // Only renormalize if different from canonical if (min !== canonicalMin || max !== canonicalMax) { // Renormalize score to canonical scale const oldScore = v.result.score_details.score const normalized = (oldScore - min) / (max - min) v.result.score_details.score = normalized * (canonicalMax - canonicalMin) + canonicalMin v.result.metric_config.min_score = canonicalMin v.result.metric_config.max_score = canonicalMax } } } } for (const variant of group.variants) { const variantKey = getVariantDedupKey(variant) const existingVariant = dedupedVariants.get(variantKey) if (!existingVariant) { dedupedVariants.set(variantKey, variant) continue } if ( toComparableTimestamp(variant.evaluation.retrieved_timestamp) >= toComparableTimestamp(existingVariant.evaluation.retrieved_timestamp) ) { dedupedVariants.set(variantKey, variant) } } group.variants = Array.from(dedupedVariants.values()) group.variants.sort((a, b) => { const aIsSlice = Boolean(a.evaluation.slice_key) const bIsSlice = Boolean(b.evaluation.slice_key) if (aIsSlice !== bIsSlice) { return aIsSlice ? 1 : -1 } const aPrimaryLabel = getVariantPrimaryLabel(a, group.title) const bPrimaryLabel = getVariantPrimaryLabel(b, group.title) if (aPrimaryLabel !== bPrimaryLabel) { return aPrimaryLabel.localeCompare(bPrimaryLabel) } return b.normalizedScore - a.normalizedScore }) group.avgRawScore = group.variants.reduce((sum, variant) => sum + variant.result.score_details.score, 0) / group.variants.length group.avgNormalizedScore = group.variants.reduce((sum, variant) => sum + variant.normalizedScore, 0) / group.variants.length group.avgDisplayScore = formatRawScoreValue(group.avgRawScore) const rankedVariants = group.variants .filter((variant) => variant.rankRatio != null) .sort((a, b) => (a.rankRatio ?? Number.POSITIVE_INFINITY) - (b.rankRatio ?? Number.POSITIVE_INFINITY)) group.bestRankPosition = rankedVariants[0]?.rankPosition ?? null group.bestRankTotal = rankedVariants[0]?.rankTotal ?? null group.bestRankRatio = rankedVariants[0]?.rankRatio ?? null // Attach a warning if scales were mixed if (scaleWarning) { // Patch: extend group with a warning property for UI (group as any).__scaleWarning = true } return group }) .sort((a, b) => b.avgNormalizedScore - a.avgNormalizedScore) } function getEvaluationVariantLabel(evaluation: BenchmarkEvaluation) { const evaluationIdWithoutTimestamp = evaluation.evaluation_id.replace(/\/[^/]+$/, "") const modelSlug = evaluation.model_info.id.replace(/\//g, "_") let evaluationPrefix = evaluationIdWithoutTimestamp if (evaluationPrefix.endsWith(`__${modelSlug}`)) { evaluationPrefix = evaluationPrefix.slice(0, -(`__${modelSlug}`.length)) } else if (evaluationPrefix.endsWith(`/${modelSlug}`)) { evaluationPrefix = evaluationPrefix.slice(0, -(`/${modelSlug}`.length)) } const benchmarkName = evaluation.benchmark if (benchmarkName && evaluationPrefix.startsWith(`${benchmarkName}/`)) { const variant = evaluationPrefix.slice(benchmarkName.length + 1) return variant.split("/").filter(Boolean).pop() || null } if (benchmarkName && evaluationPrefix === benchmarkName) { return null } return evaluationPrefix.split("/").filter(Boolean).pop() || null } export function BenchmarkDetail({ summary, benchmarkCards, modelCards, evalHierarchy, comparisonIndex, }: BenchmarkDetailProps) { const { mode } = useAudienceMode() const isResearchView = mode === "research" const pathname = usePathname() const searchParams = useSearchParams() const [benchmarkSearch, setBenchmarkSearch] = useState("") // Sort dropdown was removed — ordering is driven by the source/category // grouping itself, not a user-selected sort. const [selectedCategories, setSelectedCategories] = useState([]) const [selectedFamilies, setSelectedFamilies] = useState([]) const [expandedSuites, setExpandedSuites] = useState>(new Set()) const [activeBenchmarkGroupKey, setActiveBenchmarkGroupKey] = useState(null) const [benchmarkViewMode, setBenchmarkViewMode] = useState<"grid" | "list">("grid") // Tri-state view selector. "source" = the warehouse's natural shape: // family-rooted plotboxes / family-grouped accordions, no cross-family // collapse. "category" = same composite/standalone units, but the top- // level grouping switches to the curated category tag (data/benchmarks/ // categories.json) so similarly-tagged benchmarks cluster across // families. "overlaps" = cross-family duplicates only, rendered as a // table (no plotbox/list toggle) with mean and 95% CI for the model's // score across each canonical's appearances. const [groupingMode, setGroupingMode] = useState<"source" | "category" | "overlaps">("source") const [expandedFamilies, setExpandedFamilies] = useState>(new Set()) const toggleFamily = (key: string) => setExpandedFamilies((prev) => { const next = new Set(prev) if (next.has(key)) next.delete(key) else next.add(key) return next }) const currentDetailHref = useMemo(() => { const query = searchParams.toString() return query ? `${pathname}?${query}` : pathname }, [pathname, searchParams]) const modelId = summary.model_info.id // Collect all known model IDs for peer rank lookup (family ID + raw variant IDs) const modelIds = useMemo(() => { const ids = new Set([modelId]) if ('raw_model_ids' in summary) { for (const id of (summary as any).raw_model_ids ?? []) { ids.add(id) } } // Also add IDs from individual evaluations, including pipeline-computed family_id for (const evals of Object.values(summary.evaluations_by_category)) { for (const e of evals) { if (e.model_info?.id) ids.add(e.model_info.id) const familyId = (e.model_info as any)?.family_id if (familyId) ids.add(familyId) } } return Array.from(ids) }, [modelId, summary]) const [peerRanks, setPeerRanks] = useState({}) // Load peer-ranks.json once and store in state so the table can use them useEffect(() => { loadPeerRanks().then(setPeerRanks) }, []) // Build an eval_summary_id → family/composite lookup from hierarchy.json. // ~31 eval_summary_ids appear in multiple families (e.g. mmlu-pro under // both `mmlu` and `artificial-analysis`); use the eval row's own family_id // as the disambiguating preference when present. const hierarchyIndex = useMemo(() => { if (!evalHierarchy) { return null } const familyIdByEvalSummaryId = new Map() for (const evals of Object.values(summary.evaluations_by_category)) { for (const evaluation of evals) { if (evaluation.eval_summary_id && evaluation.family_id) { familyIdByEvalSummaryId.set(evaluation.eval_summary_id, evaluation.family_id) } } } return buildHierarchyEvalIndex( evalHierarchy, (evalSummaryId) => familyIdByEvalSummaryId.get(evalSummaryId) ?? null, ) }, [evalHierarchy, summary.evaluations_by_category]) // Source-prefix → hierarchy-family lookup. The producer ships // benchmark-canonical `family_id`s on each comparison-index entry // (e.g. `family_id="aime"` for every AIME variant across sources) // alongside source-leaderboard families in hierarchy.json (e.g. // `artificial-analysis`, `vals-ai`, `llm-stats`). Most evals are // listed in `family.eval_summary_ids` and resolve via // `hierarchyIndex` directly, but variant rows the producer // emits under the same canonical (e.g. `aime-2025`, `aime-2024`) // are NOT enumerated at family level — they fall back to // `evalEntry.family_id` and synthesise a phantom "aime" parent // section. This map provides a second fallback: pick the // hierarchy family whose own listed eval ids share this id's // source prefix. const sourcePrefixFamily = useMemo(() => { const out = new Map() for (const fam of evalHierarchy?.families ?? []) { for (const id of fam.eval_summary_ids ?? []) { const prefix = id.includes("%2F") ? id.split("%2F")[0] : null if (!prefix) continue if (!out.has(prefix)) { out.set(prefix, { key: fam.key, displayName: fam.display_name }) } } } return out }, [evalHierarchy]) // Lookup eval_summary_id -> canonical benchmark info from hierarchy.json's // `benchmark_index[]`. Used by the "group duplicates" toggle in the // list view AND by the histogram cross-family whisker overlay. // // The hierarchy is pre-cleaned by `cleanHierarchy` (lib/clean-hierarchy.ts) // server-side: family-rollup entries are dropped, (family_key, // eval_summary_id) pairs deduped, degenerate entries filtered out. So // we can iterate the entries directly here without per-entry filtering. const benchmarkIndexLookup = useMemo(() => { const out = new Map< string, { canonicalKey: string; canonicalDisplayName: string; siblingEvalIds: string[] } >() for (const entry of evalHierarchy?.benchmark_index ?? []) { const idSet = new Set() for (const app of entry.appearances ?? []) { for (const id of app.eval_summary_ids ?? []) idSet.add(id) } const ids = Array.from(idSet) for (const id of ids) { if (!out.has(id)) { out.set(id, { canonicalKey: entry.key, canonicalDisplayName: entry.display_name, siblingEvalIds: ids, }) } } } return out }, [evalHierarchy]) // List-view-only consolidation state. Active when `groupingMode === "benchmark"`. // `mergedRowState` precomputes (across families in display order) which // rows render as the consolidated representative ("merged"), which collapse // into a previously-rendered representative ("skip"), and which stay as // single per-eval rows. Aggregates carry mean/min/max + per-source // breakdown (one entry per contributing variant) for the hover tooltip. type MergedRowAggregate = { canonicalKey: string canonicalDisplayName: string mean: number min: number max: number sources: Array<{ familyKey: string familyName: string score: number displayScore: string group: BenchmarkGroup variant: BenchmarkVariant }> } type RowDisposition = "single" | "merged" | "skip" // Composite relevance score for benchmark ordering // relevance = population × 0.4 + rank_extremity × 0.3 + has_metadata × 0.2 + recency × 0.1 const getRelevanceScore = useMemo(() => { // Find max population across all peer-ranked benchmarks let maxPop = 1 for (const evalRanks of Object.values(peerRanks)) { const pop = Object.keys(evalRanks).length if (pop > maxPop) maxPop = pop } // Find latest timestamp across all evaluations for recency normalization const allTimestamps: number[] = [] for (const evals of Object.values(summary.evaluations_by_category)) { for (const e of evals) { const ts = parseFloat(e.retrieved_timestamp) if (Number.isFinite(ts)) allTimestamps.push(ts) } } const maxTs = allTimestamps.length > 0 ? Math.max(...allTimestamps) : 0 const minTs = allTimestamps.length > 0 ? Math.min(...allTimestamps) : 0 const tsRange = maxTs - minTs || 1 return (group: BenchmarkGroup): number => { const rank = getGroupPeerRank(group, modelIds, peerRanks) // Population: how many models were compared (0-1) const population = rank ? Math.min(rank.total / maxPop, 1) : 0 // Rank extremity: how far from median — |0.5 - percentile| × 2 (0-1) const percentile = rank ? rank.position / rank.total : 0.5 const rankExtremity = Math.abs(0.5 - percentile) * 2 // Rich metadata: has benchmark card (0 or 1) const hasMetadata = group.benchmarkCard ? 1 : 0 // Recency: how recent is the latest evaluation (0-1) let latestTs = 0 for (const v of group.variants) { const ts = parseFloat(v.evaluation.retrieved_timestamp) if (Number.isFinite(ts) && ts > latestTs) latestTs = ts } const recency = maxTs > minTs ? (latestTs - minTs) / tsRange : 0.5 return population * 0.4 + rankExtremity * 0.3 + hasMetadata * 0.2 + recency * 0.1 } }, [peerRanks, modelIds, summary.evaluations_by_category]) const allEvaluations = useMemo( () => Object.values(summary.evaluations_by_category).flat(), [summary.evaluations_by_category] ) const reportingStats = useMemo(() => { const organizations = new Set() const sourceTypes = new Set() const libraries = new Set() let missingGenerationConfigs = 0 let thirdPartyEvaluations = 0 allEvaluations.forEach((evaluation) => { organizations.add(getOrganizationDisplayName(evaluation.source_metadata.source_organization_name)) sourceTypes.add(evaluation.source_metadata.source_type) if (evaluation.eval_library?.name) { libraries.add(formatEvalLibrary(evaluation.eval_library)) } if (evaluation.source_metadata.evaluator_relationship === "third_party") { thirdPartyEvaluations += 1 } missingGenerationConfigs += evaluation.evaluation_results.filter((result) => !result.generation_config).length }) return { organizationNames: Array.from(organizations).sort((a, b) => a.localeCompare(b)), organizationCount: organizations.size, sourceTypeCount: sourceTypes.size, libraryCount: libraries.size, libraryList: Array.from(libraries).sort((a, b) => a.localeCompare(b)), missingGenerationConfigs, thirdPartyEvaluations, } }, [allEvaluations]) const reproducibilityGapCount = summary.reproducibility_summary?.has_reproducibility_gap_count ?? reportingStats.missingGenerationConfigs const reproducibilityResultsTotal = summary.reproducibility_summary?.results_total ?? summary.total_evaluations const allCategoryResults = useMemo( () => Object.entries(summary.evaluations_by_category).flatMap(([fallbackCategory, evals]) => evals.flatMap((evaluation) => { // Re-bucket by curated tag from data/benchmarks/categories.json. // The hierarchy lookup gives us the leaf benchmark's derivedTags; // the first tag becomes the displayed category. Fall back to the // legacy 5-bucket category only when no tag is found, so existing // ordering / filter wiring still works. // // Normalise both branches into the lowercase-snake_case form used // by categories.json so visually-identical categories (the // legacy "General" fallback and the curated "general" tag, the // legacy "Safety" and "safety", etc.) collapse to the same // CategoryType — otherwise downstream surfaces show two // adjacent rows / pills with the same label. const evalSummaryId = evaluation.eval_summary_id const tags = evalSummaryId ? hierarchyIndex?.get(evalSummaryId)?.tags : undefined const primaryTag = tags && tags.length > 0 ? tags[0] : null const normalisedFallback = fallbackCategory .toLowerCase() .trim() .replace(/\s+/g, "_") const category = (primaryTag ?? normalisedFallback) as CategoryType return evaluation.evaluation_results.map((result) => ({ evaluation, result, category, })) }) ), [summary.evaluations_by_category, hierarchyIndex] ) const policyHighlights = useMemo(() => { const groups = buildBenchmarkGroups(allCategoryResults, benchmarkCards) const seenLabels = new Set() return groups .filter((group) => { const narrative = getPolicyBenchmarkNarrative(group.title) if (seenLabels.has(narrative.label)) { return false } seenLabels.add(narrative.label) return true }) .slice(0, 6) .map((group) => { const narrative = getPolicyBenchmarkNarrative(group.title) const level = getPolicySignalLevel(group.avgNormalizedScore) return { key: group.key, title: group.canonicalTitle, label: narrative.label, description: narrative.description, scoreText: group.avgDisplayScore, level, } }) }, [allCategoryResults, benchmarkCards]) const policySummary = useMemo(() => { const benchmarkCount = new Set( allCategoryResults.map((entry) => entry.evaluation.benchmark || entry.evaluation.benchmark_parent_name || entry.evaluation.eval_summary_id || getResultBenchmarkName(entry.evaluation, entry.result)) ).size const allThirdParty = allEvaluations.length > 0 && reportingStats.thirdPartyEvaluations === allEvaluations.length const leadOrganization = reportingStats.organizationNames[0] const modelScaleDescription = getModelScaleDescription(summary.model_info.additional_details?.params_billions) const compactParamCount = formatParamsBillions(summary.model_info.additional_details?.params_billions) const normalizedModelName = getModelDisplayName(summary.model_info.name) const compactModelName = compactParamCount ? `${normalizedModelName} · ${compactParamCount}` : normalizedModelName let testedByCopy = `Reported across ${benchmarkCount} standardized benchmark${benchmarkCount === 1 ? "" : "s"}.` if (leadOrganization && reportingStats.organizationCount === 1) { testedByCopy = allThirdParty ? `Tested independently by ${leadOrganization} (a third party, distinct from the model's developer) using ${benchmarkCount} standardized benchmark${benchmarkCount === 1 ? "" : "s"}.` : `Reported by ${leadOrganization} using ${benchmarkCount} standardized benchmark${benchmarkCount === 1 ? "" : "s"}.` } else if (leadOrganization) { testedByCopy = allThirdParty ? `Tested by ${leadOrganization} and ${reportingStats.organizationCount - 1} other reporting organization${reportingStats.organizationCount - 1 === 1 ? "" : "s"} using ${benchmarkCount} standardized benchmark${benchmarkCount === 1 ? "" : "s"}.` : `Reported by ${reportingStats.organizationCount} organizations using ${benchmarkCount} benchmark views.` } const reproducibilityCopy = reproducibilityGapCount === 0 ? null : reproducibilityGapCount === reproducibilityResultsTotal ? "How this model was prompted during testing is not documented. Scores cannot be independently confirmed." : `${reproducibilityGapCount} of ${reproducibilityResultsTotal} reported scores are missing enough setup detail to be re-run as-is.` const comparabilityCopy = reproducibilityGapCount > 0 ? `${benchmarkCount > 0 ? `These results cover ${benchmarkCount} benchmark${benchmarkCount === 1 ? "" : "s"},` : "These results"} but missing prompting details mean apparent score gaps may partly reflect setup differences as well as capability.` : "Shared benchmark coverage helps, but evaluator choices, benchmark mix, and model size can still limit direct apples-to-apples comparison." const sizeCaveat = modelScaleDescription ? `${modelScaleDescription}. Comparisons against much smaller or larger systems should be interpreted with care.` : null return { compactModelName, modelScaleDescription, testedByCopy, reproducibilityCopy, comparabilityCopy, sizeCaveat, independentlyVerified: allThirdParty || reportingStats.thirdPartyEvaluations > 0, benchmarkCount, } }, [ allCategoryResults, allEvaluations.length, reportingStats, reproducibilityGapCount, reproducibilityResultsTotal, summary.model_info.additional_details?.params_billions, summary.model_info.name, ]) const benchmarkGroups = useMemo( () => buildBenchmarkGroups(allCategoryResults, benchmarkCards, currentDetailHref), [allCategoryResults, benchmarkCards, currentDetailHref] ) // Categories actually present in this model's benchmark groups, derived // from the curated tag bucketing in `allCategoryResults`. We no longer // trust `summary.categories_covered` (legacy 5-bucket) for ordering / // filtering; build the list locally so the new tag vocabulary surfaces. // Sorted alphabetically by display label for stable filter-pill order. const availableCategories = useMemo(() => { const seen = new Set() const cats: string[] = [] for (const group of benchmarkGroups) { const cat = group.category as unknown as string if (!seen.has(cat)) { seen.add(cat) cats.push(cat) } } cats.sort((a, b) => formatTagLabel(a).localeCompare(formatTagLabel(b))) return cats as unknown as CategoryType[] }, [benchmarkGroups]) // Family names present in this model's benchmark groups — used for // the Source-view filter chips. Sorted alphabetically by display name. const availableFamilies = useMemo(() => { const seen = new Map() for (const group of benchmarkGroups) { const evalId = group.variants.find((v) => v.evaluation.eval_summary_id)?.evaluation.eval_summary_id const hierarchyLocation = evalId ? hierarchyIndex?.get(evalId) ?? null : null const sourcePrefix = evalId?.includes("%2F") ? evalId.split("%2F")[0] : null const inferred = !hierarchyLocation && sourcePrefix ? sourcePrefixFamily.get(sourcePrefix) ?? null : null const famKey = hierarchyLocation?.familyKey ?? inferred?.key ?? (sourcePrefix ?? group.key) const famName = hierarchyLocation?.familyDisplayName || inferred?.displayName || group.title if (!seen.has(famKey)) seen.set(famKey, famName) } return Array.from(seen.entries()) .map(([key, name]) => ({ key, name })) .sort((a, b) => a.name.localeCompare(b.name)) }, [benchmarkGroups, hierarchyIndex, sourcePrefixFamily]) // First-party vs third-party split per category (for the donut + bars). const evaluatorMix = useMemo(() => { // Bucket counts per category, then re-bucket by display label so // visually-identical labels collapse: the curated tag vocab can // produce two distinct CategoryType strings ("general" vs // "general_other") that both render as "General". Without this the // donut shows two "General" / "Safety" rows. const byCat = new Map() let firstTotal = 0 let thirdTotal = 0 let collabTotal = 0 let otherTotal = 0 for (const group of benchmarkGroups) { const slot = byCat.get(group.category) ?? { first: 0, third: 0, collab: 0, other: 0 } for (const variant of group.variants) { const rel = variant.evaluation.source_metadata.evaluator_relationship if (rel === "first_party") { slot.first++; firstTotal++ } else if (rel === "third_party") { slot.third++; thirdTotal++ } else if (rel === "collaborative") { slot.collab++; collabTotal++ } else { slot.other++; otherTotal++ } } byCat.set(group.category, slot) } type Row = { category: CategoryType label: string first: number third: number collab: number other: number total: number } const byLabel = new Map() for (const [category, counts] of byCat) { const label = formatTagLabel(category as unknown as string) const existing = byLabel.get(label) ?? { category, label, first: 0, third: 0, collab: 0, other: 0, total: 0, } existing.first += counts.first existing.third += counts.third existing.collab += counts.collab existing.other += counts.other existing.total = existing.first + existing.third + existing.collab + existing.other byLabel.set(label, existing) } const rows = Array.from(byLabel.values()) .filter((row) => row.total > 0) .sort((a, b) => a.label.localeCompare(b.label)) const grand = firstTotal + thirdTotal + collabTotal + otherTotal return { rows, firstTotal, thirdTotal, collabTotal, otherTotal, grand, } }, [benchmarkGroups]) const filteredBenchmarkGroups = useMemo(() => { const query = benchmarkSearch.trim().toLowerCase() const filtered = benchmarkGroups.filter((group) => { if (selectedCategories.length > 0 && !selectedCategories.includes(group.category)) { return false } if (selectedFamilies.length > 0) { const evalId = group.variants.find((v) => v.evaluation.eval_summary_id)?.evaluation.eval_summary_id const hierarchyLocation = evalId ? hierarchyIndex?.get(evalId) ?? null : null const sourcePrefix = evalId?.includes("%2F") ? evalId.split("%2F")[0] : null const inferred = !hierarchyLocation && sourcePrefix ? sourcePrefixFamily.get(sourcePrefix) ?? null : null const famKey = hierarchyLocation?.familyKey ?? inferred?.key ?? (sourcePrefix ?? group.key) if (!selectedFamilies.includes(famKey)) return false } if (!query) { return true } return ( group.title.toLowerCase().includes(query) || group.canonicalTitle.toLowerCase().includes(query) || group.description.toLowerCase().includes(query) || group.variants.some((variant) => variant.label.toLowerCase().includes(query)) ) }) // Default ordering: relevance score (most-reported / most-extreme rank / // richest metadata / most recent first). The source / category grouping // applied downstream may regroup but doesn't re-sort within a group. filtered.sort((a, b) => getRelevanceScore(b) - getRelevanceScore(a)) return filtered }, [benchmarkGroups, benchmarkSearch, selectedCategories, selectedFamilies, hierarchyIndex, sourcePrefixFamily, modelId, peerRanks, getRelevanceScore]) const groupedFilteredBenchmarkGroups = useMemo(() => { const order = new Map(availableCategories.map((category, index) => [category, index])) const groups = new Map() for (const benchmarkGroup of filteredBenchmarkGroups) { const bucket = groups.get(benchmarkGroup.category) ?? [] bucket.push(benchmarkGroup) groups.set(benchmarkGroup.category, bucket) } return Array.from(groups.entries()) .sort((a, b) => (order.get(a[0]) ?? 999) - (order.get(b[0]) ?? 999)) .map(([category, groups]) => ({ category, groups })) }, [filteredBenchmarkGroups, availableCategories]) // Family-bucketed groups for the list view, mirroring plotboxUnits logic. // When comparisonIndex is available we use the backend-authoritative // family_id; otherwise we fall back to the group's own key so each // BenchmarkGroup forms its own family. type ListFamily = { familyKey: string familyName: string kind: "single-eval" | "multi-eval" groups: BenchmarkGroup[] totalRows: number } const listFamiliesByCategory = useMemo(() => { const order = new Map( availableCategories.map((category, index) => [category, index]) ) const byCategory = new Map>() for (const group of filteredBenchmarkGroups) { const evalId = group.variants.find((v) => v.evaluation.eval_summary_id) ?.evaluation.eval_summary_id const evalEntry = evalId && comparisonIndex ? comparisonIndex.evals[evalId] : null const hierarchyLocation = evalId ? hierarchyIndex?.get(evalId) ?? null : null const sourcePrefix = evalId && evalId.includes("%2F") ? evalId.split("%2F")[0] : null const inferredSourceFamily = !hierarchyLocation && sourcePrefix ? sourcePrefixFamily.get(sourcePrefix) ?? null : null const famKey = hierarchyLocation?.familyKey ?? inferredSourceFamily?.key ?? evalEntry?.family_id ?? group.key const famName = hierarchyLocation?.familyDisplayName || inferredSourceFamily?.displayName || evalEntry?.family_display_name || evalEntry?.display_name || group.title const catBucket = byCategory.get(group.category) ?? new Map() const family = catBucket.get(famKey) ?? { familyKey: famKey, familyName: famName, kind: "single-eval" as const, groups: [] as BenchmarkGroup[], totalRows: 0, } family.groups.push(group) family.totalRows += group.variants.length catBucket.set(famKey, family) byCategory.set(group.category, catBucket) } return Array.from(byCategory.entries()) .sort((a, b) => (order.get(a[0]) ?? 999) - (order.get(b[0]) ?? 999)) .map(([category, fams]) => ({ category, families: Array.from(fams.values()).map((f) => ({ ...f, kind: f.groups.length > 1 ? "multi-eval" as const : "single-eval" as const, })), })) }, [filteredBenchmarkGroups, comparisonIndex, hierarchyIndex, sourcePrefixFamily, availableCategories]) // Precompute "merged" / "skip" / "single" disposition per row when the // list-view duplicate-grouping toggle is on. Walks the categories → // families → groups → variants in display order; the first variant we // encounter for each canonical benchmark renders the merged // representative (showing mean + range + per-source breakdown), and // every later variant of the same canonical benchmark is suppressed. // The aggregate contains contributions from every sibling regardless of // family, so the merged row is a true cross-family consolidation. // Cross-family duplicate consolidation in the source/category list views // is gone — overlaps now have their own dedicated table view. The list // view always renders one row per (family, group, variant) so this // returns null and the renderRow path treats every row as `single`. // The variables below are referenced by `listFamiliesByCategory`'s // useMemo deps but kept for type compatibility. void benchmarkIndexLookup type _Unused = { a: MergedRowAggregate; r: RowDisposition } const mergedRowState = null as null | { aggregates: Map rowDisposition: Map } // OverlapsRow types and the useMemo that builds the data live further // down — they need `currentModelRouteId` and `currentModelIdentityKeys`. type OverlapAppearance = { familyKey: string familyName: string evalSummaryId: string metricSummaryId: string metricName: string score: number displayScore: string unit: string | null } type OverlapRow = { canonicalKey: string canonicalDisplayName: string appearances: OverlapAppearance[] mean: number stddev: number min: number max: number ci95: { low: number; high: number } | null /** Tagged 0-1 (proportion) vs 0-100 (percent) — drives display. */ isPercentScale: boolean } const compositeGroups = useMemo(() => { const groups = groupByComposite(filteredBenchmarkGroups, modelIds, peerRanks, hierarchyIndex) // Re-sort composites by max relevance of their benchmarks return groups.sort((a, b) => { const aMax = Math.max(...a.benchmarks.map(getRelevanceScore)) const bMax = Math.max(...b.benchmarks.map(getRelevanceScore)) return bMax - aMax }) }, [filteredBenchmarkGroups, modelIds, peerRanks, getRelevanceScore, hierarchyIndex]) const categoryCompositeSections = useMemo( () => groupedFilteredBenchmarkGroups .map(({ category, groups }) => ({ category, composites: groupByComposite(groups, modelIds, peerRanks, hierarchyIndex).sort((a, b) => { const aMax = Math.max(...a.benchmarks.map(getRelevanceScore)) const bMax = Math.max(...b.benchmarks.map(getRelevanceScore)) return bMax - aMax }), })) .filter((section) => section.composites.length > 0), [groupedFilteredBenchmarkGroups, modelIds, peerRanks, getRelevanceScore, hierarchyIndex] ) const categoryScoreRanges = useMemo(() => { const ranges = new Map() for (const section of categoryCompositeSections) { ranges.set( section.category, getScoreRange(section.composites.map((composite) => composite.avgNormalizedScore)) ) } return ranges }, [categoryCompositeSections]) const compositeBenchmarkScoreRanges = useMemo(() => { const ranges = new Map() for (const section of categoryCompositeSections) { for (const composite of section.composites) { ranges.set( composite.compositeKey, getScoreRange(composite.benchmarks.map((group) => group.avgNormalizedScore)) ) } } return ranges }, [categoryCompositeSections]) const benchmarkGroupLookup = useMemo( () => new Map(benchmarkGroups.map((group) => [group.key, group] as const)), [benchmarkGroups] ) const activeBenchmarkGroup = activeBenchmarkGroupKey ? benchmarkGroupLookup.get(activeBenchmarkGroupKey) ?? null : null const toggleSuite = (compositeKey: string) => { setExpandedSuites((prev) => { const next = new Set(prev) if (next.has(compositeKey)) next.delete(compositeKey) else next.add(compositeKey) return next }) } const overviewBenchmarkGroups = selectedCategories.length > 0 || benchmarkSearch.trim() ? filteredBenchmarkGroups : benchmarkGroups // Only groups with actual sidecar ordinal rank data (total > 0) qualify // for the "ranks high / low in" summary. Fallback bestRankPosition values // are raw scores (0–1), not ordinal positions, so they must be excluded. const rankedBenchmarkGroups = useMemo( () => overviewBenchmarkGroups.filter((group) => { const rank = getGroupPeerRank(group, modelIds, peerRanks) return rank != null && rank.total > 0 }), [overviewBenchmarkGroups, modelId, peerRanks] ) const strongRankedBenchmarks = useMemo(() => { const sorted = [...rankedBenchmarkGroups].sort((a, b) => { const aRank = getGroupPeerRank(a, modelIds, peerRanks)! const bRank = getGroupPeerRank(b, modelIds, peerRanks)! return aRank.position / aRank.total - bRank.position / bRank.total }) return sorted.slice(0, 3) }, [rankedBenchmarkGroups, modelId, peerRanks]) const weakRankedBenchmarks = useMemo(() => { const strongKeys = new Set(strongRankedBenchmarks.map((g) => g.key)) const sorted = [...rankedBenchmarkGroups] .filter((g) => !strongKeys.has(g.key)) .sort((a, b) => { const aRank = getGroupPeerRank(a, modelIds, peerRanks)! const bRank = getGroupPeerRank(b, modelIds, peerRanks)! return bRank.position / bRank.total - aRank.position / aRank.total }) return sorted.slice(0, 3) }, [rankedBenchmarkGroups, strongRankedBenchmarks, modelId, peerRanks]) const repeatedBenchmarkCount = overviewBenchmarkGroups.filter((group) => group.variants.length > 1).length const setupDrivenBenchmarkCount = overviewBenchmarkGroups.filter((group) => group.variants.some((variant) => variant.variantType === "setup" || variant.variantType === "setup+slice") ).length const sliceDrivenBenchmarkCount = overviewBenchmarkGroups.filter((group) => group.variants.some((variant) => variant.variantType === "slice" || variant.variantType === "setup+slice") ).length useEffect(() => { setSelectedCategories((current) => current.filter((category) => availableCategories.includes(category)) ) }, [availableCategories]) useEffect(() => { const keys = new Set(availableFamilies.map((f) => f.key)) setSelectedFamilies((current) => current.filter((k) => keys.has(k))) }, [availableFamilies]) // Shared YYYY-MM-DD formatter (lib/utils#formatDateISO). Used for the // "Updated"
at line ~3514, the "Released" line, and any other // date-cell render in this component. Other surfaces (eval-detail // table, model-table summary) use the same helper so the corpus // renders one consistent date style. const formatDate = formatDateISO const jumpToDeepDive = (groupKey: string) => { if (benchmarkGroupLookup.has(groupKey)) { setActiveBenchmarkGroupKey(groupKey) } } // Model comparison logic const comparisonModels = useMemo(() => { if (!modelCards || modelCards.length === 0) return [] // Exclude the current model return modelCards.filter(m => m.id !== modelId) }, [modelCards, modelId]) // Per-benchmark extras added via the "+" button on each histogram. const [extraModelsByBenchmark, setExtraModelsByBenchmark] = useState>({}) // All histogram data now comes from `comparisonIndex` (comparison-index.json), // the backend-authoritative per-(eval, metric) leaderboard artifact. The old // `top_scores`-on-model-cards and per-eval-detail fan-out paths are retired. type HistogramBar = { modelId: string modelName: string score: number isCurrent: boolean isDefault: boolean submissionCount: number submissionAxis: SubmissionAxis headlineRunLabel?: string submissions?: ComparisonScoreEntry["submissions"] variantKey?: string } type BenchmarkHistogram = { histKey: string evalSummaryId: string metricSummaryId: string metricName: string metricGroup: ComparisonMetricEntry["group"] lowerIsBetter: boolean unit: string | null bars: HistogramBar[] availableModels: Array<{ id: string name: string score: number submissionCount: number submissionAxis: SubmissionAxis }> defaultIds: Set currentModelRank: { position: number; total: number } | null } const histKeyFor = (evalSummaryId: string, metricSummaryId: string) => `${evalSummaryId}::${metricSummaryId}` // Every identifier the current model may appear under in comparison-index. // Used to (a) pull our own score out of `by_model` and (b) drop ourselves // out of the peer score list. const currentModelIdentityKeys = useMemo(() => { const keys = new Set( [ summary.model_info.id, (summary as any).model_family_id, (summary.model_info as any).family_id, (summary.model_info as any).model_route_id, ...((summary as any).raw_model_ids ?? []), ].filter(Boolean) as string[] ) return keys }, [summary]) // The primary model_route_id that keys into comparison-index.by_model for // this page. Prefer an explicit route id; otherwise derive one. const currentModelRouteId = useMemo(() => { const explicit = (summary.model_info as any).model_route_id || (summary as any).model_route_id if (typeof explicit === "string" && explicit.length > 0) return explicit const id = summary.model_info.id || "" return id.replace(/[/]/g, "__") }, [summary]) // Cross-suite overlaps: walk `benchmark_index[]` (already pre-filtered by // `cleanHierarchy` to canonicals appearing in ≥2 distinct families) and // resolve this model's score in each appearance via `comparisonIndex`. // Aggregate per canonical with mean, SD, and 95% CI from Student's-t // (df=N-1). N=2 widths are very wide on purpose: with two samples we // genuinely don't know the spread, and surfacing that beats fake // precision. const overlapsRows = useMemo(() => { if (!evalHierarchy?.benchmark_index || !comparisonIndex) return [] const familyDisplayByKey = new Map() for (const fam of evalHierarchy.families ?? []) { familyDisplayByKey.set(fam.key, fam.display_name) } const byModel = comparisonIndex.by_model[currentModelRouteId] ?? {} const lookupModelScore = ( evalId: string, metric: ComparisonMetricEntry, ): number | null => { const cell = byModel[evalId]?.[metric.metric_summary_id] if (cell != null && Number.isFinite(cell.score)) return cell.score for (const row of metric.scores) { if ( currentModelIdentityKeys.has(row.model_route_id) || currentModelIdentityKeys.has(row.model_family_id) ) { if (Number.isFinite(row.score)) return row.score } } return null } const tCrit95: Record = { 1: 12.706, 2: 4.303, 3: 3.182, 4: 2.776, 5: 2.571, 6: 2.447, 7: 2.365, 8: 2.306, 9: 2.262, 10: 2.228, 15: 2.131, 20: 2.086, 29: 2.045, } const tFor = (df: number): number => { if (df <= 0) return 12.706 if (df >= 30) return 2.0 const known = [29, 20, 15, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1] for (const k of known) if (df >= k) return tCrit95[k] return 12.706 } const out: OverlapRow[] = [] for (const entry of evalHierarchy.benchmark_index) { const bestPerFamily = new Map() for (const appearance of entry.appearances ?? []) { const familyKey = appearance.family_key const familyName = familyDisplayByKey.get(familyKey) ?? familyKey for (const evalId of appearance.eval_summary_ids ?? []) { const evalEntry = comparisonIndex.evals[evalId] if (!evalEntry) continue const targetMetric = evalEntry.metrics.find( (m) => !isStderrMetricId(m.metric_summary_id) && /accuracy|score|exact|pass|win|mean/i.test(m.metric_name ?? ""), ) ?? evalEntry.metrics.find((m) => !isStderrMetricId(m.metric_summary_id)) ?? evalEntry.metrics[0] if (!targetMetric) continue const score = lookupModelScore(evalId, targetMetric) if (score == null || !Number.isFinite(score)) continue const unit = targetMetric.unit ?? null const isPercent = (unit ?? "").toLowerCase().match(/percent|%|pct/) != null const display = isPercent || score > 1.5 ? `${score.toFixed(1)}%` : `${(score * 100).toFixed(1)}%` if (!bestPerFamily.has(familyKey)) { bestPerFamily.set(familyKey, { familyKey, familyName, evalSummaryId: evalId, metricSummaryId: targetMetric.metric_summary_id, metricName: targetMetric.metric_name ?? "", score, displayScore: display, unit, }) } } } // Dedupe appearances whose score is byte-identical: same model // scoring exactly the same value across two "different" families // is the canonical false-flag for a benchmark surfaced twice // under different family wrappers (BBH was listed under both // big-bench and big-bench-hard with the same eval_summary_id and // therefore the same scores). Comparing scores at full precision // avoids collapsing genuinely-different reports that happen to // round to the same display value. // // Tie-break: when two appearances have identical scores, prefer // the non-llm-stats one. llm-stats is an aggregator and likely // republished the canonical source's number — so when an // independent family reports the same value, the llm-stats copy // is the duplicate, not the source. Sorting llm-stats to the back // before the seen-score scan makes the first-wins dedupe drop the // llm-stats appearance. const allRaw = Array.from(bestPerFamily.values()) const isAggregator = (familyKey: string) => familyKey === "llm-stats" allRaw.sort((a, b) => { const aAgg = isAggregator(a.familyKey) ? 1 : 0 const bAgg = isAggregator(b.familyKey) ? 1 : 0 return aAgg - bAgg }) const seenScores = new Set() const collected: OverlapAppearance[] = [] for (const c of allRaw) { if (seenScores.has(c.score)) continue seenScores.add(c.score) collected.push(c) } if (collected.length < 2) continue const highCount = collected.filter((c) => Math.abs(c.score) > 1.5).length const lowCount = collected.length - highCount const useHigh = highCount >= lowCount const scaled = collected.map((c) => { const isHigh = Math.abs(c.score) > 1.5 const score = useHigh ? isHigh ? c.score : c.score * 100 : isHigh ? c.score / 100 : c.score return { ...c, score } }) const scores = scaled.map((s) => s.score) const mean = scores.reduce((a, b) => a + b, 0) / scores.length const variance = scores.length > 1 ? scores.reduce((a, b) => a + (b - mean) ** 2, 0) / (scores.length - 1) : 0 const stddev = Math.sqrt(variance) const ci95 = scores.length >= 2 ? { low: mean - tFor(scores.length - 1) * (stddev / Math.sqrt(scores.length)), high: mean + tFor(scores.length - 1) * (stddev / Math.sqrt(scores.length)), } : null out.push({ canonicalKey: entry.key, canonicalDisplayName: entry.display_name, appearances: scaled.sort((a, b) => b.score - a.score), mean, stddev, min: Math.min(...scores), max: Math.max(...scores), ci95, isPercentScale: useHigh, }) } out.sort( (a, b) => b.appearances.length - a.appearances.length || a.canonicalDisplayName.localeCompare(b.canonicalDisplayName), ) return out }, [ evalHierarchy, comparisonIndex, currentModelRouteId, currentModelIdentityKeys, ]) // Per-(eval, metric) leaderboards sourced from comparison-index.json. const benchmarkHistograms = useMemo>(() => { const result = new Map() if (!comparisonIndex) return result const currentModelName = getModelDisplayName(summary.model_info.name) // Resolve every eval_summary_id we care about from the current model's // benchmarkGroups — this is the intersection of "what this model reports" // and "what comparison-index covers". const wantedEvalIds = new Set() for (const group of benchmarkGroups) { for (const variant of group.variants) { if (variant.evaluation.eval_summary_id) { wantedEvalIds.add(variant.evaluation.eval_summary_id) } } } const byModelForCurrent = comparisonIndex.by_model[currentModelRouteId] ?? {} for (const evalId of wantedEvalIds) { const evalEntry = comparisonIndex.evals[evalId] if (!evalEntry) continue for (const metric of evalEntry.metrics) { if (isStderrMetricId(metric.metric_summary_id)) continue const histKey = histKeyFor(evalId, metric.metric_summary_id) const lowerIsBetter = Boolean(metric.lower_is_better) // The current model's own row (if present) lives both in scores[] and // in by_model. We look it up by any of the known identity keys and // pull out its score/rank/submission info. let currentRow: ComparisonScoreEntry | undefined for (const s of metric.scores) { if ( currentModelIdentityKeys.has(s.model_route_id) || currentModelIdentityKeys.has(s.model_family_id) ) { currentRow = s break } } const byModelRow = byModelForCurrent[evalId]?.[metric.metric_summary_id] const currentScore = currentRow?.score ?? byModelRow?.score if (currentScore == null || !Number.isFinite(currentScore)) { // We don't have a score on this (eval, metric) — skip the histogram. // The tab will just not render. continue } const currentModelRank = currentRow != null ? { position: currentRow.rank, total: currentRow.total } : byModelRow != null ? { position: byModelRow.rank, total: byModelRow.total } : null // Peer rows = everything in scores[] that isn't us. Backend already // sorts best-first in the metric's own direction; we preserve that. const peerRows = metric.scores.filter( (s) => !currentModelIdentityKeys.has(s.model_route_id) && !currentModelIdentityKeys.has(s.model_family_id) ) const defaults = new Set() if (peerRows.length > 0) { // Best and worst come straight off the pre-sorted list. defaults.add(peerRows[0].model_route_id) defaults.add(peerRows[peerRows.length - 1].model_route_id) // Two peers closest to the current score. const closest = [...peerRows] .sort( (a, b) => Math.abs(a.score - currentScore) - Math.abs(b.score - currentScore) ) .filter((p) => !defaults.has(p.model_route_id)) .slice(0, 2) for (const p of closest) defaults.add(p.model_route_id) } const extras = extraModelsByBenchmark[histKey] ?? [] const selectedIds = new Set([...defaults, ...extras]) const peerBars: HistogramBar[] = peerRows .filter((p) => selectedIds.has(p.model_route_id)) .map((p) => ({ modelId: p.model_route_id, // Fall back to model_family_id when the registry has no display // name for this model. ~86% of score entries today land here; // root cause (registry coverage) is Step 2 work. The id is // already human-readable in this codebase ("anthropic/Sonnet 4.5"), // so the fallback is more useful than "Unknown Model". modelName: getModelDisplayName(p.model_family_name || p.model_family_id), score: p.score, isCurrent: false, isDefault: defaults.has(p.model_route_id), submissionCount: p.submission_count, submissionAxis: p.submission_axis, headlineRunLabel: p.headline_run_label, submissions: p.submissions, variantKey: p.variant_key, })) const currentBar: HistogramBar = { modelId: currentModelRouteId, modelName: currentModelName, score: currentScore, isCurrent: true, isDefault: true, submissionCount: currentRow?.submission_count ?? byModelRow?.submission_count ?? 1, submissionAxis: currentRow?.submission_axis ?? byModelRow?.submission_axis ?? "default", headlineRunLabel: currentRow?.headline_run_label, submissions: currentRow?.submissions, variantKey: currentRow?.variant_key, } const bars = [currentBar, ...peerBars].sort((a, b) => lowerIsBetter ? a.score - b.score : b.score - a.score ) const availableModels = peerRows .filter((p) => !selectedIds.has(p.model_route_id)) .map((p) => ({ id: p.model_route_id, name: p.model_family_name, score: p.score, submissionCount: p.submission_count, submissionAxis: p.submission_axis, })) result.set(histKey, { histKey, evalSummaryId: evalId, metricSummaryId: metric.metric_summary_id, metricName: metric.metric_name, metricGroup: metric.group, lowerIsBetter, unit: metric.unit, bars, availableModels, defaultIds: defaults, currentModelRank, }) } } return result }, [ benchmarkGroups, comparisonIndex, currentModelIdentityKeys, currentModelRouteId, extraModelsByBenchmark, summary.model_info.name, ]) // A plotbox can expose a top-level "view" selector (slices, child // benchmarks, components) and an optional metric tab rail beneath the chart. // Plotbox grouping is driven entirely by comparison-index's own // family_id so it stays in sync with the backend. type PlotboxMetricTab = { tabKey: string label: string histKey: string evalSummaryId: string metricSummaryId: string evalDisplayName: string evalEntry: ComparisonEvalEntry metricEntry: ComparisonMetricEntry isRollup: boolean group: BenchmarkGroup variant: BenchmarkVariant } type PlotboxView = { viewKey: string label: string evalDisplayName: string evalEntry: ComparisonEvalEntry isRollup: boolean group: BenchmarkGroup tabs: PlotboxMetricTab[] } type PlotboxUnit = { unitKey: string /** Plotbox-scope key. Composite key for composite-rooted plotboxes, * otherwise the benchmark / family key. Drives the dropdown and * the in-card title. */ familyKey: string /** Plotbox-scope display name (composite name for composites, * benchmark/family name for standalone benchmarks). */ familyName: string /** Outer family that owns this plotbox in the hierarchy. The grid * render groups composite plotboxes by `parentFamilyKey` so the * HELM family header sits above its `HELM Classic` / `HELM Safety` * plotboxes. Falls back to `familyKey` for standalone families. */ parentFamilyKey: string parentFamilyDisplayName: string category: CategoryType kind: "single-eval" | "multi-eval" childKindLabel: "metric" | "benchmark" | "component" | "slice" | null views: PlotboxView[] primaryGroup: BenchmarkGroup } // Strip the family name from a child's display so tabs read "Korean" rather // than "Global MMLU Lite Korean" and "Math" rather than "Reward Bench 2 Math". const stripFamilyPrefix = (label: string, familyName: string): string => { if (!familyName) return label const trimmed = label.trim() const fam = familyName.trim() if (trimmed.toLowerCase() === fam.toLowerCase()) return "Overall" if (trimmed.toLowerCase().startsWith(fam.toLowerCase() + " ")) { return trimmed.slice(fam.length).trim() } return trimmed } const plotboxUnits = useMemo(() => { if (!comparisonIndex) return [] type ResolvedGroup = { group: BenchmarkGroup evalEntry: ComparisonEvalEntry } type Bucket = { bucketKey: string parentFamilyKey: string parentFamilyDisplayName: string compositeKey: string | null compositeDisplayName: string | null bucketDisplayName: string category: CategoryType resolved: ResolvedGroup[] } // Composite-level bucketing. For evals that hierarchy.json places under a // composite (e.g. HELM Classic, HELM Safety), all evals in the same // composite share a bucket. Evals with no composite (standalone // benchmarks like AIME, MATH-500) each get their own bucket. The // grid render then groups bucket plotboxes by `parentFamilyKey`, so a // family like HELM shows one section header with N composite plotboxes // beneath it; singleton families show one plotbox under their own // header. const buckets = new Map() for (const group of filteredBenchmarkGroups) { const evalId = group.variants.find((v) => v.evaluation.eval_summary_id) ?.evaluation.eval_summary_id if (!evalId) continue const evalEntry = comparisonIndex.evals[evalId] if (!evalEntry) continue // Prefer hierarchy.json grouping. The comparison-index family_id is // null for ~7% of evals (e.g. CySE2 composites) and points at the // leaf for singleton families, so the hierarchy is the only source // that captures family→composite groupings authoritatively. // For evals not enumerated in any family's `eval_summary_ids` // (e.g. `artificial-analysis-llms%2Faime-2025`, where only the // base `…%2Faime` variant is listed at family level), prefer // source-prefix inference over `evalEntry.family_id` so the // variant lands under the correct organizational family // (`artificial-analysis`) instead of synthesising a phantom // `aime` parent section. const hierarchyLocation = hierarchyIndex?.get(evalId) ?? null const sourcePrefix = evalId.includes("%2F") ? evalId.split("%2F")[0] : null const inferredSourceFamily = !hierarchyLocation && sourcePrefix ? sourcePrefixFamily.get(sourcePrefix) ?? null : null const parentFamilyKey = hierarchyLocation?.familyKey ?? inferredSourceFamily?.key ?? evalEntry.family_id ?? evalId const parentFamilyDisplayName = hierarchyLocation?.familyDisplayName || inferredSourceFamily?.displayName || evalEntry.family_display_name || evalEntry.display_name || parentFamilyKey const compositeKey = hierarchyLocation?.compositeKey ?? null const compositeDisplayName = hierarchyLocation?.compositeDisplayName ?? null // Bucketing precedence: // composite > hierarchy benchmark > group key // Standalone benchmarks with N split eval rows (Fibble Arena's // 1-/2-/3-/4-/5-lies, AgentHarm's category siblings) all resolve // to the same `benchmarkKey` post-cleanHierarchy, so bucketing on // it groups every split into one plotbox. Without this they each // get their own group key and render as N separate plotboxes, // which contradicts the cleaned hierarchy's standalone-with- // splits intent. const benchmarkKey = hierarchyLocation?.benchmarkKey ?? null const benchmarkDisplayName = hierarchyLocation?.benchmarkDisplayName ?? null const bucketKey = compositeKey ? `${parentFamilyKey}::comp::${compositeKey}` : benchmarkKey ? `${parentFamilyKey}::bench::${benchmarkKey}` : `${parentFamilyKey}::bench::${group.key}` const bucketDisplayName = compositeDisplayName ?? benchmarkDisplayName ?? evalEntry.display_name ?? group.title ?? parentFamilyDisplayName const bucket = buckets.get(bucketKey) ?? { bucketKey, parentFamilyKey, parentFamilyDisplayName, compositeKey, compositeDisplayName, bucketDisplayName, category: group.category, resolved: [] as ResolvedGroup[], } bucket.resolved.push({ group, evalEntry }) buckets.set(bucketKey, bucket) } const variantFor = ( group: BenchmarkGroup, metricSummaryId: string ): BenchmarkVariant => { return ( group.variants.find( (v) => v.result.metric_summary_id === metricSummaryId ) ?? group.variants[0] ) } const metricTabFor = ( group: BenchmarkGroup, evalEntry: ComparisonEvalEntry, metric: ComparisonMetricEntry, variant: BenchmarkVariant, isRollup: boolean ): PlotboxMetricTab => ({ tabKey: `${evalEntry.eval_summary_id}::${metric.metric_summary_id}`, label: deriveMetricTabLabel(metric.metric_name, metric.metric_summary_id), histKey: histKeyFor(evalEntry.eval_summary_id, metric.metric_summary_id), evalSummaryId: evalEntry.eval_summary_id, metricSummaryId: metric.metric_summary_id, evalDisplayName: evalEntry.display_name || group.title, evalEntry, metricEntry: metric, isRollup, group, variant, }) const units: PlotboxUnit[] = [] for (const bucket of buckets.values()) { const { bucketKey, parentFamilyKey, parentFamilyDisplayName, bucketDisplayName, category, resolved, } = bucket if (resolved.length === 1) { // One eval in scope — slices/splits become the view selector while // metrics move to a compact tab rail beneath the chart. const { group, evalEntry } = resolved[0] const evalDisplay = evalEntry.display_name || group.title const singleEvalViewBuckets = new Map< string, { viewKey: string; label: string; variants: BenchmarkVariant[] } >() for (const variant of group.variants) { const viewKey = variant.sliceLabel ? `slice:${normalizeDisplayKey(variant.sliceLabel)}` : "default" const label = variant.sliceLabel || "Overall" const bucketForView = singleEvalViewBuckets.get(viewKey) ?? { viewKey, label, variants: [], } bucketForView.variants.push(variant) singleEvalViewBuckets.set(viewKey, bucketForView) } const views: PlotboxView[] = Array.from(singleEvalViewBuckets.values()) .map((viewBucket) => { const tabs = evalEntry.metrics .filter((metric) => !isStderrMetricId(metric.metric_summary_id)) .map((metric) => { const metricVariants = viewBucket.variants.filter( (variant) => variant.result.metric_summary_id === metric.metric_summary_id ) const variant = metricVariants.find((candidate) => !candidate.setupLabel) ?? metricVariants[0] ?? null if (!variant) return null return metricTabFor(group, evalEntry, metric, variant, false) }) .filter((tab): tab is PlotboxMetricTab => tab != null) if (tabs.length === 0) return null return { viewKey: viewBucket.viewKey, label: viewBucket.label, evalDisplayName: evalDisplay, evalEntry, isRollup: viewBucket.viewKey === "default", group, tabs, } }) .filter((view): view is PlotboxView => view != null) .sort((a, b) => { if (a.viewKey === "default") return -1 if (b.viewKey === "default") return 1 return a.label.localeCompare(b.label) }) if (views.length === 0) continue units.push({ unitKey: `eval:${bucketKey}`, familyKey: bucketKey, familyName: evalDisplay, parentFamilyKey, parentFamilyDisplayName, category, kind: "single-eval", childKindLabel: views.length > 1 ? "slice" : null, views, primaryGroup: group, }) continue } // Multi-eval composite — the view selector chooses among child evals // and each view exposes that eval's metrics in the bottom tab rail. // Rollup row = this eval IS the family root, i.e. its benchmark id // matches the family id. For multi-benchmark composites this is rare // (HELM Classic has no "helm-classic" benchmark), so rollup typically // stays null and the children render as siblings. const rollup = resolved.find( (r) => r.evalEntry.benchmark_id != null && r.evalEntry.benchmark_id === r.evalEntry.family_id, ) ?? null const children = rollup ? resolved.filter((r) => r !== rollup) : resolved const ordered: ResolvedGroup[] = rollup ? [rollup, ...children] : children const views: PlotboxView[] = ordered .map((r) => { const rawLabel = r.evalEntry.display_name || r.group.title const label = r === rollup ? "Overall" : stripFamilyPrefix(rawLabel, bucketDisplayName) const tabs = r.evalEntry.metrics .filter((metric) => !isStderrMetricId(metric.metric_summary_id)) .map((metric) => { const variant = variantFor(r.group, metric.metric_summary_id) if (!variant) return null return metricTabFor( r.group, r.evalEntry, metric, variant, r === rollup ) }) .filter((tab): tab is PlotboxMetricTab => tab != null) if (tabs.length === 0) return null return { viewKey: r.evalEntry.eval_summary_id, label: label || rawLabel, evalDisplayName: rawLabel, evalEntry: r.evalEntry, isRollup: r === rollup, group: r.group, tabs, } }) .filter((view): view is PlotboxView => view != null) if (views.length === 0) continue let hasComponent = false let hasSlice = false let hasDistinctLeaves = false for (const r of children) { // "Distinct leaf" = this child has an identity distinct from the // family root. Two cases: // 1. Non-slice child (e.g. HELM/MMLU under HELM family). // 2. Slice whose parent benchmark is not the family itself // (e.g. HELM/MMLU/anatomy — slice of MMLU under HELM, where // parent="mmlu" ≠ family="helm"). In a singleton family the // slice's parent equals the family and this case collapses // back to "slice", as expected. const evalEntry = r.evalEntry const isDistinctLeaf = evalEntry.is_slice === false || (evalEntry.parent_benchmark_id != null && evalEntry.parent_benchmark_id !== evalEntry.family_id) if (isDistinctLeaf) { hasDistinctLeaves = true } if (r.group.variants[0]?.evaluation.benchmark_component_key ?? null) { hasComponent = true } else { hasSlice = true } } const childKindLabel: PlotboxUnit["childKindLabel"] = hasComponent && hasSlice ? "component" : hasComponent ? "metric" : hasDistinctLeaves ? "benchmark" : "slice" units.push({ unitKey: `composite:${bucketKey}`, familyKey: bucketKey, familyName: bucketDisplayName, parentFamilyKey, parentFamilyDisplayName, category, kind: "multi-eval", childKindLabel, views, primaryGroup: (rollup ?? children[0] ?? resolved[0]).group, }) } return units }, [comparisonIndex, filteredBenchmarkGroups, hierarchyIndex, sourcePrefixFamily]) // Benchmark-mode units: derived from `plotboxUnits` (already composite- // rooted, so splits like Fibble Arena's 1-/2-/3-lies variants and // AIR-Bench's per-category slices are bundled inside ONE plotbox via // the cleaner's flatten + slice-folding) with a canonical-dedupe pass // layered on top. When two composite units resolve to the same // benchmark_index canonical (e.g. MMLU appearing under HELM and // lighteval), the first occurrence wins and the rest re-enter as // whisker overlays inside `renderPlotbox`. Splits never merge across // suites — only benchmark-level identities do. const benchmarkLeafUnits = useMemo(() => { if (plotboxUnits.length === 0) return [] const evalIdsForUnit = (unit: PlotboxUnit): string[] => { const out: string[] = [] for (const view of unit.views) { for (const tab of view.tabs) { if (tab.evalSummaryId) out.push(tab.evalSummaryId) } } return out } const canonicalForUnit = (unit: PlotboxUnit): string | null => { // Pick the canonical identity by polling each eval id under the // unit and taking the first benchmark_index hit. Within a single // composite the producer often groups several near-canonical // variants (helm-classic carries both `mmlu` and `mmlu-pro`), // so we don't insist they all agree — the first wins. for (const evalId of evalIdsForUnit(unit)) { const indexEntry = benchmarkIndexLookup.get(evalId) if (indexEntry) return indexEntry.canonicalKey } return null } const seenCanonical = new Set() const out: PlotboxUnit[] = [] for (const unit of plotboxUnits) { const canonical = canonicalForUnit(unit) if (canonical) { if (seenCanonical.has(canonical)) continue seenCanonical.add(canonical) } out.push(unit) } return out }, [plotboxUnits, benchmarkIndexLookup]) const [activeViewByUnit, setActiveViewByUnit] = useState>({}) const [activeMetricByUnit, setActiveMetricByUnit] = useState>({}) const getActiveView = (unit: PlotboxUnit): PlotboxView => { const explicit = activeViewByUnit[unit.unitKey] if (explicit) { const match = unit.views.find((view) => view.viewKey === explicit) if (match) return match } return unit.views[0] } const getActiveMetricTab = ( unit: PlotboxUnit, view: PlotboxView ): PlotboxMetricTab => { const explicit = activeMetricByUnit[unit.unitKey] if (explicit) { const match = view.tabs.find((tab) => tab.tabKey === explicit) if (match) return match } return view.tabs[0] } const submissionChipCopy = ( axis: SubmissionAxis, count: number, headlineLabel?: string ): { short: string; long: string } | null => { if (count <= 1 || axis === "default") return null const others = count - 1 const variantNoun = (n: number) => (n === 1 ? "variant" : "variants") switch (axis) { case "harness": return { short: `+${others} harness${others === 1 ? "" : "es"}`, long: headlineLabel ? `${headlineLabel} · +${others} harness${others === 1 ? "" : "es"}` : `+${others} harness${others === 1 ? "" : "es"}`, } case "variant": return { short: `+${others} ${variantNoun(others)}`, long: headlineLabel ? `${headlineLabel} · +${others} ${variantNoun(others)}` : `+${others} ${variantNoun(others)}`, } case "rerun": return { short: `+${others} re-run${others === 1 ? "" : "s"}`, long: headlineLabel ? `${headlineLabel} · +${others} re-run${others === 1 ? "" : "s"}` : `+${others} re-run${others === 1 ? "" : "s"}`, } case "mixed": return { short: `+${others} submissions`, long: `+${others} submissions`, } } } const reconcileHistogramScales = ( hist: BenchmarkHistogram ): { hist: BenchmarkHistogram; rescaled: boolean; averaged: boolean } => { // Collect every numeric score (primary bar + per-submission re-runs) so we // can tell whether some records come in on a 0-1 scale while others are // on a 0-100 scale. const allScores: number[] = [] for (const bar of hist.bars) { if (Number.isFinite(bar.score)) allScores.push(bar.score) for (const sub of bar.submissions ?? []) { if (Number.isFinite(sub.score)) allScores.push(sub.score) } } // Only reconcile score-like metrics (accuracy / proportion / percentage / // bare-number). Leave physical units like seconds, USD, ranks alone. const unitLc = (hist.unit ?? "").trim().toLowerCase() const isScoreLikeUnit = !unitLc || unitLc === "percentage" || unitLc === "percent" || unitLc === "%" || unitLc === "pct" || unitLc === "accuracy" || unitLc === "proportion" || unitLc === "pass@1" || unitLc === "score" || unitLc === "rate" const lowCount = allScores.filter((s) => s > 0 && s <= 1).length const highCount = allScores.filter((s) => s > 1).length const needsRescale = isScoreLikeUnit && allScores.length >= 2 && lowCount > 0 && highCount > 0 // Rescale the minority to match the majority. Break ties in favour of the // 0-100 scale since that's the app's default display. const rescaleLowTo100 = highCount >= lowCount const rescale = (score: number) => { if (!needsRescale || !Number.isFinite(score)) return score if (rescaleLowTo100 && score > 0 && score <= 1) return score * 100 if (!rescaleLowTo100 && score > 1) return score / 100 return score } let averaged = false const bars = hist.bars.map((bar) => { const rescaledSubmissions = bar.submissions?.map((sub) => ({ ...sub, score: rescale(sub.score), })) const rescaledHeadline = rescale(bar.score) // If the model has more than one submission after rescaling, use their // mean as the headline bar value. Readers can still see the per-run // spread through the whisker and the dropdown. const submissionValues = (rescaledSubmissions ?? []) .map((sub) => sub.score) .filter((score) => Number.isFinite(score)) const headlineScore = submissionValues.length > 1 ? submissionValues.reduce((sum, s) => sum + s, 0) / submissionValues.length : rescaledHeadline if (submissionValues.length > 1) averaged = true return { ...bar, score: headlineScore, submissions: rescaledSubmissions, } }) if (!needsRescale && !averaged) { return { hist, rescaled: false, averaged: false } } return { hist: { ...hist, bars }, rescaled: needsRescale, averaged } } const renderPlotbox = (unit: PlotboxUnit, enableWhisker: boolean = false) => { const activeView = getActiveView(unit) const activeTab = getActiveMetricTab(unit, activeView) if (!activeView || !activeTab) return null const hist = benchmarkHistograms.get(activeTab.histKey) // Fallback: no comparison rows loaded yet, or the metric has zero peers. // Still draw the current model's own bar from BenchmarkGroup data. const rawHist: BenchmarkHistogram = hist ?? { histKey: activeTab.histKey, evalSummaryId: activeTab.evalSummaryId, metricSummaryId: activeTab.metricSummaryId, metricName: activeTab.metricEntry.metric_name, metricGroup: activeTab.metricEntry.group, lowerIsBetter: Boolean(activeTab.metricEntry.lower_is_better), unit: activeTab.metricEntry.unit, bars: [ { modelId: currentModelRouteId, modelName: getModelDisplayName(summary.model_info.name), score: activeTab.variant.result.score_details.score, isCurrent: true, isDefault: true, submissionCount: 1, submissionAxis: "default", }, ], availableModels: [], defaultIds: new Set(), currentModelRank: null, } // When a benchmark is reported on both 0-1 and 0-100 scales (e.g. Wordle // Arena's win_rate surfaces as 76.9 in one submission and 0.409 in another), // plotting raw scores squashes the 0-1 submissions to a flat zero. Rescale // the minority scale onto the majority before computing the chart domain. const { hist: activeHist, rescaled, averaged } = reconcileHistogramScales(rawHist) const scores = activeHist.bars.map((b) => b.score) const rawMax = Math.max(...scores) const rawMin = Math.min(...scores) const hasSpread = rawMax !== rawMin const domainMin = hasSpread ? rawMin - (rawMax - rawMin) * 0.15 : Math.min(0, rawMin) const domainMax = hasSpread ? rawMax + (rawMax - rawMin) * 0.15 : rawMax === 0 ? 1 : rawMax * 1.2 const range = domainMax - domainMin || 1 const bestScore = activeHist.lowerIsBetter ? rawMin : rawMax const worstScore = activeHist.lowerIsBetter ? rawMax : rawMin let bestBarId: string | null = null let worstBarId: string | null = null if (hasSpread) { for (const b of activeHist.bars) { if (!bestBarId && b.score === bestScore) bestBarId = b.modelId if (!worstBarId && b.score === worstScore) worstBarId = b.modelId } } // Cross-family score range: when this benchmark also reports under // sibling families (per hierarchy.json's `benchmark_index[]`), look up // this model's score on each sibling and treat the resulting set as a // whisker overlay on the current bar. The match is by metric_name — // sibling evals use different metric_summary_ids but the same metric // (e.g. "accuracy" on AIME shows up in both artificial-analysis and // llm-stats). Best-effort: any sibling without a name-matching metric // simply doesn't contribute. const crossFamilyContribs = (() => { if (!enableWhisker) return [] as Array<{ familyName: string; score: number }> if (!comparisonIndex) return [] as Array<{ familyName: string; score: number }> const indexEntry = benchmarkIndexLookup.get(activeTab.evalSummaryId) if (!indexEntry) return [] const targetMetricName = activeTab.metricEntry.metric_name?.toLowerCase().trim() ?? "" // Producers label the same metric differently across families // ("Score" / "Accuracy" / "Acc" all refer to AIME's pass rate). To // make the whisker fire across these, match siblings in priority: // 1. exact metric_name (case-insensitive) // 2. metric_summary_id local-part (e.g. both end in `:score`) // 3. fall back to the sibling's first metric — best-effort, the // benchmark_index already vouches for canonical equality. const targetMetricLocal = activeTab.metricEntry.metric_summary_id ?.split("%3A") .pop() ?.toLowerCase() .trim() // `by_model` is keyed by URL-encoded model_route_id (`anthropic%2F…`) // but the page-level fallback for `currentModelRouteId` produces the // underscore form when no explicit route id is in `summary.model_info`. // Mirror the histogram-builder's two-pronged identity match so the // whisker fires even when the by_model lookup misses: scan the // sibling metric's `scores[]` and accept any row whose route or family // id is in `currentModelIdentityKeys`. const byModel = comparisonIndex.by_model[currentModelRouteId] ?? {} const familyDisplayByKey = new Map() for (const fam of evalHierarchy?.families ?? []) { familyDisplayByKey.set(fam.key, fam.display_name) } // Cross-family scores often arrive on different scales: vals-ai // reports AIME accuracy as a 0–100 percent (22.292) while // artificial-analysis reports the same benchmark as a 0–1 proportion // (0.355). Reconcile to the active histogram's scale before // computing the whisker so the band reflects real spread, not unit // mismatch. Same heuristic the bar-rescaler uses: if the active // histogram's bars are mostly >1, treat sibling raw scores ≤1 as // proportions and bump them up to match. const histScores = activeHist.bars .map((b) => b.score) .filter((s) => Number.isFinite(s)) const histMaxAbs = histScores.length ? Math.max(...histScores.map(Math.abs)) : 1 const histIsPercent = histMaxAbs > 1.5 const reconcileSibling = (score: number, siblingMetricUnit: string | null | undefined): number => { const u = (siblingMetricUnit ?? "").toLowerCase().trim() const siblingIsPercent = u === "percent" || u === "percentage" || u === "%" || u === "pct" || (!["proportion", "rate"].includes(u) && Math.abs(score) > 1.5) if (histIsPercent === siblingIsPercent) return score return histIsPercent ? score * 100 : score / 100 } const out: Array<{ familyName: string; score: number }> = [] for (const siblingId of indexEntry.siblingEvalIds) { if (siblingId === activeTab.evalSummaryId) continue const siblingEval = comparisonIndex.evals[siblingId] if (!siblingEval || siblingEval.metrics.length === 0) continue const matchByName = siblingEval.metrics.find( (m) => m.metric_name?.toLowerCase().trim() === targetMetricName, ) const matchByLocal = !matchByName && targetMetricLocal ? siblingEval.metrics.find( (m) => m.metric_summary_id?.split("%3A").pop()?.toLowerCase().trim() === targetMetricLocal, ) : null const siblingMetric = matchByName ?? matchByLocal ?? siblingEval.metrics[0] let siblingScore: number | null = null const byModelCell = byModel[siblingId]?.[siblingMetric.metric_summary_id] if (byModelCell != null && Number.isFinite(byModelCell.score)) { siblingScore = byModelCell.score } else { // Fallback: scan scores[] for a row matching any of the model's // identity keys. Covers route-id encoding mismatches. for (const row of siblingMetric.scores) { if ( currentModelIdentityKeys.has(row.model_route_id) || currentModelIdentityKeys.has(row.model_family_id) ) { if (Number.isFinite(row.score)) { siblingScore = row.score break } } } } if (siblingScore == null) continue const reconciledScore = reconcileSibling(siblingScore, siblingMetric.unit) // siblingId looks like "%2F"; recover the // family name from the hierarchy where possible, fall back to the slug. const familyKey = siblingId.split("%2F")[0] const familyName = familyDisplayByKey.get(familyKey) ?? familyKey out.push({ familyName, score: reconciledScore }) } return out })() const ownScore = activeTab.variant.result.score_details.score const crossFamilyScores = [ ...crossFamilyContribs.map((c) => c.score), ...(Number.isFinite(ownScore) ? [ownScore] : []), ] const hasCrossFamilyWhisker = crossFamilyContribs.length > 0 const crossFamilyMin = hasCrossFamilyWhisker ? Math.min(...crossFamilyScores) : null const crossFamilyMax = hasCrossFamilyWhisker ? Math.max(...crossFamilyScores) : null // Stderr-based whisker for the current model's bar. With cross-family // overlays moved into the dedicated Overlaps view, stderr (when the // producer ships a paired `_stderr` metric) is the more useful // statistical cue here: it's the metric's own sampling-error, // independent of how many other suites also report this benchmark. // Reconciles to the histogram's display scale exactly like the // primary bar does so the whisker doesn't shrink when the score is // rescaled from 0-1 to 0-100. const stderrRaw = activeTab.variant.auxStderr const stderrUnit = activeTab.variant.auxStderrUnit const stderrIsPercent = (() => { const u = (stderrUnit ?? "").toLowerCase().trim() if (u === "percent" || u === "percentage" || u === "%" || u === "pct") return true if (u === "proportion" || u === "rate") return false return null })() const stderrReconciled = (() => { if (stderrRaw == null || !Number.isFinite(stderrRaw)) return null // Match the active histogram's scale: if bars are mostly 0-100 and the // stderr looks like a 0-1 proportion, scale up. Same heuristic as the // sibling reconciler, applied to a single value. const histScores = activeHist.bars .map((b) => b.score) .filter((s) => Number.isFinite(s)) const histMaxAbs = histScores.length ? Math.max(...histScores.map(Math.abs)) : 1 const histIsPercent = histMaxAbs > 1.5 let isPercent = stderrIsPercent if (isPercent == null) { // Fallback: assume stderr matches the bar's score scale. isPercent = Math.abs(activeTab.variant.result.score_details.score) > 1.5 } if (histIsPercent === isPercent) return stderrRaw return histIsPercent ? stderrRaw * 100 : stderrRaw / 100 })() const hasStderrWhisker = stderrReconciled != null && Number.isFinite(stderrReconciled) && stderrReconciled > 0 && Number.isFinite(ownScore) const stderrLow = hasStderrWhisker ? ownScore - stderrReconciled! : null const stderrHigh = hasStderrWhisker ? ownScore + stderrReconciled! : null void enableWhisker const rank = activeHist.currentModelRank const plotboxKey = unit.unitKey const hasViewSelector = unit.views.length > 1 const hasMetricTabs = activeView.tabs.length > 1 const activeViewIndex = Math.max( 0, unit.views.findIndex((view) => view.viewKey === activeView.viewKey) ) const childKindCount = unit.views.length - (unit.views.some((view) => view.isRollup) ? 1 : 0) const showChildKindBadge = hasViewSelector && unit.childKindLabel != null && childKindCount > 0 const childKindPlural = unit.childKindLabel === "metric" ? childKindCount === 1 ? "metric" : "metrics" : unit.childKindLabel === "slice" ? childKindCount === 1 ? "slice" : "slices" : unit.childKindLabel === "benchmark" ? childKindCount === 1 ? "benchmark" : "benchmarks" : childKindCount === 1 ? "component" : "components" const setPlotboxActiveView = (nextViewKey: string) => setActiveViewByUnit((prev) => ({ ...prev, [unit.unitKey]: nextViewKey, })) const setPlotboxActiveMetric = (nextTabKey: string) => setActiveMetricByUnit((prev) => ({ ...prev, [unit.unitKey]: nextTabKey, })) return (
{/* Header */}
{formatTagLabel(unit.category as unknown as string)} {showChildKindBadge && ( · {childKindCount} {childKindPlural} )} {rank && ( #{rank.position}{rank.total ? `/${rank.total}` : ""} )} {(() => { const relationship = activeTab.variant.evaluation.source_metadata.evaluator_relationship if (!relationship) return null const isFirst = relationship === "first_party" return ( · {getRelationshipShortLabel(relationship)} ) })()}
{activeHist.lowerIsBetter ? "Lower is better" : "Higher is better"} {hasStderrWhisker && ( · ↕ ±σ )} {!hasStderrWhisker && hasCrossFamilyWhisker && ( · ↕ {crossFamilyContribs.length + 1} reports )} {(averaged || rescaled) && (
{averaged && (
Bar = mean of re-runs
Bars show the mean across submissions for each model. Whiskers and the dropdown show individual re-runs.
)} {rescaled && (
Scales aligned
Submissions arrived on different scales (e.g. 0-1 and 0-100). The minority scale was auto-rescaled to match the majority.
)}
)}
{activeHist.availableModels.length > 0 && ( Add model {activeHist.availableModels.slice(0, 80).map((m) => ( { setExtraModelsByBenchmark((prev) => { const current = prev[activeHist.histKey] ?? [] if (current.includes(m.id)) return prev return { ...prev, [activeHist.histKey]: [...current, m.id] } }) }} className="flex items-center justify-between gap-4 text-xs" > {getModelDisplayName(m.name)} {formatRawScoreValue(m.score)} ))} )}
{/* View selector */} {hasViewSelector && (
View {activeViewIndex + 1}/{unit.views.length}
)} {/* Spacer pushes the chart to the bottom of the card so bar baselines align across plotboxes regardless of whether a tab row is present. */}
{/* Chart */}
{activeHist.bars.map((bar) => { const normalized = (bar.score - domainMin) / range const clampedNorm = Math.max(Math.min(normalized, 1), 0) const heightPct = Math.max(clampedNorm * 100, 4) const isExtra = !activeHist.defaultIds.has(bar.modelId) && !bar.isCurrent const isBest = bar.modelId === bestBarId && !bar.isCurrent const isWorst = bar.modelId === worstBarId && !bar.isCurrent && bestBarId !== worstBarId const submissionScores = (bar.submissions ?? []) .map((submission) => submission.score) .filter((score) => Number.isFinite(score)) const minSubmissionScore = submissionScores.length >= 2 ? Math.min(...submissionScores) : null const maxSubmissionScore = submissionScores.length >= 2 ? Math.max(...submissionScores) : null const minSubmissionPct = minSubmissionScore != null ? Math.max(0, Math.min(100, ((minSubmissionScore - domainMin) / range) * 100)) : null const maxSubmissionPct = maxSubmissionScore != null ? Math.max(0, Math.min(100, ((maxSubmissionScore - domainMin) / range) * 100)) : null const submissionSpanPct = minSubmissionPct != null && maxSubmissionPct != null ? Math.max(maxSubmissionPct - minSubmissionPct, 0.8) : 0 return (
{minSubmissionPct != null && maxSubmissionPct != null && ( )} {bar.isCurrent && hasStderrWhisker && (() => { const minPct = Math.max( 0, Math.min(100, ((stderrLow! - domainMin) / range) * 100), ) const maxPct = Math.max( 0, Math.min(100, ((stderrHigh! - domainMin) / range) * 100), ) const spanPct = Math.max(maxPct - minPct, 0.8) const tooltip = `±1 stderr: ${formatRawScoreValue(stderrLow!, activeHist.unit ?? undefined)}` + ` – ${formatRawScoreValue(stderrHigh!, activeHist.unit ?? undefined)}` + ` (σ ${formatRawScoreValue(stderrReconciled!, activeHist.unit ?? undefined)})` return ( ) })()} {bar.isCurrent && !hasStderrWhisker && hasCrossFamilyWhisker && (() => { const minPct = Math.max( 0, Math.min(100, ((crossFamilyMin! - domainMin) / range) * 100), ) const maxPct = Math.max( 0, Math.min(100, ((crossFamilyMax! - domainMin) / range) * 100), ) const spanPct = Math.max(maxPct - minPct, 0.8) const tooltip = `Cross-family range: ${formatRawScoreValue(crossFamilyMin!, activeHist.unit ?? undefined)}` + ` – ${formatRawScoreValue(crossFamilyMax!, activeHist.unit ?? undefined)}` + ` across ${crossFamilyContribs.length + 1} family appearance${crossFamilyContribs.length === 0 ? "" : "s"}: ` + crossFamilyContribs .map((c) => `${c.familyName} ${formatRawScoreValue(c.score, activeHist.unit ?? undefined)}`) .join(", ") return ( ) })()}
{formatRawScoreValue(bar.score)}
{isExtra && ( )}
{bar.modelName}
{(() => { const chip = submissionChipCopy( bar.submissionAxis, bar.submissionCount, bar.headlineRunLabel ) if (!chip) return null const submissions = bar.submissions ?? [] const trigger = ( ) if (submissions.length === 0) return trigger return ( {trigger} {bar.modelName} {chip.short} {submissions.map((s, i) => ( e.preventDefault()} > {s.run_kind} {s.run_label} {formatRawScoreValue(s.score, activeHist.unit ?? undefined)} ))} ) })()}
) })}
{hasSpread && domainMin > 0.0001 && (
axis zoomed: {formatRawScoreValue(domainMin, activeHist.unit ?? undefined)} –{" "} {formatRawScoreValue(domainMax, activeHist.unit ?? undefined)}
)} {hasMetricTabs && (
{activeView.tabs.map((tab) => ( ))}
)}
{!hist && (
{comparisonIndex ? "No peer scores for this metric." : "Loading comparison data…"}
)}
) } const documentedPct = Math.round( (summary.total_evaluations > 0 && reproducibilityResultsTotal > 0 ? Math.max(0, reproducibilityResultsTotal - reproducibilityGapCount) / reproducibilityResultsTotal : 1) * 100 ) return (
{/* ============================================================ Header — paper-style document hero ============================================================ */}
Eval Card · Registry Entry

{getModelDisplayName(summary.model_info.name)}

{getOrganizationDisplayName(summary.model_info.developer)} {summary.model_info.release_date && ( <> · Released {formatDate(summary.model_info.release_date).split(",")[0]} )} {summary.model_info.additional_details?.deployment_context && ( <> · {summary.model_info.additional_details.deployment_context} )} {formatParamsBillions(summary.model_info.additional_details?.params_billions) && ( <> · {formatParamsBillions(summary.model_info.additional_details?.params_billions)} )}
Registry ID
ec/models/{summary.model_info.id}
Documented
{documentedPct}%
{Math.max(0, reproducibilityResultsTotal - reproducibilityGapCount)} / {reproducibilityResultsTotal} reported
{/* ============================================================ Lede — paper-style abstract ============================================================ */}
{isResearchView ? (

{getModelDisplayName(summary.model_info.name)} reports{" "} {summary.total_evaluations} result{summary.total_evaluations === 1 ? "" : "s"} across{" "} {benchmarkGroups.length} benchmark{benchmarkGroups.length === 1 ? "" : "s"}, sourced from{" "} {reportingStats.organizationCount} reporting organization{reportingStats.organizationCount === 1 ? "" : "s"} ({reportingStats.sourceTypeCount} source type{reportingStats.sourceTypeCount === 1 ? "" : "s"}).{" "} {reportingStats.missingGenerationConfigs > 0 ? `${reportingStats.missingGenerationConfigs} entries are missing generation config, limiting cross-slice comparability.` : "Generation configuration is present across the result set."}

{(setupDrivenBenchmarkCount > 0 || sliceDrivenBenchmarkCount > 0) && (

Decomposition: {setupDrivenBenchmarkCount} setup-aware ·{" "} {sliceDrivenBenchmarkCount} slice-aware. {reportingStats.libraryList.length > 0 && ( <> {" "}Eval libraries: {reportingStats.libraryList.join(", ")}. )}

)}
) : (

{policySummary.testedByCopy}

{policySummary.reproducibilityCopy && (
Reproducibility gap {policySummary.reproducibilityCopy}
)}

{policySummary.comparabilityCopy} {policySummary.sizeCaveat ? ` ${policySummary.sizeCaveat}` : ""}

)}
{/* ============================================================ §1 Identification — hairline data list ============================================================ */}

§1 Identification

Model record
Model name
{getModelDisplayName(summary.model_info.name)}
Developer
{getOrganizationDisplayName(summary.model_info.developer)}
{summary.model_info.model_version && ( <>
Version
{summary.model_info.model_version}
)} {summary.model_info.release_date && ( <>
Released
{formatDate(summary.model_info.release_date).split(",")[0]}
)} {formatParamsBillions(summary.model_info.additional_details?.params_billions) && ( <>
Parameters
{formatParamsBillions(summary.model_info.additional_details?.params_billions)}
)} {(summary.model_info.architecture || summary.model_info.inference_engine) && ( <>
Architecture
{summary.model_info.architecture || summary.model_info.inference_engine}
)} {(summary.model_info.modalities?.input?.length || summary.model_info.modalities?.output?.length) && ( <>
Modalities
{(summary.model_info.modalities?.input?.join(", ") || "Text")} → {(summary.model_info.modalities?.output?.join(", ") || "Text")}
)} {summary.model_info.additional_details?.deployment_context && ( <>
Access
{summary.model_info.additional_details.deployment_context}
)}
System ID
{summary.model_info.id}
{summary.model_info.model_url && ( <>
Reference
{summary.model_info.model_url.replace(/^https?:\/\//, "").replace(/\/$/, "")}
)}
Updated
{formatDate(summary.last_updated).split(",")[0]}
{/* ============================================================ §2 Coverage of registry benchmarks ============================================================ */}

§2 Coverage of registry benchmarks

{filteredBenchmarkGroups.length} shown · {benchmarkGroups.length} reported

{isResearchView ? "Benchmark-first view of this model's reported results, grouped by category. Setup spread and slice-vs-setup differences surface up-front." : "The public evidence behind this model, grouped by category. The strongest and most variable signals are listed first."} {policyHighlights.length > 0 && !isResearchView && ( <> {" "} {policyHighlights.length} headline finding{policyHighlights.length === 1 ? "" : "s"}. )}

{/* Strong / Weak / Spread — hairline rows */} {(strongRankedBenchmarks.length > 0 || weakRankedBenchmarks.length > 0 || repeatedBenchmarkCount > 0) && (
{strongRankedBenchmarks.length > 0 && ( <>
Ranks high in
{strongRankedBenchmarks.map((group) => { const rank = getGroupPeerRank(group, modelIds, peerRanks) return ( ) })}
)} {weakRankedBenchmarks.length > 0 && ( <>
Ranks low in
{weakRankedBenchmarks.map((group) => { const rank = getGroupPeerRank(group, modelIds, peerRanks) return ( ) })}
)} {repeatedBenchmarkCount > 0 && ( <>
Slice spread
{repeatedBenchmarkCount} benchmark{repeatedBenchmarkCount === 1 ? "" : "s"} include multiple slices or setups.
)} {benchmarkGroups.some((g) => (g as { __scaleWarning?: boolean }).__scaleWarning) && ( <>
Scale notes
Some scores were auto-renormalized due to mixed scales (e.g., 0–1 vs 0–100).
)}
)}
{/* ============================================================ §3 Who reports what — evaluator-mix donut + per-category bars ============================================================ */} {evaluatorMix.grand > 0 && (

§3 Who reports what

First-party · third-party · per category
)} {/* ============================================================ §4 Reported metrics — filter bar + view toggle + grid/list ============================================================ */}

§4 Reported metrics

{filteredBenchmarkGroups.length} shown
{/* Filter bar */}
setBenchmarkSearch(event.target.value)} placeholder="Search benchmarks or setups…" />
{/* Grid/list toggle lives here so it can be hidden in Overlaps mode without yanking layout in the section header above. */} {groupingMode !== "overlaps" && (
)}
{groupingMode === "source" && availableFamilies.length > 0 && (
Family {availableFamilies.map(({ key, name }) => { const isSelected = selectedFamilies.includes(key) return ( ) })}
)} {groupingMode === "category" && availableCategories.length > 0 && (
Category {availableCategories.map((category) => { const isSelected = selectedCategories.includes(category) return ( ) })}
)} {groupingMode === "overlaps" ? (() => { const query = benchmarkSearch.trim().toLowerCase() const visibleOverlaps = query ? overlapsRows.filter( (r) => r.canonicalDisplayName.toLowerCase().includes(query) || r.canonicalKey.toLowerCase().includes(query) || r.appearances.some((a) => a.familyName.toLowerCase().includes(query)), ) : overlapsRows return visibleOverlaps.length === 0 ? (
{query ? "No overlaps match your search" : "No cross-suite overlaps found for this model"}
) : (
Benchmark
N
Mean (95% CI)
Range
Sources
{visibleOverlaps.map((row, idx) => { const fmt = (v: number) => row.isPercentScale ? `${v.toFixed(1)}%` : `${(v * 100).toFixed(1)}%` const ciLabel = row.ci95 ? row.appearances.length === 2 ? `±${(((row.ci95.high - row.ci95.low) / 2) || 0).toFixed(1)} (n=2, wide)` : `[${fmt(row.ci95.low)}, ${fmt(row.ci95.high)}]` : "—" return (
{row.canonicalDisplayName}
{row.canonicalKey}
{row.appearances.length}
{fmt(row.mean)}
{ciLabel}
{fmt(row.min)} – {fmt(row.max)}
Δ {fmt(row.max - row.min)}
{row.appearances.map((app) => ( {app.familyName} · {fmt(app.score)} ))}
) })}
) })() : filteredBenchmarkGroups.length === 0 || (benchmarkViewMode === "grid" && plotboxUnits.length === 0) ? (
No benchmarks match the current search or category filters
) : benchmarkViewMode === "grid" && groupingMode === "source" ? ( /* Hierarchy mode — section per family, one plotbox per composite (or per standalone benchmark) under it. Inside each plotbox the view selector still drills into the composite's benchmarks / slices. No cross-family whisker. */ (() => { const byFamily = new Map< string, { familyKey: string familyDisplayName: string category: CategoryType units: PlotboxUnit[] } >() for (const unit of plotboxUnits) { const entry = byFamily.get(unit.parentFamilyKey) ?? { familyKey: unit.parentFamilyKey, familyDisplayName: unit.parentFamilyDisplayName, category: unit.category, units: [] as PlotboxUnit[], } entry.units.push(unit) byFamily.set(unit.parentFamilyKey, entry) } const families = Array.from(byFamily.values()) return (
{families.map((fam) => { const compositeCount = fam.units.length const totalBenchmarks = fam.units.reduce( (sum, u) => sum + u.views.reduce((vs, view) => vs + view.tabs.length, 0), 0, ) return (
{fam.familyDisplayName} {compositeCount}{" "} {compositeCount === 1 ? "plot" : "plots"} {totalBenchmarks !== compositeCount && ( <> {" "}· {totalBenchmarks} benchmark {totalBenchmarks === 1 ? "" : "s"} )}
{fam.units.map((unit) => renderPlotbox(unit, false))}
) })}
) })() ) : benchmarkViewMode === "grid" ? ( /* Category mode — same composite/standalone units as Source mode, but the top-level grouping switches to the curated category tag so similarly-tagged benchmarks cluster across families. No cross-family dedup, no whiskers — overlaps live in their own dedicated view. */ (() => { const categoryOrder = new Map( availableCategories.map((cat, i) => [cat, i]) ) const byCategory = new Map() for (const unit of plotboxUnits) { const list = byCategory.get(unit.category) ?? [] list.push(unit) byCategory.set(unit.category, list) } const orderedCategories = Array.from(byCategory.keys()).sort( (a, b) => (categoryOrder.get(a) ?? 999) - (categoryOrder.get(b) ?? 999) ) return (
{orderedCategories.map((category) => { const units = byCategory.get(category) ?? [] const totalBenchmarks = units.length return (
{formatTagLabel(category)} {totalBenchmarks} benchmark{totalBenchmarks === 1 ? "" : "s"}
{units.map((unit) => renderPlotbox(unit, false))}
) })}
) })() ) : ( /* List view — accordions per benchmark family, grouped by category. Family bucketing mirrors the grid view's plotboxUnits logic. */
{(() => { const allFamilyKeys = listFamiliesByCategory.flatMap(({ families }) => families.map((f) => f.familyKey) ) const allExpanded = allFamilyKeys.length > 0 && allFamilyKeys.every((k) => expandedFamilies.has(k)) return (
) })()} {listFamiliesByCategory.map(({ category, families }) => { const totalRows = families.reduce((sum, f) => sum + f.totalRows, 0) const totalBenchmarks = families.reduce((sum, f) => sum + f.groups.length, 0) type ListRow = { group: BenchmarkGroup variant: BenchmarkVariant /** Present when this row is the consolidated representative * for a multi-source canonical benchmark — see the * `groupDuplicatesInList` toggle. */ aggregate?: MergedRowAggregate } const renderRow = (row: ListRow, isLast: boolean) => { const unit = row.variant.result.metric_config.unit const lower = row.variant.result.metric_config.lower_is_better const variantLabel = getVariantPrimaryLabel(row.variant, row.group.title) const rel = row.variant.evaluation.source_metadata.evaluator_relationship const agg = row.aggregate const meanDisplay = agg ? `${(agg.mean * 100).toFixed(1)}%` : row.variant.displayScore const rangeDisplay = agg ? `${(agg.min * 100).toFixed(1)}–${(agg.max * 100).toFixed(1)}%` : null const button = ( ) if (!agg) return (
{button}
) return (
Per-source breakdown
{agg.sources.map((src, i) => (
{src.familyName} {src.displayScore}
))}
Mean (range) {meanDisplay} ({rangeDisplay})
} > {button}
) } const partyRowsFor = (rowsAll: ListRow[]) => ({ firstParty: rowsAll.filter( (r) => r.variant.evaluation.source_metadata.evaluator_relationship === "first_party" ), thirdParty: rowsAll.filter( (r) => r.variant.evaluation.source_metadata.evaluator_relationship === "third_party" ), otherRows: rowsAll.filter((r) => { const rel = r.variant.evaluation.source_metadata.evaluator_relationship return rel !== "first_party" && rel !== "third_party" }), }) const renderPartyBreakdown = (rowsAll: ListRow[]) => { const { firstParty, thirdParty, otherRows } = partyRowsFor(rowsAll) return ( <> {firstParty.length > 0 && ( <>
First-party · {firstParty.length} row{firstParty.length === 1 ? "" : "s"}
{firstParty.map((row, i) => renderRow(row, i === firstParty.length - 1))}
)} {thirdParty.length > 0 && ( <>
Third-party · independent evaluators · {thirdParty.length} row{thirdParty.length === 1 ? "" : "s"}
{thirdParty.map((row, i) => renderRow(row, i === thirdParty.length - 1))}
)} {otherRows.length > 0 && ( <>
Other / unspecified · {otherRows.length} row{otherRows.length === 1 ? "" : "s"}
{otherRows.map((row, i) => renderRow(row, i === otherRows.length - 1))}
)} ) } return (
{formatTagLabel(category as unknown as string)} {families.length} {families.length === 1 ? "family" : "families"} · {totalBenchmarks} benchmark{totalBenchmarks === 1 ? "" : "s"} · {totalRows} row{totalRows === 1 ? "" : "s"}
{families.map((family) => { const isOpen = expandedFamilies.has(family.familyKey) // Build the per-family row list, then apply duplicate // grouping when the toggle is on. "skip" rows drop out // entirely (they're absorbed into a representative row // shown earlier in display order, possibly under a // different family). "merged" rows carry a // MergedRowAggregate so renderRow knows to display the // mean + range + tooltip breakdown. const allRows: ListRow[] = [] for (const g of family.groups) { for (const v of g.variants) { if (mergedRowState) { const rowKey = `${family.familyKey}::${g.key}::${v.evaluation.evaluation_id}::${v.label}` const disposition = mergedRowState.rowDisposition.get(rowKey) if (disposition === "skip") continue if (disposition === "merged") { const evalId = v.evaluation.eval_summary_id const indexEntry = evalId ? benchmarkIndexLookup.get(evalId) : undefined const aggregate = indexEntry ? mergedRowState.aggregates.get(indexEntry.canonicalKey) : undefined allRows.push({ group: g, variant: v, aggregate }) continue } } allRows.push({ group: g, variant: v }) } } // Skip empty families when grouping is on (every row // got absorbed into an earlier family's merged row). if (allRows.length === 0) return null const { firstParty, thirdParty } = partyRowsFor(allRows) // Family-level summary score = avg of avgs across child groups const avgScores = family.groups .map((g) => g.avgNormalizedScore) .filter((v) => Number.isFinite(v) && v >= 0) const familyAvg = avgScores.length ? avgScores.reduce((s, v) => s + v, 0) / avgScores.length : null const familyAvgDisplay = familyAvg != null ? `${(familyAvg * 100).toFixed(1)}%` : family.groups[0]?.avgDisplayScore ?? "–" // Best peer rank across this family's groups let bestRank: { position: number; total: number } | null = null for (const g of family.groups) { const r = getGroupPeerRank(g, modelIds, peerRanks) if (!r) continue if (!bestRank || r.position < bestRank.position) bestRank = r } return (
{isOpen && (
{family.kind === "single-eval" ? ( renderPartyBreakdown(allRows) ) : ( /* Multi-eval — sub-section per child eval. */
{family.groups.map((childGroup) => { const childRows = childGroup.variants.map((v) => ({ group: childGroup, variant: v, })) const childRank = getGroupPeerRank(childGroup, modelIds, peerRanks) return (
{childGroup.avgDisplayScore} {childRank && ( #{childRank.position}{childRank.total ? `/${childRank.total}` : ""} )}
{renderPartyBreakdown(childRows)}
) })}
)}
)}
) })}
) })}
)}
{ if (!open) { setActiveBenchmarkGroupKey(null) } }} > {activeBenchmarkGroup && ( )}
) } function SampleDataDialog({ samples: initialSamples, evaluationName, fullDataUrl, }: { samples: any[], evaluationName: string fullDataUrl?: string }) { const [open, setOpen] = useState(false) const [searchTerm, setSearchTerm] = useState("") const [currentPage, setCurrentPage] = useState(1) const [allSamples, setAllSamples] = useState(initialSamples) const [isLoadingAll, setIsLoadingAll] = useState(false) const [hasLoadedAll, setHasLoadedAll] = useState(false) const itemsPerPage = 10 const filteredSamples = allSamples.filter(sample => { const term = searchTerm.toLowerCase() return ( (sample.input ?? "").toLowerCase().includes(term) || (sample.response ?? "").toLowerCase().includes(term) || (sample.ground_truth ?? "").toLowerCase().includes(term) ) }) const totalPages = Math.ceil(filteredSamples.length / itemsPerPage) const startIndex = (currentPage - 1) * itemsPerPage const currentSamples = filteredSamples.slice(startIndex, startIndex + itemsPerPage) // Reset page when search changes useEffect(() => { setCurrentPage(1) }, [searchTerm]) const [loadError, setLoadError] = useState(null) const handleLoadAll = async () => { if (!fullDataUrl) { setLoadError("No data URL available for this benchmark") return } if (hasLoadedAll) return setIsLoadingAll(true) setLoadError(null) try { const res = await fetch(`/api/instance-data?url=${encodeURIComponent(fullDataUrl)}`) const data = await res.json() if (data.error) { setLoadError(data.error) } else if (data.samples && data.samples.length > 0) { setAllSamples(data.samples) setHasLoadedAll(true) } else { setLoadError("No samples found in the full dataset") } } catch (err) { setLoadError(`Failed to load: ${err instanceof Error ? err.message : "unknown error"}`) } finally { setIsLoadingAll(false) } } const handleOpenToggle = () => { const nextOpen = !open setOpen(nextOpen) if (nextOpen && fullDataUrl && !hasLoadedAll && !isLoadingAll) { void handleLoadAll() } } return ( <> {open && (
Sample Level Data
{hasLoadedAll ? `All ${allSamples.length} samples from ${evaluationName}` : `Showing ${allSamples.length} preview samples from ${evaluationName}`}
setSearchTerm(e.target.value)} className="pl-8" />
{isLoadingAll && (
Loading all instances…
)} {loadError && (
{loadError}
)}
Showing {filteredSamples.length > 0 ? startIndex + 1 : 0}-{Math.min(startIndex + itemsPerPage, filteredSamples.length)} of {filteredSamples.length}
ID Input Model Response Ground Truth Score {currentSamples.length > 0 ? ( currentSamples.map((sample, idx) => ( {sample.sample_id || idx}
{sample.input}
{sample.response}
{sample.ground_truth}
{typeof sample.score === 'number' ? formatRawScoreValue(sample.score) : sample.score || 'N/A'}
)) ) : ( No results found. )}
Page {currentPage} of {totalPages || 1}
)} ) } function BenchmarkResultCard({ evaluation, result, titleOverride, showSetupBadge = true, }: { evaluation: BenchmarkEvaluation, result: EvaluationResult titleOverride?: string showSetupBadge?: boolean }) { const [isOpen, setIsOpen] = useState(false) // Inline samples from the dataset are shown immediately const inlineSamples = evaluation.detailed_evaluation_results_per_samples const detailedUrl = result.detailed_evaluation_results_url const randomSample = useMemo(() => { if (!inlineSamples || inlineSamples.length === 0) return null; const randomIndex = Math.floor(Math.random() * inlineSamples.length); return inlineSamples[randomIndex]; }, [inlineSamples]); const formatDate = formatDateISO const { score } = result.score_details const { min_score = 0, max_score = 1, unit, lower_is_better } = result.metric_config const detailEntries = result.score_details.details ? Object.entries(result.score_details.details) : [] const numericBreakdown = detailEntries.filter(([, value]) => typeof value === "number") const structuredBreakdown = detailEntries.filter(([, value]) => typeof value !== "number") // Normalize to 0-1 for color coding let normalized = (score - min_score) / (max_score - min_score) if (lower_is_better) normalized = 1 - normalized const isHigh = normalized >= 0.8 const isMedium = normalized >= 0.6 let displayScore = formatRawScoreValue(score) let displayUnit = normalizeDisplayLabel(unit) || "Score" const evaluationVariant = getEvaluationVariantLabel(evaluation) if (unit === 'points') { displayScore = score.toFixed(1) displayUnit = "/ 10" } else if (unit === 'accuracy' || unit === 'pass@1' || !unit) { displayUnit = normalizeDisplayLabel(unit) || "Accuracy" } return (

{titleOverride || getResultDisplayName(evaluation, result)}

{result.metric_config.score_type} {showSetupBadge && evaluationVariant && ( Setup: {evaluationVariant} )}

{result.metric_config.evaluation_description}

{displayScore}
{displayUnit}
{/* Source Provenance */}
Source Provenance
{/* Source Metadata */}

Evaluator Metadata

Organization: {getOrganizationDisplayName(evaluation.source_metadata.source_organization_name)}
Relationship: {getRelationshipShortLabel(evaluation.source_metadata.evaluator_relationship)}
Source Type: {getSourceTypeDisplayName(evaluation.source_metadata.source_type)}
{evaluationVariant && (
Evaluation Setup: {evaluationVariant}
)} {evaluation.source_metadata.source_url && (
URL: Link
)}
Date: {formatDate(evaluation.retrieved_timestamp)}
{/* Source Data */}

Dataset Information

Name: {Array.isArray(evaluation.source_data) ? 'Multiple Sources' : evaluation.source_data.dataset_name}
{!Array.isArray(evaluation.source_data) && ( <> {evaluation.source_data.hf_repo && ( )} {evaluation.source_data.hf_split && (
Split: {evaluation.source_data.hf_split}
)}
Samples: {evaluation.source_data.samples_number?.toLocaleString()}
)}
{/* Evaluation Results */}
Evaluation Results
Overall Score
{result.metric_config.score_type} • {result.metric_config.min_score}-{result.metric_config.max_score} • {result.metric_config.lower_is_better ? 'Lower is better' : 'Higher is better'}
{displayScore}
{detailEntries.length > 0 && ( <>
Detailed Breakdown
Scores and structured metadata for individual slices or metrics
{numericBreakdown.length > 0 && (
{numericBreakdown.map(([key, value]) => { let valDisplay = typeof value === 'number' ? value.toFixed(2) : value; let normalized_slice = 0; if (typeof value === 'number') { if (unit === 'accuracy' || !unit || unit === 'pass@1') { valDisplay = formatRawScoreValue(value); normalized_slice = value; } else { valDisplay = value.toFixed(2); normalized_slice = (value - min_score) / (max_score - min_score); } } // Format the key nicely const formattedKey = key.replace(/_/g, ' ').replace(/\b\w/g, l => l.toUpperCase()); return (
{formattedKey}
{valDisplay}
{typeof value === 'number' && ( )}
)})}
)} {structuredBreakdown.length > 0 && (
Structured Detail Fields
Field Value {structuredBreakdown.map(([key, value]) => { const formattedKey = key.replace(/_/g, ' ').replace(/\b\w/g, l => l.toUpperCase()); const formattedValue = formatMetadataValue(value) ?? "N/A" return ( {formattedKey}
                                        {formattedValue}
                                      
) })}
)} )}
{/* Generation Configuration */} {result.generation_config && (
Generation Configuration
{result.generation_config.additional_details && (
Description
{formatMetadataValue(result.generation_config.additional_details)}
)} {result.generation_config.generation_args && (
{Object.entries(result.generation_config.generation_args).map(([key, value]) => (
{key}
{formatMetadataValue(value)}
))}
)}
)} {/* Sample Level Data — inline samples from the dataset show immediately */} {inlineSamples && inlineSamples.length > 0 && randomSample && (
Sample Level Data (Random Sample)
{inlineSamples.length} Samples
ID: {randomSample.sample_id}
Input
{randomSample.input}
Model Response
{randomSample.response}
Ground Truth
{randomSample.ground_truth}
)} {/* Footer Links */}
{result.detailed_evaluation_results_url && ( View detailed per-sample results )}
) } function AggregatedBenchmarkCard({ group, anchorId, isOpen, onOpenChange, motionIndex = 0, }: { group: BenchmarkGroup anchorId: string isOpen: boolean onOpenChange: (open: boolean) => void motionIndex?: number }) { const { mode } = useAudienceMode() const isResearchView = mode === "research" const [expandedRows, setExpandedRows] = useState>({}) const [selectedFilters, setSelectedFilters] = useState>({}) const variantRows = useMemo( () => group.variants.map((variant, index) => { const configMap = getVariantConfigMap(variant) return { rowKey: `${variant.evaluation.evaluation_id}-${index}`, variant, configMap, configEntries: Object.entries(configMap), sampleCount: Array.isArray(variant.evaluation.source_data) ? null : variant.evaluation.source_data.samples_number ?? null, } }), [group.variants] ) const filterDefinitions = useMemo(() => { const valuesByKey = new Map>() for (const row of variantRows) { for (const [key, value] of row.configEntries) { if (!valuesByKey.has(key)) { valuesByKey.set(key, new Set()) } valuesByKey.get(key)?.add(value) } } return Array.from(valuesByKey.entries()) .filter(([, values]) => values.size > 1) .sort(([a], [b]) => { if (a === "setup") return -1 if (b === "setup") return 1 return a.localeCompare(b) }) .map(([key, values]) => ({ key, label: key === "setup" ? "Setup" : formatConfigLabel(key), values: Array.from(values).sort((a, b) => a.localeCompare(b)), })) }, [variantRows]) const filteredRows = useMemo( () => variantRows.filter((row) => filterDefinitions.every((definition) => { const selectedValue = selectedFilters[definition.key] if (!selectedValue || selectedValue === "all") { return true } return row.configMap[definition.key] === selectedValue }) ), [filterDefinitions, selectedFilters, variantRows] ) const activeFilterCount = Object.values(selectedFilters).filter((value) => value && value !== "all").length const leaderNormalizedScore = filteredRows[0]?.variant.normalizedScore ?? 0 const spread = getBenchmarkSpread(group) const sourceOrganizations = new Set(group.variants.map((variant) => getOrganizationDisplayName(variant.evaluation.source_metadata.source_organization_name))) const latestTimestamp = group.variants.reduce((latest, variant) => { const value = Number.parseFloat(variant.evaluation.retrieved_timestamp) return Number.isFinite(value) ? Math.max(latest, value) : latest }, Number.NEGATIVE_INFINITY) const latestReportedLabel = Number.isFinite(latestTimestamp) ? formatCompactDate(String(latestTimestamp)) : formatCompactDate(group.variants[0]?.evaluation.retrieved_timestamp ?? "") const compactDomains = group.domains.slice(0, 2) const progressWidth = Math.max(4, Math.min(100, group.avgNormalizedScore * 100)) const sliceCount = getGroupSliceCount(group) const toggleRow = (rowKey: string) => { setExpandedRows((current) => ({ ...current, [rowKey]: !current[rowKey], })) } return (
onOpenChange(!isOpen)} onKeyDown={(event) => { if (event.key === "Enter" || event.key === " ") { event.preventDefault() onOpenChange(!isOpen) } }} className="block w-full cursor-pointer px-3.5 py-2.5 text-left transition-colors hover:bg-muted/20 focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2" > {/* Compact single-row layout */}
{/* Category dot */} {formatTagLabel(group.category as unknown as string)} {/* Name + domains */}
event.stopPropagation()} className="text-sm font-semibold tracking-tight text-foreground/95 underline decoration-dotted underline-offset-4 hover:text-primary" > {group.title} {group.benchmarkCard && ( card )} {sliceCount > 0 && ( {sliceCount} slice{sliceCount === 1 ? "" : "s"} )} {compactDomains.map((domain) => ( {domain} ))} {group.domains.length > compactDomains.length && ( +{group.domains.length - compactDomains.length} )}
{group.avgDisplayScore} {group.bestRankPosition != null && ( {`#${group.bestRankPosition}${group.bestRankTotal ? `/${group.bestRankTotal}` : ""}`} )}
{/* Slice count */} {group.variants.length} {group.variants.length === 1 ? "row" : "rows"}
{isOpen ? : }
{group.benchmarkCard && (
Benchmark context
{group.benchmarkCard.benchmark_details.name}

{group.benchmarkCard.benchmark_details.overview}

{group.benchmarkCard.benchmark_details.data_type} {group.benchmarkCard.methodology.metrics.slice(0, 2).map((metric) => ( {metric} ))}
Goal
{group.benchmarkCard.purpose_and_intended_users.goal}
Methods
{group.benchmarkCard.methodology.methods.slice(0, 2).join(", ") || "Not specified"}
Caveat
{group.benchmarkCard.purpose_and_intended_users.limitations}
)}
Metrics & Breakdown
{isResearchView ? "Benchmark-level metrics and benchmark breakdowns are shown separately from setup changes." : "Root benchmark metrics and real benchmark breakdowns are shown without inventing extra hierarchy in the UI."}
{filterDefinitions.length > 0 && (
Comparison Filters
{isResearchView ? "Narrow to matching setup or generation config values for apples-to-apples comparison" : "Narrow to matching setup and reporting conditions for more comparable policy review"}
{filteredRows.length} of {variantRows.length} shown {activeFilterCount > 0 && ( )}
{filterDefinitions.map((definition) => (
{definition.label}
))}
)} {variantRows.length === 1 ? (
Reported Details
) : (
{filteredRows.map((row, index) => { const { rowKey, variant } = row const isRowOpen = expandedRows[rowKey] ?? false const hasSourceLink = Boolean(variant.evaluation.source_metadata.source_url) const leaderRawScore = filteredRows[0]?.variant.result.score_details.score ?? variant.result.score_details.score const gapToLeader = Math.max(0, leaderRawScore - variant.result.score_details.score) const evidenceStatus = hasSourceLink ? "Linked" : "Inline" return (
{isRowOpen && (
)}
) })} {filteredRows.length === 0 && (
No rows match the current filters.
)}
)}
) } function BenchmarkDeepDiveDialogPanel({ group, comparisonIndex, evalHierarchy, hierarchyIndex, }: { group: BenchmarkGroup comparisonIndex?: ComparisonIndex | null evalHierarchy?: EvalHierarchy | null hierarchyIndex?: Map | null }) { const { mode } = useAudienceMode() const isResearchView = mode === "research" const [resolvedRanks, setResolvedRanks] = useState>({}) const [isResolvingRanks, setIsResolvingRanks] = useState(false) const compactDomains = group.domains.slice(0, 2) const sliceCount = getGroupSliceCount(group) const hasSliceMatrix = sliceCount > 0 const sourceOrganizations = useMemo( () => new Set(group.variants.map((variant) => getOrganizationDisplayName(variant.evaluation.source_metadata.source_organization_name))), [group.variants] ) const rankedVariants = useMemo( () => [...group.variants].sort((a, b) => { const aRank = a.rankRatio ?? Number.POSITIVE_INFINITY const bRank = b.rankRatio ?? Number.POSITIVE_INFINITY if (aRank !== bRank) { return aRank - bRank } return b.normalizedScore - a.normalizedScore }), [group.variants] ) // Cross-family appearances: hierarchy.json's `benchmark_index[]` cross-links // a canonical benchmark across multiple families (e.g. AIME appears in // artificial-analysis, llm-stats, and vals-ai). When any of this group's // variant eval_summary_ids show up in a benchmark_index entry, surface the // other appearances as a "this benchmark also reports as" panel so the // reader sees the duplication without leaving the dialog. const crossFamilyAppearances = useMemo(() => { const benchmarkIndex = evalHierarchy?.benchmark_index if (!benchmarkIndex || benchmarkIndex.length === 0) return [] as Array<{ canonicalDisplayName: string appearances: Array<{ familyKey: string familyDisplayName: string evalSummaryId: string isCurrent: boolean }> }> const groupEvalIds = new Set( group.variants .map((v) => v.evaluation.eval_summary_id) .filter((id): id is string => Boolean(id)), ) if (groupEvalIds.size === 0) return [] const familyDisplayByKey = new Map() for (const fam of evalHierarchy?.families ?? []) { familyDisplayByKey.set(fam.key, fam.display_name) } const seen = new Set() const out: Array<{ canonicalDisplayName: string appearances: Array<{ familyKey: string familyDisplayName: string evalSummaryId: string isCurrent: boolean }> }> = [] // benchmark_index is pre-cleaned server-side (cleanHierarchy): // family-rollup entries are dropped, (family, eval_id) pairs are // deduped, degenerate entries are filtered. So we just walk it. for (const entry of benchmarkIndex) { const flat: Array<{ familyKey: string; evalSummaryId: string }> = [] for (const app of entry.appearances ?? []) { for (const id of app.eval_summary_ids ?? []) { flat.push({ familyKey: app.family_key, evalSummaryId: id }) } } const matches = flat.some((f) => groupEvalIds.has(f.evalSummaryId)) if (!matches) continue if (seen.has(entry.key)) continue seen.add(entry.key) out.push({ canonicalDisplayName: entry.display_name, appearances: flat.map((f) => ({ familyKey: f.familyKey, familyDisplayName: familyDisplayByKey.get(f.familyKey) ?? f.familyKey, evalSummaryId: f.evalSummaryId, isCurrent: groupEvalIds.has(f.evalSummaryId), })), }) } return out }, [evalHierarchy, group.variants]) void hierarchyIndex const variantRows = useMemo( () => rankedVariants.map((variant, index) => { const rowKey = `${variant.evaluation.evaluation_id}-${index}` const evalHref = getEvalDetailHref(variant.evaluation, variant.result) const evalSummaryId = variant.evaluation.eval_summary_id ?? getEvalSummaryIdFromHref(evalHref) const configMap = getVariantConfigMap(variant) return { rowKey, variant, evalSummaryId, configMap, configEntries: Object.entries(configMap), } }), [rankedVariants] ) const rowsByPrimaryLabel = useMemo(() => { const groupedRows = new Map() for (const row of variantRows) { const primaryLabel = getVariantPrimaryLabel(row.variant, group.title) const existing = groupedRows.get(primaryLabel) ?? [] existing.push(row) groupedRows.set(primaryLabel, existing) } return groupedRows }, [group.title, variantRows]) const hasAmbiguousPrimaryLabels = useMemo( () => Array.from(rowsByPrimaryLabel.values()).some((rows) => rows.length > 1), [rowsByPrimaryLabel] ) const rowDisambiguationLabels = useMemo(() => { const labels = new Map() for (const [primaryLabel, rows] of rowsByPrimaryLabel.entries()) { if (rows.length <= 1) { continue } const runLabelsByRow = new Map() const distinctRunLabels = new Set() for (const row of rows) { const runLabel = getVariantRunLabels(row, comparisonIndex)[0] if (!runLabel) { continue } runLabelsByRow.set(row.rowKey, runLabel) distinctRunLabels.add(runLabel) } if (distinctRunLabels.size === rows.length) { for (const row of rows) { const runLabel = runLabelsByRow.get(row.rowKey) if (runLabel) { labels.set(row.rowKey, runLabel) } } continue } const configLabelsByRow = new Map() const distinctConfigLabels = new Set() for (const row of rows) { const configLabel = getVariantConfigDisambiguation(row, rows).join(" · ") if (!configLabel) { continue } configLabelsByRow.set(row.rowKey, configLabel) distinctConfigLabels.add(configLabel) } if (distinctConfigLabels.size === rows.length) { for (const row of rows) { const configLabel = configLabelsByRow.get(row.rowKey) if (configLabel) { labels.set(row.rowKey, configLabel) } } continue } rows.forEach((row, index) => { labels.set(row.rowKey, `${primaryLabel} run ${index + 1}`) }) } return labels }, [comparisonIndex, rowsByPrimaryLabel]) const bestResolvedRank = useMemo(() => { const candidates = variantRows .map((row) => { const resolved = resolvedRanks[row.rowKey] if (resolved) return resolved if (row.variant.rankPosition != null) { return { position: row.variant.rankPosition, total: row.variant.rankTotal } } return null }) .filter((r): r is { position: number; total: number | null } => r != null) .sort((a, b) => { const aRatio = a.total != null && a.total > 0 ? a.position / a.total : a.position const bRatio = b.total != null && b.total > 0 ? b.position / b.total : b.position return aRatio - bRatio }) return candidates[0] ?? null }, [resolvedRanks, variantRows]) // Kept only to drive the single-setup overview: when every reported row is // reported under the same setup, the detail table collapses into a compact // view that drops the redundant "Reporting setup" column. const sliceSetups = useMemo(() => { if (!hasSliceMatrix) return null const setupOrder: string[] = [] for (const row of variantRows) { const setupDisplayLabel = formatSetupDisplayLabel(row.variant.setupLabel) if (!setupOrder.includes(setupDisplayLabel)) setupOrder.push(setupDisplayLabel) } return { setupOrder } }, [hasSliceMatrix, variantRows]) const useSingleSetupOverview = Boolean(sliceSetups && sliceSetups.setupOrder.length === 1) const singleSetupDisplayLabel = useSingleSetupOverview ? sliceSetups?.setupOrder[0] ?? null : null useEffect(() => { const pendingRows = variantRows.filter( (row) => row.variant.rankPosition == null && !resolvedRanks[row.rowKey] && row.evalSummaryId ) if (pendingRows.length === 0) { return } let isCancelled = false const resolveRanks = async () => { setIsResolvingRanks(true) const nextResolvedEntries = await Promise.all( pendingRows.map(async (row) => { const rank = await fetchPeerRankForModel(row.evalSummaryId, row.variant.evaluation.model_info.id) return rank ? ([row.rowKey, rank] as const) : null }) ) if (isCancelled) { return } setResolvedRanks((current) => { const patch: Record = {} for (const entry of nextResolvedEntries) { if (!entry) { continue } patch[entry[0]] = entry[1] } return Object.keys(patch).length > 0 ? { ...current, ...patch } : current }) setIsResolvingRanks(false) } resolveRanks() return () => { isCancelled = true } }, [resolvedRanks, variantRows]) return ( <>
{formatTagLabel(group.category as unknown as string)} · Benchmark deep dive
{getBenchmarkGroupHeading(group)} {isResearchView ? "Inspect setup slices, score details, and source provenance in one focused view." : "Inspect reporting setup and evidence details before interpreting benchmark position."} {(compactDomains.length > 0 || group.benchmarkCard) && (
{compactDomains.map((domain) => ( {domain} ))} {group.domains.length > compactDomains.length && ( +{group.domains.length - compactDomains.length} )} {group.benchmarkCard && · Card available}
)}
Avg score
{group.avgDisplayScore}
{group.bestRankPosition != null && (
#{group.bestRankPosition}{group.bestRankTotal ? `/${group.bestRankTotal}` : ""}
)}
{/* Stat strip — paper-style hairline grid */}
Reported rows
{group.variants.length}
Best rank
{bestResolvedRank != null ? `#${bestResolvedRank.position}${bestResolvedRank.total ? `/${bestResolvedRank.total}` : ""}` : isResolvingRanks ? "…" : "N/A"}
Sources
{sourceOrganizations.size} {hasSliceMatrix && ( · {sliceCount} slice{sliceCount === 1 ? "" : "s"} )}
{group.benchmarkCard && (
Benchmark context

{group.benchmarkCard.benchmark_details.overview}

)} {/* Sources — distinct reporting orgs and dataset links for this group. Pulled up to the top of the deep-dive so the per-row table can stay focused on slice / setup / score. */} {(() => { type SourceEntry = { key: string orgName: string orgHref: string | null relationship: string | null | undefined datasetHref: string | null } const entries: SourceEntry[] = [] const seen = new Set() for (const variant of group.variants) { const meta = variant.evaluation.source_metadata const orgName = getOrganizationDisplayName(meta.source_organization_name) const orgHref = meta.source_organization_url || null const rawSource = variant.result.source_data ?? variant.evaluation.source_data const sourceData = !Array.isArray(rawSource) ? rawSource : null const datasetHref = sourceData?.dataset_url || (Array.isArray(sourceData?.url) ? sourceData?.url?.[0] : sourceData?.url) || (sourceData?.hf_repo ? `https://huggingface.co/datasets/${sourceData.hf_repo}` : null) || null const relationship = meta.evaluator_relationship const key = `${orgName}::${orgHref ?? ""}::${relationship ?? ""}::${datasetHref ?? ""}` if (seen.has(key)) continue seen.add(key) entries.push({ key, orgName, orgHref, relationship, datasetHref }) } if (entries.length === 0) return null return (
Sources
    {entries.map((entry, i) => { const showDataset = Boolean(entry.datasetHref) && entry.datasetHref !== entry.orgHref const isFirst = entry.relationship === "first_party" const isThird = entry.relationship === "third_party" return (
  • {entry.orgHref ? ( {entry.orgName} ) : ( {entry.orgName} )} · {getRelationshipShortLabel(entry.relationship)} {showDataset && ( · Dataset )}
  • ) })}
) })()} {crossFamilyAppearances.length > 0 && (

Also reports this benchmark

{crossFamilyAppearances.reduce((sum, e) => sum + e.appearances.length, 0)} entries

The same canonical benchmark appears under multiple families. Each entry below is a separate eval row; scores from these siblings can be compared but are recorded independently.

{crossFamilyAppearances.flatMap((entry) => entry.appearances.map((app) => (
{app.familyDisplayName} {app.isCurrent && ( · current )}
{humanizeEvaluationId(app.evalSummaryId)}
Open )), )}
)} {useSingleSetupOverview ? (

{hasAmbiguousPrimaryLabels ? "Reported runs" : "Slice overview"}

{singleSetupDisplayLabel ? `${singleSetupDisplayLabel} · ` : ""} {variantRows.length} row{variantRows.length === 1 ? "" : "s"}

{hasAmbiguousPrimaryLabels ? isResearchView ? "These rows share the same benchmark label, so run names or differing config fields are surfaced to show what changed across reports." : "These rows describe the same benchmark view, so the table surfaces the reported run name or setup differences that separate them." : isResearchView ? "This benchmark reports one setup, so slices, scores, and provenance are merged into one comparison view." : "This benchmark only reports one setup, so the slice evidence is consolidated into a single reader-friendly view."}

{variantRows.map((row, index) => { const { rowKey, variant, configEntries } = row const resolvedRank = resolvedRanks[rowKey] const primaryLabel = getVariantPrimaryLabel(variant, group.title) const filteredConfigEntries = configEntries.filter(([key]) => key.toLowerCase() !== "setup") const disambiguationLabel = rowDisambiguationLabels.get(rowKey) const leadLabel = hasAmbiguousPrimaryLabels ? disambiguationLabel ?? `Reported run ${index + 1}` : primaryLabel const supportingLabel = hasAmbiguousPrimaryLabels ? primaryLabel : disambiguationLabel && disambiguationLabel !== primaryLabel ? disambiguationLabel : null return ( ) })}
{hasAmbiguousPrimaryLabels ? "Reported row" : "Slice"} Setup detail Score Rank
{leadLabel}
{supportingLabel && (
{supportingLabel}
)} {(!variant.evaluation.slice_key || variant.variantType !== "default") && (
{!variant.evaluation.slice_key && Benchmark-level metric} {variant.variantType !== "default" && · {getVariantTypeLabel(variant.variantType)}}
)}
{singleSetupDisplayLabel}
{filteredConfigEntries.length > 0 && (
{filteredConfigEntries .slice(0, 2) .map(([key, value]) => `${formatConfigLabel(key)}=${getConfigDisplayValue(value)}`) .join(" · ")}
)}
{variant.displayScore}
{(variant.rankPosition != null || resolvedRank) ? `#${resolvedRank?.position ?? variant.rankPosition}${(resolvedRank?.total ?? variant.rankTotal) ? `/${resolvedRank?.total ?? variant.rankTotal}` : ""}` : "N/A"}
) : null} {!useSingleSetupOverview && (

Benchmark breakdown

{variantRows.length} row{variantRows.length === 1 ? "" : "s"}

Primary row labels show the benchmark slice or slice. Setup and source details sit alongside each row.

{variantRows.map((row) => { const { rowKey, variant, configEntries } = row const resolvedRank = resolvedRanks[rowKey] const primaryLabel = getVariantPrimaryLabel(variant, group.title) const setupDisplayLabel = formatSetupDisplayLabel(variant.setupLabel) const rawVariantLabel = variant.label !== primaryLabel ? variant.label : null return ( ) })}
Slice Reporting setup Score Rank
{primaryLabel}
{getVariantTypeLabel(variant.variantType)} {rawVariantLabel && · {rawVariantLabel}}
{setupDisplayLabel}
{configEntries.length > 0 && (
{configEntries.slice(0, 3).map(([key, value]) => `${formatConfigLabel(key)}=${getConfigDisplayValue(value)}`).join(" · ")}
)}
{variant.displayScore}
{(variant.rankPosition != null || resolvedRank) ? `#${resolvedRank?.position ?? variant.rankPosition}${(resolvedRank?.total ?? variant.rankTotal) ? `/${resolvedRank?.total ?? variant.rankTotal}` : ""}` : "N/A"}
)} {(() => { const variantWithSamples = group.variants.find(v => v.evaluation.detailed_evaluation_results_per_samples && v.evaluation.detailed_evaluation_results_per_samples.length > 0) if (!variantWithSamples) return null const samples = variantWithSamples.evaluation.detailed_evaluation_results_per_samples! const fullDataUrl = variantWithSamples.result.detailed_evaluation_results_url ?? variantWithSamples.evaluation.evaluation_results.find(r => r.detailed_evaluation_results_url)?.detailed_evaluation_results_url return (

Sample data preview

{samples.length} example{samples.length === 1 ? "" : "s"}
{samples.slice(0, INSTANCE_PREVIEW_LIMIT).map((sample, idx) => (
{sample.input && (
Input
{sample.input.slice(0, 400)}{sample.input.length > 400 ? "..." : ""}
)} {sample.response && (
Response
{sample.response.slice(0, 300)}{sample.response.length > 300 ? "..." : ""}
)} {sample.ground_truth && (
Ground truth
{sample.ground_truth.slice(0, 200)}
)}
))}
{(samples.length > INSTANCE_PREVIEW_LIMIT || fullDataUrl) && (
)}
) })()}
View full leaderboard
) } function VariantExpandedDetail({ row, group, mode, }: { row: VariantRowData group: BenchmarkGroup mode: "research" | "policy" }) { const isResearchView = mode === "research" const { variant, configEntries, sampleCount } = row const { numericBreakdown, helmMetrics, structuredBreakdown } = buildVariantStructuredSections(variant) const sourceTypeLabel = getSourceTypeDisplayName(variant.evaluation.source_metadata.source_type) const sourceData = !Array.isArray(variant.result.source_data ?? variant.evaluation.source_data) ? (variant.result.source_data ?? variant.evaluation.source_data) as import("@/lib/benchmark-schema").SourceData : null const evalLibrary = variant.evaluation.eval_library const uncertainty = (variant.result.score_details as any).uncertainty as { standard_error?: { value: number }; num_samples?: number } | undefined const confidenceInterval = variant.result.score_details.confidence_interval const numSamples = uncertainty?.num_samples ?? variant.result.score_details.sample_size ?? sourceData?.samples_number ?? sampleCount const stdError = uncertainty?.standard_error?.value const inferencePlatform = variant.evaluation.model_info.inference_platform // Source URLs for linking const sourceUrls: string[] = Array.isArray(sourceData?.url) ? (sourceData.url as string[]) : sourceData?.url ? [sourceData.url as string] : sourceData?.dataset_url ? [sourceData.dataset_url] : [] return (
{variant.label}
{getVariantTypeLabel(variant.variantType)} {group.title} {variant.displayScore}
{variant.result.metric_config.evaluation_description}
{formatCompactDate(variant.evaluation.retrieved_timestamp)} {getRelationshipShortLabel(variant.evaluation.source_metadata.evaluator_relationship)} {numSamples != null && {Number(numSamples).toLocaleString()} samples} {evalLibrary && ( {formatEvalLibrary(evalLibrary)} )}
{isResearchView ? "Provenance & Dataset" : "Reporting Context"}
{sourceData?.hf_repo && ( {sourceData.hf_repo} } /> )} {sourceData?.dataset_version && } {sourceData?.hf_split && } {variant.sliceLabel && } {variant.setupLabel && } {inferencePlatform && } {variant.evaluation.source_metadata.source_name && ( )} {numSamples != null && } {stdError != null && } {confidenceInterval && ( )} {sourceUrls.length > 0 && ( {sourceUrls.map((url, i) => ( {url.replace(/^https?:\/\//, "").slice(0, 50)}{url.length > 57 ? "…" : ""} ))}
} /> )}
{isResearchView ? "Config Snapshot" : "Evaluation Setup"}
{configEntries.length > 0 ? ( configEntries.map(([key, value]) => ( {formatConfigLabel(key)}: {getConfigDisplayValue(value)} )) ) : ( No explicit config recorded )}
{numericBreakdown.length > 0 && (
{isResearchView ? "Slice Scores" : "Reported Metrics"}
{numericBreakdown.map(([key, value]) => { const numericValue = value as number const minScore = variant.result.metric_config.min_score ?? 0 const maxScore = variant.result.metric_config.max_score ?? 1 const range = maxScore - minScore const normalizedValue = range > 0 ? ((numericValue - minScore) / range) * 100 : numericValue * 100 return (
{formatConfigLabel(key)}
{formatRawScoreValue(numericValue, variant.result.metric_config.unit)}
) })}
)} {helmMetrics.length > 0 && (
Additional Metrics
Metric Category Value {helmMetrics.map(({ label, tab, score }) => { const numericScore = Number.parseFloat(score) return ( {label} {tab} {Number.isFinite(numericScore) ? numericScore.toFixed(3) : score} ) })}
)} {structuredBreakdown.length > 0 && (
Supporting Detail
Field Value {structuredBreakdown.map(([key, value]) => ( {formatConfigLabel(key)}
                        {formatMetadataValue(value)}
                      
))}
)} {/* Instance-level sample data */} {variant.evaluation.detailed_evaluation_results_per_samples && variant.evaluation.detailed_evaluation_results_per_samples.length > 0 && (
Sample data ({variant.evaluation.detailed_evaluation_results_per_samples.length} examples)
{variant.evaluation.detailed_evaluation_results_per_samples.slice(0, INSTANCE_PREVIEW_LIMIT).map((sample, idx) => (
{sample.input && (
Input
{sample.input.slice(0, 500)}
)} {sample.response && (
Response
{sample.response.slice(0, 500)}
)} {sample.ground_truth && (
Ground truth
{sample.ground_truth.slice(0, 300)}
)} {sample.is_correct != null && (
{sample.is_correct ? "Correct" : "Incorrect"}
)}
))}
{(variant.evaluation.detailed_evaluation_results_per_samples.length > INSTANCE_PREVIEW_LIMIT || variant.result.detailed_evaluation_results_url || variant.evaluation.evaluation_results.find(r => r.detailed_evaluation_results_url)?.detailed_evaluation_results_url) && ( r.detailed_evaluation_results_url)?.detailed_evaluation_results_url} /> )}
)} {variant.evaluation.source_metadata.source_url && ( )}
) } function InlineMeta({ label, value }: { label: string; value: React.ReactNode }) { return (
{label}
{value}
) } /** * Compact inline metadata pair used in the model header strip. * Renders nothing when children is empty / null. */ function MetaFact({ label, children, mono = false, }: { label: string children: React.ReactNode mono?: boolean }) { if (children == null || children === "") return null return (
{label}
{children}
) } /** * Compact hero stat cell used in the model header. * Tones are subtle backgrounds; cells share a single bordered container. */ function HeroStat({ label, value, tone, }: { label: string value: number | string tone: "amber" | "emerald" | "sky" | "slate" }) { const toneClass = { amber: "bg-amber-50/70 text-amber-900 dark:bg-amber-950/25 dark:text-amber-100", emerald: "bg-emerald-50/70 text-emerald-900 dark:bg-emerald-950/25 dark:text-emerald-100", sky: "bg-sky-50/70 text-sky-900 dark:bg-sky-950/25 dark:text-sky-100", slate: "bg-muted/30 text-foreground", }[tone] return (
{label}
{typeof value === "number" ? value.toLocaleString() : value}
) } type EvaluatorMixData = { rows: Array<{ category: CategoryType label: string first: number third: number collab: number other: number total: number }> firstTotal: number thirdTotal: number collabTotal: number otherTotal: number grand: number } /** * Donut + per-category bars showing first-party vs third-party row counts. * Adapted from mock_design/model_detail_a.jsx#EvaluatorMix. */ function EvaluatorMix({ mix }: { mix: EvaluatorMixData }) { const { rows, firstTotal, thirdTotal, collabTotal, otherTotal, grand } = mix const R = 64 const sw = 18 const C = 2 * Math.PI * R const fFirst = firstTotal / grand const fThird = thirdTotal / grand const fCollab = collabTotal / grand const fOther = otherTotal / grand const lFirst = C * fFirst const lThird = C * fThird const lCollab = C * fCollab const lOther = C * fOther return (
{/* Donut */}
Total rows
{grand}
{/* Legend + per-category bars */}
{firstTotal} first-party {Math.round(fFirst * 100)}%
{thirdTotal} third-party · independent {Math.round(fThird * 100)}%
{collabTotal > 0 && (
{collabTotal} collaborative
)} {otherTotal > 0 && (
{otherTotal} unspecified
)}
{rows.map((row, i) => { const f = row.first / row.total const t = row.third / row.total const c = row.collab / row.total const o = row.other / row.total return (
{row.label}
{row.total} row{row.total === 1 ? "" : "s"}
{f > 0 &&
} {t > 0 &&
} {c > 0 &&
} {o > 0 &&
}
{row.first} · {row.third} {(row.collab > 0 || row.other > 0) && ( <> · {row.collab + row.other} )}
) })}
) } function AllEvaluationsView({ evaluations }: { evaluations: BenchmarkEvaluation[] }) { return (
{evaluations.map((eval_, idx) => (
{eval_.evaluation_results.map((result, ridx) => ( ))}
))}
) } function CategoryStatsView({ stats, summary }: { stats: { category: CategoryType; count: number; avg_score: number }[] summary: ModelSummaryCore }) { const getCategoryColor = (score: number) => { if (score >= 0.8) return 'text-green-600' if (score >= 0.6) return 'text-yellow-600' return 'text-red-600' } const getCategoryLabel = (category: CategoryType): string => { return category.split('-').map(w => w.charAt(0).toUpperCase() + w.slice(1)).join(' ') } return (
{stats.map((stat) => { const evals = summary.evaluations_by_category[stat.category] || [] return (
{getCategoryLabel(stat.category)}
{formatRawScoreValue(stat.avg_score)}
{stat.count} evaluation{stat.count !== 1 ? 's' : ''}
{evals.map((eval_: BenchmarkEvaluation, idx: number) => { // Filter results to only show those that match this category const relevantResults = eval_.evaluation_results.filter((result: any) => { const resultCategory = inferCategoryFromBenchmark(result.evaluation_name) return resultCategory === stat.category }) if (relevantResults.length === 0) return null return relevantResults.map((result: any, ridx: number) => (
{getResultDisplayName(eval_, result)}
{((getEvaluationVariantLabel(eval_) ? `Setup: ${formatSetupDisplayLabel(getEvaluationVariantLabel(eval_))}` : null)) || (Array.isArray(eval_.source_data) ? (normalizeDisplayLabel(eval_.source_metadata.source_name) || 'Unknown') : normalizeDisplayLabel(eval_.source_data.dataset_name))}
{formatRawScoreValue(result.score_details.score, result.metric_config.unit)}
)) })}
) })}
) }