"use client" import { useAudienceMode } from "@/components/audience-mode-provider" import { Fragment, useEffect, useMemo, useState } from "react" import Link from "next/link" import { Badge } from "@/components/ui/badge" import { Button } from "@/components/ui/button" import { CompletenessPanel } from "@/components/signals/completeness-panel" import { ComparabilityPanel } from "@/components/signals/comparability-panel" import { ReproducibilityPanel } from "@/components/signals/reproducibility-panel" import { SignalsRowBadges } from "@/components/signals/signals-row-badges" import { RowSignalsCompact } from "@/components/signals/row-signals-compact" import { SignalTooltip } from "@/components/signals/signal-tooltip" import { getCompletenessPopulatedCount } from "@/components/signals/signal-utils" import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card" import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "@/components/ui/collapsible" import { Dialog, DialogContent, DialogDescription, DialogHeader, DialogTitle, } from "@/components/ui/dialog" import { Input } from "@/components/ui/input" import { DropdownMenu, DropdownMenuCheckboxItem, DropdownMenuContent, DropdownMenuItem, DropdownMenuLabel, DropdownMenuSeparator, DropdownMenuTrigger, } from "@/components/ui/dropdown-menu" import { Progress } from "@/components/ui/progress" import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table" import { getModelFamilyRouteId } from "@/lib/model-family" import { cn } from "@/lib/utils" import { AlertTriangle, BarChart3, BookOpen, ChevronDown, ChevronUp, Database, ExternalLink, FileText, Globe, Medal, Scale, Search, Shield, SlidersHorizontal, Tag, X, } from "lucide-react" import type { BenchmarkCard } from "@/lib/benchmark-schema" import type { BenchmarkEvalSummary, ModelResultForBenchmark } from "@/lib/eval-processing" import { PolicyOverview } from "@/components/policy-overview" import { ResearcherReproducibilityCard } from "@/components/researcher-reproducibility-card" import { KnownIssuesPanel } from "@/components/known-issues-panel" import { getKnownIssues, type KnownIssue } from "@/lib/known-issues" import { ApplesToApplesBanner } from "@/components/apples-to-apples-banner" import { FlagScoreButton } from "@/components/flag-score-button" interface EvalDetailProps { summary: BenchmarkEvalSummary } interface LeaderboardRow { key: string rank: number modelResult: ModelResultForBenchmark normalizedScore: number } type LeaderboardMetric = NonNullable[number] type LeaderboardMatrixRow = NonNullable[number] /** * Pick a representative row-level annotation for the matrix view. * * Reproducibility and provenance are typically constant across all metrics for * a given (model, benchmark) pair, so rendering them in every cell is just * noise. This helper grabs the first non-null annotation across visible metrics * and returns it for the row-level badge strip. */ function getRowLevelAnnotations( row: LeaderboardMatrixRow, visibleMetrics: LeaderboardMetric[] ) { const annotationsByMetric = row.annotations_by_metric if (!annotationsByMetric) { return null } for (const metric of visibleMetrics) { const annotations = annotationsByMetric[metric.column_key] if (annotations) { return annotations } } return null } const SLICE_PILL_THRESHOLD = 5 interface SliceTab { key: string label: string } /** * Slice picker that adapts to slice count. * * - <= SLICE_PILL_THRESHOLD: render every slice as a pill (current familiar UX). * - > SLICE_PILL_THRESHOLD: render "All slices" + currently-selected pill + * a "Browse N slices" button that opens a searchable dialog. Hundreds of * subtasks (e.g. AIRBench's 374) fit cleanly. */ function SliceSelector({ activeSubtaskTab, onChange, tabs, }: { activeSubtaskTab: string onChange: (key: string) => void tabs: SliceTab[] }) { const [browserOpen, setBrowserOpen] = useState(false) const [search, setSearch] = useState("") const useBrowser = tabs.length > SLICE_PILL_THRESHOLD const activeTab = tabs.find((tab) => tab.key === activeSubtaskTab) const filteredTabs = useMemo(() => { const query = search.trim().toLowerCase() if (!query) return tabs return tabs.filter((tab) => tab.label.toLowerCase().includes(query)) }, [search, tabs]) if (!useBrowser) { return (
Benchmark slices
{tabs.map((tab) => ( ))}
) } return (
Benchmark slices
{tabs.length} total
{activeTab && ( )}
{ setBrowserOpen(open) if (!open) setSearch("") }} > Browse benchmark slices {tabs.length} slices in this benchmark. Pick one to filter the leaderboard, or close to keep showing all slices. setSearch(event.target.value)} placeholder="Search slices..." autoFocus />
{filteredTabs.length === 0 ? (
No slices match "{search}".
) : ( filteredTabs.map((tab) => ( )) )}
) } const PARAM_RANGE_VALUES = [1, 2, 3, 4, 6, 8, 10, 12, 16, 24, 32, 48, 64, 96, 128, 192, 256, 384, 500] as const const PARAM_RANGE_MARKERS = [ { label: "< 1B", step: 0 }, { label: "6B", step: PARAM_RANGE_VALUES.indexOf(6) }, { label: "12B", step: PARAM_RANGE_VALUES.indexOf(12) }, { label: "32B", step: PARAM_RANGE_VALUES.indexOf(32) }, { label: "128B", step: PARAM_RANGE_VALUES.indexOf(128) }, { label: "> 500B", step: PARAM_RANGE_VALUES.length - 1 }, ] as const function formatParamBoundLabel(step: number, bound: "min" | "max") { const maxStepIndex = PARAM_RANGE_VALUES.length - 1 if (bound === "min" && step <= 0) { return "< 1B" } if (bound === "max" && step >= maxStepIndex) { return "> 500B" } const value = PARAM_RANGE_VALUES[step] return value != null ? `${value}B` : "Not reported" } function parseParamsBillionsFromText(value: string | null | undefined) { if (!value) { return null } const normalized = value.trim().toLowerCase() if (!normalized) { return null } const compact = normalized.replace(/,/g, "") const tokenMatch = compact.match(/(\d+(?:\.\d+)?)\s*(trillion|tn|t|billion|bn|b|million|mn|m|thousand|k)\b/) if (tokenMatch) { const amount = Number.parseFloat(tokenMatch[1]) if (!Number.isFinite(amount)) { return null } const unit = tokenMatch[2] if (unit === "trillion" || unit === "tn" || unit === "t") { return amount * 1000 } if (unit === "billion" || unit === "bn" || unit === "b") { return amount } if (unit === "million" || unit === "mn" || unit === "m") { return amount / 1000 } if (unit === "thousand" || unit === "k") { return amount / 1_000_000 } } const numeric = Number.parseFloat(compact) return Number.isFinite(numeric) ? numeric : null } function parseParamsBillionsFromModelName(modelName: string | null | undefined) { if (!modelName) { return null } const sizeTokens = Array.from(modelName.matchAll(/\b(\d+(?:\.\d+)?)\s*([tmbk])\b/gi)) if (sizeTokens.length === 0) { return null } const lastToken = sizeTokens[sizeTokens.length - 1] const numericValue = Number.parseFloat(lastToken[1]) if (!Number.isFinite(numericValue)) { return null } const unit = lastToken[2].toLowerCase() if (unit === "t") { return numericValue * 1000 } if (unit === "b") { return numericValue } if (unit === "m") { return numericValue / 1000 } if (unit === "k") { return numericValue / 1_000_000 } return null } function getParamsBillionsFromModelInfo(modelInfo: ModelResultForBenchmark["model_info"]) { const additionalDetails = modelInfo.additional_details const rawParamsBillions = additionalDetails?.params_billions ?? additionalDetails?.parameter_count ?? additionalDetails?.num_parameters ?? additionalDetails?.params if (typeof rawParamsBillions === "number") { return rawParamsBillions } if (typeof rawParamsBillions === "string") { const parsed = parseParamsBillionsFromText(rawParamsBillions) if (Number.isFinite(parsed)) { return parsed } } if (typeof modelInfo.parameter_count === "string") { const parsed = parseParamsBillionsFromText(modelInfo.parameter_count) if (Number.isFinite(parsed)) { return parsed } } return parseParamsBillionsFromModelName(modelInfo.name) } function getParamsBillions(modelResult: ModelResultForBenchmark) { return getParamsBillionsFromModelInfo(modelResult.model_info) } function formatMetadataValue(value: unknown): string { if (value == null) { return "N/A" } if (typeof value === "string") { return value } if (typeof value === "number") { return Number.isFinite(value) ? value.toLocaleString() : String(value) } if (typeof value === "boolean") { return value ? "true" : "false" } if (Array.isArray(value)) { return value.map((item) => formatMetadataValue(item)).join(", ") } try { return JSON.stringify(value) } catch { return String(value) } } function formatDate(ts: string) { if (!ts || !ts.trim()) { return "Unknown" } const numeric = Number(ts) const parsedDate = !Number.isNaN(numeric) && !ts.includes("-") ? new Date(numeric * 1000) : new Date(ts) if (Number.isNaN(parsedDate.getTime())) { return "Unknown" } try { return parsedDate.toLocaleDateString("en-US", { year: "numeric", month: "short", day: "numeric", }) } catch { return ts } } function formatRawScore(score: number, unit?: string) { const suffix = unit ? ` ${unit}` : "" return `${score.toFixed(2)}${suffix}` } function isNumericScore(value: number | null | undefined): value is number { return typeof value === "number" && Number.isFinite(value) } function metricLabelReadsAsPercentage(metricLabel: string, unit?: string) { const normalized = `${metricLabel} ${unit ?? ""}`.toLowerCase() return unit === "%" || /percent|percentage|accuracy|exact match|win rate|pass@|precision|recall|f1/.test(normalized) } function describeLeaderboardMetric(metric: LeaderboardMetric) { const metricLabel = getCompactMetricLabel(metric.display_name) const metricPhrase = metricLabelReadsAsPercentage(metricLabel, metric.unit) ? `${metricLabel} percentage` : metricLabel if (metric.scope === "subtask" && metric.subtask_name) { return `${metricPhrase} for ${metric.subtask_name}` } return metric.canonical_display_name || metric.display_name } function getCompactMetricLabel(value: string | undefined) { if (!value) { return "Metric" } const parts = value .split("/") .map((part) => part.trim()) .filter(Boolean) return parts[parts.length - 1] ?? value } function getRankBadgeClass(rank: number) { if (rank === 1) { return "border-amber-300 bg-amber-100 text-amber-800" } if (rank === 2) { return "border-slate-300 bg-slate-100 text-slate-700" } if (rank === 3) { return "border-orange-300 bg-orange-100 text-orange-800" } return "border-border bg-background text-foreground" } export function EvalDetail({ summary }: EvalDetailProps) { const { mode } = useAudienceMode() const isResearchView = mode === "research" const hasMultiMetricLeaderboard = (summary.leaderboard_metrics?.length ?? 0) > 1 && (summary.leaderboard_rows?.length ?? 0) > 0 const [overviewOpen, setOverviewOpen] = useState(true) // Collapse the dense technical overview by default in policy mode; expand // for researchers. Reset whenever the user switches modes. useEffect(() => { setOverviewOpen(isResearchView) }, [isResearchView]) const [expandedRows, setExpandedRows] = useState>({}) const [leaderboardPage, setLeaderboardPage] = useState(1) const [minParamStep, setMinParamStep] = useState(0) const [maxParamStep, setMaxParamStep] = useState(PARAM_RANGE_VALUES.length - 1) const maxScore = summary.metric_config.max_score ?? 1 const minScore = summary.metric_config.min_score ?? 0 const range = maxScore - minScore const normalizeScore = (raw: number) => (range > 0 ? (raw - minScore) / range : raw) const formatPercent = (normalized: number) => `${(normalized * 100).toFixed(1)}%` const maxParamStepIndex = PARAM_RANGE_VALUES.length - 1 const minHandlePercent = (minParamStep / maxParamStepIndex) * 100 const maxHandlePercent = (maxParamStep / maxParamStepIndex) * 100 const numericMinParams = useMemo(() => { if (minParamStep <= 0) { return null } return PARAM_RANGE_VALUES[minParamStep] ?? null }, [minParamStep]) const numericMaxParams = useMemo(() => { if (maxParamStep >= PARAM_RANGE_VALUES.length - 1) { return null } return PARAM_RANGE_VALUES[maxParamStep] ?? null }, [maxParamStep]) const sortedResults = useMemo( () => [...summary.model_results].sort((a, b) => summary.metric_config.lower_is_better ? a.score - b.score : b.score - a.score ), [summary.model_results, summary.metric_config.lower_is_better] ) const hasParameterData = useMemo( () => sortedResults.some((result) => getParamsBillions(result) != null), [sortedResults] ) const filteredResults = useMemo(() => { return sortedResults.filter((modelResult) => { const paramsBillions = getParamsBillions(modelResult) if (numericMinParams != null && (paramsBillions == null || paramsBillions < numericMinParams)) { return false } if (numericMaxParams != null && (paramsBillions == null || paramsBillions > numericMaxParams)) { return false } return true }) }, [numericMaxParams, numericMinParams, sortedResults]) const leaderboardRows = useMemo(() => { let currentRank = 0 let previousScore: number | null = null return filteredResults.map((modelResult, index) => { if (previousScore === null || Math.abs(modelResult.score - previousScore) > 1e-9) { currentRank = index + 1 previousScore = modelResult.score } return { key: `${modelResult.model_info.id}-${index}`, rank: currentRank, modelResult, normalizedScore: normalizeScore(modelResult.score), } }) }, [filteredResults]) const LEADERBOARD_PAGE_SIZE = 50 const pagedLeaderboardRows = useMemo( () => leaderboardRows.slice(0, leaderboardPage * LEADERBOARD_PAGE_SIZE), [leaderboardRows, leaderboardPage] ) const avgScoreLabel = formatRawScore(summary.avg_score, summary.metric_config.unit) const scoreDirectionLabel = summary.metric_config.lower_is_better ? "Lower scores rank higher" : "Higher scores rank higher" const leaderboardTitle = isResearchView ? "Leaderboard" : "Reporting Comparison" const sourceDatasetLabel = summary.source_data?.hf_repo ?? summary.source_data?.dataset_name ?? "Summary source" const instanceDataLabel = summary.instance_data?.available ? `${summary.instance_data.url_count.toLocaleString()} linked URL${summary.instance_data.url_count === 1 ? "" : "s"}` : "Not linked" const leaderboardDescription = isResearchView ? summary.is_aggregated ? "Models ranked by average raw score across the contributing composite benchmarks." : "Models ranked by raw score for this benchmark." : summary.is_aggregated ? "Averaged model results across the contributing composite benchmarks, with drill-down to each component score." : "Model results with benchmark context, source dataset detail, and optional instance-data links." const reportingCompleteness = summary.evalcards?.annotations?.reporting_completeness const benchmarkComparability = summary.evalcards?.annotations?.benchmark_comparability const documentationPopulatedCount = reportingCompleteness ? getCompletenessPopulatedCount(reportingCompleteness) : null const toggleRow = (key: string) => setExpandedRows((current) => ({ ...current, [key]: !current[key], })) return (
{!isResearchView && }
{summary.is_aggregated ? "Merged Benchmark" : "Single Benchmark"} {summary.is_aggregated ? ( {summary.aggregate_sources?.length ?? 0} composite benchmarks ) : ( Composite: {summary.composite_benchmark_name} )} {summary.metric_config.score_type} {summary.metric_config.lower_is_better ? "Lower is better" : "Higher is better"} {summary.tags?.languages && summary.tags.languages.length > 0 && ( {summary.tags.languages.join(", ")} )}

{summary.metric_config.evaluation_description}

Models
{summary.models_count}
{hasMultiMetricLeaderboard ? "Measures" : isResearchView ? "Avg score" : "Measures"}
{hasMultiMetricLeaderboard ? summary.metrics_count ?? summary.leaderboard_metrics?.length ?? 1 : isResearchView ? avgScoreLabel : summary.metrics_count ?? 1}
{hasMultiMetricLeaderboard || !isResearchView ? "Source dataset" : "Top model"}
{hasMultiMetricLeaderboard || !isResearchView ? sourceDatasetLabel : summary.best_model?.name ?? "Unknown"}
{!hasMultiMetricLeaderboard && isResearchView && summary.best_model && (
{formatRawScore(summary.best_model.score, summary.metric_config.unit)}
)}
{hasMultiMetricLeaderboard || !isResearchView ? "Instance data" : "Bottom model"}
{hasMultiMetricLeaderboard || !isResearchView ? instanceDataLabel : summary.worst_model?.name ?? "Unknown"}
{!hasMultiMetricLeaderboard && isResearchView && summary.worst_model && (
{formatRawScore(summary.worst_model.score, summary.metric_config.unit)}
)}
{isResearchView ? "Metric specification" : "Reading context"}
Composite benchmark
{summary.is_aggregated ? summary.aggregate_sources?.map((source) => source.composite_benchmark_name).join(", ") || "Multiple composite benchmarks" : summary.composite_benchmark_name}
{isResearchView ? "Single benchmark ID" : "What this covers"}
{isResearchView ? summary.evaluation_id : summary.metric_config.evaluation_description}
{isResearchView ? "Score scale" : "How to read scores"}
{isResearchView ? `${summary.metric_config.min_score ?? 0} - ${summary.metric_config.max_score ?? 1}` : scoreDirectionLabel}
{summary.tags?.domains && summary.tags.domains.length > 0 && (
Domain tags
{summary.tags.domains.slice(0, 2).join(", ")} {summary.tags.domains.length > 2 ? ` +${summary.tags.domains.length - 2} more` : ""}
)}
{isResearchView ? "Source dataset" : "Instance data"}
{isResearchView ? sourceDatasetLabel : instanceDataLabel}
{!hasMultiMetricLeaderboard && (summary.root_metrics?.length || summary.subtasks?.length) ? (
Benchmark structure
Benchmark-level summary metrics and subtask slices grouped in one compact section.
{summary.root_metrics && summary.root_metrics.length > 0 && (
Benchmark-level metrics
Benchmark summary metrics used in this evaluation view.
{summary.root_metrics.map((metric) => ( {getCompactMetricLabel(metric.display_name)} {typeof metric.top_score === "number" ? ` · ${formatRawScore(metric.top_score, metric.unit)}` : ""} ))}
)} {summary.subtasks && summary.subtasks.length > 0 && (
Subtask breakdown
{summary.subtasks.map((subtask) => (
{subtask.display_name || subtask.subtask_name}
{subtask.canonical_display_name || subtask.display_name}
{subtask.metrics.map((metric) => ( {getCompactMetricLabel(metric.display_name)} {typeof metric.top_score === "number" ? ` · ${formatRawScore(metric.top_score, metric.unit)}` : ""} ))}
))}
)}
) : null} {summary.benchmark_card && ( )}
{hasMultiMetricLeaderboard ? ( ) : (
{leaderboardTitle}
{leaderboardDescription}
{leaderboardRows.length === summary.models_count ? `${summary.models_count} models` : `${leaderboardRows.length} of ${summary.models_count} models`} {scoreDirectionLabel} {hasParameterData && (numericMinParams != null || numericMaxParams != null) && ( Params {formatParamBoundLabel(minParamStep, "min")} to {formatParamBoundLabel(maxParamStep, "max")} )} {isResearchView && ( Scale {summary.metric_config.min_score ?? 0} - {summary.metric_config.max_score ?? 1} )}
{hasParameterData && (
Parameter range
Narrow the leaderboard to comparable model sizes.
{PARAM_RANGE_MARKERS.map((marker) => ( {marker.label} ))}
{PARAM_RANGE_VALUES.map((_, stepIndex) => (
{ const nextMin = Number(event.target.value) setMinParamStep(Math.min(nextMin, maxParamStep)) }} className="param-range-input" aria-label="Minimum parameter filter" /> { const nextMax = Number(event.target.value) setMaxParamStep(Math.max(nextMax, minParamStep)) }} className="param-range-input" aria-label="Maximum parameter filter" />
{formatParamBoundLabel(minParamStep, "min")} to {formatParamBoundLabel(maxParamStep, "max")}
)} Rank Model {isResearchView ? "Developer" : "Provider"} Score {isResearchView ? ( Performance ) : ( Source type )} {isResearchView ? "Evaluator" : "Reporting Org"} Updated {pagedLeaderboardRows.map(({ key, rank, modelResult, normalizedScore }) => { const isExpanded = expandedRows[key] ?? false const subtasks = modelResult.score_details.details ? Object.entries(modelResult.score_details.details).filter(([, value]) => typeof value === "number") : [] const hasExpandableDetails = isResearchView || (modelResult.aggregate_components && modelResult.aggregate_components.length > 1) || subtasks.length > 1 const datasetName = Array.isArray(modelResult.source_data) ? undefined : modelResult.source_data.dataset_name const samples = Array.isArray(modelResult.source_data) ? undefined : modelResult.source_data.samples_number const rowAnnotations = modelResult.result.evalcards?.annotations return (
{rank}
{hasExpandableDetails && ( )} {modelResult.model_info.name}
{modelResult.model_info.parameter_count && ( {modelResult.model_info.parameter_count} )} {modelResult.model_info.architecture && ( {modelResult.model_info.architecture} )} {modelResult.model_info.developer ?? "Unknown developer"} {modelResult.aggregate_components && modelResult.aggregate_components.length > 1 && ( Avg of {modelResult.aggregate_components.length} )}
{modelResult.model_info.developer ?? "Unknown developer"}
{formatRawScore(modelResult.score, summary.metric_config.unit)}
{isResearchView ? (
) : (
{modelResult.aggregate_components && modelResult.aggregate_components.length > 1 ? `average of ${modelResult.aggregate_components.length} composite scores` : datasetName ?? "Detailed result source"}
)} {modelResult.aggregate_components && modelResult.aggregate_components.length > 1 ? (
{Array.from(new Set(modelResult.aggregate_components.map((component) => component.source_organization_name))).join(", ")}
{modelResult.aggregate_components .map((component) => component.composite_benchmark_name) .join(", ")}
) : (
{datasetName ?? sourceDatasetLabel}
{Array.isArray(modelResult.source_data) ? "Detailed result source" : modelResult.source_data.hf_repo ?? modelResult.source_data.source_type ?? "Detailed result source"}
)}
{formatDate(modelResult.evaluation_timestamp)}
{isExpanded && (
{isResearchView && } {modelResult.source_metadata.source_url && ( View source } /> )} {!isResearchView && ( )} {!isResearchView && ( <> {modelResult.score_details.confidence_interval && ( )} )}
{modelResult.aggregate_components && modelResult.aggregate_components.length > 1 && (
Composite Score Breakdown
{modelResult.aggregate_components.map((component, i) => ( ))}
Benchmark Source Raw
{component.composite_benchmark_name} {component.source_organization_name} {formatRawScore(component.score)}
)} {subtasks.length > 1 && (
Subtask Breakdown
{subtasks.map(([subtaskName, value]) => { const numericValue = value as number return ( ) })}
Subtask Raw
{subtaskName.replace(/_/g, " ")} {formatRawScore(numericValue, summary.metric_config.unit)}
)} {isResearchView ? (
) : ( modelResult.result.generation_config && (
Generation Config
Evaluation-time generation parameters.
{modelResult.result.generation_config.generation_args && Object.entries(modelResult.result.generation_config.generation_args).map(([key, value]) => (
{key.replace(/_/g, " ")}
{formatMetadataValue(value)}
))} {modelResult.result.generation_config.additional_details && (
Additional Details
{formatMetadataValue(modelResult.result.generation_config.additional_details)}
)} {modelResult.result.generation_config.prompt_template && (
Prompt Template
{formatMetadataValue(modelResult.result.generation_config.prompt_template)}
)}
) )}
)} ) })} {leaderboardRows.length === 0 && ( No leaderboard entries match the selected parameter range. )} {pagedLeaderboardRows.length < leaderboardRows.length && (
)} )} ) } function MultiMetricLeaderboard({ summary, isResearchView, }: { summary: BenchmarkEvalSummary isResearchView: boolean }) { const [page, setPage] = useState(1) const [sortKey, setSortKey] = useState("coverage") const [sortDirection, setSortDirection] = useState<"asc" | "desc">("desc") const [activeSubtaskTab, setActiveSubtaskTab] = useState("all") const [minParamStep, setMinParamStep] = useState(0) const [maxParamStep, setMaxParamStep] = useState(PARAM_RANGE_VALUES.length - 1) const [expandedRows, setExpandedRows] = useState>({}) // Index ModelResultForBenchmark entries by model_info.id so we can power the // research-mode reproducibility card from a multi-metric row. There may be // several entries per model (one per metric); we prefer one with a recorded // generation_config so the card has the most data to show. const modelResultByModelId = useMemo(() => { const map = new Map() for (const result of summary.model_results) { const id = result.model_info.id const existing = map.get(id) if (!existing) { map.set(id, result) continue } const existingHasGen = existing.result.generation_config != null const candidateHasGen = result.result.generation_config != null if (!existingHasGen && candidateHasGen) { map.set(id, result) } } return map }, [summary.model_results]) const toggleExpandedRow = (key: string) => setExpandedRows((current) => ({ ...current, [key]: !current[key] })) const leaderboardMetrics = summary.leaderboard_metrics ?? [] const leaderboardRows = summary.leaderboard_rows ?? [] const allMetricKeys = useMemo(() => leaderboardMetrics.map((metric) => metric.column_key), [leaderboardMetrics]) // Cap default visible columns to avoid hangs on benchmarks with hundreds of metrics // (e.g. helm_air_bench has 374 subtask×metric pairs). Users can opt in to more. const DEFAULT_VISIBLE_METRIC_CAP = 24 const defaultVisibleMetricKeys = useMemo( () => allMetricKeys.slice(0, DEFAULT_VISIBLE_METRIC_CAP), [allMetricKeys] ) const [visibleMetricKeys, setVisibleMetricKeys] = useState(() => defaultVisibleMetricKeys) const maxParamStepIndex = PARAM_RANGE_VALUES.length - 1 const leaderboardMetricMap = useMemo( () => new Map(leaderboardMetrics.map((metric) => [metric.column_key, metric])), [leaderboardMetrics] ) const visibleMetricKeySet = useMemo(() => new Set(visibleMetricKeys), [visibleMetricKeys]) const subtaskMetricCounts = useMemo(() => { const counts = new Map() for (const metric of leaderboardMetrics) { if (metric.scope === "subtask" && metric.subtask_key) { counts.set(metric.subtask_key, (counts.get(metric.subtask_key) ?? 0) + 1) } } return counts }, [leaderboardMetrics]) const singleMetricSubtaskTabs = useMemo(() => { return leaderboardMetrics .filter((metric) => metric.scope === "subtask" && metric.subtask_key && subtaskMetricCounts.get(metric.subtask_key) === 1) .map((metric) => ({ key: metric.subtask_key as string, label: metric.subtask_name ?? getCompactMetricLabel(metric.display_name), })) }, [leaderboardMetrics, subtaskMetricCounts]) const hasSubtaskTabs = singleMetricSubtaskTabs.length > 1 const visibleMetrics = useMemo( () => leaderboardMetrics.filter((metric) => { if (!visibleMetricKeySet.has(metric.column_key)) { return false } if (!hasSubtaskTabs || activeSubtaskTab === "all") { return true } return metric.scope === "subtask" && metric.subtask_key === activeSubtaskTab }), [activeSubtaskTab, hasSubtaskTabs, leaderboardMetrics, visibleMetricKeySet] ) const visibleMetricColumnKeySet = useMemo( () => new Set(visibleMetrics.map((metric) => metric.column_key)), [visibleMetrics] ) const numericMinParams = useMemo(() => { if (minParamStep <= 0) { return null } return PARAM_RANGE_VALUES[minParamStep] ?? null }, [minParamStep]) const numericMaxParams = useMemo(() => { if (maxParamStep >= PARAM_RANGE_VALUES.length - 1) { return null } return PARAM_RANGE_VALUES[maxParamStep] ?? null }, [maxParamStep]) const filteredRows = useMemo(() => { return leaderboardRows .filter((row) => { const paramsBillions = getParamsBillionsFromModelInfo(row.model_info) if (numericMinParams != null && (paramsBillions == null || paramsBillions < numericMinParams)) { return false } if (numericMaxParams != null && (paramsBillions == null || paramsBillions > numericMaxParams)) { return false } return true }) }, [leaderboardRows, numericMaxParams, numericMinParams]) const sortedRows = useMemo(() => { const rows = [...filteredRows] const compareNames = (left: LeaderboardMatrixRow, right: LeaderboardMatrixRow) => left.model_info.name.localeCompare(right.model_info.name) || (left.model_info.developer ?? "").localeCompare(right.model_info.developer ?? "") const compareTimestamps = (left: string, right: string) => { const leftNumeric = Number(left) const rightNumeric = Number(right) const leftTimestamp = !Number.isNaN(leftNumeric) && !left.includes("-") ? leftNumeric * 1000 : new Date(left).getTime() const rightTimestamp = !Number.isNaN(rightNumeric) && !right.includes("-") ? rightNumeric * 1000 : new Date(right).getTime() return leftTimestamp - rightTimestamp } rows.sort((left, right) => { if (sortKey === "model") { const comparison = compareNames(left, right) return sortDirection === "asc" ? comparison : -comparison } if (sortKey === "developer") { const comparison = (left.model_info.developer ?? "").localeCompare(right.model_info.developer ?? "") || compareNames(left, right) return sortDirection === "asc" ? comparison : -comparison } if (sortKey === "coverage") { const comparison = left.metrics_present - right.metrics_present || compareNames(left, right) return sortDirection === "asc" ? comparison : -comparison } if (sortKey === "updated") { const comparison = compareTimestamps(left.evaluation_timestamp, right.evaluation_timestamp) || compareNames(left, right) return sortDirection === "asc" ? comparison : -comparison } const metric = leaderboardMetricMap.get(sortKey) if (metric) { const leftValue = left.values[sortKey] const rightValue = right.values[sortKey] const leftHasValue = isNumericScore(leftValue) const rightHasValue = isNumericScore(rightValue) if (leftHasValue && rightHasValue) { const comparison = leftValue - rightValue || compareNames(left, right) return sortDirection === "asc" ? comparison : -comparison } if (leftHasValue !== rightHasValue) { return leftHasValue ? -1 : 1 } } return compareNames(left, right) }) return rows }, [filteredRows, leaderboardMetricMap, sortDirection, sortKey]) useEffect(() => { setPage(1) }, [maxParamStep, minParamStep, sortDirection, sortKey]) useEffect(() => { setVisibleMetricKeys(defaultVisibleMetricKeys) }, [defaultVisibleMetricKeys, summary.evaluation_id]) useEffect(() => { setActiveSubtaskTab("all") }, [summary.evaluation_id]) useEffect(() => { if (leaderboardMetricMap.has(sortKey) && !visibleMetricColumnKeySet.has(sortKey)) { setSortKey("coverage") setSortDirection("desc") } }, [leaderboardMetricMap, sortKey, visibleMetricColumnKeySet]) useEffect(() => { if (!hasSubtaskTabs) { if (activeSubtaskTab !== "all") { setActiveSubtaskTab("all") } return } if (activeSubtaskTab === "all") { return } if (!singleMetricSubtaskTabs.some((tab) => tab.key === activeSubtaskTab)) { setActiveSubtaskTab("all") } }, [activeSubtaskTab, hasSubtaskTabs, singleMetricSubtaskTabs]) const hasParameterData = useMemo( () => leaderboardRows.some((row) => getParamsBillionsFromModelInfo(row.model_info) != null), [leaderboardRows] ) const metricRanges = useMemo(() => { const ranges = new Map() for (const metric of leaderboardMetrics) { const scores = filteredRows .map((row) => row.values[metric.column_key]) .filter(isNumericScore) if (scores.length > 0) { ranges.set(metric.column_key, { min: Math.min(...scores), max: Math.max(...scores), }) } } return ranges }, [filteredRows, leaderboardMetrics]) const pagedRows = useMemo( () => sortedRows.slice(0, page * 50), [page, sortedRows] ) const rankByModelId = useMemo( () => new Map(sortedRows.map((row, index) => [row.model_info.id, index + 1])), [sortedRows] ) const setMetricVisibility = (metricKey: string, nextVisible: boolean) => { setVisibleMetricKeys((current) => { if (nextVisible) { return allMetricKeys.filter((key) => key === metricKey || current.includes(key)) } return current.filter((key) => key !== metricKey) }) } const getVisibleMetricCount = (row: LeaderboardMatrixRow) => visibleMetrics.reduce( (count, metric) => count + (isNumericScore(row.values[metric.column_key]) ? 1 : 0), 0 ) const getDefaultSortDirection = (key: string): "asc" | "desc" => { if (key === "model" || key === "developer") { return "asc" } if (key === "updated" || key === "coverage") { return "desc" } return leaderboardMetricMap.get(key)?.lower_is_better ? "asc" : "desc" } const handleSort = (key: string) => { if (sortKey === key) { setSortDirection((current) => (current === "asc" ? "desc" : "asc")) return } setSortKey(key) setSortDirection(getDefaultSortDirection(key)) } const getSortIndicator = (key: string) => { if (sortKey !== key) { return "" } return sortDirection === "asc" ? " ▲" : " ▼" } return (
{isResearchView ? "Leaderboard" : "Reporting Comparison"}
{isResearchView ? "Each column is a reported benchmark measure. Distinct measures stay separate instead of collapsing into a single raw score." : "Each column is a separately reported measure so the benchmark can be read without flattening different results into one number."}
{filteredRows.length === leaderboardRows.length ? `${leaderboardRows.length} models` : `${filteredRows.length} of ${leaderboardRows.length} models`} {visibleMetrics.length === leaderboardMetrics.length ? `${leaderboardMetrics.length} measures` : `${visibleMetrics.length} of ${leaderboardMetrics.length} measures`} {hasParameterData && (numericMinParams != null || numericMaxParams != null) && ( Params {formatParamBoundLabel(minParamStep, "min")} to {formatParamBoundLabel(maxParamStep, "max")} )} Visible measure columns setVisibleMetricKeys(allMetricKeys)}> Show all {leaderboardMetrics.map((metric) => { const isVisible = visibleMetricKeySet.has(metric.column_key) const isLastVisible = isVisible && visibleMetrics.length === 1 const visibleLabel = metric.scope === "subtask" && metric.subtask_key && subtaskMetricCounts.get(metric.subtask_key) === 1 && metric.subtask_name ? metric.subtask_name : getCompactMetricLabel(metric.display_name) return ( setMetricVisibility(metric.column_key, checked === true)} className="items-start" >
{visibleLabel} {describeLeaderboardMetric(metric)}
) })}
{hasSubtaskTabs && (
)} {hasParameterData && (
Parameter range
Narrow the matrix to comparable model sizes.
{PARAM_RANGE_MARKERS.map((marker) => ( {marker.label} ))}
{PARAM_RANGE_VALUES.map((_, stepIndex) => (
{ const nextMin = Number(event.target.value) setMinParamStep(Math.min(nextMin, maxParamStep)) }} className="param-range-input" aria-label="Minimum parameter filter" /> { const nextMax = Number(event.target.value) setMaxParamStep(Math.max(nextMax, minParamStep)) }} className="param-range-input" aria-label="Maximum parameter filter" />
{formatParamBoundLabel(minParamStep, "min")} to {formatParamBoundLabel(maxParamStep, "max")}
)}
Rank {visibleMetrics.map((metric) => ( ))} {pagedRows.map((row) => { const rank = rankByModelId.get(row.model_info.id) ?? 0 const expandKey = row.model_info.id const isExpanded = expandedRows[expandKey] ?? false const matchingResult = modelResultByModelId.get(row.model_info.id) return (
{rank}
{isResearchView && matchingResult && ( )} {row.model_info.name}
{row.model_info.parameter_count && ( {row.model_info.parameter_count} )} {row.model_info.architecture && ( {row.model_info.architecture} )} {row.model_info.developer ?? "Unknown developer"}
{row.model_info.developer ?? "Unknown developer"}
{getVisibleMetricCount(row)}/{visibleMetrics.length} {visibleMetrics.map((metric) => { const score = row.values[metric.column_key] const annotations = row.annotations_by_metric?.[metric.column_key] return (
{isNumericScore(score) ? formatRawScore(score, metric.unit) : "—"}
) })} {formatDate(row.evaluation_timestamp)}
{isResearchView && isExpanded && matchingResult && (
)}
)})} {filteredRows.length === 0 && ( No models match the selected parameter range. )}
{pagedRows.length < filteredRows.length && (
)} ) } function BenchmarkCardCollapsible({ card, isResearchView, defaultOpen = true, defaultRisksOpen = false, knownIssues = [], }: { card: BenchmarkCard isResearchView: boolean defaultOpen?: boolean defaultRisksOpen?: boolean knownIssues?: KnownIssue[] }) { const [open, setOpen] = useState(defaultOpen) return ( ) } function DetailPanel({ title, subtitle, children, }: { title: string subtitle: string children: React.ReactNode }) { return (
{title}
{subtitle}
{children}
) } function MetaRow({ label, value, }: { label: string value: React.ReactNode }) { // Hide rows whose value is missing or a generic placeholder. This keeps // the detail panels focused on fields we actually have data for. if (value == null) return null if (typeof value === "string") { const normalized = value.trim().toLowerCase() if ( normalized === "" || normalized === "unknown" || normalized === "n/a" || normalized === "not recorded" || normalized === "not specified" || normalized === "not linked" ) { return null } } return (
{label}
{value}
) } function toStringArray(value: unknown): string[] { const result = new Set() const visit = (candidate: unknown) => { if (!candidate) { return } if (Array.isArray(candidate)) { for (const item of candidate) { visit(item) } return } if (typeof candidate === "object") { for (const item of Object.values(candidate)) { visit(item) } return } if (typeof candidate !== "string") { return } const normalized = candidate.trim() if (!normalized || normalized === "Not specified") { return } for (const part of normalized.split(/[,;|]/)) { const token = part.trim() if (token && token !== "Not specified") { result.add(token) } } } visit(value) return Array.from(result) } function BenchmarkCardPanel({ card, isResearchView, defaultRisksOpen = false, knownIssues = [], }: { card: BenchmarkCard isResearchView: boolean defaultRisksOpen?: boolean knownIssues?: KnownIssue[] }) { const [risksOpen, setRisksOpen] = useState(defaultRisksOpen) const details = card.benchmark_details const purpose = card.purpose_and_intended_users const methodology = card.methodology const data = card.data const ethical = card.ethical_and_legal_considerations const risks = card.possible_risks ?? [] const flaggedFields = Object.entries(card.flagged_fields ?? {}) const missingFields = card.missing_fields ?? [] const domains = toStringArray(details.domains) const languages = toStringArray(details.languages) const resources = (details.resources ?? []).filter(Boolean) const tasks = toStringArray(purpose.tasks) const audience = toStringArray(purpose.audience) const license = ethical.data_licensing ?? "" const shortLicense = license && license !== "Not specified" ? license : null return (
Benchmark Card {shortLicense && ( {shortLicense} )} {(flaggedFields.length > 0 || missingFields.length > 0) && ( {flaggedFields.length} flagged · {missingFields.length} missing )}
Structured metadata about this benchmark: what it measures, how it was built, and known limitations. {card.card_info?.llm && ( Card generated by {card.card_info.llm}. )}
{knownIssues.length > 0 && } {/* Overview + domains */}

{details.overview}

{domains.map((d) => ( {d} ))} {languages.map((l) => ( {l} ))}
{/* Goal */}
Goal

{purpose.goal}

{/* Metric interpretation */}
Score interpretation

{methodology.interpretation}

{/* Limitations */}
Limitations

{purpose.limitations}

{(methodology.methods?.length > 0 || (methodology.calculation && methodology.calculation !== "Not specified") || (methodology.validation && methodology.validation !== "Not specified")) && (
How tasks were sourced and scored
{methodology.methods?.length > 0 && (
Task setup
    {methodology.methods.map((m, i) => (
  1. {m}
  2. ))}
)}
{methodology.calculation && methodology.calculation !== "Not specified" && (
Score calculation

{methodology.calculation}

)} {methodology.validation && methodology.validation !== "Not specified" && (
Validation

{methodology.validation}

)}
)} {/* Research-only: methodology + dataset details */} {isResearchView && (
Dataset
Size
{data.size}
Format
{data.format}
Source
{data.source}
Methodology
{methodology.metrics.length > 0 && (
Metrics
{methodology.metrics.join(", ")}
)} {tasks.length > 0 && (
Tasks
{tasks.join(", ")}
)} {audience.length > 0 && (
Audience
{audience.join("; ")}
)}
)} {/* Generic IBM-style AI risks. These are boilerplate (per audit feedback: "least useful feature for policy users"), so in policy mode we hide them entirely — the curated known-issues panel above carries the benchmark-specific concerns. Researchers still get the full collapsible list. */} {risks.length > 0 && isResearchView && (
{risks.map((risk, i) => (
{risk.description?.[0] && (

{risk.description[0]}

)}
))}
)} {/* Compliance / ethical notes (policy view emphasis) */} {!isResearchView && (
Ethical & legal
{shortLicense && (
License
{license}
)} {ethical.compliance_with_regulations && ethical.compliance_with_regulations !== "Not specified" && (
Compliance
{ethical.compliance_with_regulations}
)} {ethical.privacy_and_anonymity && ethical.privacy_and_anonymity !== "Not specified" && (
Privacy
{ethical.privacy_and_anonymity}
)}
)} {/* Flagged / missing fields warning */} {(flaggedFields.length > 0 || missingFields.length > 0) && isResearchView && (
Card quality notes
{flaggedFields.length > 0 && (
    {flaggedFields.map(([field, note]) => (
  • {field}: {note}
  • ))}
)} {missingFields.length > 0 && (

Missing: {missingFields.join(", ")}

)}
)} {/* External resources */} {resources.length > 0 && ( )}
) }