"use client" import { Fragment, useMemo, useState } from "react" import Link from "next/link" import { ChevronDown, ExternalLink, } from "lucide-react" import type { BenchmarkEvaluationCardData } from "@/components/benchmark-evaluation-card" import { routeIdToPath } from "@/lib/utils" import { Badge } from "@/components/ui/badge" import { Button } from "@/components/ui/button" import { Collapsible, CollapsibleContent, CollapsibleTrigger, } from "@/components/ui/collapsible" import { Dialog, DialogContent, DialogDescription, DialogHeader, DialogTitle, } from "@/components/ui/dialog" import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table" function formatDate(isoString: string | null | undefined) { if (!isoString) return "—" const numeric = Number(isoString) const parsedDate = !Number.isNaN(numeric) && !isoString.includes("-") ? new Date(numeric * 1000) : new Date(isoString) try { return parsedDate.toLocaleDateString("en-US", { year: "numeric", month: "short", day: "numeric", }) } catch { return isoString } } function parseParamsBillionsFromModelName(modelName: string | null | undefined) { if (!modelName) return null // Parse explicit size tokens like 7B, 40B, or 560M from model names. const sizeTokens = Array.from(modelName.matchAll(/\b(\d+(?:\.\d+)?)\s*([bm])\b/gi)) if (sizeTokens.length === 0) return null const lastToken = sizeTokens[sizeTokens.length - 1] const numericValue = Number(lastToken[1]) if (!Number.isFinite(numericValue)) return null const unit = lastToken[2].toLowerCase() if (unit === "b") return numericValue if (unit === "m") return numericValue / 1000 return null } function formatParamsBillions(value: number | null | undefined, modelName?: string) { const resolvedValue = value == null || Number.isNaN(value) ? parseParamsBillionsFromModelName(modelName) : value if (resolvedValue == null || Number.isNaN(resolvedValue)) return "Not reported" if (resolvedValue >= 100) return `${Math.round(resolvedValue)}B` return `${resolvedValue.toFixed(1)}B` } function formatBenchmarkScore(score: number, unit?: string) { if (unit === "accuracy" || unit === "pass@1" || (!unit && score >= 0 && score <= 1)) { return `${(score * 100).toFixed(1)}` } if (unit === "points") { return score.toFixed(1) } return score.toFixed(2) } function formatSummaryScore(value: number | null | undefined) { if (value == null || !Number.isFinite(value)) return "Not summarized" if (value >= 0 && value <= 1) return `${(value * 100).toFixed(1)}%` if (Math.abs(value) >= 100) return value.toFixed(0) return value.toFixed(2) } function getBenchmarkSection(name: string) { const value = name.toLowerCase() if ( value.includes("tau") || value.includes("swe-bench") || value.includes("browsecomp") || value.includes("agent") ) { return "Agentic" } if ( value.includes("aime") || value.includes("gpqa") || value.includes("hmmt") || value.includes("beyond aime") || value.includes("reason") ) { return "Reasoning" } if ( value.includes("math") || value.includes("mmlu") || value.includes("ifeval") || value.includes("arena") || value.includes("live code") || value.includes("humaneval") || value.includes("mbpp") || value.includes("code") ) { return "General" } return "Other" } const SECTION_ORDER = ["General", "Reasoning", "Agentic", "Other"] const CONTEXT_ROWS = [ { key: "developer", label: "Developer" }, { key: "params", label: "Parameter range" }, { key: "benchmarks", label: "Benchmark coverage" }, { key: "variants", label: "Versions" }, { key: "score_summary", label: "Score range" }, { key: "reproducibility", label: "Re-runnability" }, { key: "latest", label: "Latest summary" }, { key: "updated", label: "Updated" }, ] as const interface ModelCompareDialogProps { models: BenchmarkEvaluationCardData[] open: boolean onOpenChange: (open: boolean) => void } export function ModelCompareDialog({ models, open, onOpenChange, }: ModelCompareDialogProps) { const [sharedOnly, setSharedOnly] = useState(false) const benchmarkRows = useMemo(() => { const rows = new Map< string, { benchmark: string metric: string section: string values: Record order: number } >() let order = 0 for (const model of models) { for (const score of model.top_scores) { const existing = rows.get(score.benchmark) if (!existing) { rows.set(score.benchmark, { benchmark: score.benchmark, metric: score.metric, section: getBenchmarkSection(score.benchmark), values: { [model.id]: { score: score.score, unit: score.unit, }, }, order, }) order += 1 continue } existing.values[model.id] = { score: score.score, unit: score.unit, } } } return Array.from(rows.values()).sort((a, b) => { const sectionDiff = SECTION_ORDER.indexOf(a.section) - SECTION_ORDER.indexOf(b.section) if (sectionDiff !== 0) { return sectionDiff } return a.order - b.order }) }, [models]) const benchmarkSections = useMemo(() => { return SECTION_ORDER.map((section) => ({ section, rows: benchmarkRows.filter((row) => row.section === section), })).filter((group) => group.rows.length > 0) }, [benchmarkRows]) const sharedBenchmarkCount = useMemo( () => benchmarkRows.filter((row) => Object.keys(row.values).length === models.length).length, [benchmarkRows, models.length] ) const visibleBenchmarkSections = useMemo(() => { if (!sharedOnly) { return benchmarkSections } return benchmarkSections .map((group) => ({ ...group, rows: group.rows.filter((row) => Object.keys(row.values).length === models.length), })) .filter((group) => group.rows.length > 0) }, [benchmarkSections, models.length, sharedOnly]) return (
Side-By-Side Comparison
Compare Selected Models Start with the benchmark table. Use the context table when you need coverage breadth, version spread, or score range detail.

Benchmark comparison

Rows are drawn from the most relevant surfaced benchmarks across the selected models, closer to how release posts present comparison tables.

{benchmarkRows.length} surfaced benchmarks · {sharedBenchmarkCount} shared across all selected models
{models.map((model) => ( ))} {visibleBenchmarkSections.map((group) => ( {group.rows.map((row) => { const rowValues = Object.values(row.values).map((value) => value.score) const maxScore = rowValues.length > 0 ? Math.max(...rowValues) : null return ( {models.map((model) => { const value = row.values[model.id] const isBest = value && maxScore != null && value.score === maxScore return ( ) })} ) })} ))}
Benchmark
{model.model_name}
{model.developer || "Unknown"}
{formatParamsBillions(model.params_billions, model.model_name)} View →
{group.section}
{row.benchmark}
{row.metric}
 {value ? (
{formatBenchmarkScore(value.score, value.unit)}
) : (
––
)}
{models.map((model) => ( ))} {CONTEXT_ROWS.map((row) => ( {models.map((model) => ( ))} ))}
Signal
{model.model_name}
{row.label} {row.key === "developer" ? model.developer || "Unknown developer" : null} {row.key === "params" ? formatParamsBillions(model.params_billions, model.model_name) : null} {row.key === "benchmarks" ? (
{model.benchmarks_count} covered benchmarks
{(model.benchmark_names ?? []).slice(0, 4).join(", ") || `${model.evaluations_count} reported result${model.evaluations_count !== 1 ? "s" : ""}`}
) : null} {row.key === "variants" ? (
{model.variant_count} version{model.variant_count !== 1 ? "s" : ""}
{model.variant_count > 1 ? "Family-level summary spans multiple published variants" : "Single summarized variant"}
) : null} {row.key === "score_summary" ? (
Range {formatSummaryScore(model.score_summary?.min ?? null)} to {formatSummaryScore(model.score_summary?.max ?? null)} across {model.score_summary?.count ?? 0} surfaced scores
) : null} {row.key === "reproducibility" ? ( model.reproducibility_summary && model.reproducibility_summary.has_reproducibility_gap_count > 0 ? (
{model.reproducibility_summary.has_reproducibility_gap_count} setup gaps
Out of {model.reproducibility_summary.results_total} reported scores
) : ( No setup gaps reported ) ) : null} {row.key === "latest" ? (
{model.latest_source_name || `${model.benchmarks_count} benchmark composites summarized`} {model.source_urls?.[0] ? ( ) : null}
) : null} {row.key === "updated" ? formatDate(model.latest_timestamp) : null}
) }