"use client" import { useMemo, useState } from "react" import type { CSSProperties, FocusEvent as ReactFocusEvent, MouseEvent as ReactMouseEvent, } from "react" import { feedbackConditionDescription, type ComputeMark, type FeedbackCondition, type ProtocolSeries, } from "@/lib/collections" interface ScoreSeries { /** Stable key — used by the metric dropdown to switch series. */ key: string /** Short label shown in the dropdown and as the panel sub-title. */ label: string /** Optional longer description shown next to the label. */ caption?: string values: number[] unit?: string lowerIsBetter?: boolean /** * Per-model rows for the optional frontier-plot view. When provided * (and at least one row carries a parseable releaseDate), the panel * exposes a chip toggle that swaps the density curve for a * release-date frontier (cumulative best score over time). */ points?: Array<{ score: number releaseDate?: string | null modelName?: string | null }> } interface ScoreDistributionProps { /** Single-series shorthand. Either pass `values` (single) or `series` (multi). */ values?: number[] label?: string unit?: string lowerIsBetter?: boolean /** Multi-series — when provided, a dropdown picker swaps between them. */ series?: ScoreSeries[] /** Initial selected key when multi-series. Defaults to first. */ initialKey?: string /** Compact variant — shorter, used for matrix per-column distributions. */ compact?: boolean /** Initial view when the active series supports both modes. Defaults * to "distribution". The /embed/.../frontier route passes "frontier" * to open directly on the Pareto-frontier view. */ defaultView?: "distribution" | "frontier" /** When false, the Distribution/Frontier chip toggle is hidden so the * caller can lock the panel to a single view (e.g. inside an embed * iframe that the user explicitly chose to embed as Distribution * *or* Frontier). The metric chips above still appear when the panel * carries more than one series. Defaults to true. */ showViewToggle?: boolean /** Optional Compute view. Only the eval-detail single-metric * call site passes this — matrix/embed sites must not (their row * shapes carry no protocol fields). */ protocol?: ProtocolSeries } interface SummaryStats { n: number min: number max: number mean: number median: number q1: number q3: number } function parseReleaseDate(value: string | null | undefined): number | null { if (!value) return null const raw = String(value).trim() if (!raw) return null // Numeric epoch — treat seconds-since-epoch values as such, ms otherwise. const numeric = Number(raw) if (!Number.isNaN(numeric) && !raw.includes("-")) { const ms = numeric > 1_000_000_000_000 ? numeric : numeric * 1000 return Number.isFinite(ms) ? ms : null } const parsed = new Date(raw).getTime() return Number.isFinite(parsed) ? parsed : null } const MONTH_LABELS = ["Jan","Feb","Mar","Apr","May","Jun","Jul","Aug","Sep","Oct","Nov","Dec"] function formatMonthYear(ms: number): string { const d = new Date(ms) if (Number.isNaN(d.getTime())) return "" return `${MONTH_LABELS[d.getUTCMonth()]} ${d.getUTCFullYear()}` } function computeStats(values: number[]): SummaryStats | null { const sorted = values.filter((v) => Number.isFinite(v)).slice().sort((a, b) => a - b) const n = sorted.length if (n === 0) return null const min = sorted[0] const max = sorted[n - 1] const mean = sorted.reduce((acc, v) => acc + v, 0) / n const quantile = (p: number) => { if (n === 1) return sorted[0] const pos = (n - 1) * p const base = Math.floor(pos) const rest = pos - base return sorted[base + 1] != null ? sorted[base] + rest * (sorted[base + 1] - sorted[base]) : sorted[base] } return { n, min, max, mean, median: quantile(0.5), q1: quantile(0.25), q3: quantile(0.75), } } function formatValue(v: number, unit?: string) { const abs = Math.abs(v) let formatted: string if (abs >= 100) formatted = v.toFixed(1) else if (abs >= 10) formatted = v.toFixed(2) else formatted = v.toFixed(3).replace(/0+$/g, "").replace(/\.$/, "") return unit ? `${formatted} ${unit}` : formatted } /** * Continuous-density distribution plot. * * Builds a smoothed kernel density estimate (KDE) from the raw values rather * than a binned histogram, which reads as a continuous probability-weight * curve in the paper's hairline style. Median and mean are rendered as * vertical rules on top of the curve; IQR is a bracket along the baseline. * * Multi-series mode shows a small dropdown inside the panel header so a * caller (e.g. a multi-metric leaderboard) can stack metrics into one * visualization the user swaps between, instead of rendering N panels. */ export function ScoreDistribution({ values, label, unit, lowerIsBetter, series, initialKey, compact = false, defaultView, showViewToggle = true, protocol, }: ScoreDistributionProps) { // Normalize: either we got a single series (via values) or many. const seriesList: ScoreSeries[] = useMemo(() => { if (series && series.length > 0) return series if (values && values.length > 0) { return [{ key: "__single", label: label ?? "Score", values, unit, lowerIsBetter }] } return [] }, [series, values, label, unit, lowerIsBetter]) const [activeKey, setActiveKey] = useState( () => initialKey ?? series?.[0]?.key ?? "__single", ) const active = seriesList.find((s) => s.key === activeKey) ?? seriesList[0] const stats = useMemo(() => (active ? computeStats(active.values) : null), [active]) // Frontier-plot data: parse release dates, sort by time, then walk the // sequence emitting an event whenever a model improves on the best // score seen so far. Honours lowerIsBetter so e.g. "Mean Response // Time · ms" shows the frontier descending instead of climbing. const frontier = useMemo(() => { if (!active?.points || active.points.length === 0) return null const lowerIsBetter = active.lowerIsBetter ?? false const parsed = active.points .map((p) => { const t = parseReleaseDate(p.releaseDate) if (t == null) return null if (!Number.isFinite(p.score)) return null return { time: t, score: p.score, name: p.modelName ?? "" } }) .filter((p): p is { time: number; score: number; name: string } => p !== null) .sort((a, b) => a.time - b.time) if (parsed.length < 2) return null let best = lowerIsBetter ? Number.POSITIVE_INFINITY : Number.NEGATIVE_INFINITY const events: typeof parsed = [] for (const p of parsed) { const better = lowerIsBetter ? p.score < best : p.score > best if (better) { best = p.score events.push(p) } } if (events.length < 2) return null return { events, samples: parsed } }, [active]) const canShowFrontier = frontier != null const canShowCompute = (protocol?.marks.length ?? 0) > 0 const [view, setView] = useState<"distribution" | "frontier" | "compute">( defaultView ?? "distribution", ) // If the active series doesn't support the selected view (e.g. user // switched to a metric whose models don't carry release_date), fall // back to the distribution view rather than rendering an empty panel. const effectiveView = view === "frontier" && canShowFrontier ? "frontier" : view === "compute" && canShowCompute ? "compute" : "distribution" // When the caller hides the toggle (embed locks to one view), force the // panel to whatever defaultView/view it was created with — the user // can't switch, so any "frontier" inference must come from props. const renderViewToggle = showViewToggle && (canShowFrontier || canShowCompute) const availableViews = [ "distribution" as const, ...(canShowFrontier ? ["frontier" as const] : []), ...(canShowCompute ? ["compute" as const] : []), ] const density = useMemo(() => { if (!active || !stats) return null if (stats.max === stats.min) { return { points: [{ x: stats.min, y: 1 }], maxY: 1 } } const sorted = active.values .filter((v) => Number.isFinite(v)) .slice() .sort((a, b) => a - b) const n = sorted.length if (n === 0) return null // Silverman's rule of thumb for bandwidth. const variance = sorted.reduce((acc, v) => acc + (v - stats.mean) ** 2, 0) / n const stdDev = Math.sqrt(variance) const iqr = stats.q3 - stats.q1 const sigma = iqr > 0 ? Math.min(stdDev, iqr / 1.34) : stdDev || (stats.max - stats.min) / 6 const bandwidth = Math.max( 1.06 * sigma * Math.pow(n, -0.2), (stats.max - stats.min) / 80, ) const sampleCount = compact ? 80 : 140 const range = stats.max - stats.min const xs: number[] = [] for (let i = 0; i < sampleCount; i++) { xs.push(stats.min + (range * i) / (sampleCount - 1)) } const ys = xs.map((x) => { let sum = 0 for (const v of sorted) { const u = (x - v) / bandwidth sum += Math.exp(-0.5 * u * u) } return sum / (n * bandwidth * Math.sqrt(2 * Math.PI)) }) const maxY = Math.max(...ys, 1e-9) const points = xs.map((x, i) => ({ x, y: ys[i] })) return { points, maxY } }, [active, stats, compact]) if (!active || !stats || !density) return null const width = 100 const plotHeight = compact ? 28 : 56 const fullRange = stats.max - stats.min || 1 const markerX = (v: number) => ((v - stats.min) / fullRange) * width const markerY = (y: number) => plotHeight - (y / density.maxY) * (plotHeight - 4) const path = density.points .map((p, i) => { const x = markerX(p.x) const y = markerY(p.y) return `${i === 0 ? "M" : "L"}${x.toFixed(3)},${y.toFixed(3)}` }) .join(" ") const fillPath = `${path} L${width.toFixed(3)},${plotHeight.toFixed(3)} L0,${plotHeight.toFixed(3)} Z` const captionItems: Array<{ label: string; value: string; key: string }> = [ { key: "n", label: "n", value: stats.n.toString() }, { key: "min", label: "min", value: formatValue(stats.min, active.unit) }, { key: "q1", label: "q1", value: formatValue(stats.q1, active.unit) }, { key: "median", label: "median", value: formatValue(stats.median, active.unit) }, { key: "mean", label: "mean", value: formatValue(stats.mean, active.unit) }, { key: "q3", label: "q3", value: formatValue(stats.q3, active.unit) }, { key: "max", label: "max", value: formatValue(stats.max, active.unit) }, ] const directionHint = active.lowerIsBetter ? "lower is better ←" : "higher is better →" const showPicker = seriesList.length > 1 return (
{!compact && (
{/* Row 1 — View toggle (left) + direction hint (right). The kicker label makes it clear that these chips switch the chart type, distinguishing them from the metric chips below. */}
{renderViewToggle ? "View" : "Score distribution"} {renderViewToggle && (
{availableViews.map((view) => { const on = effectiveView === view const label = view === "distribution" ? "Distribution" : view === "frontier" ? "Frontier" : "Compute" return ( ) })}
)}
{directionHint}
{/* Row 2 — Metric chips. Only shown when there's more than one series; otherwise the active label gets a quiet inline caption next to the view kicker. */} {showPicker ? (
Metric
{seriesList.map((s) => { const on = s.key === active.key return ( ) })}
) : (
{active.label} {active.unit ? {" · " + active.unit} : null}
)}
)} {compact && (
{active.label}
)} {effectiveView === "compute" && protocol ? ( ) : effectiveView === "frontier" && frontier ? ( ) : ( {/* Filled density area */} {/* Density curve */} {/* Baseline */} {/* IQR bracket along baseline */} {/* Median vertical rule (accent) */} {/* Mean tick (dashed) */} )} {/* Caption row — distribution view only; the frontier and compute panels render their own captions. */} {effectiveView === "distribution" && (
{captionItems.map((item, i) => ( {i > 0 && ·} {item.label} {item.value} ))}
)} {!compact && effectiveView === "distribution" && (
median mean IQR (q1–q3)
)}
) } interface FrontierPlotProps { /** Strictly-improving subset of the input — each entry pushes the * cumulative best score further. Already sorted ascending by time. */ events: Array<{ time: number; score: number; name: string }> /** Every dated sample (improving or not), used as background dots. */ samples: Array<{ time: number; score: number; name: string }> unit?: string lowerIsBetter: boolean label: string } function FrontierPlot({ events, samples, unit, lowerIsBetter, label }: FrontierPlotProps) { const PLOT_HEIGHT = 180 const PAD_T = 8 const PAD_B = 22 // room for year labels under the axis const PAD_L_PCT = 1 const PAD_R_PCT = 1 const tMin = Math.min(...samples.map((s) => s.time)) const tMaxData = Math.max(...samples.map((s) => s.time)) // Always extend the rightmost edge to "now" so the user sees how // long the current frontier holder has been on top. const tMax = Math.max(tMaxData, Date.now()) const tRange = tMax - tMin || 1 const sValues = samples.map((s) => s.score) const sMin = Math.min(...sValues) const sMax = Math.max(...sValues) const sRange = sMax - sMin || Math.abs(sMax) || 1 // Pad y so dots don't kiss the borders. const yLo = sMin - sRange * 0.05 const yHi = sMax + sRange * 0.05 const yRange = yHi - yLo || 1 // Percent helpers — used for both HTML overlay positioning and the // SVG path (which uses a 0-100 viewBox so the line scales with the // container without distorting other glyphs). const xPct = (t: number) => PAD_L_PCT + ((t - tMin) / tRange) * (100 - PAD_L_PCT - PAD_R_PCT) const yPct = (s: number) => 100 - ((s - yLo) / yRange) * 100 // 0 at top, 100 at bottom // Pixel helpers for the step-line SVG. Keep its viewBox at 100x100 // so it overlays the container 1:1, while strokeWidth uses // vectorEffect=non-scaling-stroke so the line stays crisp. let d = "" for (let i = 0; i < events.length; i++) { const e = events[i] const x = xPct(e.time) const y = yPct(e.score) if (i === 0) { d += `M${x.toFixed(3)},${y.toFixed(3)} ` } else { const prev = events[i - 1] const yPrev = yPct(prev.score) d += `L${x.toFixed(3)},${yPrev.toFixed(3)} L${x.toFixed(3)},${y.toFixed(3)} ` } } if (events.length > 0) { const last = events[events.length - 1] d += `L${xPct(tMax).toFixed(3)},${yPct(last.score).toFixed(3)}` } // Year tick marks along the x-axis. Keep at most 6 to avoid label // collisions on narrow viewports. const startYear = new Date(tMin).getUTCFullYear() const endYear = new Date(tMax).getUTCFullYear() const yearSpan = endYear - startYear const tickStep = yearSpan <= 6 ? 1 : Math.ceil(yearSpan / 6) const yearTicks: number[] = [] for (let y = startYear; y <= endYear; y += tickStep) yearTicks.push(y) // Pre-bucket samples that *aren't* on the frontier so we don't // double-render them (the frontier dots are emphasised separately). const eventTimes = new Set(events.map((e) => `${e.time}|${e.score}`)) const bgSamples = samples.filter((s) => !eventTimes.has(`${s.time}|${s.score}`)) // Local hover state so we can render a richer label than the native // `title=` tooltip — keeps the dot and the popup nameplate in sync // even when the cursor sits right between two dots. const [hover, setHover] = useState<{ x: number y: number name: string when: string score: string onFrontier: boolean } | null>(null) return (
setHover(null)} > {/* Inner plot canvas (the area minus axis padding). */}
{/* Step line. The SVG uses a 0-100 viewBox so its path lines up with HTML overlays positioned via the same xPct/yPct helpers; non-scaling-stroke keeps the stroke crisp. */} {/* Background sample dots — every dated model that's NOT on the frontier. Rendered as HTML so they're crisp circles and individually clickable / focusable. */} {bgSamples.map((s, i) => (
frontier {events.length} step{events.length === 1 ? "" : "s"} · best {formatValue(events[events.length - 1]?.score, unit)} by {events[events.length - 1]?.name || "—"} · since {formatMonthYear(events[0].time)} · {lowerIsBetter ? "frontier descends: lower is better" : "frontier ascends: higher is better"}
) } // --------------------------------------------------------------------------- // Compute view: per-run scores over // the study's nominal budget axis. Honest-estimator rules: no trend // lines (2–3 nominal levels per axis, conditions at unmatched budgets — a line // would assert a compute trend the study says must be compared at // matched budgets); highlight scopes to one (model, condition), never across // conditions; the condition legend is always visible. // --------------------------------------------------------------------------- const CONDITION_LEGEND: Record = { none: "no feedback", unknown: "condition unknown", answer_feedback: "oracle score feedback (assisted)", } function formatTokenTick(v: number): string { const trim = (n: number) => { const s = n.toFixed(n >= 10 ? 0 : 1) return s.replace(/\.0$/, "") } if (v >= 1_000_000) return `${trim(v / 1_000_000)}M` if (v >= 1_000) return `${trim(v / 1_000)}k` return String(v) } function ConditionGlyph({ condition, highlighted, }: { condition: FeedbackCondition highlighted?: boolean }) { const ink = highlighted ? "var(--accent)" : "var(--fg-muted)" const base: CSSProperties = { display: "inline-block", boxSizing: "border-box" } if (condition === "none") { return ( ) } if (condition === "unknown") { return ( ) } return ( ) } export function ComputePlot({ protocol, unit, label, }: { protocol: ProtocolSeries unit?: string label: string }) { const { marks, axisLabel, omitted, mismatchedConditionBudgets, researcherMode } = protocol const PLOT_HEIGHT = 190 const PAD_T = 8 const PAD_B = 24 const logs = marks.map((m) => Math.log10(m.x)) let xLo = Math.min(...logs) let xHi = Math.max(...logs) if (xHi - xLo < 1e-9) { xLo -= 0.5 xHi += 0.5 } const xPad = (xHi - xLo) * 0.06 xLo -= xPad xHi += xPad const scores = marks.map((m) => m.score) const sMin = Math.min(...scores) const sMax = Math.max(...scores) const sRange = sMax - sMin || Math.abs(sMax) || 1 const yLo = sMin - sRange * 0.05 const yHi = sMax + sRange * 0.05 const yRange = yHi - yLo || 1 const xPct = (v: number) => ((Math.log10(v) - xLo) / (xHi - xLo)) * 98 + 1 const yPct = (s: number) => 100 - ((s - yLo) / yRange) * 100 // Ticks at the distinct NOMINAL levels actually present (2–6 values). const ticks = Array.from(new Set(marks.map((m) => m.x))).sort((a, b) => a - b) const conditionsPresent = (["none", "unknown", "answer_feedback"] as FeedbackCondition[]).filter( (condition) => marks.some((m) => m.condition === condition), ) const [highlightKey, setHighlightKey] = useState(null) const [hover, setHover] = useState<{ x: number; y: number; mark: ComputeMark } | null>(null) const markKey = (mark: ComputeMark) => `${mark.modelName}|${mark.condition}` const hoverDetail = (mark: ComputeMark): string => { if (researcherMode) { return Object.entries(mark.protocolFields) .filter(([, v]) => v != null) .map(([k, v]) => `${k}=${String(v)}`) .join(" · ") } return feedbackConditionDescription(mark.condition) } return (
{ setHover(null) setHighlightKey(null) }} >
{/* Tick rules at the nominal levels */} {ticks.map((t) => (
))} {marks.map((mark, i) => { const highlighted = highlightKey === markKey(mark) const dimmed = highlightKey != null && !highlighted const enter = ( event: ReactMouseEvent | ReactFocusEvent, ) => { const rect = event.currentTarget.parentElement!.getBoundingClientRect() const dot = event.currentTarget.getBoundingClientRect() setHover({ x: dot.left + dot.width / 2 - rect.left, y: dot.top + dot.height / 2 - rect.top, mark, }) // Highlight every mark of this (model, condition) pair — // never across conditions, which would imply a // cross-condition per-model trend. setHighlightKey(markKey(mark)) } return ( ) })} {hover && (
{hover.mark.modelName}
{formatValue(hover.mark.score, unit)} · {formatTokenTick(hover.mark.x)}{" "} {axisLabel}
{hoverDetail(hover.mark)}
)} {/* Baseline */}
{/* Nominal-level labels under the baseline (log-positioned) */} {ticks.map((t) => (
{formatTokenTick(t)}
))}
{/* Condition legend — ALWAYS visible: the study's own figures use solid/hollow marks with a different meaning, so arriving readers need the encoding spelled out. */}
{conditionsPresent.map((condition) => ( {CONDITION_LEGEND[condition]} ))}
{marks.length} runs · x: {axisLabel} (log scale) · y: {label}
{omitted > 0 && (
{omitted} {omitted === 1 ? "run has" : "runs have"} no recorded {axisLabel} and{" "} {omitted === 1 ? "is" : "are"} left out
)} {mismatchedConditionBudgets && (
feedback conditions ran at different budgets; compare them only where budgets match
)}
) }