"use client" // Trajectory panels for protocol-varied collection pages // (notes/collection-benchmark-page-spec.md R2). Mounted below the // plotbox on per-source pages whose curated collection ships a // trajectory extract; every panel is independently gated by the shape of // the served payload and the whole section disappears when the route // serves nothing. Scores arrive pre-shaped from the server — nothing is // recomputed or converted here. import { useEffect, useMemo, useRef, useState } from "react" import type { CSSProperties, MouseEvent as ReactMouseEvent, ReactNode } from "react" import { EmbedButton } from "@/components/embed-button" import { fetchEvalTrajectories } from "@/lib/dashboard-data-client" import { availableTrajectoryPanels, TRAJECTORY_PANEL_LABELS, type EvalTrajectoriesPayload, type ReliabilityPanel, type TerminationSummary, type TokensToSuccessPanel, type TrajectoryModelEntry, type TrajectoryPanelKey, } from "@/lib/collection-trajectories" import type { FeedbackCondition } from "@/lib/collections" import { routeIdToPath } from "@/lib/utils" const CONDITION_LABELS: Record = { none: "No feedback", answer_feedback: "Oracle score feedback (assisted)", unknown: "Condition unknown", } // Categorical series colors for the step curves — one hue per model, // assigned in the page's fixed release order (color follows the entity, // never its rank, and hues are never cycled: a 7th model falls back to // neutral ink). Light/dark sets are the dataviz reference palette's // first six slots, validated with its gate script against this app's // actual surfaces (#ffffff / #111110): adjacent-pair CVD ΔE ≥ 8.4 and // normal-vision ΔE ≥ 19.3 in both modes. Three light-mode slots sit // below 3:1 contrast — relieved by the always-visible named legend and // the per-curve tooltips (identity never rides on color alone). const SERIES_LIGHT = ["#2a78d6", "#eb6834", "#1baf7a", "#eda100", "#e87ba4", "#008300"] const SERIES_DARK = ["#3987e5", "#d95926", "#199e70", "#c98500", "#d55181", "#008300"] function TrajectorySeriesStyle() { return ( ) } function seriesColor(index: number): string { return index >= 0 && index < SERIES_LIGHT.length ? `var(--traj-c${index})` : "var(--fg-muted)" } /** Cursor-anchored hover nameplate shared by the trajectory panels * (same visual as the plotbox tooltips). Mount inside a * position:relative container; pass coordinates relative to it. */ function Nameplate({ x, y, wide, children }: { x: number; y: number; wide?: boolean; children: ReactNode }) { return (
{children}
) } /** Keep a centered nameplate from clipping at the container's edges. */ function clampTipX(x: number, rect: DOMRect, halfWidth = 190): number { const margin = Math.min(halfWidth, rect.width / 2) return Math.min(Math.max(x, margin), rect.width - margin) } const nameplateLine: CSSProperties = { fontFamily: "var(--font-mono)", fontSize: 10, letterSpacing: "0.04em", opacity: 0.85, marginTop: 1, } function formatTokens(v: number): string { const trim = (n: number) => n.toFixed(n >= 10 ? 0 : 1).replace(/\.0$/, "") if (v >= 1_000_000) return `${trim(v / 1_000_000)}M` if (v >= 1_000) return `${trim(v / 1_000)}k` return String(v) } function formatPct(v: number): string { return `${(v * 100).toFixed(v * 100 >= 10 ? 0 : 1)}%` } function modelLabel(models: TrajectoryModelEntry[], key: string): string { const entry = models.find((m) => m.key === key) if (!entry) return key return entry.unmatched ? `${entry.label} (unmatched id)` : entry.label } function PanelCard({ kicker, title, children, }: { kicker: string title: string children: ReactNode }) { return (
{kicker} {title}
{children}
) } function ConditionChips({ conditions, active, onChange, }: { conditions: FeedbackCondition[] active: FeedbackCondition onChange: (condition: FeedbackCondition) => void }) { if (conditions.length <= 1) { return (
{CONDITION_LABELS[conditions[0] ?? "unknown"]}
) } return (
{conditions.map((condition) => ( ))}
) } // --------------------------------------------------------------------------- // R2a — lowest observed tokens to success (step curves, oracle-feedback condition). // --------------------------------------------------------------------------- function TokensToSuccessCard({ panel, models, taskCount, }: { panel: TokensToSuccessPanel models: TrajectoryModelEntry[] taskCount: number }) { const [highlight, setHighlight] = useState(null) const [hover, setHover] = useState<{ x: number; y: number; modelKey: string; tokens: number } | null>(null) const plotRef = useRef(null) // Draw and label in the page's release order so each model keeps one // color everywhere it appears. const modelIndex = (key: string) => models.findIndex((m) => m.key === key) const curves = [...panel.curves].sort((a, b) => modelIndex(a.modelKey) - modelIndex(b.modelKey)) const allTokens = curves.flatMap((c) => [ ...c.steps.map((s) => s.tokens), ...(c.censorTokens != null ? [c.censorTokens] : []), ]) if (allTokens.length === 0) return null let xLo = Math.log10(Math.min(...allTokens)) let xHi = Math.log10(Math.max(...allTokens)) if (xHi - xLo < 1e-9) { xLo -= 0.5 xHi += 0.5 } const pad = (xHi - xLo) * 0.04 xLo -= pad xHi += pad const xPct = (tokens: number) => ((Math.log10(tokens) - xLo) / (xHi - xLo)) * 98 + 1 const yPct = (rate: number) => 100 - rate * 100 const curvePath = (curve: (typeof curves)[number]): string => { let d = "" let prevRate = 0 for (const step of curve.steps) { const x = xPct(step.tokens) if (d === "") d = `M${x.toFixed(3)},${yPct(prevRate).toFixed(3)} ` else d += `L${x.toFixed(3)},${yPct(prevRate).toFixed(3)} ` d += `L${x.toFixed(3)},${yPct(step.rate).toFixed(3)} ` prevRate = step.rate } // Extend the plateau to the censoring extent (largest observed // consumed tokens among unsolved attempts) — never a nominal budget. const lastStep = curve.steps[curve.steps.length - 1] const extendTo = Math.max(curve.censorTokens ?? 0, lastStep?.tokens ?? 0) if (d !== "" && extendTo > (lastStep?.tokens ?? 0)) { d += `L${xPct(extendTo).toFixed(3)},${yPct(prevRate).toFixed(3)}` } return d } // Per-curve hover: track the cursor along an invisible fat hit path so // the tooltip can read out the curve AT the cursor's token count. const handleCurveMove = (modelKey: string) => (event: ReactMouseEvent) => { const rect = plotRef.current?.getBoundingClientRect() if (!rect || rect.width === 0) return const xfrac = Math.min(1, Math.max(0, (event.clientX - rect.left) / rect.width)) setHover({ x: clampTipX(event.clientX - rect.left, rect), y: event.clientY - rect.top, modelKey, tokens: 10 ** (xLo + xfrac * (xHi - xLo)), }) setHighlight(modelKey) } // Log-decade ticks inside the visible range. const ticks: number[] = [] for (let e = Math.ceil(xLo); e <= Math.floor(xHi); e++) ticks.push(10 ** e) const censoredTotal = curves.reduce((acc, c) => acc + c.censoredTasks, 0) const hoverCurve = hover ? curves.find((c) => c.modelKey === hover.modelKey) : null const hoverSolvedAt = hover && hoverCurve ? hoverCurve.steps.filter((s) => s.tokens <= hover.tokens).length : 0 return (
{ setHover(null) setHighlight(null) }} > {[0.25, 0.5, 0.75].map((rate) => ( ))} {curves.map((curve) => { const on = highlight === curve.modelKey const dim = highlight != null && !on return ( ) })} {/* Invisible fat hit paths — a hover target wider than the 1.6px mark, per curve, above the visible strokes. */} {curves.map((curve) => ( ))} {ticks.map((t) => (
{formatTokens(t)}
))} {hover && hoverCurve && (
{modelLabel(models, hover.modelKey)}
{formatPct(hoverSolvedAt / hoverCurve.attemptedTasks)} of{" "} {hoverCurve.attemptedTasks} attempted solved within {formatTokens(hover.tokens)}{" "} tokens
{hoverCurve.solvedTasks} solved · {hoverCurve.censoredTasks} unsolved
)}
{curves.map((curve) => ( ))}
x: lowest observed tokens to success (log) · y: cumulative success rate over attempted tasks
{censoredTotal > 0 && (
unsolved tasks keep a curve flat; each curve ends at the highest token count reached
)}
runs used oracle score feedback and expanded budgets; compare with published fixed-budget scores with caution
{curves.length + panel.droppedModels.length < models.length && (
the study ran oracle score feedback for{" "} {curves.length + panel.droppedModels.length} of this page's {models.length}{" "} models on this benchmark
)} {panel.droppedModels.length > 0 && (
curves need at least 5 attempted tasks;{" "} {panel.droppedModels .map((d) => `${modelLabel(models, d.modelKey)} attempted ${d.attemptedTasks}`) .join(", ")}
)}
) } // --------------------------------------------------------------------------- // R2b — reliability heatmap (models × task-difficulty fifths). // --------------------------------------------------------------------------- function ReliabilityCard({ panels, models, isResearchView, initialCondition, }: { panels: ReliabilityPanel[] models: TrajectoryModelEntry[] isResearchView: boolean initialCondition?: FeedbackCondition }) { const conditions = panels.map((p) => p.condition) const [condition, setCondition] = useState( initialCondition && conditions.includes(initialCondition) ? initialCondition : conditions.includes("none") ? "none" : conditions[0], ) const panel = panels.find((p) => p.condition === condition) ?? panels[0] const gridRef = useRef(null) const [hover, setHover] = useState<{ x: number; y: number; modelKey: string; binKey: string } | null>(null) if (!panel) return null // Columns: the page's models in release order, restricted to models // present in this condition. const presentKeys = new Set(panel.cells.map((c) => c.modelKey)) const columns = models.filter((m) => presentKeys.has(m.key)) const cellFor = (modelKey: string, binKey: string) => panel.cells.find((c) => c.modelKey === modelKey && c.binKey === binKey) const handleCellMove = (modelKey: string, binKey: string) => (event: ReactMouseEvent) => { const rect = gridRef.current?.getBoundingClientRect() if (!rect) return setHover({ x: clampTipX(event.clientX - rect.left, rect), y: event.clientY - rect.top, modelKey, binKey }) } const hoverBin = hover ? panel.bins.find((b) => b.key === hover.binKey) : null const hoverCell = hover ? cellFor(hover.modelKey, hover.binKey) : null return (
setHover(null)}> {hover && hoverBin && (
{modelLabel(models, hover.modelKey)}
{hoverBin.label} · {hoverBin.taskCount} tasks
{hoverCell?.solveRate != null ? `solved ${formatPct(hoverCell.solveRate)} of ${hoverCell.scoredAttempts} runs on ${hoverCell.attemptedTasks} attempted tasks` : "this model has no runs on these tasks in this condition"}
{isResearchView && (
difficulty {hoverBin.difficultyRange[0].toFixed(2)} to{" "} {hoverBin.difficultyRange[1].toFixed(2)} · tasks:{" "} {hoverBin.taskIds.slice(0, 6).join(", ")} {hoverBin.taskIds.length > 6 ? ` +${hoverBin.taskIds.length - 6} more` : ""}
)}
)}
{columns.map((model) => ( ))} {panel.bins.map((bin) => ( {columns.map((model) => { const cell = cellFor(model.key, bin.key) const rate = cell?.solveRate ?? null return ( ) })} ))}
Task difficulty {modelLabel(models, model.key)}
{bin.label} 0.55 ? "var(--bg)" : "var(--fg)", background: rate == null ? "repeating-linear-gradient(45deg, transparent, transparent 4px, var(--border-soft) 4px, var(--border-soft) 5px)" : `color-mix(in srgb, var(--accent) ${Math.round(8 + rate * 72)}%, transparent)`, }} > {rate != null ? formatPct(rate) : "—"}
difficulty = 1 − the task's median solve rate across models, over both feedback conditions (the study's definition). Rows keep the same tasks when you switch condition; hatched cells have no runs.
) } // --------------------------------------------------------------------------- // R2c — how runs ended: the study's termination-table grain — a // PARTITION of runs by termination cause per feedback condition — plus a // separately-labeled outcome line (owner decision (c), 2026-08-21). // --------------------------------------------------------------------------- function TerminationCard({ summaries, models, isResearchView, initialCondition, }: { summaries: TerminationSummary[] models: TrajectoryModelEntry[] isResearchView: boolean initialCondition?: FeedbackCondition }) { const conditions = summaries.map((s) => s.condition) const [condition, setCondition] = useState( initialCondition && conditions.includes(initialCondition) ? initialCondition : conditions.includes("none") ? "none" : conditions[0], ) const [breakdownOpen, setBreakdownOpen] = useState(false) const rowsRef = useRef(null) const [hover, setHover] = useState<{ x: number; y: number; key: string } | null>(null) const summary = summaries.find((s) => s.condition === condition) ?? summaries[0] if (!summary) return null // Partition rows sum to 100%. "Ended on a correct submission" is a // termination cause only under oracle score feedback; elsewhere it // renders "—" (the category cannot occur), mirroring the study's own // table. const rows: Array<{ key: string label: string value: { n: number; denominator: number } | null dashNote?: string }> = [ { key: "correct-submit", label: "ended on a correct submission", value: summary.endedOnCorrectSubmission, dashNote: "only occurs under oracle score feedback", }, { key: "guard", label: "repeated similar answers", value: summary.repetitionGuard }, { key: "budget", label: "token budget exhausted", value: summary.budgetExhausted }, { key: "other", label: "other endings", value: summary.otherEndings }, ] const reasonKeys = Array.from( new Set(summary.byModel.flatMap((m) => Object.keys(m.reasons))), ).sort() const outcome = summary.reachedCorrectAnswer // What sits inside "other endings", pooled across models, for its // hover breakdown. const TABLE_REASONS = ["completed_on_successful_submit", "repetition_guard", "token_limit"] const otherBreakdown: Array<[string, number]> = [] { const pooled = new Map() for (const model of summary.byModel) { for (const [reason, n] of Object.entries(model.reasons)) { pooled.set(reason, (pooled.get(reason) ?? 0) + n) } } for (const [reason, n] of pooled) { if (!TABLE_REASONS.includes(reason) && n > 0) otherBreakdown.push([reason, n]) } otherBreakdown.sort((a, b) => b[1] - a[1]) } const handleRowMove = (key: string) => (event: ReactMouseEvent) => { const rect = rowsRef.current?.getBoundingClientRect() if (!rect) return setHover({ x: clampTipX(event.clientX - rect.left, rect), y: event.clientY - rect.top, key }) } const hoverContent = (key: string): ReactNode => { const row = rows.find((r) => r.key === key) if (key === "outcome" && outcome) { const excluded = summary.runCount - outcome.denominator return ( <>
runs whose outcome was correct
{outcome.n} of {outcome.denominator} runs with a recorded outcome ( {formatPct(outcome.n / outcome.denominator)})
{excluded > 0 && (
{excluded} runs have no recorded outcome and are left out
)} ) } if (!row) return null if (!row.value) { return ( <>
{row.label}
this can only happen under oracle score feedback
) } const detail = key === "correct-submit" ? "the oracle ended these runs after a correct submission" : key === "guard" ? "the run started repeating similar answers and was stopped" : key === "budget" ? "the run used its full token budget" : "the run record stops here, usually during a tool call" return ( <>
{row.label}
{row.value.n} of {row.value.denominator} runs ( {formatPct(row.value.denominator > 0 ? row.value.n / row.value.denominator : 0)})
{detail}
{key === "other" && otherBreakdown.length > 0 && (
{otherBreakdown.map(([reason, n]) => `${reason} ${n}`).join(" · ")}
)} ) } const proportionRow = ( key: string, label: string, value: { n: number; denominator: number } | null, dashNote?: string, ) => { const pct = value && value.denominator > 0 ? value.n / value.denominator : 0 return (
{label}
{value && (
)}
{value ? ( <> {formatPct(pct)} · {value.n} ) : ( "—" )}
) } return (
setHover(null)}> {hover && {hoverContent(hover.key)}}
{rows.map((row) => proportionRow(row.key, row.label, row.value, row.dashNote))}
{/* Outcome line, shown apart from the termination categories: under no feedback a correct run still ends at the guard or the budget, so its ending says nothing about its outcome. */}
Outcome
{outcome ? ( proportionRow( "outcome", "runs whose outcome was correct", outcome, ) ) : (
this benchmark grades each run on a scale, so there is no correct/incorrect count
)} {outcome && outcome.denominator < summary.runCount && (
over {outcome.denominator} runs with a recorded outcome
)}

The outcome is listed separately because many correct runs still end at the repetition guard or the token limit. “Other endings” counts runs whose record stops outside the three study categories, usually during a tool call.

{isResearchView && reasonKeys.length > 0 && (
{breakdownOpen && (
{reasonKeys.map((reason) => ( ))} {summary.byModel.map((model) => ( {reasonKeys.map((reason) => { const n = model.reasons[reason] ?? 0 return ( ) })} ))}
model {reason}
{modelLabel(models, model.modelKey)} 0 ? "var(--fg)" : "var(--fg-subtle)", textAlign: "right", padding: "3px 10px" }} title={`${n} of ${model.total} runs`} > {model.total > 0 ? formatPct(n / model.total) : "—"}
)}
)}
) } // --------------------------------------------------------------------------- // Section wrapper. // --------------------------------------------------------------------------- export function CollectionTrajectories({ evaluationId, isResearchView, payload: servedPayload, panels: panelFilter, initialCondition, showEmbedButton = false, }: { evaluationId: string isResearchView: boolean /** Pre-fetched route payload. When provided the section renders it * directly and never fetches — the trajectories embed owns its own * fetch so it can render absence copy instead of nothing. Leaving * the prop off means the section fetches for itself; a parent that * owns the fetch must therefore hold off mounting the section until * it has a payload, rather than passing an in-flight placeholder. */ payload?: EvalTrajectoriesPayload /** Restrict to these panels; omitted = all the payload carries. */ panels?: TrajectoryPanelKey[] /** Preselect this feedback condition on chip-bearing panels that * carry it; the chips stay interactive. */ initialCondition?: FeedbackCondition /** Renders an embed-this button row above the cards. Kept off inside * the embed route itself, so an iframe never offers to re-embed. */ showEmbedButton?: boolean }) { const ownsFetch = servedPayload === undefined const [fetched, setFetched] = useState(null) useEffect(() => { if (!ownsFetch) return let cancelled = false setFetched(null) fetchEvalTrajectories(evaluationId).then((data) => { if (!cancelled) setFetched(data) }) return () => { cancelled = true } }, [evaluationId, ownsFetch]) const payload = ownsFetch ? fetched : servedPayload const visiblePanels = useMemo(() => { const available = availableTrajectoryPanels(payload) return panelFilter ? available.filter((p) => panelFilter.includes(p)) : available }, [payload, panelFilter]) // Route absent / table absent / zero rows → the page renders exactly // as it does without this feature. if (!payload || visiblePanels.length === 0) return null return (
{showEmbedButton && (
({ id: panel, label: TRAJECTORY_PANEL_LABELS[panel], embedPath: `/embed/eval/trajectories/${routeIdToPath(evaluationId)}?panel=${panel}`, })), ]} />
)} {visiblePanels.includes("tokens") && payload.tokens_to_success && ( )} {visiblePanels.includes("reliability") && ( )} {visiblePanels.includes("termination") && ( )}
) }