Assisted runs: shown, labeled, and unranked by default

Protocol-varied collections (first: the UK AISI inference-scaling study)
deliver several results per model, including oracle answer-feedback runs.
The backend never serves those a rank, but the single-metric table (and
the merged page, which renders through it) re-rank client-side by score —
which would have crowned an assisted 1.00 as #1.

- Plumb collection_id / protocol_condition through the eval-summary and
merged queries, probe-gated so pre-collections snapshots keep NULL
aliases (no binder errors, feature self-disabling).
- Assisted rows keep their place in the table but take no rank (em dash +
"assisted" chip); an explicit banner control re-includes them in
ranking (default off).
- Study-protocol caveat banner on pages carrying protocol rows: expanded
budgets are not comparable to standard published results.
- Distribution/Frontier plotbox stats exclude assisted rows — a "best"
that needed the answer oracle is not a frontier.

Pages without protocol data render identically (single activation gate;
NULL columns on old snapshots).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

components/eval-detail.tsx

@@ -53,6 +53,7 @@ import {
 } from "lucide-react"
 import type { BenchmarkCard, SourceData } from "@/lib/benchmark-schema"
 import { tagLabel } from "@/lib/benchmark-schema"
+import { isAssistedResult } from "@/lib/eval-processing"
 import type { BenchmarkEvalSummary, ModelResultForBenchmark } from "@/lib/eval-processing"
 import { isRecognizedEvaluator } from "@/lib/evaluators"
 import { useEvaluatorSlug } from "@/components/org-metadata-provider"
@@ -851,6 +852,21 @@ export function EvalDetail({
 
   const [showUnknownSize, setShowUnknownSize] = useState(true)
 
+  // Protocol-varied collections (e.g. the AISI inference-scaling study):
+  // rows may carry a protocol_condition. Assisted (answer-feedback) runs
+  // are shown but UNRANKED by default — mirroring the backend, which never
+  // serves them a rank — with an explicit opt-in that re-includes them in
+  // the client-side ranking.
+  const hasProtocolRows = useMemo(
+    () => lb.model_results.some((r) => r.protocol_condition != null),
+    [lb.model_results]
+  )
+  const hasAssistedRows = useMemo(
+    () => lb.model_results.some((r) => isAssistedResult(r.protocol_condition)),
+    [lb.model_results]
+  )
+  const [includeAssistedInRanking, setIncludeAssistedInRanking] = useState(false)
+
   const hasParameterData = useMemo(
     () => sortedResults.some((result) => getParamsBillions(result) != null),
     [sortedResults]
@@ -871,21 +887,39 @@ export function EvalDetail({
   const leaderboardRows = useMemo<LeaderboardRow[]>(() => {
     let currentRank = 0
     let previousScore: number | null = null
+    let rankedCount = 0
 
     return filteredResults.map((modelResult, index) => {
-      if (previousScore === null || Math.abs(modelResult.score - previousScore) > 1e-9) {
-        currentRank = index + 1
-        previousScore = modelResult.score
+      // Assisted runs are visible but take no rank (rank 0 sentinel)
+      // unless the reader explicitly re-includes them.
+      const unranked =
+        !includeAssistedInRanking && isAssistedResult(modelResult.protocol_condition)
+      let rank = 0
+      if (!unranked) {
+        rankedCount += 1
+        if (previousScore === null || Math.abs(modelResult.score - previousScore) > 1e-9) {
+          currentRank = rankedCount
+          previousScore = modelResult.score
+        }
+        rank = currentRank
       }
 
       return {
         key: `${modelResult.model_info.id}-${index}`,
-        rank: currentRank,
+        rank,
         modelResult,
         normalizedScore: normalizeScore(modelResult.score),
       }
     })
-  }, [filteredResults])
+  }, [filteredResults, includeAssistedInRanking])
+
+  // Plotbox input: assisted runs stay out of the distribution stats and
+  // the frontier cumulative-best regardless of the ranking toggle — a
+  // "best" that needed the answer oracle is not a frontier.
+  const unassistedLeaderboardRows = useMemo(
+    () => leaderboardRows.filter((r) => !isAssistedResult(r.modelResult.protocol_condition)),
+    [leaderboardRows]
+  )
 
   // Optional user-driven sort. `default` keeps the score-ordered rows
   // the ranker already produced. The rank label is always by score
@@ -1560,7 +1594,43 @@ export function EvalDetail({
 
           {/* Score distribution — paper-themed mean/median/quartile summary,
               with an optional Frontier toggle when models carry release dates. */}
-          {leaderboardRows.length >= 3 && (
+          {hasProtocolRows && (
+            <div
+              className="mb-4 flex items-start gap-3 px-4 py-3"
+              style={{
+                border: "1px solid var(--border-soft)",
+                borderLeft: "2px solid var(--fg-muted)",
+                background: "var(--bg-warm)",
+                color: "var(--fg)",
+              }}
+            >
+              <div className="text-[13px] leading-relaxed">
+                <span style={{ fontWeight: 600 }}>Study-specific protocol.</span>{" "}
+                These results were run under expanded, study-specific inference
+                budgets and are not directly comparable to standard published
+                benchmark results.
+                {hasAssistedRows && (
+                  <>
+                    {" "}Assisted runs — where the model is told when its answer
+                    is correct — are labeled and excluded from ranking
+                    {includeAssistedInRanking ? " (currently included)" : ""}.{" "}
+                    <button
+                      type="button"
+                      onClick={() => setIncludeAssistedInRanking((v) => !v)}
+                      className="underline underline-offset-2 hover:text-[color:var(--accent)]"
+                      style={{ color: "var(--fg-muted)" }}
+                    >
+                      {includeAssistedInRanking
+                        ? "Exclude assisted runs from ranking"
+                        : "Include assisted runs in ranking"}
+                    </button>
+                  </>
+                )}
+              </div>
+            </div>
+          )}
+
+          {unassistedLeaderboardRows.length >= 3 && (
             <div className="mb-4">
               <div className="flex justify-end mb-2">
                 <EmbedButton
@@ -1590,10 +1660,10 @@ export function EvalDetail({
                 series={[{
                   key: "primary",
                   label: lb.metric_config.unit ?? "Score",
-                  values: leaderboardRows.map((r) => r.modelResult.score),
+                  values: unassistedLeaderboardRows.map((r) => r.modelResult.score),
                   unit: lb.metric_config.unit,
                   lowerIsBetter: lb.metric_config.lower_is_better,
-                  points: leaderboardRows.map((r) => ({
+                  points: unassistedLeaderboardRows.map((r) => ({
                     score: r.modelResult.score,
                     releaseDate: r.modelResult.model_info.release_date,
                     modelName: r.modelResult.model_info.name,
@@ -1866,6 +1936,7 @@ export function EvalDetail({
                     : null
                   const isTopRank = rank === 1
                   const rankColor = rank === 1 ? "var(--accent)" : "var(--fg-muted)"
+                  const isAssisted = isAssistedResult(modelResult.protocol_condition)
 
                   return (
                     <Fragment key={key}>
@@ -1882,10 +1953,13 @@ export function EvalDetail({
                             style={{
                               fontSize: 14,
                               fontWeight: isTopRank ? 600 : 500,
-                              color: rankColor,
+                              color: rank === 0 ? "var(--fg-subtle)" : rankColor,
                             }}
+                            title={rank === 0
+                              ? "Assisted run (answer feedback) — shown, not ranked"
+                              : undefined}
                           >
-                            #{rank}
+                            {rank === 0 ? "—" : `#${rank}`}
                           </span>
                         </td>
 
@@ -1911,6 +1985,22 @@ export function EvalDetail({
                               >
                                 {modelResult.model_info.name}
                               </Link>
+                              {isAssisted && (
+                                <span
+                                  className="ml-1.5 inline-block align-middle font-mono uppercase"
+                                  style={{
+                                    fontSize: 9,
+                                    letterSpacing: "0.08em",
+                                    color: "var(--fg-muted)",
+                                    border: "1px solid var(--border-soft)",
+                                    borderRadius: 3,
+                                    padding: "1px 4px",
+                                  }}
+                                  title="The model was told when its answer was correct (oracle answer feedback)."
+                                >
+                                  assisted
+                                </span>
+                              )}
                               {familyLabel && (
                                 <div
                                   className="mt-0.5 font-mono uppercase truncate"

lib/eval-processing.ts

@@ -23,6 +23,23 @@ import type { EvalcardsAnnotations, RowAnnotations, SignalSummaries } from './ba
 export type { BenchmarkCard }
 export type { ModelEvaluationSummary }
 
+
+/**
+ * True when a result ran under the oracle answer-feedback arm of a
+ * protocol-varied collection (the model was told when its answer was
+ * correct). Assisted rows are shown but excluded from client-side ranking
+ * by default — mirroring the backend, which never serves them a rank.
+ */
+export function isAssistedResult(protocolCondition: string | null | undefined): boolean {
+  if (!protocolCondition) return false
+  try {
+    const parsed = JSON.parse(protocolCondition) as { feedback?: unknown } | null
+    return parsed != null && typeof parsed === "object" && parsed.feedback === "answer_feedback"
+  } catch {
+    return false
+  }
+}
+
 export interface ModelResultForBenchmark {
   model_info: ModelInfo
   model_route_id?: string
@@ -273,6 +290,11 @@ export interface MergedObservationRow {
   source_metadata: SourceMetadata
   generation_config?: GenerationConfig
   is_verified_evaluator?: boolean
+  /** Collections: submission-channel id of the observation's source row. */
+  collection_id?: string
+  /** Protocol point (canonical sorted-key JSON) for protocol-varied
+   *  collections; absent/null for ordinary observations. */
+  protocol_condition?: string | null
 }
 
 export interface MergedBenchmarkSummary {

lib/merged-adapter.ts

@@ -99,6 +99,8 @@ export function mergedSummaryToEvalSummary(merged: MergedBenchmarkSummary): Benc
       source_data: sourceData,
       merged_source_slug: row.composite_slug,
       is_verified_evaluator: row.is_verified_evaluator,
+      collection_id: row.collection_id,
+      protocol_condition: row.protocol_condition,
       result: {
         evaluation_name: metricDisplayName,
         display_name: metricDisplayName,

lib/view-data.ts

@@ -641,6 +641,8 @@ function reshapeCellToModelResult(row: Row): ModelResultForBenchmark {
     source_data: sourceDataFromRow(row),
     source_record_url: optionalString(row.source_record_url),
     eee_record_url: optionalString(row.eee_record_url),
+    collection_id: optionalString(row.collection_id),
+    protocol_condition: optionalString(row.protocol_condition) ?? undefined,
     aggregate_components: asArray<NonNullable<ModelResultForBenchmark["aggregate_components"]>[number]>(
       aggregateComponents
     ),
@@ -780,6 +782,32 @@ async function evalsViewHasParentDisplayName(): Promise<boolean> {
   return evalsViewParentDisplayNameCache
 }
 
+// Probe (once per process) whether the loaded snapshot's eval_results_view
+// carries the additive collections columns (collection_id /
+// protocol_condition), so projections degrade to NULL aliases on older
+// snapshots instead of binder-erroring the query. Same lifecycle as the
+// parent_display_name probe above.
+let ervCollectionColumnsCache: boolean | undefined
+async function evalResultsViewHasCollectionColumns(): Promise<boolean> {
+  if (ervCollectionColumnsCache === undefined) {
+    try {
+      const columns = await readRows<{ column_name: string }>("DESCRIBE eval_results_view")
+      const names = new Set(columns.map((column) => column.column_name))
+      ervCollectionColumnsCache = names.has("collection_id") && names.has("protocol_condition")
+    } catch {
+      return false
+    }
+  }
+  return ervCollectionColumnsCache
+}
+
+function evalCellJoinColumns(hasCollections: boolean) {
+  return `${EVAL_CELL_JOIN_COLUMNS},
+  ${hasCollections
+    ? "r.collection_id, r.protocol_condition"
+    : "CAST(NULL AS VARCHAR) AS collection_id, CAST(NULL AS VARCHAR) AS protocol_condition"}`
+}
+
 async function getModelEvaluationRows(modelKey: string): Promise<Row[]> {
   const hasParentDisplayName = await evalsViewHasParentDisplayName()
   // model_key is the producer's addressable identifier — non-null for both
@@ -964,8 +992,9 @@ export async function getEvalSummaryById(evalId: string): Promise<BenchmarkEvalS
   const evalRow = evalRows[0]
   if (!evalRow) return null
 
+  const hasCollectionColumns = await evalResultsViewHasCollectionColumns()
   let cellRows = await readRows<Row>(
-    `SELECT ${EVAL_CELL_JOIN_COLUMNS}
+    `SELECT ${evalCellJoinColumns(hasCollectionColumns)}
      FROM eval_results_view r
      LEFT JOIN evals_view e ON r.evaluation_id = e.evaluation_id
      WHERE r.evaluation_id = ?
@@ -978,7 +1007,7 @@ export async function getEvalSummaryById(evalId: string): Promise<BenchmarkEvalS
 
   if (cellRows.length === 0) {
     cellRows = await readRows<Row>(
-      `SELECT ${EVAL_CELL_JOIN_COLUMNS}
+      `SELECT ${evalCellJoinColumns(hasCollectionColumns)}
        FROM eval_results_view r
        LEFT JOIN evals_view e ON r.evaluation_id = e.evaluation_id
        WHERE r.evaluation_id = ?
@@ -1170,6 +1199,13 @@ const MERGED_ROW_COLUMNS = `
   CAST(to_json(slices) AS VARCHAR) AS slices
 `
 
+function mergedResultColumns(hasCollections: boolean) {
+  return `${MERGED_RESULT_COLUMNS},
+  ${hasCollections
+    ? "r.collection_id, r.protocol_condition"
+    : "CAST(NULL AS VARCHAR) AS collection_id, CAST(NULL AS VARCHAR) AS protocol_condition"}`
+}
+
 const MERGED_RESULT_COLUMNS = `
   r.evaluation_id,
   r.benchmark_id,
@@ -1206,6 +1242,8 @@ function mergedObservationFromRow(row: Row): MergedObservationRow {
     generation_config: (generationConfig ?? undefined) as GenerationConfig | undefined,
     is_verified_evaluator:
       row.is_verified_evaluator == null ? undefined : Boolean(row.is_verified_evaluator),
+    collection_id: optionalString(row.collection_id),
+    protocol_condition: optionalString(row.protocol_condition) ?? undefined,
   }
 }
 
@@ -1277,7 +1315,7 @@ export async function getMergedBenchmarkSummary(
   if (grain === "benchmark" || selectedSliceId) {
     const targetBenchmarkId = grain === "slice" ? selectedSliceId : asString(row.benchmark_id)
     resultRows = await readRows<Row>(
-      `SELECT ${MERGED_RESULT_COLUMNS}
+      `SELECT ${mergedResultColumns(await evalResultsViewHasCollectionColumns())}
        FROM eval_results_view r
        WHERE r.benchmark_id = ?
          AND r.metric_id_effective = ?

tests/assisted-results.test.ts

@@ -0,0 +1,30 @@
+import { describe, expect, it } from "vitest"
+
+import { isAssistedResult } from "@/lib/eval-processing"
+
+// Assisted (oracle answer-feedback) classification drives shown-but-unranked
+// leaderboard handling; misclassification either ranks an assisted run or
+// strips a rank from a clean one.
+describe("isAssistedResult", () => {
+  it("classifies the answer_feedback arm", () => {
+    expect(
+      isAssistedResult(
+        '{"compaction":false,"feedback":"answer_feedback","scaffold":"S-adaptive","token_limit":5000000}',
+      ),
+    ).toBe(true)
+  })
+
+  it("keeps clean and unknown arms unassisted", () => {
+    expect(isAssistedResult('{"feedback":"none"}')).toBe(false)
+    expect(isAssistedResult('{"feedback":"unknown"}')).toBe(false)
+    expect(isAssistedResult('{"token_limit":5000000}')).toBe(false)
+  })
+
+  it("treats absent/invalid protocol data as unassisted", () => {
+    expect(isAssistedResult(undefined)).toBe(false)
+    expect(isAssistedResult(null)).toBe(false)
+    expect(isAssistedResult("")).toBe(false)
+    expect(isAssistedResult("not json")).toBe(false)
+    expect(isAssistedResult('"answer_feedback"')).toBe(false)
+  })
+})