Use model_key as the addressable identifier and wire comparison-index sidecar

model_key is non-null for both resolved and unresolved models (the latter
fall back to the raw source name); querying by model_id alone silently
dropped unresolved models. Update model lookups, eval-row joins, and
ModelInfo derivation to prefer model_key with model_id as fallback.

Also point the Dockerfile snapshot URL at the temp dataset
(j-chim/temp_evalcard_backend) and add fetchComparisonIndex sidecar
support so v2 mode no longer hits the legacy JSON path.

Dockerfile

@@ -14,7 +14,7 @@ ARG PNPM_VERSION=10.25.0
 # Override at build time via `--build-arg ...`.
 ARG DATA_BACKEND=v2
 ARG HF_DATASET_REPO=https://huggingface.co/datasets/evaleval/card_backend
-ARG SNAPSHOT_URL=https://huggingface.co/datasets/evaleval/eval-cards-data/resolve/main/warehouse/2026-05-03T21-46-50Z
+ARG SNAPSHOT_URL=https://huggingface.co/datasets/j-chim/temp_evalcard_backend/resolve/main/warehouse/2026-05-03T21-46-50Z
 # Static prerender (`next build`) executes route handlers. In legacy mode the
 # cache populated by `cache-hf-data.mjs` lives at `/app/.cache/hf-data`; in v2
 # the cache step is skipped and the app reads the pinned Stage J snapshot.
@@ -48,7 +48,7 @@ FROM node:18-bullseye-slim AS runner
 WORKDIR /app
 
 ARG DATA_BACKEND=v2
-ARG SNAPSHOT_URL=https://huggingface.co/datasets/evaleval/eval-cards-data/resolve/main/warehouse/2026-05-03T21-46-50Z
+ARG SNAPSHOT_URL=https://huggingface.co/datasets/j-chim/temp_evalcard_backend/resolve/main/warehouse/2026-05-03T21-46-50Z
 
 # Runtime needs the same data-source envs that the builder used. Docker
 # multi-stage doesn't carry ENVs across stages, so keep backend selection and

lib/hf-data.ts

@@ -1005,6 +1005,10 @@ function adaptEvalHierarchy(raw: EvalHierarchy): EvalHierarchy {
 }
 
 export async function fetchComparisonIndex(): Promise<ComparisonIndex> {
+  if (useViewLayerBackend()) {
+    return (await fetchSnapshotSidecars()).fetchComparisonIndex()
+  }
+
   return fetchHFJson<ComparisonIndex>("comparison-index.json")
 }
 

lib/sidecars.ts

@@ -2,6 +2,7 @@ import "server-only"
 
 import type {
   BackendManifest,
+  ComparisonIndex,
   CorpusAggregates,
   EvalHierarchy,
 } from "@/lib/backend-artifacts"
@@ -10,6 +11,7 @@ let cache: {
   manifest?: Promise<BackendManifest>
   headline?: Promise<CorpusAggregates>
   hierarchy?: Promise<EvalHierarchy>
+  comparisonIndex?: Promise<ComparisonIndex>
 } = {}
 
 function getSnapshotUrl() {
@@ -54,6 +56,10 @@ export function fetchHierarchy(): Promise<EvalHierarchy> {
   return (cache.hierarchy ??= fetchJson<EvalHierarchy>("hierarchy.json"))
 }
 
+export function fetchComparisonIndex(): Promise<ComparisonIndex> {
+  return (cache.comparisonIndex ??= fetchJson<ComparisonIndex>("comparison-index.json"))
+}
+
 export function resetSidecarCacheForTests() {
   cache = {}
 }

lib/view-data.ts

@@ -28,7 +28,7 @@ import type {
 type Row = Record<string, any>
 
 const MODEL_CARD_COLUMNS = `
-  id, route_id, model_name, model_id, canonical_model_name, developer,
+  id, model_key, route_id, model_name, model_id, canonical_model_name, developer,
   evaluations_count, benchmarks_count, variant_count,
   categories, category_stats, latest_timestamp,
   evaluator_count, evaluator_names, source_type_count, source_types,
@@ -250,8 +250,8 @@ function metricConfigFromRow(row: Row): MetricConfig {
 
 function modelInfoFromModelRow(row: Row): ModelInfo {
   return {
-    name: asString(row.model_name ?? row.model_family_name ?? row.model_id, "Unknown model"),
-    id: asString(row.model_id ?? row.id ?? row.route_id, "unknown-model"),
+    name: asString(row.model_name ?? row.model_family_name ?? row.model_id ?? row.model_key, "Unknown model"),
+    id: asString(row.model_key ?? row.model_id ?? row.id ?? row.route_id, "unknown-model"),
     developer: optionalString(row.developer),
     inference_platform: optionalString(row.inference_platform),
     inference_engine: optionalString(row.inference_engine),
@@ -390,23 +390,26 @@ function modelSummaryFromRows(modelRow: Row, cellRows: Row[]): ModelEvaluationSu
 
   return {
     ...core,
-    model_family_id: asString(modelRow.model_family_id ?? modelRow.model_id, modelRow.model_id),
+    model_family_id: asString(modelRow.model_family_id ?? modelRow.model_key ?? modelRow.model_id, modelRow.model_key ?? modelRow.model_id),
     model_route_id: asString(modelRow.model_route_id ?? modelRow.route_id, modelRow.route_id),
     model_family_name: asString(modelRow.model_family_name ?? modelRow.model_name, modelRow.model_name),
-    raw_model_ids: rawModelIds.length > 0 ? rawModelIds : [asString(modelRow.model_id, "")].filter(Boolean),
+    raw_model_ids: rawModelIds.length > 0 ? rawModelIds : [asString(modelRow.model_key ?? modelRow.model_id, "")].filter(Boolean),
     variants,
   }
 }
 
-async function getModelEvaluationRows(modelId: string): Promise<Row[]> {
+async function getModelEvaluationRows(modelKey: string): Promise<Row[]> {
+  // model_key is the producer's addressable identifier — non-null for both
+  // resolved and unresolved models (the latter fall back to the raw source
+  // name). Querying by model_id alone would silently miss unresolved models.
   return readRows<Row>(
     `SELECT ${CELL_JOIN_COLUMNS}
      FROM eval_results_view r
      LEFT JOIN evals_view e ON r.evaluation_id = e.evaluation_id
-     WHERE r.model_id = ?
+     WHERE r.model_key = ?
        AND r.score IS NOT NULL
      ORDER BY r.category, r.percentile DESC NULLS LAST`,
-    [modelId]
+    [modelKey]
   )
 }
 
@@ -466,17 +469,21 @@ export async function getDashboardData() {
 }
 
 export async function getModelSummaryById(routeId: string): Promise<ModelEvaluationSummary | null> {
+  // Lookups use the addressable identifier (`model_key`/`route_id`/
+  // `model_route_id`/`model_family_id`) so unresolved models — whose
+  // `model_id` is NULL — are still findable. `model_id` is kept in the
+  // OR chain as a back-compat fallback for old links.
   const rows = await readRows<Row>(
     `SELECT *
      FROM models_view
-     WHERE route_id = ? OR model_route_id = ? OR model_family_id = ? OR model_id = ?
+     WHERE model_key = ? OR route_id = ? OR model_route_id = ? OR model_family_id = ? OR model_id = ?
      LIMIT 1`,
-    [routeId, routeId, routeId, routeId]
+    [routeId, routeId, routeId, routeId, routeId]
   )
   const modelRow = rows[0]
   if (!modelRow) return null
 
-  const cellRows = await getModelEvaluationRows(asString(modelRow.model_id, routeId))
+  const cellRows = await getModelEvaluationRows(asString(modelRow.model_key ?? modelRow.model_id, routeId))
   return modelSummaryFromRows(modelRow, cellRows)
 }
 

notes/backend-v2-migration.md

@@ -0,0 +1,616 @@
+# Frontend migration to backend v2 (Stage J view layer)
+
+> **Status:** spec, drafted 2026-05-03 against `eval_card_backend`'s
+> Stage J view-layer contract.
+>
+> **Sources:**
+> - Backend spec (the contract this consumes):
+>   `../eval_card_backend/notes/08-frontend-view-layer.md`
+> - Canonical schema (audit/debug only; not in hot path):
+>   `../eval_card_backend/notes/01-schema-from-frontend.md`
+
+---
+
+## Context
+
+The legacy producer (`eval_cards_backend_pipeline`) emitted ten
+parquets where each row carried a `payload_json` VARCHAR with the
+post-TS-adapter shape baked in. The frontend's "DuckDB backend"
+(`lib/duckdb-data.ts`) read these blobs and `JSON.parse`d them — column
+projection, filter pushdown, and type contracts were all forfeited.
+
+The new producer (`eval_card_backend`) emits a typed view layer over
+its canonical normalised tables. Three Parquet files cover every page
+shape, three small JSON sidecars cover corpus-level scalars and the
+hierarchy tree. Column names match the frontend's TS interfaces
+field-for-field, so the row→object cast is a typed spread for most
+accessors. Two interfaces (`ModelResultForBenchmark` and the
+`evaluations_by_category` body of `ModelEvaluationSummary`) require a
+small mechanical reshape over the row, since one nests fields that the
+view stores flat — see the per-accessor sections below. No
+HF-record-to-display adapter logic survives.
+
+This document specifies what changes in `general-eval-card` once
+backend v2 is faithfully implemented. **The visual frontend, page
+renderers, and TS interface shapes do not change.** Only the I/O
+boundary moves.
+
+---
+
+## What changes (overview)
+
+| layer | before (v1) | after (v2) |
+|---|---|---|
+| Distribution | `LOCAL_PIPELINE_OUTPUT` env var pointing at a producer output dir; `duckdb/v1/` subpath; implicit "warehouse/latest/" coupling | `SNAPSHOT_URL` env var (file:// or HF dataset URL); one snapshot pinned per deploy |
+| Storage shape | 10 parquets each with one `payload_json` column | 3 typed-column view parquets + 3 JSON sidecars |
+| Read pattern | `SELECT payload_json FROM read_parquet(?) WHERE id = ?`, then `JSON.parse` | `SELECT col1, col2, ... FROM <view> WHERE id = ?`, typed row spread |
+| List vs detail | Separate `*_lite.parquet` files | Column projection on the same parquet |
+| Suite/aggregate dispatch | Eval id prefix (`aggregate__`, `matrix__`) → different parquet | `is_summary_score` flag + `parent_benchmark_id` on `evals_view` |
+| Slug rule | Custom `replace('/', '__')` escapes; per-page slug helpers | Producer-owned RFC 3986 percent-encoded `route_id` / `evaluation_id` / `metric_summary_id`; frontend decodes only on `<Link>` href |
+| Corpus aggregates | `corpus-aggregates.json` over HF JSON loader | `headline.json` sidecar in the snapshot dir |
+| Hierarchy | Synthesised in the producer's `eval_hierarchy` JSON | `hierarchy.json` sidecar |
+| Backend manifest | `manifest.json` fetched from upstream HF dataset root via `lib/hf-data.ts` | `manifest.json` sidecar inside the snapshot dir, read via `SNAPSHOT_URL` |
+
+The TS interfaces (`EvaluationCardData`, `BenchmarkEvalSummary`,
+`ModelEvaluationSummary`, `ModelResultForBenchmark`, `CorpusAggregates`,
+`EvalHierarchy`, `BackendManifest`) stay as-is — the producer agreed to
+emit columns under those exact names.
+
+---
+
+## What does not change
+
+- All page components under `app/`. The renderer trees are unchanged.
+- TS interface declarations in `lib/benchmark-schema.ts`,
+  `lib/eval-processing.ts`, `lib/backend-artifacts.ts`. These are now
+  the contract surface — column names match field names by agreement
+  with the producer.
+- Component files under `components/`.
+- `lib/glossary.ts`, `lib/known-issues.ts`, `lib/utils.ts`,
+  `lib/na-utils.ts` — these are pure presentation helpers.
+- `app/api/*/route.ts` handlers stay as thin pass-throughs to
+  `lib/data-backend.ts`.
+
+---
+
+## Distribution: `SNAPSHOT_URL`
+
+Frontend reads `SNAPSHOT_URL` from env at process start. One deploy =
+one snapshot. The URL points at a directory containing the six
+artifacts the frontend reads:
+
+```
+$SNAPSHOT_URL/
+├── models_view.parquet
+├── evals_view.parquet
+├── eval_results_view.parquet
+├── headline.json
+├── hierarchy.json
+└── manifest.json
+```
+
+Examples:
+
+- Local dev: `SNAPSHOT_URL=file:///path/to/eval_card_backend/warehouse/2026-05-03T15-48-59Z`
+- Production (pinned snapshot): `SNAPSHOT_URL=https://huggingface.co/datasets/evaleval/eval-cards-data/resolve/<rev>/warehouse/<snapshot_id>`
+- Production (rolling): `SNAPSHOT_URL=https://huggingface.co/datasets/evaleval/eval-cards-data/resolve/main/warehouse/latest`
+
+`LOCAL_PIPELINE_OUTPUT` is removed. The `duckdb/v1/` subpath is
+removed. The producer maintains a `warehouse/latest/` alias that
+points at the most recent snapshot, so deploys can pin either to a
+timestamped snapshot (immutable, redeploy required to roll forward)
+or to `latest` (auto-rolls forward on the next Space rebuild). Within
+a running process the snapshot is still effectively constant — sidecar
+caches in `lib/sidecars.ts` are first-write-wins per process.
+
+---
+
+## DuckDB connection lifecycle
+
+`lib/duckdb.ts` (new file; replaces the connection-management portion
+of `lib/duckdb-data.ts`):
+
+```ts
+import "server-only"
+import { DuckDBConnection } from "@duckdb/node-api"
+
+let connectionPromise: Promise<DuckDBConnection> | null = null
+
+const SNAPSHOT_URL = process.env.SNAPSHOT_URL
+if (!SNAPSHOT_URL) {
+  throw new Error("SNAPSHOT_URL must be set; see notes/backend-v2-migration.md")
+}
+
+const VIEWS = {
+  models_view:       `${SNAPSHOT_URL}/models_view.parquet`,
+  evals_view:        `${SNAPSHOT_URL}/evals_view.parquet`,
+  eval_results_view: `${SNAPSHOT_URL}/eval_results_view.parquet`,
+} as const
+
+export async function getConnection(): Promise<DuckDBConnection> {
+  if (!connectionPromise) {
+    connectionPromise = (async () => {
+      const conn = await DuckDBConnection.create()
+      // httpfs is built into duckdb-node-api; no INSTALL needed.
+      // Register each parquet as a view so callers write `FROM models_view`,
+      // not the full URL.
+      for (const [name, path] of Object.entries(VIEWS)) {
+        await conn.run(
+          `CREATE OR REPLACE VIEW ${name} AS SELECT * FROM read_parquet(?)`,
+          [path]
+        )
+      }
+      return conn
+    })()
+  }
+  return connectionPromise
+}
+```
+
+One connection per Node process. Views are registered once at
+startup; subsequent queries write `FROM models_view` rather than
+re-passing the parquet URL. DuckDB's column projection means the cost
+of `SELECT route_id, model_name FROM models_view` is independent of
+how wide `models_view` is.
+
+---
+
+## Per-accessor mapping
+
+`lib/data-backend.ts` keeps its current export names. `lib/duckdb-data.ts`
+gets gutted; each function becomes a thin typed `SELECT`. The mapping
+below uses the column names spec'd in
+`../eval_card_backend/notes/08-frontend-view-layer.md` — the row
+returned by DuckDB casts directly to the TS interface.
+
+### Models
+
+```ts
+// getModelCards / getModelCardsLite — list pages
+export async function getModelCards(): Promise<EvaluationCardData[]> {
+  const conn = await getConnection()
+  const reader = await conn.runAndReadAll(`
+    SELECT id, route_id, model_name, model_id, canonical_model_name, developer,
+           evaluations_count, benchmarks_count, variant_count,
+           categories, category_stats, latest_timestamp,
+           evaluator_count, evaluator_names, source_type_count, source_types,
+           evidence_count, missing_generation_config_count,
+           third_party_eval_count, independent_verification_ratio,
+           reproducibility_status, eval_libraries, latest_source_name,
+           params_billions, benchmark_names, score_summary,
+           reproducibility_summary, provenance_summary, comparability_summary,
+           top_scores, source_urls, detail_urls,
+           model_url, release_date, input_modalities, output_modalities,
+           architecture, params, inference_engine, inference_platform
+    FROM models_view
+    ORDER BY latest_timestamp DESC
+  `)
+  return reader.getRowObjects() as EvaluationCardData[]
+}
+
+// "Lite" is just narrower projection — same parquet, fewer columns.
+export async function getModelCardsLite(): Promise<EvaluationCardData[]> {
+  const conn = await getConnection()
+  const reader = await conn.runAndReadAll(`
+    SELECT id, route_id, model_name, model_id, developer,
+           evaluations_count, benchmarks_count, categories,
+           latest_timestamp, third_party_eval_count,
+           independent_verification_ratio, reproducibility_status,
+           latest_source_name, params_billions
+    FROM models_view
+    ORDER BY benchmarks_count DESC, evaluations_count DESC, model_name ASC
+  `)
+  return reader.getRowObjects() as EvaluationCardData[]
+}
+
+// getModelSummaryById — detail page.
+//
+// The row carries the metadata shell (variants[], categories,
+// category_stats, signal summaries, model_family_id, raw_model_ids,
+// total_evaluations, last_updated). The full `ModelEvaluationSummary`
+// also requires `evaluations_by_category: Record<CategoryType,
+// BenchmarkEvaluation[]>`, which is a heavyweight per-cell breakdown —
+// produced by a separate join over `eval_results_view`, see
+// `getModelEvaluationCells` below.
+//
+// Returning a `ModelSummaryShell` (Omit-ed type, defined alongside the
+// existing TS interface) makes the contract explicit and stops the cast
+// from lying. The model-detail page composes the full
+// `ModelEvaluationSummary` from `shell` + `cells`.
+export type ModelSummaryShell = Omit<
+  ModelEvaluationSummary,
+  "evaluations_by_category"
+>
+
+export async function getModelSummaryById(routeId: string): Promise<ModelSummaryShell | null> {
+  const conn = await getConnection()
+  const reader = await conn.runAndReadAll(
+    `SELECT * FROM models_view WHERE route_id = ? OR model_family_id = ? LIMIT 1`,
+    [routeId, routeId]
+  )
+  const rows = reader.getRowObjects()
+  if (rows.length === 0) return null
+  return rows[0] as unknown as ModelSummaryShell
+}
+
+// Per-cell reshape helper. `eval_results_view` rows carry the per-cell
+// fields scattered (model_info, score_details, evaluation_timestamp,
+// source_metadata, source_data, metric_*, etc.) rather than under a
+// nested `result: EvaluationResult` STRUCT. Reshape into the
+// `ModelResultForBenchmark` shape the leaderboard / model-detail
+// renderers expect. Single helper; reused by getEvalSummaryById and
+// getModelEvaluationCells. No HF-record-to-display logic survives.
+function reshapeCellToModelResult(row: Record<string, any>): ModelResultForBenchmark {
+  return {
+    model_info:           row.model_info,
+    model_route_id:       row.model_route_id,
+    score:                row.score,
+    score_details:        row.score_details,
+    evaluation_timestamp: row.evaluation_timestamp,
+    source_metadata:      row.source_metadata,
+    source_data:          row.source_data,
+    source_record_url:    row.source_record_url,
+    aggregate_components: row.aggregate_components,
+    result: {
+      evaluation_name:      row.metric_display_name,
+      metric_summary_id:    row.metric_summary_id,
+      metric_key:           row.metric_id,
+      evaluation_timestamp: row.evaluation_timestamp,
+      metric_config:        { lower_is_better: row.lower_is_better, unit: row.metric_unit, /* …denormalised meta… */ },
+      score_details:        row.score_details,
+      evalcards:            row.evalcards_annotations ? { annotations: row.evalcards_annotations } : undefined,
+    },
+  }
+}
+
+// Helper for the model-detail page's evaluations_by_category body.
+// The page groups by `category` in TS after this returns.
+export async function getModelEvaluationCells(modelId: string): Promise<ModelResultForBenchmark[]> {
+  const conn = await getConnection()
+  const reader = await conn.runAndReadAll(
+    `SELECT * FROM eval_results_view WHERE model_id = ? ORDER BY category, percentile DESC`,
+    [modelId]
+  )
+  return reader.getRowObjects().map(reshapeCellToModelResult)
+}
+```
+
+### Evals
+
+```ts
+// getEvalListData / getEvalListLiteData — list pages
+export async function getEvalListData(): Promise<{
+  evals: BenchmarkEvalListItem[]
+  totalModels: number
+}> {
+  const conn = await getConnection()
+  const [evalsReader, modelsReader] = await Promise.all([
+    conn.runAndReadAll(`
+      SELECT evaluation_id, evaluation_name, canonical_display_name,
+             composite_benchmark_key, composite_benchmark_name,
+             benchmark_family_key, benchmark_leaf_key, category,
+             metric_config, models_count, evaluator_names, source_types,
+             latest_source_name, third_party_ratio,
+             missing_generation_config_count, best_model, worst_model,
+             avg_score, avg_score_norm, has_card,
+             is_aggregated, aggregate_sources, tags,
+             metrics_count, metric_names, instance_data, top_score,
+             subtasks_count, is_summary_score, summary_eval_ids,
+             root_metrics, subtasks, leaderboard_metrics,
+             reproducibility_summary, provenance_summary, comparability_summary,
+             source_data
+      FROM evals_view
+      ORDER BY evaluation_name ASC
+    `),
+    conn.runAndReadAll(`SELECT COUNT(*) AS n FROM models_view`),
+  ])
+  return {
+    evals: evalsReader.getRowObjects() as BenchmarkEvalListItem[],
+    totalModels: Number(modelsReader.getRowObjects()[0].n),
+  }
+}
+
+// getEvalSummaryById — detail page.
+//
+// No more aggregate__/matrix__ id-prefix dispatch — `evals_view` is the
+// single source for all eval shapes. Suite-vs-leaf is a column
+// (`is_summary_score`, `is_aggregated`) on the same parquet.
+//
+// `model_results[]` rows go through the same reshape helper as
+// `getModelEvaluationCells` (defined below) — they share the
+// ModelResultForBenchmark target shape, so the eval/metric/cell
+// → BenchmarkEvaluation reshape is one helper, two callers.
+export async function getEvalSummaryById(evalId: string): Promise<BenchmarkEvalSummary | null> {
+  const conn = await getConnection()
+  const [evalReader, cellsReader] = await Promise.all([
+    conn.runAndReadAll(
+      `SELECT * FROM evals_view WHERE evaluation_id = ? LIMIT 1`,
+      [evalId]
+    ),
+    conn.runAndReadAll(
+      `SELECT * FROM eval_results_view
+       WHERE evaluation_id = ?
+         AND metric_id     = (SELECT primary_metric_id FROM evals_view WHERE evaluation_id = ?)
+       ORDER BY position ASC`,
+      [evalId, evalId]
+    ),
+  ])
+  const evalRows = evalReader.getRowObjects()
+  if (evalRows.length === 0) return null
+  return {
+    ...(evalRows[0] as Omit<BenchmarkEvalSummary, "model_results">),
+    model_results: cellsReader.getRowObjects().map(reshapeCellToModelResult),
+  } as BenchmarkEvalSummary
+}
+```
+
+### Developers
+
+```ts
+// getDeveloperList — list page; reads from headline.json (precomputed,
+// including producer-owned route_id, model/benchmark/evaluation counts,
+// and popular_evals). DeveloperListEntry is satisfied directly by the
+// headline entry shape.
+export async function getDeveloperList(): Promise<DeveloperListEntry[]> {
+  const headline = await fetchHeadline()
+  return headline.developers as DeveloperListEntry[]
+}
+
+// getDeveloperSummaryById — detail page; reads models_view filtered by developer.
+// The route_id on headline.developers[] is the canonical lookup key — we don't
+// re-derive `developer` from the URL slug, since percent-decoding may not
+// round-trip exactly to the producer's source string.
+export async function getDeveloperSummaryById(routeId: string) {
+  const headline = await fetchHeadline()
+  const headlineEntry = headline.developers.find((d) => d.route_id === routeId)
+  if (!headlineEntry) return null
+  const conn = await getConnection()
+  const reader = await conn.runAndReadAll(
+    `SELECT * FROM models_view WHERE developer = ?`,
+    [headlineEntry.developer]
+  )
+  const models = reader.getRowObjects() as EvaluationCardData[]
+  return { ...headlineEntry, models }
+}
+```
+
+### Dashboard convenience accessor
+
+```ts
+// Was: { models, evals } over both legacy parquets; same shape, new sources.
+export async function getDashboardData() {
+  const [models, evalListData] = await Promise.all([
+    getModelCards(),
+    getEvalListData(),
+  ])
+  return { models, evals: evalListData.evals }
+}
+```
+
+---
+
+## Sidecar fetchers (replace `lib/hf-data.ts` corpus calls)
+
+Three small JSON files live in the snapshot dir alongside the
+parquets. New module `lib/sidecars.ts` exposes typed fetchers.
+`lib/hf-data.ts`'s `fetchCorpusAggregates`, `fetchEvalHierarchy`,
+`fetchBackendManifest`, and `fetchBackendManifestStatus` get their
+implementations replaced — same export names, new sources.
+
+```ts
+// lib/sidecars.ts
+import "server-only"
+import type {
+  CorpusAggregates,
+  EvalHierarchy,
+  BackendManifest,
+} from "@/lib/backend-artifacts"
+
+const SNAPSHOT_URL = process.env.SNAPSHOT_URL!
+
+let cache: {
+  manifest?: Promise<BackendManifest>
+  headline?: Promise<CorpusAggregates>
+  hierarchy?: Promise<EvalHierarchy>
+} = {}
+
+async function fetchJson<T>(name: string): Promise<T> {
+  const url = `${SNAPSHOT_URL}/${name}`
+  const res = url.startsWith("file://")
+    ? await import("fs/promises").then((fs) => fs.readFile(new URL(url), "utf8"))
+    : await fetch(url, { next: { revalidate: 3600 } }).then((r) => r.text())
+  return JSON.parse(typeof res === "string" ? res : res.toString()) as T
+}
+
+export function fetchManifest(): Promise<BackendManifest> {
+  return (cache.manifest ??= fetchJson<BackendManifest>("manifest.json"))
+}
+
+export function fetchHeadline(): Promise<CorpusAggregates> {
+  return (cache.headline ??= fetchJson<CorpusAggregates>("headline.json"))
+}
+
+export function fetchHierarchy(): Promise<EvalHierarchy> {
+  return (cache.hierarchy ??= fetchJson<EvalHierarchy>("hierarchy.json"))
+}
+```
+
+Then in `lib/hf-data.ts`:
+
+```ts
+// fetchBackendManifest: was a fetchHFJsonSafe call; now reads the snapshot sidecar.
+export const fetchBackendManifest = fetchManifest
+export const fetchCorpusAggregates = fetchHeadline
+export const fetchEvalHierarchy = fetchHierarchy
+
+// fetchBackendManifestStatus: simplified — single snapshot pin, no "latest" comparison.
+export async function fetchBackendManifestStatus(): Promise<BackendManifestStatus> {
+  const m = await fetchManifest()
+  return {
+    currentManifest: m,
+    latestManifest: m,                          // no separate "latest" — snapshot is pinned
+    currentManifestSignature: m.generated_at,
+    latestManifestSignature: m.generated_at,
+    updateAvailable: false,
+    refreshing: false,
+    pendingRefreshCount: 0,
+  }
+}
+```
+
+---
+
+## What deletes
+
+After v2 is live, the following code is dead and can be removed in a
+follow-up cleanup:
+
+- `lib/duckdb-data.ts` — replaced by typed SELECTs split between
+  `lib/duckdb.ts` (connection) and `lib/data-backend.ts` (queries).
+- The `payload_json` parser helpers (`parsePayload`, `readPayloads`,
+  `readPayloadById`, `assertDeveloperListShape`) — no JSON blobs to
+  parse.
+- The `aggregate__` / `matrix__` eval-id prefix dispatch in
+  `getEvalSummaryByIdFromDuckDB` — the typed view is the only path.
+- `lib/model-data.ts` — most of its functions exist to convert HF
+  JSON records into `BenchmarkEvaluation` / `EvaluationCardData`. Once
+  the producer emits those shapes directly, the adapter logic deletes.
+  Keep only the helpers that don't touch HF records (slug parsing,
+  display formatters).
+- `lib/eval-processing.ts` — the `groupEvaluationsByModel`,
+  `createModelSummary`, `createBenchmarkEvalSummary`, and
+  `inferCategoryFromBenchmark` adapter functions are no longer called
+  in the data path. The exported types stay.
+- `scripts/audit-adapters.mjs`, `scripts/dump-adapter-outputs.mts`,
+  `scripts/compare-data-backends.mjs`, `scripts/refresh-fixtures.mjs`,
+  `scripts/cache-hf-data.mjs` — adapter / parity-check tooling for the
+  legacy pipeline. Delete once v1 is retired.
+- `data/models/`, `data/developers/`, `data/benchmarks.json`,
+  `data/models.json`, `data/developers.json` — bundled snapshots of
+  v1 output for fixture tests. Replace with v2 fixtures if needed.
+- `LOCAL_PIPELINE_OUTPUT` env var, `duckdb/v1/` subpath conventions,
+  and the parity-emitter expectations documented in
+  `lib/duckdb-data.ts`'s preamble.
+- `inferCategoryFromBenchmark` regex chain in
+  `lib/benchmark-schema.ts` — producer is the source of truth for
+  category. Keep the `EVALUATION_CATEGORIES` const + `CategoryType`
+  type; delete the inference function and `BENCHMARK_PRIORITY_RULES`.
+
+---
+
+## Slug rule
+
+Producer emits all URL-bearing identifiers in
+RFC 3986 percent-encoded form (`route_id`, `evaluation_id`,
+`metric_summary_id`). Frontend treats them as opaque except for
+`<Link>` href construction:
+
+```tsx
+// Old: href={`/models/${model.route_id}`}  // route_id was already escaped via __ rule
+// New: href={`/models/${model.route_id}`}  // same code; route_id is now percent-encoded
+```
+
+Decode happens inside the route handler when looking up by slug:
+
+```ts
+// app/models/[id]/page.tsx
+export default async function ModelDetailPage({ params }: { params: { id: string } }) {
+  const summary = await getModelSummaryById(params.id)  // pass encoded form straight through
+  ...
+}
+```
+
+`getModelSummaryById` looks up by `route_id = ?` directly without
+decoding — the producer's `route_id` column matches the URL path
+segment byte-for-byte. The legacy `replace('/', '__')` and
+`replace(/\//g, ...)` helpers in `lib/utils.ts` and `lib/model-family.ts`
+become dead code; remove them in the cleanup pass.
+
+---
+
+## Migration strategy
+
+A feature flag gates v1 vs v2 during the transition:
+
+```ts
+// lib/data-backend.ts
+const BACKEND_VERSION = process.env.DATA_BACKEND ?? "v1"
+
+export const getModelCards =
+  BACKEND_VERSION === "v2"
+    ? (await import("@/lib/duckdb")).getModelCards
+    : (await import("@/lib/duckdb-data")).getModelCardsFromDuckDB
+// ... same pattern for other accessors
+```
+
+Phase plan:
+
+1. **Producer ships Stage J.** `eval_card_backend` emits the six
+   v2 artifacts in `warehouse/<snapshot_id>/`. Existing canonical
+   parquets stay alongside.
+2. **Frontend lands `lib/duckdb.ts` + `lib/sidecars.ts`** behind the
+   `DATA_BACKEND=v2` flag. CI builds both backends; default stays v1.
+3. **Smoke test in dev with `DATA_BACKEND=v2`,
+   `SNAPSHOT_URL=file://...`.** Verify each page renders identical
+   bytes (modulo source-of-data labels). Where they diverge, file
+   producer issues — do not patch the frontend to paper over.
+4. **Flip the production default to v2.** Keep v1 path compilable but
+   unreachable. Monitor for a release.
+5. **Delete v1 path** (the "What deletes" list above).
+
+The flag is intentionally process-wide, not per-accessor. Mixing
+backends within one render produces inconsistent snapshots.
+
+---
+
+## What doesn't move
+
+- **Instance-level data fetching** (`fetchInstanceLevelData` in
+  `lib/hf-data.ts`). Instance JSONL is referenced by URL in
+  `eval_results_view.instance_file_path`; the lazy-load stays. Pointer
+  shape on the row is unchanged from v1.
+- **Benchmark card metadata** lives inside `evals_view.benchmark_card`
+  STRUCT now, not a separate `benchmark_card_*.json` per file. The
+  page reads it from the eval row directly. Adapter-style readers
+  (`fetchBenchmarkMetadataMap`) become a `SELECT benchmark_id, benchmark_card
+  FROM evals_view` aggregation if anything still calls them — most
+  callers should fold into `getEvalSummaryById`.
+- **EvalCards annotations** (`evalcards.annotations`) live on
+  `eval_results_view.evalcards_annotations` per-row. The eval-detail
+  page reads them inline; no separate fetcher.
+
+---
+
+## Open questions / risks
+
+- **httpfs cold-start latency.** First query against an HF-hosted
+  parquet pays a round trip per file. Mitigate by pre-registering all
+  three views at process start (above), so the first user query hits
+  warm metadata. Measure on the production HF Space; if too slow,
+  consider downloading the snapshot to local disk at container start
+  (~MB per snapshot).
+- **Connection lifetime in serverless.** Vercel's serverless
+  runtime tears down the Node process per request; the
+  `connectionPromise` cache doesn't help. The HF Space deployment
+  (Docker, long-lived) is unaffected. If we ever target serverless,
+  switch to `duckdb-wasm` in the browser or a separate serving
+  process.
+- **`aggregate_components[]` on `eval_results_view`.** This array is
+  the per-suite-component breakdown for rollup rows. For non-rollup
+  rows it's always empty. If suite rollups grow common, the storage
+  cost of trailing-empty arrays is non-trivial; consider splitting
+  into a dedicated parquet at that point.
+- **Category drift.** Producer's `category_mapping.json` will lag real
+  benchmark tag changes. The mapping is producer-owned, so the
+  frontend can't patch around drift — this is a feature, not a bug,
+  but it requires operator discipline. Surface "uncategorised
+  benchmark count" in the producer's run summary and the home-page
+  manifest banner.
+- **Type widening for `score_summary` etc.** The producer emits these
+  as DuckDB STRUCTs; the TS interface declares them as nested
+  `{ count, min, max, average }`. `runAndReadAll` returns nested
+  STRUCTs as plain JS objects, so the cast works — but if duckdb-node
+  changes its STRUCT serialisation, audit the `as` casts here. Add a
+  dev-only validator that runs `EvaluationCardData`'s shape check at
+  the row level on the first `getModelCards()` call after process
+  start.

notes/merge-cheatsheet-backend-v2.md

@@ -0,0 +1,193 @@
+# Merge cheatsheet: pulling `main` into `feat/use-new-backend-data`
+
+> Drafted 2026-05-04, before pulling. Companion to `backend-v2-migration.md`
+> (which is the design doc). This file is just a per-file conflict guide.
+>
+> Branch: `feat/use-new-backend-data` (2 commits ahead of `main`:
+> `7635aee` Integrate with test backend data, `bfce8f2` Drop
+> input/output_modalities from MODEL_CARD_COLUMNS).
+
+## Triage at a glance
+
+| File | Risk | Strategy |
+|---|---|---|
+| `lib/data-backend.ts` | **High** | Keep ours wholesale; re-port any new accessors main added |
+| `lib/backend-artifacts.ts` | **High** | Keep our schema renames; reconcile any *new* main-side fields against producer output |
+| `components/signals/corpus-dashboard.tsx` | **Med** | Keep main's UI structure; rewire data fields to v2 names |
+| `components/signals/corpus-signals-strip.tsx` | **Med** | Same as above |
+| `lib/hf-data.ts` | **Med** | Keep `useViewLayerBackend()` short-circuits at top of 5 fetchers |
+| `Dockerfile` | **Med** | Keep our `DATA_BACKEND=v2` + `SNAPSHOT_URL` wiring; layer main's other changes on top |
+| `lib/benchmark-schema.ts` | **Low** | Trivial 1-line addition (`num_few_shot?`) |
+| `app/page.tsx` | **Low** | One-line copy change (`corpus-aggregates.json` → `headline.json`) |
+
+New files (no conflict possible): `lib/view-data.ts`, `lib/duckdb.ts`,
+`lib/sidecars.ts`, `tests/view-data.test.ts`,
+`notes/backend-v2-migration.md`.
+
+---
+
+## `lib/data-backend.ts` — High
+
+**What we did:** Replaced static re-exports from `lib/duckdb-data` with
+a `BACKEND_VERSION` env-flag dispatcher. Each accessor now branches on
+`useViewLayerBackend()` (true when `DATA_BACKEND=v2` or `stage-j`) and
+lazy-imports either `@/lib/view-data` or `@/lib/duckdb-data`.
+Manifest/hierarchy accessors branch between `@/lib/sidecars` and
+`@/lib/hf-data`.
+
+**Reconcile:**
+- Conflict almost certain if main touched any export wiring here.
+- **Keep our file as-is.** The dispatcher pattern is load-bearing.
+- If main added a new accessor (e.g. `getFooBar`), add a new dispatcher
+  function following the same pattern — only the legacy branch needs
+  to be wired immediately; v2 branch can throw `Not implemented` until
+  `lib/view-data.ts` adds it.
+
+---
+
+## `lib/backend-artifacts.ts` — High
+
+**What we did:** Renamed corpus-block fields to match what the v2
+producer emits:
+
+| Block | v1 (main) | v2 (ours) |
+|---|---|---|
+| Completeness | `total_benchmarks`, `completeness_score_mean`, `completeness_score_median`, `per_field_population{}` | `total_triples`, `completeness_avg`, `completeness_min`, `completeness_max` |
+| Provenance | `multi_source_groups`, `multi_source_rate`, `first_party_only_groups`, `first_party_only_rate`, `total_groups` | `multi_source_triples`, `first_party_only_triples`, `total_triples` (rates dropped — derived in components via local `rate()` helper) |
+| Comparability | `variant_eligible_groups`, `variant_divergent_groups`, `variant_divergence_rate`, `cross_party_eligible_groups`, `cross_party_divergent_groups`, `cross_party_divergence_rate`, `total_groups` | `total_triples`, `variant_divergent_count`, `cross_party_divergent_count`, `groups_with_variant_check`, `groups_with_cross_party_check` |
+
+Also added: `DeveloperListEntry` interface, optional
+`developers/families/categories` arrays on `CorpusAggregates`,
+optional `eval_hierarchy` key in `BackendManifest.summary_artifacts`.
+
+**Reconcile:**
+- Producer is the source of truth for v2 field names — do **not** add
+  back v1 names to satisfy a main-side change. If main added a field
+  the v2 producer doesn't emit, either drop it or check
+  `eval_card_backend/notes/08-frontend-view-layer.md` first.
+- Keep all three new optional sections on `CorpusAggregates`
+  (developers, families, categories) — they back the new
+  developer-list path.
+- The `summary_artifacts.eval_hierarchy` key is additive; safe to keep
+  alongside whatever main added there.
+
+---
+
+## `components/signals/corpus-dashboard.tsx` — Medium
+
+**What we did:** Mechanical rewrite of every field reference in this
+file to use the v2 names from `lib/backend-artifacts.ts` (above).
+Removed the `per_field_population` per-field grid and replaced it with
+a `min / avg / max` MiniMetric trio. Added a local `rate(num, denom)`
+helper (returns null if either side is null/zero) since v2 stores
+counts, not pre-computed rates. Title-cased `CATEGORY_ORDER`
+(`"Agentic"`, `"General"`, …) and made the keys-to-render set extend
+gracefully to unknown categories.
+
+**Reconcile:**
+- If main touched this file for design/UX reasons, **prefer main's
+  visual structure** — but keep our field accessors. The recipe is:
+  - Anywhere main reads `multi_source_rate`, replace with `rate(prov.multi_source_triples, prov.total_triples)`.
+  - Anywhere main reads `completeness_score_mean`, replace with `comp.completeness_avg`.
+  - Anywhere main reads `*_eligible_groups` / `*_divergent_groups`, swap to `groups_with_*_check` / `*_divergent_count`.
+  - Drop any new code that reads `per_field_population` — gone in v2.
+- Keep the local `rate()` helper at the bottom of the file.
+- Category lookup must use the new title-cased keys (or stay tolerant
+  via the `available` set logic we added).
+
+---
+
+## `components/signals/corpus-signals-strip.tsx` — Medium
+
+**What we did:** Same field renames as above, same local `rate()`
+helper added. Headline copy updated from "groups" → "triples" where
+the underlying unit changed.
+
+**Reconcile:** Apply the same recipe as `corpus-dashboard.tsx`. The
+two files share field names and the `rate()` helper.
+
+---
+
+## `lib/hf-data.ts` — Medium
+
+**What we did:** Added an early-return guard at the top of five
+functions:
+- `fetchBackendManifestStatus` — synthesizes a status from the v2 manifest sidecar
+- `fetchBenchmarkMetadataMap` — delegates to `view-data.getBenchmarkMetadataMap`
+- `fetchBackendManifest` — delegates to `sidecars.fetchManifest`
+- `fetchEvalHierarchy` — delegates to `sidecars.fetchHierarchy` (still wraps in `adaptEvalHierarchy`)
+- `fetchCorpusAggregates` — delegates to `sidecars.fetchHeadline`
+
+Plus a module-level `useViewLayerBackend()` helper and a lazy
+`fetchSnapshotSidecars()` importer near the top of the file.
+
+**Reconcile:**
+- These are all additive guards at the start of existing functions —
+  conflicts are likely only if main re-shaped the same function
+  bodies.
+- Pattern: `if (useViewLayerBackend()) { return <v2 path> }` then fall
+  through to the existing v1 implementation untouched.
+- If main renamed one of these functions, port the guard into the
+  renamed version. Don't drop the guard.
+
+---
+
+## `Dockerfile` — Medium
+
+**What we did:**
+- Default `ARG DATA_BACKEND` flipped from `duckdb` → `v2` in **both**
+  stages (builder and runner).
+- Added `ARG SNAPSHOT_URL` + `ENV SNAPSHOT_URL` in both stages,
+  defaulting to a pinned `evaleval/eval-cards-data` warehouse path.
+- Comment block rewritten to reflect v2 + legacy coexistence.
+- Kept legacy `LOCAL_PIPELINE_OUTPUT`, `HF_DATA_LOCAL_DIR`,
+  `HF_DATA_OFFLINE=1` envs intact (legacy backend still compilable).
+
+**Uncommitted tweak (working tree):** `SNAPSHOT_URL` default points at
+`j-chim/temp_evalcard_backend` instead of `evaleval/eval-cards-data` —
+this is the dev/test dataset for the temp HF Space deploy. Do **not**
+commit this override; revert before merging to main, or keep it only
+on local working copy.
+
+**Reconcile:**
+- Keep our `DATA_BACKEND=v2` default and `SNAPSHOT_URL` plumbing.
+- Layer main's non-data changes (base image bumps, `pnpm` version,
+  build commands) on top.
+
+---
+
+## `lib/benchmark-schema.ts` — Low
+
+**What we did:** Added one optional field, `num_few_shot?: number`, on
+`GenerationConfig`. That's it.
+
+**Reconcile:** Trivially additive. Keep our line; merge tool should
+handle it cleanly unless main touched the same struct.
+
+---
+
+## `app/page.tsx` — Low
+
+**What we did:** One-line copy change in the empty-state banner —
+`corpus-aggregates.json` → `headline.json` (the v2 sidecar name).
+
+**Reconcile:** Trivial. Keep ours.
+
+---
+
+## Order of operations after `git pull`
+
+1. Resolve `lib/backend-artifacts.ts` first — it's the schema source
+   of truth that the components depend on.
+2. Resolve `lib/data-backend.ts` and `lib/hf-data.ts` — backend wiring.
+3. Resolve the two `components/signals/*` files using the rename recipe.
+4. Resolve `Dockerfile` — keep our v2 envs.
+5. `app/page.tsx` and `lib/benchmark-schema.ts` — should auto-merge or
+   be trivial.
+6. Run `pnpm tsc --noEmit` (or whatever the project's typecheck is) to
+   catch any v1 field references main introduced that didn't conflict
+   textually but break against our renamed types.
+7. Run `pnpm test` — `tests/view-data.test.ts` and
+   `tests/duckdb-data.test.ts` should both still pass.
+8. Smoke test with `DATA_BACKEND=v2 SNAPSHOT_URL=file://…` and again
+   without (legacy path) — both must render.

tests/view-data.test.ts

@@ -22,6 +22,7 @@ async function writeSyntheticStageJSnapshot(snapshotDir: string) {
     `
       SELECT
         TIMESTAMP '2026-05-03 00:00:00' AS snapshot_id,
+        'openai/gpt-5' AS model_key,
         'openai/gpt-5' AS model_id,
         'openai/gpt-5' AS id,
         'openai%2Fgpt-5' AS route_id,
@@ -205,6 +206,7 @@ async function writeSyntheticStageJSnapshot(snapshotDir: string) {
         'mmlu%3Aaccuracy' AS metric_summary_id,
         'mmlu' AS benchmark_id,
         'accuracy' AS metric_id,
+        'openai/gpt-5' AS model_key,
         'openai/gpt-5' AS model_id,
         'openai%2Fgpt-5' AS model_route_id,
         struct_pack(