j-chim commited on
Commit
7635aee
·
1 Parent(s): dbdd6d1

Integrate with test backend data

Browse files
Dockerfile CHANGED
@@ -9,20 +9,18 @@ ARG PNPM_VERSION=10.25.0
9
 
10
  # Build-time data-source configuration. HF Spaces "Variables" are NOT injected
11
  # into Docker RUN steps automatically — only into the final runtime — so we
12
- # bake the DuckDB-mode defaults here. `cache-hf-data.mjs` reads these to know
13
- # which dataset to clone and to apply lean cache mode (skip JSON-fallback
14
- # artifacts). Override at build time via `--build-arg HF_DATASET_REPO=...`.
15
- ARG DATA_BACKEND=duckdb
16
  ARG HF_DATASET_REPO=https://huggingface.co/datasets/evaleval/card_backend
17
- # Static prerender (`next build`) executes route handlers, which call
18
- # `getModelCards` etc. `lib/duckdb-data.ts`, which requires
19
- # `LOCAL_PIPELINE_OUTPUT`. The cache populated by `cache-hf-data.mjs`
20
- # lives at `/app/.cache/hf-data`. `HF_DATA_OFFLINE=1` keeps the metadata
21
- # fetchers (`lib/hf-data.ts`) from attempting `evaleval/card_backend`
22
- # network reads with `revalidate: 0` (which Next 15 treats as dynamic
23
- # and fails the static export of `/`).
24
  ENV DATA_BACKEND=${DATA_BACKEND} \
25
  HF_DATASET_REPO=${HF_DATASET_REPO} \
 
26
  LOCAL_PIPELINE_OUTPUT=/app/.cache/hf-data \
27
  HF_DATA_LOCAL_DIR=/app/.cache/hf-data \
28
  HF_DATA_OFFLINE=1
@@ -49,13 +47,15 @@ RUN pnpm run build
49
  FROM node:18-bullseye-slim AS runner
50
  WORKDIR /app
51
 
52
- # Runtime needs the same DuckDB-mode envs that the builder used. HF Space
53
- # Variables aren't set on this Space, and Docker multi-stage doesn't carry
54
- # ENVs across stages — without these, lib/duckdb-data.ts throws
55
- # "DATA_BACKEND=duckdb requires LOCAL_PIPELINE_OUTPUT" at request time and
56
- # every model/eval/developer endpoint returns empty.
 
57
  ENV NODE_ENV=production \
58
- DATA_BACKEND=duckdb \
 
59
  LOCAL_PIPELINE_OUTPUT=/app/.cache/hf-data \
60
  HF_DATA_LOCAL_DIR=/app/.cache/hf-data \
61
  HF_DATA_OFFLINE=1
 
9
 
10
  # Build-time data-source configuration. HF Spaces "Variables" are NOT injected
11
  # into Docker RUN steps automatically — only into the final runtime — so we
12
+ # bake the selected backend here. `DATA_BACKEND=v2` reads `SNAPSHOT_URL`
13
+ # directly; legacy DuckDB mode still clones `HF_DATASET_REPO` into the cache.
14
+ # Override at build time via `--build-arg ...`.
15
+ ARG DATA_BACKEND=v2
16
  ARG HF_DATASET_REPO=https://huggingface.co/datasets/evaleval/card_backend
17
+ ARG SNAPSHOT_URL=https://huggingface.co/datasets/evaleval/eval-cards-data/resolve/main/warehouse/2026-05-03T21-46-50Z
18
+ # Static prerender (`next build`) executes route handlers. In legacy mode the
19
+ # cache populated by `cache-hf-data.mjs` lives at `/app/.cache/hf-data`; in v2
20
+ # the cache step is skipped and the app reads the pinned Stage J snapshot.
 
 
 
21
  ENV DATA_BACKEND=${DATA_BACKEND} \
22
  HF_DATASET_REPO=${HF_DATASET_REPO} \
23
+ SNAPSHOT_URL=${SNAPSHOT_URL} \
24
  LOCAL_PIPELINE_OUTPUT=/app/.cache/hf-data \
25
  HF_DATA_LOCAL_DIR=/app/.cache/hf-data \
26
  HF_DATA_OFFLINE=1
 
47
  FROM node:18-bullseye-slim AS runner
48
  WORKDIR /app
49
 
50
+ ARG DATA_BACKEND=v2
51
+ ARG SNAPSHOT_URL=https://huggingface.co/datasets/evaleval/eval-cards-data/resolve/main/warehouse/2026-05-03T21-46-50Z
52
+
53
+ # Runtime needs the same data-source envs that the builder used. Docker
54
+ # multi-stage doesn't carry ENVs across stages, so keep backend selection and
55
+ # snapshot/cache pointers explicit here too.
56
  ENV NODE_ENV=production \
57
+ DATA_BACKEND=${DATA_BACKEND} \
58
+ SNAPSHOT_URL=${SNAPSHOT_URL} \
59
  LOCAL_PIPELINE_OUTPUT=/app/.cache/hf-data \
60
  HF_DATA_LOCAL_DIR=/app/.cache/hf-data \
61
  HF_DATA_OFFLINE=1
app/page.tsx CHANGED
@@ -244,7 +244,7 @@ export default async function HomePage() {
244
  <p className="mx-auto mt-2 max-w-2xl text-sm leading-6 text-[color:var(--fg-muted)]">
245
  The current backend snapshot does not include{" "}
246
  <code className="rounded-sm bg-[color:var(--bg-surface)] px-1.5 py-0.5 font-mono text-xs">
247
- corpus-aggregates.json
248
  </code>
249
  . When it does, this section will render the four corpus-level rollups.
250
  </p>
 
244
  <p className="mx-auto mt-2 max-w-2xl text-sm leading-6 text-[color:var(--fg-muted)]">
245
  The current backend snapshot does not include{" "}
246
  <code className="rounded-sm bg-[color:var(--bg-surface)] px-1.5 py-0.5 font-mono text-xs">
247
+ headline.json
248
  </code>
249
  . When it does, this section will render the four corpus-level rollups.
250
  </p>
components/signals/corpus-dashboard.tsx CHANGED
@@ -20,7 +20,7 @@ import {
20
  formatPercent,
21
  } from "./signal-utils"
22
 
23
- const CATEGORY_ORDER = ["agentic", "general", "knowledge", "reasoning", "safety", "other"]
24
 
25
  const SOURCE_COLORS: Record<string, string> = {
26
  first_party: "bg-amber-500",
@@ -51,13 +51,21 @@ export function CorpusDashboard({
51
  }, [mode])
52
 
53
  const categoryKeys = useMemo(
54
- () =>
55
- CATEGORY_ORDER.filter((category) =>
56
- aggregates.reproducibility.by_category[category] ||
57
- aggregates.completeness.by_category[category] ||
58
- aggregates.provenance.by_category[category] ||
59
- aggregates.comparability.by_category[category]
60
- ),
 
 
 
 
 
 
 
 
61
  [aggregates]
62
  )
63
 
@@ -190,25 +198,14 @@ function CompletenessSection({
190
  icon={<ClipboardCheck className="h-5 w-5" />}
191
  title="Reporting Completeness"
192
  subtitle="How much benchmark documentation is populated."
193
- headline={formatPercent(block.completeness_score_mean)}
194
- headlineLabel={`Median ${formatPercent(block.completeness_score_median)} across ${block.total_benchmarks.toLocaleString()} benchmarks`}
195
  >
196
  {scores.length > 0 && <Histogram scores={scores} />}
197
- <div className="mt-4 grid gap-2">
198
- {Object.entries(block.per_field_population).slice(0, 10).map(([field, value]) => (
199
- <div key={field} className="rounded-xl border border-border/60 bg-background px-3 py-2">
200
- <div className="flex items-start justify-between gap-3 text-sm">
201
- <span className="font-medium">{formatFieldLabel(field)}</span>
202
- <span className="shrink-0 tabular-nums text-muted-foreground">
203
- {formatPercent(value.mean_score)}
204
- </span>
205
- </div>
206
- <div className="mt-2 grid gap-1.5">
207
- <MetricBar label="Any data" value={value.populated_rate} compact />
208
- <MetricBar label="Fully populated" value={value.fully_populated_rate} compact />
209
- </div>
210
- </div>
211
- ))}
212
  </div>
213
  </DashboardSection>
214
  )
@@ -217,14 +214,16 @@ function CompletenessSection({
217
  function ProvenanceSection({ block }: { block: ProvenanceCorpusBlock }) {
218
  const distribution = block.source_type_distribution
219
  const total = Object.values(distribution).reduce((sum, value) => sum + value, 0)
 
 
220
 
221
  return (
222
  <DashboardSection
223
  icon={<BarChart3 className="h-5 w-5" />}
224
  title="Provenance"
225
  subtitle="Who reported the scores, and whether groups have multiple sources."
226
- headline={formatPercent(block.multi_source_rate)}
227
- headlineLabel="of (model, benchmark, metric) groups have multiple reporting sources"
228
  >
229
  <div className="overflow-hidden rounded-full border border-border/70 bg-muted/30">
230
  <div className="flex h-4 w-full">
@@ -240,34 +239,40 @@ function ProvenanceSection({ block }: { block: ProvenanceCorpusBlock }) {
240
  </div>
241
 
242
  <div className="mt-3 grid gap-2 sm:grid-cols-2">
243
- <RatioTile label="Multi-source groups" value={block.multi_source_rate} count={block.multi_source_groups} />
244
- <RatioTile label="First-party only groups" value={block.first_party_only_rate} count={block.first_party_only_groups} />
245
  </div>
246
  </DashboardSection>
247
  )
248
  }
249
 
250
  function ComparabilitySection({ block }: { block: ComparabilityCorpusBlock }) {
 
 
 
 
 
 
251
  return (
252
  <DashboardSection
253
  icon={<GitCompareArrows className="h-5 w-5" />}
254
  title="Comparability"
255
  subtitle="Eligible groups where scores diverge across setups or reporting organizations."
256
- headline={formatNullableRate(block.variant_divergence_rate)}
257
- headlineLabel={`${block.variant_divergent_groups.toLocaleString()} of ${block.variant_eligible_groups.toLocaleString()} setup-eligible groups diverge`}
258
  >
259
  <div className="grid gap-3 md:grid-cols-2">
260
  <ComparabilityRateCard
261
  title="Variant divergence"
262
- rate={block.variant_divergence_rate}
263
- eligible={block.variant_eligible_groups}
264
- divergent={block.variant_divergent_groups}
265
  />
266
  <ComparabilityRateCard
267
  title="Cross-party divergence"
268
- rate={block.cross_party_divergence_rate}
269
- eligible={block.cross_party_eligible_groups}
270
- divergent={block.cross_party_divergent_groups}
271
  />
272
  </div>
273
  </DashboardSection>
@@ -288,6 +293,15 @@ function CategoryPanel({
288
  comparability?: ComparabilityCorpusBlock
289
  }) {
290
  const categoryLabel = `${category.charAt(0).toUpperCase()}${category.slice(1)}`
 
 
 
 
 
 
 
 
 
291
 
292
  return (
293
  <section className="rounded-2xl border border-border/70 bg-card p-4 shadow-sm">
@@ -297,11 +311,11 @@ function CategoryPanel({
297
  </div>
298
  <div className="grid gap-3 sm:grid-cols-2">
299
  <MiniMetric label="Reproducibility gaps" value={formatPercent(reproducibility?.reproducibility_gap_rate)} />
300
- <MiniMetric label="Documentation mean" value={formatPercent(completeness?.completeness_score_mean)} />
301
- <MiniMetric label="Multi-source groups" value={formatPercent(provenance?.multi_source_rate)} />
302
- <MiniMetric label="Variant divergence" value={formatNullableRate(comparability?.variant_divergence_rate)} />
303
  </div>
304
- {comparability?.cross_party_divergence_rate == null && (
305
  <div className="mt-3 rounded-xl border border-dashed border-border/70 bg-muted/10 px-3 py-2 text-sm text-muted-foreground">
306
  Cross-party divergence: N/A - not enough multi-org coverage.
307
  </div>
@@ -411,7 +425,7 @@ function RatioTile({ label, value, count }: { label: string; value: number | nul
411
  <div className="text-sm font-medium">{label}</div>
412
  <div className="mt-1 flex items-baseline justify-between gap-2">
413
  <span className="text-xl font-semibold tabular-nums">{formatPercent(value)}</span>
414
- <span className="text-xs text-muted-foreground">{count.toLocaleString()} groups</span>
415
  </div>
416
  </div>
417
  )
@@ -463,6 +477,11 @@ function formatNullableRate(value: number | null | undefined) {
463
  return value == null ? "N/A" : formatPercent(value)
464
  }
465
 
 
 
 
 
 
466
  function formatGeneratedDate(value: string) {
467
  const date = new Date(value)
468
  if (Number.isNaN(date.getTime())) {
 
20
  formatPercent,
21
  } from "./signal-utils"
22
 
23
+ const CATEGORY_ORDER = ["Agentic", "General", "Knowledge", "Reasoning", "Safety", "Other"]
24
 
25
  const SOURCE_COLORS: Record<string, string> = {
26
  first_party: "bg-amber-500",
 
51
  }, [mode])
52
 
53
  const categoryKeys = useMemo(
54
+ () => {
55
+ const available = new Set([
56
+ ...Object.keys(aggregates.reproducibility.by_category),
57
+ ...Object.keys(aggregates.completeness.by_category),
58
+ ...Object.keys(aggregates.provenance.by_category),
59
+ ...Object.keys(aggregates.comparability.by_category),
60
+ ])
61
+
62
+ return [
63
+ ...CATEGORY_ORDER.filter((category) => available.has(category)),
64
+ ...Array.from(available)
65
+ .filter((category) => !CATEGORY_ORDER.includes(category))
66
+ .sort((a, b) => a.localeCompare(b)),
67
+ ]
68
+ },
69
  [aggregates]
70
  )
71
 
 
198
  icon={<ClipboardCheck className="h-5 w-5" />}
199
  title="Reporting Completeness"
200
  subtitle="How much benchmark documentation is populated."
201
+ headline={formatPercent(block.completeness_avg)}
202
+ headlineLabel={`Range ${formatPercent(block.completeness_min)} to ${formatPercent(block.completeness_max)} across ${block.total_triples.toLocaleString()} reported score triples`}
203
  >
204
  {scores.length > 0 && <Histogram scores={scores} />}
205
+ <div className="mt-4 grid gap-2 sm:grid-cols-3">
206
+ <MiniMetric label="Minimum" value={formatPercent(block.completeness_min)} />
207
+ <MiniMetric label="Average" value={formatPercent(block.completeness_avg)} />
208
+ <MiniMetric label="Maximum" value={formatPercent(block.completeness_max)} />
 
 
 
 
 
 
 
 
 
 
 
209
  </div>
210
  </DashboardSection>
211
  )
 
214
  function ProvenanceSection({ block }: { block: ProvenanceCorpusBlock }) {
215
  const distribution = block.source_type_distribution
216
  const total = Object.values(distribution).reduce((sum, value) => sum + value, 0)
217
+ const multiSourceRate = rate(block.multi_source_triples, block.total_triples)
218
+ const firstPartyOnlyRate = rate(block.first_party_only_triples, block.total_triples)
219
 
220
  return (
221
  <DashboardSection
222
  icon={<BarChart3 className="h-5 w-5" />}
223
  title="Provenance"
224
  subtitle="Who reported the scores, and whether groups have multiple sources."
225
+ headline={formatPercent(multiSourceRate)}
226
+ headlineLabel="of reported score triples have multiple reporting sources"
227
  >
228
  <div className="overflow-hidden rounded-full border border-border/70 bg-muted/30">
229
  <div className="flex h-4 w-full">
 
239
  </div>
240
 
241
  <div className="mt-3 grid gap-2 sm:grid-cols-2">
242
+ <RatioTile label="Multi-source triples" value={multiSourceRate} count={block.multi_source_triples} />
243
+ <RatioTile label="First-party only triples" value={firstPartyOnlyRate} count={block.first_party_only_triples} />
244
  </div>
245
  </DashboardSection>
246
  )
247
  }
248
 
249
  function ComparabilitySection({ block }: { block: ComparabilityCorpusBlock }) {
250
+ const variantRate = rate(block.variant_divergent_count, block.groups_with_variant_check)
251
+ const crossPartyRate = rate(
252
+ block.cross_party_divergent_count,
253
+ block.groups_with_cross_party_check
254
+ )
255
+
256
  return (
257
  <DashboardSection
258
  icon={<GitCompareArrows className="h-5 w-5" />}
259
  title="Comparability"
260
  subtitle="Eligible groups where scores diverge across setups or reporting organizations."
261
+ headline={formatNullableRate(variantRate)}
262
+ headlineLabel={`${block.variant_divergent_count.toLocaleString()} of ${block.groups_with_variant_check.toLocaleString()} setup-eligible groups diverge`}
263
  >
264
  <div className="grid gap-3 md:grid-cols-2">
265
  <ComparabilityRateCard
266
  title="Variant divergence"
267
+ rate={variantRate}
268
+ eligible={block.groups_with_variant_check}
269
+ divergent={block.variant_divergent_count}
270
  />
271
  <ComparabilityRateCard
272
  title="Cross-party divergence"
273
+ rate={crossPartyRate}
274
+ eligible={block.groups_with_cross_party_check}
275
+ divergent={block.cross_party_divergent_count}
276
  />
277
  </div>
278
  </DashboardSection>
 
293
  comparability?: ComparabilityCorpusBlock
294
  }) {
295
  const categoryLabel = `${category.charAt(0).toUpperCase()}${category.slice(1)}`
296
+ const multiSourceRate = rate(provenance?.multi_source_triples, provenance?.total_triples)
297
+ const variantRate = rate(
298
+ comparability?.variant_divergent_count,
299
+ comparability?.groups_with_variant_check
300
+ )
301
+ const crossPartyRate = rate(
302
+ comparability?.cross_party_divergent_count,
303
+ comparability?.groups_with_cross_party_check
304
+ )
305
 
306
  return (
307
  <section className="rounded-2xl border border-border/70 bg-card p-4 shadow-sm">
 
311
  </div>
312
  <div className="grid gap-3 sm:grid-cols-2">
313
  <MiniMetric label="Reproducibility gaps" value={formatPercent(reproducibility?.reproducibility_gap_rate)} />
314
+ <MiniMetric label="Documentation mean" value={formatPercent(completeness?.completeness_avg)} />
315
+ <MiniMetric label="Multi-source triples" value={formatPercent(multiSourceRate)} />
316
+ <MiniMetric label="Variant divergence" value={formatNullableRate(variantRate)} />
317
  </div>
318
+ {crossPartyRate == null && (
319
  <div className="mt-3 rounded-xl border border-dashed border-border/70 bg-muted/10 px-3 py-2 text-sm text-muted-foreground">
320
  Cross-party divergence: N/A - not enough multi-org coverage.
321
  </div>
 
425
  <div className="text-sm font-medium">{label}</div>
426
  <div className="mt-1 flex items-baseline justify-between gap-2">
427
  <span className="text-xl font-semibold tabular-nums">{formatPercent(value)}</span>
428
+ <span className="text-xs text-muted-foreground">{count.toLocaleString()} triples</span>
429
  </div>
430
  </div>
431
  )
 
477
  return value == null ? "N/A" : formatPercent(value)
478
  }
479
 
480
+ function rate(numerator: number | null | undefined, denominator: number | null | undefined) {
481
+ if (numerator == null || denominator == null || denominator <= 0) return null
482
+ return numerator / denominator
483
+ }
484
+
485
  function formatGeneratedDate(value: string) {
486
  const date = new Date(value)
487
  if (Number.isNaN(date.getTime())) {
components/signals/corpus-signals-strip.tsx CHANGED
@@ -39,8 +39,13 @@ export function CorpusSignalsStrip({
39
  const tpShare = totalReports > 0 ? prov.source_type_distribution.third_party / totalReports : 0
40
  const fpShare = totalReports > 0 ? prov.source_type_distribution.first_party / totalReports : 0
41
 
42
- const cmpRate = cmp.variant_divergence_rate
43
- const crossPartyAvailable = cmp.cross_party_eligible_groups > 0
 
 
 
 
 
44
 
45
  return (
46
  <div className="signals-grid">
@@ -58,29 +63,29 @@ export function CorpusSignalsStrip({
58
  />
59
  <SignalTile
60
  id="completeness"
61
- statValue={pctNum(comp.completeness_score_mean)}
62
  statUnit="%"
63
- headline={`mean across ${comp.total_benchmarks.toLocaleString()} benchmarks (median ${formatPct(comp.completeness_score_median)}).`}
64
- detail="Source-provenance fields populate fully; preregistration fields are unmet."
65
  asks="Is the benchmark itself documented well enough to interpret a score on it?"
66
  />
67
  <SignalTile
68
  id="provenance"
69
- statValue={pctNum(prov.multi_source_rate)}
70
  statUnit="%"
71
- headline="of (model, benchmark) groups have reports from more than one party."
72
- detail={`${formatPct(tpShare)} third-party, ${formatPct(fpShare)} first-party of ${totalReports.toLocaleString()} results.`}
73
  asks="Who reported this score, and have others reproduced it?"
74
  />
75
  <SignalTile
76
  id="comparability"
77
  statValue={pctNum(cmpRate)}
78
  statUnit="%"
79
- headline={`of setup-eligible groups diverge across variants (${cmp.variant_divergent_groups.toLocaleString()} of ${cmp.variant_eligible_groups.toLocaleString()}).`}
80
  detail={
81
  crossPartyAvailable
82
- ? `Cross-party divergence: ${formatPct(cmp.cross_party_divergence_rate)}.`
83
- : "Cross-party divergence not yet computable too few multi-org reports."
84
  }
85
  asks="Are scores on the same benchmark actually measuring the same thing?"
86
  />
@@ -154,6 +159,11 @@ function formatPct(value: number | null | undefined): string {
154
  return `${Math.round(value * 100)}%`
155
  }
156
 
 
 
 
 
 
157
  const FIELD_LABELS: Record<string, string> = {
158
  temperature: "temperature",
159
  max_tokens: "max tokens",
 
39
  const tpShare = totalReports > 0 ? prov.source_type_distribution.third_party / totalReports : 0
40
  const fpShare = totalReports > 0 ? prov.source_type_distribution.first_party / totalReports : 0
41
 
42
+ const multiSourceRate = rate(prov.multi_source_triples, prov.total_triples)
43
+ const cmpRate = rate(cmp.variant_divergent_count, cmp.groups_with_variant_check)
44
+ const crossPartyRate = rate(
45
+ cmp.cross_party_divergent_count,
46
+ cmp.groups_with_cross_party_check
47
+ )
48
+ const crossPartyAvailable = cmp.groups_with_cross_party_check > 0
49
 
50
  return (
51
  <div className="signals-grid">
 
63
  />
64
  <SignalTile
65
  id="completeness"
66
+ statValue={pctNum(comp.completeness_avg)}
67
  statUnit="%"
68
+ headline={`mean across ${comp.total_triples.toLocaleString()} reported score triples.`}
69
+ detail={`Observed range: ${formatPct(comp.completeness_min)} to ${formatPct(comp.completeness_max)}.`}
70
  asks="Is the benchmark itself documented well enough to interpret a score on it?"
71
  />
72
  <SignalTile
73
  id="provenance"
74
+ statValue={pctNum(multiSourceRate)}
75
  statUnit="%"
76
+ headline="of reported score triples have reports from more than one party."
77
+ detail={`${formatPct(tpShare)} third-party, ${formatPct(fpShare)} first-party of ${totalReports.toLocaleString()} triples.`}
78
  asks="Who reported this score, and have others reproduced it?"
79
  />
80
  <SignalTile
81
  id="comparability"
82
  statValue={pctNum(cmpRate)}
83
  statUnit="%"
84
+ headline={`of setup-eligible groups diverge across variants (${cmp.variant_divergent_count.toLocaleString()} of ${cmp.groups_with_variant_check.toLocaleString()}).`}
85
  detail={
86
  crossPartyAvailable
87
+ ? `Cross-party divergence: ${formatPct(crossPartyRate)}.`
88
+ : "Cross-party divergence not yet computable: too few multi-org reports."
89
  }
90
  asks="Are scores on the same benchmark actually measuring the same thing?"
91
  />
 
159
  return `${Math.round(value * 100)}%`
160
  }
161
 
162
+ function rate(numerator: number | null | undefined, denominator: number | null | undefined) {
163
+ if (numerator == null || denominator == null || denominator <= 0) return null
164
+ return numerator / denominator
165
+ }
166
+
167
  const FIELD_LABELS: Record<string, string> = {
168
  temperature: "temperature",
169
  max_tokens: "max tokens",
lib/backend-artifacts.ts CHANGED
@@ -12,6 +12,7 @@ export interface BackendManifest {
12
  skipped_config_count?: number
13
  summary_artifacts?: {
14
  corpus_aggregates?: string
 
15
  [key: string]: string | undefined
16
  }
17
  }
@@ -177,6 +178,27 @@ export interface CorpusAggregates {
177
  completeness: Stratified<CompletenessCorpusBlock>
178
  provenance: Stratified<ProvenanceCorpusBlock>
179
  comparability: Stratified<ComparabilityCorpusBlock>
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
180
  }
181
 
182
  export interface Stratified<T> {
@@ -198,35 +220,25 @@ export interface ReproducibilityCorpusBlock {
198
  }
199
 
200
  export interface CompletenessCorpusBlock {
201
- total_benchmarks: number
202
- completeness_score_mean: number | null
203
- completeness_score_median: number | null
204
- per_field_population: Record<string, {
205
- mean_score: number
206
- populated_rate: number
207
- fully_populated_rate: number
208
- benchmark_count: number
209
- }>
210
  }
211
 
212
  export interface ProvenanceCorpusBlock {
213
  total_triples: number
214
- total_groups: number
215
- multi_source_groups: number
216
- multi_source_rate: number | null
217
- first_party_only_groups: number
218
- first_party_only_rate: number | null
219
  source_type_distribution: Record<ProvenanceSourceType, number>
220
  }
221
 
222
  export interface ComparabilityCorpusBlock {
223
- total_groups: number
224
- variant_eligible_groups: number
225
- variant_divergent_groups: number
226
- variant_divergence_rate: number | null
227
- cross_party_eligible_groups: number
228
- cross_party_divergent_groups: number
229
- cross_party_divergence_rate: number | null
230
  }
231
 
232
  export interface HierarchyTags {
 
12
  skipped_config_count?: number
13
  summary_artifacts?: {
14
  corpus_aggregates?: string
15
+ eval_hierarchy?: string
16
  [key: string]: string | undefined
17
  }
18
  }
 
178
  completeness: Stratified<CompletenessCorpusBlock>
179
  provenance: Stratified<ProvenanceCorpusBlock>
180
  comparability: Stratified<ComparabilityCorpusBlock>
181
+ developers?: DeveloperListEntry[]
182
+ families?: Array<{
183
+ family_key: string
184
+ display_name: string
185
+ model_count: number
186
+ eval_count: number
187
+ }>
188
+ categories?: Array<{
189
+ category: string
190
+ model_count: number
191
+ eval_count: number
192
+ }>
193
+ }
194
+
195
+ export interface DeveloperListEntry {
196
+ developer: string
197
+ route_id: string
198
+ model_count: number
199
+ benchmark_count: number
200
+ evaluation_count: number
201
+ popular_evals: Array<{ benchmark: string; model_count: number }>
202
  }
203
 
204
  export interface Stratified<T> {
 
220
  }
221
 
222
  export interface CompletenessCorpusBlock {
223
+ total_triples: number
224
+ completeness_avg: number | null
225
+ completeness_min: number | null
226
+ completeness_max: number | null
 
 
 
 
 
227
  }
228
 
229
  export interface ProvenanceCorpusBlock {
230
  total_triples: number
231
+ multi_source_triples: number
232
+ first_party_only_triples: number
 
 
 
233
  source_type_distribution: Record<ProvenanceSourceType, number>
234
  }
235
 
236
  export interface ComparabilityCorpusBlock {
237
+ total_triples: number
238
+ variant_divergent_count: number
239
+ cross_party_divergent_count: number
240
+ groups_with_variant_check: number
241
+ groups_with_cross_party_check: number
 
 
242
  }
243
 
244
  export interface HierarchyTags {
lib/benchmark-schema.ts CHANGED
@@ -124,6 +124,7 @@ export interface ScoreDetails {
124
  }
125
 
126
  export interface GenerationConfig {
 
127
  generation_args?: {
128
  temperature?: number
129
  top_p?: number
 
124
  }
125
 
126
  export interface GenerationConfig {
127
+ num_few_shot?: number
128
  generation_args?: {
129
  temperature?: number
130
  top_p?: number
lib/data-backend.ts CHANGED
@@ -1,27 +1,138 @@
1
  import "server-only"
2
 
3
- import {
4
- fetchBackendManifest,
5
- fetchBackendManifestStatus,
6
- fetchEvalHierarchy,
7
- } from "@/lib/hf-data"
8
-
9
- export {
10
- getDashboardDataFromDuckDB as getDashboardData,
11
- getModelCardsFromDuckDB as getModelCards,
12
- getModelCardsLiteFromDuckDB as getModelCardsLite,
13
- getEvalListDataFromDuckDB as getEvalListData,
14
- getEvalListLiteDataFromDuckDB as getEvalListLiteData,
15
- getEvalListFromDuckDB as getEvalList,
16
- getDeveloperListFromDuckDB as getDeveloperList,
17
- getDeveloperSummaryByIdFromDuckDB as getDeveloperSummaryById,
18
- getModelSummaryByIdFromDuckDB as getModelSummaryById,
19
- getEvalSummaryByIdFromDuckDB as getEvalSummaryById,
20
- } from "@/lib/duckdb-data"
21
-
22
- // Metadata-style artifacts are still read through the existing JSON/HF path.
23
- // They are not request-time processing hotspots and the DuckDB shadow doesn't
24
- // re-shape them, so calling lib/hf-data directly avoids needless indirection.
25
- export const getBackendManifestData = fetchBackendManifest
26
- export const getBackendManifestStatusData = fetchBackendManifestStatus
27
- export const getEvalHierarchyData = fetchEvalHierarchy
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  import "server-only"
2
 
3
+ import type { BackendManifestStatus } from "@/lib/backend-artifacts"
4
+
5
+ const BACKEND_VERSION = process.env.DATA_BACKEND?.trim().toLowerCase() ?? "duckdb"
6
+
7
+ function useViewLayerBackend() {
8
+ return BACKEND_VERSION === "v2" || BACKEND_VERSION === "stage-j"
9
+ }
10
+
11
+ async function legacyBackend() {
12
+ return import("@/lib/duckdb-data")
13
+ }
14
+
15
+ async function viewBackend() {
16
+ return import("@/lib/view-data")
17
+ }
18
+
19
+ async function sidecars() {
20
+ return import("@/lib/sidecars")
21
+ }
22
+
23
+ async function hfData() {
24
+ return import("@/lib/hf-data")
25
+ }
26
+
27
+ export async function getModelCards() {
28
+ if (useViewLayerBackend()) {
29
+ return (await viewBackend()).getModelCards()
30
+ }
31
+
32
+ return (await legacyBackend()).getModelCardsFromDuckDB()
33
+ }
34
+
35
+ export async function getModelCardsLite() {
36
+ if (useViewLayerBackend()) {
37
+ return (await viewBackend()).getModelCardsLite()
38
+ }
39
+
40
+ return (await legacyBackend()).getModelCardsLiteFromDuckDB()
41
+ }
42
+
43
+ export async function getEvalListData() {
44
+ if (useViewLayerBackend()) {
45
+ return (await viewBackend()).getEvalListData()
46
+ }
47
+
48
+ return (await legacyBackend()).getEvalListDataFromDuckDB()
49
+ }
50
+
51
+ export async function getEvalListLiteData() {
52
+ if (useViewLayerBackend()) {
53
+ return (await viewBackend()).getEvalListLiteData()
54
+ }
55
+
56
+ return (await legacyBackend()).getEvalListLiteDataFromDuckDB()
57
+ }
58
+
59
+ export async function getEvalList() {
60
+ if (useViewLayerBackend()) {
61
+ return (await viewBackend()).getEvalList()
62
+ }
63
+
64
+ return (await legacyBackend()).getEvalListFromDuckDB()
65
+ }
66
+
67
+ export async function getDashboardData() {
68
+ if (useViewLayerBackend()) {
69
+ return (await viewBackend()).getDashboardData()
70
+ }
71
+
72
+ return (await legacyBackend()).getDashboardDataFromDuckDB()
73
+ }
74
+
75
+ export async function getDeveloperList() {
76
+ if (useViewLayerBackend()) {
77
+ return (await viewBackend()).getDeveloperList()
78
+ }
79
+
80
+ return (await legacyBackend()).getDeveloperListFromDuckDB()
81
+ }
82
+
83
+ export async function getDeveloperSummaryById(routeId: string) {
84
+ if (useViewLayerBackend()) {
85
+ return (await viewBackend()).getDeveloperSummaryById(routeId)
86
+ }
87
+
88
+ return (await legacyBackend()).getDeveloperSummaryByIdFromDuckDB(routeId)
89
+ }
90
+
91
+ export async function getModelSummaryById(modelId: string) {
92
+ if (useViewLayerBackend()) {
93
+ return (await viewBackend()).getModelSummaryById(modelId)
94
+ }
95
+
96
+ return (await legacyBackend()).getModelSummaryByIdFromDuckDB(modelId)
97
+ }
98
+
99
+ export async function getEvalSummaryById(evalId: string) {
100
+ if (useViewLayerBackend()) {
101
+ return (await viewBackend()).getEvalSummaryById(evalId)
102
+ }
103
+
104
+ return (await legacyBackend()).getEvalSummaryByIdFromDuckDB(evalId)
105
+ }
106
+
107
+ export async function getBackendManifestData() {
108
+ if (useViewLayerBackend()) {
109
+ return (await sidecars()).fetchManifest()
110
+ }
111
+
112
+ return (await hfData()).fetchBackendManifest()
113
+ }
114
+
115
+ export async function getBackendManifestStatusData(): Promise<BackendManifestStatus> {
116
+ if (useViewLayerBackend()) {
117
+ const manifest = await (await sidecars()).fetchManifest()
118
+ return {
119
+ currentManifest: manifest,
120
+ latestManifest: manifest,
121
+ currentManifestSignature: manifest.generated_at,
122
+ latestManifestSignature: manifest.generated_at,
123
+ updateAvailable: false,
124
+ refreshing: false,
125
+ pendingRefreshCount: 0,
126
+ }
127
+ }
128
+
129
+ return (await hfData()).fetchBackendManifestStatus()
130
+ }
131
+
132
+ export async function getEvalHierarchyData() {
133
+ if (useViewLayerBackend()) {
134
+ return (await sidecars()).fetchHierarchy()
135
+ }
136
+
137
+ return (await hfData()).fetchEvalHierarchy()
138
+ }
lib/duckdb.ts ADDED
@@ -0,0 +1,46 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import "server-only"
2
+
3
+ import { DuckDBConnection } from "@duckdb/node-api"
4
+
5
+ let connectionPromise: Promise<DuckDBConnection> | null = null
6
+
7
+ function getSnapshotUrl() {
8
+ const snapshotUrl = process.env.SNAPSHOT_URL?.trim()
9
+ if (!snapshotUrl) {
10
+ throw new Error("DATA_BACKEND=v2 requires SNAPSHOT_URL to point at a Stage J snapshot directory")
11
+ }
12
+
13
+ return snapshotUrl.replace(/\/+$/, "")
14
+ }
15
+
16
+ function snapshotArtifact(name: string) {
17
+ return `${getSnapshotUrl()}/${name}`
18
+ }
19
+
20
+ function sqlString(value: string) {
21
+ return `'${value.replace(/'/g, "''")}'`
22
+ }
23
+
24
+ const VIEW_FILES = {
25
+ models_view: "models_view.parquet",
26
+ evals_view: "evals_view.parquet",
27
+ eval_results_view: "eval_results_view.parquet",
28
+ } as const
29
+
30
+ export async function getConnection(): Promise<DuckDBConnection> {
31
+ if (!connectionPromise) {
32
+ connectionPromise = (async () => {
33
+ const connection = await DuckDBConnection.create()
34
+
35
+ for (const [viewName, fileName] of Object.entries(VIEW_FILES)) {
36
+ await connection.run(
37
+ `CREATE OR REPLACE VIEW ${viewName} AS SELECT * FROM read_parquet(${sqlString(snapshotArtifact(fileName))})`
38
+ )
39
+ }
40
+
41
+ return connection
42
+ })()
43
+ }
44
+
45
+ return connectionPromise
46
+ }
lib/hf-data.ts CHANGED
@@ -138,6 +138,15 @@ function getManifestSignature(manifest: BackendManifest | null | undefined) {
138
  // reading the same on-disk artifacts cannot diverge mid-test via background
139
  // refresh, and useful generally for offline development.
140
  const OFFLINE = process.env.HF_DATA_OFFLINE === "1"
 
 
 
 
 
 
 
 
 
141
 
142
  async function fetchRemoteJson<T>(relativePath: string): Promise<T> {
143
  if (OFFLINE) {
@@ -423,6 +432,19 @@ async function fetchHFJson<T>(relativePath: string): Promise<T> {
423
  }
424
 
425
  export async function fetchBackendManifestStatus(): Promise<BackendManifestStatus> {
 
 
 
 
 
 
 
 
 
 
 
 
 
426
  const snapshot = await getManifestSnapshot()
427
  const currentManifest = getCurrentManifestFromSnapshot(snapshot)
428
  const currentManifestSignature = getManifestSignature(currentManifest)
@@ -864,14 +886,26 @@ export async function fetchDevelopersList(): Promise<HFDeveloperEntry[]> {
864
  }
865
 
866
  export async function fetchBenchmarkMetadataMap(): Promise<Record<string, BenchmarkCard>> {
 
 
 
 
867
  return fetchHFJson<Record<string, BenchmarkCard>>("benchmark-metadata.json")
868
  }
869
 
870
  export async function fetchBackendManifest(): Promise<BackendManifest> {
 
 
 
 
871
  return fetchHFJson<BackendManifest>("manifest.json")
872
  }
873
 
874
  export async function fetchEvalHierarchy(): Promise<EvalHierarchy> {
 
 
 
 
875
  const raw = await fetchHFJson<EvalHierarchy>("eval-hierarchy.json")
876
  return adaptEvalHierarchy(raw)
877
  }
@@ -975,6 +1009,10 @@ export async function fetchComparisonIndex(): Promise<ComparisonIndex> {
975
  }
976
 
977
  export async function fetchCorpusAggregates(): Promise<CorpusAggregates | null> {
 
 
 
 
978
  return fetchHFJsonSafe<CorpusAggregates>("corpus-aggregates.json")
979
  }
980
 
 
138
  // reading the same on-disk artifacts cannot diverge mid-test via background
139
  // refresh, and useful generally for offline development.
140
  const OFFLINE = process.env.HF_DATA_OFFLINE === "1"
141
+ const DATA_BACKEND_VERSION = process.env.DATA_BACKEND?.trim().toLowerCase()
142
+
143
+ function useViewLayerBackend() {
144
+ return DATA_BACKEND_VERSION === "v2" || DATA_BACKEND_VERSION === "stage-j"
145
+ }
146
+
147
+ async function fetchSnapshotSidecars() {
148
+ return import("@/lib/sidecars")
149
+ }
150
 
151
  async function fetchRemoteJson<T>(relativePath: string): Promise<T> {
152
  if (OFFLINE) {
 
432
  }
433
 
434
  export async function fetchBackendManifestStatus(): Promise<BackendManifestStatus> {
435
+ if (useViewLayerBackend()) {
436
+ const manifest = await (await fetchSnapshotSidecars()).fetchManifest()
437
+ return {
438
+ currentManifest: manifest,
439
+ latestManifest: manifest,
440
+ currentManifestSignature: manifest.generated_at,
441
+ latestManifestSignature: manifest.generated_at,
442
+ updateAvailable: false,
443
+ refreshing: false,
444
+ pendingRefreshCount: 0,
445
+ }
446
+ }
447
+
448
  const snapshot = await getManifestSnapshot()
449
  const currentManifest = getCurrentManifestFromSnapshot(snapshot)
450
  const currentManifestSignature = getManifestSignature(currentManifest)
 
886
  }
887
 
888
  export async function fetchBenchmarkMetadataMap(): Promise<Record<string, BenchmarkCard>> {
889
+ if (useViewLayerBackend()) {
890
+ return (await import("@/lib/view-data")).getBenchmarkMetadataMap()
891
+ }
892
+
893
  return fetchHFJson<Record<string, BenchmarkCard>>("benchmark-metadata.json")
894
  }
895
 
896
  export async function fetchBackendManifest(): Promise<BackendManifest> {
897
+ if (useViewLayerBackend()) {
898
+ return (await fetchSnapshotSidecars()).fetchManifest()
899
+ }
900
+
901
  return fetchHFJson<BackendManifest>("manifest.json")
902
  }
903
 
904
  export async function fetchEvalHierarchy(): Promise<EvalHierarchy> {
905
+ if (useViewLayerBackend()) {
906
+ return adaptEvalHierarchy(await (await fetchSnapshotSidecars()).fetchHierarchy())
907
+ }
908
+
909
  const raw = await fetchHFJson<EvalHierarchy>("eval-hierarchy.json")
910
  return adaptEvalHierarchy(raw)
911
  }
 
1009
  }
1010
 
1011
  export async function fetchCorpusAggregates(): Promise<CorpusAggregates | null> {
1012
+ if (useViewLayerBackend()) {
1013
+ return (await fetchSnapshotSidecars()).fetchHeadline()
1014
+ }
1015
+
1016
  return fetchHFJsonSafe<CorpusAggregates>("corpus-aggregates.json")
1017
  }
1018
 
lib/sidecars.ts ADDED
@@ -0,0 +1,59 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import "server-only"
2
+
3
+ import type {
4
+ BackendManifest,
5
+ CorpusAggregates,
6
+ EvalHierarchy,
7
+ } from "@/lib/backend-artifacts"
8
+
9
+ let cache: {
10
+ manifest?: Promise<BackendManifest>
11
+ headline?: Promise<CorpusAggregates>
12
+ hierarchy?: Promise<EvalHierarchy>
13
+ } = {}
14
+
15
+ function getSnapshotUrl() {
16
+ const snapshotUrl = process.env.SNAPSHOT_URL?.trim()
17
+ if (!snapshotUrl) {
18
+ throw new Error("DATA_BACKEND=v2 requires SNAPSHOT_URL to point at a Stage J snapshot directory")
19
+ }
20
+
21
+ return snapshotUrl.replace(/\/+$/, "")
22
+ }
23
+
24
+ function sidecarUrl(name: string) {
25
+ return `${getSnapshotUrl()}/${name}`
26
+ }
27
+
28
+ async function fetchJson<T>(name: string): Promise<T> {
29
+ const url = sidecarUrl(name)
30
+
31
+ if (url.startsWith("file://")) {
32
+ const fs = await import("fs/promises")
33
+ const text = await fs.readFile(new URL(url), "utf8")
34
+ return JSON.parse(text) as T
35
+ }
36
+
37
+ const response = await fetch(url, { next: { revalidate: 3600 } })
38
+ if (!response.ok) {
39
+ throw new Error(`Snapshot sidecar fetch failed: ${response.status} ${response.statusText} for ${url}`)
40
+ }
41
+
42
+ return (await response.json()) as T
43
+ }
44
+
45
+ export function fetchManifest(): Promise<BackendManifest> {
46
+ return (cache.manifest ??= fetchJson<BackendManifest>("manifest.json"))
47
+ }
48
+
49
+ export function fetchHeadline(): Promise<CorpusAggregates> {
50
+ return (cache.headline ??= fetchJson<CorpusAggregates>("headline.json"))
51
+ }
52
+
53
+ export function fetchHierarchy(): Promise<EvalHierarchy> {
54
+ return (cache.hierarchy ??= fetchJson<EvalHierarchy>("hierarchy.json"))
55
+ }
56
+
57
+ export function resetSidecarCacheForTests() {
58
+ cache = {}
59
+ }
lib/view-data.ts ADDED
@@ -0,0 +1,569 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import "server-only"
2
+
3
+ import { getConnection } from "@/lib/duckdb"
4
+ import { fetchHeadline } from "@/lib/sidecars"
5
+ import {
6
+ EVALUATION_CATEGORIES,
7
+ type BenchmarkCard,
8
+ type BenchmarkEvaluation,
9
+ type CategoryType,
10
+ type EvaluationCardData,
11
+ type EvaluationResult,
12
+ type GenerationConfig,
13
+ type MetricConfig,
14
+ type ModelInfo,
15
+ type ModelEvaluationSummary,
16
+ type ModelVariantSummary,
17
+ type ScoreDetails,
18
+ type SourceData,
19
+ type SourceMetadata,
20
+ } from "@/lib/benchmark-schema"
21
+ import type { DeveloperListEntry } from "@/lib/backend-artifacts"
22
+ import type {
23
+ BenchmarkEvalListItem,
24
+ BenchmarkEvalSummary,
25
+ ModelResultForBenchmark,
26
+ } from "@/lib/eval-processing"
27
+
28
+ type Row = Record<string, any>
29
+
30
+ const MODEL_CARD_COLUMNS = `
31
+ id, route_id, model_name, model_id, canonical_model_name, developer,
32
+ evaluations_count, benchmarks_count, variant_count,
33
+ categories, category_stats, latest_timestamp,
34
+ evaluator_count, evaluator_names, source_type_count, source_types,
35
+ evidence_count, missing_generation_config_count,
36
+ third_party_eval_count, independent_verification_ratio,
37
+ reproducibility_status, eval_libraries, latest_source_name,
38
+ params_billions, benchmark_names, score_summary,
39
+ reproducibility_summary, provenance_summary, comparability_summary,
40
+ top_scores, source_urls, detail_urls,
41
+ model_url, release_date, input_modalities, output_modalities,
42
+ architecture, params, inference_engine, inference_platform
43
+ `
44
+
45
+ const EVAL_LIST_COLUMNS = `
46
+ evaluation_id, evaluation_name, canonical_display_name,
47
+ composite_benchmark_key, composite_benchmark_name,
48
+ benchmark_family_key, benchmark_leaf_key, category,
49
+ metric_config, models_count, evaluator_names, source_types,
50
+ latest_source_name, third_party_ratio,
51
+ missing_generation_config_count, best_model, worst_model,
52
+ avg_score, avg_score_norm, has_card, benchmark_card,
53
+ is_aggregated, aggregate_sources, tags,
54
+ metrics_count, metric_names, instance_data, top_score,
55
+ subtasks_count, is_summary_score, summary_eval_ids,
56
+ root_metrics, subtasks, leaderboard_metrics,
57
+ reproducibility_summary, provenance_summary, comparability_summary,
58
+ source_data
59
+ `
60
+
61
+ const CELL_JOIN_COLUMNS = `
62
+ r.*,
63
+ e.evaluation_name AS eval_evaluation_name,
64
+ e.canonical_display_name AS eval_canonical_display_name,
65
+ e.composite_benchmark_key AS eval_composite_benchmark_key,
66
+ e.composite_benchmark_name AS eval_composite_benchmark_name,
67
+ e.benchmark_family_key AS eval_benchmark_family_key,
68
+ e.benchmark_leaf_key AS eval_benchmark_leaf_key,
69
+ e.category AS eval_category,
70
+ e.metric_config AS eval_metric_config,
71
+ e.source_data AS eval_source_data,
72
+ e.benchmark_card AS eval_benchmark_card,
73
+ e.tags AS eval_tags,
74
+ e.is_summary_score AS eval_is_summary_score,
75
+ e.summary_eval_ids AS eval_summary_eval_ids
76
+ `
77
+
78
+ function normalizeDuckDBValue(value: unknown): unknown {
79
+ if (typeof value === "bigint") {
80
+ return Number(value)
81
+ }
82
+
83
+ if (value instanceof Date) {
84
+ return value.toISOString()
85
+ }
86
+
87
+ if (value instanceof Map) {
88
+ return Object.fromEntries(
89
+ Array.from(value.entries()).map(([key, mapValue]) => [String(key), normalizeDuckDBValue(mapValue)])
90
+ )
91
+ }
92
+
93
+ if (Array.isArray(value)) {
94
+ return value.map(normalizeDuckDBValue)
95
+ }
96
+
97
+ if (value && typeof value === "object") {
98
+ const duckValue = value as {
99
+ constructor?: { name?: string }
100
+ entries?: unknown
101
+ items?: unknown
102
+ scale?: unknown
103
+ value?: unknown
104
+ toString?: () => string
105
+ }
106
+ const constructorName = duckValue.constructor?.name ?? ""
107
+
108
+ if (constructorName === "DuckDBStructValue" && duckValue.entries && typeof duckValue.entries === "object") {
109
+ return normalizeDuckDBValue(duckValue.entries)
110
+ }
111
+
112
+ if (
113
+ (constructorName === "DuckDBListValue" || constructorName === "DuckDBArrayValue") &&
114
+ Array.isArray(duckValue.items)
115
+ ) {
116
+ return duckValue.items.map(normalizeDuckDBValue)
117
+ }
118
+
119
+ if (constructorName === "DuckDBMapValue" && Array.isArray(duckValue.entries)) {
120
+ return Object.fromEntries(
121
+ duckValue.entries.map((entry) => {
122
+ const pair = entry as { key: unknown; value: unknown }
123
+ return [String(pair.key), normalizeDuckDBValue(pair.value)]
124
+ })
125
+ )
126
+ }
127
+
128
+ if (constructorName === "DuckDBDecimalValue" && typeof duckValue.toString === "function") {
129
+ return Number(duckValue.toString())
130
+ }
131
+
132
+ if (constructorName.startsWith("DuckDB") && typeof duckValue.toString === "function") {
133
+ return duckValue.toString()
134
+ }
135
+
136
+ return Object.fromEntries(
137
+ Object.entries(value).map(([key, objectValue]) => [key, normalizeDuckDBValue(objectValue)])
138
+ )
139
+ }
140
+
141
+ return value
142
+ }
143
+
144
+ async function readRows<T = Row>(sql: string, params: unknown[] = []): Promise<T[]> {
145
+ const connection = await getConnection()
146
+ const reader = params.length > 0
147
+ ? await connection.runAndReadAll(sql, params as any[])
148
+ : await connection.runAndReadAll(sql)
149
+ return reader.getRowObjects().map((row) => normalizeDuckDBValue(row) as T)
150
+ }
151
+
152
+ function asNumber(value: unknown, fallback = 0) {
153
+ if (typeof value === "number" && Number.isFinite(value)) return value
154
+ if (typeof value === "bigint") return Number(value)
155
+ if (typeof value === "string" && value.trim() !== "") {
156
+ const parsed = Number(value)
157
+ if (Number.isFinite(parsed)) return parsed
158
+ }
159
+ return fallback
160
+ }
161
+
162
+ function optionalNumber(value: unknown) {
163
+ if (value == null) return undefined
164
+ const parsed = asNumber(value, Number.NaN)
165
+ return Number.isFinite(parsed) ? parsed : undefined
166
+ }
167
+
168
+ function asString(value: unknown, fallback = "") {
169
+ return typeof value === "string" ? value : fallback
170
+ }
171
+
172
+ function optionalString(value: unknown) {
173
+ return typeof value === "string" && value.length > 0 ? value : undefined
174
+ }
175
+
176
+ function asArray<T>(value: unknown): T[] {
177
+ return Array.isArray(value) ? value as T[] : []
178
+ }
179
+
180
+ function normalizeCategory(value: unknown): CategoryType {
181
+ return EVALUATION_CATEGORIES.includes(value as CategoryType)
182
+ ? value as CategoryType
183
+ : "General"
184
+ }
185
+
186
+ function emptyEvaluationsByCategory(): Record<CategoryType, BenchmarkEvaluation[]> {
187
+ return EVALUATION_CATEGORIES.reduce((acc, category) => {
188
+ acc[category] = []
189
+ return acc
190
+ }, {} as Record<CategoryType, BenchmarkEvaluation[]>)
191
+ }
192
+
193
+ function sourceMetadataFromRow(row: Row): SourceMetadata {
194
+ if (row.source_metadata && typeof row.source_metadata === "object") {
195
+ return row.source_metadata as SourceMetadata
196
+ }
197
+
198
+ return {
199
+ source_type: "documentation",
200
+ source_organization_name: asString(row.latest_source_name, "Unknown"),
201
+ evaluator_relationship: "other",
202
+ }
203
+ }
204
+
205
+ function sourceDataFromRow(row: Row): BenchmarkEvaluation["source_data"] {
206
+ const sourceData = row.source_data ?? row.eval_source_data
207
+ if (sourceData) {
208
+ return sourceData as BenchmarkEvaluation["source_data"]
209
+ }
210
+
211
+ return {
212
+ dataset_name: asString(row.eval_evaluation_name ?? row.evaluation_name ?? row.benchmark_id, "Unknown dataset"),
213
+ } satisfies SourceData
214
+ }
215
+
216
+ function scoreDetailsFromRow(row: Row): ScoreDetails {
217
+ const details = row.score_details && typeof row.score_details === "object"
218
+ ? row.score_details as Partial<ScoreDetails>
219
+ : {}
220
+ const score = asNumber(details.score ?? row.score)
221
+
222
+ return {
223
+ ...details,
224
+ score,
225
+ } as ScoreDetails
226
+ }
227
+
228
+ function metricConfigFromRow(row: Row): MetricConfig {
229
+ const config = (row.metric_config ?? row.eval_metric_config ?? {}) as Partial<MetricConfig>
230
+ const scoreType = config.score_type === "binary" || config.score_type === "discrete"
231
+ ? config.score_type
232
+ : "continuous"
233
+
234
+ return {
235
+ evaluation_description: asString(
236
+ config.evaluation_description ??
237
+ row.metric_description ??
238
+ row.metric_display_name ??
239
+ row.eval_evaluation_name ??
240
+ row.evaluation_name,
241
+ ""
242
+ ),
243
+ lower_is_better: Boolean(row.lower_is_better ?? config.lower_is_better ?? false),
244
+ score_type: scoreType,
245
+ min_score: optionalNumber(config.min_score ?? row.min_score),
246
+ max_score: optionalNumber(config.max_score ?? row.max_score),
247
+ unit: optionalString(row.metric_unit ?? config.unit),
248
+ }
249
+ }
250
+
251
+ function modelInfoFromModelRow(row: Row): ModelInfo {
252
+ return {
253
+ name: asString(row.model_name ?? row.model_family_name ?? row.model_id, "Unknown model"),
254
+ id: asString(row.model_id ?? row.id ?? row.route_id, "unknown-model"),
255
+ developer: optionalString(row.developer),
256
+ inference_platform: optionalString(row.inference_platform),
257
+ inference_engine: optionalString(row.inference_engine),
258
+ architecture: optionalString(row.architecture),
259
+ parameter_count: optionalString(row.params),
260
+ release_date: optionalString(row.release_date),
261
+ model_url: optionalString(row.model_url),
262
+ additional_details: {
263
+ params_billions: row.params_billions,
264
+ },
265
+ modalities: {
266
+ input: asArray<string>(row.input_modalities),
267
+ output: asArray<string>(row.output_modalities),
268
+ },
269
+ }
270
+ }
271
+
272
+ function resultFromCell(row: Row): EvaluationResult {
273
+ const scoreDetails = scoreDetailsFromRow(row)
274
+ const generationConfig = row.generation_config as GenerationConfig | undefined
275
+ const annotations = row.evalcards_annotations
276
+
277
+ return {
278
+ evaluation_name: asString(row.metric_display_name ?? row.eval_evaluation_name ?? row.metric_id, "Score"),
279
+ display_name: optionalString(row.metric_display_name),
280
+ canonical_display_name: optionalString(row.metric_display_name),
281
+ metric_summary_id: optionalString(row.metric_summary_id),
282
+ metric_key: optionalString(row.metric_id),
283
+ evaluation_timestamp: asString(row.evaluation_timestamp, ""),
284
+ source_data: sourceDataFromRow(row),
285
+ metric_config: metricConfigFromRow(row),
286
+ score_details: scoreDetails,
287
+ generation_config: generationConfig,
288
+ detailed_evaluation_results_url: optionalString(row.instance_file_path),
289
+ evalcards: annotations ? { annotations } : undefined,
290
+ }
291
+ }
292
+
293
+ function reshapeCellToModelResult(row: Row): ModelResultForBenchmark {
294
+ const scoreDetails = scoreDetailsFromRow(row)
295
+
296
+ return {
297
+ model_info: (row.model_info ?? modelInfoFromModelRow(row)) as ModelInfo,
298
+ model_route_id: optionalString(row.model_route_id),
299
+ score: scoreDetails.score,
300
+ score_details: scoreDetails,
301
+ evaluation_timestamp: asString(row.evaluation_timestamp, ""),
302
+ source_metadata: sourceMetadataFromRow(row),
303
+ source_data: sourceDataFromRow(row),
304
+ source_record_url: optionalString(row.source_record_url),
305
+ aggregate_components: asArray<NonNullable<ModelResultForBenchmark["aggregate_components"]>[number]>(
306
+ row.aggregate_components
307
+ ),
308
+ result: resultFromCell(row),
309
+ }
310
+ }
311
+
312
+ function reshapeCellToBenchmarkEvaluation(row: Row): BenchmarkEvaluation {
313
+ const result = resultFromCell(row)
314
+ const modelInfo = (row.model_info ?? modelInfoFromModelRow(row)) as ModelInfo
315
+
316
+ return {
317
+ schema_version: "1.0",
318
+ eval_summary_id: optionalString(row.evaluation_id),
319
+ evaluation_id: asString(row.evaluation_id ?? row.benchmark_id, "unknown-evaluation"),
320
+ retrieved_timestamp: asString(row.evaluation_timestamp, ""),
321
+ benchmark: optionalString(row.eval_evaluation_name ?? row.benchmark_id),
322
+ display_name: optionalString(row.eval_evaluation_name),
323
+ canonical_display_name: optionalString(row.eval_canonical_display_name),
324
+ category: normalizeCategory(row.eval_category ?? row.category),
325
+ benchmark_family_key: optionalString(row.eval_benchmark_family_key),
326
+ benchmark_family_name: optionalString(row.eval_composite_benchmark_name),
327
+ benchmark_parent_key: optionalString(row.eval_composite_benchmark_key),
328
+ benchmark_parent_name: optionalString(row.eval_composite_benchmark_name),
329
+ benchmark_leaf_key: optionalString(row.eval_benchmark_leaf_key),
330
+ benchmark_leaf_name: optionalString(row.eval_evaluation_name),
331
+ is_summary_score: Boolean(row.eval_is_summary_score ?? row.is_summary_score),
332
+ source_data: sourceDataFromRow(row),
333
+ source_metadata: sourceMetadataFromRow(row),
334
+ eval_library: row.eval_library,
335
+ model_info: modelInfo,
336
+ generation_config: row.generation_config,
337
+ evaluation_results: [result],
338
+ }
339
+ }
340
+
341
+ function modelSummaryFromRows(modelRow: Row, cellRows: Row[]): ModelEvaluationSummary {
342
+ const evaluationsByCategory = emptyEvaluationsByCategory()
343
+ for (const cellRow of cellRows) {
344
+ const evaluation = reshapeCellToBenchmarkEvaluation(cellRow)
345
+ const category = normalizeCategory(evaluation.category)
346
+ evaluationsByCategory[category].push(evaluation)
347
+ }
348
+
349
+ const categoriesCovered = asArray<CategoryType>(modelRow.categories).filter((category) =>
350
+ EVALUATION_CATEGORIES.includes(category)
351
+ )
352
+ const modelInfo = (modelRow.model_info ?? modelInfoFromModelRow(modelRow)) as ModelInfo
353
+ const totalEvaluations = asNumber(modelRow.total_evaluations ?? modelRow.evaluations_count)
354
+ const lastUpdated = asString(modelRow.last_updated ?? modelRow.latest_timestamp, "")
355
+ const rawModelIds = asArray<string>(modelRow.raw_model_ids)
356
+
357
+ const core = {
358
+ model_info: modelInfo,
359
+ evaluations_by_category: evaluationsByCategory,
360
+ total_evaluations: totalEvaluations,
361
+ last_updated: lastUpdated,
362
+ categories_covered: categoriesCovered.length > 0
363
+ ? categoriesCovered
364
+ : EVALUATION_CATEGORIES.filter((category) => evaluationsByCategory[category].length > 0),
365
+ reproducibility_summary: modelRow.reproducibility_summary,
366
+ provenance_summary: modelRow.provenance_summary,
367
+ comparability_summary: modelRow.comparability_summary,
368
+ }
369
+
370
+ const variants = asArray<Row>(modelRow.variants).map((variant, index) => ({
371
+ ...core,
372
+ ...variant,
373
+ variant_id: asString(variant.variant_id ?? variant.variant_key, `variant-${index}`),
374
+ variant_key: asString(variant.variant_key, `variant-${index}`),
375
+ variant_label: asString(variant.variant_label ?? variant.variant_display_name, "Default"),
376
+ variant_display_name: asString(variant.variant_display_name ?? variant.variant_label ?? modelRow.model_name, modelRow.model_name),
377
+ raw_model_ids: asArray<string>(variant.raw_model_ids),
378
+ family_id: asString(variant.family_id ?? modelRow.model_family_id, modelRow.model_family_id),
379
+ family_name: asString(variant.family_name ?? modelRow.model_family_name, modelRow.model_family_name),
380
+ total_evaluations: asNumber(variant.total_evaluations ?? totalEvaluations),
381
+ last_updated: asString(variant.last_updated ?? lastUpdated, lastUpdated),
382
+ categories_covered: asArray<CategoryType>(variant.categories_covered).length > 0
383
+ ? asArray<CategoryType>(variant.categories_covered)
384
+ : core.categories_covered,
385
+ model_info: {
386
+ ...modelInfo,
387
+ name: asString(variant.variant_display_name ?? variant.variant_label ?? modelInfo.name, modelInfo.name),
388
+ },
389
+ })) as ModelVariantSummary[]
390
+
391
+ return {
392
+ ...core,
393
+ model_family_id: asString(modelRow.model_family_id ?? modelRow.model_id, modelRow.model_id),
394
+ model_route_id: asString(modelRow.model_route_id ?? modelRow.route_id, modelRow.route_id),
395
+ model_family_name: asString(modelRow.model_family_name ?? modelRow.model_name, modelRow.model_name),
396
+ raw_model_ids: rawModelIds.length > 0 ? rawModelIds : [asString(modelRow.model_id, "")].filter(Boolean),
397
+ variants,
398
+ }
399
+ }
400
+
401
+ async function getModelEvaluationRows(modelId: string): Promise<Row[]> {
402
+ return readRows<Row>(
403
+ `SELECT ${CELL_JOIN_COLUMNS}
404
+ FROM eval_results_view r
405
+ LEFT JOIN evals_view e ON r.evaluation_id = e.evaluation_id
406
+ WHERE r.model_id = ?
407
+ AND r.score IS NOT NULL
408
+ ORDER BY r.category, r.percentile DESC NULLS LAST`,
409
+ [modelId]
410
+ )
411
+ }
412
+
413
+ export async function getModelCards(): Promise<EvaluationCardData[]> {
414
+ return readRows<EvaluationCardData>(
415
+ `SELECT ${MODEL_CARD_COLUMNS}
416
+ FROM models_view
417
+ ORDER BY latest_timestamp DESC NULLS LAST`
418
+ )
419
+ }
420
+
421
+ export async function getModelCardsLite(): Promise<EvaluationCardData[]> {
422
+ return readRows<EvaluationCardData>(
423
+ `SELECT ${MODEL_CARD_COLUMNS}
424
+ FROM models_view
425
+ ORDER BY benchmarks_count DESC NULLS LAST, evaluations_count DESC NULLS LAST, model_name ASC`
426
+ )
427
+ }
428
+
429
+ export async function getEvalListData(): Promise<{
430
+ evals: BenchmarkEvalListItem[]
431
+ totalModels: number
432
+ }> {
433
+ const [evals, countRows] = await Promise.all([
434
+ readRows<BenchmarkEvalListItem>(
435
+ `SELECT ${EVAL_LIST_COLUMNS}
436
+ FROM evals_view
437
+ ORDER BY evaluation_name ASC`
438
+ ),
439
+ readRows<{ n: number }>("SELECT COUNT(*) AS n FROM models_view"),
440
+ ])
441
+
442
+ return {
443
+ evals,
444
+ totalModels: asNumber(countRows[0]?.n),
445
+ }
446
+ }
447
+
448
+ export async function getEvalListLiteData(): Promise<{
449
+ evals: BenchmarkEvalListItem[]
450
+ totalModels: number
451
+ }> {
452
+ return getEvalListData()
453
+ }
454
+
455
+ export async function getEvalList() {
456
+ const { evals } = await getEvalListData()
457
+ return evals
458
+ }
459
+
460
+ export async function getDashboardData() {
461
+ const [models, evals] = await Promise.all([
462
+ getModelCards(),
463
+ getEvalList(),
464
+ ])
465
+ return { models, evals }
466
+ }
467
+
468
+ export async function getModelSummaryById(routeId: string): Promise<ModelEvaluationSummary | null> {
469
+ const rows = await readRows<Row>(
470
+ `SELECT *
471
+ FROM models_view
472
+ WHERE route_id = ? OR model_route_id = ? OR model_family_id = ? OR model_id = ?
473
+ LIMIT 1`,
474
+ [routeId, routeId, routeId, routeId]
475
+ )
476
+ const modelRow = rows[0]
477
+ if (!modelRow) return null
478
+
479
+ const cellRows = await getModelEvaluationRows(asString(modelRow.model_id, routeId))
480
+ return modelSummaryFromRows(modelRow, cellRows)
481
+ }
482
+
483
+ export async function getEvalSummaryById(evalId: string): Promise<BenchmarkEvalSummary | null> {
484
+ const evalRows = await readRows<Row>(
485
+ "SELECT * FROM evals_view WHERE evaluation_id = ? LIMIT 1",
486
+ [evalId]
487
+ )
488
+ const evalRow = evalRows[0]
489
+ if (!evalRow) return null
490
+
491
+ let cellRows = await readRows<Row>(
492
+ `SELECT ${CELL_JOIN_COLUMNS}
493
+ FROM eval_results_view r
494
+ LEFT JOIN evals_view e ON r.evaluation_id = e.evaluation_id
495
+ WHERE r.evaluation_id = ?
496
+ AND r.metric_id = (SELECT primary_metric_id FROM evals_view WHERE evaluation_id = ?)
497
+ AND r.score IS NOT NULL
498
+ ORDER BY r.position ASC NULLS LAST`,
499
+ [evalId, evalId]
500
+ )
501
+
502
+ if (cellRows.length === 0) {
503
+ cellRows = await readRows<Row>(
504
+ `SELECT ${CELL_JOIN_COLUMNS}
505
+ FROM eval_results_view r
506
+ LEFT JOIN evals_view e ON r.evaluation_id = e.evaluation_id
507
+ WHERE r.evaluation_id = ?
508
+ AND r.score IS NOT NULL
509
+ ORDER BY r.position ASC NULLS LAST`,
510
+ [evalId]
511
+ )
512
+ }
513
+
514
+ return {
515
+ ...evalRow,
516
+ model_results: cellRows.map(reshapeCellToModelResult),
517
+ } as BenchmarkEvalSummary
518
+ }
519
+
520
+ export async function getDeveloperList(): Promise<DeveloperListEntry[]> {
521
+ const headline = await fetchHeadline()
522
+ return [...(headline.developers ?? [])].sort((a, b) => a.developer.localeCompare(b.developer))
523
+ }
524
+
525
+ export async function getDeveloperSummaryById(routeId: string) {
526
+ const developers = await getDeveloperList()
527
+ const developer = developers.find((entry) => entry.route_id === routeId)
528
+ if (!developer) return null
529
+
530
+ const models = await readRows<EvaluationCardData>(
531
+ `SELECT ${MODEL_CARD_COLUMNS}
532
+ FROM models_view
533
+ WHERE developer = ?
534
+ ORDER BY benchmarks_count DESC NULLS LAST, evaluations_count DESC NULLS LAST, model_name ASC`,
535
+ [developer.developer]
536
+ )
537
+
538
+ return {
539
+ ...developer,
540
+ models,
541
+ }
542
+ }
543
+
544
+ export async function getBenchmarkMetadataMap(): Promise<Record<string, BenchmarkCard>> {
545
+ const rows = await readRows<Row>(
546
+ `SELECT evaluation_id, evaluation_name, composite_benchmark_key, benchmark_card
547
+ FROM evals_view
548
+ WHERE benchmark_card IS NOT NULL`
549
+ )
550
+ const result: Record<string, BenchmarkCard> = {}
551
+
552
+ for (const row of rows) {
553
+ const card = row.benchmark_card as BenchmarkCard | null | undefined
554
+ if (!card) continue
555
+
556
+ const keys = [
557
+ row.evaluation_id,
558
+ row.evaluation_name,
559
+ row.composite_benchmark_key,
560
+ card.benchmark_details?.name,
561
+ ].filter((key): key is string => typeof key === "string" && key.length > 0)
562
+
563
+ for (const key of keys) {
564
+ result[key] = card
565
+ }
566
+ }
567
+
568
+ return result
569
+ }
scripts/cache-hf-data.mjs CHANGED
@@ -18,6 +18,13 @@ import { promisify } from "util"
18
  const root = path.resolve(new URL(import.meta.url).pathname, "..", "..")
19
  const cacheDir = path.join(root, ".cache", "hf-data")
20
  const publicDir = path.join(root, "public")
 
 
 
 
 
 
 
21
  const HF_DATASET_REPO = process.env.HF_DATASET_REPO?.trim()
22
  || "https://huggingface.co/datasets/evaleval/card_backend"
23
  const HF_RESOLVE_BASE = `${HF_DATASET_REPO}/resolve/main`
 
18
  const root = path.resolve(new URL(import.meta.url).pathname, "..", "..")
19
  const cacheDir = path.join(root, ".cache", "hf-data")
20
  const publicDir = path.join(root, "public")
21
+ const dataBackend = process.env.DATA_BACKEND?.trim().toLowerCase()
22
+ if (dataBackend === "v2" || dataBackend === "stage-j") {
23
+ await fs.mkdir(cacheDir, { recursive: true })
24
+ console.log("[cache-hf-data] DATA_BACKEND=v2: skipping legacy HF cache; runtime reads SNAPSHOT_URL")
25
+ process.exit(0)
26
+ }
27
+
28
  const HF_DATASET_REPO = process.env.HF_DATASET_REPO?.trim()
29
  || "https://huggingface.co/datasets/evaleval/card_backend"
30
  const HF_RESOLVE_BASE = `${HF_DATASET_REPO}/resolve/main`
tests/duckdb-data.test.ts CHANGED
@@ -12,27 +12,13 @@ function sqlString(value: string) {
12
  }
13
 
14
  async function writeParquetPayload(outputDir: string, fileName: string, payloads: unknown[]) {
15
- const parquetDir = path.join(outputDir, "experimental", "parquet")
16
  await mkdir(parquetDir, { recursive: true })
17
 
18
  const selects = payloads
19
- .map((payload, index) => {
20
- const record = payload as Record<string, unknown>
21
  const payloadJson = JSON.stringify(payload)
22
- return [
23
- `SELECT 'model_card_lite' AS record_type`,
24
- `${sqlString(String(record.model_route_id ?? index))} AS model_route_id`,
25
- `${sqlString(String(record.model_family_id ?? ""))} AS model_family_id`,
26
- `${sqlString(String(record.developer ?? ""))} AS developer`,
27
- `NULL AS eval_summary_id`,
28
- `NULL AS developer_route_id`,
29
- `NULL AS category`,
30
- `NULL AS benchmark_family_key`,
31
- `${Number(record.benchmark_family_count ?? 0)} AS models_count`,
32
- `${Number(record.total_evaluations ?? 0)} AS total_evaluations`,
33
- `${sqlString(String(record.last_updated ?? ""))} AS last_updated`,
34
- `${sqlString(payloadJson)} AS payload_json`,
35
- ].join(", ")
36
  })
37
  .join(" UNION ALL ")
38
 
@@ -49,22 +35,36 @@ describe("DuckDB local data backend", () => {
49
  process.env.LOCAL_PIPELINE_OUTPUT = outputDir
50
  await writeParquetPayload(outputDir, "model_cards_lite.parquet", [
51
  {
52
- model_family_id: "openai/gpt-5",
53
- model_route_id: "openai__gpt-5",
54
- model_family_name: "GPT 5",
55
- developer: "openai",
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
56
  params_billions: 100,
57
- total_evaluations: 3,
58
- benchmark_count: 2,
59
- benchmark_family_count: 2,
60
- categories_covered: ["reasoning"],
61
- last_updated: "2026-01-01T00:00:00Z",
62
- variants: [],
63
  score_summary: { count: 1, min: 0.7, max: 0.9, average: 0.8 },
64
  benchmark_names: ["mmlu"],
65
- top_benchmark_scores: [
66
  { benchmark: "mmlu", score: 0.9, metric: "accuracy" },
67
  ],
 
 
68
  },
69
  ])
70
 
@@ -93,7 +93,7 @@ describe("DuckDB local data backend", () => {
93
  try {
94
  process.env.LOCAL_PIPELINE_OUTPUT = outputDir
95
  await expect(getModelCardsLiteFromDuckDB()).rejects.toThrow(
96
- /EXPORT_EXPERIMENTAL_PARQUET=1/
97
  )
98
  } finally {
99
  if (previousOutput == null) {
 
12
  }
13
 
14
  async function writeParquetPayload(outputDir: string, fileName: string, payloads: unknown[]) {
15
+ const parquetDir = path.join(outputDir, "duckdb", "v1")
16
  await mkdir(parquetDir, { recursive: true })
17
 
18
  const selects = payloads
19
+ .map((payload) => {
 
20
  const payloadJson = JSON.stringify(payload)
21
+ return `SELECT ${sqlString(payloadJson)} AS payload_json`
 
 
 
 
 
 
 
 
 
 
 
 
 
22
  })
23
  .join(" UNION ALL ")
24
 
 
35
  process.env.LOCAL_PIPELINE_OUTPUT = outputDir
36
  await writeParquetPayload(outputDir, "model_cards_lite.parquet", [
37
  {
38
+ id: "openai/gpt-5",
39
+ route_id: "openai__gpt-5",
40
+ model_name: "GPT 5",
41
+ model_id: "openai/gpt-5",
42
+ canonical_model_name: "GPT 5",
43
+ developer: "OpenAI",
44
+ evaluations_count: 3,
45
+ benchmarks_count: 2,
46
+ variant_count: 1,
47
+ categories: ["Reasoning"],
48
+ category_stats: { General: 0, Reasoning: 2, Agentic: 0, Safety: 0, Knowledge: 0 },
49
+ latest_timestamp: "2026-01-01T00:00:00Z",
50
+ evaluator_count: 1,
51
+ evaluator_names: ["OpenAI"],
52
+ source_type_count: 1,
53
+ source_types: ["documentation"],
54
+ evidence_count: 3,
55
+ missing_generation_config_count: 0,
56
+ third_party_eval_count: 0,
57
+ independent_verification_ratio: 0,
58
+ reproducibility_status: "complete",
59
+ eval_libraries: [],
60
  params_billions: 100,
 
 
 
 
 
 
61
  score_summary: { count: 1, min: 0.7, max: 0.9, average: 0.8 },
62
  benchmark_names: ["mmlu"],
63
+ top_scores: [
64
  { benchmark: "mmlu", score: 0.9, metric: "accuracy" },
65
  ],
66
+ source_urls: [],
67
+ detail_urls: [],
68
  },
69
  ])
70
 
 
93
  try {
94
  process.env.LOCAL_PIPELINE_OUTPUT = outputDir
95
  await expect(getModelCardsLiteFromDuckDB()).rejects.toThrow(
96
+ /duckdb\/v1\/model_cards_lite\.parquet/
97
  )
98
  } finally {
99
  if (previousOutput == null) {
tests/view-data.test.ts ADDED
@@ -0,0 +1,464 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import { mkdir, mkdtemp, rm, writeFile } from "fs/promises"
2
+ import os from "os"
3
+ import path from "path"
4
+
5
+ import { DuckDBConnection } from "@duckdb/node-api"
6
+ import { describe, expect, it } from "vitest"
7
+
8
+ function sqlString(value: string) {
9
+ return `'${value.replace(/'/g, "''")}'`
10
+ }
11
+
12
+ async function copyParquet(connection: DuckDBConnection, sql: string, outputPath: string) {
13
+ await connection.run(`COPY (${sql}) TO ${sqlString(outputPath)} (FORMAT parquet)`)
14
+ }
15
+
16
+ async function writeSyntheticStageJSnapshot(snapshotDir: string) {
17
+ await mkdir(snapshotDir, { recursive: true })
18
+ const connection = await DuckDBConnection.create()
19
+
20
+ await copyParquet(
21
+ connection,
22
+ `
23
+ SELECT
24
+ TIMESTAMP '2026-05-03 00:00:00' AS snapshot_id,
25
+ 'openai/gpt-5' AS model_id,
26
+ 'openai/gpt-5' AS id,
27
+ 'openai%2Fgpt-5' AS route_id,
28
+ 'openai%2Fgpt-5' AS model_route_id,
29
+ 'openai/gpt-5' AS model_family_id,
30
+ 'GPT 5' AS model_name,
31
+ 'GPT 5' AS canonical_model_name,
32
+ 'GPT 5' AS model_family_name,
33
+ 'OpenAI' AS developer,
34
+ DATE '2026-01-01' AS release_date,
35
+ 'https://example.test/model' AS model_url,
36
+ 'transformer' AS architecture,
37
+ '100B' AS params,
38
+ 100.0 AS params_billions,
39
+ ['text']::VARCHAR[] AS input_modalities,
40
+ ['text']::VARCHAR[] AS output_modalities,
41
+ 'engine' AS inference_engine,
42
+ 'platform' AS inference_platform,
43
+ 1::BIGINT AS evaluations_count,
44
+ 1::BIGINT AS benchmarks_count,
45
+ 1::INTEGER AS variant_count,
46
+ 1::BIGINT AS evaluator_count,
47
+ ['OpenAI']::VARCHAR[] AS evaluator_names,
48
+ 1::INTEGER AS source_type_count,
49
+ ['documentation']::VARCHAR[] AS source_types,
50
+ 0::BIGINT AS third_party_eval_count,
51
+ 0.0 AS independent_verification_ratio,
52
+ 1::BIGINT AS evidence_count,
53
+ 0::INTEGER AS missing_generation_config_count,
54
+ TIMESTAMP '2026-05-03 00:00:00' AS latest_timestamp,
55
+ 'OpenAI' AS latest_source_name,
56
+ ['MMLU']::VARCHAR[] AS benchmark_names,
57
+ ['Reasoning']::VARCHAR[] AS categories,
58
+ struct_pack("General" := 0, "Reasoning" := 1, "Agentic" := 0, "Safety" := 0, "Knowledge" := 0) AS category_stats,
59
+ 'complete' AS reproducibility_status,
60
+ struct_pack(results_total := 1, has_reproducibility_gap_count := 0, populated_ratio_avg := 1.0) AS reproducibility_summary,
61
+ struct_pack(
62
+ total_results := 1,
63
+ total_groups := 1,
64
+ multi_source_groups := 0,
65
+ first_party_only_groups := 1,
66
+ source_type_distribution := struct_pack(first_party := 1, third_party := 0, collaborative := 0, unspecified := 0)
67
+ ) AS provenance_summary,
68
+ struct_pack(
69
+ total_groups := 1,
70
+ groups_with_variant_check := 0,
71
+ groups_with_cross_party_check := 0,
72
+ variant_divergent_count := 0,
73
+ cross_party_divergent_count := 0
74
+ ) AS comparability_summary,
75
+ [struct_pack(name := 'openai-evals', version := '1.0', fork := NULL::VARCHAR)] AS eval_libraries,
76
+ struct_pack(count := 1, min := 0.8, max := 0.8, average := 0.8) AS score_summary,
77
+ [struct_pack(benchmark := 'MMLU', benchmarkKey := 'mmlu', score := 0.8, metric := 'accuracy')] AS top_scores,
78
+ ['https://example.test/source']::VARCHAR[] AS source_urls,
79
+ []::VARCHAR[] AS detail_urls,
80
+ [struct_pack(
81
+ variant_id := 'default',
82
+ variant_key := 'default',
83
+ variant_label := 'Default',
84
+ variant_display_name := 'GPT 5',
85
+ raw_model_ids := ['openai/gpt-5']::VARCHAR[],
86
+ family_id := 'openai/gpt-5',
87
+ family_name := 'GPT 5',
88
+ version_date := NULL::VARCHAR,
89
+ version_qualifier := NULL::VARCHAR,
90
+ total_evaluations := 1,
91
+ last_updated := TIMESTAMP '2026-05-03 00:00:00',
92
+ categories_covered := ['Reasoning']::VARCHAR[]
93
+ )] AS variants,
94
+ ['openai/gpt-5']::VARCHAR[] AS raw_model_ids
95
+ `,
96
+ path.join(snapshotDir, "models_view.parquet")
97
+ )
98
+
99
+ await copyParquet(
100
+ connection,
101
+ `
102
+ SELECT
103
+ TIMESTAMP '2026-05-03 00:00:00' AS snapshot_id,
104
+ 'mmlu' AS evaluation_id,
105
+ 'mmlu' AS benchmark_id,
106
+ 'accuracy' AS primary_metric_id,
107
+ 'MMLU' AS evaluation_name,
108
+ 'MMLU' AS canonical_display_name,
109
+ 'mmlu' AS composite_benchmark_key,
110
+ 'MMLU' AS composite_benchmark_name,
111
+ 'mmlu' AS benchmark_family_key,
112
+ 'mmlu' AS benchmark_leaf_key,
113
+ 'Reasoning' AS category,
114
+ struct_pack(
115
+ evaluation_description := 'Accuracy on MMLU',
116
+ lower_is_better := false,
117
+ score_type := 'continuous',
118
+ min_score := 0.0,
119
+ max_score := 1.0,
120
+ unit := 'proportion'
121
+ ) AS metric_config,
122
+ 1::BIGINT AS models_count,
123
+ ['OpenAI']::VARCHAR[] AS evaluator_names,
124
+ ['documentation']::VARCHAR[] AS source_types,
125
+ 'OpenAI' AS latest_source_name,
126
+ 0.0 AS third_party_ratio,
127
+ 0::INTEGER AS missing_generation_config_count,
128
+ struct_pack(name := 'GPT 5', score := 0.8) AS best_model,
129
+ struct_pack(name := 'GPT 5', score := 0.8) AS worst_model,
130
+ 0.8 AS avg_score,
131
+ 0.8 AS avg_score_norm,
132
+ 0.8 AS top_score,
133
+ false AS has_card,
134
+ NULL AS benchmark_card,
135
+ false AS is_aggregated,
136
+ [] AS aggregate_sources,
137
+ false AS is_summary_score,
138
+ []::VARCHAR[] AS summary_eval_ids,
139
+ struct_pack(domains := ['knowledge']::VARCHAR[], languages := ['en']::VARCHAR[], tasks := ['qa']::VARCHAR[]) AS tags,
140
+ struct_pack(
141
+ dataset_name := 'MMLU',
142
+ source_type := 'documentation',
143
+ hf_repo := NULL::VARCHAR,
144
+ hf_split := NULL::VARCHAR,
145
+ samples_number := 10,
146
+ url := ['https://example.test/mmlu']::VARCHAR[],
147
+ dataset_url := 'https://example.test/mmlu',
148
+ dataset_version := 'v1'
149
+ ) AS source_data,
150
+ struct_pack(results_total := 1, has_reproducibility_gap_count := 0, populated_ratio_avg := 1.0) AS reproducibility_summary,
151
+ struct_pack(
152
+ total_results := 1,
153
+ total_groups := 1,
154
+ multi_source_groups := 0,
155
+ first_party_only_groups := 1,
156
+ source_type_distribution := struct_pack(first_party := 1, third_party := 0, collaborative := 0, unspecified := 0)
157
+ ) AS provenance_summary,
158
+ struct_pack(
159
+ total_groups := 1,
160
+ groups_with_variant_check := 0,
161
+ groups_with_cross_party_check := 0,
162
+ variant_divergent_count := 0,
163
+ cross_party_divergent_count := 0
164
+ ) AS comparability_summary,
165
+ struct_pack(available := false, url_count := 0::BIGINT, sample_urls := []::VARCHAR[], models_with_loaded_instances := 0) AS instance_data,
166
+ 1::INTEGER AS metrics_count,
167
+ ['Accuracy']::VARCHAR[] AS metric_names,
168
+ [struct_pack(
169
+ column_key := 'root:accuracy',
170
+ metric_summary_id := 'mmlu%3Aaccuracy',
171
+ metric_id := 'accuracy',
172
+ metric_name := 'accuracy',
173
+ display_name := 'Accuracy',
174
+ canonical_display_name := 'Accuracy',
175
+ lower_is_better := false,
176
+ unit := 'proportion',
177
+ scope := 'root',
178
+ subtask_key := NULL::VARCHAR,
179
+ subtask_name := NULL::VARCHAR
180
+ )] AS leaderboard_metrics,
181
+ [] AS leaderboard_rows,
182
+ [struct_pack(
183
+ metric_summary_id := 'mmlu%3Aaccuracy',
184
+ metric_name := 'accuracy',
185
+ display_name := 'Accuracy',
186
+ canonical_display_name := 'Accuracy',
187
+ metric_key := 'accuracy',
188
+ lower_is_better := false,
189
+ models_count := 1,
190
+ top_score := 0.8,
191
+ unit := 'proportion'
192
+ )] AS root_metrics,
193
+ [] AS subtasks,
194
+ 0::INTEGER AS subtasks_count
195
+ `,
196
+ path.join(snapshotDir, "evals_view.parquet")
197
+ )
198
+
199
+ await copyParquet(
200
+ connection,
201
+ `
202
+ SELECT
203
+ TIMESTAMP '2026-05-03 00:00:00' AS snapshot_id,
204
+ 'mmlu' AS evaluation_id,
205
+ 'mmlu%3Aaccuracy' AS metric_summary_id,
206
+ 'mmlu' AS benchmark_id,
207
+ 'accuracy' AS metric_id,
208
+ 'openai/gpt-5' AS model_id,
209
+ 'openai%2Fgpt-5' AS model_route_id,
210
+ struct_pack(
211
+ name := 'GPT 5',
212
+ id := 'openai/gpt-5',
213
+ developer := 'OpenAI',
214
+ inference_platform := 'platform',
215
+ inference_engine := 'engine',
216
+ model_version := NULL::VARCHAR,
217
+ architecture := 'transformer',
218
+ parameter_count := '100B',
219
+ release_date := '2026-01-01',
220
+ model_url := 'https://example.test/model',
221
+ modalities := struct_pack(input := ['text']::VARCHAR[], output := ['text']::VARCHAR[])
222
+ ) AS model_info,
223
+ 'Accuracy' AS metric_display_name,
224
+ 'proportion' AS metric_unit,
225
+ false AS lower_is_better,
226
+ 'Reasoning' AS category,
227
+ 0.8 AS score,
228
+ struct_pack(
229
+ score := 0.8,
230
+ standard_error := 0.01,
231
+ sample_size := 10,
232
+ confidence_interval := struct_pack(lower := 0.7, upper := 0.9, confidence_level := 0.95)
233
+ ) AS score_details,
234
+ 1::INTEGER AS fact_row_count,
235
+ 1::INTEGER AS position,
236
+ 1::INTEGER AS total,
237
+ 1.0 AS percentile,
238
+ TIMESTAMP '2026-05-03 00:00:00' AS evaluation_timestamp,
239
+ struct_pack(
240
+ source_name := 'OpenAI report',
241
+ source_type := 'documentation',
242
+ source_organization_name := 'OpenAI',
243
+ source_organization_url := 'https://example.test',
244
+ evaluator_relationship := 'first_party',
245
+ source_url := 'https://example.test/report',
246
+ publication_date := DATE '2026-05-03'
247
+ ) AS source_metadata,
248
+ struct_pack(
249
+ dataset_name := 'MMLU',
250
+ source_type := 'documentation',
251
+ hf_repo := NULL::VARCHAR,
252
+ hf_split := NULL::VARCHAR,
253
+ samples_number := 10,
254
+ url := ['https://example.test/mmlu']::VARCHAR[],
255
+ dataset_url := 'https://example.test/mmlu',
256
+ dataset_version := 'v1'
257
+ ) AS source_data,
258
+ 'https://example.test/record.json' AS source_record_url,
259
+ struct_pack(name := 'openai-evals', version := '1.0', fork := NULL::VARCHAR) AS eval_library,
260
+ ['first_party']::VARCHAR[] AS evaluator_relationships,
261
+ true AS has_first_party,
262
+ false AS has_third_party,
263
+ 'self' AS coverage_cell,
264
+ ['OpenAI']::VARCHAR[] AS reporting_orgs,
265
+ map(['OpenAI'], [0.8]) AS scores_by_organization,
266
+ false AS is_summary_score,
267
+ NULL::VARCHAR AS summary_score_for,
268
+ [] AS aggregate_components,
269
+ false AS has_reproducibility_gap,
270
+ 1.0 AS completeness_score,
271
+ false AS is_multi_source,
272
+ true AS first_party_only,
273
+ false AS has_variant_divergence,
274
+ false AS has_cross_party_divergence,
275
+ NULL AS evalcards_annotations,
276
+ NULL::VARCHAR AS instance_file_path,
277
+ NULL::VARCHAR AS instance_file_format,
278
+ 0::INTEGER AS instance_rows
279
+ `,
280
+ path.join(snapshotDir, "eval_results_view.parquet")
281
+ )
282
+
283
+ await writeFile(
284
+ path.join(snapshotDir, "manifest.json"),
285
+ JSON.stringify({
286
+ generated_at: "2026-05-03T00:00:00Z",
287
+ config_version: 2,
288
+ skipped_configs: [],
289
+ model_count: 1,
290
+ eval_count: 1,
291
+ metric_eval_count: 1,
292
+ source_config_count: 1,
293
+ skipped_config_count: 0,
294
+ summary_artifacts: {
295
+ corpus_aggregates: "headline.json",
296
+ eval_hierarchy: "hierarchy.json",
297
+ },
298
+ })
299
+ )
300
+
301
+ const reproducibilityBlock = {
302
+ total_triples: 1,
303
+ triples_with_reproducibility_gap: 0,
304
+ reproducibility_gap_rate: 0,
305
+ agentic_triples: 0,
306
+ per_field_missingness: {
307
+ temperature: {
308
+ missing_count: 0,
309
+ missing_rate: 0,
310
+ denominator: "all_triples",
311
+ denominator_count: 1,
312
+ },
313
+ },
314
+ }
315
+ const completenessBlock = {
316
+ total_triples: 1,
317
+ completeness_avg: 0.75,
318
+ completeness_min: 0.75,
319
+ completeness_max: 0.75,
320
+ }
321
+ const provenanceBlock = {
322
+ total_triples: 1,
323
+ multi_source_triples: 0,
324
+ first_party_only_triples: 1,
325
+ source_type_distribution: {
326
+ first_party: 1,
327
+ third_party: 0,
328
+ collaborative: 0,
329
+ unspecified: 0,
330
+ },
331
+ }
332
+ const comparabilityBlock = {
333
+ total_triples: 1,
334
+ variant_divergent_count: 0,
335
+ cross_party_divergent_count: 0,
336
+ groups_with_variant_check: 1,
337
+ groups_with_cross_party_check: 0,
338
+ }
339
+ await writeFile(
340
+ path.join(snapshotDir, "headline.json"),
341
+ JSON.stringify({
342
+ generated_at: "2026-05-03T00:00:00Z",
343
+ signal_version: "1.0",
344
+ stratification_dimensions: ["category"],
345
+ reproducibility: {
346
+ overall: reproducibilityBlock,
347
+ by_category: { Reasoning: reproducibilityBlock },
348
+ },
349
+ completeness: {
350
+ overall: completenessBlock,
351
+ by_category: { Reasoning: completenessBlock },
352
+ },
353
+ provenance: {
354
+ overall: provenanceBlock,
355
+ by_category: { Reasoning: provenanceBlock },
356
+ },
357
+ comparability: {
358
+ overall: comparabilityBlock,
359
+ by_category: { Reasoning: comparabilityBlock },
360
+ },
361
+ developers: [
362
+ {
363
+ developer: "OpenAI",
364
+ route_id: "OpenAI",
365
+ model_count: 1,
366
+ benchmark_count: 1,
367
+ evaluation_count: 1,
368
+ popular_evals: [{ benchmark: "MMLU", model_count: 1 }],
369
+ },
370
+ ],
371
+ })
372
+ )
373
+
374
+ await writeFile(
375
+ path.join(snapshotDir, "hierarchy.json"),
376
+ JSON.stringify({
377
+ stats: {
378
+ family_count: 1,
379
+ composite_count: 0,
380
+ standalone_benchmark_count: 1,
381
+ single_benchmark_count: 1,
382
+ slice_count: 0,
383
+ metric_count: 1,
384
+ metric_rows_scanned: 1,
385
+ },
386
+ families: [],
387
+ })
388
+ )
389
+ }
390
+
391
+ describe("Stage J view-layer backend", () => {
392
+ it("reads a pinned snapshot through the v2 accessors", async () => {
393
+ const snapshotDir = await mkdtemp(path.join(os.tmpdir(), "eval-card-stage-j-"))
394
+ const previousBackend = process.env.DATA_BACKEND
395
+ const previousSnapshotUrl = process.env.SNAPSHOT_URL
396
+
397
+ try {
398
+ await writeSyntheticStageJSnapshot(snapshotDir)
399
+ process.env.DATA_BACKEND = "v2"
400
+ process.env.SNAPSHOT_URL = `file://${snapshotDir}`
401
+
402
+ const dataBackend = await import("../lib/data-backend")
403
+ const hfData = await import("../lib/hf-data")
404
+
405
+ const [models, evalListData, modelSummary, evalSummary, developers, developerSummary, manifest, hierarchy, aggregates] =
406
+ await Promise.all([
407
+ dataBackend.getModelCardsLite(),
408
+ dataBackend.getEvalListLiteData(),
409
+ dataBackend.getModelSummaryById("openai%2Fgpt-5"),
410
+ dataBackend.getEvalSummaryById("mmlu"),
411
+ dataBackend.getDeveloperList(),
412
+ dataBackend.getDeveloperSummaryById("OpenAI"),
413
+ dataBackend.getBackendManifestData(),
414
+ dataBackend.getEvalHierarchyData(),
415
+ hfData.fetchCorpusAggregates(),
416
+ ])
417
+
418
+ expect(models[0]).toMatchObject({
419
+ route_id: "openai%2Fgpt-5",
420
+ model_name: "GPT 5",
421
+ evaluations_count: 1,
422
+ })
423
+ expect(evalListData).toMatchObject({
424
+ totalModels: 1,
425
+ evals: [{ evaluation_id: "mmlu", evaluation_name: "MMLU", models_count: 1 }],
426
+ })
427
+ expect(modelSummary?.evaluations_by_category.Reasoning).toHaveLength(1)
428
+ expect(evalSummary?.model_results[0]).toMatchObject({
429
+ model_route_id: "openai%2Fgpt-5",
430
+ score: 0.8,
431
+ result: { metric_summary_id: "mmlu%3Aaccuracy" },
432
+ })
433
+ expect(developers[0]).toMatchObject({ developer: "OpenAI", route_id: "OpenAI" })
434
+ expect(developerSummary?.models).toHaveLength(1)
435
+ expect(manifest.model_count).toBe(1)
436
+ expect(hierarchy.stats?.metric_rows_scanned).toBe(1)
437
+ expect(aggregates?.completeness.overall).toMatchObject({
438
+ total_triples: 1,
439
+ completeness_avg: 0.75,
440
+ })
441
+ expect(aggregates?.provenance.overall).toMatchObject({
442
+ total_triples: 1,
443
+ first_party_only_triples: 1,
444
+ })
445
+ expect(aggregates?.comparability.overall).toMatchObject({
446
+ groups_with_variant_check: 1,
447
+ variant_divergent_count: 0,
448
+ })
449
+ expect(aggregates?.comparability.by_category.Reasoning).toBeDefined()
450
+ } finally {
451
+ if (previousBackend == null) {
452
+ delete process.env.DATA_BACKEND
453
+ } else {
454
+ process.env.DATA_BACKEND = previousBackend
455
+ }
456
+ if (previousSnapshotUrl == null) {
457
+ delete process.env.SNAPSHOT_URL
458
+ } else {
459
+ process.env.SNAPSHOT_URL = previousSnapshotUrl
460
+ }
461
+ await rm(snapshotDir, { recursive: true, force: true })
462
+ }
463
+ })
464
+ })