Spaces:
Running
Running
Integrate with test backend data
Browse files- Dockerfile +17 -17
- app/page.tsx +1 -1
- components/signals/corpus-dashboard.tsx +61 -42
- components/signals/corpus-signals-strip.tsx +21 -11
- lib/backend-artifacts.ts +33 -21
- lib/benchmark-schema.ts +1 -0
- lib/data-backend.ts +136 -25
- lib/duckdb.ts +46 -0
- lib/hf-data.ts +38 -0
- lib/sidecars.ts +59 -0
- lib/view-data.ts +569 -0
- scripts/cache-hf-data.mjs +7 -0
- tests/duckdb-data.test.ts +29 -29
- tests/view-data.test.ts +464 -0
Dockerfile
CHANGED
|
@@ -9,20 +9,18 @@ ARG PNPM_VERSION=10.25.0
|
|
| 9 |
|
| 10 |
# Build-time data-source configuration. HF Spaces "Variables" are NOT injected
|
| 11 |
# into Docker RUN steps automatically — only into the final runtime — so we
|
| 12 |
-
# bake the
|
| 13 |
-
#
|
| 14 |
-
#
|
| 15 |
-
ARG DATA_BACKEND=
|
| 16 |
ARG HF_DATASET_REPO=https://huggingface.co/datasets/evaleval/card_backend
|
| 17 |
-
|
| 18 |
-
# `
|
| 19 |
-
#
|
| 20 |
-
#
|
| 21 |
-
# fetchers (`lib/hf-data.ts`) from attempting `evaleval/card_backend`
|
| 22 |
-
# network reads with `revalidate: 0` (which Next 15 treats as dynamic
|
| 23 |
-
# and fails the static export of `/`).
|
| 24 |
ENV DATA_BACKEND=${DATA_BACKEND} \
|
| 25 |
HF_DATASET_REPO=${HF_DATASET_REPO} \
|
|
|
|
| 26 |
LOCAL_PIPELINE_OUTPUT=/app/.cache/hf-data \
|
| 27 |
HF_DATA_LOCAL_DIR=/app/.cache/hf-data \
|
| 28 |
HF_DATA_OFFLINE=1
|
|
@@ -49,13 +47,15 @@ RUN pnpm run build
|
|
| 49 |
FROM node:18-bullseye-slim AS runner
|
| 50 |
WORKDIR /app
|
| 51 |
|
| 52 |
-
|
| 53 |
-
|
| 54 |
-
|
| 55 |
-
#
|
| 56 |
-
#
|
|
|
|
| 57 |
ENV NODE_ENV=production \
|
| 58 |
-
DATA_BACKEND=
|
|
|
|
| 59 |
LOCAL_PIPELINE_OUTPUT=/app/.cache/hf-data \
|
| 60 |
HF_DATA_LOCAL_DIR=/app/.cache/hf-data \
|
| 61 |
HF_DATA_OFFLINE=1
|
|
|
|
| 9 |
|
| 10 |
# Build-time data-source configuration. HF Spaces "Variables" are NOT injected
|
| 11 |
# into Docker RUN steps automatically — only into the final runtime — so we
|
| 12 |
+
# bake the selected backend here. `DATA_BACKEND=v2` reads `SNAPSHOT_URL`
|
| 13 |
+
# directly; legacy DuckDB mode still clones `HF_DATASET_REPO` into the cache.
|
| 14 |
+
# Override at build time via `--build-arg ...`.
|
| 15 |
+
ARG DATA_BACKEND=v2
|
| 16 |
ARG HF_DATASET_REPO=https://huggingface.co/datasets/evaleval/card_backend
|
| 17 |
+
ARG SNAPSHOT_URL=https://huggingface.co/datasets/evaleval/eval-cards-data/resolve/main/warehouse/2026-05-03T21-46-50Z
|
| 18 |
+
# Static prerender (`next build`) executes route handlers. In legacy mode the
|
| 19 |
+
# cache populated by `cache-hf-data.mjs` lives at `/app/.cache/hf-data`; in v2
|
| 20 |
+
# the cache step is skipped and the app reads the pinned Stage J snapshot.
|
|
|
|
|
|
|
|
|
|
| 21 |
ENV DATA_BACKEND=${DATA_BACKEND} \
|
| 22 |
HF_DATASET_REPO=${HF_DATASET_REPO} \
|
| 23 |
+
SNAPSHOT_URL=${SNAPSHOT_URL} \
|
| 24 |
LOCAL_PIPELINE_OUTPUT=/app/.cache/hf-data \
|
| 25 |
HF_DATA_LOCAL_DIR=/app/.cache/hf-data \
|
| 26 |
HF_DATA_OFFLINE=1
|
|
|
|
| 47 |
FROM node:18-bullseye-slim AS runner
|
| 48 |
WORKDIR /app
|
| 49 |
|
| 50 |
+
ARG DATA_BACKEND=v2
|
| 51 |
+
ARG SNAPSHOT_URL=https://huggingface.co/datasets/evaleval/eval-cards-data/resolve/main/warehouse/2026-05-03T21-46-50Z
|
| 52 |
+
|
| 53 |
+
# Runtime needs the same data-source envs that the builder used. Docker
|
| 54 |
+
# multi-stage doesn't carry ENVs across stages, so keep backend selection and
|
| 55 |
+
# snapshot/cache pointers explicit here too.
|
| 56 |
ENV NODE_ENV=production \
|
| 57 |
+
DATA_BACKEND=${DATA_BACKEND} \
|
| 58 |
+
SNAPSHOT_URL=${SNAPSHOT_URL} \
|
| 59 |
LOCAL_PIPELINE_OUTPUT=/app/.cache/hf-data \
|
| 60 |
HF_DATA_LOCAL_DIR=/app/.cache/hf-data \
|
| 61 |
HF_DATA_OFFLINE=1
|
app/page.tsx
CHANGED
|
@@ -244,7 +244,7 @@ export default async function HomePage() {
|
|
| 244 |
<p className="mx-auto mt-2 max-w-2xl text-sm leading-6 text-[color:var(--fg-muted)]">
|
| 245 |
The current backend snapshot does not include{" "}
|
| 246 |
<code className="rounded-sm bg-[color:var(--bg-surface)] px-1.5 py-0.5 font-mono text-xs">
|
| 247 |
-
|
| 248 |
</code>
|
| 249 |
. When it does, this section will render the four corpus-level rollups.
|
| 250 |
</p>
|
|
|
|
| 244 |
<p className="mx-auto mt-2 max-w-2xl text-sm leading-6 text-[color:var(--fg-muted)]">
|
| 245 |
The current backend snapshot does not include{" "}
|
| 246 |
<code className="rounded-sm bg-[color:var(--bg-surface)] px-1.5 py-0.5 font-mono text-xs">
|
| 247 |
+
headline.json
|
| 248 |
</code>
|
| 249 |
. When it does, this section will render the four corpus-level rollups.
|
| 250 |
</p>
|
components/signals/corpus-dashboard.tsx
CHANGED
|
@@ -20,7 +20,7 @@ import {
|
|
| 20 |
formatPercent,
|
| 21 |
} from "./signal-utils"
|
| 22 |
|
| 23 |
-
const CATEGORY_ORDER = ["
|
| 24 |
|
| 25 |
const SOURCE_COLORS: Record<string, string> = {
|
| 26 |
first_party: "bg-amber-500",
|
|
@@ -51,13 +51,21 @@ export function CorpusDashboard({
|
|
| 51 |
}, [mode])
|
| 52 |
|
| 53 |
const categoryKeys = useMemo(
|
| 54 |
-
() =>
|
| 55 |
-
|
| 56 |
-
aggregates.reproducibility.by_category
|
| 57 |
-
aggregates.completeness.by_category
|
| 58 |
-
aggregates.provenance.by_category
|
| 59 |
-
aggregates.comparability.by_category
|
| 60 |
-
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 61 |
[aggregates]
|
| 62 |
)
|
| 63 |
|
|
@@ -190,25 +198,14 @@ function CompletenessSection({
|
|
| 190 |
icon={<ClipboardCheck className="h-5 w-5" />}
|
| 191 |
title="Reporting Completeness"
|
| 192 |
subtitle="How much benchmark documentation is populated."
|
| 193 |
-
headline={formatPercent(block.
|
| 194 |
-
headlineLabel={`
|
| 195 |
>
|
| 196 |
{scores.length > 0 && <Histogram scores={scores} />}
|
| 197 |
-
<div className="mt-4 grid gap-2">
|
| 198 |
-
{
|
| 199 |
-
|
| 200 |
-
|
| 201 |
-
<span className="font-medium">{formatFieldLabel(field)}</span>
|
| 202 |
-
<span className="shrink-0 tabular-nums text-muted-foreground">
|
| 203 |
-
{formatPercent(value.mean_score)}
|
| 204 |
-
</span>
|
| 205 |
-
</div>
|
| 206 |
-
<div className="mt-2 grid gap-1.5">
|
| 207 |
-
<MetricBar label="Any data" value={value.populated_rate} compact />
|
| 208 |
-
<MetricBar label="Fully populated" value={value.fully_populated_rate} compact />
|
| 209 |
-
</div>
|
| 210 |
-
</div>
|
| 211 |
-
))}
|
| 212 |
</div>
|
| 213 |
</DashboardSection>
|
| 214 |
)
|
|
@@ -217,14 +214,16 @@ function CompletenessSection({
|
|
| 217 |
function ProvenanceSection({ block }: { block: ProvenanceCorpusBlock }) {
|
| 218 |
const distribution = block.source_type_distribution
|
| 219 |
const total = Object.values(distribution).reduce((sum, value) => sum + value, 0)
|
|
|
|
|
|
|
| 220 |
|
| 221 |
return (
|
| 222 |
<DashboardSection
|
| 223 |
icon={<BarChart3 className="h-5 w-5" />}
|
| 224 |
title="Provenance"
|
| 225 |
subtitle="Who reported the scores, and whether groups have multiple sources."
|
| 226 |
-
headline={formatPercent(
|
| 227 |
-
headlineLabel="of
|
| 228 |
>
|
| 229 |
<div className="overflow-hidden rounded-full border border-border/70 bg-muted/30">
|
| 230 |
<div className="flex h-4 w-full">
|
|
@@ -240,34 +239,40 @@ function ProvenanceSection({ block }: { block: ProvenanceCorpusBlock }) {
|
|
| 240 |
</div>
|
| 241 |
|
| 242 |
<div className="mt-3 grid gap-2 sm:grid-cols-2">
|
| 243 |
-
<RatioTile label="Multi-source
|
| 244 |
-
<RatioTile label="First-party only
|
| 245 |
</div>
|
| 246 |
</DashboardSection>
|
| 247 |
)
|
| 248 |
}
|
| 249 |
|
| 250 |
function ComparabilitySection({ block }: { block: ComparabilityCorpusBlock }) {
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 251 |
return (
|
| 252 |
<DashboardSection
|
| 253 |
icon={<GitCompareArrows className="h-5 w-5" />}
|
| 254 |
title="Comparability"
|
| 255 |
subtitle="Eligible groups where scores diverge across setups or reporting organizations."
|
| 256 |
-
headline={formatNullableRate(
|
| 257 |
-
headlineLabel={`${block.
|
| 258 |
>
|
| 259 |
<div className="grid gap-3 md:grid-cols-2">
|
| 260 |
<ComparabilityRateCard
|
| 261 |
title="Variant divergence"
|
| 262 |
-
rate={
|
| 263 |
-
eligible={block.
|
| 264 |
-
divergent={block.
|
| 265 |
/>
|
| 266 |
<ComparabilityRateCard
|
| 267 |
title="Cross-party divergence"
|
| 268 |
-
rate={
|
| 269 |
-
eligible={block.
|
| 270 |
-
divergent={block.
|
| 271 |
/>
|
| 272 |
</div>
|
| 273 |
</DashboardSection>
|
|
@@ -288,6 +293,15 @@ function CategoryPanel({
|
|
| 288 |
comparability?: ComparabilityCorpusBlock
|
| 289 |
}) {
|
| 290 |
const categoryLabel = `${category.charAt(0).toUpperCase()}${category.slice(1)}`
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 291 |
|
| 292 |
return (
|
| 293 |
<section className="rounded-2xl border border-border/70 bg-card p-4 shadow-sm">
|
|
@@ -297,11 +311,11 @@ function CategoryPanel({
|
|
| 297 |
</div>
|
| 298 |
<div className="grid gap-3 sm:grid-cols-2">
|
| 299 |
<MiniMetric label="Reproducibility gaps" value={formatPercent(reproducibility?.reproducibility_gap_rate)} />
|
| 300 |
-
<MiniMetric label="Documentation mean" value={formatPercent(completeness?.
|
| 301 |
-
<MiniMetric label="Multi-source
|
| 302 |
-
<MiniMetric label="Variant divergence" value={formatNullableRate(
|
| 303 |
</div>
|
| 304 |
-
{
|
| 305 |
<div className="mt-3 rounded-xl border border-dashed border-border/70 bg-muted/10 px-3 py-2 text-sm text-muted-foreground">
|
| 306 |
Cross-party divergence: N/A - not enough multi-org coverage.
|
| 307 |
</div>
|
|
@@ -411,7 +425,7 @@ function RatioTile({ label, value, count }: { label: string; value: number | nul
|
|
| 411 |
<div className="text-sm font-medium">{label}</div>
|
| 412 |
<div className="mt-1 flex items-baseline justify-between gap-2">
|
| 413 |
<span className="text-xl font-semibold tabular-nums">{formatPercent(value)}</span>
|
| 414 |
-
<span className="text-xs text-muted-foreground">{count.toLocaleString()}
|
| 415 |
</div>
|
| 416 |
</div>
|
| 417 |
)
|
|
@@ -463,6 +477,11 @@ function formatNullableRate(value: number | null | undefined) {
|
|
| 463 |
return value == null ? "N/A" : formatPercent(value)
|
| 464 |
}
|
| 465 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 466 |
function formatGeneratedDate(value: string) {
|
| 467 |
const date = new Date(value)
|
| 468 |
if (Number.isNaN(date.getTime())) {
|
|
|
|
| 20 |
formatPercent,
|
| 21 |
} from "./signal-utils"
|
| 22 |
|
| 23 |
+
const CATEGORY_ORDER = ["Agentic", "General", "Knowledge", "Reasoning", "Safety", "Other"]
|
| 24 |
|
| 25 |
const SOURCE_COLORS: Record<string, string> = {
|
| 26 |
first_party: "bg-amber-500",
|
|
|
|
| 51 |
}, [mode])
|
| 52 |
|
| 53 |
const categoryKeys = useMemo(
|
| 54 |
+
() => {
|
| 55 |
+
const available = new Set([
|
| 56 |
+
...Object.keys(aggregates.reproducibility.by_category),
|
| 57 |
+
...Object.keys(aggregates.completeness.by_category),
|
| 58 |
+
...Object.keys(aggregates.provenance.by_category),
|
| 59 |
+
...Object.keys(aggregates.comparability.by_category),
|
| 60 |
+
])
|
| 61 |
+
|
| 62 |
+
return [
|
| 63 |
+
...CATEGORY_ORDER.filter((category) => available.has(category)),
|
| 64 |
+
...Array.from(available)
|
| 65 |
+
.filter((category) => !CATEGORY_ORDER.includes(category))
|
| 66 |
+
.sort((a, b) => a.localeCompare(b)),
|
| 67 |
+
]
|
| 68 |
+
},
|
| 69 |
[aggregates]
|
| 70 |
)
|
| 71 |
|
|
|
|
| 198 |
icon={<ClipboardCheck className="h-5 w-5" />}
|
| 199 |
title="Reporting Completeness"
|
| 200 |
subtitle="How much benchmark documentation is populated."
|
| 201 |
+
headline={formatPercent(block.completeness_avg)}
|
| 202 |
+
headlineLabel={`Range ${formatPercent(block.completeness_min)} to ${formatPercent(block.completeness_max)} across ${block.total_triples.toLocaleString()} reported score triples`}
|
| 203 |
>
|
| 204 |
{scores.length > 0 && <Histogram scores={scores} />}
|
| 205 |
+
<div className="mt-4 grid gap-2 sm:grid-cols-3">
|
| 206 |
+
<MiniMetric label="Minimum" value={formatPercent(block.completeness_min)} />
|
| 207 |
+
<MiniMetric label="Average" value={formatPercent(block.completeness_avg)} />
|
| 208 |
+
<MiniMetric label="Maximum" value={formatPercent(block.completeness_max)} />
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 209 |
</div>
|
| 210 |
</DashboardSection>
|
| 211 |
)
|
|
|
|
| 214 |
function ProvenanceSection({ block }: { block: ProvenanceCorpusBlock }) {
|
| 215 |
const distribution = block.source_type_distribution
|
| 216 |
const total = Object.values(distribution).reduce((sum, value) => sum + value, 0)
|
| 217 |
+
const multiSourceRate = rate(block.multi_source_triples, block.total_triples)
|
| 218 |
+
const firstPartyOnlyRate = rate(block.first_party_only_triples, block.total_triples)
|
| 219 |
|
| 220 |
return (
|
| 221 |
<DashboardSection
|
| 222 |
icon={<BarChart3 className="h-5 w-5" />}
|
| 223 |
title="Provenance"
|
| 224 |
subtitle="Who reported the scores, and whether groups have multiple sources."
|
| 225 |
+
headline={formatPercent(multiSourceRate)}
|
| 226 |
+
headlineLabel="of reported score triples have multiple reporting sources"
|
| 227 |
>
|
| 228 |
<div className="overflow-hidden rounded-full border border-border/70 bg-muted/30">
|
| 229 |
<div className="flex h-4 w-full">
|
|
|
|
| 239 |
</div>
|
| 240 |
|
| 241 |
<div className="mt-3 grid gap-2 sm:grid-cols-2">
|
| 242 |
+
<RatioTile label="Multi-source triples" value={multiSourceRate} count={block.multi_source_triples} />
|
| 243 |
+
<RatioTile label="First-party only triples" value={firstPartyOnlyRate} count={block.first_party_only_triples} />
|
| 244 |
</div>
|
| 245 |
</DashboardSection>
|
| 246 |
)
|
| 247 |
}
|
| 248 |
|
| 249 |
function ComparabilitySection({ block }: { block: ComparabilityCorpusBlock }) {
|
| 250 |
+
const variantRate = rate(block.variant_divergent_count, block.groups_with_variant_check)
|
| 251 |
+
const crossPartyRate = rate(
|
| 252 |
+
block.cross_party_divergent_count,
|
| 253 |
+
block.groups_with_cross_party_check
|
| 254 |
+
)
|
| 255 |
+
|
| 256 |
return (
|
| 257 |
<DashboardSection
|
| 258 |
icon={<GitCompareArrows className="h-5 w-5" />}
|
| 259 |
title="Comparability"
|
| 260 |
subtitle="Eligible groups where scores diverge across setups or reporting organizations."
|
| 261 |
+
headline={formatNullableRate(variantRate)}
|
| 262 |
+
headlineLabel={`${block.variant_divergent_count.toLocaleString()} of ${block.groups_with_variant_check.toLocaleString()} setup-eligible groups diverge`}
|
| 263 |
>
|
| 264 |
<div className="grid gap-3 md:grid-cols-2">
|
| 265 |
<ComparabilityRateCard
|
| 266 |
title="Variant divergence"
|
| 267 |
+
rate={variantRate}
|
| 268 |
+
eligible={block.groups_with_variant_check}
|
| 269 |
+
divergent={block.variant_divergent_count}
|
| 270 |
/>
|
| 271 |
<ComparabilityRateCard
|
| 272 |
title="Cross-party divergence"
|
| 273 |
+
rate={crossPartyRate}
|
| 274 |
+
eligible={block.groups_with_cross_party_check}
|
| 275 |
+
divergent={block.cross_party_divergent_count}
|
| 276 |
/>
|
| 277 |
</div>
|
| 278 |
</DashboardSection>
|
|
|
|
| 293 |
comparability?: ComparabilityCorpusBlock
|
| 294 |
}) {
|
| 295 |
const categoryLabel = `${category.charAt(0).toUpperCase()}${category.slice(1)}`
|
| 296 |
+
const multiSourceRate = rate(provenance?.multi_source_triples, provenance?.total_triples)
|
| 297 |
+
const variantRate = rate(
|
| 298 |
+
comparability?.variant_divergent_count,
|
| 299 |
+
comparability?.groups_with_variant_check
|
| 300 |
+
)
|
| 301 |
+
const crossPartyRate = rate(
|
| 302 |
+
comparability?.cross_party_divergent_count,
|
| 303 |
+
comparability?.groups_with_cross_party_check
|
| 304 |
+
)
|
| 305 |
|
| 306 |
return (
|
| 307 |
<section className="rounded-2xl border border-border/70 bg-card p-4 shadow-sm">
|
|
|
|
| 311 |
</div>
|
| 312 |
<div className="grid gap-3 sm:grid-cols-2">
|
| 313 |
<MiniMetric label="Reproducibility gaps" value={formatPercent(reproducibility?.reproducibility_gap_rate)} />
|
| 314 |
+
<MiniMetric label="Documentation mean" value={formatPercent(completeness?.completeness_avg)} />
|
| 315 |
+
<MiniMetric label="Multi-source triples" value={formatPercent(multiSourceRate)} />
|
| 316 |
+
<MiniMetric label="Variant divergence" value={formatNullableRate(variantRate)} />
|
| 317 |
</div>
|
| 318 |
+
{crossPartyRate == null && (
|
| 319 |
<div className="mt-3 rounded-xl border border-dashed border-border/70 bg-muted/10 px-3 py-2 text-sm text-muted-foreground">
|
| 320 |
Cross-party divergence: N/A - not enough multi-org coverage.
|
| 321 |
</div>
|
|
|
|
| 425 |
<div className="text-sm font-medium">{label}</div>
|
| 426 |
<div className="mt-1 flex items-baseline justify-between gap-2">
|
| 427 |
<span className="text-xl font-semibold tabular-nums">{formatPercent(value)}</span>
|
| 428 |
+
<span className="text-xs text-muted-foreground">{count.toLocaleString()} triples</span>
|
| 429 |
</div>
|
| 430 |
</div>
|
| 431 |
)
|
|
|
|
| 477 |
return value == null ? "N/A" : formatPercent(value)
|
| 478 |
}
|
| 479 |
|
| 480 |
+
function rate(numerator: number | null | undefined, denominator: number | null | undefined) {
|
| 481 |
+
if (numerator == null || denominator == null || denominator <= 0) return null
|
| 482 |
+
return numerator / denominator
|
| 483 |
+
}
|
| 484 |
+
|
| 485 |
function formatGeneratedDate(value: string) {
|
| 486 |
const date = new Date(value)
|
| 487 |
if (Number.isNaN(date.getTime())) {
|
components/signals/corpus-signals-strip.tsx
CHANGED
|
@@ -39,8 +39,13 @@ export function CorpusSignalsStrip({
|
|
| 39 |
const tpShare = totalReports > 0 ? prov.source_type_distribution.third_party / totalReports : 0
|
| 40 |
const fpShare = totalReports > 0 ? prov.source_type_distribution.first_party / totalReports : 0
|
| 41 |
|
| 42 |
-
const
|
| 43 |
-
const
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 44 |
|
| 45 |
return (
|
| 46 |
<div className="signals-grid">
|
|
@@ -58,29 +63,29 @@ export function CorpusSignalsStrip({
|
|
| 58 |
/>
|
| 59 |
<SignalTile
|
| 60 |
id="completeness"
|
| 61 |
-
statValue={pctNum(comp.
|
| 62 |
statUnit="%"
|
| 63 |
-
headline={`mean across ${comp.
|
| 64 |
-
detail=
|
| 65 |
asks="Is the benchmark itself documented well enough to interpret a score on it?"
|
| 66 |
/>
|
| 67 |
<SignalTile
|
| 68 |
id="provenance"
|
| 69 |
-
statValue={pctNum(
|
| 70 |
statUnit="%"
|
| 71 |
-
headline="of
|
| 72 |
-
detail={`${formatPct(tpShare)} third-party, ${formatPct(fpShare)} first-party of ${totalReports.toLocaleString()}
|
| 73 |
asks="Who reported this score, and have others reproduced it?"
|
| 74 |
/>
|
| 75 |
<SignalTile
|
| 76 |
id="comparability"
|
| 77 |
statValue={pctNum(cmpRate)}
|
| 78 |
statUnit="%"
|
| 79 |
-
headline={`of setup-eligible groups diverge across variants (${cmp.
|
| 80 |
detail={
|
| 81 |
crossPartyAvailable
|
| 82 |
-
? `Cross-party divergence: ${formatPct(
|
| 83 |
-
: "Cross-party divergence not yet computable
|
| 84 |
}
|
| 85 |
asks="Are scores on the same benchmark actually measuring the same thing?"
|
| 86 |
/>
|
|
@@ -154,6 +159,11 @@ function formatPct(value: number | null | undefined): string {
|
|
| 154 |
return `${Math.round(value * 100)}%`
|
| 155 |
}
|
| 156 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 157 |
const FIELD_LABELS: Record<string, string> = {
|
| 158 |
temperature: "temperature",
|
| 159 |
max_tokens: "max tokens",
|
|
|
|
| 39 |
const tpShare = totalReports > 0 ? prov.source_type_distribution.third_party / totalReports : 0
|
| 40 |
const fpShare = totalReports > 0 ? prov.source_type_distribution.first_party / totalReports : 0
|
| 41 |
|
| 42 |
+
const multiSourceRate = rate(prov.multi_source_triples, prov.total_triples)
|
| 43 |
+
const cmpRate = rate(cmp.variant_divergent_count, cmp.groups_with_variant_check)
|
| 44 |
+
const crossPartyRate = rate(
|
| 45 |
+
cmp.cross_party_divergent_count,
|
| 46 |
+
cmp.groups_with_cross_party_check
|
| 47 |
+
)
|
| 48 |
+
const crossPartyAvailable = cmp.groups_with_cross_party_check > 0
|
| 49 |
|
| 50 |
return (
|
| 51 |
<div className="signals-grid">
|
|
|
|
| 63 |
/>
|
| 64 |
<SignalTile
|
| 65 |
id="completeness"
|
| 66 |
+
statValue={pctNum(comp.completeness_avg)}
|
| 67 |
statUnit="%"
|
| 68 |
+
headline={`mean across ${comp.total_triples.toLocaleString()} reported score triples.`}
|
| 69 |
+
detail={`Observed range: ${formatPct(comp.completeness_min)} to ${formatPct(comp.completeness_max)}.`}
|
| 70 |
asks="Is the benchmark itself documented well enough to interpret a score on it?"
|
| 71 |
/>
|
| 72 |
<SignalTile
|
| 73 |
id="provenance"
|
| 74 |
+
statValue={pctNum(multiSourceRate)}
|
| 75 |
statUnit="%"
|
| 76 |
+
headline="of reported score triples have reports from more than one party."
|
| 77 |
+
detail={`${formatPct(tpShare)} third-party, ${formatPct(fpShare)} first-party of ${totalReports.toLocaleString()} triples.`}
|
| 78 |
asks="Who reported this score, and have others reproduced it?"
|
| 79 |
/>
|
| 80 |
<SignalTile
|
| 81 |
id="comparability"
|
| 82 |
statValue={pctNum(cmpRate)}
|
| 83 |
statUnit="%"
|
| 84 |
+
headline={`of setup-eligible groups diverge across variants (${cmp.variant_divergent_count.toLocaleString()} of ${cmp.groups_with_variant_check.toLocaleString()}).`}
|
| 85 |
detail={
|
| 86 |
crossPartyAvailable
|
| 87 |
+
? `Cross-party divergence: ${formatPct(crossPartyRate)}.`
|
| 88 |
+
: "Cross-party divergence not yet computable: too few multi-org reports."
|
| 89 |
}
|
| 90 |
asks="Are scores on the same benchmark actually measuring the same thing?"
|
| 91 |
/>
|
|
|
|
| 159 |
return `${Math.round(value * 100)}%`
|
| 160 |
}
|
| 161 |
|
| 162 |
+
function rate(numerator: number | null | undefined, denominator: number | null | undefined) {
|
| 163 |
+
if (numerator == null || denominator == null || denominator <= 0) return null
|
| 164 |
+
return numerator / denominator
|
| 165 |
+
}
|
| 166 |
+
|
| 167 |
const FIELD_LABELS: Record<string, string> = {
|
| 168 |
temperature: "temperature",
|
| 169 |
max_tokens: "max tokens",
|
lib/backend-artifacts.ts
CHANGED
|
@@ -12,6 +12,7 @@ export interface BackendManifest {
|
|
| 12 |
skipped_config_count?: number
|
| 13 |
summary_artifacts?: {
|
| 14 |
corpus_aggregates?: string
|
|
|
|
| 15 |
[key: string]: string | undefined
|
| 16 |
}
|
| 17 |
}
|
|
@@ -177,6 +178,27 @@ export interface CorpusAggregates {
|
|
| 177 |
completeness: Stratified<CompletenessCorpusBlock>
|
| 178 |
provenance: Stratified<ProvenanceCorpusBlock>
|
| 179 |
comparability: Stratified<ComparabilityCorpusBlock>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 180 |
}
|
| 181 |
|
| 182 |
export interface Stratified<T> {
|
|
@@ -198,35 +220,25 @@ export interface ReproducibilityCorpusBlock {
|
|
| 198 |
}
|
| 199 |
|
| 200 |
export interface CompletenessCorpusBlock {
|
| 201 |
-
|
| 202 |
-
|
| 203 |
-
|
| 204 |
-
|
| 205 |
-
mean_score: number
|
| 206 |
-
populated_rate: number
|
| 207 |
-
fully_populated_rate: number
|
| 208 |
-
benchmark_count: number
|
| 209 |
-
}>
|
| 210 |
}
|
| 211 |
|
| 212 |
export interface ProvenanceCorpusBlock {
|
| 213 |
total_triples: number
|
| 214 |
-
|
| 215 |
-
|
| 216 |
-
multi_source_rate: number | null
|
| 217 |
-
first_party_only_groups: number
|
| 218 |
-
first_party_only_rate: number | null
|
| 219 |
source_type_distribution: Record<ProvenanceSourceType, number>
|
| 220 |
}
|
| 221 |
|
| 222 |
export interface ComparabilityCorpusBlock {
|
| 223 |
-
|
| 224 |
-
|
| 225 |
-
|
| 226 |
-
|
| 227 |
-
|
| 228 |
-
cross_party_divergent_groups: number
|
| 229 |
-
cross_party_divergence_rate: number | null
|
| 230 |
}
|
| 231 |
|
| 232 |
export interface HierarchyTags {
|
|
|
|
| 12 |
skipped_config_count?: number
|
| 13 |
summary_artifacts?: {
|
| 14 |
corpus_aggregates?: string
|
| 15 |
+
eval_hierarchy?: string
|
| 16 |
[key: string]: string | undefined
|
| 17 |
}
|
| 18 |
}
|
|
|
|
| 178 |
completeness: Stratified<CompletenessCorpusBlock>
|
| 179 |
provenance: Stratified<ProvenanceCorpusBlock>
|
| 180 |
comparability: Stratified<ComparabilityCorpusBlock>
|
| 181 |
+
developers?: DeveloperListEntry[]
|
| 182 |
+
families?: Array<{
|
| 183 |
+
family_key: string
|
| 184 |
+
display_name: string
|
| 185 |
+
model_count: number
|
| 186 |
+
eval_count: number
|
| 187 |
+
}>
|
| 188 |
+
categories?: Array<{
|
| 189 |
+
category: string
|
| 190 |
+
model_count: number
|
| 191 |
+
eval_count: number
|
| 192 |
+
}>
|
| 193 |
+
}
|
| 194 |
+
|
| 195 |
+
export interface DeveloperListEntry {
|
| 196 |
+
developer: string
|
| 197 |
+
route_id: string
|
| 198 |
+
model_count: number
|
| 199 |
+
benchmark_count: number
|
| 200 |
+
evaluation_count: number
|
| 201 |
+
popular_evals: Array<{ benchmark: string; model_count: number }>
|
| 202 |
}
|
| 203 |
|
| 204 |
export interface Stratified<T> {
|
|
|
|
| 220 |
}
|
| 221 |
|
| 222 |
export interface CompletenessCorpusBlock {
|
| 223 |
+
total_triples: number
|
| 224 |
+
completeness_avg: number | null
|
| 225 |
+
completeness_min: number | null
|
| 226 |
+
completeness_max: number | null
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 227 |
}
|
| 228 |
|
| 229 |
export interface ProvenanceCorpusBlock {
|
| 230 |
total_triples: number
|
| 231 |
+
multi_source_triples: number
|
| 232 |
+
first_party_only_triples: number
|
|
|
|
|
|
|
|
|
|
| 233 |
source_type_distribution: Record<ProvenanceSourceType, number>
|
| 234 |
}
|
| 235 |
|
| 236 |
export interface ComparabilityCorpusBlock {
|
| 237 |
+
total_triples: number
|
| 238 |
+
variant_divergent_count: number
|
| 239 |
+
cross_party_divergent_count: number
|
| 240 |
+
groups_with_variant_check: number
|
| 241 |
+
groups_with_cross_party_check: number
|
|
|
|
|
|
|
| 242 |
}
|
| 243 |
|
| 244 |
export interface HierarchyTags {
|
lib/benchmark-schema.ts
CHANGED
|
@@ -124,6 +124,7 @@ export interface ScoreDetails {
|
|
| 124 |
}
|
| 125 |
|
| 126 |
export interface GenerationConfig {
|
|
|
|
| 127 |
generation_args?: {
|
| 128 |
temperature?: number
|
| 129 |
top_p?: number
|
|
|
|
| 124 |
}
|
| 125 |
|
| 126 |
export interface GenerationConfig {
|
| 127 |
+
num_few_shot?: number
|
| 128 |
generation_args?: {
|
| 129 |
temperature?: number
|
| 130 |
top_p?: number
|
lib/data-backend.ts
CHANGED
|
@@ -1,27 +1,138 @@
|
|
| 1 |
import "server-only"
|
| 2 |
|
| 3 |
-
import {
|
| 4 |
-
|
| 5 |
-
|
| 6 |
-
|
| 7 |
-
|
| 8 |
-
|
| 9 |
-
|
| 10 |
-
|
| 11 |
-
|
| 12 |
-
|
| 13 |
-
|
| 14 |
-
|
| 15 |
-
|
| 16 |
-
|
| 17 |
-
|
| 18 |
-
|
| 19 |
-
|
| 20 |
-
|
| 21 |
-
|
| 22 |
-
|
| 23 |
-
|
| 24 |
-
|
| 25 |
-
|
| 26 |
-
|
| 27 |
-
export
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
import "server-only"
|
| 2 |
|
| 3 |
+
import type { BackendManifestStatus } from "@/lib/backend-artifacts"
|
| 4 |
+
|
| 5 |
+
const BACKEND_VERSION = process.env.DATA_BACKEND?.trim().toLowerCase() ?? "duckdb"
|
| 6 |
+
|
| 7 |
+
function useViewLayerBackend() {
|
| 8 |
+
return BACKEND_VERSION === "v2" || BACKEND_VERSION === "stage-j"
|
| 9 |
+
}
|
| 10 |
+
|
| 11 |
+
async function legacyBackend() {
|
| 12 |
+
return import("@/lib/duckdb-data")
|
| 13 |
+
}
|
| 14 |
+
|
| 15 |
+
async function viewBackend() {
|
| 16 |
+
return import("@/lib/view-data")
|
| 17 |
+
}
|
| 18 |
+
|
| 19 |
+
async function sidecars() {
|
| 20 |
+
return import("@/lib/sidecars")
|
| 21 |
+
}
|
| 22 |
+
|
| 23 |
+
async function hfData() {
|
| 24 |
+
return import("@/lib/hf-data")
|
| 25 |
+
}
|
| 26 |
+
|
| 27 |
+
export async function getModelCards() {
|
| 28 |
+
if (useViewLayerBackend()) {
|
| 29 |
+
return (await viewBackend()).getModelCards()
|
| 30 |
+
}
|
| 31 |
+
|
| 32 |
+
return (await legacyBackend()).getModelCardsFromDuckDB()
|
| 33 |
+
}
|
| 34 |
+
|
| 35 |
+
export async function getModelCardsLite() {
|
| 36 |
+
if (useViewLayerBackend()) {
|
| 37 |
+
return (await viewBackend()).getModelCardsLite()
|
| 38 |
+
}
|
| 39 |
+
|
| 40 |
+
return (await legacyBackend()).getModelCardsLiteFromDuckDB()
|
| 41 |
+
}
|
| 42 |
+
|
| 43 |
+
export async function getEvalListData() {
|
| 44 |
+
if (useViewLayerBackend()) {
|
| 45 |
+
return (await viewBackend()).getEvalListData()
|
| 46 |
+
}
|
| 47 |
+
|
| 48 |
+
return (await legacyBackend()).getEvalListDataFromDuckDB()
|
| 49 |
+
}
|
| 50 |
+
|
| 51 |
+
export async function getEvalListLiteData() {
|
| 52 |
+
if (useViewLayerBackend()) {
|
| 53 |
+
return (await viewBackend()).getEvalListLiteData()
|
| 54 |
+
}
|
| 55 |
+
|
| 56 |
+
return (await legacyBackend()).getEvalListLiteDataFromDuckDB()
|
| 57 |
+
}
|
| 58 |
+
|
| 59 |
+
export async function getEvalList() {
|
| 60 |
+
if (useViewLayerBackend()) {
|
| 61 |
+
return (await viewBackend()).getEvalList()
|
| 62 |
+
}
|
| 63 |
+
|
| 64 |
+
return (await legacyBackend()).getEvalListFromDuckDB()
|
| 65 |
+
}
|
| 66 |
+
|
| 67 |
+
export async function getDashboardData() {
|
| 68 |
+
if (useViewLayerBackend()) {
|
| 69 |
+
return (await viewBackend()).getDashboardData()
|
| 70 |
+
}
|
| 71 |
+
|
| 72 |
+
return (await legacyBackend()).getDashboardDataFromDuckDB()
|
| 73 |
+
}
|
| 74 |
+
|
| 75 |
+
export async function getDeveloperList() {
|
| 76 |
+
if (useViewLayerBackend()) {
|
| 77 |
+
return (await viewBackend()).getDeveloperList()
|
| 78 |
+
}
|
| 79 |
+
|
| 80 |
+
return (await legacyBackend()).getDeveloperListFromDuckDB()
|
| 81 |
+
}
|
| 82 |
+
|
| 83 |
+
export async function getDeveloperSummaryById(routeId: string) {
|
| 84 |
+
if (useViewLayerBackend()) {
|
| 85 |
+
return (await viewBackend()).getDeveloperSummaryById(routeId)
|
| 86 |
+
}
|
| 87 |
+
|
| 88 |
+
return (await legacyBackend()).getDeveloperSummaryByIdFromDuckDB(routeId)
|
| 89 |
+
}
|
| 90 |
+
|
| 91 |
+
export async function getModelSummaryById(modelId: string) {
|
| 92 |
+
if (useViewLayerBackend()) {
|
| 93 |
+
return (await viewBackend()).getModelSummaryById(modelId)
|
| 94 |
+
}
|
| 95 |
+
|
| 96 |
+
return (await legacyBackend()).getModelSummaryByIdFromDuckDB(modelId)
|
| 97 |
+
}
|
| 98 |
+
|
| 99 |
+
export async function getEvalSummaryById(evalId: string) {
|
| 100 |
+
if (useViewLayerBackend()) {
|
| 101 |
+
return (await viewBackend()).getEvalSummaryById(evalId)
|
| 102 |
+
}
|
| 103 |
+
|
| 104 |
+
return (await legacyBackend()).getEvalSummaryByIdFromDuckDB(evalId)
|
| 105 |
+
}
|
| 106 |
+
|
| 107 |
+
export async function getBackendManifestData() {
|
| 108 |
+
if (useViewLayerBackend()) {
|
| 109 |
+
return (await sidecars()).fetchManifest()
|
| 110 |
+
}
|
| 111 |
+
|
| 112 |
+
return (await hfData()).fetchBackendManifest()
|
| 113 |
+
}
|
| 114 |
+
|
| 115 |
+
export async function getBackendManifestStatusData(): Promise<BackendManifestStatus> {
|
| 116 |
+
if (useViewLayerBackend()) {
|
| 117 |
+
const manifest = await (await sidecars()).fetchManifest()
|
| 118 |
+
return {
|
| 119 |
+
currentManifest: manifest,
|
| 120 |
+
latestManifest: manifest,
|
| 121 |
+
currentManifestSignature: manifest.generated_at,
|
| 122 |
+
latestManifestSignature: manifest.generated_at,
|
| 123 |
+
updateAvailable: false,
|
| 124 |
+
refreshing: false,
|
| 125 |
+
pendingRefreshCount: 0,
|
| 126 |
+
}
|
| 127 |
+
}
|
| 128 |
+
|
| 129 |
+
return (await hfData()).fetchBackendManifestStatus()
|
| 130 |
+
}
|
| 131 |
+
|
| 132 |
+
export async function getEvalHierarchyData() {
|
| 133 |
+
if (useViewLayerBackend()) {
|
| 134 |
+
return (await sidecars()).fetchHierarchy()
|
| 135 |
+
}
|
| 136 |
+
|
| 137 |
+
return (await hfData()).fetchEvalHierarchy()
|
| 138 |
+
}
|
lib/duckdb.ts
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import "server-only"
|
| 2 |
+
|
| 3 |
+
import { DuckDBConnection } from "@duckdb/node-api"
|
| 4 |
+
|
| 5 |
+
let connectionPromise: Promise<DuckDBConnection> | null = null
|
| 6 |
+
|
| 7 |
+
function getSnapshotUrl() {
|
| 8 |
+
const snapshotUrl = process.env.SNAPSHOT_URL?.trim()
|
| 9 |
+
if (!snapshotUrl) {
|
| 10 |
+
throw new Error("DATA_BACKEND=v2 requires SNAPSHOT_URL to point at a Stage J snapshot directory")
|
| 11 |
+
}
|
| 12 |
+
|
| 13 |
+
return snapshotUrl.replace(/\/+$/, "")
|
| 14 |
+
}
|
| 15 |
+
|
| 16 |
+
function snapshotArtifact(name: string) {
|
| 17 |
+
return `${getSnapshotUrl()}/${name}`
|
| 18 |
+
}
|
| 19 |
+
|
| 20 |
+
function sqlString(value: string) {
|
| 21 |
+
return `'${value.replace(/'/g, "''")}'`
|
| 22 |
+
}
|
| 23 |
+
|
| 24 |
+
const VIEW_FILES = {
|
| 25 |
+
models_view: "models_view.parquet",
|
| 26 |
+
evals_view: "evals_view.parquet",
|
| 27 |
+
eval_results_view: "eval_results_view.parquet",
|
| 28 |
+
} as const
|
| 29 |
+
|
| 30 |
+
export async function getConnection(): Promise<DuckDBConnection> {
|
| 31 |
+
if (!connectionPromise) {
|
| 32 |
+
connectionPromise = (async () => {
|
| 33 |
+
const connection = await DuckDBConnection.create()
|
| 34 |
+
|
| 35 |
+
for (const [viewName, fileName] of Object.entries(VIEW_FILES)) {
|
| 36 |
+
await connection.run(
|
| 37 |
+
`CREATE OR REPLACE VIEW ${viewName} AS SELECT * FROM read_parquet(${sqlString(snapshotArtifact(fileName))})`
|
| 38 |
+
)
|
| 39 |
+
}
|
| 40 |
+
|
| 41 |
+
return connection
|
| 42 |
+
})()
|
| 43 |
+
}
|
| 44 |
+
|
| 45 |
+
return connectionPromise
|
| 46 |
+
}
|
lib/hf-data.ts
CHANGED
|
@@ -138,6 +138,15 @@ function getManifestSignature(manifest: BackendManifest | null | undefined) {
|
|
| 138 |
// reading the same on-disk artifacts cannot diverge mid-test via background
|
| 139 |
// refresh, and useful generally for offline development.
|
| 140 |
const OFFLINE = process.env.HF_DATA_OFFLINE === "1"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 141 |
|
| 142 |
async function fetchRemoteJson<T>(relativePath: string): Promise<T> {
|
| 143 |
if (OFFLINE) {
|
|
@@ -423,6 +432,19 @@ async function fetchHFJson<T>(relativePath: string): Promise<T> {
|
|
| 423 |
}
|
| 424 |
|
| 425 |
export async function fetchBackendManifestStatus(): Promise<BackendManifestStatus> {
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 426 |
const snapshot = await getManifestSnapshot()
|
| 427 |
const currentManifest = getCurrentManifestFromSnapshot(snapshot)
|
| 428 |
const currentManifestSignature = getManifestSignature(currentManifest)
|
|
@@ -864,14 +886,26 @@ export async function fetchDevelopersList(): Promise<HFDeveloperEntry[]> {
|
|
| 864 |
}
|
| 865 |
|
| 866 |
export async function fetchBenchmarkMetadataMap(): Promise<Record<string, BenchmarkCard>> {
|
|
|
|
|
|
|
|
|
|
|
|
|
| 867 |
return fetchHFJson<Record<string, BenchmarkCard>>("benchmark-metadata.json")
|
| 868 |
}
|
| 869 |
|
| 870 |
export async function fetchBackendManifest(): Promise<BackendManifest> {
|
|
|
|
|
|
|
|
|
|
|
|
|
| 871 |
return fetchHFJson<BackendManifest>("manifest.json")
|
| 872 |
}
|
| 873 |
|
| 874 |
export async function fetchEvalHierarchy(): Promise<EvalHierarchy> {
|
|
|
|
|
|
|
|
|
|
|
|
|
| 875 |
const raw = await fetchHFJson<EvalHierarchy>("eval-hierarchy.json")
|
| 876 |
return adaptEvalHierarchy(raw)
|
| 877 |
}
|
|
@@ -975,6 +1009,10 @@ export async function fetchComparisonIndex(): Promise<ComparisonIndex> {
|
|
| 975 |
}
|
| 976 |
|
| 977 |
export async function fetchCorpusAggregates(): Promise<CorpusAggregates | null> {
|
|
|
|
|
|
|
|
|
|
|
|
|
| 978 |
return fetchHFJsonSafe<CorpusAggregates>("corpus-aggregates.json")
|
| 979 |
}
|
| 980 |
|
|
|
|
| 138 |
// reading the same on-disk artifacts cannot diverge mid-test via background
|
| 139 |
// refresh, and useful generally for offline development.
|
| 140 |
const OFFLINE = process.env.HF_DATA_OFFLINE === "1"
|
| 141 |
+
const DATA_BACKEND_VERSION = process.env.DATA_BACKEND?.trim().toLowerCase()
|
| 142 |
+
|
| 143 |
+
function useViewLayerBackend() {
|
| 144 |
+
return DATA_BACKEND_VERSION === "v2" || DATA_BACKEND_VERSION === "stage-j"
|
| 145 |
+
}
|
| 146 |
+
|
| 147 |
+
async function fetchSnapshotSidecars() {
|
| 148 |
+
return import("@/lib/sidecars")
|
| 149 |
+
}
|
| 150 |
|
| 151 |
async function fetchRemoteJson<T>(relativePath: string): Promise<T> {
|
| 152 |
if (OFFLINE) {
|
|
|
|
| 432 |
}
|
| 433 |
|
| 434 |
export async function fetchBackendManifestStatus(): Promise<BackendManifestStatus> {
|
| 435 |
+
if (useViewLayerBackend()) {
|
| 436 |
+
const manifest = await (await fetchSnapshotSidecars()).fetchManifest()
|
| 437 |
+
return {
|
| 438 |
+
currentManifest: manifest,
|
| 439 |
+
latestManifest: manifest,
|
| 440 |
+
currentManifestSignature: manifest.generated_at,
|
| 441 |
+
latestManifestSignature: manifest.generated_at,
|
| 442 |
+
updateAvailable: false,
|
| 443 |
+
refreshing: false,
|
| 444 |
+
pendingRefreshCount: 0,
|
| 445 |
+
}
|
| 446 |
+
}
|
| 447 |
+
|
| 448 |
const snapshot = await getManifestSnapshot()
|
| 449 |
const currentManifest = getCurrentManifestFromSnapshot(snapshot)
|
| 450 |
const currentManifestSignature = getManifestSignature(currentManifest)
|
|
|
|
| 886 |
}
|
| 887 |
|
| 888 |
export async function fetchBenchmarkMetadataMap(): Promise<Record<string, BenchmarkCard>> {
|
| 889 |
+
if (useViewLayerBackend()) {
|
| 890 |
+
return (await import("@/lib/view-data")).getBenchmarkMetadataMap()
|
| 891 |
+
}
|
| 892 |
+
|
| 893 |
return fetchHFJson<Record<string, BenchmarkCard>>("benchmark-metadata.json")
|
| 894 |
}
|
| 895 |
|
| 896 |
export async function fetchBackendManifest(): Promise<BackendManifest> {
|
| 897 |
+
if (useViewLayerBackend()) {
|
| 898 |
+
return (await fetchSnapshotSidecars()).fetchManifest()
|
| 899 |
+
}
|
| 900 |
+
|
| 901 |
return fetchHFJson<BackendManifest>("manifest.json")
|
| 902 |
}
|
| 903 |
|
| 904 |
export async function fetchEvalHierarchy(): Promise<EvalHierarchy> {
|
| 905 |
+
if (useViewLayerBackend()) {
|
| 906 |
+
return adaptEvalHierarchy(await (await fetchSnapshotSidecars()).fetchHierarchy())
|
| 907 |
+
}
|
| 908 |
+
|
| 909 |
const raw = await fetchHFJson<EvalHierarchy>("eval-hierarchy.json")
|
| 910 |
return adaptEvalHierarchy(raw)
|
| 911 |
}
|
|
|
|
| 1009 |
}
|
| 1010 |
|
| 1011 |
export async function fetchCorpusAggregates(): Promise<CorpusAggregates | null> {
|
| 1012 |
+
if (useViewLayerBackend()) {
|
| 1013 |
+
return (await fetchSnapshotSidecars()).fetchHeadline()
|
| 1014 |
+
}
|
| 1015 |
+
|
| 1016 |
return fetchHFJsonSafe<CorpusAggregates>("corpus-aggregates.json")
|
| 1017 |
}
|
| 1018 |
|
lib/sidecars.ts
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import "server-only"
|
| 2 |
+
|
| 3 |
+
import type {
|
| 4 |
+
BackendManifest,
|
| 5 |
+
CorpusAggregates,
|
| 6 |
+
EvalHierarchy,
|
| 7 |
+
} from "@/lib/backend-artifacts"
|
| 8 |
+
|
| 9 |
+
let cache: {
|
| 10 |
+
manifest?: Promise<BackendManifest>
|
| 11 |
+
headline?: Promise<CorpusAggregates>
|
| 12 |
+
hierarchy?: Promise<EvalHierarchy>
|
| 13 |
+
} = {}
|
| 14 |
+
|
| 15 |
+
function getSnapshotUrl() {
|
| 16 |
+
const snapshotUrl = process.env.SNAPSHOT_URL?.trim()
|
| 17 |
+
if (!snapshotUrl) {
|
| 18 |
+
throw new Error("DATA_BACKEND=v2 requires SNAPSHOT_URL to point at a Stage J snapshot directory")
|
| 19 |
+
}
|
| 20 |
+
|
| 21 |
+
return snapshotUrl.replace(/\/+$/, "")
|
| 22 |
+
}
|
| 23 |
+
|
| 24 |
+
function sidecarUrl(name: string) {
|
| 25 |
+
return `${getSnapshotUrl()}/${name}`
|
| 26 |
+
}
|
| 27 |
+
|
| 28 |
+
async function fetchJson<T>(name: string): Promise<T> {
|
| 29 |
+
const url = sidecarUrl(name)
|
| 30 |
+
|
| 31 |
+
if (url.startsWith("file://")) {
|
| 32 |
+
const fs = await import("fs/promises")
|
| 33 |
+
const text = await fs.readFile(new URL(url), "utf8")
|
| 34 |
+
return JSON.parse(text) as T
|
| 35 |
+
}
|
| 36 |
+
|
| 37 |
+
const response = await fetch(url, { next: { revalidate: 3600 } })
|
| 38 |
+
if (!response.ok) {
|
| 39 |
+
throw new Error(`Snapshot sidecar fetch failed: ${response.status} ${response.statusText} for ${url}`)
|
| 40 |
+
}
|
| 41 |
+
|
| 42 |
+
return (await response.json()) as T
|
| 43 |
+
}
|
| 44 |
+
|
| 45 |
+
export function fetchManifest(): Promise<BackendManifest> {
|
| 46 |
+
return (cache.manifest ??= fetchJson<BackendManifest>("manifest.json"))
|
| 47 |
+
}
|
| 48 |
+
|
| 49 |
+
export function fetchHeadline(): Promise<CorpusAggregates> {
|
| 50 |
+
return (cache.headline ??= fetchJson<CorpusAggregates>("headline.json"))
|
| 51 |
+
}
|
| 52 |
+
|
| 53 |
+
export function fetchHierarchy(): Promise<EvalHierarchy> {
|
| 54 |
+
return (cache.hierarchy ??= fetchJson<EvalHierarchy>("hierarchy.json"))
|
| 55 |
+
}
|
| 56 |
+
|
| 57 |
+
export function resetSidecarCacheForTests() {
|
| 58 |
+
cache = {}
|
| 59 |
+
}
|
lib/view-data.ts
ADDED
|
@@ -0,0 +1,569 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import "server-only"
|
| 2 |
+
|
| 3 |
+
import { getConnection } from "@/lib/duckdb"
|
| 4 |
+
import { fetchHeadline } from "@/lib/sidecars"
|
| 5 |
+
import {
|
| 6 |
+
EVALUATION_CATEGORIES,
|
| 7 |
+
type BenchmarkCard,
|
| 8 |
+
type BenchmarkEvaluation,
|
| 9 |
+
type CategoryType,
|
| 10 |
+
type EvaluationCardData,
|
| 11 |
+
type EvaluationResult,
|
| 12 |
+
type GenerationConfig,
|
| 13 |
+
type MetricConfig,
|
| 14 |
+
type ModelInfo,
|
| 15 |
+
type ModelEvaluationSummary,
|
| 16 |
+
type ModelVariantSummary,
|
| 17 |
+
type ScoreDetails,
|
| 18 |
+
type SourceData,
|
| 19 |
+
type SourceMetadata,
|
| 20 |
+
} from "@/lib/benchmark-schema"
|
| 21 |
+
import type { DeveloperListEntry } from "@/lib/backend-artifacts"
|
| 22 |
+
import type {
|
| 23 |
+
BenchmarkEvalListItem,
|
| 24 |
+
BenchmarkEvalSummary,
|
| 25 |
+
ModelResultForBenchmark,
|
| 26 |
+
} from "@/lib/eval-processing"
|
| 27 |
+
|
| 28 |
+
type Row = Record<string, any>
|
| 29 |
+
|
| 30 |
+
const MODEL_CARD_COLUMNS = `
|
| 31 |
+
id, route_id, model_name, model_id, canonical_model_name, developer,
|
| 32 |
+
evaluations_count, benchmarks_count, variant_count,
|
| 33 |
+
categories, category_stats, latest_timestamp,
|
| 34 |
+
evaluator_count, evaluator_names, source_type_count, source_types,
|
| 35 |
+
evidence_count, missing_generation_config_count,
|
| 36 |
+
third_party_eval_count, independent_verification_ratio,
|
| 37 |
+
reproducibility_status, eval_libraries, latest_source_name,
|
| 38 |
+
params_billions, benchmark_names, score_summary,
|
| 39 |
+
reproducibility_summary, provenance_summary, comparability_summary,
|
| 40 |
+
top_scores, source_urls, detail_urls,
|
| 41 |
+
model_url, release_date, input_modalities, output_modalities,
|
| 42 |
+
architecture, params, inference_engine, inference_platform
|
| 43 |
+
`
|
| 44 |
+
|
| 45 |
+
const EVAL_LIST_COLUMNS = `
|
| 46 |
+
evaluation_id, evaluation_name, canonical_display_name,
|
| 47 |
+
composite_benchmark_key, composite_benchmark_name,
|
| 48 |
+
benchmark_family_key, benchmark_leaf_key, category,
|
| 49 |
+
metric_config, models_count, evaluator_names, source_types,
|
| 50 |
+
latest_source_name, third_party_ratio,
|
| 51 |
+
missing_generation_config_count, best_model, worst_model,
|
| 52 |
+
avg_score, avg_score_norm, has_card, benchmark_card,
|
| 53 |
+
is_aggregated, aggregate_sources, tags,
|
| 54 |
+
metrics_count, metric_names, instance_data, top_score,
|
| 55 |
+
subtasks_count, is_summary_score, summary_eval_ids,
|
| 56 |
+
root_metrics, subtasks, leaderboard_metrics,
|
| 57 |
+
reproducibility_summary, provenance_summary, comparability_summary,
|
| 58 |
+
source_data
|
| 59 |
+
`
|
| 60 |
+
|
| 61 |
+
const CELL_JOIN_COLUMNS = `
|
| 62 |
+
r.*,
|
| 63 |
+
e.evaluation_name AS eval_evaluation_name,
|
| 64 |
+
e.canonical_display_name AS eval_canonical_display_name,
|
| 65 |
+
e.composite_benchmark_key AS eval_composite_benchmark_key,
|
| 66 |
+
e.composite_benchmark_name AS eval_composite_benchmark_name,
|
| 67 |
+
e.benchmark_family_key AS eval_benchmark_family_key,
|
| 68 |
+
e.benchmark_leaf_key AS eval_benchmark_leaf_key,
|
| 69 |
+
e.category AS eval_category,
|
| 70 |
+
e.metric_config AS eval_metric_config,
|
| 71 |
+
e.source_data AS eval_source_data,
|
| 72 |
+
e.benchmark_card AS eval_benchmark_card,
|
| 73 |
+
e.tags AS eval_tags,
|
| 74 |
+
e.is_summary_score AS eval_is_summary_score,
|
| 75 |
+
e.summary_eval_ids AS eval_summary_eval_ids
|
| 76 |
+
`
|
| 77 |
+
|
| 78 |
+
function normalizeDuckDBValue(value: unknown): unknown {
|
| 79 |
+
if (typeof value === "bigint") {
|
| 80 |
+
return Number(value)
|
| 81 |
+
}
|
| 82 |
+
|
| 83 |
+
if (value instanceof Date) {
|
| 84 |
+
return value.toISOString()
|
| 85 |
+
}
|
| 86 |
+
|
| 87 |
+
if (value instanceof Map) {
|
| 88 |
+
return Object.fromEntries(
|
| 89 |
+
Array.from(value.entries()).map(([key, mapValue]) => [String(key), normalizeDuckDBValue(mapValue)])
|
| 90 |
+
)
|
| 91 |
+
}
|
| 92 |
+
|
| 93 |
+
if (Array.isArray(value)) {
|
| 94 |
+
return value.map(normalizeDuckDBValue)
|
| 95 |
+
}
|
| 96 |
+
|
| 97 |
+
if (value && typeof value === "object") {
|
| 98 |
+
const duckValue = value as {
|
| 99 |
+
constructor?: { name?: string }
|
| 100 |
+
entries?: unknown
|
| 101 |
+
items?: unknown
|
| 102 |
+
scale?: unknown
|
| 103 |
+
value?: unknown
|
| 104 |
+
toString?: () => string
|
| 105 |
+
}
|
| 106 |
+
const constructorName = duckValue.constructor?.name ?? ""
|
| 107 |
+
|
| 108 |
+
if (constructorName === "DuckDBStructValue" && duckValue.entries && typeof duckValue.entries === "object") {
|
| 109 |
+
return normalizeDuckDBValue(duckValue.entries)
|
| 110 |
+
}
|
| 111 |
+
|
| 112 |
+
if (
|
| 113 |
+
(constructorName === "DuckDBListValue" || constructorName === "DuckDBArrayValue") &&
|
| 114 |
+
Array.isArray(duckValue.items)
|
| 115 |
+
) {
|
| 116 |
+
return duckValue.items.map(normalizeDuckDBValue)
|
| 117 |
+
}
|
| 118 |
+
|
| 119 |
+
if (constructorName === "DuckDBMapValue" && Array.isArray(duckValue.entries)) {
|
| 120 |
+
return Object.fromEntries(
|
| 121 |
+
duckValue.entries.map((entry) => {
|
| 122 |
+
const pair = entry as { key: unknown; value: unknown }
|
| 123 |
+
return [String(pair.key), normalizeDuckDBValue(pair.value)]
|
| 124 |
+
})
|
| 125 |
+
)
|
| 126 |
+
}
|
| 127 |
+
|
| 128 |
+
if (constructorName === "DuckDBDecimalValue" && typeof duckValue.toString === "function") {
|
| 129 |
+
return Number(duckValue.toString())
|
| 130 |
+
}
|
| 131 |
+
|
| 132 |
+
if (constructorName.startsWith("DuckDB") && typeof duckValue.toString === "function") {
|
| 133 |
+
return duckValue.toString()
|
| 134 |
+
}
|
| 135 |
+
|
| 136 |
+
return Object.fromEntries(
|
| 137 |
+
Object.entries(value).map(([key, objectValue]) => [key, normalizeDuckDBValue(objectValue)])
|
| 138 |
+
)
|
| 139 |
+
}
|
| 140 |
+
|
| 141 |
+
return value
|
| 142 |
+
}
|
| 143 |
+
|
| 144 |
+
async function readRows<T = Row>(sql: string, params: unknown[] = []): Promise<T[]> {
|
| 145 |
+
const connection = await getConnection()
|
| 146 |
+
const reader = params.length > 0
|
| 147 |
+
? await connection.runAndReadAll(sql, params as any[])
|
| 148 |
+
: await connection.runAndReadAll(sql)
|
| 149 |
+
return reader.getRowObjects().map((row) => normalizeDuckDBValue(row) as T)
|
| 150 |
+
}
|
| 151 |
+
|
| 152 |
+
function asNumber(value: unknown, fallback = 0) {
|
| 153 |
+
if (typeof value === "number" && Number.isFinite(value)) return value
|
| 154 |
+
if (typeof value === "bigint") return Number(value)
|
| 155 |
+
if (typeof value === "string" && value.trim() !== "") {
|
| 156 |
+
const parsed = Number(value)
|
| 157 |
+
if (Number.isFinite(parsed)) return parsed
|
| 158 |
+
}
|
| 159 |
+
return fallback
|
| 160 |
+
}
|
| 161 |
+
|
| 162 |
+
function optionalNumber(value: unknown) {
|
| 163 |
+
if (value == null) return undefined
|
| 164 |
+
const parsed = asNumber(value, Number.NaN)
|
| 165 |
+
return Number.isFinite(parsed) ? parsed : undefined
|
| 166 |
+
}
|
| 167 |
+
|
| 168 |
+
function asString(value: unknown, fallback = "") {
|
| 169 |
+
return typeof value === "string" ? value : fallback
|
| 170 |
+
}
|
| 171 |
+
|
| 172 |
+
function optionalString(value: unknown) {
|
| 173 |
+
return typeof value === "string" && value.length > 0 ? value : undefined
|
| 174 |
+
}
|
| 175 |
+
|
| 176 |
+
function asArray<T>(value: unknown): T[] {
|
| 177 |
+
return Array.isArray(value) ? value as T[] : []
|
| 178 |
+
}
|
| 179 |
+
|
| 180 |
+
function normalizeCategory(value: unknown): CategoryType {
|
| 181 |
+
return EVALUATION_CATEGORIES.includes(value as CategoryType)
|
| 182 |
+
? value as CategoryType
|
| 183 |
+
: "General"
|
| 184 |
+
}
|
| 185 |
+
|
| 186 |
+
function emptyEvaluationsByCategory(): Record<CategoryType, BenchmarkEvaluation[]> {
|
| 187 |
+
return EVALUATION_CATEGORIES.reduce((acc, category) => {
|
| 188 |
+
acc[category] = []
|
| 189 |
+
return acc
|
| 190 |
+
}, {} as Record<CategoryType, BenchmarkEvaluation[]>)
|
| 191 |
+
}
|
| 192 |
+
|
| 193 |
+
function sourceMetadataFromRow(row: Row): SourceMetadata {
|
| 194 |
+
if (row.source_metadata && typeof row.source_metadata === "object") {
|
| 195 |
+
return row.source_metadata as SourceMetadata
|
| 196 |
+
}
|
| 197 |
+
|
| 198 |
+
return {
|
| 199 |
+
source_type: "documentation",
|
| 200 |
+
source_organization_name: asString(row.latest_source_name, "Unknown"),
|
| 201 |
+
evaluator_relationship: "other",
|
| 202 |
+
}
|
| 203 |
+
}
|
| 204 |
+
|
| 205 |
+
function sourceDataFromRow(row: Row): BenchmarkEvaluation["source_data"] {
|
| 206 |
+
const sourceData = row.source_data ?? row.eval_source_data
|
| 207 |
+
if (sourceData) {
|
| 208 |
+
return sourceData as BenchmarkEvaluation["source_data"]
|
| 209 |
+
}
|
| 210 |
+
|
| 211 |
+
return {
|
| 212 |
+
dataset_name: asString(row.eval_evaluation_name ?? row.evaluation_name ?? row.benchmark_id, "Unknown dataset"),
|
| 213 |
+
} satisfies SourceData
|
| 214 |
+
}
|
| 215 |
+
|
| 216 |
+
function scoreDetailsFromRow(row: Row): ScoreDetails {
|
| 217 |
+
const details = row.score_details && typeof row.score_details === "object"
|
| 218 |
+
? row.score_details as Partial<ScoreDetails>
|
| 219 |
+
: {}
|
| 220 |
+
const score = asNumber(details.score ?? row.score)
|
| 221 |
+
|
| 222 |
+
return {
|
| 223 |
+
...details,
|
| 224 |
+
score,
|
| 225 |
+
} as ScoreDetails
|
| 226 |
+
}
|
| 227 |
+
|
| 228 |
+
function metricConfigFromRow(row: Row): MetricConfig {
|
| 229 |
+
const config = (row.metric_config ?? row.eval_metric_config ?? {}) as Partial<MetricConfig>
|
| 230 |
+
const scoreType = config.score_type === "binary" || config.score_type === "discrete"
|
| 231 |
+
? config.score_type
|
| 232 |
+
: "continuous"
|
| 233 |
+
|
| 234 |
+
return {
|
| 235 |
+
evaluation_description: asString(
|
| 236 |
+
config.evaluation_description ??
|
| 237 |
+
row.metric_description ??
|
| 238 |
+
row.metric_display_name ??
|
| 239 |
+
row.eval_evaluation_name ??
|
| 240 |
+
row.evaluation_name,
|
| 241 |
+
""
|
| 242 |
+
),
|
| 243 |
+
lower_is_better: Boolean(row.lower_is_better ?? config.lower_is_better ?? false),
|
| 244 |
+
score_type: scoreType,
|
| 245 |
+
min_score: optionalNumber(config.min_score ?? row.min_score),
|
| 246 |
+
max_score: optionalNumber(config.max_score ?? row.max_score),
|
| 247 |
+
unit: optionalString(row.metric_unit ?? config.unit),
|
| 248 |
+
}
|
| 249 |
+
}
|
| 250 |
+
|
| 251 |
+
function modelInfoFromModelRow(row: Row): ModelInfo {
|
| 252 |
+
return {
|
| 253 |
+
name: asString(row.model_name ?? row.model_family_name ?? row.model_id, "Unknown model"),
|
| 254 |
+
id: asString(row.model_id ?? row.id ?? row.route_id, "unknown-model"),
|
| 255 |
+
developer: optionalString(row.developer),
|
| 256 |
+
inference_platform: optionalString(row.inference_platform),
|
| 257 |
+
inference_engine: optionalString(row.inference_engine),
|
| 258 |
+
architecture: optionalString(row.architecture),
|
| 259 |
+
parameter_count: optionalString(row.params),
|
| 260 |
+
release_date: optionalString(row.release_date),
|
| 261 |
+
model_url: optionalString(row.model_url),
|
| 262 |
+
additional_details: {
|
| 263 |
+
params_billions: row.params_billions,
|
| 264 |
+
},
|
| 265 |
+
modalities: {
|
| 266 |
+
input: asArray<string>(row.input_modalities),
|
| 267 |
+
output: asArray<string>(row.output_modalities),
|
| 268 |
+
},
|
| 269 |
+
}
|
| 270 |
+
}
|
| 271 |
+
|
| 272 |
+
function resultFromCell(row: Row): EvaluationResult {
|
| 273 |
+
const scoreDetails = scoreDetailsFromRow(row)
|
| 274 |
+
const generationConfig = row.generation_config as GenerationConfig | undefined
|
| 275 |
+
const annotations = row.evalcards_annotations
|
| 276 |
+
|
| 277 |
+
return {
|
| 278 |
+
evaluation_name: asString(row.metric_display_name ?? row.eval_evaluation_name ?? row.metric_id, "Score"),
|
| 279 |
+
display_name: optionalString(row.metric_display_name),
|
| 280 |
+
canonical_display_name: optionalString(row.metric_display_name),
|
| 281 |
+
metric_summary_id: optionalString(row.metric_summary_id),
|
| 282 |
+
metric_key: optionalString(row.metric_id),
|
| 283 |
+
evaluation_timestamp: asString(row.evaluation_timestamp, ""),
|
| 284 |
+
source_data: sourceDataFromRow(row),
|
| 285 |
+
metric_config: metricConfigFromRow(row),
|
| 286 |
+
score_details: scoreDetails,
|
| 287 |
+
generation_config: generationConfig,
|
| 288 |
+
detailed_evaluation_results_url: optionalString(row.instance_file_path),
|
| 289 |
+
evalcards: annotations ? { annotations } : undefined,
|
| 290 |
+
}
|
| 291 |
+
}
|
| 292 |
+
|
| 293 |
+
function reshapeCellToModelResult(row: Row): ModelResultForBenchmark {
|
| 294 |
+
const scoreDetails = scoreDetailsFromRow(row)
|
| 295 |
+
|
| 296 |
+
return {
|
| 297 |
+
model_info: (row.model_info ?? modelInfoFromModelRow(row)) as ModelInfo,
|
| 298 |
+
model_route_id: optionalString(row.model_route_id),
|
| 299 |
+
score: scoreDetails.score,
|
| 300 |
+
score_details: scoreDetails,
|
| 301 |
+
evaluation_timestamp: asString(row.evaluation_timestamp, ""),
|
| 302 |
+
source_metadata: sourceMetadataFromRow(row),
|
| 303 |
+
source_data: sourceDataFromRow(row),
|
| 304 |
+
source_record_url: optionalString(row.source_record_url),
|
| 305 |
+
aggregate_components: asArray<NonNullable<ModelResultForBenchmark["aggregate_components"]>[number]>(
|
| 306 |
+
row.aggregate_components
|
| 307 |
+
),
|
| 308 |
+
result: resultFromCell(row),
|
| 309 |
+
}
|
| 310 |
+
}
|
| 311 |
+
|
| 312 |
+
function reshapeCellToBenchmarkEvaluation(row: Row): BenchmarkEvaluation {
|
| 313 |
+
const result = resultFromCell(row)
|
| 314 |
+
const modelInfo = (row.model_info ?? modelInfoFromModelRow(row)) as ModelInfo
|
| 315 |
+
|
| 316 |
+
return {
|
| 317 |
+
schema_version: "1.0",
|
| 318 |
+
eval_summary_id: optionalString(row.evaluation_id),
|
| 319 |
+
evaluation_id: asString(row.evaluation_id ?? row.benchmark_id, "unknown-evaluation"),
|
| 320 |
+
retrieved_timestamp: asString(row.evaluation_timestamp, ""),
|
| 321 |
+
benchmark: optionalString(row.eval_evaluation_name ?? row.benchmark_id),
|
| 322 |
+
display_name: optionalString(row.eval_evaluation_name),
|
| 323 |
+
canonical_display_name: optionalString(row.eval_canonical_display_name),
|
| 324 |
+
category: normalizeCategory(row.eval_category ?? row.category),
|
| 325 |
+
benchmark_family_key: optionalString(row.eval_benchmark_family_key),
|
| 326 |
+
benchmark_family_name: optionalString(row.eval_composite_benchmark_name),
|
| 327 |
+
benchmark_parent_key: optionalString(row.eval_composite_benchmark_key),
|
| 328 |
+
benchmark_parent_name: optionalString(row.eval_composite_benchmark_name),
|
| 329 |
+
benchmark_leaf_key: optionalString(row.eval_benchmark_leaf_key),
|
| 330 |
+
benchmark_leaf_name: optionalString(row.eval_evaluation_name),
|
| 331 |
+
is_summary_score: Boolean(row.eval_is_summary_score ?? row.is_summary_score),
|
| 332 |
+
source_data: sourceDataFromRow(row),
|
| 333 |
+
source_metadata: sourceMetadataFromRow(row),
|
| 334 |
+
eval_library: row.eval_library,
|
| 335 |
+
model_info: modelInfo,
|
| 336 |
+
generation_config: row.generation_config,
|
| 337 |
+
evaluation_results: [result],
|
| 338 |
+
}
|
| 339 |
+
}
|
| 340 |
+
|
| 341 |
+
function modelSummaryFromRows(modelRow: Row, cellRows: Row[]): ModelEvaluationSummary {
|
| 342 |
+
const evaluationsByCategory = emptyEvaluationsByCategory()
|
| 343 |
+
for (const cellRow of cellRows) {
|
| 344 |
+
const evaluation = reshapeCellToBenchmarkEvaluation(cellRow)
|
| 345 |
+
const category = normalizeCategory(evaluation.category)
|
| 346 |
+
evaluationsByCategory[category].push(evaluation)
|
| 347 |
+
}
|
| 348 |
+
|
| 349 |
+
const categoriesCovered = asArray<CategoryType>(modelRow.categories).filter((category) =>
|
| 350 |
+
EVALUATION_CATEGORIES.includes(category)
|
| 351 |
+
)
|
| 352 |
+
const modelInfo = (modelRow.model_info ?? modelInfoFromModelRow(modelRow)) as ModelInfo
|
| 353 |
+
const totalEvaluations = asNumber(modelRow.total_evaluations ?? modelRow.evaluations_count)
|
| 354 |
+
const lastUpdated = asString(modelRow.last_updated ?? modelRow.latest_timestamp, "")
|
| 355 |
+
const rawModelIds = asArray<string>(modelRow.raw_model_ids)
|
| 356 |
+
|
| 357 |
+
const core = {
|
| 358 |
+
model_info: modelInfo,
|
| 359 |
+
evaluations_by_category: evaluationsByCategory,
|
| 360 |
+
total_evaluations: totalEvaluations,
|
| 361 |
+
last_updated: lastUpdated,
|
| 362 |
+
categories_covered: categoriesCovered.length > 0
|
| 363 |
+
? categoriesCovered
|
| 364 |
+
: EVALUATION_CATEGORIES.filter((category) => evaluationsByCategory[category].length > 0),
|
| 365 |
+
reproducibility_summary: modelRow.reproducibility_summary,
|
| 366 |
+
provenance_summary: modelRow.provenance_summary,
|
| 367 |
+
comparability_summary: modelRow.comparability_summary,
|
| 368 |
+
}
|
| 369 |
+
|
| 370 |
+
const variants = asArray<Row>(modelRow.variants).map((variant, index) => ({
|
| 371 |
+
...core,
|
| 372 |
+
...variant,
|
| 373 |
+
variant_id: asString(variant.variant_id ?? variant.variant_key, `variant-${index}`),
|
| 374 |
+
variant_key: asString(variant.variant_key, `variant-${index}`),
|
| 375 |
+
variant_label: asString(variant.variant_label ?? variant.variant_display_name, "Default"),
|
| 376 |
+
variant_display_name: asString(variant.variant_display_name ?? variant.variant_label ?? modelRow.model_name, modelRow.model_name),
|
| 377 |
+
raw_model_ids: asArray<string>(variant.raw_model_ids),
|
| 378 |
+
family_id: asString(variant.family_id ?? modelRow.model_family_id, modelRow.model_family_id),
|
| 379 |
+
family_name: asString(variant.family_name ?? modelRow.model_family_name, modelRow.model_family_name),
|
| 380 |
+
total_evaluations: asNumber(variant.total_evaluations ?? totalEvaluations),
|
| 381 |
+
last_updated: asString(variant.last_updated ?? lastUpdated, lastUpdated),
|
| 382 |
+
categories_covered: asArray<CategoryType>(variant.categories_covered).length > 0
|
| 383 |
+
? asArray<CategoryType>(variant.categories_covered)
|
| 384 |
+
: core.categories_covered,
|
| 385 |
+
model_info: {
|
| 386 |
+
...modelInfo,
|
| 387 |
+
name: asString(variant.variant_display_name ?? variant.variant_label ?? modelInfo.name, modelInfo.name),
|
| 388 |
+
},
|
| 389 |
+
})) as ModelVariantSummary[]
|
| 390 |
+
|
| 391 |
+
return {
|
| 392 |
+
...core,
|
| 393 |
+
model_family_id: asString(modelRow.model_family_id ?? modelRow.model_id, modelRow.model_id),
|
| 394 |
+
model_route_id: asString(modelRow.model_route_id ?? modelRow.route_id, modelRow.route_id),
|
| 395 |
+
model_family_name: asString(modelRow.model_family_name ?? modelRow.model_name, modelRow.model_name),
|
| 396 |
+
raw_model_ids: rawModelIds.length > 0 ? rawModelIds : [asString(modelRow.model_id, "")].filter(Boolean),
|
| 397 |
+
variants,
|
| 398 |
+
}
|
| 399 |
+
}
|
| 400 |
+
|
| 401 |
+
async function getModelEvaluationRows(modelId: string): Promise<Row[]> {
|
| 402 |
+
return readRows<Row>(
|
| 403 |
+
`SELECT ${CELL_JOIN_COLUMNS}
|
| 404 |
+
FROM eval_results_view r
|
| 405 |
+
LEFT JOIN evals_view e ON r.evaluation_id = e.evaluation_id
|
| 406 |
+
WHERE r.model_id = ?
|
| 407 |
+
AND r.score IS NOT NULL
|
| 408 |
+
ORDER BY r.category, r.percentile DESC NULLS LAST`,
|
| 409 |
+
[modelId]
|
| 410 |
+
)
|
| 411 |
+
}
|
| 412 |
+
|
| 413 |
+
export async function getModelCards(): Promise<EvaluationCardData[]> {
|
| 414 |
+
return readRows<EvaluationCardData>(
|
| 415 |
+
`SELECT ${MODEL_CARD_COLUMNS}
|
| 416 |
+
FROM models_view
|
| 417 |
+
ORDER BY latest_timestamp DESC NULLS LAST`
|
| 418 |
+
)
|
| 419 |
+
}
|
| 420 |
+
|
| 421 |
+
export async function getModelCardsLite(): Promise<EvaluationCardData[]> {
|
| 422 |
+
return readRows<EvaluationCardData>(
|
| 423 |
+
`SELECT ${MODEL_CARD_COLUMNS}
|
| 424 |
+
FROM models_view
|
| 425 |
+
ORDER BY benchmarks_count DESC NULLS LAST, evaluations_count DESC NULLS LAST, model_name ASC`
|
| 426 |
+
)
|
| 427 |
+
}
|
| 428 |
+
|
| 429 |
+
export async function getEvalListData(): Promise<{
|
| 430 |
+
evals: BenchmarkEvalListItem[]
|
| 431 |
+
totalModels: number
|
| 432 |
+
}> {
|
| 433 |
+
const [evals, countRows] = await Promise.all([
|
| 434 |
+
readRows<BenchmarkEvalListItem>(
|
| 435 |
+
`SELECT ${EVAL_LIST_COLUMNS}
|
| 436 |
+
FROM evals_view
|
| 437 |
+
ORDER BY evaluation_name ASC`
|
| 438 |
+
),
|
| 439 |
+
readRows<{ n: number }>("SELECT COUNT(*) AS n FROM models_view"),
|
| 440 |
+
])
|
| 441 |
+
|
| 442 |
+
return {
|
| 443 |
+
evals,
|
| 444 |
+
totalModels: asNumber(countRows[0]?.n),
|
| 445 |
+
}
|
| 446 |
+
}
|
| 447 |
+
|
| 448 |
+
export async function getEvalListLiteData(): Promise<{
|
| 449 |
+
evals: BenchmarkEvalListItem[]
|
| 450 |
+
totalModels: number
|
| 451 |
+
}> {
|
| 452 |
+
return getEvalListData()
|
| 453 |
+
}
|
| 454 |
+
|
| 455 |
+
export async function getEvalList() {
|
| 456 |
+
const { evals } = await getEvalListData()
|
| 457 |
+
return evals
|
| 458 |
+
}
|
| 459 |
+
|
| 460 |
+
export async function getDashboardData() {
|
| 461 |
+
const [models, evals] = await Promise.all([
|
| 462 |
+
getModelCards(),
|
| 463 |
+
getEvalList(),
|
| 464 |
+
])
|
| 465 |
+
return { models, evals }
|
| 466 |
+
}
|
| 467 |
+
|
| 468 |
+
export async function getModelSummaryById(routeId: string): Promise<ModelEvaluationSummary | null> {
|
| 469 |
+
const rows = await readRows<Row>(
|
| 470 |
+
`SELECT *
|
| 471 |
+
FROM models_view
|
| 472 |
+
WHERE route_id = ? OR model_route_id = ? OR model_family_id = ? OR model_id = ?
|
| 473 |
+
LIMIT 1`,
|
| 474 |
+
[routeId, routeId, routeId, routeId]
|
| 475 |
+
)
|
| 476 |
+
const modelRow = rows[0]
|
| 477 |
+
if (!modelRow) return null
|
| 478 |
+
|
| 479 |
+
const cellRows = await getModelEvaluationRows(asString(modelRow.model_id, routeId))
|
| 480 |
+
return modelSummaryFromRows(modelRow, cellRows)
|
| 481 |
+
}
|
| 482 |
+
|
| 483 |
+
export async function getEvalSummaryById(evalId: string): Promise<BenchmarkEvalSummary | null> {
|
| 484 |
+
const evalRows = await readRows<Row>(
|
| 485 |
+
"SELECT * FROM evals_view WHERE evaluation_id = ? LIMIT 1",
|
| 486 |
+
[evalId]
|
| 487 |
+
)
|
| 488 |
+
const evalRow = evalRows[0]
|
| 489 |
+
if (!evalRow) return null
|
| 490 |
+
|
| 491 |
+
let cellRows = await readRows<Row>(
|
| 492 |
+
`SELECT ${CELL_JOIN_COLUMNS}
|
| 493 |
+
FROM eval_results_view r
|
| 494 |
+
LEFT JOIN evals_view e ON r.evaluation_id = e.evaluation_id
|
| 495 |
+
WHERE r.evaluation_id = ?
|
| 496 |
+
AND r.metric_id = (SELECT primary_metric_id FROM evals_view WHERE evaluation_id = ?)
|
| 497 |
+
AND r.score IS NOT NULL
|
| 498 |
+
ORDER BY r.position ASC NULLS LAST`,
|
| 499 |
+
[evalId, evalId]
|
| 500 |
+
)
|
| 501 |
+
|
| 502 |
+
if (cellRows.length === 0) {
|
| 503 |
+
cellRows = await readRows<Row>(
|
| 504 |
+
`SELECT ${CELL_JOIN_COLUMNS}
|
| 505 |
+
FROM eval_results_view r
|
| 506 |
+
LEFT JOIN evals_view e ON r.evaluation_id = e.evaluation_id
|
| 507 |
+
WHERE r.evaluation_id = ?
|
| 508 |
+
AND r.score IS NOT NULL
|
| 509 |
+
ORDER BY r.position ASC NULLS LAST`,
|
| 510 |
+
[evalId]
|
| 511 |
+
)
|
| 512 |
+
}
|
| 513 |
+
|
| 514 |
+
return {
|
| 515 |
+
...evalRow,
|
| 516 |
+
model_results: cellRows.map(reshapeCellToModelResult),
|
| 517 |
+
} as BenchmarkEvalSummary
|
| 518 |
+
}
|
| 519 |
+
|
| 520 |
+
export async function getDeveloperList(): Promise<DeveloperListEntry[]> {
|
| 521 |
+
const headline = await fetchHeadline()
|
| 522 |
+
return [...(headline.developers ?? [])].sort((a, b) => a.developer.localeCompare(b.developer))
|
| 523 |
+
}
|
| 524 |
+
|
| 525 |
+
export async function getDeveloperSummaryById(routeId: string) {
|
| 526 |
+
const developers = await getDeveloperList()
|
| 527 |
+
const developer = developers.find((entry) => entry.route_id === routeId)
|
| 528 |
+
if (!developer) return null
|
| 529 |
+
|
| 530 |
+
const models = await readRows<EvaluationCardData>(
|
| 531 |
+
`SELECT ${MODEL_CARD_COLUMNS}
|
| 532 |
+
FROM models_view
|
| 533 |
+
WHERE developer = ?
|
| 534 |
+
ORDER BY benchmarks_count DESC NULLS LAST, evaluations_count DESC NULLS LAST, model_name ASC`,
|
| 535 |
+
[developer.developer]
|
| 536 |
+
)
|
| 537 |
+
|
| 538 |
+
return {
|
| 539 |
+
...developer,
|
| 540 |
+
models,
|
| 541 |
+
}
|
| 542 |
+
}
|
| 543 |
+
|
| 544 |
+
export async function getBenchmarkMetadataMap(): Promise<Record<string, BenchmarkCard>> {
|
| 545 |
+
const rows = await readRows<Row>(
|
| 546 |
+
`SELECT evaluation_id, evaluation_name, composite_benchmark_key, benchmark_card
|
| 547 |
+
FROM evals_view
|
| 548 |
+
WHERE benchmark_card IS NOT NULL`
|
| 549 |
+
)
|
| 550 |
+
const result: Record<string, BenchmarkCard> = {}
|
| 551 |
+
|
| 552 |
+
for (const row of rows) {
|
| 553 |
+
const card = row.benchmark_card as BenchmarkCard | null | undefined
|
| 554 |
+
if (!card) continue
|
| 555 |
+
|
| 556 |
+
const keys = [
|
| 557 |
+
row.evaluation_id,
|
| 558 |
+
row.evaluation_name,
|
| 559 |
+
row.composite_benchmark_key,
|
| 560 |
+
card.benchmark_details?.name,
|
| 561 |
+
].filter((key): key is string => typeof key === "string" && key.length > 0)
|
| 562 |
+
|
| 563 |
+
for (const key of keys) {
|
| 564 |
+
result[key] = card
|
| 565 |
+
}
|
| 566 |
+
}
|
| 567 |
+
|
| 568 |
+
return result
|
| 569 |
+
}
|
scripts/cache-hf-data.mjs
CHANGED
|
@@ -18,6 +18,13 @@ import { promisify } from "util"
|
|
| 18 |
const root = path.resolve(new URL(import.meta.url).pathname, "..", "..")
|
| 19 |
const cacheDir = path.join(root, ".cache", "hf-data")
|
| 20 |
const publicDir = path.join(root, "public")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 21 |
const HF_DATASET_REPO = process.env.HF_DATASET_REPO?.trim()
|
| 22 |
|| "https://huggingface.co/datasets/evaleval/card_backend"
|
| 23 |
const HF_RESOLVE_BASE = `${HF_DATASET_REPO}/resolve/main`
|
|
|
|
| 18 |
const root = path.resolve(new URL(import.meta.url).pathname, "..", "..")
|
| 19 |
const cacheDir = path.join(root, ".cache", "hf-data")
|
| 20 |
const publicDir = path.join(root, "public")
|
| 21 |
+
const dataBackend = process.env.DATA_BACKEND?.trim().toLowerCase()
|
| 22 |
+
if (dataBackend === "v2" || dataBackend === "stage-j") {
|
| 23 |
+
await fs.mkdir(cacheDir, { recursive: true })
|
| 24 |
+
console.log("[cache-hf-data] DATA_BACKEND=v2: skipping legacy HF cache; runtime reads SNAPSHOT_URL")
|
| 25 |
+
process.exit(0)
|
| 26 |
+
}
|
| 27 |
+
|
| 28 |
const HF_DATASET_REPO = process.env.HF_DATASET_REPO?.trim()
|
| 29 |
|| "https://huggingface.co/datasets/evaleval/card_backend"
|
| 30 |
const HF_RESOLVE_BASE = `${HF_DATASET_REPO}/resolve/main`
|
tests/duckdb-data.test.ts
CHANGED
|
@@ -12,27 +12,13 @@ function sqlString(value: string) {
|
|
| 12 |
}
|
| 13 |
|
| 14 |
async function writeParquetPayload(outputDir: string, fileName: string, payloads: unknown[]) {
|
| 15 |
-
const parquetDir = path.join(outputDir, "
|
| 16 |
await mkdir(parquetDir, { recursive: true })
|
| 17 |
|
| 18 |
const selects = payloads
|
| 19 |
-
.map((payload
|
| 20 |
-
const record = payload as Record<string, unknown>
|
| 21 |
const payloadJson = JSON.stringify(payload)
|
| 22 |
-
return
|
| 23 |
-
`SELECT 'model_card_lite' AS record_type`,
|
| 24 |
-
`${sqlString(String(record.model_route_id ?? index))} AS model_route_id`,
|
| 25 |
-
`${sqlString(String(record.model_family_id ?? ""))} AS model_family_id`,
|
| 26 |
-
`${sqlString(String(record.developer ?? ""))} AS developer`,
|
| 27 |
-
`NULL AS eval_summary_id`,
|
| 28 |
-
`NULL AS developer_route_id`,
|
| 29 |
-
`NULL AS category`,
|
| 30 |
-
`NULL AS benchmark_family_key`,
|
| 31 |
-
`${Number(record.benchmark_family_count ?? 0)} AS models_count`,
|
| 32 |
-
`${Number(record.total_evaluations ?? 0)} AS total_evaluations`,
|
| 33 |
-
`${sqlString(String(record.last_updated ?? ""))} AS last_updated`,
|
| 34 |
-
`${sqlString(payloadJson)} AS payload_json`,
|
| 35 |
-
].join(", ")
|
| 36 |
})
|
| 37 |
.join(" UNION ALL ")
|
| 38 |
|
|
@@ -49,22 +35,36 @@ describe("DuckDB local data backend", () => {
|
|
| 49 |
process.env.LOCAL_PIPELINE_OUTPUT = outputDir
|
| 50 |
await writeParquetPayload(outputDir, "model_cards_lite.parquet", [
|
| 51 |
{
|
| 52 |
-
|
| 53 |
-
|
| 54 |
-
|
| 55 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 56 |
params_billions: 100,
|
| 57 |
-
total_evaluations: 3,
|
| 58 |
-
benchmark_count: 2,
|
| 59 |
-
benchmark_family_count: 2,
|
| 60 |
-
categories_covered: ["reasoning"],
|
| 61 |
-
last_updated: "2026-01-01T00:00:00Z",
|
| 62 |
-
variants: [],
|
| 63 |
score_summary: { count: 1, min: 0.7, max: 0.9, average: 0.8 },
|
| 64 |
benchmark_names: ["mmlu"],
|
| 65 |
-
|
| 66 |
{ benchmark: "mmlu", score: 0.9, metric: "accuracy" },
|
| 67 |
],
|
|
|
|
|
|
|
| 68 |
},
|
| 69 |
])
|
| 70 |
|
|
@@ -93,7 +93,7 @@ describe("DuckDB local data backend", () => {
|
|
| 93 |
try {
|
| 94 |
process.env.LOCAL_PIPELINE_OUTPUT = outputDir
|
| 95 |
await expect(getModelCardsLiteFromDuckDB()).rejects.toThrow(
|
| 96 |
-
/
|
| 97 |
)
|
| 98 |
} finally {
|
| 99 |
if (previousOutput == null) {
|
|
|
|
| 12 |
}
|
| 13 |
|
| 14 |
async function writeParquetPayload(outputDir: string, fileName: string, payloads: unknown[]) {
|
| 15 |
+
const parquetDir = path.join(outputDir, "duckdb", "v1")
|
| 16 |
await mkdir(parquetDir, { recursive: true })
|
| 17 |
|
| 18 |
const selects = payloads
|
| 19 |
+
.map((payload) => {
|
|
|
|
| 20 |
const payloadJson = JSON.stringify(payload)
|
| 21 |
+
return `SELECT ${sqlString(payloadJson)} AS payload_json`
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 22 |
})
|
| 23 |
.join(" UNION ALL ")
|
| 24 |
|
|
|
|
| 35 |
process.env.LOCAL_PIPELINE_OUTPUT = outputDir
|
| 36 |
await writeParquetPayload(outputDir, "model_cards_lite.parquet", [
|
| 37 |
{
|
| 38 |
+
id: "openai/gpt-5",
|
| 39 |
+
route_id: "openai__gpt-5",
|
| 40 |
+
model_name: "GPT 5",
|
| 41 |
+
model_id: "openai/gpt-5",
|
| 42 |
+
canonical_model_name: "GPT 5",
|
| 43 |
+
developer: "OpenAI",
|
| 44 |
+
evaluations_count: 3,
|
| 45 |
+
benchmarks_count: 2,
|
| 46 |
+
variant_count: 1,
|
| 47 |
+
categories: ["Reasoning"],
|
| 48 |
+
category_stats: { General: 0, Reasoning: 2, Agentic: 0, Safety: 0, Knowledge: 0 },
|
| 49 |
+
latest_timestamp: "2026-01-01T00:00:00Z",
|
| 50 |
+
evaluator_count: 1,
|
| 51 |
+
evaluator_names: ["OpenAI"],
|
| 52 |
+
source_type_count: 1,
|
| 53 |
+
source_types: ["documentation"],
|
| 54 |
+
evidence_count: 3,
|
| 55 |
+
missing_generation_config_count: 0,
|
| 56 |
+
third_party_eval_count: 0,
|
| 57 |
+
independent_verification_ratio: 0,
|
| 58 |
+
reproducibility_status: "complete",
|
| 59 |
+
eval_libraries: [],
|
| 60 |
params_billions: 100,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 61 |
score_summary: { count: 1, min: 0.7, max: 0.9, average: 0.8 },
|
| 62 |
benchmark_names: ["mmlu"],
|
| 63 |
+
top_scores: [
|
| 64 |
{ benchmark: "mmlu", score: 0.9, metric: "accuracy" },
|
| 65 |
],
|
| 66 |
+
source_urls: [],
|
| 67 |
+
detail_urls: [],
|
| 68 |
},
|
| 69 |
])
|
| 70 |
|
|
|
|
| 93 |
try {
|
| 94 |
process.env.LOCAL_PIPELINE_OUTPUT = outputDir
|
| 95 |
await expect(getModelCardsLiteFromDuckDB()).rejects.toThrow(
|
| 96 |
+
/duckdb\/v1\/model_cards_lite\.parquet/
|
| 97 |
)
|
| 98 |
} finally {
|
| 99 |
if (previousOutput == null) {
|
tests/view-data.test.ts
ADDED
|
@@ -0,0 +1,464 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import { mkdir, mkdtemp, rm, writeFile } from "fs/promises"
|
| 2 |
+
import os from "os"
|
| 3 |
+
import path from "path"
|
| 4 |
+
|
| 5 |
+
import { DuckDBConnection } from "@duckdb/node-api"
|
| 6 |
+
import { describe, expect, it } from "vitest"
|
| 7 |
+
|
| 8 |
+
function sqlString(value: string) {
|
| 9 |
+
return `'${value.replace(/'/g, "''")}'`
|
| 10 |
+
}
|
| 11 |
+
|
| 12 |
+
async function copyParquet(connection: DuckDBConnection, sql: string, outputPath: string) {
|
| 13 |
+
await connection.run(`COPY (${sql}) TO ${sqlString(outputPath)} (FORMAT parquet)`)
|
| 14 |
+
}
|
| 15 |
+
|
| 16 |
+
async function writeSyntheticStageJSnapshot(snapshotDir: string) {
|
| 17 |
+
await mkdir(snapshotDir, { recursive: true })
|
| 18 |
+
const connection = await DuckDBConnection.create()
|
| 19 |
+
|
| 20 |
+
await copyParquet(
|
| 21 |
+
connection,
|
| 22 |
+
`
|
| 23 |
+
SELECT
|
| 24 |
+
TIMESTAMP '2026-05-03 00:00:00' AS snapshot_id,
|
| 25 |
+
'openai/gpt-5' AS model_id,
|
| 26 |
+
'openai/gpt-5' AS id,
|
| 27 |
+
'openai%2Fgpt-5' AS route_id,
|
| 28 |
+
'openai%2Fgpt-5' AS model_route_id,
|
| 29 |
+
'openai/gpt-5' AS model_family_id,
|
| 30 |
+
'GPT 5' AS model_name,
|
| 31 |
+
'GPT 5' AS canonical_model_name,
|
| 32 |
+
'GPT 5' AS model_family_name,
|
| 33 |
+
'OpenAI' AS developer,
|
| 34 |
+
DATE '2026-01-01' AS release_date,
|
| 35 |
+
'https://example.test/model' AS model_url,
|
| 36 |
+
'transformer' AS architecture,
|
| 37 |
+
'100B' AS params,
|
| 38 |
+
100.0 AS params_billions,
|
| 39 |
+
['text']::VARCHAR[] AS input_modalities,
|
| 40 |
+
['text']::VARCHAR[] AS output_modalities,
|
| 41 |
+
'engine' AS inference_engine,
|
| 42 |
+
'platform' AS inference_platform,
|
| 43 |
+
1::BIGINT AS evaluations_count,
|
| 44 |
+
1::BIGINT AS benchmarks_count,
|
| 45 |
+
1::INTEGER AS variant_count,
|
| 46 |
+
1::BIGINT AS evaluator_count,
|
| 47 |
+
['OpenAI']::VARCHAR[] AS evaluator_names,
|
| 48 |
+
1::INTEGER AS source_type_count,
|
| 49 |
+
['documentation']::VARCHAR[] AS source_types,
|
| 50 |
+
0::BIGINT AS third_party_eval_count,
|
| 51 |
+
0.0 AS independent_verification_ratio,
|
| 52 |
+
1::BIGINT AS evidence_count,
|
| 53 |
+
0::INTEGER AS missing_generation_config_count,
|
| 54 |
+
TIMESTAMP '2026-05-03 00:00:00' AS latest_timestamp,
|
| 55 |
+
'OpenAI' AS latest_source_name,
|
| 56 |
+
['MMLU']::VARCHAR[] AS benchmark_names,
|
| 57 |
+
['Reasoning']::VARCHAR[] AS categories,
|
| 58 |
+
struct_pack("General" := 0, "Reasoning" := 1, "Agentic" := 0, "Safety" := 0, "Knowledge" := 0) AS category_stats,
|
| 59 |
+
'complete' AS reproducibility_status,
|
| 60 |
+
struct_pack(results_total := 1, has_reproducibility_gap_count := 0, populated_ratio_avg := 1.0) AS reproducibility_summary,
|
| 61 |
+
struct_pack(
|
| 62 |
+
total_results := 1,
|
| 63 |
+
total_groups := 1,
|
| 64 |
+
multi_source_groups := 0,
|
| 65 |
+
first_party_only_groups := 1,
|
| 66 |
+
source_type_distribution := struct_pack(first_party := 1, third_party := 0, collaborative := 0, unspecified := 0)
|
| 67 |
+
) AS provenance_summary,
|
| 68 |
+
struct_pack(
|
| 69 |
+
total_groups := 1,
|
| 70 |
+
groups_with_variant_check := 0,
|
| 71 |
+
groups_with_cross_party_check := 0,
|
| 72 |
+
variant_divergent_count := 0,
|
| 73 |
+
cross_party_divergent_count := 0
|
| 74 |
+
) AS comparability_summary,
|
| 75 |
+
[struct_pack(name := 'openai-evals', version := '1.0', fork := NULL::VARCHAR)] AS eval_libraries,
|
| 76 |
+
struct_pack(count := 1, min := 0.8, max := 0.8, average := 0.8) AS score_summary,
|
| 77 |
+
[struct_pack(benchmark := 'MMLU', benchmarkKey := 'mmlu', score := 0.8, metric := 'accuracy')] AS top_scores,
|
| 78 |
+
['https://example.test/source']::VARCHAR[] AS source_urls,
|
| 79 |
+
[]::VARCHAR[] AS detail_urls,
|
| 80 |
+
[struct_pack(
|
| 81 |
+
variant_id := 'default',
|
| 82 |
+
variant_key := 'default',
|
| 83 |
+
variant_label := 'Default',
|
| 84 |
+
variant_display_name := 'GPT 5',
|
| 85 |
+
raw_model_ids := ['openai/gpt-5']::VARCHAR[],
|
| 86 |
+
family_id := 'openai/gpt-5',
|
| 87 |
+
family_name := 'GPT 5',
|
| 88 |
+
version_date := NULL::VARCHAR,
|
| 89 |
+
version_qualifier := NULL::VARCHAR,
|
| 90 |
+
total_evaluations := 1,
|
| 91 |
+
last_updated := TIMESTAMP '2026-05-03 00:00:00',
|
| 92 |
+
categories_covered := ['Reasoning']::VARCHAR[]
|
| 93 |
+
)] AS variants,
|
| 94 |
+
['openai/gpt-5']::VARCHAR[] AS raw_model_ids
|
| 95 |
+
`,
|
| 96 |
+
path.join(snapshotDir, "models_view.parquet")
|
| 97 |
+
)
|
| 98 |
+
|
| 99 |
+
await copyParquet(
|
| 100 |
+
connection,
|
| 101 |
+
`
|
| 102 |
+
SELECT
|
| 103 |
+
TIMESTAMP '2026-05-03 00:00:00' AS snapshot_id,
|
| 104 |
+
'mmlu' AS evaluation_id,
|
| 105 |
+
'mmlu' AS benchmark_id,
|
| 106 |
+
'accuracy' AS primary_metric_id,
|
| 107 |
+
'MMLU' AS evaluation_name,
|
| 108 |
+
'MMLU' AS canonical_display_name,
|
| 109 |
+
'mmlu' AS composite_benchmark_key,
|
| 110 |
+
'MMLU' AS composite_benchmark_name,
|
| 111 |
+
'mmlu' AS benchmark_family_key,
|
| 112 |
+
'mmlu' AS benchmark_leaf_key,
|
| 113 |
+
'Reasoning' AS category,
|
| 114 |
+
struct_pack(
|
| 115 |
+
evaluation_description := 'Accuracy on MMLU',
|
| 116 |
+
lower_is_better := false,
|
| 117 |
+
score_type := 'continuous',
|
| 118 |
+
min_score := 0.0,
|
| 119 |
+
max_score := 1.0,
|
| 120 |
+
unit := 'proportion'
|
| 121 |
+
) AS metric_config,
|
| 122 |
+
1::BIGINT AS models_count,
|
| 123 |
+
['OpenAI']::VARCHAR[] AS evaluator_names,
|
| 124 |
+
['documentation']::VARCHAR[] AS source_types,
|
| 125 |
+
'OpenAI' AS latest_source_name,
|
| 126 |
+
0.0 AS third_party_ratio,
|
| 127 |
+
0::INTEGER AS missing_generation_config_count,
|
| 128 |
+
struct_pack(name := 'GPT 5', score := 0.8) AS best_model,
|
| 129 |
+
struct_pack(name := 'GPT 5', score := 0.8) AS worst_model,
|
| 130 |
+
0.8 AS avg_score,
|
| 131 |
+
0.8 AS avg_score_norm,
|
| 132 |
+
0.8 AS top_score,
|
| 133 |
+
false AS has_card,
|
| 134 |
+
NULL AS benchmark_card,
|
| 135 |
+
false AS is_aggregated,
|
| 136 |
+
[] AS aggregate_sources,
|
| 137 |
+
false AS is_summary_score,
|
| 138 |
+
[]::VARCHAR[] AS summary_eval_ids,
|
| 139 |
+
struct_pack(domains := ['knowledge']::VARCHAR[], languages := ['en']::VARCHAR[], tasks := ['qa']::VARCHAR[]) AS tags,
|
| 140 |
+
struct_pack(
|
| 141 |
+
dataset_name := 'MMLU',
|
| 142 |
+
source_type := 'documentation',
|
| 143 |
+
hf_repo := NULL::VARCHAR,
|
| 144 |
+
hf_split := NULL::VARCHAR,
|
| 145 |
+
samples_number := 10,
|
| 146 |
+
url := ['https://example.test/mmlu']::VARCHAR[],
|
| 147 |
+
dataset_url := 'https://example.test/mmlu',
|
| 148 |
+
dataset_version := 'v1'
|
| 149 |
+
) AS source_data,
|
| 150 |
+
struct_pack(results_total := 1, has_reproducibility_gap_count := 0, populated_ratio_avg := 1.0) AS reproducibility_summary,
|
| 151 |
+
struct_pack(
|
| 152 |
+
total_results := 1,
|
| 153 |
+
total_groups := 1,
|
| 154 |
+
multi_source_groups := 0,
|
| 155 |
+
first_party_only_groups := 1,
|
| 156 |
+
source_type_distribution := struct_pack(first_party := 1, third_party := 0, collaborative := 0, unspecified := 0)
|
| 157 |
+
) AS provenance_summary,
|
| 158 |
+
struct_pack(
|
| 159 |
+
total_groups := 1,
|
| 160 |
+
groups_with_variant_check := 0,
|
| 161 |
+
groups_with_cross_party_check := 0,
|
| 162 |
+
variant_divergent_count := 0,
|
| 163 |
+
cross_party_divergent_count := 0
|
| 164 |
+
) AS comparability_summary,
|
| 165 |
+
struct_pack(available := false, url_count := 0::BIGINT, sample_urls := []::VARCHAR[], models_with_loaded_instances := 0) AS instance_data,
|
| 166 |
+
1::INTEGER AS metrics_count,
|
| 167 |
+
['Accuracy']::VARCHAR[] AS metric_names,
|
| 168 |
+
[struct_pack(
|
| 169 |
+
column_key := 'root:accuracy',
|
| 170 |
+
metric_summary_id := 'mmlu%3Aaccuracy',
|
| 171 |
+
metric_id := 'accuracy',
|
| 172 |
+
metric_name := 'accuracy',
|
| 173 |
+
display_name := 'Accuracy',
|
| 174 |
+
canonical_display_name := 'Accuracy',
|
| 175 |
+
lower_is_better := false,
|
| 176 |
+
unit := 'proportion',
|
| 177 |
+
scope := 'root',
|
| 178 |
+
subtask_key := NULL::VARCHAR,
|
| 179 |
+
subtask_name := NULL::VARCHAR
|
| 180 |
+
)] AS leaderboard_metrics,
|
| 181 |
+
[] AS leaderboard_rows,
|
| 182 |
+
[struct_pack(
|
| 183 |
+
metric_summary_id := 'mmlu%3Aaccuracy',
|
| 184 |
+
metric_name := 'accuracy',
|
| 185 |
+
display_name := 'Accuracy',
|
| 186 |
+
canonical_display_name := 'Accuracy',
|
| 187 |
+
metric_key := 'accuracy',
|
| 188 |
+
lower_is_better := false,
|
| 189 |
+
models_count := 1,
|
| 190 |
+
top_score := 0.8,
|
| 191 |
+
unit := 'proportion'
|
| 192 |
+
)] AS root_metrics,
|
| 193 |
+
[] AS subtasks,
|
| 194 |
+
0::INTEGER AS subtasks_count
|
| 195 |
+
`,
|
| 196 |
+
path.join(snapshotDir, "evals_view.parquet")
|
| 197 |
+
)
|
| 198 |
+
|
| 199 |
+
await copyParquet(
|
| 200 |
+
connection,
|
| 201 |
+
`
|
| 202 |
+
SELECT
|
| 203 |
+
TIMESTAMP '2026-05-03 00:00:00' AS snapshot_id,
|
| 204 |
+
'mmlu' AS evaluation_id,
|
| 205 |
+
'mmlu%3Aaccuracy' AS metric_summary_id,
|
| 206 |
+
'mmlu' AS benchmark_id,
|
| 207 |
+
'accuracy' AS metric_id,
|
| 208 |
+
'openai/gpt-5' AS model_id,
|
| 209 |
+
'openai%2Fgpt-5' AS model_route_id,
|
| 210 |
+
struct_pack(
|
| 211 |
+
name := 'GPT 5',
|
| 212 |
+
id := 'openai/gpt-5',
|
| 213 |
+
developer := 'OpenAI',
|
| 214 |
+
inference_platform := 'platform',
|
| 215 |
+
inference_engine := 'engine',
|
| 216 |
+
model_version := NULL::VARCHAR,
|
| 217 |
+
architecture := 'transformer',
|
| 218 |
+
parameter_count := '100B',
|
| 219 |
+
release_date := '2026-01-01',
|
| 220 |
+
model_url := 'https://example.test/model',
|
| 221 |
+
modalities := struct_pack(input := ['text']::VARCHAR[], output := ['text']::VARCHAR[])
|
| 222 |
+
) AS model_info,
|
| 223 |
+
'Accuracy' AS metric_display_name,
|
| 224 |
+
'proportion' AS metric_unit,
|
| 225 |
+
false AS lower_is_better,
|
| 226 |
+
'Reasoning' AS category,
|
| 227 |
+
0.8 AS score,
|
| 228 |
+
struct_pack(
|
| 229 |
+
score := 0.8,
|
| 230 |
+
standard_error := 0.01,
|
| 231 |
+
sample_size := 10,
|
| 232 |
+
confidence_interval := struct_pack(lower := 0.7, upper := 0.9, confidence_level := 0.95)
|
| 233 |
+
) AS score_details,
|
| 234 |
+
1::INTEGER AS fact_row_count,
|
| 235 |
+
1::INTEGER AS position,
|
| 236 |
+
1::INTEGER AS total,
|
| 237 |
+
1.0 AS percentile,
|
| 238 |
+
TIMESTAMP '2026-05-03 00:00:00' AS evaluation_timestamp,
|
| 239 |
+
struct_pack(
|
| 240 |
+
source_name := 'OpenAI report',
|
| 241 |
+
source_type := 'documentation',
|
| 242 |
+
source_organization_name := 'OpenAI',
|
| 243 |
+
source_organization_url := 'https://example.test',
|
| 244 |
+
evaluator_relationship := 'first_party',
|
| 245 |
+
source_url := 'https://example.test/report',
|
| 246 |
+
publication_date := DATE '2026-05-03'
|
| 247 |
+
) AS source_metadata,
|
| 248 |
+
struct_pack(
|
| 249 |
+
dataset_name := 'MMLU',
|
| 250 |
+
source_type := 'documentation',
|
| 251 |
+
hf_repo := NULL::VARCHAR,
|
| 252 |
+
hf_split := NULL::VARCHAR,
|
| 253 |
+
samples_number := 10,
|
| 254 |
+
url := ['https://example.test/mmlu']::VARCHAR[],
|
| 255 |
+
dataset_url := 'https://example.test/mmlu',
|
| 256 |
+
dataset_version := 'v1'
|
| 257 |
+
) AS source_data,
|
| 258 |
+
'https://example.test/record.json' AS source_record_url,
|
| 259 |
+
struct_pack(name := 'openai-evals', version := '1.0', fork := NULL::VARCHAR) AS eval_library,
|
| 260 |
+
['first_party']::VARCHAR[] AS evaluator_relationships,
|
| 261 |
+
true AS has_first_party,
|
| 262 |
+
false AS has_third_party,
|
| 263 |
+
'self' AS coverage_cell,
|
| 264 |
+
['OpenAI']::VARCHAR[] AS reporting_orgs,
|
| 265 |
+
map(['OpenAI'], [0.8]) AS scores_by_organization,
|
| 266 |
+
false AS is_summary_score,
|
| 267 |
+
NULL::VARCHAR AS summary_score_for,
|
| 268 |
+
[] AS aggregate_components,
|
| 269 |
+
false AS has_reproducibility_gap,
|
| 270 |
+
1.0 AS completeness_score,
|
| 271 |
+
false AS is_multi_source,
|
| 272 |
+
true AS first_party_only,
|
| 273 |
+
false AS has_variant_divergence,
|
| 274 |
+
false AS has_cross_party_divergence,
|
| 275 |
+
NULL AS evalcards_annotations,
|
| 276 |
+
NULL::VARCHAR AS instance_file_path,
|
| 277 |
+
NULL::VARCHAR AS instance_file_format,
|
| 278 |
+
0::INTEGER AS instance_rows
|
| 279 |
+
`,
|
| 280 |
+
path.join(snapshotDir, "eval_results_view.parquet")
|
| 281 |
+
)
|
| 282 |
+
|
| 283 |
+
await writeFile(
|
| 284 |
+
path.join(snapshotDir, "manifest.json"),
|
| 285 |
+
JSON.stringify({
|
| 286 |
+
generated_at: "2026-05-03T00:00:00Z",
|
| 287 |
+
config_version: 2,
|
| 288 |
+
skipped_configs: [],
|
| 289 |
+
model_count: 1,
|
| 290 |
+
eval_count: 1,
|
| 291 |
+
metric_eval_count: 1,
|
| 292 |
+
source_config_count: 1,
|
| 293 |
+
skipped_config_count: 0,
|
| 294 |
+
summary_artifacts: {
|
| 295 |
+
corpus_aggregates: "headline.json",
|
| 296 |
+
eval_hierarchy: "hierarchy.json",
|
| 297 |
+
},
|
| 298 |
+
})
|
| 299 |
+
)
|
| 300 |
+
|
| 301 |
+
const reproducibilityBlock = {
|
| 302 |
+
total_triples: 1,
|
| 303 |
+
triples_with_reproducibility_gap: 0,
|
| 304 |
+
reproducibility_gap_rate: 0,
|
| 305 |
+
agentic_triples: 0,
|
| 306 |
+
per_field_missingness: {
|
| 307 |
+
temperature: {
|
| 308 |
+
missing_count: 0,
|
| 309 |
+
missing_rate: 0,
|
| 310 |
+
denominator: "all_triples",
|
| 311 |
+
denominator_count: 1,
|
| 312 |
+
},
|
| 313 |
+
},
|
| 314 |
+
}
|
| 315 |
+
const completenessBlock = {
|
| 316 |
+
total_triples: 1,
|
| 317 |
+
completeness_avg: 0.75,
|
| 318 |
+
completeness_min: 0.75,
|
| 319 |
+
completeness_max: 0.75,
|
| 320 |
+
}
|
| 321 |
+
const provenanceBlock = {
|
| 322 |
+
total_triples: 1,
|
| 323 |
+
multi_source_triples: 0,
|
| 324 |
+
first_party_only_triples: 1,
|
| 325 |
+
source_type_distribution: {
|
| 326 |
+
first_party: 1,
|
| 327 |
+
third_party: 0,
|
| 328 |
+
collaborative: 0,
|
| 329 |
+
unspecified: 0,
|
| 330 |
+
},
|
| 331 |
+
}
|
| 332 |
+
const comparabilityBlock = {
|
| 333 |
+
total_triples: 1,
|
| 334 |
+
variant_divergent_count: 0,
|
| 335 |
+
cross_party_divergent_count: 0,
|
| 336 |
+
groups_with_variant_check: 1,
|
| 337 |
+
groups_with_cross_party_check: 0,
|
| 338 |
+
}
|
| 339 |
+
await writeFile(
|
| 340 |
+
path.join(snapshotDir, "headline.json"),
|
| 341 |
+
JSON.stringify({
|
| 342 |
+
generated_at: "2026-05-03T00:00:00Z",
|
| 343 |
+
signal_version: "1.0",
|
| 344 |
+
stratification_dimensions: ["category"],
|
| 345 |
+
reproducibility: {
|
| 346 |
+
overall: reproducibilityBlock,
|
| 347 |
+
by_category: { Reasoning: reproducibilityBlock },
|
| 348 |
+
},
|
| 349 |
+
completeness: {
|
| 350 |
+
overall: completenessBlock,
|
| 351 |
+
by_category: { Reasoning: completenessBlock },
|
| 352 |
+
},
|
| 353 |
+
provenance: {
|
| 354 |
+
overall: provenanceBlock,
|
| 355 |
+
by_category: { Reasoning: provenanceBlock },
|
| 356 |
+
},
|
| 357 |
+
comparability: {
|
| 358 |
+
overall: comparabilityBlock,
|
| 359 |
+
by_category: { Reasoning: comparabilityBlock },
|
| 360 |
+
},
|
| 361 |
+
developers: [
|
| 362 |
+
{
|
| 363 |
+
developer: "OpenAI",
|
| 364 |
+
route_id: "OpenAI",
|
| 365 |
+
model_count: 1,
|
| 366 |
+
benchmark_count: 1,
|
| 367 |
+
evaluation_count: 1,
|
| 368 |
+
popular_evals: [{ benchmark: "MMLU", model_count: 1 }],
|
| 369 |
+
},
|
| 370 |
+
],
|
| 371 |
+
})
|
| 372 |
+
)
|
| 373 |
+
|
| 374 |
+
await writeFile(
|
| 375 |
+
path.join(snapshotDir, "hierarchy.json"),
|
| 376 |
+
JSON.stringify({
|
| 377 |
+
stats: {
|
| 378 |
+
family_count: 1,
|
| 379 |
+
composite_count: 0,
|
| 380 |
+
standalone_benchmark_count: 1,
|
| 381 |
+
single_benchmark_count: 1,
|
| 382 |
+
slice_count: 0,
|
| 383 |
+
metric_count: 1,
|
| 384 |
+
metric_rows_scanned: 1,
|
| 385 |
+
},
|
| 386 |
+
families: [],
|
| 387 |
+
})
|
| 388 |
+
)
|
| 389 |
+
}
|
| 390 |
+
|
| 391 |
+
describe("Stage J view-layer backend", () => {
|
| 392 |
+
it("reads a pinned snapshot through the v2 accessors", async () => {
|
| 393 |
+
const snapshotDir = await mkdtemp(path.join(os.tmpdir(), "eval-card-stage-j-"))
|
| 394 |
+
const previousBackend = process.env.DATA_BACKEND
|
| 395 |
+
const previousSnapshotUrl = process.env.SNAPSHOT_URL
|
| 396 |
+
|
| 397 |
+
try {
|
| 398 |
+
await writeSyntheticStageJSnapshot(snapshotDir)
|
| 399 |
+
process.env.DATA_BACKEND = "v2"
|
| 400 |
+
process.env.SNAPSHOT_URL = `file://${snapshotDir}`
|
| 401 |
+
|
| 402 |
+
const dataBackend = await import("../lib/data-backend")
|
| 403 |
+
const hfData = await import("../lib/hf-data")
|
| 404 |
+
|
| 405 |
+
const [models, evalListData, modelSummary, evalSummary, developers, developerSummary, manifest, hierarchy, aggregates] =
|
| 406 |
+
await Promise.all([
|
| 407 |
+
dataBackend.getModelCardsLite(),
|
| 408 |
+
dataBackend.getEvalListLiteData(),
|
| 409 |
+
dataBackend.getModelSummaryById("openai%2Fgpt-5"),
|
| 410 |
+
dataBackend.getEvalSummaryById("mmlu"),
|
| 411 |
+
dataBackend.getDeveloperList(),
|
| 412 |
+
dataBackend.getDeveloperSummaryById("OpenAI"),
|
| 413 |
+
dataBackend.getBackendManifestData(),
|
| 414 |
+
dataBackend.getEvalHierarchyData(),
|
| 415 |
+
hfData.fetchCorpusAggregates(),
|
| 416 |
+
])
|
| 417 |
+
|
| 418 |
+
expect(models[0]).toMatchObject({
|
| 419 |
+
route_id: "openai%2Fgpt-5",
|
| 420 |
+
model_name: "GPT 5",
|
| 421 |
+
evaluations_count: 1,
|
| 422 |
+
})
|
| 423 |
+
expect(evalListData).toMatchObject({
|
| 424 |
+
totalModels: 1,
|
| 425 |
+
evals: [{ evaluation_id: "mmlu", evaluation_name: "MMLU", models_count: 1 }],
|
| 426 |
+
})
|
| 427 |
+
expect(modelSummary?.evaluations_by_category.Reasoning).toHaveLength(1)
|
| 428 |
+
expect(evalSummary?.model_results[0]).toMatchObject({
|
| 429 |
+
model_route_id: "openai%2Fgpt-5",
|
| 430 |
+
score: 0.8,
|
| 431 |
+
result: { metric_summary_id: "mmlu%3Aaccuracy" },
|
| 432 |
+
})
|
| 433 |
+
expect(developers[0]).toMatchObject({ developer: "OpenAI", route_id: "OpenAI" })
|
| 434 |
+
expect(developerSummary?.models).toHaveLength(1)
|
| 435 |
+
expect(manifest.model_count).toBe(1)
|
| 436 |
+
expect(hierarchy.stats?.metric_rows_scanned).toBe(1)
|
| 437 |
+
expect(aggregates?.completeness.overall).toMatchObject({
|
| 438 |
+
total_triples: 1,
|
| 439 |
+
completeness_avg: 0.75,
|
| 440 |
+
})
|
| 441 |
+
expect(aggregates?.provenance.overall).toMatchObject({
|
| 442 |
+
total_triples: 1,
|
| 443 |
+
first_party_only_triples: 1,
|
| 444 |
+
})
|
| 445 |
+
expect(aggregates?.comparability.overall).toMatchObject({
|
| 446 |
+
groups_with_variant_check: 1,
|
| 447 |
+
variant_divergent_count: 0,
|
| 448 |
+
})
|
| 449 |
+
expect(aggregates?.comparability.by_category.Reasoning).toBeDefined()
|
| 450 |
+
} finally {
|
| 451 |
+
if (previousBackend == null) {
|
| 452 |
+
delete process.env.DATA_BACKEND
|
| 453 |
+
} else {
|
| 454 |
+
process.env.DATA_BACKEND = previousBackend
|
| 455 |
+
}
|
| 456 |
+
if (previousSnapshotUrl == null) {
|
| 457 |
+
delete process.env.SNAPSHOT_URL
|
| 458 |
+
} else {
|
| 459 |
+
process.env.SNAPSHOT_URL = previousSnapshotUrl
|
| 460 |
+
}
|
| 461 |
+
await rm(snapshotDir, { recursive: true, force: true })
|
| 462 |
+
}
|
| 463 |
+
})
|
| 464 |
+
})
|