Spaces:
Running
Running
File size: 3,823 Bytes
2ed4959 d249d5b 2ed4959 d249d5b 2ed4959 d249d5b 2ed4959 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 | import { mkdir, mkdtemp, rm, writeFile } from "fs/promises"
import os from "os"
import path from "path"
import { describe, expect, it } from "vitest"
import type { ComparisonIndex } from "../lib/backend-artifacts"
import { assertComparisonIndexShape } from "../lib/sidecars"
// Shared fixture: a comparison-index in the current (v3) shape. Tests
// mutate copies of this to exercise contract assertions.
const validFixture: ComparisonIndex = {
generated_at: "2026-05-04T00:00:00Z",
config_version: 2,
metric_group_order: ["capability", "robustness", "efficiency", "cost", "latency", "rank", "other"],
evals: {
"helm-classic%2Fmmlu": {
evaluation_id: "helm-classic%2Fmmlu",
benchmark_id: "mmlu",
family_id: "helm",
family_display_name: "HELM",
composite_slug: "helm-classic",
composite_display_name: "HELM Classic",
parent_benchmark_id: null,
display_name: "MMLU",
category: "Knowledge",
is_slice: false,
is_summary_score: false,
summary_score_for: null,
metrics: [],
},
"helm-classic%2Fmmlu%2Fanatomy": {
evaluation_id: "helm-classic%2Fmmlu%2Fanatomy",
benchmark_id: "mmlu",
family_id: "helm",
family_display_name: "HELM",
composite_slug: "helm-classic",
composite_display_name: "HELM Classic",
parent_benchmark_id: "mmlu",
display_name: "MMLU Anatomy",
category: "Knowledge",
is_slice: true,
is_summary_score: false,
summary_score_for: null,
metrics: [],
},
},
by_model: {},
}
describe("assertComparisonIndexShape", () => {
it("accepts entries with family_id present", () => {
expect(() => assertComparisonIndexShape(validFixture)).not.toThrow()
})
it("accepts the new parent_benchmark_id field on slice entries", () => {
const sliceEntry = validFixture.evals["helm-classic%2Fmmlu%2Fanatomy"]
expect(sliceEntry.parent_benchmark_id).toBe("mmlu")
expect(sliceEntry.is_slice).toBe(true)
})
it("rejects entries that are missing family_id", () => {
const broken = JSON.parse(JSON.stringify(validFixture)) as ComparisonIndex
delete (broken.evals["helm-classic%2Fmmlu"] as unknown as Record<string, unknown>).family_id
expect(() => assertComparisonIndexShape(broken)).toThrow(
/missing family_id/,
)
})
})
describe("fetchComparisonIndex (v2 sidecar)", () => {
it("validates the loaded sidecar against the contract", async () => {
const snapshotDir = await mkdtemp(path.join(os.tmpdir(), "eval-card-cmp-"))
const previousBackend = process.env.DATA_BACKEND
const previousSnapshotUrl = process.env.SNAPSHOT_URL
try {
await mkdir(snapshotDir, { recursive: true })
const broken = JSON.parse(JSON.stringify(validFixture)) as ComparisonIndex
delete (broken.evals["helm-classic%2Fmmlu"] as unknown as Record<string, unknown>).family_id
await writeFile(
path.join(snapshotDir, "comparison-index.json"),
JSON.stringify(broken),
)
process.env.DATA_BACKEND = "v2"
process.env.SNAPSHOT_URL = `file://${snapshotDir}`
const sidecars = await import("../lib/sidecars")
sidecars.resetSidecarCacheForTests()
await expect(sidecars.fetchComparisonIndex()).rejects.toThrow(
/missing family_id/,
)
} finally {
const sidecars = await import("../lib/sidecars")
sidecars.resetSidecarCacheForTests()
if (previousBackend == null) {
delete process.env.DATA_BACKEND
} else {
process.env.DATA_BACKEND = previousBackend
}
if (previousSnapshotUrl == null) {
delete process.env.SNAPSHOT_URL
} else {
process.env.SNAPSHOT_URL = previousSnapshotUrl
}
await rm(snapshotDir, { recursive: true, force: true })
}
})
})
|