Spaces:
Running
Running
File size: 3,677 Bytes
2fcae3f fe99ffa 2fcae3f fe99ffa 2fcae3f fe99ffa 2fcae3f fe99ffa 2fcae3f fe99ffa 2fcae3f fe99ffa 2fcae3f fe99ffa 2fcae3f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 | import { mkdtemp, mkdir, rm } from "fs/promises"
import os from "os"
import path from "path"
import { DuckDBConnection } from "@duckdb/node-api"
import { describe, expect, it } from "vitest"
import { getModelCardsLiteFromDuckDB } from "../lib/duckdb-data"
function sqlString(value: string) {
return `'${value.replace(/'/g, "''")}'`
}
async function writeParquetPayload(outputDir: string, fileName: string, payloads: unknown[]) {
const parquetDir = path.join(outputDir, "duckdb", "v1")
await mkdir(parquetDir, { recursive: true })
const selects = payloads
.map((payload) => {
const payloadJson = JSON.stringify(payload)
return `SELECT ${sqlString(payloadJson)} AS payload_json`
})
.join(" UNION ALL ")
const connection = await DuckDBConnection.create()
await connection.run(`COPY (${selects}) TO ${sqlString(path.join(parquetDir, fileName))} (FORMAT parquet)`)
}
describe("DuckDB local data backend", () => {
it("reads model-card lite payloads from local Parquet", async () => {
const outputDir = await mkdtemp(path.join(os.tmpdir(), "eval-card-duckdb-"))
const previousOutput = process.env.LOCAL_PIPELINE_OUTPUT
try {
process.env.LOCAL_PIPELINE_OUTPUT = outputDir
await writeParquetPayload(outputDir, "model_cards_lite.parquet", [
{
id: "openai/gpt-5",
route_id: "openai__gpt-5",
model_name: "GPT 5",
model_id: "openai/gpt-5",
canonical_model_name: "GPT 5",
developer: "OpenAI",
evaluations_count: 3,
benchmarks_count: 2,
variant_count: 1,
categories: ["Reasoning"],
category_stats: { General: 0, Reasoning: 2, Agentic: 0, Safety: 0, Knowledge: 0 },
latest_timestamp: "2026-01-01T00:00:00Z",
evaluator_count: 1,
evaluator_names: ["OpenAI"],
source_type_count: 1,
source_types: ["documentation"],
evidence_count: 3,
missing_generation_config_count: 0,
third_party_eval_count: 0,
independent_verification_ratio: 0,
reproducibility_status: "complete",
eval_libraries: [],
params_billions: 100,
score_summary: { count: 1, min: 0.7, max: 0.9, average: 0.8 },
benchmark_names: ["mmlu"],
top_scores: [
{ benchmark: "mmlu", score: 0.9, metric: "accuracy" },
],
source_urls: [],
detail_urls: [],
},
])
const cards = await getModelCardsLiteFromDuckDB()
expect(cards).toHaveLength(1)
expect(cards[0]).toMatchObject({
route_id: "openai__gpt-5",
model_name: "GPT 5",
developer: "OpenAI",
evaluations_count: 3,
})
} finally {
if (previousOutput == null) {
delete process.env.LOCAL_PIPELINE_OUTPUT
} else {
process.env.LOCAL_PIPELINE_OUTPUT = previousOutput
}
await rm(outputDir, { recursive: true, force: true })
}
})
it("fails clearly when the expected Parquet file is missing", async () => {
const outputDir = await mkdtemp(path.join(os.tmpdir(), "eval-card-duckdb-missing-"))
const previousOutput = process.env.LOCAL_PIPELINE_OUTPUT
try {
process.env.LOCAL_PIPELINE_OUTPUT = outputDir
await expect(getModelCardsLiteFromDuckDB()).rejects.toThrow(
/duckdb\/v1\/model_cards_lite\.parquet/
)
} finally {
if (previousOutput == null) {
delete process.env.LOCAL_PIPELINE_OUTPUT
} else {
process.env.LOCAL_PIPELINE_OUTPUT = previousOutput
}
await rm(outputDir, { recursive: true, force: true })
}
})
})
|