Spaces:
Running
Running
Update to uniform US spelling
Browse files- README.md +1 -1
- app/developers/[...id]/layout.tsx +1 -1
- app/layout.tsx +1 -1
- app/models/[...id]/layout.tsx +1 -1
- app/opengraph-image.tsx +1 -1
- app/page.tsx +1 -1
- components/quick-start.tsx +1 -1
- components/signals/comparability-panel.tsx +1 -1
- content/tutorials/what-its-built-on.md +1 -1
- lib/benchmark-tags.ts +14 -14
README.md
CHANGED
|
@@ -33,7 +33,7 @@ The Eval Cards project aims to:
|
|
| 33 |
## Architecture
|
| 34 |
|
| 35 |
This app is a **read-only consumer** of a snapshot. The producer (`eval_card_backend`)
|
| 36 |
-
|
| 37 |
layer (`*.parquet` view tables + JSON sidecars), and the frontend reads that snapshot at
|
| 38 |
runtime via DuckDB — it performs no identity resolution or aggregation of its own. The
|
| 39 |
view-layer column names match this app's TypeScript interfaces by contract (declared in
|
|
|
|
| 33 |
## Architecture
|
| 34 |
|
| 35 |
This app is a **read-only consumer** of a snapshot. The producer (`eval_card_backend`)
|
| 36 |
+
canonicalizes raw evaluation data into a typed Parquet warehouse plus a Stage J view
|
| 37 |
layer (`*.parquet` view tables + JSON sidecars), and the frontend reads that snapshot at
|
| 38 |
runtime via DuckDB — it performs no identity resolution or aggregation of its own. The
|
| 39 |
view-layer column names match this app's TypeScript interfaces by contract (declared in
|
app/developers/[...id]/layout.tsx
CHANGED
|
@@ -32,7 +32,7 @@ export async function generateMetadata(props: {
|
|
| 32 |
const title = developerName
|
| 33 |
const description =
|
| 34 |
modelCount != null
|
| 35 |
-
? `${developerName} — evaluation coverage across ${modelCount.toLocaleString("en-US")} ${modelCount === 1 ? "model" : "models"},
|
| 36 |
: `${developerName} — model evaluation coverage on Evaluation Cards.`
|
| 37 |
const imageUrl = `/api/og/developers/${idSlug}`
|
| 38 |
|
|
|
|
| 32 |
const title = developerName
|
| 33 |
const description =
|
| 34 |
modelCount != null
|
| 35 |
+
? `${developerName} — evaluation coverage across ${modelCount.toLocaleString("en-US")} ${modelCount === 1 ? "model" : "models"}, organized under Evaluation Cards' five-level hierarchy and four interpretive signals.`
|
| 36 |
: `${developerName} — model evaluation coverage on Evaluation Cards.`
|
| 37 |
const imageUrl = `/api/og/developers/${idSlug}`
|
| 38 |
|
app/layout.tsx
CHANGED
|
@@ -13,7 +13,7 @@ const SITE_URL = "https://evalcards.evalevalai.com"
|
|
| 13 |
const SITE_NAME = "Evaluation Cards"
|
| 14 |
const SITE_TITLE = "Evaluation Cards — a reporting layer for AI evaluations"
|
| 15 |
const SITE_DESCRIPTION =
|
| 16 |
-
"A public collection of reported model–benchmark results,
|
| 17 |
|
| 18 |
export const metadata: Metadata = {
|
| 19 |
metadataBase: new URL(SITE_URL),
|
|
|
|
| 13 |
const SITE_NAME = "Evaluation Cards"
|
| 14 |
const SITE_TITLE = "Evaluation Cards — a reporting layer for AI evaluations"
|
| 15 |
const SITE_DESCRIPTION =
|
| 16 |
+
"A public collection of reported model–benchmark results, organized under a five-level rollout hierarchy and four interpretive signals: reproducibility, completeness, provenance, and comparability."
|
| 17 |
|
| 18 |
export const metadata: Metadata = {
|
| 19 |
metadataBase: new URL(SITE_URL),
|
app/models/[...id]/layout.tsx
CHANGED
|
@@ -31,7 +31,7 @@ export async function generateMetadata(props: {
|
|
| 31 |
const idSlug = routeIdToPath(routeId)
|
| 32 |
const title = developer ? `${modelName} — ${developer}` : modelName
|
| 33 |
const description = developer
|
| 34 |
-
? `${modelName} (${developer}) — every reported model–benchmark result,
|
| 35 |
: `${modelName} — every reported model–benchmark result on Evaluation Cards.`
|
| 36 |
const imageUrl = `/api/og/models/${idSlug}`
|
| 37 |
|
|
|
|
| 31 |
const idSlug = routeIdToPath(routeId)
|
| 32 |
const title = developer ? `${modelName} — ${developer}` : modelName
|
| 33 |
const description = developer
|
| 34 |
+
? `${modelName} (${developer}) — every reported model–benchmark result, organized under Evaluation Cards' five-level hierarchy and four interpretive signals.`
|
| 35 |
: `${modelName} — every reported model–benchmark result on Evaluation Cards.`
|
| 36 |
const imageUrl = `/api/og/models/${idSlug}`
|
| 37 |
|
app/opengraph-image.tsx
CHANGED
|
@@ -108,7 +108,7 @@ export default function OpenGraphImage() {
|
|
| 108 |
maxWidth: "920px",
|
| 109 |
}}
|
| 110 |
>
|
| 111 |
-
A collection of reported model–benchmark results,
|
| 112 |
rollout hierarchy and four interpretive signals.
|
| 113 |
</div>
|
| 114 |
</div>
|
|
|
|
| 108 |
maxWidth: "920px",
|
| 109 |
}}
|
| 110 |
>
|
| 111 |
+
A collection of reported model–benchmark results, organized under a five-level
|
| 112 |
rollout hierarchy and four interpretive signals.
|
| 113 |
</div>
|
| 114 |
</div>
|
app/page.tsx
CHANGED
|
@@ -119,7 +119,7 @@ export default async function HomePage() {
|
|
| 119 |
</h1>
|
| 120 |
<p className="home-hero-lede">
|
| 121 |
<strong>Evaluation Cards</strong> is a collection of reported model–benchmark results,
|
| 122 |
-
|
| 123 |
computed over the joined record.
|
| 124 |
</p>
|
| 125 |
<div className="flex flex-wrap items-center gap-3">
|
|
|
|
| 119 |
</h1>
|
| 120 |
<p className="home-hero-lede">
|
| 121 |
<strong>Evaluation Cards</strong> is a collection of reported model–benchmark results,
|
| 122 |
+
organized under a five-level rollout hierarchy and four interpretive signals
|
| 123 |
computed over the joined record.
|
| 124 |
</p>
|
| 125 |
<div className="flex flex-wrap items-center gap-3">
|
components/quick-start.tsx
CHANGED
|
@@ -45,7 +45,7 @@ const SLIDES: Slide[] = [
|
|
| 45 |
icon: BookOpen,
|
| 46 |
kicker: "Welcome",
|
| 47 |
title: "What this is",
|
| 48 |
-
body: "Evaluation Cards collects how AI models have been tested across lots of benchmarks. A single score rarely tells you much on its own, so we give you everything you need to read one: the evaluation results, the benchmark's metadata, how the run was set up, and our interpretive signals. And unlike any other
|
| 49 |
},
|
| 50 |
{
|
| 51 |
icon: Layers,
|
|
|
|
| 45 |
icon: BookOpen,
|
| 46 |
kicker: "Welcome",
|
| 47 |
title: "What this is",
|
| 48 |
+
body: "Evaluation Cards collects how AI models have been tested across lots of benchmarks. A single score rarely tells you much on its own, so we give you everything you need to read one: the evaluation results, the benchmark's metadata, how the run was set up, and our interpretive signals. And unlike any other artifact out there, you can see those results reported side by side, which makes them easy to compare.",
|
| 49 |
},
|
| 50 |
{
|
| 51 |
icon: Layers,
|
components/signals/comparability-panel.tsx
CHANGED
|
@@ -398,7 +398,7 @@ function DivergenceGroupItem({
|
|
| 398 |
const fieldLabels = fields.map((f) => formatFieldLabel(f.field))
|
| 399 |
const summarySentence =
|
| 400 |
fieldLabels.length === 0
|
| 401 |
-
? `Reported scores diverge by ${formatSignalNumber(magnitude)}, above the ${formatSignalNumber(threshold)} threshold. The setup difference is not
|
| 402 |
: `Reported scores diverge by ${formatSignalNumber(magnitude)} (threshold ${formatSignalNumber(threshold)}) because the runs differ on ${
|
| 403 |
fieldLabels.length === 1
|
| 404 |
? fieldLabels[0]
|
|
|
|
| 398 |
const fieldLabels = fields.map((f) => formatFieldLabel(f.field))
|
| 399 |
const summarySentence =
|
| 400 |
fieldLabels.length === 0
|
| 401 |
+
? `Reported scores diverge by ${formatSignalNumber(magnitude)}, above the ${formatSignalNumber(threshold)} threshold. The setup difference is not labeled.`
|
| 402 |
: `Reported scores diverge by ${formatSignalNumber(magnitude)} (threshold ${formatSignalNumber(threshold)}) because the runs differ on ${
|
| 403 |
fieldLabels.length === 1
|
| 404 |
? fieldLabels[0]
|
content/tutorials/what-its-built-on.md
CHANGED
|
@@ -10,7 +10,7 @@ A schema for benchmark-level metadata: what a benchmark measures, its splits, in
|
|
| 10 |
|
| 11 |
## Every Eval Ever
|
| 12 |
|
| 13 |
-
A run-level corpus of public evaluation results: `(model, benchmark, metric-path, value, source)` tuples extracted from papers, model cards, and leaderboards. It provides the raw rows that Evaluation Cards
|
| 14 |
|
| 15 |
## IBM Risk Atlas alignment
|
| 16 |
|
|
|
|
| 10 |
|
| 11 |
## Every Eval Ever
|
| 12 |
|
| 13 |
+
A run-level corpus of public evaluation results: `(model, benchmark, metric-path, value, source)` tuples extracted from papers, model cards, and leaderboards. It provides the raw rows that Evaluation Cards canonicalizes and joins. [Every Eval Ever](https://evalevalai.com/every_eval_ever/) is a sister [EvalEval](https://evalevalai.com/) project.
|
| 14 |
|
| 15 |
## IBM Risk Atlas alignment
|
| 16 |
|
lib/benchmark-tags.ts
CHANGED
|
@@ -24,18 +24,18 @@ import { inferTagsFromBenchmark } from "@/lib/benchmark-schema"
|
|
| 24 |
|
| 25 |
const REF: Record<string, string[]> = categoriesJson as Record<string, string[]>
|
| 26 |
|
| 27 |
-
// Two
|
| 28 |
// keeps spaces (so "MMLU Pro" still differs from "MMLUPro" if both
|
| 29 |
// were ever in the file); the second strips everything non-alphanumeric
|
| 30 |
// for a tolerant fallback ("ARC-C" ↔ "arc c" ↔ "arcc").
|
| 31 |
-
const
|
| 32 |
-
const
|
| 33 |
|
| 34 |
-
function
|
| 35 |
return name.trim().toLowerCase().replace(/\s+/g, " ")
|
| 36 |
}
|
| 37 |
|
| 38 |
-
function
|
| 39 |
return name.toLowerCase().replace(/[^a-z0-9]/g, "")
|
| 40 |
}
|
| 41 |
|
|
@@ -56,9 +56,9 @@ function stripParenSuffix(name: string): string {
|
|
| 56 |
}
|
| 57 |
|
| 58 |
for (const [name, tags] of Object.entries(REF)) {
|
| 59 |
-
|
| 60 |
-
const tight =
|
| 61 |
-
if (tight && !
|
| 62 |
}
|
| 63 |
|
| 64 |
/**
|
|
@@ -78,15 +78,15 @@ export function getBenchmarkTags(
|
|
| 78 |
): string[] {
|
| 79 |
const names = candidates.filter((n): n is string => typeof n === "string" && n.trim().length > 0)
|
| 80 |
for (const name of names) {
|
| 81 |
-
const loose =
|
| 82 |
if (loose) return loose
|
| 83 |
-
const tight =
|
| 84 |
if (tight) return tight
|
| 85 |
const stripped = stripParenSuffix(name)
|
| 86 |
if (stripped && stripped !== name) {
|
| 87 |
-
const loose2 =
|
| 88 |
if (loose2) return loose2
|
| 89 |
-
const tight2 =
|
| 90 |
if (tight2) return tight2
|
| 91 |
}
|
| 92 |
}
|
|
@@ -107,8 +107,8 @@ export function getBenchmarkTags(
|
|
| 107 |
export function hasCuratedTags(...candidates: Array<string | null | undefined>): boolean {
|
| 108 |
for (const name of candidates) {
|
| 109 |
if (typeof name !== "string" || !name.trim()) continue
|
| 110 |
-
if (
|
| 111 |
-
if (
|
| 112 |
}
|
| 113 |
return false
|
| 114 |
}
|
|
|
|
| 24 |
|
| 25 |
const REF: Record<string, string[]> = categoriesJson as Record<string, string[]>
|
| 26 |
|
| 27 |
+
// Two normalized lookup tables built once at module load. The first
|
| 28 |
// keeps spaces (so "MMLU Pro" still differs from "MMLUPro" if both
|
| 29 |
// were ever in the file); the second strips everything non-alphanumeric
|
| 30 |
// for a tolerant fallback ("ARC-C" ↔ "arc c" ↔ "arcc").
|
| 31 |
+
const NORMALIZED_LOOSE: Map<string, string[]> = new Map()
|
| 32 |
+
const NORMALIZED_TIGHT: Map<string, string[]> = new Map()
|
| 33 |
|
| 34 |
+
function normalizeLoose(name: string): string {
|
| 35 |
return name.trim().toLowerCase().replace(/\s+/g, " ")
|
| 36 |
}
|
| 37 |
|
| 38 |
+
function normalizeTight(name: string): string {
|
| 39 |
return name.toLowerCase().replace(/[^a-z0-9]/g, "")
|
| 40 |
}
|
| 41 |
|
|
|
|
| 56 |
}
|
| 57 |
|
| 58 |
for (const [name, tags] of Object.entries(REF)) {
|
| 59 |
+
NORMALIZED_LOOSE.set(normalizeLoose(name), tags)
|
| 60 |
+
const tight = normalizeTight(name)
|
| 61 |
+
if (tight && !NORMALIZED_TIGHT.has(tight)) NORMALIZED_TIGHT.set(tight, tags)
|
| 62 |
}
|
| 63 |
|
| 64 |
/**
|
|
|
|
| 78 |
): string[] {
|
| 79 |
const names = candidates.filter((n): n is string => typeof n === "string" && n.trim().length > 0)
|
| 80 |
for (const name of names) {
|
| 81 |
+
const loose = NORMALIZED_LOOSE.get(normalizeLoose(name))
|
| 82 |
if (loose) return loose
|
| 83 |
+
const tight = NORMALIZED_TIGHT.get(normalizeTight(name))
|
| 84 |
if (tight) return tight
|
| 85 |
const stripped = stripParenSuffix(name)
|
| 86 |
if (stripped && stripped !== name) {
|
| 87 |
+
const loose2 = NORMALIZED_LOOSE.get(normalizeLoose(stripped))
|
| 88 |
if (loose2) return loose2
|
| 89 |
+
const tight2 = NORMALIZED_TIGHT.get(normalizeTight(stripped))
|
| 90 |
if (tight2) return tight2
|
| 91 |
}
|
| 92 |
}
|
|
|
|
| 107 |
export function hasCuratedTags(...candidates: Array<string | null | undefined>): boolean {
|
| 108 |
for (const name of candidates) {
|
| 109 |
if (typeof name !== "string" || !name.trim()) continue
|
| 110 |
+
if (NORMALIZED_LOOSE.has(normalizeLoose(name))) return true
|
| 111 |
+
if (NORMALIZED_TIGHT.has(normalizeTight(name))) return true
|
| 112 |
}
|
| 113 |
return false
|
| 114 |
}
|