j-chim commited on
Commit
2fd6a71
·
1 Parent(s): 01f29b6

Update to uniform US spelling

Browse files
README.md CHANGED
@@ -33,7 +33,7 @@ The Eval Cards project aims to:
33
  ## Architecture
34
 
35
  This app is a **read-only consumer** of a snapshot. The producer (`eval_card_backend`)
36
- canonicalises raw evaluation data into a typed Parquet warehouse plus a Stage J view
37
  layer (`*.parquet` view tables + JSON sidecars), and the frontend reads that snapshot at
38
  runtime via DuckDB — it performs no identity resolution or aggregation of its own. The
39
  view-layer column names match this app's TypeScript interfaces by contract (declared in
 
33
  ## Architecture
34
 
35
  This app is a **read-only consumer** of a snapshot. The producer (`eval_card_backend`)
36
+ canonicalizes raw evaluation data into a typed Parquet warehouse plus a Stage J view
37
  layer (`*.parquet` view tables + JSON sidecars), and the frontend reads that snapshot at
38
  runtime via DuckDB — it performs no identity resolution or aggregation of its own. The
39
  view-layer column names match this app's TypeScript interfaces by contract (declared in
app/developers/[...id]/layout.tsx CHANGED
@@ -32,7 +32,7 @@ export async function generateMetadata(props: {
32
  const title = developerName
33
  const description =
34
  modelCount != null
35
- ? `${developerName} — evaluation coverage across ${modelCount.toLocaleString("en-US")} ${modelCount === 1 ? "model" : "models"}, organised under Evaluation Cards' five-level hierarchy and four interpretive signals.`
36
  : `${developerName} — model evaluation coverage on Evaluation Cards.`
37
  const imageUrl = `/api/og/developers/${idSlug}`
38
 
 
32
  const title = developerName
33
  const description =
34
  modelCount != null
35
+ ? `${developerName} — evaluation coverage across ${modelCount.toLocaleString("en-US")} ${modelCount === 1 ? "model" : "models"}, organized under Evaluation Cards' five-level hierarchy and four interpretive signals.`
36
  : `${developerName} — model evaluation coverage on Evaluation Cards.`
37
  const imageUrl = `/api/og/developers/${idSlug}`
38
 
app/layout.tsx CHANGED
@@ -13,7 +13,7 @@ const SITE_URL = "https://evalcards.evalevalai.com"
13
  const SITE_NAME = "Evaluation Cards"
14
  const SITE_TITLE = "Evaluation Cards — a reporting layer for AI evaluations"
15
  const SITE_DESCRIPTION =
16
- "A public collection of reported model–benchmark results, organised under a five-level rollout hierarchy and four interpretive signals: reproducibility, completeness, provenance, and comparability."
17
 
18
  export const metadata: Metadata = {
19
  metadataBase: new URL(SITE_URL),
 
13
  const SITE_NAME = "Evaluation Cards"
14
  const SITE_TITLE = "Evaluation Cards — a reporting layer for AI evaluations"
15
  const SITE_DESCRIPTION =
16
+ "A public collection of reported model–benchmark results, organized under a five-level rollout hierarchy and four interpretive signals: reproducibility, completeness, provenance, and comparability."
17
 
18
  export const metadata: Metadata = {
19
  metadataBase: new URL(SITE_URL),
app/models/[...id]/layout.tsx CHANGED
@@ -31,7 +31,7 @@ export async function generateMetadata(props: {
31
  const idSlug = routeIdToPath(routeId)
32
  const title = developer ? `${modelName} — ${developer}` : modelName
33
  const description = developer
34
- ? `${modelName} (${developer}) — every reported model–benchmark result, organised under Evaluation Cards' five-level hierarchy and four interpretive signals.`
35
  : `${modelName} — every reported model–benchmark result on Evaluation Cards.`
36
  const imageUrl = `/api/og/models/${idSlug}`
37
 
 
31
  const idSlug = routeIdToPath(routeId)
32
  const title = developer ? `${modelName} — ${developer}` : modelName
33
  const description = developer
34
+ ? `${modelName} (${developer}) — every reported model–benchmark result, organized under Evaluation Cards' five-level hierarchy and four interpretive signals.`
35
  : `${modelName} — every reported model–benchmark result on Evaluation Cards.`
36
  const imageUrl = `/api/og/models/${idSlug}`
37
 
app/opengraph-image.tsx CHANGED
@@ -108,7 +108,7 @@ export default function OpenGraphImage() {
108
  maxWidth: "920px",
109
  }}
110
  >
111
- A collection of reported model–benchmark results, organised under a five-level
112
  rollout hierarchy and four interpretive signals.
113
  </div>
114
  </div>
 
108
  maxWidth: "920px",
109
  }}
110
  >
111
+ A collection of reported model–benchmark results, organized under a five-level
112
  rollout hierarchy and four interpretive signals.
113
  </div>
114
  </div>
app/page.tsx CHANGED
@@ -119,7 +119,7 @@ export default async function HomePage() {
119
  </h1>
120
  <p className="home-hero-lede">
121
  <strong>Evaluation Cards</strong> is a collection of reported model–benchmark results,
122
- organised under a five-level rollout hierarchy and four interpretive signals
123
  computed over the joined record.
124
  </p>
125
  <div className="flex flex-wrap items-center gap-3">
 
119
  </h1>
120
  <p className="home-hero-lede">
121
  <strong>Evaluation Cards</strong> is a collection of reported model–benchmark results,
122
+ organized under a five-level rollout hierarchy and four interpretive signals
123
  computed over the joined record.
124
  </p>
125
  <div className="flex flex-wrap items-center gap-3">
components/quick-start.tsx CHANGED
@@ -45,7 +45,7 @@ const SLIDES: Slide[] = [
45
  icon: BookOpen,
46
  kicker: "Welcome",
47
  title: "What this is",
48
- body: "Evaluation Cards collects how AI models have been tested across lots of benchmarks. A single score rarely tells you much on its own, so we give you everything you need to read one: the evaluation results, the benchmark's metadata, how the run was set up, and our interpretive signals. And unlike any other artefact out there, you can see those results reported side by side, which makes them easy to compare.",
49
  },
50
  {
51
  icon: Layers,
 
45
  icon: BookOpen,
46
  kicker: "Welcome",
47
  title: "What this is",
48
+ body: "Evaluation Cards collects how AI models have been tested across lots of benchmarks. A single score rarely tells you much on its own, so we give you everything you need to read one: the evaluation results, the benchmark's metadata, how the run was set up, and our interpretive signals. And unlike any other artifact out there, you can see those results reported side by side, which makes them easy to compare.",
49
  },
50
  {
51
  icon: Layers,
components/signals/comparability-panel.tsx CHANGED
@@ -398,7 +398,7 @@ function DivergenceGroupItem({
398
  const fieldLabels = fields.map((f) => formatFieldLabel(f.field))
399
  const summarySentence =
400
  fieldLabels.length === 0
401
- ? `Reported scores diverge by ${formatSignalNumber(magnitude)}, above the ${formatSignalNumber(threshold)} threshold. The setup difference is not labelled.`
402
  : `Reported scores diverge by ${formatSignalNumber(magnitude)} (threshold ${formatSignalNumber(threshold)}) because the runs differ on ${
403
  fieldLabels.length === 1
404
  ? fieldLabels[0]
 
398
  const fieldLabels = fields.map((f) => formatFieldLabel(f.field))
399
  const summarySentence =
400
  fieldLabels.length === 0
401
+ ? `Reported scores diverge by ${formatSignalNumber(magnitude)}, above the ${formatSignalNumber(threshold)} threshold. The setup difference is not labeled.`
402
  : `Reported scores diverge by ${formatSignalNumber(magnitude)} (threshold ${formatSignalNumber(threshold)}) because the runs differ on ${
403
  fieldLabels.length === 1
404
  ? fieldLabels[0]
content/tutorials/what-its-built-on.md CHANGED
@@ -10,7 +10,7 @@ A schema for benchmark-level metadata: what a benchmark measures, its splits, in
10
 
11
  ## Every Eval Ever
12
 
13
- A run-level corpus of public evaluation results: `(model, benchmark, metric-path, value, source)` tuples extracted from papers, model cards, and leaderboards. It provides the raw rows that Evaluation Cards canonicalises and joins. [Every Eval Ever](https://evalevalai.com/every_eval_ever/) is a sister [EvalEval](https://evalevalai.com/) project.
14
 
15
  ## IBM Risk Atlas alignment
16
 
 
10
 
11
  ## Every Eval Ever
12
 
13
+ A run-level corpus of public evaluation results: `(model, benchmark, metric-path, value, source)` tuples extracted from papers, model cards, and leaderboards. It provides the raw rows that Evaluation Cards canonicalizes and joins. [Every Eval Ever](https://evalevalai.com/every_eval_ever/) is a sister [EvalEval](https://evalevalai.com/) project.
14
 
15
  ## IBM Risk Atlas alignment
16
 
lib/benchmark-tags.ts CHANGED
@@ -24,18 +24,18 @@ import { inferTagsFromBenchmark } from "@/lib/benchmark-schema"
24
 
25
  const REF: Record<string, string[]> = categoriesJson as Record<string, string[]>
26
 
27
- // Two normalised lookup tables built once at module load. The first
28
  // keeps spaces (so "MMLU Pro" still differs from "MMLUPro" if both
29
  // were ever in the file); the second strips everything non-alphanumeric
30
  // for a tolerant fallback ("ARC-C" ↔ "arc c" ↔ "arcc").
31
- const NORMALISED_LOOSE: Map<string, string[]> = new Map()
32
- const NORMALISED_TIGHT: Map<string, string[]> = new Map()
33
 
34
- function normaliseLoose(name: string): string {
35
  return name.trim().toLowerCase().replace(/\s+/g, " ")
36
  }
37
 
38
- function normaliseTight(name: string): string {
39
  return name.toLowerCase().replace(/[^a-z0-9]/g, "")
40
  }
41
 
@@ -56,9 +56,9 @@ function stripParenSuffix(name: string): string {
56
  }
57
 
58
  for (const [name, tags] of Object.entries(REF)) {
59
- NORMALISED_LOOSE.set(normaliseLoose(name), tags)
60
- const tight = normaliseTight(name)
61
- if (tight && !NORMALISED_TIGHT.has(tight)) NORMALISED_TIGHT.set(tight, tags)
62
  }
63
 
64
  /**
@@ -78,15 +78,15 @@ export function getBenchmarkTags(
78
  ): string[] {
79
  const names = candidates.filter((n): n is string => typeof n === "string" && n.trim().length > 0)
80
  for (const name of names) {
81
- const loose = NORMALISED_LOOSE.get(normaliseLoose(name))
82
  if (loose) return loose
83
- const tight = NORMALISED_TIGHT.get(normaliseTight(name))
84
  if (tight) return tight
85
  const stripped = stripParenSuffix(name)
86
  if (stripped && stripped !== name) {
87
- const loose2 = NORMALISED_LOOSE.get(normaliseLoose(stripped))
88
  if (loose2) return loose2
89
- const tight2 = NORMALISED_TIGHT.get(normaliseTight(stripped))
90
  if (tight2) return tight2
91
  }
92
  }
@@ -107,8 +107,8 @@ export function getBenchmarkTags(
107
  export function hasCuratedTags(...candidates: Array<string | null | undefined>): boolean {
108
  for (const name of candidates) {
109
  if (typeof name !== "string" || !name.trim()) continue
110
- if (NORMALISED_LOOSE.has(normaliseLoose(name))) return true
111
- if (NORMALISED_TIGHT.has(normaliseTight(name))) return true
112
  }
113
  return false
114
  }
 
24
 
25
  const REF: Record<string, string[]> = categoriesJson as Record<string, string[]>
26
 
27
+ // Two normalized lookup tables built once at module load. The first
28
  // keeps spaces (so "MMLU Pro" still differs from "MMLUPro" if both
29
  // were ever in the file); the second strips everything non-alphanumeric
30
  // for a tolerant fallback ("ARC-C" ↔ "arc c" ↔ "arcc").
31
+ const NORMALIZED_LOOSE: Map<string, string[]> = new Map()
32
+ const NORMALIZED_TIGHT: Map<string, string[]> = new Map()
33
 
34
+ function normalizeLoose(name: string): string {
35
  return name.trim().toLowerCase().replace(/\s+/g, " ")
36
  }
37
 
38
+ function normalizeTight(name: string): string {
39
  return name.toLowerCase().replace(/[^a-z0-9]/g, "")
40
  }
41
 
 
56
  }
57
 
58
  for (const [name, tags] of Object.entries(REF)) {
59
+ NORMALIZED_LOOSE.set(normalizeLoose(name), tags)
60
+ const tight = normalizeTight(name)
61
+ if (tight && !NORMALIZED_TIGHT.has(tight)) NORMALIZED_TIGHT.set(tight, tags)
62
  }
63
 
64
  /**
 
78
  ): string[] {
79
  const names = candidates.filter((n): n is string => typeof n === "string" && n.trim().length > 0)
80
  for (const name of names) {
81
+ const loose = NORMALIZED_LOOSE.get(normalizeLoose(name))
82
  if (loose) return loose
83
+ const tight = NORMALIZED_TIGHT.get(normalizeTight(name))
84
  if (tight) return tight
85
  const stripped = stripParenSuffix(name)
86
  if (stripped && stripped !== name) {
87
+ const loose2 = NORMALIZED_LOOSE.get(normalizeLoose(stripped))
88
  if (loose2) return loose2
89
+ const tight2 = NORMALIZED_TIGHT.get(normalizeTight(stripped))
90
  if (tight2) return tight2
91
  }
92
  }
 
107
  export function hasCuratedTags(...candidates: Array<string | null | undefined>): boolean {
108
  for (const name of candidates) {
109
  if (typeof name !== "string" || !name.trim()) continue
110
+ if (NORMALIZED_LOOSE.has(normalizeLoose(name))) return true
111
+ if (NORMALIZED_TIGHT.has(normalizeTight(name))) return true
112
  }
113
  return false
114
  }