Spaces:
Running
Running
Switch family/model views to curated category tags
Browse filesSanitize family/composite/benchmark display names against their key
(falling back to a humanized slug when the warehouse leaks an unrelated
display_name), then propagate derivedTags bottom-up so a family inherits
the union of its children's tags. The /evals family table now shows tag
chips instead of the raw "category" column, and selecting a category
pill auto-expands matching families and filters their leaves. The model
view re-buckets benchmark groups by the leaf's curated tag (via the
hierarchy lookup) instead of the legacy 5-bucket category.
- app/evals/page.tsx +1 -0
- components/benchmark-detail.tsx +47 -25
- components/family-table.tsx +69 -13
- lib/benchmark-tags.ts +91 -1
- lib/hierarchy-lookup.ts +35 -1
app/evals/page.tsx
CHANGED
|
@@ -535,6 +535,7 @@ function EvalsPageInner() {
|
|
| 535 |
evalItems={evalItems}
|
| 536 |
benchmarkCards={benchmarkCards}
|
| 537 |
domainFilter={domainFilter}
|
|
|
|
| 538 |
/>
|
| 539 |
)}
|
| 540 |
|
|
|
|
| 535 |
evalItems={evalItems}
|
| 536 |
benchmarkCards={benchmarkCards}
|
| 537 |
domainFilter={domainFilter}
|
| 538 |
+
categoryFilter={new Set(selectedCategories)}
|
| 539 |
/>
|
| 540 |
)}
|
| 541 |
|
components/benchmark-detail.tsx
CHANGED
|
@@ -37,6 +37,7 @@ import {
|
|
| 37 |
} from "lucide-react"
|
| 38 |
import type { BenchmarkCard, BenchmarkEvaluation, CategoryType, EvaluationResult } from "@/lib/benchmark-schema"
|
| 39 |
import { getCategoryColor as getCategoryTone, inferCategoryFromBenchmark } from "@/lib/benchmark-schema"
|
|
|
|
| 40 |
import type { BenchmarkEvalSummary } from "@/lib/eval-processing"
|
| 41 |
import type { ModelSummaryCore } from "@/lib/benchmark-schema"
|
| 42 |
import { lookupBenchmarkCard } from "@/lib/benchmark-metadata-utils"
|
|
@@ -1819,16 +1820,25 @@ export function BenchmarkDetail({
|
|
| 1819 |
|
| 1820 |
const allCategoryResults = useMemo(
|
| 1821 |
() =>
|
| 1822 |
-
Object.entries(summary.evaluations_by_category).flatMap(([
|
| 1823 |
-
evals.flatMap((evaluation) =>
|
| 1824 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1825 |
evaluation,
|
| 1826 |
result,
|
| 1827 |
-
category
|
| 1828 |
}))
|
| 1829 |
-
)
|
| 1830 |
),
|
| 1831 |
-
[summary.evaluations_by_category]
|
| 1832 |
)
|
| 1833 |
|
| 1834 |
const policyHighlights = useMemo(() => {
|
|
@@ -1927,9 +1937,26 @@ export function BenchmarkDetail({
|
|
| 1927 |
[allCategoryResults, benchmarkCards, currentDetailHref]
|
| 1928 |
)
|
| 1929 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1930 |
// First-party vs third-party split per category (for the donut + bars).
|
| 1931 |
const evaluatorMix = useMemo(() => {
|
| 1932 |
-
const order = new Map(
|
| 1933 |
const byCat = new Map<CategoryType, { first: number; third: number; collab: number; other: number }>()
|
| 1934 |
let firstTotal = 0
|
| 1935 |
let thirdTotal = 0
|
|
@@ -1963,12 +1990,7 @@ export function BenchmarkDetail({
|
|
| 1963 |
otherTotal,
|
| 1964 |
grand,
|
| 1965 |
}
|
| 1966 |
-
}, [benchmarkGroups,
|
| 1967 |
-
|
| 1968 |
-
const availableCategories = useMemo(() => {
|
| 1969 |
-
const presentCategories = new Set(benchmarkGroups.map((group) => group.category))
|
| 1970 |
-
return summary.categories_covered.filter((category) => presentCategories.has(category))
|
| 1971 |
-
}, [benchmarkGroups, summary.categories_covered])
|
| 1972 |
|
| 1973 |
const filteredBenchmarkGroups = useMemo(() => {
|
| 1974 |
const query = benchmarkSearch.trim().toLowerCase()
|
|
@@ -2017,7 +2039,7 @@ export function BenchmarkDetail({
|
|
| 2017 |
}, [benchmarkGroups, benchmarkSearch, benchmarkSort, selectedCategories, modelId, peerRanks])
|
| 2018 |
|
| 2019 |
const groupedFilteredBenchmarkGroups = useMemo(() => {
|
| 2020 |
-
const order = new Map(
|
| 2021 |
const groups = new Map<CategoryType, BenchmarkGroup[]>()
|
| 2022 |
|
| 2023 |
for (const benchmarkGroup of filteredBenchmarkGroups) {
|
|
@@ -2029,7 +2051,7 @@ export function BenchmarkDetail({
|
|
| 2029 |
return Array.from(groups.entries())
|
| 2030 |
.sort((a, b) => (order.get(a[0]) ?? 999) - (order.get(b[0]) ?? 999))
|
| 2031 |
.map(([category, groups]) => ({ category, groups }))
|
| 2032 |
-
}, [filteredBenchmarkGroups,
|
| 2033 |
|
| 2034 |
// Family-bucketed groups for the list view, mirroring plotboxUnits logic.
|
| 2035 |
// When comparisonIndex is available we use the backend-authoritative
|
|
@@ -2044,7 +2066,7 @@ export function BenchmarkDetail({
|
|
| 2044 |
}
|
| 2045 |
const listFamiliesByCategory = useMemo(() => {
|
| 2046 |
const order = new Map(
|
| 2047 |
-
|
| 2048 |
)
|
| 2049 |
const byCategory = new Map<CategoryType, Map<string, ListFamily>>()
|
| 2050 |
|
|
@@ -2087,7 +2109,7 @@ export function BenchmarkDetail({
|
|
| 2087 |
kind: f.groups.length > 1 ? "multi-eval" as const : "single-eval" as const,
|
| 2088 |
})),
|
| 2089 |
}))
|
| 2090 |
-
}, [filteredBenchmarkGroups, comparisonIndex, hierarchyIndex,
|
| 2091 |
|
| 2092 |
const compositeGroups = useMemo(() => {
|
| 2093 |
const groups = groupByComposite(filteredBenchmarkGroups, modelIds, peerRanks, hierarchyIndex)
|
|
@@ -2998,7 +3020,7 @@ export function BenchmarkDetail({
|
|
| 2998 |
<div className="min-w-0 flex-1">
|
| 2999 |
<div className="flex flex-wrap items-center gap-2">
|
| 3000 |
<span className="font-mono text-[9px] uppercase tracking-[0.18em] text-[color:var(--fg-subtle)] font-semibold">
|
| 3001 |
-
{unit.category}
|
| 3002 |
</span>
|
| 3003 |
{showChildKindBadge && (
|
| 3004 |
<span className="font-mono text-[9px] uppercase tracking-[0.15em] text-[color:var(--fg-subtle)]">
|
|
@@ -3794,7 +3816,7 @@ export function BenchmarkDetail({
|
|
| 3794 |
}
|
| 3795 |
className={`ec-pill ${isSelected ? "on" : ""}`}
|
| 3796 |
>
|
| 3797 |
-
{category}
|
| 3798 |
</button>
|
| 3799 |
)
|
| 3800 |
})}
|
|
@@ -3808,7 +3830,7 @@ export function BenchmarkDetail({
|
|
| 3808 |
) : benchmarkViewMode === "grid" ? (
|
| 3809 |
(() => {
|
| 3810 |
const categoryOrder = new Map(
|
| 3811 |
-
|
| 3812 |
)
|
| 3813 |
const byCategory = new Map<CategoryType, PlotboxUnit[]>()
|
| 3814 |
for (const unit of plotboxUnits) {
|
|
@@ -3841,7 +3863,7 @@ export function BenchmarkDetail({
|
|
| 3841 |
<div className="flex items-baseline justify-between gap-3 border-b border-[color:var(--border-soft)] pb-2">
|
| 3842 |
<div className="flex items-baseline gap-3">
|
| 3843 |
<span className="font-mono text-[10px] uppercase tracking-[0.2em] text-[color:var(--accent)] font-semibold">
|
| 3844 |
-
{category}
|
| 3845 |
</span>
|
| 3846 |
<span className="font-mono text-[10px] uppercase tracking-[0.15em] text-[color:var(--fg-subtle)]">
|
| 3847 |
{totalBenchmarks} benchmark{totalBenchmarks === 1 ? "" : "s"}
|
|
@@ -4042,7 +4064,7 @@ export function BenchmarkDetail({
|
|
| 4042 |
<section key={`list-cat-${category}`}>
|
| 4043 |
<div className="mb-3 flex items-baseline justify-between gap-3 border-b border-[color:var(--fg)] pb-2">
|
| 4044 |
<span className="font-mono text-[10px] uppercase tracking-[0.2em] text-[color:var(--accent)] font-semibold">
|
| 4045 |
-
{category}
|
| 4046 |
</span>
|
| 4047 |
<span className="font-mono text-[10px] uppercase tracking-[0.15em] text-[color:var(--fg-subtle)]">
|
| 4048 |
{families.length} {families.length === 1 ? "family" : "families"} · {totalBenchmarks} benchmark{totalBenchmarks === 1 ? "" : "s"} · {totalRows} row{totalRows === 1 ? "" : "s"}
|
|
@@ -4883,7 +4905,7 @@ function AggregatedBenchmarkCard({
|
|
| 4883 |
<div className="flex items-center gap-3">
|
| 4884 |
{/* Category dot */}
|
| 4885 |
<span className={`shrink-0 inline-flex items-center rounded-full border px-2 py-0.5 text-[10px] font-semibold ${getCategoryTone(group.category)}`}>
|
| 4886 |
-
{group.category}
|
| 4887 |
</span>
|
| 4888 |
|
| 4889 |
{/* Name + domains */}
|
|
@@ -5453,7 +5475,7 @@ function BenchmarkDeepDiveDialogPanel({
|
|
| 5453 |
<div className="flex items-start justify-between gap-4 pr-6">
|
| 5454 |
<div className="min-w-0 flex-1">
|
| 5455 |
<div className="kicker mb-2">
|
| 5456 |
-
<span className="text-[color:var(--accent)] font-semibold mr-2">{group.category}</span>
|
| 5457 |
<span className="text-[color:var(--fg-subtle)]">· Benchmark deep dive</span>
|
| 5458 |
</div>
|
| 5459 |
<DialogTitle className="text-[28px] leading-[1.05] tracking-[-0.02em] font-bold text-[color:var(--fg)]">
|
|
@@ -6357,7 +6379,7 @@ function EvaluatorMix({ mix }: { mix: EvaluatorMixData }) {
|
|
| 6357 |
style={{ borderBottom: i < rows.length - 1 ? "1px solid var(--border-soft)" : "none" }}
|
| 6358 |
>
|
| 6359 |
<div>
|
| 6360 |
-
<div className="text-[13px] font-medium capitalize">{row.category}</div>
|
| 6361 |
<div className="font-mono text-[10px] uppercase tracking-[0.1em] text-[color:var(--fg-subtle)] mt-0.5">
|
| 6362 |
{row.total} row{row.total === 1 ? "" : "s"}
|
| 6363 |
</div>
|
|
|
|
| 37 |
} from "lucide-react"
|
| 38 |
import type { BenchmarkCard, BenchmarkEvaluation, CategoryType, EvaluationResult } from "@/lib/benchmark-schema"
|
| 39 |
import { getCategoryColor as getCategoryTone, inferCategoryFromBenchmark } from "@/lib/benchmark-schema"
|
| 40 |
+
import { formatTagLabel } from "@/lib/benchmark-tags"
|
| 41 |
import type { BenchmarkEvalSummary } from "@/lib/eval-processing"
|
| 42 |
import type { ModelSummaryCore } from "@/lib/benchmark-schema"
|
| 43 |
import { lookupBenchmarkCard } from "@/lib/benchmark-metadata-utils"
|
|
|
|
| 1820 |
|
| 1821 |
const allCategoryResults = useMemo(
|
| 1822 |
() =>
|
| 1823 |
+
Object.entries(summary.evaluations_by_category).flatMap(([fallbackCategory, evals]) =>
|
| 1824 |
+
evals.flatMap((evaluation) => {
|
| 1825 |
+
// Re-bucket by curated tag from data/benchmarks/categories.json.
|
| 1826 |
+
// The hierarchy lookup gives us the leaf benchmark's derivedTags;
|
| 1827 |
+
// the first tag becomes the displayed category. Fall back to the
|
| 1828 |
+
// legacy 5-bucket category only when no tag is found, so existing
|
| 1829 |
+
// ordering / filter wiring still works.
|
| 1830 |
+
const evalSummaryId = evaluation.eval_summary_id
|
| 1831 |
+
const tags = evalSummaryId ? hierarchyIndex?.get(evalSummaryId)?.tags : undefined
|
| 1832 |
+
const primaryTag = tags && tags.length > 0 ? tags[0] : null
|
| 1833 |
+
const category = (primaryTag ?? fallbackCategory) as CategoryType
|
| 1834 |
+
return evaluation.evaluation_results.map((result) => ({
|
| 1835 |
evaluation,
|
| 1836 |
result,
|
| 1837 |
+
category,
|
| 1838 |
}))
|
| 1839 |
+
})
|
| 1840 |
),
|
| 1841 |
+
[summary.evaluations_by_category, hierarchyIndex]
|
| 1842 |
)
|
| 1843 |
|
| 1844 |
const policyHighlights = useMemo(() => {
|
|
|
|
| 1937 |
[allCategoryResults, benchmarkCards, currentDetailHref]
|
| 1938 |
)
|
| 1939 |
|
| 1940 |
+
// Categories actually present in this model's benchmark groups, derived
|
| 1941 |
+
// from the curated tag bucketing in `allCategoryResults`. We no longer
|
| 1942 |
+
// trust `summary.categories_covered` (legacy 5-bucket) for ordering /
|
| 1943 |
+
// filtering; build the list locally so the new tag vocabulary surfaces.
|
| 1944 |
+
const availableCategories = useMemo(() => {
|
| 1945 |
+
const order: string[] = []
|
| 1946 |
+
const seen = new Set<string>()
|
| 1947 |
+
for (const group of benchmarkGroups) {
|
| 1948 |
+
const cat = group.category as unknown as string
|
| 1949 |
+
if (!seen.has(cat)) {
|
| 1950 |
+
seen.add(cat)
|
| 1951 |
+
order.push(cat)
|
| 1952 |
+
}
|
| 1953 |
+
}
|
| 1954 |
+
return order as unknown as CategoryType[]
|
| 1955 |
+
}, [benchmarkGroups])
|
| 1956 |
+
|
| 1957 |
// First-party vs third-party split per category (for the donut + bars).
|
| 1958 |
const evaluatorMix = useMemo(() => {
|
| 1959 |
+
const order = new Map(availableCategories.map((cat, i) => [cat, i]))
|
| 1960 |
const byCat = new Map<CategoryType, { first: number; third: number; collab: number; other: number }>()
|
| 1961 |
let firstTotal = 0
|
| 1962 |
let thirdTotal = 0
|
|
|
|
| 1990 |
otherTotal,
|
| 1991 |
grand,
|
| 1992 |
}
|
| 1993 |
+
}, [benchmarkGroups, availableCategories])
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1994 |
|
| 1995 |
const filteredBenchmarkGroups = useMemo(() => {
|
| 1996 |
const query = benchmarkSearch.trim().toLowerCase()
|
|
|
|
| 2039 |
}, [benchmarkGroups, benchmarkSearch, benchmarkSort, selectedCategories, modelId, peerRanks])
|
| 2040 |
|
| 2041 |
const groupedFilteredBenchmarkGroups = useMemo(() => {
|
| 2042 |
+
const order = new Map(availableCategories.map((category, index) => [category, index]))
|
| 2043 |
const groups = new Map<CategoryType, BenchmarkGroup[]>()
|
| 2044 |
|
| 2045 |
for (const benchmarkGroup of filteredBenchmarkGroups) {
|
|
|
|
| 2051 |
return Array.from(groups.entries())
|
| 2052 |
.sort((a, b) => (order.get(a[0]) ?? 999) - (order.get(b[0]) ?? 999))
|
| 2053 |
.map(([category, groups]) => ({ category, groups }))
|
| 2054 |
+
}, [filteredBenchmarkGroups, availableCategories])
|
| 2055 |
|
| 2056 |
// Family-bucketed groups for the list view, mirroring plotboxUnits logic.
|
| 2057 |
// When comparisonIndex is available we use the backend-authoritative
|
|
|
|
| 2066 |
}
|
| 2067 |
const listFamiliesByCategory = useMemo(() => {
|
| 2068 |
const order = new Map(
|
| 2069 |
+
availableCategories.map((category, index) => [category, index])
|
| 2070 |
)
|
| 2071 |
const byCategory = new Map<CategoryType, Map<string, ListFamily>>()
|
| 2072 |
|
|
|
|
| 2109 |
kind: f.groups.length > 1 ? "multi-eval" as const : "single-eval" as const,
|
| 2110 |
})),
|
| 2111 |
}))
|
| 2112 |
+
}, [filteredBenchmarkGroups, comparisonIndex, hierarchyIndex, availableCategories])
|
| 2113 |
|
| 2114 |
const compositeGroups = useMemo(() => {
|
| 2115 |
const groups = groupByComposite(filteredBenchmarkGroups, modelIds, peerRanks, hierarchyIndex)
|
|
|
|
| 3020 |
<div className="min-w-0 flex-1">
|
| 3021 |
<div className="flex flex-wrap items-center gap-2">
|
| 3022 |
<span className="font-mono text-[9px] uppercase tracking-[0.18em] text-[color:var(--fg-subtle)] font-semibold">
|
| 3023 |
+
{formatTagLabel(unit.category as unknown as string)}
|
| 3024 |
</span>
|
| 3025 |
{showChildKindBadge && (
|
| 3026 |
<span className="font-mono text-[9px] uppercase tracking-[0.15em] text-[color:var(--fg-subtle)]">
|
|
|
|
| 3816 |
}
|
| 3817 |
className={`ec-pill ${isSelected ? "on" : ""}`}
|
| 3818 |
>
|
| 3819 |
+
{formatTagLabel(category as unknown as string)}
|
| 3820 |
</button>
|
| 3821 |
)
|
| 3822 |
})}
|
|
|
|
| 3830 |
) : benchmarkViewMode === "grid" ? (
|
| 3831 |
(() => {
|
| 3832 |
const categoryOrder = new Map(
|
| 3833 |
+
availableCategories.map((cat, i) => [cat, i])
|
| 3834 |
)
|
| 3835 |
const byCategory = new Map<CategoryType, PlotboxUnit[]>()
|
| 3836 |
for (const unit of plotboxUnits) {
|
|
|
|
| 3863 |
<div className="flex items-baseline justify-between gap-3 border-b border-[color:var(--border-soft)] pb-2">
|
| 3864 |
<div className="flex items-baseline gap-3">
|
| 3865 |
<span className="font-mono text-[10px] uppercase tracking-[0.2em] text-[color:var(--accent)] font-semibold">
|
| 3866 |
+
{formatTagLabel(category)}
|
| 3867 |
</span>
|
| 3868 |
<span className="font-mono text-[10px] uppercase tracking-[0.15em] text-[color:var(--fg-subtle)]">
|
| 3869 |
{totalBenchmarks} benchmark{totalBenchmarks === 1 ? "" : "s"}
|
|
|
|
| 4064 |
<section key={`list-cat-${category}`}>
|
| 4065 |
<div className="mb-3 flex items-baseline justify-between gap-3 border-b border-[color:var(--fg)] pb-2">
|
| 4066 |
<span className="font-mono text-[10px] uppercase tracking-[0.2em] text-[color:var(--accent)] font-semibold">
|
| 4067 |
+
{formatTagLabel(category as unknown as string)}
|
| 4068 |
</span>
|
| 4069 |
<span className="font-mono text-[10px] uppercase tracking-[0.15em] text-[color:var(--fg-subtle)]">
|
| 4070 |
{families.length} {families.length === 1 ? "family" : "families"} · {totalBenchmarks} benchmark{totalBenchmarks === 1 ? "" : "s"} · {totalRows} row{totalRows === 1 ? "" : "s"}
|
|
|
|
| 4905 |
<div className="flex items-center gap-3">
|
| 4906 |
{/* Category dot */}
|
| 4907 |
<span className={`shrink-0 inline-flex items-center rounded-full border px-2 py-0.5 text-[10px] font-semibold ${getCategoryTone(group.category)}`}>
|
| 4908 |
+
{formatTagLabel(group.category as unknown as string)}
|
| 4909 |
</span>
|
| 4910 |
|
| 4911 |
{/* Name + domains */}
|
|
|
|
| 5475 |
<div className="flex items-start justify-between gap-4 pr-6">
|
| 5476 |
<div className="min-w-0 flex-1">
|
| 5477 |
<div className="kicker mb-2">
|
| 5478 |
+
<span className="text-[color:var(--accent)] font-semibold mr-2">{formatTagLabel(group.category as unknown as string)}</span>
|
| 5479 |
<span className="text-[color:var(--fg-subtle)]">· Benchmark deep dive</span>
|
| 5480 |
</div>
|
| 5481 |
<DialogTitle className="text-[28px] leading-[1.05] tracking-[-0.02em] font-bold text-[color:var(--fg)]">
|
|
|
|
| 6379 |
style={{ borderBottom: i < rows.length - 1 ? "1px solid var(--border-soft)" : "none" }}
|
| 6380 |
>
|
| 6381 |
<div>
|
| 6382 |
+
<div className="text-[13px] font-medium capitalize">{formatTagLabel(row.category as unknown as string)}</div>
|
| 6383 |
<div className="font-mono text-[10px] uppercase tracking-[0.1em] text-[color:var(--fg-subtle)] mt-0.5">
|
| 6384 |
{row.total} row{row.total === 1 ? "" : "s"}
|
| 6385 |
</div>
|
components/family-table.tsx
CHANGED
|
@@ -5,7 +5,8 @@ import { useRouter } from "next/navigation"
|
|
| 5 |
import { ArrowUpRight, ChevronDown, ChevronRight } from "lucide-react"
|
| 6 |
|
| 7 |
import type { HierarchyBenchmark, HierarchyFamily } from "@/lib/backend-artifacts"
|
| 8 |
-
import
|
|
|
|
| 9 |
import type { BenchmarkEvalListItem } from "@/lib/eval-processing"
|
| 10 |
import { humanizeEvaluationId } from "@/lib/utils"
|
| 11 |
|
|
@@ -35,6 +36,11 @@ interface FamilyTableProps {
|
|
| 35 |
* families are kept only when their domains intersect the filter.
|
| 36 |
* Pass `null`/`undefined` to disable filtering. */
|
| 37 |
domainFilter?: Set<string> | null
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 38 |
}
|
| 39 |
|
| 40 |
function slugify(value: string | null | undefined): string {
|
|
@@ -66,6 +72,7 @@ interface LeafEntry {
|
|
| 66 |
leafName: string
|
| 67 |
evalsCount: number
|
| 68 |
domains: string[]
|
|
|
|
| 69 |
}
|
| 70 |
|
| 71 |
/**
|
|
@@ -115,6 +122,7 @@ function collectLeafEntries(
|
|
| 115 |
leafName: benchmark.display_name || benchmark.key,
|
| 116 |
evalsCount: ids.length,
|
| 117 |
domains: Array.from(collected),
|
|
|
|
| 118 |
})
|
| 119 |
}
|
| 120 |
|
|
@@ -231,7 +239,7 @@ interface RowData {
|
|
| 231 |
navId: string | null
|
| 232 |
name: string
|
| 233 |
keySlug: string
|
| 234 |
-
|
| 235 |
benchmarks: number
|
| 236 |
evalsCount: number
|
| 237 |
leaves: LeafEntry[]
|
|
@@ -247,24 +255,48 @@ export function FamilyTable({
|
|
| 247 |
evalItems,
|
| 248 |
benchmarkCards,
|
| 249 |
domainFilter,
|
|
|
|
| 250 |
}: FamilyTableProps) {
|
| 251 |
const router = useRouter()
|
| 252 |
const [expanded, setExpanded] = useState<Record<string, boolean>>({})
|
| 253 |
|
| 254 |
-
const
|
|
|
|
|
|
|
| 255 |
|
| 256 |
-
function
|
| 257 |
-
if (!
|
| 258 |
return leaf.domains.some((d) => domainFilter.has(d))
|
| 259 |
}
|
| 260 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 261 |
function familyMatchesFilter(
|
| 262 |
fam: HierarchyFamily,
|
| 263 |
navId: string | null,
|
| 264 |
leafEntries: LeafEntry[],
|
| 265 |
): boolean {
|
| 266 |
-
if (!filterActive
|
| 267 |
if (leafEntries.some(leafMatchesFilter)) return true
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 268 |
const candidates: BenchmarkCard | undefined = (() => {
|
| 269 |
if (navId) {
|
| 270 |
const fromList = evalItems?.get(navId)?.benchmark_card
|
|
@@ -336,7 +368,7 @@ export function FamilyTable({
|
|
| 336 |
navId,
|
| 337 |
name: displayName,
|
| 338 |
keySlug: fam.key,
|
| 339 |
-
|
| 340 |
benchmarks: benchmarkCount,
|
| 341 |
evalsCount: fam.evals_count ?? metricCount,
|
| 342 |
leaves: visibleLeafEntries,
|
|
@@ -346,15 +378,15 @@ export function FamilyTable({
|
|
| 346 |
}
|
| 347 |
return out
|
| 348 |
// eslint-disable-next-line react-hooks/exhaustive-deps
|
| 349 |
-
}, [families, evalItems, benchmarkCards, domainFilter])
|
| 350 |
|
| 351 |
return (
|
| 352 |
<div className="overflow-x-auto">
|
| 353 |
<table className="ec-htable">
|
| 354 |
<thead>
|
| 355 |
<tr>
|
| 356 |
-
<th style={{ width: "
|
| 357 |
-
<th>
|
| 358 |
<th className="num">Benchmarks</th>
|
| 359 |
<th className="num">Reported results</th>
|
| 360 |
<th style={{ width: 90 }} />
|
|
@@ -435,9 +467,33 @@ export function FamilyTable({
|
|
| 435 |
</div>
|
| 436 |
</td>
|
| 437 |
<td>
|
| 438 |
-
|
| 439 |
-
|
| 440 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 441 |
</td>
|
| 442 |
<td className="num font-mono text-[13px]">
|
| 443 |
{row.benchmarks.toLocaleString()}
|
|
|
|
| 5 |
import { ArrowUpRight, ChevronDown, ChevronRight } from "lucide-react"
|
| 6 |
|
| 7 |
import type { HierarchyBenchmark, HierarchyFamily } from "@/lib/backend-artifacts"
|
| 8 |
+
import { formatTagLabel } from "@/lib/benchmark-tags"
|
| 9 |
+
import type { BenchmarkCard } from "@/lib/benchmark-schema"
|
| 10 |
import type { BenchmarkEvalListItem } from "@/lib/eval-processing"
|
| 11 |
import { humanizeEvaluationId } from "@/lib/utils"
|
| 12 |
|
|
|
|
| 36 |
* families are kept only when their domains intersect the filter.
|
| 37 |
* Pass `null`/`undefined` to disable filtering. */
|
| 38 |
domainFilter?: Set<string> | null
|
| 39 |
+
/** Curated category-tag slugs (data/benchmarks/categories.json
|
| 40 |
+
* vocabulary) to filter on. Same expand-and-filter behaviour as
|
| 41 |
+
* `domainFilter`: matching families auto-expand, leaves are
|
| 42 |
+
* restricted to those whose `derivedTags` intersect the selection. */
|
| 43 |
+
categoryFilter?: Set<string> | null
|
| 44 |
}
|
| 45 |
|
| 46 |
function slugify(value: string | null | undefined): string {
|
|
|
|
| 72 |
leafName: string
|
| 73 |
evalsCount: number
|
| 74 |
domains: string[]
|
| 75 |
+
tags: string[]
|
| 76 |
}
|
| 77 |
|
| 78 |
/**
|
|
|
|
| 122 |
leafName: benchmark.display_name || benchmark.key,
|
| 123 |
evalsCount: ids.length,
|
| 124 |
domains: Array.from(collected),
|
| 125 |
+
tags: benchmark.derivedTags ?? [],
|
| 126 |
})
|
| 127 |
}
|
| 128 |
|
|
|
|
| 239 |
navId: string | null
|
| 240 |
name: string
|
| 241 |
keySlug: string
|
| 242 |
+
tags: string[]
|
| 243 |
benchmarks: number
|
| 244 |
evalsCount: number
|
| 245 |
leaves: LeafEntry[]
|
|
|
|
| 255 |
evalItems,
|
| 256 |
benchmarkCards,
|
| 257 |
domainFilter,
|
| 258 |
+
categoryFilter,
|
| 259 |
}: FamilyTableProps) {
|
| 260 |
const router = useRouter()
|
| 261 |
const [expanded, setExpanded] = useState<Record<string, boolean>>({})
|
| 262 |
|
| 263 |
+
const domainFilterActive = Boolean(domainFilter && domainFilter.size > 0)
|
| 264 |
+
const categoryFilterActive = Boolean(categoryFilter && categoryFilter.size > 0)
|
| 265 |
+
const filterActive = domainFilterActive || categoryFilterActive
|
| 266 |
|
| 267 |
+
function leafMatchesDomain(leaf: LeafEntry): boolean {
|
| 268 |
+
if (!domainFilterActive || !domainFilter) return true
|
| 269 |
return leaf.domains.some((d) => domainFilter.has(d))
|
| 270 |
}
|
| 271 |
|
| 272 |
+
function leafMatchesCategory(leaf: LeafEntry): boolean {
|
| 273 |
+
if (!categoryFilterActive || !categoryFilter) return true
|
| 274 |
+
return leaf.tags.some((t) => categoryFilter.has(t))
|
| 275 |
+
}
|
| 276 |
+
|
| 277 |
+
function leafMatchesFilter(leaf: LeafEntry): boolean {
|
| 278 |
+
return leafMatchesDomain(leaf) && leafMatchesCategory(leaf)
|
| 279 |
+
}
|
| 280 |
+
|
| 281 |
function familyMatchesFilter(
|
| 282 |
fam: HierarchyFamily,
|
| 283 |
navId: string | null,
|
| 284 |
leafEntries: LeafEntry[],
|
| 285 |
): boolean {
|
| 286 |
+
if (!filterActive) return true
|
| 287 |
if (leafEntries.some(leafMatchesFilter)) return true
|
| 288 |
+
if (categoryFilterActive && categoryFilter) {
|
| 289 |
+
// Family-level tag union (from derivedTags) — covers single-benchmark
|
| 290 |
+
// families and aggregator families whose own bucket holds the tag
|
| 291 |
+
// even if no leaf row carries it.
|
| 292 |
+
for (const tag of fam.derivedTags ?? []) {
|
| 293 |
+
if (categoryFilter.has(tag)) {
|
| 294 |
+
// Only counts if the domain side also matches (or is inactive).
|
| 295 |
+
if (!domainFilterActive) return true
|
| 296 |
+
}
|
| 297 |
+
}
|
| 298 |
+
}
|
| 299 |
+
if (!domainFilterActive || !domainFilter) return false
|
| 300 |
const candidates: BenchmarkCard | undefined = (() => {
|
| 301 |
if (navId) {
|
| 302 |
const fromList = evalItems?.get(navId)?.benchmark_card
|
|
|
|
| 368 |
navId,
|
| 369 |
name: displayName,
|
| 370 |
keySlug: fam.key,
|
| 371 |
+
tags: fam.derivedTags ?? [],
|
| 372 |
benchmarks: benchmarkCount,
|
| 373 |
evalsCount: fam.evals_count ?? metricCount,
|
| 374 |
leaves: visibleLeafEntries,
|
|
|
|
| 378 |
}
|
| 379 |
return out
|
| 380 |
// eslint-disable-next-line react-hooks/exhaustive-deps
|
| 381 |
+
}, [families, evalItems, benchmarkCards, domainFilter, categoryFilter])
|
| 382 |
|
| 383 |
return (
|
| 384 |
<div className="overflow-x-auto">
|
| 385 |
<table className="ec-htable">
|
| 386 |
<thead>
|
| 387 |
<tr>
|
| 388 |
+
<th style={{ width: "55%" }}>Family</th>
|
| 389 |
+
<th>Categories</th>
|
| 390 |
<th className="num">Benchmarks</th>
|
| 391 |
<th className="num">Reported results</th>
|
| 392 |
<th style={{ width: 90 }} />
|
|
|
|
| 467 |
</div>
|
| 468 |
</td>
|
| 469 |
<td>
|
| 470 |
+
{row.tags.length === 0 ? (
|
| 471 |
+
<span
|
| 472 |
+
className="inline-flex items-center font-mono text-[10px] uppercase tracking-[0.12em] text-[color:var(--fg-subtle)]"
|
| 473 |
+
>
|
| 474 |
+
—
|
| 475 |
+
</span>
|
| 476 |
+
) : (
|
| 477 |
+
<div className="flex flex-wrap gap-1">
|
| 478 |
+
{row.tags.map((tag) => {
|
| 479 |
+
const highlighted =
|
| 480 |
+
categoryFilter && categoryFilter.has(tag)
|
| 481 |
+
return (
|
| 482 |
+
<span
|
| 483 |
+
key={tag}
|
| 484 |
+
className="inline-flex items-center font-mono text-[10px] uppercase tracking-[0.12em] border px-1.5 py-0.5"
|
| 485 |
+
style={{
|
| 486 |
+
color: highlighted ? "var(--bg)" : "var(--fg-muted)",
|
| 487 |
+
borderColor: highlighted ? "var(--fg)" : "var(--border-soft)",
|
| 488 |
+
background: highlighted ? "var(--fg)" : "var(--bg)",
|
| 489 |
+
}}
|
| 490 |
+
>
|
| 491 |
+
{formatTagLabel(tag)}
|
| 492 |
+
</span>
|
| 493 |
+
)
|
| 494 |
+
})}
|
| 495 |
+
</div>
|
| 496 |
+
)}
|
| 497 |
</td>
|
| 498 |
<td className="num font-mono text-[13px]">
|
| 499 |
{row.benchmarks.toLocaleString()}
|
lib/benchmark-tags.ts
CHANGED
|
@@ -146,31 +146,121 @@ export function hasCuratedTags(...candidates: Array<string | null | undefined>):
|
|
| 146 |
* benchmarks, 709 slices): 95.8% / 98.6% / 99.7% respectively.
|
| 147 |
*/
|
| 148 |
export function decorateHierarchyDerivedTags(h: EvalHierarchy): EvalHierarchy {
|
| 149 |
-
for (const fam of h.families ?? [])
|
|
|
|
|
|
|
|
|
|
| 150 |
return h
|
| 151 |
}
|
| 152 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 153 |
function decorateFamily(fam: HierarchyFamily): void {
|
| 154 |
fam.derivedTags = getBenchmarkTags(null, fam.display_name, fam.key)
|
| 155 |
for (const b of fam.standalone_benchmarks ?? []) decorateBenchmark(b, fam.derivedTags)
|
| 156 |
for (const b of fam.benchmarks ?? []) decorateBenchmark(b, fam.derivedTags)
|
| 157 |
for (const c of fam.composites ?? []) decorateComposite(c, fam.derivedTags)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 158 |
}
|
| 159 |
|
| 160 |
function decorateComposite(comp: HierarchyComposite, parentTags: string[]): void {
|
| 161 |
comp.derivedTags = getBenchmarkTags(parentTags, comp.display_name, comp.key)
|
| 162 |
for (const b of comp.benchmarks ?? []) decorateBenchmark(b, comp.derivedTags)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 163 |
}
|
| 164 |
|
| 165 |
function decorateBenchmark(b: HierarchyBenchmark, parentTags: string[]): void {
|
| 166 |
b.derivedTags = getBenchmarkTags(parentTags, b.display_name, b.key)
|
| 167 |
for (const s of b.slices ?? []) decorateSlice(s, b.derivedTags)
|
|
|
|
| 168 |
}
|
| 169 |
|
| 170 |
function decorateSlice(s: HierarchySlice, parentTags: string[]): void {
|
| 171 |
s.derivedTags = getBenchmarkTags(parentTags, s.display_name, s.key)
|
| 172 |
}
|
| 173 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 174 |
/**
|
| 175 |
* Render a tag for display: snake_case → Sentence case.
|
| 176 |
* "software_engineering" → "Software engineering";
|
|
|
|
| 146 |
* benchmarks, 709 slices): 95.8% / 98.6% / 99.7% respectively.
|
| 147 |
*/
|
| 148 |
export function decorateHierarchyDerivedTags(h: EvalHierarchy): EvalHierarchy {
|
| 149 |
+
for (const fam of h.families ?? []) {
|
| 150 |
+
sanitizeFamilyDisplayNames(fam)
|
| 151 |
+
decorateFamily(fam)
|
| 152 |
+
}
|
| 153 |
return h
|
| 154 |
}
|
| 155 |
|
| 156 |
+
// Workaround for an upstream warehouse bug where some families inherit a
|
| 157 |
+
// sibling family's `display_name` (e.g. `math-mc` and `gsm-mc` both ship
|
| 158 |
+
// with "wasp (Writer's Assessor of System Performance)"). When the
|
| 159 |
+
// display_name shares no token with the entry's `key`, fall back to a
|
| 160 |
+
// readable rendering of the key.
|
| 161 |
+
function shareToken(displayName: string, key: string): boolean {
|
| 162 |
+
const norm = (s: string) => s.toLowerCase().replace(/[^a-z0-9]+/g, "")
|
| 163 |
+
const k = norm(key)
|
| 164 |
+
if (!k) return true
|
| 165 |
+
return norm(displayName).includes(k)
|
| 166 |
+
}
|
| 167 |
+
|
| 168 |
+
// Acronyms that should stay uppercase when humanizing a slug. Mirrors
|
| 169 |
+
// the set used in family-table.tsx; kept here so the sanitiser can
|
| 170 |
+
// produce the same output across surfaces.
|
| 171 |
+
const ACRONYMS = new Set([
|
| 172 |
+
"ai", "aa", "api", "arc", "bbh", "bfcl", "cli", "cv", "gpqa", "gpt",
|
| 173 |
+
"gsm", "hf", "hle", "llm", "llms", "mc", "ml", "mt", "nlp", "qa",
|
| 174 |
+
"rl", "sql", "swe", "vlm", "vqa",
|
| 175 |
+
])
|
| 176 |
+
|
| 177 |
+
function humanizeKey(key: string): string {
|
| 178 |
+
const parts = key.split(/[_\-\s]+/).filter(Boolean)
|
| 179 |
+
if (parts.length === 0) return key
|
| 180 |
+
return parts
|
| 181 |
+
.map((word) => {
|
| 182 |
+
const lower = word.toLowerCase()
|
| 183 |
+
if (ACRONYMS.has(lower)) return word.toUpperCase()
|
| 184 |
+
// Treat short all-letter parts (≤4 chars) as acronym-like.
|
| 185 |
+
if (word.length <= 4 && /^[a-zA-Z]+$/.test(word)) return word.toUpperCase()
|
| 186 |
+
return word.charAt(0).toUpperCase() + word.slice(1).toLowerCase()
|
| 187 |
+
})
|
| 188 |
+
.join("-")
|
| 189 |
+
}
|
| 190 |
+
|
| 191 |
+
function sanitizeName(displayName: string | null | undefined, key: string): string {
|
| 192 |
+
if (!displayName || !displayName.trim()) return humanizeKey(key)
|
| 193 |
+
if (!shareToken(displayName, key)) return humanizeKey(key)
|
| 194 |
+
return displayName
|
| 195 |
+
}
|
| 196 |
+
|
| 197 |
+
function sanitizeFamilyDisplayNames(fam: HierarchyFamily): void {
|
| 198 |
+
fam.display_name = sanitizeName(fam.display_name, fam.key)
|
| 199 |
+
for (const c of fam.composites ?? []) {
|
| 200 |
+
c.display_name = sanitizeName(c.display_name, c.key)
|
| 201 |
+
for (const b of c.benchmarks ?? []) {
|
| 202 |
+
b.display_name = sanitizeName(b.display_name, b.key)
|
| 203 |
+
for (const s of b.slices ?? []) s.display_name = sanitizeName(s.display_name, s.key)
|
| 204 |
+
}
|
| 205 |
+
}
|
| 206 |
+
for (const b of fam.standalone_benchmarks ?? []) {
|
| 207 |
+
b.display_name = sanitizeName(b.display_name, b.key)
|
| 208 |
+
for (const s of b.slices ?? []) s.display_name = sanitizeName(s.display_name, s.key)
|
| 209 |
+
}
|
| 210 |
+
for (const b of fam.benchmarks ?? []) {
|
| 211 |
+
b.display_name = sanitizeName(b.display_name, b.key)
|
| 212 |
+
for (const s of b.slices ?? []) s.display_name = sanitizeName(s.display_name, s.key)
|
| 213 |
+
}
|
| 214 |
+
}
|
| 215 |
+
|
| 216 |
function decorateFamily(fam: HierarchyFamily): void {
|
| 217 |
fam.derivedTags = getBenchmarkTags(null, fam.display_name, fam.key)
|
| 218 |
for (const b of fam.standalone_benchmarks ?? []) decorateBenchmark(b, fam.derivedTags)
|
| 219 |
for (const b of fam.benchmarks ?? []) decorateBenchmark(b, fam.derivedTags)
|
| 220 |
for (const c of fam.composites ?? []) decorateComposite(c, fam.derivedTags)
|
| 221 |
+
// Bottom-up union: parents accumulate their descendants' tags so a
|
| 222 |
+
// family-level filter ("mathematics") matches families whose own name
|
| 223 |
+
// doesn't, but whose children do.
|
| 224 |
+
fam.derivedTags = unionTags(
|
| 225 |
+
fam.derivedTags,
|
| 226 |
+
...(fam.standalone_benchmarks ?? []).map((b) => b.derivedTags ?? []),
|
| 227 |
+
...(fam.benchmarks ?? []).map((b) => b.derivedTags ?? []),
|
| 228 |
+
...(fam.composites ?? []).map((c) => c.derivedTags ?? []),
|
| 229 |
+
)
|
| 230 |
}
|
| 231 |
|
| 232 |
function decorateComposite(comp: HierarchyComposite, parentTags: string[]): void {
|
| 233 |
comp.derivedTags = getBenchmarkTags(parentTags, comp.display_name, comp.key)
|
| 234 |
for (const b of comp.benchmarks ?? []) decorateBenchmark(b, comp.derivedTags)
|
| 235 |
+
comp.derivedTags = unionTags(
|
| 236 |
+
comp.derivedTags,
|
| 237 |
+
...(comp.benchmarks ?? []).map((b) => b.derivedTags ?? []),
|
| 238 |
+
)
|
| 239 |
}
|
| 240 |
|
| 241 |
function decorateBenchmark(b: HierarchyBenchmark, parentTags: string[]): void {
|
| 242 |
b.derivedTags = getBenchmarkTags(parentTags, b.display_name, b.key)
|
| 243 |
for (const s of b.slices ?? []) decorateSlice(s, b.derivedTags)
|
| 244 |
+
b.derivedTags = unionTags(b.derivedTags, ...(b.slices ?? []).map((s) => s.derivedTags ?? []))
|
| 245 |
}
|
| 246 |
|
| 247 |
function decorateSlice(s: HierarchySlice, parentTags: string[]): void {
|
| 248 |
s.derivedTags = getBenchmarkTags(parentTags, s.display_name, s.key)
|
| 249 |
}
|
| 250 |
|
| 251 |
+
function unionTags(...lists: Array<string[] | null | undefined>): string[] {
|
| 252 |
+
const seen = new Set<string>()
|
| 253 |
+
const out: string[] = []
|
| 254 |
+
for (const list of lists) {
|
| 255 |
+
for (const tag of list ?? []) {
|
| 256 |
+
if (!tag || seen.has(tag)) continue
|
| 257 |
+
seen.add(tag)
|
| 258 |
+
out.push(tag)
|
| 259 |
+
}
|
| 260 |
+
}
|
| 261 |
+
return out
|
| 262 |
+
}
|
| 263 |
+
|
| 264 |
/**
|
| 265 |
* Render a tag for display: snake_case → Sentence case.
|
| 266 |
* "software_engineering" → "Software engineering";
|
lib/hierarchy-lookup.ts
CHANGED
|
@@ -9,11 +9,17 @@ export interface HierarchyEvalLocation {
|
|
| 9 |
familyDisplayName: string
|
| 10 |
compositeKey?: string
|
| 11 |
compositeDisplayName?: string
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 12 |
}
|
| 13 |
|
| 14 |
interface FamilyAppearance {
|
| 15 |
family: HierarchyFamily
|
| 16 |
composite?: HierarchyComposite
|
|
|
|
| 17 |
}
|
| 18 |
|
| 19 |
function findComposite(
|
|
@@ -28,6 +34,24 @@ function findComposite(
|
|
| 28 |
return composites.find((composite) => composite.key === sourcePrefix)
|
| 29 |
}
|
| 30 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 31 |
function buildAppearancesIndex(
|
| 32 |
hierarchy: EvalHierarchy | null | undefined,
|
| 33 |
): Map<string, FamilyAppearance[]> {
|
|
@@ -39,8 +63,9 @@ function buildAppearancesIndex(
|
|
| 39 |
for (const family of hierarchy.families) {
|
| 40 |
for (const evalSummaryId of family.eval_summary_ids ?? []) {
|
| 41 |
const composite = findComposite(family, evalSummaryId)
|
|
|
|
| 42 |
const list = index.get(evalSummaryId) ?? []
|
| 43 |
-
list.push({ family, composite })
|
| 44 |
index.set(evalSummaryId, list)
|
| 45 |
}
|
| 46 |
}
|
|
@@ -80,11 +105,20 @@ export function buildHierarchyEvalIndex(
|
|
| 80 |
}
|
| 81 |
}
|
| 82 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 83 |
index.set(evalSummaryId, {
|
| 84 |
familyKey: chosen.family.key,
|
| 85 |
familyDisplayName: chosen.family.display_name,
|
| 86 |
compositeKey: chosen.composite?.key,
|
| 87 |
compositeDisplayName: chosen.composite?.display_name,
|
|
|
|
| 88 |
})
|
| 89 |
}
|
| 90 |
|
|
|
|
| 9 |
familyDisplayName: string
|
| 10 |
compositeKey?: string
|
| 11 |
compositeDisplayName?: string
|
| 12 |
+
/** Curated category tags (data/benchmarks/categories.json vocabulary)
|
| 13 |
+
* for the leaf benchmark this eval belongs to, falling back to its
|
| 14 |
+
* composite/family. Decorated by `decorateHierarchyDerivedTags` at
|
| 15 |
+
* hydration time. */
|
| 16 |
+
tags?: string[]
|
| 17 |
}
|
| 18 |
|
| 19 |
interface FamilyAppearance {
|
| 20 |
family: HierarchyFamily
|
| 21 |
composite?: HierarchyComposite
|
| 22 |
+
benchmarkTags?: string[]
|
| 23 |
}
|
| 24 |
|
| 25 |
function findComposite(
|
|
|
|
| 34 |
return composites.find((composite) => composite.key === sourcePrefix)
|
| 35 |
}
|
| 36 |
|
| 37 |
+
function findBenchmarkTags(
|
| 38 |
+
family: HierarchyFamily,
|
| 39 |
+
composite: HierarchyComposite | undefined,
|
| 40 |
+
evalSummaryId: string,
|
| 41 |
+
): string[] | undefined {
|
| 42 |
+
const benchmarks = [
|
| 43 |
+
...(composite?.benchmarks ?? []),
|
| 44 |
+
...(family.standalone_benchmarks ?? []),
|
| 45 |
+
...(family.benchmarks ?? []),
|
| 46 |
+
]
|
| 47 |
+
for (const benchmark of benchmarks) {
|
| 48 |
+
if (benchmark.summary_eval_ids?.includes(evalSummaryId)) {
|
| 49 |
+
return benchmark.derivedTags
|
| 50 |
+
}
|
| 51 |
+
}
|
| 52 |
+
return undefined
|
| 53 |
+
}
|
| 54 |
+
|
| 55 |
function buildAppearancesIndex(
|
| 56 |
hierarchy: EvalHierarchy | null | undefined,
|
| 57 |
): Map<string, FamilyAppearance[]> {
|
|
|
|
| 63 |
for (const family of hierarchy.families) {
|
| 64 |
for (const evalSummaryId of family.eval_summary_ids ?? []) {
|
| 65 |
const composite = findComposite(family, evalSummaryId)
|
| 66 |
+
const benchmarkTags = findBenchmarkTags(family, composite, evalSummaryId)
|
| 67 |
const list = index.get(evalSummaryId) ?? []
|
| 68 |
+
list.push({ family, composite, benchmarkTags })
|
| 69 |
index.set(evalSummaryId, list)
|
| 70 |
}
|
| 71 |
}
|
|
|
|
| 105 |
}
|
| 106 |
}
|
| 107 |
|
| 108 |
+
// Tag preference order for the leaf: benchmark > composite > family.
|
| 109 |
+
// We want the most specific tags available so the model-view bucketing
|
| 110 |
+
// groups by leaf semantics, not by the family-level union.
|
| 111 |
+
const tags =
|
| 112 |
+
chosen.benchmarkTags && chosen.benchmarkTags.length > 0
|
| 113 |
+
? chosen.benchmarkTags
|
| 114 |
+
: chosen.composite?.derivedTags ?? chosen.family.derivedTags ?? []
|
| 115 |
+
|
| 116 |
index.set(evalSummaryId, {
|
| 117 |
familyKey: chosen.family.key,
|
| 118 |
familyDisplayName: chosen.family.display_name,
|
| 119 |
compositeKey: chosen.composite?.key,
|
| 120 |
compositeDisplayName: chosen.composite?.display_name,
|
| 121 |
+
tags,
|
| 122 |
})
|
| 123 |
}
|
| 124 |
|