"use client" import { Suspense, useCallback, useDeferredValue, useEffect, useMemo, useState } from "react" import { useRouter, useSearchParams } from "next/navigation" import { Search } from "lucide-react" import { EvaluatorTable, type EvaluatorTableSortCol } from "@/components/evaluator-table" import { FamilyTable, getFamilyNavId, type FamilySortCol } from "@/components/family-table" import { InfiniteScrollSentinel } from "@/components/infinite-scroll" import { Navigation } from "@/components/navigation" import { PageLoadingState, type PageLoadingStage } from "@/components/page-loading-state" import type { EvalHierarchy, HierarchyFamily } from "@/lib/backend-artifacts" import { fetchBenchmarkMetadata, fetchEvalHierarchy, fetchEvalList } from "@/lib/dashboard-data-client" import type { BenchmarkEvalListItem } from "@/lib/eval-processing" import type { BenchmarkCard } from "@/lib/benchmark-schema" import { formatTagLabel } from "@/lib/benchmark-tags" import { groupEvalsByEvaluator, verifiedEvalIds } from "@/lib/evaluators" const PAGE_SIZE = 60 function getFamilyBenchmarkCount(fam: HierarchyFamily): number { return ( (fam.standalone_benchmarks?.length ?? 0) + (fam.benchmarks?.length ?? 0) + (fam.composites ?? []).reduce((sum, c) => sum + (c.benchmarks?.length ?? 0), 0) ) } function getFamilyEvalsCount(fam: HierarchyFamily): number { if (fam.evals_count != null) return fam.evals_count const all = [ ...(fam.standalone_benchmarks ?? []), ...(fam.benchmarks ?? []), ...(fam.composites ?? []).flatMap((c) => c.benchmarks ?? []), ] return all.reduce((sum, b) => sum + (b.metrics?.length ?? 0), 0) } function EvalsPageInner() { const router = useRouter() const searchParams = useSearchParams() const familyParam = searchParams.get("family") const queryParam = searchParams.get("q") const groupByParam = searchParams.get("groupBy") const verifiedParam = searchParams.get("verified") const [hierarchy, setHierarchy] = useState(null) const [totalModels, setTotalModels] = useState(0) const [evalItems, setEvalItems] = useState>(new Map()) const [benchmarkCards, setBenchmarkCards] = useState>({}) const [hierarchyStageDone, setHierarchyStageDone] = useState(false) const [evalListStageDone, setEvalListStageDone] = useState(false) const [metadataStageDone, setMetadataStageDone] = useState(false) const [loading, setLoading] = useState(true) const [searchQuery, setSearchQuery] = useState("") const [visibleCount, setVisibleCount] = useState(PAGE_SIZE) const [selectedCategories, setSelectedCategories] = useState([]) const [agentMode, setAgentMode] = useState<"all" | "agentic" | "non-agentic">("all") const [sortCol, setSortCol] = useState("name") const [sortDir, setSortDir] = useState<"asc" | "desc">("asc") const [groupBy, setGroupBy] = useState<"family" | "evaluator">( groupByParam === "evaluator" ? "evaluator" : "family", ) const [verifiedOnly, setVerifiedOnly] = useState( verifiedParam === "1" || verifiedParam === "true", ) const [evaluatorSortCol, setEvaluatorSortCol] = useState("evals") const [evaluatorSortDir, setEvaluatorSortDir] = useState<"asc" | "desc">("desc") const deferredSearchQuery = useDeferredValue(searchQuery) const handleEvaluatorSort = useCallback((col: EvaluatorTableSortCol) => { setEvaluatorSortCol((current) => { if (current === col) { setEvaluatorSortDir((dir) => (dir === "asc" ? "desc" : "asc")) return current } setEvaluatorSortDir(col === "name" ? "asc" : "desc") return col }) }, []) const handleSort = useCallback((col: FamilySortCol) => { if (sortCol === col) { setSortDir((d) => (d === "asc" ? "desc" : "asc")) } else { setSortCol(col) setSortDir("asc") } }, [sortCol]) useEffect(() => { const hierarchyRequest = fetchEvalHierarchy() .then((h) => { setHierarchy(h) setHierarchyStageDone(true) }) .catch(console.error) const evalListRequest = fetchEvalList() .then((list) => { setTotalModels(list.totalModels) const map = new Map() for (const item of list.evals) map.set(item.evaluation_id, item) setEvalItems(map) setEvalListStageDone(true) }) .catch(console.error) const benchmarkMetadataRequest = fetchBenchmarkMetadata() .then((metadata) => { setBenchmarkCards(metadata) setMetadataStageDone(true) }) .catch(console.error) Promise.allSettled([hierarchyRequest, evalListRequest, benchmarkMetadataRequest]) .finally(() => setLoading(false)) }, []) const loadingStages = useMemo(() => [ { label: "Evaluation hierarchy", done: hierarchyStageDone }, { label: `Evaluation index${totalModels > 0 ? ` (${totalModels.toLocaleString()} models)` : ""}`, done: evalListStageDone }, { label: "Benchmark metadata", done: metadataStageDone }, ], [evalListStageDone, hierarchyStageDone, metadataStageDone, totalModels]) // Resolve the `?q=` deep link from tag chips on benchmark pages. useEffect(() => { if (queryParam) setSearchQuery(queryParam) }, [queryParam]) // Resolve the `?family=` deep link from the home page family cards. // For families with a clean family-level summary we redirect to the // detail page; for aggregator families (no nav target) we seed the // search box so the listing narrows to that family and the user can // expand it. Runs once per `family` param value, after data loads. useEffect(() => { if (!familyParam || !hierarchy) return const fam = hierarchy.families.find((f) => f.key === familyParam) if (!fam) return const navId = getFamilyNavId(fam, benchmarkCards) if (navId) { router.replace(`/evals/${navId.replace(/%2F/g, "/")}`) return } setSearchQuery(fam.display_name || fam.key) }, [familyParam, hierarchy, benchmarkCards, router]) // Reflect groupBy / verified into the URL (nice-to-have deep link). // Shallow replace so the back button isn't spammed and data isn't refetched. useEffect(() => { const params = new URLSearchParams(searchParams.toString()) if (groupBy === "evaluator") params.set("groupBy", "evaluator") else params.delete("groupBy") if (verifiedOnly) params.set("verified", "1") else params.delete("verified") const qs = params.toString() router.replace(qs ? `/evals?${qs}` : "/evals", { scroll: false }) // searchParams intentionally omitted — we only push when our own toggles change. // eslint-disable-next-line react-hooks/exhaustive-deps }, [groupBy, verifiedOnly]) const allEvals = useMemo(() => Array.from(evalItems.values()), [evalItems]) // Verified-eval id universe — drives the Family-mode "Verified only" gate. const verifiedIds = useMemo(() => verifiedEvalIds(allEvals), [allEvals]) // Evaluator groups (group-by-Evaluator mode). Verified filter is // evaluator-aware: counts only (eval, org) pairs where org is verified. const evaluatorGroups = useMemo( () => groupEvalsByEvaluator(allEvals, { verifiedOnly }), [allEvals, verifiedOnly], ) const filteredEvaluators = useMemo(() => { const query = deferredSearchQuery.trim().toLowerCase() let list = evaluatorGroups if (query) list = list.filter((g) => g.name.toLowerCase().includes(query)) const dirMul = evaluatorSortDir === "asc" ? 1 : -1 return list.slice().sort((a, b) => { let cmp = 0 if (evaluatorSortCol === "name") cmp = a.name.localeCompare(b.name) else if (evaluatorSortCol === "verified") cmp = a.verifiedCount - b.verifiedCount else cmp = a.evalCount - b.evalCount if (cmp === 0) cmp = a.name.localeCompare(b.name) return cmp * dirMul }) }, [evaluatorGroups, deferredSearchQuery, evaluatorSortCol, evaluatorSortDir]) const families = hierarchy?.families ?? [] // Tags per family — union of derivedTags across the family and every // nested benchmark/composite. derivedTags is attached at hydration // time by decorateHierarchyDerivedTags (lib/benchmark-tags.ts) so all // the lookup, inheritance, and fallback logic lives in one place. // Drives both the pill selector below and the filter predicate. const familyTags = useMemo(() => { const out = new Map>() for (const fam of families) { const tags = new Set(fam.derivedTags ?? []) for (const b of fam.standalone_benchmarks ?? []) for (const t of b.derivedTags ?? []) tags.add(t) for (const b of fam.benchmarks ?? []) for (const t of b.derivedTags ?? []) tags.add(t) for (const c of fam.composites ?? []) { for (const t of c.derivedTags ?? []) tags.add(t) for (const b of c.benchmarks ?? []) for (const t of b.derivedTags ?? []) tags.add(t) } out.set(fam.key, tags) } return out }, [families]) // Tag → family-count map, sorted by descending count so the most // common tags surface first in the pill bar. Excludes "agentic" — it // lives on its own dedicated toggle since it's an orthogonal axis // (interaction style) rather than a category. const availableTags = useMemo(() => { const counts = new Map() for (const tags of familyTags.values()) { for (const tag of tags) { if (tag.toLowerCase() === "agentic") continue counts.set(tag, (counts.get(tag) ?? 0) + 1) } } return Array.from(counts.entries()) .sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])) .map(([tag]) => tag) }, [familyTags]) const filteredFamilies = useMemo(() => { const query = deferredSearchQuery.trim().toLowerCase() let list = families if (query) { // The query has to also reach nested benchmark keys / display names / // evaluation ids — otherwise typing "mmlu-pro" misses // helm-capabilities/mmlu-pro and friends because they live as leaves // under the MMLU family, whose family-level metadata doesn't contain // the substring. Decoding %2F so percent-encoded ids also match // human-typed slashes. const matchesNestedBenchmark = (fam: HierarchyFamily): boolean => { const benches = [ ...(fam.standalone_benchmarks ?? []), ...(fam.benchmarks ?? []), ...(fam.composites ?? []).flatMap((c) => c.benchmarks ?? []), ] for (const b of benches) { if (b.key && b.key.toLowerCase().includes(query)) return true if (b.display_name && b.display_name.toLowerCase().includes(query)) return true for (const id of b.constituent_evaluation_ids ?? []) { const decoded = decodeURIComponent(id).toLowerCase() if (decoded.includes(query) || id.toLowerCase().includes(query)) return true } } return false } list = list.filter((fam) => { if (fam.display_name.toLowerCase().includes(query)) return true if (fam.key.toLowerCase().includes(query)) return true if (fam.category?.toLowerCase().includes(query)) return true // Also match the curated tag set so the search box and the // category pill bar agree on what is filterable. Without this, // typing "finance" returned zero families even though the // Finance pill catches several. const tags = familyTags.get(fam.key) if (tags) { for (const tag of tags) { if (tag.toLowerCase().includes(query)) return true if (formatTagLabel(tag).toLowerCase().includes(query)) return true } } return matchesNestedBenchmark(fam) }) } if (selectedCategories.length > 0) { const set = new Set(selectedCategories) list = list.filter((fam) => { const tags = familyTags.get(fam.key) if (!tags) return false for (const tag of tags) if (set.has(tag)) return true return false }) } if (agentMode !== "all") { list = list.filter((fam) => { const tags = familyTags.get(fam.key) const hasAgentic = !!tags && Array.from(tags).some((t) => t.toLowerCase() === "agentic") return agentMode === "agentic" ? hasAgentic : !hasAgentic }) } return list.slice().sort((a, b) => { let cmp = 0 switch (sortCol) { case "name": cmp = a.display_name.localeCompare(b.display_name) break case "benchmarks": cmp = getFamilyBenchmarkCount(a) - getFamilyBenchmarkCount(b) break case "results": cmp = getFamilyEvalsCount(a) - getFamilyEvalsCount(b) break } return sortDir === "asc" ? cmp : -cmp }) }, [families, deferredSearchQuery, selectedCategories, agentMode, familyTags, sortCol, sortDir]) useEffect(() => { setVisibleCount(PAGE_SIZE) }, [deferredSearchQuery, selectedCategories, agentMode, sortCol, sortDir, groupBy, verifiedOnly, evaluatorSortCol, evaluatorSortDir]) const visibleFamilies = useMemo( () => filteredFamilies.slice(0, visibleCount), [filteredFamilies, visibleCount], ) const visibleEvaluators = useMemo( () => filteredEvaluators.slice(0, visibleCount), [filteredEvaluators, visibleCount], ) const totalRows = groupBy === "evaluator" ? filteredEvaluators.length : filteredFamilies.length const hasMore = visibleCount < totalRows const handleLoadMore = useCallback(() => { setVisibleCount((current) => Math.min(current + PAGE_SIZE, totalRows)) }, [totalRows]) return (
{/* HEADER --------------------------------------------------- */}
Index

Evaluations

{groupBy === "evaluator" ? ( <> Evaluations grouped by the organisation that reported them. A verified evaluator submitted the results from the org that ran the evaluation. ) : ( <> Evaluations are grouped into families. A family may hold a single standalone benchmark or many related ones; each benchmark has one or more slices, and each slice reports one or more metrics. )}

{/* MODE + FILTER ROW --------------------------------------- */}
setSearchQuery(event.target.value)} placeholder={ groupBy === "evaluator" ? "Search evaluator…" : "Search family, benchmark, or category…" } />
{/* Family-mode-only filters: interaction style + category pills. These operate on the family hierarchy and have no meaning in the evaluator grouping. */} {groupBy === "family" && ( <> {/* INTERACTION STYLE TOGGLE — orthogonal axis from category, surfaced on its own so users don't mix "is this an agent benchmark?" with "what category is this in?" */}
Interaction style
{/* CATEGORY PILLS — multi-select filter by curated benchmark tag (data/benchmarks/categories.json), with the legacy inferCategoryFromBenchmark buckets mixed in for benchmarks not present in the curated file. Agentic is excluded — it has its own dedicated toggle above. */} {availableTags.length > 0 && (
Category {availableTags.map((tag) => { const isSelected = selectedCategories.includes(tag) return ( ) })}
)} )} {/* TABLE ---------------------------------------------------- */} {loading ? ( ) : totalRows === 0 ? (

{groupBy === "evaluator" ? "No evaluators found matching your filters." : "No families found matching your filters."}

) : groupBy === "evaluator" ? ( ) : ( )}
) } export default function EvalsPage() { return ( ) }