"use client" import { useEffect, useMemo, useState } from "react" import Link from "next/link" import { useSearchParams } from "next/navigation" import { Search, X } from "lucide-react" import { useAudienceMode } from "@/components/audience-mode-provider" import { ListPagination } from "@/components/list-pagination" import { Navigation } from "@/components/navigation" import { PageHeader } from "@/components/page-header" import { Input } from "@/components/ui/input" import type { BenchmarkCard, CategoryType } from "@/lib/benchmark-schema" import type { BenchmarkEvalListItem } from "@/lib/eval-processing" import { fetchBenchmarkMetadata, fetchEvalList } from "@/lib/dashboard-data-client" import { getCategoryColor } from "@/lib/benchmark-schema" import { lookupBenchmarkCard, normalizeBenchmarkKey } from "@/lib/benchmark-metadata-utils" import { cn } from "@/lib/utils" const PAGE_SIZE = 40 function shortenLicense(license: string): string { if (!license || license === "Not specified") return "" if (license.toLowerCase().includes("creative commons attribution 4")) return "CC BY 4.0" if (license.toLowerCase().includes("creative commons zero")) return "CC0" if (license.toLowerCase().includes("apache license 2") || license.toLowerCase().includes("apache 2")) return "Apache 2.0" if (license.toLowerCase().includes("mit license")) return "MIT" if (license.toLowerCase().includes("cc-by-sa")) return "CC BY-SA" if (license.length > 24) return `${license.slice(0, 22)}…` return license } const LICENSE_COLORS: Record = { mit: "bg-emerald-100 text-emerald-800 border-emerald-200 dark:bg-emerald-950/40 dark:text-emerald-200", apache: "bg-sky-100 text-sky-800 border-sky-200 dark:bg-sky-950/40 dark:text-sky-200", "cc by": "bg-violet-100 text-violet-800 border-violet-200 dark:bg-violet-950/40 dark:text-violet-200", cc0: "bg-teal-100 text-teal-800 border-teal-200 dark:bg-teal-950/40 dark:text-teal-200", "cc-by-sa": "bg-indigo-100 text-indigo-800 border-indigo-200 dark:bg-indigo-950/40 dark:text-indigo-200", } function licenseBadgeClass(license: string): string { const normalized = license.toLowerCase() for (const [key, className] of Object.entries(LICENSE_COLORS)) { if (normalized.includes(key)) return className } return "bg-muted text-muted-foreground border-border" } function slugifyAggregateId(value: string) { return `aggregate__${value.toLowerCase().replace(/[^a-z0-9]+/g, "_").replace(/^_|_$/g, "")}` } export default function EvalsPage() { const { mode } = useAudienceMode() const searchParams = useSearchParams() const [summaries, setSummaries] = useState([]) const [benchmarkCards, setBenchmarkCards] = useState>({}) const [loading, setLoading] = useState(true) const [totalModels, setTotalModels] = useState(0) const [searchQuery, setSearchQuery] = useState("") const [selectedDomain, setSelectedDomain] = useState(null) const [selectedCategory, setSelectedCategory] = useState(null) const [showWithoutMetadata, setShowWithoutMetadata] = useState(false) const [page, setPage] = useState(1) useEffect(() => { Promise.all([fetchEvalList(), fetchBenchmarkMetadata()]) .then(([data, cards]) => { setSummaries(data.evals) setTotalModels(data.totalModels) setBenchmarkCards(cards) }) .catch(console.error) .finally(() => setLoading(false)) }, []) useEffect(() => { const incomingSearch = searchParams.get("search") ?? "" if (incomingSearch) { setSearchQuery(incomingSearch) } }, [searchParams]) const summariesWithCards = useMemo(() => { return summaries.map((summary) => { if (summary.benchmark_card) { return summary } const fallbackCard = lookupBenchmarkCard(benchmarkCards, summary.evaluation_name) ?? lookupBenchmarkCard(benchmarkCards, summary.composite_benchmark_name) ?? lookupBenchmarkCard(benchmarkCards, summary.composite_benchmark_key) return fallbackCard ? { ...summary, benchmark_card: fallbackCard } : summary }) }, [benchmarkCards, summaries]) const aggregatedSummaries = useMemo(() => { const grouped = new Map() const passthrough: BenchmarkEvalListItem[] = [] for (const summary of summariesWithCards) { const cardName = summary.benchmark_card?.benchmark_details?.name if (!cardName) { passthrough.push(summary) continue } const groupKey = normalizeBenchmarkKey(cardName) const existing = grouped.get(groupKey) ?? [] existing.push(summary) grouped.set(groupKey, existing) } const merged = Array.from(grouped.entries()).map(([groupKey, items]) => { if (items.length === 1) { return items[0] } const first = items[0] const aggregateSources = Array.from( new Map( items.map((item) => [ item.evaluation_id, { evaluation_id: item.evaluation_id, composite_benchmark_key: item.composite_benchmark_key, composite_benchmark_name: item.composite_benchmark_name, models_count: item.models_count, avg_score_norm: item.avg_score_norm, }, ]) ).values() ).sort((a, b) => a.composite_benchmark_name.localeCompare(b.composite_benchmark_name)) const dominantCategory = Object.entries( items.reduce>((counts, item) => { counts[item.category] = (counts[item.category] ?? 0) + 1 return counts }, {}) ).sort((a, b) => b[1] - a[1])[0]?.[0] ?? first.category return { ...first, evaluation_name: first.benchmark_card?.benchmark_details?.name ?? first.evaluation_name, evaluation_id: slugifyAggregateId(groupKey), composite_benchmark_key: aggregateSources.length === 1 ? aggregateSources[0].composite_benchmark_key : "multiple", composite_benchmark_name: aggregateSources.length === 1 ? aggregateSources[0].composite_benchmark_name : `${aggregateSources.length} composite benchmarks`, category: dominantCategory as CategoryType, models_count: Math.max(...items.map((item) => item.models_count)), evaluator_names: Array.from(new Set(items.flatMap((item) => item.evaluator_names))).sort((a, b) => a.localeCompare(b)), source_types: Array.from(new Set(items.flatMap((item) => item.source_types))).sort((a, b) => a.localeCompare(b)), latest_source_name: aggregateSources.length === 1 ? aggregateSources[0].composite_benchmark_name : "Multiple sources", third_party_ratio: items.reduce((sum, item) => sum + item.third_party_ratio, 0) / items.length, missing_generation_config_count: items.reduce( (sum, item) => sum + item.missing_generation_config_count, 0 ), avg_score: items.reduce((sum, item) => sum + item.avg_score_norm, 0) / items.length, avg_score_norm: items.reduce((sum, item) => sum + item.avg_score_norm, 0) / items.length, best_model: null, worst_model: null, is_aggregated: true, aggregate_sources: aggregateSources, } }) return [...merged, ...passthrough] }, [summariesWithCards]) const allDomains = useMemo(() => { const domainSet = new Set() for (const summary of aggregatedSummaries.filter((entry) => entry.benchmark_card)) { for (const domain of summary.benchmark_card?.benchmark_details?.domains ?? []) { domainSet.add(domain) } } return Array.from(domainSet).sort((a, b) => a.localeCompare(b)) }, [aggregatedSummaries]) const allCategories = useMemo(() => { const categorySet = new Set() for (const summary of aggregatedSummaries.filter((entry) => entry.benchmark_card)) { if (summary.category) { categorySet.add(summary.category) } } return Array.from(categorySet).sort((a, b) => a.localeCompare(b)) }, [aggregatedSummaries]) const metadataRichCount = useMemo( () => aggregatedSummaries.filter((summary) => summary.benchmark_card).length, [aggregatedSummaries] ) const metadataPoorCount = aggregatedSummaries.length - metadataRichCount const filtered = useMemo(() => { const query = searchQuery.trim().toLowerCase() let list = showWithoutMetadata ? [...aggregatedSummaries] : aggregatedSummaries.filter((summary) => summary.benchmark_card) if (query) { list = list.filter((summary) => { const haystacks = [ summary.evaluation_name, summary.composite_benchmark_name, summary.metric_config.evaluation_description, summary.benchmark_card?.benchmark_details?.overview, ...(summary.benchmark_card?.benchmark_details?.domains ?? []), ] return haystacks.some((value) => value?.toLowerCase().includes(query)) }) } if (selectedDomain) { list = list.filter((summary) => (summary.benchmark_card?.benchmark_details?.domains ?? []).some( (domain) => domain.toLowerCase() === selectedDomain.toLowerCase() ) ) } if (selectedCategory) { list = list.filter((summary) => summary.category === selectedCategory) } list.sort((a, b) => a.evaluation_name.localeCompare(b.evaluation_name)) return list }, [aggregatedSummaries, searchQuery, selectedCategory, selectedDomain, showWithoutMetadata]) useEffect(() => { setPage(1) }, [searchQuery, selectedCategory, selectedDomain, showWithoutMetadata]) const pagedSummaries = useMemo( () => filtered.slice((page - 1) * PAGE_SIZE, page * PAGE_SIZE), [filtered, page] ) if (loading) { return (
Loading evaluations...
) } return (
setSearchQuery(event.target.value)} placeholder="Search by name, domain, or overview" className="pl-9" />
{allDomains.length > 0 && (
Domain {allDomains.map((domain) => ( ))} {selectedDomain && ( )}
)} {allCategories.length > 0 && (
Category {allCategories.map((category) => ( ))} {selectedCategory && ( )}
)}
{filtered.length === 0 ? (
No evaluations found.
) : (
{pagedSummaries.map((summary) => { const card = summary.benchmark_card const title = card?.benchmark_details?.name ?? summary.evaluation_name const overview = card?.benchmark_details?.overview ?? summary.metric_config.evaluation_description const domains = card?.benchmark_details?.domains ?? [] const dataType = card?.benchmark_details?.data_type ?? "" const license = card?.ethical_and_legal_considerations?.data_licensing ?? "" const shortLicense = shortenLicense(license) const showCompositeLabel = summary.composite_benchmark_name && summary.composite_benchmark_name.toLowerCase() !== title.toLowerCase() const compositeLabel = summary.is_aggregated ? summary.aggregate_sources?.map((source) => source.composite_benchmark_name).join(", ") : summary.composite_benchmark_name return (
{card ? "Benchmark" : "Benchmark without rich metadata"}
{dataType && ( {dataType} )} {shortLicense && ( {shortLicense} )} {summary.models_count.toLocaleString()} models

{title}

{showCompositeLabel && compositeLabel && (
{compositeLabel}
)} {overview && (

{overview}

)} {domains.length > 0 && (
{domains.slice(0, 5).map((domain) => ( {domain} ))} {domains.length > 5 && ( +{domains.length - 5} )}
)} ) })}
)}
) }