Spaces:
Running
Running
feat: refine model and benchmark exploration
Browse files- app/about/page.tsx +65 -0
- app/api/developer-summary/route.ts +20 -0
- app/api/developers/route.ts +8 -0
- app/developers/[id]/page.tsx +192 -0
- app/developers/page.tsx +5 -0
- app/evals/[id]/page.tsx +3 -3
- app/evals/page.tsx +123 -23
- app/{evaluations → models}/[id]/page.tsx +65 -41
- app/models/page.tsx +183 -42
- app/page.tsx +5 -5
- components/benchmark-evaluation-card.tsx +24 -13
- components/developer-card.tsx +107 -0
- components/eval-card.tsx +5 -2
- components/eval-detail.tsx +13 -4
- components/navigation.tsx +4 -1
- lib/dashboard-data-client.ts +35 -0
- lib/eval-processing.ts +8 -2
- lib/model-data.ts +624 -11
- lib/model-family.ts +15 -0
app/about/page.tsx
CHANGED
|
@@ -150,6 +150,47 @@ export default function AboutPage() {
|
|
| 150 |
/>
|
| 151 |
</section>
|
| 152 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 153 |
<section>
|
| 154 |
<Card className="border-border/70">
|
| 155 |
<CardHeader className="border-b bg-muted/20 pb-4">
|
|
@@ -208,6 +249,30 @@ export default function AboutPage() {
|
|
| 208 |
)
|
| 209 |
}
|
| 210 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 211 |
function SurfaceNote({
|
| 212 |
icon,
|
| 213 |
title,
|
|
|
|
| 150 |
/>
|
| 151 |
</section>
|
| 152 |
|
| 153 |
+
<section>
|
| 154 |
+
<Card className="border-border/70">
|
| 155 |
+
<CardHeader className="border-b bg-muted/20 pb-4">
|
| 156 |
+
<CardTitle className="text-xl">Terminology We Use</CardTitle>
|
| 157 |
+
</CardHeader>
|
| 158 |
+
<CardContent className="space-y-5 pt-6">
|
| 159 |
+
<p className="text-sm leading-7 text-muted-foreground">
|
| 160 |
+
The evaluation community often uses <span className="font-medium text-foreground">benchmark</span>, <span className="font-medium text-foreground">eval</span>, <span className="font-medium text-foreground">metric</span>, and <span className="font-medium text-foreground">task</span> interchangeably. That ambiguity showed up repeatedly in this project, so we use a more operational set of definitions in the interface.
|
| 161 |
+
</p>
|
| 162 |
+
|
| 163 |
+
<div className="grid gap-4 md:grid-cols-3">
|
| 164 |
+
<DefinitionCard
|
| 165 |
+
title="Single benchmark"
|
| 166 |
+
definition="An individual evaluation with a defined dataset and scoring method."
|
| 167 |
+
examples={["GSM8K", "IFEval", "MMLU-Pro"]}
|
| 168 |
+
/>
|
| 169 |
+
<DefinitionCard
|
| 170 |
+
title="Composite benchmark"
|
| 171 |
+
definition="A collection of single benchmarks reported together, often under a unified leaderboard."
|
| 172 |
+
examples={["Open LLM Leaderboard", "HELM Instruct", "HF Open LLM v2"]}
|
| 173 |
+
/>
|
| 174 |
+
<DefinitionCard
|
| 175 |
+
title="Metric"
|
| 176 |
+
definition="Strictly what is measured and how; not a benchmark nested inside a composite."
|
| 177 |
+
examples={["Accuracy", "pass@1", "F1", "binary accuracy"]}
|
| 178 |
+
/>
|
| 179 |
+
</div>
|
| 180 |
+
|
| 181 |
+
<div className="rounded-2xl border bg-muted/10 p-4 text-sm leading-7 text-muted-foreground">
|
| 182 |
+
<div className="font-medium text-foreground">Important example</div>
|
| 183 |
+
<p className="mt-2">
|
| 184 |
+
If Reward Bench lists <span className="font-medium text-foreground">factuality</span> under a “metrics” heading, we treat that as a <span className="font-medium text-foreground">benchmark</span> in this interface. The <span className="font-medium text-foreground">metric</span> is the scoring rule attached to it, such as binary accuracy.
|
| 185 |
+
</p>
|
| 186 |
+
<p className="mt-2">
|
| 187 |
+
This is also why the Evaluations page can now group <span className="font-medium text-foreground">single benchmarks</span> underneath a <span className="font-medium text-foreground">composite benchmark</span> like HF Open LLM v2 instead of conflating the two.
|
| 188 |
+
</p>
|
| 189 |
+
</div>
|
| 190 |
+
</CardContent>
|
| 191 |
+
</Card>
|
| 192 |
+
</section>
|
| 193 |
+
|
| 194 |
<section>
|
| 195 |
<Card className="border-border/70">
|
| 196 |
<CardHeader className="border-b bg-muted/20 pb-4">
|
|
|
|
| 249 |
)
|
| 250 |
}
|
| 251 |
|
| 252 |
+
function DefinitionCard({
|
| 253 |
+
title,
|
| 254 |
+
definition,
|
| 255 |
+
examples,
|
| 256 |
+
}: {
|
| 257 |
+
title: string
|
| 258 |
+
definition: string
|
| 259 |
+
examples: string[]
|
| 260 |
+
}) {
|
| 261 |
+
return (
|
| 262 |
+
<div className="rounded-2xl border bg-muted/10 p-4">
|
| 263 |
+
<div className="text-sm font-semibold">{title}</div>
|
| 264 |
+
<p className="mt-2 text-sm leading-6 text-muted-foreground">{definition}</p>
|
| 265 |
+
<div className="mt-3 flex flex-wrap gap-2">
|
| 266 |
+
{examples.map((example) => (
|
| 267 |
+
<Badge key={example} variant="outline">
|
| 268 |
+
{example}
|
| 269 |
+
</Badge>
|
| 270 |
+
))}
|
| 271 |
+
</div>
|
| 272 |
+
</div>
|
| 273 |
+
)
|
| 274 |
+
}
|
| 275 |
+
|
| 276 |
function SurfaceNote({
|
| 277 |
icon,
|
| 278 |
title,
|
app/api/developer-summary/route.ts
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import { NextResponse } from "next/server"
|
| 2 |
+
|
| 3 |
+
import { getDeveloperSummaryById } from "@/lib/model-data"
|
| 4 |
+
|
| 5 |
+
export async function GET(request: Request) {
|
| 6 |
+
const { searchParams } = new URL(request.url)
|
| 7 |
+
const id = searchParams.get("id")
|
| 8 |
+
|
| 9 |
+
if (!id) {
|
| 10 |
+
return NextResponse.json({ error: "Missing developer id" }, { status: 400 })
|
| 11 |
+
}
|
| 12 |
+
|
| 13 |
+
const summary = await getDeveloperSummaryById(id)
|
| 14 |
+
|
| 15 |
+
if (!summary) {
|
| 16 |
+
return NextResponse.json({ error: "Developer not found" }, { status: 404 })
|
| 17 |
+
}
|
| 18 |
+
|
| 19 |
+
return NextResponse.json(summary)
|
| 20 |
+
}
|
app/api/developers/route.ts
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import { NextResponse } from "next/server"
|
| 2 |
+
|
| 3 |
+
import { getDeveloperList } from "@/lib/model-data"
|
| 4 |
+
|
| 5 |
+
export async function GET() {
|
| 6 |
+
const developers = await getDeveloperList()
|
| 7 |
+
return NextResponse.json(developers)
|
| 8 |
+
}
|
app/developers/[id]/page.tsx
ADDED
|
@@ -0,0 +1,192 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"use client"
|
| 2 |
+
|
| 3 |
+
import { useCallback, useEffect, useMemo, useState } from "react"
|
| 4 |
+
import { useParams, useRouter } from "next/navigation"
|
| 5 |
+
import { ArrowLeft, ArrowUpDown, Search } from "lucide-react"
|
| 6 |
+
|
| 7 |
+
import { BenchmarkEvaluationCard, type BenchmarkEvaluationCardData } from "@/components/benchmark-evaluation-card"
|
| 8 |
+
import { ListPagination } from "@/components/list-pagination"
|
| 9 |
+
import { Navigation } from "@/components/navigation"
|
| 10 |
+
import { PageHeader } from "@/components/page-header"
|
| 11 |
+
import { Button } from "@/components/ui/button"
|
| 12 |
+
import { Input } from "@/components/ui/input"
|
| 13 |
+
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"
|
| 14 |
+
import { fetchDeveloperSummary } from "@/lib/dashboard-data-client"
|
| 15 |
+
|
| 16 |
+
const PAGE_SIZE = 40
|
| 17 |
+
|
| 18 |
+
export default function DeveloperDetailPage() {
|
| 19 |
+
const params = useParams()
|
| 20 |
+
const router = useRouter()
|
| 21 |
+
const [developer, setDeveloper] = useState<string>("")
|
| 22 |
+
const [models, setModels] = useState<BenchmarkEvaluationCardData[]>([])
|
| 23 |
+
const [loading, setLoading] = useState(true)
|
| 24 |
+
const [error, setError] = useState<string | null>(null)
|
| 25 |
+
const [searchQuery, setSearchQuery] = useState("")
|
| 26 |
+
const [sortBy, setSortBy] = useState<"date" | "name" | "benchmarks">("date")
|
| 27 |
+
const [page, setPage] = useState(1)
|
| 28 |
+
|
| 29 |
+
const routeId = params.id as string
|
| 30 |
+
|
| 31 |
+
const handleBack = useCallback(() => {
|
| 32 |
+
router.push("/developers")
|
| 33 |
+
}, [router])
|
| 34 |
+
|
| 35 |
+
useEffect(() => {
|
| 36 |
+
fetchDeveloperSummary(routeId)
|
| 37 |
+
.then((summary) => {
|
| 38 |
+
setDeveloper(summary.developer)
|
| 39 |
+
setModels(summary.models)
|
| 40 |
+
})
|
| 41 |
+
.catch((err) => {
|
| 42 |
+
console.error(err)
|
| 43 |
+
setError("Developer not found")
|
| 44 |
+
})
|
| 45 |
+
.finally(() => setLoading(false))
|
| 46 |
+
}, [routeId])
|
| 47 |
+
|
| 48 |
+
const filteredModels = useMemo(() => {
|
| 49 |
+
const query = searchQuery.trim().toLowerCase()
|
| 50 |
+
const filtered = query
|
| 51 |
+
? models.filter((model) => {
|
| 52 |
+
const haystacks = [
|
| 53 |
+
model.model_name,
|
| 54 |
+
model.canonical_model_name,
|
| 55 |
+
model.developer,
|
| 56 |
+
...model.top_scores.map((score) => score.benchmark),
|
| 57 |
+
]
|
| 58 |
+
|
| 59 |
+
return haystacks.some((value) => value?.toLowerCase().includes(query))
|
| 60 |
+
})
|
| 61 |
+
: [...models]
|
| 62 |
+
|
| 63 |
+
switch (sortBy) {
|
| 64 |
+
case "date":
|
| 65 |
+
filtered.sort(
|
| 66 |
+
(a, b) =>
|
| 67 |
+
new Date(b.latest_timestamp).getTime() -
|
| 68 |
+
new Date(a.latest_timestamp).getTime()
|
| 69 |
+
)
|
| 70 |
+
break
|
| 71 |
+
case "name":
|
| 72 |
+
filtered.sort((a, b) => a.model_name.localeCompare(b.model_name))
|
| 73 |
+
break
|
| 74 |
+
case "benchmarks":
|
| 75 |
+
filtered.sort((a, b) => b.benchmarks_count - a.benchmarks_count)
|
| 76 |
+
break
|
| 77 |
+
}
|
| 78 |
+
|
| 79 |
+
return filtered
|
| 80 |
+
}, [models, searchQuery, sortBy])
|
| 81 |
+
|
| 82 |
+
useEffect(() => {
|
| 83 |
+
setPage(1)
|
| 84 |
+
}, [searchQuery, sortBy])
|
| 85 |
+
|
| 86 |
+
const pagedModels = useMemo(
|
| 87 |
+
() => filteredModels.slice((page - 1) * PAGE_SIZE, page * PAGE_SIZE),
|
| 88 |
+
[filteredModels, page]
|
| 89 |
+
)
|
| 90 |
+
|
| 91 |
+
if (loading) {
|
| 92 |
+
return (
|
| 93 |
+
<div className="min-h-screen bg-background">
|
| 94 |
+
<Navigation />
|
| 95 |
+
<main className="container mx-auto px-4 py-8">
|
| 96 |
+
<div className="flex h-96 items-center justify-center">
|
| 97 |
+
<div className="text-lg text-muted-foreground">Loading developer...</div>
|
| 98 |
+
</div>
|
| 99 |
+
</main>
|
| 100 |
+
</div>
|
| 101 |
+
)
|
| 102 |
+
}
|
| 103 |
+
|
| 104 |
+
if (error) {
|
| 105 |
+
return (
|
| 106 |
+
<div className="min-h-screen bg-background">
|
| 107 |
+
<Navigation />
|
| 108 |
+
<main className="container mx-auto px-4 py-8">
|
| 109 |
+
<div className="flex flex-col items-center justify-center h-96 space-y-4">
|
| 110 |
+
<div className="text-lg text-muted-foreground">{error}</div>
|
| 111 |
+
<Button onClick={handleBack}>
|
| 112 |
+
<ArrowLeft className="mr-2 h-4 w-4" />
|
| 113 |
+
Back to Developers
|
| 114 |
+
</Button>
|
| 115 |
+
</div>
|
| 116 |
+
</main>
|
| 117 |
+
</div>
|
| 118 |
+
)
|
| 119 |
+
}
|
| 120 |
+
|
| 121 |
+
return (
|
| 122 |
+
<div className="min-h-screen bg-background">
|
| 123 |
+
<Navigation />
|
| 124 |
+
<main className="container mx-auto px-4 py-8">
|
| 125 |
+
<PageHeader
|
| 126 |
+
eyebrow="Developer"
|
| 127 |
+
title={developer}
|
| 128 |
+
description="Model cards loaded from this developer’s index and detail files, without scanning the entire corpus."
|
| 129 |
+
metaItems={[
|
| 130 |
+
{ label: "Models", value: models.length.toString() },
|
| 131 |
+
{
|
| 132 |
+
label: "Reported Results",
|
| 133 |
+
value: models.reduce((sum, model) => sum + model.evaluations_count, 0).toString(),
|
| 134 |
+
},
|
| 135 |
+
]}
|
| 136 |
+
>
|
| 137 |
+
<Button variant="outline" onClick={handleBack}>
|
| 138 |
+
<ArrowLeft className="mr-2 h-4 w-4" />
|
| 139 |
+
Back
|
| 140 |
+
</Button>
|
| 141 |
+
</PageHeader>
|
| 142 |
+
|
| 143 |
+
<div className="mb-8 mt-8 flex flex-col gap-4 border-b border-border/50 pb-6 sm:flex-row">
|
| 144 |
+
<div className="relative w-full sm:max-w-sm">
|
| 145 |
+
<Search className="pointer-events-none absolute left-3 top-1/2 h-4 w-4 -translate-y-1/2 text-muted-foreground" />
|
| 146 |
+
<Input
|
| 147 |
+
value={searchQuery}
|
| 148 |
+
onChange={(event) => setSearchQuery(event.target.value)}
|
| 149 |
+
placeholder="Search models or benchmarks"
|
| 150 |
+
className="pl-9"
|
| 151 |
+
/>
|
| 152 |
+
</div>
|
| 153 |
+
<Select value={sortBy} onValueChange={(value) => setSortBy(value as typeof sortBy)}>
|
| 154 |
+
<SelectTrigger className="w-[180px]">
|
| 155 |
+
<ArrowUpDown className="mr-2 h-4 w-4" />
|
| 156 |
+
<SelectValue placeholder="Sort by" />
|
| 157 |
+
</SelectTrigger>
|
| 158 |
+
<SelectContent>
|
| 159 |
+
<SelectItem value="date">Latest First</SelectItem>
|
| 160 |
+
<SelectItem value="name">Name (A-Z)</SelectItem>
|
| 161 |
+
<SelectItem value="benchmarks">Most Benchmark Coverage</SelectItem>
|
| 162 |
+
</SelectContent>
|
| 163 |
+
</Select>
|
| 164 |
+
</div>
|
| 165 |
+
|
| 166 |
+
{filteredModels.length === 0 ? (
|
| 167 |
+
<div className="py-12 text-center text-lg text-muted-foreground">
|
| 168 |
+
No models found matching your filters
|
| 169 |
+
</div>
|
| 170 |
+
) : (
|
| 171 |
+
<div className="grid grid-cols-1 gap-6 md:grid-cols-2 lg:grid-cols-2">
|
| 172 |
+
{pagedModels.map((model, index) => (
|
| 173 |
+
<BenchmarkEvaluationCard
|
| 174 |
+
key={model.id}
|
| 175 |
+
data={model}
|
| 176 |
+
delayMs={Math.min(index * 45, 240)}
|
| 177 |
+
/>
|
| 178 |
+
))}
|
| 179 |
+
</div>
|
| 180 |
+
)}
|
| 181 |
+
|
| 182 |
+
<ListPagination
|
| 183 |
+
page={page}
|
| 184 |
+
pageSize={PAGE_SIZE}
|
| 185 |
+
totalItems={filteredModels.length}
|
| 186 |
+
itemLabel="models"
|
| 187 |
+
onPageChange={setPage}
|
| 188 |
+
/>
|
| 189 |
+
</main>
|
| 190 |
+
</div>
|
| 191 |
+
)
|
| 192 |
+
}
|
app/developers/page.tsx
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import { redirect } from "next/navigation"
|
| 2 |
+
|
| 3 |
+
export default function DevelopersRedirectPage() {
|
| 4 |
+
redirect("/models?group=developer")
|
| 5 |
+
}
|
app/evals/[id]/page.tsx
CHANGED
|
@@ -42,7 +42,7 @@ export default function EvalDetailPage() {
|
|
| 42 |
const evalId = decodeURIComponent(params.id as string)
|
| 43 |
const found = await fetchEvalSummary(evalId)
|
| 44 |
setSummary(found)
|
| 45 |
-
document.title = `${found.evaluation_name}
|
| 46 |
} catch (err) {
|
| 47 |
console.error(err)
|
| 48 |
setError("Evaluation not found")
|
|
@@ -94,7 +94,7 @@ export default function EvalDetailPage() {
|
|
| 94 |
<ArrowLeft className="h-4 w-4" />
|
| 95 |
</Button>
|
| 96 |
<div className="flex-1 text-center">
|
| 97 |
-
<h2 className="text-base font-medium tracking-tight text-foreground/90 sm:text-lg">
|
| 98 |
</div>
|
| 99 |
</div>
|
| 100 |
{/* Desktop */}
|
|
@@ -105,7 +105,7 @@ export default function EvalDetailPage() {
|
|
| 105 |
</Button>
|
| 106 |
<div className="text-center">
|
| 107 |
<h2 className="text-xl font-medium tracking-tight text-foreground/90 md:text-2xl">
|
| 108 |
-
|
| 109 |
</h2>
|
| 110 |
</div>
|
| 111 |
<div />
|
|
|
|
| 42 |
const evalId = decodeURIComponent(params.id as string)
|
| 43 |
const found = await fetchEvalSummary(evalId)
|
| 44 |
setSummary(found)
|
| 45 |
+
document.title = `${found.evaluation_name} | Single Benchmark`
|
| 46 |
} catch (err) {
|
| 47 |
console.error(err)
|
| 48 |
setError("Evaluation not found")
|
|
|
|
| 94 |
<ArrowLeft className="h-4 w-4" />
|
| 95 |
</Button>
|
| 96 |
<div className="flex-1 text-center">
|
| 97 |
+
<h2 className="text-base font-medium tracking-tight text-foreground/90 sm:text-lg">Single benchmark details</h2>
|
| 98 |
</div>
|
| 99 |
</div>
|
| 100 |
{/* Desktop */}
|
|
|
|
| 105 |
</Button>
|
| 106 |
<div className="text-center">
|
| 107 |
<h2 className="text-xl font-medium tracking-tight text-foreground/90 md:text-2xl">
|
| 108 |
+
Single benchmark details
|
| 109 |
</h2>
|
| 110 |
</div>
|
| 111 |
<div />
|
app/evals/page.tsx
CHANGED
|
@@ -20,6 +20,7 @@ export default function EvalsPage() {
|
|
| 20 |
const [loading, setLoading] = useState(true)
|
| 21 |
const [totalModels, setTotalModels] = useState(0)
|
| 22 |
const [sortBy, setSortBy] = useState<"name" | "models" | "score">("name")
|
|
|
|
| 23 |
const [searchQuery, setSearchQuery] = useState("")
|
| 24 |
const [page, setPage] = useState(1)
|
| 25 |
|
|
@@ -40,6 +41,7 @@ export default function EvalsPage() {
|
|
| 40 |
if (query) {
|
| 41 |
list = list.filter((summary) => {
|
| 42 |
const haystacks = [
|
|
|
|
| 43 |
summary.evaluation_name,
|
| 44 |
summary.metric_config.evaluation_description,
|
| 45 |
summary.latest_source_name,
|
|
@@ -67,15 +69,43 @@ export default function EvalsPage() {
|
|
| 67 |
return list
|
| 68 |
}, [searchQuery, summaries, sortBy])
|
| 69 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 70 |
useEffect(() => {
|
| 71 |
setPage(1)
|
| 72 |
-
}, [sortBy, searchQuery])
|
| 73 |
|
| 74 |
const pagedSummaries = useMemo(
|
| 75 |
() => filtered.slice((page - 1) * PAGE_SIZE, page * PAGE_SIZE),
|
| 76 |
[filtered, page]
|
| 77 |
)
|
| 78 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 79 |
if (loading) {
|
| 80 |
return (
|
| 81 |
<div className="min-h-screen bg-background">
|
|
@@ -97,50 +127,120 @@ export default function EvalsPage() {
|
|
| 97 |
title="Explore Evaluations"
|
| 98 |
description={
|
| 99 |
mode === "research"
|
| 100 |
-
? "Compare benchmark behavior, methodological framing, and performance spread."
|
| 101 |
-
: "Review evaluation reporting with emphasis on coverage, evidence, and accountable documentation."
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 102 |
}
|
| 103 |
-
metaItems={[
|
| 104 |
-
{ label: "Benchmarks", value: summaries.length.toString() },
|
| 105 |
-
{ label: "Models", value: totalModels.toString() },
|
| 106 |
-
{ label: "View", value: mode === "research" ? "Research" : "Policy" },
|
| 107 |
-
]}
|
| 108 |
/>
|
| 109 |
<main className="container mx-auto px-4 py-8">
|
| 110 |
-
<div className="mb-8 flex flex-col gap-3 border-b border-border/50 pb-6 sm:flex-row">
|
| 111 |
<div className="relative w-full sm:max-w-sm">
|
| 112 |
<Search className="pointer-events-none absolute left-3 top-1/2 h-4 w-4 -translate-y-1/2 text-muted-foreground" />
|
| 113 |
<Input
|
| 114 |
value={searchQuery}
|
| 115 |
onChange={(event) => setSearchQuery(event.target.value)}
|
| 116 |
-
placeholder=
|
|
|
|
|
|
|
|
|
|
|
|
|
| 117 |
className="pl-9"
|
| 118 |
/>
|
| 119 |
</div>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 120 |
<Select value={sortBy} onValueChange={v => setSortBy(v as any)}>
|
| 121 |
<SelectTrigger className="w-[200px]">
|
| 122 |
<ArrowUpDown className="h-4 w-4 mr-2" />
|
| 123 |
<SelectValue placeholder="Sort" />
|
| 124 |
</SelectTrigger>
|
| 125 |
<SelectContent>
|
| 126 |
-
<SelectItem value="name">
|
| 127 |
<SelectItem value="models">Most Models</SelectItem>
|
| 128 |
<SelectItem value="score">Highest Avg Score</SelectItem>
|
| 129 |
</SelectContent>
|
| 130 |
</Select>
|
| 131 |
</div>
|
| 132 |
|
| 133 |
-
|
| 134 |
-
|
| 135 |
-
|
| 136 |
-
|
| 137 |
-
|
| 138 |
-
|
| 139 |
-
|
| 140 |
-
|
| 141 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 142 |
|
| 143 |
-
{filtered.length === 0 && (
|
| 144 |
<div className="text-center py-12 text-muted-foreground">
|
| 145 |
No evaluations found.
|
| 146 |
</div>
|
|
@@ -149,8 +249,8 @@ export default function EvalsPage() {
|
|
| 149 |
<ListPagination
|
| 150 |
page={page}
|
| 151 |
pageSize={PAGE_SIZE}
|
| 152 |
-
totalItems={filtered.length}
|
| 153 |
-
itemLabel="
|
| 154 |
onPageChange={setPage}
|
| 155 |
/>
|
| 156 |
</main>
|
|
|
|
| 20 |
const [loading, setLoading] = useState(true)
|
| 21 |
const [totalModels, setTotalModels] = useState(0)
|
| 22 |
const [sortBy, setSortBy] = useState<"name" | "models" | "score">("name")
|
| 23 |
+
const [groupByComposite, setGroupByComposite] = useState(true)
|
| 24 |
const [searchQuery, setSearchQuery] = useState("")
|
| 25 |
const [page, setPage] = useState(1)
|
| 26 |
|
|
|
|
| 41 |
if (query) {
|
| 42 |
list = list.filter((summary) => {
|
| 43 |
const haystacks = [
|
| 44 |
+
summary.composite_benchmark_name,
|
| 45 |
summary.evaluation_name,
|
| 46 |
summary.metric_config.evaluation_description,
|
| 47 |
summary.latest_source_name,
|
|
|
|
| 69 |
return list
|
| 70 |
}, [searchQuery, summaries, sortBy])
|
| 71 |
|
| 72 |
+
const groupedSummaries = useMemo(() => {
|
| 73 |
+
const groups = new Map<
|
| 74 |
+
string,
|
| 75 |
+
{
|
| 76 |
+
key: string
|
| 77 |
+
name: string
|
| 78 |
+
items: BenchmarkEvalListItem[]
|
| 79 |
+
}
|
| 80 |
+
>()
|
| 81 |
+
|
| 82 |
+
for (const summary of filtered) {
|
| 83 |
+
const existing = groups.get(summary.composite_benchmark_key) ?? {
|
| 84 |
+
key: summary.composite_benchmark_key,
|
| 85 |
+
name: summary.composite_benchmark_name,
|
| 86 |
+
items: [],
|
| 87 |
+
}
|
| 88 |
+
existing.items.push(summary)
|
| 89 |
+
groups.set(summary.composite_benchmark_key, existing)
|
| 90 |
+
}
|
| 91 |
+
|
| 92 |
+
return Array.from(groups.values()).sort((a, b) => a.name.localeCompare(b.name))
|
| 93 |
+
}, [filtered])
|
| 94 |
+
|
| 95 |
useEffect(() => {
|
| 96 |
setPage(1)
|
| 97 |
+
}, [groupByComposite, sortBy, searchQuery])
|
| 98 |
|
| 99 |
const pagedSummaries = useMemo(
|
| 100 |
() => filtered.slice((page - 1) * PAGE_SIZE, page * PAGE_SIZE),
|
| 101 |
[filtered, page]
|
| 102 |
)
|
| 103 |
|
| 104 |
+
const pagedGroups = useMemo(
|
| 105 |
+
() => groupedSummaries.slice((page - 1) * PAGE_SIZE, page * PAGE_SIZE),
|
| 106 |
+
[groupedSummaries, page]
|
| 107 |
+
)
|
| 108 |
+
|
| 109 |
if (loading) {
|
| 110 |
return (
|
| 111 |
<div className="min-h-screen bg-background">
|
|
|
|
| 127 |
title="Explore Evaluations"
|
| 128 |
description={
|
| 129 |
mode === "research"
|
| 130 |
+
? "Compare benchmark behavior, methodological framing, and performance spread, or group single benchmarks under their composite leaderboards."
|
| 131 |
+
: "Review evaluation reporting with emphasis on coverage, evidence, and accountable documentation, including how single benchmarks roll up into composite leaderboards."
|
| 132 |
+
}
|
| 133 |
+
metaItems={
|
| 134 |
+
groupByComposite
|
| 135 |
+
? [
|
| 136 |
+
{ label: "Composite Benchmarks", value: groupedSummaries.length.toString() },
|
| 137 |
+
{ label: "Single Benchmarks", value: filtered.length.toString() },
|
| 138 |
+
{ label: "Models", value: totalModels.toString() },
|
| 139 |
+
{ label: "View", value: mode === "research" ? "Research" : "Policy" },
|
| 140 |
+
]
|
| 141 |
+
: [
|
| 142 |
+
{ label: "Single Benchmarks", value: summaries.length.toString() },
|
| 143 |
+
{ label: "Models", value: totalModels.toString() },
|
| 144 |
+
{ label: "View", value: mode === "research" ? "Research" : "Policy" },
|
| 145 |
+
]
|
| 146 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 147 |
/>
|
| 148 |
<main className="container mx-auto px-4 py-8">
|
| 149 |
+
<div className="mb-8 flex flex-col gap-3 border-b border-border/50 pb-6 sm:flex-row sm:flex-wrap sm:items-center">
|
| 150 |
<div className="relative w-full sm:max-w-sm">
|
| 151 |
<Search className="pointer-events-none absolute left-3 top-1/2 h-4 w-4 -translate-y-1/2 text-muted-foreground" />
|
| 152 |
<Input
|
| 153 |
value={searchQuery}
|
| 154 |
onChange={(event) => setSearchQuery(event.target.value)}
|
| 155 |
+
placeholder={
|
| 156 |
+
groupByComposite
|
| 157 |
+
? "Search composite benchmarks, single benchmarks, purpose, or source"
|
| 158 |
+
: "Search single benchmarks, purpose, or source"
|
| 159 |
+
}
|
| 160 |
className="pl-9"
|
| 161 |
/>
|
| 162 |
</div>
|
| 163 |
+
<div className="inline-flex w-fit rounded-full border bg-muted/20 p-1">
|
| 164 |
+
<button
|
| 165 |
+
type="button"
|
| 166 |
+
onClick={() => setGroupByComposite(false)}
|
| 167 |
+
className={`inline-flex items-center rounded-full px-3 py-1.5 text-sm font-medium transition-colors ${
|
| 168 |
+
!groupByComposite
|
| 169 |
+
? "bg-background text-foreground shadow-sm"
|
| 170 |
+
: "text-muted-foreground hover:text-foreground"
|
| 171 |
+
}`}
|
| 172 |
+
>
|
| 173 |
+
Flat list
|
| 174 |
+
</button>
|
| 175 |
+
<button
|
| 176 |
+
type="button"
|
| 177 |
+
onClick={() => setGroupByComposite(true)}
|
| 178 |
+
className={`inline-flex items-center rounded-full px-3 py-1.5 text-sm font-medium transition-colors ${
|
| 179 |
+
groupByComposite
|
| 180 |
+
? "bg-background text-foreground shadow-sm"
|
| 181 |
+
: "text-muted-foreground hover:text-foreground"
|
| 182 |
+
}`}
|
| 183 |
+
>
|
| 184 |
+
Group by composite benchmark
|
| 185 |
+
</button>
|
| 186 |
+
</div>
|
| 187 |
<Select value={sortBy} onValueChange={v => setSortBy(v as any)}>
|
| 188 |
<SelectTrigger className="w-[200px]">
|
| 189 |
<ArrowUpDown className="h-4 w-4 mr-2" />
|
| 190 |
<SelectValue placeholder="Sort" />
|
| 191 |
</SelectTrigger>
|
| 192 |
<SelectContent>
|
| 193 |
+
<SelectItem value="name">Single Benchmark (A-Z)</SelectItem>
|
| 194 |
<SelectItem value="models">Most Models</SelectItem>
|
| 195 |
<SelectItem value="score">Highest Avg Score</SelectItem>
|
| 196 |
</SelectContent>
|
| 197 |
</Select>
|
| 198 |
</div>
|
| 199 |
|
| 200 |
+
{groupByComposite ? (
|
| 201 |
+
<div className="space-y-8">
|
| 202 |
+
{pagedGroups.map((group, groupIndex) => (
|
| 203 |
+
<section
|
| 204 |
+
key={group.key}
|
| 205 |
+
className="rounded-[1.5rem] border border-border/70 bg-muted/10 p-5"
|
| 206 |
+
>
|
| 207 |
+
<div className="mb-4 flex flex-col gap-2 sm:flex-row sm:items-end sm:justify-between">
|
| 208 |
+
<div>
|
| 209 |
+
<div className="text-[11px] font-semibold uppercase tracking-[0.22em] text-muted-foreground">
|
| 210 |
+
Composite Benchmark
|
| 211 |
+
</div>
|
| 212 |
+
<h2 className="mt-1 text-xl font-bold tracking-tight">{group.name}</h2>
|
| 213 |
+
<p className="mt-1 text-sm text-muted-foreground">
|
| 214 |
+
{group.items.length} single benchmark{group.items.length !== 1 ? "s" : ""} grouped under this composite benchmark.
|
| 215 |
+
</p>
|
| 216 |
+
</div>
|
| 217 |
+
</div>
|
| 218 |
+
|
| 219 |
+
<div className="grid grid-cols-1 gap-6 md:grid-cols-2">
|
| 220 |
+
{group.items.map((summary, index) => (
|
| 221 |
+
<EvalCard
|
| 222 |
+
key={summary.evaluation_id}
|
| 223 |
+
summary={summary}
|
| 224 |
+
delayMs={Math.min((groupIndex * 2 + index) * 35, 240)}
|
| 225 |
+
/>
|
| 226 |
+
))}
|
| 227 |
+
</div>
|
| 228 |
+
</section>
|
| 229 |
+
))}
|
| 230 |
+
</div>
|
| 231 |
+
) : (
|
| 232 |
+
<div className="grid grid-cols-1 gap-6 md:grid-cols-2">
|
| 233 |
+
{pagedSummaries.map((summary, index) => (
|
| 234 |
+
<EvalCard
|
| 235 |
+
key={summary.evaluation_id}
|
| 236 |
+
summary={summary}
|
| 237 |
+
delayMs={Math.min(index * 45, 240)}
|
| 238 |
+
/>
|
| 239 |
+
))}
|
| 240 |
+
</div>
|
| 241 |
+
)}
|
| 242 |
|
| 243 |
+
{(groupByComposite ? groupedSummaries.length === 0 : filtered.length === 0) && (
|
| 244 |
<div className="text-center py-12 text-muted-foreground">
|
| 245 |
No evaluations found.
|
| 246 |
</div>
|
|
|
|
| 249 |
<ListPagination
|
| 250 |
page={page}
|
| 251 |
pageSize={PAGE_SIZE}
|
| 252 |
+
totalItems={groupByComposite ? groupedSummaries.length : filtered.length}
|
| 253 |
+
itemLabel={groupByComposite ? "composite benchmarks" : "single benchmarks"}
|
| 254 |
onPageChange={setPage}
|
| 255 |
/>
|
| 256 |
</main>
|
app/{evaluations → models}/[id]/page.tsx
RENAMED
|
@@ -10,7 +10,7 @@ import type { ModelEvaluationSummary } from "@/lib/eval-processing"
|
|
| 10 |
import { fetchModelSummary } from "@/lib/dashboard-data-client"
|
| 11 |
import { Tabs, TabsList, TabsTrigger } from "@/components/ui/tabs"
|
| 12 |
|
| 13 |
-
export default function
|
| 14 |
const params = useParams()
|
| 15 |
const router = useRouter()
|
| 16 |
const searchParams = useSearchParams()
|
|
@@ -59,22 +59,69 @@ export default function BenchmarkDetailPage() {
|
|
| 59 |
router.push("/models")
|
| 60 |
}, [router])
|
| 61 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 62 |
useEffect(() => {
|
|
|
|
|
|
|
| 63 |
const loadData = async () => {
|
| 64 |
try {
|
| 65 |
const modelSummary = await fetchModelSummary(routeId)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 66 |
setSummary(modelSummary)
|
| 67 |
-
setSelectedVariantId(
|
| 68 |
} catch (err) {
|
| 69 |
-
|
| 70 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 71 |
} finally {
|
| 72 |
-
|
|
|
|
|
|
|
| 73 |
}
|
| 74 |
}
|
| 75 |
-
|
| 76 |
loadData()
|
| 77 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 78 |
|
| 79 |
useEffect(() => {
|
| 80 |
if (!summary?.variants.length) {
|
|
@@ -85,7 +132,7 @@ export default function BenchmarkDetailPage() {
|
|
| 85 |
if (requestedVariant && requestedVariant.variant_id !== selectedVariantId) {
|
| 86 |
setSelectedVariantId(requestedVariant.variant_id)
|
| 87 |
}
|
| 88 |
-
}, [getVariantFromQuery, searchParams,
|
| 89 |
|
| 90 |
const selectedVariant = useMemo(() => {
|
| 91 |
if (!summary) {
|
|
@@ -115,34 +162,13 @@ export default function BenchmarkDetailPage() {
|
|
| 115 |
document.title = `${titleParts.join(" · ")} - AI Evaluation Dashboard`
|
| 116 |
}, [selectedVariant, summary])
|
| 117 |
|
| 118 |
-
useEffect(() => {
|
| 119 |
-
if (!summary || summary.variants.length <= 1 || !selectedVariant || !routeId) {
|
| 120 |
-
return
|
| 121 |
-
}
|
| 122 |
-
|
| 123 |
-
const nextParams = new URLSearchParams(searchParams.toString())
|
| 124 |
-
const currentVersion = nextParams.get("version")
|
| 125 |
-
const nextVersion = selectedVariant.variant_key
|
| 126 |
-
|
| 127 |
-
if (currentVersion === nextVersion) {
|
| 128 |
-
return
|
| 129 |
-
}
|
| 130 |
-
|
| 131 |
-
nextParams.set("version", nextVersion)
|
| 132 |
-
const nextQuery = nextParams.toString()
|
| 133 |
-
router.replace(
|
| 134 |
-
nextQuery ? `/evaluations/${routeId}?${nextQuery}` : `/evaluations/${routeId}`,
|
| 135 |
-
{ scroll: false }
|
| 136 |
-
)
|
| 137 |
-
}, [routeId, router, searchParams, selectedVariant, summary])
|
| 138 |
-
|
| 139 |
if (loading) {
|
| 140 |
return (
|
| 141 |
<div className="min-h-screen bg-background">
|
| 142 |
<Navigation />
|
| 143 |
<main className="container mx-auto px-4 py-8">
|
| 144 |
<div className="flex items-center justify-center h-96">
|
| 145 |
-
<div className="text-lg text-muted-foreground">Loading
|
| 146 |
</div>
|
| 147 |
</main>
|
| 148 |
</div>
|
|
@@ -155,7 +181,7 @@ export default function BenchmarkDetailPage() {
|
|
| 155 |
<Navigation />
|
| 156 |
<main className="container mx-auto px-4 py-8">
|
| 157 |
<div className="flex flex-col items-center justify-center h-96 space-y-4">
|
| 158 |
-
<div className="text-lg text-muted-foreground">{error || "
|
| 159 |
<Button onClick={handleBack}>
|
| 160 |
<ArrowLeft className="mr-2 h-4 w-4" />
|
| 161 |
Back
|
|
@@ -174,10 +200,9 @@ export default function BenchmarkDetailPage() {
|
|
| 174 |
<Navigation />
|
| 175 |
<div className="border-b bg-muted/30">
|
| 176 |
<div className="container mx-auto px-4 sm:px-6 py-4 sm:py-6">
|
| 177 |
-
{/* Mobile layout - Back button + centered title */}
|
| 178 |
<div className="flex items-center gap-3 sm:hidden">
|
| 179 |
-
<Button
|
| 180 |
-
variant="ghost"
|
| 181 |
size="sm"
|
| 182 |
onClick={handleBack}
|
| 183 |
className="shrink-0"
|
|
@@ -186,15 +211,14 @@ export default function BenchmarkDetailPage() {
|
|
| 186 |
</Button>
|
| 187 |
<div className="flex-1 text-center">
|
| 188 |
<h2 className="text-base font-medium tracking-tight text-foreground/90 sm:text-lg">
|
| 189 |
-
|
| 190 |
</h2>
|
| 191 |
</div>
|
| 192 |
</div>
|
| 193 |
-
|
| 194 |
-
{/* Desktop layout - Grid with back button, centered title, empty space */}
|
| 195 |
<div className="hidden sm:grid sm:grid-cols-[auto_1fr_auto] sm:items-center sm:gap-4">
|
| 196 |
-
<Button
|
| 197 |
-
variant="ghost"
|
| 198 |
onClick={handleBack}
|
| 199 |
>
|
| 200 |
<ArrowLeft className="mr-2 h-4 w-4" />
|
|
@@ -202,7 +226,7 @@ export default function BenchmarkDetailPage() {
|
|
| 202 |
</Button>
|
| 203 |
<div className="text-center">
|
| 204 |
<h2 className="text-xl font-medium tracking-tight text-foreground/90 md:text-2xl">
|
| 205 |
-
|
| 206 |
</h2>
|
| 207 |
</div>
|
| 208 |
<div />
|
|
@@ -215,7 +239,7 @@ export default function BenchmarkDetailPage() {
|
|
| 215 |
</div>
|
| 216 |
<Tabs
|
| 217 |
value={selectedVariant?.variant_id ?? summary.variants[0].variant_id}
|
| 218 |
-
onValueChange={
|
| 219 |
className="gap-0"
|
| 220 |
>
|
| 221 |
<TabsList className="flex w-full flex-wrap gap-2">
|
|
|
|
| 10 |
import { fetchModelSummary } from "@/lib/dashboard-data-client"
|
| 11 |
import { Tabs, TabsList, TabsTrigger } from "@/components/ui/tabs"
|
| 12 |
|
| 13 |
+
export default function ModelDetailPage() {
|
| 14 |
const params = useParams()
|
| 15 |
const router = useRouter()
|
| 16 |
const searchParams = useSearchParams()
|
|
|
|
| 59 |
router.push("/models")
|
| 60 |
}, [router])
|
| 61 |
|
| 62 |
+
const handleVariantChange = useCallback(
|
| 63 |
+
(nextVariantId: string) => {
|
| 64 |
+
setSelectedVariantId(nextVariantId)
|
| 65 |
+
|
| 66 |
+
if (!summary || summary.variants.length <= 1 || !routeId) {
|
| 67 |
+
return
|
| 68 |
+
}
|
| 69 |
+
|
| 70 |
+
const nextVariant = summary.variants.find((variant) => variant.variant_id === nextVariantId)
|
| 71 |
+
if (!nextVariant) {
|
| 72 |
+
return
|
| 73 |
+
}
|
| 74 |
+
|
| 75 |
+
const nextParams = new URLSearchParams(searchParams.toString())
|
| 76 |
+
const currentVersion = nextParams.get("version")
|
| 77 |
+
const nextVersion = nextVariant.variant_key
|
| 78 |
+
|
| 79 |
+
if (currentVersion === nextVersion) {
|
| 80 |
+
return
|
| 81 |
+
}
|
| 82 |
+
|
| 83 |
+
nextParams.set("version", nextVersion)
|
| 84 |
+
const nextQuery = nextParams.toString()
|
| 85 |
+
router.replace(
|
| 86 |
+
nextQuery ? `/models/${routeId}?${nextQuery}` : `/models/${routeId}`,
|
| 87 |
+
{ scroll: false }
|
| 88 |
+
)
|
| 89 |
+
},
|
| 90 |
+
[routeId, router, searchParams, summary]
|
| 91 |
+
)
|
| 92 |
+
|
| 93 |
useEffect(() => {
|
| 94 |
+
let isCancelled = false
|
| 95 |
+
|
| 96 |
const loadData = async () => {
|
| 97 |
try {
|
| 98 |
const modelSummary = await fetchModelSummary(routeId)
|
| 99 |
+
if (isCancelled) {
|
| 100 |
+
return
|
| 101 |
+
}
|
| 102 |
+
|
| 103 |
setSummary(modelSummary)
|
| 104 |
+
setSelectedVariantId((current) => current ?? modelSummary.variants[0]?.variant_id ?? null)
|
| 105 |
} catch (err) {
|
| 106 |
+
if (isCancelled) {
|
| 107 |
+
return
|
| 108 |
+
}
|
| 109 |
+
|
| 110 |
+
console.error("Failed to load model:", err)
|
| 111 |
+
setError("Failed to load model data")
|
| 112 |
} finally {
|
| 113 |
+
if (!isCancelled) {
|
| 114 |
+
setLoading(false)
|
| 115 |
+
}
|
| 116 |
}
|
| 117 |
}
|
| 118 |
+
|
| 119 |
loadData()
|
| 120 |
+
|
| 121 |
+
return () => {
|
| 122 |
+
isCancelled = true
|
| 123 |
+
}
|
| 124 |
+
}, [routeId])
|
| 125 |
|
| 126 |
useEffect(() => {
|
| 127 |
if (!summary?.variants.length) {
|
|
|
|
| 132 |
if (requestedVariant && requestedVariant.variant_id !== selectedVariantId) {
|
| 133 |
setSelectedVariantId(requestedVariant.variant_id)
|
| 134 |
}
|
| 135 |
+
}, [getVariantFromQuery, searchParams, summary])
|
| 136 |
|
| 137 |
const selectedVariant = useMemo(() => {
|
| 138 |
if (!summary) {
|
|
|
|
| 162 |
document.title = `${titleParts.join(" · ")} - AI Evaluation Dashboard`
|
| 163 |
}, [selectedVariant, summary])
|
| 164 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 165 |
if (loading) {
|
| 166 |
return (
|
| 167 |
<div className="min-h-screen bg-background">
|
| 168 |
<Navigation />
|
| 169 |
<main className="container mx-auto px-4 py-8">
|
| 170 |
<div className="flex items-center justify-center h-96">
|
| 171 |
+
<div className="text-lg text-muted-foreground">Loading model details...</div>
|
| 172 |
</div>
|
| 173 |
</main>
|
| 174 |
</div>
|
|
|
|
| 181 |
<Navigation />
|
| 182 |
<main className="container mx-auto px-4 py-8">
|
| 183 |
<div className="flex flex-col items-center justify-center h-96 space-y-4">
|
| 184 |
+
<div className="text-lg text-muted-foreground">{error || "Model not found"}</div>
|
| 185 |
<Button onClick={handleBack}>
|
| 186 |
<ArrowLeft className="mr-2 h-4 w-4" />
|
| 187 |
Back
|
|
|
|
| 200 |
<Navigation />
|
| 201 |
<div className="border-b bg-muted/30">
|
| 202 |
<div className="container mx-auto px-4 sm:px-6 py-4 sm:py-6">
|
|
|
|
| 203 |
<div className="flex items-center gap-3 sm:hidden">
|
| 204 |
+
<Button
|
| 205 |
+
variant="ghost"
|
| 206 |
size="sm"
|
| 207 |
onClick={handleBack}
|
| 208 |
className="shrink-0"
|
|
|
|
| 211 |
</Button>
|
| 212 |
<div className="flex-1 text-center">
|
| 213 |
<h2 className="text-base font-medium tracking-tight text-foreground/90 sm:text-lg">
|
| 214 |
+
Model details
|
| 215 |
</h2>
|
| 216 |
</div>
|
| 217 |
</div>
|
| 218 |
+
|
|
|
|
| 219 |
<div className="hidden sm:grid sm:grid-cols-[auto_1fr_auto] sm:items-center sm:gap-4">
|
| 220 |
+
<Button
|
| 221 |
+
variant="ghost"
|
| 222 |
onClick={handleBack}
|
| 223 |
>
|
| 224 |
<ArrowLeft className="mr-2 h-4 w-4" />
|
|
|
|
| 226 |
</Button>
|
| 227 |
<div className="text-center">
|
| 228 |
<h2 className="text-xl font-medium tracking-tight text-foreground/90 md:text-2xl">
|
| 229 |
+
Model details
|
| 230 |
</h2>
|
| 231 |
</div>
|
| 232 |
<div />
|
|
|
|
| 239 |
</div>
|
| 240 |
<Tabs
|
| 241 |
value={selectedVariant?.variant_id ?? summary.variants[0].variant_id}
|
| 242 |
+
onValueChange={handleVariantChange}
|
| 243 |
className="gap-0"
|
| 244 |
>
|
| 245 |
<TabsList className="flex w-full flex-wrap gap-2">
|
app/models/page.tsx
CHANGED
|
@@ -7,34 +7,49 @@ import { Input } from "@/components/ui/input"
|
|
| 7 |
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"
|
| 8 |
import { ArrowUpDown, Search } from "lucide-react"
|
| 9 |
import { BenchmarkEvaluationCard, type BenchmarkEvaluationCardData } from "@/components/benchmark-evaluation-card"
|
|
|
|
| 10 |
import { ListPagination } from "@/components/list-pagination"
|
| 11 |
import { Navigation } from "@/components/navigation"
|
| 12 |
import { PageHeader } from "@/components/page-header"
|
| 13 |
-
import { fetchModelCards } from "@/lib/dashboard-data-client"
|
| 14 |
|
| 15 |
const PAGE_SIZE = 40
|
| 16 |
|
| 17 |
export default function ModelsPage() {
|
| 18 |
const { mode } = useAudienceMode()
|
| 19 |
const [evaluations, setEvaluations] = useState<BenchmarkEvaluationCardData[]>([])
|
| 20 |
-
const [
|
| 21 |
-
const [
|
|
|
|
|
|
|
|
|
|
|
|
|
| 22 |
const [searchQuery, setSearchQuery] = useState("")
|
| 23 |
const [page, setPage] = useState(1)
|
| 24 |
|
| 25 |
useEffect(() => {
|
| 26 |
-
|
| 27 |
-
|
| 28 |
-
|
| 29 |
-
setEvaluations(data)
|
| 30 |
-
} catch (error) {
|
| 31 |
console.error("Failed to load evaluations:", error)
|
| 32 |
-
}
|
| 33 |
-
|
| 34 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 35 |
}
|
| 36 |
|
| 37 |
-
|
|
|
|
| 38 |
}, [])
|
| 39 |
|
| 40 |
const filteredEvaluations = useMemo(() => {
|
|
@@ -62,7 +77,7 @@ export default function ModelsPage() {
|
|
| 62 |
const sortedEvaluations = useMemo(() => {
|
| 63 |
const sorted = [...filteredEvaluations]
|
| 64 |
|
| 65 |
-
switch (
|
| 66 |
case "date":
|
| 67 |
sorted.sort((a, b) =>
|
| 68 |
new Date(b.latest_timestamp).getTime() - new Date(a.latest_timestamp).getTime()
|
|
@@ -77,28 +92,81 @@ export default function ModelsPage() {
|
|
| 77 |
}
|
| 78 |
|
| 79 |
return sorted
|
| 80 |
-
}, [filteredEvaluations,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 81 |
|
| 82 |
useEffect(() => {
|
| 83 |
setPage(1)
|
| 84 |
-
}, [
|
| 85 |
|
| 86 |
const pagedEvaluations = useMemo(
|
| 87 |
() => sortedEvaluations.slice((page - 1) * PAGE_SIZE, page * PAGE_SIZE),
|
| 88 |
[sortedEvaluations, page]
|
| 89 |
)
|
| 90 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 91 |
const handleDelete = (id: string) => {
|
| 92 |
setEvaluations((prev) => prev.filter((e) => e.id !== id))
|
| 93 |
}
|
| 94 |
|
|
|
|
|
|
|
| 95 |
if (loading) {
|
| 96 |
return (
|
| 97 |
<div className="min-h-screen bg-background">
|
| 98 |
<Navigation />
|
| 99 |
<main className="container mx-auto px-4 py-8">
|
| 100 |
<div className="flex items-center justify-center h-96">
|
| 101 |
-
<div className="text-lg text-muted-foreground">Loading
|
| 102 |
</div>
|
| 103 |
</main>
|
| 104 |
</div>
|
|
@@ -111,50 +179,115 @@ export default function ModelsPage() {
|
|
| 111 |
<main className="container mx-auto px-4 py-8">
|
| 112 |
<PageHeader
|
| 113 |
eyebrow="Models"
|
| 114 |
-
title="AI Model Evaluations"
|
| 115 |
description={
|
| 116 |
-
|
| 117 |
-
? "
|
| 118 |
-
:
|
|
|
|
|
|
|
| 119 |
}
|
| 120 |
metaItems={[
|
| 121 |
-
|
| 122 |
-
|
| 123 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 124 |
]}
|
| 125 |
/>
|
| 126 |
|
| 127 |
-
<div className="mb-8 flex flex-col gap-4 border-b border-border/50 pb-6 sm:flex-row">
|
| 128 |
<div className="relative w-full sm:max-w-sm">
|
| 129 |
<Search className="pointer-events-none absolute left-3 top-1/2 h-4 w-4 -translate-y-1/2 text-muted-foreground" />
|
| 130 |
<Input
|
| 131 |
value={searchQuery}
|
| 132 |
onChange={(event) => setSearchQuery(event.target.value)}
|
| 133 |
-
placeholder=
|
|
|
|
|
|
|
|
|
|
|
|
|
| 134 |
className="pl-9"
|
| 135 |
/>
|
| 136 |
</div>
|
| 137 |
-
<
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 138 |
<SelectTrigger className="w-[180px]">
|
| 139 |
<ArrowUpDown className="mr-2 h-4 w-4" />
|
| 140 |
<SelectValue placeholder="Sort by" />
|
| 141 |
</SelectTrigger>
|
| 142 |
<SelectContent>
|
| 143 |
-
|
| 144 |
-
|
| 145 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 146 |
</SelectContent>
|
| 147 |
</Select>
|
| 148 |
</div>
|
| 149 |
|
| 150 |
-
{sortedEvaluations.length === 0 ? (
|
| 151 |
<div className="py-12 text-center">
|
| 152 |
<p className="mb-4 text-lg text-muted-foreground">
|
| 153 |
-
|
|
|
|
|
|
|
| 154 |
</p>
|
| 155 |
<Button
|
| 156 |
onClick={() => {
|
| 157 |
-
|
|
|
|
| 158 |
setSearchQuery("")
|
| 159 |
}}
|
| 160 |
>
|
|
@@ -163,22 +296,30 @@ export default function ModelsPage() {
|
|
| 163 |
</div>
|
| 164 |
) : (
|
| 165 |
<div className="grid grid-cols-1 gap-6 md:grid-cols-2 lg:grid-cols-2">
|
| 166 |
-
{
|
| 167 |
-
|
| 168 |
-
|
| 169 |
-
|
| 170 |
-
|
| 171 |
-
|
| 172 |
-
|
| 173 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 174 |
</div>
|
| 175 |
)}
|
| 176 |
|
| 177 |
<ListPagination
|
| 178 |
page={page}
|
| 179 |
pageSize={PAGE_SIZE}
|
| 180 |
-
totalItems={sortedEvaluations.length}
|
| 181 |
-
itemLabel="models"
|
| 182 |
onPageChange={setPage}
|
| 183 |
/>
|
| 184 |
</main>
|
|
|
|
| 7 |
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"
|
| 8 |
import { ArrowUpDown, Search } from "lucide-react"
|
| 9 |
import { BenchmarkEvaluationCard, type BenchmarkEvaluationCardData } from "@/components/benchmark-evaluation-card"
|
| 10 |
+
import { DeveloperCard } from "@/components/developer-card"
|
| 11 |
import { ListPagination } from "@/components/list-pagination"
|
| 12 |
import { Navigation } from "@/components/navigation"
|
| 13 |
import { PageHeader } from "@/components/page-header"
|
| 14 |
+
import { fetchDevelopers, fetchModelCards, type DeveloperListItem } from "@/lib/dashboard-data-client"
|
| 15 |
|
| 16 |
const PAGE_SIZE = 40
|
| 17 |
|
| 18 |
export default function ModelsPage() {
|
| 19 |
const { mode } = useAudienceMode()
|
| 20 |
const [evaluations, setEvaluations] = useState<BenchmarkEvaluationCardData[]>([])
|
| 21 |
+
const [developers, setDevelopers] = useState<DeveloperListItem[]>([])
|
| 22 |
+
const [loadingModels, setLoadingModels] = useState(true)
|
| 23 |
+
const [loadingDevelopers, setLoadingDevelopers] = useState(true)
|
| 24 |
+
const [groupByDeveloper, setGroupByDeveloper] = useState(false)
|
| 25 |
+
const [modelSortBy, setModelSortBy] = useState<"date" | "name" | "benchmarks">("date")
|
| 26 |
+
const [developerSortBy, setDeveloperSortBy] = useState<"coverage" | "evaluated" | "models" | "name">("coverage")
|
| 27 |
const [searchQuery, setSearchQuery] = useState("")
|
| 28 |
const [page, setPage] = useState(1)
|
| 29 |
|
| 30 |
useEffect(() => {
|
| 31 |
+
fetchModelCards()
|
| 32 |
+
.then(setEvaluations)
|
| 33 |
+
.catch((error) => {
|
|
|
|
|
|
|
| 34 |
console.error("Failed to load evaluations:", error)
|
| 35 |
+
})
|
| 36 |
+
.finally(() => setLoadingModels(false))
|
| 37 |
+
|
| 38 |
+
fetchDevelopers()
|
| 39 |
+
.then(setDevelopers)
|
| 40 |
+
.catch((error) => {
|
| 41 |
+
console.error("Failed to load developers:", error)
|
| 42 |
+
})
|
| 43 |
+
.finally(() => setLoadingDevelopers(false))
|
| 44 |
+
}, [])
|
| 45 |
+
|
| 46 |
+
useEffect(() => {
|
| 47 |
+
if (typeof window === "undefined") {
|
| 48 |
+
return
|
| 49 |
}
|
| 50 |
|
| 51 |
+
const params = new URLSearchParams(window.location.search)
|
| 52 |
+
setGroupByDeveloper(params.get("group") === "developer")
|
| 53 |
}, [])
|
| 54 |
|
| 55 |
const filteredEvaluations = useMemo(() => {
|
|
|
|
| 77 |
const sortedEvaluations = useMemo(() => {
|
| 78 |
const sorted = [...filteredEvaluations]
|
| 79 |
|
| 80 |
+
switch (modelSortBy) {
|
| 81 |
case "date":
|
| 82 |
sorted.sort((a, b) =>
|
| 83 |
new Date(b.latest_timestamp).getTime() - new Date(a.latest_timestamp).getTime()
|
|
|
|
| 92 |
}
|
| 93 |
|
| 94 |
return sorted
|
| 95 |
+
}, [filteredEvaluations, modelSortBy])
|
| 96 |
+
|
| 97 |
+
const filteredDevelopers = useMemo(() => {
|
| 98 |
+
const query = searchQuery.trim().toLowerCase()
|
| 99 |
+
const filtered = query
|
| 100 |
+
? developers.filter((developer) => {
|
| 101 |
+
const haystacks = [
|
| 102 |
+
developer.developer,
|
| 103 |
+
...developer.popular_evals.map((evaluation) => evaluation.benchmark),
|
| 104 |
+
]
|
| 105 |
+
|
| 106 |
+
return haystacks.some((value) => value?.toLowerCase().includes(query))
|
| 107 |
+
})
|
| 108 |
+
: [...developers]
|
| 109 |
+
|
| 110 |
+
filtered.sort((a, b) => {
|
| 111 |
+
switch (developerSortBy) {
|
| 112 |
+
case "coverage":
|
| 113 |
+
if (b.benchmark_count !== a.benchmark_count) {
|
| 114 |
+
return b.benchmark_count - a.benchmark_count
|
| 115 |
+
}
|
| 116 |
+
if (b.evaluation_count !== a.evaluation_count) {
|
| 117 |
+
return b.evaluation_count - a.evaluation_count
|
| 118 |
+
}
|
| 119 |
+
if (b.model_count !== a.model_count) {
|
| 120 |
+
return b.model_count - a.model_count
|
| 121 |
+
}
|
| 122 |
+
break
|
| 123 |
+
case "evaluated":
|
| 124 |
+
if (b.evaluation_count !== a.evaluation_count) {
|
| 125 |
+
return b.evaluation_count - a.evaluation_count
|
| 126 |
+
}
|
| 127 |
+
break
|
| 128 |
+
case "models":
|
| 129 |
+
if (b.model_count !== a.model_count) {
|
| 130 |
+
return b.model_count - a.model_count
|
| 131 |
+
}
|
| 132 |
+
break
|
| 133 |
+
case "name":
|
| 134 |
+
break
|
| 135 |
+
}
|
| 136 |
+
|
| 137 |
+
return a.developer.localeCompare(b.developer)
|
| 138 |
+
})
|
| 139 |
+
|
| 140 |
+
return filtered
|
| 141 |
+
}, [developerSortBy, developers, searchQuery])
|
| 142 |
|
| 143 |
useEffect(() => {
|
| 144 |
setPage(1)
|
| 145 |
+
}, [developerSortBy, groupByDeveloper, modelSortBy, searchQuery])
|
| 146 |
|
| 147 |
const pagedEvaluations = useMemo(
|
| 148 |
() => sortedEvaluations.slice((page - 1) * PAGE_SIZE, page * PAGE_SIZE),
|
| 149 |
[sortedEvaluations, page]
|
| 150 |
)
|
| 151 |
|
| 152 |
+
const pagedDevelopers = useMemo(
|
| 153 |
+
() => filteredDevelopers.slice((page - 1) * PAGE_SIZE, page * PAGE_SIZE),
|
| 154 |
+
[filteredDevelopers, page]
|
| 155 |
+
)
|
| 156 |
+
|
| 157 |
const handleDelete = (id: string) => {
|
| 158 |
setEvaluations((prev) => prev.filter((e) => e.id !== id))
|
| 159 |
}
|
| 160 |
|
| 161 |
+
const loading = loadingModels || loadingDevelopers
|
| 162 |
+
|
| 163 |
if (loading) {
|
| 164 |
return (
|
| 165 |
<div className="min-h-screen bg-background">
|
| 166 |
<Navigation />
|
| 167 |
<main className="container mx-auto px-4 py-8">
|
| 168 |
<div className="flex items-center justify-center h-96">
|
| 169 |
+
<div className="text-lg text-muted-foreground">Loading models...</div>
|
| 170 |
</div>
|
| 171 |
</main>
|
| 172 |
</div>
|
|
|
|
| 179 |
<main className="container mx-auto px-4 py-8">
|
| 180 |
<PageHeader
|
| 181 |
eyebrow="Models"
|
| 182 |
+
title={groupByDeveloper ? "Model Developers" : "AI Model Evaluations"}
|
| 183 |
description={
|
| 184 |
+
groupByDeveloper
|
| 185 |
+
? "Group the model corpus by developer to compare how many models each team ships and which eval suites show up most often."
|
| 186 |
+
: mode === "research"
|
| 187 |
+
? "Browse model cards with benchmark breadth, result density, and technical highlights."
|
| 188 |
+
: "Browse model cards with stronger emphasis on reporting breadth, evidence, and evaluation accountability."
|
| 189 |
}
|
| 190 |
metaItems={[
|
| 191 |
+
groupByDeveloper
|
| 192 |
+
? { label: "Developers", value: filteredDevelopers.length.toString() }
|
| 193 |
+
: { label: "Models", value: sortedEvaluations.length.toString() },
|
| 194 |
+
groupByDeveloper
|
| 195 |
+
? {
|
| 196 |
+
label: "Indexed Models",
|
| 197 |
+
value: filteredDevelopers.reduce((sum, developer) => sum + developer.model_count, 0).toString(),
|
| 198 |
+
}
|
| 199 |
+
: { label: "Reported results", value: sortedEvaluations.reduce((sum, e) => sum + e.evaluations_count, 0).toString() },
|
| 200 |
+
groupByDeveloper
|
| 201 |
+
? {
|
| 202 |
+
label: "Benchmarks",
|
| 203 |
+
value: filteredDevelopers.reduce((sum, developer) => sum + developer.benchmark_count, 0).toString(),
|
| 204 |
+
}
|
| 205 |
+
: { label: "Reporting orgs", value: new Set(sortedEvaluations.flatMap((e) => e.evaluator_names)).size.toString() },
|
| 206 |
]}
|
| 207 |
/>
|
| 208 |
|
| 209 |
+
<div className="mb-8 flex flex-col gap-4 border-b border-border/50 pb-6 sm:flex-row sm:flex-wrap sm:items-center">
|
| 210 |
<div className="relative w-full sm:max-w-sm">
|
| 211 |
<Search className="pointer-events-none absolute left-3 top-1/2 h-4 w-4 -translate-y-1/2 text-muted-foreground" />
|
| 212 |
<Input
|
| 213 |
value={searchQuery}
|
| 214 |
onChange={(event) => setSearchQuery(event.target.value)}
|
| 215 |
+
placeholder={
|
| 216 |
+
groupByDeveloper
|
| 217 |
+
? "Search developers or popular evals"
|
| 218 |
+
: "Search models, developers, or benchmarks"
|
| 219 |
+
}
|
| 220 |
className="pl-9"
|
| 221 |
/>
|
| 222 |
</div>
|
| 223 |
+
<div className="inline-flex w-fit rounded-full border bg-muted/20 p-1">
|
| 224 |
+
<button
|
| 225 |
+
type="button"
|
| 226 |
+
onClick={() => setGroupByDeveloper(false)}
|
| 227 |
+
className={`inline-flex items-center rounded-full px-3 py-1.5 text-sm font-medium transition-colors ${
|
| 228 |
+
!groupByDeveloper
|
| 229 |
+
? "bg-background text-foreground shadow-sm"
|
| 230 |
+
: "text-muted-foreground hover:text-foreground"
|
| 231 |
+
}`}
|
| 232 |
+
>
|
| 233 |
+
Models
|
| 234 |
+
</button>
|
| 235 |
+
<button
|
| 236 |
+
type="button"
|
| 237 |
+
onClick={() => setGroupByDeveloper(true)}
|
| 238 |
+
className={`inline-flex items-center rounded-full px-3 py-1.5 text-sm font-medium transition-colors ${
|
| 239 |
+
groupByDeveloper
|
| 240 |
+
? "bg-background text-foreground shadow-sm"
|
| 241 |
+
: "text-muted-foreground hover:text-foreground"
|
| 242 |
+
}`}
|
| 243 |
+
>
|
| 244 |
+
Group by model developer
|
| 245 |
+
</button>
|
| 246 |
+
</div>
|
| 247 |
+
<Select
|
| 248 |
+
value={groupByDeveloper ? developerSortBy : modelSortBy}
|
| 249 |
+
onValueChange={(value) => {
|
| 250 |
+
if (groupByDeveloper) {
|
| 251 |
+
setDeveloperSortBy(value as typeof developerSortBy)
|
| 252 |
+
} else {
|
| 253 |
+
setModelSortBy(value as typeof modelSortBy)
|
| 254 |
+
}
|
| 255 |
+
}}
|
| 256 |
+
>
|
| 257 |
<SelectTrigger className="w-[180px]">
|
| 258 |
<ArrowUpDown className="mr-2 h-4 w-4" />
|
| 259 |
<SelectValue placeholder="Sort by" />
|
| 260 |
</SelectTrigger>
|
| 261 |
<SelectContent>
|
| 262 |
+
{groupByDeveloper ? (
|
| 263 |
+
<>
|
| 264 |
+
<SelectItem value="coverage">Most Coverage</SelectItem>
|
| 265 |
+
<SelectItem value="evaluated">Most Results</SelectItem>
|
| 266 |
+
<SelectItem value="models">Most Models</SelectItem>
|
| 267 |
+
<SelectItem value="name">Name (A-Z)</SelectItem>
|
| 268 |
+
</>
|
| 269 |
+
) : (
|
| 270 |
+
<>
|
| 271 |
+
<SelectItem value="date">Latest First</SelectItem>
|
| 272 |
+
<SelectItem value="name">Name (A-Z)</SelectItem>
|
| 273 |
+
<SelectItem value="benchmarks">Most Benchmark Coverage</SelectItem>
|
| 274 |
+
</>
|
| 275 |
+
)}
|
| 276 |
</SelectContent>
|
| 277 |
</Select>
|
| 278 |
</div>
|
| 279 |
|
| 280 |
+
{(groupByDeveloper ? filteredDevelopers.length === 0 : sortedEvaluations.length === 0) ? (
|
| 281 |
<div className="py-12 text-center">
|
| 282 |
<p className="mb-4 text-lg text-muted-foreground">
|
| 283 |
+
{groupByDeveloper
|
| 284 |
+
? "No developers found matching your filters"
|
| 285 |
+
: "No evaluations found matching your filters"}
|
| 286 |
</p>
|
| 287 |
<Button
|
| 288 |
onClick={() => {
|
| 289 |
+
setModelSortBy("date")
|
| 290 |
+
setDeveloperSortBy("coverage")
|
| 291 |
setSearchQuery("")
|
| 292 |
}}
|
| 293 |
>
|
|
|
|
| 296 |
</div>
|
| 297 |
) : (
|
| 298 |
<div className="grid grid-cols-1 gap-6 md:grid-cols-2 lg:grid-cols-2">
|
| 299 |
+
{groupByDeveloper
|
| 300 |
+
? pagedDevelopers.map((developer, index) => (
|
| 301 |
+
<DeveloperCard
|
| 302 |
+
key={developer.route_id}
|
| 303 |
+
developer={developer}
|
| 304 |
+
delayMs={Math.min(index * 45, 240)}
|
| 305 |
+
/>
|
| 306 |
+
))
|
| 307 |
+
: pagedEvaluations.map((evaluation, index) => (
|
| 308 |
+
<BenchmarkEvaluationCard
|
| 309 |
+
key={evaluation.id}
|
| 310 |
+
data={evaluation}
|
| 311 |
+
onDelete={handleDelete}
|
| 312 |
+
delayMs={Math.min(index * 45, 240)}
|
| 313 |
+
/>
|
| 314 |
+
))}
|
| 315 |
</div>
|
| 316 |
)}
|
| 317 |
|
| 318 |
<ListPagination
|
| 319 |
page={page}
|
| 320 |
pageSize={PAGE_SIZE}
|
| 321 |
+
totalItems={groupByDeveloper ? filteredDevelopers.length : sortedEvaluations.length}
|
| 322 |
+
itemLabel={groupByDeveloper ? "developers" : "models"}
|
| 323 |
onPageChange={setPage}
|
| 324 |
/>
|
| 325 |
</main>
|
app/page.tsx
CHANGED
|
@@ -126,7 +126,7 @@ export default function HomePage() {
|
|
| 126 |
<Info className="h-4 w-4 text-amber-600 dark:text-amber-400" />
|
| 127 |
<AlertTitle>Demo Environment</AlertTitle>
|
| 128 |
<AlertDescription>
|
| 129 |
-
This is a
|
| 130 |
</AlertDescription>
|
| 131 |
</Alert>
|
| 132 |
|
|
@@ -249,8 +249,8 @@ export default function HomePage() {
|
|
| 249 |
<section className="grid gap-6 xl:grid-cols-2">
|
| 250 |
<OverviewPanel
|
| 251 |
eyebrow="Accountability"
|
| 252 |
-
title="
|
| 253 |
-
description="Models with the
|
| 254 |
href="/models"
|
| 255 |
cta="Browse models"
|
| 256 |
>
|
|
@@ -260,7 +260,7 @@ export default function HomePage() {
|
|
| 260 |
key={model.id}
|
| 261 |
rank={index + 1}
|
| 262 |
model={model}
|
| 263 |
-
metricLabel="
|
| 264 |
metricValue={`${Math.round(model.independent_verification_ratio * 100)}%`}
|
| 265 |
secondaryLabel={`${model.benchmarks_count} benchmarks`}
|
| 266 |
highlight={model.independent_verification_ratio > 0.5}
|
|
@@ -392,7 +392,7 @@ function ModelOverviewRow({
|
|
| 392 |
highlight?: boolean
|
| 393 |
}) {
|
| 394 |
return (
|
| 395 |
-
<Link href={`/
|
| 396 |
<div className="flex items-center gap-4 rounded-2xl px-3 py-3 transition-colors hover:bg-muted/30">
|
| 397 |
<div className="flex h-8 w-8 shrink-0 items-center justify-center rounded-full border text-xs font-semibold">
|
| 398 |
{rank}
|
|
|
|
| 126 |
<Info className="h-4 w-4 text-amber-600 dark:text-amber-400" />
|
| 127 |
<AlertTitle>Demo Environment</AlertTitle>
|
| 128 |
<AlertDescription>
|
| 129 |
+
This is a research preview with sample data for demonstration purposes.
|
| 130 |
</AlertDescription>
|
| 131 |
</Alert>
|
| 132 |
|
|
|
|
| 249 |
<section className="grid gap-6 xl:grid-cols-2">
|
| 250 |
<OverviewPanel
|
| 251 |
eyebrow="Accountability"
|
| 252 |
+
title="Highest third-party reporting share"
|
| 253 |
+
description="Models with the largest share of third-party benchmark reporting in the current corpus."
|
| 254 |
href="/models"
|
| 255 |
cta="Browse models"
|
| 256 |
>
|
|
|
|
| 260 |
key={model.id}
|
| 261 |
rank={index + 1}
|
| 262 |
model={model}
|
| 263 |
+
metricLabel="Third-party share"
|
| 264 |
metricValue={`${Math.round(model.independent_verification_ratio * 100)}%`}
|
| 265 |
secondaryLabel={`${model.benchmarks_count} benchmarks`}
|
| 266 |
highlight={model.independent_verification_ratio > 0.5}
|
|
|
|
| 392 |
highlight?: boolean
|
| 393 |
}) {
|
| 394 |
return (
|
| 395 |
+
<Link href={`/models/${model.route_id}`} className="block">
|
| 396 |
<div className="flex items-center gap-4 rounded-2xl px-3 py-3 transition-colors hover:bg-muted/30">
|
| 397 |
<div className="flex h-8 w-8 shrink-0 items-center justify-center rounded-full border text-xs font-semibold">
|
| 398 |
{rank}
|
components/benchmark-evaluation-card.tsx
CHANGED
|
@@ -139,6 +139,20 @@ function getReportingSummaryLabel(data: BenchmarkEvaluationCardData) {
|
|
| 139 |
return "Aggregated reporting view"
|
| 140 |
}
|
| 141 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 142 |
export function BenchmarkEvaluationCard({ data, onDelete, delayMs = 0 }: BenchmarkEvaluationCardProps) {
|
| 143 |
const router = useRouter()
|
| 144 |
const { mode } = useAudienceMode()
|
|
@@ -152,7 +166,7 @@ export function BenchmarkEvaluationCard({ data, onDelete, delayMs = 0 }: Benchma
|
|
| 152 |
<Card
|
| 153 |
className="motion-academic-enter motion-academic-surface motion-academic-hover group cursor-pointer overflow-hidden border-border/70 bg-card hover:shadow-xl"
|
| 154 |
style={{ "--enter-delay": `${delayMs}ms` } as CSSProperties}
|
| 155 |
-
onClick={() => router.push(`/
|
| 156 |
>
|
| 157 |
<CardHeader className="space-y-4 border-b border-border/60 pb-4">
|
| 158 |
<div className="flex flex-wrap items-center justify-between gap-2 text-[10px] font-semibold uppercase tracking-[0.24em] text-muted-foreground">
|
|
@@ -184,12 +198,7 @@ export function BenchmarkEvaluationCard({ data, onDelete, delayMs = 0 }: Benchma
|
|
| 184 |
{data.input_modalities && data.input_modalities.length > 1 && (
|
| 185 |
<Badge variant="secondary">Multimodal</Badge>
|
| 186 |
)}
|
| 187 |
-
{data.
|
| 188 |
-
<Badge className="bg-emerald-100 text-emerald-800 dark:bg-emerald-950/50 dark:text-emerald-300">
|
| 189 |
-
<BadgeCheck className="mr-1 h-3 w-3" />
|
| 190 |
-
Independent reporting
|
| 191 |
-
</Badge>
|
| 192 |
-
) : data.evaluator_count > 0 ? (
|
| 193 |
<Badge variant="secondary">{reportingSummaryLabel}</Badge>
|
| 194 |
) : null}
|
| 195 |
</div>
|
|
@@ -207,7 +216,7 @@ export function BenchmarkEvaluationCard({ data, onDelete, delayMs = 0 }: Benchma
|
|
| 207 |
</Button>
|
| 208 |
</DropdownMenuTrigger>
|
| 209 |
<DropdownMenuContent align="end">
|
| 210 |
-
<DropdownMenuItem onClick={() => router.push(`/
|
| 211 |
<Eye className="mr-2 h-4 w-4" />
|
| 212 |
View Details
|
| 213 |
</DropdownMenuItem>
|
|
@@ -241,8 +250,8 @@ export function BenchmarkEvaluationCard({ data, onDelete, delayMs = 0 }: Benchma
|
|
| 241 |
tone="bg-stone-100 text-stone-900 ring-1 ring-stone-200/80 dark:bg-stone-900/40 dark:text-stone-100 dark:ring-stone-800/70"
|
| 242 |
/>
|
| 243 |
<CompactStat
|
| 244 |
-
label=
|
| 245 |
-
value={
|
| 246 |
tone="bg-emerald-50 text-emerald-900 ring-1 ring-emerald-200/70 dark:bg-emerald-950/25 dark:text-emerald-100 dark:ring-emerald-900/50"
|
| 247 |
/>
|
| 248 |
</div>
|
|
@@ -310,9 +319,11 @@ export function BenchmarkEvaluationCard({ data, onDelete, delayMs = 0 }: Benchma
|
|
| 310 |
<div className="text-sm font-semibold">Reporting summary</div>
|
| 311 |
<div className="text-sm text-muted-foreground">
|
| 312 |
This model has reported results from {reportingSummaryLabel.toLowerCase()} across {data.benchmarks_count} benchmark{data.benchmarks_count !== 1 ? "s" : ""}.
|
| 313 |
-
{data.independent_verification_ratio > 0
|
| 314 |
-
? ` ${Math.round(data.independent_verification_ratio * 100)}%
|
| 315 |
-
:
|
|
|
|
|
|
|
| 316 |
</div>
|
| 317 |
<div className="flex flex-wrap gap-2 pt-1">
|
| 318 |
{data.evaluator_names.slice(0, 2).map((evaluator) => (
|
|
|
|
| 139 |
return "Aggregated reporting view"
|
| 140 |
}
|
| 141 |
|
| 142 |
+
function getReportingMixLabel(data: BenchmarkEvaluationCardData) {
|
| 143 |
+
const thirdPartyShare = Math.round(data.independent_verification_ratio * 100)
|
| 144 |
+
|
| 145 |
+
if (thirdPartyShare <= 0) {
|
| 146 |
+
return "Self-reported evidence"
|
| 147 |
+
}
|
| 148 |
+
|
| 149 |
+
if (thirdPartyShare >= 100) {
|
| 150 |
+
return "Third-party reported"
|
| 151 |
+
}
|
| 152 |
+
|
| 153 |
+
return `${thirdPartyShare}% third-party mix`
|
| 154 |
+
}
|
| 155 |
+
|
| 156 |
export function BenchmarkEvaluationCard({ data, onDelete, delayMs = 0 }: BenchmarkEvaluationCardProps) {
|
| 157 |
const router = useRouter()
|
| 158 |
const { mode } = useAudienceMode()
|
|
|
|
| 166 |
<Card
|
| 167 |
className="motion-academic-enter motion-academic-surface motion-academic-hover group cursor-pointer overflow-hidden border-border/70 bg-card hover:shadow-xl"
|
| 168 |
style={{ "--enter-delay": `${delayMs}ms` } as CSSProperties}
|
| 169 |
+
onClick={() => router.push(`/models/${data.route_id}`)}
|
| 170 |
>
|
| 171 |
<CardHeader className="space-y-4 border-b border-border/60 pb-4">
|
| 172 |
<div className="flex flex-wrap items-center justify-between gap-2 text-[10px] font-semibold uppercase tracking-[0.24em] text-muted-foreground">
|
|
|
|
| 198 |
{data.input_modalities && data.input_modalities.length > 1 && (
|
| 199 |
<Badge variant="secondary">Multimodal</Badge>
|
| 200 |
)}
|
| 201 |
+
{data.evaluator_count > 0 ? (
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 202 |
<Badge variant="secondary">{reportingSummaryLabel}</Badge>
|
| 203 |
) : null}
|
| 204 |
</div>
|
|
|
|
| 216 |
</Button>
|
| 217 |
</DropdownMenuTrigger>
|
| 218 |
<DropdownMenuContent align="end">
|
| 219 |
+
<DropdownMenuItem onClick={() => router.push(`/models/${data.route_id}`)}>
|
| 220 |
<Eye className="mr-2 h-4 w-4" />
|
| 221 |
View Details
|
| 222 |
</DropdownMenuItem>
|
|
|
|
| 250 |
tone="bg-stone-100 text-stone-900 ring-1 ring-stone-200/80 dark:bg-stone-900/40 dark:text-stone-100 dark:ring-stone-800/70"
|
| 251 |
/>
|
| 252 |
<CompactStat
|
| 253 |
+
label="Reporting Mix"
|
| 254 |
+
value={getReportingMixLabel(data)}
|
| 255 |
tone="bg-emerald-50 text-emerald-900 ring-1 ring-emerald-200/70 dark:bg-emerald-950/25 dark:text-emerald-100 dark:ring-emerald-900/50"
|
| 256 |
/>
|
| 257 |
</div>
|
|
|
|
| 319 |
<div className="text-sm font-semibold">Reporting summary</div>
|
| 320 |
<div className="text-sm text-muted-foreground">
|
| 321 |
This model has reported results from {reportingSummaryLabel.toLowerCase()} across {data.benchmarks_count} benchmark{data.benchmarks_count !== 1 ? "s" : ""}.
|
| 322 |
+
{data.independent_verification_ratio > 0 && data.independent_verification_ratio < 1
|
| 323 |
+
? ` The current record mixes self-reported and third-party benchmark results, with ${Math.round(data.independent_verification_ratio * 100)}% coming from third-party reporting.`
|
| 324 |
+
: data.independent_verification_ratio >= 1
|
| 325 |
+
? " The current record is fully backed by third-party reporting."
|
| 326 |
+
: " The current record is self-reported."}
|
| 327 |
</div>
|
| 328 |
<div className="flex flex-wrap gap-2 pt-1">
|
| 329 |
{data.evaluator_names.slice(0, 2).map((evaluator) => (
|
components/developer-card.tsx
ADDED
|
@@ -0,0 +1,107 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"use client"
|
| 2 |
+
|
| 3 |
+
import type { ComponentType, CSSProperties } from "react"
|
| 4 |
+
import { useRouter } from "next/navigation"
|
| 5 |
+
import { BarChart3, Boxes, ChevronRight, Sparkles } from "lucide-react"
|
| 6 |
+
|
| 7 |
+
import { Badge } from "@/components/ui/badge"
|
| 8 |
+
import { Card, CardContent, CardHeader } from "@/components/ui/card"
|
| 9 |
+
import type { DeveloperListItem } from "@/lib/dashboard-data-client"
|
| 10 |
+
|
| 11 |
+
interface DeveloperCardProps {
|
| 12 |
+
developer: DeveloperListItem
|
| 13 |
+
delayMs?: number
|
| 14 |
+
}
|
| 15 |
+
|
| 16 |
+
export function DeveloperCard({ developer, delayMs = 0 }: DeveloperCardProps) {
|
| 17 |
+
const router = useRouter()
|
| 18 |
+
|
| 19 |
+
return (
|
| 20 |
+
<Card
|
| 21 |
+
className="motion-academic-enter motion-academic-surface motion-academic-hover group cursor-pointer overflow-hidden border-border/70 bg-card hover:shadow-lg"
|
| 22 |
+
style={{ "--enter-delay": `${delayMs}ms` } as CSSProperties}
|
| 23 |
+
onClick={() => router.push(`/developers/${developer.route_id}`)}
|
| 24 |
+
>
|
| 25 |
+
<CardHeader className="space-y-3 border-b border-border/60 pb-4">
|
| 26 |
+
<div className="text-[10px] font-semibold uppercase tracking-[0.24em] text-muted-foreground">
|
| 27 |
+
Developer Summary
|
| 28 |
+
</div>
|
| 29 |
+
|
| 30 |
+
<div className="flex items-start justify-between gap-3">
|
| 31 |
+
<div className="min-w-0">
|
| 32 |
+
<div className="truncate text-xl font-bold">{developer.developer}</div>
|
| 33 |
+
<div className="mt-1 text-sm text-muted-foreground">
|
| 34 |
+
Aggregated reporting across published model variants and their most common single benchmarks
|
| 35 |
+
</div>
|
| 36 |
+
</div>
|
| 37 |
+
<ChevronRight className="mt-1 h-5 w-5 shrink-0 text-muted-foreground transition-transform group-hover:translate-x-0.5" />
|
| 38 |
+
</div>
|
| 39 |
+
</CardHeader>
|
| 40 |
+
|
| 41 |
+
<CardContent className="space-y-4 pt-4">
|
| 42 |
+
<div className="grid gap-2 sm:grid-cols-3">
|
| 43 |
+
<MetricPill
|
| 44 |
+
icon={Boxes}
|
| 45 |
+
label="Models"
|
| 46 |
+
value={developer.model_count.toLocaleString()}
|
| 47 |
+
tone="bg-sky-100/80 text-sky-900 dark:bg-sky-950/40 dark:text-sky-100"
|
| 48 |
+
/>
|
| 49 |
+
<MetricPill
|
| 50 |
+
icon={BarChart3}
|
| 51 |
+
label="Benchmarks"
|
| 52 |
+
value={developer.benchmark_count.toLocaleString()}
|
| 53 |
+
tone="bg-amber-100/80 text-amber-900 dark:bg-amber-950/40 dark:text-amber-100"
|
| 54 |
+
/>
|
| 55 |
+
<MetricPill
|
| 56 |
+
icon={Sparkles}
|
| 57 |
+
label="Results"
|
| 58 |
+
value={developer.evaluation_count.toLocaleString()}
|
| 59 |
+
tone="bg-emerald-100/80 text-emerald-900 dark:bg-emerald-950/40 dark:text-emerald-100"
|
| 60 |
+
/>
|
| 61 |
+
</div>
|
| 62 |
+
|
| 63 |
+
<div className="rounded-xl border bg-muted/10 p-3">
|
| 64 |
+
<div className="mb-2 text-xs font-semibold uppercase tracking-[0.18em] text-muted-foreground">
|
| 65 |
+
Scores From
|
| 66 |
+
</div>
|
| 67 |
+
<div className="flex flex-wrap gap-2">
|
| 68 |
+
{developer.popular_evals.length > 0 ? (
|
| 69 |
+
developer.popular_evals.map((evaluation) => (
|
| 70 |
+
<Badge key={evaluation.benchmark} variant="secondary">
|
| 71 |
+
{evaluation.benchmark}
|
| 72 |
+
<span className="ml-1 text-muted-foreground">· {evaluation.model_count}</span>
|
| 73 |
+
</Badge>
|
| 74 |
+
))
|
| 75 |
+
) : (
|
| 76 |
+
<span className="text-sm text-muted-foreground">No benchmark coverage indexed yet</span>
|
| 77 |
+
)}
|
| 78 |
+
</div>
|
| 79 |
+
</div>
|
| 80 |
+
</CardContent>
|
| 81 |
+
</Card>
|
| 82 |
+
)
|
| 83 |
+
}
|
| 84 |
+
|
| 85 |
+
function MetricPill({
|
| 86 |
+
icon: Icon,
|
| 87 |
+
label,
|
| 88 |
+
value,
|
| 89 |
+
tone,
|
| 90 |
+
}: {
|
| 91 |
+
icon: ComponentType<{ className?: string }>
|
| 92 |
+
label: string
|
| 93 |
+
value: string
|
| 94 |
+
tone: string
|
| 95 |
+
}) {
|
| 96 |
+
return (
|
| 97 |
+
<div className={`flex items-center justify-between rounded-xl px-3 py-2 ${tone}`}>
|
| 98 |
+
<div className="flex items-center gap-2">
|
| 99 |
+
<Icon className="h-4 w-4" />
|
| 100 |
+
<span className="text-[11px] font-semibold uppercase tracking-[0.18em] opacity-80">
|
| 101 |
+
{label}
|
| 102 |
+
</span>
|
| 103 |
+
</div>
|
| 104 |
+
<span className="text-sm font-bold">{value}</span>
|
| 105 |
+
</div>
|
| 106 |
+
)
|
| 107 |
+
}
|
components/eval-card.tsx
CHANGED
|
@@ -26,7 +26,7 @@ export function EvalCard({ summary, delayMs = 0 }: EvalCardProps) {
|
|
| 26 |
const { mode } = useAudienceMode()
|
| 27 |
const isResearchView = mode === "research"
|
| 28 |
const scorePercent = `${Math.round(summary.avg_score_norm * 100)}%`
|
| 29 |
-
const purpose = summary.factsheet?.purpose ?? "General
|
| 30 |
|
| 31 |
return (
|
| 32 |
<Card
|
|
@@ -36,11 +36,14 @@ export function EvalCard({ summary, delayMs = 0 }: EvalCardProps) {
|
|
| 36 |
>
|
| 37 |
<CardHeader className="space-y-3 border-b border-border/60 pb-4">
|
| 38 |
<div className="text-[10px] font-semibold uppercase tracking-[0.24em] text-muted-foreground">
|
| 39 |
-
|
| 40 |
</div>
|
| 41 |
|
| 42 |
<div className="min-w-0">
|
| 43 |
<div className="text-xl font-bold">{summary.evaluation_name}</div>
|
|
|
|
|
|
|
|
|
|
| 44 |
<div className="mt-1 text-sm text-muted-foreground line-clamp-2">
|
| 45 |
{summary.metric_config.evaluation_description}
|
| 46 |
</div>
|
|
|
|
| 26 |
const { mode } = useAudienceMode()
|
| 27 |
const isResearchView = mode === "research"
|
| 28 |
const scorePercent = `${Math.round(summary.avg_score_norm * 100)}%`
|
| 29 |
+
const purpose = summary.factsheet?.purpose ?? "General single-benchmark evaluation"
|
| 30 |
|
| 31 |
return (
|
| 32 |
<Card
|
|
|
|
| 36 |
>
|
| 37 |
<CardHeader className="space-y-3 border-b border-border/60 pb-4">
|
| 38 |
<div className="text-[10px] font-semibold uppercase tracking-[0.24em] text-muted-foreground">
|
| 39 |
+
Single Benchmark
|
| 40 |
</div>
|
| 41 |
|
| 42 |
<div className="min-w-0">
|
| 43 |
<div className="text-xl font-bold">{summary.evaluation_name}</div>
|
| 44 |
+
<div className="mt-1 text-xs font-medium uppercase tracking-[0.18em] text-muted-foreground">
|
| 45 |
+
Composite benchmark: {summary.composite_benchmark_name}
|
| 46 |
+
</div>
|
| 47 |
<div className="mt-1 text-sm text-muted-foreground line-clamp-2">
|
| 48 |
{summary.metric_config.evaluation_description}
|
| 49 |
</div>
|
components/eval-detail.tsx
CHANGED
|
@@ -137,7 +137,7 @@ export function EvalDetail({ summary }: EvalDetailProps) {
|
|
| 137 |
const scoreDirectionLabel = summary.metric_config.lower_is_better ? "Lower scores rank higher" : "Higher scores rank higher"
|
| 138 |
const leaderboardTitle = isResearchView ? "Leaderboard" : "Reporting Comparison"
|
| 139 |
const leaderboardDescription = isResearchView
|
| 140 |
-
? "Models ranked by normalized score for this
|
| 141 |
: "Model results with stronger emphasis on reporting context and evaluator provenance."
|
| 142 |
|
| 143 |
const toggleRow = (key: string) =>
|
|
@@ -154,7 +154,10 @@ export function EvalDetail({ summary }: EvalDetailProps) {
|
|
| 154 |
<div className="space-y-3">
|
| 155 |
<div className="flex flex-wrap items-center gap-2">
|
| 156 |
<Badge variant="outline" className="border-border/60 bg-background/80 text-[11px] uppercase tracking-[0.18em]">
|
| 157 |
-
|
|
|
|
|
|
|
|
|
|
| 158 |
</Badge>
|
| 159 |
<Badge variant="secondary" className="font-normal capitalize">
|
| 160 |
{summary.metric_config.score_type}
|
|
@@ -227,10 +230,16 @@ export function EvalDetail({ summary }: EvalDetailProps) {
|
|
| 227 |
<div className="text-[11px] font-semibold uppercase tracking-[0.2em] text-muted-foreground">
|
| 228 |
{isResearchView ? "Metric specification" : "Reading context"}
|
| 229 |
</div>
|
| 230 |
-
<dl className="mt-3 grid gap-x-6 gap-y-3 text-sm sm:grid-cols-2 xl:grid-cols-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 231 |
<div>
|
| 232 |
<dt className="text-[11px] font-semibold uppercase tracking-[0.16em] text-muted-foreground">
|
| 233 |
-
{isResearchView ? "
|
| 234 |
</dt>
|
| 235 |
<dd className="mt-1 break-words font-medium">
|
| 236 |
{isResearchView ? summary.evaluation_id : summary.metric_config.evaluation_description}
|
|
|
|
| 137 |
const scoreDirectionLabel = summary.metric_config.lower_is_better ? "Lower scores rank higher" : "Higher scores rank higher"
|
| 138 |
const leaderboardTitle = isResearchView ? "Leaderboard" : "Reporting Comparison"
|
| 139 |
const leaderboardDescription = isResearchView
|
| 140 |
+
? "Models ranked by normalized score for this benchmark."
|
| 141 |
: "Model results with stronger emphasis on reporting context and evaluator provenance."
|
| 142 |
|
| 143 |
const toggleRow = (key: string) =>
|
|
|
|
| 154 |
<div className="space-y-3">
|
| 155 |
<div className="flex flex-wrap items-center gap-2">
|
| 156 |
<Badge variant="outline" className="border-border/60 bg-background/80 text-[11px] uppercase tracking-[0.18em]">
|
| 157 |
+
Single Benchmark
|
| 158 |
+
</Badge>
|
| 159 |
+
<Badge variant="secondary" className="font-normal">
|
| 160 |
+
Composite: {summary.composite_benchmark_name}
|
| 161 |
</Badge>
|
| 162 |
<Badge variant="secondary" className="font-normal capitalize">
|
| 163 |
{summary.metric_config.score_type}
|
|
|
|
| 230 |
<div className="text-[11px] font-semibold uppercase tracking-[0.2em] text-muted-foreground">
|
| 231 |
{isResearchView ? "Metric specification" : "Reading context"}
|
| 232 |
</div>
|
| 233 |
+
<dl className="mt-3 grid gap-x-6 gap-y-3 text-sm sm:grid-cols-2 xl:grid-cols-5">
|
| 234 |
+
<div>
|
| 235 |
+
<dt className="text-[11px] font-semibold uppercase tracking-[0.16em] text-muted-foreground">
|
| 236 |
+
Composite benchmark
|
| 237 |
+
</dt>
|
| 238 |
+
<dd className="mt-1 break-words font-medium">{summary.composite_benchmark_name}</dd>
|
| 239 |
+
</div>
|
| 240 |
<div>
|
| 241 |
<dt className="text-[11px] font-semibold uppercase tracking-[0.16em] text-muted-foreground">
|
| 242 |
+
{isResearchView ? "Single benchmark ID" : "What this covers"}
|
| 243 |
</dt>
|
| 244 |
<dd className="mt-1 break-words font-medium">
|
| 245 |
{isResearchView ? summary.evaluation_id : summary.metric_config.evaluation_description}
|
components/navigation.tsx
CHANGED
|
@@ -24,7 +24,10 @@ export function Navigation() {
|
|
| 24 |
href: "/models",
|
| 25 |
label: "Models",
|
| 26 |
icon: LayoutGrid,
|
| 27 |
-
isActive:
|
|
|
|
|
|
|
|
|
|
| 28 |
},
|
| 29 |
{
|
| 30 |
href: "/evals",
|
|
|
|
| 24 |
href: "/models",
|
| 25 |
label: "Models",
|
| 26 |
icon: LayoutGrid,
|
| 27 |
+
isActive:
|
| 28 |
+
pathname === "/models" ||
|
| 29 |
+
pathname?.startsWith("/models/") ||
|
| 30 |
+
pathname?.startsWith("/developers/")
|
| 31 |
},
|
| 32 |
{
|
| 33 |
href: "/evals",
|
lib/dashboard-data-client.ts
CHANGED
|
@@ -15,6 +15,31 @@ export interface EvalListResponse {
|
|
| 15 |
totalModels: number
|
| 16 |
}
|
| 17 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 18 |
async function fetchJson<T>(input: string): Promise<T> {
|
| 19 |
const response = await fetch(input)
|
| 20 |
|
|
@@ -48,3 +73,13 @@ export function fetchEvalSummary(evalId: string) {
|
|
| 48 |
`/api/eval-summary?id=${encodeURIComponent(evalId)}`
|
| 49 |
)
|
| 50 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 15 |
totalModels: number
|
| 16 |
}
|
| 17 |
|
| 18 |
+
export interface DeveloperListItem {
|
| 19 |
+
developer: string
|
| 20 |
+
route_id: string
|
| 21 |
+
model_count: number
|
| 22 |
+
benchmark_count: number
|
| 23 |
+
evaluation_count: number
|
| 24 |
+
popular_evals: Array<{
|
| 25 |
+
benchmark: string
|
| 26 |
+
model_count: number
|
| 27 |
+
}>
|
| 28 |
+
}
|
| 29 |
+
|
| 30 |
+
export interface DeveloperSummaryResponse {
|
| 31 |
+
developer: string
|
| 32 |
+
route_id: string
|
| 33 |
+
model_count: number
|
| 34 |
+
benchmark_count: number
|
| 35 |
+
evaluation_count: number
|
| 36 |
+
popular_evals: Array<{
|
| 37 |
+
benchmark: string
|
| 38 |
+
model_count: number
|
| 39 |
+
}>
|
| 40 |
+
models: BenchmarkEvaluationCardData[]
|
| 41 |
+
}
|
| 42 |
+
|
| 43 |
async function fetchJson<T>(input: string): Promise<T> {
|
| 44 |
const response = await fetch(input)
|
| 45 |
|
|
|
|
| 73 |
`/api/eval-summary?id=${encodeURIComponent(evalId)}`
|
| 74 |
)
|
| 75 |
}
|
| 76 |
+
|
| 77 |
+
export function fetchDevelopers() {
|
| 78 |
+
return fetchJson<DeveloperListItem[]>("/api/developers")
|
| 79 |
+
}
|
| 80 |
+
|
| 81 |
+
export function fetchDeveloperSummary(developerId: string) {
|
| 82 |
+
return fetchJson<DeveloperSummaryResponse>(
|
| 83 |
+
`/api/developer-summary?id=${encodeURIComponent(developerId)}`
|
| 84 |
+
)
|
| 85 |
+
}
|
lib/eval-processing.ts
CHANGED
|
@@ -76,7 +76,8 @@ function getEvaluationSummaryId(
|
|
| 76 |
evaluation: BenchmarkEvaluation,
|
| 77 |
result: EvaluationResult
|
| 78 |
): string {
|
| 79 |
-
|
|
|
|
| 80 |
}
|
| 81 |
|
| 82 |
// ── Eval-centric (per-benchmark) types ────────────────────────────────────────
|
|
@@ -95,6 +96,8 @@ export interface BenchmarkEvalSummary {
|
|
| 95 |
evaluation_name: string
|
| 96 |
/** URL-safe slug derived from evaluation_name */
|
| 97 |
evaluation_id: string
|
|
|
|
|
|
|
| 98 |
category: CategoryType
|
| 99 |
metric_config: MetricConfig
|
| 100 |
factsheet: EvaluationResult['factsheet'] | undefined
|
|
@@ -722,9 +725,10 @@ export function groupEvaluationsByBenchmark(
|
|
| 722 |
|
| 723 |
for (const eval_ of evaluations) {
|
| 724 |
for (const result of eval_.evaluation_results) {
|
| 725 |
-
const name = result.evaluation_name
|
| 726 |
const displayName = getEvaluationDisplayName(eval_, result)
|
| 727 |
const evalId = getEvaluationSummaryId(eval_, result)
|
|
|
|
|
|
|
| 728 |
|
| 729 |
if (!summaries[evalId]) {
|
| 730 |
// Determine category from factsheet first, then infer
|
|
@@ -743,6 +747,8 @@ export function groupEvaluationsByBenchmark(
|
|
| 743 |
summaries[evalId] = {
|
| 744 |
evaluation_name: displayName,
|
| 745 |
evaluation_id: evalId,
|
|
|
|
|
|
|
| 746 |
category,
|
| 747 |
metric_config: result.metric_config,
|
| 748 |
factsheet: result.factsheet,
|
|
|
|
| 76 |
evaluation: BenchmarkEvaluation,
|
| 77 |
result: EvaluationResult
|
| 78 |
): string {
|
| 79 |
+
const benchmarkKey = evaluation.benchmark || getBenchmarkName(evaluation, result)
|
| 80 |
+
return slugify(`${benchmarkKey}__${result.evaluation_name}`)
|
| 81 |
}
|
| 82 |
|
| 83 |
// ── Eval-centric (per-benchmark) types ────────────────────────────────────────
|
|
|
|
| 96 |
evaluation_name: string
|
| 97 |
/** URL-safe slug derived from evaluation_name */
|
| 98 |
evaluation_id: string
|
| 99 |
+
composite_benchmark_key: string
|
| 100 |
+
composite_benchmark_name: string
|
| 101 |
category: CategoryType
|
| 102 |
metric_config: MetricConfig
|
| 103 |
factsheet: EvaluationResult['factsheet'] | undefined
|
|
|
|
| 725 |
|
| 726 |
for (const eval_ of evaluations) {
|
| 727 |
for (const result of eval_.evaluation_results) {
|
|
|
|
| 728 |
const displayName = getEvaluationDisplayName(eval_, result)
|
| 729 |
const evalId = getEvaluationSummaryId(eval_, result)
|
| 730 |
+
const compositeBenchmarkKey = eval_.benchmark || getBenchmarkName(eval_, result)
|
| 731 |
+
const compositeBenchmarkName = getBenchmarkDisplayName(compositeBenchmarkKey)
|
| 732 |
|
| 733 |
if (!summaries[evalId]) {
|
| 734 |
// Determine category from factsheet first, then infer
|
|
|
|
| 747 |
summaries[evalId] = {
|
| 748 |
evaluation_name: displayName,
|
| 749 |
evaluation_id: evalId,
|
| 750 |
+
composite_benchmark_key: compositeBenchmarkKey,
|
| 751 |
+
composite_benchmark_name: compositeBenchmarkName,
|
| 752 |
category,
|
| 753 |
metric_config: result.metric_config,
|
| 754 |
factsheet: result.factsheet,
|
lib/model-data.ts
CHANGED
|
@@ -8,13 +8,15 @@ import type {
|
|
| 8 |
EvalLibrary,
|
| 9 |
EvaluationResult,
|
| 10 |
GenerationConfig,
|
|
|
|
| 11 |
ModelInfo,
|
| 12 |
SampleResult,
|
| 13 |
-
ScoreDetails,
|
| 14 |
SourceData,
|
| 15 |
SourceMetadata,
|
| 16 |
} from "@/lib/benchmark-schema"
|
|
|
|
| 17 |
import {
|
|
|
|
| 18 |
createEvaluationCard,
|
| 19 |
createModelFamilySummary,
|
| 20 |
groupEvaluationsByBenchmark,
|
|
@@ -46,21 +48,469 @@ interface RawEvaluationResult
|
|
| 46 |
evaluation_timestamp?: string
|
| 47 |
}
|
| 48 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 49 |
function getDataDirectory() {
|
| 50 |
return path.join(process.cwd(), "data")
|
| 51 |
}
|
| 52 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 53 |
function shouldCacheModelData() {
|
| 54 |
return process.env.NODE_ENV === "production"
|
| 55 |
}
|
| 56 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 57 |
async function listModelDataFiles(): Promise<string[]> {
|
| 58 |
-
const
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 59 |
|
| 60 |
-
|
| 61 |
-
|
| 62 |
-
|
| 63 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 64 |
}
|
| 65 |
|
| 66 |
function getFallbackSourceData(
|
|
@@ -157,14 +607,15 @@ export async function getDashboardData() {
|
|
| 157 |
|
| 158 |
export async function getModelCards() {
|
| 159 |
const evaluations = await loadAllEvaluationsFromDataDirectory()
|
| 160 |
-
|
| 161 |
-
|
| 162 |
-
return Object.values(groupedByModel).map((modelEvaluations) =>
|
| 163 |
-
createEvaluationCard(createModelFamilySummary(modelEvaluations))
|
| 164 |
-
)
|
| 165 |
}
|
| 166 |
|
| 167 |
export async function getEvalListData() {
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 168 |
const evaluations = await loadAllEvaluationsFromDataDirectory()
|
| 169 |
const summaries = Object.values(groupEvaluationsByBenchmark(evaluations))
|
| 170 |
const totalModels = Object.keys(groupEvaluationsByModelFamily(evaluations)).length
|
|
@@ -180,7 +631,169 @@ export async function getEvalList() {
|
|
| 180 |
return evals
|
| 181 |
}
|
| 182 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 183 |
export async function getModelSummaryById(modelId: string) {
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 184 |
const evaluations = await loadAllEvaluationsFromDataDirectory()
|
| 185 |
const groupedByFamily = groupEvaluationsByModelFamily(evaluations)
|
| 186 |
const directFamilyMatch = groupedByFamily[modelId]
|
|
|
|
| 8 |
EvalLibrary,
|
| 9 |
EvaluationResult,
|
| 10 |
GenerationConfig,
|
| 11 |
+
MetricConfig,
|
| 12 |
ModelInfo,
|
| 13 |
SampleResult,
|
|
|
|
| 14 |
SourceData,
|
| 15 |
SourceMetadata,
|
| 16 |
} from "@/lib/benchmark-schema"
|
| 17 |
+
import { inferCategoryFromBenchmark } from "@/lib/benchmark-schema"
|
| 18 |
import {
|
| 19 |
+
type BenchmarkEvalListItem,
|
| 20 |
createEvaluationCard,
|
| 21 |
createModelFamilySummary,
|
| 22 |
groupEvaluationsByBenchmark,
|
|
|
|
| 48 |
evaluation_timestamp?: string
|
| 49 |
}
|
| 50 |
|
| 51 |
+
interface IndexedModelSummary {
|
| 52 |
+
id: string
|
| 53 |
+
name: string
|
| 54 |
+
developer?: string
|
| 55 |
+
evaluator_relationship?: string | null
|
| 56 |
+
benchmark_scores?: Record<string, number>
|
| 57 |
+
}
|
| 58 |
+
|
| 59 |
+
interface IndexedBenchmarkEntry {
|
| 60 |
+
benchmark: string
|
| 61 |
+
model_count: number
|
| 62 |
+
}
|
| 63 |
+
|
| 64 |
+
interface IndexedBenchmarkDetail {
|
| 65 |
+
models: Array<{
|
| 66 |
+
model_id: string
|
| 67 |
+
name: string
|
| 68 |
+
developer?: string
|
| 69 |
+
scores?: Record<string, number>
|
| 70 |
+
}>
|
| 71 |
+
}
|
| 72 |
+
|
| 73 |
+
interface IndexedDeveloperEntry {
|
| 74 |
+
developer: string
|
| 75 |
+
model_count: number
|
| 76 |
+
}
|
| 77 |
+
|
| 78 |
+
interface IndexedDeveloperDetail {
|
| 79 |
+
developer: string
|
| 80 |
+
models: IndexedModelSummary[]
|
| 81 |
+
}
|
| 82 |
+
|
| 83 |
+
interface DeveloperAggregateSummary {
|
| 84 |
+
model_count: number
|
| 85 |
+
benchmark_count: number
|
| 86 |
+
evaluation_count: number
|
| 87 |
+
popular_evals: Array<{
|
| 88 |
+
benchmark: string
|
| 89 |
+
model_count: number
|
| 90 |
+
}>
|
| 91 |
+
}
|
| 92 |
+
|
| 93 |
function getDataDirectory() {
|
| 94 |
return path.join(process.cwd(), "data")
|
| 95 |
}
|
| 96 |
|
| 97 |
+
function getModelSubdirectory() {
|
| 98 |
+
return path.join(getDataDirectory(), "models")
|
| 99 |
+
}
|
| 100 |
+
|
| 101 |
+
function getBenchmarkSubdirectory() {
|
| 102 |
+
return path.join(getDataDirectory(), "benchmarks")
|
| 103 |
+
}
|
| 104 |
+
|
| 105 |
+
function getModelsIndexPath() {
|
| 106 |
+
return path.join(getDataDirectory(), "models.json")
|
| 107 |
+
}
|
| 108 |
+
|
| 109 |
+
function getBenchmarksIndexPath() {
|
| 110 |
+
return path.join(getDataDirectory(), "benchmarks.json")
|
| 111 |
+
}
|
| 112 |
+
|
| 113 |
+
function getDeveloperSubdirectory() {
|
| 114 |
+
return path.join(getDataDirectory(), "developers")
|
| 115 |
+
}
|
| 116 |
+
|
| 117 |
+
function getDevelopersIndexPath() {
|
| 118 |
+
return path.join(getDataDirectory(), "developers.json")
|
| 119 |
+
}
|
| 120 |
+
|
| 121 |
function shouldCacheModelData() {
|
| 122 |
return process.env.NODE_ENV === "production"
|
| 123 |
}
|
| 124 |
|
| 125 |
+
function shouldCacheIndexes() {
|
| 126 |
+
return process.env.NODE_ENV === "production"
|
| 127 |
+
}
|
| 128 |
+
|
| 129 |
+
async function listJsonFiles(directory: string): Promise<string[]> {
|
| 130 |
+
try {
|
| 131 |
+
const entries = await fs.readdir(directory, { withFileTypes: true })
|
| 132 |
+
|
| 133 |
+
return entries
|
| 134 |
+
.filter((entry) => entry.isFile() && entry.name.endsWith(".json"))
|
| 135 |
+
.map((entry) => path.join(directory, entry.name))
|
| 136 |
+
} catch {
|
| 137 |
+
return []
|
| 138 |
+
}
|
| 139 |
+
}
|
| 140 |
+
|
| 141 |
async function listModelDataFiles(): Promise<string[]> {
|
| 142 |
+
const [rootFiles, modelFiles] = await Promise.all([
|
| 143 |
+
listJsonFiles(getDataDirectory()),
|
| 144 |
+
listJsonFiles(getModelSubdirectory()),
|
| 145 |
+
])
|
| 146 |
+
|
| 147 |
+
// Prefer the pipeline layout:
|
| 148 |
+
// data/models/*.json
|
| 149 |
+
// Fall back to the legacy flat layout:
|
| 150 |
+
// data/*.json
|
| 151 |
+
// Root-level index files such as data/models.json are not raw model detail files.
|
| 152 |
+
const preferredFiles = modelFiles.length > 0 ? modelFiles : rootFiles
|
| 153 |
+
|
| 154 |
+
return preferredFiles.sort((a, b) => a.localeCompare(b))
|
| 155 |
+
}
|
| 156 |
+
|
| 157 |
+
async function readJsonFile<T>(filePath: string): Promise<T | null> {
|
| 158 |
+
try {
|
| 159 |
+
return JSON.parse(await fs.readFile(filePath, "utf8")) as T
|
| 160 |
+
} catch {
|
| 161 |
+
return null
|
| 162 |
+
}
|
| 163 |
+
}
|
| 164 |
+
|
| 165 |
+
let cachedModelIndexPromise: Promise<IndexedModelSummary[] | null> | null = null
|
| 166 |
+
|
| 167 |
+
async function readModelsIndex() {
|
| 168 |
+
const load = async () => {
|
| 169 |
+
const parsed = await readJsonFile<IndexedModelSummary[]>(getModelsIndexPath())
|
| 170 |
+
return Array.isArray(parsed) ? parsed : null
|
| 171 |
+
}
|
| 172 |
+
|
| 173 |
+
if (!shouldCacheIndexes()) {
|
| 174 |
+
return load()
|
| 175 |
+
}
|
| 176 |
+
|
| 177 |
+
if (!cachedModelIndexPromise) {
|
| 178 |
+
cachedModelIndexPromise = load()
|
| 179 |
+
}
|
| 180 |
+
|
| 181 |
+
return cachedModelIndexPromise
|
| 182 |
+
}
|
| 183 |
+
|
| 184 |
+
let cachedBenchmarkIndexPromise: Promise<IndexedBenchmarkEntry[] | null> | null = null
|
| 185 |
+
|
| 186 |
+
async function readBenchmarksIndex() {
|
| 187 |
+
const load = async () => {
|
| 188 |
+
const parsed = await readJsonFile<IndexedBenchmarkEntry[]>(getBenchmarksIndexPath())
|
| 189 |
+
return Array.isArray(parsed) ? parsed : null
|
| 190 |
+
}
|
| 191 |
+
|
| 192 |
+
if (!shouldCacheIndexes()) {
|
| 193 |
+
return load()
|
| 194 |
+
}
|
| 195 |
+
|
| 196 |
+
if (!cachedBenchmarkIndexPromise) {
|
| 197 |
+
cachedBenchmarkIndexPromise = load()
|
| 198 |
+
}
|
| 199 |
+
|
| 200 |
+
return cachedBenchmarkIndexPromise
|
| 201 |
+
}
|
| 202 |
+
|
| 203 |
+
let cachedDeveloperIndexPromise: Promise<IndexedDeveloperEntry[] | null> | null = null
|
| 204 |
+
|
| 205 |
+
async function readDevelopersIndex() {
|
| 206 |
+
const load = async () => {
|
| 207 |
+
const parsed = await readJsonFile<IndexedDeveloperEntry[]>(getDevelopersIndexPath())
|
| 208 |
+
return Array.isArray(parsed) ? parsed : null
|
| 209 |
+
}
|
| 210 |
+
|
| 211 |
+
if (!shouldCacheIndexes()) {
|
| 212 |
+
return load()
|
| 213 |
+
}
|
| 214 |
+
|
| 215 |
+
if (!cachedDeveloperIndexPromise) {
|
| 216 |
+
cachedDeveloperIndexPromise = load()
|
| 217 |
+
}
|
| 218 |
+
|
| 219 |
+
return cachedDeveloperIndexPromise
|
| 220 |
+
}
|
| 221 |
+
|
| 222 |
+
function humanizeToken(token: string) {
|
| 223 |
+
return token
|
| 224 |
+
.split(/[_-]+/g)
|
| 225 |
+
.filter(Boolean)
|
| 226 |
+
.map((part) => part.charAt(0).toUpperCase() + part.slice(1))
|
| 227 |
+
.join(" ")
|
| 228 |
+
}
|
| 229 |
+
|
| 230 |
+
function getBenchmarkDisplayName(benchmark: string) {
|
| 231 |
+
if (benchmark === "hfopenllm_v2") return "HF Open LLM v2"
|
| 232 |
+
return humanizeToken(benchmark)
|
| 233 |
+
}
|
| 234 |
+
|
| 235 |
+
function getBenchmarkMetricDisplayName(benchmark: string, metric: string) {
|
| 236 |
+
const normalized = metric.trim()
|
| 237 |
+
const genericMetrics = new Set([
|
| 238 |
+
"score",
|
| 239 |
+
"accuracy",
|
| 240 |
+
"mean win rate",
|
| 241 |
+
"exact match",
|
| 242 |
+
"f1",
|
| 243 |
+
"pass@1",
|
| 244 |
+
])
|
| 245 |
+
|
| 246 |
+
if (genericMetrics.has(normalized.toLowerCase())) {
|
| 247 |
+
return `${getBenchmarkDisplayName(benchmark)} - ${normalized}`
|
| 248 |
+
}
|
| 249 |
+
|
| 250 |
+
return normalized
|
| 251 |
+
}
|
| 252 |
+
|
| 253 |
+
function slugifyEvalId(value: string) {
|
| 254 |
+
return value.toLowerCase().replace(/[^a-z0-9]+/g, "_").replace(/^_|_$/g, "")
|
| 255 |
+
}
|
| 256 |
+
|
| 257 |
+
function inferMetricConfig(scores: number[], benchmark: string, metric: string): MetricConfig {
|
| 258 |
+
const finiteScores = scores.filter((score) => Number.isFinite(score))
|
| 259 |
+
const maxScore = finiteScores.length > 0 ? Math.max(...finiteScores) : 1
|
| 260 |
+
const minScore = finiteScores.length > 0 ? Math.min(...finiteScores) : 0
|
| 261 |
+
const appearsNormalized = minScore >= 0 && maxScore <= 1.05
|
| 262 |
+
|
| 263 |
+
return {
|
| 264 |
+
evaluation_description: `${metric} on ${getBenchmarkDisplayName(benchmark)}`,
|
| 265 |
+
lower_is_better: false,
|
| 266 |
+
score_type: "continuous",
|
| 267 |
+
min_score: appearsNormalized ? 0 : Math.min(0, minScore),
|
| 268 |
+
max_score: appearsNormalized ? 1 : Math.max(100, maxScore),
|
| 269 |
+
unit: appearsNormalized ? "accuracy" : undefined,
|
| 270 |
+
}
|
| 271 |
+
}
|
| 272 |
+
|
| 273 |
+
async function buildEvalListDataFromBenchmarkIndexes(): Promise<{
|
| 274 |
+
evals: BenchmarkEvalListItem[]
|
| 275 |
+
totalModels: number
|
| 276 |
+
} | null> {
|
| 277 |
+
const [benchmarkIndex, modelIndex] = await Promise.all([
|
| 278 |
+
readBenchmarksIndex(),
|
| 279 |
+
readModelsIndex(),
|
| 280 |
+
])
|
| 281 |
+
|
| 282 |
+
if (!benchmarkIndex?.length) {
|
| 283 |
+
return null
|
| 284 |
+
}
|
| 285 |
+
|
| 286 |
+
const benchmarkDetails = await Promise.all(
|
| 287 |
+
benchmarkIndex.map(async ({ benchmark }) => ({
|
| 288 |
+
benchmark,
|
| 289 |
+
detail: await readJsonFile<IndexedBenchmarkDetail>(
|
| 290 |
+
path.join(getBenchmarkSubdirectory(), `${benchmark}.json`)
|
| 291 |
+
),
|
| 292 |
+
}))
|
| 293 |
+
)
|
| 294 |
+
|
| 295 |
+
const evals: BenchmarkEvalListItem[] = []
|
| 296 |
+
|
| 297 |
+
for (const { benchmark, detail } of benchmarkDetails) {
|
| 298 |
+
if (!detail?.models?.length) {
|
| 299 |
+
continue
|
| 300 |
+
}
|
| 301 |
+
|
| 302 |
+
const metricScores = new Map<string, number[]>()
|
| 303 |
+
|
| 304 |
+
for (const model of detail.models) {
|
| 305 |
+
for (const [metric, score] of Object.entries(model.scores ?? {})) {
|
| 306 |
+
if (!Number.isFinite(score)) {
|
| 307 |
+
continue
|
| 308 |
+
}
|
| 309 |
+
|
| 310 |
+
const bucket = metricScores.get(metric) ?? []
|
| 311 |
+
bucket.push(score)
|
| 312 |
+
metricScores.set(metric, bucket)
|
| 313 |
+
}
|
| 314 |
+
}
|
| 315 |
+
|
| 316 |
+
for (const [metric, scores] of metricScores) {
|
| 317 |
+
if (scores.length === 0) {
|
| 318 |
+
continue
|
| 319 |
+
}
|
| 320 |
+
|
| 321 |
+
const displayName = getBenchmarkMetricDisplayName(benchmark, metric)
|
| 322 |
+
const metricConfig = inferMetricConfig(scores, benchmark, metric)
|
| 323 |
+
const avgScore = scores.reduce((sum, score) => sum + score, 0) / scores.length
|
| 324 |
+
const maxScore = metricConfig.max_score ?? 1
|
| 325 |
+
const minScore = metricConfig.min_score ?? 0
|
| 326 |
+
const range = maxScore - minScore
|
| 327 |
+
|
| 328 |
+
evals.push({
|
| 329 |
+
evaluation_name: displayName,
|
| 330 |
+
evaluation_id: slugifyEvalId(`${benchmark}__${metric}`),
|
| 331 |
+
composite_benchmark_key: benchmark,
|
| 332 |
+
composite_benchmark_name: getBenchmarkDisplayName(benchmark),
|
| 333 |
+
category: inferCategoryFromBenchmark(displayName),
|
| 334 |
+
metric_config: metricConfig,
|
| 335 |
+
factsheet: undefined,
|
| 336 |
+
models_count: scores.length,
|
| 337 |
+
evaluator_names: [],
|
| 338 |
+
source_types: [],
|
| 339 |
+
latest_source_name: getBenchmarkDisplayName(benchmark),
|
| 340 |
+
third_party_ratio: 0,
|
| 341 |
+
missing_generation_config_count: 0,
|
| 342 |
+
best_model: null,
|
| 343 |
+
worst_model: null,
|
| 344 |
+
avg_score: avgScore,
|
| 345 |
+
avg_score_norm: range > 0 ? (avgScore - minScore) / range : 0,
|
| 346 |
+
})
|
| 347 |
+
}
|
| 348 |
+
}
|
| 349 |
+
|
| 350 |
+
return {
|
| 351 |
+
evals: evals.sort((a, b) => a.evaluation_name.localeCompare(b.evaluation_name)),
|
| 352 |
+
totalModels: modelIndex?.length ?? 0,
|
| 353 |
+
}
|
| 354 |
+
}
|
| 355 |
+
|
| 356 |
+
function createIndexedModelInfo(summary: IndexedModelSummary): ModelInfo {
|
| 357 |
+
return {
|
| 358 |
+
id: summary.id,
|
| 359 |
+
name: summary.name || summary.id,
|
| 360 |
+
developer: summary.developer,
|
| 361 |
+
}
|
| 362 |
+
}
|
| 363 |
+
|
| 364 |
+
function pipelineSlugify(text: string) {
|
| 365 |
+
return (
|
| 366 |
+
text
|
| 367 |
+
.replace(/[\x00-\x1f\x7f]/g, "")
|
| 368 |
+
.replace(/[^a-zA-Z0-9._-]/g, "_")
|
| 369 |
+
.replace(/^_+|_+$/g, "") || "unknown"
|
| 370 |
+
)
|
| 371 |
+
}
|
| 372 |
+
|
| 373 |
+
function getDeveloperRouteId(developer: string) {
|
| 374 |
+
return pipelineSlugify(developer)
|
| 375 |
+
}
|
| 376 |
+
|
| 377 |
+
function getModelDetailSlugCandidates(modelId: string) {
|
| 378 |
+
const normalized = modelId.trim()
|
| 379 |
+
const lowercased = normalized.toLowerCase()
|
| 380 |
+
const candidates = new Set([
|
| 381 |
+
pipelineSlugify(normalized),
|
| 382 |
+
pipelineSlugify(lowercased),
|
| 383 |
+
])
|
| 384 |
+
|
| 385 |
+
return Array.from(candidates)
|
| 386 |
+
}
|
| 387 |
+
|
| 388 |
+
async function loadEvaluationsForModelId(modelId: string) {
|
| 389 |
+
for (const slug of getModelDetailSlugCandidates(modelId)) {
|
| 390 |
+
const raw = await readJsonFile<RawModelFile>(
|
| 391 |
+
path.join(getModelSubdirectory(), `${slug}.json`)
|
| 392 |
+
)
|
| 393 |
+
|
| 394 |
+
if (!raw?.model_info || !Array.isArray(raw.evaluations)) {
|
| 395 |
+
continue
|
| 396 |
+
}
|
| 397 |
+
|
| 398 |
+
return raw.evaluations.map((evaluation) =>
|
| 399 |
+
normalizeEvaluation(raw.model_info, evaluation)
|
| 400 |
+
)
|
| 401 |
+
}
|
| 402 |
+
|
| 403 |
+
return []
|
| 404 |
+
}
|
| 405 |
+
|
| 406 |
+
async function loadEvaluationsForModelIds(modelIds: string[]) {
|
| 407 |
+
const loaded = await Promise.all(modelIds.map((modelId) => loadEvaluationsForModelId(modelId)))
|
| 408 |
+
return loaded.flat()
|
| 409 |
+
}
|
| 410 |
+
|
| 411 |
+
function buildModelCardsFromEvaluations(evaluations: BenchmarkEvaluation[]) {
|
| 412 |
+
const groupedByModel = groupEvaluationsByModelFamily(evaluations)
|
| 413 |
+
|
| 414 |
+
return Object.values(groupedByModel)
|
| 415 |
+
.map((modelEvaluations) =>
|
| 416 |
+
createEvaluationCard(createModelFamilySummary(modelEvaluations))
|
| 417 |
+
)
|
| 418 |
+
.sort(
|
| 419 |
+
(a, b) =>
|
| 420 |
+
new Date(b.latest_timestamp).getTime() - new Date(a.latest_timestamp).getTime()
|
| 421 |
+
)
|
| 422 |
+
}
|
| 423 |
+
|
| 424 |
+
function buildDeveloperListFromModelsIndex(models: IndexedModelSummary[]) {
|
| 425 |
+
const counts = new Map<string, number>()
|
| 426 |
+
|
| 427 |
+
for (const model of models) {
|
| 428 |
+
const developer = model.developer?.trim() || "unknown"
|
| 429 |
+
counts.set(developer, (counts.get(developer) ?? 0) + 1)
|
| 430 |
+
}
|
| 431 |
+
|
| 432 |
+
return Array.from(counts.entries())
|
| 433 |
+
.map(([developer, model_count]) => ({
|
| 434 |
+
developer,
|
| 435 |
+
route_id: getDeveloperRouteId(developer),
|
| 436 |
+
model_count,
|
| 437 |
+
}))
|
| 438 |
+
.sort((a, b) => a.developer.localeCompare(b.developer))
|
| 439 |
+
}
|
| 440 |
+
|
| 441 |
+
function summarizeDeveloperModels(models: IndexedModelSummary[]): DeveloperAggregateSummary {
|
| 442 |
+
const benchmarkCounts = new Map<string, number>()
|
| 443 |
+
let evaluationCount = 0
|
| 444 |
+
|
| 445 |
+
for (const model of models) {
|
| 446 |
+
const seenBenchmarks = new Set<string>()
|
| 447 |
+
|
| 448 |
+
for (const key of Object.keys(model.benchmark_scores ?? {})) {
|
| 449 |
+
evaluationCount += 1
|
| 450 |
+
|
| 451 |
+
const [benchmark] = key.split("/", 1)
|
| 452 |
+
if (!benchmark || seenBenchmarks.has(benchmark)) {
|
| 453 |
+
continue
|
| 454 |
+
}
|
| 455 |
+
|
| 456 |
+
seenBenchmarks.add(benchmark)
|
| 457 |
+
benchmarkCounts.set(benchmark, (benchmarkCounts.get(benchmark) ?? 0) + 1)
|
| 458 |
+
}
|
| 459 |
+
}
|
| 460 |
+
|
| 461 |
+
const popularEvals = Array.from(benchmarkCounts.entries())
|
| 462 |
+
.sort((a, b) => {
|
| 463 |
+
if (b[1] !== a[1]) {
|
| 464 |
+
return b[1] - a[1]
|
| 465 |
+
}
|
| 466 |
+
|
| 467 |
+
return a[0].localeCompare(b[0])
|
| 468 |
+
})
|
| 469 |
+
.slice(0, 3)
|
| 470 |
+
.map(([benchmark, model_count]) => ({
|
| 471 |
+
benchmark: getBenchmarkDisplayName(benchmark),
|
| 472 |
+
model_count,
|
| 473 |
+
}))
|
| 474 |
+
|
| 475 |
+
return {
|
| 476 |
+
model_count: models.length,
|
| 477 |
+
benchmark_count: benchmarkCounts.size,
|
| 478 |
+
evaluation_count: evaluationCount,
|
| 479 |
+
popular_evals: popularEvals,
|
| 480 |
+
}
|
| 481 |
+
}
|
| 482 |
+
|
| 483 |
+
async function readDeveloperDetail(routeId: string) {
|
| 484 |
+
const direct = await readJsonFile<IndexedDeveloperDetail>(
|
| 485 |
+
path.join(getDeveloperSubdirectory(), `${routeId}.json`)
|
| 486 |
+
)
|
| 487 |
|
| 488 |
+
if (direct?.developer && Array.isArray(direct.models)) {
|
| 489 |
+
return direct
|
| 490 |
+
}
|
| 491 |
+
|
| 492 |
+
const developersIndex = await readDevelopersIndex()
|
| 493 |
+
const matchedDeveloper = developersIndex?.find(
|
| 494 |
+
(entry) =>
|
| 495 |
+
entry.developer === routeId || getDeveloperRouteId(entry.developer) === routeId
|
| 496 |
+
)
|
| 497 |
+
|
| 498 |
+
if (!matchedDeveloper) {
|
| 499 |
+
return null
|
| 500 |
+
}
|
| 501 |
+
|
| 502 |
+
const resolved = await readJsonFile<IndexedDeveloperDetail>(
|
| 503 |
+
path.join(
|
| 504 |
+
getDeveloperSubdirectory(),
|
| 505 |
+
`${getDeveloperRouteId(matchedDeveloper.developer)}.json`
|
| 506 |
+
)
|
| 507 |
+
)
|
| 508 |
+
|
| 509 |
+
if (resolved?.developer && Array.isArray(resolved.models)) {
|
| 510 |
+
return resolved
|
| 511 |
+
}
|
| 512 |
+
|
| 513 |
+
return null
|
| 514 |
}
|
| 515 |
|
| 516 |
function getFallbackSourceData(
|
|
|
|
| 607 |
|
| 608 |
export async function getModelCards() {
|
| 609 |
const evaluations = await loadAllEvaluationsFromDataDirectory()
|
| 610 |
+
return buildModelCardsFromEvaluations(evaluations)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 611 |
}
|
| 612 |
|
| 613 |
export async function getEvalListData() {
|
| 614 |
+
const indexed = await buildEvalListDataFromBenchmarkIndexes()
|
| 615 |
+
if (indexed) {
|
| 616 |
+
return indexed
|
| 617 |
+
}
|
| 618 |
+
|
| 619 |
const evaluations = await loadAllEvaluationsFromDataDirectory()
|
| 620 |
const summaries = Object.values(groupEvaluationsByBenchmark(evaluations))
|
| 621 |
const totalModels = Object.keys(groupEvaluationsByModelFamily(evaluations)).length
|
|
|
|
| 631 |
return evals
|
| 632 |
}
|
| 633 |
|
| 634 |
+
export async function getDeveloperList() {
|
| 635 |
+
const [developersIndex, modelsIndex] = await Promise.all([
|
| 636 |
+
readDevelopersIndex(),
|
| 637 |
+
readModelsIndex(),
|
| 638 |
+
])
|
| 639 |
+
|
| 640 |
+
if (developersIndex?.length) {
|
| 641 |
+
const details = await Promise.all(
|
| 642 |
+
developersIndex.map(async (entry) => ({
|
| 643 |
+
developer: entry.developer,
|
| 644 |
+
detail: await readJsonFile<IndexedDeveloperDetail>(
|
| 645 |
+
path.join(
|
| 646 |
+
getDeveloperSubdirectory(),
|
| 647 |
+
`${getDeveloperRouteId(entry.developer)}.json`
|
| 648 |
+
)
|
| 649 |
+
),
|
| 650 |
+
}))
|
| 651 |
+
)
|
| 652 |
+
|
| 653 |
+
return details
|
| 654 |
+
.map(({ developer, detail }) => {
|
| 655 |
+
const aggregate = summarizeDeveloperModels(detail?.models ?? [])
|
| 656 |
+
|
| 657 |
+
return {
|
| 658 |
+
developer,
|
| 659 |
+
route_id: getDeveloperRouteId(developer),
|
| 660 |
+
model_count: detail?.models?.length ?? aggregate.model_count ?? 0,
|
| 661 |
+
benchmark_count: aggregate.benchmark_count,
|
| 662 |
+
evaluation_count: aggregate.evaluation_count,
|
| 663 |
+
popular_evals: aggregate.popular_evals,
|
| 664 |
+
}
|
| 665 |
+
})
|
| 666 |
+
.sort((a, b) => a.developer.localeCompare(b.developer))
|
| 667 |
+
}
|
| 668 |
+
|
| 669 |
+
if (modelsIndex?.length) {
|
| 670 |
+
return buildDeveloperListFromModelsIndex(modelsIndex).map((entry) => {
|
| 671 |
+
const developerModels = modelsIndex.filter(
|
| 672 |
+
(model) => (model.developer?.trim() || "unknown") === entry.developer
|
| 673 |
+
)
|
| 674 |
+
const aggregate = summarizeDeveloperModels(developerModels)
|
| 675 |
+
|
| 676 |
+
return {
|
| 677 |
+
...entry,
|
| 678 |
+
benchmark_count: aggregate.benchmark_count,
|
| 679 |
+
evaluation_count: aggregate.evaluation_count,
|
| 680 |
+
popular_evals: aggregate.popular_evals,
|
| 681 |
+
}
|
| 682 |
+
})
|
| 683 |
+
}
|
| 684 |
+
|
| 685 |
+
const evaluations = await loadAllEvaluationsFromDataDirectory()
|
| 686 |
+
const groupedByModel = groupEvaluationsByModel(evaluations)
|
| 687 |
+
const counts = new Map<string, number>()
|
| 688 |
+
|
| 689 |
+
for (const modelEvaluations of Object.values(groupedByModel)) {
|
| 690 |
+
const developer = modelEvaluations[0]?.model_info.developer?.trim() || "unknown"
|
| 691 |
+
counts.set(developer, (counts.get(developer) ?? 0) + 1)
|
| 692 |
+
}
|
| 693 |
+
|
| 694 |
+
return Array.from(counts.entries())
|
| 695 |
+
.map(([developer, model_count]) => ({
|
| 696 |
+
developer,
|
| 697 |
+
route_id: getDeveloperRouteId(developer),
|
| 698 |
+
model_count,
|
| 699 |
+
benchmark_count: 0,
|
| 700 |
+
evaluation_count: 0,
|
| 701 |
+
popular_evals: [],
|
| 702 |
+
}))
|
| 703 |
+
.sort((a, b) => a.developer.localeCompare(b.developer))
|
| 704 |
+
}
|
| 705 |
+
|
| 706 |
+
export async function getDeveloperSummaryById(routeId: string) {
|
| 707 |
+
const detail = await readDeveloperDetail(routeId)
|
| 708 |
+
|
| 709 |
+
if (detail) {
|
| 710 |
+
const evaluations = await loadEvaluationsForModelIds(
|
| 711 |
+
detail.models.map((model) => model.id)
|
| 712 |
+
)
|
| 713 |
+
const aggregate = summarizeDeveloperModels(detail.models)
|
| 714 |
+
|
| 715 |
+
return {
|
| 716 |
+
developer: detail.developer,
|
| 717 |
+
route_id: getDeveloperRouteId(detail.developer),
|
| 718 |
+
model_count: aggregate.model_count,
|
| 719 |
+
benchmark_count: aggregate.benchmark_count,
|
| 720 |
+
evaluation_count: aggregate.evaluation_count,
|
| 721 |
+
popular_evals: aggregate.popular_evals,
|
| 722 |
+
models: buildModelCardsFromEvaluations(evaluations),
|
| 723 |
+
}
|
| 724 |
+
}
|
| 725 |
+
|
| 726 |
+
const evaluations = await loadAllEvaluationsFromDataDirectory()
|
| 727 |
+
const groupedByModel = groupEvaluationsByModel(evaluations)
|
| 728 |
+
const matchedEvaluations = Object.values(groupedByModel)
|
| 729 |
+
.filter((modelEvaluations) => {
|
| 730 |
+
const developer = modelEvaluations[0]?.model_info.developer?.trim() || "unknown"
|
| 731 |
+
return developer === routeId || getDeveloperRouteId(developer) === routeId
|
| 732 |
+
})
|
| 733 |
+
.flat()
|
| 734 |
+
|
| 735 |
+
if (matchedEvaluations.length === 0) {
|
| 736 |
+
return null
|
| 737 |
+
}
|
| 738 |
+
|
| 739 |
+
const developer = matchedEvaluations[0]?.model_info.developer?.trim() || "unknown"
|
| 740 |
+
const groupedModels = groupEvaluationsByModel(matchedEvaluations)
|
| 741 |
+
const fallbackModels: IndexedModelSummary[] = Object.values(groupedModels).map((items) => ({
|
| 742 |
+
id: items[0]?.model_info.id ?? "unknown",
|
| 743 |
+
name: items[0]?.model_info.name ?? items[0]?.model_info.id ?? "unknown",
|
| 744 |
+
developer,
|
| 745 |
+
benchmark_scores: Object.fromEntries(
|
| 746 |
+
items.flatMap((evaluation) =>
|
| 747 |
+
evaluation.evaluation_results
|
| 748 |
+
.map((result) => {
|
| 749 |
+
const score = result.score_details?.score
|
| 750 |
+
if (!Number.isFinite(score)) {
|
| 751 |
+
return null
|
| 752 |
+
}
|
| 753 |
+
|
| 754 |
+
const benchmarkName =
|
| 755 |
+
!Array.isArray(result.source_data) && result.source_data?.dataset_name
|
| 756 |
+
? result.source_data.dataset_name
|
| 757 |
+
: evaluation.benchmark ?? result.evaluation_name
|
| 758 |
+
|
| 759 |
+
return [[`${benchmarkName}/${result.evaluation_name}`, score]] as const
|
| 760 |
+
})
|
| 761 |
+
.filter((entry): entry is readonly [string, number][] => entry !== null)
|
| 762 |
+
),
|
| 763 |
+
),
|
| 764 |
+
}))
|
| 765 |
+
const aggregate = summarizeDeveloperModels(fallbackModels)
|
| 766 |
+
|
| 767 |
+
return {
|
| 768 |
+
developer,
|
| 769 |
+
route_id: getDeveloperRouteId(developer),
|
| 770 |
+
model_count: aggregate.model_count,
|
| 771 |
+
benchmark_count: aggregate.benchmark_count,
|
| 772 |
+
evaluation_count: aggregate.evaluation_count,
|
| 773 |
+
popular_evals: aggregate.popular_evals,
|
| 774 |
+
models: buildModelCardsFromEvaluations(matchedEvaluations),
|
| 775 |
+
}
|
| 776 |
+
}
|
| 777 |
+
|
| 778 |
export async function getModelSummaryById(modelId: string) {
|
| 779 |
+
const indexedModels = await readModelsIndex()
|
| 780 |
+
|
| 781 |
+
if (indexedModels?.length) {
|
| 782 |
+
const matchingModelIds = indexedModels
|
| 783 |
+
.filter((summary) => {
|
| 784 |
+
const familyId = getCanonicalModelIdentity(createIndexedModelInfo(summary)).familyId
|
| 785 |
+
return familyId === modelId || getModelFamilyRouteId(familyId) === modelId || summary.id === modelId
|
| 786 |
+
})
|
| 787 |
+
.map((summary) => summary.id)
|
| 788 |
+
|
| 789 |
+
if (matchingModelIds.length > 0) {
|
| 790 |
+
const evaluations = await loadEvaluationsForModelIds(matchingModelIds)
|
| 791 |
+
if (evaluations.length > 0) {
|
| 792 |
+
return createModelFamilySummary(evaluations)
|
| 793 |
+
}
|
| 794 |
+
}
|
| 795 |
+
}
|
| 796 |
+
|
| 797 |
const evaluations = await loadAllEvaluationsFromDataDirectory()
|
| 798 |
const groupedByFamily = groupEvaluationsByModelFamily(evaluations)
|
| 799 |
const directFamilyMatch = groupedByFamily[modelId]
|
lib/model-family.ts
CHANGED
|
@@ -191,11 +191,26 @@ export function getCanonicalModelIdentity(modelInfo: ModelInfo): ParsedModelIden
|
|
| 191 |
|
| 192 |
export function normalizeModelInfo(modelInfo: ModelInfo): ModelInfo {
|
| 193 |
const identity = getCanonicalModelIdentity(modelInfo)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 194 |
|
| 195 |
return {
|
| 196 |
...modelInfo,
|
| 197 |
id: modelInfo.id.trim(),
|
| 198 |
name: identity.variantDisplayName,
|
|
|
|
|
|
|
|
|
|
|
|
|
| 199 |
model_version:
|
| 200 |
modelInfo.model_version ??
|
| 201 |
(identity.variantKey === "base" ? undefined : identity.variantLabel),
|
|
|
|
| 191 |
|
| 192 |
export function normalizeModelInfo(modelInfo: ModelInfo): ModelInfo {
|
| 193 |
const identity = getCanonicalModelIdentity(modelInfo)
|
| 194 |
+
const additionalArchitecture =
|
| 195 |
+
typeof modelInfo.additional_details?.architecture === "string"
|
| 196 |
+
? modelInfo.additional_details.architecture
|
| 197 |
+
: undefined
|
| 198 |
+
const rawParamsBillions = modelInfo.additional_details?.params_billions
|
| 199 |
+
const parsedParamsBillions =
|
| 200 |
+
typeof rawParamsBillions === "number"
|
| 201 |
+
? rawParamsBillions
|
| 202 |
+
: typeof rawParamsBillions === "string"
|
| 203 |
+
? Number.parseFloat(rawParamsBillions)
|
| 204 |
+
: null
|
| 205 |
|
| 206 |
return {
|
| 207 |
...modelInfo,
|
| 208 |
id: modelInfo.id.trim(),
|
| 209 |
name: identity.variantDisplayName,
|
| 210 |
+
architecture: modelInfo.architecture ?? additionalArchitecture,
|
| 211 |
+
parameter_count:
|
| 212 |
+
modelInfo.parameter_count ??
|
| 213 |
+
(Number.isFinite(parsedParamsBillions ?? NaN) ? `${parsedParamsBillions}B` : undefined),
|
| 214 |
model_version:
|
| 215 |
modelInfo.model_version ??
|
| 216 |
(identity.variantKey === "base" ? undefined : identity.variantLabel),
|