Spaces:
Running
Running
ux changes
Browse files- app/globals.css +68 -0
- app/models/page.tsx +319 -12
- components/benchmark-evaluation-card.tsx +195 -129
- components/model-compare-dialog.tsx +459 -0
- lib/eval-processing.ts +45 -2
app/globals.css
CHANGED
|
@@ -212,6 +212,74 @@
|
|
| 212 |
}
|
| 213 |
}
|
| 214 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 215 |
@media (prefers-reduced-motion: reduce) {
|
| 216 |
.motion-academic-enter,
|
| 217 |
.motion-academic-enter-soft {
|
|
|
|
| 212 |
}
|
| 213 |
}
|
| 214 |
|
| 215 |
+
.param-range-input {
|
| 216 |
+
appearance: none;
|
| 217 |
+
position: absolute;
|
| 218 |
+
left: 0.375rem;
|
| 219 |
+
right: 0.375rem;
|
| 220 |
+
top: 50%;
|
| 221 |
+
transform: translateY(-50%);
|
| 222 |
+
width: calc(100% - 0.75rem);
|
| 223 |
+
height: 3px;
|
| 224 |
+
background: transparent;
|
| 225 |
+
pointer-events: none;
|
| 226 |
+
}
|
| 227 |
+
|
| 228 |
+
.param-range-input::-webkit-slider-runnable-track {
|
| 229 |
+
height: 3px;
|
| 230 |
+
background: transparent;
|
| 231 |
+
}
|
| 232 |
+
|
| 233 |
+
.param-range-input::-webkit-slider-thumb {
|
| 234 |
+
appearance: none;
|
| 235 |
+
pointer-events: auto;
|
| 236 |
+
height: 12px;
|
| 237 |
+
width: 12px;
|
| 238 |
+
border-radius: 9999px;
|
| 239 |
+
border: 2px solid var(--color-foreground);
|
| 240 |
+
background: var(--color-foreground);
|
| 241 |
+
margin-top: -4.5px;
|
| 242 |
+
cursor: pointer;
|
| 243 |
+
box-shadow: 0 0 0 3px color-mix(in srgb, var(--color-background) 80%, transparent);
|
| 244 |
+
transition:
|
| 245 |
+
box-shadow 180ms var(--ease-out-quart),
|
| 246 |
+
background-color 180ms var(--ease-out-quart),
|
| 247 |
+
border-color 180ms var(--ease-out-quart);
|
| 248 |
+
}
|
| 249 |
+
|
| 250 |
+
.param-range-input::-moz-range-track {
|
| 251 |
+
height: 3px;
|
| 252 |
+
border: 0;
|
| 253 |
+
background: transparent;
|
| 254 |
+
}
|
| 255 |
+
|
| 256 |
+
.param-range-input::-moz-range-thumb {
|
| 257 |
+
pointer-events: auto;
|
| 258 |
+
height: 12px;
|
| 259 |
+
width: 12px;
|
| 260 |
+
border-radius: 9999px;
|
| 261 |
+
border: 2px solid var(--color-foreground);
|
| 262 |
+
background: var(--color-foreground);
|
| 263 |
+
cursor: pointer;
|
| 264 |
+
box-shadow: 0 0 0 3px color-mix(in srgb, var(--color-background) 80%, transparent);
|
| 265 |
+
transition:
|
| 266 |
+
box-shadow 180ms var(--ease-out-quart),
|
| 267 |
+
background-color 180ms var(--ease-out-quart),
|
| 268 |
+
border-color 180ms var(--ease-out-quart);
|
| 269 |
+
}
|
| 270 |
+
|
| 271 |
+
.param-range-input:focus-visible::-webkit-slider-thumb {
|
| 272 |
+
box-shadow:
|
| 273 |
+
0 0 0 3px color-mix(in srgb, var(--color-background) 80%, transparent),
|
| 274 |
+
0 0 0 6px color-mix(in srgb, var(--color-ring) 35%, transparent);
|
| 275 |
+
}
|
| 276 |
+
|
| 277 |
+
.param-range-input:focus-visible::-moz-range-thumb {
|
| 278 |
+
box-shadow:
|
| 279 |
+
0 0 0 3px color-mix(in srgb, var(--color-background) 80%, transparent),
|
| 280 |
+
0 0 0 6px color-mix(in srgb, var(--color-ring) 35%, transparent);
|
| 281 |
+
}
|
| 282 |
+
|
| 283 |
@media (prefers-reduced-motion: reduce) {
|
| 284 |
.motion-academic-enter,
|
| 285 |
.motion-academic-enter-soft {
|
app/models/page.tsx
CHANGED
|
@@ -5,15 +5,53 @@ import { useAudienceMode } from "@/components/audience-mode-provider"
|
|
| 5 |
import { Button } from "@/components/ui/button"
|
| 6 |
import { Input } from "@/components/ui/input"
|
| 7 |
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"
|
| 8 |
-
import { ArrowUpDown, Search } from "lucide-react"
|
| 9 |
import { BenchmarkEvaluationCard, type BenchmarkEvaluationCardData } from "@/components/benchmark-evaluation-card"
|
| 10 |
import { DeveloperCard } from "@/components/developer-card"
|
| 11 |
import { ListPagination } from "@/components/list-pagination"
|
|
|
|
| 12 |
import { Navigation } from "@/components/navigation"
|
| 13 |
import { PageHeader } from "@/components/page-header"
|
|
|
|
| 14 |
import { fetchDevelopers, fetchModelCards, type DeveloperListItem } from "@/lib/dashboard-data-client"
|
| 15 |
|
| 16 |
const PAGE_SIZE = 40
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 17 |
|
| 18 |
export default function ModelsPage() {
|
| 19 |
const { mode } = useAudienceMode()
|
|
@@ -22,9 +60,13 @@ export default function ModelsPage() {
|
|
| 22 |
const [loadingModels, setLoadingModels] = useState(true)
|
| 23 |
const [loadingDevelopers, setLoadingDevelopers] = useState(true)
|
| 24 |
const [groupByDeveloper, setGroupByDeveloper] = useState(false)
|
| 25 |
-
const [modelSortBy, setModelSortBy] = useState<"date" | "name" | "benchmarks">("
|
| 26 |
const [developerSortBy, setDeveloperSortBy] = useState<"coverage" | "evaluated" | "models" | "name">("coverage")
|
| 27 |
const [searchQuery, setSearchQuery] = useState("")
|
|
|
|
|
|
|
|
|
|
|
|
|
| 28 |
const [page, setPage] = useState(1)
|
| 29 |
|
| 30 |
useEffect(() => {
|
|
@@ -52,27 +94,62 @@ export default function ModelsPage() {
|
|
| 52 |
setGroupByDeveloper(params.get("group") === "developer")
|
| 53 |
}, [])
|
| 54 |
|
| 55 |
-
|
| 56 |
-
|
|
|
|
|
|
|
|
|
|
| 57 |
|
| 58 |
-
|
| 59 |
-
|
|
|
|
| 60 |
}
|
| 61 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 62 |
return evaluations.filter((evaluation) => {
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 63 |
const haystacks = [
|
| 64 |
evaluation.model_name,
|
| 65 |
evaluation.canonical_model_name,
|
| 66 |
evaluation.developer,
|
| 67 |
evaluation.architecture,
|
| 68 |
evaluation.latest_source_name,
|
|
|
|
| 69 |
...evaluation.evaluator_names,
|
| 70 |
...evaluation.top_scores.map((score) => score.benchmark),
|
| 71 |
]
|
| 72 |
|
| 73 |
return haystacks.some((value) => value?.toLowerCase().includes(query))
|
| 74 |
})
|
| 75 |
-
}, [evaluations, searchQuery])
|
| 76 |
|
| 77 |
const sortedEvaluations = useMemo(() => {
|
| 78 |
const sorted = [...filteredEvaluations]
|
|
@@ -89,6 +166,34 @@ export default function ModelsPage() {
|
|
| 89 |
case "benchmarks":
|
| 90 |
sorted.sort((a, b) => b.benchmarks_count - a.benchmarks_count)
|
| 91 |
break
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 92 |
}
|
| 93 |
|
| 94 |
return sorted
|
|
@@ -142,7 +247,7 @@ export default function ModelsPage() {
|
|
| 142 |
|
| 143 |
useEffect(() => {
|
| 144 |
setPage(1)
|
| 145 |
-
}, [developerSortBy, groupByDeveloper, modelSortBy, searchQuery])
|
| 146 |
|
| 147 |
const pagedEvaluations = useMemo(
|
| 148 |
() => sortedEvaluations.slice((page - 1) * PAGE_SIZE, page * PAGE_SIZE),
|
|
@@ -156,9 +261,41 @@ export default function ModelsPage() {
|
|
| 156 |
|
| 157 |
const handleDelete = (id: string) => {
|
| 158 |
setEvaluations((prev) => prev.filter((e) => e.id !== id))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 159 |
}
|
| 160 |
|
| 161 |
const loading = loadingModels || loadingDevelopers
|
|
|
|
|
|
|
|
|
|
| 162 |
|
| 163 |
if (loading) {
|
| 164 |
return (
|
|
@@ -184,8 +321,8 @@ export default function ModelsPage() {
|
|
| 184 |
groupByDeveloper
|
| 185 |
? "Group the model corpus by developer to compare how many models each team ships and which eval suites show up most often."
|
| 186 |
: mode === "research"
|
| 187 |
-
? "Browse model cards with benchmark breadth,
|
| 188 |
-
: "Browse model cards with stronger emphasis on reporting breadth, evidence, and evaluation accountability."
|
| 189 |
}
|
| 190 |
metaItems={[
|
| 191 |
groupByDeveloper
|
|
@@ -203,9 +340,33 @@ export default function ModelsPage() {
|
|
| 203 |
value: filteredDevelopers.reduce((sum, developer) => sum + developer.benchmark_count, 0).toString(),
|
| 204 |
}
|
| 205 |
: { label: "Reporting orgs", value: new Set(sortedEvaluations.flatMap((e) => e.evaluator_names)).size.toString() },
|
|
|
|
|
|
|
|
|
|
| 206 |
]}
|
| 207 |
/>
|
| 208 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 209 |
<div className="mb-8 flex flex-col gap-4 border-b border-border/50 pb-6 sm:flex-row sm:flex-wrap sm:items-center">
|
| 210 |
<div className="relative w-full sm:max-w-sm">
|
| 211 |
<Search className="pointer-events-none absolute left-3 top-1/2 h-4 w-4 -translate-y-1/2 text-muted-foreground" />
|
|
@@ -220,6 +381,91 @@ export default function ModelsPage() {
|
|
| 220 |
className="pl-9"
|
| 221 |
/>
|
| 222 |
</div>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 223 |
<div className="inline-flex w-fit rounded-full border bg-muted/20 p-1">
|
| 224 |
<button
|
| 225 |
type="button"
|
|
@@ -268,9 +514,12 @@ export default function ModelsPage() {
|
|
| 268 |
</>
|
| 269 |
) : (
|
| 270 |
<>
|
|
|
|
|
|
|
|
|
|
|
|
|
| 271 |
<SelectItem value="date">Latest First</SelectItem>
|
| 272 |
<SelectItem value="name">Name (A-Z)</SelectItem>
|
| 273 |
-
<SelectItem value="benchmarks">Most Benchmark Coverage</SelectItem>
|
| 274 |
</>
|
| 275 |
)}
|
| 276 |
</SelectContent>
|
|
@@ -286,9 +535,11 @@ export default function ModelsPage() {
|
|
| 286 |
</p>
|
| 287 |
<Button
|
| 288 |
onClick={() => {
|
| 289 |
-
setModelSortBy("
|
| 290 |
setDeveloperSortBy("coverage")
|
| 291 |
setSearchQuery("")
|
|
|
|
|
|
|
| 292 |
}}
|
| 293 |
>
|
| 294 |
Reset Filters
|
|
@@ -309,6 +560,8 @@ export default function ModelsPage() {
|
|
| 309 |
key={evaluation.id}
|
| 310 |
data={evaluation}
|
| 311 |
onDelete={handleDelete}
|
|
|
|
|
|
|
| 312 |
delayMs={Math.min(index * 45, 240)}
|
| 313 |
/>
|
| 314 |
))}
|
|
@@ -322,7 +575,61 @@ export default function ModelsPage() {
|
|
| 322 |
itemLabel={groupByDeveloper ? "developers" : "models"}
|
| 323 |
onPageChange={setPage}
|
| 324 |
/>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 325 |
</main>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 326 |
</div>
|
| 327 |
)
|
| 328 |
}
|
|
|
|
| 5 |
import { Button } from "@/components/ui/button"
|
| 6 |
import { Input } from "@/components/ui/input"
|
| 7 |
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"
|
| 8 |
+
import { ArrowUpDown, ArrowRightLeft, Search, X } from "lucide-react"
|
| 9 |
import { BenchmarkEvaluationCard, type BenchmarkEvaluationCardData } from "@/components/benchmark-evaluation-card"
|
| 10 |
import { DeveloperCard } from "@/components/developer-card"
|
| 11 |
import { ListPagination } from "@/components/list-pagination"
|
| 12 |
+
import { ModelCompareDialog } from "@/components/model-compare-dialog"
|
| 13 |
import { Navigation } from "@/components/navigation"
|
| 14 |
import { PageHeader } from "@/components/page-header"
|
| 15 |
+
import { Badge } from "@/components/ui/badge"
|
| 16 |
import { fetchDevelopers, fetchModelCards, type DeveloperListItem } from "@/lib/dashboard-data-client"
|
| 17 |
|
| 18 |
const PAGE_SIZE = 40
|
| 19 |
+
const MAX_COMPARE_MODELS = 4
|
| 20 |
+
const PARAM_RANGE_VALUES = [1, 2, 3, 4, 6, 8, 10, 12, 16, 24, 32, 48, 64, 96, 128, 192, 256, 384, 500] as const
|
| 21 |
+
const PARAM_RANGE_MARKERS = [
|
| 22 |
+
{ label: "< 1B", step: 0 },
|
| 23 |
+
{ label: "6B", step: PARAM_RANGE_VALUES.indexOf(6) },
|
| 24 |
+
{ label: "12B", step: PARAM_RANGE_VALUES.indexOf(12) },
|
| 25 |
+
{ label: "32B", step: PARAM_RANGE_VALUES.indexOf(32) },
|
| 26 |
+
{ label: "128B", step: PARAM_RANGE_VALUES.indexOf(128) },
|
| 27 |
+
{ label: "> 500B", step: PARAM_RANGE_VALUES.length - 1 },
|
| 28 |
+
] as const
|
| 29 |
+
|
| 30 |
+
function formatParamBoundLabel(step: number, bound: "min" | "max") {
|
| 31 |
+
const maxStepIndex = PARAM_RANGE_VALUES.length - 1
|
| 32 |
+
|
| 33 |
+
if (bound === "min" && step <= 0) {
|
| 34 |
+
return "< 1B"
|
| 35 |
+
}
|
| 36 |
+
|
| 37 |
+
if (bound === "max" && step >= maxStepIndex) {
|
| 38 |
+
return "> 500B"
|
| 39 |
+
}
|
| 40 |
+
|
| 41 |
+
const value = PARAM_RANGE_VALUES[step]
|
| 42 |
+
return value != null ? `${value}B` : "Not reported"
|
| 43 |
+
}
|
| 44 |
+
|
| 45 |
+
function getReproducibilitySortValue(status: BenchmarkEvaluationCardData["reproducibility_status"]) {
|
| 46 |
+
switch (status) {
|
| 47 |
+
case "complete":
|
| 48 |
+
return 2
|
| 49 |
+
case "partial":
|
| 50 |
+
return 1
|
| 51 |
+
default:
|
| 52 |
+
return 0
|
| 53 |
+
}
|
| 54 |
+
}
|
| 55 |
|
| 56 |
export default function ModelsPage() {
|
| 57 |
const { mode } = useAudienceMode()
|
|
|
|
| 60 |
const [loadingModels, setLoadingModels] = useState(true)
|
| 61 |
const [loadingDevelopers, setLoadingDevelopers] = useState(true)
|
| 62 |
const [groupByDeveloper, setGroupByDeveloper] = useState(false)
|
| 63 |
+
const [modelSortBy, setModelSortBy] = useState<"date" | "name" | "benchmarks" | "reporting" | "reproducibility" | "size">("benchmarks")
|
| 64 |
const [developerSortBy, setDeveloperSortBy] = useState<"coverage" | "evaluated" | "models" | "name">("coverage")
|
| 65 |
const [searchQuery, setSearchQuery] = useState("")
|
| 66 |
+
const [minParamStep, setMinParamStep] = useState(0)
|
| 67 |
+
const [maxParamStep, setMaxParamStep] = useState(PARAM_RANGE_VALUES.length - 1)
|
| 68 |
+
const [selectedModelIds, setSelectedModelIds] = useState<string[]>([])
|
| 69 |
+
const [compareOpen, setCompareOpen] = useState(false)
|
| 70 |
const [page, setPage] = useState(1)
|
| 71 |
|
| 72 |
useEffect(() => {
|
|
|
|
| 94 |
setGroupByDeveloper(params.get("group") === "developer")
|
| 95 |
}, [])
|
| 96 |
|
| 97 |
+
useEffect(() => {
|
| 98 |
+
setSelectedModelIds((current) =>
|
| 99 |
+
current.filter((id) => evaluations.some((evaluation) => evaluation.id === id))
|
| 100 |
+
)
|
| 101 |
+
}, [evaluations])
|
| 102 |
|
| 103 |
+
const numericMinParams = useMemo(() => {
|
| 104 |
+
if (minParamStep <= 0) {
|
| 105 |
+
return null
|
| 106 |
}
|
| 107 |
|
| 108 |
+
return PARAM_RANGE_VALUES[minParamStep] ?? null
|
| 109 |
+
}, [minParamStep])
|
| 110 |
+
|
| 111 |
+
const numericMaxParams = useMemo(() => {
|
| 112 |
+
if (maxParamStep >= PARAM_RANGE_VALUES.length - 1) {
|
| 113 |
+
return null
|
| 114 |
+
}
|
| 115 |
+
|
| 116 |
+
return PARAM_RANGE_VALUES[maxParamStep] ?? null
|
| 117 |
+
}, [maxParamStep])
|
| 118 |
+
|
| 119 |
+
const filteredEvaluations = useMemo(() => {
|
| 120 |
+
const query = searchQuery.trim().toLowerCase()
|
| 121 |
+
|
| 122 |
return evaluations.filter((evaluation) => {
|
| 123 |
+
if (numericMinParams != null) {
|
| 124 |
+
if (evaluation.params_billions == null || evaluation.params_billions < numericMinParams) {
|
| 125 |
+
return false
|
| 126 |
+
}
|
| 127 |
+
}
|
| 128 |
+
|
| 129 |
+
if (numericMaxParams != null) {
|
| 130 |
+
if (evaluation.params_billions == null || evaluation.params_billions > numericMaxParams) {
|
| 131 |
+
return false
|
| 132 |
+
}
|
| 133 |
+
}
|
| 134 |
+
|
| 135 |
+
if (!query) {
|
| 136 |
+
return true
|
| 137 |
+
}
|
| 138 |
+
|
| 139 |
const haystacks = [
|
| 140 |
evaluation.model_name,
|
| 141 |
evaluation.canonical_model_name,
|
| 142 |
evaluation.developer,
|
| 143 |
evaluation.architecture,
|
| 144 |
evaluation.latest_source_name,
|
| 145 |
+
evaluation.reproducibility_status,
|
| 146 |
...evaluation.evaluator_names,
|
| 147 |
...evaluation.top_scores.map((score) => score.benchmark),
|
| 148 |
]
|
| 149 |
|
| 150 |
return haystacks.some((value) => value?.toLowerCase().includes(query))
|
| 151 |
})
|
| 152 |
+
}, [evaluations, numericMaxParams, numericMinParams, searchQuery])
|
| 153 |
|
| 154 |
const sortedEvaluations = useMemo(() => {
|
| 155 |
const sorted = [...filteredEvaluations]
|
|
|
|
| 166 |
case "benchmarks":
|
| 167 |
sorted.sort((a, b) => b.benchmarks_count - a.benchmarks_count)
|
| 168 |
break
|
| 169 |
+
case "reporting":
|
| 170 |
+
sorted.sort((a, b) => {
|
| 171 |
+
if (b.evaluator_count !== a.evaluator_count) {
|
| 172 |
+
return b.evaluator_count - a.evaluator_count
|
| 173 |
+
}
|
| 174 |
+
if (b.independent_verification_ratio !== a.independent_verification_ratio) {
|
| 175 |
+
return b.independent_verification_ratio - a.independent_verification_ratio
|
| 176 |
+
}
|
| 177 |
+
return b.benchmarks_count - a.benchmarks_count
|
| 178 |
+
})
|
| 179 |
+
break
|
| 180 |
+
case "reproducibility":
|
| 181 |
+
sorted.sort((a, b) => {
|
| 182 |
+
const reproducibilityDiff =
|
| 183 |
+
getReproducibilitySortValue(b.reproducibility_status) -
|
| 184 |
+
getReproducibilitySortValue(a.reproducibility_status)
|
| 185 |
+
if (reproducibilityDiff !== 0) {
|
| 186 |
+
return reproducibilityDiff
|
| 187 |
+
}
|
| 188 |
+
if (b.independent_verification_ratio !== a.independent_verification_ratio) {
|
| 189 |
+
return b.independent_verification_ratio - a.independent_verification_ratio
|
| 190 |
+
}
|
| 191 |
+
return b.benchmarks_count - a.benchmarks_count
|
| 192 |
+
})
|
| 193 |
+
break
|
| 194 |
+
case "size":
|
| 195 |
+
sorted.sort((a, b) => (b.params_billions ?? -1) - (a.params_billions ?? -1))
|
| 196 |
+
break
|
| 197 |
}
|
| 198 |
|
| 199 |
return sorted
|
|
|
|
| 247 |
|
| 248 |
useEffect(() => {
|
| 249 |
setPage(1)
|
| 250 |
+
}, [developerSortBy, groupByDeveloper, maxParamStep, minParamStep, modelSortBy, searchQuery])
|
| 251 |
|
| 252 |
const pagedEvaluations = useMemo(
|
| 253 |
() => sortedEvaluations.slice((page - 1) * PAGE_SIZE, page * PAGE_SIZE),
|
|
|
|
| 261 |
|
| 262 |
const handleDelete = (id: string) => {
|
| 263 |
setEvaluations((prev) => prev.filter((e) => e.id !== id))
|
| 264 |
+
setSelectedModelIds((prev) => prev.filter((selectedId) => selectedId !== id))
|
| 265 |
+
}
|
| 266 |
+
|
| 267 |
+
const selectedModels = useMemo(
|
| 268 |
+
() =>
|
| 269 |
+
selectedModelIds
|
| 270 |
+
.map((id) => evaluations.find((evaluation) => evaluation.id === id))
|
| 271 |
+
.filter((evaluation): evaluation is BenchmarkEvaluationCardData => Boolean(evaluation)),
|
| 272 |
+
[evaluations, selectedModelIds]
|
| 273 |
+
)
|
| 274 |
+
|
| 275 |
+
useEffect(() => {
|
| 276 |
+
if (compareOpen && selectedModels.length < 2) {
|
| 277 |
+
setCompareOpen(false)
|
| 278 |
+
}
|
| 279 |
+
}, [compareOpen, selectedModels.length])
|
| 280 |
+
|
| 281 |
+
const toggleModelSelection = (id: string) => {
|
| 282 |
+
setSelectedModelIds((current) => {
|
| 283 |
+
if (current.includes(id)) {
|
| 284 |
+
return current.filter((selectedId) => selectedId !== id)
|
| 285 |
+
}
|
| 286 |
+
|
| 287 |
+
if (current.length >= MAX_COMPARE_MODELS) {
|
| 288 |
+
return current
|
| 289 |
+
}
|
| 290 |
+
|
| 291 |
+
return [...current, id]
|
| 292 |
+
})
|
| 293 |
}
|
| 294 |
|
| 295 |
const loading = loadingModels || loadingDevelopers
|
| 296 |
+
const maxParamStepIndex = PARAM_RANGE_VALUES.length - 1
|
| 297 |
+
const minHandlePercent = (minParamStep / maxParamStepIndex) * 100
|
| 298 |
+
const maxHandlePercent = (maxParamStep / maxParamStepIndex) * 100
|
| 299 |
|
| 300 |
if (loading) {
|
| 301 |
return (
|
|
|
|
| 321 |
groupByDeveloper
|
| 322 |
? "Group the model corpus by developer to compare how many models each team ships and which eval suites show up most often."
|
| 323 |
: mode === "research"
|
| 324 |
+
? "Browse model cards with benchmark breadth, comparison-ready context, and methodological trust signals."
|
| 325 |
+
: "Browse model cards with stronger emphasis on reporting breadth, evidence quality, and evaluation accountability."
|
| 326 |
}
|
| 327 |
metaItems={[
|
| 328 |
groupByDeveloper
|
|
|
|
| 340 |
value: filteredDevelopers.reduce((sum, developer) => sum + developer.benchmark_count, 0).toString(),
|
| 341 |
}
|
| 342 |
: { label: "Reporting orgs", value: new Set(sortedEvaluations.flatMap((e) => e.evaluator_names)).size.toString() },
|
| 343 |
+
!groupByDeveloper
|
| 344 |
+
? { label: "Compare tray", value: selectedModels.length.toString() }
|
| 345 |
+
: { label: "View", value: "Developer" },
|
| 346 |
]}
|
| 347 |
/>
|
| 348 |
|
| 349 |
+
{!groupByDeveloper ? (
|
| 350 |
+
<div className="mb-6 rounded-[1.5rem] border border-border/70 bg-muted/10 px-5 py-4">
|
| 351 |
+
<div className="flex flex-col gap-3 lg:flex-row lg:items-center lg:justify-between">
|
| 352 |
+
<div>
|
| 353 |
+
<div className="flex items-center gap-2 text-[11px] font-semibold uppercase tracking-[0.22em] text-muted-foreground">
|
| 354 |
+
<ArrowRightLeft className="h-3.5 w-3.5" />
|
| 355 |
+
Compare Workflow
|
| 356 |
+
</div>
|
| 357 |
+
<div className="mt-2 text-sm text-muted-foreground">
|
| 358 |
+
Show the most useful information first: narrow to a similar parameter range, scan key benchmarks, then select up to {MAX_COMPARE_MODELS} models for a table comparison.
|
| 359 |
+
</div>
|
| 360 |
+
</div>
|
| 361 |
+
<div className="flex flex-wrap gap-2">
|
| 362 |
+
<Badge variant="outline">Parameter range filter</Badge>
|
| 363 |
+
<Badge variant="outline">Table comparison</Badge>
|
| 364 |
+
<Badge variant="outline">Trust signals first</Badge>
|
| 365 |
+
</div>
|
| 366 |
+
</div>
|
| 367 |
+
</div>
|
| 368 |
+
) : null}
|
| 369 |
+
|
| 370 |
<div className="mb-8 flex flex-col gap-4 border-b border-border/50 pb-6 sm:flex-row sm:flex-wrap sm:items-center">
|
| 371 |
<div className="relative w-full sm:max-w-sm">
|
| 372 |
<Search className="pointer-events-none absolute left-3 top-1/2 h-4 w-4 -translate-y-1/2 text-muted-foreground" />
|
|
|
|
| 381 |
className="pl-9"
|
| 382 |
/>
|
| 383 |
</div>
|
| 384 |
+
{!groupByDeveloper ? (
|
| 385 |
+
<div className="rounded-xl border border-border/70 bg-muted/15 px-4 py-2">
|
| 386 |
+
<div className="flex items-center gap-3">
|
| 387 |
+
<span className="shrink-0 text-sm font-medium text-foreground">Parameters</span>
|
| 388 |
+
|
| 389 |
+
<div className="min-w-0 flex-1 w-[min(92vw,360px)]">
|
| 390 |
+
<div className="relative mb-1 h-4 text-[11px] text-muted-foreground">
|
| 391 |
+
{PARAM_RANGE_MARKERS.map((marker) => (
|
| 392 |
+
<span
|
| 393 |
+
key={marker.label}
|
| 394 |
+
className="absolute top-0 whitespace-nowrap"
|
| 395 |
+
style={{
|
| 396 |
+
left: `${(marker.step / maxParamStepIndex) * 100}%`,
|
| 397 |
+
transform:
|
| 398 |
+
marker.step === 0
|
| 399 |
+
? "translateX(0)"
|
| 400 |
+
: marker.step === maxParamStepIndex
|
| 401 |
+
? "translateX(-100%)"
|
| 402 |
+
: "translateX(-50%)",
|
| 403 |
+
}}
|
| 404 |
+
>
|
| 405 |
+
{marker.label}
|
| 406 |
+
</span>
|
| 407 |
+
))}
|
| 408 |
+
</div>
|
| 409 |
+
|
| 410 |
+
<div className="relative h-4">
|
| 411 |
+
<div className="absolute inset-x-1.5 top-1/2 h-[3px] -translate-y-1/2 rounded-full bg-border/80" />
|
| 412 |
+
<div className="absolute inset-x-1.5 top-1/2 h-[3px] -translate-y-1/2">
|
| 413 |
+
<div
|
| 414 |
+
className="absolute inset-y-0 rounded-full bg-foreground transition-[left,right] duration-300 ease-[var(--ease-out-quint)]"
|
| 415 |
+
style={{
|
| 416 |
+
left: `${minHandlePercent}%`,
|
| 417 |
+
right: `${Math.max(100 - maxHandlePercent, 0)}%`,
|
| 418 |
+
}}
|
| 419 |
+
/>
|
| 420 |
+
</div>
|
| 421 |
+
|
| 422 |
+
<div className="absolute inset-x-1.5 top-1/2 -translate-y-1/2">
|
| 423 |
+
{PARAM_RANGE_VALUES.map((_, stepIndex) => (
|
| 424 |
+
<span
|
| 425 |
+
key={`param-tick-${stepIndex}`}
|
| 426 |
+
className="absolute top-0 h-2 w-px -translate-x-1/2 rounded-full bg-border"
|
| 427 |
+
style={{ left: `${(stepIndex / maxParamStepIndex) * 100}%` }}
|
| 428 |
+
aria-hidden="true"
|
| 429 |
+
/>
|
| 430 |
+
))}
|
| 431 |
+
</div>
|
| 432 |
+
|
| 433 |
+
<input
|
| 434 |
+
type="range"
|
| 435 |
+
min={0}
|
| 436 |
+
max={maxParamStepIndex}
|
| 437 |
+
step={1}
|
| 438 |
+
value={minParamStep}
|
| 439 |
+
onChange={(event) => {
|
| 440 |
+
const nextMin = Number(event.target.value)
|
| 441 |
+
setMinParamStep(Math.min(nextMin, maxParamStep))
|
| 442 |
+
}}
|
| 443 |
+
className="param-range-input"
|
| 444 |
+
aria-label="Minimum parameter filter"
|
| 445 |
+
/>
|
| 446 |
+
|
| 447 |
+
<input
|
| 448 |
+
type="range"
|
| 449 |
+
min={0}
|
| 450 |
+
max={maxParamStepIndex}
|
| 451 |
+
step={1}
|
| 452 |
+
value={maxParamStep}
|
| 453 |
+
onChange={(event) => {
|
| 454 |
+
const nextMax = Number(event.target.value)
|
| 455 |
+
setMaxParamStep(Math.max(nextMax, minParamStep))
|
| 456 |
+
}}
|
| 457 |
+
className="param-range-input"
|
| 458 |
+
aria-label="Maximum parameter filter"
|
| 459 |
+
/>
|
| 460 |
+
</div>
|
| 461 |
+
</div>
|
| 462 |
+
|
| 463 |
+
<span className="shrink-0 text-[11px] text-muted-foreground">
|
| 464 |
+
{formatParamBoundLabel(minParamStep, "min")} to {formatParamBoundLabel(maxParamStep, "max")}
|
| 465 |
+
</span>
|
| 466 |
+
</div>
|
| 467 |
+
</div>
|
| 468 |
+
) : null}
|
| 469 |
<div className="inline-flex w-fit rounded-full border bg-muted/20 p-1">
|
| 470 |
<button
|
| 471 |
type="button"
|
|
|
|
| 514 |
</>
|
| 515 |
) : (
|
| 516 |
<>
|
| 517 |
+
<SelectItem value="benchmarks">Most Benchmark Coverage</SelectItem>
|
| 518 |
+
<SelectItem value="reporting">Most Reporting Context</SelectItem>
|
| 519 |
+
<SelectItem value="reproducibility">Best Reproducibility</SelectItem>
|
| 520 |
+
<SelectItem value="size">Largest Models</SelectItem>
|
| 521 |
<SelectItem value="date">Latest First</SelectItem>
|
| 522 |
<SelectItem value="name">Name (A-Z)</SelectItem>
|
|
|
|
| 523 |
</>
|
| 524 |
)}
|
| 525 |
</SelectContent>
|
|
|
|
| 535 |
</p>
|
| 536 |
<Button
|
| 537 |
onClick={() => {
|
| 538 |
+
setModelSortBy("benchmarks")
|
| 539 |
setDeveloperSortBy("coverage")
|
| 540 |
setSearchQuery("")
|
| 541 |
+
setMinParamStep(0)
|
| 542 |
+
setMaxParamStep(PARAM_RANGE_VALUES.length - 1)
|
| 543 |
}}
|
| 544 |
>
|
| 545 |
Reset Filters
|
|
|
|
| 560 |
key={evaluation.id}
|
| 561 |
data={evaluation}
|
| 562 |
onDelete={handleDelete}
|
| 563 |
+
selectedForCompare={selectedModelIds.includes(evaluation.id)}
|
| 564 |
+
onToggleCompare={toggleModelSelection}
|
| 565 |
delayMs={Math.min(index * 45, 240)}
|
| 566 |
/>
|
| 567 |
))}
|
|
|
|
| 575 |
itemLabel={groupByDeveloper ? "developers" : "models"}
|
| 576 |
onPageChange={setPage}
|
| 577 |
/>
|
| 578 |
+
|
| 579 |
+
{!groupByDeveloper && selectedModels.length > 0 ? (
|
| 580 |
+
<div className="pointer-events-none fixed inset-x-0 bottom-4 z-40 flex justify-center px-4">
|
| 581 |
+
<div className="pointer-events-auto w-full max-w-5xl rounded-[1.5rem] border border-border/80 bg-background/95 p-4 shadow-2xl backdrop-blur">
|
| 582 |
+
<div className="flex flex-col gap-3 lg:flex-row lg:items-center lg:justify-between">
|
| 583 |
+
<div className="space-y-2">
|
| 584 |
+
<div className="text-[11px] font-semibold uppercase tracking-[0.22em] text-muted-foreground">
|
| 585 |
+
Compare Tray
|
| 586 |
+
</div>
|
| 587 |
+
<div className="flex flex-wrap gap-2">
|
| 588 |
+
{selectedModels.map((model) => (
|
| 589 |
+
<span
|
| 590 |
+
key={model.id}
|
| 591 |
+
className="inline-flex items-center gap-2 rounded-full border border-border/70 bg-muted/20 px-3 py-1.5 text-sm"
|
| 592 |
+
>
|
| 593 |
+
<span className="font-medium">{model.model_name}</span>
|
| 594 |
+
<button
|
| 595 |
+
type="button"
|
| 596 |
+
onClick={() => toggleModelSelection(model.id)}
|
| 597 |
+
className="text-muted-foreground transition-colors hover:text-foreground"
|
| 598 |
+
aria-label={`Remove ${model.model_name} from compare`}
|
| 599 |
+
>
|
| 600 |
+
<X className="h-3.5 w-3.5" />
|
| 601 |
+
</button>
|
| 602 |
+
</span>
|
| 603 |
+
))}
|
| 604 |
+
</div>
|
| 605 |
+
<div className="text-sm text-muted-foreground">
|
| 606 |
+
Select up to {MAX_COMPARE_MODELS} models. The compare view is most useful when you keep the parameter range tight.
|
| 607 |
+
</div>
|
| 608 |
+
</div>
|
| 609 |
+
|
| 610 |
+
<div className="flex flex-wrap items-center gap-2">
|
| 611 |
+
<Button variant="ghost" onClick={() => setSelectedModelIds([])}>
|
| 612 |
+
Clear
|
| 613 |
+
</Button>
|
| 614 |
+
<Button
|
| 615 |
+
onClick={() => setCompareOpen(true)}
|
| 616 |
+
disabled={selectedModels.length < 2}
|
| 617 |
+
>
|
| 618 |
+
<ArrowRightLeft className="h-4 w-4" />
|
| 619 |
+
Compare {selectedModels.length} model{selectedModels.length !== 1 ? "s" : ""}
|
| 620 |
+
</Button>
|
| 621 |
+
</div>
|
| 622 |
+
</div>
|
| 623 |
+
</div>
|
| 624 |
+
</div>
|
| 625 |
+
) : null}
|
| 626 |
</main>
|
| 627 |
+
|
| 628 |
+
<ModelCompareDialog
|
| 629 |
+
models={selectedModels}
|
| 630 |
+
open={compareOpen}
|
| 631 |
+
onOpenChange={setCompareOpen}
|
| 632 |
+
/>
|
| 633 |
</div>
|
| 634 |
)
|
| 635 |
}
|
components/benchmark-evaluation-card.tsx
CHANGED
|
@@ -6,18 +6,22 @@ import { useRouter } from "next/navigation"
|
|
| 6 |
import {
|
| 7 |
Award,
|
| 8 |
BookOpenText,
|
|
|
|
|
|
|
| 9 |
ExternalLink,
|
| 10 |
Eye,
|
| 11 |
FlaskConical,
|
| 12 |
LibraryBig,
|
| 13 |
MoreHorizontal,
|
| 14 |
-
|
|
|
|
| 15 |
} from "lucide-react"
|
| 16 |
|
| 17 |
import type { CategoryType } from "@/lib/benchmark-schema"
|
| 18 |
import { Badge } from "@/components/ui/badge"
|
| 19 |
import { Button } from "@/components/ui/button"
|
| 20 |
import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"
|
|
|
|
| 21 |
import { DropdownMenu, DropdownMenuContent, DropdownMenuItem, DropdownMenuTrigger } from "@/components/ui/dropdown-menu"
|
| 22 |
import { Progress } from "@/components/ui/progress"
|
| 23 |
|
|
@@ -74,6 +78,8 @@ interface BenchmarkEvaluationCardProps {
|
|
| 74 |
data: BenchmarkEvaluationCardData
|
| 75 |
onDelete?: (id: string) => void
|
| 76 |
delayMs?: number
|
|
|
|
|
|
|
| 77 |
}
|
| 78 |
|
| 79 |
function formatDate(isoString: string) {
|
|
@@ -138,7 +144,50 @@ function getReportingSummaryLabel(data: BenchmarkEvaluationCardData) {
|
|
| 138 |
return "Aggregated reporting view"
|
| 139 |
}
|
| 140 |
|
| 141 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 142 |
const router = useRouter()
|
| 143 |
const { mode } = useAudienceMode()
|
| 144 |
const isResearchView = mode === "research"
|
|
@@ -146,6 +195,8 @@ export function BenchmarkEvaluationCard({ data, onDelete, delayMs = 0 }: Benchma
|
|
| 146 |
const library = data.eval_libraries[0]
|
| 147 |
const paramsBillions = formatParamsBillions(data.params_billions)
|
| 148 |
const reportingSummaryLabel = getReportingSummaryLabel(data)
|
|
|
|
|
|
|
| 149 |
|
| 150 |
return (
|
| 151 |
<Card
|
|
@@ -179,46 +230,63 @@ export function BenchmarkEvaluationCard({ data, onDelete, delayMs = 0 }: Benchma
|
|
| 179 |
<Badge variant="secondary">{data.variant_count} versions</Badge>
|
| 180 |
)}
|
| 181 |
{paramsBillions && <Badge variant="secondary">{paramsBillions} parameters</Badge>}
|
| 182 |
-
|
| 183 |
-
|
| 184 |
-
|
| 185 |
-
|
| 186 |
-
{data.
|
| 187 |
-
<
|
| 188 |
-
|
|
|
|
| 189 |
</div>
|
| 190 |
</div>
|
| 191 |
|
| 192 |
-
<
|
| 193 |
-
|
| 194 |
<Button
|
| 195 |
-
variant="
|
| 196 |
-
size="
|
| 197 |
-
className="
|
| 198 |
-
onClick={(event) =>
|
|
|
|
|
|
|
|
|
|
| 199 |
>
|
| 200 |
-
|
| 201 |
</Button>
|
| 202 |
-
|
| 203 |
-
|
| 204 |
-
|
| 205 |
-
|
| 206 |
-
|
| 207 |
-
|
| 208 |
-
|
| 209 |
-
|
| 210 |
-
|
| 211 |
-
|
| 212 |
-
|
| 213 |
-
|
| 214 |
-
|
| 215 |
-
|
| 216 |
-
|
| 217 |
-
|
|
|
|
| 218 |
</DropdownMenuItem>
|
| 219 |
-
|
| 220 |
-
|
| 221 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 222 |
</div>
|
| 223 |
</CardHeader>
|
| 224 |
|
|
@@ -241,105 +309,103 @@ export function BenchmarkEvaluationCard({ data, onDelete, delayMs = 0 }: Benchma
|
|
| 241 |
/>
|
| 242 |
</div>
|
| 243 |
|
| 244 |
-
|
| 245 |
-
<div className="
|
| 246 |
-
<
|
| 247 |
-
|
| 248 |
-
<
|
| 249 |
-
|
| 250 |
-
|
| 251 |
-
|
| 252 |
-
|
| 253 |
-
|
| 254 |
-
|
| 255 |
-
|
| 256 |
-
|
| 257 |
-
|
| 258 |
-
|
| 259 |
-
|
| 260 |
-
|
| 261 |
-
|
| 262 |
-
<KeyValueRow label="Missing config" value={`${data.missing_generation_config_count} result${data.missing_generation_config_count !== 1 ? "s" : ""}`} />
|
| 263 |
-
)}
|
| 264 |
-
{library?.fork && (
|
| 265 |
-
<div className="flex items-start gap-2 rounded-xl bg-amber-50/80 px-3 py-2 text-xs text-amber-900 dark:bg-amber-950/20 dark:text-amber-200">
|
| 266 |
-
<LibraryBig className="mt-0.5 h-3.5 w-3.5 shrink-0 text-amber-600 dark:text-amber-300" />
|
| 267 |
-
<span>Non-standard eval library fork</span>
|
| 268 |
-
</div>
|
| 269 |
-
)}
|
| 270 |
-
</div>
|
| 271 |
-
</div>
|
| 272 |
-
</section>
|
| 273 |
|
| 274 |
-
|
| 275 |
-
|
| 276 |
-
|
| 277 |
-
|
| 278 |
-
|
| 279 |
-
|
| 280 |
-
<
|
| 281 |
-
|
| 282 |
-
|
| 283 |
-
|
| 284 |
-
|
| 285 |
-
|
| 286 |
-
|
| 287 |
-
|
| 288 |
-
|
| 289 |
-
|
| 290 |
-
|
| 291 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 292 |
</div>
|
| 293 |
-
|
| 294 |
-
|
| 295 |
-
</div>
|
| 296 |
-
) : (
|
| 297 |
-
<div className="grid gap-4 md:grid-cols-[0.92fr_1.08fr]">
|
| 298 |
-
<section className="space-y-3">
|
| 299 |
-
<div className="text-[11px] font-semibold uppercase tracking-[0.18em] text-muted-foreground">Reporting Context</div>
|
| 300 |
-
<div className="rounded-2xl border border-amber-200/70 bg-amber-50/70 p-3 dark:border-amber-900/40 dark:bg-amber-950/15">
|
| 301 |
-
<div className="flex items-start gap-2">
|
| 302 |
-
<Scale className="mt-0.5 h-4 w-4 shrink-0 text-amber-600" />
|
| 303 |
-
<div className="space-y-2">
|
| 304 |
-
<div className="text-sm font-semibold">Reporting summary</div>
|
| 305 |
-
<div className="text-sm text-muted-foreground">
|
| 306 |
-
This model has reported results from {reportingSummaryLabel.toLowerCase()} across {data.benchmarks_count} benchmark{data.benchmarks_count !== 1 ? "s" : ""}. Benchmark detail pages break out reporting provenance benchmark by benchmark.
|
| 307 |
-
</div>
|
| 308 |
-
<div className="flex flex-wrap gap-2 pt-1">
|
| 309 |
-
{data.evaluator_names.slice(0, 2).map((evaluator) => (
|
| 310 |
-
<Badge key={evaluator} variant="secondary" className="font-normal">
|
| 311 |
-
{evaluator}
|
| 312 |
-
</Badge>
|
| 313 |
-
))}
|
| 314 |
-
</div>
|
| 315 |
-
</div>
|
| 316 |
</div>
|
| 317 |
</div>
|
| 318 |
-
|
| 319 |
-
|
| 320 |
-
|
| 321 |
-
|
| 322 |
-
|
| 323 |
-
|
| 324 |
-
|
| 325 |
-
</
|
| 326 |
-
|
| 327 |
-
|
| 328 |
-
|
| 329 |
-
|
| 330 |
-
|
| 331 |
-
|
| 332 |
-
|
| 333 |
-
|
| 334 |
-
|
| 335 |
-
|
| 336 |
-
|
| 337 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 338 |
</div>
|
| 339 |
-
|
| 340 |
-
|
| 341 |
-
</
|
| 342 |
-
|
| 343 |
</CardContent>
|
| 344 |
</Card>
|
| 345 |
)
|
|
|
|
| 6 |
import {
|
| 7 |
Award,
|
| 8 |
BookOpenText,
|
| 9 |
+
ChevronDown,
|
| 10 |
+
CheckCircle2,
|
| 11 |
ExternalLink,
|
| 12 |
Eye,
|
| 13 |
FlaskConical,
|
| 14 |
LibraryBig,
|
| 15 |
MoreHorizontal,
|
| 16 |
+
ShieldCheck,
|
| 17 |
+
TriangleAlert,
|
| 18 |
} from "lucide-react"
|
| 19 |
|
| 20 |
import type { CategoryType } from "@/lib/benchmark-schema"
|
| 21 |
import { Badge } from "@/components/ui/badge"
|
| 22 |
import { Button } from "@/components/ui/button"
|
| 23 |
import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"
|
| 24 |
+
import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "@/components/ui/collapsible"
|
| 25 |
import { DropdownMenu, DropdownMenuContent, DropdownMenuItem, DropdownMenuTrigger } from "@/components/ui/dropdown-menu"
|
| 26 |
import { Progress } from "@/components/ui/progress"
|
| 27 |
|
|
|
|
| 78 |
data: BenchmarkEvaluationCardData
|
| 79 |
onDelete?: (id: string) => void
|
| 80 |
delayMs?: number
|
| 81 |
+
selectedForCompare?: boolean
|
| 82 |
+
onToggleCompare?: (id: string) => void
|
| 83 |
}
|
| 84 |
|
| 85 |
function formatDate(isoString: string) {
|
|
|
|
| 144 |
return "Aggregated reporting view"
|
| 145 |
}
|
| 146 |
|
| 147 |
+
function getReproducibilitySummary(data: BenchmarkEvaluationCardData) {
|
| 148 |
+
switch (data.reproducibility_status) {
|
| 149 |
+
case "complete":
|
| 150 |
+
return {
|
| 151 |
+
label: "Full config coverage",
|
| 152 |
+
tone: "secondary" as const,
|
| 153 |
+
icon: CheckCircle2,
|
| 154 |
+
}
|
| 155 |
+
case "partial":
|
| 156 |
+
return {
|
| 157 |
+
label: "Partial config coverage",
|
| 158 |
+
tone: "outline" as const,
|
| 159 |
+
icon: FlaskConical,
|
| 160 |
+
}
|
| 161 |
+
default:
|
| 162 |
+
return {
|
| 163 |
+
label: "Config mostly missing",
|
| 164 |
+
tone: "destructive" as const,
|
| 165 |
+
icon: TriangleAlert,
|
| 166 |
+
}
|
| 167 |
+
}
|
| 168 |
+
}
|
| 169 |
+
|
| 170 |
+
function getIndependentSummary(data: BenchmarkEvaluationCardData) {
|
| 171 |
+
const percent = Math.round(data.independent_verification_ratio * 100)
|
| 172 |
+
|
| 173 |
+
if (data.independent_verification_ratio >= 0.75) {
|
| 174 |
+
return `${percent}% independent`
|
| 175 |
+
}
|
| 176 |
+
|
| 177 |
+
if (data.independent_verification_ratio > 0) {
|
| 178 |
+
return `${percent}% independent`
|
| 179 |
+
}
|
| 180 |
+
|
| 181 |
+
return "Self-reported only"
|
| 182 |
+
}
|
| 183 |
+
|
| 184 |
+
export function BenchmarkEvaluationCard({
|
| 185 |
+
data,
|
| 186 |
+
onDelete,
|
| 187 |
+
delayMs = 0,
|
| 188 |
+
selectedForCompare = false,
|
| 189 |
+
onToggleCompare,
|
| 190 |
+
}: BenchmarkEvaluationCardProps) {
|
| 191 |
const router = useRouter()
|
| 192 |
const { mode } = useAudienceMode()
|
| 193 |
const isResearchView = mode === "research"
|
|
|
|
| 195 |
const library = data.eval_libraries[0]
|
| 196 |
const paramsBillions = formatParamsBillions(data.params_billions)
|
| 197 |
const reportingSummaryLabel = getReportingSummaryLabel(data)
|
| 198 |
+
const reproducibility = getReproducibilitySummary(data)
|
| 199 |
+
const independentSummary = getIndependentSummary(data)
|
| 200 |
|
| 201 |
return (
|
| 202 |
<Card
|
|
|
|
| 230 |
<Badge variant="secondary">{data.variant_count} versions</Badge>
|
| 231 |
)}
|
| 232 |
{paramsBillions && <Badge variant="secondary">{paramsBillions} parameters</Badge>}
|
| 233 |
+
<Badge variant={reproducibility.tone}>
|
| 234 |
+
<reproducibility.icon className="h-3.5 w-3.5" />
|
| 235 |
+
{reproducibility.label}
|
| 236 |
+
</Badge>
|
| 237 |
+
<Badge variant={data.independent_verification_ratio > 0 ? "secondary" : "outline"}>
|
| 238 |
+
<ShieldCheck className="h-3.5 w-3.5" />
|
| 239 |
+
{independentSummary}
|
| 240 |
+
</Badge>
|
| 241 |
</div>
|
| 242 |
</div>
|
| 243 |
|
| 244 |
+
<div className="flex items-center gap-2">
|
| 245 |
+
{onToggleCompare ? (
|
| 246 |
<Button
|
| 247 |
+
variant={selectedForCompare ? "default" : "outline"}
|
| 248 |
+
size="sm"
|
| 249 |
+
className="shrink-0"
|
| 250 |
+
onClick={(event) => {
|
| 251 |
+
event.stopPropagation()
|
| 252 |
+
onToggleCompare(data.id)
|
| 253 |
+
}}
|
| 254 |
>
|
| 255 |
+
{selectedForCompare ? "Selected" : "Compare"}
|
| 256 |
</Button>
|
| 257 |
+
) : null}
|
| 258 |
+
|
| 259 |
+
<DropdownMenu>
|
| 260 |
+
<DropdownMenuTrigger asChild>
|
| 261 |
+
<Button
|
| 262 |
+
variant="ghost"
|
| 263 |
+
size="icon"
|
| 264 |
+
className="motion-academic-button opacity-0 transition-opacity group-hover:opacity-100"
|
| 265 |
+
onClick={(event) => event.stopPropagation()}
|
| 266 |
+
>
|
| 267 |
+
<MoreHorizontal className="h-4 w-4" />
|
| 268 |
+
</Button>
|
| 269 |
+
</DropdownMenuTrigger>
|
| 270 |
+
<DropdownMenuContent align="end">
|
| 271 |
+
<DropdownMenuItem onClick={() => router.push(`/models/${data.route_id}`)}>
|
| 272 |
+
<Eye className="mr-2 h-4 w-4" />
|
| 273 |
+
View Details
|
| 274 |
</DropdownMenuItem>
|
| 275 |
+
{data.source_urls.length > 0 && (
|
| 276 |
+
<DropdownMenuItem onClick={() => window.open(data.source_urls[0], "_blank")}>
|
| 277 |
+
<ExternalLink className="mr-2 h-4 w-4" />
|
| 278 |
+
View Source
|
| 279 |
+
</DropdownMenuItem>
|
| 280 |
+
)}
|
| 281 |
+
{onDelete && (
|
| 282 |
+
<DropdownMenuItem onClick={() => onDelete(data.id)} className="text-destructive">
|
| 283 |
+
<Award className="mr-2 h-4 w-4" />
|
| 284 |
+
Remove
|
| 285 |
+
</DropdownMenuItem>
|
| 286 |
+
)}
|
| 287 |
+
</DropdownMenuContent>
|
| 288 |
+
</DropdownMenu>
|
| 289 |
+
</div>
|
| 290 |
</div>
|
| 291 |
</CardHeader>
|
| 292 |
|
|
|
|
| 309 |
/>
|
| 310 |
</div>
|
| 311 |
|
| 312 |
+
<div className="rounded-2xl border border-border/70 bg-muted/10 px-4 py-3">
|
| 313 |
+
<div className="flex flex-wrap items-center gap-2">
|
| 314 |
+
<Badge variant="outline">{reportingSummaryLabel}</Badge>
|
| 315 |
+
<Badge variant={reproducibility.tone}>
|
| 316 |
+
<reproducibility.icon className="h-3.5 w-3.5" />
|
| 317 |
+
{reproducibility.label}
|
| 318 |
+
</Badge>
|
| 319 |
+
<Badge variant={data.independent_verification_ratio > 0 ? "secondary" : "outline"}>
|
| 320 |
+
<ShieldCheck className="h-3.5 w-3.5" />
|
| 321 |
+
{independentSummary}
|
| 322 |
+
</Badge>
|
| 323 |
+
</div>
|
| 324 |
+
<div className="mt-2 text-sm text-muted-foreground">
|
| 325 |
+
{isResearchView
|
| 326 |
+
? "Most useful signals first: benchmark coverage, reproducibility, and benchmark-level performance. Open the details panel only when you need methodology or provenance."
|
| 327 |
+
: "Most useful signals first: benchmark coverage, reporting posture, and what was actually tested. Open the details panel if you need source or methodology context."}
|
| 328 |
+
</div>
|
| 329 |
+
</div>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 330 |
|
| 331 |
+
{highlights.length > 0 ? (
|
| 332 |
+
<section className="space-y-3">
|
| 333 |
+
<div className="flex items-center gap-2 text-[11px] font-semibold uppercase tracking-[0.18em] text-muted-foreground">
|
| 334 |
+
{isResearchView ? (
|
| 335 |
+
<FlaskConical className="h-3.5 w-3.5" />
|
| 336 |
+
) : (
|
| 337 |
+
<BookOpenText className="h-3.5 w-3.5" />
|
| 338 |
+
)}
|
| 339 |
+
{isResearchView ? "Most Relevant Benchmarks" : "What Was Tested"}
|
| 340 |
+
</div>
|
| 341 |
+
<div className="overflow-hidden rounded-2xl border border-border/70">
|
| 342 |
+
{highlights.map((item, index) => (
|
| 343 |
+
<SignalRow
|
| 344 |
+
key={item.benchmark}
|
| 345 |
+
rank={index + 1}
|
| 346 |
+
label={isResearchView ? item.benchmark : getPolicyBenchmarkLabel(item.benchmark)}
|
| 347 |
+
rawLabel={
|
| 348 |
+
isResearchView
|
| 349 |
+
? item.metric !== item.benchmark
|
| 350 |
+
? item.metric
|
| 351 |
+
: undefined
|
| 352 |
+
: item.benchmark
|
| 353 |
+
}
|
| 354 |
+
scoreLabel={formatHighlightScore(item.score, item.unit)}
|
| 355 |
+
scorePercent={scoreToPercent(item.score, item.unit)}
|
| 356 |
+
isLast={index === highlights.length - 1}
|
| 357 |
+
/>
|
| 358 |
+
))}
|
| 359 |
+
</div>
|
| 360 |
+
</section>
|
| 361 |
+
) : null}
|
| 362 |
+
|
| 363 |
+
<Collapsible className="rounded-2xl border border-border/70 bg-background">
|
| 364 |
+
<CollapsibleTrigger asChild>
|
| 365 |
+
<button
|
| 366 |
+
type="button"
|
| 367 |
+
onClick={(event) => event.stopPropagation()}
|
| 368 |
+
className="flex w-full items-center justify-between px-4 py-3 text-left"
|
| 369 |
+
>
|
| 370 |
+
<div>
|
| 371 |
+
<div className="text-[11px] font-semibold uppercase tracking-[0.18em] text-muted-foreground">
|
| 372 |
+
Dive Deeper
|
| 373 |
</div>
|
| 374 |
+
<div className="mt-1 text-sm font-semibold text-foreground">
|
| 375 |
+
Show reporting and methodology details
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 376 |
</div>
|
| 377 |
</div>
|
| 378 |
+
<ChevronDown className="h-4 w-4 text-muted-foreground" />
|
| 379 |
+
</button>
|
| 380 |
+
</CollapsibleTrigger>
|
| 381 |
+
<CollapsibleContent onClick={(event) => event.stopPropagation()} className="border-t border-border/60 px-4 py-4">
|
| 382 |
+
<div className="space-y-0 text-sm">
|
| 383 |
+
<KeyValueRow label="Reporting sources" value={reportingSummaryLabel} />
|
| 384 |
+
{library && (
|
| 385 |
+
<KeyValueRow label="Library" value={`${library.name}${library.version ? ` ${library.version}` : ""}`} />
|
| 386 |
+
)}
|
| 387 |
+
{data.latest_source_name && (
|
| 388 |
+
<KeyValueRow label="Latest report" value={data.latest_source_name} />
|
| 389 |
+
)}
|
| 390 |
+
<KeyValueRow label="Updated" value={formatDate(data.latest_timestamp)} />
|
| 391 |
+
<KeyValueRow label="Reproducibility" value={reproducibility.label} />
|
| 392 |
+
<KeyValueRow label="Independence" value={independentSummary} />
|
| 393 |
+
{data.source_types.length > 0 && (
|
| 394 |
+
<KeyValueRow label="Source types" value={data.source_types.map((s) => s.replace(/_/g, " ")).join(", ")} />
|
| 395 |
+
)}
|
| 396 |
+
{data.architecture && <KeyValueRow label="Architecture" value={data.architecture} />}
|
| 397 |
+
{data.missing_generation_config_count > 0 && (
|
| 398 |
+
<KeyValueRow label="Missing config" value={`${data.missing_generation_config_count} result${data.missing_generation_config_count !== 1 ? "s" : ""}`} />
|
| 399 |
+
)}
|
| 400 |
+
{library?.fork && (
|
| 401 |
+
<div className="mt-3 flex items-start gap-2 rounded-xl bg-amber-50/80 px-3 py-2 text-xs text-amber-900 dark:bg-amber-950/20 dark:text-amber-200">
|
| 402 |
+
<LibraryBig className="mt-0.5 h-3.5 w-3.5 shrink-0 text-amber-600 dark:text-amber-300" />
|
| 403 |
+
<span>Non-standard eval library fork</span>
|
| 404 |
</div>
|
| 405 |
+
)}
|
| 406 |
+
</div>
|
| 407 |
+
</CollapsibleContent>
|
| 408 |
+
</Collapsible>
|
| 409 |
</CardContent>
|
| 410 |
</Card>
|
| 411 |
)
|
components/model-compare-dialog.tsx
ADDED
|
@@ -0,0 +1,459 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"use client"
|
| 2 |
+
|
| 3 |
+
import { Fragment, useMemo } from "react"
|
| 4 |
+
import Link from "next/link"
|
| 5 |
+
import {
|
| 6 |
+
ChevronDown,
|
| 7 |
+
ExternalLink,
|
| 8 |
+
FlaskConical,
|
| 9 |
+
ShieldCheck,
|
| 10 |
+
TriangleAlert,
|
| 11 |
+
} from "lucide-react"
|
| 12 |
+
import type { BenchmarkEvaluationCardData } from "@/components/benchmark-evaluation-card"
|
| 13 |
+
import { Badge } from "@/components/ui/badge"
|
| 14 |
+
import { Button } from "@/components/ui/button"
|
| 15 |
+
import {
|
| 16 |
+
Collapsible,
|
| 17 |
+
CollapsibleContent,
|
| 18 |
+
CollapsibleTrigger,
|
| 19 |
+
} from "@/components/ui/collapsible"
|
| 20 |
+
import {
|
| 21 |
+
Dialog,
|
| 22 |
+
DialogContent,
|
| 23 |
+
DialogDescription,
|
| 24 |
+
DialogHeader,
|
| 25 |
+
DialogTitle,
|
| 26 |
+
} from "@/components/ui/dialog"
|
| 27 |
+
import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table"
|
| 28 |
+
|
| 29 |
+
function formatDate(isoString: string) {
|
| 30 |
+
const numeric = Number(isoString)
|
| 31 |
+
const parsedDate =
|
| 32 |
+
!Number.isNaN(numeric) && !isoString.includes("-")
|
| 33 |
+
? new Date(numeric * 1000)
|
| 34 |
+
: new Date(isoString)
|
| 35 |
+
|
| 36 |
+
try {
|
| 37 |
+
return parsedDate.toLocaleDateString("en-US", {
|
| 38 |
+
year: "numeric",
|
| 39 |
+
month: "short",
|
| 40 |
+
day: "numeric",
|
| 41 |
+
})
|
| 42 |
+
} catch {
|
| 43 |
+
return isoString
|
| 44 |
+
}
|
| 45 |
+
}
|
| 46 |
+
|
| 47 |
+
function parseParamsBillionsFromModelName(modelName: string | null | undefined) {
|
| 48 |
+
if (!modelName) return null
|
| 49 |
+
|
| 50 |
+
// Parse explicit size tokens like 7B, 40B, or 560M from model names.
|
| 51 |
+
const sizeTokens = Array.from(modelName.matchAll(/\b(\d+(?:\.\d+)?)\s*([bm])\b/gi))
|
| 52 |
+
if (sizeTokens.length === 0) return null
|
| 53 |
+
|
| 54 |
+
const lastToken = sizeTokens[sizeTokens.length - 1]
|
| 55 |
+
const numericValue = Number(lastToken[1])
|
| 56 |
+
if (!Number.isFinite(numericValue)) return null
|
| 57 |
+
|
| 58 |
+
const unit = lastToken[2].toLowerCase()
|
| 59 |
+
if (unit === "b") return numericValue
|
| 60 |
+
if (unit === "m") return numericValue / 1000
|
| 61 |
+
|
| 62 |
+
return null
|
| 63 |
+
}
|
| 64 |
+
|
| 65 |
+
function formatParamsBillions(value: number | null | undefined, modelName?: string) {
|
| 66 |
+
const resolvedValue =
|
| 67 |
+
value == null || Number.isNaN(value) ? parseParamsBillionsFromModelName(modelName) : value
|
| 68 |
+
|
| 69 |
+
if (resolvedValue == null || Number.isNaN(resolvedValue)) return "Not reported"
|
| 70 |
+
if (resolvedValue >= 100) return `${Math.round(resolvedValue)}B`
|
| 71 |
+
return `${resolvedValue.toFixed(1)}B`
|
| 72 |
+
}
|
| 73 |
+
|
| 74 |
+
function formatBenchmarkScore(score: number, unit?: string) {
|
| 75 |
+
if (unit === "accuracy" || unit === "pass@1" || (!unit && score >= 0 && score <= 1)) {
|
| 76 |
+
return `${(score * 100).toFixed(1)}`
|
| 77 |
+
}
|
| 78 |
+
|
| 79 |
+
if (unit === "points") {
|
| 80 |
+
return score.toFixed(1)
|
| 81 |
+
}
|
| 82 |
+
|
| 83 |
+
return score.toFixed(2)
|
| 84 |
+
}
|
| 85 |
+
|
| 86 |
+
function getReproducibilityLabel(status: BenchmarkEvaluationCardData["reproducibility_status"]) {
|
| 87 |
+
switch (status) {
|
| 88 |
+
case "complete":
|
| 89 |
+
return "Full config coverage"
|
| 90 |
+
case "partial":
|
| 91 |
+
return "Partial config coverage"
|
| 92 |
+
default:
|
| 93 |
+
return "Config mostly missing"
|
| 94 |
+
}
|
| 95 |
+
}
|
| 96 |
+
|
| 97 |
+
function getIndependentLabel(model: BenchmarkEvaluationCardData) {
|
| 98 |
+
if (model.independent_verification_ratio >= 0.75) return "Mostly third-party"
|
| 99 |
+
if (model.independent_verification_ratio > 0) return "Mixed reporting"
|
| 100 |
+
return "Self-reported only"
|
| 101 |
+
}
|
| 102 |
+
|
| 103 |
+
function getIndependentValue(model: BenchmarkEvaluationCardData) {
|
| 104 |
+
return `${Math.round(model.independent_verification_ratio * 100)}% independent`
|
| 105 |
+
}
|
| 106 |
+
|
| 107 |
+
function getBenchmarkSection(name: string) {
|
| 108 |
+
const value = name.toLowerCase()
|
| 109 |
+
|
| 110 |
+
if (
|
| 111 |
+
value.includes("tau") ||
|
| 112 |
+
value.includes("swe-bench") ||
|
| 113 |
+
value.includes("browsecomp") ||
|
| 114 |
+
value.includes("agent")
|
| 115 |
+
) {
|
| 116 |
+
return "Agentic"
|
| 117 |
+
}
|
| 118 |
+
|
| 119 |
+
if (
|
| 120 |
+
value.includes("aime") ||
|
| 121 |
+
value.includes("gpqa") ||
|
| 122 |
+
value.includes("hmmt") ||
|
| 123 |
+
value.includes("beyond aime") ||
|
| 124 |
+
value.includes("reason")
|
| 125 |
+
) {
|
| 126 |
+
return "Reasoning"
|
| 127 |
+
}
|
| 128 |
+
|
| 129 |
+
if (
|
| 130 |
+
value.includes("math") ||
|
| 131 |
+
value.includes("mmlu") ||
|
| 132 |
+
value.includes("ifeval") ||
|
| 133 |
+
value.includes("arena") ||
|
| 134 |
+
value.includes("live code") ||
|
| 135 |
+
value.includes("humaneval") ||
|
| 136 |
+
value.includes("mbpp") ||
|
| 137 |
+
value.includes("code")
|
| 138 |
+
) {
|
| 139 |
+
return "General"
|
| 140 |
+
}
|
| 141 |
+
|
| 142 |
+
return "Other"
|
| 143 |
+
}
|
| 144 |
+
|
| 145 |
+
const SECTION_ORDER = ["General", "Reasoning", "Agentic", "Other"]
|
| 146 |
+
|
| 147 |
+
const CONTEXT_ROWS = [
|
| 148 |
+
{ key: "developer", label: "Developer" },
|
| 149 |
+
{ key: "params", label: "Parameter range" },
|
| 150 |
+
{ key: "benchmarks", label: "Benchmark coverage" },
|
| 151 |
+
{ key: "reporting", label: "Reporting orgs" },
|
| 152 |
+
{ key: "independence", label: "Reporting posture" },
|
| 153 |
+
{ key: "reproducibility", label: "Reproducibility" },
|
| 154 |
+
{ key: "latest", label: "Latest report" },
|
| 155 |
+
{ key: "updated", label: "Updated" },
|
| 156 |
+
] as const
|
| 157 |
+
|
| 158 |
+
interface ModelCompareDialogProps {
|
| 159 |
+
models: BenchmarkEvaluationCardData[]
|
| 160 |
+
open: boolean
|
| 161 |
+
onOpenChange: (open: boolean) => void
|
| 162 |
+
}
|
| 163 |
+
|
| 164 |
+
export function ModelCompareDialog({
|
| 165 |
+
models,
|
| 166 |
+
open,
|
| 167 |
+
onOpenChange,
|
| 168 |
+
}: ModelCompareDialogProps) {
|
| 169 |
+
const benchmarkRows = useMemo(() => {
|
| 170 |
+
const rows = new Map<
|
| 171 |
+
string,
|
| 172 |
+
{
|
| 173 |
+
benchmark: string
|
| 174 |
+
metric: string
|
| 175 |
+
section: string
|
| 176 |
+
values: Record<string, { score: number; unit?: string }>
|
| 177 |
+
order: number
|
| 178 |
+
}
|
| 179 |
+
>()
|
| 180 |
+
|
| 181 |
+
let order = 0
|
| 182 |
+
|
| 183 |
+
for (const model of models) {
|
| 184 |
+
for (const score of model.top_scores) {
|
| 185 |
+
const existing = rows.get(score.benchmark)
|
| 186 |
+
if (!existing) {
|
| 187 |
+
rows.set(score.benchmark, {
|
| 188 |
+
benchmark: score.benchmark,
|
| 189 |
+
metric: score.metric,
|
| 190 |
+
section: getBenchmarkSection(score.benchmark),
|
| 191 |
+
values: {
|
| 192 |
+
[model.id]: {
|
| 193 |
+
score: score.score,
|
| 194 |
+
unit: score.unit,
|
| 195 |
+
},
|
| 196 |
+
},
|
| 197 |
+
order,
|
| 198 |
+
})
|
| 199 |
+
order += 1
|
| 200 |
+
continue
|
| 201 |
+
}
|
| 202 |
+
|
| 203 |
+
existing.values[model.id] = {
|
| 204 |
+
score: score.score,
|
| 205 |
+
unit: score.unit,
|
| 206 |
+
}
|
| 207 |
+
}
|
| 208 |
+
}
|
| 209 |
+
|
| 210 |
+
return Array.from(rows.values()).sort((a, b) => {
|
| 211 |
+
const sectionDiff = SECTION_ORDER.indexOf(a.section) - SECTION_ORDER.indexOf(b.section)
|
| 212 |
+
if (sectionDiff !== 0) {
|
| 213 |
+
return sectionDiff
|
| 214 |
+
}
|
| 215 |
+
|
| 216 |
+
return a.order - b.order
|
| 217 |
+
})
|
| 218 |
+
}, [models])
|
| 219 |
+
|
| 220 |
+
const benchmarkSections = useMemo(() => {
|
| 221 |
+
return SECTION_ORDER.map((section) => ({
|
| 222 |
+
section,
|
| 223 |
+
rows: benchmarkRows.filter((row) => row.section === section),
|
| 224 |
+
})).filter((group) => group.rows.length > 0)
|
| 225 |
+
}, [benchmarkRows])
|
| 226 |
+
|
| 227 |
+
return (
|
| 228 |
+
<Dialog open={open} onOpenChange={onOpenChange}>
|
| 229 |
+
<DialogContent className="h-[85dvh] max-h-[85dvh] max-w-[min(96vw,1220px)] overflow-hidden p-0 sm:max-w-[min(96vw,1220px)]">
|
| 230 |
+
<div className="flex h-full min-h-0 flex-col">
|
| 231 |
+
<DialogHeader className="border-b border-border/70 px-6 py-5">
|
| 232 |
+
<div className="text-[11px] font-semibold uppercase tracking-[0.22em] text-muted-foreground">
|
| 233 |
+
Side-By-Side Comparison
|
| 234 |
+
</div>
|
| 235 |
+
<DialogTitle>Compare Selected Models</DialogTitle>
|
| 236 |
+
<DialogDescription>
|
| 237 |
+
Start with the benchmark table. Use the context table only when you need reporting, reproducibility, or provenance detail.
|
| 238 |
+
</DialogDescription>
|
| 239 |
+
</DialogHeader>
|
| 240 |
+
|
| 241 |
+
<div className="min-h-0 flex-1 overflow-auto px-6 py-6">
|
| 242 |
+
<div className="min-w-[920px] space-y-6">
|
| 243 |
+
<div className="rounded-[1.5rem] border border-border/70 bg-background">
|
| 244 |
+
<div className="border-b border-border/60 px-5 py-4">
|
| 245 |
+
<div className="text-[11px] font-semibold uppercase tracking-[0.22em] text-muted-foreground">
|
| 246 |
+
Benchmark Comparison
|
| 247 |
+
</div>
|
| 248 |
+
<div className="mt-2 text-sm text-muted-foreground">
|
| 249 |
+
Rows are drawn from the most relevant surfaced benchmarks across the selected models, closer to how release posts present comparison tables.
|
| 250 |
+
</div>
|
| 251 |
+
</div>
|
| 252 |
+
|
| 253 |
+
<div className="overflow-auto">
|
| 254 |
+
<Table>
|
| 255 |
+
<TableHeader>
|
| 256 |
+
<TableRow>
|
| 257 |
+
<TableHead className="w-[260px]">Benchmark</TableHead>
|
| 258 |
+
{models.map((model) => (
|
| 259 |
+
<TableHead key={model.id} className="min-w-[170px] align-top">
|
| 260 |
+
<div className="space-y-2">
|
| 261 |
+
<div>
|
| 262 |
+
<div className="font-semibold text-foreground">{model.model_name}</div>
|
| 263 |
+
<div className="text-xs font-medium uppercase tracking-[0.14em] text-muted-foreground">
|
| 264 |
+
{model.developer || "Unknown developer"}
|
| 265 |
+
</div>
|
| 266 |
+
</div>
|
| 267 |
+
<div className="flex flex-wrap gap-2">
|
| 268 |
+
<Badge variant="outline">
|
| 269 |
+
{formatParamsBillions(model.params_billions, model.model_name)}
|
| 270 |
+
</Badge>
|
| 271 |
+
<Button asChild variant="ghost" size="sm">
|
| 272 |
+
<Link href={`/models/${model.route_id}`}>View</Link>
|
| 273 |
+
</Button>
|
| 274 |
+
</div>
|
| 275 |
+
</div>
|
| 276 |
+
</TableHead>
|
| 277 |
+
))}
|
| 278 |
+
</TableRow>
|
| 279 |
+
</TableHeader>
|
| 280 |
+
<TableBody>
|
| 281 |
+
{benchmarkSections.map((group) => (
|
| 282 |
+
<Fragment key={group.section}>
|
| 283 |
+
<TableRow key={`${group.section}-heading`}>
|
| 284 |
+
<TableCell
|
| 285 |
+
colSpan={models.length + 1}
|
| 286 |
+
className="bg-muted/25 text-[11px] font-semibold uppercase tracking-[0.22em] text-muted-foreground"
|
| 287 |
+
>
|
| 288 |
+
{group.section}
|
| 289 |
+
</TableCell>
|
| 290 |
+
</TableRow>
|
| 291 |
+
{group.rows.map((row) => {
|
| 292 |
+
const rowValues = Object.values(row.values).map((value) => value.score)
|
| 293 |
+
const maxScore = rowValues.length > 0 ? Math.max(...rowValues) : null
|
| 294 |
+
|
| 295 |
+
return (
|
| 296 |
+
<TableRow key={row.benchmark}>
|
| 297 |
+
<TableCell className="whitespace-normal">
|
| 298 |
+
<div className="font-medium">{row.benchmark}</div>
|
| 299 |
+
<div className="mt-1 text-xs text-muted-foreground">{row.metric}</div>
|
| 300 |
+
</TableCell>
|
| 301 |
+
{models.map((model) => {
|
| 302 |
+
const value = row.values[model.id]
|
| 303 |
+
const isBest = value && maxScore != null && value.score === maxScore
|
| 304 |
+
|
| 305 |
+
return (
|
| 306 |
+
<TableCell
|
| 307 |
+
key={`${row.benchmark}-${model.id}`}
|
| 308 |
+
className={isBest ? "bg-emerald-50/70 dark:bg-emerald-950/20" : ""}
|
| 309 |
+
>
|
| 310 |
+
{value ? (
|
| 311 |
+
<div className="text-base font-semibold tabular-nums">
|
| 312 |
+
{formatBenchmarkScore(value.score, value.unit)}
|
| 313 |
+
</div>
|
| 314 |
+
) : (
|
| 315 |
+
<div className="text-muted-foreground">--</div>
|
| 316 |
+
)}
|
| 317 |
+
</TableCell>
|
| 318 |
+
)
|
| 319 |
+
})}
|
| 320 |
+
</TableRow>
|
| 321 |
+
)
|
| 322 |
+
})}
|
| 323 |
+
</Fragment>
|
| 324 |
+
))}
|
| 325 |
+
</TableBody>
|
| 326 |
+
</Table>
|
| 327 |
+
</div>
|
| 328 |
+
</div>
|
| 329 |
+
|
| 330 |
+
<Collapsible className="rounded-[1.5rem] border border-border/70 bg-muted/10">
|
| 331 |
+
<CollapsibleTrigger asChild>
|
| 332 |
+
<button
|
| 333 |
+
type="button"
|
| 334 |
+
className="flex w-full items-center justify-between px-5 py-4 text-left"
|
| 335 |
+
>
|
| 336 |
+
<div>
|
| 337 |
+
<div className="text-[11px] font-semibold uppercase tracking-[0.22em] text-muted-foreground">
|
| 338 |
+
Dive Deeper
|
| 339 |
+
</div>
|
| 340 |
+
<div className="mt-1 font-semibold text-foreground">
|
| 341 |
+
Show reporting and reproducibility context
|
| 342 |
+
</div>
|
| 343 |
+
</div>
|
| 344 |
+
<ChevronDown className="h-4 w-4 text-muted-foreground" />
|
| 345 |
+
</button>
|
| 346 |
+
</CollapsibleTrigger>
|
| 347 |
+
<CollapsibleContent className="border-t border-border/60 px-5 py-5">
|
| 348 |
+
<div className="overflow-auto rounded-[1.25rem] border border-border/70 bg-background">
|
| 349 |
+
<Table>
|
| 350 |
+
<TableHeader>
|
| 351 |
+
<TableRow>
|
| 352 |
+
<TableHead className="w-[220px]">Signal</TableHead>
|
| 353 |
+
{models.map((model) => (
|
| 354 |
+
<TableHead key={`${model.id}-context`} className="min-w-[220px]">
|
| 355 |
+
<div className="font-semibold">{model.model_name}</div>
|
| 356 |
+
</TableHead>
|
| 357 |
+
))}
|
| 358 |
+
</TableRow>
|
| 359 |
+
</TableHeader>
|
| 360 |
+
<TableBody>
|
| 361 |
+
{CONTEXT_ROWS.map((row) => (
|
| 362 |
+
<TableRow key={row.key}>
|
| 363 |
+
<TableCell className="font-medium text-muted-foreground">{row.label}</TableCell>
|
| 364 |
+
{models.map((model) => (
|
| 365 |
+
<TableCell key={`${model.id}-${row.key}`} className="align-top whitespace-normal">
|
| 366 |
+
{row.key === "developer" ? model.developer || "Unknown developer" : null}
|
| 367 |
+
{row.key === "params"
|
| 368 |
+
? formatParamsBillions(model.params_billions, model.model_name)
|
| 369 |
+
: null}
|
| 370 |
+
{row.key === "benchmarks" ? (
|
| 371 |
+
<div className="space-y-1">
|
| 372 |
+
<div className="font-medium">{model.benchmarks_count} covered benchmarks</div>
|
| 373 |
+
<div className="text-sm text-muted-foreground">
|
| 374 |
+
{model.evaluations_count} reported result{model.evaluations_count !== 1 ? "s" : ""}
|
| 375 |
+
</div>
|
| 376 |
+
</div>
|
| 377 |
+
) : null}
|
| 378 |
+
{row.key === "reporting" ? (
|
| 379 |
+
<div className="space-y-1">
|
| 380 |
+
<div className="font-medium">
|
| 381 |
+
{model.evaluator_count} reporting org{model.evaluator_count !== 1 ? "s" : ""}
|
| 382 |
+
</div>
|
| 383 |
+
<div className="text-sm text-muted-foreground">
|
| 384 |
+
{model.evaluator_names.slice(0, 3).join(", ") || "Not named"}
|
| 385 |
+
</div>
|
| 386 |
+
</div>
|
| 387 |
+
) : null}
|
| 388 |
+
{row.key === "independence" ? (
|
| 389 |
+
<div className="space-y-2">
|
| 390 |
+
<Badge
|
| 391 |
+
variant={model.independent_verification_ratio > 0 ? "secondary" : "outline"}
|
| 392 |
+
className="font-medium"
|
| 393 |
+
>
|
| 394 |
+
<ShieldCheck className="h-3.5 w-3.5" />
|
| 395 |
+
{getIndependentLabel(model)}
|
| 396 |
+
</Badge>
|
| 397 |
+
<div className="text-sm text-muted-foreground">
|
| 398 |
+
{getIndependentValue(model)}
|
| 399 |
+
</div>
|
| 400 |
+
</div>
|
| 401 |
+
) : null}
|
| 402 |
+
{row.key === "reproducibility" ? (
|
| 403 |
+
<div className="space-y-2">
|
| 404 |
+
<Badge
|
| 405 |
+
variant={
|
| 406 |
+
model.reproducibility_status === "complete"
|
| 407 |
+
? "secondary"
|
| 408 |
+
: model.reproducibility_status === "partial"
|
| 409 |
+
? "outline"
|
| 410 |
+
: "destructive"
|
| 411 |
+
}
|
| 412 |
+
className="font-medium"
|
| 413 |
+
>
|
| 414 |
+
{model.reproducibility_status === "missing" ? (
|
| 415 |
+
<TriangleAlert className="h-3.5 w-3.5" />
|
| 416 |
+
) : (
|
| 417 |
+
<FlaskConical className="h-3.5 w-3.5" />
|
| 418 |
+
)}
|
| 419 |
+
{getReproducibilityLabel(model.reproducibility_status)}
|
| 420 |
+
</Badge>
|
| 421 |
+
<div className="text-sm text-muted-foreground">
|
| 422 |
+
{model.missing_generation_config_count > 0
|
| 423 |
+
? `${model.missing_generation_config_count} result${model.missing_generation_config_count !== 1 ? "s" : ""} without generation config`
|
| 424 |
+
: "No missing generation config in current corpus"}
|
| 425 |
+
</div>
|
| 426 |
+
</div>
|
| 427 |
+
) : null}
|
| 428 |
+
{row.key === "latest" ? (
|
| 429 |
+
<div className="flex items-center gap-2">
|
| 430 |
+
<span>{model.latest_source_name || "No named source"}</span>
|
| 431 |
+
{model.source_urls[0] ? (
|
| 432 |
+
<a
|
| 433 |
+
href={model.source_urls[0]}
|
| 434 |
+
target="_blank"
|
| 435 |
+
rel="noopener noreferrer"
|
| 436 |
+
className="text-muted-foreground hover:text-foreground"
|
| 437 |
+
>
|
| 438 |
+
<ExternalLink className="h-3.5 w-3.5" />
|
| 439 |
+
</a>
|
| 440 |
+
) : null}
|
| 441 |
+
</div>
|
| 442 |
+
) : null}
|
| 443 |
+
{row.key === "updated" ? formatDate(model.latest_timestamp) : null}
|
| 444 |
+
</TableCell>
|
| 445 |
+
))}
|
| 446 |
+
</TableRow>
|
| 447 |
+
))}
|
| 448 |
+
</TableBody>
|
| 449 |
+
</Table>
|
| 450 |
+
</div>
|
| 451 |
+
</CollapsibleContent>
|
| 452 |
+
</Collapsible>
|
| 453 |
+
</div>
|
| 454 |
+
</div>
|
| 455 |
+
</div>
|
| 456 |
+
</DialogContent>
|
| 457 |
+
</Dialog>
|
| 458 |
+
)
|
| 459 |
+
}
|
lib/eval-processing.ts
CHANGED
|
@@ -29,6 +29,15 @@ const GENERIC_EVALUATION_NAMES = new Set([
|
|
| 29 |
"pass@1",
|
| 30 |
])
|
| 31 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 32 |
function slugify(value: string): string {
|
| 33 |
return value.toLowerCase().replace(/[^a-z0-9]+/g, "_").replace(/^_|_$/g, "")
|
| 34 |
}
|
|
@@ -80,6 +89,18 @@ function getEvaluationSummaryId(
|
|
| 80 |
return slugify(`${benchmarkKey}__${result.evaluation_name}`)
|
| 81 |
}
|
| 82 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 83 |
// ββ Eval-centric (per-benchmark) types ββββββββββββββββββββββββββββββββββββββββ
|
| 84 |
|
| 85 |
export interface ModelResultForBenchmark {
|
|
@@ -339,6 +360,7 @@ export function createEvaluationCard(
|
|
| 339 |
const benchmarksSet = new Set<string>()
|
| 340 |
const allScores: Array<{
|
| 341 |
benchmark: string
|
|
|
|
| 342 |
score: number
|
| 343 |
metric: string
|
| 344 |
unit?: string
|
|
@@ -426,6 +448,7 @@ export function createEvaluationCard(
|
|
| 426 |
|
| 427 |
allScores.push({
|
| 428 |
benchmark: getEvaluationDisplayName(eval_, result),
|
|
|
|
| 429 |
score: result.score_details.score,
|
| 430 |
metric: result.metric_config.evaluation_description || result.evaluation_name,
|
| 431 |
unit: result.metric_config.unit
|
|
@@ -435,7 +458,10 @@ export function createEvaluationCard(
|
|
| 435 |
}
|
| 436 |
|
| 437 |
// Deduplicate by benchmark name, keeping highest score for each
|
| 438 |
-
const scoresByBenchmark = new Map<
|
|
|
|
|
|
|
|
|
|
| 439 |
for (const scoreData of allScores) {
|
| 440 |
const existing = scoresByBenchmark.get(scoreData.benchmark)
|
| 441 |
if (!existing || scoreData.score > existing.score) {
|
|
@@ -492,8 +518,25 @@ export function createEvaluationCard(
|
|
| 492 |
|
| 493 |
// Get top 5 unique benchmarks by score
|
| 494 |
const topScores = Array.from(scoresByBenchmark.values())
|
| 495 |
-
.sort((a, b) =>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 496 |
.slice(0, 5)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 497 |
|
| 498 |
const paramsBillionsRaw = summary.model_info.additional_details?.params_billions
|
| 499 |
const paramsBillions =
|
|
|
|
| 29 |
"pass@1",
|
| 30 |
])
|
| 31 |
|
| 32 |
+
const BENCHMARK_PRIORITY_RULES: Array<{ pattern: RegExp; priority: number }> = [
|
| 33 |
+
{ pattern: /\b(swe-bench|terminal-bench|tau-bench|agent|browsecomp)\b/, priority: 10 },
|
| 34 |
+
{ pattern: /\b(gpqa|mmlu-pro|mmlu|bbh|ifeval|math|aime|gsm8k|minerva)\b/, priority: 9 },
|
| 35 |
+
{ pattern: /\b(humaneval|livecodebench|mbpp|codecontests|apps)\b/, priority: 8 },
|
| 36 |
+
{ pattern: /\b(mmmu|mmmu-pro|seed-bench|vision|vqa|multimodal)\b/, priority: 7 },
|
| 37 |
+
{ pattern: /\b(mt-bench|arena-hard|alpacaeval|reward-bench|truthfulqa)\b/, priority: 6 },
|
| 38 |
+
{ pattern: /\b(fairness|bias|safety|toxic|harmful|robust|privacy)\b/, priority: 5 },
|
| 39 |
+
]
|
| 40 |
+
|
| 41 |
function slugify(value: string): string {
|
| 42 |
return value.toLowerCase().replace(/[^a-z0-9]+/g, "_").replace(/^_|_$/g, "")
|
| 43 |
}
|
|
|
|
| 89 |
return slugify(`${benchmarkKey}__${result.evaluation_name}`)
|
| 90 |
}
|
| 91 |
|
| 92 |
+
function getBenchmarkPriority(value: string): number {
|
| 93 |
+
const normalized = value.toLowerCase()
|
| 94 |
+
|
| 95 |
+
for (const rule of BENCHMARK_PRIORITY_RULES) {
|
| 96 |
+
if (rule.pattern.test(normalized)) {
|
| 97 |
+
return rule.priority
|
| 98 |
+
}
|
| 99 |
+
}
|
| 100 |
+
|
| 101 |
+
return 0
|
| 102 |
+
}
|
| 103 |
+
|
| 104 |
// ββ Eval-centric (per-benchmark) types ββββββββββββββββββββββββββββββββββββββββ
|
| 105 |
|
| 106 |
export interface ModelResultForBenchmark {
|
|
|
|
| 360 |
const benchmarksSet = new Set<string>()
|
| 361 |
const allScores: Array<{
|
| 362 |
benchmark: string
|
| 363 |
+
benchmarkKey: string
|
| 364 |
score: number
|
| 365 |
metric: string
|
| 366 |
unit?: string
|
|
|
|
| 448 |
|
| 449 |
allScores.push({
|
| 450 |
benchmark: getEvaluationDisplayName(eval_, result),
|
| 451 |
+
benchmarkKey: getBenchmarkName(eval_, result),
|
| 452 |
score: result.score_details.score,
|
| 453 |
metric: result.metric_config.evaluation_description || result.evaluation_name,
|
| 454 |
unit: result.metric_config.unit
|
|
|
|
| 458 |
}
|
| 459 |
|
| 460 |
// Deduplicate by benchmark name, keeping highest score for each
|
| 461 |
+
const scoresByBenchmark = new Map<
|
| 462 |
+
string,
|
| 463 |
+
{ benchmark: string; benchmarkKey: string; score: number; metric: string; unit?: string }
|
| 464 |
+
>()
|
| 465 |
for (const scoreData of allScores) {
|
| 466 |
const existing = scoresByBenchmark.get(scoreData.benchmark)
|
| 467 |
if (!existing || scoreData.score > existing.score) {
|
|
|
|
| 518 |
|
| 519 |
// Get top 5 unique benchmarks by score
|
| 520 |
const topScores = Array.from(scoresByBenchmark.values())
|
| 521 |
+
.sort((a, b) => {
|
| 522 |
+
const priorityDiff = getBenchmarkPriority(b.benchmarkKey) - getBenchmarkPriority(a.benchmarkKey)
|
| 523 |
+
if (priorityDiff !== 0) {
|
| 524 |
+
return priorityDiff
|
| 525 |
+
}
|
| 526 |
+
|
| 527 |
+
if (b.score !== a.score) {
|
| 528 |
+
return b.score - a.score
|
| 529 |
+
}
|
| 530 |
+
|
| 531 |
+
return a.benchmark.localeCompare(b.benchmark)
|
| 532 |
+
})
|
| 533 |
.slice(0, 5)
|
| 534 |
+
.map(({ benchmark, score, metric, unit }) => ({
|
| 535 |
+
benchmark,
|
| 536 |
+
score,
|
| 537 |
+
metric,
|
| 538 |
+
unit,
|
| 539 |
+
}))
|
| 540 |
|
| 541 |
const paramsBillionsRaw = summary.model_info.additional_details?.params_billions
|
| 542 |
const paramsBillions =
|