"use client" import { useMemo, useState } from "react" import { BookOpen, ChevronDown, ChevronUp, ExternalLink, FileText, Globe, Layers, ScrollText, Tag, Users } from "lucide-react" import { SignalTooltip } from "@/components/signals/signal-tooltip" import type { BenchmarkEvalSummary } from "@/lib/eval-processing" import { getKnownIssues } from "@/lib/known-issues" import { KnownIssuesPanel } from "@/components/known-issues-panel" interface PolicyOverviewProps { summary: BenchmarkEvalSummary } const SUMMARY_PREVIEW_CHARS = 280 function classifyResource(url: string): { kind: "paper" | "dataset" | "leaderboard" | "site"; label: string } { const lower = url.toLowerCase() if (lower.includes("arxiv.org") || lower.endsWith(".pdf") || lower.includes("/papers/")) { return { kind: "paper", label: "Paper" } } if (lower.includes("huggingface.co/datasets") || lower.includes("/dataset")) { return { kind: "dataset", label: "Dataset" } } if (lower.includes("leaderboard")) { return { kind: "leaderboard", label: "Leaderboard" } } return { kind: "site", label: "Source" } } function shortHost(url: string) { return url.replace(/^https?:\/\//, "").replace(/\/.*$/, "") } /** * Plain-language summary surface shown at the top of the benchmark page in * policy mode. Designed to answer three questions a non-technical reader has: * what does this measure, who built it, and where does it come from? * * Technical detail (variants, metric specifications, score scales) lives in * the existing overview card below this and is collapsed by default. */ function normalizeId(value: string | undefined | null): string { if (!value) return "" return value .toLowerCase() .trim() .replace(/[\s_\-/]+/g, "") } export function PolicyOverview({ summary }: PolicyOverviewProps) { const card = summary.benchmark_card // Defensive check: pipelines older than the "ancestor card leak" fix // sometimes attach the parent suite's card to a leaf benchmark (e.g. // helm_classic's card embedded under XSUM). Detect when the card's own // name is clearly not this benchmark and ignore its narrative text — the // synthesized fallback below produces something accurate instead. const cardName = card?.benchmark_details?.name const cardNameNorm = normalizeId(cardName) const evalIdentifiers = [ summary.evaluation_name, summary.benchmark_leaf_key, summary.composite_benchmark_key, summary.composite_benchmark_name, summary.canonical_display_name, summary.evaluation_id, ].map(normalizeId) // Treat the card as belonging to this eval when its name fuzzily appears in // any of the eval's identifiers (or vice versa). Otherwise the card is from // a different (typically ancestor) benchmark. const cardMatchesEval = !cardName || evalIdentifiers.some( (id) => id.length > 0 && cardNameNorm.length > 0 && (id.includes(cardNameNorm) || cardNameNorm.includes(id)), ) const overview = cardMatchesEval ? card?.benchmark_details?.overview?.trim() || "" : "" const goal = cardMatchesEval ? card?.purpose_and_intended_users?.goal?.trim() || "" : "" // Detect "parent" benchmark pages — either an aggregated composite or a // multi-metric matrix where each column is a subtask. In both cases the // per-evaluation `metric_config.evaluation_description` belongs to whichever // component was processed first (e.g. just the "airline" subset of Tau // Bench 2) and would mislead a policy reader. Synthesize parent framing // instead and surface the subtasks separately. const isAggregated = summary.is_aggregated === true const aggregateNames = (summary.aggregate_sources ?? []) .map((s) => s.composite_benchmark_name) .filter((s): s is string => typeof s === "string" && s.length > 0) const subtaskLabels = useMemo(() => { const seen = new Set() const labels: string[] = [] const add = (raw: string | undefined | null) => { if (!raw) return const trimmed = raw.trim() if (!trimmed) return const key = trimmed.toLowerCase() if (seen.has(key)) return seen.add(key) labels.push(trimmed) } for (const subtask of summary.subtasks ?? []) { add(subtask.display_name || subtask.subtask_name) } for (const metric of summary.leaderboard_metrics ?? []) { if (metric.scope === "subtask") { add(metric.subtask_name || metric.display_name) } } for (const name of aggregateNames) add(name) return labels }, [summary.subtasks, summary.leaderboard_metrics, aggregateNames]) const isMatrix = (summary.leaderboard_metrics?.length ?? 0) > 1 const isParentPage = isAggregated || (isMatrix && subtaskLabels.length > 1) const useComponentDescription = !isParentPage const parentFallback = isParentPage && subtaskLabels.length > 1 ? `${summary.evaluation_name} reports results across ${subtaskLabels.length} ${ isAggregated ? "component benchmarks" : "subtasks" }. Each is evaluated separately; the score shown is the ${ isAggregated ? "average" : "per-subtask result" }.` : null const summaryText = overview || goal || parentFallback || (useComponentDescription ? summary.metric_config.evaluation_description : summary.evaluation_name) const [expanded, setExpanded] = useState(false) const [subtasksOpen, setSubtasksOpen] = useState(false) const isLong = summaryText.length > SUMMARY_PREVIEW_CHARS const visibleText = expanded || !isLong ? summaryText : summaryText.slice(0, SUMMARY_PREVIEW_CHARS).replace(/\s+\S*$/, "") + "…" const domains = useMemo(() => { const fromTags = summary.tags?.domains ?? [] const fromCard = cardMatchesEval ? card?.benchmark_details?.domains ?? [] : [] return Array.from(new Set([...fromTags, ...fromCard].map((d) => d.trim()).filter(Boolean))).slice(0, 6) }, [summary.tags?.domains, card?.benchmark_details?.domains, cardMatchesEval]) const languages = useMemo(() => { const fromTags = summary.tags?.languages ?? [] const fromCard = cardMatchesEval ? card?.benchmark_details?.languages ?? [] : [] return Array.from(new Set([...fromTags, ...fromCard].map((d) => d.trim()).filter(Boolean))).slice(0, 4) }, [summary.tags?.languages, card?.benchmark_details?.languages, cardMatchesEval]) const license = card?.ethical_and_legal_considerations?.data_licensing const showLicense = license && license !== "Not specified" const resources = useMemo(() => { if (!cardMatchesEval) return [] const urls = (card?.benchmark_details?.resources ?? []).filter((r) => typeof r === "string" && r.startsWith("http")) const seen = new Map["kind"]; label: string; url: string }>() for (const url of urls) { const c = classifyResource(url) if (!seen.has(c.kind)) { seen.set(c.kind, { ...c, url }) } } return Array.from(seen.values()) }, [card?.benchmark_details?.resources, cardMatchesEval]) const evaluators = (summary.evaluator_names ?? []).slice(0, 3) const hasMoreEvaluators = (summary.evaluator_names?.length ?? 0) > evaluators.length const knownIssues = useMemo( () => getKnownIssues( summary.evaluation_name, summary.composite_benchmark_name, summary.composite_benchmark_key, summary.benchmark_family_key, summary.benchmark_leaf_key, card?.benchmark_details?.name, ), [ summary.evaluation_name, summary.composite_benchmark_name, summary.composite_benchmark_key, summary.benchmark_family_key, summary.benchmark_leaf_key, card?.benchmark_details?.name, ], ) const directionLabel = summary.metric_config.lower_is_better ? "Lower scores are better" : "Higher scores are better" // Policy-note triple (paper §4.2.2): What it measures · Main caveat · Intended for. const measuresText = visibleText const caveatText = card?.purpose_and_intended_users?.limitations?.trim() || null const audienceArr = card?.purpose_and_intended_users?.audience const audienceText = Array.isArray(audienceArr) ? audienceArr.filter(Boolean).join("; ") : (typeof audienceArr === "string" ? audienceArr : "") return (
Policy note {summary.evaluation_name} · in plain language
{knownIssues.length > 0 && (
)}
Measures
{measuresText} {isLong && ( )}
{caveatText && ( <>
Caveat
{caveatText}
)} {audienceText && ( <>
Intended for
{audienceText}
)}
How to read
{directionLabel}. {" "} Compared across {summary.models_count} model{summary.models_count === 1 ? "" : "s"}.
{isParentPage && subtaskLabels.length > 1 && (
{subtasksOpen && (
    {subtaskLabels.map((name) => (
  • {name}
  • ))}
)}
)} {(domains.length > 0 || languages.length > 0 || showLicense) && (
{domains.map((d) => ( {d} ))} {languages.map((l) => ( {l} ))} {showLicense && ( {license} )}
)} {(resources.length > 0 || evaluators.length > 0) && (
Where this comes from
{resources.map((r) => ( {r.label} {shortHost(r.url)} ))} {evaluators.length > 0 && ( Who reported these scores {summary.evaluator_names?.join(", ")} {summary.third_party_ratio > 0 && ( {Math.round(summary.third_party_ratio * 100)}% of results come from independent evaluators (not the model's own developer). )} } > Reported by {evaluators.join(", ")} {hasMoreEvaluators ? ` +${(summary.evaluator_names?.length ?? 0) - evaluators.length} more` : ""} )}
)}
) }