Spaces:
Running
Reconcile UI with v2 backend payload + drop redundant signal cards
Browse filesThe Swap backend data merge (fe99ffa) routed the app through the new
view-data layer where corpus-aggregates and hierarchy.json have
partially renamed shapes. Three concrete regressions from that merge
plus one duplicate-card cleanup are addressed here.
components/signals/corpus-signals-strip.tsx
- Crash fix. The strip was throwing
`TypeError: Cannot read properties of undefined (reading
'toLocaleString')` on the home / corpus routes because the new
CompletenessCorpusBlock and ComparabilityCorpusBlock no longer
expose `total_benchmarks`, `completeness_score_mean`,
`completeness_score_median`, `variant_divergence_rate`,
`cross_party_eligible_groups`, `variant_eligible_groups`, or
`cross_party_divergence_rate`. The renamed fields (total_triples,
completeness_avg/min/max, variant_divergent_count,
groups_with_variant_check, groups_with_cross_party_check, etc.)
are already in lib/backend-artifacts.ts post-merge but the strip
still read the old names.
- Each block read now goes through `aggregates.{block}?.overall`
with optional chaining on every nested field. When a value is
absent the tile shows a non-numeric headline ("mean across
reported score triples." instead of "mean across X reported β¦")
rather than crashing the route.
- source_type_distribution shares (tpShare/fpShare) only compute
when both totalReports > 0 and the corresponding bucket exists.
components/eval-detail.tsx
- Drops the standalone rounded shadcn-style <CompletenessPanel>
and <ComparabilityPanel> cards from the technical-details
accordion. The compact BenchmarkSignalsStrip introduced in
32864b0 already conveys both signals in the paper-aligned 4-tile
framing; the standalone cards were a duplicate that visually
re-introduced the old design language ("Comparability β Groups
where reported scores divergeβ¦").
- Removes the now-unused imports (CompletenessPanel,
ComparabilityPanel) and the dead `benchmarkComparability` local.
components/family-table.tsx
- Hardens collectLeafEntries against hierarchy snapshots that ship
leaves without an explicit `eval_summary_ids` array (the v2
snapshot doesn't always populate it). When that field is empty
we now synthesise the id as `${family_key}_${leaf_key}` (the
pipeline's standard naming) and fall back to the bare leaf key.
Without this guard, every leaf was silently dropped and the
inline "Benchmarks in this family Β· N" grid disappeared
completely on rows like Helm classic / LLM-Stats.
|
@@ -4,8 +4,6 @@ import { useAudienceMode } from "@/components/audience-mode-provider"
|
|
| 4 |
import { Fragment, useEffect, useMemo, useState } from "react"
|
| 5 |
import Link from "next/link"
|
| 6 |
import { BenchmarkSignalsStrip } from "@/components/signals/benchmark-signals-strip"
|
| 7 |
-
import { CompletenessPanel } from "@/components/signals/completeness-panel"
|
| 8 |
-
import { ComparabilityPanel } from "@/components/signals/comparability-panel"
|
| 9 |
import { SignalsRowBadges } from "@/components/signals/signals-row-badges"
|
| 10 |
import { getCompletenessPopulatedCount } from "@/components/signals/signal-utils"
|
| 11 |
import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "@/components/ui/collapsible"
|
|
@@ -551,7 +549,6 @@ export function EvalDetail({ summary }: EvalDetailProps) {
|
|
| 551 |
? "Averaged model results across the suite's component benchmarks, with drill-down to each component score."
|
| 552 |
: "Model results with benchmark context, source dataset detail, and optional instance-data links."
|
| 553 |
const reportingCompleteness = summary.evalcards?.annotations?.reporting_completeness
|
| 554 |
-
const benchmarkComparability = summary.evalcards?.annotations?.benchmark_comparability
|
| 555 |
const documentationPopulatedCount = reportingCompleteness
|
| 556 |
? getCompletenessPopulatedCount(reportingCompleteness)
|
| 557 |
: null
|
|
@@ -732,11 +729,10 @@ export function EvalDetail({ summary }: EvalDetailProps) {
|
|
| 732 |
</dl>
|
| 733 |
</div>
|
| 734 |
|
| 735 |
-
|
| 736 |
-
|
| 737 |
-
|
| 738 |
-
|
| 739 |
-
/>
|
| 740 |
|
| 741 |
{!hasMultiMetricLeaderboard && (summary.root_metrics?.length || summary.subtasks?.length) ? (
|
| 742 |
<section
|
|
|
|
| 4 |
import { Fragment, useEffect, useMemo, useState } from "react"
|
| 5 |
import Link from "next/link"
|
| 6 |
import { BenchmarkSignalsStrip } from "@/components/signals/benchmark-signals-strip"
|
|
|
|
|
|
|
| 7 |
import { SignalsRowBadges } from "@/components/signals/signals-row-badges"
|
| 8 |
import { getCompletenessPopulatedCount } from "@/components/signals/signal-utils"
|
| 9 |
import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "@/components/ui/collapsible"
|
|
|
|
| 549 |
? "Averaged model results across the suite's component benchmarks, with drill-down to each component score."
|
| 550 |
: "Model results with benchmark context, source dataset detail, and optional instance-data links."
|
| 551 |
const reportingCompleteness = summary.evalcards?.annotations?.reporting_completeness
|
|
|
|
| 552 |
const documentationPopulatedCount = reportingCompleteness
|
| 553 |
? getCompletenessPopulatedCount(reportingCompleteness)
|
| 554 |
: null
|
|
|
|
| 729 |
</dl>
|
| 730 |
</div>
|
| 731 |
|
| 732 |
+
{/* The compact BenchmarkSignalsStrip above already covers
|
| 733 |
+
* completeness and comparability with paper-aligned framing,
|
| 734 |
+
* so the standalone rounded shadcn cards that used to live
|
| 735 |
+
* here are intentionally dropped. */}
|
|
|
|
| 736 |
|
| 737 |
{!hasMultiMetricLeaderboard && (summary.root_metrics?.length || summary.subtasks?.length) ? (
|
| 738 |
<section
|
|
@@ -73,7 +73,19 @@ function collectLeafEntries(
|
|
| 73 |
): LeafEntry[] {
|
| 74 |
const out: LeafEntry[] = []
|
| 75 |
for (const leaf of fam.leaves ?? []) {
|
| 76 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 77 |
if (ids.length === 0) continue
|
| 78 |
// Domain sources, in order of trust:
|
| 79 |
// (1) hierarchy `leaf.tags.domains` β sometimes absent
|
|
|
|
| 73 |
): LeafEntry[] {
|
| 74 |
const out: LeafEntry[] = []
|
| 75 |
for (const leaf of fam.leaves ?? []) {
|
| 76 |
+
// Backends differ in whether leaves carry an explicit
|
| 77 |
+
// `eval_summary_ids` array. When absent, fall back to the
|
| 78 |
+
// pipeline's standard `${fam.key}_${leaf.key}` naming, then to the
|
| 79 |
+
// bare leaf key β both are stable enough for the detail page to
|
| 80 |
+
// resolve. This stops the inline benchmarks grid from disappearing
|
| 81 |
+
// on a backend that ships hierarchy.json without leaf eval ids.
|
| 82 |
+
const explicit = leaf.eval_summary_ids ?? []
|
| 83 |
+
const ids =
|
| 84 |
+
explicit.length > 0
|
| 85 |
+
? explicit
|
| 86 |
+
: leaf.key
|
| 87 |
+
? [`${fam.key}_${leaf.key}`, leaf.key]
|
| 88 |
+
: []
|
| 89 |
if (ids.length === 0) continue
|
| 90 |
// Domain sources, in order of trust:
|
| 91 |
// (1) hierarchy `leaf.tags.domains` β sometimes absent
|
|
@@ -23,29 +23,38 @@ export function CorpusSignalsStrip({
|
|
| 23 |
}: {
|
| 24 |
aggregates: CorpusAggregates
|
| 25 |
}) {
|
| 26 |
-
|
| 27 |
-
|
| 28 |
-
|
| 29 |
-
const
|
|
|
|
|
|
|
|
|
|
| 30 |
|
| 31 |
// Invert the gap rate to read as "documented", which matches reader intuition.
|
| 32 |
-
const
|
| 33 |
-
|
| 34 |
-
|
| 35 |
-
|
| 36 |
-
const
|
| 37 |
-
|
| 38 |
-
const
|
| 39 |
-
|
| 40 |
-
|
| 41 |
-
|
| 42 |
-
|
| 43 |
-
|
|
|
|
|
|
|
|
|
|
| 44 |
const crossPartyRate = rate(
|
| 45 |
-
cmp.cross_party_divergent_count,
|
| 46 |
-
cmp.groups_with_cross_party_check
|
| 47 |
)
|
| 48 |
-
const crossPartyAvailable = cmp.groups_with_cross_party_check > 0
|
|
|
|
|
|
|
|
|
|
| 49 |
|
| 50 |
return (
|
| 51 |
<div className="signals-grid">
|
|
@@ -56,17 +65,21 @@ export function CorpusSignalsStrip({
|
|
| 56 |
headline="of reported scores have a complete setup recorded β the rest cannot be independently re-run."
|
| 57 |
detail={
|
| 58 |
reproDetail
|
| 59 |
-
? `${formatPct(
|
| 60 |
-
: `${formatPct(
|
| 61 |
}
|
| 62 |
asks="Can someone else run this evaluation and get the same number?"
|
| 63 |
/>
|
| 64 |
<SignalTile
|
| 65 |
id="completeness"
|
| 66 |
-
statValue={pctNum(comp.completeness_avg)}
|
| 67 |
statUnit="%"
|
| 68 |
-
headline={
|
| 69 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 70 |
asks="Is the benchmark itself documented well enough to interpret a score on it?"
|
| 71 |
/>
|
| 72 |
<SignalTile
|
|
@@ -81,7 +94,11 @@ export function CorpusSignalsStrip({
|
|
| 81 |
id="comparability"
|
| 82 |
statValue={pctNum(cmpRate)}
|
| 83 |
statUnit="%"
|
| 84 |
-
headline={
|
|
|
|
|
|
|
|
|
|
|
|
|
| 85 |
detail={
|
| 86 |
crossPartyAvailable
|
| 87 |
? `Cross-party divergence: ${formatPct(crossPartyRate)}.`
|
|
|
|
| 23 |
}: {
|
| 24 |
aggregates: CorpusAggregates
|
| 25 |
}) {
|
| 26 |
+
// Each block can be partly missing if the backend hasn't computed it
|
| 27 |
+
// for the snapshot in use; guard every read so a partial payload
|
| 28 |
+
// renders as "β" instead of crashing the route.
|
| 29 |
+
const repro = aggregates.reproducibility?.overall ?? null
|
| 30 |
+
const comp = aggregates.completeness?.overall ?? null
|
| 31 |
+
const prov = aggregates.provenance?.overall ?? null
|
| 32 |
+
const cmp = aggregates.comparability?.overall ?? null
|
| 33 |
|
| 34 |
// Invert the gap rate to read as "documented", which matches reader intuition.
|
| 35 |
+
const reproGapRate = repro?.reproducibility_gap_rate ?? null
|
| 36 |
+
const reproDocumented = reproGapRate == null ? null : Math.max(0, 1 - reproGapRate)
|
| 37 |
+
const reproDetail = topMissingFields(repro?.per_field_missingness ?? {}, 2)
|
| 38 |
+
|
| 39 |
+
const totalReports = prov?.total_triples ?? 0
|
| 40 |
+
const sourceDist = prov?.source_type_distribution
|
| 41 |
+
const tpShare = totalReports > 0 && sourceDist?.third_party != null
|
| 42 |
+
? sourceDist.third_party / totalReports
|
| 43 |
+
: null
|
| 44 |
+
const fpShare = totalReports > 0 && sourceDist?.first_party != null
|
| 45 |
+
? sourceDist.first_party / totalReports
|
| 46 |
+
: null
|
| 47 |
+
|
| 48 |
+
const multiSourceRate = rate(prov?.multi_source_triples, prov?.total_triples)
|
| 49 |
+
const cmpRate = rate(cmp?.variant_divergent_count, cmp?.groups_with_variant_check)
|
| 50 |
const crossPartyRate = rate(
|
| 51 |
+
cmp?.cross_party_divergent_count,
|
| 52 |
+
cmp?.groups_with_cross_party_check,
|
| 53 |
)
|
| 54 |
+
const crossPartyAvailable = (cmp?.groups_with_cross_party_check ?? 0) > 0
|
| 55 |
+
const variantDivergent = cmp?.variant_divergent_count ?? null
|
| 56 |
+
const variantEligible = cmp?.groups_with_variant_check ?? null
|
| 57 |
+
const completenessTotal = comp?.total_triples ?? null
|
| 58 |
|
| 59 |
return (
|
| 60 |
<div className="signals-grid">
|
|
|
|
| 65 |
headline="of reported scores have a complete setup recorded β the rest cannot be independently re-run."
|
| 66 |
detail={
|
| 67 |
reproDetail
|
| 68 |
+
? `${formatPct(reproGapRate)} have at least one undocumented field. Most often missing: ${reproDetail}.`
|
| 69 |
+
: `${formatPct(reproGapRate)} have at least one undocumented field.`
|
| 70 |
}
|
| 71 |
asks="Can someone else run this evaluation and get the same number?"
|
| 72 |
/>
|
| 73 |
<SignalTile
|
| 74 |
id="completeness"
|
| 75 |
+
statValue={pctNum(comp?.completeness_avg)}
|
| 76 |
statUnit="%"
|
| 77 |
+
headline={
|
| 78 |
+
completenessTotal != null
|
| 79 |
+
? `mean across ${completenessTotal.toLocaleString()} reported score triples.`
|
| 80 |
+
: "mean across reported score triples."
|
| 81 |
+
}
|
| 82 |
+
detail={`Observed range: ${formatPct(comp?.completeness_min)} to ${formatPct(comp?.completeness_max)}.`}
|
| 83 |
asks="Is the benchmark itself documented well enough to interpret a score on it?"
|
| 84 |
/>
|
| 85 |
<SignalTile
|
|
|
|
| 94 |
id="comparability"
|
| 95 |
statValue={pctNum(cmpRate)}
|
| 96 |
statUnit="%"
|
| 97 |
+
headline={
|
| 98 |
+
variantEligible != null && variantDivergent != null
|
| 99 |
+
? `of setup-eligible groups diverge across variants (${variantDivergent.toLocaleString()} of ${variantEligible.toLocaleString()}).`
|
| 100 |
+
: "of setup-eligible groups diverge across variants."
|
| 101 |
+
}
|
| 102 |
detail={
|
| 103 |
crossPartyAvailable
|
| 104 |
? `Cross-party divergence: ${formatPct(crossPartyRate)}.`
|