j-chim commited on
Commit
2b5a024
Β·
1 Parent(s): 8a710d4

Update verified counts for grey validated orgs

Browse files
app/evaluators/[...id]/page.tsx CHANGED
@@ -91,9 +91,14 @@ function EvaluatorDetailInner() {
91
  // intersect this evaluator's set), so the header agrees with the accordion
92
  // below it rather than the finer family_display_name grouping.
93
  const { familyCount, verifiedCount } = useMemo(() => {
 
 
 
 
 
94
  let verified = 0
95
  for (const ev of evals) {
96
- if (name && (ev.verified_evaluator_names ?? []).includes(name)) verified += 1
97
  }
98
  let familyCount = 0
99
  for (const fam of families) {
 
91
  // intersect this evaluator's set), so the header agrees with the accordion
92
  // below it rather than the finer family_display_name grouping.
93
  const { familyCount, verifiedCount } = useMemo(() => {
94
+ // "Verified" spans both trust tiers: blue (org verified for this eval) and
95
+ // grey (a recognized source β€” the tier applies to all of the org's evals).
96
+ // So a recognized-only org like Hugging Face reports its full eval count
97
+ // rather than 0. Mirrors getEvalsForEvaluator's verified-only filter.
98
+ const recognized = isRecognizedEvaluator(name)
99
  let verified = 0
100
  for (const ev of evals) {
101
+ if (name && (recognized || (ev.verified_evaluator_names ?? []).includes(name))) verified += 1
102
  }
103
  let familyCount = 0
104
  for (const fam of families) {
components/evaluator-table.tsx CHANGED
@@ -105,9 +105,17 @@ export function EvaluatorTable({ rows, sortCol, sortDir, onSort, verifiedOnly }:
105
  {!verifiedOnly && (
106
  <td className="num font-mono text-[13px]">
107
  {row.verifiedCount > 0 ? (
108
- <span className="inline-flex items-center gap-1 text-[color:var(--accent)]">
 
 
 
109
  {row.verifiedCount.toLocaleString()}
110
- <VerifiedBadge verified size="sm" withTooltip={false} />
 
 
 
 
 
111
  </span>
112
  ) : (
113
  <span className="text-[color:var(--fg-subtle)]">β€”</span>
 
105
  {!verifiedOnly && (
106
  <td className="num font-mono text-[13px]">
107
  {row.verifiedCount > 0 ? (
108
+ <span
109
+ className="inline-flex items-center gap-1"
110
+ style={{ color: row.isVerified ? "var(--accent)" : "var(--fg-muted)" }}
111
+ >
112
  {row.verifiedCount.toLocaleString()}
113
+ <VerifiedBadge
114
+ verified={row.isVerified}
115
+ recognized={isRecognizedEvaluator(row.name)}
116
+ size="sm"
117
+ withTooltip={false}
118
+ />
119
  </span>
120
  ) : (
121
  <span className="text-[color:var(--fg-subtle)]">β€”</span>
lib/evaluators.ts CHANGED
@@ -50,7 +50,9 @@ export interface EvaluatorGroup {
50
  slug: string
51
  /** Number of evals this org reported (respecting the verified filter). */
52
  evalCount: number
53
- /** Number of those evals where this org is a *verified* evaluator. */
 
 
54
  verifiedCount: number
55
  /** True when the org is a verified evaluator for β‰₯1 eval (regardless of filter). */
56
  isVerified: boolean
@@ -137,16 +139,18 @@ export function groupEvalsByEvaluator(
137
  const org = (raw ?? "").trim()
138
  if (!org) continue
139
  const isVerifiedHere = verifiedSet.has(org)
 
140
  // In verified-only mode, the membership counts when the org carries a
141
  // trust badge for this eval β€” either blue (verified *for this eval*) or
142
  // grey (a recognized source). Both are surfaced.
143
- if (verifiedOnly && !isVerifiedHere && !isRecognizedEvaluator(org)) continue
144
  const cur = acc.get(org) ?? { evalCount: 0, verifiedCount: 0, isVerified: false }
145
  cur.evalCount += 1
146
- if (isVerifiedHere) {
147
- cur.verifiedCount += 1
148
- cur.isVerified = true
149
- }
 
150
  acc.set(org, cur)
151
  }
152
  }
 
50
  slug: string
51
  /** Number of evals this org reported (respecting the verified filter). */
52
  evalCount: number
53
+ /** Number of those evals carrying a trust badge for this org β€” blue
54
+ * (verified evaluator for the eval) or grey (org is a recognized source).
55
+ * For a recognized-only org this equals evalCount. */
56
  verifiedCount: number
57
  /** True when the org is a verified evaluator for β‰₯1 eval (regardless of filter). */
58
  isVerified: boolean
 
139
  const org = (raw ?? "").trim()
140
  if (!org) continue
141
  const isVerifiedHere = verifiedSet.has(org)
142
+ const recognized = isRecognizedEvaluator(org)
143
  // In verified-only mode, the membership counts when the org carries a
144
  // trust badge for this eval β€” either blue (verified *for this eval*) or
145
  // grey (a recognized source). Both are surfaced.
146
+ if (verifiedOnly && !isVerifiedHere && !recognized) continue
147
  const cur = acc.get(org) ?? { evalCount: 0, verifiedCount: 0, isVerified: false }
148
  cur.evalCount += 1
149
+ // verifiedCount spans both trust tiers (blue verified-here or grey
150
+ // recognized) so a recognized-only org reports its full count, not 0.
151
+ // isVerified stays blue-only β€” it drives the blue vs grey badge tier.
152
+ if (isVerifiedHere || recognized) cur.verifiedCount += 1
153
+ if (isVerifiedHere) cur.isVerified = true
154
  acc.set(org, cur)
155
  }
156
  }
tests/evaluators.test.ts CHANGED
@@ -67,7 +67,10 @@ describe("groupEvalsByEvaluator", () => {
67
  const groups = groupEvalsByEvaluator(withGrey, { verifiedOnly: true })
68
  const aa = groups.find((g) => g.name === "Artificial Analysis")!
69
  expect(aa.evalCount).toBe(2)
70
- expect(aa.verifiedCount).toBe(0) // grey is not blue
 
 
 
71
  // OpenAI is neither verified nor recognized β†’ still absent.
72
  expect(groups.find((g) => g.name === "OpenAI")).toBeUndefined()
73
  })
 
67
  const groups = groupEvalsByEvaluator(withGrey, { verifiedOnly: true })
68
  const aa = groups.find((g) => g.name === "Artificial Analysis")!
69
  expect(aa.evalCount).toBe(2)
70
+ // Grey recognized tier counts toward verifiedCount (both g1 & g2), but the
71
+ // org is not blue-verified so the badge tier stays grey.
72
+ expect(aa.verifiedCount).toBe(2)
73
+ expect(aa.isVerified).toBe(false)
74
  // OpenAI is neither verified nor recognized β†’ still absent.
75
  expect(groups.find((g) => g.name === "OpenAI")).toBeUndefined()
76
  })