j-chim commited on
Commit
8a710d4
Β·
1 Parent(s): f7f3c2c

Update verified list

Browse files
Files changed (2) hide show
  1. lib/evaluators.ts +8 -4
  2. tests/evaluators.test.ts +16 -0
lib/evaluators.ts CHANGED
@@ -137,9 +137,10 @@ export function groupEvalsByEvaluator(
137
  const org = (raw ?? "").trim()
138
  if (!org) continue
139
  const isVerifiedHere = verifiedSet.has(org)
140
- // In verified-only mode, the membership itself only counts when the
141
- // org is verified *for this eval*.
142
- if (verifiedOnly && !isVerifiedHere) continue
 
143
  const cur = acc.get(org) ?? { evalCount: 0, verifiedCount: 0, isVerified: false }
144
  cur.evalCount += 1
145
  if (isVerifiedHere) {
@@ -196,6 +197,7 @@ export function getEvalsForEvaluator(
196
  if (!name) return { name: null, isVerified: false, evals: [] }
197
 
198
  let isVerified = false
 
199
  const evals = allEvals.filter((ev) => {
200
  // Exclude slices β€” evaluator surfaces show root benchmarks only.
201
  if (ev.is_slice) return false
@@ -203,7 +205,9 @@ export function getEvalsForEvaluator(
203
  if (!orgs.includes(name)) return false
204
  const verifiedHere = (ev.verified_evaluator_names ?? []).includes(name)
205
  if (verifiedHere) isVerified = true
206
- if (verifiedOnly) return verifiedHere
 
 
207
  return true
208
  })
209
 
 
137
  const org = (raw ?? "").trim()
138
  if (!org) continue
139
  const isVerifiedHere = verifiedSet.has(org)
140
+ // In verified-only mode, the membership counts when the org carries a
141
+ // trust badge for this eval β€” either blue (verified *for this eval*) or
142
+ // grey (a recognized source). Both are surfaced.
143
+ if (verifiedOnly && !isVerifiedHere && !isRecognizedEvaluator(org)) continue
144
  const cur = acc.get(org) ?? { evalCount: 0, verifiedCount: 0, isVerified: false }
145
  cur.evalCount += 1
146
  if (isVerifiedHere) {
 
197
  if (!name) return { name: null, isVerified: false, evals: [] }
198
 
199
  let isVerified = false
200
+ const recognized = isRecognizedEvaluator(name)
201
  const evals = allEvals.filter((ev) => {
202
  // Exclude slices β€” evaluator surfaces show root benchmarks only.
203
  if (ev.is_slice) return false
 
205
  if (!orgs.includes(name)) return false
206
  const verifiedHere = (ev.verified_evaluator_names ?? []).includes(name)
207
  if (verifiedHere) isVerified = true
208
+ // Verified-only keeps both trust tiers: blue (verified for this eval) and
209
+ // grey (recognized source β€” applies to all of the org's evals).
210
+ if (verifiedOnly) return verifiedHere || recognized
211
  return true
212
  })
213
 
tests/evaluators.test.ts CHANGED
@@ -55,6 +55,22 @@ describe("groupEvalsByEvaluator", () => {
55
  // OpenAI never verified β†’ absent
56
  expect(groups.find((g) => g.name === "OpenAI")).toBeUndefined()
57
  })
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
58
  })
59
 
60
  describe("buildEvaluatorSlugMap", () => {
 
55
  // OpenAI never verified β†’ absent
56
  expect(groups.find((g) => g.name === "OpenAI")).toBeUndefined()
57
  })
58
+
59
+ it("verified-only also keeps recognized (grey-tier) sources", () => {
60
+ // "Artificial Analysis" is a recognized source β€” grey badge, never in
61
+ // verified_evaluator_names β€” so it must still surface under verified-only.
62
+ const withGrey = [
63
+ ...corpus,
64
+ evalRow("g1", ["Artificial Analysis"]),
65
+ evalRow("g2", ["Artificial Analysis", "OpenAI"]),
66
+ ]
67
+ const groups = groupEvalsByEvaluator(withGrey, { verifiedOnly: true })
68
+ const aa = groups.find((g) => g.name === "Artificial Analysis")!
69
+ expect(aa.evalCount).toBe(2)
70
+ expect(aa.verifiedCount).toBe(0) // grey is not blue
71
+ // OpenAI is neither verified nor recognized β†’ still absent.
72
+ expect(groups.find((g) => g.name === "OpenAI")).toBeUndefined()
73
+ })
74
  })
75
 
76
  describe("buildEvaluatorSlugMap", () => {