Spaces:
Running
Running
Update verified list
Browse files- lib/evaluators.ts +8 -4
- tests/evaluators.test.ts +16 -0
lib/evaluators.ts
CHANGED
|
@@ -137,9 +137,10 @@ export function groupEvalsByEvaluator(
|
|
| 137 |
const org = (raw ?? "").trim()
|
| 138 |
if (!org) continue
|
| 139 |
const isVerifiedHere = verifiedSet.has(org)
|
| 140 |
-
// In verified-only mode, the membership
|
| 141 |
-
//
|
| 142 |
-
|
|
|
|
| 143 |
const cur = acc.get(org) ?? { evalCount: 0, verifiedCount: 0, isVerified: false }
|
| 144 |
cur.evalCount += 1
|
| 145 |
if (isVerifiedHere) {
|
|
@@ -196,6 +197,7 @@ export function getEvalsForEvaluator(
|
|
| 196 |
if (!name) return { name: null, isVerified: false, evals: [] }
|
| 197 |
|
| 198 |
let isVerified = false
|
|
|
|
| 199 |
const evals = allEvals.filter((ev) => {
|
| 200 |
// Exclude slices β evaluator surfaces show root benchmarks only.
|
| 201 |
if (ev.is_slice) return false
|
|
@@ -203,7 +205,9 @@ export function getEvalsForEvaluator(
|
|
| 203 |
if (!orgs.includes(name)) return false
|
| 204 |
const verifiedHere = (ev.verified_evaluator_names ?? []).includes(name)
|
| 205 |
if (verifiedHere) isVerified = true
|
| 206 |
-
|
|
|
|
|
|
|
| 207 |
return true
|
| 208 |
})
|
| 209 |
|
|
|
|
| 137 |
const org = (raw ?? "").trim()
|
| 138 |
if (!org) continue
|
| 139 |
const isVerifiedHere = verifiedSet.has(org)
|
| 140 |
+
// In verified-only mode, the membership counts when the org carries a
|
| 141 |
+
// trust badge for this eval β either blue (verified *for this eval*) or
|
| 142 |
+
// grey (a recognized source). Both are surfaced.
|
| 143 |
+
if (verifiedOnly && !isVerifiedHere && !isRecognizedEvaluator(org)) continue
|
| 144 |
const cur = acc.get(org) ?? { evalCount: 0, verifiedCount: 0, isVerified: false }
|
| 145 |
cur.evalCount += 1
|
| 146 |
if (isVerifiedHere) {
|
|
|
|
| 197 |
if (!name) return { name: null, isVerified: false, evals: [] }
|
| 198 |
|
| 199 |
let isVerified = false
|
| 200 |
+
const recognized = isRecognizedEvaluator(name)
|
| 201 |
const evals = allEvals.filter((ev) => {
|
| 202 |
// Exclude slices β evaluator surfaces show root benchmarks only.
|
| 203 |
if (ev.is_slice) return false
|
|
|
|
| 205 |
if (!orgs.includes(name)) return false
|
| 206 |
const verifiedHere = (ev.verified_evaluator_names ?? []).includes(name)
|
| 207 |
if (verifiedHere) isVerified = true
|
| 208 |
+
// Verified-only keeps both trust tiers: blue (verified for this eval) and
|
| 209 |
+
// grey (recognized source β applies to all of the org's evals).
|
| 210 |
+
if (verifiedOnly) return verifiedHere || recognized
|
| 211 |
return true
|
| 212 |
})
|
| 213 |
|
tests/evaluators.test.ts
CHANGED
|
@@ -55,6 +55,22 @@ describe("groupEvalsByEvaluator", () => {
|
|
| 55 |
// OpenAI never verified β absent
|
| 56 |
expect(groups.find((g) => g.name === "OpenAI")).toBeUndefined()
|
| 57 |
})
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 58 |
})
|
| 59 |
|
| 60 |
describe("buildEvaluatorSlugMap", () => {
|
|
|
|
| 55 |
// OpenAI never verified β absent
|
| 56 |
expect(groups.find((g) => g.name === "OpenAI")).toBeUndefined()
|
| 57 |
})
|
| 58 |
+
|
| 59 |
+
it("verified-only also keeps recognized (grey-tier) sources", () => {
|
| 60 |
+
// "Artificial Analysis" is a recognized source β grey badge, never in
|
| 61 |
+
// verified_evaluator_names β so it must still surface under verified-only.
|
| 62 |
+
const withGrey = [
|
| 63 |
+
...corpus,
|
| 64 |
+
evalRow("g1", ["Artificial Analysis"]),
|
| 65 |
+
evalRow("g2", ["Artificial Analysis", "OpenAI"]),
|
| 66 |
+
]
|
| 67 |
+
const groups = groupEvalsByEvaluator(withGrey, { verifiedOnly: true })
|
| 68 |
+
const aa = groups.find((g) => g.name === "Artificial Analysis")!
|
| 69 |
+
expect(aa.evalCount).toBe(2)
|
| 70 |
+
expect(aa.verifiedCount).toBe(0) // grey is not blue
|
| 71 |
+
// OpenAI is neither verified nor recognized β still absent.
|
| 72 |
+
expect(groups.find((g) => g.name === "OpenAI")).toBeUndefined()
|
| 73 |
+
})
|
| 74 |
})
|
| 75 |
|
| 76 |
describe("buildEvaluatorSlugMap", () => {
|