Spaces:
Running
Running
| <html lang="en"> | |
| <head> | |
| <meta charset="utf-8"> | |
| <link rel="icon" href="favicon.svg"> | |
| <meta name="viewport" content="width=device-width, initial-scale=1"> | |
| <title>HealthBench config check v2: our spaces vs OpenAI's published numbers</title> | |
| <style> | |
| :root{ | |
| --ivory:#FAF9F5; --slate:#141413; --gray-700:#3D3D3A; --gray-500:#87867F; | |
| --gray-300:#D1CFC5; --gray-150:#F0EEE6; --clay:#D97757; --oat:#E3DACC; | |
| --olive:#788C5D; | |
| } | |
| *{box-sizing:border-box} | |
| html{scroll-behavior:smooth} | |
| body{ | |
| margin:0; background:var(--ivory); color:var(--gray-700); | |
| font-family:system-ui,-apple-system,"Segoe UI",Roboto,sans-serif; | |
| font-size:15px; line-height:1.65; padding:56px 24px 120px; | |
| } | |
| .page{max-width:1100px; margin:0 auto; display:grid; | |
| grid-template-columns:200px minmax(0,1fr); gap:48px} | |
| h1,h2,h3,dt{font-family:ui-serif,Georgia,"Times New Roman",serif; font-weight:500; color:var(--slate)} | |
| h1{font-size:32px; line-height:1.2; margin:0} | |
| h2{font-size:22px; margin:40px 0 12px; scroll-margin-top:24px} | |
| h3{font-size:17px; margin:26px 0 8px} | |
| p{max-width:680px} | |
| a{color:inherit} | |
| code{font-family:ui-monospace,"SF Mono",Menlo,Consolas,monospace; font-size:13px} | |
| .eyebrow{font-family:ui-monospace,"SF Mono",Menlo,Consolas,monospace; | |
| text-transform:uppercase; letter-spacing:.08em; font-size:11px; color:var(--gray-500); margin:0 0 14px} | |
| .dateline{font-family:ui-monospace,"SF Mono",Menlo,Consolas,monospace; font-size:11px; color:var(--gray-500); margin-top:10px} | |
| header{margin-bottom:8px} | |
| nav{position:sticky; top:32px; align-self:start; font-size:13px} | |
| nav .navlinks{display:flex; flex-direction:column; border-left:2px solid var(--gray-300)} | |
| nav a{display:block; padding:5px 0 5px 14px; margin-left:-2px; | |
| border-left:2px solid transparent; color:var(--gray-700); text-decoration:none} | |
| nav a:hover{border-left-color:var(--slate); color:var(--slate)} | |
| nav a.l2{padding-left:26px; font-size:12px; color:var(--gray-500)} | |
| nav .files{margin-top:22px} | |
| nav .files .lbl{font-family:ui-monospace,Menlo,monospace; font-size:10px; text-transform:uppercase; | |
| letter-spacing:.08em; color:var(--gray-500); margin-bottom:6px} | |
| nav .files code{display:block; font-size:11px; color:var(--gray-500); word-break:break-all; margin-bottom:4px} | |
| table{border-collapse:collapse; width:100%; max-width:860px; margin:14px 0; font-size:13.5px} | |
| th,td{text-align:left; padding:8px 12px; border-bottom:1px solid var(--gray-300)} | |
| th{font-family:ui-monospace,Menlo,monospace; font-size:11px; text-transform:uppercase; | |
| letter-spacing:.05em; color:var(--gray-500); font-weight:600; border-bottom:1.5px solid var(--gray-300)} | |
| td.num{font-family:ui-monospace,Menlo,monospace; text-align:right} | |
| tr.ours{background:rgba(217,119,87,.09)} | |
| tr.ours td{font-weight:600; color:var(--slate)} | |
| tr.human td{color:var(--gray-500); font-style:italic} | |
| .tag{font-family:ui-monospace,Menlo,monospace; font-size:10px; padding:1px 6px; border-radius:4px; | |
| border:1px solid var(--gray-300); color:var(--gray-500); white-space:nowrap} | |
| .tag.same{border-color:var(--olive); color:var(--olive)} | |
| .tag.diff{border-color:var(--clay); color:var(--clay)} | |
| .callout{border:1.5px solid var(--oat); background:rgba(227,218,204,.35); border-radius:10px; | |
| padding:16px 18px; margin:20px 0; max-width:800px; display:flex; gap:14px} | |
| .callout .star{color:var(--clay); font-size:18px; line-height:1.4} | |
| .callout .ct{font-size:14px} | |
| .callout .ct b{color:var(--slate)} | |
| details{background:#fff; border:1px solid var(--gray-300); border-radius:10px; margin:10px 0; max-width:860px} | |
| summary{cursor:pointer; padding:12px 16px; font-family:ui-serif,Georgia,serif; font-size:15px; | |
| color:var(--slate); list-style:none; display:flex; align-items:center; gap:10px} | |
| summary::-webkit-details-marker{display:none} | |
| summary::before{content:"\25B8"; color:var(--clay); transition:transform .15s; font-size:13px} | |
| details[open] summary::before{transform:rotate(90deg)} | |
| summary .where{margin-left:auto; font-family:ui-monospace,Menlo,monospace; font-size:11px; color:var(--gray-500)} | |
| details .body{padding:2px 16px 14px; font-size:14px} | |
| details .body p{margin:8px 0} | |
| details .body pre{background:var(--gray-150); border-radius:6px; padding:12px 14px; overflow-x:auto; | |
| font-family:ui-monospace,Menlo,monospace; font-size:12px; line-height:1.5; margin:10px 0} | |
| .hl{color:var(--clay)} | |
| .ok{color:var(--olive); font-weight:600} | |
| .warn{color:var(--clay); font-weight:600} | |
| .bad{color:#B0533A; font-weight:600} | |
| .muted{color:var(--gray-500)} | |
| .note{font-size:12.5px; color:var(--gray-500); max-width:800px} | |
| .ladder{display:grid; grid-template-columns:repeat(auto-fill,minmax(184px,1fr)); gap:7px; max-width:800px; margin:14px 0} | |
| .chip{font-family:ui-monospace,Menlo,monospace; font-size:10.5px; border:1px solid var(--gray-300); | |
| border-radius:6px; padding:6px 9px; color:var(--gray-500)} | |
| .chip.hit{border-color:var(--olive); background:rgba(120,140,93,.10); color:var(--slate); font-weight:600} | |
| .chip.miss{border-color:var(--clay); background:rgba(217,119,87,.08); color:var(--slate); font-weight:600} | |
| .chip.done{border-color:var(--olive); background:rgba(120,140,93,.10); color:var(--slate); font-weight:600; | |
| text-decoration:line-through; text-decoration-color:var(--gray-500)} | |
| .chip.part{border-color:var(--gray-500); background:rgba(135,134,127,.10); color:var(--slate); font-weight:600} | |
| tr.new td{background:rgba(120,140,93,.10)} | |
| .vtag{font-family:ui-monospace,Menlo,monospace; font-size:10px; padding:1px 6px; border-radius:4px; | |
| border:1px solid var(--olive); color:var(--olive); white-space:nowrap; margin-left:6px} | |
| .ladder-legend{font-family:ui-monospace,Menlo,monospace; font-size:10.5px; color:var(--gray-500); margin-top:6px} | |
| .cmp-chart{width:100%; max-width:760px; height:auto; margin:10px 0 4px; font-family:system-ui} | |
| .tc-ax{font-size:11px; fill:var(--gray-500); font-family:ui-monospace,Menlo,monospace} | |
| .tc-ml{font-size:11px; fill:var(--slate); font-family:ui-monospace,Menlo,monospace} | |
| .tc-lg{font-size:11px; fill:var(--gray-700)} | |
| .tc-val{font-size:10px; fill:var(--slate); font-family:ui-monospace,Menlo,monospace} | |
| .tc-dl{font-size:10.5px; font-family:ui-monospace,Menlo,monospace; font-weight:600} | |
| @media (max-width:920px){ | |
| .page{grid-template-columns:1fr} | |
| nav{display:none} | |
| } | |
| </style> | |
| <style id="hbnav-css"> | |
| .hbnav{position:sticky;top:0;z-index:50;display:flex;flex-wrap:wrap;align-items:center;gap:4px; | |
| padding:8px 16px;margin:-56px -24px 40px;background:rgba(250,249,245,.94); | |
| backdrop-filter:saturate(180%) blur(8px);border-bottom:1px solid #D1CFC5; | |
| font-family:ui-monospace,"SF Mono",Menlo,Consolas,monospace;font-size:11.5px} | |
| .hbnav .home{color:#87867F;margin-right:6px;text-decoration:none} | |
| .hbnav .home:hover{color:#141413} | |
| .hbnav a.pill{color:#3D3D3A;text-decoration:none;padding:3px 9px;border:1px solid #D1CFC5;border-radius:6px} | |
| .hbnav a.pill:hover{border-color:#D97757;color:#141413} | |
| .hbnav a.pill.here{background:#141413;border-color:#141413;color:#FAF9F5} | |
| .hbnav .sp{flex:1} | |
| @media(max-width:640px){.hbnav{margin:-56px -24px 28px}} | |
| </style> | |
| </head> | |
| <body> | |
| <div class="hbnav"><a class="home" href="index.html">← HealthBench logs</a><a class="pill" href="matrix.html">Coverage matrix</a><a class="pill here" href="config-check-v2.html">Config check v2</a><a class="pill" href="config-check-v1.html">Config check v1</a><span class="sp"></span><a class="pill" href="viewer/index.html">Log viewer →</a></div> | |
| <div class="page"> | |
| <nav> | |
| <div class="navlinks"> | |
| <a href="#summary">Summary</a> | |
| <a href="#v2">v2: DeepSeek re-run</a> | |
| <a href="#v2-verdict" class="l2">What it fixed</a> | |
| <a href="#anchor">The anchor check</a> | |
| <a href="#baseline" class="l2">Physician baseline</a> | |
| <a href="#config">Config matrix</a> | |
| <a href="#judge">Issue 1: judge model</a> | |
| <a href="#effort">Issue 2: reasoning effort</a> | |
| <a href="#headline">Issue 3: headline metric</a> | |
| <a href="#subscores">Issue 4: subscore clipping</a> | |
| <a href="#floored">Issue 5: floored zeros</a> | |
| <a href="#hygiene">Run hygiene</a> | |
| <a href="#fixes">What to change</a> | |
| <a href="#full">Full results</a> | |
| <a href="#method">How this was checked</a> | |
| </div> | |
| <div class="files"> | |
| <div class="lbl">Run artifacts</div> | |
| <code>hb-spaces/hs-non-professional/logs/</code> | |
| <code>hb-spaces/hs-hard/logs/</code> | |
| <code>hb-spaces/hs-consensus/logs/</code> | |
| <code>hb-spaces/hs-prof-subsets/logs/</code> | |
| <code>hb-spaces/healthbench-professional-deepseek-v4-pro/logs/</code> | |
| <div class="lbl" style="margin-top:14px">Reference</div> | |
| <code>arXiv:2505.08775 (HealthBench)</code> | |
| <code>arXiv:2604.27470 (Professional)</code> | |
| <code>GPT-5.6 system card, Table 6</code> | |
| <code>openai/simple-evals</code> | |
| </div> | |
| </nav> | |
| <main> | |
| <header> | |
| <p class="eyebrow">HealthBench · config sanity check · v2</p> | |
| <h1>HealthBench config check: our spaces vs OpenAI's published numbers</h1> | |
| <div class="dateline">Created 2026-07-27 · v2 2026-08-06 (DeepSeek Professional re-run folded in)</div> | |
| </header> | |
| <section id="summary"> | |
| <div class="callout"> | |
| <span class="star">★</span> | |
| <span class="ct"><b>What is new in v2 (2026-08-06).</b> One re-run has landed since v1: DeepSeek v4 Pro on | |
| HealthBench Professional, published as its own space. It is folded in below. Two things change as a result. | |
| DeepSeek's Professional overall moves from <b>27.66 to 30.99</b>, because its old consult figure turned out | |
| to be a cache replay rather than a real run. And <a href="#subscores">issue 4</a> is now diagnosed exactly: | |
| the inflated per-use-case subscores are not clipped versions of the length-adjusted score, they discard the | |
| length adjustment altogether. Nothing else in v1 changes. The two flags that matter most for comparability | |
| — <b>eight samples per example</b> and <b>reasoning effort</b> — are still unset everywhere, | |
| including in the re-run.</span> | |
| </div> | |
| <p> | |
| We have a real anchor for this check: OpenAI's GPT-5.6 system card publishes <b>gpt-5.5 scores for all four | |
| HealthBench variants</b>, and gpt-5.5 is one of the models we ran. Lining our numbers up against theirs | |
| gives a per-variant verdict rather than a guess. | |
| </p> | |
| <p> | |
| The result splits cleanly. <b>The two spaces where we used the correct grader reproduce OpenAI within a | |
| point.</b> The full HealthBench run lands at <b>55.8</b> against their <b>56.5</b>, and our HealthBench | |
| Professional physician baseline lands at <b>43.87</b> against their published <b>43.7</b>. That is close | |
| enough to say the dataset, prompting, rubric scoring, length adjustment and aggregation are all wired up | |
| right. | |
| </p> | |
| <p> | |
| <b>The two spaces where we used the wrong grader are badly off.</b> HealthBench Hard is <b>5.5 points low</b> | |
| and HealthBench Consensus is <b>13.6 points low</b>. Both were graded by <code>gpt-4o-mini</code> instead of | |
| GPT-4.1, not by choice but because the <code>inspect_evals</code> task wrappers for those two variants do not | |
| expose a <code>judge_model</code> argument and silently fall back to the package default. Consensus is the | |
| loudest signal: every frontier model in OpenAI's table sits in a 94 to 96 band, and ours sits at 82. | |
| </p> | |
| <p> | |
| A separate, smaller gap shows up on the inference side. Our gpt-5.5 Professional score is <b>4 points below</b> | |
| OpenAI's, even though the grader is correct there. The physician baseline matching to 0.2 points rules out | |
| the grading pipeline, which points the finger at <b>reasoning effort</b>: we pass none, while OpenAI evaluates | |
| "at the highest reasoning effort option available via each model's API." Claude was run with extended thinking | |
| off entirely. | |
| </p> | |
| <div class="callout"> | |
| <span class="star">★</span> | |
| <span class="ct"><b>The one-line verdict.</b> The scoring machinery is correct and provably so. The | |
| configuration around it is not: two variants use a mini-tier grader, no run sets reasoning effort, and three | |
| of the four spaces show the unadjusted score as the headline where OpenAI shows the length-adjusted one. | |
| None of this requires re-implementing anything, only re-running with four flags set.</span> | |
| </div> | |
| </section> | |
| <section id="v2"> | |
| <h2>v2 update: the DeepSeek Professional re-run</h2> | |
| <p> | |
| Ajay re-ran HealthBench Professional for DeepSeek on 2026-08-06 and published it as its own space, | |
| <code>ajay-citadel/healthbench-professional-deepseek-v4-pro</code>. It is the first run that addresses | |
| anything on the <a href="#fixes">what to change</a> list, so this version of the report folds it in and | |
| corrects the numbers it supersedes. Everything else in v1 stands unchanged. | |
| </p> | |
| <p> | |
| The re-run is a <b>single combined task over all 525 examples</b> rather than five separate subset tasks, | |
| which is what OpenAI's reference implementation does. That removes the manual sample-weighting v1 had to | |
| apply, and it gives DeepSeek a genuine <code>consult</code> run for the first time. | |
| </p> | |
| <table> | |
| <thead><tr><th>DeepSeek v4 Pro, Professional</th><th class="num">v1 (hs-prof-subsets)</th><th class="num">v2 (re-run)</th><th class="num">Δ</th><th>note</th></tr></thead> | |
| <tbody> | |
| <tr class="new"><td><b>Overall, length-adjusted</b></td><td class="num">27.66</td><td class="num">30.99</td><td class="num" style="color:#788C5D">+3.3</td><td class="note">v1 was stitched from subsets; v2 is native</td></tr> | |
| <tr><td>Overall, unadjusted</td><td class="num muted">n/a</td><td class="num">37.81</td><td class="num muted">—</td><td class="note">not computable from v1 subsets</td></tr> | |
| <tr><td>Care consult</td><td class="num">25.59</td><td class="num">31.83</td><td class="num" style="color:#788C5D">+6.2</td><td class="note">v1 figure was a cache replay, see below</td></tr> | |
| <tr><td>Writing and documentation</td><td class="num">4.95</td><td class="num">6.70</td><td class="num">+1.8</td><td class="note">agrees within sampling noise</td></tr> | |
| <tr><td>Medical research</td><td class="num">52.91</td><td class="num">53.10</td><td class="num">+0.2</td><td class="note">agrees</td></tr> | |
| <tr><td>Red teaming (unclipped)</td><td class="num">−6.87</td><td class="num">−5.19</td><td class="num">+1.7</td><td class="note">still floored to 0.00 in the log</td></tr> | |
| <tr><td>Mean response length (chars)</td><td class="num muted">3069–5665</td><td class="num">4320</td><td class="num muted">—</td><td class="note">vs 3818 for gpt-5.5 in OpenAI's table</td></tr> | |
| </tbody> | |
| </table> | |
| <p class="note"> | |
| The v1 sample-weighted arithmetic checks out against the native number: (236×31.83 + 142×6.70 + | |
| 147×53.10) / 525 = 30.99, exactly the re-run's own <code>bootstrap_score</code>. So the stitching method | |
| v1 used was sound; the 3.3-point move comes entirely from the consult subset being re-generated rather than | |
| replayed. DeepSeek does not change rank — it stays third behind claude-opus-4-7 (48.02) and gpt-5.5 (47.81). | |
| </p> | |
| <h3 id="v2-verdict">What the re-run fixed, and what it did not</h3> | |
| <table> | |
| <thead><tr><th>v1 finding</th><th>status in the re-run</th><th>evidence</th></tr></thead> | |
| <tbody> | |
| <tr><td>DeepSeek consult had no real epochs≥1 run of its own</td><td><span class="tag same">fixed</span></td><td class="note">967k output tokens billed to deepseek; the v1 epochs=1 logs report zero model usage</td></tr> | |
| <tr><td>Duplicate, conflicting DeepSeek entries in <code>listing.json</code></td><td><span class="tag same">fixed</span></td><td class="note">one log, one task, its own space</td></tr> | |
| <tr><td>Grader config implicit (relied on the package default)</td><td><span class="tag same">fixed</span></td><td class="note"><code>judge_model: openai/gpt-5.4</code>, <code>judge_reasoning_effort: low</code> now explicit in <code>task_args</code></td></tr> | |
| <tr><td>Headline metric should be length-adjusted</td><td><span class="tag same">already ok</span></td><td class="note"><code>use_length_adjusted: true</code> on the primary metric</td></tr> | |
| <tr><td><b>Samples per example should be 8</b></td><td><span class="tag diff">not fixed</span></td><td class="note"><code>epochs: 1</code>. Bootstrap std is 2.8 points on a 31-point score</td></tr> | |
| <tr><td><b>Reasoning effort should be the highest available</b></td><td><span class="tag diff">not fixed</span></td><td class="note">generate config is <code>{cache: true}</code>; 69.7% of samples reasoned, at DeepSeek's own default</td></tr> | |
| <tr><td><b>Per-use-case subscores are inflated</b></td><td><span class="tag diff">not fixed</span></td><td class="note">reproduced exactly, and the mechanism is now pinned — see <a href="#subscores">issue 4</a></td></tr> | |
| </tbody> | |
| </table> | |
| <div class="callout"> | |
| <span class="star">★</span> | |
| <span class="ct"><b>Verdict on the re-run: clean, and it moves the number, but it is not yet the paper's | |
| configuration.</b> The grading half is exactly OpenAI's internal reference — gpt-5.4-2026-03-05 at low | |
| reasoning effort, temperature 0, "You are a helpful assistant." — and every score in the log | |
| re-aggregates to the reported value from per-sample data. The inference half still runs one sample per | |
| example instead of eight and passes no reasoning effort, so the two headline items from | |
| <a href="#effort">issue 2</a> and the Professional row of <a href="#config">the config matrix</a> remain | |
| open for DeepSeek exactly as they do for every other model.</span> | |
| </div> | |
| <details> | |
| <summary>Sanity checks run against the new log <span class="where">525 samples, 0 errors</span></summary> | |
| <div class="body"> | |
| <pre>reported bootstrap_score 0.3099 <span class="muted"># length-adjusted</span> | |
| recomputed mean(adjusted) 0.3099 <span class="muted"># match</span> | |
| recomputed mean(unadjusted) 0.3781 <span class="muted"># matches the log's own mean metric</span> | |
| bootstrap 1000x, own resample 0.3104 +/- 0.0283 <span class="muted"># log: 0.3099 +/- 0.0279</span> | |
| criteria_met_rate 0.5921 <span class="muted"># match; 1135 criteria over 525 samples</span> | |
| length adjustment: adj = raw - 0.0147 * ((chars - 2000) / 500) | |
| mismatches over 525 samples 0 <span class="muted"># arithmetic verified per sample</span> | |
| served model deepseek/deepseek-v4-pro <span class="muted"># matches requested, no silent substitution</span> | |
| judge served gpt-5.4-2026-03-05 <span class="muted"># 1135 calls, one per rubric criterion</span> | |
| judge config {system_message: "You are a helpful assistant.", temperature: 0.0, | |
| reasoning_effort: "low"} <span class="muted"># matches OpenAI's internal reference</span> | |
| candidate config {cache: true} <span class="muted"># no reasoning effort passed</span> | |
| rubric coverage 1135 graded / 1135 in dataset metadata <span class="muted"># no criteria dropped</span> | |
| unique prompt ids 525 / 525 | |
| empty completions 0 | |
| mean chars 4320 median 4053 p99 11620 max 17982 | |
| frac. samples with length-adjusted score < 0 0.303</pre> | |
| <p>The one number worth watching is that last line: 30% of DeepSeek's Professional responses score below zero | |
| once the length penalty applies, against 16.4% for the physician baseline. DeepSeek averages 4320 characters | |
| where the penalty centre is 2000, so it is paying about 6.8 points of pure verbosity tax. That is a real | |
| property of the model under this metric, not a config error, but it does mean DeepSeek's Professional score is | |
| unusually sensitive to the length-adjustment constant.</p> | |
| </div> | |
| </details> | |
| </section> | |
| <section id="anchor"> | |
| <h2>The anchor check</h2> | |
| <p> | |
| All numbers are the <b>length-adjusted score on the 0 to 100 scale</b>, which is what OpenAI reports as the | |
| primary metric. Their figures come from the GPT-5.6 system card Table 6, given as | |
| <code>length-adjusted (unadjusted, mean chars)</code>. Ours are recomputed from the downloaded | |
| <code>.eval</code> logs. | |
| </p> | |
| <svg viewBox="0 0 760 430" xmlns="http://www.w3.org/2000/svg" class="cmp-chart" role="img" | |
| aria-label="gpt-5.5 length-adjusted scores, OpenAI published versus our runs, across five checks"> | |
| <line x1="60" y1="40" x2="60" y2="340" stroke="#141413" stroke-width="1"/> | |
| <line x1="60" y1="340" x2="740" y2="340" stroke="#141413" stroke-width="1"/> | |
| <line x1="60" y1="280" x2="740" y2="280" stroke="#F0EEE6" stroke-width="1"/> | |
| <line x1="60" y1="220" x2="740" y2="220" stroke="#F0EEE6" stroke-width="1"/> | |
| <line x1="60" y1="160" x2="740" y2="160" stroke="#F0EEE6" stroke-width="1"/> | |
| <line x1="60" y1="100" x2="740" y2="100" stroke="#F0EEE6" stroke-width="1"/> | |
| <line x1="60" y1="40" x2="740" y2="40" stroke="#F0EEE6" stroke-width="1"/> | |
| <text x="52" y="343.5" text-anchor="end" class="tc-ax">0</text> | |
| <text x="52" y="283.5" text-anchor="end" class="tc-ax">20</text> | |
| <text x="52" y="223.5" text-anchor="end" class="tc-ax">40</text> | |
| <text x="52" y="163.5" text-anchor="end" class="tc-ax">60</text> | |
| <text x="52" y="103.5" text-anchor="end" class="tc-ax">80</text> | |
| <text x="52" y="43.5" text-anchor="end" class="tc-ax">100</text> | |
| <!-- group 0: physician baseline --> | |
| <rect x="88" y="208.9" width="40" height="131.1" fill="#87867F"/> | |
| <rect x="132" y="208.4" width="40" height="131.6" fill="#D97757"/> | |
| <text x="108" y="203" text-anchor="middle" class="tc-val">43.7</text> | |
| <text x="152" y="203" text-anchor="middle" class="tc-val">43.9</text> | |
| <text x="130" y="358" text-anchor="middle" class="tc-ml">Physician</text> | |
| <text x="130" y="372" text-anchor="middle" class="tc-ml">baseline</text> | |
| <text x="130" y="390" text-anchor="middle" class="tc-dl" fill="#788C5D">+0.2</text> | |
| <!-- group 1: professional --> | |
| <rect x="224" y="184.6" width="40" height="155.4" fill="#87867F"/> | |
| <rect x="268" y="196.6" width="40" height="143.4" fill="#D97757"/> | |
| <text x="244" y="178.6" text-anchor="middle" class="tc-val">51.8</text> | |
| <text x="288" y="190.6" text-anchor="middle" class="tc-val">47.8</text> | |
| <text x="266" y="358" text-anchor="middle" class="tc-ml">Professional</text> | |
| <text x="266" y="372" text-anchor="middle" class="tc-ax">grader ok</text> | |
| <text x="266" y="390" text-anchor="middle" class="tc-dl" fill="#D97757">−4.0</text> | |
| <!-- group 2: healthbench full --> | |
| <rect x="360" y="170.5" width="40" height="169.5" fill="#87867F"/> | |
| <rect x="404" y="172.5" width="40" height="167.5" fill="#D97757"/> | |
| <text x="380" y="164.5" text-anchor="middle" class="tc-val">56.5</text> | |
| <text x="424" y="166.5" text-anchor="middle" class="tc-val">55.8</text> | |
| <text x="402" y="358" text-anchor="middle" class="tc-ml">HealthBench</text> | |
| <text x="402" y="372" text-anchor="middle" class="tc-ax">grader ok</text> | |
| <text x="402" y="390" text-anchor="middle" class="tc-dl" fill="#788C5D">−0.7</text> | |
| <!-- group 3: hard --> | |
| <rect x="496" y="245.5" width="40" height="94.5" fill="#87867F"/> | |
| <rect x="540" y="261.9" width="40" height="78.1" fill="#D97757"/> | |
| <text x="516" y="239.5" text-anchor="middle" class="tc-val">31.5</text> | |
| <text x="560" y="255.9" text-anchor="middle" class="tc-val">26.0</text> | |
| <text x="538" y="358" text-anchor="middle" class="tc-ml">Hard</text> | |
| <text x="538" y="372" text-anchor="middle" class="tc-ax">wrong grader</text> | |
| <text x="538" y="390" text-anchor="middle" class="tc-dl" fill="#B0533A">−5.5</text> | |
| <!-- group 4: consensus --> | |
| <rect x="632" y="53.2" width="40" height="286.8" fill="#87867F"/> | |
| <rect x="676" y="93.9" width="40" height="246.1" fill="#D97757"/> | |
| <text x="652" y="47.2" text-anchor="middle" class="tc-val">95.6</text> | |
| <text x="696" y="87.9" text-anchor="middle" class="tc-val">82.0</text> | |
| <text x="674" y="358" text-anchor="middle" class="tc-ml">Consensus</text> | |
| <text x="674" y="372" text-anchor="middle" class="tc-ax">wrong grader</text> | |
| <text x="674" y="390" text-anchor="middle" class="tc-dl" fill="#B0533A">−13.6</text> | |
| <rect x="60" y="408" width="12" height="12" fill="#87867F"/> | |
| <text x="78" y="418" class="tc-lg">OpenAI published</text> | |
| <rect x="200" y="408" width="12" height="12" fill="#D97757"/> | |
| <text x="218" y="418" class="tc-lg">our run</text> | |
| <text x="320" y="418" class="tc-ax">gpt-5.5, length-adjusted score (0 to 100)</text> | |
| </svg> | |
| <table> | |
| <thead><tr><th>Check</th><th>Grader used</th><th class="num">OpenAI adj</th><th class="num">ours adj</th><th class="num">Δ</th><th class="num">OpenAI raw</th><th class="num">ours raw</th><th class="num">OpenAI chars</th><th class="num">ours chars</th></tr></thead> | |
| <tbody> | |
| <tr><td>Physician baseline (Prof)</td><td><span class="tag same">gpt-5.4 low</span></td><td class="num">43.7</td><td class="num">43.87</td><td class="num" style="color:#788C5D">+0.2</td><td class="num muted">n/a</td><td class="num">44.29</td><td class="num muted">n/a</td><td class="num muted">n/a</td></tr> | |
| <tr><td>HealthBench (full)</td><td><span class="tag same">gpt-4.1</span></td><td class="num">56.5</td><td class="num">55.83</td><td class="num" style="color:#788C5D">−0.7</td><td class="num">58.4</td><td class="num">56.87</td><td class="num">2313</td><td class="num">2175</td></tr> | |
| <tr><td>HealthBench Professional</td><td><span class="tag same">gpt-5.4 low</span></td><td class="num">51.8</td><td class="num">47.81</td><td class="num" style="color:#D97757">−4.0</td><td class="num">57.2</td><td class="num muted">n/a</td><td class="num">3818</td><td class="num">3748</td></tr> | |
| <tr><td>HealthBench Hard</td><td><span class="tag diff">gpt-4o-mini</span></td><td class="num">31.5</td><td class="num">26.03</td><td class="num" style="color:#B0533A">−5.5</td><td class="num">33.8</td><td class="num">27.27</td><td class="num">2289</td><td class="num">2158</td></tr> | |
| <tr><td>HealthBench Consensus</td><td><span class="tag diff">gpt-4o-mini</span></td><td class="num">95.6</td><td class="num">82.02</td><td class="num" style="color:#B0533A">−13.6</td><td class="num">95.7</td><td class="num">82.08</td><td class="num">2259</td><td class="num">2143</td></tr> | |
| </tbody> | |
| </table> | |
| <p class="note"> | |
| Our Professional overall is a sample-weighted mean of the three use-case subsets (consult 236, writing 142, | |
| research 147, total 525), since we ran them as separate tasks rather than one benchmark. Red teaming is a | |
| cross-cutting slice of those same 525 examples, not a fourth use case, so it is excluded from the weighting. | |
| </p> | |
| <p> | |
| Two things worth noticing beyond the scores. First, <b>mean response lengths track closely</b> (2175 vs 2313, | |
| 2158 vs 2289, 2143 vs 2259, 3748 vs 3818), consistently about 5% shorter but never structurally different. | |
| Since length is the one output property that would move if our prompting or harness diverged, this is good | |
| evidence the request side is faithful. Second, the two <span class="tag same">correct grader</span> rows and | |
| the two <span class="tag diff">wrong grader</span> rows separate perfectly, which is what makes the diagnosis | |
| confident rather than speculative. | |
| </p> | |
| <h3 id="baseline">The physician baseline is the cleanest check we have</h3> | |
| <p> | |
| HealthBench Professional ships a set of <b>525 physician-written responses</b>. Scoring them involves no model | |
| inference at all: the responses are fixed text, and the only moving part is the grader plus the aggregation. | |
| So if our number matches OpenAI's, the entire scoring half of the pipeline is verified, and any remaining | |
| discrepancy on real models has to live on the inference side. | |
| </p> | |
| <table> | |
| <thead><tr><th>Physician baseline</th><th class="num">OpenAI published</th><th class="num">ours (recomputed)</th><th class="num">Δ</th></tr></thead> | |
| <tbody> | |
| <tr class="ours"><td>Overall</td><td class="num">43.7</td><td class="num">43.87</td><td class="num">+0.17</td></tr> | |
| <tr><td>Care consult</td><td class="num">42.7</td><td class="num">42.54</td><td class="num">−0.16</td></tr> | |
| <tr><td>Writing and documentation</td><td class="num">32.1</td><td class="num">32.91</td><td class="num">+0.81</td></tr> | |
| <tr><td>Medical research</td><td class="num">56.3</td><td class="num">56.60</td><td class="num">+0.30</td></tr> | |
| </tbody> | |
| </table> | |
| <p class="note"> | |
| Every cell is within a point. Note that the per-use-case rows here are <b>recomputed from per-sample scores</b>, | |
| not read off the log's own subset metrics, which are inflated for a reason covered in | |
| <a href="#subscores">issue 4</a>. | |
| </p> | |
| <div class="callout"> | |
| <span class="star">★</span> | |
| <span class="ct"><b>This is the finding that makes the rest interpretable.</b> Because the judge-only path | |
| reproduces OpenAI to 0.2 points, the 4-point Professional gap on gpt-5.5 cannot be blamed on the grader, | |
| the rubric, the length adjustment, or the aggregation. It is an inference-side difference, and reasoning | |
| effort is the only inference-side knob that differs.</span> | |
| </div> | |
| </section> | |
| <section id="config"> | |
| <h2>Config matrix</h2> | |
| <p>What we set against what OpenAI's reference implementation sets, per variant.</p> | |
| <table> | |
| <thead><tr><th>Setting</th><th>OpenAI reference</th><th>hs-non-professional</th><th>hs-hard</th><th>hs-consensus</th><th>hs-prof-subsets</th><th>ds-v4-pro re-run <span class="vtag">v2</span></th></tr></thead> | |
| <tbody> | |
| <tr><td>Grader model</td><td>gpt-4.1 / gpt-5.4 low</td><td><span class="tag same">gpt-4.1</span></td><td><span class="tag diff">gpt-4o-mini</span></td><td><span class="tag diff">gpt-4o-mini</span></td><td><span class="tag same">gpt-5.4 low</span></td><td><span class="tag same">gpt-5.4 low</span></td></tr> | |
| <tr><td>Grader system msg</td><td>"You are a helpful assistant."</td><td><span class="tag same">same</span></td><td><span class="tag same">same</span></td><td><span class="tag same">same</span></td><td><span class="tag same">same</span></td><td><span class="tag same">same</span></td></tr> | |
| <tr><td>Length adj. center</td><td>2000 chars</td><td><span class="tag same">2000</span></td><td><span class="tag same">2000</span></td><td><span class="tag same">2000</span></td><td><span class="tag same">2000</span></td><td><span class="tag same">2000</span></td></tr> | |
| <tr><td>Length adj. penalty</td><td>2.99 / 3.92 / 0.20 / 1.47</td><td><span class="tag same">0.0299</span></td><td><span class="tag same">0.0392</span></td><td><span class="tag same">0.002</span></td><td><span class="tag same">0.0147</span></td><td><span class="tag same">0.0147</span></td></tr> | |
| <tr><td>Per-sample clipping</td><td>none (mean is clipped)</td><td><span class="tag same">none</span></td><td><span class="tag same">none</span></td><td><span class="tag same">none</span></td><td><span class="tag diff">subscores clipped</span></td><td><span class="tag diff">subscores clipped</span></td></tr> | |
| <tr><td>Dataset size</td><td>5000 / 1000 / 3671 / 525</td><td><span class="tag same">5000</span></td><td><span class="tag same">1000</span></td><td><span class="tag same">3671</span></td><td><span class="tag same">525</span></td><td><span class="tag same">525</span></td></tr> | |
| <tr><td>Samples per example</td><td>1 (main), 8 (Prof)</td><td><span class="tag same">1</span></td><td><span class="tag same">1</span></td><td><span class="tag same">1</span></td><td><span class="tag same">8</span> <span class="tag diff">1 for deepseek</span></td><td><span class="tag diff">1</span></td></tr> | |
| <tr><td>Reasoning effort</td><td>highest available</td><td><span class="tag diff">unset</span></td><td><span class="tag diff">unset</span></td><td><span class="tag diff">unset</span></td><td><span class="tag diff">unset</span></td><td><span class="tag diff">unset</span></td></tr> | |
| <tr><td>Headline metric</td><td>length-adjusted</td><td><span class="tag diff">unadjusted</span></td><td><span class="tag diff">unadjusted</span></td><td><span class="tag diff">unadjusted</span></td><td><span class="tag same">length-adjusted</span></td><td><span class="tag same">length-adjusted</span></td></tr> | |
| <tr><td>Task shape</td><td>one benchmark</td><td><span class="tag same">one task</span></td><td><span class="tag same">one task</span></td><td><span class="tag same">one task</span></td><td><span class="tag diff">5 subset tasks</span></td><td><span class="tag same">one task</span></td></tr> | |
| <tr><td>Bootstrap resamples</td><td>1000</td><td><span class="tag same">1000</span></td><td><span class="tag same">1000</span></td><td><span class="tag same">1000</span></td><td><span class="tag same">1000</span></td><td><span class="tag same">1000</span></td></tr> | |
| </tbody> | |
| </table> | |
| <p class="note"> | |
| <span class="ok">v2:</span> the re-run column is DeepSeek only. It closes the task-shape and explicit-grader | |
| gaps and leaves the two that matter most for score comparability — <b>samples per example</b> and | |
| <b>reasoning effort</b> — exactly where they were. | |
| </p> | |
| <p class="note"> | |
| The length adjustment is worth calling out as correct-by-verification, not just correct-by-declaration: we | |
| confirmed the arithmetic per sample against | |
| <code>score − penalty × ((len − center) / 500)</code>, OpenAI's | |
| <code>calculate_length_adjusted_score</code>. On one Hard sample: raw 0.6102, completion 13,776 chars, giving | |
| 0.6102 − 0.0392 × 23.552 = −0.3131, which is exactly the stored value. | |
| </p> | |
| </section> | |
| <section id="judge"> | |
| <h2>Issue 1: Hard and Consensus were graded by gpt-4o-mini</h2> | |
| <p> | |
| This is the largest error and it was not a deliberate choice. The <code>inspect_evals</code> task wrappers for | |
| these two variants accept only the length-adjustment arguments and pass nothing else through, so | |
| <code>judge_model</code> silently keeps the package default of <code>openai/gpt-4o-mini</code>: | |
| </p> | |
| <details> | |
| <summary>The wrapper that drops the judge argument <span class="where">inspect_evals/healthbench/healthbench.py</span></summary> | |
| <div class="body"> | |
| <pre>def healthbench( | |
| ... | |
| judge_model: str | Model = "openai/gpt-4o-mini", <span class="muted"># package default</span> | |
| ... | |
| ) | |
| @task | |
| def healthbench_hard( | |
| length_adjustment_center: float | None = None, | |
| length_adjustment_penalty_per_500_chars: float | None = None, | |
| ) -> Task: | |
| return healthbench( <span class="muted"># judge_model never forwarded</span> | |
| subset="hard", | |
| length_adjustment_center=length_adjustment_center, | |
| length_adjustment_penalty_per_500_chars=length_adjustment_penalty_per_500_chars, | |
| )</pre> | |
| <p>Our <code>hs-non-professional</code> run passed <code>judge_model: "openai/gpt-4.1"</code> explicitly and got | |
| it. The Hard and Consensus runs declared no judge at all in <code>task_args</code>, and inspecting the model | |
| events in the logs confirms every grading call went to <code>gpt-4o-mini-2024-07-18</code>.</p> | |
| </div> | |
| </details> | |
| <p> | |
| OpenAI never grades any variant with a mini-tier model. In <code>simple_evals.py</code> the same | |
| <code>healthbench_grading_sampler</code> is handed to <code>healthbench</code>, <code>healthbench_hard</code> | |
| and <code>healthbench_consensus</code> alike, so all three get GPT-4.1, or all three get GPT-5.4 low when the | |
| <code>--healthbench-use-gpt-5-4-low-grader</code> flag is set. | |
| </p> | |
| <p> | |
| The HealthBench paper measured what a weaker grader costs. Its meta-evaluation ranks candidate graders by | |
| agreement with physicians (Macro-F1): <b>GPT-4.1 0.709</b>, o4-mini 0.692, o3 0.681, GPT-4.1 mini 0.661, | |
| GPT-4.1 nano 0.580, and calls the smaller variants "substantially worse". <code>gpt-4o-mini</code> is not in | |
| that table but sits below GPT-4.1 mini in capability, so 0.661 is the optimistic bound. | |
| </p> | |
| <div class="callout"> | |
| <span class="star">★</span> | |
| <span class="ct"><b>Consensus is the smoking gun.</b> OpenAI's Consensus scores are 94 to 96 for every model | |
| from gpt-5 through gpt-5.6, because the subset is deliberately low-noise and near-saturated. Our whole | |
| Consensus space sits between <b>71 and 82</b>. That is not our models underperforming, that is the grader | |
| failing to recognise criteria that a stronger grader marks as met.</span> | |
| </div> | |
| </section> | |
| <section id="effort"> | |
| <h2>Issue 2: reasoning effort is never set, and the models diverge because of it</h2> | |
| <p> | |
| The HealthBench Professional paper states that models are "evaluated at the highest reasoning effort option | |
| available via each model's API (e.g., xhigh for GPT-5.4)", and measures a <b>5.6 to 7.3 point</b> gain moving | |
| from low to xhigh. None of our runs pass a reasoning effort, so each provider's default applies, and the | |
| defaults are not comparable to each other: | |
| </p> | |
| <table> | |
| <thead><tr><th>Model (HealthBench Hard, 1000 samples)</th><th class="num">frac. with reasoning</th><th class="num">mean reasoning tokens</th><th>generate config sent</th></tr></thead> | |
| <tbody> | |
| <tr><td>openai/gpt-5.5</td><td class="num">0.97</td><td class="num">291</td><td><code>{cache: true}</code></td></tr> | |
| <tr><td>openrouter/deepseek-v4-pro</td><td class="num">0.86</td><td class="num">489</td><td><code>{cache: true}</code></td></tr> | |
| <tr class="ours"><td>anthropic/claude-opus-4-7</td><td class="num">0.00</td><td class="num">0</td><td><code>{cache: true, max_tokens: 32000}</code></td></tr> | |
| <tr><td>plamo-3.0-prime</td><td class="num">0.00</td><td class="num">0</td><td><code>{cache: true}</code></td></tr> | |
| <tr><td>medgemma-4b / 27b</td><td class="num">0.00</td><td class="num">0</td><td><code>{cache: true}</code></td></tr> | |
| </tbody> | |
| </table> | |
| <p> | |
| Claude ran with <b>extended thinking off on every sample of every log</b>, while gpt-5.5 and DeepSeek reasoned | |
| by default. Whatever the intended comparison was, this is not it: the reasoning models get their default | |
| budget and Claude gets none. Claude is the most disadvantaged model in the set for a reason that has nothing | |
| to do with Claude. | |
| </p> | |
| <p> | |
| This also explains the one gap the grader cannot account for. On Professional, where our grader is correct and | |
| the physician baseline matches to 0.2 points, gpt-5.5 still lands 4 points low. A default-effort versus | |
| highest-effort difference of that size is consistent with the 5.6 to 7.3 point low-to-xhigh delta OpenAI | |
| reports. | |
| </p> | |
| </section> | |
| <section id="headline"> | |
| <h2>Issue 3: three spaces show the unadjusted score as the headline</h2> | |
| <p> | |
| OpenAI reports "length-adjusted score (unadjusted, mean response length in characters)", so the adjusted | |
| number is the primary one. In our <code>listing.json</code> the <code>primary_metric</code> for | |
| <code>hs-non-professional</code>, <code>hs-hard</code> and <code>hs-consensus</code> is the | |
| <b>unadjusted</b> <code>bootstrap_score</code>. Only <code>hs-prof-subsets</code> carries | |
| <code>use_length_adjusted: true</code>. | |
| </p> | |
| <p> | |
| This is not cosmetic. Length adjustment reorders the Hard leaderboard almost completely, because our models | |
| differ enormously in verbosity: | |
| </p> | |
| <table> | |
| <thead><tr><th>Model</th><th class="num">mean chars</th><th class="num">raw</th><th class="num">length-adj</th><th class="num">shift</th><th class="num">rank raw</th><th class="num">rank adj</th></tr></thead> | |
| <tbody> | |
| <tr><td>anthropic/claude-opus-4-7</td><td class="num">1849</td><td class="num">26.61</td><td class="num">27.80</td><td class="num" style="color:#788C5D">+1.2</td><td class="num">2</td><td class="num">1</td></tr> | |
| <tr><td>openai/gpt-5.5</td><td class="num">2158</td><td class="num">27.27</td><td class="num">26.03</td><td class="num">−1.2</td><td class="num">1</td><td class="num">2</td></tr> | |
| <tr><td>openrouter/deepseek-v4-pro</td><td class="num">3408</td><td class="num">24.84</td><td class="num">13.80</td><td class="num" style="color:#B0533A">−11.0</td><td class="num">3</td><td class="num">3</td></tr> | |
| <tr><td>plamo-3.0-prime</td><td class="num">2997</td><td class="num">17.43</td><td class="num">9.61</td><td class="num" style="color:#B0533A">−7.8</td><td class="num">5</td><td class="num">4</td></tr> | |
| <tr><td>vllm/medgemma-27b</td><td class="num">4073</td><td class="num">21.09</td><td class="num">4.83</td><td class="num" style="color:#B0533A">−16.3</td><td class="num">4</td><td class="num">5</td></tr> | |
| <tr><td>vllm/medgemma-4b</td><td class="num">3192</td><td class="num">10.60</td><td class="num">1.26</td><td class="num" style="color:#B0533A">−9.3</td><td class="num">6</td><td class="num">6</td></tr> | |
| </tbody> | |
| </table> | |
| <p class="note"> | |
| Claude gains because it is the only model averaging under the 2000-character centre. medgemma-27b loses 16 | |
| points. Anyone reading the space's default metric is reading a different ranking from the one OpenAI's | |
| methodology produces. | |
| </p> | |
| </section> | |
| <section id="subscores"> | |
| <h2>Issue 4: the Professional per-use-case subscores are inflated</h2> | |
| <p> | |
| The custom <code>healthbench_professional.py</code> emits <code>use_case_*_score</code>, | |
| <code>type_*_score</code> and <code>difficulty_*_score</code> metrics alongside the headline. <b>These do not | |
| use the same aggregation as the headline and should not be quoted.</b> | |
| </p> | |
| <div class="callout"> | |
| <span class="star">★</span> | |
| <span class="ct"><b>v2 correction: the mechanism is now pinned exactly, and it is worse than v1 said.</b> | |
| v1 described these subscores as "consistent with per-sample clipping" of the length-adjusted score, with a | |
| residual attributed to bootstrap noise. The DeepSeek re-run separates the two candidate formulas cleanly, | |
| because DeepSeek is verbose enough that adjusted and unadjusted scores diverge sharply. Every one of the ten | |
| subset metrics in that log reproduces to six decimal places as | |
| <code>mean(clip(<b>unadjusted</b> score, 0, 1))</code>. So the subscores are not a clipped version of the | |
| headline metric — <b>they ignore the length adjustment entirely</b>, despite the run declaring | |
| <code>use_length_adjusted: true</code>, and then clip per sample on top. Both errors push the same way.</span> | |
| </div> | |
| <table> | |
| <thead><tr><th>DeepSeek re-run, by slice</th><th class="num">as reported in the log</th><th class="num">mean(clip(raw))</th><th class="num">mean(clip(adj))</th><th class="num">correct (mean adj)</th><th class="num">inflation</th></tr></thead> | |
| <tbody> | |
| <tr><td>Care consult</td><td class="num" style="color:#B0533A">46.05</td><td class="num">46.05</td><td class="num muted">42.70</td><td class="num">31.83</td><td class="num">+14.2</td></tr> | |
| <tr><td>Writing and documentation</td><td class="num" style="color:#B0533A">35.43</td><td class="num">35.43</td><td class="num muted">32.42</td><td class="num">6.70</td><td class="num">+28.7</td></tr> | |
| <tr><td>Medical research</td><td class="num" style="color:#B0533A">66.60</td><td class="num">66.60</td><td class="num muted">57.00</td><td class="num">53.10</td><td class="num">+13.5</td></tr> | |
| <tr class="new"><td>Red teaming</td><td class="num" style="color:#B0533A">24.94</td><td class="num">24.94</td><td class="num muted">23.98</td><td class="num">−5.19</td><td class="num">+30.1</td></tr> | |
| <tr><td>Difficult</td><td class="num" style="color:#B0533A">28.67</td><td class="num">28.67</td><td class="num muted">27.38</td><td class="num">3.51</td><td class="num">+25.2</td></tr> | |
| <tr><td>Typical</td><td class="num" style="color:#B0533A">70.23</td><td class="num">70.23</td><td class="num muted">61.30</td><td class="num">59.86</td><td class="num">+10.4</td></tr> | |
| </tbody> | |
| </table> | |
| <p class="note"> | |
| The <code>mean(clip(raw))</code> column matches the reported column exactly, to six decimals, on all ten subset | |
| metrics the log emits — there is no bootstrap noise in these numbers at all. Red teaming is the clearest | |
| illustration of the damage: the log presents it as <b>24.94</b> when the correctly aggregated value is | |
| <b>−5.19</b>, a 30-point swing that flips the slice from mediocre to negative. | |
| </p> | |
| <p> | |
| The same error was present in the v1 spaces; it was simply harder to characterise there, because the | |
| physician-baseline responses are short enough that the adjusted and unadjusted scores nearly coincide. | |
| </p> | |
| <p> | |
| The physician baseline makes the error measurable, because OpenAI publishes the ground truth for exactly these | |
| three cells: | |
| </p> | |
| <table> | |
| <thead><tr><th>Physician baseline by use case</th><th class="num">OpenAI</th><th class="num">recomputed correctly</th><th class="num">as reported in our log</th><th class="num">inflation</th></tr></thead> | |
| <tbody> | |
| <tr><td>Care consult</td><td class="num">42.7</td><td class="num">42.54</td><td class="num" style="color:#B0533A">48.4</td><td class="num">+5.9</td></tr> | |
| <tr><td>Writing and documentation</td><td class="num">32.1</td><td class="num">32.91</td><td class="num" style="color:#B0533A">44.9</td><td class="num">+12.0</td></tr> | |
| <tr><td>Medical research</td><td class="num">56.3</td><td class="num">56.60</td><td class="num" style="color:#B0533A">59.0</td><td class="num">+2.4</td></tr> | |
| </tbody> | |
| </table> | |
| <details> | |
| <summary>The arithmetic that identifies the cause <span class="where">525 examples × 8 epochs = 4200</span></summary> | |
| <div class="body"> | |
| <pre>overall mean raw = 44.29 <span class="muted"># matches reported 44.29</span> | |
| overall mean adj = 43.87 <span class="muted"># matches reported 43.87, and OpenAI's 43.7</span> | |
| overall mean adj, clipped per-sample= 49.71 | |
| frac. of samples with adj < 0 = 0.164 | |
| by use case n mean_adj clipped_per_sample | |
| consult 1888 42.54 48.09 | |
| research 1176 56.60 57.47 | |
| writing 1136 32.91 44.39</pre> | |
| <p>16.4% of physician responses score below zero once the length penalty applies, mostly short writing-task | |
| answers that trip negative rubric criteria. Clipping those to zero before averaging is what lifts writing from | |
| 32.9 to roughly 44. <span class="ok">v2:</span> on the physician baseline the clipped-raw and clipped-adjusted | |
| columns are within 0.002 of each other, which is why v1 could not tell them apart and read the ~0.3 residual as | |
| bootstrap noise. The DeepSeek re-run resolves it: the formula is clipped <b>raw</b>.</p> | |
| </div> | |
| </details> | |
| <p> | |
| The headline <code>bootstrap_score</code> is unaffected and remains correct. Only the subset breakdowns are | |
| wrong, which matters because the by-use-case split is the most quoted view of Professional results. | |
| </p> | |
| </section> | |
| <section id="floored"> | |
| <h2>Issue 5: five Professional scores are floored at zero</h2> | |
| <p> | |
| OpenAI clips the aggregate mean to [0, 1], and our implementation follows suit, so this is faithful behaviour | |
| rather than a bug. But it means five reported numbers are all displayed as <code>0.00</code> while their true | |
| values differ by 7 points, which hides real ranking information: | |
| </p> | |
| <table> | |
| <thead><tr><th>Model</th><th>Subset</th><th class="num">reported</th><th class="num">true unclipped mean</th></tr></thead> | |
| <tbody> | |
| <tr><td>plamo-3.0-prime</td><td>red teaming</td><td class="num">0.00</td><td class="num">−11.82</td></tr> | |
| <tr><td>openrouter/deepseek-v4-pro</td><td>red teaming (ep 1)</td><td class="num">0.00</td><td class="num">−8.29</td></tr> | |
| <tr><td>openrouter/deepseek-v4-pro</td><td>red teaming (ep 8)</td><td class="num">0.00</td><td class="num">−6.87</td></tr> | |
| <tr><td>vllm/medgemma-27b</td><td>red teaming</td><td class="num">0.00</td><td class="num">−6.80</td></tr> | |
| <tr><td>plamo-3.0-prime</td><td>writing</td><td class="num">0.00</td><td class="num">−4.31</td></tr> | |
| </tbody> | |
| </table> | |
| <p class="note"> | |
| Worth reporting the unclipped value alongside the clipped one for these, or at least noting that a zero means | |
| "at or below zero" rather than "scored nothing". | |
| </p> | |
| </section> | |
| <section id="hygiene"> | |
| <h2>Run hygiene</h2> | |
| <details> | |
| <summary>DeepSeek's Professional runs are under-sampled, and its consult run is the only one <span class="where">hs-prof-subsets · superseded in v2</span></summary> | |
| <div class="body"> | |
| <p>Every model ran <code>epochs=8</code>, matching the paper's "8 samples per example". The 2026-07-25 DeepSeek | |
| batch ran <code>epochs=1</code>. For <code>writing</code>, <code>research</code>, <code>red_teaming</code> and | |
| <code>physician_baseline</code> both variants exist in <code>listing.json</code>, so the viewer shows duplicate | |
| conflicting entries. For <code>consult</code> the epochs=1 run is the <b>only</b> DeepSeek run, so it sits in | |
| the same chart as everyone else at one eighth the sampling.</p> | |
| <p>The good news is the two variants agree closely where both exist (research 51.85 vs 52.91, writing 5.16 vs | |
| 4.95, red teaming floored in both), so the practical distortion is small. It is still worth deleting the | |
| epochs=1 logs and re-running consult at 8.</p> | |
| <p><span class="ok">v2 update, and one thing v1 got wrong.</span> The 2026-07-25 epochs=1 batch was worse than | |
| under-sampled: <b>it made no model calls at all</b>. Every one of those five logs reports an empty | |
| <code>model_usage</code>, meaning both the candidate generations and the grader responses were served from | |
| Inspect's response cache. They are cache replays of an earlier run, not independent runs, which is why they | |
| completed in seconds. v1 read the close agreement between the epochs=1 and epochs=8 variants as reassuring; | |
| it was tautological. The <a href="#v2">2026-08-06 re-run</a> is a genuine run — 967k output tokens | |
| billed to DeepSeek, 476 cached input tokens — and it moves consult from 25.59 to 31.83. The | |
| duplicate-entry and no-real-consult-run problems are resolved. <b>Epochs is still 1</b>, so the | |
| under-sampling itself is not.</p> | |
| </div> | |
| </details> | |
| <details> | |
| <summary>medgemma-4b is missing four of five Professional subsets <span class="where">hs-prof-subsets</span></summary> | |
| <div class="body"> | |
| <p>Only <code>consult</code> exists (score 8.19). Writing, research, red teaming and physician baseline were | |
| never run, so medgemma-4b has no Professional overall and cannot appear in a like-for-like comparison.</p> | |
| </div> | |
| </details> | |
| <details> | |
| <summary>Response caching is on and demonstrably active <span class="where">plan: generate(cache=true)</span></summary> | |
| <div class="body"> | |
| <p>Four of the five physician-baseline runs report byte-identical scores of 0.44291, which only happens if the | |
| grader responses came from cache. Inspect's cache key includes the epoch by default | |
| (<code>CachePolicy.per_epoch = True</code>), so the 8 epochs are not collapsing into one, which was the real | |
| risk. The remaining caveat is that a re-run inside the one-week TTL is not an independent sample, so a repeat | |
| run cannot be used as a variance estimate.</p> | |
| </div> | |
| </details> | |
| <details> | |
| <summary>Two different inspect_evals versions inside the same space <span class="where">0.14.3 vs 0.16.0</span></summary> | |
| <div class="body"> | |
| <p>The frontier-model runs (July 15 to 16) used <code>inspect_evals 0.14.3</code> with | |
| <code>inspect_ai 0.3.246</code>; the medgemma runs (July 24) used <code>0.16.0</code> with | |
| <code>0.3.249</code>. Same space, same chart, different scorer code. Worth confirming the healthbench scorer | |
| did not change across those releases before comparing medgemma against the frontier models.</p> | |
| </div> | |
| </details> | |
| <details> | |
| <summary>Outlier generations and empty completions <span class="where">hs-hard</span></summary> | |
| <div class="body"> | |
| <p>All runs completed with zero sample errors. Two small things: medgemma-4b produced one 110,722-character | |
| response on Hard, which carries a length penalty of −8.5 on its own and moves the 1000-sample mean by | |
| about 0.9 points; and claude-opus-4-7 returned 7 empty completions on Hard, each scored zero. Neither is | |
| fatal, both are worth a spot check.</p> | |
| </div> | |
| </details> | |
| </section> | |
| <section id="fixes"> | |
| <h2>What to change</h2> | |
| <div class="ladder"> | |
| <div class="chip miss">judge: gpt-4.1 or gpt-5.4-low on hard</div> | |
| <div class="chip miss">judge: gpt-4.1 or gpt-5.4-low on consensus</div> | |
| <div class="chip miss">reasoning effort: highest, all models</div> | |
| <div class="chip miss">claude: enable extended thinking</div> | |
| <div class="chip miss">headline: length-adjusted, 3 spaces</div> | |
| <div class="chip miss">prof subscores: use adj score, no clip</div> | |
| <div class="chip part">deepseek consult: real run ✓, epochs 8 ✗</div> | |
| <div class="chip miss">medgemma-4b: run 4 missing subsets</div> | |
| <div class="chip done">deepseek: duplicate listing entries</div> | |
| <div class="chip done">prof: one task, not 5 subset tasks</div> | |
| <div class="chip hit">length adjustment constants</div> | |
| <div class="chip hit">dataset sizes and splits</div> | |
| <div class="chip hit">grader system message and temp</div> | |
| <div class="chip hit">bootstrap and mean clipping</div> | |
| <div class="chip hit">epochs 8 on Professional</div> | |
| <div class="chip hit">prompting and harness (length match)</div> | |
| </div> | |
| <div class="ladder-legend">clay = needs changing · olive = verified correct, leave alone · | |
| struck through = fixed by the v2 re-run · grey = partly fixed</div> | |
| <p>Concretely, for the two broken variants, stop using the wrapper tasks and call the parent task instead:</p> | |
| <details> | |
| <summary>Getting a real judge into Hard and Consensus <span class="where">the wrapper cannot do it</span></summary> | |
| <div class="body"> | |
| <pre><span class="muted"># broken: judge_model silently defaults to gpt-4o-mini</span> | |
| inspect eval inspect_evals/healthbench_hard \ | |
| -T length_adjustment_center=2000 \ | |
| -T length_adjustment_penalty_per_500_chars=0.0392 | |
| <span class="muted"># works: call healthbench() directly with subset=</span> | |
| inspect eval inspect_evals/healthbench \ | |
| -T subset=hard \ | |
| -T judge_model=<span class="hl">openai/gpt-4.1</span> \ | |
| -T length_adjustment_center=2000 \ | |
| -T length_adjustment_penalty_per_500_chars=0.0392</pre> | |
| <p>Worth considering GPT-5.4 at low reasoning for all four variants instead. The Professional paper notes the | |
| external implementation now has "an option to use settings matching our internal implementation (e.g., GPT-5.4 | |
| at low reasoning effort as a grader)" and that OpenAI intends to keep reporting from the internal one. Using it | |
| everywhere would both fix the deviation and make our four spaces internally comparable for the first time.</p> | |
| </div> | |
| </details> | |
| <p class="note"> | |
| Re-running with the correct grader and highest reasoning effort should move Hard up by roughly 5 points and | |
| Consensus up by roughly 13 toward the published band. If it does not, that is the signal something else is | |
| wrong. If it does, the config is settled. | |
| </p> | |
| </section> | |
| <section id="full"> | |
| <h2>Full results</h2> | |
| <h3>Main variants, all six models</h3> | |
| <table> | |
| <thead><tr><th>Model</th><th class="num">full raw</th><th class="num">full adj</th><th class="num">hard raw</th><th class="num">hard adj</th><th class="num">cons. raw</th><th class="num">cons. adj</th></tr></thead> | |
| <tbody> | |
| <tr class="ours"><td>openai/gpt-5.5</td><td class="num">56.87</td><td class="num">55.83</td><td class="num">27.27</td><td class="num">26.03</td><td class="num">82.08</td><td class="num">82.02</td></tr> | |
| <tr class="ours"><td>anthropic/claude-opus-4-7</td><td class="num">53.44</td><td class="num">54.31</td><td class="num">26.61</td><td class="num">27.80</td><td class="num">80.17</td><td class="num">80.23</td></tr> | |
| <tr><td>openrouter/deepseek-v4-pro</td><td class="num">51.37</td><td class="num">41.73</td><td class="num">24.84</td><td class="num">13.80</td><td class="num">79.10</td><td class="num">78.46</td></tr> | |
| <tr><td>vllm/medgemma-27b-text-it</td><td class="num">47.20</td><td class="num">33.21</td><td class="num">21.09</td><td class="num">4.83</td><td class="num">77.58</td><td class="num">76.61</td></tr> | |
| <tr><td>openai-api/plamo-3.0-prime</td><td class="num">39.42</td><td class="num">32.43</td><td class="num">17.43</td><td class="num">9.61</td><td class="num">75.25</td><td class="num">74.78</td></tr> | |
| <tr><td>vllm/medgemma-4b-it</td><td class="num">26.97</td><td class="num">18.32</td><td class="num">10.60</td><td class="num">1.26</td><td class="num">71.37</td><td class="num">70.77</td></tr> | |
| <tr class="human"><td>OpenAI published, gpt-5.5</td><td class="num">58.4</td><td class="num">56.5</td><td class="num">33.8</td><td class="num">31.5</td><td class="num">95.7</td><td class="num">95.6</td></tr> | |
| </tbody> | |
| </table> | |
| <p class="note"> | |
| Judge: gpt-4.1 for full, gpt-4o-mini for hard and consensus. Epochs 1 throughout. All runs completed with zero | |
| errors. Only the full column is grader-comparable to the published row. | |
| </p> | |
| <h3>Professional subsets, length-adjusted</h3> | |
| <table> | |
| <thead><tr><th>Model</th><th class="num">consult</th><th class="num">writing</th><th class="num">research</th><th class="num">red team</th><th class="num">weighted overall</th></tr></thead> | |
| <tbody> | |
| <tr class="ours"><td>anthropic/claude-opus-4-7</td><td class="num">47.04</td><td class="num">36.15</td><td class="num">61.07</td><td class="num">26.74</td><td class="num">48.02</td></tr> | |
| <tr class="ours"><td>openai/gpt-5.5</td><td class="num">48.64</td><td class="num">35.96</td><td class="num">57.92</td><td class="num">28.19</td><td class="num">47.81</td></tr> | |
| <tr class="new"><td>openrouter/deepseek-v4-pro <span class="vtag">v2 re-run</span></td><td class="num">31.83</td><td class="num">6.70</td><td class="num">53.10</td><td class="num">−5.19</td><td class="num">30.99</td></tr> | |
| <tr><td class="muted">openrouter/deepseek-v4-pro <span class="note">(v1, superseded)</span></td><td class="num muted">25.59</td><td class="num muted">4.95</td><td class="num muted">52.91</td><td class="num muted">−6.87</td><td class="num muted">27.66</td></tr> | |
| <tr><td>vllm/medgemma-27b-text-it</td><td class="num">17.78</td><td class="num">9.13</td><td class="num">34.40</td><td class="num">−6.80</td><td class="num">20.09</td></tr> | |
| <tr><td>openai-api/plamo-3.0-prime</td><td class="num">15.36</td><td class="num">−4.31</td><td class="num">28.58</td><td class="num">−11.82</td><td class="num">13.74</td></tr> | |
| <tr><td>vllm/medgemma-4b-it</td><td class="num">8.19</td><td class="num muted">not run</td><td class="num muted">not run</td><td class="num muted">not run</td><td class="num muted">n/a</td></tr> | |
| <tr class="human"><td>Physician baseline (ours)</td><td class="num">42.54</td><td class="num">32.91</td><td class="num">56.60</td><td class="num muted">n/a</td><td class="num">43.87</td></tr> | |
| <tr class="human"><td>Physician baseline (OpenAI)</td><td class="num">42.7</td><td class="num">32.1</td><td class="num">56.3</td><td class="num muted">n/a</td><td class="num">43.7</td></tr> | |
| <tr class="human"><td>OpenAI published, gpt-5.5</td><td class="num muted">n/a</td><td class="num muted">n/a</td><td class="num muted">n/a</td><td class="num muted">n/a</td><td class="num">51.8</td></tr> | |
| </tbody> | |
| </table> | |
| <p class="note"> | |
| Red teaming values are shown unclipped so the ordering is visible; the logs report these as 0.00. Weighted | |
| overall uses consult 236, writing 142, research 147. Only gpt-5.5 and claude-opus-4-7 beat the physician | |
| baseline overall, and both do so narrowly. | |
| </p> | |
| <p class="note"> | |
| <span class="ok">v2:</span> the DeepSeek row now comes from the 2026-08-06 single-task re-run, whose 30.99 is a | |
| native overall rather than a weighted stitch. Its subset figures are recomputed from per-sample data, not read | |
| off the log's own subset metrics, which are inflated — see <a href="#subscores">issue 4</a>. The | |
| superseded v1 row is kept for traceability: its consult and writing figures came from the 2026-07-25 cache | |
| replay, and its research and red-teaming figures from the 2026-07-24 epochs=8 runs. Every other model's row is | |
| still epochs=8 and unchanged, so DeepSeek is the only single-sample row in this table. | |
| </p> | |
| </section> | |
| <section id="method"> | |
| <h2>How this was checked</h2> | |
| <p> | |
| The four HuggingFace spaces were cloned and their <code>.eval</code> logs pulled as real LFS blobs (the plain | |
| clone yields 133-byte pointers). Headers gave the declared config; per-sample model events gave the config | |
| actually sent, including which model served every grading call; per-sample scores were re-aggregated | |
| independently to confirm the reported metrics and to recompute the subset breakdowns. | |
| </p> | |
| <table> | |
| <thead><tr><th>Space</th><th class="num">logs</th><th class="num">size</th><th>task</th><th class="num">samples/run</th></tr></thead> | |
| <tbody> | |
| <tr><td>hs-non-professional</td><td class="num">6</td><td class="num">573 MB</td><td><code>inspect_evals/healthbench</code></td><td class="num">5000</td></tr> | |
| <tr><td>hs-hard</td><td class="num">6</td><td class="num">113 MB</td><td><code>inspect_evals/healthbench_hard</code></td><td class="num">1000</td></tr> | |
| <tr><td>hs-consensus</td><td class="num">6</td><td class="num">256 MB</td><td><code>inspect_evals/healthbench_consensus</code></td><td class="num">3671</td></tr> | |
| <tr><td>hs-prof-subsets</td><td class="num">30</td><td class="num">714 MB</td><td><code>healthbench_professional_*</code></td><td class="num">525</td></tr> | |
| <tr class="new"><td>healthbench-professional-deepseek-v4-pro <span class="vtag">v2</span></td><td class="num">1</td><td class="num">10 MB</td><td><code>healthbench_professional</code></td><td class="num">525</td></tr> | |
| </tbody> | |
| </table> | |
| <p class="note"> | |
| Local copies at <code>/Users/kazuki/Sandbox/hb-spaces/</code>. The <code>hs-original</code> space was excluded, | |
| its link is broken. The v2 space's <code>.eval</code> was fetched over the HuggingFace | |
| <code>resolve/main</code> endpoint rather than <code>git lfs pull</code>, since git-lfs is not installed on | |
| this machine; the file is a valid zstd-compressed Inspect log and reads cleanly. Reference sources: HealthBench (<a href="https://arxiv.org/abs/2505.08775">arXiv:2505.08775</a>), | |
| HealthBench Professional (<a href="https://arxiv.org/abs/2604.27470">arXiv:2604.27470</a>), the | |
| <a href="https://deploymentsafety.openai.com/gpt-5-6-preview/healthbench">GPT-5.6 Preview system card</a> Table 6, | |
| and <a href="https://github.com/openai/simple-evals">openai/simple-evals</a>. | |
| </p> | |
| </section> | |
| </main> | |
| </div> | |
| </body> | |
| </html> | |