Spaces:
Running
Running
| <html lang="en"> | |
| <head> | |
| <meta charset="utf-8"> | |
| <link rel="icon" href="favicon.svg"> | |
| <meta name="viewport" content="width=device-width, initial-scale=1"> | |
| <title>HealthBench eval logs: 68 runs, 6 models, 8 benches</title> | |
| <style> | |
| :root{--ivory:#FAF9F5;--slate:#141413;--clay:#D97757;--oat:#E3DACC;--olive:#788C5D; | |
| --gray-150:#F0EEE6;--gray-300:#D1CFC5;--gray-500:#87867F;--gray-700:#3D3D3A; | |
| --serif:ui-serif,Georgia,"Times New Roman",serif;--sans:system-ui,-apple-system,"Segoe UI",Roboto,sans-serif; | |
| --mono:ui-monospace,"SF Mono",Menlo,Consolas,monospace} | |
| *{box-sizing:border-box;margin:0;padding:0}html{scroll-behavior:smooth} | |
| body{background:var(--ivory);color:var(--gray-700);font-family:var(--sans);font-size:15px;line-height:1.65; | |
| -webkit-font-smoothing:antialiased;padding:56px 24px 120px} | |
| .page{max-width:1160px;margin:0 auto;display:grid;grid-template-columns:210px minmax(0,1fr);gap:44px} | |
| @media(max-width:920px){.page{grid-template-columns:1fr}nav{display:none}} | |
| nav{position:sticky;top:32px;align-self:start;font-size:13px} | |
| nav .label{font-family:var(--mono);font-size:10px;letter-spacing:.1em;text-transform:uppercase;color:var(--gray-500);margin-bottom:12px} | |
| nav a{display:block;padding:5px 0 5px 12px;border-left:2px solid var(--gray-300);color:var(--gray-700);text-decoration:none} | |
| nav a:hover{color:var(--slate);border-color:var(--slate)} | |
| nav a.l2{padding-left:20px;font-size:12px;color:var(--gray-500)} | |
| nav a .ct{font-family:var(--mono);font-size:10px;color:var(--gray-300)} | |
| nav .files{margin-top:26px;border-top:1px solid var(--gray-300);padding-top:14px} | |
| header{margin-bottom:12px} | |
| .eyebrow{font-family:var(--mono);font-size:11px;letter-spacing:.08em;text-transform:uppercase;color:var(--gray-500);margin-bottom:10px} | |
| h1{font-family:var(--serif);font-weight:500;font-size:32px;color:var(--slate);letter-spacing:-.01em;margin-bottom:14px} | |
| .dateline{font-family:var(--mono);font-size:11px;letter-spacing:.04em;color:var(--gray-500);margin-top:-4px} | |
| h2{font-family:var(--serif);font-weight:500;font-size:22px;color:var(--slate);margin:40px 0 14px;scroll-margin-top:24px} | |
| h3{font-family:var(--serif);font-weight:500;font-size:16px;color:var(--slate);margin:26px 0 2px;scroll-margin-top:24px} | |
| h3 .cnt{font-family:var(--mono);font-size:11px;color:var(--gray-500);font-weight:400;margin-left:6px} | |
| p{margin-bottom:12px;max-width:680px} | |
| p.blurb{font-size:13px;color:var(--gray-500);margin-bottom:8px} | |
| ul{margin:0 0 12px 22px;max-width:680px}li{margin-bottom:6px} | |
| code{font-family:var(--mono);font-size:13px} | |
| a{color:var(--slate);text-decoration:underline;text-decoration-color:var(--gray-300)} | |
| a:hover{text-decoration-color:var(--clay)} | |
| .cards{display:grid;grid-template-columns:1fr 1fr;gap:14px;margin:18px 0;max-width:760px} | |
| @media(max-width:640px){.cards{grid-template-columns:1fr}} | |
| .card{border:1.5px solid var(--gray-300);border-radius:10px;background:#fff;padding:16px 18px;text-decoration:none;display:block} | |
| .card:hover{border-color:var(--clay)} | |
| .card .k{font-family:var(--mono);font-size:10px;letter-spacing:.08em;text-transform:uppercase;color:var(--gray-500);margin-bottom:6px} | |
| .card .t{font-family:var(--serif);font-size:18px;color:var(--slate);margin-bottom:6px} | |
| .card .b{font-size:13px;color:var(--gray-700)} | |
| .callout{display:flex;gap:12px;border:1.5px solid var(--oat);background:rgba(227,218,204,.35);border-radius:10px; | |
| padding:14px 16px;margin:18px 0;font-size:14px;max-width:760px} | |
| .callout .ico{color:var(--clay);font-weight:600} | |
| .callout p{margin-bottom:6px}.callout p:last-child{margin-bottom:0} | |
| .stats{display:grid;grid-template-columns:repeat(4,1fr);gap:8px;margin:16px 0;max-width:600px} | |
| @media(max-width:640px){.stats{grid-template-columns:repeat(2,1fr)}} | |
| .stat{border:1px solid var(--gray-300);border-radius:8px;background:#fff;padding:10px 12px} | |
| .stat .v{font-family:var(--serif);font-size:22px;color:var(--slate);line-height:1.2} | |
| .stat .l{font-family:var(--mono);font-size:9.5px;letter-spacing:.06em;text-transform:uppercase;color:var(--gray-500)} | |
| #filter{font-family:var(--mono);font-size:13px;padding:9px 12px;border:1.5px solid var(--gray-300);border-radius:8px; | |
| background:#fff;width:100%;max-width:420px;color:var(--slate)} | |
| #filter:focus{outline:none;border-color:var(--clay)} | |
| .fcount{font-family:var(--mono);font-size:11px;color:var(--gray-500);margin-top:6px} | |
| table.logs{border-collapse:collapse;width:100%;max-width:900px;margin:6px 0 4px;font-size:13px; | |
| border:1.5px solid var(--gray-300);border-radius:8px;background:#fff} | |
| table.logs th,table.logs td{padding:6px 9px;border-bottom:1px solid var(--gray-300);text-align:left;white-space:nowrap} | |
| table.logs thead th{font-family:var(--mono);font-size:9.5px;letter-spacing:.05em;text-transform:uppercase; | |
| color:var(--gray-500);background:var(--gray-150)} | |
| table.logs th.n,table.logs td.n{text-align:right;font-variant-numeric:tabular-nums} | |
| table.logs tbody tr:last-child td{border-bottom:none} | |
| table.logs td.m{font-weight:500;color:var(--slate)} | |
| table.logs td.d,table.logs td.j,table.logs td.n{font-family:var(--mono);font-size:12px} | |
| table.logs td.f a{font-family:var(--mono);font-size:11.5px;color:var(--clay);text-decoration:none; | |
| border:1px solid var(--gray-300);border-radius:5px;padding:2px 7px} | |
| table.logs td.f a:hover{border-color:var(--clay);background:rgba(217,119,87,.08)} | |
| .tag{font-family:var(--mono);font-size:10px;padding:2px 6px;border-radius:4px;border:1px solid var(--gray-300);color:var(--gray-500)} | |
| .t-ok{border-color:var(--olive);color:#5d6e47;background:rgba(120,140,93,.1)} | |
| .t-warn{border-color:var(--clay);color:#b0533a;background:rgba(217,119,87,.1)} | |
| .t-bad{border-color:#B0533A;color:#B0533A;background:rgba(176,83,58,.14)} | |
| .t-neutral{background:var(--gray-150)} | |
| .inf{color:var(--clay);font-size:10px} | |
| table.plain{border-collapse:collapse;width:100%;max-width:760px;margin:14px 0;font-size:14px} | |
| table.plain th,table.plain td{padding:8px 12px;border-bottom:1px solid var(--gray-300);text-align:left;vertical-align:top} | |
| table.plain th{font-family:var(--mono);font-size:11px;letter-spacing:.06em;text-transform:uppercase;color:var(--gray-500); | |
| font-weight:600;border-bottom:1.5px solid var(--gray-300)} | |
| .legend{font-family:var(--mono);font-size:11px;color:var(--gray-500);margin:10px 0 0;max-width:900px;line-height:1.9} | |
| </style> | |
| </head> | |
| <body> | |
| <div class="page"> | |
| <nav> | |
| <div class="label">On this page</div> | |
| <a href="#start">Start here</a> | |
| <a href="#logs">All 68 logs</a> | |
| <a href="#full" class="l2">HealthBench full <span class="ct">10</span></a> | |
| <a href="#consensus" class="l2">HealthBench consensus <span class="ct">8</span></a> | |
| <a href="#hard" class="l2">HealthBench hard <span class="ct">13</span></a> | |
| <a href="#professional" class="l2">HealthBench Professional <span class="ct">7</span></a> | |
| <a href="#professional-consult" class="l2">Professional: consult <span class="ct">6</span></a> | |
| <a href="#professional-writing" class="l2">Professional: writing <span class="ct">6</span></a> | |
| <a href="#professional-research" class="l2">Professional: research <span class="ct">6</span></a> | |
| <a href="#professional-redteam" class="l2">Professional: red-teaming <span class="ct">6</span></a> | |
| <a href="#professional-baseline" class="l2">Professional: physician baseline <span class="ct">6</span></a> | |
| <a href="#reports">Analysis</a> | |
| <a href="#trust">Before you quote a number</a> | |
| <a href="#how">How this was assembled</a> | |
| <div class="files"> | |
| <div class="label">Data files</div> | |
| <a href="data/log_mapping.csv">log_mapping.csv</a> | |
| <a href="data/MANIFEST.csv">MANIFEST.csv</a> | |
| <a href="data/INDEX.md">INDEX.md</a> | |
| </div> | |
| </nav> | |
| <main> | |
| <header> | |
| <div class="eyebrow">HealthBench · Inspect eval logs</div> | |
| <h1>HealthBench eval logs: 68 runs, 6 models, 8 benches</h1> | |
| <div class="dateline">Created 2026-08-07</div> | |
| </header> | |
| <h2 id="start">Start here</h2> | |
| <p>Every HealthBench run we have, as Inspect <code>.eval</code> logs, in one place. Six models | |
| (GPT-5.5, Claude Opus 4.7, DeepSeek-V4-Pro, PLaMo-3.0-Prime, MedGemma-27B, MedGemma-4B) across | |
| HealthBench full, consensus, hard, and Professional with its four use-case slices.</p> | |
| <div class="stats"> | |
| <div class="stat"><div class="v">68</div><div class="l">eval logs</div></div> | |
| <div class="stat"><div class="v">2.3 GB</div><div class="l">total size</div></div> | |
| <div class="stat"><div class="v">23</div><div class="l">fresh runs</div></div> | |
| <div class="stat"><div class="v">34</div><div class="l">cache replays</div></div> | |
| </div> | |
| <div class="cards"> | |
| <a class="card" href="viewer/index.html"> | |
| <div class="k">browse in-browser</div> | |
| <div class="t">Open the log viewer →</div> | |
| <div class="b">The Inspect viewer, loaded with the 7 Professional whole-set runs (Spaces cap out at 1 GB, so the full 2.1 GB set lives in the dataset). Click any run to read | |
| individual samples, the model's answer, and every rubric verdict the judge made.</div> | |
| </a> | |
| <a class="card" href="matrix.html"> | |
| <div class="k">what exists, what does not</div> | |
| <div class="t">Coverage matrix →</div> | |
| <div class="b">Model × bench grid: which cells are complete, which are off-config, and | |
| the 5 runs still missing. Read this before comparing any two numbers.</div> | |
| </a> | |
| </div> | |
| <p>To pull a single log straight down, use the <code>.eval</code> button in the tables below, or | |
| fetch it directly:</p> | |
| <table class="plain"> | |
| <tbody> | |
| <tr><td><code>huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/<bench>__<model>__<date>.eval</code></td></tr> | |
| </tbody> | |
| </table> | |
| <p>Filenames carry the config, so <code>professional__gpt-5.5__2026-07-24.eval</code> needs no | |
| lookup. A <code>-replay</code> or <code>-cached</code> suffix means the model responses came from | |
| Inspect's cache rather than a fresh generation; <code>-FAILED</code> means the run errored and is | |
| kept only for provenance.</p> | |
| <h2 id="logs">All 68 logs</h2> | |
| <p>Scores are ×100. <code>raw</code> is the HealthBench score, <code>adj</code> is the | |
| length-adjusted one. Grouped by bench, sorted by model.</p> | |
| <input id="filter" type="search" placeholder="filter: try "professional medgemma" or "replay"" autocomplete="off"> | |
| <div class="fcount" id="fcount">68 logs</div> | |
| <h3 id="full">HealthBench full <span class="cnt">10 logs</span></h3> | |
| <p class="blurb">5000 samples, the whole open-ended set</p> | |
| <table class="logs"><thead><tr><th>Model</th><th>Date</th><th class="n">ep</th><th class="n">n</th><th>Judge</th><th class="n">raw</th><th class="n">adj</th><th>Provenance</th><th>File</th></tr></thead><tbody> | |
| <tr data-s="gpt-5.5 full 2026-07-09 openai/gpt-4o-mini not-a-run"><td class="m">gpt-5.5</td><td class="d">2026-07-09</td><td class="n">1</td><td class="n">—</td><td class="j">openai/gpt-4o-mini</td><td class="n">—</td><td class="n">—</td><td><span class="tag t-bad">failed</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/full__gpt-5.5__2026-07-09-FAILED.eval" download>.eval</a></td></tr> | |
| <tr data-s="gpt-5.5 full 2026-07-09 openai/gpt-4o-mini not-a-run"><td class="m">gpt-5.5</td><td class="d">2026-07-09</td><td class="n">1</td><td class="n">—</td><td class="j">openai/gpt-4o-mini</td><td class="n">—</td><td class="n">—</td><td><span class="tag t-bad">failed</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/full__gpt-5.5__2026-07-09-FAILED__0905.eval" download>.eval</a></td></tr> | |
| <tr data-s="gpt-5.5 full 2026-07-09 openai/gpt-4o-mini fresh"><td class="m">gpt-5.5</td><td class="d">2026-07-09</td><td class="n">1</td><td class="n">5000</td><td class="j">openai/gpt-4o-mini</td><td class="n">48.7</td><td class="n">—</td><td><span class="tag t-ok">fresh</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/full__gpt-5.5__2026-07-09.eval" download>.eval</a></td></tr> | |
| <tr data-s="gpt-5.5 full 2026-07-15 gpt-4.1 full cache replay"><td class="m">gpt-5.5</td><td class="d">2026-07-15</td><td class="n">1</td><td class="n">5000</td><td class="j">gpt-4.1</td><td class="n">56.9</td><td class="n">55.8</td><td><span class="tag t-warn">replay</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/full__gpt-5.5__2026-07-15-replay.eval" download>.eval</a></td></tr> | |
| <tr data-s="opus-4.7 full 2026-07-09 openai/gpt-4o-mini fresh"><td class="m">opus-4.7</td><td class="d">2026-07-09</td><td class="n">1</td><td class="n">5000</td><td class="j">openai/gpt-4o-mini</td><td class="n">47.6</td><td class="n">—</td><td><span class="tag t-ok">fresh</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/full__opus-4.7__2026-07-09.eval" download>.eval</a></td></tr> | |
| <tr data-s="opus-4.7 full 2026-07-15 gpt-4.1 full cache replay"><td class="m">opus-4.7</td><td class="d">2026-07-15</td><td class="n">1</td><td class="n">5000</td><td class="j">gpt-4.1</td><td class="n">53.4</td><td class="n">54.3</td><td><span class="tag t-warn">replay</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/full__opus-4.7__2026-07-15-replay.eval" download>.eval</a></td></tr> | |
| <tr data-s="deepseek-v4-pro full 2026-07-16 gpt-4.1 fresh"><td class="m">deepseek-v4-pro</td><td class="d">2026-07-16</td><td class="n">1</td><td class="n">5000</td><td class="j">gpt-4.1</td><td class="n">51.4</td><td class="n">41.7</td><td><span class="tag t-ok">fresh</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/full__deepseek-v4-pro__2026-07-16.eval" download>.eval</a></td></tr> | |
| <tr data-s="plamo-3.0-prime full 2026-07-16 gpt-4.1 fresh"><td class="m">plamo-3.0-prime</td><td class="d">2026-07-16</td><td class="n">1</td><td class="n">5000</td><td class="j">gpt-4.1</td><td class="n">39.4</td><td class="n">32.4</td><td><span class="tag t-ok">fresh</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/full__plamo-3.0-prime__2026-07-16.eval" download>.eval</a></td></tr> | |
| <tr data-s="medgemma-27b full 2026-07-24 gpt-4.1 fresh"><td class="m">medgemma-27b</td><td class="d">2026-07-24</td><td class="n">1</td><td class="n">5000</td><td class="j">gpt-4.1</td><td class="n">47.2</td><td class="n">33.2</td><td><span class="tag t-ok">fresh</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/full__medgemma-27b__2026-07-24.eval" download>.eval</a></td></tr> | |
| <tr data-s="medgemma-4b full 2026-07-24 gpt-4.1 mostly cached"><td class="m">medgemma-4b</td><td class="d">2026-07-24</td><td class="n">1</td><td class="n">5000</td><td class="j">gpt-4.1</td><td class="n">27.0</td><td class="n">18.3</td><td><span class="tag t-warn">cached</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/full__medgemma-4b__2026-07-24-cached.eval" download>.eval</a></td></tr> | |
| </tbody></table> | |
| <h3 id="consensus">HealthBench consensus <span class="cnt">8 logs</span></h3> | |
| <p class="blurb">3671 samples, criteria physicians agreed on</p> | |
| <table class="logs"><thead><tr><th>Model</th><th>Date</th><th class="n">ep</th><th class="n">n</th><th>Judge</th><th class="n">raw</th><th class="n">adj</th><th>Provenance</th><th>File</th></tr></thead><tbody> | |
| <tr data-s="gpt-5.5 consensus 2026-07-16 gpt-4o-mini full cache replay"><td class="m">gpt-5.5</td><td class="d">2026-07-16</td><td class="n">1</td><td class="n">3671</td><td class="j">gpt-4o-mini <span class="inf">?</span></td><td class="n">82.1</td><td class="n">82.0</td><td><span class="tag t-warn">replay</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/consensus__gpt-5.5__2026-07-16-replay.eval" download>.eval</a></td></tr> | |
| <tr data-s="opus-4.7 consensus 2026-07-16 gpt-4o-mini full cache replay"><td class="m">opus-4.7</td><td class="d">2026-07-16</td><td class="n">1</td><td class="n">3671</td><td class="j">gpt-4o-mini <span class="inf">?</span></td><td class="n">80.2</td><td class="n">80.2</td><td><span class="tag t-warn">replay</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/consensus__opus-4.7__2026-07-16-replay.eval" download>.eval</a></td></tr> | |
| <tr data-s="deepseek-v4-pro consensus 2026-07-16 gpt-4o-mini full cache replay"><td class="m">deepseek-v4-pro</td><td class="d">2026-07-16</td><td class="n">1</td><td class="n">3671</td><td class="j">gpt-4o-mini <span class="inf">?</span></td><td class="n">79.1</td><td class="n">78.5</td><td><span class="tag t-warn">replay</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/consensus__deepseek-v4-pro__2026-07-16-replay.eval" download>.eval</a></td></tr> | |
| <tr data-s="plamo-3.0-prime consensus 2026-07-16 gpt-4o-mini full cache replay"><td class="m">plamo-3.0-prime</td><td class="d">2026-07-16</td><td class="n">1</td><td class="n">3671</td><td class="j">gpt-4o-mini <span class="inf">?</span></td><td class="n">75.2</td><td class="n">74.8</td><td><span class="tag t-warn">replay</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/consensus__plamo-3.0-prime__2026-07-16-replay.eval" download>.eval</a></td></tr> | |
| <tr data-s="medgemma-27b consensus 2026-07-24 gpt-4o-mini fresh"><td class="m">medgemma-27b</td><td class="d">2026-07-24</td><td class="n">1</td><td class="n">3671</td><td class="j">gpt-4o-mini <span class="inf">?</span></td><td class="n">77.6</td><td class="n">76.6</td><td><span class="tag t-ok">fresh</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/consensus__medgemma-27b__2026-07-24.eval" download>.eval</a></td></tr> | |
| <tr data-s="medgemma-27b consensus 2026-08-05 gpt-4.1 fresh"><td class="m">medgemma-27b</td><td class="d">2026-08-05</td><td class="n">1</td><td class="n">3671</td><td class="j">gpt-4.1</td><td class="n">91.0</td><td class="n">90.1</td><td><span class="tag t-ok">fresh</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/consensus__medgemma-27b__2026-08-05.eval" download>.eval</a></td></tr> | |
| <tr data-s="medgemma-4b consensus 2026-07-24 gpt-4o-mini fresh"><td class="m">medgemma-4b</td><td class="d">2026-07-24</td><td class="n">1</td><td class="n">3671</td><td class="j">gpt-4o-mini <span class="inf">?</span></td><td class="n">71.4</td><td class="n">70.8</td><td><span class="tag t-ok">fresh</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/consensus__medgemma-4b__2026-07-24.eval" download>.eval</a></td></tr> | |
| <tr data-s="medgemma-4b consensus 2026-08-05 gpt-4.1 fresh"><td class="m">medgemma-4b</td><td class="d">2026-08-05</td><td class="n">1</td><td class="n">3671</td><td class="j">gpt-4.1</td><td class="n">75.8</td><td class="n">75.2</td><td><span class="tag t-ok">fresh</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/consensus__medgemma-4b__2026-08-05.eval" download>.eval</a></td></tr> | |
| </tbody></table> | |
| <h3 id="hard">HealthBench hard <span class="cnt">13 logs</span></h3> | |
| <p class="blurb">1000 samples, the hardest slice</p> | |
| <table class="logs"><thead><tr><th>Model</th><th>Date</th><th class="n">ep</th><th class="n">n</th><th>Judge</th><th class="n">raw</th><th class="n">adj</th><th>Provenance</th><th>File</th></tr></thead><tbody> | |
| <tr data-s="gpt-5.5 hard 2026-07-09 &mdash; not-a-run"><td class="m">gpt-5.5</td><td class="d">2026-07-09</td><td class="n">1</td><td class="n">—</td><td class="j">—</td><td class="n">—</td><td class="n">—</td><td><span class="tag t-bad">failed</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/hard__gpt-5.5__2026-07-09-FAILED.eval" download>.eval</a></td></tr> | |
| <tr data-s="gpt-5.5 hard 2026-07-16 gpt-4o-mini full cache replay"><td class="m">gpt-5.5</td><td class="d">2026-07-16</td><td class="n">1</td><td class="n">1000</td><td class="j">gpt-4o-mini <span class="inf">?</span></td><td class="n">27.3</td><td class="n">26.0</td><td><span class="tag t-warn">replay</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/hard__gpt-5.5__2026-07-16-replay.eval" download>.eval</a></td></tr> | |
| <tr data-s="opus-4.7 hard 2026-07-09 &mdash; not-a-run"><td class="m">opus-4.7</td><td class="d">2026-07-09</td><td class="n">1</td><td class="n">—</td><td class="j">—</td><td class="n">—</td><td class="n">—</td><td><span class="tag t-bad">failed</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/hard__opus-4.7__2026-07-09-FAILED.eval" download>.eval</a></td></tr> | |
| <tr data-s="opus-4.7 hard 2026-07-16 gpt-4o-mini full cache replay"><td class="m">opus-4.7</td><td class="d">2026-07-16</td><td class="n">1</td><td class="n">1000</td><td class="j">gpt-4o-mini <span class="inf">?</span></td><td class="n">26.6</td><td class="n">27.8</td><td><span class="tag t-warn">replay</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/hard__opus-4.7__2026-07-16-replay.eval" download>.eval</a></td></tr> | |
| <tr data-s="deepseek-v4-pro hard 2026-07-16 gpt-4o-mini full cache replay"><td class="m">deepseek-v4-pro</td><td class="d">2026-07-16</td><td class="n">1</td><td class="n">1000</td><td class="j">gpt-4o-mini <span class="inf">?</span></td><td class="n">24.8</td><td class="n">13.8</td><td><span class="tag t-warn">replay</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/hard__deepseek-v4-pro__2026-07-16-replay.eval" download>.eval</a></td></tr> | |
| <tr data-s="plamo-3.0-prime hard 2026-07-16 gpt-4o-mini full cache replay"><td class="m">plamo-3.0-prime</td><td class="d">2026-07-16</td><td class="n">1</td><td class="n">1000</td><td class="j">gpt-4o-mini <span class="inf">?</span></td><td class="n">17.4</td><td class="n">9.6</td><td><span class="tag t-warn">replay</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/hard__plamo-3.0-prime__2026-07-16-replay.eval" download>.eval</a></td></tr> | |
| <tr data-s="medgemma-27b hard 2026-07-24 gpt-4o-mini mostly cached"><td class="m">medgemma-27b</td><td class="d">2026-07-24</td><td class="n">1</td><td class="n">1000</td><td class="j">gpt-4o-mini <span class="inf">?</span></td><td class="n">21.1</td><td class="n">4.8</td><td><span class="tag t-warn">cached</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/hard__medgemma-27b__2026-07-24-cached.eval" download>.eval</a></td></tr> | |
| <tr data-s="medgemma-27b hard 2026-08-05 gpt-4.1 fresh"><td class="m">medgemma-27b</td><td class="d">2026-08-05</td><td class="n">1</td><td class="n">1000</td><td class="j">gpt-4.1</td><td class="n">13.3</td><td class="n">-4.4</td><td><span class="tag t-ok">fresh</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/hard__medgemma-27b__2026-08-05.eval" download>.eval</a></td></tr> | |
| <tr data-s="medgemma-27b hard 2026-08-05 gpt-4.1 fresh"><td class="m">medgemma-27b</td><td class="d">2026-08-05</td><td class="n">1</td><td class="n">1000</td><td class="j">gpt-4.1</td><td class="n">14.1</td><td class="n">-2.3</td><td><span class="tag t-ok">fresh</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/hard__medgemma-27b__2026-08-05__0807.eval" download>.eval</a></td></tr> | |
| <tr data-s="medgemma-4b hard 2026-07-24 gpt-4o-mini fresh"><td class="m">medgemma-4b</td><td class="d">2026-07-24</td><td class="n">1</td><td class="n">1000</td><td class="j">gpt-4o-mini <span class="inf">?</span></td><td class="n">10.6</td><td class="n">1.3</td><td><span class="tag t-ok">fresh</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/hard__medgemma-4b__2026-07-24.eval" download>.eval</a></td></tr> | |
| <tr data-s="medgemma-4b hard 2026-08-05 gpt-4.1 fresh"><td class="m">medgemma-4b</td><td class="d">2026-08-05</td><td class="n">1</td><td class="n">1000</td><td class="j">gpt-4.1</td><td class="n">-3.1</td><td class="n">-16.4</td><td><span class="tag t-ok">fresh</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/hard__medgemma-4b__2026-08-05.eval" download>.eval</a></td></tr> | |
| <tr data-s="medgemma-4b hard 2026-08-05 gpt-4.1 fresh"><td class="m">medgemma-4b</td><td class="d">2026-08-05</td><td class="n">1</td><td class="n">1000</td><td class="j">gpt-4.1</td><td class="n">-3.5</td><td class="n">-12.6</td><td><span class="tag t-ok">fresh</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/hard__medgemma-4b__2026-08-05__0754.eval" download>.eval</a></td></tr> | |
| <tr data-s="gpt-5-nano hard 2026-07-09 &mdash; not-a-run"><td class="m">gpt-5-nano</td><td class="d">2026-07-09</td><td class="n">1</td><td class="n">—</td><td class="j">—</td><td class="n">—</td><td class="n">—</td><td><span class="tag t-bad">failed</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/hard__gpt-5-nano__2026-07-09-FAILED.eval" download>.eval</a></td></tr> | |
| </tbody></table> | |
| <h3 id="professional">HealthBench Professional <span class="cnt">7 logs</span></h3> | |
| <p class="blurb">525 samples, physician-written, has a human baseline</p> | |
| <table class="logs"><thead><tr><th>Model</th><th>Date</th><th class="n">ep</th><th class="n">n</th><th>Judge</th><th class="n">raw</th><th class="n">adj</th><th>Provenance</th><th>File</th></tr></thead><tbody> | |
| <tr data-s="gpt-5.5 professional 2026-07-24 gpt-5.4 fresh"><td class="m">gpt-5.5</td><td class="d">2026-07-24</td><td class="n">8</td><td class="n">4200</td><td class="j">gpt-5.4</td><td class="n">52.9</td><td class="n">47.8</td><td><span class="tag t-ok">fresh</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/professional__gpt-5.5__2026-07-24.eval" download>.eval</a></td></tr> | |
| <tr data-s="opus-4.7 professional 2026-07-24 gpt-5.4 fresh"><td class="m">opus-4.7</td><td class="d">2026-07-24</td><td class="n">8</td><td class="n">4200</td><td class="j">gpt-5.4</td><td class="n">50.8</td><td class="n">48.0</td><td><span class="tag t-ok">fresh</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/professional__opus-4.7__2026-07-24.eval" download>.eval</a></td></tr> | |
| <tr data-s="deepseek-v4-pro professional 2026-07-25 gpt-5.4 mostly cached"><td class="m">deepseek-v4-pro</td><td class="d">2026-07-25</td><td class="n">1</td><td class="n">525</td><td class="j">gpt-5.4</td><td class="n">34.3</td><td class="n">27.4</td><td><span class="tag t-warn">cached</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/professional__deepseek-v4-pro__2026-07-25-cached.eval" download>.eval</a></td></tr> | |
| <tr data-s="deepseek-v4-pro professional 2026-08-06 gpt-5.4 fresh"><td class="m">deepseek-v4-pro</td><td class="d">2026-08-06</td><td class="n">1</td><td class="n">525</td><td class="j">gpt-5.4</td><td class="n">37.8</td><td class="n">31.0</td><td><span class="tag t-ok">fresh</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/professional__deepseek-v4-pro__2026-08-06.eval" download>.eval</a></td></tr> | |
| <tr data-s="plamo-3.0-prime professional 2026-07-24 gpt-5.4 fresh"><td class="m">plamo-3.0-prime</td><td class="d">2026-07-24</td><td class="n">8</td><td class="n">4200</td><td class="j">gpt-5.4</td><td class="n">20.8</td><td class="n">13.7</td><td><span class="tag t-ok">fresh</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/professional__plamo-3.0-prime__2026-07-24.eval" download>.eval</a></td></tr> | |
| <tr data-s="medgemma-27b professional 2026-07-24 gpt-5.4 fresh"><td class="m">medgemma-27b</td><td class="d">2026-07-24</td><td class="n">8</td><td class="n">4200</td><td class="j">gpt-5.4</td><td class="n">31.2</td><td class="n">20.0</td><td><span class="tag t-ok">fresh</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/professional__medgemma-27b__2026-07-24.eval" download>.eval</a></td></tr> | |
| <tr data-s="medgemma-4b professional 2026-07-25 gpt-5.4 fresh"><td class="m">medgemma-4b</td><td class="d">2026-07-25</td><td class="n">8</td><td class="n">4200</td><td class="j">gpt-5.4</td><td class="n">16.5</td><td class="n">9.0</td><td><span class="tag t-ok">fresh</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/professional__medgemma-4b__2026-07-25.eval" download>.eval</a></td></tr> | |
| </tbody></table> | |
| <h3 id="professional-consult">Professional: consult <span class="cnt">6 logs</span></h3> | |
| <p class="blurb">236 samples</p> | |
| <table class="logs"><thead><tr><th>Model</th><th>Date</th><th class="n">ep</th><th class="n">n</th><th>Judge</th><th class="n">raw</th><th class="n">adj</th><th>Provenance</th><th>File</th></tr></thead><tbody> | |
| <tr data-s="gpt-5.5 professional-consult 2026-07-24 gpt-5.4 full cache replay"><td class="m">gpt-5.5</td><td class="d">2026-07-24</td><td class="n">8</td><td class="n">1888</td><td class="j">gpt-5.4</td><td class="n">51.0</td><td class="n">48.6</td><td><span class="tag t-warn">replay</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/professional-consult__gpt-5.5__2026-07-24-replay.eval" download>.eval</a></td></tr> | |
| <tr data-s="opus-4.7 professional-consult 2026-07-24 gpt-5.4 full cache replay"><td class="m">opus-4.7</td><td class="d">2026-07-24</td><td class="n">8</td><td class="n">1888</td><td class="j">gpt-5.4</td><td class="n">49.1</td><td class="n">47.0</td><td><span class="tag t-warn">replay</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/professional-consult__opus-4.7__2026-07-24-replay.eval" download>.eval</a></td></tr> | |
| <tr data-s="deepseek-v4-pro professional-consult 2026-07-25 gpt-5.4 full cache replay"><td class="m">deepseek-v4-pro</td><td class="d">2026-07-25</td><td class="n">1</td><td class="n">236</td><td class="j">gpt-5.4</td><td class="n">31.2</td><td class="n">25.6</td><td><span class="tag t-warn">replay</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/professional-consult__deepseek-v4-pro__2026-07-25-replay.eval" download>.eval</a></td></tr> | |
| <tr data-s="plamo-3.0-prime professional-consult 2026-07-24 gpt-5.4 full cache replay"><td class="m">plamo-3.0-prime</td><td class="d">2026-07-24</td><td class="n">8</td><td class="n">1888</td><td class="j">gpt-5.4</td><td class="n">21.8</td><td class="n">15.4</td><td><span class="tag t-warn">replay</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/professional-consult__plamo-3.0-prime__2026-07-24-replay.eval" download>.eval</a></td></tr> | |
| <tr data-s="medgemma-27b professional-consult 2026-07-24 gpt-5.4 mostly cached"><td class="m">medgemma-27b</td><td class="d">2026-07-24</td><td class="n">8</td><td class="n">1888</td><td class="j">gpt-5.4</td><td class="n">28.5</td><td class="n">17.8</td><td><span class="tag t-warn">cached</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/professional-consult__medgemma-27b__2026-07-24-cached.eval" download>.eval</a></td></tr> | |
| <tr data-s="medgemma-4b professional-consult 2026-07-25 gpt-5.4 mostly cached"><td class="m">medgemma-4b</td><td class="d">2026-07-25</td><td class="n">8</td><td class="n">1888</td><td class="j">gpt-5.4</td><td class="n">15.2</td><td class="n">8.2</td><td><span class="tag t-warn">cached</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/professional-consult__medgemma-4b__2026-07-25-cached.eval" download>.eval</a></td></tr> | |
| </tbody></table> | |
| <h3 id="professional-writing">Professional: writing <span class="cnt">6 logs</span></h3> | |
| <p class="blurb">142 samples</p> | |
| <table class="logs"><thead><tr><th>Model</th><th>Date</th><th class="n">ep</th><th class="n">n</th><th>Judge</th><th class="n">raw</th><th class="n">adj</th><th>Provenance</th><th>File</th></tr></thead><tbody> | |
| <tr data-s="gpt-5.5 professional-writing 2026-07-24 gpt-5.4 full cache replay"><td class="m">gpt-5.5</td><td class="d">2026-07-24</td><td class="n">8</td><td class="n">1136</td><td class="j">gpt-5.4</td><td class="n">40.6</td><td class="n">36.0</td><td><span class="tag t-warn">replay</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/professional-writing__gpt-5.5__2026-07-24-replay.eval" download>.eval</a></td></tr> | |
| <tr data-s="opus-4.7 professional-writing 2026-07-24 gpt-5.4 full cache replay"><td class="m">opus-4.7</td><td class="d">2026-07-24</td><td class="n">8</td><td class="n">1136</td><td class="j">gpt-5.4</td><td class="n">39.5</td><td class="n">36.1</td><td><span class="tag t-warn">replay</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/professional-writing__opus-4.7__2026-07-24-replay.eval" download>.eval</a></td></tr> | |
| <tr data-s="deepseek-v4-pro professional-writing 2026-07-24 gpt-5.4 fresh"><td class="m">deepseek-v4-pro</td><td class="d">2026-07-24</td><td class="n">8</td><td class="n">1136</td><td class="j">gpt-5.4</td><td class="n">9.5</td><td class="n">5.0</td><td><span class="tag t-ok">fresh</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/professional-writing__deepseek-v4-pro__2026-07-24.eval" download>.eval</a></td></tr> | |
| <tr data-s="deepseek-v4-pro professional-writing 2026-07-25 gpt-5.4 full cache replay"><td class="m">deepseek-v4-pro</td><td class="d">2026-07-25</td><td class="n">1</td><td class="n">142</td><td class="j">gpt-5.4</td><td class="n">9.7</td><td class="n">5.2</td><td><span class="tag t-warn">replay</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/professional-writing__deepseek-v4-pro__2026-07-25-replay.eval" download>.eval</a></td></tr> | |
| <tr data-s="plamo-3.0-prime professional-writing 2026-07-24 gpt-5.4 full cache replay"><td class="m">plamo-3.0-prime</td><td class="d">2026-07-24</td><td class="n">8</td><td class="n">1136</td><td class="j">gpt-5.4</td><td class="n">-2.8</td><td class="n">-4.3</td><td><span class="tag t-warn">replay</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/professional-writing__plamo-3.0-prime__2026-07-24-replay.eval" download>.eval</a></td></tr> | |
| <tr data-s="medgemma-27b professional-writing 2026-07-24 gpt-5.4 mostly cached"><td class="m">medgemma-27b</td><td class="d">2026-07-24</td><td class="n">8</td><td class="n">1136</td><td class="j">gpt-5.4</td><td class="n">18.9</td><td class="n">9.1</td><td><span class="tag t-warn">cached</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/professional-writing__medgemma-27b__2026-07-24-cached.eval" download>.eval</a></td></tr> | |
| </tbody></table> | |
| <h3 id="professional-research">Professional: research <span class="cnt">6 logs</span></h3> | |
| <p class="blurb">147 samples</p> | |
| <table class="logs"><thead><tr><th>Model</th><th>Date</th><th class="n">ep</th><th class="n">n</th><th>Judge</th><th class="n">raw</th><th class="n">adj</th><th>Provenance</th><th>File</th></tr></thead><tbody> | |
| <tr data-s="gpt-5.5 professional-research 2026-07-24 gpt-5.4 full cache replay"><td class="m">gpt-5.5</td><td class="d">2026-07-24</td><td class="n">8</td><td class="n">1176</td><td class="j">gpt-5.4</td><td class="n">68.0</td><td class="n">57.9</td><td><span class="tag t-warn">replay</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/professional-research__gpt-5.5__2026-07-24-replay.eval" download>.eval</a></td></tr> | |
| <tr data-s="opus-4.7 professional-research 2026-07-24 gpt-5.4 full cache replay"><td class="m">opus-4.7</td><td class="d">2026-07-24</td><td class="n">8</td><td class="n">1176</td><td class="j">gpt-5.4</td><td class="n">64.6</td><td class="n">61.1</td><td><span class="tag t-warn">replay</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/professional-research__opus-4.7__2026-07-24-replay.eval" download>.eval</a></td></tr> | |
| <tr data-s="deepseek-v4-pro professional-research 2026-07-24 gpt-5.4 fresh"><td class="m">deepseek-v4-pro</td><td class="d">2026-07-24</td><td class="n">8</td><td class="n">1176</td><td class="j">gpt-5.4</td><td class="n">63.7</td><td class="n">52.9</td><td><span class="tag t-ok">fresh</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/professional-research__deepseek-v4-pro__2026-07-24.eval" download>.eval</a></td></tr> | |
| <tr data-s="deepseek-v4-pro professional-research 2026-07-25 gpt-5.4 full cache replay"><td class="m">deepseek-v4-pro</td><td class="d">2026-07-25</td><td class="n">1</td><td class="n">147</td><td class="j">gpt-5.4</td><td class="n">63.0</td><td class="n">51.8</td><td><span class="tag t-warn">replay</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/professional-research__deepseek-v4-pro__2026-07-25-replay.eval" download>.eval</a></td></tr> | |
| <tr data-s="plamo-3.0-prime professional-research 2026-07-24 gpt-5.4 full cache replay"><td class="m">plamo-3.0-prime</td><td class="d">2026-07-24</td><td class="n">8</td><td class="n">1176</td><td class="j">gpt-5.4</td><td class="n">41.8</td><td class="n">28.6</td><td><span class="tag t-warn">replay</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/professional-research__plamo-3.0-prime__2026-07-24-replay.eval" download>.eval</a></td></tr> | |
| <tr data-s="medgemma-27b professional-research 2026-07-24 gpt-5.4 mostly cached"><td class="m">medgemma-27b</td><td class="d">2026-07-24</td><td class="n">8</td><td class="n">1176</td><td class="j">gpt-5.4</td><td class="n">47.8</td><td class="n">34.4</td><td><span class="tag t-warn">cached</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/professional-research__medgemma-27b__2026-07-24-cached.eval" download>.eval</a></td></tr> | |
| </tbody></table> | |
| <h3 id="professional-redteam">Professional: red-teaming <span class="cnt">6 logs</span></h3> | |
| <p class="blurb">191 samples</p> | |
| <table class="logs"><thead><tr><th>Model</th><th>Date</th><th class="n">ep</th><th class="n">n</th><th>Judge</th><th class="n">raw</th><th class="n">adj</th><th>Provenance</th><th>File</th></tr></thead><tbody> | |
| <tr data-s="gpt-5.5 professional-redteam 2026-07-24 gpt-5.4 full cache replay"><td class="m">gpt-5.5</td><td class="d">2026-07-24</td><td class="n">8</td><td class="n">1528</td><td class="j">gpt-5.4</td><td class="n">29.9</td><td class="n">28.2</td><td><span class="tag t-warn">replay</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/professional-redteam__gpt-5.5__2026-07-24-replay.eval" download>.eval</a></td></tr> | |
| <tr data-s="opus-4.7 professional-redteam 2026-07-24 gpt-5.4 full cache replay"><td class="m">opus-4.7</td><td class="d">2026-07-24</td><td class="n">8</td><td class="n">1528</td><td class="j">gpt-5.4</td><td class="n">28.3</td><td class="n">26.7</td><td><span class="tag t-warn">replay</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/professional-redteam__opus-4.7__2026-07-24-replay.eval" download>.eval</a></td></tr> | |
| <tr data-s="deepseek-v4-pro professional-redteam 2026-07-24 gpt-5.4 fresh"><td class="m">deepseek-v4-pro</td><td class="d">2026-07-24</td><td class="n">8</td><td class="n">1528</td><td class="j">gpt-5.4</td><td class="n">-3.7</td><td class="n">-6.9</td><td><span class="tag t-ok">fresh</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/professional-redteam__deepseek-v4-pro__2026-07-24.eval" download>.eval</a></td></tr> | |
| <tr data-s="deepseek-v4-pro professional-redteam 2026-07-25 gpt-5.4 full cache replay"><td class="m">deepseek-v4-pro</td><td class="d">2026-07-25</td><td class="n">1</td><td class="n">191</td><td class="j">gpt-5.4</td><td class="n">-5.3</td><td class="n">-8.3</td><td><span class="tag t-warn">replay</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/professional-redteam__deepseek-v4-pro__2026-07-25-replay.eval" download>.eval</a></td></tr> | |
| <tr data-s="plamo-3.0-prime professional-redteam 2026-07-24 gpt-5.4 full cache replay"><td class="m">plamo-3.0-prime</td><td class="d">2026-07-24</td><td class="n">8</td><td class="n">1528</td><td class="j">gpt-5.4</td><td class="n">-9.9</td><td class="n">-11.8</td><td><span class="tag t-warn">replay</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/professional-redteam__plamo-3.0-prime__2026-07-24-replay.eval" download>.eval</a></td></tr> | |
| <tr data-s="medgemma-27b professional-redteam 2026-07-24 gpt-5.4 full cache replay"><td class="m">medgemma-27b</td><td class="d">2026-07-24</td><td class="n">8</td><td class="n">1528</td><td class="j">gpt-5.4</td><td class="n">1.5</td><td class="n">-6.8</td><td><span class="tag t-warn">replay</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/professional-redteam__medgemma-27b__2026-07-24-replay.eval" download>.eval</a></td></tr> | |
| </tbody></table> | |
| <h3 id="professional-baseline">Professional: physician baseline <span class="cnt">6 logs</span></h3> | |
| <p class="blurb">the human reference, model-independent</p> | |
| <table class="logs"><thead><tr><th>Model</th><th>Date</th><th class="n">ep</th><th class="n">n</th><th>Judge</th><th class="n">raw</th><th class="n">adj</th><th>Provenance</th><th>File</th></tr></thead><tbody> | |
| <tr data-s="gpt-5.5 professional-baseline 2026-07-24 gpt-5.4 baseline(no model gen by design)"><td class="m">gpt-5.5</td><td class="d">2026-07-24</td><td class="n">8</td><td class="n">4200</td><td class="j">gpt-5.4</td><td class="n">44.3</td><td class="n">43.9</td><td><span class="tag t-neutral">baseline</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/professional-baseline__gpt-5.5__2026-07-24.eval" download>.eval</a></td></tr> | |
| <tr data-s="opus-4.7 professional-baseline 2026-07-24 gpt-5.4 baseline(no model gen by design)"><td class="m">opus-4.7</td><td class="d">2026-07-24</td><td class="n">8</td><td class="n">4200</td><td class="j">gpt-5.4</td><td class="n">44.3</td><td class="n">43.9</td><td><span class="tag t-neutral">baseline</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/professional-baseline__opus-4.7__2026-07-24.eval" download>.eval</a></td></tr> | |
| <tr data-s="deepseek-v4-pro professional-baseline 2026-07-24 gpt-5.4 baseline(no model gen by design)"><td class="m">deepseek-v4-pro</td><td class="d">2026-07-24</td><td class="n">8</td><td class="n">4200</td><td class="j">gpt-5.4</td><td class="n">44.3</td><td class="n">43.9</td><td><span class="tag t-neutral">baseline</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/professional-baseline__deepseek-v4-pro__2026-07-24.eval" download>.eval</a></td></tr> | |
| <tr data-s="deepseek-v4-pro professional-baseline 2026-07-25 gpt-5.4 baseline(no model gen by design)"><td class="m">deepseek-v4-pro</td><td class="d">2026-07-25</td><td class="n">1</td><td class="n">525</td><td class="j">gpt-5.4</td><td class="n">43.3</td><td class="n">42.9</td><td><span class="tag t-neutral">baseline</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/professional-baseline__deepseek-v4-pro__2026-07-25.eval" download>.eval</a></td></tr> | |
| <tr data-s="plamo-3.0-prime professional-baseline 2026-07-24 gpt-5.4 baseline(no model gen by design)"><td class="m">plamo-3.0-prime</td><td class="d">2026-07-24</td><td class="n">8</td><td class="n">4200</td><td class="j">gpt-5.4</td><td class="n">44.3</td><td class="n">43.9</td><td><span class="tag t-neutral">baseline</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/professional-baseline__plamo-3.0-prime__2026-07-24.eval" download>.eval</a></td></tr> | |
| <tr data-s="medgemma-27b professional-baseline 2026-07-24 gpt-5.4 baseline(no model gen by design)"><td class="m">medgemma-27b</td><td class="d">2026-07-24</td><td class="n">8</td><td class="n">4200</td><td class="j">gpt-5.4</td><td class="n">43.9</td><td class="n">43.5</td><td><span class="tag t-neutral">baseline</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/professional-baseline__medgemma-27b__2026-07-24.eval" download>.eval</a></td></tr> | |
| </tbody></table> | |
| <p class="legend"> | |
| <span class="tag t-ok">fresh</span> model responses generated in this run | |
| <span class="tag t-warn">replay</span> every call served from cache | |
| <span class="tag t-warn">cached</span> under 200 candidate tokens per sample | |
| <span class="tag t-neutral">baseline</span> human responses, no generation by design | |
| <span class="tag t-bad">failed</span> errored or cancelled<br> | |
| <span class="inf">?</span> judge not recorded in <code>task_args</code>, inferred from the | |
| scorer default and confirmed against <code>stats.model_usage</code> | |
| </p> | |
| <h2 id="reports">Analysis</h2> | |
| <table class="plain"> | |
| <thead><tr><th>Document</th><th>What it answers</th></tr></thead> | |
| <tbody> | |
| <tr> | |
| <td><a href="matrix.html">Coverage matrix</a></td> | |
| <td>Which model ran which bench, which cells are comparable, and what is still missing.</td> | |
| </tr> | |
| <tr> | |
| <td><a href="config-check-v2.html">Config check v2</a></td> | |
| <td>Do our numbers reproduce OpenAI's published HealthBench results? Anchored on the | |
| physician baseline (ours 43.9 against their 43.7).</td> | |
| </tr> | |
| <tr> | |
| <td><a href="config-check-v1.html">Config check v1</a></td> | |
| <td>The earlier pass over the first four spaces. Superseded by v2, kept for history.</td> | |
| </tr> | |
| </tbody> | |
| </table> | |
| <h2 id="trust">Before you quote a number</h2> | |
| <p>Four things will bite you if you take a score straight out of a log.</p> | |
| <table class="plain"> | |
| <thead><tr><th>Issue</th><th>What to do</th></tr></thead> | |
| <tbody> | |
| <tr> | |
| <td><b>The judge is not constant.</b> Three graders are in play: <code>gpt-4o-mini</code> | |
| (hard, consensus), <code>gpt-4.1</code> (full, and the Aug-05 MedGemma re-runs), | |
| <code>gpt-5.4</code> (all Professional). Swapping the judge moves a score by up to 14 | |
| points, and not always in the same direction.</td> | |
| <td>Only compare runs sharing a judge. The <code>Judge</code> column above is the check.</td> | |
| </tr> | |
| <tr> | |
| <td><b>Professional epochs are inconsistent.</b> 8 epochs for most models, 1 for DeepSeek.</td> | |
| <td>Check the <code>ep</code> column before putting two Professional rows side by side.</td> | |
| </tr> | |
| <tr> | |
| <td><b>In-log subset metrics are wrong.</b> <code>use_case_*_score</code>, | |
| <code>specialty_*_score</code>, <code>difficulty_*_score</code> and | |
| <code>source_slice_*_score</code> drop the length adjustment and clip each sample to | |
| [0,1] first. Errors run up to +32 points, always upward.</td> | |
| <td>Use the standalone <code>professional-*</code> logs above for the four use-case | |
| slices. For specialty and difficulty, re-aggregate from per-sample scores yourself.</td> | |
| </tr> | |
| <tr> | |
| <td><b><code>cache=true</code> on every run.</b> 34 of 68 logs served some | |
| or all calls from cache, so an empty <code>stats.model_usage</code> is a replay, not a run.</td> | |
| <td>The <code>Provenance</code> column above already classifies this.</td> | |
| </tr> | |
| </tbody> | |
| </table> | |
| <div class="callout"> | |
| <span class="ico">★</span> | |
| <div> | |
| <p>The pipeline itself is validated: the physician baseline on Professional lands at | |
| <b>43.9</b> against OpenAI's published <b>43.7</b>. Discrepancies in the model numbers are | |
| config drift, not a broken harness.</p> | |
| </div> | |
| </div> | |
| <h2 id="how">How this was assembled</h2> | |
| <p>The runs were executed by Ajay between 2026-07-09 and 2026-08-06 and originally published as | |
| ten separate HuggingFace Spaces under <code>ajay-citadel</code>. This Space consolidates all of | |
| them into one viewer, renames the logs so the config is legible from the filename, and adds the | |
| provenance classification that the raw logs do not carry.</p> | |
| <p><code>data/log_mapping.csv</code> maps every renamed file back to its original space and | |
| filename, so nothing here is a dead end. <code>data/MANIFEST.csv</code> is the full per-run | |
| header dump: judge, epochs, token counts, package versions. | |
| <code>data/INDEX.md</code> is the short version of the traps list.</p> | |
| <p>The original spaces remain the upstream source. If a number here disagrees with one there, | |
| the logs are byte-identical, so the difference is in which run you are reading, not in the data.</p> | |
| </main> | |
| </div> | |
| <script> | |
| const f=document.getElementById('filter'),rows=[...document.querySelectorAll('table.logs tbody tr')], | |
| cnt=document.getElementById('fcount'); | |
| f.addEventListener('input',()=>{ | |
| const q=f.value.trim().toLowerCase().split(/\s+/).filter(Boolean); | |
| let shown=0; | |
| rows.forEach(r=>{const s=r.dataset.s,hit=q.every(t=>s.includes(t));r.style.display=hit?'':'none';if(hit)shown++;}); | |
| document.querySelectorAll('table.logs').forEach(t=>{ | |
| const any=[...t.querySelectorAll('tbody tr')].some(r=>r.style.display!=='none'); | |
| const h3=t.previousElementSibling.previousElementSibling; | |
| t.style.display=any?'':'none'; | |
| if(h3&&h3.tagName==='H3'){h3.style.display=any?'':'none';t.previousElementSibling.style.display=any?'':'none';} | |
| }); | |
| cnt.textContent=q.length?shown+' of '+rows.length+' logs match':rows.length+' logs'; | |
| }); | |
| </script> | |
| </body> | |
| </html> | |