Hugging Face's logo

Spaces:
kirby44
/
healthbench-eval-logs
Running

App Files Community
healthbench-eval-logs
File size: 51,983 Bytes
cb839ca
c189142
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
cb839ca
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<link rel="icon" href="favicon.svg">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>HealthBench eval logs: 68 runs, 6 models, 8 benches</title>
<style>
:root{--ivory:#FAF9F5;--slate:#141413;--clay:#D97757;--oat:#E3DACC;--olive:#788C5D;
--gray-150:#F0EEE6;--gray-300:#D1CFC5;--gray-500:#87867F;--gray-700:#3D3D3A;
--serif:ui-serif,Georgia,"Times New Roman",serif;--sans:system-ui,-apple-system,"Segoe UI",Roboto,sans-serif;
--mono:ui-monospace,"SF Mono",Menlo,Consolas,monospace}
*{box-sizing:border-box;margin:0;padding:0}html{scroll-behavior:smooth}
body{background:var(--ivory);color:var(--gray-700);font-family:var(--sans);font-size:15px;line-height:1.65;
-webkit-font-smoothing:antialiased;padding:56px 24px 120px}
.page{max-width:1160px;margin:0 auto;display:grid;grid-template-columns:210px minmax(0,1fr);gap:44px}
@media(max-width:920px){.page{grid-template-columns:1fr}nav{display:none}}
nav{position:sticky;top:32px;align-self:start;font-size:13px}
nav .label{font-family:var(--mono);font-size:10px;letter-spacing:.1em;text-transform:uppercase;color:var(--gray-500);margin-bottom:12px}
nav a{display:block;padding:5px 0 5px 12px;border-left:2px solid var(--gray-300);color:var(--gray-700);text-decoration:none}
nav a:hover{color:var(--slate);border-color:var(--slate)}
nav a.l2{padding-left:20px;font-size:12px;color:var(--gray-500)}
nav a .ct{font-family:var(--mono);font-size:10px;color:var(--gray-300)}
nav .files{margin-top:26px;border-top:1px solid var(--gray-300);padding-top:14px}
header{margin-bottom:12px}
.eyebrow{font-family:var(--mono);font-size:11px;letter-spacing:.08em;text-transform:uppercase;color:var(--gray-500);margin-bottom:10px}
h1{font-family:var(--serif);font-weight:500;font-size:32px;color:var(--slate);letter-spacing:-.01em;margin-bottom:14px}
.dateline{font-family:var(--mono);font-size:11px;letter-spacing:.04em;color:var(--gray-500);margin-top:-4px}
h2{font-family:var(--serif);font-weight:500;font-size:22px;color:var(--slate);margin:40px 0 14px;scroll-margin-top:24px}
h3{font-family:var(--serif);font-weight:500;font-size:16px;color:var(--slate);margin:26px 0 2px;scroll-margin-top:24px}
h3 .cnt{font-family:var(--mono);font-size:11px;color:var(--gray-500);font-weight:400;margin-left:6px}
p{margin-bottom:12px;max-width:680px}
p.blurb{font-size:13px;color:var(--gray-500);margin-bottom:8px}
ul{margin:0 0 12px 22px;max-width:680px}li{margin-bottom:6px}
code{font-family:var(--mono);font-size:13px}
a{color:var(--slate);text-decoration:underline;text-decoration-color:var(--gray-300)}
a:hover{text-decoration-color:var(--clay)}
.cards{display:grid;grid-template-columns:1fr 1fr;gap:14px;margin:18px 0;max-width:760px}
@media(max-width:640px){.cards{grid-template-columns:1fr}}
.card{border:1.5px solid var(--gray-300);border-radius:10px;background:#fff;padding:16px 18px;text-decoration:none;display:block}
.card:hover{border-color:var(--clay)}
.card .k{font-family:var(--mono);font-size:10px;letter-spacing:.08em;text-transform:uppercase;color:var(--gray-500);margin-bottom:6px}
.card .t{font-family:var(--serif);font-size:18px;color:var(--slate);margin-bottom:6px}
.card .b{font-size:13px;color:var(--gray-700)}
.callout{display:flex;gap:12px;border:1.5px solid var(--oat);background:rgba(227,218,204,.35);border-radius:10px;
padding:14px 16px;margin:18px 0;font-size:14px;max-width:760px}
.callout .ico{color:var(--clay);font-weight:600}
.callout p{margin-bottom:6px}.callout p:last-child{margin-bottom:0}
.stats{display:grid;grid-template-columns:repeat(4,1fr);gap:8px;margin:16px 0;max-width:600px}
@media(max-width:640px){.stats{grid-template-columns:repeat(2,1fr)}}
.stat{border:1px solid var(--gray-300);border-radius:8px;background:#fff;padding:10px 12px}
.stat .v{font-family:var(--serif);font-size:22px;color:var(--slate);line-height:1.2}
.stat .l{font-family:var(--mono);font-size:9.5px;letter-spacing:.06em;text-transform:uppercase;color:var(--gray-500)}
#filter{font-family:var(--mono);font-size:13px;padding:9px 12px;border:1.5px solid var(--gray-300);border-radius:8px;
background:#fff;width:100%;max-width:420px;color:var(--slate)}
#filter:focus{outline:none;border-color:var(--clay)}
.fcount{font-family:var(--mono);font-size:11px;color:var(--gray-500);margin-top:6px}
table.logs{border-collapse:collapse;width:100%;max-width:900px;margin:6px 0 4px;font-size:13px;
border:1.5px solid var(--gray-300);border-radius:8px;background:#fff}
table.logs th,table.logs td{padding:6px 9px;border-bottom:1px solid var(--gray-300);text-align:left;white-space:nowrap}
table.logs thead th{font-family:var(--mono);font-size:9.5px;letter-spacing:.05em;text-transform:uppercase;
color:var(--gray-500);background:var(--gray-150)}
table.logs th.n,table.logs td.n{text-align:right;font-variant-numeric:tabular-nums}
table.logs tbody tr:last-child td{border-bottom:none}
table.logs td.m{font-weight:500;color:var(--slate)}
table.logs td.d,table.logs td.j,table.logs td.n{font-family:var(--mono);font-size:12px}
table.logs td.f a{font-family:var(--mono);font-size:11.5px;color:var(--clay);text-decoration:none;
border:1px solid var(--gray-300);border-radius:5px;padding:2px 7px}
table.logs td.f a:hover{border-color:var(--clay);background:rgba(217,119,87,.08)}
.tag{font-family:var(--mono);font-size:10px;padding:2px 6px;border-radius:4px;border:1px solid var(--gray-300);color:var(--gray-500)}
.t-ok{border-color:var(--olive);color:#5d6e47;background:rgba(120,140,93,.1)}
.t-warn{border-color:var(--clay);color:#b0533a;background:rgba(217,119,87,.1)}
.t-bad{border-color:#B0533A;color:#B0533A;background:rgba(176,83,58,.14)}
.t-neutral{background:var(--gray-150)}
.inf{color:var(--clay);font-size:10px}
table.plain{border-collapse:collapse;width:100%;max-width:760px;margin:14px 0;font-size:14px}
table.plain th,table.plain td{padding:8px 12px;border-bottom:1px solid var(--gray-300);text-align:left;vertical-align:top}
table.plain th{font-family:var(--mono);font-size:11px;letter-spacing:.06em;text-transform:uppercase;color:var(--gray-500);
font-weight:600;border-bottom:1.5px solid var(--gray-300)}
.legend{font-family:var(--mono);font-size:11px;color:var(--gray-500);margin:10px 0 0;max-width:900px;line-height:1.9}
</style>
</head>
<body>
<div class="page">
  <nav>
    <div class="label">On this page</div>
    <a href="#start">Start here</a>
    <a href="#logs">All 68 logs</a>
        <a href="#full" class="l2">HealthBench full <span class="ct">10</span></a>
    <a href="#consensus" class="l2">HealthBench consensus <span class="ct">8</span></a>
    <a href="#hard" class="l2">HealthBench hard <span class="ct">13</span></a>
    <a href="#professional" class="l2">HealthBench Professional <span class="ct">7</span></a>
    <a href="#professional-consult" class="l2">Professional: consult <span class="ct">6</span></a>
    <a href="#professional-writing" class="l2">Professional: writing <span class="ct">6</span></a>
    <a href="#professional-research" class="l2">Professional: research <span class="ct">6</span></a>
    <a href="#professional-redteam" class="l2">Professional: red-teaming <span class="ct">6</span></a>
    <a href="#professional-baseline" class="l2">Professional: physician baseline <span class="ct">6</span></a>
    <a href="#reports">Analysis</a>
    <a href="#trust">Before you quote a number</a>
    <a href="#how">How this was assembled</a>
    <div class="files">
      <div class="label">Data files</div>
      <a href="data/log_mapping.csv">log_mapping.csv</a>
      <a href="data/MANIFEST.csv">MANIFEST.csv</a>
      <a href="data/INDEX.md">INDEX.md</a>
    </div>
  </nav>

  <main>
    <header>
      <div class="eyebrow">HealthBench &middot; Inspect eval logs</div>
      <h1>HealthBench eval logs: 68 runs, 6 models, 8 benches</h1>
      <div class="dateline">Created 2026-08-07</div>
    </header>

    <h2 id="start">Start here</h2>

    <p>Every HealthBench run we have, as Inspect <code>.eval</code> logs, in one place. Six models
    (GPT-5.5, Claude Opus 4.7, DeepSeek-V4-Pro, PLaMo-3.0-Prime, MedGemma-27B, MedGemma-4B) across
    HealthBench full, consensus, hard, and Professional with its four use-case slices.</p>

    <div class="stats">
      <div class="stat"><div class="v">68</div><div class="l">eval logs</div></div>
      <div class="stat"><div class="v">2.3 GB</div><div class="l">total size</div></div>
      <div class="stat"><div class="v">23</div><div class="l">fresh runs</div></div>
      <div class="stat"><div class="v">34</div><div class="l">cache replays</div></div>
    </div>

    <div class="cards">
      <a class="card" href="viewer/index.html">
        <div class="k">browse in-browser</div>
        <div class="t">Open the log viewer &rarr;</div>
        <div class="b">The Inspect viewer, loaded with the 7 Professional whole-set runs (Spaces cap out at 1 GB, so the full 2.1 GB set lives in the dataset). Click any run to read
        individual samples, the model's answer, and every rubric verdict the judge made.</div>
      </a>
      <a class="card" href="matrix.html">
        <div class="k">what exists, what does not</div>
        <div class="t">Coverage matrix &rarr;</div>
        <div class="b">Model &times; bench grid: which cells are complete, which are off-config, and
        the 5 runs still missing. Read this before comparing any two numbers.</div>
      </a>
    </div>

    <p>To pull a single log straight down, use the <code>.eval</code> button in the tables below, or
    fetch it directly:</p>

    <table class="plain">
      <tbody>
        <tr><td><code>huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/&lt;bench&gt;__&lt;model&gt;__&lt;date&gt;.eval</code></td></tr>
      </tbody>
    </table>

    <p>Filenames carry the config, so <code>professional__gpt-5.5__2026-07-24.eval</code> needs no
    lookup. A <code>-replay</code> or <code>-cached</code> suffix means the model responses came from
    Inspect's cache rather than a fresh generation; <code>-FAILED</code> means the run errored and is
    kept only for provenance.</p>

    <h2 id="logs">All 68 logs</h2>

    <p>Scores are &times;100. <code>raw</code> is the HealthBench score, <code>adj</code> is the
    length-adjusted one. Grouped by bench, sorted by model.</p>

    <input id="filter" type="search" placeholder="filter: try &quot;professional medgemma&quot; or &quot;replay&quot;" autocomplete="off">
    <div class="fcount" id="fcount">68 logs</div>

<h3 id="full">HealthBench full <span class="cnt">10 logs</span></h3>
<p class="blurb">5000 samples, the whole open-ended set</p>
<table class="logs"><thead><tr><th>Model</th><th>Date</th><th class="n">ep</th><th class="n">n</th><th>Judge</th><th class="n">raw</th><th class="n">adj</th><th>Provenance</th><th>File</th></tr></thead><tbody>
<tr data-s="gpt-5.5 full 2026-07-09 openai/gpt-4o-mini not-a-run"><td class="m">gpt-5.5</td><td class="d">2026-07-09</td><td class="n">1</td><td class="n">&mdash;</td><td class="j">openai/gpt-4o-mini</td><td class="n">&mdash;</td><td class="n">&mdash;</td><td><span class="tag t-bad">failed</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/full__gpt-5.5__2026-07-09-FAILED.eval" download>.eval</a></td></tr>
<tr data-s="gpt-5.5 full 2026-07-09 openai/gpt-4o-mini not-a-run"><td class="m">gpt-5.5</td><td class="d">2026-07-09</td><td class="n">1</td><td class="n">&mdash;</td><td class="j">openai/gpt-4o-mini</td><td class="n">&mdash;</td><td class="n">&mdash;</td><td><span class="tag t-bad">failed</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/full__gpt-5.5__2026-07-09-FAILED__0905.eval" download>.eval</a></td></tr>
<tr data-s="gpt-5.5 full 2026-07-09 openai/gpt-4o-mini fresh"><td class="m">gpt-5.5</td><td class="d">2026-07-09</td><td class="n">1</td><td class="n">5000</td><td class="j">openai/gpt-4o-mini</td><td class="n">48.7</td><td class="n">&mdash;</td><td><span class="tag t-ok">fresh</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/full__gpt-5.5__2026-07-09.eval" download>.eval</a></td></tr>
<tr data-s="gpt-5.5 full 2026-07-15 gpt-4.1 full cache replay"><td class="m">gpt-5.5</td><td class="d">2026-07-15</td><td class="n">1</td><td class="n">5000</td><td class="j">gpt-4.1</td><td class="n">56.9</td><td class="n">55.8</td><td><span class="tag t-warn">replay</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/full__gpt-5.5__2026-07-15-replay.eval" download>.eval</a></td></tr>
<tr data-s="opus-4.7 full 2026-07-09 openai/gpt-4o-mini fresh"><td class="m">opus-4.7</td><td class="d">2026-07-09</td><td class="n">1</td><td class="n">5000</td><td class="j">openai/gpt-4o-mini</td><td class="n">47.6</td><td class="n">&mdash;</td><td><span class="tag t-ok">fresh</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/full__opus-4.7__2026-07-09.eval" download>.eval</a></td></tr>
<tr data-s="opus-4.7 full 2026-07-15 gpt-4.1 full cache replay"><td class="m">opus-4.7</td><td class="d">2026-07-15</td><td class="n">1</td><td class="n">5000</td><td class="j">gpt-4.1</td><td class="n">53.4</td><td class="n">54.3</td><td><span class="tag t-warn">replay</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/full__opus-4.7__2026-07-15-replay.eval" download>.eval</a></td></tr>
<tr data-s="deepseek-v4-pro full 2026-07-16 gpt-4.1 fresh"><td class="m">deepseek-v4-pro</td><td class="d">2026-07-16</td><td class="n">1</td><td class="n">5000</td><td class="j">gpt-4.1</td><td class="n">51.4</td><td class="n">41.7</td><td><span class="tag t-ok">fresh</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/full__deepseek-v4-pro__2026-07-16.eval" download>.eval</a></td></tr>
<tr data-s="plamo-3.0-prime full 2026-07-16 gpt-4.1 fresh"><td class="m">plamo-3.0-prime</td><td class="d">2026-07-16</td><td class="n">1</td><td class="n">5000</td><td class="j">gpt-4.1</td><td class="n">39.4</td><td class="n">32.4</td><td><span class="tag t-ok">fresh</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/full__plamo-3.0-prime__2026-07-16.eval" download>.eval</a></td></tr>
<tr data-s="medgemma-27b full 2026-07-24 gpt-4.1 fresh"><td class="m">medgemma-27b</td><td class="d">2026-07-24</td><td class="n">1</td><td class="n">5000</td><td class="j">gpt-4.1</td><td class="n">47.2</td><td class="n">33.2</td><td><span class="tag t-ok">fresh</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/full__medgemma-27b__2026-07-24.eval" download>.eval</a></td></tr>
<tr data-s="medgemma-4b full 2026-07-24 gpt-4.1 mostly cached"><td class="m">medgemma-4b</td><td class="d">2026-07-24</td><td class="n">1</td><td class="n">5000</td><td class="j">gpt-4.1</td><td class="n">27.0</td><td class="n">18.3</td><td><span class="tag t-warn">cached</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/full__medgemma-4b__2026-07-24-cached.eval" download>.eval</a></td></tr>
</tbody></table>
<h3 id="consensus">HealthBench consensus <span class="cnt">8 logs</span></h3>
<p class="blurb">3671 samples, criteria physicians agreed on</p>
<table class="logs"><thead><tr><th>Model</th><th>Date</th><th class="n">ep</th><th class="n">n</th><th>Judge</th><th class="n">raw</th><th class="n">adj</th><th>Provenance</th><th>File</th></tr></thead><tbody>
<tr data-s="gpt-5.5 consensus 2026-07-16 gpt-4o-mini full cache replay"><td class="m">gpt-5.5</td><td class="d">2026-07-16</td><td class="n">1</td><td class="n">3671</td><td class="j">gpt-4o-mini <span class="inf">?</span></td><td class="n">82.1</td><td class="n">82.0</td><td><span class="tag t-warn">replay</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/consensus__gpt-5.5__2026-07-16-replay.eval" download>.eval</a></td></tr>
<tr data-s="opus-4.7 consensus 2026-07-16 gpt-4o-mini full cache replay"><td class="m">opus-4.7</td><td class="d">2026-07-16</td><td class="n">1</td><td class="n">3671</td><td class="j">gpt-4o-mini <span class="inf">?</span></td><td class="n">80.2</td><td class="n">80.2</td><td><span class="tag t-warn">replay</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/consensus__opus-4.7__2026-07-16-replay.eval" download>.eval</a></td></tr>
<tr data-s="deepseek-v4-pro consensus 2026-07-16 gpt-4o-mini full cache replay"><td class="m">deepseek-v4-pro</td><td class="d">2026-07-16</td><td class="n">1</td><td class="n">3671</td><td class="j">gpt-4o-mini <span class="inf">?</span></td><td class="n">79.1</td><td class="n">78.5</td><td><span class="tag t-warn">replay</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/consensus__deepseek-v4-pro__2026-07-16-replay.eval" download>.eval</a></td></tr>
<tr data-s="plamo-3.0-prime consensus 2026-07-16 gpt-4o-mini full cache replay"><td class="m">plamo-3.0-prime</td><td class="d">2026-07-16</td><td class="n">1</td><td class="n">3671</td><td class="j">gpt-4o-mini <span class="inf">?</span></td><td class="n">75.2</td><td class="n">74.8</td><td><span class="tag t-warn">replay</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/consensus__plamo-3.0-prime__2026-07-16-replay.eval" download>.eval</a></td></tr>
<tr data-s="medgemma-27b consensus 2026-07-24 gpt-4o-mini fresh"><td class="m">medgemma-27b</td><td class="d">2026-07-24</td><td class="n">1</td><td class="n">3671</td><td class="j">gpt-4o-mini <span class="inf">?</span></td><td class="n">77.6</td><td class="n">76.6</td><td><span class="tag t-ok">fresh</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/consensus__medgemma-27b__2026-07-24.eval" download>.eval</a></td></tr>
<tr data-s="medgemma-27b consensus 2026-08-05 gpt-4.1 fresh"><td class="m">medgemma-27b</td><td class="d">2026-08-05</td><td class="n">1</td><td class="n">3671</td><td class="j">gpt-4.1</td><td class="n">91.0</td><td class="n">90.1</td><td><span class="tag t-ok">fresh</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/consensus__medgemma-27b__2026-08-05.eval" download>.eval</a></td></tr>
<tr data-s="medgemma-4b consensus 2026-07-24 gpt-4o-mini fresh"><td class="m">medgemma-4b</td><td class="d">2026-07-24</td><td class="n">1</td><td class="n">3671</td><td class="j">gpt-4o-mini <span class="inf">?</span></td><td class="n">71.4</td><td class="n">70.8</td><td><span class="tag t-ok">fresh</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/consensus__medgemma-4b__2026-07-24.eval" download>.eval</a></td></tr>
<tr data-s="medgemma-4b consensus 2026-08-05 gpt-4.1 fresh"><td class="m">medgemma-4b</td><td class="d">2026-08-05</td><td class="n">1</td><td class="n">3671</td><td class="j">gpt-4.1</td><td class="n">75.8</td><td class="n">75.2</td><td><span class="tag t-ok">fresh</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/consensus__medgemma-4b__2026-08-05.eval" download>.eval</a></td></tr>
</tbody></table>
<h3 id="hard">HealthBench hard <span class="cnt">13 logs</span></h3>
<p class="blurb">1000 samples, the hardest slice</p>
<table class="logs"><thead><tr><th>Model</th><th>Date</th><th class="n">ep</th><th class="n">n</th><th>Judge</th><th class="n">raw</th><th class="n">adj</th><th>Provenance</th><th>File</th></tr></thead><tbody>
<tr data-s="gpt-5.5 hard 2026-07-09 &amp;mdash; not-a-run"><td class="m">gpt-5.5</td><td class="d">2026-07-09</td><td class="n">1</td><td class="n">&mdash;</td><td class="j">&mdash;</td><td class="n">&mdash;</td><td class="n">&mdash;</td><td><span class="tag t-bad">failed</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/hard__gpt-5.5__2026-07-09-FAILED.eval" download>.eval</a></td></tr>
<tr data-s="gpt-5.5 hard 2026-07-16 gpt-4o-mini full cache replay"><td class="m">gpt-5.5</td><td class="d">2026-07-16</td><td class="n">1</td><td class="n">1000</td><td class="j">gpt-4o-mini <span class="inf">?</span></td><td class="n">27.3</td><td class="n">26.0</td><td><span class="tag t-warn">replay</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/hard__gpt-5.5__2026-07-16-replay.eval" download>.eval</a></td></tr>
<tr data-s="opus-4.7 hard 2026-07-09 &amp;mdash; not-a-run"><td class="m">opus-4.7</td><td class="d">2026-07-09</td><td class="n">1</td><td class="n">&mdash;</td><td class="j">&mdash;</td><td class="n">&mdash;</td><td class="n">&mdash;</td><td><span class="tag t-bad">failed</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/hard__opus-4.7__2026-07-09-FAILED.eval" download>.eval</a></td></tr>
<tr data-s="opus-4.7 hard 2026-07-16 gpt-4o-mini full cache replay"><td class="m">opus-4.7</td><td class="d">2026-07-16</td><td class="n">1</td><td class="n">1000</td><td class="j">gpt-4o-mini <span class="inf">?</span></td><td class="n">26.6</td><td class="n">27.8</td><td><span class="tag t-warn">replay</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/hard__opus-4.7__2026-07-16-replay.eval" download>.eval</a></td></tr>
<tr data-s="deepseek-v4-pro hard 2026-07-16 gpt-4o-mini full cache replay"><td class="m">deepseek-v4-pro</td><td class="d">2026-07-16</td><td class="n">1</td><td class="n">1000</td><td class="j">gpt-4o-mini <span class="inf">?</span></td><td class="n">24.8</td><td class="n">13.8</td><td><span class="tag t-warn">replay</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/hard__deepseek-v4-pro__2026-07-16-replay.eval" download>.eval</a></td></tr>
<tr data-s="plamo-3.0-prime hard 2026-07-16 gpt-4o-mini full cache replay"><td class="m">plamo-3.0-prime</td><td class="d">2026-07-16</td><td class="n">1</td><td class="n">1000</td><td class="j">gpt-4o-mini <span class="inf">?</span></td><td class="n">17.4</td><td class="n">9.6</td><td><span class="tag t-warn">replay</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/hard__plamo-3.0-prime__2026-07-16-replay.eval" download>.eval</a></td></tr>
<tr data-s="medgemma-27b hard 2026-07-24 gpt-4o-mini mostly cached"><td class="m">medgemma-27b</td><td class="d">2026-07-24</td><td class="n">1</td><td class="n">1000</td><td class="j">gpt-4o-mini <span class="inf">?</span></td><td class="n">21.1</td><td class="n">4.8</td><td><span class="tag t-warn">cached</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/hard__medgemma-27b__2026-07-24-cached.eval" download>.eval</a></td></tr>
<tr data-s="medgemma-27b hard 2026-08-05 gpt-4.1 fresh"><td class="m">medgemma-27b</td><td class="d">2026-08-05</td><td class="n">1</td><td class="n">1000</td><td class="j">gpt-4.1</td><td class="n">13.3</td><td class="n">-4.4</td><td><span class="tag t-ok">fresh</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/hard__medgemma-27b__2026-08-05.eval" download>.eval</a></td></tr>
<tr data-s="medgemma-27b hard 2026-08-05 gpt-4.1 fresh"><td class="m">medgemma-27b</td><td class="d">2026-08-05</td><td class="n">1</td><td class="n">1000</td><td class="j">gpt-4.1</td><td class="n">14.1</td><td class="n">-2.3</td><td><span class="tag t-ok">fresh</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/hard__medgemma-27b__2026-08-05__0807.eval" download>.eval</a></td></tr>
<tr data-s="medgemma-4b hard 2026-07-24 gpt-4o-mini fresh"><td class="m">medgemma-4b</td><td class="d">2026-07-24</td><td class="n">1</td><td class="n">1000</td><td class="j">gpt-4o-mini <span class="inf">?</span></td><td class="n">10.6</td><td class="n">1.3</td><td><span class="tag t-ok">fresh</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/hard__medgemma-4b__2026-07-24.eval" download>.eval</a></td></tr>
<tr data-s="medgemma-4b hard 2026-08-05 gpt-4.1 fresh"><td class="m">medgemma-4b</td><td class="d">2026-08-05</td><td class="n">1</td><td class="n">1000</td><td class="j">gpt-4.1</td><td class="n">-3.1</td><td class="n">-16.4</td><td><span class="tag t-ok">fresh</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/hard__medgemma-4b__2026-08-05.eval" download>.eval</a></td></tr>
<tr data-s="medgemma-4b hard 2026-08-05 gpt-4.1 fresh"><td class="m">medgemma-4b</td><td class="d">2026-08-05</td><td class="n">1</td><td class="n">1000</td><td class="j">gpt-4.1</td><td class="n">-3.5</td><td class="n">-12.6</td><td><span class="tag t-ok">fresh</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/hard__medgemma-4b__2026-08-05__0754.eval" download>.eval</a></td></tr>
<tr data-s="gpt-5-nano hard 2026-07-09 &amp;mdash; not-a-run"><td class="m">gpt-5-nano</td><td class="d">2026-07-09</td><td class="n">1</td><td class="n">&mdash;</td><td class="j">&mdash;</td><td class="n">&mdash;</td><td class="n">&mdash;</td><td><span class="tag t-bad">failed</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/hard__gpt-5-nano__2026-07-09-FAILED.eval" download>.eval</a></td></tr>
</tbody></table>
<h3 id="professional">HealthBench Professional <span class="cnt">7 logs</span></h3>
<p class="blurb">525 samples, physician-written, has a human baseline</p>
<table class="logs"><thead><tr><th>Model</th><th>Date</th><th class="n">ep</th><th class="n">n</th><th>Judge</th><th class="n">raw</th><th class="n">adj</th><th>Provenance</th><th>File</th></tr></thead><tbody>
<tr data-s="gpt-5.5 professional 2026-07-24 gpt-5.4 fresh"><td class="m">gpt-5.5</td><td class="d">2026-07-24</td><td class="n">8</td><td class="n">4200</td><td class="j">gpt-5.4</td><td class="n">52.9</td><td class="n">47.8</td><td><span class="tag t-ok">fresh</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/professional__gpt-5.5__2026-07-24.eval" download>.eval</a></td></tr>
<tr data-s="opus-4.7 professional 2026-07-24 gpt-5.4 fresh"><td class="m">opus-4.7</td><td class="d">2026-07-24</td><td class="n">8</td><td class="n">4200</td><td class="j">gpt-5.4</td><td class="n">50.8</td><td class="n">48.0</td><td><span class="tag t-ok">fresh</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/professional__opus-4.7__2026-07-24.eval" download>.eval</a></td></tr>
<tr data-s="deepseek-v4-pro professional 2026-07-25 gpt-5.4 mostly cached"><td class="m">deepseek-v4-pro</td><td class="d">2026-07-25</td><td class="n">1</td><td class="n">525</td><td class="j">gpt-5.4</td><td class="n">34.3</td><td class="n">27.4</td><td><span class="tag t-warn">cached</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/professional__deepseek-v4-pro__2026-07-25-cached.eval" download>.eval</a></td></tr>
<tr data-s="deepseek-v4-pro professional 2026-08-06 gpt-5.4 fresh"><td class="m">deepseek-v4-pro</td><td class="d">2026-08-06</td><td class="n">1</td><td class="n">525</td><td class="j">gpt-5.4</td><td class="n">37.8</td><td class="n">31.0</td><td><span class="tag t-ok">fresh</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/professional__deepseek-v4-pro__2026-08-06.eval" download>.eval</a></td></tr>
<tr data-s="plamo-3.0-prime professional 2026-07-24 gpt-5.4 fresh"><td class="m">plamo-3.0-prime</td><td class="d">2026-07-24</td><td class="n">8</td><td class="n">4200</td><td class="j">gpt-5.4</td><td class="n">20.8</td><td class="n">13.7</td><td><span class="tag t-ok">fresh</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/professional__plamo-3.0-prime__2026-07-24.eval" download>.eval</a></td></tr>
<tr data-s="medgemma-27b professional 2026-07-24 gpt-5.4 fresh"><td class="m">medgemma-27b</td><td class="d">2026-07-24</td><td class="n">8</td><td class="n">4200</td><td class="j">gpt-5.4</td><td class="n">31.2</td><td class="n">20.0</td><td><span class="tag t-ok">fresh</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/professional__medgemma-27b__2026-07-24.eval" download>.eval</a></td></tr>
<tr data-s="medgemma-4b professional 2026-07-25 gpt-5.4 fresh"><td class="m">medgemma-4b</td><td class="d">2026-07-25</td><td class="n">8</td><td class="n">4200</td><td class="j">gpt-5.4</td><td class="n">16.5</td><td class="n">9.0</td><td><span class="tag t-ok">fresh</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/professional__medgemma-4b__2026-07-25.eval" download>.eval</a></td></tr>
</tbody></table>
<h3 id="professional-consult">Professional: consult <span class="cnt">6 logs</span></h3>
<p class="blurb">236 samples</p>
<table class="logs"><thead><tr><th>Model</th><th>Date</th><th class="n">ep</th><th class="n">n</th><th>Judge</th><th class="n">raw</th><th class="n">adj</th><th>Provenance</th><th>File</th></tr></thead><tbody>
<tr data-s="gpt-5.5 professional-consult 2026-07-24 gpt-5.4 full cache replay"><td class="m">gpt-5.5</td><td class="d">2026-07-24</td><td class="n">8</td><td class="n">1888</td><td class="j">gpt-5.4</td><td class="n">51.0</td><td class="n">48.6</td><td><span class="tag t-warn">replay</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/professional-consult__gpt-5.5__2026-07-24-replay.eval" download>.eval</a></td></tr>
<tr data-s="opus-4.7 professional-consult 2026-07-24 gpt-5.4 full cache replay"><td class="m">opus-4.7</td><td class="d">2026-07-24</td><td class="n">8</td><td class="n">1888</td><td class="j">gpt-5.4</td><td class="n">49.1</td><td class="n">47.0</td><td><span class="tag t-warn">replay</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/professional-consult__opus-4.7__2026-07-24-replay.eval" download>.eval</a></td></tr>
<tr data-s="deepseek-v4-pro professional-consult 2026-07-25 gpt-5.4 full cache replay"><td class="m">deepseek-v4-pro</td><td class="d">2026-07-25</td><td class="n">1</td><td class="n">236</td><td class="j">gpt-5.4</td><td class="n">31.2</td><td class="n">25.6</td><td><span class="tag t-warn">replay</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/professional-consult__deepseek-v4-pro__2026-07-25-replay.eval" download>.eval</a></td></tr>
<tr data-s="plamo-3.0-prime professional-consult 2026-07-24 gpt-5.4 full cache replay"><td class="m">plamo-3.0-prime</td><td class="d">2026-07-24</td><td class="n">8</td><td class="n">1888</td><td class="j">gpt-5.4</td><td class="n">21.8</td><td class="n">15.4</td><td><span class="tag t-warn">replay</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/professional-consult__plamo-3.0-prime__2026-07-24-replay.eval" download>.eval</a></td></tr>
<tr data-s="medgemma-27b professional-consult 2026-07-24 gpt-5.4 mostly cached"><td class="m">medgemma-27b</td><td class="d">2026-07-24</td><td class="n">8</td><td class="n">1888</td><td class="j">gpt-5.4</td><td class="n">28.5</td><td class="n">17.8</td><td><span class="tag t-warn">cached</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/professional-consult__medgemma-27b__2026-07-24-cached.eval" download>.eval</a></td></tr>
<tr data-s="medgemma-4b professional-consult 2026-07-25 gpt-5.4 mostly cached"><td class="m">medgemma-4b</td><td class="d">2026-07-25</td><td class="n">8</td><td class="n">1888</td><td class="j">gpt-5.4</td><td class="n">15.2</td><td class="n">8.2</td><td><span class="tag t-warn">cached</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/professional-consult__medgemma-4b__2026-07-25-cached.eval" download>.eval</a></td></tr>
</tbody></table>
<h3 id="professional-writing">Professional: writing <span class="cnt">6 logs</span></h3>
<p class="blurb">142 samples</p>
<table class="logs"><thead><tr><th>Model</th><th>Date</th><th class="n">ep</th><th class="n">n</th><th>Judge</th><th class="n">raw</th><th class="n">adj</th><th>Provenance</th><th>File</th></tr></thead><tbody>
<tr data-s="gpt-5.5 professional-writing 2026-07-24 gpt-5.4 full cache replay"><td class="m">gpt-5.5</td><td class="d">2026-07-24</td><td class="n">8</td><td class="n">1136</td><td class="j">gpt-5.4</td><td class="n">40.6</td><td class="n">36.0</td><td><span class="tag t-warn">replay</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/professional-writing__gpt-5.5__2026-07-24-replay.eval" download>.eval</a></td></tr>
<tr data-s="opus-4.7 professional-writing 2026-07-24 gpt-5.4 full cache replay"><td class="m">opus-4.7</td><td class="d">2026-07-24</td><td class="n">8</td><td class="n">1136</td><td class="j">gpt-5.4</td><td class="n">39.5</td><td class="n">36.1</td><td><span class="tag t-warn">replay</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/professional-writing__opus-4.7__2026-07-24-replay.eval" download>.eval</a></td></tr>
<tr data-s="deepseek-v4-pro professional-writing 2026-07-24 gpt-5.4 fresh"><td class="m">deepseek-v4-pro</td><td class="d">2026-07-24</td><td class="n">8</td><td class="n">1136</td><td class="j">gpt-5.4</td><td class="n">9.5</td><td class="n">5.0</td><td><span class="tag t-ok">fresh</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/professional-writing__deepseek-v4-pro__2026-07-24.eval" download>.eval</a></td></tr>
<tr data-s="deepseek-v4-pro professional-writing 2026-07-25 gpt-5.4 full cache replay"><td class="m">deepseek-v4-pro</td><td class="d">2026-07-25</td><td class="n">1</td><td class="n">142</td><td class="j">gpt-5.4</td><td class="n">9.7</td><td class="n">5.2</td><td><span class="tag t-warn">replay</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/professional-writing__deepseek-v4-pro__2026-07-25-replay.eval" download>.eval</a></td></tr>
<tr data-s="plamo-3.0-prime professional-writing 2026-07-24 gpt-5.4 full cache replay"><td class="m">plamo-3.0-prime</td><td class="d">2026-07-24</td><td class="n">8</td><td class="n">1136</td><td class="j">gpt-5.4</td><td class="n">-2.8</td><td class="n">-4.3</td><td><span class="tag t-warn">replay</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/professional-writing__plamo-3.0-prime__2026-07-24-replay.eval" download>.eval</a></td></tr>
<tr data-s="medgemma-27b professional-writing 2026-07-24 gpt-5.4 mostly cached"><td class="m">medgemma-27b</td><td class="d">2026-07-24</td><td class="n">8</td><td class="n">1136</td><td class="j">gpt-5.4</td><td class="n">18.9</td><td class="n">9.1</td><td><span class="tag t-warn">cached</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/professional-writing__medgemma-27b__2026-07-24-cached.eval" download>.eval</a></td></tr>
</tbody></table>
<h3 id="professional-research">Professional: research <span class="cnt">6 logs</span></h3>
<p class="blurb">147 samples</p>
<table class="logs"><thead><tr><th>Model</th><th>Date</th><th class="n">ep</th><th class="n">n</th><th>Judge</th><th class="n">raw</th><th class="n">adj</th><th>Provenance</th><th>File</th></tr></thead><tbody>
<tr data-s="gpt-5.5 professional-research 2026-07-24 gpt-5.4 full cache replay"><td class="m">gpt-5.5</td><td class="d">2026-07-24</td><td class="n">8</td><td class="n">1176</td><td class="j">gpt-5.4</td><td class="n">68.0</td><td class="n">57.9</td><td><span class="tag t-warn">replay</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/professional-research__gpt-5.5__2026-07-24-replay.eval" download>.eval</a></td></tr>
<tr data-s="opus-4.7 professional-research 2026-07-24 gpt-5.4 full cache replay"><td class="m">opus-4.7</td><td class="d">2026-07-24</td><td class="n">8</td><td class="n">1176</td><td class="j">gpt-5.4</td><td class="n">64.6</td><td class="n">61.1</td><td><span class="tag t-warn">replay</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/professional-research__opus-4.7__2026-07-24-replay.eval" download>.eval</a></td></tr>
<tr data-s="deepseek-v4-pro professional-research 2026-07-24 gpt-5.4 fresh"><td class="m">deepseek-v4-pro</td><td class="d">2026-07-24</td><td class="n">8</td><td class="n">1176</td><td class="j">gpt-5.4</td><td class="n">63.7</td><td class="n">52.9</td><td><span class="tag t-ok">fresh</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/professional-research__deepseek-v4-pro__2026-07-24.eval" download>.eval</a></td></tr>
<tr data-s="deepseek-v4-pro professional-research 2026-07-25 gpt-5.4 full cache replay"><td class="m">deepseek-v4-pro</td><td class="d">2026-07-25</td><td class="n">1</td><td class="n">147</td><td class="j">gpt-5.4</td><td class="n">63.0</td><td class="n">51.8</td><td><span class="tag t-warn">replay</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/professional-research__deepseek-v4-pro__2026-07-25-replay.eval" download>.eval</a></td></tr>
<tr data-s="plamo-3.0-prime professional-research 2026-07-24 gpt-5.4 full cache replay"><td class="m">plamo-3.0-prime</td><td class="d">2026-07-24</td><td class="n">8</td><td class="n">1176</td><td class="j">gpt-5.4</td><td class="n">41.8</td><td class="n">28.6</td><td><span class="tag t-warn">replay</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/professional-research__plamo-3.0-prime__2026-07-24-replay.eval" download>.eval</a></td></tr>
<tr data-s="medgemma-27b professional-research 2026-07-24 gpt-5.4 mostly cached"><td class="m">medgemma-27b</td><td class="d">2026-07-24</td><td class="n">8</td><td class="n">1176</td><td class="j">gpt-5.4</td><td class="n">47.8</td><td class="n">34.4</td><td><span class="tag t-warn">cached</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/professional-research__medgemma-27b__2026-07-24-cached.eval" download>.eval</a></td></tr>
</tbody></table>
<h3 id="professional-redteam">Professional: red-teaming <span class="cnt">6 logs</span></h3>
<p class="blurb">191 samples</p>
<table class="logs"><thead><tr><th>Model</th><th>Date</th><th class="n">ep</th><th class="n">n</th><th>Judge</th><th class="n">raw</th><th class="n">adj</th><th>Provenance</th><th>File</th></tr></thead><tbody>
<tr data-s="gpt-5.5 professional-redteam 2026-07-24 gpt-5.4 full cache replay"><td class="m">gpt-5.5</td><td class="d">2026-07-24</td><td class="n">8</td><td class="n">1528</td><td class="j">gpt-5.4</td><td class="n">29.9</td><td class="n">28.2</td><td><span class="tag t-warn">replay</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/professional-redteam__gpt-5.5__2026-07-24-replay.eval" download>.eval</a></td></tr>
<tr data-s="opus-4.7 professional-redteam 2026-07-24 gpt-5.4 full cache replay"><td class="m">opus-4.7</td><td class="d">2026-07-24</td><td class="n">8</td><td class="n">1528</td><td class="j">gpt-5.4</td><td class="n">28.3</td><td class="n">26.7</td><td><span class="tag t-warn">replay</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/professional-redteam__opus-4.7__2026-07-24-replay.eval" download>.eval</a></td></tr>
<tr data-s="deepseek-v4-pro professional-redteam 2026-07-24 gpt-5.4 fresh"><td class="m">deepseek-v4-pro</td><td class="d">2026-07-24</td><td class="n">8</td><td class="n">1528</td><td class="j">gpt-5.4</td><td class="n">-3.7</td><td class="n">-6.9</td><td><span class="tag t-ok">fresh</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/professional-redteam__deepseek-v4-pro__2026-07-24.eval" download>.eval</a></td></tr>
<tr data-s="deepseek-v4-pro professional-redteam 2026-07-25 gpt-5.4 full cache replay"><td class="m">deepseek-v4-pro</td><td class="d">2026-07-25</td><td class="n">1</td><td class="n">191</td><td class="j">gpt-5.4</td><td class="n">-5.3</td><td class="n">-8.3</td><td><span class="tag t-warn">replay</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/professional-redteam__deepseek-v4-pro__2026-07-25-replay.eval" download>.eval</a></td></tr>
<tr data-s="plamo-3.0-prime professional-redteam 2026-07-24 gpt-5.4 full cache replay"><td class="m">plamo-3.0-prime</td><td class="d">2026-07-24</td><td class="n">8</td><td class="n">1528</td><td class="j">gpt-5.4</td><td class="n">-9.9</td><td class="n">-11.8</td><td><span class="tag t-warn">replay</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/professional-redteam__plamo-3.0-prime__2026-07-24-replay.eval" download>.eval</a></td></tr>
<tr data-s="medgemma-27b professional-redteam 2026-07-24 gpt-5.4 full cache replay"><td class="m">medgemma-27b</td><td class="d">2026-07-24</td><td class="n">8</td><td class="n">1528</td><td class="j">gpt-5.4</td><td class="n">1.5</td><td class="n">-6.8</td><td><span class="tag t-warn">replay</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/professional-redteam__medgemma-27b__2026-07-24-replay.eval" download>.eval</a></td></tr>
</tbody></table>
<h3 id="professional-baseline">Professional: physician baseline <span class="cnt">6 logs</span></h3>
<p class="blurb">the human reference, model-independent</p>
<table class="logs"><thead><tr><th>Model</th><th>Date</th><th class="n">ep</th><th class="n">n</th><th>Judge</th><th class="n">raw</th><th class="n">adj</th><th>Provenance</th><th>File</th></tr></thead><tbody>
<tr data-s="gpt-5.5 professional-baseline 2026-07-24 gpt-5.4 baseline(no model gen by design)"><td class="m">gpt-5.5</td><td class="d">2026-07-24</td><td class="n">8</td><td class="n">4200</td><td class="j">gpt-5.4</td><td class="n">44.3</td><td class="n">43.9</td><td><span class="tag t-neutral">baseline</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/professional-baseline__gpt-5.5__2026-07-24.eval" download>.eval</a></td></tr>
<tr data-s="opus-4.7 professional-baseline 2026-07-24 gpt-5.4 baseline(no model gen by design)"><td class="m">opus-4.7</td><td class="d">2026-07-24</td><td class="n">8</td><td class="n">4200</td><td class="j">gpt-5.4</td><td class="n">44.3</td><td class="n">43.9</td><td><span class="tag t-neutral">baseline</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/professional-baseline__opus-4.7__2026-07-24.eval" download>.eval</a></td></tr>
<tr data-s="deepseek-v4-pro professional-baseline 2026-07-24 gpt-5.4 baseline(no model gen by design)"><td class="m">deepseek-v4-pro</td><td class="d">2026-07-24</td><td class="n">8</td><td class="n">4200</td><td class="j">gpt-5.4</td><td class="n">44.3</td><td class="n">43.9</td><td><span class="tag t-neutral">baseline</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/professional-baseline__deepseek-v4-pro__2026-07-24.eval" download>.eval</a></td></tr>
<tr data-s="deepseek-v4-pro professional-baseline 2026-07-25 gpt-5.4 baseline(no model gen by design)"><td class="m">deepseek-v4-pro</td><td class="d">2026-07-25</td><td class="n">1</td><td class="n">525</td><td class="j">gpt-5.4</td><td class="n">43.3</td><td class="n">42.9</td><td><span class="tag t-neutral">baseline</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/professional-baseline__deepseek-v4-pro__2026-07-25.eval" download>.eval</a></td></tr>
<tr data-s="plamo-3.0-prime professional-baseline 2026-07-24 gpt-5.4 baseline(no model gen by design)"><td class="m">plamo-3.0-prime</td><td class="d">2026-07-24</td><td class="n">8</td><td class="n">4200</td><td class="j">gpt-5.4</td><td class="n">44.3</td><td class="n">43.9</td><td><span class="tag t-neutral">baseline</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/professional-baseline__plamo-3.0-prime__2026-07-24.eval" download>.eval</a></td></tr>
<tr data-s="medgemma-27b professional-baseline 2026-07-24 gpt-5.4 baseline(no model gen by design)"><td class="m">medgemma-27b</td><td class="d">2026-07-24</td><td class="n">8</td><td class="n">4200</td><td class="j">gpt-5.4</td><td class="n">43.9</td><td class="n">43.5</td><td><span class="tag t-neutral">baseline</span></td><td class="f"><a href="https://huggingface.co/datasets/kirby44/healthbench-eval-logs/resolve/main/logs/professional-baseline__medgemma-27b__2026-07-24.eval" download>.eval</a></td></tr>
</tbody></table>

    <p class="legend">
      <span class="tag t-ok">fresh</span> model responses generated in this run &nbsp;
      <span class="tag t-warn">replay</span> every call served from cache &nbsp;
      <span class="tag t-warn">cached</span> under 200 candidate tokens per sample &nbsp;
      <span class="tag t-neutral">baseline</span> human responses, no generation by design &nbsp;
      <span class="tag t-bad">failed</span> errored or cancelled<br>
      <span class="inf">?</span> judge not recorded in <code>task_args</code>, inferred from the
      scorer default and confirmed against <code>stats.model_usage</code>
    </p>

    <h2 id="reports">Analysis</h2>

    <table class="plain">
      <thead><tr><th>Document</th><th>What it answers</th></tr></thead>
      <tbody>
        <tr>
          <td><a href="matrix.html">Coverage matrix</a></td>
          <td>Which model ran which bench, which cells are comparable, and what is still missing.</td>
        </tr>
        <tr>
          <td><a href="config-check-v2.html">Config check v2</a></td>
          <td>Do our numbers reproduce OpenAI's published HealthBench results? Anchored on the
          physician baseline (ours 43.9 against their 43.7).</td>
        </tr>
        <tr>
          <td><a href="config-check-v1.html">Config check v1</a></td>
          <td>The earlier pass over the first four spaces. Superseded by v2, kept for history.</td>
        </tr>
      </tbody>
    </table>

    <h2 id="trust">Before you quote a number</h2>

    <p>Four things will bite you if you take a score straight out of a log.</p>

    <table class="plain">
      <thead><tr><th>Issue</th><th>What to do</th></tr></thead>
      <tbody>
        <tr>
          <td><b>The judge is not constant.</b> Three graders are in play: <code>gpt-4o-mini</code>
          (hard, consensus), <code>gpt-4.1</code> (full, and the Aug-05 MedGemma re-runs),
          <code>gpt-5.4</code> (all Professional). Swapping the judge moves a score by up to 14
          points, and not always in the same direction.</td>
          <td>Only compare runs sharing a judge. The <code>Judge</code> column above is the check.</td>
        </tr>
        <tr>
          <td><b>Professional epochs are inconsistent.</b> 8 epochs for most models, 1 for DeepSeek.</td>
          <td>Check the <code>ep</code> column before putting two Professional rows side by side.</td>
        </tr>
        <tr>
          <td><b>In-log subset metrics are wrong.</b> <code>use_case_*_score</code>,
          <code>specialty_*_score</code>, <code>difficulty_*_score</code> and
          <code>source_slice_*_score</code> drop the length adjustment and clip each sample to
          [0,1] first. Errors run up to +32 points, always upward.</td>
          <td>Use the standalone <code>professional-*</code> logs above for the four use-case
          slices. For specialty and difficulty, re-aggregate from per-sample scores yourself.</td>
        </tr>
        <tr>
          <td><b><code>cache=true</code> on every run.</b> 34 of 68 logs served some
          or all calls from cache, so an empty <code>stats.model_usage</code> is a replay, not a run.</td>
          <td>The <code>Provenance</code> column above already classifies this.</td>
        </tr>
      </tbody>
    </table>

    <div class="callout">
      <span class="ico">&#9733;</span>
      <div>
        <p>The pipeline itself is validated: the physician baseline on Professional lands at
        <b>43.9</b> against OpenAI's published <b>43.7</b>. Discrepancies in the model numbers are
        config drift, not a broken harness.</p>
      </div>
    </div>

    <h2 id="how">How this was assembled</h2>

    <p>The runs were executed by Ajay between 2026-07-09 and 2026-08-06 and originally published as
    ten separate HuggingFace Spaces under <code>ajay-citadel</code>. This Space consolidates all of
    them into one viewer, renames the logs so the config is legible from the filename, and adds the
    provenance classification that the raw logs do not carry.</p>

    <p><code>data/log_mapping.csv</code> maps every renamed file back to its original space and
    filename, so nothing here is a dead end. <code>data/MANIFEST.csv</code> is the full per-run
    header dump: judge, epochs, token counts, package versions.
    <code>data/INDEX.md</code> is the short version of the traps list.</p>

    <p>The original spaces remain the upstream source. If a number here disagrees with one there,
    the logs are byte-identical, so the difference is in which run you are reading, not in the data.</p>
  </main>
</div>
<script>
const f=document.getElementById('filter'),rows=[...document.querySelectorAll('table.logs tbody tr')],
      cnt=document.getElementById('fcount');
f.addEventListener('input',()=>{
  const q=f.value.trim().toLowerCase().split(/\s+/).filter(Boolean);
  let shown=0;
  rows.forEach(r=>{const s=r.dataset.s,hit=q.every(t=>s.includes(t));r.style.display=hit?'':'none';if(hit)shown++;});
  document.querySelectorAll('table.logs').forEach(t=>{
    const any=[...t.querySelectorAll('tbody tr')].some(r=>r.style.display!=='none');
    const h3=t.previousElementSibling.previousElementSibling;
    t.style.display=any?'':'none';
    if(h3&&h3.tagName==='H3'){h3.style.display=any?'':'none';t.previousElementSibling.style.display=any?'':'none';}
  });
  cnt.textContent=q.length?shown+' of '+rows.length+' logs match':rows.length+' logs';
});
</script>
</body>
</html>