DarwinX project page
Browse files- .gitattributes +5 -0
- README.md +167 -5
- assets/data.js +558 -0
- assets/full/overview.png +3 -0
- assets/full/population.png +3 -0
- assets/full/teaser.png +3 -0
- assets/overview.png +3 -0
- assets/population.png +3 -0
- assets/tb21_category_gain.png +0 -0
- assets/tb21_compute.png +0 -0
- assets/teaser.png +0 -0
- assets/teaser_schematic.png +0 -0
- assets/tw_heldout_specialists.png +0 -0
- assets/wai_audit_by_app.png +0 -0
- assets/wai_evolution_curve.png +0 -0
- assets/wai_invalid_composition.png +0 -0
- assets/wai_tree.png +0 -0
- index.html +1704 -17
- tools/build_data.py +223 -0
- tools/deploy_hf.sh +58 -0
- tools/interaction_test.html +193 -0
.gitattributes
CHANGED
|
@@ -33,3 +33,8 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
+
assets/full/overview.png filter=lfs diff=lfs merge=lfs -text
|
| 37 |
+
assets/full/population.png filter=lfs diff=lfs merge=lfs -text
|
| 38 |
+
assets/full/teaser.png filter=lfs diff=lfs merge=lfs -text
|
| 39 |
+
assets/overview.png filter=lfs diff=lfs merge=lfs -text
|
| 40 |
+
assets/population.png filter=lfs diff=lfs merge=lfs -text
|
README.md
CHANGED
|
@@ -1,10 +1,172 @@
|
|
| 1 |
---
|
| 2 |
-
title:
|
| 3 |
-
emoji:
|
| 4 |
-
colorFrom:
|
| 5 |
-
colorTo:
|
| 6 |
sdk: static
|
|
|
|
| 7 |
pinned: false
|
|
|
|
|
|
|
| 8 |
---
|
| 9 |
|
| 10 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
---
|
| 2 |
+
title: DarwinX
|
| 3 |
+
emoji: 🧬
|
| 4 |
+
colorFrom: blue
|
| 5 |
+
colorTo: indigo
|
| 6 |
sdk: static
|
| 7 |
+
app_file: index.html
|
| 8 |
pinned: false
|
| 9 |
+
license: cc-by-4.0
|
| 10 |
+
short_description: Evolving Agent Harnesses Through Natural Selection
|
| 11 |
---
|
| 12 |
|
| 13 |
+
# DarwinX: Evolving Agent Harnesses Through Natural Selection
|
| 14 |
+
|
| 15 |
+
Project page for [arXiv:2608.07545](https://arxiv.org/abs/2608.07545).
|
| 16 |
+
|
| 17 |
+
DarwinX treats agent self-evolution as selection over a *population* of harnesses with the base
|
| 18 |
+
model frozen. Across four benchmarks, one loop adds ~17 points on average:
|
| 19 |
+
|
| 20 |
+
| Benchmark | Result | Gain | What it isolates |
|
| 21 |
+
| --- | --- | --- | --- |
|
| 22 |
+
| Terminal-Bench 2.1 (avg@5) | 84.7% | +7.7 on matched base | in-domain evolution |
|
| 23 |
+
| TerminalWorld (held-out) | 68.3% | +7.3 | disjoint held-out task split |
|
| 24 |
+
| WebArena-Infinity (audit-clean pass@1) | 93.0% | +49.5 pp | synthetic → real intent shift |
|
| 25 |
+
| SWE-bench Verified (zero-shot transfer) | 84.2% | +3.4 | cross-benchmark transfer |
|
| 26 |
+
|
| 27 |
+
## Page sections
|
| 28 |
+
|
| 29 |
+
`Abstract` · `Method` · `Results at a glance` · `Benchmark detail` · `Ablation: what evolution
|
| 30 |
+
changes` · `Limitations` · `BibTeX`
|
| 31 |
+
|
| 32 |
+
The ablation section reproduces the paper's framing: it is an *exploratory attribution, not a
|
| 33 |
+
per-skill causal ablation*, since the skills were co-selected rather than independently randomized.
|
| 34 |
+
The limitations section is carried over from the paper's Discussion.
|
| 35 |
+
|
| 36 |
+
## Interactive figures
|
| 37 |
+
|
| 38 |
+
Eight figures are interactive, and each is driven by real run artifacts or the paper's own plotting
|
| 39 |
+
scripts rather than illustrative numbers. `tools/build_data.py` regenerates `assets/data.js` from the
|
| 40 |
+
original files, so nothing is hand-transcribed.
|
| 41 |
+
|
| 42 |
+
**1. TerminalWorld merge explorer.** Toggle any subset of the four evolved specialists and watch
|
| 43 |
+
which of the 41 held-out tasks the selection covers. Built from the `per_task_results` arrays of the
|
| 44 |
+
five evaluation runs, joined on `task_id` (the run files list the tasks in different orders, so
|
| 45 |
+
position-based joining would silently misalign them).
|
| 46 |
+
|
| 47 |
+
It surfaces something the paper states only in aggregate. The four specialists solve 24/25/26/27
|
| 48 |
+
tasks and their **union is 29**, but the harness recombination actually produced solves **28**:
|
| 49 |
+
|
| 50 |
+
| | tasks |
|
| 51 |
+
| --- | --- |
|
| 52 |
+
| best single specialist (D) | 27/41 |
|
| 53 |
+
| union of all four | 29/41 |
|
| 54 |
+
| realized merge | 28/41 |
|
| 55 |
+
| solved by merge, by no specialist | `tw_448247` |
|
| 56 |
+
| solved by some specialist, not by merge | `tw_449421`, `tw_498533` |
|
| 57 |
+
| solved by all four | 21/41 |
|
| 58 |
+
| solved by none | 12/41 |
|
| 59 |
+
|
| 60 |
+
So recombination is not a free set union: it adds a capability no parent had and loses two. These are
|
| 61 |
+
single attempts and one task is worth 2.4 points, so individual flips sit inside the noise band —
|
| 62 |
+
the page says so next to the widget.
|
| 63 |
+
|
| 64 |
+
**2. TB2.1 cluster explorer.** Per-cluster base vs. evolved avg@5, sortable by gain, base rate,
|
| 65 |
+
cluster size, or name, with exact rates on hover. Numbers from `notes/TB21_RESULTS.md` (paired
|
| 66 |
+
protocol, 88 tasks). Deltas are carried over as reported rather than recomputed, because the source
|
| 67 |
+
rounds them from unrounded rates.
|
| 68 |
+
|
| 69 |
+
**3. WebArena-Infinity evolution curve.** All 37 evaluated variants with a best-so-far envelope,
|
| 70 |
+
from `notes/tw_dynamics.json` → `wai_adaptive_scores`. Toggle either series.
|
| 71 |
+
|
| 72 |
+
**4. Headline four-benchmark panels.** Replaces the right half of the teaser. From
|
| 73 |
+
`scripts/gen_summary_figure.py`. The reason to make this one interactive is that the paper's figure
|
| 74 |
+
has to *state* its truncation in the caption: on a shared axis WebArena-Infinity's +49.5 flattens the
|
| 75 |
+
two terminal benchmarks into slivers, so each panel is scaled to its own range. Here the reader can
|
| 76 |
+
switch between per-panel and shared 0–100 axes and see both framings. The best-prior-agent bar can be
|
| 77 |
+
hidden, since those systems use different models and effort settings and are context rather than a
|
| 78 |
+
controlled comparison. Per-panel labels carry the exceptions: on SWE-bench Verified the grey bar is
|
| 79 |
+
the fix-skill reference, not an unevolved Monet, and there is no prior-agent bar.
|
| 80 |
+
|
| 81 |
+
**5. TerminalWorld specialist bars.** From `notes/tw_dynamics.json` → `tw_heldout`. Same axis
|
| 82 |
+
question in miniature: the six arms span 58.5–68.3, so a 0–100 track renders them near-identical.
|
| 83 |
+
Defaults to a 55–70 axis with the full axis one click away and the truncation named in both notes.
|
| 84 |
+
Spec. D and the Claude Code reference are both 27/41, so D's bar landing exactly on the dashed
|
| 85 |
+
reference line is a built-in check that the axis transform is right.
|
| 86 |
+
|
| 87 |
+
**6. TB2.1 compute.** From `scripts/gen_tb21_compute.py` (medians over clean attempts). Switch
|
| 88 |
+
between turns and tokens; the bars share one scale across both task groups so the newly-solved vs.
|
| 89 |
+
already-solved contrast is not rescaled away.
|
| 90 |
+
|
| 91 |
+
**7. WAI invalid-trajectory composition.** From `scripts/gen_wai_invalid_composition.py`. Break the
|
| 92 |
+
293 → 17 collapse down by application or by mechanism. Both rows sit on a shared absolute scale by
|
| 93 |
+
default, which makes the "after" row a near-invisible sliver — that *is* the finding; normalizing
|
| 94 |
+
each row to its own total then shows what the remainder consists of. `build_data.py` asserts both
|
| 95 |
+
decompositions still total 293 and 17.
|
| 96 |
+
|
| 97 |
+
**8. WAI audit dumbbells.** From `scripts/gen_wai_audit_by_app.py`. One line per application from raw
|
| 98 |
+
pass@1 to audited pass@1, so line length reads directly as how much a harness was leaning on
|
| 99 |
+
trajectories the audit rejects. Sortable by audit loss. `build_data.py` cross-checks the audited
|
| 100 |
+
DarwinX column against the per-application table rendered elsewhere on the page.
|
| 101 |
+
|
| 102 |
+
Three figures stay static because they are conceptual diagrams with no underlying data: the teaser's
|
| 103 |
+
selection schematic, the method overview, and the per-generation operators. The archive lineage tree
|
| 104 |
+
also stays static for a different reason — its node/edge data lives in a `state.db` on the cluster,
|
| 105 |
+
not on this machine, so there is nothing truthful to make interactive yet.
|
| 106 |
+
|
| 107 |
+
Tables are click-to-sort. Numeric columns open descending, text columns A–Z, and `Overall` rows stay
|
| 108 |
+
pinned to the bottom.
|
| 109 |
+
|
| 110 |
+
## Local preview
|
| 111 |
+
|
| 112 |
+
```bash
|
| 113 |
+
python3 -m http.server 8000
|
| 114 |
+
# open http://localhost:8000
|
| 115 |
+
```
|
| 116 |
+
|
| 117 |
+
## Regenerate and test
|
| 118 |
+
|
| 119 |
+
```bash
|
| 120 |
+
python3 tools/build_data.py # rebuild assets/data.js from the run artifacts
|
| 121 |
+
# then, with the server running, open:
|
| 122 |
+
# http://localhost:8000/tools/interaction_test.html
|
| 123 |
+
# it drives every widget with real click events and writes PASS/FAIL into the page title
|
| 124 |
+
```
|
| 125 |
+
|
| 126 |
+
Headless run:
|
| 127 |
+
|
| 128 |
+
```bash
|
| 129 |
+
python3 -m http.server 8000 &
|
| 130 |
+
"/Applications/Google Chrome.app/Contents/MacOS/Google Chrome" --headless --disable-gpu \
|
| 131 |
+
--virtual-time-budget=6000 --dump-dom http://localhost:8000/tools/interaction_test.html \
|
| 132 |
+
| grep -o '<title>.*</title>'
|
| 133 |
+
```
|
| 134 |
+
|
| 135 |
+
## Layout
|
| 136 |
+
|
| 137 |
+
```
|
| 138 |
+
index.html the page: inline CSS + vanilla JS, no build step, no external
|
| 139 |
+
requests, no web fonts, no chart library
|
| 140 |
+
assets/data.js generated — real numbers behind the interactive figures
|
| 141 |
+
assets/*.png web-sized figures (max 2000px wide, 256-color) — 736 KB total
|
| 142 |
+
assets/full/*.png full-resolution originals for the three dense diagrams,
|
| 143 |
+
linked from the page via click-to-zoom
|
| 144 |
+
tools/build_data.py regenerates assets/data.js from the original run artifacts
|
| 145 |
+
tools/interaction_test.html click-driven test suite for the interactive figures (49 checks)
|
| 146 |
+
```
|
| 147 |
+
|
| 148 |
+
| Figure | Content |
|
| 149 |
+
| --- | --- |
|
| 150 |
+
| `teaser.png` | four-benchmark teaser (full figure, click-to-zoom target) |
|
| 151 |
+
| `teaser_schematic.png` | teaser's left half — the selection schematic shown on the page |
|
| 152 |
+
| `overview.png` | method overview |
|
| 153 |
+
| `population.png` | per-generation operators (mutation, classification, merge) |
|
| 154 |
+
| `tb21_category_gain.png` | per-cluster TB2.1 gain |
|
| 155 |
+
| `tb21_compute.png` | per-task compute: newly-solved vs already-solved |
|
| 156 |
+
| `tw_heldout_specialists.png` | held-out generalization vs archive diversity |
|
| 157 |
+
| `wai_evolution_curve.png` | best-so-far screening score |
|
| 158 |
+
| `wai_tree.png` | archive lineage tree |
|
| 159 |
+
| `wai_invalid_composition.png` | invalid trajectories, 293 → 17 |
|
| 160 |
+
| `wai_audit_by_app.png` | raw vs audited pass@1 per application |
|
| 161 |
+
|
| 162 |
+
Result figures are rendered from the paper's PDF sources at 200 DPI, then palette-reduced without
|
| 163 |
+
dithering (RMSE vs. truecolor < 0.5%, which is imperceptible for line art but 4.7x smaller).
|
| 164 |
+
|
| 165 |
+
## Hosting
|
| 166 |
+
|
| 167 |
+
The frontmatter above configures this directory as a Hugging Face **static Space**. The same files
|
| 168 |
+
work unchanged as a GitHub Pages site or behind any static file server.
|
| 169 |
+
|
| 170 |
+
One thing to change when hosting: `og:image` and `twitter:image` are relative paths, which most
|
| 171 |
+
crawlers resolve but X/Twitter wants absolute. Replace them with the deployed absolute URL to get a
|
| 172 |
+
large social card.
|
assets/data.js
ADDED
|
@@ -0,0 +1,558 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
/* Generated by tools/build_data.py - do not edit by hand. */
|
| 2 |
+
window.DARWINX = {
|
| 3 |
+
"generated": "2026-08-14 01:23 UTC",
|
| 4 |
+
"tw": {
|
| 5 |
+
"arms": [
|
| 6 |
+
{
|
| 7 |
+
"key": "A",
|
| 8 |
+
"label": "Spec. A"
|
| 9 |
+
},
|
| 10 |
+
{
|
| 11 |
+
"key": "B",
|
| 12 |
+
"label": "Spec. B"
|
| 13 |
+
},
|
| 14 |
+
{
|
| 15 |
+
"key": "C",
|
| 16 |
+
"label": "Spec. C"
|
| 17 |
+
},
|
| 18 |
+
{
|
| 19 |
+
"key": "D",
|
| 20 |
+
"label": "Spec. D"
|
| 21 |
+
},
|
| 22 |
+
{
|
| 23 |
+
"key": "M",
|
| 24 |
+
"label": "Merged"
|
| 25 |
+
}
|
| 26 |
+
],
|
| 27 |
+
"tasks": [
|
| 28 |
+
{
|
| 29 |
+
"id": "tw_433818",
|
| 30 |
+
"bits": "11111"
|
| 31 |
+
},
|
| 32 |
+
{
|
| 33 |
+
"id": "tw_90030",
|
| 34 |
+
"bits": "00011"
|
| 35 |
+
},
|
| 36 |
+
{
|
| 37 |
+
"id": "tw_152535",
|
| 38 |
+
"bits": "11111"
|
| 39 |
+
},
|
| 40 |
+
{
|
| 41 |
+
"id": "tw_326675",
|
| 42 |
+
"bits": "11111"
|
| 43 |
+
},
|
| 44 |
+
{
|
| 45 |
+
"id": "tw_204417",
|
| 46 |
+
"bits": "11111"
|
| 47 |
+
},
|
| 48 |
+
{
|
| 49 |
+
"id": "tw_649919",
|
| 50 |
+
"bits": "11111"
|
| 51 |
+
},
|
| 52 |
+
{
|
| 53 |
+
"id": "tw_661947",
|
| 54 |
+
"bits": "11111"
|
| 55 |
+
},
|
| 56 |
+
{
|
| 57 |
+
"id": "tw_494670",
|
| 58 |
+
"bits": "00000"
|
| 59 |
+
},
|
| 60 |
+
{
|
| 61 |
+
"id": "tw_241711",
|
| 62 |
+
"bits": "00000"
|
| 63 |
+
},
|
| 64 |
+
{
|
| 65 |
+
"id": "tw_448247",
|
| 66 |
+
"bits": "00001"
|
| 67 |
+
},
|
| 68 |
+
{
|
| 69 |
+
"id": "tw_713365",
|
| 70 |
+
"bits": "00000"
|
| 71 |
+
},
|
| 72 |
+
{
|
| 73 |
+
"id": "tw_212956",
|
| 74 |
+
"bits": "11111"
|
| 75 |
+
},
|
| 76 |
+
{
|
| 77 |
+
"id": "tw_165299",
|
| 78 |
+
"bits": "11111"
|
| 79 |
+
},
|
| 80 |
+
{
|
| 81 |
+
"id": "tw_17407",
|
| 82 |
+
"bits": "00000"
|
| 83 |
+
},
|
| 84 |
+
{
|
| 85 |
+
"id": "tw_449421",
|
| 86 |
+
"bits": "00110"
|
| 87 |
+
},
|
| 88 |
+
{
|
| 89 |
+
"id": "tw_360189",
|
| 90 |
+
"bits": "11111"
|
| 91 |
+
},
|
| 92 |
+
{
|
| 93 |
+
"id": "tw_274399",
|
| 94 |
+
"bits": "11111"
|
| 95 |
+
},
|
| 96 |
+
{
|
| 97 |
+
"id": "tw_739013",
|
| 98 |
+
"bits": "10111"
|
| 99 |
+
},
|
| 100 |
+
{
|
| 101 |
+
"id": "tw_11884",
|
| 102 |
+
"bits": "00000"
|
| 103 |
+
},
|
| 104 |
+
{
|
| 105 |
+
"id": "tw_768161",
|
| 106 |
+
"bits": "00000"
|
| 107 |
+
},
|
| 108 |
+
{
|
| 109 |
+
"id": "tw_296822",
|
| 110 |
+
"bits": "11111"
|
| 111 |
+
},
|
| 112 |
+
{
|
| 113 |
+
"id": "tw_418270",
|
| 114 |
+
"bits": "11101"
|
| 115 |
+
},
|
| 116 |
+
{
|
| 117 |
+
"id": "tw_576752",
|
| 118 |
+
"bits": "11111"
|
| 119 |
+
},
|
| 120 |
+
{
|
| 121 |
+
"id": "tw_203087",
|
| 122 |
+
"bits": "01011"
|
| 123 |
+
},
|
| 124 |
+
{
|
| 125 |
+
"id": "tw_123683",
|
| 126 |
+
"bits": "00000"
|
| 127 |
+
},
|
| 128 |
+
{
|
| 129 |
+
"id": "tw_627639",
|
| 130 |
+
"bits": "11111"
|
| 131 |
+
},
|
| 132 |
+
{
|
| 133 |
+
"id": "tw_8593",
|
| 134 |
+
"bits": "11111"
|
| 135 |
+
},
|
| 136 |
+
{
|
| 137 |
+
"id": "tw_321885",
|
| 138 |
+
"bits": "11111"
|
| 139 |
+
},
|
| 140 |
+
{
|
| 141 |
+
"id": "tw_481830",
|
| 142 |
+
"bits": "11111"
|
| 143 |
+
},
|
| 144 |
+
{
|
| 145 |
+
"id": "tw_117921",
|
| 146 |
+
"bits": "11111"
|
| 147 |
+
},
|
| 148 |
+
{
|
| 149 |
+
"id": "tw_19076",
|
| 150 |
+
"bits": "11111"
|
| 151 |
+
},
|
| 152 |
+
{
|
| 153 |
+
"id": "tw_474864",
|
| 154 |
+
"bits": "00000"
|
| 155 |
+
},
|
| 156 |
+
{
|
| 157 |
+
"id": "tw_414193",
|
| 158 |
+
"bits": "00000"
|
| 159 |
+
},
|
| 160 |
+
{
|
| 161 |
+
"id": "tw_15649",
|
| 162 |
+
"bits": "11111"
|
| 163 |
+
},
|
| 164 |
+
{
|
| 165 |
+
"id": "tw_453279",
|
| 166 |
+
"bits": "01111"
|
| 167 |
+
},
|
| 168 |
+
{
|
| 169 |
+
"id": "tw_569867",
|
| 170 |
+
"bits": "11111"
|
| 171 |
+
},
|
| 172 |
+
{
|
| 173 |
+
"id": "tw_498533",
|
| 174 |
+
"bits": "00010"
|
| 175 |
+
},
|
| 176 |
+
{
|
| 177 |
+
"id": "tw_433816",
|
| 178 |
+
"bits": "11101"
|
| 179 |
+
},
|
| 180 |
+
{
|
| 181 |
+
"id": "tw_486982",
|
| 182 |
+
"bits": "11111"
|
| 183 |
+
},
|
| 184 |
+
{
|
| 185 |
+
"id": "tw_247983",
|
| 186 |
+
"bits": "00000"
|
| 187 |
+
},
|
| 188 |
+
{
|
| 189 |
+
"id": "tw_39289",
|
| 190 |
+
"bits": "00000"
|
| 191 |
+
}
|
| 192 |
+
],
|
| 193 |
+
"counts": {
|
| 194 |
+
"A": 24,
|
| 195 |
+
"B": 25,
|
| 196 |
+
"C": 26,
|
| 197 |
+
"D": 27,
|
| 198 |
+
"M": 28
|
| 199 |
+
}
|
| 200 |
+
},
|
| 201 |
+
"tb21Clusters": [
|
| 202 |
+
{
|
| 203 |
+
"name": "ML & scientific computing",
|
| 204 |
+
"n": 23,
|
| 205 |
+
"base": 60.1,
|
| 206 |
+
"evolved": 74.9,
|
| 207 |
+
"delta": 14.8
|
| 208 |
+
},
|
| 209 |
+
{
|
| 210 |
+
"name": "Data & databases",
|
| 211 |
+
"n": 9,
|
| 212 |
+
"base": 83.9,
|
| 213 |
+
"evolved": 97.8,
|
| 214 |
+
"delta": 13.8
|
| 215 |
+
},
|
| 216 |
+
{
|
| 217 |
+
"name": "Algorithms & coding",
|
| 218 |
+
"n": 14,
|
| 219 |
+
"base": 83.1,
|
| 220 |
+
"evolved": 90.0,
|
| 221 |
+
"delta": 6.9
|
| 222 |
+
},
|
| 223 |
+
{
|
| 224 |
+
"name": "System admin & networking",
|
| 225 |
+
"n": 11,
|
| 226 |
+
"base": 92.0,
|
| 227 |
+
"evolved": 97.6,
|
| 228 |
+
"delta": 5.7
|
| 229 |
+
},
|
| 230 |
+
{
|
| 231 |
+
"name": "Parsing, text & media",
|
| 232 |
+
"n": 11,
|
| 233 |
+
"base": 67.3,
|
| 234 |
+
"evolved": 72.2,
|
| 235 |
+
"delta": 5.0
|
| 236 |
+
},
|
| 237 |
+
{
|
| 238 |
+
"name": "Systems & build",
|
| 239 |
+
"n": 11,
|
| 240 |
+
"base": 79.3,
|
| 241 |
+
"evolved": 81.8,
|
| 242 |
+
"delta": 2.5
|
| 243 |
+
},
|
| 244 |
+
{
|
| 245 |
+
"name": "Security & crypto",
|
| 246 |
+
"n": 9,
|
| 247 |
+
"base": 85.1,
|
| 248 |
+
"evolved": 84.4,
|
| 249 |
+
"delta": -0.6
|
| 250 |
+
}
|
| 251 |
+
],
|
| 252 |
+
"waiCurve": [
|
| 253 |
+
19.67,
|
| 254 |
+
58.33,
|
| 255 |
+
41.67,
|
| 256 |
+
50.0,
|
| 257 |
+
50.0,
|
| 258 |
+
50.0,
|
| 259 |
+
50.0,
|
| 260 |
+
58.33,
|
| 261 |
+
70.0,
|
| 262 |
+
51.79,
|
| 263 |
+
58.33,
|
| 264 |
+
57.14,
|
| 265 |
+
51.79,
|
| 266 |
+
67.86,
|
| 267 |
+
55.56,
|
| 268 |
+
58.93,
|
| 269 |
+
60.71,
|
| 270 |
+
70.0,
|
| 271 |
+
57.14,
|
| 272 |
+
67.86,
|
| 273 |
+
55.56,
|
| 274 |
+
38.42,
|
| 275 |
+
70.0,
|
| 276 |
+
77.78,
|
| 277 |
+
70.0,
|
| 278 |
+
34.42,
|
| 279 |
+
77.78,
|
| 280 |
+
77.78,
|
| 281 |
+
70.0,
|
| 282 |
+
62.5,
|
| 283 |
+
70.0,
|
| 284 |
+
77.78,
|
| 285 |
+
70.0,
|
| 286 |
+
77.78,
|
| 287 |
+
70.0,
|
| 288 |
+
77.78,
|
| 289 |
+
70.0
|
| 290 |
+
],
|
| 291 |
+
"headline": [
|
| 292 |
+
{
|
| 293 |
+
"short": "TB 2.1",
|
| 294 |
+
"name": "Terminal-Bench 2.1",
|
| 295 |
+
"metric": "avg@5",
|
| 296 |
+
"model": "GPT-5.5",
|
| 297 |
+
"scope": "89 tasks",
|
| 298 |
+
"baseLabel": "Monet (base)",
|
| 299 |
+
"base": 75.5,
|
| 300 |
+
"evolvedLabel": "Monet (DarwinX)",
|
| 301 |
+
"evolved": 83.2,
|
| 302 |
+
"priorLabel": "Codex",
|
| 303 |
+
"prior": 83.1,
|
| 304 |
+
"delta": 7.7,
|
| 305 |
+
"lo": 70,
|
| 306 |
+
"hi": 88
|
| 307 |
+
},
|
| 308 |
+
{
|
| 309 |
+
"short": "T-World",
|
| 310 |
+
"name": "TerminalWorld",
|
| 311 |
+
"metric": "pass@1",
|
| 312 |
+
"model": "Opus 4.8",
|
| 313 |
+
"scope": "41 held-out tasks",
|
| 314 |
+
"baseLabel": "Monet (base)",
|
| 315 |
+
"base": 61.0,
|
| 316 |
+
"evolvedLabel": "Monet (DarwinX)",
|
| 317 |
+
"evolved": 68.3,
|
| 318 |
+
"priorLabel": "Claude Code",
|
| 319 |
+
"prior": 65.9,
|
| 320 |
+
"delta": 7.3,
|
| 321 |
+
"lo": 55,
|
| 322 |
+
"hi": 75
|
| 323 |
+
},
|
| 324 |
+
{
|
| 325 |
+
"short": "WA-Inf.",
|
| 326 |
+
"name": "WebArena-Infinity",
|
| 327 |
+
"metric": "audit-clean pass@1",
|
| 328 |
+
"model": "GPT-5.5",
|
| 329 |
+
"scope": "1,260 real tasks",
|
| 330 |
+
"baseLabel": "Monet (base)",
|
| 331 |
+
"base": 43.5,
|
| 332 |
+
"evolvedLabel": "Monet (DarwinX)",
|
| 333 |
+
"evolved": 93.0,
|
| 334 |
+
"priorLabel": "GPT-5.5 + Browser Use",
|
| 335 |
+
"prior": 86.1,
|
| 336 |
+
"delta": 49.5,
|
| 337 |
+
"lo": 35,
|
| 338 |
+
"hi": 100
|
| 339 |
+
},
|
| 340 |
+
{
|
| 341 |
+
"short": "SWE-V",
|
| 342 |
+
"name": "SWE-bench Verified",
|
| 343 |
+
"metric": "official pass@1",
|
| 344 |
+
"model": "Opus 4.8",
|
| 345 |
+
"scope": "500 instances, transfer only",
|
| 346 |
+
"baseLabel": "Fix-skill reference",
|
| 347 |
+
"base": 80.8,
|
| 348 |
+
"evolvedLabel": "Monet (DarwinX), transferred",
|
| 349 |
+
"evolved": 84.2,
|
| 350 |
+
"priorLabel": null,
|
| 351 |
+
"prior": null,
|
| 352 |
+
"delta": 3.4,
|
| 353 |
+
"lo": 75,
|
| 354 |
+
"hi": 88
|
| 355 |
+
}
|
| 356 |
+
],
|
| 357 |
+
"twBars": {
|
| 358 |
+
"n": 41,
|
| 359 |
+
"claudeCode": 27,
|
| 360 |
+
"bars": [
|
| 361 |
+
{
|
| 362 |
+
"label": "Base",
|
| 363 |
+
"solved": 25
|
| 364 |
+
},
|
| 365 |
+
{
|
| 366 |
+
"label": "Spec. A",
|
| 367 |
+
"solved": 24
|
| 368 |
+
},
|
| 369 |
+
{
|
| 370 |
+
"label": "Spec. B",
|
| 371 |
+
"solved": 25
|
| 372 |
+
},
|
| 373 |
+
{
|
| 374 |
+
"label": "Spec. C",
|
| 375 |
+
"solved": 26
|
| 376 |
+
},
|
| 377 |
+
{
|
| 378 |
+
"label": "Spec. D",
|
| 379 |
+
"solved": 27
|
| 380 |
+
},
|
| 381 |
+
{
|
| 382 |
+
"label": "Monet (DarwinX)",
|
| 383 |
+
"solved": 28
|
| 384 |
+
}
|
| 385 |
+
]
|
| 386 |
+
},
|
| 387 |
+
"tb21Compute": {
|
| 388 |
+
"groups": [
|
| 389 |
+
"Newly solved (6 tasks)",
|
| 390 |
+
"Already solved (69 tasks)"
|
| 391 |
+
],
|
| 392 |
+
"turns": {
|
| 393 |
+
"base": [
|
| 394 |
+
11,
|
| 395 |
+
12
|
| 396 |
+
],
|
| 397 |
+
"evolved": [
|
| 398 |
+
22,
|
| 399 |
+
13
|
| 400 |
+
]
|
| 401 |
+
},
|
| 402 |
+
"tokens": {
|
| 403 |
+
"base": [
|
| 404 |
+
89,
|
| 405 |
+
125
|
| 406 |
+
],
|
| 407 |
+
"evolved": [
|
| 408 |
+
380,
|
| 409 |
+
172
|
| 410 |
+
]
|
| 411 |
+
}
|
| 412 |
+
},
|
| 413 |
+
"waiInvalid": {
|
| 414 |
+
"application": {
|
| 415 |
+
"labels": [
|
| 416 |
+
"Elation Clinical",
|
| 417 |
+
"Elation Prescription",
|
| 418 |
+
"GitLab Plan & Track",
|
| 419 |
+
"Gmail",
|
| 420 |
+
"Gmail Acct & Contact",
|
| 421 |
+
"Handshake",
|
| 422 |
+
"Linear Acct Mgmt",
|
| 423 |
+
"PayPal Wallet",
|
| 424 |
+
"Super-human",
|
| 425 |
+
"Xero Invoicing"
|
| 426 |
+
],
|
| 427 |
+
"before": [
|
| 428 |
+
29,
|
| 429 |
+
27,
|
| 430 |
+
38,
|
| 431 |
+
21,
|
| 432 |
+
44,
|
| 433 |
+
54,
|
| 434 |
+
26,
|
| 435 |
+
10,
|
| 436 |
+
44,
|
| 437 |
+
0
|
| 438 |
+
],
|
| 439 |
+
"after": [
|
| 440 |
+
2,
|
| 441 |
+
3,
|
| 442 |
+
0,
|
| 443 |
+
0,
|
| 444 |
+
3,
|
| 445 |
+
0,
|
| 446 |
+
0,
|
| 447 |
+
0,
|
| 448 |
+
8,
|
| 449 |
+
1
|
| 450 |
+
],
|
| 451 |
+
"colors": [
|
| 452 |
+
"#4C72B0",
|
| 453 |
+
"#A6C8E0",
|
| 454 |
+
"#DD8452",
|
| 455 |
+
"#E3B23C",
|
| 456 |
+
"#55A868",
|
| 457 |
+
"#8FD19E",
|
| 458 |
+
"#C44E52",
|
| 459 |
+
"#E79EA6",
|
| 460 |
+
"#8172B3",
|
| 461 |
+
"#C7BFE0"
|
| 462 |
+
]
|
| 463 |
+
},
|
| 464 |
+
"mechanism": {
|
| 465 |
+
"labels": [
|
| 466 |
+
"Evaluation plane access",
|
| 467 |
+
"Privileged knowledge",
|
| 468 |
+
"Exploit or privilege escalation",
|
| 469 |
+
"Raw state mutation"
|
| 470 |
+
],
|
| 471 |
+
"before": [
|
| 472 |
+
155,
|
| 473 |
+
97,
|
| 474 |
+
26,
|
| 475 |
+
15
|
| 476 |
+
],
|
| 477 |
+
"after": [
|
| 478 |
+
0,
|
| 479 |
+
0,
|
| 480 |
+
0,
|
| 481 |
+
17
|
| 482 |
+
],
|
| 483 |
+
"colors": [
|
| 484 |
+
"#55A868",
|
| 485 |
+
"#DD8452",
|
| 486 |
+
"#CE6DBD",
|
| 487 |
+
"#8FA6D1"
|
| 488 |
+
]
|
| 489 |
+
}
|
| 490 |
+
},
|
| 491 |
+
"waiAudit": {
|
| 492 |
+
"apps": [
|
| 493 |
+
"Overall",
|
| 494 |
+
"Elation-Clin",
|
| 495 |
+
"Elation-Rx",
|
| 496 |
+
"GitLab",
|
| 497 |
+
"Gmail",
|
| 498 |
+
"Gmail-Acct",
|
| 499 |
+
"Handshake",
|
| 500 |
+
"Linear",
|
| 501 |
+
"PayPal",
|
| 502 |
+
"Superhuman",
|
| 503 |
+
"Xero"
|
| 504 |
+
],
|
| 505 |
+
"rawBase": [
|
| 506 |
+
53.0,
|
| 507 |
+
95.8,
|
| 508 |
+
28.3,
|
| 509 |
+
75.7,
|
| 510 |
+
45.0,
|
| 511 |
+
33.3,
|
| 512 |
+
44.0,
|
| 513 |
+
54.2,
|
| 514 |
+
50.0,
|
| 515 |
+
53.3,
|
| 516 |
+
49.2
|
| 517 |
+
],
|
| 518 |
+
"rawDx": [
|
| 519 |
+
94.4,
|
| 520 |
+
98.3,
|
| 521 |
+
97.5,
|
| 522 |
+
97.9,
|
| 523 |
+
98.3,
|
| 524 |
+
94.2,
|
| 525 |
+
84.0,
|
| 526 |
+
94.2,
|
| 527 |
+
95.7,
|
| 528 |
+
94.2,
|
| 529 |
+
97.5
|
| 530 |
+
],
|
| 531 |
+
"auditedBase": [
|
| 532 |
+
43.5,
|
| 533 |
+
95.8,
|
| 534 |
+
20.0,
|
| 535 |
+
63.6,
|
| 536 |
+
25.0,
|
| 537 |
+
21.7,
|
| 538 |
+
36.5,
|
| 539 |
+
43.3,
|
| 540 |
+
49.3,
|
| 541 |
+
31.7,
|
| 542 |
+
39.2
|
| 543 |
+
],
|
| 544 |
+
"auditedDx": [
|
| 545 |
+
93.0,
|
| 546 |
+
96.7,
|
| 547 |
+
95.0,
|
| 548 |
+
97.9,
|
| 549 |
+
98.3,
|
| 550 |
+
91.7,
|
| 551 |
+
84.0,
|
| 552 |
+
94.2,
|
| 553 |
+
95.7,
|
| 554 |
+
87.5,
|
| 555 |
+
96.7
|
| 556 |
+
]
|
| 557 |
+
}
|
| 558 |
+
};
|
assets/full/overview.png
ADDED
|
Git LFS Details
|
assets/full/population.png
ADDED
|
Git LFS Details
|
assets/full/teaser.png
ADDED
|
Git LFS Details
|
assets/overview.png
ADDED
|
Git LFS Details
|
assets/population.png
ADDED
|
Git LFS Details
|
assets/tb21_category_gain.png
ADDED
|
assets/tb21_compute.png
ADDED
|
assets/teaser.png
ADDED
|
assets/teaser_schematic.png
ADDED
|
assets/tw_heldout_specialists.png
ADDED
|
assets/wai_audit_by_app.png
ADDED
|
assets/wai_evolution_curve.png
ADDED
|
assets/wai_invalid_composition.png
ADDED
|
assets/wai_tree.png
ADDED
|
index.html
CHANGED
|
@@ -1,19 +1,1706 @@
|
|
| 1 |
<!doctype html>
|
| 2 |
-
<html>
|
| 3 |
-
|
| 4 |
-
|
| 5 |
-
|
| 6 |
-
|
| 7 |
-
|
| 8 |
-
|
| 9 |
-
|
| 10 |
-
|
| 11 |
-
|
| 12 |
-
|
| 13 |
-
|
| 14 |
-
|
| 15 |
-
|
| 16 |
-
|
| 17 |
-
|
| 18 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 19 |
</html>
|
|
|
|
| 1 |
<!doctype html>
|
| 2 |
+
<html lang="en">
|
| 3 |
+
<head>
|
| 4 |
+
<meta charset="utf-8" />
|
| 5 |
+
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
| 6 |
+
<title>DarwinX: Evolving Agent Harnesses Through Natural Selection</title>
|
| 7 |
+
<meta name="description" content="DarwinX treats agent self-evolution as selection over a population of harnesses with the model frozen. 84.7% on Terminal-Bench 2.1, 68.3% on held-out TerminalWorld, 93.0% audit-clean on WebArena-Infinity, 84.2% zero-shot transfer to SWE-bench Verified." />
|
| 8 |
+
<meta property="og:type" content="article" />
|
| 9 |
+
<meta property="og:title" content="DarwinX: Evolving Agent Harnesses Through Natural Selection" />
|
| 10 |
+
<meta property="og:description" content="Population selection over agent harnesses with the model frozen. Terminal-Bench 2.1 frontier at 84.7%." />
|
| 11 |
+
<meta property="og:image" content="assets/teaser.png" />
|
| 12 |
+
<meta name="twitter:card" content="summary_large_image" />
|
| 13 |
+
<meta name="twitter:title" content="DarwinX: Evolving Agent Harnesses Through Natural Selection" />
|
| 14 |
+
<meta name="twitter:description" content="A frozen model need not be a fixed agent: harness selection turns evaluation compute into durable capability." />
|
| 15 |
+
<meta name="twitter:image" content="assets/teaser.png" />
|
| 16 |
+
<link rel="icon" href="data:image/svg+xml,<svg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 32 32'><rect width='32' height='32' rx='7' fill='%23032d60'/><path d='M12.6 14.8 19.4 10.6M12.6 17.2 19.4 21.4' stroke='%23fff' stroke-width='1.7'/><circle cx='10' cy='16' r='3.3' fill='%230176d3'/><circle cx='22' cy='9' r='3.3' fill='%23fff'/><circle cx='22' cy='23' r='3.3' fill='%23fff'/></svg>" />
|
| 17 |
+
<style>
|
| 18 |
+
:root {
|
| 19 |
+
--navy: #032d60;
|
| 20 |
+
--blue: #0176d3;
|
| 21 |
+
--ink: #17202c;
|
| 22 |
+
--muted: #5a6b7d;
|
| 23 |
+
--line: #dfe6ee;
|
| 24 |
+
--tint: #f2f7fc;
|
| 25 |
+
--bg: #ffffff;
|
| 26 |
+
--maxw: 920px;
|
| 27 |
+
--navh: 48px;
|
| 28 |
+
}
|
| 29 |
+
* { box-sizing: border-box; }
|
| 30 |
+
html { -webkit-text-size-adjust: 100%; scroll-behavior: smooth; }
|
| 31 |
+
body {
|
| 32 |
+
margin: 0;
|
| 33 |
+
background: var(--bg);
|
| 34 |
+
color: var(--ink);
|
| 35 |
+
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", "Helvetica Neue", Arial, sans-serif;
|
| 36 |
+
font-size: 17px;
|
| 37 |
+
line-height: 1.65;
|
| 38 |
+
}
|
| 39 |
+
.wrap { max-width: var(--maxw); margin: 0 auto; padding: 0 22px; }
|
| 40 |
+
a { color: var(--blue); text-decoration: none; }
|
| 41 |
+
a:hover { text-decoration: underline; }
|
| 42 |
+
|
| 43 |
+
/* ---------- sticky nav ---------- */
|
| 44 |
+
nav.top {
|
| 45 |
+
position: sticky; top: 0; z-index: 50;
|
| 46 |
+
background: rgba(255,255,255,0.94);
|
| 47 |
+
backdrop-filter: saturate(180%) blur(8px);
|
| 48 |
+
border-bottom: 1px solid var(--line);
|
| 49 |
+
height: var(--navh);
|
| 50 |
+
}
|
| 51 |
+
nav.top .wrap { display: flex; align-items: center; gap: 4px; height: 100%; overflow-x: auto; }
|
| 52 |
+
nav.top .brand { font-weight: 700; color: var(--navy); margin-right: 12px; white-space: nowrap; }
|
| 53 |
+
nav.top a {
|
| 54 |
+
color: var(--muted); font-size: 0.85rem; padding: 5px 9px; border-radius: 5px; white-space: nowrap;
|
| 55 |
+
}
|
| 56 |
+
nav.top a:hover { background: var(--tint); color: var(--navy); text-decoration: none; }
|
| 57 |
+
nav.top .spacer { flex: 1 1 auto; }
|
| 58 |
+
nav.top a.cta { color: #fff; background: var(--navy); font-weight: 500; }
|
| 59 |
+
nav.top a.cta:hover { background: #06407f; color: #fff; }
|
| 60 |
+
section[id] { scroll-margin-top: calc(var(--navh) + 14px); }
|
| 61 |
+
|
| 62 |
+
/* ---------- header ---------- */
|
| 63 |
+
header { padding: 52px 0 34px; border-bottom: 1px solid var(--line); }
|
| 64 |
+
h1 {
|
| 65 |
+
font-size: 2.05rem; line-height: 1.25; letter-spacing: -0.015em;
|
| 66 |
+
color: var(--navy); margin: 0 0 20px; font-weight: 700;
|
| 67 |
+
}
|
| 68 |
+
h1 .sub { display: block; font-size: 1.32rem; font-weight: 600; margin-top: 6px; color: var(--ink); }
|
| 69 |
+
.authors { font-size: 1.02rem; margin: 0 0 8px; }
|
| 70 |
+
.authors b { font-weight: 600; }
|
| 71 |
+
sup { font-size: 0.7em; color: var(--blue); }
|
| 72 |
+
.affil, .legend { color: var(--muted); font-size: 0.9rem; margin: 4px 0 0; }
|
| 73 |
+
.legend { margin-top: 12px; font-size: 0.84rem; }
|
| 74 |
+
|
| 75 |
+
.links { display: flex; flex-wrap: wrap; gap: 10px; margin: 26px 0 0; }
|
| 76 |
+
.btn {
|
| 77 |
+
display: inline-flex; align-items: center; padding: 9px 17px; border-radius: 7px;
|
| 78 |
+
background: var(--navy); color: #fff; font-size: 0.94rem; font-weight: 500;
|
| 79 |
+
border: 1px solid var(--navy); cursor: pointer;
|
| 80 |
+
}
|
| 81 |
+
.btn:hover { background: #06407f; text-decoration: none; }
|
| 82 |
+
.btn.ghost { background: #fff; color: var(--navy); }
|
| 83 |
+
.btn.ghost:hover { background: var(--tint); }
|
| 84 |
+
|
| 85 |
+
/* ---------- sections ---------- */
|
| 86 |
+
section { padding: 42px 0; border-bottom: 1px solid var(--line); }
|
| 87 |
+
section:last-of-type { border-bottom: 0; }
|
| 88 |
+
h2 {
|
| 89 |
+
font-size: 0.83rem; text-transform: uppercase; letter-spacing: 0.1em;
|
| 90 |
+
color: var(--muted); font-weight: 700; margin: 0 0 20px;
|
| 91 |
+
}
|
| 92 |
+
h3 { font-size: 1.16rem; color: var(--navy); margin: 0 0 4px; font-weight: 600; letter-spacing: -0.01em; }
|
| 93 |
+
h3 .tag { font-weight: 400; color: var(--muted); font-size: 0.8rem; letter-spacing: 0.02em; }
|
| 94 |
+
h4 { font-size: 1rem; color: var(--ink); margin: 0 0 5px; font-weight: 600; }
|
| 95 |
+
p { margin: 0 0 15px; }
|
| 96 |
+
p:last-child { margin-bottom: 0; }
|
| 97 |
+
|
| 98 |
+
figure { margin: 0; }
|
| 99 |
+
figure img { width: 100%; height: auto; display: block; border: 1px solid var(--line); border-radius: 8px; background: #fff; }
|
| 100 |
+
figure a.zoom { display: block; position: relative; }
|
| 101 |
+
figure a.zoom:hover img { border-color: var(--blue); }
|
| 102 |
+
figure a.zoom::after {
|
| 103 |
+
content: "\2317"; position: absolute; right: 9px; bottom: 9px;
|
| 104 |
+
font-size: 0.85rem; line-height: 1; padding: 5px 7px; border-radius: 5px;
|
| 105 |
+
background: rgba(3,45,96,0.82); color: #fff; opacity: 0; transition: opacity .15s;
|
| 106 |
+
}
|
| 107 |
+
figure a.zoom:hover::after { opacity: 1; }
|
| 108 |
+
figcaption { color: var(--muted); font-size: 0.87rem; margin-top: 10px; line-height: 1.55; }
|
| 109 |
+
.figrow { display: grid; grid-template-columns: 1fr 1fr; gap: 20px; align-items: start; }
|
| 110 |
+
.abstract { font-size: 1.02rem; }
|
| 111 |
+
|
| 112 |
+
/* ---------- headline cards ---------- */
|
| 113 |
+
.cards { display: grid; grid-template-columns: repeat(2, 1fr); gap: 14px; }
|
| 114 |
+
.card { border: 1px solid var(--line); border-radius: 8px; padding: 17px 18px; background: var(--tint); }
|
| 115 |
+
.card .label { font-size: 0.79rem; color: var(--muted); text-transform: uppercase; letter-spacing: 0.06em; font-weight: 600; }
|
| 116 |
+
.card .big { font-size: 1.75rem; font-weight: 700; color: var(--navy); margin: 7px 0 2px; letter-spacing: -0.02em; }
|
| 117 |
+
.card .big .delta { font-size: 0.9rem; font-weight: 600; color: var(--blue); margin-left: 7px; letter-spacing: 0; }
|
| 118 |
+
.card .note { font-size: 0.85rem; color: var(--muted); line-height: 1.5; }
|
| 119 |
+
|
| 120 |
+
/* ---------- per-benchmark blocks ---------- */
|
| 121 |
+
.bench { padding-top: 34px; margin-top: 34px; border-top: 1px solid var(--line); }
|
| 122 |
+
.bench:first-of-type { padding-top: 0; margin-top: 0; border-top: 0; }
|
| 123 |
+
.bench > .lede { color: var(--muted); font-size: 0.93rem; margin: 0 0 20px; }
|
| 124 |
+
.block { margin-top: 28px; }
|
| 125 |
+
|
| 126 |
+
table { width: 100%; border-collapse: collapse; font-size: 0.94rem; }
|
| 127 |
+
caption { text-align: left; color: var(--muted); font-size: 0.87rem; padding-bottom: 10px; line-height: 1.5; }
|
| 128 |
+
th, td { text-align: left; padding: 8px 9px; border-bottom: 1px solid var(--line); vertical-align: top; }
|
| 129 |
+
th { font-size: 0.78rem; text-transform: uppercase; letter-spacing: 0.04em; color: var(--muted); border-bottom: 1.5px solid #c9d4e0; font-weight: 700; }
|
| 130 |
+
td.num, th.num { text-align: right; font-variant-numeric: tabular-nums; }
|
| 131 |
+
tr.ours td { background: var(--tint); font-weight: 600; color: var(--navy); }
|
| 132 |
+
tr.base td { color: var(--muted); font-style: italic; }
|
| 133 |
+
tr.total td { border-top: 1.5px solid #c9d4e0; font-weight: 700; }
|
| 134 |
+
td.delta { color: var(--blue); font-weight: 600; }
|
| 135 |
+
.tnote { color: var(--muted); font-size: 0.85rem; margin-top: 11px; line-height: 1.55; }
|
| 136 |
+
.scroller { overflow-x: auto; }
|
| 137 |
+
.scroller table { min-width: 640px; }
|
| 138 |
+
code {
|
| 139 |
+
font-family: ui-monospace, SFMono-Regular, "SF Mono", Menlo, monospace;
|
| 140 |
+
font-size: 0.85em; background: #eef3f8; padding: 1px 5px; border-radius: 4px; color: var(--navy);
|
| 141 |
+
white-space: nowrap;
|
| 142 |
+
}
|
| 143 |
+
|
| 144 |
+
ul.mech { list-style: none; padding: 0; margin: 0; }
|
| 145 |
+
ul.mech li { padding: 0 0 18px 0; }
|
| 146 |
+
ul.mech li:last-child { padding-bottom: 0; }
|
| 147 |
+
ul.lim { padding-left: 20px; margin: 0; }
|
| 148 |
+
ul.lim li { padding-bottom: 12px; }
|
| 149 |
+
ul.lim li:last-child { padding-bottom: 0; }
|
| 150 |
+
ul.lim b { color: var(--navy); }
|
| 151 |
+
|
| 152 |
+
/* ---------- interactive widgets ---------- */
|
| 153 |
+
.widget { border: 1px solid var(--line); border-radius: 9px; padding: 18px; background: #fcfdfe; }
|
| 154 |
+
.widget .wtop {
|
| 155 |
+
display: flex; flex-wrap: wrap; align-items: center; gap: 8px 14px;
|
| 156 |
+
padding-bottom: 15px; margin-bottom: 15px; border-bottom: 1px solid var(--line);
|
| 157 |
+
}
|
| 158 |
+
.widget .wtop .wlabel { font-size: 0.78rem; text-transform: uppercase; letter-spacing: 0.06em; color: var(--muted); font-weight: 700; }
|
| 159 |
+
.chips { display: flex; flex-wrap: wrap; gap: 7px; }
|
| 160 |
+
.chip {
|
| 161 |
+
font: inherit; font-size: 0.85rem; padding: 5px 12px; border-radius: 20px;
|
| 162 |
+
border: 1px solid var(--line); background: #fff; color: var(--muted); cursor: pointer;
|
| 163 |
+
transition: background .12s, border-color .12s, color .12s;
|
| 164 |
+
}
|
| 165 |
+
.chip:hover { border-color: #b9c7d6; }
|
| 166 |
+
.chip[aria-pressed="true"] { background: var(--navy); border-color: var(--navy); color: #fff; }
|
| 167 |
+
.chip .cnt { opacity: 0.62; font-variant-numeric: tabular-nums; }
|
| 168 |
+
.toggle { display: inline-flex; align-items: center; gap: 7px; font-size: 0.86rem; color: var(--muted); cursor: pointer; }
|
| 169 |
+
.toggle input { accent-color: var(--navy); width: 15px; height: 15px; }
|
| 170 |
+
|
| 171 |
+
.readout { display: flex; flex-wrap: wrap; gap: 10px; margin-bottom: 16px; }
|
| 172 |
+
.ro {
|
| 173 |
+
flex: 1 1 150px; border: 1px solid var(--line); border-radius: 7px;
|
| 174 |
+
padding: 11px 13px; background: #fff;
|
| 175 |
+
}
|
| 176 |
+
.ro .k { font-size: 0.75rem; text-transform: uppercase; letter-spacing: 0.05em; color: var(--muted); font-weight: 600; }
|
| 177 |
+
.ro .v { font-size: 1.35rem; font-weight: 700; color: var(--navy); font-variant-numeric: tabular-nums; letter-spacing: -0.01em; }
|
| 178 |
+
.ro.hi { background: var(--tint); border-color: #c9dcf0; }
|
| 179 |
+
.ro .v small { font-size: 0.62em; font-weight: 600; color: var(--muted); }
|
| 180 |
+
|
| 181 |
+
.tgrid { display: grid; grid-template-columns: repeat(auto-fill, minmax(26px, 1fr)); gap: 5px; }
|
| 182 |
+
.tcell {
|
| 183 |
+
position: relative; aspect-ratio: 1; border-radius: 4px; background: #e6ecf2;
|
| 184 |
+
border: 1px solid transparent; cursor: default; transition: background .12s;
|
| 185 |
+
}
|
| 186 |
+
.tcell.on { background: var(--blue); }
|
| 187 |
+
.tcell.gain { box-shadow: 0 0 0 2px #e8a33d; z-index: 1; }
|
| 188 |
+
.tcell.loss { box-shadow: 0 0 0 2px #d05a4e; z-index: 1; }
|
| 189 |
+
.tcell:hover { border-color: var(--navy); }
|
| 190 |
+
|
| 191 |
+
.wlegend { display: flex; flex-wrap: wrap; gap: 14px; margin-top: 14px; font-size: 0.8rem; color: var(--muted); }
|
| 192 |
+
.wlegend span { display: inline-flex; align-items: center; gap: 6px; }
|
| 193 |
+
.sw { width: 13px; height: 13px; border-radius: 3px; display: inline-block; }
|
| 194 |
+
|
| 195 |
+
#tip {
|
| 196 |
+
position: fixed; z-index: 100; pointer-events: none; opacity: 0;
|
| 197 |
+
background: var(--navy); color: #fff; font-size: 0.78rem; line-height: 1.45;
|
| 198 |
+
padding: 7px 10px; border-radius: 6px; max-width: 260px;
|
| 199 |
+
transition: opacity .1s; box-shadow: 0 4px 14px rgba(3,45,96,0.25);
|
| 200 |
+
}
|
| 201 |
+
#tip b { color: #9ecbf5; font-weight: 600; }
|
| 202 |
+
|
| 203 |
+
/* cluster bars */
|
| 204 |
+
.cbars { display: flex; flex-direction: column; gap: 9px; }
|
| 205 |
+
.cbar { display: grid; grid-template-columns: minmax(120px, 1.5fr) 3fr auto; gap: 12px; align-items: center; }
|
| 206 |
+
.cbar .cname { font-size: 0.86rem; color: var(--ink); }
|
| 207 |
+
.cbar .cname .cn { color: var(--muted); font-size: 0.78rem; }
|
| 208 |
+
.cbar .track { position: relative; height: 26px; background: #f0f4f8; border-radius: 4px; overflow: hidden; }
|
| 209 |
+
.cbar .fill { position: absolute; top: 0; height: 100%; border-radius: 4px; transition: width .35s ease, left .35s ease; }
|
| 210 |
+
.cbar .fill.b { background: #c6d3e0; }
|
| 211 |
+
.cbar .fill.e { background: var(--blue); opacity: 0.92; }
|
| 212 |
+
.cbar .cd { font-size: 0.84rem; font-weight: 700; font-variant-numeric: tabular-nums; min-width: 52px; text-align: right; }
|
| 213 |
+
.cbar .cd.up { color: var(--blue); }
|
| 214 |
+
.cbar .cd.down { color: #b3453a; }
|
| 215 |
+
|
| 216 |
+
/* line chart */
|
| 217 |
+
.chart { width: 100%; height: auto; display: block; overflow: visible; }
|
| 218 |
+
.chart .ax { stroke: #ccd7e2; stroke-width: 1; }
|
| 219 |
+
.chart .gl { stroke: #eef2f6; stroke-width: 1; }
|
| 220 |
+
.chart .lbl { font-size: 10.5px; fill: var(--muted); }
|
| 221 |
+
.chart .best { fill: none; stroke: var(--blue); stroke-width: 2.2; }
|
| 222 |
+
.chart .dot { fill: #b9c7d6; }
|
| 223 |
+
.chart .dot:hover, .chart .dot.act { fill: var(--navy); }
|
| 224 |
+
.chart .hit { fill: transparent; cursor: pointer; }
|
| 225 |
+
|
| 226 |
+
/* headline panels */
|
| 227 |
+
.panels { display: grid; grid-template-columns: repeat(4, 1fr); gap: 12px; }
|
| 228 |
+
.panel { text-align: center; }
|
| 229 |
+
.panel .pt { font-size: 0.95rem; font-weight: 700; color: var(--navy); }
|
| 230 |
+
.panel .ps { font-size: 0.74rem; color: var(--muted); line-height: 1.4; margin-bottom: 6px; }
|
| 231 |
+
.panel svg { width: 100%; height: auto; display: block; overflow: visible; }
|
| 232 |
+
.panel .bar { transition: y .35s ease, height .35s ease; }
|
| 233 |
+
.panel .bar.b { fill: #c6d3e0; }
|
| 234 |
+
.panel .bar.p { fill: #8fb4e8; }
|
| 235 |
+
.panel .bar.e { fill: var(--blue); }
|
| 236 |
+
.panel .dl { font-size: 11.5px; font-weight: 700; fill: var(--blue); }
|
| 237 |
+
.panel .tick { font-size: 9.5px; fill: var(--muted); }
|
| 238 |
+
.panel .axl { stroke: #ccd7e2; stroke-width: 1; }
|
| 239 |
+
.panel .foot { font-size: 0.72rem; color: var(--muted); margin-top: 4px; }
|
| 240 |
+
|
| 241 |
+
/* horizontal bar rows (specialists, compute) */
|
| 242 |
+
.hrows { display: flex; flex-direction: column; gap: 8px; }
|
| 243 |
+
.hrow { display: grid; grid-template-columns: minmax(96px, 1.1fr) 3fr auto; gap: 12px; align-items: center; }
|
| 244 |
+
.hrow .hn { font-size: 0.86rem; }
|
| 245 |
+
.hrow .ht { position: relative; height: 24px; background: #f0f4f8; border-radius: 4px; }
|
| 246 |
+
.hrow .hf { position: absolute; left: 0; top: 0; height: 100%; border-radius: 4px; transition: width .35s ease; }
|
| 247 |
+
.hrow .hv { font-size: 0.84rem; font-weight: 700; font-variant-numeric: tabular-nums; min-width: 68px; text-align: right; color: var(--navy); }
|
| 248 |
+
.hrow.dim .hn, .hrow.dim .hv { color: var(--muted); }
|
| 249 |
+
.refline { position: absolute; top: -3px; bottom: -3px; width: 0; border-left: 2px dashed #e4572e; }
|
| 250 |
+
.reftag { position: absolute; top: -17px; transform: translateX(-50%); font-size: 0.7rem; color: #e4572e; white-space: nowrap; }
|
| 251 |
+
|
| 252 |
+
/* stacked composition */
|
| 253 |
+
.stack { display: flex; height: 30px; border-radius: 5px; overflow: hidden; background: #f0f4f8; }
|
| 254 |
+
.stack i { display: block; height: 100%; transition: width .35s ease; }
|
| 255 |
+
.srow { margin-bottom: 16px; }
|
| 256 |
+
.srow .sh { display: flex; justify-content: space-between; font-size: 0.82rem; color: var(--muted); margin-bottom: 5px; }
|
| 257 |
+
.srow .sh b { color: var(--navy); font-size: 0.95rem; }
|
| 258 |
+
.keys { display: flex; flex-wrap: wrap; gap: 6px 14px; margin-top: 12px; font-size: 0.78rem; color: var(--muted); }
|
| 259 |
+
.keys span { display: inline-flex; align-items: center; gap: 6px; }
|
| 260 |
+
|
| 261 |
+
/* dumbbell */
|
| 262 |
+
.dumb { display: flex; flex-direction: column; gap: 7px; }
|
| 263 |
+
.drow { display: grid; grid-template-columns: minmax(84px, 1fr) 4fr; gap: 12px; align-items: center; }
|
| 264 |
+
.drow.tot { padding-bottom: 7px; border-bottom: 1px solid var(--line); }
|
| 265 |
+
.drow.tot .dn { font-weight: 700; color: var(--navy); }
|
| 266 |
+
.drow .dn { font-size: 0.82rem; }
|
| 267 |
+
.drow .dt { position: relative; height: 30px;
|
| 268 |
+
background-image: repeating-linear-gradient(to right, #edf1f5 0 1px, transparent 1px 25%); }
|
| 269 |
+
.drow.dax { align-items: end; }
|
| 270 |
+
.drow.dax .dt { height: 16px; background: none; }
|
| 271 |
+
.drow.dax .dt span { position: absolute; bottom: 0; transform: translateX(-50%);
|
| 272 |
+
font-size: 0.7rem; color: var(--muted); }
|
| 273 |
+
.drow .seg { position: absolute; height: 3px; border-radius: 2px; }
|
| 274 |
+
.drow .pt2 { position: absolute; width: 9px; height: 9px; border-radius: 50%; transform: translate(-50%, -50%); }
|
| 275 |
+
.drow .hollow { background: #fff; }
|
| 276 |
+
|
| 277 |
+
.nojs { display: none; }
|
| 278 |
+
.nojs img { width: 100%; height: auto; border: 1px solid var(--line); border-radius: 8px; }
|
| 279 |
+
|
| 280 |
+
th.sortable { cursor: pointer; user-select: none; white-space: nowrap; }
|
| 281 |
+
th.sortable:hover { color: var(--navy); }
|
| 282 |
+
th.sortable::after { content: " \2195"; opacity: 0.35; font-size: 0.9em; }
|
| 283 |
+
th.sortable.asc::after { content: " \2191"; opacity: 1; color: var(--blue); }
|
| 284 |
+
th.sortable.desc::after { content: " \2193"; opacity: 1; color: var(--blue); }
|
| 285 |
+
|
| 286 |
+
.bibhead { display: flex; align-items: center; justify-content: space-between; gap: 12px; margin-bottom: 12px; }
|
| 287 |
+
.bibhead h2 { margin: 0; }
|
| 288 |
+
#copybtn {
|
| 289 |
+
font: inherit; font-size: 0.84rem; padding: 6px 13px; border-radius: 6px;
|
| 290 |
+
border: 1px solid var(--line); background: #fff; color: var(--navy); cursor: pointer;
|
| 291 |
+
}
|
| 292 |
+
#copybtn:hover { background: var(--tint); border-color: #c9d4e0; }
|
| 293 |
+
pre {
|
| 294 |
+
background: #f7f9fb; border: 1px solid var(--line); border-radius: 8px;
|
| 295 |
+
padding: 16px; overflow-x: auto; font-size: 0.85rem; line-height: 1.55;
|
| 296 |
+
font-family: ui-monospace, SFMono-Regular, "SF Mono", Menlo, monospace;
|
| 297 |
+
color: #223; margin: 0;
|
| 298 |
+
}
|
| 299 |
+
footer { padding: 30px 0 60px; color: var(--muted); font-size: 0.88rem; }
|
| 300 |
+
|
| 301 |
+
@media (max-width: 700px) {
|
| 302 |
+
body { font-size: 16px; }
|
| 303 |
+
h1 { font-size: 1.6rem; }
|
| 304 |
+
header { padding: 32px 0 26px; }
|
| 305 |
+
.cards, .figrow { grid-template-columns: 1fr; }
|
| 306 |
+
nav.top .hideable { display: none; }
|
| 307 |
+
.panels { grid-template-columns: repeat(2, 1fr); gap: 18px 10px; }
|
| 308 |
+
.hrow { grid-template-columns: minmax(72px, 1fr) 2fr auto; gap: 8px; }
|
| 309 |
+
.hrow .hn, .drow .dn { font-size: 0.78rem; }
|
| 310 |
+
.hrow .hv { min-width: 54px; font-size: 0.78rem; }
|
| 311 |
+
.drow { grid-template-columns: minmax(66px, 1fr) 3fr; gap: 8px; }
|
| 312 |
+
}
|
| 313 |
+
</style>
|
| 314 |
+
</head>
|
| 315 |
+
<body>
|
| 316 |
+
<noscript><style>.widget{display:none}.nojs{display:block}</style></noscript>
|
| 317 |
+
|
| 318 |
+
<nav class="top">
|
| 319 |
+
<div class="wrap">
|
| 320 |
+
<span class="brand">DarwinX</span>
|
| 321 |
+
<a href="#abstract">Abstract</a>
|
| 322 |
+
<a href="#method">Method</a>
|
| 323 |
+
<a href="#results">Results</a>
|
| 324 |
+
<a href="#benchmarks" class="hideable">Benchmarks</a>
|
| 325 |
+
<a href="#ablation" class="hideable">Ablation</a>
|
| 326 |
+
<a href="#limitations" class="hideable">Limitations</a>
|
| 327 |
+
<a href="#bibtex" class="hideable">BibTeX</a>
|
| 328 |
+
<span class="spacer"></span>
|
| 329 |
+
<a class="cta" href="https://arxiv.org/abs/2608.07545">arXiv</a>
|
| 330 |
+
</div>
|
| 331 |
+
</nav>
|
| 332 |
+
|
| 333 |
+
<header>
|
| 334 |
+
<div class="wrap">
|
| 335 |
+
<h1>DarwinX<span class="sub">Evolving Agent Harnesses Through Natural Selection</span></h1>
|
| 336 |
+
|
| 337 |
+
<p class="authors">
|
| 338 |
+
<b>Yifan Zhang</b><sup>◦1</sup>,
|
| 339 |
+
<b>Yutong Dai</b><sup>◦1</sup>,
|
| 340 |
+
Juntao Tan<sup>*1</sup>,
|
| 341 |
+
Luyu Yang<sup>*1</sup>,
|
| 342 |
+
Rishi Mullur<sup>1</sup>,
|
| 343 |
+
Thai Hoang<sup>1</sup>,
|
| 344 |
+
Zhiyuan Hu<sup>1</sup>,
|
| 345 |
+
James Zhu<sup>†2</sup>,
|
| 346 |
+
Phil Mui<sup>†2</sup>,
|
| 347 |
+
Silvio Savarese<sup>†1</sup>,
|
| 348 |
+
Ran Xu<sup>†1</sup>,
|
| 349 |
+
Zeyuan Chen<sup>†1</sup>
|
| 350 |
+
</p>
|
| 351 |
+
<p class="affil"><sup>1</sup>Salesforce AI Research <sup>2</sup>Salesforce Agentforce</p>
|
| 352 |
+
<p class="legend"><sup>◦</sup>First authors <sup>*</sup>Core authors <sup>†</sup>Senior authors</p>
|
| 353 |
+
|
| 354 |
+
<div class="links">
|
| 355 |
+
<a class="btn" href="https://arxiv.org/abs/2608.07545">arXiv:2608.07545</a>
|
| 356 |
+
<a class="btn ghost" href="https://arxiv.org/pdf/2608.07545">PDF</a>
|
| 357 |
+
<a class="btn ghost" href="https://huggingface.co/papers/2608.07545">Hugging Face</a>
|
| 358 |
+
<a class="btn ghost" href="mailto:yifan.zhang@salesforce.com?subject=DarwinX%20code%20request">Code available on request</a>
|
| 359 |
+
</div>
|
| 360 |
+
</div>
|
| 361 |
+
</header>
|
| 362 |
+
|
| 363 |
+
<section>
|
| 364 |
+
<div class="wrap">
|
| 365 |
+
<figure>
|
| 366 |
+
<a class="zoom" href="assets/full/teaser.png" target="_blank" rel="noopener">
|
| 367 |
+
<img src="assets/teaser_schematic.png" alt="Variants are evolved from a base agent, survivors are kept and merged, pruned lineages are dropped, all with the model frozen." />
|
| 368 |
+
</a>
|
| 369 |
+
<figcaption>
|
| 370 |
+
<b>With the base model frozen, evolving the harness alone matches or beats the strongest prior
|
| 371 |
+
agent on four benchmarks.</b> Variants survive on measured fitness (avg@<i>k</i>, no gold solutions)
|
| 372 |
+
and complementary survivors are merged. Click for the paper's full figure.
|
| 373 |
+
</figcaption>
|
| 374 |
+
</figure>
|
| 375 |
+
|
| 376 |
+
<div class="block">
|
| 377 |
+
<div class="widget" id="hl-widget">
|
| 378 |
+
<div class="wtop">
|
| 379 |
+
<span class="wlabel">y-axis</span>
|
| 380 |
+
<div class="chips" id="hl-chips"></div>
|
| 381 |
+
<span style="flex:1"></span>
|
| 382 |
+
<label class="toggle"><input type="checkbox" id="hl-prior" checked /> best prior agent</label>
|
| 383 |
+
</div>
|
| 384 |
+
<div class="panels" id="hl-panels"></div>
|
| 385 |
+
<div class="wlegend" id="hl-legend"></div>
|
| 386 |
+
<p class="tnote" id="hl-note"></p>
|
| 387 |
+
</div>
|
| 388 |
+
<noscript class="nojs"><img src="assets/teaser.png" alt="Four-benchmark summary: base vs. evolved vs. best prior agent." /></noscript>
|
| 389 |
+
</div>
|
| 390 |
+
</div>
|
| 391 |
+
</section>
|
| 392 |
+
|
| 393 |
+
<section id="abstract">
|
| 394 |
+
<div class="wrap">
|
| 395 |
+
<h2>Abstract</h2>
|
| 396 |
+
<div class="abstract">
|
| 397 |
+
<p>
|
| 398 |
+
An LLM agent's capability depends not only on model weights but on its <i>harness</i>: prompts,
|
| 399 |
+
tools, skills, and control flow. Self-improvement loops already edit harnesses, yet single-lineage
|
| 400 |
+
search is path-dependent and local wins often regress other tasks. We introduce <b>DarwinX</b>, which
|
| 401 |
+
treats self-evolution as <b>selection over a population of harnesses</b> with the model frozen: a
|
| 402 |
+
preserve-and-extend contract admits only variants that extend coverage without regressing, an archive
|
| 403 |
+
keeps alternative lineages for recombination, and failure-, teacher-, and self-derived evidence share
|
| 404 |
+
one edit interface. Fitness comes from each benchmark's own verifier: no gold solutions, no
|
| 405 |
+
hand-picked winners.
|
| 406 |
+
</p>
|
| 407 |
+
<p>
|
| 408 |
+
Across four benchmarks that progressively separate the evolution signal from the test, one loop adds
|
| 409 |
+
about 17 points on average: Terminal-Bench 2.1 rises +7.7 to 83.2% on a matched base and to the
|
| 410 |
+
verified frontier at <b>84.7%</b> on a stronger one; TerminalWorld's held-out split reaches
|
| 411 |
+
<b>68.3%</b>, ahead of every off-the-shelf agent; WebArena-Infinity real-task pass@1 rises from 43.5%
|
| 412 |
+
to <b>93.0%</b> audit-clean; and a Terminal-Bench 2.1 harness transfers unchanged to SWE-bench
|
| 413 |
+
Verified. What evolves is general agent competence, not benchmark-specific patches, so it survives
|
| 414 |
+
changes of task, verifier, and base model. A frozen model need not be a fixed agent: harness selection
|
| 415 |
+
turns evaluation compute into durable capability.
|
| 416 |
+
</p>
|
| 417 |
+
</div>
|
| 418 |
+
</div>
|
| 419 |
+
</section>
|
| 420 |
+
|
| 421 |
+
<section id="method">
|
| 422 |
+
<div class="wrap">
|
| 423 |
+
<h2>Method</h2>
|
| 424 |
+
|
| 425 |
+
<figure>
|
| 426 |
+
<a class="zoom" href="assets/full/overview.png" target="_blank" rel="noopener">
|
| 427 |
+
<img src="assets/overview.png" alt="Overview of the DarwinX loop." loading="lazy" />
|
| 428 |
+
</a>
|
| 429 |
+
</figure>
|
| 430 |
+
|
| 431 |
+
<ul class="mech" style="margin-top:26px">
|
| 432 |
+
<li>
|
| 433 |
+
<h4>Selection over a population, not one lineage</h4>
|
| 434 |
+
<p>A run maintains an archive shaped as a tree, where each node is a harness snapshot with its edit
|
| 435 |
+
delta, per-task scores, trial evidence, and distilled lessons. Nothing is discarded: a variant that
|
| 436 |
+
loses overall may still hold the one edit that, combined with another branch's, unlocks a task
|
| 437 |
+
neither solves alone. Parallel branches target different capability clusters, so the archive grows
|
| 438 |
+
specialists with different solved-task signatures.</p>
|
| 439 |
+
</li>
|
| 440 |
+
<li>
|
| 441 |
+
<h4>A preserve-and-extend contract</h4>
|
| 442 |
+
<p>A child is admitted only when the measured evidence shows it solved something new
|
| 443 |
+
<i>without breaking what it already solved</i>, which bounds what a win is allowed to cost
|
| 444 |
+
elsewhere. When variants solve complementary tasks, their additive edits are merged, and the merged
|
| 445 |
+
child is kept only if it covers the union of its parents' wins. Fitness is the benchmark's own
|
| 446 |
+
verifier at avg@<i>k</i> — no gold solutions and no hand-picked winners.</p>
|
| 447 |
+
</li>
|
| 448 |
+
<li>
|
| 449 |
+
<h4>One edit interface for three signals</h4>
|
| 450 |
+
<p>Failure-derived diagnosis, teacher-derived demonstrations, and self-derived rollout contrast are
|
| 451 |
+
all converted into harness edits rather than model-weight updates. The base model never changes, so
|
| 452 |
+
every gain reported here is a statement about the harness alone.</p>
|
| 453 |
+
</li>
|
| 454 |
+
</ul>
|
| 455 |
+
|
| 456 |
+
<figure style="margin-top:30px">
|
| 457 |
+
<a class="zoom" href="assets/full/population.png" target="_blank" rel="noopener">
|
| 458 |
+
<img src="assets/population.png" alt="DarwinX per-generation operators: mutation loop, variant classification, and the merge operator." loading="lazy" />
|
| 459 |
+
</a>
|
| 460 |
+
<figcaption>
|
| 461 |
+
<b>DarwinX's per-generation operators.</b> <b>Left:</b> the mutation loop and the three learning
|
| 462 |
+
signals that drive it. <b>Middle:</b> variants classified by how their solved set changes, where those
|
| 463 |
+
preserving inherited solves stay eligible for recombination while the rest contribute only distilled
|
| 464 |
+
lessons. <b>Right:</b> the merge operator and its acceptance criterion.
|
| 465 |
+
</figcaption>
|
| 466 |
+
</figure>
|
| 467 |
+
</div>
|
| 468 |
+
</section>
|
| 469 |
+
|
| 470 |
+
<section id="results">
|
| 471 |
+
<div class="wrap">
|
| 472 |
+
<h2>Results at a glance</h2>
|
| 473 |
+
<div class="cards">
|
| 474 |
+
<div class="card">
|
| 475 |
+
<div class="label">Terminal-Bench 2.1 · avg@5</div>
|
| 476 |
+
<div class="big">84.7%<span class="delta">+7.7 on matched base</span></div>
|
| 477 |
+
<div class="note">Verified-leaderboard frontier on a frozen GPT-5.6 Sol at <i>medium</i> effort.
|
| 478 |
+
On GPT-5.5, 75.5% → 83.2%.</div>
|
| 479 |
+
</div>
|
| 480 |
+
<div class="card">
|
| 481 |
+
<div class="label">TerminalWorld · held-out split</div>
|
| 482 |
+
<div class="big">68.3%<span class="delta">+7.3</span></div>
|
| 483 |
+
<div class="note">28/41 held-out tasks on a frozen Opus 4.8, above every off-the-shelf agent
|
| 484 |
+
we measured.</div>
|
| 485 |
+
</div>
|
| 486 |
+
<div class="card">
|
| 487 |
+
<div class="label">WebArena-Infinity · audit-clean pass@1</div>
|
| 488 |
+
<div class="big">93.0%<span class="delta">+49.5 pp</span></div>
|
| 489 |
+
<div class="note">1,260 real tasks, evolved on synthetic intents only. Base 43.5%; best same-model
|
| 490 |
+
baseline 86.1%.</div>
|
| 491 |
+
</div>
|
| 492 |
+
<div class="card">
|
| 493 |
+
<div class="label">SWE-bench Verified · zero-shot transfer</div>
|
| 494 |
+
<div class="big">84.2%<span class="delta">+3.4</span></div>
|
| 495 |
+
<div class="note">421/500 official pass@1 from the Terminal-Bench 2.1 harness run
|
| 496 |
+
<i>unchanged</i>, with no SWE-V feedback.</div>
|
| 497 |
+
</div>
|
| 498 |
+
</div>
|
| 499 |
+
<p class="tnote">
|
| 500 |
+
The four benchmarks progressively separate the evolution signal from the test: in-domain
|
| 501 |
+
(Terminal-Bench 2.1), held-out task split (TerminalWorld), synthetic → real intent
|
| 502 |
+
shift (WebArena-Infinity), and zero-shot cross-benchmark transfer (SWE-bench Verified).
|
| 503 |
+
</p>
|
| 504 |
+
</div>
|
| 505 |
+
</section>
|
| 506 |
+
|
| 507 |
+
<section id="benchmarks">
|
| 508 |
+
<div class="wrap">
|
| 509 |
+
<h2>Benchmark detail</h2>
|
| 510 |
+
|
| 511 |
+
<!-- ============ TB 2.1 ============ -->
|
| 512 |
+
<div class="bench">
|
| 513 |
+
<h3>Terminal-Bench 2.1 <span class="tag">— in-domain evolution</span></h3>
|
| 514 |
+
<p class="lede">89 tasks. Evolution and evaluation share the task distribution; the model is frozen and
|
| 515 |
+
only the harness changes.</p>
|
| 516 |
+
|
| 517 |
+
<table class="sortable">
|
| 518 |
+
<caption>Terminal-Bench 2.1, avg@5 under the strict leaderboard rule (binary avg@5,
|
| 519 |
+
<i>k</i>=5, errored trials score zero). Both DarwinX rows are leaderboard submissions, reported
|
| 520 |
+
before the leaderboard's uniform reward-hacking pass.</caption>
|
| 521 |
+
<thead>
|
| 522 |
+
<tr><th>Agent</th><th>Model / effort</th><th class="num">avg@5</th></tr>
|
| 523 |
+
</thead>
|
| 524 |
+
<tbody>
|
| 525 |
+
<tr class="ours"><td>Monet (DarwinX)</td><td>GPT-5.6 Sol / medium</td><td class="num">84.7 ± 1.2</td></tr>
|
| 526 |
+
<tr><td>Claude Code</td><td>Fable 5 / xhigh</td><td class="num">83.8 ± 1.2</td></tr>
|
| 527 |
+
<tr class="ours"><td>Monet (DarwinX)</td><td>GPT-5.5 / high</td><td class="num">83.2 ± 1.2</td></tr>
|
| 528 |
+
<tr><td>Codex</td><td>GPT-5.5 / xhigh</td><td class="num">83.1 ± 1.1</td></tr>
|
| 529 |
+
<tr><td>OpenAI reference</td><td>GPT-5.6 Sol / medium</td><td class="num">81.8</td></tr>
|
| 530 |
+
<tr><td>Terminus 2</td><td>GPT-5.5 / xhigh</td><td class="num">78.0 ± 1.2</td></tr>
|
| 531 |
+
<tr class="base"><td>Monet (base)</td><td>GPT-5.5 / default</td><td class="num">75.5 ± 3.5</td></tr>
|
| 532 |
+
</tbody>
|
| 533 |
+
</table>
|
| 534 |
+
<p class="tnote">
|
| 535 |
+
At 84.7% DarwinX matches or exceeds the verified leader (Claude Code + Fable 5, 83.8% at
|
| 536 |
+
<i>xhigh</i>) while running at a <i>lower</i> effort setting, and adds +2.9 points over OpenAI's own
|
| 537 |
+
native single-agent GPT-5.6 Sol at the same medium effort. Against a neutral harness on the
|
| 538 |
+
<i>same</i> GPT-5.5 base (Terminus 2, 78.0%), the pure harness gain is +5.2 points — so the
|
| 539 |
+
improvement is the harness, not the model or the effort setting.
|
| 540 |
+
</p>
|
| 541 |
+
|
| 542 |
+
<div class="block">
|
| 543 |
+
<h4>Where evolution helps</h4>
|
| 544 |
+
<p class="tnote" style="margin-top:0;margin-bottom:14px">
|
| 545 |
+
Per-cluster avg@5 on a frozen GPT-5.5. Sort by any column and hover a bar for exact rates.
|
| 546 |
+
</p>
|
| 547 |
+
<div class="widget" id="cl-widget">
|
| 548 |
+
<div class="wtop">
|
| 549 |
+
<span class="wlabel">Sort by</span>
|
| 550 |
+
<div class="chips" id="cl-chips"></div>
|
| 551 |
+
<span class="spacer" style="flex:1"></span>
|
| 552 |
+
<label class="toggle"><input type="checkbox" id="cl-base" checked /> show base</label>
|
| 553 |
+
</div>
|
| 554 |
+
<div class="cbars" id="cl-bars"></div>
|
| 555 |
+
<div class="wlegend">
|
| 556 |
+
<span><i class="sw" style="background:#c6d3e0"></i> Monet (base)</span>
|
| 557 |
+
<span><i class="sw" style="background:var(--blue)"></i> Monet (DarwinX)</span>
|
| 558 |
+
</div>
|
| 559 |
+
</div>
|
| 560 |
+
<noscript class="nojs"><img src="assets/tb21_category_gain.png" alt="Per-cluster Terminal-Bench 2.1 gain." /></noscript>
|
| 561 |
+
<p class="tnote">
|
| 562 |
+
Gains concentrate where a frozen base has the most headroom: <i>ML & scientific computing</i>
|
| 563 |
+
(+14.8, the largest cluster at 23 tasks) and <i>data & databases</i> (+13.8). Clusters where the
|
| 564 |
+
base is already strong barely move, and none regresses beyond the per-task noise band. Paired
|
| 565 |
+
protocol over 88 tasks, which excludes one task with no paired evolved trial; the headline 83.2%
|
| 566 |
+
uses the stricter 89-task leaderboard rule above.
|
| 567 |
+
</p>
|
| 568 |
+
</div>
|
| 569 |
+
|
| 570 |
+
<div class="block">
|
| 571 |
+
<h4>The gain is the harness, not compute</h4>
|
| 572 |
+
<p class="tnote" style="margin-top:0;margin-bottom:14px">
|
| 573 |
+
Per-task compute, median over clean attempts.
|
| 574 |
+
</p>
|
| 575 |
+
<div class="widget" id="cp-widget">
|
| 576 |
+
<div class="wtop">
|
| 577 |
+
<span class="wlabel">Metric</span>
|
| 578 |
+
<div class="chips" id="cp-chips"></div>
|
| 579 |
+
</div>
|
| 580 |
+
<div class="hrows" id="cp-rows"></div>
|
| 581 |
+
<div class="wlegend">
|
| 582 |
+
<span><i class="sw" style="background:#c6d3e0"></i> Monet (base)</span>
|
| 583 |
+
<span><i class="sw" style="background:var(--blue)"></i> Monet (DarwinX)</span>
|
| 584 |
+
</div>
|
| 585 |
+
<p class="tnote" id="cp-note"></p>
|
| 586 |
+
</div>
|
| 587 |
+
<noscript class="nojs"><img src="assets/tb21_compute.png" alt="Per-task compute, newly solved vs. already solved." /></noscript>
|
| 588 |
+
</div>
|
| 589 |
+
</div>
|
| 590 |
+
|
| 591 |
+
<!-- ============ TerminalWorld ============ -->
|
| 592 |
+
<div class="bench">
|
| 593 |
+
<h3>TerminalWorld <span class="tag">— held-out task split</span></h3>
|
| 594 |
+
<p class="lede">Evolve on 94 verifier-scored training tasks, then freeze the harness and evaluate on 41
|
| 595 |
+
disjoint held-out tasks. Single-attempt pass@1, no retries and no best-of-<i>k</i>, so no held-out
|
| 596 |
+
reward can feed back into the archive.</p>
|
| 597 |
+
|
| 598 |
+
<table class="sortable">
|
| 599 |
+
<caption>TerminalWorld held-out pass@1 (41 tasks, single attempt, frozen base models, identical
|
| 600 |
+
verifier and infrastructure).</caption>
|
| 601 |
+
<thead>
|
| 602 |
+
<tr><th>Agent</th><th>Model</th><th class="num">pass@1</th></tr>
|
| 603 |
+
</thead>
|
| 604 |
+
<tbody>
|
| 605 |
+
<tr class="ours"><td>Monet (DarwinX)</td><td>Opus 4.8</td><td class="num">68.3%</td></tr>
|
| 606 |
+
<tr><td>Claude Code</td><td>Opus 4.8</td><td class="num">65.9%</td></tr>
|
| 607 |
+
<tr class="base"><td>Monet (base)</td><td>Opus 4.8</td><td class="num">61.0%</td></tr>
|
| 608 |
+
<tr><td>Terminus-2</td><td>GPT-5.5</td><td class="num">61.0%</td></tr>
|
| 609 |
+
<tr><td>Terminus-2</td><td>Opus 4.8</td><td class="num">58.5%</td></tr>
|
| 610 |
+
<tr><td>Monet (DarwinX)</td><td>GPT-5.5</td><td class="num">56.1%</td></tr>
|
| 611 |
+
<tr><td>Codex</td><td>GPT-5.5</td><td class="num">51.2%</td></tr>
|
| 612 |
+
<tr class="base"><td>Monet (base)</td><td>GPT-5.5</td><td class="num">48.8%</td></tr>
|
| 613 |
+
</tbody>
|
| 614 |
+
</table>
|
| 615 |
+
|
| 616 |
+
<div class="figrow block">
|
| 617 |
+
<div>
|
| 618 |
+
<div class="widget" id="sp-widget">
|
| 619 |
+
<div class="wtop">
|
| 620 |
+
<span class="wlabel">x-axis</span>
|
| 621 |
+
<div class="chips" id="sp-chips"></div>
|
| 622 |
+
<span style="flex:1"></span>
|
| 623 |
+
<label class="toggle"><input type="checkbox" id="sp-counts" /> task counts</label>
|
| 624 |
+
</div>
|
| 625 |
+
<div class="hrows" id="sp-rows"></div>
|
| 626 |
+
<p class="tnote" id="sp-note"></p>
|
| 627 |
+
</div>
|
| 628 |
+
<noscript class="nojs"><img src="assets/tw_heldout_specialists.png" alt="Held-out pass@1 of four specialists and their merge." /></noscript>
|
| 629 |
+
<p class="tnote">
|
| 630 |
+
Held-out generalization reflects <b>archive diversity</b>: four evolved specialists each solve a
|
| 631 |
+
different subset of the 41 tasks, and the merged Monet (DarwinX) exceeds every specialist and the
|
| 632 |
+
Claude Code reference (dashed).
|
| 633 |
+
</p>
|
| 634 |
+
</div>
|
| 635 |
+
<div>
|
| 636 |
+
<h4>The in-loop proxy overfits; the population absorbs it</h4>
|
| 637 |
+
<p style="font-size:0.95rem">
|
| 638 |
+
During evolution the training-subset score saturates from 0.505 to 1.000, yet held-out pass@1 is
|
| 639 |
+
68.3% — a <b>31.7-point gap</b> between the proxy the search maximizes and the held-out
|
| 640 |
+
truth it never sees. Crucially, the variant that best fits the proxy is <i>not</i> the best
|
| 641 |
+
generalizer: four high-scoring specialists solve 24, 25, 26, and 27 of the 41 held-out tasks on
|
| 642 |
+
overlapping but distinct subsets, and their merge reaches 28/41. Keeping a population, rather than
|
| 643 |
+
following the single best-scoring lineage, is what converts an overfit proxy into held-out gain.
|
| 644 |
+
</p>
|
| 645 |
+
</div>
|
| 646 |
+
</div>
|
| 647 |
+
|
| 648 |
+
<div class="block">
|
| 649 |
+
<h4>Explore the merge: which specialists cover which held-out tasks</h4>
|
| 650 |
+
<p class="tnote" style="margin-top:0;margin-bottom:14px">
|
| 651 |
+
Every cell is one of the 41 held-out tasks, filled when at least one selected specialist solves it.
|
| 652 |
+
Toggle specialists to watch coverage grow, and turn on the comparison to see how the harness DarwinX
|
| 653 |
+
actually produced differs from the plain union. Hover any cell for its task id.
|
| 654 |
+
Built from the per-task records of the five evaluation runs.
|
| 655 |
+
</p>
|
| 656 |
+
<div class="widget" id="tw-widget">
|
| 657 |
+
<div class="wtop">
|
| 658 |
+
<span class="wlabel">Specialists</span>
|
| 659 |
+
<div class="chips" id="tw-chips"></div>
|
| 660 |
+
<span class="spacer" style="flex:1"></span>
|
| 661 |
+
<label class="toggle"><input type="checkbox" id="tw-cmp" /> compare with realized merge</label>
|
| 662 |
+
</div>
|
| 663 |
+
<div class="readout" id="tw-readout"></div>
|
| 664 |
+
<div class="tgrid" id="tw-grid"></div>
|
| 665 |
+
<div class="wlegend" id="tw-legend">
|
| 666 |
+
<span><i class="sw" style="background:var(--blue)"></i> solved by selection</span>
|
| 667 |
+
<span><i class="sw" style="background:#e6ecf2"></i> unsolved</span>
|
| 668 |
+
</div>
|
| 669 |
+
<p class="tnote" id="tw-note"></p>
|
| 670 |
+
</div>
|
| 671 |
+
</div>
|
| 672 |
+
</div>
|
| 673 |
+
|
| 674 |
+
<!-- ============ WebArena-Infinity ============ -->
|
| 675 |
+
<div class="bench">
|
| 676 |
+
<h3>WebArena-Infinity <span class="tag">— synthetic → real intent shift</span></h3>
|
| 677 |
+
<p class="lede">Evolution sees synthetic intents only; evaluation is the official 1,260-task real suite
|
| 678 |
+
across 10 applications. We report <i>audit-clean</i> pass@1 throughout: trajectories flagged invalid
|
| 679 |
+
by the validity audit count as failures even when the verifier passes them.</p>
|
| 680 |
+
|
| 681 |
+
<div class="scroller">
|
| 682 |
+
<table class="sortable">
|
| 683 |
+
<caption>WebArena-Infinity audit-clean pass@1 by application. Click any column header to sort. External baselines are as-reported
|
| 684 |
+
without re-auditing, which makes this comparison conservative for DarwinX. BU = Browser Use.</caption>
|
| 685 |
+
<thead>
|
| 686 |
+
<tr>
|
| 687 |
+
<th>Application</th>
|
| 688 |
+
<th class="num">Kimi</th><th class="num">Qwen</th><th class="num">Gemini+BU</th>
|
| 689 |
+
<th class="num">GPT-5.5+BU</th><th class="num">Monet (base)</th>
|
| 690 |
+
<th class="num">Monet (DarwinX)</th><th class="num">Δ</th>
|
| 691 |
+
</tr>
|
| 692 |
+
</thead>
|
| 693 |
+
<tbody>
|
| 694 |
+
<tr><td>Elation clinical records</td><td class="num">50.0</td><td class="num">54.2</td><td class="num">81.7</td><td class="num">92.5</td><td class="num">95.8</td><td class="num"><b>96.7</b></td><td class="num delta">+0.9</td></tr>
|
| 695 |
+
<tr><td>Elation prescriptions</td><td class="num">23.3</td><td class="num">41.7</td><td class="num">80.8</td><td class="num">90.8</td><td class="num">20.0</td><td class="num"><b>95.0</b></td><td class="num delta">+75.0</td></tr>
|
| 696 |
+
<tr><td>GitLab plan and track</td><td class="num">39.3</td><td class="num">37.1</td><td class="num">63.6</td><td class="num">77.9</td><td class="num">63.6</td><td class="num"><b>97.9</b></td><td class="num delta">+34.3</td></tr>
|
| 697 |
+
<tr><td>Gmail</td><td class="num">70.0</td><td class="num">56.7</td><td class="num">75.0</td><td class="num">85.0</td><td class="num">25.0</td><td class="num"><b>98.3</b></td><td class="num delta">+73.3</td></tr>
|
| 698 |
+
<tr><td>Gmail accounts and contacts</td><td class="num">40.0</td><td class="num">33.3</td><td class="num">61.7</td><td class="num">87.5</td><td class="num">21.7</td><td class="num"><b>91.7</b></td><td class="num delta">+70.0</td></tr>
|
| 699 |
+
<tr><td>Handshake career exploration</td><td class="num">50.0</td><td class="num">50.5</td><td class="num">50.5</td><td class="num">83.5</td><td class="num">36.5</td><td class="num"><b>84.0</b></td><td class="num delta">+47.5</td></tr>
|
| 700 |
+
<tr><td>Linear account settings</td><td class="num">54.2</td><td class="num">65.8</td><td class="num">73.3</td><td class="num">81.7</td><td class="num">43.3</td><td class="num"><b>94.2</b></td><td class="num delta">+50.9</td></tr>
|
| 701 |
+
<tr><td>PayPal wallet</td><td class="num">70.7</td><td class="num">71.4</td><td class="num">88.6</td><td class="num">90.0</td><td class="num">49.3</td><td class="num"><b>95.7</b></td><td class="num delta">+46.4</td></tr>
|
| 702 |
+
<tr><td>Superhuman general</td><td class="num">15.0</td><td class="num">25.8</td><td class="num">50.0</td><td class="num">80.8</td><td class="num">31.7</td><td class="num"><b>87.5</b></td><td class="num delta">+55.8</td></tr>
|
| 703 |
+
<tr><td>Xero invoicing</td><td class="num">52.5</td><td class="num">55.8</td><td class="num">80.8</td><td class="num">93.3</td><td class="num">39.2</td><td class="num"><b>96.7</b></td><td class="num delta">+57.5</td></tr>
|
| 704 |
+
<tr class="total"><td>Overall</td><td class="num">43.3</td><td class="num">48.3</td><td class="num">69.3</td><td class="num">86.1</td><td class="num">43.5</td><td class="num">93.0</td><td class="num delta">+49.5</td></tr>
|
| 705 |
+
</tbody>
|
| 706 |
+
</table>
|
| 707 |
+
</div>
|
| 708 |
+
<p class="tnote">
|
| 709 |
+
93.0% audit-clean beats the strongest same-model baseline (GPT-5.5 + Browser Use, 86.1%) by
|
| 710 |
+
6.9 points and the top public agent (Gemini 3 Flash + Browser Use, 69.3%) by
|
| 711 |
+
23.7 points. Given differing action spaces across public agents, we treat the same-model gap as
|
| 712 |
+
the primary comparison. In counts, the base solves 548/1,260 and the evolved harness 1,171/1,260.
|
| 713 |
+
</p>
|
| 714 |
+
|
| 715 |
+
<div class="block">
|
| 716 |
+
<h4>Evolution as optimization</h4>
|
| 717 |
+
<p class="tnote" style="margin-top:0;margin-bottom:14px">
|
| 718 |
+
Screening score of every evaluated variant in the WebArena-Infinity run, in archive order. Hover a
|
| 719 |
+
point for its score; the line is the best-so-far envelope.
|
| 720 |
+
</p>
|
| 721 |
+
<div class="widget" id="cv-widget">
|
| 722 |
+
<div class="wtop">
|
| 723 |
+
<span class="wlabel">Show</span>
|
| 724 |
+
<label class="toggle"><input type="checkbox" id="cv-all" checked /> every evaluated variant</label>
|
| 725 |
+
<label class="toggle"><input type="checkbox" id="cv-best" checked /> best so far</label>
|
| 726 |
+
</div>
|
| 727 |
+
<svg class="chart" id="cv-chart" viewBox="0 0 860 320" role="img"
|
| 728 |
+
aria-label="Screening score per evaluated variant, with the best-so-far envelope."></svg>
|
| 729 |
+
<p class="tnote" id="cv-note"></p>
|
| 730 |
+
</div>
|
| 731 |
+
<noscript class="nojs"><img src="assets/wai_evolution_curve.png" alt="Best-so-far screening score across accepted generations." /></noscript>
|
| 732 |
+
</div>
|
| 733 |
+
|
| 734 |
+
<figure class="block">
|
| 735 |
+
<a class="zoom" href="assets/wai_tree.png" target="_blank" rel="noopener">
|
| 736 |
+
<img src="assets/wai_tree.png" alt="Archive lineage tree with accepted and reverted variants, the primary lineage, and recombination edges." loading="lazy" />
|
| 737 |
+
</a>
|
| 738 |
+
<figcaption>
|
| 739 |
+
The same run as an archive lineage tree (node size ∝ screening score): accepted (blue) and
|
| 740 |
+
reverted (grey) variants, the primary lineage (gold, base → evolved), and
|
| 741 |
+
recombination edges (dashed). The reverted nodes are the preserve-and-extend contract doing its job:
|
| 742 |
+
a variant that broke an inherited solve contributes its distilled lesson but not its edit.
|
| 743 |
+
</figcaption>
|
| 744 |
+
</figure>
|
| 745 |
+
|
| 746 |
+
<div class="block">
|
| 747 |
+
<h4>Anti-cheating audit</h4>
|
| 748 |
+
<table style="margin-top:12px">
|
| 749 |
+
<caption>Validity audit on the 1,260 real tasks. Trajectories that reach the goal through the
|
| 750 |
+
evaluation plane, privileged knowledge, or exploits are counted as failures.</caption>
|
| 751 |
+
<thead>
|
| 752 |
+
<tr><th>Measure</th><th class="num">Monet (base)</th><th class="num">Monet (DarwinX)</th><th class="num">Δ</th></tr>
|
| 753 |
+
</thead>
|
| 754 |
+
<tbody>
|
| 755 |
+
<tr class="ours"><td>Audit-clean pass@1</td><td class="num">43.5</td><td class="num">93.0</td><td class="num delta">+49.5 pp</td></tr>
|
| 756 |
+
<tr><td>Confirmed invalid</td><td class="num">23.5%</td><td class="num">1.4%</td><td class="num">−22.1 pp</td></tr>
|
| 757 |
+
<tr><td>Human review</td><td class="num">5.1%</td><td class="num">0.1%</td><td class="num">−5.0 pp</td></tr>
|
| 758 |
+
<tr><td>Blocked attempt</td><td class="num">14.3%</td><td class="num">0.1%</td><td class="num">−14.2 pp</td></tr>
|
| 759 |
+
</tbody>
|
| 760 |
+
</table>
|
| 761 |
+
<p class="tnote">
|
| 762 |
+
Evolution does not buy the gain with shortcuts: it cuts invalid trajectories from 293 to 17, the
|
| 763 |
+
evaluation-plane, privileged-knowledge, and exploit mechanisms disappear entirely, and the residual
|
| 764 |
+
17 are all raw-state mutations concentrated in a single application. Dropping unaudited successes
|
| 765 |
+
too still leaves 92.9%.
|
| 766 |
+
</p>
|
| 767 |
+
<div class="block">
|
| 768 |
+
<div class="widget" id="iv-widget">
|
| 769 |
+
<div class="wtop">
|
| 770 |
+
<span class="wlabel">Break down by</span>
|
| 771 |
+
<div class="chips" id="iv-chips"></div>
|
| 772 |
+
<span style="flex:1"></span>
|
| 773 |
+
<label class="toggle"><input type="checkbox" id="iv-norm" /> scale each row to its own total</label>
|
| 774 |
+
</div>
|
| 775 |
+
<div id="iv-rows"></div>
|
| 776 |
+
<div class="keys" id="iv-keys"></div>
|
| 777 |
+
<p class="tnote" id="iv-note"></p>
|
| 778 |
+
</div>
|
| 779 |
+
<noscript class="nojs"><img src="assets/wai_invalid_composition.png" alt="Invalid trajectories before vs. after evolution, 293 to 17." /></noscript>
|
| 780 |
+
<p class="tnote">
|
| 781 |
+
Invalid trajectories before vs. after evolution. On a shared scale the second row is almost
|
| 782 |
+
invisible, which is the result: 293 invalid trajectories become 17. Scale each row to its own
|
| 783 |
+
total to inspect what the remainder is made of.
|
| 784 |
+
</p>
|
| 785 |
+
</div>
|
| 786 |
+
|
| 787 |
+
<div class="block">
|
| 788 |
+
<h4>What the audit costs each harness</h4>
|
| 789 |
+
<p class="tnote" style="margin-top:0;margin-bottom:14px">
|
| 790 |
+
Each line runs from raw pass@1 to pass@1 after the validity audit. A long line means the harness
|
| 791 |
+
was leaning on trajectories the audit rejects.
|
| 792 |
+
</p>
|
| 793 |
+
<div class="widget" id="ad-widget">
|
| 794 |
+
<div class="wtop">
|
| 795 |
+
<span class="wlabel">Show</span>
|
| 796 |
+
<label class="toggle"><input type="checkbox" id="ad-base" checked /> Monet (base)</label>
|
| 797 |
+
<label class="toggle"><input type="checkbox" id="ad-dx" checked /> Monet (DarwinX)</label>
|
| 798 |
+
<span style="flex:1"></span>
|
| 799 |
+
<label class="toggle"><input type="checkbox" id="ad-sort" /> sort by audit loss</label>
|
| 800 |
+
</div>
|
| 801 |
+
<div class="dumb" id="ad-rows"></div>
|
| 802 |
+
<div class="wlegend">
|
| 803 |
+
<span><i class="sw" style="background:#fff;box-shadow:inset 0 0 0 2px #9aa8b8;border-radius:50%"></i> raw pass@1</span>
|
| 804 |
+
<span><i class="sw" style="background:#8794a6;border-radius:50%"></i> base, audited</span>
|
| 805 |
+
<span><i class="sw" style="background:var(--blue);border-radius:50%"></i> DarwinX, audited</span>
|
| 806 |
+
</div>
|
| 807 |
+
<p class="tnote" id="ad-note"></p>
|
| 808 |
+
</div>
|
| 809 |
+
<noscript class="nojs"><img src="assets/wai_audit_by_app.png" alt="Raw vs. audited pass@1 per application." /></noscript>
|
| 810 |
+
</div>
|
| 811 |
+
</div>
|
| 812 |
+
</div>
|
| 813 |
+
|
| 814 |
+
<!-- ============ SWE-bench Verified ============ -->
|
| 815 |
+
<div class="bench">
|
| 816 |
+
<h3>SWE-bench Verified <span class="tag">— zero-shot cross-benchmark transfer</span></h3>
|
| 817 |
+
<p class="lede">The best Terminal-Bench 2.1 harness is run <i>unchanged</i> on all 500 SWE-bench
|
| 818 |
+
Verified instances. No SWE-V feedback ever enters the loop; we report it as a transfer target only and
|
| 819 |
+
run no in-domain SWE-V evolution.</p>
|
| 820 |
+
|
| 821 |
+
<table>
|
| 822 |
+
<caption>SWE-bench Verified official pass@1, graded by SWE-V's own test harness.</caption>
|
| 823 |
+
<thead>
|
| 824 |
+
<tr><th>Harness</th><th class="num">Solved</th><th class="num">pass@1</th></tr>
|
| 825 |
+
</thead>
|
| 826 |
+
<tbody>
|
| 827 |
+
<tr class="ours"><td>Monet (DarwinX), TB2.1-specialized, transferred unchanged</td><td class="num">421/500</td><td class="num">84.2%</td></tr>
|
| 828 |
+
<tr><td>Fix-skill reference</td><td class="num">—</td><td class="num">80.8%</td></tr>
|
| 829 |
+
</tbody>
|
| 830 |
+
</table>
|
| 831 |
+
<p class="tnote">
|
| 832 |
+
+3.4 points over the reference without receiving any SWE-V feedback. The transferred agent preserves
|
| 833 |
+
strong repository-level competence, so what evolved on terminal tasks is general agent capability
|
| 834 |
+
rather than benchmark-specific patching.
|
| 835 |
+
</p>
|
| 836 |
+
</div>
|
| 837 |
+
</div>
|
| 838 |
+
</section>
|
| 839 |
+
|
| 840 |
+
<section id="ablation">
|
| 841 |
+
<div class="wrap">
|
| 842 |
+
<h2>Ablation: what evolution changes</h2>
|
| 843 |
+
<p>
|
| 844 |
+
Anchored on Terminal-Bench 2.1, we compare base Monet (75.5%) with the evolved best (83.2%) along
|
| 845 |
+
two axes: the <b>skill-bundle diff</b> between the two harnesses, and <b>where</b> on the benchmark the
|
| 846 |
+
gains land. This is an <i>exploratory attribution, not a per-skill causal ablation</i> — the skills
|
| 847 |
+
were co-selected, not independently randomized.
|
| 848 |
+
</p>
|
| 849 |
+
|
| 850 |
+
<table style="margin-top:20px">
|
| 851 |
+
<caption>The seven skills the evolved lineage adds over base Monet. Every one belongs to a single
|
| 852 |
+
family: verification / artifact-contract. None adds domain knowledge.</caption>
|
| 853 |
+
<thead>
|
| 854 |
+
<tr><th style="width:42%">Evolved skills</th><th>Role</th></tr>
|
| 855 |
+
</thead>
|
| 856 |
+
<tbody>
|
| 857 |
+
<tr>
|
| 858 |
+
<td><code>verifier-contract</code><br /><code>contract-candidate</code></td>
|
| 859 |
+
<td>Derive the task's acceptance contract and check the solution against it before finalizing.</td>
|
| 860 |
+
</tr>
|
| 861 |
+
<tr>
|
| 862 |
+
<td><code>graded-artifact-final-check</code><br /><code>artifact-verification-loop</code></td>
|
| 863 |
+
<td>Verify the graded artifact (output file, format, and values) and iterate a fix-and-recheck loop.</td>
|
| 864 |
+
</tr>
|
| 865 |
+
<tr>
|
| 866 |
+
<td><code>real-tool-artifact</code><br /><code>tool-grounded-artifact</code></td>
|
| 867 |
+
<td>Ground outputs in real tool execution rather than asserted or simulated results.</td>
|
| 868 |
+
</tr>
|
| 869 |
+
<tr>
|
| 870 |
+
<td><code>security-contract-repair</code></td>
|
| 871 |
+
<td>Repair the solution against security and contract checks.</td>
|
| 872 |
+
</tr>
|
| 873 |
+
</tbody>
|
| 874 |
+
</table>
|
| 875 |
+
|
| 876 |
+
<div class="block" style="margin-top:30px">
|
| 877 |
+
<h4>Where the gains land</h4>
|
| 878 |
+
<p>
|
| 879 |
+
Gains concentrate where a frozen base has the most headroom and where verifying against a contract
|
| 880 |
+
unblocks fragile multi-step work: <i>ML & scientific-computing</i> (+14.8, 60→75%) and
|
| 881 |
+
<i>data/database</i> (+13.8, 84→98%). Clusters where the base is already strong barely move
|
| 882 |
+
(system administration 92→98%, security 85→84%, within noise), and no cluster regresses
|
| 883 |
+
beyond the per-task noise band. <b>That asymmetry is the empirical footprint of the
|
| 884 |
+
preserve-and-extend rule</b>: extend the fragile clusters, hold the solved ones fixed. The unblocked
|
| 885 |
+
difficulty is procedural — long dependency installs, environment setup, output verification,
|
| 886 |
+
multi-step tool use — rather than knowledge-bound, which matches a verification/artifact-contract
|
| 887 |
+
bundle rather than a stronger model.
|
| 888 |
+
</p>
|
| 889 |
+
</div>
|
| 890 |
+
|
| 891 |
+
<div class="block">
|
| 892 |
+
<h4>Cross-benchmark cross-checks</h4>
|
| 893 |
+
<p>
|
| 894 |
+
<b>WebArena-Infinity, the same family in a different modality.</b> The evolved browser harness adds
|
| 895 |
+
the same kind of skills (state and action contracts), and its confirmed-invalid rate falls from 23.5%
|
| 896 |
+
to 1.4% while audit-clean pass@1 rises 49.5 points, with the largest gains on state-changing
|
| 897 |
+
applications. Verification-before-finalization is thus visible under a different interaction modality
|
| 898 |
+
and reward source.
|
| 899 |
+
</p>
|
| 900 |
+
<p>
|
| 901 |
+
<b>TerminalWorld, diversity rather than a single skill.</b> Individual specialists solve 24–27
|
| 902 |
+
held-out tasks and the merged harness solves 28, so the archive is valuable as a source of
|
| 903 |
+
complementary behaviors even when the training-subset ranking is noisy.
|
| 904 |
+
</p>
|
| 905 |
+
<p class="tnote">
|
| 906 |
+
Together these make verification-before-finalization and contract-aware tool use a plausible shared
|
| 907 |
+
mechanism across benchmarks, offered as an explanation rather than a per-skill causal estimate.
|
| 908 |
+
</p>
|
| 909 |
+
</div>
|
| 910 |
+
</div>
|
| 911 |
+
</section>
|
| 912 |
+
|
| 913 |
+
<section id="limitations">
|
| 914 |
+
<div class="wrap">
|
| 915 |
+
<h2>Limitations</h2>
|
| 916 |
+
<ul class="lim">
|
| 917 |
+
<li><b>Transfer is measured in one direction, and is much smaller.</b> A TB2.1-evolved harness reaches
|
| 918 |
+
84.2% on SWE-bench Verified, ahead of the fix-skill reference but inside a narrow band around it
|
| 919 |
+
(official scores across the harnesses we compare span just 80.8–84.2%). The transferred gain is
|
| 920 |
+
far smaller than the in-domain ones, and we make no in-domain SWE-V claim.</li>
|
| 921 |
+
<li><b>Recombination's own contribution is not isolated.</b> The archive and merge machinery can
|
| 922 |
+
preserve and combine variants, but population search needs diverse wins before inheritance is useful,
|
| 923 |
+
and the contribution of recombination relative to single-lineage mutation still requires controlled
|
| 924 |
+
ablation.</li>
|
| 925 |
+
<li><b>The claim is system-level, not per-operator.</b> The experiments evaluate the complete system:
|
| 926 |
+
archive, parent selector, recombination operator, and inference effort are not independently
|
| 927 |
+
randomized, and public leaderboard rows use different models and effort settings. Matched-model deltas
|
| 928 |
+
support a harness claim; any individual operator's contribution, and the attribution to a
|
| 929 |
+
verification/contract mechanism, remain plausible rather than causal.</li>
|
| 930 |
+
<li><b>TerminalWorld is small.</b> With only 41 held-out tasks, one solve moves pass@1 by 2.4 points,
|
| 931 |
+
and the matched Opus comparison (25/41 vs. 28/41, McNemar <i>p</i>=0.45) is suggestive rather than
|
| 932 |
+
decisive.</li>
|
| 933 |
+
<li><b>The validity audit is not a formal sandbox.</b> Our WAI policy permits client-visible observation
|
| 934 |
+
and semantic application operations but rejects privileged knowledge, evaluation-plane access,
|
| 935 |
+
raw-state fabrication, database manipulation, and exploits. The static-plus-LLM audit is far stronger
|
| 936 |
+
than a keyword heuristic, yet deeply dynamic construction can require human review and a small number
|
| 937 |
+
of trajectories were unavailable, so we report both raw and conservative audit-clean scores.</li>
|
| 938 |
+
<li><b>Freezing the base model is a choice, not a ceiling.</b> It is what makes these deltas
|
| 939 |
+
attributable to the harness. Coupling harness selection with weight updates, and treating the harness
|
| 940 |
+
as an asset that outlives a base-model generation, are left to future work.</li>
|
| 941 |
+
</ul>
|
| 942 |
+
</div>
|
| 943 |
+
</section>
|
| 944 |
+
|
| 945 |
+
<section id="bibtex">
|
| 946 |
+
<div class="wrap">
|
| 947 |
+
<div class="bibhead">
|
| 948 |
+
<h2>BibTeX</h2>
|
| 949 |
+
<button id="copybtn" type="button">Copy</button>
|
| 950 |
+
</div>
|
| 951 |
+
<pre id="bib">@article{zhang2026darwinx,
|
| 952 |
+
title = {DarwinX: Evolving Agent Harnesses Through Natural Selection},
|
| 953 |
+
author = {Zhang, Yifan and Dai, Yutong and Tan, Juntao and Yang, Luyu and
|
| 954 |
+
Mullur, Rishi and Hoang, Thai and Hu, Zhiyuan and Zhu, James and
|
| 955 |
+
Mui, Phil and Savarese, Silvio and Xu, Ran and Chen, Zeyuan},
|
| 956 |
+
journal = {arXiv preprint arXiv:2608.07545},
|
| 957 |
+
year = {2026}
|
| 958 |
+
}</pre>
|
| 959 |
+
</div>
|
| 960 |
+
</section>
|
| 961 |
+
|
| 962 |
+
<footer>
|
| 963 |
+
<div class="wrap">
|
| 964 |
+
Salesforce AI Research · 2026. All figures and numbers are from
|
| 965 |
+
<a href="https://arxiv.org/abs/2608.07545">arXiv:2608.07545</a>.
|
| 966 |
+
Click any figure to open it at full resolution.
|
| 967 |
+
</div>
|
| 968 |
+
</footer>
|
| 969 |
+
|
| 970 |
+
<div id="tip" role="tooltip"></div>
|
| 971 |
+
|
| 972 |
+
<script src="assets/data.js"></script>
|
| 973 |
+
<script>
|
| 974 |
+
(function () {
|
| 975 |
+
"use strict";
|
| 976 |
+
var D = window.DARWINX;
|
| 977 |
+
var $ = function (id) { return document.getElementById(id); };
|
| 978 |
+
|
| 979 |
+
/* ---------------- shared tooltip ---------------- */
|
| 980 |
+
var tip = $("tip");
|
| 981 |
+
function showTip(html, evt) {
|
| 982 |
+
tip.innerHTML = html;
|
| 983 |
+
tip.style.opacity = "1";
|
| 984 |
+
var r = tip.getBoundingClientRect();
|
| 985 |
+
var x = Math.min(Math.max(8, evt.clientX - r.width / 2), window.innerWidth - r.width - 8);
|
| 986 |
+
var y = evt.clientY - r.height - 12;
|
| 987 |
+
if (y < 8) { y = evt.clientY + 16; }
|
| 988 |
+
tip.style.left = x + "px";
|
| 989 |
+
tip.style.top = y + "px";
|
| 990 |
+
}
|
| 991 |
+
function hideTip() { tip.style.opacity = "0"; }
|
| 992 |
+
function bindTip(el, htmlFn) {
|
| 993 |
+
el.addEventListener("mousemove", function (e) { showTip(htmlFn(), e); });
|
| 994 |
+
el.addEventListener("mouseleave", hideTip);
|
| 995 |
+
}
|
| 996 |
+
|
| 997 |
+
/* ---------------- 0a. headline panels ---------------- */
|
| 998 |
+
(function () {
|
| 999 |
+
if (!D || !D.headline) { return; }
|
| 1000 |
+
var P = D.headline;
|
| 1001 |
+
var modes = [
|
| 1002 |
+
{ id: "own", label: "per panel" },
|
| 1003 |
+
{ id: "shared", label: "shared 0\u2013100" }
|
| 1004 |
+
];
|
| 1005 |
+
var mode = "own";
|
| 1006 |
+
|
| 1007 |
+
var chips = $("hl-chips");
|
| 1008 |
+
modes.forEach(function (m) {
|
| 1009 |
+
var b = document.createElement("button");
|
| 1010 |
+
b.className = "chip";
|
| 1011 |
+
b.type = "button";
|
| 1012 |
+
b.textContent = m.label;
|
| 1013 |
+
b.setAttribute("aria-pressed", m.id === mode ? "true" : "false");
|
| 1014 |
+
b.addEventListener("click", function () {
|
| 1015 |
+
mode = m.id;
|
| 1016 |
+
Array.prototype.forEach.call(chips.children, function (c) {
|
| 1017 |
+
c.setAttribute("aria-pressed", c === b ? "true" : "false");
|
| 1018 |
+
});
|
| 1019 |
+
render();
|
| 1020 |
+
});
|
| 1021 |
+
chips.appendChild(b);
|
| 1022 |
+
});
|
| 1023 |
+
|
| 1024 |
+
var NS = "http://www.w3.org/2000/svg";
|
| 1025 |
+
function mk(n, a) {
|
| 1026 |
+
var e = document.createElementNS(NS, n);
|
| 1027 |
+
Object.keys(a).forEach(function (k) { e.setAttribute(k, a[k]); });
|
| 1028 |
+
return e;
|
| 1029 |
+
}
|
| 1030 |
+
|
| 1031 |
+
function render() {
|
| 1032 |
+
var showPrior = $("hl-prior").checked;
|
| 1033 |
+
var box = $("hl-panels");
|
| 1034 |
+
box.innerHTML = "";
|
| 1035 |
+
|
| 1036 |
+
P.forEach(function (p) {
|
| 1037 |
+
var lo = mode === "shared" ? 0 : p.lo;
|
| 1038 |
+
var hi = mode === "shared" ? 100 : p.hi;
|
| 1039 |
+
var W = 190, H = 168, T = 20, B = 22, L = 26;
|
| 1040 |
+
var ih = H - T - B;
|
| 1041 |
+
var y = function (v) { return T + ih - ((v - lo) / (hi - lo)) * ih; };
|
| 1042 |
+
|
| 1043 |
+
var series = [{ c: "b", label: p.baseLabel, v: p.base }];
|
| 1044 |
+
if (showPrior && p.prior !== null) { series.push({ c: "p", label: p.priorLabel, v: p.prior }); }
|
| 1045 |
+
series.push({ c: "e", label: p.evolvedLabel, v: p.evolved });
|
| 1046 |
+
|
| 1047 |
+
var wrapEl = document.createElement("div");
|
| 1048 |
+
wrapEl.className = "panel";
|
| 1049 |
+
wrapEl.innerHTML = "<div class=\"pt\">" + p.short + "</div><div class=\"ps\">" +
|
| 1050 |
+
p.metric + "<br>on " + p.model + "</div>";
|
| 1051 |
+
|
| 1052 |
+
var svg = mk("svg", { viewBox: "0 0 " + W + " " + H });
|
| 1053 |
+
[lo, hi].forEach(function (v) {
|
| 1054 |
+
var t = mk("text", { class: "tick", x: L - 6, y: y(v) + 3.5, "text-anchor": "end" });
|
| 1055 |
+
t.textContent = v;
|
| 1056 |
+
svg.appendChild(t);
|
| 1057 |
+
});
|
| 1058 |
+
svg.appendChild(mk("line", { class: "axl", x1: L, x2: W - 6, y1: y(lo), y2: y(lo) }));
|
| 1059 |
+
|
| 1060 |
+
var slot = (W - 6 - L) / series.length;
|
| 1061 |
+
var bw = Math.min(34, slot * 0.62);
|
| 1062 |
+
series.forEach(function (s, i) {
|
| 1063 |
+
var cx = L + slot * (i + 0.5);
|
| 1064 |
+
var top = y(Math.max(s.v, lo));
|
| 1065 |
+
var rect = mk("rect", {
|
| 1066 |
+
class: "bar " + s.c, x: cx - bw / 2, y: top, width: bw,
|
| 1067 |
+
height: Math.max(0, y(lo) - top), rx: 2
|
| 1068 |
+
});
|
| 1069 |
+
svg.appendChild(rect);
|
| 1070 |
+
var hit = mk("rect", { x: cx - slot / 2, y: T, width: slot, height: ih, fill: "transparent" });
|
| 1071 |
+
hit.addEventListener("mousemove", function (e) {
|
| 1072 |
+
showTip("<b>" + p.name + "</b><br>" + s.label + ": " + s.v.toFixed(1) + "%<br>" +
|
| 1073 |
+
p.metric + ", " + p.scope, e);
|
| 1074 |
+
});
|
| 1075 |
+
hit.addEventListener("mouseleave", hideTip);
|
| 1076 |
+
svg.appendChild(hit);
|
| 1077 |
+
});
|
| 1078 |
+
|
| 1079 |
+
var ev = series[series.length - 1];
|
| 1080 |
+
var dl = mk("text", { class: "dl", x: L + slot * (series.length - 0.5),
|
| 1081 |
+
y: y(ev.v) - 6, "text-anchor": "middle" });
|
| 1082 |
+
dl.textContent = "+" + p.delta.toFixed(1);
|
| 1083 |
+
svg.appendChild(dl);
|
| 1084 |
+
|
| 1085 |
+
wrapEl.appendChild(svg);
|
| 1086 |
+
var foot = document.createElement("div");
|
| 1087 |
+
foot.className = "foot";
|
| 1088 |
+
foot.textContent = p.prior === null ? "transfer only" : "vs " + p.priorLabel.split(" + ")[0];
|
| 1089 |
+
wrapEl.appendChild(foot);
|
| 1090 |
+
box.appendChild(wrapEl);
|
| 1091 |
+
});
|
| 1092 |
+
|
| 1093 |
+
$("hl-legend").innerHTML =
|
| 1094 |
+
"<span><i class=\"sw\" style=\"background:#c6d3e0\"></i> base / reference</span>" +
|
| 1095 |
+
(showPrior ? "<span><i class=\"sw\" style=\"background:#8fb4e8\"></i> best prior agent</span>" : "") +
|
| 1096 |
+
"<span><i class=\"sw\" style=\"background:var(--blue)\"></i> Monet (DarwinX)</span>";
|
| 1097 |
+
|
| 1098 |
+
$("hl-note").innerHTML = mode === "own"
|
| 1099 |
+
? "Each panel is scaled to its own range, which is how the paper draws it: on a shared axis " +
|
| 1100 |
+
"WebArena-Infinity's +49.5 flattens the two terminal benchmarks into slivers. Switch to the " +
|
| 1101 |
+
"shared axis to see the honest relative magnitudes \u2014 the annotated deltas are the same either way."
|
| 1102 |
+
: "On one 0\u2013100 axis the four regimes are directly comparable, and it is clear how much of the " +
|
| 1103 |
+
"range each benchmark actually occupies. Bars within a panel share one frozen base model; the " +
|
| 1104 |
+
"prior-agent bar uses a different model and effort setting, so it is context rather than a " +
|
| 1105 |
+
"controlled comparison. On SWE-bench Verified the grey bar is the fix-skill reference, not an " +
|
| 1106 |
+
"unevolved Monet, and no prior-agent bar is drawn.";
|
| 1107 |
+
}
|
| 1108 |
+
|
| 1109 |
+
$("hl-prior").addEventListener("change", render);
|
| 1110 |
+
render();
|
| 1111 |
+
})();
|
| 1112 |
+
|
| 1113 |
+
/* ---------------- 0b. TerminalWorld specialist bars ---------------- */
|
| 1114 |
+
(function () {
|
| 1115 |
+
if (!D || !D.twBars) { return; }
|
| 1116 |
+
var T = D.twBars;
|
| 1117 |
+
function pct(s) { return (100 * s / T.n); }
|
| 1118 |
+
// The arms span 58.5-68.3, so a full 0-100 track renders them as six near-identical
|
| 1119 |
+
// bars. Zoomed is the default (and how the paper draws it); the full axis stays one
|
| 1120 |
+
// click away because a truncated axis exaggerates the spread.
|
| 1121 |
+
var scales = [{ id: "zoom", label: "55\u201370", lo: 55, hi: 70 },
|
| 1122 |
+
{ id: "full", label: "0\u2013100", lo: 0, hi: 100 }];
|
| 1123 |
+
var scale = scales[0];
|
| 1124 |
+
|
| 1125 |
+
var chips = $("sp-chips");
|
| 1126 |
+
scales.forEach(function (s) {
|
| 1127 |
+
var b = document.createElement("button");
|
| 1128 |
+
b.className = "chip";
|
| 1129 |
+
b.type = "button";
|
| 1130 |
+
b.textContent = s.label;
|
| 1131 |
+
b.setAttribute("aria-pressed", s === scale ? "true" : "false");
|
| 1132 |
+
b.addEventListener("click", function () {
|
| 1133 |
+
scale = s;
|
| 1134 |
+
Array.prototype.forEach.call(chips.children, function (c) {
|
| 1135 |
+
c.setAttribute("aria-pressed", c === b ? "true" : "false");
|
| 1136 |
+
});
|
| 1137 |
+
render();
|
| 1138 |
+
});
|
| 1139 |
+
chips.appendChild(b);
|
| 1140 |
+
});
|
| 1141 |
+
|
| 1142 |
+
function render() {
|
| 1143 |
+
var counts = $("sp-counts").checked;
|
| 1144 |
+
var pos = function (p) {
|
| 1145 |
+
return Math.max(0, Math.min(100, 100 * (p - scale.lo) / (scale.hi - scale.lo)));
|
| 1146 |
+
};
|
| 1147 |
+
var refPos = pos(pct(T.claudeCode));
|
| 1148 |
+
var box = $("sp-rows");
|
| 1149 |
+
box.innerHTML = "";
|
| 1150 |
+
T.bars.forEach(function (b) {
|
| 1151 |
+
var isDx = /DarwinX/.test(b.label);
|
| 1152 |
+
var isBase = b.label === "Base";
|
| 1153 |
+
var row = document.createElement("div");
|
| 1154 |
+
row.className = "hrow" + (isBase ? " dim" : "");
|
| 1155 |
+
var color = isDx ? "var(--blue)" : (isBase ? "#c6d3e0" : "#8fb4e8");
|
| 1156 |
+
row.innerHTML =
|
| 1157 |
+
"<div class=\"hn\"" + (isDx ? " style=\"font-weight:600\"" : "") + ">" + b.label + "</div>" +
|
| 1158 |
+
"<div class=\"ht\"><div class=\"hf\" style=\"width:" + pos(pct(b.solved)).toFixed(1) +
|
| 1159 |
+
"%;background:" + color + "\"></div></div>" +
|
| 1160 |
+
"<div class=\"hv\">" + (counts ? b.solved + "/" + T.n : pct(b.solved).toFixed(1) + "%") + "</div>";
|
| 1161 |
+
var track = row.querySelector(".ht");
|
| 1162 |
+
var ref = document.createElement("div");
|
| 1163 |
+
ref.className = "refline";
|
| 1164 |
+
ref.style.left = refPos.toFixed(1) + "%";
|
| 1165 |
+
track.appendChild(ref);
|
| 1166 |
+
bindTip(track, function () {
|
| 1167 |
+
return "<b>" + b.label + "</b><br>" + b.solved + " of " + T.n + " held-out tasks (" +
|
| 1168 |
+
pct(b.solved).toFixed(1) + "%)<br>Claude Code reference: " + T.claudeCode + "/" + T.n +
|
| 1169 |
+
" (" + pct(T.claudeCode).toFixed(1) + "%)";
|
| 1170 |
+
});
|
| 1171 |
+
box.appendChild(row);
|
| 1172 |
+
});
|
| 1173 |
+
var tag = document.createElement("div");
|
| 1174 |
+
tag.className = "hrow";
|
| 1175 |
+
tag.innerHTML = "<div></div><div style=\"position:relative;height:14px\">" +
|
| 1176 |
+
"<div class=\"reftag\" style=\"left:" + refPos.toFixed(1) + "%;top:0\">Claude Code " +
|
| 1177 |
+
pct(T.claudeCode).toFixed(1) + "%</div></div><div></div>";
|
| 1178 |
+
box.appendChild(tag);
|
| 1179 |
+
|
| 1180 |
+
$("sp-note").innerHTML = scale.id === "zoom"
|
| 1181 |
+
? "Bars start at 55%, not 0, so the arms are distinguishable \u2014 the whole spread is 9.8 points. " +
|
| 1182 |
+
"Switch to the full axis to see that spread in proportion."
|
| 1183 |
+
: "On the full axis every arm looks alike, which is the honest framing of the magnitude: the merge " +
|
| 1184 |
+
"buys 2.4 points over the best single specialist. The ordering is what matters here, not the size.";
|
| 1185 |
+
}
|
| 1186 |
+
$("sp-counts").addEventListener("change", render);
|
| 1187 |
+
render();
|
| 1188 |
+
})();
|
| 1189 |
+
|
| 1190 |
+
/* ---------------- 0c. TB2.1 compute ---------------- */
|
| 1191 |
+
(function () {
|
| 1192 |
+
if (!D || !D.tb21Compute) { return; }
|
| 1193 |
+
var C = D.tb21Compute;
|
| 1194 |
+
var metrics = [
|
| 1195 |
+
{ id: "turns", label: "turns", unit: "", fmt: function (v) { return v; } },
|
| 1196 |
+
{ id: "tokens", label: "tokens", unit: "K", fmt: function (v) { return v + "K"; } }
|
| 1197 |
+
];
|
| 1198 |
+
var active = "turns";
|
| 1199 |
+
|
| 1200 |
+
var chips = $("cp-chips");
|
| 1201 |
+
metrics.forEach(function (m) {
|
| 1202 |
+
var b = document.createElement("button");
|
| 1203 |
+
b.className = "chip";
|
| 1204 |
+
b.type = "button";
|
| 1205 |
+
b.textContent = m.label;
|
| 1206 |
+
b.setAttribute("aria-pressed", m.id === active ? "true" : "false");
|
| 1207 |
+
b.addEventListener("click", function () {
|
| 1208 |
+
active = m.id;
|
| 1209 |
+
Array.prototype.forEach.call(chips.children, function (c) {
|
| 1210 |
+
c.setAttribute("aria-pressed", c === b ? "true" : "false");
|
| 1211 |
+
});
|
| 1212 |
+
render();
|
| 1213 |
+
});
|
| 1214 |
+
chips.appendChild(b);
|
| 1215 |
+
});
|
| 1216 |
+
|
| 1217 |
+
function render() {
|
| 1218 |
+
var m = metrics.filter(function (x) { return x.id === active; })[0];
|
| 1219 |
+
var d = C[active];
|
| 1220 |
+
var max = Math.max.apply(null, d.base.concat(d.evolved));
|
| 1221 |
+
var box = $("cp-rows");
|
| 1222 |
+
box.innerHTML = "";
|
| 1223 |
+
C.groups.forEach(function (g, gi) {
|
| 1224 |
+
[["Monet (base)", d.base[gi], "#c6d3e0"], ["Monet (DarwinX)", d.evolved[gi], "var(--blue)"]]
|
| 1225 |
+
.forEach(function (s, si) {
|
| 1226 |
+
var row = document.createElement("div");
|
| 1227 |
+
row.className = "hrow";
|
| 1228 |
+
row.innerHTML =
|
| 1229 |
+
"<div class=\"hn\">" + (si === 0 ? g : "") + "</div>" +
|
| 1230 |
+
"<div class=\"ht\"><div class=\"hf\" style=\"width:" + (100 * s[1] / max).toFixed(1) +
|
| 1231 |
+
"%;background:" + s[2] + "\"></div></div>" +
|
| 1232 |
+
"<div class=\"hv\">" + m.fmt(s[1]) + "</div>";
|
| 1233 |
+
bindTip(row.querySelector(".ht"), function () {
|
| 1234 |
+
var other = si === 0 ? d.evolved[gi] : d.base[gi];
|
| 1235 |
+
var ratio = si === 0 ? (other / s[1]) : (s[1] / other);
|
| 1236 |
+
return "<b>" + g + "</b><br>" + s[0] + ": " + m.fmt(s[1]) + " median " + m.label +
|
| 1237 |
+
"<br>evolved / base = " + ratio.toFixed(1) + "x";
|
| 1238 |
+
});
|
| 1239 |
+
box.appendChild(row);
|
| 1240 |
+
});
|
| 1241 |
+
});
|
| 1242 |
+
$("cp-note").innerHTML = active === "turns"
|
| 1243 |
+
? "On the six tasks that flip from failing to solved the evolved harness roughly doubles its turns " +
|
| 1244 |
+
"(22 vs. 11). On the 69 both already solve, turns barely move (13 vs. 12): the extra effort is " +
|
| 1245 |
+
"spent where it changes the outcome, not spread across the benchmark."
|
| 1246 |
+
: "Tokens tell the same story more sharply: 4.3x on the newly solved six (380K vs. 89K) against " +
|
| 1247 |
+
"1.4x on the 69 already solved (172K vs. 125K). This is verify-and-retry effort converting a " +
|
| 1248 |
+
"near-miss into a pass, not uniformly more compute.";
|
| 1249 |
+
}
|
| 1250 |
+
render();
|
| 1251 |
+
})();
|
| 1252 |
+
|
| 1253 |
+
/* ---------------- 0d. WAI invalid composition ---------------- */
|
| 1254 |
+
(function () {
|
| 1255 |
+
if (!D || !D.waiInvalid) { return; }
|
| 1256 |
+
var V = D.waiInvalid;
|
| 1257 |
+
var views = [{ id: "application", label: "application" }, { id: "mechanism", label: "mechanism" }];
|
| 1258 |
+
var active = "mechanism";
|
| 1259 |
+
|
| 1260 |
+
var chips = $("iv-chips");
|
| 1261 |
+
views.forEach(function (v) {
|
| 1262 |
+
var b = document.createElement("button");
|
| 1263 |
+
b.className = "chip";
|
| 1264 |
+
b.type = "button";
|
| 1265 |
+
b.textContent = v.label;
|
| 1266 |
+
b.setAttribute("aria-pressed", v.id === active ? "true" : "false");
|
| 1267 |
+
b.addEventListener("click", function () {
|
| 1268 |
+
active = v.id;
|
| 1269 |
+
Array.prototype.forEach.call(chips.children, function (c) {
|
| 1270 |
+
c.setAttribute("aria-pressed", c === b ? "true" : "false");
|
| 1271 |
+
});
|
| 1272 |
+
render();
|
| 1273 |
+
});
|
| 1274 |
+
chips.appendChild(b);
|
| 1275 |
+
});
|
| 1276 |
+
|
| 1277 |
+
function render() {
|
| 1278 |
+
var v = V[active];
|
| 1279 |
+
var norm = $("iv-norm").checked;
|
| 1280 |
+
var totalBefore = v.before.reduce(function (a, b) { return a + b; }, 0);
|
| 1281 |
+
var box = $("iv-rows");
|
| 1282 |
+
box.innerHTML = "";
|
| 1283 |
+
|
| 1284 |
+
[["Before evolution", v.before], ["After evolution", v.after]].forEach(function (pair) {
|
| 1285 |
+
var vals = pair[1];
|
| 1286 |
+
var sum = vals.reduce(function (a, b) { return a + b; }, 0);
|
| 1287 |
+
var denom = norm ? (sum || 1) : totalBefore;
|
| 1288 |
+
var row = document.createElement("div");
|
| 1289 |
+
row.className = "srow";
|
| 1290 |
+
row.innerHTML = "<div class=\"sh\"><span>" + pair[0] +
|
| 1291 |
+
"</span><span><b>" + sum + "</b> invalid</span></div>";
|
| 1292 |
+
var st = document.createElement("div");
|
| 1293 |
+
st.className = "stack";
|
| 1294 |
+
vals.forEach(function (val, i) {
|
| 1295 |
+
if (!val) { return; }
|
| 1296 |
+
var seg = document.createElement("i");
|
| 1297 |
+
seg.style.width = (100 * val / denom).toFixed(2) + "%";
|
| 1298 |
+
seg.style.background = v.colors[i];
|
| 1299 |
+
bindTip(seg, function () {
|
| 1300 |
+
return "<b>" + v.labels[i] + "</b><br>" + val + " invalid trajector" +
|
| 1301 |
+
(val === 1 ? "y" : "ies") + "<br>" + (100 * val / (sum || 1)).toFixed(1) +
|
| 1302 |
+
"% of this row";
|
| 1303 |
+
});
|
| 1304 |
+
st.appendChild(seg);
|
| 1305 |
+
});
|
| 1306 |
+
row.appendChild(st);
|
| 1307 |
+
box.appendChild(row);
|
| 1308 |
+
});
|
| 1309 |
+
|
| 1310 |
+
$("iv-keys").innerHTML = v.labels.map(function (l, i) {
|
| 1311 |
+
if (!v.before[i] && !v.after[i]) { return ""; }
|
| 1312 |
+
return "<span><i class=\"sw\" style=\"background:" + v.colors[i] + "\"></i>" + l + "</span>";
|
| 1313 |
+
}).join("");
|
| 1314 |
+
|
| 1315 |
+
$("iv-note").innerHTML = active === "mechanism"
|
| 1316 |
+
? "Evaluation-plane access, privileged knowledge, and exploits go to <b>zero</b>. Every one of the " +
|
| 1317 |
+
"17 remaining invalid trajectories is a raw-state mutation, up from 15 of 293 before \u2014 the " +
|
| 1318 |
+
"category that survives is the mildest one."
|
| 1319 |
+
: "The remainder concentrates: 8 of the 17 sit in a single application (Super-human), and five of " +
|
| 1320 |
+
"the ten applications end with no invalid trajectories at all.";
|
| 1321 |
+
}
|
| 1322 |
+
$("iv-norm").addEventListener("change", render);
|
| 1323 |
+
render();
|
| 1324 |
+
})();
|
| 1325 |
+
|
| 1326 |
+
/* ---------------- 0e. WAI audit dumbbell ---------------- */
|
| 1327 |
+
(function () {
|
| 1328 |
+
if (!D || !D.waiAudit) { return; }
|
| 1329 |
+
var A = D.waiAudit;
|
| 1330 |
+
|
| 1331 |
+
function render() {
|
| 1332 |
+
var showBase = $("ad-base").checked, showDx = $("ad-dx").checked;
|
| 1333 |
+
var idx = A.apps.map(function (_, i) { return i; });
|
| 1334 |
+
var overall = idx.shift();
|
| 1335 |
+
if ($("ad-sort").checked) {
|
| 1336 |
+
idx.sort(function (a, b) {
|
| 1337 |
+
return (A.rawBase[b] - A.auditedBase[b]) - (A.rawBase[a] - A.auditedBase[a]);
|
| 1338 |
+
});
|
| 1339 |
+
}
|
| 1340 |
+
var box = $("ad-rows");
|
| 1341 |
+
box.innerHTML = "";
|
| 1342 |
+
|
| 1343 |
+
var ax = document.createElement("div");
|
| 1344 |
+
ax.className = "drow dax";
|
| 1345 |
+
ax.innerHTML = "<div class=\"dn\" style=\"color:var(--muted);font-size:0.7rem\">pass@1 (%)</div>" +
|
| 1346 |
+
"<div class=\"dt\">" + [0, 25, 50, 75, 100].map(function (t) {
|
| 1347 |
+
return "<span style=\"left:" + t + "%\">" + t + "</span>";
|
| 1348 |
+
}).join("") + "</div>";
|
| 1349 |
+
box.appendChild(ax);
|
| 1350 |
+
|
| 1351 |
+
[overall].concat(idx).forEach(function (i, pos) {
|
| 1352 |
+
var row = document.createElement("div");
|
| 1353 |
+
row.className = "drow" + (pos === 0 ? " tot" : "");
|
| 1354 |
+
row.innerHTML = "<div class=\"dn\">" + A.apps[i] + "</div><div class=\"dt\"></div>";
|
| 1355 |
+
var t = row.querySelector(".dt");
|
| 1356 |
+
|
| 1357 |
+
function lane(raw, aud, color, top) {
|
| 1358 |
+
var x1 = Math.min(raw, aud), x2 = Math.max(raw, aud);
|
| 1359 |
+
var seg = document.createElement("div");
|
| 1360 |
+
seg.className = "seg";
|
| 1361 |
+
seg.style.left = x1 + "%";
|
| 1362 |
+
seg.style.width = (x2 - x1) + "%";
|
| 1363 |
+
seg.style.top = top + "px";
|
| 1364 |
+
seg.style.background = color;
|
| 1365 |
+
t.appendChild(seg);
|
| 1366 |
+
var p1 = document.createElement("div");
|
| 1367 |
+
p1.className = "pt2 hollow";
|
| 1368 |
+
p1.style.left = raw + "%";
|
| 1369 |
+
p1.style.top = (top + 1.5) + "px";
|
| 1370 |
+
p1.style.boxShadow = "inset 0 0 0 2px " + color;
|
| 1371 |
+
t.appendChild(p1);
|
| 1372 |
+
var p2 = document.createElement("div");
|
| 1373 |
+
p2.className = "pt2";
|
| 1374 |
+
p2.style.left = aud + "%";
|
| 1375 |
+
p2.style.top = (top + 1.5) + "px";
|
| 1376 |
+
p2.style.background = color;
|
| 1377 |
+
t.appendChild(p2);
|
| 1378 |
+
}
|
| 1379 |
+
|
| 1380 |
+
if (showBase) { lane(A.rawBase[i], A.auditedBase[i], "#8794a6", 8); }
|
| 1381 |
+
if (showDx) { lane(A.rawDx[i], A.auditedDx[i], "#0176d3", 19); }
|
| 1382 |
+
|
| 1383 |
+
bindTip(t, function () {
|
| 1384 |
+
return "<b>" + A.apps[i] + "</b><br>base " + A.rawBase[i].toFixed(1) + " \u2192 " +
|
| 1385 |
+
A.auditedBase[i].toFixed(1) + " (\u2212" + (A.rawBase[i] - A.auditedBase[i]).toFixed(1) +
|
| 1386 |
+
")<br>DarwinX " + A.rawDx[i].toFixed(1) + " \u2192 " + A.auditedDx[i].toFixed(1) +
|
| 1387 |
+
" (\u2212" + (A.rawDx[i] - A.auditedDx[i]).toFixed(1) + ")";
|
| 1388 |
+
});
|
| 1389 |
+
box.appendChild(row);
|
| 1390 |
+
});
|
| 1391 |
+
|
| 1392 |
+
var lossB = A.rawBase[0] - A.auditedBase[0], lossD = A.rawDx[0] - A.auditedDx[0];
|
| 1393 |
+
$("ad-note").innerHTML = "Overall the audit costs the base <b>" + lossB.toFixed(1) +
|
| 1394 |
+
"</b> points and DarwinX <b>" + lossD.toFixed(1) + "</b>. The gap therefore <i>widens</i> under " +
|
| 1395 |
+
"scrutiny, from " + (A.rawDx[0] - A.rawBase[0]).toFixed(1) + " points raw to " +
|
| 1396 |
+
(A.auditedDx[0] - A.auditedBase[0]).toFixed(1) + " audited, which is why the page reports the " +
|
| 1397 |
+
"audited number. Audit coverage is 99.0% for the base and 99.4% for the evolved harness.";
|
| 1398 |
+
}
|
| 1399 |
+
["ad-base", "ad-dx", "ad-sort"].forEach(function (id) {
|
| 1400 |
+
$(id).addEventListener("change", render);
|
| 1401 |
+
});
|
| 1402 |
+
render();
|
| 1403 |
+
})();
|
| 1404 |
+
|
| 1405 |
+
/* ---------------- 1. TerminalWorld merge explorer ---------------- */
|
| 1406 |
+
(function () {
|
| 1407 |
+
if (!D || !D.tw) { return; }
|
| 1408 |
+
var tasks = D.tw.tasks;
|
| 1409 |
+
var counts = D.tw.counts;
|
| 1410 |
+
var KEYS = ["A", "B", "C", "D"];
|
| 1411 |
+
var sel = { A: true, B: true, C: true, D: true };
|
| 1412 |
+
|
| 1413 |
+
var chips = $("tw-chips");
|
| 1414 |
+
KEYS.forEach(function (k, i) {
|
| 1415 |
+
var b = document.createElement("button");
|
| 1416 |
+
b.className = "chip";
|
| 1417 |
+
b.type = "button";
|
| 1418 |
+
b.setAttribute("aria-pressed", "true");
|
| 1419 |
+
b.innerHTML = "Spec. " + k + " <span class=\"cnt\">" + counts[k] + "/41</span>";
|
| 1420 |
+
b.addEventListener("click", function () {
|
| 1421 |
+
sel[k] = !sel[k];
|
| 1422 |
+
b.setAttribute("aria-pressed", sel[k] ? "true" : "false");
|
| 1423 |
+
render();
|
| 1424 |
+
});
|
| 1425 |
+
chips.appendChild(b);
|
| 1426 |
+
void i;
|
| 1427 |
+
});
|
| 1428 |
+
|
| 1429 |
+
var grid = $("tw-grid");
|
| 1430 |
+
var cells = tasks.map(function (t) {
|
| 1431 |
+
var c = document.createElement("div");
|
| 1432 |
+
c.className = "tcell";
|
| 1433 |
+
grid.appendChild(c);
|
| 1434 |
+
bindTip(c, function () {
|
| 1435 |
+
var solvers = KEYS.filter(function (k) { return t.bits[KEYS.indexOf(k)] === "1"; });
|
| 1436 |
+
return "<b>" + t.id + "</b><br>solved by: " +
|
| 1437 |
+
(solvers.length ? "Spec. " + solvers.join(", ") : "no specialist") +
|
| 1438 |
+
"<br>realized merge: " + (t.bits[4] === "1" ? "solved" : "unsolved");
|
| 1439 |
+
});
|
| 1440 |
+
return c;
|
| 1441 |
+
});
|
| 1442 |
+
|
| 1443 |
+
function unionOf(keys) {
|
| 1444 |
+
return tasks.filter(function (t) {
|
| 1445 |
+
return keys.some(function (k) { return t.bits[KEYS.indexOf(k)] === "1"; });
|
| 1446 |
+
});
|
| 1447 |
+
}
|
| 1448 |
+
|
| 1449 |
+
function render() {
|
| 1450 |
+
var on = KEYS.filter(function (k) { return sel[k]; });
|
| 1451 |
+
var u = unionOf(on);
|
| 1452 |
+
var uIds = {};
|
| 1453 |
+
u.forEach(function (t) { uIds[t.id] = 1; });
|
| 1454 |
+
var cmp = $("tw-cmp").checked;
|
| 1455 |
+
|
| 1456 |
+
cells.forEach(function (c, i) {
|
| 1457 |
+
var t = tasks[i];
|
| 1458 |
+
var inU = !!uIds[t.id];
|
| 1459 |
+
var inM = t.bits[4] === "1";
|
| 1460 |
+
c.className = "tcell" + (inU ? " on" : "") +
|
| 1461 |
+
(cmp && inM && !inU ? " gain" : "") + (cmp && inU && !inM ? " loss" : "");
|
| 1462 |
+
});
|
| 1463 |
+
|
| 1464 |
+
var best = on.length ? Math.max.apply(null, on.map(function (k) { return counts[k]; })) : 0;
|
| 1465 |
+
$("tw-readout").innerHTML =
|
| 1466 |
+
ro("Selected", on.length ? on.join(" + ") : "none", true) +
|
| 1467 |
+
ro("Union coverage", u.length + "<small>/41</small>") +
|
| 1468 |
+
ro("Best single selected", (best || "\u2014") + (best ? "<small>/41</small>" : "")) +
|
| 1469 |
+
ro("Realized merge", counts.M + "<small>/41</small>");
|
| 1470 |
+
|
| 1471 |
+
var note = "";
|
| 1472 |
+
if (on.length === 4) {
|
| 1473 |
+
note = "All four specialists together cover <b>29/41</b>, two more than the best single lineage " +
|
| 1474 |
+
"(Spec. D at 27). The harness DarwinX actually produced solves <b>28/41</b>: recombination is not " +
|
| 1475 |
+
"a free set union. Turn on the comparison to see the difference \u2014 the merge picks up " +
|
| 1476 |
+
"<code>tw_448247</code>, which <i>no</i> specialist solves, and drops <code>tw_449421</code> and " +
|
| 1477 |
+
"<code>tw_498533</code>. These are single attempts, so individual flips sit inside the noise band " +
|
| 1478 |
+
"(one task = 2.4 points).";
|
| 1479 |
+
} else if (on.length === 0) {
|
| 1480 |
+
note = "Nothing selected. 12 of the 41 held-out tasks are solved by none of the four specialists.";
|
| 1481 |
+
} else if (u.length > best) {
|
| 1482 |
+
note = "This selection covers <b>" + u.length + "/41</b>, which is " + (u.length - best) +
|
| 1483 |
+
" more than its best individual member \u2014 the specialists solve overlapping but distinct subsets.";
|
| 1484 |
+
} else {
|
| 1485 |
+
note = "This selection covers <b>" + u.length + "/41</b>, no more than its best individual member: " +
|
| 1486 |
+
"these lineages are not complementary.";
|
| 1487 |
+
}
|
| 1488 |
+
$("tw-note").innerHTML = note;
|
| 1489 |
+
|
| 1490 |
+
var lg = $("tw-legend");
|
| 1491 |
+
lg.innerHTML =
|
| 1492 |
+
"<span><i class=\"sw\" style=\"background:var(--blue)\"></i> solved by selection</span>" +
|
| 1493 |
+
"<span><i class=\"sw\" style=\"background:#e6ecf2\"></i> unsolved</span>" +
|
| 1494 |
+
(cmp ? "<span><i class=\"sw\" style=\"box-shadow:0 0 0 2px #e8a33d;background:#e6ecf2\"></i>" +
|
| 1495 |
+
" merge solves, selection does not</span>" +
|
| 1496 |
+
"<span><i class=\"sw\" style=\"box-shadow:0 0 0 2px #d05a4e;background:var(--blue)\"></i>" +
|
| 1497 |
+
" selection solves, merge does not</span>" : "");
|
| 1498 |
+
}
|
| 1499 |
+
|
| 1500 |
+
function ro(k, v, plain) {
|
| 1501 |
+
return "<div class=\"ro" + (plain ? "" : " hi") + "\"><div class=\"k\">" + k +
|
| 1502 |
+
"</div><div class=\"v\"" + (plain ? " style=\"font-size:0.95rem;font-weight:600\"" : "") + ">" +
|
| 1503 |
+
v + "</div></div>";
|
| 1504 |
+
}
|
| 1505 |
+
|
| 1506 |
+
$("tw-cmp").addEventListener("change", render);
|
| 1507 |
+
render();
|
| 1508 |
+
})();
|
| 1509 |
+
|
| 1510 |
+
/* ---------------- 2. TB2.1 cluster explorer ---------------- */
|
| 1511 |
+
(function () {
|
| 1512 |
+
if (!D || !D.tb21Clusters) { return; }
|
| 1513 |
+
var data = D.tb21Clusters.slice();
|
| 1514 |
+
var modes = [
|
| 1515 |
+
{ id: "delta", label: "gain", cmp: function (a, b) { return b.delta - a.delta; } },
|
| 1516 |
+
{ id: "base", label: "base rate", cmp: function (a, b) { return a.base - b.base; } },
|
| 1517 |
+
{ id: "n", label: "cluster size", cmp: function (a, b) { return b.n - a.n; } },
|
| 1518 |
+
{ id: "name", label: "name", cmp: function (a, b) { return a.name.localeCompare(b.name); } }
|
| 1519 |
+
];
|
| 1520 |
+
var active = "delta";
|
| 1521 |
+
|
| 1522 |
+
var chips = $("cl-chips");
|
| 1523 |
+
modes.forEach(function (m) {
|
| 1524 |
+
var b = document.createElement("button");
|
| 1525 |
+
b.className = "chip";
|
| 1526 |
+
b.type = "button";
|
| 1527 |
+
b.textContent = m.label;
|
| 1528 |
+
b.setAttribute("aria-pressed", m.id === active ? "true" : "false");
|
| 1529 |
+
b.addEventListener("click", function () {
|
| 1530 |
+
active = m.id;
|
| 1531 |
+
Array.prototype.forEach.call(chips.children, function (c) {
|
| 1532 |
+
c.setAttribute("aria-pressed", c === b ? "true" : "false");
|
| 1533 |
+
});
|
| 1534 |
+
render();
|
| 1535 |
+
});
|
| 1536 |
+
chips.appendChild(b);
|
| 1537 |
+
});
|
| 1538 |
+
|
| 1539 |
+
function render() {
|
| 1540 |
+
var mode = modes.filter(function (m) { return m.id === active; })[0];
|
| 1541 |
+
var rows = data.slice().sort(mode.cmp);
|
| 1542 |
+
var showBase = $("cl-base").checked;
|
| 1543 |
+
var box = $("cl-bars");
|
| 1544 |
+
box.innerHTML = "";
|
| 1545 |
+
rows.forEach(function (d) {
|
| 1546 |
+
var el = document.createElement("div");
|
| 1547 |
+
el.className = "cbar";
|
| 1548 |
+
el.innerHTML =
|
| 1549 |
+
"<div class=\"cname\">" + d.name + " <span class=\"cn\">n=" + d.n + "</span></div>" +
|
| 1550 |
+
"<div class=\"track\">" +
|
| 1551 |
+
(showBase ? "<div class=\"fill b\" style=\"left:0;width:" + d.base + "%\"></div>" : "") +
|
| 1552 |
+
"<div class=\"fill e\" style=\"left:0;width:" + d.evolved + "%;height:" +
|
| 1553 |
+
(showBase ? "58%;top:21%" : "100%;top:0") + "\"></div>" +
|
| 1554 |
+
"</div>" +
|
| 1555 |
+
"<div class=\"cd " + (d.delta >= 0 ? "up" : "down") + "\">" +
|
| 1556 |
+
(d.delta >= 0 ? "+" : "\u2212") + Math.abs(d.delta).toFixed(1) + "</div>";
|
| 1557 |
+
bindTip(el.querySelector(".track"), function () {
|
| 1558 |
+
return "<b>" + d.name + "</b><br>base " + d.base.toFixed(1) + "% \u2192 evolved " +
|
| 1559 |
+
d.evolved.toFixed(1) + "%<br>" + d.n + " tasks, \u0394 " +
|
| 1560 |
+
(d.delta >= 0 ? "+" : "\u2212") + Math.abs(d.delta).toFixed(1) + " points";
|
| 1561 |
+
});
|
| 1562 |
+
box.appendChild(el);
|
| 1563 |
+
});
|
| 1564 |
+
}
|
| 1565 |
+
|
| 1566 |
+
$("cl-base").addEventListener("change", render);
|
| 1567 |
+
render();
|
| 1568 |
+
})();
|
| 1569 |
+
|
| 1570 |
+
/* ---------------- 3. WAI evolution curve ---------------- */
|
| 1571 |
+
(function () {
|
| 1572 |
+
if (!D || !D.waiCurve) { return; }
|
| 1573 |
+
var s = D.waiCurve;
|
| 1574 |
+
var svg = $("cv-chart");
|
| 1575 |
+
var W = 860, H = 320, L = 44, R = 14, T = 14, B = 34;
|
| 1576 |
+
var iw = W - L - R, ih = H - T - B;
|
| 1577 |
+
var NS = "http://www.w3.org/2000/svg";
|
| 1578 |
+
|
| 1579 |
+
function x(i) { return L + (s.length === 1 ? iw / 2 : (i / (s.length - 1)) * iw); }
|
| 1580 |
+
function y(v) { return T + ih - (v / 100) * ih; }
|
| 1581 |
+
function mk(n, a) {
|
| 1582 |
+
var e = document.createElementNS(NS, n);
|
| 1583 |
+
Object.keys(a).forEach(function (k) { e.setAttribute(k, a[k]); });
|
| 1584 |
+
return e;
|
| 1585 |
+
}
|
| 1586 |
+
|
| 1587 |
+
function render() {
|
| 1588 |
+
while (svg.firstChild) { svg.removeChild(svg.firstChild); }
|
| 1589 |
+
|
| 1590 |
+
[0, 25, 50, 75, 100].forEach(function (v) {
|
| 1591 |
+
svg.appendChild(mk("line", { class: "gl", x1: L, x2: W - R, y1: y(v), y2: y(v) }));
|
| 1592 |
+
var t = mk("text", { class: "lbl", x: L - 8, y: y(v) + 3.5, "text-anchor": "end" });
|
| 1593 |
+
t.textContent = v;
|
| 1594 |
+
svg.appendChild(t);
|
| 1595 |
+
});
|
| 1596 |
+
svg.appendChild(mk("line", { class: "ax", x1: L, x2: L, y1: T, y2: T + ih }));
|
| 1597 |
+
svg.appendChild(mk("line", { class: "ax", x1: L, x2: W - R, y1: T + ih, y2: T + ih }));
|
| 1598 |
+
|
| 1599 |
+
[0, 9, 18, 27, 36].forEach(function (i) {
|
| 1600 |
+
if (i >= s.length) { return; }
|
| 1601 |
+
var t = mk("text", { class: "lbl", x: x(i), y: T + ih + 17, "text-anchor": "middle" });
|
| 1602 |
+
t.textContent = i;
|
| 1603 |
+
svg.appendChild(t);
|
| 1604 |
+
});
|
| 1605 |
+
var xl = mk("text", { class: "lbl", x: L + iw / 2, y: H - 3, "text-anchor": "middle" });
|
| 1606 |
+
xl.textContent = "archive node";
|
| 1607 |
+
svg.appendChild(xl);
|
| 1608 |
+
var yl = mk("text", { class: "lbl", x: 12, y: T + ih / 2, "text-anchor": "middle",
|
| 1609 |
+
transform: "rotate(-90 12 " + (T + ih / 2) + ")" });
|
| 1610 |
+
yl.textContent = "screening score (%)";
|
| 1611 |
+
svg.appendChild(yl);
|
| 1612 |
+
|
| 1613 |
+
if ($("cv-best").checked) {
|
| 1614 |
+
var m = -1, pts = [];
|
| 1615 |
+
s.forEach(function (v, i) {
|
| 1616 |
+
m = Math.max(m, v);
|
| 1617 |
+
if (i > 0) { pts.push(x(i) + "," + y(m)); }
|
| 1618 |
+
pts.push(x(i) + "," + y(m));
|
| 1619 |
+
});
|
| 1620 |
+
svg.appendChild(mk("polyline", { class: "best", points: pts.join(" ") }));
|
| 1621 |
+
}
|
| 1622 |
+
|
| 1623 |
+
if ($("cv-all").checked) {
|
| 1624 |
+
s.forEach(function (v, i) {
|
| 1625 |
+
var dot = mk("circle", { class: "dot", cx: x(i), cy: y(v), r: 3.4 });
|
| 1626 |
+
svg.appendChild(dot);
|
| 1627 |
+
var hit = mk("circle", { class: "hit", cx: x(i), cy: y(v), r: 11 });
|
| 1628 |
+
hit.addEventListener("mousemove", function (e) {
|
| 1629 |
+
dot.classList.add("act");
|
| 1630 |
+
showTip("<b>archive node " + i + "</b><br>screening score " + v.toFixed(1) + "%", e);
|
| 1631 |
+
});
|
| 1632 |
+
hit.addEventListener("mouseleave", function () { dot.classList.remove("act"); hideTip(); });
|
| 1633 |
+
svg.appendChild(hit);
|
| 1634 |
+
});
|
| 1635 |
+
}
|
| 1636 |
+
}
|
| 1637 |
+
|
| 1638 |
+
var best = Math.max.apply(null, s);
|
| 1639 |
+
$("cv-note").innerHTML = "The run evaluates <b>" + s.length + "</b> variants. The screening score " +
|
| 1640 |
+
"climbs from " + s[0].toFixed(1) + "% to <b>" + best.toFixed(1) + "%</b>, but not monotonically: " +
|
| 1641 |
+
"individual variants score well below the running best, and those are the ones the " +
|
| 1642 |
+
"preserve-and-extend contract reverts. Screening runs on synthetic intents, so this axis is the " +
|
| 1643 |
+
"in-loop proxy, not the 1,260-task real result.";
|
| 1644 |
+
|
| 1645 |
+
$("cv-all").addEventListener("change", render);
|
| 1646 |
+
$("cv-best").addEventListener("change", render);
|
| 1647 |
+
render();
|
| 1648 |
+
})();
|
| 1649 |
+
|
| 1650 |
+
/* ---------------- 4. sortable tables ---------------- */
|
| 1651 |
+
(function () {
|
| 1652 |
+
function num(td) {
|
| 1653 |
+
var v = parseFloat(td.textContent.replace(/[^0-9.\-\u2212]/g, "").replace("\u2212", "-"));
|
| 1654 |
+
return isNaN(v) ? null : v;
|
| 1655 |
+
}
|
| 1656 |
+
Array.prototype.forEach.call(document.querySelectorAll("table.sortable"), function (tb) {
|
| 1657 |
+
var ths = tb.querySelectorAll("thead th");
|
| 1658 |
+
Array.prototype.forEach.call(ths, function (th, ci) {
|
| 1659 |
+
th.classList.add("sortable");
|
| 1660 |
+
th.addEventListener("click", function () {
|
| 1661 |
+
var body = tb.querySelector("tbody");
|
| 1662 |
+
var rows = Array.prototype.slice.call(body.rows);
|
| 1663 |
+
var pinned = rows.filter(function (r) { return r.classList.contains("total"); });
|
| 1664 |
+
var movable = rows.filter(function (r) { return !r.classList.contains("total"); });
|
| 1665 |
+
if (!movable.length) { return; }
|
| 1666 |
+
|
| 1667 |
+
// Numeric columns open descending (best first); text columns open A-Z.
|
| 1668 |
+
var numeric = movable.some(function (r) { return num(r.cells[ci]) !== null; });
|
| 1669 |
+
var desc;
|
| 1670 |
+
if (th.classList.contains("desc")) { desc = false; }
|
| 1671 |
+
else if (th.classList.contains("asc")) { desc = true; }
|
| 1672 |
+
else { desc = numeric; }
|
| 1673 |
+
|
| 1674 |
+
Array.prototype.forEach.call(ths, function (o) { o.classList.remove("asc", "desc"); });
|
| 1675 |
+
th.classList.add(desc ? "desc" : "asc");
|
| 1676 |
+
|
| 1677 |
+
movable.sort(function (a, b) {
|
| 1678 |
+
var x = num(a.cells[ci]), y = num(b.cells[ci]);
|
| 1679 |
+
if (x === null || y === null) {
|
| 1680 |
+
var av = a.cells[ci].textContent.trim(), bv = b.cells[ci].textContent.trim();
|
| 1681 |
+
return desc ? bv.localeCompare(av) : av.localeCompare(bv);
|
| 1682 |
+
}
|
| 1683 |
+
return desc ? y - x : x - y;
|
| 1684 |
+
});
|
| 1685 |
+
movable.concat(pinned).forEach(function (r) { body.appendChild(r); });
|
| 1686 |
+
});
|
| 1687 |
+
});
|
| 1688 |
+
});
|
| 1689 |
+
})();
|
| 1690 |
+
|
| 1691 |
+
/* ---------------- 5. BibTeX copy ---------------- */
|
| 1692 |
+
$("copybtn").addEventListener("click", function () {
|
| 1693 |
+
var btn = this;
|
| 1694 |
+
navigator.clipboard.writeText($("bib").textContent).then(
|
| 1695 |
+
function () {
|
| 1696 |
+
btn.textContent = "Copied";
|
| 1697 |
+
setTimeout(function () { btn.textContent = "Copy"; }, 1600);
|
| 1698 |
+
},
|
| 1699 |
+
function () { btn.textContent = "Press \u2318C"; }
|
| 1700 |
+
);
|
| 1701 |
+
});
|
| 1702 |
+
})();
|
| 1703 |
+
</script>
|
| 1704 |
+
|
| 1705 |
+
</body>
|
| 1706 |
</html>
|
tools/build_data.py
ADDED
|
@@ -0,0 +1,223 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Emit assets/data.js for the DarwinX project page from the original run artifacts.
|
| 3 |
+
|
| 4 |
+
Every number the interactive figures display comes from the files listed in SOURCES,
|
| 5 |
+
so the page can be re-derived rather than hand-transcribed. Run from anywhere:
|
| 6 |
+
|
| 7 |
+
python3 tools/build_data.py
|
| 8 |
+
"""
|
| 9 |
+
|
| 10 |
+
import json
|
| 11 |
+
import os
|
| 12 |
+
import sys
|
| 13 |
+
from datetime import datetime, timezone
|
| 14 |
+
|
| 15 |
+
TW_DIR = ("/Users/yifan.zhang/Desktop/techreport/darwinx_zeyuan_feedback/"
|
| 16 |
+
"version_control/version_0717_local/results/tw/results")
|
| 17 |
+
NOTES = "/Users/yifan.zhang/Desktop/techreport/darwinx_zeyuan_feedback/notes"
|
| 18 |
+
|
| 19 |
+
TW_ARMS = [
|
| 20 |
+
("A", "Spec. A", "opus_node_34f71461_run.json"),
|
| 21 |
+
("B", "Spec. B", "opus_node_cab04955_run.json"),
|
| 22 |
+
("C", "Spec. C", "opus_node_56fc77cc_run.json"),
|
| 23 |
+
("D", "Spec. D", "opus_node_eb39901b_run.json"),
|
| 24 |
+
("M", "Merged", "TW_HEADLINE_merge_opus_28of41_run.json"),
|
| 25 |
+
]
|
| 26 |
+
|
| 27 |
+
# From notes/TB21_RESULTS.md (paired protocol, 88 tasks). Deltas are carried over as
|
| 28 |
+
# reported rather than recomputed: the source rounds the delta from unrounded rates,
|
| 29 |
+
# so evolved-minus-base on the displayed 1-decimal values differs by up to 0.1.
|
| 30 |
+
TB21_CLUSTERS = [
|
| 31 |
+
("ML & scientific computing", 23, 60.1, 74.9, 14.8),
|
| 32 |
+
("Data & databases", 9, 83.9, 97.8, 13.8),
|
| 33 |
+
("Algorithms & coding", 14, 83.1, 90.0, 6.9),
|
| 34 |
+
("System admin & networking", 11, 92.0, 97.6, 5.7),
|
| 35 |
+
("Parsing, text & media", 11, 67.3, 72.2, 5.0),
|
| 36 |
+
("Systems & build", 11, 79.3, 81.8, 2.5),
|
| 37 |
+
("Security & crypto", 9, 85.1, 84.4, -0.6),
|
| 38 |
+
]
|
| 39 |
+
|
| 40 |
+
# The four headline panels, from scripts/gen_summary_figure.py GROUPS. Each panel is
|
| 41 |
+
# matched-model: base and evolved share one frozen base, so the gap is the harness.
|
| 42 |
+
# The prior-agent value is context, not a controlled comparison, because those
|
| 43 |
+
# systems use different models and effort settings. SWE-V has no prior-agent bar and
|
| 44 |
+
# its "base" is the fix-skill reference rather than an unevolved Monet, so the label
|
| 45 |
+
# travels with the number.
|
| 46 |
+
HEADLINE = [
|
| 47 |
+
{"short": "TB 2.1", "name": "Terminal-Bench 2.1", "metric": "avg@5",
|
| 48 |
+
"model": "GPT-5.5", "scope": "89 tasks",
|
| 49 |
+
"baseLabel": "Monet (base)", "base": 75.5,
|
| 50 |
+
"evolvedLabel": "Monet (DarwinX)", "evolved": 83.2,
|
| 51 |
+
"priorLabel": "Codex", "prior": 83.1,
|
| 52 |
+
"delta": 7.7, "lo": 70, "hi": 88},
|
| 53 |
+
{"short": "T-World", "name": "TerminalWorld", "metric": "pass@1",
|
| 54 |
+
"model": "Opus 4.8", "scope": "41 held-out tasks",
|
| 55 |
+
"baseLabel": "Monet (base)", "base": 61.0,
|
| 56 |
+
"evolvedLabel": "Monet (DarwinX)", "evolved": 68.3,
|
| 57 |
+
"priorLabel": "Claude Code", "prior": 65.9,
|
| 58 |
+
"delta": 7.3, "lo": 55, "hi": 75},
|
| 59 |
+
{"short": "WA-Inf.", "name": "WebArena-Infinity", "metric": "audit-clean pass@1",
|
| 60 |
+
"model": "GPT-5.5", "scope": "1,260 real tasks",
|
| 61 |
+
"baseLabel": "Monet (base)", "base": 43.5,
|
| 62 |
+
"evolvedLabel": "Monet (DarwinX)", "evolved": 93.0,
|
| 63 |
+
"priorLabel": "GPT-5.5 + Browser Use", "prior": 86.1,
|
| 64 |
+
"delta": 49.5, "lo": 35, "hi": 100},
|
| 65 |
+
{"short": "SWE-V", "name": "SWE-bench Verified", "metric": "official pass@1",
|
| 66 |
+
"model": "Opus 4.8", "scope": "500 instances, transfer only",
|
| 67 |
+
"baseLabel": "Fix-skill reference", "base": 80.8,
|
| 68 |
+
"evolvedLabel": "Monet (DarwinX), transferred", "evolved": 84.2,
|
| 69 |
+
"priorLabel": None, "prior": None,
|
| 70 |
+
"delta": 3.4, "lo": 75, "hi": 88},
|
| 71 |
+
]
|
| 72 |
+
|
| 73 |
+
# scripts/gen_tw_dynamics.py + notes/tw_dynamics.json. Base is the unevolved v1.0.0
|
| 74 |
+
# harness on Opus 4.8 (25/41); its per-task run.json is not on this machine, so the
|
| 75 |
+
# merge explorer covers only the arms that have per-task records.
|
| 76 |
+
TW_HELDOUT_BARS = [("Base", 25), ("Spec. A", 24), ("Spec. B", 25),
|
| 77 |
+
("Spec. C", 26), ("Spec. D", 27), ("Monet (DarwinX)", 28)]
|
| 78 |
+
TW_CLAUDE_CODE = 27 # Claude Code (Opus 4.8) reference, 27/41
|
| 79 |
+
TW_N = 41
|
| 80 |
+
|
| 81 |
+
# scripts/gen_tb21_compute.py: medians precomputed from the trial archive.
|
| 82 |
+
TB21_COMPUTE = {
|
| 83 |
+
"groups": ["Newly solved (6 tasks)", "Already solved (69 tasks)"],
|
| 84 |
+
"turns": {"base": [11, 12], "evolved": [22, 13]},
|
| 85 |
+
"tokens": {"base": [89, 125], "evolved": [380, 172]}, # thousands
|
| 86 |
+
}
|
| 87 |
+
|
| 88 |
+
# scripts/gen_wai_invalid_composition.py: 293 invalid trajectories before evolution,
|
| 89 |
+
# 17 after, decomposed two ways over the same totals.
|
| 90 |
+
WAI_INVALID = {
|
| 91 |
+
"application": {
|
| 92 |
+
"labels": ["Elation Clinical", "Elation Prescription", "GitLab Plan & Track",
|
| 93 |
+
"Gmail", "Gmail Acct & Contact", "Handshake", "Linear Acct Mgmt",
|
| 94 |
+
"PayPal Wallet", "Super-human", "Xero Invoicing"],
|
| 95 |
+
"before": [29, 27, 38, 21, 44, 54, 26, 10, 44, 0],
|
| 96 |
+
"after": [2, 3, 0, 0, 3, 0, 0, 0, 8, 1],
|
| 97 |
+
"colors": ["#4C72B0", "#A6C8E0", "#DD8452", "#E3B23C", "#55A868",
|
| 98 |
+
"#8FD19E", "#C44E52", "#E79EA6", "#8172B3", "#C7BFE0"],
|
| 99 |
+
},
|
| 100 |
+
"mechanism": {
|
| 101 |
+
"labels": ["Evaluation plane access", "Privileged knowledge",
|
| 102 |
+
"Exploit or privilege escalation", "Raw state mutation"],
|
| 103 |
+
"before": [155, 97, 26, 15],
|
| 104 |
+
"after": [0, 0, 0, 17],
|
| 105 |
+
"colors": ["#55A868", "#DD8452", "#CE6DBD", "#8FA6D1"],
|
| 106 |
+
},
|
| 107 |
+
}
|
| 108 |
+
|
| 109 |
+
# scripts/gen_wai_audit_by_app.py: pass@1 before and after the validity audit.
|
| 110 |
+
WAI_AUDIT = {
|
| 111 |
+
"apps": ["Overall", "Elation-Clin", "Elation-Rx", "GitLab", "Gmail",
|
| 112 |
+
"Gmail-Acct", "Handshake", "Linear", "PayPal", "Superhuman", "Xero"],
|
| 113 |
+
"rawBase": [53.0, 95.8, 28.3, 75.7, 45.0, 33.3, 44.0, 54.2, 50.0, 53.3, 49.2],
|
| 114 |
+
"rawDx": [94.4, 98.3, 97.5, 97.9, 98.3, 94.2, 84.0, 94.2, 95.7, 94.2, 97.5],
|
| 115 |
+
"auditedBase": [43.5, 95.8, 20.0, 63.6, 25.0, 21.7, 36.5, 43.3, 49.3, 31.7, 39.2],
|
| 116 |
+
"auditedDx": [93.0, 96.7, 95.0, 97.9, 98.3, 91.7, 84.0, 94.2, 95.7, 87.5, 96.7],
|
| 117 |
+
}
|
| 118 |
+
|
| 119 |
+
|
| 120 |
+
def load_tw():
|
| 121 |
+
"""Per-task resolved flags for the four specialists and the merged harness.
|
| 122 |
+
|
| 123 |
+
The run files list the 41 tasks in different orders, so every arm is joined on
|
| 124 |
+
task_id and the emitted order is canonical (the first arm's file order).
|
| 125 |
+
"""
|
| 126 |
+
order, universe, sets = None, None, {}
|
| 127 |
+
for key, _label, fname in TW_ARMS:
|
| 128 |
+
path = os.path.join(TW_DIR, fname)
|
| 129 |
+
if not os.path.exists(path):
|
| 130 |
+
sys.exit(f"missing TerminalWorld run file: {path}")
|
| 131 |
+
rows = json.load(open(path))["per_task_results"]
|
| 132 |
+
ids = {r["task_id"] for r in rows}
|
| 133 |
+
if len(ids) != len(rows):
|
| 134 |
+
sys.exit(f"duplicate task_id in {fname}")
|
| 135 |
+
if order is None:
|
| 136 |
+
order, universe = [r["task_id"] for r in rows], ids
|
| 137 |
+
elif ids != universe:
|
| 138 |
+
sys.exit(f"task set differs in {fname}: "
|
| 139 |
+
f"{sorted(ids ^ universe)} not shared; refusing to emit")
|
| 140 |
+
sets[key] = {r["task_id"] for r in rows if r["resolved"]}
|
| 141 |
+
|
| 142 |
+
tasks = [{"id": t, "bits": "".join("1" if t in sets[k] else "0"
|
| 143 |
+
for k, _l, _f in TW_ARMS)} for t in order]
|
| 144 |
+
counts = {k: len(sets[k]) for k, _l, _f in TW_ARMS}
|
| 145 |
+
if [counts[k] for k in "ABCD"] != [24, 25, 26, 27] or counts["M"] != 28:
|
| 146 |
+
sys.exit(f"unexpected TerminalWorld totals {counts}; paper reports 24/25/26/27 and 28")
|
| 147 |
+
return tasks, counts
|
| 148 |
+
|
| 149 |
+
|
| 150 |
+
def load_wai_curve():
|
| 151 |
+
path = os.path.join(NOTES, "tw_dynamics.json")
|
| 152 |
+
if not os.path.exists(path):
|
| 153 |
+
sys.exit(f"missing {path}")
|
| 154 |
+
scores = json.load(open(path))["wai_adaptive_scores"]
|
| 155 |
+
return [round(float(s), 2) for s in scores]
|
| 156 |
+
|
| 157 |
+
|
| 158 |
+
def main():
|
| 159 |
+
tasks, counts = load_tw()
|
| 160 |
+
curve = load_wai_curve()
|
| 161 |
+
|
| 162 |
+
payload = {
|
| 163 |
+
"generated": datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC"),
|
| 164 |
+
"tw": {
|
| 165 |
+
"arms": [{"key": k, "label": l} for k, l, _f in TW_ARMS],
|
| 166 |
+
"tasks": tasks,
|
| 167 |
+
"counts": counts,
|
| 168 |
+
},
|
| 169 |
+
"tb21Clusters": [
|
| 170 |
+
{"name": n, "n": c, "base": b, "evolved": e, "delta": d}
|
| 171 |
+
for n, c, b, e, d in TB21_CLUSTERS
|
| 172 |
+
],
|
| 173 |
+
"waiCurve": curve,
|
| 174 |
+
"headline": HEADLINE,
|
| 175 |
+
"twBars": {
|
| 176 |
+
"n": TW_N,
|
| 177 |
+
"claudeCode": TW_CLAUDE_CODE,
|
| 178 |
+
"bars": [{"label": l, "solved": s} for l, s in TW_HELDOUT_BARS],
|
| 179 |
+
},
|
| 180 |
+
"tb21Compute": TB21_COMPUTE,
|
| 181 |
+
"waiInvalid": WAI_INVALID,
|
| 182 |
+
"waiAudit": WAI_AUDIT,
|
| 183 |
+
}
|
| 184 |
+
|
| 185 |
+
# Guard the invariants the page states in prose.
|
| 186 |
+
for view in ("application", "mechanism"):
|
| 187 |
+
v = WAI_INVALID[view]
|
| 188 |
+
if sum(v["before"]) != 293 or sum(v["after"]) != 17:
|
| 189 |
+
sys.exit(f"WAI_INVALID[{view}] must total 293 before and 17 after, got "
|
| 190 |
+
f"{sum(v['before'])} and {sum(v['after'])}")
|
| 191 |
+
if not (len(v["labels"]) == len(v["before"]) == len(v["after"]) == len(v["colors"])):
|
| 192 |
+
sys.exit(f"WAI_INVALID[{view}] has ragged columns")
|
| 193 |
+
for k in ("rawBase", "rawDx", "auditedBase", "auditedDx"):
|
| 194 |
+
if len(WAI_AUDIT[k]) != len(WAI_AUDIT["apps"]):
|
| 195 |
+
sys.exit(f"WAI_AUDIT[{k}] length does not match apps")
|
| 196 |
+
# The audited numbers must agree with the per-application table on the page.
|
| 197 |
+
page_table = [96.7, 95.0, 97.9, 98.3, 91.7, 84.0, 94.2, 95.7, 87.5, 96.7]
|
| 198 |
+
if WAI_AUDIT["auditedDx"][1:] != page_table:
|
| 199 |
+
sys.exit("WAI_AUDIT auditedDx disagrees with the page's per-application table")
|
| 200 |
+
if [b["solved"] for b in payload["twBars"]["bars"]] != [25, 24, 25, 26, 27, 28]:
|
| 201 |
+
sys.exit("twBars solved counts changed unexpectedly")
|
| 202 |
+
|
| 203 |
+
out = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
|
| 204 |
+
"assets", "data.js")
|
| 205 |
+
with open(out, "w") as f:
|
| 206 |
+
f.write("/* Generated by tools/build_data.py - do not edit by hand. */\n")
|
| 207 |
+
f.write("window.DARWINX = ")
|
| 208 |
+
json.dump(payload, f, indent=1, sort_keys=False)
|
| 209 |
+
f.write(";\n")
|
| 210 |
+
|
| 211 |
+
union = {t["id"] for t in tasks if "1" in t["bits"][:4]}
|
| 212 |
+
merged = {t["id"] for t in tasks if t["bits"][4] == "1"}
|
| 213 |
+
print(f"wrote {out}")
|
| 214 |
+
print(f" TerminalWorld: {len(tasks)} tasks, counts={counts}")
|
| 215 |
+
print(f" union(A..D)={len(union)} merged={len(merged)}")
|
| 216 |
+
print(f" merged only: {sorted(merged - union)}")
|
| 217 |
+
print(f" union only : {sorted(union - merged)}")
|
| 218 |
+
print(f" TB2.1 clusters: {len(payload['tb21Clusters'])}")
|
| 219 |
+
print(f" WAI curve: {len(curve)} nodes, max={max(curve)}")
|
| 220 |
+
|
| 221 |
+
|
| 222 |
+
if __name__ == "__main__":
|
| 223 |
+
main()
|
tools/deploy_hf.sh
ADDED
|
@@ -0,0 +1,58 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env bash
|
| 2 |
+
# Publish this directory to a Hugging Face static Space.
|
| 3 |
+
#
|
| 4 |
+
# Prerequisite: the Space must already exist. Create it once at
|
| 5 |
+
# https://huggingface.co/new-space -> SDK: "Static"
|
| 6 |
+
# Then run, from anywhere:
|
| 7 |
+
# tools/deploy_hf.sh <user>/<space>
|
| 8 |
+
#
|
| 9 |
+
# Auth: uses $HF_TOKEN if set, otherwise prompts. Needs a token with write
|
| 10 |
+
# access (https://huggingface.co/settings/tokens).
|
| 11 |
+
set -euo pipefail
|
| 12 |
+
|
| 13 |
+
SPACE="${1:-}"
|
| 14 |
+
if [[ -z "$SPACE" || "$SPACE" != */* ]]; then
|
| 15 |
+
echo "usage: $0 <user>/<space> e.g. $0 yifanzhang/darwinx" >&2
|
| 16 |
+
exit 2
|
| 17 |
+
fi
|
| 18 |
+
USER="${SPACE%%/*}"
|
| 19 |
+
|
| 20 |
+
SRC="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
| 21 |
+
cd "$SRC"
|
| 22 |
+
[[ -f index.html && -f README.md ]] || { echo "run this from the project page directory" >&2; exit 1; }
|
| 23 |
+
|
| 24 |
+
if [[ -z "${HF_TOKEN:-}" ]]; then
|
| 25 |
+
read -rsp "Hugging Face write token (hf_...): " HF_TOKEN; echo
|
| 26 |
+
fi
|
| 27 |
+
[[ -n "$HF_TOKEN" ]] || { echo "no token given" >&2; exit 1; }
|
| 28 |
+
|
| 29 |
+
WORK="$(mktemp -d)"
|
| 30 |
+
trap 'rm -rf "$WORK"' EXIT
|
| 31 |
+
echo "==> cloning https://huggingface.co/spaces/$SPACE"
|
| 32 |
+
git -c credential.helper= \
|
| 33 |
+
clone --quiet "https://$USER:$HF_TOKEN@huggingface.co/spaces/$SPACE" "$WORK/space" 2>&1 \
|
| 34 |
+
| sed 's/'"$HF_TOKEN"'/***/g' \
|
| 35 |
+
|| { echo "clone failed - does the Space exist, and is the token writable?" >&2; exit 1; }
|
| 36 |
+
|
| 37 |
+
echo "==> copying the page"
|
| 38 |
+
# Ship only what the page needs; drop local scratch files.
|
| 39 |
+
rsync -a --delete \
|
| 40 |
+
--exclude '.git' --exclude '.DS_Store' --exclude '_*' --exclude '*.log' \
|
| 41 |
+
./ "$WORK/space/"
|
| 42 |
+
|
| 43 |
+
cd "$WORK/space"
|
| 44 |
+
git add -A
|
| 45 |
+
if git diff --cached --quiet; then
|
| 46 |
+
echo "==> nothing changed; Space is already up to date"
|
| 47 |
+
exit 0
|
| 48 |
+
fi
|
| 49 |
+
echo "==> files being published:"
|
| 50 |
+
git diff --cached --name-status | sed 's/^/ /'
|
| 51 |
+
|
| 52 |
+
git -c user.name="$USER" -c user.email="$USER@users.noreply.huggingface.co" \
|
| 53 |
+
commit --quiet -m "DarwinX project page"
|
| 54 |
+
git push --quiet origin HEAD:main 2>&1 | sed 's/'"$HF_TOKEN"'/***/g'
|
| 55 |
+
|
| 56 |
+
echo
|
| 57 |
+
echo "==> live at https://huggingface.co/spaces/$SPACE"
|
| 58 |
+
echo " (a static Space builds in a few seconds; hard-refresh if you see the old copy)"
|
tools/interaction_test.html
ADDED
|
@@ -0,0 +1,193 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
<!doctype html>
|
| 2 |
+
<html><head><meta charset="utf-8"><title>pending</title></head>
|
| 3 |
+
<body>
|
| 4 |
+
<div id="host"></div>
|
| 5 |
+
<script>
|
| 6 |
+
/* Loads ../index.html into this document, then drives the widgets with real click
|
| 7 |
+
events and asserts the readouts. Results are written into document.title so
|
| 8 |
+
`chrome --headless --dump-dom` can report them. */
|
| 9 |
+
var results = [];
|
| 10 |
+
function check(name, got, want) {
|
| 11 |
+
results.push((String(got) === String(want) ? "PASS" : "FAIL") + " " + name +
|
| 12 |
+
" got=" + got + " want=" + want);
|
| 13 |
+
}
|
| 14 |
+
|
| 15 |
+
fetch("../index.html").then(function (r) { return r.text(); }).then(function (html) {
|
| 16 |
+
var doc = new DOMParser().parseFromString(html, "text/html");
|
| 17 |
+
|
| 18 |
+
// move body content in, then re-execute scripts (innerHTML does not run them)
|
| 19 |
+
document.getElementById("host").innerHTML = doc.body.innerHTML;
|
| 20 |
+
var scripts = Array.prototype.slice.call(document.querySelectorAll("#host script"));
|
| 21 |
+
var chain = Promise.resolve();
|
| 22 |
+
scripts.forEach(function (old) {
|
| 23 |
+
chain = chain.then(function () {
|
| 24 |
+
return new Promise(function (done) {
|
| 25 |
+
var s = document.createElement("script");
|
| 26 |
+
// getAttribute keeps the raw relative path; .src would already be resolved
|
| 27 |
+
// against this page's directory, which is one level too deep.
|
| 28 |
+
var raw = old.getAttribute("src");
|
| 29 |
+
if (raw) { s.src = "../" + raw; s.onload = done; s.onerror = done; }
|
| 30 |
+
else { s.textContent = old.textContent; }
|
| 31 |
+
document.head.appendChild(s);
|
| 32 |
+
if (!raw) { done(); }
|
| 33 |
+
});
|
| 34 |
+
});
|
| 35 |
+
});
|
| 36 |
+
|
| 37 |
+
return chain.then(function () { return new Promise(function (r2) { setTimeout(r2, 120); }); });
|
| 38 |
+
}).then(function () {
|
| 39 |
+
var chips = document.querySelectorAll("#tw-chips .chip");
|
| 40 |
+
var unionVal = function () {
|
| 41 |
+
return document.querySelectorAll("#tw-grid .tcell.on").length;
|
| 42 |
+
};
|
| 43 |
+
var readout = function () { return document.getElementById("tw-readout").textContent; };
|
| 44 |
+
|
| 45 |
+
check("widgets built: chips", chips.length, 4);
|
| 46 |
+
check("grid cells", document.querySelectorAll("#tw-grid .tcell").length, 41);
|
| 47 |
+
check("all-4 union highlighted", unionVal(), 29);
|
| 48 |
+
check("readout shows 29", /29\/41/.test(readout()), true);
|
| 49 |
+
check("readout shows merge 28", /28\/41/.test(readout()), true);
|
| 50 |
+
|
| 51 |
+
// turn OFF Spec. A -> union(B,C,D) should still be 29
|
| 52 |
+
chips[0].click();
|
| 53 |
+
check("A off -> union(BCD)", unionVal(), 29);
|
| 54 |
+
|
| 55 |
+
// turn OFF B and C -> D alone = 27
|
| 56 |
+
chips[1].click(); chips[2].click();
|
| 57 |
+
check("D alone", unionVal(), 27);
|
| 58 |
+
|
| 59 |
+
// D off too -> nothing selected
|
| 60 |
+
chips[3].click();
|
| 61 |
+
check("none selected", unionVal(), 0);
|
| 62 |
+
|
| 63 |
+
// back to A + D only -> 29 (complementary pair)
|
| 64 |
+
chips[0].click(); chips[3].click();
|
| 65 |
+
check("A+D complementary pair", unionVal(), 29);
|
| 66 |
+
|
| 67 |
+
// full set again, then compare-with-merge ring counts
|
| 68 |
+
chips[1].click(); chips[2].click();
|
| 69 |
+
check("back to all four", unionVal(), 29);
|
| 70 |
+
document.getElementById("tw-cmp").click();
|
| 71 |
+
check("merge-only ring (gain)", document.querySelectorAll("#tw-grid .tcell.gain").length, 1);
|
| 72 |
+
check("union-only ring (loss)", document.querySelectorAll("#tw-grid .tcell.loss").length, 2);
|
| 73 |
+
|
| 74 |
+
// cluster widget
|
| 75 |
+
var bars = document.querySelectorAll("#cl-bars .cbar");
|
| 76 |
+
check("cluster rows", bars.length, 7);
|
| 77 |
+
var firstDelta = bars[0].querySelector(".cd").textContent.trim();
|
| 78 |
+
check("default sort = biggest gain first", firstDelta, "+14.8");
|
| 79 |
+
var sortChips = document.querySelectorAll("#cl-chips .chip");
|
| 80 |
+
sortChips[1].click(); // by base rate ascending
|
| 81 |
+
var b2 = document.querySelectorAll("#cl-bars .cbar");
|
| 82 |
+
check("sort by base rate -> lowest base first",
|
| 83 |
+
/ML & scientific/.test(b2[0].querySelector(".cname").textContent), true);
|
| 84 |
+
sortChips[2].click(); // by cluster size
|
| 85 |
+
var b3 = document.querySelectorAll("#cl-bars .cbar");
|
| 86 |
+
check("sort by size -> n=23 first",
|
| 87 |
+
/n=23/.test(b3[0].querySelector(".cname").textContent), true);
|
| 88 |
+
document.getElementById("cl-base").click(); // hide base
|
| 89 |
+
check("hide base removes grey fills", document.querySelectorAll("#cl-bars .fill.b").length, 0);
|
| 90 |
+
|
| 91 |
+
// curve widget
|
| 92 |
+
check("curve dots", document.querySelectorAll("#cv-chart .dot").length, 37);
|
| 93 |
+
check("best-so-far polyline", document.querySelectorAll("#cv-chart .best").length, 1);
|
| 94 |
+
document.getElementById("cv-all").click();
|
| 95 |
+
check("hide variants", document.querySelectorAll("#cv-chart .dot").length, 0);
|
| 96 |
+
document.getElementById("cv-best").click();
|
| 97 |
+
check("hide envelope", document.querySelectorAll("#cv-chart .best").length, 0);
|
| 98 |
+
|
| 99 |
+
// sortable table: WAI overall row must stay pinned last
|
| 100 |
+
var wai = document.querySelectorAll("table.sortable")[2];
|
| 101 |
+
var ths = wai.querySelectorAll("thead th");
|
| 102 |
+
ths[6].click(); // sort by Monet (DarwinX)
|
| 103 |
+
var rows = wai.querySelectorAll("tbody tr");
|
| 104 |
+
check("total row stays last after sort",
|
| 105 |
+
rows[rows.length - 1].classList.contains("total"), true);
|
| 106 |
+
var firstCell = rows[0].cells[6].textContent.trim();
|
| 107 |
+
check("descending sort puts 98.3 first", firstCell, "98.3");
|
| 108 |
+
ths[6].click(); // ascending
|
| 109 |
+
var rows2 = wai.querySelectorAll("tbody tr");
|
| 110 |
+
check("ascending sort puts 84.0 first", rows2[0].cells[6].textContent.trim(), "84.0");
|
| 111 |
+
check("total still last", rows2[rows2.length - 1].classList.contains("total"), true);
|
| 112 |
+
|
| 113 |
+
// ---- headline panels ----
|
| 114 |
+
check("headline panels", document.querySelectorAll("#hl-panels .panel").length, 4);
|
| 115 |
+
check("prior bars present", document.querySelectorAll("#hl-panels .bar.p").length, 3);
|
| 116 |
+
var wa = document.querySelectorAll("#hl-panels .panel")[2];
|
| 117 |
+
var hOwn = parseFloat(wa.querySelector(".bar.e").getAttribute("height"));
|
| 118 |
+
document.querySelectorAll("#hl-chips .chip")[1].click(); // shared 0-100
|
| 119 |
+
var wa2 = document.querySelectorAll("#hl-panels .panel")[2];
|
| 120 |
+
var hShared = parseFloat(wa2.querySelector(".bar.e").getAttribute("height"));
|
| 121 |
+
check("shared axis changes bar geometry", hShared !== hOwn, true);
|
| 122 |
+
var tb = document.querySelectorAll("#hl-panels .panel")[0];
|
| 123 |
+
var tbH = parseFloat(tb.querySelector(".bar.e").getAttribute("height"));
|
| 124 |
+
check("shared axis: TB2.1 bar shorter than WAI bar", tbH < hShared, true);
|
| 125 |
+
document.getElementById("hl-prior").click();
|
| 126 |
+
check("hide prior removes its bars", document.querySelectorAll("#hl-panels .bar.p").length, 0);
|
| 127 |
+
document.querySelectorAll("#hl-chips .chip")[0].click(); // back to per-panel
|
| 128 |
+
|
| 129 |
+
// ---- TerminalWorld specialist bars ----
|
| 130 |
+
var spRows = document.querySelectorAll("#sp-rows .hrow");
|
| 131 |
+
check("specialist rows (6 bars + ref tag)", spRows.length, 7);
|
| 132 |
+
check("percent shown by default", /68\.3%/.test(document.getElementById("sp-rows").textContent), true);
|
| 133 |
+
var spW = function (n) {
|
| 134 |
+
return parseFloat(document.querySelectorAll("#sp-rows .hf")[n].style.width);
|
| 135 |
+
};
|
| 136 |
+
var zoomSpread = spW(5) - spW(1); // merged minus weakest specialist
|
| 137 |
+
document.querySelectorAll("#sp-chips .chip")[1].click(); // full 0-100 axis
|
| 138 |
+
var fullSpread = spW(5) - spW(1);
|
| 139 |
+
check("zoomed axis magnifies the spread", zoomSpread > fullSpread * 3, true);
|
| 140 |
+
check("full axis puts merged at its true 68.3%", Math.round(spW(5) * 10) / 10, 68.3);
|
| 141 |
+
document.querySelectorAll("#sp-chips .chip")[0].click();
|
| 142 |
+
document.getElementById("sp-counts").click();
|
| 143 |
+
check("counts toggle shows 28/41", /28\/41/.test(document.getElementById("sp-rows").textContent), true);
|
| 144 |
+
|
| 145 |
+
// ---- TB2.1 compute ----
|
| 146 |
+
check("compute rows (2 groups x 2 series)", document.querySelectorAll("#cp-rows .hrow").length, 4);
|
| 147 |
+
var cpVals = function () {
|
| 148 |
+
return Array.prototype.map.call(document.querySelectorAll("#cp-rows .hv"),
|
| 149 |
+
function (e) { return e.textContent.trim(); }).join(",");
|
| 150 |
+
};
|
| 151 |
+
check("turns shown first", cpVals(), "11,22,12,13");
|
| 152 |
+
document.querySelectorAll("#cp-chips .chip")[1].click();
|
| 153 |
+
check("tokens toggle shows K values", cpVals(), "89K,380K,125K,172K");
|
| 154 |
+
|
| 155 |
+
// ---- WAI invalid composition ----
|
| 156 |
+
check("two stacked rows", document.querySelectorAll("#iv-rows .stack").length, 2);
|
| 157 |
+
var mechSegs = document.querySelectorAll("#iv-rows .stack i").length;
|
| 158 |
+
check("mechanism view: 4 before + 1 after segments", mechSegs, 5);
|
| 159 |
+
check("totals rendered 293 and 17",
|
| 160 |
+
/293/.test(document.getElementById("iv-rows").textContent) &&
|
| 161 |
+
/17/.test(document.getElementById("iv-rows").textContent), true);
|
| 162 |
+
var afterStack = document.querySelectorAll("#iv-rows .stack")[1];
|
| 163 |
+
var wShared = afterStack.querySelector("i").style.width;
|
| 164 |
+
document.getElementById("iv-norm").click();
|
| 165 |
+
var wNorm = document.querySelectorAll("#iv-rows .stack")[1].querySelector("i").style.width;
|
| 166 |
+
check("normalising widens the after row", parseFloat(wNorm) > parseFloat(wShared), true);
|
| 167 |
+
check("after row fills the width when normalised", parseFloat(wNorm), 100);
|
| 168 |
+
document.querySelectorAll("#iv-chips .chip")[0].click(); // by application
|
| 169 |
+
check("application view segment count",
|
| 170 |
+
document.querySelectorAll("#iv-rows .stack i").length, 9 + 5);
|
| 171 |
+
|
| 172 |
+
// ---- WAI audit dumbbell ----
|
| 173 |
+
check("audit lanes (11 apps x 2)", document.querySelectorAll("#ad-rows .seg").length, 22);
|
| 174 |
+
check("axis row precedes the data rows",
|
| 175 |
+
/pass@1/.test(document.querySelectorAll("#ad-rows .drow")[0].textContent), true);
|
| 176 |
+
check("overall row pinned first",
|
| 177 |
+
document.querySelectorAll("#ad-rows .drow")[1].textContent.indexOf("Overall"), 0);
|
| 178 |
+
document.getElementById("ad-base").click();
|
| 179 |
+
check("hiding base halves the lanes", document.querySelectorAll("#ad-rows .seg").length, 11);
|
| 180 |
+
document.getElementById("ad-base").click();
|
| 181 |
+
document.getElementById("ad-sort").click();
|
| 182 |
+
var firstAfterOverall = document.querySelectorAll("#ad-rows .drow")[2].querySelector(".dn").textContent;
|
| 183 |
+
// Superhuman loses the most to the audit: 53.3 -> 31.7 = 21.6, ahead of Gmail's 20.0.
|
| 184 |
+
check("sort by audit loss puts Superhuman first", firstAfterOverall, "Superhuman");
|
| 185 |
+
|
| 186 |
+
var fails = results.filter(function (r) { return r.indexOf("FAIL") === 0; });
|
| 187 |
+
document.title = "RESULT " + (fails.length ? "FAILURES=" + fails.length : "ALL PASS") +
|
| 188 |
+
" || " + results.join(" || ");
|
| 189 |
+
}).catch(function (e) {
|
| 190 |
+
document.title = "RESULT ERROR " + e.message + " || " + results.join(" || ");
|
| 191 |
+
});
|
| 192 |
+
</script>
|
| 193 |
+
</body></html>
|