#!/usr/bin/env python3 """Emit assets/data.js for the DarwinX project page from the original run artifacts. Every number the interactive figures display comes from the files listed in SOURCES, so the page can be re-derived rather than hand-transcribed. Run from anywhere: python3 tools/build_data.py """ import json import os import sys from datetime import datetime, timezone TW_DIR = ("/Users/yifan.zhang/Desktop/techreport/darwinx_zeyuan_feedback/" "version_control/version_0717_local/results/tw/results") NOTES = "/Users/yifan.zhang/Desktop/techreport/darwinx_zeyuan_feedback/notes" TW_ARMS = [ ("A", "Spec. A", "opus_node_34f71461_run.json"), ("B", "Spec. B", "opus_node_cab04955_run.json"), ("C", "Spec. C", "opus_node_56fc77cc_run.json"), ("D", "Spec. D", "opus_node_eb39901b_run.json"), ("M", "Merged", "TW_HEADLINE_merge_opus_28of41_run.json"), ] # From notes/TB21_RESULTS.md (paired protocol, 88 tasks). Deltas are carried over as # reported rather than recomputed: the source rounds the delta from unrounded rates, # so evolved-minus-base on the displayed 1-decimal values differs by up to 0.1. TB21_CLUSTERS = [ ("ML & scientific computing", 23, 60.1, 74.9, 14.8), ("Data & databases", 9, 83.9, 97.8, 13.8), ("Algorithms & coding", 14, 83.1, 90.0, 6.9), ("System admin & networking", 11, 92.0, 97.6, 5.7), ("Parsing, text & media", 11, 67.3, 72.2, 5.0), ("Systems & build", 11, 79.3, 81.8, 2.5), ("Security & crypto", 9, 85.1, 84.4, -0.6), ] # The four headline panels, from scripts/gen_summary_figure.py GROUPS. Each panel is # matched-model: base and evolved share one frozen base, so the gap is the harness. # The prior-agent value is context, not a controlled comparison, because those # systems use different models and effort settings. SWE-V has no prior-agent bar and # its "base" is the fix-skill reference rather than an unevolved Monet, so the label # travels with the number. HEADLINE = [ {"short": "TB 2.1", "name": "Terminal-Bench 2.1", "metric": "avg@5", "model": "GPT-5.5", "scope": "89 tasks", "baseLabel": "Monet (base)", "base": 75.5, "evolvedLabel": "Monet (DarwinX)", "evolved": 83.2, "priorLabel": "Codex", "prior": 83.1, "delta": 7.7, "lo": 70, "hi": 88}, {"short": "T-World", "name": "TerminalWorld", "metric": "pass@1", "model": "Opus 4.8", "scope": "41 held-out tasks", "baseLabel": "Monet (base)", "base": 61.0, "evolvedLabel": "Monet (DarwinX)", "evolved": 68.3, "priorLabel": "Claude Code", "prior": 65.9, "delta": 7.3, "lo": 55, "hi": 75}, {"short": "WA-Inf.", "name": "WebArena-Infinity", "metric": "audit-clean pass@1", "model": "GPT-5.5", "scope": "1,260 real tasks", "baseLabel": "Monet (base)", "base": 43.5, "evolvedLabel": "Monet (DarwinX)", "evolved": 93.0, "priorLabel": "GPT-5.5 + Browser Use", "prior": 86.1, "delta": 49.5, "lo": 35, "hi": 100}, {"short": "SWE-V", "name": "SWE-bench Verified", "metric": "official pass@1", "model": "Opus 4.8", "scope": "500 instances, transfer only", "baseLabel": "Fix-skill reference", "base": 80.8, "evolvedLabel": "Monet (DarwinX), transferred", "evolved": 84.2, "priorLabel": None, "prior": None, "delta": 3.4, "lo": 75, "hi": 88}, ] # scripts/gen_tw_dynamics.py + notes/tw_dynamics.json. Base is the unevolved v1.0.0 # harness on Opus 4.8 (25/41); its per-task run.json is not on this machine, so the # merge explorer covers only the arms that have per-task records. TW_HELDOUT_BARS = [("Base", 25), ("Spec. A", 24), ("Spec. B", 25), ("Spec. C", 26), ("Spec. D", 27), ("Monet (DarwinX)", 28)] TW_CLAUDE_CODE = 27 # Claude Code (Opus 4.8) reference, 27/41 TW_N = 41 # scripts/gen_tb21_compute.py: medians precomputed from the trial archive. TB21_COMPUTE = { "groups": ["Newly solved (6 tasks)", "Already solved (69 tasks)"], "turns": {"base": [11, 12], "evolved": [22, 13]}, "tokens": {"base": [89, 125], "evolved": [380, 172]}, # thousands } # scripts/gen_wai_invalid_composition.py: 293 invalid trajectories before evolution, # 17 after, decomposed two ways over the same totals. WAI_INVALID = { "application": { "labels": ["Elation Clinical", "Elation Prescription", "GitLab Plan & Track", "Gmail", "Gmail Acct & Contact", "Handshake", "Linear Acct Mgmt", "PayPal Wallet", "Super-human", "Xero Invoicing"], "before": [29, 27, 38, 21, 44, 54, 26, 10, 44, 0], "after": [2, 3, 0, 0, 3, 0, 0, 0, 8, 1], "colors": ["#4C72B0", "#A6C8E0", "#DD8452", "#E3B23C", "#55A868", "#8FD19E", "#C44E52", "#E79EA6", "#8172B3", "#C7BFE0"], }, "mechanism": { "labels": ["Evaluation plane access", "Privileged knowledge", "Exploit or privilege escalation", "Raw state mutation"], "before": [155, 97, 26, 15], "after": [0, 0, 0, 17], "colors": ["#55A868", "#DD8452", "#CE6DBD", "#8FA6D1"], }, } # scripts/gen_wai_audit_by_app.py: pass@1 before and after the validity audit. WAI_AUDIT = { "apps": ["Overall", "Elation-Clin", "Elation-Rx", "GitLab", "Gmail", "Gmail-Acct", "Handshake", "Linear", "PayPal", "Superhuman", "Xero"], "rawBase": [53.0, 95.8, 28.3, 75.7, 45.0, 33.3, 44.0, 54.2, 50.0, 53.3, 49.2], "rawDx": [94.4, 98.3, 97.5, 97.9, 98.3, 94.2, 84.0, 94.2, 95.7, 94.2, 97.5], "auditedBase": [43.5, 95.8, 20.0, 63.6, 25.0, 21.7, 36.5, 43.3, 49.3, 31.7, 39.2], "auditedDx": [93.0, 96.7, 95.0, 97.9, 98.3, 91.7, 84.0, 94.2, 95.7, 87.5, 96.7], } def load_tw(): """Per-task resolved flags for the four specialists and the merged harness. The run files list the 41 tasks in different orders, so every arm is joined on task_id and the emitted order is canonical (the first arm's file order). """ order, universe, sets = None, None, {} for key, _label, fname in TW_ARMS: path = os.path.join(TW_DIR, fname) if not os.path.exists(path): sys.exit(f"missing TerminalWorld run file: {path}") rows = json.load(open(path))["per_task_results"] ids = {r["task_id"] for r in rows} if len(ids) != len(rows): sys.exit(f"duplicate task_id in {fname}") if order is None: order, universe = [r["task_id"] for r in rows], ids elif ids != universe: sys.exit(f"task set differs in {fname}: " f"{sorted(ids ^ universe)} not shared; refusing to emit") sets[key] = {r["task_id"] for r in rows if r["resolved"]} tasks = [{"id": t, "bits": "".join("1" if t in sets[k] else "0" for k, _l, _f in TW_ARMS)} for t in order] counts = {k: len(sets[k]) for k, _l, _f in TW_ARMS} if [counts[k] for k in "ABCD"] != [24, 25, 26, 27] or counts["M"] != 28: sys.exit(f"unexpected TerminalWorld totals {counts}; paper reports 24/25/26/27 and 28") return tasks, counts def load_wai_curve(): path = os.path.join(NOTES, "tw_dynamics.json") if not os.path.exists(path): sys.exit(f"missing {path}") scores = json.load(open(path))["wai_adaptive_scores"] return [round(float(s), 2) for s in scores] def main(): tasks, counts = load_tw() curve = load_wai_curve() payload = { "generated": datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC"), "tw": { "arms": [{"key": k, "label": l} for k, l, _f in TW_ARMS], "tasks": tasks, "counts": counts, }, "tb21Clusters": [ {"name": n, "n": c, "base": b, "evolved": e, "delta": d} for n, c, b, e, d in TB21_CLUSTERS ], "waiCurve": curve, "headline": HEADLINE, "twBars": { "n": TW_N, "claudeCode": TW_CLAUDE_CODE, "bars": [{"label": l, "solved": s} for l, s in TW_HELDOUT_BARS], }, "tb21Compute": TB21_COMPUTE, "waiInvalid": WAI_INVALID, "waiAudit": WAI_AUDIT, } # Guard the invariants the page states in prose. for view in ("application", "mechanism"): v = WAI_INVALID[view] if sum(v["before"]) != 293 or sum(v["after"]) != 17: sys.exit(f"WAI_INVALID[{view}] must total 293 before and 17 after, got " f"{sum(v['before'])} and {sum(v['after'])}") if not (len(v["labels"]) == len(v["before"]) == len(v["after"]) == len(v["colors"])): sys.exit(f"WAI_INVALID[{view}] has ragged columns") for k in ("rawBase", "rawDx", "auditedBase", "auditedDx"): if len(WAI_AUDIT[k]) != len(WAI_AUDIT["apps"]): sys.exit(f"WAI_AUDIT[{k}] length does not match apps") # The audited numbers must agree with the per-application table on the page. page_table = [96.7, 95.0, 97.9, 98.3, 91.7, 84.0, 94.2, 95.7, 87.5, 96.7] if WAI_AUDIT["auditedDx"][1:] != page_table: sys.exit("WAI_AUDIT auditedDx disagrees with the page's per-application table") if [b["solved"] for b in payload["twBars"]["bars"]] != [25, 24, 25, 26, 27, 28]: sys.exit("twBars solved counts changed unexpectedly") out = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "assets", "data.js") with open(out, "w") as f: f.write("/* Generated by tools/build_data.py - do not edit by hand. */\n") f.write("window.DARWINX = ") json.dump(payload, f, indent=1, sort_keys=False) f.write(";\n") union = {t["id"] for t in tasks if "1" in t["bits"][:4]} merged = {t["id"] for t in tasks if t["bits"][4] == "1"} print(f"wrote {out}") print(f" TerminalWorld: {len(tasks)} tasks, counts={counts}") print(f" union(A..D)={len(union)} merged={len(merged)}") print(f" merged only: {sorted(merged - union)}") print(f" union only : {sorted(union - merged)}") print(f" TB2.1 clusters: {len(payload['tb21Clusters'])}") print(f" WAI curve: {len(curve)} nodes, max={max(curve)}") if __name__ == "__main__": main()