benchmark/figures/make_more_figures.py

"""Four more figures for the model card. Same surface, palette and rcParams as figures 01-05."""
from pathlib import Path
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
import numpy as np

OUT = Path("/tmp/claude-1000/-home-mp-ubuntu-Projects-paper-qwen-intent-classifier/"
           "0af32b5a-ac18-4e73-a3ad-1c863d0154af/scratchpad/fig2")

SURFACE = "#fcfcfb"
INK, INK2, MUTED = "#1a1a19", "#4a4a47", "#8a8a85"
GRID = "#e5e5e1"
FLOOR = 0.4756270160201846
C = {"blue": "#2a78d6", "orange": "#eb6834", "aqua": "#1baf7a",
     "yellow": "#eda100", "violet": "#4a3aa7"}

plt.rcParams.update({
    "figure.facecolor": SURFACE, "axes.facecolor": SURFACE, "savefig.facecolor": SURFACE,
    "axes.edgecolor": GRID, "axes.linewidth": 0.8, "axes.labelcolor": INK2,
    "xtick.color": MUTED, "ytick.color": MUTED, "text.color": INK,
    "font.size": 9, "axes.titlesize": 11, "axes.titleweight": "semibold",
    "grid.color": GRID, "grid.linewidth": 0.7,
    "xtick.major.size": 0, "ytick.major.size": 0,
    "legend.frameon": False, "figure.dpi": 160,
})


def bare(ax, axis="y"):
    ax.set_axisbelow(True)
    ax.grid(True, axis=axis, alpha=0.9)
    for side in ("top", "right", "left"):
        ax.spines[side].set_visible(False)
    ax.spines["bottom"].set_color(GRID)


# ---------------------------------------------------------------- figure 06
# The card's spine. Four configurations, one protocol, three seeds each.
SPINE = [
    ("24L\n248k",            0.548740, 0.022761, 1435.1, 774.2),
    ("4L\n248k",             0.560165, 0.039089,  637.3, 349.2),
    ("4L\n128k",             0.563683, 0.032115,  402.3, 218.3),
    ("4L\n39,866 EN/KO",     0.572054, 0.008704,  230.1, 124.0),
]
PRECISION = [("BF16", 0.560165, 0.039089), ("F16", 0.559200, 0.039800), ("Q8_0", 0.561200, 0.035600)]


def fig06():
    fig, axes = plt.subplots(1, 3, figsize=(11.4, 4.1),
                             gridspec_kw={"width_ratios": [1.15, 1.0, 0.72], "wspace": 0.28})
    labels = [s[0] for s in SPINE]
    x = np.arange(len(SPINE))

    ax = axes[0]
    bare(ax)
    w = 0.36
    bf = [s[3] for s in SPINE]
    q8 = [s[4] for s in SPINE]
    ax.bar(x - w / 2 - 0.01, bf, w, color=C["blue"], label="BF16 safetensors", zorder=3)
    ax.bar(x + w / 2 + 0.01, q8, w, color=C["orange"], label="Q8_0 GGUF", zorder=3)
    for xi, v in zip(x - w / 2 - 0.01, bf):
        ax.annotate(f"{v:,.0f}", (xi, v), textcoords="offset points", xytext=(0, 3),
                    ha="center", fontsize=7.5, color=INK2)
    for xi, v in zip(x + w / 2 + 0.01, q8):
        ax.annotate(f"{v:,.0f}", (xi, v), textcoords="offset points", xytext=(0, 3),
                    ha="center", fontsize=7.5, color=INK2)
    ax.set_xticks(x); ax.set_xticklabels(labels, fontsize=8)
    ax.set_ylabel("weight file, MiB")
    ax.set_ylim(0, 1620)
    ax.set_title("Artifact size down the path")
    ax.legend(loc="upper right", fontsize=8)
    ax.annotate("1,435 MiB \u2192 124 MiB, 11.6x", (1.55, 1180), fontsize=8.5, color=MUTED,
                ha="center", va="center")

    ax = axes[1]
    bare(ax)
    m = np.array([s[1] for s in SPINE])
    sd = np.array([s[2] for s in SPINE])
    med_sd = float(np.median(sd))
    ax.axhspan(m.mean() - med_sd, m.mean() + med_sd, color=C["blue"], alpha=0.09, zorder=1,
               label=f"mean ± median seed SD ({med_sd:.4f})")
    ax.errorbar(x, m, yerr=sd, fmt="o", ms=7, color=C["blue"], ecolor=C["blue"],
                elinewidth=1.6, capsize=4, capthick=1.6, zorder=4, label="BF16, 3 seeds")
    ax.axhline(FLOOR, color=MUTED, lw=1.2, ls=(0, (4, 3)), zorder=2)
    ax.annotate("all-positive predictor floor, 0.4756", (0.02, FLOOR + 0.004),
                xycoords=("axes fraction", "data"), fontsize=7.5, color=MUTED)
    ax.set_xticks(x); ax.set_xticklabels(labels, fontsize=8)
    ax.set_xlim(-0.55, len(SPINE) - 0.45)
    ax.set_ylim(FLOOR - 0.018, 0.652)
    ax.set_ylabel("macro F1, opened 56-doc calibration split")
    ax.set_title("Quality down the same path")
    ax.annotate(f"BF16, 3 seeds, whiskers = sample SD\nband = mean \u00b1 median seed SD ({med_sd:.4f})",
                (0.03, 0.975), xycoords="axes fraction", va="top", fontsize=7.5, color=INK2)

    ax = axes[2]
    bare(ax)
    xs = np.arange(len(PRECISION))
    pm = [p[1] for p in PRECISION]
    ps = [p[2] for p in PRECISION]
    ax.errorbar(xs, pm, yerr=ps, fmt="s", ms=6, color=C["violet"], ecolor=C["violet"],
                elinewidth=1.6, capsize=4, capthick=1.6, zorder=4)
    ax.set_xticks(xs); ax.set_xticklabels([p[0] for p in PRECISION], fontsize=8)
    ax.set_xlim(-0.6, len(PRECISION) - 0.4)
    ax.set_ylim(FLOOR - 0.018, 0.652)
    ax.set_yticklabels([])
    ax.set_title("Precision, at 4L / 248k")
    ax.annotate("spread 0.0020\nagainst SD ~0.037", (1, 0.636), ha="center", va="top", fontsize=8, color=MUTED)

    fig.suptitle("The path, in one picture: four configurations, one frozen protocol, three seeds each",
                 fontsize=11.5, fontweight="semibold", y=0.99)
    fig.text(0.5, 0.062, "Quality is the transfer probe with a fresh 14-label head, not the headless root.",
             ha="center", fontsize=7.5, color=MUTED)
    fig.text(0.5, 0.030, "Stage 1 (prompting) is a different task on different hardware and is not on this axis.",
             ha="center", fontsize=7.5, color=MUTED)
    fig.subplots_adjust(left=0.065, right=0.985, top=0.855, bottom=0.215, wspace=0.28)
    fig.savefig(OUT / "06_stage_path.png")
    plt.close(fig)


# ---------------------------------------------------------------- figure 07
# v3 vs v2 on the 128k model (level 6 -> 12), then the port on cut vocabularies (level 0 -> 12).
V3_VS_V2 = [(33, 15.7626, 12.2856), (64, 20.9628, 19.8497), (129, 50.8364, 37.4436),
            (233, 92.1428, 70.4744), (256, 77.3285, 73.3632)]
PORT = [("v16k", 33, 14.94, 10.68), ("v32k", 25, 11.47, 7.36), ("v64k", 23, 10.32, 7.26)]


def fig07():
    fig, axes = plt.subplots(1, 2, figsize=(11.0, 4.3),
                             gridspec_kw={"width_ratios": [1.25, 1.0], "wspace": 0.22})

    ax = axes[0]
    bare(ax)
    x = np.arange(len(V3_VS_V2))
    w = 0.36
    a = [r[1] for r in V3_VS_V2]
    b = [r[2] for r in V3_VS_V2]
    ax.bar(x - w / 2 - 0.01, a, w, color=MUTED, label="level 6  (v2 path: 4x4 Q8 kernel)", zorder=3)
    ax.bar(x + w / 2 + 0.01, b, w, color=C["aqua"], label="level 12 (full specialisation)", zorder=3)
    for xi, va, vb in zip(x, a, b):
        red = (va - vb) / va * 100
        ax.annotate(f"−{red:.0f}%", (xi + w / 2 + 0.01, vb), textcoords="offset points",
                    xytext=(0, 4), ha="center", fontsize=8, color=C["aqua"], fontweight="semibold")
    ax.set_xticks(x); ax.set_xticklabels([f"{r[0]}" for r in V3_VS_V2])
    ax.set_xlabel("tokens in the probe window")
    ax.set_ylabel("native compute, ms (median of 5 blocks)")
    ax.set_ylim(0, 108)
    ax.set_title("On the 128k model: what the specialisation buys")
    ax.legend(loc="upper left", fontsize=8)

    ax = axes[1]
    bare(ax)
    x = np.arange(len(PORT))
    a = [r[2] for r in PORT]
    b = [r[3] for r in PORT]
    ax.bar(x - w / 2 - 0.01, a, w, color=MUTED, label="level 0  (generic path)", zorder=3)
    ax.bar(x + w / 2 + 0.01, b, w, color=C["orange"], label="level 12 (full specialisation)", zorder=3)
    for xi, va, vb in zip(x, a, b):
        red = (va - vb) / va * 100
        ax.annotate(f"−{red:.1f}%", (xi + w / 2 + 0.01, vb), textcoords="offset points",
                    xytext=(0, 4), ha="center", fontsize=8, color=C["orange"], fontweight="semibold")
    ax.set_xticks(x)
    ax.set_xticklabels([f"{r[0]}\n{r[1]} tokens" for r in PORT], fontsize=8)
    ax.set_ylabel("native compute, ms")
    ax.set_ylim(0, 19)
    ax.set_title("After the port: the cuts load and run specialised")
    ax.legend(loc="upper right", fontsize=8)
    ax.annotate("output bit-identical\nto level 0, all three cuts",
                (0.03, 0.90), xycoords="axes fraction", ha="left", va="top",
                fontsize=8, color=INK2)

    fig.suptitle("Dedicated runtime: a patched llama.cpp that runs the head natively and caches no KV",
                 fontsize=11.5, fontweight="semibold", y=0.985)
    fig.text(0.5, 0.062, "Token counts differ per cut for the same sentence, so compare within a bar pair, not across them.",
             ha="center", fontsize=7.5, color=MUTED)
    fig.text(0.5, 0.030, "Single-sentence medians on a host with a desktop session running. No model-card number came from this runtime.",
             ha="center", fontsize=7.5, color=MUTED)
    fig.subplots_adjust(left=0.065, right=0.985, top=0.845, bottom=0.215, wspace=0.22)
    fig.savefig(OUT / "07_runtime_specialization.png")
    plt.close(fig)


# ---------------------------------------------------------------- figure 08
SWEEP = {
    "qwen3.5 4L, 39,866 (262,144 ctx)": ([256, 512, 1024, 2048, 4096],
        [0.5815932, 0.5633398, 0.5722053, 0.5408152, 0.5479988], C["blue"], 262144),
    "modernbert-base (8,192 ctx)": ([256, 512, 1024, 2048, 4096],
        [0.5125712, 0.4904162, 0.4855341, 0.4794665, 0.4787691], C["orange"], 8192),
    "roberta-base (514 ctx)": ([256, 512], [0.5380876, 0.5509372], C["aqua"], 514),
    "deberta-v3-base (512 ctx)": ([256, 512], [0.5329868, 0.5375809], C["yellow"], 512),
}


def fig08():
    fig, ax = plt.subplots(figsize=(7.8, 4.7))
    bare(ax, axis="both")
    for lab, (xs, ys, col, cap) in SWEEP.items():
        ax.plot(xs, ys, "-o", lw=2, ms=6, color=col, label=lab, zorder=4,
                markeredgecolor=SURFACE, markeredgewidth=1.4)
        if len(xs) == 2:
            ax.annotate("position limit", (xs[-1], ys[-1]), textcoords="offset points",
                        xytext=(9, -2), fontsize=7.5, color=col, va="center")
    ax.axhline(FLOOR, color=MUTED, lw=1.2, ls=(0, (4, 3)), zorder=2)
    ax.annotate("all-positive predictor floor", (0.015, FLOOR - 0.0085),
                xycoords=("axes fraction", "data"), fontsize=7.5, color=MUTED, ha="left")
    ax.set_xscale("log", base=2)
    ax.set_xticks([256, 512, 1024, 2048, 4096])
    ax.set_xticklabels(["256", "512", "1,024", "2,048", "4,096"])
    ax.set_xlim(215, 11000)
    ax.set_ylim(FLOOR - 0.016, 0.605)
    ax.set_xlabel("reading window, tokens (every head was TRAINED at 256 / stride 128)")
    ax.set_ylabel("macro F1, opened 56-doc calibration split")
    ax.set_title("Reading wider does not help, and the 8,192 encoder loses exactly as much")
    ax.legend(loc="upper right", fontsize=8)
    ax.annotate("−0.0336\nover 256→4,096", (4096, 0.5480), textcoords="offset points",
                xytext=(12, 0), fontsize=8, color=C["blue"], ha="left", va="center")
    ax.annotate("−0.0338", (4096, 0.4788), textcoords="offset points",
                xytext=(12, 0), fontsize=8, color=C["orange"], ha="left", va="center")
    fig.text(0.5, 0.058, "Seed 41, same trained heads, inference-window sweep only.",
             ha="center", fontsize=7.5, color=MUTED)
    fig.text(0.5, 0.032, "Selected thresholds fall with the window (qwen 0.20→0.05, modernbert 0.50→0.30)",
             ha="center", fontsize=7.5, color=MUTED)
    fig.text(0.5, 0.006, "and window counts collapse 562→58, so this reads as distribution shift, not a context limit.",
             ha="center", fontsize=7.5, color=MUTED)
    fig.subplots_adjust(left=0.105, right=0.985, top=0.925, bottom=0.235)
    fig.savefig(OUT / "08_long_context_window.png")
    plt.close(fig)


# ---------------------------------------------------------------- figure 09
TOK = [("Korean", 16, 16, 1.00), ("English", 9, 12, 1.33),
       ("Japanese", 10, 44, 4.40), ("Chinese", 8, 58, 7.25)]


def fig09():
    fig, ax = plt.subplots(figsize=(7.4, 3.9))
    bare(ax, axis="x")
    y = np.arange(len(TOK))
    ratios = [t[3] for t in TOK]
    cols = [C["aqua"] if r <= 1.35 else C["orange"] for r in ratios]
    ax.barh(y, ratios, 0.55, color=cols, zorder=3)
    ax.axvline(1.0, color=MUTED, lw=1.2, ls=(0, (4, 3)), zorder=2)
    for yi, t in zip(y, TOK):
        ax.annotate(f"{t[3]:.2f}x   ({t[1]} → {t[2]} tokens)", (t[3], yi),
                    textcoords="offset points", xytext=(7, 0), va="center",
                    fontsize=8.5, color=INK2)
    ax.set_yticks(y); ax.set_yticklabels([t[0] for t in TOK])
    ax.invert_yaxis()
    ax.set_xlim(0, 10.8)
    ax.set_xlabel("tokens for the same sentence, relative to the upstream 248k tokenizer")
    ax.set_title("What the vocabulary cut costs per language")
    ax.annotate("unchanged \u2014 the rule keeps every Hangul-carrying token",
                (1.05, 0.42), fontsize=7.5, color=MUTED, va="center")
    fig.text(0.5, 0.055, "Every Han-carrying token upstream sits at BPE id 95,726 or above, so none survives any grid point",
             ha="center", fontsize=7.5, color=MUTED)
    fig.text(0.5, 0.020, "and CJK ideographs fall back to byte tokens. Compute scales with token count.",
             ha="center", fontsize=7.5, color=MUTED)
    fig.subplots_adjust(left=0.135, right=0.985, top=0.895, bottom=0.275)
    fig.savefig(OUT / "09_tokenizer_cost.png")
    plt.close(fig)


for f in (fig06, fig07, fig08, fig09):
    f()
    print("wrote", f.__name__)