#!/usr/bin/env python3 """Regenerate the model-card figures for Ornith-1.0-35B-GGUF. Figures are written to ``assets/NN_name.png`` so the README image links resolve unchanged. All numbers are transcribed from the committed benchmark docs: - next-token top-64 KLD ladder: ``benchmarks/kld-quant-vs-bf16-top64.md`` plus the integrated IQ4_XS-MTP graft from ``benchmarks/mtp-kld-eval-2026-06-28.md``. - draft-head distillation progress: ``benchmarks/mtp-kld-eval-2026-06-28.md`` (2d). Run: python scripts/make_charts.py """ import os import matplotlib matplotlib.use("Agg") import matplotlib.pyplot as plt # --- shared style ---------------------------------------------------------- INK = "#111827" MUTED = "#6B7280" GRID = "#E5E7EB" # per-quant palette PALETTE = { "Q3_K_M": "#E11D48", "IQ4_XS": "#D97706", "Q4_K_M": "#4F46E5", "Q5_K_M": "#0D9488", "Q6_K": "#16A34A", "Q8_0": "#475569", } MTP = "#7C3AED" MTP_LIGHT = "#A78BFA" FOOTER = "Ornith-1.0-35B-GGUF · FirstPoint Labs · single-GPU (tp1) · llama.cpp" plt.rcParams.update({ "figure.dpi": 200, "savefig.dpi": 200, "font.family": "sans-serif", "font.sans-serif": ["DejaVu Sans"], "figure.facecolor": "white", "axes.facecolor": "white", "savefig.facecolor": "white", "axes.edgecolor": MUTED, "axes.labelcolor": INK, "text.color": INK, "xtick.color": INK, "ytick.color": INK, "axes.grid": True, "grid.color": GRID, "grid.linewidth": 0.6, "axes.axisbelow": True, "axes.spines.top": False, "axes.spines.right": False, }) REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) ASSETS = os.path.join(REPO_ROOT, "assets") os.makedirs(ASSETS, exist_ok=True) def _footer(fig): fig.text(0.01, 0.012, FOOTER, fontsize=7, color=MUTED, ha="left") # --- Figure 02: next-token fidelity ladder --------------------------------- def fidelity_ladder(): # (label, mean KLD nats, top-1 annotation, color) rows = [ ("Q3_K_M", 0.3620, "27/32 (84.4%)", PALETTE["Q3_K_M"]), ("IQ4_XS", 0.1426, "27/32 (84.4%)", PALETTE["IQ4_XS"]), ("Q4_K_M", 0.0864, "29/32 (90.6%)", PALETTE["Q4_K_M"]), ("IQ4_XS-MTP (graft)", 0.0731382442, "29/32 (90.6%)", MTP), ("Q5_K_M", 0.0354, "30/32 (93.8%)", PALETTE["Q5_K_M"]), ("Q6_K", 0.0165, "32/32 (100.0%)", PALETTE["Q6_K"]), ("Q8_0", 0.0108, "31/32 (96.9%)", PALETTE["Q8_0"]), ] # sort best fidelity (lowest KLD) on top rows = sorted(rows, key=lambda r: r[1]) labels = [r[0] for r in rows] vals = [r[1] for r in rows] tops = [r[2] for r in rows] colors = [r[3] for r in rows] fig, ax = plt.subplots(figsize=(9.0, 5.0)) y = range(len(rows)) bars = ax.barh(list(y), vals, color=colors, height=0.62, zorder=3) ax.set_yticks(list(y)) ax.set_yticklabels(labels) ax.invert_yaxis() # lowest KLD (best) on top ax.set_xlabel("Mean top-64 KLD vs BF16 (nats) — lower is better") ax.set_xlim(0, max(vals) * 1.55) # headroom for end-of-bar annotations # emphasize the MTP graft label for tick in ax.get_yticklabels(): if tick.get_text().startswith("IQ4_XS-MTP"): tick.set_color(MTP) tick.set_fontweight("bold") for bar, v, t in zip(bars, vals, tops): ax.text(bar.get_width() + max(vals) * 0.012, bar.get_y() + bar.get_height() / 2, f"{v:.4f} · {t}", va="center", ha="left", fontsize=8.5, color=INK) ax.set_title("Next-token fidelity ladder — top-64 KLD vs BF16", fontweight="bold", loc="left", fontsize=13, color=INK, pad=14) fig.subplots_adjust(left=0.215, right=0.97, top=0.88, bottom=0.13) _footer(fig) out = os.path.join(ASSETS, "02_fidelity_ladder.png") fig.savefig(out) plt.close(fig) print("wrote", out) # --- Figure 05: MTP draft-head distillation progress ----------------------- def draft_head_distill(): # two simple panels instead of one dual-axis chart labels = ["s500\n(distill)", "s1000\n(live-a)", "s1000\n(live-b)"] teacher_kl = [1.9222025748, 1.5038724942, 0.7591610373] teacher_top1 = [0.5360250917, 0.6876808388, 0.8908070915] gold_top1 = [0.4425939833, 0.6764816013, 0.8076183982] xi = list(range(len(labels))) fig, (axL, axR) = plt.subplots(1, 2, figsize=(11.0, 4.8)) # left: teacher KL bars (lower is better) bars = axL.bar(xi, teacher_kl, width=0.6, color=MTP, zorder=3) axL.set_xticks(xi); axL.set_xticklabels(labels) axL.set_ylabel("Teacher KL vs cached target states (nats)") axL.set_ylim(0, max(teacher_kl) * 1.16) for b, v in zip(bars, teacher_kl): axL.text(b.get_x() + b.get_width() / 2, b.get_height() + max(teacher_kl) * 0.015, f"{v:.4f}", ha="center", va="bottom", fontsize=9, color=INK) axL.set_title("Teacher KL — lower is better", fontweight="bold", loc="left", fontsize=12, color=INK, pad=10) # right: top-1 agreement grouped bars (higher is better) w = 0.38 b1 = axR.bar([x - w / 2 for x in xi], teacher_top1, w, color="#0D9488", zorder=3, label="Teacher argmax top-1") b2 = axR.bar([x + w / 2 for x in xi], gold_top1, w, color="#D97706", zorder=3, label="Gold top-1") axR.set_xticks(xi); axR.set_xticklabels(labels) axR.set_ylabel("argmax top-1 agreement") axR.set_ylim(0, 1.0) for grp in (b1, b2): for b in grp: axR.text(b.get_x() + b.get_width() / 2, b.get_height() + 0.015, f"{b.get_height():.2f}", ha="center", va="bottom", fontsize=8.5, color=INK) axR.set_title("Top-1 agreement — higher is better", fontweight="bold", loc="left", fontsize=12, color=INK, pad=10) axR.legend(loc="upper left", frameon=False, fontsize=8.5) fig.suptitle("MTP draft-head distillation progress", fontweight="bold", fontsize=14, color=INK, x=0.01, ha="left") fig.subplots_adjust(left=0.075, right=0.985, top=0.85, bottom=0.12, wspace=0.22) _footer(fig) out = os.path.join(ASSETS, "05_mtp_draft_head_distill.png") fig.savefig(out) plt.close(fig) print("wrote", out) # --- shared serving-benchmark data (benchmarks/llamacpp-quant-benchmarks.md) --- # short-context throughput profile: CTX_SIZE=8192 PARALLEL=16 (n_ctx=512/slot), # fixed 256-token generation sweep, ~23-27 token prompts. CONC = [1, 4, 8, 16] TPS = { # aggregate output tok/s "Q3_K_M": [240.5, 422.0, 464.4, 493.0], "IQ4_XS": [234.1, 297.7, 411.5, 476.0], "Q4_K_M": [243.3, 458.3, 615.0, 655.6], "Q5_K_M": [236.7, 311.0, 439.0, 638.6], "Q6_K": [225.9, 295.8, 409.6, 603.3], "Q8_0": [208.5, 281.5, 405.8, 601.4], } TTFT = { # p95 TTFT ms "Q3_K_M": [77.9, 170.7, 344.9, 493.7], "IQ4_XS": [75.1, 159.7, 330.2, 541.5], "Q4_K_M": [76.3, 192.3, 361.8, 650.0], "Q5_K_M": [75.1, 198.8, 383.6, 620.4], "Q6_K": [76.8, 194.2, 394.8, 657.4], "Q8_0": [76.9, 190.8, 389.1, 725.8], } # draw order high->low fidelity for stable legend QORDER = ["Q8_0", "Q6_K", "Q5_K_M", "Q4_K_M", "IQ4_XS", "Q3_K_M"] def _conc_line(metric, ylabel, title, subtitle, out, lower_better=False): xi = list(range(len(CONC))) fig, ax = plt.subplots(figsize=(9.0, 5.0)) for q in QORDER: ax.plot(xi, metric[q], "-o", color=PALETTE[q], linewidth=2, markersize=5, zorder=3, label=q) ax.set_xticks(xi) ax.set_xticklabels([f"c{c}" for c in CONC]) ax.set_xlabel("Concurrent requests") ax.set_ylabel(ylabel) ax.set_title(title, fontweight="bold", loc="left", fontsize=13, color=INK, pad=14) ax.margins(x=0.04) ax.legend(loc="upper left", frameon=False, fontsize=8.5, ncol=2, title=("lower is better" if lower_better else "higher is better"), title_fontsize=8) fig.subplots_adjust(left=0.10, right=0.97, top=0.90, bottom=0.12) _footer(fig) fig.savefig(out) plt.close(fig) print("wrote", out) def throughput_tps(): out = os.path.join(ASSETS, "03_throughput_tps.png") _conc_line(TPS, "Aggregate decode throughput (tok/s)", "Serving throughput vs concurrency — llama.cpp tp=1", None, out) def ttft_p95(): out = os.path.join(ASSETS, "04_ttft_p95.png") _conc_line(TTFT, "p95 time-to-first-token (ms)", "p95 TTFT vs concurrency — llama.cpp tp=1 (short-context profile)", None, out, lower_better=True) def mtp_tps(): # single-stream low-concurrency decode, deterministic 8x64 probe: # target-only vs active MTP graft (sourced from mtp-kld-eval-2026-06-28.md). # MTP wins at single-user; it is NOT faster at saturated c16 (see card). groups = ["Client aggregate\ntok/s", "Server decode\ntok/s (/metrics)"] target = [172.57, 210.0] mtp = [233.81, 325.70] ratios = ["1.35x", "~1.55x"] approx = [False, True] # ~210 is approximate per-request timing xi = list(range(len(groups))) w = 0.36 fig, ax = plt.subplots(figsize=(8.4, 5.0)) b1 = ax.bar([x - w / 2 for x in xi], target, w, color="#475569", zorder=3, label="Target-only") b2 = ax.bar([x + w / 2 for x in xi], mtp, w, color=MTP, zorder=3, label="Active MTP graft") ax.set_xticks(xi) ax.set_xticklabels(groups) ax.set_ylabel("Single-stream decode throughput (tok/s)") ax.set_ylim(0, max(mtp) * 1.2) for b, v, ap in zip(b1, target, approx): ax.text(b.get_x() + b.get_width() / 2, b.get_height() + max(mtp) * 0.015, (f"~{int(v)}" if ap else f"{v:.2f}"), ha="center", va="bottom", fontsize=9, color=INK) for b, v in zip(b2, mtp): ax.text(b.get_x() + b.get_width() / 2, b.get_height() + max(mtp) * 0.015, f"{v:.2f}", ha="center", va="bottom", fontsize=9, color=INK) for x, r, m in zip(xi, ratios, mtp): ax.text(x + w / 2, m + max(mtp) * 0.07, r, ha="center", va="bottom", fontsize=10, color=MTP, fontweight="bold") ax.set_title("MTP single-stream decode speedup — deterministic 8x64 probe", fontweight="bold", loc="left", fontsize=12, color=INK, pad=14) ax.legend(loc="upper left", frameon=False, fontsize=9) fig.subplots_adjust(left=0.10, right=0.97, top=0.90, bottom=0.13) _footer(fig) out = os.path.join(ASSETS, "06_mtp_tps.png") fig.savefig(out) plt.close(fig) print("wrote", out) # --- long-context TTFT (benchmarks/llamacpp-longctx-ttft.md) ----------------- # single-stream (PARALLEL=1, n_ctx=131072), p95 TTFT ms, exact prompt lengths. LONGCTX_LABELS = ["512", "1K", "2K", "4K", "8K", "16K", "32K"] TTFT_LC = { # p95 TTFT ms "Q4_K_M": [94.1, 172.2, 346.1, 702.2, 1458.0, 3030.4, 6313.0], "IQ4_XS": [88.7, 161.6, 318.7, 653.4, 1348.5, 2804.0, 5853.1], "IQ4_XS-MTP (graft)": [87.8, 159.0, 308.9, 631.3, 1304.0, 2737.3, 5696.6], } LC_COLORS = {"Q4_K_M": PALETTE["Q4_K_M"], "IQ4_XS": PALETTE["IQ4_XS"], "IQ4_XS-MTP (graft)": MTP} def longctx_ttft(): xi = list(range(len(LONGCTX_LABELS))) fig, ax = plt.subplots(figsize=(9.0, 5.0)) for q, ys in TTFT_LC.items(): ax.plot(xi, ys, "-o", color=LC_COLORS[q], linewidth=2, markersize=5, zorder=3, label=q) ax.set_xticks(xi) ax.set_xticklabels(LONGCTX_LABELS) ax.set_xlabel("Prompt / context length (tokens)") ax.set_ylabel("p95 time-to-first-token (ms)") ax.set_title("Long-context p95 TTFT vs prompt length — llama.cpp tp=1, single stream", fontweight="bold", loc="left", fontsize=12, color=INK, pad=14) ax.legend(loc="upper left", frameon=False, fontsize=9, title="lower is better", title_fontsize=8) ax.margins(x=0.04) fig.subplots_adjust(left=0.11, right=0.97, top=0.90, bottom=0.12) _footer(fig) out = os.path.join(ASSETS, "07_longctx_ttft.png") fig.savefig(out) plt.close(fig) print("wrote", out) if __name__ == "__main__": fidelity_ladder() throughput_tps() ttft_p95() draft_head_distill() mtp_tps() longctx_ttft()