Add llama.cpp VRAM calculator app and core logic

Gradio UI (app.py) plus the pure vramcalc package: quant BPW table,
KV cache + compute-scratch math, multi-GPU proportional split, YaRN
context-extension checks, minimal GGUF v3 header parser (HF range-read),
model presets, and 23 unit tests.

Co-Authored-By: Claude <noreply@anthropic.com>

.gitignore

@@ -0,0 +1,21 @@
+# Python
+__pycache__/
+*.py[cod]
+*.egg-info/
+.eggs/
+build/
+dist/
+
+# Test / tooling caches
+.pytest_cache/
+.coverage
+htmlcov/
+
+# venv
+.venv/
+venv/
+
+# OS / editor
+.DS_Store
+.idea/
+.vscode/

app.py

@@ -0,0 +1,377 @@
+"""llama.cpp VRAM Calculator — Hugging Face Space.
+
+A Gradio app that estimates VRAM usage for a Hugging Face GGUF model given
+quantization type, context length, KV cache options, YaRN context-extension
+parameters, MTP heads, and a multi-GPU budget. Architecture is auto-fetched
+from the GGUF header (range-read, no full download) with a manual-override
+tab and presets for offline use.
+"""
+
+from __future__ import annotations
+
+import gradio as gr
+from huggingface_hub import HfApi
+
+from vramcalc import (
+    QUANT_BPW,
+    ModelArch,
+    Inputs,
+    estimate,
+    command_preview,
+    format_bytes,
+    quant_from_filename,
+    parse_hf_range,
+)
+from vramcalc.presets import PRESETS, PRESET_NAMES
+
+
+QUANT_CHOICES = list(QUANT_BPW.keys())
+CACHE_DTYPES = ["f16", "bf16", "f32", "q8_0", "q8_1", "q4_0", "q4_1", "q5_0", "q5_1"]
+COMPUTE_DTYPES = ["f16", "bf16", "f32"]
+
+
+def list_gguf_files(repo_id: str, hf_token: str):
+    """List *.gguf files in a repo and auto-detect their quant names."""
+    if not repo_id or not repo_id.strip():
+        return [], "Enter a Hugging Face repo id."
+    try:
+        api = HfApi(token=hf_token or None)
+        files = api.list_repo_files(repo_id=repo_id.strip())
+    except Exception as e:  # noqa: BLE001
+        return [], f"Error listing repo: {e}"
+    ggufs = sorted(f for f in files if f.lower().endswith(".gguf"))
+    if not ggufs:
+        return [], f"No .gguf files found in {repo_id!r}."
+    choices = []
+    for f in ggufs:
+        q = quant_from_filename(f)
+        label = f"{f}" + (f"  [{q}]" if q else "")
+        choices.append((label, f))
+    return choices, f"Found {len(ggufs)} GGUF file(s)."
+
+
+def fetch_arch(repo_id: str, filename: str, hf_token: str):
+    """Range-read a GGUF header from HF and return editable arch fields."""
+    if not repo_id or not filename:
+        return _empty_arch_fields(), "Pick a GGUF file first."
+    try:
+        meta = parse_hf_range(
+            repo_id.strip(), filename, token=hf_token or None
+        )
+    except Exception as e:  # noqa: BLE001
+        return _empty_arch_fields(), f"Error reading GGUF header: {e}"
+    if not meta.n_layer:
+        return _arch_to_fields(meta), (
+            "Parsed header but architecture fields look empty; "
+            "edit them manually below."
+        )
+    return _arch_to_fields(meta), (
+        f"Fetched {meta.architecture or 'model'}: "
+        f"{meta.n_layer} layers, {meta.n_embd} embd, "
+        f"{meta.n_head}/{meta.n_head_kv} heads, ctx {meta.training_ctx}, "
+        f"params {meta.params or 'n/a'}."
+    )
+
+
+def _empty_arch_fields():
+    return _arch_to_fields(ModelArch())
+
+
+def _arch_to_fields(m: ModelArch):
+    return [
+        m.name, m.architecture, m.n_layer, m.n_embd, m.n_head, m.n_head_kv,
+        m.training_ctx, m.params, m.rope_freq_base, m.n_expert,
+        m.n_expert_used, m.n_mtp,
+    ]
+
+
+ARCH_FIELD_NAMES = [
+    "name", "architecture", "n_layer", "n_embd", "n_head", "n_head_kv",
+    "training_ctx", "params", "rope_freq_base", "n_expert",
+    "n_expert_used", "n_mtp",
+]
+
+
+def _fields_to_arch(fields) -> ModelArch:
+    values = {k: v for k, v in zip(ARCH_FIELD_NAMES, fields)}
+    return ModelArch(
+        name=str(values["name"] or ""),
+        architecture=str(values["architecture"] or ""),
+        n_layer=int(values["n_layer"] or 0),
+        n_embd=int(values["n_embd"] or 0),
+        n_head=int(values["n_head"] or 0),
+        n_head_kv=int(values["n_head_kv"] or 0),
+        training_ctx=int(values["training_ctx"] or 0),
+        params=int(values["params"] or 0),
+        rope_freq_base=float(values["rope_freq_base"] or 10000.0),
+        n_expert=int(values["n_expert"] or 0),
+        n_expert_used=int(values["n_expert_used"] or 0),
+        n_mtp=int(values["n_mtp"] or 0),
+    )
+
+
+def load_preset(name: str):
+    if name and name in PRESETS:
+        return _arch_to_fields(PRESETS[name]), f"Loaded preset: {name}"
+    return _empty_arch_fields(), ""
+
+
+def _parse_gpu_list(text: str) -> list[float]:
+    out = []
+    for tok in (text or "").replace(";", ",").split(","):
+        tok = tok.strip()
+        if tok:
+            try:
+                out.append(float(tok))
+            except ValueError:
+                pass
+    return out or [24.0]
+
+
+def compute(
+    arch_fields,
+    quant, n_ctx, cache_dtype, flash_attn, compute_dtype,
+    n_batch, n_prompt,
+    rope_freq_scale, yarn_ext_factor, yarn_attn_factor,
+    yarn_beta_fast, yarn_beta_slow,
+    gpu_vram_text, kv_on_largest, safety_margin,
+):
+    arch = _fields_to_arch(arch_fields)
+    gpus = _parse_gpu_list(gpu_vram_text)
+    inp = Inputs(
+        quant=quant,
+        n_ctx=int(n_ctx),
+        cache_dtype=cache_dtype,
+        flash_attn=bool(flash_attn),
+        compute_dtype=compute_dtype,
+        n_batch=int(n_batch),
+        n_prompt=int(n_prompt),
+        rope_freq_scale=float(rope_freq_scale),
+        yarn_ext_factor=float(yarn_ext_factor),
+        yarn_attn_factor=float(yarn_attn_factor),
+        yarn_beta_fast=float(yarn_beta_fast),
+        yarn_beta_slow=float(yarn_beta_slow),
+        gpu_vram_gb=gpus,
+        kv_on_largest=bool(kv_on_largest),
+        safety_margin_pct=float(safety_margin),
+    )
+    if arch.n_layer <= 0 or arch.n_embd <= 0 or arch.params <= 0:
+        return (
+            "⚠️ Model architecture is incomplete. Fill n_layer, n_embd, and "
+            "params (or fetch from a GGUF / load a preset).",
+            "", "",
+        )
+    bd = estimate(arch, inp)
+
+    # breakdown table
+    rows = [
+        ["Weights (GGUF, " + quant + ")", format_bytes(bd.weights_bytes)],
+        ["KV cache (" + cache_dtype + ")", format_bytes(bd.kv_cache_bytes)],
+        ["Compute / scratch", format_bytes(bd.compute_scratch_bytes)],
+        ["MTP overhead", format_bytes(bd.mtp_overhead_bytes)],
+        ["GGUF header / overhead", format_bytes(bd.gguf_overhead_bytes)],
+        ["Safety margin (" + str(inp.safety_margin_pct) + "%)",
+         format_bytes(bd.safety_margin_bytes)],
+        ["**Total**", f"**{format_bytes(bd.total_bytes)}**"],
+    ]
+    breakdown_md = "| Component | Size |\n|---|---|\n" + "\n".join(
+        f"| {a} | {b} |" for a, b in rows
+    )
+
+    # warnings
+    warn_md = ""
+    if bd.warnings:
+        warn_md = "\n\n**⚠️ YaRN / context notes:**\n" + "\n".join(
+            f"- {w}" for w in bd.warnings
+        )
+    eff_md = (
+        f"\n\nEffective context (training_ctx / rope_freq_scale): "
+        f"**{bd.effective_context}**"
+    )
+
+    # per-GPU table
+    gpu_md = ""
+    if bd.gpu and bd.gpu.assignments:
+        a = bd.gpu.assignments
+        header = "| GPU | VRAM | Weights | KV+Compute | Used | Free | Fits? |"
+        sep = "|---|---|---|---|---|---|---|"
+        body = []
+        for g in a:
+            kv = format_bytes(g.kv_compute_bytes) if g.kv_compute_bytes else "—"
+            badge = "✅" if g.fits else "❌"
+            body.append(
+                f"| {g.index} | {format_bytes(g.vram_bytes)} | "
+                f"{format_bytes(g.weight_bytes)} | {kv} | "
+                f"{format_bytes(g.used_bytes)} | {format_bytes(g.free_bytes)} "
+                f"| {badge} |"
+            )
+        total_badge = "✅ all fit" if bd.gpu.all_fit else "❌ over budget"
+        gpu_md = (
+            "**Per-GPU split (estimate, proportional weight split):**\n\n"
+            + header + "\n" + sep + "\n" + "\n".join(body)
+            + f"\n\nTotal VRAM: {format_bytes(bd.gpu.total_vram_bytes)} · "
+            f"Total used: {format_bytes(bd.gpu.total_used_bytes)} · "
+            f"{total_badge} · KV on GPU {bd.gpu.kv_gpu_index}"
+        )
+
+    cmd = command_preview(arch, inp)
+    summary = (
+        f"**{arch.name or arch.architecture or 'Model'}** @ {quant}, "
+        f"ctx {inp.n_ctx} ({cache_dtype} KV"
+        + (", FA" if inp.flash_attn else ", no FA")
+        + f"), {len(gpus)} GPU(s) → "
+        f"**{format_bytes(bd.total_bytes)}** total"
+    )
+    return summary + "\n\n" + breakdown_md + eff_md + warn_md + "\n\n" + gpu_md, cmd, cmd
+
+
+def build_ui():
+    with gr.Blocks(title="llama.cpp VRAM Calculator") as demo:
+        gr.Markdown(
+            "# 🦀 llama.cpp VRAM Calculator\n"
+            "Estimate VRAM for a Hugging Face GGUF model: quant size, "
+            "context, KV cache options, YaRN context extension, MTP heads, "
+            "and multi-GPU split. Architecture is auto-fetched from the GGUF "
+            "header (range-read — no full model download)."
+        )
+
+        arch_state = gr.State(_empty_arch_fields())
+
+        with gr.Row():
+            with gr.Column(scale=1):
+                gr.Markdown("### 1. Model source")
+                with gr.Tab("Auto-fetch from HF"):
+                    repo_id = gr.Textbox(
+                        label="HF repo id",
+                        placeholder="e.g. bartowski/Llama-3-8B-Instruct-GGUF",
+                    )
+                    hf_token = gr.Textbox(
+                        label="HF token (optional, for gated/private repos)",
+                        type="password",
+                    )
+                    list_btn = gr.Button("List GGUF files")
+                    file_picker = gr.Dropdown(
+                        label="GGUF file", choices=[], interactive=True
+                    )
+                    fetch_btn = gr.Button("Fetch architecture from GGUF header")
+                    fetch_status = gr.Markdown("")
+
+                with gr.Tab("Presets / manual"):
+                    preset_dd = gr.Dropdown(
+                        label="Quick preset", choices=PRESET_NAMES, interactive=True
+                    )
+                    load_preset_btn = gr.Button("Load preset")
+
+                gr.Markdown("### Architecture (editable)")
+                arch_inputs = [
+                    gr.Textbox(label="name", value=""),
+                    gr.Textbox(label="architecture", value=""),
+                    gr.Number(label="n_layer", value=0, precision=0),
+                    gr.Number(label="n_embd", value=0, precision=0),
+                    gr.Number(label="n_head", value=0, precision=0),
+                    gr.Number(label="n_head_kv", value=0, precision=0),
+                    gr.Number(label="training_ctx", value=0, precision=0),
+                    gr.Number(label="params", value=0, precision=0),
+                    gr.Number(label="rope_freq_base", value=10000.0),
+                    gr.Number(label="n_expert (MoE)", value=0, precision=0),
+                    gr.Number(label="n_expert_used", value=0, precision=0),
+                    gr.Number(label="n_mtp", value=0, precision=0),
+                ]
+
+            with gr.Column(scale=1):
+                gr.Markdown("### 2. Inference options")
+                quant = gr.Dropdown(
+                    label="Quantization", choices=QUANT_CHOICES, value="Q4_K_M"
+                )
+                n_ctx = gr.Number(label="Target context (n_ctx)", value=8192, precision=0)
+                with gr.Row():
+                    cache_dtype = gr.Dropdown(
+                        label="KV cache dtype", choices=CACHE_DTYPES, value="f16"
+                    )
+                    compute_dtype = gr.Dropdown(
+                        label="Compute dtype", choices=COMPUTE_DTYPES, value="f16"
+                    )
+                flash_attn = gr.Checkbox(label="Flash attention", value=True)
+                with gr.Row():
+                    n_batch = gr.Number(label="n_batch", value=512, precision=0)
+                    n_prompt = gr.Number(label="n_prompt (active)", value=0, precision=0)
+
+                gr.Markdown("### YaRN / RoPE context extension")
+                with gr.Row():
+                    rope_freq_scale = gr.Number(label="rope_freq_scale", value=1.0)
+                    yarn_ext_factor = gr.Number(label="yarn_ext_factor", value=-1.0)
+                    yarn_attn_factor = gr.Number(label="yarn_attn_factor", value=1.0)
+                with gr.Row():
+                    yarn_beta_fast = gr.Number(label="yarn_beta_fast", value=32.0)
+                    yarn_beta_slow = gr.Number(label="yarn_beta_slow", value=1.0)
+
+                gr.Markdown("### 3. Multi-GPU budget")
+                gpu_vram_text = gr.Textbox(
+                    label="Per-GPU VRAM (GB, comma-separated)",
+                    value="24",
+                    placeholder="e.g. 24,24,16",
+                )
+                with gr.Row():
+                    kv_on_largest = gr.Checkbox(
+                        label="Place KV cache on largest GPU", value=False
+                    )
+                    safety_margin = gr.Number(label="Safety margin %", value=5.0)
+
+                compute_btn = gr.Button("Compute VRAM", variant="primary")
+
+        gr.Markdown("### Results")
+        result_md = gr.Markdown("")
+        with gr.Accordion("llama.cpp launch command preview", open=False):
+            cmd_md = gr.Markdown("")
+            cmd_text = gr.Textbox(
+                label="Command (copyable)", lines=8, interactive=False
+            )
+
+        # --- wiring ---
+        def _store_arch(*fields):
+            return list(fields)
+
+        arch_inputs_and_state = [*arch_inputs, arch_state]
+        # keep arch_state synced whenever arch fields change
+        for comp in arch_inputs:
+            comp.change(
+                fn=_store_arch, inputs=arch_inputs, outputs=arch_state
+            )
+
+        list_btn.click(
+            fn=list_gguf_files, inputs=[repo_id, hf_token],
+            outputs=[file_picker, fetch_status],
+        )
+        fetch_btn.click(
+            fn=fetch_arch, inputs=[repo_id, file_picker, hf_token],
+            outputs=[*arch_inputs, fetch_status],
+        ).then(
+            fn=_store_arch, inputs=arch_inputs, outputs=arch_state
+        )
+        load_preset_btn.click(
+            fn=load_preset, inputs=[preset_dd],
+            outputs=[*arch_inputs, fetch_status],
+        ).then(
+            fn=_store_arch, inputs=arch_inputs, outputs=arch_state
+        )
+
+        compute_btn.click(
+            fn=compute,
+            inputs=[
+                arch_state, quant, n_ctx, cache_dtype, flash_attn, compute_dtype,
+                n_batch, n_prompt,
+                rope_freq_scale, yarn_ext_factor, yarn_attn_factor,
+                yarn_beta_fast, yarn_beta_slow,
+                gpu_vram_text, kv_on_largest, safety_margin,
+            ],
+            outputs=[result_md, cmd_md, cmd_text],
+        )
+
+    return demo
+
+
+demo = build_ui()
+
+
+if __name__ == "__main__":
+    demo.launch()

requirements.txt

@@ -0,0 +1,3 @@
+gradio==6.20.0
+huggingface_hub>=0.20.0
+requests>=2.28.0

tests/test_vramcalc.py

@@ -0,0 +1,259 @@
+"""Unit tests for the pure calculation logic in vramcalc."""
+
+import pytest
+
+from vramcalc import (
+    QUANT_BPW,
+    weight_bytes,
+    quant_from_filename,
+    kv_cache_bytes,
+    compute_scratch_bytes,
+    yarn_effective_context,
+    yarn_warnings,
+    gpu_split,
+    estimate,
+    command_preview,
+    format_bytes,
+)
+from vramcalc.presets import PRESETS
+from vramcalc.gguf import parse_header_bytes, metadata_to_arch
+
+
+def test_quant_table_has_common_types():
+    for q in ["Q4_K_M", "Q5_K_M", "Q8_0", "Q2_K", "F16", "BF16", "Q6_K"]:
+        assert q in QUANT_BPW
+
+
+def test_quant_from_filename():
+    assert quant_from_filename("Llama-3-8B-Instruct-Q4_K_M.gguf") == "Q4_K_M"
+    assert quant_from_filename("model.Q5_K_S.gguf") == "Q5_K_S"
+    assert quant_from_filename("model.Q8_0.gguf") == "Q8_0"
+    assert quant_from_filename("model.F16.gguf") == "F16"
+    assert quant_from_filename("model.gguf") is None
+
+
+def test_weight_bytes():
+    # 8B params at Q4_K_M (4.84375 bpw) -> ~4.84 GB
+    b = weight_bytes(8_000_000_000, "Q4_K_M")
+    assert 4.7e9 < b < 4.9e9
+    # F16 -> 16 bytes/param
+    assert weight_bytes(1_000_000, "F16") == pytest.approx(2_000_000)
+
+
+def test_weight_bytes_unknown_raises():
+    with pytest.raises(ValueError):
+        weight_bytes(1, "BOGUS")
+
+
+def test_kv_cache_basic():
+    # Llama-3 8B: 32 layers, 4096 embd, 32 heads, 8 kv heads, ctx 8192, f16
+    # head_dim = 128; per layer = 8192*2*8*128*2 = 33,554,432 bytes
+    # * 32 layers = ~1.07 GB
+    b = kv_cache_bytes(
+        n_layer=32, n_embd=4096, n_head=32, n_head_kv=8,
+        n_ctx=8192, cache_dtype="f16",
+    )
+    assert 1.0e9 < b < 1.1e9
+
+
+def test_kv_cache_scales_with_ctx():
+    b1 = kv_cache_bytes(n_layer=32, n_embd=4096, n_head=32, n_head_kv=8,
+                        n_ctx=4096, cache_dtype="f16")
+    b2 = kv_cache_bytes(n_layer=32, n_embd=4096, n_head=32, n_head_kv=8,
+                        n_ctx=8192, cache_dtype="f16")
+    assert b2 == pytest.approx(2 * b1)
+
+
+def test_kv_cache_mtp():
+    base = kv_cache_bytes(n_layer=61, n_embd=7168, n_head=128, n_head_kv=128,
+                          n_ctx=8192, cache_dtype="f16", n_mtp=0)
+    mtp = kv_cache_bytes(n_layer=61, n_embd=7168, n_head=128, n_head_kv=128,
+                         n_ctx=8192, cache_dtype="f16", n_mtp=1)
+    # n_mtp=1 adds one layer's worth: ratio = 62/61
+    assert mtp == pytest.approx(base * 62 / 61)
+
+
+def test_kv_cache_quantized_smaller():
+    f16 = kv_cache_bytes(n_layer=32, n_embd=4096, n_head=32, n_head_kv=8,
+                         n_ctx=8192, cache_dtype="f16")
+    q8 = kv_cache_bytes(n_layer=32, n_embd=4096, n_head=32, n_head_kv=8,
+                        n_ctx=8192, cache_dtype="q8_0")
+    q4 = kv_cache_bytes(n_layer=32, n_embd=4096, n_head=32, n_head_kv=8,
+                        n_ctx=8192, cache_dtype="q4_0")
+    assert q8 < f16
+    assert q4 < q8
+    assert q8 == pytest.approx(f16 / 2)
+    assert q4 == pytest.approx(f16 / 4)
+
+
+def test_compute_scratch_positive():
+    s = compute_scratch_bytes(
+        n_layer=32, n_embd=4096, n_head=32, n_head_kv=8,
+        n_batch=512, compute_dtype="f16",
+    )
+    assert s > 0
+
+
+def test_compute_scratch_no_fa_quantized_adds_dequant():
+    base = compute_scratch_bytes(
+        n_layer=32, n_embd=4096, n_head=32, n_head_kv=8,
+        n_batch=512, compute_dtype="f16", cache_dtype="q8_0", flash_attn=True,
+    )
+    nofa = compute_scratch_bytes(
+        n_layer=32, n_embd=4096, n_head=32, n_head_kv=8,
+        n_batch=512, compute_dtype="f16", cache_dtype="q8_0", flash_attn=False,
+    )
+    assert nofa > base
+
+
+def test_yarn_effective_context():
+    # scale 0.5 doubles effective context
+    assert yarn_effective_context(8192, 0.5) == 16384
+    assert yarn_effective_context(8192, 1.0) == 8192
+    # zero/negative scale falls back to training ctx
+    assert yarn_effective_context(8192, 0.0) == 8192
+
+
+def test_yarn_warnings_target_exceeds():
+    w = yarn_warnings(
+        training_ctx=8192, target_ctx=32768,
+        rope_freq_scale=1.0, yarn_ext_factor=-1.0, yarn_attn_factor=1.0,
+    )
+    assert any("exceeds training context" in x for x in w)
+    assert any("Effective context" in x for x in w)
+
+
+def test_yarn_warnings_no_warning_when_within():
+    w = yarn_warnings(
+        training_ctx=131072, target_ctx=8192,
+        rope_freq_scale=1.0, yarn_ext_factor=-1.0, yarn_attn_factor=1.0,
+    )
+    assert w == []
+
+
+def test_gpu_split_proportional_sums():
+    res = gpu_split(
+        gpu_vram_bytes=[24 << 30, 24 << 30, 16 << 30],
+        weights_bytes=40 << 30,
+        kv_compute_bytes=4 << 30,
+    )
+    # weight shares sum to weights
+    total_w = sum(a.weight_bytes for a in res.assignments)
+    assert total_w == pytest.approx(40 << 30, rel=1e-6)
+    # KV on exactly one GPU (default GPU 0)
+    kv_gpus = [a for a in res.assignments if a.kv_compute_bytes > 0]
+    assert len(kv_gpus) == 1
+    assert kv_gpus[0].index == 0
+    # used+free == vram for each gpu
+    for a in res.assignments:
+        assert a.used_bytes + a.free_bytes == pytest.approx(a.vram_bytes, rel=1e-6)
+
+
+def test_gpu_split_kv_on_largest():
+    res = gpu_split(
+        gpu_vram_bytes=[16 << 30, 24 << 30],
+        weights_bytes=10 << 30,
+        kv_compute_bytes=3 << 30,
+        kv_on_largest=True,
+    )
+    kv_gpus = [a for a in res.assignments if a.kv_compute_bytes > 0]
+    assert len(kv_gpus) == 1
+    assert kv_gpus[0].index == 1
+
+
+def test_gpu_split_fit_badge():
+    res = gpu_split(
+        gpu_vram_bytes=[4 << 30],  # tiny
+        weights_bytes=10 << 30,
+        kv_compute_bytes=1 << 30,
+    )
+    assert res.all_fit is False
+    assert res.assignments[0].fits is False
+
+
+def test_estimate_llama3_8b():
+    arch = PRESETS["Llama-3 8B"]
+    from vramcalc import Inputs
+    bd = estimate(arch, Inputs(quant="Q4_K_M", n_ctx=8192, cache_dtype="f16",
+                                flash_attn=True, compute_dtype="f16",
+                                n_batch=512, gpu_vram_gb=[24.0]))
+    # weights ~4.8 GB
+    assert 4.5e9 < bd.weights_bytes < 5.0e9
+    # KV ~1 GB
+    assert 0.9e9 < bd.kv_cache_bytes < 1.1e9
+    # total fits on 24 GB
+    assert bd.gpu.all_fit is True
+    assert bd.total_bytes < 24 << 30
+
+
+def test_estimate_mtp_overhead():
+    arch = PRESETS["DeepSeek-V3 (MoE)"]
+    from vramcalc import Inputs
+    bd = estimate(arch, Inputs(quant="Q4_K_M", n_ctx=8192, cache_dtype="f16",
+                               gpu_vram_gb=[80.0]))
+    assert bd.mtp_overhead_bytes > 0
+
+
+def test_command_preview_basic():
+    arch = PRESETS["Llama-3 8B"]
+    from vramcalc import Inputs
+    cmd = command_preview(arch, Inputs(quant="Q4_K_M", n_ctx=8192,
+                                       cache_dtype="q8_0", flash_attn=True,
+                                       gpu_vram_gb=[24.0]))
+    assert "llama-server" in cmd
+    assert "-c 8192" in cmd
+    assert "--cache-type-k q8_0" in cmd
+    assert "--flash-attn" in cmd
+
+
+def test_command_preview_multigpu_split():
+    arch = PRESETS["Llama-3 8B"]
+    from vramcalc import Inputs
+    cmd = command_preview(arch, Inputs(quant="Q4_K_M", n_ctx=8192,
+                                       gpu_vram_gb=[24.0, 24.0]))
+    assert "--tensor-split 24.0,24.0" in cmd
+
+
+def test_command_preview_yarn_when_target_exceeds():
+    arch = PRESETS["Llama-3 8B"]  # training_ctx 8192
+    from vramcalc import Inputs
+    cmd = command_preview(arch, Inputs(quant="Q4_K_M", n_ctx=32768,
+                                       rope_freq_scale=0.5,
+                                       gpu_vram_gb=[24.0]))
+    assert "--rope-scaling yarn" in cmd
+    assert "--rope-freq-scale 0.5" in cmd
+
+
+def test_format_bytes():
+    assert "GiB" in format_bytes(5 << 30)
+    assert "MiB" in format_bytes(5 << 20)
+    assert "KiB" in format_bytes(5 << 10)
+    assert format_bytes(0).endswith("B")
+
+
+def test_gguf_parse_header_minimal():
+    # Build a tiny valid GGUF header in memory: magic, version, tensor_count=0,
+    # kv_count=2, with one string and one int metadata entry.
+    import struct
+    def s(x): return x.encode("utf-8")
+    buf = b""
+    buf += struct.pack("<I", 0x46554747)  # GGUF
+    buf += struct.pack("<I", 3)           # version
+    buf += struct.pack("<Q", 0)           # tensor_count
+    buf += struct.pack("<Q", 2)           # kv_count
+    # kv 1: "general.architecture" = "llama"
+    name = s("general.architecture")
+    buf += struct.pack("<Q", len(name)) + name
+    buf += struct.pack("<I", 8)  # STRING
+    val = s("llama")
+    buf += struct.pack("<Q", len(val)) + val
+    # kv 2: "llama.block_count" = 32
+    name2 = s("llama.block_count")
+    buf += struct.pack("<Q", len(name2)) + name2
+    buf += struct.pack("<I", 4)  # UINT32
+    buf += struct.pack("<I", 32)
+    meta = parse_header_bytes(buf)
+    assert meta["general.architecture"] == "llama"
+    arch = metadata_to_arch(meta)
+    assert arch.architecture == "llama"
+    assert arch.n_layer == 32

vramcalc/__init__.py

@@ -0,0 +1,50 @@
+"""llama.cpp VRAM estimation helpers.
+
+Pure calculation logic, importable without Gradio so it can be unit-tested.
+"""
+
+from .quant import QUANT_BPW, weight_bytes, quant_from_filename
+from .kv import kv_cache_bytes, cache_dtype_bytes, compute_scratch_bytes
+from .yarn import yarn_effective_context, yarn_warnings
+from .gpu import GpuBudget, gpu_split, fit_gpus, GpuSplitResult
+from .gguf import (
+    GGUFMetadata,
+    parse_header_bytes,
+    parse_local_file,
+    parse_hf_range,
+    metadata_to_arch,
+)
+from .report import (
+    ModelArch,
+    Inputs,
+    Breakdown,
+    estimate,
+    command_preview,
+    format_bytes,
+)
+
+__all__ = [
+    "QUANT_BPW",
+    "weight_bytes",
+    "quant_from_filename",
+    "kv_cache_bytes",
+    "cache_dtype_bytes",
+    "compute_scratch_bytes",
+    "yarn_effective_context",
+    "yarn_warnings",
+    "GpuBudget",
+    "GpuSplitResult",
+    "gpu_split",
+    "fit_gpus",
+    "GGUFMetadata",
+    "parse_header_bytes",
+    "parse_local_file",
+    "parse_hf_range",
+    "metadata_to_arch",
+    "ModelArch",
+    "Inputs",
+    "Breakdown",
+    "estimate",
+    "command_preview",
+    "format_bytes",
+]

vramcalc/gguf.py

@@ -0,0 +1,218 @@
+"""Minimal GGUF v3 header parser.
+
+Reads only the metadata key/value section (no tensor data) from a GGUF file,
+either from a local path or via a ranged HTTP GET against the Hugging Face
+resolve URL so we never download the whole model.
+
+GGUF layout (little-endian):
+    magic: u32 = 'GGUF'
+    version: u32
+    tensor_count: u64
+    metadata_kv_count: u64
+    metadata_kv[]: { name_len: u64, name: bytes, value_type: u32, value: ... }
+    tensor_info[]: (not parsed here)
+
+We parse just enough metadata to recover architecture parameters.
+"""
+
+from __future__ import annotations
+
+import io
+import struct
+from dataclasses import dataclass, field
+
+GGUF_MAGIC = 0x46554747  # "GGUF"
+GGUF_DEFAULT_VERSION = 3
+
+GGUF_TYPE_UINT8 = 0
+GGUF_TYPE_INT8 = 1
+GGUF_TYPE_UINT16 = 2
+GGUF_TYPE_INT16 = 3
+GGUF_TYPE_UINT32 = 4
+GGUF_TYPE_INT32 = 5
+GGUF_TYPE_FLOAT32 = 6
+GGUF_TYPE_BOOL = 7
+GGUF_TYPE_STRING = 8
+GGUF_TYPE_ARRAY = 9
+GGUF_TYPE_UINT64 = 10
+GGUF_TYPE_INT64 = 11
+GGUF_TYPE_FLOAT64 = 12
+
+_SCALAR_FMT = {
+    GGUF_TYPE_UINT8: ("<B", 1),
+    GGUF_TYPE_INT8: ("<b", 1),
+    GGUF_TYPE_UINT16: ("<H", 2),
+    GGUF_TYPE_INT16: ("<h", 2),
+    GGUF_TYPE_UINT32: ("<I", 4),
+    GGUF_TYPE_INT32: ("<i", 4),
+    GGUF_TYPE_FLOAT32: ("<f", 4),
+    GGUF_TYPE_BOOL: ("<?", 1),
+    GGUF_TYPE_UINT64: ("<Q", 8),
+    GGUF_TYPE_INT64: ("<q", 8),
+    GGUF_TYPE_FLOAT64: ("<d", 8),
+}
+
+
+@dataclass
+class GGUFMetadata:
+    raw: dict[str, object] = field(default_factory=dict)
+    architecture: str = ""
+    n_layer: int = 0
+    n_embd: int = 0
+    n_head: int = 0
+    n_head_kv: int = 0
+    training_ctx: int = 0
+    rope_freq_base: float = 10000.0
+    n_expert: int = 0
+    n_expert_used: int = 0
+    n_mtp: int = 0
+    params: int = 0
+
+
+class _Reader:
+    def __init__(self, buf: bytes):
+        self.buf = buf
+        self.pos = 0
+
+    def eof(self) -> bool:
+        return self.pos >= len(self.buf)
+
+    def need(self, n: int):
+        if self.pos + n > len(self.buf):
+            raise EOFError("buffer exhausted while parsing GGUF header")
+
+    def u32(self) -> int:
+        self.need(4)
+        v = struct.unpack_from("<I", self.buf, self.pos)[0]
+        self.pos += 4
+        return v
+
+    def u64(self) -> int:
+        self.need(8)
+        v = struct.unpack_from("<Q", self.buf, self.pos)[0]
+        self.pos += 8
+        return v
+
+    def string(self) -> str:
+        n = self.u64()
+        self.need(n)
+        s = self.buf[self.pos:self.pos + n].decode("utf-8", errors="replace")
+        self.pos += n
+        return s
+
+    def scalar(self, t: int):
+        fmt, size = _SCALAR_FMT[t]
+        self.need(size)
+        v = struct.unpack_from(fmt, self.buf, self.pos)[0]
+        self.pos += size
+        return v
+
+    def value(self, t: int):
+        if t == GGUF_TYPE_STRING:
+            return self.string()
+        if t in _SCALAR_FMT:
+            return self.scalar(t)
+        if t == GGUF_TYPE_ARRAY:
+            inner = self.u32()
+            count = self.u64()
+            return [self.value(inner) for _ in range(count)]
+        raise ValueError(f"unsupported GGUF value type {t}")
+
+
+def parse_header_bytes(buf: bytes) -> dict[str, object]:
+    """Parse the metadata KV map from a buffer containing the GGUF header."""
+    r = _Reader(buf)
+    magic = r.u32()
+    if magic != GGUF_MAGIC:
+        raise ValueError(f"not a GGUF file (magic={magic:#x})")
+    version = r.u32()
+    if version < 1 or version > 3:
+        raise ValueError(f"unsupported GGUF version {version}")
+    # tensor_count (u64) - skipped
+    r.u64()
+    kv_count = r.u64()
+    meta: dict[str, object] = {}
+    for _ in range(kv_count):
+        if r.eof():
+            break
+        key = r.string()
+        t = r.u32()
+        try:
+            meta[key] = r.value(t)
+        except (EOFError, ValueError):
+            # ran out of buffered bytes (range read too small) - stop
+            break
+    return meta
+
+
+def _coerce_int(v: object) -> int:
+    if isinstance(v, bool):
+        return int(v)
+    if isinstance(v, (int, float)):
+        return int(v)
+    return 0
+
+
+def _coerce_float(v: object) -> float:
+    if isinstance(v, (int, float)):
+        return float(v)
+    return 0.0
+
+
+def metadata_to_arch(meta: dict[str, object]) -> GGUFMetadata:
+    """Project raw GGUF metadata into the architecture fields we need."""
+    arch_name = str(meta.get("general.architecture", "") or "")
+    p = f"{arch_name}." if arch_name else ""
+
+    def g(key: str, default=0):
+        return meta.get(f"{p}{key}", meta.get(key, default))
+
+    m = GGUFMetadata(raw=meta, architecture=arch_name)
+    m.n_layer = _coerce_int(g("block_count"))
+    m.n_embd = _coerce_int(g("embedding_length"))
+    m.n_head = _coerce_int(g("attention.head_count"))
+    n_head_kv = g("attention.head_count_kv")
+    m.n_head_kv = _coerce_int(n_head_kv) if n_head_kv is not None else m.n_head
+    if m.n_head_kv == 0:
+        m.n_head_kv = m.n_head
+    m.training_ctx = _coerce_int(g("context_length"))
+    m.rope_freq_base = _coerce_float(g("rope.freq_base", 10000.0))
+    m.n_expert = _coerce_int(g("expert_count"))
+    m.n_expert_used = _coerce_int(g("expert_used_count"))
+    # MTP: not always in GGUF metadata; leave 0 unless present
+    m.n_mtp = _coerce_int(g("mtp.count") or g("n_mtp"))
+    # parameter count
+    m.params = _coerce_int(meta.get("general.parameter_count", 0))
+    return m
+
+
+def parse_local_file(path: str, max_bytes: int = 1 << 20) -> GGUFMetadata:
+    with open(path, "rb") as f:
+        buf = f.read(max_bytes)
+    return metadata_to_arch(parse_header_bytes(buf))
+
+
+def parse_hf_range(
+    repo_id: str,
+    filename: str,
+    *,
+    max_bytes: int = 1 << 20,
+    token: str | None = None,
+    revision: str = "main",
+    timeout: float = 30.0,
+) -> GGUFMetadata:
+    """Range-read the first `max_bytes` of a GGUF file from the HF Hub.
+
+    Uses a HTTP Range request against the resolve URL so we never download the
+    full model. Requires the `requests` package.
+    """
+    import requests
+
+    url = f"https://huggingface.co/{repo_id}/resolve/{revision}/{filename}"
+    headers = {"Range": f"bytes=0-{max_bytes - 1}"}
+    if token:
+        headers["Authorization"] = f"Bearer {token}"
+    resp = requests.get(url, headers=headers, timeout=timeout, stream=True)
+    resp.raise_for_status()
+    buf = resp.content
+    return metadata_to_arch(parse_header_bytes(buf))

vramcalc/gpu.py

@@ -0,0 +1,104 @@
+"""Multi-GPU VRAM budgeting and tensor-split estimation.
+
+llama.cpp's real `--tensor-split` distributes tensor rows across devices; we
+approximate it by splitting the **weights** proportionally to each GPU's VRAM
+(this is the common case when users want an even load). The KV cache and
+compute scratch are placed on a single device (GPU 0 by default, or the GPU
+with the most free space) because llama.cpp keeps the cache on the first
+device unless manually offloaded.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+
+@dataclass
+class GpuBudget:
+    vram_bytes: list[int]
+
+
+@dataclass
+class GpuAssignment:
+    index: int
+    vram_bytes: int
+    weight_bytes: float
+    kv_compute_bytes: float
+    used_bytes: float
+    free_bytes: float
+    fits: bool
+
+
+@dataclass
+class GpuSplitResult:
+    assignments: list[GpuAssignment] = field(default_factory=list)
+    total_vram_bytes: int = 0
+    total_used_bytes: float = 0.0
+    total_weights_bytes: float = 0.0
+    total_kv_compute_bytes: float = 0.0
+    all_fit: bool = False
+    kv_gpu_index: int = 0
+
+
+def gpu_split(
+    *,
+    gpu_vram_bytes: list[int],
+    weights_bytes: float,
+    kv_compute_bytes: float,
+    kv_on_largest: bool = False,
+) -> GpuSplitResult:
+    """Split weights proportionally to VRAM; place KV+compute on one GPU."""
+    if not gpu_vram_bytes:
+        return GpuSplitResult()
+
+    total_vram = sum(gpu_vram_bytes)
+    if total_vram <= 0:
+        return GpuSplitResult(
+            assignments=[
+                GpuAssignment(i, v, 0.0, 0.0, 0.0, float(v), True)
+                for i, v in enumerate(gpu_vram_bytes)
+            ],
+            total_vram_bytes=total_vram,
+        )
+
+    # weight share per gpu proportional to vram
+    weight_shares = [weights_bytes * (v / total_vram) for v in gpu_vram_bytes]
+
+    # choose KV host
+    if kv_on_largest:
+        kv_idx = max(range(len(gpu_vram_bytes)), key=lambda i: gpu_vram_bytes[i])
+    else:
+        kv_idx = 0
+
+    assignments = []
+    for i, vram in enumerate(gpu_vram_bytes):
+        w = weight_shares[i]
+        kv = kv_compute_bytes if i == kv_idx else 0.0
+        used = w + kv
+        free = vram - used
+        fits = used <= vram
+        assignments.append(
+            GpuAssignment(
+                index=i,
+                vram_bytes=vram,
+                weight_bytes=w,
+                kv_compute_bytes=kv,
+                used_bytes=used,
+                free_bytes=free,
+                fits=fits,
+            )
+        )
+
+    return GpuSplitResult(
+        assignments=assignments,
+        total_vram_bytes=total_vram,
+        total_used_bytes=sum(a.used_bytes for a in assignments),
+        total_weights_bytes=weights_bytes,
+        total_kv_compute_bytes=kv_compute_bytes,
+        all_fit=all(a.fits for a in assignments),
+        kv_gpu_index=kv_idx,
+    )
+
+
+def fit_gpus(result: GpuSplitResult) -> bool:
+    return result.all_fit

vramcalc/kv.py

@@ -0,0 +1,106 @@
+"""KV cache and compute-scratch memory math.
+
+Based on llama.cpp's KV cache layout:
+
+    KV = n_layer * n_ctx * 2 * n_head_kv * head_dim * sizeof(cache_dtype)
+
+where head_dim = n_embd / n_head. The factor 2 covers both K and V. With MTP
+heads (e.g. DeepSeek-V3 n_mtp=1) each head adds ~one extra layer's worth of
+KV, so the layer count becomes (n_layer + n_mtp).
+"""
+
+from __future__ import annotations
+
+
+# cache dtype -> bytes per element (effective, including scales for quantized)
+CACHE_DTYPE_BYTES: dict[str, float] = {
+    "f16": 2.0,
+    "bf16": 2.0,
+    "f32": 4.0,
+    "q8_0": 1.0,
+    "q8_1": 1.0625,
+    "q4_0": 0.5,
+    "q4_1": 0.5625,
+    "q5_0": 0.625,
+    "q5_1": 0.6875,
+}
+
+# compute dtype -> bytes per element
+COMPUTE_DTYPE_BYTES: dict[str, float] = {
+    "f16": 2.0,
+    "bf16": 2.0,
+    "f32": 4.0,
+}
+
+
+def cache_dtype_bytes(dtype: str) -> float:
+    if dtype not in CACHE_DTYPE_BYTES:
+        raise ValueError(f"Unknown cache dtype: {dtype!r}")
+    return CACHE_DTYPE_BYTES[dtype]
+
+
+def _head_dim(n_embd: int, n_head: int) -> int:
+    if n_head <= 0:
+        return 0
+    return n_embd // n_head
+
+
+def kv_cache_bytes(
+    *,
+    n_layer: int,
+    n_embd: int,
+    n_head: int,
+    n_head_kv: int,
+    n_ctx: int,
+    cache_dtype: str,
+    n_mtp: int = 0,
+    flash_attn: bool = True,
+) -> float:
+    """Total KV cache size in bytes for the whole context window."""
+    head_dim = _head_dim(n_embd, n_head)
+    per_layer = n_ctx * 2 * n_head_kv * head_dim * cache_dtype_bytes(cache_dtype)
+    layers = n_layer + max(0, n_mtp)
+    base = per_layer * layers
+
+    # When flash attention is off AND the cache is quantized, llama.cpp keeps a
+    # dequantized f32 scratch for the active batch slice. We approximate the
+    # worst-case scratch as one batch-sized slice in f32, per layer (K+V).
+    if not flash_attn and cache_dtype not in ("f16", "bf16", "f32"):
+        # scratch is per-batch, not per-ctx; caller passes batch separately via
+        # compute_scratch_bytes. Here we leave it out to avoid double counting.
+        pass
+    return base
+
+
+def compute_scratch_bytes(
+    *,
+    n_layer: int,
+    n_embd: int,
+    n_head: int,
+    n_head_kv: int,
+    n_batch: int,
+    compute_dtype: str = "f16",
+    cache_dtype: str = "f16",
+    flash_attn: bool = True,
+    n_mtp: int = 0,
+) -> float:
+    """Working/compute scratch memory in bytes.
+
+    Includes the batch activations and, when FA is off with a quantized cache,
+    the per-batch f32 dequantization scratch for K and V across all layers.
+    """
+    if compute_dtype not in COMPUTE_DTYPE_BYTES:
+        raise ValueError(f"Unknown compute dtype: {compute_dtype!r}")
+    cb = COMPUTE_DTYPE_BYTES[compute_dtype]
+
+    head_dim = _head_dim(n_embd, n_head)
+    layers = n_layer + max(0, n_mtp)
+
+    # activations: roughly batch * n_embd * compute_dtype (a couple of buffers)
+    activations = n_batch * n_embd * cb * 2
+
+    scratch = 0.0
+    if not flash_attn and cache_dtype not in ("f16", "bf16", "f32"):
+        # f32 dequant buffer for one batch across all layers, K and V
+        scratch = layers * n_batch * 2 * n_head_kv * head_dim * 4.0
+    return activations + scratch

vramcalc/presets.py

@@ -0,0 +1,62 @@
+"""Built-in model architecture presets for offline quick-start."""
+
+from __future__ import annotations
+
+from .report import ModelArch
+
+
+PRESETS: dict[str, ModelArch] = {
+    "Llama-3 8B": ModelArch(
+        name="Llama-3 8B", architecture="llama",
+        n_layer=32, n_embd=4096, n_head=32, n_head_kv=8,
+        training_ctx=8192, params=8_030_000_000, rope_freq_base=500000.0,
+    ),
+    "Llama-3 70B": ModelArch(
+        name="Llama-3 70B", architecture="llama",
+        n_layer=80, n_embd=8192, n_head=64, n_head_kv=8,
+        training_ctx=8192, params=70_000_000_000, rope_freq_base=500000.0,
+    ),
+    "Llama-3.1 8B": ModelArch(
+        name="Llama-3.1 8B", architecture="llama",
+        n_layer=32, n_embd=4096, n_head=32, n_head_kv=8,
+        training_ctx=131072, params=8_030_000_000, rope_freq_base=500000.0,
+    ),
+    "Llama-3.1 70B": ModelArch(
+        name="Llama-3.1 70B", architecture="llama",
+        n_layer=80, n_embd=8192, n_head=64, n_head_kv=8,
+        training_ctx=131072, params=70_000_000_000, rope_freq_base=500000.0,
+    ),
+    "Mistral 7B": ModelArch(
+        name="Mistral 7B", architecture="llama",
+        n_layer=32, n_embd=4096, n_head=32, n_head_kv=8,
+        training_ctx=32768, params=7_240_000_000, rope_freq_base=10000.0,
+    ),
+    "Qwen2 7B": ModelArch(
+        name="Qwen2 7B", architecture="qwen2",
+        n_layer=28, n_embd=3584, n_head=28, n_head_kv=4,
+        training_ctx=32768, params=7_620_000_000, rope_freq_base=1000000.0,
+    ),
+    "Qwen2 72B": ModelArch(
+        name="Qwen2 72B", architecture="qwen2",
+        n_layer=80, n_embd=8192, n_head=64, n_head_kv=8,
+        training_ctx=131072, params=72_700_000_000, rope_freq_base=1000000.0,
+    ),
+    "DeepSeek-V3 (MoE)": ModelArch(
+        name="DeepSeek-V3", architecture="deepseek2",
+        n_layer=61, n_embd=7168, n_head=128, n_head_kv=128,
+        training_ctx=131072, params=671_000_000_000, rope_freq_base=10000.0,
+        n_expert=256, n_expert_used=8, n_mtp=1,
+    ),
+    "Phi-3 mini 3.8B": ModelArch(
+        name="Phi-3 mini", architecture="phi3",
+        n_layer=32, n_embd=3072, n_head=32, n_head_kv=32,
+        training_ctx=131072, params=3_800_000_000, rope_freq_base=10000.0,
+    ),
+    "Gemma-2 9B": ModelArch(
+        name="Gemma-2 9B", architecture="gemma2",
+        n_layer=42, n_embd=3584, n_head=16, n_head_kv=8,
+        training_ctx=8192, params=9_240_000_000, rope_freq_base=10000.0,
+    ),
+}
+
+PRESET_NAMES = list(PRESETS.keys())

vramcalc/quant.py

@@ -0,0 +1,86 @@
+"""Quantization -> bits-per-weight table and weight memory math.
+
+Values are the effective bits per weight (including block scales/min) as used
+by llama.cpp's k_quants / imatrix quants. Sourced from the llama.cpp docs and
+the widely-cited community tables. These are estimates; real file sizes vary a
+little due to alignment and metadata.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+# quant name -> effective bits per weight
+QUANT_BPW: dict[str, float] = {
+    # 2-bit
+    "Q2_K": 2.5625,
+    "IQ2_XXS": 2.0625,
+    "IQ2_XS": 2.3125,
+    "IQ2_S": 2.5,
+    "IQ2_M": 2.7,
+    # 3-bit
+    "Q3_K_S": 3.4375,
+    "Q3_K_M": 3.84375,
+    "Q3_K_L": 4.125,
+    "IQ3_XXS": 3.0625,
+    "IQ3_XS": 3.25,
+    "IQ3_S": 3.5,
+    "IQ3_M": 3.7,
+    # 4-bit
+    "Q4_0": 4.5,
+    "Q4_1": 5.0,
+    "Q4_K_S": 4.5,
+    "Q4_K_M": 4.84375,
+    "IQ4_NL": 4.5,
+    "IQ4_XS": 4.25,
+    # 5-bit
+    "Q5_0": 5.5,
+    "Q5_1": 6.0,
+    "Q5_K_S": 5.5,
+    "Q5_K_M": 5.6875,
+    # 6-bit
+    "Q6_K": 6.5625,
+    # 8-bit
+    "Q8_0": 8.5,
+    "Q8_1": 9.0,
+    # unquantized
+    "F16": 16.0,
+    "BF16": 16.0,
+    "F32": 32.0,
+}
+
+
+@dataclass(frozen=True)
+class QuantInfo:
+    name: str
+    bpw: float
+
+
+def quant_from_filename(filename: str) -> str | None:
+    """Detect a quant name from a GGUF filename like `Model-Q4_K_M.gguf`.
+
+    Returns the canonical quant name (matching a key in QUANT_BPW) or None.
+    Matching is case-insensitive on the quant token.
+    """
+    import re
+
+    upper = filename.upper()
+    # Try longer / more specific names first so e.g. Q4_K_S isn't caught by Q4_K.
+    ordered = sorted(QUANT_BPW.keys(), key=len, reverse=True)
+    for q in ordered:
+        # match as a token: surrounded by non-alphanumeric or string edges
+        if re.search(rf"(^|[^A-Z0-9]){re.escape(q.upper())}([^A-Z0-9]|$)", upper):
+            return q
+    return None
+
+
+def weight_bytes(params: int, quant: str) -> float:
+    """Estimated weight memory in bytes for `params` parameters at `quant`.
+
+    `params` is the total parameter count (including all MoE experts; the GGUF
+    already encodes them so this is the natural input).
+    """
+    if quant not in QUANT_BPW:
+        raise ValueError(f"Unknown quant type: {quant!r}")
+    return params * QUANT_BPW[quant] / 8.0

vramcalc/report.py

@@ -0,0 +1,199 @@
+"""Assemble a full VRAM breakdown and a llama.cpp launch-command preview."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+from .quant import weight_bytes
+from .kv import kv_cache_bytes, compute_scratch_bytes
+from .yarn import yarn_effective_context, yarn_warnings
+from .gpu import gpu_split, GpuSplitResult
+
+
+@dataclass
+class ModelArch:
+    """Architecture parameters (fetched from GGUF or entered manually)."""
+    name: str = ""
+    architecture: str = ""
+    n_layer: int = 0
+    n_embd: int = 0
+    n_head: int = 0
+    n_head_kv: int = 0
+    training_ctx: int = 0
+    params: int = 0          # total params including MoE experts
+    rope_freq_base: float = 10000.0
+    n_expert: int = 0        # MoE
+    n_expert_used: int = 0
+    n_mtp: int = 0           # MTP heads (e.g. DeepSeek-V3 = 1)
+
+
+@dataclass
+class Inputs:
+    quant: str = "Q4_K_M"
+    n_ctx: int = 8192
+    cache_dtype: str = "f16"
+    flash_attn: bool = True
+    compute_dtype: str = "f16"
+    n_batch: int = 512
+    n_prompt: int = 0
+    # YaRN
+    rope_freq_scale: float = 1.0
+    yarn_ext_factor: float = -1.0
+    yarn_attn_factor: float = 1.0
+    yarn_beta_fast: float = 32.0
+    yarn_beta_slow: float = 1.0
+    # GPU
+    gpu_vram_gb: list[float] = field(default_factory=lambda: [24.0])
+    kv_on_largest: bool = False
+    # margin
+    safety_margin_pct: float = 5.0
+
+
+@dataclass
+class Breakdown:
+    weights_bytes: float = 0.0
+    kv_cache_bytes: float = 0.0
+    compute_scratch_bytes: float = 0.0
+    mtp_overhead_bytes: float = 0.0
+    gguf_overhead_bytes: float = 0.0
+    safety_margin_bytes: float = 0.0
+    total_bytes: float = 0.0
+    effective_context: int = 0
+    warnings: list[str] = field(default_factory=list)
+    gpu: GpuSplitResult | None = None
+
+
+def estimate(arch: ModelArch, inp: Inputs) -> Breakdown:
+    weights = weight_bytes(arch.params, inp.quant)
+
+    # KV cache (with MTP layers folded in)
+    kv = kv_cache_bytes(
+        n_layer=arch.n_layer,
+        n_embd=arch.n_embd,
+        n_head=arch.n_head,
+        n_head_kv=arch.n_head_kv,
+        n_ctx=inp.n_ctx,
+        cache_dtype=inp.cache_dtype,
+        n_mtp=arch.n_mtp,
+        flash_attn=inp.flash_attn,
+    )
+
+    # compute / activation scratch
+    scratch = compute_scratch_bytes(
+        n_layer=arch.n_layer,
+        n_embd=arch.n_embd,
+        n_head=arch.n_head,
+        n_head_kv=arch.n_head_kv,
+        n_batch=inp.n_batch,
+        compute_dtype=inp.compute_dtype,
+        cache_dtype=inp.cache_dtype,
+        flash_attn=inp.flash_attn,
+        n_mtp=arch.n_mtp,
+    )
+
+    # isolate MTP overhead for display: the extra layer's weights + its KV share
+    mtp_overhead = 0.0
+    if arch.n_mtp > 0:
+        # approximate extra weight as one layer's worth: params/n_layer * bpw/8
+        if arch.n_layer > 0:
+            per_layer_params = arch.params / arch.n_layer
+            mtp_weights = per_layer_params * (
+                __import__("vramcalc.quant", fromlist=["QUANT_BPW"]).QUANT_BPW[inp.quant] / 8.0
+            )
+        else:
+            mtp_weights = 0.0
+        # KV for the extra layers
+        head_dim = arch.n_embd // arch.n_head if arch.n_head > 0 else 0
+        mtp_kv = (
+            arch.n_mtp * inp.n_ctx * 2 * arch.n_head_kv * head_dim
+            * __import__("vramcalc.kv", fromlist=["cache_dtype_bytes"]).cache_dtype_bytes(inp.cache_dtype)
+        )
+        mtp_overhead = mtp_weights + mtp_kv
+
+    # GGUF header / alignment overhead: small constant per file, rough estimate
+    gguf_overhead = max(arch.n_layer * 4096, 1 << 20)  # >= 1 MiB
+
+    subtotal = weights + kv + scratch + gguf_overhead
+    margin = subtotal * (inp.safety_margin_pct / 100.0)
+    total = subtotal + margin
+
+    eff_ctx = yarn_effective_context(arch.training_ctx, inp.rope_freq_scale)
+    warns = yarn_warnings(
+        training_ctx=arch.training_ctx,
+        target_ctx=inp.n_ctx,
+        rope_freq_scale=inp.rope_freq_scale,
+        yarn_ext_factor=inp.yarn_ext_factor,
+        yarn_attn_factor=inp.yarn_attn_factor,
+    )
+
+    gpu_vram_bytes = [int(g * (1 << 30)) for g in inp.gpu_vram_gb]
+    gpu = gpu_split(
+        gpu_vram_bytes=gpu_vram_bytes,
+        weights_bytes=weights,
+        kv_compute_bytes=kv + scratch,
+        kv_on_largest=inp.kv_on_largest,
+    )
+
+    return Breakdown(
+        weights_bytes=weights,
+        kv_cache_bytes=kv,
+        compute_scratch_bytes=scratch,
+        mtp_overhead_bytes=mtp_overhead,
+        gguf_overhead_bytes=gguf_overhead,
+        safety_margin_bytes=margin,
+        total_bytes=total,
+        effective_context=eff_ctx,
+        warnings=warns,
+        gpu=gpu,
+    )
+
+
+def format_bytes(n: float) -> str:
+    """Human-readable byte size."""
+    n = float(n)
+    if n < 0:
+        return "-" + format_bytes(-n)
+    units = [("GiB", 1 << 30), ("MiB", 1 << 20), ("KiB", 1 << 10)]
+    for label, size in units:
+        if n >= size:
+            return f"{n / size:.2f} {label}"
+    return f"{n:.0f} B"
+
+
+def command_preview(arch: ModelArch, inp: Inputs) -> str:
+    """Generate a llama.cpp launch command from the current inputs."""
+    parts = ["llama-server"]
+    parts.append(f"-m model-{inp.quant}.gguf")
+    parts.append(f"-c {inp.n_ctx}")
+    parts.append("-ngl 999")  # full offload assumption
+    parts.append(f"-b {inp.n_batch}")
+    if inp.cache_dtype != "f16":
+        parts.append(f"--cache-type-k {inp.cache_dtype}")
+        parts.append(f"--cache-type-v {inp.cache_dtype}")
+    if inp.flash_attn:
+        parts.append("--flash-attn")
+    if len(inp.gpu_vram_gb) > 1:
+        # tensor-split proportional to vram
+        split = ",".join(f"{g}" for g in inp.gpu_vram_gb)
+        parts.append(f"--tensor-split {split}")
+    # YaRN / rope
+    yarn_args = []
+    if inp.rope_freq_scale != 1.0:
+        yarn_args.append(f"--rope-freq-scale {inp.rope_freq_scale}")
+    if arch.rope_freq_base != 10000.0:
+        yarn_args.append(f"--rope-freq-base {arch.rope_freq_base}")
+    if inp.yarn_ext_factor >= 0.0:
+        yarn_args.append(f"--yarn-ext-factor {inp.yarn_ext_factor}")
+    if inp.yarn_attn_factor != 1.0:
+        yarn_args.append(f"--yarn-attn-factor {inp.yarn_attn_factor}")
+    if inp.yarn_beta_fast != 32.0:
+        yarn_args.append(f"--yarn-beta-fast {inp.yarn_beta_fast}")
+    if inp.yarn_beta_slow != 1.0:
+        yarn_args.append(f"--yarn-beta-slow {inp.yarn_beta_slow}")
+    if inp.n_ctx > arch.training_ctx and arch.training_ctx > 0:
+        yarn_args.append("--rope-scaling yarn")
+    if yarn_args:
+        parts.extend(yarn_args)
+    if arch.n_mtp > 0:
+        parts.append(f"--mtp {arch.n_mtp}")
+    return " \\\n  ".join(parts)

vramcalc/yarn.py

@@ -0,0 +1,59 @@
+"""YaRN / RoPE context-extension sanity checks.
+
+YaRN changes how the model *interprets* positions, not the per-token KV size,
+so KV cache memory is driven by the target context length the user requests.
+These helpers surface warnings when the target context exceeds the model's
+training context (YaRN scaling required) and show the effective context as a
+sanity check.
+"""
+
+from __future__ import annotations
+
+
+def yarn_effective_context(training_ctx: int, rope_freq_scale: float) -> int:
+    """Effective (interpolated) context given a rope_freq_scale.
+
+    effective = training_ctx / rope_freq_scale (rope_freq_scale < 1 extends).
+    """
+    if rope_freq_scale <= 0:
+        return training_ctx
+    return int(round(training_ctx / rope_freq_scale))
+
+
+def yarn_warnings(
+    *,
+    training_ctx: int,
+    target_ctx: int,
+    rope_freq_scale: float,
+    yarn_ext_factor: float,
+    yarn_attn_factor: float,
+) -> list[str]:
+    """Return human-readable warnings about the YaRN / context config."""
+    warns: list[str] = []
+
+    if target_ctx > training_ctx:
+        warns.append(
+            f"Target context {target_ctx} exceeds training context "
+            f"{training_ctx}; YaRN/RoPE scaling is required for coherent "
+            f"long-context output."
+        )
+
+    eff = yarn_effective_context(training_ctx, rope_freq_scale)
+    if target_ctx > training_ctx and eff < target_ctx:
+        warns.append(
+            f"Effective context from rope_freq_scale={rope_freq_scale} is "
+            f"{eff}, which is below the target {target_ctx}. The model may "
+            f"not fully cover the requested window; lower rope_freq_scale "
+            f"to interpolate more aggressively."
+        )
+
+    if target_ctx > training_ctx and yarn_ext_factor < 0.0:
+        warns.append("yarn_ext_factor is negative; YaRN extrapolation is off.")
+
+    if yarn_attn_factor < 1.0:
+        warns.append(
+            "yarn_attn_factor < 1.0 scales attention down; use ~1.0 for "
+            "normal long-context YaRN (lower only for high-context YaRN fine-tunes)."
+        )
+
+    return warns